nghttpx: Add TLS ticket key sharing among nghttpx instances using memcached

gennghttpxfun.py

@@ -94,6 +94,7 @@ OPTIONS = [
     "tls-ticket-cipher",
     "host-rewrite",
     "tls-session-cache-memcached",
+    "tls-ticket-key-memcached",
     "conf",
 ]
 

src/shrpx.cc

@@ -83,6 +83,8 @@
 #include "shrpx_accept_handler.h"
 #include "shrpx_http2_upstream.h"
 #include "shrpx_http2_session.h"
+#include "shrpx_memcached_dispatcher.h"
+#include "shrpx_memcached_request.h"
 #include "util.h"
 #include "app_helper.h"
 #include "ssl.h"
@@ -689,6 +691,116 @@ void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) {
 }
 } // namespace
 
+namespace {
+void memcached_get_ticket_key_cb(struct ev_loop *loop, ev_timer *w,
+                                 int revents) {
+  auto conn_handler = static_cast<ConnectionHandler *>(w->data);
+  auto dispatcher = conn_handler->get_tls_ticket_key_memcached_dispatcher();
+
+  auto req = make_unique<MemcachedRequest>();
+  req->key = "nghttpx:tls-ticket-key";
+  req->op = MEMCACHED_OP_GET;
+  req->cb = [conn_handler, dispatcher, w](MemcachedRequest *req,
+                                          MemcachedResult res) {
+    switch (res.status_code) {
+    case MEMCACHED_ERR_NO_ERROR:
+      break;
+    case MEMCACHED_ERR_EXT_NETWORK_ERROR:
+      conn_handler->on_tls_ticket_key_network_error(w);
+      return;
+    default:
+      conn_handler->on_tls_ticket_key_not_found(w);
+      return;
+    }
+
+    // |version (4bytes)|len (2bytes)|key (variable length)|...
+    // (len, key) pairs are repeated as necessary.
+
+    auto &value = res.value;
+    if (value.size() < 4) {
+      LOG(WARN) << "Memcached: tls ticket key value is too small: got "
+                << value.size();
+      conn_handler->on_tls_ticket_key_not_found(w);
+      return;
+    }
+    auto p = value.data();
+    auto version = util::get_uint32(p);
+    // Currently supported version is 1.
+    if (version != 1) {
+      LOG(WARN) << "Memcached: tls ticket key version: want 1, got " << version;
+      conn_handler->on_tls_ticket_key_not_found(w);
+      return;
+    }
+
+    auto end = p + value.size();
+    p += 4;
+
+    size_t expectedlen;
+    size_t enc_keylen;
+    size_t hmac_keylen;
+    if (get_config()->tls_ticket_cipher == EVP_aes_128_cbc()) {
+      expectedlen = 48;
+      enc_keylen = 16;
+      hmac_keylen = 16;
+    } else if (get_config()->tls_ticket_cipher == EVP_aes_256_cbc()) {
+      expectedlen = 80;
+      enc_keylen = 32;
+      hmac_keylen = 32;
+    } else {
+      return;
+    }
+
+    auto ticket_keys = std::make_shared<TicketKeys>();
+
+    for (; p != end;) {
+      if (end - p < 2) {
+        LOG(WARN) << "Memcached: tls ticket key data is too small";
+        conn_handler->on_tls_ticket_key_not_found(w);
+        return;
+      }
+      auto len = util::get_uint16(p);
+      p += 2;
+      if (len != expectedlen) {
+        LOG(WARN) << "Memcached: wrong tls ticket key size: want "
+                  << expectedlen << ", got " << len;
+        conn_handler->on_tls_ticket_key_not_found(w);
+        return;
+      }
+      if (p + len > end) {
+        LOG(WARN) << "Memcached: too short tls ticket key payload: want " << len
+                  << ", got " << (end - p);
+        conn_handler->on_tls_ticket_key_not_found(w);
+        return;
+      }
+      auto key = TicketKey();
+      key.cipher = get_config()->tls_ticket_cipher;
+      key.hmac = EVP_sha256();
+      key.hmac_keylen = EVP_MD_size(key.hmac);
+
+      std::copy_n(p, key.data.name.size(), key.data.name.data());
+      p += key.data.name.size();
+
+      std::copy_n(p, enc_keylen, key.data.enc_key.data());
+      p += enc_keylen;
+
+      std::copy_n(p, hmac_keylen, key.data.hmac_key.data());
+      p += hmac_keylen;
+
+      ticket_keys->keys.push_back(std::move(key));
+    }
+
+    conn_handler->on_tls_ticket_key_get_success(ticket_keys, w);
+  };
+
+  if (LOG_ENABLED(INFO)) {
+    LOG(INFO) << "Memcached: tls ticket key get request sent";
+  }
+
+  dispatcher->add_request(std::move(req));
+}
+
+} // namespace
+
 namespace {
 int call_daemon() {
 #ifdef __sgi
@@ -749,34 +861,47 @@ int event_loop() {
 
   ev_timer renew_ticket_key_timer;
   if (!get_config()->upstream_no_tls) {
-    bool auto_tls_ticket_key = true;
+    if (get_config()->tls_ticket_key_memcached_host) {
-    if (!get_config()->tls_ticket_key_files.empty()) {
+      conn_handler->set_tls_ticket_key_memcached_dispatcher(
-      if (!get_config()->tls_ticket_cipher_given) {
+          make_unique<MemcachedDispatcher>(
-        LOG(WARN) << "It is strongly recommended to specify "
+              &get_config()->tls_ticket_key_memcached_addr, loop));
-                     "--tls-ticket-cipher=aes-128-cbc (or "
-                     "tls-ticket-cipher=aes-128-cbc in configuration file) "
-                     "when --tls-ticket-key-file is used for the smooth "
-                     "transition when the default value of --tls-ticket-cipher "
-                     "becomes aes-256-cbc";
-      }
-      auto ticket_keys = read_tls_ticket_key_file(
-          get_config()->tls_ticket_key_files, get_config()->tls_ticket_cipher,
-          EVP_sha256());
-      if (!ticket_keys) {
-        LOG(WARN) << "Use internal session ticket key generator";
-      } else {
-        conn_handler->set_ticket_keys(std::move(ticket_keys));
-        auto_tls_ticket_key = false;
-      }
-    }
-    if (auto_tls_ticket_key) {
-      // Generate new ticket key every 1hr.
-      ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 1_h);
-      renew_ticket_key_timer.data = conn_handler.get();
-      ev_timer_again(loop, &renew_ticket_key_timer);
 
-      // Generate first session ticket key before running workers.
+      ev_timer_init(&renew_ticket_key_timer, memcached_get_ticket_key_cb, 0.,
-      renew_ticket_key_cb(loop, &renew_ticket_key_timer, 0);
+                    0.);
+      renew_ticket_key_timer.data = conn_handler.get();
+      // Get first ticket keys.
+      memcached_get_ticket_key_cb(loop, &renew_ticket_key_timer, 0);
+    } else {
+      bool auto_tls_ticket_key = true;
+      if (!get_config()->tls_ticket_key_files.empty()) {
+        if (!get_config()->tls_ticket_cipher_given) {
+          LOG(WARN)
+              << "It is strongly recommended to specify "
+                 "--tls-ticket-cipher=aes-128-cbc (or "
+                 "tls-ticket-cipher=aes-128-cbc in configuration file) "
+                 "when --tls-ticket-key-file is used for the smooth "
+                 "transition when the default value of --tls-ticket-cipher "
+                 "becomes aes-256-cbc";
+        }
+        auto ticket_keys = read_tls_ticket_key_file(
+            get_config()->tls_ticket_key_files, get_config()->tls_ticket_cipher,
+            EVP_sha256());
+        if (!ticket_keys) {
+          LOG(WARN) << "Use internal session ticket key generator";
+        } else {
+          conn_handler->set_ticket_keys(std::move(ticket_keys));
+          auto_tls_ticket_key = false;
+        }
+      }
+      if (auto_tls_ticket_key) {
+        // Generate new ticket key every 1hr.
+        ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 1_h);
+        renew_ticket_key_timer.data = conn_handler.get();
+        ev_timer_again(loop, &renew_ticket_key_timer);
+
+        // Generate first session ticket key before running workers.
+        renew_ticket_key_cb(loop, &renew_ticket_key_timer, 0);
+      }
     }
   }
 
@@ -1020,6 +1145,9 @@ void fill_default_config() {
   mod_config()->tls_ticket_cipher = EVP_aes_128_cbc();
   mod_config()->tls_ticket_cipher_given = false;
   mod_config()->tls_session_timeout = std::chrono::hours(12);
+  mod_config()->tls_ticket_key_memcached_max_retry = 3;
+  mod_config()->tls_ticket_key_memcached_max_fail = 2;
+  mod_config()->tls_ticket_key_memcached_interval = 10_min;
 }
 } // namespace
 
@@ -1368,6 +1496,15 @@ SSL/TLS:
               Specify  address of  memcached server  to store  session
               cache.   This  enables   shared  session  cache  between
               multiple nghttpx instances.
+  --tls-ticket-key-memcached=<HOST>,<PORT>
+              Specify  address of  memcached server  to store  session
+              cache.   This  enables  shared TLS  ticket  key  between
+              multiple nghttpx  instances.  nghttpx  does not  set TLS
+              ticket  key  to  memcached.   The  external  ticket  key
+              generator  is required.   nghttpx just  gets TLS  ticket
+              keys from  memcached, and  use them,  possibly replacing
+              current set of keys.  It is  up to extern TLS ticket key
+              generator to rotate keys frequently.
 
 HTTP/2 and SPDY:
   -c, --http2-max-concurrent-streams=<N>
@@ -1732,6 +1869,7 @@ int main(int argc, char **argv) {
         {SHRPX_OPT_TLS_TICKET_CIPHER, required_argument, &flag, 84},
         {SHRPX_OPT_HOST_REWRITE, no_argument, &flag, 85},
         {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED, required_argument, &flag, 86},
+        {SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED, required_argument, &flag, 87},
         {nullptr, 0, nullptr, 0}};
 
     int option_index = 0;
@@ -2110,6 +2248,10 @@ int main(int argc, char **argv) {
         // --tls-session-cache-memcached
         cmdcfgs.emplace_back(SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED, optarg);
         break;
+      case 87:
+        // --tls-ticket-key-memcached
+        cmdcfgs.emplace_back(SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED, optarg);
+        break;
       default:
         break;
       }
@@ -2396,6 +2538,15 @@ int main(int argc, char **argv) {
     }
   }
 
+  if (get_config()->tls_ticket_key_memcached_host) {
+    if (resolve_hostname(&mod_config()->tls_ticket_key_memcached_addr,
+                         get_config()->tls_ticket_key_memcached_host.get(),
+                         get_config()->tls_ticket_key_memcached_port,
+                         AF_UNSPEC) == -1) {
+      exit(EXIT_FAILURE);
+    }
+  }
+
   if (get_config()->rlimit_nofile) {
     struct rlimit lim = {static_cast<rlim_t>(get_config()->rlimit_nofile),
                          static_cast<rlim_t>(get_config()->rlimit_nofile)};

src/shrpx_config.cc

@@ -706,6 +706,7 @@ enum {
   SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED,
   SHRPX_OPTID_TLS_TICKET_CIPHER,
   SHRPX_OPTID_TLS_TICKET_KEY_FILE,
+  SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED,
   SHRPX_OPTID_USER,
   SHRPX_OPTID_VERIFY_CLIENT,
   SHRPX_OPTID_VERIFY_CLIENT_CACERT,
@@ -1138,6 +1139,11 @@ int option_lookup_token(const char *name, size_t namelen) {
     break;
   case 24:
     switch (name[23]) {
+    case 'd':
+      if (util::strieq_l("tls-ticket-key-memcache", name, 23)) {
+        return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED;
+      }
+      break;
     case 'e':
       if (util::strieq_l("fetch-ocsp-response-fil", name, 23)) {
         return SHRPX_OPTID_FETCH_OCSP_RESPONSE_FILE;
@@ -1881,6 +1887,17 @@ int parse_config(const char *opt, const char *optarg,
 
     return 0;
   }
+  case SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED: {
+    if (split_host_port(host, sizeof(host), &port, optarg, strlen(optarg)) ==
+        -1) {
+      return -1;
+    }
+
+    mod_config()->tls_ticket_key_memcached_host = strcopy(host);
+    mod_config()->tls_ticket_key_memcached_port = port;
+
+    return 0;
+  }
   case SHRPX_OPTID_CONF:
     LOG(WARN) << "conf: ignored";
 

src/shrpx_config.h

@@ -175,6 +175,8 @@ constexpr char SHRPX_OPT_TLS_TICKET_CIPHER[] = "tls-ticket-cipher";
 constexpr char SHRPX_OPT_HOST_REWRITE[] = "host-rewrite";
 constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED[] =
     "tls-session-cache-memcached";
+constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED[] =
+    "tls-ticket-key-memcached";
 
 union sockaddr_union {
   sockaddr_storage storage;
@@ -260,6 +262,7 @@ struct Config {
   // binary form of http proxy host and port
   Address downstream_http_proxy_addr;
   Address session_cache_memcached_addr;
+  Address tls_ticket_key_memcached_addr;
   std::chrono::seconds tls_session_timeout;
   ev_tstamp http2_upstream_read_timeout;
   ev_tstamp upstream_read_timeout;
@@ -271,6 +274,7 @@ struct Config {
   ev_tstamp downstream_idle_read_timeout;
   ev_tstamp listener_disable_timeout;
   ev_tstamp ocsp_update_interval;
+  ev_tstamp tls_ticket_key_memcached_interval;
   // address of frontend connection.  This could be a path to UNIX
   // domain socket.  In this case, |host_unix| must be true.
   std::unique_ptr<char[]> host;
@@ -303,6 +307,7 @@ struct Config {
   std::unique_ptr<char[]> fetch_ocsp_response_file;
   std::unique_ptr<char[]> user;
   std::unique_ptr<char[]> session_cache_memcached_host;
+  std::unique_ptr<char[]> tls_ticket_key_memcached_host;
   FILE *http2_upstream_dump_request_header;
   FILE *http2_upstream_dump_response_header;
   nghttp2_session_callbacks *http2_upstream_callbacks;
@@ -339,6 +344,12 @@ struct Config {
   size_t max_header_fields;
   // The index of catch-all group in downstream_addr_groups.
   size_t downstream_addr_group_catch_all;
+  // Maximum number of retries when getting TLS ticket key from
+  // mamcached, due to network error.
+  size_t tls_ticket_key_memcached_max_retry;
+  // Maximum number of consecutive error from memcached, when this
+  // limit reached, TLS ticket is disabled.
+  size_t tls_ticket_key_memcached_max_fail;
   // Bit mask to disable SSL/TLS protocol versions.  This will be
   // passed to SSL_CTX_set_options().
   long int tls_proto_mask;
@@ -356,6 +367,7 @@ struct Config {
   // port in http proxy URI
   uint16_t downstream_http_proxy_port;
   uint16_t session_cache_memcached_port;
+  uint16_t tls_ticket_key_memcached_port;
   bool verbose;
   bool daemon;
   bool verify_client;

src/shrpx_connection_handler.cc

@@ -32,6 +32,7 @@
 
 #include <cerrno>
 #include <thread>
+#include <random>
 
 #include "shrpx_client_handler.h"
 #include "shrpx_ssl.h"
@@ -41,6 +42,7 @@
 #include "shrpx_connect_blocker.h"
 #include "shrpx_downstream_connection.h"
 #include "shrpx_accept_handler.h"
+#include "shrpx_memcached_dispatcher.h"
 #include "util.h"
 #include "template.h"
 
@@ -94,7 +96,9 @@ void ocsp_chld_cb(struct ev_loop *loop, ev_child *w, int revent) {
 } // namespace
 
 ConnectionHandler::ConnectionHandler(struct ev_loop *loop)
-    : single_worker_(nullptr), loop_(loop), worker_round_robin_cnt_(0),
+    : single_worker_(nullptr), loop_(loop),
+      tls_ticket_key_memcached_get_retry_count_(0),
+      tls_ticket_key_memcached_fail_count_(0), worker_round_robin_cnt_(0),
       graceful_shutdown_(false) {
   ev_timer_init(&disable_acceptor_timer_, acceptor_disable_cb, 0., 0.);
   disable_acceptor_timer_.data = this;
@@ -553,4 +557,95 @@ void ConnectionHandler::proceed_next_cert_ocsp() {
   }
 }
 
+void ConnectionHandler::set_tls_ticket_key_memcached_dispatcher(
+    std::unique_ptr<MemcachedDispatcher> dispatcher) {
+  tls_ticket_key_memcached_dispatcher_ = std::move(dispatcher);
+}
+
+MemcachedDispatcher *
+ConnectionHandler::get_tls_ticket_key_memcached_dispatcher() const {
+  return tls_ticket_key_memcached_dispatcher_.get();
+}
+
+namespace {
+std::random_device rd;
+} // namespace
+
+void ConnectionHandler::on_tls_ticket_key_network_error(ev_timer *w) {
+  if (++tls_ticket_key_memcached_get_retry_count_ >=
+      get_config()->tls_ticket_key_memcached_max_retry) {
+    LOG(WARN) << "Memcached: tls ticket get retry all failed "
+              << tls_ticket_key_memcached_get_retry_count_ << " times.";
+
+    on_tls_ticket_key_not_found(w);
+    return;
+  }
+
+  auto dist = std::uniform_int_distribution<int>(
+      1, std::min(60, 1 << tls_ticket_key_memcached_get_retry_count_));
+  auto t = dist(rd);
+
+  LOG(WARN)
+      << "Memcached: tls ticket get failed due to network error, retrying in "
+      << t << " seconds";
+
+  ev_timer_set(w, t, 0.);
+  ev_timer_start(loop_, w);
+}
+
+void ConnectionHandler::on_tls_ticket_key_not_found(ev_timer *w) {
+  tls_ticket_key_memcached_get_retry_count_ = 0;
+
+  if (++tls_ticket_key_memcached_fail_count_ >=
+      get_config()->tls_ticket_key_memcached_max_fail) {
+    LOG(WARN) << "Memcached: could not get tls ticket; disable tls ticket";
+
+    tls_ticket_key_memcached_fail_count_ = 0;
+
+    set_ticket_keys(nullptr);
+    set_ticket_keys_to_worker(nullptr);
+  }
+
+  LOG(WARN) << "Memcached: tls ticket get failed, schedule next";
+  schedule_next_tls_ticket_key_memcached_get(w);
+}
+
+void ConnectionHandler::on_tls_ticket_key_get_success(
+    const std::shared_ptr<TicketKeys> &ticket_keys, ev_timer *w) {
+  if (LOG_ENABLED(INFO)) {
+    LOG(INFO) << "Memcached: tls ticket get success";
+  }
+
+  tls_ticket_key_memcached_get_retry_count_ = 0;
+  tls_ticket_key_memcached_fail_count_ = 0;
+
+  schedule_next_tls_ticket_key_memcached_get(w);
+
+  if (!ticket_keys || ticket_keys->keys.empty()) {
+    LOG(WARN) << "Memcached: tls ticket keys are empty; tls ticket disabled";
+    set_ticket_keys(nullptr);
+    set_ticket_keys_to_worker(nullptr);
+    return;
+  }
+
+  if (LOG_ENABLED(INFO)) {
+    LOG(INFO) << "ticket keys get done";
+    LOG(INFO) << 0 << " enc+dec: "
+              << util::format_hex(ticket_keys->keys[0].data.name);
+    for (size_t i = 1; i < ticket_keys->keys.size(); ++i) {
+      auto &key = ticket_keys->keys[i];
+      LOG(INFO) << i << " dec: " << util::format_hex(key.data.name);
+    }
+  }
+
+  set_ticket_keys(ticket_keys);
+  set_ticket_keys_to_worker(ticket_keys);
+}
+
+void
+ConnectionHandler::schedule_next_tls_ticket_key_memcached_get(ev_timer *w) {
+  ev_timer_set(w, get_config()->tls_ticket_key_memcached_interval, 0.);
+  ev_timer_start(loop_, w);
+}
+
 } // namespace shrpx

src/shrpx_connection_handler.h

@@ -49,6 +49,7 @@ class AcceptHandler;
 class Worker;
 struct WorkerStat;
 struct TicketKeys;
+class MemcachedDispatcher;
 
 struct OCSPUpdateContext {
   // ocsp response buffer
@@ -111,6 +112,17 @@ public:
   // update.
   void proceed_next_cert_ocsp();
 
+  void set_tls_ticket_key_memcached_dispatcher(
+      std::unique_ptr<MemcachedDispatcher> dispatcher);
+
+  MemcachedDispatcher *get_tls_ticket_key_memcached_dispatcher() const;
+  void on_tls_ticket_key_network_error(ev_timer *w);
+  void on_tls_ticket_key_not_found(ev_timer *w);
+  void
+  on_tls_ticket_key_get_success(const std::shared_ptr<TicketKeys> &ticket_keys,
+                                ev_timer *w);
+  void schedule_next_tls_ticket_key_memcached_get(ev_timer *w);
+
 private:
   // Stores all SSL_CTX objects.
   std::vector<SSL_CTX *> all_ssl_ctx_;
@@ -120,6 +132,7 @@ private:
   // Worker instance used when single threaded mode (-n1) is used.
   // Otherwise, nullptr and workers_ has instances of Worker instead.
   std::unique_ptr<Worker> single_worker_;
+  std::unique_ptr<MemcachedDispatcher> tls_ticket_key_memcached_dispatcher_;
   // Current TLS session ticket keys.  Note that TLS connection does
   // not refer to this field directly.  They use TicketKeys object in
   // Worker object.
@@ -131,6 +144,8 @@ private:
   std::unique_ptr<AcceptHandler> acceptor6_;
   ev_timer disable_acceptor_timer_;
   ev_timer ocsp_timer_;
+  size_t tls_ticket_key_memcached_get_retry_count_;
+  size_t tls_ticket_key_memcached_fail_count_;
   unsigned int worker_round_robin_cnt_;
   bool graceful_shutdown_;
 };

src/shrpx_memcached_connection.cc

@@ -100,7 +100,7 @@ namespace {
 void clear_request(std::deque<std::unique_ptr<MemcachedRequest>> &q) {
   for (auto &req : q) {
     if (req->cb) {
-      req->cb(req.get(), MemcachedResult(MEMCACHED_ERR_ERROR));
+      req->cb(req.get(), MemcachedResult(MEMCACHED_ERR_EXT_NETWORK_ERROR));
     }
   }
   q.clear();

src/shrpx_memcached_result.h

@@ -32,8 +32,8 @@
 namespace shrpx {
 
 enum MemcachedStatusCode {
-  MEMCACHED_ERR_OK,
+  MEMCACHED_ERR_NO_ERROR,
-  MEMCACHED_ERR_ERROR = 0x1001,
+  MEMCACHED_ERR_EXT_NETWORK_ERROR = 0x1001,
 };
 
 struct MemcachedResult {