Merge branch 'nghttpx-memcached-tls'

gennghttpxfun.py

@@ -92,6 +92,7 @@ OPTIONS = [
     "tls-ticket-key-cipher",
     "host-rewrite",
     "tls-session-cache-memcached",
+    "tls-session-cache-memcached-tls",
     "tls-ticket-key-memcached",
     "tls-ticket-key-memcached-interval",
     "tls-ticket-key-memcached-max-retry",
@@ -114,7 +115,14 @@ OPTIONS = [
     "max-header-fields",
     "no-http2-cipher-black-list",
     "backend-http1-tls",
-    "backend-tls-session-cache-per-worker"
+    "backend-tls-session-cache-per-worker",
+    "tls-session-cache-memcached-cert-file",
+    "tls-session-cache-memcached-private-key-file",
+    "tls-session-cache-memcached-address-family",
+    "tls-ticket-key-memcached-tls",
+    "tls-ticket-key-memcached-cert-file",
+    "tls-ticket-key-memcached-private-key-file",
+    "tls-ticket-key-memcached-address-family",
 ]
 
 LOGVARS = [

src/shrpx.cc

@@ -1051,6 +1051,13 @@ void fill_default_config() {
       memcachedconf.max_retry = 3;
       memcachedconf.max_fail = 2;
       memcachedconf.interval = 10_min;
+      memcachedconf.family = AF_UNSPEC;
+    }
+
+    auto &session_cacheconf = tlsconf.session_cache;
+    {
+      auto &memcachedconf = session_cacheconf.memcached;
+      memcachedconf.family = AF_UNSPEC;
     }
 
     ticketconf.cipher = EVP_aes_128_cbc();
@@ -1520,16 +1527,23 @@ SSL/TLS:
               ticket  key sharing  between  nghttpx  instances is  not
               required.
   --tls-ticket-key-memcached=<HOST>,<PORT>
-              Specify  address of  memcached server  to store  session
+              Specify address  of memcached  server to get  TLS ticket
-              cache.   This  enables  shared TLS  ticket  key  between
+              keys for  session resumption.   This enables  shared TLS
-              multiple nghttpx  instances.  nghttpx  does not  set TLS
+              ticket key between  multiple nghttpx instances.  nghttpx
-              ticket  key  to  memcached.   The  external  ticket  key
+              does not set TLS ticket  key to memcached.  The external
-              generator  is required.   nghttpx just  gets TLS  ticket
+              ticket key generator is required.  nghttpx just gets TLS
-              keys from  memcached, and  use them,  possibly replacing
+              ticket  keys  from  memcached, and  use  them,  possibly
-              current set of keys.  It is  up to extern TLS ticket key
+              replacing current set  of keys.  It is up  to extern TLS
-              generator to  rotate keys frequently.  See  "TLS SESSION
+              ticket  key generator  to rotate  keys frequently.   See
-              TICKET RESUMPTION"  section in  manual page to  know the
+              "TLS SESSION  TICKET RESUMPTION" section in  manual page
-              data format in memcached entry.
+              to know the data format in memcached entry.
+  --tls-ticket-key-memcached-address-family=(auto|IPv4|IPv6)
+              Specify address  family of memcached connections  to get
+              TLS ticket keys.  If "auto" is given, both IPv4 and IPv6
+              are considered.   If "IPv4" is given,  only IPv4 address
+              is considered.  If "IPv6" is given, only IPv6 address is
+              considered.
+              Default: auto
   --tls-ticket-key-memcached-interval=<DURATION>
               Set interval to get TLS ticket keys from memcached.
               Default: )"
@@ -1550,6 +1564,15 @@ SSL/TLS:
               Specify cipher  to encrypt TLS session  ticket.  Specify
               either   aes-128-cbc   or  aes-256-cbc.    By   default,
               aes-128-cbc is used.
+  --tls-ticket-key-memcached-tls
+              Enable  SSL/TLS  on  memcached connections  to  get  TLS
+              ticket keys.
+  --tls-ticket-key-memcached-cert-file=<PATH>
+              Path to client certificate  for memcached connections to
+              get TLS ticket keys.
+  --tls-ticket-key-memcached-private-key-file=<PATH>
+              Path to client private  key for memcached connections to
+              get TLS ticket keys.
   --fetch-ocsp-response-file=<PATH>
               Path to  fetch-ocsp-response script file.  It  should be
               absolute path.
@@ -1564,6 +1587,22 @@ SSL/TLS:
               Specify  address of  memcached server  to store  session
               cache.   This  enables   shared  session  cache  between
               multiple nghttpx instances.
+  --tls-session-cache-memcached-address-family=(auto|IPv4|IPv6)
+              Specify address family of memcached connections to store
+              session cache.  If  "auto" is given, both  IPv4 and IPv6
+              are considered.   If "IPv4" is given,  only IPv4 address
+              is considered.  If "IPv6" is given, only IPv6 address is
+              considered.
+              Default: auto
+  --tls-session-cache-memcached-tls
+              Enable SSL/TLS on memcached connections to store session
+              cache.
+  --tls-session-cache-memcached-cert-file=<PATH>
+              Path to client certificate  for memcached connections to
+              store session cache.
+  --tls-session-cache-memcached-private-key-file=<PATH>
+              Path to client private  key for memcached connections to
+              store session cache.
   --tls-dyn-rec-warmup-threshold=<SIZE>
               Specify the  threshold size for TLS  dynamic record size
               behaviour.  During  a TLS  session, after  the threshold
@@ -2181,7 +2220,7 @@ void process_options(
     auto &memcachedconf = tlsconf.session_cache.memcached;
     if (memcachedconf.host) {
       if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.get(),
-                           memcachedconf.port, AF_UNSPEC) == -1) {
+                           memcachedconf.port, memcachedconf.family) == -1) {
         exit(EXIT_FAILURE);
       }
     }
@@ -2191,7 +2230,7 @@ void process_options(
     auto &memcachedconf = tlsconf.ticket.memcached;
     if (memcachedconf.host) {
       if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.get(),
-                           memcachedconf.port, AF_UNSPEC) == -1) {
+                           memcachedconf.port, memcachedconf.family) == -1) {
         exit(EXIT_FAILURE);
       }
     }
@@ -2400,6 +2439,20 @@ int main(int argc, char **argv) {
         {SHRPX_OPT_BACKEND_HTTP1_TLS, no_argument, &flag, 106},
         {SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER, required_argument,
          &flag, 107},
+        {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_TLS, no_argument, &flag, 108},
+        {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE, required_argument,
+         &flag, 109},
+        {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE,
+         required_argument, &flag, 110},
+        {SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_TLS, no_argument, &flag, 111},
+        {SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_CERT_FILE, required_argument, &flag,
+         112},
+        {SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE, required_argument,
+         &flag, 113},
+        {SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY, required_argument,
+         &flag, 114},
+        {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY,
+         required_argument, &flag, 115},
         {nullptr, 0, nullptr, 0}};
 
     int option_index = 0;
@@ -2858,6 +2911,44 @@ int main(int argc, char **argv) {
         cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER,
                              optarg);
         break;
+      case 108:
+        // --tls-session-cache-memcached-tls
+        cmdcfgs.emplace_back(SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_TLS, "yes");
+        break;
+      case 109:
+        // --tls-session-cache-memcached-cert-file
+        cmdcfgs.emplace_back(SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE,
+                             optarg);
+        break;
+      case 110:
+        // --tls-session-cache-memcached-private-key-file
+        cmdcfgs.emplace_back(
+            SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE, optarg);
+        break;
+      case 111:
+        // --tls-ticket-key-memcached-tls
+        cmdcfgs.emplace_back(SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_TLS, "yes");
+        break;
+      case 112:
+        // --tls-ticket-key-memcached-cert-file
+        cmdcfgs.emplace_back(SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_CERT_FILE,
+                             optarg);
+        break;
+      case 113:
+        // --tls-ticket-key-memcached-private-key-file
+        cmdcfgs.emplace_back(
+            SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE, optarg);
+        break;
+      case 114:
+        // --tls-ticket-key-memcached-address-family
+        cmdcfgs.emplace_back(SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY,
+                             optarg);
+        break;
+      case 115:
+        // --tls-session-cache-memcached-address-family
+        cmdcfgs.emplace_back(
+            SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY, optarg);
+        break;
       default:
         break;
       }

src/shrpx_config.cc

@@ -575,6 +575,26 @@ std::vector<LogFragment> parse_log_format(const char *optarg) {
   return res;
 }
 
+namespace {
+int parse_address_family(int *dest, const char *opt, const char *optarg) {
+  if (util::strieq("auto", optarg)) {
+    *dest = AF_UNSPEC;
+    return 0;
+  }
+  if (util::strieq("IPv4", optarg)) {
+    *dest = AF_INET;
+    return 0;
+  }
+  if (util::strieq("IPv6", optarg)) {
+    *dest = AF_INET6;
+    return 0;
+  }
+
+  LOG(ERROR) << opt << ": bad value: '" << optarg << "'";
+  return -1;
+}
+} // namespace
+
 namespace {
 int parse_duration(ev_tstamp *dest, const char *opt, const char *optarg) {
   auto t = util::parse_duration_with_unit(optarg);
@@ -758,12 +778,20 @@ enum {
   SHRPX_OPTID_TLS_DYN_REC_WARMUP_THRESHOLD,
   SHRPX_OPTID_TLS_PROTO_LIST,
   SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED,
+  SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY,
+  SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE,
+  SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE,
+  SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_TLS,
   SHRPX_OPTID_TLS_TICKET_KEY_CIPHER,
   SHRPX_OPTID_TLS_TICKET_KEY_FILE,
   SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED,
+  SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY,
+  SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_CERT_FILE,
   SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_INTERVAL,
   SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_MAX_FAIL,
   SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_MAX_RETRY,
+  SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE,
+  SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_TLS,
   SHRPX_OPTID_USER,
   SHRPX_OPTID_VERIFY_CLIENT,
   SHRPX_OPTID_VERIFY_CLIENT_CACERT,
@@ -1325,6 +1353,9 @@ int option_lookup_token(const char *name, size_t namelen) {
       if (util::strieq_l("http2-max-concurrent-stream", name, 27)) {
         return SHRPX_OPTID_HTTP2_MAX_CONCURRENT_STREAMS;
       }
+      if (util::strieq_l("tls-ticket-key-memcached-tl", name, 27)) {
+        return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_TLS;
+      }
       break;
     }
     break;
@@ -1337,6 +1368,15 @@ int option_lookup_token(const char *name, size_t namelen) {
       break;
     }
     break;
+  case 31:
+    switch (name[30]) {
+    case 's':
+      if (util::strieq_l("tls-session-cache-memcached-tl", name, 30)) {
+        return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_TLS;
+      }
+      break;
+    }
+    break;
   case 33:
     switch (name[32]) {
     case 'l':
@@ -1351,6 +1391,11 @@ int option_lookup_token(const char *name, size_t namelen) {
     break;
   case 34:
     switch (name[33]) {
+    case 'e':
+      if (util::strieq_l("tls-ticket-key-memcached-cert-fil", name, 33)) {
+        return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_CERT_FILE;
+      }
+      break;
     case 'r':
       if (util::strieq_l("frontend-http2-dump-request-heade", name, 33)) {
         return SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER;
@@ -1396,6 +1441,11 @@ int option_lookup_token(const char *name, size_t namelen) {
     break;
   case 37:
     switch (name[36]) {
+    case 'e':
+      if (util::strieq_l("tls-session-cache-memcached-cert-fil", name, 36)) {
+        return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE;
+      }
+      break;
     case 's':
       if (util::strieq_l("frontend-http2-connection-window-bit", name, 36)) {
         return SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS;
@@ -1412,6 +1462,45 @@ int option_lookup_token(const char *name, size_t namelen) {
       break;
     }
     break;
+  case 39:
+    switch (name[38]) {
+    case 'y':
+      if (util::strieq_l("tls-ticket-key-memcached-address-famil", name, 38)) {
+        return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY;
+      }
+      break;
+    }
+    break;
+  case 41:
+    switch (name[40]) {
+    case 'e':
+      if (util::strieq_l("tls-ticket-key-memcached-private-key-fil", name,
+                         40)) {
+        return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE;
+      }
+      break;
+    }
+    break;
+  case 42:
+    switch (name[41]) {
+    case 'y':
+      if (util::strieq_l("tls-session-cache-memcached-address-famil", name,
+                         41)) {
+        return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY;
+      }
+      break;
+    }
+    break;
+  case 44:
+    switch (name[43]) {
+    case 'e':
+      if (util::strieq_l("tls-session-cache-memcached-private-key-fil", name,
+                         43)) {
+        return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE;
+      }
+      break;
+    }
+    break;
   }
   return -1;
 }
@@ -2229,6 +2318,36 @@ int parse_config(const char *opt, const char *optarg,
   case SHRPX_OPTID_BACKEND_TLS_SESSION_CACHE_PER_WORKER:
     return parse_uint(&mod_config()->tls.downstream_session_cache_per_worker,
                       opt, optarg);
+  case SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_TLS:
+    mod_config()->tls.session_cache.memcached.tls = util::strieq(optarg, "yes");
+
+    return 0;
+  case SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE:
+    mod_config()->tls.session_cache.memcached.cert_file = optarg;
+
+    return 0;
+  case SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE:
+    mod_config()->tls.session_cache.memcached.private_key_file = optarg;
+
+    return 0;
+  case SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_TLS:
+    mod_config()->tls.ticket.memcached.tls = util::strieq(optarg, "yes");
+
+    return 0;
+  case SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_CERT_FILE:
+    mod_config()->tls.ticket.memcached.cert_file = optarg;
+
+    return 0;
+  case SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE:
+    mod_config()->tls.ticket.memcached.private_key_file = optarg;
+
+    return 0;
+  case SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY:
+    return parse_address_family(&mod_config()->tls.ticket.memcached.family, opt,
+                                optarg);
+  case SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY:
+    return parse_address_family(
+        &mod_config()->tls.session_cache.memcached.family, opt, optarg);
   case SHRPX_OPTID_CONF:
     LOG(WARN) << "conf: ignored";
 

src/shrpx_config.h

@@ -209,6 +209,22 @@ constexpr char SHRPX_OPT_NO_HTTP2_CIPHER_BLACK_LIST[] =
 constexpr char SHRPX_OPT_BACKEND_HTTP1_TLS[] = "backend-http1-tls";
 constexpr char SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER[] =
     "backend-tls-session-cache-per-worker";
+constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_TLS[] =
+    "tls-session-cache-memcached-tls";
+constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE[] =
+    "tls-session-cache-memcached-cert-file";
+constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE[] =
+    "tls-session-cache-memcached-private-key-file";
+constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY[] =
+    "tls-session-cache-memcached-address-family";
+constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_TLS[] =
+    "tls-ticket-key-memcached-tls";
+constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_CERT_FILE[] =
+    "tls-ticket-key-memcached-cert-file";
+constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE[] =
+    "tls-ticket-key-memcached-private-key-file";
+constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY[] =
+    "tls-ticket-key-memcached-address-family";
 
 constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8;
 
@@ -335,6 +351,9 @@ struct TLSConfig {
       Address addr;
       uint16_t port;
       std::unique_ptr<char[]> host;
+      // Client private key and certificate for authentication
+      ImmutableString private_key_file;
+      ImmutableString cert_file;
       ev_tstamp interval;
       // Maximum number of retries when getting TLS ticket key from
       // mamcached, due to network error.
@@ -342,6 +361,10 @@ struct TLSConfig {
       // Maximum number of consecutive error from memcached, when this
       // limit reached, TLS ticket is disabled.
       size_t max_fail;
+      // Address family of memcached connection.  One of either
+      // AF_INET, AF_INET6 or AF_UNSPEC.
+      int family;
+      bool tls;
     } memcached;
     std::vector<std::string> files;
     const EVP_CIPHER *cipher;
@@ -355,6 +378,13 @@ struct TLSConfig {
       Address addr;
       uint16_t port;
       std::unique_ptr<char[]> host;
+      // Client private key and certificate for authentication
+      ImmutableString private_key_file;
+      ImmutableString cert_file;
+      // Address family of memcached connection.  One of either
+      // AF_INET, AF_INET6 or AF_UNSPEC.
+      int family;
+      bool tls;
     } memcached;
   } session_cache;
 

src/shrpx_connection_handler.cc

@@ -193,8 +193,24 @@ int ConnectionHandler::create_single_worker() {
     all_ssl_ctx_.push_back(cl_ssl_ctx);
   }
 
-  single_worker_ = make_unique<Worker>(loop_, sv_ssl_ctx, cl_ssl_ctx, cert_tree,
+  auto &tlsconf = get_config()->tls;
-                                       ticket_keys_);
+  auto &memcachedconf = get_config()->tls.session_cache.memcached;
+
+  SSL_CTX *session_cache_ssl_ctx = nullptr;
+  if (memcachedconf.tls) {
+    session_cache_ssl_ctx = ssl::create_ssl_client_context(
+#ifdef HAVE_NEVERBLEED
+        nb_.get(),
+#endif // HAVE_NEVERBLEED
+        StringRef::from_maybe_nullptr(tlsconf.cacert.get()),
+        StringRef(memcachedconf.cert_file),
+        StringRef(memcachedconf.private_key_file), StringRef(), nullptr);
+    all_ssl_ctx_.push_back(session_cache_ssl_ctx);
+  }
+
+  single_worker_ =
+      make_unique<Worker>(loop_, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx,
+                          cert_tree, ticket_keys_);
 #ifdef HAVE_MRUBY
   if (single_worker_->create_mruby_context() != 0) {
     return -1;
@@ -225,11 +241,26 @@ int ConnectionHandler::create_worker_thread(size_t num) {
     all_ssl_ctx_.push_back(cl_ssl_ctx);
   }
 
+  auto &tlsconf = get_config()->tls;
+  auto &memcachedconf = get_config()->tls.session_cache.memcached;
+
   for (size_t i = 0; i < num; ++i) {
     auto loop = ev_loop_new(0);
 
-    auto worker = make_unique<Worker>(loop, sv_ssl_ctx, cl_ssl_ctx, cert_tree,
+    SSL_CTX *session_cache_ssl_ctx = nullptr;
-                                      ticket_keys_);
+    if (memcachedconf.tls) {
+      session_cache_ssl_ctx = ssl::create_ssl_client_context(
+#ifdef HAVE_NEVERBLEED
+          nb_.get(),
+#endif // HAVE_NEVERBLEED
+          StringRef::from_maybe_nullptr(tlsconf.cacert.get()),
+          StringRef(memcachedconf.cert_file),
+          StringRef(memcachedconf.private_key_file), StringRef(), nullptr);
+      all_ssl_ctx_.push_back(session_cache_ssl_ctx);
+    }
+    auto worker =
+        make_unique<Worker>(loop, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx,
+                            cert_tree, ticket_keys_);
 #ifdef HAVE_MRUBY
     if (worker->create_mruby_context() != 0) {
       return -1;
@@ -728,6 +759,23 @@ void ConnectionHandler::schedule_next_tls_ticket_key_memcached_get(
   ev_timer_start(loop_, w);
 }
 
+SSL_CTX *ConnectionHandler::create_tls_ticket_key_memcached_ssl_ctx() {
+  auto &tlsconf = get_config()->tls;
+  auto &memcachedconf = get_config()->tls.ticket.memcached;
+
+  auto ssl_ctx = ssl::create_ssl_client_context(
+#ifdef HAVE_NEVERBLEED
+      nb_.get(),
+#endif // HAVE_NEVERBLEED
+      StringRef::from_maybe_nullptr(tlsconf.cacert.get()),
+      StringRef(memcachedconf.cert_file),
+      StringRef(memcachedconf.private_key_file), StringRef(), nullptr);
+
+  all_ssl_ctx_.push_back(ssl_ctx);
+
+  return ssl_ctx;
+}
+
 #ifdef HAVE_NEVERBLEED
 void ConnectionHandler::set_neverbleed(std::unique_ptr<neverbleed_t> nb) {
   nb_ = std::move(nb);

src/shrpx_connection_handler.h

@@ -129,6 +129,7 @@ public:
   on_tls_ticket_key_get_success(const std::shared_ptr<TicketKeys> &ticket_keys,
                                 ev_timer *w);
   void schedule_next_tls_ticket_key_memcached_get(ev_timer *w);
+  SSL_CTX *create_tls_ticket_key_memcached_ssl_ctx();
 
 #ifdef HAVE_NEVERBLEED
   void set_neverbleed(std::unique_ptr<neverbleed_t> nb);

src/shrpx_memcached_connection.cc

@@ -32,6 +32,7 @@
 #include "shrpx_memcached_request.h"
 #include "shrpx_memcached_result.h"
 #include "shrpx_config.h"
+#include "shrpx_ssl.h"
 #include "util.h"
 
 namespace shrpx {
@@ -78,7 +79,7 @@ void connectcb(struct ev_loop *loop, ev_io *w, int revents) {
   auto conn = static_cast<Connection *>(w->data);
   auto mconn = static_cast<MemcachedConnection *>(conn->data);
 
-  if (mconn->on_connect() != 0) {
+  if (mconn->connected() != 0) {
     mconn->disconnect();
     return;
   }
@@ -91,11 +92,17 @@ constexpr ev_tstamp write_timeout = 10.;
 constexpr ev_tstamp read_timeout = 10.;
 
 MemcachedConnection::MemcachedConnection(const Address *addr,
-                                         struct ev_loop *loop)
+                                         struct ev_loop *loop, SSL_CTX *ssl_ctx,
-    : conn_(loop, -1, nullptr, nullptr, write_timeout, read_timeout, {}, {},
+                                         const StringRef &sni_name,
+                                         MemchunkPool *mcpool)
+    : conn_(loop, -1, nullptr, mcpool, write_timeout, read_timeout, {}, {},
             connectcb, readcb, timeoutcb, this, 0, 0.),
+      do_read_(&MemcachedConnection::noop),
+      do_write_(&MemcachedConnection::noop),
+      sni_name_(sni_name.str()),
       parse_state_{},
       addr_(addr),
+      ssl_ctx_(ssl_ctx),
       sendsum_(0),
       connected_(false) {}
 
@@ -127,11 +134,21 @@ void MemcachedConnection::disconnect() {
 
   assert(recvbuf_.rleft() == 0);
   recvbuf_.reset();
+
+  do_read_ = do_write_ = &MemcachedConnection::noop;
 }
 
 int MemcachedConnection::initiate_connection() {
   assert(conn_.fd == -1);
 
+  if (ssl_ctx_ && !conn_.tls.ssl) {
+    auto ssl = ssl::create_ssl(ssl_ctx_);
+    if (!ssl) {
+      return -1;
+    }
+    conn_.set_ssl(ssl);
+  }
+
   conn_.fd = util::create_nonblock_socket(addr_->su.storage.ss_family);
 
   if (conn_.fd == -1) {
@@ -153,6 +170,14 @@ int MemcachedConnection::initiate_connection() {
     return -1;
   }
 
+  if (ssl_ctx_) {
+    if (!util::numeric_host(sni_name_.c_str())) {
+      SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name_.c_str());
+    }
+
+    conn_.prepare_client_handshake();
+  }
+
   if (LOG_ENABLED(INFO)) {
     MCLOG(INFO, this) << "Connecting to memcached server";
   }
@@ -168,7 +193,7 @@ int MemcachedConnection::initiate_connection() {
   return 0;
 }
 
-int MemcachedConnection::on_connect() {
+int MemcachedConnection::connected() {
   if (!util::check_socket_connected(conn_.fd)) {
     conn_.wlimit.stopw();
 
@@ -185,15 +210,59 @@ int MemcachedConnection::on_connect() {
 
   connected_ = true;
 
-  ev_set_cb(&conn_.wev, writecb);
-
   conn_.rlimit.startw();
   ev_timer_again(conn_.loop, &conn_.rt);
 
+  ev_set_cb(&conn_.wev, writecb);
+
+  if (conn_.tls.ssl) {
+    do_read_ = &MemcachedConnection::tls_handshake;
+    do_write_ = &MemcachedConnection::tls_handshake;
+
+    return 0;
+  }
+
+  do_read_ = &MemcachedConnection::read_clear;
+  do_write_ = &MemcachedConnection::write_clear;
+
   return 0;
 }
 
-int MemcachedConnection::on_write() {
+int MemcachedConnection::on_write() { return do_write_(*this); }
+int MemcachedConnection::on_read() { return do_read_(*this); }
+
+int MemcachedConnection::tls_handshake() {
+  ERR_clear_error();
+
+  ev_timer_again(conn_.loop, &conn_.rt);
+
+  auto rv = conn_.tls_handshake();
+  if (rv == SHRPX_ERR_INPROGRESS) {
+    return 0;
+  }
+
+  if (rv < 0) {
+    return rv;
+  }
+
+  if (LOG_ENABLED(INFO)) {
+    LOG(INFO) << "SSL/TLS handshake completed";
+  }
+
+  auto &tlsconf = get_config()->tls;
+
+  if (!tlsconf.insecure &&
+      ssl::check_cert(conn_.tls.ssl, addr_, StringRef(sni_name_)) != 0) {
+    return -1;
+  }
+
+  do_read_ = &MemcachedConnection::read_tls;
+  do_write_ = &MemcachedConnection::write_tls;
+
+  return on_write();
+}
+
+int MemcachedConnection::write_tls() {
   if (!connected_) {
     return 0;
   }
@@ -207,19 +276,30 @@ int MemcachedConnection::on_write() {
     return 0;
   }
 
-  int rv;
+  std::array<struct iovec, MAX_WR_IOVCNT> iov;
+  std::array<uint8_t, 16_k> buf;
 
   for (; !sendq_.empty();) {
-    rv = send_request();
+    auto iovcnt = fill_request_buffer(iov.data(), iov.size());
+    auto p = std::begin(buf);
+    for (size_t i = 0; i < iovcnt; ++i) {
+      auto &v = iov[i];
+      auto n = std::min(static_cast<size_t>(std::end(buf) - p), v.iov_len);
+      p = std::copy_n(static_cast<uint8_t *>(v.iov_base), n, p);
+      if (p == std::end(buf)) {
+        break;
+      }
+    }
 
-    if (rv < 0) {
+    auto nwrite = conn_.write_tls(buf.data(), p - std::begin(buf));
+    if (nwrite < 0) {
       return -1;
     }
-
+    if (nwrite == 0) {
-    if (rv == 1) {
-      // blocked
       return 0;
     }
+
+    drain_send_queue(nwrite);
   }
 
   conn_.wlimit.stopw();
@@ -228,7 +308,70 @@ int MemcachedConnection::on_write() {
   return 0;
 }
 
-int MemcachedConnection::on_read() {
+int MemcachedConnection::read_tls() {
+  if (!connected_) {
+    return 0;
+  }
+
+  ev_timer_again(conn_.loop, &conn_.rt);
+
+  for (;;) {
+    auto nread = conn_.read_tls(recvbuf_.last, recvbuf_.wleft());
+
+    if (nread == 0) {
+      return 0;
+    }
+
+    if (nread < 0) {
+      return -1;
+    }
+
+    recvbuf_.write(nread);
+
+    if (parse_packet() != 0) {
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+int MemcachedConnection::write_clear() {
+  if (!connected_) {
+    return 0;
+  }
+
+  ev_timer_again(conn_.loop, &conn_.rt);
+
+  if (sendq_.empty()) {
+    conn_.wlimit.stopw();
+    ev_timer_stop(conn_.loop, &conn_.wt);
+
+    return 0;
+  }
+
+  std::array<struct iovec, MAX_WR_IOVCNT> iov;
+
+  for (; !sendq_.empty();) {
+    auto iovcnt = fill_request_buffer(iov.data(), iov.size());
+    auto nwrite = conn_.writev_clear(iov.data(), iovcnt);
+    if (nwrite < 0) {
+      return -1;
+    }
+    if (nwrite == 0) {
+      return 0;
+    }
+
+    drain_send_queue(nwrite);
+  }
+
+  conn_.wlimit.stopw();
+  ev_timer_stop(conn_.loop, &conn_.wt);
+
+  return 0;
+}
+
+int MemcachedConnection::read_clear() {
   if (!connected_) {
     return 0;
   }
@@ -415,9 +558,8 @@ int MemcachedConnection::parse_packet() {
 #define MAX_WR_IOVCNT DEFAULT_WR_IOVCNT
 #endif // !defined(IOV_MAX) || IOV_MAX >= DEFAULT_WR_IOVCNT
 
-int MemcachedConnection::send_request() {
+size_t MemcachedConnection::fill_request_buffer(struct iovec *iov,
-  ssize_t nwrite;
+                                                size_t iovlen) {
-
   if (sendsum_ == 0) {
     for (auto &req : sendq_) {
       if (req->canceled) {
@@ -438,32 +580,27 @@ int MemcachedConnection::send_request() {
     }
   }
 
-  std::array<struct iovec, DEFAULT_WR_IOVCNT> iov;
+  size_t iovcnt = 0;
-  size_t iovlen = 0;
   for (auto &buf : sendbufv_) {
-    if (iovlen + 2 > iov.size()) {
+    if (iovcnt + 2 > iovlen) {
       break;
     }
 
     auto req = buf.req;
     if (buf.headbuf.rleft()) {
-      iov[iovlen++] = {buf.headbuf.pos, buf.headbuf.rleft()};
+      iov[iovcnt++] = {buf.headbuf.pos, buf.headbuf.rleft()};
     }
     if (buf.send_value_left) {
-      iov[iovlen++] = {req->value.data() + req->value.size() -
+      iov[iovcnt++] = {req->value.data() + req->value.size() -
                            buf.send_value_left,
                        buf.send_value_left};
     }
   }
 
-  nwrite = conn_.writev_clear(iov.data(), iovlen);
+  return iovcnt;
-  if (nwrite < 0) {
+}
-    return -1;
-  }
-  if (nwrite == 0) {
-    return 1;
-  }
 
+void MemcachedConnection::drain_send_queue(size_t nwrite) {
   sendsum_ -= nwrite;
 
   while (nwrite > 0) {
@@ -488,8 +625,6 @@ int MemcachedConnection::send_request() {
     recvq_.push_back(std::move(sendq_.front()));
     sendq_.pop_front();
   }
-
-  return 0;
 }
 
 size_t MemcachedConnection::serialized_size(MemcachedRequest *req) {
@@ -549,4 +684,6 @@ int MemcachedConnection::add_request(std::unique_ptr<MemcachedRequest> req) {
 // TODO should we start write timer too?
 void MemcachedConnection::signal_write() { conn_.wlimit.startw(); }
 
+int MemcachedConnection::noop() { return 0; }
+
 } // namespace shrpx

src/shrpx_memcached_connection.h

@@ -93,7 +93,9 @@ constexpr uint8_t MEMCACHED_RES_MAGIC = 0x81;
 // https://code.google.com/p/memcached/wiki/MemcacheBinaryProtocol
 class MemcachedConnection {
 public:
-  MemcachedConnection(const Address *addr, struct ev_loop *loop);
+  MemcachedConnection(const Address *addr, struct ev_loop *loop,
+                      SSL_CTX *ssl_ctx, const StringRef &sni_name,
+                      MemchunkPool *mcpool);
   ~MemcachedConnection();
 
   void disconnect();
@@ -101,23 +103,38 @@ public:
   int add_request(std::unique_ptr<MemcachedRequest> req);
   int initiate_connection();
 
-  int on_connect();
+  int connected();
   int on_write();
   int on_read();
-  int send_request();
+
+  int write_clear();
+  int read_clear();
+
+  int tls_handshake();
+  int write_tls();
+  int read_tls();
+
+  size_t fill_request_buffer(struct iovec *iov, size_t iovlen);
+  void drain_send_queue(size_t nwrite);
+
   void make_request(MemcachedSendbuf *sendbuf, MemcachedRequest *req);
   int parse_packet();
   size_t serialized_size(MemcachedRequest *req);
 
   void signal_write();
 
+  int noop();
+
 private:
   Connection conn_;
   std::deque<std::unique_ptr<MemcachedRequest>> recvq_;
   std::deque<std::unique_ptr<MemcachedRequest>> sendq_;
   std::deque<MemcachedSendbuf> sendbufv_;
+  std::function<int(MemcachedConnection &)> do_read_, do_write_;
+  std::string sni_name_;
   MemcachedParseState parse_state_;
   const Address *addr_;
+  SSL_CTX *ssl_ctx_;
   // Sum of the bytes to be transmitted in sendbufv_.
   size_t sendsum_;
   bool connected_;

src/shrpx_memcached_dispatcher.cc

@@ -31,8 +31,12 @@
 namespace shrpx {
 
 MemcachedDispatcher::MemcachedDispatcher(const Address *addr,
-                                         struct ev_loop *loop)
+                                         struct ev_loop *loop, SSL_CTX *ssl_ctx,
-    : loop_(loop), mconn_(make_unique<MemcachedConnection>(addr, loop_)) {}
+                                         const StringRef &sni_name,
+                                         MemchunkPool *mcpool)
+    : loop_(loop),
+      mconn_(make_unique<MemcachedConnection>(addr, loop_, ssl_ctx, sni_name,
+                                              mcpool)) {}
 
 MemcachedDispatcher::~MemcachedDispatcher() {}
 

src/shrpx_memcached_dispatcher.h

@@ -31,6 +31,10 @@
 
 #include <ev.h>
 
+#include <openssl/ssl.h>
+
+#include "memchunk.h"
+
 namespace shrpx {
 
 struct MemcachedRequest;
@@ -39,7 +43,9 @@ struct Address;
 
 class MemcachedDispatcher {
 public:
-  MemcachedDispatcher(const Address *addr, struct ev_loop *loop);
+  MemcachedDispatcher(const Address *addr, struct ev_loop *loop,
+                      SSL_CTX *ssl_ctx, const StringRef &sni_name,
+                      MemchunkPool *mcpool);
   ~MemcachedDispatcher();
 
   int add_request(std::unique_ptr<MemcachedRequest> req);

src/shrpx_ssl.cc

@@ -662,8 +662,8 @@ SSL_CTX *create_ssl_client_context(
 #ifdef HAVE_NEVERBLEED
     neverbleed_t *nb,
 #endif // HAVE_NEVERBLEED
-    const char *cacert, const char *cert_file, const char *private_key_file,
+    const StringRef &cacert, const StringRef &cert_file,
-    const StringRef &alpn,
+    const StringRef &private_key_file, const StringRef &alpn,
     int (*next_proto_select_cb)(SSL *s, unsigned char **out,
                                 unsigned char *outlen, const unsigned char *in,
                                 unsigned int inlen, void *arg)) {
@@ -702,8 +702,8 @@ SSL_CTX *create_ssl_client_context(
               << ERR_error_string(ERR_get_error(), nullptr);
   }
 
-  if (cacert) {
+  if (!cacert.empty()) {
-    if (SSL_CTX_load_verify_locations(ssl_ctx, cacert, nullptr) != 1) {
+    if (SSL_CTX_load_verify_locations(ssl_ctx, cacert.c_str(), nullptr) != 1) {
 
       LOG(FATAL) << "Could not load trusted ca certificates from " << cacert
                  << ": " << ERR_error_string(ERR_get_error(), nullptr);
@@ -711,8 +711,8 @@ SSL_CTX *create_ssl_client_context(
     }
   }
 
-  if (cert_file) {
+  if (!cert_file.empty()) {
-    if (SSL_CTX_use_certificate_chain_file(ssl_ctx, cert_file) != 1) {
+    if (SSL_CTX_use_certificate_chain_file(ssl_ctx, cert_file.c_str()) != 1) {
 
       LOG(FATAL) << "Could not load client certificate from " << cert_file
                  << ": " << ERR_error_string(ERR_get_error(), nullptr);
@@ -720,9 +720,9 @@ SSL_CTX *create_ssl_client_context(
     }
   }
 
-  if (private_key_file) {
+  if (!private_key_file.empty()) {
 #ifndef HAVE_NEVERBLEED
-    if (SSL_CTX_use_PrivateKey_file(ssl_ctx, private_key_file,
+    if (SSL_CTX_use_PrivateKey_file(ssl_ctx, private_key_file.c_str(),
                                     SSL_FILETYPE_PEM) != 1) {
       LOG(FATAL) << "Could not load client private key from "
                  << private_key_file << ": "
@@ -731,7 +731,7 @@ SSL_CTX *create_ssl_client_context(
     }
 #else  // HAVE_NEVERBLEED
     std::array<char, NEVERBLEED_ERRBUF_SIZE> errbuf;
-    if (neverbleed_load_private_key_file(nb, ssl_ctx, private_key_file,
+    if (neverbleed_load_private_key_file(nb, ssl_ctx, private_key_file.c_str(),
                                          errbuf.data()) != 1) {
       LOG(FATAL) << "neverbleed_load_private_key_file: could not load client "
                     "private key from " << private_key_file << ": "
@@ -982,7 +982,7 @@ int verify_hostname(X509 *cert, const char *hostname, size_t hlen,
 }
 } // namespace
 
-int check_cert(SSL *ssl, const DownstreamAddr *addr) {
+int check_cert(SSL *ssl, const Address *addr, const StringRef &host) {
   auto cert = SSL_get_peer_certificate(ssl);
   if (!cert) {
     LOG(ERROR) << "No certificate found";
@@ -996,18 +996,21 @@ int check_cert(SSL *ssl, const DownstreamAddr *addr) {
     return -1;
   }
 
-  auto &backend_sni_name = get_config()->tls.backend_sni_name;
+  if (verify_hostname(cert, host.c_str(), host.size(), addr) != 0) {
-
-  auto hostname = !backend_sni_name.empty() ? StringRef(backend_sni_name)
-                                            : StringRef(addr->host);
-  if (verify_hostname(cert, hostname.c_str(), hostname.size(), &addr->addr) !=
-      0) {
     LOG(ERROR) << "Certificate verification failed: hostname does not match";
     return -1;
   }
   return 0;
 }
 
+int check_cert(SSL *ssl, const DownstreamAddr *addr) {
+  auto &backend_sni_name = get_config()->tls.backend_sni_name;
+
+  auto hostname = !backend_sni_name.empty() ? StringRef(backend_sni_name)
+                                            : StringRef(addr->host);
+  return check_cert(ssl, &addr->addr, hostname);
+}
+
 CertLookupTree::CertLookupTree() {
   root_.ssl_ctx = nullptr;
   root_.str = nullptr;
@@ -1320,8 +1323,10 @@ SSL_CTX *setup_downstream_client_ssl_context(
 #ifdef HAVE_NEVERBLEED
       nb,
 #endif // HAVE_NEVERBLEED
-      tlsconf.cacert.get(), tlsconf.client.cert_file.get(),
+      StringRef::from_maybe_nullptr(tlsconf.cacert.get()),
-      tlsconf.client.private_key_file.get(), alpn, next_proto_select_cb);
+      StringRef::from_maybe_nullptr(tlsconf.client.cert_file.get()),
+      StringRef::from_maybe_nullptr(tlsconf.client.private_key_file.get()),
+      alpn, next_proto_select_cb);
 }
 
 CertLookupTree *create_cert_lookup_tree() {

src/shrpx_ssl.h

@@ -46,6 +46,7 @@ class Worker;
 class DownstreamConnectionPool;
 struct DownstreamAddr;
 struct UpstreamAddr;
+struct Address;
 
 namespace ssl {
 
@@ -74,8 +75,8 @@ SSL_CTX *create_ssl_client_context(
 #ifdef HAVE_NEVERBLEED
     neverbleed_t *nb,
 #endif // HAVE_NEVERBLEED
-    const char *cacert, const char *cert_file, const char *private_key_file,
+    const StringRef &cacert, const StringRef &cert_file,
-    const StringRef &alpn,
+    const StringRef &private_key_file, const StringRef &alpn,
     int (*next_proto_select_cb)(SSL *s, unsigned char **out,
                                 unsigned char *outlen, const unsigned char *in,
                                 unsigned int inlen, void *arg));
@@ -83,9 +84,8 @@ SSL_CTX *create_ssl_client_context(
 ClientHandler *accept_connection(Worker *worker, int fd, sockaddr *addr,
                                  int addrlen, const UpstreamAddr *faddr);
 
-// Check peer's certificate against first downstream address in
+// Check peer's certificate against given |address| and |host|.
-// Config::downstream_addrs.  We only consider first downstream since
+int check_cert(SSL *ssl, const Address *addr, const StringRef &host);
-// we use this function for HTTP/2 downstream link only.
 int check_cert(SSL *ssl, const DownstreamAddr *addr);
 
 // Retrieves DNS and IP address in subjectAltNames and commonName from

src/shrpx_worker.cc

@@ -68,6 +68,7 @@ std::random_device rd;
 } // namespace
 
 Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
+               SSL_CTX *tls_session_cache_memcached_ssl_ctx,
                ssl::CertLookupTree *cert_tree,
                const std::shared_ptr<TicketKeys> &ticket_keys)
     : randgen_(rd()),
@@ -92,7 +93,9 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
 
   if (session_cacheconf.memcached.host) {
     session_cache_memcached_dispatcher_ = make_unique<MemcachedDispatcher>(
-        &session_cacheconf.memcached.addr, loop);
+        &session_cacheconf.memcached.addr, loop,
+        tls_session_cache_memcached_ssl_ctx,
+        session_cacheconf.memcached.host.get(), &mcpool_);
   }
 
   auto &downstreamconf = get_config()->conn.downstream;

src/shrpx_worker.h

@@ -112,6 +112,7 @@ struct SessionCacheEntry {
 class Worker {
 public:
   Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
+         SSL_CTX *tls_session_cache_memcached_ssl_ctx,
          ssl::CertLookupTree *cert_tree,
          const std::shared_ptr<TicketKeys> &ticket_keys);
   ~Worker();

src/shrpx_worker_process.cc

@@ -420,13 +420,24 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) {
 
 #endif // HAVE_NEVERBLEED
 
+  MemchunkPool mcpool;
+
   ev_timer renew_ticket_key_timer;
   if (!upstreamconf.no_tls) {
     auto &ticketconf = get_config()->tls.ticket;
+    auto &memcachedconf = ticketconf.memcached;
 
     if (ticketconf.memcached.host) {
+      SSL_CTX *ssl_ctx = nullptr;
+
+      if (memcachedconf.tls) {
+        ssl_ctx = conn_handler.create_tls_ticket_key_memcached_ssl_ctx();
+      }
+
       conn_handler.set_tls_ticket_key_memcached_dispatcher(
-          make_unique<MemcachedDispatcher>(&ticketconf.memcached.addr, loop));
+          make_unique<MemcachedDispatcher>(
+              &ticketconf.memcached.addr, loop, ssl_ctx,
+              StringRef(memcachedconf.host.get()), &mcpool));
 
       ev_timer_init(&renew_ticket_key_timer, memcached_get_ticket_key_cb, 0.,
                     0.);

src/template.h

@@ -402,6 +402,13 @@ public:
   static StringRef from_lit(const CharT(&s)[N]) {
     return StringRef(s, N - 1);
   }
+  static StringRef from_maybe_nullptr(const char *s) {
+    if (s == nullptr) {
+      return StringRef();
+    }
+
+    return StringRef(s);
+  }
 
   const_iterator begin() const { return base; };
   const_iterator cbegin() const { return base; };