diff --git a/gennghttpxfun.py b/gennghttpxfun.py index f75add0b..cda5bb5c 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -92,6 +92,7 @@ OPTIONS = [ "tls-ticket-key-cipher", "host-rewrite", "tls-session-cache-memcached", + "tls-session-cache-memcached-tls", "tls-ticket-key-memcached", "tls-ticket-key-memcached-interval", "tls-ticket-key-memcached-max-retry", @@ -114,7 +115,14 @@ OPTIONS = [ "max-header-fields", "no-http2-cipher-black-list", "backend-http1-tls", - "backend-tls-session-cache-per-worker" + "backend-tls-session-cache-per-worker", + "tls-session-cache-memcached-cert-file", + "tls-session-cache-memcached-private-key-file", + "tls-session-cache-memcached-address-family", + "tls-ticket-key-memcached-tls", + "tls-ticket-key-memcached-cert-file", + "tls-ticket-key-memcached-private-key-file", + "tls-ticket-key-memcached-address-family", ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index e93c5d67..4561f833 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1051,6 +1051,13 @@ void fill_default_config() { memcachedconf.max_retry = 3; memcachedconf.max_fail = 2; memcachedconf.interval = 10_min; + memcachedconf.family = AF_UNSPEC; + } + + auto &session_cacheconf = tlsconf.session_cache; + { + auto &memcachedconf = session_cacheconf.memcached; + memcachedconf.family = AF_UNSPEC; } ticketconf.cipher = EVP_aes_128_cbc(); @@ -1520,16 +1527,23 @@ SSL/TLS: ticket key sharing between nghttpx instances is not required. --tls-ticket-key-memcached=, - Specify address of memcached server to store session - cache. This enables shared TLS ticket key between - multiple nghttpx instances. nghttpx does not set TLS - ticket key to memcached. The external ticket key - generator is required. nghttpx just gets TLS ticket - keys from memcached, and use them, possibly replacing - current set of keys. It is up to extern TLS ticket key - generator to rotate keys frequently. See "TLS SESSION - TICKET RESUMPTION" section in manual page to know the - data format in memcached entry. + Specify address of memcached server to get TLS ticket + keys for session resumption. This enables shared TLS + ticket key between multiple nghttpx instances. nghttpx + does not set TLS ticket key to memcached. The external + ticket key generator is required. nghttpx just gets TLS + ticket keys from memcached, and use them, possibly + replacing current set of keys. It is up to extern TLS + ticket key generator to rotate keys frequently. See + "TLS SESSION TICKET RESUMPTION" section in manual page + to know the data format in memcached entry. + --tls-ticket-key-memcached-address-family=(auto|IPv4|IPv6) + Specify address family of memcached connections to get + TLS ticket keys. If "auto" is given, both IPv4 and IPv6 + are considered. If "IPv4" is given, only IPv4 address + is considered. If "IPv6" is given, only IPv6 address is + considered. + Default: auto --tls-ticket-key-memcached-interval= Set interval to get TLS ticket keys from memcached. Default: )" @@ -1550,6 +1564,15 @@ SSL/TLS: Specify cipher to encrypt TLS session ticket. Specify either aes-128-cbc or aes-256-cbc. By default, aes-128-cbc is used. + --tls-ticket-key-memcached-tls + Enable SSL/TLS on memcached connections to get TLS + ticket keys. + --tls-ticket-key-memcached-cert-file= + Path to client certificate for memcached connections to + get TLS ticket keys. + --tls-ticket-key-memcached-private-key-file= + Path to client private key for memcached connections to + get TLS ticket keys. --fetch-ocsp-response-file= Path to fetch-ocsp-response script file. It should be absolute path. @@ -1564,6 +1587,22 @@ SSL/TLS: Specify address of memcached server to store session cache. This enables shared session cache between multiple nghttpx instances. + --tls-session-cache-memcached-address-family=(auto|IPv4|IPv6) + Specify address family of memcached connections to store + session cache. If "auto" is given, both IPv4 and IPv6 + are considered. If "IPv4" is given, only IPv4 address + is considered. If "IPv6" is given, only IPv6 address is + considered. + Default: auto + --tls-session-cache-memcached-tls + Enable SSL/TLS on memcached connections to store session + cache. + --tls-session-cache-memcached-cert-file= + Path to client certificate for memcached connections to + store session cache. + --tls-session-cache-memcached-private-key-file= + Path to client private key for memcached connections to + store session cache. --tls-dyn-rec-warmup-threshold= Specify the threshold size for TLS dynamic record size behaviour. During a TLS session, after the threshold @@ -2181,7 +2220,7 @@ void process_options( auto &memcachedconf = tlsconf.session_cache.memcached; if (memcachedconf.host) { if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.get(), - memcachedconf.port, AF_UNSPEC) == -1) { + memcachedconf.port, memcachedconf.family) == -1) { exit(EXIT_FAILURE); } } @@ -2191,7 +2230,7 @@ void process_options( auto &memcachedconf = tlsconf.ticket.memcached; if (memcachedconf.host) { if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.get(), - memcachedconf.port, AF_UNSPEC) == -1) { + memcachedconf.port, memcachedconf.family) == -1) { exit(EXIT_FAILURE); } } @@ -2400,6 +2439,20 @@ int main(int argc, char **argv) { {SHRPX_OPT_BACKEND_HTTP1_TLS, no_argument, &flag, 106}, {SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER, required_argument, &flag, 107}, + {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_TLS, no_argument, &flag, 108}, + {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE, required_argument, + &flag, 109}, + {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE, + required_argument, &flag, 110}, + {SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_TLS, no_argument, &flag, 111}, + {SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_CERT_FILE, required_argument, &flag, + 112}, + {SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE, required_argument, + &flag, 113}, + {SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY, required_argument, + &flag, 114}, + {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY, + required_argument, &flag, 115}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2858,6 +2911,44 @@ int main(int argc, char **argv) { cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER, optarg); break; + case 108: + // --tls-session-cache-memcached-tls + cmdcfgs.emplace_back(SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_TLS, "yes"); + break; + case 109: + // --tls-session-cache-memcached-cert-file + cmdcfgs.emplace_back(SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE, + optarg); + break; + case 110: + // --tls-session-cache-memcached-private-key-file + cmdcfgs.emplace_back( + SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE, optarg); + break; + case 111: + // --tls-ticket-key-memcached-tls + cmdcfgs.emplace_back(SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_TLS, "yes"); + break; + case 112: + // --tls-ticket-key-memcached-cert-file + cmdcfgs.emplace_back(SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_CERT_FILE, + optarg); + break; + case 113: + // --tls-ticket-key-memcached-private-key-file + cmdcfgs.emplace_back( + SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE, optarg); + break; + case 114: + // --tls-ticket-key-memcached-address-family + cmdcfgs.emplace_back(SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY, + optarg); + break; + case 115: + // --tls-session-cache-memcached-address-family + cmdcfgs.emplace_back( + SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY, optarg); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 102319e5..b7409358 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -575,6 +575,26 @@ std::vector parse_log_format(const char *optarg) { return res; } +namespace { +int parse_address_family(int *dest, const char *opt, const char *optarg) { + if (util::strieq("auto", optarg)) { + *dest = AF_UNSPEC; + return 0; + } + if (util::strieq("IPv4", optarg)) { + *dest = AF_INET; + return 0; + } + if (util::strieq("IPv6", optarg)) { + *dest = AF_INET6; + return 0; + } + + LOG(ERROR) << opt << ": bad value: '" << optarg << "'"; + return -1; +} +} // namespace + namespace { int parse_duration(ev_tstamp *dest, const char *opt, const char *optarg) { auto t = util::parse_duration_with_unit(optarg); @@ -758,12 +778,20 @@ enum { SHRPX_OPTID_TLS_DYN_REC_WARMUP_THRESHOLD, SHRPX_OPTID_TLS_PROTO_LIST, SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED, + SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY, + SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE, + SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE, + SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_TLS, SHRPX_OPTID_TLS_TICKET_KEY_CIPHER, SHRPX_OPTID_TLS_TICKET_KEY_FILE, SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED, + SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY, + SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_CERT_FILE, SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_INTERVAL, SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_MAX_FAIL, SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_MAX_RETRY, + SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE, + SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_TLS, SHRPX_OPTID_USER, SHRPX_OPTID_VERIFY_CLIENT, SHRPX_OPTID_VERIFY_CLIENT_CACERT, @@ -1325,6 +1353,9 @@ int option_lookup_token(const char *name, size_t namelen) { if (util::strieq_l("http2-max-concurrent-stream", name, 27)) { return SHRPX_OPTID_HTTP2_MAX_CONCURRENT_STREAMS; } + if (util::strieq_l("tls-ticket-key-memcached-tl", name, 27)) { + return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_TLS; + } break; } break; @@ -1337,6 +1368,15 @@ int option_lookup_token(const char *name, size_t namelen) { break; } break; + case 31: + switch (name[30]) { + case 's': + if (util::strieq_l("tls-session-cache-memcached-tl", name, 30)) { + return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_TLS; + } + break; + } + break; case 33: switch (name[32]) { case 'l': @@ -1351,6 +1391,11 @@ int option_lookup_token(const char *name, size_t namelen) { break; case 34: switch (name[33]) { + case 'e': + if (util::strieq_l("tls-ticket-key-memcached-cert-fil", name, 33)) { + return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_CERT_FILE; + } + break; case 'r': if (util::strieq_l("frontend-http2-dump-request-heade", name, 33)) { return SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER; @@ -1396,6 +1441,11 @@ int option_lookup_token(const char *name, size_t namelen) { break; case 37: switch (name[36]) { + case 'e': + if (util::strieq_l("tls-session-cache-memcached-cert-fil", name, 36)) { + return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE; + } + break; case 's': if (util::strieq_l("frontend-http2-connection-window-bit", name, 36)) { return SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS; @@ -1412,6 +1462,45 @@ int option_lookup_token(const char *name, size_t namelen) { break; } break; + case 39: + switch (name[38]) { + case 'y': + if (util::strieq_l("tls-ticket-key-memcached-address-famil", name, 38)) { + return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY; + } + break; + } + break; + case 41: + switch (name[40]) { + case 'e': + if (util::strieq_l("tls-ticket-key-memcached-private-key-fil", name, + 40)) { + return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE; + } + break; + } + break; + case 42: + switch (name[41]) { + case 'y': + if (util::strieq_l("tls-session-cache-memcached-address-famil", name, + 41)) { + return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY; + } + break; + } + break; + case 44: + switch (name[43]) { + case 'e': + if (util::strieq_l("tls-session-cache-memcached-private-key-fil", name, + 43)) { + return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE; + } + break; + } + break; } return -1; } @@ -2229,6 +2318,36 @@ int parse_config(const char *opt, const char *optarg, case SHRPX_OPTID_BACKEND_TLS_SESSION_CACHE_PER_WORKER: return parse_uint(&mod_config()->tls.downstream_session_cache_per_worker, opt, optarg); + case SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_TLS: + mod_config()->tls.session_cache.memcached.tls = util::strieq(optarg, "yes"); + + return 0; + case SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE: + mod_config()->tls.session_cache.memcached.cert_file = optarg; + + return 0; + case SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE: + mod_config()->tls.session_cache.memcached.private_key_file = optarg; + + return 0; + case SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_TLS: + mod_config()->tls.ticket.memcached.tls = util::strieq(optarg, "yes"); + + return 0; + case SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_CERT_FILE: + mod_config()->tls.ticket.memcached.cert_file = optarg; + + return 0; + case SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE: + mod_config()->tls.ticket.memcached.private_key_file = optarg; + + return 0; + case SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY: + return parse_address_family(&mod_config()->tls.ticket.memcached.family, opt, + optarg); + case SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY: + return parse_address_family( + &mod_config()->tls.session_cache.memcached.family, opt, optarg); case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 2d511fda..fd2564b9 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -209,6 +209,22 @@ constexpr char SHRPX_OPT_NO_HTTP2_CIPHER_BLACK_LIST[] = constexpr char SHRPX_OPT_BACKEND_HTTP1_TLS[] = "backend-http1-tls"; constexpr char SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER[] = "backend-tls-session-cache-per-worker"; +constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_TLS[] = + "tls-session-cache-memcached-tls"; +constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE[] = + "tls-session-cache-memcached-cert-file"; +constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE[] = + "tls-session-cache-memcached-private-key-file"; +constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY[] = + "tls-session-cache-memcached-address-family"; +constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_TLS[] = + "tls-ticket-key-memcached-tls"; +constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_CERT_FILE[] = + "tls-ticket-key-memcached-cert-file"; +constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE[] = + "tls-ticket-key-memcached-private-key-file"; +constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY[] = + "tls-ticket-key-memcached-address-family"; constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -335,6 +351,9 @@ struct TLSConfig { Address addr; uint16_t port; std::unique_ptr host; + // Client private key and certificate for authentication + ImmutableString private_key_file; + ImmutableString cert_file; ev_tstamp interval; // Maximum number of retries when getting TLS ticket key from // mamcached, due to network error. @@ -342,6 +361,10 @@ struct TLSConfig { // Maximum number of consecutive error from memcached, when this // limit reached, TLS ticket is disabled. size_t max_fail; + // Address family of memcached connection. One of either + // AF_INET, AF_INET6 or AF_UNSPEC. + int family; + bool tls; } memcached; std::vector files; const EVP_CIPHER *cipher; @@ -355,6 +378,13 @@ struct TLSConfig { Address addr; uint16_t port; std::unique_ptr host; + // Client private key and certificate for authentication + ImmutableString private_key_file; + ImmutableString cert_file; + // Address family of memcached connection. One of either + // AF_INET, AF_INET6 or AF_UNSPEC. + int family; + bool tls; } memcached; } session_cache; diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index c9066b69..588e814b 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -193,8 +193,24 @@ int ConnectionHandler::create_single_worker() { all_ssl_ctx_.push_back(cl_ssl_ctx); } - single_worker_ = make_unique(loop_, sv_ssl_ctx, cl_ssl_ctx, cert_tree, - ticket_keys_); + auto &tlsconf = get_config()->tls; + auto &memcachedconf = get_config()->tls.session_cache.memcached; + + SSL_CTX *session_cache_ssl_ctx = nullptr; + if (memcachedconf.tls) { + session_cache_ssl_ctx = ssl::create_ssl_client_context( +#ifdef HAVE_NEVERBLEED + nb_.get(), +#endif // HAVE_NEVERBLEED + StringRef::from_maybe_nullptr(tlsconf.cacert.get()), + StringRef(memcachedconf.cert_file), + StringRef(memcachedconf.private_key_file), StringRef(), nullptr); + all_ssl_ctx_.push_back(session_cache_ssl_ctx); + } + + single_worker_ = + make_unique(loop_, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, + cert_tree, ticket_keys_); #ifdef HAVE_MRUBY if (single_worker_->create_mruby_context() != 0) { return -1; @@ -225,11 +241,26 @@ int ConnectionHandler::create_worker_thread(size_t num) { all_ssl_ctx_.push_back(cl_ssl_ctx); } + auto &tlsconf = get_config()->tls; + auto &memcachedconf = get_config()->tls.session_cache.memcached; + for (size_t i = 0; i < num; ++i) { auto loop = ev_loop_new(0); - auto worker = make_unique(loop, sv_ssl_ctx, cl_ssl_ctx, cert_tree, - ticket_keys_); + SSL_CTX *session_cache_ssl_ctx = nullptr; + if (memcachedconf.tls) { + session_cache_ssl_ctx = ssl::create_ssl_client_context( +#ifdef HAVE_NEVERBLEED + nb_.get(), +#endif // HAVE_NEVERBLEED + StringRef::from_maybe_nullptr(tlsconf.cacert.get()), + StringRef(memcachedconf.cert_file), + StringRef(memcachedconf.private_key_file), StringRef(), nullptr); + all_ssl_ctx_.push_back(session_cache_ssl_ctx); + } + auto worker = + make_unique(loop, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, + cert_tree, ticket_keys_); #ifdef HAVE_MRUBY if (worker->create_mruby_context() != 0) { return -1; @@ -728,6 +759,23 @@ void ConnectionHandler::schedule_next_tls_ticket_key_memcached_get( ev_timer_start(loop_, w); } +SSL_CTX *ConnectionHandler::create_tls_ticket_key_memcached_ssl_ctx() { + auto &tlsconf = get_config()->tls; + auto &memcachedconf = get_config()->tls.ticket.memcached; + + auto ssl_ctx = ssl::create_ssl_client_context( +#ifdef HAVE_NEVERBLEED + nb_.get(), +#endif // HAVE_NEVERBLEED + StringRef::from_maybe_nullptr(tlsconf.cacert.get()), + StringRef(memcachedconf.cert_file), + StringRef(memcachedconf.private_key_file), StringRef(), nullptr); + + all_ssl_ctx_.push_back(ssl_ctx); + + return ssl_ctx; +} + #ifdef HAVE_NEVERBLEED void ConnectionHandler::set_neverbleed(std::unique_ptr nb) { nb_ = std::move(nb); diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h index 81d872f5..8602b359 100644 --- a/src/shrpx_connection_handler.h +++ b/src/shrpx_connection_handler.h @@ -129,6 +129,7 @@ public: on_tls_ticket_key_get_success(const std::shared_ptr &ticket_keys, ev_timer *w); void schedule_next_tls_ticket_key_memcached_get(ev_timer *w); + SSL_CTX *create_tls_ticket_key_memcached_ssl_ctx(); #ifdef HAVE_NEVERBLEED void set_neverbleed(std::unique_ptr nb); diff --git a/src/shrpx_memcached_connection.cc b/src/shrpx_memcached_connection.cc index 8c4d1cc2..4df031d8 100644 --- a/src/shrpx_memcached_connection.cc +++ b/src/shrpx_memcached_connection.cc @@ -32,6 +32,7 @@ #include "shrpx_memcached_request.h" #include "shrpx_memcached_result.h" #include "shrpx_config.h" +#include "shrpx_ssl.h" #include "util.h" namespace shrpx { @@ -78,7 +79,7 @@ void connectcb(struct ev_loop *loop, ev_io *w, int revents) { auto conn = static_cast(w->data); auto mconn = static_cast(conn->data); - if (mconn->on_connect() != 0) { + if (mconn->connected() != 0) { mconn->disconnect(); return; } @@ -91,11 +92,17 @@ constexpr ev_tstamp write_timeout = 10.; constexpr ev_tstamp read_timeout = 10.; MemcachedConnection::MemcachedConnection(const Address *addr, - struct ev_loop *loop) - : conn_(loop, -1, nullptr, nullptr, write_timeout, read_timeout, {}, {}, + struct ev_loop *loop, SSL_CTX *ssl_ctx, + const StringRef &sni_name, + MemchunkPool *mcpool) + : conn_(loop, -1, nullptr, mcpool, write_timeout, read_timeout, {}, {}, connectcb, readcb, timeoutcb, this, 0, 0.), + do_read_(&MemcachedConnection::noop), + do_write_(&MemcachedConnection::noop), + sni_name_(sni_name.str()), parse_state_{}, addr_(addr), + ssl_ctx_(ssl_ctx), sendsum_(0), connected_(false) {} @@ -127,11 +134,21 @@ void MemcachedConnection::disconnect() { assert(recvbuf_.rleft() == 0); recvbuf_.reset(); + + do_read_ = do_write_ = &MemcachedConnection::noop; } int MemcachedConnection::initiate_connection() { assert(conn_.fd == -1); + if (ssl_ctx_ && !conn_.tls.ssl) { + auto ssl = ssl::create_ssl(ssl_ctx_); + if (!ssl) { + return -1; + } + conn_.set_ssl(ssl); + } + conn_.fd = util::create_nonblock_socket(addr_->su.storage.ss_family); if (conn_.fd == -1) { @@ -153,6 +170,14 @@ int MemcachedConnection::initiate_connection() { return -1; } + if (ssl_ctx_) { + if (!util::numeric_host(sni_name_.c_str())) { + SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name_.c_str()); + } + + conn_.prepare_client_handshake(); + } + if (LOG_ENABLED(INFO)) { MCLOG(INFO, this) << "Connecting to memcached server"; } @@ -168,7 +193,7 @@ int MemcachedConnection::initiate_connection() { return 0; } -int MemcachedConnection::on_connect() { +int MemcachedConnection::connected() { if (!util::check_socket_connected(conn_.fd)) { conn_.wlimit.stopw(); @@ -185,15 +210,59 @@ int MemcachedConnection::on_connect() { connected_ = true; - ev_set_cb(&conn_.wev, writecb); - conn_.rlimit.startw(); ev_timer_again(conn_.loop, &conn_.rt); + ev_set_cb(&conn_.wev, writecb); + + if (conn_.tls.ssl) { + do_read_ = &MemcachedConnection::tls_handshake; + do_write_ = &MemcachedConnection::tls_handshake; + + return 0; + } + + do_read_ = &MemcachedConnection::read_clear; + do_write_ = &MemcachedConnection::write_clear; + return 0; } -int MemcachedConnection::on_write() { +int MemcachedConnection::on_write() { return do_write_(*this); } +int MemcachedConnection::on_read() { return do_read_(*this); } + +int MemcachedConnection::tls_handshake() { + ERR_clear_error(); + + ev_timer_again(conn_.loop, &conn_.rt); + + auto rv = conn_.tls_handshake(); + if (rv == SHRPX_ERR_INPROGRESS) { + return 0; + } + + if (rv < 0) { + return rv; + } + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "SSL/TLS handshake completed"; + } + + auto &tlsconf = get_config()->tls; + + if (!tlsconf.insecure && + ssl::check_cert(conn_.tls.ssl, addr_, StringRef(sni_name_)) != 0) { + return -1; + } + + do_read_ = &MemcachedConnection::read_tls; + do_write_ = &MemcachedConnection::write_tls; + + return on_write(); +} + +int MemcachedConnection::write_tls() { if (!connected_) { return 0; } @@ -207,19 +276,30 @@ int MemcachedConnection::on_write() { return 0; } - int rv; + std::array iov; + std::array buf; for (; !sendq_.empty();) { - rv = send_request(); + auto iovcnt = fill_request_buffer(iov.data(), iov.size()); + auto p = std::begin(buf); + for (size_t i = 0; i < iovcnt; ++i) { + auto &v = iov[i]; + auto n = std::min(static_cast(std::end(buf) - p), v.iov_len); + p = std::copy_n(static_cast(v.iov_base), n, p); + if (p == std::end(buf)) { + break; + } + } - if (rv < 0) { + auto nwrite = conn_.write_tls(buf.data(), p - std::begin(buf)); + if (nwrite < 0) { return -1; } - - if (rv == 1) { - // blocked + if (nwrite == 0) { return 0; } + + drain_send_queue(nwrite); } conn_.wlimit.stopw(); @@ -228,7 +308,70 @@ int MemcachedConnection::on_write() { return 0; } -int MemcachedConnection::on_read() { +int MemcachedConnection::read_tls() { + if (!connected_) { + return 0; + } + + ev_timer_again(conn_.loop, &conn_.rt); + + for (;;) { + auto nread = conn_.read_tls(recvbuf_.last, recvbuf_.wleft()); + + if (nread == 0) { + return 0; + } + + if (nread < 0) { + return -1; + } + + recvbuf_.write(nread); + + if (parse_packet() != 0) { + return -1; + } + } + + return 0; +} + +int MemcachedConnection::write_clear() { + if (!connected_) { + return 0; + } + + ev_timer_again(conn_.loop, &conn_.rt); + + if (sendq_.empty()) { + conn_.wlimit.stopw(); + ev_timer_stop(conn_.loop, &conn_.wt); + + return 0; + } + + std::array iov; + + for (; !sendq_.empty();) { + auto iovcnt = fill_request_buffer(iov.data(), iov.size()); + auto nwrite = conn_.writev_clear(iov.data(), iovcnt); + if (nwrite < 0) { + return -1; + } + if (nwrite == 0) { + return 0; + } + + drain_send_queue(nwrite); + } + + conn_.wlimit.stopw(); + ev_timer_stop(conn_.loop, &conn_.wt); + + return 0; +} + +int MemcachedConnection::read_clear() { if (!connected_) { return 0; } @@ -415,9 +558,8 @@ int MemcachedConnection::parse_packet() { #define MAX_WR_IOVCNT DEFAULT_WR_IOVCNT #endif // !defined(IOV_MAX) || IOV_MAX >= DEFAULT_WR_IOVCNT -int MemcachedConnection::send_request() { - ssize_t nwrite; - +size_t MemcachedConnection::fill_request_buffer(struct iovec *iov, + size_t iovlen) { if (sendsum_ == 0) { for (auto &req : sendq_) { if (req->canceled) { @@ -438,32 +580,27 @@ int MemcachedConnection::send_request() { } } - std::array iov; - size_t iovlen = 0; + size_t iovcnt = 0; for (auto &buf : sendbufv_) { - if (iovlen + 2 > iov.size()) { + if (iovcnt + 2 > iovlen) { break; } auto req = buf.req; if (buf.headbuf.rleft()) { - iov[iovlen++] = {buf.headbuf.pos, buf.headbuf.rleft()}; + iov[iovcnt++] = {buf.headbuf.pos, buf.headbuf.rleft()}; } if (buf.send_value_left) { - iov[iovlen++] = {req->value.data() + req->value.size() - + iov[iovcnt++] = {req->value.data() + req->value.size() - buf.send_value_left, buf.send_value_left}; } } - nwrite = conn_.writev_clear(iov.data(), iovlen); - if (nwrite < 0) { - return -1; - } - if (nwrite == 0) { - return 1; - } + return iovcnt; +} +void MemcachedConnection::drain_send_queue(size_t nwrite) { sendsum_ -= nwrite; while (nwrite > 0) { @@ -488,8 +625,6 @@ int MemcachedConnection::send_request() { recvq_.push_back(std::move(sendq_.front())); sendq_.pop_front(); } - - return 0; } size_t MemcachedConnection::serialized_size(MemcachedRequest *req) { @@ -549,4 +684,6 @@ int MemcachedConnection::add_request(std::unique_ptr req) { // TODO should we start write timer too? void MemcachedConnection::signal_write() { conn_.wlimit.startw(); } +int MemcachedConnection::noop() { return 0; } + } // namespace shrpx diff --git a/src/shrpx_memcached_connection.h b/src/shrpx_memcached_connection.h index 43d27c46..c9552198 100644 --- a/src/shrpx_memcached_connection.h +++ b/src/shrpx_memcached_connection.h @@ -93,7 +93,9 @@ constexpr uint8_t MEMCACHED_RES_MAGIC = 0x81; // https://code.google.com/p/memcached/wiki/MemcacheBinaryProtocol class MemcachedConnection { public: - MemcachedConnection(const Address *addr, struct ev_loop *loop); + MemcachedConnection(const Address *addr, struct ev_loop *loop, + SSL_CTX *ssl_ctx, const StringRef &sni_name, + MemchunkPool *mcpool); ~MemcachedConnection(); void disconnect(); @@ -101,23 +103,38 @@ public: int add_request(std::unique_ptr req); int initiate_connection(); - int on_connect(); + int connected(); int on_write(); int on_read(); - int send_request(); + + int write_clear(); + int read_clear(); + + int tls_handshake(); + int write_tls(); + int read_tls(); + + size_t fill_request_buffer(struct iovec *iov, size_t iovlen); + void drain_send_queue(size_t nwrite); + void make_request(MemcachedSendbuf *sendbuf, MemcachedRequest *req); int parse_packet(); size_t serialized_size(MemcachedRequest *req); void signal_write(); + int noop(); + private: Connection conn_; std::deque> recvq_; std::deque> sendq_; std::deque sendbufv_; + std::function do_read_, do_write_; + std::string sni_name_; MemcachedParseState parse_state_; const Address *addr_; + SSL_CTX *ssl_ctx_; // Sum of the bytes to be transmitted in sendbufv_. size_t sendsum_; bool connected_; diff --git a/src/shrpx_memcached_dispatcher.cc b/src/shrpx_memcached_dispatcher.cc index 28c00f13..796fef8e 100644 --- a/src/shrpx_memcached_dispatcher.cc +++ b/src/shrpx_memcached_dispatcher.cc @@ -31,8 +31,12 @@ namespace shrpx { MemcachedDispatcher::MemcachedDispatcher(const Address *addr, - struct ev_loop *loop) - : loop_(loop), mconn_(make_unique(addr, loop_)) {} + struct ev_loop *loop, SSL_CTX *ssl_ctx, + const StringRef &sni_name, + MemchunkPool *mcpool) + : loop_(loop), + mconn_(make_unique(addr, loop_, ssl_ctx, sni_name, + mcpool)) {} MemcachedDispatcher::~MemcachedDispatcher() {} diff --git a/src/shrpx_memcached_dispatcher.h b/src/shrpx_memcached_dispatcher.h index b3ea4866..64021c68 100644 --- a/src/shrpx_memcached_dispatcher.h +++ b/src/shrpx_memcached_dispatcher.h @@ -31,6 +31,10 @@ #include +#include + +#include "memchunk.h" + namespace shrpx { struct MemcachedRequest; @@ -39,7 +43,9 @@ struct Address; class MemcachedDispatcher { public: - MemcachedDispatcher(const Address *addr, struct ev_loop *loop); + MemcachedDispatcher(const Address *addr, struct ev_loop *loop, + SSL_CTX *ssl_ctx, const StringRef &sni_name, + MemchunkPool *mcpool); ~MemcachedDispatcher(); int add_request(std::unique_ptr req); diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 66c703ad..5d8a2360 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -662,8 +662,8 @@ SSL_CTX *create_ssl_client_context( #ifdef HAVE_NEVERBLEED neverbleed_t *nb, #endif // HAVE_NEVERBLEED - const char *cacert, const char *cert_file, const char *private_key_file, - const StringRef &alpn, + const StringRef &cacert, const StringRef &cert_file, + const StringRef &private_key_file, const StringRef &alpn, int (*next_proto_select_cb)(SSL *s, unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, void *arg)) { @@ -702,8 +702,8 @@ SSL_CTX *create_ssl_client_context( << ERR_error_string(ERR_get_error(), nullptr); } - if (cacert) { - if (SSL_CTX_load_verify_locations(ssl_ctx, cacert, nullptr) != 1) { + if (!cacert.empty()) { + if (SSL_CTX_load_verify_locations(ssl_ctx, cacert.c_str(), nullptr) != 1) { LOG(FATAL) << "Could not load trusted ca certificates from " << cacert << ": " << ERR_error_string(ERR_get_error(), nullptr); @@ -711,8 +711,8 @@ SSL_CTX *create_ssl_client_context( } } - if (cert_file) { - if (SSL_CTX_use_certificate_chain_file(ssl_ctx, cert_file) != 1) { + if (!cert_file.empty()) { + if (SSL_CTX_use_certificate_chain_file(ssl_ctx, cert_file.c_str()) != 1) { LOG(FATAL) << "Could not load client certificate from " << cert_file << ": " << ERR_error_string(ERR_get_error(), nullptr); @@ -720,9 +720,9 @@ SSL_CTX *create_ssl_client_context( } } - if (private_key_file) { + if (!private_key_file.empty()) { #ifndef HAVE_NEVERBLEED - if (SSL_CTX_use_PrivateKey_file(ssl_ctx, private_key_file, + if (SSL_CTX_use_PrivateKey_file(ssl_ctx, private_key_file.c_str(), SSL_FILETYPE_PEM) != 1) { LOG(FATAL) << "Could not load client private key from " << private_key_file << ": " @@ -731,7 +731,7 @@ SSL_CTX *create_ssl_client_context( } #else // HAVE_NEVERBLEED std::array errbuf; - if (neverbleed_load_private_key_file(nb, ssl_ctx, private_key_file, + if (neverbleed_load_private_key_file(nb, ssl_ctx, private_key_file.c_str(), errbuf.data()) != 1) { LOG(FATAL) << "neverbleed_load_private_key_file: could not load client " "private key from " << private_key_file << ": " @@ -982,7 +982,7 @@ int verify_hostname(X509 *cert, const char *hostname, size_t hlen, } } // namespace -int check_cert(SSL *ssl, const DownstreamAddr *addr) { +int check_cert(SSL *ssl, const Address *addr, const StringRef &host) { auto cert = SSL_get_peer_certificate(ssl); if (!cert) { LOG(ERROR) << "No certificate found"; @@ -996,18 +996,21 @@ int check_cert(SSL *ssl, const DownstreamAddr *addr) { return -1; } - auto &backend_sni_name = get_config()->tls.backend_sni_name; - - auto hostname = !backend_sni_name.empty() ? StringRef(backend_sni_name) - : StringRef(addr->host); - if (verify_hostname(cert, hostname.c_str(), hostname.size(), &addr->addr) != - 0) { + if (verify_hostname(cert, host.c_str(), host.size(), addr) != 0) { LOG(ERROR) << "Certificate verification failed: hostname does not match"; return -1; } return 0; } +int check_cert(SSL *ssl, const DownstreamAddr *addr) { + auto &backend_sni_name = get_config()->tls.backend_sni_name; + + auto hostname = !backend_sni_name.empty() ? StringRef(backend_sni_name) + : StringRef(addr->host); + return check_cert(ssl, &addr->addr, hostname); +} + CertLookupTree::CertLookupTree() { root_.ssl_ctx = nullptr; root_.str = nullptr; @@ -1320,8 +1323,10 @@ SSL_CTX *setup_downstream_client_ssl_context( #ifdef HAVE_NEVERBLEED nb, #endif // HAVE_NEVERBLEED - tlsconf.cacert.get(), tlsconf.client.cert_file.get(), - tlsconf.client.private_key_file.get(), alpn, next_proto_select_cb); + StringRef::from_maybe_nullptr(tlsconf.cacert.get()), + StringRef::from_maybe_nullptr(tlsconf.client.cert_file.get()), + StringRef::from_maybe_nullptr(tlsconf.client.private_key_file.get()), + alpn, next_proto_select_cb); } CertLookupTree *create_cert_lookup_tree() { diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h index da7c1667..ca93da14 100644 --- a/src/shrpx_ssl.h +++ b/src/shrpx_ssl.h @@ -46,6 +46,7 @@ class Worker; class DownstreamConnectionPool; struct DownstreamAddr; struct UpstreamAddr; +struct Address; namespace ssl { @@ -74,8 +75,8 @@ SSL_CTX *create_ssl_client_context( #ifdef HAVE_NEVERBLEED neverbleed_t *nb, #endif // HAVE_NEVERBLEED - const char *cacert, const char *cert_file, const char *private_key_file, - const StringRef &alpn, + const StringRef &cacert, const StringRef &cert_file, + const StringRef &private_key_file, const StringRef &alpn, int (*next_proto_select_cb)(SSL *s, unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, void *arg)); @@ -83,9 +84,8 @@ SSL_CTX *create_ssl_client_context( ClientHandler *accept_connection(Worker *worker, int fd, sockaddr *addr, int addrlen, const UpstreamAddr *faddr); -// Check peer's certificate against first downstream address in -// Config::downstream_addrs. We only consider first downstream since -// we use this function for HTTP/2 downstream link only. +// Check peer's certificate against given |address| and |host|. +int check_cert(SSL *ssl, const Address *addr, const StringRef &host); int check_cert(SSL *ssl, const DownstreamAddr *addr); // Retrieves DNS and IP address in subjectAltNames and commonName from diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 5e46e398..e663eace 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -68,6 +68,7 @@ std::random_device rd; } // namespace Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, + SSL_CTX *tls_session_cache_memcached_ssl_ctx, ssl::CertLookupTree *cert_tree, const std::shared_ptr &ticket_keys) : randgen_(rd()), @@ -92,7 +93,9 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, if (session_cacheconf.memcached.host) { session_cache_memcached_dispatcher_ = make_unique( - &session_cacheconf.memcached.addr, loop); + &session_cacheconf.memcached.addr, loop, + tls_session_cache_memcached_ssl_ctx, + session_cacheconf.memcached.host.get(), &mcpool_); } auto &downstreamconf = get_config()->conn.downstream; diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index c578d9fb..8d352429 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -112,6 +112,7 @@ struct SessionCacheEntry { class Worker { public: Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, + SSL_CTX *tls_session_cache_memcached_ssl_ctx, ssl::CertLookupTree *cert_tree, const std::shared_ptr &ticket_keys); ~Worker(); diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc index 0e05e19f..26863d82 100644 --- a/src/shrpx_worker_process.cc +++ b/src/shrpx_worker_process.cc @@ -420,13 +420,24 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { #endif // HAVE_NEVERBLEED + MemchunkPool mcpool; + ev_timer renew_ticket_key_timer; if (!upstreamconf.no_tls) { auto &ticketconf = get_config()->tls.ticket; + auto &memcachedconf = ticketconf.memcached; if (ticketconf.memcached.host) { + SSL_CTX *ssl_ctx = nullptr; + + if (memcachedconf.tls) { + ssl_ctx = conn_handler.create_tls_ticket_key_memcached_ssl_ctx(); + } + conn_handler.set_tls_ticket_key_memcached_dispatcher( - make_unique(&ticketconf.memcached.addr, loop)); + make_unique( + &ticketconf.memcached.addr, loop, ssl_ctx, + StringRef(memcachedconf.host.get()), &mcpool)); ev_timer_init(&renew_ticket_key_timer, memcached_get_ticket_key_cb, 0., 0.); diff --git a/src/template.h b/src/template.h index 0d4c4ba2..1a48d787 100644 --- a/src/template.h +++ b/src/template.h @@ -402,6 +402,13 @@ public: static StringRef from_lit(const CharT(&s)[N]) { return StringRef(s, N - 1); } + static StringRef from_maybe_nullptr(const char *s) { + if (s == nullptr) { + return StringRef(); + } + + return StringRef(s); + } const_iterator begin() const { return base; }; const_iterator cbegin() const { return base; };