diff --git a/gennghttpxfun.py b/gennghttpxfun.py index f322e5c1..bba4e8a0 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -94,6 +94,7 @@ OPTIONS = [ "tls-ticket-cipher", "host-rewrite", "tls-session-cache-memcached", + "tls-ticket-key-memcached", "conf", ] diff --git a/src/shrpx.cc b/src/shrpx.cc index d828084a..ed6a46dc 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -83,6 +83,8 @@ #include "shrpx_accept_handler.h" #include "shrpx_http2_upstream.h" #include "shrpx_http2_session.h" +#include "shrpx_memcached_dispatcher.h" +#include "shrpx_memcached_request.h" #include "util.h" #include "app_helper.h" #include "ssl.h" @@ -689,6 +691,116 @@ void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) { } } // namespace +namespace { +void memcached_get_ticket_key_cb(struct ev_loop *loop, ev_timer *w, + int revents) { + auto conn_handler = static_cast(w->data); + auto dispatcher = conn_handler->get_tls_ticket_key_memcached_dispatcher(); + + auto req = make_unique(); + req->key = "nghttpx:tls-ticket-key"; + req->op = MEMCACHED_OP_GET; + req->cb = [conn_handler, dispatcher, w](MemcachedRequest *req, + MemcachedResult res) { + switch (res.status_code) { + case MEMCACHED_ERR_NO_ERROR: + break; + case MEMCACHED_ERR_EXT_NETWORK_ERROR: + conn_handler->on_tls_ticket_key_network_error(w); + return; + default: + conn_handler->on_tls_ticket_key_not_found(w); + return; + } + + // |version (4bytes)|len (2bytes)|key (variable length)|... + // (len, key) pairs are repeated as necessary. + + auto &value = res.value; + if (value.size() < 4) { + LOG(WARN) << "Memcached: tls ticket key value is too small: got " + << value.size(); + conn_handler->on_tls_ticket_key_not_found(w); + return; + } + auto p = value.data(); + auto version = util::get_uint32(p); + // Currently supported version is 1. + if (version != 1) { + LOG(WARN) << "Memcached: tls ticket key version: want 1, got " << version; + conn_handler->on_tls_ticket_key_not_found(w); + return; + } + + auto end = p + value.size(); + p += 4; + + size_t expectedlen; + size_t enc_keylen; + size_t hmac_keylen; + if (get_config()->tls_ticket_cipher == EVP_aes_128_cbc()) { + expectedlen = 48; + enc_keylen = 16; + hmac_keylen = 16; + } else if (get_config()->tls_ticket_cipher == EVP_aes_256_cbc()) { + expectedlen = 80; + enc_keylen = 32; + hmac_keylen = 32; + } else { + return; + } + + auto ticket_keys = std::make_shared(); + + for (; p != end;) { + if (end - p < 2) { + LOG(WARN) << "Memcached: tls ticket key data is too small"; + conn_handler->on_tls_ticket_key_not_found(w); + return; + } + auto len = util::get_uint16(p); + p += 2; + if (len != expectedlen) { + LOG(WARN) << "Memcached: wrong tls ticket key size: want " + << expectedlen << ", got " << len; + conn_handler->on_tls_ticket_key_not_found(w); + return; + } + if (p + len > end) { + LOG(WARN) << "Memcached: too short tls ticket key payload: want " << len + << ", got " << (end - p); + conn_handler->on_tls_ticket_key_not_found(w); + return; + } + auto key = TicketKey(); + key.cipher = get_config()->tls_ticket_cipher; + key.hmac = EVP_sha256(); + key.hmac_keylen = EVP_MD_size(key.hmac); + + std::copy_n(p, key.data.name.size(), key.data.name.data()); + p += key.data.name.size(); + + std::copy_n(p, enc_keylen, key.data.enc_key.data()); + p += enc_keylen; + + std::copy_n(p, hmac_keylen, key.data.hmac_key.data()); + p += hmac_keylen; + + ticket_keys->keys.push_back(std::move(key)); + } + + conn_handler->on_tls_ticket_key_get_success(ticket_keys, w); + }; + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Memcached: tls ticket key get request sent"; + } + + dispatcher->add_request(std::move(req)); +} + +} // namespace + namespace { int call_daemon() { #ifdef __sgi @@ -749,34 +861,47 @@ int event_loop() { ev_timer renew_ticket_key_timer; if (!get_config()->upstream_no_tls) { - bool auto_tls_ticket_key = true; - if (!get_config()->tls_ticket_key_files.empty()) { - if (!get_config()->tls_ticket_cipher_given) { - LOG(WARN) << "It is strongly recommended to specify " - "--tls-ticket-cipher=aes-128-cbc (or " - "tls-ticket-cipher=aes-128-cbc in configuration file) " - "when --tls-ticket-key-file is used for the smooth " - "transition when the default value of --tls-ticket-cipher " - "becomes aes-256-cbc"; - } - auto ticket_keys = read_tls_ticket_key_file( - get_config()->tls_ticket_key_files, get_config()->tls_ticket_cipher, - EVP_sha256()); - if (!ticket_keys) { - LOG(WARN) << "Use internal session ticket key generator"; - } else { - conn_handler->set_ticket_keys(std::move(ticket_keys)); - auto_tls_ticket_key = false; - } - } - if (auto_tls_ticket_key) { - // Generate new ticket key every 1hr. - ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 1_h); - renew_ticket_key_timer.data = conn_handler.get(); - ev_timer_again(loop, &renew_ticket_key_timer); + if (get_config()->tls_ticket_key_memcached_host) { + conn_handler->set_tls_ticket_key_memcached_dispatcher( + make_unique( + &get_config()->tls_ticket_key_memcached_addr, loop)); - // Generate first session ticket key before running workers. - renew_ticket_key_cb(loop, &renew_ticket_key_timer, 0); + ev_timer_init(&renew_ticket_key_timer, memcached_get_ticket_key_cb, 0., + 0.); + renew_ticket_key_timer.data = conn_handler.get(); + // Get first ticket keys. + memcached_get_ticket_key_cb(loop, &renew_ticket_key_timer, 0); + } else { + bool auto_tls_ticket_key = true; + if (!get_config()->tls_ticket_key_files.empty()) { + if (!get_config()->tls_ticket_cipher_given) { + LOG(WARN) + << "It is strongly recommended to specify " + "--tls-ticket-cipher=aes-128-cbc (or " + "tls-ticket-cipher=aes-128-cbc in configuration file) " + "when --tls-ticket-key-file is used for the smooth " + "transition when the default value of --tls-ticket-cipher " + "becomes aes-256-cbc"; + } + auto ticket_keys = read_tls_ticket_key_file( + get_config()->tls_ticket_key_files, get_config()->tls_ticket_cipher, + EVP_sha256()); + if (!ticket_keys) { + LOG(WARN) << "Use internal session ticket key generator"; + } else { + conn_handler->set_ticket_keys(std::move(ticket_keys)); + auto_tls_ticket_key = false; + } + } + if (auto_tls_ticket_key) { + // Generate new ticket key every 1hr. + ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 1_h); + renew_ticket_key_timer.data = conn_handler.get(); + ev_timer_again(loop, &renew_ticket_key_timer); + + // Generate first session ticket key before running workers. + renew_ticket_key_cb(loop, &renew_ticket_key_timer, 0); + } } } @@ -1020,6 +1145,9 @@ void fill_default_config() { mod_config()->tls_ticket_cipher = EVP_aes_128_cbc(); mod_config()->tls_ticket_cipher_given = false; mod_config()->tls_session_timeout = std::chrono::hours(12); + mod_config()->tls_ticket_key_memcached_max_retry = 3; + mod_config()->tls_ticket_key_memcached_max_fail = 2; + mod_config()->tls_ticket_key_memcached_interval = 10_min; } } // namespace @@ -1368,6 +1496,15 @@ SSL/TLS: Specify address of memcached server to store session cache. This enables shared session cache between multiple nghttpx instances. + --tls-ticket-key-memcached=, + Specify address of memcached server to store session + cache. This enables shared TLS ticket key between + multiple nghttpx instances. nghttpx does not set TLS + ticket key to memcached. The external ticket key + generator is required. nghttpx just gets TLS ticket + keys from memcached, and use them, possibly replacing + current set of keys. It is up to extern TLS ticket key + generator to rotate keys frequently. HTTP/2 and SPDY: -c, --http2-max-concurrent-streams= @@ -1732,6 +1869,7 @@ int main(int argc, char **argv) { {SHRPX_OPT_TLS_TICKET_CIPHER, required_argument, &flag, 84}, {SHRPX_OPT_HOST_REWRITE, no_argument, &flag, 85}, {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED, required_argument, &flag, 86}, + {SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED, required_argument, &flag, 87}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2110,6 +2248,10 @@ int main(int argc, char **argv) { // --tls-session-cache-memcached cmdcfgs.emplace_back(SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED, optarg); break; + case 87: + // --tls-ticket-key-memcached + cmdcfgs.emplace_back(SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED, optarg); + break; default: break; } @@ -2396,6 +2538,15 @@ int main(int argc, char **argv) { } } + if (get_config()->tls_ticket_key_memcached_host) { + if (resolve_hostname(&mod_config()->tls_ticket_key_memcached_addr, + get_config()->tls_ticket_key_memcached_host.get(), + get_config()->tls_ticket_key_memcached_port, + AF_UNSPEC) == -1) { + exit(EXIT_FAILURE); + } + } + if (get_config()->rlimit_nofile) { struct rlimit lim = {static_cast(get_config()->rlimit_nofile), static_cast(get_config()->rlimit_nofile)}; diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 51daf44a..55da8c12 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -706,6 +706,7 @@ enum { SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED, SHRPX_OPTID_TLS_TICKET_CIPHER, SHRPX_OPTID_TLS_TICKET_KEY_FILE, + SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED, SHRPX_OPTID_USER, SHRPX_OPTID_VERIFY_CLIENT, SHRPX_OPTID_VERIFY_CLIENT_CACERT, @@ -1138,6 +1139,11 @@ int option_lookup_token(const char *name, size_t namelen) { break; case 24: switch (name[23]) { + case 'd': + if (util::strieq_l("tls-ticket-key-memcache", name, 23)) { + return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED; + } + break; case 'e': if (util::strieq_l("fetch-ocsp-response-fil", name, 23)) { return SHRPX_OPTID_FETCH_OCSP_RESPONSE_FILE; @@ -1881,6 +1887,17 @@ int parse_config(const char *opt, const char *optarg, return 0; } + case SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED: { + if (split_host_port(host, sizeof(host), &port, optarg, strlen(optarg)) == + -1) { + return -1; + } + + mod_config()->tls_ticket_key_memcached_host = strcopy(host); + mod_config()->tls_ticket_key_memcached_port = port; + + return 0; + } case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 742ceada..48f3be71 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -175,6 +175,8 @@ constexpr char SHRPX_OPT_TLS_TICKET_CIPHER[] = "tls-ticket-cipher"; constexpr char SHRPX_OPT_HOST_REWRITE[] = "host-rewrite"; constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED[] = "tls-session-cache-memcached"; +constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED[] = + "tls-ticket-key-memcached"; union sockaddr_union { sockaddr_storage storage; @@ -260,6 +262,7 @@ struct Config { // binary form of http proxy host and port Address downstream_http_proxy_addr; Address session_cache_memcached_addr; + Address tls_ticket_key_memcached_addr; std::chrono::seconds tls_session_timeout; ev_tstamp http2_upstream_read_timeout; ev_tstamp upstream_read_timeout; @@ -271,6 +274,7 @@ struct Config { ev_tstamp downstream_idle_read_timeout; ev_tstamp listener_disable_timeout; ev_tstamp ocsp_update_interval; + ev_tstamp tls_ticket_key_memcached_interval; // address of frontend connection. This could be a path to UNIX // domain socket. In this case, |host_unix| must be true. std::unique_ptr host; @@ -303,6 +307,7 @@ struct Config { std::unique_ptr fetch_ocsp_response_file; std::unique_ptr user; std::unique_ptr session_cache_memcached_host; + std::unique_ptr tls_ticket_key_memcached_host; FILE *http2_upstream_dump_request_header; FILE *http2_upstream_dump_response_header; nghttp2_session_callbacks *http2_upstream_callbacks; @@ -339,6 +344,12 @@ struct Config { size_t max_header_fields; // The index of catch-all group in downstream_addr_groups. size_t downstream_addr_group_catch_all; + // Maximum number of retries when getting TLS ticket key from + // mamcached, due to network error. + size_t tls_ticket_key_memcached_max_retry; + // Maximum number of consecutive error from memcached, when this + // limit reached, TLS ticket is disabled. + size_t tls_ticket_key_memcached_max_fail; // Bit mask to disable SSL/TLS protocol versions. This will be // passed to SSL_CTX_set_options(). long int tls_proto_mask; @@ -356,6 +367,7 @@ struct Config { // port in http proxy URI uint16_t downstream_http_proxy_port; uint16_t session_cache_memcached_port; + uint16_t tls_ticket_key_memcached_port; bool verbose; bool daemon; bool verify_client; diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 64260e9d..1d79fe81 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -32,6 +32,7 @@ #include #include +#include #include "shrpx_client_handler.h" #include "shrpx_ssl.h" @@ -41,6 +42,7 @@ #include "shrpx_connect_blocker.h" #include "shrpx_downstream_connection.h" #include "shrpx_accept_handler.h" +#include "shrpx_memcached_dispatcher.h" #include "util.h" #include "template.h" @@ -94,7 +96,9 @@ void ocsp_chld_cb(struct ev_loop *loop, ev_child *w, int revent) { } // namespace ConnectionHandler::ConnectionHandler(struct ev_loop *loop) - : single_worker_(nullptr), loop_(loop), worker_round_robin_cnt_(0), + : single_worker_(nullptr), loop_(loop), + tls_ticket_key_memcached_get_retry_count_(0), + tls_ticket_key_memcached_fail_count_(0), worker_round_robin_cnt_(0), graceful_shutdown_(false) { ev_timer_init(&disable_acceptor_timer_, acceptor_disable_cb, 0., 0.); disable_acceptor_timer_.data = this; @@ -553,4 +557,95 @@ void ConnectionHandler::proceed_next_cert_ocsp() { } } +void ConnectionHandler::set_tls_ticket_key_memcached_dispatcher( + std::unique_ptr dispatcher) { + tls_ticket_key_memcached_dispatcher_ = std::move(dispatcher); +} + +MemcachedDispatcher * +ConnectionHandler::get_tls_ticket_key_memcached_dispatcher() const { + return tls_ticket_key_memcached_dispatcher_.get(); +} + +namespace { +std::random_device rd; +} // namespace + +void ConnectionHandler::on_tls_ticket_key_network_error(ev_timer *w) { + if (++tls_ticket_key_memcached_get_retry_count_ >= + get_config()->tls_ticket_key_memcached_max_retry) { + LOG(WARN) << "Memcached: tls ticket get retry all failed " + << tls_ticket_key_memcached_get_retry_count_ << " times."; + + on_tls_ticket_key_not_found(w); + return; + } + + auto dist = std::uniform_int_distribution( + 1, std::min(60, 1 << tls_ticket_key_memcached_get_retry_count_)); + auto t = dist(rd); + + LOG(WARN) + << "Memcached: tls ticket get failed due to network error, retrying in " + << t << " seconds"; + + ev_timer_set(w, t, 0.); + ev_timer_start(loop_, w); +} + +void ConnectionHandler::on_tls_ticket_key_not_found(ev_timer *w) { + tls_ticket_key_memcached_get_retry_count_ = 0; + + if (++tls_ticket_key_memcached_fail_count_ >= + get_config()->tls_ticket_key_memcached_max_fail) { + LOG(WARN) << "Memcached: could not get tls ticket; disable tls ticket"; + + tls_ticket_key_memcached_fail_count_ = 0; + + set_ticket_keys(nullptr); + set_ticket_keys_to_worker(nullptr); + } + + LOG(WARN) << "Memcached: tls ticket get failed, schedule next"; + schedule_next_tls_ticket_key_memcached_get(w); +} + +void ConnectionHandler::on_tls_ticket_key_get_success( + const std::shared_ptr &ticket_keys, ev_timer *w) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Memcached: tls ticket get success"; + } + + tls_ticket_key_memcached_get_retry_count_ = 0; + tls_ticket_key_memcached_fail_count_ = 0; + + schedule_next_tls_ticket_key_memcached_get(w); + + if (!ticket_keys || ticket_keys->keys.empty()) { + LOG(WARN) << "Memcached: tls ticket keys are empty; tls ticket disabled"; + set_ticket_keys(nullptr); + set_ticket_keys_to_worker(nullptr); + return; + } + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "ticket keys get done"; + LOG(INFO) << 0 << " enc+dec: " + << util::format_hex(ticket_keys->keys[0].data.name); + for (size_t i = 1; i < ticket_keys->keys.size(); ++i) { + auto &key = ticket_keys->keys[i]; + LOG(INFO) << i << " dec: " << util::format_hex(key.data.name); + } + } + + set_ticket_keys(ticket_keys); + set_ticket_keys_to_worker(ticket_keys); +} + +void +ConnectionHandler::schedule_next_tls_ticket_key_memcached_get(ev_timer *w) { + ev_timer_set(w, get_config()->tls_ticket_key_memcached_interval, 0.); + ev_timer_start(loop_, w); +} + } // namespace shrpx diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h index c6488854..d46f059c 100644 --- a/src/shrpx_connection_handler.h +++ b/src/shrpx_connection_handler.h @@ -49,6 +49,7 @@ class AcceptHandler; class Worker; struct WorkerStat; struct TicketKeys; +class MemcachedDispatcher; struct OCSPUpdateContext { // ocsp response buffer @@ -111,6 +112,17 @@ public: // update. void proceed_next_cert_ocsp(); + void set_tls_ticket_key_memcached_dispatcher( + std::unique_ptr dispatcher); + + MemcachedDispatcher *get_tls_ticket_key_memcached_dispatcher() const; + void on_tls_ticket_key_network_error(ev_timer *w); + void on_tls_ticket_key_not_found(ev_timer *w); + void + on_tls_ticket_key_get_success(const std::shared_ptr &ticket_keys, + ev_timer *w); + void schedule_next_tls_ticket_key_memcached_get(ev_timer *w); + private: // Stores all SSL_CTX objects. std::vector all_ssl_ctx_; @@ -120,6 +132,7 @@ private: // Worker instance used when single threaded mode (-n1) is used. // Otherwise, nullptr and workers_ has instances of Worker instead. std::unique_ptr single_worker_; + std::unique_ptr tls_ticket_key_memcached_dispatcher_; // Current TLS session ticket keys. Note that TLS connection does // not refer to this field directly. They use TicketKeys object in // Worker object. @@ -131,6 +144,8 @@ private: std::unique_ptr acceptor6_; ev_timer disable_acceptor_timer_; ev_timer ocsp_timer_; + size_t tls_ticket_key_memcached_get_retry_count_; + size_t tls_ticket_key_memcached_fail_count_; unsigned int worker_round_robin_cnt_; bool graceful_shutdown_; }; diff --git a/src/shrpx_memcached_connection.cc b/src/shrpx_memcached_connection.cc index ace5928c..74497334 100644 --- a/src/shrpx_memcached_connection.cc +++ b/src/shrpx_memcached_connection.cc @@ -100,7 +100,7 @@ namespace { void clear_request(std::deque> &q) { for (auto &req : q) { if (req->cb) { - req->cb(req.get(), MemcachedResult(MEMCACHED_ERR_ERROR)); + req->cb(req.get(), MemcachedResult(MEMCACHED_ERR_EXT_NETWORK_ERROR)); } } q.clear(); diff --git a/src/shrpx_memcached_result.h b/src/shrpx_memcached_result.h index c2cd70d0..a5ac7591 100644 --- a/src/shrpx_memcached_result.h +++ b/src/shrpx_memcached_result.h @@ -32,8 +32,8 @@ namespace shrpx { enum MemcachedStatusCode { - MEMCACHED_ERR_OK, - MEMCACHED_ERR_ERROR = 0x1001, + MEMCACHED_ERR_NO_ERROR, + MEMCACHED_ERR_EXT_NETWORK_ERROR = 0x1001, }; struct MemcachedResult {