From fcddb5c06ce3a2d89b1b09c3ba06022f788818f2 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 8 Jan 2015 21:15:45 +0900 Subject: [PATCH] nghttpx: Distribute session ticket keys to workers without mutex --- src/shrpx.cc | 31 +++++++++++++++++-------------- src/shrpx_config.h | 4 ---- src/shrpx_listen_handler.cc | 18 +++++++++++++++--- src/shrpx_listen_handler.h | 2 ++ src/shrpx_ssl.cc | 13 ++----------- src/shrpx_worker.cc | 34 +++++++++++++++++++--------------- src/shrpx_worker.h | 17 ++++++++--------- src/shrpx_worker_config.h | 3 +++ 8 files changed, 66 insertions(+), 56 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index e03d94b4..ff147c1b 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -438,10 +438,8 @@ void refresh_cb(struct ev_loop *loop, ev_timer *w, int revents) { namespace { void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) { -#ifndef NOTHREADS - std::lock_guard g(mod_config()->ticket_keys_lock); -#endif // !NOTHREADS - auto old_ticket_keys = get_config()->ticket_keys; + auto listener_handler = static_cast(w->data); + const auto &old_ticket_keys = worker_config->ticket_keys; auto ticket_keys = std::make_shared(); if (LOG_ENABLED(INFO)) { @@ -475,7 +473,9 @@ void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) { } } - mod_config()->ticket_keys = ticket_keys; + worker_config->ticket_keys = ticket_keys; + + listener_handler->worker_renew_ticket_keys(ticket_keys); } } // namespace @@ -529,6 +529,17 @@ int event_loop() { // After that, we drop the root privileges if needed. drop_privileges(); + ev_timer renew_ticket_key_timer; + if (sv_ssl_ctx && get_config()->auto_tls_ticket_key) { + // Renew ticket key every 12hrs + ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 12 * 3600.); + renew_ticket_key_timer.data = listener_handler.get(); + ev_timer_again(loop, &renew_ticket_key_timer); + + // Generate first session ticket key before running workers. + renew_ticket_key_cb(loop, &renew_ticket_key_timer, 0); + } + #ifndef NOTHREADS int rv; sigset_t signals; @@ -578,14 +589,6 @@ int event_loop() { refresh_timer.data = listener_handler.get(); ev_timer_again(loop, &refresh_timer); - ev_timer renew_ticket_key_timer; - if (sv_ssl_ctx && get_config()->auto_tls_ticket_key) { - // Renew ticket key every 12hrs - ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 12 * 3600.); - ev_timer_again(loop, &renew_ticket_key_timer); - renew_ticket_key_cb(loop, &renew_ticket_key_timer, 0); - } - if (LOG_ENABLED(INFO)) { LOG(INFO) << "Entering event loop"; } @@ -1755,7 +1758,7 @@ int main(int argc, char **argv) { if (!ticket_keys) { LOG(WARN) << "Use internal session ticket key generator"; } else { - mod_config()->ticket_keys = std::move(ticket_keys); + worker_config->ticket_keys = std::move(ticket_keys); mod_config()->auto_tls_ticket_key = false; } } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index f18cb93b..d739398d 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -35,8 +35,6 @@ #include #include #include -#include -#include #include @@ -184,8 +182,6 @@ struct Config { std::vector accesslog_format; std::vector downstream_addrs; std::vector tls_ticket_key_files; - std::mutex ticket_keys_lock; - std::shared_ptr ticket_keys; // binary form of http proxy host and port sockaddr_union downstream_http_proxy_addr; ev_tstamp http2_upstream_read_timeout; diff --git a/src/shrpx_listen_handler.cc b/src/shrpx_listen_handler.cc index 55cfa9bc..76a2f720 100644 --- a/src/shrpx_listen_handler.cc +++ b/src/shrpx_listen_handler.cc @@ -85,14 +85,26 @@ void ListenHandler::worker_reopen_log_files() { } } +void ListenHandler::worker_renew_ticket_keys( + const std::shared_ptr &ticket_keys) { + WorkerEvent wev; + + memset(&wev, 0, sizeof(wev)); + wev.type = RENEW_TICKET_KEYS; + wev.ticket_keys = ticket_keys; + + for (auto &worker : workers_) { + worker->send(wev); + } +} + void ListenHandler::create_worker_thread(size_t num) { #ifndef NOTHREADS assert(workers_.size() == 0); for (size_t i = 0; i < num; ++i) { - auto worker = util::make_unique(sv_ssl_ctx_, cl_ssl_ctx_); - worker->run(); - workers_.push_back(std::move(worker)); + workers_.push_back(util::make_unique(sv_ssl_ctx_, cl_ssl_ctx_, + worker_config->ticket_keys)); if (LOG_ENABLED(INFO)) { LLOG(INFO, this) << "Created thread #" << workers_.size() - 1; diff --git a/src/shrpx_listen_handler.h b/src/shrpx_listen_handler.h index e6b5ff60..2da967f3 100644 --- a/src/shrpx_listen_handler.h +++ b/src/shrpx_listen_handler.h @@ -46,6 +46,7 @@ class ConnectBlocker; class AcceptHandler; class Worker; struct WorkerStat; +struct TicketKeys; // TODO should be renamed as ConnectionHandler class ListenHandler { @@ -55,6 +56,7 @@ public: int handle_connection(int fd, sockaddr *addr, int addrlen); void create_worker_thread(size_t num); void worker_reopen_log_files(); + void worker_renew_ticket_keys(const std::shared_ptr &ticket_keys); struct ev_loop *get_loop() const; void create_http2_session(); void create_http1_connect_blocker(); diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 89bce944..1c62f707 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -47,6 +47,7 @@ #include "shrpx_client_handler.h" #include "shrpx_config.h" #include "shrpx_worker.h" +#include "shrpx_worker_config.h" #include "shrpx_downstream_connection_pool.h" #include "util.h" #include "ssl.h" @@ -147,17 +148,7 @@ namespace { int ticket_key_cb(SSL *ssl, unsigned char *key_name, unsigned char *iv, EVP_CIPHER_CTX *ctx, HMAC_CTX *hctx, int enc) { auto handler = static_cast(SSL_get_app_data(ssl)); -#ifndef NOTHREADS - std::shared_ptr ticket_keys; - if (get_config()->auto_tls_ticket_key) { - std::lock_guard g(mod_config()->ticket_keys_lock); - ticket_keys = get_config()->ticket_keys; - } else { - ticket_keys = get_config()->ticket_keys; - } -#else // NOTHREADS - auto ticket_keys = get_config()->ticket_keys; -#endif // NOTHREADS + auto ticket_keys = worker_config->ticket_keys; if (!ticket_keys) { // No ticket keys available. diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index cff05487..ec8e51c0 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -47,7 +47,8 @@ void eventcb(struct ev_loop *loop, ev_async *w, int revents) { } } // namespace -Worker::Worker(SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx) +Worker::Worker(SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, + const std::shared_ptr &ticket_keys) : loop_(ev_loop_new(0)), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), worker_stat_(util::make_unique()) { ev_async_init(&w_, eventcb); @@ -59,22 +60,15 @@ Worker::Worker(SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx) } else { http1_connect_blocker_ = util::make_unique(loop_); } + + fut_ = std::async(std::launch::async, [this, &ticket_keys] { + worker_config->ticket_keys = ticket_keys; + (void)reopen_log_files(); + ev_run(loop_); + }); } -Worker::~Worker() { - ev_async_stop(loop_, &w_); -} - -void Worker::run() { -#ifndef NOTHREADS - fut_ = std::async(std::launch::async, [this] { this->run_loop(); }); -#endif // !NOTHREADS -} - -void Worker::run_loop() { - (void)reopen_log_files(); - ev_run(loop_); -} +Worker::~Worker() { ev_async_stop(loop_, &w_); } void Worker::wait() { #ifndef NOTHREADS @@ -99,6 +93,16 @@ void Worker::process_events() { q.swap(q_); } for (auto &wev : q) { + if (wev.type == RENEW_TICKET_KEYS) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Renew ticket keys: worker_info(" << worker_config << ")"; + } + + worker_config->ticket_keys = wev.ticket_keys; + + continue; + } + if (wev.type == REOPEN_LOG) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Reopening log files: worker_info(" << worker_config diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 4d441349..71ae3bd3 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -61,25 +61,24 @@ enum WorkerEventType { NEW_CONNECTION = 0x01, REOPEN_LOG = 0x02, GRACEFUL_SHUTDOWN = 0x03, + RENEW_TICKET_KEYS = 0x04, }; struct WorkerEvent { WorkerEventType type; - union { - struct { - sockaddr_union client_addr; - size_t client_addrlen; - int client_fd; - }; + struct { + sockaddr_union client_addr; + size_t client_addrlen; + int client_fd; }; + std::shared_ptr ticket_keys; }; class Worker { public: - Worker(SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx); + Worker(SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, + const std::shared_ptr &ticket_keys); ~Worker(); - void run(); - void run_loop(); void wait(); void process_events(); void send(const WorkerEvent &event); diff --git a/src/shrpx_worker_config.h b/src/shrpx_worker_config.h index 761fb45a..8730f062 100644 --- a/src/shrpx_worker_config.h +++ b/src/shrpx_worker_config.h @@ -29,7 +29,10 @@ namespace shrpx { +struct TicketKeys; + struct WorkerConfig { + std::shared_ptr ticket_keys; int accesslog_fd; int errorlog_fd; // true if errorlog_fd is referring to a terminal.