diff --git a/src/shrpx.cc b/src/shrpx.cc index 689c9ad8..6219683c 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -454,12 +454,12 @@ void graceful_shutdown_signal_cb(struct ev_loop *loop, ev_signal *w, namespace { void refresh_cb(struct ev_loop *loop, ev_timer *w, int revents) { auto conn_handler = static_cast(w->data); - auto worker_stat = conn_handler->get_worker_stat(); + auto worker = conn_handler->get_single_worker(); // In multi threaded mode (get_config()->num_worker > 1), we have to // wait for event notification to workers to finish. if (get_config()->num_worker == 1 && worker_config->graceful_shutdown && - (!worker_stat || worker_stat->num_connections == 0)) { + (!worker || worker->get_worker_stat()->num_connections == 0)) { ev_break(loop); } } @@ -468,7 +468,7 @@ void refresh_cb(struct ev_loop *loop, ev_timer *w, int revents) { namespace { void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) { auto conn_handler = static_cast(w->data); - const auto &old_ticket_keys = worker_config->ticket_keys; + const auto &old_ticket_keys = conn_handler->get_ticket_keys(); auto ticket_keys = std::make_shared(); if (LOG_ENABLED(INFO)) { @@ -502,8 +502,7 @@ void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) { } } - worker_config->ticket_keys = ticket_keys; - + conn_handler->set_ticket_keys(ticket_keys); conn_handler->worker_renew_ticket_keys(ticket_keys); } } // namespace @@ -539,22 +538,35 @@ int event_loop() { conn_handler->set_acceptor4(std::move(acceptor4)); conn_handler->set_acceptor6(std::move(acceptor6)); + ev_timer renew_ticket_key_timer; + if (!get_config()->upstream_no_tls) { + bool auto_tls_ticket_key = true; + if (!get_config()->tls_ticket_key_files.empty()) { + auto ticket_keys = + read_tls_ticket_key_file(get_config()->tls_ticket_key_files); + if (!ticket_keys) { + LOG(WARN) << "Use internal session ticket key generator"; + } else { + conn_handler->set_ticket_keys(std::move(ticket_keys)); + auto_tls_ticket_key = false; + } + } + if (auto_tls_ticket_key) { + // Renew ticket key every 12hrs + ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., + 12 * 3600.); + renew_ticket_key_timer.data = conn_handler.get(); + ev_timer_again(loop, &renew_ticket_key_timer); + + // Generate first session ticket key before running workers. + renew_ticket_key_cb(loop, &renew_ticket_key_timer, 0); + } + } + // ListenHandler loads private key, and we listen on a priveleged port. // After that, we drop the root privileges if needed. drop_privileges(); - ev_timer renew_ticket_key_timer; - if (!get_config()->client_mode && !get_config()->upstream_no_tls && - get_config()->auto_tls_ticket_key) { - // Renew ticket key every 12hrs - ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 12 * 3600.); - renew_ticket_key_timer.data = conn_handler.get(); - ev_timer_again(loop, &renew_ticket_key_timer); - - // Generate first session ticket key before running workers. - renew_ticket_key_cb(loop, &renew_ticket_key_timer, 0); - } - #ifndef NOTHREADS int rv; sigset_t signals; @@ -568,18 +580,10 @@ int event_loop() { } #endif // !NOTHREADS - if (get_config()->num_worker > 1) { - if (!get_config()->tls_ctx_per_worker) { - conn_handler->create_ssl_context(); - } - conn_handler->create_worker_thread(get_config()->num_worker); + if (get_config()->num_worker == 1) { + conn_handler->create_single_worker(); } else { - conn_handler->create_ssl_context(); - if (get_config()->downstream_proto == PROTO_HTTP2) { - conn_handler->create_http2_session(); - } else { - conn_handler->create_http1_connect_blocker(); - } + conn_handler->create_worker_thread(get_config()->num_worker); } #ifndef NOTHREADS @@ -778,7 +782,6 @@ void fill_default_config() { mod_config()->downstream_connections_per_host = 8; mod_config()->downstream_connections_per_frontend = 0; mod_config()->listener_disable_timeout = 0.; - mod_config()->auto_tls_ticket_key = true; mod_config()->tls_ctx_per_worker = false; mod_config()->downstream_request_buffer_size = 16 * 1024; mod_config()->downstream_response_buffer_size = 16 * 1024; @@ -1817,17 +1820,6 @@ int main(int argc, char **argv) { mod_config()->alpn_prefs = ssl::set_alpn_prefs(get_config()->npn_list); - if (!get_config()->tls_ticket_key_files.empty()) { - auto ticket_keys = - read_tls_ticket_key_file(get_config()->tls_ticket_key_files); - if (!ticket_keys) { - LOG(WARN) << "Use internal session ticket key generator"; - } else { - worker_config->ticket_keys = std::move(ticket_keys); - mod_config()->auto_tls_ticket_key = false; - } - } - if (get_config()->backend_ipv4 && get_config()->backend_ipv6) { LOG(FATAL) << "--backend-ipv4 and --backend-ipv6 cannot be used at the " << "same time."; @@ -1849,6 +1841,7 @@ int main(int argc, char **argv) { if (get_config()->client || get_config()->client_proxy) { mod_config()->client_mode = true; + mod_config()->upstream_no_tls = true; } if (get_config()->client_mode || get_config()->http2_bridge) { @@ -1857,12 +1850,11 @@ int main(int argc, char **argv) { mod_config()->downstream_proto = PROTO_HTTP; } - if (!get_config()->client_mode && !get_config()->upstream_no_tls) { - if (!get_config()->private_key_file || !get_config()->cert_file) { - print_usage(std::cerr); - LOG(FATAL) << "Too few arguments"; - exit(EXIT_FAILURE); - } + if (!get_config()->upstream_no_tls && + (!get_config()->private_key_file || !get_config()->cert_file)) { + print_usage(std::cerr); + LOG(FATAL) << "Too few arguments"; + exit(EXIT_FAILURE); } if (get_config()->downstream_addrs.empty()) { diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 5cdfd70a..b611632e 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -352,21 +352,17 @@ int ClientHandler::upstream_http1_connhd_read() { return 0; } -ClientHandler::ClientHandler(struct ev_loop *loop, int fd, SSL *ssl, - const char *ipaddr, const char *port, - WorkerStat *worker_stat, - DownstreamConnectionPool *dconn_pool) - : conn_(loop, fd, ssl, get_config()->upstream_write_timeout, +ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl, + const char *ipaddr, const char *port) + : conn_(worker->get_loop(), fd, ssl, get_config()->upstream_write_timeout, get_config()->upstream_read_timeout, get_config()->write_rate, get_config()->write_burst, get_config()->read_rate, get_config()->read_burst, writecb, readcb, timeoutcb, this), - ipaddr_(ipaddr), port_(port), dconn_pool_(dconn_pool), - http2session_(nullptr), http1_connect_blocker_(nullptr), - worker_stat_(worker_stat), + ipaddr_(ipaddr), port_(port), worker_(worker), left_connhd_len_(NGHTTP2_CLIENT_CONNECTION_PREFACE_LEN), should_close_after_write_(false) { - ++worker_stat->num_connections; + ++worker_->get_worker_stat()->num_connections; ev_timer_init(&reneg_shutdown_timer_, shutdowncb, 0., 0.); @@ -402,13 +398,14 @@ ClientHandler::~ClientHandler() { upstream_->on_handler_delete(); } - --worker_stat_->num_connections; + auto worker_stat = worker_->get_worker_stat(); + --worker_stat->num_connections; ev_timer_stop(conn_.loop, &reneg_shutdown_timer_); // TODO If backend is http/2, and it is in CONNECTED state, signal // it and make it loopbreak when output is zero. - if (worker_config->graceful_shutdown && worker_stat_->num_connections == 0) { + if (worker_config->graceful_shutdown && worker_stat->num_connections == 0) { ev_break(conn_.loop); } @@ -579,7 +576,8 @@ void ClientHandler::pool_downstream_connection( CLOG(INFO, this) << "Pooling downstream connection DCONN:" << dconn.get(); } dconn->set_client_handler(nullptr); - dconn_pool_->add_downstream_connection(std::move(dconn)); + auto dconn_pool = worker_->get_dconn_pool(); + dconn_pool->add_downstream_connection(std::move(dconn)); } void ClientHandler::remove_downstream_connection(DownstreamConnection *dconn) { @@ -587,12 +585,14 @@ void ClientHandler::remove_downstream_connection(DownstreamConnection *dconn) { CLOG(INFO, this) << "Removing downstream connection DCONN:" << dconn << " from pool"; } - dconn_pool_->remove_downstream_connection(dconn); + auto dconn_pool = worker_->get_dconn_pool(); + dconn_pool->remove_downstream_connection(dconn); } std::unique_ptr ClientHandler::get_downstream_connection() { - auto dconn = dconn_pool_->pop_downstream_connection(); + auto dconn_pool = worker_->get_dconn_pool(); + auto dconn = dconn_pool->pop_downstream_connection(); if (!dconn) { if (LOG_ENABLED(INFO)) { @@ -600,11 +600,13 @@ ClientHandler::get_downstream_connection() { << " Create new one"; } - if (http2session_) { - dconn = - make_unique(dconn_pool_, http2session_); + auto dconn_pool = worker_->get_dconn_pool(); + auto http2session = worker_->get_http2_session(); + + if (http2session) { + dconn = make_unique(dconn_pool, http2session); } else { - dconn = make_unique(dconn_pool_, conn_.loop); + dconn = make_unique(dconn_pool, conn_.loop); } dconn->set_client_handler(this); return dconn; @@ -622,19 +624,12 @@ ClientHandler::get_downstream_connection() { SSL *ClientHandler::get_ssl() const { return conn_.tls.ssl; } -void ClientHandler::set_http2_session(Http2Session *http2session) { - http2session_ = http2session; -} - -Http2Session *ClientHandler::get_http2_session() const { return http2session_; } - -void ClientHandler::set_http1_connect_blocker( - ConnectBlocker *http1_connect_blocker) { - http1_connect_blocker_ = http1_connect_blocker; +Http2Session *ClientHandler::get_http2_session() const { + return worker_->get_http2_session(); } ConnectBlocker *ClientHandler::get_http1_connect_blocker() const { - return http1_connect_blocker_; + return worker_->get_http1_connect_blocker(); } void ClientHandler::direct_http2_upgrade() { @@ -724,8 +719,6 @@ void ClientHandler::write_accesslog(int major, int minor, unsigned int status, upstream_accesslog(get_config()->accesslog_format, &lgsp); } -WorkerStat *ClientHandler::get_worker_stat() const { return worker_stat_; } - ClientHandler::WriteBuf *ClientHandler::get_wb() { return &wb_; } ClientHandler::ReadBuf *ClientHandler::get_rb() { return &rb_; } @@ -737,4 +730,6 @@ RateLimit *ClientHandler::get_wlimit() { return &conn_.wlimit; } ev_io *ClientHandler::get_wev() { return &conn_.wev; } +Worker *ClientHandler::get_worker() const { return worker_; } + } // namespace shrpx diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index 44662c6a..96021a59 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -47,13 +47,13 @@ class Http2Session; class HttpsUpstream; class ConnectBlocker; class DownstreamConnectionPool; +class Worker; struct WorkerStat; class ClientHandler { public: - ClientHandler(struct ev_loop *loop, int fd, SSL *ssl, const char *ipaddr, - const char *port, WorkerStat *worker_stat, - DownstreamConnectionPool *dconn_pool); + ClientHandler(Worker *worker, int fd, SSL *ssl, const char *ipaddr, + const char *port); ~ClientHandler(); // Performs clear text I/O @@ -93,9 +93,7 @@ public: void remove_downstream_connection(DownstreamConnection *dconn); std::unique_ptr get_downstream_connection(); SSL *get_ssl() const; - void set_http2_session(Http2Session *http2session); Http2Session *get_http2_session() const; - void set_http1_connect_blocker(ConnectBlocker *http1_connect_blocker); ConnectBlocker *get_http1_connect_blocker() const; // Call this function when HTTP/2 connection header is received at // the start of the connection. @@ -117,7 +115,7 @@ public: // corresponding Downstream object is not available. void write_accesslog(int major, int minor, unsigned int status, int64_t body_bytes_sent); - WorkerStat *get_worker_stat() const; + Worker *get_worker() const; using WriteBuf = Buffer<32768>; using ReadBuf = Buffer<8192>; @@ -141,12 +139,7 @@ private: std::string alpn_; std::function read_, write_; std::function on_read_, on_write_; - DownstreamConnectionPool *dconn_pool_; - // Shared HTTP2 session for each thread. NULL if backend is not - // HTTP2. Not deleted by this object. - Http2Session *http2session_; - ConnectBlocker *http1_connect_blocker_; - WorkerStat *worker_stat_; + Worker *worker_; // The number of bytes of HTTP/2 client connection header to read size_t left_connhd_len_; bool should_close_after_write_; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index ac920b43..e2c75514 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -303,7 +303,6 @@ struct Config { bool upstream_frame_debug; bool no_location_rewrite; bool no_host_rewrite; - bool auto_tls_ticket_key; bool tls_ctx_per_worker; bool no_server_push; }; diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 930ea077..3290ba1a 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -60,24 +60,15 @@ void acceptor_disable_cb(struct ev_loop *loop, ev_timer *w, int revent) { } // namespace ConnectionHandler::ConnectionHandler(struct ev_loop *loop) - : loop_(loop), sv_ssl_ctx_(nullptr), cl_ssl_ctx_(nullptr), - // rate_limit_group_(bufferevent_rate_limit_group_new( - // evbase, get_config()->worker_rate_limit_cfg)), - worker_stat_(make_unique()), worker_round_robin_cnt_(0) { + : single_worker_(nullptr), loop_(loop), worker_round_robin_cnt_(0) { ev_timer_init(&disable_acceptor_timer_, acceptor_disable_cb, 0., 0.); disable_acceptor_timer_.data = this; } ConnectionHandler::~ConnectionHandler() { - // bufferevent_rate_limit_group_free(rate_limit_group_); ev_timer_stop(loop_, &disable_acceptor_timer_); } -void ConnectionHandler::create_ssl_context() { - sv_ssl_ctx_ = ssl::setup_server_ssl_context(); - cl_ssl_ctx_ = ssl::setup_client_ssl_context(); -} - void ConnectionHandler::worker_reopen_log_files() { WorkerEvent wev; @@ -102,14 +93,41 @@ void ConnectionHandler::worker_renew_ticket_keys( } } +void ConnectionHandler::create_single_worker() { + auto cert_tree = ssl::create_cert_lookup_tree(); + auto sv_ssl_ctx = ssl::setup_server_ssl_context(cert_tree); + auto cl_ssl_ctx = ssl::setup_client_ssl_context(); + + single_worker_ = make_unique(loop_, sv_ssl_ctx, cl_ssl_ctx, cert_tree, + ticket_keys_); +} + void ConnectionHandler::create_worker_thread(size_t num) { #ifndef NOTHREADS assert(workers_.size() == 0); + SSL_CTX *sv_ssl_ctx = nullptr, *cl_ssl_ctx = nullptr; + ssl::CertLookupTree *cert_tree = nullptr; + + if (!get_config()->tls_ctx_per_worker) { + cert_tree = ssl::create_cert_lookup_tree(); + sv_ssl_ctx = ssl::setup_server_ssl_context(cert_tree); + cl_ssl_ctx = ssl::setup_client_ssl_context(); + } + for (size_t i = 0; i < num; ++i) { - workers_.push_back(make_unique(sv_ssl_ctx_, cl_ssl_ctx_, - worker_config->cert_tree, - worker_config->ticket_keys)); + auto loop = ev_loop_new(0); + + if (get_config()->tls_ctx_per_worker) { + cert_tree = ssl::create_cert_lookup_tree(); + sv_ssl_ctx = ssl::setup_server_ssl_context(cert_tree); + cl_ssl_ctx = ssl::setup_client_ssl_context(); + } + + auto worker = make_unique(loop, sv_ssl_ctx, cl_ssl_ctx, cert_tree, + ticket_keys_); + worker->run_async(); + workers_.push_back(std::move(worker)); if (LOG_ENABLED(INFO)) { LLOG(INFO, this) << "Created thread #" << workers_.size() - 1; @@ -163,7 +181,7 @@ int ConnectionHandler::handle_connection(int fd, sockaddr *addr, int addrlen) { if (get_config()->num_worker == 1) { - if (worker_stat_->num_connections >= + if (single_worker_->get_worker_stat()->num_connections >= get_config()->worker_frontend_connections) { if (LOG_ENABLED(INFO)) { @@ -175,8 +193,8 @@ int ConnectionHandler::handle_connection(int fd, sockaddr *addr, int addrlen) { return -1; } - auto client = ssl::accept_connection(loop_, sv_ssl_ctx_, fd, addr, addrlen, - worker_stat_.get(), &dconn_pool_); + auto client = + ssl::accept_connection(single_worker_.get(), fd, addr, addrlen); if (!client) { LLOG(ERROR, this) << "ClientHandler creation failed"; @@ -184,13 +202,13 @@ int ConnectionHandler::handle_connection(int fd, sockaddr *addr, int addrlen) { return -1; } - client->set_http2_session(http2session_.get()); - client->set_http1_connect_blocker(http1_connect_blocker_.get()); - return 0; } size_t idx = worker_round_robin_cnt_ % workers_.size(); + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Dispatch connection to worker #" << idx; + } ++worker_round_robin_cnt_; WorkerEvent wev; memset(&wev, 0, sizeof(wev)); @@ -208,16 +226,8 @@ struct ev_loop *ConnectionHandler::get_loop() const { return loop_; } -void ConnectionHandler::create_http2_session() { - http2session_ = make_unique(loop_, cl_ssl_ctx_); -} - -void ConnectionHandler::create_http1_connect_blocker() { - http1_connect_blocker_ = make_unique(loop_); -} - -const WorkerStat *ConnectionHandler::get_worker_stat() const { - return worker_stat_.get(); +Worker *ConnectionHandler::get_single_worker() const { + return single_worker_.get(); } void ConnectionHandler::set_acceptor4(std::unique_ptr h) { @@ -276,4 +286,16 @@ void ConnectionHandler::accept_pending_connection() { } } +void +ConnectionHandler::set_ticket_keys(std::shared_ptr ticket_keys) { + ticket_keys_ = std::move(ticket_keys); + if (single_worker_) { + single_worker_->set_ticket_keys(ticket_keys_); + } +} + +const std::shared_ptr &ConnectionHandler::get_ticket_keys() const { + return ticket_keys_; +} + } // namespace shrpx diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h index bac31d9d..955f978a 100644 --- a/src/shrpx_connection_handler.h +++ b/src/shrpx_connection_handler.h @@ -54,14 +54,17 @@ public: ConnectionHandler(struct ev_loop *loop); ~ConnectionHandler(); int handle_connection(int fd, sockaddr *addr, int addrlen); - void create_ssl_context(); + // Creates Worker object for single threaded configuration. + void create_single_worker(); + // Creates |num| Worker objects for multi threaded configuration. + // The |num| must be strictly more than 1. void create_worker_thread(size_t num); void worker_reopen_log_files(); void worker_renew_ticket_keys(const std::shared_ptr &ticket_keys); + void set_ticket_keys(std::shared_ptr ticket_keys); + const std::shared_ptr &get_ticket_keys() const; struct ev_loop *get_loop() const; - void create_http2_session(); - void create_http1_connect_blocker(); - const WorkerStat *get_worker_stat() const; + Worker *get_single_worker() const; void set_acceptor4(std::unique_ptr h); AcceptHandler *get_acceptor4() const; void set_acceptor6(std::unique_ptr h); @@ -74,22 +77,19 @@ public: void join_worker(); private: - DownstreamConnectionPool dconn_pool_; + // Worker instances when multi threaded mode (-nN, N >= 2) is used. std::vector> workers_; + // Worker instance used when single threaded mode (-n1) is used. + // Otherwise, nullptr and workers_ has instances of Worker instead. + std::unique_ptr single_worker_; + // Current TLS session ticket keys. Note that TLS connection does + // not refer to this field directly. They use TicketKeys object in + // Worker object. + std::shared_ptr ticket_keys_; struct ev_loop *loop_; - // The frontend server SSL_CTX - SSL_CTX *sv_ssl_ctx_; - // The backend server SSL_CTX - SSL_CTX *cl_ssl_ctx_; - // Shared backend HTTP2 session. NULL if multi-threaded. In - // multi-threaded case, see shrpx_worker.cc. - std::unique_ptr http2session_; - std::unique_ptr http1_connect_blocker_; - // bufferevent_rate_limit_group *rate_limit_group_; std::unique_ptr acceptor4_; std::unique_ptr acceptor6_; ev_timer disable_acceptor_timer_; - std::unique_ptr worker_stat_; unsigned int worker_round_robin_cnt_; }; diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 5dd90151..db353dfc 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -140,7 +140,8 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { return -1; } - auto worker_stat = client_handler_->get_worker_stat(); + auto worker = client_handler_->get_worker(); + auto worker_stat = worker->get_worker_stat(); auto end = worker_stat->next_downstream; for (;;) { auto i = worker_stat->next_downstream; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 886b63ca..e7926b98 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -47,7 +47,6 @@ #include "shrpx_client_handler.h" #include "shrpx_config.h" #include "shrpx_worker.h" -#include "shrpx_worker_config.h" #include "shrpx_downstream_connection_pool.h" #include "util.h" #include "ssl.h" @@ -130,7 +129,9 @@ int ssl_pem_passwd_cb(char *buf, int size, int rwflag, void *user_data) { namespace { int servername_callback(SSL *ssl, int *al, void *arg) { - auto cert_tree = worker_config->cert_tree; + auto handler = static_cast(SSL_get_app_data(ssl)); + auto worker = handler->get_worker(); + auto cert_tree = worker->get_cert_lookup_tree(); if (cert_tree) { const char *hostname = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); if (hostname) { @@ -149,7 +150,8 @@ namespace { int ticket_key_cb(SSL *ssl, unsigned char *key_name, unsigned char *iv, EVP_CIPHER_CTX *ctx, HMAC_CTX *hctx, int enc) { auto handler = static_cast(SSL_get_app_data(ssl)); - const auto &ticket_keys = worker_config->ticket_keys; + auto worker = handler->get_worker(); + const auto &ticket_keys = worker->get_ticket_keys(); if (!ticket_keys) { // No ticket keys available. @@ -515,10 +517,8 @@ SSL_CTX *create_ssl_client_context() { return ssl_ctx; } -ClientHandler *accept_connection(struct ev_loop *loop, SSL_CTX *ssl_ctx, int fd, - sockaddr *addr, int addrlen, - WorkerStat *worker_stat, - DownstreamConnectionPool *dconn_pool) { +ClientHandler *accept_connection(Worker *worker, int fd, sockaddr *addr, + int addrlen) { char host[NI_MAXHOST]; char service[NI_MAXSERV]; int rv; @@ -537,6 +537,7 @@ ClientHandler *accept_connection(struct ev_loop *loop, SSL_CTX *ssl_ctx, int fd, LOG(WARN) << "Setting option TCP_NODELAY failed: errno=" << errno; } SSL *ssl = nullptr; + auto ssl_ctx = worker->get_sv_ssl_ctx(); if (ssl_ctx) { ssl = SSL_new(ssl_ctx); if (!ssl) { @@ -555,8 +556,7 @@ ClientHandler *accept_connection(struct ev_loop *loop, SSL_CTX *ssl_ctx, int fd, SSL_set_accept_state(ssl); } - return new ClientHandler(loop, fd, ssl, host, service, worker_stat, - dconn_pool); + return new ClientHandler(worker, fd, ssl, host, service); } namespace { @@ -927,7 +927,7 @@ bool check_http2_requirement(SSL *ssl) { return true; } -SSL_CTX *setup_server_ssl_context() { +SSL_CTX *setup_server_ssl_context(CertLookupTree *cert_tree) { if (get_config()->upstream_no_tls) { return nullptr; } @@ -939,9 +939,11 @@ SSL_CTX *setup_server_ssl_context() { return ssl_ctx; } - auto cert_tree = new CertLookupTree(); - - worker_config->cert_tree = cert_tree; + if (!cert_tree) { + LOG(WARN) << "We have multiple additional certificates (--subcert), but " + "cert_tree is not given. SNI may not work."; + return ssl_ctx; + } for (auto &keycert : get_config()->subcerts) { auto ssl_ctx = @@ -973,6 +975,13 @@ SSL_CTX *setup_client_ssl_context() { : nullptr; } +CertLookupTree *create_cert_lookup_tree() { + if (get_config()->upstream_no_tls || get_config()->subcerts.empty()) { + return nullptr; + } + return new ssl::CertLookupTree(); +} + } // namespace ssl } // namespace shrpx diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h index 68d3182e..710a999e 100644 --- a/src/shrpx_ssl.h +++ b/src/shrpx_ssl.h @@ -37,7 +37,7 @@ namespace shrpx { class ClientHandler; -struct WorkerStat; +class Worker; class DownstreamConnectionPool; namespace ssl { @@ -49,10 +49,8 @@ SSL_CTX *create_ssl_context(const char *private_key_file, // Create client side SSL_CTX SSL_CTX *create_ssl_client_context(); -ClientHandler *accept_connection(struct ev_loop *loop, SSL_CTX *ssl_ctx, int fd, - sockaddr *addr, int addrlen, - WorkerStat *worker_stat, - DownstreamConnectionPool *dconn_pool); +ClientHandler *accept_connection(Worker *worker, int fd, sockaddr *addr, + int addrlen); // Check peer's certificate against first downstream address in // Config::downstream_addrs. We only consider first downstream since @@ -143,15 +141,20 @@ std::vector set_alpn_prefs(const std::vector &protos); // Setups server side SSL_CTX. This function inspects get_config() // and if upstream_no_tls is true, returns nullptr. Otherwise -// construct default SSL_CTX. If subcerts are not empty, create -// SSL_CTX for them. All created SSL_CTX are added to CertLookupTree. -SSL_CTX *setup_server_ssl_context(); +// construct default SSL_CTX. If subcerts are available +// (get_config()->subcerts), caller should provide CertLookupTree +// object as |cert_tree| parameter, otherwise SNI does not work. +SSL_CTX *setup_server_ssl_context(CertLookupTree *cert_tree); // Setups client side SSL_CTX. This function inspects get_config() // and if downstream_no_tls is true, returns nullptr. Otherwise, only // construct SSL_CTX if either client_mode or http2_bridge is true. SSL_CTX *setup_client_ssl_context(); +// Creates CertLookupTree. If frontend is configured not to use TLS, +// this function returns nullptr. +CertLookupTree *create_cert_lookup_tree(); + } // namespace ssl } // namespace shrpx diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 9f5682ca..abe75165 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -48,35 +48,20 @@ void eventcb(struct ev_loop *loop, ev_async *w, int revents) { } } // namespace -Worker::Worker(SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, +Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, ssl::CertLookupTree *cert_tree, const std::shared_ptr &ticket_keys) - : loop_(ev_loop_new(0)), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), - worker_stat_(make_unique()) { + : loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), + cert_tree_(cert_tree), ticket_keys_(ticket_keys) { ev_async_init(&w_, eventcb); w_.data = this; ev_async_start(loop_, &w_); -#ifndef NOTHREADS - fut_ = std::async(std::launch::async, [this, cert_tree, &ticket_keys] { - if (get_config()->tls_ctx_per_worker) { - sv_ssl_ctx_ = ssl::setup_server_ssl_context(); - cl_ssl_ctx_ = ssl::setup_client_ssl_context(); - } else { - worker_config->cert_tree = cert_tree; - } - - if (get_config()->downstream_proto == PROTO_HTTP2) { - http2session_ = make_unique(loop_, cl_ssl_ctx_); - } else { - http1_connect_blocker_ = make_unique(loop_); - } - - worker_config->ticket_keys = ticket_keys; - (void)reopen_log_files(); - ev_run(loop_); - }); -#endif // !NOTHREADS + if (get_config()->downstream_proto == PROTO_HTTP2) { + http2session_ = make_unique(loop_, cl_ssl_ctx_); + } else { + http1_connect_blocker_ = make_unique(loop_); + } } Worker::~Worker() { ev_async_stop(loop_, &w_); } @@ -87,6 +72,15 @@ void Worker::wait() { #endif // !NOTHREADS } +void Worker::run_async() { +#ifndef NOTHREADS + fut_ = std::async(std::launch::async, [this] { + (void)reopen_log_files(); + ev_run(loop_); + }); +#endif // !NOTHREADS +} + void Worker::send(const WorkerEvent &event) { { std::lock_guard g(m_); @@ -111,7 +105,7 @@ void Worker::process_events() { << ", addrlen=" << wev.client_addrlen; } - if (worker_stat_->num_connections >= + if (worker_stat_.num_connections >= get_config()->worker_frontend_connections) { if (LOG_ENABLED(INFO)) { @@ -125,8 +119,7 @@ void Worker::process_events() { } auto client_handler = ssl::accept_connection( - loop_, sv_ssl_ctx_, wev.client_fd, &wev.client_addr.sa, - wev.client_addrlen, worker_stat_.get(), &dconn_pool_); + this, wev.client_fd, &wev.client_addr.sa, wev.client_addrlen); if (!client_handler) { if (LOG_ENABLED(INFO)) { WLOG(ERROR, this) << "ClientHandler creation failed"; @@ -135,9 +128,6 @@ void Worker::process_events() { break; } - client_handler->set_http2_session(http2session_.get()); - client_handler->set_http1_connect_blocker(http1_connect_blocker_.get()); - if (LOG_ENABLED(INFO)) { WLOG(INFO, this) << "CLIENT_HANDLER:" << client_handler << " created "; } @@ -150,7 +140,7 @@ void Worker::process_events() { << ")"; } - worker_config->ticket_keys = wev.ticket_keys; + ticket_keys_ = wev.ticket_keys; break; case REOPEN_LOG: @@ -167,7 +157,7 @@ void Worker::process_events() { worker_config->graceful_shutdown = true; - if (worker_stat_->num_connections == 0) { + if (worker_stat_.num_connections == 0) { ev_break(loop_); return; @@ -182,4 +172,30 @@ void Worker::process_events() { } } +ssl::CertLookupTree *Worker::get_cert_lookup_tree() const { return cert_tree_; } + +const std::shared_ptr &Worker::get_ticket_keys() const { + return ticket_keys_; +} + +void Worker::set_ticket_keys(std::shared_ptr ticket_keys) { + ticket_keys_ = std::move(ticket_keys); +} + +WorkerStat *Worker::get_worker_stat() { return &worker_stat_; } + +DownstreamConnectionPool *Worker::get_dconn_pool() { return &dconn_pool_; } + +Http2Session *Worker::get_http2_session() const { return http2session_.get(); } + +ConnectBlocker *Worker::get_http1_connect_blocker() const { + return http1_connect_blocker_.get(); +} + +struct ev_loop *Worker::get_loop() const { + return loop_; +} + +SSL_CTX *Worker::get_sv_ssl_ctx() const { return sv_ssl_ctx_; } + } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index cb527eda..9e0935d0 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -80,14 +80,25 @@ struct WorkerEvent { class Worker { public: - Worker(SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, + Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, ssl::CertLookupTree *cert_tree, const std::shared_ptr &ticket_keys); ~Worker(); + void run_async(); void wait(); void process_events(); void send(const WorkerEvent &event); + ssl::CertLookupTree *get_cert_lookup_tree() const; + const std::shared_ptr &get_ticket_keys() const; + void set_ticket_keys(std::shared_ptr ticket_keys); + WorkerStat *get_worker_stat(); + DownstreamConnectionPool *get_dconn_pool(); + Http2Session *get_http2_session() const; + ConnectBlocker *get_http1_connect_blocker() const; + struct ev_loop *get_loop() const; + SSL_CTX *get_sv_ssl_ctx() const; + private: #ifndef NOTHREADS std::future fut_; @@ -96,12 +107,18 @@ private: std::deque q_; ev_async w_; DownstreamConnectionPool dconn_pool_; + WorkerStat worker_stat_; struct ev_loop *loop_; + + // Following fields are shared across threads if + // get_config()->tls_ctx_per_worker == true. SSL_CTX *sv_ssl_ctx_; SSL_CTX *cl_ssl_ctx_; + ssl::CertLookupTree *cert_tree_; + + std::shared_ptr ticket_keys_; std::unique_ptr http2session_; std::unique_ptr http1_connect_blocker_; - std::unique_ptr worker_stat_; }; } // namespace shrpx diff --git a/src/shrpx_worker_config.cc b/src/shrpx_worker_config.cc index 4e95b23d..24332aaa 100644 --- a/src/shrpx_worker_config.cc +++ b/src/shrpx_worker_config.cc @@ -30,8 +30,8 @@ using namespace nghttp2; namespace shrpx { WorkerConfig::WorkerConfig() - : cert_tree(nullptr), accesslog_fd(-1), errorlog_fd(-1), - errorlog_tty(false), graceful_shutdown(false) {} + : accesslog_fd(-1), errorlog_fd(-1), errorlog_tty(false), + graceful_shutdown(false) {} #ifndef NOTHREADS thread_local diff --git a/src/shrpx_worker_config.h b/src/shrpx_worker_config.h index b281c2f8..57ed6c63 100644 --- a/src/shrpx_worker_config.h +++ b/src/shrpx_worker_config.h @@ -38,11 +38,9 @@ class CertLookupTree; struct TicketKeys; struct WorkerConfig { - std::shared_ptr ticket_keys; std::chrono::system_clock::time_point time_str_updated_; std::string time_local_str; std::string time_iso8601_str; - ssl::CertLookupTree *cert_tree; int accesslog_fd; int errorlog_fd; // true if errorlog_fd is referring to a terminal.