From b161dfe57374f1bb184cec42f4e0d95dd3e45387 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 25 Feb 2015 22:53:23 +0900 Subject: [PATCH] nghttpx: Move graceful_shutdown flag from WorkerConfig to Worker A part of an effort to eliminate thread_local WorkerConfig --- src/shrpx.cc | 6 +++--- src/shrpx_client_handler.cc | 2 +- src/shrpx_connection_handler.cc | 16 ++++++++++++++-- src/shrpx_connection_handler.h | 3 +++ src/shrpx_http2_upstream.cc | 6 +++++- src/shrpx_https_upstream.cc | 5 ++++- src/shrpx_worker.cc | 9 +++++++-- src/shrpx_worker.h | 5 +++++ src/shrpx_worker_config.cc | 3 +-- src/shrpx_worker_config.h | 1 - 10 files changed, 43 insertions(+), 13 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 5ca15b04..30162219 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -537,13 +537,13 @@ void graceful_shutdown_signal_cb(struct ev_loop *loop, ev_signal *w, int revents) { auto conn_handler = static_cast(w->data); - if (worker_config->graceful_shutdown) { + if (conn_handler->get_graceful_shutdown()) { return; } LOG(NOTICE) << "Graceful shutdown signal received"; - worker_config->graceful_shutdown = true; + conn_handler->set_graceful_shutdown(true); conn_handler->disable_acceptor(); @@ -571,7 +571,7 @@ void refresh_cb(struct ev_loop *loop, ev_timer *w, int revents) { // In multi threaded mode (get_config()->num_worker > 1), we have to // wait for event notification to workers to finish. - if (get_config()->num_worker == 1 && worker_config->graceful_shutdown && + if (get_config()->num_worker == 1 && conn_handler->get_graceful_shutdown() && (!worker || worker->get_worker_stat()->num_connections == 0)) { ev_break(loop); } diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 6c059b4d..a0e1b131 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -410,7 +410,7 @@ ClientHandler::~ClientHandler() { // TODO If backend is http/2, and it is in CONNECTED state, signal // it and make it loopbreak when output is zero. - if (worker_config->graceful_shutdown && worker_stat->num_connections == 0) { + if (worker_->get_graceful_shutdown() && worker_stat->num_connections == 0) { ev_break(conn_.loop); } diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 1bb1b184..29442891 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -51,7 +51,7 @@ void acceptor_disable_cb(struct ev_loop *loop, ev_timer *w, int revent) { // If we are in graceful shutdown period, we must not enable // acceptors again. - if (worker_config->graceful_shutdown) { + if (h->get_graceful_shutdown()) { return; } @@ -60,7 +60,8 @@ void acceptor_disable_cb(struct ev_loop *loop, ev_timer *w, int revent) { } // namespace ConnectionHandler::ConnectionHandler(struct ev_loop *loop) - : single_worker_(nullptr), loop_(loop), worker_round_robin_cnt_(0) { + : single_worker_(nullptr), loop_(loop), worker_round_robin_cnt_(0), + graceful_shutdown_(false) { ev_timer_init(&disable_acceptor_timer_, acceptor_disable_cb, 0., 0.); disable_acceptor_timer_.data = this; } @@ -298,4 +299,15 @@ const std::shared_ptr &ConnectionHandler::get_ticket_keys() const { return ticket_keys_; } +void ConnectionHandler::set_graceful_shutdown(bool f) { + graceful_shutdown_ = f; + if (single_worker_) { + single_worker_->set_graceful_shutdown(f); + } +} + +bool ConnectionHandler::get_graceful_shutdown() const { + return graceful_shutdown_; +} + } // namespace shrpx diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h index 9b127552..16847090 100644 --- a/src/shrpx_connection_handler.h +++ b/src/shrpx_connection_handler.h @@ -74,6 +74,8 @@ public: void disable_acceptor_temporary(ev_tstamp t); void accept_pending_connection(); void graceful_shutdown_worker(); + void set_graceful_shutdown(bool f); + bool get_graceful_shutdown() const; void join_worker(); private: @@ -93,6 +95,7 @@ private: std::unique_ptr acceptor6_; ev_timer disable_acceptor_timer_; unsigned int worker_round_robin_cnt_; + bool graceful_shutdown_; }; } // namespace shrpx diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 0ec365a2..eae890cd 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -36,6 +36,7 @@ #include "shrpx_config.h" #include "shrpx_http.h" #include "shrpx_worker_config.h" +#include "shrpx_worker.h" #include "http2.h" #include "util.h" #include "base64.h" @@ -592,7 +593,10 @@ void Http2Upstream::check_shutdown() { if (shutdown_handled_) { return; } - if (worker_config->graceful_shutdown) { + + auto worker = handler_->get_worker(); + + if (worker->get_graceful_shutdown()) { shutdown_handled_ = true; rv = nghttp2_submit_shutdown_notice(session_); if (rv != 0) { diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 08c832df..7bb954d9 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -35,6 +35,7 @@ #include "shrpx_config.h" #include "shrpx_error.h" #include "shrpx_worker_config.h" +#include "shrpx_worker.h" #include "http2.h" #include "util.h" #include "template.h" @@ -665,9 +666,11 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) { return 0; } + auto worker = handler_->get_worker(); + // after graceful shutdown commenced, add connection: close header // field. - if (worker_config->graceful_shutdown) { + if (worker->get_graceful_shutdown()) { downstream->set_response_connection_close(true); } diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index abe75165..7cfa58ba 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -52,7 +52,8 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, ssl::CertLookupTree *cert_tree, const std::shared_ptr &ticket_keys) : loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), - cert_tree_(cert_tree), ticket_keys_(ticket_keys) { + cert_tree_(cert_tree), ticket_keys_(ticket_keys), + graceful_shutdown_(false) { ev_async_init(&w_, eventcb); w_.data = this; ev_async_start(loop_, &w_); @@ -155,7 +156,7 @@ void Worker::process_events() { case GRACEFUL_SHUTDOWN: WLOG(NOTICE, this) << "Graceful shutdown commencing"; - worker_config->graceful_shutdown = true; + graceful_shutdown_ = true; if (worker_stat_.num_connections == 0) { ev_break(loop_); @@ -198,4 +199,8 @@ struct ev_loop *Worker::get_loop() const { SSL_CTX *Worker::get_sv_ssl_ctx() const { return sv_ssl_ctx_; } +void Worker::set_graceful_shutdown(bool f) { graceful_shutdown_ = f; } + +bool Worker::get_graceful_shutdown() const { return graceful_shutdown_; } + } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 9e0935d0..f9f4bb8a 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -99,6 +99,9 @@ public: struct ev_loop *get_loop() const; SSL_CTX *get_sv_ssl_ctx() const; + void set_graceful_shutdown(bool f); + bool get_graceful_shutdown() const; + private: #ifndef NOTHREADS std::future fut_; @@ -119,6 +122,8 @@ private: std::shared_ptr ticket_keys_; std::unique_ptr http2session_; std::unique_ptr http1_connect_blocker_; + + bool graceful_shutdown_; }; } // namespace shrpx diff --git a/src/shrpx_worker_config.cc b/src/shrpx_worker_config.cc index 24332aaa..5507c8d0 100644 --- a/src/shrpx_worker_config.cc +++ b/src/shrpx_worker_config.cc @@ -30,8 +30,7 @@ using namespace nghttp2; namespace shrpx { WorkerConfig::WorkerConfig() - : accesslog_fd(-1), errorlog_fd(-1), errorlog_tty(false), - graceful_shutdown(false) {} + : accesslog_fd(-1), errorlog_fd(-1), errorlog_tty(false) {} #ifndef NOTHREADS thread_local diff --git a/src/shrpx_worker_config.h b/src/shrpx_worker_config.h index 57ed6c63..273ab904 100644 --- a/src/shrpx_worker_config.h +++ b/src/shrpx_worker_config.h @@ -45,7 +45,6 @@ struct WorkerConfig { int errorlog_fd; // true if errorlog_fd is referring to a terminal. bool errorlog_tty; - bool graceful_shutdown; WorkerConfig(); void update_tstamp(const std::chrono::system_clock::time_point &now);