nghttpx: Move graceful_shutdown flag from WorkerConfig to Worker

A part of an effort to eliminate thread_local WorkerConfig

src/shrpx.cc

@@ -537,13 +537,13 @@ void graceful_shutdown_signal_cb(struct ev_loop *loop, ev_signal *w,
                                  int revents) {
   auto conn_handler = static_cast<ConnectionHandler *>(w->data);
 
-  if (worker_config->graceful_shutdown) {
+  if (conn_handler->get_graceful_shutdown()) {
     return;
   }
 
   LOG(NOTICE) << "Graceful shutdown signal received";
 
-  worker_config->graceful_shutdown = true;
+  conn_handler->set_graceful_shutdown(true);
 
   conn_handler->disable_acceptor();
 
@@ -571,7 +571,7 @@ void refresh_cb(struct ev_loop *loop, ev_timer *w, int revents) {
 
   // In multi threaded mode (get_config()->num_worker > 1), we have to
   // wait for event notification to workers to finish.
-  if (get_config()->num_worker == 1 && worker_config->graceful_shutdown &&
+  if (get_config()->num_worker == 1 && conn_handler->get_graceful_shutdown() &&
       (!worker || worker->get_worker_stat()->num_connections == 0)) {
     ev_break(loop);
   }

src/shrpx_client_handler.cc

@@ -410,7 +410,7 @@ ClientHandler::~ClientHandler() {
 
   // TODO If backend is http/2, and it is in CONNECTED state, signal
   // it and make it loopbreak when output is zero.
-  if (worker_config->graceful_shutdown && worker_stat->num_connections == 0) {
+  if (worker_->get_graceful_shutdown() && worker_stat->num_connections == 0) {
     ev_break(conn_.loop);
   }
 

src/shrpx_connection_handler.cc

@@ -51,7 +51,7 @@ void acceptor_disable_cb(struct ev_loop *loop, ev_timer *w, int revent) {
 
   // If we are in graceful shutdown period, we must not enable
   // acceptors again.
-  if (worker_config->graceful_shutdown) {
+  if (h->get_graceful_shutdown()) {
     return;
   }
 
@@ -60,7 +60,8 @@ void acceptor_disable_cb(struct ev_loop *loop, ev_timer *w, int revent) {
 } // namespace
 
 ConnectionHandler::ConnectionHandler(struct ev_loop *loop)
-    : single_worker_(nullptr), loop_(loop), worker_round_robin_cnt_(0) {
+    : single_worker_(nullptr), loop_(loop), worker_round_robin_cnt_(0),
+      graceful_shutdown_(false) {
   ev_timer_init(&disable_acceptor_timer_, acceptor_disable_cb, 0., 0.);
   disable_acceptor_timer_.data = this;
 }
@@ -298,4 +299,15 @@ const std::shared_ptr<TicketKeys> &ConnectionHandler::get_ticket_keys() const {
   return ticket_keys_;
 }
 
+void ConnectionHandler::set_graceful_shutdown(bool f) {
+  graceful_shutdown_ = f;
+  if (single_worker_) {
+    single_worker_->set_graceful_shutdown(f);
+  }
+}
+
+bool ConnectionHandler::get_graceful_shutdown() const {
+  return graceful_shutdown_;
+}
+
 } // namespace shrpx

src/shrpx_connection_handler.h

@@ -74,6 +74,8 @@ public:
   void disable_acceptor_temporary(ev_tstamp t);
   void accept_pending_connection();
   void graceful_shutdown_worker();
+  void set_graceful_shutdown(bool f);
+  bool get_graceful_shutdown() const;
   void join_worker();
 
 private:
@@ -93,6 +95,7 @@ private:
   std::unique_ptr<AcceptHandler> acceptor6_;
   ev_timer disable_acceptor_timer_;
   unsigned int worker_round_robin_cnt_;
+  bool graceful_shutdown_;
 };
 
 } // namespace shrpx

src/shrpx_http2_upstream.cc

@@ -36,6 +36,7 @@
 #include "shrpx_config.h"
 #include "shrpx_http.h"
 #include "shrpx_worker_config.h"
+#include "shrpx_worker.h"
 #include "http2.h"
 #include "util.h"
 #include "base64.h"
@@ -592,7 +593,10 @@ void Http2Upstream::check_shutdown() {
   if (shutdown_handled_) {
     return;
   }
-  if (worker_config->graceful_shutdown) {
+
+  auto worker = handler_->get_worker();
+
+  if (worker->get_graceful_shutdown()) {
     shutdown_handled_ = true;
     rv = nghttp2_submit_shutdown_notice(session_);
     if (rv != 0) {

src/shrpx_https_upstream.cc

@@ -35,6 +35,7 @@
 #include "shrpx_config.h"
 #include "shrpx_error.h"
 #include "shrpx_worker_config.h"
+#include "shrpx_worker.h"
 #include "http2.h"
 #include "util.h"
 #include "template.h"
@@ -665,9 +666,11 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) {
     return 0;
   }
 
+  auto worker = handler_->get_worker();
+
   // after graceful shutdown commenced, add connection: close header
   // field.
-  if (worker_config->graceful_shutdown) {
+  if (worker->get_graceful_shutdown()) {
     downstream->set_response_connection_close(true);
   }
 

src/shrpx_worker.cc

@@ -52,7 +52,8 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
                ssl::CertLookupTree *cert_tree,
                const std::shared_ptr<TicketKeys> &ticket_keys)
     : loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx),
-      cert_tree_(cert_tree), ticket_keys_(ticket_keys) {
+      cert_tree_(cert_tree), ticket_keys_(ticket_keys),
+      graceful_shutdown_(false) {
   ev_async_init(&w_, eventcb);
   w_.data = this;
   ev_async_start(loop_, &w_);
@@ -155,7 +156,7 @@ void Worker::process_events() {
     case GRACEFUL_SHUTDOWN:
       WLOG(NOTICE, this) << "Graceful shutdown commencing";
 
-      worker_config->graceful_shutdown = true;
+      graceful_shutdown_ = true;
 
       if (worker_stat_.num_connections == 0) {
         ev_break(loop_);
@@ -198,4 +199,8 @@ struct ev_loop *Worker::get_loop() const {
 
 SSL_CTX *Worker::get_sv_ssl_ctx() const { return sv_ssl_ctx_; }
 
+void Worker::set_graceful_shutdown(bool f) { graceful_shutdown_ = f; }
+
+bool Worker::get_graceful_shutdown() const { return graceful_shutdown_; }
+
 } // namespace shrpx

src/shrpx_worker.h

@@ -99,6 +99,9 @@ public:
   struct ev_loop *get_loop() const;
   SSL_CTX *get_sv_ssl_ctx() const;
 
+  void set_graceful_shutdown(bool f);
+  bool get_graceful_shutdown() const;
+
 private:
 #ifndef NOTHREADS
   std::future<void> fut_;
@@ -119,6 +122,8 @@ private:
   std::shared_ptr<TicketKeys> ticket_keys_;
   std::unique_ptr<Http2Session> http2session_;
   std::unique_ptr<ConnectBlocker> http1_connect_blocker_;
+
+  bool graceful_shutdown_;
 };
 
 } // namespace shrpx

src/shrpx_worker_config.cc

@@ -30,8 +30,7 @@ using namespace nghttp2;
 namespace shrpx {
 
 WorkerConfig::WorkerConfig()
-    : accesslog_fd(-1), errorlog_fd(-1), errorlog_tty(false),
+    : accesslog_fd(-1), errorlog_fd(-1), errorlog_tty(false) {}
-      graceful_shutdown(false) {}
 
 #ifndef NOTHREADS
 thread_local

src/shrpx_worker_config.h

@@ -45,7 +45,6 @@ struct WorkerConfig {
   int errorlog_fd;
   // true if errorlog_fd is referring to a terminal.
   bool errorlog_tty;
-  bool graceful_shutdown;
 
   WorkerConfig();
   void update_tstamp(const std::chrono::system_clock::time_point &now);