From 75ff04f87a7211c5e1b19788aa17a5f5ab3e0859 Mon Sep 17 00:00:00 2001
From: Tatsuhiro Tsujikawa <tatsuhiro.t@gmail.com>
Date: Tue, 29 Sep 2015 00:40:33 +0900
Subject: [PATCH] nghttpx: Don't stop default loop of worker process on
 graceful shutdown

To keep ipc channel being read from worker process, default loop
should not be stopped.  To join all worker threads, now we use
dedicated thread to do this.  When all worker threads are joined,
ev_async_send sends message to default loop, and it is finally
stopped.
---
 src/shrpx_connection_handler.cc | 21 ++++++++++++++++++++-
 src/shrpx_connection_handler.h  |  7 +++++++
 src/shrpx_worker_process.cc     | 18 ++++++------------
 3 files changed, 33 insertions(+), 13 deletions(-)

diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc
index 51232b1f..960beed0 100644
--- a/src/shrpx_connection_handler.cc
+++ b/src/shrpx_connection_handler.cc
@@ -96,6 +96,12 @@ void ocsp_chld_cb(struct ev_loop *loop, ev_child *w, int revent) {
 }
 } // namespace
 
+namespace {
+void thread_join_async_cb(struct ev_loop *loop, ev_async *w, int revent) {
+  ev_break(loop);
+}
+} // namespace
+
 ConnectionHandler::ConnectionHandler(struct ev_loop *loop)
     : single_worker_(nullptr), loop_(loop),
       tls_ticket_key_memcached_get_retry_count_(0),
@@ -110,6 +116,8 @@ ConnectionHandler::ConnectionHandler(struct ev_loop *loop)
   ev_io_init(&ocsp_.rev, ocsp_read_cb, -1, EV_READ);
   ocsp_.rev.data = this;
 
+  ev_async_init(&thread_join_asyncev_, thread_join_async_cb);
+
   ev_child_init(&ocsp_.chldev, ocsp_chld_cb, 0, 0);
   ocsp_.chldev.data = this;
 
@@ -120,6 +128,7 @@ ConnectionHandler::ConnectionHandler(struct ev_loop *loop)
 }
 
 ConnectionHandler::~ConnectionHandler() {
+  ev_async_stop(loop_, &thread_join_asyncev_);
   ev_timer_stop(loop_, &disable_acceptor_timer_);
   ev_timer_stop(loop_, &ocsp_timer_);
 
@@ -264,9 +273,19 @@ void ConnectionHandler::graceful_shutdown_worker() {
   }
 
   for (auto &worker : workers_) {
-
     worker->send(wev);
   }
+
+#ifndef NOTHREADS
+  ev_async_start(loop_, &thread_join_asyncev_);
+
+  thread_join_fut_ = std::async(std::launch::async, [this]() {
+    (void)reopen_log_files();
+    join_worker();
+    ev_async_send(get_loop(), &thread_join_asyncev_);
+    delete log_config();
+  });
+#endif // NOTHREADS
 }
 
 int ConnectionHandler::handle_connection(int fd, sockaddr *addr, int addrlen) {
diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h
index 51a746ca..f9c5b03a 100644
--- a/src/shrpx_connection_handler.h
+++ b/src/shrpx_connection_handler.h
@@ -34,6 +34,9 @@
 
 #include <memory>
 #include <vector>
+#ifndef NOTHREADS
+#include <future>
+#endif // NOTHREADS
 
 #include <openssl/ssl.h>
 
@@ -158,6 +161,10 @@ private:
 #endif // HAVE_NEVERBLEED
   ev_timer disable_acceptor_timer_;
   ev_timer ocsp_timer_;
+  ev_async thread_join_asyncev_;
+#ifndef NOTHREADS
+  std::future<void> thread_join_fut_;
+#endif // NOTHREADS
   size_t tls_ticket_key_memcached_get_retry_count_;
   size_t tls_ticket_key_memcached_fail_count_;
   unsigned int worker_round_robin_cnt_;
diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc
index 0dc3926c..c2358c58 100644
--- a/src/shrpx_worker_process.cc
+++ b/src/shrpx_worker_process.cc
@@ -112,19 +112,14 @@ void graceful_shutdown(ConnectionHandler *conn_handler) {
 
   conn_handler->graceful_shutdown_worker();
 
-  if (get_config()->num_worker == 1 &&
-      conn_handler->get_single_worker()->get_worker_stat()->num_connections >
-          0) {
+  if (get_config()->num_worker == 1) {
+    if (conn_handler->get_single_worker()->get_worker_stat()->num_connections ==
+        0) {
+      ev_break(conn_handler->get_loop());
+    }
+
     return;
   }
-
-  // We have accepted all pending connections.  Shutdown main event
-  // loop.
-
-  // TODO this makes IPC from master process impossible.  Perhaps, we
-  // should keep alive default loop, and break it once all connections
-  // are terminated somehow.
-  ev_break(conn_handler->get_loop());
 }
 } // namespace
 
@@ -527,7 +522,6 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) {
 
   ev_run(loop, 0);
 
-  conn_handler.join_worker();
   conn_handler.cancel_ocsp_update();
 
 #ifdef HAVE_NEVERBLEED