nghttpx: Add --single-worker option

Previously, nghttpx will use only one single thread inside the worker
process if --workers=1 (this is default).  If --workers=N, N > 1, we
use additional threads for accepting connections, or API request
processing, etc.

With this commit, we use the same processing model for N > 1 even if N
== 1.  To restore the original single thread execution mode,
--single-worker option is added.  If threading is disabled
--single-worker is always true.

gennghttpxfun.py

@@ -162,6 +162,7 @@ OPTIONS = [
     "tls-max-proto-version",
     "redirect-https-port",
     "frontend-max-requests",
+    "single-thread",
 ]
 
 LOGVARS = [

src/shrpx.cc

@@ -1387,6 +1387,10 @@ void fill_default_config(Config *config) {
   config->conf_path = StringRef::from_lit("/etc/nghttpx/nghttpx.conf");
   config->pid = getpid();
 
+#ifdef NOTHREADS
+  config->single_thread = true;
+#endif // NOTHREADS
+
   if (ev_supported_backends() & ~ev_recommended_backends() & EVBACKEND_KQUEUE) {
     config->ev_loop_flags = ev_recommended_backends() | EVBACKEND_KQUEUE;
   }
@@ -1824,6 +1828,12 @@ Performance:
               Set the number of worker threads.
               Default: )"
       << config->num_worker << R"(
+  --single-thread
+              Run everything in one  thread inside the worker process.
+              This   feature   is   provided  for   better   debugging
+              experience,  or  for  the platforms  which  lack  thread
+              support.   If  threading  is disabled,  this  option  is
+              always enabled.
   --read-rate=<SIZE>
               Set maximum  average read  rate on  frontend connection.
               Setting 0 to this option means read rate is unlimited.
@@ -2707,6 +2717,11 @@ int process_options(Config *config,
     }
   }
 
+  if (config->single_thread) {
+    LOG(WARN) << "single-thread: Set workers to 1";
+    config->num_worker = 1;
+  }
+
   auto &http2conf = config->http2;
   {
     auto &dumpconf = http2conf.upstream.debug.dump;
@@ -3276,6 +3291,7 @@ int main(int argc, char **argv) {
         {SHRPX_OPT_REDIRECT_HTTPS_PORT.c_str(), required_argument, &flag, 154},
         {SHRPX_OPT_FRONTEND_MAX_REQUESTS.c_str(), required_argument, &flag,
          155},
+        {SHRPX_OPT_SINGLE_THREAD.c_str(), no_argument, &flag, 156},
         {nullptr, 0, nullptr, 0}};
 
     int option_index = 0;
@@ -4007,6 +4023,11 @@ int main(int argc, char **argv) {
         cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_MAX_REQUESTS,
                              StringRef{optarg});
         break;
+      case 156:
+        // --single-thread
+        cmdcfgs.emplace_back(SHRPX_OPT_SINGLE_THREAD,
+                             StringRef::from_lit("yes"));
+        break;
       default:
         break;
       }

src/shrpx_config.cc

@@ -1573,6 +1573,9 @@ int option_lookup_token(const char *name, size_t namelen) {
       if (util::strieq_l("add-forwarde", name, 12)) {
         return SHRPX_OPTID_ADD_FORWARDED;
       }
+      if (util::strieq_l("single-threa", name, 12)) {
+        return SHRPX_OPTID_SINGLE_THREAD;
+      }
       break;
     case 'e':
       if (util::strieq_l("dh-param-fil", name, 12)) {
@@ -3376,6 +3379,10 @@ int parse_config(Config *config, int optid, const StringRef &opt,
   }
   case SHRPX_OPTID_FRONTEND_MAX_REQUESTS:
     return parse_uint(&config->http.max_requests, opt, optarg);
+  case SHRPX_OPTID_SINGLE_THREAD:
+    config->single_thread = util::strieq_l("yes", optarg);
+
+    return 0;
   case SHRPX_OPTID_CONF:
     LOG(WARN) << "conf: ignored";
 

src/shrpx_config.h

@@ -335,6 +335,7 @@ constexpr auto SHRPX_OPT_REDIRECT_HTTPS_PORT =
     StringRef::from_lit("redirect-https-port");
 constexpr auto SHRPX_OPT_FRONTEND_MAX_REQUESTS =
     StringRef::from_lit("frontend-max-requests");
+constexpr auto SHRPX_OPT_SINGLE_THREAD = StringRef::from_lit("single-thread");
 
 constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8;
 
@@ -863,6 +864,7 @@ struct Config {
         verbose{false},
         daemon{false},
         http2_proxy{false},
+        single_thread{false},
         ev_loop_flags{0} {}
   ~Config();
 
@@ -903,6 +905,7 @@ struct Config {
   bool verbose;
   bool daemon;
   bool http2_proxy;
+  bool single_thread;
   // flags passed to ev_default_loop() and ev_loop_new()
   int ev_loop_flags;
 };
@@ -1037,6 +1040,7 @@ enum {
   SHRPX_OPTID_RESPONSE_HEADER_FIELD_BUFFER,
   SHRPX_OPTID_RLIMIT_NOFILE,
   SHRPX_OPTID_SERVER_NAME,
+  SHRPX_OPTID_SINGLE_THREAD,
   SHRPX_OPTID_STREAM_READ_TIMEOUT,
   SHRPX_OPTID_STREAM_WRITE_TIMEOUT,
   SHRPX_OPTID_STRIP_INCOMING_FORWARDED,

src/shrpx_connection_handler.cc

@@ -333,7 +333,7 @@ void ConnectionHandler::join_worker() {
 }
 
 void ConnectionHandler::graceful_shutdown_worker() {
-  if (get_config()->num_worker == 1) {
+  if (single_worker_) {
     return;
   }
 
@@ -369,7 +369,7 @@ int ConnectionHandler::handle_connection(int fd, sockaddr *addr, int addrlen,
 
   auto config = get_config();
 
-  if (config->num_worker == 1) {
+  if (single_worker_) {
     auto &upstreamconf = config->conn.upstream;
     if (single_worker_->get_worker_stat()->num_connections >=
         upstreamconf.worker_connections) {

src/shrpx_worker_process.cc

@@ -123,9 +123,9 @@ void graceful_shutdown(ConnectionHandler *conn_handler) {
 
   conn_handler->graceful_shutdown_worker();
 
-  if (get_config()->num_worker == 1) {
+  auto single_worker = conn_handler->get_single_worker();
-    if (conn_handler->get_single_worker()->get_worker_stat()->num_connections ==
+  if (single_worker) {
-        0) {
+    if (single_worker->get_worker_stat()->num_connections == 0) {
       ev_break(conn_handler->get_loop());
     }
 
@@ -144,9 +144,7 @@ void reopen_log(ConnectionHandler *conn_handler) {
   (void)reopen_log_files(loggingconf);
   redirect_stderr_to_errorlog(loggingconf);
 
-  if (get_config()->num_worker > 1) {
+  conn_handler->worker_reopen_log_files();
-    conn_handler->worker_reopen_log_files();
-  }
 }
 } // namespace
 
@@ -508,7 +506,7 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) {
     }
   }
 
-  if (config->num_worker == 1) {
+  if (config->single_thread) {
     rv = conn_handler.create_single_worker();
     if (rv != 0) {
       return -1;