nghttpx: Reload configuration with SIGHUP

This commit implements configuration reloading with SIGHUP.
There are rough edges left:

* Rename SignalServer with more meaningful name, say, WorkerProcess.
* We should introduce global configuration object which is not
  affected by configuration reloading.  It should hold cmdcfgs, argc,
  argv, and last worker PID.
* We should close the listener file descriptor when some operation was
  failed after that.

src/shrpx.cc

@@ -130,22 +130,138 @@ constexpr auto ENV_ACCEPT_PREFIX = StringRef::from_lit("NGHTTPX_ACCEPT_");
 #endif
 #endif
 
+struct InheritedAddr {
+  // IP address if TCP socket.  Otherwise, UNIX domain socket path.
+  ImmutableString host;
+  uint16_t port;
+  // true if UNIX domain socket path
+  bool host_unix;
+  int fd;
+  bool used;
+};
+
+namespace {
+std::random_device rd;
+} // namespace
+
+namespace {
+// This contains all options given in command-line.  Make it static so
+// that we can use it in reloading.
+std::vector<std::pair<StringRef, StringRef>> cmdcfgs;
+} // namespace
+
+namespace {
+void signal_cb(struct ev_loop *loop, ev_signal *w, int revents);
+} // namespace
+
+namespace {
+void worker_process_child_cb(struct ev_loop *loop, ev_child *w, int revents);
+} // namespace
+
 struct SignalServer {
-  SignalServer() : ipc_fd{{-1, -1}}, worker_process_pid(-1) {}
+  SignalServer(struct ev_loop *loop, pid_t worker_pid, int ipc_fd)
-  ~SignalServer() {
+      : loop(loop), worker_pid(worker_pid), ipc_fd(ipc_fd) {
-    if (ipc_fd[0] != -1) {
+    ev_signal_init(&reopen_log_signalev, signal_cb, REOPEN_LOG_SIGNAL);
-      close(ipc_fd[0]);
+    reopen_log_signalev.data = this;
+    ev_signal_start(loop, &reopen_log_signalev);
+
+    ev_signal_init(&exec_binary_signalev, signal_cb, EXEC_BINARY_SIGNAL);
+    exec_binary_signalev.data = this;
+    ev_signal_start(loop, &exec_binary_signalev);
+
+    ev_signal_init(&graceful_shutdown_signalev, signal_cb,
+                   GRACEFUL_SHUTDOWN_SIGNAL);
+    graceful_shutdown_signalev.data = this;
+    ev_signal_start(loop, &graceful_shutdown_signalev);
+
+    ev_signal_init(&reload_signalev, signal_cb, RELOAD_SIGNAL);
+    reload_signalev.data = this;
+    ev_signal_start(loop, &reload_signalev);
+
+    ev_child_init(&worker_process_childev, worker_process_child_cb, worker_pid,
+                  0);
+    worker_process_childev.data = this;
+    ev_child_start(loop, &worker_process_childev);
   }
-    if (ipc_fd[1] != -1) {
+
-      shutdown(ipc_fd[1], SHUT_WR);
+  ~SignalServer() {
-      close(ipc_fd[1]);
+    shutdown_signal_watchers();
+
+    ev_child_stop(loop, &worker_process_childev);
+
+    if (ipc_fd != -1) {
+      shutdown(ipc_fd, SHUT_WR);
+      close(ipc_fd);
     }
   }
 
-  std::array<int, 2> ipc_fd;
+  void shutdown_signal_watchers() {
-  pid_t worker_process_pid;
+    ev_signal_stop(loop, &reopen_log_signalev);
+    ev_signal_stop(loop, &exec_binary_signalev);
+    ev_signal_stop(loop, &graceful_shutdown_signalev);
+    ev_signal_stop(loop, &reload_signalev);
+  }
+
+  ev_signal reopen_log_signalev;
+  ev_signal exec_binary_signalev;
+  ev_signal graceful_shutdown_signalev;
+  ev_signal reload_signalev;
+  ev_child worker_process_childev;
+  struct ev_loop *loop;
+  pid_t worker_pid;
+  int ipc_fd;
 };
 
+namespace {
+void reload_config(SignalServer *ssv);
+} // namespace
+
+namespace {
+std::deque<std::unique_ptr<SignalServer>> signal_servers;
+} // namespace
+
+namespace {
+void signal_server_add(std::unique_ptr<SignalServer> ssv) {
+  signal_servers.push_back(std::move(ssv));
+}
+} // namespace
+
+namespace {
+void signal_server_remove(const SignalServer *ssv) {
+  for (auto it = std::begin(signal_servers); it != std::end(signal_servers);
+       ++it) {
+    auto &s = *it;
+
+    if (s.get() != ssv) {
+      continue;
+    }
+
+    signal_servers.erase(it);
+    break;
+  }
+}
+} // namespace
+
+namespace {
+void signal_server_remove_all() {
+  std::deque<std::unique_ptr<SignalServer>>().swap(signal_servers);
+}
+} // namespace
+
+namespace {
+// Send signal |signum| to all worker processes, and clears
+// signal_servers.
+void signal_server_kill(int signum) {
+  for (auto &s : signal_servers) {
+    if (s->worker_pid == -1) {
+      continue;
+    }
+    kill(s->worker_pid, signum);
+  }
+  signal_servers.clear();
+}
+} // namespace
+
 namespace {
 int chown_to_running_user(const char *path) {
   return chown(path, get_config()->uid, get_config()->gid);
@@ -346,8 +462,7 @@ void exec_binary(SignalServer *ssv) {
 namespace {
 void ipc_send(SignalServer *ssv, uint8_t ipc_event) {
   ssize_t nwrite;
-  while ((nwrite = write(ssv->ipc_fd[1], &ipc_event, 1)) == -1 &&
+  while ((nwrite = write(ssv->ipc_fd, &ipc_event, 1)) == -1 && errno == EINTR)
-         errno == EINTR)
     ;
 
   if (nwrite < 0) {
@@ -377,7 +492,7 @@ void reopen_log(SignalServer *ssv) {
 namespace {
 void signal_cb(struct ev_loop *loop, ev_signal *w, int revents) {
   auto ssv = static_cast<SignalServer *>(w->data);
-  if (ssv->worker_process_pid == -1) {
+  if (ssv->worker_pid == -1) {
     ev_break(loop);
     return;
   }
@@ -392,8 +507,11 @@ void signal_cb(struct ev_loop *loop, ev_signal *w, int revents) {
   case GRACEFUL_SHUTDOWN_SIGNAL:
     ipc_send(ssv, SHRPX_IPC_GRACEFUL_SHUTDOWN);
     return;
+  case RELOAD_SIGNAL:
+    reload_config(ssv);
+    return;
   default:
-    kill(ssv->worker_process_pid, w->signum);
+    signal_server_kill(w->signum);
     ev_break(loop);
     return;
   }
@@ -402,24 +520,20 @@ void signal_cb(struct ev_loop *loop, ev_signal *w, int revents) {
 
 namespace {
 void worker_process_child_cb(struct ev_loop *loop, ev_child *w, int revents) {
+  auto ssv = static_cast<SignalServer *>(w->data);
+
   log_chld(w->rpid, w->rstatus, "Worker process");
 
-  ev_child_stop(loop, w);
+  auto pid = ssv->worker_pid;
 
+  signal_server_remove(ssv);
+
+  if (get_config()->last_worker_pid == pid) {
     ev_break(loop);
+  }
 }
 } // namespace
 
-struct InheritedAddr {
-  // IP address if TCP socket.  Otherwise, UNIX domain socket path.
-  ImmutableString host;
-  uint16_t port;
-  // true if UNIX domain socket path
-  bool host_unix;
-  int fd;
-  bool used;
-};
-
 namespace {
 int create_unix_domain_server_socket(UpstreamAddr &faddr,
                                      std::vector<InheritedAddr> &iaddrs) {
@@ -660,6 +774,70 @@ int create_tcp_server_socket(UpstreamAddr &faddr,
 } // namespace
 
 namespace {
+// Returns array of InheritedAddr constructed from |config|.  This
+// function is intended to be used when reloading configuration, and
+// |config| is usually a current configuration.
+std::vector<InheritedAddr>
+get_inherited_addr_from_config(const Config *config) {
+  int rv;
+  std::vector<InheritedAddr> iaddrs;
+
+  auto &listenerconf = config->conn.listener;
+
+  for (auto &addr : listenerconf.addrs) {
+    iaddrs.emplace_back();
+    auto &iaddr = iaddrs.back();
+
+    if (addr.host_unix) {
+      iaddr.host = addr.host;
+      iaddr.host_unix = true;
+      iaddr.fd = addr.fd;
+
+      continue;
+    }
+
+    iaddr.port = addr.port;
+    iaddr.fd = addr.fd;
+
+    // We have to getsockname/getnameinfo for fd, since we may have
+    // '*' appear in addr.host, which makes comparison against "real"
+    // address fail.
+
+    sockaddr_union su;
+    socklen_t salen = sizeof(su);
+
+    // We already added entry to iaddrs.  Even if we got errors, we
+    // don't remove it.  This is required because we have to close the
+    // socket if it is not reused.  The empty host name usually does
+    // not match anything.
+
+    if (getsockname(addr.fd, &su.sa, &salen) != 0) {
+      auto error = errno;
+      LOG(WARN) << "getsockname() syscall failed (fd=" << addr.fd
+                << "): " << strerror(error);
+      continue;
+    }
+
+    std::array<char, NI_MAXHOST> host;
+    rv = getnameinfo(&su.sa, salen, host.data(), host.size(), nullptr, 0,
+                     NI_NUMERICHOST);
+    if (rv != 0) {
+      LOG(WARN) << "getnameinfo() failed (fd=" << addr.fd
+                << "): " << gai_strerror(rv);
+      continue;
+    }
+
+    iaddr.host = host.data();
+  }
+
+  return iaddrs;
+}
+} // namespace
+
+namespace {
+// Returns array of InheritedAddr constructed from environment
+// variables.  This function handles the old environment variable
+// names used in 1.7.0 or earlier.
 std::vector<InheritedAddr> get_inherited_addr_from_env() {
   int rv;
   std::vector<InheritedAddr> iaddrs;
@@ -808,7 +986,8 @@ std::vector<InheritedAddr> get_inherited_addr_from_env() {
 } // namespace
 
 namespace {
-void closeUnusedInheritedAddr(const std::vector<InheritedAddr> &iaddrs) {
+// Closes all sockets which are not reused.
+void close_unused_inherited_addr(const std::vector<InheritedAddr> &iaddrs) {
   for (auto &ia : iaddrs) {
     if (ia.used) {
       continue;
@@ -820,8 +999,8 @@ void closeUnusedInheritedAddr(const std::vector<InheritedAddr> &iaddrs) {
 } // namespace
 
 namespace {
-int create_acceptor_socket(std::vector<InheritedAddr> &iaddrs) {
+int create_acceptor_socket(Config *config, std::vector<InheritedAddr> &iaddrs) {
-  auto &listenerconf = mod_config()->conn.listener;
+  auto &listenerconf = config->conn.listener;
 
   for (auto &addr : listenerconf.addrs) {
     if (addr.host_unix) {
@@ -829,7 +1008,7 @@ int create_acceptor_socket(std::vector<InheritedAddr> &iaddrs) {
         return -1;
       }
 
-      if (get_config()->uid != 0) {
+      if (config->uid != 0) {
         // fd is not associated to inode, so we cannot use fchown(2)
         // here.  https://lkml.org/lkml/2004/11/1/84
         if (chown_to_running_user(addr.host.c_str()) == -1) {
@@ -861,15 +1040,54 @@ int call_daemon() {
 } // namespace
 
 namespace {
-pid_t fork_worker_process(SignalServer *ssv) {
+// Opens IPC socket used to communicate with worker proess.  The
+// communication is unidirectional; that is main process sends
+// messages to the worker process.  On success, ipc_fd[0] is for
+// reading, and ipc_fd[1] for writing, just like pipe(2).
+int create_ipc_socket(std::array<int, 2> &ipc_fd) {
+  int rv;
+
+  rv = pipe(ipc_fd.data());
+  if (rv == -1) {
+    auto error = errno;
+    LOG(WARN) << "Failed to create pipe to communicate worker process: "
+              << strerror(error);
+    return -1;
+  }
+
+  for (int i = 0; i < 2; ++i) {
+    auto fd = ipc_fd[i];
+    util::make_socket_nonblocking(fd);
+    util::make_socket_closeonexec(fd);
+  }
+
+  return 0;
+}
+} // namespace
+
+namespace {
+// Creates worker process, and returns PID of worker process.  On
+// success, file descriptor for IPC (send only) is assigned to
+// |main_ipc_fd|.
+pid_t fork_worker_process(int &main_ipc_fd) {
   int rv;
   sigset_t oldset;
 
+  std::array<int, 2> ipc_fd;
+
+  rv = create_ipc_socket(ipc_fd);
+  if (rv != 0) {
+    return -1;
+  }
+
   rv = shrpx_signal_block_all(&oldset);
   if (rv != 0) {
     auto error = errno;
     LOG(ERROR) << "Blocking all signals failed: " << strerror(error);
 
+    close(ipc_fd[0]);
+    close(ipc_fd[1]);
+
     return -1;
   }
 
@@ -878,6 +1096,10 @@ pid_t fork_worker_process(SignalServer *ssv) {
   if (pid == 0) {
     ev_loop_fork(EV_DEFAULT);
 
+    // Remove all SignalServers to stop any registered watcher on
+    // default loop.
+    signal_server_remove_all();
+
     shrpx_signal_set_worker_proc_ign_handler();
 
     rv = shrpx_signal_unblock_all();
@@ -888,8 +1110,8 @@ pid_t fork_worker_process(SignalServer *ssv) {
       _Exit(EXIT_FAILURE);
     }
 
-    close(ssv->ipc_fd[1]);
+    close(ipc_fd[1]);
-    WorkerProcessConfig wpconf{ssv->ipc_fd[0]};
+    WorkerProcessConfig wpconf{ipc_fd[0]};
     rv = worker_process_event_loop(&wpconf);
     if (rv != 0) {
       LOG(FATAL) << "Worker process returned error";
@@ -918,10 +1140,15 @@ pid_t fork_worker_process(SignalServer *ssv) {
   }
 
   if (pid == -1) {
+    close(ipc_fd[0]);
+    close(ipc_fd[1]);
+
     return -1;
   }
 
-  close(ssv->ipc_fd[0]);
+  close(ipc_fd[0]);
+
+  main_ipc_fd = ipc_fd[1];
 
   LOG(NOTICE) << "Worker process [" << pid << "] spawned";
 
@@ -931,8 +1158,6 @@ pid_t fork_worker_process(SignalServer *ssv) {
 
 namespace {
 int event_loop() {
-  int rv;
-
   shrpx_signal_set_master_proc_ign_handler();
 
   if (get_config()->daemon) {
@@ -950,55 +1175,27 @@ int event_loop() {
     redirect_stderr_to_errorlog();
   }
 
-  SignalServer ssv;
-
-  rv = pipe(ssv.ipc_fd.data());
-  if (rv == -1) {
-    auto error = errno;
-    LOG(WARN) << "Failed to create pipe to communicate worker process: "
-              << strerror(error);
-    return -1;
-  }
-
-  for (int i = 0; i < 2; ++i) {
-    auto fd = ssv.ipc_fd[i];
-    util::make_socket_nonblocking(fd);
-    util::make_socket_closeonexec(fd);
-  }
-
   auto iaddrs = get_inherited_addr_from_env();
 
-  if (create_acceptor_socket(iaddrs) != 0) {
+  if (create_acceptor_socket(mod_config(), iaddrs) != 0) {
     return -1;
   }
 
-  closeUnusedInheritedAddr(iaddrs);
+  close_unused_inherited_addr(iaddrs);
 
   auto loop = ev_default_loop(get_config()->ev_loop_flags);
 
-  auto pid = fork_worker_process(&ssv);
+  int ipc_fd;
+
+  auto pid = fork_worker_process(ipc_fd);
 
   if (pid == -1) {
     return -1;
   }
 
-  ssv.worker_process_pid = pid;
+  signal_server_add(make_unique<SignalServer>(loop, pid, ipc_fd));
 
-  constexpr auto signals = std::array<int, 3>{
+  mod_config()->last_worker_pid = pid;
-      {REOPEN_LOG_SIGNAL, EXEC_BINARY_SIGNAL, GRACEFUL_SHUTDOWN_SIGNAL}};
-  auto sigevs = std::array<ev_signal, signals.size()>();
-
-  for (size_t i = 0; i < signals.size(); ++i) {
-    auto sigev = &sigevs[i];
-    ev_signal_init(sigev, signal_cb, signals[i]);
-    sigev->data = &ssv;
-    ev_signal_start(loop, sigev);
-  }
-
-  ev_child worker_process_childev;
-  ev_child_init(&worker_process_childev, worker_process_child_cb, pid, 0);
-  worker_process_childev.data = nullptr;
-  ev_child_start(loop, &worker_process_childev);
 
   // Write PID file when we are ready to accept connection from peer.
   // This makes easier to write restart script for nghttpx.  Because
@@ -1043,18 +1240,19 @@ constexpr auto DEFAULT_ACCESSLOG_FORMAT = StringRef::from_lit(
 } // namespace
 
 namespace {
-void fill_default_config() {
+void fill_default_config(Config *config) {
-  *mod_config() = {};
+  *config = {};
 
-  mod_config()->num_worker = 1;
+  config->num_worker = 1;
-  mod_config()->conf_path = "/etc/nghttpx/nghttpx.conf";
+  config->conf_path = "/etc/nghttpx/nghttpx.conf";
-  mod_config()->pid = getpid();
+  config->pid = getpid();
+  config->last_worker_pid = -1;
 
   if (ev_supported_backends() & ~ev_recommended_backends() & EVBACKEND_KQUEUE) {
-    mod_config()->ev_loop_flags = ev_recommended_backends() | EVBACKEND_KQUEUE;
+    config->ev_loop_flags = ev_recommended_backends() | EVBACKEND_KQUEUE;
   }
 
-  auto &tlsconf = mod_config()->tls;
+  auto &tlsconf = config->tls;
   {
     auto &ticketconf = tlsconf.ticket;
     {
@@ -1089,7 +1287,7 @@ void fill_default_config() {
 
   tlsconf.session_timeout = std::chrono::hours(12);
 
-  auto &httpconf = mod_config()->http;
+  auto &httpconf = config->http;
   httpconf.server_name =
       StringRef::from_lit("nghttpx nghttp2/" NGHTTP2_VERSION);
   httpconf.no_host_rewrite = true;
@@ -1098,7 +1296,7 @@ void fill_default_config() {
   httpconf.response_header_field_buffer = 64_k;
   httpconf.max_response_header_fields = 500;
 
-  auto &http2conf = mod_config()->http2;
+  auto &http2conf = config->http2;
   {
     auto &upstreamconf = http2conf.upstream;
 
@@ -1143,7 +1341,7 @@ void fill_default_config() {
     nghttp2_option_set_peer_max_concurrent_streams(downstreamconf.option, 100);
   }
 
-  auto &loggingconf = mod_config()->logging;
+  auto &loggingconf = config->logging;
   {
     auto &accessconf = loggingconf.access;
     accessconf.format = parse_log_format(DEFAULT_ACCESSLOG_FORMAT);
@@ -1154,7 +1352,7 @@ void fill_default_config() {
 
   loggingconf.syslog_facility = LOG_DAEMON;
 
-  auto &connconf = mod_config()->conn;
+  auto &connconf = config->conn;
   {
     auto &listenerconf = connconf.listener;
     {
@@ -1198,7 +1396,7 @@ void fill_default_config() {
     downstreamconf.family = AF_UNSPEC;
   }
 
-  auto &apiconf = mod_config()->api;
+  auto &apiconf = config->api;
   apiconf.max_request_body = 16_k;
 }
 
@@ -2038,23 +2236,17 @@ Misc:
 } // namespace
 
 namespace {
-void process_options(int argc, char **argv,
+int process_options(Config *config,
                     std::vector<std::pair<StringRef, StringRef>> &cmdcfgs) {
-  if (conf_exists(get_config()->conf_path.c_str())) {
+  if (conf_exists(config->conf_path.c_str())) {
     std::set<StringRef> include_set;
-    if (load_config(get_config()->conf_path.c_str(), include_set) == -1) {
+    if (load_config(config, config->conf_path.c_str(), include_set) == -1) {
-      LOG(FATAL) << "Failed to load configuration from "
+      LOG(FATAL) << "Failed to load configuration from " << config->conf_path;
-                 << get_config()->conf_path;
+      return -1;
-      exit(EXIT_FAILURE);
     }
     assert(include_set.empty());
   }
 
-  if (argc - optind >= 2) {
-    cmdcfgs.emplace_back(SHRPX_OPT_PRIVATE_KEY_FILE, StringRef{argv[optind++]});
-    cmdcfgs.emplace_back(SHRPX_OPT_CERTIFICATE_FILE, StringRef{argv[optind++]});
-  }
-
   // Reopen log files using configurations in file
   reopen_log_files();
 
@@ -2062,16 +2254,16 @@ void process_options(int argc, char **argv,
     std::set<StringRef> include_set;
 
     for (auto &p : cmdcfgs) {
-      if (parse_config(mod_config(), p.first, p.second, include_set) == -1) {
+      if (parse_config(config, p.first, p.second, include_set) == -1) {
         LOG(FATAL) << "Failed to parse command-line argument.";
-        exit(EXIT_FAILURE);
+        return -1;
       }
     }
 
     assert(include_set.empty());
   }
 
-  auto &loggingconf = get_config()->logging;
+  auto &loggingconf = config->logging;
 
   if (loggingconf.access.syslog || loggingconf.error.syslog) {
     openlog("nghttpx", LOG_NDELAY | LOG_NOWAIT | LOG_PID,
@@ -2080,29 +2272,27 @@ void process_options(int argc, char **argv,
 
   if (reopen_log_files() != 0) {
     LOG(FATAL) << "Failed to open log file";
-    exit(EXIT_FAILURE);
+    return -1;
   }
 
   redirect_stderr_to_errorlog();
 
-  if (get_config()->uid != 0) {
+  if (config->uid != 0) {
     if (log_config()->accesslog_fd != -1 &&
-        fchown(log_config()->accesslog_fd, get_config()->uid,
+        fchown(log_config()->accesslog_fd, config->uid, config->gid) == -1) {
-               get_config()->gid) == -1) {
       auto error = errno;
       LOG(WARN) << "Changing owner of access log file failed: "
                 << strerror(error);
     }
     if (log_config()->errorlog_fd != -1 &&
-        fchown(log_config()->errorlog_fd, get_config()->uid,
+        fchown(log_config()->errorlog_fd, config->uid, config->gid) == -1) {
-               get_config()->gid) == -1) {
       auto error = errno;
       LOG(WARN) << "Changing owner of error log file failed: "
                 << strerror(error);
     }
   }
 
-  auto &http2conf = mod_config()->http2;
+  auto &http2conf = config->http2;
   {
     auto &dumpconf = http2conf.upstream.debug.dump;
 
@@ -2113,12 +2303,12 @@ void process_options(int argc, char **argv,
       if (f == nullptr) {
         LOG(FATAL) << "Failed to open http2 upstream request header file: "
                    << path;
-        exit(EXIT_FAILURE);
+        return -1;
       }
 
       dumpconf.request_header = f;
 
-      if (get_config()->uid != 0) {
+      if (config->uid != 0) {
         if (chown_to_running_user(path) == -1) {
           auto error = errno;
           LOG(WARN) << "Changing owner of http2 upstream request header file "
@@ -2134,12 +2324,12 @@ void process_options(int argc, char **argv,
       if (f == nullptr) {
         LOG(FATAL) << "Failed to open http2 upstream response header file: "
                    << path;
-        exit(EXIT_FAILURE);
+        return -1;
       }
 
       dumpconf.response_header = f;
 
-      if (get_config()->uid != 0) {
+      if (config->uid != 0) {
         if (chown_to_running_user(path) == -1) {
           auto error = errno;
           LOG(WARN) << "Changing owner of http2 upstream response header file"
@@ -2149,7 +2339,7 @@ void process_options(int argc, char **argv,
     }
   }
 
-  auto &tlsconf = mod_config()->tls;
+  auto &tlsconf = config->tls;
 
   if (tlsconf.npn_list.empty()) {
     tlsconf.npn_list = util::parse_config_str_list(DEFAULT_NPN_LIST);
@@ -2165,8 +2355,8 @@ void process_options(int argc, char **argv,
 
   tlsconf.bio_method = create_bio_method();
 
-  auto &listenerconf = mod_config()->conn.listener;
+  auto &listenerconf = config->conn.listener;
-  auto &upstreamconf = mod_config()->conn.upstream;
+  auto &upstreamconf = config->conn.upstream;
 
   if (listenerconf.addrs.empty()) {
     UpstreamAddr addr{};
@@ -2187,7 +2377,7 @@ void process_options(int argc, char **argv,
       (tlsconf.private_key_file.empty() || tlsconf.cert_file.empty())) {
     print_usage(std::cerr);
     LOG(FATAL) << "Too few arguments";
-    exit(EXIT_FAILURE);
+    return -1;
   }
 
   if (ssl::upstream_tls_enabled() && !tlsconf.ocsp.disabled) {
@@ -2200,18 +2390,18 @@ void process_options(int argc, char **argv,
     }
   }
 
-  if (configure_downstream_group(mod_config(), get_config()->http2_proxy, false,
+  if (configure_downstream_group(config, config->http2_proxy, false, tlsconf) !=
-                                 tlsconf) != 0) {
+      0) {
-    exit(EXIT_FAILURE);
+    return -1;
   }
 
-  auto &proxy = mod_config()->downstream_http_proxy;
+  auto &proxy = config->downstream_http_proxy;
   if (!proxy.host.empty()) {
     auto hostport = util::make_hostport(StringRef{proxy.host}, proxy.port);
     if (resolve_hostname(&proxy.addr, proxy.host.c_str(), proxy.port,
                          AF_UNSPEC) == -1) {
       LOG(FATAL) << "Resolving backend HTTP proxy address failed: " << hostport;
-      exit(EXIT_FAILURE);
+      return -1;
     }
     LOG(NOTICE) << "Backend HTTP proxy address: " << hostport << " -> "
                 << util::to_numeric_addr(&proxy.addr);
@@ -2227,7 +2417,7 @@ void process_options(int argc, char **argv,
         LOG(FATAL)
             << "Resolving memcached address for TLS session cache failed: "
             << hostport;
-        exit(EXIT_FAILURE);
+        return -1;
       }
       LOG(NOTICE) << "Memcached address for TLS session cache: " << hostport
                   << " -> " << util::to_numeric_addr(&memcachedconf.addr);
@@ -2247,7 +2437,7 @@ void process_options(int argc, char **argv,
                            memcachedconf.port, memcachedconf.family) == -1) {
         LOG(FATAL) << "Resolving memcached address for TLS ticket key failed: "
                    << hostport;
-        exit(EXIT_FAILURE);
+        return -1;
       }
       LOG(NOTICE) << "Memcached address for TLS ticket key: " << hostport
                   << " -> " << util::to_numeric_addr(&memcachedconf.addr);
@@ -2258,27 +2448,26 @@ void process_options(int argc, char **argv,
     }
   }
 
-  if (get_config()->rlimit_nofile) {
+  if (config->rlimit_nofile) {
-    struct rlimit lim = {static_cast<rlim_t>(get_config()->rlimit_nofile),
+    struct rlimit lim = {static_cast<rlim_t>(config->rlimit_nofile),
-                         static_cast<rlim_t>(get_config()->rlimit_nofile)};
+                         static_cast<rlim_t>(config->rlimit_nofile)};
     if (setrlimit(RLIMIT_NOFILE, &lim) != 0) {
       auto error = errno;
       LOG(WARN) << "Setting rlimit-nofile failed: " << strerror(error);
     }
   }
 
-  auto &fwdconf = mod_config()->http.forwarded;
+  auto &fwdconf = config->http.forwarded;
 
   if (fwdconf.by_node_type == FORWARDED_NODE_OBFUSCATED &&
       fwdconf.by_obfuscated.empty()) {
-    std::random_device rd;
     std::mt19937 gen(rd());
     auto &dst = fwdconf.by_obfuscated;
     dst = "_";
     dst += util::random_alpha_digit(gen, SHRPX_OBFUSCATED_NODE_LENGTH);
   }
 
-  if (get_config()->http2.upstream.debug.frame_debug) {
+  if (config->http2.upstream.debug.frame_debug) {
     // To make it sync to logging
     set_output(stderr);
     if (isatty(fileno(stdout))) {
@@ -2287,13 +2476,88 @@ void process_options(int argc, char **argv,
     reset_timer();
   }
 
-  mod_config()->http2.upstream.callbacks = create_http2_upstream_callbacks();
+  config->http2.upstream.callbacks = create_http2_upstream_callbacks();
-  mod_config()->http2.downstream.callbacks =
+  config->http2.downstream.callbacks = create_http2_downstream_callbacks();
-      create_http2_downstream_callbacks();
+
+  return 0;
+}
+} // namespace
+
+namespace {
+void reload_config(SignalServer *ssv) {
+  int rv;
+
+  LOG(NOTICE) << "Reloading configuration";
+
+  auto cur_config = get_config();
+  auto new_config = make_unique<Config>();
+
+  fill_default_config(new_config.get());
+
+  new_config->conf_path = cur_config->conf_path;
+  new_config->argc = cur_config->argc;
+  new_config->argv = cur_config->argv;
+  new_config->original_argv = cur_config->original_argv;
+  new_config->cwd = cur_config->cwd;
+
+  rv = process_options(new_config.get(), cmdcfgs);
+  if (rv != 0) {
+    LOG(ERROR) << "Failed to process new configuration";
+    return;
+  }
+
+  // daemon option is ignored here.
+
+  auto iaddrs = get_inherited_addr_from_config(cur_config);
+
+  if (create_acceptor_socket(new_config.get(), iaddrs) != 0) {
+    return;
+  }
+
+  // TODO may leak socket if we get error later in this function
+
+  // TODO loop is reused, and new_config->ev_loop_flags gets ignored
+
+  auto loop = ssv->loop;
+
+  int ipc_fd;
+
+  auto pid = fork_worker_process(ipc_fd);
+
+  if (pid == -1) {
+    LOG(ERROR) << "Failed to process new configuration";
+    return;
+  }
+
+  close_unused_inherited_addr(iaddrs);
+
+  // Send last worker process a graceful shutdown notice
+  auto &last_ssv = signal_servers.back();
+  ipc_send(last_ssv.get(), SHRPX_IPC_GRACEFUL_SHUTDOWN);
+  // We no longer use signals for this worker.
+  last_ssv->shutdown_signal_watchers();
+
+  signal_server_add(make_unique<SignalServer>(loop, pid, ipc_fd));
+
+  new_config->last_worker_pid = pid;
+
+  auto old_config = replace_config(new_config.release());
+
+  assert(cur_config == old_config);
+
+  delete_config(old_config);
+
+  // Now we use new configuration
+
+  if (!get_config()->pid_file.empty()) {
+    save_pid();
+  }
 }
 } // namespace
 
 int main(int argc, char **argv) {
+  int rv;
+
   nghttp2::ssl::libssl_init();
 
 #ifndef NOTHREADS
@@ -2302,7 +2566,7 @@ int main(int argc, char **argv) {
 
   Log::set_severity_level(NOTICE);
   create_config();
-  fill_default_config();
+  fill_default_config(mod_config());
 
   // make copy of stderr
   util::store_original_fds();
@@ -2333,7 +2597,6 @@ int main(int argc, char **argv) {
     exit(EXIT_FAILURE);
   }
 
-  std::vector<std::pair<StringRef, StringRef>> cmdcfgs;
   while (1) {
     static int flag = 0;
     static option long_options[] = {
@@ -3121,7 +3384,15 @@ int main(int argc, char **argv) {
     }
   }
 
-  process_options(argc, argv, cmdcfgs);
+  if (argc - optind >= 2) {
+    cmdcfgs.emplace_back(SHRPX_OPT_PRIVATE_KEY_FILE, StringRef{argv[optind++]});
+    cmdcfgs.emplace_back(SHRPX_OPT_CERTIFICATE_FILE, StringRef{argv[optind++]});
+  }
+
+  rv = process_options(mod_config(), cmdcfgs);
+  if (rv != 0) {
+    return -1;
+  }
 
   if (event_loop() != 0) {
     return -1;

src/shrpx_config.cc

@@ -69,8 +69,44 @@ const Config *get_config() { return config; }
 
 Config *mod_config() { return config; }
 
+Config *replace_config(Config *new_config) {
+  std::swap(config, new_config);
+  return new_config;
+}
+
 void create_config() { config = new Config(); }
 
+void delete_config(Config *config) {
+  if (config == nullptr) {
+    return;
+  }
+
+  auto &http2conf = config->http2;
+
+  auto &upstreamconf = http2conf.upstream;
+
+  nghttp2_option_del(upstreamconf.option);
+  nghttp2_option_del(upstreamconf.alt_mode_option);
+  nghttp2_session_callbacks_del(upstreamconf.callbacks);
+
+  auto &downstreamconf = http2conf.downstream;
+
+  nghttp2_option_del(downstreamconf.option);
+  nghttp2_session_callbacks_del(downstreamconf.callbacks);
+
+  auto &dumpconf = http2conf.upstream.debug.dump;
+
+  if (dumpconf.request_header) {
+    fclose(dumpconf.request_header);
+  }
+
+  if (dumpconf.response_header) {
+    fclose(dumpconf.response_header);
+  }
+
+  delete config;
+}
+
 TicketKeys::~TicketKeys() {
   /* Erase keys from memory */
   for (auto &key : keys) {
@@ -2398,7 +2434,7 @@ int parse_config(Config *config, int optid, const StringRef &opt,
     }
 
     included_set.insert(optarg);
-    auto rv = load_config(optarg.c_str(), included_set);
+    auto rv = load_config(config, optarg.c_str(), included_set);
     included_set.erase(optarg);
 
     if (rv != 0) {
@@ -2648,7 +2684,8 @@ int parse_config(Config *config, int optid, const StringRef &opt,
   return -1;
 }
 
-int load_config(const char *filename, std::set<StringRef> &include_set) {
+int load_config(Config *config, const char *filename,
+                std::set<StringRef> &include_set) {
   std::ifstream in(filename, std::ios::binary);
   if (!in) {
     LOG(ERROR) << "Could not open config file " << filename;
@@ -2669,7 +2706,7 @@ int load_config(const char *filename, std::set<StringRef> &include_set) {
     }
     *eq = '\0';
 
-    if (parse_config(mod_config(), StringRef{std::begin(line), eq},
+    if (parse_config(config, StringRef{std::begin(line), eq},
                      StringRef{eq + 1, std::end(line)}, include_set) != 0) {
       return -1;
     }

src/shrpx_config.h

@@ -727,6 +727,9 @@ struct Config {
   uid_t uid;
   gid_t gid;
   pid_t pid;
+  // With reloading feature, we may have multiple worker PIDs at the
+  // given moment.  This field tracks the last worker PID.
+  pid_t last_worker_pid;
   bool verbose;
   bool daemon;
   bool http2_proxy;
@@ -736,7 +739,11 @@ struct Config {
 
 const Config *get_config();
 Config *mod_config();
+// Replaces the current config with given |new_config|.  The old config is
+// returned.
+Config *replace_config(Config *new_config);
 void create_config();
+void delete_config(Config *config);
 
 // generated by gennghttpxfun.py
 enum {
@@ -890,10 +897,11 @@ int parse_config(Config *config, const StringRef &opt, const StringRef &optarg,
 int parse_config(Config *config, int optid, const StringRef &opt,
                  const StringRef &optarg, std::set<StringRef> &included_set);
 
-// Loads configurations from |filename| and stores them in statically
+// Loads configurations from |filename| and stores them in |config|.
-// allocated Config object. This function returns 0 if it succeeds, or
+// This function returns 0 if it succeeds, or -1.  See parse_config()
-// -1.  See parse_config() for |include_set|.
+// for |include_set|.
-int load_config(const char *filename, std::set<StringRef> &include_set);
+int load_config(Config *config, const char *filename,
+                std::set<StringRef> &include_set);
 
 // Parses header field in |optarg|.  We expect header field is formed
 // like "NAME: VALUE".  We require that NAME is non empty string.  ":"

src/shrpx_signal.cc

@@ -114,8 +114,9 @@ constexpr auto master_proc_ign_signals = std::array<int, 1>{{SIGPIPE}};
 } // namespace
 
 namespace {
-constexpr auto worker_proc_ign_signals = std::array<int, 4>{
+constexpr auto worker_proc_ign_signals =
-    {REOPEN_LOG_SIGNAL, EXEC_BINARY_SIGNAL, GRACEFUL_SHUTDOWN_SIGNAL, SIGPIPE}};
+    std::array<int, 5>{{REOPEN_LOG_SIGNAL, EXEC_BINARY_SIGNAL,
+                        GRACEFUL_SHUTDOWN_SIGNAL, RELOAD_SIGNAL, SIGPIPE}};
 } // namespace
 
 void shrpx_signal_set_master_proc_ign_handler() {

src/shrpx_signal.h

@@ -34,6 +34,7 @@ namespace shrpx {
 constexpr int REOPEN_LOG_SIGNAL = SIGUSR1;
 constexpr int EXEC_BINARY_SIGNAL = SIGUSR2;
 constexpr int GRACEFUL_SHUTDOWN_SIGNAL = SIGQUIT;
+constexpr int RELOAD_SIGNAL = SIGHUP;
 
 // Blocks all signals.  The previous signal mask is stored into
 // |oldset| if it is not nullptr.  This function returns 0 if it