From ce1b11e3a061e8b977f033f31dc00110804e6c52 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 18 Sep 2015 01:16:49 +0900 Subject: [PATCH] nghttpx: Split monolithic one process into control and worker processes The control process handles signals, reads configuration, reads private keys, and bind port (which may be privileged one). It never drop privileges, so that it can execute new binary with the same privilege. It forks worker process. The worker process handles all incoming connections. It drops privilege. --- doc/nghttpx.h2r | 8 + src/Makefile.am | 2 + src/shrpx.cc | 903 ++++++++++++++---------------------- src/shrpx_config.h | 1 + src/shrpx_process.h | 43 ++ src/shrpx_worker.cc | 3 +- src/shrpx_worker_process.cc | 467 +++++++++++++++++++ src/shrpx_worker_process.h | 49 ++ 8 files changed, 909 insertions(+), 567 deletions(-) create mode 100644 src/shrpx_process.h create mode 100644 src/shrpx_worker_process.cc create mode 100644 src/shrpx_worker_process.h diff --git a/doc/nghttpx.h2r b/doc/nghttpx.h2r index c3e6dec5..ed97fad2 100644 --- a/doc/nghttpx.h2r +++ b/doc/nghttpx.h2r @@ -42,6 +42,14 @@ SIGUSR2 After new process comes up, sending SIGQUIT to the original process to perform hot swapping. +.. note:: + + nghttpx consists of 2 processes: one process for processing these + signals, and another one for processing requests. The former spawns + the latter. The former is called master process, and the latter is + called worker process. The above signal must be sent to the master + process. If the worker process receives one of them, it is ignored. + SERVER PUSH ----------- diff --git a/src/Makefile.am b/src/Makefile.am index 8ee76239..358e9d8d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -125,6 +125,8 @@ NGHTTPX_SRCS = \ shrpx_memcached_connection.cc shrpx_memcached_connection.h \ shrpx_memcached_request.h \ shrpx_memcached_result.h \ + shrpx_worker_process.cc shrpx_worker_process.h \ + shrpx_process.h \ buffer.h memchunk.h template.h if HAVE_SPDYLAY diff --git a/src/shrpx.cc b/src/shrpx.cc index 635e78e6..5675668b 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -48,7 +48,6 @@ #ifdef HAVE_SYSLOG_H #include #endif // HAVE_SYSLOG_H -#include #ifdef HAVE_LIMITS_H #include #endif // HAVE_LIMITS_H @@ -56,7 +55,6 @@ #include #endif // HAVE_SYS_TIME_H #include -#include #include #include @@ -69,22 +67,19 @@ #include #include #include -#include #include #include #include "shrpx_config.h" -#include "shrpx_connection_handler.h" #include "shrpx_ssl.h" #include "shrpx_log_config.h" #include "shrpx_worker.h" -#include "shrpx_accept_handler.h" #include "shrpx_http2_upstream.h" #include "shrpx_http2_session.h" -#include "shrpx_memcached_dispatcher.h" -#include "shrpx_memcached_request.h" +#include "shrpx_worker_process.h" +#include "shrpx_process.h" #include "util.h" #include "app_helper.h" #include "ssl.h" @@ -96,12 +91,6 @@ using namespace nghttp2; namespace shrpx { -namespace { -const int REOPEN_LOG_SIGNAL = SIGUSR1; -const int EXEC_BINARY_SIGNAL = SIGUSR2; -const int GRACEFUL_SHUTDOWN_SIGNAL = SIGQUIT; -} // namespace - // Environment variables to tell new binary the listening socket's // file descriptors. They are not close-on-exec. #define ENV_LISTENER4_FD "NGHTTPX_LISTENER4_FD" @@ -118,6 +107,34 @@ const int GRACEFUL_SHUTDOWN_SIGNAL = SIGQUIT; // path. #define ENV_UNIX_PATH "NGHTTP2_UNIX_PATH" +struct SignalServer { + SignalServer() + : ipc_fd{{-1, -1}}, server_fd(-1), server_fd6(-1), + worker_process_pid(-1) {} + ~SignalServer() { + if (server_fd6 != -1) { + close(server_fd6); + } + if (server_fd != -1) { + close(server_fd); + } + if (ipc_fd[0] != -1) { + close(ipc_fd[0]); + } + if (ipc_fd[1] != -1) { + shutdown(ipc_fd[1], SHUT_WR); + close(ipc_fd[1]); + } + } + + std::array ipc_fd; + // server socket, either IPv4 or UNIX domain + int server_fd; + // server socket IPv6 + int server_fd6; + pid_t worker_process_pid; +}; + namespace { int resolve_hostname(Address *addr, const char *hostname, uint16_t port, int family) { @@ -140,6 +157,8 @@ int resolve_hostname(Address *addr, const char *hostname, uint16_t port, return -1; } + auto res_d = defer(freeaddrinfo, res); + char host[NI_MAXHOST]; rv = getnameinfo(res->ai_addr, res->ai_addrlen, host, sizeof(host), 0, 0, NI_NUMERICHOST); @@ -147,8 +166,6 @@ int resolve_hostname(Address *addr, const char *hostname, uint16_t port, LOG(FATAL) << "Address resolution for " << hostname << " failed: " << gai_strerror(rv); - freeaddrinfo(res); - return -1; } @@ -159,27 +176,204 @@ int resolve_hostname(Address *addr, const char *hostname, uint16_t port, memcpy(&addr->su, res->ai_addr, res->ai_addrlen); addr->len = res->ai_addrlen; - freeaddrinfo(res); + return 0; } } // namespace namespace { -void close_env_fd(std::initializer_list envnames) { - for (auto envname : envnames) { - auto envfd = getenv(envname); - if (!envfd) { - continue; +void save_pid() { + std::ofstream out(get_config()->pid_file.get(), std::ios::binary); + out << get_config()->pid << "\n"; + out.close(); + if (!out) { + LOG(ERROR) << "Could not save PID to file " << get_config()->pid_file.get(); + exit(EXIT_FAILURE); + } + + if (get_config()->uid != 0) { + if (chown(get_config()->pid_file.get(), get_config()->uid, + get_config()->gid) == -1) { + auto error = errno; + LOG(WARN) << "Changing owner of pid file " << get_config()->pid_file.get() + << " failed: " << strerror(error); } - auto fd = strtol(envfd, nullptr, 10); - close(fd); } } } // namespace namespace { -std::unique_ptr -create_unix_domain_acceptor(ConnectionHandler *handler) { +void exec_binary(SignalServer *ssv) { + LOG(NOTICE) << "Executing new binary"; + + auto pid = fork(); + + if (pid == -1) { + auto error = errno; + LOG(ERROR) << "fork() failed errno=" << error; + return; + } + + if (pid != 0) { + return; + } + + auto exec_path = util::get_exec_path(get_config()->argc, get_config()->argv, + get_config()->cwd); + + if (!exec_path) { + LOG(ERROR) << "Could not resolve the executable path"; + return; + } + + auto argv = make_unique(get_config()->argc + 1); + + argv[0] = exec_path; + for (int i = 1; i < get_config()->argc; ++i) { + argv[i] = strdup(get_config()->argv[i]); + } + argv[get_config()->argc] = nullptr; + + size_t envlen = 0; + for (char **p = environ; *p; ++p, ++envlen) + ; + // 3 for missing (fd4, fd6 and port) or (unix fd and unix path) + auto envp = make_unique(envlen + 3 + 1); + size_t envidx = 0; + + if (get_config()->host_unix) { + std::string fd = ENV_UNIX_FD "="; + fd += util::utos(ssv->server_fd); + envp[envidx++] = strdup(fd.c_str()); + + std::string path = ENV_UNIX_PATH "="; + path += get_config()->host.get(); + envp[envidx++] = strdup(path.c_str()); + } else { + if (ssv->server_fd) { + std::string fd4 = ENV_LISTENER4_FD "="; + fd4 += util::utos(ssv->server_fd); + envp[envidx++] = strdup(fd4.c_str()); + } + + if (ssv->server_fd6) { + std::string fd6 = ENV_LISTENER6_FD "="; + fd6 += util::utos(ssv->server_fd6); + envp[envidx++] = strdup(fd6.c_str()); + } + + std::string port = ENV_PORT "="; + port += util::utos(get_config()->port); + envp[envidx++] = strdup(port.c_str()); + } + + for (size_t i = 0; i < envlen; ++i) { + if (util::startsWith(environ[i], ENV_LISTENER4_FD) || + util::startsWith(environ[i], ENV_LISTENER6_FD) || + util::startsWith(environ[i], ENV_PORT) || + util::startsWith(environ[i], ENV_UNIX_FD) || + util::startsWith(environ[i], ENV_UNIX_PATH)) { + continue; + } + + envp[envidx++] = environ[i]; + } + + envp[envidx++] = nullptr; + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "cmdline"; + for (int i = 0; argv[i]; ++i) { + LOG(INFO) << i << ": " << argv[i]; + } + LOG(INFO) << "environ"; + for (int i = 0; envp[i]; ++i) { + LOG(INFO) << i << ": " << envp[i]; + } + } + + // restores original stderr + util::restore_original_fds(); + + if (execve(argv[0], argv.get(), envp.get()) == -1) { + auto error = errno; + LOG(ERROR) << "execve failed: errno=" << error; + _Exit(EXIT_FAILURE); + } +} +} // namespace + +namespace { +void ipc_send(SignalServer *ssv, uint8_t ipc_event) { + ssize_t nwrite; + while ((nwrite = write(ssv->ipc_fd[1], &ipc_event, 1)) == -1 && + errno == EINTR) + ; + + if (nwrite < 0) { + auto error = errno; + LOG(ERROR) << "Could not send IPC event to worker process: " + << strerror(error); + return; + } + + if (nwrite == 0) { + LOG(ERROR) << "Could not send IPC event due to pipe overflow"; + return; + } +} +} // namespace + +namespace { +void reopen_log(SignalServer *ssv) { + LOG(NOTICE) << "Reopening log files: master process"; + + (void)reopen_log_files(); + redirect_stderr_to_errorlog(); + ipc_send(ssv, SHRPX_IPC_REOPEN_LOG); +} +} // namespace + +namespace { +void signal_cb(struct ev_loop *loop, ev_signal *w, int revents) { + auto ssv = static_cast(w->data); + if (ssv->worker_process_pid == -1) { + ev_break(loop); + return; + } + + switch (w->signum) { + case REOPEN_LOG_SIGNAL: + reopen_log(ssv); + return; + case EXEC_BINARY_SIGNAL: + exec_binary(ssv); + return; + case GRACEFUL_SHUTDOWN_SIGNAL: + ipc_send(ssv, SHRPX_IPC_GRACEFUL_SHUTDOWN); + return; + default: + kill(ssv->worker_process_pid, w->signum); + ev_break(loop); + return; + } +} +} // namespace + +namespace { +void worker_process_child_cb(struct ev_loop *loop, ev_child *w, int revents) { + LOG(NOTICE) << "Worker process (" << w->rpid << ") exited " + << (WIFEXITED(w->rstatus) ? "normally" : "abnormally") + << " with status " << std::hex << w->rstatus << std::oct + << "; exit status " << WEXITSTATUS(w->rstatus) << "; signal " + << (WIFSIGNALED(w->rstatus) ? WTERMSIG(w->rstatus) : 0); + + ev_break(loop); +} +} // namespace + +namespace { +int create_unix_domain_server_socket() { auto path = get_config()->host.get(); auto pathlen = strlen(path); { @@ -191,7 +385,7 @@ create_unix_domain_acceptor(ConnectionHandler *handler) { if (util::streq(envpath, path)) { LOG(NOTICE) << "Listening on UNIX domain socket " << path; - return make_unique(fd, handler); + return fd; } LOG(WARN) << "UNIX domain socket path was changed between old binary (" @@ -203,12 +397,12 @@ create_unix_domain_acceptor(ConnectionHandler *handler) { #ifdef SOCK_NONBLOCK auto fd = socket(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, 0); if (fd == -1) { - return nullptr; + return -1; } #else // !SOCK_NONBLOCK auto fd = socket(AF_UNIX, SOCK_STREAM, 0); if (fd == -1) { - return nullptr; + return -1; } util::make_socket_nonblocking(fd); #endif // !SOCK_NONBLOCK @@ -216,7 +410,7 @@ create_unix_domain_acceptor(ConnectionHandler *handler) { if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val, static_cast(sizeof(val))) == -1) { close(fd); - return nullptr; + return -1; } sockaddr_union addr; @@ -225,7 +419,7 @@ create_unix_domain_acceptor(ConnectionHandler *handler) { LOG(FATAL) << "UNIX domain socket path " << path << " is too long > " << sizeof(addr.un.sun_path); close(fd); - return nullptr; + return -1; } // copy path including terminal NULL std::copy_n(path, pathlen + 1, addr.un.sun_path); @@ -237,25 +431,24 @@ create_unix_domain_acceptor(ConnectionHandler *handler) { auto error = errno; LOG(FATAL) << "Failed to bind UNIX domain socket, error=" << error; close(fd); - return nullptr; + return -1; } if (listen(fd, get_config()->backlog) != 0) { auto error = errno; LOG(FATAL) << "Failed to listen to UNIX domain socket, error=" << error; close(fd); - return nullptr; + return -1; } LOG(NOTICE) << "Listening on UNIX domain socket " << path; - return make_unique(fd, handler); + return fd; } } // namespace namespace { -std::unique_ptr create_acceptor(ConnectionHandler *handler, - int family) { +int create_tcp_server_socket(int family) { { auto envfd = getenv(family == AF_INET ? ENV_LISTENER4_FD : ENV_LISTENER6_FD); @@ -271,7 +464,7 @@ std::unique_ptr create_acceptor(ConnectionHandler *handler, if (port == get_config()->port) { LOG(NOTICE) << "Listening on port " << get_config()->port; - return make_unique(fd, handler); + return fd; } LOG(WARN) << "Port was changed between old binary (" << port @@ -304,8 +497,11 @@ std::unique_ptr create_acceptor(ConnectionHandler *handler, << " address for " << get_config()->host.get() << ": " << gai_strerror(rv); } - return nullptr; + return -1; } + + auto res_d = defer(freeaddrinfo, res); + for (rp = res; rp; rp = rp->ai_next) { #ifdef SOCK_NONBLOCK fd = @@ -381,429 +577,27 @@ std::unique_ptr create_acceptor(ConnectionHandler *handler, LOG(WARN) << "Listening " << (family == AF_INET ? "IPv4" : "IPv6") << " socket failed"; - freeaddrinfo(res); - - return nullptr; + return -1; } char host[NI_MAXHOST]; rv = getnameinfo(rp->ai_addr, rp->ai_addrlen, host, sizeof(host), nullptr, 0, NI_NUMERICHOST); - freeaddrinfo(res); - if (rv != 0) { LOG(WARN) << gai_strerror(rv); close(fd); - return nullptr; + return -1; } LOG(NOTICE) << "Listening on " << host << ", port " << get_config()->port; - return make_unique(fd, handler); + return fd; } } // namespace -namespace { -void drop_privileges() { - if (getuid() == 0 && get_config()->uid != 0) { - if (initgroups(get_config()->user.get(), get_config()->gid) != 0) { - auto error = errno; - LOG(FATAL) << "Could not change supplementary groups: " - << strerror(error); - exit(EXIT_FAILURE); - } - if (setgid(get_config()->gid) != 0) { - auto error = errno; - LOG(FATAL) << "Could not change gid: " << strerror(error); - exit(EXIT_FAILURE); - } - if (setuid(get_config()->uid) != 0) { - auto error = errno; - LOG(FATAL) << "Could not change uid: " << strerror(error); - exit(EXIT_FAILURE); - } - if (setuid(0) != -1) { - LOG(FATAL) << "Still have root privileges?"; - exit(EXIT_FAILURE); - } - } -} -} // namespace - -namespace { -void save_pid() { - std::ofstream out(get_config()->pid_file.get(), std::ios::binary); - out << get_config()->pid << "\n"; - out.close(); - if (!out) { - LOG(ERROR) << "Could not save PID to file " << get_config()->pid_file.get(); - exit(EXIT_FAILURE); - } - - if (get_config()->uid != 0) { - if (chown(get_config()->pid_file.get(), get_config()->uid, - get_config()->gid) == -1) { - auto error = errno; - LOG(WARN) << "Changing owner of pid file " << get_config()->pid_file.get() - << " failed: " << strerror(error); - } - } -} -} // namespace - -namespace { -void reopen_log_signal_cb(struct ev_loop *loop, ev_signal *w, int revents) { - auto conn_handler = static_cast(w->data); - - LOG(NOTICE) << "Reopening log files: main"; - - (void)reopen_log_files(); - redirect_stderr_to_errorlog(); - - if (get_config()->num_worker > 1) { - conn_handler->worker_reopen_log_files(); - } -} -} // namespace - -namespace { -void exec_binary_signal_cb(struct ev_loop *loop, ev_signal *w, int revents) { - auto conn_handler = static_cast(w->data); - - LOG(NOTICE) << "Executing new binary"; - - auto pid = fork(); - - if (pid == -1) { - auto error = errno; - LOG(ERROR) << "fork() failed errno=" << error; - return; - } - - if (pid != 0) { - return; - } - - auto exec_path = util::get_exec_path(get_config()->argc, get_config()->argv, - get_config()->cwd); - - if (!exec_path) { - LOG(ERROR) << "Could not resolve the executable path"; - return; - } - - auto argv = make_unique(get_config()->argc + 1); - - argv[0] = exec_path; - for (int i = 1; i < get_config()->argc; ++i) { - argv[i] = strdup(get_config()->argv[i]); - } - argv[get_config()->argc] = nullptr; - - size_t envlen = 0; - for (char **p = environ; *p; ++p, ++envlen) - ; - // 3 for missing (fd4, fd6 and port) or (unix fd and unix path) - auto envp = make_unique(envlen + 3 + 1); - size_t envidx = 0; - - if (get_config()->host_unix) { - auto acceptor = conn_handler->get_acceptor(); - std::string fd = ENV_UNIX_FD "="; - fd += util::utos(acceptor->get_fd()); - envp[envidx++] = strdup(fd.c_str()); - - std::string path = ENV_UNIX_PATH "="; - path += get_config()->host.get(); - envp[envidx++] = strdup(path.c_str()); - } else { - auto acceptor4 = conn_handler->get_acceptor(); - if (acceptor4) { - std::string fd4 = ENV_LISTENER4_FD "="; - fd4 += util::utos(acceptor4->get_fd()); - envp[envidx++] = strdup(fd4.c_str()); - } - - auto acceptor6 = conn_handler->get_acceptor6(); - if (acceptor6) { - std::string fd6 = ENV_LISTENER6_FD "="; - fd6 += util::utos(acceptor6->get_fd()); - envp[envidx++] = strdup(fd6.c_str()); - } - - std::string port = ENV_PORT "="; - port += util::utos(get_config()->port); - envp[envidx++] = strdup(port.c_str()); - } - - for (size_t i = 0; i < envlen; ++i) { - if (util::startsWith(environ[i], ENV_LISTENER4_FD) || - util::startsWith(environ[i], ENV_LISTENER6_FD) || - util::startsWith(environ[i], ENV_PORT) || - util::startsWith(environ[i], ENV_UNIX_FD) || - util::startsWith(environ[i], ENV_UNIX_PATH)) { - continue; - } - - envp[envidx++] = environ[i]; - } - - envp[envidx++] = nullptr; - - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "cmdline"; - for (int i = 0; argv[i]; ++i) { - LOG(INFO) << i << ": " << argv[i]; - } - LOG(INFO) << "environ"; - for (int i = 0; envp[i]; ++i) { - LOG(INFO) << i << ": " << envp[i]; - } - } - - // restores original stderr - util::restore_original_fds(); - - if (execve(argv[0], argv.get(), envp.get()) == -1) { - auto error = errno; - LOG(ERROR) << "execve failed: errno=" << error; - _Exit(EXIT_FAILURE); - } -} -} // namespace - -namespace { -void graceful_shutdown_signal_cb(struct ev_loop *loop, ev_signal *w, - int revents) { - auto conn_handler = static_cast(w->data); - - if (conn_handler->get_graceful_shutdown()) { - return; - } - - LOG(NOTICE) << "Graceful shutdown signal received"; - - conn_handler->set_graceful_shutdown(true); - - conn_handler->disable_acceptor(); - - // After disabling accepting new connection, disptach incoming - // connection in backlog. - - conn_handler->accept_pending_connection(); - - conn_handler->graceful_shutdown_worker(); - - if (get_config()->num_worker == 1 && - conn_handler->get_single_worker()->get_worker_stat()->num_connections > - 0) { - return; - } - - // We have accepted all pending connections. Shutdown main event - // loop. - ev_break(loop); -} -} // namespace - -namespace { -int generate_ticket_key(TicketKey &ticket_key) { - ticket_key.cipher = get_config()->tls_ticket_key_cipher; - ticket_key.hmac = EVP_sha256(); - ticket_key.hmac_keylen = EVP_MD_size(ticket_key.hmac); - - assert(static_cast(EVP_CIPHER_key_length(ticket_key.cipher)) <= - ticket_key.data.enc_key.size()); - assert(ticket_key.hmac_keylen <= ticket_key.data.hmac_key.size()); - - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "enc_keylen=" << EVP_CIPHER_key_length(ticket_key.cipher) - << ", hmac_keylen=" << ticket_key.hmac_keylen; - } - - if (RAND_bytes(reinterpret_cast(&ticket_key.data), - sizeof(ticket_key.data)) == 0) { - return -1; - } - - return 0; -} -} // namespace - -namespace { -void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) { - auto conn_handler = static_cast(w->data); - const auto &old_ticket_keys = conn_handler->get_ticket_keys(); - - auto ticket_keys = std::make_shared(); - LOG(NOTICE) << "Renew new ticket keys"; - - // If old_ticket_keys is not empty, it should contain at least 2 - // keys: one for encryption, and last one for the next encryption - // key but decryption only. The keys in between are old keys and - // decryption only. The next key is provided to ensure to mitigate - // possible problem when one worker encrypt new key, but one worker, - // which did not take the that key yet, and cannot decrypt it. - // - // We keep keys for get_config()->tls_session_timeout seconds. The - // default is 12 hours. Thus the maximum ticket vector size is 12. - if (old_ticket_keys) { - auto &old_keys = old_ticket_keys->keys; - auto &new_keys = ticket_keys->keys; - - assert(!old_keys.empty()); - - auto max_tickets = - static_cast(std::chrono::duration_cast( - get_config()->tls_session_timeout).count()); - - new_keys.resize(std::min(max_tickets, old_keys.size() + 1)); - std::copy_n(std::begin(old_keys), new_keys.size() - 1, - std::begin(new_keys) + 1); - } else { - ticket_keys->keys.resize(1); - } - - auto &new_key = ticket_keys->keys[0]; - - if (generate_ticket_key(new_key) != 0) { - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "failed to generate ticket key"; - } - conn_handler->set_ticket_keys(nullptr); - conn_handler->set_ticket_keys_to_worker(nullptr); - return; - } - - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "ticket keys generation done"; - assert(ticket_keys->keys.size() >= 1); - LOG(INFO) << 0 << " enc+dec: " - << util::format_hex(ticket_keys->keys[0].data.name); - for (size_t i = 1; i < ticket_keys->keys.size(); ++i) { - auto &key = ticket_keys->keys[i]; - LOG(INFO) << i << " dec: " << util::format_hex(key.data.name); - } - } - - conn_handler->set_ticket_keys(ticket_keys); - conn_handler->set_ticket_keys_to_worker(ticket_keys); -} -} // namespace - -namespace { -void memcached_get_ticket_key_cb(struct ev_loop *loop, ev_timer *w, - int revents) { - auto conn_handler = static_cast(w->data); - auto dispatcher = conn_handler->get_tls_ticket_key_memcached_dispatcher(); - - auto req = make_unique(); - req->key = "nghttpx:tls-ticket-key"; - req->op = MEMCACHED_OP_GET; - req->cb = [conn_handler, dispatcher, w](MemcachedRequest *req, - MemcachedResult res) { - switch (res.status_code) { - case MEMCACHED_ERR_NO_ERROR: - break; - case MEMCACHED_ERR_EXT_NETWORK_ERROR: - conn_handler->on_tls_ticket_key_network_error(w); - return; - default: - conn_handler->on_tls_ticket_key_not_found(w); - return; - } - - // |version (4bytes)|len (2bytes)|key (variable length)|... - // (len, key) pairs are repeated as necessary. - - auto &value = res.value; - if (value.size() < 4) { - LOG(WARN) << "Memcached: tls ticket key value is too small: got " - << value.size(); - conn_handler->on_tls_ticket_key_not_found(w); - return; - } - auto p = value.data(); - auto version = util::get_uint32(p); - // Currently supported version is 1. - if (version != 1) { - LOG(WARN) << "Memcached: tls ticket key version: want 1, got " << version; - conn_handler->on_tls_ticket_key_not_found(w); - return; - } - - auto end = p + value.size(); - p += 4; - - size_t expectedlen; - size_t enc_keylen; - size_t hmac_keylen; - if (get_config()->tls_ticket_key_cipher == EVP_aes_128_cbc()) { - expectedlen = 48; - enc_keylen = 16; - hmac_keylen = 16; - } else if (get_config()->tls_ticket_key_cipher == EVP_aes_256_cbc()) { - expectedlen = 80; - enc_keylen = 32; - hmac_keylen = 32; - } else { - return; - } - - auto ticket_keys = std::make_shared(); - - for (; p != end;) { - if (end - p < 2) { - LOG(WARN) << "Memcached: tls ticket key data is too small"; - conn_handler->on_tls_ticket_key_not_found(w); - return; - } - auto len = util::get_uint16(p); - p += 2; - if (len != expectedlen) { - LOG(WARN) << "Memcached: wrong tls ticket key size: want " - << expectedlen << ", got " << len; - conn_handler->on_tls_ticket_key_not_found(w); - return; - } - if (p + len > end) { - LOG(WARN) << "Memcached: too short tls ticket key payload: want " << len - << ", got " << (end - p); - conn_handler->on_tls_ticket_key_not_found(w); - return; - } - auto key = TicketKey(); - key.cipher = get_config()->tls_ticket_key_cipher; - key.hmac = EVP_sha256(); - key.hmac_keylen = hmac_keylen; - - std::copy_n(p, key.data.name.size(), key.data.name.data()); - p += key.data.name.size(); - - std::copy_n(p, enc_keylen, key.data.enc_key.data()); - p += enc_keylen; - - std::copy_n(p, hmac_keylen, key.data.hmac_key.data()); - p += hmac_keylen; - - ticket_keys->keys.push_back(std::move(key)); - } - - conn_handler->on_tls_ticket_key_get_success(ticket_keys, w); - }; - - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Memcached: tls ticket key get request sent"; - } - - dispatcher->add_request(std::move(req)); -} - -} // namespace - namespace { int call_daemon() { #ifdef __sgi @@ -815,15 +609,54 @@ int call_daemon() { } // namespace namespace { -int event_loop() { - auto loop = EV_DEFAULT; +void close_env_fd(std::initializer_list envnames) { + for (auto envname : envnames) { + auto envfd = getenv(envname); + if (!envfd) { + continue; + } + auto fd = strtol(envfd, nullptr, 10); + close(fd); + } +} +} // namespace + +namespace { +pid_t fork_worker_process(SignalServer *ssv) { + int rv; + auto pid = fork(); + + if (pid == -1) { + return -1; + } + + if (pid == 0) { + close(ssv->ipc_fd[1]); + WorkerProcessConfig wpconf{ssv->ipc_fd[0], ssv->server_fd, ssv->server_fd6}; + rv = worker_process_event_loop(&wpconf); + if (rv != 0) { + LOG(ERROR) << "Worker process returned error"; + } + return 0; + } + + close(ssv->ipc_fd[0]); + + LOG(NOTICE) << "Worker process (" << pid << ") spawned"; + + return pid; +} +} // namespace + +namespace { +int event_loop() { + int rv; - auto conn_handler = make_unique(loop); if (get_config()->daemon) { if (call_daemon() == -1) { auto error = errno; LOG(FATAL) << "Failed to daemonize: " << strerror(error); - exit(EXIT_FAILURE); + return -1; } // We get new PID after successful daemon(). @@ -838,140 +671,76 @@ int event_loop() { save_pid(); } - if (get_config()->host_unix) { - close_env_fd({ENV_LISTENER4_FD, ENV_LISTENER6_FD}); - auto acceptor = create_unix_domain_acceptor(conn_handler.get()); - if (!acceptor) { - LOG(FATAL) << "Failed to listen on UNIX domain socket " - << get_config()->host.get(); - exit(EXIT_FAILURE); - } + SignalServer ssv; - conn_handler->set_acceptor(std::move(acceptor)); - } else { - close_env_fd({ENV_UNIX_FD}); - auto acceptor6 = create_acceptor(conn_handler.get(), AF_INET6); - auto acceptor4 = create_acceptor(conn_handler.get(), AF_INET); - if (!acceptor6 && !acceptor4) { - LOG(FATAL) << "Failed to listen on address " << get_config()->host.get() - << ", port " << get_config()->port; - exit(EXIT_FAILURE); - } - - conn_handler->set_acceptor(std::move(acceptor4)); - conn_handler->set_acceptor6(std::move(acceptor6)); - } - - ev_timer renew_ticket_key_timer; - if (!get_config()->upstream_no_tls) { - if (get_config()->tls_ticket_key_memcached_host) { - conn_handler->set_tls_ticket_key_memcached_dispatcher( - make_unique( - &get_config()->tls_ticket_key_memcached_addr, loop)); - - ev_timer_init(&renew_ticket_key_timer, memcached_get_ticket_key_cb, 0., - 0.); - renew_ticket_key_timer.data = conn_handler.get(); - // Get first ticket keys. - memcached_get_ticket_key_cb(loop, &renew_ticket_key_timer, 0); - } else { - bool auto_tls_ticket_key = true; - if (!get_config()->tls_ticket_key_files.empty()) { - if (!get_config()->tls_ticket_key_cipher_given) { - LOG(WARN) - << "It is strongly recommended to specify " - "--tls-ticket-key-cipher=aes-128-cbc (or " - "tls-ticket-key-cipher=aes-128-cbc in configuration file) " - "when --tls-ticket-key-file is used for the smooth " - "transition when the default value of --tls-ticket-key-cipher " - "becomes aes-256-cbc"; - } - auto ticket_keys = read_tls_ticket_key_file( - get_config()->tls_ticket_key_files, - get_config()->tls_ticket_key_cipher, EVP_sha256()); - if (!ticket_keys) { - LOG(WARN) << "Use internal session ticket key generator"; - } else { - conn_handler->set_ticket_keys(std::move(ticket_keys)); - auto_tls_ticket_key = false; - } - } - if (auto_tls_ticket_key) { - // Generate new ticket key every 1hr. - ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 1_h); - renew_ticket_key_timer.data = conn_handler.get(); - ev_timer_again(loop, &renew_ticket_key_timer); - - // Generate first session ticket key before running workers. - renew_ticket_key_cb(loop, &renew_ticket_key_timer, 0); - } - } - } - - // ListenHandler loads private key, and we listen on a priveleged port. - // After that, we drop the root privileges if needed. - drop_privileges(); - - int rv; - -#ifndef NOTHREADS - sigset_t signals; - sigemptyset(&signals); - sigaddset(&signals, REOPEN_LOG_SIGNAL); - sigaddset(&signals, EXEC_BINARY_SIGNAL); - sigaddset(&signals, GRACEFUL_SHUTDOWN_SIGNAL); - rv = pthread_sigmask(SIG_BLOCK, &signals, nullptr); - if (rv != 0) { - LOG(ERROR) << "Blocking signals failed: " << strerror(rv); - } -#endif // !NOTHREADS - - if (get_config()->num_worker == 1) { - rv = conn_handler->create_single_worker(); - } else { - rv = conn_handler->create_worker_thread(get_config()->num_worker); - } - - if (rv != 0) { + rv = pipe(ssv.ipc_fd.data()); + if (rv == -1) { + auto error = errno; + LOG(WARN) << "Failed to create pipe to communicate worker process: " + << strerror(error); return -1; } -#ifndef NOTHREADS - rv = pthread_sigmask(SIG_UNBLOCK, &signals, nullptr); - if (rv != 0) { - LOG(ERROR) << "Unblocking signals failed: " << strerror(rv); - } -#endif // !NOTHREADS + util::make_socket_nonblocking(ssv.ipc_fd[0]); + util::make_socket_nonblocking(ssv.ipc_fd[1]); - ev_signal reopen_log_sig; - ev_signal_init(&reopen_log_sig, reopen_log_signal_cb, REOPEN_LOG_SIGNAL); - reopen_log_sig.data = conn_handler.get(); - ev_signal_start(loop, &reopen_log_sig); + auto loop = EV_DEFAULT; - ev_signal exec_bin_sig; - ev_signal_init(&exec_bin_sig, exec_binary_signal_cb, EXEC_BINARY_SIGNAL); - exec_bin_sig.data = conn_handler.get(); - ev_signal_start(loop, &exec_bin_sig); + if (get_config()->host_unix) { + close_env_fd({ENV_LISTENER4_FD, ENV_LISTENER6_FD}); + auto fd = create_unix_domain_server_socket(); + if (fd == -1) { + LOG(FATAL) << "Failed to listen on UNIX domain socket " + << get_config()->host.get(); + return -1; + } - ev_signal graceful_shutdown_sig; - ev_signal_init(&graceful_shutdown_sig, graceful_shutdown_signal_cb, - GRACEFUL_SHUTDOWN_SIGNAL); - graceful_shutdown_sig.data = conn_handler.get(); - ev_signal_start(loop, &graceful_shutdown_sig); + ssv.server_fd = fd; + } else { + close_env_fd({ENV_UNIX_FD}); + auto fd6 = create_tcp_server_socket(AF_INET6); + auto fd4 = create_tcp_server_socket(AF_INET); + if (fd6 == -1 && fd4 == -1) { + LOG(FATAL) << "Failed to listen on address " << get_config()->host.get() + << ", port " << get_config()->port; + return -1; + } - if (!get_config()->upstream_no_tls && !get_config()->no_ocsp) { - conn_handler->proceed_next_cert_ocsp(); + ssv.server_fd = fd4; + ssv.server_fd6 = fd6; } - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Entering event loop"; + auto pid = fork_worker_process(&ssv); + + switch (pid) { + case -1: + return -1; + case 0: + // worker process (child) + return 0; } + ssv.worker_process_pid = pid; + + auto signals = + std::array{{REOPEN_LOG_SIGNAL, EXEC_BINARY_SIGNAL, + GRACEFUL_SHUTDOWN_SIGNAL, SIGINT, SIGTERM}}; + auto sigevs = std::array(); + + for (size_t i = 0; i < signals.size(); ++i) { + auto sigev = &sigevs[i]; + ev_signal_init(sigev, signal_cb, signals[i]); + sigev->data = &ssv; + ev_signal_start(loop, sigev); + } + + ev_child worker_process_childev; + ev_child_init(&worker_process_childev, worker_process_child_cb, pid, 0); + worker_process_childev.data = nullptr; + ev_child_start(loop, &worker_process_childev); + ev_run(loop, 0); - conn_handler->join_worker(); - conn_handler->cancel_ocsp_update(); - return 0; } } // namespace @@ -1790,6 +1559,8 @@ int main(int argc, char **argv) { // log errors/warnings while reading configuration files. reopen_log_files(); + mod_config()->original_argv = argv; + // We have to copy argv, since getopt_long may change its content. mod_config()->argc = argc; mod_config()->argv = new char *[argc]; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 15107575..f147c61f 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -327,6 +327,7 @@ struct Config { nghttp2_option *http2_client_option; const EVP_CIPHER *tls_ticket_key_cipher; const char *server_name; + char **original_argv; char **argv; char *cwd; size_t num_worker; diff --git a/src/shrpx_process.h b/src/shrpx_process.h new file mode 100644 index 00000000..47fc5b89 --- /dev/null +++ b/src/shrpx_process.h @@ -0,0 +1,43 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2015 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef SHRPX_PROCESS_H +#define SHRPX_PROCESS_H + +#include "shrpx.h" + +#include + +namespace shrpx { + +constexpr uint8_t SHRPX_IPC_REOPEN_LOG = 1; +constexpr uint8_t SHRPX_IPC_GRACEFUL_SHUTDOWN = 2; + +constexpr int REOPEN_LOG_SIGNAL = SIGUSR1; +constexpr int EXEC_BINARY_SIGNAL = SIGUSR2; +constexpr int GRACEFUL_SHUTDOWN_SIGNAL = SIGQUIT; + +} // namespace shrpx + +#endif // SHRPX_PROCESS_H diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index db236649..c116cec0 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -183,7 +183,8 @@ void Worker::process_events() { break; } case REOPEN_LOG: - WLOG(NOTICE, this) << "Reopening log files: worker(" << this << ")"; + WLOG(NOTICE, this) << "Reopening log files: worker process (thread " + << this << ")"; reopen_log_files(); diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc new file mode 100644 index 00000000..76411c02 --- /dev/null +++ b/src/shrpx_worker_process.cc @@ -0,0 +1,467 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2015 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "shrpx_worker_process.h" + +#include +#ifdef HAVE_UNISTD_H +#include +#endif // HAVE_UNISTD_H +#include +#include + +#include +#include + +#include + +#include + +#include "shrpx_config.h" +#include "shrpx_connection_handler.h" +#include "shrpx_log_config.h" +#include "shrpx_worker.h" +#include "shrpx_accept_handler.h" +#include "shrpx_http2_upstream.h" +#include "shrpx_http2_session.h" +#include "shrpx_memcached_dispatcher.h" +#include "shrpx_memcached_request.h" +#include "shrpx_process.h" +#include "util.h" +#include "app_helper.h" +#include "template.h" + +using namespace nghttp2; + +namespace shrpx { + +namespace { +void drop_privileges() { + if (getuid() == 0 && get_config()->uid != 0) { + if (initgroups(get_config()->user.get(), get_config()->gid) != 0) { + auto error = errno; + LOG(FATAL) << "Could not change supplementary groups: " + << strerror(error); + exit(EXIT_FAILURE); + } + if (setgid(get_config()->gid) != 0) { + auto error = errno; + LOG(FATAL) << "Could not change gid: " << strerror(error); + exit(EXIT_FAILURE); + } + if (setuid(get_config()->uid) != 0) { + auto error = errno; + LOG(FATAL) << "Could not change uid: " << strerror(error); + exit(EXIT_FAILURE); + } + if (setuid(0) != -1) { + LOG(FATAL) << "Still have root privileges?"; + exit(EXIT_FAILURE); + } + } +} +} // namespace + +namespace { +void graceful_shutdown(ConnectionHandler *conn_handler) { + if (conn_handler->get_graceful_shutdown()) { + return; + } + + LOG(NOTICE) << "Graceful shutdown signal received"; + + conn_handler->set_graceful_shutdown(true); + + conn_handler->disable_acceptor(); + + // After disabling accepting new connection, disptach incoming + // connection in backlog. + + conn_handler->accept_pending_connection(); + + conn_handler->graceful_shutdown_worker(); + + if (get_config()->num_worker == 1 && + conn_handler->get_single_worker()->get_worker_stat()->num_connections > + 0) { + return; + } + + // We have accepted all pending connections. Shutdown main event + // loop. + ev_break(conn_handler->get_loop()); +} +} // namespace + +namespace { +void reopen_log(ConnectionHandler *conn_handler) { + LOG(NOTICE) << "Reopening log files: worker process (thread main)"; + + (void)reopen_log_files(); + redirect_stderr_to_errorlog(); + + if (get_config()->num_worker > 1) { + conn_handler->worker_reopen_log_files(); + } +} +} // namespace + +namespace { +void ipc_readcb(struct ev_loop *loop, ev_io *w, int revents) { + auto conn_handler = static_cast(w->data); + std::array buf; + ssize_t nread; + while ((nread = read(w->fd, buf.data(), buf.size())) == -1 && errno == EINTR) + ; + if (nread == -1) { + auto error = errno; + LOG(ERROR) << "Failed to read data from ipc channel: errno=" << error; + return; + } + + if (nread == 0) { + // IPC socket closed. Perform immediate shutdown. + LOG(FATAL) << "IPC socket is closed. Perform immediate shutdown."; + ev_break(conn_handler->get_loop()); + return; + } + + for (ssize_t i = 0; i < nread; ++i) { + switch (buf[i]) { + case SHRPX_IPC_GRACEFUL_SHUTDOWN: + graceful_shutdown(conn_handler); + break; + case SHRPX_IPC_REOPEN_LOG: + reopen_log(conn_handler); + break; + } + } +} +} // namespace + +namespace { +int generate_ticket_key(TicketKey &ticket_key) { + ticket_key.cipher = get_config()->tls_ticket_key_cipher; + ticket_key.hmac = EVP_sha256(); + ticket_key.hmac_keylen = EVP_MD_size(ticket_key.hmac); + + assert(static_cast(EVP_CIPHER_key_length(ticket_key.cipher)) <= + ticket_key.data.enc_key.size()); + assert(ticket_key.hmac_keylen <= ticket_key.data.hmac_key.size()); + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "enc_keylen=" << EVP_CIPHER_key_length(ticket_key.cipher) + << ", hmac_keylen=" << ticket_key.hmac_keylen; + } + + if (RAND_bytes(reinterpret_cast(&ticket_key.data), + sizeof(ticket_key.data)) == 0) { + return -1; + } + + return 0; +} +} // namespace + +namespace { +void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) { + auto conn_handler = static_cast(w->data); + const auto &old_ticket_keys = conn_handler->get_ticket_keys(); + + auto ticket_keys = std::make_shared(); + LOG(NOTICE) << "Renew new ticket keys"; + + // If old_ticket_keys is not empty, it should contain at least 2 + // keys: one for encryption, and last one for the next encryption + // key but decryption only. The keys in between are old keys and + // decryption only. The next key is provided to ensure to mitigate + // possible problem when one worker encrypt new key, but one worker, + // which did not take the that key yet, and cannot decrypt it. + // + // We keep keys for get_config()->tls_session_timeout seconds. The + // default is 12 hours. Thus the maximum ticket vector size is 12. + if (old_ticket_keys) { + auto &old_keys = old_ticket_keys->keys; + auto &new_keys = ticket_keys->keys; + + assert(!old_keys.empty()); + + auto max_tickets = + static_cast(std::chrono::duration_cast( + get_config()->tls_session_timeout).count()); + + new_keys.resize(std::min(max_tickets, old_keys.size() + 1)); + std::copy_n(std::begin(old_keys), new_keys.size() - 1, + std::begin(new_keys) + 1); + } else { + ticket_keys->keys.resize(1); + } + + auto &new_key = ticket_keys->keys[0]; + + if (generate_ticket_key(new_key) != 0) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "failed to generate ticket key"; + } + conn_handler->set_ticket_keys(nullptr); + conn_handler->set_ticket_keys_to_worker(nullptr); + return; + } + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "ticket keys generation done"; + assert(ticket_keys->keys.size() >= 1); + LOG(INFO) << 0 << " enc+dec: " + << util::format_hex(ticket_keys->keys[0].data.name); + for (size_t i = 1; i < ticket_keys->keys.size(); ++i) { + auto &key = ticket_keys->keys[i]; + LOG(INFO) << i << " dec: " << util::format_hex(key.data.name); + } + } + + conn_handler->set_ticket_keys(ticket_keys); + conn_handler->set_ticket_keys_to_worker(ticket_keys); +} +} // namespace + +namespace { +void memcached_get_ticket_key_cb(struct ev_loop *loop, ev_timer *w, + int revents) { + auto conn_handler = static_cast(w->data); + auto dispatcher = conn_handler->get_tls_ticket_key_memcached_dispatcher(); + + auto req = make_unique(); + req->key = "nghttpx:tls-ticket-key"; + req->op = MEMCACHED_OP_GET; + req->cb = [conn_handler, dispatcher, w](MemcachedRequest *req, + MemcachedResult res) { + switch (res.status_code) { + case MEMCACHED_ERR_NO_ERROR: + break; + case MEMCACHED_ERR_EXT_NETWORK_ERROR: + conn_handler->on_tls_ticket_key_network_error(w); + return; + default: + conn_handler->on_tls_ticket_key_not_found(w); + return; + } + + // |version (4bytes)|len (2bytes)|key (variable length)|... + // (len, key) pairs are repeated as necessary. + + auto &value = res.value; + if (value.size() < 4) { + LOG(WARN) << "Memcached: tls ticket key value is too small: got " + << value.size(); + conn_handler->on_tls_ticket_key_not_found(w); + return; + } + auto p = value.data(); + auto version = util::get_uint32(p); + // Currently supported version is 1. + if (version != 1) { + LOG(WARN) << "Memcached: tls ticket key version: want 1, got " << version; + conn_handler->on_tls_ticket_key_not_found(w); + return; + } + + auto end = p + value.size(); + p += 4; + + size_t expectedlen; + size_t enc_keylen; + size_t hmac_keylen; + if (get_config()->tls_ticket_key_cipher == EVP_aes_128_cbc()) { + expectedlen = 48; + enc_keylen = 16; + hmac_keylen = 16; + } else if (get_config()->tls_ticket_key_cipher == EVP_aes_256_cbc()) { + expectedlen = 80; + enc_keylen = 32; + hmac_keylen = 32; + } else { + return; + } + + auto ticket_keys = std::make_shared(); + + for (; p != end;) { + if (end - p < 2) { + LOG(WARN) << "Memcached: tls ticket key data is too small"; + conn_handler->on_tls_ticket_key_not_found(w); + return; + } + auto len = util::get_uint16(p); + p += 2; + if (len != expectedlen) { + LOG(WARN) << "Memcached: wrong tls ticket key size: want " + << expectedlen << ", got " << len; + conn_handler->on_tls_ticket_key_not_found(w); + return; + } + if (p + len > end) { + LOG(WARN) << "Memcached: too short tls ticket key payload: want " << len + << ", got " << (end - p); + conn_handler->on_tls_ticket_key_not_found(w); + return; + } + auto key = TicketKey(); + key.cipher = get_config()->tls_ticket_key_cipher; + key.hmac = EVP_sha256(); + key.hmac_keylen = hmac_keylen; + + std::copy_n(p, key.data.name.size(), key.data.name.data()); + p += key.data.name.size(); + + std::copy_n(p, enc_keylen, key.data.enc_key.data()); + p += enc_keylen; + + std::copy_n(p, hmac_keylen, key.data.hmac_key.data()); + p += hmac_keylen; + + ticket_keys->keys.push_back(std::move(key)); + } + + conn_handler->on_tls_ticket_key_get_success(ticket_keys, w); + }; + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Memcached: tls ticket key get request sent"; + } + + dispatcher->add_request(std::move(req)); +} + +} // namespace + +int worker_process_event_loop(WorkerProcessConfig *wpconf) { + if (reopen_log_files() != 0) { + LOG(FATAL) << "Failed to open log file"; + return -1; + } + + auto loop = EV_DEFAULT; + + ConnectionHandler conn_handler(loop); + + if (wpconf->server_fd6 != -1) { + conn_handler.set_acceptor6( + make_unique(wpconf->server_fd6, &conn_handler)); + } + if (wpconf->server_fd != 1) { + conn_handler.set_acceptor( + make_unique(wpconf->server_fd, &conn_handler)); + } + + ev_timer renew_ticket_key_timer; + if (!get_config()->upstream_no_tls) { + if (get_config()->tls_ticket_key_memcached_host) { + conn_handler.set_tls_ticket_key_memcached_dispatcher( + make_unique( + &get_config()->tls_ticket_key_memcached_addr, loop)); + + ev_timer_init(&renew_ticket_key_timer, memcached_get_ticket_key_cb, 0., + 0.); + renew_ticket_key_timer.data = &conn_handler; + // Get first ticket keys. + memcached_get_ticket_key_cb(loop, &renew_ticket_key_timer, 0); + } else { + bool auto_tls_ticket_key = true; + if (!get_config()->tls_ticket_key_files.empty()) { + if (!get_config()->tls_ticket_key_cipher_given) { + LOG(WARN) + << "It is strongly recommended to specify " + "--tls-ticket-key-cipher=aes-128-cbc (or " + "tls-ticket-key-cipher=aes-128-cbc in configuration file) " + "when --tls-ticket-key-file is used for the smooth " + "transition when the default value of --tls-ticket-key-cipher " + "becomes aes-256-cbc"; + } + auto ticket_keys = read_tls_ticket_key_file( + get_config()->tls_ticket_key_files, + get_config()->tls_ticket_key_cipher, EVP_sha256()); + if (!ticket_keys) { + LOG(WARN) << "Use internal session ticket key generator"; + } else { + conn_handler.set_ticket_keys(std::move(ticket_keys)); + auto_tls_ticket_key = false; + } + } + if (auto_tls_ticket_key) { + // Generate new ticket key every 1hr. + ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 1_h); + renew_ticket_key_timer.data = &conn_handler; + ev_timer_again(loop, &renew_ticket_key_timer); + + // Generate first session ticket key before running workers. + renew_ticket_key_cb(loop, &renew_ticket_key_timer, 0); + } + } + } + + int rv; + + // It is good to ignore these control signals since user may use + // "killall .. nghttpx". We don't want catch this signal in worker + // process. + struct sigaction act {}; + act.sa_handler = SIG_IGN; + sigaction(REOPEN_LOG_SIGNAL, &act, nullptr); + sigaction(EXEC_BINARY_SIGNAL, &act, nullptr); + sigaction(GRACEFUL_SHUTDOWN_SIGNAL, &act, nullptr); + + if (get_config()->num_worker == 1) { + rv = conn_handler.create_single_worker(); + } else { + rv = conn_handler.create_worker_thread(get_config()->num_worker); + } + + if (rv != 0) { + return -1; + } + + drop_privileges(); + + ev_io ipcev; + ev_io_init(&ipcev, ipc_readcb, wpconf->ipc_fd, EV_READ); + ipcev.data = &conn_handler; + ev_io_start(loop, &ipcev); + + if (!get_config()->upstream_no_tls && !get_config()->no_ocsp) { + conn_handler.proceed_next_cert_ocsp(); + } + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Entering event loop"; + } + + ev_run(loop, 0); + + conn_handler.join_worker(); + conn_handler.cancel_ocsp_update(); + + return 0; +} + +} // namespace shrpx diff --git a/src/shrpx_worker_process.h b/src/shrpx_worker_process.h new file mode 100644 index 00000000..77634f00 --- /dev/null +++ b/src/shrpx_worker_process.h @@ -0,0 +1,49 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2015 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef SHRPX_WORKER_PROCESS_H +#define SHRPX_WORKER_PROCESS_H + +#include "shrpx.h" + +using namespace nghttp2; + +namespace shrpx { + +class ConnectionHandler; + +struct WorkerProcessConfig { + // IPC socket to read event from master process + int ipc_fd; + // IPv4 or UNIX domain socket, or -1 if not used + int server_fd; + // IPv6 socket, or -1 if not used + int server_fd6; +}; + +int worker_process_event_loop(WorkerProcessConfig *wpconf); + +} // namespace shrpx + +#endif // SHRPX_WORKER_PROCESS_H