From 044385ab6ef54e02cd3c62d4b887886c1793197e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 23 Sep 2015 19:45:53 +0900 Subject: [PATCH] Add neverbleed support neverbleed is disabled by default. To enable it, use --with-neverbleed configure option. --- configure.ac | 15 +++++++++ src/Makefile.am | 10 ++++++ src/shrpx.cc | 23 ++----------- src/shrpx_connection_handler.cc | 23 +++++++++++-- src/shrpx_connection_handler.h | 12 +++++++ src/shrpx_log.cc | 23 +++++++++++++ src/shrpx_log.h | 4 +++ src/shrpx_ssl.cc | 39 +++++++++++++++++++--- src/shrpx_ssl.h | 19 +++++++++-- src/shrpx_worker_process.cc | 58 +++++++++++++++++++++++++++++++-- third-party/Makefile.am | 5 +++ third-party/neverbleed | 2 +- 12 files changed, 200 insertions(+), 33 deletions(-) diff --git a/configure.ac b/configure.ac index 3a0961f5..a37d7e16 100644 --- a/configure.ac +++ b/configure.ac @@ -124,6 +124,11 @@ AC_ARG_WITH([mruby], [Use mruby [default=no]])], [request_mruby=$withval], [request_mruby=no]) +AC_ARG_WITH([neverbleed], + [AS_HELP_STRING([--with-neverbleed], + [Use neverbleed [default=no]])], + [request_neverbleed=$withval], [request_neverbleed=no]) + AC_ARG_WITH([cython], [AS_HELP_STRING([--with-cython=PATH], [Use cython in given PATH])], @@ -391,6 +396,15 @@ fi AM_CONDITIONAL([HAVE_MRUBY], [test "x${have_mruby}" = "xyes"]) +# neverbleed (for src/nghttpx) +have_neverbleed=no +if test "x${request_neverbleed}" = "xyes"; then + have_neverbleed=yes + AC_DEFINE([HAVE_NEVERBLEED], [1], [Define to 1 if you have `neverbleed` library.]) +fi + +AM_CONDITIONAL([HAVE_NEVERBLEED], [test "x${have_neverbleed}" = "xyes"]) + # Check Boost Asio library have_asio_lib=no @@ -755,6 +769,7 @@ AC_MSG_NOTICE([summary of build options: Libevent(SSL): ${have_libevent_openssl} Spdylay: ${have_spdylay} MRuby: ${have_mruby} + Neverbleed: ${have_neverbleed} Jansson: ${have_jansson} Jemalloc: ${have_jemalloc} Zlib: ${have_zlib} diff --git a/src/Makefile.am b/src/Makefile.am index 3635b633..85405cb4 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -157,6 +157,11 @@ libnghttpx_a_CPPFLAGS += \ nghttpx_LDADD += -L${top_builddir}/third-party/mruby/build/lib @LIBMRUBY_LIBS@ endif # HAVE_MRUBY +if HAVE_NEVERBLEED +libnghttpx_a_CPPFLAGS += -I${top_srcdir}/third-party/neverbleed +nghttpx_LDADD += ${top_builddir}/third-party/libneverbleed.la +endif # HAVE_NEVERBLEED + if HAVE_CUNIT check_PROGRAMS += nghttpx-unittest nghttpx_unittest_SOURCES = shrpx-unittest.cc \ @@ -180,6 +185,11 @@ nghttpx_unittest_LDADD += \ -L${top_builddir}/third-party/mruby/build/lib @LIBMRUBY_LIBS@ endif # HAVE_MRUBY +if HAVE_NEVERBLEED +nghttpx_unittest_CPPFLAGS += -I${top_srcdir}/third-party/neverbleed +nghttpx_unittest_LDADD += ${top_builddir}/third-party/libneverbleed.la +endif # HAVE_NEVERBLEED + TESTS += nghttpx-unittest endif # HAVE_CUNIT diff --git a/src/shrpx.cc b/src/shrpx.cc index 5aa5885f..5a3a48a6 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -408,26 +408,9 @@ void signal_cb(struct ev_loop *loop, ev_signal *w, int revents) { namespace { void worker_process_child_cb(struct ev_loop *loop, ev_child *w, int revents) { - std::string signalstr; - if (WIFSIGNALED(w->rstatus)) { - signalstr += "; signal "; - auto sig = WTERMSIG(w->rstatus); - auto s = strsignal(sig); - if (s) { - signalstr += s; - signalstr += "("; - } else { - signalstr += "UNKNOWN("; - } - signalstr += util::utos(sig); - signalstr += ")"; - } + log_chld(w->rpid, w->rstatus, "Worker process"); - LOG(NOTICE) << "Worker process (" << w->rpid << ") exited " - << (WIFEXITED(w->rstatus) ? "normally" : "abnormally") - << " with status " << std::hex << w->rstatus << std::oct - << "; exit status " << WEXITSTATUS(w->rstatus) - << (signalstr.empty() ? "" : signalstr.c_str()); + ev_child_stop(loop, w); ev_break(loop); } @@ -744,7 +727,7 @@ pid_t fork_worker_process(SignalServer *ssv) { close(ssv->ipc_fd[0]); - LOG(NOTICE) << "Worker process (" << pid << ") spawned"; + LOG(NOTICE) << "Worker process [" << pid << "] spawned"; return pid; } diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 167ecc19..52c5c94d 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -159,7 +159,12 @@ void ConnectionHandler::worker_reopen_log_files() { int ConnectionHandler::create_single_worker() { auto cert_tree = ssl::create_cert_lookup_tree(); - auto sv_ssl_ctx = ssl::setup_server_ssl_context(all_ssl_ctx_, cert_tree); + auto sv_ssl_ctx = ssl::setup_server_ssl_context(all_ssl_ctx_, cert_tree +#ifdef HAVE_NEVERBLEED + , + nb_.get() +#endif // HAVE_NEVERBLEED + ); auto cl_ssl_ctx = ssl::setup_client_ssl_context(); if (cl_ssl_ctx) { @@ -182,7 +187,12 @@ int ConnectionHandler::create_worker_thread(size_t num) { assert(workers_.size() == 0); auto cert_tree = ssl::create_cert_lookup_tree(); - auto sv_ssl_ctx = ssl::setup_server_ssl_context(all_ssl_ctx_, cert_tree); + auto sv_ssl_ctx = ssl::setup_server_ssl_context(all_ssl_ctx_, cert_tree +#ifdef HAVE_NEVERBLEED + , + nb_.get() +#endif // HAVE_NEVERBLEED + ); auto cl_ssl_ctx = ssl::setup_client_ssl_context(); if (cl_ssl_ctx) { @@ -703,4 +713,13 @@ ConnectionHandler::schedule_next_tls_ticket_key_memcached_get(ev_timer *w) { ev_timer_start(loop_, w); } +#ifdef HAVE_NEVERBLEED +void ConnectionHandler::set_neverbleed(std::unique_ptr nb) { + nb_ = std::move(nb); +} + +neverbleed_t *ConnectionHandler::get_neverbleed() const { return nb_.get(); } + +#endif // HAVE_NEVERBLEED + } // namespace shrpx diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h index 8ce25eec..51a746ca 100644 --- a/src/shrpx_connection_handler.h +++ b/src/shrpx_connection_handler.h @@ -39,6 +39,10 @@ #include +#ifdef HAVE_NEVERBLEED +#include +#endif // HAVE_NEVERBLEED + #include "shrpx_downstream_connection_pool.h" namespace shrpx { @@ -123,6 +127,11 @@ public: ev_timer *w); void schedule_next_tls_ticket_key_memcached_get(ev_timer *w); +#ifdef HAVE_NEVERBLEED + void set_neverbleed(std::unique_ptr nb); + neverbleed_t *get_neverbleed() const; +#endif // HAVE_NEVERBLEED + private: // Stores all SSL_CTX objects. std::vector all_ssl_ctx_; @@ -144,6 +153,9 @@ private: std::unique_ptr acceptor_; // acceptor for IPv6 address std::unique_ptr acceptor6_; +#ifdef HAVE_NEVERBLEED + std::unique_ptr nb_; +#endif // HAVE_NEVERBLEED ev_timer disable_acceptor_timer_; ev_timer ocsp_timer_; size_t tls_ticket_key_memcached_get_retry_count_; diff --git a/src/shrpx_log.cc b/src/shrpx_log.cc index 5c853e9b..71714bfa 100644 --- a/src/shrpx_log.cc +++ b/src/shrpx_log.cc @@ -370,6 +370,29 @@ int reopen_log_files() { return res; } +void log_chld(pid_t pid, int rstatus, const char *msg) { + std::string signalstr; + if (WIFSIGNALED(rstatus)) { + signalstr += "; signal "; + auto sig = WTERMSIG(rstatus); + auto s = strsignal(sig); + if (s) { + signalstr += s; + signalstr += "("; + } else { + signalstr += "UNKNOWN("; + } + signalstr += util::utos(sig); + signalstr += ")"; + } + + LOG(NOTICE) << msg << ": [" << pid << "] exited " + << (WIFEXITED(rstatus) ? "normally" : "abnormally") + << " with status " << std::hex << rstatus << std::oct + << "; exit status " << WEXITSTATUS(rstatus) + << (signalstr.empty() ? "" : signalstr.c_str()); +} + void redirect_stderr_to_errorlog() { auto lgconf = log_config(); diff --git a/src/shrpx_log.h b/src/shrpx_log.h index bb86a92e..33336913 100644 --- a/src/shrpx_log.h +++ b/src/shrpx_log.h @@ -156,6 +156,10 @@ void upstream_accesslog(const std::vector &lf, int reopen_log_files(); +// Logs message when process whose pid is |pid| and exist status is +// |rstatus| exited. The |msg| is prepended to the log message. +void log_chld(pid_t pid, int rstatus, const char *msg); + void redirect_stderr_to_errorlog(); } // namespace shrpx diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index dc85b43e..6b517b84 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -456,8 +456,12 @@ long int create_tls_proto_mask(const std::vector &tls_proto_list) { return res; } -SSL_CTX *create_ssl_context(const char *private_key_file, - const char *cert_file) { +SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file +#ifdef HAVE_NEVERBLEED + , + neverbleed_t *nb +#endif // HAVE_NEVERBLEED + ) { auto ssl_ctx = SSL_CTX_new(SSLv23_server_method()); if (!ssl_ctx) { LOG(FATAL) << ERR_error_string(ERR_get_error(), nullptr); @@ -543,12 +547,22 @@ SSL_CTX *create_ssl_context(const char *private_key_file, SSL_CTX_set_default_passwd_cb(ssl_ctx, ssl_pem_passwd_cb); SSL_CTX_set_default_passwd_cb_userdata(ssl_ctx, (void *)get_config()); } + +#ifndef HAVE_NEVERBLEED if (SSL_CTX_use_PrivateKey_file(ssl_ctx, private_key_file, SSL_FILETYPE_PEM) != 1) { LOG(FATAL) << "SSL_CTX_use_PrivateKey_file failed: " << ERR_error_string(ERR_get_error(), nullptr); + } +#else // HAVE_NEVERBLEED + std::array errbuf; + if (neverbleed_load_private_key_file(nb, ssl_ctx, private_key_file, + errbuf.data()) != 1) { + LOG(FATAL) << "neverbleed_load_private_key_file failed: " << errbuf.data(); DIE(); } +#endif // HAVE_NEVERBLEED + if (SSL_CTX_use_certificate_chain_file(ssl_ctx, cert_file) != 1) { LOG(FATAL) << "SSL_CTX_use_certificate_file failed: " << ERR_error_string(ERR_get_error(), nullptr); @@ -1096,13 +1110,23 @@ bool in_proto_list(const std::vector &protos, } SSL_CTX *setup_server_ssl_context(std::vector &all_ssl_ctx, - CertLookupTree *cert_tree) { + CertLookupTree *cert_tree +#ifdef HAVE_NEVERBLEED + , + neverbleed_t *nb +#endif // HAVE_NEVERBLEED + ) { if (get_config()->upstream_no_tls) { return nullptr; } auto ssl_ctx = ssl::create_ssl_context(get_config()->private_key_file.get(), - get_config()->cert_file.get()); + get_config()->cert_file.get() +#ifdef HAVE_NEVERBLEED + , + nb +#endif // HAVE_NEVERBLEED + ); all_ssl_ctx.push_back(ssl_ctx); @@ -1118,7 +1142,12 @@ SSL_CTX *setup_server_ssl_context(std::vector &all_ssl_ctx, for (auto &keycert : get_config()->subcerts) { auto ssl_ctx = - ssl::create_ssl_context(keycert.first.c_str(), keycert.second.c_str()); + ssl::create_ssl_context(keycert.first.c_str(), keycert.second.c_str() +#ifdef HAVE_NEVERBLEED + , + nb +#endif // HAVE_NEVERBLEED + ); all_ssl_ctx.push_back(ssl_ctx); if (ssl::cert_lookup_tree_add_cert_from_file( cert_tree, ssl_ctx, keycert.second.c_str()) == -1) { diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h index 7fdbbd67..2a722c07 100644 --- a/src/shrpx_ssl.h +++ b/src/shrpx_ssl.h @@ -35,6 +35,10 @@ #include +#ifdef HAVE_NEVERBLEED +#include +#endif // HAVE_NEVERBLEED + namespace shrpx { class ClientHandler; @@ -57,8 +61,12 @@ struct TLSContextData { }; // Create server side SSL_CTX -SSL_CTX *create_ssl_context(const char *private_key_file, - const char *cert_file); +SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file +#ifdef HAVE_NEVERBLEED + , + neverbleed_t *nb +#endif // HAVE_NEVERBLEED + ); // Create client side SSL_CTX SSL_CTX *create_ssl_client_context(); @@ -161,7 +169,12 @@ set_alpn_prefs(const std::vector &protos); // object as |cert_tree| parameter, otherwise SNI does not work. All // the created SSL_CTX is stored into |all_ssl_ctx|. SSL_CTX *setup_server_ssl_context(std::vector &all_ssl_ctx, - CertLookupTree *cert_tree); + CertLookupTree *cert_tree +#ifdef HAVE_NEVERBLEED + , + neverbleed_t *nb +#endif // HAVE_NEVERBLEED + ); // Setups client side SSL_CTX. This function inspects get_config() // and if downstream_no_tls is true, returns nullptr. Otherwise, only diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc index 8452256e..f37ced59 100644 --- a/src/shrpx_worker_process.cc +++ b/src/shrpx_worker_process.cc @@ -57,7 +57,11 @@ using namespace nghttp2; namespace shrpx { namespace { -void drop_privileges() { +void drop_privileges( +#ifdef HAVE_NEVERBLEED + neverbleed_t *nb +#endif // HAVE_NEVERBLEED + ) { if (getuid() == 0 && get_config()->uid != 0) { if (initgroups(get_config()->user.get(), get_config()->gid) != 0) { auto error = errno; @@ -79,6 +83,9 @@ void drop_privileges() { LOG(FATAL) << "Still have root privileges?"; exit(EXIT_FAILURE); } +#ifdef HAVE_NEVERBLEED + neverbleed_setuidgid(nb, get_config()->user.get(), 1); +#endif // HAVE_NEVERBLEED } } } // namespace @@ -359,6 +366,20 @@ void memcached_get_ticket_key_cb(struct ev_loop *loop, ev_timer *w, } // namespace +#ifdef HAVE_NEVERBLEED +namespace { +void nb_child_cb(struct ev_loop *loop, ev_child *w, int revents) { + log_chld(w->rpid, w->rstatus, "neverbleed process"); + + ev_child_stop(loop, w); + + LOG(FATAL) << "neverbleed process exitted; aborting now"; + + _Exit(EXIT_FAILURE); +} +} // namespace +#endif // HAVE_NEVERBLEED + int worker_process_event_loop(WorkerProcessConfig *wpconf) { if (reopen_log_files() != 0) { LOG(FATAL) << "Failed to open log file"; @@ -378,6 +399,29 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { make_unique(wpconf->server_fd, &conn_handler)); } +#ifdef HAVE_NEVERBLEED + std::array errbuf; + { + auto nb = make_unique(); + if (neverbleed_init(nb.get(), errbuf.data()) != 0) { + LOG(FATAL) << "neverbleed_init failed: " << errbuf.data(); + return -1; + } + + LOG(NOTICE) << "neverbleed process [" << nb->daemon_pid << "] spawned"; + + conn_handler.set_neverbleed(std::move(nb)); + } + + auto nb = conn_handler.get_neverbleed(); + + ev_child nb_childev; + ev_child_init(&nb_childev, nb_child_cb, nb->daemon_pid, 0); + nb_childev.data = nullptr; + ev_child_start(loop, &nb_childev); + +#endif // HAVE_NEVERBLEED + ev_timer renew_ticket_key_timer; if (!get_config()->upstream_no_tls) { if (get_config()->tls_ticket_key_memcached_host) { @@ -458,7 +502,11 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { #endif // !NOTHREADS } - drop_privileges(); + drop_privileges( +#ifdef HAVE_NEVERBLEED + nb +#endif // HAVE_NEVERBLEED + ); ev_io ipcev; ev_io_init(&ipcev, ipc_readcb, wpconf->ipc_fd, EV_READ); @@ -478,6 +526,12 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { conn_handler.join_worker(); conn_handler.cancel_ocsp_update(); +#ifdef HAVE_NEVERBLEED + if (nb && nb->daemon_pid != -1) { + kill(nb->daemon_pid, SIGTERM); + } +#endif // HAVE_NEVERBLEED + return 0; } diff --git a/third-party/Makefile.am b/third-party/Makefile.am index af1424bd..371a19b0 100644 --- a/third-party/Makefile.am +++ b/third-party/Makefile.am @@ -30,6 +30,11 @@ libhttp_parser_la_SOURCES = \ http-parser/http_parser.c \ http-parser/http_parser.h +if HAVE_NEVERBLEED +noinst_LTLIBRARIES += libneverbleed.la +libneverbleed_la_SOURCES = neverbleed/neverbleed.c neverbleed/neverbleed.h +endif # HAVE_NEVERBLEED + if HAVE_MRUBY EXTRA_DIST = build_config.rb mruby/* diff --git a/third-party/neverbleed b/third-party/neverbleed index 3d567817..81eff20b 160000 --- a/third-party/neverbleed +++ b/third-party/neverbleed @@ -1 +1 @@ -Subproject commit 3d56781771a60a9c343e2b80849d81992c925779 +Subproject commit 81eff20bd84b4d0dce2cbbd1a5ad1384d086423b