walkero
/
nghttp2
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

nghttpx: Add --tls-ctx-per-worker option 

When same SSL_CTX is used by multiple thread simultaneously we have to
setup some number of mutex locks for it.  We could not check how this
locking affects scalability since we have 4 cores at best in our
development machine.  Good side of sharing SSL_CTX across threads is
we can share session ID pool.

If --tls-ctx-per-worker is enabled, SSL_CTX is created per thread
basis and we can eliminate mutex locks.  The downside is session ID is
no longer shared, which means if session ID generated by one thread
cannot be acceptable by another thread.  But we have now session
ticket enabled and its keys are shared by all threads.
This commit is contained in:
Tatsuhiro Tsujikawa 2015-01-13 00:18:27 +09:00
parent 0ea041e8cb
commit 1e4f8f27fd
11 changed files with 135 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
src/shrpx.cc
View File

@ -485,23 +485,8 @@ void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) {
namespace {
int event_loop() {
auto loop = EV_DEFAULT;
SSL_CTX *sv_ssl_ctx, *cl_ssl_ctx;
if (get_config()->client_mode) {
sv_ssl_ctx = nullptr;
cl_ssl_ctx = get_config()->downstream_no_tls
? nullptr
: ssl::create_ssl_client_context();
} else {
sv_ssl_ctx =
get_config()->upstream_no_tls ? nullptr : get_config()->default_ssl_ctx;
cl_ssl_ctx = get_config()->http2_bridge && !get_config()->downstream_no_tls
? ssl::create_ssl_client_context()
: nullptr;
}
auto conn_handler =
util::make_unique<ConnectionHandler>(loop, sv_ssl_ctx, cl_ssl_ctx);
auto conn_handler = util::make_unique<ConnectionHandler>(loop);
if (get_config()->daemon) {
if (daemon(0, 0) == -1) {
auto error = errno;
@ -533,7 +518,8 @@ int event_loop() {
drop_privileges();
ev_timer renew_ticket_key_timer;
if (sv_ssl_ctx && get_config()->auto_tls_ticket_key) {
if (!get_config()->client_mode && !get_config()->upstream_no_tls &&
get_config()->auto_tls_ticket_key) {
// Renew ticket key every 12hrs
ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 12 * 3600.);
renew_ticket_key_timer.data = conn_handler.get();
@ -556,6 +542,10 @@ int event_loop() {
}
#endif // !NOTHREADS
if (!get_config()->tls_ctx_per_worker) {
conn_handler->create_ssl_context();
}
if (get_config()->num_worker > 1) {
conn_handler->create_worker_thread(get_config()->num_worker);
} else if (get_config()->downstream_proto == PROTO_HTTP2) {
@ -719,7 +709,6 @@ void fill_default_config() {
mod_config()->pid = getpid();
mod_config()->backend_ipv4 = false;
mod_config()->backend_ipv6 = false;
mod_config()->cert_tree = nullptr;
mod_config()->downstream_http_proxy_userinfo = nullptr;
mod_config()->downstream_http_proxy_host = nullptr;
mod_config()->downstream_http_proxy_port = 0;
@ -755,6 +744,7 @@ void fill_default_config() {
mod_config()->downstream_connections_per_frontend = 0;
mod_config()->listener_disable_timeout = 0.;
mod_config()->auto_tls_ticket_key = true;
mod_config()->tls_ctx_per_worker = false;
}
} // namespace
@ -1016,6 +1006,16 @@ SSL/TLS:
while opening or reading a file, key is generated
automatically and renewed every 12hrs. At most 2
keys are stored in memory.
--tls-ctx-per-worker
Create OpenSSL's SSL_CTX per worker, so that no
internal locking is required. This may improve
scalability with multi threaded configuration.
If this option is enabled, session ID is no
longer shared accross SSL_CTX objects, which
means session ID generated by one worker is not
acceptable by another worker. On the other hand,
session ticket key is shared across all worker
threads.
HTTP/2 and SPDY:
-c, --http2-max-concurrent-streams=<NUM>
@ -1288,6 +1288,7 @@ int main(int argc, char **argv) {
67},
{"tls-ticket-key-file", required_argument, &flag, 68},
{"rlimit-nofile", required_argument, &flag, 69},
{"tls-ctx-per-worker", no_argument, &flag, 70},
{nullptr, 0, nullptr, 0}};
int option_index = 0;
@ -1605,6 +1606,10 @@ int main(int argc, char **argv) {
// --rlimit-nofile
cmdcfgs.emplace_back(SHRPX_OPT_RLIMIT_NOFILE, optarg);
break;
case 70:
// --tls-ctx-per-worker
cmdcfgs.emplace_back(SHRPX_OPT_TLS_CTX_PER_WORKER, "yes");
break;
default:
break;
}
@ -1620,9 +1625,6 @@ int main(int argc, char **argv) {
OpenSSL_add_all_algorithms();
SSL_load_error_strings();
SSL_library_init();
#ifndef NOTHREADS
nghttp2::ssl::LibsslGlobalLock lock;
#endif // NOTHREADS
if (conf_exists(get_config()->conf_path.get())) {
if (load_config(get_config()->conf_path.get()) == -1) {
@ -1648,6 +1650,13 @@ int main(int argc, char **argv) {
}
}
#ifndef NOTHREADS
std::unique_ptr<nghttp2::ssl::LibsslGlobalLock> lock;
if (!get_config()->tls_ctx_per_worker) {
lock = util::make_unique<nghttp2::ssl::LibsslGlobalLock>();
}
#endif // NOTHREADS
if (get_config()->accesslog_syslog || get_config()->errorlog_syslog) {
openlog("nghttpx", LOG_NDELAY | LOG_NOWAIT | LOG_PID,
get_config()->syslog_facility);
@ -1730,33 +1739,6 @@ int main(int argc, char **argv) {
mod_config()->alpn_prefs = ssl::set_alpn_prefs(get_config()->npn_list);
if (!get_config()->subcerts.empty()) {
mod_config()->cert_tree = ssl::cert_lookup_tree_new();
}
for (auto &keycert : get_config()->subcerts) {
auto ssl_ctx =
ssl::create_ssl_context(keycert.first.c_str(), keycert.second.c_str());
if (ssl::cert_lookup_tree_add_cert_from_file(
get_config()->cert_tree, ssl_ctx, keycert.second.c_str()) == -1) {
LOG(FATAL) << "Failed to add sub certificate.";
exit(EXIT_FAILURE);
}
}
if (get_config()->cert_file && get_config()->private_key_file) {
mod_config()->default_ssl_ctx = ssl::create_ssl_context(
get_config()->private_key_file.get(), get_config()->cert_file.get());
if (get_config()->cert_tree) {
if (ssl::cert_lookup_tree_add_cert_from_file(
get_config()->cert_tree, get_config()->default_ssl_ctx,
get_config()->cert_file.get()) == -1) {
LOG(FATAL) << "Failed to add server certificate to the lookup tree.";
exit(EXIT_FAILURE);
}
}
}
if (!get_config()->tls_ticket_key_files.empty()) {
auto ticket_keys =
read_tls_ticket_key_file(get_config()->tls_ticket_key_files);

7
src/shrpx_config.cc
View File

@ -140,6 +140,7 @@ const char SHRPX_OPT_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND[] =
const char SHRPX_OPT_LISTENER_DISABLE_TIMEOUT[] = "listener-disable-timeout";
const char SHRPX_OPT_TLS_TICKET_KEY_FILE[] = "tls-ticket-key-file";
const char SHRPX_OPT_RLIMIT_NOFILE[] = "rlimit-nofile";
const char SHRPX_OPT_TLS_CTX_PER_WORKER[] = "tls-ctx-per-worker";
namespace {
Config *config = nullptr;
@ -1128,6 +1129,12 @@ int parse_config(const char *opt, const char *optarg) {
return 0;
}
if (util::strieq(opt, SHRPX_OPT_TLS_CTX_PER_WORKER)) {
mod_config()->tls_ctx_per_worker = util::strieq(optarg, "yes");
return 0;
}
if (util::strieq(opt, "conf")) {
LOG(WARN) << "conf: ignored";

4
src/shrpx_config.h
View File

@ -128,6 +128,7 @@ extern const char SHRPX_OPT_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND[];
extern const char SHRPX_OPT_LISTENER_DISABLE_TIMEOUT[];
extern const char SHRPX_OPT_TLS_TICKET_KEY_FILE[];
extern const char SHRPX_OPT_RLIMIT_NOFILE[];
extern const char SHRPX_OPT_TLS_CTX_PER_WORKER[];
union sockaddr_union {
sockaddr sa;
@ -199,8 +200,6 @@ struct Config {
std::unique_ptr<char[]> private_key_passwd;
std::unique_ptr<char[]> cert_file;
std::unique_ptr<char[]> dh_param_file;
SSL_CTX *default_ssl_ctx;
ssl::CertLookupTree *cert_tree;
const char *server_name;
std::unique_ptr<char[]> backend_tls_sni_name;
std::unique_ptr<char[]> pid_file;
@ -296,6 +295,7 @@ struct Config {
bool upstream_frame_debug;
bool no_location_rewrite;
bool auto_tls_ticket_key;
bool tls_ctx_per_worker;
};
const Config *get_config();

11
src/shrpx_connection_handler.cc
View File

@ -58,9 +58,8 @@ void acceptor_disable_cb(struct ev_loop *loop, ev_timer *w, int revent) {
}
} // namespace
ConnectionHandler::ConnectionHandler(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx,
SSL_CTX *cl_ssl_ctx)
: loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx),
ConnectionHandler::ConnectionHandler(struct ev_loop *loop)
: loop_(loop), sv_ssl_ctx_(nullptr), cl_ssl_ctx_(nullptr),
// rate_limit_group_(bufferevent_rate_limit_group_new(
// evbase, get_config()->worker_rate_limit_cfg)),
worker_stat_(util::make_unique<WorkerStat>()),
@ -74,6 +73,11 @@ ConnectionHandler::~ConnectionHandler() {
ev_timer_stop(loop_, &disable_acceptor_timer_);
}
void ConnectionHandler::create_ssl_context() {
sv_ssl_ctx_ = ssl::setup_server_ssl_context();
cl_ssl_ctx_ = ssl::setup_client_ssl_context();
}
void ConnectionHandler::worker_reopen_log_files() {
WorkerEvent wev;
@ -104,6 +108,7 @@ void ConnectionHandler::create_worker_thread(size_t num) {
for (size_t i = 0; i < num; ++i) {
workers_.push_back(util::make_unique<Worker>(sv_ssl_ctx_, cl_ssl_ctx_,
worker_config->cert_tree,
worker_config->ticket_keys));
if (LOG_ENABLED(INFO)) {

4
src/shrpx_connection_handler.h
View File

@ -51,10 +51,10 @@ struct TicketKeys;
// TODO should be renamed as ConnectionHandler
class ConnectionHandler {
public:
ConnectionHandler(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx,
SSL_CTX *cl_ssl_ctx);
ConnectionHandler(struct ev_loop *loop);
~ConnectionHandler();
int handle_connection(int fd, sockaddr *addr, int addrlen);
void create_ssl_context();
void create_worker_thread(size_t num);
void worker_reopen_log_files();
void worker_renew_ticket_keys(const std::shared_ptr<TicketKeys> &ticket_keys);

51
src/shrpx_ssl.cc
View File

@ -129,11 +129,12 @@ int ssl_pem_passwd_cb(char *buf, int size, int rwflag, void *user_data) {
namespace {
int servername_callback(SSL *ssl, int *al, void *arg) {
if (get_config()->cert_tree) {
auto cert_tree = worker_config->cert_tree;
if (cert_tree) {
const char *hostname = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name);
if (hostname) {
auto ssl_ctx = cert_lookup_tree_lookup(get_config()->cert_tree, hostname,
strlen(hostname));
auto ssl_ctx =
cert_lookup_tree_lookup(cert_tree, hostname, strlen(hostname));
if (ssl_ctx) {
SSL_set_SSL_CTX(ssl, ssl_ctx);
}
@ -943,6 +944,50 @@ bool check_http2_requirement(SSL *ssl) {
return true;
}
SSL_CTX *setup_server_ssl_context() {
if (get_config()->upstream_no_tls) {
return nullptr;
}
auto ssl_ctx = ssl::create_ssl_context(get_config()->private_key_file.get(),
get_config()->cert_file.get());
auto cert_tree =
get_config()->subcerts.empty() ? nullptr : cert_lookup_tree_new();
worker_config->cert_tree = cert_tree;
for (auto &keycert : get_config()->subcerts) {
auto ssl_ctx =
ssl::create_ssl_context(keycert.first.c_str(), keycert.second.c_str());
if (ssl::cert_lookup_tree_add_cert_from_file(
cert_tree, ssl_ctx, keycert.second.c_str()) == -1) {
LOG(FATAL) << "Failed to add sub certificate.";
DIE();
}
}
if (cert_tree) {
if (ssl::cert_lookup_tree_add_cert_from_file(
cert_tree, ssl_ctx, get_config()->cert_file.get()) == -1) {
LOG(FATAL) << "Failed to add default certificate.";
DIE();
}
}
return ssl_ctx;
}
SSL_CTX *setup_client_ssl_context() {
if (get_config()->client_mode) {
return get_config()->downstream_no_tls ? nullptr
: ssl::create_ssl_client_context();
}
return get_config()->http2_bridge && !get_config()->downstream_no_tls
? ssl::create_ssl_client_context()
: nullptr;
}
} // namespace ssl
} // namespace shrpx

11
src/shrpx_ssl.h
View File

@ -141,6 +141,17 @@ long int create_tls_proto_mask(const std::vector<char *> &tls_proto_list);
std::vector<unsigned char> set_alpn_prefs(const std::vector<char *> &protos);
// Setups server side SSL_CTX. This function inspects get_config()
// and if upstream_no_tls is true, returns nullptr. Otherwise
// construct default SSL_CTX. If subcerts are not empty, create
// SSL_CTX for them. All created SSL_CTX are added to CertLookupTree.
SSL_CTX *setup_server_ssl_context();
// Setups client side SSL_CTX. This function inspects get_config()
// and if downstream_no_tls is true, returns nullptr. Otherwise, only
// construct SSL_CTX if either client_mode or http2_bridge is true.
SSL_CTX *setup_client_ssl_context();
} // namespace ssl
} // namespace shrpx

22
src/shrpx_worker.cc
View File

@ -48,6 +48,7 @@ void eventcb(struct ev_loop *loop, ev_async *w, int revents) {
} // namespace
Worker::Worker(SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
ssl::CertLookupTree *cert_tree,
const std::shared_ptr<TicketKeys> &ticket_keys)
: loop_(ev_loop_new(0)), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx),
worker_stat_(util::make_unique<WorkerStat>()) {
@ -55,14 +56,21 @@ Worker::Worker(SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
w_.data = this;
ev_async_start(loop_, &w_);
if (get_config()->downstream_proto == PROTO_HTTP2) {
http2session_ = util::make_unique<Http2Session>(loop_, cl_ssl_ctx_);
} else {
http1_connect_blocker_ = util::make_unique<ConnectBlocker>(loop_);
}
#ifndef NOTHREADS
fut_ = std::async(std::launch::async, [this, &ticket_keys] {
fut_ = std::async(std::launch::async, [this, cert_tree, &ticket_keys] {
if (get_config()->tls_ctx_per_worker) {
sv_ssl_ctx_ = ssl::setup_server_ssl_context();
cl_ssl_ctx_ = ssl::setup_client_ssl_context();
} else {
worker_config->cert_tree = cert_tree;
}
if (get_config()->downstream_proto == PROTO_HTTP2) {
http2session_ = util::make_unique<Http2Session>(loop_, cl_ssl_ctx_);
} else {
http1_connect_blocker_ = util::make_unique<ConnectBlocker>(loop_);
}
worker_config->ticket_keys = ticket_keys;
(void)reopen_log_files();
ev_run(loop_);

5
src/shrpx_worker.h
View File

@ -47,6 +47,10 @@ namespace shrpx {
class Http2Session;
class ConnectBlocker;
namespace ssl {
struct CertLookupTree;
} // namespace ssl
struct WorkerStat {
WorkerStat() : num_connections(0), next_downstream(0) {}
@ -77,6 +81,7 @@ struct WorkerEvent {
class Worker {
public:
Worker(SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
ssl::CertLookupTree *cert_tree,
const std::shared_ptr<TicketKeys> &ticket_keys);
~Worker();
void wait();

4
src/shrpx_worker_config.cc
View File

@ -27,8 +27,8 @@
namespace shrpx {
WorkerConfig::WorkerConfig()
: accesslog_fd(-1), errorlog_fd(-1), errorlog_tty(false),
graceful_shutdown(false) {}
: cert_tree(nullptr), accesslog_fd(-1), errorlog_fd(-1),
errorlog_tty(false), graceful_shutdown(false) {}
#ifndef NOTHREADS
thread_local

5
src/shrpx_worker_config.h
View File

@ -29,10 +29,15 @@
namespace shrpx {
namespace ssl {
struct CertLookupTree;
} // namespace ssl
struct TicketKeys;
struct WorkerConfig {
std::shared_ptr<TicketKeys> ticket_keys;
ssl::CertLookupTree *cert_tree;
int accesslog_fd;
int errorlog_fd;
// true if errorlog_fd is referring to a terminal.