From 26d49c1dc39ea0f22d23eb6a1b62ea79f0e6cd5a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 6 Feb 2016 23:07:00 +0900 Subject: [PATCH] nghttpx: Cache client session --- src/shrpx_client_handler.cc | 5 +-- src/shrpx_http_downstream_connection.cc | 43 ++++++++++++-------- src/shrpx_http_downstream_connection.h | 5 ++- src/shrpx_worker.cc | 52 +++++++++++++++++++++++++ src/shrpx_worker.h | 8 ++++ 5 files changed, 92 insertions(+), 21 deletions(-) diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 1a5b06b9..30f2fc4e 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -722,9 +722,8 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { } dconn = make_unique(dconn_pool, http2session); } else { - dconn = make_unique( - dconn_pool, group, conn_.loop, worker_->get_cl_ssl_ctx(), - worker_->get_mcpool()); + dconn = make_unique(dconn_pool, group, + conn_.loop, worker_); } dconn->set_client_handler(this); return dconn; diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index c2fd17f8..532de0dd 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -113,23 +113,34 @@ void connectcb(struct ev_loop *loop, ev_io *w, int revents) { HttpDownstreamConnection::HttpDownstreamConnection( DownstreamConnectionPool *dconn_pool, size_t group, struct ev_loop *loop, - SSL_CTX *ssl_ctx, MemchunkPool *mcpool) + Worker *worker) : DownstreamConnection(dconn_pool), - conn_(loop, -1, nullptr, mcpool, + conn_(loop, -1, nullptr, worker->get_mcpool(), get_config()->conn.downstream.timeout.write, get_config()->conn.downstream.timeout.read, {}, {}, connectcb, readcb, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold, get_config()->tls.dyn_rec.idle_timeout), do_read_(&HttpDownstreamConnection::noop), do_write_(&HttpDownstreamConnection::noop), - ssl_ctx_(ssl_ctx), + worker_(worker), + ssl_ctx_(worker->get_cl_ssl_ctx()), ioctrl_(&conn_.rlimit), response_htp_{0}, group_(group), addr_idx_(0), connected_(false) {} -HttpDownstreamConnection::~HttpDownstreamConnection() {} +HttpDownstreamConnection::~HttpDownstreamConnection() { + if (conn_.tls.ssl) { + auto session = SSL_get1_session(conn_.tls.ssl); + if (session) { + auto &downstreamconf = get_config()->conn.downstream; + auto &addr = downstreamconf.addr_groups[group_].addrs[addr_idx_]; + + worker_->cache_cl_tls_session(&addr, session); + } + } +} int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { if (LOG_ENABLED(INFO)) { @@ -149,22 +160,16 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { return -1; } - // TODO This does not do any good, since once connection is - // dropped, HttpDownstreamConnection is destroyed. We need to - // cache SSL session somewhere, possibly worker scope cache. if (ssl_ctx_) { - if (!conn_.tls.ssl) { - auto ssl = ssl::create_ssl(ssl_ctx_); - if (!ssl) { - return -1; - } - - conn_.set_ssl(ssl); + auto ssl = ssl::create_ssl(ssl_ctx_); + if (!ssl) { + return -1; } + + conn_.set_ssl(ssl); } - auto worker = client_handler_->get_worker(); - auto &next_downstream = worker->get_dgrp(group_)->next; + auto &next_downstream = worker_->get_dgrp(group_)->next; auto end = next_downstream; auto &addrs = downstreamconf.addr_groups[group_].addrs; for (;;) { @@ -217,6 +222,12 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str()); } + auto session = worker_->reuse_cl_tls_session(&addrs[i]); + if (session) { + SSL_set_session(conn_.tls.ssl, session); + SSL_SESSION_free(session); + } + conn_.prepare_client_handshake(); } diff --git a/src/shrpx_http_downstream_connection.h b/src/shrpx_http_downstream_connection.h index 8b1dabab..24f678b2 100644 --- a/src/shrpx_http_downstream_connection.h +++ b/src/shrpx_http_downstream_connection.h @@ -36,12 +36,12 @@ namespace shrpx { class DownstreamConnectionPool; +class Worker; class HttpDownstreamConnection : public DownstreamConnection { public: HttpDownstreamConnection(DownstreamConnectionPool *dconn_pool, size_t group, - struct ev_loop *loop, SSL_CTX *ssl_ctx, - MemchunkPool *mcpool); + struct ev_loop *loop, Worker *worker); virtual ~HttpDownstreamConnection(); virtual int attach_downstream(Downstream *downstream); virtual void detach_downstream(Downstream *downstream); @@ -83,6 +83,7 @@ private: size_t buflen)> read_; std::function write_; + Worker *worker_; // nullptr if TLS is not used. SSL_CTX *ssl_ctx_; IOControl ioctrl_; diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 255d23aa..a4ee502e 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -116,6 +116,12 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, Worker::~Worker() { ev_async_stop(loop_, &w_); ev_timer_stop(loop_, &mcpool_clear_timer_); + + for (auto &p : cl_tls_session_cache_) { + for (auto session : p.second) { + SSL_SESSION_free(session); + } + } } void Worker::schedule_clear_mcpool() { @@ -300,4 +306,50 @@ mruby::MRubyContext *Worker::get_mruby_context() const { } #endif // HAVE_MRUBY +void Worker::cache_cl_tls_session(const DownstreamAddr *addr, + SSL_SESSION *session) { + if (cl_tls_session_order_.size() >= 10000) { + auto addrkey = cl_tls_session_order_.front(); + cl_tls_session_order_.pop_front(); + auto it = cl_tls_session_cache_.find(addrkey); + assert(it != std::end(cl_tls_session_cache_)); + auto &v = (*it).second; + assert(!v.empty()); + auto sess = v.front(); + SSL_SESSION_free(sess); + v.pop_front(); + if (v.empty()) { + cl_tls_session_cache_.erase(it); + } + } + + cl_tls_session_order_.push_back(addr); + auto it = cl_tls_session_cache_.find(addr); + if (it == std::end(cl_tls_session_cache_)) { + std::tie(it, std::ignore) = + cl_tls_session_cache_.emplace(addr, std::deque()); + } + (*it).second.push_back(session); +} + +SSL_SESSION *Worker::reuse_cl_tls_session(const DownstreamAddr *addr) { + auto it = cl_tls_session_cache_.find(addr); + if (it == std::end(cl_tls_session_cache_)) { + return nullptr; + } + + assert((*it).first == cl_tls_session_order_.back()); + + auto &v = (*it).second; + assert(!v.empty()); + auto session = v.back(); + v.pop_back(); + if (v.empty()) { + cl_tls_session_cache_.erase(it); + } + cl_tls_session_order_.pop_back(); + + return session; +} + } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 367416f6..74867e71 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -30,6 +30,8 @@ #include #include #include +#include +#include #include #ifndef NOTHREADS #include @@ -143,6 +145,9 @@ public: mruby::MRubyContext *get_mruby_context() const; #endif // HAVE_MRUBY + void cache_cl_tls_session(const DownstreamAddr *addr, SSL_SESSION *session); + SSL_SESSION *reuse_cl_tls_session(const DownstreamAddr *addr); + private: #ifndef NOTHREADS std::future fut_; @@ -156,6 +161,9 @@ private: DownstreamConnectionPool dconn_pool_; WorkerStat worker_stat_; std::vector dgrps_; + std::unordered_map> + cl_tls_session_cache_; + std::deque cl_tls_session_order_; std::unique_ptr session_cache_memcached_dispatcher_; #ifdef HAVE_MRUBY std::unique_ptr mruby_ctx_;