nghttpx: Cache client session

src/shrpx_client_handler.cc

@@ -722,9 +722,8 @@ ClientHandler::get_downstream_connection(Downstream *downstream) {
       }
       dconn = make_unique<Http2DownstreamConnection>(dconn_pool, http2session);
     } else {
-      dconn = make_unique<HttpDownstreamConnection>(
-          dconn_pool, group, conn_.loop, worker_->get_cl_ssl_ctx(),
-          worker_->get_mcpool());
+      dconn = make_unique<HttpDownstreamConnection>(dconn_pool, group,
+                                                    conn_.loop, worker_);
     }
     dconn->set_client_handler(this);
     return dconn;

src/shrpx_http_downstream_connection.cc

@@ -113,23 +113,34 @@ void connectcb(struct ev_loop *loop, ev_io *w, int revents) {
 
 HttpDownstreamConnection::HttpDownstreamConnection(
     DownstreamConnectionPool *dconn_pool, size_t group, struct ev_loop *loop,
-    SSL_CTX *ssl_ctx, MemchunkPool *mcpool)
+    Worker *worker)
     : DownstreamConnection(dconn_pool),
-      conn_(loop, -1, nullptr, mcpool,
+      conn_(loop, -1, nullptr, worker->get_mcpool(),
             get_config()->conn.downstream.timeout.write,
             get_config()->conn.downstream.timeout.read, {}, {}, connectcb,
             readcb, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold,
             get_config()->tls.dyn_rec.idle_timeout),
       do_read_(&HttpDownstreamConnection::noop),
       do_write_(&HttpDownstreamConnection::noop),
-      ssl_ctx_(ssl_ctx),
+      worker_(worker),
+      ssl_ctx_(worker->get_cl_ssl_ctx()),
       ioctrl_(&conn_.rlimit),
       response_htp_{0},
       group_(group),
       addr_idx_(0),
       connected_(false) {}
 
-HttpDownstreamConnection::~HttpDownstreamConnection() {}
+HttpDownstreamConnection::~HttpDownstreamConnection() {
+  if (conn_.tls.ssl) {
+    auto session = SSL_get1_session(conn_.tls.ssl);
+    if (session) {
+      auto &downstreamconf = get_config()->conn.downstream;
+      auto &addr = downstreamconf.addr_groups[group_].addrs[addr_idx_];
+
+      worker_->cache_cl_tls_session(&addr, session);
+    }
+  }
+}
 
 int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
   if (LOG_ENABLED(INFO)) {
@@ -149,11 +160,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
       return -1;
     }
 
-    // TODO This does not do any good, since once connection is
-    // dropped, HttpDownstreamConnection is destroyed.  We need to
-    // cache SSL session somewhere, possibly worker scope cache.
     if (ssl_ctx_) {
-      if (!conn_.tls.ssl) {
       auto ssl = ssl::create_ssl(ssl_ctx_);
       if (!ssl) {
         return -1;
@@ -161,10 +168,8 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
 
       conn_.set_ssl(ssl);
     }
-    }
 
-    auto worker = client_handler_->get_worker();
-    auto &next_downstream = worker->get_dgrp(group_)->next;
+    auto &next_downstream = worker_->get_dgrp(group_)->next;
     auto end = next_downstream;
     auto &addrs = downstreamconf.addr_groups[group_].addrs;
     for (;;) {
@@ -217,6 +222,12 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
           SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str());
         }
 
+        auto session = worker_->reuse_cl_tls_session(&addrs[i]);
+        if (session) {
+          SSL_set_session(conn_.tls.ssl, session);
+          SSL_SESSION_free(session);
+        }
+
         conn_.prepare_client_handshake();
       }
 

src/shrpx_http_downstream_connection.h

@@ -36,12 +36,12 @@
 namespace shrpx {
 
 class DownstreamConnectionPool;
+class Worker;
 
 class HttpDownstreamConnection : public DownstreamConnection {
 public:
   HttpDownstreamConnection(DownstreamConnectionPool *dconn_pool, size_t group,
-                           struct ev_loop *loop, SSL_CTX *ssl_ctx,
-                           MemchunkPool *mcpool);
+                           struct ev_loop *loop, Worker *worker);
   virtual ~HttpDownstreamConnection();
   virtual int attach_downstream(Downstream *downstream);
   virtual void detach_downstream(Downstream *downstream);
@@ -83,6 +83,7 @@ private:
                         size_t buflen)> read_;
   std::function<ssize_t(HttpDownstreamConnection &, struct iovec *iov,
                         size_t iovlen)> write_;
+  Worker *worker_;
   // nullptr if TLS is not used.
   SSL_CTX *ssl_ctx_;
   IOControl ioctrl_;

src/shrpx_worker.cc

@@ -116,6 +116,12 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
 Worker::~Worker() {
   ev_async_stop(loop_, &w_);
   ev_timer_stop(loop_, &mcpool_clear_timer_);
+
+  for (auto &p : cl_tls_session_cache_) {
+    for (auto session : p.second) {
+      SSL_SESSION_free(session);
+    }
+  }
 }
 
 void Worker::schedule_clear_mcpool() {
@@ -300,4 +306,50 @@ mruby::MRubyContext *Worker::get_mruby_context() const {
 }
 #endif // HAVE_MRUBY
 
+void Worker::cache_cl_tls_session(const DownstreamAddr *addr,
+                                  SSL_SESSION *session) {
+  if (cl_tls_session_order_.size() >= 10000) {
+    auto addrkey = cl_tls_session_order_.front();
+    cl_tls_session_order_.pop_front();
+    auto it = cl_tls_session_cache_.find(addrkey);
+    assert(it != std::end(cl_tls_session_cache_));
+    auto &v = (*it).second;
+    assert(!v.empty());
+    auto sess = v.front();
+    SSL_SESSION_free(sess);
+    v.pop_front();
+    if (v.empty()) {
+      cl_tls_session_cache_.erase(it);
+    }
+  }
+
+  cl_tls_session_order_.push_back(addr);
+  auto it = cl_tls_session_cache_.find(addr);
+  if (it == std::end(cl_tls_session_cache_)) {
+    std::tie(it, std::ignore) =
+        cl_tls_session_cache_.emplace(addr, std::deque<SSL_SESSION *>());
+  }
+  (*it).second.push_back(session);
+}
+
+SSL_SESSION *Worker::reuse_cl_tls_session(const DownstreamAddr *addr) {
+  auto it = cl_tls_session_cache_.find(addr);
+  if (it == std::end(cl_tls_session_cache_)) {
+    return nullptr;
+  }
+
+  assert((*it).first == cl_tls_session_order_.back());
+
+  auto &v = (*it).second;
+  assert(!v.empty());
+  auto session = v.back();
+  v.pop_back();
+  if (v.empty()) {
+    cl_tls_session_cache_.erase(it);
+  }
+  cl_tls_session_order_.pop_back();
+
+  return session;
+}
+
 } // namespace shrpx

src/shrpx_worker.h

@@ -30,6 +30,8 @@
 #include <mutex>
 #include <vector>
 #include <random>
+#include <unordered_map>
+#include <deque>
 #include <thread>
 #ifndef NOTHREADS
 #include <future>
@@ -143,6 +145,9 @@ public:
   mruby::MRubyContext *get_mruby_context() const;
 #endif // HAVE_MRUBY
 
+  void cache_cl_tls_session(const DownstreamAddr *addr, SSL_SESSION *session);
+  SSL_SESSION *reuse_cl_tls_session(const DownstreamAddr *addr);
+
 private:
 #ifndef NOTHREADS
   std::future<void> fut_;
@@ -156,6 +161,9 @@ private:
   DownstreamConnectionPool dconn_pool_;
   WorkerStat worker_stat_;
   std::vector<DownstreamGroup> dgrps_;
+  std::unordered_map<const DownstreamAddr *, std::deque<SSL_SESSION *>>
+      cl_tls_session_cache_;
+  std::deque<const DownstreamAddr *> cl_tls_session_order_;
   std::unique_ptr<MemcachedDispatcher> session_cache_memcached_dispatcher_;
 #ifdef HAVE_MRUBY
   std::unique_ptr<mruby::MRubyContext> mruby_ctx_;