nghttpx: Cache client side session inside openssl callback

src/shrpx_connection.h

@@ -41,6 +41,10 @@ namespace shrpx {
 
 struct MemcachedRequest;
 
+namespace ssl {
+struct TLSSessionCache;
+} // namespace ssl
+
 enum {
   TLS_CONN_NORMAL,
   TLS_CONN_WAIT_FOR_SESSION_CACHE,
@@ -55,6 +59,7 @@ struct TLSConnection {
   SSL *ssl;
   SSL_SESSION *cached_session;
   MemcachedRequest *cached_session_lookup_req;
+  ssl::TLSSessionCache *client_session_cache;
   ev_tstamp last_write_idle;
   size_t warmup_writelen;
   // length passed to SSL_write and SSL_read last time.  This is

src/shrpx_http2_session.cc

@@ -437,6 +437,7 @@ int Http2Session::initiate_connection() {
         ssl::setup_downstream_http2_alpn(ssl);
 
         conn_.set_ssl(ssl);
+        conn_.tls.client_session_cache = &addr_->tls_session_cache;
 
         auto sni_name =
             addr_->sni.empty() ? StringRef{addr_->host} : StringRef{addr_->sni};
@@ -2076,14 +2077,6 @@ int Http2Session::tls_handshake() {
     return -1;
   }
 
-  if (!SSL_session_reused(conn_.tls.ssl)) {
-    auto tls_session = SSL_get0_session(conn_.tls.ssl);
-    if (tls_session) {
-      ssl::try_cache_tls_session(addr_->tls_session_cache, *raddr_, tls_session,
-                                 ev_now(conn_.loop));
-    }
-  }
-
   read_ = &Http2Session::read_tls;
   write_ = &Http2Session::write_tls;
 

src/shrpx_http_downstream_connection.cc

@@ -431,6 +431,7 @@ int HttpDownstreamConnection::initiate_connection() {
         ssl::setup_downstream_http1_alpn(ssl);
 
         conn_.set_ssl(ssl);
+        conn_.tls.client_session_cache = &addr_->tls_session_cache;
 
         auto sni_name =
             addr_->sni.empty() ? StringRef{addr_->host} : StringRef{addr_->sni};
@@ -1229,14 +1230,6 @@ int HttpDownstreamConnection::tls_handshake() {
     return -1;
   }
 
-  if (!SSL_session_reused(conn_.tls.ssl)) {
-    auto session = SSL_get0_session(conn_.tls.ssl);
-    if (session) {
-      ssl::try_cache_tls_session(addr_->tls_session_cache, *raddr_, session,
-                                 ev_now(conn_.loop));
-    }
-  }
-
   auto &connect_blocker = addr_->connect_blocker;
 
   signal_write_ = &HttpDownstreamConnection::actual_signal_write;

src/shrpx_live_check.cc

@@ -222,6 +222,7 @@ int LiveCheck::initiate_connection() {
     }
 
     conn_.set_ssl(ssl);
+    conn_.tls.client_session_cache = &addr_->tls_session_cache;
   }
 
   if (addr_->dns) {
@@ -400,14 +401,6 @@ int LiveCheck::tls_handshake() {
     return -1;
   }
 
-  if (!SSL_session_reused(conn_.tls.ssl)) {
-    auto tls_session = SSL_get0_session(conn_.tls.ssl);
-    if (tls_session) {
-      ssl::try_cache_tls_session(addr_->tls_session_cache, *raddr_, tls_session,
-                                 ev_now(conn_.loop));
-    }
-  }
-
   // Check negotiated ALPN
 
   const unsigned char *next_proto = nullptr;

src/shrpx_memcached_connection.cc

@@ -155,6 +155,7 @@ int MemcachedConnection::initiate_connection() {
       return -1;
     }
     conn_.set_ssl(ssl);
+    conn_.tls.client_session_cache = &tls_session_cache_;
   }
 
   conn_.fd = util::create_nonblock_socket(addr_->su.storage.ss_family);
@@ -280,14 +281,6 @@ int MemcachedConnection::tls_handshake() {
     return -1;
   }
 
-  if (!SSL_session_reused(conn_.tls.ssl)) {
-    auto tls_session = SSL_get0_session(conn_.tls.ssl);
-    if (tls_session) {
-      ssl::try_cache_tls_session(tls_session_cache_, *addr_, tls_session,
-                                 ev_now(conn_.loop));
-    }
-  }
-
   ev_timer_stop(conn_.loop, &conn_.rt);
   ev_timer_stop(conn_.loop, &conn_.wt);
 

src/shrpx_ssl.cc

@@ -261,6 +261,20 @@ int ocsp_resp_cb(SSL *ssl, void *arg) {
 constexpr auto MEMCACHED_SESSION_CACHE_KEY_PREFIX =
     StringRef::from_lit("nghttpx:tls-session-cache:");
 
+namespace {
+int tls_session_client_new_cb(SSL *ssl, SSL_SESSION *session) {
+  auto conn = static_cast<Connection *>(SSL_get_app_data(ssl));
+  if (conn->tls.client_session_cache == nullptr) {
+    return 0;
+  }
+
+  try_cache_tls_session(conn->tls.client_session_cache, session,
+                        ev_now(conn->loop));
+
+  return 0;
+}
+} // namespace
+
 namespace {
 int tls_session_new_cb(SSL *ssl, SSL_SESSION *session) {
   auto conn = static_cast<Connection *>(SSL_get_app_data(ssl));
@@ -917,6 +931,10 @@ SSL_CTX *create_ssl_client_context(
 
   SSL_CTX_set_options(ssl_ctx, ssl_opts | tlsconf.tls_proto_mask);
 
+  SSL_CTX_set_session_cache_mode(ssl_ctx, SSL_SESS_CACHE_CLIENT |
+                                              SSL_SESS_CACHE_NO_INTERNAL_STORE);
+  SSL_CTX_sess_set_new_cb(ssl_ctx, tls_session_client_new_cb);
+
   if (nghttp2::ssl::ssl_ctx_set_proto_versions(
           ssl_ctx, tlsconf.min_proto_version, tlsconf.max_proto_version) != 0) {
     LOG(FATAL) << "Could not set TLS protocol version";
@@ -1678,24 +1696,22 @@ std::vector<uint8_t> serialize_ssl_session(SSL_SESSION *session) {
 }
 } // namespace
 
-void try_cache_tls_session(TLSSessionCache &cache, const Address &addr,
-                           SSL_SESSION *session, ev_tstamp t) {
-  if (cache.last_updated + 1_min > t) {
+void try_cache_tls_session(TLSSessionCache *cache, SSL_SESSION *session,
+                           ev_tstamp t) {
+  if (cache->last_updated + 1_min > t) {
     if (LOG_ENABLED(INFO)) {
-      LOG(INFO) << "Cache for addr=" << util::to_numeric_addr(&addr)
-                << " is still host.  Not updating.";
+      LOG(INFO) << "Client session cache entry is still fresh.";
     }
     return;
   }
 
   if (LOG_ENABLED(INFO)) {
-    LOG(INFO) << "Update cache entry for SSL_SESSION=" << session
-              << ", addr=" << util::to_numeric_addr(&addr)
-              << ", timestamp=" << std::fixed << std::setprecision(6) << t;
+    LOG(INFO) << "Update client cache entry "
+              << "timestamp = " << std::fixed << std::setprecision(6) << t;
   }
 
-  cache.session_data = serialize_ssl_session(session);
-  cache.last_updated = t;
+  cache->session_data = serialize_ssl_session(session);
+  cache->last_updated = t;
 }
 
 SSL_SESSION *reuse_tls_session(const TLSSessionCache &cache) {

src/shrpx_ssl.h

@@ -244,12 +244,12 @@ bool upstream_tls_enabled(const ConnectionConfig &connconf);
 // is based on RFC 6125.
 bool tls_hostname_match(const StringRef &pattern, const StringRef &hostname);
 
-// Caches |session| which is associated to remote address |addr|.
-// |session| is serialized into ASN1 representation, and stored.  |t|
-// is used as a time stamp.  Depending on the existing cache's time
-// stamp, |session| might not be cached.
-void try_cache_tls_session(TLSSessionCache &cache, const Address &addr,
-                           SSL_SESSION *session, ev_tstamp t);
+// Caches |session|.  |session| is serialized into ASN1
+// representation, and stored.  |t| is used as a time stamp.
+// Depending on the existing cache's time stamp, |session| might not
+// be cached.
+void try_cache_tls_session(TLSSessionCache *cache, SSL_SESSION *session,
+                           ev_tstamp t);
 
 // Returns cached session associated |addr|.  If no cache entry is
 // found associated to |addr|, nullptr will be returned.