diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index f5f17dec..166269ea 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -131,9 +131,10 @@ HttpDownstreamConnection::HttpDownstreamConnection( HttpDownstreamConnection::~HttpDownstreamConnection() { if (conn_.tls.ssl) { - auto session = SSL_get1_session(conn_.tls.ssl); + auto session = SSL_get0_session(conn_.tls.ssl); if (session) { - worker_->cache_downstream_tls_session(&addr_->addr, session); + worker_->cache_downstream_tls_session(&addr_->addr, session, + ev_now(conn_.loop)); } } } diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 15083347..9521e5ac 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -29,6 +29,7 @@ #endif // HAVE_UNISTD_H #include +#include #include "shrpx_ssl.h" #include "shrpx_log.h" @@ -73,7 +74,6 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, dconn_pool_(get_config()->conn.downstream.addr_groups.size()), worker_stat_(get_config()->conn.downstream.addr_groups.size()), dgrps_(get_config()->conn.downstream.addr_groups.size()), - downstream_tls_session_cache_size_(0), loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), @@ -117,12 +117,6 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, Worker::~Worker() { ev_async_stop(loop_, &w_); ev_timer_stop(loop_, &mcpool_clear_timer_); - - for (auto &p : downstream_tls_session_cache_) { - for (auto session : p.second) { - SSL_SESSION_free(session); - } - } } void Worker::schedule_clear_mcpool() { @@ -307,38 +301,51 @@ mruby::MRubyContext *Worker::get_mruby_context() const { } #endif // HAVE_MRUBY -void Worker::cache_downstream_tls_session(const Address *addr, - SSL_SESSION *session) { - auto &tlsconf = get_config()->tls; +namespace { +std::vector serialize_ssl_session(SSL_SESSION *session) { + auto len = i2d_SSL_SESSION(session, nullptr); + auto buf = std::vector(len); + auto p = buf.data(); + i2d_SSL_SESSION(session, &p); - auto max = tlsconf.downstream_session_cache_per_worker; - if (max == 0) { + return buf; +} +} // namespace + +void Worker::cache_downstream_tls_session(const Address *addr, + SSL_SESSION *session, ev_tstamp t) { + auto it = downstream_tls_session_cache_.find(addr); + if (it == std::end(downstream_tls_session_cache_)) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Create cache entry for SSL_SESSION=" << session + << ", addr=" << util::numeric_hostport(&addr->su.sa, addr->len) + << "(" << addr << "), timestamp=" << std::fixed + << std::setprecision(6) << t; + } + downstream_tls_session_cache_.emplace( + addr, SessionCacheEntry{serialize_ssl_session(session), t}); return; } - if (downstream_tls_session_cache_size_ >= max) { - // It is implementation dependent which item is returned from - // std::begin(). Probably, this depends on hash algorithm. If it - // is random fashion, then we are mostly OK. - auto it = std::begin(downstream_tls_session_cache_); - assert(it != std::end(downstream_tls_session_cache_)); - auto &v = (*it).second; - assert(!v.empty()); - auto sess = v.front(); - v.pop_front(); - SSL_SESSION_free(sess); - if (v.empty()) { - downstream_tls_session_cache_.erase(it); + auto &ent = (*it).second; + if (ent.last_updated + 1_min > t) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Cache for addr=" + << util::numeric_hostport(&addr->su.sa, addr->len) << "(" + << addr << ") is still host. Not updating."; } + return; } - auto it = downstream_tls_session_cache_.find(addr); - if (it == std::end(downstream_tls_session_cache_)) { - std::tie(it, std::ignore) = downstream_tls_session_cache_.emplace( - addr, std::deque()); + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Update cache entry for SSL_SESSION=" << session + << ", addr=" << util::numeric_hostport(&addr->su.sa, addr->len) + << "(" << addr << "), timestamp=" << std::fixed + << std::setprecision(6) << t; } - (*it).second.push_back(session); - ++downstream_tls_session_cache_size_; + + ent.session_data = serialize_ssl_session(session); + ent.last_updated = t; } SSL_SESSION *Worker::reuse_downstream_tls_session(const Address *addr) { @@ -347,17 +354,9 @@ SSL_SESSION *Worker::reuse_downstream_tls_session(const Address *addr) { return nullptr; } - auto &v = (*it).second; - assert(!v.empty()); - auto session = v.back(); - v.pop_back(); - --downstream_tls_session_cache_size_; - - if (v.empty()) { - downstream_tls_session_cache_.erase(it); - } - - return session; + const auto &ent = (*it).second; + auto p = ent.session_data.data(); + return d2i_SSL_SESSION(nullptr, &p, ent.session_data.size()); } } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index a967caf6..e4861c03 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -101,6 +101,14 @@ struct WorkerEvent { std::shared_ptr ticket_keys; }; +struct SessionCacheEntry { + // ASN1 representation of SSL_SESSION object. See + // i2d_SSL_SESSION(3SSL). + std::vector session_data; + // The last time stamp when this cache entry is created or updated. + ev_tstamp last_updated; +}; + class Worker { public: Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, @@ -146,13 +154,13 @@ public: #endif // HAVE_MRUBY // Caches |session| which is associated to remote address |addr|. - // The caller is responsible to increment the reference count of - // |session|, since this function does not do so. - void cache_downstream_tls_session(const Address *addr, SSL_SESSION *session); - // Returns cached session associated |addr|. If non-nullptr value - // is returned, its cache entry was successfully removed from cache. - // If no cache entry is found associated to |addr|, nullptr will be - // returned. + // |session| is serialized into ASN1 representation, and stored. + // |t| is used as a time stamp. Depending on the existing cache's + // time stamp, |session| might not be cached. + void cache_downstream_tls_session(const Address *addr, SSL_SESSION *session, + ev_tstamp t); + // Returns cached session associated |addr|. If no cache entry is + // found associated to |addr|, nullptr will be returned. SSL_SESSION *reuse_downstream_tls_session(const Address *addr); private: @@ -170,13 +178,9 @@ private: std::vector dgrps_; // Client side SSL_SESSION cache. SSL_SESSION is associated to - // remote address. One address has multiple SSL_SESSION objects. - // New SSL_SESSION is appended to the deque. When doing eviction - // due to storage limitation, the SSL_SESSION which sits at the - // front of deque is removed. - std::unordered_map> + // remote address. + std::unordered_map downstream_tls_session_cache_; - size_t downstream_tls_session_cache_size_; std::unique_ptr session_cache_memcached_dispatcher_; #ifdef HAVE_MRUBY diff --git a/src/util.cc b/src/util.cc index b2d0ca1c..2f120887 100644 --- a/src/util.cc +++ b/src/util.cc @@ -651,6 +651,41 @@ std::string numeric_name(const struct sockaddr *sa, socklen_t salen) { return host.data(); } +std::string numeric_hostport(const struct sockaddr *sa, socklen_t salen) { + if (sa->sa_family == AF_UNIX) { + return "localhost"; + } + + std::array host; + std::array serv; + auto rv = getnameinfo(sa, salen, host.data(), host.size(), serv.data(), + serv.size(), NI_NUMERICHOST | NI_NUMERICSERV); + if (rv != 0) { + return "unknown"; + } + + auto hostlen = strlen(host.data()); + auto servlen = strlen(serv.data()); + + std::string s; + char *p; + if (sa->sa_family == AF_INET6) { + s.resize(hostlen + servlen + 2 + 1); + p = &s[0]; + *p++ = '['; + p = std::copy_n(host.data(), hostlen, p); + *p++ = ']'; + } else { + s.resize(hostlen + servlen + 1); + p = &s[0]; + p = std::copy_n(host.data(), hostlen, p); + } + *p++ = ':'; + std::copy_n(serv.data(), servlen, p); + + return s; +} + static int STDERR_COPY = -1; static int STDOUT_COPY = -1; diff --git a/src/util.h b/src/util.h index 52b6eb02..ad3a60dc 100644 --- a/src/util.h +++ b/src/util.h @@ -459,6 +459,11 @@ bool numeric_host(const char *hostname, int family); // failed, "unknown" is returned. std::string numeric_name(const struct sockaddr *sa, socklen_t salen); +// Returns string representation of numeric address and port of |addr| +// of length |salen|. The format is like :. For IPv6 +// address, address is enclosed by square brackets ([]). +std::string numeric_hostport(const struct sockaddr *sa, socklen_t salen); + // Makes internal copy of stderr (and possibly stdout in the future), // which is then used as pointer to /dev/stderr or /proc/self/fd/2 void store_original_fds();