diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc
index 9cb5eee0..f5f17dec 100644
--- a/src/shrpx_http_downstream_connection.cc
+++ b/src/shrpx_http_downstream_connection.cc
@@ -133,7 +133,7 @@ HttpDownstreamConnection::~HttpDownstreamConnection() {
   if (conn_.tls.ssl) {
     auto session = SSL_get1_session(conn_.tls.ssl);
     if (session) {
-      worker_->cache_downstream_tls_session(addr_, session);
+      worker_->cache_downstream_tls_session(&addr_->addr, session);
     }
   }
 }
@@ -218,7 +218,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
           SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str());
         }
 
-        auto session = worker_->reuse_downstream_tls_session(addr_);
+        auto session = worker_->reuse_downstream_tls_session(&addr_->addr);
         if (session) {
           SSL_set_session(conn_.tls.ssl, session);
           SSL_SESSION_free(session);
diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc
index 2204b9af..15083347 100644
--- a/src/shrpx_worker.cc
+++ b/src/shrpx_worker.cc
@@ -307,7 +307,7 @@ mruby::MRubyContext *Worker::get_mruby_context() const {
 }
 #endif // HAVE_MRUBY
 
-void Worker::cache_downstream_tls_session(const DownstreamAddr *addr,
+void Worker::cache_downstream_tls_session(const Address *addr,
                                           SSL_SESSION *session) {
   auto &tlsconf = get_config()->tls;
 
@@ -341,7 +341,7 @@ void Worker::cache_downstream_tls_session(const DownstreamAddr *addr,
   ++downstream_tls_session_cache_size_;
 }
 
-SSL_SESSION *Worker::reuse_downstream_tls_session(const DownstreamAddr *addr) {
+SSL_SESSION *Worker::reuse_downstream_tls_session(const Address *addr) {
   auto it = downstream_tls_session_cache_.find(addr);
   if (it == std::end(downstream_tls_session_cache_)) {
     return nullptr;
diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h
index 3939b4c8..a967caf6 100644
--- a/src/shrpx_worker.h
+++ b/src/shrpx_worker.h
@@ -145,16 +145,15 @@ public:
   mruby::MRubyContext *get_mruby_context() const;
 #endif // HAVE_MRUBY
 
-  // Caches |session| which is associated to downstream address
-  // |addr|.  The caller is responsible to increment the reference
-  // count of |session|, since this function does not do so.
-  void cache_downstream_tls_session(const DownstreamAddr *addr,
-                                    SSL_SESSION *session);
+  // Caches |session| which is associated to remote address |addr|.
+  // The caller is responsible to increment the reference count of
+  // |session|, since this function does not do so.
+  void cache_downstream_tls_session(const Address *addr, SSL_SESSION *session);
   // Returns cached session associated |addr|.  If non-nullptr value
   // is returned, its cache entry was successfully removed from cache.
   // If no cache entry is found associated to |addr|, nullptr will be
   // returned.
-  SSL_SESSION *reuse_downstream_tls_session(const DownstreamAddr *addr);
+  SSL_SESSION *reuse_downstream_tls_session(const Address *addr);
 
 private:
 #ifndef NOTHREADS
@@ -170,12 +169,12 @@ private:
   WorkerStat worker_stat_;
   std::vector<DownstreamGroup> dgrps_;
 
-  // Cache for SSL_SESSION for downstream connections.  SSL_SESSION is
-  // associated to downstream address.  One address has multiple
-  // SSL_SESSION objects.  New SSL_SESSION is appended to the deque.
-  // When doing eviction due to storage limitation, the SSL_SESSION
-  // which sits at the front of deque is removed.
-  std::unordered_map<const DownstreamAddr *, std::deque<SSL_SESSION *>>
+  // Client side SSL_SESSION cache.  SSL_SESSION is associated to
+  // remote address.  One address has multiple SSL_SESSION objects.
+  // New SSL_SESSION is appended to the deque.  When doing eviction
+  // due to storage limitation, the SSL_SESSION which sits at the
+  // front of deque is removed.
+  std::unordered_map<const Address *, std::deque<SSL_SESSION *>>
       downstream_tls_session_cache_;
   size_t downstream_tls_session_cache_size_;