diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc
index ceebf61b..0ea8f370 100644
--- a/src/shrpx_connection_handler.cc
+++ b/src/shrpx_connection_handler.cc
@@ -117,6 +117,15 @@ ConnectionHandler::ConnectionHandler(struct ev_loop *loop)
 ConnectionHandler::~ConnectionHandler() {
   ev_timer_stop(loop_, &disable_acceptor_timer_);
   ev_timer_stop(loop_, &ocsp_timer_);
+
+  for (auto ssl_ctx : all_ssl_ctx_) {
+    auto tls_ctx_data =
+        static_cast<ssl::TLSContextData *>(SSL_CTX_get_app_data(ssl_ctx));
+    if (tls_ctx_data) {
+      delete tls_ctx_data;
+    }
+    SSL_CTX_free(ssl_ctx);
+  }
 }
 
 void ConnectionHandler::worker_reopen_log_files() {
@@ -148,6 +157,10 @@ void ConnectionHandler::create_single_worker() {
   auto sv_ssl_ctx = ssl::setup_server_ssl_context(all_ssl_ctx_, cert_tree);
   auto cl_ssl_ctx = ssl::setup_client_ssl_context();
 
+  if (cl_ssl_ctx) {
+    all_ssl_ctx_.push_back(cl_ssl_ctx);
+  }
+
   single_worker_ = make_unique<Worker>(loop_, sv_ssl_ctx, cl_ssl_ctx, cert_tree,
                                        ticket_keys_);
 }
@@ -160,6 +173,10 @@ void ConnectionHandler::create_worker_thread(size_t num) {
   auto sv_ssl_ctx = ssl::setup_server_ssl_context(all_ssl_ctx_, cert_tree);
   auto cl_ssl_ctx = ssl::setup_client_ssl_context();
 
+  if (cl_ssl_ctx) {
+    all_ssl_ctx_.push_back(cl_ssl_ctx);
+  }
+
   for (size_t i = 0; i < num; ++i) {
     auto loop = ev_loop_new(0);
 
diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h
index 00e4be49..5b17fef2 100644
--- a/src/shrpx_connection_handler.h
+++ b/src/shrpx_connection_handler.h
@@ -111,6 +111,7 @@ public:
   void proceed_next_cert_ocsp();
 
 private:
+  // Stores all SSL_CTX objects.
   std::vector<SSL_CTX *> all_ssl_ctx_;
   OCSPUpdateContext ocsp_;
   // Worker instances when multi threaded mode (-nN, N >= 2) is used.