diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc
index 8b82db5f..bce7e0db 100644
--- a/src/shrpx_connection_handler.cc
+++ b/src/shrpx_connection_handler.cc
@@ -130,6 +130,13 @@ ConnectionHandler::~ConnectionHandler() {
     }
     SSL_CTX_free(ssl_ctx);
   }
+
+  // Free workers before destroying ev_loop
+  workers_.clear();
+
+  for (auto loop : worker_loops_) {
+    ev_loop_destroy(loop);
+  }
 }
 
 void ConnectionHandler::set_ticket_keys_to_worker(
@@ -193,6 +200,7 @@ int ConnectionHandler::create_worker_thread(size_t num) {
 #endif // HAVE_MRUBY
 
     workers_.push_back(std::move(worker));
+    worker_loops_.push_back(loop);
 
     if (LOG_ENABLED(INFO)) {
       LLOG(INFO, this) << "Created thread #" << workers_.size() - 1;
diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h
index dc33eb61..8ce25eec 100644
--- a/src/shrpx_connection_handler.h
+++ b/src/shrpx_connection_handler.h
@@ -127,6 +127,8 @@ private:
   // Stores all SSL_CTX objects.
   std::vector<SSL_CTX *> all_ssl_ctx_;
   OCSPUpdateContext ocsp_;
+  // ev_loop for each worker
+  std::vector<struct ev_loop *> worker_loops_;
   // Worker instances when multi threaded mode (-nN, N >= 2) is used.
   std::vector<std::unique_ptr<Worker>> workers_;
   // Worker instance used when single threaded mode (-n1) is used.
diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc
index fadd6494..db236649 100644
--- a/src/shrpx_worker.cc
+++ b/src/shrpx_worker.cc
@@ -104,7 +104,6 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
 Worker::~Worker() {
   ev_async_stop(loop_, &w_);
   ev_timer_stop(loop_, &mcpool_clear_timer_);
-  ev_loop_destroy(loop_);
 }
 
 void Worker::schedule_clear_mcpool() {