diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc
index dc7762d0..2984776c 100644
--- a/src/shrpx_downstream.cc
+++ b/src/shrpx_downstream.cc
@@ -240,6 +240,10 @@ void Downstream::detach_downstream_connection() {
 
   auto handler = dconn_->get_client_handler();
 
+  if (!handler) {
+    return;
+  }
+
   handler->pool_downstream_connection(
       std::unique_ptr<DownstreamConnection>(dconn_.release()));
 }