diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc
index 53787bb2..30a9f4c9 100644
--- a/src/shrpx_client_handler.cc
+++ b/src/shrpx_client_handler.cc
@@ -361,7 +361,13 @@ void ClientHandler::set_bev_cb
 void ClientHandler::set_upstream_timeouts(const timeval *read_timeout,
                                           const timeval *write_timeout)
 {
-  bufferevent_set_timeouts(bev_, read_timeout, write_timeout);
+  auto bev = bufferevent_get_underlying(bev_);
+
+  if(!bev) {
+    bev = bev_;
+  }
+
+  bufferevent_set_timeouts(bev, read_timeout, write_timeout);
 }
 
 int ClientHandler::validate_next_proto()