diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc
index a557ed78..3052b8e5 100644
--- a/src/shrpx_http2_session.cc
+++ b/src/shrpx_http2_session.cc
@@ -229,6 +229,7 @@ void readcb(bufferevent *bev, void *ptr)
 {
   int rv;
   auto http2session = static_cast<Http2Session*>(ptr);
+  http2session->reset_timeouts();
   rv = http2session->on_read();
   if(rv != 0) {
     http2session->disconnect();
@@ -239,11 +240,12 @@ void readcb(bufferevent *bev, void *ptr)
 namespace {
 void writecb(bufferevent *bev, void *ptr)
 {
+  auto http2session = static_cast<Http2Session*>(ptr);
+  http2session->reset_timeouts();
   if(evbuffer_get_length(bufferevent_get_output(bev)) > 0) {
     return;
   }
   int rv;
-  auto http2session = static_cast<Http2Session*>(ptr);
   rv = http2session->on_write();
   if(rv != 0) {
     http2session->disconnect();
@@ -535,8 +537,7 @@ int Http2Session::initiate_connection()
     bufferevent_enable(bev_, EV_READ);
     bufferevent_setcb(bev_, readcb, writecb, eventcb, this);
     // Set timeout for HTTP2 session
-    bufferevent_set_timeouts(bev_, &get_config()->downstream_read_timeout,
-                             &get_config()->downstream_write_timeout);
+    reset_timeouts();
 
     // We have been already connected when no TLS and proxy is used.
     if(state_ != CONNECTED) {
@@ -1656,4 +1657,10 @@ int Http2Session::consume(int32_t stream_id, size_t len)
   return 0;
 }
 
+void Http2Session::reset_timeouts()
+{
+  bufferevent_set_timeouts(bev_, &get_config()->downstream_read_timeout,
+                           &get_config()->downstream_write_timeout);
+}
+
 } // namespace shrpx
diff --git a/src/shrpx_http2_session.h b/src/shrpx_http2_session.h
index 226585f6..dae01b71 100644
--- a/src/shrpx_http2_session.h
+++ b/src/shrpx_http2_session.h
@@ -106,6 +106,8 @@ public:
 
   int consume(int32_t stream_id, size_t len);
 
+  void reset_timeouts();
+
   enum {
     // Disconnected
     DISCONNECTED,
diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc
index d70ab25f..cf120589 100644
--- a/src/shrpx_http_downstream_connection.cc
+++ b/src/shrpx_http_downstream_connection.cc
@@ -43,12 +43,6 @@ namespace {
 const size_t OUTBUF_MAX_THRES = 64*1024;
 } // namespace
 
-// Workaround for the inability for Bufferevent to remove timeout from
-// bufferevent. Specify this long timeout instead of removing.
-namespace {
-timeval max_timeout = { 86400, 0 };
-} // namespace
-
 HttpDownstreamConnection::HttpDownstreamConnection
 (ClientHandler *client_handler)
   : DownstreamConnection(client_handler),
@@ -138,11 +132,9 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream)
                     upstream->get_downstream_readcb(),
                     upstream->get_downstream_writecb(),
                     upstream->get_downstream_eventcb(), this);
-  // HTTP request/response model, we first issue request to downstream
-  // server, so just enable write timeout here.
-  bufferevent_set_timeouts(bev_,
-                           &max_timeout,
-                           &get_config()->downstream_write_timeout);
+
+  reset_timeouts();
+
   return 0;
 }
 
@@ -287,16 +279,6 @@ int HttpDownstreamConnection::push_request_headers()
     return -1;
   }
 
-  // When downstream request is issued, set read timeout. We don't
-  // know when the request is completely received by the downstream
-  // server. This function may be called before that happens. Overall
-  // it does not cause problem for most of the time.  If the
-  // downstream server is too slow to recv/send, the connection will
-  // be dropped by read timeout.
-  bufferevent_set_timeouts(bev_,
-                           &get_config()->downstream_read_timeout,
-                           &get_config()->downstream_write_timeout);
-
   return 0;
 }
 
@@ -586,6 +568,8 @@ http_parser_settings htp_hooks = {
 
 int HttpDownstreamConnection::on_read()
 {
+  reset_timeouts();
+
   auto input = bufferevent_get_input(bev_);
 
   if(downstream_->get_upgraded()) {
@@ -650,6 +634,8 @@ int HttpDownstreamConnection::on_read()
 
 int HttpDownstreamConnection::on_write()
 {
+  reset_timeouts();
+
   auto upstream = downstream_->get_upstream();
   upstream->resume_read(SHRPX_NO_BUFFER, downstream_,
                         downstream_->get_request_datalen());
@@ -664,4 +650,11 @@ void HttpDownstreamConnection::on_upstream_change(Upstream *upstream)
                     upstream->get_downstream_eventcb(), this);
 }
 
+void HttpDownstreamConnection::reset_timeouts()
+{
+  bufferevent_set_timeouts(bev_,
+                           &get_config()->downstream_read_timeout,
+                           &get_config()->downstream_write_timeout);
+}
+
 } // namespace shrpx
diff --git a/src/shrpx_http_downstream_connection.h b/src/shrpx_http_downstream_connection.h
index 5a69ec17..7beb4992 100644
--- a/src/shrpx_http_downstream_connection.h
+++ b/src/shrpx_http_downstream_connection.h
@@ -64,6 +64,8 @@ public:
   }
 
   bufferevent* get_bev();
+
+  void reset_timeouts();
 private:
   bufferevent *bev_;
   IOControl ioctrl_;