diff --git a/src/shrpx_connection.cc b/src/shrpx_connection.cc
index 0f40959b..d9597901 100644
--- a/src/shrpx_connection.cc
+++ b/src/shrpx_connection.cc
@@ -421,6 +421,10 @@ int Connection::tls_handshake() {
     ev_feed_event(loop, &rev, EV_READ);
   }
 
+  // We may have stopped reading
+  rlimit.startw();
+  ev_timer_again(loop, &rt);
+
   if (LOG_ENABLED(INFO)) {
     LOG(INFO) << "SSL/TLS handshake completed";
     if (SSL_session_reused(tls.ssl)) {