diff --git a/src/shrpx_rate_limit.cc b/src/shrpx_rate_limit.cc
index f05ae64c..8e3134d3 100644
--- a/src/shrpx_rate_limit.cc
+++ b/src/shrpx_rate_limit.cc
@@ -108,7 +108,8 @@ void RateLimit::stopw() {
 }
 
 void RateLimit::handle_tls_pending_read() {
-  if (!conn_ || !conn_->tls.ssl || !conn_->tls.initial_handshake_done ||
+  if (!conn_ || !conn_->tls.ssl ||
+      (conn_->tls.rbuf.rleft() == 0 && !conn_->tls.initial_handshake_done) ||
       (SSL_pending(conn_->tls.ssl) == 0 && conn_->tls.rbuf.rleft() == 0 &&
        conn_->tls.earlybuf.rleft() == 0)) {
     return;