diff --git a/src/shrpx_rate_limit.cc b/src/shrpx_rate_limit.cc
index 8e3134d3..0d4f9212 100644
--- a/src/shrpx_rate_limit.cc
+++ b/src/shrpx_rate_limit.cc
@@ -109,9 +109,9 @@ void RateLimit::stopw() {
 
 void RateLimit::handle_tls_pending_read() {
   if (!conn_ || !conn_->tls.ssl ||
-      (conn_->tls.rbuf.rleft() == 0 && !conn_->tls.initial_handshake_done) ||
       (SSL_pending(conn_->tls.ssl) == 0 && conn_->tls.rbuf.rleft() == 0 &&
-       conn_->tls.earlybuf.rleft() == 0)) {
+       (!conn_->tls.initial_handshake_done ||
+        conn_->tls.earlybuf.rleft() == 0))) {
     return;
   }