diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc
index 3c57a54e..ba32323f 100644
--- a/src/shrpx_client_handler.cc
+++ b/src/shrpx_client_handler.cc
@@ -247,6 +247,7 @@ int ClientHandler::write_tls() {
       return -1;
     }
     if (wb_.rleft() == 0) {
+      conn_.start_tls_write_idle();
       break;
     }
   }
diff --git a/src/shrpx_connection.cc b/src/shrpx_connection.cc
index 196b0ea4..a2a3b91a 100644
--- a/src/shrpx_connection.cc
+++ b/src/shrpx_connection.cc
@@ -61,7 +61,7 @@ Connection::Connection(struct ev_loop *loop, int fd, SSL *ssl,
   rt.data = this;
 
   // set 0. to double field explicitly just in case
-  tls.last_write_time = 0.;
+  tls.last_write_idle = 0.;
 
   if (ssl) {
     set_ssl(ssl);
@@ -402,7 +402,7 @@ const size_t SHRPX_WARMUP_THRESHOLD = 1 << 20;
 size_t Connection::get_tls_write_limit() {
   auto t = ev_now(loop);
 
-  if (t - tls.last_write_time > 1.) {
+  if (tls.last_write_idle >= 0. && t - tls.last_write_idle > 1.) {
     // Time out, use small record size
     tls.warmup_writelen = 0;
     return SHRPX_SMALL_WRITE_LIMIT;
@@ -421,6 +421,12 @@ void Connection::update_tls_warmup_writelen(size_t n) {
   }
 }
 
+void Connection::start_tls_write_idle() {
+  if (tls.last_write_idle < 0.) {
+    tls.last_write_idle = ev_now(loop);
+  }
+}
+
 ssize_t Connection::write_tls(const void *data, size_t len) {
   // SSL_write requires the same arguments (buf pointer and its
   // length) on SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE.
@@ -439,14 +445,14 @@ ssize_t Connection::write_tls(const void *data, size_t len) {
     tls.last_writelen = 0;
   }
 
+  tls.last_write_idle = -1.;
+
   auto rv = SSL_write(tls.ssl, data, len);
 
   if (rv == 0) {
     return SHRPX_ERR_NETWORK;
   }
 
-  tls.last_write_time = ev_now(loop);
-
   if (rv < 0) {
     auto err = SSL_get_error(tls.ssl, rv);
     switch (err) {
diff --git a/src/shrpx_connection.h b/src/shrpx_connection.h
index 7b73bb85..f9e18fd2 100644
--- a/src/shrpx_connection.h
+++ b/src/shrpx_connection.h
@@ -52,7 +52,7 @@ struct TLSConnection {
   SSL *ssl;
   SSL_SESSION *cached_session;
   MemcachedRequest *cached_session_lookup_req;
-  ev_tstamp last_write_time;
+  ev_tstamp last_write_idle;
   size_t warmup_writelen;
   // length passed to SSL_write and SSL_read last time.  This is
   // required since these functions require the exact same parameters
@@ -100,6 +100,9 @@ struct Connection {
   size_t get_tls_write_limit();
   // Updates the number of bytes written in warm up period.
   void update_tls_warmup_writelen(size_t n);
+  // Tells there is no immediate write now.  This triggers timer to
+  // determine fallback to short record size mode.
+  void start_tls_write_idle();
 
   ssize_t write_clear(const void *data, size_t len);
   ssize_t writev_clear(struct iovec *iov, int iovcnt);
diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc
index 036f8d5e..79628543 100644
--- a/src/shrpx_http2_session.cc
+++ b/src/shrpx_http2_session.cc
@@ -1729,6 +1729,7 @@ int Http2Session::write_tls() {
       return -1;
     }
     if (wb_.rleft() == 0) {
+      conn_.start_tls_write_idle();
       break;
     }
   }