diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc
index 29d3d9f8..ea73d0ee 100644
--- a/src/shrpx_http3_upstream.cc
+++ b/src/shrpx_http3_upstream.cc
@@ -781,10 +781,11 @@ int Http3Upstream::write_streams() {
                          prev_ps.path.local.addr, prev_ps.path.local.addrlen,
                          buf.data(), bufpos - buf.data(), max_udp_payload_size);
 
-        ngtcp2_conn_update_pkt_tx_time(conn_, ts);
         reset_idle_timer();
       }
 
+      ngtcp2_conn_update_pkt_tx_time(conn_, ts);
+
       handler_->get_connection()->wlimit.stopw();
 
       return 0;