Bump ngtcp2 and nghttp3

.github/workflows/build.yml

@@ -116,7 +116,7 @@ jobs:
     - name: Build nghttp3
       if: matrix.http3 == 'http3'
       run: |
-        git clone --depth 1 -b v0.3.0 https://github.com/ngtcp2/nghttp3
+        git clone --depth 1 -b v0.4.0 https://github.com/ngtcp2/nghttp3
         cd nghttp3
         autoreconf -i
         ./configure --prefix=$PWD/build --enable-lib-only
@@ -125,7 +125,7 @@ jobs:
     - name: Build ngtcp2
       if: matrix.http3 == 'http3'
       run: |
-        git clone --depth 1 -b v0.3.0 https://github.com/ngtcp2/ngtcp2
+        git clone --depth 1 -b v0.4.0 https://github.com/ngtcp2/ngtcp2
         cd ngtcp2
         autoreconf -i
         ./configure --prefix=$PWD/build --enable-lib-only PKG_CONFIG_PATH="../openssl/build/lib/pkgconfig"

README.rst

@@ -154,8 +154,8 @@ following libraries are required:
   <https://github.com/quictls/openssl/tree/OpenSSL_1_1_1n+quic>`_; or
   `BoringSSL <https://boringssl.googlesource.com/boringssl/>`_ (commit
   36a41bf0bf2dd3176f8780e09c03585351f29963)
-* `ngtcp2 <https://github.com/ngtcp2/ngtcp2>`_ >= 0.3.0
+* `ngtcp2 <https://github.com/ngtcp2/ngtcp2>`_ >= 0.4.0
-* `nghttp3 <https://github.com/ngtcp2/nghttp3>`_ >= 0.2.0
+* `nghttp3 <https://github.com/ngtcp2/nghttp3>`_ >= 0.4.0
 
 Use ``--enable-http3`` configure option to enable HTTP/3 feature for
 h2load and nghttpx.
@@ -366,7 +366,7 @@ Build nghttp3:
 
 .. code-block:: text
 
-   $ git clone --depth 1 -b v0.2.0 https://github.com/ngtcp2/nghttp3
+   $ git clone --depth 1 -b v0.4.0 https://github.com/ngtcp2/nghttp3
    $ cd nghttp3
    $ autoreconf -i
    $ ./configure --prefix=$PWD/build --enable-lib-only
@@ -378,7 +378,7 @@ Build ngtcp2:
 
 .. code-block:: text
 
-   $ git clone --depth 1 -b v0.3.0 https://github.com/ngtcp2/ngtcp2
+   $ git clone --depth 1 -b v0.4.0 https://github.com/ngtcp2/ngtcp2
    $ cd ngtcp2
    $ autoreconf -i
    $ ./configure --prefix=$PWD/build --enable-lib-only \

configure.ac

@@ -540,7 +540,7 @@ fi
 # ngtcp2 (for src)
 have_libngtcp2=no
 if test "x${request_libngtcp2}" != "xno"; then
-  PKG_CHECK_MODULES([LIBNGTCP2], [libngtcp2 >= 0.3.0], [have_libngtcp2=yes],
+  PKG_CHECK_MODULES([LIBNGTCP2], [libngtcp2 >= 0.4.0], [have_libngtcp2=yes],
                     [have_libngtcp2=no])
   if test "x${have_libngtcp2}" = "xno"; then
     AC_MSG_NOTICE($LIBNGTCP2_PKG_ERRORS)
@@ -557,7 +557,7 @@ have_libngtcp2_crypto_openssl=no
 if test "x${have_ssl_is_quic}" = "xyes" &&
    test "x${request_libngtcp2}" != "xno"; then
   PKG_CHECK_MODULES([LIBNGTCP2_CRYPTO_OPENSSL],
-                    [libngtcp2_crypto_openssl >= 0.3.0],
+                    [libngtcp2_crypto_openssl >= 0.4.0],
                     [have_libngtcp2_crypto_openssl=yes],
                     [have_libngtcp2_crypto_openssl=no])
   if test "x${have_libngtcp2_crypto_openssl}" = "xno"; then
@@ -599,7 +599,7 @@ fi
 # nghttp3 (for src)
 have_libnghttp3=no
 if test "x${request_libnghttp3}" != "xno"; then
-  PKG_CHECK_MODULES([LIBNGHTTP3], [libnghttp3 >= 0.2.0], [have_libnghttp3=yes],
+  PKG_CHECK_MODULES([LIBNGHTTP3], [libnghttp3 >= 0.4.0], [have_libnghttp3=yes],
                     [have_libnghttp3=no])
   if test "x${have_libnghttp3}" = "xno"; then
     AC_MSG_NOTICE($LIBNGHTTP3_PKG_ERRORS)

src/h2load_quic.cc

@@ -786,7 +786,7 @@ int Client::write_quic() {
 
     if (pktcnt == 0) {
       gso_size = nwrite;
-    } else if (static_cast<size_t>(nwrite) > gso_size) {
+    } else if (static_cast<size_t>(nwrite) != gso_size) {
       auto data = quic.tx.data.get();
       auto datalen = bufpos - quic.tx.data.get() - nwrite;
       rv = write_udp(ps.path.remote.addr, ps.path.remote.addrlen, data, datalen,
@@ -808,7 +808,7 @@ int Client::write_quic() {
     }
 
     // Assume that the path does not change.
-    if (++pktcnt == max_pktcnt || static_cast<size_t>(nwrite) < gso_size) {
+    if (++pktcnt == max_pktcnt) {
       auto data = quic.tx.data.get();
       auto datalen = bufpos - quic.tx.data.get();
       rv = write_udp(ps.path.remote.addr, ps.path.remote.addrlen, data, datalen,

src/shrpx_http3_upstream.cc

@@ -49,22 +49,6 @@
 
 namespace shrpx {
 
-namespace {
-void idle_timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) {
-  auto upstream = static_cast<Http3Upstream *>(w->data);
-
-  if (LOG_ENABLED(INFO)) {
-    ULOG(INFO, upstream) << "QUIC idle timeout";
-  }
-
-  upstream->idle_close();
-
-  auto handler = upstream->get_client_handler();
-
-  delete handler;
-}
-} // namespace
-
 namespace {
 void timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) {
   auto upstream = static_cast<Http3Upstream *>(w->data);
@@ -125,7 +109,6 @@ Http3Upstream::Http3Upstream(ClientHandler *handler)
       httpconn_{nullptr},
       downstream_queue_{downstream_queue_size(handler->get_worker()),
                         !get_config()->http2_proxy},
-      idle_close_{false},
       retry_close_{false},
       tx_{
           .data = std::unique_ptr<uint8_t[]>(new uint8_t[64_k]),
@@ -135,13 +118,6 @@ Http3Upstream::Http3Upstream(ClientHandler *handler)
 
   ngtcp2_connection_close_error_default(&last_error_);
 
-  auto config = get_config();
-  auto &quicconf = config->quic;
-
-  ev_timer_init(&idle_timer_, idle_timeoutcb, 0.,
-                quicconf.upstream.timeout.idle);
-  idle_timer_.data = this;
-
   ev_timer_init(&shutdown_timer_, shutdown_timeout_cb, 0., 0.);
   shutdown_timer_.data = this;
 
@@ -155,7 +131,6 @@ Http3Upstream::~Http3Upstream() {
 
   ev_prepare_stop(loop, &prep_);
   ev_timer_stop(loop, &shutdown_timer_);
-  ev_timer_stop(loop, &idle_timer_);
   ev_timer_stop(loop, &timer_);
 
   nghttp3_conn_del(httpconn_);
@@ -862,14 +837,11 @@ int Http3Upstream::write_streams() {
                           prev_pi, data, datalen, gso_size);
 
           ngtcp2_conn_update_pkt_tx_time(conn_, ts);
-          reset_idle_timer();
 
           signal_write_upstream_addr(faddr);
 
           return 0;
         }
-
-        reset_idle_timer();
       }
 
       ngtcp2_conn_update_pkt_tx_time(conn_, ts);
@@ -888,7 +860,7 @@ int Http3Upstream::write_streams() {
       gso_size = nwrite;
     } else if (!ngtcp2_path_eq(&prev_ps.path, &ps.path) ||
                prev_pi.ecn != pi.ecn ||
-               static_cast<size_t>(nwrite) > gso_size) {
+               static_cast<size_t>(nwrite) != gso_size) {
       auto faddr = static_cast<UpstreamAddr *>(prev_ps.path.user_data);
       auto data = tx_.data.get();
       auto datalen = bufpos - data - nwrite;
@@ -926,12 +898,11 @@ int Http3Upstream::write_streams() {
       }
 
       ngtcp2_conn_update_pkt_tx_time(conn_, ts);
-      reset_idle_timer();
 
       return 0;
     }
 
-    if (++pktcnt == max_pktcnt || static_cast<size_t>(nwrite) < gso_size) {
+    if (++pktcnt == max_pktcnt) {
       auto faddr = static_cast<UpstreamAddr *>(ps.path.user_data);
       auto data = tx_.data.get();
       auto datalen = bufpos - data;
@@ -945,7 +916,6 @@ int Http3Upstream::write_streams() {
       }
 
       ngtcp2_conn_update_pkt_tx_time(conn_, ts);
-      reset_idle_timer();
 
       signal_write_upstream_addr(faddr);
 
@@ -964,7 +934,6 @@ int Http3Upstream::write_streams() {
                       0);
 
       ngtcp2_conn_update_pkt_tx_time(conn_, ts);
-      reset_idle_timer();
 
       signal_write_upstream_addr(faddr);
 
@@ -973,7 +942,6 @@ int Http3Upstream::write_streams() {
 
     if (++pktcnt == max_pktcnt) {
       ngtcp2_conn_update_pkt_tx_time(conn_, ts);
-      reset_idle_timer();
 
       signal_write_upstream_addr(faddr);
 
@@ -1508,7 +1476,9 @@ void Http3Upstream::on_handler_delete() {
     quic_conn_handler->remove_connection_id(cid);
   }
 
-  if (idle_close_ || retry_close_) {
+  if (retry_close_ ||
+      last_error_.type ==
+          NGTCP2_CONNECTION_CLOSE_ERROR_CODE_TYPE_TRANSPORT_IDLE_CLOSE) {
     return;
   }
 
@@ -1839,8 +1809,6 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr,
     return handle_error();
   }
 
-  reset_idle_timer();
-
   return 0;
 }
 
@@ -1937,7 +1905,8 @@ void Http3Upstream::signal_write_upstream_addr(const UpstreamAddr *faddr) {
 }
 
 int Http3Upstream::handle_error() {
-  if (ngtcp2_conn_is_in_closing_period(conn_)) {
+  if (ngtcp2_conn_is_in_closing_period(conn_) ||
+      ngtcp2_conn_is_in_draining_period(conn_)) {
     return -1;
   }
 
@@ -2020,7 +1989,11 @@ int Http3Upstream::handle_expiry() {
 
   rv = ngtcp2_conn_handle_expiry(conn_, ts);
   if (rv != 0) {
-    ULOG(ERROR, this) << "ngtcp2_conn_handle_expiry: " << ngtcp2_strerror(rv);
+    if (rv == NGTCP2_ERR_IDLE_CLOSE) {
+      ULOG(INFO, this) << "Idle connection timeout";
+    } else {
+      ULOG(ERROR, this) << "ngtcp2_conn_handle_expiry: " << ngtcp2_strerror(rv);
+    }
     ngtcp2_connection_close_error_set_transport_error_liberr(&last_error_, rv,
                                                              nullptr, 0);
     return handle_error();
@@ -2029,26 +2002,19 @@ int Http3Upstream::handle_expiry() {
   return 0;
 }
 
-void Http3Upstream::reset_idle_timer() {
-  auto ts = quic_timestamp();
-  auto idle_ts = ngtcp2_conn_get_idle_expiry(conn_);
-
-  idle_timer_.repeat =
-      idle_ts > ts ? static_cast<ev_tstamp>(idle_ts - ts) / NGTCP2_SECONDS
-                   : 1e-9;
-
-  ev_timer_again(handler_->get_loop(), &idle_timer_);
-}
-
 void Http3Upstream::reset_timer() {
   auto ts = quic_timestamp();
   auto expiry_ts = ngtcp2_conn_get_expiry(conn_);
+  auto loop = handler_->get_loop();
 
-  timer_.repeat = expiry_ts > ts
+  if (expiry_ts <= ts) {
-                      ? static_cast<ev_tstamp>(expiry_ts - ts) / NGTCP2_SECONDS
+    ev_feed_event(loop, &timer_, EV_TIMER);
-                      : 1e-9;
+    return;
+  }
 
-  ev_timer_again(handler_->get_loop(), &timer_);
+  timer_.repeat = static_cast<ev_tstamp>(expiry_ts - ts) / NGTCP2_SECONDS;
+
+  ev_timer_again(loop, &timer_);
 }
 
 namespace {
@@ -2922,8 +2888,6 @@ int Http3Upstream::submit_goaway() {
   return 0;
 }
 
-void Http3Upstream::idle_close() { idle_close_ = true; }
-
 int Http3Upstream::open_qlog_file(const StringRef &dir,
                                   const ngtcp2_cid &scid) const {
   std::array<char, sizeof("20141115T125824.741+0900")> buf;

src/shrpx_http3_upstream.h

@@ -110,7 +110,6 @@ public:
   int handle_error();
 
   int handle_expiry();
-  void reset_idle_timer();
   void reset_timer();
 
   int setup_httpconn();
@@ -148,7 +147,6 @@ public:
   int check_shutdown();
   int start_graceful_shutdown();
   int submit_goaway();
-  void idle_close();
   int send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
                   size_t remote_salen, const sockaddr *local_sa,
                   size_t local_salen, const ngtcp2_pkt_info &pi,
@@ -167,7 +165,6 @@ public:
 private:
   ClientHandler *handler_;
   ev_timer timer_;
-  ev_timer idle_timer_;
   ev_timer shutdown_timer_;
   ev_prepare prep_;
   int qlog_fd_;
@@ -177,7 +174,6 @@ private:
   uint8_t tls_alert_;
   nghttp3_conn *httpconn_;
   DownstreamQueue downstream_queue_;
-  bool idle_close_;
   bool retry_close_;
   std::vector<uint8_t> conn_close_;
 

src/shrpx_quic.cc

@@ -270,16 +270,16 @@ int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid &cid,
   return 0;
 }
 
-int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
+int generate_retry_token(uint8_t *token, size_t &tokenlen, uint32_t version,
-                         socklen_t salen, const ngtcp2_cid &retry_scid,
+                         const sockaddr *sa, socklen_t salen,
-                         const ngtcp2_cid &odcid, const uint8_t *secret,
+                         const ngtcp2_cid &retry_scid, const ngtcp2_cid &odcid,
-                         size_t secretlen) {
+                         const uint8_t *secret, size_t secretlen) {
   auto t = std::chrono::duration_cast<std::chrono::nanoseconds>(
                std::chrono::system_clock::now().time_since_epoch())
                .count();
 
   auto stokenlen = ngtcp2_crypto_generate_retry_token(
-      token, secret, secretlen, sa, salen, &retry_scid, &odcid, t);
+      token, secret, secretlen, version, sa, salen, &retry_scid, &odcid, t);
   if (stokenlen < 0) {
     return -1;
   }
@@ -290,16 +290,16 @@ int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
 }
 
 int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen,
-                       const ngtcp2_cid &dcid, const sockaddr *sa,
+                       uint32_t version, const ngtcp2_cid &dcid,
-                       socklen_t salen, const uint8_t *secret,
+                       const sockaddr *sa, socklen_t salen,
-                       size_t secretlen) {
+                       const uint8_t *secret, size_t secretlen) {
 
   auto t = std::chrono::duration_cast<std::chrono::nanoseconds>(
                std::chrono::system_clock::now().time_since_epoch())
                .count();
 
   if (ngtcp2_crypto_verify_retry_token(&odcid, token, tokenlen, secret,
-                                       secretlen, sa, salen, &dcid,
+                                       secretlen, version, sa, salen, &dcid,
                                        10 * NGTCP2_SECONDS, t) != 0) {
     return -1;
   }

src/shrpx_quic.h

@@ -107,15 +107,15 @@ int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid &cid,
                                         const uint8_t *secret,
                                         size_t secretlen);
 
-int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
+int generate_retry_token(uint8_t *token, size_t &tokenlen, uint32_t version,
-                         socklen_t salen, const ngtcp2_cid &retry_scid,
+                         const sockaddr *sa, socklen_t salen,
-                         const ngtcp2_cid &odcid, const uint8_t *secret,
+                         const ngtcp2_cid &retry_scid, const ngtcp2_cid &odcid,
-                         size_t secretlen);
+                         const uint8_t *secret, size_t secretlen);
 
 int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen,
-                       const ngtcp2_cid &dcid, const sockaddr *sa,
+                       uint32_t version, const ngtcp2_cid &dcid,
-                       socklen_t salen, const uint8_t *secret,
+                       const sockaddr *sa, socklen_t salen,
-                       size_t secretlen);
+                       const uint8_t *secret, size_t secretlen);
 
 int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
                    size_t salen, const uint8_t *secret, size_t secretlen);

src/shrpx_quic_connection_handler.cc

@@ -225,8 +225,8 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
 
       switch (hd.token.base[0]) {
       case NGTCP2_CRYPTO_TOKEN_MAGIC_RETRY:
-        if (verify_retry_token(odcid, hd.token.base, hd.token.len, hd.dcid,
+        if (verify_retry_token(odcid, hd.token.base, hd.token.len, hd.version,
-                               &remote_addr.su.sa, remote_addr.len,
+                               hd.dcid, &remote_addr.su.sa, remote_addr.len,
                                qkm->secret.data(), qkm->secret.size()) != 0) {
           if (LOG_ENABLED(INFO)) {
             LOG(INFO) << "Failed to validate Retry token from remote="
@@ -473,7 +473,7 @@ int QUICConnectionHandler::send_retry(
   ngtcp2_cid_init(&idcid, ini_dcid, ini_dcidlen);
   ngtcp2_cid_init(&iscid, ini_scid, ini_scidlen);
 
-  if (generate_retry_token(token.data(), tokenlen, &remote_addr.su.sa,
+  if (generate_retry_token(token.data(), tokenlen, version, &remote_addr.su.sa,
                            remote_addr.len, retry_scid, idcid,
                            qkm.secret.data(), qkm.secret.size()) != 0) {
     return -1;