Merge branch 'master' into master

.github/workflows/build.yml

@@ -60,8 +60,6 @@ jobs:
           pkg-config \
           libtool
         echo 'PKG_CONFIG_PATH=/usr/local/opt/libressl/lib/pkgconfig:/usr/local/opt/libxml2/lib/pkgconfig' >> $GITHUB_ENV
-        # This fixes infamous 'stdio.h not found' error.
-        echo 'SDKROOT='"$(xcrun --sdk macosx --show-sdk-path)" >> $GITHUB_ENV
     - name: Setup clang (Linux)
       if: runner.os == 'Linux' && matrix.compiler == 'clang'
       run: |
@@ -87,28 +85,28 @@ jobs:
       run: |
         git clone -b v0.4.0 https://github.com/libbpf/libbpf
         cd libbpf
-        export PREFIX=$PWD/build
+        PREFIX=$PWD/build make -C src install
-        cd src
-        make install
 
         EXTRA_AUTOTOOLS_OPTS="--with-libbpf"
+        EXTRA_CMAKE_OPTS="-DWITH_LIBBPF=1"
 
         echo 'EXTRA_AUTOTOOLS_OPTS='"$EXTRA_AUTOTOOLS_OPTS" >> $GITHUB_ENV
+        echo 'EXTRA_CMAKE_OPTS='"$EXTRA_CMAKE_OPTS" >> $GITHUB_ENV
     - name: Build quictls/openssl v1.1.1
       if: matrix.http3 == 'http3' && matrix.openssl == 'openssl1'
       run: |
-        git clone --depth 1 -b OpenSSL_1_1_1l+quic https://github.com/quictls/openssl
+        git clone --depth 1 -b OpenSSL_1_1_1m+quic https://github.com/quictls/openssl
         cd openssl
         ./config enable-tls1_3 --prefix=$PWD/build
         make -j$(nproc)
         make install_sw
-    - name: Build quictls/openssl v3.0.0
+    - name: Build quictls/openssl v3.0.x
       if: matrix.http3 == 'http3' && matrix.openssl == 'openssl3'
       run: |
         unset CPPFLAGS
         unset LDFLAGS
 
-        git clone --depth 1 -b openssl-3.0.0+quic https://github.com/quictls/openssl
+        git clone --depth 1 -b openssl-3.0.1+quic https://github.com/quictls/openssl
         cd openssl
         ./config enable-tls1_3 --prefix=$PWD/build --libdir=$PWD/build/lib
         make -j$(nproc)
@@ -118,6 +116,7 @@ jobs:
       run: |
         git clone https://github.com/ngtcp2/nghttp3
         cd nghttp3
+        git checkout 74a222fe0c89b7202bcdaf6ef27a232edffc85e3
         autoreconf -i
         ./configure --prefix=$PWD/build --enable-lib-only
         make -j$(nproc) check
@@ -125,7 +124,7 @@ jobs:
     - name: Build ngtcp2
       if: matrix.http3 == 'http3'
       run: |
-        git clone https://github.com/ngtcp2/ngtcp2
+        git clone --depth 1 -b v0.1.0 https://github.com/ngtcp2/ngtcp2
         cd ngtcp2
         autoreconf -i
         ./configure --prefix=$PWD/build --enable-lib-only PKG_CONFIG_PATH="../openssl/build/lib/pkgconfig"
@@ -137,31 +136,41 @@ jobs:
         PKG_CONFIG_PATH="$PWD/openssl/build/lib/pkgconfig:$PWD/nghttp3/build/lib/pkgconfig:$PWD/ngtcp2/build/lib/pkgconfig:$PWD/libbpf/build/lib64/pkgconfig:$PKG_CONFIG_PATH"
         LDFLAGS="$LDFLAGS -Wl,-rpath,$PWD/openssl/build/lib -Wl,-rpath,$PWD/libbpf/build/lib64"
         EXTRA_AUTOTOOLS_OPTS="--enable-http3 $EXTRA_AUTOTOOLS_OPTS"
+        EXTRA_CMAKE_OPTS="-DENABLE_HTTP3=1 $EXTRA_CMAKE_OPTS"
 
         echo 'PKG_CONFIG_PATH='"$PKG_CONFIG_PATH" >> $GITHUB_ENV
         echo 'LDFLAGS='"$LDFLAGS" >> $GITHUB_ENV
         echo 'EXTRA_AUTOTOOLS_OPTS='"$EXTRA_AUTOTOOLS_OPTS" >> $GITHUB_ENV
-        echo 'EXTRA_CMAKE_OPTS=-DENABLE_HTTP3=ON' >> $GITHUB_ENV
+        echo 'EXTRA_CMAKE_OPTS='"$EXTRA_CMAKE_OPTS" >> $GITHUB_ENV
     - name: Setup git submodules
       run: |
         git submodule update --init
     - name: Configure autotools
-      if: matrix.buildtool == 'autotools'
       run: |
         autoreconf -i
-        ./configure --enable-werror --with-mruby $EXTRA_AUTOTOOLS_OPTS
+        ./configure
     - name: Configure cmake
       if: matrix.buildtool == 'cmake'
       run: |
-        cmake -DENABLE_WERROR=1 -DWITH_MRUBY=1 -DWITH_NEVERBLEED=1 $EXTRA_CMAKE_OPTS -DCPPFLAGS="$CPPFLAGS" -DLDFLAGS="$LDFLAGS" .
+        make dist
+        VERSION=$(grep PACKAGE_VERSION config.h | cut -d' ' -f3 | tr -d '"')
+        tar xf nghttp2-$VERSION.tar.gz
+        cd nghttp2-$VERSION
+        echo 'NGHTTP2_CMAKE_DIR='"$PWD" >> $GITHUB_ENV
+
+        # This fixes infamous 'stdio.h not found' error.
+        echo 'SDKROOT='"$(xcrun --sdk macosx --show-sdk-path)" >> $GITHUB_ENV
+
+        cmake -DENABLE_WERROR=1 -DWITH_MRUBY=1 -DWITH_NEVERBLEED=1 -DENABLE_APP=1 $EXTRA_CMAKE_OPTS -DCPPFLAGS="$CPPFLAGS" -DLDFLAGS="$LDFLAGS" .
     - name: Build nghttp2 with autotools
       if: matrix.buildtool == 'autotools'
       run: |
         make distcheck \
-          DISTCHECK_CONFIGURE_FLAGS="--with-mruby --with-neverbleed --enable-werror $EXTRA_AUTOTOOLS_OPTS CPPFLAGS=\"$CPPFLAGS\" LDFLAGS=\"$LDFLAGS\""
+          DISTCHECK_CONFIGURE_FLAGS="--with-mruby --with-neverbleed --with-libev --enable-werror $EXTRA_AUTOTOOLS_OPTS CPPFLAGS=\"$CPPFLAGS\" LDFLAGS=\"$LDFLAGS\""
     - name: Build nghttp2 with cmake
       if: matrix.buildtool == 'cmake'
       run: |
+        cd $NGHTTP2_CMAKE_DIR
         make
         make check
     - name: Integration test
@@ -169,5 +178,5 @@ jobs:
       # artifacts.
       if: matrix.buildtool == 'cmake'
       run: |
-        cd integration-tests
+        cd $NGHTTP2_CMAKE_DIR/integration-tests
         make itprep it

AUTHORS

@@ -19,6 +19,7 @@ Alek Storm
 Alex Nalivko
 Alexandros Konstantinakis-Karmis
 Alexis La Goutte
+Amir Livneh
 Amir Pakdel
 Anders Bakken
 Andreas Pohl
@@ -34,11 +35,13 @@ Bernard Spil
 Brendan Heinonen
 Brian Card
 Brian Suh
+Daniel Bevenius
 Daniel Evers
 Daniel Stenberg
 Dave Reisner
 David Beitey
 David Weekly
+Dmitri Tikhonov
 Dmitriy Vetutnev
 Don
 Dylan Plecki
@@ -48,9 +51,12 @@ Fabian Wiesel
 Gabi Davar
 Gaël PORTAY
 Geoff Hill
+George Liu
 Gitai
 Google Inc.
+Hajime Fujita
 Jacky Tian
+Jacky_Yin
 Jacob Champion
 James M Snell
 Jan Kundrát
@@ -76,6 +82,7 @@ MATSUMOTO Ryosuke
 Marc Bachmann
 Matt Rudary
 Matt Way
+Michael Kaufmann
 Mike Conlen
 Mike Frysinger
 Mike Lothian
@@ -127,6 +134,7 @@ es
 fangdingjun
 jwchoi
 kumagi
+lhuang04
 lstefani
 makovich
 mod-h2-dev

CMakeLists.txt

@@ -24,13 +24,13 @@
 
 cmake_minimum_required(VERSION 3.0)
 # XXX using 1.8.90 instead of 1.9.0-DEV
-project(nghttp2 VERSION 1.44.90)
+project(nghttp2 VERSION 1.46.90)
 
 # See versioning rule:
-#  http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
+#  https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
-set(LT_CURRENT  34)
+set(LT_CURRENT  35)
-set(LT_REVISION 2)
+set(LT_REVISION 1)
-set(LT_AGE      20)
+set(LT_AGE      21)
 
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
 include(Version)
@@ -63,8 +63,16 @@ find_package(Libcares 1.7.5)
 find_package(ZLIB 1.2.3)
 find_package(Libngtcp2 0.0.0)
 find_package(Libngtcp2_crypto_openssl 0.0.0)
+if(LIBNGTCP2_CRYPTO_OPENSSL_FOUND)
+  set(HAVE_LIBNGTCP2_CRYPTO_OPENSSL 1)
+endif()
 find_package(Libnghttp3 0.0.0)
-find_package(Libbpf 0.4.0)
+if(WITH_LIBBPF)
+  find_package(Libbpf 0.4.0)
+  if(NOT LIBBPF_FOUND)
+    message(FATAL_ERROR "libbpf was requested (WITH_LIBBPF=1) but not found.")
+  endif()
+endif()
 if(OPENSSL_FOUND AND LIBEV_FOUND AND ZLIB_FOUND)
   set(ENABLE_APP_DEFAULT ON)
 else()
@@ -171,7 +179,7 @@ endif()
 # case "$host" in
 #   *android*)
 #     android_build=yes
-#     # android does not need -pthread, but needs followng 3 libs for C++
+#     # android does not need -pthread, but needs following 3 libs for C++
 #     APPLDFLAGS="$APPLDFLAGS -lstdc++ -latomic -lsupc++"
 
 # dl: openssl requires libdl when it is statically linked.

CMakeOptions.txt

@@ -23,9 +23,8 @@ option(WITH_LIBXML2     "Use libxml2"
   ${WITH_LIBXML2_DEFAULT})
 option(WITH_JEMALLOC    "Use jemalloc"
   ${WITH_JEMALLOC_DEFAULT})
-option(WITH_SPDYLAY     "Use spdylay"
-  ${WITH_SPDYLAY_DEFAULT})
 option(WITH_MRUBY       "Use mruby")
 option(WITH_NEVERBLEED  "Use neverbleed")
+option(WITH_LIBBPF      "Use libbpf")
 
 # vim: ft=cmake:

Makefile.am

@@ -46,16 +46,20 @@ EXTRA_DIST = nghttpx.conf.sample proxy.pac.sample android-config android-make \
 	cmake/FindLibevent.cmake \
 	cmake/FindJansson.cmake \
 	cmake/FindLibcares.cmake \
-	cmake/FindSystemd.cmake
+	cmake/FindSystemd.cmake \
+	cmake/FindLibbpf.cmake \
+	cmake/FindLibnghttp3.cmake \
+	cmake/FindLibngtcp2.cmake \
+	cmake/FindLibngtcp2_crypto_openssl.cmake
 
 .PHONY: clang-format
 
 # Format source files using clang-format.  Don't format source files
-# under third-party directory since we are not responsible for thier
+# under third-party directory since we are not responsible for their
 # coding style.
 clang-format:
 	CLANGFORMAT=`git config --get clangformat.binary`; \
 	test -z $${CLANGFORMAT} && CLANGFORMAT="clang-format"; \
 	$${CLANGFORMAT} -i lib/*.{c,h} lib/includes/nghttp2/*.h \
 	src/*.{c,cc,h} src/includes/nghttp2/*.h examples/*.{c,cc} \
-	tests/*.{c,h}
+	tests/*.{c,h} bpf/*.c

README.rst

@@ -32,12 +32,14 @@ Public Test Server
 The following endpoints are available to try out our nghttp2
 implementation.
 
-* https://nghttp2.org/ (TLS + ALPN/NPN)
+* https://nghttp2.org/ (TLS + ALPN/NPN and HTTP/3)
 
   This endpoint supports ``h2``, ``h2-16``, ``h2-14``, and
   ``http/1.1`` via ALPN/NPN and requires TLSv1.2 for HTTP/2
   connection.
 
+  It also supports HTTP/3.
+
 * http://nghttp2.org/ (HTTP Upgrade and HTTP/2 Direct)
 
   ``h2c`` and ``http/1.1``.
@@ -149,16 +151,24 @@ To enable the experimental HTTP/3 support for h2load and nghttpx, the
 following libraries are required:
 
 * `OpenSSL with QUIC support
-  <https://github.com/quictls/openssl/tree/OpenSSL_1_1_1k+quic>`_
+  <https://github.com/quictls/openssl/tree/OpenSSL_1_1_1m+quic>`_; or
+  `BoringSSL <https://boringssl.googlesource.com/boringssl/>`_ (commit
+  f6ef1c560ae5af51e2df5d8d2175bed207b28b8f)
 * `ngtcp2 <https://github.com/ngtcp2/ngtcp2>`_
 * `nghttp3 <https://github.com/ngtcp2/nghttp3>`_
 
+Use ``--enable-http3`` configure option to enable HTTP/3 feature for
+h2load and nghttpx.
+
 In order to build optional eBPF program to direct an incoming QUIC UDP
 datagram to a correct socket for nghttpx, the following libraries are
 required:
 
 * libbpf-dev >= 0.4.0
 
+Use ``--with-libbpf`` configure option to build eBPF program.
+libelf-dev is needed to build libbpf.
+
 For Ubuntu 20.04, you can build libbpf from `the source code
 <https://github.com/libbpf/libbpf/releases/tag/v0.4.0>`_.  nghttpx
 requires eBPF program for reloading its configuration and hot swapping
@@ -326,6 +336,89 @@ The generated documents will not be installed with ``make install``.
 The online documentation is available at
 https://nghttp2.org/documentation/
 
+Build HTTP/3 enabled h2load and nghttpx
+---------------------------------------
+
+To build h2load and nghttpx with HTTP/3 feature enabled, run the
+configure script with ``--enable-http3``.
+
+For nghttpx to reload configurations and swapping its executable while
+gracefully terminating old worker processes, eBPF is required.  Run
+the configure script with ``--enable-http3 --with-libbpf`` to build
+eBPF program.  The QUIC keying material must be set with
+``--frontend-quic-secret-file`` in order to keep the existing
+connections alive during reload.
+
+The detailed steps to build HTTP/3 enabled h2load and nghttpx follow.
+
+Build custom OpenSSL:
+
+.. code-block:: text
+
+   $ git clone --depth 1 -b OpenSSL_1_1_1m+quic https://github.com/quictls/openssl
+   $ cd openssl
+   $ ./config --prefix=$PWD/build --openssldir=/etc/ssl
+   $ make -j$(nproc)
+   $ make install_sw
+   $ cd ..
+
+Build nghttp3:
+
+.. code-block:: text
+
+   $ git clone https://github.com/ngtcp2/nghttp3
+   $ cd nghttp3
+   $ git checkout 74a222fe0c89b7202bcdaf6ef27a232edffc85e3
+   $ autoreconf -i
+   $ ./configure --prefix=$PWD/build --enable-lib-only
+   $ make -j$(nproc)
+   $ make install
+   $ cd ..
+
+Build ngtcp2:
+
+.. code-block:: text
+
+   $ git clone --depth 1 -b v0.1.0 https://github.com/ngtcp2/ngtcp2
+   $ cd ngtcp2
+   $ autoreconf -i
+   $ ./configure --prefix=$PWD/build --enable-lib-only \
+         PKG_CONFIG_PATH="$PWD/../openssl/build/lib/pkgconfig"
+   $ make -j$(nproc)
+   $ make install
+   $ cd ..
+
+If your Linux distribution does not have libbpf-dev >= 0.4.0, build
+from source:
+
+.. code-block:: text
+
+   $ git clone --depth 1 -b v0.4.0 https://github.com/libbpf/libbpf
+   $ cd libbpf
+   $ PREFIX=$PWD/build make -C src install
+   $ cd ..
+
+Build nghttp2:
+
+.. code-block:: text
+
+   $ git clone https://github.com/nghttp2/nghttp2
+   $ cd nghttp2
+   $ git submodule update --init
+   $ autoreconf -i
+   $ ./configure --with-mruby --with-neverbleed --enable-http3 --with-libbpf \
+         --disable-python-bindings \
+         CC=clang-12 CXX=clang++-12 \
+         PKG_CONFIG_PATH="$PWD/../openssl/build/lib/pkgconfig:$PWD/../nghttp3/build/lib/pkgconfig:$PWD/../ngtcp2/build/lib/pkgconfig:$PWD/../libbpf/build/lib64/pkgconfig" \
+         LDFLAGS="$LDFLAGS -Wl,-rpath,$PWD/../openssl/build/lib -Wl,-rpath,$PWD/../libbpf/build/lib64"
+   $ make -j$(nproc)
+
+The eBPF program ``reuseport_kern.o`` should be found under bpf
+directory.  Pass ``--quic-bpf-program-file=bpf/reuseport_kern.o``
+option to nghttpx to load it.  See also `HTTP/3 section in nghttpx -
+HTTP/2 proxy - HOW-TO
+<https://nghttp2.org/documentation/nghttpx-howto.html#http-3>`_.
+
 Unit tests
 ----------
 
@@ -753,7 +846,7 @@ information.  Here is sample output from ``nghttpd``:
 nghttpx - proxy
 +++++++++++++++
 
-``nghttpx`` is a multi-threaded reverse proxy for HTTP/2, and
+``nghttpx`` is a multi-threaded reverse proxy for HTTP/3, HTTP/2, and
 HTTP/1.1, and powers http://nghttp2.org and supports HTTP/2 server
 push.
 
@@ -774,16 +867,16 @@ ticket keys among multiple ``nghttpx`` instances via memcached.
 
 ``nghttpx`` has 2 operation modes:
 
-================== ================ ================ =============
+================== ======================== ================ =============
-Mode option        Frontend         Backend          Note
+Mode option        Frontend                 Backend          Note
-================== ================ ================ =============
+================== ======================== ================ =============
-default mode       HTTP/2, HTTP/1.1 HTTP/1.1, HTTP/2 Reverse proxy
+default mode       HTTP/3, HTTP/2, HTTP/1.1 HTTP/1.1, HTTP/2 Reverse proxy
-``--http2-proxy``  HTTP/2, HTTP/1.1 HTTP/1.1, HTTP/2 Forward proxy
+``--http2-proxy``  HTTP/3, HTTP/2, HTTP/1.1 HTTP/1.1, HTTP/2 Forward proxy
-================== ================ ================ =============
+================== ======================== ================ =============
 
 The interesting mode at the moment is the default mode.  It works like
-a reverse proxy and listens for HTTP/2, and HTTP/1.1 and can be
+a reverse proxy and listens for HTTP/3, HTTP/2, and HTTP/1.1 and can
-deployed as a SSL/TLS terminator for existing web server.
+be deployed as a SSL/TLS terminator for existing web server.
 
 In all modes, the frontend connections are encrypted by SSL/TLS by
 default.  To disable encryption, use the ``no-tls`` keyword in
@@ -801,16 +894,16 @@ server:
 
 .. code-block:: text
 
-    Client <-- (HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/1.1, HTTP/2) --> Web Server
+    Client <-- (HTTP/3, HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/1.1, HTTP/2) --> Web Server
-                                    [reverse proxy]
+                                            [reverse proxy]
 
 With the ``--http2-proxy`` option, it works as forward proxy, and it
 is so called secure HTTP/2 proxy:
 
 .. code-block:: text
 
-    Client <-- (HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/1.1) --> Proxy
+    Client <-- (HTTP/3, HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/1.1) --> Proxy
-                                     [secure proxy]          (e.g., Squid, ATS)
+                                             [secure proxy]          (e.g., Squid, ATS)
 
 The ``Client`` in the above example needs to be configured to use
 ``nghttpx`` as secure proxy.
@@ -842,7 +935,7 @@ proxy through an HTTP proxy:
 
 .. code-block:: text
 
-    Client <-- (HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/2) --
+    Client <-- (HTTP/3, HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/2) --
 
             --===================---> HTTP/2 Proxy
               (HTTP proxy tunnel)     (e.g., nghttpx -s)
@@ -850,8 +943,8 @@ proxy through an HTTP proxy:
 Benchmarking tool
 -----------------
 
-The ``h2load`` program is a benchmarking tool for HTTP/2.  The UI of
+The ``h2load`` program is a benchmarking tool for HTTP/3, HTTP/2, and
-``h2load`` is heavily inspired by ``weighttp``
+HTTP/1.1.  The UI of ``h2load`` is heavily inspired by ``weighttp``
 (https://github.com/lighttpd/weighttp).  The typical usage is as
 follows:
 
@@ -902,17 +995,6 @@ like so:
 
     $ h2load --npn-list h3 https://127.0.0.1:4433
 
-HTTP/3
-------
-
-To build h2load and nghttpx with HTTP/3 feature enabled, run the
-configure script with ``--enable-http3``.
-
-For nghttpx to reload configurations and swapping its executable while
-gracefully terminating old worker processes, eBPF is required.  Run
-the configure script with ``--enable-http3 --with-libbpf`` to build
-eBPF program.
-
 HPACK tools
 -----------
 

bpf/Makefile.am

@@ -21,9 +21,9 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-if HAVE_LIBBPF
+EXTRA_DIST = CMakeLists.txt reuseport_kern.c
 
-EXTRA_DIST = reuseport_kern.c
+if HAVE_LIBBPF
 
 bpf_pkglibdir = $(pkglibdir)
 bpf_pkglib_DATA = reuseport_kern.o

bpf/reuseport_kern.c

@@ -38,6 +38,320 @@
  * how to install kernel header files.
  */
 
+/* AES_CBC_decrypt_buffer: https://github.com/kokke/tiny-AES-c
+   License is Public Domain.  Commit hash:
+   12e7744b4919e9d55de75b7ab566326a1c8e7a67 */
+
+#define AES_BLOCKLEN                                                           \
+  16 /* Block length in bytes - AES is 128b block                              \
+        only */
+
+#define AES_KEYLEN 16 /* Key length in bytes */
+#define AES_keyExpSize 176
+
+struct AES_ctx {
+  __u8 RoundKey[AES_keyExpSize];
+};
+
+/* The number of columns comprising a state in AES. This is a constant
+   in AES. Value=4 */
+#define Nb 4
+
+#define Nk 4  /* The number of 32 bit words in a key. */
+#define Nr 10 /* The number of rounds in AES Cipher. */
+
+/* state - array holding the intermediate results during
+   decryption. */
+typedef __u8 state_t[4][4];
+
+/* The lookup-tables are marked const so they can be placed in
+   read-only storage instead of RAM The numbers below can be computed
+   dynamically trading ROM for RAM - This can be useful in (embedded)
+   bootloader applications, where ROM is often limited. */
+static const __u8 sbox[256] = {
+    /* 0 1 2 3 4 5 6 7 8 9 A B C D E F */
+    0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b,
+    0xfe, 0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0,
+    0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 0xb7, 0xfd, 0x93, 0x26,
+    0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
+    0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2,
+    0xeb, 0x27, 0xb2, 0x75, 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0,
+    0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 0x53, 0xd1, 0x00, 0xed,
+    0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
+    0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f,
+    0x50, 0x3c, 0x9f, 0xa8, 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5,
+    0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 0xcd, 0x0c, 0x13, 0xec,
+    0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
+    0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14,
+    0xde, 0x5e, 0x0b, 0xdb, 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c,
+    0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 0xe7, 0xc8, 0x37, 0x6d,
+    0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
+    0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f,
+    0x4b, 0xbd, 0x8b, 0x8a, 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e,
+    0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 0xe1, 0xf8, 0x98, 0x11,
+    0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
+    0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f,
+    0xb0, 0x54, 0xbb, 0x16};
+
+static const __u8 rsbox[256] = {
+    0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e,
+    0x81, 0xf3, 0xd7, 0xfb, 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87,
+    0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 0x54, 0x7b, 0x94, 0x32,
+    0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e,
+    0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49,
+    0x6d, 0x8b, 0xd1, 0x25, 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16,
+    0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, 0x6c, 0x70, 0x48, 0x50,
+    0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84,
+    0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05,
+    0xb8, 0xb3, 0x45, 0x06, 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02,
+    0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, 0x3a, 0x91, 0x11, 0x41,
+    0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73,
+    0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8,
+    0x1c, 0x75, 0xdf, 0x6e, 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89,
+    0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 0xfc, 0x56, 0x3e, 0x4b,
+    0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4,
+    0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59,
+    0x27, 0x80, 0xec, 0x5f, 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d,
+    0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, 0xa0, 0xe0, 0x3b, 0x4d,
+    0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61,
+    0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63,
+    0x55, 0x21, 0x0c, 0x7d};
+
+/* The round constant word array, Rcon[i], contains the values given
+   by x to the power (i-1) being powers of x (x is denoted as {02}) in
+   the field GF(2^8) */
+static const __u8 Rcon[11] = {0x8d, 0x01, 0x02, 0x04, 0x08, 0x10,
+                              0x20, 0x40, 0x80, 0x1b, 0x36};
+
+#define getSBoxValue(num) (sbox[(num)])
+
+/* This function produces Nb(Nr+1) round keys. The round keys are used
+   in each round to decrypt the states. */
+static void KeyExpansion(__u8 *RoundKey, const __u8 *Key) {
+  unsigned i, j, k;
+  __u8 tempa[4]; /* Used for the column/row operations */
+
+  /* The first round key is the key itself. */
+  for (i = 0; i < Nk; ++i) {
+    RoundKey[(i * 4) + 0] = Key[(i * 4) + 0];
+    RoundKey[(i * 4) + 1] = Key[(i * 4) + 1];
+    RoundKey[(i * 4) + 2] = Key[(i * 4) + 2];
+    RoundKey[(i * 4) + 3] = Key[(i * 4) + 3];
+  }
+
+  /* All other round keys are found from the previous round keys. */
+  for (i = Nk; i < Nb * (Nr + 1); ++i) {
+    {
+      k = (i - 1) * 4;
+      tempa[0] = RoundKey[k + 0];
+      tempa[1] = RoundKey[k + 1];
+      tempa[2] = RoundKey[k + 2];
+      tempa[3] = RoundKey[k + 3];
+    }
+
+    if (i % Nk == 0) {
+      /* This function shifts the 4 bytes in a word to the left once.
+         [a0,a1,a2,a3] becomes [a1,a2,a3,a0] */
+
+      /* Function RotWord() */
+      {
+        const __u8 u8tmp = tempa[0];
+        tempa[0] = tempa[1];
+        tempa[1] = tempa[2];
+        tempa[2] = tempa[3];
+        tempa[3] = u8tmp;
+      }
+
+      /* SubWord() is a function that takes a four-byte input word and
+         applies the S-box to each of the four bytes to produce an
+         output word. */
+
+      /* Function Subword() */
+      {
+        tempa[0] = getSBoxValue(tempa[0]);
+        tempa[1] = getSBoxValue(tempa[1]);
+        tempa[2] = getSBoxValue(tempa[2]);
+        tempa[3] = getSBoxValue(tempa[3]);
+      }
+
+      tempa[0] = tempa[0] ^ Rcon[i / Nk];
+    }
+    j = i * 4;
+    k = (i - Nk) * 4;
+    RoundKey[j + 0] = RoundKey[k + 0] ^ tempa[0];
+    RoundKey[j + 1] = RoundKey[k + 1] ^ tempa[1];
+    RoundKey[j + 2] = RoundKey[k + 2] ^ tempa[2];
+    RoundKey[j + 3] = RoundKey[k + 3] ^ tempa[3];
+  }
+}
+
+static void AES_init_ctx(struct AES_ctx *ctx, const __u8 *key) {
+  KeyExpansion(ctx->RoundKey, key);
+}
+
+/* This function adds the round key to state.  The round key is added
+   to the state by an XOR function. */
+static void AddRoundKey(__u8 round, state_t *state, const __u8 *RoundKey) {
+  __u8 i, j;
+  for (i = 0; i < 4; ++i) {
+    for (j = 0; j < 4; ++j) {
+      (*state)[i][j] ^= RoundKey[(round * Nb * 4) + (i * Nb) + j];
+    }
+  }
+}
+
+static __u8 xtime(__u8 x) { return ((x << 1) ^ (((x >> 7) & 1) * 0x1b)); }
+
+#define Multiply(x, y)                                                         \
+  (((y & 1) * x) ^ ((y >> 1 & 1) * xtime(x)) ^                                 \
+   ((y >> 2 & 1) * xtime(xtime(x))) ^                                          \
+   ((y >> 3 & 1) * xtime(xtime(xtime(x)))) ^                                   \
+   ((y >> 4 & 1) * xtime(xtime(xtime(xtime(x))))))
+
+#define getSBoxInvert(num) (rsbox[(num)])
+
+/* MixColumns function mixes the columns of the state matrix.  The
+   method used to multiply may be difficult to understand for the
+   inexperienced. Please use the references to gain more
+   information. */
+static void InvMixColumns(state_t *state) {
+  int i;
+  __u8 a, b, c, d;
+  for (i = 0; i < 4; ++i) {
+    a = (*state)[i][0];
+    b = (*state)[i][1];
+    c = (*state)[i][2];
+    d = (*state)[i][3];
+
+    (*state)[i][0] = Multiply(a, 0x0e) ^ Multiply(b, 0x0b) ^ Multiply(c, 0x0d) ^
+                     Multiply(d, 0x09);
+    (*state)[i][1] = Multiply(a, 0x09) ^ Multiply(b, 0x0e) ^ Multiply(c, 0x0b) ^
+                     Multiply(d, 0x0d);
+    (*state)[i][2] = Multiply(a, 0x0d) ^ Multiply(b, 0x09) ^ Multiply(c, 0x0e) ^
+                     Multiply(d, 0x0b);
+    (*state)[i][3] = Multiply(a, 0x0b) ^ Multiply(b, 0x0d) ^ Multiply(c, 0x09) ^
+                     Multiply(d, 0x0e);
+  }
+}
+
+extern __u32 LINUX_KERNEL_VERSION __kconfig;
+
+/* The SubBytes Function Substitutes the values in the state matrix
+   with values in an S-box. */
+static void InvSubBytes(state_t *state) {
+  __u8 i, j;
+  if (LINUX_KERNEL_VERSION < KERNEL_VERSION(5, 10, 0)) {
+    for (i = 0; i < 4; ++i) {
+      for (j = 0; j < 4; ++j) {
+        /* Ubuntu 20.04 LTS kernel 5.4.0 needs this workaround
+           otherwise "math between map_value pointer and register with
+           unbounded min value is not allowed".  5.10.0 is a kernel
+           version that works but it might not be the minimum
+           version.  */
+        __u8 k = (*state)[j][i];
+        (*state)[j][i] = k ? getSBoxInvert(k) : getSBoxInvert(0);
+      }
+    }
+  } else {
+    for (i = 0; i < 4; ++i) {
+      for (j = 0; j < 4; ++j) {
+        (*state)[j][i] = getSBoxInvert((*state)[j][i]);
+      }
+    }
+  }
+}
+
+static void InvShiftRows(state_t *state) {
+  __u8 temp;
+
+  /* Rotate first row 1 columns to right */
+  temp = (*state)[3][1];
+  (*state)[3][1] = (*state)[2][1];
+  (*state)[2][1] = (*state)[1][1];
+  (*state)[1][1] = (*state)[0][1];
+  (*state)[0][1] = temp;
+
+  /* Rotate second row 2 columns to right */
+  temp = (*state)[0][2];
+  (*state)[0][2] = (*state)[2][2];
+  (*state)[2][2] = temp;
+
+  temp = (*state)[1][2];
+  (*state)[1][2] = (*state)[3][2];
+  (*state)[3][2] = temp;
+
+  /* Rotate third row 3 columns to right */
+  temp = (*state)[0][3];
+  (*state)[0][3] = (*state)[1][3];
+  (*state)[1][3] = (*state)[2][3];
+  (*state)[2][3] = (*state)[3][3];
+  (*state)[3][3] = temp;
+}
+
+static void InvCipher(state_t *state, const __u8 *RoundKey) {
+  /* Add the First round key to the state before starting the
+     rounds. */
+  AddRoundKey(Nr, state, RoundKey);
+
+  /* There will be Nr rounds.  The first Nr-1 rounds are identical.
+     These Nr rounds are executed in the loop below.  Last one without
+     InvMixColumn() */
+  InvShiftRows(state);
+  InvSubBytes(state);
+  AddRoundKey(Nr - 1, state, RoundKey);
+  InvMixColumns(state);
+
+  InvShiftRows(state);
+  InvSubBytes(state);
+  AddRoundKey(Nr - 2, state, RoundKey);
+  InvMixColumns(state);
+
+  InvShiftRows(state);
+  InvSubBytes(state);
+  AddRoundKey(Nr - 3, state, RoundKey);
+  InvMixColumns(state);
+
+  InvShiftRows(state);
+  InvSubBytes(state);
+  AddRoundKey(Nr - 4, state, RoundKey);
+  InvMixColumns(state);
+
+  InvShiftRows(state);
+  InvSubBytes(state);
+  AddRoundKey(Nr - 5, state, RoundKey);
+  InvMixColumns(state);
+
+  InvShiftRows(state);
+  InvSubBytes(state);
+  AddRoundKey(Nr - 6, state, RoundKey);
+  InvMixColumns(state);
+
+  InvShiftRows(state);
+  InvSubBytes(state);
+  AddRoundKey(Nr - 7, state, RoundKey);
+  InvMixColumns(state);
+
+  InvShiftRows(state);
+  InvSubBytes(state);
+  AddRoundKey(Nr - 8, state, RoundKey);
+  InvMixColumns(state);
+
+  InvShiftRows(state);
+  InvSubBytes(state);
+  AddRoundKey(Nr - 9, state, RoundKey);
+  InvMixColumns(state);
+
+  InvShiftRows(state);
+  InvSubBytes(state);
+  AddRoundKey(Nr - 10, state, RoundKey);
+}
+
+static void AES_ECB_decrypt(const struct AES_ctx *ctx, __u8 *buf) {
+  /* The next function call decrypts the PlainText with the Key using
+     AES algorithm. */
+  InvCipher((state_t *)buf, ctx->RoundKey);
+}
+
 /* rol32: From linux kernel source code */
 
 /**
@@ -125,13 +439,13 @@ struct bpf_map_def SEC("maps") reuseport_array = {
 
 struct bpf_map_def SEC("maps") sk_info = {
     .type = BPF_MAP_TYPE_ARRAY,
-    .max_entries = 1,
+    .max_entries = 3,
     .key_size = sizeof(__u32),
-    .value_size = sizeof(__u32),
+    .value_size = sizeof(__u64),
 };
 
 typedef struct quic_hd {
-  const __u8 *dcid;
+  __u8 *dcid;
   __u32 dcidlen;
   __u32 dcid_offset;
   __u8 type;
@@ -141,6 +455,7 @@ typedef struct quic_hd {
 #define MAX_DCIDLEN 20
 #define MIN_DCIDLEN 8
 #define CID_PREFIXLEN 8
+#define CID_PREFIX_OFFSET 1
 
 enum {
   NGTCP2_PKT_INITIAL = 0x0,
@@ -149,9 +464,8 @@ enum {
   NGTCP2_PKT_SHORT = 0x40,
 };
 
-static inline int parse_quic(quic_hd *qhd, const __u8 *data,
+static inline int parse_quic(quic_hd *qhd, __u8 *data, __u8 *data_end) {
-                             const __u8 *data_end) {
+  __u8 *p;
-  const __u8 *p;
   __u64 dcidlen;
 
   if (*data & 0x80) {
@@ -192,7 +506,7 @@ static __u32 hash(const __u8 *data, __u32 datalen, __u32 initval) {
 
 static __u32 sk_index_from_dcid(const quic_hd *qhd,
                                 const struct sk_reuseport_md *reuse_md,
-                                __u32 num_socks) {
+                                __u64 num_socks) {
   __u32 len = qhd->dcidlen;
   __u32 h = reuse_md->hash;
   __u8 hbuf[8];
@@ -259,11 +573,14 @@ static __u32 sk_index_from_dcid(const quic_hd *qhd,
 SEC("sk_reuseport")
 int select_reuseport(struct sk_reuseport_md *reuse_md) {
   __u32 sk_index, *psk_index;
-  __u32 *pnum_socks;
+  __u64 *pnum_socks, *pkey;
-  __u32 zero = 0;
+  __u32 zero = 0, key_high_idx = 1, key_low_idx = 2;
   int rv;
   quic_hd qhd;
   __u8 qpktbuf[6 + MAX_DCIDLEN];
+  struct AES_ctx aes_ctx;
+  __u8 key[AES_KEYLEN];
+  __u8 *cid_prefix;
 
   if (bpf_skb_load_bytes(reuse_md, sizeof(struct udphdr), qpktbuf,
                          sizeof(qpktbuf)) != 0) {
@@ -275,16 +592,35 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) {
     return SK_DROP;
   }
 
+  pkey = bpf_map_lookup_elem(&sk_info, &key_high_idx);
+  if (pkey == NULL) {
+    return SK_DROP;
+  }
+
+  __builtin_memcpy(key, pkey, sizeof(*pkey));
+
+  pkey = bpf_map_lookup_elem(&sk_info, &key_low_idx);
+  if (pkey == NULL) {
+    return SK_DROP;
+  }
+
+  __builtin_memcpy(key + sizeof(*pkey), pkey, sizeof(*pkey));
+
   rv = parse_quic(&qhd, qpktbuf, qpktbuf + sizeof(qpktbuf));
   if (rv != 0) {
     return SK_DROP;
   }
 
+  AES_init_ctx(&aes_ctx, key);
+
   switch (qhd.type) {
   case NGTCP2_PKT_INITIAL:
   case NGTCP2_PKT_0RTT:
     if (qhd.dcidlen == SV_DCIDLEN) {
-      psk_index = bpf_map_lookup_elem(&cid_prefix_map, qhd.dcid);
+      cid_prefix = qhd.dcid + CID_PREFIX_OFFSET;
+      AES_ECB_decrypt(&aes_ctx, cid_prefix);
+
+      psk_index = bpf_map_lookup_elem(&cid_prefix_map, cid_prefix);
       if (psk_index != NULL) {
         sk_index = *psk_index;
 
@@ -301,7 +637,10 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) {
       return SK_DROP;
     }
 
-    psk_index = bpf_map_lookup_elem(&cid_prefix_map, qhd.dcid);
+    cid_prefix = qhd.dcid + CID_PREFIX_OFFSET;
+    AES_ECB_decrypt(&aes_ctx, cid_prefix);
+
+    psk_index = bpf_map_lookup_elem(&cid_prefix_map, cid_prefix);
     if (psk_index == NULL) {
       sk_index = sk_index_from_dcid(&qhd, reuse_md, *pnum_socks);
 

cmakeconfig.h.in

@@ -87,3 +87,6 @@
 
 /* Define to 1 if you have enum bpf_stats_type in linux/bpf.h. */
 #cmakedefine HAVE_BPF_STATS_TYPE 1
+
+/* Define to 1 if you have `libngtcp2_crypto_openssl` library. */
+#cmakedefine HAVE_LIBNGTCP2_CRYPTO_OPENSSL

configure.ac

@@ -22,10 +22,10 @@ dnl OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 dnl WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 dnl Do not change user variables!
-dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html
+dnl https://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html
 
 AC_PREREQ(2.61)
-AC_INIT([nghttp2], [1.45.0-DEV], [t-tujikawa@users.sourceforge.net])
+AC_INIT([nghttp2], [1.47.0-DEV], [t-tujikawa@users.sourceforge.net])
 AC_CONFIG_AUX_DIR([.])
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS([config.h])
@@ -43,10 +43,10 @@ AM_INIT_AUTOMAKE([subdir-objects])
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 
 dnl See versioning rule:
-dnl  http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
+dnl  https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
-AC_SUBST(LT_CURRENT, 34)
+AC_SUBST(LT_CURRENT, 35)
-AC_SUBST(LT_REVISION, 2)
+AC_SUBST(LT_REVISION, 1)
-AC_SUBST(LT_AGE, 20)
+AC_SUBST(LT_AGE, 21)
 
 major=`echo $PACKAGE_VERSION |cut -d. -f1 | sed -e "s/[^0-9]//g"`
 minor=`echo $PACKAGE_VERSION |cut -d. -f2 | sed -e "s/[^0-9]//g"`
@@ -220,6 +220,11 @@ PKG_PROG_PKG_CONFIG([0.20])
 
 AM_PATH_PYTHON([3.8],, [:])
 
+if test "x$request_python_bindings" = "xyes" &&
+   test "x$PYTHON" = "x:"; then
+  AC_MSG_ERROR([python was requested (enable-python-bindings) but not found])
+fi
+
 if [test "x$request_lib_only" = "xyes"]; then
   request_app=no
   request_hpack_tools=no
@@ -227,8 +232,10 @@ if [test "x$request_lib_only" = "xyes"]; then
   request_python_bindings=no
 fi
 
-if [test "x$request_python_bindings" != "xno"]; then
+if test "x$request_python_bindings" != "xno" &&
-  AX_PYTHON_DEVEL([>= '3.8'])
+   test "x$PYTHON" != "x:"; then
+  # version check is broken
+  AX_PYTHON_DEVEL()
 fi
 
 if test "x${cython_path}" = "x"; then
@@ -342,7 +349,7 @@ APPLDFLAGS=
 case "$host_os" in
   *android*)
     android_build=yes
-    # android does not need -pthread, but needs followng 3 libs for C++
+    # android does not need -pthread, but needs following 3 libs for C++
     APPLDFLAGS="$APPLDFLAGS -lstdc++ -latomic -lsupc++"
     ;;
   *)
@@ -467,6 +474,7 @@ if test "x${request_openssl}" != "xno"; then
     CFLAGS="$OPENSSL_CFLAGS $CFLAGS"
     LIBS="$OPENSSL_LIBS $LIBS"
 
+    # quictls/openssl has SSL_is_quic.
     have_ssl_is_quic=no
     AC_MSG_CHECKING([for SSL_is_quic])
     AC_LINK_IFELSE([AC_LANG_PROGRAM([[
@@ -478,6 +486,17 @@ if test "x${request_openssl}" != "xno"; then
     [AC_MSG_RESULT([yes]); have_ssl_is_quic=yes],
     [AC_MSG_RESULT([no]); have_ssl_is_quic=no])
 
+    # boringssl has SSL_set_quic_early_data_context.
+    AC_MSG_CHECKING([for SSL_set_quic_early_data_context])
+    AC_LINK_IFELSE([AC_LANG_PROGRAM([[
+      #include <openssl/ssl.h>
+    ]], [[
+      SSL *ssl = NULL;
+      SSL_set_quic_early_data_context(ssl, NULL, 0);
+    ]])],
+    [AC_MSG_RESULT([yes]); have_boringssl_quic=yes],
+    [AC_MSG_RESULT([no]); have_boringssl_quic=no])
+
     CFLAGS="$save_CFLAGS"
     LIBS="$save_LIBS"
   fi
@@ -506,7 +525,7 @@ fi
 # ngtcp2 (for src)
 have_libngtcp2=no
 if test "x${request_libngtcp2}" != "xno"; then
-  PKG_CHECK_MODULES([LIBNGTCP2], [libngtcp2 >= 0.0.0], [have_libngtcp2=yes],
+  PKG_CHECK_MODULES([LIBNGTCP2], [libngtcp2 >= 0.1.0], [have_libngtcp2=yes],
                     [have_libngtcp2=no])
   if test "x${have_libngtcp2}" = "xno"; then
     AC_MSG_NOTICE($LIBNGTCP2_PKG_ERRORS)
@@ -520,25 +539,52 @@ fi
 
 # ngtcp2_crypto_openssl (for src)
 have_libngtcp2_crypto_openssl=no
-if test "x${request_libngtcp2}" != "xno"; then
+if test "x${have_ssl_is_quic}" = "xyes" &&
+   test "x${request_libngtcp2}" != "xno"; then
   PKG_CHECK_MODULES([LIBNGTCP2_CRYPTO_OPENSSL],
                     [libngtcp2_crypto_openssl >= 0.0.0],
                     [have_libngtcp2_crypto_openssl=yes],
                     [have_libngtcp2_crypto_openssl=no])
   if test "x${have_libngtcp2_crypto_openssl}" = "xno"; then
     AC_MSG_NOTICE($LIBNGTCP2_CRYPTO_OPENSSL_PKG_ERRORS)
+  else
+    AC_DEFINE([HAVE_LIBNGTCP2_CRYPTO_OPENSSL], [1],
+              [Define to 1 if you have `libngtcp2_crypto_openssl` library.])
   fi
 fi
 
-if test "x${request_libngtcp2}" = "xyes" &&
+if test "x${have_ssl_is_quic}" = "xyes" &&
+   test "x${request_libngtcp2}" = "xyes" &&
    test "x${have_libngtcp2_crypto_openssl}" != "xyes"; then
   AC_MSG_ERROR([libngtcp2_crypto_openssl was requested (--with-libngtcp2) but not found])
 fi
 
+# ngtcp2_crypto_boringssl (for src)
+have_libngtcp2_crypto_boringssl=no
+if test "x${have_boringssl_quic}" = "xyes" &&
+   test "x${request_libngtcp2}" != "xno"; then
+  PKG_CHECK_MODULES([LIBNGTCP2_CRYPTO_BORINGSSL],
+                    [libngtcp2_crypto_boringssl >= 0.0.0],
+                    [have_libngtcp2_crypto_boringssl=yes],
+                    [have_libngtcp2_crypto_boringssl=no])
+  if test "x${have_libngtcp2_crypto_boringssl}" = "xno"; then
+    AC_MSG_NOTICE($LIBNGTCP2_CRYPTO_BORINGSSL_PKG_ERRORS)
+  else
+    AC_DEFINE([HAVE_LIBNGTCP2_CRYPTO_BORINGSSL], [1],
+              [Define to 1 if you have `libngtcp2_crypto_boringssl` library.])
+  fi
+fi
+
+if test "x${have_boringssl_quic}" = "xyes" &&
+   test "x${request_libngtcp2}" = "xyes" &&
+   test "x${have_libngtcp2_crypto_boringssl}" != "xyes"; then
+  AC_MSG_ERROR([libngtcp2_crypto_boringssl was requested (--with-libngtcp2) but not found])
+fi
+
 # nghttp3 (for src)
 have_libnghttp3=no
 if test "x${request_libnghttp3}" != "xno"; then
-  PKG_CHECK_MODULES([LIBNGHTTP3], [libnghttp3 >= 0.0.0], [have_libnghttp3=yes],
+  PKG_CHECK_MODULES([LIBNGHTTP3], [libnghttp3 >= 0.1.0], [have_libnghttp3=yes],
                     [have_libnghttp3=no])
   if test "x${have_libnghttp3}" = "xno"; then
     AC_MSG_NOTICE($LIBNGHTTP3_PKG_ERRORS)
@@ -742,9 +788,11 @@ AM_CONDITIONAL([ENABLE_APP], [ test "x${enable_app}" = "xyes" ])
 # Check HTTP/3 support
 enable_http3=no
 if test "x${request_http3}" != "xno" &&
-   test "x${have_ssl_is_quic}" = "xyes" &&
+   (test "x${have_ssl_is_quic}" = "xyes" ||
+    test "x${have_boringssl_quic}" = "xyes") &&
    test "x${have_libngtcp2}" = "xyes" &&
-   test "x${have_libngtcp2_crypto_openssl}" = "xyes" &&
+   (test "x${have_libngtcp2_crypto_openssl}" = "xyes" ||
+    test "x${have_libngtcp2_crypto_boringssl}" = "xyes") &&
    test "x${have_libnghttp3}" = "xyes"; then
   enable_http3=yes
   AC_DEFINE([ENABLE_HTTP3], [1], [Define to 1 if HTTP/3 is enabled.])
@@ -983,31 +1031,6 @@ AC_CHECK_DECLS([initgroups], [], [], [[
   #include <grp.h>
 ]])
 
-have_netinet_udp_h_udp_segment=no
-AC_CHECK_DECL([UDP_SEGMENT], [have_netinet_udp_h_udp_segment=yes],
-              [have_netinet_udp_h_udp_segment=no], [[
-  #include <netinet/udp.h>
-]])
-
-if test "x$have_netinet_udp_h_udp_segment" = "xno"; then
-  have_linux_udp_h_udp_segment=no
-  AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
-                     [[
-                       #include <linux/udp.h>
-                     ]],
-                     [[
-		       #if UDP_SEGMENT != 103
-		       exit(1)
-		       #endif
-                     ]])],
-		    [have_linux_udp_h_udp_segment=yes],
-		    [have_linux_udp_h_udp_segment=no])
-
-  if test "x$have_linux_udp_h_udp_segment" = "xyes"; then
-    EXTRA_DEFS="$EXTRA_DEFS -DUDP_SEGMENT=103"
-  fi
-fi
-
 save_CFLAGS=$CFLAGS
 save_CXXFLAGS=$CXXFLAGS
 
@@ -1202,6 +1225,7 @@ AC_MSG_NOTICE([summary of build options:
       Libc-ares:      ${have_libcares} (CFLAGS='${LIBCARES_CFLAGS}' LIBS='${LIBCARES_LIBS}')
       libngtcp2:      ${have_libngtcp2} (CFLAGS='${LIBNGTCP2_CFLAGS}' LIBS='${LIBNGTCP2_LIBS}')
       libngtcp2_crypto_openssl: ${have_libngtcp2_crypto_openssl} (CFLAGS='${LIBNGTCP2_CRYPTO_OPENSSL_CFLAGS}' LIBS='${LIBNGTCP2_CRYPTO_OPENSSL_LIBS}')
+      libngtcp2_crypto_boringssl: ${have_libngtcp2_crypto_boringssl} (CFLAGS='${LIBNGTCP2_CRYPTO_BORINGSSL_CFLAGS}' LIBS='${LIBNGTCP2_CRYPTO_BORINGSSL_LIBS}')
       libnghttp3:     ${have_libnghttp3} (CFLAGS='${LIBNGHTTP3_CFLAGS}' LIBS='${LIBNGHTTP3_LIBS}')
       libbpf:         ${have_libbpf} (CFLAGS='${LIBBPF_CFLAGS}' LIBS='${LIBBPF_LIBS}')
       Libevent(SSL):  ${have_libevent_openssl} (CFLAGS='${LIBEVENT_OPENSSL_CFLAGS}' LIBS='${LIBEVENT_OPENSSL_LIBS}')

doc/CMakeLists.txt

@@ -184,9 +184,9 @@ set(EXTRA_DIST
   sources/python-apiref.rst
   sources/building-android-binary.rst
   sources/contribute.rst
-  _exts/sphinxcontrib/LICENSE.rubydomain
+  _exts/rubydomain/LICENSE.rubydomain
-  _exts/sphinxcontrib/__init__.py
+  _exts/rubydomain/__init__.py
-  _exts/sphinxcontrib/rubydomain.py
+  _exts/rubydomain/rubydomain.py
   _themes/sphinx_rtd_theme/__init__.py
   _themes/sphinx_rtd_theme/breadcrumbs.html
   _themes/sphinx_rtd_theme/footer.html

doc/Makefile.am

@@ -206,9 +206,9 @@ EXTRA_DIST = \
 	sources/building-android-binary.rst \
 	sources/contribute.rst \
 	sources/security.rst \
-	_exts/sphinxcontrib/LICENSE.rubydomain \
+	_exts/rubydomain/LICENSE.rubydomain \
-	_exts/sphinxcontrib/__init__.py \
+	_exts/rubydomain/__init__.py \
-	_exts/sphinxcontrib/rubydomain.py \
+	_exts/rubydomain/rubydomain.py \
 	_themes/sphinx_rtd_theme/__init__.py \
 	_themes/sphinx_rtd_theme/breadcrumbs.html \
 	_themes/sphinx_rtd_theme/footer.html \
@@ -272,7 +272,7 @@ EXTRA_DIST = \
 
 # You can set these variables from the command line.
 SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
+SPHINXBUILD   ?= sphinx-build
 PAPER         =
 BUILDDIR      = manual
 

doc/_exts/rubydomain/rubydomain.py

@@ -493,7 +493,7 @@ class RubyModuleIndex(Index):
         # list of all modules, sorted by module name
         modules = sorted(_iteritems(self.domain.data['modules']),
                          key=lambda x: x[0].lower())
-        # sort out collapsable modules
+        # sort out collapsible modules
         prev_modname = ''
         num_toplevels = 0
         for modname, (docname, synopsis, platforms, deprecated) in modules:

doc/bash_completion/h2load

@@ -8,7 +8,7 @@ _h2load()
     _get_comp_words_by_ref cur prev
     case $cur in
         -*)
-            COMPREPLY=( $( compgen -W '--connection-window-bits --clients --verbose --ciphers --rate --no-tls-proto --connect-to --header-table-size --requests --log-file --base-uri --no-udp-gso --h1 --threads --npn-list --rate-period --rps --data --tls13-ciphers --version --connection-inactivity-timeout --timing-script-file --encoder-header-table-size --max-concurrent-streams --connection-active-timeout --input-file --help --groups --window-bits --qlog-file-base --warm-up-time --duration --header ' -- "$cur" ) )
+            COMPREPLY=( $( compgen -W '--requests --clients --threads --input-file --max-concurrent-streams --window-bits --connection-window-bits --header --ciphers --tls13-ciphers --no-tls-proto --data --rate --rate-period --duration --warm-up-time --connection-active-timeout --connection-inactivity-timeout --timing-script-file --base-uri --npn-list --h1 --header-table-size --encoder-header-table-size --log-file --qlog-file-base --connect-to --rps --groups --no-udp-gso --max-udp-payload-size --verbose --version --help ' -- "$cur" ) )
             ;;
         *)
             _filedir

doc/bash_completion/make_bash_completion.py

@@ -1,8 +1,6 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-from __future__ import unicode_literals
-from __future__ import print_function
 import subprocess
 import io
 import re

doc/bash_completion/nghttp

@@ -8,7 +8,7 @@ _nghttp()
     _get_comp_words_by_ref cur prev
     case $cur in
         -*)
-            COMPREPLY=( $( compgen -W '--no-push --verbose --no-dep --get-assets --har --header-table-size --multiply --encoder-header-table-size --padding --hexdump --max-concurrent-streams --continuation --connection-window-bits --peer-max-concurrent-streams --timeout --data --no-content-length --version --color --cert --upgrade --remote-name --trailer --weight --help --key --null-out --window-bits --expect-continue --stat --no-verify-peer --header ' -- "$cur" ) )
+            COMPREPLY=( $( compgen -W '--verbose --null-out --remote-name --timeout --window-bits --connection-window-bits --get-assets --stat --header --trailer --cert --key --data --multiply --upgrade --weight --peer-max-concurrent-streams --header-table-size --encoder-header-table-size --padding --har --color --continuation --no-content-length --no-dep --hexdump --no-push --max-concurrent-streams --expect-continue --no-verify-peer --version --help ' -- "$cur" ) )
             ;;
         *)
             _filedir

doc/bash_completion/nghttpd

@@ -8,7 +8,7 @@ _nghttpd()
     _get_comp_words_by_ref cur prev
     case $cur in
         -*)
-            COMPREPLY=( $( compgen -W '--htdocs --verbose --daemon --echo-upload --error-gzip --push --header-table-size --encoder-header-table-size --padding --hexdump --max-concurrent-streams --no-tls --connection-window-bits --mime-types-file --no-content-length --workers --version --color --early-response --dh-param-file --trailer --address --window-bits --verify-client --help ' -- "$cur" ) )
+            COMPREPLY=( $( compgen -W '--address --daemon --verify-client --htdocs --verbose --no-tls --header-table-size --encoder-header-table-size --color --push --padding --max-concurrent-streams --workers --error-gzip --window-bits --connection-window-bits --dh-param-file --early-response --trailer --hexdump --echo-upload --mime-types-file --no-content-length --version --help ' -- "$cur" ) )
             ;;
         *)
             _filedir

doc/bash_completion/nghttpx

@@ -8,7 +8,7 @@ _nghttpx()
     _get_comp_words_by_ref cur prev
     case $cur in
         -*)
-            COMPREPLY=( $( compgen -W '--worker-read-rate --include --frontend-http2-dump-response-header --tls-ticket-key-file --verify-client-cacert --max-response-header-fields --backend-http2-window-size --tls13-client-ciphers --frontend-keep-alive-timeout --backend-request-buffer --max-request-header-fields --backend-connect-timeout --tls-max-proto-version --frontend-quic-idle-timeout --frontend-http3-read-timeout --conf --dns-lookup-timeout --frontend-http3-connection-window-size --backend-http2-max-concurrent-streams --worker-write-burst --no-quic-bpf --npn-list --dns-max-try --fetch-ocsp-response-file --tls-session-cache-memcached-cert-file --no-via --mruby-file --no-server-push --stream-read-timeout --client-ciphers --ocsp-update-interval --forwarded-for --accesslog-syslog --dns-cache-timeout --frontend-http2-read-timeout --listener-disable-timeout --frontend-http3-window-size --ciphers --client-psk-secrets --strip-incoming-x-forwarded-for --no-server-rewrite --private-key-passwd-file --backend-keep-alive-timeout --backend-http-proxy-uri --frontend-max-requests --tls-no-postpone-early-data --rlimit-nofile --no-strip-incoming-x-forwarded-proto --tls-ticket-key-memcached-cert-file --no-verify-ocsp --forwarded-by --tls-session-cache-memcached-private-key-file --error-page --ocsp-startup --backend-write-timeout --tls-dyn-rec-warmup-threshold --frontend-http2-window-size --tls-ticket-key-memcached-max-retry --http2-no-cookie-crumbling --frontend-http3-max-concurrent-streams --worker-read-burst --dh-param-file --accesslog-format --errorlog-syslog --redirect-https-port --request-header-field-buffer --api-max-request-body --frontend-http2-decoder-dynamic-table-size --errorlog-file --frontend-http2-max-concurrent-streams --psk-secrets --frontend-write-timeout --tls-ticket-key-cipher --read-burst --no-add-x-forwarded-proto --backend --server-name --insecure --backend-max-backoff --log-level --host-rewrite --quic-bpf-program-file --http2-altsvc --frontend-http3-max-connection-window-size --tls-ticket-key-memcached-interval --frontend-http2-setting-timeout --frontend-http2-connection-window-size --worker-frontend-connections --syslog-facility --fastopen --no-location-rewrite --single-thread --no-http2-cipher-block-list --tls-session-cache-memcached --no-ocsp --backend-response-buffer --tls-min-proto-version --workers --add-x-forwarded-for --add-forwarded --frontend-http3-max-window-size --worker-write-rate --add-request-header --backend-http2-settings-timeout --subcert --ignore-per-pattern-mruby-error --ecdh-curves --no-kqueue --help --frontend-frame-debug --tls-sct-dir --pid-file --frontend-http2-dump-request-header --daemon --write-rate --altsvc --backend-http2-decoder-dynamic-table-size --no-strip-incoming-early-data --user --verify-client-tolerate-expired --frontend-read-timeout --tls-ticket-key-memcached-max-fail --backlog --write-burst --backend-connections-per-host --tls-max-early-data --response-header-field-buffer --tls-ticket-key-memcached-address-family --padding --tls-session-cache-memcached-address-family --stream-write-timeout --cacert --frontend-quic-debug-log --tls-ticket-key-memcached-private-key-file --accesslog-write-early --backend-address-family --backend-http2-connection-window-size --tls13-ciphers --client-no-http2-cipher-block-list --version --add-response-header --backend-read-timeout --frontend-http2-optimize-window-size --frontend --accesslog-file --http2-proxy --backend-http2-encoder-dynamic-table-size --client-private-key-file --single-process --client-cert-file --tls-ticket-key-memcached --tls-dyn-rec-idle-timeout --frontend-http2-optimize-write-buffer-size --verify-client --frontend-http2-encoder-dynamic-table-size --read-rate --backend-connections-per-frontend --strip-incoming-forwarded ' -- "$cur" ) )
+            COMPREPLY=( $( compgen -W '--backend --frontend --backlog --backend-address-family --backend-http-proxy-uri --workers --single-thread --read-rate --read-burst --write-rate --write-burst --worker-read-rate --worker-read-burst --worker-write-rate --worker-write-burst --worker-frontend-connections --backend-connections-per-host --backend-connections-per-frontend --rlimit-nofile --rlimit-memlock --backend-request-buffer --backend-response-buffer --fastopen --no-kqueue --frontend-http2-read-timeout --frontend-http3-read-timeout --frontend-read-timeout --frontend-write-timeout --frontend-keep-alive-timeout --stream-read-timeout --stream-write-timeout --backend-read-timeout --backend-write-timeout --backend-connect-timeout --backend-keep-alive-timeout --listener-disable-timeout --frontend-http2-setting-timeout --backend-http2-settings-timeout --backend-max-backoff --ciphers --tls13-ciphers --client-ciphers --tls13-client-ciphers --ecdh-curves --insecure --cacert --private-key-passwd-file --subcert --dh-param-file --npn-list --verify-client --verify-client-cacert --verify-client-tolerate-expired --client-private-key-file --client-cert-file --tls-min-proto-version --tls-max-proto-version --tls-ticket-key-file --tls-ticket-key-memcached --tls-ticket-key-memcached-address-family --tls-ticket-key-memcached-interval --tls-ticket-key-memcached-max-retry --tls-ticket-key-memcached-max-fail --tls-ticket-key-cipher --tls-ticket-key-memcached-cert-file --tls-ticket-key-memcached-private-key-file --fetch-ocsp-response-file --ocsp-update-interval --ocsp-startup --no-verify-ocsp --no-ocsp --tls-session-cache-memcached --tls-session-cache-memcached-address-family --tls-session-cache-memcached-cert-file --tls-session-cache-memcached-private-key-file --tls-dyn-rec-warmup-threshold --tls-dyn-rec-idle-timeout --no-http2-cipher-block-list --client-no-http2-cipher-block-list --tls-sct-dir --psk-secrets --client-psk-secrets --tls-no-postpone-early-data --tls-max-early-data --frontend-http2-max-concurrent-streams --backend-http2-max-concurrent-streams --frontend-http2-window-size --frontend-http2-connection-window-size --backend-http2-window-size --backend-http2-connection-window-size --http2-no-cookie-crumbling --padding --no-server-push --frontend-http2-optimize-write-buffer-size --frontend-http2-optimize-window-size --frontend-http2-encoder-dynamic-table-size --frontend-http2-decoder-dynamic-table-size --backend-http2-encoder-dynamic-table-size --backend-http2-decoder-dynamic-table-size --http2-proxy --log-level --accesslog-file --accesslog-syslog --accesslog-format --accesslog-write-early --errorlog-file --errorlog-syslog --syslog-facility --add-x-forwarded-for --strip-incoming-x-forwarded-for --no-add-x-forwarded-proto --no-strip-incoming-x-forwarded-proto --add-forwarded --strip-incoming-forwarded --forwarded-by --forwarded-for --no-via --no-strip-incoming-early-data --no-location-rewrite --host-rewrite --altsvc --http2-altsvc --add-request-header --add-response-header --request-header-field-buffer --max-request-header-fields --response-header-field-buffer --max-response-header-fields --error-page --server-name --no-server-rewrite --redirect-https-port --api-max-request-body --dns-cache-timeout --dns-lookup-timeout --dns-max-try --frontend-max-requests --frontend-http2-dump-request-header --frontend-http2-dump-response-header --frontend-frame-debug --daemon --pid-file --user --single-process --max-worker-processes --worker-process-grace-shutdown-period --mruby-file --ignore-per-pattern-mruby-error --frontend-quic-idle-timeout --frontend-quic-debug-log --quic-bpf-program-file --frontend-quic-early-data --frontend-quic-qlog-dir --frontend-quic-require-token --frontend-quic-congestion-controller --frontend-quic-secret-file --quic-server-id --frontend-quic-initial-rtt --no-quic-bpf --frontend-http3-window-size --frontend-http3-connection-window-size --frontend-http3-max-window-size --frontend-http3-max-connection-window-size --frontend-http3-max-concurrent-streams --conf --include --version --help ' -- "$cur" ) )
             ;;
         *)
             _filedir

doc/conf.py.in

@@ -41,7 +41,7 @@ import sys, os
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #sys.path.insert(0, os.path.abspath('.'))
 
-sys.path.append(os.path.abspath('@top_srcdir@/doc/_exts'))
+sys.path.insert(0, os.path.abspath('@top_srcdir@/doc/_exts'))
 
 # -- General configuration -----------------------------------------------------
 
@@ -50,7 +50,7 @@ sys.path.append(os.path.abspath('@top_srcdir@/doc/_exts'))
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinxcontrib.rubydomain']
+extensions = ['rubydomain.rubydomain']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['@top_srcdir@/_templates']

doc/h2load.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "H2LOAD" "1" "Aug 31, 2021" "1.45.0-DEV" "nghttp2"
+.TH "H2LOAD" "1" "Oct 19, 2021" "1.46.0" "nghttp2"
 .SH NAME
 h2load \- HTTP/2 benchmarking tool
 .
@@ -328,6 +328,11 @@ Disable UDP GSO.
 .UNINDENT
 .INDENT 0.0
 .TP
+.B \-\-max\-udp\-payload\-size=<SIZE>
+Specify the maximum outgoing UDP datagram payload size.
+.UNINDENT
+.INDENT 0.0
+.TP
 .B \-v, \-\-verbose
 Output debug information.
 .UNINDENT

doc/h2load.1.rst

@@ -277,6 +277,10 @@ OPTIONS
 
     Disable UDP GSO.
 
+.. option:: --max-udp-payload-size=<SIZE>
+
+    Specify the maximum outgoing UDP datagram payload size.
+
 .. option:: -v, --verbose
 
     Output debug information.

doc/nghttp.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "NGHTTP" "1" "Aug 31, 2021" "1.45.0-DEV" "nghttp2"
+.TH "NGHTTP" "1" "Oct 19, 2021" "1.46.0" "nghttp2"
 .SH NAME
 nghttp \- HTTP/2 client
 .

doc/nghttpd.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "NGHTTPD" "1" "Aug 31, 2021" "1.45.0-DEV" "nghttp2"
+.TH "NGHTTPD" "1" "Oct 19, 2021" "1.46.0" "nghttp2"
 .SH NAME
 nghttpd \- HTTP/2 server
 .

doc/nghttpx.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "NGHTTPX" "1" "Aug 31, 2021" "1.45.0-DEV" "nghttp2"
+.TH "NGHTTPX" "1" "Oct 19, 2021" "1.46.0" "nghttp2"
 .SH NAME
 nghttpx \- HTTP/2 proxy
 .
@@ -35,7 +35,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 \fBnghttpx\fP [OPTIONS]... [<PRIVATE_KEY> <CERT>]
 .SH DESCRIPTION
 .sp
-A reverse proxy for HTTP/2, and HTTP/1.
+A reverse proxy for HTTP/3, HTTP/2, and HTTP/1.
 .INDENT 0.0
 .TP
 .B <PRIVATE_KEY>
@@ -503,6 +503,15 @@ Default: \fB0\fP
 .UNINDENT
 .INDENT 0.0
 .TP
+.B \-\-rlimit\-memlock=<N>
+Set maximum number of bytes of memory that may be locked
+into  RAM.  If  0 is  given,  nghttpx does  not set  the
+limit.
+.sp
+Default: \fB0\fP
+.UNINDENT
+.INDENT 0.0
+.TP
 .B \-\-backend\-request\-buffer=<SIZE>
 Set buffer size used to store backend request.
 .sp
@@ -1089,12 +1098,13 @@ option.  But be aware its implications.
 .INDENT 0.0
 .TP
 .B \-\-tls\-no\-postpone\-early\-data
-By default,  nghttpx postpones forwarding  HTTP requests
+By  default,   except  for  QUIC   connections,  nghttpx
-sent in early data, including those sent in partially in
+postpones forwarding  HTTP requests sent in  early data,
-it, until TLS handshake finishes.  If all backend server
+including  those  sent in  partially  in  it, until  TLS
-recognizes "Early\-Data" header  field, using this option
+handshake  finishes.  If  all backend  server recognizes
-makes nghttpx  not postpone  forwarding request  and get
+"Early\-Data"  header  field,  using  this  option  makes
-full potential of 0\-RTT data.
+nghttpx  not postpone  forwarding request  and get  full
+potential of 0\-RTT data.
 .UNINDENT
 .INDENT 0.0
 .TP
@@ -1507,7 +1517,7 @@ they  are treated  as  nothing is  specified.  They  are
 advertised  in alt\-svc  header  field  only in  HTTP/1.1
 frontend.   This option  can be  used multiple  times to
 specify multiple alternative services.
-Example: \fI\%\-\-altsvc\fP="h2,443,,,ma=3600; persist=1\(aq
+Example: \fI\%\-\-altsvc\fP="h2,443,,,ma=3600; persist=1"
 .UNINDENT
 .INDENT 0.0
 .TP
@@ -1704,6 +1714,33 @@ process.   nghttpx still  spawns  additional process  if
 neverbleed  is used.   In the  single process  mode, the
 signal handling feature is disabled.
 .UNINDENT
+.INDENT 0.0
+.TP
+.B \-\-max\-worker\-processes=<N>
+The maximum number of  worker processes.  nghttpx spawns
+new worker  process when  it reloads  its configuration.
+The previous worker  process enters graceful termination
+period and will terminate  when it finishes handling the
+existing    connections.     However,    if    reloading
+configurations  happen   very  frequently,   the  worker
+processes might be piled up if they take a bit long time
+to finish  the existing connections.  With  this option,
+if  the number  of  worker processes  exceeds the  given
+value,   the  oldest   worker   process  is   terminated
+immediately.  Specifying 0 means no  limit and it is the
+default behaviour.
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-\-worker\-process\-grace\-shutdown\-period=<DURATION>
+Maximum  period  for  a   worker  process  to  terminate
+gracefully.  When  a worker  process enters  in graceful
+shutdown   period  (e.g.,   when  nghttpx   reloads  its
+configuration)  and  it  does not  finish  handling  the
+existing connections in the given  period of time, it is
+immediately terminated.  Specifying 0 means no limit and
+it is the default behaviour.
+.UNINDENT
 .SS Scripting
 .INDENT 0.0
 .TP
@@ -1741,6 +1778,86 @@ Default: \fB/usr/local/lib/nghttp2/reuseport_kern.o\fP
 .UNINDENT
 .INDENT 0.0
 .TP
+.B \-\-frontend\-quic\-early\-data
+Enable early data on frontend QUIC connections.  nghttpx
+sends "Early\-Data" header field to a backend server if a
+request is received in early  data and handshake has not
+finished.  All backend servers should deal with possibly
+replayed requests.
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-\-frontend\-quic\-qlog\-dir=<DIR>
+Specify a  directory where  a qlog  file is  written for
+frontend QUIC  connections.  A qlog file  is created per
+each QUIC  connection.  The  file name is  ISO8601 basic
+format, followed by "\-", server Source Connection ID and
+".qlog".
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-\-frontend\-quic\-require\-token
+Require an address validation  token for a frontend QUIC
+connection.   Server sends  a token  in Retry  packet or
+NEW_TOKEN frame in the previous connection.
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-\-frontend\-quic\-congestion\-controller=<CC>
+Specify a congestion controller algorithm for a frontend
+QUIC  connection.   <CC>  should be  either  "cubic"  or
+"bbr".
+.sp
+Default: \fBcubic\fP
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-\-frontend\-quic\-secret\-file=<PATH>
+Path to file that contains secure random data to be used
+as QUIC keying materials.  It is used to derive keys for
+encrypting tokens and Connection IDs.  It is not used to
+encrypt  QUIC  packets.  Each  line  of  this file  must
+contain  exactly  136  bytes  hex\-encoded  string  (when
+decoded the byte string is  68 bytes long).  The first 2
+bits of  decoded byte  string are  used to  identify the
+keying material.  An  empty line or a  line which starts
+\(aq#\(aq  is ignored.   The file  can contain  more than  one
+keying materials.  Because the  identifier is 2 bits, at
+most 4 keying materials are  read and the remaining data
+is discarded.  The first keying  material in the file is
+primarily  used for  encryption and  decryption for  new
+connection.  The other ones are used to decrypt data for
+the  existing connections.   Specifying multiple  keying
+materials enables  key rotation.   Please note  that key
+rotation  does  not  occur automatically.   User  should
+update  files  or  change  options  values  and  restart
+nghttpx gracefully.   If opening  or reading  given file
+fails, all loaded keying  materials are discarded and it
+is treated as if none of  this option is given.  If this
+option is not  given or an error  occurred while opening
+or  reading  a  file,  a keying  material  is  generated
+internally on startup and reload.
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-\-quic\-server\-id=<HEXSTRING>
+Specify server  ID encoded in Connection  ID to identify
+this  particular  server  instance.   Connection  ID  is
+encrypted and  this part is  not visible in  public.  It
+must be 4  bytes long and must be encoded  in hex string
+(which is 8  bytes long).  If this option  is omitted, a
+random   server  ID   is   generated   on  startup   and
+configuration reload.
+.UNINDENT
+.INDENT 0.0
+.TP
+.B \-\-frontend\-quic\-initial\-rtt=<DURATION>
+Specify the initial RTT of the frontend QUIC connection.
+.sp
+Default: \fB333ms\fP
+.UNINDENT
+.INDENT 0.0
+.TP
 .B \-\-no\-quic\-bpf
 Disable eBPF.
 .UNINDENT

doc/nghttpx.1.rst

@@ -14,7 +14,7 @@ SYNOPSIS
 DESCRIPTION
 -----------
 
-A reverse proxy for HTTP/2, and HTTP/1.
+A reverse proxy for HTTP/3, HTTP/2, and HTTP/1.
 
 .. describe:: <PRIVATE_KEY>
 
@@ -472,6 +472,14 @@ Performance
 
     Default: ``0``
 
+.. option:: --rlimit-memlock=<N>
+
+    Set maximum number of bytes of memory that may be locked
+    into  RAM.  If  0 is  given,  nghttpx does  not set  the
+    limit.
+
+    Default: ``0``
+
 .. option:: --backend-request-buffer=<SIZE>
 
     Set buffer size used to store backend request.
@@ -1003,12 +1011,13 @@ SSL/TLS
 
 .. option:: --tls-no-postpone-early-data
 
-    By default,  nghttpx postpones forwarding  HTTP requests
+    By  default,   except  for  QUIC   connections,  nghttpx
-    sent in early data, including those sent in partially in
+    postpones forwarding  HTTP requests sent in  early data,
-    it, until TLS handshake finishes.  If all backend server
+    including  those  sent in  partially  in  it, until  TLS
-    recognizes "Early-Data" header  field, using this option
+    handshake  finishes.  If  all backend  server recognizes
-    makes nghttpx  not postpone  forwarding request  and get
+    "Early-Data"  header  field,  using  this  option  makes
-    full potential of 0-RTT data.
+    nghttpx  not postpone  forwarding request  and get  full
+    potential of 0-RTT data.
 
 .. option:: --tls-max-early-data=<SIZE>
 
@@ -1367,7 +1376,7 @@ HTTP
     advertised  in alt-svc  header  field  only in  HTTP/1.1
     frontend.   This option  can be  used multiple  times to
     specify multiple alternative services.
-    Example: :option:`--altsvc`\="h2,443,,,ma=3600; persist=1'
+    Example: :option:`--altsvc`\="h2,443,,,ma=3600; persist=1"
 
 .. option:: --http2-altsvc=<PROTOID,PORT[,HOST,[ORIGIN[,PARAMS]]]>
 
@@ -1553,6 +1562,31 @@ Process
     neverbleed  is used.   In the  single process  mode, the
     signal handling feature is disabled.
 
+.. option:: --max-worker-processes=<N>
+
+    The maximum number of  worker processes.  nghttpx spawns
+    new worker  process when  it reloads  its configuration.
+    The previous worker  process enters graceful termination
+    period and will terminate  when it finishes handling the
+    existing    connections.     However,    if    reloading
+    configurations  happen   very  frequently,   the  worker
+    processes might be piled up if they take a bit long time
+    to finish  the existing connections.  With  this option,
+    if  the number  of  worker processes  exceeds the  given
+    value,   the  oldest   worker   process  is   terminated
+    immediately.  Specifying 0 means no  limit and it is the
+    default behaviour.
+
+.. option:: --worker-process-grace-shutdown-period=<DURATION>
+
+    Maximum  period  for  a   worker  process  to  terminate
+    gracefully.  When  a worker  process enters  in graceful
+    shutdown   period  (e.g.,   when  nghttpx   reloads  its
+    configuration)  and  it  does not  finish  handling  the
+    existing connections in the given  period of time, it is
+    immediately terminated.  Specifying 0 means no limit and
+    it is the default behaviour.
+
 
 Scripting
 ~~~~~~~~~
@@ -1589,6 +1623,79 @@ HTTP/3 and QUIC
 
     Default: ``/usr/local/lib/nghttp2/reuseport_kern.o``
 
+.. option:: --frontend-quic-early-data
+
+    Enable early data on frontend QUIC connections.  nghttpx
+    sends "Early-Data" header field to a backend server if a
+    request is received in early  data and handshake has not
+    finished.  All backend servers should deal with possibly
+    replayed requests.
+
+.. option:: --frontend-quic-qlog-dir=<DIR>
+
+    Specify a  directory where  a qlog  file is  written for
+    frontend QUIC  connections.  A qlog file  is created per
+    each QUIC  connection.  The  file name is  ISO8601 basic
+    format, followed by "-", server Source Connection ID and
+    ".qlog".
+
+.. option:: --frontend-quic-require-token
+
+    Require an address validation  token for a frontend QUIC
+    connection.   Server sends  a token  in Retry  packet or
+    NEW_TOKEN frame in the previous connection.
+
+.. option:: --frontend-quic-congestion-controller=<CC>
+
+    Specify a congestion controller algorithm for a frontend
+    QUIC  connection.   <CC>  should be  either  "cubic"  or
+    "bbr".
+
+    Default: ``cubic``
+
+.. option:: --frontend-quic-secret-file=<PATH>
+
+    Path to file that contains secure random data to be used
+    as QUIC keying materials.  It is used to derive keys for
+    encrypting tokens and Connection IDs.  It is not used to
+    encrypt  QUIC  packets.  Each  line  of  this file  must
+    contain  exactly  136  bytes  hex-encoded  string  (when
+    decoded the byte string is  68 bytes long).  The first 2
+    bits of  decoded byte  string are  used to  identify the
+    keying material.  An  empty line or a  line which starts
+    '#'  is ignored.   The file  can contain  more than  one
+    keying materials.  Because the  identifier is 2 bits, at
+    most 4 keying materials are  read and the remaining data
+    is discarded.  The first keying  material in the file is
+    primarily  used for  encryption and  decryption for  new
+    connection.  The other ones are used to decrypt data for
+    the  existing connections.   Specifying multiple  keying
+    materials enables  key rotation.   Please note  that key
+    rotation  does  not  occur automatically.   User  should
+    update  files  or  change  options  values  and  restart
+    nghttpx gracefully.   If opening  or reading  given file
+    fails, all loaded keying  materials are discarded and it
+    is treated as if none of  this option is given.  If this
+    option is not  given or an error  occurred while opening
+    or  reading  a  file,  a keying  material  is  generated
+    internally on startup and reload.
+
+.. option:: --quic-server-id=<HEXSTRING>
+
+    Specify server  ID encoded in Connection  ID to identify
+    this  particular  server  instance.   Connection  ID  is
+    encrypted and  this part is  not visible in  public.  It
+    must be 4  bytes long and must be encoded  in hex string
+    (which is 8  bytes long).  If this option  is omitted, a
+    random   server  ID   is   generated   on  startup   and
+    configuration reload.
+
+.. option:: --frontend-quic-initial-rtt=<DURATION>
+
+    Specify the initial RTT of the frontend QUIC connection.
+
+    Default: ``333ms``
+
 .. option:: --no-quic-bpf
 
     Disable eBPF.

doc/sources/nghttpx-howto.rst

@@ -14,8 +14,8 @@ Default mode
 
 If nghttpx is invoked without :option:`--http2-proxy`, it operates in
 default mode.  In this mode, it works as reverse proxy (gateway) for
-both HTTP/2 and HTTP/1 clients to backend servers.  This is also known
+HTTP/3, HTTP/2 and HTTP/1 clients to backend servers.  This is also
-as "HTTP/2 router".
+known as "HTTP/2 router".
 
 By default, frontend connection is encrypted using SSL/TLS.  So
 server's private key and certificate must be supplied to the command
@@ -28,6 +28,9 @@ the frontend, and an HTTP/1 connection can be upgraded to HTTP/2 using
 HTTP Upgrade.  Starting HTTP/2 connection by sending HTTP/2 connection
 preface is also supported.
 
+In order to receive HTTP/3 traffic, use ``quic`` parameter in
+:option:`--frontend` option (.e.g, ``--frontend='*,443;quic'``)
+
 nghttpx can listen on multiple frontend addresses.  This is achieved
 by using multiple :option:`--frontend` options.  For each frontend
 address, TLS can be enabled or disabled.
@@ -509,6 +512,8 @@ Bootstrapping WebSockets with HTTP/2 for both frontend and backend
 connections.  This feature is enabled by default and no configuration
 is required.
 
+WebSockets over HTTP/3 is also supported.
+
 HTTP/3
 ------
 
@@ -529,7 +534,28 @@ nghttpx does not support HTTP/3 on backend connection.
 
 Hot swapping (SIGUSR2) or configuration reload (SIGHUP) require eBPF
 program.  Without eBPF, old worker processes keep getting HTTP/3
-traffic and do not work as intended.
+traffic and do not work as intended.  The QUIC keying material to
+encrypt Connection ID must be set with
+:option:`--frontend-quic-secret-file` and must provide the existing
+keys in order to keep the existing connections alive during reload.
+
+The construction of Connection ID closely follows Block Cipher CID
+Algorithm described in `QUIC-LB draft
+<https://datatracker.ietf.org/doc/html/draft-ietf-quic-load-balancers>`_.
+A Connection ID that nghttpx generates is always 20 bytes long.  It
+uses first 2 bits as a configuration ID.  The remaining bits in the
+first byte are reserved and random.  The next 4 bytes are server ID.
+The next 4 bytes are used to route UDP datagram to a correct
+``SO_REUSEPORT`` socket.  The remaining bytes are randomly generated.
+The server ID and the next 12 bytes are encrypted with AES-ECB.  The
+key is derived from the keying materials stored in a file specified by
+:option:`--frontend-quic-secret-file`.  The first 2 bits of keying
+material in the file is used as a configuration ID.  The remaining
+bits and following 3 bytes are reserved and unused.  The next 32 bytes
+are used as an initial secret.  The remaining 32 bytes are used as a
+salt.  The encryption key is generated by `HKDF
+<https://datatracker.ietf.org/doc/html/rfc5869>`_ with SHA256 and
+these keying materials and ``connection id encryption key`` as info.
 
 In order announce that HTTP/3 endpoint is available, you should
 specify alt-svc header field.  For example, the following options send

docker/Dockerfile

@@ -0,0 +1,78 @@
+FROM debian:11 as build
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        git clang make binutils autoconf automake autotools-dev libtool \
+        pkg-config \
+        zlib1g-dev libev-dev libjemalloc-dev ruby-dev libc-ares-dev bison \
+        libelf-dev
+
+RUN git clone --depth 1 -b OpenSSL_1_1_1m+quic https://github.com/quictls/openssl && \
+    cd openssl && \
+    ./config --openssldir=/etc/ssl && \
+    make -j$(nproc) && \
+    make install_sw && \
+    cd .. && \
+    rm -rf openssl
+
+RUN git clone --depth 1 -b v0.1.1 https://github.com/ngtcp2/nghttp3 && \
+    cd nghttp3 && \
+    autoreconf -i && \
+    ./configure --enable-lib-only && \
+    make -j$(nproc) && \
+    make install-strip && \
+    cd .. && \
+    rm -rf nghttp3
+
+RUN git clone --depth 1 -b v0.1.0 https://github.com/ngtcp2/ngtcp2 && \
+    cd ngtcp2 && \
+    autoreconf -i && \
+    ./configure --enable-lib-only \
+        LIBTOOL_LDFLAGS="-static-libtool-libs" \
+        OPENSSL_LIBS="-l:libssl.a -l:libcrypto.a -ldl -lpthread" \
+        PKG_CONFIG_PATH="/usr/local/lib64/pkgconfig" && \
+    make -j$(nproc) && \
+    make install-strip && \
+    cd .. && \
+    rm -rf ngtcp2
+
+RUN git clone --depth 1 -b v0.4.0 https://github.com/libbpf/libbpf && \
+    cd libbpf && \
+    PREFIX=/usr/local make -C src install && \
+    cd .. && \
+    rm -rf libbpf
+
+RUN git clone --depth 1 https://github.com/nghttp2/nghttp2.git && \
+    cd nghttp2 && \
+    git submodule update --init && \
+    autoreconf -i && \
+    ./configure --disable-examples --disable-hpack-tools \
+        --disable-python-bindings --with-mruby --with-neverbleed \
+        --enable-http3 --with-libbpf \
+        CC=clang CXX=clang++ \
+        LIBTOOL_LDFLAGS="-static-libtool-libs" \
+        OPENSSL_LIBS="-l:libssl.a -l:libcrypto.a -ldl -pthread" \
+        LIBEV_LIBS="-l:libev.a" \
+        JEMALLOC_LIBS="-l:libjemalloc.a" \
+        LIBCARES_LIBS="-l:libcares.a" \
+        ZLIB_LIBS="-l:libz.a" \
+        LIBBPF_LIBS="-L/usr/local/lib64 -l:libbpf.a -l:libelf.a" \
+        LDFLAGS="-static-libgcc -static-libstdc++" \
+        PKG_CONFIG_PATH="/usr/local/lib64/pkgconfig" && \
+    make -j$(nproc) install-strip && \
+    cd .. && \
+    rm -rf nghttp2
+
+FROM gcr.io/distroless/base-debian11
+
+COPY --from=build \
+    /usr/local/share/nghttp2/ \
+    /usr/local/share/nghttp2/
+COPY --from=build \
+    /usr/local/bin/h2load \
+    /usr/local/bin/nghttpx \
+    /usr/local/bin/nghttp \
+    /usr/local/bin/nghttpd \
+    /usr/local/bin/
+COPY --from=build /usr/local/lib/nghttp2/reuseport_kern.o \
+    /usr/local/lib/nghttp2/

docker/Dockerfile-h2load-http3

@@ -1,39 +0,0 @@
-FROM debian:10 as build
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-        git g++ make binutils autoconf automake autotools-dev libtool \
-        pkg-config \
-        zlib1g-dev libev-dev libjemalloc-dev ruby-dev libc-ares-dev bison && \
-    git clone --depth 1 -b OpenSSL_1_1_1l+quic https://github.com/quictls/openssl && \
-    cd openssl && ./config enable-tls1_3 --openssldir=/etc/ssl && make -j$(nproc) && make install_sw && cd .. && rm -rf openssl && \
-    git clone --depth 1 https://github.com/ngtcp2/nghttp3 && \
-    cd nghttp3 && autoreconf -i && \
-    ./configure --enable-lib-only && \
-    make -j$(nproc) && make install-strip && cd .. && rm -rf nghttp3 && \
-    git clone --depth 1 https://github.com/ngtcp2/ngtcp2 && \
-    cd ngtcp2 && autoreconf -i && \
-    ./configure --enable-lib-only \
-        LIBTOOL_LDFLAGS="-static-libtool-libs" \
-        OPENSSL_LIBS="-l:libssl.a -l:libcrypto.a -ldl -lpthread" && \
-    make -j$(nproc) && make install-strip && cd .. && rm -rf ngtcp2 && \
-    git clone --depth 1 https://github.com/nghttp2/nghttp2.git && \
-    cd nghttp2 && \
-    git submodule update --init && autoreconf -i && \
-    ./configure --disable-examples --disable-hpack-tools \
-        --disable-python-bindings --with-mruby --with-neverbleed \
-        --enable-http3 \
-        LIBTOOL_LDFLAGS="-static-libtool-libs" \
-        LIBS="-ldl -pthread" \
-        OPENSSL_LIBS="-l:libssl.a -l:libcrypto.a" \
-        LIBEV_LIBS="-l:libev.a" \
-        JEMALLOC_LIBS="-l:libjemalloc.a" \
-        LIBCARES_LIBS="-l:libcares.a" \
-        ZLIB_LIBS="-l:libz.a" && \
-    make -j$(nproc) install-strip
-
-FROM gcr.io/distroless/cc-debian10
-
-COPY --from=build /usr/local/bin/h2load /usr/local/bin/
-
-ENTRYPOINT ["/usr/local/bin/h2load"]

docker/README.rst

@@ -0,0 +1,25 @@
+Dockerfile
+==========
+
+Dockerfile creates the applications bundled with nghttp2.
+These applications are:
+
+- nghttp
+- nghttpd
+- nghttpx
+- h2load
+
+HTTP/3 and eBPF features are enabled.
+
+In order to run nghttpx with HTTP/3 endpoint, you need to run the
+image with the escalated privilege and higher memlock value.  Here is
+the example command-line to run nghttpx to listen to HTTP/3 on port
+443, assuming that the current directory contains a private key and a
+certificate in server.key and server.crt respectively :
+
+.. code-block:: text
+
+   $ docker run --rm -it -v $PWD:/shared --net=host --privileged \
+         nghttp2 nghttpx \
+         /shared/server.key /shared/server.crt \
+         -f'*,443;quic' --rlimit-memlock 524288

examples/client.c

@@ -380,6 +380,10 @@ static void init_ssl_ctx(SSL_CTX *ssl_ctx) {
 #ifndef OPENSSL_NO_NEXTPROTONEG
   SSL_CTX_set_next_proto_select_cb(ssl_ctx, select_next_proto_cb, NULL);
 #endif /* !OPENSSL_NO_NEXTPROTONEG */
+
+#if OPENSSL_VERSION_NUMBER >= 0x10002000L
+  SSL_CTX_set_alpn_protos(ssl_ctx, (const unsigned char *)"\x02h2", 3);
+#endif /* OPENSSL_VERSION_NUMBER >= 0x10002000L */
 }
 
 static void ssl_handshake(SSL *ssl, int fd) {
@@ -544,7 +548,7 @@ static void fetch_uri(const struct URI *uri) {
   if (fd == -1) {
     die("Could not open file descriptor");
   }
-  ssl_ctx = SSL_CTX_new(SSLv23_client_method());
+  ssl_ctx = SSL_CTX_new(TLS_client_method());
   if (ssl_ctx == NULL) {
     dief("SSL_CTX_new", ERR_error_string(ERR_get_error(), NULL));
   }
@@ -715,8 +719,18 @@ int main(int argc, char **argv) {
   act.sa_handler = SIG_IGN;
   sigaction(SIGPIPE, &act, 0);
 
+#if OPENSSL_VERSION_NUMBER >= 0x1010000fL
+  /* No explicit initialization is required. */
+#elif defined(OPENSSL_IS_BORINGSSL)
+  CRYPTO_library_init();
+#else  /* !(OPENSSL_VERSION_NUMBER >= 0x1010000fL) &&                          \
+          !defined(OPENSSL_IS_BORINGSSL) */
+  OPENSSL_config(NULL);
   SSL_load_error_strings();
   SSL_library_init();
+  OpenSSL_add_all_algorithms();
+#endif /* !(OPENSSL_VERSION_NUMBER >= 0x1010000fL) &&                          \
+          !defined(OPENSSL_IS_BORINGSSL) */
 
   rv = parse_uri(&uri, argv[1]);
   if (rv != 0) {

examples/deflate.c

@@ -44,7 +44,7 @@ static void deflate(nghttp2_hd_deflater *deflater,
 static int inflate_header_block(nghttp2_hd_inflater *inflater, uint8_t *in,
                                 size_t inlen, int final);
 
-int main() {
+int main(void) {
   int rv;
   nghttp2_hd_deflater *deflater;
   nghttp2_hd_inflater *inflater;

examples/libevent-client.c

@@ -328,7 +328,7 @@ static int select_next_proto_cb(SSL *ssl, unsigned char **out,
 /* Create SSL_CTX. */
 static SSL_CTX *create_ssl_ctx(void) {
   SSL_CTX *ssl_ctx;
-  ssl_ctx = SSL_CTX_new(SSLv23_client_method());
+  ssl_ctx = SSL_CTX_new(TLS_client_method());
   if (!ssl_ctx) {
     errx(1, "Could not create SSL/TLS context: %s",
          ERR_error_string(ERR_get_error(), NULL));
@@ -617,8 +617,18 @@ int main(int argc, char **argv) {
   act.sa_handler = SIG_IGN;
   sigaction(SIGPIPE, &act, NULL);
 
+#if OPENSSL_VERSION_NUMBER >= 0x1010000fL
+  /* No explicit initialization is required. */
+#elif defined(OPENSSL_IS_BORINGSSL)
+  CRYPTO_library_init();
+#else  /* !(OPENSSL_VERSION_NUMBER >= 0x1010000fL) &&                          \
+          !defined(OPENSSL_IS_BORINGSSL) */
+  OPENSSL_config(NULL);
   SSL_load_error_strings();
   SSL_library_init();
+  OpenSSL_add_all_algorithms();
+#endif /* !(OPENSSL_VERSION_NUMBER >= 0x1010000fL) &&                          \
+          !defined(OPENSSL_IS_BORINGSSL) */
 
   run(argv[1]);
   return 0;

examples/libevent-server.c

@@ -143,7 +143,7 @@ static int alpn_select_proto_cb(SSL *ssl, const unsigned char **out,
 static SSL_CTX *create_ssl_ctx(const char *key_file, const char *cert_file) {
   SSL_CTX *ssl_ctx;
 
-  ssl_ctx = SSL_CTX_new(SSLv23_server_method());
+  ssl_ctx = SSL_CTX_new(TLS_server_method());
   if (!ssl_ctx) {
     errx(1, "Could not create SSL/TLS context: %s",
          ERR_error_string(ERR_get_error(), NULL));
@@ -153,14 +153,9 @@ static SSL_CTX *create_ssl_ctx(const char *key_file, const char *cert_file) {
                           SSL_OP_NO_COMPRESSION |
                           SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION);
 #if OPENSSL_VERSION_NUMBER >= 0x30000000L
-  {
+  if (SSL_CTX_set1_curves_list(ssl_ctx, "P-256") != 1) {
-    EVP_PKEY *ecdh;
+    errx(1, "SSL_CTX_set1_curves_list failed: %s",
-    ecdh = EVP_EC_gen("P-256");
+         ERR_error_string(ERR_get_error(), NULL));
-    if (!ecdh) {
-      errx(1, "EVP_EC_gen failed: %s", ERR_error_string(ERR_get_error(), NULL));
-    }
-    SSL_CTX_set_tmp_ecdh(ssl_ctx, ecdh);
-    EVP_PKEY_free(ecdh);
   }
 #else  /* !(OPENSSL_VERSION_NUMBER >= 0x30000000L) */
   {
@@ -822,8 +817,18 @@ int main(int argc, char **argv) {
   act.sa_handler = SIG_IGN;
   sigaction(SIGPIPE, &act, NULL);
 
+#if OPENSSL_VERSION_NUMBER >= 0x1010000fL
+  /* No explicit initialization is required. */
+#elif defined(OPENSSL_IS_BORINGSSL)
+  CRYPTO_library_init();
+#else  /* !(OPENSSL_VERSION_NUMBER >= 0x1010000fL) &&                          \
+          !defined(OPENSSL_IS_BORINGSSL) */
+  OPENSSL_config(NULL);
   SSL_load_error_strings();
   SSL_library_init();
+  OpenSSL_add_all_algorithms();
+#endif /* !(OPENSSL_VERSION_NUMBER >= 0x1010000fL) &&                          \
+          !defined(OPENSSL_IS_BORINGSSL) */
 
   run(argv[1], argv[2], argv[3]);
   return 0;

gennghttpxfun.py

@@ -190,6 +190,14 @@ OPTIONS = [
     "frontend-http3-max-concurrent-streams",
     "frontend-quic-early-data",
     "frontend-quic-qlog-dir",
+    "frontend-quic-require-token",
+    "frontend-quic-congestion-controller",
+    "quic-server-id",
+    "frontend-quic-secret-file",
+    "rlimit-memlock",
+    "max-worker-processes",
+    "worker-process-grace-shutdown-period",
+    "frontend-quic-initial-rtt",
 ]
 
 LOGVARS = [

integration-tests/nghttpx_http1_test.go

@@ -5,14 +5,15 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"golang.org/x/net/http2/hpack"
-	"golang.org/x/net/websocket"
 	"io"
 	"net/http"
 	"regexp"
 	"syscall"
 	"testing"
 	"time"
+
+	"golang.org/x/net/http2/hpack"
+	"golang.org/x/net/websocket"
 )
 
 // TestH1H1PlainGET tests whether simple HTTP/1 GET request works.
@@ -34,7 +35,7 @@ func TestH1H1PlainGET(t *testing.T) {
 }
 
 // TestH1H1PlainGETClose tests whether simple HTTP/1 GET request with
-// Connetion: close request header field works.
+// Connection: close request header field works.
 func TestH1H1PlainGETClose(t *testing.T) {
 	st := newServerTester(nil, t, noopHandler)
 	defer st.Close()
@@ -1171,3 +1172,31 @@ Content-Length: 1000000
 		t.Errorf("status: %v; want %v", got, want)
 	}
 }
+
+// TestH1H1ChunkedEndsPrematurely tests that an HTTP/1.1 request fails
+// if the backend chunked encoded response ends prematurely.
+func TestH1H1ChunkedEndsPrematurely(t *testing.T) {
+	st := newServerTester(nil, t, func(w http.ResponseWriter, r *http.Request) {
+		hj, ok := w.(http.Hijacker)
+		if !ok {
+			http.Error(w, "Could not hijack the connection", http.StatusInternalServerError)
+			return
+		}
+		conn, bufrw, err := hj.Hijack()
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		defer conn.Close()
+		bufrw.WriteString("HTTP/1.1 200\r\nTransfer-Encoding: chunked\r\n\r\n")
+		bufrw.Flush()
+	})
+	defer st.Close()
+
+	_, err := st.http1(requestParam{
+		name: "TestH1H1ChunkedEndsPrematurely",
+	})
+	if err == nil {
+		t.Fatal("st.http1() should fail")
+	}
+}

integration-tests/nghttpx_http2_test.go

@@ -565,7 +565,7 @@ func TestH2H1BadResponseCL(t *testing.T) {
 		t.Fatalf("Error st.http2() = %v", err)
 	}
 
-	want := http2.ErrCodeProtocol
+	want := http2.ErrCodeInternal
 	if res.errCode != want {
 		t.Errorf("res.errCode = %v; want %v", res.errCode, want)
 	}
@@ -2838,3 +2838,35 @@ func TestH2ResponseBeforeRequestEnd(t *testing.T) {
 		t.Errorf("res.status: %v; want %v", got, want)
 	}
 }
+
+// TestH2H1ChunkedEndsPrematurely tests that a stream is reset if the
+// backend chunked encoded response ends prematurely.
+func TestH2H1ChunkedEndsPrematurely(t *testing.T) {
+	st := newServerTester(nil, t, func(w http.ResponseWriter, r *http.Request) {
+		hj, ok := w.(http.Hijacker)
+		if !ok {
+			http.Error(w, "Could not hijack the connection", http.StatusInternalServerError)
+			return
+		}
+		conn, bufrw, err := hj.Hijack()
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		defer conn.Close()
+		bufrw.WriteString("HTTP/1.1 200\r\nTransfer-Encoding: chunked\r\n\r\n")
+		bufrw.Flush()
+	})
+	defer st.Close()
+
+	res, err := st.http2(requestParam{
+		name: "TestH2H1ChunkedEndsPrematurely",
+	})
+	if err != nil {
+		t.Fatalf("Error st.http2() = %v", err)
+	}
+
+	if got, want := res.errCode, http2.ErrCodeInternal; got != want {
+		t.Errorf("res.errCode = %v; want %v", got, want)
+	}
+}

lib/nghttp2_buf.h

@@ -99,7 +99,7 @@ void nghttp2_buf_free(nghttp2_buf *buf, nghttp2_mem *mem);
  * |new_cap|. If extensions took place, buffer pointers in |buf| will
  * change.
  *
- * This function returns 0 if it succeeds, or one of the followings
+ * This function returns 0 if it succeeds, or one of the following
  * negative error codes:
  *
  * NGHTTP2_ERR_NOMEM

lib/nghttp2_frame.c

@@ -654,8 +654,6 @@ int nghttp2_frame_unpack_goaway_payload2(nghttp2_goaway *frame,
     var_gift_payloadlen = 0;
   }
 
-  payloadlen -= var_gift_payloadlen;
-
   if (!var_gift_payloadlen) {
     var_gift_payload = NULL;
   } else {

lib/nghttp2_frame.h

@@ -46,7 +46,7 @@
 #define NGHTTP2_MAX_FRAME_SIZE_MIN (1 << 14)
 
 #define NGHTTP2_MAX_PAYLOADLEN 16384
-/* The one frame buffer length for tranmission.  We may use several of
+/* The one frame buffer length for transmission.  We may use several of
    them to support CONTINUATION.  To account for Pad Length field, we
    allocate extra 1 byte, which saves extra large memcopying. */
 #define NGHTTP2_FRAMEBUF_CHUNKLEN                                              \

lib/nghttp2_hd.c

@@ -1263,6 +1263,8 @@ int nghttp2_hd_inflate_change_table_size(
     return NGHTTP2_ERR_INVALID_STATE;
   }
 
+  inflater->settings_hd_table_bufsize_max = settings_max_dynamic_table_size;
+
   /* It seems that encoder is not required to send dynamic table size
      update if the table size is not changed after applying
      SETTINGS_HEADER_TABLE_SIZE.  RFC 7541 is ambiguous here, but this
@@ -1275,13 +1277,12 @@ int nghttp2_hd_inflate_change_table_size(
     /* Remember minimum value, and validate that encoder sends the
        value less than or equal to this. */
     inflater->min_hd_table_bufsize_max = settings_max_dynamic_table_size;
+
+    inflater->ctx.hd_table_bufsize_max = settings_max_dynamic_table_size;
+
+    hd_context_shrink_table_size(&inflater->ctx, NULL);
   }
 
-  inflater->settings_hd_table_bufsize_max = settings_max_dynamic_table_size;
-
-  inflater->ctx.hd_table_bufsize_max = settings_max_dynamic_table_size;
-
-  hd_context_shrink_table_size(&inflater->ctx, NULL);
   return 0;
 }
 

lib/nghttp2_map.c

@@ -189,6 +189,7 @@ static int map_resize(nghttp2_map *map, uint32_t new_tablelen,
   nghttp2_map_bucket *new_table;
   nghttp2_map_bucket *bkt;
   int rv;
+  (void)rv;
 
   new_table =
       nghttp2_mem_calloc(map->mem, new_tablelen, sizeof(nghttp2_map_bucket));

lib/nghttp2_net.h

@@ -42,7 +42,7 @@
 #if defined(WIN32)
 /* Windows requires ws2_32 library for ntonl family functions.  We
    define inline functions for those function so that we don't have
-   dependeny on that lib. */
+   dependency on that lib. */
 
 #  ifdef _MSC_VER
 #    define STIN static __inline

lib/nghttp2_outbound_item.h

@@ -111,7 +111,7 @@ struct nghttp2_outbound_item {
      to this structure to avoid frequent memory allocation. */
   nghttp2_ext_frame_payload ext_frame_payload;
   nghttp2_aux_data aux_data;
-  /* The priority used in priority comparion.  Smaller is served
+  /* The priority used in priority comparison.  Smaller is served
      earlier.  For PING, SETTINGS and non-DATA frames (excluding
      response HEADERS frame) have dedicated cycle value defined above.
      For DATA frame, cycle is computed by taking into account of

lib/nghttp2_pq.h

@@ -114,7 +114,7 @@ typedef int (*nghttp2_pq_item_cb)(nghttp2_pq_entry *item, void *arg);
 void nghttp2_pq_update(nghttp2_pq *pq, nghttp2_pq_item_cb fun, void *arg);
 
 /*
- * Applys |fun| to each item in |pq|.  The |arg| is passed as arg
+ * Applies |fun| to each item in |pq|.  The |arg| is passed as arg
  * parameter to callback function.  This function must not change the
  * ordering key.  If the return value from callback is nonzero, this
  * function returns 1 immediately without iterating remaining items.

lib/nghttp2_session.c

@@ -5341,7 +5341,7 @@ static ssize_t inbound_frame_compute_pad(nghttp2_inbound_frame *iframe) {
 
 /*
  * This function returns the effective payload length in the data of
- * length |readlen| when the remaning payload is |payloadleft|. The
+ * length |readlen| when the remaining payload is |payloadleft|. The
  * |payloadleft| does not include |readlen|. If padding was started
  * strictly before this data chunk, this function returns -1.
  */

lib/nghttp2_session.h

@@ -408,7 +408,7 @@ int nghttp2_session_add_rst_stream(nghttp2_session *session, int32_t stream_id,
                                    uint32_t error_code);
 
 /*
- * Adds PING frame. This is a convenient functin built on top of
+ * Adds PING frame. This is a convenient function built on top of
  * nghttp2_session_add_frame() to add PING easily.
  *
  * If the |opaque_data| is not NULL, it must point to 8 bytes memory

lib/nghttp2_stream.c

@@ -33,7 +33,7 @@
 #include "nghttp2_frame.h"
 
 /* Maximum distance between any two stream's cycle in the same
-   prirority queue.  Imagine stream A's cycle is A, and stream B's
+   priority queue.  Imagine stream A's cycle is A, and stream B's
    cycle is B, and A < B.  The cycle is unsigned 32 bit integer, it
    may get overflow.  Because of how we calculate the next cycle
    value, if B - A is less than or equals to

lib/nghttp2_submit.c

@@ -492,8 +492,6 @@ int nghttp2_session_set_local_window_size(nghttp2_session *session,
     return nghttp2_session_update_recv_stream_window_size(session, stream, 0,
                                                           1);
   }
-
-  return 0;
 }
 
 int nghttp2_submit_altsvc(nghttp2_session *session, uint8_t flags,

m4/ax_python_devel.m4

@@ -67,7 +67,7 @@
 #   modified version of the Autoconf Macro, you may extend this special
 #   exception to the GPL to apply to your modified version as well.
 
-#serial 21
+#serial 23
 
 AU_ALIAS([AC_PYTHON_DEVEL], [AX_PYTHON_DEVEL])
 AC_DEFUN([AX_PYTHON_DEVEL],[
@@ -135,16 +135,25 @@ variable to configure. See ``configure --help'' for reference.
 	#
 	# Check if you have distutils, else fail
 	#
-	AC_MSG_CHECKING([for the distutils Python package])
+	AC_MSG_CHECKING([for the sysconfig Python package])
-	ac_distutils_result=`$PYTHON -c "import distutils" 2>&1`
+	ac_sysconfig_result=`$PYTHON -c "import sysconfig" 2>&1`
 	if test $? -eq 0; then
 		AC_MSG_RESULT([yes])
+		IMPORT_SYSCONFIG="import sysconfig"
 	else
 		AC_MSG_RESULT([no])
-		AC_MSG_ERROR([cannot import Python module "distutils".
+
+		AC_MSG_CHECKING([for the distutils Python package])
+		ac_sysconfig_result=`$PYTHON -c "from distutils import sysconfig" 2>&1`
+		if test $? -eq 0; then
+			AC_MSG_RESULT([yes])
+			IMPORT_SYSCONFIG="from distutils import sysconfig"
+		else
+			AC_MSG_ERROR([cannot import Python module "distutils".
 Please check your Python installation. The error was:
-$ac_distutils_result])
+$ac_sysconfig_result])
-		PYTHON_VERSION=""
+			PYTHON_VERSION=""
+		fi
 	fi
 
 	#
@@ -152,10 +161,19 @@ $ac_distutils_result])
 	#
 	AC_MSG_CHECKING([for Python include path])
 	if test -z "$PYTHON_CPPFLAGS"; then
-		python_path=`$PYTHON -c "import distutils.sysconfig; \
+		if test "$IMPORT_SYSCONFIG" = "import sysconfig"; then
-			print (distutils.sysconfig.get_python_inc ());"`
+			# sysconfig module has different functions
-		plat_python_path=`$PYTHON -c "import distutils.sysconfig; \
+			python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \
-			print (distutils.sysconfig.get_python_inc (plat_specific=1));"`
+				print (sysconfig.get_path ('include'));"`
+			plat_python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \
+				print (sysconfig.get_path ('platinclude'));"`
+		else
+			# old distutils way
+			python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \
+				print (sysconfig.get_python_inc ());"`
+			plat_python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \
+				print (sysconfig.get_python_inc (plat_specific=1));"`
+		fi
 		if test -n "${python_path}"; then
 			if test "${plat_python_path}" != "${python_path}"; then
 				python_path="-I$python_path -I$plat_python_path"
@@ -179,7 +197,7 @@ $ac_distutils_result])
 
 # join all versioning strings, on some systems
 # major/minor numbers could be in different list elements
-from distutils.sysconfig import *
+from sysconfig import *
 e = get_config_var('VERSION')
 if e is not None:
 	print(e)
@@ -202,8 +220,8 @@ EOD`
 		ac_python_libdir=`cat<<EOD | $PYTHON -
 
 # There should be only one
-import distutils.sysconfig
+$IMPORT_SYSCONFIG
-e = distutils.sysconfig.get_config_var('LIBDIR')
+e = sysconfig.get_config_var('LIBDIR')
 if e is not None:
 	print (e)
 EOD`
@@ -211,8 +229,8 @@ EOD`
 		# Now, for the library:
 		ac_python_library=`cat<<EOD | $PYTHON -
 
-import distutils.sysconfig
+$IMPORT_SYSCONFIG
-c = distutils.sysconfig.get_config_vars()
+c = sysconfig.get_config_vars()
 if 'LDVERSION' in c:
 	print ('python'+c[['LDVERSION']])
 else:
@@ -231,7 +249,7 @@ EOD`
 		else
 			# old way: use libpython from python_configdir
 			ac_python_libdir=`$PYTHON -c \
-			  "from distutils.sysconfig import get_python_lib as f; \
+			  "from sysconfig import get_python_lib as f; \
 			  import os; \
 			  print (os.path.join(f(plat_specific=1, standard_lib=1), 'config'));"`
 			PYTHON_LIBS="-L$ac_python_libdir -lpython$ac_python_version"
@@ -252,19 +270,42 @@ EOD`
 	#
 	AC_MSG_CHECKING([for Python site-packages path])
 	if test -z "$PYTHON_SITE_PKG"; then
-		PYTHON_SITE_PKG=`$PYTHON -c "import distutils.sysconfig; \
+		if test "$IMPORT_SYSCONFIG" = "import sysconfig"; then
-			print (distutils.sysconfig.get_python_lib(0,0));"`
+			PYTHON_SITE_PKG=`$PYTHON -c "$IMPORT_SYSCONFIG; \
+				print (sysconfig.get_path('purelib'));"`
+		else
+			# distutils.sysconfig way
+			PYTHON_SITE_PKG=`$PYTHON -c "$IMPORT_SYSCONFIG; \
+				print (sysconfig.get_python_lib(0,0));"`
+		fi
 	fi
 	AC_MSG_RESULT([$PYTHON_SITE_PKG])
 	AC_SUBST([PYTHON_SITE_PKG])
 
+	#
+	# Check for platform-specific site packages
+	#
+	AC_MSG_CHECKING([for Python platform specific site-packages path])
+	if test -z "$PYTHON_SITE_PKG"; then
+		if test "$IMPORT_SYSCONFIG" = "import sysconfig"; then
+			PYTHON_PLATFORM_SITE_PKG=`$PYTHON -c "$IMPORT_SYSCONFIG; \
+				print (sysconfig.get_path('platlib'));"`
+		else
+			# distutils.sysconfig way
+			PYTHON_PLATFORM_SITE_PKG=`$PYTHON -c "$IMPORT_SYSCONFIG; \
+				print (sysconfig.get_python_lib(1,0));"`
+		fi
+	fi
+	AC_MSG_RESULT([$PYTHON_PLATFORM_SITE_PKG])
+	AC_SUBST([PYTHON_PLATFORM_SITE_PKG])
+
 	#
 	# libraries which must be linked in when embedding
 	#
 	AC_MSG_CHECKING(python extra libraries)
 	if test -z "$PYTHON_EXTRA_LIBS"; then
-	   PYTHON_EXTRA_LIBS=`$PYTHON -c "import distutils.sysconfig; \
+	   PYTHON_EXTRA_LIBS=`$PYTHON -c "$IMPORT_SYSCONFIG; \
-                conf = distutils.sysconfig.get_config_var; \
+                conf = sysconfig.get_config_var; \
                 print (conf('LIBS') + ' ' + conf('SYSLIBS'))"`
 	fi
 	AC_MSG_RESULT([$PYTHON_EXTRA_LIBS])
@@ -275,8 +316,8 @@ EOD`
 	#
 	AC_MSG_CHECKING(python extra linking flags)
 	if test -z "$PYTHON_EXTRA_LDFLAGS"; then
-		PYTHON_EXTRA_LDFLAGS=`$PYTHON -c "import distutils.sysconfig; \
+		PYTHON_EXTRA_LDFLAGS=`$PYTHON -c "$IMPORT_SYSCONFIG; \
-			conf = distutils.sysconfig.get_config_var; \
+			conf = sysconfig.get_config_var; \
 			print (conf('LINKFORSHARED'))"`
 	fi
 	AC_MSG_RESULT([$PYTHON_EXTRA_LDFLAGS])

python/nghttp2.pyx

@@ -701,7 +701,7 @@ cdef class _HTTP2SessionCoreBase:
             if outbuflen == 0:
                 break
             if outbuflen < 0:
-                raise Exception('nghttp2_session_mem_send faild: {}'.format\
+                raise Exception('nghttp2_session_mem_send failed: {}'.format\
                                 (_strerror(outbuflen)))
             self.transport.write(outbuf[:outbuflen])
 
@@ -1057,8 +1057,7 @@ if asyncio:
         """HTTP/2 request (stream) handler base class.
 
         The class is used to handle the HTTP/2 stream. By default, it does
-        not nothing. It must be subclassed to handle each event callback
+        nothing. It must be subclassed to handle each event callback method.
-        method.
 
         The first callback method invoked is on_headers(). It is called
         when HEADERS frame, which includes request header fields, is
@@ -1084,7 +1083,7 @@ if asyncio:
           address.
 
         client_certificate
-          May contain the client certifcate in its non-binary form
+          May contain the client certificate in its non-binary form
 
         stream_id
           Stream ID of this stream

src/HttpServer.cc

@@ -2110,7 +2110,7 @@ int HttpServer::run() {
   std::vector<unsigned char> next_proto;
 
   if (!config_->no_tls) {
-    ssl_ctx = SSL_CTX_new(SSLv23_server_method());
+    ssl_ctx = SSL_CTX_new(TLS_server_method());
     if (!ssl_ctx) {
       std::cerr << ERR_error_string(ERR_get_error(), nullptr) << std::endl;
       return -1;
@@ -2143,22 +2143,13 @@ int HttpServer::run() {
     SSL_CTX_set_session_cache_mode(ssl_ctx, SSL_SESS_CACHE_SERVER);
 
 #ifndef OPENSSL_NO_EC
-    // Disabled SSL_CTX_set_ecdh_auto, because computational cost of
+#  if !LIBRESSL_LEGACY_API && OPENSSL_VERSION_NUMBER >= 0x10002000L
-    // chosen curve is much higher than P-256.
+    if (SSL_CTX_set1_curves_list(ssl_ctx, "P-256") != 1) {
-
+      std::cerr << "SSL_CTX_set1_curves_list failed: "
-    //     SSL_CTX_set_ecdh_auto(ssl_ctx, 1);
-    // Use P-256, which is sufficiently secure at the time of this
-    // writing.
-#  if OPENSSL_3_0_0_API
-    auto ecdh = EVP_EC_gen("P-256");
-    if (ecdh == nullptr) {
-      std::cerr << "EC_KEY_new_by_curv_name failed: "
                 << ERR_error_string(ERR_get_error(), nullptr);
       return -1;
     }
-    SSL_CTX_set_tmp_ecdh(ssl_ctx, ecdh);
+#  else  // !(!LIBRESSL_LEGACY_API && OPENSSL_VERSION_NUMBER >= 0x10002000L)
-    EVP_PKEY_free(ecdh);
-#  else  // !OPENSSL_3_0_0_API
     auto ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1);
     if (ecdh == nullptr) {
       std::cerr << "EC_KEY_new_by_curv_name failed: "
@@ -2167,7 +2158,7 @@ int HttpServer::run() {
     }
     SSL_CTX_set_tmp_ecdh(ssl_ctx, ecdh);
     EC_KEY_free(ecdh);
-#  endif // !OPENSSL_3_0_0_API
+#  endif // !(!LIBRESSL_LEGACY_API && OPENSSL_VERSION_NUMBER >= 0x10002000L)
 #endif   // OPENSSL_NO_EC
 
     if (!config_->dh_param_file.empty()) {
@@ -2191,8 +2182,11 @@ int HttpServer::run() {
         return -1;
       }
 
-      SSL_CTX_set_tmp_dh(ssl_ctx, dh);
+      if (SSL_CTX_set0_tmp_dh_pkey(ssl_ctx, dh) != 1) {
-      EVP_PKEY_free(dh);
+        std::cerr << "SSL_CTX_set0_tmp_dh_pkey failed: "
+                  << ERR_error_string(ERR_get_error(), nullptr) << std::endl;
+        return -1;
+      }
 #else  // !OPENSSL_3_0_0_API
       auto dh = PEM_read_bio_DHparams(bio, nullptr, nullptr, nullptr);
 

src/Makefile.am

@@ -47,6 +47,7 @@ AM_CPPFLAGS = \
 	@LIBEV_CFLAGS@ \
 	@LIBNGHTTP3_CFLAGS@ \
 	@LIBNGTCP2_CRYPTO_OPENSSL_CFLAGS@ \
+	@LIBNGTCP2_CRYPTO_BORINGSSL_CFLAGS@ \
 	@LIBNGTCP2_CFLAGS@ \
 	@OPENSSL_CFLAGS@ \
 	@LIBCARES_CFLAGS@ \
@@ -65,6 +66,7 @@ LDADD = $(top_builddir)/lib/libnghttp2.la \
 	@LIBEV_LIBS@ \
 	@LIBNGHTTP3_LIBS@ \
 	@LIBNGTCP2_CRYPTO_OPENSSL_LIBS@ \
+	@LIBNGTCP2_CRYPTO_BORINGSSL_LIBS@ \
 	@LIBNGTCP2_LIBS@ \
 	@OPENSSL_LIBS@ \
 	@LIBCARES_LIBS@ \

src/h2load.cc

@@ -81,6 +81,7 @@ bool recorded(const std::chrono::steady_clock::time_point &t) {
 }
 } // namespace
 
+#if OPENSSL_1_1_1_API
 namespace {
 std::ofstream keylog_file;
 void keylog_callback(const SSL *ssl, const char *line) {
@@ -89,6 +90,7 @@ void keylog_callback(const SSL *ssl, const char *line) {
   keylog_file.flush();
 }
 } // namespace
+#endif // OPENSSL_1_1_1_API
 
 Config::Config()
     : ciphers(tls::DEFAULT_CIPHER_LIST),
@@ -104,6 +106,7 @@ Config::Config()
       max_concurrent_streams(1),
       window_bits(30),
       connection_window_bits(30),
+      max_frame_size(16_k),
       rate(0),
       rate_period(1.0),
       duration(0.0),
@@ -1514,7 +1517,8 @@ int get_ev_loop_flags() {
 
 Worker::Worker(uint32_t id, SSL_CTX *ssl_ctx, size_t req_todo, size_t nclients,
                size_t rate, size_t max_samples, Config *config)
-    : stats(req_todo, nclients),
+    : randgen(util::make_mt19937()),
+      stats(req_todo, nclients),
       loop(ev_loop_new(get_ev_loop_flags())),
       ssl_ctx(ssl_ctx),
       config(config),
@@ -2106,6 +2110,11 @@ Options:
               http/1.1  is used,  this  specifies the  number of  HTTP
               pipelining requests in-flight.
               Default: 1
+  -f, --max-frame-size=<SIZE>
+              Maximum frame size that the local endpoint is willing to
+              receive.
+              Default: )"
+      << util::utos_unit(config.max_frame_size) << R"(
   -w, --window-bits=<N>
               Sets the stream level initial window size to (2**<N>)-1.
               For QUIC, <N> is capped to 26 (roughly 64MiB).
@@ -2119,7 +2128,7 @@ Options:
   -H, --header=<HEADER>
               Add/Override a header to the requests.
   --ciphers=<SUITE>
-              Set  allowed cipher  list  for TLSv1.2  or ealier.   The
+              Set  allowed cipher  list  for TLSv1.2  or earlier.   The
               format of the string is described in OpenSSL ciphers(1).
               Default: )"
       << config.ciphers << R"(
@@ -2243,11 +2252,10 @@ Options:
               to buffering.  Status code is -1 for failed streams.
   --qlog-file-base=<PATH>
               Enable qlog output and specify base file name for qlogs.
-              Qlog  is emitted  for each connection.
+              Qlog is emitted  for each connection.  For  a given base
-              For  a  given  base  name "base", each  output file name
+              name   "base",    each   output   file    name   becomes
-              becomes  "base.M.N.qlog"  where M is worker ID  and N is
+              "base.M.N.sqlog" where M is worker ID and N is client ID
-              client ID (e.g. "base.0.3.qlog").
+              (e.g. "base.0.3.sqlog").  Only effective in QUIC runs.
-              Only effective in QUIC runs.
   --connect-to=<HOST>[:<PORT>]
               Host and port to connect  instead of using the authority
               in <URI>.
@@ -2299,6 +2307,7 @@ int main(int argc, char **argv) {
         {"threads", required_argument, nullptr, 't'},
         {"max-concurrent-streams", required_argument, nullptr, 'm'},
         {"window-bits", required_argument, nullptr, 'w'},
+        {"max-frame-size", required_argument, nullptr, 'f'},
         {"connection-window-bits", required_argument, nullptr, 'W'},
         {"input-file", required_argument, nullptr, 'i'},
         {"header", required_argument, nullptr, 'H'},
@@ -2330,7 +2339,7 @@ int main(int argc, char **argv) {
         {nullptr, 0, nullptr, 0}};
     int option_index = 0;
     auto c = getopt_long(argc, argv,
-                         "hvW:c:d:m:n:p:t:w:H:i:r:T:N:D:B:", long_options,
+                         "hvW:c:d:m:n:p:t:w:f:H:i:r:T:N:D:B:", long_options,
                          &option_index);
     if (c == -1) {
       break;
@@ -2376,6 +2385,24 @@ int main(int argc, char **argv) {
       }
       break;
     }
+    case 'f': {
+      auto n = util::parse_uint_with_unit(optarg);
+      if (n == -1) {
+        std::cerr << "--max-frame-size: bad option value: " << optarg
+                  << std::endl;
+        exit(EXIT_FAILURE);
+      }
+      if (static_cast<uint64_t>(n) < 16_k) {
+        std::cerr << "--max-frame-size: minimum 16384" << std::endl;
+        exit(EXIT_FAILURE);
+      }
+      if (static_cast<uint64_t>(n) > 16_m - 1) {
+        std::cerr << "--max-frame-size: maximum 16777215" << std::endl;
+        exit(EXIT_FAILURE);
+      }
+      config.max_frame_size = n;
+      break;
+    }
     case 'H': {
       char *header = optarg;
       // Skip first possible ':' in the header name
@@ -2809,7 +2836,7 @@ int main(int argc, char **argv) {
   act.sa_handler = SIG_IGN;
   sigaction(SIGPIPE, &act, nullptr);
 
-  auto ssl_ctx = SSL_CTX_new(SSLv23_client_method());
+  auto ssl_ctx = SSL_CTX_new(TLS_client_method());
   if (!ssl_ctx) {
     std::cerr << "Failed to create SSL_CTX: "
               << ERR_error_string(ERR_get_error(), nullptr) << std::endl;
@@ -2843,19 +2870,26 @@ int main(int argc, char **argv) {
     exit(EXIT_FAILURE);
   }
 
-#if OPENSSL_1_1_1_API
+#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
   if (SSL_CTX_set_ciphersuites(ssl_ctx, config.tls13_ciphers.c_str()) == 0) {
     std::cerr << "SSL_CTX_set_ciphersuites with " << config.tls13_ciphers
               << " failed: " << ERR_error_string(ERR_get_error(), nullptr)
               << std::endl;
     exit(EXIT_FAILURE);
   }
-#endif // OPENSSL_1_1_1_API
+#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
 
+#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
   if (SSL_CTX_set1_groups_list(ssl_ctx, config.groups.c_str()) != 1) {
     std::cerr << "SSL_CTX_set1_groups_list failed" << std::endl;
     exit(EXIT_FAILURE);
   }
+#else  // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
+  if (SSL_CTX_set1_curves_list(ssl_ctx, config.groups.c_str()) != 1) {
+    std::cerr << "SSL_CTX_set1_curves_list failed" << std::endl;
+    exit(EXIT_FAILURE);
+  }
+#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
 
 #ifndef OPENSSL_NO_NEXTPROTONEG
   SSL_CTX_set_next_proto_select_cb(ssl_ctx, client_select_next_proto_cb,

src/h2load.h

@@ -95,6 +95,7 @@ struct Config {
   ssize_t max_concurrent_streams;
   size_t window_bits;
   size_t connection_window_bits;
+  size_t max_frame_size;
   // rate at which connections should be made
   size_t rate;
   ev_tstamp rate_period;
@@ -269,6 +270,7 @@ struct Sampling {
 
 struct Worker {
   MemchunkPool mcpool;
+  std::mt19937 randgen;
   Stats stats;
   Sampling request_times_smp;
   Sampling client_smp;
@@ -474,8 +476,10 @@ struct Client {
   int quic_stream_stop_sending(int64_t stream_id, uint64_t app_error_code);
   int quic_extend_max_local_streams();
 
-  int quic_on_key(ngtcp2_crypto_level level, const uint8_t *rx_secret,
+  int quic_on_rx_secret(ngtcp2_crypto_level level, const uint8_t *secret,
-                  const uint8_t *tx_secret, size_t secretlen);
+                        size_t secretlen);
+  int quic_on_tx_secret(ngtcp2_crypto_level level, const uint8_t *secret,
+                        size_t secretlen);
   void quic_set_tls_alert(uint8_t alert);
 
   void quic_write_client_handshake(ngtcp2_crypto_level level,

src/h2load_http2_session.cc

@@ -215,7 +215,7 @@ void Http2Session::on_connect() {
 
   nghttp2_option_del(opt);
 
-  std::array<nghttp2_settings_entry, 3> iv;
+  std::array<nghttp2_settings_entry, 4> iv;
   size_t niv = 2;
   iv[0].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH;
   iv[0].value = 0;
@@ -227,6 +227,11 @@ void Http2Session::on_connect() {
     iv[niv].value = config->header_table_size;
     ++niv;
   }
+  if (config->max_frame_size != 16_k) {
+    iv[niv].settings_id = NGHTTP2_SETTINGS_MAX_FRAME_SIZE;
+    iv[niv].value = config->max_frame_size;
+    ++niv;
+  }
 
   rv = nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, iv.data(), niv);
 

src/h2load_http3_session.cc

@@ -213,19 +213,17 @@ void Http3Session::recv_header(int64_t stream_id, const nghttp3_vec *name,
 }
 
 namespace {
-int send_stop_sending(nghttp3_conn *conn, int64_t stream_id,
+int stop_sending(nghttp3_conn *conn, int64_t stream_id, uint64_t app_error_code,
-                      uint64_t app_error_code, void *user_data,
+                 void *user_data, void *stream_user_data) {
-                      void *stream_user_data) {
   auto s = static_cast<Http3Session *>(user_data);
-  if (s->send_stop_sending(stream_id, app_error_code) != 0) {
+  if (s->stop_sending(stream_id, app_error_code) != 0) {
     return NGHTTP3_ERR_CALLBACK_FAILURE;
   }
   return 0;
 }
 } // namespace
 
-int Http3Session::send_stop_sending(int64_t stream_id,
+int Http3Session::stop_sending(int64_t stream_id, uint64_t app_error_code) {
-                                    uint64_t app_error_code) {
   auto rv = ngtcp2_conn_shutdown_stream_read(client_->quic.conn, stream_id,
                                              app_error_code);
   if (rv != 0) {
@@ -300,14 +298,14 @@ int Http3Session::init_conn() {
       nullptr, // begin_trailers
       h2load::recv_header,
       nullptr, // end_trailers
-      h2load::send_stop_sending,
+      h2load::stop_sending,
   };
 
   auto config = client_->worker->config;
 
   nghttp3_settings settings;
   nghttp3_settings_default(&settings);
-  settings.qpack_max_table_capacity = config->header_table_size;
+  settings.qpack_max_dtable_capacity = config->header_table_size;
   settings.qpack_blocked_streams = 100;
 
   auto mem = nghttp3_mem_default();

src/h2load_http3_session.h

@@ -51,7 +51,7 @@ public:
   void begin_headers(int64_t stream_id);
   void recv_header(int64_t stream_id, const nghttp3_vec *name,
                    const nghttp3_vec *value);
-  int send_stop_sending(int64_t stream_id, uint64_t app_error_code);
+  int stop_sending(int64_t stream_id, uint64_t app_error_code);
 
   int close_stream(int64_t stream_id, uint64_t app_error_code);
   int shutdown_stream_read(int64_t stream_id);

src/h2load_quic.cc

@@ -28,18 +28,20 @@
 
 #include <iostream>
 
-#include <ngtcp2/ngtcp2_crypto_openssl.h>
+#ifdef HAVE_LIBNGTCP2_CRYPTO_OPENSSL
+#  include <ngtcp2/ngtcp2_crypto_openssl.h>
+#endif // HAVE_LIBNGTCP2_CRYPTO_OPENSSL
+#ifdef HAVE_LIBNGTCP2_CRYPTO_BORINGSSL
+#  include <ngtcp2/ngtcp2_crypto_boringssl.h>
+#endif // HAVE_LIBNGTCP2_CRYPTO_BORINGSSL
 
 #include <openssl/err.h>
+#include <openssl/rand.h>
 
 #include "h2load_http3_session.h"
 
 namespace h2load {
 
-namespace {
-auto randgen = util::make_mt19937();
-} // namespace
-
 namespace {
 int handshake_completed(ngtcp2_conn *conn, void *user_data) {
   auto c = static_cast<Client *>(user_data);
@@ -118,7 +120,7 @@ int stream_close(ngtcp2_conn *conn, uint32_t flags, int64_t stream_id,
   }
 
   if (c->quic_stream_close(stream_id, app_error_code) != 0) {
-    return -1;
+    return NGTCP2_ERR_CALLBACK_FAILURE;
   }
   return 0;
 }
@@ -138,7 +140,7 @@ int stream_reset(ngtcp2_conn *conn, int64_t stream_id, uint64_t final_size,
                  void *stream_user_data) {
   auto c = static_cast<Client *>(user_data);
   if (c->quic_stream_reset(stream_id, app_error_code) != 0) {
-    return -1;
+    return NGTCP2_ERR_CALLBACK_FAILURE;
   }
   return 0;
 }
@@ -158,7 +160,7 @@ int stream_stop_sending(ngtcp2_conn *conn, int64_t stream_id,
                         void *stream_user_data) {
   auto c = static_cast<Client *>(user_data);
   if (c->quic_stream_stop_sending(stream_id, app_error_code) != 0) {
-    return -1;
+    return NGTCP2_ERR_CALLBACK_FAILURE;
   }
   return 0;
 }
@@ -197,13 +199,15 @@ int Client::quic_extend_max_local_streams() {
 namespace {
 int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token,
                           size_t cidlen, void *user_data) {
-  auto dis = std::uniform_int_distribution<uint8_t>(
+  if (RAND_bytes(cid->data, cidlen) != 1) {
-      0, std::numeric_limits<uint8_t>::max());
+    return NGTCP2_ERR_CALLBACK_FAILURE;
-  auto f = [&dis]() { return dis(randgen); };
+  }
 
-  std::generate_n(cid->data, cidlen, f);
   cid->datalen = cidlen;
-  std::generate_n(token, NGTCP2_STATELESS_RESET_TOKENLEN, f);
+
+  if (RAND_bytes(token, NGTCP2_STATELESS_RESET_TOKENLEN) != 1) {
+    return NGTCP2_ERR_CALLBACK_FAILURE;
+  }
 
   return 0;
 }
@@ -222,11 +226,14 @@ void debug_log_printf(void *user_data, const char *fmt, ...) {
 } // namespace
 
 namespace {
-void generate_cid(ngtcp2_cid &dest) {
+int generate_cid(ngtcp2_cid &dest) {
-  auto dis = std::uniform_int_distribution<uint8_t>(
-      0, std::numeric_limits<uint8_t>::max());
   dest.datalen = 8;
-  std::generate_n(dest.data, dest.datalen, [&dis]() { return dis(randgen); });
+
+  if (RAND_bytes(dest.data, dest.datalen) != 1) {
+    return -1;
+  }
+
+  return 0;
 }
 } // namespace
 
@@ -236,15 +243,19 @@ ngtcp2_tstamp timestamp(struct ev_loop *loop) {
 }
 } // namespace
 
+#ifdef HAVE_LIBNGTCP2_CRYPTO_OPENSSL
 namespace {
 int set_encryption_secrets(SSL *ssl, OSSL_ENCRYPTION_LEVEL ossl_level,
                            const uint8_t *rx_secret, const uint8_t *tx_secret,
                            size_t secret_len) {
   auto c = static_cast<Client *>(SSL_get_app_data(ssl));
+  auto level = ngtcp2_crypto_openssl_from_ossl_encryption_level(ossl_level);
 
-  if (c->quic_on_key(
+  if (c->quic_on_rx_secret(level, rx_secret, secret_len) != 0) {
-          ngtcp2_crypto_openssl_from_ossl_encryption_level(ossl_level),
+    return 0;
-          rx_secret, tx_secret, secret_len) != 0) {
+  }
+
+  if (c->quic_on_tx_secret(level, tx_secret, secret_len) != 0) {
     return 0;
   }
 
@@ -282,6 +293,70 @@ auto quic_method = SSL_QUIC_METHOD{
     send_alert,
 };
 } // namespace
+#endif // HAVE_LIBNGTCP2_CRYPTO_OPENSSL
+
+#ifdef HAVE_LIBNGTCP2_CRYPTO_BORINGSSL
+namespace {
+int set_read_secret(SSL *ssl, ssl_encryption_level_t ssl_level,
+                    const SSL_CIPHER *cipher, const uint8_t *secret,
+                    size_t secretlen) {
+  auto c = static_cast<Client *>(SSL_get_app_data(ssl));
+
+  if (c->quic_on_rx_secret(
+          ngtcp2_crypto_boringssl_from_ssl_encryption_level(ssl_level), secret,
+          secretlen) != 0) {
+    return 0;
+  }
+
+  return 1;
+}
+} // namespace
+
+namespace {
+int set_write_secret(SSL *ssl, ssl_encryption_level_t ssl_level,
+                     const SSL_CIPHER *cipher, const uint8_t *secret,
+                     size_t secretlen) {
+  auto c = static_cast<Client *>(SSL_get_app_data(ssl));
+
+  if (c->quic_on_tx_secret(
+          ngtcp2_crypto_boringssl_from_ssl_encryption_level(ssl_level), secret,
+          secretlen) != 0) {
+    return 0;
+  }
+
+  return 1;
+}
+} // namespace
+
+namespace {
+int add_handshake_data(SSL *ssl, ssl_encryption_level_t ssl_level,
+                       const uint8_t *data, size_t len) {
+  auto c = static_cast<Client *>(SSL_get_app_data(ssl));
+  c->quic_write_client_handshake(
+      ngtcp2_crypto_boringssl_from_ssl_encryption_level(ssl_level), data, len);
+  return 1;
+}
+} // namespace
+
+namespace {
+int flush_flight(SSL *ssl) { return 1; }
+} // namespace
+
+namespace {
+int send_alert(SSL *ssl, ssl_encryption_level_t level, uint8_t alert) {
+  auto c = static_cast<Client *>(SSL_get_app_data(ssl));
+  c->quic_set_tls_alert(alert);
+  return 1;
+}
+} // namespace
+
+namespace {
+auto quic_method = SSL_QUIC_METHOD{
+    set_read_secret, set_write_secret, add_handshake_data,
+    flush_flight,    send_alert,
+};
+} // namespace
+#endif // HAVE_LIBNGTCP2_CRYPTO_BORINGSSL
 
 // qlog write callback -- excerpted from ngtcp2/examples/client_base.cc
 namespace {
@@ -297,6 +372,13 @@ void Client::quic_write_qlog(const void *data, size_t datalen) {
   fwrite(data, 1, datalen, quic.qlog_file);
 }
 
+namespace {
+void rand(uint8_t *dest, size_t destlen, const ngtcp2_rand_ctx *rand_ctx) {
+  util::random_bytes(dest, dest + destlen,
+                     *static_cast<std::mt19937 *>(rand_ctx->native_handle));
+}
+} // namespace
+
 int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen,
                       const sockaddr *remote_addr, socklen_t remote_addrlen) {
   int rv;
@@ -327,7 +409,7 @@ int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen,
       ngtcp2_crypto_recv_retry_cb,
       h2load::extend_max_local_streams_bidi,
       nullptr, // extend_max_local_streams_uni
-      nullptr, // rand
+      h2load::rand,
       get_new_connection_id,
       nullptr, // remove_connection_id
       ngtcp2_crypto_update_key_cb,
@@ -345,13 +427,17 @@ int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen,
       nullptr, // recv_datagram
       nullptr, // ack_datagram
       nullptr, // lost_datagram
-      nullptr, // get_path_challenge_data
+      ngtcp2_crypto_get_path_challenge_data_cb,
       h2load::stream_stop_sending,
   };
 
   ngtcp2_cid scid, dcid;
-  generate_cid(scid);
+  if (generate_cid(scid) != 0) {
-  generate_cid(dcid);
+    return -1;
+  }
+  if (generate_cid(dcid) != 0) {
+    return -1;
+  }
 
   auto config = worker->config;
 
@@ -361,6 +447,7 @@ int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen,
     settings.log_printf = debug_log_printf;
   }
   settings.initial_ts = timestamp(worker->loop);
+  settings.rand_ctx.native_handle = &worker->randgen;
   if (!config->qlog_file_base.empty()) {
     assert(quic.qlog_file == nullptr);
     auto path = config->qlog_file_base;
@@ -368,7 +455,7 @@ int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen,
     path += util::utos(worker->id);
     path += '.';
     path += util::utos(id);
-    path += ".qlog";
+    path += ".sqlog";
     quic.qlog_file = fopen(path.c_str(), "w");
     if (quic.qlog_file == nullptr) {
       std::cerr << "Failed to open a qlog file: " << path << std::endl;
@@ -393,8 +480,14 @@ int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen,
   params.max_idle_timeout = 30 * NGTCP2_SECONDS;
 
   auto path = ngtcp2_path{
-      {local_addrlen, const_cast<sockaddr *>(local_addr)},
+      {
-      {remote_addrlen, const_cast<sockaddr *>(remote_addr)},
+          const_cast<sockaddr *>(local_addr),
+          local_addrlen,
+      },
+      {
+          const_cast<sockaddr *>(remote_addr),
+          remote_addrlen,
+      },
   };
 
   assert(config->npn_list.size());
@@ -442,15 +535,16 @@ void Client::quic_close_connection() {
   case quic::ErrorType::Transport:
     nwrite = ngtcp2_conn_write_connection_close(
         quic.conn, &ps.path, nullptr, buf.data(), buf.size(),
-        quic.last_error.code, timestamp(worker->loop));
+        quic.last_error.code, nullptr, 0, timestamp(worker->loop));
     break;
   case quic::ErrorType::Application:
     nwrite = ngtcp2_conn_write_application_close(
         quic.conn, &ps.path, nullptr, buf.data(), buf.size(),
-        quic.last_error.code, timestamp(worker->loop));
+        quic.last_error.code, nullptr, 0, timestamp(worker->loop));
     break;
   default:
     assert(0);
+    abort();
   }
 
   if (nwrite < 0) {
@@ -461,24 +555,16 @@ void Client::quic_close_connection() {
             ps.path.remote.addrlen, buf.data(), nwrite, 0);
 }
 
-int Client::quic_on_key(ngtcp2_crypto_level level, const uint8_t *rx_secret,
+int Client::quic_on_rx_secret(ngtcp2_crypto_level level, const uint8_t *secret,
-                        const uint8_t *tx_secret, size_t secretlen) {
+                              size_t secretlen) {
   if (ngtcp2_crypto_derive_and_install_rx_key(quic.conn, nullptr, nullptr,
-                                              nullptr, level, rx_secret,
+                                              nullptr, level, secret,
                                               secretlen) != 0) {
     std::cerr << "ngtcp2_crypto_derive_and_install_rx_key() failed"
               << std::endl;
     return -1;
   }
 
-  if (ngtcp2_crypto_derive_and_install_tx_key(quic.conn, nullptr, nullptr,
-                                              nullptr, level, tx_secret,
-                                              secretlen) != 0) {
-    std::cerr << "ngtcp2_crypto_derive_and_install_tx_key() failed"
-              << std::endl;
-    return -1;
-  }
-
   if (level == NGTCP2_CRYPTO_LEVEL_APPLICATION) {
     auto s = std::make_unique<Http3Session>(this);
     if (s->init_conn() == -1) {
@@ -490,6 +576,19 @@ int Client::quic_on_key(ngtcp2_crypto_level level, const uint8_t *rx_secret,
   return 0;
 }
 
+int Client::quic_on_tx_secret(ngtcp2_crypto_level level, const uint8_t *secret,
+                              size_t secretlen) {
+  if (ngtcp2_crypto_derive_and_install_tx_key(quic.conn, nullptr, nullptr,
+                                              nullptr, level, secret,
+                                              secretlen) != 0) {
+    std::cerr << "ngtcp2_crypto_derive_and_install_tx_key() failed"
+              << std::endl;
+    return -1;
+  }
+
+  return 0;
+}
+
 void Client::quic_set_tls_alert(uint8_t alert) {
   quic.last_error = quic::err_transport_tls(alert);
 }
@@ -554,14 +653,25 @@ int Client::read_quic() {
     ++worker->stats.udp_dgram_recv;
 
     auto path = ngtcp2_path{
-        {local_addr.len, &local_addr.su.sa},
+        {
-        {addrlen, &su.sa},
+            &local_addr.su.sa,
+            local_addr.len,
+        },
+        {
+            &su.sa,
+            addrlen,
+        },
     };
 
     rv = ngtcp2_conn_read_pkt(quic.conn, &path, &pi, buf.data(), nread,
                               timestamp(worker->loop));
     if (rv != 0) {
       std::cerr << "ngtcp2_conn_read_pkt: " << ngtcp2_strerror(rv) << std::endl;
+
+      if (!quic.last_error.code) {
+        quic.last_error = quic::err_transport(rv);
+      }
+
       return -1;
     }
 

src/http2_test.cc

@@ -916,7 +916,7 @@ void test_http2_rewrite_clean_path(void) {
   CU_ASSERT("/delta%3A" == http2::rewrite_clean_path(
                                balloc, StringRef::from_lit("/delta%3a")));
 
-  // path component is normalized before mathcing
+  // path component is normalized before matching
   CU_ASSERT(
       "/alpha/bravo/" ==
       http2::rewrite_clean_path(

src/nghttp.cc

@@ -2268,7 +2268,7 @@ int communicate(
   auto loop = EV_DEFAULT;
   SSL_CTX *ssl_ctx = nullptr;
   if (scheme == "https") {
-    ssl_ctx = SSL_CTX_new(SSLv23_client_method());
+    ssl_ctx = SSL_CTX_new(TLS_client_method());
     if (!ssl_ctx) {
       std::cerr << "[ERROR] Failed to create SSL_CTX: "
                 << ERR_error_string(ERR_get_error(), nullptr) << std::endl;

src/shrpx.cc

@@ -76,6 +76,11 @@
 
 #include <nghttp2/nghttp2.h>
 
+#ifdef ENABLE_HTTP3
+#  include <ngtcp2/ngtcp2.h>
+#  include <nghttp3/nghttp3.h>
+#endif // ENABLE_HTTP3
+
 #include "shrpx_config.h"
 #include "shrpx_tls.h"
 #include "shrpx_log_config.h"
@@ -202,7 +207,8 @@ struct WorkerProcess {
                 )
       : loop(loop),
         worker_pid(worker_pid),
-        ipc_fd(ipc_fd)
+        ipc_fd(ipc_fd),
+        termination_deadline(0.)
 #ifdef ENABLE_HTTP3
         ,
         quic_ipc_fd(quic_ipc_fd),
@@ -264,6 +270,7 @@ struct WorkerProcess {
   struct ev_loop *loop;
   pid_t worker_pid;
   int ipc_fd;
+  ev_tstamp termination_deadline;
 #ifdef ENABLE_HTTP3
   int quic_ipc_fd;
   std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> cid_prefixes;
@@ -278,6 +285,74 @@ namespace {
 std::deque<std::unique_ptr<WorkerProcess>> worker_processes;
 } // namespace
 
+namespace {
+ev_timer worker_process_grace_period_timer;
+} // namespace
+
+namespace {
+void worker_process_grace_period_timercb(struct ev_loop *loop, ev_timer *w,
+                                         int revents) {
+  auto now = ev_now(loop);
+  ev_tstamp next_repeat = 0.;
+
+  for (auto it = std::begin(worker_processes);
+       it != std::end(worker_processes);) {
+    auto &wp = *it;
+    if (!(wp->termination_deadline > 0.)) {
+      ++it;
+
+      continue;
+    }
+
+    auto d = wp->termination_deadline - now;
+    if (d > 0) {
+      if (!(next_repeat > 0.) || d < next_repeat) {
+        next_repeat = d;
+      }
+
+      ++it;
+
+      continue;
+    }
+
+    LOG(NOTICE) << "Deleting worker process pid=" << wp->worker_pid
+                << " because its grace shutdown period is over";
+
+    it = worker_processes.erase(it);
+  }
+
+  if (next_repeat > 0.) {
+    w->repeat = next_repeat;
+    ev_timer_again(loop, w);
+
+    return;
+  }
+
+  ev_timer_stop(loop, w);
+}
+} // namespace
+
+namespace {
+void worker_process_set_termination_deadline(WorkerProcess *wp,
+                                             struct ev_loop *loop) {
+  auto config = get_config();
+
+  if (!(config->worker_process_grace_shutdown_period > 0.)) {
+    return;
+  }
+
+  wp->termination_deadline =
+      ev_now(loop) + config->worker_process_grace_shutdown_period;
+
+  if (!ev_is_active(&worker_process_grace_period_timer)) {
+    worker_process_grace_period_timer.repeat =
+        config->worker_process_grace_shutdown_period;
+
+    ev_timer_again(loop, &worker_process_grace_period_timer);
+  }
+}
+} // namespace
+
 namespace {
 void worker_process_add(std::unique_ptr<WorkerProcess> wp) {
   worker_processes.push_back(std::move(wp));
@@ -285,7 +360,7 @@ void worker_process_add(std::unique_ptr<WorkerProcess> wp) {
 } // namespace
 
 namespace {
-void worker_process_remove(const WorkerProcess *wp) {
+void worker_process_remove(const WorkerProcess *wp, struct ev_loop *loop) {
   for (auto it = std::begin(worker_processes); it != std::end(worker_processes);
        ++it) {
     auto &s = *it;
@@ -295,28 +370,46 @@ void worker_process_remove(const WorkerProcess *wp) {
     }
 
     worker_processes.erase(it);
+
+    if (worker_processes.empty()) {
+      ev_timer_stop(loop, &worker_process_grace_period_timer);
+    }
+
     break;
   }
 }
 } // namespace
 
 namespace {
-void worker_process_remove_all() {
+void worker_process_adjust_limit() {
+  auto config = get_config();
+
+  if (config->max_worker_processes &&
+      worker_processes.size() > config->max_worker_processes) {
+    worker_processes.pop_front();
+  }
+}
+} // namespace
+
+namespace {
+void worker_process_remove_all(struct ev_loop *loop) {
   std::deque<std::unique_ptr<WorkerProcess>>().swap(worker_processes);
+
+  ev_timer_stop(loop, &worker_process_grace_period_timer);
 }
 } // namespace
 
 namespace {
 // Send signal |signum| to all worker processes, and clears
 // worker_processes.
-void worker_process_kill(int signum) {
+void worker_process_kill(int signum, struct ev_loop *loop) {
   for (auto &s : worker_processes) {
     if (s->worker_pid == -1) {
       continue;
     }
     kill(s->worker_pid, signum);
   }
-  worker_process_remove_all();
+  worker_process_remove_all(loop);
 }
 } // namespace
 
@@ -635,13 +728,14 @@ void signal_cb(struct ev_loop *loop, ev_signal *w, int revents) {
       close(addr.fd);
     }
     ipc_send(wp, SHRPX_IPC_GRACEFUL_SHUTDOWN);
+    worker_process_set_termination_deadline(wp, loop);
     return;
   }
   case RELOAD_SIGNAL:
     reload_config(wp);
     return;
   default:
-    worker_process_kill(w->signum);
+    worker_process_kill(w->signum, loop);
     ev_break(loop);
     return;
   }
@@ -656,7 +750,7 @@ void worker_process_child_cb(struct ev_loop *loop, ev_child *w, int revents) {
 
   auto pid = wp->worker_pid;
 
-  worker_process_remove(wp);
+  worker_process_remove(wp, loop);
 
   if (worker_process_last_pid() == pid) {
     ev_break(loop);
@@ -1342,6 +1436,7 @@ int generate_cid_prefix(
     std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>> &cid_prefixes,
     const Config *config) {
   auto &apiconf = config->api;
+  auto &quicconf = config->quic;
 
   size_t num_cid_prefix;
   if (config->single_thread) {
@@ -1360,7 +1455,7 @@ int generate_cid_prefix(
   cid_prefixes.resize(num_cid_prefix);
 
   for (auto &cid_prefix : cid_prefixes) {
-    if (create_cid_prefix(cid_prefix.data()) != 0) {
+    if (create_cid_prefix(cid_prefix.data(), quicconf.server_id.data()) != 0) {
       return -1;
     }
   }
@@ -1469,7 +1564,7 @@ pid_t fork_worker_process(
 
     // Remove all WorkerProcesses to stop any registered watcher on
     // default loop.
-    worker_process_remove_all();
+    worker_process_remove_all(EV_DEFAULT);
 
     close_unused_inherited_addr(iaddrs);
 
@@ -1644,6 +1739,9 @@ int event_loop() {
     return -1;
   }
 
+  ev_timer_init(&worker_process_grace_period_timer,
+                worker_process_grace_period_timercb, 0., 0.);
+
   worker_process_add(std::make_unique<WorkerProcess>(loop, pid, ipc_fd
 #ifdef ENABLE_HTTP3
                                                      ,
@@ -1670,6 +1768,8 @@ int event_loop() {
 
   ev_run(loop, 0);
 
+  ev_timer_stop(loop, &worker_process_grace_period_timer);
+
   return 0;
 }
 } // namespace
@@ -1854,6 +1954,16 @@ void fill_default_config(Config *config) {
 
     auto &bpfconf = quicconf.bpf;
     bpfconf.prog_file = StringRef::from_lit(PKGLIBDIR "/reuseport_kern.o");
+
+    upstreamconf.congestion_controller = NGTCP2_CC_ALGO_CUBIC;
+
+    upstreamconf.initial_rtt =
+        static_cast<ev_tstamp>(NGTCP2_DEFAULT_INITIAL_RTT) / NGTCP2_SECONDS;
+  }
+
+  if (RAND_bytes(quicconf.server_id.data(), quicconf.server_id.size()) != 1) {
+    assert(0);
+    abort();
   }
 
   auto &http3conf = config->http3;
@@ -1948,14 +2058,18 @@ void fill_default_config(Config *config) {
 
 namespace {
 void print_version(std::ostream &out) {
-  out << "nghttpx nghttp2/" NGHTTP2_VERSION << std::endl;
+  out << "nghttpx nghttp2/" NGHTTP2_VERSION
+#ifdef ENABLE_HTTP3
+         " ngtcp2/" NGTCP2_VERSION " nghttp3/" NGHTTP3_VERSION
+#endif // ENABLE_HTTP3
+      << std::endl;
 }
 } // namespace
 
 namespace {
 void print_usage(std::ostream &out) {
   out << R"(Usage: nghttpx [OPTIONS]... [<PRIVATE_KEY> <CERT>]
-A reverse proxy for HTTP/2, and HTTP/1.)"
+A reverse proxy for HTTP/3, HTTP/2, and HTTP/1.)"
       << std::endl;
 }
 } // namespace
@@ -2366,6 +2480,12 @@ Performance:
               If 0 is given, nghttpx does not set the limit.
               Default: )"
       << config->rlimit_nofile << R"(
+  --rlimit-memlock=<N>
+              Set maximum number of bytes of memory that may be locked
+              into  RAM.  If  0 is  given,  nghttpx does  not set  the
+              limit.
+              Default: )"
+      << config->rlimit_memlock << R"(
   --backend-request-buffer=<SIZE>
               Set buffer size used to store backend request.
               Default: )"
@@ -3054,7 +3174,7 @@ HTTP:
               advertised  in alt-svc  header  field  only in  HTTP/1.1
               frontend.   This option  can be  used multiple  times to
               specify multiple alternative services.
-              Example: --altsvc="h2,443,,,ma=3600; persist=1'
+              Example: --altsvc="h2,443,,,ma=3600; persist=1"
   --http2-altsvc=<PROTOID,PORT[,HOST,[ORIGIN[,PARAMS]]]>
               Just like --altsvc option, but  this altsvc is only sent
               in HTTP/2 frontend.
@@ -3185,6 +3305,27 @@ Process:
               process.   nghttpx still  spawns  additional process  if
               neverbleed  is used.   In the  single process  mode, the
               signal handling feature is disabled.
+  --max-worker-processes=<N>
+              The maximum number of  worker processes.  nghttpx spawns
+              new worker  process when  it reloads  its configuration.
+              The previous worker  process enters graceful termination
+              period and will terminate  when it finishes handling the
+              existing    connections.     However,    if    reloading
+              configurations  happen   very  frequently,   the  worker
+              processes might be piled up if they take a bit long time
+              to finish  the existing connections.  With  this option,
+              if  the number  of  worker processes  exceeds the  given
+              value,   the  oldest   worker   process  is   terminated
+              immediately.  Specifying 0 means no  limit and it is the
+              default behaviour.
+  --worker-process-grace-shutdown-period=<DURATION>
+              Maximum  period  for  a   worker  process  to  terminate
+              gracefully.  When  a worker  process enters  in graceful
+              shutdown   period  (e.g.,   when  nghttpx   reloads  its
+              configuration)  and  it  does not  finish  handling  the
+              existing connections in the given  period of time, it is
+              immediately terminated.  Specifying 0 means no limit and
+              it is the default behaviour.
 
 Scripting:
   --mruby-file=<PATH>
@@ -3221,7 +3362,57 @@ HTTP/3 and QUIC:
               frontend QUIC  connections.  A qlog file  is created per
               each QUIC  connection.  The  file name is  ISO8601 basic
               format, followed by "-", server Source Connection ID and
-              ".qlog".
+              ".sqlog".
+  --frontend-quic-require-token
+              Require an address validation  token for a frontend QUIC
+              connection.   Server sends  a token  in Retry  packet or
+              NEW_TOKEN frame in the previous connection.
+  --frontend-quic-congestion-controller=<CC>
+              Specify a congestion controller algorithm for a frontend
+              QUIC  connection.   <CC>  should be  either  "cubic"  or
+              "bbr".
+              Default: )"
+      << (config->quic.upstream.congestion_controller == NGTCP2_CC_ALGO_CUBIC
+              ? "cubic"
+              : "bbr")
+      << R"(
+  --frontend-quic-secret-file=<PATH>
+              Path to file that contains secure random data to be used
+              as QUIC keying materials.  It is used to derive keys for
+              encrypting tokens and Connection IDs.  It is not used to
+              encrypt  QUIC  packets.  Each  line  of  this file  must
+              contain  exactly  136  bytes  hex-encoded  string  (when
+              decoded the byte string is  68 bytes long).  The first 2
+              bits of  decoded byte  string are  used to  identify the
+              keying material.  An  empty line or a  line which starts
+              '#'  is ignored.   The file  can contain  more than  one
+              keying materials.  Because the  identifier is 2 bits, at
+              most 4 keying materials are  read and the remaining data
+              is discarded.  The first keying  material in the file is
+              primarily  used for  encryption and  decryption for  new
+              connection.  The other ones are used to decrypt data for
+              the  existing connections.   Specifying multiple  keying
+              materials enables  key rotation.   Please note  that key
+              rotation  does  not  occur automatically.   User  should
+              update  files  or  change  options  values  and  restart
+              nghttpx gracefully.   If opening  or reading  given file
+              fails, all loaded keying  materials are discarded and it
+              is treated as if none of  this option is given.  If this
+              option is not  given or an error  occurred while opening
+              or  reading  a  file,  a keying  material  is  generated
+              internally on startup and reload.
+  --quic-server-id=<HEXSTRING>
+              Specify server  ID encoded in Connection  ID to identify
+              this  particular  server  instance.   Connection  ID  is
+              encrypted and  this part is  not visible in  public.  It
+              must be 4  bytes long and must be encoded  in hex string
+              (which is 8  bytes long).  If this option  is omitted, a
+              random   server  ID   is   generated   on  startup   and
+              configuration reload.
+  --frontend-quic-initial-rtt=<DURATION>
+              Specify the initial RTT of the frontend QUIC connection.
+              Default: )"
+      << util::duration_str(config->quic.upstream.initial_rtt) << R"(
   --no-quic-bpf
               Disable eBPF.
   --frontend-http3-window-size=<SIZE>
@@ -3475,9 +3666,12 @@ int process_options(Config *config,
     return -1;
   }
 
+  std::array<char, util::max_hostport> hostport_buf;
+
   auto &proxy = config->downstream_http_proxy;
   if (!proxy.host.empty()) {
-    auto hostport = util::make_hostport(StringRef{proxy.host}, proxy.port);
+    auto hostport = util::make_hostport(std::begin(hostport_buf),
+                                        StringRef{proxy.host}, proxy.port);
     if (resolve_hostname(&proxy.addr, proxy.host.c_str(), proxy.port,
                          AF_UNSPEC) == -1) {
       LOG(FATAL) << "Resolving backend HTTP proxy address failed: " << hostport;
@@ -3490,7 +3684,8 @@ int process_options(Config *config,
   {
     auto &memcachedconf = tlsconf.session_cache.memcached;
     if (!memcachedconf.host.empty()) {
-      auto hostport = util::make_hostport(StringRef{memcachedconf.host},
+      auto hostport = util::make_hostport(std::begin(hostport_buf),
+                                          StringRef{memcachedconf.host},
                                           memcachedconf.port);
       if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.c_str(),
                            memcachedconf.port, memcachedconf.family) == -1) {
@@ -3511,7 +3706,8 @@ int process_options(Config *config,
   {
     auto &memcachedconf = tlsconf.ticket.memcached;
     if (!memcachedconf.host.empty()) {
-      auto hostport = util::make_hostport(StringRef{memcachedconf.host},
+      auto hostport = util::make_hostport(std::begin(hostport_buf),
+                                          StringRef{memcachedconf.host},
                                           memcachedconf.port);
       if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.c_str(),
                            memcachedconf.port, memcachedconf.family) == -1) {
@@ -3538,6 +3734,18 @@ int process_options(Config *config,
     }
   }
 
+#ifdef RLIMIT_MEMLOCK
+  if (config->rlimit_memlock) {
+    struct rlimit lim = {static_cast<rlim_t>(config->rlimit_memlock),
+                         static_cast<rlim_t>(config->rlimit_memlock)};
+    if (setrlimit(RLIMIT_MEMLOCK, &lim) != 0) {
+      auto error = errno;
+      LOG(WARN) << "Setting rlimit-memlock failed: "
+                << xsi_strerror(error, errbuf.data(), errbuf.size());
+    }
+  }
+#endif // RLIMIT_MEMLOCK
+
   auto &fwdconf = config->http.forwarded;
 
   if (fwdconf.by_node_type == ForwardedNode::OBFUSCATED &&
@@ -3682,6 +3890,7 @@ void reload_config(WorkerProcess *wp) {
   // Send last worker process a graceful shutdown notice
   auto &last_wp = worker_processes.back();
   ipc_send(last_wp.get(), SHRPX_IPC_GRACEFUL_SHUTDOWN);
+  worker_process_set_termination_deadline(last_wp.get(), loop);
   // We no longer use signals for this worker.
   last_wp->shutdown_signal_watchers();
 
@@ -3692,6 +3901,8 @@ void reload_config(WorkerProcess *wp) {
 #endif // ENABLE_HTTP3
                                                      ));
 
+  worker_process_adjust_limit();
+
   if (!get_config()->pid_file.empty()) {
     save_pid();
   }
@@ -4011,6 +4222,19 @@ int main(int argc, char **argv) {
         {SHRPX_OPT_FRONTEND_QUIC_EARLY_DATA.c_str(), no_argument, &flag, 180},
         {SHRPX_OPT_FRONTEND_QUIC_QLOG_DIR.c_str(), required_argument, &flag,
          181},
+        {SHRPX_OPT_FRONTEND_QUIC_REQUIRE_TOKEN.c_str(), no_argument, &flag,
+         182},
+        {SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER.c_str(),
+         required_argument, &flag, 183},
+        {SHRPX_OPT_QUIC_SERVER_ID.c_str(), required_argument, &flag, 185},
+        {SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE.c_str(), required_argument, &flag,
+         186},
+        {SHRPX_OPT_RLIMIT_MEMLOCK.c_str(), required_argument, &flag, 187},
+        {SHRPX_OPT_MAX_WORKER_PROCESSES.c_str(), required_argument, &flag, 188},
+        {SHRPX_OPT_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD.c_str(),
+         required_argument, &flag, 189},
+        {SHRPX_OPT_FRONTEND_QUIC_INITIAL_RTT.c_str(), required_argument, &flag,
+         190},
         {nullptr, 0, nullptr, 0}};
 
     int option_index = 0;
@@ -4878,6 +5102,43 @@ int main(int argc, char **argv) {
         cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_QLOG_DIR,
                              StringRef{optarg});
         break;
+      case 182:
+        // --frontend-quic-require-token
+        cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_REQUIRE_TOKEN,
+                             StringRef::from_lit("yes"));
+        break;
+      case 183:
+        // --frontend-quic-congestion-controller
+        cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER,
+                             StringRef{optarg});
+        break;
+      case 185:
+        // --quic-server-id
+        cmdcfgs.emplace_back(SHRPX_OPT_QUIC_SERVER_ID, StringRef{optarg});
+        break;
+      case 186:
+        // --frontend-quic-secret-file
+        cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE,
+                             StringRef{optarg});
+        break;
+      case 187:
+        // --rlimit-memlock
+        cmdcfgs.emplace_back(SHRPX_OPT_RLIMIT_MEMLOCK, StringRef{optarg});
+        break;
+      case 188:
+        // --max-worker-processes
+        cmdcfgs.emplace_back(SHRPX_OPT_MAX_WORKER_PROCESSES, StringRef{optarg});
+        break;
+      case 189:
+        // --worker-process-grace-shutdown-period
+        cmdcfgs.emplace_back(SHRPX_OPT_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD,
+                             StringRef{optarg});
+        break;
+      case 190:
+        // --frontend-quic-initial-rtt
+        cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_INITIAL_RTT,
+                             StringRef{optarg});
+        break;
       default:
         break;
       }

src/shrpx_api_downstream_connection.cc

@@ -463,7 +463,7 @@ int APIDownstreamConnection::on_read() { return 0; }
 
 int APIDownstreamConnection::on_write() { return 0; }
 
-void APIDownstreamConnection::on_upstream_change(Upstream *uptream) {}
+void APIDownstreamConnection::on_upstream_change(Upstream *upstream) {}
 
 bool APIDownstreamConnection::poolable() const { return false; }
 

src/shrpx_api_downstream_connection.h

@@ -81,7 +81,7 @@ public:
   virtual int on_read();
   virtual int on_write();
 
-  virtual void on_upstream_change(Upstream *uptream);
+  virtual void on_upstream_change(Upstream *upstream);
 
   // true if this object is poolable.
   virtual bool poolable() const;

src/shrpx_client_handler.cc

@@ -292,11 +292,12 @@ int ClientHandler::write_tls() {
 #ifdef ENABLE_HTTP3
 int ClientHandler::read_quic(const UpstreamAddr *faddr,
                              const Address &remote_addr,
-                             const Address &local_addr, const uint8_t *data,
+                             const Address &local_addr,
+                             const ngtcp2_pkt_info &pi, const uint8_t *data,
                              size_t datalen) {
   auto upstream = static_cast<Http3Upstream *>(upstream_.get());
 
-  return upstream->on_read(faddr, remote_addr, local_addr, data, datalen);
+  return upstream->on_read(faddr, remote_addr, local_addr, pi, data, datalen);
 }
 
 int ClientHandler::write_quic() { return upstream_->on_write(); }
@@ -517,7 +518,6 @@ void ClientHandler::setup_upstream_io_callback() {
 void ClientHandler::setup_http3_upstream(
     std::unique_ptr<Http3Upstream> &&upstream) {
   upstream_ = std::move(upstream);
-  alpn_ = StringRef::from_lit("h3");
   write_ = &ClientHandler::write_quic;
 
   auto config = get_config();
@@ -884,7 +884,6 @@ DownstreamAddr *ClientHandler::get_downstream_addr(int &err,
         err = -1;
         return nullptr;
       }
-      aff_idx = i;
     }
 
     return addr;
@@ -1599,4 +1598,13 @@ StringRef ClientHandler::get_alpn() const { return alpn_; }
 
 BlockAllocator &ClientHandler::get_block_allocator() { return balloc_; }
 
+void ClientHandler::set_alpn_from_conn() {
+  const unsigned char *alpn;
+  unsigned int alpnlen;
+
+  SSL_get0_alpn_selected(conn_.tls.ssl, &alpn, &alpnlen);
+
+  alpn_ = make_string_ref(balloc_, StringRef{alpn, alpnlen});
+}
+
 } // namespace shrpx

src/shrpx_client_handler.h

@@ -149,7 +149,8 @@ public:
 #ifdef ENABLE_HTTP3
   void setup_http3_upstream(std::unique_ptr<Http3Upstream> &&upstream);
   int read_quic(const UpstreamAddr *faddr, const Address &remote_addr,
-                const Address &local_addr, const uint8_t *data, size_t datalen);
+                const Address &local_addr, const ngtcp2_pkt_info &pi,
+                const uint8_t *data, size_t datalen);
   int write_quic();
 #endif // ENABLE_HTTP3
 
@@ -187,6 +188,8 @@ public:
 
   BlockAllocator &get_block_allocator();
 
+  void set_alpn_from_conn();
+
 private:
   // Allocator to allocate memory for connection-wide objects.  Make
   // sure that the allocations must be bounded, and not proportional

src/shrpx_config.cc

@@ -230,10 +230,81 @@ read_tls_ticket_key_file(const std::vector<StringRef> &files,
   return ticket_keys;
 }
 
+#ifdef ENABLE_HTTP3
+std::shared_ptr<QUICKeyingMaterials>
+read_quic_secret_file(const StringRef &path) {
+  constexpr size_t expectedlen =
+      SHRPX_QUIC_SECRET_RESERVEDLEN + SHRPX_QUIC_SECRETLEN + SHRPX_QUIC_SALTLEN;
+
+  auto qkms = std::make_shared<QUICKeyingMaterials>();
+  auto &kms = qkms->keying_materials;
+
+  std::ifstream f(path.c_str());
+  if (!f) {
+    LOG(ERROR) << "frontend-quic-secret-file: could not open file " << path;
+    return nullptr;
+  }
+
+  std::array<char, 4096> buf;
+
+  while (f.getline(buf.data(), buf.size())) {
+    auto len = strlen(buf.data());
+    if (len == 0 || buf[0] == '#') {
+      continue;
+    }
+
+    auto s = StringRef{std::begin(buf), std::begin(buf) + len};
+    if (s.size() != expectedlen * 2 || !util::is_hex_string(s)) {
+      LOG(ERROR) << "frontend-quic-secret-file: each line must be a "
+                 << expectedlen * 2 << " bytes hex encoded string";
+      return nullptr;
+    }
+
+    kms.emplace_back();
+    auto &qkm = kms.back();
+
+    auto p = std::begin(s);
+
+    util::decode_hex(std::begin(qkm.reserved),
+                     StringRef{p, p + qkm.reserved.size()});
+    p += qkm.reserved.size() * 2;
+    util::decode_hex(std::begin(qkm.secret),
+                     StringRef{p, p + qkm.secret.size()});
+    p += qkm.secret.size() * 2;
+    util::decode_hex(std::begin(qkm.salt), StringRef{p, p + qkm.salt.size()});
+    p += qkm.salt.size() * 2;
+
+    assert(static_cast<size_t>(p - std::begin(s)) == expectedlen * 2);
+
+    qkm.id = qkm.reserved[0] & 0xc0;
+
+    if (kms.size() == 4) {
+      break;
+    }
+  }
+
+  if (f.bad() || (!f.eof() && f.fail())) {
+    LOG(ERROR)
+        << "frontend-quic-secret-file: error occurred while reading file "
+        << path;
+    return nullptr;
+  }
+
+  if (kms.empty()) {
+    LOG(WARN)
+        << "frontend-quic-secret-file: no keying materials are present in file "
+        << path;
+    return nullptr;
+  }
+
+  return qkms;
+}
+#endif // ENABLE_HTTP3
+
 FILE *open_file_for_write(const char *filename) {
   std::array<char, STRERROR_BUFSIZE> errbuf;
 
-#if defined O_CLOEXEC
+#ifdef O_CLOEXEC
   auto fd = open(filename, O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC,
                  S_IRUSR | S_IWUSR);
 #else
@@ -1983,6 +2054,11 @@ int option_lookup_token(const char *name, size_t namelen) {
     break;
   case 14:
     switch (name[13]) {
+    case 'd':
+      if (util::strieq_l("quic-server-i", name, 13)) {
+        return SHRPX_OPTID_QUIC_SERVER_ID;
+      }
+      break;
     case 'e':
       if (util::strieq_l("accesslog-fil", name, 13)) {
         return SHRPX_OPTID_ACCESSLOG_FILE;
@@ -1993,6 +2069,11 @@ int option_lookup_token(const char *name, size_t namelen) {
         return SHRPX_OPTID_NO_SERVER_PUSH;
       }
       break;
+    case 'k':
+      if (util::strieq_l("rlimit-memloc", name, 13)) {
+        return SHRPX_OPTID_RLIMIT_MEMLOCK;
+      }
+      break;
     case 'p':
       if (util::strieq_l("no-verify-ocs", name, 13)) {
         return SHRPX_OPTID_NO_VERIFY_OCSP;
@@ -2172,6 +2253,9 @@ int option_lookup_token(const char *name, size_t namelen) {
       }
       break;
     case 's':
+      if (util::strieq_l("max-worker-processe", name, 19)) {
+        return SHRPX_OPTID_MAX_WORKER_PROCESSES;
+      }
       if (util::strieq_l("tls13-client-cipher", name, 19)) {
         return SHRPX_OPTID_TLS13_CLIENT_CIPHERS;
       }
@@ -2339,6 +2423,9 @@ int option_lookup_token(const char *name, size_t namelen) {
       if (util::strieq_l("backend-http2-window-siz", name, 24)) {
         return SHRPX_OPTID_BACKEND_HTTP2_WINDOW_SIZE;
       }
+      if (util::strieq_l("frontend-quic-secret-fil", name, 24)) {
+        return SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE;
+      }
       break;
     case 'g':
       if (util::strieq_l("http2-no-cookie-crumblin", name, 24)) {
@@ -2353,6 +2440,11 @@ int option_lookup_token(const char *name, size_t namelen) {
         return SHRPX_OPTID_MAX_REQUEST_HEADER_FIELDS;
       }
       break;
+    case 't':
+      if (util::strieq_l("frontend-quic-initial-rt", name, 24)) {
+        return SHRPX_OPTID_FRONTEND_QUIC_INITIAL_RTT;
+      }
+      break;
     }
     break;
   case 26:
@@ -2401,6 +2493,11 @@ int option_lookup_token(const char *name, size_t namelen) {
         return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED;
       }
       break;
+    case 'n':
+      if (util::strieq_l("frontend-quic-require-toke", name, 26)) {
+        return SHRPX_OPTID_FRONTEND_QUIC_REQUIRE_TOKEN;
+      }
+      break;
     case 'r':
       if (util::strieq_l("request-header-field-buffe", name, 26)) {
         return SHRPX_OPTID_REQUEST_HEADER_FIELD_BUFFER;
@@ -2566,11 +2663,19 @@ int option_lookup_token(const char *name, size_t namelen) {
       if (util::strieq_l("frontend-http2-dump-response-heade", name, 34)) {
         return SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER;
       }
+      if (util::strieq_l("frontend-quic-congestion-controlle", name, 34)) {
+        return SHRPX_OPTID_FRONTEND_QUIC_CONGESTION_CONTROLLER;
+      }
       break;
     }
     break;
   case 36:
     switch (name[35]) {
+    case 'd':
+      if (util::strieq_l("worker-process-grace-shutdown-perio", name, 35)) {
+        return SHRPX_OPTID_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD;
+      }
+      break;
     case 'e':
       if (util::strieq_l("backend-http2-connection-window-siz", name, 35)) {
         return SHRPX_OPTID_BACKEND_HTTP2_CONNECTION_WINDOW_SIZE;
@@ -2765,9 +2870,16 @@ int parse_config(Config *config, int optid, const StringRef &opt,
       return -1;
     }
 
-    if (params.quic && params.alt_mode != UpstreamAltMode::NONE) {
+    if (params.quic) {
-      LOG(ERROR) << "frontend: api or healthmon cannot be used with quic";
+      if (params.alt_mode != UpstreamAltMode::NONE) {
-      return -1;
+        LOG(ERROR) << "frontend: api or healthmon cannot be used with quic";
+        return -1;
+      }
+
+      if (!params.tls) {
+        LOG(ERROR) << "frontend: quic requires TLS";
+        return -1;
+      }
     }
 
     UpstreamAddr addr{};
@@ -3842,7 +3954,7 @@ int parse_config(Config *config, int optid, const StringRef &opt,
                     "65535], inclusive";
       return -1;
     }
-    config->http.redirect_https_port = optarg;
+    config->http.redirect_https_port = make_string_ref(config->balloc, optarg);
     return 0;
   }
   case SHRPX_OPTID_FRONTEND_MAX_REQUESTS:
@@ -3983,10 +4095,75 @@ int parse_config(Config *config, int optid, const StringRef &opt,
     return 0;
   case SHRPX_OPTID_FRONTEND_QUIC_QLOG_DIR:
 #ifdef ENABLE_HTTP3
-    config->quic.upstream.qlog.dir = optarg;
+    config->quic.upstream.qlog.dir = make_string_ref(config->balloc, optarg);
 #endif // ENABLE_HTTP3
 
     return 0;
+  case SHRPX_OPTID_FRONTEND_QUIC_REQUIRE_TOKEN:
+#ifdef ENABLE_HTTP3
+    config->quic.upstream.require_token = util::strieq_l("yes", optarg);
+#endif // ENABLE_HTTP3
+
+    return 0;
+  case SHRPX_OPTID_FRONTEND_QUIC_CONGESTION_CONTROLLER:
+#ifdef ENABLE_HTTP3
+    if (util::strieq_l("cubic", optarg)) {
+      config->quic.upstream.congestion_controller = NGTCP2_CC_ALGO_CUBIC;
+    } else if (util::strieq_l("bbr", optarg)) {
+      config->quic.upstream.congestion_controller = NGTCP2_CC_ALGO_BBR;
+    } else {
+      LOG(ERROR) << opt << ": must be either cubic or bbr";
+      return -1;
+    }
+#endif // ENABLE_HTTP3
+
+    return 0;
+  case SHRPX_OPTID_QUIC_SERVER_ID:
+#ifdef ENABLE_HTTP3
+    if (optarg.size() != config->quic.server_id.size() * 2 ||
+        !util::is_hex_string(optarg)) {
+      LOG(ERROR) << opt << ": must be a hex-string";
+      return -1;
+    }
+    util::decode_hex(std::begin(config->quic.server_id), optarg);
+#endif // ENABLE_HTTP3
+
+    return 0;
+  case SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE:
+#ifdef ENABLE_HTTP3
+    config->quic.upstream.secret_file = make_string_ref(config->balloc, optarg);
+#endif // ENABLE_HTTP3
+
+    return 0;
+  case SHRPX_OPTID_RLIMIT_MEMLOCK: {
+    int n;
+
+    if (parse_uint(&n, opt, optarg) != 0) {
+      return -1;
+    }
+
+    if (n < 0) {
+      LOG(ERROR) << opt << ": specify the integer more than or equal to 0";
+
+      return -1;
+    }
+
+    config->rlimit_memlock = n;
+
+    return 0;
+  }
+  case SHRPX_OPTID_MAX_WORKER_PROCESSES:
+    return parse_uint(&config->max_worker_processes, opt, optarg);
+  case SHRPX_OPTID_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD:
+    return parse_duration(&config->worker_process_grace_shutdown_period, opt,
+                          optarg);
+  case SHRPX_OPTID_FRONTEND_QUIC_INITIAL_RTT: {
+#ifdef ENABLE_HTTP3
+    return parse_duration(&config->quic.upstream.initial_rtt, opt, optarg);
+#endif // ENABLE_HTTP3
+
+    return 0;
+  }
   case SHRPX_OPTID_CONF:
     LOG(WARN) << "conf: ignored";
 
@@ -4286,7 +4463,7 @@ int configure_downstream_group(Config *config, bool http2_proxy,
     if (!g.mruby_file.empty()) {
       if (mruby::create_mruby_context(g.mruby_file) == nullptr) {
         LOG(config->ignore_per_pattern_mruby_error ? ERROR : FATAL)
-            << "backend: Could not compile mruby flie for pattern "
+            << "backend: Could not compile mruby file for pattern "
             << g.pattern;
         if (!config->ignore_per_pattern_mruby_error) {
           return -1;
@@ -4321,6 +4498,8 @@ int configure_downstream_group(Config *config, bool http2_proxy,
 
   auto resolve_flags = numeric_addr_only ? AI_NUMERICHOST | AI_NUMERICSERV : 0;
 
+  std::array<char, util::max_hostport> hostport_buf;
+
   for (auto &g : addr_groups) {
     std::unordered_map<StringRef, uint32_t> wgchk;
     for (auto &addr : g.addrs) {
@@ -4366,7 +4545,7 @@ int configure_downstream_group(Config *config, bool http2_proxy,
           util::make_http_hostport(downstreamconf.balloc, addr.host, addr.port);
 
       auto hostport =
-          util::make_hostport(downstreamconf.balloc, addr.host, addr.port);
+          util::make_hostport(std::begin(hostport_buf), addr.host, addr.port);
 
       if (!addr.dns) {
         if (resolve_hostname(&addr.addr, addr.host.c_str(), addr.port,

src/shrpx_config.h

@@ -387,6 +387,20 @@ constexpr auto SHRPX_OPT_FRONTEND_QUIC_EARLY_DATA =
     StringRef::from_lit("frontend-quic-early-data");
 constexpr auto SHRPX_OPT_FRONTEND_QUIC_QLOG_DIR =
     StringRef::from_lit("frontend-quic-qlog-dir");
+constexpr auto SHRPX_OPT_FRONTEND_QUIC_REQUIRE_TOKEN =
+    StringRef::from_lit("frontend-quic-require-token");
+constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER =
+    StringRef::from_lit("frontend-quic-congestion-controller");
+constexpr auto SHRPX_OPT_QUIC_SERVER_ID = StringRef::from_lit("quic-server-id");
+constexpr auto SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE =
+    StringRef::from_lit("frontend-quic-secret-file");
+constexpr auto SHRPX_OPT_RLIMIT_MEMLOCK = StringRef::from_lit("rlimit-memlock");
+constexpr auto SHRPX_OPT_MAX_WORKER_PROCESSES =
+    StringRef::from_lit("max-worker-processes");
+constexpr auto SHRPX_OPT_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD =
+    StringRef::from_lit("worker-process-grace-shutdown-period");
+constexpr auto SHRPX_OPT_FRONTEND_QUIC_INITIAL_RTT =
+    StringRef::from_lit("frontend-quic-initial-rtt");
 
 constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8;
 
@@ -598,10 +612,18 @@ struct TLSCertificate {
 };
 
 #ifdef ENABLE_HTTP3
-struct QUICSecret {
+struct QUICKeyingMaterial {
-  std::array<uint8_t, SHRPX_QUIC_STATELESS_RESET_SECRETLEN>
+  std::array<uint8_t, SHRPX_QUIC_SECRET_RESERVEDLEN> reserved;
-      stateless_reset_secret;
+  std::array<uint8_t, SHRPX_QUIC_SECRETLEN> secret;
-  std::array<uint8_t, SHRPX_QUIC_TOKEN_SECRETLEN> token_secret;
+  std::array<uint8_t, SHRPX_QUIC_SALTLEN> salt;
+  std::array<uint8_t, SHRPX_QUIC_CID_ENCRYPTION_KEYLEN> cid_encryption_key;
+  // Identifier of this keying material.  Only the first 2 bits are
+  // used.
+  uint8_t id;
+};
+
+struct QUICKeyingMaterials {
+  std::vector<QUICKeyingMaterial> keying_materials;
 };
 #endif // ENABLE_HTTP3
 
@@ -669,7 +691,7 @@ struct TLSConfig {
     ev_tstamp idle_timeout;
   } dyn_rec;
 
-  // OCSP realted configurations
+  // OCSP related configurations
   struct {
     ev_tstamp update_interval;
     StringRef fetch_ocsp_response_file;
@@ -754,12 +776,17 @@ struct QUICConfig {
     struct {
       StringRef dir;
     } qlog;
+    ngtcp2_cc_algo congestion_controller;
     bool early_data;
+    bool require_token;
+    StringRef secret_file;
+    ev_tstamp initial_rtt;
   } upstream;
   struct {
     StringRef prog_file;
     bool disabled;
   } bpf;
+  std::array<uint8_t, SHRPX_QUIC_SERVER_IDLEN> server_id;
 };
 
 struct Http3Config {
@@ -1044,6 +1071,7 @@ struct Config {
         num_worker{0},
         padding{0},
         rlimit_nofile{0},
+        rlimit_memlock{0},
         uid{0},
         gid{0},
         pid{0},
@@ -1053,7 +1081,9 @@ struct Config {
         single_process{false},
         single_thread{false},
         ignore_per_pattern_mruby_error{false},
-        ev_loop_flags{0} {
+        ev_loop_flags{0},
+        max_worker_processes{0},
+        worker_process_grace_shutdown_period{0.} {
   }
   ~Config();
 
@@ -1092,6 +1122,7 @@ struct Config {
   size_t num_worker;
   size_t padding;
   size_t rlimit_nofile;
+  size_t rlimit_memlock;
   uid_t uid;
   gid_t gid;
   pid_t pid;
@@ -1106,6 +1137,8 @@ struct Config {
   bool ignore_per_pattern_mruby_error;
   // flags passed to ev_default_loop() and ev_loop_new()
   int ev_loop_flags;
+  size_t max_worker_processes;
+  ev_tstamp worker_process_grace_shutdown_period;
 };
 
 const Config *get_config();
@@ -1207,10 +1240,14 @@ enum {
   SHRPX_OPTID_FRONTEND_KEEP_ALIVE_TIMEOUT,
   SHRPX_OPTID_FRONTEND_MAX_REQUESTS,
   SHRPX_OPTID_FRONTEND_NO_TLS,
+  SHRPX_OPTID_FRONTEND_QUIC_CONGESTION_CONTROLLER,
   SHRPX_OPTID_FRONTEND_QUIC_DEBUG_LOG,
   SHRPX_OPTID_FRONTEND_QUIC_EARLY_DATA,
   SHRPX_OPTID_FRONTEND_QUIC_IDLE_TIMEOUT,
+  SHRPX_OPTID_FRONTEND_QUIC_INITIAL_RTT,
   SHRPX_OPTID_FRONTEND_QUIC_QLOG_DIR,
+  SHRPX_OPTID_FRONTEND_QUIC_REQUIRE_TOKEN,
+  SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE,
   SHRPX_OPTID_FRONTEND_READ_TIMEOUT,
   SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT,
   SHRPX_OPTID_HEADER_FIELD_BUFFER,
@@ -1228,6 +1265,7 @@ enum {
   SHRPX_OPTID_MAX_HEADER_FIELDS,
   SHRPX_OPTID_MAX_REQUEST_HEADER_FIELDS,
   SHRPX_OPTID_MAX_RESPONSE_HEADER_FIELDS,
+  SHRPX_OPTID_MAX_WORKER_PROCESSES,
   SHRPX_OPTID_MRUBY_FILE,
   SHRPX_OPTID_NO_ADD_X_FORWARDED_PROTO,
   SHRPX_OPTID_NO_HOST_REWRITE,
@@ -1252,11 +1290,13 @@ enum {
   SHRPX_OPTID_PRIVATE_KEY_PASSWD_FILE,
   SHRPX_OPTID_PSK_SECRETS,
   SHRPX_OPTID_QUIC_BPF_PROGRAM_FILE,
+  SHRPX_OPTID_QUIC_SERVER_ID,
   SHRPX_OPTID_READ_BURST,
   SHRPX_OPTID_READ_RATE,
   SHRPX_OPTID_REDIRECT_HTTPS_PORT,
   SHRPX_OPTID_REQUEST_HEADER_FIELD_BUFFER,
   SHRPX_OPTID_RESPONSE_HEADER_FIELD_BUFFER,
+  SHRPX_OPTID_RLIMIT_MEMLOCK,
   SHRPX_OPTID_RLIMIT_NOFILE,
   SHRPX_OPTID_SERVER_NAME,
   SHRPX_OPTID_SINGLE_PROCESS,
@@ -1297,6 +1337,7 @@ enum {
   SHRPX_OPTID_VERIFY_CLIENT_CACERT,
   SHRPX_OPTID_VERIFY_CLIENT_TOLERATE_EXPIRED,
   SHRPX_OPTID_WORKER_FRONTEND_CONNECTIONS,
+  SHRPX_OPTID_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD,
   SHRPX_OPTID_WORKER_READ_BURST,
   SHRPX_OPTID_WORKER_READ_RATE,
   SHRPX_OPTID_WORKER_WRITE_BURST,
@@ -1361,6 +1402,11 @@ std::unique_ptr<TicketKeys>
 read_tls_ticket_key_file(const std::vector<StringRef> &files,
                          const EVP_CIPHER *cipher, const EVP_MD *hmac);
 
+#ifdef ENABLE_HTTP3
+std::shared_ptr<QUICKeyingMaterials>
+read_quic_secret_file(const StringRef &path);
+#endif // ENABLE_HTTP3
+
 // Returns string representation of |proto|.
 StringRef strproto(Proto proto);
 

src/shrpx_connection.cc

@@ -397,11 +397,14 @@ int Connection::tls_handshake() {
 
   ERR_clear_error();
 
-#if OPENSSL_1_1_1_API
+#if OPENSSL_1_1_1_API || defined(OPENSSL_IS_BORINGSSL)
+  auto &tlsconf = get_config()->tls;
+#endif // OPENSSL_1_1_1_API || defined(OPENSSL_IS_BORINGSSL)
+
+#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
   if (!tls.server_handshake || tls.early_data_finish) {
     rv = SSL_do_handshake(tls.ssl);
   } else {
-    auto &tlsconf = get_config()->tls;
     for (;;) {
       size_t nread;
 
@@ -449,9 +452,9 @@ int Connection::tls_handshake() {
       }
     }
   }
-#else  // !OPENSSL_1_1_1_API
+#else  // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
   rv = SSL_do_handshake(tls.ssl);
-#endif // !OPENSSL_1_1_1_API
+#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
 
   if (rv <= 0) {
     auto err = SSL_get_error(tls.ssl, rv);
@@ -499,7 +502,12 @@ int Connection::tls_handshake() {
   // Don't send handshake data if handshake was completed in OpenSSL
   // routine.  We have to check HTTP/2 requirement if HTTP/2 was
   // negotiated before sending finished message to the peer.
-  if (rv != 1 && tls.wbuf.rleft()) {
+  if ((rv != 1
+#ifdef OPENSSL_IS_BORINGSSL
+       || SSL_in_init(tls.ssl)
+#endif // OPENSSL_IS_BORINGSSL
+           ) &&
+      tls.wbuf.rleft()) {
     // First write indicates that resumption stuff has done.
     if (tls.handshake_state != TLSHandshakeState::WRITE_STARTED) {
       tls.handshake_state = TLSHandshakeState::WRITE_STARTED;
@@ -535,6 +543,40 @@ int Connection::tls_handshake() {
     return SHRPX_ERR_INPROGRESS;
   }
 
+#ifdef OPENSSL_IS_BORINGSSL
+  if (!tlsconf.no_postpone_early_data && SSL_in_early_data(tls.ssl) &&
+      SSL_in_init(tls.ssl)) {
+    auto nread = SSL_read(tls.ssl, buf.data(), buf.size());
+    if (nread <= 0) {
+      auto err = SSL_get_error(tls.ssl, nread);
+      switch (err) {
+      case SSL_ERROR_WANT_READ:
+      case SSL_ERROR_WANT_WRITE:
+        break;
+      case SSL_ERROR_ZERO_RETURN:
+        return SHRPX_ERR_EOF;
+      case SSL_ERROR_SSL:
+        if (LOG_ENABLED(INFO)) {
+          LOG(INFO) << "SSL_read: "
+                    << ERR_error_string(ERR_get_error(), nullptr);
+        }
+        return SHRPX_ERR_NETWORK;
+      default:
+        if (LOG_ENABLED(INFO)) {
+          LOG(INFO) << "SSL_read: SSL_get_error returned " << err;
+        }
+        return SHRPX_ERR_NETWORK;
+      }
+    } else {
+      tls.earlybuf.append(buf.data(), nread);
+    }
+
+    if (SSL_in_init(tls.ssl)) {
+      return SHRPX_ERR_INPROGRESS;
+    }
+  }
+#endif // OPENSSL_IS_BORINGSSL
+
   // Handshake was done
 
   rv = check_http2_requirement();
@@ -571,6 +613,36 @@ int Connection::write_tls_pending_handshake() {
     tls.wbuf.drain(nwrite);
   }
 
+#ifdef OPENSSL_IS_BORINGSSL
+  if (!SSL_in_init(tls.ssl)) {
+    // This will send a session ticket.
+    auto nwrite = SSL_write(tls.ssl, "", 0);
+    if (nwrite < 0) {
+      auto err = SSL_get_error(tls.ssl, nwrite);
+      switch (err) {
+      case SSL_ERROR_WANT_READ:
+        if (LOG_ENABLED(INFO)) {
+          LOG(INFO) << "Close connection due to TLS renegotiation";
+        }
+        return SHRPX_ERR_NETWORK;
+      case SSL_ERROR_WANT_WRITE:
+        break;
+      case SSL_ERROR_SSL:
+        if (LOG_ENABLED(INFO)) {
+          LOG(INFO) << "SSL_write: "
+                    << ERR_error_string(ERR_get_error(), nullptr);
+        }
+        return SHRPX_ERR_NETWORK;
+      default:
+        if (LOG_ENABLED(INFO)) {
+          LOG(INFO) << "SSL_write: SSL_get_error returned " << err;
+        }
+        return SHRPX_ERR_NETWORK;
+      }
+    }
+  }
+#endif // OPENSSL_IS_BORINGSSL
+
   // We have to start read watcher, since later stage of code expects
   // this.
   rlimit.startw();
@@ -681,7 +753,7 @@ ssize_t Connection::write_tls(const void *data, size_t len) {
   // length) on SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE.
   // get_write_limit() may return smaller length than previously
   // passed to SSL_write, which violates OpenSSL assumption.  To avoid
-  // this, we keep last legnth passed to SSL_write to
+  // this, we keep last length passed to SSL_write to
   // tls.last_writelen if SSL_write indicated I/O blocking.
   if (tls.last_writelen == 0) {
     len = std::min(len, wlimit.avail());
@@ -698,7 +770,7 @@ ssize_t Connection::write_tls(const void *data, size_t len) {
 
   ERR_clear_error();
 
-#if OPENSSL_1_1_1_API
+#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
   int rv;
   if (SSL_is_init_finished(tls.ssl)) {
     rv = SSL_write(tls.ssl, data, len);
@@ -710,9 +782,9 @@ ssize_t Connection::write_tls(const void *data, size_t len) {
       rv = nwrite;
     }
   }
-#else  // !OPENSSL_1_1_1_API
+#else  // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
   auto rv = SSL_write(tls.ssl, data, len);
-#endif // !OPENSSL_1_1_1_API
+#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
 
   if (rv <= 0) {
     auto err = SSL_get_error(tls.ssl, rv);
@@ -759,7 +831,7 @@ ssize_t Connection::read_tls(void *data, size_t len) {
   // length) on SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE.
   // rlimit_.avail() or rlimit_.avail() may return different length
   // than the length previously passed to SSL_read, which violates
-  // OpenSSL assumption.  To avoid this, we keep last legnth passed
+  // OpenSSL assumption.  To avoid this, we keep last length passed
   // to SSL_read to tls_last_readlen_ if SSL_read indicated I/O
   // blocking.
   if (tls.last_readlen == 0) {
@@ -772,7 +844,7 @@ ssize_t Connection::read_tls(void *data, size_t len) {
     tls.last_readlen = 0;
   }
 
-#if OPENSSL_1_1_1_API
+#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
   if (!tls.early_data_finish) {
     // TLSv1.3 handshake is still going on.
     size_t nread;
@@ -811,7 +883,7 @@ ssize_t Connection::read_tls(void *data, size_t len) {
     }
     return nread;
   }
-#endif // OPENSSL_1_1_1_API
+#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
 
   auto rv = SSL_read(tls.ssl, data, len);
 

src/shrpx_connection_handler.cc

@@ -298,8 +298,6 @@ int ConnectionHandler::create_single_worker() {
 #endif // HAVE_MRUBY
 
 #ifdef ENABLE_HTTP3
-  single_worker_->set_quic_secret(quic_secret_);
-
   if (single_worker_->setup_quic_server_socket() != 0) {
     return -1;
   }
@@ -404,8 +402,6 @@ int ConnectionHandler::create_worker_thread(size_t num) {
 #  endif // HAVE_MRUBY
 
 #  ifdef ENABLE_HTTP3
-    worker->set_quic_secret(quic_secret_);
-
     if ((!apiconf.enabled || i != 0) &&
         worker->setup_quic_server_socket() != 0) {
       return -1;
@@ -744,9 +740,9 @@ void ConnectionHandler::handle_ocsp_complete() {
     // that case we get nullptr.
     auto quic_ssl_ctx = quic_all_ssl_ctx_[ocsp_.next];
     if (quic_ssl_ctx) {
+#  ifndef OPENSSL_IS_BORINGSSL
       auto quic_tls_ctx_data = static_cast<tls::TLSContextData *>(
           SSL_CTX_get_app_data(quic_ssl_ctx));
-#  ifndef OPENSSL_IS_BORINGSSL
 #    ifdef HAVE_ATOMIC_STD_SHARED_PTR
       std::atomic_store_explicit(
           &quic_tls_ctx_data->ocsp_data,
@@ -758,7 +754,8 @@ void ConnectionHandler::handle_ocsp_complete() {
           std::make_shared<std::vector<uint8_t>>(ocsp_.resp);
 #    endif // !HAVE_ATOMIC_STD_SHARED_PTR
 #  else    // OPENSSL_IS_BORINGSSL
-      SSL_CTX_set_ocsp_response(ssl_ctx, ocsp_.resp.data(), ocsp_.resp.size());
+      SSL_CTX_set_ocsp_response(quic_ssl_ctx, ocsp_.resp.data(),
+                                ocsp_.resp.size());
 #  endif   // OPENSSL_IS_BORINGSSL
     }
 #endif // ENABLE_HTTP3
@@ -1020,12 +1017,10 @@ void ConnectionHandler::set_enable_acceptor_on_ocsp_completion(bool f) {
 }
 
 #ifdef ENABLE_HTTP3
-int ConnectionHandler::forward_quic_packet(const UpstreamAddr *faddr,
+int ConnectionHandler::forward_quic_packet(
-                                           const Address &remote_addr,
+    const UpstreamAddr *faddr, const Address &remote_addr,
-                                           const Address &local_addr,
+    const Address &local_addr, const ngtcp2_pkt_info &pi,
-                                           const uint8_t *cid_prefix,
+    const uint8_t *cid_prefix, const uint8_t *data, size_t datalen) {
-                                           const uint8_t *data,
-                                           size_t datalen) {
   assert(!get_config()->single_thread);
 
   for (auto &worker : workers_) {
@@ -1037,7 +1032,7 @@ int ConnectionHandler::forward_quic_packet(const UpstreamAddr *faddr,
     WorkerEvent wev{};
     wev.type = WorkerEventType::QUIC_PKT_FORWARD;
     wev.quic_pkt = std::make_unique<QUICPacket>(faddr->index, remote_addr,
-                                                local_addr, data, datalen);
+                                                local_addr, pi, data, datalen);
 
     worker->send(std::move(wev));
 
@@ -1047,25 +1042,14 @@ int ConnectionHandler::forward_quic_packet(const UpstreamAddr *faddr,
   return -1;
 }
 
-int ConnectionHandler::create_quic_secret() {
+void ConnectionHandler::set_quic_keying_materials(
-  auto quic_secret = std::make_shared<QUICSecret>();
+    std::shared_ptr<QUICKeyingMaterials> qkms) {
+  quic_keying_materials_ = std::move(qkms);
+}
 
-  if (generate_quic_stateless_reset_secret(
+const std::shared_ptr<QUICKeyingMaterials> &
-          quic_secret->stateless_reset_secret.data()) != 0) {
+ConnectionHandler::get_quic_keying_materials() const {
-    LOG(ERROR) << "Failed to generate QUIC Stateless Reset secret";
+  return quic_keying_materials_;
-
-    return -1;
-  }
-
-  if (generate_quic_token_secret(quic_secret->token_secret.data()) != 0) {
-    LOG(ERROR) << "Failed to generate QUIC token secret";
-
-    return -1;
-  }
-
-  quic_secret_ = std::move(quic_secret);
-
-  return 0;
 }
 
 void ConnectionHandler::set_cid_prefixes(
@@ -1094,6 +1078,26 @@ ConnectionHandler::match_quic_lingering_worker_process_cid_prefix(
 std::vector<BPFRef> &ConnectionHandler::get_quic_bpf_refs() {
   return quic_bpf_refs_;
 }
+
+void ConnectionHandler::unload_bpf_objects() {
+  std::array<char, STRERROR_BUFSIZE> errbuf;
+
+  LOG(NOTICE) << "Unloading BPF objects";
+
+  for (auto &ref : quic_bpf_refs_) {
+    if (ref.obj == nullptr) {
+      continue;
+    }
+
+    if (bpf_object__unload(ref.obj) != 0) {
+      LOG(WARN) << "Failed to unload bpf object: "
+                << xsi_strerror(errno, errbuf.data(), errbuf.size());
+      continue;
+    }
+
+    ref.obj = nullptr;
+  }
+}
 #  endif // HAVE_LIBBPF
 
 void ConnectionHandler::set_quic_ipc_fd(int fd) { quic_ipc_fd_ = fd; }
@@ -1105,10 +1109,11 @@ void ConnectionHandler::set_quic_lingering_worker_processes(
 
 int ConnectionHandler::forward_quic_packet_to_lingering_worker_process(
     QUICLingeringWorkerProcess *quic_lwp, const Address &remote_addr,
-    const Address &local_addr, const uint8_t *data, size_t datalen) {
+    const Address &local_addr, const ngtcp2_pkt_info &pi, const uint8_t *data,
+    size_t datalen) {
   std::array<uint8_t, 512> header;
 
-  assert(header.size() >= 1 + 1 + 1 + sizeof(sockaddr_storage) * 2);
+  assert(header.size() >= 1 + 1 + 1 + 1 + sizeof(sockaddr_storage) * 2);
   assert(remote_addr.len > 0);
   assert(local_addr.len > 0);
 
@@ -1121,6 +1126,7 @@ int ConnectionHandler::forward_quic_packet_to_lingering_worker_process(
   *p++ = static_cast<uint8_t>(local_addr.len - 1);
   p = std::copy_n(reinterpret_cast<const uint8_t *>(&local_addr.su),
                   local_addr.len, p);
+  *p++ = pi.ecn;
 
   iovec msg_iov[] = {
       {
@@ -1179,14 +1185,14 @@ int ConnectionHandler::quic_ipc_read() {
     return 0;
   }
 
-  size_t len = 1 + 1 + 1;
+  size_t len = 1 + 1 + 1 + 1;
 
   // Wire format:
-  // TYPE(1) REMOTE_ADDRLEN(1) REMOTE_ADDR(N) LOCAL_ADDRLEN(1) REMOTE_ADDR(N)
+  // TYPE(1) REMOTE_ADDRLEN(1) REMOTE_ADDR(N) LOCAL_ADDRLEN(1) LOCAL_ADDR(N)
-  // DGRAM_PAYLAOD(N)
+  // ECN(1) DGRAM_PAYLOAD(N)
   //
-  // When encoding, REMOTE_ADDRLEN and LOCAL_ADDRLEN is decremented by
+  // When encoding, REMOTE_ADDRLEN and LOCAL_ADDRLEN are decremented
-  // 1.
+  // by 1.
   if (static_cast<size_t>(nread) < len) {
     return 0;
   }
@@ -1243,6 +1249,8 @@ int ConnectionHandler::quic_ipc_read() {
 
   p += local_addrlen;
 
+  pkt->pi.ecn = *p++;
+
   auto datalen = nread - (p - buf.data());
 
   pkt->data.assign(p, p + datalen);
@@ -1265,8 +1273,8 @@ int ConnectionHandler::quic_ipc_read() {
     return -1;
   }
 
-  if (dcidlen < SHRPX_QUIC_CID_PREFIXLEN) {
+  if (dcidlen != SHRPX_QUIC_SCIDLEN) {
-    LOG(ERROR) << "DCID is too short";
+    LOG(ERROR) << "DCID length is invalid";
     return -1;
   }
 
@@ -1282,13 +1290,25 @@ int ConnectionHandler::quic_ipc_read() {
 
     // Ignore return value
     quic_conn_handler->handle_packet(faddr, pkt->remote_addr, pkt->local_addr,
-                                     pkt->data.data(), pkt->data.size());
+                                     pkt->pi, pkt->data.data(),
+                                     pkt->data.size());
 
     return 0;
   }
 
+  auto &qkm = quic_keying_materials_->keying_materials.front();
+
+  std::array<uint8_t, SHRPX_QUIC_DECRYPTED_DCIDLEN> decrypted_dcid;
+
+  if (decrypt_quic_connection_id(decrypted_dcid.data(),
+                                 dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
+                                 qkm.cid_encryption_key.data()) != 0) {
+    return -1;
+  }
+
   for (auto &worker : workers_) {
-    if (!std::equal(dcid, dcid + SHRPX_QUIC_CID_PREFIXLEN,
+    if (!std::equal(std::begin(decrypted_dcid),
+                    std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
                     worker->get_cid_prefix())) {
       continue;
     }

src/shrpx_connection_handler.h

@@ -106,6 +106,7 @@ struct SerialEvent {
 #ifdef ENABLE_HTTP3
 #  ifdef HAVE_LIBBPF
 struct BPFRef {
+  bpf_object *obj;
   int reuseport_array;
   int cid_prefix_map;
 };
@@ -162,7 +163,7 @@ public:
 
   // Cancels ocsp update process
   void cancel_ocsp_update();
-  // Starts ocsp update for certficate |cert_file|.
+  // Starts ocsp update for certificate |cert_file|.
   int start_ocsp_update(const char *cert_file);
   // Reads incoming data from ocsp update process
   void read_ocsp_chunk();
@@ -195,10 +196,12 @@ public:
   const std::vector<SSL_CTX *> &get_quic_indexed_ssl_ctx(size_t idx) const;
 
   int forward_quic_packet(const UpstreamAddr *faddr, const Address &remote_addr,
-                          const Address &local_addr, const uint8_t *cid_prefix,
+                          const Address &local_addr, const ngtcp2_pkt_info &pi,
-                          const uint8_t *data, size_t datalen);
+                          const uint8_t *cid_prefix, const uint8_t *data,
+                          size_t datalen);
 
-  int create_quic_secret();
+  void set_quic_keying_materials(std::shared_ptr<QUICKeyingMaterials> qkms);
+  const std::shared_ptr<QUICKeyingMaterials> &get_quic_keying_materials() const;
 
   void set_cid_prefixes(
       const std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>>
@@ -216,7 +219,8 @@ public:
 
   int forward_quic_packet_to_lingering_worker_process(
       QUICLingeringWorkerProcess *quic_lwp, const Address &remote_addr,
-      const Address &local_addr, const uint8_t *data, size_t datalen);
+      const Address &local_addr, const ngtcp2_pkt_info &pi, const uint8_t *data,
+      size_t datalen);
 
   void set_quic_ipc_fd(int fd);
 
@@ -224,6 +228,7 @@ public:
 
 #  ifdef HAVE_LIBBPF
   std::vector<BPFRef> &get_quic_bpf_refs();
+  void unload_bpf_objects();
 #  endif // HAVE_LIBBPF
 #endif   // ENABLE_HTTP3
 
@@ -263,7 +268,7 @@ private:
 #  ifdef HAVE_LIBBPF
   std::vector<BPFRef> quic_bpf_refs_;
 #  endif // HAVE_LIBBPF
-  std::shared_ptr<QUICSecret> quic_secret_;
+  std::shared_ptr<QUICKeyingMaterials> quic_keying_materials_;
   std::vector<SSL_CTX *> quic_all_ssl_ctx_;
   std::vector<std::vector<SSL_CTX *>> quic_indexed_ssl_ctx_;
 #endif // ENABLE_HTTP3

src/shrpx_dns_resolver.h

@@ -88,7 +88,7 @@ public:
   int on_write(int fd);
   int on_timeout();
   // Calls this function when DNS query finished.
-  void on_result(int staus, hostent *hostent);
+  void on_result(int status, hostent *hostent);
   void reset_timeout();
 
   void start_rev(int fd);

src/shrpx_downstream_connection.h

@@ -58,7 +58,7 @@ public:
   virtual int on_write() = 0;
   virtual int on_timeout() { return 0; }
 
-  virtual void on_upstream_change(Upstream *uptream) = 0;
+  virtual void on_upstream_change(Upstream *upstream) = 0;
 
   // true if this object is poolable.
   virtual bool poolable() const = 0;

src/shrpx_health_monitor_downstream_connection.cc

@@ -98,7 +98,8 @@ int HealthMonitorDownstreamConnection::on_read() { return 0; }
 
 int HealthMonitorDownstreamConnection::on_write() { return 0; }
 
-void HealthMonitorDownstreamConnection::on_upstream_change(Upstream *uptream) {}
+void HealthMonitorDownstreamConnection::on_upstream_change(Upstream *upstream) {
+}
 
 bool HealthMonitorDownstreamConnection::poolable() const { return false; }
 

src/shrpx_health_monitor_downstream_connection.h

@@ -49,7 +49,7 @@ public:
   virtual int on_read();
   virtual int on_write();
 
-  virtual void on_upstream_change(Upstream *uptream);
+  virtual void on_upstream_change(Upstream *upstream);
 
   // true if this object is poolable.
   virtual bool poolable() const;

src/shrpx_http2_downstream_connection.h

@@ -60,7 +60,7 @@ public:
 
   virtual void on_upstream_change(Upstream *upstream) {}
 
-  // This object is not poolable because we dont' have facility to
+  // This object is not poolable because we don't have facility to
   // migrate to another Http2Session object.
   virtual bool poolable() const { return false; }
 

src/shrpx_http2_session.cc

@@ -206,13 +206,13 @@ Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx,
   on_read_ = &Http2Session::read_noop;
   on_write_ = &Http2Session::write_noop;
 
-  // We will resuse this many times, so use repeat timeout value.  The
+  // We will reuse this many times, so use repeat timeout value.  The
   // timeout value is set later.
   ev_timer_init(&connchk_timer_, connchk_timeout_cb, 0., 0.);
 
   connchk_timer_.data = this;
 
-  // SETTINGS ACK timeout is 10 seconds for now.  We will resuse this
+  // SETTINGS ACK timeout is 10 seconds for now.  We will reuse this
   // many times, so use repeat timeout value.
   ev_timer_init(&settings_timer_, settings_timeout_cb, 0., 0.);
 

src/shrpx_http2_upstream.cc

@@ -1317,7 +1317,7 @@ int Http2Upstream::downstream_eof(DownstreamConnection *dconn) {
   downstream->pop_downstream_connection();
   // dconn was deleted
   dconn = nullptr;
-  // downstream wil be deleted in on_stream_close_callback.
+  // downstream will be deleted in on_stream_close_callback.
   if (downstream->get_response_state() == DownstreamState::HEADER_COMPLETE) {
     // Server may indicate the end of the request by EOF
     if (LOG_ENABLED(INFO)) {
@@ -2183,7 +2183,7 @@ int Http2Upstream::submit_push_promise(const StringRef &scheme,
   // 4 for :method, :scheme, :path and :authority
   nva.reserve(4 + req.fs.headers().size());
 
-  // juse use "GET" for now
+  // just use "GET" for now
   nva.push_back(http2::make_nv_ll(":method", "GET"));
   nva.push_back(http2::make_nv_ls_nocopy(":scheme", scheme));
   nva.push_back(http2::make_nv_ls_nocopy(":path", path));

src/shrpx_http3_upstream.cc

@@ -27,6 +27,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
+#include <netinet/udp.h>
 
 #include <cstdio>
 
@@ -39,6 +40,7 @@
 #include "shrpx_quic.h"
 #include "shrpx_worker.h"
 #include "shrpx_http.h"
+#include "shrpx_connection_handler.h"
 #ifdef HAVE_MRUBY
 #  include "shrpx_mruby.h"
 #endif // HAVE_MRUBY
@@ -116,13 +118,16 @@ size_t downstream_queue_size(Worker *worker) {
 
 Http3Upstream::Http3Upstream(ClientHandler *handler)
     : handler_{handler},
+      max_udp_payload_size_{SHRPX_QUIC_MAX_UDP_PAYLOAD_SIZE},
       qlog_fd_{-1},
+      hashed_scid_{},
       conn_{nullptr},
       tls_alert_{0},
       httpconn_{nullptr},
       downstream_queue_{downstream_queue_size(handler->get_worker()),
                         !get_config()->http2_proxy},
-      idle_close_{false} {
+      idle_close_{false},
+      retry_close_{false} {
   ev_timer_init(&timer_, timeoutcb, 0., 0.);
   timer_.data = this;
 
@@ -151,9 +156,7 @@ Http3Upstream::~Http3Upstream() {
 
   nghttp3_conn_del(httpconn_);
 
-  if (conn_) {
+  ngtcp2_conn_del(conn_);
-    ngtcp2_conn_del(conn_);
-  }
 
   if (qlog_fd_ != -1) {
     close(qlog_fd_);
@@ -192,9 +195,7 @@ void qlog_write(void *user_data, uint32_t flags, const void *data,
 void Http3Upstream::qlog_write(const void *data, size_t datalen, bool fin) {
   assert(qlog_fd_ != -1);
 
-  ssize_t nwrite;
+  while (write(qlog_fd_, data, datalen) == -1 && errno == EINTR)
-
-  while ((nwrite = write(qlog_fd_, data, datalen)) == -1 && errno == EINTR)
     ;
 
   if (fin) {
@@ -216,22 +217,23 @@ int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token,
   auto upstream = static_cast<Http3Upstream *>(user_data);
   auto handler = upstream->get_client_handler();
   auto worker = handler->get_worker();
+  auto conn_handler = worker->get_connection_handler();
+  auto &qkms = conn_handler->get_quic_keying_materials();
+  auto &qkm = qkms->keying_materials.front();
 
-  if (generate_quic_connection_id(cid, cidlen, worker->get_cid_prefix()) != 0) {
+  if (generate_quic_connection_id(*cid, cidlen, worker->get_cid_prefix(),
+                                  qkm.id, qkm.cid_encryption_key.data()) != 0) {
     return NGTCP2_ERR_CALLBACK_FAILURE;
   }
 
-  auto &quic_secret = worker->get_quic_secret();
+  if (generate_quic_stateless_reset_token(token, *cid, qkm.secret.data(),
-  auto &secret = quic_secret->stateless_reset_secret;
+                                          qkm.secret.size()) != 0) {
-
-  if (generate_quic_stateless_reset_token(token, cid, secret.data(),
-                                          secret.size()) != 0) {
     return NGTCP2_ERR_CALLBACK_FAILURE;
   }
 
   auto quic_connection_handler = worker->get_quic_connection_handler();
 
-  quic_connection_handler->add_connection_id(cid, handler);
+  quic_connection_handler->add_connection_id(*cid, handler);
 
   return 0;
 }
@@ -245,7 +247,7 @@ int remove_connection_id(ngtcp2_conn *conn, const ngtcp2_cid *cid,
   auto worker = handler->get_worker();
   auto quic_conn_handler = worker->get_quic_connection_handler();
 
-  quic_conn_handler->remove_connection_id(cid);
+  quic_conn_handler->remove_connection_id(*cid);
 
   return 0;
 }
@@ -476,16 +478,26 @@ int handshake_completed(ngtcp2_conn *conn, void *user_data) {
 } // namespace
 
 int Http3Upstream::handshake_completed() {
+  handler_->set_alpn_from_conn();
+
+  auto alpn = handler_->get_alpn();
+  if (alpn.empty()) {
+    ULOG(ERROR, this) << "NO ALPN was negotiated";
+    return -1;
+  }
+
   std::array<uint8_t, NGTCP2_CRYPTO_MAX_REGULAR_TOKENLEN> token;
   size_t tokenlen;
 
   auto path = ngtcp2_conn_get_path(conn_);
   auto worker = handler_->get_worker();
-  auto &quic_secret = worker->get_quic_secret();
+  auto conn_handler = worker->get_connection_handler();
-  auto &secret = quic_secret->token_secret;
+  auto &qkms = conn_handler->get_quic_keying_materials();
+  auto &qkm = qkms->keying_materials.front();
 
   if (generate_token(token.data(), tokenlen, path->remote.addr,
-                     path->remote.addrlen, secret.data()) != 0) {
+                     path->remote.addrlen, qkm.secret.data(),
+                     qkm.secret.size()) != 0) {
     return 0;
   }
 
@@ -507,6 +519,7 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
   int rv;
 
   auto worker = handler_->get_worker();
+  auto conn_handler = worker->get_connection_handler();
 
   auto callbacks = ngtcp2_callbacks{
       nullptr, // client_initial
@@ -547,17 +560,21 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
       shrpx::stream_stop_sending,
   };
 
-  ngtcp2_cid scid;
-
-  if (generate_quic_connection_id(&scid, SHRPX_QUIC_SCIDLEN,
-                                  worker->get_cid_prefix()) != 0) {
-    return -1;
-  }
-
   auto config = get_config();
   auto &quicconf = config->quic;
   auto &http3conf = config->http3;
 
+  auto &qkms = conn_handler->get_quic_keying_materials();
+  auto &qkm = qkms->keying_materials.front();
+
+  ngtcp2_cid scid;
+
+  if (generate_quic_connection_id(scid, SHRPX_QUIC_SCIDLEN,
+                                  worker->get_cid_prefix(), qkm.id,
+                                  qkm.cid_encryption_key.data()) != 0) {
+    return -1;
+  }
+
   ngtcp2_settings settings;
   ngtcp2_settings_default(&settings);
   if (quicconf.upstream.debug.log) {
@@ -574,7 +591,9 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
   }
 
   settings.initial_ts = quic_timestamp();
-  settings.cc_algo = NGTCP2_CC_ALGO_BBR;
+  settings.initial_rtt = static_cast<ngtcp2_tstamp>(
+      quicconf.upstream.initial_rtt * NGTCP2_SECONDS);
+  settings.cc_algo = quicconf.upstream.congestion_controller;
   settings.max_window = http3conf.upstream.max_connection_window_size;
   settings.max_stream_window = http3conf.upstream.max_window_size;
   settings.max_udp_payload_size = SHRPX_QUIC_MAX_UDP_PAYLOAD_SIZE;
@@ -593,6 +612,40 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
   params.max_idle_timeout = static_cast<ngtcp2_tstamp>(
       quicconf.upstream.timeout.idle * NGTCP2_SECONDS);
 
+#ifdef OPENSSL_IS_BORINGSSL
+  if (quicconf.upstream.early_data) {
+    ngtcp2_transport_params early_data_params{
+        .initial_max_stream_data_bidi_local =
+            params.initial_max_stream_data_bidi_local,
+        .initial_max_stream_data_bidi_remote =
+            params.initial_max_stream_data_bidi_remote,
+        .initial_max_stream_data_uni = params.initial_max_stream_data_uni,
+        .initial_max_data = params.initial_max_data,
+        .initial_max_streams_bidi = params.initial_max_streams_bidi,
+        .initial_max_streams_uni = params.initial_max_streams_uni,
+    };
+
+    // TODO include HTTP/3 SETTINGS
+
+    std::array<uint8_t, 128> quic_early_data_ctx;
+
+    auto quic_early_data_ctxlen = ngtcp2_encode_transport_params(
+        quic_early_data_ctx.data(), quic_early_data_ctx.size(),
+        NGTCP2_TRANSPORT_PARAMS_TYPE_ENCRYPTED_EXTENSIONS, &early_data_params);
+
+    assert(quic_early_data_ctxlen > 0);
+    assert(static_cast<size_t>(quic_early_data_ctxlen) <=
+           quic_early_data_ctx.size());
+
+    if (SSL_set_quic_early_data_context(handler_->get_ssl(),
+                                        quic_early_data_ctx.data(),
+                                        quic_early_data_ctxlen) != 1) {
+      ULOG(ERROR, this) << "SSL_set_quic_early_data_context failed";
+      return -1;
+    }
+  }
+#endif // OPENSSL_IS_BORINGSSL
+
   if (odcid) {
     params.original_dcid = *odcid;
     params.retry_scid = initial_hd.dcid;
@@ -601,12 +654,8 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
     params.original_dcid = initial_hd.dcid;
   }
 
-  auto &quic_secret = worker->get_quic_secret();
+  rv = generate_quic_stateless_reset_token(
-  auto &stateless_reset_secret = quic_secret->stateless_reset_secret;
+      params.stateless_reset_token, scid, qkm.secret.data(), qkm.secret.size());
-
-  rv = generate_quic_stateless_reset_token(params.stateless_reset_token, &scid,
-                                           stateless_reset_secret.data(),
-                                           stateless_reset_secret.size());
   if (rv != 0) {
     ULOG(ERROR, this) << "generate_quic_stateless_reset_token failed";
     return -1;
@@ -614,8 +663,14 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
   params.stateless_reset_token_present = 1;
 
   auto path = ngtcp2_path{
-      {local_addr.len, const_cast<sockaddr *>(&local_addr.su.sa)},
+      {
-      {remote_addr.len, const_cast<sockaddr *>(&remote_addr.su.sa)},
+          const_cast<sockaddr *>(&local_addr.su.sa),
+          local_addr.len,
+      },
+      {
+          const_cast<sockaddr *>(&remote_addr.su.sa),
+          remote_addr.len,
+      },
       const_cast<UpstreamAddr *>(faddr),
   };
 
@@ -631,8 +686,13 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
 
   auto quic_connection_handler = worker->get_quic_connection_handler();
 
-  quic_connection_handler->add_connection_id(&initial_hd.dcid, handler_);
+  if (generate_quic_hashed_connection_id(hashed_scid_, remote_addr, local_addr,
-  quic_connection_handler->add_connection_id(&scid, handler_);
+                                         initial_hd.dcid) != 0) {
+    return -1;
+  }
+
+  quic_connection_handler->add_connection_id(hashed_scid_, handler_);
+  quic_connection_handler->add_connection_id(scid, handler_);
 
   return 0;
 }
@@ -652,11 +712,12 @@ int Http3Upstream::on_write() {
 int Http3Upstream::write_streams() {
   std::array<nghttp3_vec, 16> vec;
   std::array<uint8_t, 64_k> buf;
-  auto max_udp_payload_size = ngtcp2_conn_get_path_max_udp_payload_size(conn_);
+  auto max_udp_payload_size = std::min(
+      max_udp_payload_size_, ngtcp2_conn_get_path_max_udp_payload_size(conn_));
   size_t max_pktcnt =
       std::min(static_cast<size_t>(64_k), ngtcp2_conn_get_send_quantum(conn_)) /
       max_udp_payload_size;
-  ngtcp2_pkt_info pi;
+  ngtcp2_pkt_info pi, prev_pi;
   uint8_t *bufpos = buf.data();
   ngtcp2_path_storage ps, prev_ps;
   size_t pktcnt = 0;
@@ -666,6 +727,13 @@ int Http3Upstream::write_streams() {
   ngtcp2_path_storage_zero(&ps);
   ngtcp2_path_storage_zero(&prev_ps);
 
+  auto config = get_config();
+  auto &quicconf = config->quic;
+
+  if (quicconf.upstream.congestion_controller != NGTCP2_CC_ALGO_BBR) {
+    max_pktcnt = std::min(max_pktcnt, static_cast<size_t>(10));
+  }
+
   for (;;) {
     int64_t stream_id = -1;
     int fin = 0;
@@ -750,15 +818,17 @@ int Http3Upstream::write_streams() {
 
     if (nwrite == 0) {
       if (bufpos - buf.data()) {
-        quic_send_packet(static_cast<UpstreamAddr *>(prev_ps.path.user_data),
+        send_packet(static_cast<UpstreamAddr *>(prev_ps.path.user_data),
-                         prev_ps.path.remote.addr, prev_ps.path.remote.addrlen,
+                    prev_ps.path.remote.addr, prev_ps.path.remote.addrlen,
-                         prev_ps.path.local.addr, prev_ps.path.local.addrlen,
+                    prev_ps.path.local.addr, prev_ps.path.local.addrlen,
-                         buf.data(), bufpos - buf.data(), max_udp_payload_size);
+                    prev_pi, buf.data(), bufpos - buf.data(),
+                    max_udp_payload_size);
 
-        ngtcp2_conn_update_pkt_tx_time(conn_, ts);
         reset_idle_timer();
       }
 
+      ngtcp2_conn_update_pkt_tx_time(conn_, ts);
+
       handler_->get_connection()->wlimit.stopw();
 
       return 0;
@@ -769,17 +839,19 @@ int Http3Upstream::write_streams() {
 #ifdef UDP_SEGMENT
     if (pktcnt == 0) {
       ngtcp2_path_copy(&prev_ps.path, &ps.path);
-    } else if (!ngtcp2_path_eq(&prev_ps.path, &ps.path)) {
+      prev_pi = pi;
-      quic_send_packet(static_cast<UpstreamAddr *>(prev_ps.path.user_data),
+    } else if (!ngtcp2_path_eq(&prev_ps.path, &ps.path) ||
-                       prev_ps.path.remote.addr, prev_ps.path.remote.addrlen,
+               prev_pi.ecn != pi.ecn) {
-                       prev_ps.path.local.addr, prev_ps.path.local.addrlen,
+      send_packet(static_cast<UpstreamAddr *>(prev_ps.path.user_data),
-                       buf.data(), bufpos - buf.data() - nwrite,
+                  prev_ps.path.remote.addr, prev_ps.path.remote.addrlen,
-                       max_udp_payload_size);
+                  prev_ps.path.local.addr, prev_ps.path.local.addrlen, prev_pi,
+                  buf.data(), bufpos - buf.data() - nwrite,
+                  max_udp_payload_size);
 
-      quic_send_packet(static_cast<UpstreamAddr *>(ps.path.user_data),
+      send_packet(static_cast<UpstreamAddr *>(ps.path.user_data),
-                       ps.path.remote.addr, ps.path.remote.addrlen,
+                  ps.path.remote.addr, ps.path.remote.addrlen,
-                       ps.path.local.addr, ps.path.local.addrlen,
+                  ps.path.local.addr, ps.path.local.addrlen, pi,
-                       bufpos - nwrite, nwrite, max_udp_payload_size);
+                  bufpos - nwrite, nwrite, max_udp_payload_size);
 
       ngtcp2_conn_update_pkt_tx_time(conn_, ts);
       reset_idle_timer();
@@ -791,10 +863,10 @@ int Http3Upstream::write_streams() {
 
     if (++pktcnt == max_pktcnt ||
         static_cast<size_t>(nwrite) < max_udp_payload_size) {
-      quic_send_packet(static_cast<UpstreamAddr *>(ps.path.user_data),
+      send_packet(static_cast<UpstreamAddr *>(ps.path.user_data),
-                       ps.path.remote.addr, ps.path.remote.addrlen,
+                  ps.path.remote.addr, ps.path.remote.addrlen,
-                       ps.path.local.addr, ps.path.local.addrlen, buf.data(),
+                  ps.path.local.addr, ps.path.local.addrlen, pi, buf.data(),
-                       bufpos - buf.data(), max_udp_payload_size);
+                  bufpos - buf.data(), max_udp_payload_size);
 
       ngtcp2_conn_update_pkt_tx_time(conn_, ts);
       reset_idle_timer();
@@ -804,10 +876,9 @@ int Http3Upstream::write_streams() {
       return 0;
     }
 #else  // !UDP_SEGMENT
-    quic_send_packet(static_cast<UpstreamAddr *>(ps.path.user_data),
+    send_packet(static_cast<UpstreamAddr *>(ps.path.user_data),
-                     ps.path.remote.addr, ps.path.remote.addrlen,
+                ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr,
-                     ps.path.local.addr, ps.path.local.addrlen, buf.data(),
+                ps.path.local.addrlen, pi, buf.data(), bufpos - buf.data(), 0);
-                     bufpos - buf.data(), 0);
 
     if (++pktcnt == max_pktcnt) {
       ngtcp2_conn_update_pkt_tx_time(conn_, ts);
@@ -952,7 +1023,7 @@ int Http3Upstream::downstream_eof(DownstreamConnection *dconn) {
   downstream->pop_downstream_connection();
   // dconn was deleted
   dconn = nullptr;
-  // downstream wil be deleted in on_stream_close_callback.
+  // downstream will be deleted in on_stream_close_callback.
   if (downstream->get_response_state() == DownstreamState::HEADER_COMPLETE) {
     // Server may indicate the end of the request by EOF
     if (LOG_ENABLED(INFO)) {
@@ -964,7 +1035,9 @@ int Http3Upstream::downstream_eof(DownstreamConnection *dconn) {
     // downstream_read_data_callback to send RST_STREAM after pending
     // response body is sent. This is needed to ensure that RST_STREAM
     // is sent after all pending data are sent.
-    on_downstream_body_complete(downstream);
+    if (on_downstream_body_complete(downstream) != 0) {
+      return -1;
+    }
   } else if (downstream->get_response_state() !=
              DownstreamState::MSG_COMPLETE) {
     // If stream was not closed, then we set MSG_COMPLETE and let
@@ -1008,7 +1081,9 @@ int Http3Upstream::downstream_error(DownstreamConnection *dconn, int events) {
   } else {
     if (downstream->get_response_state() == DownstreamState::HEADER_COMPLETE) {
       if (downstream->get_upgraded()) {
-        on_downstream_body_complete(downstream);
+        if (on_downstream_body_complete(downstream) != 0) {
+          return -1;
+        }
       } else {
         shutdown_stream(downstream, NGHTTP3_H3_INTERNAL_ERROR);
       }
@@ -1295,6 +1370,24 @@ int Http3Upstream::on_downstream_body_complete(Downstream *downstream) {
     return 0;
   }
 
+  if (!downstream->get_upgraded()) {
+    const auto &trailers = resp.fs.trailers();
+    if (!trailers.empty()) {
+      std::vector<nghttp3_nv> nva;
+      nva.reserve(trailers.size());
+      http3::copy_headers_to_nva_nocopy(nva, trailers, http2::HDOP_STRIP_ALL);
+      if (!nva.empty()) {
+        auto rv = nghttp3_conn_submit_trailers(
+            httpconn_, downstream->get_stream_id(), nva.data(), nva.size());
+        if (rv != 0) {
+          ULOG(FATAL, this) << "nghttp3_conn_submit_trailers() failed: "
+                            << nghttp3_strerror(rv);
+          return -1;
+        }
+      }
+    }
+  }
+
   nghttp3_conn_resume_stream(httpconn_, downstream->get_stream_id());
   downstream->ensure_upstream_wtimer();
 
@@ -1312,17 +1405,15 @@ void Http3Upstream::on_handler_delete() {
   auto worker = handler_->get_worker();
   auto quic_conn_handler = worker->get_quic_connection_handler();
 
-  quic_conn_handler->remove_connection_id(
+  std::vector<ngtcp2_cid> scids(ngtcp2_conn_get_num_scid(conn_) + 1);
-      ngtcp2_conn_get_client_initial_dcid(conn_));
-
-  std::vector<ngtcp2_cid> scids(ngtcp2_conn_get_num_scid(conn_));
   ngtcp2_conn_get_scid(conn_, scids.data());
+  scids.back() = hashed_scid_;
 
   for (auto &cid : scids) {
-    quic_conn_handler->remove_connection_id(&cid);
+    quic_conn_handler->remove_connection_id(cid);
   }
 
-  if (idle_close_) {
+  if (idle_close_ || retry_close_) {
     return;
   }
 
@@ -1337,7 +1428,7 @@ void Http3Upstream::on_handler_delete() {
 
     auto nwrite = ngtcp2_conn_write_connection_close(
         conn_, &ps.path, &pi, conn_close_.data(), conn_close_.size(),
-        NGTCP2_NO_ERROR, quic_timestamp());
+        NGTCP2_NO_ERROR, nullptr, 0, quic_timestamp());
     if (nwrite < 0) {
       if (nwrite != NGTCP2_ERR_INVALID_STATE) {
         ULOG(ERROR, this) << "ngtcp2_conn_write_connection_close: "
@@ -1349,10 +1440,9 @@ void Http3Upstream::on_handler_delete() {
 
     conn_close_.resize(nwrite);
 
-    quic_send_packet(static_cast<UpstreamAddr *>(ps.path.user_data),
+    send_packet(static_cast<UpstreamAddr *>(ps.path.user_data),
-                     ps.path.remote.addr, ps.path.remote.addrlen,
+                ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr,
-                     ps.path.local.addr, ps.path.local.addrlen,
+                ps.path.local.addrlen, pi, conn_close_.data(), nwrite, 0);
-                     conn_close_.data(), nwrite, 0);
   }
 
   auto d =
@@ -1571,34 +1661,63 @@ void Http3Upstream::cancel_premature_downstream(
 
 int Http3Upstream::on_read(const UpstreamAddr *faddr,
                            const Address &remote_addr,
-                           const Address &local_addr, const uint8_t *data,
+                           const Address &local_addr, const ngtcp2_pkt_info &pi,
-                           size_t datalen) {
+                           const uint8_t *data, size_t datalen) {
   int rv;
-  ngtcp2_pkt_info pi{};
 
   auto path = ngtcp2_path{
       {
-          local_addr.len,
           const_cast<sockaddr *>(&local_addr.su.sa),
+          local_addr.len,
       },
       {
-          remote_addr.len,
           const_cast<sockaddr *>(&remote_addr.su.sa),
+          remote_addr.len,
       },
       const_cast<UpstreamAddr *>(faddr),
   };
 
   rv = ngtcp2_conn_read_pkt(conn_, &path, &pi, data, datalen, quic_timestamp());
   if (rv != 0) {
-    ULOG(ERROR, this) << "ngtcp2_conn_read_pkt: " << ngtcp2_strerror(rv);
-
     switch (rv) {
     case NGTCP2_ERR_DRAINING:
-      // TODO Start drain period
       return -1;
-    case NGTCP2_ERR_RETRY:
+    case NGTCP2_ERR_RETRY: {
-      // TODO Send Retry packet
+      auto worker = handler_->get_worker();
+      auto quic_conn_handler = worker->get_quic_connection_handler();
+
+      uint32_t version;
+      const uint8_t *dcid, *scid;
+      size_t dcidlen, scidlen;
+
+      rv = ngtcp2_pkt_decode_version_cid(&version, &dcid, &dcidlen, &scid,
+                                         &scidlen, data, datalen,
+                                         SHRPX_QUIC_SCIDLEN);
+      if (rv != 0) {
+        return -1;
+      }
+
+      if (worker->get_graceful_shutdown()) {
+        ngtcp2_cid ini_dcid, ini_scid;
+
+        ngtcp2_cid_init(&ini_dcid, dcid, dcidlen);
+        ngtcp2_cid_init(&ini_scid, scid, scidlen);
+
+        quic_conn_handler->send_connection_close(
+            faddr, version, ini_dcid, ini_scid, remote_addr, local_addr,
+            NGTCP2_CONNECTION_REFUSED);
+
+        return -1;
+      }
+
+      retry_close_ = true;
+
+      quic_conn_handler->send_retry(handler_->get_upstream_addr(), version,
+                                    dcid, dcidlen, scid, scidlen, remote_addr,
+                                    local_addr);
+
       return -1;
+    }
     case NGTCP2_ERR_REQUIRED_TRANSPORT_PARAM:
     case NGTCP2_ERR_MALFORMED_TRANSPORT_PARAM:
     case NGTCP2_ERR_TRANSPORT_PARAM:
@@ -1615,7 +1734,8 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr,
       }
     }
 
-    // TODO Send connection close
+    ULOG(ERROR, this) << "ngtcp2_conn_read_pkt: " << ngtcp2_strerror(rv);
+
     return handle_error();
   }
 
@@ -1624,6 +1744,29 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr,
   return 0;
 }
 
+int Http3Upstream::send_packet(const UpstreamAddr *faddr,
+                               const sockaddr *remote_sa, size_t remote_salen,
+                               const sockaddr *local_sa, size_t local_salen,
+                               const ngtcp2_pkt_info &pi, const uint8_t *data,
+                               size_t datalen, size_t gso_size) {
+  auto rv = quic_send_packet(faddr, remote_sa, remote_salen, local_sa,
+                             local_salen, pi, data, datalen, gso_size);
+  switch (rv) {
+  case 0:
+    return 0;
+    // With GSO, sendmsg may fail with EINVAL if UDP payload is too
+    // large.
+  case -EINVAL:
+  case -EMSGSIZE:
+    max_udp_payload_size_ = NGTCP2_MAX_UDP_PAYLOAD_SIZE;
+    break;
+  default:
+    break;
+  }
+
+  return -1;
+}
+
 int Http3Upstream::handle_error() {
   if (ngtcp2_conn_is_in_closing_period(conn_)) {
     return -1;
@@ -1643,7 +1786,7 @@ int Http3Upstream::handle_error() {
   if (last_error_.type == quic::ErrorType::Transport) {
     nwrite = ngtcp2_conn_write_connection_close(
         conn_, &ps.path, &pi, conn_close_.data(), conn_close_.size(),
-        last_error_.code, ts);
+        last_error_.code, nullptr, 0, ts);
     if (nwrite < 0) {
       ULOG(ERROR, this) << "ngtcp2_conn_write_connection_close: "
                         << ngtcp2_strerror(nwrite);
@@ -1652,7 +1795,7 @@ int Http3Upstream::handle_error() {
   } else {
     nwrite = ngtcp2_conn_write_application_close(
         conn_, &ps.path, &pi, conn_close_.data(), conn_close_.size(),
-        last_error_.code, ts);
+        last_error_.code, nullptr, 0, ts);
     if (nwrite < 0) {
       ULOG(ERROR, this) << "ngtcp2_conn_write_application_close: "
                         << ngtcp2_strerror(nwrite);
@@ -1662,10 +1805,9 @@ int Http3Upstream::handle_error() {
 
   conn_close_.resize(nwrite);
 
-  quic_send_packet(static_cast<UpstreamAddr *>(ps.path.user_data),
+  send_packet(static_cast<UpstreamAddr *>(ps.path.user_data),
-                   ps.path.remote.addr, ps.path.remote.addrlen,
+              ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr,
-                   ps.path.local.addr, ps.path.local.addrlen,
+              ps.path.local.addrlen, pi, conn_close_.data(), nwrite, 0);
-                   conn_close_.data(), nwrite, 0);
 
   return -1;
 }
@@ -1762,7 +1904,7 @@ int http_deferred_consume(nghttp3_conn *conn, int64_t stream_id,
 
 namespace {
 int http_acked_stream_data(nghttp3_conn *conn, int64_t stream_id,
-                           size_t datalen, void *user_data,
+                           uint64_t datalen, void *user_data,
                            void *stream_user_data) {
   auto upstream = static_cast<Http3Upstream *>(user_data);
   auto downstream = static_cast<Downstream *>(stream_user_data);
@@ -1778,7 +1920,7 @@ int http_acked_stream_data(nghttp3_conn *conn, int64_t stream_id,
 } // namespace
 
 int Http3Upstream::http_acked_stream_data(Downstream *downstream,
-                                          size_t datalen) {
+                                          uint64_t datalen) {
   if (LOG_ENABLED(INFO)) {
     ULOG(INFO, this) << "Stream " << downstream->get_stream_id() << " "
                      << datalen << " bytes acknowledged";
@@ -1786,6 +1928,7 @@ int Http3Upstream::http_acked_stream_data(Downstream *downstream,
 
   auto body = downstream->get_response_buf();
   auto drained = body->drain_mark(datalen);
+  (void)drained;
 
   assert(datalen == drained);
 
@@ -1822,8 +1965,29 @@ int http_recv_request_header(nghttp3_conn *conn, int64_t stream_id,
     return 0;
   }
 
-  if (upstream->http_recv_request_header(downstream, token, name, value,
+  if (upstream->http_recv_request_header(downstream, token, name, value, flags,
-                                         flags) != 0) {
+                                         /* trailer = */ false) != 0) {
+    return NGHTTP3_ERR_CALLBACK_FAILURE;
+  }
+
+  return 0;
+}
+} // namespace
+
+namespace {
+int http_recv_request_trailer(nghttp3_conn *conn, int64_t stream_id,
+                              int32_t token, nghttp3_rcbuf *name,
+                              nghttp3_rcbuf *value, uint8_t flags,
+                              void *user_data, void *stream_user_data) {
+  auto upstream = static_cast<Http3Upstream *>(user_data);
+  auto downstream = static_cast<Downstream *>(stream_user_data);
+
+  if (!downstream || downstream->get_stop_reading()) {
+    return 0;
+  }
+
+  if (upstream->http_recv_request_header(downstream, token, name, value, flags,
+                                         /* trailer = */ true) != 0) {
     return NGHTTP3_ERR_CALLBACK_FAILURE;
   }
 
@@ -1834,8 +1998,8 @@ int http_recv_request_header(nghttp3_conn *conn, int64_t stream_id,
 int Http3Upstream::http_recv_request_header(Downstream *downstream,
                                             int32_t h3token,
                                             nghttp3_rcbuf *name,
-                                            nghttp3_rcbuf *value,
+                                            nghttp3_rcbuf *value, uint8_t flags,
-                                            uint8_t flags) {
+                                            bool trailer) {
   auto namebuf = nghttp3_rcbuf_get_buf(name);
   auto valuebuf = nghttp3_rcbuf_get_buf(value);
   auto &req = downstream->request();
@@ -1857,8 +2021,13 @@ int Http3Upstream::http_recv_request_header(Downstream *downstream,
                        << ", num=" << req.fs.num_fields() + 1;
     }
 
+    // just ignore if this is a trailer part.
+    if (trailer) {
+      return 0;
+    }
+
     if (error_reply(downstream, 431) != 0) {
-      return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
+      return -1;
     }
 
     return 0;
@@ -1870,6 +2039,13 @@ int Http3Upstream::http_recv_request_header(Downstream *downstream,
   downstream->add_rcbuf(name);
   downstream->add_rcbuf(value);
 
+  if (trailer) {
+    req.fs.add_trailer_token(StringRef{namebuf.base, namebuf.len},
+                             StringRef{valuebuf.base, valuebuf.len}, no_index,
+                             token);
+    return 0;
+  }
+
   req.fs.add_header_token(StringRef{namebuf.base, namebuf.len},
                           StringRef{valuebuf.base, valuebuf.len}, no_index,
                           token);
@@ -1877,7 +2053,7 @@ int Http3Upstream::http_recv_request_header(Downstream *downstream,
 }
 
 namespace {
-int http_end_request_headers(nghttp3_conn *conn, int64_t stream_id,
+int http_end_request_headers(nghttp3_conn *conn, int64_t stream_id, int fin,
                              void *user_data, void *stream_user_data) {
   auto upstream = static_cast<Http3Upstream *>(user_data);
   auto handler = upstream->get_client_handler();
@@ -1887,7 +2063,7 @@ int http_end_request_headers(nghttp3_conn *conn, int64_t stream_id,
     return 0;
   }
 
-  if (upstream->http_end_request_headers(downstream) != 0) {
+  if (upstream->http_end_request_headers(downstream, fin) != 0) {
     return NGHTTP3_ERR_CALLBACK_FAILURE;
   }
 
@@ -1898,7 +2074,7 @@ int http_end_request_headers(nghttp3_conn *conn, int64_t stream_id,
 }
 } // namespace
 
-int Http3Upstream::http_end_request_headers(Downstream *downstream) {
+int Http3Upstream::http_end_request_headers(Downstream *downstream, int fin) {
   auto lgconf = log_config();
   lgconf->update_tstamp(std::chrono::system_clock::now());
   auto &req = downstream->request();
@@ -1939,7 +2115,7 @@ int Http3Upstream::http_end_request_headers(Downstream *downstream) {
   auto method_token = http2::lookup_method_token(method->value);
   if (method_token == -1) {
     if (error_reply(downstream, 501) != 0) {
-      return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
+      return -1;
     }
     return 0;
   }
@@ -1951,7 +2127,7 @@ int Http3Upstream::http_end_request_headers(Downstream *downstream) {
   // For HTTP/2 proxy, we require :authority.
   if (method_token != HTTP_CONNECT && config->http2_proxy &&
       faddr->alt_mode == UpstreamAltMode::NONE && !authority) {
-    shutdown_stream(downstream, NGHTTP2_PROTOCOL_ERROR);
+    shutdown_stream(downstream, NGHTTP3_H3_GENERAL_PROTOCOL_ERROR);
     return 0;
   }
 
@@ -1987,15 +2163,18 @@ int Http3Upstream::http_end_request_headers(Downstream *downstream) {
   if (connect_proto) {
     if (connect_proto->value != "websocket") {
       if (error_reply(downstream, 400) != 0) {
-        return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
+        return -1;
       }
       return 0;
     }
     req.connect_proto = ConnectProto::WEBSOCKET;
   }
 
-  // We are not sure that request has body or not at the moment.
+  if (!fin) {
-  req.http2_expect_body = true;
+    req.http2_expect_body = true;
+  } else if (req.fs.content_length == -1) {
+    req.fs.content_length = 0;
+  }
 
   downstream->inspect_http2_request();
 
@@ -2009,7 +2188,7 @@ int Http3Upstream::http_end_request_headers(Downstream *downstream) {
 
   if (mruby_ctx->run_on_request_proc(downstream) != 0) {
     if (error_reply(downstream, 500) != 0) {
-      return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
+      return -1;
     }
     return 0;
   }
@@ -2162,7 +2341,7 @@ int Http3Upstream::http_end_stream(Downstream *downstream) {
 
   if (downstream->end_upload_data() != 0) {
     if (downstream->get_response_state() != DownstreamState::MSG_COMPLETE) {
-      shutdown_stream(downstream, NGHTTP2_INTERNAL_ERROR);
+      shutdown_stream(downstream, NGHTTP3_H3_INTERNAL_ERROR);
     }
   }
 
@@ -2238,12 +2417,12 @@ int Http3Upstream::http_stream_close(Downstream *downstream,
 }
 
 namespace {
-int http_send_stop_sending(nghttp3_conn *conn, int64_t stream_id,
+int http_stop_sending(nghttp3_conn *conn, int64_t stream_id,
-                           uint64_t app_error_code, void *user_data,
+                      uint64_t app_error_code, void *user_data,
-                           void *stream_user_data) {
+                      void *stream_user_data) {
   auto upstream = static_cast<Http3Upstream *>(user_data);
 
-  if (upstream->http_send_stop_sending(stream_id, app_error_code) != 0) {
+  if (upstream->http_stop_sending(stream_id, app_error_code) != 0) {
     return NGHTTP3_ERR_CALLBACK_FAILURE;
   }
 
@@ -2251,8 +2430,8 @@ int http_send_stop_sending(nghttp3_conn *conn, int64_t stream_id,
 }
 } // namespace
 
-int Http3Upstream::http_send_stop_sending(int64_t stream_id,
+int Http3Upstream::http_stop_sending(int64_t stream_id,
-                                          uint64_t app_error_code) {
+                                     uint64_t app_error_code) {
   auto rv = ngtcp2_conn_shutdown_stream_read(conn_, stream_id, app_error_code);
   if (ngtcp2_err_is_fatal(rv)) {
     ULOG(ERROR, this) << "ngtcp2_conn_shutdown_stream_read: "
@@ -2305,9 +2484,9 @@ int Http3Upstream::setup_httpconn() {
       shrpx::http_recv_request_header,
       shrpx::http_end_request_headers,
       nullptr, // begin_trailers
-      nullptr, // recv_trailer
+      shrpx::http_recv_request_trailer,
       nullptr, // end_trailers
-      shrpx::http_send_stop_sending,
+      shrpx::http_stop_sending,
       shrpx::http_end_stream,
       shrpx::http_reset_stream,
   };
@@ -2316,7 +2495,7 @@ int Http3Upstream::setup_httpconn() {
 
   nghttp3_settings settings;
   nghttp3_settings_default(&settings);
-  settings.qpack_max_table_capacity = 4_k;
+  settings.qpack_max_dtable_capacity = 4_k;
 
   if (!config->http2_proxy) {
     settings.enable_connect_protocol = 1;
@@ -2589,7 +2768,7 @@ int Http3Upstream::open_qlog_file(const StringRef &dir,
       util::format_iso8601_basic(buf.data(), std::chrono::system_clock::now());
   path += '-';
   path += util::format_hex(scid.data, scid.datalen);
-  path += ".qlog";
+  path += ".sqlog";
 
   int fd;
 

src/shrpx_http3_upstream.h

@@ -92,7 +92,8 @@ public:
            const ngtcp2_cid *odcid, const uint8_t *token, size_t tokenlen);
 
   int on_read(const UpstreamAddr *faddr, const Address &remote_addr,
-              const Address &local_addr, const uint8_t *data, size_t datalen);
+              const Address &local_addr, const ngtcp2_pkt_info &pi,
+              const uint8_t *data, size_t datalen);
 
   int write_streams();
 
@@ -123,8 +124,8 @@ public:
   void http_begin_request_headers(int64_t stream_id);
   int http_recv_request_header(Downstream *downstream, int32_t token,
                                nghttp3_rcbuf *name, nghttp3_rcbuf *value,
-                               uint8_t flags);
+                               uint8_t flags, bool trailer);
-  int http_end_request_headers(Downstream *downstream);
+  int http_end_request_headers(Downstream *downstream, int fin);
   int http_end_stream(Downstream *downstream);
   void start_downstream(Downstream *downstream);
   void initiate_downstream(Downstream *downstream);
@@ -137,10 +138,10 @@ public:
   int stream_close(int64_t stream_id, uint64_t app_error_code);
   void log_response_headers(Downstream *downstream,
                             const std::vector<nghttp3_nv> &nva) const;
-  int http_acked_stream_data(Downstream *downstream, size_t datalen);
+  int http_acked_stream_data(Downstream *downstream, uint64_t datalen);
   int http_shutdown_stream_read(int64_t stream_id);
   int http_reset_stream(int64_t stream_id, uint64_t app_error_code);
-  int http_send_stop_sending(int64_t stream_id, uint64_t app_error_code);
+  int http_stop_sending(int64_t stream_id, uint64_t app_error_code);
   int http_recv_data(Downstream *downstream, const uint8_t *data,
                      size_t datalen);
   int handshake_completed();
@@ -148,6 +149,10 @@ public:
   int start_graceful_shutdown();
   int submit_goaway();
   void idle_close();
+  int send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
+                  size_t remote_salen, const sockaddr *local_sa,
+                  size_t local_salen, const ngtcp2_pkt_info &pi,
+                  const uint8_t *data, size_t datalen, size_t gso_size);
 
   void qlog_write(const void *data, size_t datalen, bool fin);
   int open_qlog_file(const StringRef &dir, const ngtcp2_cid &scid) const;
@@ -158,13 +163,16 @@ private:
   ev_timer idle_timer_;
   ev_timer shutdown_timer_;
   ev_prepare prep_;
+  size_t max_udp_payload_size_;
   int qlog_fd_;
+  ngtcp2_cid hashed_scid_;
   ngtcp2_conn *conn_;
   quic::Error last_error_;
   uint8_t tls_alert_;
   nghttp3_conn *httpconn_;
   DownstreamQueue downstream_queue_;
   bool idle_close_;
+  bool retry_close_;
   std::vector<uint8_t> conn_close_;
 };
 

src/shrpx_http_downstream_connection.cc

@@ -1218,6 +1218,18 @@ int HttpDownstreamConnection::read_clear() {
     }
 
     if (nread < 0) {
+      if (nread == SHRPX_ERR_EOF && !downstream_->get_upgraded()) {
+        auto htperr = llhttp_finish(&response_htp_);
+        if (htperr != HPE_OK) {
+          if (LOG_ENABLED(INFO)) {
+            DCLOG(INFO, this) << "HTTP response ended prematurely: "
+                              << llhttp_errno_name(htperr);
+          }
+
+          return -1;
+        }
+      }
+
       return nread;
     }
 
@@ -1337,6 +1349,18 @@ int HttpDownstreamConnection::read_tls() {
     }
 
     if (nread < 0) {
+      if (nread == SHRPX_ERR_EOF && !downstream_->get_upgraded()) {
+        auto htperr = llhttp_finish(&response_htp_);
+        if (htperr != HPE_OK) {
+          if (LOG_ENABLED(INFO)) {
+            DCLOG(INFO, this) << "HTTP response ended prematurely: "
+                              << llhttp_errno_name(htperr);
+          }
+
+          return -1;
+        }
+      }
+
       return nread;
     }
 

src/shrpx_log.cc

@@ -755,7 +755,11 @@ void upstream_accesslog(const std::vector<LogFragment> &lfv,
         std::tie(p, last) = copy('-', p, last);
         break;
       }
+#if OPENSSL_3_0_0_API
+      auto x = SSL_get0_peer_certificate(lgsp.ssl);
+#else  // !OPENSSL_3_0_0_API
       auto x = SSL_get_peer_certificate(lgsp.ssl);
+#endif // !OPENSSL_3_0_0_API
       if (!x) {
         std::tie(p, last) = copy('-', p, last);
         break;
@@ -766,7 +770,9 @@ void upstream_accesslog(const std::vector<LogFragment> &lfv,
           lf.type == LogFragmentType::TLS_CLIENT_FINGERPRINT_SHA256
               ? EVP_sha256()
               : EVP_sha1());
+#if !OPENSSL_3_0_0_API
       X509_free(x);
+#endif // !OPENSSL_3_0_0_API
       if (len <= 0) {
         std::tie(p, last) = copy('-', p, last);
         break;
@@ -780,7 +786,11 @@ void upstream_accesslog(const std::vector<LogFragment> &lfv,
         std::tie(p, last) = copy('-', p, last);
         break;
       }
+#if OPENSSL_3_0_0_API
+      auto x = SSL_get0_peer_certificate(lgsp.ssl);
+#else  // !OPENSSL_3_0_0_API
       auto x = SSL_get_peer_certificate(lgsp.ssl);
+#endif // !OPENSSL_3_0_0_API
       if (!x) {
         std::tie(p, last) = copy('-', p, last);
         break;
@@ -788,7 +798,9 @@ void upstream_accesslog(const std::vector<LogFragment> &lfv,
       auto name = lf.type == LogFragmentType::TLS_CLIENT_ISSUER_NAME
                       ? tls::get_x509_issuer_name(balloc, x)
                       : tls::get_x509_subject_name(balloc, x);
+#if !OPENSSL_3_0_0_API
       X509_free(x);
+#endif // !OPENSSL_3_0_0_API
       if (name.empty()) {
         std::tie(p, last) = copy('-', p, last);
         break;
@@ -801,13 +813,19 @@ void upstream_accesslog(const std::vector<LogFragment> &lfv,
         std::tie(p, last) = copy('-', p, last);
         break;
       }
+#if OPENSSL_3_0_0_API
+      auto x = SSL_get0_peer_certificate(lgsp.ssl);
+#else  // !OPENSSL_3_0_0_API
       auto x = SSL_get_peer_certificate(lgsp.ssl);
+#endif // !OPENSSL_3_0_0_API
       if (!x) {
         std::tie(p, last) = copy('-', p, last);
         break;
       }
       auto sn = tls::get_x509_serial(balloc, x);
+#if !OPENSSL_3_0_0_API
       X509_free(x);
+#endif // !OPENSSL_3_0_0_API
       if (sn.empty()) {
         std::tie(p, last) = copy('-', p, last);
         break;
@@ -964,7 +982,7 @@ int open_log_file(const char *path) {
       strcmp(path, "/proc/self/fd/2") == 0) {
     return STDERR_COPY;
   }
-#if defined O_CLOEXEC
+#ifdef O_CLOEXEC
 
   auto fd = open(path, O_WRONLY | O_APPEND | O_CREAT | O_CLOEXEC,
                  S_IRUSR | S_IWUSR | S_IRGRP);

src/shrpx_mruby.cc

@@ -75,6 +75,7 @@ int MRubyContext::run_app(Downstream *downstream, int phase) {
     break;
   default:
     assert(0);
+    abort();
   }
 
   auto res = mrb_funcall(mrb_, app_, method, 1, env_);

src/shrpx_mruby_module_env.cc

@@ -153,7 +153,11 @@ mrb_value env_get_tls_client_fingerprint_md(mrb_state *mrb, const EVP_MD *md) {
     return mrb_str_new_static(mrb, "", 0);
   }
 
+#if OPENSSL_3_0_0_API
+  auto x = SSL_get0_peer_certificate(ssl);
+#else  // !OPENSSL_3_0_0_API
   auto x = SSL_get_peer_certificate(ssl);
+#endif // !OPENSSL_3_0_0_API
   if (!x) {
     return mrb_str_new_static(mrb, "", 0);
   }
@@ -161,7 +165,9 @@ mrb_value env_get_tls_client_fingerprint_md(mrb_state *mrb, const EVP_MD *md) {
   // Currently the largest hash value is SHA-256, which is 32 bytes.
   std::array<uint8_t, 32> buf;
   auto slen = tls::get_x509_fingerprint(buf.data(), buf.size(), x, md);
+#if !OPENSSL_3_0_0_API
   X509_free(x);
+#endif // !OPENSSL_3_0_0_API
   if (slen == -1) {
     mrb_raise(mrb, E_RUNTIME_ERROR, "could not compute client fingerprint");
   }
@@ -199,14 +205,20 @@ mrb_value env_get_tls_client_subject_name(mrb_state *mrb, mrb_value self) {
     return mrb_str_new_static(mrb, "", 0);
   }
 
+#if OPENSSL_3_0_0_API
+  auto x = SSL_get0_peer_certificate(ssl);
+#else  // !OPENSSL_3_0_0_API
   auto x = SSL_get_peer_certificate(ssl);
+#endif // !OPENSSL_3_0_0_API
   if (!x) {
     return mrb_str_new_static(mrb, "", 0);
   }
 
   auto &balloc = downstream->get_block_allocator();
   auto name = tls::get_x509_subject_name(balloc, x);
+#if !OPENSSL_3_0_0_API
   X509_free(x);
+#endif // !OPENSSL_3_0_0_API
   return mrb_str_new(mrb, name.c_str(), name.size());
 }
 } // namespace
@@ -223,14 +235,20 @@ mrb_value env_get_tls_client_issuer_name(mrb_state *mrb, mrb_value self) {
     return mrb_str_new_static(mrb, "", 0);
   }
 
+#if OPENSSL_3_0_0_API
+  auto x = SSL_get0_peer_certificate(ssl);
+#else  // !OPENSSL_3_0_0_API
   auto x = SSL_get_peer_certificate(ssl);
+#endif // !OPENSSL_3_0_0_API
   if (!x) {
     return mrb_str_new_static(mrb, "", 0);
   }
 
   auto &balloc = downstream->get_block_allocator();
   auto name = tls::get_x509_issuer_name(balloc, x);
+#if !OPENSSL_3_0_0_API
   X509_free(x);
+#endif // !OPENSSL_3_0_0_API
   return mrb_str_new(mrb, name.c_str(), name.size());
 }
 } // namespace
@@ -247,14 +265,20 @@ mrb_value env_get_tls_client_serial(mrb_state *mrb, mrb_value self) {
     return mrb_str_new_static(mrb, "", 0);
   }
 
+#if OPENSSL_3_0_0_API
+  auto x = SSL_get0_peer_certificate(ssl);
+#else  // !OPENSSL_3_0_0_API
   auto x = SSL_get_peer_certificate(ssl);
+#endif // !OPENSSL_3_0_0_API
   if (!x) {
     return mrb_str_new_static(mrb, "", 0);
   }
 
   auto &balloc = downstream->get_block_allocator();
   auto sn = tls::get_x509_serial(balloc, x);
+#if !OPENSSL_3_0_0_API
   X509_free(x);
+#endif // !OPENSSL_3_0_0_API
   return mrb_str_new(mrb, sn.c_str(), sn.size());
 }
 } // namespace
@@ -271,16 +295,24 @@ mrb_value env_get_tls_client_not_before(mrb_state *mrb, mrb_value self) {
     return mrb_fixnum_value(0);
   }
 
+#if OPENSSL_3_0_0_API
+  auto x = SSL_get0_peer_certificate(ssl);
+#else  // !OPENSSL_3_0_0_API
   auto x = SSL_get_peer_certificate(ssl);
+#endif // !OPENSSL_3_0_0_API
   if (!x) {
     return mrb_fixnum_value(0);
   }
 
   time_t t;
   if (tls::get_x509_not_before(t, x) != 0) {
-    return mrb_fixnum_value(0);
+    t = 0;
   }
 
+#if !OPENSSL_3_0_0_API
+  X509_free(x);
+#endif // !OPENSSL_3_0_0_API
+
   return mrb_fixnum_value(t);
 }
 } // namespace
@@ -297,16 +329,24 @@ mrb_value env_get_tls_client_not_after(mrb_state *mrb, mrb_value self) {
     return mrb_fixnum_value(0);
   }
 
+#if OPENSSL_3_0_0_API
+  auto x = SSL_get0_peer_certificate(ssl);
+#else  // !OPENSSL_3_0_0_API
   auto x = SSL_get_peer_certificate(ssl);
+#endif // !OPENSSL_3_0_0_API
   if (!x) {
     return mrb_fixnum_value(0);
   }
 
   time_t t;
   if (tls::get_x509_not_after(t, x) != 0) {
-    return mrb_fixnum_value(0);
+    t = 0;
   }
 
+#if !OPENSSL_3_0_0_API
+  X509_free(x);
+#endif // !OPENSSL_3_0_0_API
+
   return mrb_fixnum_value(t);
 }
 } // namespace

src/shrpx_null_downstream_connection.cc

@@ -74,7 +74,7 @@ int NullDownstreamConnection::on_read() { return 0; }
 
 int NullDownstreamConnection::on_write() { return 0; }
 
-void NullDownstreamConnection::on_upstream_change(Upstream *uptream) {}
+void NullDownstreamConnection::on_upstream_change(Upstream *upstream) {}
 
 bool NullDownstreamConnection::poolable() const { return false; }
 

src/shrpx_null_downstream_connection.h

@@ -50,7 +50,7 @@ public:
   virtual int on_read();
   virtual int on_write();
 
-  virtual void on_upstream_change(Upstream *uptream);
+  virtual void on_upstream_change(Upstream *upstream);
 
   // true if this object is poolable.
   virtual bool poolable() const;

src/shrpx_quic.cc

@@ -43,8 +43,6 @@
 #include "util.h"
 #include "xsi_strerror.h"
 
-using namespace nghttp2;
-
 bool operator==(const ngtcp2_cid &lhs, const ngtcp2_cid &rhs) {
   return ngtcp2_cid_eq(&lhs, &rhs);
 }
@@ -59,8 +57,8 @@ ngtcp2_tstamp quic_timestamp() {
 
 int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
                      size_t remote_salen, const sockaddr *local_sa,
-                     size_t local_salen, const uint8_t *data, size_t datalen,
+                     size_t local_salen, const ngtcp2_pkt_info &pi,
-                     size_t gso_size) {
+                     const uint8_t *data, size_t datalen, size_t gso_size) {
   iovec msg_iov = {const_cast<uint8_t *>(data), datalen};
   msghdr msg{};
   msg.msg_name = const_cast<sockaddr *>(remote_sa);
@@ -125,6 +123,8 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
 
   msg.msg_controllen = controllen;
 
+  util::fd_set_send_ecn(faddr->fd, local_sa->sa_family, pi.ecn);
+
   ssize_t nwrite;
 
   do {
@@ -132,65 +132,138 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
   } while (nwrite == -1 && errno == EINTR);
 
   if (nwrite == -1) {
-    return -1;
+    if (LOG_ENABLED(INFO)) {
+      auto error = errno;
+      LOG(INFO) << "sendmsg failed: errno=" << error;
+    }
+
+    return -errno;
   }
 
   if (LOG_ENABLED(INFO)) {
     LOG(INFO) << "QUIC sent packet: local="
               << util::to_numeric_addr(local_sa, local_salen)
               << " remote=" << util::to_numeric_addr(remote_sa, remote_salen)
-              << " " << nwrite << " bytes";
+              << " ecn=" << log::hex << pi.ecn << log::dec << " " << nwrite
+              << " bytes";
   }
 
   return 0;
 }
 
-int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen) {
+int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen,
-  if (RAND_bytes(cid->data, cidlen) != 1) {
+                                      const uint8_t *server_id, uint8_t km_id,
+                                      const uint8_t *key) {
+  assert(cidlen == SHRPX_QUIC_SCIDLEN);
+
+  if (RAND_bytes(cid.data, cidlen) != 1) {
     return -1;
   }
 
-  cid->datalen = cidlen;
+  cid.datalen = cidlen;
 
-  return 0;
+  cid.data[0] = (cid.data[0] & 0x3f) | km_id;
+
+  auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET;
+
+  std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, p);
+
+  return encrypt_quic_connection_id(p, p, key);
 }
 
-int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen,
+int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen,
-                                const uint8_t *cid_prefix) {
+                                const uint8_t *cid_prefix, uint8_t km_id,
-  assert(cidlen > SHRPX_QUIC_CID_PREFIXLEN);
+                                const uint8_t *key) {
+  assert(cidlen == SHRPX_QUIC_SCIDLEN);
 
-  auto p = std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, cid->data);
+  if (RAND_bytes(cid.data, cidlen) != 1) {
-
-  if (RAND_bytes(p, cidlen - SHRPX_QUIC_CID_PREFIXLEN) != 1) {
     return -1;
   }
 
-  cid->datalen = cidlen;
+  cid.datalen = cidlen;
+
+  cid.data[0] = (cid.data[0] & 0x3f) | km_id;
+
+  auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET;
+
+  std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, p);
+
+  return encrypt_quic_connection_id(p, p, key);
+}
+
+int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src,
+                               const uint8_t *key) {
+  auto ctx = EVP_CIPHER_CTX_new();
+  auto d = defer(EVP_CIPHER_CTX_free, ctx);
+
+  if (!EVP_EncryptInit_ex(ctx, EVP_aes_128_ecb(), nullptr, key, nullptr)) {
+    return -1;
+  }
+
+  EVP_CIPHER_CTX_set_padding(ctx, 0);
+
+  int len;
+
+  if (!EVP_EncryptUpdate(ctx, dest, &len, src, SHRPX_QUIC_DECRYPTED_DCIDLEN) ||
+      !EVP_EncryptFinal_ex(ctx, dest + len, &len)) {
+    return -1;
+  }
 
   return 0;
 }
 
-int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid,
+int decrypt_quic_connection_id(uint8_t *dest, const uint8_t *src,
+                               const uint8_t *key) {
+  auto ctx = EVP_CIPHER_CTX_new();
+  auto d = defer(EVP_CIPHER_CTX_free, ctx);
+
+  if (!EVP_DecryptInit_ex(ctx, EVP_aes_128_ecb(), nullptr, key, nullptr)) {
+    return -1;
+  }
+
+  EVP_CIPHER_CTX_set_padding(ctx, 0);
+
+  int len;
+
+  if (!EVP_DecryptUpdate(ctx, dest, &len, src, SHRPX_QUIC_DECRYPTED_DCIDLEN) ||
+      !EVP_DecryptFinal_ex(ctx, dest + len, &len)) {
+    return -1;
+  }
+
+  return 0;
+}
+
+int generate_quic_hashed_connection_id(ngtcp2_cid &dest,
+                                       const Address &remote_addr,
+                                       const Address &local_addr,
+                                       const ngtcp2_cid &cid) {
+  auto ctx = EVP_MD_CTX_new();
+  auto d = defer(EVP_MD_CTX_free, ctx);
+
+  std::array<uint8_t, 32> h;
+  unsigned int hlen = EVP_MD_size(EVP_sha256());
+
+  if (!EVP_DigestInit_ex(ctx, EVP_sha256(), nullptr) ||
+      !EVP_DigestUpdate(ctx, &remote_addr.su.sa, remote_addr.len) ||
+      !EVP_DigestUpdate(ctx, &local_addr.su.sa, local_addr.len) ||
+      !EVP_DigestUpdate(ctx, cid.data, cid.datalen) ||
+      !EVP_DigestFinal_ex(ctx, h.data(), &hlen)) {
+    return -1;
+  }
+
+  assert(hlen == h.size());
+
+  std::copy_n(std::begin(h), sizeof(dest.data), std::begin(dest.data));
+  dest.datalen = sizeof(dest.data);
+
+  return 0;
+}
+
+int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid &cid,
                                         const uint8_t *secret,
                                         size_t secretlen) {
   if (ngtcp2_crypto_generate_stateless_reset_token(token, secret, secretlen,
-                                                   cid) != 0) {
+                                                   &cid) != 0) {
-    return -1;
-  }
-
-  return 0;
-}
-
-int generate_quic_stateless_reset_secret(uint8_t *secret) {
-  if (RAND_bytes(secret, SHRPX_QUIC_STATELESS_RESET_SECRETLEN) != 1) {
-    return -1;
-  }
-
-  return 0;
-}
-
-int generate_quic_token_secret(uint8_t *secret) {
-  if (RAND_bytes(secret, SHRPX_QUIC_TOKEN_SECRETLEN) != 1) {
     return -1;
   }
 
@@ -198,15 +271,15 @@ int generate_quic_token_secret(uint8_t *secret) {
 }
 
 int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
-                         socklen_t salen, const ngtcp2_cid *retry_scid,
+                         socklen_t salen, const ngtcp2_cid &retry_scid,
-                         const ngtcp2_cid *odcid, const uint8_t *token_secret) {
+                         const ngtcp2_cid &odcid, const uint8_t *secret,
+                         size_t secretlen) {
   auto t = std::chrono::duration_cast<std::chrono::nanoseconds>(
                std::chrono::system_clock::now().time_since_epoch())
                .count();
 
   auto stokenlen = ngtcp2_crypto_generate_retry_token(
-      token, token_secret, SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, retry_scid,
+      token, secret, secretlen, sa, salen, &retry_scid, &odcid, t);
-      odcid, t);
   if (stokenlen < 0) {
     return -1;
   }
@@ -216,17 +289,18 @@ int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
   return 0;
 }
 
-int verify_retry_token(ngtcp2_cid *odcid, const uint8_t *token, size_t tokenlen,
+int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen,
-                       const ngtcp2_cid *dcid, const sockaddr *sa,
+                       const ngtcp2_cid &dcid, const sockaddr *sa,
-                       socklen_t salen, const uint8_t *token_secret) {
+                       socklen_t salen, const uint8_t *secret,
+                       size_t secretlen) {
 
   auto t = std::chrono::duration_cast<std::chrono::nanoseconds>(
                std::chrono::system_clock::now().time_since_epoch())
                .count();
 
-  if (ngtcp2_crypto_verify_retry_token(odcid, token, tokenlen, token_secret,
+  if (ngtcp2_crypto_verify_retry_token(&odcid, token, tokenlen, secret,
-                                       SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen,
+                                       secretlen, sa, salen, &dcid,
-                                       dcid, 10 * NGTCP2_SECONDS, t) != 0) {
+                                       10 * NGTCP2_SECONDS, t) != 0) {
     return -1;
   }
 
@@ -234,13 +308,13 @@ int verify_retry_token(ngtcp2_cid *odcid, const uint8_t *token, size_t tokenlen,
 }
 
 int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
-                   size_t salen, const uint8_t *token_secret) {
+                   size_t salen, const uint8_t *secret, size_t secretlen) {
   auto t = std::chrono::duration_cast<std::chrono::nanoseconds>(
                std::chrono::system_clock::now().time_since_epoch())
                .count();
 
   auto stokenlen = ngtcp2_crypto_generate_regular_token(
-      token, token_secret, SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, t);
+      token, secret, secretlen, sa, salen, t);
   if (stokenlen < 0) {
     return -1;
   }
@@ -251,18 +325,48 @@ int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
 }
 
 int verify_token(const uint8_t *token, size_t tokenlen, const sockaddr *sa,
-                 socklen_t salen, const uint8_t *token_secret) {
+                 socklen_t salen, const uint8_t *secret, size_t secretlen) {
   auto t = std::chrono::duration_cast<std::chrono::nanoseconds>(
                std::chrono::system_clock::now().time_since_epoch())
                .count();
 
-  if (ngtcp2_crypto_verify_regular_token(token, tokenlen, token_secret,
+  if (ngtcp2_crypto_verify_regular_token(token, tokenlen, secret, secretlen, sa,
-                                         SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen,
+                                         salen, 3600 * NGTCP2_SECONDS,
-                                         3600 * NGTCP2_SECONDS, t) != 0) {
+                                         t) != 0) {
     return -1;
   }
 
   return 0;
 }
 
+int generate_quic_connection_id_encryption_key(uint8_t *key, size_t keylen,
+                                               const uint8_t *secret,
+                                               size_t secretlen,
+                                               const uint8_t *salt,
+                                               size_t saltlen) {
+  constexpr uint8_t info[] = "connection id encryption key";
+  ngtcp2_crypto_md sha256;
+  ngtcp2_crypto_md_init(
+      &sha256, reinterpret_cast<void *>(const_cast<EVP_MD *>(EVP_sha256())));
+
+  if (ngtcp2_crypto_hkdf(key, keylen, &sha256, secret, secretlen, salt, saltlen,
+                         info, str_size(info)) != 0) {
+    return -1;
+  }
+
+  return 0;
+}
+
+const QUICKeyingMaterial *
+select_quic_keying_material(const QUICKeyingMaterials &qkms,
+                            const uint8_t *cid) {
+  for (auto &qkm : qkms.keying_materials) {
+    if (((*cid) & 0xc0) == qkm.id) {
+      return &qkm;
+    }
+  }
+
+  return &qkms.keying_materials.front();
+}
+
 } // namespace shrpx

src/shrpx_quic.h

@@ -33,6 +33,10 @@
 
 #include <ngtcp2/ngtcp2.h>
 
+#include "network.h"
+
+using namespace nghttp2;
+
 namespace std {
 template <> struct hash<ngtcp2_cid> {
   std::size_t operator()(const ngtcp2_cid &cid) const noexcept {
@@ -56,48 +60,78 @@ bool operator==(const ngtcp2_cid &lhs, const ngtcp2_cid &rhs);
 namespace shrpx {
 
 struct UpstreamAddr;
+struct QUICKeyingMaterials;
+struct QUICKeyingMaterial;
 
 constexpr size_t SHRPX_QUIC_SCIDLEN = 20;
+constexpr size_t SHRPX_QUIC_SERVER_IDLEN = 4;
+// SHRPX_QUIC_CID_PREFIXLEN includes SHRPX_QUIC_SERVER_IDLEN.
 constexpr size_t SHRPX_QUIC_CID_PREFIXLEN = 8;
+constexpr size_t SHRPX_QUIC_CID_PREFIX_OFFSET = 1;
+constexpr size_t SHRPX_QUIC_DECRYPTED_DCIDLEN = 16;
+constexpr size_t SHRPX_QUIC_CID_ENCRYPTION_KEYLEN = 16;
 constexpr size_t SHRPX_QUIC_MAX_UDP_PAYLOAD_SIZE = 1472;
-constexpr size_t SHRPX_QUIC_STATELESS_RESET_SECRETLEN = 32;
-constexpr size_t SHRPX_QUIC_TOKEN_SECRETLEN = 32;
 constexpr size_t SHRPX_QUIC_CONN_CLOSE_PKTLEN = 256;
 constexpr size_t SHRPX_QUIC_STATELESS_RESET_BURST = 100;
+constexpr size_t SHRPX_QUIC_SECRET_RESERVEDLEN = 4;
+constexpr size_t SHRPX_QUIC_SECRETLEN = 32;
+constexpr size_t SHRPX_QUIC_SALTLEN = 32;
 
 ngtcp2_tstamp quic_timestamp();
 
 int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
                      size_t remote_salen, const sockaddr *local_sa,
-                     size_t local_salen, const uint8_t *data, size_t datalen,
+                     size_t local_salen, const ngtcp2_pkt_info &pi,
-                     size_t gso_size);
+                     const uint8_t *data, size_t datalen, size_t gso_size);
 
-int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen);
+int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen,
+                                      const uint8_t *server_id, uint8_t km_id,
+                                      const uint8_t *key);
 
-int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen,
+int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen,
-                                const uint8_t *cid_prefix);
+                                const uint8_t *cid_prefix, uint8_t km_id,
+                                const uint8_t *key);
 
-int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid,
+int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src,
+                               const uint8_t *key);
+
+int decrypt_quic_connection_id(uint8_t *dest, const uint8_t *src,
+                               const uint8_t *key);
+
+int generate_quic_hashed_connection_id(ngtcp2_cid &dest,
+                                       const Address &remote_addr,
+                                       const Address &local_addr,
+                                       const ngtcp2_cid &cid);
+
+int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid &cid,
                                         const uint8_t *secret,
                                         size_t secretlen);
 
-int generate_quic_stateless_reset_secret(uint8_t *secret);
-
-int generate_quic_token_secret(uint8_t *secret);
-
 int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
-                         socklen_t salen, const ngtcp2_cid *retry_scid,
+                         socklen_t salen, const ngtcp2_cid &retry_scid,
-                         const ngtcp2_cid *odcid, const uint8_t *token_secret);
+                         const ngtcp2_cid &odcid, const uint8_t *secret,
+                         size_t secretlen);
 
-int verify_retry_token(ngtcp2_cid *odcid, const uint8_t *token, size_t tokenlen,
+int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen,
-                       const ngtcp2_cid *dcid, const sockaddr *sa,
+                       const ngtcp2_cid &dcid, const sockaddr *sa,
-                       socklen_t salen, const uint8_t *token_secret);
+                       socklen_t salen, const uint8_t *secret,
+                       size_t secretlen);
 
 int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
-                   size_t salen, const uint8_t *token_secret);
+                   size_t salen, const uint8_t *secret, size_t secretlen);
 
 int verify_token(const uint8_t *token, size_t tokenlen, const sockaddr *sa,
-                 socklen_t salen, const uint8_t *token_secret);
+                 socklen_t salen, const uint8_t *secret, size_t secretlen);
+
+int generate_quic_connection_id_encryption_key(uint8_t *key, size_t keylen,
+                                               const uint8_t *secret,
+                                               size_t secretlen,
+                                               const uint8_t *salt,
+                                               size_t saltlen);
+
+const QUICKeyingMaterial *
+select_quic_keying_material(const QUICKeyingMaterials &qkms,
+                            const uint8_t *cid);
 
 } // namespace shrpx
 

src/shrpx_quic_connection_handler.cc

@@ -61,6 +61,7 @@ QUICConnectionHandler::~QUICConnectionHandler() {
 int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
                                          const Address &remote_addr,
                                          const Address &local_addr,
+                                         const ngtcp2_pkt_info &pi,
                                          const uint8_t *data, size_t datalen) {
   int rv;
   uint32_t version;
@@ -90,32 +91,72 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
 
   ClientHandler *handler;
 
+  auto &quicconf = config->quic;
+
   auto it = connections_.find(dcid_key);
   if (it == std::end(connections_)) {
-    if (!std::equal(dcid, dcid + SHRPX_QUIC_CID_PREFIXLEN,
+    auto cwit = close_waits_.find(dcid_key);
-                    worker_->get_cid_prefix())) {
+    if (cwit != std::end(close_waits_)) {
-      auto quic_lwp =
+      auto cw = (*cwit).second;
-          conn_handler->match_quic_lingering_worker_process_cid_prefix(dcid,
-                                                                       dcidlen);
-      if (quic_lwp) {
-        if (conn_handler->forward_quic_packet_to_lingering_worker_process(
-                quic_lwp, remote_addr, local_addr, data, datalen) == 0) {
-          return 0;
-        }
 
-        return 0;
+      cw->handle_packet(faddr, remote_addr, local_addr, pi, data, datalen);
-      }
-    }
-
-    auto it = close_waits_.find(dcid_key);
-    if (it != std::end(close_waits_)) {
-      auto cw = (*it).second;
-
-      cw->handle_packet(faddr, remote_addr, local_addr, data, datalen);
 
       return 0;
     }
 
+    if (data[0] & 0x80) {
+      if (generate_quic_hashed_connection_id(dcid_key, remote_addr, local_addr,
+                                             dcid_key) != 0) {
+        return 0;
+      }
+
+      it = connections_.find(dcid_key);
+      if (it == std::end(connections_)) {
+        auto cwit = close_waits_.find(dcid_key);
+        if (cwit != std::end(close_waits_)) {
+          auto cw = (*cwit).second;
+
+          cw->handle_packet(faddr, remote_addr, local_addr, pi, data, datalen);
+
+          return 0;
+        }
+      }
+    }
+  }
+
+  if (it == std::end(connections_)) {
+    std::array<uint8_t, SHRPX_QUIC_DECRYPTED_DCIDLEN> decrypted_dcid;
+
+    auto &qkms = conn_handler->get_quic_keying_materials();
+    const QUICKeyingMaterial *qkm = nullptr;
+
+    if (dcidlen == SHRPX_QUIC_SCIDLEN) {
+      qkm = select_quic_keying_material(*qkms.get(), dcid);
+
+      if (decrypt_quic_connection_id(decrypted_dcid.data(),
+                                     dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
+                                     qkm->cid_encryption_key.data()) != 0) {
+        return 0;
+      }
+
+      if (qkm != &qkms->keying_materials.front() ||
+          !std::equal(std::begin(decrypted_dcid),
+                      std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
+                      worker_->get_cid_prefix())) {
+        auto quic_lwp =
+            conn_handler->match_quic_lingering_worker_process_cid_prefix(
+                decrypted_dcid.data(), decrypted_dcid.size());
+        if (quic_lwp) {
+          if (conn_handler->forward_quic_packet_to_lingering_worker_process(
+                  quic_lwp, remote_addr, local_addr, pi, data, datalen) == 0) {
+            return 0;
+          }
+
+          return 0;
+        }
+      }
+    }
+
     // new connection
 
     auto &upstreamconf = config->conn.upstream;
@@ -136,26 +177,56 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
 
     switch (ngtcp2_accept(&hd, data, datalen)) {
     case 0: {
-      // If we get Initial and it has the CID prefix of this worker, it
+      // If we get Initial and it has the CID prefix of this worker,
-      // is likely that client is intentionally use the our prefix.
+      // it is likely that client is intentionally use the prefix.
       // Just drop it.
-      if (std::equal(dcid, dcid + SHRPX_QUIC_CID_PREFIXLEN,
+      if (dcidlen == SHRPX_QUIC_SCIDLEN) {
-                     worker_->get_cid_prefix())) {
+        if (qkm != &qkms->keying_materials.front()) {
+          qkm = &qkms->keying_materials.front();
+
+          if (decrypt_quic_connection_id(decrypted_dcid.data(),
+                                         dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
+                                         qkm->cid_encryption_key.data()) != 0) {
+            return 0;
+          }
+        }
+
+        if (std::equal(std::begin(decrypted_dcid),
+                       std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
+                       worker_->get_cid_prefix())) {
+          return 0;
+        }
+      }
+
+      if (worker_->get_graceful_shutdown()) {
+        send_connection_close(faddr, version, hd.dcid, hd.scid, remote_addr,
+                              local_addr, NGTCP2_CONNECTION_REFUSED);
         return 0;
       }
 
       if (hd.token.len == 0) {
+        if (quicconf.upstream.require_token) {
+          send_retry(faddr, version, dcid, dcidlen, scid, scidlen, remote_addr,
+                     local_addr);
+
+          return 0;
+        }
+
         break;
       }
 
-      auto &quic_secret = worker_->get_quic_secret();
+      if (dcidlen != SHRPX_QUIC_SCIDLEN) {
-      auto &secret = quic_secret->token_secret;
+        // Initial packets with token must have DCID chosen by server.
+        return 0;
+      }
+
+      auto qkm = select_quic_keying_material(*qkms.get(), dcid);
 
       switch (hd.token.base[0]) {
       case NGTCP2_CRYPTO_TOKEN_MAGIC_RETRY:
-        if (verify_retry_token(&odcid, hd.token.base, hd.token.len, &hd.dcid,
+        if (verify_retry_token(odcid, hd.token.base, hd.token.len, hd.dcid,
                                &remote_addr.su.sa, remote_addr.len,
-                               secret.data()) != 0) {
+                               qkm->secret.data(), qkm->secret.size()) != 0) {
           if (LOG_ENABLED(INFO)) {
             LOG(INFO) << "Failed to validate Retry token from remote="
                       << util::to_numeric_addr(&remote_addr);
@@ -163,7 +234,7 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
 
           // 2nd Retry packet is not allowed, so send CONNECTION_CLOSE
           // with INVALID_TOKEN.
-          send_connection_close(faddr, version, &hd.dcid, &hd.scid, remote_addr,
+          send_connection_close(faddr, version, hd.dcid, hd.scid, remote_addr,
                                 local_addr, NGTCP2_INVALID_TOKEN);
           return 0;
         }
@@ -180,12 +251,20 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
         break;
       case NGTCP2_CRYPTO_TOKEN_MAGIC_REGULAR:
         if (verify_token(hd.token.base, hd.token.len, &remote_addr.su.sa,
-                         remote_addr.len, secret.data()) != 0) {
+                         remote_addr.len, qkm->secret.data(),
+                         qkm->secret.size()) != 0) {
           if (LOG_ENABLED(INFO)) {
             LOG(INFO) << "Failed to validate token from remote="
                       << util::to_numeric_addr(&remote_addr);
           }
 
+          if (quicconf.upstream.require_token) {
+            send_retry(faddr, version, dcid, dcidlen, scid, scidlen,
+                       remote_addr, local_addr);
+
+            return 0;
+          }
+
           break;
         }
 
@@ -199,12 +278,25 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
 
         break;
       default:
+        if (quicconf.upstream.require_token) {
+          send_retry(faddr, version, dcid, dcidlen, scid, scidlen, remote_addr,
+                     local_addr);
+
+          return 0;
+        }
+
         break;
       }
 
       break;
     }
     case NGTCP2_ERR_RETRY:
+      if (worker_->get_graceful_shutdown()) {
+        send_connection_close(faddr, version, hd.dcid, hd.scid, remote_addr,
+                              local_addr, NGTCP2_CONNECTION_REFUSED);
+        return 0;
+      }
+
       send_retry(faddr, version, dcid, dcidlen, scid, scidlen, remote_addr,
                  local_addr);
       return 0;
@@ -214,11 +306,13 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
       return 0;
     default:
       if (!config->single_thread && !(data[0] & 0x80) &&
-          dcidlen > SHRPX_QUIC_CID_PREFIXLEN &&
+          dcidlen == SHRPX_QUIC_SCIDLEN &&
-          !std::equal(dcid, dcid + SHRPX_QUIC_CID_PREFIXLEN,
+          !std::equal(std::begin(decrypted_dcid),
+                      std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
                       worker_->get_cid_prefix())) {
         if (conn_handler->forward_quic_packet(faddr, remote_addr, local_addr,
-                                              dcid, data, datalen) == 0) {
+                                              pi, decrypted_dcid.data(), data,
+                                              datalen) == 0) {
           return 0;
         }
       }
@@ -240,7 +334,8 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
     handler = (*it).second;
   }
 
-  if (handler->read_quic(faddr, remote_addr, local_addr, data, datalen) != 0) {
+  if (handler->read_quic(faddr, remote_addr, local_addr, pi, data, datalen) !=
+      0) {
     delete handler;
     return 0;
   }
@@ -276,7 +371,9 @@ ClientHandler *QUICConnectionHandler::handle_new_connection(
     return nullptr;
   }
 
+#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
   assert(SSL_is_quic(ssl));
+#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
 
   SSL_set_accept_state(ssl);
 
@@ -284,7 +381,11 @@ ClientHandler *QUICConnectionHandler::handle_new_connection(
   auto &quicconf = config->quic;
 
   if (quicconf.upstream.early_data) {
+#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
     SSL_set_quic_early_data_enabled(ssl, 1);
+#else  // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
+    SSL_set_early_data_enabled(ssl, 1);
+#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
   }
 
   // Disable TLS session ticket if we don't have working ticket
@@ -348,11 +449,18 @@ int QUICConnectionHandler::send_retry(
     return -1;
   }
 
+  auto config = get_config();
+  auto &quicconf = config->quic;
+
+  auto conn_handler = worker_->get_connection_handler();
+  auto &qkms = conn_handler->get_quic_keying_materials();
+  auto &qkm = qkms->keying_materials.front();
+
   ngtcp2_cid retry_scid;
 
-  retry_scid.datalen = SHRPX_QUIC_SCIDLEN;
+  if (generate_quic_retry_connection_id(retry_scid, SHRPX_QUIC_SCIDLEN,
-  // We do not steer packet based on Retry CID.
+                                        quicconf.server_id.data(), qkm.id,
-  if (RAND_bytes(retry_scid.data, retry_scid.datalen) != 1) {
+                                        qkm.cid_encryption_key.data()) != 0) {
     return -1;
   }
 
@@ -363,16 +471,14 @@ int QUICConnectionHandler::send_retry(
   ngtcp2_cid_init(&idcid, ini_dcid, ini_dcidlen);
   ngtcp2_cid_init(&iscid, ini_scid, ini_scidlen);
 
-  auto &quic_secret = worker_->get_quic_secret();
-  auto &secret = quic_secret->token_secret;
-
   if (generate_retry_token(token.data(), tokenlen, &remote_addr.su.sa,
-                           remote_addr.len, &retry_scid, &idcid,
+                           remote_addr.len, retry_scid, idcid,
-                           secret.data()) != 0) {
+                           qkm.secret.data(), qkm.secret.size()) != 0) {
     return -1;
   }
 
-  std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf;
+  std::vector<uint8_t> buf;
+  buf.resize(256);
 
   auto nwrite =
       ngtcp2_crypto_write_retry(buf.data(), buf.size(), version, &iscid,
@@ -382,9 +488,33 @@ int QUICConnectionHandler::send_retry(
     return -1;
   }
 
-  return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
+  buf.resize(nwrite);
-                          &local_addr.su.sa, local_addr.len, buf.data(), nwrite,
+
-                          0);
+  quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
+                   &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
+                   buf.data(), buf.size(), 0);
+
+  if (generate_quic_hashed_connection_id(idcid, remote_addr, local_addr,
+                                         idcid) != 0) {
+    return -1;
+  }
+
+  auto d =
+      static_cast<ev_tstamp>(NGTCP2_DEFAULT_INITIAL_RTT * 3) / NGTCP2_SECONDS;
+
+  if (LOG_ENABLED(INFO)) {
+    LOG(INFO) << "Enter close-wait period " << d << "s with " << buf.size()
+              << " bytes sentinel packet";
+  }
+
+  auto cw = std::make_unique<CloseWait>(worker_, std::vector<ngtcp2_cid>{idcid},
+                                        std::move(buf), d);
+
+  add_close_wait(cw.get());
+
+  cw.release();
+
+  return 0;
 }
 
 int QUICConnectionHandler::send_version_negotiation(
@@ -411,8 +541,8 @@ int QUICConnectionHandler::send_version_negotiation(
   }
 
   return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
-                          &local_addr.su.sa, local_addr.len, buf.data(), nwrite,
+                          &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
-                          0);
+                          buf.data(), nwrite, 0);
 }
 
 int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr,
@@ -440,11 +570,12 @@ int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr,
 
   ngtcp2_cid_init(&cid, dcid, dcidlen);
 
-  auto &quic_secret = worker_->get_quic_secret();
+  auto conn_handler = worker_->get_connection_handler();
-  auto &secret = quic_secret->stateless_reset_secret;
+  auto &qkms = conn_handler->get_quic_keying_materials();
+  auto &qkm = qkms->keying_materials.front();
 
-  rv = generate_quic_stateless_reset_token(token.data(), &cid, secret.data(),
+  rv = generate_quic_stateless_reset_token(token.data(), cid, qkm.secret.data(),
-                                           secret.size());
+                                           qkm.secret.size());
   if (rv != 0) {
     return -1;
   }
@@ -473,18 +604,19 @@ int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr,
   }
 
   return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
-                          &local_addr.su.sa, local_addr.len, buf.data(), nwrite,
+                          &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
-                          0);
+                          buf.data(), nwrite, 0);
 }
 
 int QUICConnectionHandler::send_connection_close(
-    const UpstreamAddr *faddr, uint32_t version, const ngtcp2_cid *ini_dcid,
+    const UpstreamAddr *faddr, uint32_t version, const ngtcp2_cid &ini_dcid,
-    const ngtcp2_cid *ini_scid, const Address &remote_addr,
+    const ngtcp2_cid &ini_scid, const Address &remote_addr,
     const Address &local_addr, uint64_t error_code) {
   std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf;
 
   auto nwrite = ngtcp2_crypto_write_connection_close(
-      buf.data(), buf.size(), version, ini_scid, ini_dcid, error_code);
+      buf.data(), buf.size(), version, &ini_scid, &ini_dcid, error_code,
+      nullptr, 0);
   if (nwrite < 0) {
     LOG(ERROR) << "ngtcp2_crypto_write_connection_close failed";
     return -1;
@@ -494,23 +626,22 @@ int QUICConnectionHandler::send_connection_close(
     LOG(INFO) << "Send Initial CONNECTION_CLOSE with error_code=" << log::hex
               << error_code << log::dec
               << " to remote=" << util::to_numeric_addr(&remote_addr)
-              << " dcid=" << util::format_hex(ini_scid->data, ini_scid->datalen)
+              << " dcid=" << util::format_hex(ini_scid.data, ini_scid.datalen)
-              << " scid="
+              << " scid=" << util::format_hex(ini_dcid.data, ini_dcid.datalen);
-              << util::format_hex(ini_dcid->data, ini_dcid->datalen);
   }
 
   return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
-                          &local_addr.su.sa, local_addr.len, buf.data(), nwrite,
+                          &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
-                          0);
+                          buf.data(), nwrite, 0);
 }
 
-void QUICConnectionHandler::add_connection_id(const ngtcp2_cid *cid,
+void QUICConnectionHandler::add_connection_id(const ngtcp2_cid &cid,
                                               ClientHandler *handler) {
-  connections_.emplace(*cid, handler);
+  connections_.emplace(cid, handler);
 }
 
-void QUICConnectionHandler::remove_connection_id(const ngtcp2_cid *cid) {
+void QUICConnectionHandler::remove_connection_id(const ngtcp2_cid &cid) {
-  connections_.erase(*cid);
+  connections_.erase(cid);
 }
 
 void QUICConnectionHandler::add_close_wait(CloseWait *cw) {
@@ -548,10 +679,10 @@ static void close_wait_timeoutcb(struct ev_loop *loop, ev_timer *w,
 }
 
 CloseWait::CloseWait(Worker *worker, std::vector<ngtcp2_cid> scids,
-                     std::vector<uint8_t> conn_close, ev_tstamp period)
+                     std::vector<uint8_t> pkt, ev_tstamp period)
     : worker{worker},
       scids{std::move(scids)},
-      conn_close{std::move(conn_close)},
+      pkt{std::move(pkt)},
       bytes_recv{0},
       bytes_sent{0},
       num_pkts_recv{0},
@@ -573,35 +704,36 @@ CloseWait::~CloseWait() {
   --worker_stat->num_close_waits;
 
   if (worker->get_graceful_shutdown() && worker_stat->num_connections == 0 &&
-      worker_stat->num_close_waits) {
+      worker_stat->num_close_waits == 0) {
     ev_break(loop);
   }
 }
 
 int CloseWait::handle_packet(const UpstreamAddr *faddr,
                              const Address &remote_addr,
-                             const Address &local_addr, const uint8_t *data,
+                             const Address &local_addr,
+                             const ngtcp2_pkt_info &pi, const uint8_t *data,
                              size_t datalen) {
-  if (conn_close.empty()) {
+  if (pkt.empty()) {
     return 0;
   }
 
   ++num_pkts_recv;
   bytes_recv += datalen;
 
-  if (bytes_sent + conn_close.size() > 3 * bytes_recv ||
+  if (bytes_sent + pkt.size() > 3 * bytes_recv ||
       next_pkts_recv > num_pkts_recv) {
     return 0;
   }
 
   if (quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
-                       &local_addr.su.sa, local_addr.len, conn_close.data(),
+                       &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
-                       conn_close.size(), 0) != 0) {
+                       pkt.data(), pkt.size(), 0) != 0) {
     return -1;
   }
 
   next_pkts_recv *= 2;
-  bytes_sent += conn_close.size();
+  bytes_sent += pkt.size();
 
   return 0;
 }

src/shrpx_quic_connection_handler.h

@@ -51,19 +51,19 @@ class Worker;
 // closing period).
 struct CloseWait {
   CloseWait(Worker *worker, std::vector<ngtcp2_cid> scids,
-            std::vector<uint8_t> conn_close, ev_tstamp period);
+            std::vector<uint8_t> pkt, ev_tstamp period);
   ~CloseWait();
 
   int handle_packet(const UpstreamAddr *faddr, const Address &remote_addr,
-                    const Address &local_addr, const uint8_t *data,
+                    const Address &local_addr, const ngtcp2_pkt_info &pi,
-                    size_t datalen);
+                    const uint8_t *data, size_t datalen);
 
   Worker *worker;
   // Source Connection IDs of the connection.
   std::vector<ngtcp2_cid> scids;
-  // QUIC packet containing CONNECTION_CLOSE.  It is empty when a
+  // QUIC packet which is sent in response to the incoming packet.  It
-  // connection entered in draining state.
+  // might be empty.
-  std::vector<uint8_t> conn_close;
+  std::vector<uint8_t> pkt;
   // Close-wait (draining or closing period) timer.
   ev_timer timer;
   // The number of bytes received during close-wait period.
@@ -82,8 +82,8 @@ public:
   QUICConnectionHandler(Worker *worker);
   ~QUICConnectionHandler();
   int handle_packet(const UpstreamAddr *faddr, const Address &remote_addr,
-                    const Address &local_addr, const uint8_t *data,
+                    const Address &local_addr, const ngtcp2_pkt_info &pi,
-                    size_t datalen);
+                    const uint8_t *data, size_t datalen);
   // Send Retry packet.  |ini_dcid| is the destination Connection ID
   // which appeared in Client Initial packet and its length is
   // |dcidlen|.  |ini_scid| is the source Connection ID which appeared
@@ -110,8 +110,8 @@ public:
   // |ini_scid| is the source Connection ID which appeared in Client
   // Initial packet.
   int send_connection_close(const UpstreamAddr *faddr, uint32_t version,
-                            const ngtcp2_cid *ini_dcid,
+                            const ngtcp2_cid &ini_dcid,
-                            const ngtcp2_cid *ini_scid,
+                            const ngtcp2_cid &ini_scid,
                             const Address &remote_addr,
                             const Address &local_addr, uint64_t error_code);
   ClientHandler *handle_new_connection(const UpstreamAddr *faddr,
@@ -120,8 +120,8 @@ public:
                                        const ngtcp2_pkt_hd &hd,
                                        const ngtcp2_cid *odcid,
                                        const uint8_t *token, size_t tokenlen);
-  void add_connection_id(const ngtcp2_cid *cid, ClientHandler *handler);
+  void add_connection_id(const ngtcp2_cid &cid, ClientHandler *handler);
-  void remove_connection_id(const ngtcp2_cid *cid);
+  void remove_connection_id(const ngtcp2_cid &cid);
 
   void add_close_wait(CloseWait *cw);
   void remove_close_wait(const CloseWait *cw);

src/shrpx_quic_listener.cc

@@ -59,7 +59,8 @@ void QUICListener::on_read() {
   msg.msg_iov = &msg_iov;
   msg.msg_iovlen = 1;
 
-  uint8_t msg_ctrl[CMSG_SPACE(sizeof(in6_pktinfo))];
+  uint8_t
+      msg_ctrl[CMSG_SPACE(sizeof(uint8_t)) + CMSG_SPACE(sizeof(in6_pktinfo))];
   msg.msg_control = msg_ctrl;
 
   auto quic_conn_handler = worker_->get_quic_connection_handler();
@@ -83,11 +84,16 @@ void QUICListener::on_read() {
 
     util::set_port(local_addr, faddr_->port);
 
+    ngtcp2_pkt_info pi{
+        .ecn = util::msghdr_get_ecn(&msg, su.storage.ss_family),
+    };
+
     if (LOG_ENABLED(INFO)) {
       LOG(INFO) << "QUIC received packet: local="
                 << util::to_numeric_addr(&local_addr)
                 << " remote=" << util::to_numeric_addr(&su.sa, msg.msg_namelen)
-                << " " << nread << " bytes";
+                << " ecn=" << log::hex << pi.ecn << log::dec << " " << nread
+                << " bytes";
     }
 
     if (nread == 0) {
@@ -98,7 +104,7 @@ void QUICListener::on_read() {
     remote_addr.su = su;
     remote_addr.len = msg.msg_namelen;
 
-    quic_conn_handler->handle_packet(faddr_, remote_addr, local_addr,
+    quic_conn_handler->handle_packet(faddr_, remote_addr, local_addr, pi,
                                      buf.data(), nread);
   }
 }

src/shrpx_tls.cc

@@ -61,8 +61,13 @@
 #ifdef ENABLE_HTTP3
 #  include <ngtcp2/ngtcp2.h>
 #  include <ngtcp2/ngtcp2_crypto.h>
-#  include <ngtcp2/ngtcp2_crypto_openssl.h>
+#  ifdef HAVE_LIBNGTCP2_CRYPTO_OPENSSL
-#endif // ENABLE_HTTP3
+#    include <ngtcp2/ngtcp2_crypto_openssl.h>
+#  endif // HAVE_LIBNGTCP2_CRYPTO_OPENSSL
+#  ifdef HAVE_LIBNGTCP2_CRYPTO_BORINGSSL
+#    include <ngtcp2/ngtcp2_crypto_boringssl.h>
+#  endif // HAVE_LIBNGTCP2_CRYPTO_BORINGSSL
+#endif   // ENABLE_HTTP3
 
 #include "shrpx_log.h"
 #include "shrpx_client_handler.h"
@@ -196,8 +201,9 @@ int servername_callback(SSL *ssl, int *al, void *arg) {
   auto hostname = StringRef{std::begin(buf), end_buf};
 
 #ifdef ENABLE_HTTP3
-  auto cert_tree = SSL_is_quic(ssl) ? worker->get_quic_cert_lookup_tree()
+  auto cert_tree = conn->proto == Proto::HTTP3
-                                    : worker->get_cert_lookup_tree();
+                       ? worker->get_quic_cert_lookup_tree()
+                       : worker->get_cert_lookup_tree();
 #else  // !ENABLE_HTTP3
   auto cert_tree = worker->get_cert_lookup_tree();
 #endif // !ENABLE_HTTP3
@@ -212,7 +218,7 @@ int servername_callback(SSL *ssl, int *al, void *arg) {
   auto conn_handler = worker->get_connection_handler();
 
 #ifdef ENABLE_HTTP3
-  const auto &ssl_ctx_list = SSL_is_quic(ssl)
+  const auto &ssl_ctx_list = conn->proto == Proto::HTTP3
                                  ? conn_handler->get_quic_indexed_ssl_ctx(idx)
                                  : conn_handler->get_indexed_ssl_ctx(idx);
 #else  // !ENABLE_HTTP3
@@ -703,29 +709,36 @@ namespace {
 int quic_alpn_select_proto_cb(SSL *ssl, const unsigned char **out,
                               unsigned char *outlen, const unsigned char *in,
                               unsigned int inlen, void *arg) {
-  for (auto p = in, end = in + inlen; p < end;) {
+  constexpr StringRef alpnlist[] = {
-    auto proto_id = p + 1;
+      StringRef::from_lit("h3"),
-    auto proto_len = *p;
+      StringRef::from_lit("h3-29"),
+  };
 
-    if (proto_id + proto_len <= end &&
+  for (auto &alpn : alpnlist) {
-        util::streq_l("h3", StringRef{proto_id, proto_len})) {
+    for (auto p = in, end = in + inlen; p < end;) {
+      auto proto_id = p + 1;
+      auto proto_len = *p;
 
-      *out = reinterpret_cast<const unsigned char *>(proto_id);
+      if (alpn.size() == proto_len &&
-      *outlen = proto_len;
+          memcmp(alpn.byte(), proto_id, alpn.size()) == 0) {
+        *out = proto_id;
+        *outlen = proto_len;
 
-      return SSL_TLSEXT_ERR_OK;
+        return SSL_TLSEXT_ERR_OK;
+      }
+
+      p += 1 + proto_len;
     }
-
-    p += 1 + proto_len;
   }
 
-  return SSL_TLSEXT_ERR_NOACK;
+  return SSL_TLSEXT_ERR_ALERT_FATAL;
 }
 } // namespace
 #  endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
 #endif   // ENABLE_HTTP3
 
-#if !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L
+#if !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L &&               \
+    !defined(OPENSSL_IS_BORINGSSL)
 
 #  ifndef TLSEXT_TYPE_signed_certificate_timestamp
 #    define TLSEXT_TYPE_signed_certificate_timestamp 18
@@ -815,7 +828,8 @@ int legacy_sct_parse_cb(SSL *ssl, unsigned int ext_type,
 } // namespace
 
 #  endif // !OPENSSL_1_1_1_API
-#endif   // !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L
+#endif   // !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L &&
+         // !defined(OPENSSL_IS_BORINGSSL)
 
 #ifndef OPENSSL_NO_PSK
 namespace {
@@ -913,7 +927,7 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file,
                             neverbleed_t *nb
 #endif // HAVE_NEVERBLEED
 ) {
-  auto ssl_ctx = SSL_CTX_new(SSLv23_server_method());
+  auto ssl_ctx = SSL_CTX_new(TLS_server_method());
   if (!ssl_ctx) {
     LOG(FATAL) << ERR_error_string(ERR_get_error(), nullptr);
     DIE();
@@ -925,14 +939,14 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file,
       SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION | SSL_OP_SINGLE_ECDH_USE |
       SSL_OP_SINGLE_DH_USE |
       SSL_OP_CIPHER_SERVER_PREFERENCE
-#if OPENSSL_1_1_1_API
+#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
       // The reason for disabling built-in anti-replay in OpenSSL is
       // that it only works if client gets back to the same server.
       // The freshness check described in
       // https://tools.ietf.org/html/rfc8446#section-8.3 is still
       // performed.
       | SSL_OP_NO_ANTI_REPLAY
-#endif // OPENSSL_1_1_1_API
+#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
       ;
 
   auto config = mod_config();
@@ -963,13 +977,13 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file,
     DIE();
   }
 
-#if OPENSSL_1_1_1_API
+#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
   if (SSL_CTX_set_ciphersuites(ssl_ctx, tlsconf.tls13_ciphers.c_str()) == 0) {
     LOG(FATAL) << "SSL_CTX_set_ciphersuites " << tlsconf.tls13_ciphers
                << " failed: " << ERR_error_string(ERR_get_error(), nullptr);
     DIE();
   }
-#endif // OPENSSL_1_1_1_API
+#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
 
 #ifndef OPENSSL_NO_EC
 #  if !LIBRESSL_LEGACY_API && OPENSSL_VERSION_NUMBER >= 0x10002000L
@@ -1017,8 +1031,11 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file,
       DIE();
     }
 
-    SSL_CTX_set_tmp_dh(ssl_ctx, dh);
+    if (SSL_CTX_set0_tmp_dh_pkey(ssl_ctx, dh) != 1) {
-    EVP_PKEY_free(dh);
+      LOG(FATAL) << "SSL_CTX_set0_tmp_dh_pkey failed: "
+                 << ERR_error_string(ERR_get_error(), nullptr);
+      DIE();
+    }
 #else  // !OPENSSL_3_0_0_API
     auto dh = PEM_read_bio_DHparams(bio, nullptr, nullptr, nullptr);
     if (dh == nullptr) {
@@ -1131,6 +1148,12 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file,
   SSL_CTX_set_alpn_select_cb(ssl_ctx, alpn_select_proto_cb, nullptr);
 #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
 
+  auto tls_ctx_data = new TLSContextData();
+  tls_ctx_data->cert_file = cert_file;
+  tls_ctx_data->sct_data = sct_data;
+
+  SSL_CTX_set_app_data(ssl_ctx, tls_ctx_data);
+
 #if !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L &&               \
     !defined(OPENSSL_IS_BORINGSSL)
   // SSL_extension_supported(TLSEXT_TYPE_signed_certificate_timestamp)
@@ -1163,31 +1186,34 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file,
     }
 #  endif // !OPENSSL_1_1_1_API
   }
-#endif // !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L &&
+#elif defined(OPENSSL_IS_BORINGSSL)
-       // !defined(OPENSSL_IS_BORINGSSL)
+  if (!tls_ctx_data->sct_data.empty() &&
+      SSL_CTX_set_signed_cert_timestamp_list(
+          ssl_ctx, tls_ctx_data->sct_data.data(),
+          tls_ctx_data->sct_data.size()) != 1) {
+    LOG(FATAL) << "SSL_CTX_set_signed_cert_timestamp_list failed: "
+               << ERR_error_string(ERR_get_error(), nullptr);
+    DIE();
+  }
+#endif // defined(OPENSSL_IS_BORINGSSL)
 
-#if OPENSSL_1_1_1_API
+#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
   if (SSL_CTX_set_max_early_data(ssl_ctx, tlsconf.max_early_data) != 1) {
     LOG(FATAL) << "SSL_CTX_set_max_early_data failed: "
                << ERR_error_string(ERR_get_error(), nullptr);
     DIE();
   }
-#endif // OPENSSL_1_1_1_API
+#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
 
 #ifndef OPENSSL_NO_PSK
   SSL_CTX_set_psk_server_callback(ssl_ctx, psk_server_cb);
 #endif // !LIBRESSL_NO_PSK
 
-  auto tls_ctx_data = new TLSContextData();
-  tls_ctx_data->cert_file = cert_file;
-  tls_ctx_data->sct_data = sct_data;
-
-  SSL_CTX_set_app_data(ssl_ctx, tls_ctx_data);
-
   return ssl_ctx;
 }
 
 #ifdef ENABLE_HTTP3
+#  ifdef HAVE_LIBNGTCP2_CRYPTO_OPENSSL
 namespace {
 int quic_set_encryption_secrets(SSL *ssl, OSSL_ENCRYPTION_LEVEL ossl_level,
                                 const uint8_t *rx_secret,
@@ -1251,6 +1277,84 @@ auto quic_method = SSL_QUIC_METHOD{
     quic_send_alert,
 };
 } // namespace
+#  endif // HAVE_LIBNGTCP2_CRYPTO_OPENSSL
+
+#  ifdef HAVE_LIBNGTCP2_CRYPTO_BORINGSSL
+namespace {
+int quic_set_read_secret(SSL *ssl, ssl_encryption_level_t ssl_level,
+                         const SSL_CIPHER *cipher, const uint8_t *secret,
+                         size_t secretlen) {
+  auto conn = static_cast<Connection *>(SSL_get_app_data(ssl));
+  auto handler = static_cast<ClientHandler *>(conn->data);
+  auto upstream = static_cast<Http3Upstream *>(handler->get_upstream());
+  auto level = ngtcp2_crypto_boringssl_from_ssl_encryption_level(ssl_level);
+
+  if (upstream->on_rx_secret(level, secret, secretlen) != 0) {
+    return 0;
+  }
+
+  return 1;
+}
+} // namespace
+
+namespace {
+int quic_set_write_secret(SSL *ssl, ssl_encryption_level_t ssl_level,
+                          const SSL_CIPHER *cipher, const uint8_t *secret,
+                          size_t secretlen) {
+  auto conn = static_cast<Connection *>(SSL_get_app_data(ssl));
+  auto handler = static_cast<ClientHandler *>(conn->data);
+  auto upstream = static_cast<Http3Upstream *>(handler->get_upstream());
+  auto level = ngtcp2_crypto_boringssl_from_ssl_encryption_level(ssl_level);
+
+  if (upstream->on_tx_secret(level, secret, secretlen) != 0) {
+    return 0;
+  }
+
+  return 1;
+}
+} // namespace
+
+namespace {
+int quic_add_handshake_data(SSL *ssl, ssl_encryption_level_t ssl_level,
+                            const uint8_t *data, size_t len) {
+  auto conn = static_cast<Connection *>(SSL_get_app_data(ssl));
+  auto handler = static_cast<ClientHandler *>(conn->data);
+  auto upstream = static_cast<Http3Upstream *>(handler->get_upstream());
+  auto level = ngtcp2_crypto_boringssl_from_ssl_encryption_level(ssl_level);
+
+  upstream->add_crypto_data(level, data, len);
+
+  return 1;
+}
+} // namespace
+
+namespace {
+int quic_flush_flight(SSL *ssl) { return 1; }
+} // namespace
+
+namespace {
+int quic_send_alert(SSL *ssl, ssl_encryption_level_t level, uint8_t alert) {
+  auto conn = static_cast<Connection *>(SSL_get_app_data(ssl));
+  auto handler = static_cast<ClientHandler *>(conn->data);
+  auto upstream = static_cast<Http3Upstream *>(handler->get_upstream());
+
+  if (!upstream) {
+    return 1;
+  }
+
+  upstream->set_tls_alert(alert);
+
+  return 1;
+}
+} // namespace
+
+namespace {
+auto quic_method = SSL_QUIC_METHOD{
+    quic_set_read_secret, quic_set_write_secret, quic_add_handshake_data,
+    quic_flush_flight,    quic_send_alert,
+};
+} // namespace
+#  endif // HAVE_LIBNGTCP2_CRYPTO_BORINGSSL
 
 SSL_CTX *create_quic_ssl_context(const char *private_key_file,
                                  const char *cert_file,
@@ -1271,14 +1375,14 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file,
       SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION | SSL_OP_SINGLE_ECDH_USE |
       SSL_OP_SINGLE_DH_USE |
       SSL_OP_CIPHER_SERVER_PREFERENCE
-#  if OPENSSL_1_1_1_API
+#  if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
       // The reason for disabling built-in anti-replay in OpenSSL is
       // that it only works if client gets back to the same server.
       // The freshness check described in
       // https://tools.ietf.org/html/rfc8446#section-8.3 is still
       // performed.
       | SSL_OP_NO_ANTI_REPLAY
-#  endif // OPENSSL_1_1_1_API
+#  endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
       ;
 
   auto config = mod_config();
@@ -1301,13 +1405,13 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file,
     DIE();
   }
 
-#  if OPENSSL_1_1_1_API
+#  if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
   if (SSL_CTX_set_ciphersuites(ssl_ctx, tlsconf.tls13_ciphers.c_str()) == 0) {
     LOG(FATAL) << "SSL_CTX_set_ciphersuites " << tlsconf.tls13_ciphers
                << " failed: " << ERR_error_string(ERR_get_error(), nullptr);
     DIE();
   }
-#  endif // OPENSSL_1_1_1_API
+#  endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
 
 #  ifndef OPENSSL_NO_EC
 #    if !LIBRESSL_LEGACY_API && OPENSSL_VERSION_NUMBER >= 0x10002000L
@@ -1355,8 +1459,11 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file,
       DIE();
     }
 
-    SSL_CTX_set_tmp_dh(ssl_ctx, dh);
+    if (SSL_CTX_set0_tmp_dh_pkey(ssl_ctx, dh) != 1) {
-    EVP_PKEY_free(dh);
+      LOG(FATAL) << "SSL_CTX_set0_tmp_dh_pkey failed: "
+                 << ERR_error_string(ERR_get_error(), nullptr);
+      DIE();
+    }
 #  else  // !OPENSSL_3_0_0_API
     auto dh = PEM_read_bio_DHparams(bio, nullptr, nullptr, nullptr);
     if (dh == nullptr) {
@@ -1460,6 +1567,12 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file,
   SSL_CTX_set_alpn_select_cb(ssl_ctx, quic_alpn_select_proto_cb, nullptr);
 #  endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
 
+  auto tls_ctx_data = new TLSContextData();
+  tls_ctx_data->cert_file = cert_file;
+  tls_ctx_data->sct_data = sct_data;
+
+  SSL_CTX_set_app_data(ssl_ctx, tls_ctx_data);
+
 #  if !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L &&             \
       !defined(OPENSSL_IS_BORINGSSL)
   // SSL_extension_supported(TLSEXT_TYPE_signed_certificate_timestamp)
@@ -1492,10 +1605,18 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file,
     }
 #    endif // !OPENSSL_1_1_1_API
   }
-#  endif // !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L &&
+#  elif defined(OPENSSL_IS_BORINGSSL)
-         // !defined(OPENSSL_IS_BORINGSSL)
+  if (!tls_ctx_data->sct_data.empty() &&
+      SSL_CTX_set_signed_cert_timestamp_list(
+          ssl_ctx, tls_ctx_data->sct_data.data(),
+          tls_ctx_data->sct_data.size()) != 1) {
+    LOG(FATAL) << "SSL_CTX_set_signed_cert_timestamp_list failed: "
+               << ERR_error_string(ERR_get_error(), nullptr);
+    DIE();
+  }
+#  endif // defined(OPENSSL_IS_BORINGSSL)
 
-#  if OPENSSL_1_1_1_API
+#  if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
   auto &quicconf = config->quic;
 
   if (quicconf.upstream.early_data &&
@@ -1505,7 +1626,7 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file,
                << ERR_error_string(ERR_get_error(), nullptr);
     DIE();
   }
-#  endif // OPENSSL_1_1_1_API
+#  endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
 
 #  ifndef OPENSSL_NO_PSK
   SSL_CTX_set_psk_server_callback(ssl_ctx, psk_server_cb);
@@ -1513,12 +1634,6 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file,
 
   SSL_CTX_set_quic_method(ssl_ctx, &quic_method);
 
-  auto tls_ctx_data = new TLSContextData();
-  tls_ctx_data->cert_file = cert_file;
-  tls_ctx_data->sct_data = sct_data;
-
-  SSL_CTX_set_app_data(ssl_ctx, tls_ctx_data);
-
   return ssl_ctx;
 }
 #endif // ENABLE_HTTP3
@@ -1579,7 +1694,7 @@ SSL_CTX *create_ssl_client_context(
     int (*next_proto_select_cb)(SSL *s, unsigned char **out,
                                 unsigned char *outlen, const unsigned char *in,
                                 unsigned int inlen, void *arg)) {
-  auto ssl_ctx = SSL_CTX_new(SSLv23_client_method());
+  auto ssl_ctx = SSL_CTX_new(TLS_client_method());
   if (!ssl_ctx) {
     LOG(FATAL) << ERR_error_string(ERR_get_error(), nullptr);
     DIE();
@@ -1610,14 +1725,14 @@ SSL_CTX *create_ssl_client_context(
     DIE();
   }
 
-#if OPENSSL_1_1_1_API
+#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
   if (SSL_CTX_set_ciphersuites(ssl_ctx, tlsconf.client.tls13_ciphers.c_str()) ==
       0) {
     LOG(FATAL) << "SSL_CTX_set_ciphersuites " << tlsconf.client.tls13_ciphers
                << " failed: " << ERR_error_string(ERR_get_error(), nullptr);
     DIE();
   }
-#endif // OPENSSL_1_1_1_API
+#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
 
   SSL_CTX_set_mode(ssl_ctx, SSL_MODE_RELEASE_BUFFERS);
 
@@ -1955,14 +2070,20 @@ int verify_hostname(X509 *cert, const StringRef &hostname,
 } // namespace
 
 int check_cert(SSL *ssl, const Address *addr, const StringRef &host) {
+#if OPENSSL_3_0_0_API
+  auto cert = SSL_get0_peer_certificate(ssl);
+#else  // !OPENSSL_3_0_0_API
   auto cert = SSL_get_peer_certificate(ssl);
+#endif // !OPENSSL_3_0_0_API
   if (!cert) {
     // By the protocol definition, TLS server always sends certificate
     // if it has.  If certificate cannot be retrieved, authentication
     // without certificate is used, such as PSK.
     return 0;
   }
+#if !OPENSSL_3_0_0_API
   auto cert_deleter = defer(X509_free, cert);
+#endif // !OPENSSL_3_0_0_API
 
   if (verify_hostname(cert, host, addr) != 0) {
     LOG(ERROR) << "Certificate verification failed: hostname does not match";
@@ -2619,7 +2740,7 @@ namespace {
 int time_t_from_asn1_time(time_t &t, const ASN1_TIME *at) {
   int rv;
 
-#if OPENSSL_1_1_1_API
+#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
   struct tm tm;
   rv = ASN1_TIME_to_tm(at, &tm);
   if (rv != 1) {
@@ -2627,7 +2748,7 @@ int time_t_from_asn1_time(time_t &t, const ASN1_TIME *at) {
   }
 
   t = nghttp2_timegm(&tm);
-#else // !OPENSSL_1_1_1_API
+#else // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
   auto b = BIO_new(BIO_s_mem());
   if (!b) {
     return -1;
@@ -2640,7 +2761,7 @@ int time_t_from_asn1_time(time_t &t, const ASN1_TIME *at) {
     return -1;
   }
 
-#  if defined(OPENSSL_IS_BORINGSSL)
+#  ifdef OPENSSL_IS_BORINGSSL
   char *s;
 #  else
   unsigned char *s;
@@ -2653,7 +2774,7 @@ int time_t_from_asn1_time(time_t &t, const ASN1_TIME *at) {
   }
 
   t = tt;
-#endif // !OPENSSL_1_1_1_API
+#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
 
   return 0;
 }

src/shrpx_tls_test.cc

@@ -121,7 +121,7 @@ void test_shrpx_tls_cert_lookup_tree_add_ssl_ctx(void) {
 
   static constexpr char nghttp2_certfile[] =
       NGHTTP2_SRC_DIR "/test.nghttp2.org.pem";
-  auto nghttp2_ssl_ctx = SSL_CTX_new(SSLv23_server_method());
+  auto nghttp2_ssl_ctx = SSL_CTX_new(TLS_server_method());
   auto nghttp2_ssl_ctx_del = defer(SSL_CTX_free, nghttp2_ssl_ctx);
   auto nghttp2_tls_ctx_data = std::make_unique<tls::TLSContextData>();
   nghttp2_tls_ctx_data->cert_file = nghttp2_certfile;
@@ -132,7 +132,7 @@ void test_shrpx_tls_cert_lookup_tree_add_ssl_ctx(void) {
 
   static constexpr char examples_certfile[] =
       NGHTTP2_SRC_DIR "/test.example.com.pem";
-  auto examples_ssl_ctx = SSL_CTX_new(SSLv23_server_method());
+  auto examples_ssl_ctx = SSL_CTX_new(TLS_server_method());
   auto examples_ssl_ctx_del = defer(SSL_CTX_free, examples_ssl_ctx);
   auto examples_tls_ctx_data = std::make_unique<tls::TLSContextData>();
   examples_tls_ctx_data->cert_file = examples_certfile;

src/shrpx_worker.cc

@@ -554,7 +554,7 @@ void Worker::process_events() {
 
     quic_conn_handler_.handle_packet(
         faddr, wev.quic_pkt->remote_addr, wev.quic_pkt->local_addr,
-        wev.quic_pkt->data.data(), wev.quic_pkt->data.size());
+        wev.quic_pkt->pi, wev.quic_pkt->data.data(), wev.quic_pkt->data.size());
 
     break;
   }
@@ -833,6 +833,28 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) {
         close(fd);
         continue;
       }
+
+      if (setsockopt(fd, IPPROTO_IPV6, IPV6_RECVTCLASS, &val,
+                     static_cast<socklen_t>(sizeof(val))) == -1) {
+        auto error = errno;
+        LOG(WARN) << "Failed to set IPV6_RECVTCLASS option to listener socket: "
+                  << xsi_strerror(error, errbuf.data(), errbuf.size());
+        close(fd);
+        continue;
+      }
+
+#  if defined(IPV6_MTU_DISCOVER) && defined(IP_PMTUDISC_DO)
+      int mtu_disc = IP_PMTUDISC_DO;
+      if (setsockopt(fd, IPPROTO_IPV6, IPV6_MTU_DISCOVER, &mtu_disc,
+                     static_cast<socklen_t>(sizeof(mtu_disc))) == -1) {
+        auto error = errno;
+        LOG(WARN)
+            << "Failed to set IPV6_MTU_DISCOVER option to listener socket: "
+            << xsi_strerror(error, errbuf.data(), errbuf.size());
+        close(fd);
+        continue;
+      }
+#  endif // defined(IPV6_MTU_DISCOVER) && defined(IP_PMTUDISC_DO)
     } else {
       if (setsockopt(fd, IPPROTO_IP, IP_PKTINFO, &val,
                      static_cast<socklen_t>(sizeof(val))) == -1) {
@@ -842,9 +864,28 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) {
         close(fd);
         continue;
       }
-    }
 
-    // TODO Enable ECN
+      if (setsockopt(fd, IPPROTO_IP, IP_RECVTOS, &val,
+                     static_cast<socklen_t>(sizeof(val))) == -1) {
+        auto error = errno;
+        LOG(WARN) << "Failed to set IP_RECVTOS option to listener socket: "
+                  << xsi_strerror(error, errbuf.data(), errbuf.size());
+        close(fd);
+        continue;
+      }
+
+#  if defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DO)
+      int mtu_disc = IP_PMTUDISC_DO;
+      if (setsockopt(fd, IPPROTO_IP, IP_MTU_DISCOVER, &mtu_disc,
+                     static_cast<socklen_t>(sizeof(mtu_disc))) == -1) {
+        auto error = errno;
+        LOG(WARN) << "Failed to set IP_MTU_DISCOVER option to listener socket: "
+                  << xsi_strerror(error, errbuf.data(), errbuf.size());
+        close(fd);
+        continue;
+      }
+#  endif // defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DO)
+    }
 
     if (bind(fd, rp->ai_addr, rp->ai_addrlen) == -1) {
       auto error = errno;
@@ -890,6 +931,8 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) {
 
       auto &ref = quic_bpf_refs[faddr.index];
 
+      ref.obj = obj;
+
       auto reuseport_array =
           bpf_object__find_map_by_name(obj, "reuseport_array");
       err = libbpf_get_error(reuseport_array);
@@ -923,7 +966,7 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) {
       }
 
       constexpr uint32_t zero = 0;
-      uint32_t num_socks = config->num_worker;
+      uint64_t num_socks = config->num_worker;
 
       if (bpf_map_update_elem(bpf_map__fd(sk_info), &zero, &num_socks,
                               BPF_ANY) != 0) {
@@ -933,6 +976,29 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) {
         return -1;
       }
 
+      constexpr uint32_t key_high_idx = 1;
+      constexpr uint32_t key_low_idx = 2;
+
+      auto &qkms = conn_handler_->get_quic_keying_materials();
+      auto &qkm = qkms->keying_materials.front();
+
+      if (bpf_map_update_elem(bpf_map__fd(sk_info), &key_high_idx,
+                              qkm.cid_encryption_key.data(), BPF_ANY) != 0) {
+        LOG(FATAL) << "Failed to update key_high_idx sk_info: "
+                   << xsi_strerror(errno, errbuf.data(), errbuf.size());
+        close(fd);
+        return -1;
+      }
+
+      if (bpf_map_update_elem(bpf_map__fd(sk_info), &key_low_idx,
+                              qkm.cid_encryption_key.data() + 8,
+                              BPF_ANY) != 0) {
+        LOG(FATAL) << "Failed to update key_low_idx sk_info: "
+                   << xsi_strerror(errno, errbuf.data(), errbuf.size());
+        close(fd);
+        return -1;
+      }
+
       auto prog_fd = bpf_program__fd(prog);
 
       if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_REUSEPORT_EBPF, &prog_fd,
@@ -987,14 +1053,6 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) {
 
 const uint8_t *Worker::get_cid_prefix() const { return cid_prefix_.data(); }
 
-void Worker::set_quic_secret(const std::shared_ptr<QUICSecret> &secret) {
-  quic_secret_ = secret;
-}
-
-const std::shared_ptr<QUICSecret> &Worker::get_quic_secret() const {
-  return quic_secret_;
-}
-
 const UpstreamAddr *Worker::find_quic_upstream_addr(const Address &local_addr) {
   std::array<char, NI_MAXHOST> host;
 
@@ -1019,9 +1077,13 @@ const UpstreamAddr *Worker::find_quic_upstream_addr(const Address &local_addr) {
     break;
   default:
     assert(0);
+    abort();
   }
 
-  auto hostport = util::make_hostport(StringRef{host.data()}, port);
+  std::array<char, util::max_hostport> hostport_buf;
+
+  auto hostport = util::make_http_hostport(std::begin(hostport_buf),
+                                           StringRef{host.data()}, port);
   const UpstreamAddr *fallback_faddr = nullptr;
 
   for (auto &faddr : quic_upstream_addrs_) {
@@ -1033,21 +1095,41 @@ const UpstreamAddr *Worker::find_quic_upstream_addr(const Address &local_addr) {
       continue;
     }
 
-    switch (faddr.family) {
+    if (faddr.port == 443 || faddr.port == 80) {
-    case AF_INET:
+      switch (faddr.family) {
-      if (util::starts_with(faddr.hostport, StringRef::from_lit("0.0.0.0:"))) {
+      case AF_INET:
-        fallback_faddr = &faddr;
+        if (util::streq(faddr.hostport, StringRef::from_lit("0.0.0.0"))) {
-      }
+          fallback_faddr = &faddr;
+        }
 
-      break;
+        break;
-    case AF_INET6:
+      case AF_INET6:
-      if (util::starts_with(faddr.hostport, StringRef::from_lit("[::]:"))) {
+        if (util::streq(faddr.hostport, StringRef::from_lit("[::]"))) {
-        fallback_faddr = &faddr;
+          fallback_faddr = &faddr;
-      }
+        }
 
-      break;
+        break;
-    default:
+      default:
-      assert(0);
+        assert(0);
+      }
+    } else {
+      switch (faddr.family) {
+      case AF_INET:
+        if (util::starts_with(faddr.hostport,
+                              StringRef::from_lit("0.0.0.0:"))) {
+          fallback_faddr = &faddr;
+        }
+
+        break;
+      case AF_INET6:
+        if (util::starts_with(faddr.hostport, StringRef::from_lit("[::]:"))) {
+          fallback_faddr = &faddr;
+        }
+
+        break;
+      default:
+        assert(0);
+      }
     }
   }
 
@@ -1227,8 +1309,10 @@ void downstream_failure(DownstreamAddr *addr, const Address *raddr) {
 }
 
 #ifdef ENABLE_HTTP3
-int create_cid_prefix(uint8_t *cid_prefix) {
+int create_cid_prefix(uint8_t *cid_prefix, const uint8_t *server_id) {
-  if (RAND_bytes(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN) != 1) {
+  auto p = std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, cid_prefix);
+
+  if (RAND_bytes(p, SHRPX_QUIC_CID_PREFIXLEN - SHRPX_QUIC_SERVER_IDLEN) != 1) {
     return -1;
   }
 

src/shrpx_worker.h

@@ -258,15 +258,18 @@ struct WorkerStat {
 #ifdef ENABLE_HTTP3
 struct QUICPacket {
   QUICPacket(size_t upstream_addr_index, const Address &remote_addr,
-             const Address &local_addr, const uint8_t *data, size_t datalen)
+             const Address &local_addr, const ngtcp2_pkt_info &pi,
+             const uint8_t *data, size_t datalen)
       : upstream_addr_index{upstream_addr_index},
         remote_addr{remote_addr},
         local_addr{local_addr},
+        pi{pi},
         data{data, data + datalen} {}
-  QUICPacket() {}
+  QUICPacket() : upstream_addr_index{}, remote_addr{}, local_addr{}, pi{} {}
   size_t upstream_addr_index;
   Address remote_addr;
   Address local_addr;
+  ngtcp2_pkt_info pi;
   std::vector<uint8_t> data;
 };
 #endif // ENABLE_HTTP3
@@ -370,10 +373,6 @@ public:
 
   const uint8_t *get_cid_prefix() const;
 
-  void set_quic_secret(const std::shared_ptr<QUICSecret> &secret);
-
-  const std::shared_ptr<QUICSecret> &get_quic_secret() const;
-
 #  ifdef HAVE_LIBBPF
   bool should_attach_bpf() const;
 
@@ -412,7 +411,6 @@ private:
   std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN> cid_prefix_;
   std::vector<UpstreamAddr> quic_upstream_addrs_;
   std::vector<std::unique_ptr<QUICListener>> quic_listeners_;
-  std::shared_ptr<QUICSecret> quic_secret_;
 #endif // ENABLE_HTTP3
 
   std::shared_ptr<DownstreamConfig> downstreamconf_;
@@ -441,7 +439,7 @@ private:
   std::shared_ptr<TicketKeys> ticket_keys_;
   std::vector<std::shared_ptr<DownstreamAddrGroup>> downstream_addr_groups_;
   // Worker level blocker for downstream connection.  For example,
-  // this is used when file decriptor is exhausted.
+  // this is used when file descriptor is exhausted.
   std::unique_ptr<ConnectBlocker> connect_blocker_;
 
   bool graceful_shutdown_;
@@ -467,8 +465,8 @@ void downstream_failure(DownstreamAddr *addr, const Address *raddr);
 #ifdef ENABLE_HTTP3
 // Creates unpredictable SHRPX_QUIC_CID_PREFIXLEN bytes sequence which
 // is used as a prefix of QUIC Connection ID.  This function returns
-// -1 on failure.
+// -1 on failure.  |server_id| must be 2 bytes long.
-int create_cid_prefix(uint8_t *cid_prefix);
+int create_cid_prefix(uint8_t *cid_prefix, const uint8_t *server_id);
 #endif // ENABLE_HTTP3
 
 } // namespace shrpx

src/shrpx_worker_process.cc

@@ -519,10 +519,51 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) {
   }
 
 #ifdef ENABLE_HTTP3
-  if (conn_handler->create_quic_secret() != 0) {
+  auto &quicconf = config->quic;
-    return -1;
+
+  std::shared_ptr<QUICKeyingMaterials> qkms;
+
+  if (!quicconf.upstream.secret_file.empty()) {
+    qkms = read_quic_secret_file(quicconf.upstream.secret_file);
+    if (!qkms) {
+      LOG(WARN) << "Use QUIC keying materials generated internally";
+    }
   }
 
+  if (!qkms) {
+    qkms = std::make_shared<QUICKeyingMaterials>();
+    qkms->keying_materials.resize(1);
+
+    auto &qkm = qkms->keying_materials.front();
+
+    if (RAND_bytes(qkm.reserved.data(), qkm.reserved.size()) != 1) {
+      LOG(ERROR) << "Failed to generate QUIC secret reserved data";
+      return -1;
+    }
+
+    if (RAND_bytes(qkm.secret.data(), qkm.secret.size()) != 1) {
+      LOG(ERROR) << "Failed to generate QUIC secret";
+      return -1;
+    }
+
+    if (RAND_bytes(qkm.salt.data(), qkm.salt.size()) != 1) {
+      LOG(ERROR) << "Failed to generate QUIC salt";
+      return -1;
+    }
+  }
+
+  for (auto &qkm : qkms->keying_materials) {
+    if (generate_quic_connection_id_encryption_key(
+            qkm.cid_encryption_key.data(), qkm.cid_encryption_key.size(),
+            qkm.secret.data(), qkm.secret.size(), qkm.salt.data(),
+            qkm.salt.size()) != 0) {
+      LOG(ERROR) << "Failed to generate QUIC Connection ID encryption key";
+      return -1;
+    }
+  }
+
+  conn_handler->set_quic_keying_materials(std::move(qkms));
+
   conn_handler->set_cid_prefixes(wpconf->cid_prefixes);
   conn_handler->set_quic_lingering_worker_processes(
       wpconf->quic_lingering_worker_processes);
@@ -562,6 +603,10 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) {
 #endif // !NOTHREADS
   }
 
+#if defined(ENABLE_HTTP3) && defined(HAVE_LIBBPF)
+  conn_handler->unload_bpf_objects();
+#endif // defined(ENABLE_HTTP3) && defined(HAVE_LIBBPF)
+
   drop_privileges(
 #ifdef HAVE_NEVERBLEED
       nb.get()

src/ssl_compat.h

@@ -26,20 +26,20 @@
 
 #  include <openssl/opensslv.h>
 
-#  if defined(LIBRESSL_VERSION_NUMBER)
+#  ifdef LIBRESSL_VERSION_NUMBER
 #    define OPENSSL_1_1_API 0
 #    define OPENSSL_1_1_1_API 0
 #    define OPENSSL_3_0_0_API 0
 #    define LIBRESSL_IN_USE 1
 #    define LIBRESSL_LEGACY_API (LIBRESSL_VERSION_NUMBER < 0x20700000L)
 #    define LIBRESSL_2_7_API (LIBRESSL_VERSION_NUMBER >= 0x20700000L)
-#  else // !defined(LIBRESSL_VERSION_NUMBER)
+#  else // !LIBRESSL_VERSION_NUMBER
 #    define OPENSSL_1_1_API (OPENSSL_VERSION_NUMBER >= 0x1010000fL)
 #    define OPENSSL_1_1_1_API (OPENSSL_VERSION_NUMBER >= 0x10101000L)
 #    define OPENSSL_3_0_0_API (OPENSSL_VERSION_NUMBER >= 0x30000000L)
 #    define LIBRESSL_IN_USE 0
 #    define LIBRESSL_LEGACY_API 0
 #    define LIBRESSL_2_7_API 0
-#  endif // !defined(LIBRESSL_VERSION_NUMBER)
+#  endif // !LIBRESSL_VERSION_NUMBER
 
 #endif // OPENSSL_COMPAT_H

src/tls.h

@@ -57,11 +57,15 @@ constexpr char DEFAULT_CIPHER_LIST[] =
     "AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256";
 
 constexpr char DEFAULT_TLS13_CIPHER_LIST[] =
-#if OPENSSL_1_1_1_API
+#if OPENSSL_3_0_0_API
+    "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256"
+#elif OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
     TLS_DEFAULT_CIPHERSUITES
-#else  // !OPENSSL_1_1_1_API
+#else  // !OPENSSL_3_0_0_API && !(OPENSSL_1_1_1_API &&
+       // !defined(OPENSSL_IS_BORINGSSL))
     ""
-#endif // !OPENSSL_1_1_1_API
+#endif // !OPENSSL_3_0_0_API && !(OPENSSL_1_1_1_API &&
+       // !defined(OPENSSL_IS_BORINGSSL))
     ;
 
 constexpr auto NGHTTP2_TLS_MIN_VERSION = TLS1_VERSION;

src/util.cc

@@ -1365,81 +1365,14 @@ std::string dtos(double n) {
 
 StringRef make_http_hostport(BlockAllocator &balloc, const StringRef &host,
                              uint16_t port) {
-  if (port != 80 && port != 443) {
+  auto iov = make_byte_ref(balloc, host.size() + 2 + 1 + 5 + 1);
-    return make_hostport(balloc, host, port);
+  return make_http_hostport(iov.base, host, port);
-  }
-
-  auto ipv6 = ipv6_numeric_addr(host.c_str());
-
-  auto iov = make_byte_ref(balloc, host.size() + (ipv6 ? 2 : 0) + 1);
-  auto p = iov.base;
-
-  if (ipv6) {
-    *p++ = '[';
-  }
-
-  p = std::copy(std::begin(host), std::end(host), p);
-
-  if (ipv6) {
-    *p++ = ']';
-  }
-
-  *p = '\0';
-
-  return StringRef{iov.base, p};
-}
-
-std::string make_hostport(const StringRef &host, uint16_t port) {
-  auto ipv6 = ipv6_numeric_addr(host.c_str());
-  auto serv = utos(port);
-
-  std::string hostport;
-  hostport.resize(host.size() + (ipv6 ? 2 : 0) + 1 + serv.size());
-
-  auto p = &hostport[0];
-
-  if (ipv6) {
-    *p++ = '[';
-  }
-
-  p = std::copy_n(host.c_str(), host.size(), p);
-
-  if (ipv6) {
-    *p++ = ']';
-  }
-
-  *p++ = ':';
-  std::copy_n(serv.c_str(), serv.size(), p);
-
-  return hostport;
 }
 
 StringRef make_hostport(BlockAllocator &balloc, const StringRef &host,
                         uint16_t port) {
-  auto ipv6 = ipv6_numeric_addr(host.c_str());
+  auto iov = make_byte_ref(balloc, host.size() + 2 + 1 + 5 + 1);
-  auto serv = utos(port);
+  return make_hostport(iov.base, host, port);
-
-  auto iov =
-      make_byte_ref(balloc, host.size() + (ipv6 ? 2 : 0) + 1 + serv.size());
-  auto p = iov.base;
-
-  if (ipv6) {
-    *p++ = '[';
-  }
-
-  p = std::copy(std::begin(host), std::end(host), p);
-
-  if (ipv6) {
-    *p++ = ']';
-  }
-
-  *p++ = ':';
-
-  p = std::copy(std::begin(serv), std::end(serv), p);
-
-  *p = '\0';
-
-  return StringRef{iov.base, p};
 }
 
 namespace {
@@ -1756,7 +1689,7 @@ std::mt19937 make_mt19937() {
 }
 
 int daemonize(int nochdir, int noclose) {
-#if defined(__APPLE__)
+#ifdef __APPLE__
   pid_t pid;
   pid = fork();
   if (pid == -1) {
@@ -1792,9 +1725,9 @@ int daemonize(int nochdir, int noclose) {
   return 0;
 #elif defined(_MSC_VER)
   return -1;
-#else  // !defined(__APPLE__)
+#else  // !__APPLE__ && !_MSC_VER
   return daemon(nochdir, noclose);
-#endif // !defined(__APPLE__)
+#endif // !__APPLE__
 }
 
 #ifdef ENABLE_HTTP3
@@ -1833,6 +1766,53 @@ int msghdr_get_local_addr(Address &dest, msghdr *msg, int family) {
 }
 #endif
 
+unsigned int msghdr_get_ecn(msghdr *msg, int family) {
+  switch (family) {
+  case AF_INET:
+    for (auto cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
+      if (cmsg->cmsg_level == IPPROTO_IP && cmsg->cmsg_type == IP_TOS &&
+          cmsg->cmsg_len) {
+        return *reinterpret_cast<uint8_t *>(CMSG_DATA(cmsg));
+      }
+    }
+
+    return 0;
+  case AF_INET6:
+    for (auto cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
+      if (cmsg->cmsg_level == IPPROTO_IPV6 && cmsg->cmsg_type == IPV6_TCLASS &&
+          cmsg->cmsg_len) {
+        return *reinterpret_cast<uint8_t *>(CMSG_DATA(cmsg));
+      }
+    }
+
+    return 0;
+  }
+
+  return 0;
+}
+
+int fd_set_send_ecn(int fd, int family, unsigned int ecn) {
+  switch (family) {
+  case AF_INET:
+    if (setsockopt(fd, IPPROTO_IP, IP_TOS, &ecn,
+                   static_cast<socklen_t>(sizeof(ecn))) == -1) {
+      return -1;
+    }
+
+    return 0;
+  case AF_INET6:
+    if (setsockopt(fd, IPPROTO_IPV6, IPV6_TCLASS, &ecn,
+                   static_cast<socklen_t>(sizeof(ecn))) == -1) {
+      return -1;
+    }
+
+    return 0;
+  }
+
+  return -1;
+}
+#endif // ENABLE_HTTP3
+
 } // namespace util
 
 } // namespace nghttp2