diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2456088a..f7502bff 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -60,8 +60,6 @@ jobs: pkg-config \ libtool echo 'PKG_CONFIG_PATH=/usr/local/opt/libressl/lib/pkgconfig:/usr/local/opt/libxml2/lib/pkgconfig' >> $GITHUB_ENV - # This fixes infamous 'stdio.h not found' error. - echo 'SDKROOT='"$(xcrun --sdk macosx --show-sdk-path)" >> $GITHUB_ENV - name: Setup clang (Linux) if: runner.os == 'Linux' && matrix.compiler == 'clang' run: | @@ -87,28 +85,28 @@ jobs: run: | git clone -b v0.4.0 https://github.com/libbpf/libbpf cd libbpf - export PREFIX=$PWD/build - cd src - make install + PREFIX=$PWD/build make -C src install EXTRA_AUTOTOOLS_OPTS="--with-libbpf" + EXTRA_CMAKE_OPTS="-DWITH_LIBBPF=1" echo 'EXTRA_AUTOTOOLS_OPTS='"$EXTRA_AUTOTOOLS_OPTS" >> $GITHUB_ENV + echo 'EXTRA_CMAKE_OPTS='"$EXTRA_CMAKE_OPTS" >> $GITHUB_ENV - name: Build quictls/openssl v1.1.1 if: matrix.http3 == 'http3' && matrix.openssl == 'openssl1' run: | - git clone --depth 1 -b OpenSSL_1_1_1l+quic https://github.com/quictls/openssl + git clone --depth 1 -b OpenSSL_1_1_1m+quic https://github.com/quictls/openssl cd openssl ./config enable-tls1_3 --prefix=$PWD/build make -j$(nproc) make install_sw - - name: Build quictls/openssl v3.0.0 + - name: Build quictls/openssl v3.0.x if: matrix.http3 == 'http3' && matrix.openssl == 'openssl3' run: | unset CPPFLAGS unset LDFLAGS - git clone --depth 1 -b openssl-3.0.0+quic https://github.com/quictls/openssl + git clone --depth 1 -b openssl-3.0.1+quic https://github.com/quictls/openssl cd openssl ./config enable-tls1_3 --prefix=$PWD/build --libdir=$PWD/build/lib make -j$(nproc) @@ -118,6 +116,7 @@ jobs: run: | git clone https://github.com/ngtcp2/nghttp3 cd nghttp3 + git checkout 74a222fe0c89b7202bcdaf6ef27a232edffc85e3 autoreconf -i ./configure --prefix=$PWD/build --enable-lib-only make -j$(nproc) check @@ -125,7 +124,7 @@ jobs: - name: Build ngtcp2 if: matrix.http3 == 'http3' run: | - git clone https://github.com/ngtcp2/ngtcp2 + git clone --depth 1 -b v0.1.0 https://github.com/ngtcp2/ngtcp2 cd ngtcp2 autoreconf -i ./configure --prefix=$PWD/build --enable-lib-only PKG_CONFIG_PATH="../openssl/build/lib/pkgconfig" @@ -137,31 +136,41 @@ jobs: PKG_CONFIG_PATH="$PWD/openssl/build/lib/pkgconfig:$PWD/nghttp3/build/lib/pkgconfig:$PWD/ngtcp2/build/lib/pkgconfig:$PWD/libbpf/build/lib64/pkgconfig:$PKG_CONFIG_PATH" LDFLAGS="$LDFLAGS -Wl,-rpath,$PWD/openssl/build/lib -Wl,-rpath,$PWD/libbpf/build/lib64" EXTRA_AUTOTOOLS_OPTS="--enable-http3 $EXTRA_AUTOTOOLS_OPTS" + EXTRA_CMAKE_OPTS="-DENABLE_HTTP3=1 $EXTRA_CMAKE_OPTS" echo 'PKG_CONFIG_PATH='"$PKG_CONFIG_PATH" >> $GITHUB_ENV echo 'LDFLAGS='"$LDFLAGS" >> $GITHUB_ENV echo 'EXTRA_AUTOTOOLS_OPTS='"$EXTRA_AUTOTOOLS_OPTS" >> $GITHUB_ENV - echo 'EXTRA_CMAKE_OPTS=-DENABLE_HTTP3=ON' >> $GITHUB_ENV + echo 'EXTRA_CMAKE_OPTS='"$EXTRA_CMAKE_OPTS" >> $GITHUB_ENV - name: Setup git submodules run: | git submodule update --init - name: Configure autotools - if: matrix.buildtool == 'autotools' run: | autoreconf -i - ./configure --enable-werror --with-mruby $EXTRA_AUTOTOOLS_OPTS + ./configure - name: Configure cmake if: matrix.buildtool == 'cmake' run: | - cmake -DENABLE_WERROR=1 -DWITH_MRUBY=1 -DWITH_NEVERBLEED=1 $EXTRA_CMAKE_OPTS -DCPPFLAGS="$CPPFLAGS" -DLDFLAGS="$LDFLAGS" . + make dist + VERSION=$(grep PACKAGE_VERSION config.h | cut -d' ' -f3 | tr -d '"') + tar xf nghttp2-$VERSION.tar.gz + cd nghttp2-$VERSION + echo 'NGHTTP2_CMAKE_DIR='"$PWD" >> $GITHUB_ENV + + # This fixes infamous 'stdio.h not found' error. + echo 'SDKROOT='"$(xcrun --sdk macosx --show-sdk-path)" >> $GITHUB_ENV + + cmake -DENABLE_WERROR=1 -DWITH_MRUBY=1 -DWITH_NEVERBLEED=1 -DENABLE_APP=1 $EXTRA_CMAKE_OPTS -DCPPFLAGS="$CPPFLAGS" -DLDFLAGS="$LDFLAGS" . - name: Build nghttp2 with autotools if: matrix.buildtool == 'autotools' run: | make distcheck \ - DISTCHECK_CONFIGURE_FLAGS="--with-mruby --with-neverbleed --enable-werror $EXTRA_AUTOTOOLS_OPTS CPPFLAGS=\"$CPPFLAGS\" LDFLAGS=\"$LDFLAGS\"" + DISTCHECK_CONFIGURE_FLAGS="--with-mruby --with-neverbleed --with-libev --enable-werror $EXTRA_AUTOTOOLS_OPTS CPPFLAGS=\"$CPPFLAGS\" LDFLAGS=\"$LDFLAGS\"" - name: Build nghttp2 with cmake if: matrix.buildtool == 'cmake' run: | + cd $NGHTTP2_CMAKE_DIR make make check - name: Integration test @@ -169,5 +178,5 @@ jobs: # artifacts. if: matrix.buildtool == 'cmake' run: | - cd integration-tests + cd $NGHTTP2_CMAKE_DIR/integration-tests make itprep it diff --git a/AUTHORS b/AUTHORS index 0d48b6eb..a7a9151b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -19,6 +19,7 @@ Alek Storm Alex Nalivko Alexandros Konstantinakis-Karmis Alexis La Goutte +Amir Livneh Amir Pakdel Anders Bakken Andreas Pohl @@ -34,11 +35,13 @@ Bernard Spil Brendan Heinonen Brian Card Brian Suh +Daniel Bevenius Daniel Evers Daniel Stenberg Dave Reisner David Beitey David Weekly +Dmitri Tikhonov Dmitriy Vetutnev Don Dylan Plecki @@ -48,9 +51,12 @@ Fabian Wiesel Gabi Davar Gaël PORTAY Geoff Hill +George Liu Gitai Google Inc. +Hajime Fujita Jacky Tian +Jacky_Yin Jacob Champion James M Snell Jan Kundrát @@ -76,6 +82,7 @@ MATSUMOTO Ryosuke Marc Bachmann Matt Rudary Matt Way +Michael Kaufmann Mike Conlen Mike Frysinger Mike Lothian @@ -127,6 +134,7 @@ es fangdingjun jwchoi kumagi +lhuang04 lstefani makovich mod-h2-dev diff --git a/CMakeLists.txt b/CMakeLists.txt index d7d589ff..e7f2a5a8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,13 +24,13 @@ cmake_minimum_required(VERSION 3.0) # XXX using 1.8.90 instead of 1.9.0-DEV -project(nghttp2 VERSION 1.44.90) +project(nghttp2 VERSION 1.46.90) # See versioning rule: -# http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html -set(LT_CURRENT 34) -set(LT_REVISION 2) -set(LT_AGE 20) +# https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html +set(LT_CURRENT 35) +set(LT_REVISION 1) +set(LT_AGE 21) set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) include(Version) @@ -63,8 +63,16 @@ find_package(Libcares 1.7.5) find_package(ZLIB 1.2.3) find_package(Libngtcp2 0.0.0) find_package(Libngtcp2_crypto_openssl 0.0.0) +if(LIBNGTCP2_CRYPTO_OPENSSL_FOUND) + set(HAVE_LIBNGTCP2_CRYPTO_OPENSSL 1) +endif() find_package(Libnghttp3 0.0.0) -find_package(Libbpf 0.4.0) +if(WITH_LIBBPF) + find_package(Libbpf 0.4.0) + if(NOT LIBBPF_FOUND) + message(FATAL_ERROR "libbpf was requested (WITH_LIBBPF=1) but not found.") + endif() +endif() if(OPENSSL_FOUND AND LIBEV_FOUND AND ZLIB_FOUND) set(ENABLE_APP_DEFAULT ON) else() @@ -171,7 +179,7 @@ endif() # case "$host" in # *android*) # android_build=yes -# # android does not need -pthread, but needs followng 3 libs for C++ +# # android does not need -pthread, but needs following 3 libs for C++ # APPLDFLAGS="$APPLDFLAGS -lstdc++ -latomic -lsupc++" # dl: openssl requires libdl when it is statically linked. diff --git a/CMakeOptions.txt b/CMakeOptions.txt index 754428a2..948e3c63 100644 --- a/CMakeOptions.txt +++ b/CMakeOptions.txt @@ -23,9 +23,8 @@ option(WITH_LIBXML2 "Use libxml2" ${WITH_LIBXML2_DEFAULT}) option(WITH_JEMALLOC "Use jemalloc" ${WITH_JEMALLOC_DEFAULT}) -option(WITH_SPDYLAY "Use spdylay" - ${WITH_SPDYLAY_DEFAULT}) option(WITH_MRUBY "Use mruby") option(WITH_NEVERBLEED "Use neverbleed") +option(WITH_LIBBPF "Use libbpf") # vim: ft=cmake: diff --git a/Makefile.am b/Makefile.am index adc597b1..a5bb7a36 100644 --- a/Makefile.am +++ b/Makefile.am @@ -46,16 +46,20 @@ EXTRA_DIST = nghttpx.conf.sample proxy.pac.sample android-config android-make \ cmake/FindLibevent.cmake \ cmake/FindJansson.cmake \ cmake/FindLibcares.cmake \ - cmake/FindSystemd.cmake + cmake/FindSystemd.cmake \ + cmake/FindLibbpf.cmake \ + cmake/FindLibnghttp3.cmake \ + cmake/FindLibngtcp2.cmake \ + cmake/FindLibngtcp2_crypto_openssl.cmake .PHONY: clang-format # Format source files using clang-format. Don't format source files -# under third-party directory since we are not responsible for thier +# under third-party directory since we are not responsible for their # coding style. clang-format: CLANGFORMAT=`git config --get clangformat.binary`; \ test -z $${CLANGFORMAT} && CLANGFORMAT="clang-format"; \ $${CLANGFORMAT} -i lib/*.{c,h} lib/includes/nghttp2/*.h \ src/*.{c,cc,h} src/includes/nghttp2/*.h examples/*.{c,cc} \ - tests/*.{c,h} + tests/*.{c,h} bpf/*.c diff --git a/README.rst b/README.rst index eaf23d0e..92f9de6a 100644 --- a/README.rst +++ b/README.rst @@ -32,12 +32,14 @@ Public Test Server The following endpoints are available to try out our nghttp2 implementation. -* https://nghttp2.org/ (TLS + ALPN/NPN) +* https://nghttp2.org/ (TLS + ALPN/NPN and HTTP/3) This endpoint supports ``h2``, ``h2-16``, ``h2-14``, and ``http/1.1`` via ALPN/NPN and requires TLSv1.2 for HTTP/2 connection. + It also supports HTTP/3. + * http://nghttp2.org/ (HTTP Upgrade and HTTP/2 Direct) ``h2c`` and ``http/1.1``. @@ -149,16 +151,24 @@ To enable the experimental HTTP/3 support for h2load and nghttpx, the following libraries are required: * `OpenSSL with QUIC support - `_ + `_; or + `BoringSSL `_ (commit + f6ef1c560ae5af51e2df5d8d2175bed207b28b8f) * `ngtcp2 `_ * `nghttp3 `_ +Use ``--enable-http3`` configure option to enable HTTP/3 feature for +h2load and nghttpx. + In order to build optional eBPF program to direct an incoming QUIC UDP datagram to a correct socket for nghttpx, the following libraries are required: * libbpf-dev >= 0.4.0 +Use ``--with-libbpf`` configure option to build eBPF program. +libelf-dev is needed to build libbpf. + For Ubuntu 20.04, you can build libbpf from `the source code `_. nghttpx requires eBPF program for reloading its configuration and hot swapping @@ -326,6 +336,89 @@ The generated documents will not be installed with ``make install``. The online documentation is available at https://nghttp2.org/documentation/ +Build HTTP/3 enabled h2load and nghttpx +--------------------------------------- + +To build h2load and nghttpx with HTTP/3 feature enabled, run the +configure script with ``--enable-http3``. + +For nghttpx to reload configurations and swapping its executable while +gracefully terminating old worker processes, eBPF is required. Run +the configure script with ``--enable-http3 --with-libbpf`` to build +eBPF program. The QUIC keying material must be set with +``--frontend-quic-secret-file`` in order to keep the existing +connections alive during reload. + +The detailed steps to build HTTP/3 enabled h2load and nghttpx follow. + +Build custom OpenSSL: + +.. code-block:: text + + $ git clone --depth 1 -b OpenSSL_1_1_1m+quic https://github.com/quictls/openssl + $ cd openssl + $ ./config --prefix=$PWD/build --openssldir=/etc/ssl + $ make -j$(nproc) + $ make install_sw + $ cd .. + +Build nghttp3: + +.. code-block:: text + + $ git clone https://github.com/ngtcp2/nghttp3 + $ cd nghttp3 + $ git checkout 74a222fe0c89b7202bcdaf6ef27a232edffc85e3 + $ autoreconf -i + $ ./configure --prefix=$PWD/build --enable-lib-only + $ make -j$(nproc) + $ make install + $ cd .. + +Build ngtcp2: + +.. code-block:: text + + $ git clone --depth 1 -b v0.1.0 https://github.com/ngtcp2/ngtcp2 + $ cd ngtcp2 + $ autoreconf -i + $ ./configure --prefix=$PWD/build --enable-lib-only \ + PKG_CONFIG_PATH="$PWD/../openssl/build/lib/pkgconfig" + $ make -j$(nproc) + $ make install + $ cd .. + +If your Linux distribution does not have libbpf-dev >= 0.4.0, build +from source: + +.. code-block:: text + + $ git clone --depth 1 -b v0.4.0 https://github.com/libbpf/libbpf + $ cd libbpf + $ PREFIX=$PWD/build make -C src install + $ cd .. + +Build nghttp2: + +.. code-block:: text + + $ git clone https://github.com/nghttp2/nghttp2 + $ cd nghttp2 + $ git submodule update --init + $ autoreconf -i + $ ./configure --with-mruby --with-neverbleed --enable-http3 --with-libbpf \ + --disable-python-bindings \ + CC=clang-12 CXX=clang++-12 \ + PKG_CONFIG_PATH="$PWD/../openssl/build/lib/pkgconfig:$PWD/../nghttp3/build/lib/pkgconfig:$PWD/../ngtcp2/build/lib/pkgconfig:$PWD/../libbpf/build/lib64/pkgconfig" \ + LDFLAGS="$LDFLAGS -Wl,-rpath,$PWD/../openssl/build/lib -Wl,-rpath,$PWD/../libbpf/build/lib64" + $ make -j$(nproc) + +The eBPF program ``reuseport_kern.o`` should be found under bpf +directory. Pass ``--quic-bpf-program-file=bpf/reuseport_kern.o`` +option to nghttpx to load it. See also `HTTP/3 section in nghttpx - +HTTP/2 proxy - HOW-TO +`_. + Unit tests ---------- @@ -753,7 +846,7 @@ information. Here is sample output from ``nghttpd``: nghttpx - proxy +++++++++++++++ -``nghttpx`` is a multi-threaded reverse proxy for HTTP/2, and +``nghttpx`` is a multi-threaded reverse proxy for HTTP/3, HTTP/2, and HTTP/1.1, and powers http://nghttp2.org and supports HTTP/2 server push. @@ -774,16 +867,16 @@ ticket keys among multiple ``nghttpx`` instances via memcached. ``nghttpx`` has 2 operation modes: -================== ================ ================ ============= -Mode option Frontend Backend Note -================== ================ ================ ============= -default mode HTTP/2, HTTP/1.1 HTTP/1.1, HTTP/2 Reverse proxy -``--http2-proxy`` HTTP/2, HTTP/1.1 HTTP/1.1, HTTP/2 Forward proxy -================== ================ ================ ============= +================== ======================== ================ ============= +Mode option Frontend Backend Note +================== ======================== ================ ============= +default mode HTTP/3, HTTP/2, HTTP/1.1 HTTP/1.1, HTTP/2 Reverse proxy +``--http2-proxy`` HTTP/3, HTTP/2, HTTP/1.1 HTTP/1.1, HTTP/2 Forward proxy +================== ======================== ================ ============= The interesting mode at the moment is the default mode. It works like -a reverse proxy and listens for HTTP/2, and HTTP/1.1 and can be -deployed as a SSL/TLS terminator for existing web server. +a reverse proxy and listens for HTTP/3, HTTP/2, and HTTP/1.1 and can +be deployed as a SSL/TLS terminator for existing web server. In all modes, the frontend connections are encrypted by SSL/TLS by default. To disable encryption, use the ``no-tls`` keyword in @@ -801,16 +894,16 @@ server: .. code-block:: text - Client <-- (HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/1.1, HTTP/2) --> Web Server - [reverse proxy] + Client <-- (HTTP/3, HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/1.1, HTTP/2) --> Web Server + [reverse proxy] With the ``--http2-proxy`` option, it works as forward proxy, and it is so called secure HTTP/2 proxy: .. code-block:: text - Client <-- (HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/1.1) --> Proxy - [secure proxy] (e.g., Squid, ATS) + Client <-- (HTTP/3, HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/1.1) --> Proxy + [secure proxy] (e.g., Squid, ATS) The ``Client`` in the above example needs to be configured to use ``nghttpx`` as secure proxy. @@ -842,7 +935,7 @@ proxy through an HTTP proxy: .. code-block:: text - Client <-- (HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/2) -- + Client <-- (HTTP/3, HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/2) -- --===================---> HTTP/2 Proxy (HTTP proxy tunnel) (e.g., nghttpx -s) @@ -850,8 +943,8 @@ proxy through an HTTP proxy: Benchmarking tool ----------------- -The ``h2load`` program is a benchmarking tool for HTTP/2. The UI of -``h2load`` is heavily inspired by ``weighttp`` +The ``h2load`` program is a benchmarking tool for HTTP/3, HTTP/2, and +HTTP/1.1. The UI of ``h2load`` is heavily inspired by ``weighttp`` (https://github.com/lighttpd/weighttp). The typical usage is as follows: @@ -902,17 +995,6 @@ like so: $ h2load --npn-list h3 https://127.0.0.1:4433 -HTTP/3 ------- - -To build h2load and nghttpx with HTTP/3 feature enabled, run the -configure script with ``--enable-http3``. - -For nghttpx to reload configurations and swapping its executable while -gracefully terminating old worker processes, eBPF is required. Run -the configure script with ``--enable-http3 --with-libbpf`` to build -eBPF program. - HPACK tools ----------- diff --git a/bpf/Makefile.am b/bpf/Makefile.am index 7aafb694..9017fd95 100644 --- a/bpf/Makefile.am +++ b/bpf/Makefile.am @@ -21,9 +21,9 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -if HAVE_LIBBPF +EXTRA_DIST = CMakeLists.txt reuseport_kern.c -EXTRA_DIST = reuseport_kern.c +if HAVE_LIBBPF bpf_pkglibdir = $(pkglibdir) bpf_pkglib_DATA = reuseport_kern.o diff --git a/bpf/reuseport_kern.c b/bpf/reuseport_kern.c index 710ff2ce..7a48afa9 100644 --- a/bpf/reuseport_kern.c +++ b/bpf/reuseport_kern.c @@ -38,6 +38,320 @@ * how to install kernel header files. */ +/* AES_CBC_decrypt_buffer: https://github.com/kokke/tiny-AES-c + License is Public Domain. Commit hash: + 12e7744b4919e9d55de75b7ab566326a1c8e7a67 */ + +#define AES_BLOCKLEN \ + 16 /* Block length in bytes - AES is 128b block \ + only */ + +#define AES_KEYLEN 16 /* Key length in bytes */ +#define AES_keyExpSize 176 + +struct AES_ctx { + __u8 RoundKey[AES_keyExpSize]; +}; + +/* The number of columns comprising a state in AES. This is a constant + in AES. Value=4 */ +#define Nb 4 + +#define Nk 4 /* The number of 32 bit words in a key. */ +#define Nr 10 /* The number of rounds in AES Cipher. */ + +/* state - array holding the intermediate results during + decryption. */ +typedef __u8 state_t[4][4]; + +/* The lookup-tables are marked const so they can be placed in + read-only storage instead of RAM The numbers below can be computed + dynamically trading ROM for RAM - This can be useful in (embedded) + bootloader applications, where ROM is often limited. */ +static const __u8 sbox[256] = { + /* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, + 0xfe, 0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, + 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 0xb7, 0xfd, 0x93, 0x26, + 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, + 0xeb, 0x27, 0xb2, 0x75, 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, + 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 0x53, 0xd1, 0x00, 0xed, + 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, + 0x50, 0x3c, 0x9f, 0xa8, 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, + 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 0xcd, 0x0c, 0x13, 0xec, + 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, + 0xde, 0x5e, 0x0b, 0xdb, 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, + 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 0xe7, 0xc8, 0x37, 0x6d, + 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, + 0x4b, 0xbd, 0x8b, 0x8a, 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, + 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 0xe1, 0xf8, 0x98, 0x11, + 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, + 0xb0, 0x54, 0xbb, 0x16}; + +static const __u8 rsbox[256] = { + 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, + 0x81, 0xf3, 0xd7, 0xfb, 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, + 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 0x54, 0x7b, 0x94, 0x32, + 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, + 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, + 0x6d, 0x8b, 0xd1, 0x25, 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, + 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, 0x6c, 0x70, 0x48, 0x50, + 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, + 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, + 0xb8, 0xb3, 0x45, 0x06, 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, + 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, 0x3a, 0x91, 0x11, 0x41, + 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, + 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, + 0x1c, 0x75, 0xdf, 0x6e, 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, + 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 0xfc, 0x56, 0x3e, 0x4b, + 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, + 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, + 0x27, 0x80, 0xec, 0x5f, 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, + 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, 0xa0, 0xe0, 0x3b, 0x4d, + 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, + 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, + 0x55, 0x21, 0x0c, 0x7d}; + +/* The round constant word array, Rcon[i], contains the values given + by x to the power (i-1) being powers of x (x is denoted as {02}) in + the field GF(2^8) */ +static const __u8 Rcon[11] = {0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, + 0x20, 0x40, 0x80, 0x1b, 0x36}; + +#define getSBoxValue(num) (sbox[(num)]) + +/* This function produces Nb(Nr+1) round keys. The round keys are used + in each round to decrypt the states. */ +static void KeyExpansion(__u8 *RoundKey, const __u8 *Key) { + unsigned i, j, k; + __u8 tempa[4]; /* Used for the column/row operations */ + + /* The first round key is the key itself. */ + for (i = 0; i < Nk; ++i) { + RoundKey[(i * 4) + 0] = Key[(i * 4) + 0]; + RoundKey[(i * 4) + 1] = Key[(i * 4) + 1]; + RoundKey[(i * 4) + 2] = Key[(i * 4) + 2]; + RoundKey[(i * 4) + 3] = Key[(i * 4) + 3]; + } + + /* All other round keys are found from the previous round keys. */ + for (i = Nk; i < Nb * (Nr + 1); ++i) { + { + k = (i - 1) * 4; + tempa[0] = RoundKey[k + 0]; + tempa[1] = RoundKey[k + 1]; + tempa[2] = RoundKey[k + 2]; + tempa[3] = RoundKey[k + 3]; + } + + if (i % Nk == 0) { + /* This function shifts the 4 bytes in a word to the left once. + [a0,a1,a2,a3] becomes [a1,a2,a3,a0] */ + + /* Function RotWord() */ + { + const __u8 u8tmp = tempa[0]; + tempa[0] = tempa[1]; + tempa[1] = tempa[2]; + tempa[2] = tempa[3]; + tempa[3] = u8tmp; + } + + /* SubWord() is a function that takes a four-byte input word and + applies the S-box to each of the four bytes to produce an + output word. */ + + /* Function Subword() */ + { + tempa[0] = getSBoxValue(tempa[0]); + tempa[1] = getSBoxValue(tempa[1]); + tempa[2] = getSBoxValue(tempa[2]); + tempa[3] = getSBoxValue(tempa[3]); + } + + tempa[0] = tempa[0] ^ Rcon[i / Nk]; + } + j = i * 4; + k = (i - Nk) * 4; + RoundKey[j + 0] = RoundKey[k + 0] ^ tempa[0]; + RoundKey[j + 1] = RoundKey[k + 1] ^ tempa[1]; + RoundKey[j + 2] = RoundKey[k + 2] ^ tempa[2]; + RoundKey[j + 3] = RoundKey[k + 3] ^ tempa[3]; + } +} + +static void AES_init_ctx(struct AES_ctx *ctx, const __u8 *key) { + KeyExpansion(ctx->RoundKey, key); +} + +/* This function adds the round key to state. The round key is added + to the state by an XOR function. */ +static void AddRoundKey(__u8 round, state_t *state, const __u8 *RoundKey) { + __u8 i, j; + for (i = 0; i < 4; ++i) { + for (j = 0; j < 4; ++j) { + (*state)[i][j] ^= RoundKey[(round * Nb * 4) + (i * Nb) + j]; + } + } +} + +static __u8 xtime(__u8 x) { return ((x << 1) ^ (((x >> 7) & 1) * 0x1b)); } + +#define Multiply(x, y) \ + (((y & 1) * x) ^ ((y >> 1 & 1) * xtime(x)) ^ \ + ((y >> 2 & 1) * xtime(xtime(x))) ^ \ + ((y >> 3 & 1) * xtime(xtime(xtime(x)))) ^ \ + ((y >> 4 & 1) * xtime(xtime(xtime(xtime(x)))))) + +#define getSBoxInvert(num) (rsbox[(num)]) + +/* MixColumns function mixes the columns of the state matrix. The + method used to multiply may be difficult to understand for the + inexperienced. Please use the references to gain more + information. */ +static void InvMixColumns(state_t *state) { + int i; + __u8 a, b, c, d; + for (i = 0; i < 4; ++i) { + a = (*state)[i][0]; + b = (*state)[i][1]; + c = (*state)[i][2]; + d = (*state)[i][3]; + + (*state)[i][0] = Multiply(a, 0x0e) ^ Multiply(b, 0x0b) ^ Multiply(c, 0x0d) ^ + Multiply(d, 0x09); + (*state)[i][1] = Multiply(a, 0x09) ^ Multiply(b, 0x0e) ^ Multiply(c, 0x0b) ^ + Multiply(d, 0x0d); + (*state)[i][2] = Multiply(a, 0x0d) ^ Multiply(b, 0x09) ^ Multiply(c, 0x0e) ^ + Multiply(d, 0x0b); + (*state)[i][3] = Multiply(a, 0x0b) ^ Multiply(b, 0x0d) ^ Multiply(c, 0x09) ^ + Multiply(d, 0x0e); + } +} + +extern __u32 LINUX_KERNEL_VERSION __kconfig; + +/* The SubBytes Function Substitutes the values in the state matrix + with values in an S-box. */ +static void InvSubBytes(state_t *state) { + __u8 i, j; + if (LINUX_KERNEL_VERSION < KERNEL_VERSION(5, 10, 0)) { + for (i = 0; i < 4; ++i) { + for (j = 0; j < 4; ++j) { + /* Ubuntu 20.04 LTS kernel 5.4.0 needs this workaround + otherwise "math between map_value pointer and register with + unbounded min value is not allowed". 5.10.0 is a kernel + version that works but it might not be the minimum + version. */ + __u8 k = (*state)[j][i]; + (*state)[j][i] = k ? getSBoxInvert(k) : getSBoxInvert(0); + } + } + } else { + for (i = 0; i < 4; ++i) { + for (j = 0; j < 4; ++j) { + (*state)[j][i] = getSBoxInvert((*state)[j][i]); + } + } + } +} + +static void InvShiftRows(state_t *state) { + __u8 temp; + + /* Rotate first row 1 columns to right */ + temp = (*state)[3][1]; + (*state)[3][1] = (*state)[2][1]; + (*state)[2][1] = (*state)[1][1]; + (*state)[1][1] = (*state)[0][1]; + (*state)[0][1] = temp; + + /* Rotate second row 2 columns to right */ + temp = (*state)[0][2]; + (*state)[0][2] = (*state)[2][2]; + (*state)[2][2] = temp; + + temp = (*state)[1][2]; + (*state)[1][2] = (*state)[3][2]; + (*state)[3][2] = temp; + + /* Rotate third row 3 columns to right */ + temp = (*state)[0][3]; + (*state)[0][3] = (*state)[1][3]; + (*state)[1][3] = (*state)[2][3]; + (*state)[2][3] = (*state)[3][3]; + (*state)[3][3] = temp; +} + +static void InvCipher(state_t *state, const __u8 *RoundKey) { + /* Add the First round key to the state before starting the + rounds. */ + AddRoundKey(Nr, state, RoundKey); + + /* There will be Nr rounds. The first Nr-1 rounds are identical. + These Nr rounds are executed in the loop below. Last one without + InvMixColumn() */ + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 1, state, RoundKey); + InvMixColumns(state); + + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 2, state, RoundKey); + InvMixColumns(state); + + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 3, state, RoundKey); + InvMixColumns(state); + + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 4, state, RoundKey); + InvMixColumns(state); + + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 5, state, RoundKey); + InvMixColumns(state); + + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 6, state, RoundKey); + InvMixColumns(state); + + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 7, state, RoundKey); + InvMixColumns(state); + + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 8, state, RoundKey); + InvMixColumns(state); + + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 9, state, RoundKey); + InvMixColumns(state); + + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 10, state, RoundKey); +} + +static void AES_ECB_decrypt(const struct AES_ctx *ctx, __u8 *buf) { + /* The next function call decrypts the PlainText with the Key using + AES algorithm. */ + InvCipher((state_t *)buf, ctx->RoundKey); +} + /* rol32: From linux kernel source code */ /** @@ -125,13 +439,13 @@ struct bpf_map_def SEC("maps") reuseport_array = { struct bpf_map_def SEC("maps") sk_info = { .type = BPF_MAP_TYPE_ARRAY, - .max_entries = 1, + .max_entries = 3, .key_size = sizeof(__u32), - .value_size = sizeof(__u32), + .value_size = sizeof(__u64), }; typedef struct quic_hd { - const __u8 *dcid; + __u8 *dcid; __u32 dcidlen; __u32 dcid_offset; __u8 type; @@ -141,6 +455,7 @@ typedef struct quic_hd { #define MAX_DCIDLEN 20 #define MIN_DCIDLEN 8 #define CID_PREFIXLEN 8 +#define CID_PREFIX_OFFSET 1 enum { NGTCP2_PKT_INITIAL = 0x0, @@ -149,9 +464,8 @@ enum { NGTCP2_PKT_SHORT = 0x40, }; -static inline int parse_quic(quic_hd *qhd, const __u8 *data, - const __u8 *data_end) { - const __u8 *p; +static inline int parse_quic(quic_hd *qhd, __u8 *data, __u8 *data_end) { + __u8 *p; __u64 dcidlen; if (*data & 0x80) { @@ -192,7 +506,7 @@ static __u32 hash(const __u8 *data, __u32 datalen, __u32 initval) { static __u32 sk_index_from_dcid(const quic_hd *qhd, const struct sk_reuseport_md *reuse_md, - __u32 num_socks) { + __u64 num_socks) { __u32 len = qhd->dcidlen; __u32 h = reuse_md->hash; __u8 hbuf[8]; @@ -259,11 +573,14 @@ static __u32 sk_index_from_dcid(const quic_hd *qhd, SEC("sk_reuseport") int select_reuseport(struct sk_reuseport_md *reuse_md) { __u32 sk_index, *psk_index; - __u32 *pnum_socks; - __u32 zero = 0; + __u64 *pnum_socks, *pkey; + __u32 zero = 0, key_high_idx = 1, key_low_idx = 2; int rv; quic_hd qhd; __u8 qpktbuf[6 + MAX_DCIDLEN]; + struct AES_ctx aes_ctx; + __u8 key[AES_KEYLEN]; + __u8 *cid_prefix; if (bpf_skb_load_bytes(reuse_md, sizeof(struct udphdr), qpktbuf, sizeof(qpktbuf)) != 0) { @@ -275,16 +592,35 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) { return SK_DROP; } + pkey = bpf_map_lookup_elem(&sk_info, &key_high_idx); + if (pkey == NULL) { + return SK_DROP; + } + + __builtin_memcpy(key, pkey, sizeof(*pkey)); + + pkey = bpf_map_lookup_elem(&sk_info, &key_low_idx); + if (pkey == NULL) { + return SK_DROP; + } + + __builtin_memcpy(key + sizeof(*pkey), pkey, sizeof(*pkey)); + rv = parse_quic(&qhd, qpktbuf, qpktbuf + sizeof(qpktbuf)); if (rv != 0) { return SK_DROP; } + AES_init_ctx(&aes_ctx, key); + switch (qhd.type) { case NGTCP2_PKT_INITIAL: case NGTCP2_PKT_0RTT: if (qhd.dcidlen == SV_DCIDLEN) { - psk_index = bpf_map_lookup_elem(&cid_prefix_map, qhd.dcid); + cid_prefix = qhd.dcid + CID_PREFIX_OFFSET; + AES_ECB_decrypt(&aes_ctx, cid_prefix); + + psk_index = bpf_map_lookup_elem(&cid_prefix_map, cid_prefix); if (psk_index != NULL) { sk_index = *psk_index; @@ -301,7 +637,10 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) { return SK_DROP; } - psk_index = bpf_map_lookup_elem(&cid_prefix_map, qhd.dcid); + cid_prefix = qhd.dcid + CID_PREFIX_OFFSET; + AES_ECB_decrypt(&aes_ctx, cid_prefix); + + psk_index = bpf_map_lookup_elem(&cid_prefix_map, cid_prefix); if (psk_index == NULL) { sk_index = sk_index_from_dcid(&qhd, reuse_md, *pnum_socks); diff --git a/cmakeconfig.h.in b/cmakeconfig.h.in index 10bb76cb..2e357d53 100644 --- a/cmakeconfig.h.in +++ b/cmakeconfig.h.in @@ -87,3 +87,6 @@ /* Define to 1 if you have enum bpf_stats_type in linux/bpf.h. */ #cmakedefine HAVE_BPF_STATS_TYPE 1 + +/* Define to 1 if you have `libngtcp2_crypto_openssl` library. */ +#cmakedefine HAVE_LIBNGTCP2_CRYPTO_OPENSSL diff --git a/configure.ac b/configure.ac index b4b1e605..06f4ae71 100644 --- a/configure.ac +++ b/configure.ac @@ -22,10 +22,10 @@ dnl OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION dnl WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. dnl Do not change user variables! -dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html +dnl https://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.45.0-DEV], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.47.0-DEV], [t-tujikawa@users.sourceforge.net]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_HEADERS([config.h]) @@ -43,10 +43,10 @@ AM_INIT_AUTOMAKE([subdir-objects]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) dnl See versioning rule: -dnl http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html -AC_SUBST(LT_CURRENT, 34) -AC_SUBST(LT_REVISION, 2) -AC_SUBST(LT_AGE, 20) +dnl https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html +AC_SUBST(LT_CURRENT, 35) +AC_SUBST(LT_REVISION, 1) +AC_SUBST(LT_AGE, 21) major=`echo $PACKAGE_VERSION |cut -d. -f1 | sed -e "s/[^0-9]//g"` minor=`echo $PACKAGE_VERSION |cut -d. -f2 | sed -e "s/[^0-9]//g"` @@ -220,6 +220,11 @@ PKG_PROG_PKG_CONFIG([0.20]) AM_PATH_PYTHON([3.8],, [:]) +if test "x$request_python_bindings" = "xyes" && + test "x$PYTHON" = "x:"; then + AC_MSG_ERROR([python was requested (enable-python-bindings) but not found]) +fi + if [test "x$request_lib_only" = "xyes"]; then request_app=no request_hpack_tools=no @@ -227,8 +232,10 @@ if [test "x$request_lib_only" = "xyes"]; then request_python_bindings=no fi -if [test "x$request_python_bindings" != "xno"]; then - AX_PYTHON_DEVEL([>= '3.8']) +if test "x$request_python_bindings" != "xno" && + test "x$PYTHON" != "x:"; then + # version check is broken + AX_PYTHON_DEVEL() fi if test "x${cython_path}" = "x"; then @@ -342,7 +349,7 @@ APPLDFLAGS= case "$host_os" in *android*) android_build=yes - # android does not need -pthread, but needs followng 3 libs for C++ + # android does not need -pthread, but needs following 3 libs for C++ APPLDFLAGS="$APPLDFLAGS -lstdc++ -latomic -lsupc++" ;; *) @@ -467,6 +474,7 @@ if test "x${request_openssl}" != "xno"; then CFLAGS="$OPENSSL_CFLAGS $CFLAGS" LIBS="$OPENSSL_LIBS $LIBS" + # quictls/openssl has SSL_is_quic. have_ssl_is_quic=no AC_MSG_CHECKING([for SSL_is_quic]) AC_LINK_IFELSE([AC_LANG_PROGRAM([[ @@ -478,6 +486,17 @@ if test "x${request_openssl}" != "xno"; then [AC_MSG_RESULT([yes]); have_ssl_is_quic=yes], [AC_MSG_RESULT([no]); have_ssl_is_quic=no]) + # boringssl has SSL_set_quic_early_data_context. + AC_MSG_CHECKING([for SSL_set_quic_early_data_context]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([[ + #include + ]], [[ + SSL *ssl = NULL; + SSL_set_quic_early_data_context(ssl, NULL, 0); + ]])], + [AC_MSG_RESULT([yes]); have_boringssl_quic=yes], + [AC_MSG_RESULT([no]); have_boringssl_quic=no]) + CFLAGS="$save_CFLAGS" LIBS="$save_LIBS" fi @@ -506,7 +525,7 @@ fi # ngtcp2 (for src) have_libngtcp2=no if test "x${request_libngtcp2}" != "xno"; then - PKG_CHECK_MODULES([LIBNGTCP2], [libngtcp2 >= 0.0.0], [have_libngtcp2=yes], + PKG_CHECK_MODULES([LIBNGTCP2], [libngtcp2 >= 0.1.0], [have_libngtcp2=yes], [have_libngtcp2=no]) if test "x${have_libngtcp2}" = "xno"; then AC_MSG_NOTICE($LIBNGTCP2_PKG_ERRORS) @@ -520,25 +539,52 @@ fi # ngtcp2_crypto_openssl (for src) have_libngtcp2_crypto_openssl=no -if test "x${request_libngtcp2}" != "xno"; then +if test "x${have_ssl_is_quic}" = "xyes" && + test "x${request_libngtcp2}" != "xno"; then PKG_CHECK_MODULES([LIBNGTCP2_CRYPTO_OPENSSL], [libngtcp2_crypto_openssl >= 0.0.0], [have_libngtcp2_crypto_openssl=yes], [have_libngtcp2_crypto_openssl=no]) if test "x${have_libngtcp2_crypto_openssl}" = "xno"; then AC_MSG_NOTICE($LIBNGTCP2_CRYPTO_OPENSSL_PKG_ERRORS) + else + AC_DEFINE([HAVE_LIBNGTCP2_CRYPTO_OPENSSL], [1], + [Define to 1 if you have `libngtcp2_crypto_openssl` library.]) fi fi -if test "x${request_libngtcp2}" = "xyes" && +if test "x${have_ssl_is_quic}" = "xyes" && + test "x${request_libngtcp2}" = "xyes" && test "x${have_libngtcp2_crypto_openssl}" != "xyes"; then AC_MSG_ERROR([libngtcp2_crypto_openssl was requested (--with-libngtcp2) but not found]) fi +# ngtcp2_crypto_boringssl (for src) +have_libngtcp2_crypto_boringssl=no +if test "x${have_boringssl_quic}" = "xyes" && + test "x${request_libngtcp2}" != "xno"; then + PKG_CHECK_MODULES([LIBNGTCP2_CRYPTO_BORINGSSL], + [libngtcp2_crypto_boringssl >= 0.0.0], + [have_libngtcp2_crypto_boringssl=yes], + [have_libngtcp2_crypto_boringssl=no]) + if test "x${have_libngtcp2_crypto_boringssl}" = "xno"; then + AC_MSG_NOTICE($LIBNGTCP2_CRYPTO_BORINGSSL_PKG_ERRORS) + else + AC_DEFINE([HAVE_LIBNGTCP2_CRYPTO_BORINGSSL], [1], + [Define to 1 if you have `libngtcp2_crypto_boringssl` library.]) + fi +fi + +if test "x${have_boringssl_quic}" = "xyes" && + test "x${request_libngtcp2}" = "xyes" && + test "x${have_libngtcp2_crypto_boringssl}" != "xyes"; then + AC_MSG_ERROR([libngtcp2_crypto_boringssl was requested (--with-libngtcp2) but not found]) +fi + # nghttp3 (for src) have_libnghttp3=no if test "x${request_libnghttp3}" != "xno"; then - PKG_CHECK_MODULES([LIBNGHTTP3], [libnghttp3 >= 0.0.0], [have_libnghttp3=yes], + PKG_CHECK_MODULES([LIBNGHTTP3], [libnghttp3 >= 0.1.0], [have_libnghttp3=yes], [have_libnghttp3=no]) if test "x${have_libnghttp3}" = "xno"; then AC_MSG_NOTICE($LIBNGHTTP3_PKG_ERRORS) @@ -742,9 +788,11 @@ AM_CONDITIONAL([ENABLE_APP], [ test "x${enable_app}" = "xyes" ]) # Check HTTP/3 support enable_http3=no if test "x${request_http3}" != "xno" && - test "x${have_ssl_is_quic}" = "xyes" && + (test "x${have_ssl_is_quic}" = "xyes" || + test "x${have_boringssl_quic}" = "xyes") && test "x${have_libngtcp2}" = "xyes" && - test "x${have_libngtcp2_crypto_openssl}" = "xyes" && + (test "x${have_libngtcp2_crypto_openssl}" = "xyes" || + test "x${have_libngtcp2_crypto_boringssl}" = "xyes") && test "x${have_libnghttp3}" = "xyes"; then enable_http3=yes AC_DEFINE([ENABLE_HTTP3], [1], [Define to 1 if HTTP/3 is enabled.]) @@ -983,31 +1031,6 @@ AC_CHECK_DECLS([initgroups], [], [], [[ #include ]]) -have_netinet_udp_h_udp_segment=no -AC_CHECK_DECL([UDP_SEGMENT], [have_netinet_udp_h_udp_segment=yes], - [have_netinet_udp_h_udp_segment=no], [[ - #include -]]) - -if test "x$have_netinet_udp_h_udp_segment" = "xno"; then - have_linux_udp_h_udp_segment=no - AC_COMPILE_IFELSE([AC_LANG_PROGRAM( - [[ - #include - ]], - [[ - #if UDP_SEGMENT != 103 - exit(1) - #endif - ]])], - [have_linux_udp_h_udp_segment=yes], - [have_linux_udp_h_udp_segment=no]) - - if test "x$have_linux_udp_h_udp_segment" = "xyes"; then - EXTRA_DEFS="$EXTRA_DEFS -DUDP_SEGMENT=103" - fi -fi - save_CFLAGS=$CFLAGS save_CXXFLAGS=$CXXFLAGS @@ -1202,6 +1225,7 @@ AC_MSG_NOTICE([summary of build options: Libc-ares: ${have_libcares} (CFLAGS='${LIBCARES_CFLAGS}' LIBS='${LIBCARES_LIBS}') libngtcp2: ${have_libngtcp2} (CFLAGS='${LIBNGTCP2_CFLAGS}' LIBS='${LIBNGTCP2_LIBS}') libngtcp2_crypto_openssl: ${have_libngtcp2_crypto_openssl} (CFLAGS='${LIBNGTCP2_CRYPTO_OPENSSL_CFLAGS}' LIBS='${LIBNGTCP2_CRYPTO_OPENSSL_LIBS}') + libngtcp2_crypto_boringssl: ${have_libngtcp2_crypto_boringssl} (CFLAGS='${LIBNGTCP2_CRYPTO_BORINGSSL_CFLAGS}' LIBS='${LIBNGTCP2_CRYPTO_BORINGSSL_LIBS}') libnghttp3: ${have_libnghttp3} (CFLAGS='${LIBNGHTTP3_CFLAGS}' LIBS='${LIBNGHTTP3_LIBS}') libbpf: ${have_libbpf} (CFLAGS='${LIBBPF_CFLAGS}' LIBS='${LIBBPF_LIBS}') Libevent(SSL): ${have_libevent_openssl} (CFLAGS='${LIBEVENT_OPENSSL_CFLAGS}' LIBS='${LIBEVENT_OPENSSL_LIBS}') diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index f3aec84d..06677731 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -184,9 +184,9 @@ set(EXTRA_DIST sources/python-apiref.rst sources/building-android-binary.rst sources/contribute.rst - _exts/sphinxcontrib/LICENSE.rubydomain - _exts/sphinxcontrib/__init__.py - _exts/sphinxcontrib/rubydomain.py + _exts/rubydomain/LICENSE.rubydomain + _exts/rubydomain/__init__.py + _exts/rubydomain/rubydomain.py _themes/sphinx_rtd_theme/__init__.py _themes/sphinx_rtd_theme/breadcrumbs.html _themes/sphinx_rtd_theme/footer.html diff --git a/doc/Makefile.am b/doc/Makefile.am index 13317991..ebbd9078 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -206,9 +206,9 @@ EXTRA_DIST = \ sources/building-android-binary.rst \ sources/contribute.rst \ sources/security.rst \ - _exts/sphinxcontrib/LICENSE.rubydomain \ - _exts/sphinxcontrib/__init__.py \ - _exts/sphinxcontrib/rubydomain.py \ + _exts/rubydomain/LICENSE.rubydomain \ + _exts/rubydomain/__init__.py \ + _exts/rubydomain/rubydomain.py \ _themes/sphinx_rtd_theme/__init__.py \ _themes/sphinx_rtd_theme/breadcrumbs.html \ _themes/sphinx_rtd_theme/footer.html \ @@ -272,7 +272,7 @@ EXTRA_DIST = \ # You can set these variables from the command line. SPHINXOPTS = -SPHINXBUILD = sphinx-build +SPHINXBUILD ?= sphinx-build PAPER = BUILDDIR = manual diff --git a/doc/_exts/sphinxcontrib/LICENSE.rubydomain b/doc/_exts/rubydomain/LICENSE.rubydomain similarity index 100% rename from doc/_exts/sphinxcontrib/LICENSE.rubydomain rename to doc/_exts/rubydomain/LICENSE.rubydomain diff --git a/doc/_exts/sphinxcontrib/__init__.py b/doc/_exts/rubydomain/__init__.py similarity index 100% rename from doc/_exts/sphinxcontrib/__init__.py rename to doc/_exts/rubydomain/__init__.py diff --git a/doc/_exts/sphinxcontrib/rubydomain.py b/doc/_exts/rubydomain/rubydomain.py similarity index 99% rename from doc/_exts/sphinxcontrib/rubydomain.py rename to doc/_exts/rubydomain/rubydomain.py index 62a04285..db352334 100644 --- a/doc/_exts/sphinxcontrib/rubydomain.py +++ b/doc/_exts/rubydomain/rubydomain.py @@ -493,7 +493,7 @@ class RubyModuleIndex(Index): # list of all modules, sorted by module name modules = sorted(_iteritems(self.domain.data['modules']), key=lambda x: x[0].lower()) - # sort out collapsable modules + # sort out collapsible modules prev_modname = '' num_toplevels = 0 for modname, (docname, synopsis, platforms, deprecated) in modules: diff --git a/doc/bash_completion/h2load b/doc/bash_completion/h2load index f90dbe2f..bb0179b3 100644 --- a/doc/bash_completion/h2load +++ b/doc/bash_completion/h2load @@ -8,7 +8,7 @@ _h2load() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--connection-window-bits --clients --verbose --ciphers --rate --no-tls-proto --connect-to --header-table-size --requests --log-file --base-uri --no-udp-gso --h1 --threads --npn-list --rate-period --rps --data --tls13-ciphers --version --connection-inactivity-timeout --timing-script-file --encoder-header-table-size --max-concurrent-streams --connection-active-timeout --input-file --help --groups --window-bits --qlog-file-base --warm-up-time --duration --header ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--requests --clients --threads --input-file --max-concurrent-streams --window-bits --connection-window-bits --header --ciphers --tls13-ciphers --no-tls-proto --data --rate --rate-period --duration --warm-up-time --connection-active-timeout --connection-inactivity-timeout --timing-script-file --base-uri --npn-list --h1 --header-table-size --encoder-header-table-size --log-file --qlog-file-base --connect-to --rps --groups --no-udp-gso --max-udp-payload-size --verbose --version --help ' -- "$cur" ) ) ;; *) _filedir diff --git a/doc/bash_completion/make_bash_completion.py b/doc/bash_completion/make_bash_completion.py index 55cbfce8..e3a535ba 100755 --- a/doc/bash_completion/make_bash_completion.py +++ b/doc/bash_completion/make_bash_completion.py @@ -1,8 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- -from __future__ import unicode_literals -from __future__ import print_function import subprocess import io import re diff --git a/doc/bash_completion/nghttp b/doc/bash_completion/nghttp index 9a4cf190..d7d66b42 100644 --- a/doc/bash_completion/nghttp +++ b/doc/bash_completion/nghttp @@ -8,7 +8,7 @@ _nghttp() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--no-push --verbose --no-dep --get-assets --har --header-table-size --multiply --encoder-header-table-size --padding --hexdump --max-concurrent-streams --continuation --connection-window-bits --peer-max-concurrent-streams --timeout --data --no-content-length --version --color --cert --upgrade --remote-name --trailer --weight --help --key --null-out --window-bits --expect-continue --stat --no-verify-peer --header ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--verbose --null-out --remote-name --timeout --window-bits --connection-window-bits --get-assets --stat --header --trailer --cert --key --data --multiply --upgrade --weight --peer-max-concurrent-streams --header-table-size --encoder-header-table-size --padding --har --color --continuation --no-content-length --no-dep --hexdump --no-push --max-concurrent-streams --expect-continue --no-verify-peer --version --help ' -- "$cur" ) ) ;; *) _filedir diff --git a/doc/bash_completion/nghttpd b/doc/bash_completion/nghttpd index 92d1dfb5..204e7db0 100644 --- a/doc/bash_completion/nghttpd +++ b/doc/bash_completion/nghttpd @@ -8,7 +8,7 @@ _nghttpd() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--htdocs --verbose --daemon --echo-upload --error-gzip --push --header-table-size --encoder-header-table-size --padding --hexdump --max-concurrent-streams --no-tls --connection-window-bits --mime-types-file --no-content-length --workers --version --color --early-response --dh-param-file --trailer --address --window-bits --verify-client --help ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--address --daemon --verify-client --htdocs --verbose --no-tls --header-table-size --encoder-header-table-size --color --push --padding --max-concurrent-streams --workers --error-gzip --window-bits --connection-window-bits --dh-param-file --early-response --trailer --hexdump --echo-upload --mime-types-file --no-content-length --version --help ' -- "$cur" ) ) ;; *) _filedir diff --git a/doc/bash_completion/nghttpx b/doc/bash_completion/nghttpx index 0e95f54b..89371abe 100644 --- a/doc/bash_completion/nghttpx +++ b/doc/bash_completion/nghttpx @@ -8,7 +8,7 @@ _nghttpx() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--worker-read-rate --include --frontend-http2-dump-response-header --tls-ticket-key-file --verify-client-cacert --max-response-header-fields --backend-http2-window-size --tls13-client-ciphers --frontend-keep-alive-timeout --backend-request-buffer --max-request-header-fields --backend-connect-timeout --tls-max-proto-version --frontend-quic-idle-timeout --frontend-http3-read-timeout --conf --dns-lookup-timeout --frontend-http3-connection-window-size --backend-http2-max-concurrent-streams --worker-write-burst --no-quic-bpf --npn-list --dns-max-try --fetch-ocsp-response-file --tls-session-cache-memcached-cert-file --no-via --mruby-file --no-server-push --stream-read-timeout --client-ciphers --ocsp-update-interval --forwarded-for --accesslog-syslog --dns-cache-timeout --frontend-http2-read-timeout --listener-disable-timeout --frontend-http3-window-size --ciphers --client-psk-secrets --strip-incoming-x-forwarded-for --no-server-rewrite --private-key-passwd-file --backend-keep-alive-timeout --backend-http-proxy-uri --frontend-max-requests --tls-no-postpone-early-data --rlimit-nofile --no-strip-incoming-x-forwarded-proto --tls-ticket-key-memcached-cert-file --no-verify-ocsp --forwarded-by --tls-session-cache-memcached-private-key-file --error-page --ocsp-startup --backend-write-timeout --tls-dyn-rec-warmup-threshold --frontend-http2-window-size --tls-ticket-key-memcached-max-retry --http2-no-cookie-crumbling --frontend-http3-max-concurrent-streams --worker-read-burst --dh-param-file --accesslog-format --errorlog-syslog --redirect-https-port --request-header-field-buffer --api-max-request-body --frontend-http2-decoder-dynamic-table-size --errorlog-file --frontend-http2-max-concurrent-streams --psk-secrets --frontend-write-timeout --tls-ticket-key-cipher --read-burst --no-add-x-forwarded-proto --backend --server-name --insecure --backend-max-backoff --log-level --host-rewrite --quic-bpf-program-file --http2-altsvc --frontend-http3-max-connection-window-size --tls-ticket-key-memcached-interval --frontend-http2-setting-timeout --frontend-http2-connection-window-size --worker-frontend-connections --syslog-facility --fastopen --no-location-rewrite --single-thread --no-http2-cipher-block-list --tls-session-cache-memcached --no-ocsp --backend-response-buffer --tls-min-proto-version --workers --add-x-forwarded-for --add-forwarded --frontend-http3-max-window-size --worker-write-rate --add-request-header --backend-http2-settings-timeout --subcert --ignore-per-pattern-mruby-error --ecdh-curves --no-kqueue --help --frontend-frame-debug --tls-sct-dir --pid-file --frontend-http2-dump-request-header --daemon --write-rate --altsvc --backend-http2-decoder-dynamic-table-size --no-strip-incoming-early-data --user --verify-client-tolerate-expired --frontend-read-timeout --tls-ticket-key-memcached-max-fail --backlog --write-burst --backend-connections-per-host --tls-max-early-data --response-header-field-buffer --tls-ticket-key-memcached-address-family --padding --tls-session-cache-memcached-address-family --stream-write-timeout --cacert --frontend-quic-debug-log --tls-ticket-key-memcached-private-key-file --accesslog-write-early --backend-address-family --backend-http2-connection-window-size --tls13-ciphers --client-no-http2-cipher-block-list --version --add-response-header --backend-read-timeout --frontend-http2-optimize-window-size --frontend --accesslog-file --http2-proxy --backend-http2-encoder-dynamic-table-size --client-private-key-file --single-process --client-cert-file --tls-ticket-key-memcached --tls-dyn-rec-idle-timeout --frontend-http2-optimize-write-buffer-size --verify-client --frontend-http2-encoder-dynamic-table-size --read-rate --backend-connections-per-frontend --strip-incoming-forwarded ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--backend --frontend --backlog --backend-address-family --backend-http-proxy-uri --workers --single-thread --read-rate --read-burst --write-rate --write-burst --worker-read-rate --worker-read-burst --worker-write-rate --worker-write-burst --worker-frontend-connections --backend-connections-per-host --backend-connections-per-frontend --rlimit-nofile --rlimit-memlock --backend-request-buffer --backend-response-buffer --fastopen --no-kqueue --frontend-http2-read-timeout --frontend-http3-read-timeout --frontend-read-timeout --frontend-write-timeout --frontend-keep-alive-timeout --stream-read-timeout --stream-write-timeout --backend-read-timeout --backend-write-timeout --backend-connect-timeout --backend-keep-alive-timeout --listener-disable-timeout --frontend-http2-setting-timeout --backend-http2-settings-timeout --backend-max-backoff --ciphers --tls13-ciphers --client-ciphers --tls13-client-ciphers --ecdh-curves --insecure --cacert --private-key-passwd-file --subcert --dh-param-file --npn-list --verify-client --verify-client-cacert --verify-client-tolerate-expired --client-private-key-file --client-cert-file --tls-min-proto-version --tls-max-proto-version --tls-ticket-key-file --tls-ticket-key-memcached --tls-ticket-key-memcached-address-family --tls-ticket-key-memcached-interval --tls-ticket-key-memcached-max-retry --tls-ticket-key-memcached-max-fail --tls-ticket-key-cipher --tls-ticket-key-memcached-cert-file --tls-ticket-key-memcached-private-key-file --fetch-ocsp-response-file --ocsp-update-interval --ocsp-startup --no-verify-ocsp --no-ocsp --tls-session-cache-memcached --tls-session-cache-memcached-address-family --tls-session-cache-memcached-cert-file --tls-session-cache-memcached-private-key-file --tls-dyn-rec-warmup-threshold --tls-dyn-rec-idle-timeout --no-http2-cipher-block-list --client-no-http2-cipher-block-list --tls-sct-dir --psk-secrets --client-psk-secrets --tls-no-postpone-early-data --tls-max-early-data --frontend-http2-max-concurrent-streams --backend-http2-max-concurrent-streams --frontend-http2-window-size --frontend-http2-connection-window-size --backend-http2-window-size --backend-http2-connection-window-size --http2-no-cookie-crumbling --padding --no-server-push --frontend-http2-optimize-write-buffer-size --frontend-http2-optimize-window-size --frontend-http2-encoder-dynamic-table-size --frontend-http2-decoder-dynamic-table-size --backend-http2-encoder-dynamic-table-size --backend-http2-decoder-dynamic-table-size --http2-proxy --log-level --accesslog-file --accesslog-syslog --accesslog-format --accesslog-write-early --errorlog-file --errorlog-syslog --syslog-facility --add-x-forwarded-for --strip-incoming-x-forwarded-for --no-add-x-forwarded-proto --no-strip-incoming-x-forwarded-proto --add-forwarded --strip-incoming-forwarded --forwarded-by --forwarded-for --no-via --no-strip-incoming-early-data --no-location-rewrite --host-rewrite --altsvc --http2-altsvc --add-request-header --add-response-header --request-header-field-buffer --max-request-header-fields --response-header-field-buffer --max-response-header-fields --error-page --server-name --no-server-rewrite --redirect-https-port --api-max-request-body --dns-cache-timeout --dns-lookup-timeout --dns-max-try --frontend-max-requests --frontend-http2-dump-request-header --frontend-http2-dump-response-header --frontend-frame-debug --daemon --pid-file --user --single-process --max-worker-processes --worker-process-grace-shutdown-period --mruby-file --ignore-per-pattern-mruby-error --frontend-quic-idle-timeout --frontend-quic-debug-log --quic-bpf-program-file --frontend-quic-early-data --frontend-quic-qlog-dir --frontend-quic-require-token --frontend-quic-congestion-controller --frontend-quic-secret-file --quic-server-id --frontend-quic-initial-rtt --no-quic-bpf --frontend-http3-window-size --frontend-http3-connection-window-size --frontend-http3-max-window-size --frontend-http3-max-connection-window-size --frontend-http3-max-concurrent-streams --conf --include --version --help ' -- "$cur" ) ) ;; *) _filedir diff --git a/doc/conf.py.in b/doc/conf.py.in index bc6401ec..d6a82b6c 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -41,7 +41,7 @@ import sys, os # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) -sys.path.append(os.path.abspath('@top_srcdir@/doc/_exts')) +sys.path.insert(0, os.path.abspath('@top_srcdir@/doc/_exts')) # -- General configuration ----------------------------------------------------- @@ -50,7 +50,7 @@ sys.path.append(os.path.abspath('@top_srcdir@/doc/_exts')) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinxcontrib.rubydomain'] +extensions = ['rubydomain.rubydomain'] # Add any paths that contain templates here, relative to this directory. templates_path = ['@top_srcdir@/_templates'] diff --git a/doc/h2load.1 b/doc/h2load.1 index eb8d7b43..2b6c2e59 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "Aug 31, 2021" "1.45.0-DEV" "nghttp2" +.TH "H2LOAD" "1" "Oct 19, 2021" "1.46.0" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . @@ -328,6 +328,11 @@ Disable UDP GSO. .UNINDENT .INDENT 0.0 .TP +.B \-\-max\-udp\-payload\-size= +Specify the maximum outgoing UDP datagram payload size. +.UNINDENT +.INDENT 0.0 +.TP .B \-v, \-\-verbose Output debug information. .UNINDENT diff --git a/doc/h2load.1.rst b/doc/h2load.1.rst index c4e733f2..66173807 100644 --- a/doc/h2load.1.rst +++ b/doc/h2load.1.rst @@ -277,6 +277,10 @@ OPTIONS Disable UDP GSO. +.. option:: --max-udp-payload-size= + + Specify the maximum outgoing UDP datagram payload size. + .. option:: -v, --verbose Output debug information. diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 72f579b8..4148abfb 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "Aug 31, 2021" "1.45.0-DEV" "nghttp2" +.TH "NGHTTP" "1" "Oct 19, 2021" "1.46.0" "nghttp2" .SH NAME nghttp \- HTTP/2 client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index 9a4efc7b..25d4f494 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "Aug 31, 2021" "1.45.0-DEV" "nghttp2" +.TH "NGHTTPD" "1" "Oct 19, 2021" "1.46.0" "nghttp2" .SH NAME nghttpd \- HTTP/2 server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index 2be0a35b..b9e0eb3b 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "Aug 31, 2021" "1.45.0-DEV" "nghttp2" +.TH "NGHTTPX" "1" "Oct 19, 2021" "1.46.0" "nghttp2" .SH NAME nghttpx \- HTTP/2 proxy . @@ -35,7 +35,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] \fBnghttpx\fP [OPTIONS]... [ ] .SH DESCRIPTION .sp -A reverse proxy for HTTP/2, and HTTP/1. +A reverse proxy for HTTP/3, HTTP/2, and HTTP/1. .INDENT 0.0 .TP .B @@ -503,6 +503,15 @@ Default: \fB0\fP .UNINDENT .INDENT 0.0 .TP +.B \-\-rlimit\-memlock= +Set maximum number of bytes of memory that may be locked +into RAM. If 0 is given, nghttpx does not set the +limit. +.sp +Default: \fB0\fP +.UNINDENT +.INDENT 0.0 +.TP .B \-\-backend\-request\-buffer= Set buffer size used to store backend request. .sp @@ -1089,12 +1098,13 @@ option. But be aware its implications. .INDENT 0.0 .TP .B \-\-tls\-no\-postpone\-early\-data -By default, nghttpx postpones forwarding HTTP requests -sent in early data, including those sent in partially in -it, until TLS handshake finishes. If all backend server -recognizes "Early\-Data" header field, using this option -makes nghttpx not postpone forwarding request and get -full potential of 0\-RTT data. +By default, except for QUIC connections, nghttpx +postpones forwarding HTTP requests sent in early data, +including those sent in partially in it, until TLS +handshake finishes. If all backend server recognizes +"Early\-Data" header field, using this option makes +nghttpx not postpone forwarding request and get full +potential of 0\-RTT data. .UNINDENT .INDENT 0.0 .TP @@ -1507,7 +1517,7 @@ they are treated as nothing is specified. They are advertised in alt\-svc header field only in HTTP/1.1 frontend. This option can be used multiple times to specify multiple alternative services. -Example: \fI\%\-\-altsvc\fP="h2,443,,,ma=3600; persist=1\(aq +Example: \fI\%\-\-altsvc\fP="h2,443,,,ma=3600; persist=1" .UNINDENT .INDENT 0.0 .TP @@ -1704,6 +1714,33 @@ process. nghttpx still spawns additional process if neverbleed is used. In the single process mode, the signal handling feature is disabled. .UNINDENT +.INDENT 0.0 +.TP +.B \-\-max\-worker\-processes= +The maximum number of worker processes. nghttpx spawns +new worker process when it reloads its configuration. +The previous worker process enters graceful termination +period and will terminate when it finishes handling the +existing connections. However, if reloading +configurations happen very frequently, the worker +processes might be piled up if they take a bit long time +to finish the existing connections. With this option, +if the number of worker processes exceeds the given +value, the oldest worker process is terminated +immediately. Specifying 0 means no limit and it is the +default behaviour. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-worker\-process\-grace\-shutdown\-period= +Maximum period for a worker process to terminate +gracefully. When a worker process enters in graceful +shutdown period (e.g., when nghttpx reloads its +configuration) and it does not finish handling the +existing connections in the given period of time, it is +immediately terminated. Specifying 0 means no limit and +it is the default behaviour. +.UNINDENT .SS Scripting .INDENT 0.0 .TP @@ -1741,6 +1778,86 @@ Default: \fB/usr/local/lib/nghttp2/reuseport_kern.o\fP .UNINDENT .INDENT 0.0 .TP +.B \-\-frontend\-quic\-early\-data +Enable early data on frontend QUIC connections. nghttpx +sends "Early\-Data" header field to a backend server if a +request is received in early data and handshake has not +finished. All backend servers should deal with possibly +replayed requests. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-frontend\-quic\-qlog\-dir= +Specify a directory where a qlog file is written for +frontend QUIC connections. A qlog file is created per +each QUIC connection. The file name is ISO8601 basic +format, followed by "\-", server Source Connection ID and +".qlog". +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-frontend\-quic\-require\-token +Require an address validation token for a frontend QUIC +connection. Server sends a token in Retry packet or +NEW_TOKEN frame in the previous connection. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-frontend\-quic\-congestion\-controller= +Specify a congestion controller algorithm for a frontend +QUIC connection. should be either "cubic" or +"bbr". +.sp +Default: \fBcubic\fP +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-frontend\-quic\-secret\-file= +Path to file that contains secure random data to be used +as QUIC keying materials. It is used to derive keys for +encrypting tokens and Connection IDs. It is not used to +encrypt QUIC packets. Each line of this file must +contain exactly 136 bytes hex\-encoded string (when +decoded the byte string is 68 bytes long). The first 2 +bits of decoded byte string are used to identify the +keying material. An empty line or a line which starts +\(aq#\(aq is ignored. The file can contain more than one +keying materials. Because the identifier is 2 bits, at +most 4 keying materials are read and the remaining data +is discarded. The first keying material in the file is +primarily used for encryption and decryption for new +connection. The other ones are used to decrypt data for +the existing connections. Specifying multiple keying +materials enables key rotation. Please note that key +rotation does not occur automatically. User should +update files or change options values and restart +nghttpx gracefully. If opening or reading given file +fails, all loaded keying materials are discarded and it +is treated as if none of this option is given. If this +option is not given or an error occurred while opening +or reading a file, a keying material is generated +internally on startup and reload. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-quic\-server\-id= +Specify server ID encoded in Connection ID to identify +this particular server instance. Connection ID is +encrypted and this part is not visible in public. It +must be 4 bytes long and must be encoded in hex string +(which is 8 bytes long). If this option is omitted, a +random server ID is generated on startup and +configuration reload. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-frontend\-quic\-initial\-rtt= +Specify the initial RTT of the frontend QUIC connection. +.sp +Default: \fB333ms\fP +.UNINDENT +.INDENT 0.0 +.TP .B \-\-no\-quic\-bpf Disable eBPF. .UNINDENT diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index fb4a0a92..ad7edbb9 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -14,7 +14,7 @@ SYNOPSIS DESCRIPTION ----------- -A reverse proxy for HTTP/2, and HTTP/1. +A reverse proxy for HTTP/3, HTTP/2, and HTTP/1. .. describe:: @@ -472,6 +472,14 @@ Performance Default: ``0`` +.. option:: --rlimit-memlock= + + Set maximum number of bytes of memory that may be locked + into RAM. If 0 is given, nghttpx does not set the + limit. + + Default: ``0`` + .. option:: --backend-request-buffer= Set buffer size used to store backend request. @@ -1003,12 +1011,13 @@ SSL/TLS .. option:: --tls-no-postpone-early-data - By default, nghttpx postpones forwarding HTTP requests - sent in early data, including those sent in partially in - it, until TLS handshake finishes. If all backend server - recognizes "Early-Data" header field, using this option - makes nghttpx not postpone forwarding request and get - full potential of 0-RTT data. + By default, except for QUIC connections, nghttpx + postpones forwarding HTTP requests sent in early data, + including those sent in partially in it, until TLS + handshake finishes. If all backend server recognizes + "Early-Data" header field, using this option makes + nghttpx not postpone forwarding request and get full + potential of 0-RTT data. .. option:: --tls-max-early-data= @@ -1367,7 +1376,7 @@ HTTP advertised in alt-svc header field only in HTTP/1.1 frontend. This option can be used multiple times to specify multiple alternative services. - Example: :option:`--altsvc`\="h2,443,,,ma=3600; persist=1' + Example: :option:`--altsvc`\="h2,443,,,ma=3600; persist=1" .. option:: --http2-altsvc= @@ -1553,6 +1562,31 @@ Process neverbleed is used. In the single process mode, the signal handling feature is disabled. +.. option:: --max-worker-processes= + + The maximum number of worker processes. nghttpx spawns + new worker process when it reloads its configuration. + The previous worker process enters graceful termination + period and will terminate when it finishes handling the + existing connections. However, if reloading + configurations happen very frequently, the worker + processes might be piled up if they take a bit long time + to finish the existing connections. With this option, + if the number of worker processes exceeds the given + value, the oldest worker process is terminated + immediately. Specifying 0 means no limit and it is the + default behaviour. + +.. option:: --worker-process-grace-shutdown-period= + + Maximum period for a worker process to terminate + gracefully. When a worker process enters in graceful + shutdown period (e.g., when nghttpx reloads its + configuration) and it does not finish handling the + existing connections in the given period of time, it is + immediately terminated. Specifying 0 means no limit and + it is the default behaviour. + Scripting ~~~~~~~~~ @@ -1589,6 +1623,79 @@ HTTP/3 and QUIC Default: ``/usr/local/lib/nghttp2/reuseport_kern.o`` +.. option:: --frontend-quic-early-data + + Enable early data on frontend QUIC connections. nghttpx + sends "Early-Data" header field to a backend server if a + request is received in early data and handshake has not + finished. All backend servers should deal with possibly + replayed requests. + +.. option:: --frontend-quic-qlog-dir= + + Specify a directory where a qlog file is written for + frontend QUIC connections. A qlog file is created per + each QUIC connection. The file name is ISO8601 basic + format, followed by "-", server Source Connection ID and + ".qlog". + +.. option:: --frontend-quic-require-token + + Require an address validation token for a frontend QUIC + connection. Server sends a token in Retry packet or + NEW_TOKEN frame in the previous connection. + +.. option:: --frontend-quic-congestion-controller= + + Specify a congestion controller algorithm for a frontend + QUIC connection. should be either "cubic" or + "bbr". + + Default: ``cubic`` + +.. option:: --frontend-quic-secret-file= + + Path to file that contains secure random data to be used + as QUIC keying materials. It is used to derive keys for + encrypting tokens and Connection IDs. It is not used to + encrypt QUIC packets. Each line of this file must + contain exactly 136 bytes hex-encoded string (when + decoded the byte string is 68 bytes long). The first 2 + bits of decoded byte string are used to identify the + keying material. An empty line or a line which starts + '#' is ignored. The file can contain more than one + keying materials. Because the identifier is 2 bits, at + most 4 keying materials are read and the remaining data + is discarded. The first keying material in the file is + primarily used for encryption and decryption for new + connection. The other ones are used to decrypt data for + the existing connections. Specifying multiple keying + materials enables key rotation. Please note that key + rotation does not occur automatically. User should + update files or change options values and restart + nghttpx gracefully. If opening or reading given file + fails, all loaded keying materials are discarded and it + is treated as if none of this option is given. If this + option is not given or an error occurred while opening + or reading a file, a keying material is generated + internally on startup and reload. + +.. option:: --quic-server-id= + + Specify server ID encoded in Connection ID to identify + this particular server instance. Connection ID is + encrypted and this part is not visible in public. It + must be 4 bytes long and must be encoded in hex string + (which is 8 bytes long). If this option is omitted, a + random server ID is generated on startup and + configuration reload. + +.. option:: --frontend-quic-initial-rtt= + + Specify the initial RTT of the frontend QUIC connection. + + Default: ``333ms`` + .. option:: --no-quic-bpf Disable eBPF. diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index bbcae298..48faddd7 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -14,8 +14,8 @@ Default mode If nghttpx is invoked without :option:`--http2-proxy`, it operates in default mode. In this mode, it works as reverse proxy (gateway) for -both HTTP/2 and HTTP/1 clients to backend servers. This is also known -as "HTTP/2 router". +HTTP/3, HTTP/2 and HTTP/1 clients to backend servers. This is also +known as "HTTP/2 router". By default, frontend connection is encrypted using SSL/TLS. So server's private key and certificate must be supplied to the command @@ -28,6 +28,9 @@ the frontend, and an HTTP/1 connection can be upgraded to HTTP/2 using HTTP Upgrade. Starting HTTP/2 connection by sending HTTP/2 connection preface is also supported. +In order to receive HTTP/3 traffic, use ``quic`` parameter in +:option:`--frontend` option (.e.g, ``--frontend='*,443;quic'``) + nghttpx can listen on multiple frontend addresses. This is achieved by using multiple :option:`--frontend` options. For each frontend address, TLS can be enabled or disabled. @@ -509,6 +512,8 @@ Bootstrapping WebSockets with HTTP/2 for both frontend and backend connections. This feature is enabled by default and no configuration is required. +WebSockets over HTTP/3 is also supported. + HTTP/3 ------ @@ -529,7 +534,28 @@ nghttpx does not support HTTP/3 on backend connection. Hot swapping (SIGUSR2) or configuration reload (SIGHUP) require eBPF program. Without eBPF, old worker processes keep getting HTTP/3 -traffic and do not work as intended. +traffic and do not work as intended. The QUIC keying material to +encrypt Connection ID must be set with +:option:`--frontend-quic-secret-file` and must provide the existing +keys in order to keep the existing connections alive during reload. + +The construction of Connection ID closely follows Block Cipher CID +Algorithm described in `QUIC-LB draft +`_. +A Connection ID that nghttpx generates is always 20 bytes long. It +uses first 2 bits as a configuration ID. The remaining bits in the +first byte are reserved and random. The next 4 bytes are server ID. +The next 4 bytes are used to route UDP datagram to a correct +``SO_REUSEPORT`` socket. The remaining bytes are randomly generated. +The server ID and the next 12 bytes are encrypted with AES-ECB. The +key is derived from the keying materials stored in a file specified by +:option:`--frontend-quic-secret-file`. The first 2 bits of keying +material in the file is used as a configuration ID. The remaining +bits and following 3 bytes are reserved and unused. The next 32 bytes +are used as an initial secret. The remaining 32 bytes are used as a +salt. The encryption key is generated by `HKDF +`_ with SHA256 and +these keying materials and ``connection id encryption key`` as info. In order announce that HTTP/3 endpoint is available, you should specify alt-svc header field. For example, the following options send diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..b66ce9f8 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,78 @@ +FROM debian:11 as build + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git clang make binutils autoconf automake autotools-dev libtool \ + pkg-config \ + zlib1g-dev libev-dev libjemalloc-dev ruby-dev libc-ares-dev bison \ + libelf-dev + +RUN git clone --depth 1 -b OpenSSL_1_1_1m+quic https://github.com/quictls/openssl && \ + cd openssl && \ + ./config --openssldir=/etc/ssl && \ + make -j$(nproc) && \ + make install_sw && \ + cd .. && \ + rm -rf openssl + +RUN git clone --depth 1 -b v0.1.1 https://github.com/ngtcp2/nghttp3 && \ + cd nghttp3 && \ + autoreconf -i && \ + ./configure --enable-lib-only && \ + make -j$(nproc) && \ + make install-strip && \ + cd .. && \ + rm -rf nghttp3 + +RUN git clone --depth 1 -b v0.1.0 https://github.com/ngtcp2/ngtcp2 && \ + cd ngtcp2 && \ + autoreconf -i && \ + ./configure --enable-lib-only \ + LIBTOOL_LDFLAGS="-static-libtool-libs" \ + OPENSSL_LIBS="-l:libssl.a -l:libcrypto.a -ldl -lpthread" \ + PKG_CONFIG_PATH="/usr/local/lib64/pkgconfig" && \ + make -j$(nproc) && \ + make install-strip && \ + cd .. && \ + rm -rf ngtcp2 + +RUN git clone --depth 1 -b v0.4.0 https://github.com/libbpf/libbpf && \ + cd libbpf && \ + PREFIX=/usr/local make -C src install && \ + cd .. && \ + rm -rf libbpf + +RUN git clone --depth 1 https://github.com/nghttp2/nghttp2.git && \ + cd nghttp2 && \ + git submodule update --init && \ + autoreconf -i && \ + ./configure --disable-examples --disable-hpack-tools \ + --disable-python-bindings --with-mruby --with-neverbleed \ + --enable-http3 --with-libbpf \ + CC=clang CXX=clang++ \ + LIBTOOL_LDFLAGS="-static-libtool-libs" \ + OPENSSL_LIBS="-l:libssl.a -l:libcrypto.a -ldl -pthread" \ + LIBEV_LIBS="-l:libev.a" \ + JEMALLOC_LIBS="-l:libjemalloc.a" \ + LIBCARES_LIBS="-l:libcares.a" \ + ZLIB_LIBS="-l:libz.a" \ + LIBBPF_LIBS="-L/usr/local/lib64 -l:libbpf.a -l:libelf.a" \ + LDFLAGS="-static-libgcc -static-libstdc++" \ + PKG_CONFIG_PATH="/usr/local/lib64/pkgconfig" && \ + make -j$(nproc) install-strip && \ + cd .. && \ + rm -rf nghttp2 + +FROM gcr.io/distroless/base-debian11 + +COPY --from=build \ + /usr/local/share/nghttp2/ \ + /usr/local/share/nghttp2/ +COPY --from=build \ + /usr/local/bin/h2load \ + /usr/local/bin/nghttpx \ + /usr/local/bin/nghttp \ + /usr/local/bin/nghttpd \ + /usr/local/bin/ +COPY --from=build /usr/local/lib/nghttp2/reuseport_kern.o \ + /usr/local/lib/nghttp2/ diff --git a/docker/Dockerfile-h2load-http3 b/docker/Dockerfile-h2load-http3 deleted file mode 100644 index b5dcd411..00000000 --- a/docker/Dockerfile-h2load-http3 +++ /dev/null @@ -1,39 +0,0 @@ -FROM debian:10 as build - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - git g++ make binutils autoconf automake autotools-dev libtool \ - pkg-config \ - zlib1g-dev libev-dev libjemalloc-dev ruby-dev libc-ares-dev bison && \ - git clone --depth 1 -b OpenSSL_1_1_1l+quic https://github.com/quictls/openssl && \ - cd openssl && ./config enable-tls1_3 --openssldir=/etc/ssl && make -j$(nproc) && make install_sw && cd .. && rm -rf openssl && \ - git clone --depth 1 https://github.com/ngtcp2/nghttp3 && \ - cd nghttp3 && autoreconf -i && \ - ./configure --enable-lib-only && \ - make -j$(nproc) && make install-strip && cd .. && rm -rf nghttp3 && \ - git clone --depth 1 https://github.com/ngtcp2/ngtcp2 && \ - cd ngtcp2 && autoreconf -i && \ - ./configure --enable-lib-only \ - LIBTOOL_LDFLAGS="-static-libtool-libs" \ - OPENSSL_LIBS="-l:libssl.a -l:libcrypto.a -ldl -lpthread" && \ - make -j$(nproc) && make install-strip && cd .. && rm -rf ngtcp2 && \ - git clone --depth 1 https://github.com/nghttp2/nghttp2.git && \ - cd nghttp2 && \ - git submodule update --init && autoreconf -i && \ - ./configure --disable-examples --disable-hpack-tools \ - --disable-python-bindings --with-mruby --with-neverbleed \ - --enable-http3 \ - LIBTOOL_LDFLAGS="-static-libtool-libs" \ - LIBS="-ldl -pthread" \ - OPENSSL_LIBS="-l:libssl.a -l:libcrypto.a" \ - LIBEV_LIBS="-l:libev.a" \ - JEMALLOC_LIBS="-l:libjemalloc.a" \ - LIBCARES_LIBS="-l:libcares.a" \ - ZLIB_LIBS="-l:libz.a" && \ - make -j$(nproc) install-strip - -FROM gcr.io/distroless/cc-debian10 - -COPY --from=build /usr/local/bin/h2load /usr/local/bin/ - -ENTRYPOINT ["/usr/local/bin/h2load"] diff --git a/docker/README.rst b/docker/README.rst new file mode 100644 index 00000000..24402165 --- /dev/null +++ b/docker/README.rst @@ -0,0 +1,25 @@ +Dockerfile +========== + +Dockerfile creates the applications bundled with nghttp2. +These applications are: + +- nghttp +- nghttpd +- nghttpx +- h2load + +HTTP/3 and eBPF features are enabled. + +In order to run nghttpx with HTTP/3 endpoint, you need to run the +image with the escalated privilege and higher memlock value. Here is +the example command-line to run nghttpx to listen to HTTP/3 on port +443, assuming that the current directory contains a private key and a +certificate in server.key and server.crt respectively : + +.. code-block:: text + + $ docker run --rm -it -v $PWD:/shared --net=host --privileged \ + nghttp2 nghttpx \ + /shared/server.key /shared/server.crt \ + -f'*,443;quic' --rlimit-memlock 524288 diff --git a/examples/client.c b/examples/client.c index 2d3c4c5d..6cc3fddf 100644 --- a/examples/client.c +++ b/examples/client.c @@ -380,6 +380,10 @@ static void init_ssl_ctx(SSL_CTX *ssl_ctx) { #ifndef OPENSSL_NO_NEXTPROTONEG SSL_CTX_set_next_proto_select_cb(ssl_ctx, select_next_proto_cb, NULL); #endif /* !OPENSSL_NO_NEXTPROTONEG */ + +#if OPENSSL_VERSION_NUMBER >= 0x10002000L + SSL_CTX_set_alpn_protos(ssl_ctx, (const unsigned char *)"\x02h2", 3); +#endif /* OPENSSL_VERSION_NUMBER >= 0x10002000L */ } static void ssl_handshake(SSL *ssl, int fd) { @@ -544,7 +548,7 @@ static void fetch_uri(const struct URI *uri) { if (fd == -1) { die("Could not open file descriptor"); } - ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + ssl_ctx = SSL_CTX_new(TLS_client_method()); if (ssl_ctx == NULL) { dief("SSL_CTX_new", ERR_error_string(ERR_get_error(), NULL)); } @@ -715,8 +719,18 @@ int main(int argc, char **argv) { act.sa_handler = SIG_IGN; sigaction(SIGPIPE, &act, 0); +#if OPENSSL_VERSION_NUMBER >= 0x1010000fL + /* No explicit initialization is required. */ +#elif defined(OPENSSL_IS_BORINGSSL) + CRYPTO_library_init(); +#else /* !(OPENSSL_VERSION_NUMBER >= 0x1010000fL) && \ + !defined(OPENSSL_IS_BORINGSSL) */ + OPENSSL_config(NULL); SSL_load_error_strings(); SSL_library_init(); + OpenSSL_add_all_algorithms(); +#endif /* !(OPENSSL_VERSION_NUMBER >= 0x1010000fL) && \ + !defined(OPENSSL_IS_BORINGSSL) */ rv = parse_uri(&uri, argv[1]); if (rv != 0) { diff --git a/examples/deflate.c b/examples/deflate.c index 63f3ea15..df1cb920 100644 --- a/examples/deflate.c +++ b/examples/deflate.c @@ -44,7 +44,7 @@ static void deflate(nghttp2_hd_deflater *deflater, static int inflate_header_block(nghttp2_hd_inflater *inflater, uint8_t *in, size_t inlen, int final); -int main() { +int main(void) { int rv; nghttp2_hd_deflater *deflater; nghttp2_hd_inflater *inflater; diff --git a/examples/libevent-client.c b/examples/libevent-client.c index f42cbdb3..2debd7b8 100644 --- a/examples/libevent-client.c +++ b/examples/libevent-client.c @@ -328,7 +328,7 @@ static int select_next_proto_cb(SSL *ssl, unsigned char **out, /* Create SSL_CTX. */ static SSL_CTX *create_ssl_ctx(void) { SSL_CTX *ssl_ctx; - ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + ssl_ctx = SSL_CTX_new(TLS_client_method()); if (!ssl_ctx) { errx(1, "Could not create SSL/TLS context: %s", ERR_error_string(ERR_get_error(), NULL)); @@ -617,8 +617,18 @@ int main(int argc, char **argv) { act.sa_handler = SIG_IGN; sigaction(SIGPIPE, &act, NULL); +#if OPENSSL_VERSION_NUMBER >= 0x1010000fL + /* No explicit initialization is required. */ +#elif defined(OPENSSL_IS_BORINGSSL) + CRYPTO_library_init(); +#else /* !(OPENSSL_VERSION_NUMBER >= 0x1010000fL) && \ + !defined(OPENSSL_IS_BORINGSSL) */ + OPENSSL_config(NULL); SSL_load_error_strings(); SSL_library_init(); + OpenSSL_add_all_algorithms(); +#endif /* !(OPENSSL_VERSION_NUMBER >= 0x1010000fL) && \ + !defined(OPENSSL_IS_BORINGSSL) */ run(argv[1]); return 0; diff --git a/examples/libevent-server.c b/examples/libevent-server.c index 0465a785..9f4e1281 100644 --- a/examples/libevent-server.c +++ b/examples/libevent-server.c @@ -143,7 +143,7 @@ static int alpn_select_proto_cb(SSL *ssl, const unsigned char **out, static SSL_CTX *create_ssl_ctx(const char *key_file, const char *cert_file) { SSL_CTX *ssl_ctx; - ssl_ctx = SSL_CTX_new(SSLv23_server_method()); + ssl_ctx = SSL_CTX_new(TLS_server_method()); if (!ssl_ctx) { errx(1, "Could not create SSL/TLS context: %s", ERR_error_string(ERR_get_error(), NULL)); @@ -153,14 +153,9 @@ static SSL_CTX *create_ssl_ctx(const char *key_file, const char *cert_file) { SSL_OP_NO_COMPRESSION | SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION); #if OPENSSL_VERSION_NUMBER >= 0x30000000L - { - EVP_PKEY *ecdh; - ecdh = EVP_EC_gen("P-256"); - if (!ecdh) { - errx(1, "EVP_EC_gen failed: %s", ERR_error_string(ERR_get_error(), NULL)); - } - SSL_CTX_set_tmp_ecdh(ssl_ctx, ecdh); - EVP_PKEY_free(ecdh); + if (SSL_CTX_set1_curves_list(ssl_ctx, "P-256") != 1) { + errx(1, "SSL_CTX_set1_curves_list failed: %s", + ERR_error_string(ERR_get_error(), NULL)); } #else /* !(OPENSSL_VERSION_NUMBER >= 0x30000000L) */ { @@ -822,8 +817,18 @@ int main(int argc, char **argv) { act.sa_handler = SIG_IGN; sigaction(SIGPIPE, &act, NULL); +#if OPENSSL_VERSION_NUMBER >= 0x1010000fL + /* No explicit initialization is required. */ +#elif defined(OPENSSL_IS_BORINGSSL) + CRYPTO_library_init(); +#else /* !(OPENSSL_VERSION_NUMBER >= 0x1010000fL) && \ + !defined(OPENSSL_IS_BORINGSSL) */ + OPENSSL_config(NULL); SSL_load_error_strings(); SSL_library_init(); + OpenSSL_add_all_algorithms(); +#endif /* !(OPENSSL_VERSION_NUMBER >= 0x1010000fL) && \ + !defined(OPENSSL_IS_BORINGSSL) */ run(argv[1], argv[2], argv[3]); return 0; diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 337c1cb4..a388b119 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -190,6 +190,14 @@ OPTIONS = [ "frontend-http3-max-concurrent-streams", "frontend-quic-early-data", "frontend-quic-qlog-dir", + "frontend-quic-require-token", + "frontend-quic-congestion-controller", + "quic-server-id", + "frontend-quic-secret-file", + "rlimit-memlock", + "max-worker-processes", + "worker-process-grace-shutdown-period", + "frontend-quic-initial-rtt", ] LOGVARS = [ diff --git a/integration-tests/nghttpx_http1_test.go b/integration-tests/nghttpx_http1_test.go index 3d416771..7159b462 100644 --- a/integration-tests/nghttpx_http1_test.go +++ b/integration-tests/nghttpx_http1_test.go @@ -5,14 +5,15 @@ import ( "bytes" "encoding/json" "fmt" - "golang.org/x/net/http2/hpack" - "golang.org/x/net/websocket" "io" "net/http" "regexp" "syscall" "testing" "time" + + "golang.org/x/net/http2/hpack" + "golang.org/x/net/websocket" ) // TestH1H1PlainGET tests whether simple HTTP/1 GET request works. @@ -34,7 +35,7 @@ func TestH1H1PlainGET(t *testing.T) { } // TestH1H1PlainGETClose tests whether simple HTTP/1 GET request with -// Connetion: close request header field works. +// Connection: close request header field works. func TestH1H1PlainGETClose(t *testing.T) { st := newServerTester(nil, t, noopHandler) defer st.Close() @@ -1171,3 +1172,31 @@ Content-Length: 1000000 t.Errorf("status: %v; want %v", got, want) } } + +// TestH1H1ChunkedEndsPrematurely tests that an HTTP/1.1 request fails +// if the backend chunked encoded response ends prematurely. +func TestH1H1ChunkedEndsPrematurely(t *testing.T) { + st := newServerTester(nil, t, func(w http.ResponseWriter, r *http.Request) { + hj, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "Could not hijack the connection", http.StatusInternalServerError) + return + } + conn, bufrw, err := hj.Hijack() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer conn.Close() + bufrw.WriteString("HTTP/1.1 200\r\nTransfer-Encoding: chunked\r\n\r\n") + bufrw.Flush() + }) + defer st.Close() + + _, err := st.http1(requestParam{ + name: "TestH1H1ChunkedEndsPrematurely", + }) + if err == nil { + t.Fatal("st.http1() should fail") + } +} diff --git a/integration-tests/nghttpx_http2_test.go b/integration-tests/nghttpx_http2_test.go index 84646fa0..a856f8fc 100644 --- a/integration-tests/nghttpx_http2_test.go +++ b/integration-tests/nghttpx_http2_test.go @@ -565,7 +565,7 @@ func TestH2H1BadResponseCL(t *testing.T) { t.Fatalf("Error st.http2() = %v", err) } - want := http2.ErrCodeProtocol + want := http2.ErrCodeInternal if res.errCode != want { t.Errorf("res.errCode = %v; want %v", res.errCode, want) } @@ -2838,3 +2838,35 @@ func TestH2ResponseBeforeRequestEnd(t *testing.T) { t.Errorf("res.status: %v; want %v", got, want) } } + +// TestH2H1ChunkedEndsPrematurely tests that a stream is reset if the +// backend chunked encoded response ends prematurely. +func TestH2H1ChunkedEndsPrematurely(t *testing.T) { + st := newServerTester(nil, t, func(w http.ResponseWriter, r *http.Request) { + hj, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "Could not hijack the connection", http.StatusInternalServerError) + return + } + conn, bufrw, err := hj.Hijack() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer conn.Close() + bufrw.WriteString("HTTP/1.1 200\r\nTransfer-Encoding: chunked\r\n\r\n") + bufrw.Flush() + }) + defer st.Close() + + res, err := st.http2(requestParam{ + name: "TestH2H1ChunkedEndsPrematurely", + }) + if err != nil { + t.Fatalf("Error st.http2() = %v", err) + } + + if got, want := res.errCode, http2.ErrCodeInternal; got != want { + t.Errorf("res.errCode = %v; want %v", got, want) + } +} diff --git a/lib/nghttp2_buf.h b/lib/nghttp2_buf.h index 06cce67a..45f62f16 100644 --- a/lib/nghttp2_buf.h +++ b/lib/nghttp2_buf.h @@ -99,7 +99,7 @@ void nghttp2_buf_free(nghttp2_buf *buf, nghttp2_mem *mem); * |new_cap|. If extensions took place, buffer pointers in |buf| will * change. * - * This function returns 0 if it succeeds, or one of the followings + * This function returns 0 if it succeeds, or one of the following * negative error codes: * * NGHTTP2_ERR_NOMEM diff --git a/lib/nghttp2_frame.c b/lib/nghttp2_frame.c index 382a26c8..3648b238 100644 --- a/lib/nghttp2_frame.c +++ b/lib/nghttp2_frame.c @@ -654,8 +654,6 @@ int nghttp2_frame_unpack_goaway_payload2(nghttp2_goaway *frame, var_gift_payloadlen = 0; } - payloadlen -= var_gift_payloadlen; - if (!var_gift_payloadlen) { var_gift_payload = NULL; } else { diff --git a/lib/nghttp2_frame.h b/lib/nghttp2_frame.h index 4b9222ac..3859926e 100644 --- a/lib/nghttp2_frame.h +++ b/lib/nghttp2_frame.h @@ -46,7 +46,7 @@ #define NGHTTP2_MAX_FRAME_SIZE_MIN (1 << 14) #define NGHTTP2_MAX_PAYLOADLEN 16384 -/* The one frame buffer length for tranmission. We may use several of +/* The one frame buffer length for transmission. We may use several of them to support CONTINUATION. To account for Pad Length field, we allocate extra 1 byte, which saves extra large memcopying. */ #define NGHTTP2_FRAMEBUF_CHUNKLEN \ diff --git a/lib/nghttp2_hd.c b/lib/nghttp2_hd.c index 5e869315..30ee9b88 100644 --- a/lib/nghttp2_hd.c +++ b/lib/nghttp2_hd.c @@ -1263,6 +1263,8 @@ int nghttp2_hd_inflate_change_table_size( return NGHTTP2_ERR_INVALID_STATE; } + inflater->settings_hd_table_bufsize_max = settings_max_dynamic_table_size; + /* It seems that encoder is not required to send dynamic table size update if the table size is not changed after applying SETTINGS_HEADER_TABLE_SIZE. RFC 7541 is ambiguous here, but this @@ -1275,13 +1277,12 @@ int nghttp2_hd_inflate_change_table_size( /* Remember minimum value, and validate that encoder sends the value less than or equal to this. */ inflater->min_hd_table_bufsize_max = settings_max_dynamic_table_size; + + inflater->ctx.hd_table_bufsize_max = settings_max_dynamic_table_size; + + hd_context_shrink_table_size(&inflater->ctx, NULL); } - inflater->settings_hd_table_bufsize_max = settings_max_dynamic_table_size; - - inflater->ctx.hd_table_bufsize_max = settings_max_dynamic_table_size; - - hd_context_shrink_table_size(&inflater->ctx, NULL); return 0; } diff --git a/lib/nghttp2_map.c b/lib/nghttp2_map.c index 5aab90b4..e5db168c 100644 --- a/lib/nghttp2_map.c +++ b/lib/nghttp2_map.c @@ -189,6 +189,7 @@ static int map_resize(nghttp2_map *map, uint32_t new_tablelen, nghttp2_map_bucket *new_table; nghttp2_map_bucket *bkt; int rv; + (void)rv; new_table = nghttp2_mem_calloc(map->mem, new_tablelen, sizeof(nghttp2_map_bucket)); diff --git a/lib/nghttp2_net.h b/lib/nghttp2_net.h index 95ffee74..582099b9 100644 --- a/lib/nghttp2_net.h +++ b/lib/nghttp2_net.h @@ -42,7 +42,7 @@ #if defined(WIN32) /* Windows requires ws2_32 library for ntonl family functions. We define inline functions for those function so that we don't have - dependeny on that lib. */ + dependency on that lib. */ # ifdef _MSC_VER # define STIN static __inline diff --git a/lib/nghttp2_outbound_item.h b/lib/nghttp2_outbound_item.h index b5f503a3..bd4611b5 100644 --- a/lib/nghttp2_outbound_item.h +++ b/lib/nghttp2_outbound_item.h @@ -111,7 +111,7 @@ struct nghttp2_outbound_item { to this structure to avoid frequent memory allocation. */ nghttp2_ext_frame_payload ext_frame_payload; nghttp2_aux_data aux_data; - /* The priority used in priority comparion. Smaller is served + /* The priority used in priority comparison. Smaller is served earlier. For PING, SETTINGS and non-DATA frames (excluding response HEADERS frame) have dedicated cycle value defined above. For DATA frame, cycle is computed by taking into account of diff --git a/lib/nghttp2_pq.h b/lib/nghttp2_pq.h index 2d7b702a..7b7b7392 100644 --- a/lib/nghttp2_pq.h +++ b/lib/nghttp2_pq.h @@ -114,7 +114,7 @@ typedef int (*nghttp2_pq_item_cb)(nghttp2_pq_entry *item, void *arg); void nghttp2_pq_update(nghttp2_pq *pq, nghttp2_pq_item_cb fun, void *arg); /* - * Applys |fun| to each item in |pq|. The |arg| is passed as arg + * Applies |fun| to each item in |pq|. The |arg| is passed as arg * parameter to callback function. This function must not change the * ordering key. If the return value from callback is nonzero, this * function returns 1 immediately without iterating remaining items. diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 36f1179f..380a47c1 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -5341,7 +5341,7 @@ static ssize_t inbound_frame_compute_pad(nghttp2_inbound_frame *iframe) { /* * This function returns the effective payload length in the data of - * length |readlen| when the remaning payload is |payloadleft|. The + * length |readlen| when the remaining payload is |payloadleft|. The * |payloadleft| does not include |readlen|. If padding was started * strictly before this data chunk, this function returns -1. */ diff --git a/lib/nghttp2_session.h b/lib/nghttp2_session.h index 07bfbb6c..907b1704 100644 --- a/lib/nghttp2_session.h +++ b/lib/nghttp2_session.h @@ -408,7 +408,7 @@ int nghttp2_session_add_rst_stream(nghttp2_session *session, int32_t stream_id, uint32_t error_code); /* - * Adds PING frame. This is a convenient functin built on top of + * Adds PING frame. This is a convenient function built on top of * nghttp2_session_add_frame() to add PING easily. * * If the |opaque_data| is not NULL, it must point to 8 bytes memory diff --git a/lib/nghttp2_stream.c b/lib/nghttp2_stream.c index 96e1d9fe..f4c80a24 100644 --- a/lib/nghttp2_stream.c +++ b/lib/nghttp2_stream.c @@ -33,7 +33,7 @@ #include "nghttp2_frame.h" /* Maximum distance between any two stream's cycle in the same - prirority queue. Imagine stream A's cycle is A, and stream B's + priority queue. Imagine stream A's cycle is A, and stream B's cycle is B, and A < B. The cycle is unsigned 32 bit integer, it may get overflow. Because of how we calculate the next cycle value, if B - A is less than or equals to diff --git a/lib/nghttp2_submit.c b/lib/nghttp2_submit.c index 744a49cf..92fb03e8 100644 --- a/lib/nghttp2_submit.c +++ b/lib/nghttp2_submit.c @@ -492,8 +492,6 @@ int nghttp2_session_set_local_window_size(nghttp2_session *session, return nghttp2_session_update_recv_stream_window_size(session, stream, 0, 1); } - - return 0; } int nghttp2_submit_altsvc(nghttp2_session *session, uint8_t flags, diff --git a/m4/ax_python_devel.m4 b/m4/ax_python_devel.m4 index 44dbd83e..9d4eecf7 100644 --- a/m4/ax_python_devel.m4 +++ b/m4/ax_python_devel.m4 @@ -67,7 +67,7 @@ # modified version of the Autoconf Macro, you may extend this special # exception to the GPL to apply to your modified version as well. -#serial 21 +#serial 23 AU_ALIAS([AC_PYTHON_DEVEL], [AX_PYTHON_DEVEL]) AC_DEFUN([AX_PYTHON_DEVEL],[ @@ -135,16 +135,25 @@ variable to configure. See ``configure --help'' for reference. # # Check if you have distutils, else fail # - AC_MSG_CHECKING([for the distutils Python package]) - ac_distutils_result=`$PYTHON -c "import distutils" 2>&1` + AC_MSG_CHECKING([for the sysconfig Python package]) + ac_sysconfig_result=`$PYTHON -c "import sysconfig" 2>&1` if test $? -eq 0; then AC_MSG_RESULT([yes]) + IMPORT_SYSCONFIG="import sysconfig" else AC_MSG_RESULT([no]) - AC_MSG_ERROR([cannot import Python module "distutils". + + AC_MSG_CHECKING([for the distutils Python package]) + ac_sysconfig_result=`$PYTHON -c "from distutils import sysconfig" 2>&1` + if test $? -eq 0; then + AC_MSG_RESULT([yes]) + IMPORT_SYSCONFIG="from distutils import sysconfig" + else + AC_MSG_ERROR([cannot import Python module "distutils". Please check your Python installation. The error was: -$ac_distutils_result]) - PYTHON_VERSION="" +$ac_sysconfig_result]) + PYTHON_VERSION="" + fi fi # @@ -152,10 +161,19 @@ $ac_distutils_result]) # AC_MSG_CHECKING([for Python include path]) if test -z "$PYTHON_CPPFLAGS"; then - python_path=`$PYTHON -c "import distutils.sysconfig; \ - print (distutils.sysconfig.get_python_inc ());"` - plat_python_path=`$PYTHON -c "import distutils.sysconfig; \ - print (distutils.sysconfig.get_python_inc (plat_specific=1));"` + if test "$IMPORT_SYSCONFIG" = "import sysconfig"; then + # sysconfig module has different functions + python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \ + print (sysconfig.get_path ('include'));"` + plat_python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \ + print (sysconfig.get_path ('platinclude'));"` + else + # old distutils way + python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \ + print (sysconfig.get_python_inc ());"` + plat_python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \ + print (sysconfig.get_python_inc (plat_specific=1));"` + fi if test -n "${python_path}"; then if test "${plat_python_path}" != "${python_path}"; then python_path="-I$python_path -I$plat_python_path" @@ -179,7 +197,7 @@ $ac_distutils_result]) # join all versioning strings, on some systems # major/minor numbers could be in different list elements -from distutils.sysconfig import * +from sysconfig import * e = get_config_var('VERSION') if e is not None: print(e) @@ -202,8 +220,8 @@ EOD` ac_python_libdir=`cat< next_proto; if (!config_->no_tls) { - ssl_ctx = SSL_CTX_new(SSLv23_server_method()); + ssl_ctx = SSL_CTX_new(TLS_server_method()); if (!ssl_ctx) { std::cerr << ERR_error_string(ERR_get_error(), nullptr) << std::endl; return -1; @@ -2143,22 +2143,13 @@ int HttpServer::run() { SSL_CTX_set_session_cache_mode(ssl_ctx, SSL_SESS_CACHE_SERVER); #ifndef OPENSSL_NO_EC - // Disabled SSL_CTX_set_ecdh_auto, because computational cost of - // chosen curve is much higher than P-256. - - // SSL_CTX_set_ecdh_auto(ssl_ctx, 1); - // Use P-256, which is sufficiently secure at the time of this - // writing. -# if OPENSSL_3_0_0_API - auto ecdh = EVP_EC_gen("P-256"); - if (ecdh == nullptr) { - std::cerr << "EC_KEY_new_by_curv_name failed: " +# if !LIBRESSL_LEGACY_API && OPENSSL_VERSION_NUMBER >= 0x10002000L + if (SSL_CTX_set1_curves_list(ssl_ctx, "P-256") != 1) { + std::cerr << "SSL_CTX_set1_curves_list failed: " << ERR_error_string(ERR_get_error(), nullptr); return -1; } - SSL_CTX_set_tmp_ecdh(ssl_ctx, ecdh); - EVP_PKEY_free(ecdh); -# else // !OPENSSL_3_0_0_API +# else // !(!LIBRESSL_LEGACY_API && OPENSSL_VERSION_NUMBER >= 0x10002000L) auto ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1); if (ecdh == nullptr) { std::cerr << "EC_KEY_new_by_curv_name failed: " @@ -2167,7 +2158,7 @@ int HttpServer::run() { } SSL_CTX_set_tmp_ecdh(ssl_ctx, ecdh); EC_KEY_free(ecdh); -# endif // !OPENSSL_3_0_0_API +# endif // !(!LIBRESSL_LEGACY_API && OPENSSL_VERSION_NUMBER >= 0x10002000L) #endif // OPENSSL_NO_EC if (!config_->dh_param_file.empty()) { @@ -2191,8 +2182,11 @@ int HttpServer::run() { return -1; } - SSL_CTX_set_tmp_dh(ssl_ctx, dh); - EVP_PKEY_free(dh); + if (SSL_CTX_set0_tmp_dh_pkey(ssl_ctx, dh) != 1) { + std::cerr << "SSL_CTX_set0_tmp_dh_pkey failed: " + << ERR_error_string(ERR_get_error(), nullptr) << std::endl; + return -1; + } #else // !OPENSSL_3_0_0_API auto dh = PEM_read_bio_DHparams(bio, nullptr, nullptr, nullptr); diff --git a/src/Makefile.am b/src/Makefile.am index a0545b4f..2fd6eadd 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -47,6 +47,7 @@ AM_CPPFLAGS = \ @LIBEV_CFLAGS@ \ @LIBNGHTTP3_CFLAGS@ \ @LIBNGTCP2_CRYPTO_OPENSSL_CFLAGS@ \ + @LIBNGTCP2_CRYPTO_BORINGSSL_CFLAGS@ \ @LIBNGTCP2_CFLAGS@ \ @OPENSSL_CFLAGS@ \ @LIBCARES_CFLAGS@ \ @@ -65,6 +66,7 @@ LDADD = $(top_builddir)/lib/libnghttp2.la \ @LIBEV_LIBS@ \ @LIBNGHTTP3_LIBS@ \ @LIBNGTCP2_CRYPTO_OPENSSL_LIBS@ \ + @LIBNGTCP2_CRYPTO_BORINGSSL_LIBS@ \ @LIBNGTCP2_LIBS@ \ @OPENSSL_LIBS@ \ @LIBCARES_LIBS@ \ diff --git a/src/h2load.cc b/src/h2load.cc index ad37144e..32cd5add 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -81,6 +81,7 @@ bool recorded(const std::chrono::steady_clock::time_point &t) { } } // namespace +#if OPENSSL_1_1_1_API namespace { std::ofstream keylog_file; void keylog_callback(const SSL *ssl, const char *line) { @@ -89,6 +90,7 @@ void keylog_callback(const SSL *ssl, const char *line) { keylog_file.flush(); } } // namespace +#endif // OPENSSL_1_1_1_API Config::Config() : ciphers(tls::DEFAULT_CIPHER_LIST), @@ -104,6 +106,7 @@ Config::Config() max_concurrent_streams(1), window_bits(30), connection_window_bits(30), + max_frame_size(16_k), rate(0), rate_period(1.0), duration(0.0), @@ -1514,7 +1517,8 @@ int get_ev_loop_flags() { Worker::Worker(uint32_t id, SSL_CTX *ssl_ctx, size_t req_todo, size_t nclients, size_t rate, size_t max_samples, Config *config) - : stats(req_todo, nclients), + : randgen(util::make_mt19937()), + stats(req_todo, nclients), loop(ev_loop_new(get_ev_loop_flags())), ssl_ctx(ssl_ctx), config(config), @@ -2106,6 +2110,11 @@ Options: http/1.1 is used, this specifies the number of HTTP pipelining requests in-flight. Default: 1 + -f, --max-frame-size= + Maximum frame size that the local endpoint is willing to + receive. + Default: )" + << util::utos_unit(config.max_frame_size) << R"( -w, --window-bits= Sets the stream level initial window size to (2**)-1. For QUIC, is capped to 26 (roughly 64MiB). @@ -2119,7 +2128,7 @@ Options: -H, --header=
Add/Override a header to the requests. --ciphers= - Set allowed cipher list for TLSv1.2 or ealier. The + Set allowed cipher list for TLSv1.2 or earlier. The format of the string is described in OpenSSL ciphers(1). Default: )" << config.ciphers << R"( @@ -2243,11 +2252,10 @@ Options: to buffering. Status code is -1 for failed streams. --qlog-file-base= Enable qlog output and specify base file name for qlogs. - Qlog is emitted for each connection. - For a given base name "base", each output file name - becomes "base.M.N.qlog" where M is worker ID and N is - client ID (e.g. "base.0.3.qlog"). - Only effective in QUIC runs. + Qlog is emitted for each connection. For a given base + name "base", each output file name becomes + "base.M.N.sqlog" where M is worker ID and N is client ID + (e.g. "base.0.3.sqlog"). Only effective in QUIC runs. --connect-to=[:] Host and port to connect instead of using the authority in . @@ -2299,6 +2307,7 @@ int main(int argc, char **argv) { {"threads", required_argument, nullptr, 't'}, {"max-concurrent-streams", required_argument, nullptr, 'm'}, {"window-bits", required_argument, nullptr, 'w'}, + {"max-frame-size", required_argument, nullptr, 'f'}, {"connection-window-bits", required_argument, nullptr, 'W'}, {"input-file", required_argument, nullptr, 'i'}, {"header", required_argument, nullptr, 'H'}, @@ -2330,7 +2339,7 @@ int main(int argc, char **argv) { {nullptr, 0, nullptr, 0}}; int option_index = 0; auto c = getopt_long(argc, argv, - "hvW:c:d:m:n:p:t:w:H:i:r:T:N:D:B:", long_options, + "hvW:c:d:m:n:p:t:w:f:H:i:r:T:N:D:B:", long_options, &option_index); if (c == -1) { break; @@ -2376,6 +2385,24 @@ int main(int argc, char **argv) { } break; } + case 'f': { + auto n = util::parse_uint_with_unit(optarg); + if (n == -1) { + std::cerr << "--max-frame-size: bad option value: " << optarg + << std::endl; + exit(EXIT_FAILURE); + } + if (static_cast(n) < 16_k) { + std::cerr << "--max-frame-size: minimum 16384" << std::endl; + exit(EXIT_FAILURE); + } + if (static_cast(n) > 16_m - 1) { + std::cerr << "--max-frame-size: maximum 16777215" << std::endl; + exit(EXIT_FAILURE); + } + config.max_frame_size = n; + break; + } case 'H': { char *header = optarg; // Skip first possible ':' in the header name @@ -2809,7 +2836,7 @@ int main(int argc, char **argv) { act.sa_handler = SIG_IGN; sigaction(SIGPIPE, &act, nullptr); - auto ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + auto ssl_ctx = SSL_CTX_new(TLS_client_method()); if (!ssl_ctx) { std::cerr << "Failed to create SSL_CTX: " << ERR_error_string(ERR_get_error(), nullptr) << std::endl; @@ -2843,19 +2870,26 @@ int main(int argc, char **argv) { exit(EXIT_FAILURE); } -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (SSL_CTX_set_ciphersuites(ssl_ctx, config.tls13_ciphers.c_str()) == 0) { std::cerr << "SSL_CTX_set_ciphersuites with " << config.tls13_ciphers << " failed: " << ERR_error_string(ERR_get_error(), nullptr) << std::endl; exit(EXIT_FAILURE); } -#endif // OPENSSL_1_1_1_API +#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (SSL_CTX_set1_groups_list(ssl_ctx, config.groups.c_str()) != 1) { std::cerr << "SSL_CTX_set1_groups_list failed" << std::endl; exit(EXIT_FAILURE); } +#else // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) + if (SSL_CTX_set1_curves_list(ssl_ctx, config.groups.c_str()) != 1) { + std::cerr << "SSL_CTX_set1_curves_list failed" << std::endl; + exit(EXIT_FAILURE); + } +#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) #ifndef OPENSSL_NO_NEXTPROTONEG SSL_CTX_set_next_proto_select_cb(ssl_ctx, client_select_next_proto_cb, diff --git a/src/h2load.h b/src/h2load.h index 895fb8eb..f1083a1d 100644 --- a/src/h2load.h +++ b/src/h2load.h @@ -95,6 +95,7 @@ struct Config { ssize_t max_concurrent_streams; size_t window_bits; size_t connection_window_bits; + size_t max_frame_size; // rate at which connections should be made size_t rate; ev_tstamp rate_period; @@ -269,6 +270,7 @@ struct Sampling { struct Worker { MemchunkPool mcpool; + std::mt19937 randgen; Stats stats; Sampling request_times_smp; Sampling client_smp; @@ -474,8 +476,10 @@ struct Client { int quic_stream_stop_sending(int64_t stream_id, uint64_t app_error_code); int quic_extend_max_local_streams(); - int quic_on_key(ngtcp2_crypto_level level, const uint8_t *rx_secret, - const uint8_t *tx_secret, size_t secretlen); + int quic_on_rx_secret(ngtcp2_crypto_level level, const uint8_t *secret, + size_t secretlen); + int quic_on_tx_secret(ngtcp2_crypto_level level, const uint8_t *secret, + size_t secretlen); void quic_set_tls_alert(uint8_t alert); void quic_write_client_handshake(ngtcp2_crypto_level level, diff --git a/src/h2load_http2_session.cc b/src/h2load_http2_session.cc index 06c474c0..9cafa0e3 100644 --- a/src/h2load_http2_session.cc +++ b/src/h2load_http2_session.cc @@ -215,7 +215,7 @@ void Http2Session::on_connect() { nghttp2_option_del(opt); - std::array iv; + std::array iv; size_t niv = 2; iv[0].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH; iv[0].value = 0; @@ -227,6 +227,11 @@ void Http2Session::on_connect() { iv[niv].value = config->header_table_size; ++niv; } + if (config->max_frame_size != 16_k) { + iv[niv].settings_id = NGHTTP2_SETTINGS_MAX_FRAME_SIZE; + iv[niv].value = config->max_frame_size; + ++niv; + } rv = nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, iv.data(), niv); diff --git a/src/h2load_http3_session.cc b/src/h2load_http3_session.cc index 40970f7a..573a8a14 100644 --- a/src/h2load_http3_session.cc +++ b/src/h2load_http3_session.cc @@ -213,19 +213,17 @@ void Http3Session::recv_header(int64_t stream_id, const nghttp3_vec *name, } namespace { -int send_stop_sending(nghttp3_conn *conn, int64_t stream_id, - uint64_t app_error_code, void *user_data, - void *stream_user_data) { +int stop_sending(nghttp3_conn *conn, int64_t stream_id, uint64_t app_error_code, + void *user_data, void *stream_user_data) { auto s = static_cast(user_data); - if (s->send_stop_sending(stream_id, app_error_code) != 0) { + if (s->stop_sending(stream_id, app_error_code) != 0) { return NGHTTP3_ERR_CALLBACK_FAILURE; } return 0; } } // namespace -int Http3Session::send_stop_sending(int64_t stream_id, - uint64_t app_error_code) { +int Http3Session::stop_sending(int64_t stream_id, uint64_t app_error_code) { auto rv = ngtcp2_conn_shutdown_stream_read(client_->quic.conn, stream_id, app_error_code); if (rv != 0) { @@ -300,14 +298,14 @@ int Http3Session::init_conn() { nullptr, // begin_trailers h2load::recv_header, nullptr, // end_trailers - h2load::send_stop_sending, + h2load::stop_sending, }; auto config = client_->worker->config; nghttp3_settings settings; nghttp3_settings_default(&settings); - settings.qpack_max_table_capacity = config->header_table_size; + settings.qpack_max_dtable_capacity = config->header_table_size; settings.qpack_blocked_streams = 100; auto mem = nghttp3_mem_default(); diff --git a/src/h2load_http3_session.h b/src/h2load_http3_session.h index 27ff025b..cdd194dd 100644 --- a/src/h2load_http3_session.h +++ b/src/h2load_http3_session.h @@ -51,7 +51,7 @@ public: void begin_headers(int64_t stream_id); void recv_header(int64_t stream_id, const nghttp3_vec *name, const nghttp3_vec *value); - int send_stop_sending(int64_t stream_id, uint64_t app_error_code); + int stop_sending(int64_t stream_id, uint64_t app_error_code); int close_stream(int64_t stream_id, uint64_t app_error_code); int shutdown_stream_read(int64_t stream_id); diff --git a/src/h2load_quic.cc b/src/h2load_quic.cc index ff63ff54..8a8f3caf 100644 --- a/src/h2load_quic.cc +++ b/src/h2load_quic.cc @@ -28,18 +28,20 @@ #include -#include +#ifdef HAVE_LIBNGTCP2_CRYPTO_OPENSSL +# include +#endif // HAVE_LIBNGTCP2_CRYPTO_OPENSSL +#ifdef HAVE_LIBNGTCP2_CRYPTO_BORINGSSL +# include +#endif // HAVE_LIBNGTCP2_CRYPTO_BORINGSSL #include +#include #include "h2load_http3_session.h" namespace h2load { -namespace { -auto randgen = util::make_mt19937(); -} // namespace - namespace { int handshake_completed(ngtcp2_conn *conn, void *user_data) { auto c = static_cast(user_data); @@ -118,7 +120,7 @@ int stream_close(ngtcp2_conn *conn, uint32_t flags, int64_t stream_id, } if (c->quic_stream_close(stream_id, app_error_code) != 0) { - return -1; + return NGTCP2_ERR_CALLBACK_FAILURE; } return 0; } @@ -138,7 +140,7 @@ int stream_reset(ngtcp2_conn *conn, int64_t stream_id, uint64_t final_size, void *stream_user_data) { auto c = static_cast(user_data); if (c->quic_stream_reset(stream_id, app_error_code) != 0) { - return -1; + return NGTCP2_ERR_CALLBACK_FAILURE; } return 0; } @@ -158,7 +160,7 @@ int stream_stop_sending(ngtcp2_conn *conn, int64_t stream_id, void *stream_user_data) { auto c = static_cast(user_data); if (c->quic_stream_stop_sending(stream_id, app_error_code) != 0) { - return -1; + return NGTCP2_ERR_CALLBACK_FAILURE; } return 0; } @@ -197,13 +199,15 @@ int Client::quic_extend_max_local_streams() { namespace { int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token, size_t cidlen, void *user_data) { - auto dis = std::uniform_int_distribution( - 0, std::numeric_limits::max()); - auto f = [&dis]() { return dis(randgen); }; + if (RAND_bytes(cid->data, cidlen) != 1) { + return NGTCP2_ERR_CALLBACK_FAILURE; + } - std::generate_n(cid->data, cidlen, f); cid->datalen = cidlen; - std::generate_n(token, NGTCP2_STATELESS_RESET_TOKENLEN, f); + + if (RAND_bytes(token, NGTCP2_STATELESS_RESET_TOKENLEN) != 1) { + return NGTCP2_ERR_CALLBACK_FAILURE; + } return 0; } @@ -222,11 +226,14 @@ void debug_log_printf(void *user_data, const char *fmt, ...) { } // namespace namespace { -void generate_cid(ngtcp2_cid &dest) { - auto dis = std::uniform_int_distribution( - 0, std::numeric_limits::max()); +int generate_cid(ngtcp2_cid &dest) { dest.datalen = 8; - std::generate_n(dest.data, dest.datalen, [&dis]() { return dis(randgen); }); + + if (RAND_bytes(dest.data, dest.datalen) != 1) { + return -1; + } + + return 0; } } // namespace @@ -236,15 +243,19 @@ ngtcp2_tstamp timestamp(struct ev_loop *loop) { } } // namespace +#ifdef HAVE_LIBNGTCP2_CRYPTO_OPENSSL namespace { int set_encryption_secrets(SSL *ssl, OSSL_ENCRYPTION_LEVEL ossl_level, const uint8_t *rx_secret, const uint8_t *tx_secret, size_t secret_len) { auto c = static_cast(SSL_get_app_data(ssl)); + auto level = ngtcp2_crypto_openssl_from_ossl_encryption_level(ossl_level); - if (c->quic_on_key( - ngtcp2_crypto_openssl_from_ossl_encryption_level(ossl_level), - rx_secret, tx_secret, secret_len) != 0) { + if (c->quic_on_rx_secret(level, rx_secret, secret_len) != 0) { + return 0; + } + + if (c->quic_on_tx_secret(level, tx_secret, secret_len) != 0) { return 0; } @@ -282,6 +293,70 @@ auto quic_method = SSL_QUIC_METHOD{ send_alert, }; } // namespace +#endif // HAVE_LIBNGTCP2_CRYPTO_OPENSSL + +#ifdef HAVE_LIBNGTCP2_CRYPTO_BORINGSSL +namespace { +int set_read_secret(SSL *ssl, ssl_encryption_level_t ssl_level, + const SSL_CIPHER *cipher, const uint8_t *secret, + size_t secretlen) { + auto c = static_cast(SSL_get_app_data(ssl)); + + if (c->quic_on_rx_secret( + ngtcp2_crypto_boringssl_from_ssl_encryption_level(ssl_level), secret, + secretlen) != 0) { + return 0; + } + + return 1; +} +} // namespace + +namespace { +int set_write_secret(SSL *ssl, ssl_encryption_level_t ssl_level, + const SSL_CIPHER *cipher, const uint8_t *secret, + size_t secretlen) { + auto c = static_cast(SSL_get_app_data(ssl)); + + if (c->quic_on_tx_secret( + ngtcp2_crypto_boringssl_from_ssl_encryption_level(ssl_level), secret, + secretlen) != 0) { + return 0; + } + + return 1; +} +} // namespace + +namespace { +int add_handshake_data(SSL *ssl, ssl_encryption_level_t ssl_level, + const uint8_t *data, size_t len) { + auto c = static_cast(SSL_get_app_data(ssl)); + c->quic_write_client_handshake( + ngtcp2_crypto_boringssl_from_ssl_encryption_level(ssl_level), data, len); + return 1; +} +} // namespace + +namespace { +int flush_flight(SSL *ssl) { return 1; } +} // namespace + +namespace { +int send_alert(SSL *ssl, ssl_encryption_level_t level, uint8_t alert) { + auto c = static_cast(SSL_get_app_data(ssl)); + c->quic_set_tls_alert(alert); + return 1; +} +} // namespace + +namespace { +auto quic_method = SSL_QUIC_METHOD{ + set_read_secret, set_write_secret, add_handshake_data, + flush_flight, send_alert, +}; +} // namespace +#endif // HAVE_LIBNGTCP2_CRYPTO_BORINGSSL // qlog write callback -- excerpted from ngtcp2/examples/client_base.cc namespace { @@ -297,6 +372,13 @@ void Client::quic_write_qlog(const void *data, size_t datalen) { fwrite(data, 1, datalen, quic.qlog_file); } +namespace { +void rand(uint8_t *dest, size_t destlen, const ngtcp2_rand_ctx *rand_ctx) { + util::random_bytes(dest, dest + destlen, + *static_cast(rand_ctx->native_handle)); +} +} // namespace + int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen, const sockaddr *remote_addr, socklen_t remote_addrlen) { int rv; @@ -327,7 +409,7 @@ int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen, ngtcp2_crypto_recv_retry_cb, h2load::extend_max_local_streams_bidi, nullptr, // extend_max_local_streams_uni - nullptr, // rand + h2load::rand, get_new_connection_id, nullptr, // remove_connection_id ngtcp2_crypto_update_key_cb, @@ -345,13 +427,17 @@ int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen, nullptr, // recv_datagram nullptr, // ack_datagram nullptr, // lost_datagram - nullptr, // get_path_challenge_data + ngtcp2_crypto_get_path_challenge_data_cb, h2load::stream_stop_sending, }; ngtcp2_cid scid, dcid; - generate_cid(scid); - generate_cid(dcid); + if (generate_cid(scid) != 0) { + return -1; + } + if (generate_cid(dcid) != 0) { + return -1; + } auto config = worker->config; @@ -361,6 +447,7 @@ int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen, settings.log_printf = debug_log_printf; } settings.initial_ts = timestamp(worker->loop); + settings.rand_ctx.native_handle = &worker->randgen; if (!config->qlog_file_base.empty()) { assert(quic.qlog_file == nullptr); auto path = config->qlog_file_base; @@ -368,7 +455,7 @@ int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen, path += util::utos(worker->id); path += '.'; path += util::utos(id); - path += ".qlog"; + path += ".sqlog"; quic.qlog_file = fopen(path.c_str(), "w"); if (quic.qlog_file == nullptr) { std::cerr << "Failed to open a qlog file: " << path << std::endl; @@ -393,8 +480,14 @@ int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen, params.max_idle_timeout = 30 * NGTCP2_SECONDS; auto path = ngtcp2_path{ - {local_addrlen, const_cast(local_addr)}, - {remote_addrlen, const_cast(remote_addr)}, + { + const_cast(local_addr), + local_addrlen, + }, + { + const_cast(remote_addr), + remote_addrlen, + }, }; assert(config->npn_list.size()); @@ -442,15 +535,16 @@ void Client::quic_close_connection() { case quic::ErrorType::Transport: nwrite = ngtcp2_conn_write_connection_close( quic.conn, &ps.path, nullptr, buf.data(), buf.size(), - quic.last_error.code, timestamp(worker->loop)); + quic.last_error.code, nullptr, 0, timestamp(worker->loop)); break; case quic::ErrorType::Application: nwrite = ngtcp2_conn_write_application_close( quic.conn, &ps.path, nullptr, buf.data(), buf.size(), - quic.last_error.code, timestamp(worker->loop)); + quic.last_error.code, nullptr, 0, timestamp(worker->loop)); break; default: assert(0); + abort(); } if (nwrite < 0) { @@ -461,24 +555,16 @@ void Client::quic_close_connection() { ps.path.remote.addrlen, buf.data(), nwrite, 0); } -int Client::quic_on_key(ngtcp2_crypto_level level, const uint8_t *rx_secret, - const uint8_t *tx_secret, size_t secretlen) { +int Client::quic_on_rx_secret(ngtcp2_crypto_level level, const uint8_t *secret, + size_t secretlen) { if (ngtcp2_crypto_derive_and_install_rx_key(quic.conn, nullptr, nullptr, - nullptr, level, rx_secret, + nullptr, level, secret, secretlen) != 0) { std::cerr << "ngtcp2_crypto_derive_and_install_rx_key() failed" << std::endl; return -1; } - if (ngtcp2_crypto_derive_and_install_tx_key(quic.conn, nullptr, nullptr, - nullptr, level, tx_secret, - secretlen) != 0) { - std::cerr << "ngtcp2_crypto_derive_and_install_tx_key() failed" - << std::endl; - return -1; - } - if (level == NGTCP2_CRYPTO_LEVEL_APPLICATION) { auto s = std::make_unique(this); if (s->init_conn() == -1) { @@ -490,6 +576,19 @@ int Client::quic_on_key(ngtcp2_crypto_level level, const uint8_t *rx_secret, return 0; } +int Client::quic_on_tx_secret(ngtcp2_crypto_level level, const uint8_t *secret, + size_t secretlen) { + if (ngtcp2_crypto_derive_and_install_tx_key(quic.conn, nullptr, nullptr, + nullptr, level, secret, + secretlen) != 0) { + std::cerr << "ngtcp2_crypto_derive_and_install_tx_key() failed" + << std::endl; + return -1; + } + + return 0; +} + void Client::quic_set_tls_alert(uint8_t alert) { quic.last_error = quic::err_transport_tls(alert); } @@ -554,14 +653,25 @@ int Client::read_quic() { ++worker->stats.udp_dgram_recv; auto path = ngtcp2_path{ - {local_addr.len, &local_addr.su.sa}, - {addrlen, &su.sa}, + { + &local_addr.su.sa, + local_addr.len, + }, + { + &su.sa, + addrlen, + }, }; rv = ngtcp2_conn_read_pkt(quic.conn, &path, &pi, buf.data(), nread, timestamp(worker->loop)); if (rv != 0) { std::cerr << "ngtcp2_conn_read_pkt: " << ngtcp2_strerror(rv) << std::endl; + + if (!quic.last_error.code) { + quic.last_error = quic::err_transport(rv); + } + return -1; } diff --git a/src/http2_test.cc b/src/http2_test.cc index b998ee71..2d1ab43b 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -916,7 +916,7 @@ void test_http2_rewrite_clean_path(void) { CU_ASSERT("/delta%3A" == http2::rewrite_clean_path( balloc, StringRef::from_lit("/delta%3a"))); - // path component is normalized before mathcing + // path component is normalized before matching CU_ASSERT( "/alpha/bravo/" == http2::rewrite_clean_path( diff --git a/src/nghttp.cc b/src/nghttp.cc index 30ef26d3..5d62baef 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -2268,7 +2268,7 @@ int communicate( auto loop = EV_DEFAULT; SSL_CTX *ssl_ctx = nullptr; if (scheme == "https") { - ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + ssl_ctx = SSL_CTX_new(TLS_client_method()); if (!ssl_ctx) { std::cerr << "[ERROR] Failed to create SSL_CTX: " << ERR_error_string(ERR_get_error(), nullptr) << std::endl; diff --git a/src/shrpx.cc b/src/shrpx.cc index 4938f5cf..f81add5d 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -76,6 +76,11 @@ #include +#ifdef ENABLE_HTTP3 +# include +# include +#endif // ENABLE_HTTP3 + #include "shrpx_config.h" #include "shrpx_tls.h" #include "shrpx_log_config.h" @@ -202,7 +207,8 @@ struct WorkerProcess { ) : loop(loop), worker_pid(worker_pid), - ipc_fd(ipc_fd) + ipc_fd(ipc_fd), + termination_deadline(0.) #ifdef ENABLE_HTTP3 , quic_ipc_fd(quic_ipc_fd), @@ -264,6 +270,7 @@ struct WorkerProcess { struct ev_loop *loop; pid_t worker_pid; int ipc_fd; + ev_tstamp termination_deadline; #ifdef ENABLE_HTTP3 int quic_ipc_fd; std::vector> cid_prefixes; @@ -278,6 +285,74 @@ namespace { std::deque> worker_processes; } // namespace +namespace { +ev_timer worker_process_grace_period_timer; +} // namespace + +namespace { +void worker_process_grace_period_timercb(struct ev_loop *loop, ev_timer *w, + int revents) { + auto now = ev_now(loop); + ev_tstamp next_repeat = 0.; + + for (auto it = std::begin(worker_processes); + it != std::end(worker_processes);) { + auto &wp = *it; + if (!(wp->termination_deadline > 0.)) { + ++it; + + continue; + } + + auto d = wp->termination_deadline - now; + if (d > 0) { + if (!(next_repeat > 0.) || d < next_repeat) { + next_repeat = d; + } + + ++it; + + continue; + } + + LOG(NOTICE) << "Deleting worker process pid=" << wp->worker_pid + << " because its grace shutdown period is over"; + + it = worker_processes.erase(it); + } + + if (next_repeat > 0.) { + w->repeat = next_repeat; + ev_timer_again(loop, w); + + return; + } + + ev_timer_stop(loop, w); +} +} // namespace + +namespace { +void worker_process_set_termination_deadline(WorkerProcess *wp, + struct ev_loop *loop) { + auto config = get_config(); + + if (!(config->worker_process_grace_shutdown_period > 0.)) { + return; + } + + wp->termination_deadline = + ev_now(loop) + config->worker_process_grace_shutdown_period; + + if (!ev_is_active(&worker_process_grace_period_timer)) { + worker_process_grace_period_timer.repeat = + config->worker_process_grace_shutdown_period; + + ev_timer_again(loop, &worker_process_grace_period_timer); + } +} +} // namespace + namespace { void worker_process_add(std::unique_ptr wp) { worker_processes.push_back(std::move(wp)); @@ -285,7 +360,7 @@ void worker_process_add(std::unique_ptr wp) { } // namespace namespace { -void worker_process_remove(const WorkerProcess *wp) { +void worker_process_remove(const WorkerProcess *wp, struct ev_loop *loop) { for (auto it = std::begin(worker_processes); it != std::end(worker_processes); ++it) { auto &s = *it; @@ -295,28 +370,46 @@ void worker_process_remove(const WorkerProcess *wp) { } worker_processes.erase(it); + + if (worker_processes.empty()) { + ev_timer_stop(loop, &worker_process_grace_period_timer); + } + break; } } } // namespace namespace { -void worker_process_remove_all() { +void worker_process_adjust_limit() { + auto config = get_config(); + + if (config->max_worker_processes && + worker_processes.size() > config->max_worker_processes) { + worker_processes.pop_front(); + } +} +} // namespace + +namespace { +void worker_process_remove_all(struct ev_loop *loop) { std::deque>().swap(worker_processes); + + ev_timer_stop(loop, &worker_process_grace_period_timer); } } // namespace namespace { // Send signal |signum| to all worker processes, and clears // worker_processes. -void worker_process_kill(int signum) { +void worker_process_kill(int signum, struct ev_loop *loop) { for (auto &s : worker_processes) { if (s->worker_pid == -1) { continue; } kill(s->worker_pid, signum); } - worker_process_remove_all(); + worker_process_remove_all(loop); } } // namespace @@ -635,13 +728,14 @@ void signal_cb(struct ev_loop *loop, ev_signal *w, int revents) { close(addr.fd); } ipc_send(wp, SHRPX_IPC_GRACEFUL_SHUTDOWN); + worker_process_set_termination_deadline(wp, loop); return; } case RELOAD_SIGNAL: reload_config(wp); return; default: - worker_process_kill(w->signum); + worker_process_kill(w->signum, loop); ev_break(loop); return; } @@ -656,7 +750,7 @@ void worker_process_child_cb(struct ev_loop *loop, ev_child *w, int revents) { auto pid = wp->worker_pid; - worker_process_remove(wp); + worker_process_remove(wp, loop); if (worker_process_last_pid() == pid) { ev_break(loop); @@ -1342,6 +1436,7 @@ int generate_cid_prefix( std::vector> &cid_prefixes, const Config *config) { auto &apiconf = config->api; + auto &quicconf = config->quic; size_t num_cid_prefix; if (config->single_thread) { @@ -1360,7 +1455,7 @@ int generate_cid_prefix( cid_prefixes.resize(num_cid_prefix); for (auto &cid_prefix : cid_prefixes) { - if (create_cid_prefix(cid_prefix.data()) != 0) { + if (create_cid_prefix(cid_prefix.data(), quicconf.server_id.data()) != 0) { return -1; } } @@ -1469,7 +1564,7 @@ pid_t fork_worker_process( // Remove all WorkerProcesses to stop any registered watcher on // default loop. - worker_process_remove_all(); + worker_process_remove_all(EV_DEFAULT); close_unused_inherited_addr(iaddrs); @@ -1644,6 +1739,9 @@ int event_loop() { return -1; } + ev_timer_init(&worker_process_grace_period_timer, + worker_process_grace_period_timercb, 0., 0.); + worker_process_add(std::make_unique(loop, pid, ipc_fd #ifdef ENABLE_HTTP3 , @@ -1670,6 +1768,8 @@ int event_loop() { ev_run(loop, 0); + ev_timer_stop(loop, &worker_process_grace_period_timer); + return 0; } } // namespace @@ -1854,6 +1954,16 @@ void fill_default_config(Config *config) { auto &bpfconf = quicconf.bpf; bpfconf.prog_file = StringRef::from_lit(PKGLIBDIR "/reuseport_kern.o"); + + upstreamconf.congestion_controller = NGTCP2_CC_ALGO_CUBIC; + + upstreamconf.initial_rtt = + static_cast(NGTCP2_DEFAULT_INITIAL_RTT) / NGTCP2_SECONDS; + } + + if (RAND_bytes(quicconf.server_id.data(), quicconf.server_id.size()) != 1) { + assert(0); + abort(); } auto &http3conf = config->http3; @@ -1948,14 +2058,18 @@ void fill_default_config(Config *config) { namespace { void print_version(std::ostream &out) { - out << "nghttpx nghttp2/" NGHTTP2_VERSION << std::endl; + out << "nghttpx nghttp2/" NGHTTP2_VERSION +#ifdef ENABLE_HTTP3 + " ngtcp2/" NGTCP2_VERSION " nghttp3/" NGHTTP3_VERSION +#endif // ENABLE_HTTP3 + << std::endl; } } // namespace namespace { void print_usage(std::ostream &out) { out << R"(Usage: nghttpx [OPTIONS]... [ ] -A reverse proxy for HTTP/2, and HTTP/1.)" +A reverse proxy for HTTP/3, HTTP/2, and HTTP/1.)" << std::endl; } } // namespace @@ -2366,6 +2480,12 @@ Performance: If 0 is given, nghttpx does not set the limit. Default: )" << config->rlimit_nofile << R"( + --rlimit-memlock= + Set maximum number of bytes of memory that may be locked + into RAM. If 0 is given, nghttpx does not set the + limit. + Default: )" + << config->rlimit_memlock << R"( --backend-request-buffer= Set buffer size used to store backend request. Default: )" @@ -3054,7 +3174,7 @@ HTTP: advertised in alt-svc header field only in HTTP/1.1 frontend. This option can be used multiple times to specify multiple alternative services. - Example: --altsvc="h2,443,,,ma=3600; persist=1' + Example: --altsvc="h2,443,,,ma=3600; persist=1" --http2-altsvc= Just like --altsvc option, but this altsvc is only sent in HTTP/2 frontend. @@ -3185,6 +3305,27 @@ Process: process. nghttpx still spawns additional process if neverbleed is used. In the single process mode, the signal handling feature is disabled. + --max-worker-processes= + The maximum number of worker processes. nghttpx spawns + new worker process when it reloads its configuration. + The previous worker process enters graceful termination + period and will terminate when it finishes handling the + existing connections. However, if reloading + configurations happen very frequently, the worker + processes might be piled up if they take a bit long time + to finish the existing connections. With this option, + if the number of worker processes exceeds the given + value, the oldest worker process is terminated + immediately. Specifying 0 means no limit and it is the + default behaviour. + --worker-process-grace-shutdown-period= + Maximum period for a worker process to terminate + gracefully. When a worker process enters in graceful + shutdown period (e.g., when nghttpx reloads its + configuration) and it does not finish handling the + existing connections in the given period of time, it is + immediately terminated. Specifying 0 means no limit and + it is the default behaviour. Scripting: --mruby-file= @@ -3221,7 +3362,57 @@ HTTP/3 and QUIC: frontend QUIC connections. A qlog file is created per each QUIC connection. The file name is ISO8601 basic format, followed by "-", server Source Connection ID and - ".qlog". + ".sqlog". + --frontend-quic-require-token + Require an address validation token for a frontend QUIC + connection. Server sends a token in Retry packet or + NEW_TOKEN frame in the previous connection. + --frontend-quic-congestion-controller= + Specify a congestion controller algorithm for a frontend + QUIC connection. should be either "cubic" or + "bbr". + Default: )" + << (config->quic.upstream.congestion_controller == NGTCP2_CC_ALGO_CUBIC + ? "cubic" + : "bbr") + << R"( + --frontend-quic-secret-file= + Path to file that contains secure random data to be used + as QUIC keying materials. It is used to derive keys for + encrypting tokens and Connection IDs. It is not used to + encrypt QUIC packets. Each line of this file must + contain exactly 136 bytes hex-encoded string (when + decoded the byte string is 68 bytes long). The first 2 + bits of decoded byte string are used to identify the + keying material. An empty line or a line which starts + '#' is ignored. The file can contain more than one + keying materials. Because the identifier is 2 bits, at + most 4 keying materials are read and the remaining data + is discarded. The first keying material in the file is + primarily used for encryption and decryption for new + connection. The other ones are used to decrypt data for + the existing connections. Specifying multiple keying + materials enables key rotation. Please note that key + rotation does not occur automatically. User should + update files or change options values and restart + nghttpx gracefully. If opening or reading given file + fails, all loaded keying materials are discarded and it + is treated as if none of this option is given. If this + option is not given or an error occurred while opening + or reading a file, a keying material is generated + internally on startup and reload. + --quic-server-id= + Specify server ID encoded in Connection ID to identify + this particular server instance. Connection ID is + encrypted and this part is not visible in public. It + must be 4 bytes long and must be encoded in hex string + (which is 8 bytes long). If this option is omitted, a + random server ID is generated on startup and + configuration reload. + --frontend-quic-initial-rtt= + Specify the initial RTT of the frontend QUIC connection. + Default: )" + << util::duration_str(config->quic.upstream.initial_rtt) << R"( --no-quic-bpf Disable eBPF. --frontend-http3-window-size= @@ -3475,9 +3666,12 @@ int process_options(Config *config, return -1; } + std::array hostport_buf; + auto &proxy = config->downstream_http_proxy; if (!proxy.host.empty()) { - auto hostport = util::make_hostport(StringRef{proxy.host}, proxy.port); + auto hostport = util::make_hostport(std::begin(hostport_buf), + StringRef{proxy.host}, proxy.port); if (resolve_hostname(&proxy.addr, proxy.host.c_str(), proxy.port, AF_UNSPEC) == -1) { LOG(FATAL) << "Resolving backend HTTP proxy address failed: " << hostport; @@ -3490,7 +3684,8 @@ int process_options(Config *config, { auto &memcachedconf = tlsconf.session_cache.memcached; if (!memcachedconf.host.empty()) { - auto hostport = util::make_hostport(StringRef{memcachedconf.host}, + auto hostport = util::make_hostport(std::begin(hostport_buf), + StringRef{memcachedconf.host}, memcachedconf.port); if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.c_str(), memcachedconf.port, memcachedconf.family) == -1) { @@ -3511,7 +3706,8 @@ int process_options(Config *config, { auto &memcachedconf = tlsconf.ticket.memcached; if (!memcachedconf.host.empty()) { - auto hostport = util::make_hostport(StringRef{memcachedconf.host}, + auto hostport = util::make_hostport(std::begin(hostport_buf), + StringRef{memcachedconf.host}, memcachedconf.port); if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.c_str(), memcachedconf.port, memcachedconf.family) == -1) { @@ -3538,6 +3734,18 @@ int process_options(Config *config, } } +#ifdef RLIMIT_MEMLOCK + if (config->rlimit_memlock) { + struct rlimit lim = {static_cast(config->rlimit_memlock), + static_cast(config->rlimit_memlock)}; + if (setrlimit(RLIMIT_MEMLOCK, &lim) != 0) { + auto error = errno; + LOG(WARN) << "Setting rlimit-memlock failed: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + } + } +#endif // RLIMIT_MEMLOCK + auto &fwdconf = config->http.forwarded; if (fwdconf.by_node_type == ForwardedNode::OBFUSCATED && @@ -3682,6 +3890,7 @@ void reload_config(WorkerProcess *wp) { // Send last worker process a graceful shutdown notice auto &last_wp = worker_processes.back(); ipc_send(last_wp.get(), SHRPX_IPC_GRACEFUL_SHUTDOWN); + worker_process_set_termination_deadline(last_wp.get(), loop); // We no longer use signals for this worker. last_wp->shutdown_signal_watchers(); @@ -3692,6 +3901,8 @@ void reload_config(WorkerProcess *wp) { #endif // ENABLE_HTTP3 )); + worker_process_adjust_limit(); + if (!get_config()->pid_file.empty()) { save_pid(); } @@ -4011,6 +4222,19 @@ int main(int argc, char **argv) { {SHRPX_OPT_FRONTEND_QUIC_EARLY_DATA.c_str(), no_argument, &flag, 180}, {SHRPX_OPT_FRONTEND_QUIC_QLOG_DIR.c_str(), required_argument, &flag, 181}, + {SHRPX_OPT_FRONTEND_QUIC_REQUIRE_TOKEN.c_str(), no_argument, &flag, + 182}, + {SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER.c_str(), + required_argument, &flag, 183}, + {SHRPX_OPT_QUIC_SERVER_ID.c_str(), required_argument, &flag, 185}, + {SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE.c_str(), required_argument, &flag, + 186}, + {SHRPX_OPT_RLIMIT_MEMLOCK.c_str(), required_argument, &flag, 187}, + {SHRPX_OPT_MAX_WORKER_PROCESSES.c_str(), required_argument, &flag, 188}, + {SHRPX_OPT_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD.c_str(), + required_argument, &flag, 189}, + {SHRPX_OPT_FRONTEND_QUIC_INITIAL_RTT.c_str(), required_argument, &flag, + 190}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -4878,6 +5102,43 @@ int main(int argc, char **argv) { cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_QLOG_DIR, StringRef{optarg}); break; + case 182: + // --frontend-quic-require-token + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_REQUIRE_TOKEN, + StringRef::from_lit("yes")); + break; + case 183: + // --frontend-quic-congestion-controller + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER, + StringRef{optarg}); + break; + case 185: + // --quic-server-id + cmdcfgs.emplace_back(SHRPX_OPT_QUIC_SERVER_ID, StringRef{optarg}); + break; + case 186: + // --frontend-quic-secret-file + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE, + StringRef{optarg}); + break; + case 187: + // --rlimit-memlock + cmdcfgs.emplace_back(SHRPX_OPT_RLIMIT_MEMLOCK, StringRef{optarg}); + break; + case 188: + // --max-worker-processes + cmdcfgs.emplace_back(SHRPX_OPT_MAX_WORKER_PROCESSES, StringRef{optarg}); + break; + case 189: + // --worker-process-grace-shutdown-period + cmdcfgs.emplace_back(SHRPX_OPT_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD, + StringRef{optarg}); + break; + case 190: + // --frontend-quic-initial-rtt + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_INITIAL_RTT, + StringRef{optarg}); + break; default: break; } diff --git a/src/shrpx_api_downstream_connection.cc b/src/shrpx_api_downstream_connection.cc index 063eed70..254ab59e 100644 --- a/src/shrpx_api_downstream_connection.cc +++ b/src/shrpx_api_downstream_connection.cc @@ -463,7 +463,7 @@ int APIDownstreamConnection::on_read() { return 0; } int APIDownstreamConnection::on_write() { return 0; } -void APIDownstreamConnection::on_upstream_change(Upstream *uptream) {} +void APIDownstreamConnection::on_upstream_change(Upstream *upstream) {} bool APIDownstreamConnection::poolable() const { return false; } diff --git a/src/shrpx_api_downstream_connection.h b/src/shrpx_api_downstream_connection.h index af6b95eb..5d4182f0 100644 --- a/src/shrpx_api_downstream_connection.h +++ b/src/shrpx_api_downstream_connection.h @@ -81,7 +81,7 @@ public: virtual int on_read(); virtual int on_write(); - virtual void on_upstream_change(Upstream *uptream); + virtual void on_upstream_change(Upstream *upstream); // true if this object is poolable. virtual bool poolable() const; diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 234c2b8c..a8975cab 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -292,11 +292,12 @@ int ClientHandler::write_tls() { #ifdef ENABLE_HTTP3 int ClientHandler::read_quic(const UpstreamAddr *faddr, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, + const Address &local_addr, + const ngtcp2_pkt_info &pi, const uint8_t *data, size_t datalen) { auto upstream = static_cast(upstream_.get()); - return upstream->on_read(faddr, remote_addr, local_addr, data, datalen); + return upstream->on_read(faddr, remote_addr, local_addr, pi, data, datalen); } int ClientHandler::write_quic() { return upstream_->on_write(); } @@ -517,7 +518,6 @@ void ClientHandler::setup_upstream_io_callback() { void ClientHandler::setup_http3_upstream( std::unique_ptr &&upstream) { upstream_ = std::move(upstream); - alpn_ = StringRef::from_lit("h3"); write_ = &ClientHandler::write_quic; auto config = get_config(); @@ -884,7 +884,6 @@ DownstreamAddr *ClientHandler::get_downstream_addr(int &err, err = -1; return nullptr; } - aff_idx = i; } return addr; @@ -1599,4 +1598,13 @@ StringRef ClientHandler::get_alpn() const { return alpn_; } BlockAllocator &ClientHandler::get_block_allocator() { return balloc_; } +void ClientHandler::set_alpn_from_conn() { + const unsigned char *alpn; + unsigned int alpnlen; + + SSL_get0_alpn_selected(conn_.tls.ssl, &alpn, &alpnlen); + + alpn_ = make_string_ref(balloc_, StringRef{alpn, alpnlen}); +} + } // namespace shrpx diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index 0c013973..c2c50c0b 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -149,7 +149,8 @@ public: #ifdef ENABLE_HTTP3 void setup_http3_upstream(std::unique_ptr &&upstream); int read_quic(const UpstreamAddr *faddr, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, size_t datalen); + const Address &local_addr, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen); int write_quic(); #endif // ENABLE_HTTP3 @@ -187,6 +188,8 @@ public: BlockAllocator &get_block_allocator(); + void set_alpn_from_conn(); + private: // Allocator to allocate memory for connection-wide objects. Make // sure that the allocations must be bounded, and not proportional diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 356c4bca..e158ba6d 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -230,10 +230,81 @@ read_tls_ticket_key_file(const std::vector &files, return ticket_keys; } +#ifdef ENABLE_HTTP3 +std::shared_ptr +read_quic_secret_file(const StringRef &path) { + constexpr size_t expectedlen = + SHRPX_QUIC_SECRET_RESERVEDLEN + SHRPX_QUIC_SECRETLEN + SHRPX_QUIC_SALTLEN; + + auto qkms = std::make_shared(); + auto &kms = qkms->keying_materials; + + std::ifstream f(path.c_str()); + if (!f) { + LOG(ERROR) << "frontend-quic-secret-file: could not open file " << path; + return nullptr; + } + + std::array buf; + + while (f.getline(buf.data(), buf.size())) { + auto len = strlen(buf.data()); + if (len == 0 || buf[0] == '#') { + continue; + } + + auto s = StringRef{std::begin(buf), std::begin(buf) + len}; + if (s.size() != expectedlen * 2 || !util::is_hex_string(s)) { + LOG(ERROR) << "frontend-quic-secret-file: each line must be a " + << expectedlen * 2 << " bytes hex encoded string"; + return nullptr; + } + + kms.emplace_back(); + auto &qkm = kms.back(); + + auto p = std::begin(s); + + util::decode_hex(std::begin(qkm.reserved), + StringRef{p, p + qkm.reserved.size()}); + p += qkm.reserved.size() * 2; + util::decode_hex(std::begin(qkm.secret), + StringRef{p, p + qkm.secret.size()}); + p += qkm.secret.size() * 2; + util::decode_hex(std::begin(qkm.salt), StringRef{p, p + qkm.salt.size()}); + p += qkm.salt.size() * 2; + + assert(static_cast(p - std::begin(s)) == expectedlen * 2); + + qkm.id = qkm.reserved[0] & 0xc0; + + if (kms.size() == 4) { + break; + } + } + + if (f.bad() || (!f.eof() && f.fail())) { + LOG(ERROR) + << "frontend-quic-secret-file: error occurred while reading file " + << path; + return nullptr; + } + + if (kms.empty()) { + LOG(WARN) + << "frontend-quic-secret-file: no keying materials are present in file " + << path; + return nullptr; + } + + return qkms; +} +#endif // ENABLE_HTTP3 + FILE *open_file_for_write(const char *filename) { std::array errbuf; -#if defined O_CLOEXEC +#ifdef O_CLOEXEC auto fd = open(filename, O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); #else @@ -1983,6 +2054,11 @@ int option_lookup_token(const char *name, size_t namelen) { break; case 14: switch (name[13]) { + case 'd': + if (util::strieq_l("quic-server-i", name, 13)) { + return SHRPX_OPTID_QUIC_SERVER_ID; + } + break; case 'e': if (util::strieq_l("accesslog-fil", name, 13)) { return SHRPX_OPTID_ACCESSLOG_FILE; @@ -1993,6 +2069,11 @@ int option_lookup_token(const char *name, size_t namelen) { return SHRPX_OPTID_NO_SERVER_PUSH; } break; + case 'k': + if (util::strieq_l("rlimit-memloc", name, 13)) { + return SHRPX_OPTID_RLIMIT_MEMLOCK; + } + break; case 'p': if (util::strieq_l("no-verify-ocs", name, 13)) { return SHRPX_OPTID_NO_VERIFY_OCSP; @@ -2172,6 +2253,9 @@ int option_lookup_token(const char *name, size_t namelen) { } break; case 's': + if (util::strieq_l("max-worker-processe", name, 19)) { + return SHRPX_OPTID_MAX_WORKER_PROCESSES; + } if (util::strieq_l("tls13-client-cipher", name, 19)) { return SHRPX_OPTID_TLS13_CLIENT_CIPHERS; } @@ -2339,6 +2423,9 @@ int option_lookup_token(const char *name, size_t namelen) { if (util::strieq_l("backend-http2-window-siz", name, 24)) { return SHRPX_OPTID_BACKEND_HTTP2_WINDOW_SIZE; } + if (util::strieq_l("frontend-quic-secret-fil", name, 24)) { + return SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE; + } break; case 'g': if (util::strieq_l("http2-no-cookie-crumblin", name, 24)) { @@ -2353,6 +2440,11 @@ int option_lookup_token(const char *name, size_t namelen) { return SHRPX_OPTID_MAX_REQUEST_HEADER_FIELDS; } break; + case 't': + if (util::strieq_l("frontend-quic-initial-rt", name, 24)) { + return SHRPX_OPTID_FRONTEND_QUIC_INITIAL_RTT; + } + break; } break; case 26: @@ -2401,6 +2493,11 @@ int option_lookup_token(const char *name, size_t namelen) { return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED; } break; + case 'n': + if (util::strieq_l("frontend-quic-require-toke", name, 26)) { + return SHRPX_OPTID_FRONTEND_QUIC_REQUIRE_TOKEN; + } + break; case 'r': if (util::strieq_l("request-header-field-buffe", name, 26)) { return SHRPX_OPTID_REQUEST_HEADER_FIELD_BUFFER; @@ -2566,11 +2663,19 @@ int option_lookup_token(const char *name, size_t namelen) { if (util::strieq_l("frontend-http2-dump-response-heade", name, 34)) { return SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER; } + if (util::strieq_l("frontend-quic-congestion-controlle", name, 34)) { + return SHRPX_OPTID_FRONTEND_QUIC_CONGESTION_CONTROLLER; + } break; } break; case 36: switch (name[35]) { + case 'd': + if (util::strieq_l("worker-process-grace-shutdown-perio", name, 35)) { + return SHRPX_OPTID_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD; + } + break; case 'e': if (util::strieq_l("backend-http2-connection-window-siz", name, 35)) { return SHRPX_OPTID_BACKEND_HTTP2_CONNECTION_WINDOW_SIZE; @@ -2765,9 +2870,16 @@ int parse_config(Config *config, int optid, const StringRef &opt, return -1; } - if (params.quic && params.alt_mode != UpstreamAltMode::NONE) { - LOG(ERROR) << "frontend: api or healthmon cannot be used with quic"; - return -1; + if (params.quic) { + if (params.alt_mode != UpstreamAltMode::NONE) { + LOG(ERROR) << "frontend: api or healthmon cannot be used with quic"; + return -1; + } + + if (!params.tls) { + LOG(ERROR) << "frontend: quic requires TLS"; + return -1; + } } UpstreamAddr addr{}; @@ -3842,7 +3954,7 @@ int parse_config(Config *config, int optid, const StringRef &opt, "65535], inclusive"; return -1; } - config->http.redirect_https_port = optarg; + config->http.redirect_https_port = make_string_ref(config->balloc, optarg); return 0; } case SHRPX_OPTID_FRONTEND_MAX_REQUESTS: @@ -3983,10 +4095,75 @@ int parse_config(Config *config, int optid, const StringRef &opt, return 0; case SHRPX_OPTID_FRONTEND_QUIC_QLOG_DIR: #ifdef ENABLE_HTTP3 - config->quic.upstream.qlog.dir = optarg; + config->quic.upstream.qlog.dir = make_string_ref(config->balloc, optarg); #endif // ENABLE_HTTP3 return 0; + case SHRPX_OPTID_FRONTEND_QUIC_REQUIRE_TOKEN: +#ifdef ENABLE_HTTP3 + config->quic.upstream.require_token = util::strieq_l("yes", optarg); +#endif // ENABLE_HTTP3 + + return 0; + case SHRPX_OPTID_FRONTEND_QUIC_CONGESTION_CONTROLLER: +#ifdef ENABLE_HTTP3 + if (util::strieq_l("cubic", optarg)) { + config->quic.upstream.congestion_controller = NGTCP2_CC_ALGO_CUBIC; + } else if (util::strieq_l("bbr", optarg)) { + config->quic.upstream.congestion_controller = NGTCP2_CC_ALGO_BBR; + } else { + LOG(ERROR) << opt << ": must be either cubic or bbr"; + return -1; + } +#endif // ENABLE_HTTP3 + + return 0; + case SHRPX_OPTID_QUIC_SERVER_ID: +#ifdef ENABLE_HTTP3 + if (optarg.size() != config->quic.server_id.size() * 2 || + !util::is_hex_string(optarg)) { + LOG(ERROR) << opt << ": must be a hex-string"; + return -1; + } + util::decode_hex(std::begin(config->quic.server_id), optarg); +#endif // ENABLE_HTTP3 + + return 0; + case SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE: +#ifdef ENABLE_HTTP3 + config->quic.upstream.secret_file = make_string_ref(config->balloc, optarg); +#endif // ENABLE_HTTP3 + + return 0; + case SHRPX_OPTID_RLIMIT_MEMLOCK: { + int n; + + if (parse_uint(&n, opt, optarg) != 0) { + return -1; + } + + if (n < 0) { + LOG(ERROR) << opt << ": specify the integer more than or equal to 0"; + + return -1; + } + + config->rlimit_memlock = n; + + return 0; + } + case SHRPX_OPTID_MAX_WORKER_PROCESSES: + return parse_uint(&config->max_worker_processes, opt, optarg); + case SHRPX_OPTID_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD: + return parse_duration(&config->worker_process_grace_shutdown_period, opt, + optarg); + case SHRPX_OPTID_FRONTEND_QUIC_INITIAL_RTT: { +#ifdef ENABLE_HTTP3 + return parse_duration(&config->quic.upstream.initial_rtt, opt, optarg); +#endif // ENABLE_HTTP3 + + return 0; + } case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; @@ -4286,7 +4463,7 @@ int configure_downstream_group(Config *config, bool http2_proxy, if (!g.mruby_file.empty()) { if (mruby::create_mruby_context(g.mruby_file) == nullptr) { LOG(config->ignore_per_pattern_mruby_error ? ERROR : FATAL) - << "backend: Could not compile mruby flie for pattern " + << "backend: Could not compile mruby file for pattern " << g.pattern; if (!config->ignore_per_pattern_mruby_error) { return -1; @@ -4321,6 +4498,8 @@ int configure_downstream_group(Config *config, bool http2_proxy, auto resolve_flags = numeric_addr_only ? AI_NUMERICHOST | AI_NUMERICSERV : 0; + std::array hostport_buf; + for (auto &g : addr_groups) { std::unordered_map wgchk; for (auto &addr : g.addrs) { @@ -4366,7 +4545,7 @@ int configure_downstream_group(Config *config, bool http2_proxy, util::make_http_hostport(downstreamconf.balloc, addr.host, addr.port); auto hostport = - util::make_hostport(downstreamconf.balloc, addr.host, addr.port); + util::make_hostport(std::begin(hostport_buf), addr.host, addr.port); if (!addr.dns) { if (resolve_hostname(&addr.addr, addr.host.c_str(), addr.port, diff --git a/src/shrpx_config.h b/src/shrpx_config.h index fdb7bae0..b5871fff 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -387,6 +387,20 @@ constexpr auto SHRPX_OPT_FRONTEND_QUIC_EARLY_DATA = StringRef::from_lit("frontend-quic-early-data"); constexpr auto SHRPX_OPT_FRONTEND_QUIC_QLOG_DIR = StringRef::from_lit("frontend-quic-qlog-dir"); +constexpr auto SHRPX_OPT_FRONTEND_QUIC_REQUIRE_TOKEN = + StringRef::from_lit("frontend-quic-require-token"); +constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER = + StringRef::from_lit("frontend-quic-congestion-controller"); +constexpr auto SHRPX_OPT_QUIC_SERVER_ID = StringRef::from_lit("quic-server-id"); +constexpr auto SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE = + StringRef::from_lit("frontend-quic-secret-file"); +constexpr auto SHRPX_OPT_RLIMIT_MEMLOCK = StringRef::from_lit("rlimit-memlock"); +constexpr auto SHRPX_OPT_MAX_WORKER_PROCESSES = + StringRef::from_lit("max-worker-processes"); +constexpr auto SHRPX_OPT_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD = + StringRef::from_lit("worker-process-grace-shutdown-period"); +constexpr auto SHRPX_OPT_FRONTEND_QUIC_INITIAL_RTT = + StringRef::from_lit("frontend-quic-initial-rtt"); constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -598,10 +612,18 @@ struct TLSCertificate { }; #ifdef ENABLE_HTTP3 -struct QUICSecret { - std::array - stateless_reset_secret; - std::array token_secret; +struct QUICKeyingMaterial { + std::array reserved; + std::array secret; + std::array salt; + std::array cid_encryption_key; + // Identifier of this keying material. Only the first 2 bits are + // used. + uint8_t id; +}; + +struct QUICKeyingMaterials { + std::vector keying_materials; }; #endif // ENABLE_HTTP3 @@ -669,7 +691,7 @@ struct TLSConfig { ev_tstamp idle_timeout; } dyn_rec; - // OCSP realted configurations + // OCSP related configurations struct { ev_tstamp update_interval; StringRef fetch_ocsp_response_file; @@ -754,12 +776,17 @@ struct QUICConfig { struct { StringRef dir; } qlog; + ngtcp2_cc_algo congestion_controller; bool early_data; + bool require_token; + StringRef secret_file; + ev_tstamp initial_rtt; } upstream; struct { StringRef prog_file; bool disabled; } bpf; + std::array server_id; }; struct Http3Config { @@ -1044,6 +1071,7 @@ struct Config { num_worker{0}, padding{0}, rlimit_nofile{0}, + rlimit_memlock{0}, uid{0}, gid{0}, pid{0}, @@ -1053,7 +1081,9 @@ struct Config { single_process{false}, single_thread{false}, ignore_per_pattern_mruby_error{false}, - ev_loop_flags{0} { + ev_loop_flags{0}, + max_worker_processes{0}, + worker_process_grace_shutdown_period{0.} { } ~Config(); @@ -1092,6 +1122,7 @@ struct Config { size_t num_worker; size_t padding; size_t rlimit_nofile; + size_t rlimit_memlock; uid_t uid; gid_t gid; pid_t pid; @@ -1106,6 +1137,8 @@ struct Config { bool ignore_per_pattern_mruby_error; // flags passed to ev_default_loop() and ev_loop_new() int ev_loop_flags; + size_t max_worker_processes; + ev_tstamp worker_process_grace_shutdown_period; }; const Config *get_config(); @@ -1207,10 +1240,14 @@ enum { SHRPX_OPTID_FRONTEND_KEEP_ALIVE_TIMEOUT, SHRPX_OPTID_FRONTEND_MAX_REQUESTS, SHRPX_OPTID_FRONTEND_NO_TLS, + SHRPX_OPTID_FRONTEND_QUIC_CONGESTION_CONTROLLER, SHRPX_OPTID_FRONTEND_QUIC_DEBUG_LOG, SHRPX_OPTID_FRONTEND_QUIC_EARLY_DATA, SHRPX_OPTID_FRONTEND_QUIC_IDLE_TIMEOUT, + SHRPX_OPTID_FRONTEND_QUIC_INITIAL_RTT, SHRPX_OPTID_FRONTEND_QUIC_QLOG_DIR, + SHRPX_OPTID_FRONTEND_QUIC_REQUIRE_TOKEN, + SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE, SHRPX_OPTID_FRONTEND_READ_TIMEOUT, SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT, SHRPX_OPTID_HEADER_FIELD_BUFFER, @@ -1228,6 +1265,7 @@ enum { SHRPX_OPTID_MAX_HEADER_FIELDS, SHRPX_OPTID_MAX_REQUEST_HEADER_FIELDS, SHRPX_OPTID_MAX_RESPONSE_HEADER_FIELDS, + SHRPX_OPTID_MAX_WORKER_PROCESSES, SHRPX_OPTID_MRUBY_FILE, SHRPX_OPTID_NO_ADD_X_FORWARDED_PROTO, SHRPX_OPTID_NO_HOST_REWRITE, @@ -1252,11 +1290,13 @@ enum { SHRPX_OPTID_PRIVATE_KEY_PASSWD_FILE, SHRPX_OPTID_PSK_SECRETS, SHRPX_OPTID_QUIC_BPF_PROGRAM_FILE, + SHRPX_OPTID_QUIC_SERVER_ID, SHRPX_OPTID_READ_BURST, SHRPX_OPTID_READ_RATE, SHRPX_OPTID_REDIRECT_HTTPS_PORT, SHRPX_OPTID_REQUEST_HEADER_FIELD_BUFFER, SHRPX_OPTID_RESPONSE_HEADER_FIELD_BUFFER, + SHRPX_OPTID_RLIMIT_MEMLOCK, SHRPX_OPTID_RLIMIT_NOFILE, SHRPX_OPTID_SERVER_NAME, SHRPX_OPTID_SINGLE_PROCESS, @@ -1297,6 +1337,7 @@ enum { SHRPX_OPTID_VERIFY_CLIENT_CACERT, SHRPX_OPTID_VERIFY_CLIENT_TOLERATE_EXPIRED, SHRPX_OPTID_WORKER_FRONTEND_CONNECTIONS, + SHRPX_OPTID_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD, SHRPX_OPTID_WORKER_READ_BURST, SHRPX_OPTID_WORKER_READ_RATE, SHRPX_OPTID_WORKER_WRITE_BURST, @@ -1361,6 +1402,11 @@ std::unique_ptr read_tls_ticket_key_file(const std::vector &files, const EVP_CIPHER *cipher, const EVP_MD *hmac); +#ifdef ENABLE_HTTP3 +std::shared_ptr +read_quic_secret_file(const StringRef &path); +#endif // ENABLE_HTTP3 + // Returns string representation of |proto|. StringRef strproto(Proto proto); diff --git a/src/shrpx_connection.cc b/src/shrpx_connection.cc index d07314b6..b4211adc 100644 --- a/src/shrpx_connection.cc +++ b/src/shrpx_connection.cc @@ -397,11 +397,14 @@ int Connection::tls_handshake() { ERR_clear_error(); -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API || defined(OPENSSL_IS_BORINGSSL) + auto &tlsconf = get_config()->tls; +#endif // OPENSSL_1_1_1_API || defined(OPENSSL_IS_BORINGSSL) + +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (!tls.server_handshake || tls.early_data_finish) { rv = SSL_do_handshake(tls.ssl); } else { - auto &tlsconf = get_config()->tls; for (;;) { size_t nread; @@ -449,9 +452,9 @@ int Connection::tls_handshake() { } } } -#else // !OPENSSL_1_1_1_API +#else // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) rv = SSL_do_handshake(tls.ssl); -#endif // !OPENSSL_1_1_1_API +#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) if (rv <= 0) { auto err = SSL_get_error(tls.ssl, rv); @@ -499,7 +502,12 @@ int Connection::tls_handshake() { // Don't send handshake data if handshake was completed in OpenSSL // routine. We have to check HTTP/2 requirement if HTTP/2 was // negotiated before sending finished message to the peer. - if (rv != 1 && tls.wbuf.rleft()) { + if ((rv != 1 +#ifdef OPENSSL_IS_BORINGSSL + || SSL_in_init(tls.ssl) +#endif // OPENSSL_IS_BORINGSSL + ) && + tls.wbuf.rleft()) { // First write indicates that resumption stuff has done. if (tls.handshake_state != TLSHandshakeState::WRITE_STARTED) { tls.handshake_state = TLSHandshakeState::WRITE_STARTED; @@ -535,6 +543,40 @@ int Connection::tls_handshake() { return SHRPX_ERR_INPROGRESS; } +#ifdef OPENSSL_IS_BORINGSSL + if (!tlsconf.no_postpone_early_data && SSL_in_early_data(tls.ssl) && + SSL_in_init(tls.ssl)) { + auto nread = SSL_read(tls.ssl, buf.data(), buf.size()); + if (nread <= 0) { + auto err = SSL_get_error(tls.ssl, nread); + switch (err) { + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + break; + case SSL_ERROR_ZERO_RETURN: + return SHRPX_ERR_EOF; + case SSL_ERROR_SSL: + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "SSL_read: " + << ERR_error_string(ERR_get_error(), nullptr); + } + return SHRPX_ERR_NETWORK; + default: + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "SSL_read: SSL_get_error returned " << err; + } + return SHRPX_ERR_NETWORK; + } + } else { + tls.earlybuf.append(buf.data(), nread); + } + + if (SSL_in_init(tls.ssl)) { + return SHRPX_ERR_INPROGRESS; + } + } +#endif // OPENSSL_IS_BORINGSSL + // Handshake was done rv = check_http2_requirement(); @@ -571,6 +613,36 @@ int Connection::write_tls_pending_handshake() { tls.wbuf.drain(nwrite); } +#ifdef OPENSSL_IS_BORINGSSL + if (!SSL_in_init(tls.ssl)) { + // This will send a session ticket. + auto nwrite = SSL_write(tls.ssl, "", 0); + if (nwrite < 0) { + auto err = SSL_get_error(tls.ssl, nwrite); + switch (err) { + case SSL_ERROR_WANT_READ: + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Close connection due to TLS renegotiation"; + } + return SHRPX_ERR_NETWORK; + case SSL_ERROR_WANT_WRITE: + break; + case SSL_ERROR_SSL: + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "SSL_write: " + << ERR_error_string(ERR_get_error(), nullptr); + } + return SHRPX_ERR_NETWORK; + default: + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "SSL_write: SSL_get_error returned " << err; + } + return SHRPX_ERR_NETWORK; + } + } + } +#endif // OPENSSL_IS_BORINGSSL + // We have to start read watcher, since later stage of code expects // this. rlimit.startw(); @@ -681,7 +753,7 @@ ssize_t Connection::write_tls(const void *data, size_t len) { // length) on SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE. // get_write_limit() may return smaller length than previously // passed to SSL_write, which violates OpenSSL assumption. To avoid - // this, we keep last legnth passed to SSL_write to + // this, we keep last length passed to SSL_write to // tls.last_writelen if SSL_write indicated I/O blocking. if (tls.last_writelen == 0) { len = std::min(len, wlimit.avail()); @@ -698,7 +770,7 @@ ssize_t Connection::write_tls(const void *data, size_t len) { ERR_clear_error(); -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) int rv; if (SSL_is_init_finished(tls.ssl)) { rv = SSL_write(tls.ssl, data, len); @@ -710,9 +782,9 @@ ssize_t Connection::write_tls(const void *data, size_t len) { rv = nwrite; } } -#else // !OPENSSL_1_1_1_API +#else // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) auto rv = SSL_write(tls.ssl, data, len); -#endif // !OPENSSL_1_1_1_API +#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) if (rv <= 0) { auto err = SSL_get_error(tls.ssl, rv); @@ -759,7 +831,7 @@ ssize_t Connection::read_tls(void *data, size_t len) { // length) on SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE. // rlimit_.avail() or rlimit_.avail() may return different length // than the length previously passed to SSL_read, which violates - // OpenSSL assumption. To avoid this, we keep last legnth passed + // OpenSSL assumption. To avoid this, we keep last length passed // to SSL_read to tls_last_readlen_ if SSL_read indicated I/O // blocking. if (tls.last_readlen == 0) { @@ -772,7 +844,7 @@ ssize_t Connection::read_tls(void *data, size_t len) { tls.last_readlen = 0; } -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (!tls.early_data_finish) { // TLSv1.3 handshake is still going on. size_t nread; @@ -811,7 +883,7 @@ ssize_t Connection::read_tls(void *data, size_t len) { } return nread; } -#endif // OPENSSL_1_1_1_API +#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) auto rv = SSL_read(tls.ssl, data, len); diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index c32eb86f..4f216a5c 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -298,8 +298,6 @@ int ConnectionHandler::create_single_worker() { #endif // HAVE_MRUBY #ifdef ENABLE_HTTP3 - single_worker_->set_quic_secret(quic_secret_); - if (single_worker_->setup_quic_server_socket() != 0) { return -1; } @@ -404,8 +402,6 @@ int ConnectionHandler::create_worker_thread(size_t num) { # endif // HAVE_MRUBY # ifdef ENABLE_HTTP3 - worker->set_quic_secret(quic_secret_); - if ((!apiconf.enabled || i != 0) && worker->setup_quic_server_socket() != 0) { return -1; @@ -744,9 +740,9 @@ void ConnectionHandler::handle_ocsp_complete() { // that case we get nullptr. auto quic_ssl_ctx = quic_all_ssl_ctx_[ocsp_.next]; if (quic_ssl_ctx) { +# ifndef OPENSSL_IS_BORINGSSL auto quic_tls_ctx_data = static_cast( SSL_CTX_get_app_data(quic_ssl_ctx)); -# ifndef OPENSSL_IS_BORINGSSL # ifdef HAVE_ATOMIC_STD_SHARED_PTR std::atomic_store_explicit( &quic_tls_ctx_data->ocsp_data, @@ -758,7 +754,8 @@ void ConnectionHandler::handle_ocsp_complete() { std::make_shared>(ocsp_.resp); # endif // !HAVE_ATOMIC_STD_SHARED_PTR # else // OPENSSL_IS_BORINGSSL - SSL_CTX_set_ocsp_response(ssl_ctx, ocsp_.resp.data(), ocsp_.resp.size()); + SSL_CTX_set_ocsp_response(quic_ssl_ctx, ocsp_.resp.data(), + ocsp_.resp.size()); # endif // OPENSSL_IS_BORINGSSL } #endif // ENABLE_HTTP3 @@ -1020,12 +1017,10 @@ void ConnectionHandler::set_enable_acceptor_on_ocsp_completion(bool f) { } #ifdef ENABLE_HTTP3 -int ConnectionHandler::forward_quic_packet(const UpstreamAddr *faddr, - const Address &remote_addr, - const Address &local_addr, - const uint8_t *cid_prefix, - const uint8_t *data, - size_t datalen) { +int ConnectionHandler::forward_quic_packet( + const UpstreamAddr *faddr, const Address &remote_addr, + const Address &local_addr, const ngtcp2_pkt_info &pi, + const uint8_t *cid_prefix, const uint8_t *data, size_t datalen) { assert(!get_config()->single_thread); for (auto &worker : workers_) { @@ -1037,7 +1032,7 @@ int ConnectionHandler::forward_quic_packet(const UpstreamAddr *faddr, WorkerEvent wev{}; wev.type = WorkerEventType::QUIC_PKT_FORWARD; wev.quic_pkt = std::make_unique(faddr->index, remote_addr, - local_addr, data, datalen); + local_addr, pi, data, datalen); worker->send(std::move(wev)); @@ -1047,25 +1042,14 @@ int ConnectionHandler::forward_quic_packet(const UpstreamAddr *faddr, return -1; } -int ConnectionHandler::create_quic_secret() { - auto quic_secret = std::make_shared(); +void ConnectionHandler::set_quic_keying_materials( + std::shared_ptr qkms) { + quic_keying_materials_ = std::move(qkms); +} - if (generate_quic_stateless_reset_secret( - quic_secret->stateless_reset_secret.data()) != 0) { - LOG(ERROR) << "Failed to generate QUIC Stateless Reset secret"; - - return -1; - } - - if (generate_quic_token_secret(quic_secret->token_secret.data()) != 0) { - LOG(ERROR) << "Failed to generate QUIC token secret"; - - return -1; - } - - quic_secret_ = std::move(quic_secret); - - return 0; +const std::shared_ptr & +ConnectionHandler::get_quic_keying_materials() const { + return quic_keying_materials_; } void ConnectionHandler::set_cid_prefixes( @@ -1094,6 +1078,26 @@ ConnectionHandler::match_quic_lingering_worker_process_cid_prefix( std::vector &ConnectionHandler::get_quic_bpf_refs() { return quic_bpf_refs_; } + +void ConnectionHandler::unload_bpf_objects() { + std::array errbuf; + + LOG(NOTICE) << "Unloading BPF objects"; + + for (auto &ref : quic_bpf_refs_) { + if (ref.obj == nullptr) { + continue; + } + + if (bpf_object__unload(ref.obj) != 0) { + LOG(WARN) << "Failed to unload bpf object: " + << xsi_strerror(errno, errbuf.data(), errbuf.size()); + continue; + } + + ref.obj = nullptr; + } +} # endif // HAVE_LIBBPF void ConnectionHandler::set_quic_ipc_fd(int fd) { quic_ipc_fd_ = fd; } @@ -1105,10 +1109,11 @@ void ConnectionHandler::set_quic_lingering_worker_processes( int ConnectionHandler::forward_quic_packet_to_lingering_worker_process( QUICLingeringWorkerProcess *quic_lwp, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, size_t datalen) { + const Address &local_addr, const ngtcp2_pkt_info &pi, const uint8_t *data, + size_t datalen) { std::array header; - assert(header.size() >= 1 + 1 + 1 + sizeof(sockaddr_storage) * 2); + assert(header.size() >= 1 + 1 + 1 + 1 + sizeof(sockaddr_storage) * 2); assert(remote_addr.len > 0); assert(local_addr.len > 0); @@ -1121,6 +1126,7 @@ int ConnectionHandler::forward_quic_packet_to_lingering_worker_process( *p++ = static_cast(local_addr.len - 1); p = std::copy_n(reinterpret_cast(&local_addr.su), local_addr.len, p); + *p++ = pi.ecn; iovec msg_iov[] = { { @@ -1179,14 +1185,14 @@ int ConnectionHandler::quic_ipc_read() { return 0; } - size_t len = 1 + 1 + 1; + size_t len = 1 + 1 + 1 + 1; // Wire format: - // TYPE(1) REMOTE_ADDRLEN(1) REMOTE_ADDR(N) LOCAL_ADDRLEN(1) REMOTE_ADDR(N) - // DGRAM_PAYLAOD(N) + // TYPE(1) REMOTE_ADDRLEN(1) REMOTE_ADDR(N) LOCAL_ADDRLEN(1) LOCAL_ADDR(N) + // ECN(1) DGRAM_PAYLOAD(N) // - // When encoding, REMOTE_ADDRLEN and LOCAL_ADDRLEN is decremented by - // 1. + // When encoding, REMOTE_ADDRLEN and LOCAL_ADDRLEN are decremented + // by 1. if (static_cast(nread) < len) { return 0; } @@ -1243,6 +1249,8 @@ int ConnectionHandler::quic_ipc_read() { p += local_addrlen; + pkt->pi.ecn = *p++; + auto datalen = nread - (p - buf.data()); pkt->data.assign(p, p + datalen); @@ -1265,8 +1273,8 @@ int ConnectionHandler::quic_ipc_read() { return -1; } - if (dcidlen < SHRPX_QUIC_CID_PREFIXLEN) { - LOG(ERROR) << "DCID is too short"; + if (dcidlen != SHRPX_QUIC_SCIDLEN) { + LOG(ERROR) << "DCID length is invalid"; return -1; } @@ -1282,13 +1290,25 @@ int ConnectionHandler::quic_ipc_read() { // Ignore return value quic_conn_handler->handle_packet(faddr, pkt->remote_addr, pkt->local_addr, - pkt->data.data(), pkt->data.size()); + pkt->pi, pkt->data.data(), + pkt->data.size()); return 0; } + auto &qkm = quic_keying_materials_->keying_materials.front(); + + std::array decrypted_dcid; + + if (decrypt_quic_connection_id(decrypted_dcid.data(), + dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, + qkm.cid_encryption_key.data()) != 0) { + return -1; + } + for (auto &worker : workers_) { - if (!std::equal(dcid, dcid + SHRPX_QUIC_CID_PREFIXLEN, + if (!std::equal(std::begin(decrypted_dcid), + std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, worker->get_cid_prefix())) { continue; } diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h index 2c8d9c9a..e89b6d52 100644 --- a/src/shrpx_connection_handler.h +++ b/src/shrpx_connection_handler.h @@ -106,6 +106,7 @@ struct SerialEvent { #ifdef ENABLE_HTTP3 # ifdef HAVE_LIBBPF struct BPFRef { + bpf_object *obj; int reuseport_array; int cid_prefix_map; }; @@ -162,7 +163,7 @@ public: // Cancels ocsp update process void cancel_ocsp_update(); - // Starts ocsp update for certficate |cert_file|. + // Starts ocsp update for certificate |cert_file|. int start_ocsp_update(const char *cert_file); // Reads incoming data from ocsp update process void read_ocsp_chunk(); @@ -195,10 +196,12 @@ public: const std::vector &get_quic_indexed_ssl_ctx(size_t idx) const; int forward_quic_packet(const UpstreamAddr *faddr, const Address &remote_addr, - const Address &local_addr, const uint8_t *cid_prefix, - const uint8_t *data, size_t datalen); + const Address &local_addr, const ngtcp2_pkt_info &pi, + const uint8_t *cid_prefix, const uint8_t *data, + size_t datalen); - int create_quic_secret(); + void set_quic_keying_materials(std::shared_ptr qkms); + const std::shared_ptr &get_quic_keying_materials() const; void set_cid_prefixes( const std::vector> @@ -216,7 +219,8 @@ public: int forward_quic_packet_to_lingering_worker_process( QUICLingeringWorkerProcess *quic_lwp, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, size_t datalen); + const Address &local_addr, const ngtcp2_pkt_info &pi, const uint8_t *data, + size_t datalen); void set_quic_ipc_fd(int fd); @@ -224,6 +228,7 @@ public: # ifdef HAVE_LIBBPF std::vector &get_quic_bpf_refs(); + void unload_bpf_objects(); # endif // HAVE_LIBBPF #endif // ENABLE_HTTP3 @@ -263,7 +268,7 @@ private: # ifdef HAVE_LIBBPF std::vector quic_bpf_refs_; # endif // HAVE_LIBBPF - std::shared_ptr quic_secret_; + std::shared_ptr quic_keying_materials_; std::vector quic_all_ssl_ctx_; std::vector> quic_indexed_ssl_ctx_; #endif // ENABLE_HTTP3 diff --git a/src/shrpx_dns_resolver.h b/src/shrpx_dns_resolver.h index 696c5430..e622f996 100644 --- a/src/shrpx_dns_resolver.h +++ b/src/shrpx_dns_resolver.h @@ -88,7 +88,7 @@ public: int on_write(int fd); int on_timeout(); // Calls this function when DNS query finished. - void on_result(int staus, hostent *hostent); + void on_result(int status, hostent *hostent); void reset_timeout(); void start_rev(int fd); diff --git a/src/shrpx_downstream_connection.h b/src/shrpx_downstream_connection.h index 12bbbc5b..8efdcbef 100644 --- a/src/shrpx_downstream_connection.h +++ b/src/shrpx_downstream_connection.h @@ -58,7 +58,7 @@ public: virtual int on_write() = 0; virtual int on_timeout() { return 0; } - virtual void on_upstream_change(Upstream *uptream) = 0; + virtual void on_upstream_change(Upstream *upstream) = 0; // true if this object is poolable. virtual bool poolable() const = 0; diff --git a/src/shrpx_health_monitor_downstream_connection.cc b/src/shrpx_health_monitor_downstream_connection.cc index 066a2af9..89e53963 100644 --- a/src/shrpx_health_monitor_downstream_connection.cc +++ b/src/shrpx_health_monitor_downstream_connection.cc @@ -98,7 +98,8 @@ int HealthMonitorDownstreamConnection::on_read() { return 0; } int HealthMonitorDownstreamConnection::on_write() { return 0; } -void HealthMonitorDownstreamConnection::on_upstream_change(Upstream *uptream) {} +void HealthMonitorDownstreamConnection::on_upstream_change(Upstream *upstream) { +} bool HealthMonitorDownstreamConnection::poolable() const { return false; } diff --git a/src/shrpx_health_monitor_downstream_connection.h b/src/shrpx_health_monitor_downstream_connection.h index 7439e83b..c0eb6334 100644 --- a/src/shrpx_health_monitor_downstream_connection.h +++ b/src/shrpx_health_monitor_downstream_connection.h @@ -49,7 +49,7 @@ public: virtual int on_read(); virtual int on_write(); - virtual void on_upstream_change(Upstream *uptream); + virtual void on_upstream_change(Upstream *upstream); // true if this object is poolable. virtual bool poolable() const; diff --git a/src/shrpx_http2_downstream_connection.h b/src/shrpx_http2_downstream_connection.h index cdc55b9f..0fc7d91c 100644 --- a/src/shrpx_http2_downstream_connection.h +++ b/src/shrpx_http2_downstream_connection.h @@ -60,7 +60,7 @@ public: virtual void on_upstream_change(Upstream *upstream) {} - // This object is not poolable because we dont' have facility to + // This object is not poolable because we don't have facility to // migrate to another Http2Session object. virtual bool poolable() const { return false; } diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 2f9d2e50..37a6eb40 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -206,13 +206,13 @@ Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, on_read_ = &Http2Session::read_noop; on_write_ = &Http2Session::write_noop; - // We will resuse this many times, so use repeat timeout value. The + // We will reuse this many times, so use repeat timeout value. The // timeout value is set later. ev_timer_init(&connchk_timer_, connchk_timeout_cb, 0., 0.); connchk_timer_.data = this; - // SETTINGS ACK timeout is 10 seconds for now. We will resuse this + // SETTINGS ACK timeout is 10 seconds for now. We will reuse this // many times, so use repeat timeout value. ev_timer_init(&settings_timer_, settings_timeout_cb, 0., 0.); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 5f8b5254..8859ec37 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1317,7 +1317,7 @@ int Http2Upstream::downstream_eof(DownstreamConnection *dconn) { downstream->pop_downstream_connection(); // dconn was deleted dconn = nullptr; - // downstream wil be deleted in on_stream_close_callback. + // downstream will be deleted in on_stream_close_callback. if (downstream->get_response_state() == DownstreamState::HEADER_COMPLETE) { // Server may indicate the end of the request by EOF if (LOG_ENABLED(INFO)) { @@ -2183,7 +2183,7 @@ int Http2Upstream::submit_push_promise(const StringRef &scheme, // 4 for :method, :scheme, :path and :authority nva.reserve(4 + req.fs.headers().size()); - // juse use "GET" for now + // just use "GET" for now nva.push_back(http2::make_nv_ll(":method", "GET")); nva.push_back(http2::make_nv_ls_nocopy(":scheme", scheme)); nva.push_back(http2::make_nv_ls_nocopy(":path", path)); diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index c6e4d1ca..1c5d18d2 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -27,6 +27,7 @@ #include #include #include +#include #include @@ -39,6 +40,7 @@ #include "shrpx_quic.h" #include "shrpx_worker.h" #include "shrpx_http.h" +#include "shrpx_connection_handler.h" #ifdef HAVE_MRUBY # include "shrpx_mruby.h" #endif // HAVE_MRUBY @@ -116,13 +118,16 @@ size_t downstream_queue_size(Worker *worker) { Http3Upstream::Http3Upstream(ClientHandler *handler) : handler_{handler}, + max_udp_payload_size_{SHRPX_QUIC_MAX_UDP_PAYLOAD_SIZE}, qlog_fd_{-1}, + hashed_scid_{}, conn_{nullptr}, tls_alert_{0}, httpconn_{nullptr}, downstream_queue_{downstream_queue_size(handler->get_worker()), !get_config()->http2_proxy}, - idle_close_{false} { + idle_close_{false}, + retry_close_{false} { ev_timer_init(&timer_, timeoutcb, 0., 0.); timer_.data = this; @@ -151,9 +156,7 @@ Http3Upstream::~Http3Upstream() { nghttp3_conn_del(httpconn_); - if (conn_) { - ngtcp2_conn_del(conn_); - } + ngtcp2_conn_del(conn_); if (qlog_fd_ != -1) { close(qlog_fd_); @@ -192,9 +195,7 @@ void qlog_write(void *user_data, uint32_t flags, const void *data, void Http3Upstream::qlog_write(const void *data, size_t datalen, bool fin) { assert(qlog_fd_ != -1); - ssize_t nwrite; - - while ((nwrite = write(qlog_fd_, data, datalen)) == -1 && errno == EINTR) + while (write(qlog_fd_, data, datalen) == -1 && errno == EINTR) ; if (fin) { @@ -216,22 +217,23 @@ int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token, auto upstream = static_cast(user_data); auto handler = upstream->get_client_handler(); auto worker = handler->get_worker(); + auto conn_handler = worker->get_connection_handler(); + auto &qkms = conn_handler->get_quic_keying_materials(); + auto &qkm = qkms->keying_materials.front(); - if (generate_quic_connection_id(cid, cidlen, worker->get_cid_prefix()) != 0) { + if (generate_quic_connection_id(*cid, cidlen, worker->get_cid_prefix(), + qkm.id, qkm.cid_encryption_key.data()) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } - auto &quic_secret = worker->get_quic_secret(); - auto &secret = quic_secret->stateless_reset_secret; - - if (generate_quic_stateless_reset_token(token, cid, secret.data(), - secret.size()) != 0) { + if (generate_quic_stateless_reset_token(token, *cid, qkm.secret.data(), + qkm.secret.size()) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } auto quic_connection_handler = worker->get_quic_connection_handler(); - quic_connection_handler->add_connection_id(cid, handler); + quic_connection_handler->add_connection_id(*cid, handler); return 0; } @@ -245,7 +247,7 @@ int remove_connection_id(ngtcp2_conn *conn, const ngtcp2_cid *cid, auto worker = handler->get_worker(); auto quic_conn_handler = worker->get_quic_connection_handler(); - quic_conn_handler->remove_connection_id(cid); + quic_conn_handler->remove_connection_id(*cid); return 0; } @@ -476,16 +478,26 @@ int handshake_completed(ngtcp2_conn *conn, void *user_data) { } // namespace int Http3Upstream::handshake_completed() { + handler_->set_alpn_from_conn(); + + auto alpn = handler_->get_alpn(); + if (alpn.empty()) { + ULOG(ERROR, this) << "NO ALPN was negotiated"; + return -1; + } + std::array token; size_t tokenlen; auto path = ngtcp2_conn_get_path(conn_); auto worker = handler_->get_worker(); - auto &quic_secret = worker->get_quic_secret(); - auto &secret = quic_secret->token_secret; + auto conn_handler = worker->get_connection_handler(); + auto &qkms = conn_handler->get_quic_keying_materials(); + auto &qkm = qkms->keying_materials.front(); if (generate_token(token.data(), tokenlen, path->remote.addr, - path->remote.addrlen, secret.data()) != 0) { + path->remote.addrlen, qkm.secret.data(), + qkm.secret.size()) != 0) { return 0; } @@ -507,6 +519,7 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, int rv; auto worker = handler_->get_worker(); + auto conn_handler = worker->get_connection_handler(); auto callbacks = ngtcp2_callbacks{ nullptr, // client_initial @@ -547,17 +560,21 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, shrpx::stream_stop_sending, }; - ngtcp2_cid scid; - - if (generate_quic_connection_id(&scid, SHRPX_QUIC_SCIDLEN, - worker->get_cid_prefix()) != 0) { - return -1; - } - auto config = get_config(); auto &quicconf = config->quic; auto &http3conf = config->http3; + auto &qkms = conn_handler->get_quic_keying_materials(); + auto &qkm = qkms->keying_materials.front(); + + ngtcp2_cid scid; + + if (generate_quic_connection_id(scid, SHRPX_QUIC_SCIDLEN, + worker->get_cid_prefix(), qkm.id, + qkm.cid_encryption_key.data()) != 0) { + return -1; + } + ngtcp2_settings settings; ngtcp2_settings_default(&settings); if (quicconf.upstream.debug.log) { @@ -574,7 +591,9 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, } settings.initial_ts = quic_timestamp(); - settings.cc_algo = NGTCP2_CC_ALGO_BBR; + settings.initial_rtt = static_cast( + quicconf.upstream.initial_rtt * NGTCP2_SECONDS); + settings.cc_algo = quicconf.upstream.congestion_controller; settings.max_window = http3conf.upstream.max_connection_window_size; settings.max_stream_window = http3conf.upstream.max_window_size; settings.max_udp_payload_size = SHRPX_QUIC_MAX_UDP_PAYLOAD_SIZE; @@ -593,6 +612,40 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, params.max_idle_timeout = static_cast( quicconf.upstream.timeout.idle * NGTCP2_SECONDS); +#ifdef OPENSSL_IS_BORINGSSL + if (quicconf.upstream.early_data) { + ngtcp2_transport_params early_data_params{ + .initial_max_stream_data_bidi_local = + params.initial_max_stream_data_bidi_local, + .initial_max_stream_data_bidi_remote = + params.initial_max_stream_data_bidi_remote, + .initial_max_stream_data_uni = params.initial_max_stream_data_uni, + .initial_max_data = params.initial_max_data, + .initial_max_streams_bidi = params.initial_max_streams_bidi, + .initial_max_streams_uni = params.initial_max_streams_uni, + }; + + // TODO include HTTP/3 SETTINGS + + std::array quic_early_data_ctx; + + auto quic_early_data_ctxlen = ngtcp2_encode_transport_params( + quic_early_data_ctx.data(), quic_early_data_ctx.size(), + NGTCP2_TRANSPORT_PARAMS_TYPE_ENCRYPTED_EXTENSIONS, &early_data_params); + + assert(quic_early_data_ctxlen > 0); + assert(static_cast(quic_early_data_ctxlen) <= + quic_early_data_ctx.size()); + + if (SSL_set_quic_early_data_context(handler_->get_ssl(), + quic_early_data_ctx.data(), + quic_early_data_ctxlen) != 1) { + ULOG(ERROR, this) << "SSL_set_quic_early_data_context failed"; + return -1; + } + } +#endif // OPENSSL_IS_BORINGSSL + if (odcid) { params.original_dcid = *odcid; params.retry_scid = initial_hd.dcid; @@ -601,12 +654,8 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, params.original_dcid = initial_hd.dcid; } - auto &quic_secret = worker->get_quic_secret(); - auto &stateless_reset_secret = quic_secret->stateless_reset_secret; - - rv = generate_quic_stateless_reset_token(params.stateless_reset_token, &scid, - stateless_reset_secret.data(), - stateless_reset_secret.size()); + rv = generate_quic_stateless_reset_token( + params.stateless_reset_token, scid, qkm.secret.data(), qkm.secret.size()); if (rv != 0) { ULOG(ERROR, this) << "generate_quic_stateless_reset_token failed"; return -1; @@ -614,8 +663,14 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, params.stateless_reset_token_present = 1; auto path = ngtcp2_path{ - {local_addr.len, const_cast(&local_addr.su.sa)}, - {remote_addr.len, const_cast(&remote_addr.su.sa)}, + { + const_cast(&local_addr.su.sa), + local_addr.len, + }, + { + const_cast(&remote_addr.su.sa), + remote_addr.len, + }, const_cast(faddr), }; @@ -631,8 +686,13 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, auto quic_connection_handler = worker->get_quic_connection_handler(); - quic_connection_handler->add_connection_id(&initial_hd.dcid, handler_); - quic_connection_handler->add_connection_id(&scid, handler_); + if (generate_quic_hashed_connection_id(hashed_scid_, remote_addr, local_addr, + initial_hd.dcid) != 0) { + return -1; + } + + quic_connection_handler->add_connection_id(hashed_scid_, handler_); + quic_connection_handler->add_connection_id(scid, handler_); return 0; } @@ -652,11 +712,12 @@ int Http3Upstream::on_write() { int Http3Upstream::write_streams() { std::array vec; std::array buf; - auto max_udp_payload_size = ngtcp2_conn_get_path_max_udp_payload_size(conn_); + auto max_udp_payload_size = std::min( + max_udp_payload_size_, ngtcp2_conn_get_path_max_udp_payload_size(conn_)); size_t max_pktcnt = std::min(static_cast(64_k), ngtcp2_conn_get_send_quantum(conn_)) / max_udp_payload_size; - ngtcp2_pkt_info pi; + ngtcp2_pkt_info pi, prev_pi; uint8_t *bufpos = buf.data(); ngtcp2_path_storage ps, prev_ps; size_t pktcnt = 0; @@ -666,6 +727,13 @@ int Http3Upstream::write_streams() { ngtcp2_path_storage_zero(&ps); ngtcp2_path_storage_zero(&prev_ps); + auto config = get_config(); + auto &quicconf = config->quic; + + if (quicconf.upstream.congestion_controller != NGTCP2_CC_ALGO_BBR) { + max_pktcnt = std::min(max_pktcnt, static_cast(10)); + } + for (;;) { int64_t stream_id = -1; int fin = 0; @@ -750,15 +818,17 @@ int Http3Upstream::write_streams() { if (nwrite == 0) { if (bufpos - buf.data()) { - quic_send_packet(static_cast(prev_ps.path.user_data), - prev_ps.path.remote.addr, prev_ps.path.remote.addrlen, - prev_ps.path.local.addr, prev_ps.path.local.addrlen, - buf.data(), bufpos - buf.data(), max_udp_payload_size); + send_packet(static_cast(prev_ps.path.user_data), + prev_ps.path.remote.addr, prev_ps.path.remote.addrlen, + prev_ps.path.local.addr, prev_ps.path.local.addrlen, + prev_pi, buf.data(), bufpos - buf.data(), + max_udp_payload_size); - ngtcp2_conn_update_pkt_tx_time(conn_, ts); reset_idle_timer(); } + ngtcp2_conn_update_pkt_tx_time(conn_, ts); + handler_->get_connection()->wlimit.stopw(); return 0; @@ -769,17 +839,19 @@ int Http3Upstream::write_streams() { #ifdef UDP_SEGMENT if (pktcnt == 0) { ngtcp2_path_copy(&prev_ps.path, &ps.path); - } else if (!ngtcp2_path_eq(&prev_ps.path, &ps.path)) { - quic_send_packet(static_cast(prev_ps.path.user_data), - prev_ps.path.remote.addr, prev_ps.path.remote.addrlen, - prev_ps.path.local.addr, prev_ps.path.local.addrlen, - buf.data(), bufpos - buf.data() - nwrite, - max_udp_payload_size); + prev_pi = pi; + } else if (!ngtcp2_path_eq(&prev_ps.path, &ps.path) || + prev_pi.ecn != pi.ecn) { + send_packet(static_cast(prev_ps.path.user_data), + prev_ps.path.remote.addr, prev_ps.path.remote.addrlen, + prev_ps.path.local.addr, prev_ps.path.local.addrlen, prev_pi, + buf.data(), bufpos - buf.data() - nwrite, + max_udp_payload_size); - quic_send_packet(static_cast(ps.path.user_data), - ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, - bufpos - nwrite, nwrite, max_udp_payload_size); + send_packet(static_cast(ps.path.user_data), + ps.path.remote.addr, ps.path.remote.addrlen, + ps.path.local.addr, ps.path.local.addrlen, pi, + bufpos - nwrite, nwrite, max_udp_payload_size); ngtcp2_conn_update_pkt_tx_time(conn_, ts); reset_idle_timer(); @@ -791,10 +863,10 @@ int Http3Upstream::write_streams() { if (++pktcnt == max_pktcnt || static_cast(nwrite) < max_udp_payload_size) { - quic_send_packet(static_cast(ps.path.user_data), - ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, buf.data(), - bufpos - buf.data(), max_udp_payload_size); + send_packet(static_cast(ps.path.user_data), + ps.path.remote.addr, ps.path.remote.addrlen, + ps.path.local.addr, ps.path.local.addrlen, pi, buf.data(), + bufpos - buf.data(), max_udp_payload_size); ngtcp2_conn_update_pkt_tx_time(conn_, ts); reset_idle_timer(); @@ -804,10 +876,9 @@ int Http3Upstream::write_streams() { return 0; } #else // !UDP_SEGMENT - quic_send_packet(static_cast(ps.path.user_data), - ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, buf.data(), - bufpos - buf.data(), 0); + send_packet(static_cast(ps.path.user_data), + ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr, + ps.path.local.addrlen, pi, buf.data(), bufpos - buf.data(), 0); if (++pktcnt == max_pktcnt) { ngtcp2_conn_update_pkt_tx_time(conn_, ts); @@ -952,7 +1023,7 @@ int Http3Upstream::downstream_eof(DownstreamConnection *dconn) { downstream->pop_downstream_connection(); // dconn was deleted dconn = nullptr; - // downstream wil be deleted in on_stream_close_callback. + // downstream will be deleted in on_stream_close_callback. if (downstream->get_response_state() == DownstreamState::HEADER_COMPLETE) { // Server may indicate the end of the request by EOF if (LOG_ENABLED(INFO)) { @@ -964,7 +1035,9 @@ int Http3Upstream::downstream_eof(DownstreamConnection *dconn) { // downstream_read_data_callback to send RST_STREAM after pending // response body is sent. This is needed to ensure that RST_STREAM // is sent after all pending data are sent. - on_downstream_body_complete(downstream); + if (on_downstream_body_complete(downstream) != 0) { + return -1; + } } else if (downstream->get_response_state() != DownstreamState::MSG_COMPLETE) { // If stream was not closed, then we set MSG_COMPLETE and let @@ -1008,7 +1081,9 @@ int Http3Upstream::downstream_error(DownstreamConnection *dconn, int events) { } else { if (downstream->get_response_state() == DownstreamState::HEADER_COMPLETE) { if (downstream->get_upgraded()) { - on_downstream_body_complete(downstream); + if (on_downstream_body_complete(downstream) != 0) { + return -1; + } } else { shutdown_stream(downstream, NGHTTP3_H3_INTERNAL_ERROR); } @@ -1295,6 +1370,24 @@ int Http3Upstream::on_downstream_body_complete(Downstream *downstream) { return 0; } + if (!downstream->get_upgraded()) { + const auto &trailers = resp.fs.trailers(); + if (!trailers.empty()) { + std::vector nva; + nva.reserve(trailers.size()); + http3::copy_headers_to_nva_nocopy(nva, trailers, http2::HDOP_STRIP_ALL); + if (!nva.empty()) { + auto rv = nghttp3_conn_submit_trailers( + httpconn_, downstream->get_stream_id(), nva.data(), nva.size()); + if (rv != 0) { + ULOG(FATAL, this) << "nghttp3_conn_submit_trailers() failed: " + << nghttp3_strerror(rv); + return -1; + } + } + } + } + nghttp3_conn_resume_stream(httpconn_, downstream->get_stream_id()); downstream->ensure_upstream_wtimer(); @@ -1312,17 +1405,15 @@ void Http3Upstream::on_handler_delete() { auto worker = handler_->get_worker(); auto quic_conn_handler = worker->get_quic_connection_handler(); - quic_conn_handler->remove_connection_id( - ngtcp2_conn_get_client_initial_dcid(conn_)); - - std::vector scids(ngtcp2_conn_get_num_scid(conn_)); + std::vector scids(ngtcp2_conn_get_num_scid(conn_) + 1); ngtcp2_conn_get_scid(conn_, scids.data()); + scids.back() = hashed_scid_; for (auto &cid : scids) { - quic_conn_handler->remove_connection_id(&cid); + quic_conn_handler->remove_connection_id(cid); } - if (idle_close_) { + if (idle_close_ || retry_close_) { return; } @@ -1337,7 +1428,7 @@ void Http3Upstream::on_handler_delete() { auto nwrite = ngtcp2_conn_write_connection_close( conn_, &ps.path, &pi, conn_close_.data(), conn_close_.size(), - NGTCP2_NO_ERROR, quic_timestamp()); + NGTCP2_NO_ERROR, nullptr, 0, quic_timestamp()); if (nwrite < 0) { if (nwrite != NGTCP2_ERR_INVALID_STATE) { ULOG(ERROR, this) << "ngtcp2_conn_write_connection_close: " @@ -1349,10 +1440,9 @@ void Http3Upstream::on_handler_delete() { conn_close_.resize(nwrite); - quic_send_packet(static_cast(ps.path.user_data), - ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, - conn_close_.data(), nwrite, 0); + send_packet(static_cast(ps.path.user_data), + ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr, + ps.path.local.addrlen, pi, conn_close_.data(), nwrite, 0); } auto d = @@ -1571,34 +1661,63 @@ void Http3Upstream::cancel_premature_downstream( int Http3Upstream::on_read(const UpstreamAddr *faddr, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, - size_t datalen) { + const Address &local_addr, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen) { int rv; - ngtcp2_pkt_info pi{}; auto path = ngtcp2_path{ { - local_addr.len, const_cast(&local_addr.su.sa), + local_addr.len, }, { - remote_addr.len, const_cast(&remote_addr.su.sa), + remote_addr.len, }, const_cast(faddr), }; rv = ngtcp2_conn_read_pkt(conn_, &path, &pi, data, datalen, quic_timestamp()); if (rv != 0) { - ULOG(ERROR, this) << "ngtcp2_conn_read_pkt: " << ngtcp2_strerror(rv); - switch (rv) { case NGTCP2_ERR_DRAINING: - // TODO Start drain period return -1; - case NGTCP2_ERR_RETRY: - // TODO Send Retry packet + case NGTCP2_ERR_RETRY: { + auto worker = handler_->get_worker(); + auto quic_conn_handler = worker->get_quic_connection_handler(); + + uint32_t version; + const uint8_t *dcid, *scid; + size_t dcidlen, scidlen; + + rv = ngtcp2_pkt_decode_version_cid(&version, &dcid, &dcidlen, &scid, + &scidlen, data, datalen, + SHRPX_QUIC_SCIDLEN); + if (rv != 0) { + return -1; + } + + if (worker->get_graceful_shutdown()) { + ngtcp2_cid ini_dcid, ini_scid; + + ngtcp2_cid_init(&ini_dcid, dcid, dcidlen); + ngtcp2_cid_init(&ini_scid, scid, scidlen); + + quic_conn_handler->send_connection_close( + faddr, version, ini_dcid, ini_scid, remote_addr, local_addr, + NGTCP2_CONNECTION_REFUSED); + + return -1; + } + + retry_close_ = true; + + quic_conn_handler->send_retry(handler_->get_upstream_addr(), version, + dcid, dcidlen, scid, scidlen, remote_addr, + local_addr); + return -1; + } case NGTCP2_ERR_REQUIRED_TRANSPORT_PARAM: case NGTCP2_ERR_MALFORMED_TRANSPORT_PARAM: case NGTCP2_ERR_TRANSPORT_PARAM: @@ -1615,7 +1734,8 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr, } } - // TODO Send connection close + ULOG(ERROR, this) << "ngtcp2_conn_read_pkt: " << ngtcp2_strerror(rv); + return handle_error(); } @@ -1624,6 +1744,29 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr, return 0; } +int Http3Upstream::send_packet(const UpstreamAddr *faddr, + const sockaddr *remote_sa, size_t remote_salen, + const sockaddr *local_sa, size_t local_salen, + const ngtcp2_pkt_info &pi, const uint8_t *data, + size_t datalen, size_t gso_size) { + auto rv = quic_send_packet(faddr, remote_sa, remote_salen, local_sa, + local_salen, pi, data, datalen, gso_size); + switch (rv) { + case 0: + return 0; + // With GSO, sendmsg may fail with EINVAL if UDP payload is too + // large. + case -EINVAL: + case -EMSGSIZE: + max_udp_payload_size_ = NGTCP2_MAX_UDP_PAYLOAD_SIZE; + break; + default: + break; + } + + return -1; +} + int Http3Upstream::handle_error() { if (ngtcp2_conn_is_in_closing_period(conn_)) { return -1; @@ -1643,7 +1786,7 @@ int Http3Upstream::handle_error() { if (last_error_.type == quic::ErrorType::Transport) { nwrite = ngtcp2_conn_write_connection_close( conn_, &ps.path, &pi, conn_close_.data(), conn_close_.size(), - last_error_.code, ts); + last_error_.code, nullptr, 0, ts); if (nwrite < 0) { ULOG(ERROR, this) << "ngtcp2_conn_write_connection_close: " << ngtcp2_strerror(nwrite); @@ -1652,7 +1795,7 @@ int Http3Upstream::handle_error() { } else { nwrite = ngtcp2_conn_write_application_close( conn_, &ps.path, &pi, conn_close_.data(), conn_close_.size(), - last_error_.code, ts); + last_error_.code, nullptr, 0, ts); if (nwrite < 0) { ULOG(ERROR, this) << "ngtcp2_conn_write_application_close: " << ngtcp2_strerror(nwrite); @@ -1662,10 +1805,9 @@ int Http3Upstream::handle_error() { conn_close_.resize(nwrite); - quic_send_packet(static_cast(ps.path.user_data), - ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, - conn_close_.data(), nwrite, 0); + send_packet(static_cast(ps.path.user_data), + ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr, + ps.path.local.addrlen, pi, conn_close_.data(), nwrite, 0); return -1; } @@ -1762,7 +1904,7 @@ int http_deferred_consume(nghttp3_conn *conn, int64_t stream_id, namespace { int http_acked_stream_data(nghttp3_conn *conn, int64_t stream_id, - size_t datalen, void *user_data, + uint64_t datalen, void *user_data, void *stream_user_data) { auto upstream = static_cast(user_data); auto downstream = static_cast(stream_user_data); @@ -1778,7 +1920,7 @@ int http_acked_stream_data(nghttp3_conn *conn, int64_t stream_id, } // namespace int Http3Upstream::http_acked_stream_data(Downstream *downstream, - size_t datalen) { + uint64_t datalen) { if (LOG_ENABLED(INFO)) { ULOG(INFO, this) << "Stream " << downstream->get_stream_id() << " " << datalen << " bytes acknowledged"; @@ -1786,6 +1928,7 @@ int Http3Upstream::http_acked_stream_data(Downstream *downstream, auto body = downstream->get_response_buf(); auto drained = body->drain_mark(datalen); + (void)drained; assert(datalen == drained); @@ -1822,8 +1965,29 @@ int http_recv_request_header(nghttp3_conn *conn, int64_t stream_id, return 0; } - if (upstream->http_recv_request_header(downstream, token, name, value, - flags) != 0) { + if (upstream->http_recv_request_header(downstream, token, name, value, flags, + /* trailer = */ false) != 0) { + return NGHTTP3_ERR_CALLBACK_FAILURE; + } + + return 0; +} +} // namespace + +namespace { +int http_recv_request_trailer(nghttp3_conn *conn, int64_t stream_id, + int32_t token, nghttp3_rcbuf *name, + nghttp3_rcbuf *value, uint8_t flags, + void *user_data, void *stream_user_data) { + auto upstream = static_cast(user_data); + auto downstream = static_cast(stream_user_data); + + if (!downstream || downstream->get_stop_reading()) { + return 0; + } + + if (upstream->http_recv_request_header(downstream, token, name, value, flags, + /* trailer = */ true) != 0) { return NGHTTP3_ERR_CALLBACK_FAILURE; } @@ -1834,8 +1998,8 @@ int http_recv_request_header(nghttp3_conn *conn, int64_t stream_id, int Http3Upstream::http_recv_request_header(Downstream *downstream, int32_t h3token, nghttp3_rcbuf *name, - nghttp3_rcbuf *value, - uint8_t flags) { + nghttp3_rcbuf *value, uint8_t flags, + bool trailer) { auto namebuf = nghttp3_rcbuf_get_buf(name); auto valuebuf = nghttp3_rcbuf_get_buf(value); auto &req = downstream->request(); @@ -1857,8 +2021,13 @@ int Http3Upstream::http_recv_request_header(Downstream *downstream, << ", num=" << req.fs.num_fields() + 1; } + // just ignore if this is a trailer part. + if (trailer) { + return 0; + } + if (error_reply(downstream, 431) != 0) { - return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + return -1; } return 0; @@ -1870,6 +2039,13 @@ int Http3Upstream::http_recv_request_header(Downstream *downstream, downstream->add_rcbuf(name); downstream->add_rcbuf(value); + if (trailer) { + req.fs.add_trailer_token(StringRef{namebuf.base, namebuf.len}, + StringRef{valuebuf.base, valuebuf.len}, no_index, + token); + return 0; + } + req.fs.add_header_token(StringRef{namebuf.base, namebuf.len}, StringRef{valuebuf.base, valuebuf.len}, no_index, token); @@ -1877,7 +2053,7 @@ int Http3Upstream::http_recv_request_header(Downstream *downstream, } namespace { -int http_end_request_headers(nghttp3_conn *conn, int64_t stream_id, +int http_end_request_headers(nghttp3_conn *conn, int64_t stream_id, int fin, void *user_data, void *stream_user_data) { auto upstream = static_cast(user_data); auto handler = upstream->get_client_handler(); @@ -1887,7 +2063,7 @@ int http_end_request_headers(nghttp3_conn *conn, int64_t stream_id, return 0; } - if (upstream->http_end_request_headers(downstream) != 0) { + if (upstream->http_end_request_headers(downstream, fin) != 0) { return NGHTTP3_ERR_CALLBACK_FAILURE; } @@ -1898,7 +2074,7 @@ int http_end_request_headers(nghttp3_conn *conn, int64_t stream_id, } } // namespace -int Http3Upstream::http_end_request_headers(Downstream *downstream) { +int Http3Upstream::http_end_request_headers(Downstream *downstream, int fin) { auto lgconf = log_config(); lgconf->update_tstamp(std::chrono::system_clock::now()); auto &req = downstream->request(); @@ -1939,7 +2115,7 @@ int Http3Upstream::http_end_request_headers(Downstream *downstream) { auto method_token = http2::lookup_method_token(method->value); if (method_token == -1) { if (error_reply(downstream, 501) != 0) { - return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + return -1; } return 0; } @@ -1951,7 +2127,7 @@ int Http3Upstream::http_end_request_headers(Downstream *downstream) { // For HTTP/2 proxy, we require :authority. if (method_token != HTTP_CONNECT && config->http2_proxy && faddr->alt_mode == UpstreamAltMode::NONE && !authority) { - shutdown_stream(downstream, NGHTTP2_PROTOCOL_ERROR); + shutdown_stream(downstream, NGHTTP3_H3_GENERAL_PROTOCOL_ERROR); return 0; } @@ -1987,15 +2163,18 @@ int Http3Upstream::http_end_request_headers(Downstream *downstream) { if (connect_proto) { if (connect_proto->value != "websocket") { if (error_reply(downstream, 400) != 0) { - return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + return -1; } return 0; } req.connect_proto = ConnectProto::WEBSOCKET; } - // We are not sure that request has body or not at the moment. - req.http2_expect_body = true; + if (!fin) { + req.http2_expect_body = true; + } else if (req.fs.content_length == -1) { + req.fs.content_length = 0; + } downstream->inspect_http2_request(); @@ -2009,7 +2188,7 @@ int Http3Upstream::http_end_request_headers(Downstream *downstream) { if (mruby_ctx->run_on_request_proc(downstream) != 0) { if (error_reply(downstream, 500) != 0) { - return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + return -1; } return 0; } @@ -2162,7 +2341,7 @@ int Http3Upstream::http_end_stream(Downstream *downstream) { if (downstream->end_upload_data() != 0) { if (downstream->get_response_state() != DownstreamState::MSG_COMPLETE) { - shutdown_stream(downstream, NGHTTP2_INTERNAL_ERROR); + shutdown_stream(downstream, NGHTTP3_H3_INTERNAL_ERROR); } } @@ -2238,12 +2417,12 @@ int Http3Upstream::http_stream_close(Downstream *downstream, } namespace { -int http_send_stop_sending(nghttp3_conn *conn, int64_t stream_id, - uint64_t app_error_code, void *user_data, - void *stream_user_data) { +int http_stop_sending(nghttp3_conn *conn, int64_t stream_id, + uint64_t app_error_code, void *user_data, + void *stream_user_data) { auto upstream = static_cast(user_data); - if (upstream->http_send_stop_sending(stream_id, app_error_code) != 0) { + if (upstream->http_stop_sending(stream_id, app_error_code) != 0) { return NGHTTP3_ERR_CALLBACK_FAILURE; } @@ -2251,8 +2430,8 @@ int http_send_stop_sending(nghttp3_conn *conn, int64_t stream_id, } } // namespace -int Http3Upstream::http_send_stop_sending(int64_t stream_id, - uint64_t app_error_code) { +int Http3Upstream::http_stop_sending(int64_t stream_id, + uint64_t app_error_code) { auto rv = ngtcp2_conn_shutdown_stream_read(conn_, stream_id, app_error_code); if (ngtcp2_err_is_fatal(rv)) { ULOG(ERROR, this) << "ngtcp2_conn_shutdown_stream_read: " @@ -2305,9 +2484,9 @@ int Http3Upstream::setup_httpconn() { shrpx::http_recv_request_header, shrpx::http_end_request_headers, nullptr, // begin_trailers - nullptr, // recv_trailer + shrpx::http_recv_request_trailer, nullptr, // end_trailers - shrpx::http_send_stop_sending, + shrpx::http_stop_sending, shrpx::http_end_stream, shrpx::http_reset_stream, }; @@ -2316,7 +2495,7 @@ int Http3Upstream::setup_httpconn() { nghttp3_settings settings; nghttp3_settings_default(&settings); - settings.qpack_max_table_capacity = 4_k; + settings.qpack_max_dtable_capacity = 4_k; if (!config->http2_proxy) { settings.enable_connect_protocol = 1; @@ -2589,7 +2768,7 @@ int Http3Upstream::open_qlog_file(const StringRef &dir, util::format_iso8601_basic(buf.data(), std::chrono::system_clock::now()); path += '-'; path += util::format_hex(scid.data, scid.datalen); - path += ".qlog"; + path += ".sqlog"; int fd; diff --git a/src/shrpx_http3_upstream.h b/src/shrpx_http3_upstream.h index 845abe92..e8c8dee4 100644 --- a/src/shrpx_http3_upstream.h +++ b/src/shrpx_http3_upstream.h @@ -92,7 +92,8 @@ public: const ngtcp2_cid *odcid, const uint8_t *token, size_t tokenlen); int on_read(const UpstreamAddr *faddr, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, size_t datalen); + const Address &local_addr, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen); int write_streams(); @@ -123,8 +124,8 @@ public: void http_begin_request_headers(int64_t stream_id); int http_recv_request_header(Downstream *downstream, int32_t token, nghttp3_rcbuf *name, nghttp3_rcbuf *value, - uint8_t flags); - int http_end_request_headers(Downstream *downstream); + uint8_t flags, bool trailer); + int http_end_request_headers(Downstream *downstream, int fin); int http_end_stream(Downstream *downstream); void start_downstream(Downstream *downstream); void initiate_downstream(Downstream *downstream); @@ -137,10 +138,10 @@ public: int stream_close(int64_t stream_id, uint64_t app_error_code); void log_response_headers(Downstream *downstream, const std::vector &nva) const; - int http_acked_stream_data(Downstream *downstream, size_t datalen); + int http_acked_stream_data(Downstream *downstream, uint64_t datalen); int http_shutdown_stream_read(int64_t stream_id); int http_reset_stream(int64_t stream_id, uint64_t app_error_code); - int http_send_stop_sending(int64_t stream_id, uint64_t app_error_code); + int http_stop_sending(int64_t stream_id, uint64_t app_error_code); int http_recv_data(Downstream *downstream, const uint8_t *data, size_t datalen); int handshake_completed(); @@ -148,6 +149,10 @@ public: int start_graceful_shutdown(); int submit_goaway(); void idle_close(); + int send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, + size_t remote_salen, const sockaddr *local_sa, + size_t local_salen, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen, size_t gso_size); void qlog_write(const void *data, size_t datalen, bool fin); int open_qlog_file(const StringRef &dir, const ngtcp2_cid &scid) const; @@ -158,13 +163,16 @@ private: ev_timer idle_timer_; ev_timer shutdown_timer_; ev_prepare prep_; + size_t max_udp_payload_size_; int qlog_fd_; + ngtcp2_cid hashed_scid_; ngtcp2_conn *conn_; quic::Error last_error_; uint8_t tls_alert_; nghttp3_conn *httpconn_; DownstreamQueue downstream_queue_; bool idle_close_; + bool retry_close_; std::vector conn_close_; }; diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 6826eb5c..8c4fe6e1 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -1218,6 +1218,18 @@ int HttpDownstreamConnection::read_clear() { } if (nread < 0) { + if (nread == SHRPX_ERR_EOF && !downstream_->get_upgraded()) { + auto htperr = llhttp_finish(&response_htp_); + if (htperr != HPE_OK) { + if (LOG_ENABLED(INFO)) { + DCLOG(INFO, this) << "HTTP response ended prematurely: " + << llhttp_errno_name(htperr); + } + + return -1; + } + } + return nread; } @@ -1337,6 +1349,18 @@ int HttpDownstreamConnection::read_tls() { } if (nread < 0) { + if (nread == SHRPX_ERR_EOF && !downstream_->get_upgraded()) { + auto htperr = llhttp_finish(&response_htp_); + if (htperr != HPE_OK) { + if (LOG_ENABLED(INFO)) { + DCLOG(INFO, this) << "HTTP response ended prematurely: " + << llhttp_errno_name(htperr); + } + + return -1; + } + } + return nread; } diff --git a/src/shrpx_log.cc b/src/shrpx_log.cc index 8fd8b11c..fe626d6b 100644 --- a/src/shrpx_log.cc +++ b/src/shrpx_log.cc @@ -755,7 +755,11 @@ void upstream_accesslog(const std::vector &lfv, std::tie(p, last) = copy('-', p, last); break; } +#if OPENSSL_3_0_0_API + auto x = SSL_get0_peer_certificate(lgsp.ssl); +#else // !OPENSSL_3_0_0_API auto x = SSL_get_peer_certificate(lgsp.ssl); +#endif // !OPENSSL_3_0_0_API if (!x) { std::tie(p, last) = copy('-', p, last); break; @@ -766,7 +770,9 @@ void upstream_accesslog(const std::vector &lfv, lf.type == LogFragmentType::TLS_CLIENT_FINGERPRINT_SHA256 ? EVP_sha256() : EVP_sha1()); +#if !OPENSSL_3_0_0_API X509_free(x); +#endif // !OPENSSL_3_0_0_API if (len <= 0) { std::tie(p, last) = copy('-', p, last); break; @@ -780,7 +786,11 @@ void upstream_accesslog(const std::vector &lfv, std::tie(p, last) = copy('-', p, last); break; } +#if OPENSSL_3_0_0_API + auto x = SSL_get0_peer_certificate(lgsp.ssl); +#else // !OPENSSL_3_0_0_API auto x = SSL_get_peer_certificate(lgsp.ssl); +#endif // !OPENSSL_3_0_0_API if (!x) { std::tie(p, last) = copy('-', p, last); break; @@ -788,7 +798,9 @@ void upstream_accesslog(const std::vector &lfv, auto name = lf.type == LogFragmentType::TLS_CLIENT_ISSUER_NAME ? tls::get_x509_issuer_name(balloc, x) : tls::get_x509_subject_name(balloc, x); +#if !OPENSSL_3_0_0_API X509_free(x); +#endif // !OPENSSL_3_0_0_API if (name.empty()) { std::tie(p, last) = copy('-', p, last); break; @@ -801,13 +813,19 @@ void upstream_accesslog(const std::vector &lfv, std::tie(p, last) = copy('-', p, last); break; } +#if OPENSSL_3_0_0_API + auto x = SSL_get0_peer_certificate(lgsp.ssl); +#else // !OPENSSL_3_0_0_API auto x = SSL_get_peer_certificate(lgsp.ssl); +#endif // !OPENSSL_3_0_0_API if (!x) { std::tie(p, last) = copy('-', p, last); break; } auto sn = tls::get_x509_serial(balloc, x); +#if !OPENSSL_3_0_0_API X509_free(x); +#endif // !OPENSSL_3_0_0_API if (sn.empty()) { std::tie(p, last) = copy('-', p, last); break; @@ -964,7 +982,7 @@ int open_log_file(const char *path) { strcmp(path, "/proc/self/fd/2") == 0) { return STDERR_COPY; } -#if defined O_CLOEXEC +#ifdef O_CLOEXEC auto fd = open(path, O_WRONLY | O_APPEND | O_CREAT | O_CLOEXEC, S_IRUSR | S_IWUSR | S_IRGRP); diff --git a/src/shrpx_mruby.cc b/src/shrpx_mruby.cc index 5f7c42ad..b5c6ed3c 100644 --- a/src/shrpx_mruby.cc +++ b/src/shrpx_mruby.cc @@ -75,6 +75,7 @@ int MRubyContext::run_app(Downstream *downstream, int phase) { break; default: assert(0); + abort(); } auto res = mrb_funcall(mrb_, app_, method, 1, env_); diff --git a/src/shrpx_mruby_module_env.cc b/src/shrpx_mruby_module_env.cc index b3ed365d..5ebd9c05 100644 --- a/src/shrpx_mruby_module_env.cc +++ b/src/shrpx_mruby_module_env.cc @@ -153,7 +153,11 @@ mrb_value env_get_tls_client_fingerprint_md(mrb_state *mrb, const EVP_MD *md) { return mrb_str_new_static(mrb, "", 0); } +#if OPENSSL_3_0_0_API + auto x = SSL_get0_peer_certificate(ssl); +#else // !OPENSSL_3_0_0_API auto x = SSL_get_peer_certificate(ssl); +#endif // !OPENSSL_3_0_0_API if (!x) { return mrb_str_new_static(mrb, "", 0); } @@ -161,7 +165,9 @@ mrb_value env_get_tls_client_fingerprint_md(mrb_state *mrb, const EVP_MD *md) { // Currently the largest hash value is SHA-256, which is 32 bytes. std::array buf; auto slen = tls::get_x509_fingerprint(buf.data(), buf.size(), x, md); +#if !OPENSSL_3_0_0_API X509_free(x); +#endif // !OPENSSL_3_0_0_API if (slen == -1) { mrb_raise(mrb, E_RUNTIME_ERROR, "could not compute client fingerprint"); } @@ -199,14 +205,20 @@ mrb_value env_get_tls_client_subject_name(mrb_state *mrb, mrb_value self) { return mrb_str_new_static(mrb, "", 0); } +#if OPENSSL_3_0_0_API + auto x = SSL_get0_peer_certificate(ssl); +#else // !OPENSSL_3_0_0_API auto x = SSL_get_peer_certificate(ssl); +#endif // !OPENSSL_3_0_0_API if (!x) { return mrb_str_new_static(mrb, "", 0); } auto &balloc = downstream->get_block_allocator(); auto name = tls::get_x509_subject_name(balloc, x); +#if !OPENSSL_3_0_0_API X509_free(x); +#endif // !OPENSSL_3_0_0_API return mrb_str_new(mrb, name.c_str(), name.size()); } } // namespace @@ -223,14 +235,20 @@ mrb_value env_get_tls_client_issuer_name(mrb_state *mrb, mrb_value self) { return mrb_str_new_static(mrb, "", 0); } +#if OPENSSL_3_0_0_API + auto x = SSL_get0_peer_certificate(ssl); +#else // !OPENSSL_3_0_0_API auto x = SSL_get_peer_certificate(ssl); +#endif // !OPENSSL_3_0_0_API if (!x) { return mrb_str_new_static(mrb, "", 0); } auto &balloc = downstream->get_block_allocator(); auto name = tls::get_x509_issuer_name(balloc, x); +#if !OPENSSL_3_0_0_API X509_free(x); +#endif // !OPENSSL_3_0_0_API return mrb_str_new(mrb, name.c_str(), name.size()); } } // namespace @@ -247,14 +265,20 @@ mrb_value env_get_tls_client_serial(mrb_state *mrb, mrb_value self) { return mrb_str_new_static(mrb, "", 0); } +#if OPENSSL_3_0_0_API + auto x = SSL_get0_peer_certificate(ssl); +#else // !OPENSSL_3_0_0_API auto x = SSL_get_peer_certificate(ssl); +#endif // !OPENSSL_3_0_0_API if (!x) { return mrb_str_new_static(mrb, "", 0); } auto &balloc = downstream->get_block_allocator(); auto sn = tls::get_x509_serial(balloc, x); +#if !OPENSSL_3_0_0_API X509_free(x); +#endif // !OPENSSL_3_0_0_API return mrb_str_new(mrb, sn.c_str(), sn.size()); } } // namespace @@ -271,16 +295,24 @@ mrb_value env_get_tls_client_not_before(mrb_state *mrb, mrb_value self) { return mrb_fixnum_value(0); } +#if OPENSSL_3_0_0_API + auto x = SSL_get0_peer_certificate(ssl); +#else // !OPENSSL_3_0_0_API auto x = SSL_get_peer_certificate(ssl); +#endif // !OPENSSL_3_0_0_API if (!x) { return mrb_fixnum_value(0); } time_t t; if (tls::get_x509_not_before(t, x) != 0) { - return mrb_fixnum_value(0); + t = 0; } +#if !OPENSSL_3_0_0_API + X509_free(x); +#endif // !OPENSSL_3_0_0_API + return mrb_fixnum_value(t); } } // namespace @@ -297,16 +329,24 @@ mrb_value env_get_tls_client_not_after(mrb_state *mrb, mrb_value self) { return mrb_fixnum_value(0); } +#if OPENSSL_3_0_0_API + auto x = SSL_get0_peer_certificate(ssl); +#else // !OPENSSL_3_0_0_API auto x = SSL_get_peer_certificate(ssl); +#endif // !OPENSSL_3_0_0_API if (!x) { return mrb_fixnum_value(0); } time_t t; if (tls::get_x509_not_after(t, x) != 0) { - return mrb_fixnum_value(0); + t = 0; } +#if !OPENSSL_3_0_0_API + X509_free(x); +#endif // !OPENSSL_3_0_0_API + return mrb_fixnum_value(t); } } // namespace diff --git a/src/shrpx_null_downstream_connection.cc b/src/shrpx_null_downstream_connection.cc index 3a80b304..cd81c8aa 100644 --- a/src/shrpx_null_downstream_connection.cc +++ b/src/shrpx_null_downstream_connection.cc @@ -74,7 +74,7 @@ int NullDownstreamConnection::on_read() { return 0; } int NullDownstreamConnection::on_write() { return 0; } -void NullDownstreamConnection::on_upstream_change(Upstream *uptream) {} +void NullDownstreamConnection::on_upstream_change(Upstream *upstream) {} bool NullDownstreamConnection::poolable() const { return false; } diff --git a/src/shrpx_null_downstream_connection.h b/src/shrpx_null_downstream_connection.h index 829d659e..7defcc33 100644 --- a/src/shrpx_null_downstream_connection.h +++ b/src/shrpx_null_downstream_connection.h @@ -50,7 +50,7 @@ public: virtual int on_read(); virtual int on_write(); - virtual void on_upstream_change(Upstream *uptream); + virtual void on_upstream_change(Upstream *upstream); // true if this object is poolable. virtual bool poolable() const; diff --git a/src/shrpx_quic.cc b/src/shrpx_quic.cc index 0cde3618..fcb2cf51 100644 --- a/src/shrpx_quic.cc +++ b/src/shrpx_quic.cc @@ -43,8 +43,6 @@ #include "util.h" #include "xsi_strerror.h" -using namespace nghttp2; - bool operator==(const ngtcp2_cid &lhs, const ngtcp2_cid &rhs) { return ngtcp2_cid_eq(&lhs, &rhs); } @@ -59,8 +57,8 @@ ngtcp2_tstamp quic_timestamp() { int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, size_t remote_salen, const sockaddr *local_sa, - size_t local_salen, const uint8_t *data, size_t datalen, - size_t gso_size) { + size_t local_salen, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen, size_t gso_size) { iovec msg_iov = {const_cast(data), datalen}; msghdr msg{}; msg.msg_name = const_cast(remote_sa); @@ -125,6 +123,8 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, msg.msg_controllen = controllen; + util::fd_set_send_ecn(faddr->fd, local_sa->sa_family, pi.ecn); + ssize_t nwrite; do { @@ -132,65 +132,138 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, } while (nwrite == -1 && errno == EINTR); if (nwrite == -1) { - return -1; + if (LOG_ENABLED(INFO)) { + auto error = errno; + LOG(INFO) << "sendmsg failed: errno=" << error; + } + + return -errno; } if (LOG_ENABLED(INFO)) { LOG(INFO) << "QUIC sent packet: local=" << util::to_numeric_addr(local_sa, local_salen) << " remote=" << util::to_numeric_addr(remote_sa, remote_salen) - << " " << nwrite << " bytes"; + << " ecn=" << log::hex << pi.ecn << log::dec << " " << nwrite + << " bytes"; } return 0; } -int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen) { - if (RAND_bytes(cid->data, cidlen) != 1) { +int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, + const uint8_t *server_id, uint8_t km_id, + const uint8_t *key) { + assert(cidlen == SHRPX_QUIC_SCIDLEN); + + if (RAND_bytes(cid.data, cidlen) != 1) { return -1; } - cid->datalen = cidlen; + cid.datalen = cidlen; - return 0; + cid.data[0] = (cid.data[0] & 0x3f) | km_id; + + auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET; + + std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, p); + + return encrypt_quic_connection_id(p, p, key); } -int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen, - const uint8_t *cid_prefix) { - assert(cidlen > SHRPX_QUIC_CID_PREFIXLEN); +int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, + const uint8_t *cid_prefix, uint8_t km_id, + const uint8_t *key) { + assert(cidlen == SHRPX_QUIC_SCIDLEN); - auto p = std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, cid->data); - - if (RAND_bytes(p, cidlen - SHRPX_QUIC_CID_PREFIXLEN) != 1) { + if (RAND_bytes(cid.data, cidlen) != 1) { return -1; } - cid->datalen = cidlen; + cid.datalen = cidlen; + + cid.data[0] = (cid.data[0] & 0x3f) | km_id; + + auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET; + + std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, p); + + return encrypt_quic_connection_id(p, p, key); +} + +int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, + const uint8_t *key) { + auto ctx = EVP_CIPHER_CTX_new(); + auto d = defer(EVP_CIPHER_CTX_free, ctx); + + if (!EVP_EncryptInit_ex(ctx, EVP_aes_128_ecb(), nullptr, key, nullptr)) { + return -1; + } + + EVP_CIPHER_CTX_set_padding(ctx, 0); + + int len; + + if (!EVP_EncryptUpdate(ctx, dest, &len, src, SHRPX_QUIC_DECRYPTED_DCIDLEN) || + !EVP_EncryptFinal_ex(ctx, dest + len, &len)) { + return -1; + } return 0; } -int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid, +int decrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, + const uint8_t *key) { + auto ctx = EVP_CIPHER_CTX_new(); + auto d = defer(EVP_CIPHER_CTX_free, ctx); + + if (!EVP_DecryptInit_ex(ctx, EVP_aes_128_ecb(), nullptr, key, nullptr)) { + return -1; + } + + EVP_CIPHER_CTX_set_padding(ctx, 0); + + int len; + + if (!EVP_DecryptUpdate(ctx, dest, &len, src, SHRPX_QUIC_DECRYPTED_DCIDLEN) || + !EVP_DecryptFinal_ex(ctx, dest + len, &len)) { + return -1; + } + + return 0; +} + +int generate_quic_hashed_connection_id(ngtcp2_cid &dest, + const Address &remote_addr, + const Address &local_addr, + const ngtcp2_cid &cid) { + auto ctx = EVP_MD_CTX_new(); + auto d = defer(EVP_MD_CTX_free, ctx); + + std::array h; + unsigned int hlen = EVP_MD_size(EVP_sha256()); + + if (!EVP_DigestInit_ex(ctx, EVP_sha256(), nullptr) || + !EVP_DigestUpdate(ctx, &remote_addr.su.sa, remote_addr.len) || + !EVP_DigestUpdate(ctx, &local_addr.su.sa, local_addr.len) || + !EVP_DigestUpdate(ctx, cid.data, cid.datalen) || + !EVP_DigestFinal_ex(ctx, h.data(), &hlen)) { + return -1; + } + + assert(hlen == h.size()); + + std::copy_n(std::begin(h), sizeof(dest.data), std::begin(dest.data)); + dest.datalen = sizeof(dest.data); + + return 0; +} + +int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid &cid, const uint8_t *secret, size_t secretlen) { if (ngtcp2_crypto_generate_stateless_reset_token(token, secret, secretlen, - cid) != 0) { - return -1; - } - - return 0; -} - -int generate_quic_stateless_reset_secret(uint8_t *secret) { - if (RAND_bytes(secret, SHRPX_QUIC_STATELESS_RESET_SECRETLEN) != 1) { - return -1; - } - - return 0; -} - -int generate_quic_token_secret(uint8_t *secret) { - if (RAND_bytes(secret, SHRPX_QUIC_TOKEN_SECRETLEN) != 1) { + &cid) != 0) { return -1; } @@ -198,15 +271,15 @@ int generate_quic_token_secret(uint8_t *secret) { } int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, - socklen_t salen, const ngtcp2_cid *retry_scid, - const ngtcp2_cid *odcid, const uint8_t *token_secret) { + socklen_t salen, const ngtcp2_cid &retry_scid, + const ngtcp2_cid &odcid, const uint8_t *secret, + size_t secretlen) { auto t = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); auto stokenlen = ngtcp2_crypto_generate_retry_token( - token, token_secret, SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, retry_scid, - odcid, t); + token, secret, secretlen, sa, salen, &retry_scid, &odcid, t); if (stokenlen < 0) { return -1; } @@ -216,17 +289,18 @@ int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, return 0; } -int verify_retry_token(ngtcp2_cid *odcid, const uint8_t *token, size_t tokenlen, - const ngtcp2_cid *dcid, const sockaddr *sa, - socklen_t salen, const uint8_t *token_secret) { +int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen, + const ngtcp2_cid &dcid, const sockaddr *sa, + socklen_t salen, const uint8_t *secret, + size_t secretlen) { auto t = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); - if (ngtcp2_crypto_verify_retry_token(odcid, token, tokenlen, token_secret, - SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, - dcid, 10 * NGTCP2_SECONDS, t) != 0) { + if (ngtcp2_crypto_verify_retry_token(&odcid, token, tokenlen, secret, + secretlen, sa, salen, &dcid, + 10 * NGTCP2_SECONDS, t) != 0) { return -1; } @@ -234,13 +308,13 @@ int verify_retry_token(ngtcp2_cid *odcid, const uint8_t *token, size_t tokenlen, } int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, - size_t salen, const uint8_t *token_secret) { + size_t salen, const uint8_t *secret, size_t secretlen) { auto t = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); auto stokenlen = ngtcp2_crypto_generate_regular_token( - token, token_secret, SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, t); + token, secret, secretlen, sa, salen, t); if (stokenlen < 0) { return -1; } @@ -251,18 +325,48 @@ int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, } int verify_token(const uint8_t *token, size_t tokenlen, const sockaddr *sa, - socklen_t salen, const uint8_t *token_secret) { + socklen_t salen, const uint8_t *secret, size_t secretlen) { auto t = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); - if (ngtcp2_crypto_verify_regular_token(token, tokenlen, token_secret, - SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, - 3600 * NGTCP2_SECONDS, t) != 0) { + if (ngtcp2_crypto_verify_regular_token(token, tokenlen, secret, secretlen, sa, + salen, 3600 * NGTCP2_SECONDS, + t) != 0) { return -1; } return 0; } +int generate_quic_connection_id_encryption_key(uint8_t *key, size_t keylen, + const uint8_t *secret, + size_t secretlen, + const uint8_t *salt, + size_t saltlen) { + constexpr uint8_t info[] = "connection id encryption key"; + ngtcp2_crypto_md sha256; + ngtcp2_crypto_md_init( + &sha256, reinterpret_cast(const_cast(EVP_sha256()))); + + if (ngtcp2_crypto_hkdf(key, keylen, &sha256, secret, secretlen, salt, saltlen, + info, str_size(info)) != 0) { + return -1; + } + + return 0; +} + +const QUICKeyingMaterial * +select_quic_keying_material(const QUICKeyingMaterials &qkms, + const uint8_t *cid) { + for (auto &qkm : qkms.keying_materials) { + if (((*cid) & 0xc0) == qkm.id) { + return &qkm; + } + } + + return &qkms.keying_materials.front(); +} + } // namespace shrpx diff --git a/src/shrpx_quic.h b/src/shrpx_quic.h index 2fa12e2d..cf1e0c1b 100644 --- a/src/shrpx_quic.h +++ b/src/shrpx_quic.h @@ -33,6 +33,10 @@ #include +#include "network.h" + +using namespace nghttp2; + namespace std { template <> struct hash { std::size_t operator()(const ngtcp2_cid &cid) const noexcept { @@ -56,48 +60,78 @@ bool operator==(const ngtcp2_cid &lhs, const ngtcp2_cid &rhs); namespace shrpx { struct UpstreamAddr; +struct QUICKeyingMaterials; +struct QUICKeyingMaterial; constexpr size_t SHRPX_QUIC_SCIDLEN = 20; +constexpr size_t SHRPX_QUIC_SERVER_IDLEN = 4; +// SHRPX_QUIC_CID_PREFIXLEN includes SHRPX_QUIC_SERVER_IDLEN. constexpr size_t SHRPX_QUIC_CID_PREFIXLEN = 8; +constexpr size_t SHRPX_QUIC_CID_PREFIX_OFFSET = 1; +constexpr size_t SHRPX_QUIC_DECRYPTED_DCIDLEN = 16; +constexpr size_t SHRPX_QUIC_CID_ENCRYPTION_KEYLEN = 16; constexpr size_t SHRPX_QUIC_MAX_UDP_PAYLOAD_SIZE = 1472; -constexpr size_t SHRPX_QUIC_STATELESS_RESET_SECRETLEN = 32; -constexpr size_t SHRPX_QUIC_TOKEN_SECRETLEN = 32; constexpr size_t SHRPX_QUIC_CONN_CLOSE_PKTLEN = 256; constexpr size_t SHRPX_QUIC_STATELESS_RESET_BURST = 100; +constexpr size_t SHRPX_QUIC_SECRET_RESERVEDLEN = 4; +constexpr size_t SHRPX_QUIC_SECRETLEN = 32; +constexpr size_t SHRPX_QUIC_SALTLEN = 32; ngtcp2_tstamp quic_timestamp(); int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, size_t remote_salen, const sockaddr *local_sa, - size_t local_salen, const uint8_t *data, size_t datalen, - size_t gso_size); + size_t local_salen, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen, size_t gso_size); -int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen); +int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, + const uint8_t *server_id, uint8_t km_id, + const uint8_t *key); -int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen, - const uint8_t *cid_prefix); +int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, + const uint8_t *cid_prefix, uint8_t km_id, + const uint8_t *key); -int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid, +int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, + const uint8_t *key); + +int decrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, + const uint8_t *key); + +int generate_quic_hashed_connection_id(ngtcp2_cid &dest, + const Address &remote_addr, + const Address &local_addr, + const ngtcp2_cid &cid); + +int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid &cid, const uint8_t *secret, size_t secretlen); -int generate_quic_stateless_reset_secret(uint8_t *secret); - -int generate_quic_token_secret(uint8_t *secret); - int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, - socklen_t salen, const ngtcp2_cid *retry_scid, - const ngtcp2_cid *odcid, const uint8_t *token_secret); + socklen_t salen, const ngtcp2_cid &retry_scid, + const ngtcp2_cid &odcid, const uint8_t *secret, + size_t secretlen); -int verify_retry_token(ngtcp2_cid *odcid, const uint8_t *token, size_t tokenlen, - const ngtcp2_cid *dcid, const sockaddr *sa, - socklen_t salen, const uint8_t *token_secret); +int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen, + const ngtcp2_cid &dcid, const sockaddr *sa, + socklen_t salen, const uint8_t *secret, + size_t secretlen); int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, - size_t salen, const uint8_t *token_secret); + size_t salen, const uint8_t *secret, size_t secretlen); int verify_token(const uint8_t *token, size_t tokenlen, const sockaddr *sa, - socklen_t salen, const uint8_t *token_secret); + socklen_t salen, const uint8_t *secret, size_t secretlen); + +int generate_quic_connection_id_encryption_key(uint8_t *key, size_t keylen, + const uint8_t *secret, + size_t secretlen, + const uint8_t *salt, + size_t saltlen); + +const QUICKeyingMaterial * +select_quic_keying_material(const QUICKeyingMaterials &qkms, + const uint8_t *cid); } // namespace shrpx diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index f7341913..955534ba 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -61,6 +61,7 @@ QUICConnectionHandler::~QUICConnectionHandler() { int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, const Address &remote_addr, const Address &local_addr, + const ngtcp2_pkt_info &pi, const uint8_t *data, size_t datalen) { int rv; uint32_t version; @@ -90,32 +91,72 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, ClientHandler *handler; + auto &quicconf = config->quic; + auto it = connections_.find(dcid_key); if (it == std::end(connections_)) { - if (!std::equal(dcid, dcid + SHRPX_QUIC_CID_PREFIXLEN, - worker_->get_cid_prefix())) { - auto quic_lwp = - conn_handler->match_quic_lingering_worker_process_cid_prefix(dcid, - dcidlen); - if (quic_lwp) { - if (conn_handler->forward_quic_packet_to_lingering_worker_process( - quic_lwp, remote_addr, local_addr, data, datalen) == 0) { - return 0; - } + auto cwit = close_waits_.find(dcid_key); + if (cwit != std::end(close_waits_)) { + auto cw = (*cwit).second; - return 0; - } - } - - auto it = close_waits_.find(dcid_key); - if (it != std::end(close_waits_)) { - auto cw = (*it).second; - - cw->handle_packet(faddr, remote_addr, local_addr, data, datalen); + cw->handle_packet(faddr, remote_addr, local_addr, pi, data, datalen); return 0; } + if (data[0] & 0x80) { + if (generate_quic_hashed_connection_id(dcid_key, remote_addr, local_addr, + dcid_key) != 0) { + return 0; + } + + it = connections_.find(dcid_key); + if (it == std::end(connections_)) { + auto cwit = close_waits_.find(dcid_key); + if (cwit != std::end(close_waits_)) { + auto cw = (*cwit).second; + + cw->handle_packet(faddr, remote_addr, local_addr, pi, data, datalen); + + return 0; + } + } + } + } + + if (it == std::end(connections_)) { + std::array decrypted_dcid; + + auto &qkms = conn_handler->get_quic_keying_materials(); + const QUICKeyingMaterial *qkm = nullptr; + + if (dcidlen == SHRPX_QUIC_SCIDLEN) { + qkm = select_quic_keying_material(*qkms.get(), dcid); + + if (decrypt_quic_connection_id(decrypted_dcid.data(), + dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, + qkm->cid_encryption_key.data()) != 0) { + return 0; + } + + if (qkm != &qkms->keying_materials.front() || + !std::equal(std::begin(decrypted_dcid), + std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, + worker_->get_cid_prefix())) { + auto quic_lwp = + conn_handler->match_quic_lingering_worker_process_cid_prefix( + decrypted_dcid.data(), decrypted_dcid.size()); + if (quic_lwp) { + if (conn_handler->forward_quic_packet_to_lingering_worker_process( + quic_lwp, remote_addr, local_addr, pi, data, datalen) == 0) { + return 0; + } + + return 0; + } + } + } + // new connection auto &upstreamconf = config->conn.upstream; @@ -136,26 +177,56 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, switch (ngtcp2_accept(&hd, data, datalen)) { case 0: { - // If we get Initial and it has the CID prefix of this worker, it - // is likely that client is intentionally use the our prefix. + // If we get Initial and it has the CID prefix of this worker, + // it is likely that client is intentionally use the prefix. // Just drop it. - if (std::equal(dcid, dcid + SHRPX_QUIC_CID_PREFIXLEN, - worker_->get_cid_prefix())) { + if (dcidlen == SHRPX_QUIC_SCIDLEN) { + if (qkm != &qkms->keying_materials.front()) { + qkm = &qkms->keying_materials.front(); + + if (decrypt_quic_connection_id(decrypted_dcid.data(), + dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, + qkm->cid_encryption_key.data()) != 0) { + return 0; + } + } + + if (std::equal(std::begin(decrypted_dcid), + std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, + worker_->get_cid_prefix())) { + return 0; + } + } + + if (worker_->get_graceful_shutdown()) { + send_connection_close(faddr, version, hd.dcid, hd.scid, remote_addr, + local_addr, NGTCP2_CONNECTION_REFUSED); return 0; } if (hd.token.len == 0) { + if (quicconf.upstream.require_token) { + send_retry(faddr, version, dcid, dcidlen, scid, scidlen, remote_addr, + local_addr); + + return 0; + } + break; } - auto &quic_secret = worker_->get_quic_secret(); - auto &secret = quic_secret->token_secret; + if (dcidlen != SHRPX_QUIC_SCIDLEN) { + // Initial packets with token must have DCID chosen by server. + return 0; + } + + auto qkm = select_quic_keying_material(*qkms.get(), dcid); switch (hd.token.base[0]) { case NGTCP2_CRYPTO_TOKEN_MAGIC_RETRY: - if (verify_retry_token(&odcid, hd.token.base, hd.token.len, &hd.dcid, + if (verify_retry_token(odcid, hd.token.base, hd.token.len, hd.dcid, &remote_addr.su.sa, remote_addr.len, - secret.data()) != 0) { + qkm->secret.data(), qkm->secret.size()) != 0) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Failed to validate Retry token from remote=" << util::to_numeric_addr(&remote_addr); @@ -163,7 +234,7 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, // 2nd Retry packet is not allowed, so send CONNECTION_CLOSE // with INVALID_TOKEN. - send_connection_close(faddr, version, &hd.dcid, &hd.scid, remote_addr, + send_connection_close(faddr, version, hd.dcid, hd.scid, remote_addr, local_addr, NGTCP2_INVALID_TOKEN); return 0; } @@ -180,12 +251,20 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, break; case NGTCP2_CRYPTO_TOKEN_MAGIC_REGULAR: if (verify_token(hd.token.base, hd.token.len, &remote_addr.su.sa, - remote_addr.len, secret.data()) != 0) { + remote_addr.len, qkm->secret.data(), + qkm->secret.size()) != 0) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Failed to validate token from remote=" << util::to_numeric_addr(&remote_addr); } + if (quicconf.upstream.require_token) { + send_retry(faddr, version, dcid, dcidlen, scid, scidlen, + remote_addr, local_addr); + + return 0; + } + break; } @@ -199,12 +278,25 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, break; default: + if (quicconf.upstream.require_token) { + send_retry(faddr, version, dcid, dcidlen, scid, scidlen, remote_addr, + local_addr); + + return 0; + } + break; } break; } case NGTCP2_ERR_RETRY: + if (worker_->get_graceful_shutdown()) { + send_connection_close(faddr, version, hd.dcid, hd.scid, remote_addr, + local_addr, NGTCP2_CONNECTION_REFUSED); + return 0; + } + send_retry(faddr, version, dcid, dcidlen, scid, scidlen, remote_addr, local_addr); return 0; @@ -214,11 +306,13 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, return 0; default: if (!config->single_thread && !(data[0] & 0x80) && - dcidlen > SHRPX_QUIC_CID_PREFIXLEN && - !std::equal(dcid, dcid + SHRPX_QUIC_CID_PREFIXLEN, + dcidlen == SHRPX_QUIC_SCIDLEN && + !std::equal(std::begin(decrypted_dcid), + std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, worker_->get_cid_prefix())) { if (conn_handler->forward_quic_packet(faddr, remote_addr, local_addr, - dcid, data, datalen) == 0) { + pi, decrypted_dcid.data(), data, + datalen) == 0) { return 0; } } @@ -240,7 +334,8 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, handler = (*it).second; } - if (handler->read_quic(faddr, remote_addr, local_addr, data, datalen) != 0) { + if (handler->read_quic(faddr, remote_addr, local_addr, pi, data, datalen) != + 0) { delete handler; return 0; } @@ -276,7 +371,9 @@ ClientHandler *QUICConnectionHandler::handle_new_connection( return nullptr; } +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) assert(SSL_is_quic(ssl)); +#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) SSL_set_accept_state(ssl); @@ -284,7 +381,11 @@ ClientHandler *QUICConnectionHandler::handle_new_connection( auto &quicconf = config->quic; if (quicconf.upstream.early_data) { +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) SSL_set_quic_early_data_enabled(ssl, 1); +#else // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) + SSL_set_early_data_enabled(ssl, 1); +#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) } // Disable TLS session ticket if we don't have working ticket @@ -348,11 +449,18 @@ int QUICConnectionHandler::send_retry( return -1; } + auto config = get_config(); + auto &quicconf = config->quic; + + auto conn_handler = worker_->get_connection_handler(); + auto &qkms = conn_handler->get_quic_keying_materials(); + auto &qkm = qkms->keying_materials.front(); + ngtcp2_cid retry_scid; - retry_scid.datalen = SHRPX_QUIC_SCIDLEN; - // We do not steer packet based on Retry CID. - if (RAND_bytes(retry_scid.data, retry_scid.datalen) != 1) { + if (generate_quic_retry_connection_id(retry_scid, SHRPX_QUIC_SCIDLEN, + quicconf.server_id.data(), qkm.id, + qkm.cid_encryption_key.data()) != 0) { return -1; } @@ -363,16 +471,14 @@ int QUICConnectionHandler::send_retry( ngtcp2_cid_init(&idcid, ini_dcid, ini_dcidlen); ngtcp2_cid_init(&iscid, ini_scid, ini_scidlen); - auto &quic_secret = worker_->get_quic_secret(); - auto &secret = quic_secret->token_secret; - if (generate_retry_token(token.data(), tokenlen, &remote_addr.su.sa, - remote_addr.len, &retry_scid, &idcid, - secret.data()) != 0) { + remote_addr.len, retry_scid, idcid, + qkm.secret.data(), qkm.secret.size()) != 0) { return -1; } - std::array buf; + std::vector buf; + buf.resize(256); auto nwrite = ngtcp2_crypto_write_retry(buf.data(), buf.size(), version, &iscid, @@ -382,9 +488,33 @@ int QUICConnectionHandler::send_retry( return -1; } - return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, - &local_addr.su.sa, local_addr.len, buf.data(), nwrite, - 0); + buf.resize(nwrite); + + quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, + &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{}, + buf.data(), buf.size(), 0); + + if (generate_quic_hashed_connection_id(idcid, remote_addr, local_addr, + idcid) != 0) { + return -1; + } + + auto d = + static_cast(NGTCP2_DEFAULT_INITIAL_RTT * 3) / NGTCP2_SECONDS; + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Enter close-wait period " << d << "s with " << buf.size() + << " bytes sentinel packet"; + } + + auto cw = std::make_unique(worker_, std::vector{idcid}, + std::move(buf), d); + + add_close_wait(cw.get()); + + cw.release(); + + return 0; } int QUICConnectionHandler::send_version_negotiation( @@ -411,8 +541,8 @@ int QUICConnectionHandler::send_version_negotiation( } return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, - &local_addr.su.sa, local_addr.len, buf.data(), nwrite, - 0); + &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{}, + buf.data(), nwrite, 0); } int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr, @@ -440,11 +570,12 @@ int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr, ngtcp2_cid_init(&cid, dcid, dcidlen); - auto &quic_secret = worker_->get_quic_secret(); - auto &secret = quic_secret->stateless_reset_secret; + auto conn_handler = worker_->get_connection_handler(); + auto &qkms = conn_handler->get_quic_keying_materials(); + auto &qkm = qkms->keying_materials.front(); - rv = generate_quic_stateless_reset_token(token.data(), &cid, secret.data(), - secret.size()); + rv = generate_quic_stateless_reset_token(token.data(), cid, qkm.secret.data(), + qkm.secret.size()); if (rv != 0) { return -1; } @@ -473,18 +604,19 @@ int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr, } return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, - &local_addr.su.sa, local_addr.len, buf.data(), nwrite, - 0); + &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{}, + buf.data(), nwrite, 0); } int QUICConnectionHandler::send_connection_close( - const UpstreamAddr *faddr, uint32_t version, const ngtcp2_cid *ini_dcid, - const ngtcp2_cid *ini_scid, const Address &remote_addr, + const UpstreamAddr *faddr, uint32_t version, const ngtcp2_cid &ini_dcid, + const ngtcp2_cid &ini_scid, const Address &remote_addr, const Address &local_addr, uint64_t error_code) { std::array buf; auto nwrite = ngtcp2_crypto_write_connection_close( - buf.data(), buf.size(), version, ini_scid, ini_dcid, error_code); + buf.data(), buf.size(), version, &ini_scid, &ini_dcid, error_code, + nullptr, 0); if (nwrite < 0) { LOG(ERROR) << "ngtcp2_crypto_write_connection_close failed"; return -1; @@ -494,23 +626,22 @@ int QUICConnectionHandler::send_connection_close( LOG(INFO) << "Send Initial CONNECTION_CLOSE with error_code=" << log::hex << error_code << log::dec << " to remote=" << util::to_numeric_addr(&remote_addr) - << " dcid=" << util::format_hex(ini_scid->data, ini_scid->datalen) - << " scid=" - << util::format_hex(ini_dcid->data, ini_dcid->datalen); + << " dcid=" << util::format_hex(ini_scid.data, ini_scid.datalen) + << " scid=" << util::format_hex(ini_dcid.data, ini_dcid.datalen); } return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, - &local_addr.su.sa, local_addr.len, buf.data(), nwrite, - 0); + &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{}, + buf.data(), nwrite, 0); } -void QUICConnectionHandler::add_connection_id(const ngtcp2_cid *cid, +void QUICConnectionHandler::add_connection_id(const ngtcp2_cid &cid, ClientHandler *handler) { - connections_.emplace(*cid, handler); + connections_.emplace(cid, handler); } -void QUICConnectionHandler::remove_connection_id(const ngtcp2_cid *cid) { - connections_.erase(*cid); +void QUICConnectionHandler::remove_connection_id(const ngtcp2_cid &cid) { + connections_.erase(cid); } void QUICConnectionHandler::add_close_wait(CloseWait *cw) { @@ -548,10 +679,10 @@ static void close_wait_timeoutcb(struct ev_loop *loop, ev_timer *w, } CloseWait::CloseWait(Worker *worker, std::vector scids, - std::vector conn_close, ev_tstamp period) + std::vector pkt, ev_tstamp period) : worker{worker}, scids{std::move(scids)}, - conn_close{std::move(conn_close)}, + pkt{std::move(pkt)}, bytes_recv{0}, bytes_sent{0}, num_pkts_recv{0}, @@ -573,35 +704,36 @@ CloseWait::~CloseWait() { --worker_stat->num_close_waits; if (worker->get_graceful_shutdown() && worker_stat->num_connections == 0 && - worker_stat->num_close_waits) { + worker_stat->num_close_waits == 0) { ev_break(loop); } } int CloseWait::handle_packet(const UpstreamAddr *faddr, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, + const Address &local_addr, + const ngtcp2_pkt_info &pi, const uint8_t *data, size_t datalen) { - if (conn_close.empty()) { + if (pkt.empty()) { return 0; } ++num_pkts_recv; bytes_recv += datalen; - if (bytes_sent + conn_close.size() > 3 * bytes_recv || + if (bytes_sent + pkt.size() > 3 * bytes_recv || next_pkts_recv > num_pkts_recv) { return 0; } if (quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, - &local_addr.su.sa, local_addr.len, conn_close.data(), - conn_close.size(), 0) != 0) { + &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{}, + pkt.data(), pkt.size(), 0) != 0) { return -1; } next_pkts_recv *= 2; - bytes_sent += conn_close.size(); + bytes_sent += pkt.size(); return 0; } diff --git a/src/shrpx_quic_connection_handler.h b/src/shrpx_quic_connection_handler.h index d6d46f05..48faba77 100644 --- a/src/shrpx_quic_connection_handler.h +++ b/src/shrpx_quic_connection_handler.h @@ -51,19 +51,19 @@ class Worker; // closing period). struct CloseWait { CloseWait(Worker *worker, std::vector scids, - std::vector conn_close, ev_tstamp period); + std::vector pkt, ev_tstamp period); ~CloseWait(); int handle_packet(const UpstreamAddr *faddr, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, - size_t datalen); + const Address &local_addr, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen); Worker *worker; // Source Connection IDs of the connection. std::vector scids; - // QUIC packet containing CONNECTION_CLOSE. It is empty when a - // connection entered in draining state. - std::vector conn_close; + // QUIC packet which is sent in response to the incoming packet. It + // might be empty. + std::vector pkt; // Close-wait (draining or closing period) timer. ev_timer timer; // The number of bytes received during close-wait period. @@ -82,8 +82,8 @@ public: QUICConnectionHandler(Worker *worker); ~QUICConnectionHandler(); int handle_packet(const UpstreamAddr *faddr, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, - size_t datalen); + const Address &local_addr, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen); // Send Retry packet. |ini_dcid| is the destination Connection ID // which appeared in Client Initial packet and its length is // |dcidlen|. |ini_scid| is the source Connection ID which appeared @@ -110,8 +110,8 @@ public: // |ini_scid| is the source Connection ID which appeared in Client // Initial packet. int send_connection_close(const UpstreamAddr *faddr, uint32_t version, - const ngtcp2_cid *ini_dcid, - const ngtcp2_cid *ini_scid, + const ngtcp2_cid &ini_dcid, + const ngtcp2_cid &ini_scid, const Address &remote_addr, const Address &local_addr, uint64_t error_code); ClientHandler *handle_new_connection(const UpstreamAddr *faddr, @@ -120,8 +120,8 @@ public: const ngtcp2_pkt_hd &hd, const ngtcp2_cid *odcid, const uint8_t *token, size_t tokenlen); - void add_connection_id(const ngtcp2_cid *cid, ClientHandler *handler); - void remove_connection_id(const ngtcp2_cid *cid); + void add_connection_id(const ngtcp2_cid &cid, ClientHandler *handler); + void remove_connection_id(const ngtcp2_cid &cid); void add_close_wait(CloseWait *cw); void remove_close_wait(const CloseWait *cw); diff --git a/src/shrpx_quic_listener.cc b/src/shrpx_quic_listener.cc index b06ec557..d22045d6 100644 --- a/src/shrpx_quic_listener.cc +++ b/src/shrpx_quic_listener.cc @@ -59,7 +59,8 @@ void QUICListener::on_read() { msg.msg_iov = &msg_iov; msg.msg_iovlen = 1; - uint8_t msg_ctrl[CMSG_SPACE(sizeof(in6_pktinfo))]; + uint8_t + msg_ctrl[CMSG_SPACE(sizeof(uint8_t)) + CMSG_SPACE(sizeof(in6_pktinfo))]; msg.msg_control = msg_ctrl; auto quic_conn_handler = worker_->get_quic_connection_handler(); @@ -83,11 +84,16 @@ void QUICListener::on_read() { util::set_port(local_addr, faddr_->port); + ngtcp2_pkt_info pi{ + .ecn = util::msghdr_get_ecn(&msg, su.storage.ss_family), + }; + if (LOG_ENABLED(INFO)) { LOG(INFO) << "QUIC received packet: local=" << util::to_numeric_addr(&local_addr) << " remote=" << util::to_numeric_addr(&su.sa, msg.msg_namelen) - << " " << nread << " bytes"; + << " ecn=" << log::hex << pi.ecn << log::dec << " " << nread + << " bytes"; } if (nread == 0) { @@ -98,7 +104,7 @@ void QUICListener::on_read() { remote_addr.su = su; remote_addr.len = msg.msg_namelen; - quic_conn_handler->handle_packet(faddr_, remote_addr, local_addr, + quic_conn_handler->handle_packet(faddr_, remote_addr, local_addr, pi, buf.data(), nread); } } diff --git a/src/shrpx_tls.cc b/src/shrpx_tls.cc index cda9bbc3..7913d86a 100644 --- a/src/shrpx_tls.cc +++ b/src/shrpx_tls.cc @@ -61,8 +61,13 @@ #ifdef ENABLE_HTTP3 # include # include -# include -#endif // ENABLE_HTTP3 +# ifdef HAVE_LIBNGTCP2_CRYPTO_OPENSSL +# include +# endif // HAVE_LIBNGTCP2_CRYPTO_OPENSSL +# ifdef HAVE_LIBNGTCP2_CRYPTO_BORINGSSL +# include +# endif // HAVE_LIBNGTCP2_CRYPTO_BORINGSSL +#endif // ENABLE_HTTP3 #include "shrpx_log.h" #include "shrpx_client_handler.h" @@ -196,8 +201,9 @@ int servername_callback(SSL *ssl, int *al, void *arg) { auto hostname = StringRef{std::begin(buf), end_buf}; #ifdef ENABLE_HTTP3 - auto cert_tree = SSL_is_quic(ssl) ? worker->get_quic_cert_lookup_tree() - : worker->get_cert_lookup_tree(); + auto cert_tree = conn->proto == Proto::HTTP3 + ? worker->get_quic_cert_lookup_tree() + : worker->get_cert_lookup_tree(); #else // !ENABLE_HTTP3 auto cert_tree = worker->get_cert_lookup_tree(); #endif // !ENABLE_HTTP3 @@ -212,7 +218,7 @@ int servername_callback(SSL *ssl, int *al, void *arg) { auto conn_handler = worker->get_connection_handler(); #ifdef ENABLE_HTTP3 - const auto &ssl_ctx_list = SSL_is_quic(ssl) + const auto &ssl_ctx_list = conn->proto == Proto::HTTP3 ? conn_handler->get_quic_indexed_ssl_ctx(idx) : conn_handler->get_indexed_ssl_ctx(idx); #else // !ENABLE_HTTP3 @@ -703,29 +709,36 @@ namespace { int quic_alpn_select_proto_cb(SSL *ssl, const unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, void *arg) { - for (auto p = in, end = in + inlen; p < end;) { - auto proto_id = p + 1; - auto proto_len = *p; + constexpr StringRef alpnlist[] = { + StringRef::from_lit("h3"), + StringRef::from_lit("h3-29"), + }; - if (proto_id + proto_len <= end && - util::streq_l("h3", StringRef{proto_id, proto_len})) { + for (auto &alpn : alpnlist) { + for (auto p = in, end = in + inlen; p < end;) { + auto proto_id = p + 1; + auto proto_len = *p; - *out = reinterpret_cast(proto_id); - *outlen = proto_len; + if (alpn.size() == proto_len && + memcmp(alpn.byte(), proto_id, alpn.size()) == 0) { + *out = proto_id; + *outlen = proto_len; - return SSL_TLSEXT_ERR_OK; + return SSL_TLSEXT_ERR_OK; + } + + p += 1 + proto_len; } - - p += 1 + proto_len; } - return SSL_TLSEXT_ERR_NOACK; + return SSL_TLSEXT_ERR_ALERT_FATAL; } } // namespace # endif // OPENSSL_VERSION_NUMBER >= 0x10002000L #endif // ENABLE_HTTP3 -#if !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L +#if !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L && \ + !defined(OPENSSL_IS_BORINGSSL) # ifndef TLSEXT_TYPE_signed_certificate_timestamp # define TLSEXT_TYPE_signed_certificate_timestamp 18 @@ -815,7 +828,8 @@ int legacy_sct_parse_cb(SSL *ssl, unsigned int ext_type, } // namespace # endif // !OPENSSL_1_1_1_API -#endif // !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L +#endif // !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L && + // !defined(OPENSSL_IS_BORINGSSL) #ifndef OPENSSL_NO_PSK namespace { @@ -913,7 +927,7 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file, neverbleed_t *nb #endif // HAVE_NEVERBLEED ) { - auto ssl_ctx = SSL_CTX_new(SSLv23_server_method()); + auto ssl_ctx = SSL_CTX_new(TLS_server_method()); if (!ssl_ctx) { LOG(FATAL) << ERR_error_string(ERR_get_error(), nullptr); DIE(); @@ -925,14 +939,14 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file, SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION | SSL_OP_SINGLE_ECDH_USE | SSL_OP_SINGLE_DH_USE | SSL_OP_CIPHER_SERVER_PREFERENCE -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) // The reason for disabling built-in anti-replay in OpenSSL is // that it only works if client gets back to the same server. // The freshness check described in // https://tools.ietf.org/html/rfc8446#section-8.3 is still // performed. | SSL_OP_NO_ANTI_REPLAY -#endif // OPENSSL_1_1_1_API +#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) ; auto config = mod_config(); @@ -963,13 +977,13 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file, DIE(); } -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (SSL_CTX_set_ciphersuites(ssl_ctx, tlsconf.tls13_ciphers.c_str()) == 0) { LOG(FATAL) << "SSL_CTX_set_ciphersuites " << tlsconf.tls13_ciphers << " failed: " << ERR_error_string(ERR_get_error(), nullptr); DIE(); } -#endif // OPENSSL_1_1_1_API +#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) #ifndef OPENSSL_NO_EC # if !LIBRESSL_LEGACY_API && OPENSSL_VERSION_NUMBER >= 0x10002000L @@ -1017,8 +1031,11 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file, DIE(); } - SSL_CTX_set_tmp_dh(ssl_ctx, dh); - EVP_PKEY_free(dh); + if (SSL_CTX_set0_tmp_dh_pkey(ssl_ctx, dh) != 1) { + LOG(FATAL) << "SSL_CTX_set0_tmp_dh_pkey failed: " + << ERR_error_string(ERR_get_error(), nullptr); + DIE(); + } #else // !OPENSSL_3_0_0_API auto dh = PEM_read_bio_DHparams(bio, nullptr, nullptr, nullptr); if (dh == nullptr) { @@ -1131,6 +1148,12 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file, SSL_CTX_set_alpn_select_cb(ssl_ctx, alpn_select_proto_cb, nullptr); #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L + auto tls_ctx_data = new TLSContextData(); + tls_ctx_data->cert_file = cert_file; + tls_ctx_data->sct_data = sct_data; + + SSL_CTX_set_app_data(ssl_ctx, tls_ctx_data); + #if !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L && \ !defined(OPENSSL_IS_BORINGSSL) // SSL_extension_supported(TLSEXT_TYPE_signed_certificate_timestamp) @@ -1163,31 +1186,34 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file, } # endif // !OPENSSL_1_1_1_API } -#endif // !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L && - // !defined(OPENSSL_IS_BORINGSSL) +#elif defined(OPENSSL_IS_BORINGSSL) + if (!tls_ctx_data->sct_data.empty() && + SSL_CTX_set_signed_cert_timestamp_list( + ssl_ctx, tls_ctx_data->sct_data.data(), + tls_ctx_data->sct_data.size()) != 1) { + LOG(FATAL) << "SSL_CTX_set_signed_cert_timestamp_list failed: " + << ERR_error_string(ERR_get_error(), nullptr); + DIE(); + } +#endif // defined(OPENSSL_IS_BORINGSSL) -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (SSL_CTX_set_max_early_data(ssl_ctx, tlsconf.max_early_data) != 1) { LOG(FATAL) << "SSL_CTX_set_max_early_data failed: " << ERR_error_string(ERR_get_error(), nullptr); DIE(); } -#endif // OPENSSL_1_1_1_API +#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) #ifndef OPENSSL_NO_PSK SSL_CTX_set_psk_server_callback(ssl_ctx, psk_server_cb); #endif // !LIBRESSL_NO_PSK - auto tls_ctx_data = new TLSContextData(); - tls_ctx_data->cert_file = cert_file; - tls_ctx_data->sct_data = sct_data; - - SSL_CTX_set_app_data(ssl_ctx, tls_ctx_data); - return ssl_ctx; } #ifdef ENABLE_HTTP3 +# ifdef HAVE_LIBNGTCP2_CRYPTO_OPENSSL namespace { int quic_set_encryption_secrets(SSL *ssl, OSSL_ENCRYPTION_LEVEL ossl_level, const uint8_t *rx_secret, @@ -1251,6 +1277,84 @@ auto quic_method = SSL_QUIC_METHOD{ quic_send_alert, }; } // namespace +# endif // HAVE_LIBNGTCP2_CRYPTO_OPENSSL + +# ifdef HAVE_LIBNGTCP2_CRYPTO_BORINGSSL +namespace { +int quic_set_read_secret(SSL *ssl, ssl_encryption_level_t ssl_level, + const SSL_CIPHER *cipher, const uint8_t *secret, + size_t secretlen) { + auto conn = static_cast(SSL_get_app_data(ssl)); + auto handler = static_cast(conn->data); + auto upstream = static_cast(handler->get_upstream()); + auto level = ngtcp2_crypto_boringssl_from_ssl_encryption_level(ssl_level); + + if (upstream->on_rx_secret(level, secret, secretlen) != 0) { + return 0; + } + + return 1; +} +} // namespace + +namespace { +int quic_set_write_secret(SSL *ssl, ssl_encryption_level_t ssl_level, + const SSL_CIPHER *cipher, const uint8_t *secret, + size_t secretlen) { + auto conn = static_cast(SSL_get_app_data(ssl)); + auto handler = static_cast(conn->data); + auto upstream = static_cast(handler->get_upstream()); + auto level = ngtcp2_crypto_boringssl_from_ssl_encryption_level(ssl_level); + + if (upstream->on_tx_secret(level, secret, secretlen) != 0) { + return 0; + } + + return 1; +} +} // namespace + +namespace { +int quic_add_handshake_data(SSL *ssl, ssl_encryption_level_t ssl_level, + const uint8_t *data, size_t len) { + auto conn = static_cast(SSL_get_app_data(ssl)); + auto handler = static_cast(conn->data); + auto upstream = static_cast(handler->get_upstream()); + auto level = ngtcp2_crypto_boringssl_from_ssl_encryption_level(ssl_level); + + upstream->add_crypto_data(level, data, len); + + return 1; +} +} // namespace + +namespace { +int quic_flush_flight(SSL *ssl) { return 1; } +} // namespace + +namespace { +int quic_send_alert(SSL *ssl, ssl_encryption_level_t level, uint8_t alert) { + auto conn = static_cast(SSL_get_app_data(ssl)); + auto handler = static_cast(conn->data); + auto upstream = static_cast(handler->get_upstream()); + + if (!upstream) { + return 1; + } + + upstream->set_tls_alert(alert); + + return 1; +} +} // namespace + +namespace { +auto quic_method = SSL_QUIC_METHOD{ + quic_set_read_secret, quic_set_write_secret, quic_add_handshake_data, + quic_flush_flight, quic_send_alert, +}; +} // namespace +# endif // HAVE_LIBNGTCP2_CRYPTO_BORINGSSL SSL_CTX *create_quic_ssl_context(const char *private_key_file, const char *cert_file, @@ -1271,14 +1375,14 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file, SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION | SSL_OP_SINGLE_ECDH_USE | SSL_OP_SINGLE_DH_USE | SSL_OP_CIPHER_SERVER_PREFERENCE -# if OPENSSL_1_1_1_API +# if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) // The reason for disabling built-in anti-replay in OpenSSL is // that it only works if client gets back to the same server. // The freshness check described in // https://tools.ietf.org/html/rfc8446#section-8.3 is still // performed. | SSL_OP_NO_ANTI_REPLAY -# endif // OPENSSL_1_1_1_API +# endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) ; auto config = mod_config(); @@ -1301,13 +1405,13 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file, DIE(); } -# if OPENSSL_1_1_1_API +# if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (SSL_CTX_set_ciphersuites(ssl_ctx, tlsconf.tls13_ciphers.c_str()) == 0) { LOG(FATAL) << "SSL_CTX_set_ciphersuites " << tlsconf.tls13_ciphers << " failed: " << ERR_error_string(ERR_get_error(), nullptr); DIE(); } -# endif // OPENSSL_1_1_1_API +# endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) # ifndef OPENSSL_NO_EC # if !LIBRESSL_LEGACY_API && OPENSSL_VERSION_NUMBER >= 0x10002000L @@ -1355,8 +1459,11 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file, DIE(); } - SSL_CTX_set_tmp_dh(ssl_ctx, dh); - EVP_PKEY_free(dh); + if (SSL_CTX_set0_tmp_dh_pkey(ssl_ctx, dh) != 1) { + LOG(FATAL) << "SSL_CTX_set0_tmp_dh_pkey failed: " + << ERR_error_string(ERR_get_error(), nullptr); + DIE(); + } # else // !OPENSSL_3_0_0_API auto dh = PEM_read_bio_DHparams(bio, nullptr, nullptr, nullptr); if (dh == nullptr) { @@ -1460,6 +1567,12 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file, SSL_CTX_set_alpn_select_cb(ssl_ctx, quic_alpn_select_proto_cb, nullptr); # endif // OPENSSL_VERSION_NUMBER >= 0x10002000L + auto tls_ctx_data = new TLSContextData(); + tls_ctx_data->cert_file = cert_file; + tls_ctx_data->sct_data = sct_data; + + SSL_CTX_set_app_data(ssl_ctx, tls_ctx_data); + # if !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L && \ !defined(OPENSSL_IS_BORINGSSL) // SSL_extension_supported(TLSEXT_TYPE_signed_certificate_timestamp) @@ -1492,10 +1605,18 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file, } # endif // !OPENSSL_1_1_1_API } -# endif // !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L && - // !defined(OPENSSL_IS_BORINGSSL) +# elif defined(OPENSSL_IS_BORINGSSL) + if (!tls_ctx_data->sct_data.empty() && + SSL_CTX_set_signed_cert_timestamp_list( + ssl_ctx, tls_ctx_data->sct_data.data(), + tls_ctx_data->sct_data.size()) != 1) { + LOG(FATAL) << "SSL_CTX_set_signed_cert_timestamp_list failed: " + << ERR_error_string(ERR_get_error(), nullptr); + DIE(); + } +# endif // defined(OPENSSL_IS_BORINGSSL) -# if OPENSSL_1_1_1_API +# if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) auto &quicconf = config->quic; if (quicconf.upstream.early_data && @@ -1505,7 +1626,7 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file, << ERR_error_string(ERR_get_error(), nullptr); DIE(); } -# endif // OPENSSL_1_1_1_API +# endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) # ifndef OPENSSL_NO_PSK SSL_CTX_set_psk_server_callback(ssl_ctx, psk_server_cb); @@ -1513,12 +1634,6 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file, SSL_CTX_set_quic_method(ssl_ctx, &quic_method); - auto tls_ctx_data = new TLSContextData(); - tls_ctx_data->cert_file = cert_file; - tls_ctx_data->sct_data = sct_data; - - SSL_CTX_set_app_data(ssl_ctx, tls_ctx_data); - return ssl_ctx; } #endif // ENABLE_HTTP3 @@ -1579,7 +1694,7 @@ SSL_CTX *create_ssl_client_context( int (*next_proto_select_cb)(SSL *s, unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, void *arg)) { - auto ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + auto ssl_ctx = SSL_CTX_new(TLS_client_method()); if (!ssl_ctx) { LOG(FATAL) << ERR_error_string(ERR_get_error(), nullptr); DIE(); @@ -1610,14 +1725,14 @@ SSL_CTX *create_ssl_client_context( DIE(); } -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (SSL_CTX_set_ciphersuites(ssl_ctx, tlsconf.client.tls13_ciphers.c_str()) == 0) { LOG(FATAL) << "SSL_CTX_set_ciphersuites " << tlsconf.client.tls13_ciphers << " failed: " << ERR_error_string(ERR_get_error(), nullptr); DIE(); } -#endif // OPENSSL_1_1_1_API +#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) SSL_CTX_set_mode(ssl_ctx, SSL_MODE_RELEASE_BUFFERS); @@ -1955,14 +2070,20 @@ int verify_hostname(X509 *cert, const StringRef &hostname, } // namespace int check_cert(SSL *ssl, const Address *addr, const StringRef &host) { +#if OPENSSL_3_0_0_API + auto cert = SSL_get0_peer_certificate(ssl); +#else // !OPENSSL_3_0_0_API auto cert = SSL_get_peer_certificate(ssl); +#endif // !OPENSSL_3_0_0_API if (!cert) { // By the protocol definition, TLS server always sends certificate // if it has. If certificate cannot be retrieved, authentication // without certificate is used, such as PSK. return 0; } +#if !OPENSSL_3_0_0_API auto cert_deleter = defer(X509_free, cert); +#endif // !OPENSSL_3_0_0_API if (verify_hostname(cert, host, addr) != 0) { LOG(ERROR) << "Certificate verification failed: hostname does not match"; @@ -2619,7 +2740,7 @@ namespace { int time_t_from_asn1_time(time_t &t, const ASN1_TIME *at) { int rv; -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) struct tm tm; rv = ASN1_TIME_to_tm(at, &tm); if (rv != 1) { @@ -2627,7 +2748,7 @@ int time_t_from_asn1_time(time_t &t, const ASN1_TIME *at) { } t = nghttp2_timegm(&tm); -#else // !OPENSSL_1_1_1_API +#else // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) auto b = BIO_new(BIO_s_mem()); if (!b) { return -1; @@ -2640,7 +2761,7 @@ int time_t_from_asn1_time(time_t &t, const ASN1_TIME *at) { return -1; } -# if defined(OPENSSL_IS_BORINGSSL) +# ifdef OPENSSL_IS_BORINGSSL char *s; # else unsigned char *s; @@ -2653,7 +2774,7 @@ int time_t_from_asn1_time(time_t &t, const ASN1_TIME *at) { } t = tt; -#endif // !OPENSSL_1_1_1_API +#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) return 0; } diff --git a/src/shrpx_tls_test.cc b/src/shrpx_tls_test.cc index 0f130f60..ef80a1a7 100644 --- a/src/shrpx_tls_test.cc +++ b/src/shrpx_tls_test.cc @@ -121,7 +121,7 @@ void test_shrpx_tls_cert_lookup_tree_add_ssl_ctx(void) { static constexpr char nghttp2_certfile[] = NGHTTP2_SRC_DIR "/test.nghttp2.org.pem"; - auto nghttp2_ssl_ctx = SSL_CTX_new(SSLv23_server_method()); + auto nghttp2_ssl_ctx = SSL_CTX_new(TLS_server_method()); auto nghttp2_ssl_ctx_del = defer(SSL_CTX_free, nghttp2_ssl_ctx); auto nghttp2_tls_ctx_data = std::make_unique(); nghttp2_tls_ctx_data->cert_file = nghttp2_certfile; @@ -132,7 +132,7 @@ void test_shrpx_tls_cert_lookup_tree_add_ssl_ctx(void) { static constexpr char examples_certfile[] = NGHTTP2_SRC_DIR "/test.example.com.pem"; - auto examples_ssl_ctx = SSL_CTX_new(SSLv23_server_method()); + auto examples_ssl_ctx = SSL_CTX_new(TLS_server_method()); auto examples_ssl_ctx_del = defer(SSL_CTX_free, examples_ssl_ctx); auto examples_tls_ctx_data = std::make_unique(); examples_tls_ctx_data->cert_file = examples_certfile; diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index e128badb..04bae7ed 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -554,7 +554,7 @@ void Worker::process_events() { quic_conn_handler_.handle_packet( faddr, wev.quic_pkt->remote_addr, wev.quic_pkt->local_addr, - wev.quic_pkt->data.data(), wev.quic_pkt->data.size()); + wev.quic_pkt->pi, wev.quic_pkt->data.data(), wev.quic_pkt->data.size()); break; } @@ -833,6 +833,28 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { close(fd); continue; } + + if (setsockopt(fd, IPPROTO_IPV6, IPV6_RECVTCLASS, &val, + static_cast(sizeof(val))) == -1) { + auto error = errno; + LOG(WARN) << "Failed to set IPV6_RECVTCLASS option to listener socket: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + close(fd); + continue; + } + +# if defined(IPV6_MTU_DISCOVER) && defined(IP_PMTUDISC_DO) + int mtu_disc = IP_PMTUDISC_DO; + if (setsockopt(fd, IPPROTO_IPV6, IPV6_MTU_DISCOVER, &mtu_disc, + static_cast(sizeof(mtu_disc))) == -1) { + auto error = errno; + LOG(WARN) + << "Failed to set IPV6_MTU_DISCOVER option to listener socket: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + close(fd); + continue; + } +# endif // defined(IPV6_MTU_DISCOVER) && defined(IP_PMTUDISC_DO) } else { if (setsockopt(fd, IPPROTO_IP, IP_PKTINFO, &val, static_cast(sizeof(val))) == -1) { @@ -842,9 +864,28 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { close(fd); continue; } - } - // TODO Enable ECN + if (setsockopt(fd, IPPROTO_IP, IP_RECVTOS, &val, + static_cast(sizeof(val))) == -1) { + auto error = errno; + LOG(WARN) << "Failed to set IP_RECVTOS option to listener socket: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + close(fd); + continue; + } + +# if defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DO) + int mtu_disc = IP_PMTUDISC_DO; + if (setsockopt(fd, IPPROTO_IP, IP_MTU_DISCOVER, &mtu_disc, + static_cast(sizeof(mtu_disc))) == -1) { + auto error = errno; + LOG(WARN) << "Failed to set IP_MTU_DISCOVER option to listener socket: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + close(fd); + continue; + } +# endif // defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DO) + } if (bind(fd, rp->ai_addr, rp->ai_addrlen) == -1) { auto error = errno; @@ -890,6 +931,8 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { auto &ref = quic_bpf_refs[faddr.index]; + ref.obj = obj; + auto reuseport_array = bpf_object__find_map_by_name(obj, "reuseport_array"); err = libbpf_get_error(reuseport_array); @@ -923,7 +966,7 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { } constexpr uint32_t zero = 0; - uint32_t num_socks = config->num_worker; + uint64_t num_socks = config->num_worker; if (bpf_map_update_elem(bpf_map__fd(sk_info), &zero, &num_socks, BPF_ANY) != 0) { @@ -933,6 +976,29 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { return -1; } + constexpr uint32_t key_high_idx = 1; + constexpr uint32_t key_low_idx = 2; + + auto &qkms = conn_handler_->get_quic_keying_materials(); + auto &qkm = qkms->keying_materials.front(); + + if (bpf_map_update_elem(bpf_map__fd(sk_info), &key_high_idx, + qkm.cid_encryption_key.data(), BPF_ANY) != 0) { + LOG(FATAL) << "Failed to update key_high_idx sk_info: " + << xsi_strerror(errno, errbuf.data(), errbuf.size()); + close(fd); + return -1; + } + + if (bpf_map_update_elem(bpf_map__fd(sk_info), &key_low_idx, + qkm.cid_encryption_key.data() + 8, + BPF_ANY) != 0) { + LOG(FATAL) << "Failed to update key_low_idx sk_info: " + << xsi_strerror(errno, errbuf.data(), errbuf.size()); + close(fd); + return -1; + } + auto prog_fd = bpf_program__fd(prog); if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_REUSEPORT_EBPF, &prog_fd, @@ -987,14 +1053,6 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { const uint8_t *Worker::get_cid_prefix() const { return cid_prefix_.data(); } -void Worker::set_quic_secret(const std::shared_ptr &secret) { - quic_secret_ = secret; -} - -const std::shared_ptr &Worker::get_quic_secret() const { - return quic_secret_; -} - const UpstreamAddr *Worker::find_quic_upstream_addr(const Address &local_addr) { std::array host; @@ -1019,9 +1077,13 @@ const UpstreamAddr *Worker::find_quic_upstream_addr(const Address &local_addr) { break; default: assert(0); + abort(); } - auto hostport = util::make_hostport(StringRef{host.data()}, port); + std::array hostport_buf; + + auto hostport = util::make_http_hostport(std::begin(hostport_buf), + StringRef{host.data()}, port); const UpstreamAddr *fallback_faddr = nullptr; for (auto &faddr : quic_upstream_addrs_) { @@ -1033,21 +1095,41 @@ const UpstreamAddr *Worker::find_quic_upstream_addr(const Address &local_addr) { continue; } - switch (faddr.family) { - case AF_INET: - if (util::starts_with(faddr.hostport, StringRef::from_lit("0.0.0.0:"))) { - fallback_faddr = &faddr; - } + if (faddr.port == 443 || faddr.port == 80) { + switch (faddr.family) { + case AF_INET: + if (util::streq(faddr.hostport, StringRef::from_lit("0.0.0.0"))) { + fallback_faddr = &faddr; + } - break; - case AF_INET6: - if (util::starts_with(faddr.hostport, StringRef::from_lit("[::]:"))) { - fallback_faddr = &faddr; - } + break; + case AF_INET6: + if (util::streq(faddr.hostport, StringRef::from_lit("[::]"))) { + fallback_faddr = &faddr; + } - break; - default: - assert(0); + break; + default: + assert(0); + } + } else { + switch (faddr.family) { + case AF_INET: + if (util::starts_with(faddr.hostport, + StringRef::from_lit("0.0.0.0:"))) { + fallback_faddr = &faddr; + } + + break; + case AF_INET6: + if (util::starts_with(faddr.hostport, StringRef::from_lit("[::]:"))) { + fallback_faddr = &faddr; + } + + break; + default: + assert(0); + } } } @@ -1227,8 +1309,10 @@ void downstream_failure(DownstreamAddr *addr, const Address *raddr) { } #ifdef ENABLE_HTTP3 -int create_cid_prefix(uint8_t *cid_prefix) { - if (RAND_bytes(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN) != 1) { +int create_cid_prefix(uint8_t *cid_prefix, const uint8_t *server_id) { + auto p = std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, cid_prefix); + + if (RAND_bytes(p, SHRPX_QUIC_CID_PREFIXLEN - SHRPX_QUIC_SERVER_IDLEN) != 1) { return -1; } diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 72ef2e68..49670630 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -258,15 +258,18 @@ struct WorkerStat { #ifdef ENABLE_HTTP3 struct QUICPacket { QUICPacket(size_t upstream_addr_index, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, size_t datalen) + const Address &local_addr, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen) : upstream_addr_index{upstream_addr_index}, remote_addr{remote_addr}, local_addr{local_addr}, + pi{pi}, data{data, data + datalen} {} - QUICPacket() {} + QUICPacket() : upstream_addr_index{}, remote_addr{}, local_addr{}, pi{} {} size_t upstream_addr_index; Address remote_addr; Address local_addr; + ngtcp2_pkt_info pi; std::vector data; }; #endif // ENABLE_HTTP3 @@ -370,10 +373,6 @@ public: const uint8_t *get_cid_prefix() const; - void set_quic_secret(const std::shared_ptr &secret); - - const std::shared_ptr &get_quic_secret() const; - # ifdef HAVE_LIBBPF bool should_attach_bpf() const; @@ -412,7 +411,6 @@ private: std::array cid_prefix_; std::vector quic_upstream_addrs_; std::vector> quic_listeners_; - std::shared_ptr quic_secret_; #endif // ENABLE_HTTP3 std::shared_ptr downstreamconf_; @@ -441,7 +439,7 @@ private: std::shared_ptr ticket_keys_; std::vector> downstream_addr_groups_; // Worker level blocker for downstream connection. For example, - // this is used when file decriptor is exhausted. + // this is used when file descriptor is exhausted. std::unique_ptr connect_blocker_; bool graceful_shutdown_; @@ -467,8 +465,8 @@ void downstream_failure(DownstreamAddr *addr, const Address *raddr); #ifdef ENABLE_HTTP3 // Creates unpredictable SHRPX_QUIC_CID_PREFIXLEN bytes sequence which // is used as a prefix of QUIC Connection ID. This function returns -// -1 on failure. -int create_cid_prefix(uint8_t *cid_prefix); +// -1 on failure. |server_id| must be 2 bytes long. +int create_cid_prefix(uint8_t *cid_prefix, const uint8_t *server_id); #endif // ENABLE_HTTP3 } // namespace shrpx diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc index 7bcdecd1..517df302 100644 --- a/src/shrpx_worker_process.cc +++ b/src/shrpx_worker_process.cc @@ -519,10 +519,51 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { } #ifdef ENABLE_HTTP3 - if (conn_handler->create_quic_secret() != 0) { - return -1; + auto &quicconf = config->quic; + + std::shared_ptr qkms; + + if (!quicconf.upstream.secret_file.empty()) { + qkms = read_quic_secret_file(quicconf.upstream.secret_file); + if (!qkms) { + LOG(WARN) << "Use QUIC keying materials generated internally"; + } } + if (!qkms) { + qkms = std::make_shared(); + qkms->keying_materials.resize(1); + + auto &qkm = qkms->keying_materials.front(); + + if (RAND_bytes(qkm.reserved.data(), qkm.reserved.size()) != 1) { + LOG(ERROR) << "Failed to generate QUIC secret reserved data"; + return -1; + } + + if (RAND_bytes(qkm.secret.data(), qkm.secret.size()) != 1) { + LOG(ERROR) << "Failed to generate QUIC secret"; + return -1; + } + + if (RAND_bytes(qkm.salt.data(), qkm.salt.size()) != 1) { + LOG(ERROR) << "Failed to generate QUIC salt"; + return -1; + } + } + + for (auto &qkm : qkms->keying_materials) { + if (generate_quic_connection_id_encryption_key( + qkm.cid_encryption_key.data(), qkm.cid_encryption_key.size(), + qkm.secret.data(), qkm.secret.size(), qkm.salt.data(), + qkm.salt.size()) != 0) { + LOG(ERROR) << "Failed to generate QUIC Connection ID encryption key"; + return -1; + } + } + + conn_handler->set_quic_keying_materials(std::move(qkms)); + conn_handler->set_cid_prefixes(wpconf->cid_prefixes); conn_handler->set_quic_lingering_worker_processes( wpconf->quic_lingering_worker_processes); @@ -562,6 +603,10 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { #endif // !NOTHREADS } +#if defined(ENABLE_HTTP3) && defined(HAVE_LIBBPF) + conn_handler->unload_bpf_objects(); +#endif // defined(ENABLE_HTTP3) && defined(HAVE_LIBBPF) + drop_privileges( #ifdef HAVE_NEVERBLEED nb.get() diff --git a/src/ssl_compat.h b/src/ssl_compat.h index e1077eef..87f326a4 100644 --- a/src/ssl_compat.h +++ b/src/ssl_compat.h @@ -26,20 +26,20 @@ # include -# if defined(LIBRESSL_VERSION_NUMBER) +# ifdef LIBRESSL_VERSION_NUMBER # define OPENSSL_1_1_API 0 # define OPENSSL_1_1_1_API 0 # define OPENSSL_3_0_0_API 0 # define LIBRESSL_IN_USE 1 # define LIBRESSL_LEGACY_API (LIBRESSL_VERSION_NUMBER < 0x20700000L) # define LIBRESSL_2_7_API (LIBRESSL_VERSION_NUMBER >= 0x20700000L) -# else // !defined(LIBRESSL_VERSION_NUMBER) +# else // !LIBRESSL_VERSION_NUMBER # define OPENSSL_1_1_API (OPENSSL_VERSION_NUMBER >= 0x1010000fL) # define OPENSSL_1_1_1_API (OPENSSL_VERSION_NUMBER >= 0x10101000L) # define OPENSSL_3_0_0_API (OPENSSL_VERSION_NUMBER >= 0x30000000L) # define LIBRESSL_IN_USE 0 # define LIBRESSL_LEGACY_API 0 # define LIBRESSL_2_7_API 0 -# endif // !defined(LIBRESSL_VERSION_NUMBER) +# endif // !LIBRESSL_VERSION_NUMBER #endif // OPENSSL_COMPAT_H diff --git a/src/tls.h b/src/tls.h index ee8a5d7f..2a6bf458 100644 --- a/src/tls.h +++ b/src/tls.h @@ -57,11 +57,15 @@ constexpr char DEFAULT_CIPHER_LIST[] = "AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256"; constexpr char DEFAULT_TLS13_CIPHER_LIST[] = -#if OPENSSL_1_1_1_API +#if OPENSSL_3_0_0_API + "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256" +#elif OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) TLS_DEFAULT_CIPHERSUITES -#else // !OPENSSL_1_1_1_API +#else // !OPENSSL_3_0_0_API && !(OPENSSL_1_1_1_API && + // !defined(OPENSSL_IS_BORINGSSL)) "" -#endif // !OPENSSL_1_1_1_API +#endif // !OPENSSL_3_0_0_API && !(OPENSSL_1_1_1_API && + // !defined(OPENSSL_IS_BORINGSSL)) ; constexpr auto NGHTTP2_TLS_MIN_VERSION = TLS1_VERSION; diff --git a/src/util.cc b/src/util.cc index 90f77777..b8a7b8fc 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1365,81 +1365,14 @@ std::string dtos(double n) { StringRef make_http_hostport(BlockAllocator &balloc, const StringRef &host, uint16_t port) { - if (port != 80 && port != 443) { - return make_hostport(balloc, host, port); - } - - auto ipv6 = ipv6_numeric_addr(host.c_str()); - - auto iov = make_byte_ref(balloc, host.size() + (ipv6 ? 2 : 0) + 1); - auto p = iov.base; - - if (ipv6) { - *p++ = '['; - } - - p = std::copy(std::begin(host), std::end(host), p); - - if (ipv6) { - *p++ = ']'; - } - - *p = '\0'; - - return StringRef{iov.base, p}; -} - -std::string make_hostport(const StringRef &host, uint16_t port) { - auto ipv6 = ipv6_numeric_addr(host.c_str()); - auto serv = utos(port); - - std::string hostport; - hostport.resize(host.size() + (ipv6 ? 2 : 0) + 1 + serv.size()); - - auto p = &hostport[0]; - - if (ipv6) { - *p++ = '['; - } - - p = std::copy_n(host.c_str(), host.size(), p); - - if (ipv6) { - *p++ = ']'; - } - - *p++ = ':'; - std::copy_n(serv.c_str(), serv.size(), p); - - return hostport; + auto iov = make_byte_ref(balloc, host.size() + 2 + 1 + 5 + 1); + return make_http_hostport(iov.base, host, port); } StringRef make_hostport(BlockAllocator &balloc, const StringRef &host, uint16_t port) { - auto ipv6 = ipv6_numeric_addr(host.c_str()); - auto serv = utos(port); - - auto iov = - make_byte_ref(balloc, host.size() + (ipv6 ? 2 : 0) + 1 + serv.size()); - auto p = iov.base; - - if (ipv6) { - *p++ = '['; - } - - p = std::copy(std::begin(host), std::end(host), p); - - if (ipv6) { - *p++ = ']'; - } - - *p++ = ':'; - - p = std::copy(std::begin(serv), std::end(serv), p); - - *p = '\0'; - - return StringRef{iov.base, p}; + auto iov = make_byte_ref(balloc, host.size() + 2 + 1 + 5 + 1); + return make_hostport(iov.base, host, port); } namespace { @@ -1756,7 +1689,7 @@ std::mt19937 make_mt19937() { } int daemonize(int nochdir, int noclose) { -#if defined(__APPLE__) +#ifdef __APPLE__ pid_t pid; pid = fork(); if (pid == -1) { @@ -1792,9 +1725,9 @@ int daemonize(int nochdir, int noclose) { return 0; #elif defined(_MSC_VER) return -1; -#else // !defined(__APPLE__) +#else // !__APPLE__ && !_MSC_VER return daemon(nochdir, noclose); -#endif // !defined(__APPLE__) +#endif // !__APPLE__ } #ifdef ENABLE_HTTP3 @@ -1833,6 +1766,53 @@ int msghdr_get_local_addr(Address &dest, msghdr *msg, int family) { } #endif +unsigned int msghdr_get_ecn(msghdr *msg, int family) { + switch (family) { + case AF_INET: + for (auto cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) { + if (cmsg->cmsg_level == IPPROTO_IP && cmsg->cmsg_type == IP_TOS && + cmsg->cmsg_len) { + return *reinterpret_cast(CMSG_DATA(cmsg)); + } + } + + return 0; + case AF_INET6: + for (auto cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) { + if (cmsg->cmsg_level == IPPROTO_IPV6 && cmsg->cmsg_type == IPV6_TCLASS && + cmsg->cmsg_len) { + return *reinterpret_cast(CMSG_DATA(cmsg)); + } + } + + return 0; + } + + return 0; +} + +int fd_set_send_ecn(int fd, int family, unsigned int ecn) { + switch (family) { + case AF_INET: + if (setsockopt(fd, IPPROTO_IP, IP_TOS, &ecn, + static_cast(sizeof(ecn))) == -1) { + return -1; + } + + return 0; + case AF_INET6: + if (setsockopt(fd, IPPROTO_IPV6, IPV6_TCLASS, &ecn, + static_cast(sizeof(ecn))) == -1) { + return -1; + } + + return 0; + } + + return -1; +} +#endif // ENABLE_HTTP3 + } // namespace util } // namespace nghttp2 diff --git a/src/util.h b/src/util.h index 17879694..34cbde57 100644 --- a/src/util.h +++ b/src/util.h @@ -572,7 +572,7 @@ std::string ascii_dump(const uint8_t *data, size_t len); // Returns absolute path of executable path. If argc == 0 or |cwd| is // nullptr, this function returns nullptr. If argv[0] starts with -// '/', this function returns argv[0]. Oterwise return cwd + "/" + +// '/', this function returns argv[0]. Otherwise return cwd + "/" + // argv[0]. If non-null is returned, it is NULL-terminated string and // dynamically allocated by malloc. The caller is responsible to free // it. @@ -762,18 +762,71 @@ std::string format_duration(const std::chrono::microseconds &u); // Just like above, but this takes |t| as seconds. std::string format_duration(double t); +// The maximum buffer size including terminal NULL to store the result +// of make_hostport. +constexpr size_t max_hostport = NI_MAXHOST + /* [] for IPv6 */ 2 + /* : */ 1 + + /* port */ 5 + /* terminal NULL */ 1; + +// Just like make_http_hostport(), but doesn't treat 80 and 443 +// specially. +StringRef make_hostport(BlockAllocator &balloc, const StringRef &host, + uint16_t port); + +template +StringRef make_hostport(OutputIt first, const StringRef &host, uint16_t port) { + auto ipv6 = ipv6_numeric_addr(host.c_str()); + auto serv = utos(port); + auto p = first; + + if (ipv6) { + *p++ = '['; + } + + p = std::copy(std::begin(host), std::end(host), p); + + if (ipv6) { + *p++ = ']'; + } + + *p++ = ':'; + + p = std::copy(std::begin(serv), std::end(serv), p); + + *p = '\0'; + + return StringRef{first, p}; +} + // Creates "host:port" string using given |host| and |port|. If // |host| is numeric IPv6 address (e.g., ::1), it is enclosed by "[" // and "]". If |port| is 80 or 443, port part is omitted. StringRef make_http_hostport(BlockAllocator &balloc, const StringRef &host, uint16_t port); -// Just like make_http_hostport(), but doesn't treat 80 and 443 -// specially. -std::string make_hostport(const StringRef &host, uint16_t port); +template +StringRef make_http_hostport(OutputIt first, const StringRef &host, + uint16_t port) { + if (port != 80 && port != 443) { + return make_hostport(first, host, port); + } -StringRef make_hostport(BlockAllocator &balloc, const StringRef &host, - uint16_t port); + auto ipv6 = ipv6_numeric_addr(host.c_str()); + auto p = first; + + if (ipv6) { + *p++ = '['; + } + + p = std::copy(std::begin(host), std::end(host), p); + + if (ipv6) { + *p++ = ']'; + } + + *p = '\0'; + + return StringRef{first, p}; +} // Dumps |src| of length |len| in the format similar to `hexdump -C`. void hexdump(FILE *out, const uint8_t *src, size_t len); @@ -867,6 +920,11 @@ int daemonize(int nochdir, int noclose); int msghdr_get_local_addr(Address &dest, msghdr *msg, int family); #endif +unsigned int msghdr_get_ecn(msghdr *msg, int family); + +int fd_set_send_ecn(int fd, int family, unsigned int ecn); +#endif // ENABLE_HTTP3 + } // namespace util } // namespace nghttp2 diff --git a/src/util_test.cc b/src/util_test.cc index dfe87e99..17d640d5 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -540,10 +540,13 @@ void test_util_make_http_hostport(void) { } void test_util_make_hostport(void) { + std::array hostport_buf; CU_ASSERT("localhost:80" == - util::make_hostport(StringRef::from_lit("localhost"), 80)); - CU_ASSERT("[::1]:443" == - util::make_hostport(StringRef::from_lit("::1"), 443)); + util::make_hostport(std::begin(hostport_buf), + StringRef::from_lit("localhost"), 80)); + CU_ASSERT("[::1]:443" == util::make_hostport(std::begin(hostport_buf), + StringRef::from_lit("::1"), + 443)); BlockAllocator balloc(4096, 4096); CU_ASSERT("localhost:80" == diff --git a/tests/failmalloc.c b/tests/failmalloc.c index 4bf83ca5..6294cfff 100644 --- a/tests/failmalloc.c +++ b/tests/failmalloc.c @@ -37,7 +37,7 @@ static int init_suite1(void) { return 0; } static int clean_suite1(void) { return 0; } -int main() { +int main(void) { CU_pSuite pSuite = NULL; unsigned int num_tests_failed; diff --git a/tests/main.c b/tests/main.c index 25cbbfd7..dc41c7c7 100644 --- a/tests/main.c +++ b/tests/main.c @@ -47,7 +47,7 @@ static int init_suite1(void) { return 0; } static int clean_suite1(void) { return 0; } -int main() { +int main(void) { CU_pSuite pSuite = NULL; unsigned int num_tests_failed; diff --git a/tests/nghttp2_hd_test.c b/tests/nghttp2_hd_test.c index 74725d40..657d895f 100644 --- a/tests/nghttp2_hd_test.c +++ b/tests/nghttp2_hd_test.c @@ -734,7 +734,7 @@ void test_nghttp2_hd_change_table_size(void) { CU_ASSERT(4096 == deflater.ctx.hd_table_bufsize_max); - CU_ASSERT(8000 == inflater.ctx.hd_table_bufsize_max); + CU_ASSERT(4096 == inflater.ctx.hd_table_bufsize_max); CU_ASSERT(8000 == inflater.settings_hd_table_bufsize_max); /* This will emit encoding context update with header table size 4096 */ @@ -830,8 +830,8 @@ void test_nghttp2_hd_change_table_size(void) { CU_ASSERT(8000 == deflater.ctx.hd_table_bufsize_max); CU_ASSERT(8000 == nghttp2_hd_deflate_get_max_dynamic_table_size(&deflater)); - CU_ASSERT(8000 == inflater.ctx.hd_table_bufsize_max); - CU_ASSERT(8000 == nghttp2_hd_inflate_get_max_dynamic_table_size(&inflater)); + CU_ASSERT(4096 == inflater.ctx.hd_table_bufsize_max); + CU_ASSERT(4096 == nghttp2_hd_inflate_get_max_dynamic_table_size(&inflater)); CU_ASSERT(8000 == inflater.settings_hd_table_bufsize_max); rv = nghttp2_hd_deflate_hd_bufs(&deflater, &bufs, nva, 2); @@ -856,8 +856,8 @@ void test_nghttp2_hd_change_table_size(void) { CU_ASSERT(8192 == deflater.ctx.hd_table_bufsize_max); CU_ASSERT(8192 == nghttp2_hd_deflate_get_max_dynamic_table_size(&deflater)); - CU_ASSERT(16383 == inflater.ctx.hd_table_bufsize_max); - CU_ASSERT(16383 == nghttp2_hd_inflate_get_max_dynamic_table_size(&inflater)); + CU_ASSERT(8000 == inflater.ctx.hd_table_bufsize_max); + CU_ASSERT(8000 == nghttp2_hd_inflate_get_max_dynamic_table_size(&inflater)); CU_ASSERT(16383 == inflater.settings_hd_table_bufsize_max); rv = nghttp2_hd_deflate_hd_bufs(&deflater, &bufs, nva, 2); diff --git a/tests/nghttp2_session_test.c b/tests/nghttp2_session_test.c index 962e3c13..cb6bdf73 100644 --- a/tests/nghttp2_session_test.c +++ b/tests/nghttp2_session_test.c @@ -2954,7 +2954,7 @@ void test_nghttp2_session_on_request_headers_received(void) { session->local_settings.max_concurrent_streams = NGHTTP2_DEFAULT_MAX_CONCURRENT_STREAMS; - /* Stream ID less than or equal to the previouly received request + /* Stream ID less than or equal to the previously received request HEADERS is just ignored due to race condition */ nghttp2_frame_headers_init(&frame.headers, NGHTTP2_FLAG_END_HEADERS | NGHTTP2_FLAG_PRIORITY, @@ -4650,7 +4650,7 @@ void test_nghttp2_session_reprioritize_stream(void) { CU_ASSERT(10 == stream->weight); CU_ASSERT(&session->root == stream->dep_prev); - /* If depenency to idle stream which is not in depdenency tree yet */ + /* If dependency to idle stream which is not in dependency tree yet */ nghttp2_priority_spec_init(&pri_spec, 3, 99, 0); @@ -5720,7 +5720,7 @@ void test_nghttp2_submit_settings(void) { nghttp2_frame_settings_free(&ack_frame.settings, mem); CU_ASSERT(16 * 1024 == session->local_settings.initial_window_size); - CU_ASSERT(1023 == session->hd_inflater.ctx.hd_table_bufsize_max); + CU_ASSERT(111 == session->hd_inflater.ctx.hd_table_bufsize_max); CU_ASSERT(111 == session->hd_inflater.min_hd_table_bufsize_max); CU_ASSERT(50 == session->local_settings.max_concurrent_streams); @@ -9070,7 +9070,7 @@ void test_nghttp2_session_stream_get_state(void) { CU_ASSERT(NGHTTP2_STREAM_STATE_RESERVED_LOCAL == nghttp2_stream_get_state(stream)); - /* Send resposne to push stream 2 with END_STREAM set */ + /* Send response to push stream 2 with END_STREAM set */ nghttp2_submit_response(session, 2, resnv, ARRLEN(resnv), NULL); rv = nghttp2_session_send(session); diff --git a/third-party/Makefile.am b/third-party/Makefile.am index 55fdad0c..2d6726d8 100644 --- a/third-party/Makefile.am +++ b/third-party/Makefile.am @@ -23,7 +23,7 @@ AM_CPPFLAGS = @DEFS@ -EXTRA_DIST = CMakeLists.txt +EXTRA_DIST = CMakeLists.txt build_config.rb mruby/* if ENABLE_THIRD_PARTY @@ -49,8 +49,6 @@ endif # HAVE_NEVERBLEED if HAVE_MRUBY -EXTRA_DIST += build_config.rb mruby/* - .PHONY: all-local clean mruby mruby: diff --git a/third-party/build_config.rb b/third-party/build_config.rb index 9e66958c..86dd8152 100644 --- a/third-party/build_config.rb +++ b/third-party/build_config.rb @@ -3,7 +3,7 @@ MRuby::Build.new do |conf| toolchain :gcc if ENV['CC'].include? "gcc" # C++ project needs this. Without this, mruby exception does not - # properly destory C++ object allocated on stack. + # properly destroy C++ object allocated on stack. conf.enable_cxx_exception conf.build_dir = ENV['BUILD_DIR']