Merge branch 'quic'

This commit is contained in:
Tatsuhiro Tsujikawa 2021-08-23 18:41:35 +09:00
commit ff389b3e97
47 changed files with 6906 additions and 104 deletions

View File

@ -11,6 +11,7 @@ jobs:
os: [ubuntu-20.04, macos-10.15]
compiler: [gcc, clang]
buildtool: [autotools, cmake]
http3: [http3, no-http3]
steps:
- uses: actions/checkout@v2
@ -74,6 +75,42 @@ jobs:
run: |
echo 'CC=gcc' >> $GITHUB_ENV
echo 'CXX=g++' >> $GITHUB_ENV
- name: Build quictls/openssl
if: matrix.http3 == 'http3'
run: |
git clone --depth 1 -b OpenSSL_1_1_1k+quic https://github.com/quictls/openssl
cd openssl
./config enable-tls1_3 --prefix=$PWD/build
make -j$(nproc)
make install_sw
- name: Build nghttp3
if: matrix.http3 == 'http3'
run: |
git clone https://github.com/ngtcp2/nghttp3
cd nghttp3
autoreconf -i
./configure --prefix=$PWD/build --enable-lib-only
make -j$(nproc) check
make install
- name: Build ngtcp2
if: matrix.http3 == 'http3'
run: |
git clone https://github.com/ngtcp2/ngtcp2
cd ngtcp2
autoreconf -i
./configure --prefix=$PWD/build --enable-lib-only PKG_CONFIG_PATH="../openssl/build/lib/pkgconfig"
make -j$(nproc) check
make install
- name: Setup extra environment variables for HTTP/3
if: matrix.http3 == 'http3'
run: |
PKG_CONFIG_PATH="$PWD/openssl/build/lib/pkgconfig:$PWD/nghttp3/build/lib/pkgconfig:$PWD/ngtcp2/build/lib/pkgconfig:$PKG_CONFIG_PATH"
LDFLAGS="$LDFLAGS -Wl,-rpath,$PWD/openssl/build/lib"
echo 'PKG_CONFIG_PATH='"$PKG_CONFIG_PATH" >> $GITHUB_ENV
echo 'LDFLAGS='"$LDFLAGS" >> $GITHUB_ENV
echo 'EXTRA_AUTOTOOLS_OPTS=--enable-http3' >> $GITHUB_ENV
echo 'EXTRA_CMAKE_OPTS=-DENABLE_HTTP3=ON' >> $GITHUB_ENV
- name: Setup git submodules
run: |
git submodule update --init
@ -81,16 +118,16 @@ jobs:
if: matrix.buildtool == 'autotools'
run: |
autoreconf -i
./configure --enable-werror --with-mruby
./configure --enable-werror --with-mruby $EXTRA_AUTOTOOLS_OPTS
- name: Configure cmake
if: matrix.buildtool == 'cmake'
run: |
cmake -DENABLE_WERROR=1 -DWITH_MRUBY=1 -DWITH_NEVERBLEED=1 -DCPPFLAGS="$CPPFLAGS" -DLDFLAGS="$LDFLAGS" .
cmake -DENABLE_WERROR=1 -DWITH_MRUBY=1 -DWITH_NEVERBLEED=1 $EXTRA_CMAKE_OPTS -DCPPFLAGS="$CPPFLAGS" -DLDFLAGS="$LDFLAGS" .
- name: Build nghttp2 with autotools
if: matrix.buildtool == 'autotools'
run: |
make distcheck \
DISTCHECK_CONFIGURE_FLAGS="--with-mruby --with-neverbleed --enable-werror CPPFLAGS=\"$CPPFLAGS\" LDFLAGS=\"$LDFLAGS\""
DISTCHECK_CONFIGURE_FLAGS="--with-mruby --with-neverbleed --enable-werror $EXTRA_AUTOTOOLS_OPTS CPPFLAGS=\"$CPPFLAGS\" LDFLAGS=\"$LDFLAGS\""
- name: Build nghttp2 with cmake
if: matrix.buildtool == 'cmake'
run: |

View File

@ -61,6 +61,9 @@ find_package(OpenSSL 1.0.1)
find_package(Libev 4.11)
find_package(Libcares 1.7.5)
find_package(ZLIB 1.2.3)
find_package(Libngtcp2 0.0.0)
find_package(Libngtcp2_crypto_openssl 0.0.0)
find_package(Libnghttp3 0.0.0)
if(OPENSSL_FOUND AND LIBEV_FOUND AND ZLIB_FOUND)
set(ENABLE_APP_DEFAULT ON)
else()
@ -182,9 +185,18 @@ if(HAVE_CUNIT)
endif()
# openssl (for src)
include(CheckSymbolExists)
set(HAVE_OPENSSL ${OPENSSL_FOUND})
if(OPENSSL_FOUND)
set(OPENSSL_INCLUDE_DIRS ${OPENSSL_INCLUDE_DIR})
cmake_push_check_state()
set(CMAKE_REQUIRED_INCLUDES "${OPENSSL_INCLUDE_DIR}")
set(CMAKE_REQUIRED_LIBRARIES "${OPENSSL_LIBRARIES}")
check_symbol_exists(SSL_is_quic "openssl/ssl.h" HAVE_SSL_IS_QUIC)
if(NOT HAVE_SSL_IS_QUIC)
message(WARNING "OpenSSL in ${OPENSSL_LIBRARIES} dose not have SSL_is_quic. HTTP/3 support cannot be enabled")
endif()
cmake_pop_check_state()
else()
set(OPENSSL_INCLUDE_DIRS "")
set(OPENSSL_LIBRARIES "")
@ -228,6 +240,12 @@ if(ENABLE_APP AND NOT (ZLIB_FOUND AND OPENSSL_FOUND AND LIBEV_FOUND))
message(FATAL_ERROR "Applications were requested (ENABLE_APP=1) but dependencies are not met.")
endif()
# HTTP/3 requires quictls/openssl, libngtcp2, libngtcp2_crypto_openssl
# and libnghttp3.
if(ENABLE_HTTP3 AND NOT (HAVE_SSL_IS_QUIC AND LIBNGTCP2_FOUND AND LIBNGTCP2_CRYPTO_OPENSSL_FOUND AND LIBNGHTTP3_FOUND))
message(FATAL_ERROR "HTTP/3 was requested (ENABLE_HTTP3=1) but dependencies are not met.")
endif()
# HPACK tools requires jansson
if(ENABLE_HPACK_TOOLS AND NOT HAVE_JANSSON)
message(FATAL_ERROR "HPACK tools were requested (ENABLE_HPACK_TOOLS=1) but dependencies are not met.")
@ -448,6 +466,10 @@ foreach(name
configure_file("${name}.in" "${name}" @ONLY)
endforeach()
if(APPLE)
add_definitions(-D__APPLE_USE_RFC_3542)
endif()
include_directories(
"${CMAKE_CURRENT_BINARY_DIR}" # for config.h
)
@ -499,6 +521,9 @@ message(STATUS "summary of build options:
Libxml2: ${HAVE_LIBXML2} (LIBS='${LIBXML2_LIBRARIES}')
Libev: ${HAVE_LIBEV} (LIBS='${LIBEV_LIBRARIES}')
Libc-ares: ${HAVE_LIBCARES} (LIBS='${LIBCARES_LIBRARIES}')
Libngtcp2: ${HAVE_LIBNGTCP2} (LIBS='${LIBNGTCP2_LIBRARIES}')
Libngtcp2_crypto_openssl: ${HAVE_LIBNGTCP2_CRYPTO_OPENSSL} (LIBS='${LIBNGTCP2_CRYPTO_OPENSSL_LIBRARIES}')
Libnghttp3: ${HAVE_LIBNGHTTP3} (LIBS='${LIBNGHTTP3_LIBRARIES}')
Libevent(SSL): ${HAVE_LIBEVENT_OPENSSL} (LIBS='${LIBEVENT_OPENSSL_LIBRARIES}')
Jansson: ${HAVE_JANSSON} (LIBS='${JANSSON_LIBRARIES}')
Jemalloc: ${HAVE_JEMALLOC} (LIBS='${JEMALLOC_LIBRARIES}')
@ -517,6 +542,7 @@ message(STATUS "summary of build options:
Examples: ${ENABLE_EXAMPLES}
Python bindings:${ENABLE_PYTHON_BINDINGS}
Threading: ${ENABLE_THREADS}
HTTP/3(EXPERIMENTAL): ${ENABLE_HTTP3}
")
if(ENABLE_LIB_ONLY_DISABLED_OTHERS)
message("Only the library will be built. To build other components "

View File

@ -17,6 +17,7 @@ option(ENABLE_LIB_ONLY "Build libnghttp2 only. This is a short hand for -DENAB
option(ENABLE_STATIC_LIB "Build libnghttp2 in static mode also")
option(ENABLE_SHARED_LIB "Build libnghttp2 as a shared library" ON)
option(ENABLE_STATIC_CRT "Build libnghttp2 against the MS LIBCMT[d]")
option(ENABLE_HTTP3 "Enable HTTP/3 support" OFF)
option(WITH_LIBXML2 "Use libxml2"
${WITH_LIBXML2_DEFAULT})

View File

@ -145,6 +145,14 @@ minimizes the risk of private key leakage when serious bug like
Heartbleed is exploited. The neverbleed is disabled by default. To
enable it, use ``--with-neverbleed`` configure option.
To enable the experimental HTTP/3 support for h2load and nghttpx, the
following libraries are required:
* `OpenSSL with QUIC support
<https://github.com/quictls/openssl/tree/OpenSSL_1_1_1k+quic>`_
* `ngtcp2 <https://github.com/ngtcp2/ngtcp2>`_
* `nghttp3 <https://github.com/ngtcp2/nghttp3>`_
Compiling libnghttp2 C source code requires a C99 compiler. gcc 4.8
is known to be adequate. In order to compile the C++ source code, gcc
>= 6.0 or clang >= 6.0 is required. C++ source code requires C++14
@ -875,6 +883,14 @@ threads to avoid saturating a single core on client side.
considered a DOS attack. Please only use it against your private
servers.
If the experimental HTTP/3 is enabled, h2load can send requests to
HTTP/3 server. To do this, specify ``h3`` to ``--npn-list`` option
like so:
.. code-block:: text
$ h2load --npn-list h3 https://127.0.0.1:4433
HPACK tools
-----------

View File

@ -0,0 +1,41 @@
# - Try to find libnghttp3
# Once done this will define
# LIBNGHTTP3_FOUND - System has libnghttp3
# LIBNGHTTP3_INCLUDE_DIRS - The libnghttp3 include directories
# LIBNGHTTP3_LIBRARIES - The libraries needed to use libnghttp3
find_package(PkgConfig QUIET)
pkg_check_modules(PC_LIBNGHTTP3 QUIET libnghttp3)
find_path(LIBNGHTTP3_INCLUDE_DIR
NAMES nghttp3/nghttp3.h
HINTS ${PC_LIBNGHTTP3_INCLUDE_DIRS}
)
find_library(LIBNGHTTP3_LIBRARY
NAMES nghttp3
HINTS ${PC_LIBNGHTTP3_LIBRARY_DIRS}
)
if(LIBNGHTTP3_INCLUDE_DIR)
set(_version_regex "^#define[ \t]+NGHTTP3_VERSION[ \t]+\"([^\"]+)\".*")
file(STRINGS "${LIBNGHTTP3_INCLUDE_DIR}/nghttp3/version.h"
LIBNGHTTP3_VERSION REGEX "${_version_regex}")
string(REGEX REPLACE "${_version_regex}" "\\1"
LIBNGHTTP3_VERSION "${LIBNGHTTP3_VERSION}")
unset(_version_regex)
endif()
include(FindPackageHandleStandardArgs)
# handle the QUIETLY and REQUIRED arguments and set LIBNGHTTP3_FOUND
# to TRUE if all listed variables are TRUE and the requested version
# matches.
find_package_handle_standard_args(Libnghttp3 REQUIRED_VARS
LIBNGHTTP3_LIBRARY LIBNGHTTP3_INCLUDE_DIR
VERSION_VAR LIBNGHTTP3_VERSION)
if(LIBNGHTTP3_FOUND)
set(LIBNGHTTP3_LIBRARIES ${LIBNGHTTP3_LIBRARY})
set(LIBNGHTTP3_INCLUDE_DIRS ${LIBNGHTTP3_INCLUDE_DIR})
endif()
mark_as_advanced(LIBNGHTTP3_INCLUDE_DIR LIBNGHTTP3_LIBRARY)

41
cmake/FindLibngtcp2.cmake Normal file
View File

@ -0,0 +1,41 @@
# - Try to find libngtcp2
# Once done this will define
# LIBNGTCP2_FOUND - System has libngtcp2
# LIBNGTCP2_INCLUDE_DIRS - The libngtcp2 include directories
# LIBNGTCP2_LIBRARIES - The libraries needed to use libngtcp2
find_package(PkgConfig QUIET)
pkg_check_modules(PC_LIBNGTCP2 QUIET libngtcp2)
find_path(LIBNGTCP2_INCLUDE_DIR
NAMES ngtcp2/ngtcp2.h
HINTS ${PC_LIBNGTCP2_INCLUDE_DIRS}
)
find_library(LIBNGTCP2_LIBRARY
NAMES ngtcp2
HINTS ${PC_LIBNGTCP2_LIBRARY_DIRS}
)
if(LIBNGTCP2_INCLUDE_DIR)
set(_version_regex "^#define[ \t]+NGTCP2_VERSION[ \t]+\"([^\"]+)\".*")
file(STRINGS "${LIBNGTCP2_INCLUDE_DIR}/ngtcp2/version.h"
LIBNGTCP2_VERSION REGEX "${_version_regex}")
string(REGEX REPLACE "${_version_regex}" "\\1"
LIBNGTCP2_VERSION "${LIBNGTCP2_VERSION}")
unset(_version_regex)
endif()
include(FindPackageHandleStandardArgs)
# handle the QUIETLY and REQUIRED arguments and set LIBNGTCP2_FOUND
# to TRUE if all listed variables are TRUE and the requested version
# matches.
find_package_handle_standard_args(Libngtcp2 REQUIRED_VARS
LIBNGTCP2_LIBRARY LIBNGTCP2_INCLUDE_DIR
VERSION_VAR LIBNGTCP2_VERSION)
if(LIBNGTCP2_FOUND)
set(LIBNGTCP2_LIBRARIES ${LIBNGTCP2_LIBRARY})
set(LIBNGTCP2_INCLUDE_DIRS ${LIBNGTCP2_INCLUDE_DIR})
endif()
mark_as_advanced(LIBNGTCP2_INCLUDE_DIR LIBNGTCP2_LIBRARY)

View File

@ -0,0 +1,43 @@
# - Try to find libngtcp2_crypto_openssl
# Once done this will define
# LIBNGTCP2_CRYPTO_OPENSSL_FOUND - System has libngtcp2_crypto_openssl
# LIBNGTCP2_CRYPTO_OPENSSL_INCLUDE_DIRS - The libngtcp2_crypto_openssl include directories
# LIBNGTCP2_CRYPTO_OPENSSL_LIBRARIES - The libraries needed to use libngtcp2_crypto_openssl
find_package(PkgConfig QUIET)
pkg_check_modules(PC_LIBNGTCP2_CRYPTO_OPENSSL QUIET libngtcp2_crypto_openssl)
find_path(LIBNGTCP2_CRYPTO_OPENSSL_INCLUDE_DIR
NAMES ngtcp2/ngtcp2_crypto_openssl.h
HINTS ${PC_LIBNGTCP2_CRYPTO_OPENSSL_INCLUDE_DIRS}
)
find_library(LIBNGTCP2_CRYPTO_OPENSSL_LIBRARY
NAMES ngtcp2_crypto_openssl
HINTS ${PC_LIBNGTCP2_CRYPTO_OPENSSL_LIBRARY_DIRS}
)
if(LIBNGTCP2_CRYPTO_OPENSSL_INCLUDE_DIR)
set(_version_regex "^#define[ \t]+NGTCP2_VERSION[ \t]+\"([^\"]+)\".*")
file(STRINGS "${LIBNGTCP2_CRYPTO_OPENSSL_INCLUDE_DIR}/ngtcp2/version.h"
LIBNGTCP2_CRYPTO_OPENSSL_VERSION REGEX "${_version_regex}")
string(REGEX REPLACE "${_version_regex}" "\\1"
LIBNGTCP2_CRYPTO_OPENSSL_VERSION "${LIBNGTCP2_CRYPTO_OPENSSL_VERSION}")
unset(_version_regex)
endif()
include(FindPackageHandleStandardArgs)
# handle the QUIETLY and REQUIRED arguments and set
# LIBNGTCP2_CRYPTO_OPENSSL_FOUND to TRUE if all listed variables are
# TRUE and the requested version matches.
find_package_handle_standard_args(Libngtcp2_crypto_openssl REQUIRED_VARS
LIBNGTCP2_CRYPTO_OPENSSL_LIBRARY
LIBNGTCP2_CRYPTO_OPENSSL_INCLUDE_DIR
VERSION_VAR LIBNGTCP2_CRYPTO_OPENSSL_VERSION)
if(LIBNGTCP2_CRYPTO_OPENSSL_FOUND)
set(LIBNGTCP2_CRYPTO_OPENSSL_LIBRARIES ${LIBNGTCP2_CRYPTO_OPENSSL_LIBRARY})
set(LIBNGTCP2_CRYPTO_OPENSSL_INCLUDE_DIRS ${LIBNGTCP2_CRYPTO_OPENSSL_INCLUDE_DIR})
endif()
mark_as_advanced(LIBNGTCP2_CRYPTO_OPENSSL_INCLUDE_DIR
LIBNGTCP2_CRYPTO_OPENSSL_LIBRARY)

View File

@ -78,3 +78,6 @@
/* Define to 1 if you have the <unistd.h> header file. */
#cmakedefine HAVE_UNISTD_H 1
/* Define to 1 if HTTP/3 is enabled. */
#cmakedefine ENABLE_HTTP3 1

View File

@ -107,6 +107,11 @@ AC_ARG_ENABLE([lib-only],
[Build libnghttp2 only. This is a short hand for --disable-app --disable-examples --disable-hpack-tools --disable-python-bindings])],
[request_lib_only=$enableval], [request_lib_only=no])
AC_ARG_ENABLE([http3],
[AS_HELP_STRING([--enable-http3],
[(EXPERIMENTAL) Enable HTTP/3. This requires ngtcp2, nghttp3, and a custom OpenSSL.])],
[request_http3=$enableval], [request_http3=no])
AC_ARG_WITH([libxml2],
[AS_HELP_STRING([--with-libxml2],
[Use libxml2 [default=check]])],
@ -172,6 +177,16 @@ AC_ARG_WITH([cython],
[Use cython in given PATH])],
[cython_path=$withval], [])
AC_ARG_WITH([libngtcp2],
[AS_HELP_STRING([--with-libngtcp2],
[Use libngtcp2 [default=check]])],
[request_libngtcp2=$withval], [request_libngtcp2=check])
AC_ARG_WITH([libnghttp3],
[AS_HELP_STRING([--with-libnghttp3],
[Use libnghttp3 [default=check]])],
[request_libnghttp3=$withval], [request_libnghttp3=check])
dnl Define variables
AC_ARG_VAR([CYTHON], [the Cython executable])
@ -334,6 +349,13 @@ case "$host_os" in
;;
esac
case "${build}" in
*-apple-darwin*)
EXTRA_DEFS="-D__APPLE_USE_RFC_3542"
AC_SUBST([EXTRA_DEFS])
;;
esac
# zlib
have_zlib=no
if test "x${request_zlib}" != "xno"; then
@ -431,6 +453,25 @@ if test "x${request_openssl}" != "xno"; then
[have_openssl=yes], [have_openssl=no])
if test "x${have_openssl}" = "xno"; then
AC_MSG_NOTICE($OPENSSL_PKG_ERRORS)
else
save_CFLAGS="$CFLAGS"
save_LIBS="$LIBS"
CFLAGS="$OPENSSL_CFLAGS $CFLAGS"
LIBS="$OPENSSL_LIBS $LIBS"
have_ssl_is_quic=no
AC_MSG_CHECKING([for SSL_is_quic])
AC_LINK_IFELSE([AC_LANG_PROGRAM([[
#include <openssl/ssl.h>
]], [[
SSL *ssl = NULL;
SSL_is_quic(ssl);
]])],
[AC_MSG_RESULT([yes]); have_ssl_is_quic=yes],
[AC_MSG_RESULT([no]); have_ssl_is_quic=no])
CFLAGS="$save_CFLAGS"
LIBS="$save_LIBS"
fi
fi
@ -454,6 +495,53 @@ if test "x${request_libcares}" = "xyes" &&
AC_MSG_ERROR([libcares was requested (--with-libcares) but not found])
fi
# ngtcp2 (for src)
have_libngtcp2=no
if test "x${request_libngtcp2}" != "xno"; then
PKG_CHECK_MODULES([LIBNGTCP2], [libngtcp2 >= 0.0.0], [have_libngtcp2=yes],
[have_libngtcp2=no])
if test "x${have_libngtcp2}" = "xno"; then
AC_MSG_NOTICE($LIBNGTCP2_PKG_ERRORS)
fi
fi
if test "x${request_libngtcp2}" = "xyes" &&
test "x${have_libngtcp2}" != "xyes"; then
AC_MSG_ERROR([libngtcp2 was requested (--with-libngtcp2) but not found])
fi
# ngtcp2_crypto_openssl (for src)
have_libngtcp2_crypto_openssl=no
if test "x${request_libngtcp2}" != "xno"; then
PKG_CHECK_MODULES([LIBNGTCP2_CRYPTO_OPENSSL],
[libngtcp2_crypto_openssl >= 0.0.0],
[have_libngtcp2_crypto_openssl=yes],
[have_libngtcp2_crypto_openssl=no])
if test "x${have_libngtcp2_crypto_openssl}" = "xno"; then
AC_MSG_NOTICE($LIBNGTCP2_CRYPTO_OPENSSL_PKG_ERRORS)
fi
fi
if test "x${request_libngtcp2}" = "xyes" &&
test "x${have_libngtcp2_crypto_openssl}" != "xyes"; then
AC_MSG_ERROR([libngtcp2_crypto_openssl was requested (--with-libngtcp2) but not found])
fi
# nghttp3 (for src)
have_libnghttp3=no
if test "x${request_libnghttp3}" != "xno"; then
PKG_CHECK_MODULES([LIBNGHTTP3], [libnghttp3 >= 0.0.0], [have_libnghttp3=yes],
[have_libnghttp3=no])
if test "x${have_libnghttp3}" = "xno"; then
AC_MSG_NOTICE($LIBNGHTTP3_PKG_ERRORS)
fi
fi
if test "x${request_libnghttp3}" = "xyes" &&
test "x${have_libnghttp3}" != "xyes"; then
AC_MSG_ERROR([libnghttp3 was requested (--with-libnghttp3) but not found])
fi
# libevent_openssl (for examples)
# 2.0.8 is required because we use evconnlistener_set_error_cb()
have_libevent_openssl=no
@ -598,6 +686,24 @@ fi
AM_CONDITIONAL([ENABLE_APP], [ test "x${enable_app}" = "xyes" ])
# Check HTTP/3 support
enable_http3=no
if test "x${request_http3}" != "xno" &&
test "x${have_ssl_is_quic}" = "xyes" &&
test "x${have_libngtcp2}" = "xyes" &&
test "x${have_libngtcp2_crypto_openssl}" = "xyes" &&
test "x${have_libnghttp3}" = "xyes"; then
enable_http3=yes
AC_DEFINE([ENABLE_HTTP3], [1], [Define to 1 if HTTP/3 is enabled.])
fi
if test "x${request_http3}" = "xyes" &&
test "x${enable_http3}" != "xyes"; then
AC_MSG_ERROR([HTTP/3 was requested (--enable-http3) but dependencies are not met.])
fi
AM_CONDITIONAL([ENABLE_HTTP3], [ test "x${enable_http3}" = "xyes" ])
enable_hpack_tools=no
# HPACK tools requires jansson
if test "x${request_hpack_tools}" != "xno" &&
@ -824,6 +930,31 @@ AC_CHECK_DECLS([initgroups], [], [], [[
#include <grp.h>
]])
have_netinet_udp_h_udp_segment=no
AC_CHECK_DECL([UDP_SEGMENT], [have_netinet_udp_h_udp_segment=yes],
[have_netinet_udp_h_udp_segment=no], [[
#include <netinet/udp.h>
]])
if test "x$have_netinet_udp_h_udp_segment" = "xno"; then
have_linux_udp_h_udp_segment=no
AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
[[
#include <linux/udp.h>
]],
[[
#if UDP_SEGMENT != 103
exit(1)
#endif
]])],
[have_linux_udp_h_udp_segment=yes],
[have_linux_udp_h_udp_segment=no])
if test "x$have_linux_udp_h_udp_segment" = "xyes"; then
EXTRA_DEFS="$EXTRA_DEFS -DUDP_SEGMENT=103"
fi
fi
save_CFLAGS=$CFLAGS
save_CXXFLAGS=$CXXFLAGS
@ -991,6 +1122,8 @@ AC_MSG_NOTICE([summary of build options:
CXX1XCXXFLAGS: ${CXX1XCXXFLAGS}
EXTRACFLAG: ${EXTRACFLAG}
LIBS: ${LIBS}
DEFS: ${DEFS}
EXTRA_DEFS: ${EXTRA_DEFS}
Library:
Shared: ${enable_shared}
Static: ${enable_static}
@ -1011,6 +1144,9 @@ AC_MSG_NOTICE([summary of build options:
Libxml2: ${have_libxml2} (CFLAGS='${LIBXML2_CFLAGS}' LIBS='${LIBXML2_LIBS}')
Libev: ${have_libev} (CFLAGS='${LIBEV_CFLAGS}' LIBS='${LIBEV_LIBS}')
Libc-ares: ${have_libcares} (CFLAGS='${LIBCARES_CFLAGS}' LIBS='${LIBCARES_LIBS}')
libngtcp2: ${have_libngtcp2} (CFLAGS='${LIBNGTCP2_CFLAGS}' LIBS='${LIBNGTCP2_LIBS}')
libngtcp2_crypto_openssl: ${have_libngtcp2_crypto_openssl} (CFLAGS='${LIBNGTCP2_CRYPTO_OPENSSL_CFLAGS}' LIBS='${LIBNGTCP2_CRYPTO_OPENSSL_LIBS}')
libnghttp3: ${have_libnghttp3} (CFLAGS='${LIBNGHTTP3_CFLAGS}' LIBS='${LIBNGHTTP3_LIBS}')
Libevent(SSL): ${have_libevent_openssl} (CFLAGS='${LIBEVENT_OPENSSL_CFLAGS}' LIBS='${LIBEVENT_OPENSSL_LIBS}')
Jansson: ${have_jansson} (CFLAGS='${JANSSON_CFLAGS}' LIBS='${JANSSON_LIBS}')
Jemalloc: ${have_jemalloc} (CFLAGS='${JEMALLOC_CFLAGS}' LIBS='${JEMALLOC_LIBS}')
@ -1032,4 +1168,5 @@ AC_MSG_NOTICE([summary of build options:
Examples: ${enable_examples}
Python bindings:${enable_python_bindings}
Threading: ${enable_threads}
HTTP/3 (EXPERIMENTAL): ${enable_http3}
])

View File

@ -0,0 +1,39 @@
FROM debian:10 as build
RUN apt-get update && \
apt-get install -y --no-install-recommends \
git g++ make binutils autoconf automake autotools-dev libtool \
pkg-config \
zlib1g-dev libev-dev libjemalloc-dev ruby-dev libc-ares-dev bison && \
git clone --depth 1 -b OpenSSL_1_1_1k+quic https://github.com/quictls/openssl && \
cd openssl && ./config enable-tls1_3 --openssldir=/etc/ssl && make -j$(nproc) && make install_sw && cd .. && rm -rf openssl && \
git clone --depth 1 https://github.com/ngtcp2/nghttp3 && \
cd nghttp3 && autoreconf -i && \
./configure --enable-lib-only && \
make -j$(nproc) && make install-strip && cd .. && rm -rf nghttp3 && \
git clone --depth 1 https://github.com/ngtcp2/ngtcp2 && \
cd ngtcp2 && autoreconf -i && \
./configure --enable-lib-only \
LIBTOOL_LDFLAGS="-static-libtool-libs" \
OPENSSL_LIBS="-l:libssl.a -l:libcrypto.a -ldl -lpthread" && \
make -j$(nproc) && make install-strip && cd .. && rm -rf ngtcp2 && \
git clone --depth 1 https://github.com/nghttp2/nghttp2.git && \
cd nghttp2 && \
git submodule update --init && autoreconf -i && \
./configure --disable-examples --disable-hpack-tools \
--disable-python-bindings --with-mruby --with-neverbleed \
--enable-http3 \
LIBTOOL_LDFLAGS="-static-libtool-libs" \
LIBS="-ldl -pthread" \
OPENSSL_LIBS="-l:libssl.a -l:libcrypto.a" \
LIBEV_LIBS="-l:libev.a" \
JEMALLOC_LIBS="-l:libjemalloc.a" \
LIBCARES_LIBS="-l:libcares.a" \
ZLIB_LIBS="-l:libz.a" && \
make -j$(nproc) install-strip
FROM gcr.io/distroless/cc-debian10
COPY --from=build /usr/local/bin/h2load /usr/local/bin/
ENTRYPOINT ["/usr/local/bin/h2load"]

View File

@ -15,6 +15,9 @@ include_directories(
${JEMALLOC_INCLUDE_DIRS}
${LIBXML2_INCLUDE_DIRS}
${LIBEV_INCLUDE_DIRS}
${LIBNGHTTP3_INCLUDE_DIRS}
${LIBNGTCP2_INCLUDE_DIRS}
${LIBNGTCP2_CRYPTO_OPENSSL_INCLUDE_DIRS}
${OPENSSL_INCLUDE_DIRS}
${LIBCARES_INCLUDE_DIRS}
${JANSSON_INCLUDE_DIRS}
@ -27,6 +30,9 @@ link_libraries(
${JEMALLOC_LIBRARIES}
${LIBXML2_LIBRARIES}
${LIBEV_LIBRARIES}
${LIBNGHTTP3_LIBRARIES}
${LIBNGTCP2_LIBRARIES}
${LIBNGTCP2_CRYPTO_OPENSSL_LIBRARIES}
${OPENSSL_LIBRARIES}
${LIBCARES_LIBRARIES}
${JANSSON_LIBRARIES}
@ -67,7 +73,13 @@ if(ENABLE_APP)
h2load_http2_session.cc
h2load_http1_session.cc
)
if(ENABLE_HTTP3)
list(APPEND H2LOAD_SOURCES
h2load_http3_session.cc
h2load_quic.cc
quic.cc
)
endif()
# Common libnhttpx sources (used for nghttpx and unit tests)
set(NGHTTPX_SRCS
@ -120,6 +132,16 @@ if(ENABLE_APP)
shrpx_mruby_module_response.cc
)
endif()
if(ENABLE_HTTP3)
list(APPEND NGHTTPX_SRCS
shrpx_quic.cc
shrpx_quic_listener.cc
shrpx_quic_connection_handler.cc
shrpx_http3_upstream.cc
http3.cc
quic.cc
)
endif()
add_library(nghttpx_static STATIC ${NGHTTPX_SRCS})
set_target_properties(nghttpx_static PROPERTIES ARCHIVE_OUTPUT_NAME nghttpx)

View File

@ -44,10 +44,14 @@ AM_CPPFLAGS = \
@JEMALLOC_CFLAGS@ \
@LIBXML2_CFLAGS@ \
@LIBEV_CFLAGS@ \
@LIBNGHTTP3_CFLAGS@ \
@LIBNGTCP2_CRYPTO_OPENSSL_CFLAGS@ \
@LIBNGTCP2_CFLAGS@ \
@OPENSSL_CFLAGS@ \
@LIBCARES_CFLAGS@ \
@JANSSON_CFLAGS@ \
@ZLIB_CFLAGS@ \
@EXTRA_DEFS@ \
@DEFS@
AM_LDFLAGS = @LIBTOOL_LDFLAGS@
@ -57,6 +61,9 @@ LDADD = $(top_builddir)/lib/libnghttp2.la \
@JEMALLOC_LIBS@ \
@LIBXML2_LIBS@ \
@LIBEV_LIBS@ \
@LIBNGHTTP3_LIBS@ \
@LIBNGTCP2_CRYPTO_OPENSSL_LIBS@ \
@LIBNGTCP2_LIBS@ \
@OPENSSL_LIBS@ \
@LIBCARES_LIBS@ \
@SYSTEMD_LIBS@ \
@ -99,6 +106,13 @@ h2load_SOURCES = util.cc util.h \
h2load_http2_session.cc h2load_http2_session.h \
h2load_http1_session.cc h2load_http1_session.h
if ENABLE_HTTP3
h2load_SOURCES += \
h2load_http3_session.cc h2load_http3_session.h \
h2load_quic.cc h2load_quic.h \
quic.cc quic.h
endif # ENABLE_HTTP3
NGHTTPX_SRCS = \
util.cc util.h http2.cc http2.h timegm.c timegm.h base64.h \
app_helper.cc app_helper.h \
@ -156,6 +170,16 @@ NGHTTPX_SRCS += \
shrpx_mruby_module_response.cc shrpx_mruby_module_response.h
endif # HAVE_MRUBY
if ENABLE_HTTP3
NGHTTPX_SRCS += \
shrpx_quic.cc shrpx_quic.h \
shrpx_quic_listener.cc shrpx_quic_listener.h \
shrpx_quic_connection_handler.cc shrpx_quic_connection_handler.h \
shrpx_http3_upstream.cc shrpx_http3_upstream.h \
http3.cc http3.h \
quic.cc quic.h
endif # ENABLE_HTTP3
noinst_LIBRARIES = libnghttpx.a
libnghttpx_a_SOURCES = ${NGHTTPX_SRCS}
libnghttpx_a_CPPFLAGS = ${AM_CPPFLAGS}

View File

@ -34,6 +34,8 @@
#ifdef HAVE_FCNTL_H
# include <fcntl.h>
#endif // HAVE_FCNTL_H
#include <sys/mman.h>
#include <netinet/udp.h>
#include <cstdio>
#include <cassert>
@ -48,10 +50,18 @@
#include <openssl/err.h>
#ifdef ENABLE_HTTP3
# include <ngtcp2/ngtcp2.h>
#endif // ENABLE_HTTP3
#include "url-parser/url_parser.h"
#include "h2load_http1_session.h"
#include "h2load_http2_session.h"
#ifdef ENABLE_HTTP3
# include "h2load_http3_session.h"
# include "h2load_quic.h"
#endif // ENABLE_HTTP3
#include "tls.h"
#include "http2.h"
#include "util.h"
@ -71,9 +81,22 @@ bool recorded(const std::chrono::steady_clock::time_point &t) {
}
} // namespace
namespace {
std::ofstream keylog_file;
void keylog_callback(const SSL *ssl, const char *line) {
keylog_file.write(line, strlen(line));
keylog_file.put('\n');
keylog_file.flush();
}
} // namespace
Config::Config()
: ciphers(tls::DEFAULT_CIPHER_LIST),
tls13_ciphers("TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_"
"CHACHA20_POLY1305_SHA256:TLS_AES_128_CCM_SHA256"),
groups("X25519:P-256:P-384:P-521"),
data_length(-1),
data(nullptr),
addrs(nullptr),
nreqs(1),
nclients(1),
@ -92,6 +115,7 @@ Config::Config()
encoder_header_table_size(4_k),
data_fd(-1),
log_fd(-1),
qlog_file_base(),
port(0),
default_port(0),
connect_to_port(0),
@ -99,7 +123,8 @@ Config::Config()
timing_script(false),
base_uri_unix(false),
unix_addr{},
rps(0.) {}
rps(0.),
no_udp_gso(false) {}
Config::~Config() {
if (addrs) {
@ -119,6 +144,14 @@ bool Config::is_rate_mode() const { return (this->rate != 0); }
bool Config::is_timing_based_mode() const { return (this->duration > 0); }
bool Config::has_base_uri() const { return (!this->base_uri.empty()); }
bool Config::rps_enabled() const { return this->rps > 0.0; }
bool Config::is_quic() const {
#ifdef ENABLE_HTTP3
return !npn_list.empty() &&
(npn_list[0] == NGHTTP3_ALPN_H3 || npn_list[0] == "\x5h3-29");
#else // !ENABLE_HTTP3
return false;
#endif // !ENABLE_HTTP3
}
Config config;
namespace {
@ -138,7 +171,9 @@ Stats::Stats(size_t req_todo, size_t nclients)
bytes_head(0),
bytes_head_decomp(0),
bytes_body(0),
status() {}
status(),
udp_dgram_recv(0),
udp_dgram_sent(0) {}
Stream::Stream() : req_stat{}, status_success(-1) {}
@ -195,8 +230,7 @@ void readcb(struct ev_loop *loop, ev_io *w, int revents) {
delete client;
return;
}
writecb(loop, &client->wev, revents);
// client->disconnect() and client->fail() may be called
client->signal_write();
}
} // namespace
@ -409,6 +443,9 @@ Client::Client(uint32_t id, Worker *worker, size_t req_todo)
cstat{},
worker(worker),
ssl(nullptr),
#ifdef ENABLE_HTTP3
quic{},
#endif // ENABLE_HTTP3
next_addr(config.addrs),
current_addr(nullptr),
reqidx(0),
@ -420,6 +457,7 @@ Client::Client(uint32_t id, Worker *worker, size_t req_todo)
req_done(0),
id(id),
fd(-1),
local_addr{},
new_connection_requested(false),
final(false),
rps_duration_started(0),
@ -449,11 +487,22 @@ Client::Client(uint32_t id, Worker *worker, size_t req_todo)
ev_timer_init(&rps_watcher, rps_cb, 0., 0.);
rps_watcher.data = this;
#ifdef ENABLE_HTTP3
ev_timer_init(&quic.pkt_timer, quic_pkt_timeout_cb, 0., 0.);
quic.pkt_timer.data = this;
#endif // ENABLE_HTTP3
}
Client::~Client() {
disconnect();
#ifdef ENABLE_HTTP3
if (config.is_quic()) {
quic_free();
}
#endif // ENABLE_HTTP3
if (ssl) {
SSL_free(ssl);
}
@ -466,26 +515,59 @@ int Client::do_read() { return readfn(*this); }
int Client::do_write() { return writefn(*this); }
int Client::make_socket(addrinfo *addr) {
fd = util::create_nonblock_socket(addr->ai_family);
if (fd == -1) {
return -1;
}
if (config.scheme == "https") {
if (!ssl) {
ssl = SSL_new(worker->ssl_ctx);
int rv;
if (config.is_quic()) {
#ifdef ENABLE_HTTP3
fd = util::create_nonblock_udp_socket(addr->ai_family);
if (fd == -1) {
return -1;
}
auto config = worker->config;
if (!util::numeric_host(config->host.c_str())) {
SSL_set_tlsext_host_name(ssl, config->host.c_str());
rv = util::bind_any_addr_udp(fd, addr->ai_family);
if (rv != 0) {
close(fd);
fd = -1;
return -1;
}
SSL_set_fd(ssl, fd);
SSL_set_connect_state(ssl);
socklen_t addrlen = sizeof(local_addr.su.storage);
rv = getsockname(fd, &local_addr.su.sa, &addrlen);
if (rv == -1) {
return -1;
}
local_addr.len = addrlen;
if (quic_init(&local_addr.su.sa, local_addr.len, addr->ai_addr,
addr->ai_addrlen) != 0) {
std::cerr << "quic_init failed" << std::endl;
return -1;
}
#endif // ENABLE_HTTP3
} else {
fd = util::create_nonblock_socket(addr->ai_family);
if (fd == -1) {
return -1;
}
if (config.scheme == "https") {
if (!ssl) {
ssl = SSL_new(worker->ssl_ctx);
}
SSL_set_fd(ssl, fd);
SSL_set_connect_state(ssl);
}
}
auto rv = ::connect(fd, addr->ai_addr, addr->ai_addrlen);
if (ssl && !util::numeric_host(config.host.c_str())) {
SSL_set_tlsext_host_name(ssl, config.host.c_str());
}
if (config.is_quic()) {
return 0;
}
rv = ::connect(fd, addr->ai_addr, addr->ai_addrlen);
if (rv != 0 && errno != EINPROGRESS) {
if (ssl) {
SSL_free(ssl);
@ -542,13 +624,22 @@ int Client::connect() {
current_addr = addr;
}
writefn = &Client::connected;
ev_io_set(&rev, fd, EV_READ);
ev_io_set(&wev, fd, EV_WRITE);
ev_io_start(worker->loop, &wev);
if (config.is_quic()) {
#ifdef ENABLE_HTTP3
ev_io_start(worker->loop, &rev);
readfn = &Client::read_quic;
writefn = &Client::write_quic;
#endif // ENABLE_HTTP3
} else {
writefn = &Client::connected;
}
return 0;
}
@ -603,6 +694,15 @@ void Client::fail() {
void Client::disconnect() {
record_client_end_time();
#ifdef ENABLE_HTTP3
if (config.is_quic()) {
quic_close_connection();
}
#endif // ENABLE_HTTP3
#ifdef ENABLE_HTTP3
ev_timer_stop(worker->loop, &quic.pkt_timer);
#endif // ENABLE_HTTP3
ev_timer_stop(worker->loop, &conn_inactivity_watcher);
ev_timer_stop(worker->loop, &conn_active_watcher);
ev_timer_stop(worker->loop, &rps_watcher);
@ -765,7 +865,14 @@ void Client::report_app_info() {
}
void Client::terminate_session() {
session->terminate();
#ifdef ENABLE_HTTP3
if (config.is_quic()) {
quic.close_requested = true;
}
#endif // ENABLE_HTTP3
if (session) {
session->terminate();
}
// http1 session needs writecb to tear down session.
signal_write();
}
@ -963,7 +1070,15 @@ int Client::connection_made() {
if (next_proto) {
auto proto = StringRef{next_proto, next_proto_len};
if (util::check_h2_is_selected(proto)) {
if (config.is_quic()) {
#ifdef ENABLE_HTTP3
assert(session);
if (!util::streq(StringRef{&NGHTTP3_ALPN_H3[1]}, proto) &&
!util::streq_l("h3-29", proto)) {
return -1;
}
#endif // ENABLE_HTTP3
} else if (util::check_h2_is_selected(proto)) {
session = std::make_unique<Http2Session>(this);
} else if (util::streq(NGHTTP2_H1_1, proto)) {
session = std::make_unique<Http1Session>(this);
@ -972,6 +1087,9 @@ int Client::connection_made() {
// Just assign next_proto to selected_proto anyway to show the
// negotiation result.
selected_proto = proto.str();
} else if (config.is_quic()) {
std::cerr << "QUIC requires ALPN negotiation" << std::endl;
return -1;
} else {
std::cout << "No protocol negotiated. Fallback behaviour may be activated"
<< std::endl;
@ -1285,6 +1403,46 @@ int Client::write_tls() {
return 0;
}
#ifdef ENABLE_HTTP3
int Client::write_udp(const sockaddr *addr, socklen_t addrlen,
const uint8_t *data, size_t datalen, size_t gso_size) {
iovec msg_iov;
msg_iov.iov_base = const_cast<uint8_t *>(data);
msg_iov.iov_len = datalen;
msghdr msg{};
msg.msg_name = const_cast<sockaddr *>(addr);
msg.msg_namelen = addrlen;
msg.msg_iov = &msg_iov;
msg.msg_iovlen = 1;
# ifdef UDP_SEGMENT
std::array<uint8_t, CMSG_SPACE(sizeof(uint16_t))> msg_ctrl{};
if (gso_size && datalen > gso_size) {
msg.msg_control = msg_ctrl.data();
msg.msg_controllen = msg_ctrl.size();
auto cm = CMSG_FIRSTHDR(&msg);
cm->cmsg_level = SOL_UDP;
cm->cmsg_type = UDP_SEGMENT;
cm->cmsg_len = CMSG_LEN(sizeof(uint16_t));
*(reinterpret_cast<uint16_t *>(CMSG_DATA(cm))) = gso_size;
}
# endif // UDP_SEGMENT
auto nwrite = sendmsg(fd, &msg, 0);
if (nwrite < 0) {
std::cerr << "sendto: errno=" << errno << std::endl;
} else {
++worker->stats.udp_dgram_sent;
}
ev_io_stop(worker->loop, &wev);
return 0;
}
#endif // ENABLE_HTTP3
void Client::record_request_time(RequestStat *req_stat) {
req_stat->request_time = std::chrono::steady_clock::now();
req_stat->request_wall_time = std::chrono::system_clock::now();
@ -1403,7 +1561,7 @@ Worker::~Worker() {
void Worker::stop_all_clients() {
for (auto client : clients) {
if (client && client->session) {
if (client) {
client->terminate_session();
}
}
@ -1938,6 +2096,7 @@ Options:
Default: 1
-w, --window-bits=<N>
Sets the stream level initial window size to (2**<N>)-1.
For QUIC, <N> is capped to 26 (roughly 64MiB).
Default: )"
<< config.window_bits << R"(
-W, --connection-window-bits=<N>
@ -1948,10 +2107,15 @@ Options:
-H, --header=<HEADER>
Add/Override a header to the requests.
--ciphers=<SUITE>
Set allowed cipher list. The format of the string is
described in OpenSSL ciphers(1).
Set allowed cipher list for TLSv1.2 or ealier. The
format of the string is described in OpenSSL ciphers(1).
Default: )"
<< config.ciphers << R"(
--tls13-ciphers=<SUITE>
Set allowed cipher list for TLSv1.3. The format of the
string is described in OpenSSL ciphers(1).
Default: )"
<< config.tls13_ciphers << R"(
-p, --no-tls-proto=<PROTOID>
Specify ALPN identifier of the protocol to be used when
accessing http URI without SSL/TLS.
@ -2065,11 +2229,24 @@ Options:
response time when using one worker thread, but may
appear slightly out of order with multiple threads due
to buffering. Status code is -1 for failed streams.
--qlog-file-base=<PATH>
Enable qlog output and specify base file name for qlogs.
Qlog is emitted for each connection.
For a given base name "base", each output file name
becomes "base.M.N.qlog" where M is worker ID and N is
client ID (e.g. "base.0.3.qlog").
Only effective in QUIC runs.
--connect-to=<HOST>[:<PORT>]
Host and port to connect instead of using the authority
in <URI>.
--rps=<N> Specify request per second for each client. --rps and
--timing-script-file are mutually exclusive.
--groups=<GROUPS>
Specify the supported groups.
Default: )"
<< config.groups << R"(
--no-udp-gso
Disable UDP GSO.
-v, --verbose
Output debug information.
--version Display version information and exit.
@ -2097,6 +2274,7 @@ int main(int argc, char **argv) {
std::string datafile;
std::string logfile;
std::string qlog_base;
bool nreqs_set_manually = false;
while (1) {
static int flag = 0;
@ -2130,6 +2308,10 @@ int main(int argc, char **argv) {
{"log-file", required_argument, &flag, 10},
{"connect-to", required_argument, &flag, 11},
{"rps", required_argument, &flag, 12},
{"groups", required_argument, &flag, 13},
{"tls13-ciphers", required_argument, &flag, 14},
{"no-udp-gso", no_argument, &flag, 15},
{"qlog-file-base", required_argument, &flag, 16},
{nullptr, 0, nullptr, 0}};
int option_index = 0;
auto c = getopt_long(argc, argv,
@ -2380,6 +2562,22 @@ int main(int argc, char **argv) {
config.rps = v;
break;
}
case 13:
// --groups
config.groups = optarg;
break;
case 14:
// --tls13-ciphers
config.tls13_ciphers = optarg;
break;
case 15:
// --no-udp-gso
config.no_udp_gso = true;
break;
case 16:
// --qlog-file-base
qlog_base = optarg;
break;
}
break;
default:
@ -2546,6 +2744,13 @@ int main(int argc, char **argv) {
exit(EXIT_FAILURE);
}
config.data_length = data_stat.st_size;
auto addr = mmap(nullptr, config.data_length, PROT_READ, MAP_SHARED,
config.data_fd, 0);
if (addr == MAP_FAILED) {
std::cerr << "-d: Could not mmap file " << datafile << std::endl;
exit(EXIT_FAILURE);
}
config.data = static_cast<uint8_t *>(addr);
}
if (!logfile.empty()) {
@ -2557,6 +2762,18 @@ int main(int argc, char **argv) {
}
}
if (!qlog_base.empty()) {
if (!config.is_quic()) {
std::cerr
<< "Warning: --qlog-file-base: only effective in quic, ignoring."
<< std::endl;
} else {
#ifdef ENABLE_HTTP3
config.qlog_file_base = qlog_base;
#endif // ENABLE_HTTP3
}
}
struct sigaction act {};
act.sa_handler = SIG_IGN;
sigaction(SIGPIPE, &act, nullptr);
@ -2576,9 +2793,14 @@ int main(int argc, char **argv) {
SSL_CTX_set_mode(ssl_ctx, SSL_MODE_AUTO_RETRY);
SSL_CTX_set_mode(ssl_ctx, SSL_MODE_RELEASE_BUFFERS);
if (nghttp2::tls::ssl_ctx_set_proto_versions(
ssl_ctx, nghttp2::tls::NGHTTP2_TLS_MIN_VERSION,
nghttp2::tls::NGHTTP2_TLS_MAX_VERSION) != 0) {
if (config.is_quic()) {
#ifdef ENABLE_HTTP3
SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_3_VERSION);
SSL_CTX_set_max_proto_version(ssl_ctx, TLS1_3_VERSION);
#endif // ENABLE_HTTP3
} else if (nghttp2::tls::ssl_ctx_set_proto_versions(
ssl_ctx, nghttp2::tls::NGHTTP2_TLS_MIN_VERSION,
nghttp2::tls::NGHTTP2_TLS_MAX_VERSION) != 0) {
std::cerr << "Could not set TLS versions" << std::endl;
exit(EXIT_FAILURE);
}
@ -2590,6 +2812,20 @@ int main(int argc, char **argv) {
exit(EXIT_FAILURE);
}
#if OPENSSL_1_1_1_API
if (SSL_CTX_set_ciphersuites(ssl_ctx, config.tls13_ciphers.c_str()) == 0) {
std::cerr << "SSL_CTX_set_ciphersuites with " << config.tls13_ciphers
<< " failed: " << ERR_error_string(ERR_get_error(), nullptr)
<< std::endl;
exit(EXIT_FAILURE);
}
#endif // OPENSSL_1_1_1_API
if (SSL_CTX_set1_groups_list(ssl_ctx, config.groups.c_str()) != 1) {
std::cerr << "SSL_CTX_set1_groups_list failed" << std::endl;
exit(EXIT_FAILURE);
}
#ifndef OPENSSL_NO_NEXTPROTONEG
SSL_CTX_set_next_proto_select_cb(ssl_ctx, client_select_next_proto_cb,
nullptr);
@ -2604,6 +2840,16 @@ int main(int argc, char **argv) {
SSL_CTX_set_alpn_protos(ssl_ctx, proto_list.data(), proto_list.size());
#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
#if OPENSSL_1_1_1_API
auto keylog_filename = getenv("SSLKEYLOGFILE");
if (keylog_filename) {
keylog_file.open(keylog_filename, std::ios_base::app);
if (keylog_file) {
SSL_CTX_set_keylog_callback(ssl_ctx, keylog_callback);
}
}
#endif // OPENSSL_1_1_1_API
std::string user_agent = "h2load nghttp2/" NGHTTP2_VERSION;
Headers shared_nva;
shared_nva.emplace_back(":scheme", config.scheme);
@ -2822,6 +3068,8 @@ int main(int argc, char **argv) {
stats.bytes_head += s.bytes_head;
stats.bytes_head_decomp += s.bytes_head_decomp;
stats.bytes_body += s.bytes_body;
stats.udp_dgram_recv += s.udp_dgram_recv;
stats.udp_dgram_sent += s.udp_dgram_sent;
for (size_t i = 0; i < stats.status.size(); ++i) {
stats.status[i] += s.status[i];
@ -2880,30 +3128,37 @@ traffic: )" << util::utos_funit(stats.bytes_total)
<< util::utos_funit(stats.bytes_head) << "B (" << stats.bytes_head
<< ") headers (space savings " << header_space_savings * 100
<< "%), " << util::utos_funit(stats.bytes_body) << "B ("
<< stats.bytes_body << R"() data
min max mean sd +/- sd
<< stats.bytes_body << R"() data)" << std::endl;
#ifdef ENABLE_HTTP3
if (config.is_quic()) {
std::cout << "UDP datagram: " << stats.udp_dgram_sent << " sent, "
<< stats.udp_dgram_recv << " received" << std::endl;
}
#endif // ENABLE_HTTP3
std::cout
<< R"( min max mean sd +/- sd
time for request: )"
<< std::setw(10) << util::format_duration(ts.request.min) << " "
<< std::setw(10) << util::format_duration(ts.request.max) << " "
<< std::setw(10) << util::format_duration(ts.request.mean) << " "
<< std::setw(10) << util::format_duration(ts.request.sd)
<< std::setw(9) << util::dtos(ts.request.within_sd) << "%"
<< "\ntime for connect: " << std::setw(10)
<< util::format_duration(ts.connect.min) << " " << std::setw(10)
<< util::format_duration(ts.connect.max) << " " << std::setw(10)
<< util::format_duration(ts.connect.mean) << " " << std::setw(10)
<< util::format_duration(ts.connect.sd) << std::setw(9)
<< util::dtos(ts.connect.within_sd) << "%"
<< "\ntime to 1st byte: " << std::setw(10)
<< util::format_duration(ts.ttfb.min) << " " << std::setw(10)
<< util::format_duration(ts.ttfb.max) << " " << std::setw(10)
<< util::format_duration(ts.ttfb.mean) << " " << std::setw(10)
<< util::format_duration(ts.ttfb.sd) << std::setw(9)
<< util::dtos(ts.ttfb.within_sd) << "%"
<< "\nreq/s : " << std::setw(10) << ts.rps.min << " "
<< std::setw(10) << ts.rps.max << " " << std::setw(10)
<< ts.rps.mean << " " << std::setw(10) << ts.rps.sd << std::setw(9)
<< util::dtos(ts.rps.within_sd) << "%" << std::endl;
<< std::setw(10) << util::format_duration(ts.request.min) << " "
<< std::setw(10) << util::format_duration(ts.request.max) << " "
<< std::setw(10) << util::format_duration(ts.request.mean) << " "
<< std::setw(10) << util::format_duration(ts.request.sd) << std::setw(9)
<< util::dtos(ts.request.within_sd) << "%"
<< "\ntime for connect: " << std::setw(10)
<< util::format_duration(ts.connect.min) << " " << std::setw(10)
<< util::format_duration(ts.connect.max) << " " << std::setw(10)
<< util::format_duration(ts.connect.mean) << " " << std::setw(10)
<< util::format_duration(ts.connect.sd) << std::setw(9)
<< util::dtos(ts.connect.within_sd) << "%"
<< "\ntime to 1st byte: " << std::setw(10)
<< util::format_duration(ts.ttfb.min) << " " << std::setw(10)
<< util::format_duration(ts.ttfb.max) << " " << std::setw(10)
<< util::format_duration(ts.ttfb.mean) << " " << std::setw(10)
<< util::format_duration(ts.ttfb.sd) << std::setw(9)
<< util::dtos(ts.ttfb.within_sd) << "%"
<< "\nreq/s : " << std::setw(10) << ts.rps.min << " "
<< std::setw(10) << ts.rps.max << " " << std::setw(10) << ts.rps.mean
<< " " << std::setw(10) << ts.rps.sd << std::setw(9)
<< util::dtos(ts.rps.within_sd) << "%" << std::endl;
SSL_CTX_free(ssl_ctx);

View File

@ -45,11 +45,19 @@
#include <nghttp2/nghttp2.h>
#ifdef ENABLE_HTTP3
# include <ngtcp2/ngtcp2.h>
# include <ngtcp2/ngtcp2_crypto.h>
#endif // ENABLE_HTTP3
#include <ev.h>
#include <openssl/ssl.h>
#include "http2.h"
#ifdef ENABLE_HTTP3
# include "quic.h"
#endif // ENABLE_HTTP3
#include "memchunk.h"
#include "template.h"
@ -72,8 +80,13 @@ struct Config {
std::string connect_to_host;
std::string ifile;
std::string ciphers;
std::string tls13_ciphers;
// supported groups (or curves).
std::string groups;
// length of upload data
int64_t data_length;
// memory mapped upload data
uint8_t *data;
addrinfo *addrs;
size_t nreqs;
size_t nclients;
@ -100,6 +113,8 @@ struct Config {
int data_fd;
// file descriptor to write per-request stats to.
int log_fd;
// base file name of qlog output files
std::string qlog_file_base;
uint16_t port;
uint16_t default_port;
uint16_t connect_to_port;
@ -116,6 +131,8 @@ struct Config {
std::vector<std::string> npn_list;
// The number of request per second for each client.
double rps;
// Disables GSO for UDP connections.
bool no_udp_gso;
Config();
~Config();
@ -124,6 +141,7 @@ struct Config {
bool is_timing_based_mode() const;
bool has_base_uri() const;
bool rps_enabled() const;
bool is_quic() const;
};
struct RequestStat {
@ -220,6 +238,10 @@ struct Stats {
std::vector<RequestStat> req_stats;
// The statistics per client
std::vector<ClientStat> client_stats;
// The number of UDP datagrams received.
size_t udp_dgram_recv;
// The number of UDP datagrams sent.
size_t udp_dgram_sent;
};
enum ClientState { CLIENT_IDLE, CLIENT_CONNECTED };
@ -309,6 +331,16 @@ struct Client {
std::function<int(Client &)> readfn, writefn;
Worker *worker;
SSL *ssl;
#ifdef ENABLE_HTTP3
struct {
ev_timer pkt_timer;
ngtcp2_conn *conn;
quic::Error last_error;
size_t max_pktlen;
bool close_requested;
FILE *qlog_file;
} quic;
#endif // ENABLE_HTTP3
ev_timer request_timeout_watcher;
addrinfo *next_addr;
// Address for the current address. When try_new_connection() is
@ -332,6 +364,7 @@ struct Client {
// The client id per worker
uint32_t id;
int fd;
Address local_addr;
ev_timer conn_active_watcher;
ev_timer conn_inactivity_watcher;
std::string selected_proto;
@ -419,6 +452,37 @@ struct Client {
void record_client_end_time();
void signal_write();
#ifdef ENABLE_HTTP3
// QUIC
int quic_init(const sockaddr *local_addr, socklen_t local_addrlen,
const sockaddr *remote_addr, socklen_t remote_addrlen);
void quic_free();
int read_quic();
int write_quic();
int write_udp(const sockaddr *addr, socklen_t addrlen, const uint8_t *data,
size_t datalen, size_t gso_size);
void quic_close_connection();
int quic_handshake_completed();
int quic_recv_stream_data(uint32_t flags, int64_t stream_id,
const uint8_t *data, size_t datalen);
int quic_acked_stream_data_offset(int64_t stream_id, size_t datalen);
int quic_stream_close(int64_t stream_id, uint64_t app_error_code);
int quic_stream_reset(int64_t stream_id, uint64_t app_error_code);
int quic_stream_stop_sending(int64_t stream_id, uint64_t app_error_code);
int quic_extend_max_local_streams();
int quic_on_key(ngtcp2_crypto_level level, const uint8_t *rx_secret,
const uint8_t *tx_secret, size_t secretlen);
void quic_set_tls_alert(uint8_t alert);
void quic_write_client_handshake(ngtcp2_crypto_level level,
const uint8_t *data, size_t datalen);
int quic_pkt_timeout();
void quic_restart_pkt_timer();
void quic_write_qlog(const void *data, size_t datalen);
#endif // ENABLE_HTTP3
};
} // namespace h2load

427
src/h2load_http3_session.cc Normal file
View File

@ -0,0 +1,427 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2019 nghttp2 contributors
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "h2load_http3_session.h"
#include <iostream>
#include <ngtcp2/ngtcp2.h>
#include "h2load.h"
namespace h2load {
Http3Session::Http3Session(Client *client)
: client_(client), conn_(nullptr), npending_request_(0), reqidx_(0) {}
Http3Session::~Http3Session() { nghttp3_conn_del(conn_); }
void Http3Session::on_connect() {}
int Http3Session::submit_request() {
if (npending_request_) {
++npending_request_;
return 0;
}
auto config = client_->worker->config;
reqidx_ = client_->reqidx;
if (++client_->reqidx == config->nva.size()) {
client_->reqidx = 0;
}
auto stream_id = submit_request_internal();
if (stream_id < 0) {
if (stream_id == NGTCP2_ERR_STREAM_ID_BLOCKED) {
++npending_request_;
return 0;
}
return -1;
}
return 0;
}
namespace {
nghttp3_ssize read_data(nghttp3_conn *conn, int64_t stream_id, nghttp3_vec *vec,
size_t veccnt, uint32_t *pflags, void *user_data,
void *stream_user_data) {
auto s = static_cast<Http3Session *>(user_data);
s->read_data(vec, veccnt, pflags);
return 1;
}
} // namespace
void Http3Session::read_data(nghttp3_vec *vec, size_t veccnt,
uint32_t *pflags) {
assert(veccnt > 0);
auto config = client_->worker->config;
vec[0].base = config->data;
vec[0].len = config->data_length;
*pflags |= NGHTTP3_DATA_FLAG_EOF;
}
int64_t Http3Session::submit_request_internal() {
int rv;
int64_t stream_id;
auto config = client_->worker->config;
auto &nva = config->nva[reqidx_];
rv = ngtcp2_conn_open_bidi_stream(client_->quic.conn, &stream_id, nullptr);
if (rv != 0) {
return rv;
}
nghttp3_data_reader dr{};
dr.read_data = h2load::read_data;
rv = nghttp3_conn_submit_request(
conn_, stream_id, reinterpret_cast<nghttp3_nv *>(nva.data()), nva.size(),
config->data_fd == -1 ? nullptr : &dr, nullptr);
if (rv != 0) {
return rv;
}
client_->on_request(stream_id);
auto req_stat = client_->get_req_stat(stream_id);
assert(req_stat);
client_->record_request_time(req_stat);
return stream_id;
}
int Http3Session::on_read(const uint8_t *data, size_t len) { return -1; }
int Http3Session::on_write() { return -1; }
void Http3Session::terminate() {}
size_t Http3Session::max_concurrent_streams() {
return (size_t)client_->worker->config->max_concurrent_streams;
}
namespace {
int stream_close(nghttp3_conn *conn, int64_t stream_id, uint64_t app_error_code,
void *user_data, void *stream_user_data) {
auto s = static_cast<Http3Session *>(user_data);
if (s->stream_close(stream_id, app_error_code) != 0) {
return NGHTTP3_ERR_CALLBACK_FAILURE;
}
return 0;
}
} // namespace
int Http3Session::stream_close(int64_t stream_id, uint64_t app_error_code) {
if (!ngtcp2_is_bidi_stream(stream_id)) {
assert(!ngtcp2_conn_is_local_stream(client_->quic.conn, stream_id));
ngtcp2_conn_extend_max_streams_uni(client_->quic.conn, 1);
}
client_->on_stream_close(stream_id, app_error_code == NGHTTP3_H3_NO_ERROR);
return 0;
}
namespace {
int recv_data(nghttp3_conn *conn, int64_t stream_id, const uint8_t *data,
size_t datalen, void *user_data, void *stream_user_data) {
auto s = static_cast<Http3Session *>(user_data);
s->recv_data(stream_id, data, datalen);
return 0;
}
} // namespace
void Http3Session::recv_data(int64_t stream_id, const uint8_t *data,
size_t datalen) {
client_->record_ttfb();
client_->worker->stats.bytes_body += datalen;
consume(stream_id, datalen);
}
namespace {
int deferred_consume(nghttp3_conn *conn, int64_t stream_id, size_t nconsumed,
void *user_data, void *stream_user_data) {
auto s = static_cast<Http3Session *>(user_data);
s->consume(stream_id, nconsumed);
return 0;
}
} // namespace
void Http3Session::consume(int64_t stream_id, size_t nconsumed) {
ngtcp2_conn_extend_max_stream_offset(client_->quic.conn, stream_id,
nconsumed);
ngtcp2_conn_extend_max_offset(client_->quic.conn, nconsumed);
}
namespace {
int begin_headers(nghttp3_conn *conn, int64_t stream_id, void *user_data,
void *stream_user_data) {
auto s = static_cast<Http3Session *>(user_data);
s->begin_headers(stream_id);
return 0;
}
} // namespace
void Http3Session::begin_headers(int64_t stream_id) {
auto payloadlen = nghttp3_conn_get_frame_payload_left(conn_, stream_id);
assert(payloadlen > 0);
client_->worker->stats.bytes_head += payloadlen;
}
namespace {
int recv_header(nghttp3_conn *conn, int64_t stream_id, int32_t token,
nghttp3_rcbuf *name, nghttp3_rcbuf *value, uint8_t flags,
void *user_data, void *stream_user_data) {
auto s = static_cast<Http3Session *>(user_data);
auto k = nghttp3_rcbuf_get_buf(name);
auto v = nghttp3_rcbuf_get_buf(value);
s->recv_header(stream_id, &k, &v);
return 0;
}
} // namespace
void Http3Session::recv_header(int64_t stream_id, const nghttp3_vec *name,
const nghttp3_vec *value) {
client_->on_header(stream_id, name->base, name->len, value->base, value->len);
client_->worker->stats.bytes_head_decomp += name->len + value->len;
}
namespace {
int send_stop_sending(nghttp3_conn *conn, int64_t stream_id,
uint64_t app_error_code, void *user_data,
void *stream_user_data) {
auto s = static_cast<Http3Session *>(user_data);
if (s->send_stop_sending(stream_id, app_error_code) != 0) {
return NGHTTP3_ERR_CALLBACK_FAILURE;
}
return 0;
}
} // namespace
int Http3Session::send_stop_sending(int64_t stream_id,
uint64_t app_error_code) {
auto rv = ngtcp2_conn_shutdown_stream_read(client_->quic.conn, stream_id,
app_error_code);
if (rv != 0) {
std::cerr << "ngtcp2_conn_shutdown_stream_read: " << ngtcp2_strerror(rv)
<< std::endl;
return -1;
}
return 0;
}
int Http3Session::close_stream(int64_t stream_id, uint64_t app_error_code) {
auto rv = nghttp3_conn_close_stream(conn_, stream_id, app_error_code);
switch (rv) {
case 0:
return 0;
case NGHTTP3_ERR_STREAM_NOT_FOUND:
if (!ngtcp2_is_bidi_stream(stream_id)) {
assert(!ngtcp2_conn_is_local_stream(client_->quic.conn, stream_id));
ngtcp2_conn_extend_max_streams_uni(client_->quic.conn, 1);
}
return 0;
default:
return -1;
}
}
int Http3Session::shutdown_stream_read(int64_t stream_id) {
auto rv = nghttp3_conn_shutdown_stream_read(conn_, stream_id);
if (rv != 0) {
return -1;
}
return 0;
}
int Http3Session::extend_max_local_streams() {
auto config = client_->worker->config;
for (; npending_request_; --npending_request_) {
auto stream_id = submit_request_internal();
if (stream_id < 0) {
if (stream_id == NGTCP2_ERR_STREAM_ID_BLOCKED) {
return 0;
}
return -1;
}
if (++reqidx_ == config->nva.size()) {
reqidx_ = 0;
}
}
return 0;
}
int Http3Session::init_conn() {
int rv;
assert(conn_ == nullptr);
if (ngtcp2_conn_get_max_local_streams_uni(client_->quic.conn) < 3) {
return -1;
}
nghttp3_callbacks callbacks{
nullptr, // acked_stream_data
h2load::stream_close,
h2load::recv_data,
h2load::deferred_consume,
h2load::begin_headers,
h2load::recv_header,
nullptr, // end_headers
nullptr, // begin_trailers
h2load::recv_header,
nullptr, // end_trailers
h2load::send_stop_sending,
};
auto config = client_->worker->config;
nghttp3_settings settings;
nghttp3_settings_default(&settings);
settings.qpack_max_table_capacity = config->header_table_size;
settings.qpack_blocked_streams = 100;
auto mem = nghttp3_mem_default();
rv = nghttp3_conn_client_new(&conn_, &callbacks, &settings, mem, this);
if (rv != 0) {
std::cerr << "nghttp3_conn_client_new: " << nghttp3_strerror(rv)
<< std::endl;
return -1;
}
int64_t ctrl_stream_id;
rv = ngtcp2_conn_open_uni_stream(client_->quic.conn, &ctrl_stream_id, NULL);
if (rv != 0) {
std::cerr << "ngtcp2_conn_open_uni_stream: " << ngtcp2_strerror(rv)
<< std::endl;
return -1;
}
rv = nghttp3_conn_bind_control_stream(conn_, ctrl_stream_id);
if (rv != 0) {
std::cerr << "nghttp3_conn_bind_control_stream: " << nghttp3_strerror(rv)
<< std::endl;
return -1;
}
int64_t qpack_enc_stream_id, qpack_dec_stream_id;
rv = ngtcp2_conn_open_uni_stream(client_->quic.conn, &qpack_enc_stream_id,
NULL);
if (rv != 0) {
std::cerr << "ngtcp2_conn_open_uni_stream: " << ngtcp2_strerror(rv)
<< std::endl;
return -1;
}
rv = ngtcp2_conn_open_uni_stream(client_->quic.conn, &qpack_dec_stream_id,
NULL);
if (rv != 0) {
std::cerr << "ngtcp2_conn_open_uni_stream: " << ngtcp2_strerror(rv)
<< std::endl;
return -1;
}
rv = nghttp3_conn_bind_qpack_streams(conn_, qpack_enc_stream_id,
qpack_dec_stream_id);
if (rv != 0) {
std::cerr << "nghttp3_conn_bind_qpack_streams: " << nghttp3_strerror(rv)
<< std::endl;
return -1;
}
return 0;
}
ssize_t Http3Session::read_stream(uint32_t flags, int64_t stream_id,
const uint8_t *data, size_t datalen) {
auto nconsumed = nghttp3_conn_read_stream(
conn_, stream_id, data, datalen, flags & NGTCP2_STREAM_DATA_FLAG_FIN);
if (nconsumed < 0) {
std::cerr << "nghttp3_conn_read_stream: " << nghttp3_strerror(nconsumed)
<< std::endl;
client_->quic.last_error = quic::err_application(nconsumed);
return -1;
}
return nconsumed;
}
ssize_t Http3Session::write_stream(int64_t &stream_id, int &fin,
nghttp3_vec *vec, size_t veccnt) {
auto sveccnt =
nghttp3_conn_writev_stream(conn_, &stream_id, &fin, vec, veccnt);
if (sveccnt < 0) {
client_->quic.last_error = quic::err_application(sveccnt);
return -1;
}
return sveccnt;
}
int Http3Session::block_stream(int64_t stream_id) {
auto rv = nghttp3_conn_block_stream(conn_, stream_id);
if (rv != 0) {
client_->quic.last_error = quic::err_application(rv);
return -1;
}
return 0;
}
int Http3Session::shutdown_stream_write(int64_t stream_id) {
auto rv = nghttp3_conn_shutdown_stream_write(conn_, stream_id);
if (rv != 0) {
client_->quic.last_error = quic::err_application(rv);
return -1;
}
return 0;
}
int Http3Session::add_write_offset(int64_t stream_id, size_t ndatalen) {
auto rv = nghttp3_conn_add_write_offset(conn_, stream_id, ndatalen);
if (rv != 0) {
client_->quic.last_error = quic::err_application(rv);
return -1;
}
return 0;
}
int Http3Session::add_ack_offset(int64_t stream_id, size_t datalen) {
auto rv = nghttp3_conn_add_ack_offset(conn_, stream_id, datalen);
if (rv != 0) {
client_->quic.last_error = quic::err_application(rv);
return -1;
}
return 0;
}
} // namespace h2load

View File

@ -0,0 +1,81 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2019 nghttp2 contributors
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef H2LOAD_HTTP3_SESSION_H
#define H2LOAD_HTTP3_SESSION_H
#include "h2load_session.h"
#include <nghttp3/nghttp3.h>
namespace h2load {
struct Client;
class Http3Session : public Session {
public:
Http3Session(Client *client);
virtual ~Http3Session();
virtual void on_connect();
virtual int submit_request();
virtual int on_read(const uint8_t *data, size_t len);
virtual int on_write();
virtual void terminate();
virtual size_t max_concurrent_streams();
int init_conn();
int stream_close(int64_t stream_id, uint64_t app_error_code);
void recv_data(int64_t stream_id, const uint8_t *data, size_t datalen);
void consume(int64_t stream_id, size_t nconsumed);
void begin_headers(int64_t stream_id);
void recv_header(int64_t stream_id, const nghttp3_vec *name,
const nghttp3_vec *value);
int send_stop_sending(int64_t stream_id, uint64_t app_error_code);
int close_stream(int64_t stream_id, uint64_t app_error_code);
int shutdown_stream_read(int64_t stream_id);
int extend_max_local_streams();
int64_t submit_request_internal();
ssize_t read_stream(uint32_t flags, int64_t stream_id, const uint8_t *data,
size_t datalen);
ssize_t write_stream(int64_t &stream_id, int &fin, nghttp3_vec *vec,
size_t veccnt);
int block_stream(int64_t stream_id);
int shutdown_stream_write(int64_t stream_id);
int add_write_offset(int64_t stream_id, size_t ndatalen);
int add_ack_offset(int64_t stream_id, size_t datalen);
void read_data(nghttp3_vec *vec, size_t veccnt, uint32_t *pflags);
private:
Client *client_;
nghttp3_conn *conn_;
size_t npending_request_;
size_t reqidx_;
};
} // namespace h2load
#endif // H2LOAD_HTTP3_SESSION_H

679
src/h2load_quic.cc Normal file
View File

@ -0,0 +1,679 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2019 nghttp2 contributors
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "h2load_quic.h"
#include <netinet/udp.h>
#include <iostream>
#include <ngtcp2/ngtcp2_crypto_openssl.h>
#include <openssl/err.h>
#include "h2load_http3_session.h"
namespace h2load {
namespace {
auto randgen = util::make_mt19937();
} // namespace
namespace {
int handshake_completed(ngtcp2_conn *conn, void *user_data) {
auto c = static_cast<Client *>(user_data);
if (c->quic_handshake_completed() != 0) {
return NGTCP2_ERR_CALLBACK_FAILURE;
}
return 0;
}
} // namespace
int Client::quic_handshake_completed() { return connection_made(); }
namespace {
int recv_stream_data(ngtcp2_conn *conn, uint32_t flags, int64_t stream_id,
uint64_t offset, const uint8_t *data, size_t datalen,
void *user_data, void *stream_user_data) {
auto c = static_cast<Client *>(user_data);
if (c->quic_recv_stream_data(flags, stream_id, data, datalen) != 0) {
// TODO Better to do this gracefully rather than
// NGTCP2_ERR_CALLBACK_FAILURE. Perhaps, call
// ngtcp2_conn_write_application_close() ?
return NGTCP2_ERR_CALLBACK_FAILURE;
}
return 0;
}
} // namespace
int Client::quic_recv_stream_data(uint32_t flags, int64_t stream_id,
const uint8_t *data, size_t datalen) {
if (worker->current_phase == Phase::MAIN_DURATION) {
worker->stats.bytes_total += datalen;
}
auto s = static_cast<Http3Session *>(session.get());
auto nconsumed = s->read_stream(flags, stream_id, data, datalen);
if (nconsumed == -1) {
return -1;
}
ngtcp2_conn_extend_max_stream_offset(quic.conn, stream_id, nconsumed);
ngtcp2_conn_extend_max_offset(quic.conn, nconsumed);
return 0;
}
namespace {
int acked_stream_data_offset(ngtcp2_conn *conn, int64_t stream_id,
uint64_t offset, uint64_t datalen, void *user_data,
void *stream_user_data) {
auto c = static_cast<Client *>(user_data);
if (c->quic_acked_stream_data_offset(stream_id, datalen) != 0) {
return NGTCP2_ERR_CALLBACK_FAILURE;
}
return 0;
}
} // namespace
int Client::quic_acked_stream_data_offset(int64_t stream_id, size_t datalen) {
auto s = static_cast<Http3Session *>(session.get());
if (s->add_ack_offset(stream_id, datalen) != 0) {
return -1;
}
return 0;
}
namespace {
int stream_close(ngtcp2_conn *conn, int64_t stream_id, uint64_t app_error_code,
void *user_data, void *stream_user_data) {
auto c = static_cast<Client *>(user_data);
if (c->quic_stream_close(stream_id, app_error_code) != 0) {
return -1;
}
return 0;
}
} // namespace
int Client::quic_stream_close(int64_t stream_id, uint64_t app_error_code) {
auto s = static_cast<Http3Session *>(session.get());
if (s->close_stream(stream_id, app_error_code == 0 ? NGHTTP3_H3_NO_ERROR
: app_error_code) != 0) {
return -1;
}
return 0;
}
namespace {
int stream_reset(ngtcp2_conn *conn, int64_t stream_id, uint64_t final_size,
uint64_t app_error_code, void *user_data,
void *stream_user_data) {
auto c = static_cast<Client *>(user_data);
if (c->quic_stream_reset(stream_id, app_error_code) != 0) {
return -1;
}
return 0;
}
} // namespace
int Client::quic_stream_reset(int64_t stream_id, uint64_t app_error_code) {
auto s = static_cast<Http3Session *>(session.get());
if (s->shutdown_stream_read(stream_id) != 0) {
return -1;
}
return 0;
}
namespace {
int stream_stop_sending(ngtcp2_conn *conn, int64_t stream_id,
uint64_t app_error_code, void *user_data,
void *stream_user_data) {
auto c = static_cast<Client *>(user_data);
if (c->quic_stream_stop_sending(stream_id, app_error_code) != 0) {
return -1;
}
return 0;
}
} // namespace
int Client::quic_stream_stop_sending(int64_t stream_id,
uint64_t app_error_code) {
auto s = static_cast<Http3Session *>(session.get());
if (s->shutdown_stream_read(stream_id) != 0) {
return -1;
}
return 0;
}
namespace {
int extend_max_local_streams_bidi(ngtcp2_conn *conn, uint64_t max_streams,
void *user_data) {
auto c = static_cast<Client *>(user_data);
if (c->quic_extend_max_local_streams() != 0) {
return NGTCP2_ERR_CALLBACK_FAILURE;
}
return 0;
}
} // namespace
int Client::quic_extend_max_local_streams() {
auto s = static_cast<Http3Session *>(session.get());
if (s->extend_max_local_streams() != 0) {
return NGTCP2_ERR_CALLBACK_FAILURE;
}
return 0;
}
namespace {
int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token,
size_t cidlen, void *user_data) {
auto dis = std::uniform_int_distribution<uint8_t>(
0, std::numeric_limits<uint8_t>::max());
auto f = [&dis]() { return dis(randgen); };
std::generate_n(cid->data, cidlen, f);
cid->datalen = cidlen;
std::generate_n(token, NGTCP2_STATELESS_RESET_TOKENLEN, f);
return 0;
}
} // namespace
namespace {
void debug_log_printf(void *user_data, const char *fmt, ...) {
va_list ap;
va_start(ap, fmt);
vfprintf(stderr, fmt, ap);
va_end(ap);
fprintf(stderr, "\n");
}
} // namespace
namespace {
void generate_cid(ngtcp2_cid &dest) {
auto dis = std::uniform_int_distribution<uint8_t>(
0, std::numeric_limits<uint8_t>::max());
dest.datalen = 8;
std::generate_n(dest.data, dest.datalen, [&dis]() { return dis(randgen); });
}
} // namespace
namespace {
ngtcp2_tstamp timestamp(struct ev_loop *loop) {
return ev_now(loop) * NGTCP2_SECONDS;
}
} // namespace
namespace {
int set_encryption_secrets(SSL *ssl, OSSL_ENCRYPTION_LEVEL ossl_level,
const uint8_t *rx_secret, const uint8_t *tx_secret,
size_t secret_len) {
auto c = static_cast<Client *>(SSL_get_app_data(ssl));
if (c->quic_on_key(
ngtcp2_crypto_openssl_from_ossl_encryption_level(ossl_level),
rx_secret, tx_secret, secret_len) != 0) {
return 0;
}
return 1;
}
} // namespace
namespace {
int add_handshake_data(SSL *ssl, OSSL_ENCRYPTION_LEVEL ossl_level,
const uint8_t *data, size_t len) {
auto c = static_cast<Client *>(SSL_get_app_data(ssl));
c->quic_write_client_handshake(
ngtcp2_crypto_openssl_from_ossl_encryption_level(ossl_level), data, len);
return 1;
}
} // namespace
namespace {
int flush_flight(SSL *ssl) { return 1; }
} // namespace
namespace {
int send_alert(SSL *ssl, enum ssl_encryption_level_t level, uint8_t alert) {
auto c = static_cast<Client *>(SSL_get_app_data(ssl));
c->quic_set_tls_alert(alert);
return 1;
}
} // namespace
namespace {
auto quic_method = SSL_QUIC_METHOD{
set_encryption_secrets,
add_handshake_data,
flush_flight,
send_alert,
};
} // namespace
// qlog write callback -- excerpted from ngtcp2/examples/client_base.cc
namespace {
void qlog_write_cb(void *user_data, uint32_t flags, const void *data,
size_t datalen) {
auto c = static_cast<Client *>(user_data);
c->quic_write_qlog(data, datalen);
}
} // namespace
void Client::quic_write_qlog(const void *data, size_t datalen) {
assert(quic.qlog_file != nullptr);
fwrite(data, 1, datalen, quic.qlog_file);
}
int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen,
const sockaddr *remote_addr, socklen_t remote_addrlen) {
int rv;
if (!ssl) {
ssl = SSL_new(worker->ssl_ctx);
SSL_set_app_data(ssl, this);
SSL_set_connect_state(ssl);
SSL_set_quic_method(ssl, &quic_method);
}
switch (remote_addr->sa_family) {
case AF_INET:
quic.max_pktlen = NGTCP2_MAX_PKTLEN_IPV4;
break;
case AF_INET6:
quic.max_pktlen = NGTCP2_MAX_PKTLEN_IPV6;
break;
default:
return -1;
}
auto callbacks = ngtcp2_callbacks{
ngtcp2_crypto_client_initial_cb,
nullptr, // recv_client_initial
ngtcp2_crypto_recv_crypto_data_cb,
h2load::handshake_completed,
nullptr, // recv_version_negotiation
ngtcp2_crypto_encrypt_cb,
ngtcp2_crypto_decrypt_cb,
ngtcp2_crypto_hp_mask_cb,
h2load::recv_stream_data,
h2load::acked_stream_data_offset,
nullptr, // stream_open
h2load::stream_close,
nullptr, // recv_stateless_reset
ngtcp2_crypto_recv_retry_cb,
h2load::extend_max_local_streams_bidi,
nullptr, // extend_max_local_streams_uni
nullptr, // rand
get_new_connection_id,
nullptr, // remove_connection_id
ngtcp2_crypto_update_key_cb,
nullptr, // path_validation
nullptr, // select_preferred_addr
h2load::stream_reset,
nullptr, // extend_max_remote_streams_bidi
nullptr, // extend_max_remote_streams_uni
nullptr, // extend_max_stream_data
nullptr, // dcid_status
nullptr, // handshake_confirmed
nullptr, // recv_new_token
ngtcp2_crypto_delete_crypto_aead_ctx_cb,
ngtcp2_crypto_delete_crypto_cipher_ctx_cb,
nullptr, // recv_datagram
nullptr, // ack_datagram
nullptr, // lost_datagram
nullptr, // get_path_challenge_data
h2load::stream_stop_sending,
};
ngtcp2_cid scid, dcid;
generate_cid(scid);
generate_cid(dcid);
auto config = worker->config;
ngtcp2_settings settings;
ngtcp2_settings_default(&settings);
if (config->verbose) {
settings.log_printf = debug_log_printf;
}
settings.initial_ts = timestamp(worker->loop);
if (!config->qlog_file_base.empty()) {
assert(quic.qlog_file == nullptr);
auto path = config->qlog_file_base;
path += '.';
path += util::utos(worker->id);
path += '.';
path += util::utos(id);
path += ".qlog";
quic.qlog_file = fopen(path.c_str(), "w");
if (quic.qlog_file == nullptr) {
std::cerr << "Failed to open a qlog file: " << path << std::endl;
return -1;
}
settings.qlog.write = qlog_write_cb;
}
ngtcp2_transport_params params;
ngtcp2_transport_params_default(&params);
auto max_stream_data =
std::min((1 << 26) - 1, (1 << config->window_bits) - 1);
params.initial_max_stream_data_bidi_local = max_stream_data;
params.initial_max_stream_data_uni = max_stream_data;
params.initial_max_data = (1 << config->connection_window_bits) - 1;
params.initial_max_streams_bidi = 0;
params.initial_max_streams_uni = 100;
params.max_idle_timeout = 30 * NGTCP2_SECONDS;
auto path = ngtcp2_path{
{local_addrlen, const_cast<sockaddr *>(local_addr)},
{remote_addrlen, const_cast<sockaddr *>(remote_addr)},
};
assert(config->npn_list.size());
uint32_t quic_version;
if (config->npn_list[0] == NGHTTP3_ALPN_H3) {
quic_version = NGTCP2_PROTO_VER_V1;
} else {
quic_version = NGTCP2_PROTO_VER_MIN;
}
rv = ngtcp2_conn_client_new(&quic.conn, &dcid, &scid, &path, quic_version,
&callbacks, &settings, &params, nullptr, this);
if (rv != 0) {
return -1;
}
ngtcp2_conn_set_tls_native_handle(quic.conn, ssl);
return 0;
}
void Client::quic_free() {
ngtcp2_conn_del(quic.conn);
if (quic.qlog_file != nullptr) {
fclose(quic.qlog_file);
quic.qlog_file = nullptr;
}
}
void Client::quic_close_connection() {
if (!quic.conn) {
return;
}
std::array<uint8_t, 1500> buf;
ngtcp2_ssize nwrite;
ngtcp2_path_storage ps;
ngtcp2_path_storage_zero(&ps);
switch (quic.last_error.type) {
case quic::ErrorType::TransportVersionNegotiation:
return;
case quic::ErrorType::Transport:
nwrite = ngtcp2_conn_write_connection_close(
quic.conn, &ps.path, nullptr, buf.data(), quic.max_pktlen,
quic.last_error.code, timestamp(worker->loop));
break;
case quic::ErrorType::Application:
nwrite = ngtcp2_conn_write_application_close(
quic.conn, &ps.path, nullptr, buf.data(), quic.max_pktlen,
quic.last_error.code, timestamp(worker->loop));
break;
default:
assert(0);
}
if (nwrite < 0) {
return;
}
write_udp(reinterpret_cast<sockaddr *>(ps.path.remote.addr),
ps.path.remote.addrlen, buf.data(), nwrite, 0);
}
int Client::quic_on_key(ngtcp2_crypto_level level, const uint8_t *rx_secret,
const uint8_t *tx_secret, size_t secretlen) {
if (ngtcp2_crypto_derive_and_install_rx_key(quic.conn, nullptr, nullptr,
nullptr, level, rx_secret,
secretlen) != 0) {
std::cerr << "ngtcp2_crypto_derive_and_install_rx_key() failed"
<< std::endl;
return -1;
}
if (ngtcp2_crypto_derive_and_install_tx_key(quic.conn, nullptr, nullptr,
nullptr, level, tx_secret,
secretlen) != 0) {
std::cerr << "ngtcp2_crypto_derive_and_install_tx_key() failed"
<< std::endl;
return -1;
}
if (level == NGTCP2_CRYPTO_LEVEL_APPLICATION) {
auto s = std::make_unique<Http3Session>(this);
if (s->init_conn() == -1) {
return -1;
}
session = std::move(s);
}
return 0;
}
void Client::quic_set_tls_alert(uint8_t alert) {
quic.last_error = quic::err_transport_tls(alert);
}
void Client::quic_write_client_handshake(ngtcp2_crypto_level level,
const uint8_t *data, size_t datalen) {
assert(level < 2);
ngtcp2_conn_submit_crypto_data(quic.conn, level, data, datalen);
}
void quic_pkt_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents) {
auto c = static_cast<Client *>(w->data);
if (c->quic_pkt_timeout() != 0) {
c->fail();
c->worker->free_client(c);
delete c;
return;
}
}
int Client::quic_pkt_timeout() {
int rv;
auto now = timestamp(worker->loop);
rv = ngtcp2_conn_handle_expiry(quic.conn, now);
if (rv != 0) {
quic.last_error = quic::err_transport(NGTCP2_ERR_INTERNAL);
return -1;
}
return write_quic();
}
void Client::quic_restart_pkt_timer() {
auto expiry = ngtcp2_conn_get_expiry(quic.conn);
auto now = timestamp(worker->loop);
auto t = expiry > now ? static_cast<ev_tstamp>(expiry - now) / NGTCP2_SECONDS
: 1e-9;
quic.pkt_timer.repeat = t;
ev_timer_again(worker->loop, &quic.pkt_timer);
}
int Client::read_quic() {
std::array<uint8_t, 65536> buf;
sockaddr_union su;
socklen_t addrlen = sizeof(su);
int rv;
size_t pktcnt = 0;
ngtcp2_pkt_info pi{};
for (;;) {
auto nread =
recvfrom(fd, buf.data(), buf.size(), MSG_DONTWAIT, &su.sa, &addrlen);
if (nread == -1) {
return 0;
}
assert(quic.conn);
++worker->stats.udp_dgram_recv;
auto path = ngtcp2_path{
{local_addr.len, &local_addr.su.sa},
{addrlen, &su.sa},
};
rv = ngtcp2_conn_read_pkt(quic.conn, &path, &pi, buf.data(), nread,
timestamp(worker->loop));
if (rv != 0) {
std::cerr << "ngtcp2_conn_read_pkt: " << ngtcp2_strerror(rv) << std::endl;
return -1;
}
if (++pktcnt == 100) {
break;
}
}
return 0;
}
int Client::write_quic() {
ev_io_stop(worker->loop, &wev);
if (quic.close_requested) {
return -1;
}
std::array<nghttp3_vec, 16> vec;
size_t pktcnt = 0;
size_t max_pktcnt =
#ifdef UDP_SEGMENT
worker->config->no_udp_gso
? 1
: std::min(static_cast<size_t>(10),
static_cast<size_t>(64_k / quic.max_pktlen));
#else // !UDP_SEGMENT
1;
#endif // !UDP_SEGMENT
std::array<uint8_t, 64_k> buf;
uint8_t *bufpos = buf.data();
ngtcp2_path_storage ps;
ngtcp2_path_storage_zero(&ps);
auto s = static_cast<Http3Session *>(session.get());
for (;;) {
int64_t stream_id = -1;
int fin = 0;
ssize_t sveccnt = 0;
if (session && ngtcp2_conn_get_max_data_left(quic.conn)) {
sveccnt = s->write_stream(stream_id, fin, vec.data(), vec.size());
if (sveccnt == -1) {
return -1;
}
}
ngtcp2_ssize ndatalen;
auto v = vec.data();
auto vcnt = static_cast<size_t>(sveccnt);
uint32_t flags = NGTCP2_WRITE_STREAM_FLAG_MORE;
if (fin) {
flags |= NGTCP2_WRITE_STREAM_FLAG_FIN;
}
auto nwrite = ngtcp2_conn_writev_stream(
quic.conn, &ps.path, nullptr, bufpos, quic.max_pktlen, &ndatalen, flags,
stream_id, reinterpret_cast<const ngtcp2_vec *>(v), vcnt,
timestamp(worker->loop));
if (nwrite < 0) {
switch (nwrite) {
case NGTCP2_ERR_STREAM_DATA_BLOCKED:
assert(ndatalen == -1);
if (s->block_stream(stream_id) != 0) {
return -1;
}
continue;
case NGTCP2_ERR_STREAM_SHUT_WR:
assert(ndatalen == -1);
if (s->shutdown_stream_write(stream_id) != 0) {
return -1;
}
continue;
case NGTCP2_ERR_WRITE_MORE:
assert(ndatalen >= 0);
if (s->add_write_offset(stream_id, ndatalen) != 0) {
return -1;
}
continue;
}
quic.last_error = quic::err_transport(nwrite);
return -1;
} else if (ndatalen >= 0 && s->add_write_offset(stream_id, ndatalen) != 0) {
return -1;
}
quic_restart_pkt_timer();
if (nwrite == 0) {
if (bufpos - buf.data()) {
write_udp(ps.path.remote.addr, ps.path.remote.addrlen, buf.data(),
bufpos - buf.data(), quic.max_pktlen);
}
return 0;
}
bufpos += nwrite;
// Assume that the path does not change.
if (++pktcnt == max_pktcnt ||
static_cast<size_t>(nwrite) < quic.max_pktlen) {
write_udp(ps.path.remote.addr, ps.path.remote.addrlen, buf.data(),
bufpos - buf.data(), quic.max_pktlen);
signal_write();
return 0;
}
}
}
} // namespace h2load

38
src/h2load_quic.h Normal file
View File

@ -0,0 +1,38 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2019 nghttp2 contributors
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef H2LOAD_QUIC_H
#define H2LOAD_QUIC_H
#include "nghttp2_config.h"
#include <ev.h>
#include "h2load.h"
namespace h2load {
void quic_pkt_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents);
} // namespace h2load
#endif // H2LOAD_QUIC_H

215
src/http3.cc Normal file
View File

@ -0,0 +1,215 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2021 Tatsuhiro Tsujikawa
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "http3.h"
namespace nghttp2 {
namespace http3 {
namespace {
nghttp3_nv make_nv_internal(const std::string &name, const std::string &value,
bool never_index, uint8_t nv_flags) {
uint8_t flags;
flags = nv_flags |
(never_index ? NGHTTP3_NV_FLAG_NEVER_INDEX : NGHTTP3_NV_FLAG_NONE);
return {(uint8_t *)name.c_str(), (uint8_t *)value.c_str(), name.size(),
value.size(), flags};
}
} // namespace
namespace {
nghttp3_nv make_nv_internal(const StringRef &name, const StringRef &value,
bool never_index, uint8_t nv_flags) {
uint8_t flags;
flags = nv_flags |
(never_index ? NGHTTP3_NV_FLAG_NEVER_INDEX : NGHTTP3_NV_FLAG_NONE);
return {(uint8_t *)name.c_str(), (uint8_t *)value.c_str(), name.size(),
value.size(), flags};
}
} // namespace
nghttp3_nv make_nv(const std::string &name, const std::string &value,
bool never_index) {
return make_nv_internal(name, value, never_index, NGHTTP3_NV_FLAG_NONE);
}
nghttp3_nv make_nv(const StringRef &name, const StringRef &value,
bool never_index) {
return make_nv_internal(name, value, never_index, NGHTTP3_NV_FLAG_NONE);
}
nghttp3_nv make_nv_nocopy(const std::string &name, const std::string &value,
bool never_index) {
return make_nv_internal(name, value, never_index,
NGHTTP3_NV_FLAG_NO_COPY_NAME |
NGHTTP3_NV_FLAG_NO_COPY_VALUE);
}
nghttp3_nv make_nv_nocopy(const StringRef &name, const StringRef &value,
bool never_index) {
return make_nv_internal(name, value, never_index,
NGHTTP3_NV_FLAG_NO_COPY_NAME |
NGHTTP3_NV_FLAG_NO_COPY_VALUE);
}
namespace {
void copy_headers_to_nva_internal(std::vector<nghttp3_nv> &nva,
const HeaderRefs &headers, uint8_t nv_flags,
uint32_t flags) {
auto it_forwarded = std::end(headers);
auto it_xff = std::end(headers);
auto it_xfp = std::end(headers);
auto it_via = std::end(headers);
for (auto it = std::begin(headers); it != std::end(headers); ++it) {
auto kv = &(*it);
if (kv->name.empty() || kv->name[0] == ':') {
continue;
}
switch (kv->token) {
case http2::HD_COOKIE:
case http2::HD_CONNECTION:
case http2::HD_HOST:
case http2::HD_HTTP2_SETTINGS:
case http2::HD_KEEP_ALIVE:
case http2::HD_PROXY_CONNECTION:
case http2::HD_SERVER:
case http2::HD_TE:
case http2::HD_TRANSFER_ENCODING:
case http2::HD_UPGRADE:
continue;
case http2::HD_EARLY_DATA:
if (flags & http2::HDOP_STRIP_EARLY_DATA) {
continue;
}
break;
case http2::HD_SEC_WEBSOCKET_ACCEPT:
if (flags & http2::HDOP_STRIP_SEC_WEBSOCKET_ACCEPT) {
continue;
}
break;
case http2::HD_SEC_WEBSOCKET_KEY:
if (flags & http2::HDOP_STRIP_SEC_WEBSOCKET_KEY) {
continue;
}
break;
case http2::HD_FORWARDED:
if (flags & http2::HDOP_STRIP_FORWARDED) {
continue;
}
if (it_forwarded == std::end(headers)) {
it_forwarded = it;
continue;
}
kv = &(*it_forwarded);
it_forwarded = it;
break;
case http2::HD_X_FORWARDED_FOR:
if (flags & http2::HDOP_STRIP_X_FORWARDED_FOR) {
continue;
}
if (it_xff == std::end(headers)) {
it_xff = it;
continue;
}
kv = &(*it_xff);
it_xff = it;
break;
case http2::HD_X_FORWARDED_PROTO:
if (flags & http2::HDOP_STRIP_X_FORWARDED_PROTO) {
continue;
}
if (it_xfp == std::end(headers)) {
it_xfp = it;
continue;
}
kv = &(*it_xfp);
it_xfp = it;
break;
case http2::HD_VIA:
if (flags & http2::HDOP_STRIP_VIA) {
continue;
}
if (it_via == std::end(headers)) {
it_via = it;
continue;
}
kv = &(*it_via);
it_via = it;
break;
}
nva.push_back(
make_nv_internal(kv->name, kv->value, kv->no_index, nv_flags));
}
}
} // namespace
void copy_headers_to_nva(std::vector<nghttp3_nv> &nva,
const HeaderRefs &headers, uint32_t flags) {
copy_headers_to_nva_internal(nva, headers, NGHTTP3_NV_FLAG_NONE, flags);
}
void copy_headers_to_nva_nocopy(std::vector<nghttp3_nv> &nva,
const HeaderRefs &headers, uint32_t flags) {
copy_headers_to_nva_internal(
nva, headers,
NGHTTP3_NV_FLAG_NO_COPY_NAME | NGHTTP3_NV_FLAG_NO_COPY_VALUE, flags);
}
int check_nv(const uint8_t *name, size_t namelen, const uint8_t *value,
size_t valuelen) {
if (!nghttp3_check_header_name(name, namelen)) {
return 0;
}
if (!nghttp3_check_header_value(value, valuelen)) {
return 0;
}
return 1;
}
void dump_nv(FILE *out, const nghttp3_nv *nva, size_t nvlen) {
auto end = nva + nvlen;
for (; nva != end; ++nva) {
fprintf(out, "%s: %s\n", nva->name, nva->value);
}
fputc('\n', out);
fflush(out);
}
} // namespace http3
} // namespace nghttp2

129
src/http3.h Normal file
View File

@ -0,0 +1,129 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2021 Tatsuhiro Tsujikawa
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef HTTP3_H
#define HTTP3_H
#include "nghttp2_config.h"
#include <cstring>
#include <string>
#include <vector>
#include <nghttp3/nghttp3.h>
#include "http2.h"
#include "template.h"
namespace nghttp2 {
namespace http3 {
// Creates nghttp3_nv using |name| and |value| and returns it. The
// returned value only references the data pointer to name.c_str() and
// value.c_str(). If |no_index| is true, nghttp3_nv flags member has
// NGHTTP3_NV_FLAG_NEVER_INDEX flag set.
nghttp3_nv make_nv(const std::string &name, const std::string &value,
bool never_index = false);
nghttp3_nv make_nv(const StringRef &name, const StringRef &value,
bool never_index = false);
nghttp3_nv make_nv_nocopy(const std::string &name, const std::string &value,
bool never_index = false);
nghttp3_nv make_nv_nocopy(const StringRef &name, const StringRef &value,
bool never_index = false);
// Create nghttp3_nv from string literal |name| and |value|.
template <size_t N, size_t M>
constexpr nghttp3_nv make_nv_ll(const char (&name)[N], const char (&value)[M]) {
return {(uint8_t *)name, (uint8_t *)value, N - 1, M - 1,
NGHTTP3_NV_FLAG_NO_COPY_NAME | NGHTTP3_NV_FLAG_NO_COPY_VALUE};
}
// Create nghttp3_nv from string literal |name| and c-string |value|.
template <size_t N>
nghttp3_nv make_nv_lc(const char (&name)[N], const char *value) {
return {(uint8_t *)name, (uint8_t *)value, N - 1, strlen(value),
NGHTTP3_NV_FLAG_NO_COPY_NAME};
}
template <size_t N>
nghttp3_nv make_nv_lc_nocopy(const char (&name)[N], const char *value) {
return {(uint8_t *)name, (uint8_t *)value, N - 1, strlen(value),
NGHTTP3_NV_FLAG_NO_COPY_NAME | NGHTTP3_NV_FLAG_NO_COPY_VALUE};
}
// Create nghttp3_nv from string literal |name| and std::string
// |value|.
template <size_t N>
nghttp3_nv make_nv_ls(const char (&name)[N], const std::string &value) {
return {(uint8_t *)name, (uint8_t *)value.c_str(), N - 1, value.size(),
NGHTTP3_NV_FLAG_NO_COPY_NAME};
}
template <size_t N>
nghttp3_nv make_nv_ls_nocopy(const char (&name)[N], const std::string &value) {
return {(uint8_t *)name, (uint8_t *)value.c_str(), N - 1, value.size(),
NGHTTP3_NV_FLAG_NO_COPY_NAME | NGHTTP3_NV_FLAG_NO_COPY_VALUE};
}
template <size_t N>
nghttp3_nv make_nv_ls_nocopy(const char (&name)[N], const StringRef &value) {
return {(uint8_t *)name, (uint8_t *)value.c_str(), N - 1, value.size(),
NGHTTP3_NV_FLAG_NO_COPY_NAME | NGHTTP3_NV_FLAG_NO_COPY_VALUE};
}
// Appends headers in |headers| to |nv|. |headers| must be indexed
// before this call (its element's token field is assigned). Certain
// headers, including disallowed headers in HTTP/3 spec and headers
// which require special handling (i.e. via), are not copied. |flags|
// is one or more of HeaderBuildOp flags. They tell function that
// certain header fields should not be added.
void copy_headers_to_nva(std::vector<nghttp3_nv> &nva,
const HeaderRefs &headers, uint32_t flags);
// Just like copy_headers_to_nva(), but this adds
// NGHTTP3_NV_FLAG_NO_COPY_NAME and NGHTTP3_NV_FLAG_NO_COPY_VALUE.
void copy_headers_to_nva_nocopy(std::vector<nghttp3_nv> &nva,
const HeaderRefs &headers, uint32_t flags);
// Dumps name/value pairs in |nva| to |out|.
void dump_nv(FILE *out, const nghttp3_nv *nva, size_t nvlen);
// Checks the header name/value pair using nghttp3_check_header_name()
// and nghttp3_check_header_value(). If both function returns nonzero,
// this function returns nonzero.
int check_nv(const uint8_t *name, size_t namelen, const uint8_t *value,
size_t valuelen);
// Dumps name/value pairs in |nva| to |out|.
void dump_nv(FILE *out, const nghttp3_nv *nva, size_t nvlen);
} // namespace http3
} // namespace nghttp2
#endif // HTTP3_H

View File

@ -113,13 +113,22 @@ template <typename T> struct Pool {
template <typename Memchunk> struct Memchunks {
Memchunks(Pool<Memchunk> *pool)
: pool(pool), head(nullptr), tail(nullptr), len(0) {}
: pool(pool),
head(nullptr),
tail(nullptr),
len(0),
mark(nullptr),
mark_pos(nullptr),
mark_offset(0) {}
Memchunks(const Memchunks &) = delete;
Memchunks(Memchunks &&other) noexcept
: pool{other.pool}, // keep other.pool
head{std::exchange(other.head, nullptr)},
tail{std::exchange(other.tail, nullptr)},
len{std::exchange(other.len, 0)} {}
len{std::exchange(other.len, 0)},
mark{std::exchange(other.mark, nullptr)},
mark_pos{std::exchange(other.mark_pos, nullptr)},
mark_offset{std::exchange(other.mark_offset, 0)} {}
Memchunks &operator=(const Memchunks &) = delete;
Memchunks &operator=(Memchunks &&other) noexcept {
if (this == &other) {
@ -132,6 +141,9 @@ template <typename Memchunk> struct Memchunks {
head = std::exchange(other.head, nullptr);
tail = std::exchange(other.tail, nullptr);
len = std::exchange(other.len, 0);
mark = std::exchange(other.mark, nullptr);
mark_pos = std::exchange(other.mark_pos, nullptr);
mark_offset = std::exchange(other.mark_offset, 0);
return *this;
}
@ -200,6 +212,8 @@ template <typename Memchunk> struct Memchunks {
return len;
}
size_t remove(void *dest, size_t count) {
assert(mark == nullptr);
if (!tail || count == 0) {
return 0;
}
@ -231,6 +245,8 @@ template <typename Memchunk> struct Memchunks {
return first - static_cast<uint8_t *>(dest);
}
size_t remove(Memchunks &dest, size_t count) {
assert(mark == nullptr);
if (!tail || count == 0) {
return 0;
}
@ -262,6 +278,7 @@ template <typename Memchunk> struct Memchunks {
}
size_t remove(Memchunks &dest) {
assert(pool == dest.pool);
assert(mark == nullptr);
if (head == nullptr) {
return 0;
@ -284,6 +301,8 @@ template <typename Memchunk> struct Memchunks {
return n;
}
size_t drain(size_t count) {
assert(mark == nullptr);
auto ndata = count;
auto m = head;
while (m) {
@ -305,6 +324,38 @@ template <typename Memchunk> struct Memchunks {
}
return ndata - count;
}
size_t drain_mark(size_t count) {
auto ndata = count;
auto m = head;
while (m) {
auto next = m->next;
auto n = std::min(count, m->len());
m->pos += n;
count -= n;
len -= n;
mark_offset -= n;
if (m->len() > 0) {
assert(mark != m || m->pos <= mark_pos);
break;
}
if (mark == m) {
assert(m->pos <= mark_pos);
mark = nullptr;
mark_pos = nullptr;
mark_offset = 0;
}
pool->recycle(m);
m = next;
}
head = m;
if (head == nullptr) {
tail = nullptr;
}
return ndata - count;
}
int riovec(struct iovec *iov, int iovcnt) const {
if (!head) {
return 0;
@ -317,7 +368,41 @@ template <typename Memchunk> struct Memchunks {
}
return i;
}
int riovec_mark(struct iovec *iov, int iovcnt) {
if (!head || iovcnt == 0) {
return 0;
}
int i = 0;
Memchunk *m;
if (mark) {
if (mark_pos != mark->last) {
iov[0].iov_base = mark_pos;
iov[0].iov_len = mark->len() - (mark_pos - mark->pos);
mark_pos = mark->last;
mark_offset += iov[0].iov_len;
i = 1;
}
m = mark->next;
} else {
i = 0;
m = head;
}
for (; i < iovcnt && m; ++i, m = m->next) {
iov[i].iov_base = m->pos;
iov[i].iov_len = m->len();
mark = m;
mark_pos = m->last;
mark_offset += m->len();
}
return i;
}
size_t rleft() const { return len; }
size_t rleft_mark() const { return len - mark_offset; }
void reset() {
for (auto m = head; m;) {
auto next = m->next;
@ -325,12 +410,17 @@ template <typename Memchunk> struct Memchunks {
m = next;
}
len = 0;
head = tail = nullptr;
head = tail = mark = nullptr;
mark_pos = nullptr;
mark_offset = 0;
}
Pool<Memchunk> *pool;
Memchunk *head, *tail;
size_t len;
Memchunk *mark;
uint8_t *mark_pos;
size_t mark_offset;
};
// Wrapper around Memchunks to offer "peeking" functionality.

60
src/quic.cc Normal file
View File

@ -0,0 +1,60 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2019 nghttp2 contributors
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "quic.h"
#include <cassert>
#include <ngtcp2/ngtcp2.h>
#include <nghttp3/nghttp3.h>
#include "template.h"
using namespace nghttp2;
namespace quic {
Error err_transport(int liberr) {
if (liberr == NGTCP2_ERR_RECV_VERSION_NEGOTIATION) {
return {ErrorType::TransportVersionNegotiation, 0};
}
return {ErrorType::Transport,
ngtcp2_err_infer_quic_transport_error_code(liberr)};
}
Error err_transport_idle_timeout() {
return {ErrorType::TransportIdleTimeout, 0};
}
Error err_transport_tls(int alert) {
return {ErrorType::Transport, ngtcp2_err_infer_quic_transport_error_code(
NGTCP2_CRYPTO_ERROR | alert)};
}
Error err_application(int liberr) {
return {ErrorType::Application,
nghttp3_err_infer_quic_app_error_code(liberr)};
}
} // namespace quic

56
src/quic.h Normal file
View File

@ -0,0 +1,56 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2019 nghttp2 contributors
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef QUIC_H
#define QUIC_H
#include "nghttp2_config.h"
#include "stdint.h"
namespace quic {
enum class ErrorType {
Transport,
TransportVersionNegotiation,
TransportIdleTimeout,
Application,
};
struct Error {
Error(ErrorType type, uint64_t code) : type(type), code(code) {}
Error() : type(ErrorType::Transport), code(0) {}
ErrorType type;
uint64_t code;
};
Error err_transport(int liberr);
Error err_transport_idle_timeout();
Error err_transport_tls(int alert);
Error err_application(int liberr);
} // namespace quic
#endif // QUIC_H

View File

@ -71,6 +71,7 @@
#include <openssl/ssl.h>
#include <openssl/err.h>
#include <openssl/rand.h>
#include <ev.h>
#include <nghttp2/nghttp2.h>
@ -1548,6 +1549,21 @@ void fill_default_config(Config *config) {
downstreamconf.option, downstreamconf.encoder_dynamic_table_size);
}
#ifdef ENABLE_HTTP3
auto &quicconf = config->quic;
{
quicconf.timeout.idle = 30_s;
auto &stateless_resetconf = quicconf.stateless_reset;
// TODO Find better place to do this and error handling.
if (RAND_bytes(stateless_resetconf.secret.data(),
stateless_resetconf.secret.size()) != 1) {
LOG(FATAL) << "Unable to generate stateless reset secret";
exit(EXIT_FAILURE);
}
}
#endif // ENABLE_HTTP3
auto &loggingconf = config->logging;
{
auto &accessconf = loggingconf.access;

View File

@ -51,6 +51,9 @@
#include "shrpx_api_downstream_connection.h"
#include "shrpx_health_monitor_downstream_connection.h"
#include "shrpx_null_downstream_connection.h"
#ifdef ENABLE_HTTP3
# include "shrpx_http3_upstream.h"
#endif // ENABLE_HTTP3
#include "shrpx_log.h"
#include "util.h"
#include "template.h"
@ -286,6 +289,19 @@ int ClientHandler::write_tls() {
}
}
#ifdef ENABLE_HTTP3
int ClientHandler::read_quic(const UpstreamAddr *faddr,
const Address &remote_addr,
const Address &local_addr, const uint8_t *data,
size_t datalen) {
auto upstream = static_cast<Http3Upstream *>(upstream_.get());
return upstream->on_read(faddr, remote_addr, local_addr, data, datalen);
}
int ClientHandler::write_quic() { return upstream_->on_write(); }
#endif // ENABLE_HTTP3
int ClientHandler::upstream_noop() { return 0; }
int ClientHandler::upstream_read() {
@ -402,7 +418,8 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl,
get_config()->conn.upstream.ratelimit.write,
get_config()->conn.upstream.ratelimit.read, writecb, readcb,
timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold,
get_config()->tls.dyn_rec.idle_timeout, Proto::NONE),
get_config()->tls.dyn_rec.idle_timeout,
faddr->quic ? Proto::HTTP3 : Proto::NONE),
ipaddr_(make_string_ref(balloc_, ipaddr)),
port_(make_string_ref(balloc_, port)),
faddr_(faddr),
@ -418,19 +435,23 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl,
reneg_shutdown_timer_.data = this;
conn_.rlimit.startw();
if (!faddr->quic) {
conn_.rlimit.startw();
}
ev_timer_again(conn_.loop, &conn_.rt);
auto config = get_config();
if (faddr_->accept_proxy_protocol ||
config->conn.upstream.accept_proxy_protocol) {
read_ = &ClientHandler::read_clear;
write_ = &ClientHandler::noop;
on_read_ = &ClientHandler::proxy_protocol_read;
on_write_ = &ClientHandler::upstream_noop;
} else {
setup_upstream_io_callback();
if (!faddr->quic) {
if (faddr_->accept_proxy_protocol ||
config->conn.upstream.accept_proxy_protocol) {
read_ = &ClientHandler::read_clear;
write_ = &ClientHandler::noop;
on_read_ = &ClientHandler::proxy_protocol_read;
on_write_ = &ClientHandler::upstream_noop;
} else {
setup_upstream_io_callback();
}
}
auto &fwdconf = config->http.forwarded;
@ -492,6 +513,15 @@ void ClientHandler::setup_upstream_io_callback() {
}
}
#ifdef ENABLE_HTTP3
void ClientHandler::setup_http3_upstream(
std::unique_ptr<Http3Upstream> &&upstream) {
upstream_ = std::move(upstream);
alpn_ = StringRef::from_lit("h3");
write_ = &ClientHandler::write_quic;
}
#endif // ENABLE_HTTP3
ClientHandler::~ClientHandler() {
if (LOG_ENABLED(INFO)) {
CLOG(INFO, this) << "Deleting";

View File

@ -53,6 +53,9 @@ class Downstream;
struct WorkerStat;
struct DownstreamAddrGroup;
struct DownstreamAddr;
#ifdef ENABLE_HTTP3
class Http3Upstream;
#endif // ENABLE_HTTP3
class ClientHandler {
public:
@ -143,6 +146,13 @@ public:
void setup_upstream_io_callback();
#ifdef ENABLE_HTTP3
void setup_http3_upstream(std::unique_ptr<Http3Upstream> &&upstream);
int read_quic(const UpstreamAddr *faddr, const Address &remote_addr,
const Address &local_addr, const uint8_t *data, size_t datalen);
int write_quic();
#endif // ENABLE_HTTP3
// Returns string suitable for use in "by" parameter of Forwarded
// header field.
StringRef get_forwarded_by() const;

View File

@ -785,6 +785,7 @@ struct UpstreamParams {
bool tls;
bool sni_fwd;
bool proxyproto;
bool quic;
};
namespace {
@ -819,6 +820,12 @@ int parse_upstream_params(UpstreamParams &out, const StringRef &src_params) {
out.alt_mode = UpstreamAltMode::HEALTHMON;
} else if (util::strieq_l("proxyproto", param)) {
out.proxyproto = true;
} else if (util::strieq_l("quic", param)) {
#ifdef ENABLE_HTTP3
out.quic = true;
#else // !ENABLE_HTTP3
LOG(ERROR) << "quic: QUIC is disabled at compile time";
#endif // !ENABLE_HTTP3
} else if (!param.empty()) {
LOG(ERROR) << "frontend: " << param << ": unknown keyword";
return -1;
@ -2637,7 +2644,6 @@ int parse_config(Config *config, int optid, const StringRef &opt,
return 0;
}
case SHRPX_OPTID_FRONTEND: {
auto &listenerconf = config->conn.listener;
auto &apiconf = config->api;
auto addr_end = std::find(std::begin(optarg), std::end(optarg), ';');
@ -2655,23 +2661,36 @@ int parse_config(Config *config, int optid, const StringRef &opt,
return -1;
}
if (params.quic && params.alt_mode != UpstreamAltMode::NONE) {
LOG(ERROR) << "frontend: api or healthmon cannot be used with quic";
return -1;
}
UpstreamAddr addr{};
addr.fd = -1;
addr.tls = params.tls;
addr.sni_fwd = params.sni_fwd;
addr.alt_mode = params.alt_mode;
addr.accept_proxy_protocol = params.proxyproto;
addr.quic = params.quic;
if (addr.alt_mode == UpstreamAltMode::API) {
apiconf.enabled = true;
}
#ifdef ENABLE_HTTP3
auto &addrs = params.quic ? config->conn.quic_listener.addrs
: config->conn.listener.addrs;
#else // !ENABLE_HTTP3
auto &addrs = config->conn.listener.addrs;
#endif // !ENABLE_HTTP3
if (util::istarts_with(optarg, SHRPX_UNIX_PATH_PREFIX)) {
auto path = std::begin(optarg) + SHRPX_UNIX_PATH_PREFIX.size();
addr.host = make_string_ref(config->balloc, StringRef{path, addr_end});
addr.host_unix = true;
listenerconf.addrs.push_back(std::move(addr));
addrs.push_back(std::move(addr));
return 0;
}
@ -2686,21 +2705,21 @@ int parse_config(Config *config, int optid, const StringRef &opt,
if (util::numeric_host(host, AF_INET)) {
addr.family = AF_INET;
listenerconf.addrs.push_back(std::move(addr));
addrs.push_back(std::move(addr));
return 0;
}
if (util::numeric_host(host, AF_INET6)) {
addr.family = AF_INET6;
listenerconf.addrs.push_back(std::move(addr));
addrs.push_back(std::move(addr));
return 0;
}
addr.family = AF_INET;
listenerconf.addrs.push_back(addr);
addrs.push_back(addr);
addr.family = AF_INET6;
listenerconf.addrs.push_back(std::move(addr));
addrs.push_back(std::move(addr));
return 0;
}
@ -3980,6 +3999,8 @@ StringRef strproto(Proto proto) {
return StringRef::from_lit("http/1.1");
case Proto::HTTP2:
return StringRef::from_lit("h2");
case Proto::HTTP3:
return StringRef::from_lit("h3");
case Proto::MEMCACHED:
return StringRef::from_lit("memcached");
}

View File

@ -370,6 +370,7 @@ enum class Proto {
NONE,
HTTP1,
HTTP2,
HTTP3,
MEMCACHED,
};
@ -458,6 +459,7 @@ struct UpstreamAddr {
bool sni_fwd;
// true if client is supposed to send PROXY protocol v1 header.
bool accept_proxy_protocol;
bool quic;
int fd;
};
@ -702,6 +704,20 @@ struct TLSConfig {
bool no_postpone_early_data;
};
#ifdef ENABLE_HTTP3
struct QUICConfig {
struct {
std::array<uint8_t, 32> secret;
} stateless_reset;
struct {
ev_tstamp idle;
} timeout;
struct {
bool log;
} debug;
};
#endif // ENABLE_HTTP3
// custom error page
struct ErrorPage {
// not NULL-terminated
@ -907,6 +923,12 @@ struct ConnectionConfig {
int fastopen;
} listener;
#ifdef ENABLE_HTTP3
struct {
std::vector<UpstreamAddr> addrs;
} quic_listener;
#endif // ENABLE_HTTP3
struct {
struct {
ev_tstamp http2_read;
@ -950,6 +972,9 @@ struct Config {
http{},
http2{},
tls{},
#ifdef ENABLE_HTTP3
quic{},
#endif // ENABLE_HTTP3
logging{},
conn{},
api{},
@ -967,7 +992,8 @@ struct Config {
single_process{false},
single_thread{false},
ignore_per_pattern_mruby_error{false},
ev_loop_flags{0} {}
ev_loop_flags{0} {
}
~Config();
Config(Config &&) = delete;
@ -983,6 +1009,9 @@ struct Config {
HttpConfig http;
Http2Config http2;
TLSConfig tls;
#ifdef ENABLE_HTTP3
QUICConfig quic;
#endif // ENABLE_HTTP3
LoggingConfig logging;
ConnectionConfig conn;
APIConfig api;

View File

@ -74,7 +74,7 @@ Connection::Connection(struct ev_loop *loop, int fd, SSL *ssl,
read_timeout(read_timeout) {
ev_io_init(&wev, writecb, fd, EV_WRITE);
ev_io_init(&rev, readcb, fd, EV_READ);
ev_io_init(&rev, readcb, proto == Proto::HTTP3 ? 0 : fd, EV_READ);
wev.data = this;
rev.data = this;
@ -128,7 +128,7 @@ void Connection::disconnect() {
tls.early_data_finish = false;
}
if (fd != -1) {
if (proto != Proto::HTTP3 && fd != -1) {
shutdown(fd, SHUT_WR);
close(fd);
fd = -1;
@ -312,10 +312,13 @@ BIO_METHOD *create_bio_method() {
void Connection::set_ssl(SSL *ssl) {
tls.ssl = ssl;
auto &tlsconf = get_config()->tls;
auto bio = BIO_new(tlsconf.bio_method);
BIO_set_data(bio, this);
SSL_set_bio(tls.ssl, bio, bio);
if (proto != Proto::HTTP3) {
auto &tlsconf = get_config()->tls;
auto bio = BIO_new(tlsconf.bio_method);
BIO_set_data(bio, this);
SSL_set_bio(tls.ssl, bio, bio);
}
SSL_set_app_data(tls.ssl, this);
}

View File

@ -156,6 +156,19 @@ ConnectionHandler::~ConnectionHandler() {
ev_timer_stop(loop_, &ocsp_timer_);
ev_timer_stop(loop_, &disable_acceptor_timer_);
#ifdef ENABLE_HTTP3
for (auto ssl_ctx : quic_all_ssl_ctx_) {
if (ssl_ctx == nullptr) {
continue;
}
auto tls_ctx_data =
static_cast<tls::TLSContextData *>(SSL_CTX_get_app_data(ssl_ctx));
delete tls_ctx_data;
SSL_CTX_free(ssl_ctx);
}
#endif // ENABLE_HTTP3
for (auto ssl_ctx : all_ssl_ctx_) {
auto tls_ctx_data =
static_cast<tls::TLSContextData *>(SSL_CTX_get_app_data(ssl_ctx));
@ -209,6 +222,18 @@ int ConnectionHandler::create_single_worker() {
nb_
#endif // HAVE_NEVERBLEED
);
#ifdef ENABLE_HTTP3
quic_cert_tree_ = tls::create_cert_lookup_tree();
auto quic_sv_ssl_ctx = tls::setup_quic_server_ssl_context(
quic_all_ssl_ctx_, quic_indexed_ssl_ctx_, quic_cert_tree_.get()
# ifdef HAVE_NEVERBLEED
,
nb_
# endif // HAVE_NEVERBLEED
);
#endif // ENABLE_HTTP3
auto cl_ssl_ctx = tls::setup_downstream_client_ssl_context(
#ifdef HAVE_NEVERBLEED
nb_
@ -217,6 +242,9 @@ int ConnectionHandler::create_single_worker() {
if (cl_ssl_ctx) {
all_ssl_ctx_.push_back(cl_ssl_ctx);
#ifdef ENABLE_HTTP3
quic_all_ssl_ctx_.push_back(nullptr);
#endif // ENABLE_HTTP3
}
auto config = get_config();
@ -233,11 +261,17 @@ int ConnectionHandler::create_single_worker() {
tlsconf.cacert, memcachedconf.cert_file,
memcachedconf.private_key_file, nullptr);
all_ssl_ctx_.push_back(session_cache_ssl_ctx);
#ifdef ENABLE_HTTP3
quic_all_ssl_ctx_.push_back(nullptr);
#endif // ENABLE_HTTP3
}
}
single_worker_ = std::make_unique<Worker>(
loop_, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(),
#ifdef ENABLE_HTTP3
quic_sv_ssl_ctx, quic_cert_tree_.get(),
#endif // ENABLE_HTTP3
ticket_keys_, this, config->conn.downstream);
#ifdef HAVE_MRUBY
if (single_worker_->create_mruby_context() != 0) {
@ -245,6 +279,12 @@ int ConnectionHandler::create_single_worker() {
}
#endif // HAVE_MRUBY
#ifdef ENABLE_HTTP3
if (single_worker_->setup_quic_server_socket() != 0) {
return -1;
}
#endif // ENABLE_HTTP3
return 0;
}
@ -260,6 +300,18 @@ int ConnectionHandler::create_worker_thread(size_t num) {
nb_
# endif // HAVE_NEVERBLEED
);
# ifdef ENABLE_HTTP3
quic_cert_tree_ = tls::create_cert_lookup_tree();
auto quic_sv_ssl_ctx = tls::setup_quic_server_ssl_context(
quic_all_ssl_ctx_, quic_indexed_ssl_ctx_, quic_cert_tree_.get()
# ifdef HAVE_NEVERBLEED
,
nb_
# endif // HAVE_NEVERBLEED
);
# endif // ENABLE_HTTP3
auto cl_ssl_ctx = tls::setup_downstream_client_ssl_context(
# ifdef HAVE_NEVERBLEED
nb_
@ -268,6 +320,9 @@ int ConnectionHandler::create_worker_thread(size_t num) {
if (cl_ssl_ctx) {
all_ssl_ctx_.push_back(cl_ssl_ctx);
# ifdef ENABLE_HTTP3
quic_all_ssl_ctx_.push_back(nullptr);
# endif // ENABLE_HTTP3
}
auto config = get_config();
@ -291,6 +346,9 @@ int ConnectionHandler::create_worker_thread(size_t num) {
tlsconf.cacert, memcachedconf.cert_file,
memcachedconf.private_key_file, nullptr);
all_ssl_ctx_.push_back(session_cache_ssl_ctx);
# ifdef ENABLE_HTTP3
quic_all_ssl_ctx_.push_back(nullptr);
# endif // ENABLE_HTTP3
}
}
@ -299,6 +357,9 @@ int ConnectionHandler::create_worker_thread(size_t num) {
auto worker = std::make_unique<Worker>(
loop, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(),
# ifdef ENABLE_HTTP3
quic_sv_ssl_ctx, quic_cert_tree_.get(),
# endif // ENABLE_HTTP3
ticket_keys_, this, config->conn.downstream);
# ifdef HAVE_MRUBY
if (worker->create_mruby_context() != 0) {
@ -306,6 +367,13 @@ int ConnectionHandler::create_worker_thread(size_t num) {
}
# endif // HAVE_MRUBY
# ifdef ENABLE_HTTP3
if ((!apiconf.enabled || i != 0) &&
worker->setup_quic_server_socket() != 0) {
return -1;
}
# endif // ENABLE_HTTP3
workers_.push_back(std::move(worker));
worker_loops_.push_back(loop);
@ -602,6 +670,9 @@ void ConnectionHandler::handle_ocsp_complete() {
ev_child_stop(loop_, &ocsp_.chldev);
assert(ocsp_.next < all_ssl_ctx_.size());
#ifdef ENABLE_HTTP3
assert(all_ssl_ctx_.size() == quic_all_ssl_ctx_.size());
#endif // ENABLE_HTTP3
auto ssl_ctx = all_ssl_ctx_[ocsp_.next];
auto tls_ctx_data =
@ -629,6 +700,31 @@ void ConnectionHandler::handle_ocsp_complete() {
if (tlsconf.ocsp.no_verify ||
tls::verify_ocsp_response(ssl_ctx, ocsp_.resp.data(),
ocsp_.resp.size()) == 0) {
#ifdef ENABLE_HTTP3
// We have list of SSL_CTX with the same certificate in
// quic_all_ssl_ctx_ as well. Some SSL_CTXs are missing there in
// that case we get nullptr.
auto quic_ssl_ctx = quic_all_ssl_ctx_[ocsp_.next];
if (quic_ssl_ctx) {
auto quic_tls_ctx_data = static_cast<tls::TLSContextData *>(
SSL_CTX_get_app_data(quic_ssl_ctx));
# ifndef OPENSSL_IS_BORINGSSL
# ifdef HAVE_ATOMIC_STD_SHARED_PTR
std::atomic_store_explicit(
&quic_tls_ctx_data->ocsp_data,
std::make_shared<std::vector<uint8_t>>(ocsp_.resp),
std::memory_order_release);
# else // !HAVE_ATOMIC_STD_SHARED_PTR
std::lock_guard<std::mutex> g(quic_tls_ctx_data->mu);
quic_tls_ctx_data->ocsp_data =
std::make_shared<std::vector<uint8_t>>(ocsp_.resp);
# endif // !HAVE_ATOMIC_STD_SHARED_PTR
# else // OPENSSL_IS_BORINGSSL
SSL_CTX_set_ocsp_response(ssl_ctx, ocsp_.resp.data(), ocsp_.resp.size());
# endif // OPENSSL_IS_BORINGSSL
}
#endif // ENABLE_HTTP3
#ifndef OPENSSL_IS_BORINGSSL
# ifdef HAVE_ATOMIC_STD_SHARED_PTR
std::atomic_store_explicit(
@ -809,6 +905,9 @@ SSL_CTX *ConnectionHandler::create_tls_ticket_key_memcached_ssl_ctx() {
nullptr);
all_ssl_ctx_.push_back(ssl_ctx);
#ifdef ENABLE_HTTP3
quic_all_ssl_ctx_.push_back(nullptr);
#endif // ENABLE_HTTP3
return ssl_ctx;
}
@ -871,6 +970,13 @@ ConnectionHandler::get_indexed_ssl_ctx(size_t idx) const {
return indexed_ssl_ctx_[idx];
}
#ifdef ENABLE_HTTP3
const std::vector<SSL_CTX *> &
ConnectionHandler::get_quic_indexed_ssl_ctx(size_t idx) const {
return quic_indexed_ssl_ctx_[idx];
}
#endif // ENABLE_HTTP3
void ConnectionHandler::set_enable_acceptor_on_ocsp_completion(bool f) {
enable_acceptor_on_ocsp_completion_ = f;
}

View File

@ -159,6 +159,9 @@ public:
SSL_CTX *get_ssl_ctx(size_t idx) const;
const std::vector<SSL_CTX *> &get_indexed_ssl_ctx(size_t idx) const;
#ifdef ENABLE_HTTP3
const std::vector<SSL_CTX *> &get_quic_indexed_ssl_ctx(size_t idx) const;
#endif // ENABLE_HTTP3
#ifdef HAVE_NEVERBLEED
void set_neverbleed(neverbleed_t *nb);
@ -187,6 +190,10 @@ private:
// selection among them are performed by hostname presented by SNI,
// and signature algorithm presented by client.
std::vector<std::vector<SSL_CTX *>> indexed_ssl_ctx_;
#ifdef ENABLE_HTTP3
std::vector<SSL_CTX *> quic_all_ssl_ctx_;
std::vector<std::vector<SSL_CTX *>> quic_indexed_ssl_ctx_;
#endif // ENABLE_HTTP3
OCSPUpdateContext ocsp_;
std::mt19937 &gen_;
// ev_loop for each worker
@ -203,6 +210,9 @@ private:
// Otherwise, nullptr and workers_ has instances of Worker instead.
std::unique_ptr<Worker> single_worker_;
std::unique_ptr<tls::CertLookupTree> cert_tree_;
#ifdef ENABLE_HTTP3
std::unique_ptr<tls::CertLookupTree> quic_cert_tree_;
#endif // ENABLE_HTTP3
std::unique_ptr<MemcachedDispatcher> tls_ticket_key_memcached_dispatcher_;
// Current TLS session ticket keys. Note that TLS connection does
// not refer to this field directly. They use TicketKeys object in

View File

@ -113,7 +113,7 @@ void downstream_wtimeoutcb(struct ev_loop *loop, ev_timer *w, int revents) {
// upstream could be nullptr for unittests
Downstream::Downstream(Upstream *upstream, MemchunkPool *mcpool,
int32_t stream_id)
int64_t stream_id)
: dlnext(nullptr),
dlprev(nullptr),
response_sent_body_length(0),
@ -145,7 +145,8 @@ Downstream::Downstream(Upstream *upstream, MemchunkPool *mcpool,
accesslog_written_(false),
new_affinity_cookie_(false),
blocked_request_data_eof_(false),
expect_100_continue_(false) {
expect_100_continue_(false),
stop_reading_(false) {
auto &timeoutconf = get_config()->http2.timeout;
@ -164,6 +165,9 @@ Downstream::Downstream(Upstream *upstream, MemchunkPool *mcpool,
downstream_wtimer_.data = this;
rcbufs_.reserve(32);
#ifdef ENABLE_HTTP3
rcbufs3_.reserve(32);
#endif // ENABLE_HTTP3
}
Downstream::~Downstream() {
@ -203,6 +207,12 @@ Downstream::~Downstream() {
// explicitly.
dconn_.reset();
#ifdef ENABLE_HTTP3
for (auto rcbuf : rcbufs3_) {
nghttp3_rcbuf_decref(rcbuf);
}
#endif // ENABLE_HTTP3
for (auto rcbuf : rcbufs_) {
nghttp2_rcbuf_decref(rcbuf);
}
@ -605,9 +615,9 @@ void Downstream::reset_upstream(Upstream *upstream) {
Upstream *Downstream::get_upstream() const { return upstream_; }
void Downstream::set_stream_id(int32_t stream_id) { stream_id_ = stream_id; }
void Downstream::set_stream_id(int64_t stream_id) { stream_id_ = stream_id; }
int32_t Downstream::get_stream_id() const { return stream_id_; }
int64_t Downstream::get_stream_id() const { return stream_id_; }
void Downstream::set_request_state(DownstreamState state) {
request_state_ = state;
@ -904,11 +914,11 @@ StringRef Downstream::get_http2_settings() const {
return http2_settings->value;
}
void Downstream::set_downstream_stream_id(int32_t stream_id) {
void Downstream::set_downstream_stream_id(int64_t stream_id) {
downstream_stream_id_ = stream_id;
}
int32_t Downstream::get_downstream_stream_id() const {
int64_t Downstream::get_downstream_stream_id() const {
return downstream_stream_id_;
}
@ -1115,11 +1125,11 @@ DefaultMemchunks Downstream::pop_response_buf() {
return std::move(response_buf_);
}
void Downstream::set_assoc_stream_id(int32_t stream_id) {
void Downstream::set_assoc_stream_id(int64_t stream_id) {
assoc_stream_id_ = stream_id;
}
int32_t Downstream::get_assoc_stream_id() const { return assoc_stream_id_; }
int64_t Downstream::get_assoc_stream_id() const { return assoc_stream_id_; }
BlockAllocator &Downstream::get_block_allocator() { return balloc_; }
@ -1128,6 +1138,13 @@ void Downstream::add_rcbuf(nghttp2_rcbuf *rcbuf) {
rcbufs_.push_back(rcbuf);
}
#ifdef ENABLE_HTTP3
void Downstream::add_rcbuf(nghttp3_rcbuf *rcbuf) {
nghttp3_rcbuf_incref(rcbuf);
rcbufs3_.push_back(rcbuf);
}
#endif // ENABLE_HTTP3
void Downstream::set_downstream_addr_group(
const std::shared_ptr<DownstreamAddrGroup> &group) {
group_ = group;
@ -1169,4 +1186,8 @@ bool Downstream::get_expect_100_continue() const {
return expect_100_continue_;
}
bool Downstream::get_stop_reading() const { return stop_reading_; }
void Downstream::set_stop_reading(bool f) { stop_reading_ = f; }
} // namespace shrpx

View File

@ -38,6 +38,10 @@
#include <nghttp2/nghttp2.h>
#ifdef ENABLE_HTTP3
# include <nghttp3/nghttp3.h>
#endif // ENABLE_HTTP3
#include "llhttp.h"
#include "shrpx_io_control.h"
@ -319,20 +323,20 @@ enum class DispatchState {
class Downstream {
public:
Downstream(Upstream *upstream, MemchunkPool *mcpool, int32_t stream_id);
Downstream(Upstream *upstream, MemchunkPool *mcpool, int64_t stream_id);
~Downstream();
void reset_upstream(Upstream *upstream);
Upstream *get_upstream() const;
void set_stream_id(int32_t stream_id);
int32_t get_stream_id() const;
void set_assoc_stream_id(int32_t stream_id);
int32_t get_assoc_stream_id() const;
void set_stream_id(int64_t stream_id);
int64_t get_stream_id() const;
void set_assoc_stream_id(int64_t stream_id);
int64_t get_assoc_stream_id() const;
void pause_read(IOCtrlReason reason);
int resume_read(IOCtrlReason reason, size_t consumed);
void force_resume_read();
// Set stream ID for downstream HTTP2 connection.
void set_downstream_stream_id(int32_t stream_id);
int32_t get_downstream_stream_id() const;
void set_downstream_stream_id(int64_t stream_id);
int64_t get_downstream_stream_id() const;
int attach_downstream_connection(std::unique_ptr<DownstreamConnection> dconn);
void detach_downstream_connection();
@ -488,6 +492,9 @@ public:
BlockAllocator &get_block_allocator();
void add_rcbuf(nghttp2_rcbuf *rcbuf);
#ifdef ENABLE_HTTP3
void add_rcbuf(nghttp3_rcbuf *rcbuf);
#endif // ENABLE_HTTP3
void
set_downstream_addr_group(const std::shared_ptr<DownstreamAddrGroup> &group);
@ -513,6 +520,9 @@ public:
bool get_expect_100_continue() const;
bool get_stop_reading() const;
void set_stop_reading(bool f);
enum {
EVENT_ERROR = 0x1,
EVENT_TIMEOUT = 0x2,
@ -527,6 +537,9 @@ private:
BlockAllocator balloc_;
std::vector<nghttp2_rcbuf *> rcbufs_;
#ifdef ENABLE_HTTP3
std::vector<nghttp3_rcbuf *> rcbufs3_;
#endif // ENABLE_HTTP3
Request req_;
Response resp_;
@ -566,12 +579,12 @@ private:
// How many times we tried in backend connection
size_t num_retry_;
// The stream ID in frontend connection
int32_t stream_id_;
int64_t stream_id_;
// The associated stream ID in frontend connection if this is pushed
// stream.
int32_t assoc_stream_id_;
int64_t assoc_stream_id_;
// stream ID in backend connection
int32_t downstream_stream_id_;
int64_t downstream_stream_id_;
// RST_STREAM error_code from downstream HTTP2 connection
uint32_t response_rst_stream_error_code_;
// An affinity cookie value.
@ -606,6 +619,7 @@ private:
bool blocked_request_data_eof_;
// true if request contains "expect: 100-continue" header field.
bool expect_100_continue_;
bool stop_reading_;
};
} // namespace shrpx

2338
src/shrpx_http3_upstream.cc Normal file

File diff suppressed because it is too large Load Diff

160
src/shrpx_http3_upstream.h Normal file
View File

@ -0,0 +1,160 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2021 Tatsuhiro Tsujikawa
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef SHRPX_HTTP3_UPSTREAM_H
#define SHRPX_HTTP3_UPSTREAM_H
#include "shrpx.h"
#include <ngtcp2/ngtcp2.h>
#include <nghttp3/nghttp3.h>
#include "shrpx_upstream.h"
#include "shrpx_downstream_queue.h"
#include "quic.h"
#include "network.h"
using namespace nghttp2;
namespace shrpx {
struct UpstreamAddr;
class Http3Upstream : public Upstream {
public:
Http3Upstream(ClientHandler *handler);
virtual ~Http3Upstream();
virtual int on_read();
virtual int on_write();
virtual int on_timeout(Downstream *downstream);
virtual int on_downstream_abort_request(Downstream *downstream,
unsigned int status_code);
virtual int
on_downstream_abort_request_with_https_redirect(Downstream *downstream);
virtual int downstream_read(DownstreamConnection *dconn);
virtual int downstream_write(DownstreamConnection *dconn);
virtual int downstream_eof(DownstreamConnection *dconn);
virtual int downstream_error(DownstreamConnection *dconn, int events);
virtual ClientHandler *get_client_handler() const;
virtual int on_downstream_header_complete(Downstream *downstream);
virtual int on_downstream_body(Downstream *downstream, const uint8_t *data,
size_t len, bool flush);
virtual int on_downstream_body_complete(Downstream *downstream);
virtual void on_handler_delete();
virtual int on_downstream_reset(Downstream *downstream, bool no_retry);
virtual void pause_read(IOCtrlReason reason);
virtual int resume_read(IOCtrlReason reason, Downstream *downstream,
size_t consumed);
virtual int send_reply(Downstream *downstream, const uint8_t *body,
size_t bodylen);
virtual int initiate_push(Downstream *downstream, const StringRef &uri);
virtual int response_riovec(struct iovec *iov, int iovcnt) const;
virtual void response_drain(size_t n);
virtual bool response_empty() const;
virtual Downstream *on_downstream_push_promise(Downstream *downstream,
int32_t promised_stream_id);
virtual int
on_downstream_push_promise_complete(Downstream *downstream,
Downstream *promised_downstream);
virtual bool push_enabled() const;
virtual void cancel_premature_downstream(Downstream *promised_downstream);
int init(const UpstreamAddr *faddr, const Address &remote_addr,
const Address &local_addr, const ngtcp2_pkt_hd &initial_hd);
int on_read(const UpstreamAddr *faddr, const Address &remote_addr,
const Address &local_addr, const uint8_t *data, size_t datalen);
int write_streams();
int on_rx_secret(ngtcp2_crypto_level level, const uint8_t *secret,
size_t secretlen);
int on_tx_secret(ngtcp2_crypto_level level, const uint8_t *secret,
size_t secretlen);
int add_crypto_data(ngtcp2_crypto_level level, const uint8_t *data,
size_t datalen);
void set_tls_alert(uint8_t alert);
int handle_error();
int handle_expiry();
void reset_idle_timer();
void reset_timer();
int setup_httpconn();
void add_pending_downstream(std::unique_ptr<Downstream> downstream);
int recv_stream_data(uint32_t flags, int64_t stream_id, const uint8_t *data,
size_t datalen);
int acked_stream_data_offset(int64_t stream_id, uint64_t datalen);
int extend_max_stream_data(int64_t stream_id);
void extend_max_remote_streams_bidi(uint64_t max_streams);
int error_reply(Downstream *downstream, unsigned int status_code);
void http_begin_request_headers(int64_t stream_id);
int http_recv_request_header(Downstream *downstream, int32_t token,
nghttp3_rcbuf *name, nghttp3_rcbuf *value,
uint8_t flags);
int http_end_request_headers(Downstream *downstream);
int http_end_stream(Downstream *downstream);
void start_downstream(Downstream *downstream);
void initiate_downstream(Downstream *downstream);
int shutdown_stream(Downstream *downstream, uint64_t app_error_code);
int shutdown_stream_read(int64_t stream_id, uint64_t app_error_code);
int redirect_to_https(Downstream *downstream);
int http_stream_close(Downstream *downstream, uint64_t app_error_code);
void consume(int64_t stream_id, size_t nconsumed);
void remove_downstream(Downstream *downstream);
int stream_close(int64_t stream_id, uint64_t app_error_code);
void log_response_headers(Downstream *downstream,
const std::vector<nghttp3_nv> &nva) const;
int http_acked_stream_data(Downstream *downstream, size_t datalen);
int http_shutdown_stream_read(int64_t stream_id);
int http_reset_stream(int64_t stream_id, uint64_t app_error_code);
int http_send_stop_sending(int64_t stream_id, uint64_t app_error_code);
int http_recv_data(Downstream *downstream, const uint8_t *data,
size_t datalen);
private:
ClientHandler *handler_;
ev_timer timer_;
ev_timer idle_timer_;
ngtcp2_cid initial_client_dcid_;
ngtcp2_conn *conn_;
quic::Error last_error_;
uint8_t tls_alert_;
nghttp3_conn *httpconn_;
DownstreamQueue downstream_queue_;
};
} // namespace shrpx
#endif // SHRPX_HTTP3_UPSTREAM_H

314
src/shrpx_quic.cc Normal file
View File

@ -0,0 +1,314 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2021 Tatsuhiro Tsujikawa
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "shrpx_quic.h"
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <netinet/udp.h>
#include <array>
#include <chrono>
#include <ngtcp2/ngtcp2_crypto.h>
#include <nghttp3/nghttp3.h>
#include <openssl/rand.h>
#include "shrpx_config.h"
#include "shrpx_log.h"
#include "util.h"
#include "xsi_strerror.h"
using namespace nghttp2;
namespace shrpx {
ngtcp2_tstamp quic_timestamp() {
return std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::steady_clock::now().time_since_epoch())
.count();
}
int create_quic_server_socket(UpstreamAddr &faddr) {
std::array<char, STRERROR_BUFSIZE> errbuf;
int fd = -1;
int rv;
auto service = util::utos(faddr.port);
addrinfo hints{};
hints.ai_family = faddr.family;
hints.ai_socktype = SOCK_DGRAM;
hints.ai_flags = AI_PASSIVE;
#ifdef AI_ADDRCONFIG
hints.ai_flags |= AI_ADDRCONFIG;
#endif // AI_ADDRCONFIG
auto node =
faddr.host == StringRef::from_lit("*") ? nullptr : faddr.host.c_str();
addrinfo *res, *rp;
rv = getaddrinfo(node, service.c_str(), &hints, &res);
#ifdef AI_ADDRCONFIG
if (rv != 0) {
// Retry without AI_ADDRCONFIG
hints.ai_flags &= ~AI_ADDRCONFIG;
rv = getaddrinfo(node, service.c_str(), &hints, &res);
}
#endif // AI_ADDRCONFIG
if (rv != 0) {
LOG(FATAL) << "Unable to get IPv" << (faddr.family == AF_INET ? "4" : "6")
<< " address for " << faddr.host << ", port " << faddr.port
<< ": " << gai_strerror(rv);
return -1;
}
auto res_d = defer(freeaddrinfo, res);
std::array<char, NI_MAXHOST> host;
for (rp = res; rp; rp = rp->ai_next) {
rv = getnameinfo(rp->ai_addr, rp->ai_addrlen, host.data(), host.size(),
nullptr, 0, NI_NUMERICHOST);
if (rv != 0) {
LOG(WARN) << "getnameinfo() failed: " << gai_strerror(rv);
continue;
}
#ifdef SOCK_NONBLOCK
fd = socket(rp->ai_family, rp->ai_socktype | SOCK_NONBLOCK | SOCK_CLOEXEC,
rp->ai_protocol);
if (fd == -1) {
auto error = errno;
LOG(WARN) << "socket() syscall failed: "
<< xsi_strerror(error, errbuf.data(), errbuf.size());
continue;
}
#else // !SOCK_NONBLOCK
fd = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);
if (fd == -1) {
auto error = errno;
LOG(WARN) << "socket() syscall failed: "
<< xsi_strerror(error, errbuf.data(), errbuf.size());
continue;
}
util::make_socket_nonblocking(fd);
util::make_socket_closeonexec(fd);
#endif // !SOCK_NONBLOCK
int val = 1;
if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val,
static_cast<socklen_t>(sizeof(val))) == -1) {
auto error = errno;
LOG(WARN) << "Failed to set SO_REUSEADDR option to listener socket: "
<< xsi_strerror(error, errbuf.data(), errbuf.size());
close(fd);
continue;
}
if (setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &val,
static_cast<socklen_t>(sizeof(val))) == -1) {
auto error = errno;
LOG(WARN) << "Failed to set SO_REUSEPORT option to listener socket: "
<< xsi_strerror(error, errbuf.data(), errbuf.size());
close(fd);
continue;
}
if (faddr.family == AF_INET6) {
#ifdef IPV6_V6ONLY
if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &val,
static_cast<socklen_t>(sizeof(val))) == -1) {
auto error = errno;
LOG(WARN) << "Failed to set IPV6_V6ONLY option to listener socket: "
<< xsi_strerror(error, errbuf.data(), errbuf.size());
close(fd);
continue;
}
#endif // IPV6_V6ONLY
if (setsockopt(fd, IPPROTO_IPV6, IPV6_RECVPKTINFO, &val,
static_cast<socklen_t>(sizeof(val))) == -1) {
auto error = errno;
LOG(WARN)
<< "Failed to set IPV6_RECVPKTINFO option to listener socket: "
<< xsi_strerror(error, errbuf.data(), errbuf.size());
close(fd);
continue;
}
} else {
if (setsockopt(fd, IPPROTO_IP, IP_PKTINFO, &val,
static_cast<socklen_t>(sizeof(val))) == -1) {
auto error = errno;
LOG(WARN) << "Failed to set IP_PKTINFO option to listener socket: "
<< xsi_strerror(error, errbuf.data(), errbuf.size());
close(fd);
continue;
}
}
// TODO Enable ECN
if (bind(fd, rp->ai_addr, rp->ai_addrlen) == -1) {
auto error = errno;
LOG(WARN) << "bind() syscall failed: "
<< xsi_strerror(error, errbuf.data(), errbuf.size());
close(fd);
continue;
}
break;
}
if (!rp) {
LOG(FATAL) << "Listening " << (faddr.family == AF_INET ? "IPv4" : "IPv6")
<< " socket failed";
return -1;
}
faddr.fd = fd;
faddr.hostport = util::make_http_hostport(mod_config()->balloc,
StringRef{host.data()}, faddr.port);
LOG(NOTICE) << "Listening on " << faddr.hostport << ", quic";
return 0;
}
int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
size_t remote_salen, const sockaddr *local_sa,
size_t local_salen, const uint8_t *data, size_t datalen,
size_t gso_size) {
iovec msg_iov = {const_cast<uint8_t *>(data), datalen};
msghdr msg{};
msg.msg_name = const_cast<sockaddr *>(remote_sa);
msg.msg_namelen = remote_salen;
msg.msg_iov = &msg_iov;
msg.msg_iovlen = 1;
uint8_t msg_ctrl[
#ifdef UDP_SEGMENT
CMSG_SPACE(sizeof(uint16_t)) +
#endif // UDP_SEGMENT
CMSG_SPACE(sizeof(in6_pktinfo))];
memset(msg_ctrl, 0, sizeof(msg_ctrl));
msg.msg_control = msg_ctrl;
msg.msg_controllen = sizeof(msg_ctrl);
size_t controllen = 0;
auto cm = CMSG_FIRSTHDR(&msg);
switch (local_sa->sa_family) {
case AF_INET: {
controllen += CMSG_SPACE(sizeof(in_pktinfo));
cm->cmsg_level = IPPROTO_IP;
cm->cmsg_type = IP_PKTINFO;
cm->cmsg_len = CMSG_LEN(sizeof(in_pktinfo));
auto pktinfo = reinterpret_cast<in_pktinfo *>(CMSG_DATA(cm));
memset(pktinfo, 0, sizeof(in_pktinfo));
auto addrin =
reinterpret_cast<sockaddr_in *>(const_cast<sockaddr *>(local_sa));
pktinfo->ipi_spec_dst = addrin->sin_addr;
break;
}
case AF_INET6: {
controllen += CMSG_SPACE(sizeof(in6_pktinfo));
cm->cmsg_level = IPPROTO_IPV6;
cm->cmsg_type = IPV6_PKTINFO;
cm->cmsg_len = CMSG_LEN(sizeof(in6_pktinfo));
auto pktinfo = reinterpret_cast<in6_pktinfo *>(CMSG_DATA(cm));
memset(pktinfo, 0, sizeof(in6_pktinfo));
auto addrin =
reinterpret_cast<sockaddr_in6 *>(const_cast<sockaddr *>(local_sa));
pktinfo->ipi6_addr = addrin->sin6_addr;
break;
}
default:
assert(0);
}
#ifdef UDP_SEGMENT
if (gso_size && datalen > gso_size) {
controllen += CMSG_SPACE(sizeof(uint16_t));
cm = CMSG_NXTHDR(&msg, cm);
cm->cmsg_level = SOL_UDP;
cm->cmsg_type = UDP_SEGMENT;
cm->cmsg_len = CMSG_LEN(sizeof(uint16_t));
*(reinterpret_cast<uint16_t *>(CMSG_DATA(cm))) = gso_size;
}
#endif // UDP_SEGMENT
msg.msg_controllen = controllen;
ssize_t nwrite;
do {
nwrite = sendmsg(faddr->fd, &msg, 0);
} while (nwrite == -1 && errno == EINTR);
if (nwrite == -1) {
return -1;
}
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "QUIC sent packet: local="
<< util::to_numeric_addr(local_sa, local_salen)
<< " remote=" << util::to_numeric_addr(remote_sa, remote_salen)
<< " " << nwrite << " bytes";
}
return 0;
}
int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen) {
if (RAND_bytes(cid->data, cidlen) != 1) {
return -1;
}
cid->datalen = cidlen;
return 0;
}
int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid,
const uint8_t *secret,
size_t secretlen) {
ngtcp2_crypto_md md;
ngtcp2_crypto_md_init(&md, const_cast<EVP_MD *>(EVP_sha256()));
if (ngtcp2_crypto_generate_stateless_reset_token(token, &md, secret,
secretlen, cid) != 0) {
return -1;
}
return 0;
}
} // namespace shrpx

58
src/shrpx_quic.h Normal file
View File

@ -0,0 +1,58 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2021 Tatsuhiro Tsujikawa
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef SHRPX_QUIC_H
#define SHRPX_QUIC_H
#include "shrpx.h"
#include <stdint.h>
#include <ngtcp2/ngtcp2.h>
namespace shrpx {
struct UpstreamAddr;
constexpr size_t SHRPX_QUIC_SCIDLEN = 20;
constexpr size_t SHRPX_MAX_UDP_PAYLOAD_SIZE = 1280;
ngtcp2_tstamp quic_timestamp();
int create_quic_server_socket(UpstreamAddr &addr);
int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
size_t remote_salen, const sockaddr *local_sa,
size_t local_salen, const uint8_t *data, size_t datalen,
size_t gso_size);
int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen);
int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid,
const uint8_t *secret,
size_t secretlen);
} // namespace shrpx
#endif // SHRPX_QUIC_H

View File

@ -0,0 +1,286 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2021 Tatsuhiro Tsujikawa
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "shrpx_quic_connection_handler.h"
#include <openssl/rand.h>
#include <ngtcp2/ngtcp2.h>
#include "shrpx_worker.h"
#include "shrpx_client_handler.h"
#include "shrpx_log.h"
#include "shrpx_quic.h"
#include "shrpx_http3_upstream.h"
namespace shrpx {
QUICConnectionHandler::QUICConnectionHandler(Worker *worker)
: worker_{worker} {}
QUICConnectionHandler::~QUICConnectionHandler() {}
namespace {
std::string make_cid_key(const uint8_t *dcid, size_t dcidlen) {
return std::string{dcid, dcid + dcidlen};
}
} // namespace
namespace {
std::string make_cid_key(const ngtcp2_cid *cid) {
return make_cid_key(cid->data, cid->datalen);
}
} // namespace
int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
const Address &remote_addr,
const Address &local_addr,
const uint8_t *data, size_t datalen) {
int rv;
uint32_t version;
const uint8_t *dcid, *scid;
size_t dcidlen, scidlen;
rv = ngtcp2_pkt_decode_version_cid(&version, &dcid, &dcidlen, &scid, &scidlen,
data, datalen, SHRPX_QUIC_SCIDLEN);
if (rv != 0) {
if (rv == 1) {
send_version_negotiation(faddr, version, scid, scidlen, dcid, dcidlen,
remote_addr, local_addr);
}
return 0;
}
auto dcid_key = make_cid_key(dcid, dcidlen);
ClientHandler *handler;
auto it = connections_.find(dcid_key);
if (it == std::end(connections_)) {
// new connection
ngtcp2_pkt_hd hd;
switch (ngtcp2_accept(&hd, data, datalen)) {
case 0:
break;
case NGTCP2_ERR_RETRY:
// TODO Send retry
return 0;
case NGTCP2_ERR_VERSION_NEGOTIATION:
send_version_negotiation(faddr, version, scid, scidlen, dcid, dcidlen,
remote_addr, local_addr);
return 0;
default:
// TODO Must be rate limited
send_stateless_reset(faddr, dcid, dcidlen, remote_addr, local_addr);
return 0;
}
handler = handle_new_connection(faddr, remote_addr, local_addr, hd);
if (handler == nullptr) {
return 0;
}
} else {
handler = (*it).second;
}
if (handler->read_quic(faddr, remote_addr, local_addr, data, datalen) != 0) {
delete handler;
return 0;
}
handler->signal_write();
return 0;
}
ClientHandler *QUICConnectionHandler::handle_new_connection(
const UpstreamAddr *faddr, const Address &remote_addr,
const Address &local_addr, const ngtcp2_pkt_hd &hd) {
std::array<char, NI_MAXHOST> host;
std::array<char, NI_MAXSERV> service;
int rv;
rv = getnameinfo(&remote_addr.su.sa, remote_addr.len, host.data(),
host.size(), service.data(), service.size(),
NI_NUMERICHOST | NI_NUMERICSERV);
if (rv != 0) {
LOG(ERROR) << "getnameinfo() failed: " << gai_strerror(rv);
return nullptr;
}
auto ssl_ctx = worker_->get_quic_sv_ssl_ctx();
assert(ssl_ctx);
auto ssl = tls::create_ssl(ssl_ctx);
if (ssl == nullptr) {
return nullptr;
}
assert(SSL_is_quic(ssl));
SSL_set_accept_state(ssl);
SSL_set_quic_early_data_enabled(ssl, 1);
// Disable TLS session ticket if we don't have working ticket
// keys.
if (!worker_->get_ticket_keys()) {
SSL_set_options(ssl, SSL_OP_NO_TICKET);
}
auto handler = std::make_unique<ClientHandler>(
worker_, faddr->fd, ssl, StringRef{host.data()},
StringRef{service.data()}, remote_addr.su.sa.sa_family, faddr);
auto upstream = std::make_unique<Http3Upstream>(handler.get());
if (upstream->init(faddr, remote_addr, local_addr, hd) != 0) {
return nullptr;
}
handler->setup_http3_upstream(std::move(upstream));
return handler.release();
}
namespace {
uint32_t generate_reserved_version(const Address &addr, uint32_t version) {
uint32_t h = 0x811C9DC5u;
const uint8_t *p = reinterpret_cast<const uint8_t *>(&addr.su.sa);
const uint8_t *ep = p + addr.len;
for (; p != ep; ++p) {
h ^= *p;
h *= 0x01000193u;
}
version = htonl(version);
p = (const uint8_t *)&version;
ep = p + sizeof(version);
for (; p != ep; ++p) {
h ^= *p;
h *= 0x01000193u;
}
h &= 0xf0f0f0f0u;
h |= 0x0a0a0a0au;
return h;
}
} // namespace
int QUICConnectionHandler::send_version_negotiation(
const UpstreamAddr *faddr, uint32_t version, const uint8_t *dcid,
size_t dcidlen, const uint8_t *scid, size_t scidlen,
const Address &remote_addr, const Address &local_addr) {
std::array<uint32_t, 2> sv;
sv[0] = generate_reserved_version(remote_addr, version);
sv[1] = NGTCP2_PROTO_VER_V1;
std::array<uint8_t, 1280> buf;
uint8_t rand_byte;
util::random_bytes(&rand_byte, &rand_byte + 1, worker_->get_randgen());
auto nwrite = ngtcp2_pkt_write_version_negotiation(
buf.data(), buf.size(), rand_byte, dcid, dcidlen, scid, scidlen,
sv.data(), sv.size());
if (nwrite < 0) {
LOG(ERROR) << "ngtcp2_pkt_write_version_negotiation: "
<< ngtcp2_strerror(nwrite);
return -1;
}
return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
&local_addr.su.sa, local_addr.len, buf.data(), nwrite,
0);
}
int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr,
const uint8_t *dcid,
size_t dcidlen,
const Address &remote_addr,
const Address &local_addr) {
int rv;
std::array<uint8_t, NGTCP2_STATELESS_RESET_TOKENLEN> token;
ngtcp2_cid cid;
ngtcp2_cid_init(&cid, dcid, dcidlen);
auto config = get_config();
auto &quicconf = config->quic;
auto &stateless_resetconf = quicconf.stateless_reset;
rv = generate_quic_stateless_reset_token(token.data(), &cid,
stateless_resetconf.secret.data(),
stateless_resetconf.secret.size());
if (rv != 0) {
return -1;
}
std::array<uint8_t, NGTCP2_MIN_STATELESS_RESET_RANDLEN> rand_bytes;
if (RAND_bytes(rand_bytes.data(), rand_bytes.size()) != 1) {
return -1;
}
std::array<uint8_t, 1280> buf;
auto nwrite =
ngtcp2_pkt_write_stateless_reset(buf.data(), buf.size(), token.data(),
rand_bytes.data(), rand_bytes.size());
if (nwrite < 0) {
LOG(ERROR) << "ngtcp2_pkt_write_stateless_reset: "
<< ngtcp2_strerror(nwrite);
return -1;
}
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Send stateless_reset to remote="
<< util::to_numeric_addr(&remote_addr)
<< " dcid=" << util::format_hex(dcid, dcidlen);
}
return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
&local_addr.su.sa, local_addr.len, buf.data(), nwrite,
0);
}
void QUICConnectionHandler::add_connection_id(const ngtcp2_cid *cid,
ClientHandler *handler) {
auto key = make_cid_key(cid);
connections_.emplace(key, handler);
}
void QUICConnectionHandler::remove_connection_id(const ngtcp2_cid *cid) {
auto key = make_cid_key(cid);
connections_.erase(key);
}
} // namespace shrpx

View File

@ -0,0 +1,75 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2021 Tatsuhiro Tsujikawa
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef SHRPX_QUIC_CONNECTION_HANDLER_H
#define SHRPX_QUIC_CONNECTION_HANDLER_H
#include "shrpx.h"
#include <memory>
#include <unordered_map>
#include <string>
#include <ngtcp2/ngtcp2.h>
#include "network.h"
using namespace nghttp2;
namespace shrpx {
struct UpstreamAddr;
class ClientHandler;
class Worker;
class QUICConnectionHandler {
public:
QUICConnectionHandler(Worker *worker);
~QUICConnectionHandler();
int handle_packet(const UpstreamAddr *faddr, const Address &remote_addr,
const Address &local_addr, const uint8_t *data,
size_t datalen);
int send_version_negotiation(const UpstreamAddr *faddr, uint32_t version,
const uint8_t *dcid, size_t dcidlen,
const uint8_t *scid, size_t scidlen,
const Address &remote_addr,
const Address &local_addr);
int send_stateless_reset(const UpstreamAddr *faddr, const uint8_t *dcid,
size_t dcidlen, const Address &remote_addr,
const Address &local_addr);
ClientHandler *handle_new_connection(const UpstreamAddr *faddr,
const Address &remote_addr,
const Address &local_addr,
const ngtcp2_pkt_hd &hd);
void add_connection_id(const ngtcp2_cid *cid, ClientHandler *handler);
void remove_connection_id(const ngtcp2_cid *cid);
private:
Worker *worker_;
std::unordered_map<std::string, ClientHandler *> connections_;
};
} // namespace shrpx
#endif // SHRPX_QUIC_CONNECTION_HANDLER_H

106
src/shrpx_quic_listener.cc Normal file
View File

@ -0,0 +1,106 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2021 Tatsuhiro Tsujikawa
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "shrpx_quic_listener.h"
#include "shrpx_worker.h"
#include "shrpx_config.h"
#include "shrpx_log.h"
namespace shrpx {
namespace {
void readcb(struct ev_loop *loop, ev_io *w, int revent) {
auto l = static_cast<QUICListener *>(w->data);
l->on_read();
}
} // namespace
QUICListener::QUICListener(const UpstreamAddr *faddr, Worker *worker)
: faddr_{faddr}, worker_{worker} {
ev_io_init(&rev_, readcb, faddr_->fd, EV_READ);
rev_.data = this;
ev_io_start(worker_->get_loop(), &rev_);
}
QUICListener::~QUICListener() {
ev_io_stop(worker_->get_loop(), &rev_);
close(faddr_->fd);
}
void QUICListener::on_read() {
sockaddr_union su;
std::array<uint8_t, 64_k> buf;
size_t pktcnt = 0;
iovec msg_iov{buf.data(), buf.size()};
msghdr msg{};
msg.msg_name = &su;
msg.msg_iov = &msg_iov;
msg.msg_iovlen = 1;
uint8_t msg_ctrl[CMSG_SPACE(sizeof(in6_pktinfo))];
msg.msg_control = msg_ctrl;
auto quic_conn_handler = worker_->get_quic_connection_handler();
for (; pktcnt < 10;) {
msg.msg_namelen = sizeof(su);
msg.msg_controllen = sizeof(msg_ctrl);
auto nread = recvmsg(faddr_->fd, &msg, 0);
if (nread == -1) {
return;
}
++pktcnt;
Address local_addr{};
if (util::msghdr_get_local_addr(local_addr, &msg, su.storage.ss_family) !=
0) {
continue;
}
util::set_port(local_addr, faddr_->port);
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "QUIC received packet: local="
<< util::to_numeric_addr(&local_addr)
<< " remote=" << util::to_numeric_addr(&su.sa, msg.msg_namelen)
<< " " << nread << " bytes";
}
if (nread == 0) {
continue;
}
Address remote_addr;
remote_addr.su = su;
remote_addr.len = msg.msg_namelen;
quic_conn_handler->handle_packet(faddr_, remote_addr, local_addr,
buf.data(), nread);
}
}
} // namespace shrpx

51
src/shrpx_quic_listener.h Normal file
View File

@ -0,0 +1,51 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2021 Tatsuhiro Tsujikawa
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef SHRPX_QUIC_LISTENER_H
#define SHRPX_QUIC_LISTENER_H
#include "shrpx.h"
#include <ev.h>
namespace shrpx {
struct UpstreamAddr;
class Worker;
class QUICListener {
public:
QUICListener(const UpstreamAddr *faddr, Worker *worker);
~QUICListener();
void on_read();
private:
const UpstreamAddr *faddr_;
Worker *worker_;
ev_io rev_;
};
} // namespace shrpx
#endif // SHRPX_QUIC_LISTENER_H

View File

@ -51,6 +51,12 @@
#include <nghttp2/nghttp2.h>
#ifdef ENABLE_HTTP3
# include <ngtcp2/ngtcp2.h>
# include <ngtcp2/ngtcp2_crypto.h>
# include <ngtcp2/ngtcp2_crypto_openssl.h>
#endif // ENABLE_HTTP3
#include "shrpx_log.h"
#include "shrpx_client_handler.h"
#include "shrpx_config.h"
@ -60,6 +66,9 @@
#include "shrpx_memcached_request.h"
#include "shrpx_memcached_dispatcher.h"
#include "shrpx_connection_handler.h"
#ifdef ENABLE_HTTP3
# include "shrpx_http3_upstream.h"
#endif // ENABLE_HTTP3
#include "util.h"
#include "tls.h"
#include "template.h"
@ -180,7 +189,12 @@ int servername_callback(SSL *ssl, int *al, void *arg) {
auto hostname = StringRef{std::begin(buf), end_buf};
#ifdef ENABLE_HTTP3
auto cert_tree = SSL_is_quic(ssl) ? worker->get_quic_cert_lookup_tree()
: worker->get_cert_lookup_tree();
#else // !ENABLE_HTTP3
auto cert_tree = worker->get_cert_lookup_tree();
#endif // !ENABLE_HTTP3
auto idx = cert_tree->lookup(hostname);
if (idx == -1) {
@ -191,7 +205,14 @@ int servername_callback(SSL *ssl, int *al, void *arg) {
auto conn_handler = worker->get_connection_handler();
#ifdef ENABLE_HTTP3
const auto &ssl_ctx_list = SSL_is_quic(ssl)
? conn_handler->get_quic_indexed_ssl_ctx(idx)
: conn_handler->get_indexed_ssl_ctx(idx);
#else // !ENABLE_HTTP3
const auto &ssl_ctx_list = conn_handler->get_indexed_ssl_ctx(idx);
#endif // !ENABLE_HTTP3
assert(!ssl_ctx_list.empty());
#if !defined(OPENSSL_IS_BORINGSSL) && !LIBRESSL_IN_USE && \
@ -600,6 +621,34 @@ int alpn_select_proto_cb(SSL *ssl, const unsigned char **out,
} // namespace
#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
#ifdef ENABLE_HTTP3
# if OPENSSL_VERSION_NUMBER >= 0x10002000L
namespace {
int quic_alpn_select_proto_cb(SSL *ssl, const unsigned char **out,
unsigned char *outlen, const unsigned char *in,
unsigned int inlen, void *arg) {
for (auto p = in, end = in + inlen; p < end;) {
auto proto_id = p + 1;
auto proto_len = *p;
if (proto_id + proto_len <= end &&
util::streq_l("h3", StringRef{proto_id, proto_len})) {
*out = reinterpret_cast<const unsigned char *>(proto_id);
*outlen = proto_len;
return SSL_TLSEXT_ERR_OK;
}
p += 1 + proto_len;
}
return SSL_TLSEXT_ERR_NOACK;
}
} // namespace
# endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
#endif // ENABLE_HTTP3
#if !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L
# ifndef TLSEXT_TYPE_signed_certificate_timestamp
@ -1042,6 +1091,328 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file,
return ssl_ctx;
}
#ifdef ENABLE_HTTP3
namespace {
int quic_set_encryption_secrets(SSL *ssl, OSSL_ENCRYPTION_LEVEL ossl_level,
const uint8_t *rx_secret,
const uint8_t *tx_secret, size_t secretlen) {
auto conn = static_cast<Connection *>(SSL_get_app_data(ssl));
auto handler = static_cast<ClientHandler *>(conn->data);
auto upstream = static_cast<Http3Upstream *>(handler->get_upstream());
auto level = ngtcp2_crypto_openssl_from_ossl_encryption_level(ossl_level);
if (upstream->on_rx_secret(level, rx_secret, secretlen) != 0) {
return 0;
}
if (tx_secret && upstream->on_tx_secret(level, tx_secret, secretlen) != 0) {
return 0;
}
return 1;
}
} // namespace
namespace {
int quic_add_handshake_data(SSL *ssl, OSSL_ENCRYPTION_LEVEL ossl_level,
const uint8_t *data, size_t len) {
auto conn = static_cast<Connection *>(SSL_get_app_data(ssl));
auto handler = static_cast<ClientHandler *>(conn->data);
auto upstream = static_cast<Http3Upstream *>(handler->get_upstream());
auto level = ngtcp2_crypto_openssl_from_ossl_encryption_level(ossl_level);
upstream->add_crypto_data(level, data, len);
return 1;
}
} // namespace
namespace {
int quic_flush_flight(SSL *ssl) { return 1; }
} // namespace
namespace {
int quic_send_alert(SSL *ssl, OSSL_ENCRYPTION_LEVEL ossl_level, uint8_t alert) {
auto conn = static_cast<Connection *>(SSL_get_app_data(ssl));
auto handler = static_cast<ClientHandler *>(conn->data);
auto upstream = static_cast<Http3Upstream *>(handler->get_upstream());
if (!upstream) {
return 1;
}
upstream->set_tls_alert(alert);
return 1;
}
} // namespace
namespace {
auto quic_method = SSL_QUIC_METHOD{
quic_set_encryption_secrets,
quic_add_handshake_data,
quic_flush_flight,
quic_send_alert,
};
} // namespace
SSL_CTX *create_quic_ssl_context(const char *private_key_file,
const char *cert_file,
const std::vector<uint8_t> &sct_data
# ifdef HAVE_NEVERBLEED
,
neverbleed_t *nb
# endif // HAVE_NEVERBLEED
) {
auto ssl_ctx = SSL_CTX_new(TLS_server_method());
if (!ssl_ctx) {
LOG(FATAL) << ERR_error_string(ERR_get_error(), nullptr);
DIE();
}
constexpr auto ssl_opts =
(SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS) |
SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION | SSL_OP_SINGLE_ECDH_USE |
SSL_OP_SINGLE_DH_USE |
SSL_OP_CIPHER_SERVER_PREFERENCE
# if OPENSSL_1_1_1_API
// The reason for disabling built-in anti-replay in OpenSSL is
// that it only works if client gets back to the same server.
// The freshness check described in
// https://tools.ietf.org/html/rfc8446#section-8.3 is still
// performed.
| SSL_OP_NO_ANTI_REPLAY
# endif // OPENSSL_1_1_1_API
;
auto config = mod_config();
auto &tlsconf = config->tls;
SSL_CTX_set_options(ssl_ctx, ssl_opts);
SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_3_VERSION);
SSL_CTX_set_max_proto_version(ssl_ctx, TLS1_3_VERSION);
const unsigned char sid_ctx[] = "shrpx";
SSL_CTX_set_session_id_context(ssl_ctx, sid_ctx, sizeof(sid_ctx) - 1);
SSL_CTX_set_session_cache_mode(ssl_ctx, SSL_SESS_CACHE_SERVER);
if (!tlsconf.session_cache.memcached.host.empty()) {
SSL_CTX_sess_set_new_cb(ssl_ctx, tls_session_new_cb);
SSL_CTX_sess_set_get_cb(ssl_ctx, tls_session_get_cb);
}
SSL_CTX_set_timeout(ssl_ctx, tlsconf.session_timeout.count());
if (SSL_CTX_set_cipher_list(ssl_ctx, tlsconf.ciphers.c_str()) == 0) {
LOG(FATAL) << "SSL_CTX_set_cipher_list " << tlsconf.ciphers
<< " failed: " << ERR_error_string(ERR_get_error(), nullptr);
DIE();
}
# if OPENSSL_1_1_1_API
if (SSL_CTX_set_ciphersuites(ssl_ctx, tlsconf.tls13_ciphers.c_str()) == 0) {
LOG(FATAL) << "SSL_CTX_set_ciphersuites " << tlsconf.tls13_ciphers
<< " failed: " << ERR_error_string(ERR_get_error(), nullptr);
DIE();
}
# endif // OPENSSL_1_1_1_API
# ifndef OPENSSL_NO_EC
# if !LIBRESSL_LEGACY_API && OPENSSL_VERSION_NUMBER >= 0x10002000L
if (SSL_CTX_set1_curves_list(ssl_ctx, tlsconf.ecdh_curves.c_str()) != 1) {
LOG(FATAL) << "SSL_CTX_set1_curves_list " << tlsconf.ecdh_curves
<< " failed";
DIE();
}
# if !defined(OPENSSL_IS_BORINGSSL) && !OPENSSL_1_1_API
// It looks like we need this function call for OpenSSL 1.0.2. This
// function was deprecated in OpenSSL 1.1.0 and BoringSSL.
SSL_CTX_set_ecdh_auto(ssl_ctx, 1);
# endif // !defined(OPENSSL_IS_BORINGSSL) && !OPENSSL_1_1_API
# else // LIBRESSL_LEGACY_API || OPENSSL_VERSION_NUBMER < 0x10002000L
// Use P-256, which is sufficiently secure at the time of this
// writing.
auto ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1);
if (ecdh == nullptr) {
LOG(FATAL) << "EC_KEY_new_by_curv_name failed: "
<< ERR_error_string(ERR_get_error(), nullptr);
DIE();
}
SSL_CTX_set_tmp_ecdh(ssl_ctx, ecdh);
EC_KEY_free(ecdh);
# endif // LIBRESSL_LEGACY_API || OPENSSL_VERSION_NUBMER < 0x10002000L
# endif // OPENSSL_NO_EC
if (!tlsconf.dh_param_file.empty()) {
// Read DH parameters from file
auto bio = BIO_new_file(tlsconf.dh_param_file.c_str(), "r");
if (bio == nullptr) {
LOG(FATAL) << "BIO_new_file() failed: "
<< ERR_error_string(ERR_get_error(), nullptr);
DIE();
}
auto dh = PEM_read_bio_DHparams(bio, nullptr, nullptr, nullptr);
if (dh == nullptr) {
LOG(FATAL) << "PEM_read_bio_DHparams() failed: "
<< ERR_error_string(ERR_get_error(), nullptr);
DIE();
}
SSL_CTX_set_tmp_dh(ssl_ctx, dh);
DH_free(dh);
BIO_free(bio);
}
SSL_CTX_set_mode(ssl_ctx, SSL_MODE_RELEASE_BUFFERS);
if (SSL_CTX_set_default_verify_paths(ssl_ctx) != 1) {
LOG(WARN) << "Could not load system trusted ca certificates: "
<< ERR_error_string(ERR_get_error(), nullptr);
}
if (!tlsconf.cacert.empty()) {
if (SSL_CTX_load_verify_locations(ssl_ctx, tlsconf.cacert.c_str(),
nullptr) != 1) {
LOG(FATAL) << "Could not load trusted ca certificates from "
<< tlsconf.cacert << ": "
<< ERR_error_string(ERR_get_error(), nullptr);
DIE();
}
}
if (!tlsconf.private_key_passwd.empty()) {
SSL_CTX_set_default_passwd_cb(ssl_ctx, ssl_pem_passwd_cb);
SSL_CTX_set_default_passwd_cb_userdata(ssl_ctx, config);
}
# ifndef HAVE_NEVERBLEED
if (SSL_CTX_use_PrivateKey_file(ssl_ctx, private_key_file,
SSL_FILETYPE_PEM) != 1) {
LOG(FATAL) << "SSL_CTX_use_PrivateKey_file failed: "
<< ERR_error_string(ERR_get_error(), nullptr);
}
# else // HAVE_NEVERBLEED
std::array<char, NEVERBLEED_ERRBUF_SIZE> errbuf;
if (neverbleed_load_private_key_file(nb, ssl_ctx, private_key_file,
errbuf.data()) != 1) {
LOG(FATAL) << "neverbleed_load_private_key_file failed: " << errbuf.data();
DIE();
}
# endif // HAVE_NEVERBLEED
if (SSL_CTX_use_certificate_chain_file(ssl_ctx, cert_file) != 1) {
LOG(FATAL) << "SSL_CTX_use_certificate_file failed: "
<< ERR_error_string(ERR_get_error(), nullptr);
DIE();
}
if (SSL_CTX_check_private_key(ssl_ctx) != 1) {
LOG(FATAL) << "SSL_CTX_check_private_key failed: "
<< ERR_error_string(ERR_get_error(), nullptr);
DIE();
}
if (tlsconf.client_verify.enabled) {
if (!tlsconf.client_verify.cacert.empty()) {
if (SSL_CTX_load_verify_locations(
ssl_ctx, tlsconf.client_verify.cacert.c_str(), nullptr) != 1) {
LOG(FATAL) << "Could not load trusted ca certificates from "
<< tlsconf.client_verify.cacert << ": "
<< ERR_error_string(ERR_get_error(), nullptr);
DIE();
}
// It is heard that SSL_CTX_load_verify_locations() may leave
// error even though it returns success. See
// http://forum.nginx.org/read.php?29,242540
ERR_clear_error();
auto list = SSL_load_client_CA_file(tlsconf.client_verify.cacert.c_str());
if (!list) {
LOG(FATAL) << "Could not load ca certificates from "
<< tlsconf.client_verify.cacert << ": "
<< ERR_error_string(ERR_get_error(), nullptr);
DIE();
}
SSL_CTX_set_client_CA_list(ssl_ctx, list);
}
SSL_CTX_set_verify(ssl_ctx,
SSL_VERIFY_PEER | SSL_VERIFY_CLIENT_ONCE |
SSL_VERIFY_FAIL_IF_NO_PEER_CERT,
verify_callback);
}
SSL_CTX_set_tlsext_servername_callback(ssl_ctx, servername_callback);
SSL_CTX_set_tlsext_ticket_key_cb(ssl_ctx, ticket_key_cb);
# ifndef OPENSSL_IS_BORINGSSL
SSL_CTX_set_tlsext_status_cb(ssl_ctx, ocsp_resp_cb);
# endif // OPENSSL_IS_BORINGSSL
# ifdef OPENSSL_IS_BORINGSSL
SSL_CTX_set_early_data_enabled(ssl_ctx, 1);
# endif // OPENSSL_IS_BORINGSSL
# if OPENSSL_VERSION_NUMBER >= 0x10002000L
// ALPN selection callback
SSL_CTX_set_alpn_select_cb(ssl_ctx, quic_alpn_select_proto_cb, nullptr);
# endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
# if !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L && \
!defined(OPENSSL_IS_BORINGSSL)
// SSL_extension_supported(TLSEXT_TYPE_signed_certificate_timestamp)
// returns 1, which means OpenSSL internally handles it. But
// OpenSSL handles signed_certificate_timestamp extension specially,
// and it lets custom handler to process the extension.
if (!sct_data.empty()) {
# if OPENSSL_1_1_1_API
// It is not entirely clear to me that SSL_EXT_CLIENT_HELLO is
// required here. sct_parse_cb is called without
// SSL_EXT_CLIENT_HELLO being set. But the passed context value
// is SSL_EXT_CLIENT_HELLO.
if (SSL_CTX_add_custom_ext(
ssl_ctx, TLSEXT_TYPE_signed_certificate_timestamp,
SSL_EXT_CLIENT_HELLO | SSL_EXT_TLS1_2_SERVER_HELLO |
SSL_EXT_TLS1_3_CERTIFICATE | SSL_EXT_IGNORE_ON_RESUMPTION,
sct_add_cb, sct_free_cb, nullptr, sct_parse_cb, nullptr) != 1) {
LOG(FATAL) << "SSL_CTX_add_custom_ext failed: "
<< ERR_error_string(ERR_get_error(), nullptr);
DIE();
}
# else // !OPENSSL_1_1_1_API
if (SSL_CTX_add_server_custom_ext(
ssl_ctx, TLSEXT_TYPE_signed_certificate_timestamp,
legacy_sct_add_cb, legacy_sct_free_cb, nullptr, legacy_sct_parse_cb,
nullptr) != 1) {
LOG(FATAL) << "SSL_CTX_add_server_custom_ext failed: "
<< ERR_error_string(ERR_get_error(), nullptr);
DIE();
}
# endif // !OPENSSL_1_1_1_API
}
# endif // !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L &&
// !defined(OPENSSL_IS_BORINGSSL)
# if OPENSSL_1_1_1_API
if (SSL_CTX_set_max_early_data(ssl_ctx,
std::numeric_limits<uint32_t>::max()) != 1) {
LOG(FATAL) << "SSL_CTX_set_max_early_data failed: "
<< ERR_error_string(ERR_get_error(), nullptr);
DIE();
}
# endif // OPENSSL_1_1_1_API
# ifndef OPENSSL_NO_PSK
SSL_CTX_set_psk_server_callback(ssl_ctx, psk_server_cb);
# endif // !LIBRESSL_NO_PSK
SSL_CTX_set_quic_method(ssl_ctx, &quic_method);
auto tls_ctx_data = new TLSContextData();
tls_ctx_data->cert_file = cert_file;
tls_ctx_data->sct_data = sct_data;
SSL_CTX_set_app_data(ssl_ctx, tls_ctx_data);
return ssl_ctx;
}
#endif // ENABLE_HTTP3
namespace {
int select_h2_next_proto_cb(SSL *ssl, unsigned char **out,
unsigned char *outlen, const unsigned char *in,
@ -1747,6 +2118,12 @@ bool in_proto_list(const std::vector<StringRef> &protos,
}
bool upstream_tls_enabled(const ConnectionConfig &connconf) {
#ifdef ENABLE_HTTP3
if (connconf.quic_listener.addrs.size()) {
return true;
}
#endif // ENABLE_HTTP3
const auto &faddrs = connconf.listener.addrs;
return std::any_of(std::begin(faddrs), std::end(faddrs),
[](const UpstreamAddr &faddr) { return faddr.tls; });
@ -1826,6 +2203,63 @@ setup_server_ssl_context(std::vector<SSL_CTX *> &all_ssl_ctx,
return ssl_ctx;
}
#ifdef ENABLE_HTTP3
SSL_CTX *setup_quic_server_ssl_context(
std::vector<SSL_CTX *> &all_ssl_ctx,
std::vector<std::vector<SSL_CTX *>> &indexed_ssl_ctx,
CertLookupTree *cert_tree
# ifdef HAVE_NEVERBLEED
,
neverbleed_t *nb
# endif // HAVE_NEVERBLEED
) {
auto config = get_config();
if (!upstream_tls_enabled(config->conn)) {
return nullptr;
}
auto &tlsconf = config->tls;
auto ssl_ctx =
create_quic_ssl_context(tlsconf.private_key_file.c_str(),
tlsconf.cert_file.c_str(), tlsconf.sct_data
# ifdef HAVE_NEVERBLEED
,
nb
# endif // HAVE_NEVERBLEED
);
all_ssl_ctx.push_back(ssl_ctx);
assert(cert_tree);
if (cert_lookup_tree_add_ssl_ctx(cert_tree, indexed_ssl_ctx, ssl_ctx) == -1) {
LOG(FATAL) << "Failed to add default certificate.";
DIE();
}
for (auto &c : tlsconf.subcerts) {
auto ssl_ctx = create_quic_ssl_context(c.private_key_file.c_str(),
c.cert_file.c_str(), c.sct_data
# ifdef HAVE_NEVERBLEED
,
nb
# endif // HAVE_NEVERBLEED
);
all_ssl_ctx.push_back(ssl_ctx);
if (cert_lookup_tree_add_ssl_ctx(cert_tree, indexed_ssl_ctx, ssl_ctx) ==
-1) {
LOG(FATAL) << "Failed to add sub certificate.";
DIE();
}
}
return ssl_ctx;
}
#endif // ENABLE_HTTP3
SSL_CTX *setup_downstream_client_ssl_context(
#ifdef HAVE_NEVERBLEED
neverbleed_t *nb

View File

@ -100,6 +100,18 @@ SSL_CTX *create_ssl_client_context(
unsigned char *outlen, const unsigned char *in,
unsigned int inlen, void *arg));
#ifdef ENABLE_HTTP3
SSL_CTX *create_quic_ssl_client_context(
# ifdef HAVE_NEVERBLEED
neverbleed_t *nb,
# endif // HAVE_NEVERBLEED
const StringRef &cacert, const StringRef &cert_file,
const StringRef &private_key_file,
int (*next_proto_select_cb)(SSL *s, unsigned char **out,
unsigned char *outlen, const unsigned char *in,
unsigned int inlen, void *arg));
#endif // ENABLE_HTTP3
ClientHandler *accept_connection(Worker *worker, int fd, sockaddr *addr,
int addrlen, const UpstreamAddr *faddr);
@ -217,6 +229,18 @@ setup_server_ssl_context(std::vector<SSL_CTX *> &all_ssl_ctx,
#endif // HAVE_NEVERBLEED
);
#ifdef ENABLE_HTTP3
SSL_CTX *setup_quic_server_ssl_context(
std::vector<SSL_CTX *> &all_ssl_ctx,
std::vector<std::vector<SSL_CTX *>> &indexed_ssl_ctx,
CertLookupTree *cert_tree
# ifdef HAVE_NEVERBLEED
,
neverbleed_t *nb
# endif // HAVE_NEVERBLEED
);
#endif // ENABLE_HTTP3
// Setups client side SSL_CTX.
SSL_CTX *setup_downstream_client_ssl_context(
#ifdef HAVE_NEVERBLEED

View File

@ -39,6 +39,10 @@
#ifdef HAVE_MRUBY
# include "shrpx_mruby.h"
#endif // HAVE_MRUBY
#ifdef ENABLE_HTTP3
# include "shrpx_quic.h"
# include "shrpx_quic_listener.h"
#endif // ENABLE_HTTP3
#include "util.h"
#include "template.h"
@ -131,17 +135,28 @@ create_downstream_key(const std::shared_ptr<SharedDownstreamAddr> &shared_addr,
Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
SSL_CTX *tls_session_cache_memcached_ssl_ctx,
tls::CertLookupTree *cert_tree,
#ifdef ENABLE_HTTP3
SSL_CTX *quic_sv_ssl_ctx, tls::CertLookupTree *quic_cert_tree,
#endif // ENABLE_HTTP3
const std::shared_ptr<TicketKeys> &ticket_keys,
ConnectionHandler *conn_handler,
std::shared_ptr<DownstreamConfig> downstreamconf)
: randgen_(util::make_mt19937()),
worker_stat_{},
dns_tracker_(loop),
#ifdef ENABLE_HTTP3
quic_upstream_addrs_{get_config()->conn.quic_listener.addrs},
#endif // ENABLE_HTTP3
loop_(loop),
sv_ssl_ctx_(sv_ssl_ctx),
cl_ssl_ctx_(cl_ssl_ctx),
cert_tree_(cert_tree),
conn_handler_(conn_handler),
#ifdef ENABLE_HTTP3
quic_sv_ssl_ctx_{quic_sv_ssl_ctx},
quic_cert_tree_{quic_cert_tree},
quic_conn_handler_{this},
#endif // ENABLE_HTTP3
ticket_keys_(ticket_keys),
connect_blocker_(
std::make_unique<ConnectBlocker>(randgen_, loop_, nullptr, nullptr)),
@ -504,6 +519,12 @@ void Worker::process_events() {
tls::CertLookupTree *Worker::get_cert_lookup_tree() const { return cert_tree_; }
#ifdef ENABLE_HTTP3
tls::CertLookupTree *Worker::get_quic_cert_lookup_tree() const {
return quic_cert_tree_;
}
#endif // ENABLE_HTTP3
std::shared_ptr<TicketKeys> Worker::get_ticket_keys() {
#ifdef HAVE_ATOMIC_STD_SHARED_PTR
return std::atomic_load_explicit(&ticket_keys_, std::memory_order_acquire);
@ -534,6 +555,10 @@ SSL_CTX *Worker::get_sv_ssl_ctx() const { return sv_ssl_ctx_; }
SSL_CTX *Worker::get_cl_ssl_ctx() const { return cl_ssl_ctx_; }
#ifdef ENABLE_HTTP3
SSL_CTX *Worker::get_quic_sv_ssl_ctx() const { return quic_sv_ssl_ctx_; }
#endif // ENABLE_HTTP3
void Worker::set_graceful_shutdown(bool f) { graceful_shutdown_ = f; }
bool Worker::get_graceful_shutdown() const { return graceful_shutdown_; }
@ -578,8 +603,29 @@ ConnectionHandler *Worker::get_connection_handler() const {
return conn_handler_;
}
#ifdef ENABLE_HTTP3
QUICConnectionHandler *Worker::get_quic_connection_handler() {
return &quic_conn_handler_;
}
#endif // ENABLE_HTTP3
DNSTracker *Worker::get_dns_tracker() { return &dns_tracker_; }
#ifdef ENABLE_HTTP3
int Worker::setup_quic_server_socket() {
for (auto &addr : quic_upstream_addrs_) {
assert(!addr.host_unix);
if (create_quic_server_socket(addr) != 0) {
return -1;
}
quic_listeners_.emplace_back(std::make_unique<QUICListener>(&addr, this));
}
return 0;
}
#endif // ENABLE_HTTP3
namespace {
size_t match_downstream_addr_group_host(
const RouterConfig &routerconf, const StringRef &host,

View File

@ -50,6 +50,9 @@
#include "shrpx_live_check.h"
#include "shrpx_connect_blocker.h"
#include "shrpx_dns_tracker.h"
#ifdef ENABLE_HTTP3
# include "shrpx_quic_connection_handler.h"
#endif // ENABLE_HTTP3
#include "allocator.h"
using namespace nghttp2;
@ -61,6 +64,9 @@ class ConnectBlocker;
class MemcachedDispatcher;
struct UpstreamAddr;
class ConnectionHandler;
#ifdef ENABLE_HTTP3
class QUICListener;
#endif // ENABLE_HTTP3
#ifdef HAVE_MRUBY
namespace mruby {
@ -270,6 +276,9 @@ public:
Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
SSL_CTX *tls_session_cache_memcached_ssl_ctx,
tls::CertLookupTree *cert_tree,
#ifdef ENABLE_HTTP3
SSL_CTX *quic_sv_ssl_ctx, tls::CertLookupTree *quic_cert_tree,
#endif // ENABLE_HTTP3
const std::shared_ptr<TicketKeys> &ticket_keys,
ConnectionHandler *conn_handler,
std::shared_ptr<DownstreamConfig> downstreamconf);
@ -280,6 +289,9 @@ public:
void send(const WorkerEvent &event);
tls::CertLookupTree *get_cert_lookup_tree() const;
#ifdef ENABLE_HTTP3
tls::CertLookupTree *get_quic_cert_lookup_tree() const;
#endif // ENABLE_HTTP3
// These 2 functions make a lock m_ to get/set ticket keys
// atomically.
@ -290,6 +302,9 @@ public:
struct ev_loop *get_loop() const;
SSL_CTX *get_sv_ssl_ctx() const;
SSL_CTX *get_cl_ssl_ctx() const;
#ifdef ENABLE_HTTP3
SSL_CTX *get_quic_sv_ssl_ctx() const;
#endif // ENABLE_HTTP3
void set_graceful_shutdown(bool f);
bool get_graceful_shutdown() const;
@ -319,6 +334,12 @@ public:
ConnectionHandler *get_connection_handler() const;
#ifdef ENABLE_HTTP3
QUICConnectionHandler *get_quic_connection_handler();
int setup_quic_server_socket();
#endif // ENABLE_HTTP3
DNSTracker *get_dns_tracker();
private:
@ -335,6 +356,11 @@ private:
WorkerStat worker_stat_;
DNSTracker dns_tracker_;
#ifdef ENABLE_HTTP3
std::vector<UpstreamAddr> quic_upstream_addrs_;
std::vector<std::unique_ptr<QUICListener>> quic_listeners_;
#endif // ENABLE_HTTP3
std::shared_ptr<DownstreamConfig> downstreamconf_;
std::unique_ptr<MemcachedDispatcher> session_cache_memcached_dispatcher_;
#ifdef HAVE_MRUBY
@ -348,6 +374,12 @@ private:
SSL_CTX *cl_ssl_ctx_;
tls::CertLookupTree *cert_tree_;
ConnectionHandler *conn_handler_;
#ifdef ENABLE_HTTP3
SSL_CTX *quic_sv_ssl_ctx_;
tls::CertLookupTree *quic_cert_tree_;
QUICConnectionHandler quic_conn_handler_;
#endif // ENABLE_HTTP3
#ifndef HAVE_ATOMIC_STD_SHARED_PTR
std::mutex ticket_keys_m_;

View File

@ -686,18 +686,21 @@ std::string numeric_name(const struct sockaddr *sa, socklen_t salen) {
}
std::string to_numeric_addr(const Address *addr) {
auto family = addr->su.storage.ss_family;
return to_numeric_addr(&addr->su.sa, addr->len);
}
std::string to_numeric_addr(const struct sockaddr *sa, socklen_t salen) {
auto family = sa->sa_family;
#ifndef _WIN32
if (family == AF_UNIX) {
return addr->su.un.sun_path;
return reinterpret_cast<const sockaddr_un *>(sa)->sun_path;
}
#endif // !_WIN32
std::array<char, NI_MAXHOST> host;
std::array<char, NI_MAXSERV> serv;
auto rv =
getnameinfo(&addr->su.sa, addr->len, host.data(), host.size(),
serv.data(), serv.size(), NI_NUMERICHOST | NI_NUMERICSERV);
auto rv = getnameinfo(sa, salen, host.data(), host.size(), serv.data(),
serv.size(), NI_NUMERICHOST | NI_NUMERICSERV);
if (rv != 0) {
return "unknown";
}
@ -945,6 +948,56 @@ int create_nonblock_socket(int family) {
return fd;
}
int create_nonblock_udp_socket(int family) {
#ifdef SOCK_NONBLOCK
auto fd = socket(family, SOCK_DGRAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0);
if (fd == -1) {
return -1;
}
#else // !SOCK_NONBLOCK
auto fd = socket(family, SOCK_DGRAM, 0);
if (fd == -1) {
return -1;
}
make_socket_nonblocking(fd);
make_socket_closeonexec(fd);
#endif // !SOCK_NONBLOCK
return fd;
}
int bind_any_addr_udp(int fd, int family) {
addrinfo hints{};
addrinfo *res, *rp;
int rv;
hints.ai_family = family;
hints.ai_socktype = SOCK_DGRAM;
hints.ai_flags = AI_PASSIVE;
rv = getaddrinfo(nullptr, "0", &hints, &res);
if (rv != 0) {
return -1;
}
for (rp = res; rp; rp = rp->ai_next) {
if (bind(fd, rp->ai_addr, rp->ai_addrlen) != -1) {
break;
}
}
freeaddrinfo(res);
if (!rp) {
return -1;
}
return 0;
}
bool check_socket_connected(int fd) {
int error;
socklen_t len = sizeof(error);
@ -1617,6 +1670,40 @@ int daemonize(int nochdir, int noclose) {
#endif // !defined(__APPLE__)
}
int msghdr_get_local_addr(Address &dest, msghdr *msg, int family) {
switch (family) {
case AF_INET:
for (auto cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
if (cmsg->cmsg_level == IPPROTO_IP && cmsg->cmsg_type == IP_PKTINFO) {
auto pktinfo = reinterpret_cast<in_pktinfo *>(CMSG_DATA(cmsg));
dest.len = sizeof(dest.su.in);
auto &sa = dest.su.in;
sa.sin_family = AF_INET;
sa.sin_addr = pktinfo->ipi_addr;
return 0;
}
}
return -1;
case AF_INET6:
for (auto cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
if (cmsg->cmsg_level == IPPROTO_IPV6 && cmsg->cmsg_type == IPV6_PKTINFO) {
auto pktinfo = reinterpret_cast<in6_pktinfo *>(CMSG_DATA(cmsg));
dest.len = sizeof(dest.su.in6);
auto &sa = dest.su.in6;
sa.sin6_family = AF_INET6;
sa.sin6_addr = pktinfo->ipi6_addr;
return 0;
}
}
return -1;
}
return -1;
}
} // namespace util
} // namespace nghttp2

View File

@ -505,6 +505,8 @@ std::string numeric_name(const struct sockaddr *sa, socklen_t salen);
// IPv6 address, address is enclosed by square brackets ([]).
std::string to_numeric_addr(const Address *addr);
std::string to_numeric_addr(const struct sockaddr *sa, socklen_t salen);
// Sets |port| to |addr|.
void set_port(Address &addr, uint16_t port);
@ -626,6 +628,9 @@ int make_socket_nonblocking(int fd);
int make_socket_nodelay(int fd);
int create_nonblock_socket(int family);
int create_nonblock_udp_socket(int family);
int bind_any_addr_udp(int fd, int family);
bool check_socket_connected(int fd);
@ -783,6 +788,8 @@ std::mt19937 make_mt19937();
// daemon() using fork().
int daemonize(int nochdir, int noclose);
int msghdr_get_local_addr(Address &dest, msghdr *msg, int family);
} // namespace util
} // namespace nghttp2