From f74b6d9a43d07315707a02edab1238a659d2d96f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 12 Sep 2021 15:04:54 +0900 Subject: [PATCH 001/142] nghttpx: Add --frontend-quic-require-token option --- gennghttpxfun.py | 1 + src/shrpx.cc | 11 +++++++++++ src/shrpx_config.cc | 11 +++++++++++ src/shrpx_config.h | 4 ++++ src/shrpx_quic_connection_handler.cc | 23 +++++++++++++++++++++++ 5 files changed, 50 insertions(+) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 337c1cb4..988cb331 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -190,6 +190,7 @@ OPTIONS = [ "frontend-http3-max-concurrent-streams", "frontend-quic-early-data", "frontend-quic-qlog-dir", + "frontend-quic-require-token", ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index 4938f5cf..15d11867 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -3222,6 +3222,10 @@ HTTP/3 and QUIC: each QUIC connection. The file name is ISO8601 basic format, followed by "-", server Source Connection ID and ".qlog". + --frontend-quic-require-token + Require an address validation token for a frontend QUIC + connection. Server sends a token in Retry packet or + NEW_TOKEN frame in the previous connection. --no-quic-bpf Disable eBPF. --frontend-http3-window-size= @@ -4011,6 +4015,8 @@ int main(int argc, char **argv) { {SHRPX_OPT_FRONTEND_QUIC_EARLY_DATA.c_str(), no_argument, &flag, 180}, {SHRPX_OPT_FRONTEND_QUIC_QLOG_DIR.c_str(), required_argument, &flag, 181}, + {SHRPX_OPT_FRONTEND_QUIC_REQUIRE_TOKEN.c_str(), no_argument, &flag, + 182}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -4878,6 +4884,11 @@ int main(int argc, char **argv) { cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_QLOG_DIR, StringRef{optarg}); break; + case 182: + // --frontend-quic-require-token + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_REQUIRE_TOKEN, + StringRef::from_lit("yes")); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 356c4bca..f07e48dd 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2401,6 +2401,11 @@ int option_lookup_token(const char *name, size_t namelen) { return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED; } break; + case 'n': + if (util::strieq_l("frontend-quic-require-toke", name, 26)) { + return SHRPX_OPTID_FRONTEND_QUIC_REQUIRE_TOKEN; + } + break; case 'r': if (util::strieq_l("request-header-field-buffe", name, 26)) { return SHRPX_OPTID_REQUEST_HEADER_FIELD_BUFFER; @@ -3986,6 +3991,12 @@ int parse_config(Config *config, int optid, const StringRef &opt, config->quic.upstream.qlog.dir = optarg; #endif // ENABLE_HTTP3 + return 0; + case SHRPX_OPTID_FRONTEND_QUIC_REQUIRE_TOKEN: +#ifdef ENABLE_HTTP3 + config->quic.upstream.require_token = util::strieq_l("yes", optarg); +#endif // ENABLE_HTTP3 + return 0; case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index fdb7bae0..6b781f38 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -387,6 +387,8 @@ constexpr auto SHRPX_OPT_FRONTEND_QUIC_EARLY_DATA = StringRef::from_lit("frontend-quic-early-data"); constexpr auto SHRPX_OPT_FRONTEND_QUIC_QLOG_DIR = StringRef::from_lit("frontend-quic-qlog-dir"); +constexpr auto SHRPX_OPT_FRONTEND_QUIC_REQUIRE_TOKEN = + StringRef::from_lit("frontend-quic-require-token"); constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -755,6 +757,7 @@ struct QUICConfig { StringRef dir; } qlog; bool early_data; + bool require_token; } upstream; struct { StringRef prog_file; @@ -1211,6 +1214,7 @@ enum { SHRPX_OPTID_FRONTEND_QUIC_EARLY_DATA, SHRPX_OPTID_FRONTEND_QUIC_IDLE_TIMEOUT, SHRPX_OPTID_FRONTEND_QUIC_QLOG_DIR, + SHRPX_OPTID_FRONTEND_QUIC_REQUIRE_TOKEN, SHRPX_OPTID_FRONTEND_READ_TIMEOUT, SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT, SHRPX_OPTID_HEADER_FIELD_BUFFER, diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index f7341913..0f3590e0 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -134,6 +134,8 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, const uint8_t *token = nullptr; size_t tokenlen = 0; + auto &quicconf = config->quic; + switch (ngtcp2_accept(&hd, data, datalen)) { case 0: { // If we get Initial and it has the CID prefix of this worker, it @@ -145,6 +147,13 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, } if (hd.token.len == 0) { + if (quicconf.upstream.require_token) { + send_retry(faddr, version, dcid, dcidlen, scid, scidlen, remote_addr, + local_addr); + + return 0; + } + break; } @@ -186,6 +195,13 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, << util::to_numeric_addr(&remote_addr); } + if (quicconf.upstream.require_token) { + send_retry(faddr, version, dcid, dcidlen, scid, scidlen, + remote_addr, local_addr); + + return 0; + } + break; } @@ -199,6 +215,13 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, break; default: + if (quicconf.upstream.require_token) { + send_retry(faddr, version, dcid, dcidlen, scid, scidlen, remote_addr, + local_addr); + + return 0; + } + break; } From fc402f580459c99505bd947a2129f5e488742739 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 12 Sep 2021 16:55:56 +0900 Subject: [PATCH 002/142] Cleanup --- src/shrpx_http3_upstream.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index c6e4d1ca..c562e2fc 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -151,9 +151,7 @@ Http3Upstream::~Http3Upstream() { nghttp3_conn_del(httpconn_); - if (conn_) { - ngtcp2_conn_del(conn_); - } + ngtcp2_conn_del(conn_); if (qlog_fd_ != -1) { close(qlog_fd_); From 00f65afe20acf59f058905008ea9e5c89c3c38e6 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 12 Sep 2021 17:48:57 +0900 Subject: [PATCH 003/142] nghttpx: Fix incorrect quic frontend address matching --- src/shrpx_worker.cc | 51 ++++++++++++++++++++++--------- src/util.cc | 73 ++++++--------------------------------------- src/util.h | 65 ++++++++++++++++++++++++++++++++++++---- src/util_test.cc | 6 ++++ 4 files changed, 111 insertions(+), 84 deletions(-) diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index e128badb..9dbdd28f 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -1021,7 +1021,10 @@ const UpstreamAddr *Worker::find_quic_upstream_addr(const Address &local_addr) { assert(0); } - auto hostport = util::make_hostport(StringRef{host.data()}, port); + std::array hostport_buf; + + auto hostport = util::make_http_hostport(std::begin(hostport_buf), + StringRef{host.data()}, port); const UpstreamAddr *fallback_faddr = nullptr; for (auto &faddr : quic_upstream_addrs_) { @@ -1033,21 +1036,41 @@ const UpstreamAddr *Worker::find_quic_upstream_addr(const Address &local_addr) { continue; } - switch (faddr.family) { - case AF_INET: - if (util::starts_with(faddr.hostport, StringRef::from_lit("0.0.0.0:"))) { - fallback_faddr = &faddr; - } + if (faddr.port == 443 || faddr.port == 80) { + switch (faddr.family) { + case AF_INET: + if (util::streq(faddr.hostport, StringRef::from_lit("0.0.0.0"))) { + fallback_faddr = &faddr; + } - break; - case AF_INET6: - if (util::starts_with(faddr.hostport, StringRef::from_lit("[::]:"))) { - fallback_faddr = &faddr; - } + break; + case AF_INET6: + if (util::streq(faddr.hostport, StringRef::from_lit("[::]"))) { + fallback_faddr = &faddr; + } - break; - default: - assert(0); + break; + default: + assert(0); + } + } else { + switch (faddr.family) { + case AF_INET: + if (util::starts_with(faddr.hostport, + StringRef::from_lit("0.0.0.0:"))) { + fallback_faddr = &faddr; + } + + break; + case AF_INET6: + if (util::starts_with(faddr.hostport, StringRef::from_lit("[::]:"))) { + fallback_faddr = &faddr; + } + + break; + default: + assert(0); + } } } diff --git a/src/util.cc b/src/util.cc index 6efc0e9e..97af8d79 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1325,81 +1325,26 @@ std::string dtos(double n) { StringRef make_http_hostport(BlockAllocator &balloc, const StringRef &host, uint16_t port) { - if (port != 80 && port != 443) { - return make_hostport(balloc, host, port); - } - - auto ipv6 = ipv6_numeric_addr(host.c_str()); - - auto iov = make_byte_ref(balloc, host.size() + (ipv6 ? 2 : 0) + 1); - auto p = iov.base; - - if (ipv6) { - *p++ = '['; - } - - p = std::copy(std::begin(host), std::end(host), p); - - if (ipv6) { - *p++ = ']'; - } - - *p = '\0'; - - return StringRef{iov.base, p}; + auto iov = make_byte_ref(balloc, host.size() + 2 + 1 + 5 + 1); + return make_http_hostport(iov.base, host, port); } std::string make_hostport(const StringRef &host, uint16_t port) { - auto ipv6 = ipv6_numeric_addr(host.c_str()); - auto serv = utos(port); - std::string hostport; - hostport.resize(host.size() + (ipv6 ? 2 : 0) + 1 + serv.size()); + // I'm not sure we can write \0 at the position std::string::size(), + // so allocate an extra byte. + hostport.resize(host.size() + 2 + 1 + 5 + 1); - auto p = &hostport[0]; - - if (ipv6) { - *p++ = '['; - } - - p = std::copy_n(host.c_str(), host.size(), p); - - if (ipv6) { - *p++ = ']'; - } - - *p++ = ':'; - std::copy_n(serv.c_str(), serv.size(), p); + auto s = make_hostport(std::begin(hostport), host, port); + hostport.resize(s.size()); return hostport; } StringRef make_hostport(BlockAllocator &balloc, const StringRef &host, uint16_t port) { - auto ipv6 = ipv6_numeric_addr(host.c_str()); - auto serv = utos(port); - - auto iov = - make_byte_ref(balloc, host.size() + (ipv6 ? 2 : 0) + 1 + serv.size()); - auto p = iov.base; - - if (ipv6) { - *p++ = '['; - } - - p = std::copy(std::begin(host), std::end(host), p); - - if (ipv6) { - *p++ = ']'; - } - - *p++ = ':'; - - p = std::copy(std::begin(serv), std::end(serv), p); - - *p = '\0'; - - return StringRef{iov.base, p}; + auto iov = make_byte_ref(balloc, host.size() + 2 + 1 + 5 + 1); + return make_hostport(iov.base, host, port); } namespace { diff --git a/src/util.h b/src/util.h index ca9b437d..d4990e03 100644 --- a/src/util.h +++ b/src/util.h @@ -762,12 +762,6 @@ std::string format_duration(const std::chrono::microseconds &u); // Just like above, but this takes |t| as seconds. std::string format_duration(double t); -// Creates "host:port" string using given |host| and |port|. If -// |host| is numeric IPv6 address (e.g., ::1), it is enclosed by "[" -// and "]". If |port| is 80 or 443, port part is omitted. -StringRef make_http_hostport(BlockAllocator &balloc, const StringRef &host, - uint16_t port); - // Just like make_http_hostport(), but doesn't treat 80 and 443 // specially. std::string make_hostport(const StringRef &host, uint16_t port); @@ -775,6 +769,65 @@ std::string make_hostport(const StringRef &host, uint16_t port); StringRef make_hostport(BlockAllocator &balloc, const StringRef &host, uint16_t port); +template +StringRef make_hostport(OutputIt first, const StringRef &host, uint16_t port) { + auto ipv6 = ipv6_numeric_addr(host.c_str()); + auto serv = utos(port); + auto p = first; + + if (ipv6) { + *p++ = '['; + } + + p = std::copy(std::begin(host), std::end(host), p); + + if (ipv6) { + *p++ = ']'; + } + + *p++ = ':'; + + p = std::copy(std::begin(serv), std::end(serv), p); + + *p = '\0'; + + return StringRef{first, p}; +} + +// Creates "host:port" string using given |host| and |port|. If +// |host| is numeric IPv6 address (e.g., ::1), it is enclosed by "[" +// and "]". If |port| is 80 or 443, port part is omitted. +StringRef make_http_hostport(BlockAllocator &balloc, const StringRef &host, + uint16_t port); + +constexpr size_t max_hostport = NI_MAXHOST + /* [] for IPv6 */ 2 + /* : */ 1 + + /* port */ 5 + /* terminal NUL */ 1; + +template +StringRef make_http_hostport(OutputIt first, const StringRef &host, + uint16_t port) { + if (port != 80 && port != 443) { + return make_hostport(first, host, port); + } + + auto ipv6 = ipv6_numeric_addr(host.c_str()); + auto p = first; + + if (ipv6) { + *p++ = '['; + } + + p = std::copy(std::begin(host), std::end(host), p); + + if (ipv6) { + *p++ = ']'; + } + + *p = '\0'; + + return StringRef{first, p}; +} + // Dumps |src| of length |len| in the format similar to `hexdump -C`. void hexdump(FILE *out, const uint8_t *src, size_t len); diff --git a/src/util_test.cc b/src/util_test.cc index dfe87e99..d5ff942f 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -550,6 +550,12 @@ void test_util_make_hostport(void) { util::make_hostport(balloc, StringRef::from_lit("localhost"), 80)); CU_ASSERT("[::1]:443" == util::make_hostport(balloc, StringRef::from_lit("::1"), 443)); + + // Check std::string version + CU_ASSERT( + "abcdefghijklmnopqrstuvwxyz0123456789:65535" == + util::make_hostport( + StringRef::from_lit("abcdefghijklmnopqrstuvwxyz0123456789"), 65535)); } void test_util_strifind(void) { From 525d59fdf6e6dec5ecd6520d07cb162714804325 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 12 Sep 2021 18:06:55 +0900 Subject: [PATCH 004/142] Remove util::make_hostport returning std::string --- src/shrpx.cc | 11 ++++++++--- src/shrpx_config.cc | 4 +++- src/util.cc | 12 ------------ src/util.h | 2 -- src/util_test.cc | 15 ++++++--------- 5 files changed, 17 insertions(+), 27 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 15d11867..ed7ff18f 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -3479,9 +3479,12 @@ int process_options(Config *config, return -1; } + std::array hostport_buf; + auto &proxy = config->downstream_http_proxy; if (!proxy.host.empty()) { - auto hostport = util::make_hostport(StringRef{proxy.host}, proxy.port); + auto hostport = util::make_hostport(std::begin(hostport_buf), + StringRef{proxy.host}, proxy.port); if (resolve_hostname(&proxy.addr, proxy.host.c_str(), proxy.port, AF_UNSPEC) == -1) { LOG(FATAL) << "Resolving backend HTTP proxy address failed: " << hostport; @@ -3494,7 +3497,8 @@ int process_options(Config *config, { auto &memcachedconf = tlsconf.session_cache.memcached; if (!memcachedconf.host.empty()) { - auto hostport = util::make_hostport(StringRef{memcachedconf.host}, + auto hostport = util::make_hostport(std::begin(hostport_buf), + StringRef{memcachedconf.host}, memcachedconf.port); if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.c_str(), memcachedconf.port, memcachedconf.family) == -1) { @@ -3515,7 +3519,8 @@ int process_options(Config *config, { auto &memcachedconf = tlsconf.ticket.memcached; if (!memcachedconf.host.empty()) { - auto hostport = util::make_hostport(StringRef{memcachedconf.host}, + auto hostport = util::make_hostport(std::begin(hostport_buf), + StringRef{memcachedconf.host}, memcachedconf.port); if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.c_str(), memcachedconf.port, memcachedconf.family) == -1) { diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index f07e48dd..a9886119 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -4332,6 +4332,8 @@ int configure_downstream_group(Config *config, bool http2_proxy, auto resolve_flags = numeric_addr_only ? AI_NUMERICHOST | AI_NUMERICSERV : 0; + std::array hostport_buf; + for (auto &g : addr_groups) { std::unordered_map wgchk; for (auto &addr : g.addrs) { @@ -4377,7 +4379,7 @@ int configure_downstream_group(Config *config, bool http2_proxy, util::make_http_hostport(downstreamconf.balloc, addr.host, addr.port); auto hostport = - util::make_hostport(downstreamconf.balloc, addr.host, addr.port); + util::make_hostport(std::begin(hostport_buf), addr.host, addr.port); if (!addr.dns) { if (resolve_hostname(&addr.addr, addr.host.c_str(), addr.port, diff --git a/src/util.cc b/src/util.cc index 97af8d79..4491a07d 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1329,18 +1329,6 @@ StringRef make_http_hostport(BlockAllocator &balloc, const StringRef &host, return make_http_hostport(iov.base, host, port); } -std::string make_hostport(const StringRef &host, uint16_t port) { - std::string hostport; - // I'm not sure we can write \0 at the position std::string::size(), - // so allocate an extra byte. - hostport.resize(host.size() + 2 + 1 + 5 + 1); - - auto s = make_hostport(std::begin(hostport), host, port); - hostport.resize(s.size()); - - return hostport; -} - StringRef make_hostport(BlockAllocator &balloc, const StringRef &host, uint16_t port) { auto iov = make_byte_ref(balloc, host.size() + 2 + 1 + 5 + 1); diff --git a/src/util.h b/src/util.h index d4990e03..9c34d9e6 100644 --- a/src/util.h +++ b/src/util.h @@ -764,8 +764,6 @@ std::string format_duration(double t); // Just like make_http_hostport(), but doesn't treat 80 and 443 // specially. -std::string make_hostport(const StringRef &host, uint16_t port); - StringRef make_hostport(BlockAllocator &balloc, const StringRef &host, uint16_t port); diff --git a/src/util_test.cc b/src/util_test.cc index d5ff942f..17d640d5 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -540,22 +540,19 @@ void test_util_make_http_hostport(void) { } void test_util_make_hostport(void) { + std::array hostport_buf; CU_ASSERT("localhost:80" == - util::make_hostport(StringRef::from_lit("localhost"), 80)); - CU_ASSERT("[::1]:443" == - util::make_hostport(StringRef::from_lit("::1"), 443)); + util::make_hostport(std::begin(hostport_buf), + StringRef::from_lit("localhost"), 80)); + CU_ASSERT("[::1]:443" == util::make_hostport(std::begin(hostport_buf), + StringRef::from_lit("::1"), + 443)); BlockAllocator balloc(4096, 4096); CU_ASSERT("localhost:80" == util::make_hostport(balloc, StringRef::from_lit("localhost"), 80)); CU_ASSERT("[::1]:443" == util::make_hostport(balloc, StringRef::from_lit("::1"), 443)); - - // Check std::string version - CU_ASSERT( - "abcdefghijklmnopqrstuvwxyz0123456789:65535" == - util::make_hostport( - StringRef::from_lit("abcdefghijklmnopqrstuvwxyz0123456789"), 65535)); } void test_util_strifind(void) { From b5e5972c2a0dc1048199a8901609166cf8ccddb9 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 12 Sep 2021 18:11:11 +0900 Subject: [PATCH 005/142] Update doc --- src/util.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/util.h b/src/util.h index 9c34d9e6..70f97f09 100644 --- a/src/util.h +++ b/src/util.h @@ -762,6 +762,11 @@ std::string format_duration(const std::chrono::microseconds &u); // Just like above, but this takes |t| as seconds. std::string format_duration(double t); +// The maximum buffer size including terminal NULL to store the result +// of make_hostport. +constexpr size_t max_hostport = NI_MAXHOST + /* [] for IPv6 */ 2 + /* : */ 1 + + /* port */ 5 + /* terminal NULL */ 1; + // Just like make_http_hostport(), but doesn't treat 80 and 443 // specially. StringRef make_hostport(BlockAllocator &balloc, const StringRef &host, @@ -798,9 +803,6 @@ StringRef make_hostport(OutputIt first, const StringRef &host, uint16_t port) { StringRef make_http_hostport(BlockAllocator &balloc, const StringRef &host, uint16_t port); -constexpr size_t max_hostport = NI_MAXHOST + /* [] for IPv6 */ 2 + /* : */ 1 + - /* port */ 5 + /* terminal NUL */ 1; - template StringRef make_http_hostport(OutputIt first, const StringRef &host, uint16_t port) { From 4541134c888380f7df362b6f415a22f21c6e79ac Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 13 Sep 2021 21:19:00 +0900 Subject: [PATCH 006/142] Add missing include --- src/shrpx_http3_upstream.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index c562e2fc..4dd035af 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -27,6 +27,7 @@ #include #include #include +#include #include From fcdac50f79b8f337cdde9f981ec542ecd7d3c07e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 13 Sep 2021 21:19:18 +0900 Subject: [PATCH 007/142] Should run program --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index b4b1e605..a31360c6 100644 --- a/configure.ac +++ b/configure.ac @@ -991,7 +991,7 @@ AC_CHECK_DECL([UDP_SEGMENT], [have_netinet_udp_h_udp_segment=yes], if test "x$have_netinet_udp_h_udp_segment" = "xno"; then have_linux_udp_h_udp_segment=no - AC_COMPILE_IFELSE([AC_LANG_PROGRAM( + AC_RUN_IFELSE([AC_LANG_PROGRAM( [[ #include ]], From 8f419a4869ae3c592d05635854a1c7f3b31422e8 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 13 Sep 2021 21:35:46 +0900 Subject: [PATCH 008/142] nghttpx: Add --frontend-quic-congestion-controller option --- gennghttpxfun.py | 1 + src/shrpx.cc | 18 ++++++++++++++++++ src/shrpx_config.cc | 16 ++++++++++++++++ src/shrpx_config.h | 4 ++++ src/shrpx_http3_upstream.cc | 9 ++++++++- 5 files changed, 47 insertions(+), 1 deletion(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 988cb331..13fecf26 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -191,6 +191,7 @@ OPTIONS = [ "frontend-quic-early-data", "frontend-quic-qlog-dir", "frontend-quic-require-token", + "frontend-quic-congestion-controller", ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index ed7ff18f..c60e4143 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1854,6 +1854,8 @@ void fill_default_config(Config *config) { auto &bpfconf = quicconf.bpf; bpfconf.prog_file = StringRef::from_lit(PKGLIBDIR "/reuseport_kern.o"); + + upstreamconf.congestion_controller = NGTCP2_CC_ALGO_CUBIC; } auto &http3conf = config->http3; @@ -3226,6 +3228,15 @@ HTTP/3 and QUIC: Require an address validation token for a frontend QUIC connection. Server sends a token in Retry packet or NEW_TOKEN frame in the previous connection. + --frontend-quic-congestion-controller= + Specify a congestion controller algorithm for a frontend + QUIC connection. should be either "cubic" or + "bbr". + Default: )" + << (config->quic.upstream.congestion_controller == NGTCP2_CC_ALGO_CUBIC + ? "cubic" + : "bbr") + << R"( --no-quic-bpf Disable eBPF. --frontend-http3-window-size= @@ -4022,6 +4033,8 @@ int main(int argc, char **argv) { 181}, {SHRPX_OPT_FRONTEND_QUIC_REQUIRE_TOKEN.c_str(), no_argument, &flag, 182}, + {SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER.c_str(), + required_argument, &flag, 183}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -4894,6 +4907,11 @@ int main(int argc, char **argv) { cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_REQUIRE_TOKEN, StringRef::from_lit("yes")); break; + case 183: + // --frontend-quic-congestion-controller + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER, + StringRef{optarg}); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index a9886119..be272f62 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2571,6 +2571,9 @@ int option_lookup_token(const char *name, size_t namelen) { if (util::strieq_l("frontend-http2-dump-response-heade", name, 34)) { return SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER; } + if (util::strieq_l("frontend-quic-congestion-controlle", name, 34)) { + return SHRPX_OPTID_FRONTEND_QUIC_CONGESTION_CONTROLLER; + } break; } break; @@ -3997,6 +4000,19 @@ int parse_config(Config *config, int optid, const StringRef &opt, config->quic.upstream.require_token = util::strieq_l("yes", optarg); #endif // ENABLE_HTTP3 + return 0; + case SHRPX_OPTID_FRONTEND_QUIC_CONGESTION_CONTROLLER: +#ifdef ENABLE_HTTP3 + if (util::strieq_l("cubic", optarg)) { + config->quic.upstream.congestion_controller = NGTCP2_CC_ALGO_CUBIC; + } else if (util::strieq_l("bbr", optarg)) { + config->quic.upstream.congestion_controller = NGTCP2_CC_ALGO_BBR; + } else { + LOG(ERROR) << opt << ": must be either cubic or bbr"; + return -1; + } +#endif // ENABLE_HTTP3 + return 0; case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 6b781f38..c46ebe5b 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -389,6 +389,8 @@ constexpr auto SHRPX_OPT_FRONTEND_QUIC_QLOG_DIR = StringRef::from_lit("frontend-quic-qlog-dir"); constexpr auto SHRPX_OPT_FRONTEND_QUIC_REQUIRE_TOKEN = StringRef::from_lit("frontend-quic-require-token"); +constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER = + StringRef::from_lit("frontend-quic-congestion-controller"); constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -756,6 +758,7 @@ struct QUICConfig { struct { StringRef dir; } qlog; + ngtcp2_cc_algo congestion_controller; bool early_data; bool require_token; } upstream; @@ -1210,6 +1213,7 @@ enum { SHRPX_OPTID_FRONTEND_KEEP_ALIVE_TIMEOUT, SHRPX_OPTID_FRONTEND_MAX_REQUESTS, SHRPX_OPTID_FRONTEND_NO_TLS, + SHRPX_OPTID_FRONTEND_QUIC_CONGESTION_CONTROLLER, SHRPX_OPTID_FRONTEND_QUIC_DEBUG_LOG, SHRPX_OPTID_FRONTEND_QUIC_EARLY_DATA, SHRPX_OPTID_FRONTEND_QUIC_IDLE_TIMEOUT, diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 4dd035af..29862e1d 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -573,7 +573,7 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, } settings.initial_ts = quic_timestamp(); - settings.cc_algo = NGTCP2_CC_ALGO_BBR; + settings.cc_algo = quicconf.upstream.congestion_controller; settings.max_window = http3conf.upstream.max_connection_window_size; settings.max_stream_window = http3conf.upstream.max_window_size; settings.max_udp_payload_size = SHRPX_QUIC_MAX_UDP_PAYLOAD_SIZE; @@ -665,6 +665,13 @@ int Http3Upstream::write_streams() { ngtcp2_path_storage_zero(&ps); ngtcp2_path_storage_zero(&prev_ps); + auto config = get_config(); + auto &quicconf = config->quic; + + if (quicconf.upstream.congestion_controller != NGTCP2_CC_ALGO_BBR) { + max_pktcnt = std::min(max_pktcnt, static_cast(10)); + } + for (;;) { int64_t stream_id = -1; int fin = 0; From 4b79a4a10de3de8275997d97226b02a0388b2f19 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 13 Sep 2021 21:46:41 +0900 Subject: [PATCH 009/142] Add message when checking UDP_SEGMENT in linux/udp.h --- configure.ac | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index a31360c6..06487861 100644 --- a/configure.ac +++ b/configure.ac @@ -991,6 +991,7 @@ AC_CHECK_DECL([UDP_SEGMENT], [have_netinet_udp_h_udp_segment=yes], if test "x$have_netinet_udp_h_udp_segment" = "xno"; then have_linux_udp_h_udp_segment=no + AC_MSG_CHECKING([whether UDP_SEGMENT is defined as 103 in linux/udp.h]) AC_RUN_IFELSE([AC_LANG_PROGRAM( [[ #include @@ -1000,8 +1001,10 @@ if test "x$have_netinet_udp_h_udp_segment" = "xno"; then exit(1) #endif ]])], - [have_linux_udp_h_udp_segment=yes], - [have_linux_udp_h_udp_segment=no]) + [have_linux_udp_h_udp_segment=yes + AC_MSG_RESULT([yes])], + [have_linux_udp_h_udp_segment=no + AC_MSG_RESULT([no])]) if test "x$have_linux_udp_h_udp_segment" = "xyes"; then EXTRA_DEFS="$EXTRA_DEFS -DUDP_SEGMENT=103" From 8903bd1e8ada0414b2d97497297fdfc9b193f333 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 13 Sep 2021 23:09:38 +0900 Subject: [PATCH 010/142] nghttpx: Deal with error from ngtcp2_conn_read_pkt --- src/shrpx_http3_upstream.cc | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 29862e1d..c98e53d2 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -1596,15 +1596,30 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr, rv = ngtcp2_conn_read_pkt(conn_, &path, &pi, data, datalen, quic_timestamp()); if (rv != 0) { - ULOG(ERROR, this) << "ngtcp2_conn_read_pkt: " << ngtcp2_strerror(rv); - switch (rv) { case NGTCP2_ERR_DRAINING: - // TODO Start drain period return -1; - case NGTCP2_ERR_RETRY: - // TODO Send Retry packet + case NGTCP2_ERR_RETRY: { + auto worker = handler_->get_worker(); + auto quic_conn_handler = worker->get_quic_connection_handler(); + + uint32_t version; + const uint8_t *dcid, *scid; + size_t dcidlen, scidlen; + + rv = ngtcp2_pkt_decode_version_cid(&version, &dcid, &dcidlen, &scid, + &scidlen, data, datalen, + SHRPX_QUIC_SCIDLEN); + if (rv != 0) { + return -1; + } + + quic_conn_handler->send_retry(handler_->get_upstream_addr(), version, + dcid, dcidlen, scid, scidlen, remote_addr, + local_addr); + return -1; + } case NGTCP2_ERR_REQUIRED_TRANSPORT_PARAM: case NGTCP2_ERR_MALFORMED_TRANSPORT_PARAM: case NGTCP2_ERR_TRANSPORT_PARAM: @@ -1621,7 +1636,8 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr, } } - // TODO Send connection close + ULOG(ERROR, this) << "ngtcp2_conn_read_pkt: " << ngtcp2_strerror(rv); + return handle_error(); } From 74162850f0a35f0154f5d058ff4ab307bcaeb4dc Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Sep 2021 16:45:11 +0900 Subject: [PATCH 011/142] nghttpx: Compile with the latest nghttp3 --- src/shrpx_http3_upstream.cc | 4 ++-- src/shrpx_http3_upstream.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index c98e53d2..492bea4b 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -1784,7 +1784,7 @@ int http_deferred_consume(nghttp3_conn *conn, int64_t stream_id, namespace { int http_acked_stream_data(nghttp3_conn *conn, int64_t stream_id, - size_t datalen, void *user_data, + uint64_t datalen, void *user_data, void *stream_user_data) { auto upstream = static_cast(user_data); auto downstream = static_cast(stream_user_data); @@ -1800,7 +1800,7 @@ int http_acked_stream_data(nghttp3_conn *conn, int64_t stream_id, } // namespace int Http3Upstream::http_acked_stream_data(Downstream *downstream, - size_t datalen) { + uint64_t datalen) { if (LOG_ENABLED(INFO)) { ULOG(INFO, this) << "Stream " << downstream->get_stream_id() << " " << datalen << " bytes acknowledged"; diff --git a/src/shrpx_http3_upstream.h b/src/shrpx_http3_upstream.h index 845abe92..9542bca6 100644 --- a/src/shrpx_http3_upstream.h +++ b/src/shrpx_http3_upstream.h @@ -137,7 +137,7 @@ public: int stream_close(int64_t stream_id, uint64_t app_error_code); void log_response_headers(Downstream *downstream, const std::vector &nva) const; - int http_acked_stream_data(Downstream *downstream, size_t datalen); + int http_acked_stream_data(Downstream *downstream, uin64_t datalen); int http_shutdown_stream_read(int64_t stream_id); int http_reset_stream(int64_t stream_id, uint64_t app_error_code); int http_send_stop_sending(int64_t stream_id, uint64_t app_error_code); From 6d29de0f1e5dd0bef3daf057e374cbaa16b0345a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Sep 2021 17:22:25 +0900 Subject: [PATCH 012/142] Fix compile error --- src/shrpx_http3_upstream.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shrpx_http3_upstream.h b/src/shrpx_http3_upstream.h index 9542bca6..0a33dd9f 100644 --- a/src/shrpx_http3_upstream.h +++ b/src/shrpx_http3_upstream.h @@ -137,7 +137,7 @@ public: int stream_close(int64_t stream_id, uint64_t app_error_code); void log_response_headers(Downstream *downstream, const std::vector &nva) const; - int http_acked_stream_data(Downstream *downstream, uin64_t datalen); + int http_acked_stream_data(Downstream *downstream, uint64_t datalen); int http_shutdown_stream_read(int64_t stream_id); int http_reset_stream(int64_t stream_id, uint64_t app_error_code); int http_send_stop_sending(int64_t stream_id, uint64_t app_error_code); From 1feeda45144fd1d4fb12621f54342114069e7ecb Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Sep 2021 21:00:41 +0900 Subject: [PATCH 013/142] nghttpx: Fix bug that worker process never exit --- src/shrpx_quic_connection_handler.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 0f3590e0..c792f489 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -596,7 +596,7 @@ CloseWait::~CloseWait() { --worker_stat->num_close_waits; if (worker->get_graceful_shutdown() && worker_stat->num_connections == 0 && - worker_stat->num_close_waits) { + worker_stat->num_close_waits == 0) { ev_break(loop); } } From fd060eb9f1490909c9022f5eb0e4938e07cdfc1b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Sep 2021 20:07:33 +0900 Subject: [PATCH 014/142] nghttpx: Connection ID encryption --- bpf/reuseport_kern.c | 337 ++++++++++++++++++++++++++- gennghttpxfun.py | 1 + src/shrpx.cc | 24 ++ src/shrpx_config.cc | 16 ++ src/shrpx_config.h | 4 + src/shrpx_connection_handler.cc | 18 +- src/shrpx_http3_upstream.cc | 22 +- src/shrpx_quic.cc | 47 +++- src/shrpx_quic.h | 13 +- src/shrpx_quic_connection_handler.cc | 47 ++-- src/shrpx_worker.cc | 25 +- 11 files changed, 513 insertions(+), 41 deletions(-) diff --git a/bpf/reuseport_kern.c b/bpf/reuseport_kern.c index 710ff2ce..25e1102a 100644 --- a/bpf/reuseport_kern.c +++ b/bpf/reuseport_kern.c @@ -38,6 +38,304 @@ * how to install kernel header files. */ +/* AES_CBC_decrypt_buffer: https://github.com/kokke/tiny-AES-c + License is Public Domain. Commit hash: + 12e7744b4919e9d55de75b7ab566326a1c8e7a67 */ + +#define AES_BLOCKLEN \ + 16 /* Block length in bytes - AES is 128b block \ + only */ + +#define AES_KEYLEN 16 /* Key length in bytes */ +#define AES_keyExpSize 176 + +struct AES_ctx { + __u8 RoundKey[AES_keyExpSize]; +}; + +/* The number of columns comprising a state in AES. This is a constant + in AES. Value=4 */ +#define Nb 4 + +#define Nk 4 /* The number of 32 bit words in a key. */ +#define Nr 10 /* The number of rounds in AES Cipher. */ + +/* state - array holding the intermediate results during + decryption. */ +typedef __u8 state_t[4][4]; + +/* The lookup-tables are marked const so they can be placed in + read-only storage instead of RAM The numbers below can be computed + dynamically trading ROM for RAM - This can be useful in (embedded) + bootloader applications, where ROM is often limited. */ +static const __u8 sbox[256] = { + /* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, + 0xfe, 0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, + 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 0xb7, 0xfd, 0x93, 0x26, + 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, + 0xeb, 0x27, 0xb2, 0x75, 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, + 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 0x53, 0xd1, 0x00, 0xed, + 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, + 0x50, 0x3c, 0x9f, 0xa8, 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, + 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 0xcd, 0x0c, 0x13, 0xec, + 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, + 0xde, 0x5e, 0x0b, 0xdb, 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, + 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 0xe7, 0xc8, 0x37, 0x6d, + 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, + 0x4b, 0xbd, 0x8b, 0x8a, 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, + 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 0xe1, 0xf8, 0x98, 0x11, + 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, + 0xb0, 0x54, 0xbb, 0x16}; + +static const __u8 rsbox[256] = { + 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, + 0x81, 0xf3, 0xd7, 0xfb, 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, + 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 0x54, 0x7b, 0x94, 0x32, + 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, + 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, + 0x6d, 0x8b, 0xd1, 0x25, 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, + 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, 0x6c, 0x70, 0x48, 0x50, + 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, + 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, + 0xb8, 0xb3, 0x45, 0x06, 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, + 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, 0x3a, 0x91, 0x11, 0x41, + 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, + 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, + 0x1c, 0x75, 0xdf, 0x6e, 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, + 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 0xfc, 0x56, 0x3e, 0x4b, + 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, + 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, + 0x27, 0x80, 0xec, 0x5f, 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, + 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, 0xa0, 0xe0, 0x3b, 0x4d, + 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, + 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, + 0x55, 0x21, 0x0c, 0x7d}; + +/* The round constant word array, Rcon[i], contains the values given + by x to the power (i-1) being powers of x (x is denoted as {02}) in + the field GF(2^8) */ +static const __u8 Rcon[11] = {0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, + 0x20, 0x40, 0x80, 0x1b, 0x36}; + +#define getSBoxValue(num) (sbox[(num)]) + +/* This function produces Nb(Nr+1) round keys. The round keys are used + in each round to decrypt the states. */ +static void KeyExpansion(__u8 *RoundKey, const __u8 *Key) { + unsigned i, j, k; + __u8 tempa[4]; /* Used for the column/row operations */ + + /* The first round key is the key itself. */ + for (i = 0; i < Nk; ++i) { + RoundKey[(i * 4) + 0] = Key[(i * 4) + 0]; + RoundKey[(i * 4) + 1] = Key[(i * 4) + 1]; + RoundKey[(i * 4) + 2] = Key[(i * 4) + 2]; + RoundKey[(i * 4) + 3] = Key[(i * 4) + 3]; + } + + /* All other round keys are found from the previous round keys. */ + for (i = Nk; i < Nb * (Nr + 1); ++i) { + { + k = (i - 1) * 4; + tempa[0] = RoundKey[k + 0]; + tempa[1] = RoundKey[k + 1]; + tempa[2] = RoundKey[k + 2]; + tempa[3] = RoundKey[k + 3]; + } + + if (i % Nk == 0) { + /* This function shifts the 4 bytes in a word to the left once. + [a0,a1,a2,a3] becomes [a1,a2,a3,a0] */ + + /* Function RotWord() */ + { + const __u8 u8tmp = tempa[0]; + tempa[0] = tempa[1]; + tempa[1] = tempa[2]; + tempa[2] = tempa[3]; + tempa[3] = u8tmp; + } + + /* SubWord() is a function that takes a four-byte input word and + applies the S-box to each of the four bytes to produce an + output word. */ + + /* Function Subword() */ + { + tempa[0] = getSBoxValue(tempa[0]); + tempa[1] = getSBoxValue(tempa[1]); + tempa[2] = getSBoxValue(tempa[2]); + tempa[3] = getSBoxValue(tempa[3]); + } + + tempa[0] = tempa[0] ^ Rcon[i / Nk]; + } + j = i * 4; + k = (i - Nk) * 4; + RoundKey[j + 0] = RoundKey[k + 0] ^ tempa[0]; + RoundKey[j + 1] = RoundKey[k + 1] ^ tempa[1]; + RoundKey[j + 2] = RoundKey[k + 2] ^ tempa[2]; + RoundKey[j + 3] = RoundKey[k + 3] ^ tempa[3]; + } +} + +static void AES_init_ctx(struct AES_ctx *ctx, const __u8 *key) { + KeyExpansion(ctx->RoundKey, key); +} + +/* This function adds the round key to state. The round key is added + to the state by an XOR function. */ +static void AddRoundKey(__u8 round, state_t *state, const __u8 *RoundKey) { + __u8 i, j; + for (i = 0; i < 4; ++i) { + for (j = 0; j < 4; ++j) { + (*state)[i][j] ^= RoundKey[(round * Nb * 4) + (i * Nb) + j]; + } + } +} + +static __u8 xtime(__u8 x) { return ((x << 1) ^ (((x >> 7) & 1) * 0x1b)); } + +#define Multiply(x, y) \ + (((y & 1) * x) ^ ((y >> 1 & 1) * xtime(x)) ^ \ + ((y >> 2 & 1) * xtime(xtime(x))) ^ \ + ((y >> 3 & 1) * xtime(xtime(xtime(x)))) ^ \ + ((y >> 4 & 1) * xtime(xtime(xtime(xtime(x)))))) + +#define getSBoxInvert(num) (rsbox[(num)]) + +/* MixColumns function mixes the columns of the state matrix. The + method used to multiply may be difficult to understand for the + inexperienced. Please use the references to gain more + information. */ +static void InvMixColumns(state_t *state) { + int i; + __u8 a, b, c, d; + for (i = 0; i < 4; ++i) { + a = (*state)[i][0]; + b = (*state)[i][1]; + c = (*state)[i][2]; + d = (*state)[i][3]; + + (*state)[i][0] = Multiply(a, 0x0e) ^ Multiply(b, 0x0b) ^ Multiply(c, 0x0d) ^ + Multiply(d, 0x09); + (*state)[i][1] = Multiply(a, 0x09) ^ Multiply(b, 0x0e) ^ Multiply(c, 0x0b) ^ + Multiply(d, 0x0d); + (*state)[i][2] = Multiply(a, 0x0d) ^ Multiply(b, 0x09) ^ Multiply(c, 0x0e) ^ + Multiply(d, 0x0b); + (*state)[i][3] = Multiply(a, 0x0b) ^ Multiply(b, 0x0d) ^ Multiply(c, 0x09) ^ + Multiply(d, 0x0e); + } +} + +/* The SubBytes Function Substitutes the values in the state matrix + with values in an S-box. */ +static void InvSubBytes(state_t *state) { + __u8 i, j; + for (i = 0; i < 4; ++i) { + for (j = 0; j < 4; ++j) { + (*state)[j][i] = getSBoxInvert((*state)[j][i]); + } + } +} + +static void InvShiftRows(state_t *state) { + __u8 temp; + + /* Rotate first row 1 columns to right */ + temp = (*state)[3][1]; + (*state)[3][1] = (*state)[2][1]; + (*state)[2][1] = (*state)[1][1]; + (*state)[1][1] = (*state)[0][1]; + (*state)[0][1] = temp; + + /* Rotate second row 2 columns to right */ + temp = (*state)[0][2]; + (*state)[0][2] = (*state)[2][2]; + (*state)[2][2] = temp; + + temp = (*state)[1][2]; + (*state)[1][2] = (*state)[3][2]; + (*state)[3][2] = temp; + + /* Rotate third row 3 columns to right */ + temp = (*state)[0][3]; + (*state)[0][3] = (*state)[1][3]; + (*state)[1][3] = (*state)[2][3]; + (*state)[2][3] = (*state)[3][3]; + (*state)[3][3] = temp; +} + +static void InvCipher(state_t *state, const __u8 *RoundKey) { + /* Add the First round key to the state before starting the + rounds. */ + AddRoundKey(Nr, state, RoundKey); + + /* There will be Nr rounds. The first Nr-1 rounds are identical. + These Nr rounds are executed in the loop below. Last one without + InvMixColumn() */ + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 1, state, RoundKey); + InvMixColumns(state); + + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 2, state, RoundKey); + InvMixColumns(state); + + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 3, state, RoundKey); + InvMixColumns(state); + + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 4, state, RoundKey); + InvMixColumns(state); + + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 5, state, RoundKey); + InvMixColumns(state); + + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 6, state, RoundKey); + InvMixColumns(state); + + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 7, state, RoundKey); + InvMixColumns(state); + + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 8, state, RoundKey); + InvMixColumns(state); + + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 9, state, RoundKey); + InvMixColumns(state); + + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(Nr - 10, state, RoundKey); +} + +static void AES_ECB_decrypt(const struct AES_ctx *ctx, __u8 *buf) { + /* The next function call decrypts the PlainText with the Key using + AES algorithm. */ + InvCipher((state_t *)buf, ctx->RoundKey); +} + /* rol32: From linux kernel source code */ /** @@ -125,13 +423,13 @@ struct bpf_map_def SEC("maps") reuseport_array = { struct bpf_map_def SEC("maps") sk_info = { .type = BPF_MAP_TYPE_ARRAY, - .max_entries = 1, + .max_entries = 3, .key_size = sizeof(__u32), - .value_size = sizeof(__u32), + .value_size = sizeof(__u64), }; typedef struct quic_hd { - const __u8 *dcid; + __u8 *dcid; __u32 dcidlen; __u32 dcid_offset; __u8 type; @@ -149,9 +447,8 @@ enum { NGTCP2_PKT_SHORT = 0x40, }; -static inline int parse_quic(quic_hd *qhd, const __u8 *data, - const __u8 *data_end) { - const __u8 *p; +static inline int parse_quic(quic_hd *qhd, __u8 *data, __u8 *data_end) { + __u8 *p; __u64 dcidlen; if (*data & 0x80) { @@ -192,7 +489,7 @@ static __u32 hash(const __u8 *data, __u32 datalen, __u32 initval) { static __u32 sk_index_from_dcid(const quic_hd *qhd, const struct sk_reuseport_md *reuse_md, - __u32 num_socks) { + __u64 num_socks) { __u32 len = qhd->dcidlen; __u32 h = reuse_md->hash; __u8 hbuf[8]; @@ -259,11 +556,13 @@ static __u32 sk_index_from_dcid(const quic_hd *qhd, SEC("sk_reuseport") int select_reuseport(struct sk_reuseport_md *reuse_md) { __u32 sk_index, *psk_index; - __u32 *pnum_socks; - __u32 zero = 0; + __u64 *pnum_socks, *pkey; + __u32 zero = 0, key_high_idx = 1, key_low_idx = 2; int rv; quic_hd qhd; __u8 qpktbuf[6 + MAX_DCIDLEN]; + struct AES_ctx aes_ctx; + __u8 key[AES_KEYLEN]; if (bpf_skb_load_bytes(reuse_md, sizeof(struct udphdr), qpktbuf, sizeof(qpktbuf)) != 0) { @@ -275,15 +574,33 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) { return SK_DROP; } + pkey = bpf_map_lookup_elem(&sk_info, &key_high_idx); + if (pkey == NULL) { + return SK_DROP; + } + + __builtin_memcpy(key, pkey, sizeof(*pkey)); + + pkey = bpf_map_lookup_elem(&sk_info, &key_low_idx); + if (pkey == NULL) { + return SK_DROP; + } + + __builtin_memcpy(key + sizeof(*pkey), pkey, sizeof(*pkey)); + rv = parse_quic(&qhd, qpktbuf, qpktbuf + sizeof(qpktbuf)); if (rv != 0) { return SK_DROP; } + AES_init_ctx(&aes_ctx, key); + switch (qhd.type) { case NGTCP2_PKT_INITIAL: case NGTCP2_PKT_0RTT: if (qhd.dcidlen == SV_DCIDLEN) { + AES_ECB_decrypt(&aes_ctx, qhd.dcid); + psk_index = bpf_map_lookup_elem(&cid_prefix_map, qhd.dcid); if (psk_index != NULL) { sk_index = *psk_index; @@ -301,6 +618,8 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) { return SK_DROP; } + AES_ECB_decrypt(&aes_ctx, qhd.dcid); + psk_index = bpf_map_lookup_elem(&cid_prefix_map, qhd.dcid); if (psk_index == NULL) { sk_index = sk_index_from_dcid(&qhd, reuse_md, *pnum_socks); diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 13fecf26..17eefd7b 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -192,6 +192,7 @@ OPTIONS = [ "frontend-quic-qlog-dir", "frontend-quic-require-token", "frontend-quic-congestion-controller", + "frontend-quic-connection-id-encryption-key", ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index c60e4143..1aa47dc4 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1856,6 +1856,14 @@ void fill_default_config(Config *config) { bpfconf.prog_file = StringRef::from_lit(PKGLIBDIR "/reuseport_kern.o"); upstreamconf.congestion_controller = NGTCP2_CC_ALGO_CUBIC; + + // TODO Not really nice to generate random key here, but fine for + // now. + if (RAND_bytes(upstreamconf.cid_encryption_key.data(), + upstreamconf.cid_encryption_key.size()) != 1) { + assert(0); + abort(); + } } auto &http3conf = config->http3; @@ -3237,6 +3245,14 @@ HTTP/3 and QUIC: ? "cubic" : "bbr") << R"( + --frontend-quic-connection-id-encryption-key= + Specify Connection ID encryption key. The encryption + key must be 16 bytes, and it must be encoded in hex + string (which is 32 bytes long). If this option is + omitted, new key is generated. In order to survive QUIC + connection in a configuration reload event, old and new + configuration must have this option and share the same + key. --no-quic-bpf Disable eBPF. --frontend-http3-window-size= @@ -4035,6 +4051,8 @@ int main(int argc, char **argv) { 182}, {SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER.c_str(), required_argument, &flag, 183}, + {SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY.c_str(), + required_argument, &flag, 184}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -4912,6 +4930,12 @@ int main(int argc, char **argv) { cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER, StringRef{optarg}); break; + case 184: + // --frontend-quic-connection-id-encryption-key + cmdcfgs.emplace_back( + SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY, + StringRef{optarg}); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index be272f62..9a2f24a7 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2684,6 +2684,10 @@ int option_lookup_token(const char *name, size_t namelen) { case 42: switch (name[41]) { case 'y': + if (util::strieq_l("frontend-quic-connection-id-encryption-ke", name, + 41)) { + return SHRPX_OPTID_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY; + } if (util::strieq_l("tls-session-cache-memcached-address-famil", name, 41)) { return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY; @@ -4013,6 +4017,18 @@ int parse_config(Config *config, int optid, const StringRef &opt, } #endif // ENABLE_HTTP3 + return 0; + case SHRPX_OPTID_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY: +#ifdef ENABLE_HTTP3 + if (optarg.size() != config->quic.upstream.cid_encryption_key.size() * 2 || + !util::is_hex_string(optarg)) { + LOG(ERROR) << opt << ": must be a hex-string"; + return -1; + } + util::decode_hex(std::begin(config->quic.upstream.cid_encryption_key), + optarg); +#endif // ENABLE_HTTP3 + return 0; case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index c46ebe5b..6202b174 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -391,6 +391,8 @@ constexpr auto SHRPX_OPT_FRONTEND_QUIC_REQUIRE_TOKEN = StringRef::from_lit("frontend-quic-require-token"); constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER = StringRef::from_lit("frontend-quic-congestion-controller"); +constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY = + StringRef::from_lit("frontend-quic-connection-id-encryption-key"); constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -761,6 +763,7 @@ struct QUICConfig { ngtcp2_cc_algo congestion_controller; bool early_data; bool require_token; + std::array cid_encryption_key; } upstream; struct { StringRef prog_file; @@ -1214,6 +1217,7 @@ enum { SHRPX_OPTID_FRONTEND_MAX_REQUESTS, SHRPX_OPTID_FRONTEND_NO_TLS, SHRPX_OPTID_FRONTEND_QUIC_CONGESTION_CONTROLLER, + SHRPX_OPTID_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY, SHRPX_OPTID_FRONTEND_QUIC_DEBUG_LOG, SHRPX_OPTID_FRONTEND_QUIC_EARLY_DATA, SHRPX_OPTID_FRONTEND_QUIC_IDLE_TIMEOUT, diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index c32eb86f..c1c72a02 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -1265,8 +1265,8 @@ int ConnectionHandler::quic_ipc_read() { return -1; } - if (dcidlen < SHRPX_QUIC_CID_PREFIXLEN) { - LOG(ERROR) << "DCID is too short"; + if (dcidlen != SHRPX_QUIC_SCIDLEN) { + LOG(ERROR) << "DCID length is invalid"; return -1; } @@ -1287,8 +1287,20 @@ int ConnectionHandler::quic_ipc_read() { return 0; } + auto config = get_config(); + auto &quicconf = config->quic; + + std::array decrypted_dcid; + + if (decrypt_quic_connection_id(decrypted_dcid.data(), dcid, + quicconf.upstream.cid_encryption_key.data()) != + 0) { + return -1; + } + for (auto &worker : workers_) { - if (!std::equal(dcid, dcid + SHRPX_QUIC_CID_PREFIXLEN, + if (!std::equal(std::begin(decrypted_dcid), + std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, worker->get_cid_prefix())) { continue; } diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 492bea4b..e21ec9bd 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -216,7 +216,12 @@ int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token, auto handler = upstream->get_client_handler(); auto worker = handler->get_worker(); - if (generate_quic_connection_id(cid, cidlen, worker->get_cid_prefix()) != 0) { + auto config = get_config(); + auto &quicconf = config->quic; + + if (generate_encrypted_quic_connection_id( + cid, cidlen, worker->get_cid_prefix(), + quicconf.upstream.cid_encryption_key.data()) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } @@ -546,17 +551,18 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, shrpx::stream_stop_sending, }; - ngtcp2_cid scid; - - if (generate_quic_connection_id(&scid, SHRPX_QUIC_SCIDLEN, - worker->get_cid_prefix()) != 0) { - return -1; - } - auto config = get_config(); auto &quicconf = config->quic; auto &http3conf = config->http3; + ngtcp2_cid scid; + + if (generate_encrypted_quic_connection_id( + &scid, SHRPX_QUIC_SCIDLEN, worker->get_cid_prefix(), + quicconf.upstream.cid_encryption_key.data()) != 0) { + return -1; + } + ngtcp2_settings settings; ngtcp2_settings_default(&settings); if (quicconf.upstream.debug.log) { diff --git a/src/shrpx_quic.cc b/src/shrpx_quic.cc index 0cde3618..1c6bee52 100644 --- a/src/shrpx_quic.cc +++ b/src/shrpx_quic.cc @@ -155,8 +155,9 @@ int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen) { return 0; } -int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen, - const uint8_t *cid_prefix) { +int generate_encrypted_quic_connection_id(ngtcp2_cid *cid, size_t cidlen, + const uint8_t *cid_prefix, + const uint8_t *key) { assert(cidlen > SHRPX_QUIC_CID_PREFIXLEN); auto p = std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, cid->data); @@ -167,6 +168,48 @@ int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen, cid->datalen = cidlen; + return encrypt_quic_connection_id(cid->data, cid->data, key); +} + +int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, + const uint8_t *key) { + auto ctx = EVP_CIPHER_CTX_new(); + auto d = defer(EVP_CIPHER_CTX_free, ctx); + + if (!EVP_EncryptInit_ex(ctx, EVP_aes_128_ecb(), nullptr, key, nullptr)) { + return -1; + } + + EVP_CIPHER_CTX_set_padding(ctx, 0); + + int len; + + if (!EVP_EncryptUpdate(ctx, dest, &len, src, SHRPX_QUIC_DECRYPTED_DCIDLEN) || + !EVP_EncryptFinal_ex(ctx, dest + len, &len)) { + return -1; + } + + return 0; +} + +int decrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, + const uint8_t *key) { + auto ctx = EVP_CIPHER_CTX_new(); + auto d = defer(EVP_CIPHER_CTX_free, ctx); + + if (!EVP_DecryptInit_ex(ctx, EVP_aes_128_ecb(), nullptr, key, nullptr)) { + return -1; + } + + EVP_CIPHER_CTX_set_padding(ctx, 0); + + int len; + + if (!EVP_DecryptUpdate(ctx, dest, &len, src, SHRPX_QUIC_DECRYPTED_DCIDLEN) || + !EVP_DecryptFinal_ex(ctx, dest + len, &len)) { + return -1; + } + return 0; } diff --git a/src/shrpx_quic.h b/src/shrpx_quic.h index 2fa12e2d..4938a15c 100644 --- a/src/shrpx_quic.h +++ b/src/shrpx_quic.h @@ -59,6 +59,8 @@ struct UpstreamAddr; constexpr size_t SHRPX_QUIC_SCIDLEN = 20; constexpr size_t SHRPX_QUIC_CID_PREFIXLEN = 8; +constexpr size_t SHRPX_QUIC_DECRYPTED_DCIDLEN = 16; +constexpr size_t SHRPX_QUIC_CID_ENCRYPTION_KEYLEN = 16; constexpr size_t SHRPX_QUIC_MAX_UDP_PAYLOAD_SIZE = 1472; constexpr size_t SHRPX_QUIC_STATELESS_RESET_SECRETLEN = 32; constexpr size_t SHRPX_QUIC_TOKEN_SECRETLEN = 32; @@ -74,8 +76,15 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen); -int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen, - const uint8_t *cid_prefix); +int generate_encrypted_quic_connection_id(ngtcp2_cid *cid, size_t cidlen, + const uint8_t *cid_prefix, + const uint8_t *key); + +int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, + const uint8_t *key); + +int decrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, + const uint8_t *key); int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid, const uint8_t *secret, diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index c792f489..49991f03 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -90,20 +90,33 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, ClientHandler *handler; + auto &quicconf = config->quic; + auto it = connections_.find(dcid_key); if (it == std::end(connections_)) { - if (!std::equal(dcid, dcid + SHRPX_QUIC_CID_PREFIXLEN, - worker_->get_cid_prefix())) { - auto quic_lwp = - conn_handler->match_quic_lingering_worker_process_cid_prefix(dcid, - dcidlen); - if (quic_lwp) { - if (conn_handler->forward_quic_packet_to_lingering_worker_process( - quic_lwp, remote_addr, local_addr, data, datalen) == 0) { + std::array decrypted_dcid; + + if (dcidlen == SHRPX_QUIC_SCIDLEN) { + if (decrypt_quic_connection_id( + decrypted_dcid.data(), dcid, + quicconf.upstream.cid_encryption_key.data()) != 0) { + return 0; + } + + if (!std::equal(std::begin(decrypted_dcid), + std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, + worker_->get_cid_prefix())) { + auto quic_lwp = + conn_handler->match_quic_lingering_worker_process_cid_prefix( + decrypted_dcid.data(), decrypted_dcid.size()); + if (quic_lwp) { + if (conn_handler->forward_quic_packet_to_lingering_worker_process( + quic_lwp, remote_addr, local_addr, data, datalen) == 0) { + return 0; + } + return 0; } - - return 0; } } @@ -134,14 +147,14 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, const uint8_t *token = nullptr; size_t tokenlen = 0; - auto &quicconf = config->quic; - switch (ngtcp2_accept(&hd, data, datalen)) { case 0: { // If we get Initial and it has the CID prefix of this worker, it // is likely that client is intentionally use the our prefix. // Just drop it. - if (std::equal(dcid, dcid + SHRPX_QUIC_CID_PREFIXLEN, + if (dcidlen == SHRPX_QUIC_SCIDLEN && + std::equal(std::begin(decrypted_dcid), + std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, worker_->get_cid_prefix())) { return 0; } @@ -237,11 +250,13 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, return 0; default: if (!config->single_thread && !(data[0] & 0x80) && - dcidlen > SHRPX_QUIC_CID_PREFIXLEN && - !std::equal(dcid, dcid + SHRPX_QUIC_CID_PREFIXLEN, + dcidlen == SHRPX_QUIC_SCIDLEN && + !std::equal(std::begin(decrypted_dcid), + std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, worker_->get_cid_prefix())) { if (conn_handler->forward_quic_packet(faddr, remote_addr, local_addr, - dcid, data, datalen) == 0) { + decrypted_dcid.data(), data, + datalen) == 0) { return 0; } } diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 9dbdd28f..75bd7325 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -923,7 +923,7 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { } constexpr uint32_t zero = 0; - uint32_t num_socks = config->num_worker; + uint64_t num_socks = config->num_worker; if (bpf_map_update_elem(bpf_map__fd(sk_info), &zero, &num_socks, BPF_ANY) != 0) { @@ -933,6 +933,29 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { return -1; } + auto &quicconf = config->quic; + + constexpr uint32_t key_high_idx = 1; + constexpr uint32_t key_low_idx = 2; + + if (bpf_map_update_elem(bpf_map__fd(sk_info), &key_high_idx, + quicconf.upstream.cid_encryption_key.data(), + BPF_ANY) != 0) { + LOG(FATAL) << "Failed to update key_high_idx sk_info: " + << xsi_strerror(errno, errbuf.data(), errbuf.size()); + close(fd); + return -1; + } + + if (bpf_map_update_elem(bpf_map__fd(sk_info), &key_low_idx, + quicconf.upstream.cid_encryption_key.data() + 8, + BPF_ANY) != 0) { + LOG(FATAL) << "Failed to update key_low_idx sk_info: " + << xsi_strerror(errno, errbuf.data(), errbuf.size()); + close(fd); + return -1; + } + auto prog_fd = bpf_program__fd(prog); if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_REUSEPORT_EBPF, &prog_fd, From 0961295a8217df4ced6ba9b8f15c9259f5834045 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Sep 2021 22:19:52 +0900 Subject: [PATCH 015/142] nghttpx: Transform odcid into hashed cid --- src/shrpx_http3_upstream.cc | 11 ++++++++--- src/shrpx_http3_upstream.h | 1 + src/shrpx_quic.cc | 28 ++++++++++++++++++++++++++-- src/shrpx_quic.h | 9 +++++++++ src/shrpx_quic_connection_handler.cc | 9 +++++++++ 5 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index e21ec9bd..c0443700 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -118,6 +118,7 @@ size_t downstream_queue_size(Worker *worker) { Http3Upstream::Http3Upstream(ClientHandler *handler) : handler_{handler}, qlog_fd_{-1}, + hashed_scid_{}, conn_{nullptr}, tls_alert_{0}, httpconn_{nullptr}, @@ -636,7 +637,12 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, auto quic_connection_handler = worker->get_quic_connection_handler(); - quic_connection_handler->add_connection_id(&initial_hd.dcid, handler_); + if (generate_quic_hashed_connection_id(hashed_scid_, remote_addr, local_addr, + initial_hd.dcid) != 0) { + return -1; + } + + quic_connection_handler->add_connection_id(&hashed_scid_, handler_); quic_connection_handler->add_connection_id(&scid, handler_); return 0; @@ -1324,8 +1330,7 @@ void Http3Upstream::on_handler_delete() { auto worker = handler_->get_worker(); auto quic_conn_handler = worker->get_quic_connection_handler(); - quic_conn_handler->remove_connection_id( - ngtcp2_conn_get_client_initial_dcid(conn_)); + quic_conn_handler->remove_connection_id(&hashed_scid_); std::vector scids(ngtcp2_conn_get_num_scid(conn_)); ngtcp2_conn_get_scid(conn_, scids.data()); diff --git a/src/shrpx_http3_upstream.h b/src/shrpx_http3_upstream.h index 0a33dd9f..0778a3f5 100644 --- a/src/shrpx_http3_upstream.h +++ b/src/shrpx_http3_upstream.h @@ -159,6 +159,7 @@ private: ev_timer shutdown_timer_; ev_prepare prep_; int qlog_fd_; + ngtcp2_cid hashed_scid_; ngtcp2_conn *conn_; quic::Error last_error_; uint8_t tls_alert_; diff --git a/src/shrpx_quic.cc b/src/shrpx_quic.cc index 1c6bee52..fc710b48 100644 --- a/src/shrpx_quic.cc +++ b/src/shrpx_quic.cc @@ -43,8 +43,6 @@ #include "util.h" #include "xsi_strerror.h" -using namespace nghttp2; - bool operator==(const ngtcp2_cid &lhs, const ngtcp2_cid &rhs) { return ngtcp2_cid_eq(&lhs, &rhs); } @@ -213,6 +211,32 @@ int decrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, return 0; } +int generate_quic_hashed_connection_id(ngtcp2_cid &dest, + const Address &remote_addr, + const Address &local_addr, + const ngtcp2_cid &cid) { + auto ctx = EVP_MD_CTX_new(); + auto d = defer(EVP_MD_CTX_free, ctx); + + std::array h; + unsigned int hlen = EVP_MD_size(EVP_sha256()); + + if (!EVP_DigestInit_ex(ctx, EVP_sha256(), nullptr) || + !EVP_DigestUpdate(ctx, &remote_addr.su.sa, remote_addr.len) || + !EVP_DigestUpdate(ctx, &local_addr.su.sa, local_addr.len) || + !EVP_DigestUpdate(ctx, cid.data, cid.datalen) || + !EVP_DigestFinal_ex(ctx, h.data(), &hlen)) { + return -1; + } + + assert(hlen == h.size()); + + std::copy_n(std::begin(h), sizeof(dest.data), std::begin(dest.data)); + dest.datalen = sizeof(dest.data); + + return 0; +} + int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid, const uint8_t *secret, size_t secretlen) { diff --git a/src/shrpx_quic.h b/src/shrpx_quic.h index 4938a15c..8bf6e4ab 100644 --- a/src/shrpx_quic.h +++ b/src/shrpx_quic.h @@ -33,6 +33,10 @@ #include +#include "network.h" + +using namespace nghttp2; + namespace std { template <> struct hash { std::size_t operator()(const ngtcp2_cid &cid) const noexcept { @@ -86,6 +90,11 @@ int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, int decrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, const uint8_t *key); +int generate_quic_hashed_connection_id(ngtcp2_cid &dest, + const Address &remote_addr, + const Address &local_addr, + const ngtcp2_cid &cid); + int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid, const uint8_t *secret, size_t secretlen); diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 49991f03..38acdf31 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -93,6 +93,15 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, auto &quicconf = config->quic; auto it = connections_.find(dcid_key); + if ((data[0] & 0x80) && it == std::end(connections_)) { + if (generate_quic_hashed_connection_id(dcid_key, remote_addr, local_addr, + dcid_key) != 0) { + return 0; + } + + it = connections_.find(dcid_key); + } + if (it == std::end(connections_)) { std::array decrypted_dcid; From 789b7a5ff19389dbed488bcabfb08c1afd51bc5e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Sep 2021 22:50:47 +0900 Subject: [PATCH 016/142] Update doc --- doc/sources/nghttpx-howto.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index bbcae298..27b4cc94 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -529,7 +529,10 @@ nghttpx does not support HTTP/3 on backend connection. Hot swapping (SIGUSR2) or configuration reload (SIGHUP) require eBPF program. Without eBPF, old worker processes keep getting HTTP/3 -traffic and do not work as intended. +traffic and do not work as intended. Connection ID encryption key +must be set with +:option:`--frontend-quic-connection-id-encryption-key` and must not +change in order to keep the existing connections alive during reload. In order announce that HTTP/3 endpoint is available, you should specify alt-svc header field. For example, the following options send From f0108ece6fafe885d97cd905ed0f803857ca6d2d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Sep 2021 22:55:21 +0900 Subject: [PATCH 017/142] Update manual pages --- doc/h2load.1 | 7 +++++- doc/h2load.1.rst | 4 ++++ doc/nghttp.1 | 2 +- doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 60 +++++++++++++++++++++++++++++++++++++++++------ doc/nghttpx.1.rst | 53 ++++++++++++++++++++++++++++++++++++----- 6 files changed, 112 insertions(+), 16 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index eb8d7b43..9b6a241c 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "Aug 31, 2021" "1.45.0-DEV" "nghttp2" +.TH "H2LOAD" "1" "Sep 15, 2021" "1.45.0-DEV" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . @@ -328,6 +328,11 @@ Disable UDP GSO. .UNINDENT .INDENT 0.0 .TP +.B \-\-max\-udp\-payload\-size= +Specify the maximum outgoing UDP datagram payload size. +.UNINDENT +.INDENT 0.0 +.TP .B \-v, \-\-verbose Output debug information. .UNINDENT diff --git a/doc/h2load.1.rst b/doc/h2load.1.rst index c4e733f2..66173807 100644 --- a/doc/h2load.1.rst +++ b/doc/h2load.1.rst @@ -277,6 +277,10 @@ OPTIONS Disable UDP GSO. +.. option:: --max-udp-payload-size= + + Specify the maximum outgoing UDP datagram payload size. + .. option:: -v, --verbose Output debug information. diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 72f579b8..10e4c5b3 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "Aug 31, 2021" "1.45.0-DEV" "nghttp2" +.TH "NGHTTP" "1" "Sep 15, 2021" "1.45.0-DEV" "nghttp2" .SH NAME nghttp \- HTTP/2 client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index 9a4efc7b..0a29da62 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "Aug 31, 2021" "1.45.0-DEV" "nghttp2" +.TH "NGHTTPD" "1" "Sep 15, 2021" "1.45.0-DEV" "nghttp2" .SH NAME nghttpd \- HTTP/2 server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index 2be0a35b..95baf730 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "Aug 31, 2021" "1.45.0-DEV" "nghttp2" +.TH "NGHTTPX" "1" "Sep 15, 2021" "1.45.0-DEV" "nghttp2" .SH NAME nghttpx \- HTTP/2 proxy . @@ -1089,12 +1089,13 @@ option. But be aware its implications. .INDENT 0.0 .TP .B \-\-tls\-no\-postpone\-early\-data -By default, nghttpx postpones forwarding HTTP requests -sent in early data, including those sent in partially in -it, until TLS handshake finishes. If all backend server -recognizes "Early\-Data" header field, using this option -makes nghttpx not postpone forwarding request and get -full potential of 0\-RTT data. +By default, except for QUIC connections, nghttpx +postpones forwarding HTTP requests sent in early data, +including those sent in partially in it, until TLS +handshake finishes. If all backend server recognizes +"Early\-Data" header field, using this option makes +nghttpx not postpone forwarding request and get full +potential of 0\-RTT data. .UNINDENT .INDENT 0.0 .TP @@ -1741,6 +1742,51 @@ Default: \fB/usr/local/lib/nghttp2/reuseport_kern.o\fP .UNINDENT .INDENT 0.0 .TP +.B \-\-frontend\-quic\-early\-data +Enable early data on frontend QUIC connections. nghttpx +sends "Early\-Data" header field to a backend server if a +request is received in early data and handshake has not +finished. All backend servers should deal with possibly +replayed requests. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-frontend\-quic\-qlog\-dir= +Specify a directory where a qlog file is written for +frontend QUIC connections. A qlog file is created per +each QUIC connection. The file name is ISO8601 basic +format, followed by "\-", server Source Connection ID and +".qlog". +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-frontend\-quic\-require\-token +Require an address validation token for a frontend QUIC +connection. Server sends a token in Retry packet or +NEW_TOKEN frame in the previous connection. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-frontend\-quic\-congestion\-controller= +Specify a congestion controller algorithm for a frontend +QUIC connection. should be either "cubic" or +"bbr". +.sp +Default: \fBcubic\fP +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-frontend\-quic\-connection\-id\-encryption\-key= +Specify Connection ID encryption key. The encryption +key must be 16 bytes, and it must be encoded in hex +string (which is 32 bytes long). If this option is +omitted, new key is generated. In order to survive QUIC +connection in a configuration reload event, old and new +configuration must have this option and share the same +key. +.UNINDENT +.INDENT 0.0 +.TP .B \-\-no\-quic\-bpf Disable eBPF. .UNINDENT diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index fb4a0a92..21592553 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -1003,12 +1003,13 @@ SSL/TLS .. option:: --tls-no-postpone-early-data - By default, nghttpx postpones forwarding HTTP requests - sent in early data, including those sent in partially in - it, until TLS handshake finishes. If all backend server - recognizes "Early-Data" header field, using this option - makes nghttpx not postpone forwarding request and get - full potential of 0-RTT data. + By default, except for QUIC connections, nghttpx + postpones forwarding HTTP requests sent in early data, + including those sent in partially in it, until TLS + handshake finishes. If all backend server recognizes + "Early-Data" header field, using this option makes + nghttpx not postpone forwarding request and get full + potential of 0-RTT data. .. option:: --tls-max-early-data= @@ -1589,6 +1590,46 @@ HTTP/3 and QUIC Default: ``/usr/local/lib/nghttp2/reuseport_kern.o`` +.. option:: --frontend-quic-early-data + + Enable early data on frontend QUIC connections. nghttpx + sends "Early-Data" header field to a backend server if a + request is received in early data and handshake has not + finished. All backend servers should deal with possibly + replayed requests. + +.. option:: --frontend-quic-qlog-dir= + + Specify a directory where a qlog file is written for + frontend QUIC connections. A qlog file is created per + each QUIC connection. The file name is ISO8601 basic + format, followed by "-", server Source Connection ID and + ".qlog". + +.. option:: --frontend-quic-require-token + + Require an address validation token for a frontend QUIC + connection. Server sends a token in Retry packet or + NEW_TOKEN frame in the previous connection. + +.. option:: --frontend-quic-congestion-controller= + + Specify a congestion controller algorithm for a frontend + QUIC connection. should be either "cubic" or + "bbr". + + Default: ``cubic`` + +.. option:: --frontend-quic-connection-id-encryption-key= + + Specify Connection ID encryption key. The encryption + key must be 16 bytes, and it must be encoded in hex + string (which is 32 bytes long). If this option is + omitted, new key is generated. In order to survive QUIC + connection in a configuration reload event, old and new + configuration must have this option and share the same + key. + .. option:: --no-quic-bpf Disable eBPF. From e2e6d827c7836c0ec6f2789a42695cfcdff02c73 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Sep 2021 22:55:38 +0900 Subject: [PATCH 018/142] Update bash_completion --- doc/bash_completion/h2load | 2 +- doc/bash_completion/nghttpx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/bash_completion/h2load b/doc/bash_completion/h2load index f90dbe2f..814b4fe4 100644 --- a/doc/bash_completion/h2load +++ b/doc/bash_completion/h2load @@ -8,7 +8,7 @@ _h2load() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--connection-window-bits --clients --verbose --ciphers --rate --no-tls-proto --connect-to --header-table-size --requests --log-file --base-uri --no-udp-gso --h1 --threads --npn-list --rate-period --rps --data --tls13-ciphers --version --connection-inactivity-timeout --timing-script-file --encoder-header-table-size --max-concurrent-streams --connection-active-timeout --input-file --help --groups --window-bits --qlog-file-base --warm-up-time --duration --header ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--connection-window-bits --help --clients --verbose --ciphers --rate --no-tls-proto --connect-to --header-table-size --requests --log-file --base-uri --no-udp-gso --h1 --threads --npn-list --rate-period --rps --data --tls13-ciphers --version --connection-inactivity-timeout --timing-script-file --encoder-header-table-size --max-concurrent-streams --connection-active-timeout --input-file --max-udp-payload-size --groups --window-bits --qlog-file-base --warm-up-time --duration --header ' -- "$cur" ) ) ;; *) _filedir diff --git a/doc/bash_completion/nghttpx b/doc/bash_completion/nghttpx index 0e95f54b..4c4128ea 100644 --- a/doc/bash_completion/nghttpx +++ b/doc/bash_completion/nghttpx @@ -8,7 +8,7 @@ _nghttpx() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--worker-read-rate --include --frontend-http2-dump-response-header --tls-ticket-key-file --verify-client-cacert --max-response-header-fields --backend-http2-window-size --tls13-client-ciphers --frontend-keep-alive-timeout --backend-request-buffer --max-request-header-fields --backend-connect-timeout --tls-max-proto-version --frontend-quic-idle-timeout --frontend-http3-read-timeout --conf --dns-lookup-timeout --frontend-http3-connection-window-size --backend-http2-max-concurrent-streams --worker-write-burst --no-quic-bpf --npn-list --dns-max-try --fetch-ocsp-response-file --tls-session-cache-memcached-cert-file --no-via --mruby-file --no-server-push --stream-read-timeout --client-ciphers --ocsp-update-interval --forwarded-for --accesslog-syslog --dns-cache-timeout --frontend-http2-read-timeout --listener-disable-timeout --frontend-http3-window-size --ciphers --client-psk-secrets --strip-incoming-x-forwarded-for --no-server-rewrite --private-key-passwd-file --backend-keep-alive-timeout --backend-http-proxy-uri --frontend-max-requests --tls-no-postpone-early-data --rlimit-nofile --no-strip-incoming-x-forwarded-proto --tls-ticket-key-memcached-cert-file --no-verify-ocsp --forwarded-by --tls-session-cache-memcached-private-key-file --error-page --ocsp-startup --backend-write-timeout --tls-dyn-rec-warmup-threshold --frontend-http2-window-size --tls-ticket-key-memcached-max-retry --http2-no-cookie-crumbling --frontend-http3-max-concurrent-streams --worker-read-burst --dh-param-file --accesslog-format --errorlog-syslog --redirect-https-port --request-header-field-buffer --api-max-request-body --frontend-http2-decoder-dynamic-table-size --errorlog-file --frontend-http2-max-concurrent-streams --psk-secrets --frontend-write-timeout --tls-ticket-key-cipher --read-burst --no-add-x-forwarded-proto --backend --server-name --insecure --backend-max-backoff --log-level --host-rewrite --quic-bpf-program-file --http2-altsvc --frontend-http3-max-connection-window-size --tls-ticket-key-memcached-interval --frontend-http2-setting-timeout --frontend-http2-connection-window-size --worker-frontend-connections --syslog-facility --fastopen --no-location-rewrite --single-thread --no-http2-cipher-block-list --tls-session-cache-memcached --no-ocsp --backend-response-buffer --tls-min-proto-version --workers --add-x-forwarded-for --add-forwarded --frontend-http3-max-window-size --worker-write-rate --add-request-header --backend-http2-settings-timeout --subcert --ignore-per-pattern-mruby-error --ecdh-curves --no-kqueue --help --frontend-frame-debug --tls-sct-dir --pid-file --frontend-http2-dump-request-header --daemon --write-rate --altsvc --backend-http2-decoder-dynamic-table-size --no-strip-incoming-early-data --user --verify-client-tolerate-expired --frontend-read-timeout --tls-ticket-key-memcached-max-fail --backlog --write-burst --backend-connections-per-host --tls-max-early-data --response-header-field-buffer --tls-ticket-key-memcached-address-family --padding --tls-session-cache-memcached-address-family --stream-write-timeout --cacert --frontend-quic-debug-log --tls-ticket-key-memcached-private-key-file --accesslog-write-early --backend-address-family --backend-http2-connection-window-size --tls13-ciphers --client-no-http2-cipher-block-list --version --add-response-header --backend-read-timeout --frontend-http2-optimize-window-size --frontend --accesslog-file --http2-proxy --backend-http2-encoder-dynamic-table-size --client-private-key-file --single-process --client-cert-file --tls-ticket-key-memcached --tls-dyn-rec-idle-timeout --frontend-http2-optimize-write-buffer-size --verify-client --frontend-http2-encoder-dynamic-table-size --read-rate --backend-connections-per-frontend --strip-incoming-forwarded ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--worker-read-rate --include --frontend-http2-dump-response-header --tls-ticket-key-file --frontend-quic-qlog-dir --verify-client-cacert --max-response-header-fields --backend-http2-window-size --tls13-client-ciphers --frontend-keep-alive-timeout --backend-request-buffer --max-request-header-fields --backend-connect-timeout --frontend-quic-require-token --tls-max-proto-version --frontend-quic-idle-timeout --frontend-http3-read-timeout --conf --dns-lookup-timeout --frontend-http3-connection-window-size --backend-http2-max-concurrent-streams --worker-write-burst --no-quic-bpf --npn-list --dns-max-try --fetch-ocsp-response-file --tls-session-cache-memcached-cert-file --no-via --mruby-file --no-server-push --stream-read-timeout --client-ciphers --ocsp-update-interval --frontend-quic-connection-id-encryption-key --forwarded-for --accesslog-syslog --dns-cache-timeout --frontend-http2-read-timeout --listener-disable-timeout --frontend-http3-window-size --ciphers --client-psk-secrets --strip-incoming-x-forwarded-for --no-server-rewrite --private-key-passwd-file --backend-keep-alive-timeout --backend-http-proxy-uri --frontend-quic-early-data --frontend-max-requests --tls-no-postpone-early-data --rlimit-nofile --no-strip-incoming-x-forwarded-proto --tls-ticket-key-memcached-cert-file --no-verify-ocsp --forwarded-by --tls-session-cache-memcached-private-key-file --error-page --ocsp-startup --backend-write-timeout --tls-dyn-rec-warmup-threshold --frontend-http2-window-size --tls-ticket-key-memcached-max-retry --http2-no-cookie-crumbling --frontend-http3-max-concurrent-streams --worker-read-burst --dh-param-file --accesslog-format --errorlog-syslog --redirect-https-port --request-header-field-buffer --api-max-request-body --frontend-http2-decoder-dynamic-table-size --errorlog-file --frontend-http2-max-concurrent-streams --psk-secrets --frontend-write-timeout --tls-ticket-key-cipher --read-burst --no-add-x-forwarded-proto --backend --server-name --insecure --backend-max-backoff --log-level --host-rewrite --quic-bpf-program-file --http2-altsvc --frontend-http3-max-connection-window-size --tls-ticket-key-memcached-interval --frontend-http2-setting-timeout --frontend-http2-connection-window-size --worker-frontend-connections --syslog-facility --fastopen --no-location-rewrite --single-thread --no-http2-cipher-block-list --tls-session-cache-memcached --no-ocsp --backend-response-buffer --tls-min-proto-version --workers --add-x-forwarded-for --add-forwarded --frontend-http3-max-window-size --worker-write-rate --add-request-header --backend-http2-settings-timeout --subcert --ignore-per-pattern-mruby-error --ecdh-curves --no-kqueue --help --frontend-frame-debug --tls-sct-dir --pid-file --frontend-http2-dump-request-header --daemon --write-rate --altsvc --backend-http2-decoder-dynamic-table-size --no-strip-incoming-early-data --user --verify-client-tolerate-expired --frontend-read-timeout --tls-ticket-key-memcached-max-fail --backlog --write-burst --backend-connections-per-host --tls-max-early-data --response-header-field-buffer --tls-ticket-key-memcached-address-family --padding --tls-session-cache-memcached-address-family --frontend-quic-congestion-controller --stream-write-timeout --cacert --frontend-quic-debug-log --tls-ticket-key-memcached-private-key-file --accesslog-write-early --backend-address-family --backend-http2-connection-window-size --tls13-ciphers --client-no-http2-cipher-block-list --version --add-response-header --backend-read-timeout --frontend-http2-optimize-window-size --frontend --accesslog-file --http2-proxy --backend-http2-encoder-dynamic-table-size --client-private-key-file --single-process --client-cert-file --tls-ticket-key-memcached --tls-dyn-rec-idle-timeout --frontend-http2-optimize-write-buffer-size --verify-client --frontend-http2-encoder-dynamic-table-size --read-rate --backend-connections-per-frontend --strip-incoming-forwarded ' -- "$cur" ) ) ;; *) _filedir From 684a219e39bd2ee15876f506fd8c1b19cec4d228 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Sep 2021 23:07:46 +0900 Subject: [PATCH 019/142] nghttpx: Tweak close wait handling --- src/shrpx_quic_connection_handler.cc | 38 ++++++++++++++++++---------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 38acdf31..005c216d 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -93,13 +93,34 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, auto &quicconf = config->quic; auto it = connections_.find(dcid_key); - if ((data[0] & 0x80) && it == std::end(connections_)) { - if (generate_quic_hashed_connection_id(dcid_key, remote_addr, local_addr, - dcid_key) != 0) { + if (it == std::end(connections_)) { + auto cwit = close_waits_.find(dcid_key); + if (cwit != std::end(close_waits_)) { + auto cw = (*cwit).second; + + cw->handle_packet(faddr, remote_addr, local_addr, data, datalen); + return 0; } - it = connections_.find(dcid_key); + if (data[0] & 0x80) { + if (generate_quic_hashed_connection_id(dcid_key, remote_addr, local_addr, + dcid_key) != 0) { + return 0; + } + + it = connections_.find(dcid_key); + if (it == std::end(connections_)) { + auto cwit = close_waits_.find(dcid_key); + if (cwit != std::end(close_waits_)) { + auto cw = (*cwit).second; + + cw->handle_packet(faddr, remote_addr, local_addr, data, datalen); + + return 0; + } + } + } } if (it == std::end(connections_)) { @@ -129,15 +150,6 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, } } - auto it = close_waits_.find(dcid_key); - if (it != std::end(close_waits_)) { - auto cw = (*it).second; - - cw->handle_packet(faddr, remote_addr, local_addr, data, datalen); - - return 0; - } - // new connection auto &upstreamconf = config->conn.upstream; From 8f9744c07bf8e3bcc7a75ad760b14c7ec35314bc Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Sep 2021 23:40:46 +0900 Subject: [PATCH 020/142] nghttpx: Pass hashed_scid_ to CloseWait --- src/shrpx_http3_upstream.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index c0443700..9ed1a8cf 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -1330,10 +1330,9 @@ void Http3Upstream::on_handler_delete() { auto worker = handler_->get_worker(); auto quic_conn_handler = worker->get_quic_connection_handler(); - quic_conn_handler->remove_connection_id(&hashed_scid_); - - std::vector scids(ngtcp2_conn_get_num_scid(conn_)); + std::vector scids(ngtcp2_conn_get_num_scid(conn_) + 1); ngtcp2_conn_get_scid(conn_, scids.data()); + scids.back() = hashed_scid_; for (auto &cid : scids) { quic_conn_handler->remove_connection_id(&cid); From f3b9cd8404bd301ed7f960952548b12b7bd7310a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 16 Sep 2021 20:20:02 +0900 Subject: [PATCH 021/142] bpf: Add workaround for ubuntu 20.04 --- bpf/reuseport_kern.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bpf/reuseport_kern.c b/bpf/reuseport_kern.c index 25e1102a..a012ee9a 100644 --- a/bpf/reuseport_kern.c +++ b/bpf/reuseport_kern.c @@ -24,6 +24,7 @@ */ #include #include +#include #include @@ -240,7 +241,16 @@ static void InvSubBytes(state_t *state) { __u8 i, j; for (i = 0; i < 4; ++i) { for (j = 0; j < 4; ++j) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) + /* Ubuntu 20.04 LTS kernel 5.4.0 needs this workaround otherwise + "math between map_value pointer and register with unbounded + min value is not allowed". 5.10.0 is a kernel version that + works but it might not be a minimum version. */ + __u8 k = (*state)[j][i]; + (*state)[j][i] = k ? getSBoxInvert(k) : getSBoxInvert(0); +#else /* !(LINUX_VERSION_CDOE < KERNEL_VERSION(5, 10, 0)) */ (*state)[j][i] = getSBoxInvert((*state)[j][i]); +#endif /* !(LINUX_VERSION_CDOE < KERNEL_VERSION(5, 10, 0)) */ } } } From e167e07a9a151a511c404a9dda6915149a0567d6 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 16 Sep 2021 22:00:36 +0900 Subject: [PATCH 022/142] nghttpx: Check that HTTP response message finished safely --- src/shrpx_http_downstream_connection.cc | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 6826eb5c..8c4fe6e1 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -1218,6 +1218,18 @@ int HttpDownstreamConnection::read_clear() { } if (nread < 0) { + if (nread == SHRPX_ERR_EOF && !downstream_->get_upgraded()) { + auto htperr = llhttp_finish(&response_htp_); + if (htperr != HPE_OK) { + if (LOG_ENABLED(INFO)) { + DCLOG(INFO, this) << "HTTP response ended prematurely: " + << llhttp_errno_name(htperr); + } + + return -1; + } + } + return nread; } @@ -1337,6 +1349,18 @@ int HttpDownstreamConnection::read_tls() { } if (nread < 0) { + if (nread == SHRPX_ERR_EOF && !downstream_->get_upgraded()) { + auto htperr = llhttp_finish(&response_htp_); + if (htperr != HPE_OK) { + if (LOG_ENABLED(INFO)) { + DCLOG(INFO, this) << "HTTP response ended prematurely: " + << llhttp_errno_name(htperr); + } + + return -1; + } + } + return nread; } From 1e2081a1c554902085df7ecf6d08b6d929b623a1 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 16 Sep 2021 22:24:53 +0900 Subject: [PATCH 023/142] Fix integration test error --- integration-tests/nghttpx_http2_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/nghttpx_http2_test.go b/integration-tests/nghttpx_http2_test.go index 84646fa0..c97a45fc 100644 --- a/integration-tests/nghttpx_http2_test.go +++ b/integration-tests/nghttpx_http2_test.go @@ -565,7 +565,7 @@ func TestH2H1BadResponseCL(t *testing.T) { t.Fatalf("Error st.http2() = %v", err) } - want := http2.ErrCodeProtocol + want := http2.ErrCodeInternal if res.errCode != want { t.Errorf("res.errCode = %v; want %v", res.errCode, want) } From 095ee9683dc4f71d560331f9eaad4a6fa6205de5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 16 Sep 2021 22:37:33 +0900 Subject: [PATCH 024/142] integration: Adds tests for chunked response --- integration-tests/nghttpx_http1_test.go | 28 ++++++++++++++++++++++ integration-tests/nghttpx_http2_test.go | 32 +++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/integration-tests/nghttpx_http1_test.go b/integration-tests/nghttpx_http1_test.go index 3d416771..9eb80e60 100644 --- a/integration-tests/nghttpx_http1_test.go +++ b/integration-tests/nghttpx_http1_test.go @@ -1171,3 +1171,31 @@ Content-Length: 1000000 t.Errorf("status: %v; want %v", got, want) } } + +// TestH1H1ChunkedEndsPrematurely tests that an HTTP/1.1 request fails +// if the backend chunked encoded response ends prematurely. +func TestH1H1ChunkedEndsPrematurely(t *testing.T) { + st := newServerTester(nil, t, func(w http.ResponseWriter, r *http.Request) { + hj, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "Could not hijack the connection", http.StatusInternalServerError) + return + } + conn, bufrw, err := hj.Hijack() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer conn.Close() + bufrw.WriteString("HTTP/1.1 200\r\nTransfer-Encoding: chunked\r\n\r\n") + bufrw.Flush() + }) + defer st.Close() + + _, err := st.http1(requestParam{ + name: "TestH1H1ChunkedEndsPrematurely", + }) + if err == nil { + t.Fatal("st.http1() should fail") + } +} diff --git a/integration-tests/nghttpx_http2_test.go b/integration-tests/nghttpx_http2_test.go index c97a45fc..a856f8fc 100644 --- a/integration-tests/nghttpx_http2_test.go +++ b/integration-tests/nghttpx_http2_test.go @@ -2838,3 +2838,35 @@ func TestH2ResponseBeforeRequestEnd(t *testing.T) { t.Errorf("res.status: %v; want %v", got, want) } } + +// TestH2H1ChunkedEndsPrematurely tests that a stream is reset if the +// backend chunked encoded response ends prematurely. +func TestH2H1ChunkedEndsPrematurely(t *testing.T) { + st := newServerTester(nil, t, func(w http.ResponseWriter, r *http.Request) { + hj, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "Could not hijack the connection", http.StatusInternalServerError) + return + } + conn, bufrw, err := hj.Hijack() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer conn.Close() + bufrw.WriteString("HTTP/1.1 200\r\nTransfer-Encoding: chunked\r\n\r\n") + bufrw.Flush() + }) + defer st.Close() + + res, err := st.http2(requestParam{ + name: "TestH2H1ChunkedEndsPrematurely", + }) + if err != nil { + t.Fatalf("Error st.http2() = %v", err) + } + + if got, want := res.errCode, http2.ErrCodeInternal; got != want { + t.Errorf("res.errCode = %v; want %v", got, want) + } +} From 7cdc6cfa6d31a67c3e9e38c80da559c16522174d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 17 Sep 2021 17:52:05 +0900 Subject: [PATCH 025/142] nghttpx: Store Retry in CloseWait to rate limit its transmission --- src/shrpx_quic_connection_handler.cc | 42 ++++++++++++++++++++-------- src/shrpx_quic_connection_handler.h | 8 +++--- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 005c216d..85be3762 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -431,7 +431,8 @@ int QUICConnectionHandler::send_retry( return -1; } - std::array buf; + std::vector buf; + buf.resize(256); auto nwrite = ngtcp2_crypto_write_retry(buf.data(), buf.size(), version, &iscid, @@ -441,9 +442,28 @@ int QUICConnectionHandler::send_retry( return -1; } - return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, - &local_addr.su.sa, local_addr.len, buf.data(), nwrite, - 0); + buf.resize(nwrite); + + quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, + &local_addr.su.sa, local_addr.len, buf.data(), buf.size(), + 0); + + if (generate_quic_hashed_connection_id(idcid, remote_addr, local_addr, + idcid) != 0) { + return -1; + } + + auto d = + static_cast(NGTCP2_DEFAULT_INITIAL_RTT * 3) / NGTCP2_SECONDS; + + auto cw = std::make_unique(worker_, std::vector{idcid}, + std::move(buf), d); + + add_close_wait(cw.get()); + + cw.release(); + + return 0; } int QUICConnectionHandler::send_version_negotiation( @@ -607,10 +627,10 @@ static void close_wait_timeoutcb(struct ev_loop *loop, ev_timer *w, } CloseWait::CloseWait(Worker *worker, std::vector scids, - std::vector conn_close, ev_tstamp period) + std::vector pkt, ev_tstamp period) : worker{worker}, scids{std::move(scids)}, - conn_close{std::move(conn_close)}, + pkt{std::move(pkt)}, bytes_recv{0}, bytes_sent{0}, num_pkts_recv{0}, @@ -641,26 +661,26 @@ int CloseWait::handle_packet(const UpstreamAddr *faddr, const Address &remote_addr, const Address &local_addr, const uint8_t *data, size_t datalen) { - if (conn_close.empty()) { + if (pkt.empty()) { return 0; } ++num_pkts_recv; bytes_recv += datalen; - if (bytes_sent + conn_close.size() > 3 * bytes_recv || + if (bytes_sent + pkt.size() > 3 * bytes_recv || next_pkts_recv > num_pkts_recv) { return 0; } if (quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, - &local_addr.su.sa, local_addr.len, conn_close.data(), - conn_close.size(), 0) != 0) { + &local_addr.su.sa, local_addr.len, pkt.data(), + pkt.size(), 0) != 0) { return -1; } next_pkts_recv *= 2; - bytes_sent += conn_close.size(); + bytes_sent += pkt.size(); return 0; } diff --git a/src/shrpx_quic_connection_handler.h b/src/shrpx_quic_connection_handler.h index d6d46f05..95e598f5 100644 --- a/src/shrpx_quic_connection_handler.h +++ b/src/shrpx_quic_connection_handler.h @@ -51,7 +51,7 @@ class Worker; // closing period). struct CloseWait { CloseWait(Worker *worker, std::vector scids, - std::vector conn_close, ev_tstamp period); + std::vector pkt, ev_tstamp period); ~CloseWait(); int handle_packet(const UpstreamAddr *faddr, const Address &remote_addr, @@ -61,9 +61,9 @@ struct CloseWait { Worker *worker; // Source Connection IDs of the connection. std::vector scids; - // QUIC packet containing CONNECTION_CLOSE. It is empty when a - // connection entered in draining state. - std::vector conn_close; + // QUIC packet which is sent in response to the incoming packet. It + // might be empty. + std::vector pkt; // Close-wait (draining or closing period) timer. ev_timer timer; // The number of bytes received during close-wait period. From 1320d7efab8a29452787266a29ea14d006469c6e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 17 Sep 2021 18:30:16 +0900 Subject: [PATCH 026/142] nghttpx: Do not accept new connection during graceful shutdown period --- src/shrpx_http3_upstream.cc | 20 ++++++++++++++++++-- src/shrpx_http3_upstream.h | 1 + src/shrpx_quic_connection_handler.cc | 17 +++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 9ed1a8cf..b3c509f5 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -124,7 +124,8 @@ Http3Upstream::Http3Upstream(ClientHandler *handler) httpconn_{nullptr}, downstream_queue_{downstream_queue_size(handler->get_worker()), !get_config()->http2_proxy}, - idle_close_{false} { + idle_close_{false}, + retry_close_{false} { ev_timer_init(&timer_, timeoutcb, 0., 0.); timer_.data = this; @@ -1338,7 +1339,7 @@ void Http3Upstream::on_handler_delete() { quic_conn_handler->remove_connection_id(&cid); } - if (idle_close_) { + if (idle_close_ || retry_close_) { return; } @@ -1624,6 +1625,21 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr, return -1; } + if (worker->get_graceful_shutdown()) { + ngtcp2_cid ini_dcid, ini_scid; + + ngtcp2_cid_init(&ini_dcid, dcid, dcidlen); + ngtcp2_cid_init(&ini_scid, scid, scidlen); + + quic_conn_handler->send_connection_close( + faddr, version, &ini_dcid, &ini_scid, remote_addr, local_addr, + NGTCP2_CONNECTION_REFUSED); + + return -1; + } + + retry_close_ = true; + quic_conn_handler->send_retry(handler_->get_upstream_addr(), version, dcid, dcidlen, scid, scidlen, remote_addr, local_addr); diff --git a/src/shrpx_http3_upstream.h b/src/shrpx_http3_upstream.h index 0778a3f5..b811a4bb 100644 --- a/src/shrpx_http3_upstream.h +++ b/src/shrpx_http3_upstream.h @@ -166,6 +166,7 @@ private: nghttp3_conn *httpconn_; DownstreamQueue downstream_queue_; bool idle_close_; + bool retry_close_; std::vector conn_close_; }; diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 85be3762..28b6c17c 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -180,6 +180,12 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, return 0; } + if (worker_->get_graceful_shutdown()) { + send_connection_close(faddr, version, &hd.dcid, &hd.scid, remote_addr, + local_addr, NGTCP2_CONNECTION_REFUSED); + return 0; + } + if (hd.token.len == 0) { if (quicconf.upstream.require_token) { send_retry(faddr, version, dcid, dcidlen, scid, scidlen, remote_addr, @@ -262,6 +268,12 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, break; } case NGTCP2_ERR_RETRY: + if (worker_->get_graceful_shutdown()) { + send_connection_close(faddr, version, &hd.dcid, &hd.scid, remote_addr, + local_addr, NGTCP2_CONNECTION_REFUSED); + return 0; + } + send_retry(faddr, version, dcid, dcidlen, scid, scidlen, remote_addr, local_addr); return 0; @@ -456,6 +468,11 @@ int QUICConnectionHandler::send_retry( auto d = static_cast(NGTCP2_DEFAULT_INITIAL_RTT * 3) / NGTCP2_SECONDS; + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Enter close-wait period " << d << "s with " << buf.size() + << " bytes sentinel packet"; + } + auto cw = std::make_unique(worker_, std::vector{idcid}, std::move(buf), d); From fd107ab47cdf456c9c662f39062c3b3bc53d71c4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 17 Sep 2021 18:33:23 +0900 Subject: [PATCH 027/142] nghttpx: Refactor quic --- src/shrpx_http3_upstream.cc | 20 ++++++++-------- src/shrpx_quic.cc | 34 +++++++++++++-------------- src/shrpx_quic.h | 14 +++++------ src/shrpx_quic_connection_handler.cc | 35 +++++++++++++--------------- src/shrpx_quic_connection_handler.h | 8 +++---- 5 files changed, 54 insertions(+), 57 deletions(-) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index b3c509f5..68775eb2 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -222,7 +222,7 @@ int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token, auto &quicconf = config->quic; if (generate_encrypted_quic_connection_id( - cid, cidlen, worker->get_cid_prefix(), + *cid, cidlen, worker->get_cid_prefix(), quicconf.upstream.cid_encryption_key.data()) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } @@ -230,14 +230,14 @@ int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token, auto &quic_secret = worker->get_quic_secret(); auto &secret = quic_secret->stateless_reset_secret; - if (generate_quic_stateless_reset_token(token, cid, secret.data(), + if (generate_quic_stateless_reset_token(token, *cid, secret.data(), secret.size()) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } auto quic_connection_handler = worker->get_quic_connection_handler(); - quic_connection_handler->add_connection_id(cid, handler); + quic_connection_handler->add_connection_id(*cid, handler); return 0; } @@ -251,7 +251,7 @@ int remove_connection_id(ngtcp2_conn *conn, const ngtcp2_cid *cid, auto worker = handler->get_worker(); auto quic_conn_handler = worker->get_quic_connection_handler(); - quic_conn_handler->remove_connection_id(cid); + quic_conn_handler->remove_connection_id(*cid); return 0; } @@ -560,7 +560,7 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, ngtcp2_cid scid; if (generate_encrypted_quic_connection_id( - &scid, SHRPX_QUIC_SCIDLEN, worker->get_cid_prefix(), + scid, SHRPX_QUIC_SCIDLEN, worker->get_cid_prefix(), quicconf.upstream.cid_encryption_key.data()) != 0) { return -1; } @@ -611,7 +611,7 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, auto &quic_secret = worker->get_quic_secret(); auto &stateless_reset_secret = quic_secret->stateless_reset_secret; - rv = generate_quic_stateless_reset_token(params.stateless_reset_token, &scid, + rv = generate_quic_stateless_reset_token(params.stateless_reset_token, scid, stateless_reset_secret.data(), stateless_reset_secret.size()); if (rv != 0) { @@ -643,8 +643,8 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, return -1; } - quic_connection_handler->add_connection_id(&hashed_scid_, handler_); - quic_connection_handler->add_connection_id(&scid, handler_); + quic_connection_handler->add_connection_id(hashed_scid_, handler_); + quic_connection_handler->add_connection_id(scid, handler_); return 0; } @@ -1336,7 +1336,7 @@ void Http3Upstream::on_handler_delete() { scids.back() = hashed_scid_; for (auto &cid : scids) { - quic_conn_handler->remove_connection_id(&cid); + quic_conn_handler->remove_connection_id(cid); } if (idle_close_ || retry_close_) { @@ -1632,7 +1632,7 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr, ngtcp2_cid_init(&ini_scid, scid, scidlen); quic_conn_handler->send_connection_close( - faddr, version, &ini_dcid, &ini_scid, remote_addr, local_addr, + faddr, version, ini_dcid, ini_scid, remote_addr, local_addr, NGTCP2_CONNECTION_REFUSED); return -1; diff --git a/src/shrpx_quic.cc b/src/shrpx_quic.cc index fc710b48..aa38617a 100644 --- a/src/shrpx_quic.cc +++ b/src/shrpx_quic.cc @@ -143,30 +143,30 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, return 0; } -int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen) { - if (RAND_bytes(cid->data, cidlen) != 1) { +int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen) { + if (RAND_bytes(cid.data, cidlen) != 1) { return -1; } - cid->datalen = cidlen; + cid.datalen = cidlen; return 0; } -int generate_encrypted_quic_connection_id(ngtcp2_cid *cid, size_t cidlen, +int generate_encrypted_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, const uint8_t *cid_prefix, const uint8_t *key) { assert(cidlen > SHRPX_QUIC_CID_PREFIXLEN); - auto p = std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, cid->data); + auto p = std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, cid.data); if (RAND_bytes(p, cidlen - SHRPX_QUIC_CID_PREFIXLEN) != 1) { return -1; } - cid->datalen = cidlen; + cid.datalen = cidlen; - return encrypt_quic_connection_id(cid->data, cid->data, key); + return encrypt_quic_connection_id(cid.data, cid.data, key); } int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, @@ -237,11 +237,11 @@ int generate_quic_hashed_connection_id(ngtcp2_cid &dest, return 0; } -int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid, +int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid &cid, const uint8_t *secret, size_t secretlen) { if (ngtcp2_crypto_generate_stateless_reset_token(token, secret, secretlen, - cid) != 0) { + &cid) != 0) { return -1; } @@ -265,15 +265,15 @@ int generate_quic_token_secret(uint8_t *secret) { } int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, - socklen_t salen, const ngtcp2_cid *retry_scid, - const ngtcp2_cid *odcid, const uint8_t *token_secret) { + socklen_t salen, const ngtcp2_cid &retry_scid, + const ngtcp2_cid &odcid, const uint8_t *token_secret) { auto t = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); auto stokenlen = ngtcp2_crypto_generate_retry_token( - token, token_secret, SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, retry_scid, - odcid, t); + token, token_secret, SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, &retry_scid, + &odcid, t); if (stokenlen < 0) { return -1; } @@ -283,17 +283,17 @@ int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, return 0; } -int verify_retry_token(ngtcp2_cid *odcid, const uint8_t *token, size_t tokenlen, - const ngtcp2_cid *dcid, const sockaddr *sa, +int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen, + const ngtcp2_cid &dcid, const sockaddr *sa, socklen_t salen, const uint8_t *token_secret) { auto t = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); - if (ngtcp2_crypto_verify_retry_token(odcid, token, tokenlen, token_secret, + if (ngtcp2_crypto_verify_retry_token(&odcid, token, tokenlen, token_secret, SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, - dcid, 10 * NGTCP2_SECONDS, t) != 0) { + &dcid, 10 * NGTCP2_SECONDS, t) != 0) { return -1; } diff --git a/src/shrpx_quic.h b/src/shrpx_quic.h index 8bf6e4ab..96a109fc 100644 --- a/src/shrpx_quic.h +++ b/src/shrpx_quic.h @@ -78,9 +78,9 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, size_t local_salen, const uint8_t *data, size_t datalen, size_t gso_size); -int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen); +int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen); -int generate_encrypted_quic_connection_id(ngtcp2_cid *cid, size_t cidlen, +int generate_encrypted_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, const uint8_t *cid_prefix, const uint8_t *key); @@ -95,7 +95,7 @@ int generate_quic_hashed_connection_id(ngtcp2_cid &dest, const Address &local_addr, const ngtcp2_cid &cid); -int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid, +int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid &cid, const uint8_t *secret, size_t secretlen); @@ -104,11 +104,11 @@ int generate_quic_stateless_reset_secret(uint8_t *secret); int generate_quic_token_secret(uint8_t *secret); int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, - socklen_t salen, const ngtcp2_cid *retry_scid, - const ngtcp2_cid *odcid, const uint8_t *token_secret); + socklen_t salen, const ngtcp2_cid &retry_scid, + const ngtcp2_cid &odcid, const uint8_t *token_secret); -int verify_retry_token(ngtcp2_cid *odcid, const uint8_t *token, size_t tokenlen, - const ngtcp2_cid *dcid, const sockaddr *sa, +int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen, + const ngtcp2_cid &dcid, const sockaddr *sa, socklen_t salen, const uint8_t *token_secret); int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 28b6c17c..41b8b9e4 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -181,7 +181,7 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, } if (worker_->get_graceful_shutdown()) { - send_connection_close(faddr, version, &hd.dcid, &hd.scid, remote_addr, + send_connection_close(faddr, version, hd.dcid, hd.scid, remote_addr, local_addr, NGTCP2_CONNECTION_REFUSED); return 0; } @@ -202,7 +202,7 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, switch (hd.token.base[0]) { case NGTCP2_CRYPTO_TOKEN_MAGIC_RETRY: - if (verify_retry_token(&odcid, hd.token.base, hd.token.len, &hd.dcid, + if (verify_retry_token(odcid, hd.token.base, hd.token.len, hd.dcid, &remote_addr.su.sa, remote_addr.len, secret.data()) != 0) { if (LOG_ENABLED(INFO)) { @@ -212,7 +212,7 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, // 2nd Retry packet is not allowed, so send CONNECTION_CLOSE // with INVALID_TOKEN. - send_connection_close(faddr, version, &hd.dcid, &hd.scid, remote_addr, + send_connection_close(faddr, version, hd.dcid, hd.scid, remote_addr, local_addr, NGTCP2_INVALID_TOKEN); return 0; } @@ -269,7 +269,7 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, } case NGTCP2_ERR_RETRY: if (worker_->get_graceful_shutdown()) { - send_connection_close(faddr, version, &hd.dcid, &hd.scid, remote_addr, + send_connection_close(faddr, version, hd.dcid, hd.scid, remote_addr, local_addr, NGTCP2_CONNECTION_REFUSED); return 0; } @@ -421,9 +421,7 @@ int QUICConnectionHandler::send_retry( ngtcp2_cid retry_scid; - retry_scid.datalen = SHRPX_QUIC_SCIDLEN; - // We do not steer packet based on Retry CID. - if (RAND_bytes(retry_scid.data, retry_scid.datalen) != 1) { + if (generate_quic_connection_id(retry_scid, SHRPX_QUIC_SCIDLEN) != 0) { return -1; } @@ -438,7 +436,7 @@ int QUICConnectionHandler::send_retry( auto &secret = quic_secret->token_secret; if (generate_retry_token(token.data(), tokenlen, &remote_addr.su.sa, - remote_addr.len, &retry_scid, &idcid, + remote_addr.len, retry_scid, idcid, secret.data()) != 0) { return -1; } @@ -539,7 +537,7 @@ int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr, auto &quic_secret = worker_->get_quic_secret(); auto &secret = quic_secret->stateless_reset_secret; - rv = generate_quic_stateless_reset_token(token.data(), &cid, secret.data(), + rv = generate_quic_stateless_reset_token(token.data(), cid, secret.data(), secret.size()); if (rv != 0) { return -1; @@ -574,13 +572,13 @@ int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr, } int QUICConnectionHandler::send_connection_close( - const UpstreamAddr *faddr, uint32_t version, const ngtcp2_cid *ini_dcid, - const ngtcp2_cid *ini_scid, const Address &remote_addr, + const UpstreamAddr *faddr, uint32_t version, const ngtcp2_cid &ini_dcid, + const ngtcp2_cid &ini_scid, const Address &remote_addr, const Address &local_addr, uint64_t error_code) { std::array buf; auto nwrite = ngtcp2_crypto_write_connection_close( - buf.data(), buf.size(), version, ini_scid, ini_dcid, error_code); + buf.data(), buf.size(), version, &ini_scid, &ini_dcid, error_code); if (nwrite < 0) { LOG(ERROR) << "ngtcp2_crypto_write_connection_close failed"; return -1; @@ -590,9 +588,8 @@ int QUICConnectionHandler::send_connection_close( LOG(INFO) << "Send Initial CONNECTION_CLOSE with error_code=" << log::hex << error_code << log::dec << " to remote=" << util::to_numeric_addr(&remote_addr) - << " dcid=" << util::format_hex(ini_scid->data, ini_scid->datalen) - << " scid=" - << util::format_hex(ini_dcid->data, ini_dcid->datalen); + << " dcid=" << util::format_hex(ini_scid.data, ini_scid.datalen) + << " scid=" << util::format_hex(ini_dcid.data, ini_dcid.datalen); } return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, @@ -600,13 +597,13 @@ int QUICConnectionHandler::send_connection_close( 0); } -void QUICConnectionHandler::add_connection_id(const ngtcp2_cid *cid, +void QUICConnectionHandler::add_connection_id(const ngtcp2_cid &cid, ClientHandler *handler) { - connections_.emplace(*cid, handler); + connections_.emplace(cid, handler); } -void QUICConnectionHandler::remove_connection_id(const ngtcp2_cid *cid) { - connections_.erase(*cid); +void QUICConnectionHandler::remove_connection_id(const ngtcp2_cid &cid) { + connections_.erase(cid); } void QUICConnectionHandler::add_close_wait(CloseWait *cw) { diff --git a/src/shrpx_quic_connection_handler.h b/src/shrpx_quic_connection_handler.h index 95e598f5..39c1b7d2 100644 --- a/src/shrpx_quic_connection_handler.h +++ b/src/shrpx_quic_connection_handler.h @@ -110,8 +110,8 @@ public: // |ini_scid| is the source Connection ID which appeared in Client // Initial packet. int send_connection_close(const UpstreamAddr *faddr, uint32_t version, - const ngtcp2_cid *ini_dcid, - const ngtcp2_cid *ini_scid, + const ngtcp2_cid &ini_dcid, + const ngtcp2_cid &ini_scid, const Address &remote_addr, const Address &local_addr, uint64_t error_code); ClientHandler *handle_new_connection(const UpstreamAddr *faddr, @@ -120,8 +120,8 @@ public: const ngtcp2_pkt_hd &hd, const ngtcp2_cid *odcid, const uint8_t *token, size_t tokenlen); - void add_connection_id(const ngtcp2_cid *cid, ClientHandler *handler); - void remove_connection_id(const ngtcp2_cid *cid); + void add_connection_id(const ngtcp2_cid &cid, ClientHandler *handler); + void remove_connection_id(const ngtcp2_cid &cid); void add_close_wait(CloseWait *cw); void remove_close_wait(const CloseWait *cw); From 2d7e6fbb1106b2ca3eb74e3b56a91adb67f4f75f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 17 Sep 2021 21:21:42 +0900 Subject: [PATCH 028/142] Update doc --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index eaf23d0e..36e17060 100644 --- a/README.rst +++ b/README.rst @@ -153,12 +153,17 @@ following libraries are required: * `ngtcp2 `_ * `nghttp3 `_ +Use ``--enable-http3`` configure option to enable HTTP/3 feature for +h2load and nghttpx. + In order to build optional eBPF program to direct an incoming QUIC UDP datagram to a correct socket for nghttpx, the following libraries are required: * libbpf-dev >= 0.4.0 +Use ``--with-libbpf`` configure option to build eBPF program. + For Ubuntu 20.04, you can build libbpf from `the source code `_. nghttpx requires eBPF program for reloading its configuration and hot swapping From 0df332e7b8bdb0c4a4b252fa1b406cb5039fd26a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 17 Sep 2021 23:53:35 +0900 Subject: [PATCH 029/142] Update doc --- README.rst | 94 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 83 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index 36e17060..86ca5a6a 100644 --- a/README.rst +++ b/README.rst @@ -163,6 +163,7 @@ required: * libbpf-dev >= 0.4.0 Use ``--with-libbpf`` configure option to build eBPF program. +libelf-dev is needed to build libbpf. For Ubuntu 20.04, you can build libbpf from `the source code `_. nghttpx @@ -331,6 +332,88 @@ The generated documents will not be installed with ``make install``. The online documentation is available at https://nghttp2.org/documentation/ +Build HTTP/3 enabled h2load and nghttpx +--------------------------------------- + +To build h2load and nghttpx with HTTP/3 feature enabled, run the +configure script with ``--enable-http3``. + +For nghttpx to reload configurations and swapping its executable while +gracefully terminating old worker processes, eBPF is required. Run +the configure script with ``--enable-http3 --with-libbpf`` to build +eBPF program. The Connection ID encryption key must be set with +``--frontend-quic-connection-id-encryption-key`` and must not change +in order to keep the existing connections alive during reload. + +The detailed steps to build HTTP/3 enabled h2load and nghttpx follow. + +Build custom OpenSSL: + +.. code-block:: text + + $ git clone --depth 1 -b OpenSSL_1_1_1l+quic https://github.com/quictls/openssl + $ cd openssl + $ ./config --prefix=$PWD/build --openssldir=/etc/ssl + $ make -j$(nproc) + $ make install_sw + $ cd .. + +Build nghttp3: + +.. code-block:: text + + $ git clone --depth 1 https://github.com/ngtcp2/nghttp3 + $ cd nghttp3 + $ autoreconf -i + $ ./configure --prefix=$PWD/build --enable-lib-only + $ make -j$(nproc) + $ make install + $ cd .. + +Build ngtcp2: + +.. code-block:: text + + $ git clone --depth 1 https://github.com/ngtcp2/ngtcp2 + $ cd ngtcp2 + $ autoreconf -i + $ ./configure --prefix=$PWD/build --enable-lib-only \ + PKG_CONFIG_PATH="$PWD/../openssl/build/lib/pkgconfig" + $ make -j$(nproc) + $ make install + $ cd .. + +If your Linux distribution does not have libbpf-dev >= 0.4.0, build +from source: + +.. code-block:: text + + $ git clone --depth 1 -b v0.4.0 https://github.com/libbpf/libbpf + $ cd libbpf + $ PREFIX=$PWD/build make -C src install + $ cd .. + +Build nghttp2: + +.. code-block:: text + + $ git clone https://github.com/nghttp2/nghttp2 + $ cd nghttp2 + $ git submodule update --init + $ autoreconf -i + $ ./configure --with-mruby --with-neverbleed --enable-http3 --with-libbpf \ + --disable-python-bindings \ + CC=clang-12 CXX=clang++-12 \ + PKG_CONFIG_PATH="$PWD/../openssl/build/lib/pkgconfig:$PWD/../nghttp3/build/lib/pkgconfig:$PWD/../ngtcp2/build/lib/pkgconfig:$PWD/../libbpf/build/lib64/pkgconfig" \ + LDFLAGS="$LDFLAGS -Wl,-rpath,$PWD/../openssl/build/lib -Wl,-rpath,$PWD/../libbpf/build/lib64" + $ make -j$(nproc) + +The eBPF program ``reuseport_kern.o`` should be found under bpf +directory. Pass ``--quic-bpf-program-file=bpf/reuseport_kern.o`` +option to nghttpx to load it. See also `HTTP/3 section in nghttpx - +HTTP/2 proxy - HOW-TO +`_. + Unit tests ---------- @@ -907,17 +990,6 @@ like so: $ h2load --npn-list h3 https://127.0.0.1:4433 -HTTP/3 ------- - -To build h2load and nghttpx with HTTP/3 feature enabled, run the -configure script with ``--enable-http3``. - -For nghttpx to reload configurations and swapping its executable while -gracefully terminating old worker processes, eBPF is required. Run -the configure script with ``--enable-http3 --with-libbpf`` to build -eBPF program. - HPACK tools ----------- From 97b36b8c7436208e1d2e76a5d095a3d1c41c6150 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 17 Sep 2021 23:57:26 +0900 Subject: [PATCH 030/142] make -C --- .github/workflows/build.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2456088a..52a9a2e7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -87,9 +87,7 @@ jobs: run: | git clone -b v0.4.0 https://github.com/libbpf/libbpf cd libbpf - export PREFIX=$PWD/build - cd src - make install + PREFIX=$PWD/build make -C src install EXTRA_AUTOTOOLS_OPTS="--with-libbpf" From 9a6b623c2544c6591205cfec4087ea14e42afa7a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 18 Sep 2021 15:19:15 +0900 Subject: [PATCH 031/142] Update doc --- README.rst | 32 ++++++++++++++++---------------- doc/sources/nghttpx-howto.rst | 9 +++++++-- src/shrpx.cc | 2 +- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/README.rst b/README.rst index 86ca5a6a..d7bc328c 100644 --- a/README.rst +++ b/README.rst @@ -841,7 +841,7 @@ information. Here is sample output from ``nghttpd``: nghttpx - proxy +++++++++++++++ -``nghttpx`` is a multi-threaded reverse proxy for HTTP/2, and +``nghttpx`` is a multi-threaded reverse proxy for HTTP/3, HTTP/2, and HTTP/1.1, and powers http://nghttp2.org and supports HTTP/2 server push. @@ -862,16 +862,16 @@ ticket keys among multiple ``nghttpx`` instances via memcached. ``nghttpx`` has 2 operation modes: -================== ================ ================ ============= -Mode option Frontend Backend Note -================== ================ ================ ============= -default mode HTTP/2, HTTP/1.1 HTTP/1.1, HTTP/2 Reverse proxy -``--http2-proxy`` HTTP/2, HTTP/1.1 HTTP/1.1, HTTP/2 Forward proxy -================== ================ ================ ============= +================== ======================== ================ ============= +Mode option Frontend Backend Note +================== ======================== ================ ============= +default mode HTTP/3, HTTP/2, HTTP/1.1 HTTP/1.1, HTTP/2 Reverse proxy +``--http2-proxy`` HTTP/3, HTTP/2, HTTP/1.1 HTTP/1.1, HTTP/2 Forward proxy +================== ======================== ================ ============= The interesting mode at the moment is the default mode. It works like -a reverse proxy and listens for HTTP/2, and HTTP/1.1 and can be -deployed as a SSL/TLS terminator for existing web server. +a reverse proxy and listens for HTTP/3, HTTP/2, and HTTP/1.1 and can +be deployed as a SSL/TLS terminator for existing web server. In all modes, the frontend connections are encrypted by SSL/TLS by default. To disable encryption, use the ``no-tls`` keyword in @@ -889,16 +889,16 @@ server: .. code-block:: text - Client <-- (HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/1.1, HTTP/2) --> Web Server - [reverse proxy] + Client <-- (HTTP/3, HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/1.1, HTTP/2) --> Web Server + [reverse proxy] With the ``--http2-proxy`` option, it works as forward proxy, and it is so called secure HTTP/2 proxy: .. code-block:: text - Client <-- (HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/1.1) --> Proxy - [secure proxy] (e.g., Squid, ATS) + Client <-- (HTTP/3, HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/1.1) --> Proxy + [secure proxy] (e.g., Squid, ATS) The ``Client`` in the above example needs to be configured to use ``nghttpx`` as secure proxy. @@ -930,7 +930,7 @@ proxy through an HTTP proxy: .. code-block:: text - Client <-- (HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/2) -- + Client <-- (HTTP/3, HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/2) -- --===================---> HTTP/2 Proxy (HTTP proxy tunnel) (e.g., nghttpx -s) @@ -938,8 +938,8 @@ proxy through an HTTP proxy: Benchmarking tool ----------------- -The ``h2load`` program is a benchmarking tool for HTTP/2. The UI of -``h2load`` is heavily inspired by ``weighttp`` +The ``h2load`` program is a benchmarking tool for HTTP/3, HTTP/2, and +HTTP/1.1. The UI of ``h2load`` is heavily inspired by ``weighttp`` (https://github.com/lighttpd/weighttp). The typical usage is as follows: diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index 27b4cc94..a0f2153b 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -14,8 +14,8 @@ Default mode If nghttpx is invoked without :option:`--http2-proxy`, it operates in default mode. In this mode, it works as reverse proxy (gateway) for -both HTTP/2 and HTTP/1 clients to backend servers. This is also known -as "HTTP/2 router". +both HTTP/3, HTTP/2 and HTTP/1 clients to backend servers. This is +also known as "HTTP/2 router". By default, frontend connection is encrypted using SSL/TLS. So server's private key and certificate must be supplied to the command @@ -28,6 +28,9 @@ the frontend, and an HTTP/1 connection can be upgraded to HTTP/2 using HTTP Upgrade. Starting HTTP/2 connection by sending HTTP/2 connection preface is also supported. +In order to receive HTTP/3 traffic, use ``quic`` parameter in +:option:`--frontend` option (.e.g, ``--frontend='*,443;quic'``) + nghttpx can listen on multiple frontend addresses. This is achieved by using multiple :option:`--frontend` options. For each frontend address, TLS can be enabled or disabled. @@ -509,6 +512,8 @@ Bootstrapping WebSockets with HTTP/2 for both frontend and backend connections. This feature is enabled by default and no configuration is required. +WebSockets over HTTP/3 is also supported. + HTTP/3 ------ diff --git a/src/shrpx.cc b/src/shrpx.cc index 1aa47dc4..5189ee7b 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1965,7 +1965,7 @@ void print_version(std::ostream &out) { namespace { void print_usage(std::ostream &out) { out << R"(Usage: nghttpx [OPTIONS]... [ ] -A reverse proxy for HTTP/2, and HTTP/1.)" +A reverse proxy for HTTP/3, HTTP/2, and HTTP/1.)" << std::endl; } } // namespace From be88846972074bd3e1891f69638023f89ec7a543 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 18 Sep 2021 17:59:09 +0900 Subject: [PATCH 032/142] Build HTTP/3 and eBPF enabled nghttpx with Dockerfile --- docker/Dockerfile | 75 ++++++++++++++++++++++++++++++++++ docker/Dockerfile-h2load-http3 | 39 ------------------ docker/README.rst | 25 ++++++++++++ 3 files changed, 100 insertions(+), 39 deletions(-) create mode 100644 docker/Dockerfile delete mode 100644 docker/Dockerfile-h2load-http3 create mode 100644 docker/README.rst diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..47be5a40 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,75 @@ +FROM debian:11 as build + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git clang make binutils autoconf automake autotools-dev libtool \ + pkg-config \ + zlib1g-dev libev-dev libjemalloc-dev ruby-dev libc-ares-dev bison \ + libelf-dev + +RUN git clone --depth 1 -b openssl-3.0.0+quic https://github.com/quictls/openssl && \ + cd openssl && \ + ./config --openssldir=/etc/ssl && \ + make -j$(nproc) && \ + make install_sw && \ + cd .. && \ + rm -rf openssl + +RUN git clone --depth 1 https://github.com/ngtcp2/nghttp3 && \ + cd nghttp3 && \ + autoreconf -i && \ + ./configure --enable-lib-only && \ + make -j$(nproc) && \ + make install-strip && \ + cd .. && \ + rm -rf nghttp3 + +RUN git clone --depth 1 https://github.com/ngtcp2/ngtcp2 && \ + cd ngtcp2 && \ + autoreconf -i && \ + ./configure --enable-lib-only \ + LIBTOOL_LDFLAGS="-static-libtool-libs" \ + OPENSSL_LIBS="-l:libssl.a -l:libcrypto.a -ldl -lpthread" \ + PKG_CONFIG_PATH="/usr/local/lib64/pkgconfig" && \ + make -j$(nproc) && \ + make install-strip && \ + cd .. && \ + rm -rf ngtcp2 + +RUN git clone --depth 1 -b v0.4.0 https://github.com/libbpf/libbpf && \ + cd libbpf && \ + PREFIX=/usr/local make -C src install && \ + cd .. && \ + rm -rf libbpf + +RUN git clone --depth 1 https://github.com/nghttp2/nghttp2.git && \ + cd nghttp2 && \ + git submodule update --init && \ + autoreconf -i && \ + ./configure --disable-examples --disable-hpack-tools \ + --disable-python-bindings --with-mruby --with-neverbleed \ + --enable-http3 --with-libbpf \ + CC=clang CXX=clang++ \ + LIBTOOL_LDFLAGS="-static-libtool-libs" \ + OPENSSL_LIBS="-l:libssl.a -l:libcrypto.a -ldl -pthread" \ + LIBEV_LIBS="-l:libev.a" \ + JEMALLOC_LIBS="-l:libjemalloc.a" \ + LIBCARES_LIBS="-l:libcares.a" \ + ZLIB_LIBS="-l:libz.a" \ + LIBBPF_LIBS="-L/usr/local/lib64 -l:libbpf.a -l:libelf.a" \ + LDFLAGS="-static-libgcc -static-libstdc++" \ + PKG_CONFIG_PATH="/usr/local/lib64/pkgconfig" && \ + make -j$(nproc) install-strip && \ + cd .. && \ + rm -rf nghttp2 + +FROM gcr.io/distroless/base-debian11 + +COPY --from=build \ + /usr/local/bin/h2load \ + /usr/local/bin/nghttpx \ + /usr/local/bin/nghttp \ + /usr/local/bin/nghttpd \ + /usr/local/bin/ +COPY --from=build /usr/local/lib/nghttp2/reuseport_kern.o \ + /usr/local/lib/nghttp2/ diff --git a/docker/Dockerfile-h2load-http3 b/docker/Dockerfile-h2load-http3 deleted file mode 100644 index b5dcd411..00000000 --- a/docker/Dockerfile-h2load-http3 +++ /dev/null @@ -1,39 +0,0 @@ -FROM debian:10 as build - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - git g++ make binutils autoconf automake autotools-dev libtool \ - pkg-config \ - zlib1g-dev libev-dev libjemalloc-dev ruby-dev libc-ares-dev bison && \ - git clone --depth 1 -b OpenSSL_1_1_1l+quic https://github.com/quictls/openssl && \ - cd openssl && ./config enable-tls1_3 --openssldir=/etc/ssl && make -j$(nproc) && make install_sw && cd .. && rm -rf openssl && \ - git clone --depth 1 https://github.com/ngtcp2/nghttp3 && \ - cd nghttp3 && autoreconf -i && \ - ./configure --enable-lib-only && \ - make -j$(nproc) && make install-strip && cd .. && rm -rf nghttp3 && \ - git clone --depth 1 https://github.com/ngtcp2/ngtcp2 && \ - cd ngtcp2 && autoreconf -i && \ - ./configure --enable-lib-only \ - LIBTOOL_LDFLAGS="-static-libtool-libs" \ - OPENSSL_LIBS="-l:libssl.a -l:libcrypto.a -ldl -lpthread" && \ - make -j$(nproc) && make install-strip && cd .. && rm -rf ngtcp2 && \ - git clone --depth 1 https://github.com/nghttp2/nghttp2.git && \ - cd nghttp2 && \ - git submodule update --init && autoreconf -i && \ - ./configure --disable-examples --disable-hpack-tools \ - --disable-python-bindings --with-mruby --with-neverbleed \ - --enable-http3 \ - LIBTOOL_LDFLAGS="-static-libtool-libs" \ - LIBS="-ldl -pthread" \ - OPENSSL_LIBS="-l:libssl.a -l:libcrypto.a" \ - LIBEV_LIBS="-l:libev.a" \ - JEMALLOC_LIBS="-l:libjemalloc.a" \ - LIBCARES_LIBS="-l:libcares.a" \ - ZLIB_LIBS="-l:libz.a" && \ - make -j$(nproc) install-strip - -FROM gcr.io/distroless/cc-debian10 - -COPY --from=build /usr/local/bin/h2load /usr/local/bin/ - -ENTRYPOINT ["/usr/local/bin/h2load"] diff --git a/docker/README.rst b/docker/README.rst new file mode 100644 index 00000000..7f3a6b59 --- /dev/null +++ b/docker/README.rst @@ -0,0 +1,25 @@ +Dockerfile +========== + +Dockerfile creates the applications bundled with nghttp2. +These applications are: + +- nghttp +- nghttpd +- nghttpx +- h2load + +HTTP/3 and eBPF features are enabled. + +In order to run nghttpx with HTTP/3 endpoint, you need to run the +image with the escalated privilege and higher memlock value. Here is +the example command-line to run nghttpx to listen to HTTP/3 on port +443, assuming that the current directory contains a private key and a +certificate in server.key and server.crt respectively : + +.. code-block:: text + + $ docker run --rm -it -v $PWD:/shared --net=host --privileged \ + --ulimit memlock=2048000 nghttp2 nghttpx \ + /shared/server.key /shared/server.crt \ + -f'*,443;quic' From 6a099ee50a5bd1c7dde90fc7b2ba5b97f405060b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 18 Sep 2021 19:27:47 +0900 Subject: [PATCH 033/142] nghttpx: QUIC requires TLS --- src/shrpx_config.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 9a2f24a7..f210128b 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2777,9 +2777,16 @@ int parse_config(Config *config, int optid, const StringRef &opt, return -1; } - if (params.quic && params.alt_mode != UpstreamAltMode::NONE) { - LOG(ERROR) << "frontend: api or healthmon cannot be used with quic"; - return -1; + if (params.quic) { + if (params.alt_mode != UpstreamAltMode::NONE) { + LOG(ERROR) << "frontend: api or healthmon cannot be used with quic"; + return -1; + } + + if (!params.tls) { + LOG(ERROR) << "frontend: quic requires TLS"; + return -1; + } } UpstreamAddr addr{}; From d276ca0adc3d2a1d03e3d8388b82acc2a3c5f521 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 18 Sep 2021 19:28:03 +0900 Subject: [PATCH 034/142] Update doc --- doc/sources/nghttpx-howto.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index a0f2153b..774a3cd8 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -14,8 +14,8 @@ Default mode If nghttpx is invoked without :option:`--http2-proxy`, it operates in default mode. In this mode, it works as reverse proxy (gateway) for -both HTTP/3, HTTP/2 and HTTP/1 clients to backend servers. This is -also known as "HTTP/2 router". +HTTP/3, HTTP/2 and HTTP/1 clients to backend servers. This is also +known as "HTTP/2 router". By default, frontend connection is encrypted using SSL/TLS. So server's private key and certificate must be supplied to the command From 0264847a37a16e5cbc12782a2c274ad1e0299a85 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 19 Sep 2021 20:58:30 +0900 Subject: [PATCH 035/142] bpf: Use LINUX_KERNEL_VERSION extern variable --- bpf/reuseport_kern.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/bpf/reuseport_kern.c b/bpf/reuseport_kern.c index a012ee9a..51cbdc18 100644 --- a/bpf/reuseport_kern.c +++ b/bpf/reuseport_kern.c @@ -24,7 +24,6 @@ */ #include #include -#include #include @@ -235,22 +234,29 @@ static void InvMixColumns(state_t *state) { } } +extern __u32 LINUX_KERNEL_VERSION __kconfig; + /* The SubBytes Function Substitutes the values in the state matrix with values in an S-box. */ static void InvSubBytes(state_t *state) { __u8 i, j; - for (i = 0; i < 4; ++i) { - for (j = 0; j < 4; ++j) { -#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) - /* Ubuntu 20.04 LTS kernel 5.4.0 needs this workaround otherwise - "math between map_value pointer and register with unbounded - min value is not allowed". 5.10.0 is a kernel version that - works but it might not be a minimum version. */ - __u8 k = (*state)[j][i]; - (*state)[j][i] = k ? getSBoxInvert(k) : getSBoxInvert(0); -#else /* !(LINUX_VERSION_CDOE < KERNEL_VERSION(5, 10, 0)) */ - (*state)[j][i] = getSBoxInvert((*state)[j][i]); -#endif /* !(LINUX_VERSION_CDOE < KERNEL_VERSION(5, 10, 0)) */ + if (LINUX_KERNEL_VERSION < KERNEL_VERSION(5, 10, 0)) { + for (i = 0; i < 4; ++i) { + for (j = 0; j < 4; ++j) { + /* Ubuntu 20.04 LTS kernel 5.4.0 needs this workaround + otherwise "math between map_value pointer and register with + unbounded min value is not allowed". 5.10.0 is a kernel + version that works but it might not be the minimum + version. */ + __u8 k = (*state)[j][i]; + (*state)[j][i] = k ? getSBoxInvert(k) : getSBoxInvert(0); + } + } + } else { + for (i = 0; i < 4; ++i) { + for (j = 0; j < 4; ++j) { + (*state)[j][i] = getSBoxInvert((*state)[j][i]); + } } } } From 5b6e2cb5e080ff4abef08efd3a7598b33395d137 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 20 Sep 2021 12:19:26 +0900 Subject: [PATCH 036/142] Allow SPHINXBUILD to be overridden by environment variable --- doc/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Makefile.am b/doc/Makefile.am index 13317991..ce56b1c2 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -272,7 +272,7 @@ EXTRA_DIST = \ # You can set these variables from the command line. SPHINXOPTS = -SPHINXBUILD = sphinx-build +SPHINXBUILD ?= sphinx-build PAPER = BUILDDIR = manual From a029f6ed2cd585dd739cf9a10de9a3710cad5beb Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 20 Sep 2021 13:43:40 +0900 Subject: [PATCH 037/142] Rename sphinxcontrib to rubydomain to avoid module loading error Rename sphinxcontrib to rubydomain to avoid module loading error when sphinx-build docker image is used. --- doc/CMakeLists.txt | 6 +++--- doc/Makefile.am | 6 +++--- doc/_exts/{sphinxcontrib => rubydomain}/LICENSE.rubydomain | 0 doc/_exts/{sphinxcontrib => rubydomain}/__init__.py | 0 doc/_exts/{sphinxcontrib => rubydomain}/rubydomain.py | 0 doc/conf.py.in | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) rename doc/_exts/{sphinxcontrib => rubydomain}/LICENSE.rubydomain (100%) rename doc/_exts/{sphinxcontrib => rubydomain}/__init__.py (100%) rename doc/_exts/{sphinxcontrib => rubydomain}/rubydomain.py (100%) diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index f3aec84d..06677731 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -184,9 +184,9 @@ set(EXTRA_DIST sources/python-apiref.rst sources/building-android-binary.rst sources/contribute.rst - _exts/sphinxcontrib/LICENSE.rubydomain - _exts/sphinxcontrib/__init__.py - _exts/sphinxcontrib/rubydomain.py + _exts/rubydomain/LICENSE.rubydomain + _exts/rubydomain/__init__.py + _exts/rubydomain/rubydomain.py _themes/sphinx_rtd_theme/__init__.py _themes/sphinx_rtd_theme/breadcrumbs.html _themes/sphinx_rtd_theme/footer.html diff --git a/doc/Makefile.am b/doc/Makefile.am index ce56b1c2..ebbd9078 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -206,9 +206,9 @@ EXTRA_DIST = \ sources/building-android-binary.rst \ sources/contribute.rst \ sources/security.rst \ - _exts/sphinxcontrib/LICENSE.rubydomain \ - _exts/sphinxcontrib/__init__.py \ - _exts/sphinxcontrib/rubydomain.py \ + _exts/rubydomain/LICENSE.rubydomain \ + _exts/rubydomain/__init__.py \ + _exts/rubydomain/rubydomain.py \ _themes/sphinx_rtd_theme/__init__.py \ _themes/sphinx_rtd_theme/breadcrumbs.html \ _themes/sphinx_rtd_theme/footer.html \ diff --git a/doc/_exts/sphinxcontrib/LICENSE.rubydomain b/doc/_exts/rubydomain/LICENSE.rubydomain similarity index 100% rename from doc/_exts/sphinxcontrib/LICENSE.rubydomain rename to doc/_exts/rubydomain/LICENSE.rubydomain diff --git a/doc/_exts/sphinxcontrib/__init__.py b/doc/_exts/rubydomain/__init__.py similarity index 100% rename from doc/_exts/sphinxcontrib/__init__.py rename to doc/_exts/rubydomain/__init__.py diff --git a/doc/_exts/sphinxcontrib/rubydomain.py b/doc/_exts/rubydomain/rubydomain.py similarity index 100% rename from doc/_exts/sphinxcontrib/rubydomain.py rename to doc/_exts/rubydomain/rubydomain.py diff --git a/doc/conf.py.in b/doc/conf.py.in index bc6401ec..d6a82b6c 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -41,7 +41,7 @@ import sys, os # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) -sys.path.append(os.path.abspath('@top_srcdir@/doc/_exts')) +sys.path.insert(0, os.path.abspath('@top_srcdir@/doc/_exts')) # -- General configuration ----------------------------------------------------- @@ -50,7 +50,7 @@ sys.path.append(os.path.abspath('@top_srcdir@/doc/_exts')) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinxcontrib.rubydomain'] +extensions = ['rubydomain.rubydomain'] # Add any paths that contain templates here, relative to this directory. templates_path = ['@top_srcdir@/_templates'] From e866f9fae7a92a88a44b3c6d288d7d6ffa3b58d5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 20 Sep 2021 16:53:55 +0900 Subject: [PATCH 038/142] Update AUTHORS --- AUTHORS | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/AUTHORS b/AUTHORS index 0d48b6eb..a7a9151b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -19,6 +19,7 @@ Alek Storm Alex Nalivko Alexandros Konstantinakis-Karmis Alexis La Goutte +Amir Livneh Amir Pakdel Anders Bakken Andreas Pohl @@ -34,11 +35,13 @@ Bernard Spil Brendan Heinonen Brian Card Brian Suh +Daniel Bevenius Daniel Evers Daniel Stenberg Dave Reisner David Beitey David Weekly +Dmitri Tikhonov Dmitriy Vetutnev Don Dylan Plecki @@ -48,9 +51,12 @@ Fabian Wiesel Gabi Davar Gaël PORTAY Geoff Hill +George Liu Gitai Google Inc. +Hajime Fujita Jacky Tian +Jacky_Yin Jacob Champion James M Snell Jan Kundrát @@ -76,6 +82,7 @@ MATSUMOTO Ryosuke Marc Bachmann Matt Rudary Matt Way +Michael Kaufmann Mike Conlen Mike Frysinger Mike Lothian @@ -127,6 +134,7 @@ es fangdingjun jwchoi kumagi +lhuang04 lstefani makovich mod-h2-dev From 32ecfc6a8632affa598b8fcab7844414d48f1e59 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 20 Sep 2021 16:54:47 +0900 Subject: [PATCH 039/142] Use https --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 06487861..535a9dc2 100644 --- a/configure.ac +++ b/configure.ac @@ -43,7 +43,7 @@ AM_INIT_AUTOMAKE([subdir-objects]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) dnl See versioning rule: -dnl http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html +dnl https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html AC_SUBST(LT_CURRENT, 34) AC_SUBST(LT_REVISION, 2) AC_SUBST(LT_AGE, 20) From aab07d00d75fc5d351e0dab54d7ceacc5773d267 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 20 Sep 2021 16:58:28 +0900 Subject: [PATCH 040/142] Bump version number to 1.45.0, LT revision to 35:0:21 --- CMakeLists.txt | 8 ++++---- configure.ac | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d7d589ff..ba249a6e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,13 +24,13 @@ cmake_minimum_required(VERSION 3.0) # XXX using 1.8.90 instead of 1.9.0-DEV -project(nghttp2 VERSION 1.44.90) +project(nghttp2 VERSION 1.45.0) # See versioning rule: # http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html -set(LT_CURRENT 34) -set(LT_REVISION 2) -set(LT_AGE 20) +set(LT_CURRENT 35) +set(LT_REVISION 0) +set(LT_AGE 21) set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) include(Version) diff --git a/configure.ac b/configure.ac index 535a9dc2..50f1ea36 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.45.0-DEV], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.45.0], [t-tujikawa@users.sourceforge.net]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_HEADERS([config.h]) @@ -44,9 +44,9 @@ m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) dnl See versioning rule: dnl https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html -AC_SUBST(LT_CURRENT, 34) -AC_SUBST(LT_REVISION, 2) -AC_SUBST(LT_AGE, 20) +AC_SUBST(LT_CURRENT, 35) +AC_SUBST(LT_REVISION, 0) +AC_SUBST(LT_AGE, 21) major=`echo $PACKAGE_VERSION |cut -d. -f1 | sed -e "s/[^0-9]//g"` minor=`echo $PACKAGE_VERSION |cut -d. -f2 | sed -e "s/[^0-9]//g"` From f4515e9034f91b977df01e52aa50d485e8c75fda Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 20 Sep 2021 17:02:54 +0900 Subject: [PATCH 041/142] Update manual pages --- doc/h2load.1 | 2 +- doc/nghttp.1 | 2 +- doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 4 ++-- doc/nghttpx.1.rst | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index 9b6a241c..79b2cbda 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "Sep 15, 2021" "1.45.0-DEV" "nghttp2" +.TH "H2LOAD" "1" "Sep 20, 2021" "1.45.0" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 10e4c5b3..f0d2704e 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "Sep 15, 2021" "1.45.0-DEV" "nghttp2" +.TH "NGHTTP" "1" "Sep 20, 2021" "1.45.0" "nghttp2" .SH NAME nghttp \- HTTP/2 client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index 0a29da62..2458b9f1 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "Sep 15, 2021" "1.45.0-DEV" "nghttp2" +.TH "NGHTTPD" "1" "Sep 20, 2021" "1.45.0" "nghttp2" .SH NAME nghttpd \- HTTP/2 server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index 95baf730..ba6b8022 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "Sep 15, 2021" "1.45.0-DEV" "nghttp2" +.TH "NGHTTPX" "1" "Sep 20, 2021" "1.45.0" "nghttp2" .SH NAME nghttpx \- HTTP/2 proxy . @@ -35,7 +35,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] \fBnghttpx\fP [OPTIONS]... [ ] .SH DESCRIPTION .sp -A reverse proxy for HTTP/2, and HTTP/1. +A reverse proxy for HTTP/3, HTTP/2, and HTTP/1. .INDENT 0.0 .TP .B diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index 21592553..278ae146 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -14,7 +14,7 @@ SYNOPSIS DESCRIPTION ----------- -A reverse proxy for HTTP/2, and HTTP/1. +A reverse proxy for HTTP/3, HTTP/2, and HTTP/1. .. describe:: From afb455ef809404b582bf9819c36102befba437e2 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 20 Sep 2021 17:07:44 +0900 Subject: [PATCH 042/142] python3 --- doc/bash_completion/make_bash_completion.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/bash_completion/make_bash_completion.py b/doc/bash_completion/make_bash_completion.py index 55cbfce8..e3a535ba 100755 --- a/doc/bash_completion/make_bash_completion.py +++ b/doc/bash_completion/make_bash_completion.py @@ -1,8 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- -from __future__ import unicode_literals -from __future__ import print_function import subprocess import io import re From 58499f256b04c1687e772e97a54193d4b3d802dc Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 20 Sep 2021 17:07:58 +0900 Subject: [PATCH 043/142] Update bash_completion --- doc/bash_completion/h2load | 2 +- doc/bash_completion/nghttp | 2 +- doc/bash_completion/nghttpd | 2 +- doc/bash_completion/nghttpx | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/bash_completion/h2load b/doc/bash_completion/h2load index 814b4fe4..bb0179b3 100644 --- a/doc/bash_completion/h2load +++ b/doc/bash_completion/h2load @@ -8,7 +8,7 @@ _h2load() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--connection-window-bits --help --clients --verbose --ciphers --rate --no-tls-proto --connect-to --header-table-size --requests --log-file --base-uri --no-udp-gso --h1 --threads --npn-list --rate-period --rps --data --tls13-ciphers --version --connection-inactivity-timeout --timing-script-file --encoder-header-table-size --max-concurrent-streams --connection-active-timeout --input-file --max-udp-payload-size --groups --window-bits --qlog-file-base --warm-up-time --duration --header ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--requests --clients --threads --input-file --max-concurrent-streams --window-bits --connection-window-bits --header --ciphers --tls13-ciphers --no-tls-proto --data --rate --rate-period --duration --warm-up-time --connection-active-timeout --connection-inactivity-timeout --timing-script-file --base-uri --npn-list --h1 --header-table-size --encoder-header-table-size --log-file --qlog-file-base --connect-to --rps --groups --no-udp-gso --max-udp-payload-size --verbose --version --help ' -- "$cur" ) ) ;; *) _filedir diff --git a/doc/bash_completion/nghttp b/doc/bash_completion/nghttp index 9a4cf190..d7d66b42 100644 --- a/doc/bash_completion/nghttp +++ b/doc/bash_completion/nghttp @@ -8,7 +8,7 @@ _nghttp() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--no-push --verbose --no-dep --get-assets --har --header-table-size --multiply --encoder-header-table-size --padding --hexdump --max-concurrent-streams --continuation --connection-window-bits --peer-max-concurrent-streams --timeout --data --no-content-length --version --color --cert --upgrade --remote-name --trailer --weight --help --key --null-out --window-bits --expect-continue --stat --no-verify-peer --header ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--verbose --null-out --remote-name --timeout --window-bits --connection-window-bits --get-assets --stat --header --trailer --cert --key --data --multiply --upgrade --weight --peer-max-concurrent-streams --header-table-size --encoder-header-table-size --padding --har --color --continuation --no-content-length --no-dep --hexdump --no-push --max-concurrent-streams --expect-continue --no-verify-peer --version --help ' -- "$cur" ) ) ;; *) _filedir diff --git a/doc/bash_completion/nghttpd b/doc/bash_completion/nghttpd index 92d1dfb5..204e7db0 100644 --- a/doc/bash_completion/nghttpd +++ b/doc/bash_completion/nghttpd @@ -8,7 +8,7 @@ _nghttpd() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--htdocs --verbose --daemon --echo-upload --error-gzip --push --header-table-size --encoder-header-table-size --padding --hexdump --max-concurrent-streams --no-tls --connection-window-bits --mime-types-file --no-content-length --workers --version --color --early-response --dh-param-file --trailer --address --window-bits --verify-client --help ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--address --daemon --verify-client --htdocs --verbose --no-tls --header-table-size --encoder-header-table-size --color --push --padding --max-concurrent-streams --workers --error-gzip --window-bits --connection-window-bits --dh-param-file --early-response --trailer --hexdump --echo-upload --mime-types-file --no-content-length --version --help ' -- "$cur" ) ) ;; *) _filedir diff --git a/doc/bash_completion/nghttpx b/doc/bash_completion/nghttpx index 4c4128ea..f27ab23d 100644 --- a/doc/bash_completion/nghttpx +++ b/doc/bash_completion/nghttpx @@ -8,7 +8,7 @@ _nghttpx() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--worker-read-rate --include --frontend-http2-dump-response-header --tls-ticket-key-file --frontend-quic-qlog-dir --verify-client-cacert --max-response-header-fields --backend-http2-window-size --tls13-client-ciphers --frontend-keep-alive-timeout --backend-request-buffer --max-request-header-fields --backend-connect-timeout --frontend-quic-require-token --tls-max-proto-version --frontend-quic-idle-timeout --frontend-http3-read-timeout --conf --dns-lookup-timeout --frontend-http3-connection-window-size --backend-http2-max-concurrent-streams --worker-write-burst --no-quic-bpf --npn-list --dns-max-try --fetch-ocsp-response-file --tls-session-cache-memcached-cert-file --no-via --mruby-file --no-server-push --stream-read-timeout --client-ciphers --ocsp-update-interval --frontend-quic-connection-id-encryption-key --forwarded-for --accesslog-syslog --dns-cache-timeout --frontend-http2-read-timeout --listener-disable-timeout --frontend-http3-window-size --ciphers --client-psk-secrets --strip-incoming-x-forwarded-for --no-server-rewrite --private-key-passwd-file --backend-keep-alive-timeout --backend-http-proxy-uri --frontend-quic-early-data --frontend-max-requests --tls-no-postpone-early-data --rlimit-nofile --no-strip-incoming-x-forwarded-proto --tls-ticket-key-memcached-cert-file --no-verify-ocsp --forwarded-by --tls-session-cache-memcached-private-key-file --error-page --ocsp-startup --backend-write-timeout --tls-dyn-rec-warmup-threshold --frontend-http2-window-size --tls-ticket-key-memcached-max-retry --http2-no-cookie-crumbling --frontend-http3-max-concurrent-streams --worker-read-burst --dh-param-file --accesslog-format --errorlog-syslog --redirect-https-port --request-header-field-buffer --api-max-request-body --frontend-http2-decoder-dynamic-table-size --errorlog-file --frontend-http2-max-concurrent-streams --psk-secrets --frontend-write-timeout --tls-ticket-key-cipher --read-burst --no-add-x-forwarded-proto --backend --server-name --insecure --backend-max-backoff --log-level --host-rewrite --quic-bpf-program-file --http2-altsvc --frontend-http3-max-connection-window-size --tls-ticket-key-memcached-interval --frontend-http2-setting-timeout --frontend-http2-connection-window-size --worker-frontend-connections --syslog-facility --fastopen --no-location-rewrite --single-thread --no-http2-cipher-block-list --tls-session-cache-memcached --no-ocsp --backend-response-buffer --tls-min-proto-version --workers --add-x-forwarded-for --add-forwarded --frontend-http3-max-window-size --worker-write-rate --add-request-header --backend-http2-settings-timeout --subcert --ignore-per-pattern-mruby-error --ecdh-curves --no-kqueue --help --frontend-frame-debug --tls-sct-dir --pid-file --frontend-http2-dump-request-header --daemon --write-rate --altsvc --backend-http2-decoder-dynamic-table-size --no-strip-incoming-early-data --user --verify-client-tolerate-expired --frontend-read-timeout --tls-ticket-key-memcached-max-fail --backlog --write-burst --backend-connections-per-host --tls-max-early-data --response-header-field-buffer --tls-ticket-key-memcached-address-family --padding --tls-session-cache-memcached-address-family --frontend-quic-congestion-controller --stream-write-timeout --cacert --frontend-quic-debug-log --tls-ticket-key-memcached-private-key-file --accesslog-write-early --backend-address-family --backend-http2-connection-window-size --tls13-ciphers --client-no-http2-cipher-block-list --version --add-response-header --backend-read-timeout --frontend-http2-optimize-window-size --frontend --accesslog-file --http2-proxy --backend-http2-encoder-dynamic-table-size --client-private-key-file --single-process --client-cert-file --tls-ticket-key-memcached --tls-dyn-rec-idle-timeout --frontend-http2-optimize-write-buffer-size --verify-client --frontend-http2-encoder-dynamic-table-size --read-rate --backend-connections-per-frontend --strip-incoming-forwarded ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--backend --frontend --backlog --backend-address-family --backend-http-proxy-uri --workers --single-thread --read-rate --read-burst --write-rate --write-burst --worker-read-rate --worker-read-burst --worker-write-rate --worker-write-burst --worker-frontend-connections --backend-connections-per-host --backend-connections-per-frontend --rlimit-nofile --backend-request-buffer --backend-response-buffer --fastopen --no-kqueue --frontend-http2-read-timeout --frontend-http3-read-timeout --frontend-read-timeout --frontend-write-timeout --frontend-keep-alive-timeout --stream-read-timeout --stream-write-timeout --backend-read-timeout --backend-write-timeout --backend-connect-timeout --backend-keep-alive-timeout --listener-disable-timeout --frontend-http2-setting-timeout --backend-http2-settings-timeout --backend-max-backoff --ciphers --tls13-ciphers --client-ciphers --tls13-client-ciphers --ecdh-curves --insecure --cacert --private-key-passwd-file --subcert --dh-param-file --npn-list --verify-client --verify-client-cacert --verify-client-tolerate-expired --client-private-key-file --client-cert-file --tls-min-proto-version --tls-max-proto-version --tls-ticket-key-file --tls-ticket-key-memcached --tls-ticket-key-memcached-address-family --tls-ticket-key-memcached-interval --tls-ticket-key-memcached-max-retry --tls-ticket-key-memcached-max-fail --tls-ticket-key-cipher --tls-ticket-key-memcached-cert-file --tls-ticket-key-memcached-private-key-file --fetch-ocsp-response-file --ocsp-update-interval --ocsp-startup --no-verify-ocsp --no-ocsp --tls-session-cache-memcached --tls-session-cache-memcached-address-family --tls-session-cache-memcached-cert-file --tls-session-cache-memcached-private-key-file --tls-dyn-rec-warmup-threshold --tls-dyn-rec-idle-timeout --no-http2-cipher-block-list --client-no-http2-cipher-block-list --tls-sct-dir --psk-secrets --client-psk-secrets --tls-no-postpone-early-data --tls-max-early-data --frontend-http2-max-concurrent-streams --backend-http2-max-concurrent-streams --frontend-http2-window-size --frontend-http2-connection-window-size --backend-http2-window-size --backend-http2-connection-window-size --http2-no-cookie-crumbling --padding --no-server-push --frontend-http2-optimize-write-buffer-size --frontend-http2-optimize-window-size --frontend-http2-encoder-dynamic-table-size --frontend-http2-decoder-dynamic-table-size --backend-http2-encoder-dynamic-table-size --backend-http2-decoder-dynamic-table-size --http2-proxy --log-level --accesslog-file --accesslog-syslog --accesslog-format --accesslog-write-early --errorlog-file --errorlog-syslog --syslog-facility --add-x-forwarded-for --strip-incoming-x-forwarded-for --no-add-x-forwarded-proto --no-strip-incoming-x-forwarded-proto --add-forwarded --strip-incoming-forwarded --forwarded-by --forwarded-for --no-via --no-strip-incoming-early-data --no-location-rewrite --host-rewrite --altsvc --http2-altsvc --add-request-header --add-response-header --request-header-field-buffer --max-request-header-fields --response-header-field-buffer --max-response-header-fields --error-page --server-name --no-server-rewrite --redirect-https-port --api-max-request-body --dns-cache-timeout --dns-lookup-timeout --dns-max-try --frontend-max-requests --frontend-http2-dump-request-header --frontend-http2-dump-response-header --frontend-frame-debug --daemon --pid-file --user --single-process --mruby-file --ignore-per-pattern-mruby-error --frontend-quic-idle-timeout --frontend-quic-debug-log --quic-bpf-program-file --frontend-quic-early-data --frontend-quic-qlog-dir --frontend-quic-require-token --frontend-quic-congestion-controller --frontend-quic-connection-id-encryption-key --no-quic-bpf --frontend-http3-window-size --frontend-http3-connection-window-size --frontend-http3-max-window-size --frontend-http3-max-connection-window-size --frontend-http3-max-concurrent-streams --conf --include --version --help ' -- "$cur" ) ) ;; *) _filedir From 738b562f398e46e3af242150a7f2c92eaa24741f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 20 Sep 2021 17:52:07 +0900 Subject: [PATCH 044/142] Bump up version number to 1.46.0-DEV --- CMakeLists.txt | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ba249a6e..ef97de14 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 3.0) # XXX using 1.8.90 instead of 1.9.0-DEV -project(nghttp2 VERSION 1.45.0) +project(nghttp2 VERSION 1.45.90) # See versioning rule: # http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html diff --git a/configure.ac b/configure.ac index 50f1ea36..e7654ec3 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.45.0], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.46.0-DEV], [t-tujikawa@users.sourceforge.net]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_HEADERS([config.h]) From cdf1f269ff1336025a240a5db5cb3043d7db0249 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 21 Sep 2021 17:53:15 +0900 Subject: [PATCH 045/142] Add missing cmake files to EXTRA_DIST --- Makefile.am | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index adc597b1..99f03237 100644 --- a/Makefile.am +++ b/Makefile.am @@ -46,7 +46,11 @@ EXTRA_DIST = nghttpx.conf.sample proxy.pac.sample android-config android-make \ cmake/FindLibevent.cmake \ cmake/FindJansson.cmake \ cmake/FindLibcares.cmake \ - cmake/FindSystemd.cmake + cmake/FindSystemd.cmake \ + cmake/FindLibbpf.cmake \ + cmake/FindLibnghttp3.cmake \ + cmake/FindLibngtcp2.cmake \ + cmake/FindLibngtcp2_crypto_openssl.cmake .PHONY: clang-format From b50079524b869be871c70e95264c82be7f864770 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 21 Sep 2021 17:54:55 +0900 Subject: [PATCH 046/142] Always include optional files to EXTRA_DIST --- bpf/Makefile.am | 4 ++-- third-party/Makefile.am | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/bpf/Makefile.am b/bpf/Makefile.am index 7aafb694..9017fd95 100644 --- a/bpf/Makefile.am +++ b/bpf/Makefile.am @@ -21,9 +21,9 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -if HAVE_LIBBPF +EXTRA_DIST = CMakeLists.txt reuseport_kern.c -EXTRA_DIST = reuseport_kern.c +if HAVE_LIBBPF bpf_pkglibdir = $(pkglibdir) bpf_pkglib_DATA = reuseport_kern.o diff --git a/third-party/Makefile.am b/third-party/Makefile.am index 55fdad0c..2d6726d8 100644 --- a/third-party/Makefile.am +++ b/third-party/Makefile.am @@ -23,7 +23,7 @@ AM_CPPFLAGS = @DEFS@ -EXTRA_DIST = CMakeLists.txt +EXTRA_DIST = CMakeLists.txt build_config.rb mruby/* if ENABLE_THIRD_PARTY @@ -49,8 +49,6 @@ endif # HAVE_NEVERBLEED if HAVE_MRUBY -EXTRA_DIST += build_config.rb mruby/* - .PHONY: all-local clean mruby mruby: From 06dc7d59641a91eaa7bad7ad3995311c87ec8eed Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 21 Sep 2021 18:05:48 +0900 Subject: [PATCH 047/142] Make sure that nghttp2 can be built from tar archive --- .github/workflows/build.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 52a9a2e7..2eac7868 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -144,22 +144,28 @@ jobs: run: | git submodule update --init - name: Configure autotools - if: matrix.buildtool == 'autotools' run: | autoreconf -i - ./configure --enable-werror --with-mruby $EXTRA_AUTOTOOLS_OPTS + ./configure - name: Configure cmake if: matrix.buildtool == 'cmake' run: | - cmake -DENABLE_WERROR=1 -DWITH_MRUBY=1 -DWITH_NEVERBLEED=1 $EXTRA_CMAKE_OPTS -DCPPFLAGS="$CPPFLAGS" -DLDFLAGS="$LDFLAGS" . + make dist + VERSION=$(grep PACKAGE_VERSION config.h | cut -d' ' -f3 | tr -d '"') + tar xf nghttp2-$VERSION.tar.gz + cd nghttp2-$VERSION + echo 'NGHTTP2_CMAKE_DIR='"$PWD" >> $GITHUB_ENV + + cmake -DENABLE_WERROR=1 -DWITH_MRUBY=1 -DWITH_NEVERBLEED=1 -DENABLE_APP=1 $EXTRA_CMAKE_OPTS -DCPPFLAGS="$CPPFLAGS" -DLDFLAGS="$LDFLAGS" . - name: Build nghttp2 with autotools if: matrix.buildtool == 'autotools' run: | make distcheck \ - DISTCHECK_CONFIGURE_FLAGS="--with-mruby --with-neverbleed --enable-werror $EXTRA_AUTOTOOLS_OPTS CPPFLAGS=\"$CPPFLAGS\" LDFLAGS=\"$LDFLAGS\"" + DISTCHECK_CONFIGURE_FLAGS="--with-mruby --with-neverbleed --with-libev --enable-werror $EXTRA_AUTOTOOLS_OPTS CPPFLAGS=\"$CPPFLAGS\" LDFLAGS=\"$LDFLAGS\"" - name: Build nghttp2 with cmake if: matrix.buildtool == 'cmake' run: | + cd $NGHTTP2_CMAKE_DIR make make check - name: Integration test @@ -167,5 +173,5 @@ jobs: # artifacts. if: matrix.buildtool == 'cmake' run: | - cd integration-tests + cd $NGHTTP2_CMAKE_DIR/integration-tests make itprep it From 657d94b99283f3ec810d87b05b5f7bbc1ca50768 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 21 Sep 2021 18:59:22 +0900 Subject: [PATCH 048/142] Fix compile error with libressl --- src/h2load.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/h2load.cc b/src/h2load.cc index ad37144e..e7f368c8 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -81,6 +81,7 @@ bool recorded(const std::chrono::steady_clock::time_point &t) { } } // namespace +#if OPENSSL_1_1_1_API namespace { std::ofstream keylog_file; void keylog_callback(const SSL *ssl, const char *line) { @@ -89,6 +90,7 @@ void keylog_callback(const SSL *ssl, const char *line) { keylog_file.flush(); } } // namespace +#endif // OPENSSL_1_1_1_API Config::Config() : ciphers(tls::DEFAULT_CIPHER_LIST), From 257043b8fbe8be9ea53f1a72350a06489b255ba9 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 21 Sep 2021 19:05:09 +0900 Subject: [PATCH 049/142] Fix issue that libev cannot be found with autotools under mac osx --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2eac7868..33d8cb18 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -60,8 +60,6 @@ jobs: pkg-config \ libtool echo 'PKG_CONFIG_PATH=/usr/local/opt/libressl/lib/pkgconfig:/usr/local/opt/libxml2/lib/pkgconfig' >> $GITHUB_ENV - # This fixes infamous 'stdio.h not found' error. - echo 'SDKROOT='"$(xcrun --sdk macosx --show-sdk-path)" >> $GITHUB_ENV - name: Setup clang (Linux) if: runner.os == 'Linux' && matrix.compiler == 'clang' run: | @@ -156,6 +154,9 @@ jobs: cd nghttp2-$VERSION echo 'NGHTTP2_CMAKE_DIR='"$PWD" >> $GITHUB_ENV + # This fixes infamous 'stdio.h not found' error. + echo 'SDKROOT='"$(xcrun --sdk macosx --show-sdk-path)" >> $GITHUB_ENV + cmake -DENABLE_WERROR=1 -DWITH_MRUBY=1 -DWITH_NEVERBLEED=1 -DENABLE_APP=1 $EXTRA_CMAKE_OPTS -DCPPFLAGS="$CPPFLAGS" -DLDFLAGS="$LDFLAGS" . - name: Build nghttp2 with autotools if: matrix.buildtool == 'autotools' From 89457fd9915159c746e5503e5da58bee9fc2445e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 21 Sep 2021 20:40:57 +0900 Subject: [PATCH 050/142] More https --- CMakeLists.txt | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ef97de14..ecb90a23 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ cmake_minimum_required(VERSION 3.0) project(nghttp2 VERSION 1.45.90) # See versioning rule: -# http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html +# https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html set(LT_CURRENT 35) set(LT_REVISION 0) set(LT_AGE 21) diff --git a/configure.ac b/configure.ac index e7654ec3..7b7465d2 100644 --- a/configure.ac +++ b/configure.ac @@ -22,7 +22,7 @@ dnl OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION dnl WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. dnl Do not change user variables! -dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html +dnl https://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) AC_INIT([nghttp2], [1.46.0-DEV], [t-tujikawa@users.sourceforge.net]) From 80cc623eb2683299791b2988722ffcf0e6488d62 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 21 Sep 2021 22:46:32 +0900 Subject: [PATCH 051/142] nghttpx: Allocate server id in Connection ID --- bpf/reuseport_kern.c | 12 ++++++++---- gennghttpxfun.py | 1 + src/shrpx.cc | 25 ++++++++++++++++++++++++- src/shrpx_config.cc | 16 ++++++++++++++++ src/shrpx_config.h | 4 ++++ src/shrpx_connection_handler.cc | 6 +++--- src/shrpx_quic.cc | 24 +++++++++++++++++------- src/shrpx_quic.h | 7 ++++++- src/shrpx_quic_connection_handler.cc | 9 +++++++-- src/shrpx_worker.cc | 6 ++++-- src/shrpx_worker.h | 4 ++-- 11 files changed, 92 insertions(+), 22 deletions(-) diff --git a/bpf/reuseport_kern.c b/bpf/reuseport_kern.c index 51cbdc18..7a48afa9 100644 --- a/bpf/reuseport_kern.c +++ b/bpf/reuseport_kern.c @@ -455,6 +455,7 @@ typedef struct quic_hd { #define MAX_DCIDLEN 20 #define MIN_DCIDLEN 8 #define CID_PREFIXLEN 8 +#define CID_PREFIX_OFFSET 1 enum { NGTCP2_PKT_INITIAL = 0x0, @@ -579,6 +580,7 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) { __u8 qpktbuf[6 + MAX_DCIDLEN]; struct AES_ctx aes_ctx; __u8 key[AES_KEYLEN]; + __u8 *cid_prefix; if (bpf_skb_load_bytes(reuse_md, sizeof(struct udphdr), qpktbuf, sizeof(qpktbuf)) != 0) { @@ -615,9 +617,10 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) { case NGTCP2_PKT_INITIAL: case NGTCP2_PKT_0RTT: if (qhd.dcidlen == SV_DCIDLEN) { - AES_ECB_decrypt(&aes_ctx, qhd.dcid); + cid_prefix = qhd.dcid + CID_PREFIX_OFFSET; + AES_ECB_decrypt(&aes_ctx, cid_prefix); - psk_index = bpf_map_lookup_elem(&cid_prefix_map, qhd.dcid); + psk_index = bpf_map_lookup_elem(&cid_prefix_map, cid_prefix); if (psk_index != NULL) { sk_index = *psk_index; @@ -634,9 +637,10 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) { return SK_DROP; } - AES_ECB_decrypt(&aes_ctx, qhd.dcid); + cid_prefix = qhd.dcid + CID_PREFIX_OFFSET; + AES_ECB_decrypt(&aes_ctx, cid_prefix); - psk_index = bpf_map_lookup_elem(&cid_prefix_map, qhd.dcid); + psk_index = bpf_map_lookup_elem(&cid_prefix_map, cid_prefix); if (psk_index == NULL) { sk_index = sk_index_from_dcid(&qhd, reuse_md, *pnum_socks); diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 17eefd7b..e9810b8f 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -193,6 +193,7 @@ OPTIONS = [ "frontend-quic-require-token", "frontend-quic-congestion-controller", "frontend-quic-connection-id-encryption-key", + "frontend-quic-server-id", ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index 5189ee7b..e06d899b 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1342,6 +1342,7 @@ int generate_cid_prefix( std::vector> &cid_prefixes, const Config *config) { auto &apiconf = config->api; + auto &quicconf = config->quic; size_t num_cid_prefix; if (config->single_thread) { @@ -1360,7 +1361,8 @@ int generate_cid_prefix( cid_prefixes.resize(num_cid_prefix); for (auto &cid_prefix : cid_prefixes) { - if (create_cid_prefix(cid_prefix.data()) != 0) { + if (create_cid_prefix(cid_prefix.data(), + quicconf.upstream.server_id.data()) != 0) { return -1; } } @@ -1864,6 +1866,12 @@ void fill_default_config(Config *config) { assert(0); abort(); } + + if (RAND_bytes(upstreamconf.server_id.data(), + upstreamconf.server_id.size()) != 1) { + assert(0); + abort(); + } } auto &http3conf = config->http3; @@ -3253,6 +3261,14 @@ HTTP/3 and QUIC: connection in a configuration reload event, old and new configuration must have this option and share the same key. + --frontend-quic-server-id= + Specify server ID encoded in Connection ID to identify + this particular server instance. Connection ID is + encrypted and this part is not visible in public. It + must be 2 bytes long and must be encoded in hex string + (which is 4 bytes long). If this option is omitted, a + random server ID is generated on startup and + configuration reload. --no-quic-bpf Disable eBPF. --frontend-http3-window-size= @@ -4053,6 +4069,8 @@ int main(int argc, char **argv) { required_argument, &flag, 183}, {SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY.c_str(), required_argument, &flag, 184}, + {SHRPX_OPT_FRONTEND_QUIC_SERVER_ID.c_str(), required_argument, &flag, + 185}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -4936,6 +4954,11 @@ int main(int argc, char **argv) { SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY, StringRef{optarg}); break; + case 185: + // --frontend-quic-server-id + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_SERVER_ID, + StringRef{optarg}); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index f210128b..76704d0e 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2273,6 +2273,11 @@ int option_lookup_token(const char *name, size_t namelen) { break; case 23: switch (name[22]) { + case 'd': + if (util::strieq_l("frontend-quic-server-i", name, 22)) { + return SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID; + } + break; case 'e': if (util::strieq_l("client-private-key-fil", name, 22)) { return SHRPX_OPTID_CLIENT_PRIVATE_KEY_FILE; @@ -4036,6 +4041,17 @@ int parse_config(Config *config, int optid, const StringRef &opt, optarg); #endif // ENABLE_HTTP3 + return 0; + case SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID: +#ifdef ENABLE_HTTP3 + if (optarg.size() != config->quic.upstream.server_id.size() * 2 || + !util::is_hex_string(optarg)) { + LOG(ERROR) << opt << ": must be a hex-string"; + return -1; + } + util::decode_hex(std::begin(config->quic.upstream.server_id), optarg); +#endif // ENABLE_HTTP3 + return 0; case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 6202b174..fef79a60 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -393,6 +393,8 @@ constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER = StringRef::from_lit("frontend-quic-congestion-controller"); constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY = StringRef::from_lit("frontend-quic-connection-id-encryption-key"); +constexpr auto SHRPX_OPT_FRONTEND_QUIC_SERVER_ID = + StringRef::from_lit("frontend-quic-server-id"); constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -764,6 +766,7 @@ struct QUICConfig { bool early_data; bool require_token; std::array cid_encryption_key; + std::array server_id; } upstream; struct { StringRef prog_file; @@ -1223,6 +1226,7 @@ enum { SHRPX_OPTID_FRONTEND_QUIC_IDLE_TIMEOUT, SHRPX_OPTID_FRONTEND_QUIC_QLOG_DIR, SHRPX_OPTID_FRONTEND_QUIC_REQUIRE_TOKEN, + SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID, SHRPX_OPTID_FRONTEND_READ_TIMEOUT, SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT, SHRPX_OPTID_HEADER_FIELD_BUFFER, diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index c1c72a02..87a1efb4 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -1292,9 +1292,9 @@ int ConnectionHandler::quic_ipc_read() { std::array decrypted_dcid; - if (decrypt_quic_connection_id(decrypted_dcid.data(), dcid, - quicconf.upstream.cid_encryption_key.data()) != - 0) { + if (decrypt_quic_connection_id( + decrypted_dcid.data(), dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, + quicconf.upstream.cid_encryption_key.data()) != 0) { return -1; } diff --git a/src/shrpx_quic.cc b/src/shrpx_quic.cc index aa38617a..bf313909 100644 --- a/src/shrpx_quic.cc +++ b/src/shrpx_quic.cc @@ -143,30 +143,40 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, return 0; } -int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen) { +int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, + const uint8_t *server_id, + const uint8_t *key) { + assert(cidlen == SHRPX_QUIC_SCIDLEN); + if (RAND_bytes(cid.data, cidlen) != 1) { return -1; } cid.datalen = cidlen; - return 0; + auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET; + + std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, p); + + return encrypt_quic_connection_id(p, p, key); } int generate_encrypted_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, const uint8_t *cid_prefix, const uint8_t *key) { - assert(cidlen > SHRPX_QUIC_CID_PREFIXLEN); + assert(cidlen == SHRPX_QUIC_SCIDLEN); - auto p = std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, cid.data); - - if (RAND_bytes(p, cidlen - SHRPX_QUIC_CID_PREFIXLEN) != 1) { + if (RAND_bytes(cid.data, cidlen) != 1) { return -1; } cid.datalen = cidlen; - return encrypt_quic_connection_id(cid.data, cid.data, key); + auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET; + + std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, p); + + return encrypt_quic_connection_id(p, p, key); } int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, diff --git a/src/shrpx_quic.h b/src/shrpx_quic.h index 96a109fc..846d5a83 100644 --- a/src/shrpx_quic.h +++ b/src/shrpx_quic.h @@ -62,7 +62,10 @@ namespace shrpx { struct UpstreamAddr; constexpr size_t SHRPX_QUIC_SCIDLEN = 20; +constexpr size_t SHRPX_QUIC_SERVER_IDLEN = 2; +// SHRPX_QUIC_CID_PREFIXLEN includes SHRPX_QUIC_SERVER_IDLEN. constexpr size_t SHRPX_QUIC_CID_PREFIXLEN = 8; +constexpr size_t SHRPX_QUIC_CID_PREFIX_OFFSET = 1; constexpr size_t SHRPX_QUIC_DECRYPTED_DCIDLEN = 16; constexpr size_t SHRPX_QUIC_CID_ENCRYPTION_KEYLEN = 16; constexpr size_t SHRPX_QUIC_MAX_UDP_PAYLOAD_SIZE = 1472; @@ -78,7 +81,9 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, size_t local_salen, const uint8_t *data, size_t datalen, size_t gso_size); -int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen); +int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, + const uint8_t *server_id, + const uint8_t *key); int generate_encrypted_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, const uint8_t *cid_prefix, diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 41b8b9e4..00f5876e 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -128,7 +128,7 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, if (dcidlen == SHRPX_QUIC_SCIDLEN) { if (decrypt_quic_connection_id( - decrypted_dcid.data(), dcid, + decrypted_dcid.data(), dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, quicconf.upstream.cid_encryption_key.data()) != 0) { return 0; } @@ -419,9 +419,14 @@ int QUICConnectionHandler::send_retry( return -1; } + auto config = get_config(); + auto &quicconf = config->quic; + ngtcp2_cid retry_scid; - if (generate_quic_connection_id(retry_scid, SHRPX_QUIC_SCIDLEN) != 0) { + if (generate_quic_retry_connection_id( + retry_scid, SHRPX_QUIC_SCIDLEN, quicconf.upstream.server_id.data(), + quicconf.upstream.cid_encryption_key.data()) != 0) { return -1; } diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 75bd7325..720fd8d2 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -1273,8 +1273,10 @@ void downstream_failure(DownstreamAddr *addr, const Address *raddr) { } #ifdef ENABLE_HTTP3 -int create_cid_prefix(uint8_t *cid_prefix) { - if (RAND_bytes(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN) != 1) { +int create_cid_prefix(uint8_t *cid_prefix, const uint8_t *server_id) { + auto p = std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, cid_prefix); + + if (RAND_bytes(p, SHRPX_QUIC_CID_PREFIXLEN - SHRPX_QUIC_SERVER_IDLEN) != 1) { return -1; } diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 72ef2e68..cd53173f 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -467,8 +467,8 @@ void downstream_failure(DownstreamAddr *addr, const Address *raddr); #ifdef ENABLE_HTTP3 // Creates unpredictable SHRPX_QUIC_CID_PREFIXLEN bytes sequence which // is used as a prefix of QUIC Connection ID. This function returns -// -1 on failure. -int create_cid_prefix(uint8_t *cid_prefix); +// -1 on failure. |server_id| must be 2 bytes long. +int create_cid_prefix(uint8_t *cid_prefix, const uint8_t *server_id); #endif // ENABLE_HTTP3 } // namespace shrpx From 1c7a4ecc7f7d1b289123a0575834eb00d16bd65f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 21 Sep 2021 23:07:34 +0900 Subject: [PATCH 052/142] nghttpx: Rename generate_encrypted_quic_connection_id to generate_quic_connection_id --- src/shrpx_http3_upstream.cc | 4 ++-- src/shrpx_quic.cc | 5 ++--- src/shrpx_quic.h | 5 ++--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 68775eb2..4010e098 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -221,7 +221,7 @@ int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token, auto config = get_config(); auto &quicconf = config->quic; - if (generate_encrypted_quic_connection_id( + if (generate_quic_connection_id( *cid, cidlen, worker->get_cid_prefix(), quicconf.upstream.cid_encryption_key.data()) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; @@ -559,7 +559,7 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, ngtcp2_cid scid; - if (generate_encrypted_quic_connection_id( + if (generate_quic_connection_id( scid, SHRPX_QUIC_SCIDLEN, worker->get_cid_prefix(), quicconf.upstream.cid_encryption_key.data()) != 0) { return -1; diff --git a/src/shrpx_quic.cc b/src/shrpx_quic.cc index bf313909..529cde57 100644 --- a/src/shrpx_quic.cc +++ b/src/shrpx_quic.cc @@ -161,9 +161,8 @@ int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, return encrypt_quic_connection_id(p, p, key); } -int generate_encrypted_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, - const uint8_t *cid_prefix, - const uint8_t *key) { +int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, + const uint8_t *cid_prefix, const uint8_t *key) { assert(cidlen == SHRPX_QUIC_SCIDLEN); if (RAND_bytes(cid.data, cidlen) != 1) { diff --git a/src/shrpx_quic.h b/src/shrpx_quic.h index 846d5a83..52a4d67a 100644 --- a/src/shrpx_quic.h +++ b/src/shrpx_quic.h @@ -85,9 +85,8 @@ int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, const uint8_t *server_id, const uint8_t *key); -int generate_encrypted_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, - const uint8_t *cid_prefix, - const uint8_t *key); +int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, + const uint8_t *cid_prefix, const uint8_t *key); int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, const uint8_t *key); From c40309ae8e17d8cabb0b437fbfa3a24757a0cf46 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 22 Sep 2021 22:59:02 +0900 Subject: [PATCH 053/142] nghttpx: optarg should be allocated per configuration --- src/shrpx_config.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 76704d0e..74483d19 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -3866,7 +3866,7 @@ int parse_config(Config *config, int optid, const StringRef &opt, "65535], inclusive"; return -1; } - config->http.redirect_https_port = optarg; + config->http.redirect_https_port = make_string_ref(config->balloc, optarg); return 0; } case SHRPX_OPTID_FRONTEND_MAX_REQUESTS: @@ -4007,7 +4007,7 @@ int parse_config(Config *config, int optid, const StringRef &opt, return 0; case SHRPX_OPTID_FRONTEND_QUIC_QLOG_DIR: #ifdef ENABLE_HTTP3 - config->quic.upstream.qlog.dir = optarg; + config->quic.upstream.qlog.dir = make_string_ref(config->balloc, optarg); #endif // ENABLE_HTTP3 return 0; From 308c73bfa2aa2e191d7a0bbbf2355ca7b2ef9b10 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 22 Sep 2021 18:24:50 +0900 Subject: [PATCH 054/142] nghttpx: Read QUIC keying materials from file Add --frontend-quic-secret-file to read QUIC keying materials from file. --frontend-quic-connection-id-encryption-key was removed in favor of this new option. --- README.rst | 6 +- doc/sources/nghttpx-howto.rst | 8 +-- gennghttpxfun.py | 2 +- src/shrpx.cc | 56 ++++++++++-------- src/shrpx_config.cc | 88 +++++++++++++++++++++++----- src/shrpx_config.h | 29 ++++++--- src/shrpx_connection_handler.cc | 38 ++++-------- src/shrpx_connection_handler.h | 5 +- src/shrpx_http3_upstream.cc | 45 +++++++------- src/shrpx_quic.cc | 82 ++++++++++++++++---------- src/shrpx_quic.h | 36 ++++++++---- src/shrpx_quic_connection_handler.cc | 72 +++++++++++++++-------- src/shrpx_worker.cc | 18 ++---- src/shrpx_worker.h | 5 -- src/shrpx_worker_process.cc | 45 +++++++++++++- 15 files changed, 340 insertions(+), 195 deletions(-) diff --git a/README.rst b/README.rst index d7bc328c..2a500b63 100644 --- a/README.rst +++ b/README.rst @@ -341,9 +341,9 @@ configure script with ``--enable-http3``. For nghttpx to reload configurations and swapping its executable while gracefully terminating old worker processes, eBPF is required. Run the configure script with ``--enable-http3 --with-libbpf`` to build -eBPF program. The Connection ID encryption key must be set with -``--frontend-quic-connection-id-encryption-key`` and must not change -in order to keep the existing connections alive during reload. +eBPF program. The QUIC keying material must be set with +``--frontend-quic-secret-file`` in order to keep the existing +connections alive during reload. The detailed steps to build HTTP/3 enabled h2load and nghttpx follow. diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index 774a3cd8..90ab1b89 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -534,10 +534,10 @@ nghttpx does not support HTTP/3 on backend connection. Hot swapping (SIGUSR2) or configuration reload (SIGHUP) require eBPF program. Without eBPF, old worker processes keep getting HTTP/3 -traffic and do not work as intended. Connection ID encryption key -must be set with -:option:`--frontend-quic-connection-id-encryption-key` and must not -change in order to keep the existing connections alive during reload. +traffic and do not work as intended. The QUIC keying material to +encrypt Connection ID must be set with +:option:`--frontend-quic-secret-file` and must provide the existing +keys in order to keep the existing connections alive during reload. In order announce that HTTP/3 endpoint is available, you should specify alt-svc header field. For example, the following options send diff --git a/gennghttpxfun.py b/gennghttpxfun.py index e9810b8f..2ee66ae8 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -192,8 +192,8 @@ OPTIONS = [ "frontend-quic-qlog-dir", "frontend-quic-require-token", "frontend-quic-congestion-controller", - "frontend-quic-connection-id-encryption-key", "frontend-quic-server-id", + "frontend-quic-secret-file", ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index e06d899b..8fc8893f 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1859,14 +1859,6 @@ void fill_default_config(Config *config) { upstreamconf.congestion_controller = NGTCP2_CC_ALGO_CUBIC; - // TODO Not really nice to generate random key here, but fine for - // now. - if (RAND_bytes(upstreamconf.cid_encryption_key.data(), - upstreamconf.cid_encryption_key.size()) != 1) { - assert(0); - abort(); - } - if (RAND_bytes(upstreamconf.server_id.data(), upstreamconf.server_id.size()) != 1) { assert(0); @@ -3253,14 +3245,31 @@ HTTP/3 and QUIC: ? "cubic" : "bbr") << R"( - --frontend-quic-connection-id-encryption-key= - Specify Connection ID encryption key. The encryption - key must be 16 bytes, and it must be encoded in hex - string (which is 32 bytes long). If this option is - omitted, new key is generated. In order to survive QUIC - connection in a configuration reload event, old and new - configuration must have this option and share the same - key. + --frontend-quic-secret-file= + Path to file that contains secure random data to be used + as QUIC keying materials. It is used to derive keys for + encrypting tokens and Connection IDs. It is not used to + encrypt QUIC packets. Each line of this file must + contain exactly 136 bytes hex-encoded string (when + decoded the byte string is 68 bytes long). The first 2 + bits of decoded byte string are used to identify the + keying material. An empty line or a line which starts + '#' is ignored. The file can contain more than one + keying materials. Because the identifier is 2 bits, at + most 4 keying materials are read and the remaining data + is discarded. The first keying material in the file is + primarily used for encryption and decryption for new + connection. The other ones are used to decrypt data for + the existing connections. Specifying multiple keying + materials enables key rotation. Please note that key + rotation does not occur automatically. User should + update files or change options values and restart + nghttpx gracefully. If opening or reading given file + fails, all loaded keying materials are discarded and it + is treated as if none of this option is given. If this + option is not given or an error occurred while opening + or reading a file, a keying material is generated + internally on startup and reload. --frontend-quic-server-id= Specify server ID encoded in Connection ID to identify this particular server instance. Connection ID is @@ -4067,10 +4076,10 @@ int main(int argc, char **argv) { 182}, {SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER.c_str(), required_argument, &flag, 183}, - {SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY.c_str(), - required_argument, &flag, 184}, {SHRPX_OPT_FRONTEND_QUIC_SERVER_ID.c_str(), required_argument, &flag, 185}, + {SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE.c_str(), required_argument, &flag, + 186}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -4948,17 +4957,16 @@ int main(int argc, char **argv) { cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER, StringRef{optarg}); break; - case 184: - // --frontend-quic-connection-id-encryption-key - cmdcfgs.emplace_back( - SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY, - StringRef{optarg}); - break; case 185: // --frontend-quic-server-id cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_SERVER_ID, StringRef{optarg}); break; + case 186: + // --frontend-quic-secret-file + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE, + StringRef{optarg}); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 74483d19..d498ef02 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -230,6 +230,69 @@ read_tls_ticket_key_file(const std::vector &files, return ticket_keys; } +#ifdef ENABLE_HTTP3 +std::shared_ptr +read_quic_secret_file(const StringRef &path) { + constexpr size_t expectedlen = + SHRPX_QUIC_SECRET_RESERVEDLEN + SHRPX_QUIC_SECRETLEN + SHRPX_QUIC_SALTLEN; + + auto qkms = std::make_shared(); + auto &kms = qkms->keying_materials; + + std::ifstream f(path.c_str()); + if (!f) { + LOG(ERROR) << "frontend-quic-secret-file: could not open file " << path; + return nullptr; + } + + std::array buf; + + while (f.getline(buf.data(), buf.size())) { + if (f.gcount() == 1 || buf[0] == '#') { + continue; + } + + auto s = StringRef{std::begin(buf), std::begin(buf) + f.gcount() - 1}; + if (s.size() != expectedlen * 2 || !util::is_hex_string(s)) { + LOG(ERROR) << "frontend-quic-secret-file: each line must be a " + << expectedlen * 2 << " bytes hex encoded string"; + return nullptr; + } + + kms.emplace_back(); + auto &qkm = kms.back(); + + auto p = std::begin(s); + + util::decode_hex(std::begin(qkm.reserved), + StringRef{p, p + qkm.reserved.size()}); + p += qkm.reserved.size() * 2; + util::decode_hex(std::begin(qkm.secret), + StringRef{p, p + qkm.secret.size()}); + p += qkm.secret.size() * 2; + util::decode_hex(std::begin(qkm.salt), StringRef{p, p + qkm.salt.size()}); + p += qkm.salt.size() * 2; + + assert(static_cast(p - std::begin(s)) == expectedlen * 2); + + qkm.id = qkm.reserved[0] & 0xc0; + + if (kms.size() == 4) { + break; + } + } + + if (f.bad()) { + LOG(ERROR) + << "frontend-quic-secret-file: error occurred while reading file " + << path; + return nullptr; + } + + return qkms; +} +#endif // ENABLE_HTTP3 + FILE *open_file_for_write(const char *filename) { std::array errbuf; @@ -2344,6 +2407,9 @@ int option_lookup_token(const char *name, size_t namelen) { if (util::strieq_l("backend-http2-window-siz", name, 24)) { return SHRPX_OPTID_BACKEND_HTTP2_WINDOW_SIZE; } + if (util::strieq_l("frontend-quic-secret-fil", name, 24)) { + return SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE; + } break; case 'g': if (util::strieq_l("http2-no-cookie-crumblin", name, 24)) { @@ -2689,10 +2755,6 @@ int option_lookup_token(const char *name, size_t namelen) { case 42: switch (name[41]) { case 'y': - if (util::strieq_l("frontend-quic-connection-id-encryption-ke", name, - 41)) { - return SHRPX_OPTID_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY; - } if (util::strieq_l("tls-session-cache-memcached-address-famil", name, 41)) { return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY; @@ -4029,18 +4091,6 @@ int parse_config(Config *config, int optid, const StringRef &opt, } #endif // ENABLE_HTTP3 - return 0; - case SHRPX_OPTID_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY: -#ifdef ENABLE_HTTP3 - if (optarg.size() != config->quic.upstream.cid_encryption_key.size() * 2 || - !util::is_hex_string(optarg)) { - LOG(ERROR) << opt << ": must be a hex-string"; - return -1; - } - util::decode_hex(std::begin(config->quic.upstream.cid_encryption_key), - optarg); -#endif // ENABLE_HTTP3 - return 0; case SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID: #ifdef ENABLE_HTTP3 @@ -4052,6 +4102,12 @@ int parse_config(Config *config, int optid, const StringRef &opt, util::decode_hex(std::begin(config->quic.upstream.server_id), optarg); #endif // ENABLE_HTTP3 + return 0; + case SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE: +#ifdef ENABLE_HTTP3 + config->quic.upstream.secret_file = make_string_ref(config->balloc, optarg); +#endif // ENABLE_HTTP3 + return 0; case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index fef79a60..0c276ddd 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -391,10 +391,10 @@ constexpr auto SHRPX_OPT_FRONTEND_QUIC_REQUIRE_TOKEN = StringRef::from_lit("frontend-quic-require-token"); constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER = StringRef::from_lit("frontend-quic-congestion-controller"); -constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY = - StringRef::from_lit("frontend-quic-connection-id-encryption-key"); constexpr auto SHRPX_OPT_FRONTEND_QUIC_SERVER_ID = StringRef::from_lit("frontend-quic-server-id"); +constexpr auto SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE = + StringRef::from_lit("frontend-quic-secret-file"); constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -606,10 +606,18 @@ struct TLSCertificate { }; #ifdef ENABLE_HTTP3 -struct QUICSecret { - std::array - stateless_reset_secret; - std::array token_secret; +struct QUICKeyingMaterial { + std::array reserved; + std::array secret; + std::array salt; + std::array cid_encryption_key; + // Identifier of this keying material. Only the first 2 bits are + // used. + uint8_t id; +}; + +struct QUICKeyingMaterials { + std::vector keying_materials; }; #endif // ENABLE_HTTP3 @@ -765,8 +773,8 @@ struct QUICConfig { ngtcp2_cc_algo congestion_controller; bool early_data; bool require_token; - std::array cid_encryption_key; std::array server_id; + StringRef secret_file; } upstream; struct { StringRef prog_file; @@ -1220,12 +1228,12 @@ enum { SHRPX_OPTID_FRONTEND_MAX_REQUESTS, SHRPX_OPTID_FRONTEND_NO_TLS, SHRPX_OPTID_FRONTEND_QUIC_CONGESTION_CONTROLLER, - SHRPX_OPTID_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY, SHRPX_OPTID_FRONTEND_QUIC_DEBUG_LOG, SHRPX_OPTID_FRONTEND_QUIC_EARLY_DATA, SHRPX_OPTID_FRONTEND_QUIC_IDLE_TIMEOUT, SHRPX_OPTID_FRONTEND_QUIC_QLOG_DIR, SHRPX_OPTID_FRONTEND_QUIC_REQUIRE_TOKEN, + SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE, SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID, SHRPX_OPTID_FRONTEND_READ_TIMEOUT, SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT, @@ -1377,6 +1385,11 @@ std::unique_ptr read_tls_ticket_key_file(const std::vector &files, const EVP_CIPHER *cipher, const EVP_MD *hmac); +#ifdef ENABLE_HTTP3 +std::shared_ptr +read_quic_secret_file(const StringRef &path); +#endif // ENABLE_HTTP3 + // Returns string representation of |proto|. StringRef strproto(Proto proto); diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 87a1efb4..4d32b6c5 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -298,8 +298,6 @@ int ConnectionHandler::create_single_worker() { #endif // HAVE_MRUBY #ifdef ENABLE_HTTP3 - single_worker_->set_quic_secret(quic_secret_); - if (single_worker_->setup_quic_server_socket() != 0) { return -1; } @@ -404,8 +402,6 @@ int ConnectionHandler::create_worker_thread(size_t num) { # endif // HAVE_MRUBY # ifdef ENABLE_HTTP3 - worker->set_quic_secret(quic_secret_); - if ((!apiconf.enabled || i != 0) && worker->setup_quic_server_socket() != 0) { return -1; @@ -1047,25 +1043,14 @@ int ConnectionHandler::forward_quic_packet(const UpstreamAddr *faddr, return -1; } -int ConnectionHandler::create_quic_secret() { - auto quic_secret = std::make_shared(); +void ConnectionHandler::set_quic_keying_materials( + std::shared_ptr qkms) { + quic_keying_materials_ = std::move(qkms); +} - if (generate_quic_stateless_reset_secret( - quic_secret->stateless_reset_secret.data()) != 0) { - LOG(ERROR) << "Failed to generate QUIC Stateless Reset secret"; - - return -1; - } - - if (generate_quic_token_secret(quic_secret->token_secret.data()) != 0) { - LOG(ERROR) << "Failed to generate QUIC token secret"; - - return -1; - } - - quic_secret_ = std::move(quic_secret); - - return 0; +const std::shared_ptr & +ConnectionHandler::get_quic_keying_materials() const { + return quic_keying_materials_; } void ConnectionHandler::set_cid_prefixes( @@ -1287,14 +1272,13 @@ int ConnectionHandler::quic_ipc_read() { return 0; } - auto config = get_config(); - auto &quicconf = config->quic; + auto &qkm = quic_keying_materials_->keying_materials.front(); std::array decrypted_dcid; - if (decrypt_quic_connection_id( - decrypted_dcid.data(), dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, - quicconf.upstream.cid_encryption_key.data()) != 0) { + if (decrypt_quic_connection_id(decrypted_dcid.data(), + dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, + qkm.cid_encryption_key.data()) != 0) { return -1; } diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h index 2c8d9c9a..af391750 100644 --- a/src/shrpx_connection_handler.h +++ b/src/shrpx_connection_handler.h @@ -198,7 +198,8 @@ public: const Address &local_addr, const uint8_t *cid_prefix, const uint8_t *data, size_t datalen); - int create_quic_secret(); + void set_quic_keying_materials(std::shared_ptr qkms); + const std::shared_ptr &get_quic_keying_materials() const; void set_cid_prefixes( const std::vector> @@ -263,7 +264,7 @@ private: # ifdef HAVE_LIBBPF std::vector quic_bpf_refs_; # endif // HAVE_LIBBPF - std::shared_ptr quic_secret_; + std::shared_ptr quic_keying_materials_; std::vector quic_all_ssl_ctx_; std::vector> quic_indexed_ssl_ctx_; #endif // ENABLE_HTTP3 diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 4010e098..c67d8aae 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -40,6 +40,7 @@ #include "shrpx_quic.h" #include "shrpx_worker.h" #include "shrpx_http.h" +#include "shrpx_connection_handler.h" #ifdef HAVE_MRUBY # include "shrpx_mruby.h" #endif // HAVE_MRUBY @@ -217,21 +218,17 @@ int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token, auto upstream = static_cast(user_data); auto handler = upstream->get_client_handler(); auto worker = handler->get_worker(); + auto conn_handler = worker->get_connection_handler(); + auto &qkms = conn_handler->get_quic_keying_materials(); + auto &qkm = qkms->keying_materials.front(); - auto config = get_config(); - auto &quicconf = config->quic; - - if (generate_quic_connection_id( - *cid, cidlen, worker->get_cid_prefix(), - quicconf.upstream.cid_encryption_key.data()) != 0) { + if (generate_quic_connection_id(*cid, cidlen, worker->get_cid_prefix(), + qkm.id, qkm.cid_encryption_key.data()) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } - auto &quic_secret = worker->get_quic_secret(); - auto &secret = quic_secret->stateless_reset_secret; - - if (generate_quic_stateless_reset_token(token, *cid, secret.data(), - secret.size()) != 0) { + if (generate_quic_stateless_reset_token(token, *cid, qkm.secret.data(), + qkm.secret.size()) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } @@ -487,11 +484,13 @@ int Http3Upstream::handshake_completed() { auto path = ngtcp2_conn_get_path(conn_); auto worker = handler_->get_worker(); - auto &quic_secret = worker->get_quic_secret(); - auto &secret = quic_secret->token_secret; + auto conn_handler = worker->get_connection_handler(); + auto &qkms = conn_handler->get_quic_keying_materials(); + auto &qkm = qkms->keying_materials.front(); if (generate_token(token.data(), tokenlen, path->remote.addr, - path->remote.addrlen, secret.data()) != 0) { + path->remote.addrlen, qkm.secret.data(), + qkm.secret.size()) != 0) { return 0; } @@ -513,6 +512,7 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, int rv; auto worker = handler_->get_worker(); + auto conn_handler = worker->get_connection_handler(); auto callbacks = ngtcp2_callbacks{ nullptr, // client_initial @@ -557,11 +557,14 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, auto &quicconf = config->quic; auto &http3conf = config->http3; + auto &qkms = conn_handler->get_quic_keying_materials(); + auto &qkm = qkms->keying_materials.front(); + ngtcp2_cid scid; - if (generate_quic_connection_id( - scid, SHRPX_QUIC_SCIDLEN, worker->get_cid_prefix(), - quicconf.upstream.cid_encryption_key.data()) != 0) { + if (generate_quic_connection_id(scid, SHRPX_QUIC_SCIDLEN, + worker->get_cid_prefix(), qkm.id, + qkm.cid_encryption_key.data()) != 0) { return -1; } @@ -608,12 +611,8 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, params.original_dcid = initial_hd.dcid; } - auto &quic_secret = worker->get_quic_secret(); - auto &stateless_reset_secret = quic_secret->stateless_reset_secret; - - rv = generate_quic_stateless_reset_token(params.stateless_reset_token, scid, - stateless_reset_secret.data(), - stateless_reset_secret.size()); + rv = generate_quic_stateless_reset_token( + params.stateless_reset_token, scid, qkm.secret.data(), qkm.secret.size()); if (rv != 0) { ULOG(ERROR, this) << "generate_quic_stateless_reset_token failed"; return -1; diff --git a/src/shrpx_quic.cc b/src/shrpx_quic.cc index 529cde57..a0bf56c3 100644 --- a/src/shrpx_quic.cc +++ b/src/shrpx_quic.cc @@ -144,7 +144,7 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, } int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, - const uint8_t *server_id, + const uint8_t *server_id, uint8_t km_id, const uint8_t *key) { assert(cidlen == SHRPX_QUIC_SCIDLEN); @@ -154,6 +154,8 @@ int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, cid.datalen = cidlen; + cid.data[0] = (cid.data[0] & 0x3f) | km_id; + auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET; std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, p); @@ -162,7 +164,8 @@ int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, } int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, - const uint8_t *cid_prefix, const uint8_t *key) { + const uint8_t *cid_prefix, uint8_t km_id, + const uint8_t *key) { assert(cidlen == SHRPX_QUIC_SCIDLEN); if (RAND_bytes(cid.data, cidlen) != 1) { @@ -171,6 +174,8 @@ int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, cid.datalen = cidlen; + cid.data[0] = (cid.data[0] & 0x3f) | km_id; + auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET; std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, p); @@ -257,32 +262,16 @@ int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid &cid, return 0; } -int generate_quic_stateless_reset_secret(uint8_t *secret) { - if (RAND_bytes(secret, SHRPX_QUIC_STATELESS_RESET_SECRETLEN) != 1) { - return -1; - } - - return 0; -} - -int generate_quic_token_secret(uint8_t *secret) { - if (RAND_bytes(secret, SHRPX_QUIC_TOKEN_SECRETLEN) != 1) { - return -1; - } - - return 0; -} - int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, socklen_t salen, const ngtcp2_cid &retry_scid, - const ngtcp2_cid &odcid, const uint8_t *token_secret) { + const ngtcp2_cid &odcid, const uint8_t *secret, + size_t secretlen) { auto t = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); auto stokenlen = ngtcp2_crypto_generate_retry_token( - token, token_secret, SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, &retry_scid, - &odcid, t); + token, secret, secretlen, sa, salen, &retry_scid, &odcid, t); if (stokenlen < 0) { return -1; } @@ -294,15 +283,16 @@ int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen, const ngtcp2_cid &dcid, const sockaddr *sa, - socklen_t salen, const uint8_t *token_secret) { + socklen_t salen, const uint8_t *secret, + size_t secretlen) { auto t = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); - if (ngtcp2_crypto_verify_retry_token(&odcid, token, tokenlen, token_secret, - SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, - &dcid, 10 * NGTCP2_SECONDS, t) != 0) { + if (ngtcp2_crypto_verify_retry_token(&odcid, token, tokenlen, secret, + secretlen, sa, salen, &dcid, + 10 * NGTCP2_SECONDS, t) != 0) { return -1; } @@ -310,13 +300,13 @@ int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen, } int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, - size_t salen, const uint8_t *token_secret) { + size_t salen, const uint8_t *secret, size_t secretlen) { auto t = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); auto stokenlen = ngtcp2_crypto_generate_regular_token( - token, token_secret, SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, t); + token, secret, secretlen, sa, salen, t); if (stokenlen < 0) { return -1; } @@ -327,18 +317,48 @@ int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, } int verify_token(const uint8_t *token, size_t tokenlen, const sockaddr *sa, - socklen_t salen, const uint8_t *token_secret) { + socklen_t salen, const uint8_t *secret, size_t secretlen) { auto t = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); - if (ngtcp2_crypto_verify_regular_token(token, tokenlen, token_secret, - SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, - 3600 * NGTCP2_SECONDS, t) != 0) { + if (ngtcp2_crypto_verify_regular_token(token, tokenlen, secret, secretlen, sa, + salen, 3600 * NGTCP2_SECONDS, + t) != 0) { return -1; } return 0; } +int generate_quic_connection_id_encryption_key(uint8_t *key, size_t keylen, + const uint8_t *secret, + size_t secretlen, + const uint8_t *salt, + size_t saltlen) { + constexpr uint8_t info[] = "connection id encryption key"; + ngtcp2_crypto_md sha256; + ngtcp2_crypto_md_init( + &sha256, reinterpret_cast(const_cast(EVP_sha256()))); + + if (ngtcp2_crypto_hkdf(key, keylen, &sha256, secret, secretlen, salt, saltlen, + info, str_size(info)) != 0) { + return -1; + } + + return 0; +} + +const QUICKeyingMaterial * +select_quic_keying_material(const QUICKeyingMaterials &qkms, + const uint8_t *cid) { + for (auto &qkm : qkms.keying_materials) { + if (((*cid) & 0xc0) == qkm.id) { + return &qkm; + } + } + + return &qkms.keying_materials.front(); +} + } // namespace shrpx diff --git a/src/shrpx_quic.h b/src/shrpx_quic.h index 52a4d67a..e66018c4 100644 --- a/src/shrpx_quic.h +++ b/src/shrpx_quic.h @@ -60,6 +60,8 @@ bool operator==(const ngtcp2_cid &lhs, const ngtcp2_cid &rhs); namespace shrpx { struct UpstreamAddr; +struct QUICKeyingMaterials; +struct QUICKeyingMaterial; constexpr size_t SHRPX_QUIC_SCIDLEN = 20; constexpr size_t SHRPX_QUIC_SERVER_IDLEN = 2; @@ -69,10 +71,11 @@ constexpr size_t SHRPX_QUIC_CID_PREFIX_OFFSET = 1; constexpr size_t SHRPX_QUIC_DECRYPTED_DCIDLEN = 16; constexpr size_t SHRPX_QUIC_CID_ENCRYPTION_KEYLEN = 16; constexpr size_t SHRPX_QUIC_MAX_UDP_PAYLOAD_SIZE = 1472; -constexpr size_t SHRPX_QUIC_STATELESS_RESET_SECRETLEN = 32; -constexpr size_t SHRPX_QUIC_TOKEN_SECRETLEN = 32; constexpr size_t SHRPX_QUIC_CONN_CLOSE_PKTLEN = 256; constexpr size_t SHRPX_QUIC_STATELESS_RESET_BURST = 100; +constexpr size_t SHRPX_QUIC_SECRET_RESERVEDLEN = 4; +constexpr size_t SHRPX_QUIC_SECRETLEN = 32; +constexpr size_t SHRPX_QUIC_SALTLEN = 32; ngtcp2_tstamp quic_timestamp(); @@ -82,11 +85,12 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, size_t gso_size); int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, - const uint8_t *server_id, + const uint8_t *server_id, uint8_t km_id, const uint8_t *key); int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, - const uint8_t *cid_prefix, const uint8_t *key); + const uint8_t *cid_prefix, uint8_t km_id, + const uint8_t *key); int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, const uint8_t *key); @@ -103,23 +107,31 @@ int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid &cid, const uint8_t *secret, size_t secretlen); -int generate_quic_stateless_reset_secret(uint8_t *secret); - -int generate_quic_token_secret(uint8_t *secret); - int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, socklen_t salen, const ngtcp2_cid &retry_scid, - const ngtcp2_cid &odcid, const uint8_t *token_secret); + const ngtcp2_cid &odcid, const uint8_t *secret, + size_t secretlen); int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen, const ngtcp2_cid &dcid, const sockaddr *sa, - socklen_t salen, const uint8_t *token_secret); + socklen_t salen, const uint8_t *secret, + size_t secretlen); int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, - size_t salen, const uint8_t *token_secret); + size_t salen, const uint8_t *secret, size_t secretlen); int verify_token(const uint8_t *token, size_t tokenlen, const sockaddr *sa, - socklen_t salen, const uint8_t *token_secret); + socklen_t salen, const uint8_t *secret, size_t secretlen); + +int generate_quic_connection_id_encryption_key(uint8_t *key, size_t keylen, + const uint8_t *secret, + size_t secretlen, + const uint8_t *salt, + size_t saltlen); + +const QUICKeyingMaterial * +select_quic_keying_material(const QUICKeyingMaterials &qkms, + const uint8_t *cid); } // namespace shrpx diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 00f5876e..37116645 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -126,14 +126,20 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, if (it == std::end(connections_)) { std::array decrypted_dcid; + auto &qkms = conn_handler->get_quic_keying_materials(); + const QUICKeyingMaterial *qkm = nullptr; + if (dcidlen == SHRPX_QUIC_SCIDLEN) { - if (decrypt_quic_connection_id( - decrypted_dcid.data(), dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, - quicconf.upstream.cid_encryption_key.data()) != 0) { + qkm = select_quic_keying_material(*qkms.get(), dcid); + + if (decrypt_quic_connection_id(decrypted_dcid.data(), + dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, + qkm->cid_encryption_key.data()) != 0) { return 0; } - if (!std::equal(std::begin(decrypted_dcid), + if (qkm != &qkms->keying_materials.front() || + !std::equal(std::begin(decrypted_dcid), std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, worker_->get_cid_prefix())) { auto quic_lwp = @@ -170,14 +176,25 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, switch (ngtcp2_accept(&hd, data, datalen)) { case 0: { - // If we get Initial and it has the CID prefix of this worker, it - // is likely that client is intentionally use the our prefix. + // If we get Initial and it has the CID prefix of this worker, + // it is likely that client is intentionally use the prefix. // Just drop it. - if (dcidlen == SHRPX_QUIC_SCIDLEN && - std::equal(std::begin(decrypted_dcid), - std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, - worker_->get_cid_prefix())) { - return 0; + if (dcidlen == SHRPX_QUIC_SCIDLEN) { + if (qkm != &qkms->keying_materials.front()) { + qkm = &qkms->keying_materials.front(); + + if (decrypt_quic_connection_id(decrypted_dcid.data(), + dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, + qkm->cid_encryption_key.data()) != 0) { + return 0; + } + } + + if (std::equal(std::begin(decrypted_dcid), + std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, + worker_->get_cid_prefix())) { + return 0; + } } if (worker_->get_graceful_shutdown()) { @@ -197,14 +214,18 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, break; } - auto &quic_secret = worker_->get_quic_secret(); - auto &secret = quic_secret->token_secret; + if (dcidlen != SHRPX_QUIC_SCIDLEN) { + // Initial packets with token must have DCID chosen by server. + return 0; + } + + auto qkm = select_quic_keying_material(*qkms.get(), dcid); switch (hd.token.base[0]) { case NGTCP2_CRYPTO_TOKEN_MAGIC_RETRY: if (verify_retry_token(odcid, hd.token.base, hd.token.len, hd.dcid, &remote_addr.su.sa, remote_addr.len, - secret.data()) != 0) { + qkm->secret.data(), qkm->secret.size()) != 0) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Failed to validate Retry token from remote=" << util::to_numeric_addr(&remote_addr); @@ -229,7 +250,8 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, break; case NGTCP2_CRYPTO_TOKEN_MAGIC_REGULAR: if (verify_token(hd.token.base, hd.token.len, &remote_addr.su.sa, - remote_addr.len, secret.data()) != 0) { + remote_addr.len, qkm->secret.data(), + qkm->secret.size()) != 0) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Failed to validate token from remote=" << util::to_numeric_addr(&remote_addr); @@ -422,11 +444,15 @@ int QUICConnectionHandler::send_retry( auto config = get_config(); auto &quicconf = config->quic; + auto conn_handler = worker_->get_connection_handler(); + auto &qkms = conn_handler->get_quic_keying_materials(); + auto &qkm = qkms->keying_materials.front(); + ngtcp2_cid retry_scid; if (generate_quic_retry_connection_id( retry_scid, SHRPX_QUIC_SCIDLEN, quicconf.upstream.server_id.data(), - quicconf.upstream.cid_encryption_key.data()) != 0) { + qkm.id, qkm.cid_encryption_key.data()) != 0) { return -1; } @@ -437,12 +463,9 @@ int QUICConnectionHandler::send_retry( ngtcp2_cid_init(&idcid, ini_dcid, ini_dcidlen); ngtcp2_cid_init(&iscid, ini_scid, ini_scidlen); - auto &quic_secret = worker_->get_quic_secret(); - auto &secret = quic_secret->token_secret; - if (generate_retry_token(token.data(), tokenlen, &remote_addr.su.sa, remote_addr.len, retry_scid, idcid, - secret.data()) != 0) { + qkm.secret.data(), qkm.secret.size()) != 0) { return -1; } @@ -539,11 +562,12 @@ int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr, ngtcp2_cid_init(&cid, dcid, dcidlen); - auto &quic_secret = worker_->get_quic_secret(); - auto &secret = quic_secret->stateless_reset_secret; + auto conn_handler = worker_->get_connection_handler(); + auto &qkms = conn_handler->get_quic_keying_materials(); + auto &qkm = qkms->keying_materials.front(); - rv = generate_quic_stateless_reset_token(token.data(), cid, secret.data(), - secret.size()); + rv = generate_quic_stateless_reset_token(token.data(), cid, qkm.secret.data(), + qkm.secret.size()); if (rv != 0) { return -1; } diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 720fd8d2..b287ca50 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -933,14 +933,14 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { return -1; } - auto &quicconf = config->quic; - constexpr uint32_t key_high_idx = 1; constexpr uint32_t key_low_idx = 2; + auto &qkms = conn_handler_->get_quic_keying_materials(); + auto &qkm = qkms->keying_materials.front(); + if (bpf_map_update_elem(bpf_map__fd(sk_info), &key_high_idx, - quicconf.upstream.cid_encryption_key.data(), - BPF_ANY) != 0) { + qkm.cid_encryption_key.data(), BPF_ANY) != 0) { LOG(FATAL) << "Failed to update key_high_idx sk_info: " << xsi_strerror(errno, errbuf.data(), errbuf.size()); close(fd); @@ -948,7 +948,7 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { } if (bpf_map_update_elem(bpf_map__fd(sk_info), &key_low_idx, - quicconf.upstream.cid_encryption_key.data() + 8, + qkm.cid_encryption_key.data() + 8, BPF_ANY) != 0) { LOG(FATAL) << "Failed to update key_low_idx sk_info: " << xsi_strerror(errno, errbuf.data(), errbuf.size()); @@ -1010,14 +1010,6 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { const uint8_t *Worker::get_cid_prefix() const { return cid_prefix_.data(); } -void Worker::set_quic_secret(const std::shared_ptr &secret) { - quic_secret_ = secret; -} - -const std::shared_ptr &Worker::get_quic_secret() const { - return quic_secret_; -} - const UpstreamAddr *Worker::find_quic_upstream_addr(const Address &local_addr) { std::array host; diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index cd53173f..8cf40e97 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -370,10 +370,6 @@ public: const uint8_t *get_cid_prefix() const; - void set_quic_secret(const std::shared_ptr &secret); - - const std::shared_ptr &get_quic_secret() const; - # ifdef HAVE_LIBBPF bool should_attach_bpf() const; @@ -412,7 +408,6 @@ private: std::array cid_prefix_; std::vector quic_upstream_addrs_; std::vector> quic_listeners_; - std::shared_ptr quic_secret_; #endif // ENABLE_HTTP3 std::shared_ptr downstreamconf_; diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc index 7bcdecd1..006b8c7d 100644 --- a/src/shrpx_worker_process.cc +++ b/src/shrpx_worker_process.cc @@ -519,10 +519,51 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { } #ifdef ENABLE_HTTP3 - if (conn_handler->create_quic_secret() != 0) { - return -1; + auto &quicconf = config->quic; + + std::shared_ptr qkms; + + if (!quicconf.upstream.secret_file.empty()) { + qkms = read_quic_secret_file(quicconf.upstream.secret_file); + if (!qkms) { + LOG(WARN) << "Use QUIC keying materials generated internally"; + } } + if (!qkms) { + qkms = std::make_shared(); + qkms->keying_materials.resize(1); + + auto &qkm = qkms->keying_materials.front(); + + if (RAND_bytes(qkm.reserved.data(), qkm.reserved.size()) != 1) { + LOG(ERROR) << "Failed to generate QUIC secret reserved data"; + return -1; + } + + if (RAND_bytes(qkm.secret.data(), qkm.secret.size()) != 1) { + LOG(ERROR) << "Failed to generate QUIC secret"; + return -1; + } + + if (RAND_bytes(qkm.salt.data(), qkm.salt.size()) != 1) { + LOG(ERROR) << "Failed to generate QUIC salt"; + return -1; + } + } + + for (auto &qkm : qkms->keying_materials) { + if (generate_quic_connection_id_encryption_key( + qkm.cid_encryption_key.data(), qkm.cid_encryption_key.size(), + qkm.secret.data(), qkm.secret.size(), qkm.salt.data(), + qkm.salt.size()) != 0) { + LOG(ERROR) << "Failed to generate QUIC Connection ID encryption key"; + return -1; + } + } + + conn_handler->set_quic_keying_materials(std::move(qkms)); + conn_handler->set_cid_prefixes(wpconf->cid_prefixes); conn_handler->set_quic_lingering_worker_processes( wpconf->quic_lingering_worker_processes); From 282050c596715ac6872b16fa6bdf9720647cffcd Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 23 Sep 2021 11:45:10 +0900 Subject: [PATCH 055/142] Update manual pages --- doc/h2load.1 | 2 +- doc/nghttp.1 | 2 +- doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 46 +++++++++++++++++++++++++++++++++++++--------- doc/nghttpx.1.rst | 43 +++++++++++++++++++++++++++++++++++-------- 5 files changed, 75 insertions(+), 20 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index 79b2cbda..8ebff2bb 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "Sep 20, 2021" "1.45.0" "nghttp2" +.TH "H2LOAD" "1" "Sep 23, 2021" "1.46.0-DEV" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index f0d2704e..8915a1fd 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "Sep 20, 2021" "1.45.0" "nghttp2" +.TH "NGHTTP" "1" "Sep 23, 2021" "1.46.0-DEV" "nghttp2" .SH NAME nghttp \- HTTP/2 client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index 2458b9f1..505cf7bf 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "Sep 20, 2021" "1.45.0" "nghttp2" +.TH "NGHTTPD" "1" "Sep 23, 2021" "1.46.0-DEV" "nghttp2" .SH NAME nghttpd \- HTTP/2 server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index ba6b8022..09d31cb0 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "Sep 20, 2021" "1.45.0" "nghttp2" +.TH "NGHTTPX" "1" "Sep 23, 2021" "1.46.0-DEV" "nghttp2" .SH NAME nghttpx \- HTTP/2 proxy . @@ -1776,14 +1776,42 @@ Default: \fBcubic\fP .UNINDENT .INDENT 0.0 .TP -.B \-\-frontend\-quic\-connection\-id\-encryption\-key= -Specify Connection ID encryption key. The encryption -key must be 16 bytes, and it must be encoded in hex -string (which is 32 bytes long). If this option is -omitted, new key is generated. In order to survive QUIC -connection in a configuration reload event, old and new -configuration must have this option and share the same -key. +.B \-\-frontend\-quic\-secret\-file= +Path to file that contains secure random data to be used +as QUIC keying materials. It is used to derive keys for +encrypting tokens and Connection IDs. It is not used to +encrypt QUIC packets. Each line of this file must +contain exactly 136 bytes hex\-encoded string (when +decoded the byte string is 68 bytes long). The first 2 +bits of decoded byte string are used to identify the +keying material. An empty line or a line which starts +\(aq#\(aq is ignored. The file can contain more than one +keying materials. Because the identifier is 2 bits, at +most 4 keying materials are read and the remaining data +is discarded. The first keying material in the file is +primarily used for encryption and decryption for new +connection. The other ones are used to decrypt data for +the existing connections. Specifying multiple keying +materials enables key rotation. Please note that key +rotation does not occur automatically. User should +update files or change options values and restart +nghttpx gracefully. If opening or reading given file +fails, all loaded keying materials are discarded and it +is treated as if none of this option is given. If this +option is not given or an error occurred while opening +or reading a file, a keying material is generated +internally on startup and reload. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-frontend\-quic\-server\-id= +Specify server ID encoded in Connection ID to identify +this particular server instance. Connection ID is +encrypted and this part is not visible in public. It +must be 2 bytes long and must be encoded in hex string +(which is 4 bytes long). If this option is omitted, a +random server ID is generated on startup and +configuration reload. .UNINDENT .INDENT 0.0 .TP diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index 278ae146..b5e25ee2 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -1620,15 +1620,42 @@ HTTP/3 and QUIC Default: ``cubic`` -.. option:: --frontend-quic-connection-id-encryption-key= +.. option:: --frontend-quic-secret-file= - Specify Connection ID encryption key. The encryption - key must be 16 bytes, and it must be encoded in hex - string (which is 32 bytes long). If this option is - omitted, new key is generated. In order to survive QUIC - connection in a configuration reload event, old and new - configuration must have this option and share the same - key. + Path to file that contains secure random data to be used + as QUIC keying materials. It is used to derive keys for + encrypting tokens and Connection IDs. It is not used to + encrypt QUIC packets. Each line of this file must + contain exactly 136 bytes hex-encoded string (when + decoded the byte string is 68 bytes long). The first 2 + bits of decoded byte string are used to identify the + keying material. An empty line or a line which starts + '#' is ignored. The file can contain more than one + keying materials. Because the identifier is 2 bits, at + most 4 keying materials are read and the remaining data + is discarded. The first keying material in the file is + primarily used for encryption and decryption for new + connection. The other ones are used to decrypt data for + the existing connections. Specifying multiple keying + materials enables key rotation. Please note that key + rotation does not occur automatically. User should + update files or change options values and restart + nghttpx gracefully. If opening or reading given file + fails, all loaded keying materials are discarded and it + is treated as if none of this option is given. If this + option is not given or an error occurred while opening + or reading a file, a keying material is generated + internally on startup and reload. + +.. option:: --frontend-quic-server-id= + + Specify server ID encoded in Connection ID to identify + this particular server instance. Connection ID is + encrypted and this part is not visible in public. It + must be 2 bytes long and must be encoded in hex string + (which is 4 bytes long). If this option is omitted, a + random server ID is generated on startup and + configuration reload. .. option:: --no-quic-bpf From c5122c12cb760e52eb37768a39f1114e07954200 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 23 Sep 2021 11:45:23 +0900 Subject: [PATCH 056/142] Update bash_completion --- doc/bash_completion/nghttpx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bash_completion/nghttpx b/doc/bash_completion/nghttpx index f27ab23d..e58c1269 100644 --- a/doc/bash_completion/nghttpx +++ b/doc/bash_completion/nghttpx @@ -8,7 +8,7 @@ _nghttpx() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--backend --frontend --backlog --backend-address-family --backend-http-proxy-uri --workers --single-thread --read-rate --read-burst --write-rate --write-burst --worker-read-rate --worker-read-burst --worker-write-rate --worker-write-burst --worker-frontend-connections --backend-connections-per-host --backend-connections-per-frontend --rlimit-nofile --backend-request-buffer --backend-response-buffer --fastopen --no-kqueue --frontend-http2-read-timeout --frontend-http3-read-timeout --frontend-read-timeout --frontend-write-timeout --frontend-keep-alive-timeout --stream-read-timeout --stream-write-timeout --backend-read-timeout --backend-write-timeout --backend-connect-timeout --backend-keep-alive-timeout --listener-disable-timeout --frontend-http2-setting-timeout --backend-http2-settings-timeout --backend-max-backoff --ciphers --tls13-ciphers --client-ciphers --tls13-client-ciphers --ecdh-curves --insecure --cacert --private-key-passwd-file --subcert --dh-param-file --npn-list --verify-client --verify-client-cacert --verify-client-tolerate-expired --client-private-key-file --client-cert-file --tls-min-proto-version --tls-max-proto-version --tls-ticket-key-file --tls-ticket-key-memcached --tls-ticket-key-memcached-address-family --tls-ticket-key-memcached-interval --tls-ticket-key-memcached-max-retry --tls-ticket-key-memcached-max-fail --tls-ticket-key-cipher --tls-ticket-key-memcached-cert-file --tls-ticket-key-memcached-private-key-file --fetch-ocsp-response-file --ocsp-update-interval --ocsp-startup --no-verify-ocsp --no-ocsp --tls-session-cache-memcached --tls-session-cache-memcached-address-family --tls-session-cache-memcached-cert-file --tls-session-cache-memcached-private-key-file --tls-dyn-rec-warmup-threshold --tls-dyn-rec-idle-timeout --no-http2-cipher-block-list --client-no-http2-cipher-block-list --tls-sct-dir --psk-secrets --client-psk-secrets --tls-no-postpone-early-data --tls-max-early-data --frontend-http2-max-concurrent-streams --backend-http2-max-concurrent-streams --frontend-http2-window-size --frontend-http2-connection-window-size --backend-http2-window-size --backend-http2-connection-window-size --http2-no-cookie-crumbling --padding --no-server-push --frontend-http2-optimize-write-buffer-size --frontend-http2-optimize-window-size --frontend-http2-encoder-dynamic-table-size --frontend-http2-decoder-dynamic-table-size --backend-http2-encoder-dynamic-table-size --backend-http2-decoder-dynamic-table-size --http2-proxy --log-level --accesslog-file --accesslog-syslog --accesslog-format --accesslog-write-early --errorlog-file --errorlog-syslog --syslog-facility --add-x-forwarded-for --strip-incoming-x-forwarded-for --no-add-x-forwarded-proto --no-strip-incoming-x-forwarded-proto --add-forwarded --strip-incoming-forwarded --forwarded-by --forwarded-for --no-via --no-strip-incoming-early-data --no-location-rewrite --host-rewrite --altsvc --http2-altsvc --add-request-header --add-response-header --request-header-field-buffer --max-request-header-fields --response-header-field-buffer --max-response-header-fields --error-page --server-name --no-server-rewrite --redirect-https-port --api-max-request-body --dns-cache-timeout --dns-lookup-timeout --dns-max-try --frontend-max-requests --frontend-http2-dump-request-header --frontend-http2-dump-response-header --frontend-frame-debug --daemon --pid-file --user --single-process --mruby-file --ignore-per-pattern-mruby-error --frontend-quic-idle-timeout --frontend-quic-debug-log --quic-bpf-program-file --frontend-quic-early-data --frontend-quic-qlog-dir --frontend-quic-require-token --frontend-quic-congestion-controller --frontend-quic-connection-id-encryption-key --no-quic-bpf --frontend-http3-window-size --frontend-http3-connection-window-size --frontend-http3-max-window-size --frontend-http3-max-connection-window-size --frontend-http3-max-concurrent-streams --conf --include --version --help ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--backend --frontend --backlog --backend-address-family --backend-http-proxy-uri --workers --single-thread --read-rate --read-burst --write-rate --write-burst --worker-read-rate --worker-read-burst --worker-write-rate --worker-write-burst --worker-frontend-connections --backend-connections-per-host --backend-connections-per-frontend --rlimit-nofile --backend-request-buffer --backend-response-buffer --fastopen --no-kqueue --frontend-http2-read-timeout --frontend-http3-read-timeout --frontend-read-timeout --frontend-write-timeout --frontend-keep-alive-timeout --stream-read-timeout --stream-write-timeout --backend-read-timeout --backend-write-timeout --backend-connect-timeout --backend-keep-alive-timeout --listener-disable-timeout --frontend-http2-setting-timeout --backend-http2-settings-timeout --backend-max-backoff --ciphers --tls13-ciphers --client-ciphers --tls13-client-ciphers --ecdh-curves --insecure --cacert --private-key-passwd-file --subcert --dh-param-file --npn-list --verify-client --verify-client-cacert --verify-client-tolerate-expired --client-private-key-file --client-cert-file --tls-min-proto-version --tls-max-proto-version --tls-ticket-key-file --tls-ticket-key-memcached --tls-ticket-key-memcached-address-family --tls-ticket-key-memcached-interval --tls-ticket-key-memcached-max-retry --tls-ticket-key-memcached-max-fail --tls-ticket-key-cipher --tls-ticket-key-memcached-cert-file --tls-ticket-key-memcached-private-key-file --fetch-ocsp-response-file --ocsp-update-interval --ocsp-startup --no-verify-ocsp --no-ocsp --tls-session-cache-memcached --tls-session-cache-memcached-address-family --tls-session-cache-memcached-cert-file --tls-session-cache-memcached-private-key-file --tls-dyn-rec-warmup-threshold --tls-dyn-rec-idle-timeout --no-http2-cipher-block-list --client-no-http2-cipher-block-list --tls-sct-dir --psk-secrets --client-psk-secrets --tls-no-postpone-early-data --tls-max-early-data --frontend-http2-max-concurrent-streams --backend-http2-max-concurrent-streams --frontend-http2-window-size --frontend-http2-connection-window-size --backend-http2-window-size --backend-http2-connection-window-size --http2-no-cookie-crumbling --padding --no-server-push --frontend-http2-optimize-write-buffer-size --frontend-http2-optimize-window-size --frontend-http2-encoder-dynamic-table-size --frontend-http2-decoder-dynamic-table-size --backend-http2-encoder-dynamic-table-size --backend-http2-decoder-dynamic-table-size --http2-proxy --log-level --accesslog-file --accesslog-syslog --accesslog-format --accesslog-write-early --errorlog-file --errorlog-syslog --syslog-facility --add-x-forwarded-for --strip-incoming-x-forwarded-for --no-add-x-forwarded-proto --no-strip-incoming-x-forwarded-proto --add-forwarded --strip-incoming-forwarded --forwarded-by --forwarded-for --no-via --no-strip-incoming-early-data --no-location-rewrite --host-rewrite --altsvc --http2-altsvc --add-request-header --add-response-header --request-header-field-buffer --max-request-header-fields --response-header-field-buffer --max-response-header-fields --error-page --server-name --no-server-rewrite --redirect-https-port --api-max-request-body --dns-cache-timeout --dns-lookup-timeout --dns-max-try --frontend-max-requests --frontend-http2-dump-request-header --frontend-http2-dump-response-header --frontend-frame-debug --daemon --pid-file --user --single-process --mruby-file --ignore-per-pattern-mruby-error --frontend-quic-idle-timeout --frontend-quic-debug-log --quic-bpf-program-file --frontend-quic-early-data --frontend-quic-qlog-dir --frontend-quic-require-token --frontend-quic-congestion-controller --frontend-quic-secret-file --frontend-quic-server-id --no-quic-bpf --frontend-http3-window-size --frontend-http3-connection-window-size --frontend-http3-max-window-size --frontend-http3-max-connection-window-size --frontend-http3-max-concurrent-streams --conf --include --version --help ' -- "$cur" ) ) ;; *) _filedir From 27e6d56d83f9a0669ef901d5eac145fc2d6966af Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 23 Sep 2021 12:00:46 +0900 Subject: [PATCH 057/142] Update doc --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 2a500b63..d919940a 100644 --- a/README.rst +++ b/README.rst @@ -32,12 +32,14 @@ Public Test Server The following endpoints are available to try out our nghttp2 implementation. -* https://nghttp2.org/ (TLS + ALPN/NPN) +* https://nghttp2.org/ (TLS + ALPN/NPN and HTTP/3) This endpoint supports ``h2``, ``h2-16``, ``h2-14``, and ``http/1.1`` via ALPN/NPN and requires TLSv1.2 for HTTP/2 connection. + It also supports HTTP/3. + * http://nghttp2.org/ (HTTP Upgrade and HTTP/2 Direct) ``h2c`` and ``http/1.1``. From d0e8efac4dd0411a8dab8ede6ec5e664f73c77a2 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 24 Sep 2021 17:50:28 +0900 Subject: [PATCH 058/142] nghttpx: Fix bug that reading QUIC secret file fails without line separator --- src/shrpx_config.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index d498ef02..64c401ed 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -248,11 +248,12 @@ read_quic_secret_file(const StringRef &path) { std::array buf; while (f.getline(buf.data(), buf.size())) { - if (f.gcount() == 1 || buf[0] == '#') { + auto len = strlen(buf.data()); + if (len == 0 || buf[0] == '#') { continue; } - auto s = StringRef{std::begin(buf), std::begin(buf) + f.gcount() - 1}; + auto s = StringRef{std::begin(buf), std::begin(buf) + len}; if (s.size() != expectedlen * 2 || !util::is_hex_string(s)) { LOG(ERROR) << "frontend-quic-secret-file: each line must be a " << expectedlen * 2 << " bytes hex encoded string"; @@ -282,7 +283,7 @@ read_quic_secret_file(const StringRef &path) { } } - if (f.bad()) { + if (f.bad() || (!f.eof() && f.fail())) { LOG(ERROR) << "frontend-quic-secret-file: error occurred while reading file " << path; From 7271537a1531bad7ca0e2153b1cf188b6462dda4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 24 Sep 2021 18:01:47 +0900 Subject: [PATCH 059/142] nghttpx: Add --rlimit-memlock option --- docker/README.rst | 4 ++-- gennghttpxfun.py | 1 + src/shrpx.cc | 21 +++++++++++++++++++++ src/shrpx_config.cc | 22 ++++++++++++++++++++++ src/shrpx_config.h | 4 ++++ 5 files changed, 50 insertions(+), 2 deletions(-) diff --git a/docker/README.rst b/docker/README.rst index 7f3a6b59..2c32565a 100644 --- a/docker/README.rst +++ b/docker/README.rst @@ -20,6 +20,6 @@ certificate in server.key and server.crt respectively : .. code-block:: text $ docker run --rm -it -v $PWD:/shared --net=host --privileged \ - --ulimit memlock=2048000 nghttp2 nghttpx \ + nghttp2 nghttpx \ /shared/server.key /shared/server.crt \ - -f'*,443;quic' + -f'*,443;quic' --rlimit-memlock 262144 diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 2ee66ae8..a38fbca1 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -194,6 +194,7 @@ OPTIONS = [ "frontend-quic-congestion-controller", "frontend-quic-server-id", "frontend-quic-secret-file", + "rlimit-memlock", ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index 8fc8893f..ede06a3e 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2376,6 +2376,12 @@ Performance: If 0 is given, nghttpx does not set the limit. Default: )" << config->rlimit_nofile << R"( + --rlimit-memlock= + Set maximum number of bytes of memory that may be locked + into RAM. If 0 is given, nghttpx does not set the + limit. + Default: )" + << config->rlimit_memlock << R"( --backend-request-buffer= Set buffer size used to store backend request. Default: )" @@ -3599,6 +3605,16 @@ int process_options(Config *config, } } + if (config->rlimit_memlock) { + struct rlimit lim = {static_cast(config->rlimit_memlock), + static_cast(config->rlimit_memlock)}; + if (setrlimit(RLIMIT_MEMLOCK, &lim) != 0) { + auto error = errno; + LOG(WARN) << "Setting rlimit-memlock failed: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + } + } + auto &fwdconf = config->http.forwarded; if (fwdconf.by_node_type == ForwardedNode::OBFUSCATED && @@ -4080,6 +4096,7 @@ int main(int argc, char **argv) { 185}, {SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE.c_str(), required_argument, &flag, 186}, + {SHRPX_OPT_RLIMIT_MEMLOCK.c_str(), required_argument, &flag, 187}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -4967,6 +4984,10 @@ int main(int argc, char **argv) { cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE, StringRef{optarg}); break; + case 187: + // --rlimit-memlock + cmdcfgs.emplace_back(SHRPX_OPT_RLIMIT_MEMLOCK, StringRef{optarg}); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 64c401ed..21a2395c 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2057,6 +2057,11 @@ int option_lookup_token(const char *name, size_t namelen) { return SHRPX_OPTID_NO_SERVER_PUSH; } break; + case 'k': + if (util::strieq_l("rlimit-memloc", name, 13)) { + return SHRPX_OPTID_RLIMIT_MEMLOCK; + } + break; case 'p': if (util::strieq_l("no-verify-ocs", name, 13)) { return SHRPX_OPTID_NO_VERIFY_OCSP; @@ -4110,6 +4115,23 @@ int parse_config(Config *config, int optid, const StringRef &opt, #endif // ENABLE_HTTP3 return 0; + case SHRPX_OPTID_RLIMIT_MEMLOCK: { + int n; + + if (parse_uint(&n, opt, optarg) != 0) { + return -1; + } + + if (n < 0) { + LOG(ERROR) << opt << ": specify the integer more than or equal to 0"; + + return -1; + } + + config->rlimit_memlock = n; + + return 0; + } case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 0c276ddd..d9c6fcef 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -395,6 +395,7 @@ constexpr auto SHRPX_OPT_FRONTEND_QUIC_SERVER_ID = StringRef::from_lit("frontend-quic-server-id"); constexpr auto SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE = StringRef::from_lit("frontend-quic-secret-file"); +constexpr auto SHRPX_OPT_RLIMIT_MEMLOCK = StringRef::from_lit("rlimit-memlock"); constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -1064,6 +1065,7 @@ struct Config { num_worker{0}, padding{0}, rlimit_nofile{0}, + rlimit_memlock{0}, uid{0}, gid{0}, pid{0}, @@ -1112,6 +1114,7 @@ struct Config { size_t num_worker; size_t padding; size_t rlimit_nofile; + size_t rlimit_memlock; uid_t uid; gid_t gid; pid_t pid; @@ -1281,6 +1284,7 @@ enum { SHRPX_OPTID_REDIRECT_HTTPS_PORT, SHRPX_OPTID_REQUEST_HEADER_FIELD_BUFFER, SHRPX_OPTID_RESPONSE_HEADER_FIELD_BUFFER, + SHRPX_OPTID_RLIMIT_MEMLOCK, SHRPX_OPTID_RLIMIT_NOFILE, SHRPX_OPTID_SERVER_NAME, SHRPX_OPTID_SINGLE_PROCESS, From f6da0d342acc123ce0575146c4ab17f8b7e3146e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 24 Sep 2021 18:19:01 +0900 Subject: [PATCH 060/142] nghttpx: Fix crash if no keying materials are specified in file --- src/shrpx_config.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 21a2395c..a5689219 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -290,6 +290,13 @@ read_quic_secret_file(const StringRef &path) { return nullptr; } + if (kms.empty()) { + LOG(WARN) + << "frontend-quic-secret-file: no keying materials are present in file " + << path; + return nullptr; + } + return qkms; } #endif // ENABLE_HTTP3 From 407df2822e4b989c53c04b4f7902d2a9210ad506 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 26 Sep 2021 10:29:05 +0900 Subject: [PATCH 061/142] Remove check for UDP_SEGMENT Check for UDP_SEGMENT is for debian 10, but now that we have debian 11, remove the check because it breaks cross-build. --- configure.ac | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/configure.ac b/configure.ac index 7b7465d2..7fc90886 100644 --- a/configure.ac +++ b/configure.ac @@ -983,34 +983,6 @@ AC_CHECK_DECLS([initgroups], [], [], [[ #include ]]) -have_netinet_udp_h_udp_segment=no -AC_CHECK_DECL([UDP_SEGMENT], [have_netinet_udp_h_udp_segment=yes], - [have_netinet_udp_h_udp_segment=no], [[ - #include -]]) - -if test "x$have_netinet_udp_h_udp_segment" = "xno"; then - have_linux_udp_h_udp_segment=no - AC_MSG_CHECKING([whether UDP_SEGMENT is defined as 103 in linux/udp.h]) - AC_RUN_IFELSE([AC_LANG_PROGRAM( - [[ - #include - ]], - [[ - #if UDP_SEGMENT != 103 - exit(1) - #endif - ]])], - [have_linux_udp_h_udp_segment=yes - AC_MSG_RESULT([yes])], - [have_linux_udp_h_udp_segment=no - AC_MSG_RESULT([no])]) - - if test "x$have_linux_udp_h_udp_segment" = "xyes"; then - EXTRA_DEFS="$EXTRA_DEFS -DUDP_SEGMENT=103" - fi -fi - save_CFLAGS=$CFLAGS save_CXXFLAGS=$CXXFLAGS From 886dc93f182aa7ab6570c77013f35494889a2269 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 26 Sep 2021 15:32:44 +0900 Subject: [PATCH 062/142] nghttpx: Fail h3 connection attempt if no ALPN is negotiated --- src/shrpx_client_handler.cc | 10 +++++++++- src/shrpx_client_handler.h | 2 ++ src/shrpx_http3_upstream.cc | 8 ++++++++ src/shrpx_tls.cc | 2 +- 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 234c2b8c..8d02d740 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -517,7 +517,6 @@ void ClientHandler::setup_upstream_io_callback() { void ClientHandler::setup_http3_upstream( std::unique_ptr &&upstream) { upstream_ = std::move(upstream); - alpn_ = StringRef::from_lit("h3"); write_ = &ClientHandler::write_quic; auto config = get_config(); @@ -1599,4 +1598,13 @@ StringRef ClientHandler::get_alpn() const { return alpn_; } BlockAllocator &ClientHandler::get_block_allocator() { return balloc_; } +void ClientHandler::set_alpn_from_conn() { + const unsigned char *alpn; + unsigned int alpnlen; + + SSL_get0_alpn_selected(conn_.tls.ssl, &alpn, &alpnlen); + + alpn_ = make_string_ref(balloc_, StringRef{alpn, alpnlen}); +} + } // namespace shrpx diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index 0c013973..f20adc92 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -187,6 +187,8 @@ public: BlockAllocator &get_block_allocator(); + void set_alpn_from_conn(); + private: // Allocator to allocate memory for connection-wide objects. Make // sure that the allocations must be bounded, and not proportional diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index c67d8aae..29d3d9f8 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -479,6 +479,14 @@ int handshake_completed(ngtcp2_conn *conn, void *user_data) { } // namespace int Http3Upstream::handshake_completed() { + handler_->set_alpn_from_conn(); + + auto alpn = handler_->get_alpn(); + if (alpn.empty()) { + ULOG(ERROR, this) << "NO ALPN was negotiated"; + return -1; + } + std::array token; size_t tokenlen; diff --git a/src/shrpx_tls.cc b/src/shrpx_tls.cc index cda9bbc3..b490c107 100644 --- a/src/shrpx_tls.cc +++ b/src/shrpx_tls.cc @@ -719,7 +719,7 @@ int quic_alpn_select_proto_cb(SSL *ssl, const unsigned char **out, p += 1 + proto_len; } - return SSL_TLSEXT_ERR_NOACK; + return SSL_TLSEXT_ERR_ALERT_FATAL; } } // namespace # endif // OPENSSL_VERSION_NUMBER >= 0x10002000L From 19b4da64016f979b38a1d8eea127b153c2d22060 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 26 Sep 2021 15:59:57 +0900 Subject: [PATCH 063/142] nghttpx: Support h3-29 --- src/shrpx_tls.cc | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/shrpx_tls.cc b/src/shrpx_tls.cc index b490c107..a933b660 100644 --- a/src/shrpx_tls.cc +++ b/src/shrpx_tls.cc @@ -703,20 +703,26 @@ namespace { int quic_alpn_select_proto_cb(SSL *ssl, const unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, void *arg) { - for (auto p = in, end = in + inlen; p < end;) { - auto proto_id = p + 1; - auto proto_len = *p; + constexpr StringRef alpnlist[] = { + StringRef::from_lit("h3"), + StringRef::from_lit("h3-29"), + }; - if (proto_id + proto_len <= end && - util::streq_l("h3", StringRef{proto_id, proto_len})) { + for (auto &alpn : alpnlist) { + for (auto p = in, end = in + inlen; p < end;) { + auto proto_id = p + 1; + auto proto_len = *p; - *out = reinterpret_cast(proto_id); - *outlen = proto_len; + if (alpn.size() == proto_len && + memcmp(alpn.byte(), proto_id, alpn.size()) == 0) { + *out = proto_id; + *outlen = proto_len; - return SSL_TLSEXT_ERR_OK; + return SSL_TLSEXT_ERR_OK; + } + + p += 1 + proto_len; } - - p += 1 + proto_len; } return SSL_TLSEXT_ERR_ALERT_FATAL; From 17d5503bf244fe484cd0332158d6dac21c9487b5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 28 Sep 2021 18:07:31 +0900 Subject: [PATCH 064/142] Update doc --- doc/sources/nghttpx-howto.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index 90ab1b89..cfd9311e 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -539,6 +539,24 @@ encrypt Connection ID must be set with :option:`--frontend-quic-secret-file` and must provide the existing keys in order to keep the existing connections alive during reload. +The construction of Connection ID closely follows Block Cipher CID +Algorithm described in `QUIC-LB draft +`_. +A Connection ID that nghttpx generates is always 20 bytes long. It +uses first 2 bits as a configuration ID. The remaining bits in the +first byte are reserved and random. The next 2 bytes are server ID. +The next 6 bytes are used to route UDP datagram to a correct +``SO_REUSEPORT`` socket. The remaining bytes are randomly generated. +The server ID and the next 14 bytes are encrypted with AES-ECB. The +key is derived from the keying materials stored in a file specified by +:option:`--frontend-quic-secret-file`. The first 2 bits of keying +material in the file is used as a configuration ID. The remaining +bits and following 3 bytes are reserved and unused. The next 32 bytes +are used as an initial secret. The remaining 32 bytes are used as a +salt. The encryption key is generated by `HKDF +`_ with SHA256 and +these keying materials and ``connection id encryption key`` as info. + In order announce that HTTP/3 endpoint is available, you should specify alt-svc header field. For example, the following options send alt-svc header field in HTTP/1.1 and HTTP/2 response: From 318e0c8447e2399e7c1e95d8ae8d00118ccea5f7 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 28 Sep 2021 22:35:53 +0900 Subject: [PATCH 065/142] Guard msghdr_get_local_addr with ENABLE_HTTP3 macro --- src/util.cc | 2 ++ src/util.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/util.cc b/src/util.cc index 4491a07d..4db6325d 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1688,6 +1688,7 @@ int daemonize(int nochdir, int noclose) { #endif // !defined(__APPLE__) } +#ifdef ENABLE_HTTP3 int msghdr_get_local_addr(Address &dest, msghdr *msg, int family) { switch (family) { case AF_INET: @@ -1721,6 +1722,7 @@ int msghdr_get_local_addr(Address &dest, msghdr *msg, int family) { return -1; } +#endif // ENABLE_HTTP3 } // namespace util diff --git a/src/util.h b/src/util.h index 70f97f09..4d5cef37 100644 --- a/src/util.h +++ b/src/util.h @@ -916,7 +916,9 @@ std::mt19937 make_mt19937(); // daemon() using fork(). int daemonize(int nochdir, int noclose); +#ifdef ENABLE_HTTP3 int msghdr_get_local_addr(Address &dest, msghdr *msg, int family); +#endif // ENABLE_HTTP3 } // namespace util From df064fa2ba60d237b8f708eebe9ef6e762d06055 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 29 Sep 2021 19:33:16 +0900 Subject: [PATCH 066/142] nghttpx: Unload BPF objects on reload to avoid running out of memlock --- src/shrpx.cc | 4 ++++ src/shrpx_connection_handler.cc | 20 ++++++++++++++++++++ src/shrpx_connection_handler.h | 2 ++ src/shrpx_process.h | 1 + src/shrpx_worker.cc | 2 ++ src/shrpx_worker_process.cc | 3 +++ 6 files changed, 32 insertions(+) diff --git a/src/shrpx.cc b/src/shrpx.cc index ede06a3e..824eb739 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -3769,6 +3769,10 @@ void reload_config(WorkerProcess *wp) { #endif // ENABLE_HTTP3 )); +#ifdef ENABLE_HTTP3 + ipc_send(last_wp.get(), SHRPX_IPC_UNLOAD_BPF_OBJECT); +#endif // ENABLE_HTTP3 + if (!get_config()->pid_file.empty()) { save_pid(); } diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 4d32b6c5..3ff22740 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -1079,6 +1079,26 @@ ConnectionHandler::match_quic_lingering_worker_process_cid_prefix( std::vector &ConnectionHandler::get_quic_bpf_refs() { return quic_bpf_refs_; } + +void ConnectionHandler::unload_bpf_objects() { + std::array errbuf; + + LOG(NOTICE) << "Unloading BPF objects"; + + for (auto &ref : quic_bpf_refs_) { + if (ref.obj == nullptr) { + continue; + } + + if (bpf_object__unload(ref.obj) != 0) { + LOG(WARN) << "Failed to unload bpf object: " + << xsi_strerror(errno, errbuf.data(), errbuf.size()); + continue; + } + + ref.obj = nullptr; + } +} # endif // HAVE_LIBBPF void ConnectionHandler::set_quic_ipc_fd(int fd) { quic_ipc_fd_ = fd; } diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h index af391750..8bbf99c1 100644 --- a/src/shrpx_connection_handler.h +++ b/src/shrpx_connection_handler.h @@ -106,6 +106,7 @@ struct SerialEvent { #ifdef ENABLE_HTTP3 # ifdef HAVE_LIBBPF struct BPFRef { + bpf_object *obj; int reuseport_array; int cid_prefix_map; }; @@ -225,6 +226,7 @@ public: # ifdef HAVE_LIBBPF std::vector &get_quic_bpf_refs(); + void unload_bpf_objects(); # endif // HAVE_LIBBPF #endif // ENABLE_HTTP3 diff --git a/src/shrpx_process.h b/src/shrpx_process.h index d35461b9..b64e5c73 100644 --- a/src/shrpx_process.h +++ b/src/shrpx_process.h @@ -31,6 +31,7 @@ namespace shrpx { constexpr uint8_t SHRPX_IPC_REOPEN_LOG = 1; constexpr uint8_t SHRPX_IPC_GRACEFUL_SHUTDOWN = 2; +constexpr uint8_t SHRPX_IPC_UNLOAD_BPF_OBJECT = 3; } // namespace shrpx diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index b287ca50..255bca62 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -890,6 +890,8 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { auto &ref = quic_bpf_refs[faddr.index]; + ref.obj = obj; + auto reuseport_array = bpf_object__find_map_by_name(obj, "reuseport_array"); err = libbpf_get_error(reuseport_array); diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc index 006b8c7d..d71859a8 100644 --- a/src/shrpx_worker_process.cc +++ b/src/shrpx_worker_process.cc @@ -175,6 +175,9 @@ void ipc_readcb(struct ev_loop *loop, ev_io *w, int revents) { case SHRPX_IPC_REOPEN_LOG: reopen_log(conn_handler); break; + case SHRPX_IPC_UNLOAD_BPF_OBJECT: + conn_handler->unload_bpf_objects(); + break; } } } From d9c7631dcbf9651f7b9d7837a7c7ea1d82658ee6 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 29 Sep 2021 21:05:46 +0900 Subject: [PATCH 067/142] Fix compile error --- src/shrpx_worker_process.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc index d71859a8..8b55e386 100644 --- a/src/shrpx_worker_process.cc +++ b/src/shrpx_worker_process.cc @@ -175,9 +175,11 @@ void ipc_readcb(struct ev_loop *loop, ev_io *w, int revents) { case SHRPX_IPC_REOPEN_LOG: reopen_log(conn_handler); break; +#if defined(ENABLE_HTTP3) && defined(HAVE_LIBBPF) case SHRPX_IPC_UNLOAD_BPF_OBJECT: conn_handler->unload_bpf_objects(); break; +#endif // defined(ENABLE_HTTP3) && defined(HAVE_LIBBPF) } } } From 0266c458a34d5ef7455cffe3bf34ec9b25db99c2 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 29 Sep 2021 22:12:10 +0900 Subject: [PATCH 068/142] nghttpx: Add --max-worker-processes option --- gennghttpxfun.py | 1 + src/shrpx.cc | 31 +++++++++++++++++++++++++++++++ src/shrpx_config.cc | 5 +++++ src/shrpx_config.h | 7 ++++++- 4 files changed, 43 insertions(+), 1 deletion(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index a38fbca1..2aa76ad3 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -195,6 +195,7 @@ OPTIONS = [ "frontend-quic-server-id", "frontend-quic-secret-file", "rlimit-memlock", + "max-worker-processes", ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index 824eb739..80ee1d32 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -300,6 +300,17 @@ void worker_process_remove(const WorkerProcess *wp) { } } // namespace +namespace { +void worker_process_adjust_limit() { + auto config = get_config(); + + if (config->max_worker_processes && + worker_processes.size() > config->max_worker_processes) { + worker_processes.pop_front(); + } +} +} // namespace + namespace { void worker_process_remove_all() { std::deque>().swap(worker_processes); @@ -3201,6 +3212,19 @@ Process: process. nghttpx still spawns additional process if neverbleed is used. In the single process mode, the signal handling feature is disabled. + --max-worker-processes= + The maximum number of worker processes. nghttpx spawns + new worker process when it reloads its configuration. + The previous worker process enters graceful termination + period and will terminate when it finishes handling the + existing connections. However, if reloading + configurations happen very frequently, the worker + processes might be piled up if they take a bit long time + to finish the existing connections. With this option, + if the number of worker processes exceeds the given + value, the oldest worker process is terminated + immediately. Specifying 0 means no limit and it is the + default behaviour. Scripting: --mruby-file= @@ -3773,6 +3797,8 @@ void reload_config(WorkerProcess *wp) { ipc_send(last_wp.get(), SHRPX_IPC_UNLOAD_BPF_OBJECT); #endif // ENABLE_HTTP3 + worker_process_adjust_limit(); + if (!get_config()->pid_file.empty()) { save_pid(); } @@ -4101,6 +4127,7 @@ int main(int argc, char **argv) { {SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE.c_str(), required_argument, &flag, 186}, {SHRPX_OPT_RLIMIT_MEMLOCK.c_str(), required_argument, &flag, 187}, + {SHRPX_OPT_MAX_WORKER_PROCESSES.c_str(), required_argument, &flag, 188}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -4992,6 +5019,10 @@ int main(int argc, char **argv) { // --rlimit-memlock cmdcfgs.emplace_back(SHRPX_OPT_RLIMIT_MEMLOCK, StringRef{optarg}); break; + case 188: + // --max-worker-processes + cmdcfgs.emplace_back(SHRPX_OPT_MAX_WORKER_PROCESSES, StringRef{optarg}); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index a5689219..f4a6e67a 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2248,6 +2248,9 @@ int option_lookup_token(const char *name, size_t namelen) { } break; case 's': + if (util::strieq_l("max-worker-processe", name, 19)) { + return SHRPX_OPTID_MAX_WORKER_PROCESSES; + } if (util::strieq_l("tls13-client-cipher", name, 19)) { return SHRPX_OPTID_TLS13_CLIENT_CIPHERS; } @@ -4139,6 +4142,8 @@ int parse_config(Config *config, int optid, const StringRef &opt, return 0; } + case SHRPX_OPTID_MAX_WORKER_PROCESSES: + return parse_uint(&config->max_worker_processes, opt, optarg); case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index d9c6fcef..b70c9457 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -396,6 +396,8 @@ constexpr auto SHRPX_OPT_FRONTEND_QUIC_SERVER_ID = constexpr auto SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE = StringRef::from_lit("frontend-quic-secret-file"); constexpr auto SHRPX_OPT_RLIMIT_MEMLOCK = StringRef::from_lit("rlimit-memlock"); +constexpr auto SHRPX_OPT_MAX_WORKER_PROCESSES = + StringRef::from_lit("max-worker-processes"); constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -1075,7 +1077,8 @@ struct Config { single_process{false}, single_thread{false}, ignore_per_pattern_mruby_error{false}, - ev_loop_flags{0} { + ev_loop_flags{0}, + max_worker_processes{0} { } ~Config(); @@ -1129,6 +1132,7 @@ struct Config { bool ignore_per_pattern_mruby_error; // flags passed to ev_default_loop() and ev_loop_new() int ev_loop_flags; + size_t max_worker_processes; }; const Config *get_config(); @@ -1255,6 +1259,7 @@ enum { SHRPX_OPTID_MAX_HEADER_FIELDS, SHRPX_OPTID_MAX_REQUEST_HEADER_FIELDS, SHRPX_OPTID_MAX_RESPONSE_HEADER_FIELDS, + SHRPX_OPTID_MAX_WORKER_PROCESSES, SHRPX_OPTID_MRUBY_FILE, SHRPX_OPTID_NO_ADD_X_FORWARDED_PROTO, SHRPX_OPTID_NO_HOST_REWRITE, From 3e25ee818178eea37950f0825242f1c1465e55e8 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 29 Sep 2021 22:16:39 +0900 Subject: [PATCH 069/142] 256k memlock is not enough when reloading happens --- docker/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.rst b/docker/README.rst index 2c32565a..24402165 100644 --- a/docker/README.rst +++ b/docker/README.rst @@ -22,4 +22,4 @@ certificate in server.key and server.crt respectively : $ docker run --rm -it -v $PWD:/shared --net=host --privileged \ nghttp2 nghttpx \ /shared/server.key /shared/server.crt \ - -f'*,443;quic' --rlimit-memlock 262144 + -f'*,443;quic' --rlimit-memlock 524288 From 87bdc2166768bde482644cdf0ac139d0869fc45a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 2 Oct 2021 18:55:51 +0900 Subject: [PATCH 070/142] nghttpx: Add --worker-process-grace-shutdown-period option --- gennghttpxfun.py | 1 + src/shrpx.cc | 115 +++++++++++++++++++++++++++++++++++++++++--- src/shrpx_config.cc | 8 +++ src/shrpx_config.h | 7 ++- 4 files changed, 122 insertions(+), 9 deletions(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 2aa76ad3..af8d60a5 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -196,6 +196,7 @@ OPTIONS = [ "frontend-quic-secret-file", "rlimit-memlock", "max-worker-processes", + "worker-process-grace-shutdown-period", ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index 80ee1d32..a4faaa69 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -202,7 +202,8 @@ struct WorkerProcess { ) : loop(loop), worker_pid(worker_pid), - ipc_fd(ipc_fd) + ipc_fd(ipc_fd), + termination_deadline(0.) #ifdef ENABLE_HTTP3 , quic_ipc_fd(quic_ipc_fd), @@ -264,6 +265,7 @@ struct WorkerProcess { struct ev_loop *loop; pid_t worker_pid; int ipc_fd; + ev_tstamp termination_deadline; #ifdef ENABLE_HTTP3 int quic_ipc_fd; std::vector> cid_prefixes; @@ -278,6 +280,74 @@ namespace { std::deque> worker_processes; } // namespace +namespace { +ev_timer worker_process_grace_period_timer; +} // namespace + +namespace { +void worker_process_grace_period_timercb(struct ev_loop *loop, ev_timer *w, + int revents) { + auto now = ev_now(loop); + ev_tstamp next_repeat = 0.; + + for (auto it = std::begin(worker_processes); + it != std::end(worker_processes);) { + auto &wp = *it; + if (!(wp->termination_deadline > 0.)) { + ++it; + + continue; + } + + auto d = wp->termination_deadline - now; + if (d > 0) { + if (!(next_repeat > 0.) || d < next_repeat) { + next_repeat = d; + } + + ++it; + + continue; + } + + LOG(NOTICE) << "Deleting worker process pid=" << wp->worker_pid + << " because its grace shutdown period is over"; + + it = worker_processes.erase(it); + } + + if (next_repeat > 0.) { + w->repeat = next_repeat; + ev_timer_again(loop, w); + + return; + } + + ev_timer_stop(loop, w); +} +} // namespace + +namespace { +void worker_process_set_termination_deadline(WorkerProcess *wp, + struct ev_loop *loop) { + auto config = get_config(); + + if (!(config->worker_process_grace_shutdown_period > 0.)) { + return; + } + + wp->termination_deadline = + ev_now(loop) + config->worker_process_grace_shutdown_period; + + if (!ev_is_active(&worker_process_grace_period_timer)) { + worker_process_grace_period_timer.repeat = + config->worker_process_grace_shutdown_period; + + ev_timer_again(loop, &worker_process_grace_period_timer); + } +} +} // namespace + namespace { void worker_process_add(std::unique_ptr wp) { worker_processes.push_back(std::move(wp)); @@ -285,7 +355,7 @@ void worker_process_add(std::unique_ptr wp) { } // namespace namespace { -void worker_process_remove(const WorkerProcess *wp) { +void worker_process_remove(const WorkerProcess *wp, struct ev_loop *loop) { for (auto it = std::begin(worker_processes); it != std::end(worker_processes); ++it) { auto &s = *it; @@ -295,6 +365,11 @@ void worker_process_remove(const WorkerProcess *wp) { } worker_processes.erase(it); + + if (worker_processes.empty()) { + ev_timer_stop(loop, &worker_process_grace_period_timer); + } + break; } } @@ -312,22 +387,24 @@ void worker_process_adjust_limit() { } // namespace namespace { -void worker_process_remove_all() { +void worker_process_remove_all(struct ev_loop *loop) { std::deque>().swap(worker_processes); + + ev_timer_stop(loop, &worker_process_grace_period_timer); } } // namespace namespace { // Send signal |signum| to all worker processes, and clears // worker_processes. -void worker_process_kill(int signum) { +void worker_process_kill(int signum, struct ev_loop *loop) { for (auto &s : worker_processes) { if (s->worker_pid == -1) { continue; } kill(s->worker_pid, signum); } - worker_process_remove_all(); + worker_process_remove_all(loop); } } // namespace @@ -646,13 +723,14 @@ void signal_cb(struct ev_loop *loop, ev_signal *w, int revents) { close(addr.fd); } ipc_send(wp, SHRPX_IPC_GRACEFUL_SHUTDOWN); + worker_process_set_termination_deadline(wp, loop); return; } case RELOAD_SIGNAL: reload_config(wp); return; default: - worker_process_kill(w->signum); + worker_process_kill(w->signum, loop); ev_break(loop); return; } @@ -667,7 +745,7 @@ void worker_process_child_cb(struct ev_loop *loop, ev_child *w, int revents) { auto pid = wp->worker_pid; - worker_process_remove(wp); + worker_process_remove(wp, loop); if (worker_process_last_pid() == pid) { ev_break(loop); @@ -1482,7 +1560,7 @@ pid_t fork_worker_process( // Remove all WorkerProcesses to stop any registered watcher on // default loop. - worker_process_remove_all(); + worker_process_remove_all(EV_DEFAULT); close_unused_inherited_addr(iaddrs); @@ -1657,6 +1735,9 @@ int event_loop() { return -1; } + ev_timer_init(&worker_process_grace_period_timer, + worker_process_grace_period_timercb, 0., 0.); + worker_process_add(std::make_unique(loop, pid, ipc_fd #ifdef ENABLE_HTTP3 , @@ -1683,6 +1764,8 @@ int event_loop() { ev_run(loop, 0); + ev_timer_stop(loop, &worker_process_grace_period_timer); + return 0; } } // namespace @@ -3225,6 +3308,14 @@ Process: value, the oldest worker process is terminated immediately. Specifying 0 means no limit and it is the default behaviour. + --worker-process-grace-shutdown-period= + Maximum period for a worker process to terminate + gracefully. When a worker process enters in graceful + shutdown period (e.g., when nghttpx reloads its + configuration) and it does not finish handling the + existing connections in the given period of time, it is + immediately terminated. Specifying 0 means no limit and + it is the default behaviour. Scripting: --mruby-file= @@ -3783,6 +3874,7 @@ void reload_config(WorkerProcess *wp) { // Send last worker process a graceful shutdown notice auto &last_wp = worker_processes.back(); ipc_send(last_wp.get(), SHRPX_IPC_GRACEFUL_SHUTDOWN); + worker_process_set_termination_deadline(last_wp.get(), loop); // We no longer use signals for this worker. last_wp->shutdown_signal_watchers(); @@ -4128,6 +4220,8 @@ int main(int argc, char **argv) { 186}, {SHRPX_OPT_RLIMIT_MEMLOCK.c_str(), required_argument, &flag, 187}, {SHRPX_OPT_MAX_WORKER_PROCESSES.c_str(), required_argument, &flag, 188}, + {SHRPX_OPT_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD.c_str(), + required_argument, &flag, 189}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -5023,6 +5117,11 @@ int main(int argc, char **argv) { // --max-worker-processes cmdcfgs.emplace_back(SHRPX_OPT_MAX_WORKER_PROCESSES, StringRef{optarg}); break; + case 189: + // --worker-process-grace-shutdown-period + cmdcfgs.emplace_back(SHRPX_OPT_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD, + StringRef{optarg}); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index f4a6e67a..ac260b55 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2666,6 +2666,11 @@ int option_lookup_token(const char *name, size_t namelen) { break; case 36: switch (name[35]) { + case 'd': + if (util::strieq_l("worker-process-grace-shutdown-perio", name, 35)) { + return SHRPX_OPTID_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD; + } + break; case 'e': if (util::strieq_l("backend-http2-connection-window-siz", name, 35)) { return SHRPX_OPTID_BACKEND_HTTP2_CONNECTION_WINDOW_SIZE; @@ -4144,6 +4149,9 @@ int parse_config(Config *config, int optid, const StringRef &opt, } case SHRPX_OPTID_MAX_WORKER_PROCESSES: return parse_uint(&config->max_worker_processes, opt, optarg); + case SHRPX_OPTID_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD: + return parse_duration(&config->worker_process_grace_shutdown_period, opt, + optarg); case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index b70c9457..8ae8c26c 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -398,6 +398,8 @@ constexpr auto SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE = constexpr auto SHRPX_OPT_RLIMIT_MEMLOCK = StringRef::from_lit("rlimit-memlock"); constexpr auto SHRPX_OPT_MAX_WORKER_PROCESSES = StringRef::from_lit("max-worker-processes"); +constexpr auto SHRPX_OPT_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD = + StringRef::from_lit("worker-process-grace-shutdown-period"); constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -1078,7 +1080,8 @@ struct Config { single_thread{false}, ignore_per_pattern_mruby_error{false}, ev_loop_flags{0}, - max_worker_processes{0} { + max_worker_processes{0}, + worker_process_grace_shutdown_period{0.} { } ~Config(); @@ -1133,6 +1136,7 @@ struct Config { // flags passed to ev_default_loop() and ev_loop_new() int ev_loop_flags; size_t max_worker_processes; + ev_tstamp worker_process_grace_shutdown_period; }; const Config *get_config(); @@ -1330,6 +1334,7 @@ enum { SHRPX_OPTID_VERIFY_CLIENT_CACERT, SHRPX_OPTID_VERIFY_CLIENT_TOLERATE_EXPIRED, SHRPX_OPTID_WORKER_FRONTEND_CONNECTIONS, + SHRPX_OPTID_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD, SHRPX_OPTID_WORKER_READ_BURST, SHRPX_OPTID_WORKER_READ_RATE, SHRPX_OPTID_WORKER_WRITE_BURST, From abee658a60806b5c96d5563ae9f5a06f90610725 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 2 Oct 2021 19:22:53 +0900 Subject: [PATCH 071/142] nghttpx: Make sure that ngtcp2_conn_update_pkt_tx_time is called --- src/shrpx_http3_upstream.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 29d3d9f8..ea73d0ee 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -781,10 +781,11 @@ int Http3Upstream::write_streams() { prev_ps.path.local.addr, prev_ps.path.local.addrlen, buf.data(), bufpos - buf.data(), max_udp_payload_size); - ngtcp2_conn_update_pkt_tx_time(conn_, ts); reset_idle_timer(); } + ngtcp2_conn_update_pkt_tx_time(conn_, ts); + handler_->get_connection()->wlimit.stopw(); return 0; From 086b85b8f971b372dc061ad52966a2d92804369a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 4 Oct 2021 22:39:31 +0900 Subject: [PATCH 072/142] nghttpx: Unload BPF program after setting up all QUIC listeners --- src/shrpx.cc | 4 ---- src/shrpx_process.h | 1 - src/shrpx_worker_process.cc | 7 ++----- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index a4faaa69..f77db9fa 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -3885,10 +3885,6 @@ void reload_config(WorkerProcess *wp) { #endif // ENABLE_HTTP3 )); -#ifdef ENABLE_HTTP3 - ipc_send(last_wp.get(), SHRPX_IPC_UNLOAD_BPF_OBJECT); -#endif // ENABLE_HTTP3 - worker_process_adjust_limit(); if (!get_config()->pid_file.empty()) { diff --git a/src/shrpx_process.h b/src/shrpx_process.h index b64e5c73..d35461b9 100644 --- a/src/shrpx_process.h +++ b/src/shrpx_process.h @@ -31,7 +31,6 @@ namespace shrpx { constexpr uint8_t SHRPX_IPC_REOPEN_LOG = 1; constexpr uint8_t SHRPX_IPC_GRACEFUL_SHUTDOWN = 2; -constexpr uint8_t SHRPX_IPC_UNLOAD_BPF_OBJECT = 3; } // namespace shrpx diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc index 8b55e386..85b94c47 100644 --- a/src/shrpx_worker_process.cc +++ b/src/shrpx_worker_process.cc @@ -175,11 +175,6 @@ void ipc_readcb(struct ev_loop *loop, ev_io *w, int revents) { case SHRPX_IPC_REOPEN_LOG: reopen_log(conn_handler); break; -#if defined(ENABLE_HTTP3) && defined(HAVE_LIBBPF) - case SHRPX_IPC_UNLOAD_BPF_OBJECT: - conn_handler->unload_bpf_objects(); - break; -#endif // defined(ENABLE_HTTP3) && defined(HAVE_LIBBPF) } } } @@ -608,6 +603,8 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { #endif // !NOTHREADS } + conn_handler->unload_bpf_objects(); + drop_privileges( #ifdef HAVE_NEVERBLEED nb.get() From f4290c649761d42d17a5b09e7280d5b4c6229dd0 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 4 Oct 2021 23:23:03 +0900 Subject: [PATCH 073/142] Fix compile error --- src/shrpx_worker_process.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc index 85b94c47..517df302 100644 --- a/src/shrpx_worker_process.cc +++ b/src/shrpx_worker_process.cc @@ -603,7 +603,9 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { #endif // !NOTHREADS } +#if defined(ENABLE_HTTP3) && defined(HAVE_LIBBPF) conn_handler->unload_bpf_objects(); +#endif // defined(ENABLE_HTTP3) && defined(HAVE_LIBBPF) drop_privileges( #ifdef HAVE_NEVERBLEED From cb6aea9aa932b76fd0a57d7ace711c66c5d280f5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 6 Oct 2021 21:28:00 +0900 Subject: [PATCH 074/142] Compile with -DNDEBUG --- lib/nghttp2_map.c | 1 + src/h2load_quic.cc | 1 + src/shrpx_http3_upstream.cc | 1 + src/shrpx_mruby.cc | 1 + src/shrpx_worker.cc | 1 + 5 files changed, 5 insertions(+) diff --git a/lib/nghttp2_map.c b/lib/nghttp2_map.c index 5aab90b4..e5db168c 100644 --- a/lib/nghttp2_map.c +++ b/lib/nghttp2_map.c @@ -189,6 +189,7 @@ static int map_resize(nghttp2_map *map, uint32_t new_tablelen, nghttp2_map_bucket *new_table; nghttp2_map_bucket *bkt; int rv; + (void)rv; new_table = nghttp2_mem_calloc(map->mem, new_tablelen, sizeof(nghttp2_map_bucket)); diff --git a/src/h2load_quic.cc b/src/h2load_quic.cc index ff63ff54..b77bf6ce 100644 --- a/src/h2load_quic.cc +++ b/src/h2load_quic.cc @@ -451,6 +451,7 @@ void Client::quic_close_connection() { break; default: assert(0); + abort(); } if (nwrite < 0) { diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index ea73d0ee..aa0d1f8c 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -1842,6 +1842,7 @@ int Http3Upstream::http_acked_stream_data(Downstream *downstream, auto body = downstream->get_response_buf(); auto drained = body->drain_mark(datalen); + (void)drained; assert(datalen == drained); diff --git a/src/shrpx_mruby.cc b/src/shrpx_mruby.cc index 5f7c42ad..b5c6ed3c 100644 --- a/src/shrpx_mruby.cc +++ b/src/shrpx_mruby.cc @@ -75,6 +75,7 @@ int MRubyContext::run_app(Downstream *downstream, int phase) { break; default: assert(0); + abort(); } auto res = mrb_funcall(mrb_, app_, method, 1, env_); diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 255bca62..a0107cc9 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -1036,6 +1036,7 @@ const UpstreamAddr *Worker::find_quic_upstream_addr(const Address &local_addr) { break; default: assert(0); + abort(); } std::array hostport_buf; From 474a6db00c16a68463d39910c2a65a71041da0cb Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 6 Oct 2021 21:28:17 +0900 Subject: [PATCH 075/142] Compile with gcc --- examples/deflate.c | 2 +- tests/failmalloc.c | 2 +- tests/main.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/deflate.c b/examples/deflate.c index 63f3ea15..df1cb920 100644 --- a/examples/deflate.c +++ b/examples/deflate.c @@ -44,7 +44,7 @@ static void deflate(nghttp2_hd_deflater *deflater, static int inflate_header_block(nghttp2_hd_inflater *inflater, uint8_t *in, size_t inlen, int final); -int main() { +int main(void) { int rv; nghttp2_hd_deflater *deflater; nghttp2_hd_inflater *inflater; diff --git a/tests/failmalloc.c b/tests/failmalloc.c index 4bf83ca5..6294cfff 100644 --- a/tests/failmalloc.c +++ b/tests/failmalloc.c @@ -37,7 +37,7 @@ static int init_suite1(void) { return 0; } static int clean_suite1(void) { return 0; } -int main() { +int main(void) { CU_pSuite pSuite = NULL; unsigned int num_tests_failed; diff --git a/tests/main.c b/tests/main.c index 25cbbfd7..dc41c7c7 100644 --- a/tests/main.c +++ b/tests/main.c @@ -47,7 +47,7 @@ static int init_suite1(void) { return 0; } static int clean_suite1(void) { return 0; } -int main() { +int main(void) { CU_pSuite pSuite = NULL; unsigned int num_tests_failed; From a48e9d3d80c2c919286eac104d8df97e550bd7ca Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 6 Oct 2021 21:29:15 +0900 Subject: [PATCH 076/142] Add bpf to clang-format --- Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index 99f03237..79174a5a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -62,4 +62,4 @@ clang-format: test -z $${CLANGFORMAT} && CLANGFORMAT="clang-format"; \ $${CLANGFORMAT} -i lib/*.{c,h} lib/includes/nghttp2/*.h \ src/*.{c,cc,h} src/includes/nghttp2/*.h examples/*.{c,cc} \ - tests/*.{c,h} + tests/*.{c,h} bpf/*.c From d7af5924ff749df7887144f395f954338da0cbdd Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 10 Oct 2021 16:05:48 +0900 Subject: [PATCH 077/142] nghttpx: Extend QUIC server ID to 4 bytes --- doc/sources/nghttpx-howto.rst | 6 +++--- src/shrpx.cc | 4 ++-- src/shrpx_quic.h | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index cfd9311e..48faddd7 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -544,10 +544,10 @@ Algorithm described in `QUIC-LB draft `_. A Connection ID that nghttpx generates is always 20 bytes long. It uses first 2 bits as a configuration ID. The remaining bits in the -first byte are reserved and random. The next 2 bytes are server ID. -The next 6 bytes are used to route UDP datagram to a correct +first byte are reserved and random. The next 4 bytes are server ID. +The next 4 bytes are used to route UDP datagram to a correct ``SO_REUSEPORT`` socket. The remaining bytes are randomly generated. -The server ID and the next 14 bytes are encrypted with AES-ECB. The +The server ID and the next 12 bytes are encrypted with AES-ECB. The key is derived from the keying materials stored in a file specified by :option:`--frontend-quic-secret-file`. The first 2 bits of keying material in the file is used as a configuration ID. The remaining diff --git a/src/shrpx.cc b/src/shrpx.cc index f77db9fa..1cdc2c59 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -3395,8 +3395,8 @@ HTTP/3 and QUIC: Specify server ID encoded in Connection ID to identify this particular server instance. Connection ID is encrypted and this part is not visible in public. It - must be 2 bytes long and must be encoded in hex string - (which is 4 bytes long). If this option is omitted, a + must be 4 bytes long and must be encoded in hex string + (which is 8 bytes long). If this option is omitted, a random server ID is generated on startup and configuration reload. --no-quic-bpf diff --git a/src/shrpx_quic.h b/src/shrpx_quic.h index e66018c4..8f400254 100644 --- a/src/shrpx_quic.h +++ b/src/shrpx_quic.h @@ -64,7 +64,7 @@ struct QUICKeyingMaterials; struct QUICKeyingMaterial; constexpr size_t SHRPX_QUIC_SCIDLEN = 20; -constexpr size_t SHRPX_QUIC_SERVER_IDLEN = 2; +constexpr size_t SHRPX_QUIC_SERVER_IDLEN = 4; // SHRPX_QUIC_CID_PREFIXLEN includes SHRPX_QUIC_SERVER_IDLEN. constexpr size_t SHRPX_QUIC_CID_PREFIXLEN = 8; constexpr size_t SHRPX_QUIC_CID_PREFIX_OFFSET = 1; From 7471fa627d4992112910b0d6f068ea966992095f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 10 Oct 2021 16:08:49 +0900 Subject: [PATCH 078/142] Update ax_python_devel.m4 --- m4/ax_python_devel.m4 | 85 ++++++++++++++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 22 deletions(-) diff --git a/m4/ax_python_devel.m4 b/m4/ax_python_devel.m4 index 44dbd83e..9d4eecf7 100644 --- a/m4/ax_python_devel.m4 +++ b/m4/ax_python_devel.m4 @@ -67,7 +67,7 @@ # modified version of the Autoconf Macro, you may extend this special # exception to the GPL to apply to your modified version as well. -#serial 21 +#serial 23 AU_ALIAS([AC_PYTHON_DEVEL], [AX_PYTHON_DEVEL]) AC_DEFUN([AX_PYTHON_DEVEL],[ @@ -135,16 +135,25 @@ variable to configure. See ``configure --help'' for reference. # # Check if you have distutils, else fail # - AC_MSG_CHECKING([for the distutils Python package]) - ac_distutils_result=`$PYTHON -c "import distutils" 2>&1` + AC_MSG_CHECKING([for the sysconfig Python package]) + ac_sysconfig_result=`$PYTHON -c "import sysconfig" 2>&1` if test $? -eq 0; then AC_MSG_RESULT([yes]) + IMPORT_SYSCONFIG="import sysconfig" else AC_MSG_RESULT([no]) - AC_MSG_ERROR([cannot import Python module "distutils". + + AC_MSG_CHECKING([for the distutils Python package]) + ac_sysconfig_result=`$PYTHON -c "from distutils import sysconfig" 2>&1` + if test $? -eq 0; then + AC_MSG_RESULT([yes]) + IMPORT_SYSCONFIG="from distutils import sysconfig" + else + AC_MSG_ERROR([cannot import Python module "distutils". Please check your Python installation. The error was: -$ac_distutils_result]) - PYTHON_VERSION="" +$ac_sysconfig_result]) + PYTHON_VERSION="" + fi fi # @@ -152,10 +161,19 @@ $ac_distutils_result]) # AC_MSG_CHECKING([for Python include path]) if test -z "$PYTHON_CPPFLAGS"; then - python_path=`$PYTHON -c "import distutils.sysconfig; \ - print (distutils.sysconfig.get_python_inc ());"` - plat_python_path=`$PYTHON -c "import distutils.sysconfig; \ - print (distutils.sysconfig.get_python_inc (plat_specific=1));"` + if test "$IMPORT_SYSCONFIG" = "import sysconfig"; then + # sysconfig module has different functions + python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \ + print (sysconfig.get_path ('include'));"` + plat_python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \ + print (sysconfig.get_path ('platinclude'));"` + else + # old distutils way + python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \ + print (sysconfig.get_python_inc ());"` + plat_python_path=`$PYTHON -c "$IMPORT_SYSCONFIG; \ + print (sysconfig.get_python_inc (plat_specific=1));"` + fi if test -n "${python_path}"; then if test "${plat_python_path}" != "${python_path}"; then python_path="-I$python_path -I$plat_python_path" @@ -179,7 +197,7 @@ $ac_distutils_result]) # join all versioning strings, on some systems # major/minor numbers could be in different list elements -from distutils.sysconfig import * +from sysconfig import * e = get_config_var('VERSION') if e is not None: print(e) @@ -202,8 +220,8 @@ EOD` ac_python_libdir=`cat< Date: Sun, 10 Oct 2021 16:52:05 +0900 Subject: [PATCH 079/142] Workaround broken version check in AX_PYTHON_DEVEL --- configure.ac | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 7fc90886..5e19e1ce 100644 --- a/configure.ac +++ b/configure.ac @@ -220,6 +220,11 @@ PKG_PROG_PKG_CONFIG([0.20]) AM_PATH_PYTHON([3.8],, [:]) +if test "x$request_python_bindings" = "xyes" && + test "x$PYTHON" = "x:"; then + AC_MSG_ERROR([python was requested (enable-python-bindings) but not found]) +fi + if [test "x$request_lib_only" = "xyes"]; then request_app=no request_hpack_tools=no @@ -227,8 +232,10 @@ if [test "x$request_lib_only" = "xyes"]; then request_python_bindings=no fi -if [test "x$request_python_bindings" != "xno"]; then - AX_PYTHON_DEVEL([>= '3.8']) +if test "x$request_python_bindings" != "xno" && + test "x$PYTHON" != "x:"; then + # version check is broken + AX_PYTHON_DEVEL() fi if test "x${cython_path}" = "x"; then From 69c41871002c2ab9ac3fd4b7d720b2dda1197ec6 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 10 Oct 2021 17:24:02 +0900 Subject: [PATCH 080/142] nghttpx: Add --frontend-quic-initial-rtt option --- gennghttpxfun.py | 1 + src/shrpx.cc | 16 ++++++++++++++++ src/shrpx_config.cc | 18 ++++++++++++++++++ src/shrpx_config.h | 4 ++++ src/shrpx_http3_upstream.cc | 1 + 5 files changed, 40 insertions(+) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index af8d60a5..6f88456e 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -197,6 +197,7 @@ OPTIONS = [ "rlimit-memlock", "max-worker-processes", "worker-process-grace-shutdown-period", + "frontend-quic-initial-rtt", ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index 1cdc2c59..a36c168b 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1958,6 +1958,8 @@ void fill_default_config(Config *config) { assert(0); abort(); } + + upstreamconf.initial_rtt = NGTCP2_DEFAULT_INITIAL_RTT; } auto &http3conf = config->http3; @@ -3399,6 +3401,13 @@ HTTP/3 and QUIC: (which is 8 bytes long). If this option is omitted, a random server ID is generated on startup and configuration reload. + --frontend-quic-initial-rtt= + Specify the initial RTT of the frontend QUIC connection. + Default: )" + << util::duration_str( + static_cast(config->quic.upstream.initial_rtt) / + NGTCP2_SECONDS) + << R"( --no-quic-bpf Disable eBPF. --frontend-http3-window-size= @@ -4218,6 +4227,8 @@ int main(int argc, char **argv) { {SHRPX_OPT_MAX_WORKER_PROCESSES.c_str(), required_argument, &flag, 188}, {SHRPX_OPT_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD.c_str(), required_argument, &flag, 189}, + {SHRPX_OPT_FRONTEND_QUIC_INITIAL_RTT.c_str(), required_argument, &flag, + 190}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -5118,6 +5129,11 @@ int main(int argc, char **argv) { cmdcfgs.emplace_back(SHRPX_OPT_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD, StringRef{optarg}); break; + case 190: + // --frontend-quic-initial-rtt + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_INITIAL_RTT, + StringRef{optarg}); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index ac260b55..25576c70 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2440,6 +2440,11 @@ int option_lookup_token(const char *name, size_t namelen) { return SHRPX_OPTID_MAX_REQUEST_HEADER_FIELDS; } break; + case 't': + if (util::strieq_l("frontend-quic-initial-rt", name, 24)) { + return SHRPX_OPTID_FRONTEND_QUIC_INITIAL_RTT; + } + break; } break; case 26: @@ -4152,6 +4157,19 @@ int parse_config(Config *config, int optid, const StringRef &opt, case SHRPX_OPTID_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD: return parse_duration(&config->worker_process_grace_shutdown_period, opt, optarg); + case SHRPX_OPTID_FRONTEND_QUIC_INITIAL_RTT: { +#ifdef ENABLE_HTTP3 + ev_tstamp d; + if (parse_duration(&d, opt, optarg) != 0) { + return -1; + } + + config->quic.upstream.initial_rtt = + static_cast(d * NGTCP2_SECONDS); +#endif // ENABLE_HTTP3 + + return 0; + } case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 8ae8c26c..1b19d2f5 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -400,6 +400,8 @@ constexpr auto SHRPX_OPT_MAX_WORKER_PROCESSES = StringRef::from_lit("max-worker-processes"); constexpr auto SHRPX_OPT_WORKER_PROCESS_GRACE_SHUTDOWN_PERIOD = StringRef::from_lit("worker-process-grace-shutdown-period"); +constexpr auto SHRPX_OPT_FRONTEND_QUIC_INITIAL_RTT = + StringRef::from_lit("frontend-quic-initial-rtt"); constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -780,6 +782,7 @@ struct QUICConfig { bool require_token; std::array server_id; StringRef secret_file; + ngtcp2_duration initial_rtt; } upstream; struct { StringRef prog_file; @@ -1242,6 +1245,7 @@ enum { SHRPX_OPTID_FRONTEND_QUIC_DEBUG_LOG, SHRPX_OPTID_FRONTEND_QUIC_EARLY_DATA, SHRPX_OPTID_FRONTEND_QUIC_IDLE_TIMEOUT, + SHRPX_OPTID_FRONTEND_QUIC_INITIAL_RTT, SHRPX_OPTID_FRONTEND_QUIC_QLOG_DIR, SHRPX_OPTID_FRONTEND_QUIC_REQUIRE_TOKEN, SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE, diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index aa0d1f8c..f59d0fff 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -592,6 +592,7 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, } settings.initial_ts = quic_timestamp(); + settings.initial_rtt = quicconf.upstream.initial_rtt; settings.cc_algo = quicconf.upstream.congestion_controller; settings.max_window = http3conf.upstream.max_connection_window_size; settings.max_stream_window = http3conf.upstream.max_window_size; From fa7a916ef3b6e518d7a65e6256a9ebab1bab83ce Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 10 Oct 2021 17:34:58 +0900 Subject: [PATCH 081/142] nghttpx: Store initial_rtt in ev_tstamp for consistency --- src/shrpx.cc | 8 +++----- src/shrpx_config.cc | 8 +------- src/shrpx_config.h | 2 +- src/shrpx_http3_upstream.cc | 3 ++- 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index a36c168b..c252f61f 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1959,7 +1959,8 @@ void fill_default_config(Config *config) { abort(); } - upstreamconf.initial_rtt = NGTCP2_DEFAULT_INITIAL_RTT; + upstreamconf.initial_rtt = + static_cast(NGTCP2_DEFAULT_INITIAL_RTT) / NGTCP2_SECONDS; } auto &http3conf = config->http3; @@ -3404,10 +3405,7 @@ HTTP/3 and QUIC: --frontend-quic-initial-rtt= Specify the initial RTT of the frontend QUIC connection. Default: )" - << util::duration_str( - static_cast(config->quic.upstream.initial_rtt) / - NGTCP2_SECONDS) - << R"( + << util::duration_str(config->quic.upstream.initial_rtt) << R"( --no-quic-bpf Disable eBPF. --frontend-http3-window-size= diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 25576c70..90bdc1c2 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -4159,13 +4159,7 @@ int parse_config(Config *config, int optid, const StringRef &opt, optarg); case SHRPX_OPTID_FRONTEND_QUIC_INITIAL_RTT: { #ifdef ENABLE_HTTP3 - ev_tstamp d; - if (parse_duration(&d, opt, optarg) != 0) { - return -1; - } - - config->quic.upstream.initial_rtt = - static_cast(d * NGTCP2_SECONDS); + return parse_duration(&config->quic.upstream.initial_rtt, opt, optarg); #endif // ENABLE_HTTP3 return 0; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 1b19d2f5..27b6a9ac 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -782,7 +782,7 @@ struct QUICConfig { bool require_token; std::array server_id; StringRef secret_file; - ngtcp2_duration initial_rtt; + ev_tstamp initial_rtt; } upstream; struct { StringRef prog_file; diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index f59d0fff..662c19ae 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -592,7 +592,8 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, } settings.initial_ts = quic_timestamp(); - settings.initial_rtt = quicconf.upstream.initial_rtt; + settings.initial_rtt = static_cast( + quicconf.upstream.initial_rtt * NGTCP2_SECONDS); settings.cc_algo = quicconf.upstream.congestion_controller; settings.max_window = http3conf.upstream.max_connection_window_size; settings.max_stream_window = http3conf.upstream.max_window_size; From 59a76c6d39cb17b79e3bcab6c9f7b90fdeef754b Mon Sep 17 00:00:00 2001 From: Shelley Vohr Date: Mon, 11 Oct 2021 12:56:23 +0200 Subject: [PATCH 082/142] chore: fix -Wunreachable-code-return --- lib/nghttp2_submit.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/nghttp2_submit.c b/lib/nghttp2_submit.c index 744a49cf..92fb03e8 100644 --- a/lib/nghttp2_submit.c +++ b/lib/nghttp2_submit.c @@ -492,8 +492,6 @@ int nghttp2_session_set_local_window_size(nghttp2_session *session, return nghttp2_session_update_recv_stream_window_size(session, stream, 0, 1); } - - return 0; } int nghttp2_submit_altsvc(nghttp2_session *session, uint8_t flags, From 0b6092446b42cff5d3311a47bdb2185c647a71ea Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 14 Oct 2021 23:44:27 +0900 Subject: [PATCH 083/142] src: Compile with boringssl for non-http3 build --- src/h2load.cc | 11 +++++++++-- src/shrpx_connection.cc | 16 ++++++++-------- src/shrpx_tls.cc | 28 +++++++++++++++------------- src/tls.h | 6 +++--- 4 files changed, 35 insertions(+), 26 deletions(-) diff --git a/src/h2load.cc b/src/h2load.cc index e7f368c8..6a58d1ae 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -2845,19 +2845,26 @@ int main(int argc, char **argv) { exit(EXIT_FAILURE); } -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (SSL_CTX_set_ciphersuites(ssl_ctx, config.tls13_ciphers.c_str()) == 0) { std::cerr << "SSL_CTX_set_ciphersuites with " << config.tls13_ciphers << " failed: " << ERR_error_string(ERR_get_error(), nullptr) << std::endl; exit(EXIT_FAILURE); } -#endif // OPENSSL_1_1_1_API +#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (SSL_CTX_set1_groups_list(ssl_ctx, config.groups.c_str()) != 1) { std::cerr << "SSL_CTX_set1_groups_list failed" << std::endl; exit(EXIT_FAILURE); } +#else // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) + if (SSL_CTX_set1_curves_list(ssl_ctx, config.groups.c_str()) != 1) { + std::cerr << "SSL_CTX_set1_curves_list failed" << std::endl; + exit(EXIT_FAILURE); + } +#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) #ifndef OPENSSL_NO_NEXTPROTONEG SSL_CTX_set_next_proto_select_cb(ssl_ctx, client_select_next_proto_cb, diff --git a/src/shrpx_connection.cc b/src/shrpx_connection.cc index d07314b6..fc108bbd 100644 --- a/src/shrpx_connection.cc +++ b/src/shrpx_connection.cc @@ -397,7 +397,7 @@ int Connection::tls_handshake() { ERR_clear_error(); -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (!tls.server_handshake || tls.early_data_finish) { rv = SSL_do_handshake(tls.ssl); } else { @@ -449,9 +449,9 @@ int Connection::tls_handshake() { } } } -#else // !OPENSSL_1_1_1_API +#else // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) rv = SSL_do_handshake(tls.ssl); -#endif // !OPENSSL_1_1_1_API +#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) if (rv <= 0) { auto err = SSL_get_error(tls.ssl, rv); @@ -698,7 +698,7 @@ ssize_t Connection::write_tls(const void *data, size_t len) { ERR_clear_error(); -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) int rv; if (SSL_is_init_finished(tls.ssl)) { rv = SSL_write(tls.ssl, data, len); @@ -710,9 +710,9 @@ ssize_t Connection::write_tls(const void *data, size_t len) { rv = nwrite; } } -#else // !OPENSSL_1_1_1_API +#else // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) auto rv = SSL_write(tls.ssl, data, len); -#endif // !OPENSSL_1_1_1_API +#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) if (rv <= 0) { auto err = SSL_get_error(tls.ssl, rv); @@ -772,7 +772,7 @@ ssize_t Connection::read_tls(void *data, size_t len) { tls.last_readlen = 0; } -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (!tls.early_data_finish) { // TLSv1.3 handshake is still going on. size_t nread; @@ -811,7 +811,7 @@ ssize_t Connection::read_tls(void *data, size_t len) { } return nread; } -#endif // OPENSSL_1_1_1_API +#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) auto rv = SSL_read(tls.ssl, data, len); diff --git a/src/shrpx_tls.cc b/src/shrpx_tls.cc index a933b660..12539ad5 100644 --- a/src/shrpx_tls.cc +++ b/src/shrpx_tls.cc @@ -731,7 +731,8 @@ int quic_alpn_select_proto_cb(SSL *ssl, const unsigned char **out, # endif // OPENSSL_VERSION_NUMBER >= 0x10002000L #endif // ENABLE_HTTP3 -#if !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L +#if !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L && \ + !defined(OPENSSL_IS_BORINGSSL) # ifndef TLSEXT_TYPE_signed_certificate_timestamp # define TLSEXT_TYPE_signed_certificate_timestamp 18 @@ -821,7 +822,8 @@ int legacy_sct_parse_cb(SSL *ssl, unsigned int ext_type, } // namespace # endif // !OPENSSL_1_1_1_API -#endif // !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L +#endif // !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L && + // !defined(OPENSSL_IS_BORINGSSL) #ifndef OPENSSL_NO_PSK namespace { @@ -931,14 +933,14 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file, SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION | SSL_OP_SINGLE_ECDH_USE | SSL_OP_SINGLE_DH_USE | SSL_OP_CIPHER_SERVER_PREFERENCE -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) // The reason for disabling built-in anti-replay in OpenSSL is // that it only works if client gets back to the same server. // The freshness check described in // https://tools.ietf.org/html/rfc8446#section-8.3 is still // performed. | SSL_OP_NO_ANTI_REPLAY -#endif // OPENSSL_1_1_1_API +#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) ; auto config = mod_config(); @@ -969,13 +971,13 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file, DIE(); } -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (SSL_CTX_set_ciphersuites(ssl_ctx, tlsconf.tls13_ciphers.c_str()) == 0) { LOG(FATAL) << "SSL_CTX_set_ciphersuites " << tlsconf.tls13_ciphers << " failed: " << ERR_error_string(ERR_get_error(), nullptr); DIE(); } -#endif // OPENSSL_1_1_1_API +#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) #ifndef OPENSSL_NO_EC # if !LIBRESSL_LEGACY_API && OPENSSL_VERSION_NUMBER >= 0x10002000L @@ -1172,13 +1174,13 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file, #endif // !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L && // !defined(OPENSSL_IS_BORINGSSL) -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (SSL_CTX_set_max_early_data(ssl_ctx, tlsconf.max_early_data) != 1) { LOG(FATAL) << "SSL_CTX_set_max_early_data failed: " << ERR_error_string(ERR_get_error(), nullptr); DIE(); } -#endif // OPENSSL_1_1_1_API +#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) #ifndef OPENSSL_NO_PSK SSL_CTX_set_psk_server_callback(ssl_ctx, psk_server_cb); @@ -1616,14 +1618,14 @@ SSL_CTX *create_ssl_client_context( DIE(); } -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (SSL_CTX_set_ciphersuites(ssl_ctx, tlsconf.client.tls13_ciphers.c_str()) == 0) { LOG(FATAL) << "SSL_CTX_set_ciphersuites " << tlsconf.client.tls13_ciphers << " failed: " << ERR_error_string(ERR_get_error(), nullptr); DIE(); } -#endif // OPENSSL_1_1_1_API +#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) SSL_CTX_set_mode(ssl_ctx, SSL_MODE_RELEASE_BUFFERS); @@ -2625,7 +2627,7 @@ namespace { int time_t_from_asn1_time(time_t &t, const ASN1_TIME *at) { int rv; -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) struct tm tm; rv = ASN1_TIME_to_tm(at, &tm); if (rv != 1) { @@ -2633,7 +2635,7 @@ int time_t_from_asn1_time(time_t &t, const ASN1_TIME *at) { } t = nghttp2_timegm(&tm); -#else // !OPENSSL_1_1_1_API +#else // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) auto b = BIO_new(BIO_s_mem()); if (!b) { return -1; @@ -2659,7 +2661,7 @@ int time_t_from_asn1_time(time_t &t, const ASN1_TIME *at) { } t = tt; -#endif // !OPENSSL_1_1_1_API +#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) return 0; } diff --git a/src/tls.h b/src/tls.h index ee8a5d7f..8b785ffc 100644 --- a/src/tls.h +++ b/src/tls.h @@ -57,11 +57,11 @@ constexpr char DEFAULT_CIPHER_LIST[] = "AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256"; constexpr char DEFAULT_TLS13_CIPHER_LIST[] = -#if OPENSSL_1_1_1_API +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) TLS_DEFAULT_CIPHERSUITES -#else // !OPENSSL_1_1_1_API +#else // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) "" -#endif // !OPENSSL_1_1_1_API +#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) ; constexpr auto NGHTTP2_TLS_MIN_VERSION = TLS1_VERSION; From 3a721a9dd510f607ca6f7d57efa8bfa467fa1b33 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 14 Oct 2021 23:45:07 +0900 Subject: [PATCH 084/142] nghttpx: Send session ticket after handshake with boringssl --- src/shrpx_connection.cc | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/shrpx_connection.cc b/src/shrpx_connection.cc index fc108bbd..6a1f4419 100644 --- a/src/shrpx_connection.cc +++ b/src/shrpx_connection.cc @@ -571,6 +571,36 @@ int Connection::write_tls_pending_handshake() { tls.wbuf.drain(nwrite); } +#if defined(OPENSSL_IS_BORINGSSL) + if (!SSL_in_init(tls.ssl)) { + // This will send a session ticket. + auto nwrite = SSL_write(tls.ssl, "", 0); + if (nwrite < 0) { + auto err = SSL_get_error(tls.ssl, nwrite); + switch (err) { + case SSL_ERROR_WANT_READ: + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Close connection due to TLS renegotiation"; + } + return SHRPX_ERR_NETWORK; + case SSL_ERROR_WANT_WRITE: + break; + case SSL_ERROR_SSL: + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "SSL_write: " + << ERR_error_string(ERR_get_error(), nullptr); + } + return SHRPX_ERR_NETWORK; + default: + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "SSL_write: SSL_get_error returned " << err; + } + return SHRPX_ERR_NETWORK; + } + } + } +#endif // defined(OPENSSL_IS_BORINGSSL) + // We have to start read watcher, since later stage of code expects // this. rlimit.startw(); From d8282de2290a265956dee94bfe068d2cca36747d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 15 Oct 2021 00:11:57 +0900 Subject: [PATCH 085/142] nghttpx: Respect !tls-no-postpone-early-data with boringssl --- src/shrpx_connection.cc | 44 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/src/shrpx_connection.cc b/src/shrpx_connection.cc index 6a1f4419..de1dfbfc 100644 --- a/src/shrpx_connection.cc +++ b/src/shrpx_connection.cc @@ -397,11 +397,12 @@ int Connection::tls_handshake() { ERR_clear_error(); + auto &tlsconf = get_config()->tls; + #if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (!tls.server_handshake || tls.early_data_finish) { rv = SSL_do_handshake(tls.ssl); } else { - auto &tlsconf = get_config()->tls; for (;;) { size_t nread; @@ -499,7 +500,12 @@ int Connection::tls_handshake() { // Don't send handshake data if handshake was completed in OpenSSL // routine. We have to check HTTP/2 requirement if HTTP/2 was // negotiated before sending finished message to the peer. - if (rv != 1 && tls.wbuf.rleft()) { + if ((rv != 1 +#if defined(OPENSSL_IS_BORINGSSL) + || SSL_in_init(tls.ssl) +#endif // defined(OPENSSL_IS_BORINGSSL) + ) && + tls.wbuf.rleft()) { // First write indicates that resumption stuff has done. if (tls.handshake_state != TLSHandshakeState::WRITE_STARTED) { tls.handshake_state = TLSHandshakeState::WRITE_STARTED; @@ -535,6 +541,40 @@ int Connection::tls_handshake() { return SHRPX_ERR_INPROGRESS; } +#if defined(OPENSSL_IS_BORINGSSL) + if (!tlsconf.no_postpone_early_data && SSL_in_early_data(tls.ssl) && + SSL_in_init(tls.ssl)) { + auto nread = SSL_read(tls.ssl, buf.data(), buf.size()); + if (nread <= 0) { + auto err = SSL_get_error(tls.ssl, nread); + switch (err) { + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + break; + case SSL_ERROR_ZERO_RETURN: + return SHRPX_ERR_EOF; + case SSL_ERROR_SSL: + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "SSL_read: " + << ERR_error_string(ERR_get_error(), nullptr); + } + return SHRPX_ERR_NETWORK; + default: + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "SSL_read: SSL_get_error returned " << err; + } + return SHRPX_ERR_NETWORK; + } + } else { + tls.earlybuf.append(buf.data(), nread); + } + + if (SSL_in_init(tls.ssl)) { + return SHRPX_ERR_INPROGRESS; + } + } +#endif // defined(OPENSSL_IS_BORINGSSL) + // Handshake was done rv = check_http2_requirement(); From 3742acaf398d394a84e70f25894f17f86f8a1745 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 15 Oct 2021 18:19:07 +0900 Subject: [PATCH 086/142] nghttpx: Fix wrong SSL_CTX object usage --- src/shrpx_connection_handler.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 3ff22740..99800ca4 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -754,7 +754,8 @@ void ConnectionHandler::handle_ocsp_complete() { std::make_shared>(ocsp_.resp); # endif // !HAVE_ATOMIC_STD_SHARED_PTR # else // OPENSSL_IS_BORINGSSL - SSL_CTX_set_ocsp_response(ssl_ctx, ocsp_.resp.data(), ocsp_.resp.size()); + SSL_CTX_set_ocsp_response(quic_ssl_ctx, ocsp_.resp.data(), + ocsp_.resp.size()); # endif // OPENSSL_IS_BORINGSSL } #endif // ENABLE_HTTP3 From 9fb05d5ea258ac5707b852a856608b00193169d3 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 15 Oct 2021 19:54:08 +0900 Subject: [PATCH 087/142] Fix compile error under mac os --- src/shrpx_connection.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/shrpx_connection.cc b/src/shrpx_connection.cc index de1dfbfc..229e693c 100644 --- a/src/shrpx_connection.cc +++ b/src/shrpx_connection.cc @@ -397,7 +397,9 @@ int Connection::tls_handshake() { ERR_clear_error(); +#if OPENSSL_1_1_1_API || defined(OPENSSL_IS_BORINGSSL) auto &tlsconf = get_config()->tls; +#endif // OPENSSL_1_1_1_API || defined(OPENSSL_IS_BORINGSSL) #if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (!tls.server_handshake || tls.early_data_finish) { From c790ee64a46d6f58fc034b719c3c7808d0473a9c Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 15 Oct 2021 22:29:05 +0900 Subject: [PATCH 088/142] src: Prefer #ifdef for a single condition --- src/shrpx_config.cc | 2 +- src/shrpx_connection.cc | 12 ++++++------ src/shrpx_log.cc | 2 +- src/shrpx_tls.cc | 2 +- src/ssl_compat.h | 6 +++--- src/util.cc | 6 +++--- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 90bdc1c2..9c5d9917 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -304,7 +304,7 @@ read_quic_secret_file(const StringRef &path) { FILE *open_file_for_write(const char *filename) { std::array errbuf; -#if defined O_CLOEXEC +#ifdef O_CLOEXEC auto fd = open(filename, O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); #else diff --git a/src/shrpx_connection.cc b/src/shrpx_connection.cc index 229e693c..af3cc05a 100644 --- a/src/shrpx_connection.cc +++ b/src/shrpx_connection.cc @@ -503,9 +503,9 @@ int Connection::tls_handshake() { // routine. We have to check HTTP/2 requirement if HTTP/2 was // negotiated before sending finished message to the peer. if ((rv != 1 -#if defined(OPENSSL_IS_BORINGSSL) +#ifdef OPENSSL_IS_BORINGSSL || SSL_in_init(tls.ssl) -#endif // defined(OPENSSL_IS_BORINGSSL) +#endif // OPENSSL_IS_BORINGSSL ) && tls.wbuf.rleft()) { // First write indicates that resumption stuff has done. @@ -543,7 +543,7 @@ int Connection::tls_handshake() { return SHRPX_ERR_INPROGRESS; } -#if defined(OPENSSL_IS_BORINGSSL) +#ifdef OPENSSL_IS_BORINGSSL if (!tlsconf.no_postpone_early_data && SSL_in_early_data(tls.ssl) && SSL_in_init(tls.ssl)) { auto nread = SSL_read(tls.ssl, buf.data(), buf.size()); @@ -575,7 +575,7 @@ int Connection::tls_handshake() { return SHRPX_ERR_INPROGRESS; } } -#endif // defined(OPENSSL_IS_BORINGSSL) +#endif // OPENSSL_IS_BORINGSSL // Handshake was done @@ -613,7 +613,7 @@ int Connection::write_tls_pending_handshake() { tls.wbuf.drain(nwrite); } -#if defined(OPENSSL_IS_BORINGSSL) +#ifdef OPENSSL_IS_BORINGSSL if (!SSL_in_init(tls.ssl)) { // This will send a session ticket. auto nwrite = SSL_write(tls.ssl, "", 0); @@ -641,7 +641,7 @@ int Connection::write_tls_pending_handshake() { } } } -#endif // defined(OPENSSL_IS_BORINGSSL) +#endif // OPENSSL_IS_BORINGSSL // We have to start read watcher, since later stage of code expects // this. diff --git a/src/shrpx_log.cc b/src/shrpx_log.cc index 8fd8b11c..3a5af823 100644 --- a/src/shrpx_log.cc +++ b/src/shrpx_log.cc @@ -964,7 +964,7 @@ int open_log_file(const char *path) { strcmp(path, "/proc/self/fd/2") == 0) { return STDERR_COPY; } -#if defined O_CLOEXEC +#ifdef O_CLOEXEC auto fd = open(path, O_WRONLY | O_APPEND | O_CREAT | O_CLOEXEC, S_IRUSR | S_IWUSR | S_IRGRP); diff --git a/src/shrpx_tls.cc b/src/shrpx_tls.cc index 12539ad5..bb0017e6 100644 --- a/src/shrpx_tls.cc +++ b/src/shrpx_tls.cc @@ -2648,7 +2648,7 @@ int time_t_from_asn1_time(time_t &t, const ASN1_TIME *at) { return -1; } -# if defined(OPENSSL_IS_BORINGSSL) +# ifdef OPENSSL_IS_BORINGSSL char *s; # else unsigned char *s; diff --git a/src/ssl_compat.h b/src/ssl_compat.h index e1077eef..87f326a4 100644 --- a/src/ssl_compat.h +++ b/src/ssl_compat.h @@ -26,20 +26,20 @@ # include -# if defined(LIBRESSL_VERSION_NUMBER) +# ifdef LIBRESSL_VERSION_NUMBER # define OPENSSL_1_1_API 0 # define OPENSSL_1_1_1_API 0 # define OPENSSL_3_0_0_API 0 # define LIBRESSL_IN_USE 1 # define LIBRESSL_LEGACY_API (LIBRESSL_VERSION_NUMBER < 0x20700000L) # define LIBRESSL_2_7_API (LIBRESSL_VERSION_NUMBER >= 0x20700000L) -# else // !defined(LIBRESSL_VERSION_NUMBER) +# else // !LIBRESSL_VERSION_NUMBER # define OPENSSL_1_1_API (OPENSSL_VERSION_NUMBER >= 0x1010000fL) # define OPENSSL_1_1_1_API (OPENSSL_VERSION_NUMBER >= 0x10101000L) # define OPENSSL_3_0_0_API (OPENSSL_VERSION_NUMBER >= 0x30000000L) # define LIBRESSL_IN_USE 0 # define LIBRESSL_LEGACY_API 0 # define LIBRESSL_2_7_API 0 -# endif // !defined(LIBRESSL_VERSION_NUMBER) +# endif // !LIBRESSL_VERSION_NUMBER #endif // OPENSSL_COMPAT_H diff --git a/src/util.cc b/src/util.cc index 4db6325d..213989e4 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1649,7 +1649,7 @@ std::mt19937 make_mt19937() { } int daemonize(int nochdir, int noclose) { -#if defined(__APPLE__) +#ifdef __APPLE__ pid_t pid; pid = fork(); if (pid == -1) { @@ -1683,9 +1683,9 @@ int daemonize(int nochdir, int noclose) { } } return 0; -#else // !defined(__APPLE__) +#else // !__APPLE__ return daemon(nochdir, noclose); -#endif // !defined(__APPLE__) +#endif // !__APPLE__ } #ifdef ENABLE_HTTP3 From 7055501efd26a9b64d7d46206ded82af8d1ea2b8 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 15 Oct 2021 19:52:01 +0900 Subject: [PATCH 089/142] src: Enable HTTP/3 with boringssl --- configure.ac | 50 ++++++++++++- src/Makefile.am | 2 + src/h2load.h | 6 +- src/h2load_quic.cc | 108 +++++++++++++++++++++++---- src/shrpx_connection_handler.cc | 2 +- src/shrpx_http3_upstream.cc | 34 +++++++++ src/shrpx_quic_connection_handler.cc | 6 ++ src/shrpx_tls.cc | 107 +++++++++++++++++++++++--- 8 files changed, 282 insertions(+), 33 deletions(-) diff --git a/configure.ac b/configure.ac index 5e19e1ce..671a9484 100644 --- a/configure.ac +++ b/configure.ac @@ -474,6 +474,7 @@ if test "x${request_openssl}" != "xno"; then CFLAGS="$OPENSSL_CFLAGS $CFLAGS" LIBS="$OPENSSL_LIBS $LIBS" + # quictls/openssl has SSL_is_quic. have_ssl_is_quic=no AC_MSG_CHECKING([for SSL_is_quic]) AC_LINK_IFELSE([AC_LANG_PROGRAM([[ @@ -485,6 +486,17 @@ if test "x${request_openssl}" != "xno"; then [AC_MSG_RESULT([yes]); have_ssl_is_quic=yes], [AC_MSG_RESULT([no]); have_ssl_is_quic=no]) + # boringssl has SSL_set_quic_early_data_context. + AC_MSG_CHECKING([for SSL_set_quic_early_data_context]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([[ + #include + ]], [[ + SSL *ssl = NULL; + SSL_set_quic_early_data_context(ssl, NULL, 0); + ]])], + [AC_MSG_RESULT([yes]); have_boringssl_quic=yes], + [AC_MSG_RESULT([no]); have_boringssl_quic=no]) + CFLAGS="$save_CFLAGS" LIBS="$save_LIBS" fi @@ -527,21 +539,48 @@ fi # ngtcp2_crypto_openssl (for src) have_libngtcp2_crypto_openssl=no -if test "x${request_libngtcp2}" != "xno"; then +if test "x${have_ssl_is_quic}" = "xyes" && + test "x${request_libngtcp2}" != "xno"; then PKG_CHECK_MODULES([LIBNGTCP2_CRYPTO_OPENSSL], [libngtcp2_crypto_openssl >= 0.0.0], [have_libngtcp2_crypto_openssl=yes], [have_libngtcp2_crypto_openssl=no]) if test "x${have_libngtcp2_crypto_openssl}" = "xno"; then AC_MSG_NOTICE($LIBNGTCP2_CRYPTO_OPENSSL_PKG_ERRORS) + else + AC_DEFINE([HAVE_LIBNGTCP2_CRYPTO_OPENSSL], [1], + [Define to 1 if you have `libngtcp2_crypto_openssl` library.]) fi fi -if test "x${request_libngtcp2}" = "xyes" && +if test "x${have_ssl_is_quic}" = "xyes" && + test "x${request_libngtcp2}" = "xyes" && test "x${have_libngtcp2_crypto_openssl}" != "xyes"; then AC_MSG_ERROR([libngtcp2_crypto_openssl was requested (--with-libngtcp2) but not found]) fi +# ngtcp2_crypto_boringssl (for src) +have_libngtcp2_crypto_boringssl=no +if test "x${have_boringssl_quic}" = "xyes" && + test "x${request_libngtcp2}" != "xno"; then + PKG_CHECK_MODULES([LIBNGTCP2_CRYPTO_BORINGSSL], + [libngtcp2_crypto_boringssl >= 0.0.0], + [have_libngtcp2_crypto_boringssl=yes], + [have_libngtcp2_crypto_boringssl=no]) + if test "x${have_libngtcp2_crypto_boringssl}" = "xno"; then + AC_MSG_NOTICE($LIBNGTCP2_CRYPTO_BORINGSSL_PKG_ERRORS) + else + AC_DEFINE([HAVE_LIBNGTCP2_CRYPTO_BORINGSSL], [1], + [Define to 1 if you have `libngtcp2_crypto_boringssl` library.]) + fi +fi + +if test "x${have_boringssl_quic}" = "xyes" && + test "x${request_libngtcp2}" = "xyes" && + test "x${have_libngtcp2_crypto_boringssl}" != "xyes"; then + AC_MSG_ERROR([libngtcp2_crypto_boringssl was requested (--with-libngtcp2) but not found]) +fi + # nghttp3 (for src) have_libnghttp3=no if test "x${request_libnghttp3}" != "xno"; then @@ -749,9 +788,11 @@ AM_CONDITIONAL([ENABLE_APP], [ test "x${enable_app}" = "xyes" ]) # Check HTTP/3 support enable_http3=no if test "x${request_http3}" != "xno" && - test "x${have_ssl_is_quic}" = "xyes" && + (test "x${have_ssl_is_quic}" = "xyes" || + test "x${have_boringssl_quic}" = "xyes") && test "x${have_libngtcp2}" = "xyes" && - test "x${have_libngtcp2_crypto_openssl}" = "xyes" && + (test "x${have_libngtcp2_crypto_openssl}" = "xyes" || + test "x${have_libngtcp2_crypto_boringssl}" = "xyes") && test "x${have_libnghttp3}" = "xyes"; then enable_http3=yes AC_DEFINE([ENABLE_HTTP3], [1], [Define to 1 if HTTP/3 is enabled.]) @@ -1184,6 +1225,7 @@ AC_MSG_NOTICE([summary of build options: Libc-ares: ${have_libcares} (CFLAGS='${LIBCARES_CFLAGS}' LIBS='${LIBCARES_LIBS}') libngtcp2: ${have_libngtcp2} (CFLAGS='${LIBNGTCP2_CFLAGS}' LIBS='${LIBNGTCP2_LIBS}') libngtcp2_crypto_openssl: ${have_libngtcp2_crypto_openssl} (CFLAGS='${LIBNGTCP2_CRYPTO_OPENSSL_CFLAGS}' LIBS='${LIBNGTCP2_CRYPTO_OPENSSL_LIBS}') + libngtcp2_crypto_boringssl: ${have_libngtcp2_crypto_boringssl} (CFLAGS='${LIBNGTCP2_CRYPTO_BORINGSSL_CFLAGS}' LIBS='${LIBNGTCP2_CRYPTO_BORINGSSL_LIBS}') libnghttp3: ${have_libnghttp3} (CFLAGS='${LIBNGHTTP3_CFLAGS}' LIBS='${LIBNGHTTP3_LIBS}') libbpf: ${have_libbpf} (CFLAGS='${LIBBPF_CFLAGS}' LIBS='${LIBBPF_LIBS}') Libevent(SSL): ${have_libevent_openssl} (CFLAGS='${LIBEVENT_OPENSSL_CFLAGS}' LIBS='${LIBEVENT_OPENSSL_LIBS}') diff --git a/src/Makefile.am b/src/Makefile.am index a0545b4f..2fd6eadd 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -47,6 +47,7 @@ AM_CPPFLAGS = \ @LIBEV_CFLAGS@ \ @LIBNGHTTP3_CFLAGS@ \ @LIBNGTCP2_CRYPTO_OPENSSL_CFLAGS@ \ + @LIBNGTCP2_CRYPTO_BORINGSSL_CFLAGS@ \ @LIBNGTCP2_CFLAGS@ \ @OPENSSL_CFLAGS@ \ @LIBCARES_CFLAGS@ \ @@ -65,6 +66,7 @@ LDADD = $(top_builddir)/lib/libnghttp2.la \ @LIBEV_LIBS@ \ @LIBNGHTTP3_LIBS@ \ @LIBNGTCP2_CRYPTO_OPENSSL_LIBS@ \ + @LIBNGTCP2_CRYPTO_BORINGSSL_LIBS@ \ @LIBNGTCP2_LIBS@ \ @OPENSSL_LIBS@ \ @LIBCARES_LIBS@ \ diff --git a/src/h2load.h b/src/h2load.h index 895fb8eb..67da2b50 100644 --- a/src/h2load.h +++ b/src/h2load.h @@ -474,8 +474,10 @@ struct Client { int quic_stream_stop_sending(int64_t stream_id, uint64_t app_error_code); int quic_extend_max_local_streams(); - int quic_on_key(ngtcp2_crypto_level level, const uint8_t *rx_secret, - const uint8_t *tx_secret, size_t secretlen); + int quic_on_rx_secret(ngtcp2_crypto_level level, const uint8_t *secret, + size_t secretlen); + int quic_on_tx_secret(ngtcp2_crypto_level level, const uint8_t *secret, + size_t secretlen); void quic_set_tls_alert(uint8_t alert); void quic_write_client_handshake(ngtcp2_crypto_level level, diff --git a/src/h2load_quic.cc b/src/h2load_quic.cc index b77bf6ce..74680043 100644 --- a/src/h2load_quic.cc +++ b/src/h2load_quic.cc @@ -28,7 +28,12 @@ #include -#include +#ifdef HAVE_LIBNGTCP2_CRYPTO_OPENSSL +# include +#endif // HAVE_LIBNGTCP2_CRYPTO_OPENSSL +#ifdef HAVE_LIBNGTCP2_CRYPTO_BORINGSSL +# include +#endif // HAVE_LIBNGTCP2_CRYPTO_BORINGSSL #include @@ -236,15 +241,19 @@ ngtcp2_tstamp timestamp(struct ev_loop *loop) { } } // namespace +#ifdef HAVE_LIBNGTCP2_CRYPTO_OPENSSL namespace { int set_encryption_secrets(SSL *ssl, OSSL_ENCRYPTION_LEVEL ossl_level, const uint8_t *rx_secret, const uint8_t *tx_secret, size_t secret_len) { auto c = static_cast(SSL_get_app_data(ssl)); + auto level = ngtcp2_crypto_openssl_from_ossl_encryption_level(ossl_level); - if (c->quic_on_key( - ngtcp2_crypto_openssl_from_ossl_encryption_level(ossl_level), - rx_secret, tx_secret, secret_len) != 0) { + if (c->quic_on_rx_secret(level, rx_secret, secret_len) != 0) { + return 0; + } + + if (c->quic_on_tx_secret(level, tx_secret, secret_len) != 0) { return 0; } @@ -282,6 +291,70 @@ auto quic_method = SSL_QUIC_METHOD{ send_alert, }; } // namespace +#endif // HAVE_LIBNGTCP2_CRYPTO_OPENSSL + +#ifdef HAVE_LIBNGTCP2_CRYPTO_BORINGSSL +namespace { +int set_read_secret(SSL *ssl, ssl_encryption_level_t ssl_level, + const SSL_CIPHER *cipher, const uint8_t *secret, + size_t secretlen) { + auto c = static_cast(SSL_get_app_data(ssl)); + + if (c->quic_on_rx_secret( + ngtcp2_crypto_boringssl_from_ssl_encryption_level(ssl_level), secret, + secretlen) != 0) { + return 0; + } + + return 1; +} +} // namespace + +namespace { +int set_write_secret(SSL *ssl, ssl_encryption_level_t ssl_level, + const SSL_CIPHER *cipher, const uint8_t *secret, + size_t secretlen) { + auto c = static_cast(SSL_get_app_data(ssl)); + + if (c->quic_on_tx_secret( + ngtcp2_crypto_boringssl_from_ssl_encryption_level(ssl_level), secret, + secretlen) != 0) { + return 0; + } + + return 1; +} +} // namespace + +namespace { +int add_handshake_data(SSL *ssl, ssl_encryption_level_t ssl_level, + const uint8_t *data, size_t len) { + auto c = static_cast(SSL_get_app_data(ssl)); + c->quic_write_client_handshake( + ngtcp2_crypto_boringssl_from_ssl_encryption_level(ssl_level), data, len); + return 1; +} +} // namespace + +namespace { +int flush_flight(SSL *ssl) { return 1; } +} // namespace + +namespace { +int send_alert(SSL *ssl, ssl_encryption_level_t level, uint8_t alert) { + auto c = static_cast(SSL_get_app_data(ssl)); + c->quic_set_tls_alert(alert); + return 1; +} +} // namespace + +namespace { +auto quic_method = SSL_QUIC_METHOD{ + set_read_secret, set_write_secret, add_handshake_data, + flush_flight, send_alert, +}; +} // namespace +#endif // HAVE_LIBNGTCP2_CRYPTO_BORINGSSL // qlog write callback -- excerpted from ngtcp2/examples/client_base.cc namespace { @@ -462,24 +535,16 @@ void Client::quic_close_connection() { ps.path.remote.addrlen, buf.data(), nwrite, 0); } -int Client::quic_on_key(ngtcp2_crypto_level level, const uint8_t *rx_secret, - const uint8_t *tx_secret, size_t secretlen) { +int Client::quic_on_rx_secret(ngtcp2_crypto_level level, const uint8_t *secret, + size_t secretlen) { if (ngtcp2_crypto_derive_and_install_rx_key(quic.conn, nullptr, nullptr, - nullptr, level, rx_secret, + nullptr, level, secret, secretlen) != 0) { std::cerr << "ngtcp2_crypto_derive_and_install_rx_key() failed" << std::endl; return -1; } - if (ngtcp2_crypto_derive_and_install_tx_key(quic.conn, nullptr, nullptr, - nullptr, level, tx_secret, - secretlen) != 0) { - std::cerr << "ngtcp2_crypto_derive_and_install_tx_key() failed" - << std::endl; - return -1; - } - if (level == NGTCP2_CRYPTO_LEVEL_APPLICATION) { auto s = std::make_unique(this); if (s->init_conn() == -1) { @@ -491,6 +556,19 @@ int Client::quic_on_key(ngtcp2_crypto_level level, const uint8_t *rx_secret, return 0; } +int Client::quic_on_tx_secret(ngtcp2_crypto_level level, const uint8_t *secret, + size_t secretlen) { + if (ngtcp2_crypto_derive_and_install_tx_key(quic.conn, nullptr, nullptr, + nullptr, level, secret, + secretlen) != 0) { + std::cerr << "ngtcp2_crypto_derive_and_install_tx_key() failed" + << std::endl; + return -1; + } + + return 0; +} + void Client::quic_set_tls_alert(uint8_t alert) { quic.last_error = quic::err_transport_tls(alert); } diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 99800ca4..cf45f508 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -740,9 +740,9 @@ void ConnectionHandler::handle_ocsp_complete() { // that case we get nullptr. auto quic_ssl_ctx = quic_all_ssl_ctx_[ocsp_.next]; if (quic_ssl_ctx) { +# ifndef OPENSSL_IS_BORINGSSL auto quic_tls_ctx_data = static_cast( SSL_CTX_get_app_data(quic_ssl_ctx)); -# ifndef OPENSSL_IS_BORINGSSL # ifdef HAVE_ATOMIC_STD_SHARED_PTR std::atomic_store_explicit( &quic_tls_ctx_data->ocsp_data, diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 662c19ae..1208e3a3 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -613,6 +613,40 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, params.max_idle_timeout = static_cast( quicconf.upstream.timeout.idle * NGTCP2_SECONDS); +#ifdef OPENSSL_IS_BORINGSSL + if (quicconf.upstream.early_data) { + ngtcp2_transport_params early_data_params{ + .initial_max_stream_data_bidi_local = + params.initial_max_stream_data_bidi_local, + .initial_max_stream_data_bidi_remote = + params.initial_max_stream_data_bidi_remote, + .initial_max_stream_data_uni = params.initial_max_stream_data_uni, + .initial_max_data = params.initial_max_data, + .initial_max_streams_bidi = params.initial_max_streams_bidi, + .initial_max_streams_uni = params.initial_max_streams_uni, + }; + + // TODO include HTTP/3 SETTINGS + + std::array quic_early_data_ctx; + + auto quic_early_data_ctxlen = ngtcp2_encode_transport_params( + quic_early_data_ctx.data(), quic_early_data_ctx.size(), + NGTCP2_TRANSPORT_PARAMS_TYPE_ENCRYPTED_EXTENSIONS, &early_data_params); + + assert(quic_early_data_ctxlen > 0); + assert(static_cast(quic_early_data_ctxlen) <= + quic_early_data_ctx.size()); + + if (SSL_set_quic_early_data_context(handler_->get_ssl(), + quic_early_data_ctx.data(), + quic_early_data_ctxlen) != 1) { + ULOG(ERROR, this) << "SSL_set_quic_early_data_context failed"; + return -1; + } + } +#endif // OPENSSL_IS_BORINGSSL + if (odcid) { params.original_dcid = *odcid; params.retry_scid = initial_hd.dcid; diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 37116645..730c649f 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -369,7 +369,9 @@ ClientHandler *QUICConnectionHandler::handle_new_connection( return nullptr; } +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) assert(SSL_is_quic(ssl)); +#endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) SSL_set_accept_state(ssl); @@ -377,7 +379,11 @@ ClientHandler *QUICConnectionHandler::handle_new_connection( auto &quicconf = config->quic; if (quicconf.upstream.early_data) { +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) SSL_set_quic_early_data_enabled(ssl, 1); +#else // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) + SSL_set_early_data_enabled(ssl, 1); +#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) } // Disable TLS session ticket if we don't have working ticket diff --git a/src/shrpx_tls.cc b/src/shrpx_tls.cc index bb0017e6..cff06ddc 100644 --- a/src/shrpx_tls.cc +++ b/src/shrpx_tls.cc @@ -61,8 +61,13 @@ #ifdef ENABLE_HTTP3 # include # include -# include -#endif // ENABLE_HTTP3 +# ifdef HAVE_LIBNGTCP2_CRYPTO_OPENSSL +# include +# endif // HAVE_LIBNGTCP2_CRYPTO_OPENSSL +# ifdef HAVE_LIBNGTCP2_CRYPTO_BORINGSSL +# include +# endif // HAVE_LIBNGTCP2_CRYPTO_BORINGSSL +#endif // ENABLE_HTTP3 #include "shrpx_log.h" #include "shrpx_client_handler.h" @@ -196,8 +201,9 @@ int servername_callback(SSL *ssl, int *al, void *arg) { auto hostname = StringRef{std::begin(buf), end_buf}; #ifdef ENABLE_HTTP3 - auto cert_tree = SSL_is_quic(ssl) ? worker->get_quic_cert_lookup_tree() - : worker->get_cert_lookup_tree(); + auto cert_tree = conn->proto == Proto::HTTP3 + ? worker->get_quic_cert_lookup_tree() + : worker->get_cert_lookup_tree(); #else // !ENABLE_HTTP3 auto cert_tree = worker->get_cert_lookup_tree(); #endif // !ENABLE_HTTP3 @@ -212,7 +218,7 @@ int servername_callback(SSL *ssl, int *al, void *arg) { auto conn_handler = worker->get_connection_handler(); #ifdef ENABLE_HTTP3 - const auto &ssl_ctx_list = SSL_is_quic(ssl) + const auto &ssl_ctx_list = conn->proto == Proto::HTTP3 ? conn_handler->get_quic_indexed_ssl_ctx(idx) : conn_handler->get_indexed_ssl_ctx(idx); #else // !ENABLE_HTTP3 @@ -1196,6 +1202,7 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file, } #ifdef ENABLE_HTTP3 +# ifdef HAVE_LIBNGTCP2_CRYPTO_OPENSSL namespace { int quic_set_encryption_secrets(SSL *ssl, OSSL_ENCRYPTION_LEVEL ossl_level, const uint8_t *rx_secret, @@ -1259,6 +1266,84 @@ auto quic_method = SSL_QUIC_METHOD{ quic_send_alert, }; } // namespace +# endif // HAVE_LIBNGTCP2_CRYPTO_OPENSSL + +# ifdef HAVE_LIBNGTCP2_CRYPTO_BORINGSSL +namespace { +int quic_set_read_secret(SSL *ssl, ssl_encryption_level_t ssl_level, + const SSL_CIPHER *cipher, const uint8_t *secret, + size_t secretlen) { + auto conn = static_cast(SSL_get_app_data(ssl)); + auto handler = static_cast(conn->data); + auto upstream = static_cast(handler->get_upstream()); + auto level = ngtcp2_crypto_boringssl_from_ssl_encryption_level(ssl_level); + + if (upstream->on_rx_secret(level, secret, secretlen) != 0) { + return 0; + } + + return 1; +} +} // namespace + +namespace { +int quic_set_write_secret(SSL *ssl, ssl_encryption_level_t ssl_level, + const SSL_CIPHER *cipher, const uint8_t *secret, + size_t secretlen) { + auto conn = static_cast(SSL_get_app_data(ssl)); + auto handler = static_cast(conn->data); + auto upstream = static_cast(handler->get_upstream()); + auto level = ngtcp2_crypto_boringssl_from_ssl_encryption_level(ssl_level); + + if (upstream->on_tx_secret(level, secret, secretlen) != 0) { + return 0; + } + + return 1; +} +} // namespace + +namespace { +int quic_add_handshake_data(SSL *ssl, ssl_encryption_level_t ssl_level, + const uint8_t *data, size_t len) { + auto conn = static_cast(SSL_get_app_data(ssl)); + auto handler = static_cast(conn->data); + auto upstream = static_cast(handler->get_upstream()); + auto level = ngtcp2_crypto_boringssl_from_ssl_encryption_level(ssl_level); + + upstream->add_crypto_data(level, data, len); + + return 1; +} +} // namespace + +namespace { +int quic_flush_flight(SSL *ssl) { return 1; } +} // namespace + +namespace { +int quic_send_alert(SSL *ssl, ssl_encryption_level_t level, uint8_t alert) { + auto conn = static_cast(SSL_get_app_data(ssl)); + auto handler = static_cast(conn->data); + auto upstream = static_cast(handler->get_upstream()); + + if (!upstream) { + return 1; + } + + upstream->set_tls_alert(alert); + + return 1; +} +} // namespace + +namespace { +auto quic_method = SSL_QUIC_METHOD{ + quic_set_read_secret, quic_set_write_secret, quic_add_handshake_data, + quic_flush_flight, quic_send_alert, +}; +} // namespace +# endif // HAVE_LIBNGTCP2_CRYPTO_BORINGSSL SSL_CTX *create_quic_ssl_context(const char *private_key_file, const char *cert_file, @@ -1279,14 +1364,14 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file, SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION | SSL_OP_SINGLE_ECDH_USE | SSL_OP_SINGLE_DH_USE | SSL_OP_CIPHER_SERVER_PREFERENCE -# if OPENSSL_1_1_1_API +# if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) // The reason for disabling built-in anti-replay in OpenSSL is // that it only works if client gets back to the same server. // The freshness check described in // https://tools.ietf.org/html/rfc8446#section-8.3 is still // performed. | SSL_OP_NO_ANTI_REPLAY -# endif // OPENSSL_1_1_1_API +# endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) ; auto config = mod_config(); @@ -1309,13 +1394,13 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file, DIE(); } -# if OPENSSL_1_1_1_API +# if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (SSL_CTX_set_ciphersuites(ssl_ctx, tlsconf.tls13_ciphers.c_str()) == 0) { LOG(FATAL) << "SSL_CTX_set_ciphersuites " << tlsconf.tls13_ciphers << " failed: " << ERR_error_string(ERR_get_error(), nullptr); DIE(); } -# endif // OPENSSL_1_1_1_API +# endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) # ifndef OPENSSL_NO_EC # if !LIBRESSL_LEGACY_API && OPENSSL_VERSION_NUMBER >= 0x10002000L @@ -1503,7 +1588,7 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file, # endif // !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L && // !defined(OPENSSL_IS_BORINGSSL) -# if OPENSSL_1_1_1_API +# if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) auto &quicconf = config->quic; if (quicconf.upstream.early_data && @@ -1513,7 +1598,7 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file, << ERR_error_string(ERR_get_error(), nullptr); DIE(); } -# endif // OPENSSL_1_1_1_API +# endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) # ifndef OPENSSL_NO_PSK SSL_CTX_set_psk_server_callback(ssl_ctx, psk_server_cb); From 1ce9efc64479ad5854aab00ebc2e317f878a421e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 15 Oct 2021 23:03:09 +0900 Subject: [PATCH 090/142] nghttpx: Set SCT data when built with boringssl --- src/shrpx_tls.cc | 48 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/src/shrpx_tls.cc b/src/shrpx_tls.cc index cff06ddc..8d352727 100644 --- a/src/shrpx_tls.cc +++ b/src/shrpx_tls.cc @@ -1145,6 +1145,12 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file, SSL_CTX_set_alpn_select_cb(ssl_ctx, alpn_select_proto_cb, nullptr); #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L + auto tls_ctx_data = new TLSContextData(); + tls_ctx_data->cert_file = cert_file; + tls_ctx_data->sct_data = sct_data; + + SSL_CTX_set_app_data(ssl_ctx, tls_ctx_data); + #if !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L && \ !defined(OPENSSL_IS_BORINGSSL) // SSL_extension_supported(TLSEXT_TYPE_signed_certificate_timestamp) @@ -1177,8 +1183,16 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file, } # endif // !OPENSSL_1_1_1_API } -#endif // !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L && - // !defined(OPENSSL_IS_BORINGSSL) +#elif defined(OPENSSL_IS_BORINGSSL) + if (!tls_ctx_data->sct_data.empty() && + SSL_CTX_set_signed_cert_timestamp_list( + ssl_ctx, tls_ctx_data->sct_data.data(), + tls_ctx_data->sct_data.size()) != 1) { + LOG(FATAL) << "SSL_CTX_set_signed_cert_timestamp_list failed: " + << ERR_error_string(ERR_get_error(), nullptr); + DIE(); + } +#endif // defined(OPENSSL_IS_BORINGSSL) #if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (SSL_CTX_set_max_early_data(ssl_ctx, tlsconf.max_early_data) != 1) { @@ -1192,12 +1206,6 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file, SSL_CTX_set_psk_server_callback(ssl_ctx, psk_server_cb); #endif // !LIBRESSL_NO_PSK - auto tls_ctx_data = new TLSContextData(); - tls_ctx_data->cert_file = cert_file; - tls_ctx_data->sct_data = sct_data; - - SSL_CTX_set_app_data(ssl_ctx, tls_ctx_data); - return ssl_ctx; } @@ -1553,6 +1561,12 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file, SSL_CTX_set_alpn_select_cb(ssl_ctx, quic_alpn_select_proto_cb, nullptr); # endif // OPENSSL_VERSION_NUMBER >= 0x10002000L + auto tls_ctx_data = new TLSContextData(); + tls_ctx_data->cert_file = cert_file; + tls_ctx_data->sct_data = sct_data; + + SSL_CTX_set_app_data(ssl_ctx, tls_ctx_data); + # if !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L && \ !defined(OPENSSL_IS_BORINGSSL) // SSL_extension_supported(TLSEXT_TYPE_signed_certificate_timestamp) @@ -1585,8 +1599,16 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file, } # endif // !OPENSSL_1_1_1_API } -# endif // !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L && - // !defined(OPENSSL_IS_BORINGSSL) +# elif defined(OPENSSL_IS_BORINGSSL) + if (!tls_ctx_data->sct_data.empty() && + SSL_CTX_set_signed_cert_timestamp_list( + ssl_ctx, tls_ctx_data->sct_data.data(), + tls_ctx_data->sct_data.size()) != 1) { + LOG(FATAL) << "SSL_CTX_set_signed_cert_timestamp_list failed: " + << ERR_error_string(ERR_get_error(), nullptr); + DIE(); + } +# endif // defined(OPENSSL_IS_BORINGSSL) # if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) auto &quicconf = config->quic; @@ -1606,12 +1628,6 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file, SSL_CTX_set_quic_method(ssl_ctx, &quic_method); - auto tls_ctx_data = new TLSContextData(); - tls_ctx_data->cert_file = cert_file; - tls_ctx_data->sct_data = sct_data; - - SSL_CTX_set_app_data(ssl_ctx, tls_ctx_data); - return ssl_ctx; } #endif // ENABLE_HTTP3 From f3fca2a19a95d67a2cdcaec6e1b72ccdc9cfafe0 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 15 Oct 2021 23:24:23 +0900 Subject: [PATCH 091/142] Update doc --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d919940a..1372e3ea 100644 --- a/README.rst +++ b/README.rst @@ -151,7 +151,9 @@ To enable the experimental HTTP/3 support for h2load and nghttpx, the following libraries are required: * `OpenSSL with QUIC support - `_ + `_; or + `BoringSSL `_ (commit + f6ef1c560ae5af51e2df5d8d2175bed207b28b8f) * `ngtcp2 `_ * `nghttp3 `_ From 693431312c7b425d4aec0ae0887e331b075571f7 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 15 Oct 2021 23:39:44 +0900 Subject: [PATCH 092/142] Fix cmake build --- CMakeLists.txt | 3 +++ cmakeconfig.h.in | 3 +++ 2 files changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index ecb90a23..a5061e03 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,6 +63,9 @@ find_package(Libcares 1.7.5) find_package(ZLIB 1.2.3) find_package(Libngtcp2 0.0.0) find_package(Libngtcp2_crypto_openssl 0.0.0) +if(LIBNGTCP2_CRYPTO_OPENSSL_FOUND) + set(HAVE_LIBNGTCP2_CRYPTO_OPENSSL 1) +endif() find_package(Libnghttp3 0.0.0) find_package(Libbpf 0.4.0) if(OPENSSL_FOUND AND LIBEV_FOUND AND ZLIB_FOUND) diff --git a/cmakeconfig.h.in b/cmakeconfig.h.in index 10bb76cb..2e357d53 100644 --- a/cmakeconfig.h.in +++ b/cmakeconfig.h.in @@ -87,3 +87,6 @@ /* Define to 1 if you have enum bpf_stats_type in linux/bpf.h. */ #cmakedefine HAVE_BPF_STATS_TYPE 1 + +/* Define to 1 if you have `libngtcp2_crypto_openssl` library. */ +#cmakedefine HAVE_LIBNGTCP2_CRYPTO_OPENSSL From 8c4fbb86d81d9ba8226e2e41d2e247cd1aa62c36 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 17 Oct 2021 14:00:37 +0900 Subject: [PATCH 093/142] Bump version number to 1.46.0, LT revision to 35:1:21 --- CMakeLists.txt | 4 ++-- configure.ac | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a5061e03..bfcd07d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,12 +24,12 @@ cmake_minimum_required(VERSION 3.0) # XXX using 1.8.90 instead of 1.9.0-DEV -project(nghttp2 VERSION 1.45.90) +project(nghttp2 VERSION 1.46.0) # See versioning rule: # https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html set(LT_CURRENT 35) -set(LT_REVISION 0) +set(LT_REVISION 1) set(LT_AGE 21) set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) diff --git a/configure.ac b/configure.ac index 671a9484..bec2aa35 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl https://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.46.0-DEV], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.46.0], [t-tujikawa@users.sourceforge.net]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_HEADERS([config.h]) @@ -45,7 +45,7 @@ m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) dnl See versioning rule: dnl https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html AC_SUBST(LT_CURRENT, 35) -AC_SUBST(LT_REVISION, 0) +AC_SUBST(LT_REVISION, 1) AC_SUBST(LT_AGE, 21) major=`echo $PACKAGE_VERSION |cut -d. -f1 | sed -e "s/[^0-9]//g"` From 0cc7c598ff8ac21222c4f6d7840ce441444a9056 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 17 Oct 2021 14:27:16 +0900 Subject: [PATCH 094/142] src: TLS_DEFAULT_CIPHERSUITES was deprecated in OpenSSL 3.0.0 --- src/tls.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/tls.h b/src/tls.h index 8b785ffc..2a6bf458 100644 --- a/src/tls.h +++ b/src/tls.h @@ -57,11 +57,15 @@ constexpr char DEFAULT_CIPHER_LIST[] = "AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256"; constexpr char DEFAULT_TLS13_CIPHER_LIST[] = -#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) +#if OPENSSL_3_0_0_API + "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256" +#elif OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) TLS_DEFAULT_CIPHERSUITES -#else // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) +#else // !OPENSSL_3_0_0_API && !(OPENSSL_1_1_1_API && + // !defined(OPENSSL_IS_BORINGSSL)) "" -#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) +#endif // !OPENSSL_3_0_0_API && !(OPENSSL_1_1_1_API && + // !defined(OPENSSL_IS_BORINGSSL)) ; constexpr auto NGHTTP2_TLS_MIN_VERSION = TLS1_VERSION; From 1745a306442a8811e3c0baa48c9953fdeab8abf4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 17 Oct 2021 14:28:20 +0900 Subject: [PATCH 095/142] Update manual pages --- doc/h2load.1 | 2 +- doc/nghttp.1 | 2 +- doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 49 ++++++++++++++++++++++++++++++++++++++++++++--- doc/nghttpx.1.rst | 43 +++++++++++++++++++++++++++++++++++++++-- 5 files changed, 90 insertions(+), 8 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index 8ebff2bb..26fbb49b 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "Sep 23, 2021" "1.46.0-DEV" "nghttp2" +.TH "H2LOAD" "1" "Oct 17, 2021" "1.46.0" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 8915a1fd..58423929 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "Sep 23, 2021" "1.46.0-DEV" "nghttp2" +.TH "NGHTTP" "1" "Oct 17, 2021" "1.46.0" "nghttp2" .SH NAME nghttp \- HTTP/2 client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index 505cf7bf..42b764f9 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "Sep 23, 2021" "1.46.0-DEV" "nghttp2" +.TH "NGHTTPD" "1" "Oct 17, 2021" "1.46.0" "nghttp2" .SH NAME nghttpd \- HTTP/2 server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index 09d31cb0..a05490ea 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "Sep 23, 2021" "1.46.0-DEV" "nghttp2" +.TH "NGHTTPX" "1" "Oct 17, 2021" "1.46.0" "nghttp2" .SH NAME nghttpx \- HTTP/2 proxy . @@ -503,6 +503,15 @@ Default: \fB0\fP .UNINDENT .INDENT 0.0 .TP +.B \-\-rlimit\-memlock= +Set maximum number of bytes of memory that may be locked +into RAM. If 0 is given, nghttpx does not set the +limit. +.sp +Default: \fB0\fP +.UNINDENT +.INDENT 0.0 +.TP .B \-\-backend\-request\-buffer= Set buffer size used to store backend request. .sp @@ -1705,6 +1714,33 @@ process. nghttpx still spawns additional process if neverbleed is used. In the single process mode, the signal handling feature is disabled. .UNINDENT +.INDENT 0.0 +.TP +.B \-\-max\-worker\-processes= +The maximum number of worker processes. nghttpx spawns +new worker process when it reloads its configuration. +The previous worker process enters graceful termination +period and will terminate when it finishes handling the +existing connections. However, if reloading +configurations happen very frequently, the worker +processes might be piled up if they take a bit long time +to finish the existing connections. With this option, +if the number of worker processes exceeds the given +value, the oldest worker process is terminated +immediately. Specifying 0 means no limit and it is the +default behaviour. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-worker\-process\-grace\-shutdown\-period= +Maximum period for a worker process to terminate +gracefully. When a worker process enters in graceful +shutdown period (e.g., when nghttpx reloads its +configuration) and it does not finish handling the +existing connections in the given period of time, it is +immediately terminated. Specifying 0 means no limit and +it is the default behaviour. +.UNINDENT .SS Scripting .INDENT 0.0 .TP @@ -1808,13 +1844,20 @@ internally on startup and reload. Specify server ID encoded in Connection ID to identify this particular server instance. Connection ID is encrypted and this part is not visible in public. It -must be 2 bytes long and must be encoded in hex string -(which is 4 bytes long). If this option is omitted, a +must be 4 bytes long and must be encoded in hex string +(which is 8 bytes long). If this option is omitted, a random server ID is generated on startup and configuration reload. .UNINDENT .INDENT 0.0 .TP +.B \-\-frontend\-quic\-initial\-rtt= +Specify the initial RTT of the frontend QUIC connection. +.sp +Default: \fB333ms\fP +.UNINDENT +.INDENT 0.0 +.TP .B \-\-no\-quic\-bpf Disable eBPF. .UNINDENT diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index b5e25ee2..dd22f261 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -472,6 +472,14 @@ Performance Default: ``0`` +.. option:: --rlimit-memlock= + + Set maximum number of bytes of memory that may be locked + into RAM. If 0 is given, nghttpx does not set the + limit. + + Default: ``0`` + .. option:: --backend-request-buffer= Set buffer size used to store backend request. @@ -1554,6 +1562,31 @@ Process neverbleed is used. In the single process mode, the signal handling feature is disabled. +.. option:: --max-worker-processes= + + The maximum number of worker processes. nghttpx spawns + new worker process when it reloads its configuration. + The previous worker process enters graceful termination + period and will terminate when it finishes handling the + existing connections. However, if reloading + configurations happen very frequently, the worker + processes might be piled up if they take a bit long time + to finish the existing connections. With this option, + if the number of worker processes exceeds the given + value, the oldest worker process is terminated + immediately. Specifying 0 means no limit and it is the + default behaviour. + +.. option:: --worker-process-grace-shutdown-period= + + Maximum period for a worker process to terminate + gracefully. When a worker process enters in graceful + shutdown period (e.g., when nghttpx reloads its + configuration) and it does not finish handling the + existing connections in the given period of time, it is + immediately terminated. Specifying 0 means no limit and + it is the default behaviour. + Scripting ~~~~~~~~~ @@ -1652,11 +1685,17 @@ HTTP/3 and QUIC Specify server ID encoded in Connection ID to identify this particular server instance. Connection ID is encrypted and this part is not visible in public. It - must be 2 bytes long and must be encoded in hex string - (which is 4 bytes long). If this option is omitted, a + must be 4 bytes long and must be encoded in hex string + (which is 8 bytes long). If this option is omitted, a random server ID is generated on startup and configuration reload. +.. option:: --frontend-quic-initial-rtt= + + Specify the initial RTT of the frontend QUIC connection. + + Default: ``333ms`` + .. option:: --no-quic-bpf Disable eBPF. From 18d4a9e4ff5442b1108f9f07e307539ca2fa3a85 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 17 Oct 2021 14:28:38 +0900 Subject: [PATCH 096/142] Update bash_completion --- doc/bash_completion/nghttpx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bash_completion/nghttpx b/doc/bash_completion/nghttpx index e58c1269..04e4a225 100644 --- a/doc/bash_completion/nghttpx +++ b/doc/bash_completion/nghttpx @@ -8,7 +8,7 @@ _nghttpx() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--backend --frontend --backlog --backend-address-family --backend-http-proxy-uri --workers --single-thread --read-rate --read-burst --write-rate --write-burst --worker-read-rate --worker-read-burst --worker-write-rate --worker-write-burst --worker-frontend-connections --backend-connections-per-host --backend-connections-per-frontend --rlimit-nofile --backend-request-buffer --backend-response-buffer --fastopen --no-kqueue --frontend-http2-read-timeout --frontend-http3-read-timeout --frontend-read-timeout --frontend-write-timeout --frontend-keep-alive-timeout --stream-read-timeout --stream-write-timeout --backend-read-timeout --backend-write-timeout --backend-connect-timeout --backend-keep-alive-timeout --listener-disable-timeout --frontend-http2-setting-timeout --backend-http2-settings-timeout --backend-max-backoff --ciphers --tls13-ciphers --client-ciphers --tls13-client-ciphers --ecdh-curves --insecure --cacert --private-key-passwd-file --subcert --dh-param-file --npn-list --verify-client --verify-client-cacert --verify-client-tolerate-expired --client-private-key-file --client-cert-file --tls-min-proto-version --tls-max-proto-version --tls-ticket-key-file --tls-ticket-key-memcached --tls-ticket-key-memcached-address-family --tls-ticket-key-memcached-interval --tls-ticket-key-memcached-max-retry --tls-ticket-key-memcached-max-fail --tls-ticket-key-cipher --tls-ticket-key-memcached-cert-file --tls-ticket-key-memcached-private-key-file --fetch-ocsp-response-file --ocsp-update-interval --ocsp-startup --no-verify-ocsp --no-ocsp --tls-session-cache-memcached --tls-session-cache-memcached-address-family --tls-session-cache-memcached-cert-file --tls-session-cache-memcached-private-key-file --tls-dyn-rec-warmup-threshold --tls-dyn-rec-idle-timeout --no-http2-cipher-block-list --client-no-http2-cipher-block-list --tls-sct-dir --psk-secrets --client-psk-secrets --tls-no-postpone-early-data --tls-max-early-data --frontend-http2-max-concurrent-streams --backend-http2-max-concurrent-streams --frontend-http2-window-size --frontend-http2-connection-window-size --backend-http2-window-size --backend-http2-connection-window-size --http2-no-cookie-crumbling --padding --no-server-push --frontend-http2-optimize-write-buffer-size --frontend-http2-optimize-window-size --frontend-http2-encoder-dynamic-table-size --frontend-http2-decoder-dynamic-table-size --backend-http2-encoder-dynamic-table-size --backend-http2-decoder-dynamic-table-size --http2-proxy --log-level --accesslog-file --accesslog-syslog --accesslog-format --accesslog-write-early --errorlog-file --errorlog-syslog --syslog-facility --add-x-forwarded-for --strip-incoming-x-forwarded-for --no-add-x-forwarded-proto --no-strip-incoming-x-forwarded-proto --add-forwarded --strip-incoming-forwarded --forwarded-by --forwarded-for --no-via --no-strip-incoming-early-data --no-location-rewrite --host-rewrite --altsvc --http2-altsvc --add-request-header --add-response-header --request-header-field-buffer --max-request-header-fields --response-header-field-buffer --max-response-header-fields --error-page --server-name --no-server-rewrite --redirect-https-port --api-max-request-body --dns-cache-timeout --dns-lookup-timeout --dns-max-try --frontend-max-requests --frontend-http2-dump-request-header --frontend-http2-dump-response-header --frontend-frame-debug --daemon --pid-file --user --single-process --mruby-file --ignore-per-pattern-mruby-error --frontend-quic-idle-timeout --frontend-quic-debug-log --quic-bpf-program-file --frontend-quic-early-data --frontend-quic-qlog-dir --frontend-quic-require-token --frontend-quic-congestion-controller --frontend-quic-secret-file --frontend-quic-server-id --no-quic-bpf --frontend-http3-window-size --frontend-http3-connection-window-size --frontend-http3-max-window-size --frontend-http3-max-connection-window-size --frontend-http3-max-concurrent-streams --conf --include --version --help ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--backend --frontend --backlog --backend-address-family --backend-http-proxy-uri --workers --single-thread --read-rate --read-burst --write-rate --write-burst --worker-read-rate --worker-read-burst --worker-write-rate --worker-write-burst --worker-frontend-connections --backend-connections-per-host --backend-connections-per-frontend --rlimit-nofile --rlimit-memlock --backend-request-buffer --backend-response-buffer --fastopen --no-kqueue --frontend-http2-read-timeout --frontend-http3-read-timeout --frontend-read-timeout --frontend-write-timeout --frontend-keep-alive-timeout --stream-read-timeout --stream-write-timeout --backend-read-timeout --backend-write-timeout --backend-connect-timeout --backend-keep-alive-timeout --listener-disable-timeout --frontend-http2-setting-timeout --backend-http2-settings-timeout --backend-max-backoff --ciphers --tls13-ciphers --client-ciphers --tls13-client-ciphers --ecdh-curves --insecure --cacert --private-key-passwd-file --subcert --dh-param-file --npn-list --verify-client --verify-client-cacert --verify-client-tolerate-expired --client-private-key-file --client-cert-file --tls-min-proto-version --tls-max-proto-version --tls-ticket-key-file --tls-ticket-key-memcached --tls-ticket-key-memcached-address-family --tls-ticket-key-memcached-interval --tls-ticket-key-memcached-max-retry --tls-ticket-key-memcached-max-fail --tls-ticket-key-cipher --tls-ticket-key-memcached-cert-file --tls-ticket-key-memcached-private-key-file --fetch-ocsp-response-file --ocsp-update-interval --ocsp-startup --no-verify-ocsp --no-ocsp --tls-session-cache-memcached --tls-session-cache-memcached-address-family --tls-session-cache-memcached-cert-file --tls-session-cache-memcached-private-key-file --tls-dyn-rec-warmup-threshold --tls-dyn-rec-idle-timeout --no-http2-cipher-block-list --client-no-http2-cipher-block-list --tls-sct-dir --psk-secrets --client-psk-secrets --tls-no-postpone-early-data --tls-max-early-data --frontend-http2-max-concurrent-streams --backend-http2-max-concurrent-streams --frontend-http2-window-size --frontend-http2-connection-window-size --backend-http2-window-size --backend-http2-connection-window-size --http2-no-cookie-crumbling --padding --no-server-push --frontend-http2-optimize-write-buffer-size --frontend-http2-optimize-window-size --frontend-http2-encoder-dynamic-table-size --frontend-http2-decoder-dynamic-table-size --backend-http2-encoder-dynamic-table-size --backend-http2-decoder-dynamic-table-size --http2-proxy --log-level --accesslog-file --accesslog-syslog --accesslog-format --accesslog-write-early --errorlog-file --errorlog-syslog --syslog-facility --add-x-forwarded-for --strip-incoming-x-forwarded-for --no-add-x-forwarded-proto --no-strip-incoming-x-forwarded-proto --add-forwarded --strip-incoming-forwarded --forwarded-by --forwarded-for --no-via --no-strip-incoming-early-data --no-location-rewrite --host-rewrite --altsvc --http2-altsvc --add-request-header --add-response-header --request-header-field-buffer --max-request-header-fields --response-header-field-buffer --max-response-header-fields --error-page --server-name --no-server-rewrite --redirect-https-port --api-max-request-body --dns-cache-timeout --dns-lookup-timeout --dns-max-try --frontend-max-requests --frontend-http2-dump-request-header --frontend-http2-dump-response-header --frontend-frame-debug --daemon --pid-file --user --single-process --max-worker-processes --worker-process-grace-shutdown-period --mruby-file --ignore-per-pattern-mruby-error --frontend-quic-idle-timeout --frontend-quic-debug-log --quic-bpf-program-file --frontend-quic-early-data --frontend-quic-qlog-dir --frontend-quic-require-token --frontend-quic-congestion-controller --frontend-quic-secret-file --frontend-quic-server-id --frontend-quic-initial-rtt --no-quic-bpf --frontend-http3-window-size --frontend-http3-connection-window-size --frontend-http3-max-window-size --frontend-http3-max-connection-window-size --frontend-http3-max-concurrent-streams --conf --include --version --help ' -- "$cur" ) ) ;; *) _filedir From 9d418966633a07ac7339ac50569e40c9f8d17d58 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 17 Oct 2021 14:48:46 +0900 Subject: [PATCH 097/142] nghttpx: Rename --frontend-quic-server-id to --quic-server-id --- gennghttpxfun.py | 2 +- src/shrpx.cc | 24 ++++++++++-------------- src/shrpx_config.cc | 16 ++++++++-------- src/shrpx_config.h | 7 +++---- src/shrpx_quic_connection_handler.cc | 6 +++--- 5 files changed, 25 insertions(+), 30 deletions(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 6f88456e..a388b119 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -192,7 +192,7 @@ OPTIONS = [ "frontend-quic-qlog-dir", "frontend-quic-require-token", "frontend-quic-congestion-controller", - "frontend-quic-server-id", + "quic-server-id", "frontend-quic-secret-file", "rlimit-memlock", "max-worker-processes", diff --git a/src/shrpx.cc b/src/shrpx.cc index c252f61f..14427f92 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1450,8 +1450,7 @@ int generate_cid_prefix( cid_prefixes.resize(num_cid_prefix); for (auto &cid_prefix : cid_prefixes) { - if (create_cid_prefix(cid_prefix.data(), - quicconf.upstream.server_id.data()) != 0) { + if (create_cid_prefix(cid_prefix.data(), quicconf.server_id.data()) != 0) { return -1; } } @@ -1953,16 +1952,15 @@ void fill_default_config(Config *config) { upstreamconf.congestion_controller = NGTCP2_CC_ALGO_CUBIC; - if (RAND_bytes(upstreamconf.server_id.data(), - upstreamconf.server_id.size()) != 1) { - assert(0); - abort(); - } - upstreamconf.initial_rtt = static_cast(NGTCP2_DEFAULT_INITIAL_RTT) / NGTCP2_SECONDS; } + if (RAND_bytes(quicconf.server_id.data(), quicconf.server_id.size()) != 1) { + assert(0); + abort(); + } + auto &http3conf = config->http3; { auto &upstreamconf = http3conf.upstream; @@ -3394,7 +3392,7 @@ HTTP/3 and QUIC: option is not given or an error occurred while opening or reading a file, a keying material is generated internally on startup and reload. - --frontend-quic-server-id= + --quic-server-id= Specify server ID encoded in Connection ID to identify this particular server instance. Connection ID is encrypted and this part is not visible in public. It @@ -4217,8 +4215,7 @@ int main(int argc, char **argv) { 182}, {SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER.c_str(), required_argument, &flag, 183}, - {SHRPX_OPT_FRONTEND_QUIC_SERVER_ID.c_str(), required_argument, &flag, - 185}, + {SHRPX_OPT_QUIC_SERVER_ID.c_str(), required_argument, &flag, 185}, {SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE.c_str(), required_argument, &flag, 186}, {SHRPX_OPT_RLIMIT_MEMLOCK.c_str(), required_argument, &flag, 187}, @@ -5105,9 +5102,8 @@ int main(int argc, char **argv) { StringRef{optarg}); break; case 185: - // --frontend-quic-server-id - cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_SERVER_ID, - StringRef{optarg}); + // --quic-server-id + cmdcfgs.emplace_back(SHRPX_OPT_QUIC_SERVER_ID, StringRef{optarg}); break; case 186: // --frontend-quic-secret-file diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 9c5d9917..b8acbe50 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2054,6 +2054,11 @@ int option_lookup_token(const char *name, size_t namelen) { break; case 14: switch (name[13]) { + case 'd': + if (util::strieq_l("quic-server-i", name, 13)) { + return SHRPX_OPTID_QUIC_SERVER_ID; + } + break; case 'e': if (util::strieq_l("accesslog-fil", name, 13)) { return SHRPX_OPTID_ACCESSLOG_FILE; @@ -2352,11 +2357,6 @@ int option_lookup_token(const char *name, size_t namelen) { break; case 23: switch (name[22]) { - case 'd': - if (util::strieq_l("frontend-quic-server-i", name, 22)) { - return SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID; - } - break; case 'e': if (util::strieq_l("client-private-key-fil", name, 22)) { return SHRPX_OPTID_CLIENT_PRIVATE_KEY_FILE; @@ -4118,14 +4118,14 @@ int parse_config(Config *config, int optid, const StringRef &opt, #endif // ENABLE_HTTP3 return 0; - case SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID: + case SHRPX_OPTID_QUIC_SERVER_ID: #ifdef ENABLE_HTTP3 - if (optarg.size() != config->quic.upstream.server_id.size() * 2 || + if (optarg.size() != config->quic.server_id.size() * 2 || !util::is_hex_string(optarg)) { LOG(ERROR) << opt << ": must be a hex-string"; return -1; } - util::decode_hex(std::begin(config->quic.upstream.server_id), optarg); + util::decode_hex(std::begin(config->quic.server_id), optarg); #endif // ENABLE_HTTP3 return 0; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 27b6a9ac..d88ac7a9 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -391,8 +391,7 @@ constexpr auto SHRPX_OPT_FRONTEND_QUIC_REQUIRE_TOKEN = StringRef::from_lit("frontend-quic-require-token"); constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER = StringRef::from_lit("frontend-quic-congestion-controller"); -constexpr auto SHRPX_OPT_FRONTEND_QUIC_SERVER_ID = - StringRef::from_lit("frontend-quic-server-id"); +constexpr auto SHRPX_OPT_QUIC_SERVER_ID = StringRef::from_lit("quic-server-id"); constexpr auto SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE = StringRef::from_lit("frontend-quic-secret-file"); constexpr auto SHRPX_OPT_RLIMIT_MEMLOCK = StringRef::from_lit("rlimit-memlock"); @@ -780,7 +779,6 @@ struct QUICConfig { ngtcp2_cc_algo congestion_controller; bool early_data; bool require_token; - std::array server_id; StringRef secret_file; ev_tstamp initial_rtt; } upstream; @@ -788,6 +786,7 @@ struct QUICConfig { StringRef prog_file; bool disabled; } bpf; + std::array server_id; }; struct Http3Config { @@ -1249,7 +1248,6 @@ enum { SHRPX_OPTID_FRONTEND_QUIC_QLOG_DIR, SHRPX_OPTID_FRONTEND_QUIC_REQUIRE_TOKEN, SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE, - SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID, SHRPX_OPTID_FRONTEND_READ_TIMEOUT, SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT, SHRPX_OPTID_HEADER_FIELD_BUFFER, @@ -1292,6 +1290,7 @@ enum { SHRPX_OPTID_PRIVATE_KEY_PASSWD_FILE, SHRPX_OPTID_PSK_SECRETS, SHRPX_OPTID_QUIC_BPF_PROGRAM_FILE, + SHRPX_OPTID_QUIC_SERVER_ID, SHRPX_OPTID_READ_BURST, SHRPX_OPTID_READ_RATE, SHRPX_OPTID_REDIRECT_HTTPS_PORT, diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 730c649f..325e6957 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -456,9 +456,9 @@ int QUICConnectionHandler::send_retry( ngtcp2_cid retry_scid; - if (generate_quic_retry_connection_id( - retry_scid, SHRPX_QUIC_SCIDLEN, quicconf.upstream.server_id.data(), - qkm.id, qkm.cid_encryption_key.data()) != 0) { + if (generate_quic_retry_connection_id(retry_scid, SHRPX_QUIC_SCIDLEN, + quicconf.server_id.data(), qkm.id, + qkm.cid_encryption_key.data()) != 0) { return -1; } From 8ecacc8ed290353cd4414d6a1e70757b0a0c6309 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 17 Oct 2021 14:52:36 +0900 Subject: [PATCH 098/142] Update manual pages --- doc/nghttpx.1 | 2 +- doc/nghttpx.1.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index a05490ea..d1f8ceaf 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1840,7 +1840,7 @@ internally on startup and reload. .UNINDENT .INDENT 0.0 .TP -.B \-\-frontend\-quic\-server\-id= +.B \-\-quic\-server\-id= Specify server ID encoded in Connection ID to identify this particular server instance. Connection ID is encrypted and this part is not visible in public. It diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index dd22f261..60374068 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -1680,7 +1680,7 @@ HTTP/3 and QUIC or reading a file, a keying material is generated internally on startup and reload. -.. option:: --frontend-quic-server-id= +.. option:: --quic-server-id= Specify server ID encoded in Connection ID to identify this particular server instance. Connection ID is From ba1dff187b902f2d09b184b66fc278501eb4656e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 17 Oct 2021 14:52:55 +0900 Subject: [PATCH 099/142] Update bash_completion --- doc/bash_completion/nghttpx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bash_completion/nghttpx b/doc/bash_completion/nghttpx index 04e4a225..89371abe 100644 --- a/doc/bash_completion/nghttpx +++ b/doc/bash_completion/nghttpx @@ -8,7 +8,7 @@ _nghttpx() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--backend --frontend --backlog --backend-address-family --backend-http-proxy-uri --workers --single-thread --read-rate --read-burst --write-rate --write-burst --worker-read-rate --worker-read-burst --worker-write-rate --worker-write-burst --worker-frontend-connections --backend-connections-per-host --backend-connections-per-frontend --rlimit-nofile --rlimit-memlock --backend-request-buffer --backend-response-buffer --fastopen --no-kqueue --frontend-http2-read-timeout --frontend-http3-read-timeout --frontend-read-timeout --frontend-write-timeout --frontend-keep-alive-timeout --stream-read-timeout --stream-write-timeout --backend-read-timeout --backend-write-timeout --backend-connect-timeout --backend-keep-alive-timeout --listener-disable-timeout --frontend-http2-setting-timeout --backend-http2-settings-timeout --backend-max-backoff --ciphers --tls13-ciphers --client-ciphers --tls13-client-ciphers --ecdh-curves --insecure --cacert --private-key-passwd-file --subcert --dh-param-file --npn-list --verify-client --verify-client-cacert --verify-client-tolerate-expired --client-private-key-file --client-cert-file --tls-min-proto-version --tls-max-proto-version --tls-ticket-key-file --tls-ticket-key-memcached --tls-ticket-key-memcached-address-family --tls-ticket-key-memcached-interval --tls-ticket-key-memcached-max-retry --tls-ticket-key-memcached-max-fail --tls-ticket-key-cipher --tls-ticket-key-memcached-cert-file --tls-ticket-key-memcached-private-key-file --fetch-ocsp-response-file --ocsp-update-interval --ocsp-startup --no-verify-ocsp --no-ocsp --tls-session-cache-memcached --tls-session-cache-memcached-address-family --tls-session-cache-memcached-cert-file --tls-session-cache-memcached-private-key-file --tls-dyn-rec-warmup-threshold --tls-dyn-rec-idle-timeout --no-http2-cipher-block-list --client-no-http2-cipher-block-list --tls-sct-dir --psk-secrets --client-psk-secrets --tls-no-postpone-early-data --tls-max-early-data --frontend-http2-max-concurrent-streams --backend-http2-max-concurrent-streams --frontend-http2-window-size --frontend-http2-connection-window-size --backend-http2-window-size --backend-http2-connection-window-size --http2-no-cookie-crumbling --padding --no-server-push --frontend-http2-optimize-write-buffer-size --frontend-http2-optimize-window-size --frontend-http2-encoder-dynamic-table-size --frontend-http2-decoder-dynamic-table-size --backend-http2-encoder-dynamic-table-size --backend-http2-decoder-dynamic-table-size --http2-proxy --log-level --accesslog-file --accesslog-syslog --accesslog-format --accesslog-write-early --errorlog-file --errorlog-syslog --syslog-facility --add-x-forwarded-for --strip-incoming-x-forwarded-for --no-add-x-forwarded-proto --no-strip-incoming-x-forwarded-proto --add-forwarded --strip-incoming-forwarded --forwarded-by --forwarded-for --no-via --no-strip-incoming-early-data --no-location-rewrite --host-rewrite --altsvc --http2-altsvc --add-request-header --add-response-header --request-header-field-buffer --max-request-header-fields --response-header-field-buffer --max-response-header-fields --error-page --server-name --no-server-rewrite --redirect-https-port --api-max-request-body --dns-cache-timeout --dns-lookup-timeout --dns-max-try --frontend-max-requests --frontend-http2-dump-request-header --frontend-http2-dump-response-header --frontend-frame-debug --daemon --pid-file --user --single-process --max-worker-processes --worker-process-grace-shutdown-period --mruby-file --ignore-per-pattern-mruby-error --frontend-quic-idle-timeout --frontend-quic-debug-log --quic-bpf-program-file --frontend-quic-early-data --frontend-quic-qlog-dir --frontend-quic-require-token --frontend-quic-congestion-controller --frontend-quic-secret-file --frontend-quic-server-id --frontend-quic-initial-rtt --no-quic-bpf --frontend-http3-window-size --frontend-http3-connection-window-size --frontend-http3-max-window-size --frontend-http3-max-connection-window-size --frontend-http3-max-concurrent-streams --conf --include --version --help ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--backend --frontend --backlog --backend-address-family --backend-http-proxy-uri --workers --single-thread --read-rate --read-burst --write-rate --write-burst --worker-read-rate --worker-read-burst --worker-write-rate --worker-write-burst --worker-frontend-connections --backend-connections-per-host --backend-connections-per-frontend --rlimit-nofile --rlimit-memlock --backend-request-buffer --backend-response-buffer --fastopen --no-kqueue --frontend-http2-read-timeout --frontend-http3-read-timeout --frontend-read-timeout --frontend-write-timeout --frontend-keep-alive-timeout --stream-read-timeout --stream-write-timeout --backend-read-timeout --backend-write-timeout --backend-connect-timeout --backend-keep-alive-timeout --listener-disable-timeout --frontend-http2-setting-timeout --backend-http2-settings-timeout --backend-max-backoff --ciphers --tls13-ciphers --client-ciphers --tls13-client-ciphers --ecdh-curves --insecure --cacert --private-key-passwd-file --subcert --dh-param-file --npn-list --verify-client --verify-client-cacert --verify-client-tolerate-expired --client-private-key-file --client-cert-file --tls-min-proto-version --tls-max-proto-version --tls-ticket-key-file --tls-ticket-key-memcached --tls-ticket-key-memcached-address-family --tls-ticket-key-memcached-interval --tls-ticket-key-memcached-max-retry --tls-ticket-key-memcached-max-fail --tls-ticket-key-cipher --tls-ticket-key-memcached-cert-file --tls-ticket-key-memcached-private-key-file --fetch-ocsp-response-file --ocsp-update-interval --ocsp-startup --no-verify-ocsp --no-ocsp --tls-session-cache-memcached --tls-session-cache-memcached-address-family --tls-session-cache-memcached-cert-file --tls-session-cache-memcached-private-key-file --tls-dyn-rec-warmup-threshold --tls-dyn-rec-idle-timeout --no-http2-cipher-block-list --client-no-http2-cipher-block-list --tls-sct-dir --psk-secrets --client-psk-secrets --tls-no-postpone-early-data --tls-max-early-data --frontend-http2-max-concurrent-streams --backend-http2-max-concurrent-streams --frontend-http2-window-size --frontend-http2-connection-window-size --backend-http2-window-size --backend-http2-connection-window-size --http2-no-cookie-crumbling --padding --no-server-push --frontend-http2-optimize-write-buffer-size --frontend-http2-optimize-window-size --frontend-http2-encoder-dynamic-table-size --frontend-http2-decoder-dynamic-table-size --backend-http2-encoder-dynamic-table-size --backend-http2-decoder-dynamic-table-size --http2-proxy --log-level --accesslog-file --accesslog-syslog --accesslog-format --accesslog-write-early --errorlog-file --errorlog-syslog --syslog-facility --add-x-forwarded-for --strip-incoming-x-forwarded-for --no-add-x-forwarded-proto --no-strip-incoming-x-forwarded-proto --add-forwarded --strip-incoming-forwarded --forwarded-by --forwarded-for --no-via --no-strip-incoming-early-data --no-location-rewrite --host-rewrite --altsvc --http2-altsvc --add-request-header --add-response-header --request-header-field-buffer --max-request-header-fields --response-header-field-buffer --max-response-header-fields --error-page --server-name --no-server-rewrite --redirect-https-port --api-max-request-body --dns-cache-timeout --dns-lookup-timeout --dns-max-try --frontend-max-requests --frontend-http2-dump-request-header --frontend-http2-dump-response-header --frontend-frame-debug --daemon --pid-file --user --single-process --max-worker-processes --worker-process-grace-shutdown-period --mruby-file --ignore-per-pattern-mruby-error --frontend-quic-idle-timeout --frontend-quic-debug-log --quic-bpf-program-file --frontend-quic-early-data --frontend-quic-qlog-dir --frontend-quic-require-token --frontend-quic-congestion-controller --frontend-quic-secret-file --quic-server-id --frontend-quic-initial-rtt --no-quic-bpf --frontend-http3-window-size --frontend-http3-connection-window-size --frontend-http3-max-window-size --frontend-http3-max-connection-window-size --frontend-http3-max-concurrent-streams --conf --include --version --help ' -- "$cur" ) ) ;; *) _filedir From 8c36971ea9679f1e7b05a4a3e6db654a6d9d9019 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 17 Oct 2021 16:41:10 +0900 Subject: [PATCH 100/142] Compile with OPENSSL_NO_DEPRECATED and fix memory leaks --- examples/client.c | 10 ++++++++ examples/libevent-client.c | 10 ++++++++ examples/libevent-server.c | 21 ++++++++++------- src/HttpServer.cc | 26 ++++++++------------- src/shrpx_log.cc | 18 ++++++++++++++ src/shrpx_mruby_module_env.cc | 44 +++++++++++++++++++++++++++++++++-- src/shrpx_tls.cc | 20 ++++++++++++---- 7 files changed, 119 insertions(+), 30 deletions(-) diff --git a/examples/client.c b/examples/client.c index 2d3c4c5d..22f585f7 100644 --- a/examples/client.c +++ b/examples/client.c @@ -715,8 +715,18 @@ int main(int argc, char **argv) { act.sa_handler = SIG_IGN; sigaction(SIGPIPE, &act, 0); +#if OPENSSL_VERSION_NUMBER >= 0x1010000fL + /* No explicit initialization is required. */ +#elif defined(OPENSSL_IS_BORINGSSL) + CRYPTO_library_init(); +#else /* !(OPENSSL_VERSION_NUMBER >= 0x1010000fL) && \ + !defined(OPENSSL_IS_BORINGSSL) */ + OPENSSL_config(NULL); SSL_load_error_strings(); SSL_library_init(); + OpenSSL_add_all_algorithms(); +#endif /* !(OPENSSL_VERSION_NUMBER >= 0x1010000fL) && \ + !defined(OPENSSL_IS_BORINGSSL) */ rv = parse_uri(&uri, argv[1]); if (rv != 0) { diff --git a/examples/libevent-client.c b/examples/libevent-client.c index f42cbdb3..e7c78cf5 100644 --- a/examples/libevent-client.c +++ b/examples/libevent-client.c @@ -617,8 +617,18 @@ int main(int argc, char **argv) { act.sa_handler = SIG_IGN; sigaction(SIGPIPE, &act, NULL); +#if OPENSSL_VERSION_NUMBER >= 0x1010000fL + /* No explicit initialization is required. */ +#elif defined(OPENSSL_IS_BORINGSSL) + CRYPTO_library_init(); +#else /* !(OPENSSL_VERSION_NUMBER >= 0x1010000fL) && \ + !defined(OPENSSL_IS_BORINGSSL) */ + OPENSSL_config(NULL); SSL_load_error_strings(); SSL_library_init(); + OpenSSL_add_all_algorithms(); +#endif /* !(OPENSSL_VERSION_NUMBER >= 0x1010000fL) && \ + !defined(OPENSSL_IS_BORINGSSL) */ run(argv[1]); return 0; diff --git a/examples/libevent-server.c b/examples/libevent-server.c index 0465a785..a30a5e89 100644 --- a/examples/libevent-server.c +++ b/examples/libevent-server.c @@ -153,14 +153,9 @@ static SSL_CTX *create_ssl_ctx(const char *key_file, const char *cert_file) { SSL_OP_NO_COMPRESSION | SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION); #if OPENSSL_VERSION_NUMBER >= 0x30000000L - { - EVP_PKEY *ecdh; - ecdh = EVP_EC_gen("P-256"); - if (!ecdh) { - errx(1, "EVP_EC_gen failed: %s", ERR_error_string(ERR_get_error(), NULL)); - } - SSL_CTX_set_tmp_ecdh(ssl_ctx, ecdh); - EVP_PKEY_free(ecdh); + if (SSL_CTX_set1_curves_list(ssl_ctx, "P-256") != 1) { + errx(1, "SSL_CTX_set1_curves_list failed: %s", + ERR_error_string(ERR_get_error(), NULL)); } #else /* !(OPENSSL_VERSION_NUMBER >= 0x30000000L) */ { @@ -822,8 +817,18 @@ int main(int argc, char **argv) { act.sa_handler = SIG_IGN; sigaction(SIGPIPE, &act, NULL); +#if OPENSSL_VERSION_NUMBER >= 0x1010000fL + /* No explicit initialization is required. */ +#elif defined(OPENSSL_IS_BORINGSSL) + CRYPTO_library_init(); +#else /* !(OPENSSL_VERSION_NUMBER >= 0x1010000fL) && \ + !defined(OPENSSL_IS_BORINGSSL) */ + OPENSSL_config(NULL); SSL_load_error_strings(); SSL_library_init(); + OpenSSL_add_all_algorithms(); +#endif /* !(OPENSSL_VERSION_NUMBER >= 0x1010000fL) && \ + !defined(OPENSSL_IS_BORINGSSL) */ run(argv[1], argv[2], argv[3]); return 0; diff --git a/src/HttpServer.cc b/src/HttpServer.cc index 67bc87aa..dacb4b15 100644 --- a/src/HttpServer.cc +++ b/src/HttpServer.cc @@ -2143,22 +2143,13 @@ int HttpServer::run() { SSL_CTX_set_session_cache_mode(ssl_ctx, SSL_SESS_CACHE_SERVER); #ifndef OPENSSL_NO_EC - // Disabled SSL_CTX_set_ecdh_auto, because computational cost of - // chosen curve is much higher than P-256. - - // SSL_CTX_set_ecdh_auto(ssl_ctx, 1); - // Use P-256, which is sufficiently secure at the time of this - // writing. -# if OPENSSL_3_0_0_API - auto ecdh = EVP_EC_gen("P-256"); - if (ecdh == nullptr) { - std::cerr << "EC_KEY_new_by_curv_name failed: " +# if !LIBRESSL_LEGACY_API && OPENSSL_VERSION_NUMBER >= 0x10002000L + if (SSL_CTX_set1_curves_list(ssl_ctx, "P-256") != 1) { + std::cerr << "SSL_CTX_set1_curves_list failed: " << ERR_error_string(ERR_get_error(), nullptr); return -1; } - SSL_CTX_set_tmp_ecdh(ssl_ctx, ecdh); - EVP_PKEY_free(ecdh); -# else // !OPENSSL_3_0_0_API +# else // !(!LIBRESSL_LEGACY_API && OPENSSL_VERSION_NUMBER >= 0x10002000L) auto ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1); if (ecdh == nullptr) { std::cerr << "EC_KEY_new_by_curv_name failed: " @@ -2167,7 +2158,7 @@ int HttpServer::run() { } SSL_CTX_set_tmp_ecdh(ssl_ctx, ecdh); EC_KEY_free(ecdh); -# endif // !OPENSSL_3_0_0_API +# endif // !(!LIBRESSL_LEGACY_API && OPENSSL_VERSION_NUMBER >= 0x10002000L) #endif // OPENSSL_NO_EC if (!config_->dh_param_file.empty()) { @@ -2191,8 +2182,11 @@ int HttpServer::run() { return -1; } - SSL_CTX_set_tmp_dh(ssl_ctx, dh); - EVP_PKEY_free(dh); + if (SSL_CTX_set0_tmp_dh_pkey(ssl_ctx, dh) != 1) { + std::cerr << "SSL_CTX_set0_tmp_dh_pkey failed: " + << ERR_error_string(ERR_get_error(), nullptr) << std::endl; + return -1; + } #else // !OPENSSL_3_0_0_API auto dh = PEM_read_bio_DHparams(bio, nullptr, nullptr, nullptr); diff --git a/src/shrpx_log.cc b/src/shrpx_log.cc index 3a5af823..fe626d6b 100644 --- a/src/shrpx_log.cc +++ b/src/shrpx_log.cc @@ -755,7 +755,11 @@ void upstream_accesslog(const std::vector &lfv, std::tie(p, last) = copy('-', p, last); break; } +#if OPENSSL_3_0_0_API + auto x = SSL_get0_peer_certificate(lgsp.ssl); +#else // !OPENSSL_3_0_0_API auto x = SSL_get_peer_certificate(lgsp.ssl); +#endif // !OPENSSL_3_0_0_API if (!x) { std::tie(p, last) = copy('-', p, last); break; @@ -766,7 +770,9 @@ void upstream_accesslog(const std::vector &lfv, lf.type == LogFragmentType::TLS_CLIENT_FINGERPRINT_SHA256 ? EVP_sha256() : EVP_sha1()); +#if !OPENSSL_3_0_0_API X509_free(x); +#endif // !OPENSSL_3_0_0_API if (len <= 0) { std::tie(p, last) = copy('-', p, last); break; @@ -780,7 +786,11 @@ void upstream_accesslog(const std::vector &lfv, std::tie(p, last) = copy('-', p, last); break; } +#if OPENSSL_3_0_0_API + auto x = SSL_get0_peer_certificate(lgsp.ssl); +#else // !OPENSSL_3_0_0_API auto x = SSL_get_peer_certificate(lgsp.ssl); +#endif // !OPENSSL_3_0_0_API if (!x) { std::tie(p, last) = copy('-', p, last); break; @@ -788,7 +798,9 @@ void upstream_accesslog(const std::vector &lfv, auto name = lf.type == LogFragmentType::TLS_CLIENT_ISSUER_NAME ? tls::get_x509_issuer_name(balloc, x) : tls::get_x509_subject_name(balloc, x); +#if !OPENSSL_3_0_0_API X509_free(x); +#endif // !OPENSSL_3_0_0_API if (name.empty()) { std::tie(p, last) = copy('-', p, last); break; @@ -801,13 +813,19 @@ void upstream_accesslog(const std::vector &lfv, std::tie(p, last) = copy('-', p, last); break; } +#if OPENSSL_3_0_0_API + auto x = SSL_get0_peer_certificate(lgsp.ssl); +#else // !OPENSSL_3_0_0_API auto x = SSL_get_peer_certificate(lgsp.ssl); +#endif // !OPENSSL_3_0_0_API if (!x) { std::tie(p, last) = copy('-', p, last); break; } auto sn = tls::get_x509_serial(balloc, x); +#if !OPENSSL_3_0_0_API X509_free(x); +#endif // !OPENSSL_3_0_0_API if (sn.empty()) { std::tie(p, last) = copy('-', p, last); break; diff --git a/src/shrpx_mruby_module_env.cc b/src/shrpx_mruby_module_env.cc index b3ed365d..5ebd9c05 100644 --- a/src/shrpx_mruby_module_env.cc +++ b/src/shrpx_mruby_module_env.cc @@ -153,7 +153,11 @@ mrb_value env_get_tls_client_fingerprint_md(mrb_state *mrb, const EVP_MD *md) { return mrb_str_new_static(mrb, "", 0); } +#if OPENSSL_3_0_0_API + auto x = SSL_get0_peer_certificate(ssl); +#else // !OPENSSL_3_0_0_API auto x = SSL_get_peer_certificate(ssl); +#endif // !OPENSSL_3_0_0_API if (!x) { return mrb_str_new_static(mrb, "", 0); } @@ -161,7 +165,9 @@ mrb_value env_get_tls_client_fingerprint_md(mrb_state *mrb, const EVP_MD *md) { // Currently the largest hash value is SHA-256, which is 32 bytes. std::array buf; auto slen = tls::get_x509_fingerprint(buf.data(), buf.size(), x, md); +#if !OPENSSL_3_0_0_API X509_free(x); +#endif // !OPENSSL_3_0_0_API if (slen == -1) { mrb_raise(mrb, E_RUNTIME_ERROR, "could not compute client fingerprint"); } @@ -199,14 +205,20 @@ mrb_value env_get_tls_client_subject_name(mrb_state *mrb, mrb_value self) { return mrb_str_new_static(mrb, "", 0); } +#if OPENSSL_3_0_0_API + auto x = SSL_get0_peer_certificate(ssl); +#else // !OPENSSL_3_0_0_API auto x = SSL_get_peer_certificate(ssl); +#endif // !OPENSSL_3_0_0_API if (!x) { return mrb_str_new_static(mrb, "", 0); } auto &balloc = downstream->get_block_allocator(); auto name = tls::get_x509_subject_name(balloc, x); +#if !OPENSSL_3_0_0_API X509_free(x); +#endif // !OPENSSL_3_0_0_API return mrb_str_new(mrb, name.c_str(), name.size()); } } // namespace @@ -223,14 +235,20 @@ mrb_value env_get_tls_client_issuer_name(mrb_state *mrb, mrb_value self) { return mrb_str_new_static(mrb, "", 0); } +#if OPENSSL_3_0_0_API + auto x = SSL_get0_peer_certificate(ssl); +#else // !OPENSSL_3_0_0_API auto x = SSL_get_peer_certificate(ssl); +#endif // !OPENSSL_3_0_0_API if (!x) { return mrb_str_new_static(mrb, "", 0); } auto &balloc = downstream->get_block_allocator(); auto name = tls::get_x509_issuer_name(balloc, x); +#if !OPENSSL_3_0_0_API X509_free(x); +#endif // !OPENSSL_3_0_0_API return mrb_str_new(mrb, name.c_str(), name.size()); } } // namespace @@ -247,14 +265,20 @@ mrb_value env_get_tls_client_serial(mrb_state *mrb, mrb_value self) { return mrb_str_new_static(mrb, "", 0); } +#if OPENSSL_3_0_0_API + auto x = SSL_get0_peer_certificate(ssl); +#else // !OPENSSL_3_0_0_API auto x = SSL_get_peer_certificate(ssl); +#endif // !OPENSSL_3_0_0_API if (!x) { return mrb_str_new_static(mrb, "", 0); } auto &balloc = downstream->get_block_allocator(); auto sn = tls::get_x509_serial(balloc, x); +#if !OPENSSL_3_0_0_API X509_free(x); +#endif // !OPENSSL_3_0_0_API return mrb_str_new(mrb, sn.c_str(), sn.size()); } } // namespace @@ -271,16 +295,24 @@ mrb_value env_get_tls_client_not_before(mrb_state *mrb, mrb_value self) { return mrb_fixnum_value(0); } +#if OPENSSL_3_0_0_API + auto x = SSL_get0_peer_certificate(ssl); +#else // !OPENSSL_3_0_0_API auto x = SSL_get_peer_certificate(ssl); +#endif // !OPENSSL_3_0_0_API if (!x) { return mrb_fixnum_value(0); } time_t t; if (tls::get_x509_not_before(t, x) != 0) { - return mrb_fixnum_value(0); + t = 0; } +#if !OPENSSL_3_0_0_API + X509_free(x); +#endif // !OPENSSL_3_0_0_API + return mrb_fixnum_value(t); } } // namespace @@ -297,16 +329,24 @@ mrb_value env_get_tls_client_not_after(mrb_state *mrb, mrb_value self) { return mrb_fixnum_value(0); } +#if OPENSSL_3_0_0_API + auto x = SSL_get0_peer_certificate(ssl); +#else // !OPENSSL_3_0_0_API auto x = SSL_get_peer_certificate(ssl); +#endif // !OPENSSL_3_0_0_API if (!x) { return mrb_fixnum_value(0); } time_t t; if (tls::get_x509_not_after(t, x) != 0) { - return mrb_fixnum_value(0); + t = 0; } +#if !OPENSSL_3_0_0_API + X509_free(x); +#endif // !OPENSSL_3_0_0_API + return mrb_fixnum_value(t); } } // namespace diff --git a/src/shrpx_tls.cc b/src/shrpx_tls.cc index 8d352727..e332ba63 100644 --- a/src/shrpx_tls.cc +++ b/src/shrpx_tls.cc @@ -1031,8 +1031,11 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file, DIE(); } - SSL_CTX_set_tmp_dh(ssl_ctx, dh); - EVP_PKEY_free(dh); + if (SSL_CTX_set0_tmp_dh_pkey(ssl_ctx, dh) != 1) { + LOG(FATAL) << "SSL_CTX_set0_tmp_dh_pkey failed: " + << ERR_error_string(ERR_get_error(), nullptr); + DIE(); + } #else // !OPENSSL_3_0_0_API auto dh = PEM_read_bio_DHparams(bio, nullptr, nullptr, nullptr); if (dh == nullptr) { @@ -1456,8 +1459,11 @@ SSL_CTX *create_quic_ssl_context(const char *private_key_file, DIE(); } - SSL_CTX_set_tmp_dh(ssl_ctx, dh); - EVP_PKEY_free(dh); + if (SSL_CTX_set0_tmp_dh_pkey(ssl_ctx, dh) != 1) { + LOG(FATAL) << "SSL_CTX_set0_tmp_dh_pkey failed: " + << ERR_error_string(ERR_get_error(), nullptr); + DIE(); + } # else // !OPENSSL_3_0_0_API auto dh = PEM_read_bio_DHparams(bio, nullptr, nullptr, nullptr); if (dh == nullptr) { @@ -2064,14 +2070,20 @@ int verify_hostname(X509 *cert, const StringRef &hostname, } // namespace int check_cert(SSL *ssl, const Address *addr, const StringRef &host) { +#if OPENSSL_3_0_0_API + auto cert = SSL_get0_peer_certificate(ssl); +#else // !OPENSSL_3_0_0_API auto cert = SSL_get_peer_certificate(ssl); +#endif // !OPENSSL_3_0_0_API if (!cert) { // By the protocol definition, TLS server always sends certificate // if it has. If certificate cannot be retrieved, authentication // without certificate is used, such as PSK. return 0; } +#if !OPENSSL_3_0_0_API auto cert_deleter = defer(X509_free, cert); +#endif // !OPENSSL_3_0_0_API if (verify_hostname(cert, host, addr) != 0) { LOG(ERROR) << "Certificate verification failed: hostname does not match"; From 65d3c9047f5a254a7690178b514655d7b7d7f6f6 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 17 Oct 2021 17:21:09 +0900 Subject: [PATCH 101/142] Replace TLSv23_method with TLS_method --- examples/client.c | 2 +- examples/libevent-client.c | 2 +- examples/libevent-server.c | 2 +- src/HttpServer.cc | 2 +- src/h2load.cc | 2 +- src/nghttp.cc | 2 +- src/shrpx_tls.cc | 4 ++-- src/shrpx_tls_test.cc | 4 ++-- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/client.c b/examples/client.c index 22f585f7..40bfd238 100644 --- a/examples/client.c +++ b/examples/client.c @@ -544,7 +544,7 @@ static void fetch_uri(const struct URI *uri) { if (fd == -1) { die("Could not open file descriptor"); } - ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + ssl_ctx = SSL_CTX_new(TLS_client_method()); if (ssl_ctx == NULL) { dief("SSL_CTX_new", ERR_error_string(ERR_get_error(), NULL)); } diff --git a/examples/libevent-client.c b/examples/libevent-client.c index e7c78cf5..2debd7b8 100644 --- a/examples/libevent-client.c +++ b/examples/libevent-client.c @@ -328,7 +328,7 @@ static int select_next_proto_cb(SSL *ssl, unsigned char **out, /* Create SSL_CTX. */ static SSL_CTX *create_ssl_ctx(void) { SSL_CTX *ssl_ctx; - ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + ssl_ctx = SSL_CTX_new(TLS_client_method()); if (!ssl_ctx) { errx(1, "Could not create SSL/TLS context: %s", ERR_error_string(ERR_get_error(), NULL)); diff --git a/examples/libevent-server.c b/examples/libevent-server.c index a30a5e89..9f4e1281 100644 --- a/examples/libevent-server.c +++ b/examples/libevent-server.c @@ -143,7 +143,7 @@ static int alpn_select_proto_cb(SSL *ssl, const unsigned char **out, static SSL_CTX *create_ssl_ctx(const char *key_file, const char *cert_file) { SSL_CTX *ssl_ctx; - ssl_ctx = SSL_CTX_new(SSLv23_server_method()); + ssl_ctx = SSL_CTX_new(TLS_server_method()); if (!ssl_ctx) { errx(1, "Could not create SSL/TLS context: %s", ERR_error_string(ERR_get_error(), NULL)); diff --git a/src/HttpServer.cc b/src/HttpServer.cc index dacb4b15..e82310e9 100644 --- a/src/HttpServer.cc +++ b/src/HttpServer.cc @@ -2110,7 +2110,7 @@ int HttpServer::run() { std::vector next_proto; if (!config_->no_tls) { - ssl_ctx = SSL_CTX_new(SSLv23_server_method()); + ssl_ctx = SSL_CTX_new(TLS_server_method()); if (!ssl_ctx) { std::cerr << ERR_error_string(ERR_get_error(), nullptr) << std::endl; return -1; diff --git a/src/h2load.cc b/src/h2load.cc index 6a58d1ae..46962cd5 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -2811,7 +2811,7 @@ int main(int argc, char **argv) { act.sa_handler = SIG_IGN; sigaction(SIGPIPE, &act, nullptr); - auto ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + auto ssl_ctx = SSL_CTX_new(TLS_client_method()); if (!ssl_ctx) { std::cerr << "Failed to create SSL_CTX: " << ERR_error_string(ERR_get_error(), nullptr) << std::endl; diff --git a/src/nghttp.cc b/src/nghttp.cc index 30ef26d3..5d62baef 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -2268,7 +2268,7 @@ int communicate( auto loop = EV_DEFAULT; SSL_CTX *ssl_ctx = nullptr; if (scheme == "https") { - ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + ssl_ctx = SSL_CTX_new(TLS_client_method()); if (!ssl_ctx) { std::cerr << "[ERROR] Failed to create SSL_CTX: " << ERR_error_string(ERR_get_error(), nullptr) << std::endl; diff --git a/src/shrpx_tls.cc b/src/shrpx_tls.cc index e332ba63..7913d86a 100644 --- a/src/shrpx_tls.cc +++ b/src/shrpx_tls.cc @@ -927,7 +927,7 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file, neverbleed_t *nb #endif // HAVE_NEVERBLEED ) { - auto ssl_ctx = SSL_CTX_new(SSLv23_server_method()); + auto ssl_ctx = SSL_CTX_new(TLS_server_method()); if (!ssl_ctx) { LOG(FATAL) << ERR_error_string(ERR_get_error(), nullptr); DIE(); @@ -1694,7 +1694,7 @@ SSL_CTX *create_ssl_client_context( int (*next_proto_select_cb)(SSL *s, unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, void *arg)) { - auto ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + auto ssl_ctx = SSL_CTX_new(TLS_client_method()); if (!ssl_ctx) { LOG(FATAL) << ERR_error_string(ERR_get_error(), nullptr); DIE(); diff --git a/src/shrpx_tls_test.cc b/src/shrpx_tls_test.cc index 0f130f60..ef80a1a7 100644 --- a/src/shrpx_tls_test.cc +++ b/src/shrpx_tls_test.cc @@ -121,7 +121,7 @@ void test_shrpx_tls_cert_lookup_tree_add_ssl_ctx(void) { static constexpr char nghttp2_certfile[] = NGHTTP2_SRC_DIR "/test.nghttp2.org.pem"; - auto nghttp2_ssl_ctx = SSL_CTX_new(SSLv23_server_method()); + auto nghttp2_ssl_ctx = SSL_CTX_new(TLS_server_method()); auto nghttp2_ssl_ctx_del = defer(SSL_CTX_free, nghttp2_ssl_ctx); auto nghttp2_tls_ctx_data = std::make_unique(); nghttp2_tls_ctx_data->cert_file = nghttp2_certfile; @@ -132,7 +132,7 @@ void test_shrpx_tls_cert_lookup_tree_add_ssl_ctx(void) { static constexpr char examples_certfile[] = NGHTTP2_SRC_DIR "/test.example.com.pem"; - auto examples_ssl_ctx = SSL_CTX_new(SSLv23_server_method()); + auto examples_ssl_ctx = SSL_CTX_new(TLS_server_method()); auto examples_ssl_ctx_del = defer(SSL_CTX_free, examples_ssl_ctx); auto examples_tls_ctx_data = std::make_unique(); examples_tls_ctx_data->cert_file = examples_certfile; From 15a8d913ead5dff164ff0a4191b600b78eec1baf Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 17 Oct 2021 17:25:18 +0900 Subject: [PATCH 102/142] examples/client: Enable ALPN --- examples/client.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/client.c b/examples/client.c index 40bfd238..6cc3fddf 100644 --- a/examples/client.c +++ b/examples/client.c @@ -380,6 +380,10 @@ static void init_ssl_ctx(SSL_CTX *ssl_ctx) { #ifndef OPENSSL_NO_NEXTPROTONEG SSL_CTX_set_next_proto_select_cb(ssl_ctx, select_next_proto_cb, NULL); #endif /* !OPENSSL_NO_NEXTPROTONEG */ + +#if OPENSSL_VERSION_NUMBER >= 0x10002000L + SSL_CTX_set_alpn_protos(ssl_ctx, (const unsigned char *)"\x02h2", 3); +#endif /* OPENSSL_VERSION_NUMBER >= 0x10002000L */ } static void ssl_handshake(SSL *ssl, int fd) { From f8474b25f02c4faa6d4250800a164227cf89b327 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 18 Oct 2021 22:37:16 +0900 Subject: [PATCH 103/142] nghttpx: Reduce dgram size if sendmsg fails with EINVAL or EMSGSIZE --- src/shrpx_http3_upstream.cc | 82 +++++++++++++++++++++++-------------- src/shrpx_http3_upstream.h | 5 +++ src/shrpx_quic.cc | 7 +++- 3 files changed, 63 insertions(+), 31 deletions(-) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 1208e3a3..956215c5 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -118,6 +118,7 @@ size_t downstream_queue_size(Worker *worker) { Http3Upstream::Http3Upstream(ClientHandler *handler) : handler_{handler}, + max_udp_payload_size_{SHRPX_QUIC_MAX_UDP_PAYLOAD_SIZE}, qlog_fd_{-1}, hashed_scid_{}, conn_{nullptr}, @@ -707,7 +708,8 @@ int Http3Upstream::on_write() { int Http3Upstream::write_streams() { std::array vec; std::array buf; - auto max_udp_payload_size = ngtcp2_conn_get_path_max_udp_payload_size(conn_); + auto max_udp_payload_size = std::min( + max_udp_payload_size_, ngtcp2_conn_get_path_max_udp_payload_size(conn_)); size_t max_pktcnt = std::min(static_cast(64_k), ngtcp2_conn_get_send_quantum(conn_)) / max_udp_payload_size; @@ -812,10 +814,10 @@ int Http3Upstream::write_streams() { if (nwrite == 0) { if (bufpos - buf.data()) { - quic_send_packet(static_cast(prev_ps.path.user_data), - prev_ps.path.remote.addr, prev_ps.path.remote.addrlen, - prev_ps.path.local.addr, prev_ps.path.local.addrlen, - buf.data(), bufpos - buf.data(), max_udp_payload_size); + send_packet(static_cast(prev_ps.path.user_data), + prev_ps.path.remote.addr, prev_ps.path.remote.addrlen, + prev_ps.path.local.addr, prev_ps.path.local.addrlen, + buf.data(), bufpos - buf.data(), max_udp_payload_size); reset_idle_timer(); } @@ -833,16 +835,16 @@ int Http3Upstream::write_streams() { if (pktcnt == 0) { ngtcp2_path_copy(&prev_ps.path, &ps.path); } else if (!ngtcp2_path_eq(&prev_ps.path, &ps.path)) { - quic_send_packet(static_cast(prev_ps.path.user_data), - prev_ps.path.remote.addr, prev_ps.path.remote.addrlen, - prev_ps.path.local.addr, prev_ps.path.local.addrlen, - buf.data(), bufpos - buf.data() - nwrite, - max_udp_payload_size); + send_packet(static_cast(prev_ps.path.user_data), + prev_ps.path.remote.addr, prev_ps.path.remote.addrlen, + prev_ps.path.local.addr, prev_ps.path.local.addrlen, + buf.data(), bufpos - buf.data() - nwrite, + max_udp_payload_size); - quic_send_packet(static_cast(ps.path.user_data), - ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, - bufpos - nwrite, nwrite, max_udp_payload_size); + send_packet(static_cast(ps.path.user_data), + ps.path.remote.addr, ps.path.remote.addrlen, + ps.path.local.addr, ps.path.local.addrlen, bufpos - nwrite, + nwrite, max_udp_payload_size); ngtcp2_conn_update_pkt_tx_time(conn_, ts); reset_idle_timer(); @@ -854,10 +856,10 @@ int Http3Upstream::write_streams() { if (++pktcnt == max_pktcnt || static_cast(nwrite) < max_udp_payload_size) { - quic_send_packet(static_cast(ps.path.user_data), - ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, buf.data(), - bufpos - buf.data(), max_udp_payload_size); + send_packet(static_cast(ps.path.user_data), + ps.path.remote.addr, ps.path.remote.addrlen, + ps.path.local.addr, ps.path.local.addrlen, buf.data(), + bufpos - buf.data(), max_udp_payload_size); ngtcp2_conn_update_pkt_tx_time(conn_, ts); reset_idle_timer(); @@ -867,10 +869,9 @@ int Http3Upstream::write_streams() { return 0; } #else // !UDP_SEGMENT - quic_send_packet(static_cast(ps.path.user_data), - ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, buf.data(), - bufpos - buf.data(), 0); + send_packet(static_cast(ps.path.user_data), + ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr, + ps.path.local.addrlen, buf.data(), bufpos - buf.data(), 0); if (++pktcnt == max_pktcnt) { ngtcp2_conn_update_pkt_tx_time(conn_, ts); @@ -1410,10 +1411,9 @@ void Http3Upstream::on_handler_delete() { conn_close_.resize(nwrite); - quic_send_packet(static_cast(ps.path.user_data), - ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, - conn_close_.data(), nwrite, 0); + send_packet(static_cast(ps.path.user_data), + ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr, + ps.path.local.addrlen, conn_close_.data(), nwrite, 0); } auto d = @@ -1716,6 +1716,29 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr, return 0; } +int Http3Upstream::send_packet(const UpstreamAddr *faddr, + const sockaddr *remote_sa, size_t remote_salen, + const sockaddr *local_sa, size_t local_salen, + const uint8_t *data, size_t datalen, + size_t gso_size) { + auto rv = quic_send_packet(faddr, remote_sa, remote_salen, local_sa, + local_salen, data, datalen, gso_size); + switch (rv) { + case 0: + return 0; + // With GSO, sendmsg may fail with EINVAL if UDP payload is too + // large. + case -EINVAL: + case -EMSGSIZE: + max_udp_payload_size_ = NGTCP2_MAX_UDP_PAYLOAD_SIZE; + break; + default: + break; + } + + return -1; +} + int Http3Upstream::handle_error() { if (ngtcp2_conn_is_in_closing_period(conn_)) { return -1; @@ -1754,10 +1777,9 @@ int Http3Upstream::handle_error() { conn_close_.resize(nwrite); - quic_send_packet(static_cast(ps.path.user_data), - ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, - conn_close_.data(), nwrite, 0); + send_packet(static_cast(ps.path.user_data), + ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr, + ps.path.local.addrlen, conn_close_.data(), nwrite, 0); return -1; } diff --git a/src/shrpx_http3_upstream.h b/src/shrpx_http3_upstream.h index b811a4bb..b2935afc 100644 --- a/src/shrpx_http3_upstream.h +++ b/src/shrpx_http3_upstream.h @@ -148,6 +148,10 @@ public: int start_graceful_shutdown(); int submit_goaway(); void idle_close(); + int send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, + size_t remote_salen, const sockaddr *local_sa, + size_t local_salen, const uint8_t *data, size_t datalen, + size_t gso_size); void qlog_write(const void *data, size_t datalen, bool fin); int open_qlog_file(const StringRef &dir, const ngtcp2_cid &scid) const; @@ -158,6 +162,7 @@ private: ev_timer idle_timer_; ev_timer shutdown_timer_; ev_prepare prep_; + size_t max_udp_payload_size_; int qlog_fd_; ngtcp2_cid hashed_scid_; ngtcp2_conn *conn_; diff --git a/src/shrpx_quic.cc b/src/shrpx_quic.cc index a0bf56c3..224fbbb1 100644 --- a/src/shrpx_quic.cc +++ b/src/shrpx_quic.cc @@ -130,7 +130,12 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, } while (nwrite == -1 && errno == EINTR); if (nwrite == -1) { - return -1; + if (LOG_ENABLED(INFO)) { + auto error = errno; + LOG(INFO) << "sendmsg failed: errno=" << error; + } + + return -errno; } if (LOG_ENABLED(INFO)) { From 7af0c508be9cbec407268e2f546f597d268c104e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 19 Oct 2021 18:29:19 +0900 Subject: [PATCH 104/142] Update manual pages --- doc/h2load.1 | 2 +- doc/nghttp.1 | 2 +- doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index 26fbb49b..2b6c2e59 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "Oct 17, 2021" "1.46.0" "nghttp2" +.TH "H2LOAD" "1" "Oct 19, 2021" "1.46.0" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 58423929..4148abfb 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "Oct 17, 2021" "1.46.0" "nghttp2" +.TH "NGHTTP" "1" "Oct 19, 2021" "1.46.0" "nghttp2" .SH NAME nghttp \- HTTP/2 client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index 42b764f9..25d4f494 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "Oct 17, 2021" "1.46.0" "nghttp2" +.TH "NGHTTPD" "1" "Oct 19, 2021" "1.46.0" "nghttp2" .SH NAME nghttpd \- HTTP/2 server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index d1f8ceaf..a328ba23 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "Oct 17, 2021" "1.46.0" "nghttp2" +.TH "NGHTTPX" "1" "Oct 19, 2021" "1.46.0" "nghttp2" .SH NAME nghttpx \- HTTP/2 proxy . From 845a20b5827cf7aa8dc9e9a7956b150963e5e6a5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 19 Oct 2021 18:59:58 +0900 Subject: [PATCH 105/142] Bump version number to 1.47.0-DEV --- CMakeLists.txt | 2 +- configure.ac | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bfcd07d5..ae0700cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 3.0) # XXX using 1.8.90 instead of 1.9.0-DEV -project(nghttp2 VERSION 1.46.0) +project(nghttp2 VERSION 1.46.90) # See versioning rule: # https://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html diff --git a/configure.ac b/configure.ac index bec2aa35..91146331 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl https://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.46.0], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.47.0-DEV], [t-tujikawa@users.sourceforge.net]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_HEADERS([config.h]) From c20d175ff2011dd71f348ff1d17c9d2e68663276 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 29 Oct 2021 21:39:17 +0900 Subject: [PATCH 106/142] Guard RLIMIT_MEMLOCK usage --- src/shrpx.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/shrpx.cc b/src/shrpx.cc index 14427f92..6d810ef4 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -3725,6 +3725,7 @@ int process_options(Config *config, } } +#ifdef RLIMIT_MEMLOCK if (config->rlimit_memlock) { struct rlimit lim = {static_cast(config->rlimit_memlock), static_cast(config->rlimit_memlock)}; @@ -3734,6 +3735,7 @@ int process_options(Config *config, << xsi_strerror(error, errbuf.data(), errbuf.size()); } } +#endif // RLIMIT_MEMLOCK auto &fwdconf = config->http.forwarded; From 089fc81d729c33a7d8810837c809aa6352d2c257 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 29 Oct 2021 21:59:54 +0900 Subject: [PATCH 107/142] Change qlog file extension to .sqlog Change qlog file extension to .sqlog because upstream ngtcp2 moves to JSON-SEQ streaming qlog format. --- src/h2load.cc | 9 ++++----- src/h2load_quic.cc | 2 +- src/shrpx.cc | 2 +- src/shrpx_http3_upstream.cc | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/h2load.cc b/src/h2load.cc index 46962cd5..f00edfb0 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -2245,11 +2245,10 @@ Options: to buffering. Status code is -1 for failed streams. --qlog-file-base= Enable qlog output and specify base file name for qlogs. - Qlog is emitted for each connection. - For a given base name "base", each output file name - becomes "base.M.N.qlog" where M is worker ID and N is - client ID (e.g. "base.0.3.qlog"). - Only effective in QUIC runs. + Qlog is emitted for each connection. For a given base + name "base", each output file name becomes + "base.M.N.sqlog" where M is worker ID and N is client ID + (e.g. "base.0.3.sqlog"). Only effective in QUIC runs. --connect-to=[:] Host and port to connect instead of using the authority in . diff --git a/src/h2load_quic.cc b/src/h2load_quic.cc index 74680043..cc178d35 100644 --- a/src/h2load_quic.cc +++ b/src/h2load_quic.cc @@ -441,7 +441,7 @@ int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen, path += util::utos(worker->id); path += '.'; path += util::utos(id); - path += ".qlog"; + path += ".sqlog"; quic.qlog_file = fopen(path.c_str(), "w"); if (quic.qlog_file == nullptr) { std::cerr << "Failed to open a qlog file: " << path << std::endl; diff --git a/src/shrpx.cc b/src/shrpx.cc index 6d810ef4..e4b68488 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -3353,7 +3353,7 @@ HTTP/3 and QUIC: frontend QUIC connections. A qlog file is created per each QUIC connection. The file name is ISO8601 basic format, followed by "-", server Source Connection ID and - ".qlog". + ".sqlog". --frontend-quic-require-token Require an address validation token for a frontend QUIC connection. Server sends a token in Retry packet or diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 956215c5..681acfe3 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -2704,7 +2704,7 @@ int Http3Upstream::open_qlog_file(const StringRef &dir, util::format_iso8601_basic(buf.data(), std::chrono::system_clock::now()); path += '-'; path += util::format_hex(scid.data, scid.datalen); - path += ".qlog"; + path += ".sqlog"; int fd; From fb63ef305def619d180604474a35597b30fb3bc6 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 30 Oct 2021 18:09:06 +0900 Subject: [PATCH 108/142] Update doc --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 1372e3ea..f5bca45c 100644 --- a/README.rst +++ b/README.rst @@ -151,7 +151,7 @@ To enable the experimental HTTP/3 support for h2load and nghttpx, the following libraries are required: * `OpenSSL with QUIC support - `_; or + `_; or `BoringSSL `_ (commit f6ef1c560ae5af51e2df5d8d2175bed207b28b8f) * `ngtcp2 `_ From 0d16db2c65d8678fe7311c7db6b3aed85bac7832 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 30 Oct 2021 18:09:18 +0900 Subject: [PATCH 109/142] Use quictls 1.1.1l for now because 3.0.0 is much slower --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 47be5a40..977a5877 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update && \ zlib1g-dev libev-dev libjemalloc-dev ruby-dev libc-ares-dev bison \ libelf-dev -RUN git clone --depth 1 -b openssl-3.0.0+quic https://github.com/quictls/openssl && \ +RUN git clone --depth 1 -b OpenSSL_1_1_1l+quic https://github.com/quictls/openssl && \ cd openssl && \ ./config --openssldir=/etc/ssl && \ make -j$(nproc) && \ From c4828dbd7cc12433dc84310a5591615da04ac9a5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 3 Nov 2021 21:23:26 +0900 Subject: [PATCH 110/142] h2load: Fix assertion failure with the latest ngtcp2 --- src/h2load.cc | 3 ++- src/h2load.h | 1 + src/h2load_quic.cc | 48 ++++++++++++++++++++++++++++++---------------- 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/h2load.cc b/src/h2load.cc index f00edfb0..1ab52c0b 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -1516,7 +1516,8 @@ int get_ev_loop_flags() { Worker::Worker(uint32_t id, SSL_CTX *ssl_ctx, size_t req_todo, size_t nclients, size_t rate, size_t max_samples, Config *config) - : stats(req_todo, nclients), + : randgen(util::make_mt19937()), + stats(req_todo, nclients), loop(ev_loop_new(get_ev_loop_flags())), ssl_ctx(ssl_ctx), config(config), diff --git a/src/h2load.h b/src/h2load.h index 67da2b50..590a6cd2 100644 --- a/src/h2load.h +++ b/src/h2load.h @@ -269,6 +269,7 @@ struct Sampling { struct Worker { MemchunkPool mcpool; + std::mt19937 randgen; Stats stats; Sampling request_times_smp; Sampling client_smp; diff --git a/src/h2load_quic.cc b/src/h2load_quic.cc index cc178d35..6d729b80 100644 --- a/src/h2load_quic.cc +++ b/src/h2load_quic.cc @@ -36,15 +36,12 @@ #endif // HAVE_LIBNGTCP2_CRYPTO_BORINGSSL #include +#include #include "h2load_http3_session.h" namespace h2load { -namespace { -auto randgen = util::make_mt19937(); -} // namespace - namespace { int handshake_completed(ngtcp2_conn *conn, void *user_data) { auto c = static_cast(user_data); @@ -202,13 +199,15 @@ int Client::quic_extend_max_local_streams() { namespace { int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token, size_t cidlen, void *user_data) { - auto dis = std::uniform_int_distribution( - 0, std::numeric_limits::max()); - auto f = [&dis]() { return dis(randgen); }; + if (RAND_bytes(cid->data, cidlen) != 1) { + return NGTCP2_ERR_CALLBACK_FAILURE; + } - std::generate_n(cid->data, cidlen, f); cid->datalen = cidlen; - std::generate_n(token, NGTCP2_STATELESS_RESET_TOKENLEN, f); + + if (RAND_bytes(token, NGTCP2_STATELESS_RESET_TOKENLEN) != 1) { + return NGTCP2_ERR_CALLBACK_FAILURE; + } return 0; } @@ -227,11 +226,14 @@ void debug_log_printf(void *user_data, const char *fmt, ...) { } // namespace namespace { -void generate_cid(ngtcp2_cid &dest) { - auto dis = std::uniform_int_distribution( - 0, std::numeric_limits::max()); +int generate_cid(ngtcp2_cid &dest) { dest.datalen = 8; - std::generate_n(dest.data, dest.datalen, [&dis]() { return dis(randgen); }); + + if (RAND_bytes(dest.data, dest.datalen) != 1) { + return -1; + } + + return 0; } } // namespace @@ -370,6 +372,13 @@ void Client::quic_write_qlog(const void *data, size_t datalen) { fwrite(data, 1, datalen, quic.qlog_file); } +namespace { +void rand(uint8_t *dest, size_t destlen, const ngtcp2_rand_ctx *rand_ctx) { + util::random_bytes(dest, dest + destlen, + *static_cast(rand_ctx->native_handle)); +} +} // namespace + int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen, const sockaddr *remote_addr, socklen_t remote_addrlen) { int rv; @@ -400,7 +409,7 @@ int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen, ngtcp2_crypto_recv_retry_cb, h2load::extend_max_local_streams_bidi, nullptr, // extend_max_local_streams_uni - nullptr, // rand + h2load::rand, get_new_connection_id, nullptr, // remove_connection_id ngtcp2_crypto_update_key_cb, @@ -418,13 +427,17 @@ int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen, nullptr, // recv_datagram nullptr, // ack_datagram nullptr, // lost_datagram - nullptr, // get_path_challenge_data + ngtcp2_crypto_get_path_challenge_data_cb, h2load::stream_stop_sending, }; ngtcp2_cid scid, dcid; - generate_cid(scid); - generate_cid(dcid); + if (generate_cid(scid) != 0) { + return -1; + } + if (generate_cid(dcid) != 0) { + return -1; + } auto config = worker->config; @@ -434,6 +447,7 @@ int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen, settings.log_printf = debug_log_printf; } settings.initial_ts = timestamp(worker->loop); + settings.rand_ctx.native_handle = &worker->randgen; if (!config->qlog_file_base.empty()) { assert(quic.qlog_file == nullptr); auto path = config->qlog_file_base; From 9e154297ff414b243759a0eca34befb9fb94f823 Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Thu, 4 Nov 2021 15:42:51 +0100 Subject: [PATCH 111/142] shrpx: make nghttpx -v show ngtcp2 and nghttp3 version as well ... if HTTP/3 support is built in. --- src/shrpx.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index e4b68488..26b09925 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -76,6 +76,11 @@ #include +#ifdef ENABLE_HTTP3 +# include +# include +#endif // ENABLE_HTTP3 + #include "shrpx_config.h" #include "shrpx_tls.h" #include "shrpx_log_config.h" @@ -2053,7 +2058,11 @@ void fill_default_config(Config *config) { namespace { void print_version(std::ostream &out) { - out << "nghttpx nghttp2/" NGHTTP2_VERSION << std::endl; + out << "nghttpx nghttp2/" NGHTTP2_VERSION +#ifdef ENABLE_HTTP3 + " ngtcp2/" NGTCP2_VERSION " nghttp3/" NGHTTP3_VERSION +#endif // ENABLE_HTTP3 + << std::endl; } } // namespace From 47c33b8d03b17fb8b42e29468f63ff3359076917 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 5 Nov 2021 19:19:23 +0900 Subject: [PATCH 112/142] nghttpx: Receive ECN --- src/shrpx_client_handler.cc | 5 +-- src/shrpx_client_handler.h | 3 +- src/shrpx_connection_handler.cc | 33 ++++++++++--------- src/shrpx_connection_handler.h | 8 +++-- src/shrpx_http3_upstream.cc | 5 ++- src/shrpx_http3_upstream.h | 3 +- src/shrpx_quic_connection_handler.cc | 15 +++++---- src/shrpx_quic_connection_handler.h | 8 ++--- src/shrpx_quic_listener.cc | 12 +++++-- src/shrpx_worker.cc | 4 +-- src/shrpx_worker.h | 7 ++-- src/util.cc | 48 ++++++++++++++++++++++++++++ src/util.h | 4 +++ 13 files changed, 113 insertions(+), 42 deletions(-) diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 8d02d740..415df864 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -292,11 +292,12 @@ int ClientHandler::write_tls() { #ifdef ENABLE_HTTP3 int ClientHandler::read_quic(const UpstreamAddr *faddr, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, + const Address &local_addr, + const ngtcp2_pkt_info &pi, const uint8_t *data, size_t datalen) { auto upstream = static_cast(upstream_.get()); - return upstream->on_read(faddr, remote_addr, local_addr, data, datalen); + return upstream->on_read(faddr, remote_addr, local_addr, pi, data, datalen); } int ClientHandler::write_quic() { return upstream_->on_write(); } diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index f20adc92..c2c50c0b 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -149,7 +149,8 @@ public: #ifdef ENABLE_HTTP3 void setup_http3_upstream(std::unique_ptr &&upstream); int read_quic(const UpstreamAddr *faddr, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, size_t datalen); + const Address &local_addr, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen); int write_quic(); #endif // ENABLE_HTTP3 diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index cf45f508..4f216a5c 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -1017,12 +1017,10 @@ void ConnectionHandler::set_enable_acceptor_on_ocsp_completion(bool f) { } #ifdef ENABLE_HTTP3 -int ConnectionHandler::forward_quic_packet(const UpstreamAddr *faddr, - const Address &remote_addr, - const Address &local_addr, - const uint8_t *cid_prefix, - const uint8_t *data, - size_t datalen) { +int ConnectionHandler::forward_quic_packet( + const UpstreamAddr *faddr, const Address &remote_addr, + const Address &local_addr, const ngtcp2_pkt_info &pi, + const uint8_t *cid_prefix, const uint8_t *data, size_t datalen) { assert(!get_config()->single_thread); for (auto &worker : workers_) { @@ -1034,7 +1032,7 @@ int ConnectionHandler::forward_quic_packet(const UpstreamAddr *faddr, WorkerEvent wev{}; wev.type = WorkerEventType::QUIC_PKT_FORWARD; wev.quic_pkt = std::make_unique(faddr->index, remote_addr, - local_addr, data, datalen); + local_addr, pi, data, datalen); worker->send(std::move(wev)); @@ -1111,10 +1109,11 @@ void ConnectionHandler::set_quic_lingering_worker_processes( int ConnectionHandler::forward_quic_packet_to_lingering_worker_process( QUICLingeringWorkerProcess *quic_lwp, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, size_t datalen) { + const Address &local_addr, const ngtcp2_pkt_info &pi, const uint8_t *data, + size_t datalen) { std::array header; - assert(header.size() >= 1 + 1 + 1 + sizeof(sockaddr_storage) * 2); + assert(header.size() >= 1 + 1 + 1 + 1 + sizeof(sockaddr_storage) * 2); assert(remote_addr.len > 0); assert(local_addr.len > 0); @@ -1127,6 +1126,7 @@ int ConnectionHandler::forward_quic_packet_to_lingering_worker_process( *p++ = static_cast(local_addr.len - 1); p = std::copy_n(reinterpret_cast(&local_addr.su), local_addr.len, p); + *p++ = pi.ecn; iovec msg_iov[] = { { @@ -1185,14 +1185,14 @@ int ConnectionHandler::quic_ipc_read() { return 0; } - size_t len = 1 + 1 + 1; + size_t len = 1 + 1 + 1 + 1; // Wire format: - // TYPE(1) REMOTE_ADDRLEN(1) REMOTE_ADDR(N) LOCAL_ADDRLEN(1) REMOTE_ADDR(N) - // DGRAM_PAYLAOD(N) + // TYPE(1) REMOTE_ADDRLEN(1) REMOTE_ADDR(N) LOCAL_ADDRLEN(1) LOCAL_ADDR(N) + // ECN(1) DGRAM_PAYLOAD(N) // - // When encoding, REMOTE_ADDRLEN and LOCAL_ADDRLEN is decremented by - // 1. + // When encoding, REMOTE_ADDRLEN and LOCAL_ADDRLEN are decremented + // by 1. if (static_cast(nread) < len) { return 0; } @@ -1249,6 +1249,8 @@ int ConnectionHandler::quic_ipc_read() { p += local_addrlen; + pkt->pi.ecn = *p++; + auto datalen = nread - (p - buf.data()); pkt->data.assign(p, p + datalen); @@ -1288,7 +1290,8 @@ int ConnectionHandler::quic_ipc_read() { // Ignore return value quic_conn_handler->handle_packet(faddr, pkt->remote_addr, pkt->local_addr, - pkt->data.data(), pkt->data.size()); + pkt->pi, pkt->data.data(), + pkt->data.size()); return 0; } diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h index 8bbf99c1..8a515e4b 100644 --- a/src/shrpx_connection_handler.h +++ b/src/shrpx_connection_handler.h @@ -196,8 +196,9 @@ public: const std::vector &get_quic_indexed_ssl_ctx(size_t idx) const; int forward_quic_packet(const UpstreamAddr *faddr, const Address &remote_addr, - const Address &local_addr, const uint8_t *cid_prefix, - const uint8_t *data, size_t datalen); + const Address &local_addr, const ngtcp2_pkt_info &pi, + const uint8_t *cid_prefix, const uint8_t *data, + size_t datalen); void set_quic_keying_materials(std::shared_ptr qkms); const std::shared_ptr &get_quic_keying_materials() const; @@ -218,7 +219,8 @@ public: int forward_quic_packet_to_lingering_worker_process( QUICLingeringWorkerProcess *quic_lwp, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, size_t datalen); + const Address &local_addr, const ngtcp2_pkt_info &pi, const uint8_t *data, + size_t datalen); void set_quic_ipc_fd(int fd); diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 681acfe3..de3704f0 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -1632,10 +1632,9 @@ void Http3Upstream::cancel_premature_downstream( int Http3Upstream::on_read(const UpstreamAddr *faddr, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, - size_t datalen) { + const Address &local_addr, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen) { int rv; - ngtcp2_pkt_info pi{}; auto path = ngtcp2_path{ { diff --git a/src/shrpx_http3_upstream.h b/src/shrpx_http3_upstream.h index b2935afc..5df95283 100644 --- a/src/shrpx_http3_upstream.h +++ b/src/shrpx_http3_upstream.h @@ -92,7 +92,8 @@ public: const ngtcp2_cid *odcid, const uint8_t *token, size_t tokenlen); int on_read(const UpstreamAddr *faddr, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, size_t datalen); + const Address &local_addr, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen); int write_streams(); diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 325e6957..38249e29 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -61,6 +61,7 @@ QUICConnectionHandler::~QUICConnectionHandler() { int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, const Address &remote_addr, const Address &local_addr, + const ngtcp2_pkt_info &pi, const uint8_t *data, size_t datalen) { int rv; uint32_t version; @@ -98,7 +99,7 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, if (cwit != std::end(close_waits_)) { auto cw = (*cwit).second; - cw->handle_packet(faddr, remote_addr, local_addr, data, datalen); + cw->handle_packet(faddr, remote_addr, local_addr, pi, data, datalen); return 0; } @@ -115,7 +116,7 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, if (cwit != std::end(close_waits_)) { auto cw = (*cwit).second; - cw->handle_packet(faddr, remote_addr, local_addr, data, datalen); + cw->handle_packet(faddr, remote_addr, local_addr, pi, data, datalen); return 0; } @@ -147,7 +148,7 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, decrypted_dcid.data(), decrypted_dcid.size()); if (quic_lwp) { if (conn_handler->forward_quic_packet_to_lingering_worker_process( - quic_lwp, remote_addr, local_addr, data, datalen) == 0) { + quic_lwp, remote_addr, local_addr, pi, data, datalen) == 0) { return 0; } @@ -310,7 +311,7 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, worker_->get_cid_prefix())) { if (conn_handler->forward_quic_packet(faddr, remote_addr, local_addr, - decrypted_dcid.data(), data, + pi, decrypted_dcid.data(), data, datalen) == 0) { return 0; } @@ -333,7 +334,8 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, handler = (*it).second; } - if (handler->read_quic(faddr, remote_addr, local_addr, data, datalen) != 0) { + if (handler->read_quic(faddr, remote_addr, local_addr, pi, data, datalen) != + 0) { delete handler; return 0; } @@ -708,7 +710,8 @@ CloseWait::~CloseWait() { int CloseWait::handle_packet(const UpstreamAddr *faddr, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, + const Address &local_addr, + const ngtcp2_pkt_info &pi, const uint8_t *data, size_t datalen) { if (pkt.empty()) { return 0; diff --git a/src/shrpx_quic_connection_handler.h b/src/shrpx_quic_connection_handler.h index 39c1b7d2..48faba77 100644 --- a/src/shrpx_quic_connection_handler.h +++ b/src/shrpx_quic_connection_handler.h @@ -55,8 +55,8 @@ struct CloseWait { ~CloseWait(); int handle_packet(const UpstreamAddr *faddr, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, - size_t datalen); + const Address &local_addr, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen); Worker *worker; // Source Connection IDs of the connection. @@ -82,8 +82,8 @@ public: QUICConnectionHandler(Worker *worker); ~QUICConnectionHandler(); int handle_packet(const UpstreamAddr *faddr, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, - size_t datalen); + const Address &local_addr, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen); // Send Retry packet. |ini_dcid| is the destination Connection ID // which appeared in Client Initial packet and its length is // |dcidlen|. |ini_scid| is the source Connection ID which appeared diff --git a/src/shrpx_quic_listener.cc b/src/shrpx_quic_listener.cc index b06ec557..d22045d6 100644 --- a/src/shrpx_quic_listener.cc +++ b/src/shrpx_quic_listener.cc @@ -59,7 +59,8 @@ void QUICListener::on_read() { msg.msg_iov = &msg_iov; msg.msg_iovlen = 1; - uint8_t msg_ctrl[CMSG_SPACE(sizeof(in6_pktinfo))]; + uint8_t + msg_ctrl[CMSG_SPACE(sizeof(uint8_t)) + CMSG_SPACE(sizeof(in6_pktinfo))]; msg.msg_control = msg_ctrl; auto quic_conn_handler = worker_->get_quic_connection_handler(); @@ -83,11 +84,16 @@ void QUICListener::on_read() { util::set_port(local_addr, faddr_->port); + ngtcp2_pkt_info pi{ + .ecn = util::msghdr_get_ecn(&msg, su.storage.ss_family), + }; + if (LOG_ENABLED(INFO)) { LOG(INFO) << "QUIC received packet: local=" << util::to_numeric_addr(&local_addr) << " remote=" << util::to_numeric_addr(&su.sa, msg.msg_namelen) - << " " << nread << " bytes"; + << " ecn=" << log::hex << pi.ecn << log::dec << " " << nread + << " bytes"; } if (nread == 0) { @@ -98,7 +104,7 @@ void QUICListener::on_read() { remote_addr.su = su; remote_addr.len = msg.msg_namelen; - quic_conn_handler->handle_packet(faddr_, remote_addr, local_addr, + quic_conn_handler->handle_packet(faddr_, remote_addr, local_addr, pi, buf.data(), nread); } } diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index a0107cc9..89423537 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -554,7 +554,7 @@ void Worker::process_events() { quic_conn_handler_.handle_packet( faddr, wev.quic_pkt->remote_addr, wev.quic_pkt->local_addr, - wev.quic_pkt->data.data(), wev.quic_pkt->data.size()); + wev.quic_pkt->pi, wev.quic_pkt->data.data(), wev.quic_pkt->data.size()); break; } @@ -844,7 +844,7 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { } } - // TODO Enable ECN + util::fd_set_recv_ecn(fd, faddr.family); if (bind(fd, rp->ai_addr, rp->ai_addrlen) == -1) { auto error = errno; diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 8cf40e97..724c56ed 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -258,15 +258,18 @@ struct WorkerStat { #ifdef ENABLE_HTTP3 struct QUICPacket { QUICPacket(size_t upstream_addr_index, const Address &remote_addr, - const Address &local_addr, const uint8_t *data, size_t datalen) + const Address &local_addr, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen) : upstream_addr_index{upstream_addr_index}, remote_addr{remote_addr}, local_addr{local_addr}, + pi{pi}, data{data, data + datalen} {} - QUICPacket() {} + QUICPacket() : upstream_addr_index{}, remote_addr{}, local_addr{}, pi{} {} size_t upstream_addr_index; Address remote_addr; Address local_addr; + ngtcp2_pkt_info pi; std::vector data; }; #endif // ENABLE_HTTP3 diff --git a/src/util.cc b/src/util.cc index 213989e4..eb70bbb3 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1722,6 +1722,54 @@ int msghdr_get_local_addr(Address &dest, msghdr *msg, int family) { return -1; } + +unsigned int msghdr_get_ecn(msghdr *msg, int family) { + switch (family) { + case AF_INET: + for (auto cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) { + if (cmsg->cmsg_level == IPPROTO_IP && cmsg->cmsg_type == IP_TOS && + cmsg->cmsg_len) { + return *reinterpret_cast(CMSG_DATA(cmsg)); + } + } + + return 0; + case AF_INET6: + for (auto cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) { + if (cmsg->cmsg_level == IPPROTO_IPV6 && cmsg->cmsg_type == IPV6_TCLASS && + cmsg->cmsg_len) { + return *reinterpret_cast(CMSG_DATA(cmsg)); + } + } + + return 0; + } + + return 0; +} + +int fd_set_recv_ecn(int fd, int family) { + unsigned int tos = 1; + + switch (family) { + case AF_INET: + if (setsockopt(fd, IPPROTO_IP, IP_RECVTOS, &tos, + static_cast(sizeof(tos))) == -1) { + return -1; + } + + return 0; + case AF_INET6: + if (setsockopt(fd, IPPROTO_IPV6, IPV6_RECVTCLASS, &tos, + static_cast(sizeof(tos))) == -1) { + return -1; + } + + return 0; + } + + return -1; +} #endif // ENABLE_HTTP3 } // namespace util diff --git a/src/util.h b/src/util.h index 4d5cef37..e154b6ed 100644 --- a/src/util.h +++ b/src/util.h @@ -918,6 +918,10 @@ int daemonize(int nochdir, int noclose); #ifdef ENABLE_HTTP3 int msghdr_get_local_addr(Address &dest, msghdr *msg, int family); + +unsigned int msghdr_get_ecn(msghdr *msg, int family); + +int fd_set_recv_ecn(int fd, int family); #endif // ENABLE_HTTP3 } // namespace util From 7eb179069df4d14585a382519416dc371f61ba60 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 5 Nov 2021 22:51:36 +0900 Subject: [PATCH 113/142] Remove useless assignments --- lib/nghttp2_frame.c | 2 -- src/shrpx_client_handler.cc | 1 - src/shrpx_http3_upstream.cc | 4 +--- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/nghttp2_frame.c b/lib/nghttp2_frame.c index 382a26c8..3648b238 100644 --- a/lib/nghttp2_frame.c +++ b/lib/nghttp2_frame.c @@ -654,8 +654,6 @@ int nghttp2_frame_unpack_goaway_payload2(nghttp2_goaway *frame, var_gift_payloadlen = 0; } - payloadlen -= var_gift_payloadlen; - if (!var_gift_payloadlen) { var_gift_payload = NULL; } else { diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 415df864..a8975cab 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -884,7 +884,6 @@ DownstreamAddr *ClientHandler::get_downstream_addr(int &err, err = -1; return nullptr; } - aff_idx = i; } return addr; diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index de3704f0..733cedb4 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -195,9 +195,7 @@ void qlog_write(void *user_data, uint32_t flags, const void *data, void Http3Upstream::qlog_write(const void *data, size_t datalen, bool fin) { assert(qlog_fd_ != -1); - ssize_t nwrite; - - while ((nwrite = write(qlog_fd_, data, datalen)) == -1 && errno == EINTR) + while (write(qlog_fd_, data, datalen) == -1 && errno == EINTR) ; if (fin) { From deae6c95b176eff00f547508dc19d2fdc87b89bb Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 5 Nov 2021 23:23:00 +0900 Subject: [PATCH 114/142] nghttpx: Send ECN --- src/shrpx_http3_upstream.cc | 30 +++++++++++++++++----------- src/shrpx_http3_upstream.h | 4 ++-- src/shrpx_quic.cc | 9 ++++++--- src/shrpx_quic.h | 4 ++-- src/shrpx_quic_connection_handler.cc | 20 +++++++++---------- src/util.cc | 21 +++++++++++++++++++ src/util.h | 2 ++ 7 files changed, 61 insertions(+), 29 deletions(-) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 733cedb4..3fc40bd4 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -712,6 +712,9 @@ int Http3Upstream::write_streams() { std::min(static_cast(64_k), ngtcp2_conn_get_send_quantum(conn_)) / max_udp_payload_size; ngtcp2_pkt_info pi; +#ifdef UDP_SEGMENT + ngtcp2_pkt_info prev_pi; +#endif // UDP_SEGMENT uint8_t *bufpos = buf.data(); ngtcp2_path_storage ps, prev_ps; size_t pktcnt = 0; @@ -815,7 +818,8 @@ int Http3Upstream::write_streams() { send_packet(static_cast(prev_ps.path.user_data), prev_ps.path.remote.addr, prev_ps.path.remote.addrlen, prev_ps.path.local.addr, prev_ps.path.local.addrlen, - buf.data(), bufpos - buf.data(), max_udp_payload_size); + prev_pi, buf.data(), bufpos - buf.data(), + max_udp_payload_size); reset_idle_timer(); } @@ -832,17 +836,19 @@ int Http3Upstream::write_streams() { #ifdef UDP_SEGMENT if (pktcnt == 0) { ngtcp2_path_copy(&prev_ps.path, &ps.path); - } else if (!ngtcp2_path_eq(&prev_ps.path, &ps.path)) { + prev_pi = pi; + } else if (!ngtcp2_path_eq(&prev_ps.path, &ps.path) || + prev_pi.ecn != pi.ecn) { send_packet(static_cast(prev_ps.path.user_data), prev_ps.path.remote.addr, prev_ps.path.remote.addrlen, - prev_ps.path.local.addr, prev_ps.path.local.addrlen, + prev_ps.path.local.addr, prev_ps.path.local.addrlen, prev_pi, buf.data(), bufpos - buf.data() - nwrite, max_udp_payload_size); send_packet(static_cast(ps.path.user_data), ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, bufpos - nwrite, - nwrite, max_udp_payload_size); + ps.path.local.addr, ps.path.local.addrlen, pi, + bufpos - nwrite, nwrite, max_udp_payload_size); ngtcp2_conn_update_pkt_tx_time(conn_, ts); reset_idle_timer(); @@ -856,7 +862,7 @@ int Http3Upstream::write_streams() { static_cast(nwrite) < max_udp_payload_size) { send_packet(static_cast(ps.path.user_data), ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, buf.data(), + ps.path.local.addr, ps.path.local.addrlen, pi, buf.data(), bufpos - buf.data(), max_udp_payload_size); ngtcp2_conn_update_pkt_tx_time(conn_, ts); @@ -869,7 +875,7 @@ int Http3Upstream::write_streams() { #else // !UDP_SEGMENT send_packet(static_cast(ps.path.user_data), ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr, - ps.path.local.addrlen, buf.data(), bufpos - buf.data(), 0); + ps.path.local.addrlen, pi, buf.data(), bufpos - buf.data(), 0); if (++pktcnt == max_pktcnt) { ngtcp2_conn_update_pkt_tx_time(conn_, ts); @@ -1411,7 +1417,7 @@ void Http3Upstream::on_handler_delete() { send_packet(static_cast(ps.path.user_data), ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr, - ps.path.local.addrlen, conn_close_.data(), nwrite, 0); + ps.path.local.addrlen, pi, conn_close_.data(), nwrite, 0); } auto d = @@ -1716,10 +1722,10 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr, int Http3Upstream::send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, size_t remote_salen, const sockaddr *local_sa, size_t local_salen, - const uint8_t *data, size_t datalen, - size_t gso_size) { + const ngtcp2_pkt_info &pi, const uint8_t *data, + size_t datalen, size_t gso_size) { auto rv = quic_send_packet(faddr, remote_sa, remote_salen, local_sa, - local_salen, data, datalen, gso_size); + local_salen, pi, data, datalen, gso_size); switch (rv) { case 0: return 0; @@ -1776,7 +1782,7 @@ int Http3Upstream::handle_error() { send_packet(static_cast(ps.path.user_data), ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr, - ps.path.local.addrlen, conn_close_.data(), nwrite, 0); + ps.path.local.addrlen, pi, conn_close_.data(), nwrite, 0); return -1; } diff --git a/src/shrpx_http3_upstream.h b/src/shrpx_http3_upstream.h index 5df95283..8df63ea0 100644 --- a/src/shrpx_http3_upstream.h +++ b/src/shrpx_http3_upstream.h @@ -151,8 +151,8 @@ public: void idle_close(); int send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, size_t remote_salen, const sockaddr *local_sa, - size_t local_salen, const uint8_t *data, size_t datalen, - size_t gso_size); + size_t local_salen, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen, size_t gso_size); void qlog_write(const void *data, size_t datalen, bool fin); int open_qlog_file(const StringRef &dir, const ngtcp2_cid &scid) const; diff --git a/src/shrpx_quic.cc b/src/shrpx_quic.cc index 224fbbb1..fcb2cf51 100644 --- a/src/shrpx_quic.cc +++ b/src/shrpx_quic.cc @@ -57,8 +57,8 @@ ngtcp2_tstamp quic_timestamp() { int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, size_t remote_salen, const sockaddr *local_sa, - size_t local_salen, const uint8_t *data, size_t datalen, - size_t gso_size) { + size_t local_salen, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen, size_t gso_size) { iovec msg_iov = {const_cast(data), datalen}; msghdr msg{}; msg.msg_name = const_cast(remote_sa); @@ -123,6 +123,8 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, msg.msg_controllen = controllen; + util::fd_set_send_ecn(faddr->fd, local_sa->sa_family, pi.ecn); + ssize_t nwrite; do { @@ -142,7 +144,8 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, LOG(INFO) << "QUIC sent packet: local=" << util::to_numeric_addr(local_sa, local_salen) << " remote=" << util::to_numeric_addr(remote_sa, remote_salen) - << " " << nwrite << " bytes"; + << " ecn=" << log::hex << pi.ecn << log::dec << " " << nwrite + << " bytes"; } return 0; diff --git a/src/shrpx_quic.h b/src/shrpx_quic.h index 8f400254..cf1e0c1b 100644 --- a/src/shrpx_quic.h +++ b/src/shrpx_quic.h @@ -81,8 +81,8 @@ ngtcp2_tstamp quic_timestamp(); int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, size_t remote_salen, const sockaddr *local_sa, - size_t local_salen, const uint8_t *data, size_t datalen, - size_t gso_size); + size_t local_salen, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen, size_t gso_size); int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, const uint8_t *server_id, uint8_t km_id, diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 38249e29..2a04df9b 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -491,8 +491,8 @@ int QUICConnectionHandler::send_retry( buf.resize(nwrite); quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, - &local_addr.su.sa, local_addr.len, buf.data(), buf.size(), - 0); + &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{}, + buf.data(), buf.size(), 0); if (generate_quic_hashed_connection_id(idcid, remote_addr, local_addr, idcid) != 0) { @@ -541,8 +541,8 @@ int QUICConnectionHandler::send_version_negotiation( } return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, - &local_addr.su.sa, local_addr.len, buf.data(), nwrite, - 0); + &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{}, + buf.data(), nwrite, 0); } int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr, @@ -604,8 +604,8 @@ int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr, } return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, - &local_addr.su.sa, local_addr.len, buf.data(), nwrite, - 0); + &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{}, + buf.data(), nwrite, 0); } int QUICConnectionHandler::send_connection_close( @@ -630,8 +630,8 @@ int QUICConnectionHandler::send_connection_close( } return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, - &local_addr.su.sa, local_addr.len, buf.data(), nwrite, - 0); + &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{}, + buf.data(), nwrite, 0); } void QUICConnectionHandler::add_connection_id(const ngtcp2_cid &cid, @@ -726,8 +726,8 @@ int CloseWait::handle_packet(const UpstreamAddr *faddr, } if (quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, - &local_addr.su.sa, local_addr.len, pkt.data(), - pkt.size(), 0) != 0) { + &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{}, + pkt.data(), pkt.size(), 0) != 0) { return -1; } diff --git a/src/util.cc b/src/util.cc index eb70bbb3..bb6e5e47 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1770,6 +1770,27 @@ int fd_set_recv_ecn(int fd, int family) { return -1; } + +int fd_set_send_ecn(int fd, int family, unsigned int ecn) { + switch (family) { + case AF_INET: + if (setsockopt(fd, IPPROTO_IP, IP_TOS, &ecn, + static_cast(sizeof(ecn))) == -1) { + return -1; + } + + return 0; + case AF_INET6: + if (setsockopt(fd, IPPROTO_IPV6, IPV6_TCLASS, &ecn, + static_cast(sizeof(ecn))) == -1) { + return -1; + } + + return 0; + } + + return -1; +} #endif // ENABLE_HTTP3 } // namespace util diff --git a/src/util.h b/src/util.h index e154b6ed..359e785f 100644 --- a/src/util.h +++ b/src/util.h @@ -922,6 +922,8 @@ int msghdr_get_local_addr(Address &dest, msghdr *msg, int family); unsigned int msghdr_get_ecn(msghdr *msg, int family); int fd_set_recv_ecn(int fd, int family); + +int fd_set_send_ecn(int fd, int family, unsigned int ecn); #endif // ENABLE_HTTP3 } // namespace util From 528d177847a98f00c75a0759e1b2d885193bf6f5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 6 Nov 2021 00:16:53 +0900 Subject: [PATCH 115/142] nghttpx: Fix compile error --- src/shrpx_http3_upstream.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 3fc40bd4..97099909 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -711,10 +711,7 @@ int Http3Upstream::write_streams() { size_t max_pktcnt = std::min(static_cast(64_k), ngtcp2_conn_get_send_quantum(conn_)) / max_udp_payload_size; - ngtcp2_pkt_info pi; -#ifdef UDP_SEGMENT - ngtcp2_pkt_info prev_pi; -#endif // UDP_SEGMENT + ngtcp2_pkt_info pi, prev_pi; uint8_t *bufpos = buf.data(); ngtcp2_path_storage ps, prev_ps; size_t pktcnt = 0; From 68b2295f4eed17c08f9c75b35621bfa26c3abe43 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 6 Nov 2021 16:28:49 +0900 Subject: [PATCH 116/142] nghttpx: Set IP_PMTUDISC_DO explicitly --- src/shrpx_worker.cc | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 89423537..c0169066 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -833,6 +833,19 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { close(fd); continue; } + +# if defined(IPV6_MTU_DISCOVER) && defined(IP_PMTUDISC_DO) + int mtu_disc = IP_PMTUDISC_DO; + if (setsockopt(fd, IPPROTO_IPV6, IPV6_MTU_DISCOVER, &mtu_disc, + static_cast(sizeof(mtu_disc))) == -1) { + auto error = errno; + LOG(WARN) + << "Failed to set IPV6_MTU_DISCOVER option to listener socket: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + close(fd); + continue; + } +# endif // defined(IPV6_MTU_DISCOVER) && defined(IP_PMTUDISC_DO) } else { if (setsockopt(fd, IPPROTO_IP, IP_PKTINFO, &val, static_cast(sizeof(val))) == -1) { @@ -842,6 +855,18 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { close(fd); continue; } + +# if defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DO) + int mtu_disc = IP_PMTUDISC_DO; + if (setsockopt(fd, IPPROTO_IP, IP_MTU_DISCOVER, &mtu_disc, + static_cast(sizeof(mtu_disc))) == -1) { + auto error = errno; + LOG(WARN) << "Failed to set IP_MTU_DISCOVER option to listener socket: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + close(fd); + continue; + } +# endif // defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DO) } util::fd_set_recv_ecn(fd, faddr.family); From 2aed077761aa96f4a7891e0736e6463d2c9f2050 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 6 Nov 2021 17:19:02 +0900 Subject: [PATCH 117/142] Merge fd_set_recv_ecn to create_quic_server_socket and handle errors --- src/shrpx_worker.cc | 20 ++++++++++++++++++-- src/util.cc | 23 ----------------------- src/util.h | 2 -- 3 files changed, 18 insertions(+), 27 deletions(-) diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index c0169066..04bae7ed 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -834,6 +834,15 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { continue; } + if (setsockopt(fd, IPPROTO_IPV6, IPV6_RECVTCLASS, &val, + static_cast(sizeof(val))) == -1) { + auto error = errno; + LOG(WARN) << "Failed to set IPV6_RECVTCLASS option to listener socket: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + close(fd); + continue; + } + # if defined(IPV6_MTU_DISCOVER) && defined(IP_PMTUDISC_DO) int mtu_disc = IP_PMTUDISC_DO; if (setsockopt(fd, IPPROTO_IPV6, IPV6_MTU_DISCOVER, &mtu_disc, @@ -856,6 +865,15 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { continue; } + if (setsockopt(fd, IPPROTO_IP, IP_RECVTOS, &val, + static_cast(sizeof(val))) == -1) { + auto error = errno; + LOG(WARN) << "Failed to set IP_RECVTOS option to listener socket: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + close(fd); + continue; + } + # if defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DO) int mtu_disc = IP_PMTUDISC_DO; if (setsockopt(fd, IPPROTO_IP, IP_MTU_DISCOVER, &mtu_disc, @@ -869,8 +887,6 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { # endif // defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DO) } - util::fd_set_recv_ecn(fd, faddr.family); - if (bind(fd, rp->ai_addr, rp->ai_addrlen) == -1) { auto error = errno; LOG(WARN) << "bind() syscall failed: " diff --git a/src/util.cc b/src/util.cc index bb6e5e47..7ee5acf9 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1748,29 +1748,6 @@ unsigned int msghdr_get_ecn(msghdr *msg, int family) { return 0; } -int fd_set_recv_ecn(int fd, int family) { - unsigned int tos = 1; - - switch (family) { - case AF_INET: - if (setsockopt(fd, IPPROTO_IP, IP_RECVTOS, &tos, - static_cast(sizeof(tos))) == -1) { - return -1; - } - - return 0; - case AF_INET6: - if (setsockopt(fd, IPPROTO_IPV6, IPV6_RECVTCLASS, &tos, - static_cast(sizeof(tos))) == -1) { - return -1; - } - - return 0; - } - - return -1; -} - int fd_set_send_ecn(int fd, int family, unsigned int ecn) { switch (family) { case AF_INET: diff --git a/src/util.h b/src/util.h index 359e785f..e7a07242 100644 --- a/src/util.h +++ b/src/util.h @@ -921,8 +921,6 @@ int msghdr_get_local_addr(Address &dest, msghdr *msg, int family); unsigned int msghdr_get_ecn(msghdr *msg, int family); -int fd_set_recv_ecn(int fd, int family); - int fd_set_send_ecn(int fd, int family, unsigned int ecn); #endif // ENABLE_HTTP3 From d14d97ab682dfce91aa1ce6431fb15a925c46772 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 9 Nov 2021 11:13:23 +0900 Subject: [PATCH 118/142] Rename send_stop_sending to stop_sending --- src/h2load_http3_session.cc | 12 +++++------- src/h2load_http3_session.h | 2 +- src/shrpx_http3_upstream.cc | 14 +++++++------- src/shrpx_http3_upstream.h | 2 +- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/h2load_http3_session.cc b/src/h2load_http3_session.cc index 40970f7a..a62bb70c 100644 --- a/src/h2load_http3_session.cc +++ b/src/h2load_http3_session.cc @@ -213,19 +213,17 @@ void Http3Session::recv_header(int64_t stream_id, const nghttp3_vec *name, } namespace { -int send_stop_sending(nghttp3_conn *conn, int64_t stream_id, - uint64_t app_error_code, void *user_data, - void *stream_user_data) { +int stop_sending(nghttp3_conn *conn, int64_t stream_id, uint64_t app_error_code, + void *user_data, void *stream_user_data) { auto s = static_cast(user_data); - if (s->send_stop_sending(stream_id, app_error_code) != 0) { + if (s->stop_sending(stream_id, app_error_code) != 0) { return NGHTTP3_ERR_CALLBACK_FAILURE; } return 0; } } // namespace -int Http3Session::send_stop_sending(int64_t stream_id, - uint64_t app_error_code) { +int Http3Session::stop_sending(int64_t stream_id, uint64_t app_error_code) { auto rv = ngtcp2_conn_shutdown_stream_read(client_->quic.conn, stream_id, app_error_code); if (rv != 0) { @@ -300,7 +298,7 @@ int Http3Session::init_conn() { nullptr, // begin_trailers h2load::recv_header, nullptr, // end_trailers - h2load::send_stop_sending, + h2load::stop_sending, }; auto config = client_->worker->config; diff --git a/src/h2load_http3_session.h b/src/h2load_http3_session.h index 27ff025b..cdd194dd 100644 --- a/src/h2load_http3_session.h +++ b/src/h2load_http3_session.h @@ -51,7 +51,7 @@ public: void begin_headers(int64_t stream_id); void recv_header(int64_t stream_id, const nghttp3_vec *name, const nghttp3_vec *value); - int send_stop_sending(int64_t stream_id, uint64_t app_error_code); + int stop_sending(int64_t stream_id, uint64_t app_error_code); int close_stream(int64_t stream_id, uint64_t app_error_code); int shutdown_stream_read(int64_t stream_id); diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 97099909..87f219db 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -2353,12 +2353,12 @@ int Http3Upstream::http_stream_close(Downstream *downstream, } namespace { -int http_send_stop_sending(nghttp3_conn *conn, int64_t stream_id, - uint64_t app_error_code, void *user_data, - void *stream_user_data) { +int http_stop_sending(nghttp3_conn *conn, int64_t stream_id, + uint64_t app_error_code, void *user_data, + void *stream_user_data) { auto upstream = static_cast(user_data); - if (upstream->http_send_stop_sending(stream_id, app_error_code) != 0) { + if (upstream->http_stop_sending(stream_id, app_error_code) != 0) { return NGHTTP3_ERR_CALLBACK_FAILURE; } @@ -2366,8 +2366,8 @@ int http_send_stop_sending(nghttp3_conn *conn, int64_t stream_id, } } // namespace -int Http3Upstream::http_send_stop_sending(int64_t stream_id, - uint64_t app_error_code) { +int Http3Upstream::http_stop_sending(int64_t stream_id, + uint64_t app_error_code) { auto rv = ngtcp2_conn_shutdown_stream_read(conn_, stream_id, app_error_code); if (ngtcp2_err_is_fatal(rv)) { ULOG(ERROR, this) << "ngtcp2_conn_shutdown_stream_read: " @@ -2422,7 +2422,7 @@ int Http3Upstream::setup_httpconn() { nullptr, // begin_trailers nullptr, // recv_trailer nullptr, // end_trailers - shrpx::http_send_stop_sending, + shrpx::http_stop_sending, shrpx::http_end_stream, shrpx::http_reset_stream, }; diff --git a/src/shrpx_http3_upstream.h b/src/shrpx_http3_upstream.h index 8df63ea0..0e29356e 100644 --- a/src/shrpx_http3_upstream.h +++ b/src/shrpx_http3_upstream.h @@ -141,7 +141,7 @@ public: int http_acked_stream_data(Downstream *downstream, uint64_t datalen); int http_shutdown_stream_read(int64_t stream_id); int http_reset_stream(int64_t stream_id, uint64_t app_error_code); - int http_send_stop_sending(int64_t stream_id, uint64_t app_error_code); + int http_stop_sending(int64_t stream_id, uint64_t app_error_code); int http_recv_data(Downstream *downstream, const uint8_t *data, size_t datalen); int handshake_completed(); From 918e4ea46b53d74f3ca217237665dc5de0f2dbab Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 9 Nov 2021 15:44:06 +0900 Subject: [PATCH 119/142] Compile with the latest ngtcp2 --- src/h2load_quic.cc | 20 ++++++++++++++++---- src/shrpx_http3_upstream.cc | 14 ++++++++++---- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/h2load_quic.cc b/src/h2load_quic.cc index 6d729b80..7d3281b5 100644 --- a/src/h2load_quic.cc +++ b/src/h2load_quic.cc @@ -480,8 +480,14 @@ int Client::quic_init(const sockaddr *local_addr, socklen_t local_addrlen, params.max_idle_timeout = 30 * NGTCP2_SECONDS; auto path = ngtcp2_path{ - {local_addrlen, const_cast(local_addr)}, - {remote_addrlen, const_cast(remote_addr)}, + { + const_cast(local_addr), + local_addrlen, + }, + { + const_cast(remote_addr), + remote_addrlen, + }, }; assert(config->npn_list.size()); @@ -647,8 +653,14 @@ int Client::read_quic() { ++worker->stats.udp_dgram_recv; auto path = ngtcp2_path{ - {local_addr.len, &local_addr.su.sa}, - {addrlen, &su.sa}, + { + &local_addr.su.sa, + local_addr.len, + }, + { + &su.sa, + addrlen, + }, }; rv = ngtcp2_conn_read_pkt(quic.conn, &path, &pi, buf.data(), nread, diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 87f219db..8fdd786b 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -663,8 +663,14 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, params.stateless_reset_token_present = 1; auto path = ngtcp2_path{ - {local_addr.len, const_cast(&local_addr.su.sa)}, - {remote_addr.len, const_cast(&remote_addr.su.sa)}, + { + const_cast(&local_addr.su.sa), + local_addr.len, + }, + { + const_cast(&remote_addr.su.sa), + remote_addr.len, + }, const_cast(faddr), }; @@ -1639,12 +1645,12 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr, auto path = ngtcp2_path{ { - local_addr.len, const_cast(&local_addr.su.sa), + local_addr.len, }, { - remote_addr.len, const_cast(&remote_addr.su.sa), + remote_addr.len, }, const_cast(faddr), }; From 3c4449c046d04263da53ded448333089b9213a9d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 10 Nov 2021 19:18:48 +0900 Subject: [PATCH 120/142] nghttpx: Use nghttp3 error code --- src/shrpx_http3_upstream.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 8fdd786b..28749287 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -2072,7 +2072,7 @@ int Http3Upstream::http_end_request_headers(Downstream *downstream) { // For HTTP/2 proxy, we require :authority. if (method_token != HTTP_CONNECT && config->http2_proxy && faddr->alt_mode == UpstreamAltMode::NONE && !authority) { - shutdown_stream(downstream, NGHTTP2_PROTOCOL_ERROR); + shutdown_stream(downstream, NGHTTP3_H3_GENERAL_PROTOCOL_ERROR); return 0; } @@ -2283,7 +2283,7 @@ int Http3Upstream::http_end_stream(Downstream *downstream) { if (downstream->end_upload_data() != 0) { if (downstream->get_response_state() != DownstreamState::MSG_COMPLETE) { - shutdown_stream(downstream, NGHTTP2_INTERNAL_ERROR); + shutdown_stream(downstream, NGHTTP3_H3_INTERNAL_ERROR); } } From f92f81c05a656e6f84ba281403ca8a0207a7e6f6 Mon Sep 17 00:00:00 2001 From: robaho Date: Sat, 13 Nov 2021 18:35:15 -0600 Subject: [PATCH 121/142] allow setting max frame size for h2load --- src/h2load.cc | 27 ++++++++++++++++++++++++++- src/h2load.h | 1 + src/h2load_http2_session.cc | 7 ++++++- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/h2load.cc b/src/h2load.cc index 1ab52c0b..3fa09ab1 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -106,6 +106,7 @@ Config::Config() max_concurrent_streams(1), window_bits(30), connection_window_bits(30), + max_frame_size(16_k), rate(0), rate_period(1.0), duration(0.0), @@ -2109,6 +2110,11 @@ Options: http/1.1 is used, this specifies the number of HTTP pipelining requests in-flight. Default: 1 + -f, --max-frame-size= + Maximum frame size that the local endpoint is willing to + receive. + Default: )" + << util::utos_unit(config.max_frame_size) << R"( -w, --window-bits= Sets the stream level initial window size to (2**)-1. For QUIC, is capped to 26 (roughly 64MiB). @@ -2301,6 +2307,7 @@ int main(int argc, char **argv) { {"threads", required_argument, nullptr, 't'}, {"max-concurrent-streams", required_argument, nullptr, 'm'}, {"window-bits", required_argument, nullptr, 'w'}, + {"max-frame-size", required_argument, nullptr, 'f'}, {"connection-window-bits", required_argument, nullptr, 'W'}, {"input-file", required_argument, nullptr, 'i'}, {"header", required_argument, nullptr, 'H'}, @@ -2332,7 +2339,7 @@ int main(int argc, char **argv) { {nullptr, 0, nullptr, 0}}; int option_index = 0; auto c = getopt_long(argc, argv, - "hvW:c:d:m:n:p:t:w:H:i:r:T:N:D:B:", long_options, + "hvW:c:d:m:n:p:t:w:f:H:i:r:T:N:D:B:", long_options, &option_index); if (c == -1) { break; @@ -2378,6 +2385,24 @@ int main(int argc, char **argv) { } break; } + case 'f': { + auto n = util::parse_uint_with_unit(optarg); + if (n == -1) { + std::cerr << "--max-frame-size: bad option value: " << optarg + << std::endl; + exit(EXIT_FAILURE); + } + if (static_cast(n) < 16_k) { + std::cerr << "--max-frame-size: minimum 16384" << std::endl; + exit(EXIT_FAILURE); + } + if (static_cast(n) > 16_m - 1) { + std::cerr << "--max-frame-size: maximum 16777215" << std::endl; + exit(EXIT_FAILURE); + } + config.max_frame_size = n; + break; + } case 'H': { char *header = optarg; // Skip first possible ':' in the header name diff --git a/src/h2load.h b/src/h2load.h index 590a6cd2..f1083a1d 100644 --- a/src/h2load.h +++ b/src/h2load.h @@ -95,6 +95,7 @@ struct Config { ssize_t max_concurrent_streams; size_t window_bits; size_t connection_window_bits; + size_t max_frame_size; // rate at which connections should be made size_t rate; ev_tstamp rate_period; diff --git a/src/h2load_http2_session.cc b/src/h2load_http2_session.cc index 06c474c0..9cafa0e3 100644 --- a/src/h2load_http2_session.cc +++ b/src/h2load_http2_session.cc @@ -215,7 +215,7 @@ void Http2Session::on_connect() { nghttp2_option_del(opt); - std::array iv; + std::array iv; size_t niv = 2; iv[0].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH; iv[0].value = 0; @@ -227,6 +227,11 @@ void Http2Session::on_connect() { iv[niv].value = config->header_table_size; ++niv; } + if (config->max_frame_size != 16_k) { + iv[niv].settings_id = NGHTTP2_SETTINGS_MAX_FRAME_SIZE; + iv[niv].value = config->max_frame_size; + ++niv; + } rv = nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, iv.data(), niv); From dec233b9ef19d0cd81ea9edaa545a2723566a3f8 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 16 Nov 2021 22:03:30 +0900 Subject: [PATCH 122/142] cmake: Disable libbpf build by default --- CMakeLists.txt | 7 ++++++- CMakeOptions.txt | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ae0700cb..a4587d81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,7 +67,12 @@ if(LIBNGTCP2_CRYPTO_OPENSSL_FOUND) set(HAVE_LIBNGTCP2_CRYPTO_OPENSSL 1) endif() find_package(Libnghttp3 0.0.0) -find_package(Libbpf 0.4.0) +if(WITH_LIBBPF) + find_package(Libbpf 0.4.0) + if(NOT LIBBPF_FOUND) + message(FATAL_ERROR "libbpf was requested (WITH_LIBBPF=1) but not found.") + endif() +endif() if(OPENSSL_FOUND AND LIBEV_FOUND AND ZLIB_FOUND) set(ENABLE_APP_DEFAULT ON) else() diff --git a/CMakeOptions.txt b/CMakeOptions.txt index 754428a2..6a1a89ef 100644 --- a/CMakeOptions.txt +++ b/CMakeOptions.txt @@ -27,5 +27,6 @@ option(WITH_SPDYLAY "Use spdylay" ${WITH_SPDYLAY_DEFAULT}) option(WITH_MRUBY "Use mruby") option(WITH_NEVERBLEED "Use neverbleed") +option(WITH_LIBBPF "Use libbpf") # vim: ft=cmake: From 344d300cf95bdfbd3fe06b9012a4da375181d847 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 16 Nov 2021 22:09:48 +0900 Subject: [PATCH 123/142] Fix cmake CI build --- .github/workflows/build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 33d8cb18..dc53f626 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -88,8 +88,10 @@ jobs: PREFIX=$PWD/build make -C src install EXTRA_AUTOTOOLS_OPTS="--with-libbpf" + EXTRA_CMAKE_OPTS="-DWITH_LIBBPF=1" echo 'EXTRA_AUTOTOOLS_OPTS='"$EXTRA_AUTOTOOLS_OPTS" >> $GITHUB_ENV + echo 'EXTRA_CMAKE_OPTS='"$EXTRA_CMAKE_OPTS" >> $GITHUB_ENV - name: Build quictls/openssl v1.1.1 if: matrix.http3 == 'http3' && matrix.openssl == 'openssl1' run: | @@ -133,11 +135,12 @@ jobs: PKG_CONFIG_PATH="$PWD/openssl/build/lib/pkgconfig:$PWD/nghttp3/build/lib/pkgconfig:$PWD/ngtcp2/build/lib/pkgconfig:$PWD/libbpf/build/lib64/pkgconfig:$PKG_CONFIG_PATH" LDFLAGS="$LDFLAGS -Wl,-rpath,$PWD/openssl/build/lib -Wl,-rpath,$PWD/libbpf/build/lib64" EXTRA_AUTOTOOLS_OPTS="--enable-http3 $EXTRA_AUTOTOOLS_OPTS" + EXTRA_CMAKE_OPTS="-DENABLE_HTTP3=1 $EXTRA_CMAKE_OPTS" echo 'PKG_CONFIG_PATH='"$PKG_CONFIG_PATH" >> $GITHUB_ENV echo 'LDFLAGS='"$LDFLAGS" >> $GITHUB_ENV echo 'EXTRA_AUTOTOOLS_OPTS='"$EXTRA_AUTOTOOLS_OPTS" >> $GITHUB_ENV - echo 'EXTRA_CMAKE_OPTS=-DENABLE_HTTP3=ON' >> $GITHUB_ENV + echo 'EXTRA_CMAKE_OPTS='"$EXTRA_CMAKE_OPTS" >> $GITHUB_ENV - name: Setup git submodules run: | git submodule update --init From abc15c696d8f47257e0b592f381ba8e9af861259 Mon Sep 17 00:00:00 2001 From: Ryo Ota Date: Thu, 18 Nov 2021 09:12:05 +0900 Subject: [PATCH 124/142] fix quotes in --altsvc example --- doc/nghttpx.1 | 2 +- doc/nghttpx.1.rst | 2 +- src/shrpx.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index a328ba23..b9e0eb3b 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1517,7 +1517,7 @@ they are treated as nothing is specified. They are advertised in alt\-svc header field only in HTTP/1.1 frontend. This option can be used multiple times to specify multiple alternative services. -Example: \fI\%\-\-altsvc\fP="h2,443,,,ma=3600; persist=1\(aq +Example: \fI\%\-\-altsvc\fP="h2,443,,,ma=3600; persist=1" .UNINDENT .INDENT 0.0 .TP diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index 60374068..ad7edbb9 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -1376,7 +1376,7 @@ HTTP advertised in alt-svc header field only in HTTP/1.1 frontend. This option can be used multiple times to specify multiple alternative services. - Example: :option:`--altsvc`\="h2,443,,,ma=3600; persist=1' + Example: :option:`--altsvc`\="h2,443,,,ma=3600; persist=1" .. option:: --http2-altsvc= diff --git a/src/shrpx.cc b/src/shrpx.cc index 26b09925..f81add5d 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -3174,7 +3174,7 @@ HTTP: advertised in alt-svc header field only in HTTP/1.1 frontend. This option can be used multiple times to specify multiple alternative services. - Example: --altsvc="h2,443,,,ma=3600; persist=1' + Example: --altsvc="h2,443,,,ma=3600; persist=1" --http2-altsvc= Just like --altsvc option, but this altsvc is only sent in HTTP/2 frontend. From c3c0403dfaae0f86e5d334fd19fb93281dbcb06a Mon Sep 17 00:00:00 2001 From: Ryo Ota Date: Sun, 12 Dec 2021 23:16:10 +0900 Subject: [PATCH 125/142] Make Docker speak HTTP/3 --- docker/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 977a5877..1478b097 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -65,6 +65,9 @@ RUN git clone --depth 1 https://github.com/nghttp2/nghttp2.git && \ FROM gcr.io/distroless/base-debian11 +COPY --from=build \ + /usr/local/share/nghttp2/ \ + /usr/local/share/nghttp2/ COPY --from=build \ /usr/local/bin/h2load \ /usr/local/bin/nghttpx \ From 02e6cad1210fb2586482b2b819c8f015929bd5e1 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 19 Dec 2021 20:43:38 +0900 Subject: [PATCH 126/142] Bump quictls/openssl versions --- .github/workflows/build.yml | 6 +++--- README.rst | 4 ++-- docker/Dockerfile | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dc53f626..e29de8cc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -95,18 +95,18 @@ jobs: - name: Build quictls/openssl v1.1.1 if: matrix.http3 == 'http3' && matrix.openssl == 'openssl1' run: | - git clone --depth 1 -b OpenSSL_1_1_1l+quic https://github.com/quictls/openssl + git clone --depth 1 -b OpenSSL_1_1_1m+quic https://github.com/quictls/openssl cd openssl ./config enable-tls1_3 --prefix=$PWD/build make -j$(nproc) make install_sw - - name: Build quictls/openssl v3.0.0 + - name: Build quictls/openssl v3.0.x if: matrix.http3 == 'http3' && matrix.openssl == 'openssl3' run: | unset CPPFLAGS unset LDFLAGS - git clone --depth 1 -b openssl-3.0.0+quic https://github.com/quictls/openssl + git clone --depth 1 -b openssl-3.0.1+quic https://github.com/quictls/openssl cd openssl ./config enable-tls1_3 --prefix=$PWD/build --libdir=$PWD/build/lib make -j$(nproc) diff --git a/README.rst b/README.rst index f5bca45c..14019c69 100644 --- a/README.rst +++ b/README.rst @@ -151,7 +151,7 @@ To enable the experimental HTTP/3 support for h2load and nghttpx, the following libraries are required: * `OpenSSL with QUIC support - `_; or + `_; or `BoringSSL `_ (commit f6ef1c560ae5af51e2df5d8d2175bed207b28b8f) * `ngtcp2 `_ @@ -355,7 +355,7 @@ Build custom OpenSSL: .. code-block:: text - $ git clone --depth 1 -b OpenSSL_1_1_1l+quic https://github.com/quictls/openssl + $ git clone --depth 1 -b OpenSSL_1_1_1m+quic https://github.com/quictls/openssl $ cd openssl $ ./config --prefix=$PWD/build --openssldir=/etc/ssl $ make -j$(nproc) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1478b097..e13ca4d2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update && \ zlib1g-dev libev-dev libjemalloc-dev ruby-dev libc-ares-dev bison \ libelf-dev -RUN git clone --depth 1 -b OpenSSL_1_1_1l+quic https://github.com/quictls/openssl && \ +RUN git clone --depth 1 -b OpenSSL_1_1_1m+quic https://github.com/quictls/openssl && \ cd openssl && \ ./config --openssldir=/etc/ssl && \ make -j$(nproc) && \ From e1446fd57a4af5aa6bf02c628f47a7f664a1f83e Mon Sep 17 00:00:00 2001 From: Renaud Date: Sat, 1 Jan 2022 15:21:59 +0100 Subject: [PATCH 127/142] Remove SPDY option for CMake SPDY feature removed in #1091 and release v1.29.0 --- CMakeOptions.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/CMakeOptions.txt b/CMakeOptions.txt index 6a1a89ef..948e3c63 100644 --- a/CMakeOptions.txt +++ b/CMakeOptions.txt @@ -23,8 +23,6 @@ option(WITH_LIBXML2 "Use libxml2" ${WITH_LIBXML2_DEFAULT}) option(WITH_JEMALLOC "Use jemalloc" ${WITH_JEMALLOC_DEFAULT}) -option(WITH_SPDYLAY "Use spdylay" - ${WITH_SPDYLAY_DEFAULT}) option(WITH_MRUBY "Use mruby") option(WITH_NEVERBLEED "Use neverbleed") option(WITH_LIBBPF "Use libbpf") From 8ddb2273b960dc161dfd9b3332ae389b0b5eb11d Mon Sep 17 00:00:00 2001 From: Marcelo Trylesinski Date: Sat, 1 Jan 2022 19:18:14 +0100 Subject: [PATCH 128/142] Update nghttp2.pyx --- python/nghttp2.pyx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/nghttp2.pyx b/python/nghttp2.pyx index c374d786..81f3f3ee 100644 --- a/python/nghttp2.pyx +++ b/python/nghttp2.pyx @@ -1057,8 +1057,7 @@ if asyncio: """HTTP/2 request (stream) handler base class. The class is used to handle the HTTP/2 stream. By default, it does - not nothing. It must be subclassed to handle each event callback - method. + nothing. It must be subclassed to handle each event callback method. The first callback method invoked is on_headers(). It is called when HEADERS frame, which includes request header fields, is From d91ae6987dc73b8f16a6250e2b358ed5e403b153 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 11 Jan 2022 19:40:26 +0900 Subject: [PATCH 129/142] Compile with the latest ngtcp2 --- src/h2load_quic.cc | 4 ++-- src/shrpx_http3_upstream.cc | 6 +++--- src/shrpx_quic_connection_handler.cc | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/h2load_quic.cc b/src/h2load_quic.cc index 7d3281b5..a32b1d46 100644 --- a/src/h2load_quic.cc +++ b/src/h2load_quic.cc @@ -535,12 +535,12 @@ void Client::quic_close_connection() { case quic::ErrorType::Transport: nwrite = ngtcp2_conn_write_connection_close( quic.conn, &ps.path, nullptr, buf.data(), buf.size(), - quic.last_error.code, timestamp(worker->loop)); + quic.last_error.code, nullptr, 0, timestamp(worker->loop)); break; case quic::ErrorType::Application: nwrite = ngtcp2_conn_write_application_close( quic.conn, &ps.path, nullptr, buf.data(), buf.size(), - quic.last_error.code, timestamp(worker->loop)); + quic.last_error.code, nullptr, 0, timestamp(worker->loop)); break; default: assert(0); diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 28749287..90fb9194 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -1406,7 +1406,7 @@ void Http3Upstream::on_handler_delete() { auto nwrite = ngtcp2_conn_write_connection_close( conn_, &ps.path, &pi, conn_close_.data(), conn_close_.size(), - NGTCP2_NO_ERROR, quic_timestamp()); + NGTCP2_NO_ERROR, nullptr, 0, quic_timestamp()); if (nwrite < 0) { if (nwrite != NGTCP2_ERR_INVALID_STATE) { ULOG(ERROR, this) << "ngtcp2_conn_write_connection_close: " @@ -1764,7 +1764,7 @@ int Http3Upstream::handle_error() { if (last_error_.type == quic::ErrorType::Transport) { nwrite = ngtcp2_conn_write_connection_close( conn_, &ps.path, &pi, conn_close_.data(), conn_close_.size(), - last_error_.code, ts); + last_error_.code, nullptr, 0, ts); if (nwrite < 0) { ULOG(ERROR, this) << "ngtcp2_conn_write_connection_close: " << ngtcp2_strerror(nwrite); @@ -1773,7 +1773,7 @@ int Http3Upstream::handle_error() { } else { nwrite = ngtcp2_conn_write_application_close( conn_, &ps.path, &pi, conn_close_.data(), conn_close_.size(), - last_error_.code, ts); + last_error_.code, nullptr, 0, ts); if (nwrite < 0) { ULOG(ERROR, this) << "ngtcp2_conn_write_application_close: " << ngtcp2_strerror(nwrite); diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 2a04df9b..955534ba 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -615,7 +615,8 @@ int QUICConnectionHandler::send_connection_close( std::array buf; auto nwrite = ngtcp2_crypto_write_connection_close( - buf.data(), buf.size(), version, &ini_scid, &ini_dcid, error_code); + buf.data(), buf.size(), version, &ini_scid, &ini_dcid, error_code, + nullptr, 0); if (nwrite < 0) { LOG(ERROR) << "ngtcp2_crypto_write_connection_close failed"; return -1; From deb390cf858f4a5c31b9a69630478762041bde64 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 11 Jan 2022 19:50:45 +0900 Subject: [PATCH 130/142] Fix decoder table size update When applying new header table size acknowledged with SETTINGS ACK by an encoder, change the header table size on a decoder only when it strictly lowers the current maximum table size set by Dynamic Table Size Update from the encoder or the default size 4096 if no Dynamic Table Size Update is received. Previously, the header table size on a decoder is always changed. If a maximum size in SETTINGS are increased (e.g., 4096 -> 8192), and then decreased to the previous value, the decoder incorrectly requires Dynamic Table Size Update from an encoder. --- lib/nghttp2_hd.c | 11 ++++++----- tests/nghttp2_hd_test.c | 10 +++++----- tests/nghttp2_session_test.c | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/nghttp2_hd.c b/lib/nghttp2_hd.c index 5e869315..30ee9b88 100644 --- a/lib/nghttp2_hd.c +++ b/lib/nghttp2_hd.c @@ -1263,6 +1263,8 @@ int nghttp2_hd_inflate_change_table_size( return NGHTTP2_ERR_INVALID_STATE; } + inflater->settings_hd_table_bufsize_max = settings_max_dynamic_table_size; + /* It seems that encoder is not required to send dynamic table size update if the table size is not changed after applying SETTINGS_HEADER_TABLE_SIZE. RFC 7541 is ambiguous here, but this @@ -1275,13 +1277,12 @@ int nghttp2_hd_inflate_change_table_size( /* Remember minimum value, and validate that encoder sends the value less than or equal to this. */ inflater->min_hd_table_bufsize_max = settings_max_dynamic_table_size; + + inflater->ctx.hd_table_bufsize_max = settings_max_dynamic_table_size; + + hd_context_shrink_table_size(&inflater->ctx, NULL); } - inflater->settings_hd_table_bufsize_max = settings_max_dynamic_table_size; - - inflater->ctx.hd_table_bufsize_max = settings_max_dynamic_table_size; - - hd_context_shrink_table_size(&inflater->ctx, NULL); return 0; } diff --git a/tests/nghttp2_hd_test.c b/tests/nghttp2_hd_test.c index 74725d40..657d895f 100644 --- a/tests/nghttp2_hd_test.c +++ b/tests/nghttp2_hd_test.c @@ -734,7 +734,7 @@ void test_nghttp2_hd_change_table_size(void) { CU_ASSERT(4096 == deflater.ctx.hd_table_bufsize_max); - CU_ASSERT(8000 == inflater.ctx.hd_table_bufsize_max); + CU_ASSERT(4096 == inflater.ctx.hd_table_bufsize_max); CU_ASSERT(8000 == inflater.settings_hd_table_bufsize_max); /* This will emit encoding context update with header table size 4096 */ @@ -830,8 +830,8 @@ void test_nghttp2_hd_change_table_size(void) { CU_ASSERT(8000 == deflater.ctx.hd_table_bufsize_max); CU_ASSERT(8000 == nghttp2_hd_deflate_get_max_dynamic_table_size(&deflater)); - CU_ASSERT(8000 == inflater.ctx.hd_table_bufsize_max); - CU_ASSERT(8000 == nghttp2_hd_inflate_get_max_dynamic_table_size(&inflater)); + CU_ASSERT(4096 == inflater.ctx.hd_table_bufsize_max); + CU_ASSERT(4096 == nghttp2_hd_inflate_get_max_dynamic_table_size(&inflater)); CU_ASSERT(8000 == inflater.settings_hd_table_bufsize_max); rv = nghttp2_hd_deflate_hd_bufs(&deflater, &bufs, nva, 2); @@ -856,8 +856,8 @@ void test_nghttp2_hd_change_table_size(void) { CU_ASSERT(8192 == deflater.ctx.hd_table_bufsize_max); CU_ASSERT(8192 == nghttp2_hd_deflate_get_max_dynamic_table_size(&deflater)); - CU_ASSERT(16383 == inflater.ctx.hd_table_bufsize_max); - CU_ASSERT(16383 == nghttp2_hd_inflate_get_max_dynamic_table_size(&inflater)); + CU_ASSERT(8000 == inflater.ctx.hd_table_bufsize_max); + CU_ASSERT(8000 == nghttp2_hd_inflate_get_max_dynamic_table_size(&inflater)); CU_ASSERT(16383 == inflater.settings_hd_table_bufsize_max); rv = nghttp2_hd_deflate_hd_bufs(&deflater, &bufs, nva, 2); diff --git a/tests/nghttp2_session_test.c b/tests/nghttp2_session_test.c index 962e3c13..238ed905 100644 --- a/tests/nghttp2_session_test.c +++ b/tests/nghttp2_session_test.c @@ -5720,7 +5720,7 @@ void test_nghttp2_submit_settings(void) { nghttp2_frame_settings_free(&ack_frame.settings, mem); CU_ASSERT(16 * 1024 == session->local_settings.initial_window_size); - CU_ASSERT(1023 == session->hd_inflater.ctx.hd_table_bufsize_max); + CU_ASSERT(111 == session->hd_inflater.ctx.hd_table_bufsize_max); CU_ASSERT(111 == session->hd_inflater.min_hd_table_bufsize_max); CU_ASSERT(50 == session->local_settings.max_concurrent_streams); From ad0c9eebf780f30351998391a17b6ab618e9e1e6 Mon Sep 17 00:00:00 2001 From: Dimitris Apostolou Date: Thu, 13 Jan 2022 22:21:56 +0200 Subject: [PATCH 131/142] Fix typos --- CMakeLists.txt | 2 +- Makefile.am | 2 +- configure.ac | 2 +- doc/_exts/rubydomain/rubydomain.py | 2 +- integration-tests/nghttpx_http1_test.go | 7 ++++--- lib/nghttp2_buf.h | 2 +- lib/nghttp2_frame.h | 2 +- lib/nghttp2_net.h | 2 +- lib/nghttp2_outbound_item.h | 2 +- lib/nghttp2_pq.h | 2 +- lib/nghttp2_session.c | 2 +- lib/nghttp2_session.h | 2 +- lib/nghttp2_stream.c | 2 +- python/nghttp2.pyx | 4 ++-- src/h2load.cc | 2 +- src/http2_test.cc | 2 +- src/shrpx_api_downstream_connection.cc | 2 +- src/shrpx_api_downstream_connection.h | 2 +- src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- src/shrpx_connection.cc | 4 ++-- src/shrpx_connection_handler.h | 2 +- src/shrpx_dns_resolver.h | 2 +- src/shrpx_downstream_connection.h | 2 +- src/shrpx_health_monitor_downstream_connection.cc | 2 +- src/shrpx_health_monitor_downstream_connection.h | 2 +- src/shrpx_http2_downstream_connection.h | 2 +- src/shrpx_http2_session.cc | 4 ++-- src/shrpx_http2_upstream.cc | 4 ++-- src/shrpx_http3_upstream.cc | 2 +- src/shrpx_null_downstream_connection.cc | 2 +- src/shrpx_null_downstream_connection.h | 2 +- src/shrpx_worker.h | 2 +- src/util.h | 2 +- tests/nghttp2_session_test.c | 6 +++--- third-party/build_config.rb | 2 +- 36 files changed, 45 insertions(+), 44 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a4587d81..e7f2a5a8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -179,7 +179,7 @@ endif() # case "$host" in # *android*) # android_build=yes -# # android does not need -pthread, but needs followng 3 libs for C++ +# # android does not need -pthread, but needs following 3 libs for C++ # APPLDFLAGS="$APPLDFLAGS -lstdc++ -latomic -lsupc++" # dl: openssl requires libdl when it is statically linked. diff --git a/Makefile.am b/Makefile.am index 79174a5a..a5bb7a36 100644 --- a/Makefile.am +++ b/Makefile.am @@ -55,7 +55,7 @@ EXTRA_DIST = nghttpx.conf.sample proxy.pac.sample android-config android-make \ .PHONY: clang-format # Format source files using clang-format. Don't format source files -# under third-party directory since we are not responsible for thier +# under third-party directory since we are not responsible for their # coding style. clang-format: CLANGFORMAT=`git config --get clangformat.binary`; \ diff --git a/configure.ac b/configure.ac index 91146331..fd2b0a8a 100644 --- a/configure.ac +++ b/configure.ac @@ -349,7 +349,7 @@ APPLDFLAGS= case "$host_os" in *android*) android_build=yes - # android does not need -pthread, but needs followng 3 libs for C++ + # android does not need -pthread, but needs following 3 libs for C++ APPLDFLAGS="$APPLDFLAGS -lstdc++ -latomic -lsupc++" ;; *) diff --git a/doc/_exts/rubydomain/rubydomain.py b/doc/_exts/rubydomain/rubydomain.py index 62a04285..db352334 100644 --- a/doc/_exts/rubydomain/rubydomain.py +++ b/doc/_exts/rubydomain/rubydomain.py @@ -493,7 +493,7 @@ class RubyModuleIndex(Index): # list of all modules, sorted by module name modules = sorted(_iteritems(self.domain.data['modules']), key=lambda x: x[0].lower()) - # sort out collapsable modules + # sort out collapsible modules prev_modname = '' num_toplevels = 0 for modname, (docname, synopsis, platforms, deprecated) in modules: diff --git a/integration-tests/nghttpx_http1_test.go b/integration-tests/nghttpx_http1_test.go index 9eb80e60..7159b462 100644 --- a/integration-tests/nghttpx_http1_test.go +++ b/integration-tests/nghttpx_http1_test.go @@ -5,14 +5,15 @@ import ( "bytes" "encoding/json" "fmt" - "golang.org/x/net/http2/hpack" - "golang.org/x/net/websocket" "io" "net/http" "regexp" "syscall" "testing" "time" + + "golang.org/x/net/http2/hpack" + "golang.org/x/net/websocket" ) // TestH1H1PlainGET tests whether simple HTTP/1 GET request works. @@ -34,7 +35,7 @@ func TestH1H1PlainGET(t *testing.T) { } // TestH1H1PlainGETClose tests whether simple HTTP/1 GET request with -// Connetion: close request header field works. +// Connection: close request header field works. func TestH1H1PlainGETClose(t *testing.T) { st := newServerTester(nil, t, noopHandler) defer st.Close() diff --git a/lib/nghttp2_buf.h b/lib/nghttp2_buf.h index 06cce67a..45f62f16 100644 --- a/lib/nghttp2_buf.h +++ b/lib/nghttp2_buf.h @@ -99,7 +99,7 @@ void nghttp2_buf_free(nghttp2_buf *buf, nghttp2_mem *mem); * |new_cap|. If extensions took place, buffer pointers in |buf| will * change. * - * This function returns 0 if it succeeds, or one of the followings + * This function returns 0 if it succeeds, or one of the following * negative error codes: * * NGHTTP2_ERR_NOMEM diff --git a/lib/nghttp2_frame.h b/lib/nghttp2_frame.h index 4b9222ac..3859926e 100644 --- a/lib/nghttp2_frame.h +++ b/lib/nghttp2_frame.h @@ -46,7 +46,7 @@ #define NGHTTP2_MAX_FRAME_SIZE_MIN (1 << 14) #define NGHTTP2_MAX_PAYLOADLEN 16384 -/* The one frame buffer length for tranmission. We may use several of +/* The one frame buffer length for transmission. We may use several of them to support CONTINUATION. To account for Pad Length field, we allocate extra 1 byte, which saves extra large memcopying. */ #define NGHTTP2_FRAMEBUF_CHUNKLEN \ diff --git a/lib/nghttp2_net.h b/lib/nghttp2_net.h index 95ffee74..582099b9 100644 --- a/lib/nghttp2_net.h +++ b/lib/nghttp2_net.h @@ -42,7 +42,7 @@ #if defined(WIN32) /* Windows requires ws2_32 library for ntonl family functions. We define inline functions for those function so that we don't have - dependeny on that lib. */ + dependency on that lib. */ # ifdef _MSC_VER # define STIN static __inline diff --git a/lib/nghttp2_outbound_item.h b/lib/nghttp2_outbound_item.h index b5f503a3..bd4611b5 100644 --- a/lib/nghttp2_outbound_item.h +++ b/lib/nghttp2_outbound_item.h @@ -111,7 +111,7 @@ struct nghttp2_outbound_item { to this structure to avoid frequent memory allocation. */ nghttp2_ext_frame_payload ext_frame_payload; nghttp2_aux_data aux_data; - /* The priority used in priority comparion. Smaller is served + /* The priority used in priority comparison. Smaller is served earlier. For PING, SETTINGS and non-DATA frames (excluding response HEADERS frame) have dedicated cycle value defined above. For DATA frame, cycle is computed by taking into account of diff --git a/lib/nghttp2_pq.h b/lib/nghttp2_pq.h index 2d7b702a..7b7b7392 100644 --- a/lib/nghttp2_pq.h +++ b/lib/nghttp2_pq.h @@ -114,7 +114,7 @@ typedef int (*nghttp2_pq_item_cb)(nghttp2_pq_entry *item, void *arg); void nghttp2_pq_update(nghttp2_pq *pq, nghttp2_pq_item_cb fun, void *arg); /* - * Applys |fun| to each item in |pq|. The |arg| is passed as arg + * Applies |fun| to each item in |pq|. The |arg| is passed as arg * parameter to callback function. This function must not change the * ordering key. If the return value from callback is nonzero, this * function returns 1 immediately without iterating remaining items. diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 36f1179f..380a47c1 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -5341,7 +5341,7 @@ static ssize_t inbound_frame_compute_pad(nghttp2_inbound_frame *iframe) { /* * This function returns the effective payload length in the data of - * length |readlen| when the remaning payload is |payloadleft|. The + * length |readlen| when the remaining payload is |payloadleft|. The * |payloadleft| does not include |readlen|. If padding was started * strictly before this data chunk, this function returns -1. */ diff --git a/lib/nghttp2_session.h b/lib/nghttp2_session.h index 07bfbb6c..907b1704 100644 --- a/lib/nghttp2_session.h +++ b/lib/nghttp2_session.h @@ -408,7 +408,7 @@ int nghttp2_session_add_rst_stream(nghttp2_session *session, int32_t stream_id, uint32_t error_code); /* - * Adds PING frame. This is a convenient functin built on top of + * Adds PING frame. This is a convenient function built on top of * nghttp2_session_add_frame() to add PING easily. * * If the |opaque_data| is not NULL, it must point to 8 bytes memory diff --git a/lib/nghttp2_stream.c b/lib/nghttp2_stream.c index 96e1d9fe..f4c80a24 100644 --- a/lib/nghttp2_stream.c +++ b/lib/nghttp2_stream.c @@ -33,7 +33,7 @@ #include "nghttp2_frame.h" /* Maximum distance between any two stream's cycle in the same - prirority queue. Imagine stream A's cycle is A, and stream B's + priority queue. Imagine stream A's cycle is A, and stream B's cycle is B, and A < B. The cycle is unsigned 32 bit integer, it may get overflow. Because of how we calculate the next cycle value, if B - A is less than or equals to diff --git a/python/nghttp2.pyx b/python/nghttp2.pyx index 81f3f3ee..953a6481 100644 --- a/python/nghttp2.pyx +++ b/python/nghttp2.pyx @@ -701,7 +701,7 @@ cdef class _HTTP2SessionCoreBase: if outbuflen == 0: break if outbuflen < 0: - raise Exception('nghttp2_session_mem_send faild: {}'.format\ + raise Exception('nghttp2_session_mem_send failed: {}'.format\ (_strerror(outbuflen))) self.transport.write(outbuf[:outbuflen]) @@ -1083,7 +1083,7 @@ if asyncio: address. client_certificate - May contain the client certifcate in its non-binary form + May contain the client certificate in its non-binary form stream_id Stream ID of this stream diff --git a/src/h2load.cc b/src/h2load.cc index 3fa09ab1..32cd5add 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -2128,7 +2128,7 @@ Options: -H, --header=
Add/Override a header to the requests. --ciphers= - Set allowed cipher list for TLSv1.2 or ealier. The + Set allowed cipher list for TLSv1.2 or earlier. The format of the string is described in OpenSSL ciphers(1). Default: )" << config.ciphers << R"( diff --git a/src/http2_test.cc b/src/http2_test.cc index b998ee71..2d1ab43b 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -916,7 +916,7 @@ void test_http2_rewrite_clean_path(void) { CU_ASSERT("/delta%3A" == http2::rewrite_clean_path( balloc, StringRef::from_lit("/delta%3a"))); - // path component is normalized before mathcing + // path component is normalized before matching CU_ASSERT( "/alpha/bravo/" == http2::rewrite_clean_path( diff --git a/src/shrpx_api_downstream_connection.cc b/src/shrpx_api_downstream_connection.cc index 063eed70..254ab59e 100644 --- a/src/shrpx_api_downstream_connection.cc +++ b/src/shrpx_api_downstream_connection.cc @@ -463,7 +463,7 @@ int APIDownstreamConnection::on_read() { return 0; } int APIDownstreamConnection::on_write() { return 0; } -void APIDownstreamConnection::on_upstream_change(Upstream *uptream) {} +void APIDownstreamConnection::on_upstream_change(Upstream *upstream) {} bool APIDownstreamConnection::poolable() const { return false; } diff --git a/src/shrpx_api_downstream_connection.h b/src/shrpx_api_downstream_connection.h index af6b95eb..5d4182f0 100644 --- a/src/shrpx_api_downstream_connection.h +++ b/src/shrpx_api_downstream_connection.h @@ -81,7 +81,7 @@ public: virtual int on_read(); virtual int on_write(); - virtual void on_upstream_change(Upstream *uptream); + virtual void on_upstream_change(Upstream *upstream); // true if this object is poolable. virtual bool poolable() const; diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index b8acbe50..e158ba6d 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -4463,7 +4463,7 @@ int configure_downstream_group(Config *config, bool http2_proxy, if (!g.mruby_file.empty()) { if (mruby::create_mruby_context(g.mruby_file) == nullptr) { LOG(config->ignore_per_pattern_mruby_error ? ERROR : FATAL) - << "backend: Could not compile mruby flie for pattern " + << "backend: Could not compile mruby file for pattern " << g.pattern; if (!config->ignore_per_pattern_mruby_error) { return -1; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index d88ac7a9..b5871fff 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -691,7 +691,7 @@ struct TLSConfig { ev_tstamp idle_timeout; } dyn_rec; - // OCSP realted configurations + // OCSP related configurations struct { ev_tstamp update_interval; StringRef fetch_ocsp_response_file; diff --git a/src/shrpx_connection.cc b/src/shrpx_connection.cc index af3cc05a..b4211adc 100644 --- a/src/shrpx_connection.cc +++ b/src/shrpx_connection.cc @@ -753,7 +753,7 @@ ssize_t Connection::write_tls(const void *data, size_t len) { // length) on SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE. // get_write_limit() may return smaller length than previously // passed to SSL_write, which violates OpenSSL assumption. To avoid - // this, we keep last legnth passed to SSL_write to + // this, we keep last length passed to SSL_write to // tls.last_writelen if SSL_write indicated I/O blocking. if (tls.last_writelen == 0) { len = std::min(len, wlimit.avail()); @@ -831,7 +831,7 @@ ssize_t Connection::read_tls(void *data, size_t len) { // length) on SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE. // rlimit_.avail() or rlimit_.avail() may return different length // than the length previously passed to SSL_read, which violates - // OpenSSL assumption. To avoid this, we keep last legnth passed + // OpenSSL assumption. To avoid this, we keep last length passed // to SSL_read to tls_last_readlen_ if SSL_read indicated I/O // blocking. if (tls.last_readlen == 0) { diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h index 8a515e4b..e89b6d52 100644 --- a/src/shrpx_connection_handler.h +++ b/src/shrpx_connection_handler.h @@ -163,7 +163,7 @@ public: // Cancels ocsp update process void cancel_ocsp_update(); - // Starts ocsp update for certficate |cert_file|. + // Starts ocsp update for certificate |cert_file|. int start_ocsp_update(const char *cert_file); // Reads incoming data from ocsp update process void read_ocsp_chunk(); diff --git a/src/shrpx_dns_resolver.h b/src/shrpx_dns_resolver.h index 696c5430..e622f996 100644 --- a/src/shrpx_dns_resolver.h +++ b/src/shrpx_dns_resolver.h @@ -88,7 +88,7 @@ public: int on_write(int fd); int on_timeout(); // Calls this function when DNS query finished. - void on_result(int staus, hostent *hostent); + void on_result(int status, hostent *hostent); void reset_timeout(); void start_rev(int fd); diff --git a/src/shrpx_downstream_connection.h b/src/shrpx_downstream_connection.h index 12bbbc5b..8efdcbef 100644 --- a/src/shrpx_downstream_connection.h +++ b/src/shrpx_downstream_connection.h @@ -58,7 +58,7 @@ public: virtual int on_write() = 0; virtual int on_timeout() { return 0; } - virtual void on_upstream_change(Upstream *uptream) = 0; + virtual void on_upstream_change(Upstream *upstream) = 0; // true if this object is poolable. virtual bool poolable() const = 0; diff --git a/src/shrpx_health_monitor_downstream_connection.cc b/src/shrpx_health_monitor_downstream_connection.cc index 066a2af9..6e33bc15 100644 --- a/src/shrpx_health_monitor_downstream_connection.cc +++ b/src/shrpx_health_monitor_downstream_connection.cc @@ -98,7 +98,7 @@ int HealthMonitorDownstreamConnection::on_read() { return 0; } int HealthMonitorDownstreamConnection::on_write() { return 0; } -void HealthMonitorDownstreamConnection::on_upstream_change(Upstream *uptream) {} +void HealthMonitorDownstreamConnection::on_upstream_change(Upstream *upstream) {} bool HealthMonitorDownstreamConnection::poolable() const { return false; } diff --git a/src/shrpx_health_monitor_downstream_connection.h b/src/shrpx_health_monitor_downstream_connection.h index 7439e83b..c0eb6334 100644 --- a/src/shrpx_health_monitor_downstream_connection.h +++ b/src/shrpx_health_monitor_downstream_connection.h @@ -49,7 +49,7 @@ public: virtual int on_read(); virtual int on_write(); - virtual void on_upstream_change(Upstream *uptream); + virtual void on_upstream_change(Upstream *upstream); // true if this object is poolable. virtual bool poolable() const; diff --git a/src/shrpx_http2_downstream_connection.h b/src/shrpx_http2_downstream_connection.h index cdc55b9f..0fc7d91c 100644 --- a/src/shrpx_http2_downstream_connection.h +++ b/src/shrpx_http2_downstream_connection.h @@ -60,7 +60,7 @@ public: virtual void on_upstream_change(Upstream *upstream) {} - // This object is not poolable because we dont' have facility to + // This object is not poolable because we don't have facility to // migrate to another Http2Session object. virtual bool poolable() const { return false; } diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 2f9d2e50..37a6eb40 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -206,13 +206,13 @@ Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, on_read_ = &Http2Session::read_noop; on_write_ = &Http2Session::write_noop; - // We will resuse this many times, so use repeat timeout value. The + // We will reuse this many times, so use repeat timeout value. The // timeout value is set later. ev_timer_init(&connchk_timer_, connchk_timeout_cb, 0., 0.); connchk_timer_.data = this; - // SETTINGS ACK timeout is 10 seconds for now. We will resuse this + // SETTINGS ACK timeout is 10 seconds for now. We will reuse this // many times, so use repeat timeout value. ev_timer_init(&settings_timer_, settings_timeout_cb, 0., 0.); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 5f8b5254..8859ec37 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1317,7 +1317,7 @@ int Http2Upstream::downstream_eof(DownstreamConnection *dconn) { downstream->pop_downstream_connection(); // dconn was deleted dconn = nullptr; - // downstream wil be deleted in on_stream_close_callback. + // downstream will be deleted in on_stream_close_callback. if (downstream->get_response_state() == DownstreamState::HEADER_COMPLETE) { // Server may indicate the end of the request by EOF if (LOG_ENABLED(INFO)) { @@ -2183,7 +2183,7 @@ int Http2Upstream::submit_push_promise(const StringRef &scheme, // 4 for :method, :scheme, :path and :authority nva.reserve(4 + req.fs.headers().size()); - // juse use "GET" for now + // just use "GET" for now nva.push_back(http2::make_nv_ll(":method", "GET")); nva.push_back(http2::make_nv_ls_nocopy(":scheme", scheme)); nva.push_back(http2::make_nv_ls_nocopy(":path", path)); diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 90fb9194..89ad3086 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -1023,7 +1023,7 @@ int Http3Upstream::downstream_eof(DownstreamConnection *dconn) { downstream->pop_downstream_connection(); // dconn was deleted dconn = nullptr; - // downstream wil be deleted in on_stream_close_callback. + // downstream will be deleted in on_stream_close_callback. if (downstream->get_response_state() == DownstreamState::HEADER_COMPLETE) { // Server may indicate the end of the request by EOF if (LOG_ENABLED(INFO)) { diff --git a/src/shrpx_null_downstream_connection.cc b/src/shrpx_null_downstream_connection.cc index 3a80b304..cd81c8aa 100644 --- a/src/shrpx_null_downstream_connection.cc +++ b/src/shrpx_null_downstream_connection.cc @@ -74,7 +74,7 @@ int NullDownstreamConnection::on_read() { return 0; } int NullDownstreamConnection::on_write() { return 0; } -void NullDownstreamConnection::on_upstream_change(Upstream *uptream) {} +void NullDownstreamConnection::on_upstream_change(Upstream *upstream) {} bool NullDownstreamConnection::poolable() const { return false; } diff --git a/src/shrpx_null_downstream_connection.h b/src/shrpx_null_downstream_connection.h index 829d659e..7defcc33 100644 --- a/src/shrpx_null_downstream_connection.h +++ b/src/shrpx_null_downstream_connection.h @@ -50,7 +50,7 @@ public: virtual int on_read(); virtual int on_write(); - virtual void on_upstream_change(Upstream *uptream); + virtual void on_upstream_change(Upstream *upstream); // true if this object is poolable. virtual bool poolable() const; diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 724c56ed..49670630 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -439,7 +439,7 @@ private: std::shared_ptr ticket_keys_; std::vector> downstream_addr_groups_; // Worker level blocker for downstream connection. For example, - // this is used when file decriptor is exhausted. + // this is used when file descriptor is exhausted. std::unique_ptr connect_blocker_; bool graceful_shutdown_; diff --git a/src/util.h b/src/util.h index e7a07242..2e71f068 100644 --- a/src/util.h +++ b/src/util.h @@ -572,7 +572,7 @@ std::string ascii_dump(const uint8_t *data, size_t len); // Returns absolute path of executable path. If argc == 0 or |cwd| is // nullptr, this function returns nullptr. If argv[0] starts with -// '/', this function returns argv[0]. Oterwise return cwd + "/" + +// '/', this function returns argv[0]. Otherwise return cwd + "/" + // argv[0]. If non-null is returned, it is NULL-terminated string and // dynamically allocated by malloc. The caller is responsible to free // it. diff --git a/tests/nghttp2_session_test.c b/tests/nghttp2_session_test.c index 238ed905..cb6bdf73 100644 --- a/tests/nghttp2_session_test.c +++ b/tests/nghttp2_session_test.c @@ -2954,7 +2954,7 @@ void test_nghttp2_session_on_request_headers_received(void) { session->local_settings.max_concurrent_streams = NGHTTP2_DEFAULT_MAX_CONCURRENT_STREAMS; - /* Stream ID less than or equal to the previouly received request + /* Stream ID less than or equal to the previously received request HEADERS is just ignored due to race condition */ nghttp2_frame_headers_init(&frame.headers, NGHTTP2_FLAG_END_HEADERS | NGHTTP2_FLAG_PRIORITY, @@ -4650,7 +4650,7 @@ void test_nghttp2_session_reprioritize_stream(void) { CU_ASSERT(10 == stream->weight); CU_ASSERT(&session->root == stream->dep_prev); - /* If depenency to idle stream which is not in depdenency tree yet */ + /* If dependency to idle stream which is not in dependency tree yet */ nghttp2_priority_spec_init(&pri_spec, 3, 99, 0); @@ -9070,7 +9070,7 @@ void test_nghttp2_session_stream_get_state(void) { CU_ASSERT(NGHTTP2_STREAM_STATE_RESERVED_LOCAL == nghttp2_stream_get_state(stream)); - /* Send resposne to push stream 2 with END_STREAM set */ + /* Send response to push stream 2 with END_STREAM set */ nghttp2_submit_response(session, 2, resnv, ARRLEN(resnv), NULL); rv = nghttp2_session_send(session); diff --git a/third-party/build_config.rb b/third-party/build_config.rb index 9e66958c..86dd8152 100644 --- a/third-party/build_config.rb +++ b/third-party/build_config.rb @@ -3,7 +3,7 @@ MRuby::Build.new do |conf| toolchain :gcc if ENV['CC'].include? "gcc" # C++ project needs this. Without this, mruby exception does not - # properly destory C++ object allocated on stack. + # properly destroy C++ object allocated on stack. conf.enable_cxx_exception conf.build_dir = ENV['BUILD_DIR'] From c883b18f2d46a05915ff135c00f892e12b8ba032 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 16 Jan 2022 22:00:27 +0900 Subject: [PATCH 132/142] Pin quic dependency --- .github/workflows/build.yml | 4 ++-- README.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e29de8cc..864afb19 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -114,7 +114,7 @@ jobs: - name: Build nghttp3 if: matrix.http3 == 'http3' run: | - git clone https://github.com/ngtcp2/nghttp3 + git clone --depth 1 -b v0.1.0 https://github.com/ngtcp2/nghttp3 cd nghttp3 autoreconf -i ./configure --prefix=$PWD/build --enable-lib-only @@ -123,7 +123,7 @@ jobs: - name: Build ngtcp2 if: matrix.http3 == 'http3' run: | - git clone https://github.com/ngtcp2/ngtcp2 + git clone --depth 1 -b v0.1.0 https://github.com/ngtcp2/ngtcp2 cd ngtcp2 autoreconf -i ./configure --prefix=$PWD/build --enable-lib-only PKG_CONFIG_PATH="../openssl/build/lib/pkgconfig" diff --git a/README.rst b/README.rst index 14019c69..20f9aea9 100644 --- a/README.rst +++ b/README.rst @@ -366,7 +366,7 @@ Build nghttp3: .. code-block:: text - $ git clone --depth 1 https://github.com/ngtcp2/nghttp3 + $ git clone --depth 1 -b v0.1.0 https://github.com/ngtcp2/nghttp3 $ cd nghttp3 $ autoreconf -i $ ./configure --prefix=$PWD/build --enable-lib-only @@ -378,7 +378,7 @@ Build ngtcp2: .. code-block:: text - $ git clone --depth 1 https://github.com/ngtcp2/ngtcp2 + $ git clone --depth 1 -b v0.1.0 https://github.com/ngtcp2/ngtcp2 $ cd ngtcp2 $ autoreconf -i $ ./configure --prefix=$PWD/build --enable-lib-only \ From 6e6388e7c25e232937704460e7b50fe0ebb46943 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 22 Jan 2022 18:17:09 +0900 Subject: [PATCH 133/142] Depend on nghttp3 v0.1.1 --- .github/workflows/build.yml | 2 +- README.rst | 2 +- docker/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 864afb19..0eeb00f8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -114,7 +114,7 @@ jobs: - name: Build nghttp3 if: matrix.http3 == 'http3' run: | - git clone --depth 1 -b v0.1.0 https://github.com/ngtcp2/nghttp3 + git clone --depth 1 -b v0.1.1 https://github.com/ngtcp2/nghttp3 cd nghttp3 autoreconf -i ./configure --prefix=$PWD/build --enable-lib-only diff --git a/README.rst b/README.rst index 20f9aea9..679b9ded 100644 --- a/README.rst +++ b/README.rst @@ -366,7 +366,7 @@ Build nghttp3: .. code-block:: text - $ git clone --depth 1 -b v0.1.0 https://github.com/ngtcp2/nghttp3 + $ git clone --depth 1 -b v0.1.1 https://github.com/ngtcp2/nghttp3 $ cd nghttp3 $ autoreconf -i $ ./configure --prefix=$PWD/build --enable-lib-only diff --git a/docker/Dockerfile b/docker/Dockerfile index e13ca4d2..f75c9b37 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,7 +15,7 @@ RUN git clone --depth 1 -b OpenSSL_1_1_1m+quic https://github.com/quictls/openss cd .. && \ rm -rf openssl -RUN git clone --depth 1 https://github.com/ngtcp2/nghttp3 && \ +RUN git clone --depth 1 -b v0.1.1 https://github.com/ngtcp2/nghttp3 && \ cd nghttp3 && \ autoreconf -i && \ ./configure --enable-lib-only && \ From 3b0b9a458c9fccdb74002d54d12c6c6ed549de3e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 22 Jan 2022 18:17:30 +0900 Subject: [PATCH 134/142] Build docker image with ngtcp2 v0.1.0 --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f75c9b37..b66ce9f8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -24,7 +24,7 @@ RUN git clone --depth 1 -b v0.1.1 https://github.com/ngtcp2/nghttp3 && \ cd .. && \ rm -rf nghttp3 -RUN git clone --depth 1 https://github.com/ngtcp2/ngtcp2 && \ +RUN git clone --depth 1 -b v0.1.0 https://github.com/ngtcp2/ngtcp2 && \ cd ngtcp2 && \ autoreconf -i && \ ./configure --enable-lib-only \ From 02a564934378926881fad374167eca9e7a840080 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 23 Jan 2022 18:34:59 +0900 Subject: [PATCH 135/142] Require ngtcp2 >= 0.1.0 and nghttp3 >= 0.1.0 --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index fd2b0a8a..06f4ae71 100644 --- a/configure.ac +++ b/configure.ac @@ -525,7 +525,7 @@ fi # ngtcp2 (for src) have_libngtcp2=no if test "x${request_libngtcp2}" != "xno"; then - PKG_CHECK_MODULES([LIBNGTCP2], [libngtcp2 >= 0.0.0], [have_libngtcp2=yes], + PKG_CHECK_MODULES([LIBNGTCP2], [libngtcp2 >= 0.1.0], [have_libngtcp2=yes], [have_libngtcp2=no]) if test "x${have_libngtcp2}" = "xno"; then AC_MSG_NOTICE($LIBNGTCP2_PKG_ERRORS) @@ -584,7 +584,7 @@ fi # nghttp3 (for src) have_libnghttp3=no if test "x${request_libnghttp3}" != "xno"; then - PKG_CHECK_MODULES([LIBNGHTTP3], [libnghttp3 >= 0.0.0], [have_libnghttp3=yes], + PKG_CHECK_MODULES([LIBNGHTTP3], [libnghttp3 >= 0.1.0], [have_libnghttp3=yes], [have_libnghttp3=no]) if test "x${have_libnghttp3}" = "xno"; then AC_MSG_NOTICE($LIBNGHTTP3_PKG_ERRORS) From 227532779439062705991a86de5f335962ba9754 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 23 Jan 2022 19:28:07 +0900 Subject: [PATCH 136/142] nghttpx: Fix the issue that forwarded h3 GET request always has chunked TE --- .github/workflows/build.yml | 3 ++- README.rst | 3 ++- src/shrpx_http3_upstream.cc | 13 ++++++++----- src/shrpx_http3_upstream.h | 2 +- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0eeb00f8..cdc73ca8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -114,8 +114,9 @@ jobs: - name: Build nghttp3 if: matrix.http3 == 'http3' run: | - git clone --depth 1 -b v0.1.1 https://github.com/ngtcp2/nghttp3 + git clone https://github.com/ngtcp2/nghttp3 cd nghttp3 + git checkout 207318c92e0578ac393c31dcd797e4a4dca3e31a autoreconf -i ./configure --prefix=$PWD/build --enable-lib-only make -j$(nproc) check diff --git a/README.rst b/README.rst index 679b9ded..2c9b5f0e 100644 --- a/README.rst +++ b/README.rst @@ -366,8 +366,9 @@ Build nghttp3: .. code-block:: text - $ git clone --depth 1 -b v0.1.1 https://github.com/ngtcp2/nghttp3 + $ git clone https://github.com/ngtcp2/nghttp3 $ cd nghttp3 + $ git checkout 207318c92e0578ac393c31dcd797e4a4dca3e31a $ autoreconf -i $ ./configure --prefix=$PWD/build --enable-lib-only $ make -j$(nproc) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 89ad3086..35718a81 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -1998,7 +1998,7 @@ int Http3Upstream::http_recv_request_header(Downstream *downstream, } namespace { -int http_end_request_headers(nghttp3_conn *conn, int64_t stream_id, +int http_end_request_headers(nghttp3_conn *conn, int64_t stream_id, int fin, void *user_data, void *stream_user_data) { auto upstream = static_cast(user_data); auto handler = upstream->get_client_handler(); @@ -2008,7 +2008,7 @@ int http_end_request_headers(nghttp3_conn *conn, int64_t stream_id, return 0; } - if (upstream->http_end_request_headers(downstream) != 0) { + if (upstream->http_end_request_headers(downstream, fin) != 0) { return NGHTTP3_ERR_CALLBACK_FAILURE; } @@ -2019,7 +2019,7 @@ int http_end_request_headers(nghttp3_conn *conn, int64_t stream_id, } } // namespace -int Http3Upstream::http_end_request_headers(Downstream *downstream) { +int Http3Upstream::http_end_request_headers(Downstream *downstream, int fin) { auto lgconf = log_config(); lgconf->update_tstamp(std::chrono::system_clock::now()); auto &req = downstream->request(); @@ -2115,8 +2115,11 @@ int Http3Upstream::http_end_request_headers(Downstream *downstream) { req.connect_proto = ConnectProto::WEBSOCKET; } - // We are not sure that request has body or not at the moment. - req.http2_expect_body = true; + if (!fin) { + req.http2_expect_body = true; + } else if (req.fs.content_length == -1) { + req.fs.content_length = 0; + } downstream->inspect_http2_request(); diff --git a/src/shrpx_http3_upstream.h b/src/shrpx_http3_upstream.h index 0e29356e..9e61bb74 100644 --- a/src/shrpx_http3_upstream.h +++ b/src/shrpx_http3_upstream.h @@ -125,7 +125,7 @@ public: int http_recv_request_header(Downstream *downstream, int32_t token, nghttp3_rcbuf *name, nghttp3_rcbuf *value, uint8_t flags); - int http_end_request_headers(Downstream *downstream); + int http_end_request_headers(Downstream *downstream, int fin); int http_end_stream(Downstream *downstream); void start_downstream(Downstream *downstream); void initiate_downstream(Downstream *downstream); From 42b659354d4490ca509848935b9bc23ddd06c7f9 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 23 Jan 2022 21:32:21 +0900 Subject: [PATCH 137/142] nghttpx: Support h3 trailer fields --- src/shrpx_http3_upstream.cc | 69 +++++++++++++++++++++++++++++++++---- src/shrpx_http3_upstream.h | 2 +- 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 35718a81..8e761d8b 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -1035,7 +1035,9 @@ int Http3Upstream::downstream_eof(DownstreamConnection *dconn) { // downstream_read_data_callback to send RST_STREAM after pending // response body is sent. This is needed to ensure that RST_STREAM // is sent after all pending data are sent. - on_downstream_body_complete(downstream); + if (on_downstream_body_complete(downstream) != 0) { + return -1; + } } else if (downstream->get_response_state() != DownstreamState::MSG_COMPLETE) { // If stream was not closed, then we set MSG_COMPLETE and let @@ -1079,7 +1081,9 @@ int Http3Upstream::downstream_error(DownstreamConnection *dconn, int events) { } else { if (downstream->get_response_state() == DownstreamState::HEADER_COMPLETE) { if (downstream->get_upgraded()) { - on_downstream_body_complete(downstream); + if (on_downstream_body_complete(downstream) != 0) { + return -1; + } } else { shutdown_stream(downstream, NGHTTP3_H3_INTERNAL_ERROR); } @@ -1366,6 +1370,24 @@ int Http3Upstream::on_downstream_body_complete(Downstream *downstream) { return 0; } + if (!downstream->get_upgraded()) { + const auto &trailers = resp.fs.trailers(); + if (!trailers.empty()) { + std::vector nva; + nva.reserve(trailers.size()); + http3::copy_headers_to_nva_nocopy(nva, trailers, http2::HDOP_STRIP_ALL); + if (!nva.empty()) { + auto rv = nghttp3_conn_submit_trailers( + httpconn_, downstream->get_stream_id(), nva.data(), nva.size()); + if (rv != 0) { + ULOG(FATAL, this) << "nghttp3_conn_submit_trailers() failed: " + << nghttp3_strerror(rv); + return -1; + } + } + } + } + nghttp3_conn_resume_stream(httpconn_, downstream->get_stream_id()); downstream->ensure_upstream_wtimer(); @@ -1943,8 +1965,29 @@ int http_recv_request_header(nghttp3_conn *conn, int64_t stream_id, return 0; } - if (upstream->http_recv_request_header(downstream, token, name, value, - flags) != 0) { + if (upstream->http_recv_request_header(downstream, token, name, value, flags, + /* trailer = */ false) != 0) { + return NGHTTP3_ERR_CALLBACK_FAILURE; + } + + return 0; +} +} // namespace + +namespace { +int http_recv_request_trailer(nghttp3_conn *conn, int64_t stream_id, + int32_t token, nghttp3_rcbuf *name, + nghttp3_rcbuf *value, uint8_t flags, + void *user_data, void *stream_user_data) { + auto upstream = static_cast(user_data); + auto downstream = static_cast(stream_user_data); + + if (!downstream || downstream->get_stop_reading()) { + return 0; + } + + if (upstream->http_recv_request_header(downstream, token, name, value, flags, + /* trailer = */ true) != 0) { return NGHTTP3_ERR_CALLBACK_FAILURE; } @@ -1955,8 +1998,8 @@ int http_recv_request_header(nghttp3_conn *conn, int64_t stream_id, int Http3Upstream::http_recv_request_header(Downstream *downstream, int32_t h3token, nghttp3_rcbuf *name, - nghttp3_rcbuf *value, - uint8_t flags) { + nghttp3_rcbuf *value, uint8_t flags, + bool trailer) { auto namebuf = nghttp3_rcbuf_get_buf(name); auto valuebuf = nghttp3_rcbuf_get_buf(value); auto &req = downstream->request(); @@ -1978,6 +2021,11 @@ int Http3Upstream::http_recv_request_header(Downstream *downstream, << ", num=" << req.fs.num_fields() + 1; } + // just ignore if this is a trailer part. + if (trailer) { + return 0; + } + if (error_reply(downstream, 431) != 0) { return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; } @@ -1991,6 +2039,13 @@ int Http3Upstream::http_recv_request_header(Downstream *downstream, downstream->add_rcbuf(name); downstream->add_rcbuf(value); + if (trailer) { + req.fs.add_trailer_token(StringRef{namebuf.base, namebuf.len}, + StringRef{valuebuf.base, valuebuf.len}, no_index, + token); + return 0; + } + req.fs.add_header_token(StringRef{namebuf.base, namebuf.len}, StringRef{valuebuf.base, valuebuf.len}, no_index, token); @@ -2429,7 +2484,7 @@ int Http3Upstream::setup_httpconn() { shrpx::http_recv_request_header, shrpx::http_end_request_headers, nullptr, // begin_trailers - nullptr, // recv_trailer + shrpx::http_recv_request_trailer, nullptr, // end_trailers shrpx::http_stop_sending, shrpx::http_end_stream, diff --git a/src/shrpx_http3_upstream.h b/src/shrpx_http3_upstream.h index 9e61bb74..e8c8dee4 100644 --- a/src/shrpx_http3_upstream.h +++ b/src/shrpx_http3_upstream.h @@ -124,7 +124,7 @@ public: void http_begin_request_headers(int64_t stream_id); int http_recv_request_header(Downstream *downstream, int32_t token, nghttp3_rcbuf *name, nghttp3_rcbuf *value, - uint8_t flags); + uint8_t flags, bool trailer); int http_end_request_headers(Downstream *downstream, int fin); int http_end_stream(Downstream *downstream); void start_downstream(Downstream *downstream); From 3b549caf90475b34717fc837a1bfbe2fde02a120 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 23 Jan 2022 21:33:28 +0900 Subject: [PATCH 138/142] clang-format --- src/shrpx_health_monitor_downstream_connection.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/shrpx_health_monitor_downstream_connection.cc b/src/shrpx_health_monitor_downstream_connection.cc index 6e33bc15..89e53963 100644 --- a/src/shrpx_health_monitor_downstream_connection.cc +++ b/src/shrpx_health_monitor_downstream_connection.cc @@ -98,7 +98,8 @@ int HealthMonitorDownstreamConnection::on_read() { return 0; } int HealthMonitorDownstreamConnection::on_write() { return 0; } -void HealthMonitorDownstreamConnection::on_upstream_change(Upstream *upstream) {} +void HealthMonitorDownstreamConnection::on_upstream_change(Upstream *upstream) { +} bool HealthMonitorDownstreamConnection::poolable() const { return false; } From 809d5af43e2eba903f8e7c0ceaa5892a6cbbb1e9 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 23 Jan 2022 21:38:32 +0900 Subject: [PATCH 139/142] nghttpx: Fix invalid error code --- src/shrpx_http3_upstream.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 8e761d8b..bc890df3 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -2027,7 +2027,7 @@ int Http3Upstream::http_recv_request_header(Downstream *downstream, } if (error_reply(downstream, 431) != 0) { - return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + return -1; } return 0; @@ -2115,7 +2115,7 @@ int Http3Upstream::http_end_request_headers(Downstream *downstream, int fin) { auto method_token = http2::lookup_method_token(method->value); if (method_token == -1) { if (error_reply(downstream, 501) != 0) { - return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + return -1; } return 0; } @@ -2163,7 +2163,7 @@ int Http3Upstream::http_end_request_headers(Downstream *downstream, int fin) { if (connect_proto) { if (connect_proto->value != "websocket") { if (error_reply(downstream, 400) != 0) { - return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + return -1; } return 0; } @@ -2188,7 +2188,7 @@ int Http3Upstream::http_end_request_headers(Downstream *downstream, int fin) { if (mruby_ctx->run_on_request_proc(downstream) != 0) { if (error_reply(downstream, 500) != 0) { - return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + return -1; } return 0; } From 75272a817e9a6da4cbd42d698d576cc4f02d4917 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 23 Jan 2022 23:46:19 +0900 Subject: [PATCH 140/142] h2load: Fix ngtcp2 callback error code --- src/h2load_quic.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/h2load_quic.cc b/src/h2load_quic.cc index a32b1d46..790b7207 100644 --- a/src/h2load_quic.cc +++ b/src/h2load_quic.cc @@ -120,7 +120,7 @@ int stream_close(ngtcp2_conn *conn, uint32_t flags, int64_t stream_id, } if (c->quic_stream_close(stream_id, app_error_code) != 0) { - return -1; + return NGTCP2_ERR_CALLBACK_FAILURE; } return 0; } @@ -140,7 +140,7 @@ int stream_reset(ngtcp2_conn *conn, int64_t stream_id, uint64_t final_size, void *stream_user_data) { auto c = static_cast(user_data); if (c->quic_stream_reset(stream_id, app_error_code) != 0) { - return -1; + return NGTCP2_ERR_CALLBACK_FAILURE; } return 0; } @@ -160,7 +160,7 @@ int stream_stop_sending(ngtcp2_conn *conn, int64_t stream_id, void *stream_user_data) { auto c = static_cast(user_data); if (c->quic_stream_stop_sending(stream_id, app_error_code) != 0) { - return -1; + return NGTCP2_ERR_CALLBACK_FAILURE; } return 0; } From 3122a8390035a6c8bcbea38be542df7291d697c2 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 23 Jan 2022 23:47:40 +0900 Subject: [PATCH 141/142] h2load: Set quic error code based on error from ngtcp2_conn_read_pkt --- src/h2load_quic.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/h2load_quic.cc b/src/h2load_quic.cc index 790b7207..8a8f3caf 100644 --- a/src/h2load_quic.cc +++ b/src/h2load_quic.cc @@ -667,6 +667,11 @@ int Client::read_quic() { timestamp(worker->loop)); if (rv != 0) { std::cerr << "ngtcp2_conn_read_pkt: " << ngtcp2_strerror(rv) << std::endl; + + if (!quic.last_error.code) { + quic.last_error = quic::err_transport(rv); + } + return -1; } From 32c2557bb74b8fc12973a60e9e532ff3dbfa57b7 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 26 Jan 2022 22:58:50 +0900 Subject: [PATCH 142/142] Bump nghttp3 --- .github/workflows/build.yml | 2 +- README.rst | 2 +- src/h2load_http3_session.cc | 2 +- src/shrpx_http3_upstream.cc | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cdc73ca8..f7502bff 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -116,7 +116,7 @@ jobs: run: | git clone https://github.com/ngtcp2/nghttp3 cd nghttp3 - git checkout 207318c92e0578ac393c31dcd797e4a4dca3e31a + git checkout 74a222fe0c89b7202bcdaf6ef27a232edffc85e3 autoreconf -i ./configure --prefix=$PWD/build --enable-lib-only make -j$(nproc) check diff --git a/README.rst b/README.rst index 2c9b5f0e..92f9de6a 100644 --- a/README.rst +++ b/README.rst @@ -368,7 +368,7 @@ Build nghttp3: $ git clone https://github.com/ngtcp2/nghttp3 $ cd nghttp3 - $ git checkout 207318c92e0578ac393c31dcd797e4a4dca3e31a + $ git checkout 74a222fe0c89b7202bcdaf6ef27a232edffc85e3 $ autoreconf -i $ ./configure --prefix=$PWD/build --enable-lib-only $ make -j$(nproc) diff --git a/src/h2load_http3_session.cc b/src/h2load_http3_session.cc index a62bb70c..573a8a14 100644 --- a/src/h2load_http3_session.cc +++ b/src/h2load_http3_session.cc @@ -305,7 +305,7 @@ int Http3Session::init_conn() { nghttp3_settings settings; nghttp3_settings_default(&settings); - settings.qpack_max_table_capacity = config->header_table_size; + settings.qpack_max_dtable_capacity = config->header_table_size; settings.qpack_blocked_streams = 100; auto mem = nghttp3_mem_default(); diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index bc890df3..1c5d18d2 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -2495,7 +2495,7 @@ int Http3Upstream::setup_httpconn() { nghttp3_settings settings; nghttp3_settings_default(&settings); - settings.qpack_max_table_capacity = 4_k; + settings.qpack_max_dtable_capacity = 4_k; if (!config->http2_proxy) { settings.enable_connect_protocol = 1;