From 01da060496a28b8b23643915a44bc9df934f85e4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 15 Aug 2021 12:22:10 +0900 Subject: [PATCH] nghttpx: Create quic server socket --- src/Makefile.am | 1 + src/shrpx_config.cc | 22 +++- src/shrpx_config.h | 4 + src/shrpx_connection_handler.cc | 6 + src/shrpx_quic.cc | 188 ++++++++++++++++++++++++++++++++ src/shrpx_quic.h | 38 +++++++ src/shrpx_worker.cc | 18 +++ src/shrpx_worker.h | 4 + 8 files changed, 275 insertions(+), 6 deletions(-) create mode 100644 src/shrpx_quic.cc create mode 100644 src/shrpx_quic.h diff --git a/src/Makefile.am b/src/Makefile.am index 08797fff..d3a0b4c5 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -153,6 +153,7 @@ NGHTTPX_SRCS = \ shrpx_dns_resolver.cc shrpx_dns_resolver.h \ shrpx_dual_dns_resolver.cc shrpx_dual_dns_resolver.h \ shrpx_dns_tracker.cc shrpx_dns_tracker.h \ + shrpx_quic.cc shrpx_quic.h \ buffer.h memchunk.h template.h allocator.h \ xsi_strerror.c xsi_strerror.h diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 038c5f21..d86c61de 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -785,6 +785,7 @@ struct UpstreamParams { bool tls; bool sni_fwd; bool proxyproto; + bool quic; }; namespace { @@ -819,6 +820,8 @@ int parse_upstream_params(UpstreamParams &out, const StringRef &src_params) { out.alt_mode = UpstreamAltMode::HEALTHMON; } else if (util::strieq_l("proxyproto", param)) { out.proxyproto = true; + } else if (util::strieq_l("quic", param)) { + out.quic = true; } else if (!param.empty()) { LOG(ERROR) << "frontend: " << param << ": unknown keyword"; return -1; @@ -2637,7 +2640,6 @@ int parse_config(Config *config, int optid, const StringRef &opt, return 0; } case SHRPX_OPTID_FRONTEND: { - auto &listenerconf = config->conn.listener; auto &apiconf = config->api; auto addr_end = std::find(std::begin(optarg), std::end(optarg), ';'); @@ -2655,6 +2657,11 @@ int parse_config(Config *config, int optid, const StringRef &opt, return -1; } + if (params.quic && params.alt_mode != UpstreamAltMode::NONE) { + LOG(ERROR) << "frontend: api or healthmon cannot be used with quic"; + return -1; + } + UpstreamAddr addr{}; addr.fd = -1; addr.tls = params.tls; @@ -2666,12 +2673,15 @@ int parse_config(Config *config, int optid, const StringRef &opt, apiconf.enabled = true; } + auto &addrs = params.quic ? config->conn.quic_listener.addrs + : config->conn.listener.addrs; + if (util::istarts_with(optarg, SHRPX_UNIX_PATH_PREFIX)) { auto path = std::begin(optarg) + SHRPX_UNIX_PATH_PREFIX.size(); addr.host = make_string_ref(config->balloc, StringRef{path, addr_end}); addr.host_unix = true; - listenerconf.addrs.push_back(std::move(addr)); + addrs.push_back(std::move(addr)); return 0; } @@ -2686,21 +2696,21 @@ int parse_config(Config *config, int optid, const StringRef &opt, if (util::numeric_host(host, AF_INET)) { addr.family = AF_INET; - listenerconf.addrs.push_back(std::move(addr)); + addrs.push_back(std::move(addr)); return 0; } if (util::numeric_host(host, AF_INET6)) { addr.family = AF_INET6; - listenerconf.addrs.push_back(std::move(addr)); + addrs.push_back(std::move(addr)); return 0; } addr.family = AF_INET; - listenerconf.addrs.push_back(addr); + addrs.push_back(addr); addr.family = AF_INET6; - listenerconf.addrs.push_back(std::move(addr)); + addrs.push_back(std::move(addr)); return 0; } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index e5a0fc6a..915c9da7 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -907,6 +907,10 @@ struct ConnectionConfig { int fastopen; } listener; + struct { + std::vector addrs; + } quic_listener; + struct { struct { ev_tstamp http2_read; diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 84221ba7..f732f74a 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -244,6 +244,9 @@ int ConnectionHandler::create_single_worker() { return -1; } #endif // HAVE_MRUBY + if (single_worker_->setup_quic_server_socket() != 0) { + return -1; + } return 0; } @@ -305,6 +308,9 @@ int ConnectionHandler::create_worker_thread(size_t num) { return -1; } # endif // HAVE_MRUBY + if (worker->setup_quic_server_socket() != 0) { + return -1; + } workers_.push_back(std::move(worker)); worker_loops_.push_back(loop); diff --git a/src/shrpx_quic.cc b/src/shrpx_quic.cc new file mode 100644 index 00000000..48b9b94d --- /dev/null +++ b/src/shrpx_quic.cc @@ -0,0 +1,188 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2021 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "shrpx_quic.h" + +#include +#include +#include + +#include + +#include "shrpx_config.h" +#include "shrpx_log.h" +#include "util.h" +#include "xsi_strerror.h" + +using namespace nghttp2; + +namespace shrpx { + +int create_quic_server_socket(UpstreamAddr &faddr) { + std::array errbuf; + int fd = -1; + int rv; + + auto service = util::utos(faddr.port); + addrinfo hints{}; + hints.ai_family = faddr.family; + hints.ai_socktype = SOCK_DGRAM; + hints.ai_flags = AI_PASSIVE; +#ifdef AI_ADDRCONFIG + hints.ai_flags |= AI_ADDRCONFIG; +#endif // AI_ADDRCONFIG + + auto node = + faddr.host == StringRef::from_lit("*") ? nullptr : faddr.host.c_str(); + + addrinfo *res, *rp; + rv = getaddrinfo(node, service.c_str(), &hints, &res); +#ifdef AI_ADDRCONFIG + if (rv != 0) { + // Retry without AI_ADDRCONFIG + hints.ai_flags &= ~AI_ADDRCONFIG; + rv = getaddrinfo(node, service.c_str(), &hints, &res); + } +#endif // AI_ADDRCONFIG + if (rv != 0) { + LOG(FATAL) << "Unable to get IPv" << (faddr.family == AF_INET ? "4" : "6") + << " address for " << faddr.host << ", port " << faddr.port + << ": " << gai_strerror(rv); + return -1; + } + + auto res_d = defer(freeaddrinfo, res); + + std::array host; + + for (rp = res; rp; rp = rp->ai_next) { + rv = getnameinfo(rp->ai_addr, rp->ai_addrlen, host.data(), host.size(), + nullptr, 0, NI_NUMERICHOST); + if (rv != 0) { + LOG(WARN) << "getnameinfo() failed: " << gai_strerror(rv); + continue; + } + +#ifdef SOCK_NONBLOCK + fd = socket(rp->ai_family, rp->ai_socktype | SOCK_NONBLOCK | SOCK_CLOEXEC, + rp->ai_protocol); + if (fd == -1) { + auto error = errno; + LOG(WARN) << "socket() syscall failed: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + continue; + } +#else // !SOCK_NONBLOCK + fd = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); + if (fd == -1) { + auto error = errno; + LOG(WARN) << "socket() syscall failed: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + continue; + } + util::make_socket_nonblocking(fd); + util::make_socket_closeonexec(fd); +#endif // !SOCK_NONBLOCK + + int val = 1; + if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val, + static_cast(sizeof(val))) == -1) { + auto error = errno; + LOG(WARN) << "Failed to set SO_REUSEADDR option to listener socket: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + close(fd); + continue; + } + + if (setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &val, + static_cast(sizeof(val))) == -1) { + auto error = errno; + LOG(WARN) << "Failed to set SO_REUSEPORT option to listener socket: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + close(fd); + continue; + } + + if (faddr.family == AF_INET6) { +#ifdef IPV6_V6ONLY + if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &val, + static_cast(sizeof(val))) == -1) { + auto error = errno; + LOG(WARN) << "Failed to set IPV6_V6ONLY option to listener socket: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + close(fd); + continue; + } +#endif // IPV6_V6ONLY + + if (setsockopt(fd, IPPROTO_IPV6, IPV6_RECVPKTINFO, &val, + static_cast(sizeof(val))) == -1) { + auto error = errno; + LOG(WARN) + << "Failed to set IPV6_RECVPKTINFO option to listener socket: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + close(fd); + continue; + } + } else { + if (setsockopt(fd, IPPROTO_IP, IP_PKTINFO, &val, + static_cast(sizeof(val))) == -1) { + auto error = errno; + LOG(WARN) << "Failed to set IP_PKTINFO option to listener socket: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + close(fd); + continue; + } + } + + // TODO Enable ECN + + if (bind(fd, rp->ai_addr, rp->ai_addrlen) == -1) { + auto error = errno; + LOG(WARN) << "bind() syscall failed: " + << xsi_strerror(error, errbuf.data(), errbuf.size()); + close(fd); + continue; + } + + break; + } + + if (!rp) { + LOG(FATAL) << "Listening " << (faddr.family == AF_INET ? "IPv4" : "IPv6") + << " socket failed"; + + return -1; + } + + faddr.fd = fd; + faddr.hostport = util::make_http_hostport(mod_config()->balloc, + StringRef{host.data()}, faddr.port); + + LOG(NOTICE) << "Listening on " << faddr.hostport << ", quic"; + + return 0; +} + +} // namespace shrpx diff --git a/src/shrpx_quic.h b/src/shrpx_quic.h new file mode 100644 index 00000000..59b273e2 --- /dev/null +++ b/src/shrpx_quic.h @@ -0,0 +1,38 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2021 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef SHRPX_QUIC_H +#define SHRPX_QUIC_H + +#include "shrpx.h" + +struct UpstreamAddr; + +namespace shrpx { + +int create_quic_server_socket(UpstreamAddr &addr); + +} // namespace shrpx + +#endif // SHRPX_QUIC_H diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index cf8854b2..08966eee 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -39,6 +39,7 @@ #ifdef HAVE_MRUBY # include "shrpx_mruby.h" #endif // HAVE_MRUBY +#include "shrpx_quic.h" #include "util.h" #include "template.h" @@ -137,6 +138,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, : randgen_(util::make_mt19937()), worker_stat_{}, dns_tracker_(loop), + quic_upstream_addrs_{get_config()->conn.quic_listener.addrs}, loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), @@ -580,6 +582,22 @@ ConnectionHandler *Worker::get_connection_handler() const { DNSTracker *Worker::get_dns_tracker() { return &dns_tracker_; } +int Worker::setup_quic_server_socket() { + for (auto &addr : quic_upstream_addrs_) { + if (addr.host_unix) { + /* TODO Not implemented */ + assert(0); + continue; + } + + if (create_quic_server_socket(addr) != 0) { + return -1; + } + } + + return 0; +} + namespace { size_t match_downstream_addr_group_host( const RouterConfig &routerconf, const StringRef &host, diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index e736aad8..292deb32 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -321,6 +321,8 @@ public: DNSTracker *get_dns_tracker(); + int setup_quic_server_socket(); + private: #ifndef NOTHREADS std::future fut_; @@ -335,6 +337,8 @@ private: WorkerStat worker_stat_; DNSTracker dns_tracker_; + std::vector quic_upstream_addrs_; + std::shared_ptr downstreamconf_; std::unique_ptr session_cache_memcached_dispatcher_; #ifdef HAVE_MRUBY