From 940fdd557350a7fea72520242e1128f14fec74ee Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 16 Aug 2021 15:11:18 +0900 Subject: [PATCH] nghttpx: Read quic packet --- src/shrpx_client_handler.cc | 37 +++- src/shrpx_client_handler.h | 6 + src/shrpx_config.cc | 3 + src/shrpx_config.h | 10 ++ src/shrpx_connection.cc | 11 +- src/shrpx_connection_handler.cc | 22 +-- src/shrpx_http3_upstream.cc | 248 ++++++++++++++++++++++++++- src/shrpx_http3_upstream.h | 9 + src/shrpx_quic.cc | 59 +++++++ src/shrpx_quic.h | 33 ++++ src/shrpx_quic_connection_handler.cc | 79 ++++++++- src/shrpx_quic_connection_handler.h | 10 +- src/shrpx_quic_listener.cc | 11 +- src/shrpx_worker.cc | 16 +- src/shrpx_worker.h | 12 +- 15 files changed, 531 insertions(+), 35 deletions(-) diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 9bbec8b0..f7185f84 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -51,6 +51,7 @@ #include "shrpx_api_downstream_connection.h" #include "shrpx_health_monitor_downstream_connection.h" #include "shrpx_null_downstream_connection.h" +#include "shrpx_http3_upstream.h" #include "shrpx_log.h" #include "util.h" #include "template.h" @@ -286,6 +287,15 @@ int ClientHandler::write_tls() { } } +int ClientHandler::read_quic(const UpstreamAddr *faddr, + const Address &remote_addr, + const Address &local_addr, const uint8_t *data, + size_t datalen) { + auto upstream = static_cast(upstream_.get()); + + return upstream->on_read(faddr, remote_addr, local_addr, data, datalen); +} + int ClientHandler::upstream_noop() { return 0; } int ClientHandler::upstream_read() { @@ -402,7 +412,8 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl, get_config()->conn.upstream.ratelimit.write, get_config()->conn.upstream.ratelimit.read, writecb, readcb, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold, - get_config()->tls.dyn_rec.idle_timeout, Proto::NONE), + get_config()->tls.dyn_rec.idle_timeout, + faddr->quic ? Proto::HTTP3 : Proto::NONE), ipaddr_(make_string_ref(balloc_, ipaddr)), port_(make_string_ref(balloc_, port)), faddr_(faddr), @@ -423,14 +434,16 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl, auto config = get_config(); - if (faddr_->accept_proxy_protocol || - config->conn.upstream.accept_proxy_protocol) { - read_ = &ClientHandler::read_clear; - write_ = &ClientHandler::noop; - on_read_ = &ClientHandler::proxy_protocol_read; - on_write_ = &ClientHandler::upstream_noop; - } else { - setup_upstream_io_callback(); + if (!faddr->quic) { + if (faddr_->accept_proxy_protocol || + config->conn.upstream.accept_proxy_protocol) { + read_ = &ClientHandler::read_clear; + write_ = &ClientHandler::noop; + on_read_ = &ClientHandler::proxy_protocol_read; + on_write_ = &ClientHandler::upstream_noop; + } else { + setup_upstream_io_callback(); + } } auto &fwdconf = config->http.forwarded; @@ -492,6 +505,12 @@ void ClientHandler::setup_upstream_io_callback() { } } +void ClientHandler::setup_http3_upstream( + std::unique_ptr &&upstream) { + upstream_ = std::move(upstream); + alpn_ = StringRef::from_lit("h3"); +} + ClientHandler::~ClientHandler() { if (LOG_ENABLED(INFO)) { CLOG(INFO, this) << "Deleting"; diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index bc56d482..dbdff961 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -53,6 +53,7 @@ class Downstream; struct WorkerStat; struct DownstreamAddrGroup; struct DownstreamAddr; +class Http3Upstream; class ClientHandler { public: @@ -70,6 +71,9 @@ public: int read_tls(); int write_tls(); + int read_quic(const UpstreamAddr *faddr, const Address &remote_addr, + const Address &local_addr, const uint8_t *data, size_t datalen); + int upstream_noop(); int upstream_read(); int upstream_http2_connhd_read(); @@ -143,6 +147,8 @@ public: void setup_upstream_io_callback(); + void setup_http3_upstream(std::unique_ptr &&upstream); + // Returns string suitable for use in "by" parameter of Forwarded // header field. StringRef get_forwarded_by() const; diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index d86c61de..8742ad6a 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2668,6 +2668,7 @@ int parse_config(Config *config, int optid, const StringRef &opt, addr.sni_fwd = params.sni_fwd; addr.alt_mode = params.alt_mode; addr.accept_proxy_protocol = params.proxyproto; + addr.quic = params.quic; if (addr.alt_mode == UpstreamAltMode::API) { apiconf.enabled = true; @@ -3990,6 +3991,8 @@ StringRef strproto(Proto proto) { return StringRef::from_lit("http/1.1"); case Proto::HTTP2: return StringRef::from_lit("h2"); + case Proto::HTTP3: + return StringRef::from_lit("h3"); case Proto::MEMCACHED: return StringRef::from_lit("memcached"); } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 915c9da7..6c426685 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -370,6 +370,7 @@ enum class Proto { NONE, HTTP1, HTTP2, + HTTP3, MEMCACHED, }; @@ -458,6 +459,7 @@ struct UpstreamAddr { bool sni_fwd; // true if client is supposed to send PROXY protocol v1 header. bool accept_proxy_protocol; + bool quic; int fd; }; @@ -702,6 +704,12 @@ struct TLSConfig { bool no_postpone_early_data; }; +struct QUICConfig { + struct { + std::array secret; + } stateless_reset; +}; + // custom error page struct ErrorPage { // not NULL-terminated @@ -954,6 +962,7 @@ struct Config { http{}, http2{}, tls{}, + quic{}, logging{}, conn{}, api{}, @@ -987,6 +996,7 @@ struct Config { HttpConfig http; Http2Config http2; TLSConfig tls; + QUICConfig quic; LoggingConfig logging; ConnectionConfig conn; APIConfig api; diff --git a/src/shrpx_connection.cc b/src/shrpx_connection.cc index ec94b07c..daec6296 100644 --- a/src/shrpx_connection.cc +++ b/src/shrpx_connection.cc @@ -312,10 +312,13 @@ BIO_METHOD *create_bio_method() { void Connection::set_ssl(SSL *ssl) { tls.ssl = ssl; - auto &tlsconf = get_config()->tls; - auto bio = BIO_new(tlsconf.bio_method); - BIO_set_data(bio, this); - SSL_set_bio(tls.ssl, bio, bio); + if (proto != Proto::HTTP3) { + auto &tlsconf = get_config()->tls; + auto bio = BIO_new(tlsconf.bio_method); + BIO_set_data(bio, this); + SSL_set_bio(tls.ssl, bio, bio); + } + SSL_set_app_data(tls.ssl, this); } diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index db2ace7d..ddeb51ec 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -222,11 +222,11 @@ int ConnectionHandler::create_single_worker() { ); quic_cert_tree_ = tls::create_cert_lookup_tree(); - tls::setup_quic_server_ssl_context(quic_all_ssl_ctx_, quic_indexed_ssl_ctx_, - quic_cert_tree_.get() + auto quic_sv_ssl_ctx = tls::setup_quic_server_ssl_context( + quic_all_ssl_ctx_, quic_indexed_ssl_ctx_, quic_cert_tree_.get() #ifdef HAVE_NEVERBLEED - , - nb_ + , + nb_ #endif // HAVE_NEVERBLEED ); @@ -261,7 +261,8 @@ int ConnectionHandler::create_single_worker() { single_worker_ = std::make_unique( loop_, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(), - ticket_keys_, this, config->conn.downstream); + quic_sv_ssl_ctx, quic_cert_tree_.get(), ticket_keys_, this, + config->conn.downstream); #ifdef HAVE_MRUBY if (single_worker_->create_mruby_context() != 0) { return -1; @@ -288,11 +289,11 @@ int ConnectionHandler::create_worker_thread(size_t num) { ); quic_cert_tree_ = tls::create_cert_lookup_tree(); - tls::setup_quic_server_ssl_context(quic_all_ssl_ctx_, quic_indexed_ssl_ctx_, - quic_cert_tree_.get() + auto quic_sv_ssl_ctx = tls::setup_quic_server_ssl_context( + quic_all_ssl_ctx_, quic_indexed_ssl_ctx_, quic_cert_tree_.get() # ifdef HAVE_NEVERBLEED - , - nb_ + , + nb_ # endif // HAVE_NEVERBLEED ); @@ -337,7 +338,8 @@ int ConnectionHandler::create_worker_thread(size_t num) { auto worker = std::make_unique( loop, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(), - ticket_keys_, this, config->conn.downstream); + quic_sv_ssl_ctx, quic_cert_tree_.get(), ticket_keys_, this, + config->conn.downstream); # ifdef HAVE_MRUBY if (worker->create_mruby_context() != 0) { return -1; diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index b0da58c8..34e7e0cb 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -23,17 +23,201 @@ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "shrpx_http3_upstream.h" + +#include + +#include + #include "shrpx_client_handler.h" #include "shrpx_downstream.h" #include "shrpx_downstream_connection.h" #include "shrpx_log.h" +#include "shrpx_quic.h" +#include "shrpx_worker.h" +#include "util.h" namespace shrpx { Http3Upstream::Http3Upstream(ClientHandler *handler) - : handler_{handler}, tls_alert_{0} {} + : handler_{handler}, + conn_{nullptr}, + last_error_{QUICErrorType::Transport, 0}, + tls_alert_{0} {} -Http3Upstream::~Http3Upstream() {} +Http3Upstream::~Http3Upstream() { + if (conn_) { + auto worker = handler_->get_worker(); + auto quic_client_handler = worker->get_quic_connection_handler(); + + quic_client_handler->remove_connection_id(&initial_client_dcid_); + + std::vector scids(ngtcp2_conn_get_num_scid(conn_)); + ngtcp2_conn_get_scid(conn_, scids.data()); + + for (auto &cid : scids) { + quic_client_handler->remove_connection_id(&cid); + } + + ngtcp2_conn_del(conn_); + } +} + +namespace { +void log_printf(void *user_data, const char *fmt, ...) { + va_list ap; + std::array buf; + + va_start(ap, fmt); + auto nwrite = vsnprintf(buf.data(), buf.size(), fmt, ap); + va_end(ap); + + if (nwrite >= buf.size()) { + nwrite = buf.size() - 1; + } + + buf[nwrite++] = '\n'; + + write(fileno(stderr), buf.data(), nwrite); +} +} // namespace + +namespace { +void rand(uint8_t *dest, size_t destlen, const ngtcp2_rand_ctx *rand_ctx) { + util::random_bytes(dest, dest + destlen, + *static_cast(rand_ctx->native_handle)); +} +} // namespace + +namespace { +int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token, + size_t cidlen, void *user_data) { + if (generate_quic_connection_id(cid, cidlen) != 0) { + return NGTCP2_ERR_CALLBACK_FAILURE; + } + + auto config = get_config(); + auto &quicconf = config->quic; + auto &secret = quicconf.stateless_reset.secret; + + if (generate_quic_stateless_reset_token(token, cid, secret.data(), + secret.size()) != 0) { + return NGTCP2_ERR_CALLBACK_FAILURE; + } + + return 0; +} +} // namespace + +namespace { +int remove_connection_id(ngtcp2_conn *conn, const ngtcp2_cid *cid, + void *user_data) { + auto upstream = static_cast(user_data); + auto handler = upstream->get_client_handler(); + auto worker = handler->get_worker(); + auto quic_conn_handler = worker->get_quic_connection_handler(); + + quic_conn_handler->remove_connection_id(cid); + + return 0; +} +} // namespace + +int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, + const Address &local_addr, + const ngtcp2_pkt_hd &initial_hd) { + int rv; + + auto worker = handler_->get_worker(); + + auto callbacks = ngtcp2_callbacks{ + nullptr, // client_initial + ngtcp2_crypto_recv_client_initial_cb, + ngtcp2_crypto_recv_crypto_data_cb, + nullptr, // handshake_completed + nullptr, // recv_version_negotiation + ngtcp2_crypto_encrypt_cb, + ngtcp2_crypto_decrypt_cb, + ngtcp2_crypto_hp_mask_cb, + nullptr, // recv_stream_data + nullptr, // acked_crypto_offset + nullptr, // acked_stream_data_offset + nullptr, // stream_open + nullptr, // stream_close + nullptr, // recv_stateless_reset + nullptr, // recv_retry + nullptr, // extend_max_local_streams_bidi + nullptr, // extend_max_local_streams_uni + rand, + get_new_connection_id, + remove_connection_id, + ngtcp2_crypto_update_key_cb, + nullptr, // path_validation + nullptr, // select_preferred_addr + nullptr, // stream_reset + nullptr, // extend_max_remote_streams_bidi + nullptr, // extend_max_remote_streams_uni + nullptr, // extend_max_stream_data + nullptr, // dcid_status + nullptr, // handshake_confirmed + nullptr, // recv_new_token + ngtcp2_crypto_delete_crypto_aead_ctx_cb, + ngtcp2_crypto_delete_crypto_cipher_ctx_cb, + nullptr, // recv_datagram + nullptr, // ack_datagram + nullptr, // lost_datagram + ngtcp2_crypto_get_path_challenge_data_cb, + nullptr, // stream_stop_sending + }; + + initial_client_dcid_ = initial_hd.dcid; + + ngtcp2_cid scid; + + if (generate_quic_connection_id(&scid, SHRPX_QUIC_SCIDLEN) != 0) { + return -1; + } + + ngtcp2_settings settings; + ngtcp2_settings_default(&settings); + settings.log_printf = log_printf; + settings.initial_ts = quic_timestamp(); + settings.cc_algo = NGTCP2_CC_ALGO_BBR; + settings.max_window = 6_m; + settings.max_stream_window = 6_m; + settings.max_udp_payload_size = SHRPX_MAX_UDP_PAYLOAD_SIZE; + settings.rand_ctx = {&worker->get_randgen()}; + + ngtcp2_transport_params params; + ngtcp2_transport_params_default(¶ms); + params.initial_max_data = 1_m; + params.initial_max_stream_data_bidi_remote = 256_k; + params.initial_max_stream_data_uni = 256_k; + params.max_idle_timeout = 30 * NGTCP2_SECONDS; + params.original_dcid = initial_hd.dcid; + + auto path = ngtcp2_path{ + {local_addr.len, const_cast(&local_addr.su.sa)}, + {remote_addr.len, const_cast(&remote_addr.su.sa)}, + const_cast(faddr), + }; + + rv = ngtcp2_conn_server_new(&conn_, &initial_hd.scid, &scid, &path, + initial_hd.version, &callbacks, &settings, + ¶ms, nullptr, this); + if (rv != 0) { + LOG(ERROR) << "ngtcp2_conn_server_new: " << ngtcp2_strerror(rv); + return -1; + } + + ngtcp2_conn_set_tls_native_handle(conn_, handler_->get_ssl()); + + auto quic_connection_handler = worker->get_quic_connection_handler(); + + quic_connection_handler->add_connection_id(&initial_client_dcid_, handler_); + quic_connection_handler->add_connection_id(&scid, handler_); + + return 0; +} int Http3Upstream::on_read() { return 0; } @@ -127,16 +311,76 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr, const Address &remote_addr, const Address &local_addr, const uint8_t *data, size_t datalen) { + int rv; + ngtcp2_pkt_info pi{}; + + auto path = ngtcp2_path{ + { + local_addr.len, + const_cast(&local_addr.su.sa), + }, + { + remote_addr.len, + const_cast(&remote_addr.su.sa), + }, + const_cast(faddr), + }; + + rv = ngtcp2_conn_read_pkt(conn_, &path, &pi, data, datalen, quic_timestamp()); + if (rv != 0) { + LOG(ERROR) << "ngtcp2_conn_read_pkt: " << ngtcp2_strerror(rv); + + switch (rv) { + case NGTCP2_ERR_DRAINING: + // TODO Start drain period + return -1; + case NGTCP2_ERR_RETRY: + // TODO Send Retry packet + return -1; + case NGTCP2_ERR_REQUIRED_TRANSPORT_PARAM: + case NGTCP2_ERR_MALFORMED_TRANSPORT_PARAM: + case NGTCP2_ERR_TRANSPORT_PARAM: + // If rv indicates transport_parameters related error, we should + // send TRANSPORT_PARAMETER_ERROR even if last_error_.code is + // already set. This is because OpenSSL might set Alert. + last_error_ = quic_err_transport(rv); + break; + case NGTCP2_ERR_DROP_CONN: + return -1; + default: + if (!last_error_.code) { + last_error_ = quic_err_transport(rv); + } + } + + // TODO Send connection close + return handle_error(); + } + return 0; } +int Http3Upstream::handle_error() { return -1; } + int Http3Upstream::on_rx_secret(ngtcp2_crypto_level level, const uint8_t *secret, size_t secretlen) { + if (ngtcp2_crypto_derive_and_install_rx_key(conn_, nullptr, nullptr, nullptr, + level, secret, secretlen) != 0) { + LOG(ERROR) << "ngtcp2_crypto_derive_and_install_rx_key failed"; + return -1; + } + return 0; } int Http3Upstream::on_tx_secret(ngtcp2_crypto_level level, const uint8_t *secret, size_t secretlen) { + if (ngtcp2_crypto_derive_and_install_tx_key(conn_, nullptr, nullptr, nullptr, + level, secret, secretlen) != 0) { + LOG(ERROR) << "ngtcp2_crypto_derive_and_install_tx_key failed"; + return -1; + } + return 0; } diff --git a/src/shrpx_http3_upstream.h b/src/shrpx_http3_upstream.h index d979fff7..486f680a 100644 --- a/src/shrpx_http3_upstream.h +++ b/src/shrpx_http3_upstream.h @@ -30,6 +30,7 @@ #include #include "shrpx_upstream.h" +#include "shrpx_quic.h" #include "network.h" using namespace nghttp2; @@ -84,6 +85,9 @@ public: virtual bool push_enabled() const; virtual void cancel_premature_downstream(Downstream *promised_downstream); + int init(const UpstreamAddr *faddr, const Address &remote_addr, + const Address &local_addr, const ngtcp2_pkt_hd &initial_hd); + int on_read(const UpstreamAddr *faddr, const Address &remote_addr, const Address &local_addr, const uint8_t *data, size_t datalen); @@ -97,8 +101,13 @@ public: void set_tls_alert(uint8_t alert); + int handle_error(); + private: ClientHandler *handler_; + ngtcp2_cid initial_client_dcid_; + ngtcp2_conn *conn_; + QUICError last_error_; uint8_t tls_alert_; }; diff --git a/src/shrpx_quic.cc b/src/shrpx_quic.cc index 175ffaeb..708886c0 100644 --- a/src/shrpx_quic.cc +++ b/src/shrpx_quic.cc @@ -29,6 +29,13 @@ #include #include +#include + +#include + +#include + +#include #include "shrpx_config.h" #include "shrpx_log.h" @@ -39,6 +46,34 @@ using namespace nghttp2; namespace shrpx { +QUICError quic_err_transport(int liberr) { + if (liberr == NGTCP2_ERR_RECV_VERSION_NEGOTIATION) { + return {QUICErrorType::TransportVersionNegotiation, 0}; + } + return {QUICErrorType::Transport, + ngtcp2_err_infer_quic_transport_error_code(liberr)}; +} + +QUICError quic_err_idle_timeout() { + return {QUICErrorType::TransportIdleTimeout, 0}; +} + +QUICError quic_err_tls(int alert) { + return {QUICErrorType::Transport, + static_cast(NGTCP2_CRYPTO_ERROR | alert)}; +} + +QUICError quic_err_app(int liberr) { + return {QUICErrorType::Application, + nghttp3_err_infer_quic_app_error_code(liberr)}; +} + +ngtcp2_tstamp quic_timestamp() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} + int create_quic_server_socket(UpstreamAddr &faddr) { std::array errbuf; int fd = -1; @@ -192,4 +227,28 @@ int quic_send_packet(const UpstreamAddr *addr, const sockaddr *remote_sa, return 0; } +int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen) { + if (RAND_bytes(cid->data, cidlen) != 1) { + return -1; + } + + cid->datalen = cidlen; + + return 0; +} + +int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid, + const uint8_t *secret, + size_t secretlen) { + ngtcp2_crypto_md md; + ngtcp2_crypto_md_init(&md, const_cast(EVP_sha256())); + + if (ngtcp2_crypto_generate_stateless_reset_token(token, &md, secret, + secretlen, cid) != 0) { + return -1; + } + + return 0; +} + } // namespace shrpx diff --git a/src/shrpx_quic.h b/src/shrpx_quic.h index a575fee4..8350adc7 100644 --- a/src/shrpx_quic.h +++ b/src/shrpx_quic.h @@ -29,11 +29,38 @@ #include +#include + namespace shrpx { struct UpstreamAddr; constexpr size_t SHRPX_QUIC_SCIDLEN = 20; +constexpr size_t SHRPX_MAX_UDP_PAYLOAD_SIZE = 1280; + +enum class QUICErrorType { + Application, + Transport, + TransportVersionNegotiation, + TransportIdleTimeout, +}; + +struct QUICError { + QUICError(QUICErrorType type, uint64_t code) : type{type}, code{code} {} + + QUICErrorType type; + uint64_t code; +}; + +QUICError quic_err_transport(int liberr); + +QUICError quic_err_idle_timeout(); + +QUICError quic_err_tls(int alert); + +QUICError quic_err_app(int liberr); + +ngtcp2_tstamp quic_timestamp(); int create_quic_server_socket(UpstreamAddr &addr); @@ -42,6 +69,12 @@ int quic_send_packet(const UpstreamAddr *addr, const sockaddr *remote_sa, size_t local_salen, const uint8_t *data, size_t datalen, size_t gso_size); +int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen); + +int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid, + const uint8_t *secret, + size_t secretlen); + } // namespace shrpx #endif // SHRPX_QUIC_H diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 913da6b2..7ff08767 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -45,6 +45,12 @@ std::string make_cid_key(const uint8_t *dcid, size_t dcidlen) { } } // namespace +namespace { +std::string make_cid_key(const ngtcp2_cid *cid) { + return make_cid_key(cid->data, cid->datalen); +} +} // namespace + int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, const Address &remote_addr, const Address &local_addr, @@ -67,6 +73,8 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, auto dcid_key = make_cid_key(dcid, dcidlen); + ClientHandler *handler; + auto it = connections_.find(dcid_key); if (it == std::end(connections_)) { // new connection @@ -87,18 +95,66 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, return 0; } - return 0; + handler = handle_new_connection(faddr, remote_addr, local_addr, hd); + if (handler == nullptr) { + return 0; + } + } else { + handler = (*it).second; } - auto h = (*it).second.get(); - auto upstream = static_cast(h->get_upstream()); - - // TODO Delete h on failure - upstream->on_read(faddr, remote_addr, local_addr, data, datalen); + if (handler->read_quic(faddr, remote_addr, local_addr, data, datalen) != 0) { + delete handler; + } return 0; } +ClientHandler *QUICConnectionHandler::handle_new_connection( + const UpstreamAddr *faddr, const Address &remote_addr, + const Address &local_addr, const ngtcp2_pkt_hd &hd) { + std::array host; + std::array service; + int rv; + + rv = getnameinfo(&remote_addr.su.sa, remote_addr.len, host.data(), + host.size(), service.data(), service.size(), + NI_NUMERICHOST | NI_NUMERICSERV); + if (rv != 0) { + LOG(ERROR) << "getnameinfo() failed: " << gai_strerror(rv); + + return nullptr; + } + + auto ssl_ctx = worker_->get_quic_sv_ssl_ctx(); + + assert(ssl_ctx); + + auto ssl = tls::create_ssl(ssl_ctx); + if (ssl == nullptr) { + return nullptr; + } + + // Disable TLS session ticket if we don't have working ticket + // keys. + if (!worker_->get_ticket_keys()) { + SSL_set_options(ssl, SSL_OP_NO_TICKET); + } + + auto handler = std::make_unique( + worker_, faddr->fd, ssl, StringRef{host.data()}, + StringRef{service.data()}, remote_addr.su.sa.sa_family, faddr); + + auto upstream = std::make_unique(handler.get()); + if (upstream->init(faddr, remote_addr, local_addr, hd) != 0) { + return nullptr; + } + + handler->setup_http3_upstream(std::move(upstream)); + + return handler.release(); +} + namespace { uint32_t generate_reserved_version(const Address &addr, uint32_t version) { uint32_t h = 0x811C9DC5u; @@ -154,4 +210,15 @@ int QUICConnectionHandler::send_version_negotiation( 0); } +void QUICConnectionHandler::add_connection_id(const ngtcp2_cid *cid, + ClientHandler *handler) { + auto key = make_cid_key(cid); + connections_.emplace(key, handler); +} + +void QUICConnectionHandler::remove_connection_id(const ngtcp2_cid *cid) { + auto key = make_cid_key(cid); + connections_.erase(key); +} + } // namespace shrpx diff --git a/src/shrpx_quic_connection_handler.h b/src/shrpx_quic_connection_handler.h index b580d128..38b2f217 100644 --- a/src/shrpx_quic_connection_handler.h +++ b/src/shrpx_quic_connection_handler.h @@ -31,6 +31,8 @@ #include #include +#include + #include "network.h" using namespace nghttp2; @@ -53,10 +55,16 @@ public: const uint8_t *scid, size_t scidlen, const Address &remote_addr, const Address &local_addr); + ClientHandler *handle_new_connection(const UpstreamAddr *faddr, + const Address &remote_addr, + const Address &local_addr, + const ngtcp2_pkt_hd &hd); + void add_connection_id(const ngtcp2_cid *cid, ClientHandler *handler); + void remove_connection_id(const ngtcp2_cid *cid); private: Worker *worker_; - std::unordered_map> connections_; + std::unordered_map connections_; }; } // namespace shrpx diff --git a/src/shrpx_quic_listener.cc b/src/shrpx_quic_listener.cc index 33712a81..b06ec557 100644 --- a/src/shrpx_quic_listener.cc +++ b/src/shrpx_quic_listener.cc @@ -37,7 +37,7 @@ void readcb(struct ev_loop *loop, ev_io *w, int revent) { } // namespace QUICListener::QUICListener(const UpstreamAddr *faddr, Worker *worker) - : faddr_(faddr), worker_(worker) { + : faddr_{faddr}, worker_{worker} { ev_io_init(&rev_, readcb, faddr_->fd, EV_READ); rev_.data = this; ev_io_start(worker_->get_loop(), &rev_); @@ -62,6 +62,8 @@ void QUICListener::on_read() { uint8_t msg_ctrl[CMSG_SPACE(sizeof(in6_pktinfo))]; msg.msg_control = msg_ctrl; + auto quic_conn_handler = worker_->get_quic_connection_handler(); + for (; pktcnt < 10;) { msg.msg_namelen = sizeof(su); msg.msg_controllen = sizeof(msg_ctrl); @@ -91,6 +93,13 @@ void QUICListener::on_read() { if (nread == 0) { continue; } + + Address remote_addr; + remote_addr.su = su; + remote_addr.len = msg.msg_namelen; + + quic_conn_handler->handle_packet(faddr_, remote_addr, local_addr, + buf.data(), nread); } } diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 87235ee9..0a4ed3c7 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -132,7 +132,8 @@ create_downstream_key(const std::shared_ptr &shared_addr, Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, SSL_CTX *tls_session_cache_memcached_ssl_ctx, - tls::CertLookupTree *cert_tree, + tls::CertLookupTree *cert_tree, SSL_CTX *quic_sv_ssl_ctx, + tls::CertLookupTree *quic_cert_tree, const std::shared_ptr &ticket_keys, ConnectionHandler *conn_handler, std::shared_ptr downstreamconf) @@ -145,6 +146,9 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, cl_ssl_ctx_(cl_ssl_ctx), cert_tree_(cert_tree), conn_handler_(conn_handler), + quic_sv_ssl_ctx_{quic_sv_ssl_ctx}, + quic_cert_tree_{quic_cert_tree}, + quic_conn_handler_{this}, ticket_keys_(ticket_keys), connect_blocker_( std::make_unique(randgen_, loop_, nullptr, nullptr)), @@ -507,6 +511,10 @@ void Worker::process_events() { tls::CertLookupTree *Worker::get_cert_lookup_tree() const { return cert_tree_; } +tls::CertLookupTree *Worker::get_quic_cert_lookup_tree() const { + return quic_cert_tree_; +} + std::shared_ptr Worker::get_ticket_keys() { #ifdef HAVE_ATOMIC_STD_SHARED_PTR return std::atomic_load_explicit(&ticket_keys_, std::memory_order_acquire); @@ -537,6 +545,8 @@ SSL_CTX *Worker::get_sv_ssl_ctx() const { return sv_ssl_ctx_; } SSL_CTX *Worker::get_cl_ssl_ctx() const { return cl_ssl_ctx_; } +SSL_CTX *Worker::get_quic_sv_ssl_ctx() const { return quic_sv_ssl_ctx_; } + void Worker::set_graceful_shutdown(bool f) { graceful_shutdown_ = f; } bool Worker::get_graceful_shutdown() const { return graceful_shutdown_; } @@ -581,6 +591,10 @@ ConnectionHandler *Worker::get_connection_handler() const { return conn_handler_; } +QUICConnectionHandler *Worker::get_quic_connection_handler() { + return &quic_conn_handler_; +} + DNSTracker *Worker::get_dns_tracker() { return &dns_tracker_; } int Worker::setup_quic_server_socket() { diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index f495f8e7..02ece027 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -50,6 +50,7 @@ #include "shrpx_live_check.h" #include "shrpx_connect_blocker.h" #include "shrpx_dns_tracker.h" +#include "shrpx_quic_connection_handler.h" #include "allocator.h" using namespace nghttp2; @@ -270,7 +271,8 @@ class Worker { public: Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, SSL_CTX *tls_session_cache_memcached_ssl_ctx, - tls::CertLookupTree *cert_tree, + tls::CertLookupTree *cert_tree, SSL_CTX *quic_sv_ssl_ctx, + tls::CertLookupTree *quic_cert_tree, const std::shared_ptr &ticket_keys, ConnectionHandler *conn_handler, std::shared_ptr downstreamconf); @@ -281,6 +283,7 @@ public: void send(const WorkerEvent &event); tls::CertLookupTree *get_cert_lookup_tree() const; + tls::CertLookupTree *get_quic_cert_lookup_tree() const; // These 2 functions make a lock m_ to get/set ticket keys // atomically. @@ -291,6 +294,7 @@ public: struct ev_loop *get_loop() const; SSL_CTX *get_sv_ssl_ctx() const; SSL_CTX *get_cl_ssl_ctx() const; + SSL_CTX *get_quic_sv_ssl_ctx() const; void set_graceful_shutdown(bool f); bool get_graceful_shutdown() const; @@ -320,6 +324,8 @@ public: ConnectionHandler *get_connection_handler() const; + QUICConnectionHandler *get_quic_connection_handler(); + DNSTracker *get_dns_tracker(); int setup_quic_server_socket(); @@ -354,6 +360,10 @@ private: SSL_CTX *cl_ssl_ctx_; tls::CertLookupTree *cert_tree_; ConnectionHandler *conn_handler_; + SSL_CTX *quic_sv_ssl_ctx_; + tls::CertLookupTree *quic_cert_tree_; + + QUICConnectionHandler quic_conn_handler_; #ifndef HAVE_ATOMIC_STD_SHARED_PTR std::mutex ticket_keys_m_;