diff --git a/README.rst b/README.rst index d7bc328c..2a500b63 100644 --- a/README.rst +++ b/README.rst @@ -341,9 +341,9 @@ configure script with ``--enable-http3``. For nghttpx to reload configurations and swapping its executable while gracefully terminating old worker processes, eBPF is required. Run the configure script with ``--enable-http3 --with-libbpf`` to build -eBPF program. The Connection ID encryption key must be set with -``--frontend-quic-connection-id-encryption-key`` and must not change -in order to keep the existing connections alive during reload. +eBPF program. The QUIC keying material must be set with +``--frontend-quic-secret-file`` in order to keep the existing +connections alive during reload. The detailed steps to build HTTP/3 enabled h2load and nghttpx follow. diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index 774a3cd8..90ab1b89 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -534,10 +534,10 @@ nghttpx does not support HTTP/3 on backend connection. Hot swapping (SIGUSR2) or configuration reload (SIGHUP) require eBPF program. Without eBPF, old worker processes keep getting HTTP/3 -traffic and do not work as intended. Connection ID encryption key -must be set with -:option:`--frontend-quic-connection-id-encryption-key` and must not -change in order to keep the existing connections alive during reload. +traffic and do not work as intended. The QUIC keying material to +encrypt Connection ID must be set with +:option:`--frontend-quic-secret-file` and must provide the existing +keys in order to keep the existing connections alive during reload. In order announce that HTTP/3 endpoint is available, you should specify alt-svc header field. For example, the following options send diff --git a/gennghttpxfun.py b/gennghttpxfun.py index e9810b8f..2ee66ae8 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -192,8 +192,8 @@ OPTIONS = [ "frontend-quic-qlog-dir", "frontend-quic-require-token", "frontend-quic-congestion-controller", - "frontend-quic-connection-id-encryption-key", "frontend-quic-server-id", + "frontend-quic-secret-file", ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index e06d899b..8fc8893f 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1859,14 +1859,6 @@ void fill_default_config(Config *config) { upstreamconf.congestion_controller = NGTCP2_CC_ALGO_CUBIC; - // TODO Not really nice to generate random key here, but fine for - // now. - if (RAND_bytes(upstreamconf.cid_encryption_key.data(), - upstreamconf.cid_encryption_key.size()) != 1) { - assert(0); - abort(); - } - if (RAND_bytes(upstreamconf.server_id.data(), upstreamconf.server_id.size()) != 1) { assert(0); @@ -3253,14 +3245,31 @@ HTTP/3 and QUIC: ? "cubic" : "bbr") << R"( - --frontend-quic-connection-id-encryption-key= - Specify Connection ID encryption key. The encryption - key must be 16 bytes, and it must be encoded in hex - string (which is 32 bytes long). If this option is - omitted, new key is generated. In order to survive QUIC - connection in a configuration reload event, old and new - configuration must have this option and share the same - key. + --frontend-quic-secret-file= + Path to file that contains secure random data to be used + as QUIC keying materials. It is used to derive keys for + encrypting tokens and Connection IDs. It is not used to + encrypt QUIC packets. Each line of this file must + contain exactly 136 bytes hex-encoded string (when + decoded the byte string is 68 bytes long). The first 2 + bits of decoded byte string are used to identify the + keying material. An empty line or a line which starts + '#' is ignored. The file can contain more than one + keying materials. Because the identifier is 2 bits, at + most 4 keying materials are read and the remaining data + is discarded. The first keying material in the file is + primarily used for encryption and decryption for new + connection. The other ones are used to decrypt data for + the existing connections. Specifying multiple keying + materials enables key rotation. Please note that key + rotation does not occur automatically. User should + update files or change options values and restart + nghttpx gracefully. If opening or reading given file + fails, all loaded keying materials are discarded and it + is treated as if none of this option is given. If this + option is not given or an error occurred while opening + or reading a file, a keying material is generated + internally on startup and reload. --frontend-quic-server-id= Specify server ID encoded in Connection ID to identify this particular server instance. Connection ID is @@ -4067,10 +4076,10 @@ int main(int argc, char **argv) { 182}, {SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER.c_str(), required_argument, &flag, 183}, - {SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY.c_str(), - required_argument, &flag, 184}, {SHRPX_OPT_FRONTEND_QUIC_SERVER_ID.c_str(), required_argument, &flag, 185}, + {SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE.c_str(), required_argument, &flag, + 186}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -4948,17 +4957,16 @@ int main(int argc, char **argv) { cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER, StringRef{optarg}); break; - case 184: - // --frontend-quic-connection-id-encryption-key - cmdcfgs.emplace_back( - SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY, - StringRef{optarg}); - break; case 185: // --frontend-quic-server-id cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_SERVER_ID, StringRef{optarg}); break; + case 186: + // --frontend-quic-secret-file + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE, + StringRef{optarg}); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 74483d19..d498ef02 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -230,6 +230,69 @@ read_tls_ticket_key_file(const std::vector &files, return ticket_keys; } +#ifdef ENABLE_HTTP3 +std::shared_ptr +read_quic_secret_file(const StringRef &path) { + constexpr size_t expectedlen = + SHRPX_QUIC_SECRET_RESERVEDLEN + SHRPX_QUIC_SECRETLEN + SHRPX_QUIC_SALTLEN; + + auto qkms = std::make_shared(); + auto &kms = qkms->keying_materials; + + std::ifstream f(path.c_str()); + if (!f) { + LOG(ERROR) << "frontend-quic-secret-file: could not open file " << path; + return nullptr; + } + + std::array buf; + + while (f.getline(buf.data(), buf.size())) { + if (f.gcount() == 1 || buf[0] == '#') { + continue; + } + + auto s = StringRef{std::begin(buf), std::begin(buf) + f.gcount() - 1}; + if (s.size() != expectedlen * 2 || !util::is_hex_string(s)) { + LOG(ERROR) << "frontend-quic-secret-file: each line must be a " + << expectedlen * 2 << " bytes hex encoded string"; + return nullptr; + } + + kms.emplace_back(); + auto &qkm = kms.back(); + + auto p = std::begin(s); + + util::decode_hex(std::begin(qkm.reserved), + StringRef{p, p + qkm.reserved.size()}); + p += qkm.reserved.size() * 2; + util::decode_hex(std::begin(qkm.secret), + StringRef{p, p + qkm.secret.size()}); + p += qkm.secret.size() * 2; + util::decode_hex(std::begin(qkm.salt), StringRef{p, p + qkm.salt.size()}); + p += qkm.salt.size() * 2; + + assert(static_cast(p - std::begin(s)) == expectedlen * 2); + + qkm.id = qkm.reserved[0] & 0xc0; + + if (kms.size() == 4) { + break; + } + } + + if (f.bad()) { + LOG(ERROR) + << "frontend-quic-secret-file: error occurred while reading file " + << path; + return nullptr; + } + + return qkms; +} +#endif // ENABLE_HTTP3 + FILE *open_file_for_write(const char *filename) { std::array errbuf; @@ -2344,6 +2407,9 @@ int option_lookup_token(const char *name, size_t namelen) { if (util::strieq_l("backend-http2-window-siz", name, 24)) { return SHRPX_OPTID_BACKEND_HTTP2_WINDOW_SIZE; } + if (util::strieq_l("frontend-quic-secret-fil", name, 24)) { + return SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE; + } break; case 'g': if (util::strieq_l("http2-no-cookie-crumblin", name, 24)) { @@ -2689,10 +2755,6 @@ int option_lookup_token(const char *name, size_t namelen) { case 42: switch (name[41]) { case 'y': - if (util::strieq_l("frontend-quic-connection-id-encryption-ke", name, - 41)) { - return SHRPX_OPTID_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY; - } if (util::strieq_l("tls-session-cache-memcached-address-famil", name, 41)) { return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY; @@ -4029,18 +4091,6 @@ int parse_config(Config *config, int optid, const StringRef &opt, } #endif // ENABLE_HTTP3 - return 0; - case SHRPX_OPTID_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY: -#ifdef ENABLE_HTTP3 - if (optarg.size() != config->quic.upstream.cid_encryption_key.size() * 2 || - !util::is_hex_string(optarg)) { - LOG(ERROR) << opt << ": must be a hex-string"; - return -1; - } - util::decode_hex(std::begin(config->quic.upstream.cid_encryption_key), - optarg); -#endif // ENABLE_HTTP3 - return 0; case SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID: #ifdef ENABLE_HTTP3 @@ -4052,6 +4102,12 @@ int parse_config(Config *config, int optid, const StringRef &opt, util::decode_hex(std::begin(config->quic.upstream.server_id), optarg); #endif // ENABLE_HTTP3 + return 0; + case SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE: +#ifdef ENABLE_HTTP3 + config->quic.upstream.secret_file = make_string_ref(config->balloc, optarg); +#endif // ENABLE_HTTP3 + return 0; case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index fef79a60..0c276ddd 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -391,10 +391,10 @@ constexpr auto SHRPX_OPT_FRONTEND_QUIC_REQUIRE_TOKEN = StringRef::from_lit("frontend-quic-require-token"); constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER = StringRef::from_lit("frontend-quic-congestion-controller"); -constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY = - StringRef::from_lit("frontend-quic-connection-id-encryption-key"); constexpr auto SHRPX_OPT_FRONTEND_QUIC_SERVER_ID = StringRef::from_lit("frontend-quic-server-id"); +constexpr auto SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE = + StringRef::from_lit("frontend-quic-secret-file"); constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -606,10 +606,18 @@ struct TLSCertificate { }; #ifdef ENABLE_HTTP3 -struct QUICSecret { - std::array - stateless_reset_secret; - std::array token_secret; +struct QUICKeyingMaterial { + std::array reserved; + std::array secret; + std::array salt; + std::array cid_encryption_key; + // Identifier of this keying material. Only the first 2 bits are + // used. + uint8_t id; +}; + +struct QUICKeyingMaterials { + std::vector keying_materials; }; #endif // ENABLE_HTTP3 @@ -765,8 +773,8 @@ struct QUICConfig { ngtcp2_cc_algo congestion_controller; bool early_data; bool require_token; - std::array cid_encryption_key; std::array server_id; + StringRef secret_file; } upstream; struct { StringRef prog_file; @@ -1220,12 +1228,12 @@ enum { SHRPX_OPTID_FRONTEND_MAX_REQUESTS, SHRPX_OPTID_FRONTEND_NO_TLS, SHRPX_OPTID_FRONTEND_QUIC_CONGESTION_CONTROLLER, - SHRPX_OPTID_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY, SHRPX_OPTID_FRONTEND_QUIC_DEBUG_LOG, SHRPX_OPTID_FRONTEND_QUIC_EARLY_DATA, SHRPX_OPTID_FRONTEND_QUIC_IDLE_TIMEOUT, SHRPX_OPTID_FRONTEND_QUIC_QLOG_DIR, SHRPX_OPTID_FRONTEND_QUIC_REQUIRE_TOKEN, + SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE, SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID, SHRPX_OPTID_FRONTEND_READ_TIMEOUT, SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT, @@ -1377,6 +1385,11 @@ std::unique_ptr read_tls_ticket_key_file(const std::vector &files, const EVP_CIPHER *cipher, const EVP_MD *hmac); +#ifdef ENABLE_HTTP3 +std::shared_ptr +read_quic_secret_file(const StringRef &path); +#endif // ENABLE_HTTP3 + // Returns string representation of |proto|. StringRef strproto(Proto proto); diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 87a1efb4..4d32b6c5 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -298,8 +298,6 @@ int ConnectionHandler::create_single_worker() { #endif // HAVE_MRUBY #ifdef ENABLE_HTTP3 - single_worker_->set_quic_secret(quic_secret_); - if (single_worker_->setup_quic_server_socket() != 0) { return -1; } @@ -404,8 +402,6 @@ int ConnectionHandler::create_worker_thread(size_t num) { # endif // HAVE_MRUBY # ifdef ENABLE_HTTP3 - worker->set_quic_secret(quic_secret_); - if ((!apiconf.enabled || i != 0) && worker->setup_quic_server_socket() != 0) { return -1; @@ -1047,25 +1043,14 @@ int ConnectionHandler::forward_quic_packet(const UpstreamAddr *faddr, return -1; } -int ConnectionHandler::create_quic_secret() { - auto quic_secret = std::make_shared(); +void ConnectionHandler::set_quic_keying_materials( + std::shared_ptr qkms) { + quic_keying_materials_ = std::move(qkms); +} - if (generate_quic_stateless_reset_secret( - quic_secret->stateless_reset_secret.data()) != 0) { - LOG(ERROR) << "Failed to generate QUIC Stateless Reset secret"; - - return -1; - } - - if (generate_quic_token_secret(quic_secret->token_secret.data()) != 0) { - LOG(ERROR) << "Failed to generate QUIC token secret"; - - return -1; - } - - quic_secret_ = std::move(quic_secret); - - return 0; +const std::shared_ptr & +ConnectionHandler::get_quic_keying_materials() const { + return quic_keying_materials_; } void ConnectionHandler::set_cid_prefixes( @@ -1287,14 +1272,13 @@ int ConnectionHandler::quic_ipc_read() { return 0; } - auto config = get_config(); - auto &quicconf = config->quic; + auto &qkm = quic_keying_materials_->keying_materials.front(); std::array decrypted_dcid; - if (decrypt_quic_connection_id( - decrypted_dcid.data(), dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, - quicconf.upstream.cid_encryption_key.data()) != 0) { + if (decrypt_quic_connection_id(decrypted_dcid.data(), + dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, + qkm.cid_encryption_key.data()) != 0) { return -1; } diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h index 2c8d9c9a..af391750 100644 --- a/src/shrpx_connection_handler.h +++ b/src/shrpx_connection_handler.h @@ -198,7 +198,8 @@ public: const Address &local_addr, const uint8_t *cid_prefix, const uint8_t *data, size_t datalen); - int create_quic_secret(); + void set_quic_keying_materials(std::shared_ptr qkms); + const std::shared_ptr &get_quic_keying_materials() const; void set_cid_prefixes( const std::vector> @@ -263,7 +264,7 @@ private: # ifdef HAVE_LIBBPF std::vector quic_bpf_refs_; # endif // HAVE_LIBBPF - std::shared_ptr quic_secret_; + std::shared_ptr quic_keying_materials_; std::vector quic_all_ssl_ctx_; std::vector> quic_indexed_ssl_ctx_; #endif // ENABLE_HTTP3 diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 4010e098..c67d8aae 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -40,6 +40,7 @@ #include "shrpx_quic.h" #include "shrpx_worker.h" #include "shrpx_http.h" +#include "shrpx_connection_handler.h" #ifdef HAVE_MRUBY # include "shrpx_mruby.h" #endif // HAVE_MRUBY @@ -217,21 +218,17 @@ int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token, auto upstream = static_cast(user_data); auto handler = upstream->get_client_handler(); auto worker = handler->get_worker(); + auto conn_handler = worker->get_connection_handler(); + auto &qkms = conn_handler->get_quic_keying_materials(); + auto &qkm = qkms->keying_materials.front(); - auto config = get_config(); - auto &quicconf = config->quic; - - if (generate_quic_connection_id( - *cid, cidlen, worker->get_cid_prefix(), - quicconf.upstream.cid_encryption_key.data()) != 0) { + if (generate_quic_connection_id(*cid, cidlen, worker->get_cid_prefix(), + qkm.id, qkm.cid_encryption_key.data()) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } - auto &quic_secret = worker->get_quic_secret(); - auto &secret = quic_secret->stateless_reset_secret; - - if (generate_quic_stateless_reset_token(token, *cid, secret.data(), - secret.size()) != 0) { + if (generate_quic_stateless_reset_token(token, *cid, qkm.secret.data(), + qkm.secret.size()) != 0) { return NGTCP2_ERR_CALLBACK_FAILURE; } @@ -487,11 +484,13 @@ int Http3Upstream::handshake_completed() { auto path = ngtcp2_conn_get_path(conn_); auto worker = handler_->get_worker(); - auto &quic_secret = worker->get_quic_secret(); - auto &secret = quic_secret->token_secret; + auto conn_handler = worker->get_connection_handler(); + auto &qkms = conn_handler->get_quic_keying_materials(); + auto &qkm = qkms->keying_materials.front(); if (generate_token(token.data(), tokenlen, path->remote.addr, - path->remote.addrlen, secret.data()) != 0) { + path->remote.addrlen, qkm.secret.data(), + qkm.secret.size()) != 0) { return 0; } @@ -513,6 +512,7 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, int rv; auto worker = handler_->get_worker(); + auto conn_handler = worker->get_connection_handler(); auto callbacks = ngtcp2_callbacks{ nullptr, // client_initial @@ -557,11 +557,14 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, auto &quicconf = config->quic; auto &http3conf = config->http3; + auto &qkms = conn_handler->get_quic_keying_materials(); + auto &qkm = qkms->keying_materials.front(); + ngtcp2_cid scid; - if (generate_quic_connection_id( - scid, SHRPX_QUIC_SCIDLEN, worker->get_cid_prefix(), - quicconf.upstream.cid_encryption_key.data()) != 0) { + if (generate_quic_connection_id(scid, SHRPX_QUIC_SCIDLEN, + worker->get_cid_prefix(), qkm.id, + qkm.cid_encryption_key.data()) != 0) { return -1; } @@ -608,12 +611,8 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr, params.original_dcid = initial_hd.dcid; } - auto &quic_secret = worker->get_quic_secret(); - auto &stateless_reset_secret = quic_secret->stateless_reset_secret; - - rv = generate_quic_stateless_reset_token(params.stateless_reset_token, scid, - stateless_reset_secret.data(), - stateless_reset_secret.size()); + rv = generate_quic_stateless_reset_token( + params.stateless_reset_token, scid, qkm.secret.data(), qkm.secret.size()); if (rv != 0) { ULOG(ERROR, this) << "generate_quic_stateless_reset_token failed"; return -1; diff --git a/src/shrpx_quic.cc b/src/shrpx_quic.cc index 529cde57..a0bf56c3 100644 --- a/src/shrpx_quic.cc +++ b/src/shrpx_quic.cc @@ -144,7 +144,7 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, } int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, - const uint8_t *server_id, + const uint8_t *server_id, uint8_t km_id, const uint8_t *key) { assert(cidlen == SHRPX_QUIC_SCIDLEN); @@ -154,6 +154,8 @@ int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, cid.datalen = cidlen; + cid.data[0] = (cid.data[0] & 0x3f) | km_id; + auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET; std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, p); @@ -162,7 +164,8 @@ int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, } int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, - const uint8_t *cid_prefix, const uint8_t *key) { + const uint8_t *cid_prefix, uint8_t km_id, + const uint8_t *key) { assert(cidlen == SHRPX_QUIC_SCIDLEN); if (RAND_bytes(cid.data, cidlen) != 1) { @@ -171,6 +174,8 @@ int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, cid.datalen = cidlen; + cid.data[0] = (cid.data[0] & 0x3f) | km_id; + auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET; std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, p); @@ -257,32 +262,16 @@ int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid &cid, return 0; } -int generate_quic_stateless_reset_secret(uint8_t *secret) { - if (RAND_bytes(secret, SHRPX_QUIC_STATELESS_RESET_SECRETLEN) != 1) { - return -1; - } - - return 0; -} - -int generate_quic_token_secret(uint8_t *secret) { - if (RAND_bytes(secret, SHRPX_QUIC_TOKEN_SECRETLEN) != 1) { - return -1; - } - - return 0; -} - int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, socklen_t salen, const ngtcp2_cid &retry_scid, - const ngtcp2_cid &odcid, const uint8_t *token_secret) { + const ngtcp2_cid &odcid, const uint8_t *secret, + size_t secretlen) { auto t = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); auto stokenlen = ngtcp2_crypto_generate_retry_token( - token, token_secret, SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, &retry_scid, - &odcid, t); + token, secret, secretlen, sa, salen, &retry_scid, &odcid, t); if (stokenlen < 0) { return -1; } @@ -294,15 +283,16 @@ int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen, const ngtcp2_cid &dcid, const sockaddr *sa, - socklen_t salen, const uint8_t *token_secret) { + socklen_t salen, const uint8_t *secret, + size_t secretlen) { auto t = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); - if (ngtcp2_crypto_verify_retry_token(&odcid, token, tokenlen, token_secret, - SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, - &dcid, 10 * NGTCP2_SECONDS, t) != 0) { + if (ngtcp2_crypto_verify_retry_token(&odcid, token, tokenlen, secret, + secretlen, sa, salen, &dcid, + 10 * NGTCP2_SECONDS, t) != 0) { return -1; } @@ -310,13 +300,13 @@ int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen, } int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, - size_t salen, const uint8_t *token_secret) { + size_t salen, const uint8_t *secret, size_t secretlen) { auto t = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); auto stokenlen = ngtcp2_crypto_generate_regular_token( - token, token_secret, SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, t); + token, secret, secretlen, sa, salen, t); if (stokenlen < 0) { return -1; } @@ -327,18 +317,48 @@ int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, } int verify_token(const uint8_t *token, size_t tokenlen, const sockaddr *sa, - socklen_t salen, const uint8_t *token_secret) { + socklen_t salen, const uint8_t *secret, size_t secretlen) { auto t = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); - if (ngtcp2_crypto_verify_regular_token(token, tokenlen, token_secret, - SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, - 3600 * NGTCP2_SECONDS, t) != 0) { + if (ngtcp2_crypto_verify_regular_token(token, tokenlen, secret, secretlen, sa, + salen, 3600 * NGTCP2_SECONDS, + t) != 0) { return -1; } return 0; } +int generate_quic_connection_id_encryption_key(uint8_t *key, size_t keylen, + const uint8_t *secret, + size_t secretlen, + const uint8_t *salt, + size_t saltlen) { + constexpr uint8_t info[] = "connection id encryption key"; + ngtcp2_crypto_md sha256; + ngtcp2_crypto_md_init( + &sha256, reinterpret_cast(const_cast(EVP_sha256()))); + + if (ngtcp2_crypto_hkdf(key, keylen, &sha256, secret, secretlen, salt, saltlen, + info, str_size(info)) != 0) { + return -1; + } + + return 0; +} + +const QUICKeyingMaterial * +select_quic_keying_material(const QUICKeyingMaterials &qkms, + const uint8_t *cid) { + for (auto &qkm : qkms.keying_materials) { + if (((*cid) & 0xc0) == qkm.id) { + return &qkm; + } + } + + return &qkms.keying_materials.front(); +} + } // namespace shrpx diff --git a/src/shrpx_quic.h b/src/shrpx_quic.h index 52a4d67a..e66018c4 100644 --- a/src/shrpx_quic.h +++ b/src/shrpx_quic.h @@ -60,6 +60,8 @@ bool operator==(const ngtcp2_cid &lhs, const ngtcp2_cid &rhs); namespace shrpx { struct UpstreamAddr; +struct QUICKeyingMaterials; +struct QUICKeyingMaterial; constexpr size_t SHRPX_QUIC_SCIDLEN = 20; constexpr size_t SHRPX_QUIC_SERVER_IDLEN = 2; @@ -69,10 +71,11 @@ constexpr size_t SHRPX_QUIC_CID_PREFIX_OFFSET = 1; constexpr size_t SHRPX_QUIC_DECRYPTED_DCIDLEN = 16; constexpr size_t SHRPX_QUIC_CID_ENCRYPTION_KEYLEN = 16; constexpr size_t SHRPX_QUIC_MAX_UDP_PAYLOAD_SIZE = 1472; -constexpr size_t SHRPX_QUIC_STATELESS_RESET_SECRETLEN = 32; -constexpr size_t SHRPX_QUIC_TOKEN_SECRETLEN = 32; constexpr size_t SHRPX_QUIC_CONN_CLOSE_PKTLEN = 256; constexpr size_t SHRPX_QUIC_STATELESS_RESET_BURST = 100; +constexpr size_t SHRPX_QUIC_SECRET_RESERVEDLEN = 4; +constexpr size_t SHRPX_QUIC_SECRETLEN = 32; +constexpr size_t SHRPX_QUIC_SALTLEN = 32; ngtcp2_tstamp quic_timestamp(); @@ -82,11 +85,12 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, size_t gso_size); int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, - const uint8_t *server_id, + const uint8_t *server_id, uint8_t km_id, const uint8_t *key); int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, - const uint8_t *cid_prefix, const uint8_t *key); + const uint8_t *cid_prefix, uint8_t km_id, + const uint8_t *key); int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, const uint8_t *key); @@ -103,23 +107,31 @@ int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid &cid, const uint8_t *secret, size_t secretlen); -int generate_quic_stateless_reset_secret(uint8_t *secret); - -int generate_quic_token_secret(uint8_t *secret); - int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, socklen_t salen, const ngtcp2_cid &retry_scid, - const ngtcp2_cid &odcid, const uint8_t *token_secret); + const ngtcp2_cid &odcid, const uint8_t *secret, + size_t secretlen); int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen, const ngtcp2_cid &dcid, const sockaddr *sa, - socklen_t salen, const uint8_t *token_secret); + socklen_t salen, const uint8_t *secret, + size_t secretlen); int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa, - size_t salen, const uint8_t *token_secret); + size_t salen, const uint8_t *secret, size_t secretlen); int verify_token(const uint8_t *token, size_t tokenlen, const sockaddr *sa, - socklen_t salen, const uint8_t *token_secret); + socklen_t salen, const uint8_t *secret, size_t secretlen); + +int generate_quic_connection_id_encryption_key(uint8_t *key, size_t keylen, + const uint8_t *secret, + size_t secretlen, + const uint8_t *salt, + size_t saltlen); + +const QUICKeyingMaterial * +select_quic_keying_material(const QUICKeyingMaterials &qkms, + const uint8_t *cid); } // namespace shrpx diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 00f5876e..37116645 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -126,14 +126,20 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, if (it == std::end(connections_)) { std::array decrypted_dcid; + auto &qkms = conn_handler->get_quic_keying_materials(); + const QUICKeyingMaterial *qkm = nullptr; + if (dcidlen == SHRPX_QUIC_SCIDLEN) { - if (decrypt_quic_connection_id( - decrypted_dcid.data(), dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, - quicconf.upstream.cid_encryption_key.data()) != 0) { + qkm = select_quic_keying_material(*qkms.get(), dcid); + + if (decrypt_quic_connection_id(decrypted_dcid.data(), + dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, + qkm->cid_encryption_key.data()) != 0) { return 0; } - if (!std::equal(std::begin(decrypted_dcid), + if (qkm != &qkms->keying_materials.front() || + !std::equal(std::begin(decrypted_dcid), std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, worker_->get_cid_prefix())) { auto quic_lwp = @@ -170,14 +176,25 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, switch (ngtcp2_accept(&hd, data, datalen)) { case 0: { - // If we get Initial and it has the CID prefix of this worker, it - // is likely that client is intentionally use the our prefix. + // If we get Initial and it has the CID prefix of this worker, + // it is likely that client is intentionally use the prefix. // Just drop it. - if (dcidlen == SHRPX_QUIC_SCIDLEN && - std::equal(std::begin(decrypted_dcid), - std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, - worker_->get_cid_prefix())) { - return 0; + if (dcidlen == SHRPX_QUIC_SCIDLEN) { + if (qkm != &qkms->keying_materials.front()) { + qkm = &qkms->keying_materials.front(); + + if (decrypt_quic_connection_id(decrypted_dcid.data(), + dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, + qkm->cid_encryption_key.data()) != 0) { + return 0; + } + } + + if (std::equal(std::begin(decrypted_dcid), + std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN, + worker_->get_cid_prefix())) { + return 0; + } } if (worker_->get_graceful_shutdown()) { @@ -197,14 +214,18 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, break; } - auto &quic_secret = worker_->get_quic_secret(); - auto &secret = quic_secret->token_secret; + if (dcidlen != SHRPX_QUIC_SCIDLEN) { + // Initial packets with token must have DCID chosen by server. + return 0; + } + + auto qkm = select_quic_keying_material(*qkms.get(), dcid); switch (hd.token.base[0]) { case NGTCP2_CRYPTO_TOKEN_MAGIC_RETRY: if (verify_retry_token(odcid, hd.token.base, hd.token.len, hd.dcid, &remote_addr.su.sa, remote_addr.len, - secret.data()) != 0) { + qkm->secret.data(), qkm->secret.size()) != 0) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Failed to validate Retry token from remote=" << util::to_numeric_addr(&remote_addr); @@ -229,7 +250,8 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, break; case NGTCP2_CRYPTO_TOKEN_MAGIC_REGULAR: if (verify_token(hd.token.base, hd.token.len, &remote_addr.su.sa, - remote_addr.len, secret.data()) != 0) { + remote_addr.len, qkm->secret.data(), + qkm->secret.size()) != 0) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Failed to validate token from remote=" << util::to_numeric_addr(&remote_addr); @@ -422,11 +444,15 @@ int QUICConnectionHandler::send_retry( auto config = get_config(); auto &quicconf = config->quic; + auto conn_handler = worker_->get_connection_handler(); + auto &qkms = conn_handler->get_quic_keying_materials(); + auto &qkm = qkms->keying_materials.front(); + ngtcp2_cid retry_scid; if (generate_quic_retry_connection_id( retry_scid, SHRPX_QUIC_SCIDLEN, quicconf.upstream.server_id.data(), - quicconf.upstream.cid_encryption_key.data()) != 0) { + qkm.id, qkm.cid_encryption_key.data()) != 0) { return -1; } @@ -437,12 +463,9 @@ int QUICConnectionHandler::send_retry( ngtcp2_cid_init(&idcid, ini_dcid, ini_dcidlen); ngtcp2_cid_init(&iscid, ini_scid, ini_scidlen); - auto &quic_secret = worker_->get_quic_secret(); - auto &secret = quic_secret->token_secret; - if (generate_retry_token(token.data(), tokenlen, &remote_addr.su.sa, remote_addr.len, retry_scid, idcid, - secret.data()) != 0) { + qkm.secret.data(), qkm.secret.size()) != 0) { return -1; } @@ -539,11 +562,12 @@ int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr, ngtcp2_cid_init(&cid, dcid, dcidlen); - auto &quic_secret = worker_->get_quic_secret(); - auto &secret = quic_secret->stateless_reset_secret; + auto conn_handler = worker_->get_connection_handler(); + auto &qkms = conn_handler->get_quic_keying_materials(); + auto &qkm = qkms->keying_materials.front(); - rv = generate_quic_stateless_reset_token(token.data(), cid, secret.data(), - secret.size()); + rv = generate_quic_stateless_reset_token(token.data(), cid, qkm.secret.data(), + qkm.secret.size()); if (rv != 0) { return -1; } diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 720fd8d2..b287ca50 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -933,14 +933,14 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { return -1; } - auto &quicconf = config->quic; - constexpr uint32_t key_high_idx = 1; constexpr uint32_t key_low_idx = 2; + auto &qkms = conn_handler_->get_quic_keying_materials(); + auto &qkm = qkms->keying_materials.front(); + if (bpf_map_update_elem(bpf_map__fd(sk_info), &key_high_idx, - quicconf.upstream.cid_encryption_key.data(), - BPF_ANY) != 0) { + qkm.cid_encryption_key.data(), BPF_ANY) != 0) { LOG(FATAL) << "Failed to update key_high_idx sk_info: " << xsi_strerror(errno, errbuf.data(), errbuf.size()); close(fd); @@ -948,7 +948,7 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { } if (bpf_map_update_elem(bpf_map__fd(sk_info), &key_low_idx, - quicconf.upstream.cid_encryption_key.data() + 8, + qkm.cid_encryption_key.data() + 8, BPF_ANY) != 0) { LOG(FATAL) << "Failed to update key_low_idx sk_info: " << xsi_strerror(errno, errbuf.data(), errbuf.size()); @@ -1010,14 +1010,6 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) { const uint8_t *Worker::get_cid_prefix() const { return cid_prefix_.data(); } -void Worker::set_quic_secret(const std::shared_ptr &secret) { - quic_secret_ = secret; -} - -const std::shared_ptr &Worker::get_quic_secret() const { - return quic_secret_; -} - const UpstreamAddr *Worker::find_quic_upstream_addr(const Address &local_addr) { std::array host; diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index cd53173f..8cf40e97 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -370,10 +370,6 @@ public: const uint8_t *get_cid_prefix() const; - void set_quic_secret(const std::shared_ptr &secret); - - const std::shared_ptr &get_quic_secret() const; - # ifdef HAVE_LIBBPF bool should_attach_bpf() const; @@ -412,7 +408,6 @@ private: std::array cid_prefix_; std::vector quic_upstream_addrs_; std::vector> quic_listeners_; - std::shared_ptr quic_secret_; #endif // ENABLE_HTTP3 std::shared_ptr downstreamconf_; diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc index 7bcdecd1..006b8c7d 100644 --- a/src/shrpx_worker_process.cc +++ b/src/shrpx_worker_process.cc @@ -519,10 +519,51 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { } #ifdef ENABLE_HTTP3 - if (conn_handler->create_quic_secret() != 0) { - return -1; + auto &quicconf = config->quic; + + std::shared_ptr qkms; + + if (!quicconf.upstream.secret_file.empty()) { + qkms = read_quic_secret_file(quicconf.upstream.secret_file); + if (!qkms) { + LOG(WARN) << "Use QUIC keying materials generated internally"; + } } + if (!qkms) { + qkms = std::make_shared(); + qkms->keying_materials.resize(1); + + auto &qkm = qkms->keying_materials.front(); + + if (RAND_bytes(qkm.reserved.data(), qkm.reserved.size()) != 1) { + LOG(ERROR) << "Failed to generate QUIC secret reserved data"; + return -1; + } + + if (RAND_bytes(qkm.secret.data(), qkm.secret.size()) != 1) { + LOG(ERROR) << "Failed to generate QUIC secret"; + return -1; + } + + if (RAND_bytes(qkm.salt.data(), qkm.salt.size()) != 1) { + LOG(ERROR) << "Failed to generate QUIC salt"; + return -1; + } + } + + for (auto &qkm : qkms->keying_materials) { + if (generate_quic_connection_id_encryption_key( + qkm.cid_encryption_key.data(), qkm.cid_encryption_key.size(), + qkm.secret.data(), qkm.secret.size(), qkm.salt.data(), + qkm.salt.size()) != 0) { + LOG(ERROR) << "Failed to generate QUIC Connection ID encryption key"; + return -1; + } + } + + conn_handler->set_quic_keying_materials(std::move(qkms)); + conn_handler->set_cid_prefixes(wpconf->cid_prefixes); conn_handler->set_quic_lingering_worker_processes( wpconf->quic_lingering_worker_processes);