diff --git a/bpf/reuseport_kern.c b/bpf/reuseport_kern.c index 51cbdc18..7a48afa9 100644 --- a/bpf/reuseport_kern.c +++ b/bpf/reuseport_kern.c @@ -455,6 +455,7 @@ typedef struct quic_hd { #define MAX_DCIDLEN 20 #define MIN_DCIDLEN 8 #define CID_PREFIXLEN 8 +#define CID_PREFIX_OFFSET 1 enum { NGTCP2_PKT_INITIAL = 0x0, @@ -579,6 +580,7 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) { __u8 qpktbuf[6 + MAX_DCIDLEN]; struct AES_ctx aes_ctx; __u8 key[AES_KEYLEN]; + __u8 *cid_prefix; if (bpf_skb_load_bytes(reuse_md, sizeof(struct udphdr), qpktbuf, sizeof(qpktbuf)) != 0) { @@ -615,9 +617,10 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) { case NGTCP2_PKT_INITIAL: case NGTCP2_PKT_0RTT: if (qhd.dcidlen == SV_DCIDLEN) { - AES_ECB_decrypt(&aes_ctx, qhd.dcid); + cid_prefix = qhd.dcid + CID_PREFIX_OFFSET; + AES_ECB_decrypt(&aes_ctx, cid_prefix); - psk_index = bpf_map_lookup_elem(&cid_prefix_map, qhd.dcid); + psk_index = bpf_map_lookup_elem(&cid_prefix_map, cid_prefix); if (psk_index != NULL) { sk_index = *psk_index; @@ -634,9 +637,10 @@ int select_reuseport(struct sk_reuseport_md *reuse_md) { return SK_DROP; } - AES_ECB_decrypt(&aes_ctx, qhd.dcid); + cid_prefix = qhd.dcid + CID_PREFIX_OFFSET; + AES_ECB_decrypt(&aes_ctx, cid_prefix); - psk_index = bpf_map_lookup_elem(&cid_prefix_map, qhd.dcid); + psk_index = bpf_map_lookup_elem(&cid_prefix_map, cid_prefix); if (psk_index == NULL) { sk_index = sk_index_from_dcid(&qhd, reuse_md, *pnum_socks); diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 17eefd7b..e9810b8f 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -193,6 +193,7 @@ OPTIONS = [ "frontend-quic-require-token", "frontend-quic-congestion-controller", "frontend-quic-connection-id-encryption-key", + "frontend-quic-server-id", ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index 5189ee7b..e06d899b 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1342,6 +1342,7 @@ int generate_cid_prefix( std::vector> &cid_prefixes, const Config *config) { auto &apiconf = config->api; + auto &quicconf = config->quic; size_t num_cid_prefix; if (config->single_thread) { @@ -1360,7 +1361,8 @@ int generate_cid_prefix( cid_prefixes.resize(num_cid_prefix); for (auto &cid_prefix : cid_prefixes) { - if (create_cid_prefix(cid_prefix.data()) != 0) { + if (create_cid_prefix(cid_prefix.data(), + quicconf.upstream.server_id.data()) != 0) { return -1; } } @@ -1864,6 +1866,12 @@ void fill_default_config(Config *config) { assert(0); abort(); } + + if (RAND_bytes(upstreamconf.server_id.data(), + upstreamconf.server_id.size()) != 1) { + assert(0); + abort(); + } } auto &http3conf = config->http3; @@ -3253,6 +3261,14 @@ HTTP/3 and QUIC: connection in a configuration reload event, old and new configuration must have this option and share the same key. + --frontend-quic-server-id= + Specify server ID encoded in Connection ID to identify + this particular server instance. Connection ID is + encrypted and this part is not visible in public. It + must be 2 bytes long and must be encoded in hex string + (which is 4 bytes long). If this option is omitted, a + random server ID is generated on startup and + configuration reload. --no-quic-bpf Disable eBPF. --frontend-http3-window-size= @@ -4053,6 +4069,8 @@ int main(int argc, char **argv) { required_argument, &flag, 183}, {SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY.c_str(), required_argument, &flag, 184}, + {SHRPX_OPT_FRONTEND_QUIC_SERVER_ID.c_str(), required_argument, &flag, + 185}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -4936,6 +4954,11 @@ int main(int argc, char **argv) { SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY, StringRef{optarg}); break; + case 185: + // --frontend-quic-server-id + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_SERVER_ID, + StringRef{optarg}); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index f210128b..76704d0e 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2273,6 +2273,11 @@ int option_lookup_token(const char *name, size_t namelen) { break; case 23: switch (name[22]) { + case 'd': + if (util::strieq_l("frontend-quic-server-i", name, 22)) { + return SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID; + } + break; case 'e': if (util::strieq_l("client-private-key-fil", name, 22)) { return SHRPX_OPTID_CLIENT_PRIVATE_KEY_FILE; @@ -4036,6 +4041,17 @@ int parse_config(Config *config, int optid, const StringRef &opt, optarg); #endif // ENABLE_HTTP3 + return 0; + case SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID: +#ifdef ENABLE_HTTP3 + if (optarg.size() != config->quic.upstream.server_id.size() * 2 || + !util::is_hex_string(optarg)) { + LOG(ERROR) << opt << ": must be a hex-string"; + return -1; + } + util::decode_hex(std::begin(config->quic.upstream.server_id), optarg); +#endif // ENABLE_HTTP3 + return 0; case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 6202b174..fef79a60 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -393,6 +393,8 @@ constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER = StringRef::from_lit("frontend-quic-congestion-controller"); constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY = StringRef::from_lit("frontend-quic-connection-id-encryption-key"); +constexpr auto SHRPX_OPT_FRONTEND_QUIC_SERVER_ID = + StringRef::from_lit("frontend-quic-server-id"); constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -764,6 +766,7 @@ struct QUICConfig { bool early_data; bool require_token; std::array cid_encryption_key; + std::array server_id; } upstream; struct { StringRef prog_file; @@ -1223,6 +1226,7 @@ enum { SHRPX_OPTID_FRONTEND_QUIC_IDLE_TIMEOUT, SHRPX_OPTID_FRONTEND_QUIC_QLOG_DIR, SHRPX_OPTID_FRONTEND_QUIC_REQUIRE_TOKEN, + SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID, SHRPX_OPTID_FRONTEND_READ_TIMEOUT, SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT, SHRPX_OPTID_HEADER_FIELD_BUFFER, diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index c1c72a02..87a1efb4 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -1292,9 +1292,9 @@ int ConnectionHandler::quic_ipc_read() { std::array decrypted_dcid; - if (decrypt_quic_connection_id(decrypted_dcid.data(), dcid, - quicconf.upstream.cid_encryption_key.data()) != - 0) { + if (decrypt_quic_connection_id( + decrypted_dcid.data(), dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, + quicconf.upstream.cid_encryption_key.data()) != 0) { return -1; } diff --git a/src/shrpx_quic.cc b/src/shrpx_quic.cc index aa38617a..bf313909 100644 --- a/src/shrpx_quic.cc +++ b/src/shrpx_quic.cc @@ -143,30 +143,40 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, return 0; } -int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen) { +int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, + const uint8_t *server_id, + const uint8_t *key) { + assert(cidlen == SHRPX_QUIC_SCIDLEN); + if (RAND_bytes(cid.data, cidlen) != 1) { return -1; } cid.datalen = cidlen; - return 0; + auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET; + + std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, p); + + return encrypt_quic_connection_id(p, p, key); } int generate_encrypted_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, const uint8_t *cid_prefix, const uint8_t *key) { - assert(cidlen > SHRPX_QUIC_CID_PREFIXLEN); + assert(cidlen == SHRPX_QUIC_SCIDLEN); - auto p = std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, cid.data); - - if (RAND_bytes(p, cidlen - SHRPX_QUIC_CID_PREFIXLEN) != 1) { + if (RAND_bytes(cid.data, cidlen) != 1) { return -1; } cid.datalen = cidlen; - return encrypt_quic_connection_id(cid.data, cid.data, key); + auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET; + + std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, p); + + return encrypt_quic_connection_id(p, p, key); } int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src, diff --git a/src/shrpx_quic.h b/src/shrpx_quic.h index 96a109fc..846d5a83 100644 --- a/src/shrpx_quic.h +++ b/src/shrpx_quic.h @@ -62,7 +62,10 @@ namespace shrpx { struct UpstreamAddr; constexpr size_t SHRPX_QUIC_SCIDLEN = 20; +constexpr size_t SHRPX_QUIC_SERVER_IDLEN = 2; +// SHRPX_QUIC_CID_PREFIXLEN includes SHRPX_QUIC_SERVER_IDLEN. constexpr size_t SHRPX_QUIC_CID_PREFIXLEN = 8; +constexpr size_t SHRPX_QUIC_CID_PREFIX_OFFSET = 1; constexpr size_t SHRPX_QUIC_DECRYPTED_DCIDLEN = 16; constexpr size_t SHRPX_QUIC_CID_ENCRYPTION_KEYLEN = 16; constexpr size_t SHRPX_QUIC_MAX_UDP_PAYLOAD_SIZE = 1472; @@ -78,7 +81,9 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, size_t local_salen, const uint8_t *data, size_t datalen, size_t gso_size); -int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen); +int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, + const uint8_t *server_id, + const uint8_t *key); int generate_encrypted_quic_connection_id(ngtcp2_cid &cid, size_t cidlen, const uint8_t *cid_prefix, diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 41b8b9e4..00f5876e 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -128,7 +128,7 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, if (dcidlen == SHRPX_QUIC_SCIDLEN) { if (decrypt_quic_connection_id( - decrypted_dcid.data(), dcid, + decrypted_dcid.data(), dcid + SHRPX_QUIC_CID_PREFIX_OFFSET, quicconf.upstream.cid_encryption_key.data()) != 0) { return 0; } @@ -419,9 +419,14 @@ int QUICConnectionHandler::send_retry( return -1; } + auto config = get_config(); + auto &quicconf = config->quic; + ngtcp2_cid retry_scid; - if (generate_quic_connection_id(retry_scid, SHRPX_QUIC_SCIDLEN) != 0) { + if (generate_quic_retry_connection_id( + retry_scid, SHRPX_QUIC_SCIDLEN, quicconf.upstream.server_id.data(), + quicconf.upstream.cid_encryption_key.data()) != 0) { return -1; } diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 75bd7325..720fd8d2 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -1273,8 +1273,10 @@ void downstream_failure(DownstreamAddr *addr, const Address *raddr) { } #ifdef ENABLE_HTTP3 -int create_cid_prefix(uint8_t *cid_prefix) { - if (RAND_bytes(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN) != 1) { +int create_cid_prefix(uint8_t *cid_prefix, const uint8_t *server_id) { + auto p = std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, cid_prefix); + + if (RAND_bytes(p, SHRPX_QUIC_CID_PREFIXLEN - SHRPX_QUIC_SERVER_IDLEN) != 1) { return -1; } diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 72ef2e68..cd53173f 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -467,8 +467,8 @@ void downstream_failure(DownstreamAddr *addr, const Address *raddr); #ifdef ENABLE_HTTP3 // Creates unpredictable SHRPX_QUIC_CID_PREFIXLEN bytes sequence which // is used as a prefix of QUIC Connection ID. This function returns -// -1 on failure. -int create_cid_prefix(uint8_t *cid_prefix); +// -1 on failure. |server_id| must be 2 bytes long. +int create_cid_prefix(uint8_t *cid_prefix, const uint8_t *server_id); #endif // ENABLE_HTTP3 } // namespace shrpx