walkero
/
nghttp2
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

nghttpx: Read QUIC keying materials from file 

Add --frontend-quic-secret-file to read QUIC keying materials from
file.  --frontend-quic-connection-id-encryption-key was removed in
favor of this new option.
This commit is contained in:
Tatsuhiro Tsujikawa 2021-09-22 18:24:50 +09:00
parent c40309ae8e
commit 308c73bfa2
15 changed files with 340 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.rst
View File

@ -341,9 +341,9 @@ configure script with ``--enable-http3``.
For nghttpx to reload configurations and swapping its executable while
gracefully terminating old worker processes, eBPF is required. Run
the configure script with ``--enable-http3 --with-libbpf`` to build
eBPF program. The Connection ID encryption key must be set with
``--frontend-quic-connection-id-encryption-key`` and must not change
in order to keep the existing connections alive during reload.
eBPF program. The QUIC keying material must be set with
``--frontend-quic-secret-file`` in order to keep the existing
connections alive during reload.
The detailed steps to build HTTP/3 enabled h2load and nghttpx follow.

8
doc/sources/nghttpx-howto.rst
View File

@ -534,10 +534,10 @@ nghttpx does not support HTTP/3 on backend connection.
Hot swapping (SIGUSR2) or configuration reload (SIGHUP) require eBPF
program. Without eBPF, old worker processes keep getting HTTP/3
traffic and do not work as intended. Connection ID encryption key
must be set with
:option:`--frontend-quic-connection-id-encryption-key` and must not
change in order to keep the existing connections alive during reload.
traffic and do not work as intended. The QUIC keying material to
encrypt Connection ID must be set with
:option:`--frontend-quic-secret-file` and must provide the existing
keys in order to keep the existing connections alive during reload.
In order announce that HTTP/3 endpoint is available, you should
specify alt-svc header field. For example, the following options send

2
gennghttpxfun.py
View File

@ -192,8 +192,8 @@ OPTIONS = [
"frontend-quic-qlog-dir",
"frontend-quic-require-token",
"frontend-quic-congestion-controller",
"frontend-quic-connection-id-encryption-key",
"frontend-quic-server-id",
"frontend-quic-secret-file",
]
LOGVARS = [

56
src/shrpx.cc
View File

@ -1859,14 +1859,6 @@ void fill_default_config(Config *config) {
upstreamconf.congestion_controller = NGTCP2_CC_ALGO_CUBIC;
// TODO Not really nice to generate random key here, but fine for
// now.
if (RAND_bytes(upstreamconf.cid_encryption_key.data(),
upstreamconf.cid_encryption_key.size()) != 1) {
assert(0);
abort();
}
if (RAND_bytes(upstreamconf.server_id.data(),
upstreamconf.server_id.size()) != 1) {
assert(0);
@ -3253,14 +3245,31 @@ HTTP/3 and QUIC:
? "cubic"
: "bbr")
<< R"(
--frontend-quic-connection-id-encryption-key=<HEXSTRING>
Specify Connection ID encryption key. The encryption
key must be 16 bytes, and it must be encoded in hex
string (which is 32 bytes long). If this option is
omitted, new key is generated. In order to survive QUIC
connection in a configuration reload event, old and new
configuration must have this option and share the same
key.
--frontend-quic-secret-file=<PATH>
Path to file that contains secure random data to be used
as QUIC keying materials. It is used to derive keys for
encrypting tokens and Connection IDs. It is not used to
encrypt QUIC packets. Each line of this file must
contain exactly 136 bytes hex-encoded string (when
decoded the byte string is 68 bytes long). The first 2
bits of decoded byte string are used to identify the
keying material. An empty line or a line which starts
'#' is ignored. The file can contain more than one
keying materials. Because the identifier is 2 bits, at
most 4 keying materials are read and the remaining data
is discarded. The first keying material in the file is
primarily used for encryption and decryption for new
connection. The other ones are used to decrypt data for
the existing connections. Specifying multiple keying
materials enables key rotation. Please note that key
rotation does not occur automatically. User should
update files or change options values and restart
nghttpx gracefully. If opening or reading given file
fails, all loaded keying materials are discarded and it
is treated as if none of this option is given. If this
option is not given or an error occurred while opening
or reading a file, a keying material is generated
internally on startup and reload.
--frontend-quic-server-id=<HEXSTRING>
Specify server ID encoded in Connection ID to identify
this particular server instance. Connection ID is
@ -4067,10 +4076,10 @@ int main(int argc, char **argv) {
182},
{SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER.c_str(),
required_argument, &flag, 183},
{SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY.c_str(),
required_argument, &flag, 184},
{SHRPX_OPT_FRONTEND_QUIC_SERVER_ID.c_str(), required_argument, &flag,
185},
{SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE.c_str(), required_argument, &flag,
186},
{nullptr, 0, nullptr, 0}};
int option_index = 0;
@ -4948,17 +4957,16 @@ int main(int argc, char **argv) {
cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER,
StringRef{optarg});
break;
case 184:
// --frontend-quic-connection-id-encryption-key
cmdcfgs.emplace_back(
SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY,
StringRef{optarg});
break;
case 185:
// --frontend-quic-server-id
cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_SERVER_ID,
StringRef{optarg});
break;
case 186:
// --frontend-quic-secret-file
cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE,
StringRef{optarg});
break;
default:
break;
}

88
src/shrpx_config.cc
View File

@ -230,6 +230,69 @@ read_tls_ticket_key_file(const std::vector<StringRef> &files,
return ticket_keys;
}
#ifdef ENABLE_HTTP3
std::shared_ptr<QUICKeyingMaterials>
read_quic_secret_file(const StringRef &path) {
constexpr size_t expectedlen =
SHRPX_QUIC_SECRET_RESERVEDLEN + SHRPX_QUIC_SECRETLEN + SHRPX_QUIC_SALTLEN;
auto qkms = std::make_shared<QUICKeyingMaterials>();
auto &kms = qkms->keying_materials;
std::ifstream f(path.c_str());
if (!f) {
LOG(ERROR) << "frontend-quic-secret-file: could not open file " << path;
return nullptr;
}
std::array<char, 4096> buf;
while (f.getline(buf.data(), buf.size())) {
if (f.gcount() == 1 || buf[0] == '#') {
continue;
}
auto s = StringRef{std::begin(buf), std::begin(buf) + f.gcount() - 1};
if (s.size() != expectedlen * 2 || !util::is_hex_string(s)) {
LOG(ERROR) << "frontend-quic-secret-file: each line must be a "
<< expectedlen * 2 << " bytes hex encoded string";
return nullptr;
}
kms.emplace_back();
auto &qkm = kms.back();
auto p = std::begin(s);
util::decode_hex(std::begin(qkm.reserved),
StringRef{p, p + qkm.reserved.size()});
p += qkm.reserved.size() * 2;
util::decode_hex(std::begin(qkm.secret),
StringRef{p, p + qkm.secret.size()});
p += qkm.secret.size() * 2;
util::decode_hex(std::begin(qkm.salt), StringRef{p, p + qkm.salt.size()});
p += qkm.salt.size() * 2;
assert(static_cast<size_t>(p - std::begin(s)) == expectedlen * 2);
qkm.id = qkm.reserved[0] & 0xc0;
if (kms.size() == 4) {
break;
}
}
if (f.bad()) {
LOG(ERROR)
<< "frontend-quic-secret-file: error occurred while reading file "
<< path;
return nullptr;
}
return qkms;
}
#endif // ENABLE_HTTP3
FILE *open_file_for_write(const char *filename) {
std::array<char, STRERROR_BUFSIZE> errbuf;
@ -2344,6 +2407,9 @@ int option_lookup_token(const char *name, size_t namelen) {
if (util::strieq_l("backend-http2-window-siz", name, 24)) {
return SHRPX_OPTID_BACKEND_HTTP2_WINDOW_SIZE;
}
if (util::strieq_l("frontend-quic-secret-fil", name, 24)) {
return SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE;
}
break;
case 'g':
if (util::strieq_l("http2-no-cookie-crumblin", name, 24)) {
@ -2689,10 +2755,6 @@ int option_lookup_token(const char *name, size_t namelen) {
case 42:
switch (name[41]) {
case 'y':
if (util::strieq_l("frontend-quic-connection-id-encryption-ke", name,
41)) {
return SHRPX_OPTID_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY;
}
if (util::strieq_l("tls-session-cache-memcached-address-famil", name,
41)) {
return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY;
@ -4029,18 +4091,6 @@ int parse_config(Config *config, int optid, const StringRef &opt,
}
#endif // ENABLE_HTTP3
return 0;
case SHRPX_OPTID_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY:
#ifdef ENABLE_HTTP3
if (optarg.size() != config->quic.upstream.cid_encryption_key.size() * 2 ||
!util::is_hex_string(optarg)) {
LOG(ERROR) << opt << ": must be a hex-string";
return -1;
}
util::decode_hex(std::begin(config->quic.upstream.cid_encryption_key),
optarg);
#endif // ENABLE_HTTP3
return 0;
case SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID:
#ifdef ENABLE_HTTP3
@ -4052,6 +4102,12 @@ int parse_config(Config *config, int optid, const StringRef &opt,
util::decode_hex(std::begin(config->quic.upstream.server_id), optarg);
#endif // ENABLE_HTTP3
return 0;
case SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE:
#ifdef ENABLE_HTTP3
config->quic.upstream.secret_file = make_string_ref(config->balloc, optarg);
#endif // ENABLE_HTTP3
return 0;
case SHRPX_OPTID_CONF:
LOG(WARN) << "conf: ignored";

29
src/shrpx_config.h
View File

@ -391,10 +391,10 @@ constexpr auto SHRPX_OPT_FRONTEND_QUIC_REQUIRE_TOKEN =
StringRef::from_lit("frontend-quic-require-token");
constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER =
StringRef::from_lit("frontend-quic-congestion-controller");
constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY =
StringRef::from_lit("frontend-quic-connection-id-encryption-key");
constexpr auto SHRPX_OPT_FRONTEND_QUIC_SERVER_ID =
StringRef::from_lit("frontend-quic-server-id");
constexpr auto SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE =
StringRef::from_lit("frontend-quic-secret-file");
constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8;
@ -606,10 +606,18 @@ struct TLSCertificate {
};
#ifdef ENABLE_HTTP3
struct QUICSecret {
std::array<uint8_t, SHRPX_QUIC_STATELESS_RESET_SECRETLEN>
stateless_reset_secret;
std::array<uint8_t, SHRPX_QUIC_TOKEN_SECRETLEN> token_secret;
struct QUICKeyingMaterial {
std::array<uint8_t, SHRPX_QUIC_SECRET_RESERVEDLEN> reserved;
std::array<uint8_t, SHRPX_QUIC_SECRETLEN> secret;
std::array<uint8_t, SHRPX_QUIC_SALTLEN> salt;
std::array<uint8_t, SHRPX_QUIC_CID_ENCRYPTION_KEYLEN> cid_encryption_key;
// Identifier of this keying material. Only the first 2 bits are
// used.
uint8_t id;
};
struct QUICKeyingMaterials {
std::vector<QUICKeyingMaterial> keying_materials;
};
#endif // ENABLE_HTTP3
@ -765,8 +773,8 @@ struct QUICConfig {
ngtcp2_cc_algo congestion_controller;
bool early_data;
bool require_token;
std::array<uint8_t, SHRPX_QUIC_CID_ENCRYPTION_KEYLEN> cid_encryption_key;
std::array<uint8_t, SHRPX_QUIC_SERVER_IDLEN> server_id;
StringRef secret_file;
} upstream;
struct {
StringRef prog_file;
@ -1220,12 +1228,12 @@ enum {
SHRPX_OPTID_FRONTEND_MAX_REQUESTS,
SHRPX_OPTID_FRONTEND_NO_TLS,
SHRPX_OPTID_FRONTEND_QUIC_CONGESTION_CONTROLLER,
SHRPX_OPTID_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY,
SHRPX_OPTID_FRONTEND_QUIC_DEBUG_LOG,
SHRPX_OPTID_FRONTEND_QUIC_EARLY_DATA,
SHRPX_OPTID_FRONTEND_QUIC_IDLE_TIMEOUT,
SHRPX_OPTID_FRONTEND_QUIC_QLOG_DIR,
SHRPX_OPTID_FRONTEND_QUIC_REQUIRE_TOKEN,
SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE,
SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID,
SHRPX_OPTID_FRONTEND_READ_TIMEOUT,
SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT,
@ -1377,6 +1385,11 @@ std::unique_ptr<TicketKeys>
read_tls_ticket_key_file(const std::vector<StringRef> &files,
const EVP_CIPHER *cipher, const EVP_MD *hmac);
#ifdef ENABLE_HTTP3
std::shared_ptr<QUICKeyingMaterials>
read_quic_secret_file(const StringRef &path);
#endif // ENABLE_HTTP3
// Returns string representation of |proto|.
StringRef strproto(Proto proto);

38
src/shrpx_connection_handler.cc
View File

@ -298,8 +298,6 @@ int ConnectionHandler::create_single_worker() {
#endif // HAVE_MRUBY
#ifdef ENABLE_HTTP3
single_worker_->set_quic_secret(quic_secret_);
if (single_worker_->setup_quic_server_socket() != 0) {
return -1;
}
@ -404,8 +402,6 @@ int ConnectionHandler::create_worker_thread(size_t num) {
# endif // HAVE_MRUBY
# ifdef ENABLE_HTTP3
worker->set_quic_secret(quic_secret_);
if ((!apiconf.enabled || i != 0) &&
worker->setup_quic_server_socket() != 0) {
return -1;
@ -1047,25 +1043,14 @@ int ConnectionHandler::forward_quic_packet(const UpstreamAddr *faddr,
return -1;
}
int ConnectionHandler::create_quic_secret() {
auto quic_secret = std::make_shared<QUICSecret>();
void ConnectionHandler::set_quic_keying_materials(
std::shared_ptr<QUICKeyingMaterials> qkms) {
quic_keying_materials_ = std::move(qkms);
}
if (generate_quic_stateless_reset_secret(
quic_secret->stateless_reset_secret.data()) != 0) {
LOG(ERROR) << "Failed to generate QUIC Stateless Reset secret";
return -1;
}
if (generate_quic_token_secret(quic_secret->token_secret.data()) != 0) {
LOG(ERROR) << "Failed to generate QUIC token secret";
return -1;
}
quic_secret_ = std::move(quic_secret);
return 0;
const std::shared_ptr<QUICKeyingMaterials> &
ConnectionHandler::get_quic_keying_materials() const {
return quic_keying_materials_;
}
void ConnectionHandler::set_cid_prefixes(
@ -1287,14 +1272,13 @@ int ConnectionHandler::quic_ipc_read() {
return 0;
}
auto config = get_config();
auto &quicconf = config->quic;
auto &qkm = quic_keying_materials_->keying_materials.front();
std::array<uint8_t, SHRPX_QUIC_DECRYPTED_DCIDLEN> decrypted_dcid;
if (decrypt_quic_connection_id(
decrypted_dcid.data(), dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
quicconf.upstream.cid_encryption_key.data()) != 0) {
if (decrypt_quic_connection_id(decrypted_dcid.data(),
dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
qkm.cid_encryption_key.data()) != 0) {
return -1;
}

5
src/shrpx_connection_handler.h
View File

@ -198,7 +198,8 @@ public:
const Address &local_addr, const uint8_t *cid_prefix,
const uint8_t *data, size_t datalen);
int create_quic_secret();
void set_quic_keying_materials(std::shared_ptr<QUICKeyingMaterials> qkms);
const std::shared_ptr<QUICKeyingMaterials> &get_quic_keying_materials() const;
void set_cid_prefixes(
const std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>>
@ -263,7 +264,7 @@ private:
# ifdef HAVE_LIBBPF
std::vector<BPFRef> quic_bpf_refs_;
# endif // HAVE_LIBBPF
std::shared_ptr<QUICSecret> quic_secret_;
std::shared_ptr<QUICKeyingMaterials> quic_keying_materials_;
std::vector<SSL_CTX *> quic_all_ssl_ctx_;
std::vector<std::vector<SSL_CTX *>> quic_indexed_ssl_ctx_;
#endif // ENABLE_HTTP3

45
src/shrpx_http3_upstream.cc
View File

@ -40,6 +40,7 @@
#include "shrpx_quic.h"
#include "shrpx_worker.h"
#include "shrpx_http.h"
#include "shrpx_connection_handler.h"
#ifdef HAVE_MRUBY
# include "shrpx_mruby.h"
#endif // HAVE_MRUBY
@ -217,21 +218,17 @@ int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token,
auto upstream = static_cast<Http3Upstream *>(user_data);
auto handler = upstream->get_client_handler();
auto worker = handler->get_worker();
auto conn_handler = worker->get_connection_handler();
auto &qkms = conn_handler->get_quic_keying_materials();
auto &qkm = qkms->keying_materials.front();
auto config = get_config();
auto &quicconf = config->quic;
if (generate_quic_connection_id(
*cid, cidlen, worker->get_cid_prefix(),
quicconf.upstream.cid_encryption_key.data()) != 0) {
if (generate_quic_connection_id(*cid, cidlen, worker->get_cid_prefix(),
qkm.id, qkm.cid_encryption_key.data()) != 0) {
return NGTCP2_ERR_CALLBACK_FAILURE;
}
auto &quic_secret = worker->get_quic_secret();
auto &secret = quic_secret->stateless_reset_secret;
if (generate_quic_stateless_reset_token(token, *cid, secret.data(),
secret.size()) != 0) {
if (generate_quic_stateless_reset_token(token, *cid, qkm.secret.data(),
qkm.secret.size()) != 0) {
return NGTCP2_ERR_CALLBACK_FAILURE;
}
@ -487,11 +484,13 @@ int Http3Upstream::handshake_completed() {
auto path = ngtcp2_conn_get_path(conn_);
auto worker = handler_->get_worker();
auto &quic_secret = worker->get_quic_secret();
auto &secret = quic_secret->token_secret;
auto conn_handler = worker->get_connection_handler();
auto &qkms = conn_handler->get_quic_keying_materials();
auto &qkm = qkms->keying_materials.front();
if (generate_token(token.data(), tokenlen, path->remote.addr,
path->remote.addrlen, secret.data()) != 0) {
path->remote.addrlen, qkm.secret.data(),
qkm.secret.size()) != 0) {
return 0;
}
@ -513,6 +512,7 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
int rv;
auto worker = handler_->get_worker();
auto conn_handler = worker->get_connection_handler();
auto callbacks = ngtcp2_callbacks{
nullptr, // client_initial
@ -557,11 +557,14 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
auto &quicconf = config->quic;
auto &http3conf = config->http3;
auto &qkms = conn_handler->get_quic_keying_materials();
auto &qkm = qkms->keying_materials.front();
ngtcp2_cid scid;
if (generate_quic_connection_id(
scid, SHRPX_QUIC_SCIDLEN, worker->get_cid_prefix(),
quicconf.upstream.cid_encryption_key.data()) != 0) {
if (generate_quic_connection_id(scid, SHRPX_QUIC_SCIDLEN,
worker->get_cid_prefix(), qkm.id,
qkm.cid_encryption_key.data()) != 0) {
return -1;
}
@ -608,12 +611,8 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
params.original_dcid = initial_hd.dcid;
}
auto &quic_secret = worker->get_quic_secret();
auto &stateless_reset_secret = quic_secret->stateless_reset_secret;
rv = generate_quic_stateless_reset_token(params.stateless_reset_token, scid,
stateless_reset_secret.data(),
stateless_reset_secret.size());
rv = generate_quic_stateless_reset_token(
params.stateless_reset_token, scid, qkm.secret.data(), qkm.secret.size());
if (rv != 0) {
ULOG(ERROR, this) << "generate_quic_stateless_reset_token failed";
return -1;

82
src/shrpx_quic.cc
View File

@ -144,7 +144,7 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
}
int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen,
const uint8_t *server_id,
const uint8_t *server_id, uint8_t km_id,
const uint8_t *key) {
assert(cidlen == SHRPX_QUIC_SCIDLEN);
@ -154,6 +154,8 @@ int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen,
cid.datalen = cidlen;
cid.data[0] = (cid.data[0] & 0x3f) | km_id;
auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET;
std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, p);
@ -162,7 +164,8 @@ int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen,
}
int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen,
const uint8_t *cid_prefix, const uint8_t *key) {
const uint8_t *cid_prefix, uint8_t km_id,
const uint8_t *key) {
assert(cidlen == SHRPX_QUIC_SCIDLEN);
if (RAND_bytes(cid.data, cidlen) != 1) {
@ -171,6 +174,8 @@ int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen,
cid.datalen = cidlen;
cid.data[0] = (cid.data[0] & 0x3f) | km_id;
auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET;
std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, p);
@ -257,32 +262,16 @@ int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid &cid,
return 0;
}
int generate_quic_stateless_reset_secret(uint8_t *secret) {
if (RAND_bytes(secret, SHRPX_QUIC_STATELESS_RESET_SECRETLEN) != 1) {
return -1;
}
return 0;
}
int generate_quic_token_secret(uint8_t *secret) {
if (RAND_bytes(secret, SHRPX_QUIC_TOKEN_SECRETLEN) != 1) {
return -1;
}
return 0;
}
int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
socklen_t salen, const ngtcp2_cid &retry_scid,
const ngtcp2_cid &odcid, const uint8_t *token_secret) {
const ngtcp2_cid &odcid, const uint8_t *secret,
size_t secretlen) {
auto t = std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::system_clock::now().time_since_epoch())
.count();
auto stokenlen = ngtcp2_crypto_generate_retry_token(
token, token_secret, SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, &retry_scid,
&odcid, t);
token, secret, secretlen, sa, salen, &retry_scid, &odcid, t);
if (stokenlen < 0) {
return -1;
}
@ -294,15 +283,16 @@ int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen,
const ngtcp2_cid &dcid, const sockaddr *sa,
socklen_t salen, const uint8_t *token_secret) {
socklen_t salen, const uint8_t *secret,
size_t secretlen) {
auto t = std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::system_clock::now().time_since_epoch())
.count();
if (ngtcp2_crypto_verify_retry_token(&odcid, token, tokenlen, token_secret,
SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen,
&dcid, 10 * NGTCP2_SECONDS, t) != 0) {
if (ngtcp2_crypto_verify_retry_token(&odcid, token, tokenlen, secret,
secretlen, sa, salen, &dcid,
10 * NGTCP2_SECONDS, t) != 0) {
return -1;
}
@ -310,13 +300,13 @@ int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen,
}
int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
size_t salen, const uint8_t *token_secret) {
size_t salen, const uint8_t *secret, size_t secretlen) {
auto t = std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::system_clock::now().time_since_epoch())
.count();
auto stokenlen = ngtcp2_crypto_generate_regular_token(
token, token_secret, SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, t);
token, secret, secretlen, sa, salen, t);
if (stokenlen < 0) {
return -1;
}
@ -327,18 +317,48 @@ int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
}
int verify_token(const uint8_t *token, size_t tokenlen, const sockaddr *sa,
socklen_t salen, const uint8_t *token_secret) {
socklen_t salen, const uint8_t *secret, size_t secretlen) {
auto t = std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::system_clock::now().time_since_epoch())
.count();
if (ngtcp2_crypto_verify_regular_token(token, tokenlen, token_secret,
SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen,
3600 * NGTCP2_SECONDS, t) != 0) {
if (ngtcp2_crypto_verify_regular_token(token, tokenlen, secret, secretlen, sa,
salen, 3600 * NGTCP2_SECONDS,
t) != 0) {
return -1;
}
return 0;
}
int generate_quic_connection_id_encryption_key(uint8_t *key, size_t keylen,
const uint8_t *secret,
size_t secretlen,
const uint8_t *salt,
size_t saltlen) {
constexpr uint8_t info[] = "connection id encryption key";
ngtcp2_crypto_md sha256;
ngtcp2_crypto_md_init(
&sha256, reinterpret_cast<void *>(const_cast<EVP_MD *>(EVP_sha256())));
if (ngtcp2_crypto_hkdf(key, keylen, &sha256, secret, secretlen, salt, saltlen,
info, str_size(info)) != 0) {
return -1;
}
return 0;
}
const QUICKeyingMaterial *
select_quic_keying_material(const QUICKeyingMaterials &qkms,
const uint8_t *cid) {
for (auto &qkm : qkms.keying_materials) {
if (((*cid) & 0xc0) == qkm.id) {
return &qkm;
}
}
return &qkms.keying_materials.front();
}
} // namespace shrpx

36
src/shrpx_quic.h
View File

@ -60,6 +60,8 @@ bool operator==(const ngtcp2_cid &lhs, const ngtcp2_cid &rhs);
namespace shrpx {
struct UpstreamAddr;
struct QUICKeyingMaterials;
struct QUICKeyingMaterial;
constexpr size_t SHRPX_QUIC_SCIDLEN = 20;
constexpr size_t SHRPX_QUIC_SERVER_IDLEN = 2;
@ -69,10 +71,11 @@ constexpr size_t SHRPX_QUIC_CID_PREFIX_OFFSET = 1;
constexpr size_t SHRPX_QUIC_DECRYPTED_DCIDLEN = 16;
constexpr size_t SHRPX_QUIC_CID_ENCRYPTION_KEYLEN = 16;
constexpr size_t SHRPX_QUIC_MAX_UDP_PAYLOAD_SIZE = 1472;
constexpr size_t SHRPX_QUIC_STATELESS_RESET_SECRETLEN = 32;
constexpr size_t SHRPX_QUIC_TOKEN_SECRETLEN = 32;
constexpr size_t SHRPX_QUIC_CONN_CLOSE_PKTLEN = 256;
constexpr size_t SHRPX_QUIC_STATELESS_RESET_BURST = 100;
constexpr size_t SHRPX_QUIC_SECRET_RESERVEDLEN = 4;
constexpr size_t SHRPX_QUIC_SECRETLEN = 32;
constexpr size_t SHRPX_QUIC_SALTLEN = 32;
ngtcp2_tstamp quic_timestamp();
@ -82,11 +85,12 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
size_t gso_size);
int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen,
const uint8_t *server_id,
const uint8_t *server_id, uint8_t km_id,
const uint8_t *key);
int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen,
const uint8_t *cid_prefix, const uint8_t *key);
const uint8_t *cid_prefix, uint8_t km_id,
const uint8_t *key);
int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src,
const uint8_t *key);
@ -103,23 +107,31 @@ int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid &cid,
const uint8_t *secret,
size_t secretlen);
int generate_quic_stateless_reset_secret(uint8_t *secret);
int generate_quic_token_secret(uint8_t *secret);
int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
socklen_t salen, const ngtcp2_cid &retry_scid,
const ngtcp2_cid &odcid, const uint8_t *token_secret);
const ngtcp2_cid &odcid, const uint8_t *secret,
size_t secretlen);
int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen,
const ngtcp2_cid &dcid, const sockaddr *sa,
socklen_t salen, const uint8_t *token_secret);
socklen_t salen, const uint8_t *secret,
size_t secretlen);
int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
size_t salen, const uint8_t *token_secret);
size_t salen, const uint8_t *secret, size_t secretlen);
int verify_token(const uint8_t *token, size_t tokenlen, const sockaddr *sa,
socklen_t salen, const uint8_t *token_secret);
socklen_t salen, const uint8_t *secret, size_t secretlen);
int generate_quic_connection_id_encryption_key(uint8_t *key, size_t keylen,
const uint8_t *secret,
size_t secretlen,
const uint8_t *salt,
size_t saltlen);
const QUICKeyingMaterial *
select_quic_keying_material(const QUICKeyingMaterials &qkms,
const uint8_t *cid);
} // namespace shrpx

72
src/shrpx_quic_connection_handler.cc
View File

@ -126,14 +126,20 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
if (it == std::end(connections_)) {
std::array<uint8_t, SHRPX_QUIC_DECRYPTED_DCIDLEN> decrypted_dcid;
auto &qkms = conn_handler->get_quic_keying_materials();
const QUICKeyingMaterial *qkm = nullptr;
if (dcidlen == SHRPX_QUIC_SCIDLEN) {
if (decrypt_quic_connection_id(
decrypted_dcid.data(), dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
quicconf.upstream.cid_encryption_key.data()) != 0) {
qkm = select_quic_keying_material(*qkms.get(), dcid);
if (decrypt_quic_connection_id(decrypted_dcid.data(),
dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
qkm->cid_encryption_key.data()) != 0) {
return 0;
}
if (!std::equal(std::begin(decrypted_dcid),
if (qkm != &qkms->keying_materials.front() ||
!std::equal(std::begin(decrypted_dcid),
std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
worker_->get_cid_prefix())) {
auto quic_lwp =
@ -170,14 +176,25 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
switch (ngtcp2_accept(&hd, data, datalen)) {
case 0: {
// If we get Initial and it has the CID prefix of this worker, it
// is likely that client is intentionally use the our prefix.
// If we get Initial and it has the CID prefix of this worker,
// it is likely that client is intentionally use the prefix.
// Just drop it.
if (dcidlen == SHRPX_QUIC_SCIDLEN &&
std::equal(std::begin(decrypted_dcid),
std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
worker_->get_cid_prefix())) {
return 0;
if (dcidlen == SHRPX_QUIC_SCIDLEN) {
if (qkm != &qkms->keying_materials.front()) {
qkm = &qkms->keying_materials.front();
if (decrypt_quic_connection_id(decrypted_dcid.data(),
dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
qkm->cid_encryption_key.data()) != 0) {
return 0;
}
}
if (std::equal(std::begin(decrypted_dcid),
std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
worker_->get_cid_prefix())) {
return 0;
}
}
if (worker_->get_graceful_shutdown()) {
@ -197,14 +214,18 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
break;
}
auto &quic_secret = worker_->get_quic_secret();
auto &secret = quic_secret->token_secret;
if (dcidlen != SHRPX_QUIC_SCIDLEN) {
// Initial packets with token must have DCID chosen by server.
return 0;
}
auto qkm = select_quic_keying_material(*qkms.get(), dcid);
switch (hd.token.base[0]) {
case NGTCP2_CRYPTO_TOKEN_MAGIC_RETRY:
if (verify_retry_token(odcid, hd.token.base, hd.token.len, hd.dcid,
&remote_addr.su.sa, remote_addr.len,
secret.data()) != 0) {
qkm->secret.data(), qkm->secret.size()) != 0) {
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Failed to validate Retry token from remote="
<< util::to_numeric_addr(&remote_addr);
@ -229,7 +250,8 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
break;
case NGTCP2_CRYPTO_TOKEN_MAGIC_REGULAR:
if (verify_token(hd.token.base, hd.token.len, &remote_addr.su.sa,
remote_addr.len, secret.data()) != 0) {
remote_addr.len, qkm->secret.data(),
qkm->secret.size()) != 0) {
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Failed to validate token from remote="
<< util::to_numeric_addr(&remote_addr);
@ -422,11 +444,15 @@ int QUICConnectionHandler::send_retry(
auto config = get_config();
auto &quicconf = config->quic;
auto conn_handler = worker_->get_connection_handler();
auto &qkms = conn_handler->get_quic_keying_materials();
auto &qkm = qkms->keying_materials.front();
ngtcp2_cid retry_scid;
if (generate_quic_retry_connection_id(
retry_scid, SHRPX_QUIC_SCIDLEN, quicconf.upstream.server_id.data(),
quicconf.upstream.cid_encryption_key.data()) != 0) {
qkm.id, qkm.cid_encryption_key.data()) != 0) {
return -1;
}
@ -437,12 +463,9 @@ int QUICConnectionHandler::send_retry(
ngtcp2_cid_init(&idcid, ini_dcid, ini_dcidlen);
ngtcp2_cid_init(&iscid, ini_scid, ini_scidlen);
auto &quic_secret = worker_->get_quic_secret();
auto &secret = quic_secret->token_secret;
if (generate_retry_token(token.data(), tokenlen, &remote_addr.su.sa,
remote_addr.len, retry_scid, idcid,
secret.data()) != 0) {
qkm.secret.data(), qkm.secret.size()) != 0) {
return -1;
}
@ -539,11 +562,12 @@ int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr,
ngtcp2_cid_init(&cid, dcid, dcidlen);
auto &quic_secret = worker_->get_quic_secret();
auto &secret = quic_secret->stateless_reset_secret;
auto conn_handler = worker_->get_connection_handler();
auto &qkms = conn_handler->get_quic_keying_materials();
auto &qkm = qkms->keying_materials.front();
rv = generate_quic_stateless_reset_token(token.data(), cid, secret.data(),
secret.size());
rv = generate_quic_stateless_reset_token(token.data(), cid, qkm.secret.data(),
qkm.secret.size());
if (rv != 0) {
return -1;
}

18
src/shrpx_worker.cc
View File

@ -933,14 +933,14 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) {
return -1;
}
auto &quicconf = config->quic;
constexpr uint32_t key_high_idx = 1;
constexpr uint32_t key_low_idx = 2;
auto &qkms = conn_handler_->get_quic_keying_materials();
auto &qkm = qkms->keying_materials.front();
if (bpf_map_update_elem(bpf_map__fd(sk_info), &key_high_idx,
quicconf.upstream.cid_encryption_key.data(),
BPF_ANY) != 0) {
qkm.cid_encryption_key.data(), BPF_ANY) != 0) {
LOG(FATAL) << "Failed to update key_high_idx sk_info: "
<< xsi_strerror(errno, errbuf.data(), errbuf.size());
close(fd);
@ -948,7 +948,7 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) {
}
if (bpf_map_update_elem(bpf_map__fd(sk_info), &key_low_idx,
quicconf.upstream.cid_encryption_key.data() + 8,
qkm.cid_encryption_key.data() + 8,
BPF_ANY) != 0) {
LOG(FATAL) << "Failed to update key_low_idx sk_info: "
<< xsi_strerror(errno, errbuf.data(), errbuf.size());
@ -1010,14 +1010,6 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) {
const uint8_t *Worker::get_cid_prefix() const { return cid_prefix_.data(); }
void Worker::set_quic_secret(const std::shared_ptr<QUICSecret> &secret) {
quic_secret_ = secret;
}
const std::shared_ptr<QUICSecret> &Worker::get_quic_secret() const {
return quic_secret_;
}
const UpstreamAddr *Worker::find_quic_upstream_addr(const Address &local_addr) {
std::array<char, NI_MAXHOST> host;

5
src/shrpx_worker.h
View File

@ -370,10 +370,6 @@ public:
const uint8_t *get_cid_prefix() const;
void set_quic_secret(const std::shared_ptr<QUICSecret> &secret);
const std::shared_ptr<QUICSecret> &get_quic_secret() const;
# ifdef HAVE_LIBBPF
bool should_attach_bpf() const;
@ -412,7 +408,6 @@ private:
std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN> cid_prefix_;
std::vector<UpstreamAddr> quic_upstream_addrs_;
std::vector<std::unique_ptr<QUICListener>> quic_listeners_;
std::shared_ptr<QUICSecret> quic_secret_;
#endif // ENABLE_HTTP3
std::shared_ptr<DownstreamConfig> downstreamconf_;

45
src/shrpx_worker_process.cc
View File

@ -519,10 +519,51 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) {
}
#ifdef ENABLE_HTTP3
if (conn_handler->create_quic_secret() != 0) {
return -1;
auto &quicconf = config->quic;
std::shared_ptr<QUICKeyingMaterials> qkms;
if (!quicconf.upstream.secret_file.empty()) {
qkms = read_quic_secret_file(quicconf.upstream.secret_file);
if (!qkms) {
LOG(WARN) << "Use QUIC keying materials generated internally";
}
}
if (!qkms) {
qkms = std::make_shared<QUICKeyingMaterials>();
qkms->keying_materials.resize(1);
auto &qkm = qkms->keying_materials.front();
if (RAND_bytes(qkm.reserved.data(), qkm.reserved.size()) != 1) {
LOG(ERROR) << "Failed to generate QUIC secret reserved data";
return -1;
}
if (RAND_bytes(qkm.secret.data(), qkm.secret.size()) != 1) {
LOG(ERROR) << "Failed to generate QUIC secret";
return -1;
}
if (RAND_bytes(qkm.salt.data(), qkm.salt.size()) != 1) {
LOG(ERROR) << "Failed to generate QUIC salt";
return -1;
}
}
for (auto &qkm : qkms->keying_materials) {
if (generate_quic_connection_id_encryption_key(
qkm.cid_encryption_key.data(), qkm.cid_encryption_key.size(),
qkm.secret.data(), qkm.secret.size(), qkm.salt.data(),
qkm.salt.size()) != 0) {
LOG(ERROR) << "Failed to generate QUIC Connection ID encryption key";
return -1;
}
}
conn_handler->set_quic_keying_materials(std::move(qkms));
conn_handler->set_cid_prefixes(wpconf->cid_prefixes);
conn_handler->set_quic_lingering_worker_processes(
wpconf->quic_lingering_worker_processes);