nghttpx: Read QUIC keying materials from file

Add --frontend-quic-secret-file to read QUIC keying materials from
file.  --frontend-quic-connection-id-encryption-key was removed in
favor of this new option.

README.rst

@@ -341,9 +341,9 @@ configure script with ``--enable-http3``.
 For nghttpx to reload configurations and swapping its executable while
 gracefully terminating old worker processes, eBPF is required.  Run
 the configure script with ``--enable-http3 --with-libbpf`` to build
-eBPF program.  The Connection ID encryption key must be set with
+eBPF program.  The QUIC keying material must be set with
-``--frontend-quic-connection-id-encryption-key`` and must not change
+``--frontend-quic-secret-file`` in order to keep the existing
-in order to keep the existing connections alive during reload.
+connections alive during reload.
 
 The detailed steps to build HTTP/3 enabled h2load and nghttpx follow.
 

doc/sources/nghttpx-howto.rst

@@ -534,10 +534,10 @@ nghttpx does not support HTTP/3 on backend connection.
 
 Hot swapping (SIGUSR2) or configuration reload (SIGHUP) require eBPF
 program.  Without eBPF, old worker processes keep getting HTTP/3
-traffic and do not work as intended.  Connection ID encryption key
+traffic and do not work as intended.  The QUIC keying material to
-must be set with
+encrypt Connection ID must be set with
-:option:`--frontend-quic-connection-id-encryption-key` and must not
+:option:`--frontend-quic-secret-file` and must provide the existing
-change in order to keep the existing connections alive during reload.
+keys in order to keep the existing connections alive during reload.
 
 In order announce that HTTP/3 endpoint is available, you should
 specify alt-svc header field.  For example, the following options send

gennghttpxfun.py

@@ -192,8 +192,8 @@ OPTIONS = [
     "frontend-quic-qlog-dir",
     "frontend-quic-require-token",
     "frontend-quic-congestion-controller",
-    "frontend-quic-connection-id-encryption-key",
     "frontend-quic-server-id",
+    "frontend-quic-secret-file",
 ]
 
 LOGVARS = [

src/shrpx.cc

@@ -1859,14 +1859,6 @@ void fill_default_config(Config *config) {
 
     upstreamconf.congestion_controller = NGTCP2_CC_ALGO_CUBIC;
 
-    // TODO Not really nice to generate random key here, but fine for
-    // now.
-    if (RAND_bytes(upstreamconf.cid_encryption_key.data(),
-                   upstreamconf.cid_encryption_key.size()) != 1) {
-      assert(0);
-      abort();
-    }
-
     if (RAND_bytes(upstreamconf.server_id.data(),
                    upstreamconf.server_id.size()) != 1) {
       assert(0);
@@ -3253,14 +3245,31 @@ HTTP/3 and QUIC:
               ? "cubic"
               : "bbr")
       << R"(
-  --frontend-quic-connection-id-encryption-key=<HEXSTRING>
+  --frontend-quic-secret-file=<PATH>
-              Specify  Connection ID  encryption key.   The encryption
+              Path to file that contains secure random data to be used
-              key must  be 16  bytes, and  it must  be encoded  in hex
+              as QUIC keying materials.  It is used to derive keys for
-              string  (which is  32 bytes  long).  If  this option  is
+              encrypting tokens and Connection IDs.  It is not used to
-              omitted, new key is generated.  In order to survive QUIC
+              encrypt  QUIC  packets.  Each  line  of  this file  must
-              connection in a configuration  reload event, old and new
+              contain  exactly  136  bytes  hex-encoded  string  (when
-              configuration must  have this option and  share the same
+              decoded the byte string is  68 bytes long).  The first 2
-              key.
+              bits of  decoded byte  string are  used to  identify the
+              keying material.  An  empty line or a  line which starts
+              '#'  is ignored.   The file  can contain  more than  one
+              keying materials.  Because the  identifier is 2 bits, at
+              most 4 keying materials are  read and the remaining data
+              is discarded.  The first keying  material in the file is
+              primarily  used for  encryption and  decryption for  new
+              connection.  The other ones are used to decrypt data for
+              the  existing connections.   Specifying multiple  keying
+              materials enables  key rotation.   Please note  that key
+              rotation  does  not  occur automatically.   User  should
+              update  files  or  change  options  values  and  restart
+              nghttpx gracefully.   If opening  or reading  given file
+              fails, all loaded keying  materials are discarded and it
+              is treated as if none of  this option is given.  If this
+              option is not  given or an error  occurred while opening
+              or  reading  a  file,  a keying  material  is  generated
+              internally on startup and reload.
   --frontend-quic-server-id=<HEXSTRING>
               Specify server  ID encoded in Connection  ID to identify
               this  particular  server  instance.   Connection  ID  is
@@ -4067,10 +4076,10 @@ int main(int argc, char **argv) {
          182},
         {SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER.c_str(),
          required_argument, &flag, 183},
-        {SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY.c_str(),
-         required_argument, &flag, 184},
         {SHRPX_OPT_FRONTEND_QUIC_SERVER_ID.c_str(), required_argument, &flag,
          185},
+        {SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE.c_str(), required_argument, &flag,
+         186},
         {nullptr, 0, nullptr, 0}};
 
     int option_index = 0;
@@ -4948,17 +4957,16 @@ int main(int argc, char **argv) {
         cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER,
                              StringRef{optarg});
         break;
-      case 184:
-        // --frontend-quic-connection-id-encryption-key
-        cmdcfgs.emplace_back(
-            SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY,
-            StringRef{optarg});
-        break;
       case 185:
         // --frontend-quic-server-id
         cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_SERVER_ID,
                              StringRef{optarg});
         break;
+      case 186:
+        // --frontend-quic-secret-file
+        cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE,
+                             StringRef{optarg});
+        break;
       default:
         break;
       }

src/shrpx_config.cc

@@ -230,6 +230,69 @@ read_tls_ticket_key_file(const std::vector<StringRef> &files,
   return ticket_keys;
 }
 
+#ifdef ENABLE_HTTP3
+std::shared_ptr<QUICKeyingMaterials>
+read_quic_secret_file(const StringRef &path) {
+  constexpr size_t expectedlen =
+      SHRPX_QUIC_SECRET_RESERVEDLEN + SHRPX_QUIC_SECRETLEN + SHRPX_QUIC_SALTLEN;
+
+  auto qkms = std::make_shared<QUICKeyingMaterials>();
+  auto &kms = qkms->keying_materials;
+
+  std::ifstream f(path.c_str());
+  if (!f) {
+    LOG(ERROR) << "frontend-quic-secret-file: could not open file " << path;
+    return nullptr;
+  }
+
+  std::array<char, 4096> buf;
+
+  while (f.getline(buf.data(), buf.size())) {
+    if (f.gcount() == 1 || buf[0] == '#') {
+      continue;
+    }
+
+    auto s = StringRef{std::begin(buf), std::begin(buf) + f.gcount() - 1};
+    if (s.size() != expectedlen * 2 || !util::is_hex_string(s)) {
+      LOG(ERROR) << "frontend-quic-secret-file: each line must be a "
+                 << expectedlen * 2 << " bytes hex encoded string";
+      return nullptr;
+    }
+
+    kms.emplace_back();
+    auto &qkm = kms.back();
+
+    auto p = std::begin(s);
+
+    util::decode_hex(std::begin(qkm.reserved),
+                     StringRef{p, p + qkm.reserved.size()});
+    p += qkm.reserved.size() * 2;
+    util::decode_hex(std::begin(qkm.secret),
+                     StringRef{p, p + qkm.secret.size()});
+    p += qkm.secret.size() * 2;
+    util::decode_hex(std::begin(qkm.salt), StringRef{p, p + qkm.salt.size()});
+    p += qkm.salt.size() * 2;
+
+    assert(static_cast<size_t>(p - std::begin(s)) == expectedlen * 2);
+
+    qkm.id = qkm.reserved[0] & 0xc0;
+
+    if (kms.size() == 4) {
+      break;
+    }
+  }
+
+  if (f.bad()) {
+    LOG(ERROR)
+        << "frontend-quic-secret-file: error occurred while reading file "
+        << path;
+    return nullptr;
+  }
+
+  return qkms;
+}
+#endif // ENABLE_HTTP3
+
 FILE *open_file_for_write(const char *filename) {
   std::array<char, STRERROR_BUFSIZE> errbuf;
 
@@ -2344,6 +2407,9 @@ int option_lookup_token(const char *name, size_t namelen) {
       if (util::strieq_l("backend-http2-window-siz", name, 24)) {
         return SHRPX_OPTID_BACKEND_HTTP2_WINDOW_SIZE;
       }
+      if (util::strieq_l("frontend-quic-secret-fil", name, 24)) {
+        return SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE;
+      }
       break;
     case 'g':
       if (util::strieq_l("http2-no-cookie-crumblin", name, 24)) {
@@ -2689,10 +2755,6 @@ int option_lookup_token(const char *name, size_t namelen) {
   case 42:
     switch (name[41]) {
     case 'y':
-      if (util::strieq_l("frontend-quic-connection-id-encryption-ke", name,
-                         41)) {
-        return SHRPX_OPTID_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY;
-      }
       if (util::strieq_l("tls-session-cache-memcached-address-famil", name,
                          41)) {
         return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY;
@@ -4029,18 +4091,6 @@ int parse_config(Config *config, int optid, const StringRef &opt,
     }
 #endif // ENABLE_HTTP3
 
-    return 0;
-  case SHRPX_OPTID_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY:
-#ifdef ENABLE_HTTP3
-    if (optarg.size() != config->quic.upstream.cid_encryption_key.size() * 2 ||
-        !util::is_hex_string(optarg)) {
-      LOG(ERROR) << opt << ": must be a hex-string";
-      return -1;
-    }
-    util::decode_hex(std::begin(config->quic.upstream.cid_encryption_key),
-                     optarg);
-#endif // ENABLE_HTTP3
-
     return 0;
   case SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID:
 #ifdef ENABLE_HTTP3
@@ -4052,6 +4102,12 @@ int parse_config(Config *config, int optid, const StringRef &opt,
     util::decode_hex(std::begin(config->quic.upstream.server_id), optarg);
 #endif // ENABLE_HTTP3
 
+    return 0;
+  case SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE:
+#ifdef ENABLE_HTTP3
+    config->quic.upstream.secret_file = make_string_ref(config->balloc, optarg);
+#endif // ENABLE_HTTP3
+
     return 0;
   case SHRPX_OPTID_CONF:
     LOG(WARN) << "conf: ignored";

src/shrpx_config.h

@@ -391,10 +391,10 @@ constexpr auto SHRPX_OPT_FRONTEND_QUIC_REQUIRE_TOKEN =
     StringRef::from_lit("frontend-quic-require-token");
 constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONGESTION_CONTROLLER =
     StringRef::from_lit("frontend-quic-congestion-controller");
-constexpr auto SHRPX_OPT_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY =
-    StringRef::from_lit("frontend-quic-connection-id-encryption-key");
 constexpr auto SHRPX_OPT_FRONTEND_QUIC_SERVER_ID =
     StringRef::from_lit("frontend-quic-server-id");
+constexpr auto SHRPX_OPT_FRONTEND_QUIC_SECRET_FILE =
+    StringRef::from_lit("frontend-quic-secret-file");
 
 constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8;
 
@@ -606,10 +606,18 @@ struct TLSCertificate {
 };
 
 #ifdef ENABLE_HTTP3
-struct QUICSecret {
+struct QUICKeyingMaterial {
-  std::array<uint8_t, SHRPX_QUIC_STATELESS_RESET_SECRETLEN>
+  std::array<uint8_t, SHRPX_QUIC_SECRET_RESERVEDLEN> reserved;
-      stateless_reset_secret;
+  std::array<uint8_t, SHRPX_QUIC_SECRETLEN> secret;
-  std::array<uint8_t, SHRPX_QUIC_TOKEN_SECRETLEN> token_secret;
+  std::array<uint8_t, SHRPX_QUIC_SALTLEN> salt;
+  std::array<uint8_t, SHRPX_QUIC_CID_ENCRYPTION_KEYLEN> cid_encryption_key;
+  // Identifier of this keying material.  Only the first 2 bits are
+  // used.
+  uint8_t id;
+};
+
+struct QUICKeyingMaterials {
+  std::vector<QUICKeyingMaterial> keying_materials;
 };
 #endif // ENABLE_HTTP3
 
@@ -765,8 +773,8 @@ struct QUICConfig {
     ngtcp2_cc_algo congestion_controller;
     bool early_data;
     bool require_token;
-    std::array<uint8_t, SHRPX_QUIC_CID_ENCRYPTION_KEYLEN> cid_encryption_key;
     std::array<uint8_t, SHRPX_QUIC_SERVER_IDLEN> server_id;
+    StringRef secret_file;
   } upstream;
   struct {
     StringRef prog_file;
@@ -1220,12 +1228,12 @@ enum {
   SHRPX_OPTID_FRONTEND_MAX_REQUESTS,
   SHRPX_OPTID_FRONTEND_NO_TLS,
   SHRPX_OPTID_FRONTEND_QUIC_CONGESTION_CONTROLLER,
-  SHRPX_OPTID_FRONTEND_QUIC_CONNECTION_ID_ENCRYPTION_KEY,
   SHRPX_OPTID_FRONTEND_QUIC_DEBUG_LOG,
   SHRPX_OPTID_FRONTEND_QUIC_EARLY_DATA,
   SHRPX_OPTID_FRONTEND_QUIC_IDLE_TIMEOUT,
   SHRPX_OPTID_FRONTEND_QUIC_QLOG_DIR,
   SHRPX_OPTID_FRONTEND_QUIC_REQUIRE_TOKEN,
+  SHRPX_OPTID_FRONTEND_QUIC_SECRET_FILE,
   SHRPX_OPTID_FRONTEND_QUIC_SERVER_ID,
   SHRPX_OPTID_FRONTEND_READ_TIMEOUT,
   SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT,
@@ -1377,6 +1385,11 @@ std::unique_ptr<TicketKeys>
 read_tls_ticket_key_file(const std::vector<StringRef> &files,
                          const EVP_CIPHER *cipher, const EVP_MD *hmac);
 
+#ifdef ENABLE_HTTP3
+std::shared_ptr<QUICKeyingMaterials>
+read_quic_secret_file(const StringRef &path);
+#endif // ENABLE_HTTP3
+
 // Returns string representation of |proto|.
 StringRef strproto(Proto proto);
 

src/shrpx_connection_handler.cc

@@ -298,8 +298,6 @@ int ConnectionHandler::create_single_worker() {
 #endif // HAVE_MRUBY
 
 #ifdef ENABLE_HTTP3
-  single_worker_->set_quic_secret(quic_secret_);
-
   if (single_worker_->setup_quic_server_socket() != 0) {
     return -1;
   }
@@ -404,8 +402,6 @@ int ConnectionHandler::create_worker_thread(size_t num) {
 #  endif // HAVE_MRUBY
 
 #  ifdef ENABLE_HTTP3
-    worker->set_quic_secret(quic_secret_);
-
     if ((!apiconf.enabled || i != 0) &&
         worker->setup_quic_server_socket() != 0) {
       return -1;
@@ -1047,25 +1043,14 @@ int ConnectionHandler::forward_quic_packet(const UpstreamAddr *faddr,
   return -1;
 }
 
-int ConnectionHandler::create_quic_secret() {
+void ConnectionHandler::set_quic_keying_materials(
-  auto quic_secret = std::make_shared<QUICSecret>();
+    std::shared_ptr<QUICKeyingMaterials> qkms) {
+  quic_keying_materials_ = std::move(qkms);
+}
 
-  if (generate_quic_stateless_reset_secret(
+const std::shared_ptr<QUICKeyingMaterials> &
-          quic_secret->stateless_reset_secret.data()) != 0) {
+ConnectionHandler::get_quic_keying_materials() const {
-    LOG(ERROR) << "Failed to generate QUIC Stateless Reset secret";
+  return quic_keying_materials_;
-
-    return -1;
-  }
-
-  if (generate_quic_token_secret(quic_secret->token_secret.data()) != 0) {
-    LOG(ERROR) << "Failed to generate QUIC token secret";
-
-    return -1;
-  }
-
-  quic_secret_ = std::move(quic_secret);
-
-  return 0;
 }
 
 void ConnectionHandler::set_cid_prefixes(
@@ -1287,14 +1272,13 @@ int ConnectionHandler::quic_ipc_read() {
     return 0;
   }
 
-  auto config = get_config();
+  auto &qkm = quic_keying_materials_->keying_materials.front();
-  auto &quicconf = config->quic;
 
   std::array<uint8_t, SHRPX_QUIC_DECRYPTED_DCIDLEN> decrypted_dcid;
 
-  if (decrypt_quic_connection_id(
+  if (decrypt_quic_connection_id(decrypted_dcid.data(),
-          decrypted_dcid.data(), dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
+                                 dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
-          quicconf.upstream.cid_encryption_key.data()) != 0) {
+                                 qkm.cid_encryption_key.data()) != 0) {
     return -1;
   }
 

src/shrpx_connection_handler.h

@@ -198,7 +198,8 @@ public:
                           const Address &local_addr, const uint8_t *cid_prefix,
                           const uint8_t *data, size_t datalen);
 
-  int create_quic_secret();
+  void set_quic_keying_materials(std::shared_ptr<QUICKeyingMaterials> qkms);
+  const std::shared_ptr<QUICKeyingMaterials> &get_quic_keying_materials() const;
 
   void set_cid_prefixes(
       const std::vector<std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN>>
@@ -263,7 +264,7 @@ private:
 #  ifdef HAVE_LIBBPF
   std::vector<BPFRef> quic_bpf_refs_;
 #  endif // HAVE_LIBBPF
-  std::shared_ptr<QUICSecret> quic_secret_;
+  std::shared_ptr<QUICKeyingMaterials> quic_keying_materials_;
   std::vector<SSL_CTX *> quic_all_ssl_ctx_;
   std::vector<std::vector<SSL_CTX *>> quic_indexed_ssl_ctx_;
 #endif // ENABLE_HTTP3

src/shrpx_http3_upstream.cc

@@ -40,6 +40,7 @@
 #include "shrpx_quic.h"
 #include "shrpx_worker.h"
 #include "shrpx_http.h"
+#include "shrpx_connection_handler.h"
 #ifdef HAVE_MRUBY
 #  include "shrpx_mruby.h"
 #endif // HAVE_MRUBY
@@ -217,21 +218,17 @@ int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token,
   auto upstream = static_cast<Http3Upstream *>(user_data);
   auto handler = upstream->get_client_handler();
   auto worker = handler->get_worker();
+  auto conn_handler = worker->get_connection_handler();
+  auto &qkms = conn_handler->get_quic_keying_materials();
+  auto &qkm = qkms->keying_materials.front();
 
-  auto config = get_config();
+  if (generate_quic_connection_id(*cid, cidlen, worker->get_cid_prefix(),
-  auto &quicconf = config->quic;
+                                  qkm.id, qkm.cid_encryption_key.data()) != 0) {
-
-  if (generate_quic_connection_id(
-          *cid, cidlen, worker->get_cid_prefix(),
-          quicconf.upstream.cid_encryption_key.data()) != 0) {
     return NGTCP2_ERR_CALLBACK_FAILURE;
   }
 
-  auto &quic_secret = worker->get_quic_secret();
+  if (generate_quic_stateless_reset_token(token, *cid, qkm.secret.data(),
-  auto &secret = quic_secret->stateless_reset_secret;
+                                          qkm.secret.size()) != 0) {
-
-  if (generate_quic_stateless_reset_token(token, *cid, secret.data(),
-                                          secret.size()) != 0) {
     return NGTCP2_ERR_CALLBACK_FAILURE;
   }
 
@@ -487,11 +484,13 @@ int Http3Upstream::handshake_completed() {
 
   auto path = ngtcp2_conn_get_path(conn_);
   auto worker = handler_->get_worker();
-  auto &quic_secret = worker->get_quic_secret();
+  auto conn_handler = worker->get_connection_handler();
-  auto &secret = quic_secret->token_secret;
+  auto &qkms = conn_handler->get_quic_keying_materials();
+  auto &qkm = qkms->keying_materials.front();
 
   if (generate_token(token.data(), tokenlen, path->remote.addr,
-                     path->remote.addrlen, secret.data()) != 0) {
+                     path->remote.addrlen, qkm.secret.data(),
+                     qkm.secret.size()) != 0) {
     return 0;
   }
 
@@ -513,6 +512,7 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
   int rv;
 
   auto worker = handler_->get_worker();
+  auto conn_handler = worker->get_connection_handler();
 
   auto callbacks = ngtcp2_callbacks{
       nullptr, // client_initial
@@ -557,11 +557,14 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
   auto &quicconf = config->quic;
   auto &http3conf = config->http3;
 
+  auto &qkms = conn_handler->get_quic_keying_materials();
+  auto &qkm = qkms->keying_materials.front();
+
   ngtcp2_cid scid;
 
-  if (generate_quic_connection_id(
+  if (generate_quic_connection_id(scid, SHRPX_QUIC_SCIDLEN,
-          scid, SHRPX_QUIC_SCIDLEN, worker->get_cid_prefix(),
+                                  worker->get_cid_prefix(), qkm.id,
-          quicconf.upstream.cid_encryption_key.data()) != 0) {
+                                  qkm.cid_encryption_key.data()) != 0) {
     return -1;
   }
 
@@ -608,12 +611,8 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
     params.original_dcid = initial_hd.dcid;
   }
 
-  auto &quic_secret = worker->get_quic_secret();
+  rv = generate_quic_stateless_reset_token(
-  auto &stateless_reset_secret = quic_secret->stateless_reset_secret;
+      params.stateless_reset_token, scid, qkm.secret.data(), qkm.secret.size());
-
-  rv = generate_quic_stateless_reset_token(params.stateless_reset_token, scid,
-                                           stateless_reset_secret.data(),
-                                           stateless_reset_secret.size());
   if (rv != 0) {
     ULOG(ERROR, this) << "generate_quic_stateless_reset_token failed";
     return -1;

src/shrpx_quic.cc

@@ -144,7 +144,7 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
 }
 
 int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen,
-                                      const uint8_t *server_id,
+                                      const uint8_t *server_id, uint8_t km_id,
                                       const uint8_t *key) {
   assert(cidlen == SHRPX_QUIC_SCIDLEN);
 
@@ -154,6 +154,8 @@ int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen,
 
   cid.datalen = cidlen;
 
+  cid.data[0] = (cid.data[0] & 0x3f) | km_id;
+
   auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET;
 
   std::copy_n(server_id, SHRPX_QUIC_SERVER_IDLEN, p);
@@ -162,7 +164,8 @@ int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen,
 }
 
 int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen,
-                                const uint8_t *cid_prefix, const uint8_t *key) {
+                                const uint8_t *cid_prefix, uint8_t km_id,
+                                const uint8_t *key) {
   assert(cidlen == SHRPX_QUIC_SCIDLEN);
 
   if (RAND_bytes(cid.data, cidlen) != 1) {
@@ -171,6 +174,8 @@ int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen,
 
   cid.datalen = cidlen;
 
+  cid.data[0] = (cid.data[0] & 0x3f) | km_id;
+
   auto p = cid.data + SHRPX_QUIC_CID_PREFIX_OFFSET;
 
   std::copy_n(cid_prefix, SHRPX_QUIC_CID_PREFIXLEN, p);
@@ -257,32 +262,16 @@ int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid &cid,
   return 0;
 }
 
-int generate_quic_stateless_reset_secret(uint8_t *secret) {
-  if (RAND_bytes(secret, SHRPX_QUIC_STATELESS_RESET_SECRETLEN) != 1) {
-    return -1;
-  }
-
-  return 0;
-}
-
-int generate_quic_token_secret(uint8_t *secret) {
-  if (RAND_bytes(secret, SHRPX_QUIC_TOKEN_SECRETLEN) != 1) {
-    return -1;
-  }
-
-  return 0;
-}
-
 int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
                          socklen_t salen, const ngtcp2_cid &retry_scid,
-                         const ngtcp2_cid &odcid, const uint8_t *token_secret) {
+                         const ngtcp2_cid &odcid, const uint8_t *secret,
+                         size_t secretlen) {
   auto t = std::chrono::duration_cast<std::chrono::nanoseconds>(
                std::chrono::system_clock::now().time_since_epoch())
                .count();
 
   auto stokenlen = ngtcp2_crypto_generate_retry_token(
-      token, token_secret, SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, &retry_scid,
+      token, secret, secretlen, sa, salen, &retry_scid, &odcid, t);
-      &odcid, t);
   if (stokenlen < 0) {
     return -1;
   }
@@ -294,15 +283,16 @@ int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
 
 int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen,
                        const ngtcp2_cid &dcid, const sockaddr *sa,
-                       socklen_t salen, const uint8_t *token_secret) {
+                       socklen_t salen, const uint8_t *secret,
+                       size_t secretlen) {
 
   auto t = std::chrono::duration_cast<std::chrono::nanoseconds>(
                std::chrono::system_clock::now().time_since_epoch())
                .count();
 
-  if (ngtcp2_crypto_verify_retry_token(&odcid, token, tokenlen, token_secret,
+  if (ngtcp2_crypto_verify_retry_token(&odcid, token, tokenlen, secret,
-                                       SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen,
+                                       secretlen, sa, salen, &dcid,
-                                       &dcid, 10 * NGTCP2_SECONDS, t) != 0) {
+                                       10 * NGTCP2_SECONDS, t) != 0) {
     return -1;
   }
 
@@ -310,13 +300,13 @@ int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen,
 }
 
 int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
-                   size_t salen, const uint8_t *token_secret) {
+                   size_t salen, const uint8_t *secret, size_t secretlen) {
   auto t = std::chrono::duration_cast<std::chrono::nanoseconds>(
                std::chrono::system_clock::now().time_since_epoch())
                .count();
 
   auto stokenlen = ngtcp2_crypto_generate_regular_token(
-      token, token_secret, SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen, t);
+      token, secret, secretlen, sa, salen, t);
   if (stokenlen < 0) {
     return -1;
   }
@@ -327,18 +317,48 @@ int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
 }
 
 int verify_token(const uint8_t *token, size_t tokenlen, const sockaddr *sa,
-                 socklen_t salen, const uint8_t *token_secret) {
+                 socklen_t salen, const uint8_t *secret, size_t secretlen) {
   auto t = std::chrono::duration_cast<std::chrono::nanoseconds>(
                std::chrono::system_clock::now().time_since_epoch())
                .count();
 
-  if (ngtcp2_crypto_verify_regular_token(token, tokenlen, token_secret,
+  if (ngtcp2_crypto_verify_regular_token(token, tokenlen, secret, secretlen, sa,
-                                         SHRPX_QUIC_TOKEN_SECRETLEN, sa, salen,
+                                         salen, 3600 * NGTCP2_SECONDS,
-                                         3600 * NGTCP2_SECONDS, t) != 0) {
+                                         t) != 0) {
     return -1;
   }
 
   return 0;
 }
 
+int generate_quic_connection_id_encryption_key(uint8_t *key, size_t keylen,
+                                               const uint8_t *secret,
+                                               size_t secretlen,
+                                               const uint8_t *salt,
+                                               size_t saltlen) {
+  constexpr uint8_t info[] = "connection id encryption key";
+  ngtcp2_crypto_md sha256;
+  ngtcp2_crypto_md_init(
+      &sha256, reinterpret_cast<void *>(const_cast<EVP_MD *>(EVP_sha256())));
+
+  if (ngtcp2_crypto_hkdf(key, keylen, &sha256, secret, secretlen, salt, saltlen,
+                         info, str_size(info)) != 0) {
+    return -1;
+  }
+
+  return 0;
+}
+
+const QUICKeyingMaterial *
+select_quic_keying_material(const QUICKeyingMaterials &qkms,
+                            const uint8_t *cid) {
+  for (auto &qkm : qkms.keying_materials) {
+    if (((*cid) & 0xc0) == qkm.id) {
+      return &qkm;
+    }
+  }
+
+  return &qkms.keying_materials.front();
+}
+
 } // namespace shrpx

src/shrpx_quic.h

@@ -60,6 +60,8 @@ bool operator==(const ngtcp2_cid &lhs, const ngtcp2_cid &rhs);
 namespace shrpx {
 
 struct UpstreamAddr;
+struct QUICKeyingMaterials;
+struct QUICKeyingMaterial;
 
 constexpr size_t SHRPX_QUIC_SCIDLEN = 20;
 constexpr size_t SHRPX_QUIC_SERVER_IDLEN = 2;
@@ -69,10 +71,11 @@ constexpr size_t SHRPX_QUIC_CID_PREFIX_OFFSET = 1;
 constexpr size_t SHRPX_QUIC_DECRYPTED_DCIDLEN = 16;
 constexpr size_t SHRPX_QUIC_CID_ENCRYPTION_KEYLEN = 16;
 constexpr size_t SHRPX_QUIC_MAX_UDP_PAYLOAD_SIZE = 1472;
-constexpr size_t SHRPX_QUIC_STATELESS_RESET_SECRETLEN = 32;
-constexpr size_t SHRPX_QUIC_TOKEN_SECRETLEN = 32;
 constexpr size_t SHRPX_QUIC_CONN_CLOSE_PKTLEN = 256;
 constexpr size_t SHRPX_QUIC_STATELESS_RESET_BURST = 100;
+constexpr size_t SHRPX_QUIC_SECRET_RESERVEDLEN = 4;
+constexpr size_t SHRPX_QUIC_SECRETLEN = 32;
+constexpr size_t SHRPX_QUIC_SALTLEN = 32;
 
 ngtcp2_tstamp quic_timestamp();
 
@@ -82,11 +85,12 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
                      size_t gso_size);
 
 int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen,
-                                      const uint8_t *server_id,
+                                      const uint8_t *server_id, uint8_t km_id,
                                       const uint8_t *key);
 
 int generate_quic_connection_id(ngtcp2_cid &cid, size_t cidlen,
-                                const uint8_t *cid_prefix, const uint8_t *key);
+                                const uint8_t *cid_prefix, uint8_t km_id,
+                                const uint8_t *key);
 
 int encrypt_quic_connection_id(uint8_t *dest, const uint8_t *src,
                                const uint8_t *key);
@@ -103,23 +107,31 @@ int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid &cid,
                                         const uint8_t *secret,
                                         size_t secretlen);
 
-int generate_quic_stateless_reset_secret(uint8_t *secret);
-
-int generate_quic_token_secret(uint8_t *secret);
-
 int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
                          socklen_t salen, const ngtcp2_cid &retry_scid,
-                         const ngtcp2_cid &odcid, const uint8_t *token_secret);
+                         const ngtcp2_cid &odcid, const uint8_t *secret,
+                         size_t secretlen);
 
 int verify_retry_token(ngtcp2_cid &odcid, const uint8_t *token, size_t tokenlen,
                        const ngtcp2_cid &dcid, const sockaddr *sa,
-                       socklen_t salen, const uint8_t *token_secret);
+                       socklen_t salen, const uint8_t *secret,
+                       size_t secretlen);
 
 int generate_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
-                   size_t salen, const uint8_t *token_secret);
+                   size_t salen, const uint8_t *secret, size_t secretlen);
 
 int verify_token(const uint8_t *token, size_t tokenlen, const sockaddr *sa,
-                 socklen_t salen, const uint8_t *token_secret);
+                 socklen_t salen, const uint8_t *secret, size_t secretlen);
+
+int generate_quic_connection_id_encryption_key(uint8_t *key, size_t keylen,
+                                               const uint8_t *secret,
+                                               size_t secretlen,
+                                               const uint8_t *salt,
+                                               size_t saltlen);
+
+const QUICKeyingMaterial *
+select_quic_keying_material(const QUICKeyingMaterials &qkms,
+                            const uint8_t *cid);
 
 } // namespace shrpx
 

src/shrpx_quic_connection_handler.cc

@@ -126,14 +126,20 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
   if (it == std::end(connections_)) {
     std::array<uint8_t, SHRPX_QUIC_DECRYPTED_DCIDLEN> decrypted_dcid;
 
+    auto &qkms = conn_handler->get_quic_keying_materials();
+    const QUICKeyingMaterial *qkm = nullptr;
+
     if (dcidlen == SHRPX_QUIC_SCIDLEN) {
-      if (decrypt_quic_connection_id(
+      qkm = select_quic_keying_material(*qkms.get(), dcid);
-              decrypted_dcid.data(), dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
+
-              quicconf.upstream.cid_encryption_key.data()) != 0) {
+      if (decrypt_quic_connection_id(decrypted_dcid.data(),
+                                     dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
+                                     qkm->cid_encryption_key.data()) != 0) {
         return 0;
       }
 
-      if (!std::equal(std::begin(decrypted_dcid),
+      if (qkm != &qkms->keying_materials.front() ||
+          !std::equal(std::begin(decrypted_dcid),
                       std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
                       worker_->get_cid_prefix())) {
         auto quic_lwp =
@@ -170,14 +176,25 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
 
     switch (ngtcp2_accept(&hd, data, datalen)) {
     case 0: {
-      // If we get Initial and it has the CID prefix of this worker, it
+      // If we get Initial and it has the CID prefix of this worker,
-      // is likely that client is intentionally use the our prefix.
+      // it is likely that client is intentionally use the prefix.
       // Just drop it.
-      if (dcidlen == SHRPX_QUIC_SCIDLEN &&
+      if (dcidlen == SHRPX_QUIC_SCIDLEN) {
-          std::equal(std::begin(decrypted_dcid),
+        if (qkm != &qkms->keying_materials.front()) {
-                     std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
+          qkm = &qkms->keying_materials.front();
-                     worker_->get_cid_prefix())) {
+
-        return 0;
+          if (decrypt_quic_connection_id(decrypted_dcid.data(),
+                                         dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
+                                         qkm->cid_encryption_key.data()) != 0) {
+            return 0;
+          }
+        }
+
+        if (std::equal(std::begin(decrypted_dcid),
+                       std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
+                       worker_->get_cid_prefix())) {
+          return 0;
+        }
       }
 
       if (worker_->get_graceful_shutdown()) {
@@ -197,14 +214,18 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
         break;
       }
 
-      auto &quic_secret = worker_->get_quic_secret();
+      if (dcidlen != SHRPX_QUIC_SCIDLEN) {
-      auto &secret = quic_secret->token_secret;
+        // Initial packets with token must have DCID chosen by server.
+        return 0;
+      }
+
+      auto qkm = select_quic_keying_material(*qkms.get(), dcid);
 
       switch (hd.token.base[0]) {
       case NGTCP2_CRYPTO_TOKEN_MAGIC_RETRY:
         if (verify_retry_token(odcid, hd.token.base, hd.token.len, hd.dcid,
                                &remote_addr.su.sa, remote_addr.len,
-                               secret.data()) != 0) {
+                               qkm->secret.data(), qkm->secret.size()) != 0) {
           if (LOG_ENABLED(INFO)) {
             LOG(INFO) << "Failed to validate Retry token from remote="
                       << util::to_numeric_addr(&remote_addr);
@@ -229,7 +250,8 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
         break;
       case NGTCP2_CRYPTO_TOKEN_MAGIC_REGULAR:
         if (verify_token(hd.token.base, hd.token.len, &remote_addr.su.sa,
-                         remote_addr.len, secret.data()) != 0) {
+                         remote_addr.len, qkm->secret.data(),
+                         qkm->secret.size()) != 0) {
           if (LOG_ENABLED(INFO)) {
             LOG(INFO) << "Failed to validate token from remote="
                       << util::to_numeric_addr(&remote_addr);
@@ -422,11 +444,15 @@ int QUICConnectionHandler::send_retry(
   auto config = get_config();
   auto &quicconf = config->quic;
 
+  auto conn_handler = worker_->get_connection_handler();
+  auto &qkms = conn_handler->get_quic_keying_materials();
+  auto &qkm = qkms->keying_materials.front();
+
   ngtcp2_cid retry_scid;
 
   if (generate_quic_retry_connection_id(
           retry_scid, SHRPX_QUIC_SCIDLEN, quicconf.upstream.server_id.data(),
-          quicconf.upstream.cid_encryption_key.data()) != 0) {
+          qkm.id, qkm.cid_encryption_key.data()) != 0) {
     return -1;
   }
 
@@ -437,12 +463,9 @@ int QUICConnectionHandler::send_retry(
   ngtcp2_cid_init(&idcid, ini_dcid, ini_dcidlen);
   ngtcp2_cid_init(&iscid, ini_scid, ini_scidlen);
 
-  auto &quic_secret = worker_->get_quic_secret();
-  auto &secret = quic_secret->token_secret;
-
   if (generate_retry_token(token.data(), tokenlen, &remote_addr.su.sa,
                            remote_addr.len, retry_scid, idcid,
-                           secret.data()) != 0) {
+                           qkm.secret.data(), qkm.secret.size()) != 0) {
     return -1;
   }
 
@@ -539,11 +562,12 @@ int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr,
 
   ngtcp2_cid_init(&cid, dcid, dcidlen);
 
-  auto &quic_secret = worker_->get_quic_secret();
+  auto conn_handler = worker_->get_connection_handler();
-  auto &secret = quic_secret->stateless_reset_secret;
+  auto &qkms = conn_handler->get_quic_keying_materials();
+  auto &qkm = qkms->keying_materials.front();
 
-  rv = generate_quic_stateless_reset_token(token.data(), cid, secret.data(),
+  rv = generate_quic_stateless_reset_token(token.data(), cid, qkm.secret.data(),
-                                           secret.size());
+                                           qkm.secret.size());
   if (rv != 0) {
     return -1;
   }

src/shrpx_worker.cc

@@ -933,14 +933,14 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) {
         return -1;
       }
 
-      auto &quicconf = config->quic;
-
       constexpr uint32_t key_high_idx = 1;
       constexpr uint32_t key_low_idx = 2;
 
+      auto &qkms = conn_handler_->get_quic_keying_materials();
+      auto &qkm = qkms->keying_materials.front();
+
       if (bpf_map_update_elem(bpf_map__fd(sk_info), &key_high_idx,
-                              quicconf.upstream.cid_encryption_key.data(),
+                              qkm.cid_encryption_key.data(), BPF_ANY) != 0) {
-                              BPF_ANY) != 0) {
         LOG(FATAL) << "Failed to update key_high_idx sk_info: "
                    << xsi_strerror(errno, errbuf.data(), errbuf.size());
         close(fd);
@@ -948,7 +948,7 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) {
       }
 
       if (bpf_map_update_elem(bpf_map__fd(sk_info), &key_low_idx,
-                              quicconf.upstream.cid_encryption_key.data() + 8,
+                              qkm.cid_encryption_key.data() + 8,
                               BPF_ANY) != 0) {
         LOG(FATAL) << "Failed to update key_low_idx sk_info: "
                    << xsi_strerror(errno, errbuf.data(), errbuf.size());
@@ -1010,14 +1010,6 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) {
 
 const uint8_t *Worker::get_cid_prefix() const { return cid_prefix_.data(); }
 
-void Worker::set_quic_secret(const std::shared_ptr<QUICSecret> &secret) {
-  quic_secret_ = secret;
-}
-
-const std::shared_ptr<QUICSecret> &Worker::get_quic_secret() const {
-  return quic_secret_;
-}
-
 const UpstreamAddr *Worker::find_quic_upstream_addr(const Address &local_addr) {
   std::array<char, NI_MAXHOST> host;
 

src/shrpx_worker.h

@@ -370,10 +370,6 @@ public:
 
   const uint8_t *get_cid_prefix() const;
 
-  void set_quic_secret(const std::shared_ptr<QUICSecret> &secret);
-
-  const std::shared_ptr<QUICSecret> &get_quic_secret() const;
-
 #  ifdef HAVE_LIBBPF
   bool should_attach_bpf() const;
 
@@ -412,7 +408,6 @@ private:
   std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN> cid_prefix_;
   std::vector<UpstreamAddr> quic_upstream_addrs_;
   std::vector<std::unique_ptr<QUICListener>> quic_listeners_;
-  std::shared_ptr<QUICSecret> quic_secret_;
 #endif // ENABLE_HTTP3
 
   std::shared_ptr<DownstreamConfig> downstreamconf_;

src/shrpx_worker_process.cc

@@ -519,10 +519,51 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) {
   }
 
 #ifdef ENABLE_HTTP3
-  if (conn_handler->create_quic_secret() != 0) {
+  auto &quicconf = config->quic;
-    return -1;
+
+  std::shared_ptr<QUICKeyingMaterials> qkms;
+
+  if (!quicconf.upstream.secret_file.empty()) {
+    qkms = read_quic_secret_file(quicconf.upstream.secret_file);
+    if (!qkms) {
+      LOG(WARN) << "Use QUIC keying materials generated internally";
+    }
   }
 
+  if (!qkms) {
+    qkms = std::make_shared<QUICKeyingMaterials>();
+    qkms->keying_materials.resize(1);
+
+    auto &qkm = qkms->keying_materials.front();
+
+    if (RAND_bytes(qkm.reserved.data(), qkm.reserved.size()) != 1) {
+      LOG(ERROR) << "Failed to generate QUIC secret reserved data";
+      return -1;
+    }
+
+    if (RAND_bytes(qkm.secret.data(), qkm.secret.size()) != 1) {
+      LOG(ERROR) << "Failed to generate QUIC secret";
+      return -1;
+    }
+
+    if (RAND_bytes(qkm.salt.data(), qkm.salt.size()) != 1) {
+      LOG(ERROR) << "Failed to generate QUIC salt";
+      return -1;
+    }
+  }
+
+  for (auto &qkm : qkms->keying_materials) {
+    if (generate_quic_connection_id_encryption_key(
+            qkm.cid_encryption_key.data(), qkm.cid_encryption_key.size(),
+            qkm.secret.data(), qkm.secret.size(), qkm.salt.data(),
+            qkm.salt.size()) != 0) {
+      LOG(ERROR) << "Failed to generate QUIC Connection ID encryption key";
+      return -1;
+    }
+  }
+
+  conn_handler->set_quic_keying_materials(std::move(qkms));
+
   conn_handler->set_cid_prefixes(wpconf->cid_prefixes);
   conn_handler->set_quic_lingering_worker_processes(
       wpconf->quic_lingering_worker_processes);