nghttpx: Send Retry packet

src/shrpx_http3_upstream.cc

@@ -403,7 +403,9 @@ int stream_stop_sending(ngtcp2_conn *conn, int64_t stream_id,
 
 int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
                         const Address &local_addr,
-                        const ngtcp2_pkt_hd &initial_hd) {
+                        const ngtcp2_pkt_hd &initial_hd,
+                        const ngtcp2_cid *odcid, const uint8_t *token,
+                        size_t tokenlen) {
   int rv;
 
   auto worker = handler_->get_worker();
@@ -470,6 +472,7 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
   settings.max_stream_window = 6_m;
   settings.max_udp_payload_size = SHRPX_MAX_UDP_PAYLOAD_SIZE;
   settings.rand_ctx.native_handle = &worker->get_randgen();
+  settings.token = ngtcp2_vec{const_cast<uint8_t *>(token), tokenlen};
 
   ngtcp2_transport_params params;
   ngtcp2_transport_params_default(&params);
@@ -480,7 +483,14 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
   params.initial_max_stream_data_uni = 256_k;
   params.max_idle_timeout =
       static_cast<ngtcp2_tstamp>(quicconf.timeout.idle * NGTCP2_SECONDS);
-  params.original_dcid = initial_hd.dcid;
+
+  if (odcid) {
+    params.original_dcid = *odcid;
+    params.retry_scid = initial_hd.dcid;
+    params.retry_scid_present = 1;
+  } else {
+    params.original_dcid = initial_hd.dcid;
+  }
 
   auto &quic_secret = worker->get_quic_secret();
   auto &stateless_reset_secret = quic_secret->stateless_reset_secret;

src/shrpx_http3_upstream.h

@@ -88,7 +88,8 @@ public:
   virtual void cancel_premature_downstream(Downstream *promised_downstream);
 
   int init(const UpstreamAddr *faddr, const Address &remote_addr,
-           const Address &local_addr, const ngtcp2_pkt_hd &initial_hd);
+           const Address &local_addr, const ngtcp2_pkt_hd &initial_hd,
+           const ngtcp2_cid *odcid, const uint8_t *token, size_t tokenlen);
 
   int on_read(const UpstreamAddr *faddr, const Address &remote_addr,
               const Address &local_addr, const uint8_t *data, size_t datalen);

src/shrpx_quic.cc

@@ -181,11 +181,226 @@ int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid,
 }
 
 int generate_quic_stateless_reset_secret(uint8_t *secret) {
-  return RAND_bytes(secret, SHRPX_QUIC_STATELESS_RESET_SECRETLEN) == 1;
+  if (RAND_bytes(secret, SHRPX_QUIC_STATELESS_RESET_SECRETLEN) != 1) {
+    return -1;
+  }
+
+  return 0;
 }
 
 int generate_quic_token_secret(uint8_t *secret) {
-  return RAND_bytes(secret, SHRPX_QUIC_TOKEN_SECRETLEN) == 1;
+  if (RAND_bytes(secret, SHRPX_QUIC_TOKEN_SECRETLEN) != 1) {
+    return -1;
+  }
+
+  return 0;
+}
+
+namespace {
+int derive_token_key(uint8_t *key, size_t &keylen, uint8_t *iv, size_t &ivlen,
+                     const uint8_t *token_secret, const uint8_t *rand_data,
+                     size_t rand_datalen, const ngtcp2_crypto_aead *aead,
+                     const ngtcp2_crypto_md *md) {
+  std::array<uint8_t, 32> secret;
+
+  if (ngtcp2_crypto_hkdf_extract(secret.data(), md, token_secret,
+                                 SHRPX_QUIC_TOKEN_SECRETLEN, rand_data,
+                                 rand_datalen) != 0) {
+    return -1;
+  }
+
+  auto aead_keylen = ngtcp2_crypto_aead_keylen(aead);
+  if (keylen < aead_keylen) {
+    return -1;
+  }
+
+  keylen = aead_keylen;
+
+  auto aead_ivlen = ngtcp2_crypto_packet_protection_ivlen(aead);
+  if (ivlen < aead_ivlen) {
+    return -1;
+  }
+
+  ivlen = aead_ivlen;
+
+  if (ngtcp2_crypto_derive_packet_protection_key(
+          key, iv, nullptr, aead, md, secret.data(), secret.size()) != 0) {
+    return -1;
+  }
+
+  return 0;
+}
+} // namespace
+
+namespace {
+size_t generate_retry_token_aad(uint8_t *dest, size_t destlen,
+                                const sockaddr *sa, socklen_t salen,
+                                const ngtcp2_cid *retry_scid) {
+  assert(destlen >= salen + retry_scid->datalen);
+
+  auto p = std::copy_n(reinterpret_cast<const uint8_t *>(sa), salen, dest);
+  p = std::copy_n(retry_scid->data, retry_scid->datalen, p);
+
+  return p - dest;
+}
+} // namespace
+
+int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
+                         socklen_t salen, const ngtcp2_cid *retry_scid,
+                         const ngtcp2_cid *odcid, const uint8_t *token_secret) {
+  std::array<uint8_t, 4096> plaintext;
+
+  uint64_t t = std::chrono::duration_cast<std::chrono::nanoseconds>(
+                   std::chrono::system_clock::now().time_since_epoch())
+                   .count();
+
+  auto p = std::begin(plaintext);
+  // Host byte order
+  p = std::copy_n(reinterpret_cast<uint8_t *>(&t), sizeof(t), p);
+  p = std::copy_n(odcid->data, odcid->datalen, p);
+
+  std::array<uint8_t, SHRPX_QUIC_TOKEN_RAND_DATALEN> rand_data;
+  std::array<uint8_t, 32> key, iv;
+  auto keylen = key.size();
+  auto ivlen = iv.size();
+
+  if (RAND_bytes(rand_data.data(), rand_data.size()) != 1) {
+    return -1;
+  }
+
+  ngtcp2_crypto_aead aead;
+  ngtcp2_crypto_aead_init(&aead, const_cast<EVP_CIPHER *>(EVP_aes_128_gcm()));
+
+  ngtcp2_crypto_md md;
+  ngtcp2_crypto_md_init(&md, const_cast<EVP_MD *>(EVP_sha256()));
+
+  if (derive_token_key(key.data(), keylen, iv.data(), ivlen, token_secret,
+                       rand_data.data(), rand_data.size(), &aead, &md) != 0) {
+    return -1;
+  }
+
+  auto plaintextlen = std::distance(std::begin(plaintext), p);
+
+  std::array<uint8_t, 256> aad;
+  auto aadlen =
+      generate_retry_token_aad(aad.data(), aad.size(), sa, salen, retry_scid);
+
+  token[0] = SHRPX_QUIC_RETRY_TOKEN_MAGIC;
+
+  ngtcp2_crypto_aead_ctx aead_ctx;
+  if (ngtcp2_crypto_aead_ctx_encrypt_init(&aead_ctx, &aead, key.data(),
+                                          ivlen) != 0) {
+    return -1;
+  }
+
+  auto rv =
+      ngtcp2_crypto_encrypt(token + 1, &aead, &aead_ctx, plaintext.data(),
+                            plaintextlen, iv.data(), ivlen, aad.data(), aadlen);
+
+  ngtcp2_crypto_aead_ctx_free(&aead_ctx);
+
+  if (rv != 0) {
+    return -1;
+  }
+
+  /* 1 for magic byte */
+  tokenlen = 1 + plaintextlen + aead.max_overhead;
+  memcpy(token + tokenlen, rand_data.data(), rand_data.size());
+  tokenlen += rand_data.size();
+
+  return 0;
+}
+
+int verify_retry_token(ngtcp2_cid *odcid, const uint8_t *token, size_t tokenlen,
+                       const ngtcp2_cid *dcid, const sockaddr *sa,
+                       socklen_t salen, const uint8_t *token_secret) {
+  std::array<char, NI_MAXHOST> host;
+  std::array<char, NI_MAXSERV> port;
+
+  if (getnameinfo(sa, salen, host.data(), host.size(), port.data(), port.size(),
+                  NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
+    return -1;
+  }
+
+  /* 1 for SHRPX_QUIC_RETRY_TOKEN_MAGIC */
+  if (tokenlen < SHRPX_QUIC_TOKEN_RAND_DATALEN + 1) {
+    return -1;
+  }
+  if (tokenlen > SHRPX_QUIC_MAX_RETRY_TOKENLEN) {
+    return -1;
+  }
+
+  assert(token[0] == SHRPX_QUIC_RETRY_TOKEN_MAGIC);
+
+  auto rand_data = token + tokenlen - SHRPX_QUIC_TOKEN_RAND_DATALEN;
+  auto ciphertext = token + 1;
+  auto ciphertextlen = tokenlen - SHRPX_QUIC_TOKEN_RAND_DATALEN - 1;
+
+  std::array<uint8_t, 32> key, iv;
+  auto keylen = key.size();
+  auto ivlen = iv.size();
+
+  ngtcp2_crypto_aead aead;
+  ngtcp2_crypto_aead_init(&aead, const_cast<EVP_CIPHER *>(EVP_aes_128_gcm()));
+
+  ngtcp2_crypto_md md;
+  ngtcp2_crypto_md_init(&md, const_cast<EVP_MD *>(EVP_sha256()));
+
+  if (derive_token_key(key.data(), keylen, iv.data(), ivlen, token_secret,
+                       rand_data, SHRPX_QUIC_TOKEN_RAND_DATALEN, &aead,
+                       &md) != 0) {
+    return -1;
+  }
+
+  std::array<uint8_t, 256> aad;
+  auto aadlen =
+      generate_retry_token_aad(aad.data(), aad.size(), sa, salen, dcid);
+
+  ngtcp2_crypto_aead_ctx aead_ctx;
+  if (ngtcp2_crypto_aead_ctx_decrypt_init(&aead_ctx, &aead, key.data(),
+                                          ivlen) != 0) {
+    return -1;
+  }
+
+  std::array<uint8_t, SHRPX_QUIC_MAX_RETRY_TOKENLEN> plaintext;
+
+  auto rv = ngtcp2_crypto_decrypt(plaintext.data(), &aead, &aead_ctx,
+                                  ciphertext, ciphertextlen, iv.data(), ivlen,
+                                  aad.data(), aadlen);
+
+  ngtcp2_crypto_aead_ctx_free(&aead_ctx);
+
+  if (rv != 0) {
+    return -1;
+  }
+
+  assert(ciphertextlen >= aead.max_overhead);
+
+  auto plaintextlen = ciphertextlen - aead.max_overhead;
+  if (plaintextlen < sizeof(uint64_t)) {
+    return -1;
+  }
+
+  auto cil = plaintextlen - sizeof(uint64_t);
+  if (cil != 0 && (cil < NGTCP2_MIN_CIDLEN || cil > NGTCP2_MAX_CIDLEN)) {
+    return -1;
+  }
+
+  uint64_t t;
+  memcpy(&t, plaintext.data(), sizeof(uint64_t));
+
+  uint64_t now = std::chrono::duration_cast<std::chrono::nanoseconds>(
+                     std::chrono::system_clock::now().time_since_epoch())
+                     .count();
+
+  // Allow 10 seconds window
+  if (t + 10ULL * NGTCP2_SECONDS < now) {
+    return -1;
+  }
+
+  ngtcp2_cid_init(odcid, plaintext.data() + sizeof(uint64_t), cil);
+
+  return 0;
 }
 
 } // namespace shrpx

src/shrpx_quic.h

@@ -40,6 +40,21 @@ constexpr size_t SHRPX_QUIC_CID_PREFIXLEN = 8;
 constexpr size_t SHRPX_MAX_UDP_PAYLOAD_SIZE = 1280;
 constexpr size_t SHRPX_QUIC_STATELESS_RESET_SECRETLEN = 32;
 constexpr size_t SHRPX_QUIC_TOKEN_SECRETLEN = 32;
+constexpr size_t SHRPX_QUIC_TOKEN_RAND_DATALEN = 16;
+
+// SHRPX_QUIC_RETRY_TOKEN_MAGIC is the magic byte of Retry token.
+// Sent in plaintext.
+constexpr uint8_t SHRPX_QUIC_RETRY_TOKEN_MAGIC = 0xb6;
+constexpr size_t SHRPX_QUIC_MAX_RETRY_TOKENLEN =
+    /* magic */ 1 + sizeof(uint64_t) + NGTCP2_MAX_CIDLEN +
+    /* aead tag */ 16 + SHRPX_QUIC_TOKEN_RAND_DATALEN;
+
+// SHRPX_QUIC_TOKEN_MAGIC is the magic byte of token which is sent in
+// NEW_TOKEN frame.  Sent in plaintext.
+constexpr uint8_t SHRPX_QUIC_TOKEN_MAGIC = 0x36;
+constexpr size_t SHRPX_QUIC_MAX_TOKENLEN =
+    /* magic */ 1 + sizeof(uint64_t) + /* aead tag */ 16 +
+    SHRPX_QUIC_TOKEN_RAND_DATALEN;
 
 ngtcp2_tstamp quic_timestamp();
 
@@ -61,6 +76,14 @@ int generate_quic_stateless_reset_secret(uint8_t *secret);
 
 int generate_quic_token_secret(uint8_t *secret);
 
+int generate_retry_token(uint8_t *token, size_t &tokenlen, const sockaddr *sa,
+                         socklen_t salen, const ngtcp2_cid *retry_scid,
+                         const ngtcp2_cid *odcid, const uint8_t *token_secret);
+
+int verify_retry_token(ngtcp2_cid *odcid, const uint8_t *token, size_t tokenlen,
+                       const ngtcp2_cid *dcid, const sockaddr *sa,
+                       socklen_t salen, const uint8_t *token_secret);
+
 } // namespace shrpx
 
 #endif // SHRPX_QUIC_H

src/shrpx_quic_connection_handler.cc

@@ -27,6 +27,7 @@
 #include <openssl/rand.h>
 
 #include <ngtcp2/ngtcp2.h>
+#include <ngtcp2/ngtcp2_crypto.h>
 
 #include "shrpx_worker.h"
 #include "shrpx_client_handler.h"
@@ -83,12 +84,30 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
     // new connection
 
     ngtcp2_pkt_hd hd;
+    ngtcp2_cid odcid, *podcid = nullptr;
 
     switch (ngtcp2_accept(&hd, data, datalen)) {
-    case 0:
+    case 0: {
+      if (hd.token.len == 0) {
+        break;
+      }
+
+      auto &quic_secret = worker_->get_quic_secret();
+      auto &secret = quic_secret->token_secret;
+
+      if (hd.token.base[0] == SHRPX_QUIC_RETRY_TOKEN_MAGIC) {
+        if (verify_retry_token(&odcid, hd.token.base, hd.token.len, &hd.dcid,
+                               &remote_addr.su.sa, remote_addr.len,
+                               secret.data()) == 0) {
+          podcid = &odcid;
+        }
+      }
+
       break;
+    }
     case NGTCP2_ERR_RETRY:
-      // TODO Send retry
+      send_retry(faddr, version, dcid, dcidlen, scid, scidlen, remote_addr,
+                 local_addr);
       return 0;
     case NGTCP2_ERR_VERSION_NEGOTIATION:
       send_version_negotiation(faddr, version, scid, scidlen, dcid, dcidlen,
@@ -120,7 +139,7 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
       return 0;
     }
 
-    handler = handle_new_connection(faddr, remote_addr, local_addr, hd);
+    handler = handle_new_connection(faddr, remote_addr, local_addr, hd, podcid);
     if (handler == nullptr) {
       return 0;
     }
@@ -140,7 +159,8 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
 
 ClientHandler *QUICConnectionHandler::handle_new_connection(
     const UpstreamAddr *faddr, const Address &remote_addr,
-    const Address &local_addr, const ngtcp2_pkt_hd &hd) {
+    const Address &local_addr, const ngtcp2_pkt_hd &hd,
+    const ngtcp2_cid *odcid) {
   std::array<char, NI_MAXHOST> host;
   std::array<char, NI_MAXSERV> service;
   int rv;
@@ -178,8 +198,17 @@ ClientHandler *QUICConnectionHandler::handle_new_connection(
       worker_, faddr->fd, ssl, StringRef{host.data()},
       StringRef{service.data()}, remote_addr.su.sa.sa_family, faddr);
 
+  const uint8_t *token = nullptr;
+  size_t tokenlen = 0;
+
+  if (odcid) {
+    token = hd.token.base;
+    tokenlen = hd.token.len;
+  }
+
   auto upstream = std::make_unique<Http3Upstream>(handler.get());
-  if (upstream->init(faddr, remote_addr, local_addr, hd) != 0) {
+  if (upstream->init(faddr, remote_addr, local_addr, hd, odcid, token,
+                     tokenlen) != 0) {
     return nullptr;
   }
 
@@ -215,6 +244,58 @@ uint32_t generate_reserved_version(const Address &addr, uint32_t version) {
 }
 } // namespace
 
+int QUICConnectionHandler::send_retry(
+    const UpstreamAddr *faddr, uint32_t version, const uint8_t *initial_dcid,
+    size_t initial_dcidlen, const uint8_t *initial_scid, size_t initial_scidlen,
+    const Address &remote_addr, const Address &local_addr) {
+  std::array<char, NI_MAXHOST> host;
+  std::array<char, NI_MAXSERV> port;
+
+  if (getnameinfo(&remote_addr.su.sa, remote_addr.len, host.data(), host.size(),
+                  port.data(), port.size(),
+                  NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
+    return -1;
+  }
+
+  ngtcp2_cid retry_scid;
+
+  retry_scid.datalen = SHRPX_QUIC_SCIDLEN;
+  // We do not steer packet based on Retry CID.
+  if (RAND_bytes(retry_scid.data, retry_scid.datalen) != 1) {
+    return -1;
+  }
+
+  std::array<uint8_t, SHRPX_QUIC_MAX_RETRY_TOKENLEN> token;
+  size_t tokenlen = token.size();
+
+  ngtcp2_cid ini_dcid, ini_scid;
+  ngtcp2_cid_init(&ini_dcid, initial_dcid, initial_dcidlen);
+  ngtcp2_cid_init(&ini_scid, initial_scid, initial_scidlen);
+
+  auto &quic_secret = worker_->get_quic_secret();
+  auto &secret = quic_secret->token_secret;
+
+  if (generate_retry_token(token.data(), tokenlen, &remote_addr.su.sa,
+                           remote_addr.len, &retry_scid, &ini_dcid,
+                           secret.data()) != 0) {
+    return -1;
+  }
+
+  std::array<uint8_t, 1280> buf;
+
+  auto nwrite =
+      ngtcp2_crypto_write_retry(buf.data(), buf.size(), version, &ini_scid,
+                                &retry_scid, &ini_dcid, token.data(), tokenlen);
+  if (nwrite < 0) {
+    LOG(ERROR) << "ngtcp2_crypto_write_retry: " << ngtcp2_strerror(nwrite);
+    return -1;
+  }
+
+  return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
+                          &local_addr.su.sa, local_addr.len, buf.data(), nwrite,
+                          0);
+}
+
 int QUICConnectionHandler::send_version_negotiation(
     const UpstreamAddr *faddr, uint32_t version, const uint8_t *dcid,
     size_t dcidlen, const uint8_t *scid, size_t scidlen,

src/shrpx_quic_connection_handler.h

@@ -50,6 +50,14 @@ public:
   int handle_packet(const UpstreamAddr *faddr, const Address &remote_addr,
                     const Address &local_addr, const uint8_t *data,
                     size_t datalen);
+  // Send Retry packet.  |ini_dcid| is the destination Connection ID
+  // which appeared in Client Initial packet and its length is
+  // |dcidlen|.  |ini_scid| is the source Connection ID which appeared
+  // in Client Initial packet and its length is |scidlen|.
+  int send_retry(const UpstreamAddr *faddr, uint32_t version,
+                 const uint8_t *ini_dcid, size_t ini_dcidlen,
+                 const uint8_t *ini_scid, size_t ini_scidlen,
+                 const Address &remote_addr, const Address &local_addr);
   int send_version_negotiation(const UpstreamAddr *faddr, uint32_t version,
                                const uint8_t *dcid, size_t dcidlen,
                                const uint8_t *scid, size_t scidlen,
@@ -61,7 +69,8 @@ public:
   ClientHandler *handle_new_connection(const UpstreamAddr *faddr,
                                        const Address &remote_addr,
                                        const Address &local_addr,
-                                       const ngtcp2_pkt_hd &hd);
+                                       const ngtcp2_pkt_hd &hd,
+                                       const ngtcp2_cid *odcid);
   void add_connection_id(const ngtcp2_cid *cid, ClientHandler *handler);
   void remove_connection_id(const ngtcp2_cid *cid);
 

src/shrpx_worker_process.cc

@@ -496,6 +496,12 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) {
     }
   }
 
+#ifdef ENABLE_HTTP3
+  if (conn_handler->create_quic_secret() != 0) {
+    return -1;
+  }
+#endif // ENABLE_HTTP3
+
   if (config->single_thread) {
     rv = conn_handler->create_single_worker();
     if (rv != 0) {