nghttpx: Send ECN

src/shrpx_http3_upstream.cc

@@ -712,6 +712,9 @@ int Http3Upstream::write_streams() {
       std::min(static_cast<size_t>(64_k), ngtcp2_conn_get_send_quantum(conn_)) /
       max_udp_payload_size;
   ngtcp2_pkt_info pi;
+#ifdef UDP_SEGMENT
+  ngtcp2_pkt_info prev_pi;
+#endif // UDP_SEGMENT
   uint8_t *bufpos = buf.data();
   ngtcp2_path_storage ps, prev_ps;
   size_t pktcnt = 0;
@@ -815,7 +818,8 @@ int Http3Upstream::write_streams() {
         send_packet(static_cast<UpstreamAddr *>(prev_ps.path.user_data),
                     prev_ps.path.remote.addr, prev_ps.path.remote.addrlen,
                     prev_ps.path.local.addr, prev_ps.path.local.addrlen,
-                    buf.data(), bufpos - buf.data(), max_udp_payload_size);
+                    prev_pi, buf.data(), bufpos - buf.data(),
+                    max_udp_payload_size);
 
         reset_idle_timer();
       }
@@ -832,17 +836,19 @@ int Http3Upstream::write_streams() {
 #ifdef UDP_SEGMENT
     if (pktcnt == 0) {
       ngtcp2_path_copy(&prev_ps.path, &ps.path);
-    } else if (!ngtcp2_path_eq(&prev_ps.path, &ps.path)) {
+      prev_pi = pi;
+    } else if (!ngtcp2_path_eq(&prev_ps.path, &ps.path) ||
+               prev_pi.ecn != pi.ecn) {
       send_packet(static_cast<UpstreamAddr *>(prev_ps.path.user_data),
                   prev_ps.path.remote.addr, prev_ps.path.remote.addrlen,
-                  prev_ps.path.local.addr, prev_ps.path.local.addrlen,
+                  prev_ps.path.local.addr, prev_ps.path.local.addrlen, prev_pi,
                   buf.data(), bufpos - buf.data() - nwrite,
                   max_udp_payload_size);
 
       send_packet(static_cast<UpstreamAddr *>(ps.path.user_data),
                   ps.path.remote.addr, ps.path.remote.addrlen,
-                  ps.path.local.addr, ps.path.local.addrlen, bufpos - nwrite,
+                  ps.path.local.addr, ps.path.local.addrlen, pi,
-                  nwrite, max_udp_payload_size);
+                  bufpos - nwrite, nwrite, max_udp_payload_size);
 
       ngtcp2_conn_update_pkt_tx_time(conn_, ts);
       reset_idle_timer();
@@ -856,7 +862,7 @@ int Http3Upstream::write_streams() {
         static_cast<size_t>(nwrite) < max_udp_payload_size) {
       send_packet(static_cast<UpstreamAddr *>(ps.path.user_data),
                   ps.path.remote.addr, ps.path.remote.addrlen,
-                  ps.path.local.addr, ps.path.local.addrlen, buf.data(),
+                  ps.path.local.addr, ps.path.local.addrlen, pi, buf.data(),
                   bufpos - buf.data(), max_udp_payload_size);
 
       ngtcp2_conn_update_pkt_tx_time(conn_, ts);
@@ -869,7 +875,7 @@ int Http3Upstream::write_streams() {
 #else  // !UDP_SEGMENT
     send_packet(static_cast<UpstreamAddr *>(ps.path.user_data),
                 ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr,
-                ps.path.local.addrlen, buf.data(), bufpos - buf.data(), 0);
+                ps.path.local.addrlen, pi, buf.data(), bufpos - buf.data(), 0);
 
     if (++pktcnt == max_pktcnt) {
       ngtcp2_conn_update_pkt_tx_time(conn_, ts);
@@ -1411,7 +1417,7 @@ void Http3Upstream::on_handler_delete() {
 
     send_packet(static_cast<UpstreamAddr *>(ps.path.user_data),
                 ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr,
-                ps.path.local.addrlen, conn_close_.data(), nwrite, 0);
+                ps.path.local.addrlen, pi, conn_close_.data(), nwrite, 0);
   }
 
   auto d =
@@ -1716,10 +1722,10 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr,
 int Http3Upstream::send_packet(const UpstreamAddr *faddr,
                                const sockaddr *remote_sa, size_t remote_salen,
                                const sockaddr *local_sa, size_t local_salen,
-                               const uint8_t *data, size_t datalen,
+                               const ngtcp2_pkt_info &pi, const uint8_t *data,
-                               size_t gso_size) {
+                               size_t datalen, size_t gso_size) {
   auto rv = quic_send_packet(faddr, remote_sa, remote_salen, local_sa,
-                             local_salen, data, datalen, gso_size);
+                             local_salen, pi, data, datalen, gso_size);
   switch (rv) {
   case 0:
     return 0;
@@ -1776,7 +1782,7 @@ int Http3Upstream::handle_error() {
 
   send_packet(static_cast<UpstreamAddr *>(ps.path.user_data),
               ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr,
-              ps.path.local.addrlen, conn_close_.data(), nwrite, 0);
+              ps.path.local.addrlen, pi, conn_close_.data(), nwrite, 0);
 
   return -1;
 }

src/shrpx_http3_upstream.h

@@ -151,8 +151,8 @@ public:
   void idle_close();
   int send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
                   size_t remote_salen, const sockaddr *local_sa,
-                  size_t local_salen, const uint8_t *data, size_t datalen,
+                  size_t local_salen, const ngtcp2_pkt_info &pi,
-                  size_t gso_size);
+                  const uint8_t *data, size_t datalen, size_t gso_size);
 
   void qlog_write(const void *data, size_t datalen, bool fin);
   int open_qlog_file(const StringRef &dir, const ngtcp2_cid &scid) const;

src/shrpx_quic.cc

@@ -57,8 +57,8 @@ ngtcp2_tstamp quic_timestamp() {
 
 int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
                      size_t remote_salen, const sockaddr *local_sa,
-                     size_t local_salen, const uint8_t *data, size_t datalen,
+                     size_t local_salen, const ngtcp2_pkt_info &pi,
-                     size_t gso_size) {
+                     const uint8_t *data, size_t datalen, size_t gso_size) {
   iovec msg_iov = {const_cast<uint8_t *>(data), datalen};
   msghdr msg{};
   msg.msg_name = const_cast<sockaddr *>(remote_sa);
@@ -123,6 +123,8 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
 
   msg.msg_controllen = controllen;
 
+  util::fd_set_send_ecn(faddr->fd, local_sa->sa_family, pi.ecn);
+
   ssize_t nwrite;
 
   do {
@@ -142,7 +144,8 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
     LOG(INFO) << "QUIC sent packet: local="
               << util::to_numeric_addr(local_sa, local_salen)
               << " remote=" << util::to_numeric_addr(remote_sa, remote_salen)
-              << " " << nwrite << " bytes";
+              << " ecn=" << log::hex << pi.ecn << log::dec << " " << nwrite
+              << " bytes";
   }
 
   return 0;

src/shrpx_quic.h

@@ -81,8 +81,8 @@ ngtcp2_tstamp quic_timestamp();
 
 int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa,
                      size_t remote_salen, const sockaddr *local_sa,
-                     size_t local_salen, const uint8_t *data, size_t datalen,
+                     size_t local_salen, const ngtcp2_pkt_info &pi,
-                     size_t gso_size);
+                     const uint8_t *data, size_t datalen, size_t gso_size);
 
 int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen,
                                       const uint8_t *server_id, uint8_t km_id,

src/shrpx_quic_connection_handler.cc

@@ -491,8 +491,8 @@ int QUICConnectionHandler::send_retry(
   buf.resize(nwrite);
 
   quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
-                   &local_addr.su.sa, local_addr.len, buf.data(), buf.size(),
+                   &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
-                   0);
+                   buf.data(), buf.size(), 0);
 
   if (generate_quic_hashed_connection_id(idcid, remote_addr, local_addr,
                                          idcid) != 0) {
@@ -541,8 +541,8 @@ int QUICConnectionHandler::send_version_negotiation(
   }
 
   return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
-                          &local_addr.su.sa, local_addr.len, buf.data(), nwrite,
+                          &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
-                          0);
+                          buf.data(), nwrite, 0);
 }
 
 int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr,
@@ -604,8 +604,8 @@ int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr,
   }
 
   return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
-                          &local_addr.su.sa, local_addr.len, buf.data(), nwrite,
+                          &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
-                          0);
+                          buf.data(), nwrite, 0);
 }
 
 int QUICConnectionHandler::send_connection_close(
@@ -630,8 +630,8 @@ int QUICConnectionHandler::send_connection_close(
   }
 
   return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
-                          &local_addr.su.sa, local_addr.len, buf.data(), nwrite,
+                          &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
-                          0);
+                          buf.data(), nwrite, 0);
 }
 
 void QUICConnectionHandler::add_connection_id(const ngtcp2_cid &cid,
@@ -726,8 +726,8 @@ int CloseWait::handle_packet(const UpstreamAddr *faddr,
   }
 
   if (quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
-                       &local_addr.su.sa, local_addr.len, pkt.data(),
+                       &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{},
-                       pkt.size(), 0) != 0) {
+                       pkt.data(), pkt.size(), 0) != 0) {
     return -1;
   }
 

src/util.cc

@@ -1770,6 +1770,27 @@ int fd_set_recv_ecn(int fd, int family) {
 
   return -1;
 }
+
+int fd_set_send_ecn(int fd, int family, unsigned int ecn) {
+  switch (family) {
+  case AF_INET:
+    if (setsockopt(fd, IPPROTO_IP, IP_TOS, &ecn,
+                   static_cast<socklen_t>(sizeof(ecn))) == -1) {
+      return -1;
+    }
+
+    return 0;
+  case AF_INET6:
+    if (setsockopt(fd, IPPROTO_IPV6, IPV6_TCLASS, &ecn,
+                   static_cast<socklen_t>(sizeof(ecn))) == -1) {
+      return -1;
+    }
+
+    return 0;
+  }
+
+  return -1;
+}
 #endif // ENABLE_HTTP3
 
 } // namespace util

src/util.h

@@ -922,6 +922,8 @@ int msghdr_get_local_addr(Address &dest, msghdr *msg, int family);
 unsigned int msghdr_get_ecn(msghdr *msg, int family);
 
 int fd_set_recv_ecn(int fd, int family);
+
+int fd_set_send_ecn(int fd, int family, unsigned int ecn);
 #endif // ENABLE_HTTP3
 
 } // namespace util