From deae6c95b176eff00f547508dc19d2fdc87b89bb Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 5 Nov 2021 23:23:00 +0900 Subject: [PATCH] nghttpx: Send ECN --- src/shrpx_http3_upstream.cc | 30 +++++++++++++++++----------- src/shrpx_http3_upstream.h | 4 ++-- src/shrpx_quic.cc | 9 ++++++--- src/shrpx_quic.h | 4 ++-- src/shrpx_quic_connection_handler.cc | 20 +++++++++---------- src/util.cc | 21 +++++++++++++++++++ src/util.h | 2 ++ 7 files changed, 61 insertions(+), 29 deletions(-) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 733cedb4..3fc40bd4 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -712,6 +712,9 @@ int Http3Upstream::write_streams() { std::min(static_cast(64_k), ngtcp2_conn_get_send_quantum(conn_)) / max_udp_payload_size; ngtcp2_pkt_info pi; +#ifdef UDP_SEGMENT + ngtcp2_pkt_info prev_pi; +#endif // UDP_SEGMENT uint8_t *bufpos = buf.data(); ngtcp2_path_storage ps, prev_ps; size_t pktcnt = 0; @@ -815,7 +818,8 @@ int Http3Upstream::write_streams() { send_packet(static_cast(prev_ps.path.user_data), prev_ps.path.remote.addr, prev_ps.path.remote.addrlen, prev_ps.path.local.addr, prev_ps.path.local.addrlen, - buf.data(), bufpos - buf.data(), max_udp_payload_size); + prev_pi, buf.data(), bufpos - buf.data(), + max_udp_payload_size); reset_idle_timer(); } @@ -832,17 +836,19 @@ int Http3Upstream::write_streams() { #ifdef UDP_SEGMENT if (pktcnt == 0) { ngtcp2_path_copy(&prev_ps.path, &ps.path); - } else if (!ngtcp2_path_eq(&prev_ps.path, &ps.path)) { + prev_pi = pi; + } else if (!ngtcp2_path_eq(&prev_ps.path, &ps.path) || + prev_pi.ecn != pi.ecn) { send_packet(static_cast(prev_ps.path.user_data), prev_ps.path.remote.addr, prev_ps.path.remote.addrlen, - prev_ps.path.local.addr, prev_ps.path.local.addrlen, + prev_ps.path.local.addr, prev_ps.path.local.addrlen, prev_pi, buf.data(), bufpos - buf.data() - nwrite, max_udp_payload_size); send_packet(static_cast(ps.path.user_data), ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, bufpos - nwrite, - nwrite, max_udp_payload_size); + ps.path.local.addr, ps.path.local.addrlen, pi, + bufpos - nwrite, nwrite, max_udp_payload_size); ngtcp2_conn_update_pkt_tx_time(conn_, ts); reset_idle_timer(); @@ -856,7 +862,7 @@ int Http3Upstream::write_streams() { static_cast(nwrite) < max_udp_payload_size) { send_packet(static_cast(ps.path.user_data), ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, buf.data(), + ps.path.local.addr, ps.path.local.addrlen, pi, buf.data(), bufpos - buf.data(), max_udp_payload_size); ngtcp2_conn_update_pkt_tx_time(conn_, ts); @@ -869,7 +875,7 @@ int Http3Upstream::write_streams() { #else // !UDP_SEGMENT send_packet(static_cast(ps.path.user_data), ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr, - ps.path.local.addrlen, buf.data(), bufpos - buf.data(), 0); + ps.path.local.addrlen, pi, buf.data(), bufpos - buf.data(), 0); if (++pktcnt == max_pktcnt) { ngtcp2_conn_update_pkt_tx_time(conn_, ts); @@ -1411,7 +1417,7 @@ void Http3Upstream::on_handler_delete() { send_packet(static_cast(ps.path.user_data), ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr, - ps.path.local.addrlen, conn_close_.data(), nwrite, 0); + ps.path.local.addrlen, pi, conn_close_.data(), nwrite, 0); } auto d = @@ -1716,10 +1722,10 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr, int Http3Upstream::send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, size_t remote_salen, const sockaddr *local_sa, size_t local_salen, - const uint8_t *data, size_t datalen, - size_t gso_size) { + const ngtcp2_pkt_info &pi, const uint8_t *data, + size_t datalen, size_t gso_size) { auto rv = quic_send_packet(faddr, remote_sa, remote_salen, local_sa, - local_salen, data, datalen, gso_size); + local_salen, pi, data, datalen, gso_size); switch (rv) { case 0: return 0; @@ -1776,7 +1782,7 @@ int Http3Upstream::handle_error() { send_packet(static_cast(ps.path.user_data), ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr, - ps.path.local.addrlen, conn_close_.data(), nwrite, 0); + ps.path.local.addrlen, pi, conn_close_.data(), nwrite, 0); return -1; } diff --git a/src/shrpx_http3_upstream.h b/src/shrpx_http3_upstream.h index 5df95283..8df63ea0 100644 --- a/src/shrpx_http3_upstream.h +++ b/src/shrpx_http3_upstream.h @@ -151,8 +151,8 @@ public: void idle_close(); int send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, size_t remote_salen, const sockaddr *local_sa, - size_t local_salen, const uint8_t *data, size_t datalen, - size_t gso_size); + size_t local_salen, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen, size_t gso_size); void qlog_write(const void *data, size_t datalen, bool fin); int open_qlog_file(const StringRef &dir, const ngtcp2_cid &scid) const; diff --git a/src/shrpx_quic.cc b/src/shrpx_quic.cc index 224fbbb1..fcb2cf51 100644 --- a/src/shrpx_quic.cc +++ b/src/shrpx_quic.cc @@ -57,8 +57,8 @@ ngtcp2_tstamp quic_timestamp() { int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, size_t remote_salen, const sockaddr *local_sa, - size_t local_salen, const uint8_t *data, size_t datalen, - size_t gso_size) { + size_t local_salen, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen, size_t gso_size) { iovec msg_iov = {const_cast(data), datalen}; msghdr msg{}; msg.msg_name = const_cast(remote_sa); @@ -123,6 +123,8 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, msg.msg_controllen = controllen; + util::fd_set_send_ecn(faddr->fd, local_sa->sa_family, pi.ecn); + ssize_t nwrite; do { @@ -142,7 +144,8 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, LOG(INFO) << "QUIC sent packet: local=" << util::to_numeric_addr(local_sa, local_salen) << " remote=" << util::to_numeric_addr(remote_sa, remote_salen) - << " " << nwrite << " bytes"; + << " ecn=" << log::hex << pi.ecn << log::dec << " " << nwrite + << " bytes"; } return 0; diff --git a/src/shrpx_quic.h b/src/shrpx_quic.h index 8f400254..cf1e0c1b 100644 --- a/src/shrpx_quic.h +++ b/src/shrpx_quic.h @@ -81,8 +81,8 @@ ngtcp2_tstamp quic_timestamp(); int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, size_t remote_salen, const sockaddr *local_sa, - size_t local_salen, const uint8_t *data, size_t datalen, - size_t gso_size); + size_t local_salen, const ngtcp2_pkt_info &pi, + const uint8_t *data, size_t datalen, size_t gso_size); int generate_quic_retry_connection_id(ngtcp2_cid &cid, size_t cidlen, const uint8_t *server_id, uint8_t km_id, diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc index 38249e29..2a04df9b 100644 --- a/src/shrpx_quic_connection_handler.cc +++ b/src/shrpx_quic_connection_handler.cc @@ -491,8 +491,8 @@ int QUICConnectionHandler::send_retry( buf.resize(nwrite); quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, - &local_addr.su.sa, local_addr.len, buf.data(), buf.size(), - 0); + &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{}, + buf.data(), buf.size(), 0); if (generate_quic_hashed_connection_id(idcid, remote_addr, local_addr, idcid) != 0) { @@ -541,8 +541,8 @@ int QUICConnectionHandler::send_version_negotiation( } return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, - &local_addr.su.sa, local_addr.len, buf.data(), nwrite, - 0); + &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{}, + buf.data(), nwrite, 0); } int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr, @@ -604,8 +604,8 @@ int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr, } return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, - &local_addr.su.sa, local_addr.len, buf.data(), nwrite, - 0); + &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{}, + buf.data(), nwrite, 0); } int QUICConnectionHandler::send_connection_close( @@ -630,8 +630,8 @@ int QUICConnectionHandler::send_connection_close( } return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, - &local_addr.su.sa, local_addr.len, buf.data(), nwrite, - 0); + &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{}, + buf.data(), nwrite, 0); } void QUICConnectionHandler::add_connection_id(const ngtcp2_cid &cid, @@ -726,8 +726,8 @@ int CloseWait::handle_packet(const UpstreamAddr *faddr, } if (quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len, - &local_addr.su.sa, local_addr.len, pkt.data(), - pkt.size(), 0) != 0) { + &local_addr.su.sa, local_addr.len, ngtcp2_pkt_info{}, + pkt.data(), pkt.size(), 0) != 0) { return -1; } diff --git a/src/util.cc b/src/util.cc index eb70bbb3..bb6e5e47 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1770,6 +1770,27 @@ int fd_set_recv_ecn(int fd, int family) { return -1; } + +int fd_set_send_ecn(int fd, int family, unsigned int ecn) { + switch (family) { + case AF_INET: + if (setsockopt(fd, IPPROTO_IP, IP_TOS, &ecn, + static_cast(sizeof(ecn))) == -1) { + return -1; + } + + return 0; + case AF_INET6: + if (setsockopt(fd, IPPROTO_IPV6, IPV6_TCLASS, &ecn, + static_cast(sizeof(ecn))) == -1) { + return -1; + } + + return 0; + } + + return -1; +} #endif // ENABLE_HTTP3 } // namespace util diff --git a/src/util.h b/src/util.h index e154b6ed..359e785f 100644 --- a/src/util.h +++ b/src/util.h @@ -922,6 +922,8 @@ int msghdr_get_local_addr(Address &dest, msghdr *msg, int family); unsigned int msghdr_get_ecn(msghdr *msg, int family); int fd_set_recv_ecn(int fd, int family); + +int fd_set_send_ecn(int fd, int family, unsigned int ecn); #endif // ENABLE_HTTP3 } // namespace util