From f8474b25f02c4faa6d4250800a164227cf89b327 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 18 Oct 2021 22:37:16 +0900 Subject: [PATCH] nghttpx: Reduce dgram size if sendmsg fails with EINVAL or EMSGSIZE --- src/shrpx_http3_upstream.cc | 82 +++++++++++++++++++++++-------------- src/shrpx_http3_upstream.h | 5 +++ src/shrpx_quic.cc | 7 +++- 3 files changed, 63 insertions(+), 31 deletions(-) diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc index 1208e3a3..956215c5 100644 --- a/src/shrpx_http3_upstream.cc +++ b/src/shrpx_http3_upstream.cc @@ -118,6 +118,7 @@ size_t downstream_queue_size(Worker *worker) { Http3Upstream::Http3Upstream(ClientHandler *handler) : handler_{handler}, + max_udp_payload_size_{SHRPX_QUIC_MAX_UDP_PAYLOAD_SIZE}, qlog_fd_{-1}, hashed_scid_{}, conn_{nullptr}, @@ -707,7 +708,8 @@ int Http3Upstream::on_write() { int Http3Upstream::write_streams() { std::array vec; std::array buf; - auto max_udp_payload_size = ngtcp2_conn_get_path_max_udp_payload_size(conn_); + auto max_udp_payload_size = std::min( + max_udp_payload_size_, ngtcp2_conn_get_path_max_udp_payload_size(conn_)); size_t max_pktcnt = std::min(static_cast(64_k), ngtcp2_conn_get_send_quantum(conn_)) / max_udp_payload_size; @@ -812,10 +814,10 @@ int Http3Upstream::write_streams() { if (nwrite == 0) { if (bufpos - buf.data()) { - quic_send_packet(static_cast(prev_ps.path.user_data), - prev_ps.path.remote.addr, prev_ps.path.remote.addrlen, - prev_ps.path.local.addr, prev_ps.path.local.addrlen, - buf.data(), bufpos - buf.data(), max_udp_payload_size); + send_packet(static_cast(prev_ps.path.user_data), + prev_ps.path.remote.addr, prev_ps.path.remote.addrlen, + prev_ps.path.local.addr, prev_ps.path.local.addrlen, + buf.data(), bufpos - buf.data(), max_udp_payload_size); reset_idle_timer(); } @@ -833,16 +835,16 @@ int Http3Upstream::write_streams() { if (pktcnt == 0) { ngtcp2_path_copy(&prev_ps.path, &ps.path); } else if (!ngtcp2_path_eq(&prev_ps.path, &ps.path)) { - quic_send_packet(static_cast(prev_ps.path.user_data), - prev_ps.path.remote.addr, prev_ps.path.remote.addrlen, - prev_ps.path.local.addr, prev_ps.path.local.addrlen, - buf.data(), bufpos - buf.data() - nwrite, - max_udp_payload_size); + send_packet(static_cast(prev_ps.path.user_data), + prev_ps.path.remote.addr, prev_ps.path.remote.addrlen, + prev_ps.path.local.addr, prev_ps.path.local.addrlen, + buf.data(), bufpos - buf.data() - nwrite, + max_udp_payload_size); - quic_send_packet(static_cast(ps.path.user_data), - ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, - bufpos - nwrite, nwrite, max_udp_payload_size); + send_packet(static_cast(ps.path.user_data), + ps.path.remote.addr, ps.path.remote.addrlen, + ps.path.local.addr, ps.path.local.addrlen, bufpos - nwrite, + nwrite, max_udp_payload_size); ngtcp2_conn_update_pkt_tx_time(conn_, ts); reset_idle_timer(); @@ -854,10 +856,10 @@ int Http3Upstream::write_streams() { if (++pktcnt == max_pktcnt || static_cast(nwrite) < max_udp_payload_size) { - quic_send_packet(static_cast(ps.path.user_data), - ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, buf.data(), - bufpos - buf.data(), max_udp_payload_size); + send_packet(static_cast(ps.path.user_data), + ps.path.remote.addr, ps.path.remote.addrlen, + ps.path.local.addr, ps.path.local.addrlen, buf.data(), + bufpos - buf.data(), max_udp_payload_size); ngtcp2_conn_update_pkt_tx_time(conn_, ts); reset_idle_timer(); @@ -867,10 +869,9 @@ int Http3Upstream::write_streams() { return 0; } #else // !UDP_SEGMENT - quic_send_packet(static_cast(ps.path.user_data), - ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, buf.data(), - bufpos - buf.data(), 0); + send_packet(static_cast(ps.path.user_data), + ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr, + ps.path.local.addrlen, buf.data(), bufpos - buf.data(), 0); if (++pktcnt == max_pktcnt) { ngtcp2_conn_update_pkt_tx_time(conn_, ts); @@ -1410,10 +1411,9 @@ void Http3Upstream::on_handler_delete() { conn_close_.resize(nwrite); - quic_send_packet(static_cast(ps.path.user_data), - ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, - conn_close_.data(), nwrite, 0); + send_packet(static_cast(ps.path.user_data), + ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr, + ps.path.local.addrlen, conn_close_.data(), nwrite, 0); } auto d = @@ -1716,6 +1716,29 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr, return 0; } +int Http3Upstream::send_packet(const UpstreamAddr *faddr, + const sockaddr *remote_sa, size_t remote_salen, + const sockaddr *local_sa, size_t local_salen, + const uint8_t *data, size_t datalen, + size_t gso_size) { + auto rv = quic_send_packet(faddr, remote_sa, remote_salen, local_sa, + local_salen, data, datalen, gso_size); + switch (rv) { + case 0: + return 0; + // With GSO, sendmsg may fail with EINVAL if UDP payload is too + // large. + case -EINVAL: + case -EMSGSIZE: + max_udp_payload_size_ = NGTCP2_MAX_UDP_PAYLOAD_SIZE; + break; + default: + break; + } + + return -1; +} + int Http3Upstream::handle_error() { if (ngtcp2_conn_is_in_closing_period(conn_)) { return -1; @@ -1754,10 +1777,9 @@ int Http3Upstream::handle_error() { conn_close_.resize(nwrite); - quic_send_packet(static_cast(ps.path.user_data), - ps.path.remote.addr, ps.path.remote.addrlen, - ps.path.local.addr, ps.path.local.addrlen, - conn_close_.data(), nwrite, 0); + send_packet(static_cast(ps.path.user_data), + ps.path.remote.addr, ps.path.remote.addrlen, ps.path.local.addr, + ps.path.local.addrlen, conn_close_.data(), nwrite, 0); return -1; } diff --git a/src/shrpx_http3_upstream.h b/src/shrpx_http3_upstream.h index b811a4bb..b2935afc 100644 --- a/src/shrpx_http3_upstream.h +++ b/src/shrpx_http3_upstream.h @@ -148,6 +148,10 @@ public: int start_graceful_shutdown(); int submit_goaway(); void idle_close(); + int send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, + size_t remote_salen, const sockaddr *local_sa, + size_t local_salen, const uint8_t *data, size_t datalen, + size_t gso_size); void qlog_write(const void *data, size_t datalen, bool fin); int open_qlog_file(const StringRef &dir, const ngtcp2_cid &scid) const; @@ -158,6 +162,7 @@ private: ev_timer idle_timer_; ev_timer shutdown_timer_; ev_prepare prep_; + size_t max_udp_payload_size_; int qlog_fd_; ngtcp2_cid hashed_scid_; ngtcp2_conn *conn_; diff --git a/src/shrpx_quic.cc b/src/shrpx_quic.cc index a0bf56c3..224fbbb1 100644 --- a/src/shrpx_quic.cc +++ b/src/shrpx_quic.cc @@ -130,7 +130,12 @@ int quic_send_packet(const UpstreamAddr *faddr, const sockaddr *remote_sa, } while (nwrite == -1 && errno == EINTR); if (nwrite == -1) { - return -1; + if (LOG_ENABLED(INFO)) { + auto error = errno; + LOG(INFO) << "sendmsg failed: errno=" << error; + } + + return -errno; } if (LOG_ENABLED(INFO)) {