From 17012654e1c725f3380354565eb503f8e0b319e2 Mon Sep 17 00:00:00 2001
From: Tatsuhiro Tsujikawa <tatsuhiro.t@gmail.com>
Date: Tue, 31 Aug 2021 14:06:59 +0900
Subject: [PATCH] nghttpx: Add HTTP/3 graceful shutdown

---
 src/shrpx_http3_upstream.cc | 82 +++++++++++++++++++++++++++++++++++++
 src/shrpx_http3_upstream.h  |  5 +++
 2 files changed, 87 insertions(+)

diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc
index c6b3f612..4a4521b9 100644
--- a/src/shrpx_http3_upstream.cc
+++ b/src/shrpx_http3_upstream.cc
@@ -76,6 +76,28 @@ fail:
 }
 } // namespace
 
+namespace {
+void shutdown_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents) {
+  auto upstream = static_cast<Http3Upstream *>(w->data);
+  auto handler = upstream->get_client_handler();
+
+  if (upstream->submit_goaway() != 0) {
+    delete handler;
+  }
+}
+} // namespace
+
+namespace {
+void prepare_cb(struct ev_loop *loop, ev_prepare *w, int revent) {
+  auto upstream = static_cast<Http3Upstream *>(w->data);
+  auto handler = upstream->get_client_handler();
+
+  if (upstream->check_shutdown() != 0) {
+    delete handler;
+  }
+}
+} // namespace
+
 namespace {
 size_t downstream_queue_size(Worker *worker) {
   auto &downstreamconf = *worker->get_downstream_config();
@@ -104,11 +126,20 @@ Http3Upstream::Http3Upstream(ClientHandler *handler)
   ev_timer_init(&idle_timer_, idle_timeoutcb, 0.,
                 quicconf.upstream.timeout.idle);
   idle_timer_.data = this;
+
+  ev_timer_init(&shutdown_timer_, shutdown_timeout_cb, 0., 0.);
+  shutdown_timer_.data = this;
+
+  ev_prepare_init(&prep_, prepare_cb);
+  prep_.data = this;
+  ev_prepare_start(handler_->get_loop(), &prep_);
 }
 
 Http3Upstream::~Http3Upstream() {
   auto loop = handler_->get_loop();
 
+  ev_prepare_stop(loop, &prep_);
+  ev_timer_stop(loop, &shutdown_timer_);
   ev_timer_stop(loop, &idle_timer_);
   ev_timer_stop(loop, &timer_);
 
@@ -2400,4 +2431,55 @@ void Http3Upstream::log_response_headers(
                    << ss.str();
 }
 
+int Http3Upstream::check_shutdown() {
+  auto worker = handler_->get_worker();
+
+  if (!worker->get_graceful_shutdown()) {
+    return 0;
+  }
+
+  ev_prepare_stop(handler_->get_loop(), &prep_);
+
+  return start_graceful_shutdown();
+}
+
+int Http3Upstream::start_graceful_shutdown() {
+  int rv;
+
+  if (ev_is_active(&shutdown_timer_)) {
+    return 0;
+  }
+
+  rv = nghttp3_conn_submit_shutdown_notice(httpconn_);
+  if (rv != 0) {
+    ULOG(FATAL, this) << "nghttp3_conn_submit_shutdown_notice: "
+                      << nghttp3_strerror(rv);
+    return -1;
+  }
+
+  handler_->signal_write();
+
+  auto t = ngtcp2_conn_get_pto(conn_);
+
+  ev_timer_set(&shutdown_timer_, static_cast<ev_tstamp>(t) * 3 / NGTCP2_SECONDS,
+               0.);
+  ev_timer_start(handler_->get_loop(), &shutdown_timer_);
+
+  return 0;
+}
+
+int Http3Upstream::submit_goaway() {
+  int rv;
+
+  rv = nghttp3_conn_shutdown(httpconn_);
+  if (rv != 0) {
+    ULOG(FATAL, this) << "nghttp3_conn_shutdown: " << nghttp3_strerror(rv);
+    return -1;
+  }
+
+  handler_->signal_write();
+
+  return 0;
+}
+
 } // namespace shrpx
diff --git a/src/shrpx_http3_upstream.h b/src/shrpx_http3_upstream.h
index b3a15a0e..fcc3d122 100644
--- a/src/shrpx_http3_upstream.h
+++ b/src/shrpx_http3_upstream.h
@@ -144,11 +144,16 @@ public:
   int http_recv_data(Downstream *downstream, const uint8_t *data,
                      size_t datalen);
   int handshake_completed();
+  int check_shutdown();
+  int start_graceful_shutdown();
+  int submit_goaway();
 
 private:
   ClientHandler *handler_;
   ev_timer timer_;
   ev_timer idle_timer_;
+  ev_timer shutdown_timer_;
+  ev_prepare prep_;
   ngtcp2_cid initial_client_dcid_;
   ngtcp2_conn *conn_;
   quic::Error last_error_;