nghttpx: Implement RFC 8441 Bootstrapping WebSocket with HTTP/2

genheaderfunc.py

@@ -9,6 +9,7 @@ HEADERS = [
     ':scheme',
     ':status',
     ':host', # for spdy
+    ':protocol',
     'expect',
     'host',
     'if-modified-since',
@@ -32,6 +33,8 @@ HEADERS = [
     "date",
     "content-type",
     "early-data",
+    "sec-websocket-accept",
+    "sec-websocket-key",
     # disallowed h1 headers
     'connection',
     'keep-alive',

src/base64.h

@@ -36,14 +36,17 @@ namespace nghttp2 {
 
 namespace base64 {
 
-template <typename InputIt> std::string encode(InputIt first, InputIt last) {
+namespace {
-  static constexpr char CHAR_TABLE[] = {
+constexpr char B64_CHARS[] = {
     'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
     'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
     'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
     'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
     '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/',
 };
+} // namespace
+
+template <typename InputIt> std::string encode(InputIt first, InputIt last) {
   std::string res;
   size_t len = last - first;
   if (len == 0) {
@@ -57,29 +60,72 @@ template <typename InputIt> std::string encode(InputIt first, InputIt last) {
     uint32_t n = static_cast<uint8_t>(*first++) << 16;
     n += static_cast<uint8_t>(*first++) << 8;
     n += static_cast<uint8_t>(*first++);
-    *p++ = CHAR_TABLE[n >> 18];
+    *p++ = B64_CHARS[n >> 18];
-    *p++ = CHAR_TABLE[(n >> 12) & 0x3fu];
+    *p++ = B64_CHARS[(n >> 12) & 0x3fu];
-    *p++ = CHAR_TABLE[(n >> 6) & 0x3fu];
+    *p++ = B64_CHARS[(n >> 6) & 0x3fu];
-    *p++ = CHAR_TABLE[n & 0x3fu];
+    *p++ = B64_CHARS[n & 0x3fu];
   }
 
   if (r == 2) {
     uint32_t n = static_cast<uint8_t>(*first++) << 16;
     n += static_cast<uint8_t>(*first++) << 8;
-    *p++ = CHAR_TABLE[n >> 18];
+    *p++ = B64_CHARS[n >> 18];
-    *p++ = CHAR_TABLE[(n >> 12) & 0x3fu];
+    *p++ = B64_CHARS[(n >> 12) & 0x3fu];
-    *p++ = CHAR_TABLE[(n >> 6) & 0x3fu];
+    *p++ = B64_CHARS[(n >> 6) & 0x3fu];
     *p++ = '=';
   } else if (r == 1) {
     uint32_t n = static_cast<uint8_t>(*first++) << 16;
-    *p++ = CHAR_TABLE[n >> 18];
+    *p++ = B64_CHARS[n >> 18];
-    *p++ = CHAR_TABLE[(n >> 12) & 0x3fu];
+    *p++ = B64_CHARS[(n >> 12) & 0x3fu];
     *p++ = '=';
     *p++ = '=';
   }
   return res;
 }
 
+constexpr size_t encode_length(size_t n) { return (n + 2) / 3 * 4; }
+
+template <typename InputIt, typename OutputIt>
+OutputIt encode(InputIt first, InputIt last, OutputIt d_first) {
+  size_t len = last - first;
+  if (len == 0) {
+    return d_first;
+  }
+  auto r = len % 3;
+  auto j = last - r;
+  auto p = d_first;
+  while (first != j) {
+    uint32_t n = static_cast<uint8_t>(*first++) << 16;
+    n += static_cast<uint8_t>(*first++) << 8;
+    n += static_cast<uint8_t>(*first++);
+    *p++ = B64_CHARS[n >> 18];
+    *p++ = B64_CHARS[(n >> 12) & 0x3fu];
+    *p++ = B64_CHARS[(n >> 6) & 0x3fu];
+    *p++ = B64_CHARS[n & 0x3fu];
+  }
+
+  switch (r) {
+  case 2: {
+    uint32_t n = static_cast<uint8_t>(*first++) << 16;
+    n += static_cast<uint8_t>(*first++) << 8;
+    *p++ = B64_CHARS[n >> 18];
+    *p++ = B64_CHARS[(n >> 12) & 0x3fu];
+    *p++ = B64_CHARS[(n >> 6) & 0x3fu];
+    *p++ = '=';
+    break;
+  }
+  case 1: {
+    uint32_t n = static_cast<uint8_t>(*first++) << 16;
+    *p++ = B64_CHARS[n >> 18];
+    *p++ = B64_CHARS[(n >> 12) & 0x3fu];
+    *p++ = '=';
+    *p++ = '=';
+    break;
+  }
+  }
+  return p;
+}
+
 template <typename InputIt>
 InputIt next_decode_input(InputIt first, InputIt last, const int *tbl) {
   for (; first != last; ++first) {

src/http2.cc

@@ -394,6 +394,16 @@ void copy_headers_to_nva_internal(std::vector<nghttp2_nv> &nva,
         continue;
       }
       break;
+    case HD_SEC_WEBSOCKET_ACCEPT:
+      if (flags & HDOP_STRIP_SEC_WEBSOCKET_ACCEPT) {
+        continue;
+      }
+      break;
+    case HD_SEC_WEBSOCKET_KEY:
+      if (flags & HDOP_STRIP_SEC_WEBSOCKET_KEY) {
+        continue;
+      }
+      break;
     case HD_FORWARDED:
       if (flags & HDOP_STRIP_FORWARDED) {
         continue;
@@ -834,6 +844,11 @@ int lookup_token(const uint8_t *name, size_t namelen) {
         return HD_FORWARDED;
       }
       break;
+    case 'l':
+      if (util::streq_l(":protoco", name, 8)) {
+        return HD__PROTOCOL;
+      }
+      break;
     }
     break;
   case 10:
@@ -942,6 +957,20 @@ int lookup_token(const uint8_t *name, size_t namelen) {
         return HD_X_FORWARDED_PROTO;
       }
       break;
+    case 'y':
+      if (util::streq_l("sec-websocket-ke", name, 16)) {
+        return HD_SEC_WEBSOCKET_KEY;
+      }
+      break;
+    }
+    break;
+  case 20:
+    switch (name[19]) {
+    case 't':
+      if (util::streq_l("sec-websocket-accep", name, 19)) {
+        return HD_SEC_WEBSOCKET_ACCEPT;
+      }
+      break;
     }
     break;
   }
@@ -1331,7 +1360,8 @@ std::string path_join(const StringRef &base_path, const StringRef &base_query,
 }
 
 bool expect_response_body(int status_code) {
-  return status_code / 100 != 1 && status_code != 304 && status_code != 204;
+  return status_code == 101 ||
+         (status_code / 100 != 1 && status_code != 304 && status_code != 204);
 }
 
 bool expect_response_body(const std::string &method, int status_code) {
@@ -1829,6 +1859,21 @@ bool contains_trailers(const StringRef &s) {
   }
 }
 
+StringRef make_websocket_accept_token(uint8_t *dest, const StringRef &key) {
+  static constexpr uint8_t magic[] = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11";
+  std::array<uint8_t, base64::encode_length(16) + str_size(magic)> s;
+  auto p = std::copy(std::begin(key), std::end(key), std::begin(s));
+  std::copy_n(magic, str_size(magic), p);
+
+  std::array<uint8_t, 20> h;
+  if (util::sha1(h.data(), StringRef{std::begin(s), std::end(s)}) != 0) {
+    return StringRef{};
+  }
+
+  auto end = base64::encode(std::begin(h), std::end(h), dest);
+  return StringRef{dest, end};
+}
+
 } // namespace http2
 
 } // namespace nghttp2

src/http2.h

@@ -41,6 +41,7 @@
 #include "memchunk.h"
 #include "template.h"
 #include "allocator.h"
+#include "base64.h"
 
 namespace nghttp2 {
 
@@ -210,6 +211,12 @@ enum HeaderBuildOp {
   HDOP_STRIP_ALL = HDOP_STRIP_FORWARDED | HDOP_STRIP_X_FORWARDED_FOR |
                    HDOP_STRIP_X_FORWARDED_PROTO | HDOP_STRIP_VIA |
                    HDOP_STRIP_EARLY_DATA,
+  // Sec-WebSocket-Accept header field must be stripped.  If this flag
+  // is not set, all Sec-WebSocket-Accept header fields are added.
+  HDOP_STRIP_SEC_WEBSOCKET_ACCEPT = 1 << 5,
+  // Sec-WebSocket-Key header field must be stripped.  If this flag is
+  // not set, all Sec-WebSocket-Key header fields are added.
+  HDOP_STRIP_SEC_WEBSOCKET_KEY = 1 << 6,
 };
 
 // Appends headers in |headers| to |nv|.  |headers| must be indexed
@@ -297,6 +304,7 @@ enum {
   HD__HOST,
   HD__METHOD,
   HD__PATH,
+  HD__PROTOCOL,
   HD__SCHEME,
   HD__STATUS,
   HD_ACCEPT_ENCODING,
@@ -318,6 +326,8 @@ enum {
   HD_LINK,
   HD_LOCATION,
   HD_PROXY_CONNECTION,
+  HD_SEC_WEBSOCKET_ACCEPT,
+  HD_SEC_WEBSOCKET_KEY,
   HD_SERVER,
   HD_TE,
   HD_TRAILER,
@@ -421,6 +431,12 @@ StringRef copy_lower(BlockAllocator &balloc, const StringRef &src);
 // Returns true if te header field value |s| contains "trailers".
 bool contains_trailers(const StringRef &s);
 
+// Creates Sec-WebSocket-Accept value for |key|.  The capacity of
+// buffer pointed by |dest| must have at least 24 bytes (base64
+// encoded length of 16 bytes data).  It returns empty string in case
+// of error.
+StringRef make_websocket_accept_token(uint8_t *dest, const StringRef &key);
+
 } // namespace http2
 
 } // namespace nghttp2

src/shrpx_client_handler.cc

@@ -979,7 +979,7 @@ ClientHandler::get_downstream_connection(int &err, Downstream *downstream,
     StringRef path;
     // CONNECT method does not have path.  But we requires path in
     // host-path mapping.  As workaround, we assume that path is "/".
-    if (req.method != HTTP_CONNECT) {
+    if (!req.regular_connect_method()) {
       path = req.path;
     }
 

src/shrpx_downstream.cc

@@ -763,13 +763,47 @@ bool Downstream::validate_response_recv_body_length() const {
   return true;
 }
 
-void Downstream::check_upgrade_fulfilled() {
+void Downstream::check_upgrade_fulfilled_http2() {
   if (req_.method == HTTP_CONNECT) {
+    // This handles nonzero req_.connect_proto as well.
     upgraded_ = 200 <= resp_.http_status && resp_.http_status < 300;
 
     return;
   }
 
+  if (req_.connect_proto == CONNECT_PROTO_WEBSOCKET) {
+    // h1 frontend requests WebSocket upgrade
+    upgraded_ = resp_.http_status == 200;
+
+    return;
+  }
+}
+
+void Downstream::check_upgrade_fulfilled_http1() {
+  if (req_.method == HTTP_CONNECT) {
+    if (req_.connect_proto == CONNECT_PROTO_WEBSOCKET) {
+      if (resp_.http_status != 101) {
+        return;
+      }
+
+      // This is done for HTTP/2 frontend only.
+      auto accept = resp_.fs.header(http2::HD_SEC_WEBSOCKET_ACCEPT);
+      if (!accept) {
+        return;
+      }
+
+      std::array<uint8_t, base64::encode_length(20)> accept_buf;
+      auto expected =
+          http2::make_websocket_accept_token(accept_buf.data(), ws_key_);
+
+      upgraded_ = expected != "" && expected == accept->value;
+    } else {
+      upgraded_ = 200 <= resp_.http_status && resp_.http_status < 300;
+    }
+
+    return;
+  }
+
   if (resp_.http_status == 101) {
     // TODO Do more strict checking for upgrade headers
     upgraded_ = req_.upgrade_request;
@@ -787,7 +821,7 @@ void Downstream::inspect_http2_request() {
 void Downstream::inspect_http1_request() {
   if (req_.method == HTTP_CONNECT) {
     req_.upgrade_request = true;
-  } else {
+  } else if (req_.http_minor > 0) {
     auto upgrade = req_.fs.header(http2::HD_UPGRADE);
     if (upgrade) {
       const auto &val = upgrade->value;
@@ -797,6 +831,12 @@ void Downstream::inspect_http1_request() {
         req_.http2_upgrade_seen = true;
       } else {
         req_.upgrade_request = true;
+
+        // TODO Should we check Sec-WebSocket-Key, and
+        // Sec-WebSocket-Version as well?
+        if (util::strieq_l("websocket", val)) {
+          req_.connect_proto = CONNECT_PROTO_WEBSOCKET;
+        }
       }
     }
   }
@@ -1103,4 +1143,6 @@ bool Downstream::get_blocked_request_data_eof() const {
   return blocked_request_data_eof_;
 }
 
+void Downstream::set_ws_key(const StringRef &key) { ws_key_ = key; }
+
 } // namespace shrpx

src/shrpx_downstream.h

@@ -134,6 +134,12 @@ private:
   bool trailer_key_prev_;
 };
 
+// Protocols allowed in HTTP/2 :protocol header field.
+enum shrpx_connect_proto {
+  CONNECT_PROTO_NONE,
+  CONNECT_PROTO_WEBSOCKET,
+};
+
 struct Request {
   Request(BlockAllocator &balloc)
       : fs(balloc, 16),
@@ -142,6 +148,7 @@ struct Request {
         method(-1),
         http_major(1),
         http_minor(1),
+        connect_proto(CONNECT_PROTO_NONE),
         upgrade_request(false),
         http2_upgrade_seen(false),
         connection_close(false),
@@ -153,6 +160,12 @@ struct Request {
     unconsumed_body_length -= len;
   }
 
+  bool regular_connect_method() const {
+    return method == HTTP_CONNECT && !connect_proto;
+  }
+
+  bool extended_connect_method() const { return connect_proto; }
+
   FieldStore fs;
   // Timestamp when all request header fields are received.
   std::shared_ptr<Timestamp> tstamp;
@@ -176,6 +189,10 @@ struct Request {
   int method;
   // HTTP major and minor version
   int http_major, http_minor;
+  // connect_proto specified in HTTP/2 :protocol pseudo header field
+  // which enables extended CONNECT method.  This field is also set if
+  // WebSocket upgrade is requested in h1 frontend for convenience.
+  int connect_proto;
   // Returns true if the request is HTTP upgrade (HTTP Upgrade or
   // CONNECT method).  Upgrade to HTTP/2 is excluded.  For HTTP/2
   // Upgrade, check get_http2_upgrade_request().
@@ -283,11 +300,14 @@ public:
   // Returns true if output buffer is full. If underlying dconn_ is
   // NULL, this function always returns false.
   bool request_buf_full();
-  // Returns true if upgrade (HTTP Upgrade or CONNECT) is succeeded.
+  // Returns true if upgrade (HTTP Upgrade or CONNECT) is succeeded in
-  // This should not depend on inspect_http1_response().
+  // h1 backend.  This should not depend on inspect_http1_response().
-  void check_upgrade_fulfilled();
+  void check_upgrade_fulfilled_http1();
+  // Returns true if upgrade (HTTP Upgrade or CONNECT) is succeeded in
+  // h2 backend.
+  void check_upgrade_fulfilled_http2();
   // Returns true if the upgrade is succeeded as a result of the call
-  // check_upgrade_fulfilled().  HTTP/2 Upgrade is excluded.
+  // check_upgrade_fulfilled_http*().  HTTP/2 Upgrade is excluded.
   bool get_upgraded() const;
   // Inspects HTTP/2 request.
   void inspect_http2_request();
@@ -461,6 +481,8 @@ public:
   // field, returns 0.
   uint32_t get_affinity_cookie_to_send() const;
 
+  void set_ws_key(const StringRef &key);
+
   enum {
     EVENT_ERROR = 0x1,
     EVENT_TIMEOUT = 0x2,
@@ -500,6 +522,10 @@ private:
   DefaultMemchunks request_buf_;
   DefaultMemchunks response_buf_;
 
+  // The Sec-WebSocket-Key field sent to the peer.  This field is used
+  // if frontend uses RFC 8441 WebSocket bootstrapping via HTTP/2.
+  StringRef ws_key_;
+
   ev_timer upstream_rtimer_;
   ev_timer upstream_wtimer_;
 

src/shrpx_http2_downstream_connection.cc

@@ -105,7 +105,7 @@ int Http2DownstreamConnection::attach_downstream(Downstream *downstream) {
   auto &req = downstream_->request();
 
   // HTTP/2 disables HTTP Upgrade.
-  if (req.method != HTTP_CONNECT) {
+  if (req.method != HTTP_CONNECT && !req.connect_proto) {
     req.upgrade_request = false;
   }
 
@@ -231,7 +231,7 @@ int Http2DownstreamConnection::push_request_headers() {
   if (!downstream_) {
     return 0;
   }
-  if (!http2session_->can_push_request()) {
+  if (!http2session_->can_push_request(downstream_)) {
     // The HTTP2 session to the backend has not been established or
     // connection is now being checked.  This function will be called
     // again just after it is established.
@@ -244,6 +244,10 @@ int Http2DownstreamConnection::push_request_headers() {
 
   const auto &req = downstream_->request();
 
+  if (req.connect_proto && !http2session_->get_allow_connect_proto()) {
+    return -1;
+  }
+
   auto &balloc = downstream_->get_block_allocator();
 
   auto config = get_config();
@@ -251,7 +255,7 @@ int Http2DownstreamConnection::push_request_headers() {
   auto &http2conf = config->http2;
 
   auto no_host_rewrite = httpconf.no_host_rewrite || config->http2_proxy ||
-                         req.method == HTTP_CONNECT;
+                         req.regular_connect_method();
 
   // http2session_ has already in CONNECTED state, so we can get
   // addr_idx here.
@@ -272,25 +276,31 @@ int Http2DownstreamConnection::push_request_headers() {
     num_cookies = downstream_->count_crumble_request_cookie();
   }
 
-  // 10 means:
+  // 11 means:
   // 1. :method
   // 2. :scheme
   // 3. :path
   // 4. :authority (or host)
-  // 5. via (optional)
+  // 5. :protocol (optional)
-  // 6. x-forwarded-for (optional)
+  // 6. via (optional)
-  // 7. x-forwarded-proto (optional)
+  // 7. x-forwarded-for (optional)
-  // 8. te (optional)
+  // 8. x-forwarded-proto (optional)
-  // 9. forwarded (optional)
+  // 9. te (optional)
-  // 10. early-data (optional)
+  // 10. forwarded (optional)
+  // 11. early-data (optional)
   auto nva = std::vector<nghttp2_nv>();
-  nva.reserve(req.fs.headers().size() + 10 + num_cookies +
+  nva.reserve(req.fs.headers().size() + 11 + num_cookies +
               httpconf.add_request_headers.size());
 
-  nva.push_back(
+  if (req.connect_proto == CONNECT_PROTO_WEBSOCKET) {
-      http2::make_nv_ls_nocopy(":method", http2::to_method_string(req.method)));
+    nva.push_back(http2::make_nv_ll(":method", "CONNECT"));
+    nva.push_back(http2::make_nv_ll(":protocol", "websocket"));
+  } else {
+    nva.push_back(http2::make_nv_ls_nocopy(
+        ":method", http2::to_method_string(req.method)));
+  }
 
-  if (req.method != HTTP_CONNECT) {
+  if (!req.regular_connect_method()) {
     assert(!req.scheme.empty());
 
     auto addr = http2session_->get_addr();
@@ -308,7 +318,7 @@ int Http2DownstreamConnection::push_request_headers() {
       nva.push_back(http2::make_nv_ls_nocopy(":path", req.path));
     }
 
-    if (!req.no_authority) {
+    if (!req.no_authority || req.connect_proto) {
       nva.push_back(http2::make_nv_ls_nocopy(":authority", authority));
     } else {
       nva.push_back(http2::make_nv_ls_nocopy("host", authority));
@@ -326,7 +336,8 @@ int Http2DownstreamConnection::push_request_headers() {
       (fwdconf.strip_incoming ? http2::HDOP_STRIP_FORWARDED : 0) |
       (xffconf.strip_incoming ? http2::HDOP_STRIP_X_FORWARDED_FOR : 0) |
       (xfpconf.strip_incoming ? http2::HDOP_STRIP_X_FORWARDED_PROTO : 0) |
-      (earlydataconf.strip_incoming ? http2::HDOP_STRIP_EARLY_DATA : 0);
+      (earlydataconf.strip_incoming ? http2::HDOP_STRIP_EARLY_DATA : 0) |
+      http2::HDOP_STRIP_SEC_WEBSOCKET_KEY;
 
   http2::copy_headers_to_nva_nocopy(nva, req.fs.headers(), build_flags);
 
@@ -351,7 +362,7 @@ int Http2DownstreamConnection::push_request_headers() {
   if (fwdconf.params) {
     auto params = fwdconf.params;
 
-    if (config->http2_proxy || req.method == HTTP_CONNECT) {
+    if (config->http2_proxy || req.regular_connect_method()) {
       params &= ~FORWARDED_PROTO;
     }
 
@@ -392,7 +403,7 @@ int Http2DownstreamConnection::push_request_headers() {
     nva.push_back(http2::make_nv_ls_nocopy("x-forwarded-for", xff->value));
   }
 
-  if (!config->http2_proxy && req.method != HTTP_CONNECT) {
+  if (!config->http2_proxy && !req.regular_connect_method()) {
     auto xfp = xfpconf.strip_incoming
                    ? nullptr
                    : req.fs.header(http2::HD_X_FORWARDED_PROTO);
@@ -464,7 +475,7 @@ int Http2DownstreamConnection::push_request_headers() {
 
   // Add body as long as transfer-encoding is given even if
   // req.fs.content_length == 0 to forward trailer fields.
-  if (req.method == HTTP_CONNECT || transfer_encoding ||
+  if (req.method == HTTP_CONNECT || req.connect_proto || transfer_encoding ||
       req.fs.content_length > 0 || req.http2_expect_body) {
     // Request-body is expected.
     data_prd = {{}, http2_data_read_callback};

src/shrpx_http2_session.cc

@@ -199,7 +199,9 @@ Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx,
       raddr_(nullptr),
       state_(DISCONNECTED),
       connection_check_state_(CONNECTION_CHECK_NONE),
-      freelist_zone_(FREELIST_ZONE_NONE) {
+      freelist_zone_(FREELIST_ZONE_NONE),
+      settings_recved_(false),
+      allow_connect_proto_(false) {
   read_ = write_ = &Http2Session::noop;
 
   on_read_ = &Http2Session::read_noop;
@@ -1141,7 +1143,7 @@ int on_response_headers(Http2Session *http2session, Downstream *downstream,
   }
 
   downstream->set_response_state(Downstream::HEADER_COMPLETE);
-  downstream->check_upgrade_fulfilled();
+  downstream->check_upgrade_fulfilled_http2();
 
   if (downstream->get_upgraded()) {
     resp.connection_close = true;
@@ -1308,6 +1310,7 @@ int on_frame_recv_callback(nghttp2_session *session, const nghttp2_frame *frame,
   }
   case NGHTTP2_SETTINGS: {
     if ((frame->hd.flags & NGHTTP2_FLAG_ACK) == 0) {
+      http2session->on_settings_received(frame);
       return 0;
     }
 
@@ -1769,7 +1772,6 @@ int Http2Session::downstream_write() {
   for (;;) {
     const uint8_t *data;
     auto datalen = nghttp2_session_mem_send(session_, &data);
-
     if (datalen < 0) {
       SSLOG(ERROR, this) << "nghttp2_session_mem_send() returned error: "
                          << nghttp2_strerror(datalen);
@@ -1856,9 +1858,11 @@ int Http2Session::consume(int32_t stream_id, size_t len) {
   return 0;
 }
 
-bool Http2Session::can_push_request() const {
+bool Http2Session::can_push_request(const Downstream *downstream) const {
+  auto &req = downstream->request();
   return state_ == CONNECTED &&
-         connection_check_state_ == CONNECTION_CHECK_NONE;
+         connection_check_state_ == CONNECTION_CHECK_NONE &&
+         (!req.connect_proto || settings_recved_);
 }
 
 void Http2Session::start_checking_connection() {
@@ -1917,6 +1921,11 @@ void Http2Session::submit_pending_requests() {
       continue;
     }
 
+    auto &req = downstream->request();
+    if (req.connect_proto && !settings_recved_) {
+      continue;
+    }
+
     auto upstream = downstream->get_upstream();
 
     if (dconn->push_request_headers() != 0) {
@@ -2405,4 +2414,28 @@ void Http2Session::check_retire() {
 
 const Address *Http2Session::get_raddr() const { return raddr_; }
 
+void Http2Session::on_settings_received(const nghttp2_frame *frame) {
+  // TODO This effectively disallows nghttpx to change its behaviour
+  // based on the 2nd SETTINGS.
+  if (settings_recved_) {
+    return;
+  }
+
+  settings_recved_ = true;
+
+  for (size_t i = 0; i < frame->settings.niv; ++i) {
+    auto &ent = frame->settings.iv[i];
+    if (ent.settings_id == NGHTTP2_SETTINGS_ENABLE_CONNECT_PROTOCOL) {
+      allow_connect_proto_ = true;
+      break;
+    }
+  }
+
+  submit_pending_requests();
+}
+
+bool Http2Session::get_allow_connect_proto() const {
+  return allow_connect_proto_;
+}
+
 } // namespace shrpx

src/shrpx_http2_session.h

@@ -146,7 +146,7 @@ public:
   int consume(int32_t stream_id, size_t len);
 
   // Returns true if request can be issued on downstream connection.
-  bool can_push_request() const;
+  bool can_push_request(const Downstream *downstream) const;
   // Initiates the connection checking if downstream connection has
   // been established and connection checking is required.
   void start_checking_connection();
@@ -212,6 +212,12 @@ public:
   // Returns address used to connect to backend.  Could be nullptr.
   const Address *get_raddr() const;
 
+  // This is called when SETTINGS frame without ACK flag set is
+  // received.
+  void on_settings_received(const nghttp2_frame *frame);
+
+  bool get_allow_connect_proto() const;
+
   enum {
     // Disconnected
     DISCONNECTED,
@@ -280,6 +286,10 @@ private:
   int state_;
   int connection_check_state_;
   int freelist_zone_;
+  // true if SETTINGS without ACK is received from peer.
+  bool settings_recved_;
+  // true if peer enables RFC 8441 CONNECT protocol.
+  bool allow_connect_proto_;
 };
 
 nghttp2_session_callbacks *create_http2_downstream_callbacks();

src/shrpx_http2_upstream.cc

@@ -391,6 +391,17 @@ int Http2Upstream::on_request_headers(Downstream *downstream,
     }
   }
 
+  auto connect_proto = req.fs.header(http2::HD__PROTOCOL);
+  if (connect_proto) {
+    if (connect_proto->value != "websocket") {
+      if (error_reply(downstream, 400) != 0) {
+        return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
+      }
+      return 0;
+    }
+    req.connect_proto = CONNECT_PROTO_WEBSOCKET;
+  }
+
   if (!(frame->hd.flags & NGHTTP2_FLAG_END_STREAM)) {
     req.http2_expect_body = true;
   } else if (req.fs.content_length == -1) {
@@ -1025,7 +1036,7 @@ Http2Upstream::Http2Upstream(ClientHandler *handler)
   flow_control_ = true;
 
   // TODO Maybe call from outside?
-  std::array<nghttp2_settings_entry, 3> entry;
+  std::array<nghttp2_settings_entry, 4> entry;
   size_t nentry = 2;
 
   entry[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS;
@@ -1038,6 +1049,12 @@ Http2Upstream::Http2Upstream(ClientHandler *handler)
     entry[1].value = http2conf.upstream.window_size;
   }
 
+  if (!config->http2_proxy) {
+    entry[nentry].settings_id = NGHTTP2_SETTINGS_ENABLE_CONNECT_PROTOCOL;
+    entry[nentry].value = 1;
+    ++nentry;
+  }
+
   if (http2conf.upstream.decoder_dynamic_table_size !=
       NGHTTP2_DEFAULT_HEADER_TABLE_SIZE) {
     entry[nentry].settings_id = NGHTTP2_SETTINGS_HEADER_TABLE_SIZE;
@@ -1704,11 +1721,11 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) {
   nva.reserve(resp.fs.headers().size() + 5 +
               httpconf.add_response_headers.size());
 
+  if (downstream->get_non_final_response()) {
     auto response_status = http2::stringify_status(balloc, resp.http_status);
 
     nva.push_back(http2::make_nv_ls_nocopy(":status", response_status));
 
-  if (downstream->get_non_final_response()) {
     http2::copy_headers_to_nva_nocopy(nva, resp.fs.headers(),
                                       http2::HDOP_STRIP_ALL);
 
@@ -1730,8 +1747,19 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) {
     return 0;
   }
 
-  http2::copy_headers_to_nva_nocopy(
+  auto striphd_flags = http2::HDOP_STRIP_ALL & ~http2::HDOP_STRIP_VIA;
-      nva, resp.fs.headers(), http2::HDOP_STRIP_ALL & ~http2::HDOP_STRIP_VIA);
+  StringRef response_status;
+
+  if (req.connect_proto == CONNECT_PROTO_WEBSOCKET && resp.http_status == 101) {
+    response_status = http2::stringify_status(balloc, 200);
+    striphd_flags |= http2::HDOP_STRIP_SEC_WEBSOCKET_ACCEPT;
+  } else {
+    response_status = http2::stringify_status(balloc, resp.http_status);
+  }
+
+  nva.push_back(http2::make_nv_ls_nocopy(":status", response_status));
+
+  http2::copy_headers_to_nva_nocopy(nva, resp.fs.headers(), striphd_flags);
 
   if (!config->http2_proxy && !httpconf.no_server_rewrite) {
     nva.push_back(http2::make_nv_ls_nocopy("server", httpconf.server_name));
@@ -1742,7 +1770,7 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) {
     }
   }
 
-  if (req.method != HTTP_CONNECT || !downstream->get_upgraded()) {
+  if (!req.regular_connect_method() || !downstream->get_upgraded()) {
     auto affinity_cookie = downstream->get_affinity_cookie_to_send();
     if (affinity_cookie) {
       auto dconn = downstream->get_downstream_connection();
@@ -1916,7 +1944,7 @@ int Http2Upstream::on_downstream_abort_request_with_https_redirect(
 
 int Http2Upstream::redirect_to_https(Downstream *downstream) {
   auto &req = downstream->request();
-  if (req.method == HTTP_CONNECT || req.scheme != "http") {
+  if (req.regular_connect_method() || req.scheme != "http") {
     return error_reply(downstream, 400);
   }
 

src/shrpx_http_downstream_connection.cc

@@ -489,7 +489,7 @@ int HttpDownstreamConnection::push_request_headers() {
 
   auto &balloc = downstream_->get_block_allocator();
 
-  auto connect_method = req.method == HTTP_CONNECT;
+  auto connect_method = req.regular_connect_method();
 
   auto config = get_config();
   auto &httpconf = config->http;
@@ -513,7 +513,8 @@ int HttpDownstreamConnection::push_request_headers() {
   auto buf = downstream_->get_request_buf();
 
   // Assume that method and request path do not contain \r\n.
-  auto meth = http2::to_method_string(req.method);
+  auto meth = http2::to_method_string(
+      req.connect_proto == CONNECT_PROTO_WEBSOCKET ? HTTP_GET : req.method);
   buf->append(meth);
   buf->append(' ');
 
@@ -546,7 +547,8 @@ int HttpDownstreamConnection::push_request_headers() {
       (fwdconf.strip_incoming ? http2::HDOP_STRIP_FORWARDED : 0) |
       (xffconf.strip_incoming ? http2::HDOP_STRIP_X_FORWARDED_FOR : 0) |
       (xfpconf.strip_incoming ? http2::HDOP_STRIP_X_FORWARDED_PROTO : 0) |
-      (earlydataconf.strip_incoming ? http2::HDOP_STRIP_EARLY_DATA : 0);
+      (earlydataconf.strip_incoming ? http2::HDOP_STRIP_EARLY_DATA : 0) |
+      (req.http_major == 2 ? http2::HDOP_STRIP_SEC_WEBSOCKET_KEY : 0);
 
   http2::build_http1_headers_from_headers(buf, req.fs.headers(), build_flags);
 
@@ -559,16 +561,30 @@ int HttpDownstreamConnection::push_request_headers() {
 
   // set transfer-encoding only when content-length is unknown and
   // request body is expected.
-  if (!connect_method && req.http2_expect_body && req.fs.content_length == -1) {
+  if (req.method != HTTP_CONNECT && req.http2_expect_body &&
+      req.fs.content_length == -1) {
     downstream_->set_chunked_request(true);
     buf->append("Transfer-Encoding: chunked\r\n");
   }
 
-  if (req.connection_close) {
+  if (req.connect_proto == CONNECT_PROTO_WEBSOCKET) {
-    buf->append("Connection: close\r\n");
+    if (req.http_major == 2) {
+      std::array<uint8_t, 16> nonce;
+      util::random_bytes(std::begin(nonce), std::end(nonce),
+                         worker_->get_randgen());
+      auto iov = make_byte_ref(balloc, base64::encode_length(nonce.size()) + 1);
+      auto p = base64::encode(std::begin(nonce), std::end(nonce), iov.base);
+      *p = '\0';
+      auto key = StringRef{iov.base, p};
+      downstream_->set_ws_key(key);
+
+      buf->append("Sec-Websocket-Key: ");
+      buf->append(key);
+      buf->append("\r\n");
     }
 
-  if (!connect_method && req.upgrade_request) {
+    buf->append("Upgrade: websocket\r\nConnection: Upgrade\r\n");
+  } else if (!connect_method && req.upgrade_request) {
     auto connection = req.fs.header(http2::HD_CONNECTION);
     if (connection) {
       buf->append("Connection: ");
@@ -582,6 +598,8 @@ int HttpDownstreamConnection::push_request_headers() {
       buf->append((*upgrade).value);
       buf->append("\r\n");
     }
+  } else if (req.connection_close) {
+    buf->append("Connection: close\r\n");
   }
 
   auto upstream = downstream_->get_upstream();
@@ -708,7 +726,8 @@ int HttpDownstreamConnection::push_request_headers() {
   // Don't call signal_write() if we anticipate request body.  We call
   // signal_write() when we received request body chunk, and it
   // enables us to send headers and data in one writev system call.
-  if (connect_method || downstream_->get_blocked_request_buf()->rleft() ||
+  if (req.method == HTTP_CONNECT ||
+      downstream_->get_blocked_request_buf()->rleft() ||
       (!req.http2_expect_body && req.fs.content_length == 0)) {
     signal_write();
   }
@@ -954,7 +973,7 @@ int htp_hdrs_completecb(http_parser *htp) {
 
   // Check upgrade before processing non-final response, since if
   // upgrade succeeded, 101 response is treated as final in nghttpx.
-  downstream->check_upgrade_fulfilled();
+  downstream->check_upgrade_fulfilled_http1();
 
   if (downstream->get_non_final_response()) {
     // Reset content-length because we reuse same Downstream for the

src/shrpx_https_upstream.cc

@@ -44,6 +44,7 @@
 #include "http2.h"
 #include "util.h"
 #include "template.h"
+#include "base64.h"
 
 using namespace nghttp2;
 
@@ -1087,9 +1088,15 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) {
   buf->append('.');
   buf->append('0' + req.http_minor);
   buf->append(' ');
+  if (req.connect_proto && downstream->get_upgraded()) {
+    buf->append(http2::stringify_status(balloc, 101));
+    buf->append(' ');
+    buf->append(http2::get_reason_phrase(101));
+  } else {
     buf->append(http2::stringify_status(balloc, resp.http_status));
     buf->append(' ');
     buf->append(http2::get_reason_phrase(resp.http_status));
+  }
   buf->append("\r\n");
 
   auto config = get_config();
@@ -1139,6 +1146,22 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) {
   }
 
   if (!connect_method && downstream->get_upgraded()) {
+    if (req.connect_proto == CONNECT_PROTO_WEBSOCKET &&
+        resp.http_status == 200) {
+      buf->append("Upgrade: websocket\r\nConnection: Upgrade\r\n");
+      auto key = req.fs.header(http2::HD_SEC_WEBSOCKET_KEY);
+      if (!key || key->value.size() != base64::encode_length(16)) {
+        return -1;
+      }
+      std::array<uint8_t, base64::encode_length(20)> out;
+      auto accept = http2::make_websocket_accept_token(out.data(), key->value);
+      if (accept.empty()) {
+        return -1;
+      }
+      buf->append("Sec-WebSocket-Accept: ");
+      buf->append(accept);
+      buf->append("\r\n");
+    } else {
       auto connection = resp.fs.header(http2::HD_CONNECTION);
       if (connection) {
         buf->append("Connection: ");
@@ -1153,6 +1176,7 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) {
         buf->append("\r\n");
       }
     }
+  }
 
   if (!resp.fs.header(http2::HD_ALT_SVC)) {
     // We won't change or alter alt-svc from backend for now

src/util.cc

@@ -1453,7 +1453,8 @@ void EVP_MD_CTX_free(EVP_MD_CTX *ctx) { EVP_MD_CTX_destroy(ctx); }
 } // namespace
 #endif // !OPENSSL_1_1_API
 
-int sha256(uint8_t *res, const StringRef &s) {
+namespace {
+int message_digest(uint8_t *res, const EVP_MD *meth, const StringRef &s) {
   int rv;
 
   auto ctx = EVP_MD_CTX_new();
@@ -1463,7 +1464,7 @@ int sha256(uint8_t *res, const StringRef &s) {
 
   auto ctx_deleter = defer(EVP_MD_CTX_free, ctx);
 
-  rv = EVP_DigestInit_ex(ctx, EVP_sha256(), nullptr);
+  rv = EVP_DigestInit_ex(ctx, meth, nullptr);
   if (rv != 1) {
     return -1;
   }
@@ -1473,7 +1474,7 @@ int sha256(uint8_t *res, const StringRef &s) {
     return -1;
   }
 
-  unsigned int mdlen = 32;
+  unsigned int mdlen = EVP_MD_size(meth);
 
   rv = EVP_DigestFinal_ex(ctx, res, &mdlen);
   if (rv != 1) {
@@ -1482,6 +1483,15 @@ int sha256(uint8_t *res, const StringRef &s) {
 
   return 0;
 }
+} // namespace
+
+int sha256(uint8_t *res, const StringRef &s) {
+  return message_digest(res, EVP_sha256(), s);
+}
+
+int sha1(uint8_t *res, const StringRef &s) {
+  return message_digest(res, EVP_sha1(), s);
+}
 
 bool is_hex_string(const StringRef &s) {
   if (s.size() % 2) {

src/util.h

@@ -739,6 +739,13 @@ OutputIt random_alpha_digit(OutputIt first, OutputIt last, Generator &gen) {
   return first;
 }
 
+// Fills random bytes to the range [|first|, |last|).
+template <typename OutputIt, typename Generator>
+void random_bytes(OutputIt first, OutputIt last, Generator &gen) {
+  std::uniform_int_distribution<> dis(0, 255);
+  std::generate(first, last, [&dis, &gen]() { return dis(gen); });
+}
+
 template <typename OutputIterator, typename CharT, size_t N>
 OutputIterator copy_lit(OutputIterator it, CharT (&s)[N]) {
   return std::copy_n(s, N - 1, it);
@@ -753,6 +760,10 @@ uint32_t hash32(const StringRef &s);
 // returns 0 if it succeeds, or -1.
 int sha256(uint8_t *buf, const StringRef &s);
 
+// Computes SHA-1 of |s|, and stores it in |buf|.  This function
+// returns 0 if it succeeds, or -1.
+int sha1(uint8_t *buf, const StringRef &s);
+
 // Returns host from |hostport|.  If host cannot be found in
 // |hostport|, returns empty string.  The returned string might not be
 // NULL-terminated.