From d2a594a75340a02c312b70199a2532956968c594 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 11 Mar 2018 12:02:18 +0900 Subject: [PATCH] nghttpx: Implement RFC 8441 Bootstrapping WebSocket with HTTP/2 --- genheaderfunc.py | 3 + src/base64.h | 78 +++++++++++++++++++----- src/http2.cc | 47 +++++++++++++- src/http2.h | 16 +++++ src/shrpx_client_handler.cc | 2 +- src/shrpx_downstream.cc | 46 +++++++++++++- src/shrpx_downstream.h | 34 +++++++++-- src/shrpx_http2_downstream_connection.cc | 49 +++++++++------ src/shrpx_http2_session.cc | 43 +++++++++++-- src/shrpx_http2_session.h | 12 +++- src/shrpx_http2_upstream.cc | 46 +++++++++++--- src/shrpx_http_downstream_connection.cc | 39 +++++++++--- src/shrpx_https_upstream.cc | 50 +++++++++++---- src/util.cc | 16 ++++- src/util.h | 11 ++++ 15 files changed, 408 insertions(+), 84 deletions(-) diff --git a/genheaderfunc.py b/genheaderfunc.py index dfca88a3..9e26d931 100755 --- a/genheaderfunc.py +++ b/genheaderfunc.py @@ -9,6 +9,7 @@ HEADERS = [ ':scheme', ':status', ':host', # for spdy + ':protocol', 'expect', 'host', 'if-modified-since', @@ -32,6 +33,8 @@ HEADERS = [ "date", "content-type", "early-data", + "sec-websocket-accept", + "sec-websocket-key", # disallowed h1 headers 'connection', 'keep-alive', diff --git a/src/base64.h b/src/base64.h index c8b16e8c..1bd51afa 100644 --- a/src/base64.h +++ b/src/base64.h @@ -36,14 +36,17 @@ namespace nghttp2 { namespace base64 { +namespace { +constexpr char B64_CHARS[] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', +}; +} // namespace + template std::string encode(InputIt first, InputIt last) { - static constexpr char CHAR_TABLE[] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', - 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', - 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', - }; std::string res; size_t len = last - first; if (len == 0) { @@ -57,29 +60,72 @@ template std::string encode(InputIt first, InputIt last) { uint32_t n = static_cast(*first++) << 16; n += static_cast(*first++) << 8; n += static_cast(*first++); - *p++ = CHAR_TABLE[n >> 18]; - *p++ = CHAR_TABLE[(n >> 12) & 0x3fu]; - *p++ = CHAR_TABLE[(n >> 6) & 0x3fu]; - *p++ = CHAR_TABLE[n & 0x3fu]; + *p++ = B64_CHARS[n >> 18]; + *p++ = B64_CHARS[(n >> 12) & 0x3fu]; + *p++ = B64_CHARS[(n >> 6) & 0x3fu]; + *p++ = B64_CHARS[n & 0x3fu]; } if (r == 2) { uint32_t n = static_cast(*first++) << 16; n += static_cast(*first++) << 8; - *p++ = CHAR_TABLE[n >> 18]; - *p++ = CHAR_TABLE[(n >> 12) & 0x3fu]; - *p++ = CHAR_TABLE[(n >> 6) & 0x3fu]; + *p++ = B64_CHARS[n >> 18]; + *p++ = B64_CHARS[(n >> 12) & 0x3fu]; + *p++ = B64_CHARS[(n >> 6) & 0x3fu]; *p++ = '='; } else if (r == 1) { uint32_t n = static_cast(*first++) << 16; - *p++ = CHAR_TABLE[n >> 18]; - *p++ = CHAR_TABLE[(n >> 12) & 0x3fu]; + *p++ = B64_CHARS[n >> 18]; + *p++ = B64_CHARS[(n >> 12) & 0x3fu]; *p++ = '='; *p++ = '='; } return res; } +constexpr size_t encode_length(size_t n) { return (n + 2) / 3 * 4; } + +template +OutputIt encode(InputIt first, InputIt last, OutputIt d_first) { + size_t len = last - first; + if (len == 0) { + return d_first; + } + auto r = len % 3; + auto j = last - r; + auto p = d_first; + while (first != j) { + uint32_t n = static_cast(*first++) << 16; + n += static_cast(*first++) << 8; + n += static_cast(*first++); + *p++ = B64_CHARS[n >> 18]; + *p++ = B64_CHARS[(n >> 12) & 0x3fu]; + *p++ = B64_CHARS[(n >> 6) & 0x3fu]; + *p++ = B64_CHARS[n & 0x3fu]; + } + + switch (r) { + case 2: { + uint32_t n = static_cast(*first++) << 16; + n += static_cast(*first++) << 8; + *p++ = B64_CHARS[n >> 18]; + *p++ = B64_CHARS[(n >> 12) & 0x3fu]; + *p++ = B64_CHARS[(n >> 6) & 0x3fu]; + *p++ = '='; + break; + } + case 1: { + uint32_t n = static_cast(*first++) << 16; + *p++ = B64_CHARS[n >> 18]; + *p++ = B64_CHARS[(n >> 12) & 0x3fu]; + *p++ = '='; + *p++ = '='; + break; + } + } + return p; +} + template InputIt next_decode_input(InputIt first, InputIt last, const int *tbl) { for (; first != last; ++first) { diff --git a/src/http2.cc b/src/http2.cc index 01709a84..b88b7a66 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -394,6 +394,16 @@ void copy_headers_to_nva_internal(std::vector &nva, continue; } break; + case HD_SEC_WEBSOCKET_ACCEPT: + if (flags & HDOP_STRIP_SEC_WEBSOCKET_ACCEPT) { + continue; + } + break; + case HD_SEC_WEBSOCKET_KEY: + if (flags & HDOP_STRIP_SEC_WEBSOCKET_KEY) { + continue; + } + break; case HD_FORWARDED: if (flags & HDOP_STRIP_FORWARDED) { continue; @@ -834,6 +844,11 @@ int lookup_token(const uint8_t *name, size_t namelen) { return HD_FORWARDED; } break; + case 'l': + if (util::streq_l(":protoco", name, 8)) { + return HD__PROTOCOL; + } + break; } break; case 10: @@ -942,6 +957,20 @@ int lookup_token(const uint8_t *name, size_t namelen) { return HD_X_FORWARDED_PROTO; } break; + case 'y': + if (util::streq_l("sec-websocket-ke", name, 16)) { + return HD_SEC_WEBSOCKET_KEY; + } + break; + } + break; + case 20: + switch (name[19]) { + case 't': + if (util::streq_l("sec-websocket-accep", name, 19)) { + return HD_SEC_WEBSOCKET_ACCEPT; + } + break; } break; } @@ -1331,7 +1360,8 @@ std::string path_join(const StringRef &base_path, const StringRef &base_query, } bool expect_response_body(int status_code) { - return status_code / 100 != 1 && status_code != 304 && status_code != 204; + return status_code == 101 || + (status_code / 100 != 1 && status_code != 304 && status_code != 204); } bool expect_response_body(const std::string &method, int status_code) { @@ -1829,6 +1859,21 @@ bool contains_trailers(const StringRef &s) { } } +StringRef make_websocket_accept_token(uint8_t *dest, const StringRef &key) { + static constexpr uint8_t magic[] = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; + std::array s; + auto p = std::copy(std::begin(key), std::end(key), std::begin(s)); + std::copy_n(magic, str_size(magic), p); + + std::array h; + if (util::sha1(h.data(), StringRef{std::begin(s), std::end(s)}) != 0) { + return StringRef{}; + } + + auto end = base64::encode(std::begin(h), std::end(h), dest); + return StringRef{dest, end}; +} + } // namespace http2 } // namespace nghttp2 diff --git a/src/http2.h b/src/http2.h index be4396e7..9e7d7491 100644 --- a/src/http2.h +++ b/src/http2.h @@ -41,6 +41,7 @@ #include "memchunk.h" #include "template.h" #include "allocator.h" +#include "base64.h" namespace nghttp2 { @@ -210,6 +211,12 @@ enum HeaderBuildOp { HDOP_STRIP_ALL = HDOP_STRIP_FORWARDED | HDOP_STRIP_X_FORWARDED_FOR | HDOP_STRIP_X_FORWARDED_PROTO | HDOP_STRIP_VIA | HDOP_STRIP_EARLY_DATA, + // Sec-WebSocket-Accept header field must be stripped. If this flag + // is not set, all Sec-WebSocket-Accept header fields are added. + HDOP_STRIP_SEC_WEBSOCKET_ACCEPT = 1 << 5, + // Sec-WebSocket-Key header field must be stripped. If this flag is + // not set, all Sec-WebSocket-Key header fields are added. + HDOP_STRIP_SEC_WEBSOCKET_KEY = 1 << 6, }; // Appends headers in |headers| to |nv|. |headers| must be indexed @@ -297,6 +304,7 @@ enum { HD__HOST, HD__METHOD, HD__PATH, + HD__PROTOCOL, HD__SCHEME, HD__STATUS, HD_ACCEPT_ENCODING, @@ -318,6 +326,8 @@ enum { HD_LINK, HD_LOCATION, HD_PROXY_CONNECTION, + HD_SEC_WEBSOCKET_ACCEPT, + HD_SEC_WEBSOCKET_KEY, HD_SERVER, HD_TE, HD_TRAILER, @@ -421,6 +431,12 @@ StringRef copy_lower(BlockAllocator &balloc, const StringRef &src); // Returns true if te header field value |s| contains "trailers". bool contains_trailers(const StringRef &s); +// Creates Sec-WebSocket-Accept value for |key|. The capacity of +// buffer pointed by |dest| must have at least 24 bytes (base64 +// encoded length of 16 bytes data). It returns empty string in case +// of error. +StringRef make_websocket_accept_token(uint8_t *dest, const StringRef &key); + } // namespace http2 } // namespace nghttp2 diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 472db835..d718eddc 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -979,7 +979,7 @@ ClientHandler::get_downstream_connection(int &err, Downstream *downstream, StringRef path; // CONNECT method does not have path. But we requires path in // host-path mapping. As workaround, we assume that path is "/". - if (req.method != HTTP_CONNECT) { + if (!req.regular_connect_method()) { path = req.path; } diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index d344ef15..100cd965 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -763,13 +763,47 @@ bool Downstream::validate_response_recv_body_length() const { return true; } -void Downstream::check_upgrade_fulfilled() { +void Downstream::check_upgrade_fulfilled_http2() { if (req_.method == HTTP_CONNECT) { + // This handles nonzero req_.connect_proto as well. upgraded_ = 200 <= resp_.http_status && resp_.http_status < 300; return; } + if (req_.connect_proto == CONNECT_PROTO_WEBSOCKET) { + // h1 frontend requests WebSocket upgrade + upgraded_ = resp_.http_status == 200; + + return; + } +} + +void Downstream::check_upgrade_fulfilled_http1() { + if (req_.method == HTTP_CONNECT) { + if (req_.connect_proto == CONNECT_PROTO_WEBSOCKET) { + if (resp_.http_status != 101) { + return; + } + + // This is done for HTTP/2 frontend only. + auto accept = resp_.fs.header(http2::HD_SEC_WEBSOCKET_ACCEPT); + if (!accept) { + return; + } + + std::array accept_buf; + auto expected = + http2::make_websocket_accept_token(accept_buf.data(), ws_key_); + + upgraded_ = expected != "" && expected == accept->value; + } else { + upgraded_ = 200 <= resp_.http_status && resp_.http_status < 300; + } + + return; + } + if (resp_.http_status == 101) { // TODO Do more strict checking for upgrade headers upgraded_ = req_.upgrade_request; @@ -787,7 +821,7 @@ void Downstream::inspect_http2_request() { void Downstream::inspect_http1_request() { if (req_.method == HTTP_CONNECT) { req_.upgrade_request = true; - } else { + } else if (req_.http_minor > 0) { auto upgrade = req_.fs.header(http2::HD_UPGRADE); if (upgrade) { const auto &val = upgrade->value; @@ -797,6 +831,12 @@ void Downstream::inspect_http1_request() { req_.http2_upgrade_seen = true; } else { req_.upgrade_request = true; + + // TODO Should we check Sec-WebSocket-Key, and + // Sec-WebSocket-Version as well? + if (util::strieq_l("websocket", val)) { + req_.connect_proto = CONNECT_PROTO_WEBSOCKET; + } } } } @@ -1103,4 +1143,6 @@ bool Downstream::get_blocked_request_data_eof() const { return blocked_request_data_eof_; } +void Downstream::set_ws_key(const StringRef &key) { ws_key_ = key; } + } // namespace shrpx diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 3e121eb4..4ea9df33 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -134,6 +134,12 @@ private: bool trailer_key_prev_; }; +// Protocols allowed in HTTP/2 :protocol header field. +enum shrpx_connect_proto { + CONNECT_PROTO_NONE, + CONNECT_PROTO_WEBSOCKET, +}; + struct Request { Request(BlockAllocator &balloc) : fs(balloc, 16), @@ -142,6 +148,7 @@ struct Request { method(-1), http_major(1), http_minor(1), + connect_proto(CONNECT_PROTO_NONE), upgrade_request(false), http2_upgrade_seen(false), connection_close(false), @@ -153,6 +160,12 @@ struct Request { unconsumed_body_length -= len; } + bool regular_connect_method() const { + return method == HTTP_CONNECT && !connect_proto; + } + + bool extended_connect_method() const { return connect_proto; } + FieldStore fs; // Timestamp when all request header fields are received. std::shared_ptr tstamp; @@ -176,6 +189,10 @@ struct Request { int method; // HTTP major and minor version int http_major, http_minor; + // connect_proto specified in HTTP/2 :protocol pseudo header field + // which enables extended CONNECT method. This field is also set if + // WebSocket upgrade is requested in h1 frontend for convenience. + int connect_proto; // Returns true if the request is HTTP upgrade (HTTP Upgrade or // CONNECT method). Upgrade to HTTP/2 is excluded. For HTTP/2 // Upgrade, check get_http2_upgrade_request(). @@ -283,11 +300,14 @@ public: // Returns true if output buffer is full. If underlying dconn_ is // NULL, this function always returns false. bool request_buf_full(); - // Returns true if upgrade (HTTP Upgrade or CONNECT) is succeeded. - // This should not depend on inspect_http1_response(). - void check_upgrade_fulfilled(); + // Returns true if upgrade (HTTP Upgrade or CONNECT) is succeeded in + // h1 backend. This should not depend on inspect_http1_response(). + void check_upgrade_fulfilled_http1(); + // Returns true if upgrade (HTTP Upgrade or CONNECT) is succeeded in + // h2 backend. + void check_upgrade_fulfilled_http2(); // Returns true if the upgrade is succeeded as a result of the call - // check_upgrade_fulfilled(). HTTP/2 Upgrade is excluded. + // check_upgrade_fulfilled_http*(). HTTP/2 Upgrade is excluded. bool get_upgraded() const; // Inspects HTTP/2 request. void inspect_http2_request(); @@ -461,6 +481,8 @@ public: // field, returns 0. uint32_t get_affinity_cookie_to_send() const; + void set_ws_key(const StringRef &key); + enum { EVENT_ERROR = 0x1, EVENT_TIMEOUT = 0x2, @@ -500,6 +522,10 @@ private: DefaultMemchunks request_buf_; DefaultMemchunks response_buf_; + // The Sec-WebSocket-Key field sent to the peer. This field is used + // if frontend uses RFC 8441 WebSocket bootstrapping via HTTP/2. + StringRef ws_key_; + ev_timer upstream_rtimer_; ev_timer upstream_wtimer_; diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index beb45d9c..1a4acd00 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -105,7 +105,7 @@ int Http2DownstreamConnection::attach_downstream(Downstream *downstream) { auto &req = downstream_->request(); // HTTP/2 disables HTTP Upgrade. - if (req.method != HTTP_CONNECT) { + if (req.method != HTTP_CONNECT && !req.connect_proto) { req.upgrade_request = false; } @@ -231,7 +231,7 @@ int Http2DownstreamConnection::push_request_headers() { if (!downstream_) { return 0; } - if (!http2session_->can_push_request()) { + if (!http2session_->can_push_request(downstream_)) { // The HTTP2 session to the backend has not been established or // connection is now being checked. This function will be called // again just after it is established. @@ -244,6 +244,10 @@ int Http2DownstreamConnection::push_request_headers() { const auto &req = downstream_->request(); + if (req.connect_proto && !http2session_->get_allow_connect_proto()) { + return -1; + } + auto &balloc = downstream_->get_block_allocator(); auto config = get_config(); @@ -251,7 +255,7 @@ int Http2DownstreamConnection::push_request_headers() { auto &http2conf = config->http2; auto no_host_rewrite = httpconf.no_host_rewrite || config->http2_proxy || - req.method == HTTP_CONNECT; + req.regular_connect_method(); // http2session_ has already in CONNECTED state, so we can get // addr_idx here. @@ -272,25 +276,31 @@ int Http2DownstreamConnection::push_request_headers() { num_cookies = downstream_->count_crumble_request_cookie(); } - // 10 means: + // 11 means: // 1. :method // 2. :scheme // 3. :path // 4. :authority (or host) - // 5. via (optional) - // 6. x-forwarded-for (optional) - // 7. x-forwarded-proto (optional) - // 8. te (optional) - // 9. forwarded (optional) - // 10. early-data (optional) + // 5. :protocol (optional) + // 6. via (optional) + // 7. x-forwarded-for (optional) + // 8. x-forwarded-proto (optional) + // 9. te (optional) + // 10. forwarded (optional) + // 11. early-data (optional) auto nva = std::vector(); - nva.reserve(req.fs.headers().size() + 10 + num_cookies + + nva.reserve(req.fs.headers().size() + 11 + num_cookies + httpconf.add_request_headers.size()); - nva.push_back( - http2::make_nv_ls_nocopy(":method", http2::to_method_string(req.method))); + if (req.connect_proto == CONNECT_PROTO_WEBSOCKET) { + nva.push_back(http2::make_nv_ll(":method", "CONNECT")); + nva.push_back(http2::make_nv_ll(":protocol", "websocket")); + } else { + nva.push_back(http2::make_nv_ls_nocopy( + ":method", http2::to_method_string(req.method))); + } - if (req.method != HTTP_CONNECT) { + if (!req.regular_connect_method()) { assert(!req.scheme.empty()); auto addr = http2session_->get_addr(); @@ -308,7 +318,7 @@ int Http2DownstreamConnection::push_request_headers() { nva.push_back(http2::make_nv_ls_nocopy(":path", req.path)); } - if (!req.no_authority) { + if (!req.no_authority || req.connect_proto) { nva.push_back(http2::make_nv_ls_nocopy(":authority", authority)); } else { nva.push_back(http2::make_nv_ls_nocopy("host", authority)); @@ -326,7 +336,8 @@ int Http2DownstreamConnection::push_request_headers() { (fwdconf.strip_incoming ? http2::HDOP_STRIP_FORWARDED : 0) | (xffconf.strip_incoming ? http2::HDOP_STRIP_X_FORWARDED_FOR : 0) | (xfpconf.strip_incoming ? http2::HDOP_STRIP_X_FORWARDED_PROTO : 0) | - (earlydataconf.strip_incoming ? http2::HDOP_STRIP_EARLY_DATA : 0); + (earlydataconf.strip_incoming ? http2::HDOP_STRIP_EARLY_DATA : 0) | + http2::HDOP_STRIP_SEC_WEBSOCKET_KEY; http2::copy_headers_to_nva_nocopy(nva, req.fs.headers(), build_flags); @@ -351,7 +362,7 @@ int Http2DownstreamConnection::push_request_headers() { if (fwdconf.params) { auto params = fwdconf.params; - if (config->http2_proxy || req.method == HTTP_CONNECT) { + if (config->http2_proxy || req.regular_connect_method()) { params &= ~FORWARDED_PROTO; } @@ -392,7 +403,7 @@ int Http2DownstreamConnection::push_request_headers() { nva.push_back(http2::make_nv_ls_nocopy("x-forwarded-for", xff->value)); } - if (!config->http2_proxy && req.method != HTTP_CONNECT) { + if (!config->http2_proxy && !req.regular_connect_method()) { auto xfp = xfpconf.strip_incoming ? nullptr : req.fs.header(http2::HD_X_FORWARDED_PROTO); @@ -464,7 +475,7 @@ int Http2DownstreamConnection::push_request_headers() { // Add body as long as transfer-encoding is given even if // req.fs.content_length == 0 to forward trailer fields. - if (req.method == HTTP_CONNECT || transfer_encoding || + if (req.method == HTTP_CONNECT || req.connect_proto || transfer_encoding || req.fs.content_length > 0 || req.http2_expect_body) { // Request-body is expected. data_prd = {{}, http2_data_read_callback}; diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 20186d08..8c4ece3c 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -199,7 +199,9 @@ Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, raddr_(nullptr), state_(DISCONNECTED), connection_check_state_(CONNECTION_CHECK_NONE), - freelist_zone_(FREELIST_ZONE_NONE) { + freelist_zone_(FREELIST_ZONE_NONE), + settings_recved_(false), + allow_connect_proto_(false) { read_ = write_ = &Http2Session::noop; on_read_ = &Http2Session::read_noop; @@ -1141,7 +1143,7 @@ int on_response_headers(Http2Session *http2session, Downstream *downstream, } downstream->set_response_state(Downstream::HEADER_COMPLETE); - downstream->check_upgrade_fulfilled(); + downstream->check_upgrade_fulfilled_http2(); if (downstream->get_upgraded()) { resp.connection_close = true; @@ -1308,6 +1310,7 @@ int on_frame_recv_callback(nghttp2_session *session, const nghttp2_frame *frame, } case NGHTTP2_SETTINGS: { if ((frame->hd.flags & NGHTTP2_FLAG_ACK) == 0) { + http2session->on_settings_received(frame); return 0; } @@ -1769,7 +1772,6 @@ int Http2Session::downstream_write() { for (;;) { const uint8_t *data; auto datalen = nghttp2_session_mem_send(session_, &data); - if (datalen < 0) { SSLOG(ERROR, this) << "nghttp2_session_mem_send() returned error: " << nghttp2_strerror(datalen); @@ -1856,9 +1858,11 @@ int Http2Session::consume(int32_t stream_id, size_t len) { return 0; } -bool Http2Session::can_push_request() const { +bool Http2Session::can_push_request(const Downstream *downstream) const { + auto &req = downstream->request(); return state_ == CONNECTED && - connection_check_state_ == CONNECTION_CHECK_NONE; + connection_check_state_ == CONNECTION_CHECK_NONE && + (!req.connect_proto || settings_recved_); } void Http2Session::start_checking_connection() { @@ -1917,6 +1921,11 @@ void Http2Session::submit_pending_requests() { continue; } + auto &req = downstream->request(); + if (req.connect_proto && !settings_recved_) { + continue; + } + auto upstream = downstream->get_upstream(); if (dconn->push_request_headers() != 0) { @@ -2405,4 +2414,28 @@ void Http2Session::check_retire() { const Address *Http2Session::get_raddr() const { return raddr_; } +void Http2Session::on_settings_received(const nghttp2_frame *frame) { + // TODO This effectively disallows nghttpx to change its behaviour + // based on the 2nd SETTINGS. + if (settings_recved_) { + return; + } + + settings_recved_ = true; + + for (size_t i = 0; i < frame->settings.niv; ++i) { + auto &ent = frame->settings.iv[i]; + if (ent.settings_id == NGHTTP2_SETTINGS_ENABLE_CONNECT_PROTOCOL) { + allow_connect_proto_ = true; + break; + } + } + + submit_pending_requests(); +} + +bool Http2Session::get_allow_connect_proto() const { + return allow_connect_proto_; +} + } // namespace shrpx diff --git a/src/shrpx_http2_session.h b/src/shrpx_http2_session.h index ef2c0aff..09b865bc 100644 --- a/src/shrpx_http2_session.h +++ b/src/shrpx_http2_session.h @@ -146,7 +146,7 @@ public: int consume(int32_t stream_id, size_t len); // Returns true if request can be issued on downstream connection. - bool can_push_request() const; + bool can_push_request(const Downstream *downstream) const; // Initiates the connection checking if downstream connection has // been established and connection checking is required. void start_checking_connection(); @@ -212,6 +212,12 @@ public: // Returns address used to connect to backend. Could be nullptr. const Address *get_raddr() const; + // This is called when SETTINGS frame without ACK flag set is + // received. + void on_settings_received(const nghttp2_frame *frame); + + bool get_allow_connect_proto() const; + enum { // Disconnected DISCONNECTED, @@ -280,6 +286,10 @@ private: int state_; int connection_check_state_; int freelist_zone_; + // true if SETTINGS without ACK is received from peer. + bool settings_recved_; + // true if peer enables RFC 8441 CONNECT protocol. + bool allow_connect_proto_; }; nghttp2_session_callbacks *create_http2_downstream_callbacks(); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index fdf988cc..ac0b9438 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -391,6 +391,17 @@ int Http2Upstream::on_request_headers(Downstream *downstream, } } + auto connect_proto = req.fs.header(http2::HD__PROTOCOL); + if (connect_proto) { + if (connect_proto->value != "websocket") { + if (error_reply(downstream, 400) != 0) { + return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + } + return 0; + } + req.connect_proto = CONNECT_PROTO_WEBSOCKET; + } + if (!(frame->hd.flags & NGHTTP2_FLAG_END_STREAM)) { req.http2_expect_body = true; } else if (req.fs.content_length == -1) { @@ -1025,7 +1036,7 @@ Http2Upstream::Http2Upstream(ClientHandler *handler) flow_control_ = true; // TODO Maybe call from outside? - std::array entry; + std::array entry; size_t nentry = 2; entry[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS; @@ -1038,6 +1049,12 @@ Http2Upstream::Http2Upstream(ClientHandler *handler) entry[1].value = http2conf.upstream.window_size; } + if (!config->http2_proxy) { + entry[nentry].settings_id = NGHTTP2_SETTINGS_ENABLE_CONNECT_PROTOCOL; + entry[nentry].value = 1; + ++nentry; + } + if (http2conf.upstream.decoder_dynamic_table_size != NGHTTP2_DEFAULT_HEADER_TABLE_SIZE) { entry[nentry].settings_id = NGHTTP2_SETTINGS_HEADER_TABLE_SIZE; @@ -1704,11 +1721,11 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { nva.reserve(resp.fs.headers().size() + 5 + httpconf.add_response_headers.size()); - auto response_status = http2::stringify_status(balloc, resp.http_status); - - nva.push_back(http2::make_nv_ls_nocopy(":status", response_status)); - if (downstream->get_non_final_response()) { + auto response_status = http2::stringify_status(balloc, resp.http_status); + + nva.push_back(http2::make_nv_ls_nocopy(":status", response_status)); + http2::copy_headers_to_nva_nocopy(nva, resp.fs.headers(), http2::HDOP_STRIP_ALL); @@ -1730,8 +1747,19 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { return 0; } - http2::copy_headers_to_nva_nocopy( - nva, resp.fs.headers(), http2::HDOP_STRIP_ALL & ~http2::HDOP_STRIP_VIA); + auto striphd_flags = http2::HDOP_STRIP_ALL & ~http2::HDOP_STRIP_VIA; + StringRef response_status; + + if (req.connect_proto == CONNECT_PROTO_WEBSOCKET && resp.http_status == 101) { + response_status = http2::stringify_status(balloc, 200); + striphd_flags |= http2::HDOP_STRIP_SEC_WEBSOCKET_ACCEPT; + } else { + response_status = http2::stringify_status(balloc, resp.http_status); + } + + nva.push_back(http2::make_nv_ls_nocopy(":status", response_status)); + + http2::copy_headers_to_nva_nocopy(nva, resp.fs.headers(), striphd_flags); if (!config->http2_proxy && !httpconf.no_server_rewrite) { nva.push_back(http2::make_nv_ls_nocopy("server", httpconf.server_name)); @@ -1742,7 +1770,7 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { } } - if (req.method != HTTP_CONNECT || !downstream->get_upgraded()) { + if (!req.regular_connect_method() || !downstream->get_upgraded()) { auto affinity_cookie = downstream->get_affinity_cookie_to_send(); if (affinity_cookie) { auto dconn = downstream->get_downstream_connection(); @@ -1916,7 +1944,7 @@ int Http2Upstream::on_downstream_abort_request_with_https_redirect( int Http2Upstream::redirect_to_https(Downstream *downstream) { auto &req = downstream->request(); - if (req.method == HTTP_CONNECT || req.scheme != "http") { + if (req.regular_connect_method() || req.scheme != "http") { return error_reply(downstream, 400); } diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index f087d127..9fd3c124 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -489,7 +489,7 @@ int HttpDownstreamConnection::push_request_headers() { auto &balloc = downstream_->get_block_allocator(); - auto connect_method = req.method == HTTP_CONNECT; + auto connect_method = req.regular_connect_method(); auto config = get_config(); auto &httpconf = config->http; @@ -513,7 +513,8 @@ int HttpDownstreamConnection::push_request_headers() { auto buf = downstream_->get_request_buf(); // Assume that method and request path do not contain \r\n. - auto meth = http2::to_method_string(req.method); + auto meth = http2::to_method_string( + req.connect_proto == CONNECT_PROTO_WEBSOCKET ? HTTP_GET : req.method); buf->append(meth); buf->append(' '); @@ -546,7 +547,8 @@ int HttpDownstreamConnection::push_request_headers() { (fwdconf.strip_incoming ? http2::HDOP_STRIP_FORWARDED : 0) | (xffconf.strip_incoming ? http2::HDOP_STRIP_X_FORWARDED_FOR : 0) | (xfpconf.strip_incoming ? http2::HDOP_STRIP_X_FORWARDED_PROTO : 0) | - (earlydataconf.strip_incoming ? http2::HDOP_STRIP_EARLY_DATA : 0); + (earlydataconf.strip_incoming ? http2::HDOP_STRIP_EARLY_DATA : 0) | + (req.http_major == 2 ? http2::HDOP_STRIP_SEC_WEBSOCKET_KEY : 0); http2::build_http1_headers_from_headers(buf, req.fs.headers(), build_flags); @@ -559,16 +561,30 @@ int HttpDownstreamConnection::push_request_headers() { // set transfer-encoding only when content-length is unknown and // request body is expected. - if (!connect_method && req.http2_expect_body && req.fs.content_length == -1) { + if (req.method != HTTP_CONNECT && req.http2_expect_body && + req.fs.content_length == -1) { downstream_->set_chunked_request(true); buf->append("Transfer-Encoding: chunked\r\n"); } - if (req.connection_close) { - buf->append("Connection: close\r\n"); - } + if (req.connect_proto == CONNECT_PROTO_WEBSOCKET) { + if (req.http_major == 2) { + std::array nonce; + util::random_bytes(std::begin(nonce), std::end(nonce), + worker_->get_randgen()); + auto iov = make_byte_ref(balloc, base64::encode_length(nonce.size()) + 1); + auto p = base64::encode(std::begin(nonce), std::end(nonce), iov.base); + *p = '\0'; + auto key = StringRef{iov.base, p}; + downstream_->set_ws_key(key); - if (!connect_method && req.upgrade_request) { + buf->append("Sec-Websocket-Key: "); + buf->append(key); + buf->append("\r\n"); + } + + buf->append("Upgrade: websocket\r\nConnection: Upgrade\r\n"); + } else if (!connect_method && req.upgrade_request) { auto connection = req.fs.header(http2::HD_CONNECTION); if (connection) { buf->append("Connection: "); @@ -582,6 +598,8 @@ int HttpDownstreamConnection::push_request_headers() { buf->append((*upgrade).value); buf->append("\r\n"); } + } else if (req.connection_close) { + buf->append("Connection: close\r\n"); } auto upstream = downstream_->get_upstream(); @@ -708,7 +726,8 @@ int HttpDownstreamConnection::push_request_headers() { // Don't call signal_write() if we anticipate request body. We call // signal_write() when we received request body chunk, and it // enables us to send headers and data in one writev system call. - if (connect_method || downstream_->get_blocked_request_buf()->rleft() || + if (req.method == HTTP_CONNECT || + downstream_->get_blocked_request_buf()->rleft() || (!req.http2_expect_body && req.fs.content_length == 0)) { signal_write(); } @@ -954,7 +973,7 @@ int htp_hdrs_completecb(http_parser *htp) { // Check upgrade before processing non-final response, since if // upgrade succeeded, 101 response is treated as final in nghttpx. - downstream->check_upgrade_fulfilled(); + downstream->check_upgrade_fulfilled_http1(); if (downstream->get_non_final_response()) { // Reset content-length because we reuse same Downstream for the diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index e0765507..ce800cc8 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -44,6 +44,7 @@ #include "http2.h" #include "util.h" #include "template.h" +#include "base64.h" using namespace nghttp2; @@ -1087,9 +1088,15 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) { buf->append('.'); buf->append('0' + req.http_minor); buf->append(' '); - buf->append(http2::stringify_status(balloc, resp.http_status)); - buf->append(' '); - buf->append(http2::get_reason_phrase(resp.http_status)); + if (req.connect_proto && downstream->get_upgraded()) { + buf->append(http2::stringify_status(balloc, 101)); + buf->append(' '); + buf->append(http2::get_reason_phrase(101)); + } else { + buf->append(http2::stringify_status(balloc, resp.http_status)); + buf->append(' '); + buf->append(http2::get_reason_phrase(resp.http_status)); + } buf->append("\r\n"); auto config = get_config(); @@ -1139,18 +1146,35 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) { } if (!connect_method && downstream->get_upgraded()) { - auto connection = resp.fs.header(http2::HD_CONNECTION); - if (connection) { - buf->append("Connection: "); - buf->append((*connection).value); + if (req.connect_proto == CONNECT_PROTO_WEBSOCKET && + resp.http_status == 200) { + buf->append("Upgrade: websocket\r\nConnection: Upgrade\r\n"); + auto key = req.fs.header(http2::HD_SEC_WEBSOCKET_KEY); + if (!key || key->value.size() != base64::encode_length(16)) { + return -1; + } + std::array out; + auto accept = http2::make_websocket_accept_token(out.data(), key->value); + if (accept.empty()) { + return -1; + } + buf->append("Sec-WebSocket-Accept: "); + buf->append(accept); buf->append("\r\n"); - } + } else { + auto connection = resp.fs.header(http2::HD_CONNECTION); + if (connection) { + buf->append("Connection: "); + buf->append((*connection).value); + buf->append("\r\n"); + } - auto upgrade = resp.fs.header(http2::HD_UPGRADE); - if (upgrade) { - buf->append("Upgrade: "); - buf->append((*upgrade).value); - buf->append("\r\n"); + auto upgrade = resp.fs.header(http2::HD_UPGRADE); + if (upgrade) { + buf->append("Upgrade: "); + buf->append((*upgrade).value); + buf->append("\r\n"); + } } } diff --git a/src/util.cc b/src/util.cc index 7bbad812..b3c8de53 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1453,7 +1453,8 @@ void EVP_MD_CTX_free(EVP_MD_CTX *ctx) { EVP_MD_CTX_destroy(ctx); } } // namespace #endif // !OPENSSL_1_1_API -int sha256(uint8_t *res, const StringRef &s) { +namespace { +int message_digest(uint8_t *res, const EVP_MD *meth, const StringRef &s) { int rv; auto ctx = EVP_MD_CTX_new(); @@ -1463,7 +1464,7 @@ int sha256(uint8_t *res, const StringRef &s) { auto ctx_deleter = defer(EVP_MD_CTX_free, ctx); - rv = EVP_DigestInit_ex(ctx, EVP_sha256(), nullptr); + rv = EVP_DigestInit_ex(ctx, meth, nullptr); if (rv != 1) { return -1; } @@ -1473,7 +1474,7 @@ int sha256(uint8_t *res, const StringRef &s) { return -1; } - unsigned int mdlen = 32; + unsigned int mdlen = EVP_MD_size(meth); rv = EVP_DigestFinal_ex(ctx, res, &mdlen); if (rv != 1) { @@ -1482,6 +1483,15 @@ int sha256(uint8_t *res, const StringRef &s) { return 0; } +} // namespace + +int sha256(uint8_t *res, const StringRef &s) { + return message_digest(res, EVP_sha256(), s); +} + +int sha1(uint8_t *res, const StringRef &s) { + return message_digest(res, EVP_sha1(), s); +} bool is_hex_string(const StringRef &s) { if (s.size() % 2) { diff --git a/src/util.h b/src/util.h index 85cc6110..897f4ef4 100644 --- a/src/util.h +++ b/src/util.h @@ -739,6 +739,13 @@ OutputIt random_alpha_digit(OutputIt first, OutputIt last, Generator &gen) { return first; } +// Fills random bytes to the range [|first|, |last|). +template +void random_bytes(OutputIt first, OutputIt last, Generator &gen) { + std::uniform_int_distribution<> dis(0, 255); + std::generate(first, last, [&dis, &gen]() { return dis(gen); }); +} + template OutputIterator copy_lit(OutputIterator it, CharT (&s)[N]) { return std::copy_n(s, N - 1, it); @@ -753,6 +760,10 @@ uint32_t hash32(const StringRef &s); // returns 0 if it succeeds, or -1. int sha256(uint8_t *buf, const StringRef &s); +// Computes SHA-1 of |s|, and stores it in |buf|. This function +// returns 0 if it succeeds, or -1. +int sha1(uint8_t *buf, const StringRef &s); + // Returns host from |hostport|. If host cannot be found in // |hostport|, returns empty string. The returned string might not be // NULL-terminated.