diff --git a/doc/h2load.1 b/doc/h2load.1 index 7f4417b0..8ba07836 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "February 01, 2015" "0.7.4-DEV" "nghttp2" +.TH "H2LOAD" "1" "February 08, 2015" "0.7.4-DEV" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index f56faae3..104f6de5 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "February 01, 2015" "0.7.4-DEV" "nghttp2" +.TH "NGHTTP" "1" "February 08, 2015" "0.7.4-DEV" "nghttp2" .SH NAME nghttp \- HTTP/2 experimental client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index 1a54578b..2523ca80 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "February 01, 2015" "0.7.4-DEV" "nghttp2" +.TH "NGHTTPD" "1" "February 08, 2015" "0.7.4-DEV" "nghttp2" .SH NAME nghttpd \- HTTP/2 experimental server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index 480edd15..d6b2b446 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "February 01, 2015" "0.7.4-DEV" "nghttp2" +.TH "NGHTTPX" "1" "February 08, 2015" "0.7.4-DEV" "nghttp2" .SH NAME nghttpx \- HTTP/2 experimental proxy . @@ -500,6 +500,13 @@ padding. Specify 0 to disable padding. This option is meant for debugging purpose and not intended to enhance protocol security. .UNINDENT +.INDENT 0.0 +.TP +.B \-\-no\-server\-push +Disable HTTP/2 server push. Server push is only +supported by default mode and HTTP/2 frontend. SPDY +frontend does not support server push. +.UNINDENT .SS Mode .INDENT 0.0 .TP @@ -652,6 +659,14 @@ altered regardless of this option. .UNINDENT .INDENT 0.0 .TP +.B \-\-no\-host\-rewrite +Don\(aqt rewrite host and :authority header fields on +\fI\%\-\-http2\-bridge\fP, \fI\%\-\-client\fP and default mode. For +\fI\%\-\-http2\-proxy\fP and \fI\%\-\-client\-proxy\fP mode, these headers +will not be altered regardless of this option. +.UNINDENT +.INDENT 0.0 +.TP .B \-\-altsvc= Specify protocol ID, port, host and origin of alternative service. and are optional. @@ -782,6 +797,39 @@ path with same command\-line arguments and environment variables. After new process comes up, sending SIGQUIT to the original process to perform hot swapping. .UNINDENT +.SH SERVER PUSH +.sp +nghttpx supports HTTP/2 server push in default mode. nghttpx looks +for Link header field (\fI\%RFC 5988\fP) in response headers for +backend server and extracts URI\-reference with parameter +\fBrel=preload\fP (see \fI\%preload\fP) +and pushes those URIs to the frontend client. Here is a sample Link +header field to initiate server push: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +Link: ; rel=preload +Link: ; rel=preload +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Currently, the following restrictions are applied for server push: +.INDENT 0.0 +.IP 1. 3 +URI\-reference must not contain authority. If it exists, it is not +pushed. \fB/fonts/font.woff\fP and \fBcss/theme.css\fP are eligible to +be pushed. \fBhttps://example.org/fonts/font.woff\fP and +\fB//example.org/css/theme.css\fP are not. +.IP 2. 3 +The associated stream must have method "GET" or "POST". The +associated stream\(aqs status code must be 200. +.UNINDENT +.sp +These limitations may be loosened in the future release. .SH SEE ALSO .sp \fInghttp(1)\fP, \fInghttpd(1)\fP, \fIh2load(1)\fP diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index 240b0c3d..eff57514 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -438,6 +438,12 @@ HTTP/2 and SPDY meant for debugging purpose and not intended to enhance protocol security. +.. option:: --no-server-push + + Disable HTTP/2 server push. Server push is only + supported by default mode and HTTP/2 frontend. SPDY + frontend does not support server push. + Mode ~~~~ @@ -570,6 +576,13 @@ HTTP :option:`--client-proxy` mode, location header field will not be altered regardless of this option. +.. option:: --no-host-rewrite + + Don't rewrite host and :authority header fields on + :option:`--http2-bridge`\, :option:`--client` and default mode. For + :option:`--http2-proxy` and :option:`\--client-proxy` mode, these headers + will not be altered regardless of this option. + .. option:: --altsvc= Specify protocol ID, port, host and origin of @@ -701,6 +714,35 @@ SIGUSR2 After new process comes up, sending SIGQUIT to the original process to perform hot swapping. +SERVER PUSH +----------- + +nghttpx supports HTTP/2 server push in default mode. nghttpx looks +for Link header field (`RFC 5988 +`_) in response headers for +backend server and extracts URI-reference with parameter +``rel=preload`` (see `preload +`_) +and pushes those URIs to the frontend client. Here is a sample Link +header field to initiate server push: + +.. code-block:: http + + Link: ; rel=preload + Link: ; rel=preload + +Currently, the following restrictions are applied for server push: + +1. URI-reference must not contain authority. If it exists, it is not + pushed. ``/fonts/font.woff`` and ``css/theme.css`` are eligible to + be pushed. ``https://example.org/fonts/font.woff`` and + ``//example.org/css/theme.css`` are not. + +2. The associated stream must have method "GET" or "POST". The + associated stream's status code must be 200. + +These limitations may be loosened in the future release. + SEE ALSO -------- diff --git a/doc/nghttpx.h2r b/doc/nghttpx.h2r index 820911ed..e5766dbd 100644 --- a/doc/nghttpx.h2r +++ b/doc/nghttpx.h2r @@ -42,6 +42,35 @@ SIGUSR2 After new process comes up, sending SIGQUIT to the original process to perform hot swapping. +SERVER PUSH +----------- + +nghttpx supports HTTP/2 server push in default mode. nghttpx looks +for Link header field (`RFC 5988 +`_) in response headers for +backend server and extracts URI-reference with parameter +``rel=preload`` (see `preload +`_) +and pushes those URIs to the frontend client. Here is a sample Link +header field to initiate server push: + +.. code-block:: http + + Link: ; rel=preload + Link: ; rel=preload + +Currently, the following restrictions are applied for server push: + +1. URI-reference must not contain authority. If it exists, it is not + pushed. ``/fonts/font.woff`` and ``css/theme.css`` are eligible to + be pushed. ``https://example.org/fonts/font.woff`` and + ``//example.org/css/theme.css`` are not. + +2. The associated stream must have method "GET" or "POST". The + associated stream's status code must be 200. + +These limitations may be loosened in the future release. + SEE ALSO -------- diff --git a/genheaderfunc.py b/genheaderfunc.py index c4fba627..d8111d4c 100755 --- a/genheaderfunc.py +++ b/genheaderfunc.py @@ -21,6 +21,11 @@ HEADERS = [ "content-length", "location", "trailer", + "link", + "accept-encoding", + "accept-language", + "cache-control", + "user-agent", # disallowed h1 headers 'connection', 'keep-alive', diff --git a/integration-tests/nghttpx_http2_test.go b/integration-tests/nghttpx_http2_test.go index 3e24c912..29e5cc82 100644 --- a/integration-tests/nghttpx_http2_test.go +++ b/integration-tests/nghttpx_http2_test.go @@ -473,6 +473,9 @@ func TestH2H1TEGzip(t *testing.T) { } } +// TestH2H1SNI tests server's TLS SNI extension feature. It must +// choose appropriate certificate depending on the indicated +// server_name from client. func TestH2H1SNI(t *testing.T) { st := newServerTesterTLSConfig([]string{"--subcert=" + testDir + "/alt-server.key:" + testDir + "/alt-server.crt"}, t, noopHandler, &tls.Config{ ServerName: "alt-domain", @@ -488,6 +491,37 @@ func TestH2H1SNI(t *testing.T) { } } +// TestH2H1ServerPush tests server push using Link header field from +// backend server. +func TestH2H1ServerPush(t *testing.T) { + st := newServerTester(nil, t, func(w http.ResponseWriter, r *http.Request) { + // only resources marked as rel=preload are pushed + w.Header().Add("Link", "; rel=preload, , ; rel=preload") + }) + defer st.Close() + + res, err := st.http2(requestParam{ + name: "TestH2H1ServerPush", + }) + if err != nil { + t.Fatalf("Error st.http2() = %v", err) + } + if got, want := res.status, 200; got != want { + t.Errorf("res.status: %v; want %v", got, want) + } + if got, want := len(res.pushResponse), 2; got != want { + t.Fatalf("len(res.pushResponse): %v; want %v", got, want) + } + mainCSS := res.pushResponse[0] + if got, want := mainCSS.status, 200; got != want { + t.Errorf("mainCSS.status: %v; want %v", got, want) + } + themeCSS := res.pushResponse[1] + if got, want := themeCSS.status, 200; got != want { + t.Errorf("themeCSS.status: %v; want %v", got, want) + } +} + // TestH2H1GracefulShutdown tests graceful shutdown. func TestH2H1GracefulShutdown(t *testing.T) { st := newServerTester(nil, t, noopHandler) diff --git a/integration-tests/server_tester.go b/integration-tests/server_tester.go index 93f421eb..b8a50300 100644 --- a/integration-tests/server_tester.go +++ b/integration-tests/server_tester.go @@ -17,6 +17,7 @@ import ( "net/http/httptest" "net/url" "os/exec" + "sort" "strconv" "strings" "testing" @@ -411,7 +412,6 @@ loop: } func (st *serverTester) http2(rp requestParam) (*serverResponse, error) { - res := &serverResponse{} st.headerBlkBuf.Reset() st.header = make(http.Header) @@ -434,6 +434,13 @@ func (st *serverTester) http2(rp requestParam) (*serverResponse, error) { } } + res := &serverResponse{ + streamID: id, + } + + streams := make(map[uint32]*serverResponse) + streams[id] = res + method := "GET" if rp.method != "" { method = rp.method @@ -493,34 +500,53 @@ loop: if err != nil { return res, err } - if f.FrameHeader.StreamID != id { + sr, ok := streams[f.FrameHeader.StreamID] + if !ok { st.header = make(http.Header) break } - res.header = cloneHeader(st.header) + sr.header = cloneHeader(st.header) var status int - status, err = strconv.Atoi(res.header.Get(":status")) + status, err = strconv.Atoi(sr.header.Get(":status")) if err != nil { return res, fmt.Errorf("Error parsing status code: %v", err) } - res.status = status + sr.status = status if f.StreamEnded() { - break loop + if streamEnded(res, streams, sr) { + break loop + } } + case *http2.PushPromiseFrame: + _, err := st.dec.Write(f.HeaderBlockFragment()) + if err != nil { + return res, err + } + sr := &serverResponse{ + streamID: f.PromiseID, + reqHeader: cloneHeader(st.header), + } + streams[sr.streamID] = sr case *http2.DataFrame: - if f.FrameHeader.StreamID != id { + sr, ok := streams[f.FrameHeader.StreamID] + if !ok { break } - res.body = append(res.body, f.Data()...) + sr.body = append(sr.body, f.Data()...) if f.StreamEnded() { - break loop + if streamEnded(res, streams, sr) { + break loop + } } case *http2.RSTStreamFrame: - if f.FrameHeader.StreamID != id { + sr, ok := streams[f.FrameHeader.StreamID] + if !ok { break } - res.errCode = f.ErrCode - break loop + sr.errCode = f.ErrCode + if streamEnded(res, streams, sr) { + break loop + } case *http2.GoAwayFrame: if f.ErrCode == http2.ErrCodeNo { break @@ -535,21 +561,46 @@ loop: if err := st.fr.WriteSettingsAck(); err != nil { return res, err } - // TODO handle PUSH_PROMISE as well, since it alters HPACK context } } + sort.Sort(ByStreamID(res.pushResponse)) return res, nil } +func streamEnded(mainSr *serverResponse, streams map[uint32]*serverResponse, sr *serverResponse) bool { + delete(streams, sr.streamID) + if mainSr.streamID != sr.streamID { + mainSr.pushResponse = append(mainSr.pushResponse, sr) + } + return len(streams) == 0 +} + type serverResponse struct { status int // HTTP status code header http.Header // response header fields body []byte // response body + streamID uint32 // stream ID in HTTP/2 errCode http2.ErrCode // error code received in HTTP/2 RST_STREAM or GOAWAY connErr bool // true if HTTP/2 connection error spdyGoAwayErrCode spdy.GoAwayStatus // status code received in SPDY RST_STREAM spdyRstErrCode spdy.RstStreamStatus // status code received in SPDY GOAWAY connClose bool // Conection: close is included in response header in HTTP/1 test + reqHeader http.Header // http request header, currently only sotres pushed request header + pushResponse []*serverResponse // pushed response +} + +type ByStreamID []*serverResponse + +func (b ByStreamID) Len() int { + return len(b) +} + +func (b ByStreamID) Swap(i, j int) { + b[i], b[j] = b[j], b[i] +} + +func (b ByStreamID) Less(i, j int) bool { + return b[i].streamID < b[j].streamID } func cloneHeader(h http.Header) http.Header { diff --git a/src/HttpServer.cc b/src/HttpServer.cc index 5e21502c..d3e5c55c 100644 --- a/src/HttpServer.cc +++ b/src/HttpServer.cc @@ -85,7 +85,7 @@ template void append_nv(Stream *stream, const Array &nva) { http2::index_header(stream->hdidx, token, i); } http2::add_header(stream->headers, nv.name, nv.namelen, nv.value, - nv.valuelen, nv.flags & NGHTTP2_NV_FLAG_NO_INDEX); + nv.valuelen, nv.flags & NGHTTP2_NV_FLAG_NO_INDEX, token); } } } // namespace @@ -1061,7 +1061,7 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, http2::index_header(stream->hdidx, token, stream->headers.size()); http2::add_header(stream->headers, name, namelen, value, valuelen, - flags & NGHTTP2_NV_FLAG_NO_INDEX); + flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } } // namespace diff --git a/src/http2.cc b/src/http2.cc index a9a57dd2..6bb0ba4d 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -165,14 +165,15 @@ void copy_url_component(std::string &dest, const http_parser_url *u, int field, Headers::value_type to_header(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, - bool no_index) { + bool no_index, int16_t token) { return Header(std::string(reinterpret_cast(name), namelen), std::string(reinterpret_cast(value), valuelen), - no_index); + no_index, token); } void add_header(Headers &nva, const uint8_t *name, size_t namelen, - const uint8_t *value, size_t valuelen, bool no_index) { + const uint8_t *value, size_t valuelen, bool no_index, + int16_t token) { if (valuelen > 0) { size_t i, j; for (i = 0; i < valuelen && (value[i] == ' ' || value[i] == '\t'); ++i) @@ -182,7 +183,7 @@ void add_header(Headers &nva, const uint8_t *name, size_t namelen, value += i; valuelen -= i + (valuelen - j - 1); } - nva.push_back(to_header(name, namelen, value, valuelen, no_index)); + nva.push_back(to_header(name, namelen, value, valuelen, no_index, token)); } const Headers::value_type *get_header(const Headers &nva, const char *name) { @@ -221,7 +222,7 @@ void copy_headers_to_nva(std::vector &nva, const Headers &headers) { if (kv.name.empty() || kv.name[0] == ':') { continue; } - switch (lookup_token(kv.name)) { + switch (kv.token) { case HD_COOKIE: case HD_CONNECTION: case HD_HOST: @@ -247,7 +248,7 @@ void build_http1_headers_from_headers(std::string &hdrs, if (kv.name.empty() || kv.name[0] == ':') { continue; } - switch (lookup_token(kv.name)) { + switch (kv.token) { case HD_CONNECTION: case HD_COOKIE: case HD_HOST: @@ -438,6 +439,11 @@ int lookup_token(const uint8_t *name, size_t namelen) { break; case 4: switch (name[namelen - 1]) { + case 'k': + if (util::streq("lin", name, 3)) { + return HD_LINK; + } + break; case 't': if (util::streq("hos", name, 3)) { return HD_HOST; @@ -531,6 +537,11 @@ int lookup_token(const uint8_t *name, size_t namelen) { return HD_CONNECTION; } break; + case 't': + if (util::streq("user-agen", name, 9)) { + return HD_USER_AGENT; + } + break; case 'y': if (util::streq(":authorit", name, 9)) { return HD__AUTHORITY; @@ -538,6 +549,15 @@ int lookup_token(const uint8_t *name, size_t namelen) { break; } break; + case 13: + switch (name[namelen - 1]) { + case 'l': + if (util::streq("cache-contro", name, 12)) { + return HD_CACHE_CONTROL; + } + break; + } + break; case 14: switch (name[namelen - 1]) { case 'h': @@ -554,6 +574,16 @@ int lookup_token(const uint8_t *name, size_t namelen) { break; case 15: switch (name[namelen - 1]) { + case 'e': + if (util::streq("accept-languag", name, 14)) { + return HD_ACCEPT_LANGUAGE; + } + break; + case 'g': + if (util::streq("accept-encodin", name, 14)) { + return HD_ACCEPT_ENCODING; + } + break; case 'r': if (util::streq("x-forwarded-fo", name, 14)) { return HD_X_FORWARDED_FOR; @@ -597,18 +627,7 @@ void init_hdidx(HeaderIndex &hdidx) { std::fill(std::begin(hdidx), std::end(hdidx), -1); } -void index_headers(HeaderIndex &hdidx, const Headers &headers) { - for (size_t i = 0; i < headers.size(); ++i) { - auto &kv = headers[i]; - auto token = lookup_token( - reinterpret_cast(kv.name.c_str()), kv.name.size()); - if (token >= 0) { - http2::index_header(hdidx, token, i); - } - } -} - -void index_header(HeaderIndex &hdidx, int token, size_t idx) { +void index_header(HeaderIndex &hdidx, int16_t token, size_t idx) { if (token == -1) { return; } @@ -616,7 +635,8 @@ void index_header(HeaderIndex &hdidx, int token, size_t idx) { hdidx[token] = idx; } -bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, int token) { +bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, + int16_t token) { switch (token) { case HD__AUTHORITY: case HD__METHOD: @@ -628,7 +648,8 @@ bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, int token) { } } -bool check_http2_response_pseudo_header(const HeaderIndex &hdidx, int token) { +bool check_http2_response_pseudo_header(const HeaderIndex &hdidx, + int16_t token) { switch (token) { case HD__STATUS: return hdidx[token] == -1; @@ -637,7 +658,7 @@ bool check_http2_response_pseudo_header(const HeaderIndex &hdidx, int token) { } } -bool http2_header_allowed(int token) { +bool http2_header_allowed(int16_t token) { switch (token) { case HD_CONNECTION: case HD_KEEP_ALIVE: @@ -659,7 +680,7 @@ bool http2_mandatory_request_headers_presence(const HeaderIndex &hdidx) { return true; } -const Headers::value_type *get_header(const HeaderIndex &hdidx, int token, +const Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, const Headers &nva) { auto i = hdidx[token]; if (i == -1) { @@ -668,6 +689,336 @@ const Headers::value_type *get_header(const HeaderIndex &hdidx, int token, return &nva[i]; } +namespace { +template InputIt skip_lws(InputIt first, InputIt last) { + for (; first != last; ++first) { + switch (*first) { + case ' ': + case '\t': + continue; + default: + return first; + } + } + return first; +} +} // namespace + +namespace { +template +InputIt skip_to_next_field(InputIt first, InputIt last) { + for (; first != last; ++first) { + switch (*first) { + case ' ': + case '\t': + case ',': + continue; + default: + return first; + } + } + return first; +} +} // namespace + +namespace { +std::pair +parse_next_link_header_once(const char *first, const char *last) { + first = skip_to_next_field(first, last); + if (first == last || *first != '<') { + return {{{0, 0}}, last}; + } + auto url_first = ++first; + first = std::find(first, last, '>'); + if (first == last) { + return {{{0, 0}}, first}; + } + auto url_last = first++; + if (first == last) { + return {{{0, 0}}, first}; + } + // we expect ';' or ',' here + switch (*first) { + case ',': + return {{{0, 0}}, ++first}; + case ';': + ++first; + break; + default: + return {{{0, 0}}, last}; + } + + auto ok = false; + for (;;) { + first = skip_lws(first, last); + if (first == last) { + return {{{0, 0}}, first}; + } + // we expect link-param + + // we are only interested in rel=preload parameter. Others are + // simply skipped. + static const char PL[] = "rel=preload"; + static const size_t PLLEN = sizeof(PL) - 1; + if (first + PLLEN == last) { + if (std::equal(PL, PL + PLLEN, first)) { + ok = true; + // this is the end of sequence + return {{{url_first, url_last}}, last}; + } + } else if (first + PLLEN + 1 <= last) { + switch (*(first + PLLEN)) { + case ',': + if (!std::equal(PL, PL + PLLEN, first)) { + break; + } + ok = true; + // skip including ',' + first += PLLEN + 1; + return {{{url_first, url_last}}, first}; + case ';': + if (!std::equal(PL, PL + PLLEN, first)) { + break; + } + ok = true; + // skip including ';' + first += PLLEN + 1; + // continue parse next link-param + continue; + } + } + auto param_first = first; + for (; first != last;) { + if (util::in_attr_char(*first)) { + ++first; + continue; + } + // '*' is only allowed at the end of parameter name and must be + // followed by '=' + if (last - first >= 2 && first != param_first) { + if (*first == '*' && *(first + 1) == '=') { + ++first; + break; + } + } + if (*first == '=' || *first == ';' || *first == ',') { + break; + } + return {{{0, 0}}, last}; + } + if (param_first == first) { + // empty parmname + return {{{0, 0}}, last}; + } + // link-param without value is acceptable (see link-extension) if + // it is not followed by '=' + if (first == last || *first == ',') { + goto almost_done; + } + if (*first == ';') { + ++first; + // parse next link-param + continue; + } + // now parsing lin-param value + assert(*first == '='); + ++first; + if (first == last) { + // empty value is not acceptable + return {{{0, 0}}, first}; + } + if (*first == '"') { + // quoted-string + first = std::find(first + 1, last, '"'); + if (first == last) { + return {{{0, 0}}, first}; + } + ++first; + if (first == last || *first == ',') { + goto almost_done; + } + if (*first == ';') { + ++first; + // parse next link-param + continue; + } + return {{{0, 0}}, last}; + } + // not quoted-string, skip to next ',' or ';' + if (*first == ',' || *first == ';') { + // empty value + return {{{0, 0}}, last}; + } + for (; first != last; ++first) { + if (*first == ',' || *first == ';') { + break; + } + } + if (first == last || *first == ',') { + goto almost_done; + } + assert(*first == ';'); + ++first; + // parse next link-param + } + +almost_done: + assert(first == last || *first == ','); + + if (*first == ',') { + ++first; + } + if (ok) { + return {{{url_first, url_last}}, first}; + } + return {{{0, 0}}, first}; +} +} // namespace + +std::vector parse_link_header(const char *src, size_t len) { + auto first = src; + auto last = src + len; + std::vector res; + for (; first != last;) { + auto rv = parse_next_link_header_once(first, last); + first = rv.second; + if (rv.first.uri.first != 0 || rv.first.uri.second != 0) { + res.push_back(rv.first); + } + } + return res; +} + +namespace { +void eat_file(std::string &path) { + if (path.empty()) { + path = "/"; + return; + } + auto p = path.size() - 1; + if (path[p] == '/') { + return; + } + p = path.rfind('/', p); + if (p == std::string::npos) { + // this should not happend in normal case, where we expect path + // starts with '/' + path = "/"; + return; + } + path.erase(std::begin(path) + p + 1, std::end(path)); +} +} // namespace + +namespace { +void eat_dir(std::string &path) { + if (path.empty()) { + path = "/"; + return; + } + auto p = path.size() - 1; + if (path[p] != '/') { + p = path.rfind('/', p); + if (p == std::string::npos) { + // this should not happend in normal case, where we expect path + // starts with '/' + path = "/"; + return; + } + } + if (path[p] == '/') { + if (p == 0) { + return; + } + --p; + } + p = path.rfind('/', p); + if (p == std::string::npos) { + // this should not happend in normal case, where we expect path + // starts with '/' + path = "/"; + return; + } + path.erase(std::begin(path) + p + 1, std::end(path)); +} +} // namespace + +std::string path_join(const char *base_path, size_t base_pathlen, + const char *base_query, size_t base_querylen, + const char *rel_path, size_t rel_pathlen, + const char *rel_query, size_t rel_querylen) { + std::string res; + if (rel_pathlen == 0) { + if (base_pathlen == 0) { + res = "/"; + } else { + res.assign(base_path, base_pathlen); + } + if (rel_querylen == 0) { + if (base_querylen) { + res += "?"; + res.append(base_query, base_querylen); + } + return res; + } + res += "?"; + res.append(rel_query, rel_querylen); + return res; + } + + auto first = rel_path; + auto last = rel_path + rel_pathlen; + + if (rel_path[0] == '/') { + res = "/"; + ++first; + } else if (base_pathlen == 0) { + res = "/"; + } else { + res.assign(base_path, base_pathlen); + } + + for (; first != last;) { + if (*first == '.') { + if (first + 1 == last) { + break; + } + if (*(first + 1) == '/') { + first += 2; + continue; + } + if (*(first + 1) == '.') { + if (first + 2 == last) { + eat_dir(res); + break; + } + if (*(first + 2) == '/') { + eat_dir(res); + first += 3; + continue; + } + } + } + if (res.back() != '/') { + eat_file(res); + } + auto slash = std::find(first, last, '/'); + if (slash == last) { + res.append(first, last); + break; + } + res.append(first, slash + 1); + first = slash + 1; + for (; first != last && *first == '/'; ++first) + ; + } + if (rel_querylen) { + res += "?"; + res.append(rel_query, rel_querylen); + } + return res; +} + } // namespace http2 } // namespace nghttp2 diff --git a/src/http2.h b/src/http2.h index d93bd0ac..fcd2c420 100644 --- a/src/http2.h +++ b/src/http2.h @@ -40,10 +40,12 @@ namespace nghttp2 { struct Header { - Header(std::string name, std::string value, bool no_index = false) - : name(std::move(name)), value(std::move(value)), no_index(no_index) {} + Header(std::string name, std::string value, bool no_index = false, + int16_t token = -1) + : name(std::move(name)), value(std::move(value)), token(token), + no_index(no_index) {} - Header() : no_index(false) {} + Header() : token(-1), no_index(false) {} bool operator==(const Header &other) const { return name == other.name && value == other.value; @@ -55,6 +57,7 @@ struct Header { std::string name; std::string value; + int16_t token; bool no_index; }; @@ -77,13 +80,14 @@ void copy_url_component(std::string &dest, const http_parser_url *u, int field, Headers::value_type to_header(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, - bool no_index); + bool no_index, int16_t token); // Add name/value pairs to |nva|. If |no_index| is true, this // name/value pair won't be indexed when it is forwarded to the next // hop. This function strips white spaces around |value|. void add_header(Headers &nva, const uint8_t *name, size_t namelen, - const uint8_t *value, size_t valuelen, bool no_index); + const uint8_t *value, size_t valuelen, bool no_index, + int16_t token); // Returns pointer to the entry in |nva| which has name |name|. If // more than one entries which have the name |name|, last occurrence @@ -125,14 +129,16 @@ nghttp2_nv make_nv_ls(const char (&name)[N], const std::string &value) { NGHTTP2_NV_FLAG_NONE}; } -// Appends headers in |headers| to |nv|. Certain headers, including -// disallowed headers in HTTP/2 spec and headers which require -// special handling (i.e. via), are not copied. +// Appends headers in |headers| to |nv|. |headers| must be indexed +// before this call (its element's token field is assigned). Certain +// headers, including disallowed headers in HTTP/2 spec and headers +// which require special handling (i.e. via), are not copied. void copy_headers_to_nva(std::vector &nva, const Headers &headers); // Appends HTTP/1.1 style header lines to |hdrs| from headers in -// |headers|. Certain headers, which requires special handling -// (i.e. via and cookie), are not appended. +// |headers|. |headers| must be indexed before this call (its +// element's token field is assigned). Certain headers, which +// requires special handling (i.e. via and cookie), are not appended. void build_http1_headers_from_headers(std::string &hdrs, const Headers &headers); @@ -189,7 +195,10 @@ enum { HD__PATH, HD__SCHEME, HD__STATUS, + HD_ACCEPT_ENCODING, + HD_ACCEPT_LANGUAGE, HD_ALT_SVC, + HD_CACHE_CONTROL, HD_CONNECTION, HD_CONTENT_LENGTH, HD_COOKIE, @@ -198,6 +207,7 @@ enum { HD_HTTP2_SETTINGS, HD_IF_MODIFIED_SINCE, HD_KEEP_ALIVE, + HD_LINK, HD_LOCATION, HD_PROXY_CONNECTION, HD_SERVER, @@ -205,13 +215,14 @@ enum { HD_TRAILER, HD_TRANSFER_ENCODING, HD_UPGRADE, + HD_USER_AGENT, HD_VIA, HD_X_FORWARDED_FOR, HD_X_FORWARDED_PROTO, HD_MAXIDX, }; -using HeaderIndex = std::array; +using HeaderIndex = std::array; // Looks up header token for header name |name| of length |namelen|. // Only headers we are interested in are tokenized. If header name @@ -223,30 +234,51 @@ int lookup_token(const std::string &name); // array containing at least HD_MAXIDX elements. void init_hdidx(HeaderIndex &hdidx); // Indexes header |token| using index |idx|. -void index_header(HeaderIndex &hdidx, int token, size_t idx); -// Iterates |headers| and for each element, call index_header. -void index_headers(HeaderIndex &hdidx, const Headers &headers); +void index_header(HeaderIndex &hdidx, int16_t token, size_t idx); // Returns true if HTTP/2 request pseudo header |token| is not indexed // yet and not -1. -bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, int token); +bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, int16_t token); // Returns true if HTTP/2 response pseudo header |token| is not // indexed yet and not -1. -bool check_http2_response_pseudo_header(const HeaderIndex &hdidx, int token); +bool check_http2_response_pseudo_header(const HeaderIndex &hdidx, + int16_t token); // Returns true if header field denoted by |token| is allowed for // HTTP/2. -bool http2_header_allowed(int token); +bool http2_header_allowed(int16_t token); // Returns true that |hdidx| contains mandatory HTTP/2 request // headers. bool http2_mandatory_request_headers_presence(const HeaderIndex &hdidx); // Returns header denoted by |token| using index |hdidx|. -const Headers::value_type *get_header(const HeaderIndex &hdidx, int token, +const Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, const Headers &nva); +struct LinkHeader { + // The region of URI is [uri.first, uri.second). + std::pair uri; +}; + +// Returns next URI-reference in Link header field value |src| of +// length |len|. If no URI-reference found after searching all input, +// returned uri field is empty. This imply that empty URI-reference +// is ignored during parsing. +std::vector parse_link_header(const char *src, size_t len); + +// Constructs path by combining base path |base_path| of length +// |base_pathlen| with another path |rel_path| of length +// |rel_pathlen|. The base path and another path can have optional +// query component. This function assumes |base_path| is +// cannibalized. In other words, it does not contain ".." or "." path +// components and starts with "/" if it is not empty. +std::string path_join(const char *base_path, size_t base_pathlen, + const char *base_query, size_t base_querylen, + const char *rel_path, size_t rel_pathlen, + const char *rel_query, size_t rel_querylen); + } // namespace http2 } // namespace nghttp2 diff --git a/src/http2_test.cc b/src/http2_test.cc index 871cc392..46bccdf5 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -58,46 +58,52 @@ void test_http2_add_header(void) { auto nva = Headers(); http2::add_header(nva, (const uint8_t *)"alpha", 5, (const uint8_t *)"123", 3, - false); + false, -1); CU_ASSERT(Headers::value_type("alpha", "123") == nva[0]); CU_ASSERT(!nva[0].no_index); nva.clear(); http2::add_header(nva, (const uint8_t *)"alpha", 5, (const uint8_t *)"", 0, - true); + true, -1); CU_ASSERT(Headers::value_type("alpha", "") == nva[0]); CU_ASSERT(nva[0].no_index); nva.clear(); http2::add_header(nva, (const uint8_t *)"a", 1, (const uint8_t *)" b", 2, - false); + false, -1); CU_ASSERT(Headers::value_type("a", "b") == nva[0]); nva.clear(); http2::add_header(nva, (const uint8_t *)"a", 1, (const uint8_t *)"b ", 2, - false); + false, -1); CU_ASSERT(Headers::value_type("a", "b") == nva[0]); nva.clear(); http2::add_header(nva, (const uint8_t *)"a", 1, (const uint8_t *)" b ", 5, - false); + false, -1); CU_ASSERT(Headers::value_type("a", "b") == nva[0]); nva.clear(); http2::add_header(nva, (const uint8_t *)"a", 1, (const uint8_t *)" bravo ", - 9, false); + 9, false, -1); CU_ASSERT(Headers::value_type("a", "bravo") == nva[0]); nva.clear(); http2::add_header(nva, (const uint8_t *)"a", 1, (const uint8_t *)" ", 4, - false); + false, -1); CU_ASSERT(Headers::value_type("a", "") == nva[0]); + + nva.clear(); + + http2::add_header(nva, (const uint8_t *)"te", 2, (const uint8_t *)"trailers", + 8, false, http2::HD_TE); + CU_ASSERT(http2::HD_TE == nva[0].token); } void test_http2_get_header(void) { @@ -128,19 +134,20 @@ void test_http2_get_header(void) { } namespace { -auto headers = Headers{{"alpha", "0", true}, - {"bravo", "1"}, - {"connection", "2"}, - {"connection", "3"}, - {"delta", "4"}, - {"expect", "5"}, - {"foxtrot", "6"}, - {"tango", "7"}, - {"te", "8"}, - {"te", "9"}, - {"x-forwarded-proto", "10"}, - {"x-forwarded-proto", "11"}, - {"zulu", "12"}}; +auto headers = + Headers{{"alpha", "0", true}, + {"bravo", "1"}, + {"connection", "2", false, http2::HD_CONNECTION}, + {"connection", "3", false, http2::HD_CONNECTION}, + {"delta", "4"}, + {"expect", "5"}, + {"foxtrot", "6"}, + {"tango", "7"}, + {"te", "8", false, http2::HD_TE}, + {"te", "9", false, http2::HD_TE}, + {"x-forwarded-proto", "10", false, http2::HD_X_FORWARDED_FOR}, + {"x-forwarded-proto", "11", false, http2::HD_X_FORWARDED_FOR}, + {"zulu", "12"}}; } // namespace void test_http2_copy_headers_to_nva(void) { @@ -290,4 +297,366 @@ void test_http2_mandatory_request_headers_presence(void) { CU_ASSERT(http2::http2_mandatory_request_headers_presence(hdidx)); } +void test_http2_parse_link_header(void) { + { + // only URI appears; we don't extract URI unless it bears rel=preload + const char s[] = ""; + auto res = http2::parse_link_header(s, sizeof(s) - 1); + CU_ASSERT(0 == res.size()); + } + { + // URI url should be extracted + const char s[] = "; rel=preload"; + auto res = http2::parse_link_header(s, sizeof(s) - 1); + CU_ASSERT(1 == res.size()); + CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + } + { + // With extra link-param. URI url should be extracted + const char s[] = "; rel=preload; as=file"; + auto res = http2::parse_link_header(s, sizeof(s) - 1); + CU_ASSERT(1 == res.size()); + CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + } + { + // With extra link-param. URI url should be extracted + const char s[] = "; as=file; rel=preload"; + auto res = http2::parse_link_header(s, sizeof(s) - 1); + CU_ASSERT(1 == res.size()); + CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + } + { + // With extra link-param and quote-string. URI url should be + // extracted + const char s[] = R"(; rel=preload; title="foo,bar")"; + auto res = http2::parse_link_header(s, sizeof(s) - 1); + CU_ASSERT(1 == res.size()); + CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + } + { + // With extra link-param and quote-string. URI url should be + // extracted + const char s[] = R"(; title="foo,bar"; rel=preload)"; + auto res = http2::parse_link_header(s, sizeof(s) - 1); + CU_ASSERT(1 == res.size()); + CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + } + { + // ',' after quote-string + const char s[] = R"(; title="foo,bar", ; rel=preload)"; + auto res = http2::parse_link_header(s, sizeof(s) - 1); + CU_ASSERT(1 == res.size()); + CU_ASSERT(std::make_pair(&s[25], &s[28]) == res[0].uri); + } + { + // Only first URI should be extracted. + const char s[] = "; rel=preload, "; + auto res = http2::parse_link_header(s, sizeof(s) - 1); + CU_ASSERT(1 == res.size()); + CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + } + { + // Both have rel=preload, so both urls should be extracted + const char s[] = "; rel=preload, ; rel=preload"; + auto res = http2::parse_link_header(s, sizeof(s) - 1); + CU_ASSERT(2 == res.size()); + CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT(std::make_pair(&s[21], &s[24]) == res[1].uri); + } + { + // Second URI uri should be extracted. + const char s[] = ", ;rel=preload"; + auto res = http2::parse_link_header(s, sizeof(s) - 1); + CU_ASSERT(1 == res.size()); + CU_ASSERT(std::make_pair(&s[8], &s[11]) == res[0].uri); + } + { + // Error if input ends with ';' + const char s[] = ";rel=preload;"; + auto res = http2::parse_link_header(s, sizeof(s) - 1); + CU_ASSERT(0 == res.size()); + } + { + // OK if input ends with ',' + const char s[] = ";rel=preload,"; + auto res = http2::parse_link_header(s, sizeof(s) - 1); + CU_ASSERT(1 == res.size()); + CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + } + { + // Multiple repeated ','s between fields is OK + const char s[] = ",,,;rel=preload"; + auto res = http2::parse_link_header(s, sizeof(s) - 1); + CU_ASSERT(1 == res.size()); + CU_ASSERT(std::make_pair(&s[9], &s[12]) == res[0].uri); + } + { + // Error if url is not enclosed by <> + const char s[] = "url>;rel=preload;"; + auto res = http2::parse_link_header(s, sizeof(s) - 1); + CU_ASSERT(0 == res.size()); + } + { + // Error if url is not enclosed by <> + const char s[] = "' is not followed by ';' + const char s[] = " rel=preload"; + auto res = http2::parse_link_header(s, sizeof(s) - 1); + CU_ASSERT(0 == res.size()); + } + { + // Starting with whitespace is no problem. + const char s[] = " ; rel=preload"; + auto res = http2::parse_link_header(s, sizeof(s) - 1); + CU_ASSERT(1 == res.size()); + CU_ASSERT(std::make_pair(&s[3], &s[6]) == res[0].uri); + } + { + // preload is a prefix of bogus rel parameter value + const char s[] = "; rel=preloadx"; + auto res = http2::parse_link_header(s, sizeof(s) - 1); + CU_ASSERT(0 == res.size()); + } +} + +void test_http2_path_join(void) { + { + const char base[] = "/"; + const char rel[] = "/"; + CU_ASSERT("/" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, + sizeof(rel) - 1, nullptr, 0)); + } + { + const char base[] = "/"; + const char rel[] = "/alpha"; + CU_ASSERT("/alpha" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, + rel, sizeof(rel) - 1, nullptr, 0)); + } + { + // rel ends with trailing '/' + const char base[] = "/"; + const char rel[] = "/alpha/"; + CU_ASSERT("/alpha/" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, + rel, sizeof(rel) - 1, nullptr, 0)); + } + { + // rel contains multiple components + const char base[] = "/"; + const char rel[] = "/alpha/bravo"; + CU_ASSERT("/alpha/bravo" == http2::path_join(base, sizeof(base) - 1, + nullptr, 0, rel, + sizeof(rel) - 1, nullptr, 0)); + } + { + // rel is relative + const char base[] = "/"; + const char rel[] = "alpha/bravo"; + CU_ASSERT("/alpha/bravo" == http2::path_join(base, sizeof(base) - 1, + nullptr, 0, rel, + sizeof(rel) - 1, nullptr, 0)); + } + { + // rel is relative and base ends without /, which means it refers + // to file. + const char base[] = "/alpha"; + const char rel[] = "bravo/charlie"; + CU_ASSERT("/bravo/charlie" == + http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, + sizeof(rel) - 1, nullptr, 0)); + } + { + // rel contains repeated '/'s + const char base[] = "/"; + const char rel[] = "/alpha/////bravo/////"; + CU_ASSERT("/alpha/bravo/" == http2::path_join(base, sizeof(base) - 1, + nullptr, 0, rel, + sizeof(rel) - 1, nullptr, 0)); + } + { + // base ends with '/', so '..' eats 'bravo' + const char base[] = "/alpha/bravo/"; + const char rel[] = "../charlie/delta"; + CU_ASSERT("/alpha/charlie/delta" == + http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, + sizeof(rel) - 1, nullptr, 0)); + } + { + // base does not end with '/', so '..' eats 'alpha/bravo' + const char base[] = "/alpha/bravo"; + const char rel[] = "../charlie"; + CU_ASSERT("/charlie" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, + rel, sizeof(rel) - 1, nullptr, 0)); + } + { + // 'charlie' is eaten by following '..' + const char base[] = "/alpha/bravo/"; + const char rel[] = "../charlie/../delta"; + CU_ASSERT("/alpha/delta" == http2::path_join(base, sizeof(base) - 1, + nullptr, 0, rel, + sizeof(rel) - 1, nullptr, 0)); + } + { + // excessive '..' results in '/' + const char base[] = "/alpha/bravo/"; + const char rel[] = "../../../"; + CU_ASSERT("/" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, + sizeof(rel) - 1, nullptr, 0)); + } + { + // excessive '..' and path component + const char base[] = "/alpha/bravo/"; + const char rel[] = "../../../charlie"; + CU_ASSERT("/charlie" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, + rel, sizeof(rel) - 1, nullptr, 0)); + } + { + // rel ends with '..' + const char base[] = "/alpha/bravo/"; + const char rel[] = "charlie/.."; + CU_ASSERT("/alpha/bravo/" == http2::path_join(base, sizeof(base) - 1, + nullptr, 0, rel, + sizeof(rel) - 1, nullptr, 0)); + } + { + // base empty and rel contains '..' + const char base[] = ""; + const char rel[] = "charlie/.."; + CU_ASSERT("/" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, + sizeof(rel) - 1, nullptr, 0)); + } + { + // '.' is ignored + const char base[] = "/"; + const char rel[] = "charlie/././././delta"; + CU_ASSERT("/charlie/delta" == + http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, + sizeof(rel) - 1, nullptr, 0)); + } + { + // trailing '.' is ignored + const char base[] = "/"; + const char rel[] = "charlie/."; + CU_ASSERT("/charlie/" == http2::path_join(base, sizeof(base) - 1, nullptr, + 0, rel, sizeof(rel) - 1, nullptr, + 0)); + } + { + // query + const char base[] = "/"; + const char rel[] = "/"; + const char relq[] = "q"; + CU_ASSERT("/?q" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, + sizeof(rel) - 1, relq, + sizeof(relq) - 1)); + } + { + // empty rel and query + const char base[] = "/alpha"; + const char rel[] = ""; + const char relq[] = "q"; + CU_ASSERT("/alpha?q" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, + rel, sizeof(rel) - 1, relq, + sizeof(relq) - 1)); + } + { + // both rel and query are empty + const char base[] = "/alpha"; + const char baseq[] = "r"; + const char rel[] = ""; + const char relq[] = ""; + CU_ASSERT("/alpha?r" == + http2::path_join(base, sizeof(base) - 1, baseq, sizeof(baseq) - 1, + rel, sizeof(rel) - 1, relq, sizeof(relq) - 1)); + } + { + // empty base + const char base[] = ""; + const char rel[] = "/alpha"; + CU_ASSERT("/alpha" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, + rel, sizeof(rel) - 1, nullptr, 0)); + } + { + // everything is empty + CU_ASSERT("/" == + http2::path_join(nullptr, 0, nullptr, 0, nullptr, 0, nullptr, 0)); + } + { + // only baseq is not empty + const char base[] = ""; + const char baseq[] = "r"; + const char rel[] = ""; + CU_ASSERT("/?r" == http2::path_join(base, sizeof(base) - 1, baseq, + sizeof(baseq) - 1, rel, sizeof(rel) - 1, + nullptr, 0)); + } +} + } // namespace shrpx diff --git a/src/http2_test.h b/src/http2_test.h index f8127dc7..1171b926 100644 --- a/src/http2_test.h +++ b/src/http2_test.h @@ -39,6 +39,8 @@ void test_http2_lookup_token(void); void test_http2_check_http2_pseudo_header(void); void test_http2_http2_header_allowed(void); void test_http2_mandatory_request_headers_presence(void); +void test_http2_parse_link_header(void); +void test_http2_path_join(void); } // namespace shrpx diff --git a/src/nghttp.cc b/src/nghttp.cc index 80fad936..94150788 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -250,7 +250,7 @@ bool Request::is_ipv6_literal_addr() const { } } -bool Request::response_pseudo_header_allowed(int token) const { +bool Request::response_pseudo_header_allowed(int16_t token) const { if (!res_nva.empty() && res_nva.back().name.c_str()[0] != ':') { return false; } @@ -262,7 +262,7 @@ bool Request::response_pseudo_header_allowed(int token) const { } } -bool Request::push_request_pseudo_header_allowed(int token) const { +bool Request::push_request_pseudo_header_allowed(int16_t token) const { if (!req_nva.empty() && req_nva.back().name.c_str()[0] != ':') { return false; } @@ -277,7 +277,7 @@ bool Request::push_request_pseudo_header_allowed(int token) const { } } -Headers::value_type *Request::get_res_header(int token) { +Headers::value_type *Request::get_res_header(int16_t token) { auto idx = res_hdidx[token]; if (idx == -1) { return nullptr; @@ -285,7 +285,7 @@ Headers::value_type *Request::get_res_header(int token) { return &res_nva[idx]; } -Headers::value_type *Request::get_req_header(int token) { +Headers::value_type *Request::get_req_header(int16_t token) { auto idx = req_hdidx[token]; if (idx == -1) { return nullptr; @@ -1688,7 +1688,7 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, http2::index_header(req->res_hdidx, token, req->res_nva.size()); http2::add_header(req->res_nva, name, namelen, value, valuelen, - flags & NGHTTP2_NV_FLAG_NO_INDEX); + flags & NGHTTP2_NV_FLAG_NO_INDEX, token); break; } case NGHTTP2_PUSH_PROMISE: { @@ -1712,7 +1712,7 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, http2::index_header(req->req_hdidx, token, req->req_nva.size()); http2::add_header(req->req_nva, name, namelen, value, valuelen, - flags & NGHTTP2_NV_FLAG_NO_INDEX); + flags & NGHTTP2_NV_FLAG_NO_INDEX, token); break; } } diff --git a/src/nghttp.h b/src/nghttp.h index 14b89f94..14bbb6b2 100644 --- a/src/nghttp.h +++ b/src/nghttp.h @@ -122,11 +122,11 @@ struct Request { bool is_ipv6_literal_addr() const; - bool response_pseudo_header_allowed(int token) const; - bool push_request_pseudo_header_allowed(int token) const; + bool response_pseudo_header_allowed(int16_t token) const; + bool push_request_pseudo_header_allowed(int16_t token) const; - Headers::value_type *get_res_header(int token); - Headers::value_type *get_req_header(int token); + Headers::value_type *get_res_header(int16_t token); + Headers::value_type *get_req_header(int16_t token); void record_request_time(); void record_response_time(); diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index bb6fd05f..7e3e901a 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -93,6 +93,9 @@ int main(int argc, char *argv[]) { shrpx::test_http2_http2_header_allowed) || !CU_add_test(pSuite, "http2_mandatory_request_headers_presence", shrpx::test_http2_mandatory_request_headers_presence) || + !CU_add_test(pSuite, "http2_parse_link_header", + shrpx::test_http2_parse_link_header) || + !CU_add_test(pSuite, "http2_path_join", shrpx::test_http2_path_join) || !CU_add_test(pSuite, "downstream_index_request_headers", shrpx::test_downstream_index_request_headers) || !CU_add_test(pSuite, "downstream_index_response_headers", diff --git a/src/shrpx.cc b/src/shrpx.cc index 17896474..689c9ad8 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -782,6 +782,7 @@ void fill_default_config() { mod_config()->tls_ctx_per_worker = false; mod_config()->downstream_request_buffer_size = 16 * 1024; mod_config()->downstream_response_buffer_size = 16 * 1024; + mod_config()->no_server_push = false; } } // namespace @@ -1087,6 +1088,10 @@ HTTP/2 and SPDY: padding. Specify 0 to disable padding. This option is meant for debugging purpose and not intended to enhance protocol security. + --no-server-push + Disable HTTP/2 server push. Server push is only + supported by default mode and HTTP/2 frontend. SPDY + frontend does not support server push. Mode: (default mode) @@ -1345,6 +1350,7 @@ int main(int argc, char **argv) { {"backend-response-buffer", required_argument, &flag, 71}, {"backend-request-buffer", required_argument, &flag, 72}, {"no-host-rewrite", no_argument, &flag, 73}, + {"no-server-push", no_argument, &flag, 74}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -1678,6 +1684,10 @@ int main(int argc, char **argv) { // --no-host-rewrite cmdcfgs.emplace_back(SHRPX_OPT_NO_HOST_REWRITE, "yes"); break; + case 74: + // --no-server-push + cmdcfgs.emplace_back(SHRPX_OPT_NO_SERVER_PUSH, "yes"); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 3c94440c..96775a03 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -145,6 +145,7 @@ const char SHRPX_OPT_RLIMIT_NOFILE[] = "rlimit-nofile"; const char SHRPX_OPT_TLS_CTX_PER_WORKER[] = "tls-ctx-per-worker"; const char SHRPX_OPT_BACKEND_REQUEST_BUFFER[] = "backend-request-buffer"; const char SHRPX_OPT_BACKEND_RESPONSE_BUFFER[] = "backend-response-buffer"; +const char SHRPX_OPT_NO_SERVER_PUSH[] = "no-server-push"; namespace { Config *config = nullptr; @@ -1165,6 +1166,12 @@ int parse_config(const char *opt, const char *optarg) { return 0; } + if (util::strieq(opt, SHRPX_OPT_NO_SERVER_PUSH)) { + mod_config()->no_server_push = util::strieq(optarg, "yes"); + + return 0; + } + if (util::strieq(opt, "conf")) { LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 86eb22b8..ac920b43 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -132,6 +132,7 @@ extern const char SHRPX_OPT_RLIMIT_NOFILE[]; extern const char SHRPX_OPT_TLS_CTX_PER_WORKER[]; extern const char SHRPX_OPT_BACKEND_REQUEST_BUFFER[]; extern const char SHRPX_OPT_BACKEND_RESPONSE_BUFFER[]; +extern const char SHRPX_OPT_NO_SERVER_PUSH[]; union sockaddr_union { sockaddr_storage storage; @@ -304,6 +305,7 @@ struct Config { bool no_host_rewrite; bool auto_tls_ticket_key; bool tls_ctx_per_worker; + bool no_server_push; }; const Config *get_config(); diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index fc0fca8e..86d828dd 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -300,6 +300,7 @@ int index_headers(http2::HeaderIndex &hdidx, Headers &headers, continue; } + kv.token = token; http2::index_header(hdidx, token, i); if (token == http2::HD_CONTENT_LENGTH) { @@ -322,7 +323,7 @@ int Downstream::index_request_headers() { request_content_length_); } -const Headers::value_type *Downstream::get_request_header(int token) const { +const Headers::value_type *Downstream::get_request_header(int16_t token) const { return http2::get_header(request_hdidx_, token, request_headers_); } @@ -346,10 +347,11 @@ void Downstream::set_last_request_header_value(std::string value) { void Downstream::add_request_header(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, - bool no_index, int token) { + bool no_index, int16_t token) { http2::index_header(request_hdidx_, token, request_headers_.size()); request_headers_sum_ += namelen + valuelen; - http2::add_header(request_headers_, name, namelen, value, valuelen, no_index); + http2::add_header(request_headers_, name, namelen, value, valuelen, no_index, + token); } bool Downstream::get_request_header_key_prev() const { @@ -525,7 +527,8 @@ int Downstream::index_response_headers() { response_content_length_); } -const Headers::value_type *Downstream::get_response_header(int token) const { +const Headers::value_type * +Downstream::get_response_header(int16_t token) const { return http2::get_header(response_hdidx_, token, response_headers_); } @@ -577,13 +580,21 @@ void Downstream::set_last_response_header_value(std::string value) { item.value = std::move(value); } +void Downstream::add_response_header(std::string name, std::string value, + int16_t token) { + http2::index_header(response_hdidx_, token, response_headers_.size()); + response_headers_sum_ += name.size() + value.size(); + response_headers_.emplace_back(std::move(name), std::move(value), false, + token); +} + void Downstream::add_response_header(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, - bool no_index, int token) { + bool no_index, int16_t token) { http2::index_header(response_hdidx_, token, response_headers_.size()); response_headers_sum_ += namelen + valuelen; - http2::add_header(response_headers_, name, namelen, value, valuelen, - no_index); + http2::add_header(response_headers_, name, namelen, value, valuelen, no_index, + token); } bool Downstream::get_response_header_key_prev() const { @@ -893,14 +904,14 @@ bool pseudo_header_allowed(const Headers &headers) { } } // namespace -bool Downstream::request_pseudo_header_allowed(int token) const { +bool Downstream::request_pseudo_header_allowed(int16_t token) const { if (!pseudo_header_allowed(request_headers_)) { return false; } return http2::check_http2_request_pseudo_header(request_hdidx_, token); } -bool Downstream::response_pseudo_header_allowed(int token) const { +bool Downstream::response_pseudo_header_allowed(int16_t token) const { if (!pseudo_header_allowed(response_headers_)) { return false; } diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 83162560..f759041a 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -108,7 +108,7 @@ public: // multiple header have |name| as name, return last occurrence from // the beginning. If no such header is found, returns nullptr. // This function must be called after headers are indexed - const Headers::value_type *get_request_header(int token) const; + const Headers::value_type *get_request_header(int16_t token) const; // Returns pointer to the request header with the name |name|. If // no such header is found, returns nullptr. const Headers::value_type *get_request_header(const std::string &name) const; @@ -117,7 +117,7 @@ public: void add_request_header(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, - int token); + int16_t token); bool get_request_header_key_prev() const; void append_last_request_header_key(const char *data, size_t len); @@ -165,7 +165,7 @@ public: bool validate_request_bodylen() const; int64_t get_request_content_length() const; void set_request_content_length(int64_t len); - bool request_pseudo_header_allowed(int token) const; + bool request_pseudo_header_allowed(int16_t token) const; bool expect_response_body() const; enum { INITIAL, @@ -192,16 +192,17 @@ public: // multiple header have |name| as name, return last occurrence from // the beginning. If no such header is found, returns nullptr. // This function must be called after response headers are indexed. - const Headers::value_type *get_response_header(int token) const; + const Headers::value_type *get_response_header(int16_t token) const; // Rewrites the location response header field. void rewrite_location_response_header(const std::string &upstream_scheme, uint16_t upstream_port); void add_response_header(std::string name, std::string value); void set_last_response_header_value(std::string value); + void add_response_header(std::string name, std::string value, int16_t token); void add_response_header(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, - int token); + int16_t token); bool get_response_header_key_prev() const; void append_last_response_header_key(const char *data, size_t len); @@ -248,7 +249,7 @@ public: void dec_response_datalen(size_t len); size_t get_response_datalen() const; void reset_response_datalen(); - bool response_pseudo_header_allowed(int token) const; + bool response_pseudo_header_allowed(int16_t token) const; // Call this method when there is incoming data in downstream // connection. diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index c66c57d4..51e84f33 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -823,7 +823,8 @@ int on_response_headers(Http2Session *http2session, Downstream *downstream, // Otherwise, use chunked encoding to keep upstream connection // open. In HTTP2, we are supporsed not to receive // transfer-encoding. - downstream->add_response_header("transfer-encoding", "chunked"); + downstream->add_response_header("transfer-encoding", "chunked", + http2::HD_TRANSFER_ENCODING); downstream->set_chunked_response(true); } } diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 88828285..64d10697 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -469,14 +469,6 @@ int on_frame_recv_callback(nghttp2_session *session, const nghttp2_frame *frame, } upstream->stop_settings_timer(); break; - case NGHTTP2_PUSH_PROMISE: - rv = nghttp2_submit_rst_stream(session, NGHTTP2_FLAG_NONE, - frame->push_promise.promised_stream_id, - NGHTTP2_REFUSED_STREAM); - if (rv != 0) { - return NGHTTP2_ERR_CALLBACK_FAILURE; - } - break; case NGHTTP2_GOAWAY: if (LOG_ENABLED(INFO)) { auto debug_data = util::ascii_dump(frame->goaway.opaque_data, @@ -540,6 +532,51 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame, upstream->start_settings_timer(); } break; + case NGHTTP2_PUSH_PROMISE: { + auto downstream = make_unique( + upstream, frame->push_promise.promised_stream_id, 0); + + downstream->disable_upstream_rtimer(); + + downstream->set_request_major(2); + downstream->set_request_minor(0); + + for (size_t i = 0; i < frame->push_promise.nvlen; ++i) { + auto &nv = frame->push_promise.nva[i]; + auto token = http2::lookup_token(nv.name, nv.namelen); + switch (token) { + case http2::HD__METHOD: + downstream->set_request_method({nv.value, nv.value + nv.valuelen}); + break; + case http2::HD__SCHEME: + downstream->set_request_http2_scheme( + {nv.value, nv.value + nv.valuelen}); + break; + case http2::HD__AUTHORITY: + downstream->set_request_http2_authority( + {nv.value, nv.value + nv.valuelen}); + break; + case http2::HD__PATH: + downstream->set_request_path({nv.value, nv.value + nv.valuelen}); + break; + } + downstream->add_request_header(nv.name, nv.namelen, nv.value, nv.valuelen, + nv.flags & NGHTTP2_NV_FLAG_NO_INDEX, + token); + } + + downstream->inspect_http2_request(); + + downstream->set_request_state(Downstream::MSG_COMPLETE); + + // a bit weird but start_downstream() expects that given + // downstream is in pending queue. + auto ptr = downstream.get(); + upstream->add_pending_downstream(std::move(downstream)); + upstream->start_downstream(ptr); + + break; + } case NGHTTP2_GOAWAY: if (LOG_ENABLED(INFO)) { auto debug_data = util::ascii_dump(frame->goaway.opaque_data, @@ -1283,6 +1320,35 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { return -1; } + // We need some conditions that must be fulfilled to initiate server + // push. + // + // * Server push is disabled for http2 proxy, since incoming headers + // are mixed origins. We don't know how to reliably determine the + // authority yet. + // + // * If downstream is http/2, it is likely that PUSH_PROMISE is + // coming from there, so we don't initiate PUSH_RPOMISE here. + // + // * We need 200 response code for associated resource. This is too + // restrictive, we will review this later. + // + // * We requires GET or POST for associated resource. Probably we + // don't want to push for HEAD request. Not sure other methods + // are also eligible for push. + if (!get_config()->no_server_push && + get_config()->downstream_proto == PROTO_HTTP && + !get_config()->http2_proxy && (downstream->get_stream_id() % 2) && + downstream->get_response_header(http2::HD_LINK) && + downstream->get_response_http_status() == 200 && + (downstream->get_request_method() == "GET" || + downstream->get_request_method() == "POST")) { + + if (prepare_push_promise(downstream) != 0) { + return -1; + } + } + return 0; } @@ -1447,4 +1513,138 @@ int Http2Upstream::on_downstream_reset(bool no_retry) { MemchunkPool *Http2Upstream::get_mcpool() { return &mcpool_; } +int Http2Upstream::prepare_push_promise(Downstream *downstream) { + int rv; + http_parser_url u; + memset(&u, 0, sizeof(u)); + rv = http_parser_parse_url(downstream->get_request_path().c_str(), + downstream->get_request_path().size(), 0, &u); + if (rv != 0) { + return 0; + } + const char *base; + size_t baselen; + if (u.field_set & (1 << UF_PATH)) { + auto &f = u.field_data[UF_PATH]; + base = downstream->get_request_path().c_str() + f.off; + baselen = f.len; + } else { + base = "/"; + baselen = 1; + } + for (auto &kv : downstream->get_response_headers()) { + if (kv.token != http2::HD_LINK) { + continue; + } + for (auto &link : + http2::parse_link_header(kv.value.c_str(), kv.value.size())) { + auto link_url = link.uri.first; + auto link_urllen = link.uri.second - link.uri.first; + + const char *rel; + size_t rellen; + const char *relq = nullptr; + size_t relqlen = 0; + + http_parser_url v; + memset(&v, 0, sizeof(v)); + rv = http_parser_parse_url(link_url, link_urllen, 0, &v); + if (rv != 0) { + assert(link_urllen); + if (link_url[0] == '/') { + continue; + } + // treat link_url as relative URI. + auto end = std::find(link_url, link_url + link_urllen, '#'); + auto q = std::find(link_url, end, '?'); + rel = link_url; + rellen = q - link_url; + if (q != end) { + relq = q + 1; + relqlen = end - relq; + } + } else { + if (v.field_set & (1 << UF_HOST)) { + continue; + } + if (v.field_set & (1 << UF_PATH)) { + auto &f = v.field_data[UF_PATH]; + rel = link_url + f.off; + rellen = f.len; + } else { + rel = "/"; + rellen = 1; + } + + if (v.field_set & (1 << UF_QUERY)) { + auto &f = v.field_data[UF_QUERY]; + relq = link_url + f.off; + relqlen = f.len; + } + } + auto path = http2::path_join(base, baselen, nullptr, 0, rel, rellen, relq, + relqlen); + rv = submit_push_promise(path, downstream); + if (rv != 0) { + return -1; + } + } + } + return 0; +} + +int Http2Upstream::submit_push_promise(const std::string &path, + Downstream *downstream) { + int rv; + std::vector nva; + nva.reserve(downstream->get_request_headers().size()); + for (auto &kv : downstream->get_request_headers()) { + switch (kv.token) { + // TODO generate referer + case http2::HD__AUTHORITY: + case http2::HD__SCHEME: + case http2::HD_ACCEPT_ENCODING: + case http2::HD_ACCEPT_LANGUAGE: + case http2::HD_CACHE_CONTROL: + case http2::HD_HOST: + case http2::HD_USER_AGENT: + nva.push_back(http2::make_nv(kv.name, kv.value, kv.no_index)); + break; + case http2::HD__METHOD: + // juse use "GET" for now + nva.push_back(http2::make_nv_lc(":method", "GET")); + continue; + case http2::HD__PATH: + nva.push_back(http2::make_nv_ls(":path", path)); + continue; + } + } + + rv = nghttp2_submit_push_promise(session_, NGHTTP2_FLAG_NONE, + downstream->get_stream_id(), nva.data(), + nva.size(), nullptr); + + if (rv < 0) { + if (LOG_ENABLED(INFO)) { + ULOG(INFO, this) << "nghttp2_submit_push_promise() failed: " + << nghttp2_strerror(rv); + } + if (nghttp2_is_fatal(rv)) { + return -1; + } + return 0; + } + + if (LOG_ENABLED(INFO)) { + std::stringstream ss; + for (auto &nv : nva) { + ss << TTY_HTTP_HD << nv.name << TTY_RST << ": " << nv.value << "\n"; + } + ULOG(INFO, this) << "HTTP push request headers. promised_stream_id=" << rv + << "\n" << ss.str(); + } + + return 0; +} + } // namespace shrpx diff --git a/src/shrpx_http2_upstream.h b/src/shrpx_http2_upstream.h index 8e1eaf5d..1278ebc3 100644 --- a/src/shrpx_http2_upstream.h +++ b/src/shrpx_http2_upstream.h @@ -100,6 +100,9 @@ public: void submit_goaway(); void check_shutdown(); + int prepare_push_promise(Downstream *downstream); + int submit_push_promise(const std::string &path, Downstream *downstream); + private: // must be put before downstream_queue_ std::unique_ptr pre_upstream_; diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index ede2625b..89fdcaf7 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -860,8 +860,7 @@ int SpdyUpstream::on_downstream_header_complete(Downstream *downstream) { if (hd.name.empty() || hd.name.c_str()[0] == ':') { continue; } - auto token = http2::lookup_token(hd.name); - switch (token) { + switch (hd.token) { case http2::HD_CONNECTION: case http2::HD_KEEP_ALIVE: case http2::HD_PROXY_CONNECTION: diff --git a/src/util.cc b/src/util.cc index 9bcf1d3a..9d133e6f 100644 --- a/src/util.cc +++ b/src/util.cc @@ -99,6 +99,12 @@ bool in_token(char c) { &extra[sizeof(extra)]; } +bool in_attr_char(char c) { + static const char bad[] = {'*', '\'', '%'}; + return util::in_token(c) && + std::find(std::begin(bad), std::end(bad) - 1, c) == std::end(bad) - 1; +} + std::string percent_encode_token(const std::string &target) { auto len = target.size(); std::string dest; diff --git a/src/util.h b/src/util.h index 0d66c931..9e42b647 100644 --- a/src/util.h +++ b/src/util.h @@ -163,6 +163,8 @@ bool inRFC3986UnreservedChars(const char c); // Returns true if |c| is in token (HTTP-p1, Section 3.2.6) bool in_token(char c); +bool in_attr_char(char c); + std::string percentEncode(const unsigned char *target, size_t len); std::string percentEncode(const std::string &target);