From b1b57cc740e25d989f562d3dba5bab8cef3a7196 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 10 Mar 2016 22:42:07 +0900 Subject: [PATCH] nghttpx: Use StringRef for authority, scheme and path --- src/allocator.h | 13 ++ src/http2.cc | 272 ++++++++++++++++++++--- src/http2.h | 27 ++- src/shrpx_client_handler.cc | 37 ++- src/shrpx_client_handler.h | 2 +- src/shrpx_downstream.cc | 13 +- src/shrpx_downstream.h | 12 +- src/shrpx_downstream_queue.cc | 13 +- src/shrpx_downstream_queue.h | 6 +- src/shrpx_http2_downstream_connection.cc | 10 +- src/shrpx_http2_session.cc | 13 +- src/shrpx_http2_upstream.cc | 76 ++++--- src/shrpx_http2_upstream.h | 5 +- src/shrpx_http_downstream_connection.cc | 10 +- src/shrpx_https_upstream.cc | 99 ++++++--- src/shrpx_mruby_module_request.cc | 15 +- src/shrpx_mruby_module_response.cc | 18 +- src/shrpx_spdy_upstream.cc | 13 +- src/template.h | 3 +- src/util.h | 22 ++ 20 files changed, 511 insertions(+), 168 deletions(-) diff --git a/src/allocator.h b/src/allocator.h index 998653be..00acdbcf 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -27,6 +27,8 @@ #include "nghttp2_config.h" +#include + #include "template.h" namespace nghttp2 { @@ -110,6 +112,17 @@ StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, return StringRef{dst, a.size() + b.size()}; } +struct ByteRef { + uint8_t *base; + size_t len; +}; + +template +ByteRef make_byte_ref(BlockAllocator &alloc, size_t size) { + auto dst = static_cast(alloc.alloc(size)); + return {dst, size}; +} + } // namespace aria2 #endif // ALLOCATOR_H diff --git a/src/http2.cc b/src/http2.cc index 17de9b05..28314134 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -484,41 +484,69 @@ void dump_nv(FILE *out, const HeaderRefs &nva) { fflush(out); } -std::string rewrite_location_uri(const StringRef &uri, const http_parser_url &u, - const std::string &match_host, - const std::string &request_authority, - const std::string &upstream_scheme) { +StringRef rewrite_location_uri(BlockAllocator &balloc, const StringRef &uri, + const http_parser_url &u, + const StringRef &match_host, + const StringRef &request_authority, + const StringRef &upstream_scheme) { // We just rewrite scheme and authority. if ((u.field_set & (1 << UF_HOST)) == 0) { - return ""; + return StringRef{}; } auto field = &u.field_data[UF_HOST]; if (!util::starts_with(std::begin(match_host), std::end(match_host), &uri[field->off], &uri[field->off] + field->len) || (match_host.size() != field->len && match_host[field->len] != ':')) { - return ""; + return StringRef{}; } - std::string res; + + auto len = 0; if (!request_authority.empty()) { - res += upstream_scheme; - res += "://"; - res += request_authority; + len += upstream_scheme.size() + str_size("://") + request_authority.size(); + } + + if (u.field_set & (1 << UF_PATH)) { + field = &u.field_data[UF_PATH]; + len += field->len; + } + + if (u.field_set & (1 << UF_QUERY)) { + field = &u.field_data[UF_QUERY]; + len += 1 + field->len; + } + + if (u.field_set & (1 << UF_FRAGMENT)) { + field = &u.field_data[UF_FRAGMENT]; + len += 1 + field->len; + } + + auto iov = make_byte_ref(balloc, len + 1); + auto p = iov.base; + + if (!request_authority.empty()) { + p = std::copy(std::begin(upstream_scheme), std::end(upstream_scheme), p); + p = util::copy_lit(p, "://"); + p = std::copy(std::begin(request_authority), std::end(request_authority), + p); } if (u.field_set & (1 << UF_PATH)) { field = &u.field_data[UF_PATH]; - res.append(&uri[field->off], field->len); + p = std::copy_n(&uri[field->off], field->len, p); } if (u.field_set & (1 << UF_QUERY)) { field = &u.field_data[UF_QUERY]; - res += '?'; - res.append(&uri[field->off], field->len); + *p++ = '?'; + p = std::copy_n(&uri[field->off], field->len, p); } if (u.field_set & (1 << UF_FRAGMENT)) { field = &u.field_data[UF_FRAGMENT]; - res += '#'; - res.append(&uri[field->off], field->len); + *p++ = '#'; + p = std::copy_n(&uri[field->off], field->len, p); } - return res; + + *p = '\0'; + + return StringRef{iov.base, p}; } int check_nv(const uint8_t *name, size_t namelen, const uint8_t *value, @@ -1496,7 +1524,7 @@ StringRef to_method_string(int method_token) { return StringRef{http_method_str(static_cast(method_token))}; } -StringRef get_pure_path_component(const std::string &uri) { +StringRef get_pure_path_component(const StringRef &uri) { int rv; http_parser_url u{}; @@ -1513,9 +1541,9 @@ StringRef get_pure_path_component(const std::string &uri) { return StringRef::from_lit("/"); } -int construct_push_component(std::string &scheme, std::string &authority, - std::string &path, const StringRef &base, - const StringRef &uri) { +int construct_push_component(BlockAllocator &balloc, StringRef &scheme, + StringRef &authority, StringRef &path, + const StringRef &base, const StringRef &uri) { int rv; StringRef rel, relq; @@ -1538,15 +1566,26 @@ int construct_push_component(std::string &scheme, std::string &authority, } } else { if (u.field_set & (1 << UF_SCHEMA)) { - http2::copy_url_component(scheme, &u, UF_SCHEMA, uri.c_str()); + scheme = util::get_uri_field(uri.c_str(), u, UF_SCHEMA); } if (u.field_set & (1 << UF_HOST)) { - http2::copy_url_component(authority, &u, UF_HOST, uri.c_str()); - if (u.field_set & (1 << UF_PORT)) { - authority += ':'; - authority += util::utos(u.port); + auto auth = util::get_uri_field(uri.c_str(), u, UF_HOST); + auto len = auth.size(); + auto port_exists = u.field_set & (1 << UF_PORT); + if (port_exists) { + len += 1 + str_size("65535"); } + auto iov = make_byte_ref(balloc, len + 1); + auto p = iov.base; + p = std::copy(std::begin(auth), std::end(auth), p); + if (port_exists) { + *p++ = ':'; + p = util::utos(p, u.port); + } + *p = '\0'; + + authority = StringRef{iov.base, p}; } if (u.field_set & (1 << UF_PATH)) { @@ -1562,11 +1601,192 @@ int construct_push_component(std::string &scheme, std::string &authority, } } - path = http2::path_join(base, StringRef{}, rel, relq); + path = http2::path_join(balloc, base, StringRef{}, rel, relq); return 0; } +namespace { +template InputIt eat_file(InputIt first, InputIt last) { + if (first == last) { + *first++ = '/'; + return first; + } + + if (*(last - 1) == '/') { + return last; + } + + auto p = last; + for (; p != first && *(p - 1) != '/'; --p) + ; + if (p == first) { + // this should not happend in normal case, where we expect path + // starts with '/' + *first++ = '/'; + return first; + } + + return p; +} +} // namespace + +namespace { +template InputIt eat_dir(InputIt first, InputIt last) { + auto p = eat_file(first, last); + + --p; + + assert(*p == '/'); + + return eat_file(first, p); +} +} // namespace + +StringRef path_join(BlockAllocator &balloc, const StringRef &base_path, + const StringRef &base_query, const StringRef &rel_path, + const StringRef &rel_query) { + auto res = make_byte_ref( + balloc, std::max(static_cast(1), base_path.size()) + + rel_path.size() + 1 + + std::max(base_query.size(), rel_query.size()) + 1); + auto p = res.base; + + if (rel_path.empty()) { + if (base_path.empty()) { + *p++ = '/'; + } else { + p = std::copy(std::begin(base_path), std::end(base_path), p); + } + if (rel_query.empty()) { + if (!base_query.empty()) { + *p++ = '?'; + p = std::copy(std::begin(base_query), std::end(base_query), p); + } + *p = '\0'; + return StringRef{res.base, p}; + } + *p++ = '?'; + p = std::copy(std::begin(rel_query), std::end(rel_query), p); + *p = '\0'; + return StringRef{res.base, p}; + } + + auto first = std::begin(rel_path); + auto last = std::end(rel_path); + + if (rel_path[0] == '/') { + *p++ = '/'; + ++first; + } else if (base_path.empty()) { + *p++ = '/'; + } else { + p = std::copy(std::begin(base_path), std::end(base_path), p); + } + + for (; first != last;) { + if (*first == '.') { + if (first + 1 == last) { + break; + } + if (*(first + 1) == '/') { + first += 2; + continue; + } + if (*(first + 1) == '.') { + if (first + 2 == last) { + p = eat_dir(res.base, p); + break; + } + if (*(first + 2) == '/') { + p = eat_dir(res.base, p); + first += 3; + continue; + } + } + } + if (*(p - 1) != '/') { + p = eat_file(res.base, p); + } + auto slash = std::find(first, last, '/'); + if (slash == last) { + p = std::copy(first, last, p); + break; + } + p = std::copy(first, slash + 1, p); + first = slash + 1; + for (; first != last && *first == '/'; ++first) + ; + } + if (!rel_query.empty()) { + *p++ = '?'; + p = std::copy(std::begin(rel_query), std::end(rel_query), p); + } + *p = '\0'; + return StringRef{res.base, p}; +} + +StringRef normalize_path(BlockAllocator &balloc, const StringRef &path, + const StringRef &query) { + // First, decode %XX for unreserved characters, then do + // http2::join_path + + // We won't find %XX if length is less than 3. + if (path.size() < 3 || + std::find(std::begin(path), std::end(path), '%') == std::end(path)) { + return path_join(balloc, StringRef{}, StringRef{}, path, query); + } + + // includes last terminal NULL. + auto result = make_byte_ref(balloc, path.size() + 1); + auto p = result.base; + + auto it = std::begin(path); + for (; it + 2 != std::end(path);) { + if (*it == '%') { + if (util::is_hex_digit(*(it + 1)) && util::is_hex_digit(*(it + 2))) { + auto c = + (util::hex_to_uint(*(it + 1)) << 4) + util::hex_to_uint(*(it + 2)); + if (util::in_rfc3986_unreserved_chars(c)) { + *p++ = c; + + it += 3; + + continue; + } + *p++ = '%'; + *p++ = util::upcase(*(it + 1)); + *p++ = util::upcase(*(it + 2)); + + it += 3; + + continue; + } + } + *p++ = *it++; + } + + p = std::copy(it, std::end(path), p); + *p = '\0'; + + return path_join(balloc, StringRef{}, StringRef{}, StringRef{result.base, p}, + query); +} + +StringRef rewrite_clean_path(BlockAllocator &balloc, const StringRef &src) { + if (src.empty() || src[0] != '/') { + return src; + } + // probably, not necessary most of the case, but just in case. + auto fragment = std::find(std::begin(src), std::end(src), '#'); + auto query = std::find(std::begin(src), fragment, '?'); + if (query != fragment) { + ++query; + } + return normalize_path(balloc, StringRef{std::begin(src), query}, + StringRef{query, fragment}); +} + } // namespace http2 } // namespace nghttp2 diff --git a/src/http2.h b/src/http2.h index b4e84e8b..9b8b0bbb 100644 --- a/src/http2.h +++ b/src/http2.h @@ -40,6 +40,7 @@ #include "util.h" #include "memchunk.h" #include "template.h" +#include "allocator.h" namespace nghttp2 { @@ -241,10 +242,11 @@ void dump_nv(FILE *out, const HeaderRefs &nva); // This function returns the new rewritten URI on success. If the // location URI is not subject to the rewrite, this function returns // emtpy string. -std::string rewrite_location_uri(const StringRef &uri, const http_parser_url &u, - const std::string &match_host, - const std::string &request_authority, - const std::string &upstream_scheme); +StringRef rewrite_location_uri(BlockAllocator &balloc, const StringRef &uri, + const http_parser_url &u, + const StringRef &match_host, + const StringRef &request_authority, + const StringRef &upstream_scheme); // Checks the header name/value pair using nghttp2_check_header_name() // and nghttp2_check_header_value(). If both function returns nonzero, @@ -337,6 +339,10 @@ std::vector parse_link_header(const char *src, size_t len); std::string path_join(const StringRef &base, const StringRef &base_query, const StringRef &rel_path, const StringRef &rel_query); +StringRef path_join(BlockAllocator &balloc, const StringRef &base_path, + const StringRef &base_query, const StringRef &rel_path, + const StringRef &rel_query); + // true if response has body, taking into account the request method // and status code. bool expect_response_body(const std::string &method, int status_code); @@ -407,19 +413,24 @@ std::string rewrite_clean_path(InputIt first, InputIt last) { return path; } +StringRef normalize_path(BlockAllocator &balloc, const StringRef &path, + const StringRef &query); + +StringRef rewrite_clean_path(BlockAllocator &balloc, const StringRef &src); + // Returns path component of |uri|. The returned path does not // include query component. This function returns empty string if it // fails. -StringRef get_pure_path_component(const std::string &uri); +StringRef get_pure_path_component(const StringRef &uri); // Deduces scheme, authority and path from given |uri|, and stores // them in |scheme|, |authority|, and |path| respectively. If |uri| // is relative path, path resolution takes place using path given in // |base| of length |baselen|. This function returns 0 if it // succeeds, or -1. -int construct_push_component(std::string &scheme, std::string &authority, - std::string &path, const StringRef &base, - const StringRef &uri); +int construct_push_component(BlockAllocator &balloc, StringRef &scheme, + StringRef &authority, StringRef &path, + const StringRef &base, const StringRef &uri); } // namespace http2 diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 3326e885..d7366b04 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -825,11 +825,11 @@ int ClientHandler::perform_http2_upgrade(HttpsUpstream *http) { bool ClientHandler::get_http2_upgrade_allowed() const { return !conn_.tls.ssl; } -std::string ClientHandler::get_upstream_scheme() const { +StringRef ClientHandler::get_upstream_scheme() const { if (conn_.tls.ssl) { - return "https"; + return StringRef::from_lit("https"); } else { - return "http"; + return StringRef::from_lit("http"); } } @@ -843,24 +843,35 @@ namespace { // is mostly same routine found in // HttpDownstreamConnection::push_request_headers(), but vastly // simplified since we only care about absolute URI. -std::string construct_absolute_request_uri(const Request &req) { +StringRef construct_absolute_request_uri(BlockAllocator &balloc, + const Request &req) { if (req.authority.empty()) { return req.path; } - std::string uri; + auto len = req.authority.size() + req.path.size(); + if (req.scheme.empty()) { + len += str_size("http://"); + } else { + len += req.scheme.size() + str_size("://"); + } + + auto iov = make_byte_ref(balloc, len + 1); + auto p = iov.base; + if (req.scheme.empty()) { // We may have to log the request which lacks scheme (e.g., // http/1.1 with origin form). - uri += "http://"; + p = util::copy_lit(p, "http://"); } else { - uri += req.scheme; - uri += "://"; + p = std::copy(std::begin(req.scheme), std::end(req.scheme), p); + p = util::copy_lit(p, "://"); } - uri += req.authority; - uri += req.path; + p = std::copy(std::begin(req.authority), std::end(req.authority), p); + p = std::copy(std::begin(req.path), std::end(req.path), p); + *p = '\0'; - return uri; + return StringRef{iov.base, p}; } } // namespace @@ -869,6 +880,8 @@ void ClientHandler::write_accesslog(Downstream *downstream) { const auto &req = downstream->request(); const auto &resp = downstream->response(); + auto &balloc = downstream->get_block_allocator(); + upstream_accesslog( get_config()->logging.access.format, LogSpec{ @@ -877,7 +890,7 @@ void ClientHandler::write_accesslog(Downstream *downstream) { req.method == HTTP_CONNECT ? StringRef(req.authority) : get_config()->http2_proxy - ? StringRef(construct_absolute_request_uri(req)) + ? StringRef(construct_absolute_request_uri(balloc, req)) : req.path.empty() ? req.method == HTTP_OPTIONS ? StringRef::from_lit("*") diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index ec353576..8d1bc7c0 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -109,7 +109,7 @@ public: int perform_http2_upgrade(HttpsUpstream *http); bool get_http2_upgrade_allowed() const; // Returns upstream scheme, either "http" or "https" - std::string get_upstream_scheme() const; + StringRef get_upstream_scheme() const; void start_immediate_shutdown(); // Writes upstream accesslog using |downstream|. The |downstream| diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index ed1edf2e..f5e4f084 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -561,7 +561,7 @@ int Downstream::end_upload_data() { } void Downstream::rewrite_location_response_header( - const std::string &upstream_scheme) { + const StringRef &upstream_scheme) { auto hd = resp_.fs.header(http2::HD_LOCATION); if (!hd) { return; @@ -577,14 +577,15 @@ void Downstream::rewrite_location_response_header( return; } - auto new_uri = http2::rewrite_location_uri( - hd->value, u, request_downstream_host_, req_.authority, upstream_scheme); + auto new_uri = http2::rewrite_location_uri(balloc_, hd->value, u, + request_downstream_host_, + req_.authority, upstream_scheme); if (new_uri.empty()) { return; } - hd->value = make_string_ref(balloc_, StringRef{new_uri}); + hd->value = new_uri; } bool Downstream::get_chunked_response() const { return chunked_response_; } @@ -879,8 +880,8 @@ void Downstream::add_retry() { ++num_retry_; } bool Downstream::no_more_retry() const { return num_retry_ > 5; } -void Downstream::set_request_downstream_host(std::string host) { - request_downstream_host_ = std::move(host); +void Downstream::set_request_downstream_host(const StringRef &host) { + request_downstream_host_ = host; } void Downstream::set_request_pending(bool f) { request_pending_ = f; } diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 2ea0bf2e..cea17d86 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -147,17 +147,17 @@ struct Request { FieldStore fs; // Request scheme. For HTTP/2, this is :scheme header field value. // For HTTP/1.1, this is deduced from URI or connection. - std::string scheme; + StringRef scheme; // Request authority. This is HTTP/2 :authority header field value // or host header field value. We may deduce it from absolute-form // HTTP/1 request. We also store authority-form HTTP/1 request. // This could be empty if request comes from HTTP/1.0 without Host // header field and origin-form. - std::string authority; + StringRef authority; // Request path, including query component. For HTTP/1.1, this is // request-target. For HTTP/2, this is :path header field value. // For CONNECT request, this is empty. - std::string path; + StringRef path; // the length of request body received so far int64_t recv_body_length; // The number of bytes not consumed by the application yet. @@ -275,7 +275,7 @@ public: // Validates that received request body length and content-length // matches. bool validate_request_recv_body_length() const; - void set_request_downstream_host(std::string host); + void set_request_downstream_host(const StringRef &host); bool expect_response_body() const; enum { INITIAL, @@ -306,7 +306,7 @@ public: Response &response() { return resp_; } // Rewrites the location response header field. - void rewrite_location_response_header(const std::string &upstream_scheme); + void rewrite_location_response_header(const StringRef &upstream_scheme); bool get_chunked_response() const; void set_chunked_response(bool f); @@ -407,7 +407,7 @@ private: // host we requested to downstream. This is used to rewrite // location header field to decide the location should be rewritten // or not. - std::string request_downstream_host_; + StringRef request_downstream_host_; DefaultMemchunks request_buf_; DefaultMemchunks response_buf_; diff --git a/src/shrpx_downstream_queue.cc b/src/shrpx_downstream_queue.cc index 893effb9..91e88cff 100644 --- a/src/shrpx_downstream_queue.cc +++ b/src/shrpx_downstream_queue.cc @@ -71,14 +71,11 @@ DownstreamQueue::find_host_entry(const std::string &host) { return (*itr).second; } -const std::string & -DownstreamQueue::make_host_key(const std::string &host) const { - static std::string empty_key; - return unified_host_ ? empty_key : host; +std::string DownstreamQueue::make_host_key(const StringRef &host) const { + return unified_host_ ? "" : host.str(); } -const std::string & -DownstreamQueue::make_host_key(Downstream *downstream) const { +std::string DownstreamQueue::make_host_key(Downstream *downstream) const { return make_host_key(downstream->request().authority); } @@ -99,7 +96,7 @@ void DownstreamQueue::mark_blocked(Downstream *downstream) { ent.blocked.append(link); } -bool DownstreamQueue::can_activate(const std::string &host) const { +bool DownstreamQueue::can_activate(const StringRef &host) const { auto itr = host_entries_.find(make_host_key(host)); if (itr == std::end(host_entries_)) { return true; @@ -127,7 +124,7 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream, downstreams_.remove(downstream); - auto &host = make_host_key(downstream); + auto host = make_host_key(downstream); auto &ent = find_host_entry(host); if (downstream->get_dispatch_state() == Downstream::DISPATCH_ACTIVE) { diff --git a/src/shrpx_downstream_queue.h b/src/shrpx_downstream_queue.h index 755af10d..47e8555a 100644 --- a/src/shrpx_downstream_queue.h +++ b/src/shrpx_downstream_queue.h @@ -77,7 +77,7 @@ public: void mark_blocked(Downstream *downstream); // Returns true if we can make downstream connection to given // |host|. - bool can_activate(const std::string &host) const; + bool can_activate(const StringRef &host) const; // Removes and frees |downstream| object. If |downstream| is in // Downstream::DISPATCH_ACTIVE, and |next_blocked| is true, this // function may return Downstream object with the same target host @@ -87,8 +87,8 @@ public: bool next_blocked = true); Downstream *get_downstreams() const; HostEntry &find_host_entry(const std::string &host); - const std::string &make_host_key(const std::string &host) const; - const std::string &make_host_key(Downstream *downstream) const; + std::string make_host_key(const StringRef &host) const; + std::string make_host_key(Downstream *downstream) const; private: // Per target host structure to keep track of the number of diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index 357bd664..5e2d5129 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -282,10 +282,10 @@ int Http2DownstreamConnection::push_request_headers() { auto authority = StringRef(downstream_hostport); if (no_host_rewrite && !req.authority.empty()) { - authority = StringRef(req.authority); + authority = req.authority; } - downstream_->set_request_downstream_host(authority.str()); + downstream_->set_request_downstream_host(authority); size_t num_cookies = 0; if (!http2conf.no_cookie_crumbling) { @@ -359,9 +359,9 @@ int Http2DownstreamConnection::push_request_headers() { params &= ~FORWARDED_PROTO; } - auto value = http::create_forwarded( - params, handler->get_forwarded_by(), handler->get_forwarded_for(), - StringRef{req.authority}, StringRef{req.scheme}); + auto value = http::create_forwarded(params, handler->get_forwarded_by(), + handler->get_forwarded_for(), + req.authority, req.scheme); if (fwd || !value.empty()) { if (fwd) { forwarded_value = fwd->value.str(); diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 9a411ca1..828d888c 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -1986,6 +1986,8 @@ int Http2Session::handle_downstream_push_promise_complete( Downstream *downstream, Downstream *promised_downstream) { auto &promised_req = promised_downstream->request(); + auto &promised_balloc = promised_downstream->get_block_allocator(); + auto authority = promised_req.fs.header(http2::HD__AUTHORITY); auto path = promised_req.fs.header(http2::HD__PATH); auto method = promised_req.fs.header(http2::HD__METHOD); @@ -2007,16 +2009,19 @@ int Http2Session::handle_downstream_push_promise_complete( // TODO Rewrite authority if we enabled rewrite host. But we // really don't know how to rewrite host. Should we use the same // host in associated stream? - promised_req.authority = http2::value_to_str(authority); + if (authority) { + promised_req.authority = authority->value; + } promised_req.method = method_token; // libnghttp2 ensures that we don't have CONNECT method in // PUSH_PROMISE, and guarantees that :scheme exists. - promised_req.scheme = http2::value_to_str(scheme); + if (scheme) { + promised_req.scheme = scheme->value; + } // For server-wide OPTIONS request, path is empty. if (method_token != HTTP_OPTIONS || path->value != "*") { - promised_req.path = http2::rewrite_clean_path(std::begin(path->value), - std::end(path->value)); + promised_req.path = http2::rewrite_clean_path(promised_balloc, path->value); } promised_downstream->inspect_http2_request(); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index cf89bf3e..5bfdb826 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -303,7 +303,9 @@ int Http2Upstream::on_request_headers(Downstream *downstream, } req.method = method_token; - req.scheme = http2::value_to_str(scheme); + if (scheme) { + req.scheme = scheme->value; + } // nghttp2 library guarantees either :authority or host exist if (!authority) { @@ -311,16 +313,18 @@ int Http2Upstream::on_request_headers(Downstream *downstream, authority = req.fs.header(http2::HD_HOST); } - req.authority = http2::value_to_str(authority); + if (authority) { + req.authority = authority->value; + } if (path) { if (method_token == HTTP_OPTIONS && path->value == "*") { // Server-wide OPTIONS request. Path is empty. } else if (get_config()->http2_proxy) { - req.path = http2::value_to_str(path); + req.path = path->value; } else { - const auto &value = path->value; - req.path = http2::rewrite_clean_path(std::begin(value), std::end(value)); + req.path = http2::rewrite_clean_path(downstream->get_block_allocator(), + path->value); } } @@ -580,6 +584,8 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame, req.http_major = 2; req.http_minor = 0; + auto &promised_balloc = promised_downstream->get_block_allocator(); + for (size_t i = 0; i < frame->push_promise.nvlen; ++i) { auto &nv = frame->push_promise.nva[i]; auto token = http2::lookup_token(nv.name, nv.namelen); @@ -588,13 +594,16 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame, req.method = http2::lookup_method_token(nv.value, nv.valuelen); break; case http2::HD__SCHEME: - req.scheme.assign(nv.value, nv.value + nv.valuelen); + req.scheme = + make_string_ref(promised_balloc, StringRef{nv.value, nv.valuelen}); break; case http2::HD__AUTHORITY: - req.authority.assign(nv.value, nv.value + nv.valuelen); + req.authority = + make_string_ref(promised_balloc, StringRef{nv.value, nv.valuelen}); break; case http2::HD__PATH: - req.path = http2::rewrite_clean_path(nv.value, nv.value + nv.valuelen); + req.path = http2::rewrite_clean_path(promised_balloc, + StringRef{nv.value, nv.valuelen}); break; } req.fs.add_header_token(StringRef{nv.name, nv.namelen}, @@ -1746,6 +1755,8 @@ int Http2Upstream::prepare_push_promise(Downstream *downstream) { return 0; } + auto &balloc = downstream->get_block_allocator(); + for (auto &kv : resp.fs.headers()) { if (kv.token != http2::HD_LINK) { continue; @@ -1753,28 +1764,23 @@ int Http2Upstream::prepare_push_promise(Downstream *downstream) { for (auto &link : http2::parse_link_header(kv.value.c_str(), kv.value.size())) { - const std::string *scheme_ptr, *authority_ptr; - std::string scheme, authority, path; + StringRef scheme, authority, path; - rv = http2::construct_push_component(scheme, authority, path, base, - link.uri); + rv = http2::construct_push_component(balloc, scheme, authority, path, + base, link.uri); if (rv != 0) { continue; } if (scheme.empty()) { - scheme_ptr = &req.scheme; - } else { - scheme_ptr = &scheme; + scheme = req.scheme; } if (authority.empty()) { - authority_ptr = &req.authority; - } else { - authority_ptr = &authority; + authority = req.authority; } - rv = submit_push_promise(*scheme_ptr, *authority_ptr, path, downstream); + rv = submit_push_promise(scheme, authority, path, downstream); if (rv != 0) { return -1; } @@ -1783,9 +1789,9 @@ int Http2Upstream::prepare_push_promise(Downstream *downstream) { return 0; } -int Http2Upstream::submit_push_promise(const std::string &scheme, - const std::string &authority, - const std::string &path, +int Http2Upstream::submit_push_promise(const StringRef &scheme, + const StringRef &authority, + const StringRef &path, Downstream *downstream) { const auto &req = downstream->request(); @@ -1795,9 +1801,9 @@ int Http2Upstream::submit_push_promise(const std::string &scheme, // juse use "GET" for now nva.push_back(http2::make_nv_ll(":method", "GET")); - nva.push_back(http2::make_nv_ls(":scheme", scheme)); - nva.push_back(http2::make_nv_ls(":path", path)); - nva.push_back(http2::make_nv_ls(":authority", authority)); + nva.push_back(http2::make_nv_ls_nocopy(":scheme", scheme)); + nva.push_back(http2::make_nv_ls_nocopy(":path", path)); + nva.push_back(http2::make_nv_ls_nocopy(":authority", authority)); for (auto &kv : req.fs.headers()) { switch (kv.token) { @@ -1865,27 +1871,25 @@ int Http2Upstream::initiate_push(Downstream *downstream, const StringRef &uri) { return -1; } - const std::string *scheme_ptr, *authority_ptr; - std::string scheme, authority, path; + auto &balloc = downstream->get_block_allocator(); - rv = http2::construct_push_component(scheme, authority, path, base, uri); + StringRef scheme, authority, path; + + rv = http2::construct_push_component(balloc, scheme, authority, path, base, + uri); if (rv != 0) { return -1; } if (scheme.empty()) { - scheme_ptr = &req.scheme; - } else { - scheme_ptr = &scheme; + scheme = req.scheme; } if (authority.empty()) { - authority_ptr = &req.authority; - } else { - authority_ptr = &authority; + authority = req.authority; } - rv = submit_push_promise(*scheme_ptr, *authority_ptr, path, downstream); + rv = submit_push_promise(scheme, authority, path, downstream); if (rv != 0) { return -1; @@ -1943,7 +1947,7 @@ int Http2Upstream::on_downstream_push_promise_complete( nva.reserve(headers.size()); for (auto &kv : headers) { - nva.push_back(http2::make_nv_nocopy(kv.name, kv.value, kv.no_index)); + nva.push_back(http2::make_nv(kv.name, kv.value, kv.no_index)); } auto promised_stream_id = nghttp2_submit_push_promise( diff --git a/src/shrpx_http2_upstream.h b/src/shrpx_http2_upstream.h index bf1e1c28..1908a504 100644 --- a/src/shrpx_http2_upstream.h +++ b/src/shrpx_http2_upstream.h @@ -111,9 +111,8 @@ public: void check_shutdown(); int prepare_push_promise(Downstream *downstream); - int submit_push_promise(const std::string &scheme, - const std::string &authority, const std::string &path, - Downstream *downstream); + int submit_push_promise(const StringRef &scheme, const StringRef &authority, + const StringRef &path, Downstream *downstream); int on_request_headers(Downstream *downstream, const nghttp2_frame *frame); diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index a6c238a1..d7f5492b 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -283,10 +283,10 @@ int HttpDownstreamConnection::push_request_headers() { httpconf.no_host_rewrite || get_config()->http2_proxy || connect_method; if (no_host_rewrite && !req.authority.empty()) { - authority = StringRef(req.authority); + authority = req.authority; } - downstream_->set_request_downstream_host(authority.str()); + downstream_->set_request_downstream_host(authority); auto buf = downstream_->get_request_buf(); @@ -366,9 +366,9 @@ int HttpDownstreamConnection::push_request_headers() { params &= ~FORWARDED_PROTO; } - auto value = http::create_forwarded( - params, handler->get_forwarded_by(), handler->get_forwarded_for(), - StringRef{req.authority}, StringRef{req.scheme}); + auto value = http::create_forwarded(params, handler->get_forwarded_by(), + handler->get_forwarded_for(), + req.authority, req.scheme); if (fwd || !value.empty()) { buf->append("Forwarded: "); if (fwd) { diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 92994207..73e91a67 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -86,6 +86,8 @@ int htp_uricb(http_parser *htp, const char *data, size_t len) { auto downstream = upstream->get_downstream(); auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + // We happen to have the same value for method token. req.method = htp->method; @@ -103,9 +105,10 @@ int htp_uricb(http_parser *htp, const char *data, size_t len) { req.fs.add_extra_buffer_size(len); if (req.method == HTTP_CONNECT) { - req.authority.append(data, len); + req.authority = + concat_string_ref(balloc, req.authority, StringRef{data, len}); } else { - req.path.append(data, len); + req.path = concat_string_ref(balloc, req.path, StringRef{data, len}); } return 0; @@ -190,30 +193,51 @@ int htp_hdr_valcb(http_parser *htp, const char *data, size_t len) { } // namespace namespace { -void rewrite_request_host_path_from_uri(Request &req, const char *uri, +void rewrite_request_host_path_from_uri(BlockAllocator &balloc, Request &req, + const StringRef &uri, http_parser_url &u) { assert(u.field_set & (1 << UF_HOST)); - auto &authority = req.authority; - authority.clear(); // As per https://tools.ietf.org/html/rfc7230#section-5.4, we // rewrite host header field with authority component. - http2::copy_url_component(authority, &u, UF_HOST, uri); + auto authority = util::get_uri_field(uri.c_str(), u, UF_HOST); // TODO properly check IPv6 numeric address - if (authority.find(':') != std::string::npos) { - authority = '[' + authority; - authority += ']'; + auto ipv6 = std::find(std::begin(authority), std::end(authority), ':') != + std::end(authority); + auto authoritylen = authority.size(); + if (ipv6) { + authoritylen += 2; } if (u.field_set & (1 << UF_PORT)) { - authority += ':'; - authority += util::utos(u.port); + authoritylen += 1 + str_size("65535"); + } + if (authoritylen > authority.size()) { + auto iovec = make_byte_ref(balloc, authoritylen + 1); + auto p = iovec.base; + if (ipv6) { + *p++ = '['; + } + p = std::copy(std::begin(authority), std::end(authority), p); + if (ipv6) { + *p++ = ']'; + } + + if (u.field_set & (1 << UF_PORT)) { + *p++ = ':'; + p = util::utos(p, u.port); + } + *p = '\0'; + + req.authority = StringRef{iovec.base, p}; + } else { + req.authority = authority; } - http2::copy_url_component(req.scheme, &u, UF_SCHEMA, uri); + req.scheme = util::get_uri_field(uri.c_str(), u, UF_SCHEMA); - std::string path; + StringRef path; if (u.field_set & (1 << UF_PATH)) { - http2::copy_url_component(path, &u, UF_PATH, uri); + path = util::get_uri_field(uri.c_str(), u, UF_PATH); } else if (req.method == HTTP_OPTIONS) { // Server-wide OPTIONS takes following form in proxy request: // @@ -221,21 +245,35 @@ void rewrite_request_host_path_from_uri(Request &req, const char *uri, // // Notice that no slash after authority. See // http://tools.ietf.org/html/rfc7230#section-5.3.4 - req.path = ""; + req.path = StringRef::from_lit(""); // we ignore query component here return; } else { - path = "/"; + path = StringRef::from_lit("/"); } + if (u.field_set & (1 << UF_QUERY)) { auto &fdata = u.field_data[UF_QUERY]; - path += '?'; - path.append(uri + fdata.off, fdata.len); + + if (u.field_set & (1 << UF_PATH)) { + auto q = util::get_uri_field(uri.c_str(), u, UF_QUERY); + path = StringRef{std::begin(path), std::end(q)}; + } else { + auto iov = make_byte_ref(balloc, path.size() + 1 + fdata.len + 1); + auto p = iov.base; + + p = std::copy(std::begin(path), std::end(path), p); + *p++ = '?'; + p = std::copy_n(&uri[fdata.off], fdata.len, p); + *p = '\0'; + path = StringRef{iov.base, p}; + } } + if (get_config()->http2_proxy) { - req.path = std::move(path); + req.path = path; } else { - req.path = http2::rewrite_clean_path(std::begin(path), std::end(path)); + req.path = http2::rewrite_clean_path(balloc, path); } } } // namespace @@ -299,12 +337,11 @@ int htp_hdrs_completecb(http_parser *htp) { downstream->inspect_http1_request(); + auto &balloc = downstream->get_block_allocator(); + if (method != HTTP_CONNECT) { http_parser_url u{}; - // make a copy of request path, since we may set request path - // while we are refering to original request path. - auto path = req.path; - rv = http_parser_parse_url(path.c_str(), path.size(), 0, &u); + rv = http_parser_parse_url(req.path.c_str(), req.path.size(), 0, &u); if (rv != 0) { // Expect to respond with 400 bad request return -1; @@ -318,23 +355,23 @@ int htp_hdrs_completecb(http_parser *htp) { req.no_authority = true; - if (method == HTTP_OPTIONS && path == "*") { - req.path = ""; + if (method == HTTP_OPTIONS && req.path == "*") { + req.path = StringRef{}; } else { - req.path = http2::rewrite_clean_path(std::begin(path), std::end(path)); + req.path = http2::rewrite_clean_path(balloc, req.path); } if (host) { - req.authority = host->value.str(); + req.authority = host->value; } if (handler->get_ssl()) { - req.scheme = "https"; + req.scheme = StringRef::from_lit("https"); } else { - req.scheme = "http"; + req.scheme = StringRef::from_lit("http"); } } else { - rewrite_request_host_path_from_uri(req, path.c_str(), u); + rewrite_request_host_path_from_uri(balloc, req, req.path, u); } } diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index 74201ff2..bf83eba2 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -116,6 +116,8 @@ mrb_value request_set_authority(mrb_state *mrb, mrb_value self) { auto downstream = data->downstream; auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + check_phase(mrb, data->phase, PHASE_REQUEST); const char *authority; @@ -125,7 +127,8 @@ mrb_value request_set_authority(mrb_state *mrb, mrb_value self) { mrb_raise(mrb, E_RUNTIME_ERROR, "authority must not be empty string"); } - req.authority.assign(authority, n); + req.authority = + make_string_ref(balloc, StringRef{authority, static_cast(n)}); return self; } @@ -147,6 +150,8 @@ mrb_value request_set_scheme(mrb_state *mrb, mrb_value self) { auto downstream = data->downstream; auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + check_phase(mrb, data->phase, PHASE_REQUEST); const char *scheme; @@ -156,7 +161,8 @@ mrb_value request_set_scheme(mrb_state *mrb, mrb_value self) { mrb_raise(mrb, E_RUNTIME_ERROR, "scheme must not be empty string"); } - req.scheme.assign(scheme, n); + req.scheme = + make_string_ref(balloc, StringRef{scheme, static_cast(n)}); return self; } @@ -178,13 +184,16 @@ mrb_value request_set_path(mrb_state *mrb, mrb_value self) { auto downstream = data->downstream; auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + check_phase(mrb, data->phase, PHASE_REQUEST); const char *path; mrb_int pathlen; mrb_get_args(mrb, "s", &path, &pathlen); - req.path.assign(path, pathlen); + req.path = + make_string_ref(balloc, StringRef{path, static_cast(pathlen)}); return self; } diff --git a/src/shrpx_mruby_module_response.cc b/src/shrpx_mruby_module_response.cc index c50391e1..ada76b7c 100644 --- a/src/shrpx_mruby_module_response.cc +++ b/src/shrpx_mruby_module_response.cc @@ -187,6 +187,8 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { auto &resp = downstream->response(); int rv; + auto &balloc = downstream->get_block_allocator(); + if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { mrb_raise(mrb, E_RUNTIME_ERROR, "response has already been committed"); } @@ -207,14 +209,22 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { bodylen = vallen; } + StringRef content_length; + { + auto iov = make_byte_ref(balloc, str_size("18446744073709551615") + 1); + auto p = iov.base; + p = util::utos(p, bodylen); + *p = '\0'; + content_length = StringRef{iov.base, p}; + } + auto cl = resp.fs.header(http2::HD_CONTENT_LENGTH); if (cl) { - cl->value = make_string_ref(downstream->get_block_allocator(), - StringRef{util::utos(bodylen)}); + cl->value = content_length; } else { + // TODO we don't have to make a copy here. resp.fs.add_header_token(StringRef::from_lit("content-length"), - StringRef{util::utos(bodylen)}, false, - http2::HD_CONTENT_LENGTH); + content_length, false, http2::HD_CONTENT_LENGTH); } resp.fs.content_length = bodylen; diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index e5e1a096..89508ba8 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -159,6 +159,8 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + downstream->reset_upstream_rtimer(); auto nv = frame->syn_stream.nv; @@ -262,17 +264,16 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, req.method = method_token; if (is_connect) { - req.authority = path->value.str(); + req.authority = path->value; } else { - req.scheme = scheme->value.str(); - req.authority = host->value.str(); + req.scheme = scheme->value; + req.authority = host->value; if (get_config()->http2_proxy) { - req.path = path->value.str(); + req.path = path->value; } else if (method_token == HTTP_OPTIONS && path->value == "*") { // Server-wide OPTIONS request. Path is empty. } else { - req.path = http2::rewrite_clean_path(std::begin(path->value), - std::end(path->value)); + req.path = http2::rewrite_clean_path(balloc, path->value); } } diff --git a/src/template.h b/src/template.h index 9c52387f..b15efa02 100644 --- a/src/template.h +++ b/src/template.h @@ -409,7 +409,8 @@ public: : base(&*first), len(std::distance(first, last)) {} template StringRef(InputIt *first, InputIt *last) - : base(first), len(std::distance(first, last)) {} + : base(reinterpret_cast(first)), + len(std::distance(first, last)) {} template constexpr static StringRef from_lit(const CharT(&s)[N]) { return StringRef(s, N - 1); diff --git a/src/util.h b/src/util.h index ba3e1277..f24f8140 100644 --- a/src/util.h +++ b/src/util.h @@ -387,6 +387,23 @@ template std::string utos(T n) { return res; } +template OutputIt utos(OutputIt dst, T n) { + if (n == 0) { + *dst++ = '0'; + return dst; + } + int i = 0; + T t = n; + for (; t; t /= 10, ++i) + ; + --i; + auto p = dst + i; + for (; n; --i, n /= 10) { + *p-- = (n % 10) + '0'; + } + return dst + i + 1; +} + template std::string utos_unit(T n) { char u = 0; if (n >= (1 << 30)) { @@ -695,6 +712,11 @@ std::string random_alpha_digit(Generator &gen, size_t len) { return res; } +template +OutputIterator copy_lit(OutputIterator it, CharT(&s)[N]) { + return std::copy_n(s, N - 1, it); +} + } // namespace util } // namespace nghttp2