From bae37e3e4aec20f730560d9c0fb968824f6db6e7 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 9 Mar 2016 21:15:32 +0900 Subject: [PATCH 01/19] nghttpx: Add custom memory allocator mainly for header related objects --- src/allocator.h | 108 +++++++++++++++++ src/http2.cc | 63 ++++++++-- src/http2.h | 51 ++++++-- src/nghttp.cc | 2 +- src/shrpx_downstream.cc | 142 +++++++++++++---------- src/shrpx_downstream.h | 35 +++--- src/shrpx_http2_downstream_connection.cc | 8 +- src/shrpx_http2_upstream.cc | 4 +- src/shrpx_https_upstream.cc | 2 +- src/shrpx_mruby_module.cc | 2 +- src/shrpx_mruby_module.h | 2 +- src/shrpx_mruby_module_response.cc | 3 +- src/shrpx_spdy_upstream.cc | 10 +- src/template.h | 10 ++ src/util.cc | 4 + src/util.h | 11 ++ 16 files changed, 346 insertions(+), 111 deletions(-) create mode 100644 src/allocator.h diff --git a/src/allocator.h b/src/allocator.h new file mode 100644 index 00000000..2f48f38b --- /dev/null +++ b/src/allocator.h @@ -0,0 +1,108 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef ALLOCATOR_H +#define ALLOCATOR_H + +#include "nghttp2_config.h" + +#include "template.h" + +namespace nghttp2 { + +struct MemBlock { + MemBlock *next; + uint8_t *begin, *last, *end; +}; + +template struct BlockAllocator { + BlockAllocator() : retain(nullptr), head(nullptr) {} + + ~BlockAllocator() { + for (auto mb = retain; mb;) { + auto next = mb->next; + delete[] reinterpret_cast(mb); + mb = next; + } + } + + MemBlock *alloc_mem_block(size_t size) { + auto block = new uint8_t[sizeof(MemBlock) + size]; + auto mb = reinterpret_cast(block); + + mb->next = retain; + mb->begin = mb->last = block + sizeof(MemBlock); + mb->end = mb->begin + size; + retain = mb; + return mb; + } + + void *alloc(size_t size) { + if (size >= BLOCK_SIZE) { + auto mb = alloc_mem_block(size); + mb->last = mb->end; + return mb->begin; + } + + if (!head || head->end - head->last < static_cast(size)) { + head = alloc_mem_block(BLOCK_SIZE); + } + + auto res = head->last; + + head->last = reinterpret_cast( + (reinterpret_cast(head->last + size) + 0xf) & ~0xf); + + return res; + } + + MemBlock *retain; + MemBlock *head; +}; + +template +StringRef make_string_ref(BlockAllocator &alloc, const StringRef &src) { + auto dst = static_cast(alloc.alloc(src.size() + 1)); + auto p = dst; + p = std::copy(std::begin(src), std::end(src), p); + *p = '\0'; + return StringRef{dst, src.size()}; +} + +template +StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, + const StringRef &b) { + auto dst = static_cast(alloc.alloc(a.size() + b.size() + 1)); + auto p = dst; + p = std::copy(std::begin(a), std::end(a), p); + p = std::copy(std::begin(b), std::end(b), p); + *p = '\0'; + return StringRef{dst, a.size() + b.size()}; +} + +using DefaultBlockAllocator = BlockAllocator<1024>; + +} // namespace aria2 + +#endif // ALLOCATOR_H diff --git a/src/http2.cc b/src/http2.cc index bfad624e..17de9b05 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -238,7 +238,7 @@ const char *stringify_status(unsigned int status_code) { } } -void capitalize(DefaultMemchunks *buf, const std::string &s) { +void capitalize(DefaultMemchunks *buf, const StringRef &s) { buf->append(util::upcase(s[0])); for (size_t i = 1; i < s.size(); ++i) { if (s[i - 1] == '-') { @@ -302,14 +302,14 @@ const Headers::value_type *get_header(const Headers &nva, const char *name) { return res; } -std::string value_to_str(const Headers::value_type *nv) { +std::string value_to_str(const HeaderRefs::value_type *nv) { if (nv) { - return nv->value; + return nv->value.str(); } return ""; } -bool non_empty_value(const Headers::value_type *nv) { +bool non_empty_value(const HeaderRefs::value_type *nv) { return nv && !nv->value.empty(); } @@ -326,11 +326,29 @@ nghttp2_nv make_nv_internal(const std::string &name, const std::string &value, } } // namespace +namespace { +nghttp2_nv make_nv_internal(const StringRef &name, const StringRef &value, + bool no_index, uint8_t nv_flags) { + uint8_t flags; + + flags = + nv_flags | (no_index ? NGHTTP2_NV_FLAG_NO_INDEX : NGHTTP2_NV_FLAG_NONE); + + return {(uint8_t *)name.c_str(), (uint8_t *)value.c_str(), name.size(), + value.size(), flags}; +} +} // namespace + nghttp2_nv make_nv(const std::string &name, const std::string &value, bool no_index) { return make_nv_internal(name, value, no_index, NGHTTP2_NV_FLAG_NONE); } +nghttp2_nv make_nv(const StringRef &name, const StringRef &value, + bool no_index) { + return make_nv_internal(name, value, no_index, NGHTTP2_NV_FLAG_NONE); +} + nghttp2_nv make_nv_nocopy(const std::string &name, const std::string &value, bool no_index) { return make_nv_internal(name, value, no_index, @@ -338,9 +356,16 @@ nghttp2_nv make_nv_nocopy(const std::string &name, const std::string &value, NGHTTP2_NV_FLAG_NO_COPY_VALUE); } +nghttp2_nv make_nv_nocopy(const StringRef &name, const StringRef &value, + bool no_index) { + return make_nv_internal(name, value, no_index, + NGHTTP2_NV_FLAG_NO_COPY_NAME | + NGHTTP2_NV_FLAG_NO_COPY_VALUE); +} + namespace { void copy_headers_to_nva_internal(std::vector &nva, - const Headers &headers, uint8_t nv_flags) { + const HeaderRefs &headers, uint8_t nv_flags) { for (auto &kv : headers) { if (kv.name.empty() || kv.name[0] == ':') { continue; @@ -367,18 +392,19 @@ void copy_headers_to_nva_internal(std::vector &nva, } } // namespace -void copy_headers_to_nva(std::vector &nva, const Headers &headers) { +void copy_headers_to_nva(std::vector &nva, + const HeaderRefs &headers) { copy_headers_to_nva_internal(nva, headers, NGHTTP2_NV_FLAG_NONE); } void copy_headers_to_nva_nocopy(std::vector &nva, - const Headers &headers) { + const HeaderRefs &headers) { copy_headers_to_nva_internal(nva, headers, NGHTTP2_NV_FLAG_NO_COPY_NAME | NGHTTP2_NV_FLAG_NO_COPY_VALUE); } void build_http1_headers_from_headers(DefaultMemchunks *buf, - const Headers &headers) { + const HeaderRefs &headers) { for (auto &kv : headers) { if (kv.name.empty() || kv.name[0] == ':') { continue; @@ -450,8 +476,15 @@ void dump_nv(FILE *out, const Headers &nva) { fflush(out); } -std::string rewrite_location_uri(const std::string &uri, - const http_parser_url &u, +void dump_nv(FILE *out, const HeaderRefs &nva) { + for (auto &nv : nva) { + fprintf(out, "%s: %s\n", nv.name.c_str(), nv.value.c_str()); + } + fputc('\n', out); + fflush(out); +} + +std::string rewrite_location_uri(const StringRef &uri, const http_parser_url &u, const std::string &match_host, const std::string &request_authority, const std::string &upstream_scheme) { @@ -499,7 +532,7 @@ int check_nv(const uint8_t *name, size_t namelen, const uint8_t *value, return 1; } -int parse_http_status_code(const std::string &src) { +int parse_http_status_code(const StringRef &src) { if (src.size() != 3) { return -1; } @@ -525,6 +558,10 @@ int lookup_token(const std::string &name) { name.size()); } +int lookup_token(const StringRef &name) { + return lookup_token(name.byte(), name.size()); +} + // This function was generated by genheaderfunc.py. Inspired by h2o // header lookup. https://github.com/h2o/h2o int lookup_token(const uint8_t *name, size_t namelen) { @@ -1278,6 +1315,10 @@ int lookup_method_token(const std::string &name) { name.size()); } +int lookup_method_token(const StringRef &name) { + return lookup_method_token(name.byte(), name.size()); +} + // This function was generated by genmethodfunc.py. int lookup_method_token(const uint8_t *name, size_t namelen) { switch (namelen) { diff --git a/src/http2.h b/src/http2.h index 7a05dd45..b4e84e8b 100644 --- a/src/http2.h +++ b/src/http2.h @@ -39,6 +39,7 @@ #include "util.h" #include "memchunk.h" +#include "template.h" namespace nghttp2 { @@ -66,7 +67,29 @@ struct Header { bool no_index; }; +struct HeaderRef { + HeaderRef(const StringRef &name, const StringRef &value, + bool no_index = false, int32_t token = -1) + : name(name), value(value), token(token), no_index(no_index) {} + + HeaderRef() : token(-1), no_index(false) {} + + bool operator==(const HeaderRef &other) const { + return name == other.name && value == other.value; + } + + bool operator<(const HeaderRef &rhs) const { + return name < rhs.name || (name == rhs.name && value < rhs.value); + } + + StringRef name; + StringRef value; + int32_t token; + bool no_index; +}; + using Headers = std::vector
; +using HeaderRefs = std::vector; namespace http2 { @@ -76,7 +99,7 @@ std::string get_status_string(unsigned int status_code); // only predefined status code. Otherwise, returns nullptr. const char *stringify_status(unsigned int status_code); -void capitalize(DefaultMemchunks *buf, const std::string &s); +void capitalize(DefaultMemchunks *buf, const StringRef &s); // Returns true if |value| is LWS bool lws(const char *value); @@ -104,10 +127,10 @@ void add_header(Headers &nva, const uint8_t *name, size_t namelen, const Headers::value_type *get_header(const Headers &nva, const char *name); // Returns nv->second if nv is not nullptr. Otherwise, returns "". -std::string value_to_str(const Headers::value_type *nv); +std::string value_to_str(const HeaderRefs::value_type *nv); // Returns true if the value of |nv| is not empty. -bool non_empty_value(const Headers::value_type *nv); +bool non_empty_value(const HeaderRefs::value_type *nv); // Creates nghttp2_nv using |name| and |value| and returns it. The // returned value only references the data pointer to name.c_str() and @@ -116,9 +139,15 @@ bool non_empty_value(const Headers::value_type *nv); nghttp2_nv make_nv(const std::string &name, const std::string &value, bool no_index = false); +nghttp2_nv make_nv(const StringRef &name, const StringRef &value, + bool no_index = false); + nghttp2_nv make_nv_nocopy(const std::string &name, const std::string &value, bool no_index = false); +nghttp2_nv make_nv_nocopy(const StringRef &name, const StringRef &value, + bool no_index = false); + // Create nghttp2_nv from string literal |name| and |value|. template constexpr nghttp2_nv make_nv_ll(const char(&name)[N], const char(&value)[M]) { @@ -163,19 +192,20 @@ nghttp2_nv make_nv_ls_nocopy(const char(&name)[N], const StringRef &value) { // before this call (its element's token field is assigned). Certain // headers, including disallowed headers in HTTP/2 spec and headers // which require special handling (i.e. via), are not copied. -void copy_headers_to_nva(std::vector &nva, const Headers &headers); +void copy_headers_to_nva(std::vector &nva, + const HeaderRefs &headers); // Just like copy_headers_to_nva(), but this adds // NGHTTP2_NV_FLAG_NO_COPY_NAME and NGHTTP2_NV_FLAG_NO_COPY_VALUE. void copy_headers_to_nva_nocopy(std::vector &nva, - const Headers &headers); + const HeaderRefs &headers); // Appends HTTP/1.1 style header lines to |buf| from headers in // |headers|. |headers| must be indexed before this call (its // element's token field is assigned). Certain headers, which // requires special handling (i.e. via and cookie), are not appended. void build_http1_headers_from_headers(DefaultMemchunks *buf, - const Headers &headers); + const HeaderRefs &headers); // Return positive window_size_increment if WINDOW_UPDATE should be // sent for the stream |stream_id|. If |stream_id| == 0, this function @@ -196,6 +226,8 @@ void dump_nv(FILE *out, const nghttp2_nv *nva, size_t nvlen); // Dumps name/value pairs in |nva| to |out|. void dump_nv(FILE *out, const Headers &nva); +void dump_nv(FILE *out, const HeaderRefs &nva); + // Rewrites redirection URI which usually appears in location header // field. The |uri| is the URI in the location header field. The |u| // stores the result of parsed |uri|. The |request_authority| is the @@ -209,8 +241,7 @@ void dump_nv(FILE *out, const Headers &nva); // This function returns the new rewritten URI on success. If the // location URI is not subject to the rewrite, this function returns // emtpy string. -std::string rewrite_location_uri(const std::string &uri, - const http_parser_url &u, +std::string rewrite_location_uri(const StringRef &uri, const http_parser_url &u, const std::string &match_host, const std::string &request_authority, const std::string &upstream_scheme); @@ -222,7 +253,7 @@ int check_nv(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen); // Returns parsed HTTP status code. Returns -1 on failure. -int parse_http_status_code(const std::string &src); +int parse_http_status_code(const StringRef &src); // Header fields to be indexed, except HD_MAXIDX which is convenient // member to get maximum value. @@ -272,6 +303,7 @@ using HeaderIndex = std::array; // cannot be tokenized, returns -1. int lookup_token(const uint8_t *name, size_t namelen); int lookup_token(const std::string &name); +int lookup_token(const StringRef &name); // Initializes |hdidx|, header index. The |hdidx| must point to the // array containing at least HD_MAXIDX elements. @@ -318,6 +350,7 @@ bool expect_response_body(int status_code); // tokenized. If method name cannot be tokenized, returns -1. int lookup_method_token(const uint8_t *name, size_t namelen); int lookup_method_token(const std::string &name); +int lookup_method_token(const StringRef &name); // Returns string representation of |method_token|. This is wrapper // function over http_method_str from http-parser. If |method_token| diff --git a/src/nghttp.cc b/src/nghttp.cc index af349519..d0fc8ece 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -1600,7 +1600,7 @@ void check_response_header(nghttp2_session *session, Request *req) { return; } - auto status = http2::parse_http_status_code(status_hd->value); + auto status = http2::parse_http_status_code(StringRef{status_hd->value}); if (status == -1) { nghttp2_submit_rst_stream(session, NGHTTP2_FLAG_NONE, req->stream_id, NGHTTP2_PROTOCOL_ERROR); diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 0a8da837..11462d7c 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -116,6 +116,8 @@ Downstream::Downstream(Upstream *upstream, MemchunkPool *mcpool, : dlnext(nullptr), dlprev(nullptr), response_sent_body_length(0), + req_(balloc_), + resp_(balloc_), request_start_time_(std::chrono::high_resolution_clock::now()), request_buf_(mcpool), response_buf_(mcpool), @@ -237,8 +239,9 @@ void Downstream::force_resume_read() { } namespace { -const Headers::value_type * -search_header_linear_backwards(const Headers &headers, const StringRef &name) { +const HeaderRefs::value_type * +search_header_linear_backwards(const HeaderRefs &headers, + const StringRef &name) { for (auto it = headers.rbegin(); it != headers.rend(); ++it) { auto &kv = *it; if (kv.name == name) { @@ -258,11 +261,25 @@ std::string Downstream::assemble_request_cookie() const { continue; } - auto end = kv.value.find_last_not_of(" ;"); - if (end == std::string::npos) { + if (kv.value.empty()) { + continue; + } + + auto end = std::end(kv.value); + for (auto it = std::begin(kv.value) + kv.value.size(); + it != std::begin(kv.value); --it) { + auto c = *(it - 1); + if (c == ' ' || c == ';') { + continue; + } + end = it + 1; + break; + } + + if (end == std::end(kv.value)) { cookie += kv.value; } else { - cookie.append(std::begin(kv.value), std::begin(kv.value) + end + 1); + cookie.append(std::begin(kv.value), end); } cookie += "; "; } @@ -280,18 +297,14 @@ size_t Downstream::count_crumble_request_cookie() { !util::streq_l("cooki", kv.name.c_str(), 5)) { continue; } - size_t last = kv.value.size(); - for (size_t j = 0; j < last;) { - j = kv.value.find_first_not_of("\t ;", j); - if (j == std::string::npos) { - break; + for (auto it = std::begin(kv.value); it != std::end(kv.value);) { + if (*it == '\t' || *it == ' ' || *it == ';') { + ++it; + continue; } - j = kv.value.find(';', j); - if (j == std::string::npos) { - j = last; - } + it = std::find(it, std::end(kv.value), ';'); ++n; } @@ -305,22 +318,19 @@ void Downstream::crumble_request_cookie(std::vector &nva) { !util::streq_l("cooki", kv.name.c_str(), 5)) { continue; } - size_t last = kv.value.size(); - for (size_t j = 0; j < last;) { - j = kv.value.find_first_not_of("\t ;", j); - if (j == std::string::npos) { - break; - } - auto first = j; - - j = kv.value.find(';', j); - if (j == std::string::npos) { - j = last; + for (auto it = std::begin(kv.value); it != std::end(kv.value);) { + if (*it == '\t' || *it == ' ' || *it == ';') { + ++it; + continue; } - nva.push_back({(uint8_t *)"cookie", (uint8_t *)kv.value.c_str() + first, - str_size("cookie"), j - first, + auto first = it; + + it = std::find(it, std::end(kv.value), ';'); + + nva.push_back({(uint8_t *)"cookie", (uint8_t *)first, str_size("cookie"), + (size_t)(it - first), (uint8_t)(NGHTTP2_NV_FLAG_NO_COPY_NAME | NGHTTP2_NV_FLAG_NO_COPY_VALUE | (kv.no_index ? NGHTTP2_NV_FLAG_NO_INDEX : 0))}); @@ -329,42 +339,45 @@ void Downstream::crumble_request_cookie(std::vector &nva) { } namespace { -void add_header(bool &key_prev, size_t &sum, Headers &headers, +void add_header(bool &key_prev, size_t &sum, HeaderRefs &headers, const StringRef &name, const StringRef &value, bool no_index, int32_t token) { key_prev = true; sum += name.size() + value.size(); - headers.emplace_back(name.str(), value.str(), no_index, token); + headers.emplace_back(name, value, no_index, token); } } // namespace namespace { -void add_header(size_t &sum, Headers &headers, const StringRef &name, +void add_header(size_t &sum, HeaderRefs &headers, const StringRef &name, const StringRef &value, bool no_index, int32_t token) { sum += name.size() + value.size(); - headers.emplace_back(name.str(), value.str(), no_index, token); + headers.emplace_back(name, value, no_index, token); } } // namespace namespace { -void append_last_header_key(bool &key_prev, size_t &sum, Headers &headers, - const char *data, size_t len) { +void append_last_header_key(DefaultBlockAllocator &balloc, bool &key_prev, + size_t &sum, HeaderRefs &headers, const char *data, + size_t len) { assert(key_prev); sum += len; auto &item = headers.back(); - item.name.append(data, len); - util::inp_strlower(item.name); + item.name = concat_string_ref(balloc, item.name, StringRef{data, len}); + util::inp_strlower((uint8_t *)item.name.c_str(), + (uint8_t *)item.name.c_str() + item.name.size()); item.token = http2::lookup_token(item.name); } } // namespace namespace { -void append_last_header_value(bool &key_prev, size_t &sum, Headers &headers, +void append_last_header_value(DefaultBlockAllocator &balloc, bool &key_prev, + size_t &sum, HeaderRefs &headers, const char *data, size_t len) { key_prev = false; sum += len; auto &item = headers.back(); - item.value.append(data, len); + item.value = concat_string_ref(balloc, item.value, StringRef{data, len}); } } // namespace @@ -388,7 +401,7 @@ int FieldStore::parse_content_length() { return 0; } -const Headers::value_type *FieldStore::header(int32_t token) const { +const HeaderRefs::value_type *FieldStore::header(int32_t token) const { for (auto it = headers_.rbegin(); it != headers_.rend(); ++it) { auto &kv = *it; if (kv.token == token) { @@ -398,7 +411,7 @@ const Headers::value_type *FieldStore::header(int32_t token) const { return nullptr; } -Headers::value_type *FieldStore::header(int32_t token) { +HeaderRefs::value_type *FieldStore::header(int32_t token) { for (auto it = headers_.rbegin(); it != headers_.rend(); ++it) { auto &kv = *it; if (kv.token == token) { @@ -408,43 +421,47 @@ Headers::value_type *FieldStore::header(int32_t token) { return nullptr; } -const Headers::value_type *FieldStore::header(const StringRef &name) const { +const HeaderRefs::value_type *FieldStore::header(const StringRef &name) const { return search_header_linear_backwards(headers_, name); } void FieldStore::add_header_lower(const StringRef &name, const StringRef &value, bool no_index) { - auto low_name = name.str(); - util::inp_strlower(low_name); + auto low_name = make_string_ref(balloc_, name); + util::inp_strlower((uint8_t *)low_name.c_str(), + (uint8_t *)low_name.c_str() + low_name.size()); auto token = http2::lookup_token(low_name); - shrpx::add_header(header_key_prev_, buffer_size_, headers_, - StringRef{low_name}, value, no_index, token); + shrpx::add_header(header_key_prev_, buffer_size_, headers_, low_name, + make_string_ref(balloc_, value), no_index, token); } void FieldStore::add_header_token(const StringRef &name, const StringRef &value, bool no_index, int32_t token) { - shrpx::add_header(buffer_size_, headers_, name, value, no_index, token); + shrpx::add_header(buffer_size_, headers_, make_string_ref(balloc_, name), + make_string_ref(balloc_, value), no_index, token); + make_string_ref(balloc_, name); } void FieldStore::append_last_header_key(const char *data, size_t len) { - shrpx::append_last_header_key(header_key_prev_, buffer_size_, headers_, data, - len); + shrpx::append_last_header_key(balloc_, header_key_prev_, buffer_size_, + headers_, data, len); } void FieldStore::append_last_header_value(const char *data, size_t len) { - shrpx::append_last_header_value(header_key_prev_, buffer_size_, headers_, - data, len); + shrpx::append_last_header_value(balloc_, header_key_prev_, buffer_size_, + headers_, data, len); } void FieldStore::clear_headers() { headers_.clear(); } void FieldStore::add_trailer_lower(const StringRef &name, const StringRef &value, bool no_index) { - auto low_name = name.str(); - util::inp_strlower(low_name); + auto low_name = make_string_ref(balloc_, name); + util::inp_strlower((uint8_t *)low_name.c_str(), + (uint8_t *)low_name.c_str() + low_name.size()); auto token = http2::lookup_token(low_name); - shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, - StringRef{low_name}, value, no_index, token); + shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, low_name, + make_string_ref(balloc_, value), no_index, token); } void FieldStore::add_trailer_token(const StringRef &name, @@ -452,17 +469,18 @@ void FieldStore::add_trailer_token(const StringRef &name, int32_t token) { // Header size limit should be applied to all header and trailer // fields combined. - shrpx::add_header(buffer_size_, trailers_, name, value, no_index, token); + shrpx::add_header(buffer_size_, trailers_, make_string_ref(balloc_, name), + make_string_ref(balloc_, value), no_index, token); } void FieldStore::append_last_trailer_key(const char *data, size_t len) { - shrpx::append_last_header_key(trailer_key_prev_, buffer_size_, trailers_, - data, len); + shrpx::append_last_header_key(balloc_, trailer_key_prev_, buffer_size_, + trailers_, data, len); } void FieldStore::append_last_trailer_value(const char *data, size_t len) { - shrpx::append_last_header_value(trailer_key_prev_, buffer_size_, trailers_, - data, len); + shrpx::append_last_header_value(balloc_, trailer_key_prev_, buffer_size_, + trailers_, data, len); } void Downstream::set_request_start_time( @@ -566,7 +584,7 @@ void Downstream::rewrite_location_response_header( return; } - hd->value = std::move(new_uri); + hd->value = make_string_ref(balloc_, StringRef{new_uri}); } bool Downstream::get_chunked_response() const { return chunked_response_; } @@ -703,10 +721,10 @@ bool Downstream::get_http2_upgrade_request() const { response_state_ == INITIAL; } -const std::string &Downstream::get_http2_settings() const { +StringRef Downstream::get_http2_settings() const { auto http2_settings = req_.fs.header(http2::HD_HTTP2_SETTINGS); if (!http2_settings) { - return EMPTY_STRING; + return StringRef{}; } return http2_settings->value; } @@ -908,4 +926,6 @@ void Downstream::set_assoc_stream_id(int32_t stream_id) { int32_t Downstream::get_assoc_stream_id() const { return assoc_stream_id_; } +DefaultBlockAllocator &Downstream::get_block_allocator() { return balloc_; } + } // namespace shrpx diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 3864674f..7b9dfa83 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -40,6 +40,7 @@ #include "shrpx_io_control.h" #include "http2.h" #include "memchunk.h" +#include "allocator.h" using namespace nghttp2; @@ -51,18 +52,19 @@ struct BlockedLink; class FieldStore { public: - FieldStore(size_t headers_initial_capacity) + FieldStore(DefaultBlockAllocator &balloc, size_t headers_initial_capacity) : content_length(-1), + balloc_(balloc), buffer_size_(0), header_key_prev_(false), trailer_key_prev_(false) { headers_.reserve(headers_initial_capacity); } - const Headers &headers() const { return headers_; } - const Headers &trailers() const { return trailers_; } + const HeaderRefs &headers() const { return headers_; } + const HeaderRefs &trailers() const { return trailers_; } - Headers &headers() { return headers_; } + HeaderRefs &headers() { return headers_; } const void add_extra_buffer_size(size_t n) { buffer_size_ += n; } size_t buffer_size() const { return buffer_size_; } @@ -73,11 +75,11 @@ public: // multiple header have |name| as name, return last occurrence from // the beginning. If no such header is found, returns nullptr. // This function must be called after headers are indexed - const Headers::value_type *header(int32_t token) const; - Headers::value_type *header(int32_t token); + const HeaderRefs::value_type *header(int32_t token) const; + HeaderRefs::value_type *header(int32_t token); // Returns pointer to the header field with the name |name|. If no // such header is found, returns nullptr. - const Headers::value_type *header(const StringRef &name) const; + const HeaderRefs::value_type *header(const StringRef &name) const; void add_header_lower(const StringRef &name, const StringRef &value, bool no_index); @@ -110,10 +112,11 @@ public: int64_t content_length; private: - Headers headers_; + DefaultBlockAllocator &balloc_; + HeaderRefs headers_; // trailer fields. For HTTP/1.1, trailer fields are only included // with chunked encoding. For HTTP/2, there is no such limit. - Headers trailers_; + HeaderRefs trailers_; // Sum of the length of name and value in headers_ and trailers_. // This could also be increased by add_extra_buffer_size() to take // into account for request URI in case of HTTP/1.x request. @@ -123,8 +126,8 @@ private: }; struct Request { - Request() - : fs(16), + Request(DefaultBlockAllocator &balloc) + : fs(balloc, 16), recv_body_length(0), unconsumed_body_length(0), method(-1), @@ -179,8 +182,8 @@ struct Request { }; struct Response { - Response() - : fs(32), + Response(DefaultBlockAllocator &balloc) + : fs(balloc, 32), recv_body_length(0), unconsumed_body_length(0), http_status(0), @@ -245,7 +248,7 @@ public: // Returns true if the request is HTTP Upgrade for HTTP/2 bool get_http2_upgrade_request() const; // Returns the value of HTTP2-Settings request header field. - const std::string &get_http2_settings() const; + StringRef get_http2_settings() const; // downstream request API const Request &request() const { return req_; } @@ -373,6 +376,8 @@ public: DefaultMemchunks pop_response_buf(); + DefaultBlockAllocator &get_block_allocator(); + enum { EVENT_ERROR = 0x1, EVENT_TIMEOUT = 0x2, @@ -392,6 +397,8 @@ public: int64_t response_sent_body_length; private: + DefaultBlockAllocator balloc_; + Request req_; Response resp_; diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index 028b6e41..357bd664 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -364,7 +364,7 @@ int Http2DownstreamConnection::push_request_headers() { StringRef{req.authority}, StringRef{req.scheme}); if (fwd || !value.empty()) { if (fwd) { - forwarded_value = fwd->value; + forwarded_value = fwd->value.str(); if (!value.empty()) { forwarded_value += ", "; @@ -377,7 +377,7 @@ int Http2DownstreamConnection::push_request_headers() { } } else if (fwd) { nva.push_back(http2::make_nv_ls_nocopy("forwarded", fwd->value)); - forwarded_value = fwd->value; + forwarded_value = fwd->value.str(); } auto &xffconf = httpconf.xff; @@ -389,7 +389,7 @@ int Http2DownstreamConnection::push_request_headers() { if (xffconf.add) { if (xff) { - xff_value = (*xff).value; + xff_value = (*xff).value.str(); xff_value += ", "; } xff_value += upstream->get_client_handler()->get_ipaddr(); @@ -411,7 +411,7 @@ int Http2DownstreamConnection::push_request_headers() { } } else { if (via) { - via_value = (*via).value; + via_value = (*via).value.str(); via_value += ", "; } via_value += http::create_via_header_value(req.http_major, req.http_minor); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 9d42fe3a..cf89bf3e 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -108,7 +108,7 @@ int on_stream_close_callback(nghttp2_session *session, int32_t stream_id, int Http2Upstream::upgrade_upstream(HttpsUpstream *http) { int rv; - auto http2_settings = http->get_downstream()->get_http2_settings(); + auto http2_settings = http->get_downstream()->get_http2_settings().str(); util::to_base64(http2_settings); auto settings_payload = @@ -1434,7 +1434,7 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { } } else { if (via) { - via_value = (*via).value; + via_value = (*via).value.str(); via_value += ", "; } via_value += diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 49c9848e..92994207 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -325,7 +325,7 @@ int htp_hdrs_completecb(http_parser *htp) { } if (host) { - req.authority = host->value; + req.authority = host->value.str(); } if (handler->get_ssl()) { diff --git a/src/shrpx_mruby_module.cc b/src/shrpx_mruby_module.cc index e2f17fb1..27b7769c 100644 --- a/src/shrpx_mruby_module.cc +++ b/src/shrpx_mruby_module.cc @@ -85,7 +85,7 @@ mrb_value init_module(mrb_state *mrb) { return create_env(mrb); } -mrb_value create_headers_hash(mrb_state *mrb, const Headers &headers) { +mrb_value create_headers_hash(mrb_state *mrb, const HeaderRefs &headers) { auto hash = mrb_hash_new(mrb); for (auto &hd : headers) { diff --git a/src/shrpx_mruby_module.h b/src/shrpx_mruby_module.h index 3e417f88..a426bead 100644 --- a/src/shrpx_mruby_module.h +++ b/src/shrpx_mruby_module.h @@ -43,7 +43,7 @@ mrb_value init_module(mrb_state *mrb); void delete_downstream_from_module(mrb_state *mrb, Downstream *downstream); -mrb_value create_headers_hash(mrb_state *mrb, const Headers &headers); +mrb_value create_headers_hash(mrb_state *mrb, const HeaderRefs &headers); } // namespace mruby diff --git a/src/shrpx_mruby_module_response.cc b/src/shrpx_mruby_module_response.cc index 73dacccc..c50391e1 100644 --- a/src/shrpx_mruby_module_response.cc +++ b/src/shrpx_mruby_module_response.cc @@ -209,7 +209,8 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { auto cl = resp.fs.header(http2::HD_CONTENT_LENGTH); if (cl) { - cl->value = util::utos(bodylen); + cl->value = make_string_ref(downstream->get_block_allocator(), + StringRef{util::utos(bodylen)}); } else { resp.fs.add_header_token(StringRef::from_lit("content-length"), StringRef{util::utos(bodylen)}, false, diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index e15444a4..e5e1a096 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -262,12 +262,12 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, req.method = method_token; if (is_connect) { - req.authority = path->value; + req.authority = path->value.str(); } else { - req.scheme = scheme->value; - req.authority = host->value; + req.scheme = scheme->value.str(); + req.authority = host->value.str(); if (get_config()->http2_proxy) { - req.path = path->value; + req.path = path->value.str(); } else if (method_token == HTTP_OPTIONS && path->value == "*") { // Server-wide OPTIONS request. Path is empty. } else { @@ -1069,7 +1069,7 @@ int SpdyUpstream::on_downstream_header_complete(Downstream *downstream) { } } else { if (via) { - via_value = via->value; + via_value = via->value.str(); via_value += ", "; } via_value += diff --git a/src/template.h b/src/template.h index 6adcf0d5..9c52387f 100644 --- a/src/template.h +++ b/src/template.h @@ -443,6 +443,11 @@ private: size_type len; }; +inline bool operator==(const StringRef &lhs, const StringRef &rhs) { + return lhs.size() == rhs.size() && + std::equal(std::begin(lhs), std::end(lhs), std::begin(rhs)); +} + inline bool operator==(const StringRef &lhs, const std::string &rhs) { return lhs.size() == rhs.size() && std::equal(std::begin(lhs), std::end(lhs), std::begin(rhs)); @@ -477,6 +482,11 @@ inline bool operator!=(const char *lhs, const StringRef &rhs) { return !(rhs == lhs); } +inline bool operator<(const StringRef &lhs, const StringRef &rhs) { + return std::lexicographical_compare(std::begin(lhs), std::end(lhs), + std::begin(rhs), std::end(rhs)); +} + inline std::ostream &operator<<(std::ostream &o, const StringRef &s) { return o.write(s.c_str(), s.size()); } diff --git a/src/util.cc b/src/util.cc index 784806be..90304315 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1056,6 +1056,10 @@ int64_t parse_uint(const std::string &s) { return parse_uint(reinterpret_cast(s.c_str()), s.size()); } +int64_t parse_uint(const StringRef &s) { + return parse_uint(s.byte(), s.size()); +} + int64_t parse_uint(const uint8_t *s, size_t len) { int64_t n; size_t i; diff --git a/src/util.h b/src/util.h index d03ffd8e..ba3e1277 100644 --- a/src/util.h +++ b/src/util.h @@ -261,6 +261,11 @@ bool iends_with_l(const std::string &a, const CharT(&b)[N]) { return iends_with(std::begin(a), std::end(a), b, b + N - 1); } +template +bool iends_with_l(const StringRef &a, const CharT(&b)[N]) { + return iends_with(std::begin(a), std::end(a), b, b + N - 1); +} + int strcompare(const char *a, const uint8_t *b, size_t n); template bool strieq(const char *a, InputIt b, size_t bn) { @@ -310,6 +315,11 @@ bool strieq_l(const CharT(&a)[N], const std::string &b) { return strieq(a, N - 1, std::begin(b), b.size()); } +template +bool strieq_l(const CharT(&a)[N], const StringRef &b) { + return strieq(a, N - 1, std::begin(b), b.size()); +} + template bool streq(const char *a, InputIt b, size_t bn) { if (!a) { return false; @@ -609,6 +619,7 @@ int64_t parse_uint_with_unit(const char *s); int64_t parse_uint(const char *s); int64_t parse_uint(const uint8_t *s, size_t len); int64_t parse_uint(const std::string &s); +int64_t parse_uint(const StringRef &s); // Parses NULL terminated string |s| as unsigned integer and returns // the parsed integer casted to double. If |s| ends with "s", the From fa601e5ba3fb6af598e77923dbc21f340ffdfda9 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 9 Mar 2016 21:25:11 +0900 Subject: [PATCH 02/19] Add isolation_threshold, use field to store block size rather than template parameter --- src/allocator.h | 19 +++++++++++++------ src/shrpx_downstream.cc | 10 +++++----- src/shrpx_downstream.h | 12 ++++++------ 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/allocator.h b/src/allocator.h index 2f48f38b..998653be 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -36,8 +36,12 @@ struct MemBlock { uint8_t *begin, *last, *end; }; -template struct BlockAllocator { - BlockAllocator() : retain(nullptr), head(nullptr) {} +struct BlockAllocator { + BlockAllocator(size_t block_size, size_t isolation_threshold) + : retain(nullptr), + head(nullptr), + block_size(block_size), + isolation_threshold(std::min(block_size, isolation_threshold)) {} ~BlockAllocator() { for (auto mb = retain; mb;) { @@ -59,14 +63,14 @@ template struct BlockAllocator { } void *alloc(size_t size) { - if (size >= BLOCK_SIZE) { + if (size >= isolation_threshold) { auto mb = alloc_mem_block(size); mb->last = mb->end; return mb->begin; } if (!head || head->end - head->last < static_cast(size)) { - head = alloc_mem_block(BLOCK_SIZE); + head = alloc_mem_block(block_size); } auto res = head->last; @@ -79,6 +83,11 @@ template struct BlockAllocator { MemBlock *retain; MemBlock *head; + // size of single memory block + size_t block_size; + // if allocation greater or equal to isolation_threshold bytes is + // requested, allocate dedicated block. + size_t isolation_threshold; }; template @@ -101,8 +110,6 @@ StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, return StringRef{dst, a.size() + b.size()}; } -using DefaultBlockAllocator = BlockAllocator<1024>; - } // namespace aria2 #endif // ALLOCATOR_H diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 11462d7c..ed1edf2e 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -116,6 +116,7 @@ Downstream::Downstream(Upstream *upstream, MemchunkPool *mcpool, : dlnext(nullptr), dlprev(nullptr), response_sent_body_length(0), + balloc_(1024, 1024), req_(balloc_), resp_(balloc_), request_start_time_(std::chrono::high_resolution_clock::now()), @@ -357,9 +358,8 @@ void add_header(size_t &sum, HeaderRefs &headers, const StringRef &name, } // namespace namespace { -void append_last_header_key(DefaultBlockAllocator &balloc, bool &key_prev, - size_t &sum, HeaderRefs &headers, const char *data, - size_t len) { +void append_last_header_key(BlockAllocator &balloc, bool &key_prev, size_t &sum, + HeaderRefs &headers, const char *data, size_t len) { assert(key_prev); sum += len; auto &item = headers.back(); @@ -371,7 +371,7 @@ void append_last_header_key(DefaultBlockAllocator &balloc, bool &key_prev, } // namespace namespace { -void append_last_header_value(DefaultBlockAllocator &balloc, bool &key_prev, +void append_last_header_value(BlockAllocator &balloc, bool &key_prev, size_t &sum, HeaderRefs &headers, const char *data, size_t len) { key_prev = false; @@ -926,6 +926,6 @@ void Downstream::set_assoc_stream_id(int32_t stream_id) { int32_t Downstream::get_assoc_stream_id() const { return assoc_stream_id_; } -DefaultBlockAllocator &Downstream::get_block_allocator() { return balloc_; } +BlockAllocator &Downstream::get_block_allocator() { return balloc_; } } // namespace shrpx diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 7b9dfa83..2ea0bf2e 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -52,7 +52,7 @@ struct BlockedLink; class FieldStore { public: - FieldStore(DefaultBlockAllocator &balloc, size_t headers_initial_capacity) + FieldStore(BlockAllocator &balloc, size_t headers_initial_capacity) : content_length(-1), balloc_(balloc), buffer_size_(0), @@ -112,7 +112,7 @@ public: int64_t content_length; private: - DefaultBlockAllocator &balloc_; + BlockAllocator &balloc_; HeaderRefs headers_; // trailer fields. For HTTP/1.1, trailer fields are only included // with chunked encoding. For HTTP/2, there is no such limit. @@ -126,7 +126,7 @@ private: }; struct Request { - Request(DefaultBlockAllocator &balloc) + Request(BlockAllocator &balloc) : fs(balloc, 16), recv_body_length(0), unconsumed_body_length(0), @@ -182,7 +182,7 @@ struct Request { }; struct Response { - Response(DefaultBlockAllocator &balloc) + Response(BlockAllocator &balloc) : fs(balloc, 32), recv_body_length(0), unconsumed_body_length(0), @@ -376,7 +376,7 @@ public: DefaultMemchunks pop_response_buf(); - DefaultBlockAllocator &get_block_allocator(); + BlockAllocator &get_block_allocator(); enum { EVENT_ERROR = 0x1, @@ -397,7 +397,7 @@ public: int64_t response_sent_body_length; private: - DefaultBlockAllocator balloc_; + BlockAllocator balloc_; Request req_; Response resp_; From b1b57cc740e25d989f562d3dba5bab8cef3a7196 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 10 Mar 2016 22:42:07 +0900 Subject: [PATCH 03/19] nghttpx: Use StringRef for authority, scheme and path --- src/allocator.h | 13 ++ src/http2.cc | 272 ++++++++++++++++++++--- src/http2.h | 27 ++- src/shrpx_client_handler.cc | 37 ++- src/shrpx_client_handler.h | 2 +- src/shrpx_downstream.cc | 13 +- src/shrpx_downstream.h | 12 +- src/shrpx_downstream_queue.cc | 13 +- src/shrpx_downstream_queue.h | 6 +- src/shrpx_http2_downstream_connection.cc | 10 +- src/shrpx_http2_session.cc | 13 +- src/shrpx_http2_upstream.cc | 76 ++++--- src/shrpx_http2_upstream.h | 5 +- src/shrpx_http_downstream_connection.cc | 10 +- src/shrpx_https_upstream.cc | 99 ++++++--- src/shrpx_mruby_module_request.cc | 15 +- src/shrpx_mruby_module_response.cc | 18 +- src/shrpx_spdy_upstream.cc | 13 +- src/template.h | 3 +- src/util.h | 22 ++ 20 files changed, 511 insertions(+), 168 deletions(-) diff --git a/src/allocator.h b/src/allocator.h index 998653be..00acdbcf 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -27,6 +27,8 @@ #include "nghttp2_config.h" +#include + #include "template.h" namespace nghttp2 { @@ -110,6 +112,17 @@ StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, return StringRef{dst, a.size() + b.size()}; } +struct ByteRef { + uint8_t *base; + size_t len; +}; + +template +ByteRef make_byte_ref(BlockAllocator &alloc, size_t size) { + auto dst = static_cast(alloc.alloc(size)); + return {dst, size}; +} + } // namespace aria2 #endif // ALLOCATOR_H diff --git a/src/http2.cc b/src/http2.cc index 17de9b05..28314134 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -484,41 +484,69 @@ void dump_nv(FILE *out, const HeaderRefs &nva) { fflush(out); } -std::string rewrite_location_uri(const StringRef &uri, const http_parser_url &u, - const std::string &match_host, - const std::string &request_authority, - const std::string &upstream_scheme) { +StringRef rewrite_location_uri(BlockAllocator &balloc, const StringRef &uri, + const http_parser_url &u, + const StringRef &match_host, + const StringRef &request_authority, + const StringRef &upstream_scheme) { // We just rewrite scheme and authority. if ((u.field_set & (1 << UF_HOST)) == 0) { - return ""; + return StringRef{}; } auto field = &u.field_data[UF_HOST]; if (!util::starts_with(std::begin(match_host), std::end(match_host), &uri[field->off], &uri[field->off] + field->len) || (match_host.size() != field->len && match_host[field->len] != ':')) { - return ""; + return StringRef{}; } - std::string res; + + auto len = 0; if (!request_authority.empty()) { - res += upstream_scheme; - res += "://"; - res += request_authority; + len += upstream_scheme.size() + str_size("://") + request_authority.size(); + } + + if (u.field_set & (1 << UF_PATH)) { + field = &u.field_data[UF_PATH]; + len += field->len; + } + + if (u.field_set & (1 << UF_QUERY)) { + field = &u.field_data[UF_QUERY]; + len += 1 + field->len; + } + + if (u.field_set & (1 << UF_FRAGMENT)) { + field = &u.field_data[UF_FRAGMENT]; + len += 1 + field->len; + } + + auto iov = make_byte_ref(balloc, len + 1); + auto p = iov.base; + + if (!request_authority.empty()) { + p = std::copy(std::begin(upstream_scheme), std::end(upstream_scheme), p); + p = util::copy_lit(p, "://"); + p = std::copy(std::begin(request_authority), std::end(request_authority), + p); } if (u.field_set & (1 << UF_PATH)) { field = &u.field_data[UF_PATH]; - res.append(&uri[field->off], field->len); + p = std::copy_n(&uri[field->off], field->len, p); } if (u.field_set & (1 << UF_QUERY)) { field = &u.field_data[UF_QUERY]; - res += '?'; - res.append(&uri[field->off], field->len); + *p++ = '?'; + p = std::copy_n(&uri[field->off], field->len, p); } if (u.field_set & (1 << UF_FRAGMENT)) { field = &u.field_data[UF_FRAGMENT]; - res += '#'; - res.append(&uri[field->off], field->len); + *p++ = '#'; + p = std::copy_n(&uri[field->off], field->len, p); } - return res; + + *p = '\0'; + + return StringRef{iov.base, p}; } int check_nv(const uint8_t *name, size_t namelen, const uint8_t *value, @@ -1496,7 +1524,7 @@ StringRef to_method_string(int method_token) { return StringRef{http_method_str(static_cast(method_token))}; } -StringRef get_pure_path_component(const std::string &uri) { +StringRef get_pure_path_component(const StringRef &uri) { int rv; http_parser_url u{}; @@ -1513,9 +1541,9 @@ StringRef get_pure_path_component(const std::string &uri) { return StringRef::from_lit("/"); } -int construct_push_component(std::string &scheme, std::string &authority, - std::string &path, const StringRef &base, - const StringRef &uri) { +int construct_push_component(BlockAllocator &balloc, StringRef &scheme, + StringRef &authority, StringRef &path, + const StringRef &base, const StringRef &uri) { int rv; StringRef rel, relq; @@ -1538,15 +1566,26 @@ int construct_push_component(std::string &scheme, std::string &authority, } } else { if (u.field_set & (1 << UF_SCHEMA)) { - http2::copy_url_component(scheme, &u, UF_SCHEMA, uri.c_str()); + scheme = util::get_uri_field(uri.c_str(), u, UF_SCHEMA); } if (u.field_set & (1 << UF_HOST)) { - http2::copy_url_component(authority, &u, UF_HOST, uri.c_str()); - if (u.field_set & (1 << UF_PORT)) { - authority += ':'; - authority += util::utos(u.port); + auto auth = util::get_uri_field(uri.c_str(), u, UF_HOST); + auto len = auth.size(); + auto port_exists = u.field_set & (1 << UF_PORT); + if (port_exists) { + len += 1 + str_size("65535"); } + auto iov = make_byte_ref(balloc, len + 1); + auto p = iov.base; + p = std::copy(std::begin(auth), std::end(auth), p); + if (port_exists) { + *p++ = ':'; + p = util::utos(p, u.port); + } + *p = '\0'; + + authority = StringRef{iov.base, p}; } if (u.field_set & (1 << UF_PATH)) { @@ -1562,11 +1601,192 @@ int construct_push_component(std::string &scheme, std::string &authority, } } - path = http2::path_join(base, StringRef{}, rel, relq); + path = http2::path_join(balloc, base, StringRef{}, rel, relq); return 0; } +namespace { +template InputIt eat_file(InputIt first, InputIt last) { + if (first == last) { + *first++ = '/'; + return first; + } + + if (*(last - 1) == '/') { + return last; + } + + auto p = last; + for (; p != first && *(p - 1) != '/'; --p) + ; + if (p == first) { + // this should not happend in normal case, where we expect path + // starts with '/' + *first++ = '/'; + return first; + } + + return p; +} +} // namespace + +namespace { +template InputIt eat_dir(InputIt first, InputIt last) { + auto p = eat_file(first, last); + + --p; + + assert(*p == '/'); + + return eat_file(first, p); +} +} // namespace + +StringRef path_join(BlockAllocator &balloc, const StringRef &base_path, + const StringRef &base_query, const StringRef &rel_path, + const StringRef &rel_query) { + auto res = make_byte_ref( + balloc, std::max(static_cast(1), base_path.size()) + + rel_path.size() + 1 + + std::max(base_query.size(), rel_query.size()) + 1); + auto p = res.base; + + if (rel_path.empty()) { + if (base_path.empty()) { + *p++ = '/'; + } else { + p = std::copy(std::begin(base_path), std::end(base_path), p); + } + if (rel_query.empty()) { + if (!base_query.empty()) { + *p++ = '?'; + p = std::copy(std::begin(base_query), std::end(base_query), p); + } + *p = '\0'; + return StringRef{res.base, p}; + } + *p++ = '?'; + p = std::copy(std::begin(rel_query), std::end(rel_query), p); + *p = '\0'; + return StringRef{res.base, p}; + } + + auto first = std::begin(rel_path); + auto last = std::end(rel_path); + + if (rel_path[0] == '/') { + *p++ = '/'; + ++first; + } else if (base_path.empty()) { + *p++ = '/'; + } else { + p = std::copy(std::begin(base_path), std::end(base_path), p); + } + + for (; first != last;) { + if (*first == '.') { + if (first + 1 == last) { + break; + } + if (*(first + 1) == '/') { + first += 2; + continue; + } + if (*(first + 1) == '.') { + if (first + 2 == last) { + p = eat_dir(res.base, p); + break; + } + if (*(first + 2) == '/') { + p = eat_dir(res.base, p); + first += 3; + continue; + } + } + } + if (*(p - 1) != '/') { + p = eat_file(res.base, p); + } + auto slash = std::find(first, last, '/'); + if (slash == last) { + p = std::copy(first, last, p); + break; + } + p = std::copy(first, slash + 1, p); + first = slash + 1; + for (; first != last && *first == '/'; ++first) + ; + } + if (!rel_query.empty()) { + *p++ = '?'; + p = std::copy(std::begin(rel_query), std::end(rel_query), p); + } + *p = '\0'; + return StringRef{res.base, p}; +} + +StringRef normalize_path(BlockAllocator &balloc, const StringRef &path, + const StringRef &query) { + // First, decode %XX for unreserved characters, then do + // http2::join_path + + // We won't find %XX if length is less than 3. + if (path.size() < 3 || + std::find(std::begin(path), std::end(path), '%') == std::end(path)) { + return path_join(balloc, StringRef{}, StringRef{}, path, query); + } + + // includes last terminal NULL. + auto result = make_byte_ref(balloc, path.size() + 1); + auto p = result.base; + + auto it = std::begin(path); + for (; it + 2 != std::end(path);) { + if (*it == '%') { + if (util::is_hex_digit(*(it + 1)) && util::is_hex_digit(*(it + 2))) { + auto c = + (util::hex_to_uint(*(it + 1)) << 4) + util::hex_to_uint(*(it + 2)); + if (util::in_rfc3986_unreserved_chars(c)) { + *p++ = c; + + it += 3; + + continue; + } + *p++ = '%'; + *p++ = util::upcase(*(it + 1)); + *p++ = util::upcase(*(it + 2)); + + it += 3; + + continue; + } + } + *p++ = *it++; + } + + p = std::copy(it, std::end(path), p); + *p = '\0'; + + return path_join(balloc, StringRef{}, StringRef{}, StringRef{result.base, p}, + query); +} + +StringRef rewrite_clean_path(BlockAllocator &balloc, const StringRef &src) { + if (src.empty() || src[0] != '/') { + return src; + } + // probably, not necessary most of the case, but just in case. + auto fragment = std::find(std::begin(src), std::end(src), '#'); + auto query = std::find(std::begin(src), fragment, '?'); + if (query != fragment) { + ++query; + } + return normalize_path(balloc, StringRef{std::begin(src), query}, + StringRef{query, fragment}); +} + } // namespace http2 } // namespace nghttp2 diff --git a/src/http2.h b/src/http2.h index b4e84e8b..9b8b0bbb 100644 --- a/src/http2.h +++ b/src/http2.h @@ -40,6 +40,7 @@ #include "util.h" #include "memchunk.h" #include "template.h" +#include "allocator.h" namespace nghttp2 { @@ -241,10 +242,11 @@ void dump_nv(FILE *out, const HeaderRefs &nva); // This function returns the new rewritten URI on success. If the // location URI is not subject to the rewrite, this function returns // emtpy string. -std::string rewrite_location_uri(const StringRef &uri, const http_parser_url &u, - const std::string &match_host, - const std::string &request_authority, - const std::string &upstream_scheme); +StringRef rewrite_location_uri(BlockAllocator &balloc, const StringRef &uri, + const http_parser_url &u, + const StringRef &match_host, + const StringRef &request_authority, + const StringRef &upstream_scheme); // Checks the header name/value pair using nghttp2_check_header_name() // and nghttp2_check_header_value(). If both function returns nonzero, @@ -337,6 +339,10 @@ std::vector parse_link_header(const char *src, size_t len); std::string path_join(const StringRef &base, const StringRef &base_query, const StringRef &rel_path, const StringRef &rel_query); +StringRef path_join(BlockAllocator &balloc, const StringRef &base_path, + const StringRef &base_query, const StringRef &rel_path, + const StringRef &rel_query); + // true if response has body, taking into account the request method // and status code. bool expect_response_body(const std::string &method, int status_code); @@ -407,19 +413,24 @@ std::string rewrite_clean_path(InputIt first, InputIt last) { return path; } +StringRef normalize_path(BlockAllocator &balloc, const StringRef &path, + const StringRef &query); + +StringRef rewrite_clean_path(BlockAllocator &balloc, const StringRef &src); + // Returns path component of |uri|. The returned path does not // include query component. This function returns empty string if it // fails. -StringRef get_pure_path_component(const std::string &uri); +StringRef get_pure_path_component(const StringRef &uri); // Deduces scheme, authority and path from given |uri|, and stores // them in |scheme|, |authority|, and |path| respectively. If |uri| // is relative path, path resolution takes place using path given in // |base| of length |baselen|. This function returns 0 if it // succeeds, or -1. -int construct_push_component(std::string &scheme, std::string &authority, - std::string &path, const StringRef &base, - const StringRef &uri); +int construct_push_component(BlockAllocator &balloc, StringRef &scheme, + StringRef &authority, StringRef &path, + const StringRef &base, const StringRef &uri); } // namespace http2 diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 3326e885..d7366b04 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -825,11 +825,11 @@ int ClientHandler::perform_http2_upgrade(HttpsUpstream *http) { bool ClientHandler::get_http2_upgrade_allowed() const { return !conn_.tls.ssl; } -std::string ClientHandler::get_upstream_scheme() const { +StringRef ClientHandler::get_upstream_scheme() const { if (conn_.tls.ssl) { - return "https"; + return StringRef::from_lit("https"); } else { - return "http"; + return StringRef::from_lit("http"); } } @@ -843,24 +843,35 @@ namespace { // is mostly same routine found in // HttpDownstreamConnection::push_request_headers(), but vastly // simplified since we only care about absolute URI. -std::string construct_absolute_request_uri(const Request &req) { +StringRef construct_absolute_request_uri(BlockAllocator &balloc, + const Request &req) { if (req.authority.empty()) { return req.path; } - std::string uri; + auto len = req.authority.size() + req.path.size(); + if (req.scheme.empty()) { + len += str_size("http://"); + } else { + len += req.scheme.size() + str_size("://"); + } + + auto iov = make_byte_ref(balloc, len + 1); + auto p = iov.base; + if (req.scheme.empty()) { // We may have to log the request which lacks scheme (e.g., // http/1.1 with origin form). - uri += "http://"; + p = util::copy_lit(p, "http://"); } else { - uri += req.scheme; - uri += "://"; + p = std::copy(std::begin(req.scheme), std::end(req.scheme), p); + p = util::copy_lit(p, "://"); } - uri += req.authority; - uri += req.path; + p = std::copy(std::begin(req.authority), std::end(req.authority), p); + p = std::copy(std::begin(req.path), std::end(req.path), p); + *p = '\0'; - return uri; + return StringRef{iov.base, p}; } } // namespace @@ -869,6 +880,8 @@ void ClientHandler::write_accesslog(Downstream *downstream) { const auto &req = downstream->request(); const auto &resp = downstream->response(); + auto &balloc = downstream->get_block_allocator(); + upstream_accesslog( get_config()->logging.access.format, LogSpec{ @@ -877,7 +890,7 @@ void ClientHandler::write_accesslog(Downstream *downstream) { req.method == HTTP_CONNECT ? StringRef(req.authority) : get_config()->http2_proxy - ? StringRef(construct_absolute_request_uri(req)) + ? StringRef(construct_absolute_request_uri(balloc, req)) : req.path.empty() ? req.method == HTTP_OPTIONS ? StringRef::from_lit("*") diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index ec353576..8d1bc7c0 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -109,7 +109,7 @@ public: int perform_http2_upgrade(HttpsUpstream *http); bool get_http2_upgrade_allowed() const; // Returns upstream scheme, either "http" or "https" - std::string get_upstream_scheme() const; + StringRef get_upstream_scheme() const; void start_immediate_shutdown(); // Writes upstream accesslog using |downstream|. The |downstream| diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index ed1edf2e..f5e4f084 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -561,7 +561,7 @@ int Downstream::end_upload_data() { } void Downstream::rewrite_location_response_header( - const std::string &upstream_scheme) { + const StringRef &upstream_scheme) { auto hd = resp_.fs.header(http2::HD_LOCATION); if (!hd) { return; @@ -577,14 +577,15 @@ void Downstream::rewrite_location_response_header( return; } - auto new_uri = http2::rewrite_location_uri( - hd->value, u, request_downstream_host_, req_.authority, upstream_scheme); + auto new_uri = http2::rewrite_location_uri(balloc_, hd->value, u, + request_downstream_host_, + req_.authority, upstream_scheme); if (new_uri.empty()) { return; } - hd->value = make_string_ref(balloc_, StringRef{new_uri}); + hd->value = new_uri; } bool Downstream::get_chunked_response() const { return chunked_response_; } @@ -879,8 +880,8 @@ void Downstream::add_retry() { ++num_retry_; } bool Downstream::no_more_retry() const { return num_retry_ > 5; } -void Downstream::set_request_downstream_host(std::string host) { - request_downstream_host_ = std::move(host); +void Downstream::set_request_downstream_host(const StringRef &host) { + request_downstream_host_ = host; } void Downstream::set_request_pending(bool f) { request_pending_ = f; } diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 2ea0bf2e..cea17d86 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -147,17 +147,17 @@ struct Request { FieldStore fs; // Request scheme. For HTTP/2, this is :scheme header field value. // For HTTP/1.1, this is deduced from URI or connection. - std::string scheme; + StringRef scheme; // Request authority. This is HTTP/2 :authority header field value // or host header field value. We may deduce it from absolute-form // HTTP/1 request. We also store authority-form HTTP/1 request. // This could be empty if request comes from HTTP/1.0 without Host // header field and origin-form. - std::string authority; + StringRef authority; // Request path, including query component. For HTTP/1.1, this is // request-target. For HTTP/2, this is :path header field value. // For CONNECT request, this is empty. - std::string path; + StringRef path; // the length of request body received so far int64_t recv_body_length; // The number of bytes not consumed by the application yet. @@ -275,7 +275,7 @@ public: // Validates that received request body length and content-length // matches. bool validate_request_recv_body_length() const; - void set_request_downstream_host(std::string host); + void set_request_downstream_host(const StringRef &host); bool expect_response_body() const; enum { INITIAL, @@ -306,7 +306,7 @@ public: Response &response() { return resp_; } // Rewrites the location response header field. - void rewrite_location_response_header(const std::string &upstream_scheme); + void rewrite_location_response_header(const StringRef &upstream_scheme); bool get_chunked_response() const; void set_chunked_response(bool f); @@ -407,7 +407,7 @@ private: // host we requested to downstream. This is used to rewrite // location header field to decide the location should be rewritten // or not. - std::string request_downstream_host_; + StringRef request_downstream_host_; DefaultMemchunks request_buf_; DefaultMemchunks response_buf_; diff --git a/src/shrpx_downstream_queue.cc b/src/shrpx_downstream_queue.cc index 893effb9..91e88cff 100644 --- a/src/shrpx_downstream_queue.cc +++ b/src/shrpx_downstream_queue.cc @@ -71,14 +71,11 @@ DownstreamQueue::find_host_entry(const std::string &host) { return (*itr).second; } -const std::string & -DownstreamQueue::make_host_key(const std::string &host) const { - static std::string empty_key; - return unified_host_ ? empty_key : host; +std::string DownstreamQueue::make_host_key(const StringRef &host) const { + return unified_host_ ? "" : host.str(); } -const std::string & -DownstreamQueue::make_host_key(Downstream *downstream) const { +std::string DownstreamQueue::make_host_key(Downstream *downstream) const { return make_host_key(downstream->request().authority); } @@ -99,7 +96,7 @@ void DownstreamQueue::mark_blocked(Downstream *downstream) { ent.blocked.append(link); } -bool DownstreamQueue::can_activate(const std::string &host) const { +bool DownstreamQueue::can_activate(const StringRef &host) const { auto itr = host_entries_.find(make_host_key(host)); if (itr == std::end(host_entries_)) { return true; @@ -127,7 +124,7 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream, downstreams_.remove(downstream); - auto &host = make_host_key(downstream); + auto host = make_host_key(downstream); auto &ent = find_host_entry(host); if (downstream->get_dispatch_state() == Downstream::DISPATCH_ACTIVE) { diff --git a/src/shrpx_downstream_queue.h b/src/shrpx_downstream_queue.h index 755af10d..47e8555a 100644 --- a/src/shrpx_downstream_queue.h +++ b/src/shrpx_downstream_queue.h @@ -77,7 +77,7 @@ public: void mark_blocked(Downstream *downstream); // Returns true if we can make downstream connection to given // |host|. - bool can_activate(const std::string &host) const; + bool can_activate(const StringRef &host) const; // Removes and frees |downstream| object. If |downstream| is in // Downstream::DISPATCH_ACTIVE, and |next_blocked| is true, this // function may return Downstream object with the same target host @@ -87,8 +87,8 @@ public: bool next_blocked = true); Downstream *get_downstreams() const; HostEntry &find_host_entry(const std::string &host); - const std::string &make_host_key(const std::string &host) const; - const std::string &make_host_key(Downstream *downstream) const; + std::string make_host_key(const StringRef &host) const; + std::string make_host_key(Downstream *downstream) const; private: // Per target host structure to keep track of the number of diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index 357bd664..5e2d5129 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -282,10 +282,10 @@ int Http2DownstreamConnection::push_request_headers() { auto authority = StringRef(downstream_hostport); if (no_host_rewrite && !req.authority.empty()) { - authority = StringRef(req.authority); + authority = req.authority; } - downstream_->set_request_downstream_host(authority.str()); + downstream_->set_request_downstream_host(authority); size_t num_cookies = 0; if (!http2conf.no_cookie_crumbling) { @@ -359,9 +359,9 @@ int Http2DownstreamConnection::push_request_headers() { params &= ~FORWARDED_PROTO; } - auto value = http::create_forwarded( - params, handler->get_forwarded_by(), handler->get_forwarded_for(), - StringRef{req.authority}, StringRef{req.scheme}); + auto value = http::create_forwarded(params, handler->get_forwarded_by(), + handler->get_forwarded_for(), + req.authority, req.scheme); if (fwd || !value.empty()) { if (fwd) { forwarded_value = fwd->value.str(); diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 9a411ca1..828d888c 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -1986,6 +1986,8 @@ int Http2Session::handle_downstream_push_promise_complete( Downstream *downstream, Downstream *promised_downstream) { auto &promised_req = promised_downstream->request(); + auto &promised_balloc = promised_downstream->get_block_allocator(); + auto authority = promised_req.fs.header(http2::HD__AUTHORITY); auto path = promised_req.fs.header(http2::HD__PATH); auto method = promised_req.fs.header(http2::HD__METHOD); @@ -2007,16 +2009,19 @@ int Http2Session::handle_downstream_push_promise_complete( // TODO Rewrite authority if we enabled rewrite host. But we // really don't know how to rewrite host. Should we use the same // host in associated stream? - promised_req.authority = http2::value_to_str(authority); + if (authority) { + promised_req.authority = authority->value; + } promised_req.method = method_token; // libnghttp2 ensures that we don't have CONNECT method in // PUSH_PROMISE, and guarantees that :scheme exists. - promised_req.scheme = http2::value_to_str(scheme); + if (scheme) { + promised_req.scheme = scheme->value; + } // For server-wide OPTIONS request, path is empty. if (method_token != HTTP_OPTIONS || path->value != "*") { - promised_req.path = http2::rewrite_clean_path(std::begin(path->value), - std::end(path->value)); + promised_req.path = http2::rewrite_clean_path(promised_balloc, path->value); } promised_downstream->inspect_http2_request(); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index cf89bf3e..5bfdb826 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -303,7 +303,9 @@ int Http2Upstream::on_request_headers(Downstream *downstream, } req.method = method_token; - req.scheme = http2::value_to_str(scheme); + if (scheme) { + req.scheme = scheme->value; + } // nghttp2 library guarantees either :authority or host exist if (!authority) { @@ -311,16 +313,18 @@ int Http2Upstream::on_request_headers(Downstream *downstream, authority = req.fs.header(http2::HD_HOST); } - req.authority = http2::value_to_str(authority); + if (authority) { + req.authority = authority->value; + } if (path) { if (method_token == HTTP_OPTIONS && path->value == "*") { // Server-wide OPTIONS request. Path is empty. } else if (get_config()->http2_proxy) { - req.path = http2::value_to_str(path); + req.path = path->value; } else { - const auto &value = path->value; - req.path = http2::rewrite_clean_path(std::begin(value), std::end(value)); + req.path = http2::rewrite_clean_path(downstream->get_block_allocator(), + path->value); } } @@ -580,6 +584,8 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame, req.http_major = 2; req.http_minor = 0; + auto &promised_balloc = promised_downstream->get_block_allocator(); + for (size_t i = 0; i < frame->push_promise.nvlen; ++i) { auto &nv = frame->push_promise.nva[i]; auto token = http2::lookup_token(nv.name, nv.namelen); @@ -588,13 +594,16 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame, req.method = http2::lookup_method_token(nv.value, nv.valuelen); break; case http2::HD__SCHEME: - req.scheme.assign(nv.value, nv.value + nv.valuelen); + req.scheme = + make_string_ref(promised_balloc, StringRef{nv.value, nv.valuelen}); break; case http2::HD__AUTHORITY: - req.authority.assign(nv.value, nv.value + nv.valuelen); + req.authority = + make_string_ref(promised_balloc, StringRef{nv.value, nv.valuelen}); break; case http2::HD__PATH: - req.path = http2::rewrite_clean_path(nv.value, nv.value + nv.valuelen); + req.path = http2::rewrite_clean_path(promised_balloc, + StringRef{nv.value, nv.valuelen}); break; } req.fs.add_header_token(StringRef{nv.name, nv.namelen}, @@ -1746,6 +1755,8 @@ int Http2Upstream::prepare_push_promise(Downstream *downstream) { return 0; } + auto &balloc = downstream->get_block_allocator(); + for (auto &kv : resp.fs.headers()) { if (kv.token != http2::HD_LINK) { continue; @@ -1753,28 +1764,23 @@ int Http2Upstream::prepare_push_promise(Downstream *downstream) { for (auto &link : http2::parse_link_header(kv.value.c_str(), kv.value.size())) { - const std::string *scheme_ptr, *authority_ptr; - std::string scheme, authority, path; + StringRef scheme, authority, path; - rv = http2::construct_push_component(scheme, authority, path, base, - link.uri); + rv = http2::construct_push_component(balloc, scheme, authority, path, + base, link.uri); if (rv != 0) { continue; } if (scheme.empty()) { - scheme_ptr = &req.scheme; - } else { - scheme_ptr = &scheme; + scheme = req.scheme; } if (authority.empty()) { - authority_ptr = &req.authority; - } else { - authority_ptr = &authority; + authority = req.authority; } - rv = submit_push_promise(*scheme_ptr, *authority_ptr, path, downstream); + rv = submit_push_promise(scheme, authority, path, downstream); if (rv != 0) { return -1; } @@ -1783,9 +1789,9 @@ int Http2Upstream::prepare_push_promise(Downstream *downstream) { return 0; } -int Http2Upstream::submit_push_promise(const std::string &scheme, - const std::string &authority, - const std::string &path, +int Http2Upstream::submit_push_promise(const StringRef &scheme, + const StringRef &authority, + const StringRef &path, Downstream *downstream) { const auto &req = downstream->request(); @@ -1795,9 +1801,9 @@ int Http2Upstream::submit_push_promise(const std::string &scheme, // juse use "GET" for now nva.push_back(http2::make_nv_ll(":method", "GET")); - nva.push_back(http2::make_nv_ls(":scheme", scheme)); - nva.push_back(http2::make_nv_ls(":path", path)); - nva.push_back(http2::make_nv_ls(":authority", authority)); + nva.push_back(http2::make_nv_ls_nocopy(":scheme", scheme)); + nva.push_back(http2::make_nv_ls_nocopy(":path", path)); + nva.push_back(http2::make_nv_ls_nocopy(":authority", authority)); for (auto &kv : req.fs.headers()) { switch (kv.token) { @@ -1865,27 +1871,25 @@ int Http2Upstream::initiate_push(Downstream *downstream, const StringRef &uri) { return -1; } - const std::string *scheme_ptr, *authority_ptr; - std::string scheme, authority, path; + auto &balloc = downstream->get_block_allocator(); - rv = http2::construct_push_component(scheme, authority, path, base, uri); + StringRef scheme, authority, path; + + rv = http2::construct_push_component(balloc, scheme, authority, path, base, + uri); if (rv != 0) { return -1; } if (scheme.empty()) { - scheme_ptr = &req.scheme; - } else { - scheme_ptr = &scheme; + scheme = req.scheme; } if (authority.empty()) { - authority_ptr = &req.authority; - } else { - authority_ptr = &authority; + authority = req.authority; } - rv = submit_push_promise(*scheme_ptr, *authority_ptr, path, downstream); + rv = submit_push_promise(scheme, authority, path, downstream); if (rv != 0) { return -1; @@ -1943,7 +1947,7 @@ int Http2Upstream::on_downstream_push_promise_complete( nva.reserve(headers.size()); for (auto &kv : headers) { - nva.push_back(http2::make_nv_nocopy(kv.name, kv.value, kv.no_index)); + nva.push_back(http2::make_nv(kv.name, kv.value, kv.no_index)); } auto promised_stream_id = nghttp2_submit_push_promise( diff --git a/src/shrpx_http2_upstream.h b/src/shrpx_http2_upstream.h index bf1e1c28..1908a504 100644 --- a/src/shrpx_http2_upstream.h +++ b/src/shrpx_http2_upstream.h @@ -111,9 +111,8 @@ public: void check_shutdown(); int prepare_push_promise(Downstream *downstream); - int submit_push_promise(const std::string &scheme, - const std::string &authority, const std::string &path, - Downstream *downstream); + int submit_push_promise(const StringRef &scheme, const StringRef &authority, + const StringRef &path, Downstream *downstream); int on_request_headers(Downstream *downstream, const nghttp2_frame *frame); diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index a6c238a1..d7f5492b 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -283,10 +283,10 @@ int HttpDownstreamConnection::push_request_headers() { httpconf.no_host_rewrite || get_config()->http2_proxy || connect_method; if (no_host_rewrite && !req.authority.empty()) { - authority = StringRef(req.authority); + authority = req.authority; } - downstream_->set_request_downstream_host(authority.str()); + downstream_->set_request_downstream_host(authority); auto buf = downstream_->get_request_buf(); @@ -366,9 +366,9 @@ int HttpDownstreamConnection::push_request_headers() { params &= ~FORWARDED_PROTO; } - auto value = http::create_forwarded( - params, handler->get_forwarded_by(), handler->get_forwarded_for(), - StringRef{req.authority}, StringRef{req.scheme}); + auto value = http::create_forwarded(params, handler->get_forwarded_by(), + handler->get_forwarded_for(), + req.authority, req.scheme); if (fwd || !value.empty()) { buf->append("Forwarded: "); if (fwd) { diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 92994207..73e91a67 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -86,6 +86,8 @@ int htp_uricb(http_parser *htp, const char *data, size_t len) { auto downstream = upstream->get_downstream(); auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + // We happen to have the same value for method token. req.method = htp->method; @@ -103,9 +105,10 @@ int htp_uricb(http_parser *htp, const char *data, size_t len) { req.fs.add_extra_buffer_size(len); if (req.method == HTTP_CONNECT) { - req.authority.append(data, len); + req.authority = + concat_string_ref(balloc, req.authority, StringRef{data, len}); } else { - req.path.append(data, len); + req.path = concat_string_ref(balloc, req.path, StringRef{data, len}); } return 0; @@ -190,30 +193,51 @@ int htp_hdr_valcb(http_parser *htp, const char *data, size_t len) { } // namespace namespace { -void rewrite_request_host_path_from_uri(Request &req, const char *uri, +void rewrite_request_host_path_from_uri(BlockAllocator &balloc, Request &req, + const StringRef &uri, http_parser_url &u) { assert(u.field_set & (1 << UF_HOST)); - auto &authority = req.authority; - authority.clear(); // As per https://tools.ietf.org/html/rfc7230#section-5.4, we // rewrite host header field with authority component. - http2::copy_url_component(authority, &u, UF_HOST, uri); + auto authority = util::get_uri_field(uri.c_str(), u, UF_HOST); // TODO properly check IPv6 numeric address - if (authority.find(':') != std::string::npos) { - authority = '[' + authority; - authority += ']'; + auto ipv6 = std::find(std::begin(authority), std::end(authority), ':') != + std::end(authority); + auto authoritylen = authority.size(); + if (ipv6) { + authoritylen += 2; } if (u.field_set & (1 << UF_PORT)) { - authority += ':'; - authority += util::utos(u.port); + authoritylen += 1 + str_size("65535"); + } + if (authoritylen > authority.size()) { + auto iovec = make_byte_ref(balloc, authoritylen + 1); + auto p = iovec.base; + if (ipv6) { + *p++ = '['; + } + p = std::copy(std::begin(authority), std::end(authority), p); + if (ipv6) { + *p++ = ']'; + } + + if (u.field_set & (1 << UF_PORT)) { + *p++ = ':'; + p = util::utos(p, u.port); + } + *p = '\0'; + + req.authority = StringRef{iovec.base, p}; + } else { + req.authority = authority; } - http2::copy_url_component(req.scheme, &u, UF_SCHEMA, uri); + req.scheme = util::get_uri_field(uri.c_str(), u, UF_SCHEMA); - std::string path; + StringRef path; if (u.field_set & (1 << UF_PATH)) { - http2::copy_url_component(path, &u, UF_PATH, uri); + path = util::get_uri_field(uri.c_str(), u, UF_PATH); } else if (req.method == HTTP_OPTIONS) { // Server-wide OPTIONS takes following form in proxy request: // @@ -221,21 +245,35 @@ void rewrite_request_host_path_from_uri(Request &req, const char *uri, // // Notice that no slash after authority. See // http://tools.ietf.org/html/rfc7230#section-5.3.4 - req.path = ""; + req.path = StringRef::from_lit(""); // we ignore query component here return; } else { - path = "/"; + path = StringRef::from_lit("/"); } + if (u.field_set & (1 << UF_QUERY)) { auto &fdata = u.field_data[UF_QUERY]; - path += '?'; - path.append(uri + fdata.off, fdata.len); + + if (u.field_set & (1 << UF_PATH)) { + auto q = util::get_uri_field(uri.c_str(), u, UF_QUERY); + path = StringRef{std::begin(path), std::end(q)}; + } else { + auto iov = make_byte_ref(balloc, path.size() + 1 + fdata.len + 1); + auto p = iov.base; + + p = std::copy(std::begin(path), std::end(path), p); + *p++ = '?'; + p = std::copy_n(&uri[fdata.off], fdata.len, p); + *p = '\0'; + path = StringRef{iov.base, p}; + } } + if (get_config()->http2_proxy) { - req.path = std::move(path); + req.path = path; } else { - req.path = http2::rewrite_clean_path(std::begin(path), std::end(path)); + req.path = http2::rewrite_clean_path(balloc, path); } } } // namespace @@ -299,12 +337,11 @@ int htp_hdrs_completecb(http_parser *htp) { downstream->inspect_http1_request(); + auto &balloc = downstream->get_block_allocator(); + if (method != HTTP_CONNECT) { http_parser_url u{}; - // make a copy of request path, since we may set request path - // while we are refering to original request path. - auto path = req.path; - rv = http_parser_parse_url(path.c_str(), path.size(), 0, &u); + rv = http_parser_parse_url(req.path.c_str(), req.path.size(), 0, &u); if (rv != 0) { // Expect to respond with 400 bad request return -1; @@ -318,23 +355,23 @@ int htp_hdrs_completecb(http_parser *htp) { req.no_authority = true; - if (method == HTTP_OPTIONS && path == "*") { - req.path = ""; + if (method == HTTP_OPTIONS && req.path == "*") { + req.path = StringRef{}; } else { - req.path = http2::rewrite_clean_path(std::begin(path), std::end(path)); + req.path = http2::rewrite_clean_path(balloc, req.path); } if (host) { - req.authority = host->value.str(); + req.authority = host->value; } if (handler->get_ssl()) { - req.scheme = "https"; + req.scheme = StringRef::from_lit("https"); } else { - req.scheme = "http"; + req.scheme = StringRef::from_lit("http"); } } else { - rewrite_request_host_path_from_uri(req, path.c_str(), u); + rewrite_request_host_path_from_uri(balloc, req, req.path, u); } } diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index 74201ff2..bf83eba2 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -116,6 +116,8 @@ mrb_value request_set_authority(mrb_state *mrb, mrb_value self) { auto downstream = data->downstream; auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + check_phase(mrb, data->phase, PHASE_REQUEST); const char *authority; @@ -125,7 +127,8 @@ mrb_value request_set_authority(mrb_state *mrb, mrb_value self) { mrb_raise(mrb, E_RUNTIME_ERROR, "authority must not be empty string"); } - req.authority.assign(authority, n); + req.authority = + make_string_ref(balloc, StringRef{authority, static_cast(n)}); return self; } @@ -147,6 +150,8 @@ mrb_value request_set_scheme(mrb_state *mrb, mrb_value self) { auto downstream = data->downstream; auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + check_phase(mrb, data->phase, PHASE_REQUEST); const char *scheme; @@ -156,7 +161,8 @@ mrb_value request_set_scheme(mrb_state *mrb, mrb_value self) { mrb_raise(mrb, E_RUNTIME_ERROR, "scheme must not be empty string"); } - req.scheme.assign(scheme, n); + req.scheme = + make_string_ref(balloc, StringRef{scheme, static_cast(n)}); return self; } @@ -178,13 +184,16 @@ mrb_value request_set_path(mrb_state *mrb, mrb_value self) { auto downstream = data->downstream; auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + check_phase(mrb, data->phase, PHASE_REQUEST); const char *path; mrb_int pathlen; mrb_get_args(mrb, "s", &path, &pathlen); - req.path.assign(path, pathlen); + req.path = + make_string_ref(balloc, StringRef{path, static_cast(pathlen)}); return self; } diff --git a/src/shrpx_mruby_module_response.cc b/src/shrpx_mruby_module_response.cc index c50391e1..ada76b7c 100644 --- a/src/shrpx_mruby_module_response.cc +++ b/src/shrpx_mruby_module_response.cc @@ -187,6 +187,8 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { auto &resp = downstream->response(); int rv; + auto &balloc = downstream->get_block_allocator(); + if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { mrb_raise(mrb, E_RUNTIME_ERROR, "response has already been committed"); } @@ -207,14 +209,22 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { bodylen = vallen; } + StringRef content_length; + { + auto iov = make_byte_ref(balloc, str_size("18446744073709551615") + 1); + auto p = iov.base; + p = util::utos(p, bodylen); + *p = '\0'; + content_length = StringRef{iov.base, p}; + } + auto cl = resp.fs.header(http2::HD_CONTENT_LENGTH); if (cl) { - cl->value = make_string_ref(downstream->get_block_allocator(), - StringRef{util::utos(bodylen)}); + cl->value = content_length; } else { + // TODO we don't have to make a copy here. resp.fs.add_header_token(StringRef::from_lit("content-length"), - StringRef{util::utos(bodylen)}, false, - http2::HD_CONTENT_LENGTH); + content_length, false, http2::HD_CONTENT_LENGTH); } resp.fs.content_length = bodylen; diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index e5e1a096..89508ba8 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -159,6 +159,8 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + downstream->reset_upstream_rtimer(); auto nv = frame->syn_stream.nv; @@ -262,17 +264,16 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, req.method = method_token; if (is_connect) { - req.authority = path->value.str(); + req.authority = path->value; } else { - req.scheme = scheme->value.str(); - req.authority = host->value.str(); + req.scheme = scheme->value; + req.authority = host->value; if (get_config()->http2_proxy) { - req.path = path->value.str(); + req.path = path->value; } else if (method_token == HTTP_OPTIONS && path->value == "*") { // Server-wide OPTIONS request. Path is empty. } else { - req.path = http2::rewrite_clean_path(std::begin(path->value), - std::end(path->value)); + req.path = http2::rewrite_clean_path(balloc, path->value); } } diff --git a/src/template.h b/src/template.h index 9c52387f..b15efa02 100644 --- a/src/template.h +++ b/src/template.h @@ -409,7 +409,8 @@ public: : base(&*first), len(std::distance(first, last)) {} template StringRef(InputIt *first, InputIt *last) - : base(first), len(std::distance(first, last)) {} + : base(reinterpret_cast(first)), + len(std::distance(first, last)) {} template constexpr static StringRef from_lit(const CharT(&s)[N]) { return StringRef(s, N - 1); diff --git a/src/util.h b/src/util.h index ba3e1277..f24f8140 100644 --- a/src/util.h +++ b/src/util.h @@ -387,6 +387,23 @@ template std::string utos(T n) { return res; } +template OutputIt utos(OutputIt dst, T n) { + if (n == 0) { + *dst++ = '0'; + return dst; + } + int i = 0; + T t = n; + for (; t; t /= 10, ++i) + ; + --i; + auto p = dst + i; + for (; n; --i, n /= 10) { + *p-- = (n % 10) + '0'; + } + return dst + i + 1; +} + template std::string utos_unit(T n) { char u = 0; if (n >= (1 << 30)) { @@ -695,6 +712,11 @@ std::string random_alpha_digit(Generator &gen, size_t len) { return res; } +template +OutputIterator copy_lit(OutputIterator it, CharT(&s)[N]) { + return std::copy_n(s, N - 1, it); +} + } // namespace util } // namespace nghttp2 From 7a412df9a540dac5e3f9fbb9dad22b0ac36e6bd7 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 10 Mar 2016 23:50:04 +0900 Subject: [PATCH 04/19] nghttpx: Fix tests --- src/http2.cc | 8 +- src/http2.h | 51 +--------- src/http2_test.cc | 188 ++++++++++++++++++----------------- src/shrpx-unittest.cc | 1 + src/shrpx_config.cc | 3 +- src/shrpx_downstream.cc | 5 +- src/shrpx_downstream_test.cc | 55 +++++----- src/util.h | 3 +- src/util_test.cc | 9 ++ src/util_test.h | 1 + 10 files changed, 153 insertions(+), 171 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index 28314134..3a4474b8 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -1742,7 +1742,7 @@ StringRef normalize_path(BlockAllocator &balloc, const StringRef &path, auto p = result.base; auto it = std::begin(path); - for (; it + 2 != std::end(path);) { + for (; it + 2 < std::end(path);) { if (*it == '%') { if (util::is_hex_digit(*(it + 1)) && util::is_hex_digit(*(it + 2))) { auto c = @@ -1773,6 +1773,12 @@ StringRef normalize_path(BlockAllocator &balloc, const StringRef &path, query); } +std::string normalize_path(const StringRef &path, const StringRef &query) { + BlockAllocator balloc(1024, 1024); + + return normalize_path(balloc, path, query).str(); +} + StringRef rewrite_clean_path(BlockAllocator &balloc, const StringRef &src) { if (src.empty() || src[0] != '/') { return src; diff --git a/src/http2.h b/src/http2.h index 9b8b0bbb..cc295079 100644 --- a/src/http2.h +++ b/src/http2.h @@ -364,58 +364,11 @@ int lookup_method_token(const StringRef &name); // StringRef is guaranteed to be NULL-terminated. StringRef to_method_string(int method_token); -template -std::string normalize_path(InputIt first, InputIt last) { - // First, decode %XX for unreserved characters, then do - // http2::join_path - std::string result; - // We won't find %XX if length is less than 3. - if (last - first < 3) { - result.assign(first, last); - } else { - for (; first < last - 2;) { - if (*first == '%') { - if (util::is_hex_digit(*(first + 1)) && - util::is_hex_digit(*(first + 2))) { - auto c = (util::hex_to_uint(*(first + 1)) << 4) + - util::hex_to_uint(*(first + 2)); - if (util::in_rfc3986_unreserved_chars(c)) { - result += c; - first += 3; - continue; - } - result += '%'; - result += util::upcase(*(first + 1)); - result += util::upcase(*(first + 2)); - first += 3; - continue; - } - } - result += *first++; - } - result.append(first, last); - } - return path_join(StringRef{}, StringRef{}, StringRef{result}, StringRef{}); -} - -template -std::string rewrite_clean_path(InputIt first, InputIt last) { - if (first == last || *first != '/') { - return std::string(first, last); - } - // probably, not necessary most of the case, but just in case. - auto fragment = std::find(first, last, '#'); - auto query = std::find(first, fragment, '?'); - auto path = normalize_path(first, query); - if (query != fragment) { - path.append(query, fragment); - } - return path; -} - StringRef normalize_path(BlockAllocator &balloc, const StringRef &path, const StringRef &query); +std::string normalize_path(const StringRef &path, const StringRef &query); + StringRef rewrite_clean_path(BlockAllocator &balloc, const StringRef &src); // Returns path component of |uri|. The returned path does not diff --git a/src/http2_test.cc b/src/http2_test.cc index 4d4c7669..5ac726aa 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -46,7 +46,7 @@ using namespace nghttp2; namespace shrpx { namespace { -void check_nv(const Header &a, const nghttp2_nv *b) { +void check_nv(const HeaderRef &a, const nghttp2_nv *b) { CU_ASSERT(a.name.size() == b->namelen); CU_ASSERT(a.value.size() == b->valuelen); CU_ASSERT(memcmp(a.name.c_str(), b->name, b->namelen) == 0); @@ -134,20 +134,24 @@ void test_http2_get_header(void) { } namespace { -auto headers = - Headers{{"alpha", "0", true}, - {"bravo", "1"}, - {"connection", "2", false, http2::HD_CONNECTION}, - {"connection", "3", false, http2::HD_CONNECTION}, - {"delta", "4"}, - {"expect", "5"}, - {"foxtrot", "6"}, - {"tango", "7"}, - {"te", "8", false, http2::HD_TE}, - {"te", "9", false, http2::HD_TE}, - {"x-forwarded-proto", "10", false, http2::HD_X_FORWARDED_FOR}, - {"x-forwarded-proto", "11", false, http2::HD_X_FORWARDED_FOR}, - {"zulu", "12"}}; +auto headers = HeaderRefs{ + {StringRef::from_lit("alpha"), StringRef::from_lit("0"), true}, + {StringRef::from_lit("bravo"), StringRef::from_lit("1")}, + {StringRef::from_lit("connection"), StringRef::from_lit("2"), false, + http2::HD_CONNECTION}, + {StringRef::from_lit("connection"), StringRef::from_lit("3"), false, + http2::HD_CONNECTION}, + {StringRef::from_lit("delta"), StringRef::from_lit("4")}, + {StringRef::from_lit("expect"), StringRef::from_lit("5")}, + {StringRef::from_lit("foxtrot"), StringRef::from_lit("6")}, + {StringRef::from_lit("tango"), StringRef::from_lit("7")}, + {StringRef::from_lit("te"), StringRef::from_lit("8"), false, http2::HD_TE}, + {StringRef::from_lit("te"), StringRef::from_lit("9"), false, http2::HD_TE}, + {StringRef::from_lit("x-forwarded-proto"), StringRef::from_lit("10"), false, + http2::HD_X_FORWARDED_FOR}, + {StringRef::from_lit("x-forwarded-proto"), StringRef::from_lit("11"), false, + http2::HD_X_FORWARDED_FOR}, + {StringRef::from_lit("zulu"), StringRef::from_lit("12")}}; } // namespace void test_http2_copy_headers_to_nva(void) { @@ -209,10 +213,12 @@ void check_rewrite_location_uri(const std::string &want, const std::string &uri, const std::string &match_host, const std::string &req_authority, const std::string &upstream_scheme) { + BlockAllocator balloc(4096, 4096); http_parser_url u{}; CU_ASSERT(0 == http_parser_parse_url(uri.c_str(), uri.size(), 0, &u)); - auto got = http2::rewrite_location_uri(uri, u, match_host, req_authority, - upstream_scheme); + auto got = http2::rewrite_location_uri( + balloc, StringRef{uri}, u, StringRef{match_host}, + StringRef{req_authority}, StringRef{upstream_scheme}); CU_ASSERT(want == got); } } // namespace @@ -245,13 +251,13 @@ void test_http2_rewrite_location_uri(void) { } void test_http2_parse_http_status_code(void) { - CU_ASSERT(200 == http2::parse_http_status_code("200")); - CU_ASSERT(102 == http2::parse_http_status_code("102")); - CU_ASSERT(-1 == http2::parse_http_status_code("099")); - CU_ASSERT(-1 == http2::parse_http_status_code("99")); - CU_ASSERT(-1 == http2::parse_http_status_code("-1")); - CU_ASSERT(-1 == http2::parse_http_status_code("20a")); - CU_ASSERT(-1 == http2::parse_http_status_code("")); + CU_ASSERT(200 == http2::parse_http_status_code(StringRef::from_lit("200"))); + CU_ASSERT(102 == http2::parse_http_status_code(StringRef::from_lit("102"))); + CU_ASSERT(-1 == http2::parse_http_status_code(StringRef::from_lit("099"))); + CU_ASSERT(-1 == http2::parse_http_status_code(StringRef::from_lit("99"))); + CU_ASSERT(-1 == http2::parse_http_status_code(StringRef::from_lit("-1"))); + CU_ASSERT(-1 == http2::parse_http_status_code(StringRef::from_lit("20a"))); + CU_ASSERT(-1 == http2::parse_http_status_code(StringRef{})); } void test_http2_index_header(void) { @@ -814,137 +820,135 @@ void test_http2_path_join(void) { } void test_http2_normalize_path(void) { - std::string src; - - src = "/alpha/bravo/../charlie"; CU_ASSERT("/alpha/charlie" == - http2::normalize_path(std::begin(src), std::end(src))); + http2::normalize_path( + StringRef::from_lit("/alpha/bravo/../charlie"), StringRef{})); - src = "/a%6c%70%68%61"; - CU_ASSERT("/alpha" == http2::normalize_path(std::begin(src), std::end(src))); + CU_ASSERT("/alpha" == + http2::normalize_path(StringRef::from_lit("/a%6c%70%68%61"), + StringRef{})); - src = "/alpha%2f%3a"; - CU_ASSERT("/alpha%2F%3A" == - http2::normalize_path(std::begin(src), std::end(src))); + CU_ASSERT( + "/alpha%2F%3A" == + http2::normalize_path(StringRef::from_lit("/alpha%2f%3a"), StringRef{})); - src = "%2f"; - CU_ASSERT("/%2F" == http2::normalize_path(std::begin(src), std::end(src))); + CU_ASSERT("/%2F" == + http2::normalize_path(StringRef::from_lit("%2f"), StringRef{})); - src = "%f"; - CU_ASSERT("/%f" == http2::normalize_path(std::begin(src), std::end(src))); + CU_ASSERT("/%f" == + http2::normalize_path(StringRef::from_lit("%f"), StringRef{})); - src = "%"; - CU_ASSERT("/%" == http2::normalize_path(std::begin(src), std::end(src))); + CU_ASSERT("/%" == + http2::normalize_path(StringRef::from_lit("%"), StringRef{})); - src = ""; - CU_ASSERT("/" == http2::normalize_path(std::begin(src), std::end(src))); + CU_ASSERT("/" == http2::normalize_path(StringRef{}, StringRef{})); + + CU_ASSERT("/alpha?bravo" == + http2::normalize_path(StringRef::from_lit("/alpha"), + StringRef::from_lit("bravo"))); } void test_http2_rewrite_clean_path(void) { - std::string src; + BlockAllocator balloc(4096, 4096); // unreserved characters - src = "/alpha/%62ravo/"; CU_ASSERT("/alpha/bravo/" == - http2::rewrite_clean_path(std::begin(src), std::end(src))); + http2::rewrite_clean_path(balloc, + StringRef::from_lit("/alpha/%62ravo/"))); // percent-encoding is converted to upper case. - src = "/delta%3a"; - CU_ASSERT("/delta%3A" == - http2::rewrite_clean_path(std::begin(src), std::end(src))); + CU_ASSERT("/delta%3A" == http2::rewrite_clean_path( + balloc, StringRef::from_lit("/delta%3a"))); // path component is normalized before mathcing - src = "/alpha/charlie/%2e././bravo/delta/.."; - CU_ASSERT("/alpha/bravo/" == - http2::rewrite_clean_path(std::begin(src), std::end(src))); + CU_ASSERT( + "/alpha/bravo/" == + http2::rewrite_clean_path( + balloc, StringRef::from_lit("/alpha/charlie/%2e././bravo/delta/.."))); - src = "alpha%3a"; - CU_ASSERT(src == http2::rewrite_clean_path(std::begin(src), std::end(src))); + CU_ASSERT("alpha%3a" == + http2::rewrite_clean_path(balloc, StringRef::from_lit("alpha%3a"))); - src = ""; - CU_ASSERT(src == http2::rewrite_clean_path(std::begin(src), std::end(src))); + CU_ASSERT("" == http2::rewrite_clean_path(balloc, StringRef{})); } void test_http2_get_pure_path_component(void) { - std::string path; + CU_ASSERT("/" == http2::get_pure_path_component(StringRef::from_lit("/"))); - path = "/"; - CU_ASSERT("/" == http2::get_pure_path_component(path)); + CU_ASSERT("/foo" == + http2::get_pure_path_component(StringRef::from_lit("/foo"))); - path = "/foo"; - CU_ASSERT("/foo" == http2::get_pure_path_component(path)); + CU_ASSERT("/bar" == http2::get_pure_path_component( + StringRef::from_lit("https://example.org/bar"))); - path = "https://example.org/bar"; - CU_ASSERT("/bar" == http2::get_pure_path_component(path)); + CU_ASSERT("/alpha" == http2::get_pure_path_component(StringRef::from_lit( + "https://example.org/alpha?q=a"))); - path = "https://example.org/alpha?q=a"; - CU_ASSERT("/alpha" == http2::get_pure_path_component(path)); + CU_ASSERT("/bravo" == http2::get_pure_path_component(StringRef::from_lit( + "https://example.org/bravo?q=a#fragment"))); - path = "https://example.org/bravo?q=a#fragment"; - CU_ASSERT("/bravo" == http2::get_pure_path_component(path)); - - path = "\x01\x02"; - CU_ASSERT("" == http2::get_pure_path_component(path)); + CU_ASSERT("" == + http2::get_pure_path_component(StringRef::from_lit("\x01\x02"))); } void test_http2_construct_push_component(void) { + BlockAllocator balloc(4096, 4096); StringRef base, uri; - std::string scheme, authority, path; + StringRef scheme, authority, path; base = StringRef::from_lit("/b/"); - uri = StringRef::from_lit("https://example.org/foo"); - CU_ASSERT( - 0 == http2::construct_push_component(scheme, authority, path, base, uri)); + CU_ASSERT(0 == http2::construct_push_component(balloc, scheme, authority, + path, base, uri)); CU_ASSERT("https" == scheme); CU_ASSERT("example.org" == authority); CU_ASSERT("/foo" == path); - scheme.clear(); - authority.clear(); - path.clear(); + scheme = StringRef{}; + authority = StringRef{}; + path = StringRef{}; uri = StringRef::from_lit("/foo/bar?q=a"); - CU_ASSERT( - 0 == http2::construct_push_component(scheme, authority, path, base, uri)); + CU_ASSERT(0 == http2::construct_push_component(balloc, scheme, authority, + path, base, uri)); CU_ASSERT("" == scheme); CU_ASSERT("" == authority); CU_ASSERT("/foo/bar?q=a" == path); - scheme.clear(); - authority.clear(); - path.clear(); + scheme = StringRef{}; + authority = StringRef{}; + path = StringRef{}; uri = StringRef::from_lit("foo/../bar?q=a"); - CU_ASSERT( - 0 == http2::construct_push_component(scheme, authority, path, base, uri)); + CU_ASSERT(0 == http2::construct_push_component(balloc, scheme, authority, + path, base, uri)); CU_ASSERT("" == scheme); CU_ASSERT("" == authority); CU_ASSERT("/b/bar?q=a" == path); - scheme.clear(); - authority.clear(); - path.clear(); + scheme = StringRef{}; + authority = StringRef{}; + path = StringRef{}; uri = StringRef{}; - CU_ASSERT( - 0 == http2::construct_push_component(scheme, authority, path, base, uri)); + CU_ASSERT(0 == http2::construct_push_component(balloc, scheme, authority, + path, base, uri)); CU_ASSERT("" == scheme); CU_ASSERT("" == authority); CU_ASSERT("/" == path); - scheme.clear(); - authority.clear(); - path.clear(); + scheme = StringRef{}; + authority = StringRef{}; + path = StringRef{}; uri = StringRef::from_lit("?q=a"); - CU_ASSERT( - 0 == http2::construct_push_component(scheme, authority, path, base, uri)); + CU_ASSERT(0 == http2::construct_push_component(balloc, scheme, authority, + path, base, uri)); CU_ASSERT("" == scheme); CU_ASSERT("" == authority); CU_ASSERT("/b/?q=a" == path); diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index bebc0f28..8cd75690 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -142,6 +142,7 @@ int main(int argc, char *argv[]) { !CU_add_test(pSuite, "util_select_h2", shrpx::test_util_select_h2) || !CU_add_test(pSuite, "util_ipv6_numeric_addr", shrpx::test_util_ipv6_numeric_addr) || + !CU_add_test(pSuite, "util_utos", shrpx::test_util_utos) || !CU_add_test(pSuite, "util_utos_unit", shrpx::test_util_utos_unit) || !CU_add_test(pSuite, "util_utos_funit", shrpx::test_util_utos_funit) || !CU_add_test(pSuite, "util_parse_uint_with_unit", diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 1046fdf0..a2ad7c54 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -624,7 +624,8 @@ int parse_mapping(const DownstreamAddrConfig &addr, } else { pattern.assign(std::begin(raw_pattern), slash); util::inp_strlower(pattern); - pattern += http2::normalize_path(slash, std::end(raw_pattern)); + pattern += http2::normalize_path(StringRef{slash, std::end(raw_pattern)}, + StringRef{}); } for (auto &g : addr_groups) { if (g.pattern == pattern) { diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index f5e4f084..eadfed39 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -257,8 +257,7 @@ std::string Downstream::assemble_request_cookie() const { std::string cookie; cookie = ""; for (auto &kv : req_.fs.headers()) { - if (kv.name.size() != 6 || kv.name[5] != 'e' || - !util::streq_l("cooki", kv.name.c_str(), 5)) { + if (kv.token != http2::HD_COOKIE) { continue; } @@ -273,7 +272,7 @@ std::string Downstream::assemble_request_cookie() const { if (c == ' ' || c == ';') { continue; } - end = it + 1; + end = it; break; } diff --git a/src/shrpx_downstream_test.cc b/src/shrpx_downstream_test.cc index d9989a72..5bfcda67 100644 --- a/src/shrpx_downstream_test.cc +++ b/src/shrpx_downstream_test.cc @@ -33,7 +33,8 @@ namespace shrpx { void test_downstream_field_store_add_header_lower(void) { - FieldStore fs(0); + BlockAllocator balloc(4096, 4096); + FieldStore fs(balloc, 0); fs.add_header_lower(StringRef::from_lit("1"), StringRef::from_lit("0"), false); fs.add_header_lower(StringRef::from_lit("2"), StringRef::from_lit("1"), @@ -51,19 +52,21 @@ void test_downstream_field_store_add_header_lower(void) { fs.add_header_lower(StringRef::from_lit(":authority"), StringRef::from_lit("7"), false); - auto ans = Headers{{"1", "0"}, - {"2", "1"}, - {"charlie", "2"}, - {"alpha", "3"}, - {"delta", "4"}, - {"bravo", "5"}, - {":method", "6"}, - {":authority", "7"}}; + auto ans = + HeaderRefs{{StringRef::from_lit("1"), StringRef::from_lit("0")}, + {StringRef::from_lit("2"), StringRef::from_lit("1")}, + {StringRef::from_lit("charlie"), StringRef::from_lit("2")}, + {StringRef::from_lit("alpha"), StringRef::from_lit("3")}, + {StringRef::from_lit("delta"), StringRef::from_lit("4")}, + {StringRef::from_lit("bravo"), StringRef::from_lit("5")}, + {StringRef::from_lit(":method"), StringRef::from_lit("6")}, + {StringRef::from_lit(":authority"), StringRef::from_lit("7")}}; CU_ASSERT(ans == fs.headers()); } void test_downstream_field_store_header(void) { - FieldStore fs(0); + BlockAllocator balloc(4096, 4096); + FieldStore fs(balloc, 0); fs.add_header_token(StringRef::from_lit("alpha"), StringRef::from_lit("0"), false, -1); fs.add_header_token(StringRef::from_lit(":authority"), @@ -73,11 +76,13 @@ void test_downstream_field_store_header(void) { http2::HD_CONTENT_LENGTH); // By token - CU_ASSERT(Header(":authority", "1") == *fs.header(http2::HD__AUTHORITY)); + CU_ASSERT(HeaderRef(StringRef{":authority"}, StringRef{"1"}) == + *fs.header(http2::HD__AUTHORITY)); CU_ASSERT(nullptr == fs.header(http2::HD__METHOD)); // By name - CU_ASSERT(Header("alpha", "0") == *fs.header(StringRef::from_lit("alpha"))); + CU_ASSERT(HeaderRef(StringRef{"alpha"}, StringRef{"0"}) == + *fs.header(StringRef::from_lit("alpha"))); CU_ASSERT(nullptr == fs.header(StringRef::from_lit("bravo"))); } @@ -105,19 +110,20 @@ void test_downstream_crumble_request_cookie(void) { CU_ASSERT(5 == nva.size()); CU_ASSERT(5 == num_cookies); - Headers cookies; + HeaderRefs cookies; std::transform(std::begin(nva), std::end(nva), std::back_inserter(cookies), [](const nghttp2_nv &nv) { - return Header(std::string(nv.name, nv.name + nv.namelen), - std::string(nv.value, nv.value + nv.valuelen), - nv.flags & NGHTTP2_NV_FLAG_NO_INDEX); + return HeaderRef(StringRef{nv.name, nv.namelen}, + StringRef{nv.value, nv.valuelen}, + nv.flags & NGHTTP2_NV_FLAG_NO_INDEX); }); - Headers ans = {{"cookie", "alpha"}, - {"cookie", "bravo"}, - {"cookie", "charlie"}, - {"cookie", "delta"}, - {"cookie", "echo"}}; + HeaderRefs ans = { + {StringRef::from_lit("cookie"), StringRef::from_lit("alpha")}, + {StringRef::from_lit("cookie"), StringRef::from_lit("bravo")}, + {StringRef::from_lit("cookie"), StringRef::from_lit("charlie")}, + {StringRef::from_lit("cookie"), StringRef::from_lit("delta")}, + {StringRef::from_lit("cookie"), StringRef::from_lit("echo")}}; CU_ASSERT(ans == cookies); CU_ASSERT(cookies[0].no_index); @@ -128,6 +134,7 @@ void test_downstream_crumble_request_cookie(void) { void test_downstream_assemble_request_cookie(void) { Downstream d(nullptr, nullptr, 0); auto &req = d.request(); + req.fs.add_header_token(StringRef::from_lit(":method"), StringRef::from_lit("get"), false, -1); req.fs.add_header_token(StringRef::from_lit(":path"), @@ -151,12 +158,12 @@ void test_downstream_rewrite_location_response_header(void) { Downstream d(nullptr, nullptr, 0); auto &req = d.request(); auto &resp = d.response(); - d.set_request_downstream_host("localhost2"); - req.authority = "localhost:8443"; + d.set_request_downstream_host(StringRef::from_lit("localhost2")); + req.authority = StringRef::from_lit("localhost:8443"); resp.fs.add_header_token(StringRef::from_lit("location"), StringRef::from_lit("http://localhost2:3000/"), false, http2::HD_LOCATION); - d.rewrite_location_response_header("https"); + d.rewrite_location_response_header(StringRef::from_lit("https")); auto location = resp.fs.header(http2::HD_LOCATION); CU_ASSERT("https://localhost:8443/" == (*location).value); } diff --git a/src/util.h b/src/util.h index f24f8140..90f03f1f 100644 --- a/src/util.h +++ b/src/util.h @@ -398,10 +398,11 @@ template OutputIt utos(OutputIt dst, T n) { ; --i; auto p = dst + i; + auto res = p + 1; for (; n; --i, n /= 10) { *p-- = (n % 10) + '0'; } - return dst + i + 1; + return res; } template std::string utos_unit(T n) { diff --git a/src/util_test.cc b/src/util_test.cc index cd8e2b02..d99c3a96 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -234,6 +234,15 @@ void test_util_ipv6_numeric_addr(void) { CU_ASSERT(!util::ipv6_numeric_addr("localhost")); } +void test_util_utos(void) { + uint8_t buf[32]; + + CU_ASSERT(("0" == StringRef{buf, util::utos(buf, 0)})); + CU_ASSERT(("123" == StringRef{buf, util::utos(buf, 123)})); + CU_ASSERT(("9223372036854775808" == + StringRef{buf, util::utos(buf, 9223372036854775808ULL)})); +} + void test_util_utos_unit(void) { CU_ASSERT("0" == util::utos_unit(0)); CU_ASSERT("1023" == util::utos_unit(1023)); diff --git a/src/util_test.h b/src/util_test.h index 1b79bbae..f51692aa 100644 --- a/src/util_test.h +++ b/src/util_test.h @@ -44,6 +44,7 @@ void test_util_utox(void); void test_util_http_date(void); void test_util_select_h2(void); void test_util_ipv6_numeric_addr(void); +void test_util_utos(void); void test_util_utos_unit(void); void test_util_utos_funit(void); void test_util_parse_uint_with_unit(void); From eb393985b78c7eca4f948c1ad0ebf76930680cbc Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 11 Mar 2016 00:50:27 +0900 Subject: [PATCH 05/19] nghttpx: Make a copy before adding header to Downstream --- src/http2.cc | 9 +++++ src/http2.h | 3 ++ src/shrpx-unittest.cc | 4 +- src/shrpx_downstream.cc | 49 +++++++------------------ src/shrpx_downstream.h | 4 -- src/shrpx_downstream_test.cc | 39 +++++++------------- src/shrpx_downstream_test.h | 2 +- src/shrpx_http2_session.cc | 20 ++++++---- src/shrpx_http2_upstream.cc | 32 +++++++++------- src/shrpx_http_downstream_connection.cc | 9 ++++- src/shrpx_https_upstream.cc | 9 ++++- src/shrpx_mruby_module_request.cc | 20 ++++++---- src/shrpx_mruby_module_response.cc | 24 +++++++----- src/shrpx_spdy_upstream.cc | 4 +- 14 files changed, 118 insertions(+), 110 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index 3a4474b8..8385cafe 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -1793,6 +1793,15 @@ StringRef rewrite_clean_path(BlockAllocator &balloc, const StringRef &src) { StringRef{query, fragment}); } +StringRef copy_lower(BlockAllocator &balloc, const StringRef &src) { + auto iov = make_byte_ref(balloc, src.size() + 1); + auto p = iov.base; + p = std::copy(std::begin(src), std::end(src), p); + *p = '\0'; + util::inp_strlower(iov.base, p); + return StringRef{iov.base, p}; +} + } // namespace http2 } // namespace nghttp2 diff --git a/src/http2.h b/src/http2.h index cc295079..48d4916d 100644 --- a/src/http2.h +++ b/src/http2.h @@ -385,6 +385,9 @@ int construct_push_component(BlockAllocator &balloc, StringRef &scheme, StringRef &authority, StringRef &path, const StringRef &base, const StringRef &uri); +// Copies |src| and return its lower-cased version. +StringRef copy_lower(BlockAllocator &balloc, const StringRef &src); + } // namespace http2 } // namespace nghttp2 diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index 8cd75690..0f2e2fd2 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -101,8 +101,8 @@ int main(int argc, char *argv[]) { shrpx::test_http2_get_pure_path_component) || !CU_add_test(pSuite, "http2_construct_push_component", shrpx::test_http2_construct_push_component) || - !CU_add_test(pSuite, "downstream_field_store_add_header_lower", - shrpx::test_downstream_field_store_add_header_lower) || + !CU_add_test(pSuite, "downstream_field_store_append_last_header", + shrpx::test_downstream_field_store_append_last_header) || !CU_add_test(pSuite, "downstream_field_store_header", shrpx::test_downstream_field_store_header) || !CU_add_test(pSuite, "downstream_crumble_request_cookie", diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index eadfed39..51931aad 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -348,23 +348,21 @@ void add_header(bool &key_prev, size_t &sum, HeaderRefs &headers, } } // namespace -namespace { -void add_header(size_t &sum, HeaderRefs &headers, const StringRef &name, - const StringRef &value, bool no_index, int32_t token) { - sum += name.size() + value.size(); - headers.emplace_back(name, value, no_index, token); -} -} // namespace - namespace { void append_last_header_key(BlockAllocator &balloc, bool &key_prev, size_t &sum, HeaderRefs &headers, const char *data, size_t len) { assert(key_prev); sum += len; auto &item = headers.back(); - item.name = concat_string_ref(balloc, item.name, StringRef{data, len}); - util::inp_strlower((uint8_t *)item.name.c_str(), - (uint8_t *)item.name.c_str() + item.name.size()); + auto iov = make_byte_ref(balloc, item.name.size() + len + 1); + auto p = iov.base; + p = std::copy(std::begin(item.name), std::end(item.name), p); + p = std::copy_n(data, len, p); + util::inp_strlower(p - len, p); + *p = '\0'; + + item.name = StringRef{iov.base, p}; + item.token = http2::lookup_token(item.name); } } // namespace @@ -424,21 +422,10 @@ const HeaderRefs::value_type *FieldStore::header(const StringRef &name) const { return search_header_linear_backwards(headers_, name); } -void FieldStore::add_header_lower(const StringRef &name, const StringRef &value, - bool no_index) { - auto low_name = make_string_ref(balloc_, name); - util::inp_strlower((uint8_t *)low_name.c_str(), - (uint8_t *)low_name.c_str() + low_name.size()); - auto token = http2::lookup_token(low_name); - shrpx::add_header(header_key_prev_, buffer_size_, headers_, low_name, - make_string_ref(balloc_, value), no_index, token); -} - void FieldStore::add_header_token(const StringRef &name, const StringRef &value, bool no_index, int32_t token) { - shrpx::add_header(buffer_size_, headers_, make_string_ref(balloc_, name), - make_string_ref(balloc_, value), no_index, token); - make_string_ref(balloc_, name); + shrpx::add_header(header_key_prev_, buffer_size_, headers_, name, value, + no_index, token); } void FieldStore::append_last_header_key(const char *data, size_t len) { @@ -453,23 +440,13 @@ void FieldStore::append_last_header_value(const char *data, size_t len) { void FieldStore::clear_headers() { headers_.clear(); } -void FieldStore::add_trailer_lower(const StringRef &name, - const StringRef &value, bool no_index) { - auto low_name = make_string_ref(balloc_, name); - util::inp_strlower((uint8_t *)low_name.c_str(), - (uint8_t *)low_name.c_str() + low_name.size()); - auto token = http2::lookup_token(low_name); - shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, low_name, - make_string_ref(balloc_, value), no_index, token); -} - void FieldStore::add_trailer_token(const StringRef &name, const StringRef &value, bool no_index, int32_t token) { // Header size limit should be applied to all header and trailer // fields combined. - shrpx::add_header(buffer_size_, trailers_, make_string_ref(balloc_, name), - make_string_ref(balloc_, value), no_index, token); + shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, name, value, + no_index, token); } void FieldStore::append_last_trailer_key(const char *data, size_t len) { diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index cea17d86..a5024169 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -81,8 +81,6 @@ public: // such header is found, returns nullptr. const HeaderRefs::value_type *header(const StringRef &name) const; - void add_header_lower(const StringRef &name, const StringRef &value, - bool no_index); void add_header_token(const StringRef &name, const StringRef &value, bool no_index, int32_t token); @@ -98,8 +96,6 @@ public: // Empties headers. void clear_headers(); - void add_trailer_lower(const StringRef &name, const StringRef &value, - bool no_index); void add_trailer_token(const StringRef &name, const StringRef &value, bool no_index, int32_t token); diff --git a/src/shrpx_downstream_test.cc b/src/shrpx_downstream_test.cc index 5bfcda67..786e9046 100644 --- a/src/shrpx_downstream_test.cc +++ b/src/shrpx_downstream_test.cc @@ -32,35 +32,22 @@ namespace shrpx { -void test_downstream_field_store_add_header_lower(void) { +void test_downstream_field_store_append_last_header(void) { BlockAllocator balloc(4096, 4096); FieldStore fs(balloc, 0); - fs.add_header_lower(StringRef::from_lit("1"), StringRef::from_lit("0"), - false); - fs.add_header_lower(StringRef::from_lit("2"), StringRef::from_lit("1"), - false); - fs.add_header_lower(StringRef::from_lit("Charlie"), StringRef::from_lit("2"), - false); - fs.add_header_lower(StringRef::from_lit("Alpha"), StringRef::from_lit("3"), - false); - fs.add_header_lower(StringRef::from_lit("Delta"), StringRef::from_lit("4"), - false); - fs.add_header_lower(StringRef::from_lit("BravO"), StringRef::from_lit("5"), - false); - fs.add_header_lower(StringRef::from_lit(":method"), StringRef::from_lit("6"), - false); - fs.add_header_lower(StringRef::from_lit(":authority"), - StringRef::from_lit("7"), false); + fs.add_header_token(StringRef::from_lit("alpha"), StringRef{}, false, -1); + auto bravo = StringRef::from_lit("BRAVO"); + fs.append_last_header_key(bravo.c_str(), bravo.size()); + auto charlie = StringRef::from_lit("Charlie"); + fs.append_last_header_value(charlie.c_str(), charlie.size()); + auto delta = StringRef::from_lit("deltA"); + fs.append_last_header_value(delta.c_str(), delta.size()); + fs.add_header_token(StringRef::from_lit("echo"), + StringRef::from_lit("foxtrot"), false, -1); - auto ans = - HeaderRefs{{StringRef::from_lit("1"), StringRef::from_lit("0")}, - {StringRef::from_lit("2"), StringRef::from_lit("1")}, - {StringRef::from_lit("charlie"), StringRef::from_lit("2")}, - {StringRef::from_lit("alpha"), StringRef::from_lit("3")}, - {StringRef::from_lit("delta"), StringRef::from_lit("4")}, - {StringRef::from_lit("bravo"), StringRef::from_lit("5")}, - {StringRef::from_lit(":method"), StringRef::from_lit("6")}, - {StringRef::from_lit(":authority"), StringRef::from_lit("7")}}; + auto ans = HeaderRefs{ + {StringRef::from_lit("alphabravo"), StringRef::from_lit("CharliedeltA")}, + {StringRef::from_lit("echo"), StringRef::from_lit("foxtrot")}}; CU_ASSERT(ans == fs.headers()); } diff --git a/src/shrpx_downstream_test.h b/src/shrpx_downstream_test.h index bcae1887..f7b1e603 100644 --- a/src/shrpx_downstream_test.h +++ b/src/shrpx_downstream_test.h @@ -31,7 +31,7 @@ namespace shrpx { -void test_downstream_field_store_add_header_lower(void); +void test_downstream_field_store_append_last_header(void); void test_downstream_field_store_header(void); void test_downstream_crumble_request_cookie(void); void test_downstream_assemble_request_cookie(void); diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 828d888c..25a7f996 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -818,6 +818,7 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, auto &resp = downstream->response(); auto &httpconf = get_config()->http; + auto &balloc = downstream->get_block_allocator(); switch (frame->hd.type) { case NGHTTP2_HEADERS: { @@ -847,13 +848,15 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, if (trailer) { // just store header fields for trailer part - resp.fs.add_trailer_token(StringRef{name, namelen}, - StringRef{value, valuelen}, no_index, token); + resp.fs.add_trailer_token( + make_string_ref(balloc, StringRef{name, namelen}), + make_string_ref(balloc, StringRef{value, valuelen}), no_index, token); return 0; } - resp.fs.add_header_token(StringRef{name, namelen}, - StringRef{value, valuelen}, no_index, token); + resp.fs.add_header_token( + make_string_ref(balloc, StringRef{name, namelen}), + make_string_ref(balloc, StringRef{value, valuelen}), no_index, token); return 0; } case NGHTTP2_PUSH_PROMISE: { @@ -870,6 +873,7 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, assert(promised_downstream); auto &promised_req = promised_downstream->request(); + auto &promised_balloc = promised_downstream->get_block_allocator(); // We use request header limit for PUSH_PROMISE if (promised_req.fs.buffer_size() + namelen + valuelen > @@ -886,9 +890,11 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, } auto token = http2::lookup_token(name, namelen); - promised_req.fs.add_header_token(StringRef{name, namelen}, - StringRef{value, valuelen}, - flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + promised_req.fs.add_header_token( + make_string_ref(promised_balloc, StringRef{name, namelen}), + make_string_ref(promised_balloc, StringRef{value, valuelen}), + flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + return 0; } } diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 5bfdb826..a3c78572 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -176,6 +176,8 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, auto &httpconf = get_config()->http; + auto &balloc = downstream->get_block_allocator(); + if (req.fs.buffer_size() + namelen + valuelen > httpconf.request_header_field_buffer || req.fs.num_fields() >= httpconf.max_request_header_fields) { @@ -206,12 +208,14 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, if (frame->headers.cat == NGHTTP2_HCAT_HEADERS) { // just store header fields for trailer part - req.fs.add_trailer_token(StringRef{name, namelen}, - StringRef{value, valuelen}, no_index, token); + req.fs.add_trailer_token( + make_string_ref(balloc, StringRef{name, namelen}), + make_string_ref(balloc, StringRef{value, valuelen}), no_index, token); return 0; } - req.fs.add_header_token(StringRef{name, namelen}, StringRef{value, valuelen}, + req.fs.add_header_token(make_string_ref(balloc, StringRef{name, namelen}), + make_string_ref(balloc, StringRef{value, valuelen}), no_index, token); return 0; } @@ -588,27 +592,29 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame, for (size_t i = 0; i < frame->push_promise.nvlen; ++i) { auto &nv = frame->push_promise.nva[i]; + + auto name = + make_string_ref(promised_balloc, StringRef{nv.name, nv.namelen}); + auto value = + make_string_ref(promised_balloc, StringRef{nv.value, nv.valuelen}); + auto token = http2::lookup_token(nv.name, nv.namelen); switch (token) { case http2::HD__METHOD: - req.method = http2::lookup_method_token(nv.value, nv.valuelen); + req.method = http2::lookup_method_token(value); break; case http2::HD__SCHEME: - req.scheme = - make_string_ref(promised_balloc, StringRef{nv.value, nv.valuelen}); + req.scheme = value; break; case http2::HD__AUTHORITY: - req.authority = - make_string_ref(promised_balloc, StringRef{nv.value, nv.valuelen}); + req.authority = value; break; case http2::HD__PATH: - req.path = http2::rewrite_clean_path(promised_balloc, - StringRef{nv.value, nv.valuelen}); + req.path = http2::rewrite_clean_path(promised_balloc, value); break; } - req.fs.add_header_token(StringRef{nv.name, nv.namelen}, - StringRef{nv.value, nv.valuelen}, - nv.flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + req.fs.add_header_token(name, value, nv.flags & NGHTTP2_NV_FLAG_NO_INDEX, + token); } promised_downstream->inspect_http2_request(); diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index d7f5492b..5f4f9468 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -690,6 +690,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { auto downstream = static_cast(htp->data); auto &resp = downstream->response(); auto &httpconf = get_config()->http; + auto &balloc = downstream->get_block_allocator(); if (ensure_header_field_buffer(downstream, httpconf, len) != 0) { return -1; @@ -702,7 +703,9 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { if (ensure_max_header_fields(downstream, httpconf) != 0) { return -1; } - resp.fs.add_header_lower(StringRef{data, len}, StringRef{}, false); + auto name = http2::copy_lower(balloc, StringRef{data, len}); + auto token = http2::lookup_token(name); + resp.fs.add_header_token(name, StringRef{}, false, token); } } else { // trailer part @@ -715,7 +718,9 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { // wrong place or crash if trailer fields are currently empty. return -1; } - resp.fs.add_trailer_lower(StringRef(data, len), StringRef{}, false); + auto name = http2::copy_lower(balloc, StringRef{data, len}); + auto token = http2::lookup_token(name); + resp.fs.add_trailer_token(name, StringRef{}, false, token); } } return 0; diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 73e91a67..dd028190 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -121,6 +121,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { auto downstream = upstream->get_downstream(); auto &req = downstream->request(); auto &httpconf = get_config()->http; + auto &balloc = downstream->get_block_allocator(); if (req.fs.buffer_size() + len > httpconf.request_header_field_buffer) { if (LOG_ENABLED(INFO)) { @@ -145,7 +146,9 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { Downstream::HTTP1_REQUEST_HEADER_TOO_LARGE); return -1; } - req.fs.add_header_lower(StringRef{data, len}, StringRef{}, false); + auto name = http2::copy_lower(balloc, StringRef{data, len}); + auto token = http2::lookup_token(name); + req.fs.add_header_token(name, StringRef{}, false, token); } } else { // trailer part @@ -159,7 +162,9 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { } return -1; } - req.fs.add_trailer_lower(StringRef{data, len}, StringRef{}, false); + auto name = http2::copy_lower(balloc, StringRef{data, len}); + auto token = http2::lookup_token(name); + req.fs.add_trailer_token(name, StringRef{}, false, token); } } return 0; diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index bf83eba2..f98442a4 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -213,6 +213,7 @@ mrb_value request_mod_header(mrb_state *mrb, mrb_value self, bool repl) { auto data = static_cast(mrb->ud); auto downstream = data->downstream; auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); check_phase(mrb, data->phase, PHASE_REQUEST); @@ -226,7 +227,8 @@ mrb_value request_mod_header(mrb_state *mrb, mrb_value self, bool repl) { key = mrb_funcall(mrb, key, "downcase", 0); auto keyref = - StringRef{RSTRING_PTR(key), static_cast(RSTRING_LEN(key))}; + make_string_ref(balloc, StringRef{RSTRING_PTR(key), + static_cast(RSTRING_LEN(key))}); auto token = http2::lookup_token(keyref.byte(), keyref.size()); if (repl) { @@ -249,15 +251,19 @@ mrb_value request_mod_header(mrb_state *mrb, mrb_value self, bool repl) { for (int i = 0; i < n; ++i) { auto value = mrb_ary_entry(values, i); req.fs.add_header_token( - keyref, StringRef{RSTRING_PTR(value), - static_cast(RSTRING_LEN(value))}, + keyref, + make_string_ref(balloc, + StringRef{RSTRING_PTR(value), + static_cast(RSTRING_LEN(value))}), false, token); } } else if (!mrb_nil_p(values)) { - req.fs.add_header_token(keyref, - StringRef{RSTRING_PTR(values), - static_cast(RSTRING_LEN(values))}, - false, token); + req.fs.add_header_token( + keyref, + make_string_ref(balloc, + StringRef{RSTRING_PTR(values), + static_cast(RSTRING_LEN(values))}), + false, token); } return mrb_nil_value(); diff --git a/src/shrpx_mruby_module_response.cc b/src/shrpx_mruby_module_response.cc index ada76b7c..65fec390 100644 --- a/src/shrpx_mruby_module_response.cc +++ b/src/shrpx_mruby_module_response.cc @@ -107,6 +107,7 @@ mrb_value response_mod_header(mrb_state *mrb, mrb_value self, bool repl) { auto data = static_cast(mrb->ud); auto downstream = data->downstream; auto &resp = downstream->response(); + auto &balloc = downstream->get_block_allocator(); mrb_value key, values; mrb_get_args(mrb, "oo", &key, &values); @@ -118,7 +119,8 @@ mrb_value response_mod_header(mrb_state *mrb, mrb_value self, bool repl) { key = mrb_funcall(mrb, key, "downcase", 0); auto keyref = - StringRef{RSTRING_PTR(key), static_cast(RSTRING_LEN(key))}; + make_string_ref(balloc, StringRef{RSTRING_PTR(key), + static_cast(RSTRING_LEN(key))}); auto token = http2::lookup_token(keyref.byte(), keyref.size()); if (repl) { @@ -141,14 +143,18 @@ mrb_value response_mod_header(mrb_state *mrb, mrb_value self, bool repl) { for (int i = 0; i < n; ++i) { auto value = mrb_ary_entry(values, i); resp.fs.add_header_token( - keyref, StringRef{RSTRING_PTR(value), - static_cast(RSTRING_LEN(value))}, + keyref, + make_string_ref(balloc, + StringRef{RSTRING_PTR(value), + static_cast(RSTRING_LEN(value))}), false, token); } } else if (!mrb_nil_p(values)) { resp.fs.add_header_token( - keyref, StringRef{RSTRING_PTR(values), - static_cast(RSTRING_LEN(values))}, + keyref, + make_string_ref(balloc, + StringRef{RSTRING_PTR(values), + static_cast(RSTRING_LEN(values))}), false, token); } @@ -222,7 +228,6 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { if (cl) { cl->value = content_length; } else { - // TODO we don't have to make a copy here. resp.fs.add_header_token(StringRef::from_lit("content-length"), content_length, false, http2::HD_CONTENT_LENGTH); } @@ -232,9 +237,10 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { if (!date) { auto lgconf = log_config(); lgconf->update_tstamp(std::chrono::system_clock::now()); - resp.fs.add_header_token(StringRef::from_lit("date"), - StringRef{lgconf->time_http_str}, false, - http2::HD_DATE); + resp.fs.add_header_token( + StringRef::from_lit("date"), + make_string_ref(balloc, StringRef{lgconf->time_http_str}), false, + http2::HD_DATE); } auto upstream = downstream->get_upstream(); diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 89508ba8..4b48c93f 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -197,7 +197,9 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, auto name = StringRef{nv[i]}; auto value = StringRef{nv[i + 1]}; auto token = http2::lookup_token(name.byte(), name.size()); - req.fs.add_header_token(name, value, false, token); + req.fs.add_header_token(make_string_ref(balloc, StringRef{name}), + make_string_ref(balloc, StringRef{value}), false, + token); } if (req.fs.parse_content_length() != 0) { From 8da20975f9049b00bc27f407637815c7ee494be8 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 11 Mar 2016 01:52:55 +0900 Subject: [PATCH 06/19] Always allocate buffer for name, and value --- lib/nghttp2_buf.h | 2 +- lib/nghttp2_hd.c | 278 +++++++++++---------------------------- lib/nghttp2_hd.h | 18 +-- lib/nghttp2_hd_huffman.c | 29 +--- tests/nghttp2_hd_test.c | 13 +- 5 files changed, 97 insertions(+), 243 deletions(-) diff --git a/lib/nghttp2_buf.h b/lib/nghttp2_buf.h index 8977f2ea..7c43606e 100644 --- a/lib/nghttp2_buf.h +++ b/lib/nghttp2_buf.h @@ -73,7 +73,7 @@ typedef struct { /* * Initializes the |buf|. No memory is allocated in this function. Use - * nghttp2_buf_reserve() or nghttp2_buf_reserve2() to allocate memory. + * nghttp2_buf_reserve() to allocate memory. */ void nghttp2_buf_init(nghttp2_buf *buf); diff --git a/lib/nghttp2_hd.c b/lib/nghttp2_hd.c index 65a09a5b..c6c5b5f8 100644 --- a/lib/nghttp2_hd.c +++ b/lib/nghttp2_hd.c @@ -1099,30 +1099,24 @@ int nghttp2_hd_inflate_init(nghttp2_hd_inflater *inflater, nghttp2_mem *mem) { inflater->min_hd_table_bufsize_max = UINT32_MAX; inflater->ent_keep = NULL; - inflater->nv_keep = NULL; + inflater->nv_name_keep = NULL; + inflater->nv_value_keep = NULL; inflater->opcode = NGHTTP2_HD_OPCODE_NONE; inflater->state = NGHTTP2_HD_STATE_INFLATE_START; - rv = nghttp2_bufs_init3(&inflater->nvbufs, NGHTTP2_HD_MAX_NV / 8, 8, 1, 0, - mem); - - if (rv != 0) { - goto nvbufs_fail; - } + nghttp2_buf_init(&inflater->namebuf); + nghttp2_buf_init(&inflater->valuebuf); inflater->huffman_encoded = 0; inflater->index = 0; inflater->left = 0; inflater->shift = 0; - inflater->newnamelen = 0; inflater->index_required = 0; inflater->no_index = 0; return 0; -nvbufs_fail: - hd_context_free(&inflater->ctx); fail: return rv; } @@ -1139,8 +1133,11 @@ static void hd_inflate_keep_free(nghttp2_hd_inflater *inflater) { inflater->ent_keep = NULL; } - nghttp2_mem_free(mem, inflater->nv_keep); - inflater->nv_keep = NULL; + nghttp2_mem_free(mem, inflater->nv_value_keep); + inflater->nv_value_keep = NULL; + + nghttp2_mem_free(mem, inflater->nv_name_keep); + inflater->nv_name_keep = NULL; } void nghttp2_hd_deflate_free(nghttp2_hd_deflater *deflater) { @@ -1148,8 +1145,13 @@ void nghttp2_hd_deflate_free(nghttp2_hd_deflater *deflater) { } void nghttp2_hd_inflate_free(nghttp2_hd_inflater *inflater) { + nghttp2_mem *mem; + + mem = inflater->ctx.mem; + hd_inflate_keep_free(inflater); - nghttp2_bufs_free(&inflater->nvbufs); + nghttp2_buf_free(&inflater->valuebuf, mem); + nghttp2_buf_free(&inflater->namebuf, mem); hd_context_free(&inflater->ctx); } @@ -2036,11 +2038,9 @@ static ssize_t hd_inflate_read_len(nghttp2_hd_inflater *inflater, int *rfin, * Out of memory * NGHTTP2_ERR_HEADER_COMP * Huffman decoding failed - * NGHTTP2_ERR_BUFFER_ERROR - * Out of buffer space. */ static ssize_t hd_inflate_read_huff(nghttp2_hd_inflater *inflater, - nghttp2_bufs *bufs, uint8_t *in, + nghttp2_buf *buf, uint8_t *in, uint8_t *last) { ssize_t readlen; int final = 0; @@ -2048,7 +2048,7 @@ static ssize_t hd_inflate_read_huff(nghttp2_hd_inflater *inflater, last = in + inflater->left; final = 1; } - readlen = nghttp2_hd_huff_decode(&inflater->huff_decode_ctx, bufs, in, + readlen = nghttp2_hd_huff_decode(&inflater->huff_decode_ctx, buf, in, (size_t)(last - in), final); if (readlen < 0) { @@ -2070,17 +2070,13 @@ static ssize_t hd_inflate_read_huff(nghttp2_hd_inflater *inflater, * Out of memory * NGHTTP2_ERR_HEADER_COMP * Header decompression failed - * NGHTTP2_ERR_BUFFER_ERROR - * Out of buffer space. */ -static ssize_t hd_inflate_read(nghttp2_hd_inflater *inflater, - nghttp2_bufs *bufs, uint8_t *in, uint8_t *last) { - int rv; +static ssize_t hd_inflate_read(nghttp2_hd_inflater *inflater, nghttp2_buf *buf, + uint8_t *in, uint8_t *last) { size_t len = nghttp2_min((size_t)(last - in), inflater->left); - rv = nghttp2_bufs_add(bufs, in, len); - if (rv != 0) { - return rv; - } + + buf->last = nghttp2_cpymem(buf->last, in, len); + inflater->left -= len; return (ssize_t)len; } @@ -2105,112 +2101,6 @@ static int hd_inflate_commit_indexed(nghttp2_hd_inflater *inflater, return 0; } -static int hd_inflate_remove_bufs(nghttp2_hd_inflater *inflater, nghttp2_nv *nv, - int value_only) { - ssize_t rv; - size_t buflen; - uint8_t *buf; - nghttp2_buf *pbuf; - - if (inflater->index_required || - inflater->nvbufs.head != inflater->nvbufs.cur) { - - rv = nghttp2_bufs_remove(&inflater->nvbufs, &buf); - - if (rv < 0) { - return NGHTTP2_ERR_NOMEM; - } - - nghttp2_bufs_reset(&inflater->nvbufs); - - buflen = (size_t)rv; - - if (value_only) { - /* we don't use this value, so no need to NULL-terminate */ - nv->name = NULL; - nv->namelen = 0; - - nv->value = buf; - nv->valuelen = buflen - 1; - } else { - nv->name = buf; - nv->namelen = inflater->newnamelen; - - nv->value = buf + nv->namelen + 1; - nv->valuelen = buflen - nv->namelen - 2; - } - - return 0; - } - - /* If we are not going to store header in header table and - name/value are in first chunk, we just refer them from nv, - instead of mallocing another memory. */ - - pbuf = &inflater->nvbufs.head->buf; - - if (value_only) { - /* we don't use this value, so no need to NULL-terminate */ - nv->name = NULL; - nv->namelen = 0; - - nv->value = pbuf->pos; - nv->valuelen = nghttp2_buf_len(pbuf) - 1; - } else { - nv->name = pbuf->pos; - nv->namelen = inflater->newnamelen; - - nv->value = pbuf->pos + nv->namelen + 1; - nv->valuelen = nghttp2_buf_len(pbuf) - nv->namelen - 2; - } - - /* Resetting does not change the content of first buffer */ - nghttp2_bufs_reset(&inflater->nvbufs); - - return 0; -} - -static int hd_inflate_remove_bufs_with_name(nghttp2_hd_inflater *inflater, - nghttp2_nv *nv, - nghttp2_hd_entry *ent_name) { -#ifndef NDEBUG - size_t rv; -#endif - size_t buflen; - uint8_t *buf; - nghttp2_mem *mem; - - mem = inflater->ctx.mem; - - /* Allocate buffer including name in ent_name, plus terminating - NULL. */ - buflen = ent_name->nv.namelen + 1 + nghttp2_bufs_len(&inflater->nvbufs); - - buf = nghttp2_mem_malloc(mem, buflen); - if (buf == NULL) { - return NGHTTP2_ERR_NOMEM; - } - - /* Copy including terminal NULL */ - memcpy(buf, ent_name->nv.name, ent_name->nv.namelen + 1); -#ifndef NDEBUG - rv = -#endif - nghttp2_bufs_remove_copy(&inflater->nvbufs, - buf + ent_name->nv.namelen + 1); - assert(ent_name->nv.namelen + 1 + rv == buflen); - - nghttp2_bufs_reset(&inflater->nvbufs); - - nv->name = buf; - nv->namelen = ent_name->nv.namelen; - - nv->value = buf + nv->namelen + 1; - nv->valuelen = buflen - nv->namelen - 2; - - return 0; -} - /* * Finalize literal header representation - new name- reception. If * header is emitted, |*nv_out| is filled with that value and 0 is @@ -2224,16 +2114,18 @@ static int hd_inflate_remove_bufs_with_name(nghttp2_hd_inflater *inflater, */ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, nghttp2_nv *nv_out, int *token_out) { - int rv; nghttp2_nv nv; nghttp2_mem *mem; mem = inflater->ctx.mem; - rv = hd_inflate_remove_bufs(inflater, &nv, 0 /* name and value */); - if (rv != 0) { - return NGHTTP2_ERR_NOMEM; - } + nv.name = inflater->namebuf.pos; + nv.namelen = nghttp2_buf_len(&inflater->namebuf); + nv.value = inflater->valuebuf.pos; + nv.valuelen = nghttp2_buf_len(&inflater->valuebuf); + + nghttp2_buf_init(&inflater->valuebuf); + nghttp2_buf_init(&inflater->namebuf); if (inflater->no_index) { nv.flags = NGHTTP2_NV_FLAG_NO_INDEX; @@ -2245,10 +2137,8 @@ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, nghttp2_hd_entry *new_ent; uint8_t ent_flags; - /* nv->value points to the middle of the buffer pointed by - nv->name. So we just need to keep track of nv->name for memory - management. */ - ent_flags = NGHTTP2_HD_FLAG_NAME_ALLOC | NGHTTP2_HD_FLAG_NAME_GIFT; + ent_flags = NGHTTP2_HD_FLAG_NAME_ALLOC | NGHTTP2_HD_FLAG_NAME_GIFT | + NGHTTP2_HD_FLAG_VALUE_ALLOC | NGHTTP2_HD_FLAG_VALUE_GIFT; new_ent = add_hd_table_incremental(&inflater->ctx, &nv, lookup_token(nv.name, nv.namelen), @@ -2261,6 +2151,7 @@ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, return 0; } + nghttp2_mem_free(mem, nv.value); nghttp2_mem_free(mem, nv.name); return NGHTTP2_ERR_NOMEM; @@ -2268,9 +2159,8 @@ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, emit_literal_header(nv_out, token_out, &nv); - if (nv.name != inflater->nvbufs.head->buf.pos) { - inflater->nv_keep = nv.name; - } + inflater->nv_name_keep = nv.name; + inflater->nv_value_keep = nv.value; return 0; } @@ -2288,7 +2178,6 @@ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, */ static int hd_inflate_commit_indname(nghttp2_hd_inflater *inflater, nghttp2_nv *nv_out, int *token_out) { - int rv; nghttp2_nv nv; nghttp2_hd_entry *ent_name; nghttp2_mem *mem; @@ -2307,25 +2196,21 @@ static int hd_inflate_commit_indname(nghttp2_hd_inflater *inflater, nghttp2_hd_entry *new_ent; uint8_t ent_flags; - if (inflater->index < NGHTTP2_STATIC_TABLE_LENGTH) { - /* We don't copy name in static table */ - rv = hd_inflate_remove_bufs(inflater, &nv, 1 /* value only */); - if (rv != 0) { - return NGHTTP2_ERR_NOMEM; - } - nv.name = ent_name->nv.name; - nv.namelen = ent_name->nv.namelen; + ent_flags = NGHTTP2_HD_FLAG_VALUE_ALLOC | NGHTTP2_HD_FLAG_VALUE_GIFT; - ent_flags = NGHTTP2_HD_FLAG_VALUE_ALLOC | NGHTTP2_HD_FLAG_VALUE_GIFT; - } else { - rv = hd_inflate_remove_bufs_with_name(inflater, &nv, ent_name); - if (rv != 0) { - return NGHTTP2_ERR_NOMEM; - } - /* nv->name and nv->value are in the same buffer. */ - ent_flags = NGHTTP2_HD_FLAG_NAME_ALLOC | NGHTTP2_HD_FLAG_NAME_GIFT; + if (inflater->index >= NGHTTP2_STATIC_TABLE_LENGTH) { + /* We don't copy name in static table */ + ent_flags |= NGHTTP2_HD_FLAG_NAME_ALLOC; } + nv.name = ent_name->nv.name; + nv.namelen = ent_name->nv.namelen; + + nv.value = inflater->valuebuf.pos; + nv.valuelen = nghttp2_buf_len(&inflater->valuebuf); + + nghttp2_buf_init(&inflater->valuebuf); + new_ent = add_hd_table_incremental(&inflater->ctx, &nv, ent_name->token, ent_flags, NULL, 0); @@ -2339,28 +2224,21 @@ static int hd_inflate_commit_indname(nghttp2_hd_inflater *inflater, return 0; } - if (inflater->index < NGHTTP2_STATIC_TABLE_LENGTH) { - nghttp2_mem_free(mem, nv.value); - } else { - nghttp2_mem_free(mem, nv.name); - } + nghttp2_mem_free(mem, nv.value); return NGHTTP2_ERR_NOMEM; } - rv = hd_inflate_remove_bufs(inflater, &nv, 1 /* value only */); - if (rv != 0) { - return NGHTTP2_ERR_NOMEM; - } - nv.name = ent_name->nv.name; nv.namelen = ent_name->nv.namelen; + nv.value = inflater->valuebuf.pos; + nv.valuelen = nghttp2_buf_len(&inflater->valuebuf); + + nghttp2_buf_init(&inflater->valuebuf); emit_literal_header(nv_out, token_out, &nv); - if (nv.value != inflater->nvbufs.head->buf.pos) { - inflater->nv_keep = nv.value; - } + inflater->nv_value_keep = nv.value; return 0; } @@ -2383,6 +2261,9 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, uint8_t *last = in + inlen; int rfin = 0; int busy = 0; + nghttp2_mem *mem; + + mem = inflater->ctx.mem; if (inflater->ctx.bad) { return NGHTTP2_ERR_HEADER_COMP; @@ -2541,12 +2422,20 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, nghttp2_hd_huff_decode_context_init(&inflater->huff_decode_ctx); inflater->state = NGHTTP2_HD_STATE_NEWNAME_READ_NAMEHUFF; + rv = nghttp2_buf_reserve(&inflater->namebuf, inflater->left * 2 + 1, + mem); } else { inflater->state = NGHTTP2_HD_STATE_NEWNAME_READ_NAME; + rv = nghttp2_buf_reserve(&inflater->namebuf, inflater->left + 1, mem); } + + if (rv != 0) { + goto fail; + } + break; case NGHTTP2_HD_STATE_NEWNAME_READ_NAMEHUFF: - rv = hd_inflate_read_huff(inflater, &inflater->nvbufs, in, last); + rv = hd_inflate_read_huff(inflater, &inflater->namebuf, in, last); if (rv < 0) { goto fail; } @@ -2562,18 +2451,13 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, goto almost_ok; } - inflater->newnamelen = nghttp2_bufs_len(&inflater->nvbufs); - - rv = nghttp2_bufs_addb(&inflater->nvbufs, '\0'); - if (rv != 0) { - goto fail; - } + *inflater->namebuf.last = '\0'; inflater->state = NGHTTP2_HD_STATE_CHECK_VALUELEN; break; case NGHTTP2_HD_STATE_NEWNAME_READ_NAME: - rv = hd_inflate_read(inflater, &inflater->nvbufs, in, last); + rv = hd_inflate_read(inflater, &inflater->namebuf, in, last); if (rv < 0) { goto fail; } @@ -2588,12 +2472,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, goto almost_ok; } - inflater->newnamelen = nghttp2_bufs_len(&inflater->nvbufs); - - rv = nghttp2_bufs_addb(&inflater->nvbufs, '\0'); - if (rv != 0) { - goto fail; - } + *inflater->namebuf.last = '\0'; inflater->state = NGHTTP2_HD_STATE_CHECK_VALUELEN; @@ -2625,15 +2504,24 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, nghttp2_hd_huff_decode_context_init(&inflater->huff_decode_ctx); inflater->state = NGHTTP2_HD_STATE_READ_VALUEHUFF; + + rv = nghttp2_buf_reserve(&inflater->valuebuf, inflater->left * 2 + 1, + mem); } else { inflater->state = NGHTTP2_HD_STATE_READ_VALUE; + + rv = nghttp2_buf_reserve(&inflater->valuebuf, inflater->left + 1, mem); + } + + if (rv != 0) { + goto fail; } busy = 1; break; case NGHTTP2_HD_STATE_READ_VALUEHUFF: - rv = hd_inflate_read_huff(inflater, &inflater->nvbufs, in, last); + rv = hd_inflate_read_huff(inflater, &inflater->valuebuf, in, last); if (rv < 0) { goto fail; } @@ -2649,10 +2537,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, goto almost_ok; } - rv = nghttp2_bufs_addb(&inflater->nvbufs, '\0'); - if (rv != 0) { - goto fail; - } + *inflater->valuebuf.last = '\0'; if (inflater->opcode == NGHTTP2_HD_OPCODE_NEWNAME) { rv = hd_inflate_commit_newname(inflater, nv_out, token_out); @@ -2669,7 +2554,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, return (ssize_t)(in - first); case NGHTTP2_HD_STATE_READ_VALUE: - rv = hd_inflate_read(inflater, &inflater->nvbufs, in, last); + rv = hd_inflate_read(inflater, &inflater->valuebuf, in, last); if (rv < 0) { DEBUGF(fprintf(stderr, "inflatehd: value read failure %zd: %s\n", rv, nghttp2_strerror((int)rv))); @@ -2686,10 +2571,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, goto almost_ok; } - rv = nghttp2_bufs_addb(&inflater->nvbufs, '\0'); - if (rv != 0) { - goto fail; - } + *inflater->valuebuf.last = '\0'; if (inflater->opcode == NGHTTP2_HD_OPCODE_NEWNAME) { rv = hd_inflate_commit_newname(inflater, nv_out, token_out); diff --git a/lib/nghttp2_hd.h b/lib/nghttp2_hd.h index adf227be..fff4c5a5 100644 --- a/lib/nghttp2_hd.h +++ b/lib/nghttp2_hd.h @@ -276,7 +276,7 @@ struct nghttp2_hd_deflater { struct nghttp2_hd_inflater { nghttp2_hd_context ctx; /* header buffer */ - nghttp2_bufs nvbufs; + nghttp2_buf namebuf, valuebuf; /* Stores current state of huffman decoding */ nghttp2_hd_huff_decode_context huff_decode_ctx; /* Pointer to the nghttp2_hd_entry which is used current header @@ -285,14 +285,11 @@ struct nghttp2_hd_inflater { nghttp2_hd_entry *ent_keep; /* Pointer to the name/value pair buffer which is used in the current header emission. */ - uint8_t *nv_keep; + uint8_t *nv_name_keep, *nv_value_keep; /* The number of bytes to read */ size_t left; /* The index in indexed repr or indexed name */ size_t index; - /* The length of new name encoded in literal. For huffman encoded - string, this is the length after it is decoded. */ - size_t newnamelen; /* The maximum header table size the inflater supports. This is the same value transmitted in SETTINGS_HEADER_TABLE_SIZE */ size_t settings_hd_table_bufsize_max; @@ -470,11 +467,10 @@ int nghttp2_hd_huff_encode(nghttp2_bufs *bufs, const uint8_t *src, void nghttp2_hd_huff_decode_context_init(nghttp2_hd_huff_decode_context *ctx); /* - * Decodes the given data |src| with length |srclen|. The |ctx| must + * Decodes the given data |src| with length |srclen|. The |ctx| must * be initialized by nghttp2_hd_huff_decode_context_init(). The result - * will be added to |dest|. This function may expand |dest| as - * needed. The caller is responsible to release the memory of |dest| - * by calling nghttp2_bufs_free(). + * will be written to |buf|. This function assumes that |buf| has the + * enough room to store the decoded byte string. * * The caller must set the |final| to nonzero if the given input is * the final block. @@ -486,13 +482,11 @@ void nghttp2_hd_huff_decode_context_init(nghttp2_hd_huff_decode_context *ctx); * * NGHTTP2_ERR_NOMEM * Out of memory. - * NGHTTP2_ERR_BUFFER_ERROR - * Maximum buffer capacity size exceeded. * NGHTTP2_ERR_HEADER_COMP * Decoding process has failed. */ ssize_t nghttp2_hd_huff_decode(nghttp2_hd_huff_decode_context *ctx, - nghttp2_bufs *bufs, const uint8_t *src, + nghttp2_buf *buf, const uint8_t *src, size_t srclen, int final); #endif /* NGHTTP2_HD_H */ diff --git a/lib/nghttp2_hd_huffman.c b/lib/nghttp2_hd_huffman.c index 48638b75..3fb0d886 100644 --- a/lib/nghttp2_hd_huffman.c +++ b/lib/nghttp2_hd_huffman.c @@ -166,31 +166,10 @@ void nghttp2_hd_huff_decode_context_init(nghttp2_hd_huff_decode_context *ctx) { ctx->accept = 1; } -/* Use macro to make the code simpler..., but error case is tricky. - We spent most of the CPU in decoding, so we are doing this - thing. */ -#define hd_huff_decode_sym_emit(bufs, sym, avail) \ - do { \ - if ((avail)) { \ - nghttp2_bufs_fast_addb((bufs), (sym)); \ - --(avail); \ - } else { \ - rv = nghttp2_bufs_addb((bufs), (sym)); \ - if (rv != 0) { \ - return rv; \ - } \ - (avail) = nghttp2_bufs_cur_avail((bufs)); \ - } \ - } while (0) - ssize_t nghttp2_hd_huff_decode(nghttp2_hd_huff_decode_context *ctx, - nghttp2_bufs *bufs, const uint8_t *src, + nghttp2_buf *buf, const uint8_t *src, size_t srclen, int final) { size_t i; - int rv; - size_t avail; - - avail = nghttp2_bufs_cur_avail(bufs); /* We use the decoding algorithm described in http://graphics.ics.uci.edu/pub/Prefix.pdf */ @@ -202,8 +181,7 @@ ssize_t nghttp2_hd_huff_decode(nghttp2_hd_huff_decode_context *ctx, return NGHTTP2_ERR_HEADER_COMP; } if (t->flags & NGHTTP2_HUFF_SYM) { - /* this is macro, and may return from this function on error */ - hd_huff_decode_sym_emit(bufs, t->sym, avail); + *buf->last++ = t->sym; } t = &huff_decode_table[t->state][src[i] & 0xf]; @@ -211,8 +189,7 @@ ssize_t nghttp2_hd_huff_decode(nghttp2_hd_huff_decode_context *ctx, return NGHTTP2_ERR_HEADER_COMP; } if (t->flags & NGHTTP2_HUFF_SYM) { - /* this is macro, and may return from this function on error */ - hd_huff_decode_sym_emit(bufs, t->sym, avail); + *buf->last++ = t->sym; } ctx->state = t->state; diff --git a/tests/nghttp2_hd_test.c b/tests/nghttp2_hd_test.c index 5a4d7c5f..ed954bde 100644 --- a/tests/nghttp2_hd_test.c +++ b/tests/nghttp2_hd_test.c @@ -1352,13 +1352,15 @@ void test_nghttp2_hd_decode_length(void) { void test_nghttp2_hd_huff_encode(void) { int rv; ssize_t len; - nghttp2_bufs bufs, outbufs; + nghttp2_buf outbuf; + nghttp2_bufs bufs; nghttp2_hd_huff_decode_context ctx; const uint8_t t1[] = {22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; + uint8_t b[256]; + nghttp2_buf_wrap_init(&outbuf, b, sizeof(b)); frame_pack_bufs_init(&bufs); - frame_pack_bufs_init(&outbufs); rv = nghttp2_hd_huff_encode(&bufs, t1, sizeof(t1)); @@ -1366,14 +1368,13 @@ void test_nghttp2_hd_huff_encode(void) { nghttp2_hd_huff_decode_context_init(&ctx); - len = nghttp2_hd_huff_decode(&ctx, &outbufs, bufs.cur->buf.pos, + len = nghttp2_hd_huff_decode(&ctx, &outbuf, bufs.cur->buf.pos, nghttp2_bufs_len(&bufs), 1); CU_ASSERT((ssize_t)nghttp2_bufs_len(&bufs) == len); - CU_ASSERT((ssize_t)sizeof(t1) == nghttp2_bufs_len(&outbufs)); + CU_ASSERT((ssize_t)sizeof(t1) == nghttp2_buf_len(&outbuf)); - CU_ASSERT(0 == memcmp(t1, outbufs.cur->buf.pos, sizeof(t1))); + CU_ASSERT(0 == memcmp(t1, outbuf.pos, sizeof(t1))); nghttp2_bufs_free(&bufs); - nghttp2_bufs_free(&outbufs); } From ff0d137fb33e4302c6be010ec10030218a743efb Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 13:23:12 +0900 Subject: [PATCH 07/19] Reference counted HPACK name/value pair --- genlibtokenlookup.py | 2 +- lib/Makefile.am | 6 +- lib/includes/nghttp2/nghttp2.h | 47 +++ lib/nghttp2_hd.c | 518 ++++++++++++++------------------- lib/nghttp2_hd.h | 106 ++++--- lib/nghttp2_http.c | 87 +++--- lib/nghttp2_http.h | 5 +- lib/nghttp2_mem.c | 4 + lib/nghttp2_mem.h | 1 + lib/nghttp2_rcbuf.c | 98 +++++++ lib/nghttp2_rcbuf.h | 80 +++++ lib/nghttp2_session.c | 21 +- tests/nghttp2_hd_test.c | 12 +- 13 files changed, 557 insertions(+), 430 deletions(-) create mode 100644 lib/nghttp2_rcbuf.c create mode 100644 lib/nghttp2_rcbuf.h diff --git a/genlibtokenlookup.py b/genlibtokenlookup.py index 625e62d1..360d2198 100755 --- a/genlibtokenlookup.py +++ b/genlibtokenlookup.py @@ -162,7 +162,7 @@ def gen_enum(): def gen_index_header(): print '''\ -static inline int lookup_token(const uint8_t *name, size_t namelen) { +static inline int32_t lookup_token(const uint8_t *name, size_t namelen) { switch (namelen) {''' b = build_header(HEADERS) for size in sorted(b.keys()): diff --git a/lib/Makefile.am b/lib/Makefile.am index 3ea5bed2..17e86bfc 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -47,7 +47,8 @@ OBJECTS = nghttp2_pq.c nghttp2_map.c nghttp2_queue.c \ nghttp2_option.c \ nghttp2_callbacks.c \ nghttp2_mem.c \ - nghttp2_http.c + nghttp2_http.c \ + nghttp2_rcbuf.c HFILES = nghttp2_pq.h nghttp2_int.h nghttp2_map.h nghttp2_queue.h \ nghttp2_frame.h \ @@ -61,7 +62,8 @@ HFILES = nghttp2_pq.h nghttp2_int.h nghttp2_map.h nghttp2_queue.h \ nghttp2_option.h \ nghttp2_callbacks.h \ nghttp2_mem.h \ - nghttp2_http.h + nghttp2_http.h \ + nghttp2_rcbuf.h libnghttp2_la_SOURCES = $(HFILES) $(OBJECTS) libnghttp2_la_LDFLAGS = -no-undefined \ diff --git a/lib/includes/nghttp2/nghttp2.h b/lib/includes/nghttp2/nghttp2.h index f6133055..2a3674fb 100644 --- a/lib/includes/nghttp2/nghttp2.h +++ b/lib/includes/nghttp2/nghttp2.h @@ -419,6 +419,53 @@ typedef enum { NGHTTP2_ERR_FLOODED = -904 } nghttp2_error; +/** + * @struct + * + * The object representing single contagious buffer. + */ +typedef struct { + /** + * The pointer to the buffer. + */ + uint8_t *base; + /** + * The length of the buffer. + */ + size_t len; +} nghttp2_vec; + +struct nghttp2_rcbuf; + +/** + * @struct + * + * The object representing reference counted buffer. The details of + * this structure are intentionally hidden from the public API. + */ +typedef struct nghttp2_rcbuf nghttp2_rcbuf; + +/** + * @function + * + * Increments the reference count of |rcbuf| by 1. + */ +void nghttp2_rcbuf_incref(nghttp2_rcbuf *rcbuf); + +/** + * @function + * + * Decrements the reference count of |rcbuf| by 1. If the reference + * count becomes zero, the object pointed by |rcbuf| will be freed. + * In this case, application must not use |rcbuf| again. + */ +void nghttp2_rcbuf_decref(nghttp2_rcbuf *rcbuf); + +/** + * Returns the underlying buffer managed by |rcbuf|. + */ +nghttp2_vec nghttp2_rcbuf_get_buf(nghttp2_rcbuf *rcbuf); + /** * @enum * diff --git a/lib/nghttp2_hd.c b/lib/nghttp2_hd.c index c6c5b5f8..8da40064 100644 --- a/lib/nghttp2_hd.c +++ b/lib/nghttp2_hd.c @@ -34,15 +34,17 @@ /* Make scalar initialization form of nghttp2_hd_entry */ #define MAKE_STATIC_ENT(N, V, T, H) \ { \ - { (uint8_t *)(N), (uint8_t *)(V), sizeof((N)) - 1, sizeof((V)) - 1, 0 } \ - , NULL, 0, (H), (T), 1, NGHTTP2_HD_FLAG_NONE \ + { NULL, NULL, (uint8_t *)(N), sizeof((N)) - 1, -1 } \ + , {NULL, NULL, (uint8_t *)(V), sizeof((V)) - 1, -1}, \ + {(uint8_t *)(N), (uint8_t *)(V), sizeof((N)) - 1, sizeof((V)) - 1, 0}, \ + T, H \ } /* Generated by mkstatictbl.py */ /* 3rd parameter is nghttp2_token value for header field name. We use first enum value if same header names are repeated (e.g., :status). */ -static nghttp2_hd_entry static_table[] = { +static nghttp2_hd_static_entry static_table[] = { MAKE_STATIC_ENT(":authority", "", 0, 3153725150u), MAKE_STATIC_ENT(":method", "GET", 1, 695666056u), MAKE_STATIC_ENT(":method", "POST", 1, 695666056u), @@ -114,7 +116,7 @@ static int memeq(const void *s1, const void *s2, size_t n) { * This function was generated by genlibtokenlookup.py. Inspired by * h2o header lookup. https://github.com/h2o/h2o */ -static int lookup_token(const uint8_t *name, size_t namelen) { +static int32_t lookup_token(const uint8_t *name, size_t namelen) { switch (namelen) { case 2: switch (name[1]) { @@ -790,86 +792,33 @@ static int lookup_token(const uint8_t *name, size_t namelen) { return -1; } -int nghttp2_hd_entry_init(nghttp2_hd_entry *ent, uint8_t flags, uint8_t *name, - size_t namelen, uint8_t *value, size_t valuelen, - int token, nghttp2_mem *mem) { - int rv = 0; - - /* Since nghttp2_hd_entry is used for indexing, ent->nv.flags always - NGHTTP2_NV_FLAG_NONE */ - ent->nv.flags = NGHTTP2_NV_FLAG_NONE; - - if ((flags & NGHTTP2_HD_FLAG_NAME_ALLOC) && - (flags & NGHTTP2_HD_FLAG_NAME_GIFT) == 0) { - if (namelen == 0) { - flags = (uint8_t)(flags & ~NGHTTP2_HD_FLAG_NAME_ALLOC); - ent->nv.name = (uint8_t *)""; - } else { - /* name may not be NULL terminated on compression. */ - ent->nv.name = nghttp2_mem_malloc(mem, namelen + 1); - if (ent->nv.name == NULL) { - rv = NGHTTP2_ERR_NOMEM; - goto fail; - } - memcpy(ent->nv.name, name, namelen); - ent->nv.name[namelen] = '\0'; - } - } else { - ent->nv.name = name; - } - if ((flags & NGHTTP2_HD_FLAG_VALUE_ALLOC) && - (flags & NGHTTP2_HD_FLAG_VALUE_GIFT) == 0) { - if (valuelen == 0) { - flags = (uint8_t)(flags & ~NGHTTP2_HD_FLAG_VALUE_ALLOC); - ent->nv.value = (uint8_t *)""; - } else { - /* value may not be NULL terminated on compression. */ - ent->nv.value = nghttp2_mem_malloc(mem, valuelen + 1); - if (ent->nv.value == NULL) { - rv = NGHTTP2_ERR_NOMEM; - goto fail2; - } - memcpy(ent->nv.value, value, valuelen); - ent->nv.value[valuelen] = '\0'; - } - } else { - ent->nv.value = value; - } - ent->nv.namelen = namelen; - ent->nv.valuelen = valuelen; - ent->token = token; - ent->ref = 1; - ent->flags = flags; +void nghttp2_hd_entry_init(nghttp2_hd_entry *ent, nghttp2_hd_nv *nv) { + ent->nv = *nv; + ent->cnv.name = nv->name->base; + ent->cnv.namelen = nv->name->len; + ent->cnv.value = nv->value->base; + ent->cnv.valuelen = nv->value->len; + ent->cnv.flags = nv->flags; ent->next = NULL; ent->hash = 0; - return 0; - -fail2: - if ((flags & NGHTTP2_HD_FLAG_NAME_ALLOC) && - (flags & NGHTTP2_HD_FLAG_NAME_GIFT) == 0) { - nghttp2_mem_free(mem, ent->nv.name); - } -fail: - return rv; + nghttp2_rcbuf_incref(ent->nv.name); + nghttp2_rcbuf_incref(ent->nv.value); } -void nghttp2_hd_entry_free(nghttp2_hd_entry *ent, nghttp2_mem *mem) { - assert(ent->ref == 0); - if (ent->flags & NGHTTP2_HD_FLAG_NAME_ALLOC) { - nghttp2_mem_free(mem, ent->nv.name); - } - if (ent->flags & NGHTTP2_HD_FLAG_VALUE_ALLOC) { - nghttp2_mem_free(mem, ent->nv.value); - } +void nghttp2_hd_entry_free(nghttp2_hd_entry *ent) { + nghttp2_rcbuf_decref(ent->nv.value); + nghttp2_rcbuf_decref(ent->nv.name); } -static int name_eq(const nghttp2_nv *a, const nghttp2_nv *b) { - return a->namelen == b->namelen && memeq(a->name, b->name, a->namelen); +static int name_eq(const nghttp2_hd_nv *a, const nghttp2_nv *b) { + return a->name->len == b->namelen && + memeq(a->name->base, b->name, b->namelen); } -static int value_eq(const nghttp2_nv *a, const nghttp2_nv *b) { - return a->valuelen == b->valuelen && memeq(a->value, b->value, a->valuelen); +static int value_eq(const nghttp2_hd_nv *a, const nghttp2_nv *b) { + return a->value->len == b->valuelen && + memeq(a->value->base, b->value, b->valuelen); } static uint32_t name_hash(const nghttp2_nv *nv) { @@ -905,7 +854,7 @@ static void hd_map_insert(nghttp2_hd_map *map, nghttp2_hd_entry *ent) { } static nghttp2_hd_entry *hd_map_find(nghttp2_hd_map *map, int *exact_match, - const nghttp2_nv *nv, int token, + const nghttp2_nv *nv, int32_t token, uint32_t hash) { nghttp2_hd_entry *p; nghttp2_hd_entry *res = NULL; @@ -913,7 +862,7 @@ static nghttp2_hd_entry *hd_map_find(nghttp2_hd_map *map, int *exact_match, *exact_match = 0; for (p = map->table[hash & (HD_MAP_SIZE - 1)]; p; p = p->next) { - if (token != p->token || + if (token != p->nv.token || (token == -1 && (hash != p->hash || !name_eq(&p->nv, nv)))) { continue; } @@ -1008,8 +957,8 @@ static void hd_ringbuf_free(nghttp2_hd_ringbuf *ringbuf, nghttp2_mem *mem) { } for (i = 0; i < ringbuf->len; ++i) { nghttp2_hd_entry *ent = hd_ringbuf_get(ringbuf, i); - --ent->ref; - nghttp2_hd_entry_free(ent, mem); + + nghttp2_hd_entry_free(ent); nghttp2_mem_free(mem, ent); } nghttp2_mem_free(mem, ringbuf->buffer); @@ -1098,7 +1047,6 @@ int nghttp2_hd_inflate_init(nghttp2_hd_inflater *inflater, nghttp2_mem *mem) { inflater->settings_hd_table_bufsize_max = NGHTTP2_HD_DEFAULT_MAX_BUFFER_SIZE; inflater->min_hd_table_bufsize_max = UINT32_MAX; - inflater->ent_keep = NULL; inflater->nv_name_keep = NULL; inflater->nv_value_keep = NULL; @@ -1108,6 +1056,9 @@ int nghttp2_hd_inflate_init(nghttp2_hd_inflater *inflater, nghttp2_mem *mem) { nghttp2_buf_init(&inflater->namebuf); nghttp2_buf_init(&inflater->valuebuf); + inflater->namercbuf = NULL; + inflater->valuercbuf = NULL; + inflater->huffman_encoded = 0; inflater->index = 0; inflater->left = 0; @@ -1122,21 +1073,10 @@ fail: } static void hd_inflate_keep_free(nghttp2_hd_inflater *inflater) { - nghttp2_mem *mem; + nghttp2_rcbuf_decref(inflater->nv_value_keep); + nghttp2_rcbuf_decref(inflater->nv_name_keep); - mem = inflater->ctx.mem; - if (inflater->ent_keep) { - if (inflater->ent_keep->ref == 0) { - nghttp2_hd_entry_free(inflater->ent_keep, mem); - nghttp2_mem_free(mem, inflater->ent_keep); - } - inflater->ent_keep = NULL; - } - - nghttp2_mem_free(mem, inflater->nv_value_keep); inflater->nv_value_keep = NULL; - - nghttp2_mem_free(mem, inflater->nv_name_keep); inflater->nv_name_keep = NULL; } @@ -1145,13 +1085,11 @@ void nghttp2_hd_deflate_free(nghttp2_hd_deflater *deflater) { } void nghttp2_hd_inflate_free(nghttp2_hd_inflater *inflater) { - nghttp2_mem *mem; - - mem = inflater->ctx.mem; - hd_inflate_keep_free(inflater); - nghttp2_buf_free(&inflater->valuebuf, mem); - nghttp2_buf_free(&inflater->namebuf, mem); + + nghttp2_rcbuf_decref(inflater->valuercbuf); + nghttp2_rcbuf_decref(inflater->namercbuf); + hd_context_free(&inflater->ctx); } @@ -1159,23 +1097,13 @@ static size_t entry_room(size_t namelen, size_t valuelen) { return NGHTTP2_HD_ENTRY_OVERHEAD + namelen + valuelen; } -static int emit_indexed_header(nghttp2_nv *nv_out, int *token_out, - nghttp2_hd_entry *ent) { - DEBUGF(fprintf(stderr, "inflatehd: header emission: %s: %s\n", ent->nv.name, - ent->nv.value)); +static int emit_header(nghttp2_hd_nv *nv_out, nghttp2_hd_nv *nv) { + DEBUGF(fprintf(stderr, "inflatehd: header emission: %s: %s\n", nv->name->base, + nv->value->base)); /* ent->ref may be 0. This happens if the encoder emits literal block larger than header table capacity with indexing. */ - *nv_out = ent->nv; - *token_out = ent->token; - return 0; -} - -static int emit_literal_header(nghttp2_nv *nv_out, int *token_out, - nghttp2_nv *nv) { - DEBUGF(fprintf(stderr, "inflatehd: header emission: %s: %s\n", nv->name, - nv->value)); *nv_out = *nv; - *token_out = lookup_token(nv->name, nv->namelen); + return 0; } @@ -1483,17 +1411,16 @@ static int emit_newname_block(nghttp2_bufs *bufs, const nghttp2_nv *nv, return 0; } -static nghttp2_hd_entry * -add_hd_table_incremental(nghttp2_hd_context *context, const nghttp2_nv *nv, - int token, uint8_t entry_flags, nghttp2_hd_map *map, - uint32_t hash) { +static int add_hd_table_incremental(nghttp2_hd_context *context, + nghttp2_hd_nv *nv, nghttp2_hd_map *map, + uint32_t hash) { int rv; nghttp2_hd_entry *new_ent; size_t room; nghttp2_mem *mem; mem = context->mem; - room = entry_room(nv->namelen, nv->valuelen); + room = entry_room(nv->name->len, nv->value->len); while (context->hd_table_bufsize + room > context->hd_table_bufsize_max && context->hd_table.len > 0) { @@ -1501,72 +1428,53 @@ add_hd_table_incremental(nghttp2_hd_context *context, const nghttp2_nv *nv, size_t idx = context->hd_table.len - 1; nghttp2_hd_entry *ent = hd_ringbuf_get(&context->hd_table, idx); - context->hd_table_bufsize -= entry_room(ent->nv.namelen, ent->nv.valuelen); + context->hd_table_bufsize -= + entry_room(ent->nv.name->len, ent->nv.value->len); DEBUGF(fprintf(stderr, "hpack: remove item from header table: %s: %s\n", - ent->nv.name, ent->nv.value)); + (char *)ent->nv.name->base, (char *)ent->nv.value->base)); hd_ringbuf_pop_back(&context->hd_table); if (map) { hd_map_remove(map, ent); } - if (--ent->ref == 0) { - nghttp2_hd_entry_free(ent, mem); - nghttp2_mem_free(mem, ent); - } - } - new_ent = nghttp2_mem_malloc(mem, sizeof(nghttp2_hd_entry)); - if (new_ent == NULL) { - return NULL; - } - - rv = nghttp2_hd_entry_init(new_ent, entry_flags, nv->name, nv->namelen, - nv->value, nv->valuelen, token, mem); - if (rv != 0) { - nghttp2_mem_free(mem, new_ent); - return NULL; + nghttp2_hd_entry_free(ent); + nghttp2_mem_free(mem, ent); } if (room > context->hd_table_bufsize_max) { /* The entry taking more than NGHTTP2_HD_MAX_BUFFER_SIZE is - immediately evicted. */ - --new_ent->ref; - } else { - rv = hd_ringbuf_push_front(&context->hd_table, new_ent, mem); - - if (rv != 0) { - --new_ent->ref; - - if ((entry_flags & NGHTTP2_HD_FLAG_NAME_ALLOC) && - (entry_flags & NGHTTP2_HD_FLAG_NAME_GIFT)) { - /* nv->name are managed by caller. */ - new_ent->nv.name = NULL; - new_ent->nv.namelen = 0; - } - if ((entry_flags & NGHTTP2_HD_FLAG_VALUE_ALLOC) && - (entry_flags & NGHTTP2_HD_FLAG_VALUE_GIFT)) { - /* nv->value are managed by caller. */ - new_ent->nv.value = NULL; - new_ent->nv.valuelen = 0; - } - - nghttp2_hd_entry_free(new_ent, mem); - nghttp2_mem_free(mem, new_ent); - - return NULL; - } - - new_ent->seq = context->next_seq++; - new_ent->hash = hash; - - if (map) { - hd_map_insert(map, new_ent); - } - - context->hd_table_bufsize += room; + immediately evicted. So we don't allocate memory for it. */ + return 0; } - return new_ent; + + new_ent = nghttp2_mem_malloc(mem, sizeof(nghttp2_hd_entry)); + if (new_ent == NULL) { + return NGHTTP2_ERR_NOMEM; + } + + nghttp2_hd_entry_init(new_ent, nv); + + rv = hd_ringbuf_push_front(&context->hd_table, new_ent, mem); + + if (rv != 0) { + nghttp2_hd_entry_free(new_ent); + nghttp2_mem_free(mem, new_ent); + + return rv; + } + + new_ent->seq = context->next_seq++; + new_ent->hash = hash; + + if (map) { + hd_map_insert(map, new_ent); + } + + context->hd_table_bufsize += room; + + return 0; } typedef struct { @@ -1575,10 +1483,11 @@ typedef struct { uint8_t name_value_match; } search_result; -static search_result search_static_table(const nghttp2_nv *nv, int token, +static search_result search_static_table(const nghttp2_nv *nv, int32_t token, int indexing_mode) { search_result res = {token, 0}; int i; + nghttp2_hd_static_entry *ent; if (indexing_mode == NGHTTP2_HD_NEVER_INDEXING) { return res; @@ -1587,7 +1496,9 @@ static search_result search_static_table(const nghttp2_nv *nv, int token, for (i = token; i <= NGHTTP2_TOKEN_WWW_AUTHENTICATE && static_table[i].token == token; ++i) { - if (value_eq(&static_table[i].nv, nv)) { + ent = &static_table[i]; + if (ent->value.len == nv->valuelen && + memcmp(ent->value.base, nv->value, nv->valuelen) == 0) { res.index = i; res.name_value_match = 1; return res; @@ -1597,7 +1508,7 @@ static search_result search_static_table(const nghttp2_nv *nv, int token, } static search_result search_hd_table(nghttp2_hd_context *context, - const nghttp2_nv *nv, int token, + const nghttp2_nv *nv, int32_t token, int indexing_mode, nghttp2_hd_map *map, uint32_t hash) { search_result res = {-1, 0}; @@ -1641,15 +1552,15 @@ static void hd_context_shrink_table_size(nghttp2_hd_context *context, context->hd_table.len > 0) { size_t idx = context->hd_table.len - 1; nghttp2_hd_entry *ent = hd_ringbuf_get(&context->hd_table, idx); - context->hd_table_bufsize -= entry_room(ent->nv.namelen, ent->nv.valuelen); + context->hd_table_bufsize -= + entry_room(ent->nv.name->len, ent->nv.value->len); hd_ringbuf_pop_back(&context->hd_table); if (map) { hd_map_remove(map, ent); } - if (--ent->ref == 0) { - nghttp2_hd_entry_free(ent, mem); - nghttp2_mem_free(mem, ent); - } + + nghttp2_hd_entry_free(ent); + nghttp2_mem_free(mem, ent); } } @@ -1708,19 +1619,33 @@ static size_t get_max_index(nghttp2_hd_context *context) { return context->hd_table.len + NGHTTP2_STATIC_TABLE_LENGTH - 1; } -nghttp2_hd_entry *nghttp2_hd_table_get(nghttp2_hd_context *context, - size_t idx) { +nghttp2_hd_nv nghttp2_hd_table_get(nghttp2_hd_context *context, size_t idx) { assert(INDEX_RANGE_VALID(context, idx)); if (idx >= NGHTTP2_STATIC_TABLE_LENGTH) { - return hd_ringbuf_get(&context->hd_table, - idx - NGHTTP2_STATIC_TABLE_LENGTH); + return hd_ringbuf_get(&context->hd_table, idx - NGHTTP2_STATIC_TABLE_LENGTH) + ->nv; } else { - return &static_table[idx]; + nghttp2_hd_static_entry *ent; + + ent = &static_table[idx]; + return (nghttp2_hd_nv){&ent->name, &ent->value, ent->token, + NGHTTP2_NV_FLAG_NONE}; } } +static const nghttp2_nv *nghttp2_hd_table_get2(nghttp2_hd_context *context, + size_t idx) { + assert(INDEX_RANGE_VALID(context, idx)); + if (idx >= NGHTTP2_STATIC_TABLE_LENGTH) { + return &hd_ringbuf_get(&context->hd_table, + idx - NGHTTP2_STATIC_TABLE_LENGTH)->cnv; + } + + return &static_table[idx].cnv; +} + static int hd_deflate_decide_indexing(nghttp2_hd_deflater *deflater, - const nghttp2_nv *nv, int token) { + const nghttp2_nv *nv, int32_t token) { if (token == NGHTTP2_TOKEN__PATH || token == NGHTTP2_TOKEN_AGE || token == NGHTTP2_TOKEN_CONTENT_LENGTH || token == NGHTTP2_TOKEN_ETAG || token == NGHTTP2_TOKEN_IF_MODIFIED_SINCE || @@ -1740,7 +1665,7 @@ static int deflate_nv(nghttp2_hd_deflater *deflater, nghttp2_bufs *bufs, search_result res; ssize_t idx; int indexing_mode; - int token; + int32_t token; nghttp2_mem *mem; uint32_t hash = 0; @@ -1789,28 +1714,36 @@ static int deflate_nv(nghttp2_hd_deflater *deflater, nghttp2_bufs *bufs, } if (indexing_mode == NGHTTP2_HD_WITH_INDEXING) { - nghttp2_hd_entry *new_ent; + nghttp2_hd_nv hd_nv; + if (idx != -1 && idx < (ssize_t)NGHTTP2_STATIC_TABLE_LENGTH) { - nghttp2_nv nv_indname; - nv_indname = *nv; - nv_indname.name = - nghttp2_hd_table_get(&deflater->ctx, (size_t)idx)->nv.name; - new_ent = add_hd_table_incremental(&deflater->ctx, &nv_indname, token, - NGHTTP2_HD_FLAG_VALUE_ALLOC, - &deflater->map, hash); + hd_nv.name = nghttp2_hd_table_get(&deflater->ctx, (size_t)idx).name; + nghttp2_rcbuf_incref(hd_nv.name); } else { - new_ent = add_hd_table_incremental(&deflater->ctx, nv, token, - NGHTTP2_HD_FLAG_NAME_ALLOC | - NGHTTP2_HD_FLAG_VALUE_ALLOC, - &deflater->map, hash); + rv = nghttp2_rcbuf_new2(&hd_nv.name, nv->name, nv->namelen, mem); + if (rv != 0) { + return rv; + } } - if (!new_ent) { + + rv = nghttp2_rcbuf_new2(&hd_nv.value, nv->value, nv->valuelen, mem); + + if (rv != 0) { + nghttp2_rcbuf_decref(hd_nv.name); + return rv; + } + + hd_nv.token = token; + hd_nv.flags = NGHTTP2_NV_FLAG_NONE; + + rv = add_hd_table_incremental(&deflater->ctx, &hd_nv, &deflater->map, hash); + + nghttp2_rcbuf_decref(hd_nv.value); + nghttp2_rcbuf_decref(hd_nv.name); + + if (rv != 0) { return NGHTTP2_ERR_HEADER_COMP; } - if (new_ent->ref == 0) { - nghttp2_hd_entry_free(new_ent, mem); - nghttp2_mem_free(mem, new_ent); - } } if (idx == -1) { rv = emit_newname_block(bufs, nv, indexing_mode); @@ -2093,10 +2026,10 @@ static ssize_t hd_inflate_read(nghttp2_hd_inflater *inflater, nghttp2_buf *buf, * Out of memory */ static int hd_inflate_commit_indexed(nghttp2_hd_inflater *inflater, - nghttp2_nv *nv_out, int *token_out) { - nghttp2_hd_entry *ent = nghttp2_hd_table_get(&inflater->ctx, inflater->index); + nghttp2_hd_nv *nv_out) { + nghttp2_hd_nv nv = nghttp2_hd_table_get(&inflater->ctx, inflater->index); - emit_indexed_header(nv_out, token_out, ent); + emit_header(nv_out, &nv); return 0; } @@ -2113,19 +2046,9 @@ static int hd_inflate_commit_indexed(nghttp2_hd_inflater *inflater, * Out of memory */ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, - nghttp2_nv *nv_out, int *token_out) { - nghttp2_nv nv; - nghttp2_mem *mem; - - mem = inflater->ctx.mem; - - nv.name = inflater->namebuf.pos; - nv.namelen = nghttp2_buf_len(&inflater->namebuf); - nv.value = inflater->valuebuf.pos; - nv.valuelen = nghttp2_buf_len(&inflater->valuebuf); - - nghttp2_buf_init(&inflater->valuebuf); - nghttp2_buf_init(&inflater->namebuf); + nghttp2_hd_nv *nv_out) { + nghttp2_hd_nv nv; + int rv; if (inflater->no_index) { nv.flags = NGHTTP2_NV_FLAG_NO_INDEX; @@ -2133,35 +2056,26 @@ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, nv.flags = NGHTTP2_NV_FLAG_NONE; } + nv.name = inflater->namercbuf; + nv.value = inflater->valuercbuf; + nv.token = lookup_token(inflater->namercbuf->base, inflater->namercbuf->len); + if (inflater->index_required) { - nghttp2_hd_entry *new_ent; - uint8_t ent_flags; + rv = add_hd_table_incremental(&inflater->ctx, &nv, NULL, 0); - ent_flags = NGHTTP2_HD_FLAG_NAME_ALLOC | NGHTTP2_HD_FLAG_NAME_GIFT | - NGHTTP2_HD_FLAG_VALUE_ALLOC | NGHTTP2_HD_FLAG_VALUE_GIFT; - - new_ent = add_hd_table_incremental(&inflater->ctx, &nv, - lookup_token(nv.name, nv.namelen), - ent_flags, NULL, 0); - - if (new_ent) { - emit_indexed_header(nv_out, token_out, new_ent); - inflater->ent_keep = new_ent; - - return 0; + if (rv != 0) { + return rv; } - - nghttp2_mem_free(mem, nv.value); - nghttp2_mem_free(mem, nv.name); - - return NGHTTP2_ERR_NOMEM; } - emit_literal_header(nv_out, token_out, &nv); + emit_header(nv_out, &nv); inflater->nv_name_keep = nv.name; inflater->nv_value_keep = nv.value; + inflater->namercbuf = NULL; + inflater->valuercbuf = NULL; + return 0; } @@ -2177,12 +2091,11 @@ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, * Out of memory */ static int hd_inflate_commit_indname(nghttp2_hd_inflater *inflater, - nghttp2_nv *nv_out, int *token_out) { - nghttp2_nv nv; - nghttp2_hd_entry *ent_name; - nghttp2_mem *mem; + nghttp2_hd_nv *nv_out) { + nghttp2_hd_nv nv; + int rv; - mem = inflater->ctx.mem; + nv = nghttp2_hd_table_get(&inflater->ctx, inflater->index); if (inflater->no_index) { nv.flags = NGHTTP2_NV_FLAG_NO_INDEX; @@ -2190,72 +2103,57 @@ static int hd_inflate_commit_indname(nghttp2_hd_inflater *inflater, nv.flags = NGHTTP2_NV_FLAG_NONE; } - ent_name = nghttp2_hd_table_get(&inflater->ctx, inflater->index); + nghttp2_rcbuf_incref(nv.name); + + nv.value = inflater->valuercbuf; if (inflater->index_required) { - nghttp2_hd_entry *new_ent; - uint8_t ent_flags; - - ent_flags = NGHTTP2_HD_FLAG_VALUE_ALLOC | NGHTTP2_HD_FLAG_VALUE_GIFT; - - if (inflater->index >= NGHTTP2_STATIC_TABLE_LENGTH) { - /* We don't copy name in static table */ - ent_flags |= NGHTTP2_HD_FLAG_NAME_ALLOC; + rv = add_hd_table_incremental(&inflater->ctx, &nv, NULL, 0); + if (rv != 0) { + nghttp2_rcbuf_decref(nv.name); + return NGHTTP2_ERR_NOMEM; } - - nv.name = ent_name->nv.name; - nv.namelen = ent_name->nv.namelen; - - nv.value = inflater->valuebuf.pos; - nv.valuelen = nghttp2_buf_len(&inflater->valuebuf); - - nghttp2_buf_init(&inflater->valuebuf); - - new_ent = add_hd_table_incremental(&inflater->ctx, &nv, ent_name->token, - ent_flags, NULL, 0); - - /* At this point, ent_name might be deleted. */ - - if (new_ent) { - emit_indexed_header(nv_out, token_out, new_ent); - - inflater->ent_keep = new_ent; - - return 0; - } - - nghttp2_mem_free(mem, nv.value); - - return NGHTTP2_ERR_NOMEM; } - nv.name = ent_name->nv.name; - nv.namelen = ent_name->nv.namelen; - nv.value = inflater->valuebuf.pos; - nv.valuelen = nghttp2_buf_len(&inflater->valuebuf); - - nghttp2_buf_init(&inflater->valuebuf); - - emit_literal_header(nv_out, token_out, &nv); + emit_header(nv_out, &nv); + inflater->nv_name_keep = nv.name; inflater->nv_value_keep = nv.value; + inflater->valuercbuf = NULL; + return 0; } ssize_t nghttp2_hd_inflate_hd(nghttp2_hd_inflater *inflater, nghttp2_nv *nv_out, int *inflate_flags, uint8_t *in, size_t inlen, int in_final) { - int token; + ssize_t rv; + nghttp2_hd_nv hd_nv; - return nghttp2_hd_inflate_hd2(inflater, nv_out, inflate_flags, &token, in, - inlen, in_final); + rv = nghttp2_hd_inflate_hd2(inflater, &hd_nv, inflate_flags, in, inlen, + in_final); + + if (rv < 0) { + return rv; + } + + if (*inflate_flags & NGHTTP2_HD_INFLATE_EMIT) { + nv_out->name = hd_nv.name->base; + nv_out->namelen = hd_nv.name->len; + + nv_out->value = hd_nv.value->base; + nv_out->valuelen = hd_nv.value->len; + + nv_out->flags = hd_nv.flags; + } + + return rv; } ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, - nghttp2_nv *nv_out, int *inflate_flags, - int *token_out, uint8_t *in, size_t inlen, - int in_final) { + nghttp2_hd_nv *nv_out, int *inflate_flags, + uint8_t *in, size_t inlen, int in_final) { ssize_t rv = 0; uint8_t *first = in; uint8_t *last = in + inlen; @@ -2271,7 +2169,6 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, DEBUGF(fprintf(stderr, "inflatehd: start state=%d\n", inflater->state)); hd_inflate_keep_free(inflater); - *token_out = -1; *inflate_flags = NGHTTP2_HD_INFLATE_NONE; for (; in != last || busy;) { busy = 0; @@ -2377,7 +2274,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, inflater->index = inflater->left; --inflater->index; - rv = hd_inflate_commit_indexed(inflater, nv_out, token_out); + rv = hd_inflate_commit_indexed(inflater, nv_out); if (rv < 0) { goto fail; } @@ -2422,17 +2319,21 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, nghttp2_hd_huff_decode_context_init(&inflater->huff_decode_ctx); inflater->state = NGHTTP2_HD_STATE_NEWNAME_READ_NAMEHUFF; - rv = nghttp2_buf_reserve(&inflater->namebuf, inflater->left * 2 + 1, - mem); + + rv = nghttp2_rcbuf_new(&inflater->namercbuf, inflater->left * 2 + 1, + mem); } else { inflater->state = NGHTTP2_HD_STATE_NEWNAME_READ_NAME; - rv = nghttp2_buf_reserve(&inflater->namebuf, inflater->left + 1, mem); + rv = nghttp2_rcbuf_new(&inflater->namercbuf, inflater->left + 1, mem); } if (rv != 0) { goto fail; } + nghttp2_buf_wrap_init(&inflater->namebuf, inflater->namercbuf->base, + inflater->namercbuf->len); + break; case NGHTTP2_HD_STATE_NEWNAME_READ_NAMEHUFF: rv = hd_inflate_read_huff(inflater, &inflater->namebuf, in, last); @@ -2452,6 +2353,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, } *inflater->namebuf.last = '\0'; + inflater->namercbuf->len = nghttp2_buf_len(&inflater->namebuf); inflater->state = NGHTTP2_HD_STATE_CHECK_VALUELEN; @@ -2473,6 +2375,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, } *inflater->namebuf.last = '\0'; + inflater->namercbuf->len = nghttp2_buf_len(&inflater->namebuf); inflater->state = NGHTTP2_HD_STATE_CHECK_VALUELEN; @@ -2505,18 +2408,21 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, inflater->state = NGHTTP2_HD_STATE_READ_VALUEHUFF; - rv = nghttp2_buf_reserve(&inflater->valuebuf, inflater->left * 2 + 1, - mem); + rv = nghttp2_rcbuf_new(&inflater->valuercbuf, inflater->left * 2 + 1, + mem); } else { inflater->state = NGHTTP2_HD_STATE_READ_VALUE; - rv = nghttp2_buf_reserve(&inflater->valuebuf, inflater->left + 1, mem); + rv = nghttp2_rcbuf_new(&inflater->valuercbuf, inflater->left + 1, mem); } if (rv != 0) { goto fail; } + nghttp2_buf_wrap_init(&inflater->valuebuf, inflater->valuercbuf->base, + inflater->valuercbuf->len); + busy = 1; break; @@ -2538,11 +2444,12 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, } *inflater->valuebuf.last = '\0'; + inflater->valuercbuf->len = nghttp2_buf_len(&inflater->valuebuf); if (inflater->opcode == NGHTTP2_HD_OPCODE_NEWNAME) { - rv = hd_inflate_commit_newname(inflater, nv_out, token_out); + rv = hd_inflate_commit_newname(inflater, nv_out); } else { - rv = hd_inflate_commit_indname(inflater, nv_out, token_out); + rv = hd_inflate_commit_indname(inflater, nv_out); } if (rv != 0) { @@ -2572,11 +2479,12 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, } *inflater->valuebuf.last = '\0'; + inflater->valuercbuf->len = nghttp2_buf_len(&inflater->valuebuf); if (inflater->opcode == NGHTTP2_HD_OPCODE_NEWNAME) { - rv = hd_inflate_commit_newname(inflater, nv_out, token_out); + rv = hd_inflate_commit_newname(inflater, nv_out); } else { - rv = hd_inflate_commit_indname(inflater, nv_out, token_out); + rv = hd_inflate_commit_indname(inflater, nv_out); } if (rv != 0) { @@ -2710,7 +2618,7 @@ static const nghttp2_nv *hd_get_table_entry(nghttp2_hd_context *context, return NULL; } - return &nghttp2_hd_table_get(context, idx)->nv; + return nghttp2_hd_table_get2(context, idx); } size_t nghttp2_hd_deflate_get_num_table_entries(nghttp2_hd_deflater *deflater) { diff --git a/lib/nghttp2_hd.h b/lib/nghttp2_hd.h index fff4c5a5..cacfcf3a 100644 --- a/lib/nghttp2_hd.h +++ b/lib/nghttp2_hd.h @@ -34,6 +34,7 @@ #include "nghttp2_hd_huffman.h" #include "nghttp2_buf.h" #include "nghttp2_mem.h" +#include "nghttp2_rcbuf.h" #define NGHTTP2_HD_DEFAULT_MAX_BUFFER_SIZE NGHTTP2_DEFAULT_HEADER_TABLE_SIZE #define NGHTTP2_HD_ENTRY_OVERHEAD 32 @@ -168,25 +169,29 @@ typedef enum { NGHTTP2_TOKEN_X_XSS_PROTECTION, } nghttp2_token; -typedef enum { - NGHTTP2_HD_FLAG_NONE = 0, - /* Indicates name was dynamically allocated and must be freed */ - NGHTTP2_HD_FLAG_NAME_ALLOC = 1, - /* Indicates value was dynamically allocated and must be freed */ - NGHTTP2_HD_FLAG_VALUE_ALLOC = 1 << 1, - /* Indicates that the name was gifted to the entry and no copying - necessary. */ - NGHTTP2_HD_FLAG_NAME_GIFT = 1 << 2, - /* Indicates that the value was gifted to the entry and no copying - necessary. */ - NGHTTP2_HD_FLAG_VALUE_GIFT = 1 << 3 -} nghttp2_hd_flags; - struct nghttp2_hd_entry; typedef struct nghttp2_hd_entry nghttp2_hd_entry; +typedef struct { + /* The buffer containing header field name. NULL-termination is + guaranteed. */ + nghttp2_rcbuf *name; + /* The buffer containing header field value. NULL-termination is + guaranteed. */ + nghttp2_rcbuf *value; + /* nghttp2_token value for name. It could be -1 if we have no token + for that header field name. */ + int32_t token; + /* Bitwise OR of one or more of nghttp2_nv_flag. */ + uint8_t flags; +} nghttp2_hd_nv; + struct nghttp2_hd_entry { - nghttp2_nv nv; + /* The header field name/value pair */ + nghttp2_hd_nv nv; + /* This is solely for nghttp2_hd_{deflate,inflate}_get_table_entry + APIs to keep backward compatibility. */ + nghttp2_nv cnv; /* The next entry which shares same bucket in hash table. */ nghttp2_hd_entry *next; /* The sequence number. We will increment it by one whenever we @@ -194,14 +199,17 @@ struct nghttp2_hd_entry { uint32_t seq; /* The hash value for header name (nv.name). */ uint32_t hash; - /* nghttp2_token value for nv.name. It could be -1 if we have no - token for that header field name. */ - int token; - /* Reference count */ - uint8_t ref; - uint8_t flags; }; +/* The entry used for static header table. */ +typedef struct { + nghttp2_rcbuf name; + nghttp2_rcbuf value; + nghttp2_nv cnv; + int32_t token; + uint32_t hash; +} nghttp2_hd_static_entry; + typedef struct { nghttp2_hd_entry **buffer; size_t mask; @@ -275,17 +283,14 @@ struct nghttp2_hd_deflater { struct nghttp2_hd_inflater { nghttp2_hd_context ctx; - /* header buffer */ - nghttp2_buf namebuf, valuebuf; /* Stores current state of huffman decoding */ nghttp2_hd_huff_decode_context huff_decode_ctx; - /* Pointer to the nghttp2_hd_entry which is used current header - emission. This is required because in some cases the - ent_keep->ref == 0 and we have to keep track of it. */ - nghttp2_hd_entry *ent_keep; - /* Pointer to the name/value pair buffer which is used in the - current header emission. */ - uint8_t *nv_name_keep, *nv_value_keep; + /* header buffer */ + nghttp2_buf namebuf, valuebuf; + nghttp2_rcbuf *namercbuf, *valuercbuf; + /* Pointer to the name/value pair which are used in the current + header emission. */ + nghttp2_rcbuf *nv_name_keep, *nv_value_keep; /* The number of bytes to read */ size_t left; /* The index in indexed repr or indexed name */ @@ -309,24 +314,16 @@ struct nghttp2_hd_inflater { }; /* - * Initializes the |ent| members. If NGHTTP2_HD_FLAG_NAME_ALLOC bit - * set in the |flags|, the content pointed by the |name| with length - * |namelen| is copied. Likewise, if NGHTTP2_HD_FLAG_VALUE_ALLOC bit - * set in the |flags|, the content pointed by the |value| with length - * |valuelen| is copied. The |token| is enum number looked up by - * |name|. It could be -1 if we don't have that enum value. - * - * This function returns 0 if it succeeds, or one of the following - * negative error codes: - * - * NGHTTP2_ERR_NOMEM - * Out of memory. + * Initializes the |ent| members. The reference counts of nv->name + * and nv->value are increased by one for each. */ -int nghttp2_hd_entry_init(nghttp2_hd_entry *ent, uint8_t flags, uint8_t *name, - size_t namelen, uint8_t *value, size_t valuelen, - int token, nghttp2_mem *mem); +void nghttp2_hd_entry_init(nghttp2_hd_entry *ent, nghttp2_hd_nv *nv); -void nghttp2_hd_entry_free(nghttp2_hd_entry *ent, nghttp2_mem *mem); +/* + * This function decreases the reference counts of nv->name and + * nv->value. + */ +void nghttp2_hd_entry_free(nghttp2_hd_entry *ent); /* * Initializes |deflater| for deflating name/values pairs. @@ -407,16 +404,14 @@ int nghttp2_hd_inflate_init(nghttp2_hd_inflater *inflater, nghttp2_mem *mem); void nghttp2_hd_inflate_free(nghttp2_hd_inflater *inflater); /* - * Similar to nghttp2_hd_inflate_hd(), but this takes additional - * output parameter |token|. On successful header emission, it - * contains nghttp2_token value for nv_out->name. It could be -1 if - * we don't have enum value for the name. Other than that return - * values and semantics are the same as nghttp2_hd_inflate_hd(). + * Similar to nghttp2_hd_inflate_hd(), but this takes nghttp2_hd_nv + * instead of nghttp2_nv as output parameter |nv_out|. Other than + * that return values and semantics are the same as + * nghttp2_hd_inflate_hd(). */ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, - nghttp2_nv *nv_out, int *inflate_flags, - int *token, uint8_t *in, size_t inlen, - int in_final); + nghttp2_hd_nv *nv_out, int *inflate_flags, + uint8_t *in, size_t inlen, int in_final); /* For unittesting purpose */ int nghttp2_hd_emit_indname_block(nghttp2_bufs *bufs, size_t index, @@ -430,8 +425,7 @@ int nghttp2_hd_emit_newname_block(nghttp2_bufs *bufs, nghttp2_nv *nv, int nghttp2_hd_emit_table_size(nghttp2_bufs *bufs, size_t table_size); /* For unittesting purpose */ -nghttp2_hd_entry *nghttp2_hd_table_get(nghttp2_hd_context *context, - size_t index); +nghttp2_hd_nv nghttp2_hd_table_get(nghttp2_hd_context *context, size_t index); /* For unittesting purpose */ ssize_t nghttp2_hd_decode_length(uint32_t *res, size_t *shift_ptr, int *final, diff --git a/lib/nghttp2_http.c b/lib/nghttp2_http.c index 6aee90f0..f993c167 100644 --- a/lib/nghttp2_http.c +++ b/lib/nghttp2_http.c @@ -82,12 +82,12 @@ static int lws(const uint8_t *s, size_t n) { return 1; } -static int check_pseudo_header(nghttp2_stream *stream, const nghttp2_nv *nv, +static int check_pseudo_header(nghttp2_stream *stream, const nghttp2_hd_nv *nv, int flag) { if (stream->http_flags & flag) { return 0; } - if (lws(nv->value, nv->valuelen)) { + if (lws(nv->value->base, nv->value->len)) { return 0; } stream->http_flags = (uint16_t)(stream->http_flags | flag); @@ -112,16 +112,16 @@ static int check_path(nghttp2_stream *stream) { (stream->http_flags & NGHTTP2_HTTP_FLAG_PATH_ASTERISK))); } -static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, - int token, int trailer) { - if (nv->name[0] == ':') { +static int http_request_on_header(nghttp2_stream *stream, nghttp2_hd_nv *nv, + int trailer) { + if (nv->name->base[0] == ':') { if (trailer || (stream->http_flags & NGHTTP2_HTTP_FLAG_PSEUDO_HEADER_DISALLOWED)) { return NGHTTP2_ERR_HTTP_HEADER; } } - switch (token) { + switch (nv->token) { case NGHTTP2_TOKEN__AUTHORITY: if (!check_pseudo_header(stream, nv, NGHTTP2_HTTP_FLAG__AUTHORITY)) { return NGHTTP2_ERR_HTTP_HEADER; @@ -131,16 +131,16 @@ static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, if (!check_pseudo_header(stream, nv, NGHTTP2_HTTP_FLAG__METHOD)) { return NGHTTP2_ERR_HTTP_HEADER; } - switch (nv->valuelen) { + switch (nv->value->len) { case 4: - if (lstreq("HEAD", nv->value, nv->valuelen)) { + if (lstreq("HEAD", nv->value->base, nv->value->len)) { stream->http_flags |= NGHTTP2_HTTP_FLAG_METH_HEAD; } break; case 7: - switch (nv->value[6]) { + switch (nv->value->base[6]) { case 'T': - if (lstreq("CONNECT", nv->value, nv->valuelen)) { + if (lstreq("CONNECT", nv->value->base, nv->value->len)) { if (stream->stream_id % 2 == 0) { /* we won't allow CONNECT for push */ return NGHTTP2_ERR_HTTP_HEADER; @@ -153,7 +153,7 @@ static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, } break; case 'S': - if (lstreq("OPTIONS", nv->value, nv->valuelen)) { + if (lstreq("OPTIONS", nv->value->base, nv->value->len)) { stream->http_flags |= NGHTTP2_HTTP_FLAG_METH_OPTIONS; } break; @@ -168,9 +168,9 @@ static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, if (!check_pseudo_header(stream, nv, NGHTTP2_HTTP_FLAG__PATH)) { return NGHTTP2_ERR_HTTP_HEADER; } - if (nv->value[0] == '/') { + if (nv->value->base[0] == '/') { stream->http_flags |= NGHTTP2_HTTP_FLAG_PATH_REGULAR; - } else if (nv->valuelen == 1 && nv->value[0] == '*') { + } else if (nv->value->len == 1 && nv->value->base[0] == '*') { stream->http_flags |= NGHTTP2_HTTP_FLAG_PATH_ASTERISK; } break; @@ -181,8 +181,8 @@ static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, if (!check_pseudo_header(stream, nv, NGHTTP2_HTTP_FLAG__SCHEME)) { return NGHTTP2_ERR_HTTP_HEADER; } - if ((nv->valuelen == 4 && memieq("http", nv->value, 4)) || - (nv->valuelen == 5 && memieq("https", nv->value, 5))) { + if ((nv->value->len == 4 && memieq("http", nv->value->base, 4)) || + (nv->value->len == 5 && memieq("https", nv->value->base, 5))) { stream->http_flags |= NGHTTP2_HTTP_FLAG_SCHEME_HTTP; } break; @@ -195,7 +195,7 @@ static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, if (stream->content_length != -1) { return NGHTTP2_ERR_HTTP_HEADER; } - stream->content_length = parse_uint(nv->value, nv->valuelen); + stream->content_length = parse_uint(nv->value->base, nv->value->len); if (stream->content_length == -1) { return NGHTTP2_ERR_HTTP_HEADER; } @@ -209,41 +209,41 @@ static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, case NGHTTP2_TOKEN_UPGRADE: return NGHTTP2_ERR_HTTP_HEADER; case NGHTTP2_TOKEN_TE: - if (!lstrieq("trailers", nv->value, nv->valuelen)) { + if (!lstrieq("trailers", nv->value->base, nv->value->len)) { return NGHTTP2_ERR_HTTP_HEADER; } break; default: - if (nv->name[0] == ':') { + if (nv->name->base[0] == ':') { return NGHTTP2_ERR_HTTP_HEADER; } } - if (nv->name[0] != ':') { + if (nv->name->base[0] != ':') { stream->http_flags |= NGHTTP2_HTTP_FLAG_PSEUDO_HEADER_DISALLOWED; } return 0; } -static int http_response_on_header(nghttp2_stream *stream, nghttp2_nv *nv, - int token, int trailer) { - if (nv->name[0] == ':') { +static int http_response_on_header(nghttp2_stream *stream, nghttp2_hd_nv *nv, + int trailer) { + if (nv->name->base[0] == ':') { if (trailer || (stream->http_flags & NGHTTP2_HTTP_FLAG_PSEUDO_HEADER_DISALLOWED)) { return NGHTTP2_ERR_HTTP_HEADER; } } - switch (token) { + switch (nv->token) { case NGHTTP2_TOKEN__STATUS: { if (!check_pseudo_header(stream, nv, NGHTTP2_HTTP_FLAG__STATUS)) { return NGHTTP2_ERR_HTTP_HEADER; } - if (nv->valuelen != 3) { + if (nv->value->len != 3) { return NGHTTP2_ERR_HTTP_HEADER; } - stream->status_code = (int16_t)parse_uint(nv->value, nv->valuelen); + stream->status_code = (int16_t)parse_uint(nv->value->base, nv->value->len); if (stream->status_code == -1) { return NGHTTP2_ERR_HTTP_HEADER; } @@ -253,7 +253,7 @@ static int http_response_on_header(nghttp2_stream *stream, nghttp2_nv *nv, if (stream->content_length != -1) { return NGHTTP2_ERR_HTTP_HEADER; } - stream->content_length = parse_uint(nv->value, nv->valuelen); + stream->content_length = parse_uint(nv->value->base, nv->value->len); if (stream->content_length == -1) { return NGHTTP2_ERR_HTTP_HEADER; } @@ -267,17 +267,17 @@ static int http_response_on_header(nghttp2_stream *stream, nghttp2_nv *nv, case NGHTTP2_TOKEN_UPGRADE: return NGHTTP2_ERR_HTTP_HEADER; case NGHTTP2_TOKEN_TE: - if (!lstrieq("trailers", nv->value, nv->valuelen)) { + if (!lstrieq("trailers", nv->value->base, nv->value->len)) { return NGHTTP2_ERR_HTTP_HEADER; } break; default: - if (nv->name[0] == ':') { + if (nv->name->base[0] == ':') { return NGHTTP2_ERR_HTTP_HEADER; } } - if (nv->name[0] != ':') { + if (nv->name->base[0] != ':') { stream->http_flags |= NGHTTP2_HTTP_FLAG_PSEUDO_HEADER_DISALLOWED; } @@ -375,7 +375,7 @@ static int check_scheme(const uint8_t *value, size_t len) { } int nghttp2_http_on_header(nghttp2_session *session, nghttp2_stream *stream, - nghttp2_frame *frame, nghttp2_nv *nv, int token, + nghttp2_frame *frame, nghttp2_hd_nv *nv, int trailer) { int rv; @@ -386,14 +386,14 @@ int nghttp2_http_on_header(nghttp2_session *session, nghttp2_stream *stream, this, we may disrupt many web sites and/or libraries. So we become conservative here, and just ignore those illegal regular headers. */ - if (!nghttp2_check_header_name(nv->name, nv->namelen)) { + if (!nghttp2_check_header_name(nv->name->base, nv->name->len)) { size_t i; - if (nv->namelen > 0 && nv->name[0] == ':') { + if (nv->name->len > 0 && nv->name->base[0] == ':') { return NGHTTP2_ERR_HTTP_HEADER; } /* header field name must be lower-cased without exception */ - for (i = 0; i < nv->namelen; ++i) { - uint8_t c = nv->name[i]; + for (i = 0; i < nv->name->len; ++i) { + uint8_t c = nv->name->base[i]; if ('A' <= c && c <= 'Z') { return NGHTTP2_ERR_HTTP_HEADER; } @@ -405,17 +405,18 @@ int nghttp2_http_on_header(nghttp2_session *session, nghttp2_stream *stream, return NGHTTP2_ERR_IGN_HTTP_HEADER; } - if (token == NGHTTP2_TOKEN__AUTHORITY || token == NGHTTP2_TOKEN_HOST) { - rv = check_authority(nv->value, nv->valuelen); - } else if (token == NGHTTP2_TOKEN__SCHEME) { - rv = check_scheme(nv->value, nv->valuelen); + if (nv->token == NGHTTP2_TOKEN__AUTHORITY || + nv->token == NGHTTP2_TOKEN_HOST) { + rv = check_authority(nv->value->base, nv->value->len); + } else if (nv->token == NGHTTP2_TOKEN__SCHEME) { + rv = check_scheme(nv->value->base, nv->value->len); } else { - rv = nghttp2_check_header_value(nv->value, nv->valuelen); + rv = nghttp2_check_header_value(nv->value->base, nv->value->len); } if (rv == 0) { - assert(nv->namelen > 0); - if (nv->name[0] == ':') { + assert(nv->name->len > 0); + if (nv->name->base[0] == ':') { return NGHTTP2_ERR_HTTP_HEADER; } /* When ignoring regular headers, we set this flag so that we @@ -426,10 +427,10 @@ int nghttp2_http_on_header(nghttp2_session *session, nghttp2_stream *stream, } if (session->server || frame->hd.type == NGHTTP2_PUSH_PROMISE) { - return http_request_on_header(stream, nv, token, trailer); + return http_request_on_header(stream, nv, trailer); } - return http_response_on_header(stream, nv, token, trailer); + return http_response_on_header(stream, nv, trailer); } int nghttp2_http_on_request_headers(nghttp2_stream *stream, diff --git a/lib/nghttp2_http.h b/lib/nghttp2_http.h index f782058f..ac684c4d 100644 --- a/lib/nghttp2_http.h +++ b/lib/nghttp2_http.h @@ -36,8 +36,7 @@ /* * This function is called when HTTP header field |nv| in |frame| is * received for |stream|. This function will validate |nv| against - * the current state of stream. The |token| is nghttp2_token value - * for nv->name, or -1 if we don't have enum value for the name. + * the current state of stream. * * This function returns 0 if it succeeds, or one of the following * negative error codes: @@ -49,7 +48,7 @@ * if it was not received because of compatibility reasons. */ int nghttp2_http_on_header(nghttp2_session *session, nghttp2_stream *stream, - nghttp2_frame *frame, nghttp2_nv *nv, int token, + nghttp2_frame *frame, nghttp2_hd_nv *nv, int trailer); /* diff --git a/lib/nghttp2_mem.c b/lib/nghttp2_mem.c index e7d5aae3..317363a7 100644 --- a/lib/nghttp2_mem.c +++ b/lib/nghttp2_mem.c @@ -52,6 +52,10 @@ void nghttp2_mem_free(nghttp2_mem *mem, void *ptr) { mem->free(ptr, mem->mem_user_data); } +void nghttp2_mem_free2(nghttp2_free free, void *ptr, void *mem_user_data) { + free(ptr, mem_user_data); +} + void *nghttp2_mem_calloc(nghttp2_mem *mem, size_t nmemb, size_t size) { return mem->calloc(nmemb, size, mem->mem_user_data); } diff --git a/lib/nghttp2_mem.h b/lib/nghttp2_mem.h index d1fded4f..7709e1c5 100644 --- a/lib/nghttp2_mem.h +++ b/lib/nghttp2_mem.h @@ -38,6 +38,7 @@ nghttp2_mem *nghttp2_mem_default(void); |mem|. */ void *nghttp2_mem_malloc(nghttp2_mem *mem, size_t size); void nghttp2_mem_free(nghttp2_mem *mem, void *ptr); +void nghttp2_mem_free2(nghttp2_free free, void *ptr, void *mem_user_data); void *nghttp2_mem_calloc(nghttp2_mem *mem, size_t nmemb, size_t size); void *nghttp2_mem_realloc(nghttp2_mem *mem, void *ptr, size_t size); diff --git a/lib/nghttp2_rcbuf.c b/lib/nghttp2_rcbuf.c new file mode 100644 index 00000000..a1af1da8 --- /dev/null +++ b/lib/nghttp2_rcbuf.c @@ -0,0 +1,98 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "nghttp2_rcbuf.h" + +#include +#include + +#include "nghttp2_mem.h" + +int nghttp2_rcbuf_new(nghttp2_rcbuf **rcbuf_ptr, size_t size, + nghttp2_mem *mem) { + uint8_t *p; + + p = nghttp2_mem_malloc(mem, sizeof(nghttp2_rcbuf) + size); + if (p == NULL) { + return NGHTTP2_ERR_NOMEM; + } + + *rcbuf_ptr = (void *)p; + + (*rcbuf_ptr)->mem_user_data = mem->mem_user_data; + (*rcbuf_ptr)->free = mem->free; + (*rcbuf_ptr)->base = p + sizeof(nghttp2_rcbuf); + (*rcbuf_ptr)->len = size; + (*rcbuf_ptr)->ref = 1; + + return 0; +} + +int nghttp2_rcbuf_new2(nghttp2_rcbuf **rcbuf_ptr, const uint8_t *src, + size_t srclen, nghttp2_mem *mem) { + int rv; + + rv = nghttp2_rcbuf_new(rcbuf_ptr, srclen + 1, mem); + if (rv != 0) { + return rv; + } + + memcpy((*rcbuf_ptr)->base, src, srclen); + + (*rcbuf_ptr)->len = srclen; + (*rcbuf_ptr)->base[srclen] = '\0'; + + return 0; +} + +/* + * Frees |rcbuf| itself, regardless of its reference cout. + */ +void nghttp2_rcbuf_del(nghttp2_rcbuf *rcbuf) { + nghttp2_mem_free2(rcbuf->free, rcbuf, rcbuf->mem_user_data); +} + +void nghttp2_rcbuf_incref(nghttp2_rcbuf *rcbuf) { + if (rcbuf->ref == -1) { + return; + } + + ++rcbuf->ref; +} + +void nghttp2_rcbuf_decref(nghttp2_rcbuf *rcbuf) { + if (rcbuf == NULL || rcbuf->ref == -1) { + return; + } + + assert(rcbuf->ref > 0); + + if (--rcbuf->ref == 0) { + nghttp2_rcbuf_del(rcbuf); + } +} + +nghttp2_vec nghttp2_rcbuf_get_buf(nghttp2_rcbuf *rcbuf) { + return (nghttp2_vec){rcbuf->base, rcbuf->len}; +} diff --git a/lib/nghttp2_rcbuf.h b/lib/nghttp2_rcbuf.h new file mode 100644 index 00000000..29d1543e --- /dev/null +++ b/lib/nghttp2_rcbuf.h @@ -0,0 +1,80 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef NGHTTP2_RCBUF_H +#define NGHTTP2_RCBUF_H + +#ifdef HAVE_CONFIG_H +#include +#endif /* HAVE_CONFIG_H */ + +#include + +struct nghttp2_rcbuf { + /* custom memory allocator belongs to the mem parameter when + creating this object. */ + void *mem_user_data; + nghttp2_free free; + /* The pointer to the underlying buffer */ + uint8_t *base; + /* Size of buffer pointed by |base|. */ + size_t len; + /* Reference count */ + int32_t ref; +}; + +/* + * Allocates nghttp2_rcbuf object with |size| as initial buffer size. + * When the function succeeds, the reference count becomes 1. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * NGHTTP2_ERR_NOMEM: + * Out of memory. + */ +int nghttp2_rcbuf_new(nghttp2_rcbuf **rcbuf_ptr, size_t size, nghttp2_mem *mem); + +/* + * Like nghttp2_rcbuf_new(), but initializes the buffer with |src| of + * length |srclen|. This function allocates additional byte at the + * end and puts '\0' into it, so that the resulting buffer could be + * used as NULL-terminated string. Still (*rcbuf_ptr)->len equals to + * |srclen|. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * NGHTTP2_ERR_NOMEM: + * Out of memory. + */ +int nghttp2_rcbuf_new2(nghttp2_rcbuf **rcbuf_ptr, const uint8_t *src, + size_t srclen, nghttp2_mem *mem); + +/* + * Frees |rcbuf| itself, regardless of its reference cout. + */ +void nghttp2_rcbuf_del(nghttp2_rcbuf *rcbuf); + +#endif /* NGHTTP2_RCBUF_H */ diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index cbc1bac6..92b1c482 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -3122,12 +3122,12 @@ static int session_call_on_begin_headers(nghttp2_session *session, static int session_call_on_header(nghttp2_session *session, const nghttp2_frame *frame, - const nghttp2_nv *nv) { + const nghttp2_hd_nv *nv) { int rv; if (session->callbacks.on_header_callback) { rv = session->callbacks.on_header_callback( - session, frame, nv->name, nv->namelen, nv->value, nv->valuelen, - nv->flags, session->user_data); + session, frame, nv->name->base, nv->name->len, nv->value->base, + nv->value->len, nv->flags, session->user_data); if (rv == NGHTTP2_ERR_PAUSE || rv == NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE) { return rv; @@ -3317,11 +3317,10 @@ static int inflate_header_block(nghttp2_session *session, nghttp2_frame *frame, ssize_t proclen; int rv; int inflate_flags; - nghttp2_nv nv; + nghttp2_hd_nv nv; nghttp2_stream *stream; nghttp2_stream *subject_stream; int trailer = 0; - int token; *readlen_ptr = 0; stream = nghttp2_session_get_stream(session, frame->hd.stream_id); @@ -3338,7 +3337,7 @@ static int inflate_header_block(nghttp2_session *session, nghttp2_frame *frame, for (;;) { inflate_flags = 0; proclen = nghttp2_hd_inflate_hd2(&session->hd_inflater, &nv, &inflate_flags, - &token, in, inlen, final); + in, inlen, final); if (nghttp2_is_fatal((int)proclen)) { return (int)proclen; } @@ -3373,13 +3372,13 @@ static int inflate_header_block(nghttp2_session *session, nghttp2_frame *frame, if (call_header_cb && (inflate_flags & NGHTTP2_HD_INFLATE_EMIT)) { rv = 0; if (subject_stream && session_enforce_http_messaging(session)) { - rv = nghttp2_http_on_header(session, subject_stream, frame, &nv, token, + rv = nghttp2_http_on_header(session, subject_stream, frame, &nv, trailer); if (rv == NGHTTP2_ERR_HTTP_HEADER) { DEBUGF(fprintf( stderr, "recv: HTTP error: type=%d, id=%d, header %.*s: %.*s\n", - frame->hd.type, subject_stream->stream_id, (int)nv.namelen, - nv.name, (int)nv.valuelen, nv.value)); + frame->hd.type, subject_stream->stream_id, (int)nv.name->len, + nv.name->base, (int)nv.value->len, nv.value->base)); rv = session_handle_invalid_stream2(session, subject_stream->stream_id, @@ -3394,8 +3393,8 @@ static int inflate_header_block(nghttp2_session *session, nghttp2_frame *frame, /* header is ignored */ DEBUGF(fprintf( stderr, "recv: HTTP ignored: type=%d, id=%d, header %.*s: %.*s\n", - frame->hd.type, subject_stream->stream_id, (int)nv.namelen, - nv.name, (int)nv.valuelen, nv.value)); + frame->hd.type, subject_stream->stream_id, (int)nv.name->len, + nv.name->base, (int)nv.value->len, nv.value->base)); } } if (rv == 0) { diff --git a/tests/nghttp2_hd_test.c b/tests/nghttp2_hd_test.c index ed954bde..3d8ab83b 100644 --- a/tests/nghttp2_hd_test.c +++ b/tests/nghttp2_hd_test.c @@ -295,11 +295,6 @@ void test_nghttp2_hd_inflate_indname_inc(void) { assert_nv_equal(&nv, out.nva, 1, mem); CU_ASSERT(1 == inflater.ctx.hd_table.len); CU_ASSERT(62 == nghttp2_hd_inflate_get_num_table_entries(&inflater)); - assert_nv_equal(&nv, - &nghttp2_hd_table_get(&inflater.ctx, - NGHTTP2_STATIC_TABLE_LENGTH + - inflater.ctx.hd_table.len - 1)->nv, - 1, mem); assert_nv_equal(&nv, nghttp2_hd_inflate_get_table_entry( &inflater, NGHTTP2_STATIC_TABLE_LENGTH + inflater.ctx.hd_table.len), @@ -429,10 +424,9 @@ void test_nghttp2_hd_inflate_newname_inc(void) { CU_ASSERT(1 == out.nvlen); assert_nv_equal(&nv, out.nva, 1, mem); CU_ASSERT(1 == inflater.ctx.hd_table.len); - assert_nv_equal(&nv, - &nghttp2_hd_table_get(&inflater.ctx, - NGHTTP2_STATIC_TABLE_LENGTH + - inflater.ctx.hd_table.len - 1)->nv, + assert_nv_equal(&nv, nghttp2_hd_inflate_get_table_entry( + &inflater, NGHTTP2_STATIC_TABLE_LENGTH + + inflater.ctx.hd_table.len), 1, mem); nva_out_reset(&out, mem); From 689d2a1afbc683f80e9241440ff458d61f1e09be Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 13:24:02 +0900 Subject: [PATCH 08/19] Fix compile error with --enable-debug --- lib/nghttp2_stream.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/nghttp2_stream.c b/lib/nghttp2_stream.c index 70325bc3..3221fbf8 100644 --- a/lib/nghttp2_stream.c +++ b/lib/nghttp2_stream.c @@ -152,7 +152,7 @@ static int stream_obq_push(nghttp2_stream *dep_stream, nghttp2_stream *stream) { stream_next_cycle(stream, dep_stream->descendant_last_cycle); stream->seq = dep_stream->descendant_next_seq++; - DEBUGF(fprintf(stderr, "stream: stream=%d obq push cycle=%ld\n", + DEBUGF(fprintf(stderr, "stream: stream=%d obq push cycle=%d\n", stream->stream_id, stream->cycle)); DEBUGF(fprintf(stderr, "stream: push stream %d to stream %d\n", @@ -238,7 +238,7 @@ void nghttp2_stream_reschedule(nghttp2_stream *stream) { nghttp2_pq_push(&dep_stream->obq, &stream->pq_entry); - DEBUGF(fprintf(stderr, "stream: stream=%d obq resched cycle=%ld\n", + DEBUGF(fprintf(stderr, "stream: stream=%d obq resched cycle=%d\n", stream->stream_id, stream->cycle)); dep_stream->last_writelen = stream->last_writelen; @@ -298,7 +298,7 @@ void nghttp2_stream_change_weight(nghttp2_stream *stream, int32_t weight) { nghttp2_pq_push(&dep_stream->obq, &stream->pq_entry); - DEBUGF(fprintf(stderr, "stream: stream=%d obq resched cycle=%ld\n", + DEBUGF(fprintf(stderr, "stream: stream=%d obq resched cycle=%d\n", stream->stream_id, stream->cycle)); } From 12dad328902bdccdfbde8d803b73bd82295b7e47 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 15:05:20 +0900 Subject: [PATCH 09/19] Add nghttp2_on_header_callback2 --- doc/Makefile.am | 4 +++ lib/includes/nghttp2/nghttp2.h | 49 ++++++++++++++++++++++++--- lib/nghttp2_callbacks.c | 6 ++++ lib/nghttp2_callbacks.h | 1 + lib/nghttp2_session.c | 22 +++++++----- src/shrpx_downstream.cc | 9 +++++ src/shrpx_downstream.h | 4 +++ src/shrpx_http2_session.cc | 61 +++++++++++++++++++--------------- src/shrpx_http2_upstream.cc | 41 ++++++++++++----------- 9 files changed, 139 insertions(+), 58 deletions(-) diff --git a/doc/Makefile.am b/doc/Makefile.am index 35f1390b..c450a029 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -65,6 +65,9 @@ APIDOCS= \ nghttp2_priority_spec_check_default.rst \ nghttp2_priority_spec_default_init.rst \ nghttp2_priority_spec_init.rst \ + nghttp2_rcbuf_decref.rst \ + nghttp2_rcbuf_get_buf.rst \ + nghttp2_rcbuf_incref.rst \ nghttp2_select_next_protocol.rst \ nghttp2_session_callbacks_del.rst \ nghttp2_session_callbacks_new.rst \ @@ -78,6 +81,7 @@ APIDOCS= \ nghttp2_session_callbacks_set_on_frame_recv_callback.rst \ nghttp2_session_callbacks_set_on_frame_send_callback.rst \ nghttp2_session_callbacks_set_on_header_callback.rst \ + nghttp2_session_callbacks_set_on_header_callback2.rst \ nghttp2_session_callbacks_set_on_invalid_frame_recv_callback.rst \ nghttp2_session_callbacks_set_on_stream_close_callback.rst \ nghttp2_session_callbacks_set_pack_extension_callback.rst \ diff --git a/lib/includes/nghttp2/nghttp2.h b/lib/includes/nghttp2/nghttp2.h index 2a3674fb..0e888fa7 100644 --- a/lib/includes/nghttp2/nghttp2.h +++ b/lib/includes/nghttp2/nghttp2.h @@ -450,7 +450,7 @@ typedef struct nghttp2_rcbuf nghttp2_rcbuf; * * Increments the reference count of |rcbuf| by 1. */ -void nghttp2_rcbuf_incref(nghttp2_rcbuf *rcbuf); +NGHTTP2_EXTERN void nghttp2_rcbuf_incref(nghttp2_rcbuf *rcbuf); /** * @function @@ -459,12 +459,14 @@ void nghttp2_rcbuf_incref(nghttp2_rcbuf *rcbuf); * count becomes zero, the object pointed by |rcbuf| will be freed. * In this case, application must not use |rcbuf| again. */ -void nghttp2_rcbuf_decref(nghttp2_rcbuf *rcbuf); +NGHTTP2_EXTERN void nghttp2_rcbuf_decref(nghttp2_rcbuf *rcbuf); /** + * @function + * * Returns the underlying buffer managed by |rcbuf|. */ -nghttp2_vec nghttp2_rcbuf_get_buf(nghttp2_rcbuf *rcbuf); +NGHTTP2_EXTERN nghttp2_vec nghttp2_rcbuf_get_buf(nghttp2_rcbuf *rcbuf); /** * @enum @@ -1674,6 +1676,32 @@ typedef int (*nghttp2_on_header_callback)(nghttp2_session *session, const uint8_t *value, size_t valuelen, uint8_t flags, void *user_data); +/** + * @functypedef + * + * Callback function invoked when a header name/value pair is received + * for the |frame|. The |name| is header name. The |value| is header + * value. The |flags| is bitwise OR of one or more of + * :type:`nghttp2_nv_flag`. + * + * This callback behaves like :type:`nghttp2_on_header_callback`, + * except that |name| and |value| are stored in reference counted + * buffer. If application wishes to keep these references without + * copying them, use `nghttp2_rcbuf_incref()` to increment their + * reference count. It is the application's responsibility to call + * `nghttp2_rcbuf_decref()` if they called `nghttp2_rcbuf_incref()` so + * as not to leak memory. If the |session| is created by + * `nghttp2_session_server_new3()` or `nghttp2_session_client_new3()`, + * the function to free memory is the one belongs to the mem + * parameter. As long as this free function alives, |name| and + * |value| can live after |session| was destroyed. + */ +typedef int (*nghttp2_on_header_callback2)(nghttp2_session *session, + const nghttp2_frame *frame, + nghttp2_rcbuf *name, + nghttp2_rcbuf *value, uint8_t flags, + void *user_data); + /** * @functypedef * @@ -1988,12 +2016,25 @@ NGHTTP2_EXTERN void nghttp2_session_callbacks_set_on_begin_headers_callback( * @function * * Sets callback function invoked when a header name/value pair is - * received. + * received. If both + * `nghttp2_session_callbacks_set_on_header_callback()` and + * `nghttp2_session_callbacks_set_on_header_callback2()` are used to + * set callbacks, the latter has the precedence. */ NGHTTP2_EXTERN void nghttp2_session_callbacks_set_on_header_callback( nghttp2_session_callbacks *cbs, nghttp2_on_header_callback on_header_callback); +/** + * @function + * + * Sets callback function invoked when a header name/value pair is + * received. + */ +NGHTTP2_EXTERN void nghttp2_session_callbacks_set_on_header_callback2( + nghttp2_session_callbacks *cbs, + nghttp2_on_header_callback2 on_header_callback2); + /** * @function * diff --git a/lib/nghttp2_callbacks.c b/lib/nghttp2_callbacks.c index 3b0369c1..4bf0e7a5 100644 --- a/lib/nghttp2_callbacks.c +++ b/lib/nghttp2_callbacks.c @@ -104,6 +104,12 @@ void nghttp2_session_callbacks_set_on_header_callback( cbs->on_header_callback = on_header_callback; } +void nghttp2_session_callbacks_set_on_header_callback2( + nghttp2_session_callbacks *cbs, + nghttp2_on_header_callback2 on_header_callback2) { + cbs->on_header_callback2 = on_header_callback2; +} + void nghttp2_session_callbacks_set_select_padding_callback( nghttp2_session_callbacks *cbs, nghttp2_select_padding_callback select_padding_callback) { diff --git a/lib/nghttp2_callbacks.h b/lib/nghttp2_callbacks.h index 664bf35b..80971c54 100644 --- a/lib/nghttp2_callbacks.h +++ b/lib/nghttp2_callbacks.h @@ -91,6 +91,7 @@ struct nghttp2_session_callbacks { * received. */ nghttp2_on_header_callback on_header_callback; + nghttp2_on_header_callback2 on_header_callback2; /** * Callback function invoked when the library asks application how * many padding bytes are required for the transmission of the given diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 92b1c482..44ed7203 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -3123,19 +3123,23 @@ static int session_call_on_begin_headers(nghttp2_session *session, static int session_call_on_header(nghttp2_session *session, const nghttp2_frame *frame, const nghttp2_hd_nv *nv) { - int rv; - if (session->callbacks.on_header_callback) { + int rv = 0; + if (session->callbacks.on_header_callback2) { + rv = session->callbacks.on_header_callback2( + session, frame, nv->name, nv->value, nv->flags, session->user_data); + } else if (session->callbacks.on_header_callback) { rv = session->callbacks.on_header_callback( session, frame, nv->name->base, nv->name->len, nv->value->base, nv->value->len, nv->flags, session->user_data); - if (rv == NGHTTP2_ERR_PAUSE || - rv == NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE) { - return rv; - } - if (rv != 0) { - return NGHTTP2_ERR_CALLBACK_FAILURE; - } } + + if (rv == NGHTTP2_ERR_PAUSE || rv == NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE) { + return rv; + } + if (rv != 0) { + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + return 0; } diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 51931aad..0cdd81b8 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -182,6 +182,10 @@ Downstream::~Downstream() { // explicitly. dconn_.reset(); + for (auto rcbuf : rcbufs_) { + nghttp2_rcbuf_decref(rcbuf); + } + if (LOG_ENABLED(INFO)) { DLOG(INFO, this) << "Deleted"; } @@ -905,4 +909,9 @@ int32_t Downstream::get_assoc_stream_id() const { return assoc_stream_id_; } BlockAllocator &Downstream::get_block_allocator() { return balloc_; } +void Downstream::add_rcbuf(nghttp2_rcbuf *rcbuf) { + nghttp2_rcbuf_incref(rcbuf); + rcbufs_.push_back(rcbuf); +} + } // namespace shrpx diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index a5024169..0d987517 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -374,6 +374,8 @@ public: BlockAllocator &get_block_allocator(); + void add_rcbuf(nghttp2_rcbuf *rcbuf); + enum { EVENT_ERROR = 0x1, EVENT_TIMEOUT = 0x2, @@ -395,6 +397,8 @@ public: private: BlockAllocator balloc_; + std::vector rcbufs_; + Request req_; Response resp_; diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 25a7f996..84258896 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -801,10 +801,9 @@ void Http2Session::stop_settings_timer() { } namespace { -int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, - const uint8_t *name, size_t namelen, - const uint8_t *value, size_t valuelen, uint8_t flags, - void *user_data) { +int on_header_callback2(nghttp2_session *session, const nghttp2_frame *frame, + nghttp2_rcbuf *name, nghttp2_rcbuf *value, + uint8_t flags, void *user_data) { auto http2session = static_cast(user_data); auto sd = static_cast( nghttp2_session_get_stream_user_data(session, frame->hd.stream_id)); @@ -816,22 +815,25 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, return 0; } + auto namebuf = nghttp2_rcbuf_get_buf(name); + auto valuebuf = nghttp2_rcbuf_get_buf(value); + auto &resp = downstream->response(); auto &httpconf = get_config()->http; - auto &balloc = downstream->get_block_allocator(); switch (frame->hd.type) { case NGHTTP2_HEADERS: { auto trailer = frame->headers.cat == NGHTTP2_HCAT_HEADERS && !downstream->get_expect_final_response(); - if (resp.fs.buffer_size() + namelen + valuelen > + if (resp.fs.buffer_size() + namebuf.len + valuebuf.len > httpconf.response_header_field_buffer || resp.fs.num_fields() >= httpconf.max_response_header_fields) { if (LOG_ENABLED(INFO)) { - DLOG(INFO, downstream) << "Too large or many header field size=" - << resp.fs.buffer_size() + namelen + valuelen - << ", num=" << resp.fs.num_fields() + 1; + DLOG(INFO, downstream) + << "Too large or many header field size=" + << resp.fs.buffer_size() + namebuf.len + valuebuf.len + << ", num=" << resp.fs.num_fields() + 1; } if (trailer) { @@ -843,20 +845,23 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; } - auto token = http2::lookup_token(name, namelen); + auto token = http2::lookup_token(namebuf.base, namebuf.len); auto no_index = flags & NGHTTP2_NV_FLAG_NO_INDEX; + downstream->add_rcbuf(name); + downstream->add_rcbuf(value); + if (trailer) { // just store header fields for trailer part - resp.fs.add_trailer_token( - make_string_ref(balloc, StringRef{name, namelen}), - make_string_ref(balloc, StringRef{value, valuelen}), no_index, token); + resp.fs.add_trailer_token(StringRef{namebuf.base, namebuf.len}, + StringRef{valuebuf.base, valuebuf.len}, + no_index, token); return 0; } - resp.fs.add_header_token( - make_string_ref(balloc, StringRef{name, namelen}), - make_string_ref(balloc, StringRef{value, valuelen}), no_index, token); + resp.fs.add_header_token(StringRef{namebuf.base, namebuf.len}, + StringRef{valuebuf.base, valuebuf.len}, no_index, + token); return 0; } case NGHTTP2_PUSH_PROMISE: { @@ -870,30 +875,34 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, auto promised_downstream = promised_sd->dconn->get_downstream(); + auto namebuf = nghttp2_rcbuf_get_buf(name); + auto valuebuf = nghttp2_rcbuf_get_buf(value); + assert(promised_downstream); auto &promised_req = promised_downstream->request(); - auto &promised_balloc = promised_downstream->get_block_allocator(); // We use request header limit for PUSH_PROMISE - if (promised_req.fs.buffer_size() + namelen + valuelen > + if (promised_req.fs.buffer_size() + namebuf.len + valuebuf.len > httpconf.request_header_field_buffer || promised_req.fs.num_fields() >= httpconf.max_request_header_fields) { if (LOG_ENABLED(INFO)) { DLOG(INFO, downstream) << "Too large or many header field size=" - << promised_req.fs.buffer_size() + namelen + valuelen + << promised_req.fs.buffer_size() + namebuf.len + valuebuf.len << ", num=" << promised_req.fs.num_fields() + 1; } return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; } - auto token = http2::lookup_token(name, namelen); - promised_req.fs.add_header_token( - make_string_ref(promised_balloc, StringRef{name, namelen}), - make_string_ref(promised_balloc, StringRef{value, valuelen}), - flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + promised_downstream->add_rcbuf(name); + promised_downstream->add_rcbuf(value); + + auto token = http2::lookup_token(namebuf.base, namebuf.len); + promised_req.fs.add_header_token(StringRef{namebuf.base, namebuf.len}, + StringRef{valuebuf.base, valuebuf.len}, + flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } @@ -1402,8 +1411,8 @@ nghttp2_session_callbacks *create_http2_downstream_callbacks() { nghttp2_session_callbacks_set_on_frame_not_send_callback( callbacks, on_frame_not_send_callback); - nghttp2_session_callbacks_set_on_header_callback(callbacks, - on_header_callback); + nghttp2_session_callbacks_set_on_header_callback2(callbacks, + on_header_callback2); nghttp2_session_callbacks_set_on_begin_headers_callback( callbacks, on_begin_headers_callback); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index a3c78572..79fac5de 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -154,13 +154,15 @@ void Http2Upstream::stop_settings_timer() { } namespace { -int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, - const uint8_t *name, size_t namelen, - const uint8_t *value, size_t valuelen, uint8_t flags, - void *user_data) { +int on_header_callback2(nghttp2_session *session, const nghttp2_frame *frame, + nghttp2_rcbuf *name, nghttp2_rcbuf *value, + uint8_t flags, void *user_data) { + auto namebuf = nghttp2_rcbuf_get_buf(name); + auto valuebuf = nghttp2_rcbuf_get_buf(value); + if (get_config()->http2.upstream.debug.frame_debug) { - verbose_on_header_callback(session, frame, name, namelen, value, valuelen, - flags, user_data); + verbose_on_header_callback(session, frame, namebuf.base, namebuf.len, + valuebuf.base, valuebuf.len, flags, user_data); } if (frame->hd.type != NGHTTP2_HEADERS) { return 0; @@ -176,9 +178,7 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, auto &httpconf = get_config()->http; - auto &balloc = downstream->get_block_allocator(); - - if (req.fs.buffer_size() + namelen + valuelen > + if (req.fs.buffer_size() + namebuf.len + valuebuf.len > httpconf.request_header_field_buffer || req.fs.num_fields() >= httpconf.max_request_header_fields) { if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { @@ -187,7 +187,7 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, if (LOG_ENABLED(INFO)) { ULOG(INFO, upstream) << "Too large or many header field size=" - << req.fs.buffer_size() + namelen + valuelen + << req.fs.buffer_size() + namebuf.len + valuebuf.len << ", num=" << req.fs.num_fields() + 1; } @@ -203,20 +203,23 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, return 0; } - auto token = http2::lookup_token(name, namelen); + auto token = http2::lookup_token(namebuf.base, namebuf.len); auto no_index = flags & NGHTTP2_NV_FLAG_NO_INDEX; + downstream->add_rcbuf(name); + downstream->add_rcbuf(value); + if (frame->headers.cat == NGHTTP2_HCAT_HEADERS) { // just store header fields for trailer part - req.fs.add_trailer_token( - make_string_ref(balloc, StringRef{name, namelen}), - make_string_ref(balloc, StringRef{value, valuelen}), no_index, token); + req.fs.add_trailer_token(StringRef{namebuf.base, namebuf.len}, + StringRef{valuebuf.base, valuebuf.len}, no_index, + token); return 0; } - req.fs.add_header_token(make_string_ref(balloc, StringRef{name, namelen}), - make_string_ref(balloc, StringRef{value, valuelen}), - no_index, token); + req.fs.add_header_token(StringRef{namebuf.base, namebuf.len}, + StringRef{valuebuf.base, valuebuf.len}, no_index, + token); return 0; } } // namespace @@ -820,8 +823,8 @@ nghttp2_session_callbacks *create_http2_upstream_callbacks() { nghttp2_session_callbacks_set_on_frame_not_send_callback( callbacks, on_frame_not_send_callback); - nghttp2_session_callbacks_set_on_header_callback(callbacks, - on_header_callback); + nghttp2_session_callbacks_set_on_header_callback2(callbacks, + on_header_callback2); nghttp2_session_callbacks_set_on_begin_headers_callback( callbacks, on_begin_headers_callback); From 3db9c2c7964872239c439010303fdd062921c813 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 15:54:48 +0900 Subject: [PATCH 10/19] Remove extra tokenization of header names; they are not effective at best --- genlibtokenlookup.py | 62 +-------- lib/nghttp2_hd.c | 296 ------------------------------------------- lib/nghttp2_hd.h | 62 +-------- 3 files changed, 6 insertions(+), 414 deletions(-) diff --git a/genlibtokenlookup.py b/genlibtokenlookup.py index 360d2198..e0726c59 100755 --- a/genlibtokenlookup.py +++ b/genlibtokenlookup.py @@ -62,67 +62,11 @@ HEADERS = [ ('vary', 58), ('via', 59), ('www-authenticate', 60), - ('accept-ch', None), - ('accept-datetime', None), - ('accept-features', None), - ('accept-patch', None), - ('access-control-allow-credentials', None), - ('access-control-allow-headers', None), - ('access-control-allow-methods', None), - ('access-control-expose-headers', None), - ('access-control-max-age', None), - ('access-control-request-headers', None), - ('access-control-request-method', None), - ('alt-svc', None), - ('alternates', None), - ('connection', None), - ('content-md5', None), - ('content-security-policy', None), - ('content-security-policy-report-only', None), - ('dnt', None), - ('forwarded', None), - ('front-end-https', None), - ('keep-alive', None), - ('last-event-id', None), - ('negotiate', None), - ('origin', None), - ('p3p', None), - ('pragma', None), - ('proxy-connection', None), - ('public-key-pins', None), - ('sec-websocket-extensions', None), - ('sec-websocket-key', None), - ('sec-websocket-origin', None), - ('sec-websocket-protocol', None), - ('sec-websocket-version', None), - ('set-cookie2', None), - ('status', None), - ('tcn', None), ('te', None), - ('trailer', None), - ('tsv', None), + ('connection', None), + ('keep-alive',None), + ('proxy-connection', None), ('upgrade', None), - ('upgrade-insecure-requests', None), - ('variant-vary', None), - ('warning', None), - ('x-api-version', None), - ('x-att-deviceid', None), - ('x-cache', None), - ('x-cache-lookup', None), - ('x-content-duration', None), - ('x-content-security-policy', None), - ('x-content-type-options', None), - ('x-dnsprefetch-control', None), - ('x-forwarded-for', None), - ('x-forwarded-host', None), - ('x-forwarded-proto', None), - ('x-frame-options', None), - ('x-powered-by', None), - ('x-requested-with', None), - ('x-ua-compatible', None), - ('x-wap-profile', None), - ('x-webkit-csp', None), - ('x-xss-protection', None), ] def to_enum_hd(k): diff --git a/lib/nghttp2_hd.c b/lib/nghttp2_hd.c index 8da40064..3aedc0fd 100644 --- a/lib/nghttp2_hd.c +++ b/lib/nghttp2_hd.c @@ -139,26 +139,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_AGE; } break; - case 'n': - if (lstreq("tc", name, 2)) { - return NGHTTP2_TOKEN_TCN; - } - break; - case 'p': - if (lstreq("p3", name, 2)) { - return NGHTTP2_TOKEN_P3P; - } - break; - case 't': - if (lstreq("dn", name, 2)) { - return NGHTTP2_TOKEN_DNT; - } - break; - case 'v': - if (lstreq("ts", name, 2)) { - return NGHTTP2_TOKEN_TSV; - } - break; } break; case 4: @@ -219,31 +199,16 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { break; case 6: switch (name[5]) { - case 'a': - if (lstreq("pragm", name, 5)) { - return NGHTTP2_TOKEN_PRAGMA; - } - break; case 'e': if (lstreq("cooki", name, 5)) { return NGHTTP2_TOKEN_COOKIE; } break; - case 'n': - if (lstreq("origi", name, 5)) { - return NGHTTP2_TOKEN_ORIGIN; - } - break; case 'r': if (lstreq("serve", name, 5)) { return NGHTTP2_TOKEN_SERVER; } break; - case 's': - if (lstreq("statu", name, 5)) { - return NGHTTP2_TOKEN_STATUS; - } - break; case 't': if (lstreq("accep", name, 5)) { return NGHTTP2_TOKEN_ACCEPT; @@ -256,11 +221,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { break; case 7: switch (name[6]) { - case 'c': - if (lstreq("alt-sv", name, 6)) { - return NGHTTP2_TOKEN_ALT_SVC; - } - break; case 'd': if (lstreq(":metho", name, 6)) { return NGHTTP2_TOKEN__METHOD; @@ -279,14 +239,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("upgrad", name, 6)) { return NGHTTP2_TOKEN_UPGRADE; } - if (lstreq("x-cach", name, 6)) { - return NGHTTP2_TOKEN_X_CACHE; - } - break; - case 'g': - if (lstreq("warnin", name, 6)) { - return NGHTTP2_TOKEN_WARNING; - } break; case 'h': if (lstreq("refres", name, 6)) { @@ -297,9 +249,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("refere", name, 6)) { return NGHTTP2_TOKEN_REFERER; } - if (lstreq("traile", name, 6)) { - return NGHTTP2_TOKEN_TRAILER; - } break; case 's': if (lstreq(":statu", name, 6)) { @@ -348,25 +297,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { break; } break; - case 9: - switch (name[8]) { - case 'd': - if (lstreq("forwarde", name, 8)) { - return NGHTTP2_TOKEN_FORWARDED; - } - break; - case 'e': - if (lstreq("negotiat", name, 8)) { - return NGHTTP2_TOKEN_NEGOTIATE; - } - break; - case 'h': - if (lstreq("accept-c", name, 8)) { - return NGHTTP2_TOKEN_ACCEPT_CH; - } - break; - } - break; case 10: switch (name[9]) { case 'e': @@ -382,11 +312,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_CONNECTION; } break; - case 's': - if (lstreq("alternate", name, 9)) { - return NGHTTP2_TOKEN_ALTERNATES; - } - break; case 't': if (lstreq("user-agen", name, 9)) { return NGHTTP2_TOKEN_USER_AGENT; @@ -401,16 +326,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { break; case 11: switch (name[10]) { - case '2': - if (lstreq("set-cookie", name, 10)) { - return NGHTTP2_TOKEN_SET_COOKIE2; - } - break; - case '5': - if (lstreq("content-md", name, 10)) { - return NGHTTP2_TOKEN_CONTENT_MD5; - } - break; case 'r': if (lstreq("retry-afte", name, 10)) { return NGHTTP2_TOKEN_RETRY_AFTER; @@ -425,37 +340,16 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_CONTENT_TYPE; } break; - case 'h': - if (lstreq("accept-patc", name, 11)) { - return NGHTTP2_TOKEN_ACCEPT_PATCH; - } - break; - case 'p': - if (lstreq("x-webkit-cs", name, 11)) { - return NGHTTP2_TOKEN_X_WEBKIT_CSP; - } - break; case 's': if (lstreq("max-forward", name, 11)) { return NGHTTP2_TOKEN_MAX_FORWARDS; } break; - case 'y': - if (lstreq("variant-var", name, 11)) { - return NGHTTP2_TOKEN_VARIANT_VARY; - } - if (lstreq("x-powered-b", name, 11)) { - return NGHTTP2_TOKEN_X_POWERED_BY; - } - break; } break; case 13: switch (name[12]) { case 'd': - if (lstreq("last-event-i", name, 12)) { - return NGHTTP2_TOKEN_LAST_EVENT_ID; - } if (lstreq("last-modifie", name, 12)) { return NGHTTP2_TOKEN_LAST_MODIFIED; } @@ -464,9 +358,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("content-rang", name, 12)) { return NGHTTP2_TOKEN_CONTENT_RANGE; } - if (lstreq("x-wap-profil", name, 12)) { - return NGHTTP2_TOKEN_X_WAP_PROFILE; - } break; case 'h': if (lstreq("if-none-matc", name, 12)) { @@ -482,9 +373,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("authorizatio", name, 12)) { return NGHTTP2_TOKEN_AUTHORIZATION; } - if (lstreq("x-api-versio", name, 12)) { - return NGHTTP2_TOKEN_X_API_VERSION; - } break; case 's': if (lstreq("accept-range", name, 12)) { @@ -495,21 +383,11 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { break; case 14: switch (name[13]) { - case 'd': - if (lstreq("x-att-devicei", name, 13)) { - return NGHTTP2_TOKEN_X_ATT_DEVICEID; - } - break; case 'h': if (lstreq("content-lengt", name, 13)) { return NGHTTP2_TOKEN_CONTENT_LENGTH; } break; - case 'p': - if (lstreq("x-cache-looku", name, 13)) { - return NGHTTP2_TOKEN_X_CACHE_LOOKUP; - } - break; case 't': if (lstreq("accept-charse", name, 13)) { return NGHTTP2_TOKEN_ACCEPT_CHARSET; @@ -520,40 +398,15 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { case 15: switch (name[14]) { case 'e': - if (lstreq("accept-datetim", name, 14)) { - return NGHTTP2_TOKEN_ACCEPT_DATETIME; - } if (lstreq("accept-languag", name, 14)) { return NGHTTP2_TOKEN_ACCEPT_LANGUAGE; } - if (lstreq("x-ua-compatibl", name, 14)) { - return NGHTTP2_TOKEN_X_UA_COMPATIBLE; - } break; case 'g': if (lstreq("accept-encodin", name, 14)) { return NGHTTP2_TOKEN_ACCEPT_ENCODING; } break; - case 'r': - if (lstreq("x-forwarded-fo", name, 14)) { - return NGHTTP2_TOKEN_X_FORWARDED_FOR; - } - break; - case 's': - if (lstreq("accept-feature", name, 14)) { - return NGHTTP2_TOKEN_ACCEPT_FEATURES; - } - if (lstreq("front-end-http", name, 14)) { - return NGHTTP2_TOKEN_FRONT_END_HTTPS; - } - if (lstreq("public-key-pin", name, 14)) { - return NGHTTP2_TOKEN_PUBLIC_KEY_PINS; - } - if (lstreq("x-frame-option", name, 14)) { - return NGHTTP2_TOKEN_X_FRAME_OPTIONS; - } - break; } break; case 16: @@ -571,11 +424,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_CONTENT_ENCODING; } break; - case 'h': - if (lstreq("x-requested-wit", name, 15)) { - return NGHTTP2_TOKEN_X_REQUESTED_WITH; - } - break; case 'n': if (lstreq("content-locatio", name, 15)) { return NGHTTP2_TOKEN_CONTENT_LOCATION; @@ -583,14 +431,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("proxy-connectio", name, 15)) { return NGHTTP2_TOKEN_PROXY_CONNECTION; } - if (lstreq("x-xss-protectio", name, 15)) { - return NGHTTP2_TOKEN_X_XSS_PROTECTION; - } - break; - case 't': - if (lstreq("x-forwarded-hos", name, 15)) { - return NGHTTP2_TOKEN_X_FORWARDED_HOST; - } break; } break; @@ -606,16 +446,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_TRANSFER_ENCODING; } break; - case 'o': - if (lstreq("x-forwarded-prot", name, 16)) { - return NGHTTP2_TOKEN_X_FORWARDED_PROTO; - } - break; - case 'y': - if (lstreq("sec-websocket-ke", name, 16)) { - return NGHTTP2_TOKEN_SEC_WEBSOCKET_KEY; - } - break; } break; case 18: @@ -625,11 +455,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_PROXY_AUTHENTICATE; } break; - case 'n': - if (lstreq("x-content-duratio", name, 17)) { - return NGHTTP2_TOKEN_X_CONTENT_DURATION; - } - break; } break; case 19: @@ -649,80 +474,12 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { break; } break; - case 20: - switch (name[19]) { - case 'n': - if (lstreq("sec-websocket-origi", name, 19)) { - return NGHTTP2_TOKEN_SEC_WEBSOCKET_ORIGIN; - } - break; - } - break; - case 21: - switch (name[20]) { - case 'l': - if (lstreq("x-dnsprefetch-contro", name, 20)) { - return NGHTTP2_TOKEN_X_DNSPREFETCH_CONTROL; - } - break; - case 'n': - if (lstreq("sec-websocket-versio", name, 20)) { - return NGHTTP2_TOKEN_SEC_WEBSOCKET_VERSION; - } - break; - } - break; - case 22: - switch (name[21]) { - case 'e': - if (lstreq("access-control-max-ag", name, 21)) { - return NGHTTP2_TOKEN_ACCESS_CONTROL_MAX_AGE; - } - break; - case 'l': - if (lstreq("sec-websocket-protoco", name, 21)) { - return NGHTTP2_TOKEN_SEC_WEBSOCKET_PROTOCOL; - } - break; - case 's': - if (lstreq("x-content-type-option", name, 21)) { - return NGHTTP2_TOKEN_X_CONTENT_TYPE_OPTIONS; - } - break; - } - break; - case 23: - switch (name[22]) { - case 'y': - if (lstreq("content-security-polic", name, 22)) { - return NGHTTP2_TOKEN_CONTENT_SECURITY_POLICY; - } - break; - } - break; - case 24: - switch (name[23]) { - case 's': - if (lstreq("sec-websocket-extension", name, 23)) { - return NGHTTP2_TOKEN_SEC_WEBSOCKET_EXTENSIONS; - } - break; - } - break; case 25: switch (name[24]) { - case 's': - if (lstreq("upgrade-insecure-request", name, 24)) { - return NGHTTP2_TOKEN_UPGRADE_INSECURE_REQUESTS; - } - break; case 'y': if (lstreq("strict-transport-securit", name, 24)) { return NGHTTP2_TOKEN_STRICT_TRANSPORT_SECURITY; } - if (lstreq("x-content-security-polic", name, 24)) { - return NGHTTP2_TOKEN_X_CONTENT_SECURITY_POLICY; - } break; } break; @@ -735,59 +492,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { break; } break; - case 28: - switch (name[27]) { - case 's': - if (lstreq("access-control-allow-header", name, 27)) { - return NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_HEADERS; - } - if (lstreq("access-control-allow-method", name, 27)) { - return NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_METHODS; - } - break; - } - break; - case 29: - switch (name[28]) { - case 'd': - if (lstreq("access-control-request-metho", name, 28)) { - return NGHTTP2_TOKEN_ACCESS_CONTROL_REQUEST_METHOD; - } - break; - case 's': - if (lstreq("access-control-expose-header", name, 28)) { - return NGHTTP2_TOKEN_ACCESS_CONTROL_EXPOSE_HEADERS; - } - break; - } - break; - case 30: - switch (name[29]) { - case 's': - if (lstreq("access-control-request-header", name, 29)) { - return NGHTTP2_TOKEN_ACCESS_CONTROL_REQUEST_HEADERS; - } - break; - } - break; - case 32: - switch (name[31]) { - case 's': - if (lstreq("access-control-allow-credential", name, 31)) { - return NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_CREDENTIALS; - } - break; - } - break; - case 35: - switch (name[34]) { - case 'y': - if (lstreq("content-security-policy-report-onl", name, 34)) { - return NGHTTP2_TOKEN_CONTENT_SECURITY_POLICY_REPORT_ONLY; - } - break; - } - break; } return -1; } diff --git a/lib/nghttp2_hd.h b/lib/nghttp2_hd.h index cacfcf3a..b0e93852 100644 --- a/lib/nghttp2_hd.h +++ b/lib/nghttp2_hd.h @@ -106,67 +106,11 @@ typedef enum { NGHTTP2_TOKEN_VARY = 58, NGHTTP2_TOKEN_VIA = 59, NGHTTP2_TOKEN_WWW_AUTHENTICATE = 60, - NGHTTP2_TOKEN_ACCEPT_CH, - NGHTTP2_TOKEN_ACCEPT_DATETIME, - NGHTTP2_TOKEN_ACCEPT_FEATURES, - NGHTTP2_TOKEN_ACCEPT_PATCH, - NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_CREDENTIALS, - NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_HEADERS, - NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_METHODS, - NGHTTP2_TOKEN_ACCESS_CONTROL_EXPOSE_HEADERS, - NGHTTP2_TOKEN_ACCESS_CONTROL_MAX_AGE, - NGHTTP2_TOKEN_ACCESS_CONTROL_REQUEST_HEADERS, - NGHTTP2_TOKEN_ACCESS_CONTROL_REQUEST_METHOD, - NGHTTP2_TOKEN_ALT_SVC, - NGHTTP2_TOKEN_ALTERNATES, - NGHTTP2_TOKEN_CONNECTION, - NGHTTP2_TOKEN_CONTENT_MD5, - NGHTTP2_TOKEN_CONTENT_SECURITY_POLICY, - NGHTTP2_TOKEN_CONTENT_SECURITY_POLICY_REPORT_ONLY, - NGHTTP2_TOKEN_DNT, - NGHTTP2_TOKEN_FORWARDED, - NGHTTP2_TOKEN_FRONT_END_HTTPS, - NGHTTP2_TOKEN_KEEP_ALIVE, - NGHTTP2_TOKEN_LAST_EVENT_ID, - NGHTTP2_TOKEN_NEGOTIATE, - NGHTTP2_TOKEN_ORIGIN, - NGHTTP2_TOKEN_P3P, - NGHTTP2_TOKEN_PRAGMA, - NGHTTP2_TOKEN_PROXY_CONNECTION, - NGHTTP2_TOKEN_PUBLIC_KEY_PINS, - NGHTTP2_TOKEN_SEC_WEBSOCKET_EXTENSIONS, - NGHTTP2_TOKEN_SEC_WEBSOCKET_KEY, - NGHTTP2_TOKEN_SEC_WEBSOCKET_ORIGIN, - NGHTTP2_TOKEN_SEC_WEBSOCKET_PROTOCOL, - NGHTTP2_TOKEN_SEC_WEBSOCKET_VERSION, - NGHTTP2_TOKEN_SET_COOKIE2, - NGHTTP2_TOKEN_STATUS, - NGHTTP2_TOKEN_TCN, NGHTTP2_TOKEN_TE, - NGHTTP2_TOKEN_TRAILER, - NGHTTP2_TOKEN_TSV, + NGHTTP2_TOKEN_CONNECTION, + NGHTTP2_TOKEN_KEEP_ALIVE, + NGHTTP2_TOKEN_PROXY_CONNECTION, NGHTTP2_TOKEN_UPGRADE, - NGHTTP2_TOKEN_UPGRADE_INSECURE_REQUESTS, - NGHTTP2_TOKEN_VARIANT_VARY, - NGHTTP2_TOKEN_WARNING, - NGHTTP2_TOKEN_X_API_VERSION, - NGHTTP2_TOKEN_X_ATT_DEVICEID, - NGHTTP2_TOKEN_X_CACHE, - NGHTTP2_TOKEN_X_CACHE_LOOKUP, - NGHTTP2_TOKEN_X_CONTENT_DURATION, - NGHTTP2_TOKEN_X_CONTENT_SECURITY_POLICY, - NGHTTP2_TOKEN_X_CONTENT_TYPE_OPTIONS, - NGHTTP2_TOKEN_X_DNSPREFETCH_CONTROL, - NGHTTP2_TOKEN_X_FORWARDED_FOR, - NGHTTP2_TOKEN_X_FORWARDED_HOST, - NGHTTP2_TOKEN_X_FORWARDED_PROTO, - NGHTTP2_TOKEN_X_FRAME_OPTIONS, - NGHTTP2_TOKEN_X_POWERED_BY, - NGHTTP2_TOKEN_X_REQUESTED_WITH, - NGHTTP2_TOKEN_X_UA_COMPATIBLE, - NGHTTP2_TOKEN_X_WAP_PROFILE, - NGHTTP2_TOKEN_X_WEBKIT_CSP, - NGHTTP2_TOKEN_X_XSS_PROTECTION, } nghttp2_token; struct nghttp2_hd_entry; From 863a9441798b001001bade04cee6d5d311491d13 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 17:29:07 +0900 Subject: [PATCH 11/19] src: Add specialization for char to avoid reinterpret_cast in constexpr --- src/template.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/template.h b/src/template.h index b15efa02..1664714a 100644 --- a/src/template.h +++ b/src/template.h @@ -401,8 +401,9 @@ public: explicit StringRef(const ImmutableString &s) : base(s.c_str()), len(s.size()) {} explicit StringRef(const char *s) : base(s), len(strlen(s)) {} + constexpr StringRef(const char *s, size_t n) : base(s), len(n) {} template - constexpr StringRef(const CharT *s, size_t n) + StringRef(const CharT *s, size_t n) : base(reinterpret_cast(s)), len(n) {} template StringRef(InputIt first, InputIt last) @@ -413,7 +414,7 @@ public: len(std::distance(first, last)) {} template constexpr static StringRef from_lit(const CharT(&s)[N]) { - return StringRef(s, N - 1); + return StringRef{s, N - 1}; } static StringRef from_maybe_nullptr(const char *s) { if (s == nullptr) { From c897d5b2944a138b0649f188c224dfba72159d5d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 17:31:48 +0900 Subject: [PATCH 12/19] src: Use StringRef in parse_http_date --- src/HttpServer.cc | 2 +- src/util.cc | 2 +- src/util.h | 2 +- src/util_test.cc | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/HttpServer.cc b/src/HttpServer.cc index 03140c3b..5c65295d 100644 --- a/src/HttpServer.cc +++ b/src/HttpServer.cc @@ -1235,7 +1235,7 @@ void prepare_response(Stream *stream, Http2Handler *hd, bool last_mod_found = false; if (ims) { last_mod_found = true; - last_mod = util::parse_http_date(ims->value); + last_mod = util::parse_http_date(StringRef{ims->value}); } auto query_pos = reqpath.find("?"); std::string url; diff --git a/src/util.cc b/src/util.cc index 90304315..35fc8275 100644 --- a/src/util.cc +++ b/src/util.cc @@ -339,7 +339,7 @@ std::string iso8601_date(int64_t ms) { return res; } -time_t parse_http_date(const std::string &s) { +time_t parse_http_date(const StringRef &s) { tm tm{}; char *r = strptime(s.c_str(), "%a, %d %b %Y %H:%M:%S GMT", &tm); if (r == 0) { diff --git a/src/util.h b/src/util.h index 90f03f1f..4f926e2a 100644 --- a/src/util.h +++ b/src/util.h @@ -149,7 +149,7 @@ std::string common_log_date(time_t t); // 2014-11-15T12:58:24.741Z) std::string iso8601_date(int64_t ms); -time_t parse_http_date(const std::string &s); +time_t parse_http_date(const StringRef &s); char upcase(char c); diff --git a/src/util_test.cc b/src/util_test.cc index d99c3a96..0a17e255 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -392,8 +392,8 @@ void test_util_ends_with(void) { } void test_util_parse_http_date(void) { - CU_ASSERT(1001939696 == - util::parse_http_date("Mon, 1 Oct 2001 12:34:56 GMT")); + CU_ASSERT(1001939696 == util::parse_http_date(StringRef::from_lit( + "Mon, 1 Oct 2001 12:34:56 GMT"))); } void test_util_localtime_date(void) { From d64051fedcdafc2ca27e20c76c3da815b76bd071 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 18:00:50 +0900 Subject: [PATCH 13/19] src: Return StringRef from http2::stringify_status --- src/http2.cc | 102 ++++++++++++++--------------- src/http2.h | 2 +- src/shrpx-unittest.cc | 2 + src/shrpx_http2_upstream.cc | 54 +++++++-------- src/shrpx_mruby_module_response.cc | 9 +-- src/util.h | 10 +++ src/util_test.cc | 13 +++- src/util_test.h | 1 + 8 files changed, 105 insertions(+), 88 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index 8385cafe..445ebe44 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -134,107 +134,107 @@ std::string get_status_string(unsigned int status_code) { } } -const char *stringify_status(unsigned int status_code) { +StringRef stringify_status(unsigned int status_code) { switch (status_code) { case 100: - return "100"; + return StringRef::from_lit("100"); case 101: - return "101"; + return StringRef::from_lit("101"); case 200: - return "200"; + return StringRef::from_lit("200"); case 201: - return "201"; + return StringRef::from_lit("201"); case 202: - return "202"; + return StringRef::from_lit("202"); case 203: - return "203"; + return StringRef::from_lit("203"); case 204: - return "204"; + return StringRef::from_lit("204"); case 205: - return "205"; + return StringRef::from_lit("205"); case 206: - return "206"; + return StringRef::from_lit("206"); case 300: - return "300"; + return StringRef::from_lit("300"); case 301: - return "301"; + return StringRef::from_lit("301"); case 302: - return "302"; + return StringRef::from_lit("302"); case 303: - return "303"; + return StringRef::from_lit("303"); case 304: - return "304"; + return StringRef::from_lit("304"); case 305: - return "305"; - // case 306: return "306"; + return StringRef::from_lit("305"); + // case 306: return StringRef::from_lit("306"); case 307: - return "307"; + return StringRef::from_lit("307"); case 308: - return "308"; + return StringRef::from_lit("308"); case 400: - return "400"; + return StringRef::from_lit("400"); case 401: - return "401"; + return StringRef::from_lit("401"); case 402: - return "402"; + return StringRef::from_lit("402"); case 403: - return "403"; + return StringRef::from_lit("403"); case 404: - return "404"; + return StringRef::from_lit("404"); case 405: - return "405"; + return StringRef::from_lit("405"); case 406: - return "406"; + return StringRef::from_lit("406"); case 407: - return "407"; + return StringRef::from_lit("407"); case 408: - return "408"; + return StringRef::from_lit("408"); case 409: - return "409"; + return StringRef::from_lit("409"); case 410: - return "410"; + return StringRef::from_lit("410"); case 411: - return "411"; + return StringRef::from_lit("411"); case 412: - return "412"; + return StringRef::from_lit("412"); case 413: - return "413"; + return StringRef::from_lit("413"); case 414: - return "414"; + return StringRef::from_lit("414"); case 415: - return "415"; + return StringRef::from_lit("415"); case 416: - return "416"; + return StringRef::from_lit("416"); case 417: - return "417"; + return StringRef::from_lit("417"); case 421: - return "421"; + return StringRef::from_lit("421"); case 426: - return "426"; + return StringRef::from_lit("426"); case 428: - return "428"; + return StringRef::from_lit("428"); case 429: - return "429"; + return StringRef::from_lit("429"); case 431: - return "431"; + return StringRef::from_lit("431"); case 451: - return "451"; + return StringRef::from_lit("451"); case 500: - return "500"; + return StringRef::from_lit("500"); case 501: - return "501"; + return StringRef::from_lit("501"); case 502: - return "502"; + return StringRef::from_lit("502"); case 503: - return "503"; + return StringRef::from_lit("503"); case 504: - return "504"; + return StringRef::from_lit("504"); case 505: - return "505"; + return StringRef::from_lit("505"); case 511: - return "511"; + return StringRef::from_lit("511"); default: - return nullptr; + return StringRef{}; } } diff --git a/src/http2.h b/src/http2.h index 48d4916d..0ae9d3b3 100644 --- a/src/http2.h +++ b/src/http2.h @@ -98,7 +98,7 @@ std::string get_status_string(unsigned int status_code); // Returns string version of |status_code|. This function can handle // only predefined status code. Otherwise, returns nullptr. -const char *stringify_status(unsigned int status_code); +StringRef stringify_status(unsigned int status_code); void capitalize(DefaultMemchunks *buf, const StringRef &s); diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index 0f2e2fd2..725838f8 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -143,6 +143,8 @@ int main(int argc, char *argv[]) { !CU_add_test(pSuite, "util_ipv6_numeric_addr", shrpx::test_util_ipv6_numeric_addr) || !CU_add_test(pSuite, "util_utos", shrpx::test_util_utos) || + !CU_add_test(pSuite, "util_make_string_ref_uint", + shrpx::test_util_make_string_ref_uint) || !CU_add_test(pSuite, "util_utos_unit", shrpx::test_util_utos_unit) || !CU_add_test(pSuite, "util_utos_funit", shrpx::test_util_utos_funit) || !CU_add_test(pSuite, "util_parse_uint_with_unit", diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 79fac5de..e54b8e90 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1229,20 +1229,20 @@ int Http2Upstream::send_reply(Downstream *downstream, const uint8_t *body, const auto &resp = downstream->response(); auto &httpconf = get_config()->http; + auto &balloc = downstream->get_block_allocator(); + const auto &headers = resp.fs.headers(); auto nva = std::vector(); // 2 for :status and server nva.reserve(2 + headers.size() + httpconf.add_response_headers.size()); - std::string status_code_str; - auto response_status_const = http2::stringify_status(resp.http_status); - if (response_status_const) { - nva.push_back(http2::make_nv_lc_nocopy(":status", response_status_const)); - } else { - status_code_str = util::utos(resp.http_status); - nva.push_back(http2::make_nv_ls(":status", status_code_str)); + auto response_status = http2::stringify_status(resp.http_status); + if (response_status.empty()) { + response_status = util::make_string_ref_uint(balloc, resp.http_status); } + nva.push_back(http2::make_nv_ls_nocopy(":status", response_status)); + for (auto &kv : headers) { if (kv.name.empty() || kv.name[0] == ':') { continue; @@ -1290,6 +1290,8 @@ int Http2Upstream::error_reply(Downstream *downstream, int rv; auto &resp = downstream->response(); + auto &balloc = downstream->get_block_allocator(); + auto html = http::create_error_html(status_code); resp.http_status = status_code; auto body = downstream->get_response_buf(); @@ -1303,20 +1305,20 @@ int Http2Upstream::error_reply(Downstream *downstream, auto lgconf = log_config(); lgconf->update_tstamp(std::chrono::system_clock::now()); - auto response_status_const = http2::stringify_status(status_code); - auto content_length = util::utos(html.size()); + auto response_status = http2::stringify_status(status_code); + if (response_status.empty()) { + response_status = util::make_string_ref_uint(balloc, status_code); + } - std::string status_code_str; + auto content_length = util::make_string_ref_uint(balloc, html.size()); + auto date = make_string_ref(balloc, StringRef{lgconf->time_http_str}); - auto nva = make_array( - response_status_const - ? http2::make_nv_lc_nocopy(":status", response_status_const) - : http2::make_nv_ls(":status", - (status_code_str = util::utos(status_code))), - http2::make_nv_ll("content-type", "text/html; charset=UTF-8"), - http2::make_nv_ls_nocopy("server", get_config()->http.server_name), - http2::make_nv_ls("content-length", content_length), - http2::make_nv_ls("date", lgconf->time_http_str)); + auto nva = std::array{ + {http2::make_nv_ls_nocopy(":status", response_status), + http2::make_nv_ll("content-type", "text/html; charset=UTF-8"), + http2::make_nv_ls_nocopy("server", get_config()->http.server_name), + http2::make_nv_ls_nocopy("content-length", content_length), + http2::make_nv_ls_nocopy("date", date)}}; rv = nghttp2_submit_response(session_, downstream->get_stream_id(), nva.data(), nva.size(), &data_prd); @@ -1357,6 +1359,8 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { const auto &req = downstream->request(); auto &resp = downstream->response(); + auto &balloc = downstream->get_block_allocator(); + if (LOG_ENABLED(INFO)) { if (downstream->get_non_final_response()) { DLOG(INFO, downstream) << "HTTP non-final response header"; @@ -1396,16 +1400,14 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { nva.reserve(resp.fs.headers().size() + 4 + httpconf.add_response_headers.size()); std::string via_value; - std::string response_status; - auto response_status_const = http2::stringify_status(resp.http_status); - if (response_status_const) { - nva.push_back(http2::make_nv_lc_nocopy(":status", response_status_const)); - } else { - response_status = util::utos(resp.http_status); - nva.push_back(http2::make_nv_ls(":status", response_status)); + auto response_status = http2::stringify_status(resp.http_status); + if (response_status.empty()) { + response_status = util::make_string_ref_uint(balloc, resp.http_status); } + nva.push_back(http2::make_nv_ls_nocopy(":status", response_status)); + if (downstream->get_non_final_response()) { http2::copy_headers_to_nva(nva, resp.fs.headers()); diff --git a/src/shrpx_mruby_module_response.cc b/src/shrpx_mruby_module_response.cc index 65fec390..8a89a058 100644 --- a/src/shrpx_mruby_module_response.cc +++ b/src/shrpx_mruby_module_response.cc @@ -215,14 +215,7 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { bodylen = vallen; } - StringRef content_length; - { - auto iov = make_byte_ref(balloc, str_size("18446744073709551615") + 1); - auto p = iov.base; - p = util::utos(p, bodylen); - *p = '\0'; - content_length = StringRef{iov.base, p}; - } + auto content_length = util::make_string_ref_uint(balloc, bodylen); auto cl = resp.fs.header(http2::HD_CONTENT_LENGTH); if (cl) { diff --git a/src/util.h b/src/util.h index 4f926e2a..2d3208fc 100644 --- a/src/util.h +++ b/src/util.h @@ -51,6 +51,7 @@ #include "template.h" #include "network.h" +#include "allocator.h" namespace nghttp2 { @@ -405,6 +406,15 @@ template OutputIt utos(OutputIt dst, T n) { return res; } +template +StringRef make_string_ref_uint(BlockAllocator &balloc, T n) { + auto iov = make_byte_ref(balloc, str_size("18446744073709551615") + 1); + auto p = iov.base; + p = util::utos(p, n); + *p = '\0'; + return StringRef{iov.base, p}; +} + template std::string utos_unit(T n) { char u = 0; if (n >= (1 << 30)) { diff --git a/src/util_test.cc b/src/util_test.cc index 0a17e255..8a8ee181 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -239,8 +239,17 @@ void test_util_utos(void) { CU_ASSERT(("0" == StringRef{buf, util::utos(buf, 0)})); CU_ASSERT(("123" == StringRef{buf, util::utos(buf, 123)})); - CU_ASSERT(("9223372036854775808" == - StringRef{buf, util::utos(buf, 9223372036854775808ULL)})); + CU_ASSERT(("18446744073709551615" == + StringRef{buf, util::utos(buf, 18446744073709551615ULL)})); +} + +void test_util_make_string_ref_uint(void) { + BlockAllocator balloc(1024, 1024); + + CU_ASSERT("0" == util::make_string_ref_uint(balloc, 0)); + CU_ASSERT("123" == util::make_string_ref_uint(balloc, 123)); + CU_ASSERT("18446744073709551615" == + util::make_string_ref_uint(balloc, 18446744073709551615ULL)); } void test_util_utos_unit(void) { diff --git a/src/util_test.h b/src/util_test.h index f51692aa..b44d5b17 100644 --- a/src/util_test.h +++ b/src/util_test.h @@ -45,6 +45,7 @@ void test_util_http_date(void); void test_util_select_h2(void); void test_util_ipv6_numeric_addr(void); void test_util_utos(void); +void test_util_make_string_ref_uint(void); void test_util_utos_unit(void); void test_util_utos_funit(void); void test_util_parse_uint_with_unit(void); From 67569486d1be3d2b54f393416de89b7a7e2dad97 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 18:36:05 +0900 Subject: [PATCH 14/19] src: Rewrite http:create_via_header_value --- src/shrpx-unittest.cc | 2 ++ src/shrpx_http.cc | 11 ----------- src/shrpx_http.h | 12 +++++++++++- src/shrpx_http2_downstream_connection.cc | 21 ++++++++++++++++----- src/shrpx_http2_upstream.cc | 21 +++++++++++++++------ src/shrpx_http_downstream_connection.cc | 5 ++++- src/shrpx_http_test.cc | 14 ++++++++++++++ src/shrpx_http_test.h | 1 + src/shrpx_https_upstream.cc | 6 ++++-- src/shrpx_spdy_upstream.cc | 6 ++++-- 10 files changed, 71 insertions(+), 28 deletions(-) diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index 725838f8..6a015778 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -123,6 +123,8 @@ int main(int argc, char *argv[]) { shrpx::test_shrpx_worker_match_downstream_addr_group) || !CU_add_test(pSuite, "http_create_forwarded", shrpx::test_shrpx_http_create_forwarded) || + !CU_add_test(pSuite, "http_create_via_header_value", + shrpx::test_shrpx_http_create_via_header_value) || !CU_add_test(pSuite, "util_streq", shrpx::test_util_streq) || !CU_add_test(pSuite, "util_strieq", shrpx::test_util_strieq) || !CU_add_test(pSuite, "util_inp_strlower", diff --git a/src/shrpx_http.cc b/src/shrpx_http.cc index 7bca56dd..071f2841 100644 --- a/src/shrpx_http.cc +++ b/src/shrpx_http.cc @@ -50,17 +50,6 @@ std::string create_error_html(unsigned int status_code) { return res; } -std::string create_via_header_value(int major, int minor) { - std::string hdrs; - hdrs += static_cast(major + '0'); - if (major < 2) { - hdrs += '.'; - hdrs += static_cast(minor + '0'); - } - hdrs += " nghttpx"; - return hdrs; -} - std::string create_forwarded(int params, const StringRef &node_by, const StringRef &node_for, const StringRef &host, const StringRef &proto) { diff --git a/src/shrpx_http.h b/src/shrpx_http.h index 56705ecd..35275baf 100644 --- a/src/shrpx_http.h +++ b/src/shrpx_http.h @@ -31,13 +31,23 @@ #include +#include "util.h" + namespace shrpx { namespace http { std::string create_error_html(unsigned int status_code); -std::string create_via_header_value(int major, int minor); +template +OutputIt create_via_header_value(OutputIt dst, int major, int minor) { + *dst++ = static_cast(major + '0'); + if (major < 2) { + *dst++ = '.'; + *dst++ = static_cast(minor + '0'); + } + return util::copy_lit(dst, " nghttpx"); +} // Returns generated RFC 7239 Forwarded header field value. The // |params| is bitwise-OR of zero or more of shrpx_forwarded_param diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index 5e2d5129..2e5fb177 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -266,6 +266,8 @@ int Http2DownstreamConnection::push_request_headers() { const auto &req = downstream_->request(); + auto &balloc = downstream_->get_block_allocator(); + auto &httpconf = get_config()->http; auto &http2conf = get_config()->http2; @@ -403,19 +405,28 @@ int Http2DownstreamConnection::push_request_headers() { nva.push_back(http2::make_nv_ls_nocopy("x-forwarded-proto", req.scheme)); } - std::string via_value; auto via = req.fs.header(http2::HD_VIA); if (httpconf.no_via) { if (via) { nva.push_back(http2::make_nv_ls_nocopy("via", (*via).value)); } } else { + size_t vialen = 16; if (via) { - via_value = (*via).value.str(); - via_value += ", "; + vialen += via->value.size() + 2; } - via_value += http::create_via_header_value(req.http_major, req.http_minor); - nva.push_back(http2::make_nv_ls("via", via_value)); + + auto iov = make_byte_ref(balloc, vialen + 1); + auto p = iov.base; + + if (via) { + p = std::copy(std::begin(via->value), std::end(via->value), p); + p = util::copy_lit(p, ", "); + } + p = http::create_via_header_value(p, req.http_major, req.http_minor); + *p = '\0'; + + nva.push_back(http2::make_nv_ls_nocopy("via", StringRef{iov.base, p})); } auto te = req.fs.header(http2::HD_TE); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index e54b8e90..32905876 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1399,7 +1399,6 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { // field. nva.reserve(resp.fs.headers().size() + 4 + httpconf.add_response_headers.size()); - std::string via_value; auto response_status = http2::stringify_status(resp.http_status); if (response_status.empty()) { @@ -1453,13 +1452,23 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { nva.push_back(http2::make_nv_ls_nocopy("via", (*via).value)); } } else { + // we don't create more than 16 bytes in + // http::create_via_header_value. + size_t len = 16; if (via) { - via_value = (*via).value.str(); - via_value += ", "; + len += via->value.size() + 2; } - via_value += - http::create_via_header_value(resp.http_major, resp.http_minor); - nva.push_back(http2::make_nv_ls("via", via_value)); + + auto iov = make_byte_ref(balloc, len + 1); + auto p = iov.base; + if (via) { + p = std::copy(std::begin(via->value), std::end(via->value), p); + p = util::copy_lit(p, ", "); + } + p = http::create_via_header_value(p, resp.http_major, resp.http_minor); + *p = '\0'; + + nva.push_back(http2::make_nv_ls_nocopy("via", StringRef{iov.base, p})); } for (auto &p : httpconf.add_response_headers) { diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 5f4f9468..490fefe8 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -424,7 +424,10 @@ int HttpDownstreamConnection::push_request_headers() { buf->append((*via).value); buf->append(", "); } - buf->append(http::create_via_header_value(req.http_major, req.http_minor)); + std::array viabuf; + auto end = http::create_via_header_value(viabuf.data(), req.http_major, + req.http_minor); + buf->append(viabuf.data(), end - viabuf.data()); buf->append("\r\n"); } diff --git a/src/shrpx_http_test.cc b/src/shrpx_http_test.cc index 87fbb9c5..bee8d966 100644 --- a/src/shrpx_http_test.cc +++ b/src/shrpx_http_test.cc @@ -72,4 +72,18 @@ void test_shrpx_http_create_forwarded(void) { StringRef::from_lit(""), StringRef::from_lit(""))); } +void test_shrpx_http_create_via_header_value(void) { + std::array buf; + + auto end = http::create_via_header_value(std::begin(buf), 1, 1); + + CU_ASSERT(("1.1 nghttpx" == StringRef{std::begin(buf), end})); + + std::fill(std::begin(buf), std::end(buf), '\0'); + + end = http::create_via_header_value(std::begin(buf), 2, 0); + + CU_ASSERT(("2 nghttpx" == StringRef{std::begin(buf), end})); +} + } // namespace shrpx diff --git a/src/shrpx_http_test.h b/src/shrpx_http_test.h index d8e28a9f..7692c95a 100644 --- a/src/shrpx_http_test.h +++ b/src/shrpx_http_test.h @@ -32,6 +32,7 @@ namespace shrpx { void test_shrpx_http_create_forwarded(void); +void test_shrpx_http_create_via_header_value(void); } // namespace shrpx diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index dd028190..ac99c457 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -1074,8 +1074,10 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) { buf->append((*via).value); buf->append(", "); } - buf->append( - http::create_via_header_value(resp.http_major, resp.http_minor)); + std::array viabuf; + auto end = http::create_via_header_value(viabuf.data(), resp.http_major, + resp.http_minor); + buf->append(viabuf.data(), end - std::begin(viabuf)); buf->append("\r\n"); } diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 4b48c93f..21380afa 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -1075,8 +1075,10 @@ int SpdyUpstream::on_downstream_header_complete(Downstream *downstream) { via_value = via->value.str(); via_value += ", "; } - via_value += - http::create_via_header_value(resp.http_major, resp.http_minor); + std::array viabuf; + auto end = http::create_via_header_value(std::begin(viabuf), + resp.http_major, resp.http_minor); + via_value.append(std::begin(viabuf), end); nv[hdidx++] = "via"; nv[hdidx++] = via_value.c_str(); } From c1571a3209329798ca91eee19d2a6c20073ff60b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 18:46:58 +0900 Subject: [PATCH 15/19] src: Rewrite xff handling --- src/allocator.h | 18 ++++++++++++++++-- src/shrpx_http2_downstream_connection.cc | 15 ++++++++------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/allocator.h b/src/allocator.h index 00acdbcf..cd92d3fd 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -104,12 +104,26 @@ StringRef make_string_ref(BlockAllocator &alloc, const StringRef &src) { template StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, const StringRef &b) { - auto dst = static_cast(alloc.alloc(a.size() + b.size() + 1)); + auto len = a.size() + b.size(); + auto dst = static_cast(alloc.alloc(len + 1)); auto p = dst; p = std::copy(std::begin(a), std::end(a), p); p = std::copy(std::begin(b), std::end(b), p); *p = '\0'; - return StringRef{dst, a.size() + b.size()}; + return StringRef{dst, len}; +} + +template +StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, + const StringRef &b, const StringRef &c) { + auto len = a.size() + b.size() + c.size(); + auto dst = static_cast(alloc.alloc(len + 1)); + auto p = dst; + p = std::copy(std::begin(a), std::end(a), p); + p = std::copy(std::begin(b), std::end(b), p); + p = std::copy(std::begin(c), std::end(c), p); + *p = '\0'; + return StringRef{dst, len}; } struct ByteRef { diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index 2e5fb177..c9df0369 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -387,17 +387,18 @@ int Http2DownstreamConnection::push_request_headers() { auto xff = xffconf.strip_incoming ? nullptr : req.fs.header(http2::HD_X_FORWARDED_FOR); - std::string xff_value; - if (xffconf.add) { + StringRef xff_value; + auto addr = StringRef{upstream->get_client_handler()->get_ipaddr()}; if (xff) { - xff_value = (*xff).value.str(); - xff_value += ", "; + xff_value = concat_string_ref(balloc, xff->value, + StringRef::from_lit(", "), addr); + } else { + xff_value = addr; } - xff_value += upstream->get_client_handler()->get_ipaddr(); - nva.push_back(http2::make_nv_ls("x-forwarded-for", xff_value)); + nva.push_back(http2::make_nv_ls_nocopy("x-forwarded-for", xff_value)); } else if (xff) { - nva.push_back(http2::make_nv_ls_nocopy("x-forwarded-for", (*xff).value)); + nva.push_back(http2::make_nv_ls_nocopy("x-forwarded-for", xff->value)); } if (!get_config()->http2_proxy && req.method != HTTP_CONNECT) { From 3455cb35e4b29c6afabdde119a0613865b980bb3 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 19:07:48 +0900 Subject: [PATCH 16/19] nghttpx: Rewrite create_forwarded to use BlockAllocator --- src/shrpx_http.cc | 69 +++++++++++++++--------- src/shrpx_http.h | 7 +-- src/shrpx_http2_downstream_connection.cc | 23 ++++---- src/shrpx_http_downstream_connection.cc | 9 ++-- src/shrpx_http_test.cc | 36 +++++++------ 5 files changed, 82 insertions(+), 62 deletions(-) diff --git a/src/shrpx_http.cc b/src/shrpx_http.cc index 071f2841..c6a70e06 100644 --- a/src/shrpx_http.cc +++ b/src/shrpx_http.cc @@ -50,58 +50,75 @@ std::string create_error_html(unsigned int status_code) { return res; } -std::string create_forwarded(int params, const StringRef &node_by, - const StringRef &node_for, const StringRef &host, - const StringRef &proto) { - std::string res; +StringRef create_forwarded(BlockAllocator &balloc, int params, + const StringRef &node_by, const StringRef &node_for, + const StringRef &host, const StringRef &proto) { + size_t len = 0; + if ((params & FORWARDED_BY) && !node_by.empty()) { + len += str_size("by=\"") + node_by.size() + str_size("\";"); + } + if ((params & FORWARDED_FOR) && !node_for.empty()) { + len += str_size("for=\"") + node_for.size() + str_size("\";"); + } + if ((params & FORWARDED_HOST) && !host.empty()) { + len += str_size("host=\"") + host.size() + str_size("\";"); + } + if ((params & FORWARDED_PROTO) && !proto.empty()) { + len += str_size("proto=") + proto.size() + str_size(";"); + } + + auto iov = make_byte_ref(balloc, len + 1); + auto p = iov.base; + if ((params & FORWARDED_BY) && !node_by.empty()) { // This must be quoted-string unless it is obfuscated version // (which starts with "_") or some special value (e.g., // "localhost" for UNIX domain socket), since ':' is not allowed // in token. ':' is used to separate host and port. if (node_by[0] == '_' || node_by[0] == 'l') { - res += "by="; - res += node_by; - res += ";"; + p = util::copy_lit(p, "by="); + p = std::copy(std::begin(node_by), std::end(node_by), p); + p = util::copy_lit(p, ";"); } else { - res += "by=\""; - res += node_by; - res += "\";"; + p = util::copy_lit(p, "by=\""); + p = std::copy(std::begin(node_by), std::end(node_by), p); + p = util::copy_lit(p, "\";"); } } if ((params & FORWARDED_FOR) && !node_for.empty()) { // We only quote IPv6 literal address only, which starts with '['. if (node_for[0] == '[') { - res += "for=\""; - res += node_for; - res += "\";"; + p = util::copy_lit(p, "for=\""); + p = std::copy(std::begin(node_for), std::end(node_for), p); + p = util::copy_lit(p, "\";"); } else { - res += "for="; - res += node_for; - res += ";"; + p = util::copy_lit(p, "for="); + p = std::copy(std::begin(node_for), std::end(node_for), p); + p = util::copy_lit(p, ";"); } } if ((params & FORWARDED_HOST) && !host.empty()) { // Just be quoted to skip checking characters. - res += "host=\""; - res += host; - res += "\";"; + p = util::copy_lit(p, "host=\""); + p = std::copy(std::begin(host), std::end(host), p); + p = util::copy_lit(p, "\";"); } if ((params & FORWARDED_PROTO) && !proto.empty()) { // Scheme production rule only allow characters which are all in // token. - res += "proto="; - res += proto; - res += ";"; + p = util::copy_lit(p, "proto="); + p = std::copy(std::begin(proto), std::end(proto), p); + *p++ = ';'; } - if (res.empty()) { - return res; + if (iov.base == p) { + return StringRef{}; } - res.erase(res.size() - 1); + --p; + *p = '\0'; - return res; + return StringRef{iov.base, p}; } std::string colorizeHeaders(const char *hdrs) { diff --git a/src/shrpx_http.h b/src/shrpx_http.h index 35275baf..856f192a 100644 --- a/src/shrpx_http.h +++ b/src/shrpx_http.h @@ -32,6 +32,7 @@ #include #include "util.h" +#include "allocator.h" namespace shrpx { @@ -52,9 +53,9 @@ OutputIt create_via_header_value(OutputIt dst, int major, int minor) { // Returns generated RFC 7239 Forwarded header field value. The // |params| is bitwise-OR of zero or more of shrpx_forwarded_param // defined in shrpx_config.h. -std::string create_forwarded(int params, const StringRef &node_by, - const StringRef &node_for, const StringRef &host, - const StringRef &proto); +StringRef create_forwarded(BlockAllocator &balloc, int params, + const StringRef &node_by, const StringRef &node_for, + const StringRef &host, const StringRef &proto); // Adds ANSI color codes to HTTP headers |hdrs|. std::string colorizeHeaders(const char *hdrs); diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index c9df0369..73a43a8d 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -347,8 +347,6 @@ int Http2DownstreamConnection::push_request_headers() { auto upstream = downstream_->get_upstream(); auto handler = upstream->get_client_handler(); - std::string forwarded_value; - auto &fwdconf = httpconf.forwarded; auto fwd = @@ -361,25 +359,24 @@ int Http2DownstreamConnection::push_request_headers() { params &= ~FORWARDED_PROTO; } - auto value = http::create_forwarded(params, handler->get_forwarded_by(), - handler->get_forwarded_for(), - req.authority, req.scheme); + auto value = http::create_forwarded( + balloc, params, handler->get_forwarded_by(), + handler->get_forwarded_for(), req.authority, req.scheme); + if (fwd || !value.empty()) { if (fwd) { - forwarded_value = fwd->value.str(); - - if (!value.empty()) { - forwarded_value += ", "; + if (value.empty()) { + value = fwd->value; + } else { + value = concat_string_ref(balloc, fwd->value, + StringRef::from_lit(", "), value); } } - forwarded_value += value; - - nva.push_back(http2::make_nv_ls("forwarded", forwarded_value)); + nva.push_back(http2::make_nv_ls_nocopy("forwarded", value)); } } else if (fwd) { nva.push_back(http2::make_nv_ls_nocopy("forwarded", fwd->value)); - forwarded_value = fwd->value.str(); } auto &xffconf = httpconf.xff; diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 490fefe8..893c9efc 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -272,6 +272,8 @@ int HttpDownstreamConnection::push_request_headers() { const auto &downstream_hostport = addr_->hostport; const auto &req = downstream_->request(); + auto &balloc = downstream_->get_block_allocator(); + auto connect_method = req.method == HTTP_CONNECT; auto &httpconf = get_config()->http; @@ -366,9 +368,10 @@ int HttpDownstreamConnection::push_request_headers() { params &= ~FORWARDED_PROTO; } - auto value = http::create_forwarded(params, handler->get_forwarded_by(), - handler->get_forwarded_for(), - req.authority, req.scheme); + auto value = http::create_forwarded( + balloc, params, handler->get_forwarded_by(), + handler->get_forwarded_for(), req.authority, req.scheme); + if (fwd || !value.empty()) { buf->append("Forwarded: "); if (fwd) { diff --git a/src/shrpx_http_test.cc b/src/shrpx_http_test.cc index bee8d966..455397a6 100644 --- a/src/shrpx_http_test.cc +++ b/src/shrpx_http_test.cc @@ -38,38 +38,40 @@ namespace shrpx { void test_shrpx_http_create_forwarded(void) { + BlockAllocator balloc(1024, 1024); + CU_ASSERT("by=\"example.com:3000\";for=\"[::1]\";host=\"www.example.com\";" "proto=https" == - http::create_forwarded(FORWARDED_BY | FORWARDED_FOR | - FORWARDED_HOST | FORWARDED_PROTO, + http::create_forwarded(balloc, FORWARDED_BY | FORWARDED_FOR | + FORWARDED_HOST | FORWARDED_PROTO, StringRef::from_lit("example.com:3000"), StringRef::from_lit("[::1]"), StringRef::from_lit("www.example.com"), StringRef::from_lit("https"))); CU_ASSERT("for=192.168.0.1" == - http::create_forwarded(FORWARDED_FOR, StringRef::from_lit("alpha"), - StringRef::from_lit("192.168.0.1"), - StringRef::from_lit("bravo"), - StringRef::from_lit("charlie"))); + http::create_forwarded( + balloc, FORWARDED_FOR, StringRef::from_lit("alpha"), + StringRef::from_lit("192.168.0.1"), + StringRef::from_lit("bravo"), StringRef::from_lit("charlie"))); CU_ASSERT("by=_hidden;for=\"[::1]\"" == http::create_forwarded( - FORWARDED_BY | FORWARDED_FOR, StringRef::from_lit("_hidden"), - StringRef::from_lit("[::1]"), StringRef::from_lit(""), - StringRef::from_lit(""))); + balloc, FORWARDED_BY | FORWARDED_FOR, + StringRef::from_lit("_hidden"), StringRef::from_lit("[::1]"), + StringRef::from_lit(""), StringRef::from_lit(""))); CU_ASSERT("by=\"[::1]\";for=_hidden" == http::create_forwarded( - FORWARDED_BY | FORWARDED_FOR, StringRef::from_lit("[::1]"), - StringRef::from_lit("_hidden"), StringRef::from_lit(""), - StringRef::from_lit(""))); - - CU_ASSERT("" == - http::create_forwarded( - FORWARDED_BY | FORWARDED_FOR | FORWARDED_HOST | FORWARDED_PROTO, - StringRef::from_lit(""), StringRef::from_lit(""), + balloc, FORWARDED_BY | FORWARDED_FOR, + StringRef::from_lit("[::1]"), StringRef::from_lit("_hidden"), StringRef::from_lit(""), StringRef::from_lit(""))); + + CU_ASSERT("" == http::create_forwarded( + balloc, FORWARDED_BY | FORWARDED_FOR | FORWARDED_HOST | + FORWARDED_PROTO, + StringRef::from_lit(""), StringRef::from_lit(""), + StringRef::from_lit(""), StringRef::from_lit(""))); } void test_shrpx_http_create_via_header_value(void) { From 755b14de5d3e6d83f2cbeff4a67f93252545cfac Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 19:24:02 +0900 Subject: [PATCH 17/19] src: Unify path_join implementation --- src/http2.cc | 127 ++------------------------------------------------- 1 file changed, 3 insertions(+), 124 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index 445ebe44..092c4c8e 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -1198,132 +1198,11 @@ std::vector parse_link_header(const char *src, size_t len) { return res; } -namespace { -void eat_file(std::string &path) { - if (path.empty()) { - path = "/"; - return; - } - auto p = path.size() - 1; - if (path[p] == '/') { - return; - } - p = path.rfind('/', p); - if (p == std::string::npos) { - // this should not happend in normal case, where we expect path - // starts with '/' - path = "/"; - return; - } - path.erase(std::begin(path) + p + 1, std::end(path)); -} -} // namespace - -namespace { -void eat_dir(std::string &path) { - if (path.empty()) { - path = "/"; - return; - } - auto p = path.size() - 1; - if (path[p] != '/') { - p = path.rfind('/', p); - if (p == std::string::npos) { - // this should not happend in normal case, where we expect path - // starts with '/' - path = "/"; - return; - } - } - if (path[p] == '/') { - if (p == 0) { - return; - } - --p; - } - p = path.rfind('/', p); - if (p == std::string::npos) { - // this should not happend in normal case, where we expect path - // starts with '/' - path = "/"; - return; - } - path.erase(std::begin(path) + p + 1, std::end(path)); -} -} // namespace - std::string path_join(const StringRef &base_path, const StringRef &base_query, const StringRef &rel_path, const StringRef &rel_query) { - std::string res; - if (rel_path.empty()) { - if (base_path.empty()) { - res = "/"; - } else { - res.assign(std::begin(base_path), std::end(base_path)); - } - if (rel_query.empty()) { - if (!base_query.empty()) { - res += '?'; - res.append(std::begin(base_query), std::end(base_query)); - } - return res; - } - res += '?'; - res.append(std::begin(rel_query), std::end(rel_query)); - return res; - } + BlockAllocator balloc(1024, 1024); - auto first = std::begin(rel_path); - auto last = std::end(rel_path); - - if (rel_path[0] == '/') { - res = "/"; - ++first; - } else if (base_path.empty()) { - res = "/"; - } else { - res.assign(std::begin(base_path), std::end(base_path)); - } - - for (; first != last;) { - if (*first == '.') { - if (first + 1 == last) { - break; - } - if (*(first + 1) == '/') { - first += 2; - continue; - } - if (*(first + 1) == '.') { - if (first + 2 == last) { - eat_dir(res); - break; - } - if (*(first + 2) == '/') { - eat_dir(res); - first += 3; - continue; - } - } - } - if (res.back() != '/') { - eat_file(res); - } - auto slash = std::find(first, last, '/'); - if (slash == last) { - res.append(first, last); - break; - } - res.append(first, slash + 1); - first = slash + 1; - for (; first != last && *first == '/'; ++first) - ; - } - if (!rel_query.empty()) { - res += '?'; - res.append(std::begin(rel_query), std::end(rel_query)); - } - return res; + return path_join(balloc, base_path, base_query, rel_path, rel_query).str(); } bool expect_response_body(int status_code) { @@ -1729,7 +1608,7 @@ StringRef path_join(BlockAllocator &balloc, const StringRef &base_path, StringRef normalize_path(BlockAllocator &balloc, const StringRef &path, const StringRef &query) { // First, decode %XX for unreserved characters, then do - // http2::join_path + // http2::path_join // We won't find %XX if length is less than 3. if (path.size() < 3 || From 78fcb2143f51809a8858d2034866b3ec0b43e13f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 19:42:24 +0900 Subject: [PATCH 18/19] Update doc --- src/allocator.h | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/allocator.h b/src/allocator.h index cd92d3fd..28fe8eb0 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -34,10 +34,19 @@ namespace nghttp2 { struct MemBlock { + // The next MemBlock to chain them. This is for book keeping + // purpose to free them later. MemBlock *next; + // begin is the pointer to the beginning of buffer. last is the + // location of next write. end is the one beyond of the end of the + // buffer. uint8_t *begin, *last, *end; }; +// BlockAllocator allocates memory block with given size at once, and +// cuts the region from it when allocation is requested. If the +// requested size is larger than given threshold, it will be allocated +// in a distinct buffer on demand. struct BlockAllocator { BlockAllocator(size_t block_size, size_t isolation_threshold) : retain(nullptr), @@ -83,7 +92,9 @@ struct BlockAllocator { return res; } + // This holds live memory block to free them in dtor. MemBlock *retain; + // Current memory block to use. MemBlock *head; // size of single memory block size_t block_size; @@ -92,6 +103,8 @@ struct BlockAllocator { size_t isolation_threshold; }; +// Makes a copy of |src|. The resulting string will be +// NULL-terminated. template StringRef make_string_ref(BlockAllocator &alloc, const StringRef &src) { auto dst = static_cast(alloc.alloc(src.size() + 1)); @@ -101,6 +114,8 @@ StringRef make_string_ref(BlockAllocator &alloc, const StringRef &src) { return StringRef{dst, src.size()}; } +// Returns the string which is the concatenation of |a| and |b| in +// this order. The resulting string will be NULL-terminated. template StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, const StringRef &b) { @@ -113,6 +128,8 @@ StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, return StringRef{dst, len}; } +// Returns the string which is the concatenation of |a|, |b| and |c| +// in this order. The resulting string will be NULL-terminated. template StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, const StringRef &b, const StringRef &c) { @@ -127,10 +144,14 @@ StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, } struct ByteRef { + // The pointer to the beginning of the buffer. uint8_t *base; + // The length of the buffer. size_t len; }; +// Makes a buffer with given size. The resulting byte string might +// not be NULL-terminated. template ByteRef make_byte_ref(BlockAllocator &alloc, size_t size) { auto dst = static_cast(alloc.alloc(size)); From 0ee09320e00693b772f9d62156cce4a4bb88bebf Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 21:06:05 +0900 Subject: [PATCH 19/19] src: Remove unused value_to_str --- src/http2.cc | 7 ------- src/http2.h | 3 --- 2 files changed, 10 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index 092c4c8e..8b82dcca 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -302,13 +302,6 @@ const Headers::value_type *get_header(const Headers &nva, const char *name) { return res; } -std::string value_to_str(const HeaderRefs::value_type *nv) { - if (nv) { - return nv->value.str(); - } - return ""; -} - bool non_empty_value(const HeaderRefs::value_type *nv) { return nv && !nv->value.empty(); } diff --git a/src/http2.h b/src/http2.h index 0ae9d3b3..8ea2f91c 100644 --- a/src/http2.h +++ b/src/http2.h @@ -127,9 +127,6 @@ void add_header(Headers &nva, const uint8_t *name, size_t namelen, // in |nva| is returned. If no such entry exist, returns nullptr. const Headers::value_type *get_header(const Headers &nva, const char *name); -// Returns nv->second if nv is not nullptr. Otherwise, returns "". -std::string value_to_str(const HeaderRefs::value_type *nv); - // Returns true if the value of |nv| is not empty. bool non_empty_value(const HeaderRefs::value_type *nv);