From 3119fc259cd29513c9aeef1371bdd6807a51b850 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 10 Jul 2015 02:52:11 +0900 Subject: [PATCH 01/77] Select backend based on request host and path by extending -b option -b option syntax is now ,[;[:...]]. The optional s specify the request host and path it is used for. The can contain path, host + path or host. The matching rule is closely designed to ServeMux in Go programming language. --- src/http2.h | 36 ++++ src/http2_test.cc | 27 +++ src/http2_test.h | 1 + src/shrpx-unittest.cc | 4 + src/shrpx.cc | 151 +++++++++++---- src/shrpx_client_handler.cc | 45 ++++- src/shrpx_client_handler.h | 3 +- src/shrpx_config.cc | 226 +++++++++++++++++++++-- src/shrpx_config.h | 38 +++- src/shrpx_config_test.cc | 66 ++++++- src/shrpx_config_test.h | 1 + src/shrpx_downstream_connection.h | 1 + src/shrpx_downstream_connection_pool.cc | 27 ++- src/shrpx_downstream_connection_pool.h | 6 +- src/shrpx_http2_downstream_connection.cc | 2 +- src/shrpx_http2_downstream_connection.h | 2 + src/shrpx_http2_session.cc | 22 ++- src/shrpx_http2_upstream.cc | 4 +- src/shrpx_http_downstream_connection.cc | 32 ++-- src/shrpx_http_downstream_connection.h | 4 +- src/shrpx_https_upstream.cc | 4 +- src/shrpx_spdy_upstream.cc | 4 +- src/shrpx_ssl.cc | 6 +- src/shrpx_ssl.h | 3 +- src/shrpx_worker.cc | 7 +- src/shrpx_worker.h | 8 +- src/util.h | 6 +- 27 files changed, 613 insertions(+), 123 deletions(-) diff --git a/src/http2.h b/src/http2.h index 9b7bf94b..be79caca 100644 --- a/src/http2.h +++ b/src/http2.h @@ -37,6 +37,8 @@ #include "http-parser/http_parser.h" +#include "util.h" + namespace nghttp2 { struct Header { @@ -298,6 +300,40 @@ int lookup_method_token(const std::string &name); const char *to_method_string(int method_token); +template +std::string normalize_path(InputIt first, InputIt last) { + // First, decode %XX for unreserved characters, then do + // http2::join_path + std::string result; + // We won't find %XX if length is less than 3. + if (last - first < 3) { + result.assign(first, last); + } else { + for (; first < last - 2;) { + if (*first == '%') { + if (util::isHexDigit(*(first + 1)) && util::isHexDigit(*(first + 2))) { + auto c = (util::hex_to_uint(*(first + 1)) << 4) + + util::hex_to_uint(*(first + 2)); + if (util::inRFC3986UnreservedChars(c)) { + result += c; + first += 3; + continue; + } + result += '%'; + result += util::upcase(*(first + 1)); + result += util::upcase(*(first + 2)); + first += 3; + continue; + } + } + result += *first++; + } + result.append(first, last); + } + return path_join(nullptr, 0, nullptr, 0, result.c_str(), result.size(), + nullptr, 0); +} + } // namespace http2 } // namespace nghttp2 diff --git a/src/http2_test.cc b/src/http2_test.cc index a11df674..1937bfaf 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -829,4 +829,31 @@ void test_http2_path_join(void) { } } +void test_http2_normalize_path(void) { + std::string src; + + src = "/alpha/bravo/../charlie"; + CU_ASSERT("/alpha/charlie" == + http2::normalize_path(std::begin(src), std::end(src))); + + src = "/a%6c%70%68%61"; + CU_ASSERT("/alpha" == http2::normalize_path(std::begin(src), std::end(src))); + + src = "/alpha%2f%3a"; + CU_ASSERT("/alpha%2F%3A" == + http2::normalize_path(std::begin(src), std::end(src))); + + src = "%2f"; + CU_ASSERT("/%2F" == http2::normalize_path(std::begin(src), std::end(src))); + + src = "%f"; + CU_ASSERT("/%f" == http2::normalize_path(std::begin(src), std::end(src))); + + src = "%"; + CU_ASSERT("/%" == http2::normalize_path(std::begin(src), std::end(src))); + + src = ""; + CU_ASSERT("/" == http2::normalize_path(std::begin(src), std::end(src))); +} + } // namespace shrpx diff --git a/src/http2_test.h b/src/http2_test.h index 36c5a59a..193f23e3 100644 --- a/src/http2_test.h +++ b/src/http2_test.h @@ -45,6 +45,7 @@ void test_http2_http2_header_allowed(void); void test_http2_mandatory_request_headers_presence(void); void test_http2_parse_link_header(void); void test_http2_path_join(void); +void test_http2_normalize_path(void); } // namespace shrpx diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index d4e332ae..7030aa3d 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -96,6 +96,8 @@ int main(int argc, char *argv[]) { !CU_add_test(pSuite, "http2_parse_link_header", shrpx::test_http2_parse_link_header) || !CU_add_test(pSuite, "http2_path_join", shrpx::test_http2_path_join) || + !CU_add_test(pSuite, "http2_normalize_path", + shrpx::test_http2_normalize_path) || !CU_add_test(pSuite, "downstream_index_request_headers", shrpx::test_downstream_index_request_headers) || !CU_add_test(pSuite, "downstream_index_response_headers", @@ -118,6 +120,8 @@ int main(int argc, char *argv[]) { shrpx::test_shrpx_config_parse_log_format) || !CU_add_test(pSuite, "config_read_tls_ticket_key_file", shrpx::test_shrpx_config_read_tls_ticket_key_file) || + !CU_add_test(pSuite, "config_match_downstream_addr_group", + shrpx::test_shrpx_config_match_downstream_addr_group) || !CU_add_test(pSuite, "util_streq", shrpx::test_util_streq) || !CU_add_test(pSuite, "util_strieq", shrpx::test_util_strieq) || !CU_add_test(pSuite, "util_inp_strlower", diff --git a/src/shrpx.cc b/src/shrpx.cc index 92046aed..8bfadbc3 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -966,6 +966,7 @@ void fill_default_config() { mod_config()->no_ocsp = false; mod_config()->header_field_buffer = 64_k; mod_config()->max_header_fields = 100; + mod_config()->downstream_addr_group_catch_all = 0; } } // namespace @@ -997,14 +998,45 @@ Options: The options are categorized into several groups. Connections: - -b, --backend= + -b, --backend=,[;[:...]] Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX domain socket can be specified by prefixing path name - with "unix:" (e.g., unix:/var/run/backend.sock) + with "unix:" (e.g., unix:/var/run/backend.sock). + Optionally, if s are given, the backend address + is only used if request matches the pattern. If -s, -p, + --client or --http2-bridge is used, s are + ignored. The pattern matching is closely designed to + ServeMux in net/http package of Go programming language. + consists of path, host + path or host. The + path must starts with "/". If it ends with "/", it + matches to the request path whose prefix is the path. + If it does not end with "/", it performs exact match + against the request path. If host is given, it performs + exact match against the request host. If host alone is + given, "/" is appended to it, so that it matches all + paths under the host (e.g., specifying "nghttp2.org" + equals to "nghttp2.org/"). Longer patterns take + precedence over shorter ones, breaking a tie by the + order of the appearance in the configuration. If + is omitted, "/" is used as pattern, which + matches all paths (catch-all pattern). For example, + -b'127.0.0.1,8080;nghttp2.org/httpbin/' matches the + request host "nghttp2.org" and the request path + "/httpbin/get", but does not match the request host + "nghttp2.org" and the request path "/index.html". The + multiple s can be specified, delimiting them by + ":". Specifying + -b'127.0.0.1,8080;nghttp2.org:www.nghttp2.org' has the + same effect to specify -b'127.0.0.1,8080;nghttp2.org' + and -b'127.0.0.1,8080:www.nghttp2.org'. The backend + addresses sharing same are grouped together + forming load balancing group. Since ";" and ":" are + used as delimiter, must not contain these + characters. Default: )" << DEFAULT_DOWNSTREAM_HOST << "," << DEFAULT_DOWNSTREAM_PORT << R"( - -f, --frontend= + -f, --frontend=, Set frontend host and port. If is '*', it assumes all addresses including both IPv4 and IPv6. UNIX domain socket can be specified by prefixing path @@ -2118,55 +2150,97 @@ int main(int argc, char **argv) { } } - if (get_config()->downstream_addrs.empty()) { + if (get_config()->downstream_addr_groups.empty()) { DownstreamAddr addr; addr.host = strcopy(DEFAULT_DOWNSTREAM_HOST); addr.port = DEFAULT_DOWNSTREAM_PORT; - mod_config()->downstream_addrs.push_back(std::move(addr)); + DownstreamAddrGroup g("/"); + g.addrs.push_back(std::move(addr)); + mod_config()->downstream_addr_groups.push_back(std::move(g)); + } else if (get_config()->downstream_proto == PROTO_HTTP2 || + get_config()->http2_proxy || get_config()->client_proxy) { + // We don't support host mapping in these cases. Move all + // non-catch-all patterns to catch-all pattern for HTTP/2 backend + DownstreamAddrGroup catch_all("/"); + for (auto &g : mod_config()->downstream_addr_groups) { + std::move(std::begin(g.addrs), std::end(g.addrs), + std::back_inserter(catch_all.addrs)); + } + std::vector().swap( + mod_config()->downstream_addr_groups); + mod_config()->downstream_addr_groups.push_back(std::move(catch_all)); } if (LOG_ENABLED(INFO)) { LOG(INFO) << "Resolving backend address"; } - for (auto &addr : mod_config()->downstream_addrs) { + ssize_t catch_all_group = -1; + for (size_t i = 0; i < mod_config()->downstream_addr_groups.size(); ++i) { + auto &g = mod_config()->downstream_addr_groups[i]; + if (g.pattern == "/") { + catch_all_group = i; + } + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Host-path pattern: group " << i << ": '" << g.pattern + << "'"; + for (auto &addr : g.addrs) { + LOG(INFO) << "group " << i << " -> " << addr.host.get() + << (addr.host_unix ? "" : ":" + util::utos(addr.port)); + } + } + } - if (addr.host_unix) { - // for AF_UNIX socket, we use "localhost" as host for backend - // hostport. This is used as Host header field to backend and - // not going to be passed to any syscalls. - addr.hostport = - strcopy(util::make_hostport("localhost", get_config()->port)); + if (catch_all_group == -1) { + LOG(FATAL) << "-b: No catch-all backend address is configured"; + exit(EXIT_FAILURE); + } + mod_config()->downstream_addr_group_catch_all = catch_all_group; - auto path = addr.host.get(); - auto pathlen = strlen(path); + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Catch-all pattern is group " << catch_all_group; + } - if (pathlen + 1 > sizeof(addr.addr.un.sun_path)) { - LOG(FATAL) << "UNIX domain socket path " << path << " is too long > " - << sizeof(addr.addr.un.sun_path); - exit(EXIT_FAILURE); + for (auto &g : mod_config()->downstream_addr_groups) { + for (auto &addr : g.addrs) { + + if (addr.host_unix) { + // for AF_UNIX socket, we use "localhost" as host for backend + // hostport. This is used as Host header field to backend and + // not going to be passed to any syscalls. + addr.hostport = + strcopy(util::make_hostport("localhost", get_config()->port)); + + auto path = addr.host.get(); + auto pathlen = strlen(path); + + if (pathlen + 1 > sizeof(addr.addr.un.sun_path)) { + LOG(FATAL) << "UNIX domain socket path " << path << " is too long > " + << sizeof(addr.addr.un.sun_path); + exit(EXIT_FAILURE); + } + + LOG(INFO) << "Use UNIX domain socket path " << path + << " for backend connection"; + + addr.addr.un.sun_family = AF_UNIX; + // copy path including terminal NULL + std::copy_n(path, pathlen + 1, addr.addr.un.sun_path); + addr.addrlen = sizeof(addr.addr.un); + + continue; } - LOG(INFO) << "Use UNIX domain socket path " << path - << " for backend connection"; + addr.hostport = strcopy(util::make_hostport(addr.host.get(), addr.port)); - addr.addr.un.sun_family = AF_UNIX; - // copy path including terminal NULL - std::copy_n(path, pathlen + 1, addr.addr.un.sun_path); - addr.addrlen = sizeof(addr.addr.un); - - continue; - } - - addr.hostport = strcopy(util::make_hostport(addr.host.get(), addr.port)); - - if (resolve_hostname( - &addr.addr, &addr.addrlen, addr.host.get(), addr.port, - get_config()->backend_ipv4 - ? AF_INET - : (get_config()->backend_ipv6 ? AF_INET6 : AF_UNSPEC)) == -1) { - exit(EXIT_FAILURE); + if (resolve_hostname( + &addr.addr, &addr.addrlen, addr.host.get(), addr.port, + get_config()->backend_ipv4 ? AF_INET : (get_config()->backend_ipv6 + ? AF_INET6 + : AF_UNSPEC)) == -1) { + exit(EXIT_FAILURE); + } } } @@ -2183,9 +2257,10 @@ int main(int argc, char **argv) { } } - if (get_config()->http2_downstream_connections_per_worker == 0) { + if (get_config()->downstream_proto == PROTO_HTTP2 && + get_config()->http2_downstream_connections_per_worker == 0) { mod_config()->http2_downstream_connections_per_worker = - get_config()->downstream_addrs.size(); + get_config()->downstream_addr_groups[0].addrs.size(); } if (get_config()->rlimit_nofile) { diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index d727fe39..644a8cdc 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -588,7 +588,8 @@ void ClientHandler::pool_downstream_connection( return; } if (LOG_ENABLED(INFO)) { - CLOG(INFO, this) << "Pooling downstream connection DCONN:" << dconn.get(); + CLOG(INFO, this) << "Pooling downstream connection DCONN:" << dconn.get() + << " in group " << dconn->get_group(); } dconn->set_client_handler(nullptr); auto dconn_pool = worker_->get_dconn_pool(); @@ -605,9 +606,44 @@ void ClientHandler::remove_downstream_connection(DownstreamConnection *dconn) { } std::unique_ptr -ClientHandler::get_downstream_connection() { +ClientHandler::get_downstream_connection(Downstream *downstream) { + size_t group; + auto &groups = get_config()->downstream_addr_groups; + auto catch_all = get_config()->downstream_addr_group_catch_all; + + // Fast path. If we have one group, it must be catch-all group. + // HTTP/2 and client proxy modes fall in this case. Currently, + // HTTP/2 backend does not perform host-path mapping. + if (groups.size() == 1 || get_config()->downstream_proto == PROTO_HTTP2) { + group = 0; + } else if (downstream->get_request_method() == HTTP_CONNECT) { + // We don't know how to treat CONNECT request in host-path + // mapping. It most likely appears in proxy scenario. Since we + // have dealt with proxy case already, just use catch-all group. + group = catch_all; + } else { + if (!downstream->get_request_http2_authority().empty()) { + group = match_downstream_addr_group( + downstream->get_request_http2_authority(), + downstream->get_request_path(), groups, catch_all); + } else { + auto h = downstream->get_request_header(http2::HD_HOST); + if (h) { + group = match_downstream_addr_group( + h->value, downstream->get_request_path(), groups, catch_all); + } else { + group = match_downstream_addr_group("", downstream->get_request_path(), + groups, catch_all); + } + } + } + + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) << "Downstream address group: " << group; + } + auto dconn_pool = worker_->get_dconn_pool(); - auto dconn = dconn_pool->pop_downstream_connection(); + auto dconn = dconn_pool->pop_downstream_connection(group); if (!dconn) { if (LOG_ENABLED(INFO)) { @@ -620,7 +656,8 @@ ClientHandler::get_downstream_connection() { if (http2session_) { dconn = make_unique(dconn_pool, http2session_); } else { - dconn = make_unique(dconn_pool, conn_.loop); + dconn = + make_unique(dconn_pool, group, conn_.loop); } dconn->set_client_handler(this); return dconn; diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index a2df98eb..4f21f513 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -92,7 +92,8 @@ public: void pool_downstream_connection(std::unique_ptr dconn); void remove_downstream_connection(DownstreamConnection *dconn); - std::unique_ptr get_downstream_connection(); + std::unique_ptr + get_downstream_connection(Downstream *downstream); MemchunkPool *get_mcpool(); SSL *get_ssl() const; ConnectBlocker *get_connect_blocker() const; diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index d59a1d0d..3fdbe425 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -57,6 +57,7 @@ #include "http2.h" #include "util.h" #include "template.h" +#include "base64.h" using namespace nghttp2; @@ -79,9 +80,29 @@ TicketKeys::~TicketKeys() { } } +DownstreamAddr::DownstreamAddr(const DownstreamAddr &other) + : addr(other.addr), host(other.host ? strcopy(other.host.get()) : nullptr), + hostport(other.hostport ? strcopy(other.hostport.get()) : nullptr), + addrlen(other.addrlen), port(other.port), host_unix(other.host_unix) {} + +DownstreamAddr &DownstreamAddr::operator=(const DownstreamAddr &other) { + if (this == &other) { + return *this; + } + + addr = other.addr; + host = (other.host ? strcopy(other.host.get()) : nullptr); + hostport = (other.hostport ? strcopy(other.hostport.get()) : nullptr); + addrlen = other.addrlen; + port = other.port; + host_unix = other.host_unix; + + return *this; +} + namespace { int split_host_port(char *host, size_t hostlen, uint16_t *port_ptr, - const char *hostport) { + const char *hostport, size_t hostportlen) { // host and port in |hostport| is separated by single ','. const char *p = strchr(hostport, ','); if (!p) { @@ -97,12 +118,13 @@ int split_host_port(char *host, size_t hostlen, uint16_t *port_ptr, host[len] = '\0'; errno = 0; - unsigned long d = strtoul(p + 1, nullptr, 10); - if (errno == 0 && 1 <= d && d <= std::numeric_limits::max()) { + auto portlen = hostportlen - len - 1; + auto d = util::parse_uint(reinterpret_cast(p + 1), portlen); + if (1 <= d && d <= std::numeric_limits::max()) { *port_ptr = d; return 0; } else { - LOG(ERROR) << "Port is invalid: " << p + 1; + LOG(ERROR) << "Port is invalid: " << std::string(p + 1, portlen); return -1; } } @@ -220,16 +242,16 @@ std::unique_ptr strcopy(const std::string &val) { return strcopy(val.c_str(), val.size()); } -std::vector parse_config_str_list(const char *s) { +std::vector parse_config_str_list(const char *s, char delim) { size_t len = 1; - for (const char *first = s, *p = nullptr; (p = strchr(first, ',')); + for (const char *first = s, *p = nullptr; (p = strchr(first, delim)); ++len, first = p + 1) ; auto list = std::vector(len); auto first = strdup(s); len = 0; for (;;) { - auto p = strchr(first, ','); + auto p = strchr(first, delim); if (p == nullptr) { break; } @@ -440,30 +462,82 @@ int parse_duration(ev_tstamp *dest, const char *opt, const char *optarg) { } } // namespace +namespace { +// Parses host-path mapping patterns in |src|, and stores mappings in +// config. We will store each host-path pattern found in |src| with +// |addr|. |addr| will be copied accordingly. Also we make a group +// based on the pattern. The "/" pattern is considered as catch-all. +void parse_mapping(DownstreamAddr addr, const char *src) { + // This returns at least 1 element (it could be empty string). We + // will append '/' to all patterns, so it becomes catch-all pattern. + auto mapping = parse_config_str_list(src, ':'); + assert(!mapping.empty()); + for (auto raw_pattern : mapping) { + auto done = false; + std::string pattern; + auto slash = strchr(raw_pattern, '/'); + if (slash == nullptr) { + // This effectively makes empty pattern to "/". + pattern = raw_pattern; + util::inp_strlower(pattern); + pattern += "/"; + } else { + pattern.assign(raw_pattern, slash); + util::inp_strlower(pattern); + pattern += + http2::normalize_path(slash, raw_pattern + strlen(raw_pattern)); + } + for (auto &g : mod_config()->downstream_addr_groups) { + if (g.pattern == pattern) { + g.addrs.push_back(addr); + done = true; + break; + } + } + if (done) { + continue; + } + DownstreamAddrGroup g(pattern); + g.addrs.push_back(addr); + mod_config()->downstream_addr_groups.push_back(std::move(g)); + } + clear_config_str_list(mapping); +} +} // namespace + int parse_config(const char *opt, const char *optarg) { char host[NI_MAXHOST]; uint16_t port; + if (util::strieq(opt, SHRPX_OPT_BACKEND)) { + auto optarglen = strlen(optarg); + auto pat_delim = strchr(optarg, ';'); + if (!pat_delim) { + pat_delim = optarg + optarglen; + } + DownstreamAddr addr; if (util::istartsWith(optarg, SHRPX_UNIX_PATH_PREFIX)) { - DownstreamAddr addr; auto path = optarg + str_size(SHRPX_UNIX_PATH_PREFIX); addr.host = strcopy(path); addr.host_unix = true; + } else { + if (split_host_port(host, sizeof(host), &port, optarg, + pat_delim - optarg) == -1) { + return -1; + } - mod_config()->downstream_addrs.push_back(std::move(addr)); - - return 0; + addr.host = strcopy(host); + addr.port = port; } - if (split_host_port(host, sizeof(host), &port, optarg) == -1) { + auto mapping = pat_delim < optarg + optarglen ? pat_delim + 1 : pat_delim; + // We may introduce new parameter after additional ';', so don't + // allow extra ';' in pattern for now. + if (strchr(mapping, ';') != nullptr) { + LOG(ERROR) << opt << ": ';' must not be used in pattern"; return -1; } - - DownstreamAddr addr; - addr.host = strcopy(host); - addr.port = port; - - mod_config()->downstream_addrs.push_back(std::move(addr)); + parse_mapping(std::move(addr), mapping); return 0; } @@ -478,7 +552,8 @@ int parse_config(const char *opt, const char *optarg) { return 0; } - if (split_host_port(host, sizeof(host), &port, optarg) == -1) { + if (split_host_port(host, sizeof(host), &port, optarg, strlen(optarg)) == + -1) { return -1; } @@ -1322,4 +1397,117 @@ int int_syslog_facility(const char *strfacility) { return -1; } +namespace { +bool path_match(const std::string &pattern, const std::string &path) { + if (pattern.back() != '/') { + return pattern == path; + } + return util::startsWith(path, pattern); +} +} // namespace + +namespace { +ssize_t match(const std::string &path, + const std::vector &groups) { + ssize_t res = -1; + size_t best = 0; + for (size_t i = 0; i < groups.size(); ++i) { + auto &g = groups[i]; + auto &pattern = g.pattern; + if (!path_match(pattern, path)) { + continue; + } + if (res == -1 || best < pattern.size()) { + best = pattern.size(); + res = i; + } + } + return res; +} +} // namespace + +namespace { +size_t match_downstream_addr_group_host( + const std::string &host, const std::string &raw_path, + const std::vector &groups, size_t catch_all) { + if (raw_path == "*") { + auto group = match(host + "/", groups); + if (group != -1) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Found pattern with query " << host + << ", matched pattern=" << groups[group].pattern; + } + return group; + } + return catch_all; + } + + // probably, not necessary most of the case, but just in case. + auto fragment = std::find(std::begin(raw_path), std::end(raw_path), '#'); + auto query = std::find(std::begin(raw_path), fragment, '?'); + auto path = http2::normalize_path(std::begin(raw_path), query); + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Perform mapping selection, using host=" << host + << ", path=" << path; + } + + auto group = match(host + path, groups); + if (group != -1) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Found pattern with query " << host + path + << ", matched pattern=" << groups[group].pattern; + } + return group; + } + + group = match(path, groups); + if (group != -1) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Found pattern with query " << path + << ", matched pattern=" << groups[group].pattern; + } + return group; + } + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "None match. Use catch-all pattern"; + } + return catch_all; +} +} // namespace + +size_t match_downstream_addr_group( + const std::string &hostport, const std::string &raw_path, + const std::vector &groups, size_t catch_all) { + if (hostport.empty() || + std::find(std::begin(hostport), std::end(hostport), '/') != + std::end(hostport)) { + // We use '/' specially, and if '/' is included in host, it breaks + // our code. Select catch-all case. + return catch_all; + } + std::string host; + if (hostport[0] == '[') { + // assume this is IPv6 numeric address + auto p = std::find(std::begin(hostport), std::end(hostport), ']'); + if (p == std::end(hostport)) { + return catch_all; + } + if (p + 1 < std::end(hostport) && *(p + 1) != ':') { + return catch_all; + } + host.assign(std::begin(hostport), p + 1); + } else { + auto p = std::find(std::begin(hostport), std::end(hostport), ':'); + if (p == std::begin(hostport)) { + return catch_all; + } + host.assign(std::begin(hostport), p); + } + + util::inp_strlower(host); + return match_downstream_addr_group_host(host, raw_path, groups, catch_all); +} + } // namespace shrpx diff --git a/src/shrpx_config.h b/src/shrpx_config.h index c50ab64b..3e5c869e 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -194,6 +194,11 @@ struct AltSvc { struct DownstreamAddr { DownstreamAddr() : addr{{0}}, addrlen(0), port(0), host_unix(false) {} + DownstreamAddr(const DownstreamAddr &other); + DownstreamAddr(DownstreamAddr &&) = default; + DownstreamAddr &operator=(const DownstreamAddr &other); + DownstreamAddr &operator=(DownstreamAddr &&other) = default; + sockaddr_union addr; // backend address. If |host_unix| is true, this is UNIX domain // socket path. @@ -206,6 +211,12 @@ struct DownstreamAddr { bool host_unix; }; +struct DownstreamAddrGroup { + DownstreamAddrGroup(std::string pattern) : pattern(std::move(pattern)) {} + std::string pattern; + std::vector addrs; +}; + struct TicketKey { uint8_t name[16]; uint8_t aes_key[16]; @@ -225,7 +236,7 @@ struct Config { std::vector> add_response_headers; std::vector alpn_prefs; std::vector accesslog_format; - std::vector downstream_addrs; + std::vector downstream_addr_groups; std::vector tls_ticket_key_files; // binary form of http proxy host and port sockaddr_union downstream_http_proxy_addr; @@ -311,6 +322,8 @@ struct Config { size_t downstream_response_buffer_size; size_t header_field_buffer; size_t max_header_fields; + // The index of catch-all group in downstream_addr_groups. + size_t downstream_addr_group_catch_all; // Bit mask to disable SSL/TLS protocol versions. This will be // passed to SSL_CTX_set_options(). long int tls_proto_mask; @@ -376,15 +389,14 @@ int load_config(const char *filename); // Read passwd from |filename| std::string read_passwd_from_file(const char *filename); -// Parses comma delimited strings in |s| and returns the array of -// pointers, each element points to the each substring in |s|. The -// |s| must be comma delimited list of strings. The strings must be -// delimited by a single comma and any white spaces around it are -// treated as a part of protocol strings. This function may modify -// |s| and the caller must leave it as is after this call. This +// Parses delimited strings in |s| and returns the array of pointers, +// each element points to the each substring in |s|. The delimiter is +// given by |delim. The |s| must be comma delimited list of strings. +// The strings must be delimited by a single comma and any white +// spaces around it are treated as a part of protocol strings. This // function copies |s| and first element in the return value points to // it. It is caller's responsibility to deallocate its memory. -std::vector parse_config_str_list(const char *s); +std::vector parse_config_str_list(const char *s, char delim = ','); // Clears all elements of |list|, which is returned by // parse_config_str_list(). If list is not empty, list[0] is freed by @@ -423,6 +435,16 @@ FILE *open_file_for_write(const char *filename); std::unique_ptr read_tls_ticket_key_file(const std::vector &files); +// Selects group based on request's |hostport| and |path|. |hostport| +// is the value taken from :authority or host header field, and may +// contain port. The |path| may contain query part. We require the +// catch-all pattern in place, so this function always selects one +// group. The catch-all group index is given in |catch_all|. All +// patterns are given in |groups|. +size_t match_downstream_addr_group( + const std::string &hostport, const std::string &path, + const std::vector &groups, size_t catch_all); + } // namespace shrpx #endif // SHRPX_CONFIG_H diff --git a/src/shrpx_config_test.cc b/src/shrpx_config_test.cc index ad570bbc..4f21fc48 100644 --- a/src/shrpx_config_test.cc +++ b/src/shrpx_config_test.cc @@ -48,7 +48,7 @@ void test_shrpx_config_parse_config_str_list(void) { CU_ASSERT(0 == strcmp("", res[1])); clear_config_str_list(res); - res = parse_config_str_list(",a,,"); + res = parse_config_str_list(":a::", ':'); CU_ASSERT(4 == res.size()); CU_ASSERT(0 == strcmp("", res[0])); CU_ASSERT(0 == strcmp("a", res[1])); @@ -172,4 +172,68 @@ void test_shrpx_config_read_tls_ticket_key_file(void) { memcmp("a..............b", key->hmac_key, sizeof(key->hmac_key))); } +void test_shrpx_config_match_downstream_addr_group(void) { + auto groups = std::vector{ + {"nghttp2.org/"}, + {"nghttp2.org/alpha/bravo/"}, + {"nghttp2.org/alpha/charlie"}, + {"nghttp2.org/delta%3A"}, + {"www.nghttp2.org/"}, + {"[::1]/"}, + }; + + CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "/", groups, 255)); + + // port is removed + CU_ASSERT(0 == + match_downstream_addr_group("nghttp2.org:8080", "/", groups, 255)); + + // host is case-insensitive + CU_ASSERT(4 == match_downstream_addr_group("WWW.nghttp2.org", "/alpha", + groups, 255)); + + // path part is case-sensitive + CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "/Alpha/bravo", + groups, 255)); + + // unreserved characters are decoded before matching + CU_ASSERT(1 == match_downstream_addr_group("nghttp2.org", "/alpha/%62ravo/", + groups, 255)); + + CU_ASSERT(1 == match_downstream_addr_group( + "nghttp2.org", "/alpha/%62ravo/charlie", groups, 255)); + + CU_ASSERT(2 == match_downstream_addr_group("nghttp2.org", "/alpha/charlie", + groups, 255)); + + // pattern which does not end with '/' must match its entirely. So + // this matches to group 0, not group 2. + CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "/alpha/charlie/", + groups, 255)); + + // percent-encoding is normalized to upper case hex digits. + CU_ASSERT(3 == match_downstream_addr_group("nghttp2.org", "/delta%3a", groups, + 255)); + + // path component is normalized before mathcing + CU_ASSERT(1 == match_downstream_addr_group( + "nghttp2.org", "/alpha/charlie/%2e././bravo/delta/..", + groups, 255)); + + CU_ASSERT(255 == + match_downstream_addr_group("example.org", "/", groups, 255)); + + CU_ASSERT(255 == match_downstream_addr_group("", "/", groups, 255)); + + CU_ASSERT(255 == match_downstream_addr_group("foo/bar", "/", groups, 255)); + + // If path is "*", only match with host + "/". + CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "*", groups, 255)); + + CU_ASSERT(5 == match_downstream_addr_group("[::1]", "/", groups, 255)); + CU_ASSERT(5 == match_downstream_addr_group("[::1]:8080", "/", groups, 255)); + CU_ASSERT(255 == match_downstream_addr_group("[::1", "/", groups, 255)); + CU_ASSERT(255 == match_downstream_addr_group("[::1]8000", "/", groups, 255)); +} + } // namespace shrpx diff --git a/src/shrpx_config_test.h b/src/shrpx_config_test.h index 6d6d0353..5f9c170a 100644 --- a/src/shrpx_config_test.h +++ b/src/shrpx_config_test.h @@ -35,6 +35,7 @@ void test_shrpx_config_parse_config_str_list(void); void test_shrpx_config_parse_header(void); void test_shrpx_config_parse_log_format(void); void test_shrpx_config_read_tls_ticket_key_file(void); +void test_shrpx_config_match_downstream_addr_group(void); } // namespace shrpx diff --git a/src/shrpx_downstream_connection.h b/src/shrpx_downstream_connection.h index 49ca1beb..8d409f61 100644 --- a/src/shrpx_downstream_connection.h +++ b/src/shrpx_downstream_connection.h @@ -57,6 +57,7 @@ public: virtual void on_upstream_change(Upstream *uptream) = 0; virtual int on_priority_change(int32_t pri) = 0; + virtual size_t get_group() const = 0; // true if this object is poolable. virtual bool poolable() const = 0; diff --git a/src/shrpx_downstream_connection_pool.cc b/src/shrpx_downstream_connection_pool.cc index d762b770..eb9b1c8c 100644 --- a/src/shrpx_downstream_connection_pool.cc +++ b/src/shrpx_downstream_connection_pool.cc @@ -27,33 +27,42 @@ namespace shrpx { -DownstreamConnectionPool::DownstreamConnectionPool() {} +DownstreamConnectionPool::DownstreamConnectionPool(size_t num_groups) + : gpool_(num_groups) {} DownstreamConnectionPool::~DownstreamConnectionPool() { - for (auto dconn : pool_) { - delete dconn; + for (auto &pool : gpool_) { + for (auto dconn : pool) { + delete dconn; + } } } void DownstreamConnectionPool::add_downstream_connection( std::unique_ptr dconn) { - pool_.insert(dconn.release()); + auto group = dconn->get_group(); + assert(gpool_.size() > group); + gpool_[group].insert(dconn.release()); } std::unique_ptr -DownstreamConnectionPool::pop_downstream_connection() { - if (pool_.empty()) { +DownstreamConnectionPool::pop_downstream_connection(size_t group) { + assert(gpool_.size() > group); + auto &pool = gpool_[group]; + if (pool.empty()) { return nullptr; } - auto dconn = std::unique_ptr(*std::begin(pool_)); - pool_.erase(std::begin(pool_)); + auto dconn = std::unique_ptr(*std::begin(pool)); + pool.erase(std::begin(pool)); return dconn; } void DownstreamConnectionPool::remove_downstream_connection( DownstreamConnection *dconn) { - pool_.erase(dconn); + auto group = dconn->get_group(); + assert(gpool_.size() > group); + gpool_[group].erase(dconn); delete dconn; } diff --git a/src/shrpx_downstream_connection_pool.h b/src/shrpx_downstream_connection_pool.h index c2edce45..1fb889bb 100644 --- a/src/shrpx_downstream_connection_pool.h +++ b/src/shrpx_downstream_connection_pool.h @@ -36,15 +36,15 @@ class DownstreamConnection; class DownstreamConnectionPool { public: - DownstreamConnectionPool(); + DownstreamConnectionPool(size_t num_groups); ~DownstreamConnectionPool(); void add_downstream_connection(std::unique_ptr dconn); - std::unique_ptr pop_downstream_connection(); + std::unique_ptr pop_downstream_connection(size_t group); void remove_downstream_connection(DownstreamConnection *dconn); private: - std::set pool_; + std::vector> gpool_; }; } // namespace shrpx diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index 8a83f012..f82b5230 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -264,7 +264,7 @@ int Http2DownstreamConnection::push_request_headers() { // addr_idx here. auto addr_idx = http2session_->get_addr_idx(); auto downstream_hostport = - get_config()->downstream_addrs[addr_idx].hostport.get(); + get_config()->downstream_addr_groups[0].addrs[addr_idx].hostport.get(); const char *authority = nullptr, *host = nullptr; if (!no_host_rewrite) { diff --git a/src/shrpx_http2_downstream_connection.h b/src/shrpx_http2_downstream_connection.h index a2a78a34..736f546f 100644 --- a/src/shrpx_http2_downstream_connection.h +++ b/src/shrpx_http2_downstream_connection.h @@ -61,6 +61,8 @@ public: virtual void on_upstream_change(Upstream *upstream) {} virtual int on_priority_change(int32_t pri); + // Currently, HTTP/2 backend does not perform host-path mapping. + virtual size_t get_group() const { return 0; } // This object is not poolable because we dont' have facility to // migrate to another Http2Session object. diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index c071a722..5ed2f3c3 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -233,11 +233,16 @@ int Http2Session::disconnect(bool hard) { return 0; } -int Http2Session::check_cert() { return ssl::check_cert(conn_.tls.ssl); } +int Http2Session::check_cert() { + return ssl::check_cert( + conn_.tls.ssl, &get_config()->downstream_addr_groups[0].addrs[addr_idx_]); +} int Http2Session::initiate_connection() { int rv = 0; + auto &addrs = get_config()->downstream_addr_groups[0].addrs; + if (state_ == DISCONNECTED) { if (connect_blocker_->blocked()) { if (LOG_ENABLED(INFO)) { @@ -248,17 +253,19 @@ int Http2Session::initiate_connection() { } auto worker_stat = worker_->get_worker_stat(); - addr_idx_ = worker_stat->next_downstream; - ++worker_stat->next_downstream; - worker_stat->next_downstream %= get_config()->downstream_addrs.size(); + auto &next_downstream = worker_stat->next_downstream[0]; + addr_idx_ = next_downstream; + if (++next_downstream >= addrs.size()) { + next_downstream = 0; + } if (LOG_ENABLED(INFO)) { SSLOG(INFO, this) << "Using downstream address idx=" << addr_idx_ - << " out of " << get_config()->downstream_addrs.size(); + << " out of " << addrs.size(); } } - auto &downstream_addr = get_config()->downstream_addrs[addr_idx_]; + auto &downstream_addr = addrs[addr_idx_]; if (get_config()->downstream_http_proxy_host && state_ == DISCONNECTED) { if (LOG_ENABLED(INFO)) { @@ -503,7 +510,8 @@ int Http2Session::downstream_connect_proxy() { if (LOG_ENABLED(INFO)) { SSLOG(INFO, this) << "Connected to the proxy"; } - auto &downstream_addr = get_config()->downstream_addrs[addr_idx_]; + auto &downstream_addr = + get_config()->downstream_addr_groups[0].addrs[addr_idx_]; std::string req = "CONNECT "; req += downstream_addr.hostport.get(); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index d8016ca6..59553292 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -334,7 +334,7 @@ void Http2Upstream::initiate_downstream(Downstream *downstream) { int rv; rv = downstream->attach_downstream_connection( - handler_->get_downstream_connection()); + handler_->get_downstream_connection(downstream)); if (rv != 0) { // downstream connection fails, send error page if (error_reply(downstream, 503) != 0) { @@ -1476,7 +1476,7 @@ int Http2Upstream::on_downstream_reset(bool no_retry) { // downstream connection. rv = downstream->attach_downstream_connection( - handler_->get_downstream_connection()); + handler_->get_downstream_connection(downstream)); if (rv != 0) { goto fail; } diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 34f06b56..d0181b7e 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -109,12 +109,12 @@ void connectcb(struct ev_loop *loop, ev_io *w, int revents) { } // namespace HttpDownstreamConnection::HttpDownstreamConnection( - DownstreamConnectionPool *dconn_pool, struct ev_loop *loop) + DownstreamConnectionPool *dconn_pool, size_t group, struct ev_loop *loop) : DownstreamConnection(dconn_pool), conn_(loop, -1, nullptr, get_config()->downstream_write_timeout, get_config()->downstream_read_timeout, 0, 0, 0, 0, connectcb, readcb, timeoutcb, this), - ioctrl_(&conn_.rlimit), response_htp_{0}, addr_idx_(0), + ioctrl_(&conn_.rlimit), response_htp_{0}, group_(group), addr_idx_(0), connected_(false) {} HttpDownstreamConnection::~HttpDownstreamConnection() { @@ -143,14 +143,17 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { auto worker = client_handler_->get_worker(); auto worker_stat = worker->get_worker_stat(); - auto end = worker_stat->next_downstream; + auto &next_downstream = worker_stat->next_downstream[group_]; + auto end = next_downstream; + auto &addrs = get_config()->downstream_addr_groups[group_].addrs; for (;;) { - auto i = worker_stat->next_downstream; - ++worker_stat->next_downstream; - worker_stat->next_downstream %= get_config()->downstream_addrs.size(); + auto &addr = addrs[next_downstream]; + auto i = next_downstream; + if (++next_downstream >= addrs.size()) { + next_downstream = 0; + } - conn_.fd = util::create_nonblock_socket( - get_config()->downstream_addrs[i].addr.storage.ss_family); + conn_.fd = util::create_nonblock_socket(addr.addr.storage.ss_family); if (conn_.fd == -1) { auto error = errno; @@ -162,8 +165,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { } int rv; - rv = connect(conn_.fd, &get_config()->downstream_addrs[i].addr.sa, - get_config()->downstream_addrs[i].addrlen); + rv = connect(conn_.fd, &addr.addr.sa, addr.addrlen); if (rv != 0 && errno != EINPROGRESS) { auto error = errno; DCLOG(WARN, this) << "connect() failed; errno=" << error; @@ -172,7 +174,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { close(conn_.fd); conn_.fd = -1; - if (end == worker_stat->next_downstream) { + if (end == next_downstream) { return SHRPX_ERR_NETWORK; } @@ -212,8 +214,10 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { int HttpDownstreamConnection::push_request_headers() { const char *authority = nullptr, *host = nullptr; - auto downstream_hostport = - get_config()->downstream_addrs[addr_idx_].hostport.get(); + auto downstream_hostport = get_config() + ->downstream_addr_groups[group_] + .addrs[addr_idx_] + .hostport.get(); auto connect_method = downstream_->get_request_method() == HTTP_CONNECT; if (!get_config()->no_host_rewrite && !get_config()->http2_proxy && @@ -877,4 +881,6 @@ void HttpDownstreamConnection::on_upstream_change(Upstream *upstream) {} void HttpDownstreamConnection::signal_write() { conn_.wlimit.startw(); } +size_t HttpDownstreamConnection::get_group() const { return group_; } + } // namespace shrpx diff --git a/src/shrpx_http_downstream_connection.h b/src/shrpx_http_downstream_connection.h index 02480d5b..430a5edb 100644 --- a/src/shrpx_http_downstream_connection.h +++ b/src/shrpx_http_downstream_connection.h @@ -39,7 +39,7 @@ class DownstreamConnectionPool; class HttpDownstreamConnection : public DownstreamConnection { public: - HttpDownstreamConnection(DownstreamConnectionPool *dconn_pool, + HttpDownstreamConnection(DownstreamConnectionPool *dconn_pool, size_t group, struct ev_loop *loop); virtual ~HttpDownstreamConnection(); virtual int attach_downstream(Downstream *downstream); @@ -58,6 +58,7 @@ public: virtual void on_upstream_change(Upstream *upstream); virtual int on_priority_change(int32_t pri) { return 0; } + virtual size_t get_group() const; virtual bool poolable() const { return true; } @@ -68,6 +69,7 @@ private: Connection conn_; IOControl ioctrl_; http_parser response_htp_; + size_t group_; // index of get_config()->downstream_addrs this object is using size_t addr_idx_; bool connected_; diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 261d0c38..0fef7e6f 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -297,7 +297,7 @@ int htp_hdrs_completecb(http_parser *htp) { } rv = downstream->attach_downstream_connection( - upstream->get_client_handler()->get_downstream_connection()); + upstream->get_client_handler()->get_downstream_connection(downstream)); if (rv != 0) { downstream->set_request_state(Downstream::CONNECT_FAIL); @@ -993,7 +993,7 @@ int HttpsUpstream::on_downstream_reset(bool no_retry) { } rv = downstream_->attach_downstream_connection( - handler_->get_downstream_connection()); + handler_->get_downstream_connection(downstream_.get())); if (rv != 0) { goto fail; } diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 535909e6..eb23e107 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -271,7 +271,7 @@ void SpdyUpstream::start_downstream(Downstream *downstream) { void SpdyUpstream::initiate_downstream(Downstream *downstream) { int rv = downstream->attach_downstream_connection( - handler_->get_downstream_connection()); + handler_->get_downstream_connection(downstream)); if (rv != 0) { // If downstream connection fails, issue RST_STREAM. rst_stream(downstream, SPDYLAY_INTERNAL_ERROR); @@ -1104,7 +1104,7 @@ int SpdyUpstream::on_downstream_reset(bool no_retry) { // downstream connection. rv = downstream->attach_downstream_connection( - handler_->get_downstream_connection()); + handler_->get_downstream_connection(downstream)); if (rv != 0) { goto fail; } diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 1c04fe6c..60526653 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -743,7 +743,7 @@ void get_altnames(X509 *cert, std::vector &dns_names, } } -int check_cert(SSL *ssl) { +int check_cert(SSL *ssl, const DownstreamAddr *addr) { auto cert = SSL_get_peer_certificate(ssl); if (!cert) { LOG(ERROR) << "No certificate found"; @@ -760,9 +760,7 @@ int check_cert(SSL *ssl) { std::vector dns_names; std::vector ip_addrs; get_altnames(cert, dns_names, ip_addrs, common_name); - if (verify_hostname(get_config()->downstream_addrs[0].host.get(), - &get_config()->downstream_addrs[0].addr, - get_config()->downstream_addrs[0].addrlen, dns_names, + if (verify_hostname(addr->host.get(), &addr->addr, addr->addrlen, dns_names, ip_addrs, common_name) != 0) { LOG(ERROR) << "Certificate verification failed: hostname does not match"; return -1; diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h index ae428eaa..168b5f95 100644 --- a/src/shrpx_ssl.h +++ b/src/shrpx_ssl.h @@ -40,6 +40,7 @@ namespace shrpx { class ClientHandler; class Worker; class DownstreamConnectionPool; +struct DownstreamAddr; namespace ssl { @@ -68,7 +69,7 @@ ClientHandler *accept_connection(Worker *worker, int fd, sockaddr *addr, // Check peer's certificate against first downstream address in // Config::downstream_addrs. We only consider first downstream since // we use this function for HTTP/2 downstream link only. -int check_cert(SSL *ssl); +int check_cert(SSL *ssl, const DownstreamAddr *addr); // Retrieves DNS and IP address in subjectAltNames and commonName from // the |cert|. diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 9074b91e..a756e7c0 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -61,8 +61,11 @@ void mcpool_clear_cb(struct ev_loop *loop, ev_timer *w, int revents) { Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, ssl::CertLookupTree *cert_tree, const std::shared_ptr &ticket_keys) - : next_http2session_(0), loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), - cl_ssl_ctx_(cl_ssl_ctx), cert_tree_(cert_tree), ticket_keys_(ticket_keys), + : next_http2session_(0), + dconn_pool_(get_config()->downstream_addr_groups.size()), + worker_stat_(get_config()->downstream_addr_groups.size()), loop_(loop), + sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), cert_tree_(cert_tree), + ticket_keys_(ticket_keys), connect_blocker_(make_unique(loop_)), graceful_shutdown_(false) { ev_async_init(&w_, eventcb); diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 7a7a4e9e..f9924647 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -55,13 +55,13 @@ class CertLookupTree; } // namespace ssl struct WorkerStat { - WorkerStat() : num_connections(0), next_downstream(0) {} + WorkerStat(size_t num_groups) + : num_connections(0), next_downstream(num_groups) {} size_t num_connections; - // Next downstream index in Config::downstream_addrs. For HTTP/2 - // downstream connections, this is always 0. For HTTP/1, this is + // Next downstream index in Config::downstream_addr_groups. This is // used as load balancing. - size_t next_downstream; + std::vector next_downstream; }; enum WorkerEventType { diff --git a/src/util.h b/src/util.h index 9ee2b115..d43d3c76 100644 --- a/src/util.h +++ b/src/util.h @@ -386,9 +386,13 @@ bool streq_l(const char (&a)[N], InputIt b, size_t blen) { bool strifind(const char *a, const char *b); +template void inp_strlower(InputIt first, InputIt last) { + std::transform(first, last, first, lowcase); +} + // Lowercase |s| in place. inline void inp_strlower(std::string &s) { - std::transform(std::begin(s), std::end(s), std::begin(s), lowcase); + inp_strlower(std::begin(s), std::end(s)); } // Returns string representation of |n| with 2 fractional digits. From 6d556755ee437768569b14561fe28b6167e4b010 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 02:08:16 +0900 Subject: [PATCH 02/77] Attemp to fix travis build error --- src/shrpx_config.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 3fdbe425..11f29bb0 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -511,7 +511,7 @@ int parse_config(const char *opt, const char *optarg) { if (util::strieq(opt, SHRPX_OPT_BACKEND)) { auto optarglen = strlen(optarg); - auto pat_delim = strchr(optarg, ';'); + const char *pat_delim = strchr(optarg, ';'); if (!pat_delim) { pat_delim = optarg + optarglen; } From d457f39b1ec42498674654ff82230c9fa8872819 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 02:41:33 +0900 Subject: [PATCH 03/77] nghttpx: Fix unix domain backend --- src/shrpx_config.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 11f29bb0..5ea100f4 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -518,7 +518,7 @@ int parse_config(const char *opt, const char *optarg) { DownstreamAddr addr; if (util::istartsWith(optarg, SHRPX_UNIX_PATH_PREFIX)) { auto path = optarg + str_size(SHRPX_UNIX_PATH_PREFIX); - addr.host = strcopy(path); + addr.host = strcopy(path, pat_delim - path); addr.host_unix = true; } else { if (split_host_port(host, sizeof(host), &port, optarg, From c2e4ed96242ebdf69e8e0142558e98e5138da3f5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 12:43:48 +0900 Subject: [PATCH 04/77] nghttpx: Deal with the path without trailing slash on pattern match If pattern ends with '/', and pattern and path matches without that slash, we consider they match to deal with request to the directory without trailing slash. That is if pattern is "/foo/" and path is "/foo", we consider they match. --- src/shrpx.cc | 66 +++++++++++++++++++++++++--------------- src/shrpx_config.cc | 12 +++++++- src/shrpx_config_test.cc | 7 +++++ 3 files changed, 60 insertions(+), 25 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 8bfadbc3..c9524e25 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1002,38 +1002,56 @@ Connections: Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX domain socket can be specified by prefixing path name - with "unix:" (e.g., unix:/var/run/backend.sock). + with "unix:" (e.g., unix:/var/run/backend.sock). + Optionally, if s are given, the backend address is only used if request matches the pattern. If -s, -p, --client or --http2-bridge is used, s are ignored. The pattern matching is closely designed to ServeMux in net/http package of Go programming language. - consists of path, host + path or host. The - path must starts with "/". If it ends with "/", it + consists of path, host + path or just host. + The path must starts with "/". If it ends with "/", it matches to the request path whose prefix is the path. - If it does not end with "/", it performs exact match - against the request path. If host is given, it performs - exact match against the request host. If host alone is - given, "/" is appended to it, so that it matches all - paths under the host (e.g., specifying "nghttp2.org" - equals to "nghttp2.org/"). Longer patterns take - precedence over shorter ones, breaking a tie by the - order of the appearance in the configuration. If - is omitted, "/" is used as pattern, which - matches all paths (catch-all pattern). For example, - -b'127.0.0.1,8080;nghttp2.org/httpbin/' matches the - request host "nghttp2.org" and the request path - "/httpbin/get", but does not match the request host - "nghttp2.org" and the request path "/index.html". The - multiple s can be specified, delimiting them by - ":". Specifying + To deal with the request to the directory without + trailing slash, pattern which ends with "/" also matches + the path if pattern == path + "/" (e.g., pattern "/foo/" + matches path "/foo"). If it does not end with "/", it + performs exact match against the request path. If host + is given, it performs exact match against the request + host. If host alone is given, "/" is appended to it, so + that it matches all paths under the host (e.g., + specifying "nghttp2.org" equals to "nghttp2.org/"). + + Longer patterns take precedence over shorter ones, + breaking a tie by the order of the appearance in the + configuration. + + If is omitted, "/" is used as pattern, which + matches all paths (catch-all pattern). The catch-all + backend must be given. + + When doing a match, nghttpx made some normalization to + pattern, request host and path. For host part, they are + converted to lower case. For path part, percent-encoded + unreserved characters defined in RFC 3986 are decoded, + and any dot-segments (".." and ".") are resolved and + removed. + + For example, -b'127.0.0.1,8080;nghttp2.org/httpbin/' + matches the request host "nghttp2.org" and the request + path "/httpbin/get", but does not match the request host + "nghttp2.org" and the request path "/index.html". + + The multiple s can be specified, delimiting + them by ":". Specifying -b'127.0.0.1,8080;nghttp2.org:www.nghttp2.org' has the same effect to specify -b'127.0.0.1,8080;nghttp2.org' - and -b'127.0.0.1,8080:www.nghttp2.org'. The backend - addresses sharing same are grouped together - forming load balancing group. Since ";" and ":" are - used as delimiter, must not contain these - characters. + and -b'127.0.0.1,8080:www.nghttp2.org'. + + The backend addresses sharing same are grouped + together forming load balancing group. Since ";" and + ":" are used as delimiter, must not contain + these characters. Default: )" << DEFAULT_DOWNSTREAM_HOST << "," << DEFAULT_DOWNSTREAM_PORT << R"( -f, --frontend=, diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 5ea100f4..66b10ccf 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1402,7 +1402,17 @@ bool path_match(const std::string &pattern, const std::string &path) { if (pattern.back() != '/') { return pattern == path; } - return util::startsWith(path, pattern); + + if (util::startsWith(path, pattern)) { + return true; + } + + // If pattern ends with '/', and pattern and path matches without + // that slash, we consider they match to deal with request to the + // directory without trailing slash. That is if pattern is "/foo/" + // and path is "/foo", we consider they match. + return util::streq(std::begin(path), path.size(), std::begin(pattern), + pattern.size() - 1); } } // namespace diff --git a/src/shrpx_config_test.cc b/src/shrpx_config_test.cc index 4f21fc48..9f458d43 100644 --- a/src/shrpx_config_test.cc +++ b/src/shrpx_config_test.cc @@ -192,6 +192,13 @@ void test_shrpx_config_match_downstream_addr_group(void) { CU_ASSERT(4 == match_downstream_addr_group("WWW.nghttp2.org", "/alpha", groups, 255)); + CU_ASSERT(1 == match_downstream_addr_group("nghttp2.org", "/alpha/bravo/", + groups, 255)); + + // /alpha/bravo also matches /alpha/bravo/ + CU_ASSERT(1 == match_downstream_addr_group("nghttp2.org", "/alpha/bravo", + groups, 255)); + // path part is case-sensitive CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "/Alpha/bravo", groups, 255)); From d62e4dbc5e00523fea46fc7c0197e744c17c0249 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 12:45:23 +0900 Subject: [PATCH 05/77] Update man pages --- doc/h2load.1 | 8 +++++- doc/h2load.1.rst | 5 ++++ doc/nghttp.1 | 8 +++++- doc/nghttp.1.rst | 5 ++++ doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 66 ++++++++++++++++++++++++++++++++++++++++++++--- doc/nghttpx.1.rst | 60 +++++++++++++++++++++++++++++++++++++++--- 7 files changed, 144 insertions(+), 10 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index fd3009c6..0b2924f3 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "June 27, 2015" "1.0.5" "nghttp2" +.TH "H2LOAD" "1" "July 11, 2015" "1.0.6-DEV" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . @@ -113,6 +113,12 @@ Add/Override a header to the requests. .UNINDENT .INDENT 0.0 .TP +.B \-\-ciphers= +Set allowed cipher list. The format of the string is +described in OpenSSL ciphers(1). +.UNINDENT +.INDENT 0.0 +.TP .B \-p, \-\-no\-tls\-proto= Specify ALPN identifier of the protocol to be used when accessing http URI without SSL/TLS. diff --git a/doc/h2load.1.rst b/doc/h2load.1.rst index 21e5a425..ba1a820a 100644 --- a/doc/h2load.1.rst +++ b/doc/h2load.1.rst @@ -84,6 +84,11 @@ OPTIONS Add/Override a header to the requests. +.. option:: --ciphers= + + Set allowed cipher list. The format of the string is + described in OpenSSL ciphers(1). + .. option:: -p, --no-tls-proto= Specify ALPN identifier of the protocol to be used when diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 79b42b0c..7fb39c33 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "June 27, 2015" "1.0.5" "nghttp2" +.TH "NGHTTP" "1" "July 11, 2015" "1.0.6-DEV" "nghttp2" .SH NAME nghttp \- HTTP/2 experimental client . @@ -205,6 +205,12 @@ Disable server push. .UNINDENT .INDENT 0.0 .TP +.B \-\-max\-concurrent\-streams= +The number of concurrent pushed streams this client +accepts. +.UNINDENT +.INDENT 0.0 +.TP .B \-\-version Display version information and exit. .UNINDENT diff --git a/doc/nghttp.1.rst b/doc/nghttp.1.rst index 7f2449b2..b22d0c22 100644 --- a/doc/nghttp.1.rst +++ b/doc/nghttp.1.rst @@ -158,6 +158,11 @@ OPTIONS Disable server push. +.. option:: --max-concurrent-streams= + + The number of concurrent pushed streams this client + accepts. + .. option:: --version Display version information and exit. diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index 138a4bbf..5a477567 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "June 27, 2015" "1.0.5" "nghttp2" +.TH "NGHTTPD" "1" "July 11, 2015" "1.0.6-DEV" "nghttp2" .SH NAME nghttpd \- HTTP/2 experimental server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index 4cd1c3d9..bf67f98f 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "June 27, 2015" "1.0.5" "nghttp2" +.TH "NGHTTPX" "1" "July 11, 2015" "1.0.6-DEV" "nghttp2" .SH NAME nghttpx \- HTTP/2 experimental proxy . @@ -55,17 +55,66 @@ The options are categorized into several groups. .SS Connections .INDENT 0.0 .TP -.B \-b, \-\-backend= +.B \-b, \-\-backend=,[;[:...]] Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX domain socket can be specified by prefixing path name -with "unix:" (e.g., unix:/var/run/backend.sock) +with "unix:" (e.g., unix:/var/run/backend.sock). +.sp +Optionally, if s are given, the backend address +is only used if request matches the pattern. If \fI\%\-s\fP, \fI\%\-p\fP, +\fI\%\-\-client\fP or \fI\%\-\-http2\-bridge\fP is used, s are +ignored. The pattern matching is closely designed to +ServeMux in net/http package of Go programming language. + consists of path, host + path or just host. +The path must starts with "\fI/\fP". If it ends with "\fI/\fP", it +matches to the request path whose prefix is the path. +To deal with the request to the directory without +trailing slash, pattern which ends with "\fI/\fP" also matches +the path if pattern == path + "\fI/\fP" (e.g., pattern "\fI/foo/\fP" +matches path "\fI/foo\fP"). If it does not end with "\fI/\fP", it +performs exact match against the request path. If host +is given, it performs exact match against the request +host. If host alone is given, "\fI/\fP" is appended to it, so +that it matches all paths under the host (e.g., +specifying "nghttp2.org" equals to "nghttp2.org/"). +.sp +Longer patterns take precedence over shorter ones, +breaking a tie by the order of the appearance in the +configuration. +.sp +If is omitted, "\fI/\fP" is used as pattern, which +matches all paths (catch\-all pattern). The catch\-all +backend must be given. +.sp +When doing a match, nghttpx made some normalization to +pattern, request host and path. For host part, they are +converted to lower case. For path part, percent\-encoded +unreserved characters defined in RFC 3986 are decoded, +and any dot\-segments (".." and ".") are resolved and +removed. +.sp +For example, \fI\%\-b\fP\(aq127.0.0.1,8080;nghttp2.org/httpbin/\(aq +matches the request host "nghttp2.org" and the request +path "\fI/httpbin/get\fP", but does not match the request host +"nghttp2.org" and the request path "\fI/index.html\fP". +.sp +The multiple s can be specified, delimiting +them by ":". Specifying +\fI\%\-b\fP\(aq127.0.0.1,8080;nghttp2.org:www.nghttp2.org\(aq has the +same effect to specify \fI\%\-b\fP\(aq127.0.0.1,8080;nghttp2.org\(aq +and \fI\%\-b\fP\(aq127.0.0.1,8080:www.nghttp2.org\(aq. +.sp +The backend addresses sharing same are grouped +together forming load balancing group. Since ";" and +":" are used as delimiter, must not contain +these characters. .sp Default: \fB127.0.0.1,80\fP .UNINDENT .INDENT 0.0 .TP -.B \-f, \-\-frontend= +.B \-f, \-\-frontend=, Set frontend host and port. If is \(aq*\(aq, it assumes all addresses including both IPv4 and IPv6. UNIX domain socket can be specified by prefixing path @@ -626,6 +675,15 @@ $pid: PID of the running process. $alpn: ALPN identifier of the protocol which generates the response. For HTTP/1, ALPN is always http/1.1, regardless of minor version. +.IP \(bu 2 +$ssl_cipher: cipher used for SSL/TLS connection. +.IP \(bu 2 +$ssl_protocol: protocol for SSL/TLS connection. +.IP \(bu 2 +$ssl_session_id: session ID for SSL/TLS connection. +.IP \(bu 2 +$ssl_session_reused: "r" if SSL/TLS session was +reused. Otherwise, "." .UNINDENT .sp Default: \fB$remote_addr \- \- [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"\fP diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index ac904fb9..3132dca0 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -37,16 +37,65 @@ The options are categorized into several groups. Connections ~~~~~~~~~~~ -.. option:: -b, --backend= +.. option:: -b, --backend=,[;[:...]] Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX domain socket can be specified by prefixing path name - with "unix:" (e.g., unix:/var/run/backend.sock) + with "unix:" (e.g., unix:/var/run/backend.sock). + + Optionally, if s are given, the backend address + is only used if request matches the pattern. If :option:`-s`\, :option:`-p`\, + :option:`--client` or :option:`\--http2-bridge` is used, s are + ignored. The pattern matching is closely designed to + ServeMux in net/http package of Go programming language. + consists of path, host + path or just host. + The path must starts with "*/*". If it ends with "*/*", it + matches to the request path whose prefix is the path. + To deal with the request to the directory without + trailing slash, pattern which ends with "*/*" also matches + the path if pattern == path + "*/*" (e.g., pattern "*/foo/*" + matches path "*/foo*"). If it does not end with "*/*", it + performs exact match against the request path. If host + is given, it performs exact match against the request + host. If host alone is given, "*/*" is appended to it, so + that it matches all paths under the host (e.g., + specifying "nghttp2.org" equals to "nghttp2.org/"). + + Longer patterns take precedence over shorter ones, + breaking a tie by the order of the appearance in the + configuration. + + If is omitted, "*/*" is used as pattern, which + matches all paths (catch-all pattern). The catch-all + backend must be given. + + When doing a match, nghttpx made some normalization to + pattern, request host and path. For host part, they are + converted to lower case. For path part, percent-encoded + unreserved characters defined in RFC 3986 are decoded, + and any dot-segments (".." and ".") are resolved and + removed. + + For example, :option:`-b`\'127.0.0.1,8080;nghttp2.org/httpbin/' + matches the request host "nghttp2.org" and the request + path "*/httpbin/get*", but does not match the request host + "nghttp2.org" and the request path "*/index.html*". + + The multiple s can be specified, delimiting + them by ":". Specifying + :option:`-b`\'127.0.0.1,8080;nghttp2.org:www.nghttp2.org' has the + same effect to specify :option:`-b`\'127.0.0.1,8080;nghttp2.org' + and :option:`-b`\'127.0.0.1,8080:www.nghttp2.org'. + + The backend addresses sharing same are grouped + together forming load balancing group. Since ";" and + ":" are used as delimiter, must not contain + these characters. Default: ``127.0.0.1,80`` -.. option:: -f, --frontend= +.. option:: -f, --frontend=, Set frontend host and port. If is '\*', it assumes all addresses including both IPv4 and IPv6. @@ -550,6 +599,11 @@ Logging * $alpn: ALPN identifier of the protocol which generates the response. For HTTP/1, ALPN is always http/1.1, regardless of minor version. + * $ssl_cipher: cipher used for SSL/TLS connection. + * $ssl_protocol: protocol for SSL/TLS connection. + * $ssl_session_id: session ID for SSL/TLS connection. + * $ssl_session_reused: "r" if SSL/TLS session was + reused. Otherwise, "." Default: ``$remote_addr - - [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"`` From 19e47a192237faf8619c883b60b24a1a23bf4ac4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 16:12:35 +0900 Subject: [PATCH 06/77] nghttpx: Normalize path when setting it to Downstream --- src/http2.h | 15 +++++++++++++++ src/http2_test.cc | 25 +++++++++++++++++++++++++ src/http2_test.h | 1 + src/shrpx-unittest.cc | 2 ++ src/shrpx_config.cc | 25 ++++++++++++++----------- src/shrpx_config_test.cc | 17 +++-------------- src/shrpx_http2_upstream.cc | 9 +++++++-- src/shrpx_https_upstream.cc | 6 +++++- src/shrpx_spdy_upstream.cc | 3 ++- 9 files changed, 74 insertions(+), 29 deletions(-) diff --git a/src/http2.h b/src/http2.h index be79caca..182c45d9 100644 --- a/src/http2.h +++ b/src/http2.h @@ -334,6 +334,21 @@ std::string normalize_path(InputIt first, InputIt last) { nullptr, 0); } +template +std::string rewrite_clean_path(InputIt first, InputIt last) { + if (first == last || *first != '/') { + return std::string(first, last); + } + // probably, not necessary most of the case, but just in case. + auto fragment = std::find(first, last, '#'); + auto query = std::find(first, fragment, '?'); + auto path = normalize_path(first, query); + if (query != fragment) { + path.append(query, fragment); + } + return path; +} + } // namespace http2 } // namespace nghttp2 diff --git a/src/http2_test.cc b/src/http2_test.cc index 1937bfaf..88339045 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -856,4 +856,29 @@ void test_http2_normalize_path(void) { CU_ASSERT("/" == http2::normalize_path(std::begin(src), std::end(src))); } +void test_http2_rewrite_clean_path(void) { + std::string src; + + // unreserved characters + src = "/alpha/%62ravo/"; + CU_ASSERT("/alpha/bravo/" == + http2::rewrite_clean_path(std::begin(src), std::end(src))); + + // percent-encoding is converted to upper case. + src = "/delta%3a"; + CU_ASSERT("/delta%3A" == + http2::rewrite_clean_path(std::begin(src), std::end(src))); + + // path component is normalized before mathcing + src = "/alpha/charlie/%2e././bravo/delta/.."; + CU_ASSERT("/alpha/bravo/" == + http2::rewrite_clean_path(std::begin(src), std::end(src))); + + src = "alpha%3a"; + CU_ASSERT(src == http2::rewrite_clean_path(std::begin(src), std::end(src))); + + src = ""; + CU_ASSERT(src == http2::rewrite_clean_path(std::begin(src), std::end(src))); +} + } // namespace shrpx diff --git a/src/http2_test.h b/src/http2_test.h index 193f23e3..bc65d453 100644 --- a/src/http2_test.h +++ b/src/http2_test.h @@ -46,6 +46,7 @@ void test_http2_mandatory_request_headers_presence(void); void test_http2_parse_link_header(void); void test_http2_path_join(void); void test_http2_normalize_path(void); +void test_http2_rewrite_clean_path(void); } // namespace shrpx diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index 7030aa3d..83695204 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -98,6 +98,8 @@ int main(int argc, char *argv[]) { !CU_add_test(pSuite, "http2_path_join", shrpx::test_http2_path_join) || !CU_add_test(pSuite, "http2_normalize_path", shrpx::test_http2_normalize_path) || + !CU_add_test(pSuite, "http2_rewrite_clean_path", + shrpx::test_http2_rewrite_clean_path) || !CU_add_test(pSuite, "downstream_index_request_headers", shrpx::test_downstream_index_request_headers) || !CU_add_test(pSuite, "downstream_index_response_headers", diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 66b10ccf..ca130e77 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1438,9 +1438,9 @@ ssize_t match(const std::string &path, namespace { size_t match_downstream_addr_group_host( - const std::string &host, const std::string &raw_path, + const std::string &host, const std::string &path, const std::vector &groups, size_t catch_all) { - if (raw_path == "*") { + if (path.empty() || path[0] != '/') { auto group = match(host + "/", groups); if (group != -1) { if (LOG_ENABLED(INFO)) { @@ -1452,11 +1452,6 @@ size_t match_downstream_addr_group_host( return catch_all; } - // probably, not necessary most of the case, but just in case. - auto fragment = std::find(std::begin(raw_path), std::end(raw_path), '#'); - auto query = std::find(std::begin(raw_path), fragment, '?'); - auto path = http2::normalize_path(std::begin(raw_path), query); - if (LOG_ENABLED(INFO)) { LOG(INFO) << "Perform mapping selection, using host=" << host << ", path=" << path; @@ -1490,13 +1485,21 @@ size_t match_downstream_addr_group_host( size_t match_downstream_addr_group( const std::string &hostport, const std::string &raw_path, const std::vector &groups, size_t catch_all) { - if (hostport.empty() || - std::find(std::begin(hostport), std::end(hostport), '/') != - std::end(hostport)) { + if (std::find(std::begin(hostport), std::end(hostport), '/') != + std::end(hostport)) { // We use '/' specially, and if '/' is included in host, it breaks // our code. Select catch-all case. return catch_all; } + + auto fragment = std::find(std::begin(raw_path), std::end(raw_path), '#'); + auto query = std::find(std::begin(raw_path), fragment, '?'); + auto path = std::string(std::begin(raw_path), query); + + if (hostport.empty()) { + return match_downstream_addr_group_host(hostport, path, groups, catch_all); + } + std::string host; if (hostport[0] == '[') { // assume this is IPv6 numeric address @@ -1517,7 +1520,7 @@ size_t match_downstream_addr_group( } util::inp_strlower(host); - return match_downstream_addr_group_host(host, raw_path, groups, catch_all); + return match_downstream_addr_group_host(host, path, groups, catch_all); } } // namespace shrpx diff --git a/src/shrpx_config_test.cc b/src/shrpx_config_test.cc index 9f458d43..ab6a0f27 100644 --- a/src/shrpx_config_test.cc +++ b/src/shrpx_config_test.cc @@ -203,12 +203,8 @@ void test_shrpx_config_match_downstream_addr_group(void) { CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "/Alpha/bravo", groups, 255)); - // unreserved characters are decoded before matching - CU_ASSERT(1 == match_downstream_addr_group("nghttp2.org", "/alpha/%62ravo/", - groups, 255)); - CU_ASSERT(1 == match_downstream_addr_group( - "nghttp2.org", "/alpha/%62ravo/charlie", groups, 255)); + "nghttp2.org", "/alpha/bravo/charlie", groups, 255)); CU_ASSERT(2 == match_downstream_addr_group("nghttp2.org", "/alpha/charlie", groups, 255)); @@ -218,20 +214,13 @@ void test_shrpx_config_match_downstream_addr_group(void) { CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "/alpha/charlie/", groups, 255)); - // percent-encoding is normalized to upper case hex digits. - CU_ASSERT(3 == match_downstream_addr_group("nghttp2.org", "/delta%3a", groups, - 255)); - - // path component is normalized before mathcing - CU_ASSERT(1 == match_downstream_addr_group( - "nghttp2.org", "/alpha/charlie/%2e././bravo/delta/..", - groups, 255)); - CU_ASSERT(255 == match_downstream_addr_group("example.org", "/", groups, 255)); CU_ASSERT(255 == match_downstream_addr_group("", "/", groups, 255)); + CU_ASSERT(255 == match_downstream_addr_group("", "alpha", groups, 255)); + CU_ASSERT(255 == match_downstream_addr_group("foo/bar", "/", groups, 255)); // If path is "*", only match with host + "/". diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 59553292..f0b3b213 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -300,7 +300,11 @@ int Http2Upstream::on_request_headers(Downstream *downstream, downstream->set_request_method(method_token); downstream->set_request_http2_scheme(http2::value_to_str(scheme)); downstream->set_request_http2_authority(http2::value_to_str(authority)); - downstream->set_request_path(http2::value_to_str(path)); + if (path) { + auto &value = path->value; + downstream->set_request_path( + http2::rewrite_clean_path(std::begin(value), std::end(value))); + } if (!(frame->hd.flags & NGHTTP2_FLAG_END_STREAM)) { downstream->set_request_http2_expect_body(true); @@ -541,7 +545,8 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame, {nv.value, nv.value + nv.valuelen}); break; case http2::HD__PATH: - downstream->set_request_path({nv.value, nv.value + nv.valuelen}); + downstream->set_request_path( + http2::rewrite_clean_path(nv.value, nv.value + nv.valuelen)); break; } downstream->add_request_header(nv.name, nv.namelen, nv.value, nv.valuelen, diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 0fef7e6f..d1732e7a 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -218,7 +218,8 @@ void rewrite_request_host_path_from_uri(Downstream *downstream, const char *uri, path += '?'; path.append(uri + fdata.off, fdata.len); } - downstream->set_request_path(std::move(path)); + downstream->set_request_path( + http2::rewrite_clean_path(std::begin(path), std::end(path))); std::string scheme; http2::copy_url_component(scheme, &u, UF_SCHEMA, uri); @@ -286,6 +287,9 @@ int htp_hdrs_completecb(http_parser *htp) { return -1; } + downstream->set_request_path( + http2::rewrite_clean_path(std::begin(uri), std::end(uri))); + if (upstream->get_client_handler()->get_ssl()) { downstream->set_request_http2_scheme("https"); } else { diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index eb23e107..d6ac4f6c 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -229,7 +229,8 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, } else { downstream->set_request_http2_scheme(scheme->value); downstream->set_request_http2_authority(host->value); - downstream->set_request_path(path->value); + downstream->set_request_path(http2::rewrite_clean_path( + std::begin(path->value), std::end(path->value))); } if (!(frame->syn_stream.hd.flags & SPDYLAY_CTRL_FLAG_FIN)) { From e7724914a9aad9b9fafebeda2c0c61878b0944d4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 16:42:23 +0900 Subject: [PATCH 07/77] nghttpx: Less copy when matching path --- src/shrpx_config.cc | 54 ++++++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index ca130e77..8508cb93 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1398,12 +1398,19 @@ int int_syslog_facility(const char *strfacility) { } namespace { -bool path_match(const std::string &pattern, const std::string &path) { +template +bool path_match(const std::string &pattern, const std::string &host, + InputIt path_first, InputIt path_last) { if (pattern.back() != '/') { - return pattern == path; + return pattern.size() == host.size() + (path_last - path_first) && + std::equal(std::begin(host), std::end(host), std::begin(pattern)) && + std::equal(path_first, path_last, std::begin(pattern) + host.size()); } - if (util::startsWith(path, pattern)) { + if (pattern.size() >= host.size() && + std::equal(std::begin(host), std::end(host), std::begin(pattern)) && + util::startsWith(path_first, path_last, std::begin(pattern) + host.size(), + std::end(pattern))) { return true; } @@ -1411,20 +1418,24 @@ bool path_match(const std::string &pattern, const std::string &path) { // that slash, we consider they match to deal with request to the // directory without trailing slash. That is if pattern is "/foo/" // and path is "/foo", we consider they match. - return util::streq(std::begin(path), path.size(), std::begin(pattern), - pattern.size() - 1); + + assert(!pattern.empty()); + return pattern.size() - 1 == host.size() + (path_last - path_first) && + std::equal(std::begin(host), std::end(host), std::begin(pattern)) && + std::equal(path_first, path_last, std::begin(pattern) + host.size()); } } // namespace namespace { -ssize_t match(const std::string &path, +template +ssize_t match(const std::string &host, InputIt path_first, InputIt path_last, const std::vector &groups) { ssize_t res = -1; size_t best = 0; for (size_t i = 0; i < groups.size(); ++i) { auto &g = groups[i]; auto &pattern = g.pattern; - if (!path_match(pattern, path)) { + if (!path_match(pattern, host, path_first, path_last)) { continue; } if (res == -1 || best < pattern.size()) { @@ -1437,11 +1448,13 @@ ssize_t match(const std::string &path, } // namespace namespace { +template size_t match_downstream_addr_group_host( - const std::string &host, const std::string &path, + const std::string &host, InputIt path_first, InputIt path_last, const std::vector &groups, size_t catch_all) { - if (path.empty() || path[0] != '/') { - auto group = match(host + "/", groups); + if (path_first == path_last || *path_first != '/') { + constexpr const char P[] = "/"; + auto group = match(host, P, P + 1, groups); if (group != -1) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Found pattern with query " << host @@ -1454,22 +1467,24 @@ size_t match_downstream_addr_group_host( if (LOG_ENABLED(INFO)) { LOG(INFO) << "Perform mapping selection, using host=" << host - << ", path=" << path; + << ", path=" << std::string(path_first, path_last); } - auto group = match(host + path, groups); + auto group = match(host, path_first, path_last, groups); if (group != -1) { if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Found pattern with query " << host + path + LOG(INFO) << "Found pattern with query " << host + << std::string(path_first, path_last) << ", matched pattern=" << groups[group].pattern; } return group; } - group = match(path, groups); + group = match("", path_first, path_last, groups); if (group != -1) { if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Found pattern with query " << path + LOG(INFO) << "Found pattern with query " + << std::string(path_first, path_last) << ", matched pattern=" << groups[group].pattern; } return group; @@ -1494,10 +1509,12 @@ size_t match_downstream_addr_group( auto fragment = std::find(std::begin(raw_path), std::end(raw_path), '#'); auto query = std::find(std::begin(raw_path), fragment, '?'); - auto path = std::string(std::begin(raw_path), query); + auto path_first = std::begin(raw_path); + auto path_last = query; if (hostport.empty()) { - return match_downstream_addr_group_host(hostport, path, groups, catch_all); + return match_downstream_addr_group_host(hostport, path_first, path_last, + groups, catch_all); } std::string host; @@ -1520,7 +1537,8 @@ size_t match_downstream_addr_group( } util::inp_strlower(host); - return match_downstream_addr_group_host(host, path, groups, catch_all); + return match_downstream_addr_group_host(host, path_first, path_last, groups, + catch_all); } } // namespace shrpx From 7c216c6df8d6776582d03a39094263df08498f24 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 16:45:07 +0900 Subject: [PATCH 08/77] nghttpx: Document that patterns with host take precedence --- src/shrpx.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index c9524e25..389bf7e5 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1022,9 +1022,10 @@ Connections: that it matches all paths under the host (e.g., specifying "nghttp2.org" equals to "nghttp2.org/"). - Longer patterns take precedence over shorter ones, - breaking a tie by the order of the appearance in the - configuration. + Patterns with host take precedence over path only + patterns. Then, longer patterns take precedence over + shorter ones, breaking a tie by the order of the + appearance in the configuration. If is omitted, "/" is used as pattern, which matches all paths (catch-all pattern). The catch-all From 1a63cd94aa690df48432aa8d8ad292094c67b023 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 17:30:38 +0900 Subject: [PATCH 09/77] nghttpx: Pass by reference, since it just get copied there --- src/shrpx_config.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 8508cb93..ebaf2b23 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -467,7 +467,7 @@ namespace { // config. We will store each host-path pattern found in |src| with // |addr|. |addr| will be copied accordingly. Also we make a group // based on the pattern. The "/" pattern is considered as catch-all. -void parse_mapping(DownstreamAddr addr, const char *src) { +void parse_mapping(const DownstreamAddr &addr, const char *src) { // This returns at least 1 element (it could be empty string). We // will append '/' to all patterns, so it becomes catch-all pattern. auto mapping = parse_config_str_list(src, ':'); @@ -537,7 +537,7 @@ int parse_config(const char *opt, const char *optarg) { LOG(ERROR) << opt << ": ';' must not be used in pattern"; return -1; } - parse_mapping(std::move(addr), mapping); + parse_mapping(addr, mapping); return 0; } From fa7069a27388c085e22a3901bce744d36aa72671 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 17:50:58 +0900 Subject: [PATCH 10/77] nghttpx: Don't rewrite path if http2 proxy or client proxy is enabled There are many requests which changes its meaning when we rewrite path. This is due to bad percent-encoding in URI; reserved characters are just used without percent encoding. It seems this is common in ad services, but I suspect more to come. For reverse proxying situation, sane service most likely encodes URI properly, so probably this is not an issue. --- src/shrpx_http2_upstream.cc | 10 +++++++--- src/shrpx_https_upstream.cc | 8 ++++++-- src/shrpx_spdy_upstream.cc | 8 ++++++-- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index f0b3b213..07ac4e3d 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -301,9 +301,13 @@ int Http2Upstream::on_request_headers(Downstream *downstream, downstream->set_request_http2_scheme(http2::value_to_str(scheme)); downstream->set_request_http2_authority(http2::value_to_str(authority)); if (path) { - auto &value = path->value; - downstream->set_request_path( - http2::rewrite_clean_path(std::begin(value), std::end(value))); + if (get_config()->http2_proxy || get_config()->client_proxy) { + downstream->set_request_path(http2::value_to_str(path)); + } else { + auto &value = path->value; + downstream->set_request_path( + http2::rewrite_clean_path(std::begin(value), std::end(value))); + } } if (!(frame->hd.flags & NGHTTP2_FLAG_END_STREAM)) { diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index d1732e7a..1a832e60 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -218,8 +218,12 @@ void rewrite_request_host_path_from_uri(Downstream *downstream, const char *uri, path += '?'; path.append(uri + fdata.off, fdata.len); } - downstream->set_request_path( - http2::rewrite_clean_path(std::begin(path), std::end(path))); + if (get_config()->http2_proxy || get_config()->client_proxy) { + downstream->set_request_path(std::move(path)); + } else { + downstream->set_request_path( + http2::rewrite_clean_path(std::begin(path), std::end(path))); + } std::string scheme; http2::copy_url_component(scheme, &u, UF_SCHEMA, uri); diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index d6ac4f6c..52f267da 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -229,8 +229,12 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, } else { downstream->set_request_http2_scheme(scheme->value); downstream->set_request_http2_authority(host->value); - downstream->set_request_path(http2::rewrite_clean_path( - std::begin(path->value), std::end(path->value))); + if (get_config()->http2_proxy || get_config()->client_proxy) { + downstream->set_request_path(path->value); + } else { + downstream->set_request_path(http2::rewrite_clean_path( + std::begin(path->value), std::end(path->value))); + } } if (!(frame->syn_stream.hd.flags & SPDYLAY_CTRL_FLAG_FIN)) { From 8a2543d7b7f13ca34701829ebdfe1ad0d8b86622 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 19:37:04 +0900 Subject: [PATCH 11/77] asio: Fix custom OpenSSL build --- src/Makefile.am | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index b1336ae8..8dc53570 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -207,10 +207,10 @@ libnghttp2_asio_la_LDFLAGS = -no-undefined -version-info 1:0:0 libnghttp2_asio_la_LIBADD = \ $(top_builddir)/lib/libnghttp2.la \ $(top_builddir)/third-party/libhttp-parser.la \ + @OPENSSL_LIBS@ \ ${BOOST_LDFLAGS} \ ${BOOST_ASIO_LIB} \ ${BOOST_THREAD_LIB} \ - ${BOOST_SYSTEM_LIB} \ - @OPENSSL_LIBS@ + ${BOOST_SYSTEM_LIB} endif # ENABLE_ASIO_LIB From 6307f96fb35335ecdf2281faa2465d2c596d5cdf Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 12 Jul 2015 22:16:20 +0900 Subject: [PATCH 12/77] nghttpx: Enable host-path backend routing in HTTP/2 backend To achieve host-path backend routing, we changed behaviour of --backend-http2-connections-per-worker. It now sets the number of HTTP/2 physical connections per pattern group if pattern is used in -b option. Fixes GH-292 --- src/shrpx.cc | 54 ++++++++++++------------ src/shrpx_client_handler.cc | 12 +++--- src/shrpx_client_handler.h | 2 - src/shrpx_http2_downstream_connection.cc | 13 +++++- src/shrpx_http2_downstream_connection.h | 3 +- src/shrpx_http2_session.cc | 17 +++++--- src/shrpx_http2_session.h | 5 ++- src/shrpx_http2_upstream.cc | 1 + src/shrpx_http_downstream_connection.cc | 4 +- src/shrpx_https_upstream.cc | 1 + src/shrpx_ssl.cc | 1 + src/shrpx_worker.cc | 39 +++++++++++------ src/shrpx_worker.h | 24 +++++++---- 13 files changed, 106 insertions(+), 70 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 389bf7e5..7ce6aa2b 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1005,22 +1005,22 @@ Connections: with "unix:" (e.g., unix:/var/run/backend.sock). Optionally, if s are given, the backend address - is only used if request matches the pattern. If -s, -p, - --client or --http2-bridge is used, s are - ignored. The pattern matching is closely designed to - ServeMux in net/http package of Go programming language. - consists of path, host + path or just host. - The path must starts with "/". If it ends with "/", it - matches to the request path whose prefix is the path. - To deal with the request to the directory without - trailing slash, pattern which ends with "/" also matches - the path if pattern == path + "/" (e.g., pattern "/foo/" - matches path "/foo"). If it does not end with "/", it - performs exact match against the request path. If host - is given, it performs exact match against the request - host. If host alone is given, "/" is appended to it, so - that it matches all paths under the host (e.g., - specifying "nghttp2.org" equals to "nghttp2.org/"). + is only used if request matches the pattern. If -s or + -p is used, s are ignored. The pattern + matching is closely designed to ServeMux in net/http + package of Go programming language. consists + of path, host + path or just host. The path must starts + with "/". If it ends with "/", it matches to the + request path whose prefix is the path. To deal with the + request to the directory without trailing slash, pattern + which ends with "/" also matches the path if pattern == + path + "/" (e.g., pattern "/foo/" matches path "/foo"). + If it does not end with "/", it performs exact match + against the request path. If host is given, it performs + exact match against the request host. If host alone is + given, "/" is appended to it, so that it matches all + paths under the host (e.g., specifying "nghttp2.org" + equals to "nghttp2.org/"). Patterns with host take precedence over path only patterns. Then, longer patterns take precedence over @@ -1130,9 +1130,14 @@ Performance: accepts. Setting 0 means unlimited. Default: )" << get_config()->worker_frontend_connections << R"( --backend-http2-connections-per-worker= - Set maximum number of HTTP/2 connections per worker. - The default value is 0, which means the number of - backend addresses specified by -b option. + Set maximum number of backend HTTP/2 physical + connections per worker. If pattern is used in -b + option, this limit is applied to each pattern group (in + other words, each pattern group can have maximum + HTTP/2 connections). The default value is 0, which + means that the value is adjusted to the number of + backend addresses. If pattern is used, this adjustment + is done for each pattern group. --backend-http1-connections-per-host= Set maximum number of backend concurrent HTTP/1 connections per origin host. This option is meaningful @@ -2177,10 +2182,9 @@ int main(int argc, char **argv) { DownstreamAddrGroup g("/"); g.addrs.push_back(std::move(addr)); mod_config()->downstream_addr_groups.push_back(std::move(g)); - } else if (get_config()->downstream_proto == PROTO_HTTP2 || - get_config()->http2_proxy || get_config()->client_proxy) { + } else if (get_config()->http2_proxy || get_config()->client_proxy) { // We don't support host mapping in these cases. Move all - // non-catch-all patterns to catch-all pattern for HTTP/2 backend + // non-catch-all patterns to catch-all pattern. DownstreamAddrGroup catch_all("/"); for (auto &g : mod_config()->downstream_addr_groups) { std::move(std::begin(g.addrs), std::end(g.addrs), @@ -2276,12 +2280,6 @@ int main(int argc, char **argv) { } } - if (get_config()->downstream_proto == PROTO_HTTP2 && - get_config()->http2_downstream_connections_per_worker == 0) { - mod_config()->http2_downstream_connections_per_worker = - get_config()->downstream_addr_groups[0].addrs.size(); - } - if (get_config()->rlimit_nofile) { struct rlimit lim = {static_cast(get_config()->rlimit_nofile), static_cast(get_config()->rlimit_nofile)}; diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 644a8cdc..fcf8a235 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -39,6 +39,7 @@ #include "shrpx_worker.h" #include "shrpx_downstream_connection_pool.h" #include "shrpx_downstream.h" +#include "shrpx_http2_session.h" #ifdef HAVE_SPDYLAY #include "shrpx_spdy_upstream.h" #endif // HAVE_SPDYLAY @@ -361,7 +362,6 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl, get_config()->write_burst, get_config()->read_rate, get_config()->read_burst, writecb, readcb, timeoutcb, this), ipaddr_(ipaddr), port_(port), worker_(worker), - http2session_(worker_->next_http2_session()), left_connhd_len_(NGHTTP2_CLIENT_MAGIC_LEN), should_close_after_write_(false) { @@ -612,9 +612,8 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { auto catch_all = get_config()->downstream_addr_group_catch_all; // Fast path. If we have one group, it must be catch-all group. - // HTTP/2 and client proxy modes fall in this case. Currently, - // HTTP/2 backend does not perform host-path mapping. - if (groups.size() == 1 || get_config()->downstream_proto == PROTO_HTTP2) { + // HTTP/2 and client proxy modes fall in this case. + if (groups.size() == 1) { group = 0; } else if (downstream->get_request_method() == HTTP_CONNECT) { // We don't know how to treat CONNECT request in host-path @@ -653,8 +652,9 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { auto dconn_pool = worker_->get_dconn_pool(); - if (http2session_) { - dconn = make_unique(dconn_pool, http2session_); + if (get_config()->downstream_proto == PROTO_HTTP2) { + auto http2session = worker_->next_http2_session(group); + dconn = make_unique(dconn_pool, http2session); } else { dconn = make_unique(dconn_pool, group, conn_.loop); diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index 4f21f513..67dc0974 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -44,7 +44,6 @@ namespace shrpx { class Upstream; class DownstreamConnection; -class Http2Session; class HttpsUpstream; class ConnectBlocker; class DownstreamConnectionPool; @@ -142,7 +141,6 @@ private: std::function read_, write_; std::function on_read_, on_write_; Worker *worker_; - Http2Session *http2session_; // The number of bytes of HTTP/2 client connection header to read size_t left_connhd_len_; bool should_close_after_write_; diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index f82b5230..d6bc433f 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -263,8 +263,11 @@ int Http2DownstreamConnection::push_request_headers() { // http2session_ has already in CONNECTED state, so we can get // addr_idx here. auto addr_idx = http2session_->get_addr_idx(); - auto downstream_hostport = - get_config()->downstream_addr_groups[0].addrs[addr_idx].hostport.get(); + auto group = http2session_->get_group(); + auto downstream_hostport = get_config() + ->downstream_addr_groups[group] + .addrs[addr_idx] + .hostport.get(); const char *authority = nullptr, *host = nullptr; if (!no_host_rewrite) { @@ -557,4 +560,10 @@ int Http2DownstreamConnection::on_timeout() { return submit_rst_stream(downstream_, NGHTTP2_NO_ERROR); } +size_t Http2DownstreamConnection::get_group() const { + // HTTP/2 backend connections are managed by Http2Session object, + // and it stores group index. + return http2session_->get_group(); +} + } // namespace shrpx diff --git a/src/shrpx_http2_downstream_connection.h b/src/shrpx_http2_downstream_connection.h index 736f546f..da9f7ef5 100644 --- a/src/shrpx_http2_downstream_connection.h +++ b/src/shrpx_http2_downstream_connection.h @@ -61,8 +61,7 @@ public: virtual void on_upstream_change(Upstream *upstream) {} virtual int on_priority_change(int32_t pri); - // Currently, HTTP/2 backend does not perform host-path mapping. - virtual size_t get_group() const { return 0; } + virtual size_t get_group() const; // This object is not poolable because we dont' have facility to // migrate to another Http2Session object. diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 5ed2f3c3..64c9feb9 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -142,13 +142,14 @@ void writecb(struct ev_loop *loop, ev_io *w, int revents) { } // namespace Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, - ConnectBlocker *connect_blocker, Worker *worker) + ConnectBlocker *connect_blocker, Worker *worker, + size_t group) : conn_(loop, -1, nullptr, get_config()->downstream_write_timeout, get_config()->downstream_read_timeout, 0, 0, 0, 0, writecb, readcb, timeoutcb, this), worker_(worker), connect_blocker_(connect_blocker), ssl_ctx_(ssl_ctx), session_(nullptr), data_pending_(nullptr), data_pendinglen_(0), - addr_idx_(0), state_(DISCONNECTED), + addr_idx_(0), group_(group), state_(DISCONNECTED), connection_check_state_(CONNECTION_CHECK_NONE), flow_control_(false) { read_ = write_ = &Http2Session::noop; @@ -235,13 +236,14 @@ int Http2Session::disconnect(bool hard) { int Http2Session::check_cert() { return ssl::check_cert( - conn_.tls.ssl, &get_config()->downstream_addr_groups[0].addrs[addr_idx_]); + conn_.tls.ssl, + &get_config()->downstream_addr_groups[group_].addrs[addr_idx_]); } int Http2Session::initiate_connection() { int rv = 0; - auto &addrs = get_config()->downstream_addr_groups[0].addrs; + auto &addrs = get_config()->downstream_addr_groups[group_].addrs; if (state_ == DISCONNECTED) { if (connect_blocker_->blocked()) { @@ -252,8 +254,7 @@ int Http2Session::initiate_connection() { return -1; } - auto worker_stat = worker_->get_worker_stat(); - auto &next_downstream = worker_stat->next_downstream[0]; + auto &next_downstream = worker_->get_dgrp(group_)->next; addr_idx_ = next_downstream; if (++next_downstream >= addrs.size()) { next_downstream = 0; @@ -511,7 +512,7 @@ int Http2Session::downstream_connect_proxy() { SSLOG(INFO, this) << "Connected to the proxy"; } auto &downstream_addr = - get_config()->downstream_addr_groups[0].addrs[addr_idx_]; + get_config()->downstream_addr_groups[group_].addrs[addr_idx_]; std::string req = "CONNECT "; req += downstream_addr.hostport.get(); @@ -1752,4 +1753,6 @@ bool Http2Session::should_hard_fail() const { size_t Http2Session::get_addr_idx() const { return addr_idx_; } +size_t Http2Session::get_group() const { return group_; } + } // namespace shrpx diff --git a/src/shrpx_http2_session.h b/src/shrpx_http2_session.h index 6f642063..184252de 100644 --- a/src/shrpx_http2_session.h +++ b/src/shrpx_http2_session.h @@ -58,7 +58,7 @@ struct StreamData { class Http2Session { public: Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, - ConnectBlocker *connect_blocker, Worker *worker); + ConnectBlocker *connect_blocker, Worker *worker, size_t group); ~Http2Session(); int check_cert(); @@ -151,6 +151,8 @@ public: size_t get_addr_idx() const; + size_t get_group() const; + enum { // Disconnected DISCONNECTED, @@ -203,6 +205,7 @@ private: size_t data_pendinglen_; // index of get_config()->downstream_addrs this object uses size_t addr_idx_; + size_t group_; int state_; int connection_check_state_; bool flow_control_; diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 07ac4e3d..97ff206c 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -36,6 +36,7 @@ #include "shrpx_config.h" #include "shrpx_http.h" #include "shrpx_worker.h" +#include "shrpx_http2_session.h" #include "http2.h" #include "util.h" #include "base64.h" diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index d0181b7e..a69357a5 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -34,6 +34,7 @@ #include "shrpx_connect_blocker.h" #include "shrpx_downstream_connection_pool.h" #include "shrpx_worker.h" +#include "shrpx_http2_session.h" #include "http2.h" #include "util.h" @@ -142,8 +143,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { } auto worker = client_handler_->get_worker(); - auto worker_stat = worker->get_worker_stat(); - auto &next_downstream = worker_stat->next_downstream[group_]; + auto &next_downstream = worker->get_dgrp(group_)->next; auto end = next_downstream; auto &addrs = get_config()->downstream_addr_groups[group_].addrs; for (;;) { diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 1a832e60..22c048a3 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -36,6 +36,7 @@ #include "shrpx_error.h" #include "shrpx_log_config.h" #include "shrpx_worker.h" +#include "shrpx_http2_session.h" #include "http2.h" #include "util.h" #include "template.h" diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 60526653..9f95fa2f 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -53,6 +53,7 @@ #include "shrpx_config.h" #include "shrpx_worker.h" #include "shrpx_downstream_connection_pool.h" +#include "shrpx_http2_session.h" #include "util.h" #include "ssl.h" #include "template.h" diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index a756e7c0..e5f0f8d4 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -61,9 +61,9 @@ void mcpool_clear_cb(struct ev_loop *loop, ev_timer *w, int revents) { Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, ssl::CertLookupTree *cert_tree, const std::shared_ptr &ticket_keys) - : next_http2session_(0), - dconn_pool_(get_config()->downstream_addr_groups.size()), - worker_stat_(get_config()->downstream_addr_groups.size()), loop_(loop), + : dconn_pool_(get_config()->downstream_addr_groups.size()), + worker_stat_(get_config()->downstream_addr_groups.size()), + dgrps_(get_config()->downstream_addr_groups.size()), loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), cert_tree_(cert_tree), ticket_keys_(ticket_keys), connect_blocker_(make_unique(loop_)), @@ -77,9 +77,17 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, if (get_config()->downstream_proto == PROTO_HTTP2) { auto n = get_config()->http2_downstream_connections_per_worker; - for (; n > 0; --n) { - http2sessions_.push_back(make_unique( - loop_, cl_ssl_ctx, connect_blocker_.get(), this)); + size_t group = 0; + for (auto &dgrp : dgrps_) { + auto m = n; + if (m == 0) { + m = get_config()->downstream_addr_groups[group].addrs.size(); + } + for (; m; --m) { + dgrp.http2sessions.push_back(make_unique( + loop_, cl_ssl_ctx, connect_blocker_.get(), this, group)); + } + ++group; } } } @@ -210,15 +218,17 @@ WorkerStat *Worker::get_worker_stat() { return &worker_stat_; } DownstreamConnectionPool *Worker::get_dconn_pool() { return &dconn_pool_; } -Http2Session *Worker::next_http2_session() { - if (http2sessions_.empty()) { +Http2Session *Worker::next_http2_session(size_t group) { + auto &dgrp = dgrps_[group]; + auto &http2sessions = dgrp.http2sessions; + if (http2sessions.empty()) { return nullptr; } - auto res = http2sessions_[next_http2session_].get(); - ++next_http2session_; - if (next_http2session_ >= http2sessions_.size()) { - next_http2session_ = 0; + auto res = http2sessions[dgrp.next_http2session].get(); + ++dgrp.next_http2session; + if (dgrp.next_http2session >= http2sessions.size()) { + dgrp.next_http2session = 0; } return res; @@ -242,4 +252,9 @@ bool Worker::get_graceful_shutdown() const { return graceful_shutdown_; } MemchunkPool *Worker::get_mcpool() { return &mcpool_; } +DownstreamGroup *Worker::get_dgrp(size_t group) { + assert(group < dgrps_.size()); + return &dgrps_[group]; +} + } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index f9924647..b5f820b4 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -54,14 +54,21 @@ namespace ssl { class CertLookupTree; } // namespace ssl +struct DownstreamGroup { + DownstreamGroup() : next_http2session(0), next(0) {} + + std::vector> http2sessions; + // Next index in http2sessions. + size_t next_http2session; + // Next downstream address index corresponding to + // Config::downstream_addr_groups[]. + size_t next; +}; + struct WorkerStat { - WorkerStat(size_t num_groups) - : num_connections(0), next_downstream(num_groups) {} + WorkerStat(size_t num_groups) : num_connections(0) {} size_t num_connections; - // Next downstream index in Config::downstream_addr_groups. This is - // used as load balancing. - std::vector next_downstream; }; enum WorkerEventType { @@ -97,7 +104,7 @@ public: void set_ticket_keys(std::shared_ptr ticket_keys); WorkerStat *get_worker_stat(); DownstreamConnectionPool *get_dconn_pool(); - Http2Session *next_http2_session(); + Http2Session *next_http2_session(size_t group); ConnectBlocker *get_connect_blocker() const; struct ev_loop *get_loop() const; SSL_CTX *get_sv_ssl_ctx() const; @@ -109,9 +116,9 @@ public: MemchunkPool *get_mcpool(); void schedule_clear_mcpool(); + DownstreamGroup *get_dgrp(size_t group); + private: - std::vector> http2sessions_; - size_t next_http2session_; #ifndef NOTHREADS std::future fut_; #endif // NOTHREADS @@ -122,6 +129,7 @@ private: MemchunkPool mcpool_; DownstreamConnectionPool dconn_pool_; WorkerStat worker_stat_; + std::vector dgrps_; struct ev_loop *loop_; // Following fields are shared across threads if From 309754749140d0ac59d4f504c6f86b4026cfa72f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 12 Jul 2015 23:18:36 +0900 Subject: [PATCH 13/77] nghttpx: Add --include option to read additional configuration from given file --- src/shrpx.cc | 10 ++++++++++ src/shrpx_config.cc | 7 ++++++- src/shrpx_config.h | 1 + 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 7ce6aa2b..26326e2f 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1502,6 +1502,11 @@ Misc: --conf= Load configuration from . Default: )" << get_config()->conf_path.get() << R"( + --include= + Load additional configurations from . File + is read when configuration parser encountered this + option. This option can be used multiple times, or even + recursively. -v, --version Print version and exit. -h, --help Print this help and exit. @@ -1648,6 +1653,7 @@ int main(int argc, char **argv) { {SHRPX_OPT_HEADER_FIELD_BUFFER, required_argument, &flag, 80}, {SHRPX_OPT_MAX_HEADER_FIELDS, required_argument, &flag, 81}, {SHRPX_OPT_ADD_REQUEST_HEADER, required_argument, &flag, 82}, + {SHRPX_OPT_INCLUDE, required_argument, &flag, 83}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2010,6 +2016,10 @@ int main(int argc, char **argv) { // --add-request-header cmdcfgs.emplace_back(SHRPX_OPT_ADD_REQUEST_HEADER, optarg); break; + case 83: + // --include + cmdcfgs.emplace_back(SHRPX_OPT_INCLUDE, optarg); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index ebaf2b23..49ad49a8 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1219,6 +1219,10 @@ int parse_config(const char *opt, const char *optarg) { return parse_uint(&mod_config()->max_header_fields, opt, optarg); } + if (util::strieq(opt, SHRPX_OPT_INCLUDE)) { + return load_config(optarg); + } + if (util::strieq(opt, "conf")) { LOG(WARN) << "conf: ignored"; @@ -1248,7 +1252,8 @@ int load_config(const char *filename) { for (i = 0; i < size && line[i] != '='; ++i) ; if (i == size) { - LOG(ERROR) << "Bad configuration format at line " << linenum; + LOG(ERROR) << "Bad configuration format in " << filename << " at line " + << linenum; return -1; } line[i] = '\0'; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 3e5c869e..5f23c5b3 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -165,6 +165,7 @@ constexpr char SHRPX_OPT_OCSP_UPDATE_INTERVAL[] = "ocsp-update-interval"; constexpr char SHRPX_OPT_NO_OCSP[] = "no-ocsp"; constexpr char SHRPX_OPT_HEADER_FIELD_BUFFER[] = "header-field-buffer"; constexpr char SHRPX_OPT_MAX_HEADER_FIELDS[] = "max-header-fields"; +constexpr char SHRPX_OPT_INCLUDE[] = "include"; union sockaddr_union { sockaddr_storage storage; From 3db0badc354532509c11408b5f7b9bfa6bc1dc3d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 12 Jul 2015 23:26:45 +0900 Subject: [PATCH 14/77] Update man pages --- doc/h2load.1 | 2 +- doc/nghttp.1 | 2 +- doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 60 +++++++++++++++++++++++++++++------------------ doc/nghttpx.1.rst | 57 +++++++++++++++++++++++++++----------------- 5 files changed, 75 insertions(+), 48 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index 0b2924f3..b5b60925 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "July 11, 2015" "1.0.6-DEV" "nghttp2" +.TH "H2LOAD" "1" "July 12, 2015" "1.0.6-DEV" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 7fb39c33..9695b45a 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "July 11, 2015" "1.0.6-DEV" "nghttp2" +.TH "NGHTTP" "1" "July 12, 2015" "1.0.6-DEV" "nghttp2" .SH NAME nghttp \- HTTP/2 experimental client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index 5a477567..70a816bd 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "July 11, 2015" "1.0.6-DEV" "nghttp2" +.TH "NGHTTPD" "1" "July 12, 2015" "1.0.6-DEV" "nghttp2" .SH NAME nghttpd \- HTTP/2 experimental server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index bf67f98f..fa4e2cde 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "July 11, 2015" "1.0.6-DEV" "nghttp2" +.TH "NGHTTPX" "1" "July 12, 2015" "1.0.6-DEV" "nghttp2" .SH NAME nghttpx \- HTTP/2 experimental proxy . @@ -62,26 +62,27 @@ domain socket can be specified by prefixing path name with "unix:" (e.g., unix:/var/run/backend.sock). .sp Optionally, if s are given, the backend address -is only used if request matches the pattern. If \fI\%\-s\fP, \fI\%\-p\fP, -\fI\%\-\-client\fP or \fI\%\-\-http2\-bridge\fP is used, s are -ignored. The pattern matching is closely designed to -ServeMux in net/http package of Go programming language. - consists of path, host + path or just host. -The path must starts with "\fI/\fP". If it ends with "\fI/\fP", it -matches to the request path whose prefix is the path. -To deal with the request to the directory without -trailing slash, pattern which ends with "\fI/\fP" also matches -the path if pattern == path + "\fI/\fP" (e.g., pattern "\fI/foo/\fP" -matches path "\fI/foo\fP"). If it does not end with "\fI/\fP", it -performs exact match against the request path. If host -is given, it performs exact match against the request -host. If host alone is given, "\fI/\fP" is appended to it, so -that it matches all paths under the host (e.g., -specifying "nghttp2.org" equals to "nghttp2.org/"). +is only used if request matches the pattern. If \fI\%\-s\fP or +\fI\%\-p\fP is used, s are ignored. The pattern +matching is closely designed to ServeMux in net/http +package of Go programming language. consists +of path, host + path or just host. The path must starts +with "\fI/\fP". If it ends with "\fI/\fP", it matches to the +request path whose prefix is the path. To deal with the +request to the directory without trailing slash, pattern +which ends with "\fI/\fP" also matches the path if pattern == +path + "\fI/\fP" (e.g., pattern "\fI/foo/\fP" matches path "\fI/foo\fP"). +If it does not end with "\fI/\fP", it performs exact match +against the request path. If host is given, it performs +exact match against the request host. If host alone is +given, "\fI/\fP" is appended to it, so that it matches all +paths under the host (e.g., specifying "nghttp2.org" +equals to "nghttp2.org/"). .sp -Longer patterns take precedence over shorter ones, -breaking a tie by the order of the appearance in the -configuration. +Patterns with host take precedence over path only +patterns. Then, longer patterns take precedence over +shorter ones, breaking a tie by the order of the +appearance in the configuration. .sp If is omitted, "\fI/\fP" is used as pattern, which matches all paths (catch\-all pattern). The catch\-all @@ -244,9 +245,14 @@ Default: \fB0\fP .INDENT 0.0 .TP .B \-\-backend\-http2\-connections\-per\-worker= -Set maximum number of HTTP/2 connections per worker. -The default value is 0, which means the number of -backend addresses specified by \fI\%\-b\fP option. +Set maximum number of backend HTTP/2 physical +connections per worker. If pattern is used in \fI\%\-b\fP +option, this limit is applied to each pattern group (in +other words, each pattern group can have maximum +HTTP/2 connections). The default value is 0, which +means that the value is adjusted to the number of +backend addresses. If pattern is used, this adjustment +is done for each pattern group. .UNINDENT .INDENT 0.0 .TP @@ -844,6 +850,14 @@ Default: \fB/etc/nghttpx/nghttpx.conf\fP .UNINDENT .INDENT 0.0 .TP +.B \-\-include= +Load additional configurations from . File +is read when configuration parser encountered this +option. This option can be used multiple times, or even +recursively. +.UNINDENT +.INDENT 0.0 +.TP .B \-v, \-\-version Print version and exit. .UNINDENT diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index 3132dca0..7ff041c6 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -45,26 +45,27 @@ Connections with "unix:" (e.g., unix:/var/run/backend.sock). Optionally, if s are given, the backend address - is only used if request matches the pattern. If :option:`-s`\, :option:`-p`\, - :option:`--client` or :option:`\--http2-bridge` is used, s are - ignored. The pattern matching is closely designed to - ServeMux in net/http package of Go programming language. - consists of path, host + path or just host. - The path must starts with "*/*". If it ends with "*/*", it - matches to the request path whose prefix is the path. - To deal with the request to the directory without - trailing slash, pattern which ends with "*/*" also matches - the path if pattern == path + "*/*" (e.g., pattern "*/foo/*" - matches path "*/foo*"). If it does not end with "*/*", it - performs exact match against the request path. If host - is given, it performs exact match against the request - host. If host alone is given, "*/*" is appended to it, so - that it matches all paths under the host (e.g., - specifying "nghttp2.org" equals to "nghttp2.org/"). + is only used if request matches the pattern. If :option:`-s` or + :option:`-p` is used, s are ignored. The pattern + matching is closely designed to ServeMux in net/http + package of Go programming language. consists + of path, host + path or just host. The path must starts + with "*/*". If it ends with "*/*", it matches to the + request path whose prefix is the path. To deal with the + request to the directory without trailing slash, pattern + which ends with "*/*" also matches the path if pattern == + path + "*/*" (e.g., pattern "*/foo/*" matches path "*/foo*"). + If it does not end with "*/*", it performs exact match + against the request path. If host is given, it performs + exact match against the request host. If host alone is + given, "*/*" is appended to it, so that it matches all + paths under the host (e.g., specifying "nghttp2.org" + equals to "nghttp2.org/"). - Longer patterns take precedence over shorter ones, - breaking a tie by the order of the appearance in the - configuration. + Patterns with host take precedence over path only + patterns. Then, longer patterns take precedence over + shorter ones, breaking a tie by the order of the + appearance in the configuration. If is omitted, "*/*" is used as pattern, which matches all paths (catch-all pattern). The catch-all @@ -214,9 +215,14 @@ Performance .. option:: --backend-http2-connections-per-worker= - Set maximum number of HTTP/2 connections per worker. - The default value is 0, which means the number of - backend addresses specified by :option:`-b` option. + Set maximum number of backend HTTP/2 physical + connections per worker. If pattern is used in :option:`-b` + option, this limit is applied to each pattern group (in + other words, each pattern group can have maximum + HTTP/2 connections). The default value is 0, which + means that the value is adjusted to the number of + backend addresses. If pattern is used, this adjustment + is done for each pattern group. .. option:: --backend-http1-connections-per-host= @@ -754,6 +760,13 @@ Misc Default: ``/etc/nghttpx/nghttpx.conf`` +.. option:: --include= + + Load additional configurations from . File + is read when configuration parser encountered this + option. This option can be used multiple times, or even + recursively. + .. option:: -v, --version Print version and exit. From f96edbf987109d33b15573766be76e4c4c3c63c5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 13 Jul 2015 21:31:37 +0900 Subject: [PATCH 15/77] nghttpx: Pin frontend to specific HTTP/2 session object per group --- src/shrpx_client_handler.cc | 15 ++++++++++++++- src/shrpx_client_handler.h | 1 + src/shrpx_http2_session.cc | 6 ++++-- src/shrpx_http2_session.h | 8 +++++++- src/shrpx_worker.cc | 4 ++-- 5 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index fcf8a235..0e2c8ff9 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -361,6 +361,11 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl, get_config()->upstream_read_timeout, get_config()->write_rate, get_config()->write_burst, get_config()->read_rate, get_config()->read_burst, writecb, readcb, timeoutcb, this), + pinned_http2sessions_( + get_config()->downstream_proto == PROTO_HTTP2 + ? make_unique>( + get_config()->downstream_addr_groups.size(), -1) + : nullptr), ipaddr_(ipaddr), port_(port), worker_(worker), left_connhd_len_(NGHTTP2_CLIENT_MAGIC_LEN), should_close_after_write_(false) { @@ -653,7 +658,15 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { auto dconn_pool = worker_->get_dconn_pool(); if (get_config()->downstream_proto == PROTO_HTTP2) { - auto http2session = worker_->next_http2_session(group); + Http2Session *http2session; + auto &pinned = (*pinned_http2sessions_)[group]; + if (pinned == -1) { + http2session = worker_->next_http2_session(group); + pinned = http2session->get_index(); + } else { + auto dgrp = worker_->get_dgrp(group); + http2session = dgrp->http2sessions[pinned].get(); + } dconn = make_unique(dconn_pool, http2session); } else { dconn = diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index 67dc0974..4d4ccd9c 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -134,6 +134,7 @@ private: Connection conn_; ev_timer reneg_shutdown_timer_; std::unique_ptr upstream_; + std::unique_ptr> pinned_http2sessions_; std::string ipaddr_; std::string port_; // The ALPN identifier negotiated for this connection. diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 64c9feb9..e2b4e2ee 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -143,13 +143,13 @@ void writecb(struct ev_loop *loop, ev_io *w, int revents) { Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, ConnectBlocker *connect_blocker, Worker *worker, - size_t group) + size_t group, size_t idx) : conn_(loop, -1, nullptr, get_config()->downstream_write_timeout, get_config()->downstream_read_timeout, 0, 0, 0, 0, writecb, readcb, timeoutcb, this), worker_(worker), connect_blocker_(connect_blocker), ssl_ctx_(ssl_ctx), session_(nullptr), data_pending_(nullptr), data_pendinglen_(0), - addr_idx_(0), group_(group), state_(DISCONNECTED), + addr_idx_(0), group_(group), index_(idx), state_(DISCONNECTED), connection_check_state_(CONNECTION_CHECK_NONE), flow_control_(false) { read_ = write_ = &Http2Session::noop; @@ -1755,4 +1755,6 @@ size_t Http2Session::get_addr_idx() const { return addr_idx_; } size_t Http2Session::get_group() const { return group_; } +size_t Http2Session::get_index() const { return index_; } + } // namespace shrpx diff --git a/src/shrpx_http2_session.h b/src/shrpx_http2_session.h index 184252de..3ae3e784 100644 --- a/src/shrpx_http2_session.h +++ b/src/shrpx_http2_session.h @@ -58,7 +58,8 @@ struct StreamData { class Http2Session { public: Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, - ConnectBlocker *connect_blocker, Worker *worker, size_t group); + ConnectBlocker *connect_blocker, Worker *worker, size_t group, + size_t idx); ~Http2Session(); int check_cert(); @@ -153,6 +154,8 @@ public: size_t get_group() const; + size_t get_index() const; + enum { // Disconnected DISCONNECTED, @@ -206,6 +209,9 @@ private: // index of get_config()->downstream_addrs this object uses size_t addr_idx_; size_t group_; + // index inside group, this is used to pin frontend to certain + // HTTP/2 backend for better throughput. + size_t index_; int state_; int connection_check_state_; bool flow_control_; diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index e5f0f8d4..ecf55b51 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -83,9 +83,9 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, if (m == 0) { m = get_config()->downstream_addr_groups[group].addrs.size(); } - for (; m; --m) { + for (size_t idx = 0; idx < m; ++idx) { dgrp.http2sessions.push_back(make_unique( - loop_, cl_ssl_ctx, connect_blocker_.get(), this, group)); + loop_, cl_ssl_ctx, connect_blocker_.get(), this, group, idx)); } ++group; } From fb7775e382ef62189dfe98e93474fcefa116e0b4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 13 Jul 2015 21:44:06 +0900 Subject: [PATCH 16/77] nghttpx: Detect loop in --include paths --- src/shrpx.cc | 19 ++++++++++++++----- src/shrpx_config.cc | 22 ++++++++++++++++++---- src/shrpx_config.h | 12 ++++++++---- 3 files changed, 40 insertions(+), 13 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 26326e2f..7160bf6c 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2030,11 +2030,13 @@ int main(int argc, char **argv) { } if (conf_exists(get_config()->conf_path.get())) { - if (load_config(get_config()->conf_path.get()) == -1) { + std::set include_set; + if (load_config(get_config()->conf_path.get(), include_set) == -1) { LOG(FATAL) << "Failed to load configuration from " << get_config()->conf_path.get(); exit(EXIT_FAILURE); } + assert(include_set.empty()); } if (argc - optind >= 2) { @@ -2046,11 +2048,18 @@ int main(int argc, char **argv) { // parsing option values. reopen_log_files(); - for (size_t i = 0, len = cmdcfgs.size(); i < len; ++i) { - if (parse_config(cmdcfgs[i].first, cmdcfgs[i].second) == -1) { - LOG(FATAL) << "Failed to parse command-line argument."; - exit(EXIT_FAILURE); + { + std::set include_set; + + for (size_t i = 0, len = cmdcfgs.size(); i < len; ++i) { + if (parse_config(cmdcfgs[i].first, cmdcfgs[i].second, include_set) == + -1) { + LOG(FATAL) << "Failed to parse command-line argument."; + exit(EXIT_FAILURE); + } } + + assert(include_set.empty()); } if (get_config()->accesslog_syslog || get_config()->errorlog_syslog) { diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 49ad49a8..6a978b1c 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -505,7 +505,8 @@ void parse_mapping(const DownstreamAddr &addr, const char *src) { } } // namespace -int parse_config(const char *opt, const char *optarg) { +int parse_config(const char *opt, const char *optarg, + std::set &included_set) { char host[NI_MAXHOST]; uint16_t port; @@ -1220,7 +1221,20 @@ int parse_config(const char *opt, const char *optarg) { } if (util::strieq(opt, SHRPX_OPT_INCLUDE)) { - return load_config(optarg); + if (included_set.count(optarg)) { + LOG(ERROR) << opt << ": " << optarg << " has already been included"; + return -1; + } + + included_set.emplace(optarg); + auto rv = load_config(optarg, included_set); + included_set.erase(optarg); + + if (rv != 0) { + return -1; + } + + return 0; } if (util::strieq(opt, "conf")) { @@ -1234,7 +1248,7 @@ int parse_config(const char *opt, const char *optarg) { return -1; } -int load_config(const char *filename) { +int load_config(const char *filename, std::set &include_set) { std::ifstream in(filename, std::ios::binary); if (!in) { LOG(ERROR) << "Could not open config file " << filename; @@ -1258,7 +1272,7 @@ int load_config(const char *filename) { } line[i] = '\0'; auto s = line.c_str(); - if (parse_config(s, s + i + 1) == -1) { + if (parse_config(s, s + i + 1, include_set) == -1) { return -1; } } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 5f23c5b3..ea455f02 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -42,6 +42,7 @@ #include #include #include +#include #include @@ -379,13 +380,16 @@ void create_config(); // Parses option name |opt| and value |optarg|. The results are // stored into statically allocated Config object. This function -// returns 0 if it succeeds, or -1. -int parse_config(const char *opt, const char *optarg); +// returns 0 if it succeeds, or -1. The |included_set| contains the +// all paths already included while processing this configuration, to +// avoid loop in --include option. +int parse_config(const char *opt, const char *optarg, + std::set &included_set); // Loads configurations from |filename| and stores them in statically // allocated Config object. This function returns 0 if it succeeds, or -// -1. -int load_config(const char *filename); +// -1. See parse_config() for |include_set|. +int load_config(const char *filename, std::set &include_set); // Read passwd from |filename| std::string read_passwd_from_file(const char *filename); From 7c301defbd6f7f993bfc0448b697f2aee4d739a6 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Jul 2015 21:37:34 +0900 Subject: [PATCH 17/77] clang-format src/nghttp.cc --- src/nghttp.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/nghttp.cc b/src/nghttp.cc index 78cd5652..eb423c6d 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -1376,10 +1376,11 @@ void HttpClient::output_har(FILE *outfile) { entry, "startedDateTime", json_string(util::format_iso8601(request_time).c_str())); json_object_set_new(entry, "time", json_real(time_sum)); - + auto pushed = req->stream_id % 2 == 0; - json_object_set_new(entry, "comment", json_string(pushed ? "Pushed Object" : "")); + json_object_set_new(entry, "comment", + json_string(pushed ? "Pushed Object" : "")); auto request = json_object(); json_object_set_new(entry, "request", request); @@ -1463,7 +1464,8 @@ void HttpClient::output_har(FILE *outfile) { json_object_set_new(timings, "receive", json_real(receive_delta)); json_object_set_new(entry, "pageref", json_string(PAGE_ID)); - json_object_set_new(entry, "connection", json_string(util::utos(req->stream_id).c_str())); + json_object_set_new(entry, "connection", + json_string(util::utos(req->stream_id).c_str())); } json_dumpf(root, outfile, JSON_PRESERVE_ORDER | JSON_INDENT(2)); From 7f7b6d641d35083f4ed66c11d5c031dc7060a107 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Jul 2015 22:25:52 +0900 Subject: [PATCH 18/77] nghttpx: Allow log variable to be enclosed by curly braces --- src/shrpx.cc | 3 ++ src/shrpx_config.cc | 80 ++++++++++++++++++++++++---------------- src/shrpx_config_test.cc | 44 ++++++++++++++++++++-- 3 files changed, 93 insertions(+), 34 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 7160bf6c..d458ed68 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1407,6 +1407,9 @@ Logging: * $ssl_session_reused: "r" if SSL/TLS session was reused. Otherwise, "." + The variable can be enclosed by "{" and "}" for + disambiguation (e.g., ${remote_addr}). + Default: )" << DEFAULT_ACCESSLOG_FORMAT << R"( --errorlog-file= Set path to write error log. To reopen file, send USR1 diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 6a978b1c..c1501dff 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -370,52 +370,69 @@ std::vector parse_log_format(const char *optarg) { ++p; - for (; p != eop && var_token(*p); ++p) - ; + const char *var_name; + size_t var_namelen; + if (p != eop && *p == '{') { + var_name = ++p; + for (; p != eop && var_token(*p); ++p) + ; - auto varlen = p - var_start; + if (p == eop || *p != '}') { + LOG(WARN) << "Missing '}' after " << std::string(var_start, p); + continue; + } + + var_namelen = p - var_name; + ++p; + } else { + var_name = p; + for (; p != eop && var_token(*p); ++p) + ; + + var_namelen = p - var_name; + } auto type = SHRPX_LOGF_NONE; const char *value = nullptr; size_t valuelen = 0; - if (util::strieq_l("$remote_addr", var_start, varlen)) { + if (util::strieq_l("remote_addr", var_name, var_namelen)) { type = SHRPX_LOGF_REMOTE_ADDR; - } else if (util::strieq_l("$time_local", var_start, varlen)) { + } else if (util::strieq_l("time_local", var_name, var_namelen)) { type = SHRPX_LOGF_TIME_LOCAL; - } else if (util::strieq_l("$time_iso8601", var_start, varlen)) { + } else if (util::strieq_l("time_iso8601", var_name, var_namelen)) { type = SHRPX_LOGF_TIME_ISO8601; - } else if (util::strieq_l("$request", var_start, varlen)) { + } else if (util::strieq_l("request", var_name, var_namelen)) { type = SHRPX_LOGF_REQUEST; - } else if (util::strieq_l("$status", var_start, varlen)) { + } else if (util::strieq_l("status", var_name, var_namelen)) { type = SHRPX_LOGF_STATUS; - } else if (util::strieq_l("$body_bytes_sent", var_start, varlen)) { + } else if (util::strieq_l("body_bytes_sent", var_name, var_namelen)) { type = SHRPX_LOGF_BODY_BYTES_SENT; - } else if (util::istartsWith(var_start, varlen, "$http_")) { + } else if (util::istartsWith(var_name, var_namelen, "http_")) { type = SHRPX_LOGF_HTTP; - value = var_start + sizeof("$http_") - 1; - valuelen = varlen - (sizeof("$http_") - 1); - } else if (util::strieq_l("$remote_port", var_start, varlen)) { + value = var_name + sizeof("http_") - 1; + valuelen = var_namelen - (sizeof("http_") - 1); + } else if (util::strieq_l("remote_port", var_name, var_namelen)) { type = SHRPX_LOGF_REMOTE_PORT; - } else if (util::strieq_l("$server_port", var_start, varlen)) { + } else if (util::strieq_l("server_port", var_name, var_namelen)) { type = SHRPX_LOGF_SERVER_PORT; - } else if (util::strieq_l("$request_time", var_start, varlen)) { + } else if (util::strieq_l("request_time", var_name, var_namelen)) { type = SHRPX_LOGF_REQUEST_TIME; - } else if (util::strieq_l("$pid", var_start, varlen)) { + } else if (util::strieq_l("pid", var_name, var_namelen)) { type = SHRPX_LOGF_PID; - } else if (util::strieq_l("$alpn", var_start, varlen)) { + } else if (util::strieq_l("alpn", var_name, var_namelen)) { type = SHRPX_LOGF_ALPN; - } else if (util::strieq_l("$ssl_cipher", var_start, varlen)) { + } else if (util::strieq_l("ssl_cipher", var_name, var_namelen)) { type = SHRPX_LOGF_SSL_CIPHER; - } else if (util::strieq_l("$ssl_protocol", var_start, varlen)) { + } else if (util::strieq_l("ssl_protocol", var_name, var_namelen)) { type = SHRPX_LOGF_SSL_PROTOCOL; - } else if (util::strieq_l("$ssl_session_id", var_start, varlen)) { + } else if (util::strieq_l("ssl_session_id", var_name, var_namelen)) { type = SHRPX_LOGF_SSL_SESSION_ID; - } else if (util::strieq_l("$ssl_session_reused", var_start, varlen)) { + } else if (util::strieq_l("ssl_session_reused", var_name, var_namelen)) { type = SHRPX_LOGF_SSL_SESSION_REUSED; } else { LOG(WARN) << "Unrecognized log format variable: " - << std::string(var_start, varlen); + << std::string(var_name, var_namelen); continue; } @@ -425,19 +442,20 @@ std::vector parse_log_format(const char *optarg) { strcopy(literal_start, var_start - literal_start))); } + literal_start = p; + if (value == nullptr) { res.push_back(make_log_fragment(type)); - } else { - res.push_back(make_log_fragment(type, strcopy(value, valuelen))); - auto &v = res.back().value; - for (size_t i = 0; v[i]; ++i) { - if (v[i] == '_') { - v[i] = '-'; - } - } + continue; } - literal_start = var_start + varlen; + res.push_back(make_log_fragment(type, strcopy(value, valuelen))); + auto &v = res.back().value; + for (size_t i = 0; v[i]; ++i) { + if (v[i] == '_') { + v[i] = '-'; + } + } } if (literal_start != eop) { diff --git a/src/shrpx_config_test.cc b/src/shrpx_config_test.cc index ab6a0f27..c5116294 100644 --- a/src/shrpx_config_test.cc +++ b/src/shrpx_config_test.cc @@ -95,9 +95,9 @@ void test_shrpx_config_parse_header(void) { } void test_shrpx_config_parse_log_format(void) { - auto res = parse_log_format("$remote_addr - $remote_user [$time_local] " - "\"$request\" $status $body_bytes_sent " - "\"$http_referer\" \"$http_user_agent\""); + auto res = parse_log_format(R"($remote_addr - $remote_user [$time_local] )" + R"("$request" $status $body_bytes_sent )" + R"("${http_referer}" "$http_user_agent")"); CU_ASSERT(14 == res.size()); CU_ASSERT(SHRPX_LOGF_REMOTE_ADDR == res[0].type); @@ -136,6 +136,44 @@ void test_shrpx_config_parse_log_format(void) { CU_ASSERT(SHRPX_LOGF_LITERAL == res[13].type); CU_ASSERT(0 == strcmp("\"", res[13].value.get())); + + res = parse_log_format("$"); + + CU_ASSERT(1 == res.size()); + + CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type); + CU_ASSERT(0 == strcmp("$", res[0].value.get())); + + res = parse_log_format("${"); + + CU_ASSERT(1 == res.size()); + + CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type); + CU_ASSERT(0 == strcmp("${", res[0].value.get())); + + res = parse_log_format("${a"); + + CU_ASSERT(1 == res.size()); + + CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type); + CU_ASSERT(0 == strcmp("${a", res[0].value.get())); + + res = parse_log_format("${a "); + + CU_ASSERT(1 == res.size()); + + CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type); + CU_ASSERT(0 == strcmp("${a ", res[0].value.get())); + + res = parse_log_format("$$remote_addr"); + + CU_ASSERT(2 == res.size()); + + CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type); + CU_ASSERT(0 == strcmp("$", res[0].value.get())); + + CU_ASSERT(SHRPX_LOGF_REMOTE_ADDR == res[1].type); + CU_ASSERT(nullptr == res[1].value.get()); } void test_shrpx_config_read_tls_ticket_key_file(void) { From 0a6877d091e51ba18e971818f365643a2ba59be5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Jul 2015 22:40:33 +0900 Subject: [PATCH 19/77] nghttpx: Supply template version strcopy --- src/shrpx_config.cc | 33 +++++++-------------------------- src/shrpx_config.h | 24 ++++++++++++++++++------ 2 files changed, 25 insertions(+), 32 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index c1501dff..cb255cf3 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -59,8 +59,6 @@ #include "template.h" #include "base64.h" -using namespace nghttp2; - namespace shrpx { namespace { @@ -227,21 +225,6 @@ std::string read_passwd_from_file(const char *filename) { return line; } -std::unique_ptr strcopy(const char *val) { - return strcopy(val, strlen(val)); -} - -std::unique_ptr strcopy(const char *val, size_t len) { - auto res = make_unique(len + 1); - memcpy(res.get(), val, len); - res[len] = '\0'; - return res; -} - -std::unique_ptr strcopy(const std::string &val) { - return strcopy(val.c_str(), val.size()); -} - std::vector parse_config_str_list(const char *s, char delim) { size_t len = 1; for (const char *first = s, *p = nullptr; (p = strchr(first, delim)); @@ -394,7 +377,6 @@ std::vector parse_log_format(const char *optarg) { auto type = SHRPX_LOGF_NONE; const char *value = nullptr; - size_t valuelen = 0; if (util::strieq_l("remote_addr", var_name, var_namelen)) { type = SHRPX_LOGF_REMOTE_ADDR; @@ -411,7 +393,6 @@ std::vector parse_log_format(const char *optarg) { } else if (util::istartsWith(var_name, var_namelen, "http_")) { type = SHRPX_LOGF_HTTP; value = var_name + sizeof("http_") - 1; - valuelen = var_namelen - (sizeof("http_") - 1); } else if (util::strieq_l("remote_port", var_name, var_namelen)) { type = SHRPX_LOGF_REMOTE_PORT; } else if (util::strieq_l("server_port", var_name, var_namelen)) { @@ -437,9 +418,8 @@ std::vector parse_log_format(const char *optarg) { } if (literal_start < var_start) { - res.push_back( - make_log_fragment(SHRPX_LOGF_LITERAL, - strcopy(literal_start, var_start - literal_start))); + res.push_back(make_log_fragment(SHRPX_LOGF_LITERAL, + strcopy(literal_start, var_start))); } literal_start = p; @@ -449,7 +429,8 @@ std::vector parse_log_format(const char *optarg) { continue; } - res.push_back(make_log_fragment(type, strcopy(value, valuelen))); + res.push_back( + make_log_fragment(type, strcopy(value, var_name + var_namelen))); auto &v = res.back().value; for (size_t i = 0; v[i]; ++i) { if (v[i] == '_') { @@ -459,8 +440,8 @@ std::vector parse_log_format(const char *optarg) { } if (literal_start != eop) { - res.push_back(make_log_fragment( - SHRPX_LOGF_LITERAL, strcopy(literal_start, eop - literal_start))); + res.push_back( + make_log_fragment(SHRPX_LOGF_LITERAL, strcopy(literal_start, eop))); } return res; @@ -537,7 +518,7 @@ int parse_config(const char *opt, const char *optarg, DownstreamAddr addr; if (util::istartsWith(optarg, SHRPX_UNIX_PATH_PREFIX)) { auto path = optarg + str_size(SHRPX_UNIX_PATH_PREFIX); - addr.host = strcopy(path, pat_delim - path); + addr.host = strcopy(path, pat_delim); addr.host_unix = true; } else { if (split_host_port(host, sizeof(host), &port, optarg, diff --git a/src/shrpx_config.h b/src/shrpx_config.h index ea455f02..87af15ba 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -50,6 +50,10 @@ #include +#include "template.h" + +using namespace nghttp2; + namespace shrpx { struct LogFragment; @@ -416,15 +420,23 @@ std::pair parse_header(const char *optarg); std::vector parse_log_format(const char *optarg); -// Returns a copy of NULL-terminated string |val|. -std::unique_ptr strcopy(const char *val); +// Returns a copy of NULL-terminated string [first, last). +template +std::unique_ptr strcopy(InputIt first, InputIt last) { + auto res = make_unique(last - first + 1); + *std::copy(first, last, res.get()) = '\0'; + return res; +} -// Returns a copy of string |val| of length |n|. The returned string -// will be NULL-terminated. -std::unique_ptr strcopy(const char *val, size_t n); +// Returns a copy of NULL-terminated string |val|. +inline std::unique_ptr strcopy(const char *val) { + return strcopy(val, val + strlen(val)); +} // Returns a copy of val.c_str(). -std::unique_ptr strcopy(const std::string &val); +inline std::unique_ptr strcopy(const std::string &val) { + return strcopy(std::begin(val), std::end(val)); +} // Returns string for syslog |facility|. const char *str_syslog_facility(int facility); From 27da08ee68168eb9b548947881f6eace969c3331 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Jul 2015 22:43:02 +0900 Subject: [PATCH 20/77] nghttpx: Add inline LogFragment ctor --- src/shrpx_config.cc | 18 ++++-------------- src/shrpx_log.h | 2 ++ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index cb255cf3..37b9693b 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -323,13 +323,6 @@ int parse_int(T *dest, const char *opt, const char *optarg) { return 0; } -namespace { -LogFragment make_log_fragment(LogFragmentType type, - std::unique_ptr value = nullptr) { - return LogFragment{type, std::move(value)}; -} -} // namespace - namespace { bool var_token(char c) { return util::isAlpha(c) || util::isDigit(c) || c == '_'; @@ -418,19 +411,17 @@ std::vector parse_log_format(const char *optarg) { } if (literal_start < var_start) { - res.push_back(make_log_fragment(SHRPX_LOGF_LITERAL, - strcopy(literal_start, var_start))); + res.emplace_back(SHRPX_LOGF_LITERAL, strcopy(literal_start, var_start)); } literal_start = p; if (value == nullptr) { - res.push_back(make_log_fragment(type)); + res.emplace_back(type); continue; } - res.push_back( - make_log_fragment(type, strcopy(value, var_name + var_namelen))); + res.emplace_back(type, strcopy(value, var_name + var_namelen)); auto &v = res.back().value; for (size_t i = 0; v[i]; ++i) { if (v[i] == '_') { @@ -440,8 +431,7 @@ std::vector parse_log_format(const char *optarg) { } if (literal_start != eop) { - res.push_back( - make_log_fragment(SHRPX_LOGF_LITERAL, strcopy(literal_start, eop))); + res.emplace_back(SHRPX_LOGF_LITERAL, strcopy(literal_start, eop)); } return res; diff --git a/src/shrpx_log.h b/src/shrpx_log.h index 1d63e641..e13be41e 100644 --- a/src/shrpx_log.h +++ b/src/shrpx_log.h @@ -123,6 +123,8 @@ enum LogFragmentType { }; struct LogFragment { + LogFragment(LogFragmentType type, std::unique_ptr value = nullptr) + : type(type), value(std::move(value)) {} LogFragmentType type; std::unique_ptr value; }; From 8c1e8635230350fda69f5421b1e5b481cc2df21f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Jul 2015 23:21:38 +0900 Subject: [PATCH 21/77] nghttpx: Refactor option name lookup --- gennghttpxfun.py | 117 ++++++ gentokenlookup.py | 14 +- src/shrpx_config.cc | 978 +++++++++++++++++++++++++++++++++----------- 3 files changed, 854 insertions(+), 255 deletions(-) create mode 100755 gennghttpxfun.py diff --git a/gennghttpxfun.py b/gennghttpxfun.py new file mode 100755 index 00000000..81836fe9 --- /dev/null +++ b/gennghttpxfun.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python + +from gentokenlookup import gentokenlookup + +OPTIONS = [ + "private-key-file", + "private-key-passwd-file", + "certificate-file", + "dh-param-file", + "subcert", + "backend", + "frontend", + "workers", + "http2-max-concurrent-streams", + "log-level", + "daemon", + "http2-proxy", + "http2-bridge", + "client-proxy", + "add-x-forwarded-for", + "strip-incoming-x-forwarded-for", + "no-via", + "frontend-http2-read-timeout", + "frontend-read-timeout", + "frontend-write-timeout", + "backend-read-timeout", + "backend-write-timeout", + "stream-read-timeout", + "stream-write-timeout", + "accesslog-file", + "accesslog-syslog", + "accesslog-format", + "errorlog-file", + "errorlog-syslog", + "backend-keep-alive-timeout", + "frontend-http2-window-bits", + "backend-http2-window-bits", + "frontend-http2-connection-window-bits", + "backend-http2-connection-window-bits", + "frontend-no-tls", + "backend-no-tls", + "backend-tls-sni-field", + "pid-file", + "user", + "syslog-facility", + "backlog", + "ciphers", + "client", + "insecure", + "cacert", + "backend-ipv4", + "backend-ipv6", + "backend-http-proxy-uri", + "read-rate", + "read-burst", + "write-rate", + "write-burst", + "worker-read-rate", + "worker-read-burst", + "worker-write-rate", + "worker-write-burst", + "npn-list", + "tls-proto-list", + "verify-client", + "verify-client-cacert", + "client-private-key-file", + "client-cert-file", + "frontend-http2-dump-request-header", + "frontend-http2-dump-response-header", + "http2-no-cookie-crumbling", + "frontend-frame-debug", + "padding", + "altsvc", + "add-request-header", + "add-response-header", + "worker-frontend-connections", + "no-location-rewrite", + "no-host-rewrite", + "backend-http1-connections-per-host", + "backend-http1-connections-per-frontend", + "listener-disable-timeout", + "tls-ticket-key-file", + "rlimit-nofile", + "backend-request-buffer", + "backend-response-buffer", + "no-server-push", + "backend-http2-connections-per-worker", + "fetch-ocsp-response-file", + "ocsp-update-interval", + "no-ocsp", + "header-field-buffer", + "max-header-fields", + "include", + "conf", +] + +LOGVARS = [ + "remote_addr", + "time_local", + "time_iso8601", + "request", + "status", + "body_bytes_sent", + "remote_port", + "server_port", + "request_time", + "pid", + "alpn", + "ssl_cipher", + "ssl_protocol", + "ssl_session_id", + "ssl_session_reused", +] + +if __name__ == '__main__': + gentokenlookup(OPTIONS, 'SHRPX_OPTID', value_type='char', comp_fun='util::strieq_l') + gentokenlookup(LOGVARS, 'SHRPX_LOGF', value_type='char', comp_fun='util::strieq_l') diff --git a/gentokenlookup.py b/gentokenlookup.py index 254fd5d6..cf96bf68 100644 --- a/gentokenlookup.py +++ b/gentokenlookup.py @@ -33,10 +33,10 @@ enum {''' {}_MAXIDX, }};'''.format(prefix) -def gen_index_header(tokens, prefix): +def gen_index_header(tokens, prefix, value_type, comp_fun): print '''\ -int lookup_token(const uint8_t *name, size_t namelen) { - switch (namelen) {''' +int lookup_token(const {} *name, size_t namelen) {{ + switch (namelen) {{'''.format(value_type) b = build_header(tokens) for size in sorted(b.keys()): ents = b[size] @@ -50,9 +50,9 @@ int lookup_token(const uint8_t *name, size_t namelen) { case '{}':'''.format(c) for k in headers: print '''\ - if (util::streq_l("{}", name, {})) {{ + if ({}("{}", name, {})) {{ return {}; - }}'''.format(k[:-1], size - 1, to_enum_hd(k, prefix)) + }}'''.format(comp_fun, k[:-1], size - 1, to_enum_hd(k, prefix)) print '''\ break;''' print '''\ @@ -63,7 +63,7 @@ int lookup_token(const uint8_t *name, size_t namelen) { return -1; }''' -def gentokenlookup(tokens, prefix): +def gentokenlookup(tokens, prefix, value_type='uint8_t', comp_fun='util::streq_l'): gen_enum(tokens, prefix) print '' - gen_index_header(tokens, prefix) + gen_index_header(tokens, prefix, value_type, comp_fun) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 37b9693b..3064b5ea 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -494,12 +494,649 @@ void parse_mapping(const DownstreamAddr &addr, const char *src) { } } // namespace +enum { + SHRPX_OPTID_ACCESSLOG_FILE, + SHRPX_OPTID_ACCESSLOG_FORMAT, + SHRPX_OPTID_ACCESSLOG_SYSLOG, + SHRPX_OPTID_ADD_REQUEST_HEADER, + SHRPX_OPTID_ADD_RESPONSE_HEADER, + SHRPX_OPTID_ADD_X_FORWARDED_FOR, + SHRPX_OPTID_ALTSVC, + SHRPX_OPTID_BACKEND, + SHRPX_OPTID_BACKEND_HTTP_PROXY_URI, + SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND, + SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST, + SHRPX_OPTID_BACKEND_HTTP2_CONNECTION_WINDOW_BITS, + SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER, + SHRPX_OPTID_BACKEND_HTTP2_WINDOW_BITS, + SHRPX_OPTID_BACKEND_IPV4, + SHRPX_OPTID_BACKEND_IPV6, + SHRPX_OPTID_BACKEND_KEEP_ALIVE_TIMEOUT, + SHRPX_OPTID_BACKEND_NO_TLS, + SHRPX_OPTID_BACKEND_READ_TIMEOUT, + SHRPX_OPTID_BACKEND_REQUEST_BUFFER, + SHRPX_OPTID_BACKEND_RESPONSE_BUFFER, + SHRPX_OPTID_BACKEND_TLS_SNI_FIELD, + SHRPX_OPTID_BACKEND_WRITE_TIMEOUT, + SHRPX_OPTID_BACKLOG, + SHRPX_OPTID_CACERT, + SHRPX_OPTID_CERTIFICATE_FILE, + SHRPX_OPTID_CIPHERS, + SHRPX_OPTID_CLIENT, + SHRPX_OPTID_CLIENT_CERT_FILE, + SHRPX_OPTID_CLIENT_PRIVATE_KEY_FILE, + SHRPX_OPTID_CLIENT_PROXY, + SHRPX_OPTID_CONF, + SHRPX_OPTID_DAEMON, + SHRPX_OPTID_DH_PARAM_FILE, + SHRPX_OPTID_ERRORLOG_FILE, + SHRPX_OPTID_ERRORLOG_SYSLOG, + SHRPX_OPTID_FETCH_OCSP_RESPONSE_FILE, + SHRPX_OPTID_FRONTEND, + SHRPX_OPTID_FRONTEND_FRAME_DEBUG, + SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS, + SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER, + SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER, + SHRPX_OPTID_FRONTEND_HTTP2_READ_TIMEOUT, + SHRPX_OPTID_FRONTEND_HTTP2_WINDOW_BITS, + SHRPX_OPTID_FRONTEND_NO_TLS, + SHRPX_OPTID_FRONTEND_READ_TIMEOUT, + SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT, + SHRPX_OPTID_HEADER_FIELD_BUFFER, + SHRPX_OPTID_HTTP2_BRIDGE, + SHRPX_OPTID_HTTP2_MAX_CONCURRENT_STREAMS, + SHRPX_OPTID_HTTP2_NO_COOKIE_CRUMBLING, + SHRPX_OPTID_HTTP2_PROXY, + SHRPX_OPTID_INCLUDE, + SHRPX_OPTID_INSECURE, + SHRPX_OPTID_LISTENER_DISABLE_TIMEOUT, + SHRPX_OPTID_LOG_LEVEL, + SHRPX_OPTID_MAX_HEADER_FIELDS, + SHRPX_OPTID_NO_HOST_REWRITE, + SHRPX_OPTID_NO_LOCATION_REWRITE, + SHRPX_OPTID_NO_OCSP, + SHRPX_OPTID_NO_SERVER_PUSH, + SHRPX_OPTID_NO_VIA, + SHRPX_OPTID_NPN_LIST, + SHRPX_OPTID_OCSP_UPDATE_INTERVAL, + SHRPX_OPTID_PADDING, + SHRPX_OPTID_PID_FILE, + SHRPX_OPTID_PRIVATE_KEY_FILE, + SHRPX_OPTID_PRIVATE_KEY_PASSWD_FILE, + SHRPX_OPTID_READ_BURST, + SHRPX_OPTID_READ_RATE, + SHRPX_OPTID_RLIMIT_NOFILE, + SHRPX_OPTID_STREAM_READ_TIMEOUT, + SHRPX_OPTID_STREAM_WRITE_TIMEOUT, + SHRPX_OPTID_STRIP_INCOMING_X_FORWARDED_FOR, + SHRPX_OPTID_SUBCERT, + SHRPX_OPTID_SYSLOG_FACILITY, + SHRPX_OPTID_TLS_PROTO_LIST, + SHRPX_OPTID_TLS_TICKET_KEY_FILE, + SHRPX_OPTID_USER, + SHRPX_OPTID_VERIFY_CLIENT, + SHRPX_OPTID_VERIFY_CLIENT_CACERT, + SHRPX_OPTID_WORKER_FRONTEND_CONNECTIONS, + SHRPX_OPTID_WORKER_READ_BURST, + SHRPX_OPTID_WORKER_READ_RATE, + SHRPX_OPTID_WORKER_WRITE_BURST, + SHRPX_OPTID_WORKER_WRITE_RATE, + SHRPX_OPTID_WORKERS, + SHRPX_OPTID_WRITE_BURST, + SHRPX_OPTID_WRITE_RATE, + SHRPX_OPTID_MAXIDX, +}; + +namespace { +int option_lookup_token(const char *name, size_t namelen) { + switch (namelen) { + case 4: + switch (name[3]) { + case 'f': + if (util::strieq_l("con", name, 3)) { + return SHRPX_OPTID_CONF; + } + break; + case 'r': + if (util::strieq_l("use", name, 3)) { + return SHRPX_OPTID_USER; + } + break; + } + break; + case 6: + switch (name[5]) { + case 'a': + if (util::strieq_l("no-vi", name, 5)) { + return SHRPX_OPTID_NO_VIA; + } + break; + case 'c': + if (util::strieq_l("altsv", name, 5)) { + return SHRPX_OPTID_ALTSVC; + } + break; + case 'n': + if (util::strieq_l("daemo", name, 5)) { + return SHRPX_OPTID_DAEMON; + } + break; + case 't': + if (util::strieq_l("cacer", name, 5)) { + return SHRPX_OPTID_CACERT; + } + if (util::strieq_l("clien", name, 5)) { + return SHRPX_OPTID_CLIENT; + } + break; + } + break; + case 7: + switch (name[6]) { + case 'd': + if (util::strieq_l("backen", name, 6)) { + return SHRPX_OPTID_BACKEND; + } + break; + case 'e': + if (util::strieq_l("includ", name, 6)) { + return SHRPX_OPTID_INCLUDE; + } + break; + case 'g': + if (util::strieq_l("backlo", name, 6)) { + return SHRPX_OPTID_BACKLOG; + } + if (util::strieq_l("paddin", name, 6)) { + return SHRPX_OPTID_PADDING; + } + break; + case 'p': + if (util::strieq_l("no-ocs", name, 6)) { + return SHRPX_OPTID_NO_OCSP; + } + break; + case 's': + if (util::strieq_l("cipher", name, 6)) { + return SHRPX_OPTID_CIPHERS; + } + if (util::strieq_l("worker", name, 6)) { + return SHRPX_OPTID_WORKERS; + } + break; + case 't': + if (util::strieq_l("subcer", name, 6)) { + return SHRPX_OPTID_SUBCERT; + } + break; + } + break; + case 8: + switch (name[7]) { + case 'd': + if (util::strieq_l("fronten", name, 7)) { + return SHRPX_OPTID_FRONTEND; + } + break; + case 'e': + if (util::strieq_l("insecur", name, 7)) { + return SHRPX_OPTID_INSECURE; + } + if (util::strieq_l("pid-fil", name, 7)) { + return SHRPX_OPTID_PID_FILE; + } + break; + case 't': + if (util::strieq_l("npn-lis", name, 7)) { + return SHRPX_OPTID_NPN_LIST; + } + break; + } + break; + case 9: + switch (name[8]) { + case 'e': + if (util::strieq_l("read-rat", name, 8)) { + return SHRPX_OPTID_READ_RATE; + } + break; + case 'l': + if (util::strieq_l("log-leve", name, 8)) { + return SHRPX_OPTID_LOG_LEVEL; + } + break; + } + break; + case 10: + switch (name[9]) { + case 'e': + if (util::strieq_l("write-rat", name, 9)) { + return SHRPX_OPTID_WRITE_RATE; + } + break; + case 't': + if (util::strieq_l("read-burs", name, 9)) { + return SHRPX_OPTID_READ_BURST; + } + break; + } + break; + case 11: + switch (name[10]) { + case 't': + if (util::strieq_l("write-burs", name, 10)) { + return SHRPX_OPTID_WRITE_BURST; + } + break; + case 'y': + if (util::strieq_l("http2-prox", name, 10)) { + return SHRPX_OPTID_HTTP2_PROXY; + } + break; + } + break; + case 12: + switch (name[11]) { + case '4': + if (util::strieq_l("backend-ipv", name, 11)) { + return SHRPX_OPTID_BACKEND_IPV4; + } + break; + case '6': + if (util::strieq_l("backend-ipv", name, 11)) { + return SHRPX_OPTID_BACKEND_IPV6; + } + break; + case 'e': + if (util::strieq_l("http2-bridg", name, 11)) { + return SHRPX_OPTID_HTTP2_BRIDGE; + } + break; + case 'y': + if (util::strieq_l("client-prox", name, 11)) { + return SHRPX_OPTID_CLIENT_PROXY; + } + break; + } + break; + case 13: + switch (name[12]) { + case 'e': + if (util::strieq_l("dh-param-fil", name, 12)) { + return SHRPX_OPTID_DH_PARAM_FILE; + } + if (util::strieq_l("errorlog-fil", name, 12)) { + return SHRPX_OPTID_ERRORLOG_FILE; + } + if (util::strieq_l("rlimit-nofil", name, 12)) { + return SHRPX_OPTID_RLIMIT_NOFILE; + } + break; + case 't': + if (util::strieq_l("verify-clien", name, 12)) { + return SHRPX_OPTID_VERIFY_CLIENT; + } + break; + } + break; + case 14: + switch (name[13]) { + case 'e': + if (util::strieq_l("accesslog-fil", name, 13)) { + return SHRPX_OPTID_ACCESSLOG_FILE; + } + break; + case 'h': + if (util::strieq_l("no-server-pus", name, 13)) { + return SHRPX_OPTID_NO_SERVER_PUSH; + } + break; + case 's': + if (util::strieq_l("backend-no-tl", name, 13)) { + return SHRPX_OPTID_BACKEND_NO_TLS; + } + break; + case 't': + if (util::strieq_l("tls-proto-lis", name, 13)) { + return SHRPX_OPTID_TLS_PROTO_LIST; + } + break; + } + break; + case 15: + switch (name[14]) { + case 'e': + if (util::strieq_l("no-host-rewrit", name, 14)) { + return SHRPX_OPTID_NO_HOST_REWRITE; + } + break; + case 'g': + if (util::strieq_l("errorlog-syslo", name, 14)) { + return SHRPX_OPTID_ERRORLOG_SYSLOG; + } + break; + case 's': + if (util::strieq_l("frontend-no-tl", name, 14)) { + return SHRPX_OPTID_FRONTEND_NO_TLS; + } + break; + case 'y': + if (util::strieq_l("syslog-facilit", name, 14)) { + return SHRPX_OPTID_SYSLOG_FACILITY; + } + break; + } + break; + case 16: + switch (name[15]) { + case 'e': + if (util::strieq_l("certificate-fil", name, 15)) { + return SHRPX_OPTID_CERTIFICATE_FILE; + } + if (util::strieq_l("client-cert-fil", name, 15)) { + return SHRPX_OPTID_CLIENT_CERT_FILE; + } + if (util::strieq_l("private-key-fil", name, 15)) { + return SHRPX_OPTID_PRIVATE_KEY_FILE; + } + if (util::strieq_l("worker-read-rat", name, 15)) { + return SHRPX_OPTID_WORKER_READ_RATE; + } + break; + case 'g': + if (util::strieq_l("accesslog-syslo", name, 15)) { + return SHRPX_OPTID_ACCESSLOG_SYSLOG; + } + break; + case 't': + if (util::strieq_l("accesslog-forma", name, 15)) { + return SHRPX_OPTID_ACCESSLOG_FORMAT; + } + break; + } + break; + case 17: + switch (name[16]) { + case 'e': + if (util::strieq_l("worker-write-rat", name, 16)) { + return SHRPX_OPTID_WORKER_WRITE_RATE; + } + break; + case 's': + if (util::strieq_l("max-header-field", name, 16)) { + return SHRPX_OPTID_MAX_HEADER_FIELDS; + } + break; + case 't': + if (util::strieq_l("worker-read-burs", name, 16)) { + return SHRPX_OPTID_WORKER_READ_BURST; + } + break; + } + break; + case 18: + switch (name[17]) { + case 'r': + if (util::strieq_l("add-request-heade", name, 17)) { + return SHRPX_OPTID_ADD_REQUEST_HEADER; + } + break; + case 't': + if (util::strieq_l("worker-write-burs", name, 17)) { + return SHRPX_OPTID_WORKER_WRITE_BURST; + } + break; + } + break; + case 19: + switch (name[18]) { + case 'e': + if (util::strieq_l("no-location-rewrit", name, 18)) { + return SHRPX_OPTID_NO_LOCATION_REWRITE; + } + if (util::strieq_l("tls-ticket-key-fil", name, 18)) { + return SHRPX_OPTID_TLS_TICKET_KEY_FILE; + } + break; + case 'r': + if (util::strieq_l("add-response-heade", name, 18)) { + return SHRPX_OPTID_ADD_RESPONSE_HEADER; + } + if (util::strieq_l("add-x-forwarded-fo", name, 18)) { + return SHRPX_OPTID_ADD_X_FORWARDED_FOR; + } + if (util::strieq_l("header-field-buffe", name, 18)) { + return SHRPX_OPTID_HEADER_FIELD_BUFFER; + } + break; + case 't': + if (util::strieq_l("stream-read-timeou", name, 18)) { + return SHRPX_OPTID_STREAM_READ_TIMEOUT; + } + break; + } + break; + case 20: + switch (name[19]) { + case 'g': + if (util::strieq_l("frontend-frame-debu", name, 19)) { + return SHRPX_OPTID_FRONTEND_FRAME_DEBUG; + } + break; + case 'l': + if (util::strieq_l("ocsp-update-interva", name, 19)) { + return SHRPX_OPTID_OCSP_UPDATE_INTERVAL; + } + break; + case 't': + if (util::strieq_l("backend-read-timeou", name, 19)) { + return SHRPX_OPTID_BACKEND_READ_TIMEOUT; + } + if (util::strieq_l("stream-write-timeou", name, 19)) { + return SHRPX_OPTID_STREAM_WRITE_TIMEOUT; + } + if (util::strieq_l("verify-client-cacer", name, 19)) { + return SHRPX_OPTID_VERIFY_CLIENT_CACERT; + } + break; + } + break; + case 21: + switch (name[20]) { + case 'd': + if (util::strieq_l("backend-tls-sni-fiel", name, 20)) { + return SHRPX_OPTID_BACKEND_TLS_SNI_FIELD; + } + break; + case 't': + if (util::strieq_l("backend-write-timeou", name, 20)) { + return SHRPX_OPTID_BACKEND_WRITE_TIMEOUT; + } + if (util::strieq_l("frontend-read-timeou", name, 20)) { + return SHRPX_OPTID_FRONTEND_READ_TIMEOUT; + } + break; + } + break; + case 22: + switch (name[21]) { + case 'i': + if (util::strieq_l("backend-http-proxy-ur", name, 21)) { + return SHRPX_OPTID_BACKEND_HTTP_PROXY_URI; + } + break; + case 'r': + if (util::strieq_l("backend-request-buffe", name, 21)) { + return SHRPX_OPTID_BACKEND_REQUEST_BUFFER; + } + break; + case 't': + if (util::strieq_l("frontend-write-timeou", name, 21)) { + return SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT; + } + break; + } + break; + case 23: + switch (name[22]) { + case 'e': + if (util::strieq_l("client-private-key-fil", name, 22)) { + return SHRPX_OPTID_CLIENT_PRIVATE_KEY_FILE; + } + if (util::strieq_l("private-key-passwd-fil", name, 22)) { + return SHRPX_OPTID_PRIVATE_KEY_PASSWD_FILE; + } + break; + case 'r': + if (util::strieq_l("backend-response-buffe", name, 22)) { + return SHRPX_OPTID_BACKEND_RESPONSE_BUFFER; + } + break; + } + break; + case 24: + switch (name[23]) { + case 'e': + if (util::strieq_l("fetch-ocsp-response-fil", name, 23)) { + return SHRPX_OPTID_FETCH_OCSP_RESPONSE_FILE; + } + break; + case 't': + if (util::strieq_l("listener-disable-timeou", name, 23)) { + return SHRPX_OPTID_LISTENER_DISABLE_TIMEOUT; + } + break; + } + break; + case 25: + switch (name[24]) { + case 'g': + if (util::strieq_l("http2-no-cookie-crumblin", name, 24)) { + return SHRPX_OPTID_HTTP2_NO_COOKIE_CRUMBLING; + } + break; + case 's': + if (util::strieq_l("backend-http2-window-bit", name, 24)) { + return SHRPX_OPTID_BACKEND_HTTP2_WINDOW_BITS; + } + break; + } + break; + case 26: + switch (name[25]) { + case 's': + if (util::strieq_l("frontend-http2-window-bit", name, 25)) { + return SHRPX_OPTID_FRONTEND_HTTP2_WINDOW_BITS; + } + break; + case 't': + if (util::strieq_l("backend-keep-alive-timeou", name, 25)) { + return SHRPX_OPTID_BACKEND_KEEP_ALIVE_TIMEOUT; + } + break; + } + break; + case 27: + switch (name[26]) { + case 's': + if (util::strieq_l("worker-frontend-connection", name, 26)) { + return SHRPX_OPTID_WORKER_FRONTEND_CONNECTIONS; + } + break; + case 't': + if (util::strieq_l("frontend-http2-read-timeou", name, 26)) { + return SHRPX_OPTID_FRONTEND_HTTP2_READ_TIMEOUT; + } + break; + } + break; + case 28: + switch (name[27]) { + case 's': + if (util::strieq_l("http2-max-concurrent-stream", name, 27)) { + return SHRPX_OPTID_HTTP2_MAX_CONCURRENT_STREAMS; + } + break; + } + break; + case 30: + switch (name[29]) { + case 'r': + if (util::strieq_l("strip-incoming-x-forwarded-fo", name, 29)) { + return SHRPX_OPTID_STRIP_INCOMING_X_FORWARDED_FOR; + } + break; + } + break; + case 34: + switch (name[33]) { + case 'r': + if (util::strieq_l("frontend-http2-dump-request-heade", name, 33)) { + return SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER; + } + break; + case 't': + if (util::strieq_l("backend-http1-connections-per-hos", name, 33)) { + return SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST; + } + break; + } + break; + case 35: + switch (name[34]) { + case 'r': + if (util::strieq_l("frontend-http2-dump-response-heade", name, 34)) { + return SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER; + } + break; + } + break; + case 36: + switch (name[35]) { + case 'r': + if (util::strieq_l("backend-http2-connections-per-worke", name, 35)) { + return SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER; + } + break; + case 's': + if (util::strieq_l("backend-http2-connection-window-bit", name, 35)) { + return SHRPX_OPTID_BACKEND_HTTP2_CONNECTION_WINDOW_BITS; + } + break; + } + break; + case 37: + switch (name[36]) { + case 's': + if (util::strieq_l("frontend-http2-connection-window-bit", name, 36)) { + return SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS; + } + break; + } + break; + case 38: + switch (name[37]) { + case 'd': + if (util::strieq_l("backend-http1-connections-per-fronten", name, 37)) { + return SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND; + } + break; + } + break; + } + return -1; +} +} // namespace + int parse_config(const char *opt, const char *optarg, std::set &included_set) { char host[NI_MAXHOST]; uint16_t port; - if (util::strieq(opt, SHRPX_OPT_BACKEND)) { + auto optid = option_lookup_token(opt, strlen(opt)); + + switch (optid) { + case SHRPX_OPTID_BACKEND: { auto optarglen = strlen(optarg); const char *pat_delim = strchr(optarg, ';'); if (!pat_delim) { @@ -531,8 +1168,7 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND)) { + case SHRPX_OPTID_FRONTEND: { if (util::istartsWith(optarg, SHRPX_UNIX_PATH_PREFIX)) { auto path = optarg + str_size(SHRPX_UNIX_PATH_PREFIX); mod_config()->host = strcopy(path); @@ -553,136 +1189,88 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_WORKERS)) { + case SHRPX_OPTID_WORKERS: return parse_uint(&mod_config()->num_worker, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_HTTP2_MAX_CONCURRENT_STREAMS)) { + case SHRPX_OPTID_HTTP2_MAX_CONCURRENT_STREAMS: return parse_uint(&mod_config()->http2_max_concurrent_streams, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_LOG_LEVEL)) { + case SHRPX_OPTID_LOG_LEVEL: if (Log::set_severity_level_by_name(optarg) == -1) { LOG(ERROR) << opt << ": Invalid severity level: " << optarg; return -1; } return 0; - } - - if (util::strieq(opt, SHRPX_OPT_DAEMON)) { + case SHRPX_OPTID_DAEMON: mod_config()->daemon = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_HTTP2_PROXY)) { + case SHRPX_OPTID_HTTP2_PROXY: mod_config()->http2_proxy = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_HTTP2_BRIDGE)) { + case SHRPX_OPTID_HTTP2_BRIDGE: mod_config()->http2_bridge = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_CLIENT_PROXY)) { + case SHRPX_OPTID_CLIENT_PROXY: mod_config()->client_proxy = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_ADD_X_FORWARDED_FOR)) { + case SHRPX_OPTID_ADD_X_FORWARDED_FOR: mod_config()->add_x_forwarded_for = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_STRIP_INCOMING_X_FORWARDED_FOR)) { + case SHRPX_OPTID_STRIP_INCOMING_X_FORWARDED_FOR: mod_config()->strip_incoming_x_forwarded_for = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_NO_VIA)) { + case SHRPX_OPTID_NO_VIA: mod_config()->no_via = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND_HTTP2_READ_TIMEOUT)) { + case SHRPX_OPTID_FRONTEND_HTTP2_READ_TIMEOUT: return parse_duration(&mod_config()->http2_upstream_read_timeout, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND_READ_TIMEOUT)) { + case SHRPX_OPTID_FRONTEND_READ_TIMEOUT: return parse_duration(&mod_config()->upstream_read_timeout, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND_WRITE_TIMEOUT)) { + case SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT: return parse_duration(&mod_config()->upstream_write_timeout, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_READ_TIMEOUT)) { + case SHRPX_OPTID_BACKEND_READ_TIMEOUT: return parse_duration(&mod_config()->downstream_read_timeout, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_WRITE_TIMEOUT)) { + case SHRPX_OPTID_BACKEND_WRITE_TIMEOUT: return parse_duration(&mod_config()->downstream_write_timeout, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_STREAM_READ_TIMEOUT)) { + case SHRPX_OPTID_STREAM_READ_TIMEOUT: return parse_duration(&mod_config()->stream_read_timeout, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_STREAM_WRITE_TIMEOUT)) { + case SHRPX_OPTID_STREAM_WRITE_TIMEOUT: return parse_duration(&mod_config()->stream_write_timeout, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_ACCESSLOG_FILE)) { + case SHRPX_OPTID_ACCESSLOG_FILE: mod_config()->accesslog_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_ACCESSLOG_SYSLOG)) { + case SHRPX_OPTID_ACCESSLOG_SYSLOG: mod_config()->accesslog_syslog = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_ACCESSLOG_FORMAT)) { + case SHRPX_OPTID_ACCESSLOG_FORMAT: mod_config()->accesslog_format = parse_log_format(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_ERRORLOG_FILE)) { + case SHRPX_OPTID_ERRORLOG_FILE: mod_config()->errorlog_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_ERRORLOG_SYSLOG)) { + case SHRPX_OPTID_ERRORLOG_SYSLOG: mod_config()->errorlog_syslog = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_KEEP_ALIVE_TIMEOUT)) { + case SHRPX_OPTID_BACKEND_KEEP_ALIVE_TIMEOUT: return parse_duration(&mod_config()->downstream_idle_read_timeout, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND_HTTP2_WINDOW_BITS) || - util::strieq(opt, SHRPX_OPT_BACKEND_HTTP2_WINDOW_BITS)) { - + case SHRPX_OPTID_FRONTEND_HTTP2_WINDOW_BITS: + case SHRPX_OPTID_BACKEND_HTTP2_WINDOW_BITS: { size_t *resp; - if (util::strieq(opt, SHRPX_OPT_FRONTEND_HTTP2_WINDOW_BITS)) { + if (optid == SHRPX_OPTID_FRONTEND_HTTP2_WINDOW_BITS) { resp = &mod_config()->http2_upstream_window_bits; } else { resp = &mod_config()->http2_downstream_window_bits; @@ -706,13 +1294,11 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS) || - util::strieq(opt, SHRPX_OPT_BACKEND_HTTP2_CONNECTION_WINDOW_BITS)) { - + case SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS: + case SHRPX_OPTID_BACKEND_HTTP2_CONNECTION_WINDOW_BITS: { size_t *resp; - if (util::strieq(opt, SHRPX_OPT_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS)) { + if (optid == SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS) { resp = &mod_config()->http2_upstream_connection_window_bits; } else { resp = &mod_config()->http2_downstream_connection_window_bits; @@ -736,32 +1322,23 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND_NO_TLS)) { + case SHRPX_OPTID_FRONTEND_NO_TLS: mod_config()->upstream_no_tls = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_NO_TLS)) { + case SHRPX_OPTID_BACKEND_NO_TLS: mod_config()->downstream_no_tls = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_TLS_SNI_FIELD)) { + case SHRPX_OPTID_BACKEND_TLS_SNI_FIELD: mod_config()->backend_tls_sni_name = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_PID_FILE)) { + case SHRPX_OPTID_PID_FILE: mod_config()->pid_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_USER)) { + case SHRPX_OPTID_USER: { auto pwd = getpwnam(optarg); if (!pwd) { LOG(ERROR) << opt << ": failed to get uid from " << optarg << ": " @@ -774,14 +1351,11 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_PRIVATE_KEY_FILE)) { + case SHRPX_OPTID_PRIVATE_KEY_FILE: mod_config()->private_key_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_PRIVATE_KEY_PASSWD_FILE)) { + case SHRPX_OPTID_PRIVATE_KEY_PASSWD_FILE: { auto passwd = read_passwd_from_file(optarg); if (passwd.empty()) { LOG(ERROR) << opt << ": Couldn't read key file's passwd from " << optarg; @@ -791,20 +1365,15 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_CERTIFICATE_FILE)) { + case SHRPX_OPTID_CERTIFICATE_FILE: mod_config()->cert_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_DH_PARAM_FILE)) { + case SHRPX_OPTID_DH_PARAM_FILE: mod_config()->dh_param_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_SUBCERT)) { + case SHRPX_OPTID_SUBCERT: { // Private Key file and certificate file separated by ':'. const char *sp = strchr(optarg, ':'); if (sp) { @@ -815,8 +1384,7 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_SYSLOG_FACILITY)) { + case SHRPX_OPTID_SYSLOG_FACILITY: { int facility = int_syslog_facility(optarg); if (facility == -1) { LOG(ERROR) << opt << ": Unknown syslog facility: " << optarg; @@ -826,8 +1394,7 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_BACKLOG)) { + case SHRPX_OPTID_BACKLOG: { int n; if (parse_int(&n, opt, optarg) != 0) { return -1; @@ -843,44 +1410,31 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_CIPHERS)) { + case SHRPX_OPTID_CIPHERS: mod_config()->ciphers = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_CLIENT)) { + case SHRPX_OPTID_CLIENT: mod_config()->client = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_INSECURE)) { + case SHRPX_OPTID_INSECURE: mod_config()->insecure = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_CACERT)) { + case SHRPX_OPTID_CACERT: mod_config()->cacert = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_IPV4)) { + case SHRPX_OPTID_BACKEND_IPV4: mod_config()->backend_ipv4 = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_IPV6)) { + case SHRPX_OPTID_BACKEND_IPV6: mod_config()->backend_ipv6 = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_HTTP_PROXY_URI)) { + case SHRPX_OPTID_BACKEND_HTTP_PROXY_URI: { // parse URI and get hostname, port and optionally userinfo. http_parser_url u; memset(&u, 0, sizeof(u)); @@ -916,112 +1470,71 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_READ_RATE)) { + case SHRPX_OPTID_READ_RATE: return parse_uint_with_unit(&mod_config()->read_rate, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_READ_BURST)) { + case SHRPX_OPTID_READ_BURST: return parse_uint_with_unit(&mod_config()->read_burst, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_WRITE_RATE)) { + case SHRPX_OPTID_WRITE_RATE: return parse_uint_with_unit(&mod_config()->write_rate, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_WRITE_BURST)) { + case SHRPX_OPTID_WRITE_BURST: return parse_uint_with_unit(&mod_config()->write_burst, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_WORKER_READ_RATE)) { + case SHRPX_OPTID_WORKER_READ_RATE: LOG(WARN) << opt << ": not implemented yet"; return parse_uint_with_unit(&mod_config()->worker_read_rate, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_WORKER_READ_BURST)) { + case SHRPX_OPTID_WORKER_READ_BURST: LOG(WARN) << opt << ": not implemented yet"; return parse_uint_with_unit(&mod_config()->worker_read_burst, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_WORKER_WRITE_RATE)) { + case SHRPX_OPTID_WORKER_WRITE_RATE: LOG(WARN) << opt << ": not implemented yet"; return parse_uint_with_unit(&mod_config()->worker_write_rate, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_WORKER_WRITE_BURST)) { + case SHRPX_OPTID_WORKER_WRITE_BURST: LOG(WARN) << opt << ": not implemented yet"; return parse_uint_with_unit(&mod_config()->worker_write_burst, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_NPN_LIST)) { + case SHRPX_OPTID_NPN_LIST: clear_config_str_list(mod_config()->npn_list); - mod_config()->npn_list = parse_config_str_list(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_TLS_PROTO_LIST)) { + case SHRPX_OPTID_TLS_PROTO_LIST: clear_config_str_list(mod_config()->tls_proto_list); - mod_config()->tls_proto_list = parse_config_str_list(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_VERIFY_CLIENT)) { + case SHRPX_OPTID_VERIFY_CLIENT: mod_config()->verify_client = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_VERIFY_CLIENT_CACERT)) { + case SHRPX_OPTID_VERIFY_CLIENT_CACERT: mod_config()->verify_client_cacert = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_CLIENT_PRIVATE_KEY_FILE)) { + case SHRPX_OPTID_CLIENT_PRIVATE_KEY_FILE: mod_config()->client_private_key_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_CLIENT_CERT_FILE)) { + case SHRPX_OPTID_CLIENT_CERT_FILE: mod_config()->client_cert_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND_HTTP2_DUMP_REQUEST_HEADER)) { + case SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER: mod_config()->http2_upstream_dump_request_header_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER)) { + case SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER: mod_config()->http2_upstream_dump_response_header_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_HTTP2_NO_COOKIE_CRUMBLING)) { + case SHRPX_OPTID_HTTP2_NO_COOKIE_CRUMBLING: mod_config()->http2_no_cookie_crumbling = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND_FRAME_DEBUG)) { + case SHRPX_OPTID_FRONTEND_FRAME_DEBUG: mod_config()->upstream_frame_debug = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_PADDING)) { + case SHRPX_OPTID_PADDING: return parse_uint(&mod_config()->padding, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_ALTSVC)) { + case SHRPX_OPTID_ALTSVC: { auto tokens = parse_config_str_list(optarg); if (tokens.size() < 2) { @@ -1069,39 +1582,31 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_ADD_REQUEST_HEADER) || - util::strieq(opt, SHRPX_OPT_ADD_RESPONSE_HEADER)) { + case SHRPX_OPTID_ADD_REQUEST_HEADER: + case SHRPX_OPTID_ADD_RESPONSE_HEADER: { auto p = parse_header(optarg); if (p.first.empty()) { LOG(ERROR) << opt << ": header field name is empty: " << optarg; return -1; } - if (util::strieq(opt, SHRPX_OPT_ADD_REQUEST_HEADER)) { + if (optid == SHRPX_OPTID_ADD_REQUEST_HEADER) { mod_config()->add_request_headers.push_back(std::move(p)); } else { mod_config()->add_response_headers.push_back(std::move(p)); } return 0; } - - if (util::strieq(opt, SHRPX_OPT_WORKER_FRONTEND_CONNECTIONS)) { + case SHRPX_OPTID_WORKER_FRONTEND_CONNECTIONS: return parse_uint(&mod_config()->worker_frontend_connections, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_NO_LOCATION_REWRITE)) { + case SHRPX_OPTID_NO_LOCATION_REWRITE: mod_config()->no_location_rewrite = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_NO_HOST_REWRITE)) { + case SHRPX_OPTID_NO_HOST_REWRITE: mod_config()->no_host_rewrite = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_HTTP1_CONNECTIONS_PER_HOST)) { + case SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST: { int n; if (parse_uint(&n, opt, optarg) != 0) { @@ -1118,22 +1623,15 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND)) { + case SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND: return parse_uint(&mod_config()->downstream_connections_per_frontend, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_LISTENER_DISABLE_TIMEOUT)) { + case SHRPX_OPTID_LISTENER_DISABLE_TIMEOUT: return parse_duration(&mod_config()->listener_disable_timeout, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_TLS_TICKET_KEY_FILE)) { + case SHRPX_OPTID_TLS_TICKET_KEY_FILE: mod_config()->tls_ticket_key_files.push_back(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_RLIMIT_NOFILE)) { + case SHRPX_OPTID_RLIMIT_NOFILE: { int n; if (parse_uint(&n, opt, optarg) != 0) { @@ -1150,9 +1648,8 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_REQUEST_BUFFER) || - util::strieq(opt, SHRPX_OPT_BACKEND_RESPONSE_BUFFER)) { + case SHRPX_OPTID_BACKEND_REQUEST_BUFFER: + case SHRPX_OPTID_BACKEND_RESPONSE_BUFFER: { size_t n; if (parse_uint_with_unit(&n, opt, optarg) != 0) { return -1; @@ -1164,7 +1661,7 @@ int parse_config(const char *opt, const char *optarg, return -1; } - if (util::strieq(opt, SHRPX_OPT_BACKEND_REQUEST_BUFFER)) { + if (optid == SHRPX_OPTID_BACKEND_REQUEST_BUFFER) { mod_config()->downstream_request_buffer_size = n; } else { mod_config()->downstream_response_buffer_size = n; @@ -1173,43 +1670,29 @@ int parse_config(const char *opt, const char *optarg, return 0; } - if (util::strieq(opt, SHRPX_OPT_NO_SERVER_PUSH)) { + case SHRPX_OPTID_NO_SERVER_PUSH: mod_config()->no_server_push = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_HTTP2_CONNECTIONS_PER_WORKER)) { + case SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER: return parse_uint(&mod_config()->http2_downstream_connections_per_worker, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_FETCH_OCSP_RESPONSE_FILE)) { + case SHRPX_OPTID_FETCH_OCSP_RESPONSE_FILE: mod_config()->fetch_ocsp_response_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_OCSP_UPDATE_INTERVAL)) { + case SHRPX_OPTID_OCSP_UPDATE_INTERVAL: return parse_duration(&mod_config()->ocsp_update_interval, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_NO_OCSP)) { + case SHRPX_OPTID_NO_OCSP: mod_config()->no_ocsp = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_HEADER_FIELD_BUFFER)) { + case SHRPX_OPTID_HEADER_FIELD_BUFFER: return parse_uint_with_unit(&mod_config()->header_field_buffer, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_MAX_HEADER_FIELDS)) { + case SHRPX_OPTID_MAX_HEADER_FIELDS: return parse_uint(&mod_config()->max_header_fields, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_INCLUDE)) { + case SHRPX_OPTID_INCLUDE: { if (included_set.count(optarg)) { LOG(ERROR) << opt << ": " << optarg << " has already been included"; return -1; @@ -1225,8 +1708,7 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, "conf")) { + case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; return 0; From 4fed7a147611980003459330508e1902f645ea61 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Jul 2015 23:36:44 +0900 Subject: [PATCH 22/77] nghttpx: Refactor log format parsing --- gennghttpxfun.py | 2 +- gentokenlookup.py | 16 ++--- src/shrpx_config.cc | 170 ++++++++++++++++++++++++++++++++++---------- 3 files changed, 141 insertions(+), 47 deletions(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 81836fe9..6412ae2b 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -114,4 +114,4 @@ LOGVARS = [ if __name__ == '__main__': gentokenlookup(OPTIONS, 'SHRPX_OPTID', value_type='char', comp_fun='util::strieq_l') - gentokenlookup(LOGVARS, 'SHRPX_LOGF', value_type='char', comp_fun='util::strieq_l') + gentokenlookup(LOGVARS, 'SHRPX_LOGF', value_type='char', comp_fun='util::strieq_l', return_type='LogFragmentType', fail_value='SHRPX_LOGF_NONE') diff --git a/gentokenlookup.py b/gentokenlookup.py index cf96bf68..105eb8d2 100644 --- a/gentokenlookup.py +++ b/gentokenlookup.py @@ -33,10 +33,10 @@ enum {''' {}_MAXIDX, }};'''.format(prefix) -def gen_index_header(tokens, prefix, value_type, comp_fun): +def gen_index_header(tokens, prefix, value_type, comp_fun, return_type, fail_value): print '''\ -int lookup_token(const {} *name, size_t namelen) {{ - switch (namelen) {{'''.format(value_type) +{} lookup_token(const {} *name, size_t namelen) {{ + switch (namelen) {{'''.format(return_type, value_type) b = build_header(tokens) for size in sorted(b.keys()): ents = b[size] @@ -59,11 +59,11 @@ int lookup_token(const {} *name, size_t namelen) {{ } break;''' print '''\ - } - return -1; -}''' + }} + return {}; +}}'''.format(fail_value) -def gentokenlookup(tokens, prefix, value_type='uint8_t', comp_fun='util::streq_l'): +def gentokenlookup(tokens, prefix, value_type='uint8_t', comp_fun='util::streq_l', return_type='int', fail_value='-1'): gen_enum(tokens, prefix) print '' - gen_index_header(tokens, prefix, value_type, comp_fun) + gen_index_header(tokens, prefix, value_type, comp_fun, return_type, fail_value) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 3064b5ea..cc827f66 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -323,6 +323,127 @@ int parse_int(T *dest, const char *opt, const char *optarg) { return 0; } +namespace { +LogFragmentType log_var_lookup_token(const char *name, size_t namelen) { + switch (namelen) { + case 3: + switch (name[2]) { + case 'd': + if (util::strieq_l("pi", name, 2)) { + return SHRPX_LOGF_PID; + } + break; + } + break; + case 4: + switch (name[3]) { + case 'n': + if (util::strieq_l("alp", name, 3)) { + return SHRPX_LOGF_ALPN; + } + break; + } + break; + case 6: + switch (name[5]) { + case 's': + if (util::strieq_l("statu", name, 5)) { + return SHRPX_LOGF_STATUS; + } + break; + } + break; + case 7: + switch (name[6]) { + case 't': + if (util::strieq_l("reques", name, 6)) { + return SHRPX_LOGF_REQUEST; + } + break; + } + break; + case 10: + switch (name[9]) { + case 'l': + if (util::strieq_l("time_loca", name, 9)) { + return SHRPX_LOGF_TIME_LOCAL; + } + break; + case 'r': + if (util::strieq_l("ssl_ciphe", name, 9)) { + return SHRPX_LOGF_SSL_CIPHER; + } + break; + } + break; + case 11: + switch (name[10]) { + case 'r': + if (util::strieq_l("remote_add", name, 10)) { + return SHRPX_LOGF_REMOTE_ADDR; + } + break; + case 't': + if (util::strieq_l("remote_por", name, 10)) { + return SHRPX_LOGF_REMOTE_PORT; + } + if (util::strieq_l("server_por", name, 10)) { + return SHRPX_LOGF_SERVER_PORT; + } + break; + } + break; + case 12: + switch (name[11]) { + case '1': + if (util::strieq_l("time_iso860", name, 11)) { + return SHRPX_LOGF_TIME_ISO8601; + } + break; + case 'e': + if (util::strieq_l("request_tim", name, 11)) { + return SHRPX_LOGF_REQUEST_TIME; + } + break; + case 'l': + if (util::strieq_l("ssl_protoco", name, 11)) { + return SHRPX_LOGF_SSL_PROTOCOL; + } + break; + } + break; + case 14: + switch (name[13]) { + case 'd': + if (util::strieq_l("ssl_session_i", name, 13)) { + return SHRPX_LOGF_SSL_SESSION_ID; + } + break; + } + break; + case 15: + switch (name[14]) { + case 't': + if (util::strieq_l("body_bytes_sen", name, 14)) { + return SHRPX_LOGF_BODY_BYTES_SENT; + } + break; + } + break; + case 18: + switch (name[17]) { + case 'd': + if (util::strieq_l("ssl_session_reuse", name, 17)) { + return SHRPX_LOGF_SSL_SESSION_REUSED; + } + break; + } + break; + } + return SHRPX_LOGF_NONE; +} +} // namespace + namespace { bool var_token(char c) { return util::isAlpha(c) || util::isDigit(c) || c == '_'; @@ -368,46 +489,19 @@ std::vector parse_log_format(const char *optarg) { var_namelen = p - var_name; } - auto type = SHRPX_LOGF_NONE; const char *value = nullptr; - if (util::strieq_l("remote_addr", var_name, var_namelen)) { - type = SHRPX_LOGF_REMOTE_ADDR; - } else if (util::strieq_l("time_local", var_name, var_namelen)) { - type = SHRPX_LOGF_TIME_LOCAL; - } else if (util::strieq_l("time_iso8601", var_name, var_namelen)) { - type = SHRPX_LOGF_TIME_ISO8601; - } else if (util::strieq_l("request", var_name, var_namelen)) { - type = SHRPX_LOGF_REQUEST; - } else if (util::strieq_l("status", var_name, var_namelen)) { - type = SHRPX_LOGF_STATUS; - } else if (util::strieq_l("body_bytes_sent", var_name, var_namelen)) { - type = SHRPX_LOGF_BODY_BYTES_SENT; - } else if (util::istartsWith(var_name, var_namelen, "http_")) { - type = SHRPX_LOGF_HTTP; - value = var_name + sizeof("http_") - 1; - } else if (util::strieq_l("remote_port", var_name, var_namelen)) { - type = SHRPX_LOGF_REMOTE_PORT; - } else if (util::strieq_l("server_port", var_name, var_namelen)) { - type = SHRPX_LOGF_SERVER_PORT; - } else if (util::strieq_l("request_time", var_name, var_namelen)) { - type = SHRPX_LOGF_REQUEST_TIME; - } else if (util::strieq_l("pid", var_name, var_namelen)) { - type = SHRPX_LOGF_PID; - } else if (util::strieq_l("alpn", var_name, var_namelen)) { - type = SHRPX_LOGF_ALPN; - } else if (util::strieq_l("ssl_cipher", var_name, var_namelen)) { - type = SHRPX_LOGF_SSL_CIPHER; - } else if (util::strieq_l("ssl_protocol", var_name, var_namelen)) { - type = SHRPX_LOGF_SSL_PROTOCOL; - } else if (util::strieq_l("ssl_session_id", var_name, var_namelen)) { - type = SHRPX_LOGF_SSL_SESSION_ID; - } else if (util::strieq_l("ssl_session_reused", var_name, var_namelen)) { - type = SHRPX_LOGF_SSL_SESSION_REUSED; - } else { - LOG(WARN) << "Unrecognized log format variable: " - << std::string(var_name, var_namelen); - continue; + auto type = log_var_lookup_token(var_name, var_namelen); + + if (type == SHRPX_LOGF_NONE) { + if (util::istartsWith(var_name, var_namelen, "http_")) { + type = SHRPX_LOGF_HTTP; + value = var_name + str_size("http_"); + } else { + LOG(WARN) << "Unrecognized log format variable: " + << std::string(var_name, var_namelen); + continue; + } } if (literal_start < var_start) { From a2c78cfc69ecc8f6d1ecfa67c81306ea2bacce24 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Jul 2015 23:44:58 +0900 Subject: [PATCH 23/77] nghttpx: Update doc --- src/shrpx.cc | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index d458ed68..a5fa929f 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -998,7 +998,7 @@ Options: The options are categorized into several groups. Connections: - -b, --backend=,[;[:...]] + -b, --backend=(,|unix:)[;[:...]] Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX domain socket can be specified by prefixing path name @@ -1009,27 +1009,27 @@ Connections: -p is used, s are ignored. The pattern matching is closely designed to ServeMux in net/http package of Go programming language. consists - of path, host + path or just host. The path must starts - with "/". If it ends with "/", it matches to the - request path whose prefix is the path. To deal with the - request to the directory without trailing slash, pattern - which ends with "/" also matches the path if pattern == - path + "/" (e.g., pattern "/foo/" matches path "/foo"). - If it does not end with "/", it performs exact match - against the request path. If host is given, it performs - exact match against the request host. If host alone is - given, "/" is appended to it, so that it matches all - paths under the host (e.g., specifying "nghttp2.org" - equals to "nghttp2.org/"). + of path, host + path or just host. The path must start + with "/". If it ends with "/", it matches all request + path in its subtree. To deal with the request to the + directory without trailing slash, the path which ends + with "/" also matches the request path which only lacks + trailing '/' (e.g., path "/foo/" matches request path + "/foo"). If it does not end with "/", it performs exact + match against the request path. If host is given, it + performs exact match against the request host. If host + alone is given, "/" is appended to it, so that it + matches all request paths under the host (e.g., + specifying "nghttp2.org" equals to "nghttp2.org/"). - Patterns with host take precedence over path only - patterns. Then, longer patterns take precedence over + Patterns with host take precedence over patterns with + just path. Then, longer patterns take precedence over shorter ones, breaking a tie by the order of the appearance in the configuration. If is omitted, "/" is used as pattern, which - matches all paths (catch-all pattern). The catch-all - backend must be given. + matches all request paths (catch-all pattern). The + catch-all backend must be given. When doing a match, nghttpx made some normalization to pattern, request host and path. For host part, they are @@ -1047,15 +1047,18 @@ Connections: them by ":". Specifying -b'127.0.0.1,8080;nghttp2.org:www.nghttp2.org' has the same effect to specify -b'127.0.0.1,8080;nghttp2.org' - and -b'127.0.0.1,8080:www.nghttp2.org'. + and -b'127.0.0.1,8080;www.nghttp2.org'. The backend addresses sharing same are grouped - together forming load balancing group. Since ";" and - ":" are used as delimiter, must not contain - these characters. + together forming load balancing group. + + Since ";" and ":" are used as delimiter, must + not contain these characters. Since ";" has special + meaning in shell, the option value must be quoted. + Default: )" << DEFAULT_DOWNSTREAM_HOST << "," << DEFAULT_DOWNSTREAM_PORT << R"( - -f, --frontend=, + -f, --frontend=(,|unix:) Set frontend host and port. If is '*', it assumes all addresses including both IPv4 and IPv6. UNIX domain socket can be specified by prefixing path From 326ac31a23f1ecdf4a377f976eb4c9f0de93b324 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Jul 2015 23:45:50 +0900 Subject: [PATCH 24/77] nghttpx: Update doc --- src/shrpx_config.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index cc827f66..2b20fa93 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -324,6 +324,7 @@ int parse_int(T *dest, const char *opt, const char *optarg) { } namespace { +// generated by gennghttpxfun.py LogFragmentType log_var_lookup_token(const char *name, size_t namelen) { switch (namelen) { case 3: @@ -588,6 +589,7 @@ void parse_mapping(const DownstreamAddr &addr, const char *src) { } } // namespace +// generated by gennghttpxfun.py enum { SHRPX_OPTID_ACCESSLOG_FILE, SHRPX_OPTID_ACCESSLOG_FORMAT, @@ -682,6 +684,7 @@ enum { }; namespace { +// generated by gennghttpxfun.py int option_lookup_token(const char *name, size_t namelen) { switch (namelen) { case 4: From 860da8bf87e5d58379fa3c600c3edfe6f646b174 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Jul 2015 23:54:29 +0900 Subject: [PATCH 25/77] Bump up version number to 1.0.6, LT revision to 14:6:0 --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index d9f5e454..73f7bf67 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.0.6-DEV], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.0.6], [t-tujikawa@users.sourceforge.net]) AC_USE_SYSTEM_EXTENSIONS LT_PREREQ([2.2.6]) @@ -48,7 +48,7 @@ AC_CONFIG_HEADERS([config.h]) dnl See versioning rule: dnl http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html AC_SUBST(LT_CURRENT, 14) -AC_SUBST(LT_REVISION, 5) +AC_SUBST(LT_REVISION, 6) AC_SUBST(LT_AGE, 0) major=`echo $PACKAGE_VERSION |cut -d. -f1 | sed -e "s/[^0-9]//g"` From 9c8bc1218d49c4a161c508c45fd6041daa764289 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 00:01:04 +0900 Subject: [PATCH 26/77] Update man pages --- doc/h2load.1 | 2 +- doc/nghttp.1 | 2 +- doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 51 ++++++++++++++++++++++++++--------------------- doc/nghttpx.1.rst | 50 ++++++++++++++++++++++++++-------------------- 5 files changed, 59 insertions(+), 48 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index b5b60925..dd6776f8 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "July 12, 2015" "1.0.6-DEV" "nghttp2" +.TH "H2LOAD" "1" "July 14, 2015" "1.0.6" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 9695b45a..96d367e3 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "July 12, 2015" "1.0.6-DEV" "nghttp2" +.TH "NGHTTP" "1" "July 14, 2015" "1.0.6" "nghttp2" .SH NAME nghttp \- HTTP/2 experimental client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index 70a816bd..f2eb2f42 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "July 12, 2015" "1.0.6-DEV" "nghttp2" +.TH "NGHTTPD" "1" "July 14, 2015" "1.0.6" "nghttp2" .SH NAME nghttpd \- HTTP/2 experimental server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index fa4e2cde..055c38fc 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "July 12, 2015" "1.0.6-DEV" "nghttp2" +.TH "NGHTTPX" "1" "July 14, 2015" "1.0.6" "nghttp2" .SH NAME nghttpx \- HTTP/2 experimental proxy . @@ -55,7 +55,7 @@ The options are categorized into several groups. .SS Connections .INDENT 0.0 .TP -.B \-b, \-\-backend=,[;[:...]] +.B \-b, \-\-backend=(,|unix:)[;[:...]] Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX domain socket can be specified by prefixing path name @@ -66,27 +66,27 @@ is only used if request matches the pattern. If \fI\%\-s\fP or \fI\%\-p\fP is used, s are ignored. The pattern matching is closely designed to ServeMux in net/http package of Go programming language. consists -of path, host + path or just host. The path must starts -with "\fI/\fP". If it ends with "\fI/\fP", it matches to the -request path whose prefix is the path. To deal with the -request to the directory without trailing slash, pattern -which ends with "\fI/\fP" also matches the path if pattern == -path + "\fI/\fP" (e.g., pattern "\fI/foo/\fP" matches path "\fI/foo\fP"). -If it does not end with "\fI/\fP", it performs exact match -against the request path. If host is given, it performs -exact match against the request host. If host alone is -given, "\fI/\fP" is appended to it, so that it matches all -paths under the host (e.g., specifying "nghttp2.org" -equals to "nghttp2.org/"). +of path, host + path or just host. The path must start +with "\fI/\fP". If it ends with "\fI/\fP", it matches all request +path in its subtree. To deal with the request to the +directory without trailing slash, the path which ends +with "\fI/\fP" also matches the request path which only lacks +trailing \(aq\fI/\fP\(aq (e.g., path "\fI/foo/\fP" matches request path +"\fI/foo\fP"). If it does not end with "\fI/\fP", it performs exact +match against the request path. If host is given, it +performs exact match against the request host. If host +alone is given, "\fI/\fP" is appended to it, so that it +matches all request paths under the host (e.g., +specifying "nghttp2.org" equals to "nghttp2.org/"). .sp -Patterns with host take precedence over path only -patterns. Then, longer patterns take precedence over +Patterns with host take precedence over patterns with +just path. Then, longer patterns take precedence over shorter ones, breaking a tie by the order of the appearance in the configuration. .sp If is omitted, "\fI/\fP" is used as pattern, which -matches all paths (catch\-all pattern). The catch\-all -backend must be given. +matches all request paths (catch\-all pattern). The +catch\-all backend must be given. .sp When doing a match, nghttpx made some normalization to pattern, request host and path. For host part, they are @@ -104,18 +104,20 @@ The multiple s can be specified, delimiting them by ":". Specifying \fI\%\-b\fP\(aq127.0.0.1,8080;nghttp2.org:www.nghttp2.org\(aq has the same effect to specify \fI\%\-b\fP\(aq127.0.0.1,8080;nghttp2.org\(aq -and \fI\%\-b\fP\(aq127.0.0.1,8080:www.nghttp2.org\(aq. +and \fI\%\-b\fP\(aq127.0.0.1,8080;www.nghttp2.org\(aq. .sp The backend addresses sharing same are grouped -together forming load balancing group. Since ";" and -":" are used as delimiter, must not contain -these characters. +together forming load balancing group. +.sp +Since ";" and ":" are used as delimiter, must +not contain these characters. Since ";" has special +meaning in shell, the option value must be quoted. .sp Default: \fB127.0.0.1,80\fP .UNINDENT .INDENT 0.0 .TP -.B \-f, \-\-frontend=, +.B \-f, \-\-frontend=(,|unix:) Set frontend host and port. If is \(aq*\(aq, it assumes all addresses including both IPv4 and IPv6. UNIX domain socket can be specified by prefixing path @@ -692,6 +694,9 @@ $ssl_session_reused: "r" if SSL/TLS session was reused. Otherwise, "." .UNINDENT .sp +The variable can be enclosed by "{" and "}" for +disambiguation (e.g., ${remote_addr}). +.sp Default: \fB$remote_addr \- \- [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"\fP .UNINDENT .INDENT 0.0 diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index 7ff041c6..f88cdf8d 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -37,7 +37,7 @@ The options are categorized into several groups. Connections ~~~~~~~~~~~ -.. option:: -b, --backend=,[;[:...]] +.. option:: -b, --backend=(,|unix:)[;[:...]] Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX @@ -49,27 +49,27 @@ Connections :option:`-p` is used, s are ignored. The pattern matching is closely designed to ServeMux in net/http package of Go programming language. consists - of path, host + path or just host. The path must starts - with "*/*". If it ends with "*/*", it matches to the - request path whose prefix is the path. To deal with the - request to the directory without trailing slash, pattern - which ends with "*/*" also matches the path if pattern == - path + "*/*" (e.g., pattern "*/foo/*" matches path "*/foo*"). - If it does not end with "*/*", it performs exact match - against the request path. If host is given, it performs - exact match against the request host. If host alone is - given, "*/*" is appended to it, so that it matches all - paths under the host (e.g., specifying "nghttp2.org" - equals to "nghttp2.org/"). + of path, host + path or just host. The path must start + with "*/*". If it ends with "*/*", it matches all request + path in its subtree. To deal with the request to the + directory without trailing slash, the path which ends + with "*/*" also matches the request path which only lacks + trailing '*/*' (e.g., path "*/foo/*" matches request path + "*/foo*"). If it does not end with "*/*", it performs exact + match against the request path. If host is given, it + performs exact match against the request host. If host + alone is given, "*/*" is appended to it, so that it + matches all request paths under the host (e.g., + specifying "nghttp2.org" equals to "nghttp2.org/"). - Patterns with host take precedence over path only - patterns. Then, longer patterns take precedence over + Patterns with host take precedence over patterns with + just path. Then, longer patterns take precedence over shorter ones, breaking a tie by the order of the appearance in the configuration. If is omitted, "*/*" is used as pattern, which - matches all paths (catch-all pattern). The catch-all - backend must be given. + matches all request paths (catch-all pattern). The + catch-all backend must be given. When doing a match, nghttpx made some normalization to pattern, request host and path. For host part, they are @@ -87,16 +87,19 @@ Connections them by ":". Specifying :option:`-b`\'127.0.0.1,8080;nghttp2.org:www.nghttp2.org' has the same effect to specify :option:`-b`\'127.0.0.1,8080;nghttp2.org' - and :option:`-b`\'127.0.0.1,8080:www.nghttp2.org'. + and :option:`-b`\'127.0.0.1,8080;www.nghttp2.org'. The backend addresses sharing same are grouped - together forming load balancing group. Since ";" and - ":" are used as delimiter, must not contain - these characters. + together forming load balancing group. + + Since ";" and ":" are used as delimiter, must + not contain these characters. Since ";" has special + meaning in shell, the option value must be quoted. + Default: ``127.0.0.1,80`` -.. option:: -f, --frontend=, +.. option:: -f, --frontend=(,|unix:) Set frontend host and port. If is '\*', it assumes all addresses including both IPv4 and IPv6. @@ -611,6 +614,9 @@ Logging * $ssl_session_reused: "r" if SSL/TLS session was reused. Otherwise, "." + The variable can be enclosed by "{" and "}" for + disambiguation (e.g., ${remote_addr}). + Default: ``$remote_addr - - [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"`` From 1c7431293d29cb4b348f907a6a9bf16426389960 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 00:01:22 +0900 Subject: [PATCH 27/77] Update bash_completion --- doc/bash_completion/h2load | 2 +- doc/bash_completion/nghttp | 2 +- doc/bash_completion/nghttpx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/bash_completion/h2load b/doc/bash_completion/h2load index 2a4bf98d..c8438893 100644 --- a/doc/bash_completion/h2load +++ b/doc/bash_completion/h2load @@ -8,7 +8,7 @@ _h2load() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--threads --connection-window-bits --input-file --help --requests --data --verbose --version --window-bits --clients --no-tls-proto --header --max-concurrent-streams ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--threads --connection-window-bits --input-file --help --requests --data --verbose --ciphers --window-bits --clients --no-tls-proto --version --header --max-concurrent-streams ' -- "$cur" ) ) ;; *) _filedir diff --git a/doc/bash_completion/nghttp b/doc/bash_completion/nghttp index 6bb7a3e4..7d7da704 100644 --- a/doc/bash_completion/nghttp +++ b/doc/bash_completion/nghttp @@ -8,7 +8,7 @@ _nghttp() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--no-push --verbose --no-dep --get-assets --har --header-table-size --multiply --padding --hexdump --continuation --connection-window-bits --peer-max-concurrent-streams --timeout --data --no-content-length --version --color --cert --upgrade --remote-name --trailer --weight --help --key --null-out --window-bits --stat --header ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--no-push --verbose --no-dep --get-assets --har --header-table-size --multiply --padding --hexdump --max-concurrent-streams --continuation --connection-window-bits --peer-max-concurrent-streams --timeout --data --no-content-length --version --color --cert --upgrade --remote-name --trailer --weight --help --key --null-out --window-bits --stat --header ' -- "$cur" ) ) ;; *) _filedir diff --git a/doc/bash_completion/nghttpx b/doc/bash_completion/nghttpx index 8855b81a..64b6645e 100644 --- a/doc/bash_completion/nghttpx +++ b/doc/bash_completion/nghttpx @@ -8,7 +8,7 @@ _nghttpx() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--worker-read-rate --frontend-no-tls --frontend-http2-dump-response-header --backend-http1-connections-per-frontend --tls-ticket-key-file --verify-client-cacert --backend-request-buffer --backend-http2-connection-window-bits --conf --worker-write-burst --npn-list --fetch-ocsp-response-file --stream-read-timeout --accesslog-syslog --frontend-http2-read-timeout --listener-disable-timeout --frontend-http2-connection-window-bits --ciphers --strip-incoming-x-forwarded-for --daemon --backend-keep-alive-timeout --backend-http-proxy-uri --backend-http1-connections-per-host --rlimit-nofile --no-via --ocsp-update-interval --backend-write-timeout --client --http2-no-cookie-crumbling --worker-read-burst --client-proxy --http2-bridge --accesslog-format --errorlog-syslog --errorlog-file --http2-max-concurrent-streams --frontend-write-timeout --read-burst --backend-ipv4 --backend-ipv6 --backend --insecure --log-level --tls-proto-list --backend-http2-connections-per-worker --dh-param-file --worker-frontend-connections --header-field-buffer --no-server-push --no-location-rewrite --no-ocsp --backend-response-buffer --workers --frontend-http2-window-bits --no-host-rewrite --worker-write-rate --add-request-header --backend-tls-sni-field --subcert --help --frontend-frame-debug --pid-file --frontend-http2-dump-request-header --private-key-passwd-file --write-rate --altsvc --user --add-x-forwarded-for --syslog-facility --frontend-read-timeout --backlog --write-burst --backend-http2-window-bits --padding --stream-write-timeout --cacert --version --verify-client --backend-read-timeout --frontend --accesslog-file --http2-proxy --max-header-fields --backend-no-tls --client-private-key-file --client-cert-file --add-response-header --read-rate ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--worker-read-rate --frontend-no-tls --frontend-http2-dump-response-header --backend-http1-connections-per-frontend --tls-ticket-key-file --verify-client-cacert --include --backend-request-buffer --backend-http2-connection-window-bits --conf --worker-write-burst --npn-list --fetch-ocsp-response-file --stream-read-timeout --accesslog-syslog --frontend-http2-read-timeout --listener-disable-timeout --frontend-http2-connection-window-bits --ciphers --strip-incoming-x-forwarded-for --daemon --backend-keep-alive-timeout --backend-http-proxy-uri --backend-http1-connections-per-host --rlimit-nofile --no-via --ocsp-update-interval --backend-write-timeout --client --http2-no-cookie-crumbling --worker-read-burst --client-proxy --http2-bridge --accesslog-format --errorlog-syslog --errorlog-file --http2-max-concurrent-streams --frontend-write-timeout --read-burst --backend-ipv4 --backend-ipv6 --backend --insecure --log-level --tls-proto-list --backend-http2-connections-per-worker --dh-param-file --worker-frontend-connections --header-field-buffer --no-server-push --no-location-rewrite --no-ocsp --backend-response-buffer --workers --frontend-http2-window-bits --no-host-rewrite --worker-write-rate --add-request-header --backend-tls-sni-field --subcert --help --frontend-frame-debug --pid-file --frontend-http2-dump-request-header --private-key-passwd-file --write-rate --altsvc --user --add-x-forwarded-for --syslog-facility --frontend-read-timeout --backlog --write-burst --backend-http2-window-bits --padding --stream-write-timeout --cacert --version --verify-client --backend-read-timeout --frontend --accesslog-file --http2-proxy --max-header-fields --backend-no-tls --client-private-key-file --client-cert-file --add-response-header --read-rate ' -- "$cur" ) ) ;; *) _filedir From 5e966d000a76f7f0f89f0c97972c8b3397fc51b9 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 00:11:34 +0900 Subject: [PATCH 28/77] python: Fix out-of-tree builds failure --- python/setup.py.in | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/python/setup.py.in b/python/setup.py.in index 3abaddea..a2b4851d 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -22,8 +22,7 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. from setuptools import setup, Extension -from Cython.Build import cythonize - + LIBS = ['nghttp2'] setup( @@ -34,13 +33,13 @@ setup( author_email = 'tatsuhiro.t@gmail.com', url = 'https://nghttp2.org/', keywords = [], - ext_modules = cythonize([Extension("nghttp2", - ["nghttp2.pyx"], + ext_modules = [Extension("nghttp2", + ["nghttp2.c"], include_dirs=['@top_srcdir@/lib', '@top_srcdir@/lib/includes', '@top_builddir@/lib/includes'], library_dirs=['@top_builddir@/lib/.libs', '@top_builddir@'], - libraries=LIBS)]), + libraries=LIBS)], long_description='TBD' ) From 3db03a3f517747256e4d33df369a99c19d3a072a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 01:16:56 +0900 Subject: [PATCH 29/77] Exclude python bindings for `make distcheck` setuptools does now allow us to install custom location without hack. Also it does not provide uninstall feature, and `make uninstall` leaves several files we cannot easily remove (e.g., easy-install.pth). Therefore, it is better just exclude python bindings from `make distcheck`. --- Makefile.am | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile.am b/Makefile.am index 831b317d..d1a7dd16 100644 --- a/Makefile.am +++ b/Makefile.am @@ -23,6 +23,11 @@ SUBDIRS = lib third-party src examples python tests integration-tests \ doc contrib script +# Now with python setuptools, make uninstall will leave many files we +# cannot easily remove (e.g., easy-install.pth). Disable it for +# distcheck rule. +AM_DISTCHECK_CONFIGURE_FLAGS = --disable-python-bindings + ACLOCAL_AMFLAGS = -I m4 dist_doc_DATA = README.rst From a06c16c99a722c3727893243178710eb9e4a0e6b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 01:23:18 +0900 Subject: [PATCH 30/77] Bump up version number to 1.1.0 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 73f7bf67..e893f19e 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.0.6], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.1.0], [t-tujikawa@users.sourceforge.net]) AC_USE_SYSTEM_EXTENSIONS LT_PREREQ([2.2.6]) From da5138992729e4bf95e04141403541c8b352a852 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 01:25:27 +0900 Subject: [PATCH 31/77] Update man pages --- doc/h2load.1 | 2 +- doc/nghttp.1 | 2 +- doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index dd6776f8..95369ed9 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "July 14, 2015" "1.0.6" "nghttp2" +.TH "H2LOAD" "1" "July 15, 2015" "1.1.0" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 96d367e3..42e22100 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "July 14, 2015" "1.0.6" "nghttp2" +.TH "NGHTTP" "1" "July 15, 2015" "1.1.0" "nghttp2" .SH NAME nghttp \- HTTP/2 experimental client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index f2eb2f42..096b5e68 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "July 14, 2015" "1.0.6" "nghttp2" +.TH "NGHTTPD" "1" "July 15, 2015" "1.1.0" "nghttp2" .SH NAME nghttpd \- HTTP/2 experimental server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index 055c38fc..eb747b30 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "July 14, 2015" "1.0.6" "nghttp2" +.TH "NGHTTPX" "1" "July 15, 2015" "1.1.0" "nghttp2" .SH NAME nghttpx \- HTTP/2 experimental proxy . From f54745fe4c9a0628a28e15fdd01ea71d0fd8fb3a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 01:29:57 +0900 Subject: [PATCH 32/77] Bump up version number to 1.1.1-DEV --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index e893f19e..0e4f9788 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.1.0], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.1.1-DEV], [t-tujikawa@users.sourceforge.net]) AC_USE_SYSTEM_EXTENSIONS LT_PREREQ([2.2.6]) From 6d1079930191ea53cf1637ec624fecdc5fecddba Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 19:46:48 +0900 Subject: [PATCH 33/77] nghttpx: Don't pool failed HTTP/1 backend connection --- src/shrpx_http_downstream_connection.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index a69357a5..af16ba24 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -864,6 +864,8 @@ int HttpDownstreamConnection::on_connect() { DLOG(INFO, this) << "downstream connect failed"; } + downstream_->set_request_state(Downstream::CONNECT_FAIL); + return -1; } From 031fb31248edcfef4a45d10c2a015589ac2059a5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 19:47:15 +0900 Subject: [PATCH 34/77] nghttpx: Fix memory leak, and blocked stream dispatch --- src/shrpx_downstream_queue.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/shrpx_downstream_queue.cc b/src/shrpx_downstream_queue.cc index c70eb064..fa2ba517 100644 --- a/src/shrpx_downstream_queue.cc +++ b/src/shrpx_downstream_queue.cc @@ -124,17 +124,14 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream) { // Delete downstream when this function returns. auto delptr = std::unique_ptr(downstream); - if (downstream->get_dispatch_state() != Downstream::DISPATCH_ACTIVE) { - assert(downstream->get_dispatch_state() != Downstream::DISPATCH_NONE); - downstreams_.remove(downstream); - return nullptr; - } - downstreams_.remove(downstream); auto &host = make_host_key(downstream); auto &ent = find_host_entry(host); - --ent.num_active; + + if (downstream->get_dispatch_state() == Downstream::DISPATCH_ACTIVE) { + --ent.num_active; + } if (remove_host_entry_if_empty(ent, host_entries_, host)) { return nullptr; @@ -147,7 +144,10 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream) { for (auto link = ent.blocked.head; link;) { auto next = link->dlnext; if (!link->downstream) { + // If non-active (e.g., pending) Downstream got deleted, + // link->downstream is nullptr. ent.blocked.remove(link); + delete link; link = next; continue; } From e63e775fea41264f6970271d6c11b08fa25ca0bd Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 20:44:44 +0900 Subject: [PATCH 35/77] nghttpx: Simplify BlockedLink management --- src/shrpx_downstream.cc | 12 +++--------- src/shrpx_downstream.h | 2 +- src/shrpx_downstream_queue.cc | 35 ++++++++++++++++++----------------- 3 files changed, 22 insertions(+), 27 deletions(-) diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index e97e2913..54c29fcc 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -152,10 +152,6 @@ Downstream::~Downstream() { DLOG(INFO, this) << "Deleting"; } - if (blocked_link_) { - detach_blocked_link(blocked_link_); - } - // check nullptr for unittest if (upstream_) { auto loop = upstream_->get_client_handler()->get_loop(); @@ -1200,12 +1196,10 @@ void Downstream::attach_blocked_link(BlockedLink *l) { blocked_link_ = l; } -void Downstream::detach_blocked_link(BlockedLink *l) { - assert(blocked_link_); - assert(l->downstream == this); - - l->downstream = nullptr; +BlockedLink *Downstream::detach_blocked_link() { + auto link = blocked_link_; blocked_link_ = nullptr; + return link; } void Downstream::add_request_headers_sum(size_t amount) { diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index f0edeb2c..1294c0bd 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -332,7 +332,7 @@ public: void set_dispatch_state(int s); void attach_blocked_link(BlockedLink *l); - void detach_blocked_link(BlockedLink *l); + BlockedLink *detach_blocked_link(); enum { EVENT_ERROR = 0x1, diff --git a/src/shrpx_downstream_queue.cc b/src/shrpx_downstream_queue.cc index fa2ba517..0462d2a7 100644 --- a/src/shrpx_downstream_queue.cc +++ b/src/shrpx_downstream_queue.cc @@ -131,6 +131,12 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream) { if (downstream->get_dispatch_state() == Downstream::DISPATCH_ACTIVE) { --ent.num_active; + } else { + auto link = downstream->detach_blocked_link(); + if (link) { + ent.blocked.remove(link); + delete link; + } } if (remove_host_entry_if_empty(ent, host_entries_, host)) { @@ -141,24 +147,19 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream) { return nullptr; } - for (auto link = ent.blocked.head; link;) { - auto next = link->dlnext; - if (!link->downstream) { - // If non-active (e.g., pending) Downstream got deleted, - // link->downstream is nullptr. - ent.blocked.remove(link); - delete link; - link = next; - continue; - } - auto next_downstream = link->downstream; - next_downstream->detach_blocked_link(link); - ent.blocked.remove(link); - delete link; - remove_host_entry_if_empty(ent, host_entries_, host); - return next_downstream; + auto link = ent.blocked.head; + + if (!link) { + return nullptr; } - return nullptr; + + auto next_downstream = link->downstream; + next_downstream->detach_blocked_link(); + ent.blocked.remove(link); + delete link; + remove_host_entry_if_empty(ent, host_entries_, host); + + return next_downstream; } Downstream *DownstreamQueue::get_downstreams() const { From 20e63151a5ebecfc264199252aef3b2285ed1b91 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 23:18:32 +0900 Subject: [PATCH 36/77] nghttpx: Fix bug that idle timer is used after reuse --- src/shrpx_http_downstream_connection.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index af16ba24..eeff9312 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -205,6 +205,8 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { ev_set_cb(&conn_.rev, readcb); conn_.rt.repeat = get_config()->downstream_read_timeout; + // we may set read timer cb to idle_timeoutcb. Reset again. + ev_set_cb(&conn_.rt, timeoutcb); ev_timer_again(conn_.loop, &conn_.rt); // TODO we should have timeout for connection establishment ev_timer_again(conn_.loop, &conn_.wt); From fc062976a1ec700abe0a50fd66779d65ec45e3d6 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 23:31:32 +0900 Subject: [PATCH 37/77] nghttpx: Delete DownstreamConnection from Downstream explicitly --- src/shrpx_downstream.cc | 6 ++++-- src/shrpx_downstream.h | 2 -- src/shrpx_http2_downstream_connection.cc | 6 +----- src/shrpx_http_downstream_connection.cc | 8 +------- 4 files changed, 6 insertions(+), 16 deletions(-) diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 54c29fcc..87a2d082 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -162,6 +162,10 @@ Downstream::~Downstream() { ev_timer_stop(loop, &downstream_wtimer_); } + // DownstreamConnection may refer to this object. Delete it now + // explicitly. + dconn_.reset(); + if (LOG_ENABLED(INFO)) { DLOG(INFO, this) << "Deleted"; } @@ -191,8 +195,6 @@ void Downstream::detach_downstream_connection() { std::unique_ptr(dconn_.release())); } -void Downstream::release_downstream_connection() { dconn_.release(); } - DownstreamConnection *Downstream::get_downstream_connection() { return dconn_.get(); } diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 1294c0bd..3486621a 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -69,8 +69,6 @@ public: int attach_downstream_connection(std::unique_ptr dconn); void detach_downstream_connection(); - // Releases dconn_, without freeing it. - void release_downstream_connection(); DownstreamConnection *get_downstream_connection(); // Returns dconn_ and nullifies dconn_. std::unique_ptr pop_downstream_connection(); diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index d6bc433f..f4377b44 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -86,11 +86,7 @@ Http2DownstreamConnection::~Http2DownstreamConnection() { } } http2session_->remove_downstream_connection(this); - // Downstream and DownstreamConnection may be deleted - // asynchronously. - if (downstream_) { - downstream_->release_downstream_connection(); - } + if (LOG_ENABLED(INFO)) { DCLOG(INFO, this) << "Deleted"; } diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index eeff9312..c3f4b89e 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -118,13 +118,7 @@ HttpDownstreamConnection::HttpDownstreamConnection( ioctrl_(&conn_.rlimit), response_htp_{0}, group_(group), addr_idx_(0), connected_(false) {} -HttpDownstreamConnection::~HttpDownstreamConnection() { - // Downstream and DownstreamConnection may be deleted - // asynchronously. - if (downstream_) { - downstream_->release_downstream_connection(); - } -} +HttpDownstreamConnection::~HttpDownstreamConnection() {} int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { if (LOG_ENABLED(INFO)) { From ba31b990a25d38e09321a7c847ecbcede2903a46 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 23:33:44 +0900 Subject: [PATCH 38/77] nghttpx: Remove dead handling of Downstream::STREAM_CLOSED --- src/shrpx_http2_upstream.cc | 25 ------------------------- src/shrpx_spdy_upstream.cc | 25 ------------------------- 2 files changed, 50 deletions(-) diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 97ff206c..87c77d14 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -884,16 +884,6 @@ ClientHandler *Http2Upstream::get_client_handler() const { return handler_; } int Http2Upstream::downstream_read(DownstreamConnection *dconn) { auto downstream = dconn->get_downstream(); - if (downstream->get_request_state() == Downstream::STREAM_CLOSED) { - // If upstream HTTP2 stream was closed, we just close downstream, - // because there is no consumer now. Downstream connection is also - // closed in this case. - remove_downstream(downstream); - // downstream was deleted - - return 0; - } - if (downstream->get_response_state() == Downstream::MSG_RESET) { // The downstream stream was reset (canceled). In this case, // RST_STREAM to the upstream and delete downstream connection @@ -959,14 +949,6 @@ int Http2Upstream::downstream_eof(DownstreamConnection *dconn) { if (LOG_ENABLED(INFO)) { DCLOG(INFO, dconn) << "EOF. stream_id=" << downstream->get_stream_id(); } - if (downstream->get_request_state() == Downstream::STREAM_CLOSED) { - // If stream was closed already, we don't need to send reply at - // the first place. We can delete downstream. - remove_downstream(downstream); - // downstream was deleted - - return 0; - } // Delete downstream connection. If we don't delete it here, it will // be pooled in on_stream_close_callback. @@ -1012,13 +994,6 @@ int Http2Upstream::downstream_error(DownstreamConnection *dconn, int events) { } } - if (downstream->get_request_state() == Downstream::STREAM_CLOSED) { - remove_downstream(downstream); - // downstream was deleted - - return 0; - } - // Delete downstream connection. If we don't delete it here, it will // be pooled in on_stream_close_callback. downstream->pop_downstream_connection(); diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 52f267da..5f124296 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -560,16 +560,6 @@ ClientHandler *SpdyUpstream::get_client_handler() const { return handler_; } int SpdyUpstream::downstream_read(DownstreamConnection *dconn) { auto downstream = dconn->get_downstream(); - if (downstream->get_request_state() == Downstream::STREAM_CLOSED) { - // If upstream SPDY stream was closed, we just close downstream, - // because there is no consumer now. Downstream connection is also - // closed in this case. - remove_downstream(downstream); - // downstrea was deleted - - return 0; - } - if (downstream->get_response_state() == Downstream::MSG_RESET) { // The downstream stream was reset (canceled). In this case, // RST_STREAM to the upstream and delete downstream connection @@ -633,14 +623,6 @@ int SpdyUpstream::downstream_eof(DownstreamConnection *dconn) { if (LOG_ENABLED(INFO)) { DCLOG(INFO, dconn) << "EOF. stream_id=" << downstream->get_stream_id(); } - if (downstream->get_request_state() == Downstream::STREAM_CLOSED) { - // If stream was closed already, we don't need to send reply at - // the first place. We can delete downstream. - remove_downstream(downstream); - // downstream was deleted - - return 0; - } // Delete downstream connection. If we don't delete it here, it will // be pooled in on_stream_close_callback. @@ -686,13 +668,6 @@ int SpdyUpstream::downstream_error(DownstreamConnection *dconn, int events) { } } - if (downstream->get_request_state() == Downstream::STREAM_CLOSED) { - remove_downstream(downstream); - // downstream was deleted - - return 0; - } - // Delete downstream connection. If we don't delete it here, it will // be pooled in on_stream_close_callback. downstream->pop_downstream_connection(); From 7a3012cc1b09b0bfed7e92774d017648881457f0 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 23:44:24 +0900 Subject: [PATCH 39/77] src: Rename timegm as nghttp2_timegm --- src/timegm.c | 2 +- src/timegm.h | 2 +- src/util.cc | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/timegm.c b/src/timegm.c index b80e0789..62b14430 100644 --- a/src/timegm.c +++ b/src/timegm.c @@ -34,7 +34,7 @@ static int count_leap_year(int y) { } /* Based on the algorithm of Python 2.7 calendar.timegm. */ -time_t timegm(struct tm *tm) { +time_t nghttp2_timegm(struct tm *tm) { int days; int num_leap_year; int64_t t; diff --git a/src/timegm.h b/src/timegm.h index 2624657a..4168fdd1 100644 --- a/src/timegm.h +++ b/src/timegm.h @@ -37,7 +37,7 @@ extern "C" { #include #endif // HAVE_TIME_H -time_t timegm(struct tm *tm); +time_t nghttp2_timegm(struct tm *tm); #ifdef __cplusplus } diff --git a/src/util.cc b/src/util.cc index 2d7c7734..191bd13f 100644 --- a/src/util.cc +++ b/src/util.cc @@ -278,7 +278,7 @@ std::string common_log_date(time_t t) { #ifdef HAVE_STRUCT_TM_TM_GMTOFF auto gmtoff = tms.tm_gmtoff; #else // !HAVE_STRUCT_TM_TM_GMTOFF - auto gmtoff = timegm(&tms) - t; + auto gmtoff = nghttp2_timegm(&tms) - t; #endif // !HAVE_STRUCT_TM_TM_GMTOFF if (gmtoff >= 0) { *p++ = '+'; @@ -326,7 +326,7 @@ std::string iso8601_date(int64_t ms) { #ifdef HAVE_STRUCT_TM_TM_GMTOFF auto gmtoff = tms.tm_gmtoff; #else // !HAVE_STRUCT_TM_TM_GMTOFF - auto gmtoff = timegm(&tms) - sec; + auto gmtoff = nghttp2_timegm(&tms) - sec; #endif // !HAVE_STRUCT_TM_TM_GMTOFF if (gmtoff == 0) { *p++ = 'Z'; @@ -354,7 +354,7 @@ time_t parse_http_date(const std::string &s) { if (r == 0) { return 0; } - return timegm(&tm); + return nghttp2_timegm(&tm); } namespace { From f821f6bd80437d0a79f9464e99e25e87fa84518b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 23:46:22 +0900 Subject: [PATCH 40/77] Bump up version number to 1.1.1 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 0e4f9788..458438fe 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.1.1-DEV], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.1.1], [t-tujikawa@users.sourceforge.net]) AC_USE_SYSTEM_EXTENSIONS LT_PREREQ([2.2.6]) From 3396c71c3c0c7da46f61a1f1fa38c77a6e08ac37 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 23:48:41 +0900 Subject: [PATCH 41/77] Update man pages --- doc/h2load.1 | 2 +- doc/nghttp.1 | 2 +- doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index 95369ed9..d3f6b7a1 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "July 15, 2015" "1.1.0" "nghttp2" +.TH "H2LOAD" "1" "July 15, 2015" "1.1.1" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 42e22100..b1ee0180 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "July 15, 2015" "1.1.0" "nghttp2" +.TH "NGHTTP" "1" "July 15, 2015" "1.1.1" "nghttp2" .SH NAME nghttp \- HTTP/2 experimental client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index 096b5e68..ac31ae6b 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "July 15, 2015" "1.1.0" "nghttp2" +.TH "NGHTTPD" "1" "July 15, 2015" "1.1.1" "nghttp2" .SH NAME nghttpd \- HTTP/2 experimental server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index eb747b30..b7e2041d 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "July 15, 2015" "1.1.0" "nghttp2" +.TH "NGHTTPX" "1" "July 15, 2015" "1.1.1" "nghttp2" .SH NAME nghttpx \- HTTP/2 experimental proxy . From 2bac00ea5fe5cec6d1188e99346312b7866d4254 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 16 Jul 2015 00:01:26 +0900 Subject: [PATCH 42/77] nghttpx: Add additional assert just in case --- src/shrpx_downstream_queue.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/shrpx_downstream_queue.cc b/src/shrpx_downstream_queue.cc index 0462d2a7..28815fb4 100644 --- a/src/shrpx_downstream_queue.cc +++ b/src/shrpx_downstream_queue.cc @@ -154,7 +154,8 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream) { } auto next_downstream = link->downstream; - next_downstream->detach_blocked_link(); + auto link2 = next_downstream->detach_blocked_link(); + assert(link2 == link); ent.blocked.remove(link); delete link; remove_host_entry_if_empty(ent, host_entries_, host); From 8585599b4b252135304c9409892211cbe63cb7d1 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 16 Jul 2015 00:03:34 +0900 Subject: [PATCH 43/77] nghttpx: Add doc --- src/shrpx_downstream_queue.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/shrpx_downstream_queue.cc b/src/shrpx_downstream_queue.cc index 28815fb4..d1430e61 100644 --- a/src/shrpx_downstream_queue.cc +++ b/src/shrpx_downstream_queue.cc @@ -132,6 +132,7 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream) { if (downstream->get_dispatch_state() == Downstream::DISPATCH_ACTIVE) { --ent.num_active; } else { + // For those downstreams deleted while in blocked state auto link = downstream->detach_blocked_link(); if (link) { ent.blocked.remove(link); From 1ce3a000a1843a5266851b7b58a1856fd10be37a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 16 Jul 2015 00:36:10 +0900 Subject: [PATCH 44/77] Bump up version number to 1.1.2-DEV --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 458438fe..930c0f7c 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.1.1], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.1.2-DEV], [t-tujikawa@users.sourceforge.net]) AC_USE_SYSTEM_EXTENSIONS LT_PREREQ([2.2.6]) From c470ac7b0021d3cae80ef1c5b6460a108f2e5bdb Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 16 Jul 2015 14:01:18 +0900 Subject: [PATCH 45/77] asio: Fix missing nghttp2_timegm --- src/Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Makefile.am b/src/Makefile.am index 8dc53570..ff800463 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -176,6 +176,7 @@ lib_LTLIBRARIES = libnghttp2_asio.la libnghttp2_asio_la_SOURCES = \ util.cc util.h http2.cc http2.h \ ssl.cc ssl.h \ + timegm.c timegm.h \ asio_common.cc asio_common.h \ asio_io_service_pool.cc asio_io_service_pool.h \ asio_server_http2.cc \ From a23ab00686108eff07579bb3602bdb66452fe82c Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 18 Jul 2015 00:19:47 +0900 Subject: [PATCH 46/77] Allow custom install location for python bindings --- python/Makefile.am | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/Makefile.am b/python/Makefile.am index 7a352396..46a7ef22 100644 --- a/python/Makefile.am +++ b/python/Makefile.am @@ -33,11 +33,12 @@ all-local: nghttp2.c $(PYTHON) setup.py build install-exec-local: - $(PYTHON) setup.py install --prefix=$(DESTDIR)$(prefix) + test -d $(pyexecdir) || mkdir -p $(pyexecdir) + PYTHONUSERBASE=$(DESTDIR)/$(prefix) $(PYTHON) setup.py install --user uninstall-local: rm -f $(DESTDIR)$(libdir)/python*/site-packages/nghttp2.so - rm -f $(DESTDIR)$(libdir)/python*/site-packages/python_nghttp2-*.egg-info + rm -f $(DESTDIR)$(libdir)/python*/site-packages/python_nghttp2-*.egg clean-local: $(PYTHON) setup.py clean --all From fa872a6b48a9846d9e0c599f7be5be2c95e978b4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 18 Jul 2015 00:21:29 +0900 Subject: [PATCH 47/77] Bump up version number to 1.1.2 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 930c0f7c..d606752f 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.1.2-DEV], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.1.2], [t-tujikawa@users.sourceforge.net]) AC_USE_SYSTEM_EXTENSIONS LT_PREREQ([2.2.6]) From 03034c00812eb024f0654407ef138038b455bd3a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 18 Jul 2015 00:23:53 +0900 Subject: [PATCH 48/77] Update man pages --- doc/h2load.1 | 2 +- doc/nghttp.1 | 2 +- doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index d3f6b7a1..9a118b75 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "July 15, 2015" "1.1.1" "nghttp2" +.TH "H2LOAD" "1" "July 18, 2015" "1.1.2" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index b1ee0180..67d88a1f 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "July 15, 2015" "1.1.1" "nghttp2" +.TH "NGHTTP" "1" "July 18, 2015" "1.1.2" "nghttp2" .SH NAME nghttp \- HTTP/2 experimental client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index ac31ae6b..4152da97 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "July 15, 2015" "1.1.1" "nghttp2" +.TH "NGHTTPD" "1" "July 18, 2015" "1.1.2" "nghttp2" .SH NAME nghttpd \- HTTP/2 experimental server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index b7e2041d..8daf5d47 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "July 15, 2015" "1.1.1" "nghttp2" +.TH "NGHTTPX" "1" "July 18, 2015" "1.1.2" "nghttp2" .SH NAME nghttpx \- HTTP/2 experimental proxy . From ac301537ef3784d1f0246b6ddfaedb5f8ce59dca Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 18 Jul 2015 00:31:43 +0900 Subject: [PATCH 49/77] Bump up version number to 1.1.3-DEV --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index d606752f..a9d04ad5 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.1.2], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.1.3-DEV], [t-tujikawa@users.sourceforge.net]) AC_USE_SYSTEM_EXTENSIONS LT_PREREQ([2.2.6]) From e8167ceea71595a66a871e4091aeb505c415e0f8 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 18 Jul 2015 01:49:20 +0900 Subject: [PATCH 50/77] nghttpx: Add AES-256-CBC encryption for TLS session ticket --- gennghttpxfun.py | 1 + src/shrpx-unittest.cc | 2 + src/shrpx.cc | 50 ++++++++++++++++++++---- src/shrpx_config.cc | 83 +++++++++++++++++++++++++++++++++------- src/shrpx_config.h | 24 +++++++++--- src/shrpx_config_test.cc | 54 ++++++++++++++++++++++---- src/shrpx_config_test.h | 1 + src/shrpx_ssl.cc | 16 ++++---- src/util.h | 4 ++ 9 files changed, 193 insertions(+), 42 deletions(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 6412ae2b..9bf1c012 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -91,6 +91,7 @@ OPTIONS = [ "header-field-buffer", "max-header-fields", "include", + "tls-ticket-cipher", "conf", ] diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index 83695204..aea0eb33 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -122,6 +122,8 @@ int main(int argc, char *argv[]) { shrpx::test_shrpx_config_parse_log_format) || !CU_add_test(pSuite, "config_read_tls_ticket_key_file", shrpx::test_shrpx_config_read_tls_ticket_key_file) || + !CU_add_test(pSuite, "config_read_tls_ticket_key_file_aes_256", + shrpx::test_shrpx_config_read_tls_ticket_key_file_aes_256) || !CU_add_test(pSuite, "config_match_downstream_addr_group", shrpx::test_shrpx_config_match_downstream_addr_group) || !CU_add_test(pSuite, "util_streq", shrpx::test_util_streq) || diff --git a/src/shrpx.cc b/src/shrpx.cc index a5fa929f..d3b1c9fb 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -627,8 +627,21 @@ void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) { ticket_keys->keys.resize(1); } - if (RAND_bytes(reinterpret_cast(&ticket_keys->keys[0]), - sizeof(ticket_keys->keys[0])) == 0) { + auto &new_key = ticket_keys->keys[0]; + new_key.cipher = get_config()->tls_ticket_cipher; + new_key.hmac = EVP_sha256(); + new_key.hmac_keylen = EVP_MD_size(new_key.hmac); + + assert(EVP_CIPHER_key_length(new_key.cipher) <= sizeof(new_key.data.enc_key)); + assert(new_key.hmac_keylen <= sizeof(new_key.data.hmac_key)); + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "enc_keylen=" << EVP_CIPHER_key_length(new_key.cipher) + << ", hmac_keylen=" << new_key.hmac_keylen; + } + + if (RAND_bytes(reinterpret_cast(&new_key.data), + sizeof(new_key.data)) == 0) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "failed to renew ticket key"; } @@ -638,7 +651,7 @@ void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "ticket keys generation done"; for (auto &key : ticket_keys->keys) { - LOG(INFO) << "name: " << util::format_hex(key.name, sizeof(key.name)); + LOG(INFO) << "name: " << util::format_hex(key.data.name); } } @@ -709,8 +722,17 @@ int event_loop() { if (!get_config()->upstream_no_tls) { bool auto_tls_ticket_key = true; if (!get_config()->tls_ticket_key_files.empty()) { - auto ticket_keys = - read_tls_ticket_key_file(get_config()->tls_ticket_key_files); + if (!get_config()->tls_ticket_cipher_given) { + LOG(WARN) << "It is strongly recommended to specify " + "--tls-ticket-cipher=aes-128-cbc (or " + "tls-ticket-cipher=aes-128-cbc in configuration file) " + "when --tls-ticket-key-file is used for the smooth " + "transition when the default value of --tls-ticket-cipher " + "becomes aes-256-cbc"; + } + auto ticket_keys = read_tls_ticket_key_file( + get_config()->tls_ticket_key_files, get_config()->tls_ticket_cipher, + EVP_sha256()); if (!ticket_keys) { LOG(WARN) << "Use internal session ticket key generator"; } else { @@ -967,6 +989,8 @@ void fill_default_config() { mod_config()->header_field_buffer = 64_k; mod_config()->max_header_fields = 100; mod_config()->downstream_addr_group_catch_all = 0; + mod_config()->tls_ticket_cipher = EVP_aes_128_cbc(); + mod_config()->tls_ticket_cipher_given = false; } } // namespace @@ -1278,8 +1302,11 @@ SSL/TLS: are treated as a part of protocol string. Default: )" << DEFAULT_TLS_PROTO_LIST << R"( --tls-ticket-key-file= - Path to file that contains 48 bytes random data to - construct TLS session ticket parameters. This options + Path to file that contains random data to construct TLS + session ticket parameters. If aes-128-cbc is given in + --tls-ticket-cipher, the file must contain exactly 48 + bytes. If aes-256-cbc is given in --tls-ticket-cipher, + the file must contain exactly 80 bytes. This options can be used repeatedly to specify multiple ticket parameters. If several files are given, only the first key is used to encrypt TLS session tickets. Other keys @@ -1294,6 +1321,10 @@ SSL/TLS: while opening or reading a file, key is generated automatically and renewed every 12hrs. At most 2 keys are stored in memory. + --tls-ticket-cipher= + Specify cipher to encrypt TLS session ticket. Specify + either aes-128-cbc or aes-256-cbc. By default, + aes-128-cbc is used. --fetch-ocsp-response-file= Path to fetch-ocsp-response script file. It should be absolute path. @@ -1660,6 +1691,7 @@ int main(int argc, char **argv) { {SHRPX_OPT_MAX_HEADER_FIELDS, required_argument, &flag, 81}, {SHRPX_OPT_ADD_REQUEST_HEADER, required_argument, &flag, 82}, {SHRPX_OPT_INCLUDE, required_argument, &flag, 83}, + {SHRPX_OPT_TLS_TICKET_CIPHER, required_argument, &flag, 84}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2026,6 +2058,10 @@ int main(int argc, char **argv) { // --include cmdcfgs.emplace_back(SHRPX_OPT_INCLUDE, optarg); break; + case 84: + // --tls-ticket-cipher + cmdcfgs.emplace_back(SHRPX_OPT_TLS_TICKET_CIPHER, optarg); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 2b20fa93..5486fda8 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -144,36 +144,72 @@ bool is_secure(const char *filename) { } // namespace std::unique_ptr -read_tls_ticket_key_file(const std::vector &files) { +read_tls_ticket_key_file(const std::vector &files, + const EVP_CIPHER *cipher, const EVP_MD *hmac) { auto ticket_keys = make_unique(); auto &keys = ticket_keys->keys; keys.resize(files.size()); + auto enc_keylen = EVP_CIPHER_key_length(cipher); + auto hmac_keylen = EVP_MD_size(hmac); + if (cipher == EVP_aes_128_cbc()) { + // backward compatibility, as a legacy of using same file format + // with nginx and apache. + hmac_keylen = 16; + } + auto expectedlen = sizeof(keys[0].data.name) + enc_keylen + hmac_keylen; + char buf[256]; + assert(sizeof(buf) >= expectedlen); + size_t i = 0; for (auto &file : files) { + struct stat fst {}; + + if (stat(file.c_str(), &fst) == -1) { + auto error = errno; + LOG(ERROR) << "tls-ticket-key-file: could not stat file " << file + << ", errno=" << error; + return nullptr; + } + + if (fst.st_size != expectedlen) { + LOG(ERROR) << "tls-ticket-key-file: the expected file size is " + << expectedlen << ", the actual file size is " << fst.st_size; + return nullptr; + } + std::ifstream f(file.c_str()); if (!f) { LOG(ERROR) << "tls-ticket-key-file: could not open file " << file; return nullptr; } - char buf[48]; - f.read(buf, sizeof(buf)); - if (f.gcount() != sizeof(buf)) { - LOG(ERROR) << "tls-ticket-key-file: want to read 48 bytes but read " - << f.gcount() << " bytes from " << file; + + f.read(buf, expectedlen); + if (f.gcount() != expectedlen) { + LOG(ERROR) << "tls-ticket-key-file: want to read " << expectedlen + << " bytes but only read " << f.gcount() << " bytes from " + << file; return nullptr; } auto &key = keys[i++]; - auto p = buf; - memcpy(key.name, p, sizeof(key.name)); - p += sizeof(key.name); - memcpy(key.aes_key, p, sizeof(key.aes_key)); - p += sizeof(key.aes_key); - memcpy(key.hmac_key, p, sizeof(key.hmac_key)); + key.cipher = cipher; + key.hmac = hmac; + key.hmac_keylen = hmac_keylen; if (LOG_ENABLED(INFO)) { - LOG(INFO) << "session ticket key: " << util::format_hex(key.name, - sizeof(key.name)); + LOG(INFO) << "enc_keylen=" << enc_keylen + << ", hmac_keylen=" << key.hmac_keylen; + } + + auto p = buf; + memcpy(key.data.name, p, sizeof(key.data.name)); + p += sizeof(key.data.name); + memcpy(key.data.enc_key, p, enc_keylen); + p += enc_keylen; + memcpy(key.data.hmac_key, p, hmac_keylen); + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "session ticket key: " << util::format_hex(key.data.name); } } return ticket_keys; @@ -668,6 +704,7 @@ enum { SHRPX_OPTID_SUBCERT, SHRPX_OPTID_SYSLOG_FACILITY, SHRPX_OPTID_TLS_PROTO_LIST, + SHRPX_OPTID_TLS_TICKET_CIPHER, SHRPX_OPTID_TLS_TICKET_KEY_FILE, SHRPX_OPTID_USER, SHRPX_OPTID_VERIFY_CLIENT, @@ -959,6 +996,11 @@ int option_lookup_token(const char *name, size_t namelen) { return SHRPX_OPTID_WORKER_WRITE_RATE; } break; + case 'r': + if (util::strieq_l("tls-ticket-ciphe", name, 16)) { + return SHRPX_OPTID_TLS_TICKET_CIPHER; + } + break; case 's': if (util::strieq_l("max-header-field", name, 16)) { return SHRPX_OPTID_MAX_HEADER_FIELDS; @@ -1805,6 +1847,19 @@ int parse_config(const char *opt, const char *optarg, return 0; } + case SHRPX_OPTID_TLS_TICKET_CIPHER: + if (util::strieq(optarg, "aes-128-cbc")) { + mod_config()->tls_ticket_cipher = EVP_aes_128_cbc(); + } else if (util::strieq(optarg, "aes-256-cbc")) { + mod_config()->tls_ticket_cipher = EVP_aes_256_cbc(); + } else { + LOG(ERROR) << opt + << ": unsupported cipher for ticket encryption: " << optarg; + return -1; + } + mod_config()->tls_ticket_cipher_given = true; + + return 0; case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 87af15ba..1fee15f4 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -171,6 +171,7 @@ constexpr char SHRPX_OPT_NO_OCSP[] = "no-ocsp"; constexpr char SHRPX_OPT_HEADER_FIELD_BUFFER[] = "header-field-buffer"; constexpr char SHRPX_OPT_MAX_HEADER_FIELDS[] = "max-header-fields"; constexpr char SHRPX_OPT_INCLUDE[] = "include"; +constexpr char SHRPX_OPT_TLS_TICKET_CIPHER[] = "tls-ticket-cipher"; union sockaddr_union { sockaddr_storage storage; @@ -224,9 +225,17 @@ struct DownstreamAddrGroup { }; struct TicketKey { - uint8_t name[16]; - uint8_t aes_key[16]; - uint8_t hmac_key[16]; + const EVP_CIPHER *cipher; + const EVP_MD *hmac; + size_t hmac_keylen; + struct { + // name of this ticket configuration + uint8_t name[16]; + // encryption key for |cipher| + uint8_t enc_key[32]; + // hmac key for |hmac| + uint8_t hmac_key[32]; + } data; }; struct TicketKeys { @@ -300,6 +309,7 @@ struct Config { nghttp2_session_callbacks *http2_downstream_callbacks; nghttp2_option *http2_option; nghttp2_option *http2_client_option; + const EVP_CIPHER *tls_ticket_cipher; char **argv; char *cwd; size_t num_worker; @@ -376,6 +386,8 @@ struct Config { // true if host contains UNIX domain socket path bool host_unix; bool no_ocsp; + // true if --tls-ticket-cipher is used + bool tls_ticket_cipher_given; }; const Config *get_config(); @@ -447,10 +459,12 @@ int int_syslog_facility(const char *strfacility); FILE *open_file_for_write(const char *filename); // Reads TLS ticket key file in |files| and returns TicketKey which -// stores read key data. This function returns TicketKey if it +// stores read key data. The given |cipher| and |hmac| determine the +// expected file size. This function returns TicketKey if it // succeeds, or nullptr. std::unique_ptr -read_tls_ticket_key_file(const std::vector &files); +read_tls_ticket_key_file(const std::vector &files, + const EVP_CIPHER *cipher, const EVP_MD *hmac); // Selects group based on request's |hostport| and |path|. |hostport| // is the value taken from :authority or host header field, and may diff --git a/src/shrpx_config_test.cc b/src/shrpx_config_test.cc index c5116294..d91a706b 100644 --- a/src/shrpx_config_test.cc +++ b/src/shrpx_config_test.cc @@ -190,24 +190,62 @@ void test_shrpx_config_read_tls_ticket_key_file(void) { close(fd1); close(fd2); - auto ticket_keys = read_tls_ticket_key_file({file1, file2}); + auto ticket_keys = + read_tls_ticket_key_file({file1, file2}, EVP_aes_128_cbc(), EVP_sha256()); unlink(file1); unlink(file2); CU_ASSERT(ticket_keys.get() != nullptr); CU_ASSERT(2 == ticket_keys->keys.size()); auto key = &ticket_keys->keys[0]; - CU_ASSERT(0 == memcmp("0..............1", key->name, sizeof(key->name))); CU_ASSERT(0 == - memcmp("2..............3", key->aes_key, sizeof(key->aes_key))); - CU_ASSERT(0 == - memcmp("4..............5", key->hmac_key, sizeof(key->hmac_key))); + memcmp("0..............1", key->data.name, sizeof(key->data.name))); + CU_ASSERT(0 == memcmp("2..............3", key->data.enc_key, 16)); + CU_ASSERT(0 == memcmp("4..............5", key->data.hmac_key, 16)); key = &ticket_keys->keys[1]; - CU_ASSERT(0 == memcmp("6..............7", key->name, sizeof(key->name))); CU_ASSERT(0 == - memcmp("8..............9", key->aes_key, sizeof(key->aes_key))); + memcmp("6..............7", key->data.name, sizeof(key->data.name))); + CU_ASSERT(0 == memcmp("8..............9", key->data.enc_key, 16)); + CU_ASSERT(0 == memcmp("a..............b", key->data.hmac_key, 16)); +} + +void test_shrpx_config_read_tls_ticket_key_file_aes_256(void) { + char file1[] = "/tmp/nghttpx-unittest.XXXXXX"; + auto fd1 = mkstemp(file1); + assert(fd1 != -1); + assert(80 == write(fd1, "0..............12..............................34..." + "...........................5", + 80)); + char file2[] = "/tmp/nghttpx-unittest.XXXXXX"; + auto fd2 = mkstemp(file2); + assert(fd2 != -1); + assert(80 == write(fd2, "6..............78..............................9a..." + "...........................b", + 80)); + + close(fd1); + close(fd2); + auto ticket_keys = + read_tls_ticket_key_file({file1, file2}, EVP_aes_256_cbc(), EVP_sha256()); + unlink(file1); + unlink(file2); + CU_ASSERT(ticket_keys.get() != nullptr); + CU_ASSERT(2 == ticket_keys->keys.size()); + auto key = &ticket_keys->keys[0]; CU_ASSERT(0 == - memcmp("a..............b", key->hmac_key, sizeof(key->hmac_key))); + memcmp("0..............1", key->data.name, sizeof(key->data.name))); + CU_ASSERT(0 == + memcmp("2..............................3", key->data.enc_key, 32)); + CU_ASSERT(0 == + memcmp("4..............................5", key->data.hmac_key, 32)); + + key = &ticket_keys->keys[1]; + CU_ASSERT(0 == + memcmp("6..............7", key->data.name, sizeof(key->data.name))); + CU_ASSERT(0 == + memcmp("8..............................9", key->data.enc_key, 32)); + CU_ASSERT(0 == + memcmp("a..............................b", key->data.hmac_key, 32)); } void test_shrpx_config_match_downstream_addr_group(void) { diff --git a/src/shrpx_config_test.h b/src/shrpx_config_test.h index 5f9c170a..5db97392 100644 --- a/src/shrpx_config_test.h +++ b/src/shrpx_config_test.h @@ -35,6 +35,7 @@ void test_shrpx_config_parse_config_str_list(void); void test_shrpx_config_parse_header(void); void test_shrpx_config_parse_log_format(void); void test_shrpx_config_read_tls_ticket_key_file(void); +void test_shrpx_config_read_tls_ticket_key_file_aes_256(void); void test_shrpx_config_match_downstream_addr_group(void); } // namespace shrpx diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 9f95fa2f..121f4ce3 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -213,21 +213,21 @@ int ticket_key_cb(SSL *ssl, unsigned char *key_name, unsigned char *iv, if (LOG_ENABLED(INFO)) { CLOG(INFO, handler) << "encrypt session ticket key: " - << util::format_hex(key.name, 16); + << util::format_hex(key.data.name); } - memcpy(key_name, key.name, sizeof(key.name)); + memcpy(key_name, key.data.name, sizeof(key.data.name)); - EVP_EncryptInit_ex(ctx, EVP_aes_128_cbc(), nullptr, key.aes_key, iv); - HMAC_Init_ex(hctx, key.hmac_key, sizeof(key.hmac_key), EVP_sha256(), - nullptr); + EVP_EncryptInit_ex(ctx, get_config()->tls_ticket_cipher, nullptr, + key.data.enc_key, iv); + HMAC_Init_ex(hctx, key.data.hmac_key, key.hmac_keylen, key.hmac, nullptr); return 1; } size_t i; for (i = 0; i < keys.size(); ++i) { auto &key = keys[0]; - if (memcmp(key.name, key_name, sizeof(key.name)) == 0) { + if (memcmp(key_name, key.data.name, sizeof(key.data.name)) == 0) { break; } } @@ -246,8 +246,8 @@ int ticket_key_cb(SSL *ssl, unsigned char *key_name, unsigned char *iv, } auto &key = keys[i]; - HMAC_Init_ex(hctx, key.hmac_key, sizeof(key.hmac_key), EVP_sha256(), nullptr); - EVP_DecryptInit_ex(ctx, EVP_aes_128_cbc(), nullptr, key.aes_key, iv); + HMAC_Init_ex(hctx, key.data.hmac_key, key.hmac_keylen, key.hmac, nullptr); + EVP_DecryptInit_ex(ctx, key.cipher, nullptr, key.data.enc_key, iv); return i == 0 ? 1 : 2; } diff --git a/src/util.h b/src/util.h index d43d3c76..6c04fef0 100644 --- a/src/util.h +++ b/src/util.h @@ -212,6 +212,10 @@ std::string quote_string(const std::string &target); std::string format_hex(const unsigned char *s, size_t len); +template std::string format_hex(const unsigned char (&s)[N]) { + return format_hex(s, N); +} + std::string http_date(time_t t); // Returns given time |t| from epoch in Common Log format (e.g., From f2a1c8ee60ffd1e67e4287ea0d12a72cc7cd92f2 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 19 Jul 2015 16:11:42 +0900 Subject: [PATCH 51/77] Partially revert a23ab00686108eff07579bb3602bdb66452fe82c The workaround introduced in a23ab00686108eff07579bb3602bdb66452fe82c may cause a problem in certain platform (e.g., MacPorts, see GH-303) --- python/Makefile.am | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/Makefile.am b/python/Makefile.am index 46a7ef22..43aa5491 100644 --- a/python/Makefile.am +++ b/python/Makefile.am @@ -33,8 +33,7 @@ all-local: nghttp2.c $(PYTHON) setup.py build install-exec-local: - test -d $(pyexecdir) || mkdir -p $(pyexecdir) - PYTHONUSERBASE=$(DESTDIR)/$(prefix) $(PYTHON) setup.py install --user + $(PYTHON) setup.py install --prefix=$(DESTDIR)$(prefix) uninstall-local: rm -f $(DESTDIR)$(libdir)/python*/site-packages/nghttp2.so From 5465cdf4a40099f6abfba6ae2b2f379f6f9730cc Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 19 Jul 2015 17:14:25 +0900 Subject: [PATCH 52/77] src: Don't use struct tm.tm_yday from strptime --- src/timegm.c | 33 +++++++++++++++++++++++++++++++++ src/timegm.h | 5 +++++ src/util.cc | 2 +- 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/timegm.c b/src/timegm.c index 62b14430..e52fe74a 100644 --- a/src/timegm.c +++ b/src/timegm.c @@ -53,3 +53,36 @@ time_t nghttp2_timegm(struct tm *tm) { return (time_t)t; } + +/* Returns nonzero if the |y| is the leap year. The |y| is the year, + including century (e.g., 2012) */ +static int is_leap_year(int y) { + return y % 4 == 0 && (y % 100 != 0 || y % 400 == 0); +} + +/* The number of days before ith month begins */ +static int daysum[] = {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334}; + +time_t nghttp2_timegm_without_yday(struct tm *tm) { + int days; + int num_leap_year; + int64_t t; + if (tm->tm_mon > 11) { + return -1; + } + num_leap_year = count_leap_year(tm->tm_year + 1900) - count_leap_year(1970); + days = (tm->tm_year - 70) * 365 + num_leap_year + daysum[tm->tm_mon] + + tm->tm_mday - 1; + if (tm->tm_mon >= 2 && is_leap_year(tm->tm_year + 1900)) { + ++days; + } + t = ((int64_t)days * 24 + tm->tm_hour) * 3600 + tm->tm_min * 60 + tm->tm_sec; + +#if SIZEOF_TIME_T == 4 + if (t < INT32_MIN || t > INT32_MAX) { + return -1; + } +#endif /* SIZEOF_TIME_T == 4 */ + + return t; +} diff --git a/src/timegm.h b/src/timegm.h index 4168fdd1..4383fda1 100644 --- a/src/timegm.h +++ b/src/timegm.h @@ -39,6 +39,11 @@ extern "C" { time_t nghttp2_timegm(struct tm *tm); +/* Just like nghttp2_timegm, but without using tm->tm_yday. This is + useful if we use tm from strptime, since some platforms do not + calculate tm_yday with that call. */ +time_t nghttp2_timegm_without_yday(struct tm *tm); + #ifdef __cplusplus } #endif /* __cplusplus */ diff --git a/src/util.cc b/src/util.cc index 191bd13f..9586ab77 100644 --- a/src/util.cc +++ b/src/util.cc @@ -354,7 +354,7 @@ time_t parse_http_date(const std::string &s) { if (r == 0) { return 0; } - return nghttp2_timegm(&tm); + return nghttp2_timegm_without_yday(&tm); } namespace { From 006ac7b1b0415016d6634a207a6c5b2ea731f67b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 19 Jul 2015 17:27:38 +0900 Subject: [PATCH 53/77] h2load: Fix HTML formatting --- src/h2load.cc | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/h2load.cc b/src/h2load.cc index b1f5a672..f9519d34 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -1063,34 +1063,32 @@ Options: Available protocol: )"; #endif // !HAVE_SPDYLAY out << NGHTTP2_CLEARTEXT_PROTO_VERSION_ID << R"( - Default: )" - << NGHTTP2_CLEARTEXT_PROTO_VERSION_ID << R"( + Default: )" << NGHTTP2_CLEARTEXT_PROTO_VERSION_ID << R"( -d, --data= Post FILE to server. The request method is changed to POST. -r, --rate= - Specified the fixed rate at which connections are - created. The rate must be a positive integer, - representing the number of connections to be made per - second. When the rate is 0, the program will run as it + Specified the fixed rate at which connections are + created. The rate must be a positive integer, + representing the number of connections to be made per + second. When the rate is 0, the program will run as it normally does, creating connections at whatever variable - rate it wants. The default value for this option is 0. + rate it wants. The default value for this option is 0. -C, --num-conns= - Specifies the total number of connections to create. The - total number of connections must be a positive integer. - On each connection, '-m' requests are made. The test - stops once as soon as the N connections have either - completed or failed. When the number of connections is + Specifies the total number of connections to create. + The total number of connections must be a positive + integer. On each connection, -m requests are made. The + test stops once as soon as the N connections have either + completed or failed. When the number of connections is 0, the program will run as it normally does, creating as - many connections as it needs in order to make the '-n' - requests specified. The default value for this option is - 0. The '-n' option is not required if the '-C' option - is being used. + many connections as it needs in order to make the -n + requests specified. The default value for this option + is 0. The -n option is not required if the -C option is + being used. -v, --verbose Output debug information. --version Display version information and exit. - -h, --help Display this help and exit.)" - << std::endl; + -h, --help Display this help and exit.)" << std::endl; } } // namespace From cce64e67288ee46a12ef45b31063d4dd141130c3 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 19 Jul 2015 17:31:35 +0900 Subject: [PATCH 54/77] h2load: Call second_timeout_cb manually so that we don't waste first 1 second --- src/h2load.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/h2load.cc b/src/h2load.cc index f9519d34..918e0f42 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -1594,6 +1594,10 @@ int main(int argc, char **argv) { ev_init(&timeout_watcher, second_timeout_cb); timeout_watcher.repeat = 1.; ev_timer_again(rate_loop, &timeout_watcher); + + // call callback so that we don't waste first 1 second. + second_timeout_cb(rate_loop, &timeout_watcher, 0); + ev_run(rate_loop, 0); } // end rate mode section From 5dc060c1a2f04cc9c6f183b5822c5da2411909e4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 19 Jul 2015 17:55:37 +0900 Subject: [PATCH 55/77] src: Use C++11 value-initialization, instead of memset-ing 0 --- src/HttpServer.cc | 3 +-- src/h2load.cc | 12 ++++-------- src/http2_test.cc | 3 +-- src/nghttp.cc | 21 +++++++-------------- src/nghttpd.cc | 3 +-- src/shrpx.cc | 11 ++++------- src/shrpx_config.cc | 3 +-- src/shrpx_connection_handler.cc | 12 ++++-------- src/shrpx_downstream.cc | 3 +-- src/shrpx_http2_upstream.cc | 6 ++---- src/shrpx_spdy_upstream.cc | 3 +-- src/util.cc | 6 ++---- 12 files changed, 29 insertions(+), 57 deletions(-) diff --git a/src/HttpServer.cc b/src/HttpServer.cc index 02279ad1..6936bc85 100644 --- a/src/HttpServer.cc +++ b/src/HttpServer.cc @@ -1666,7 +1666,6 @@ int verify_callback(int preverify_ok, X509_STORE_CTX *ctx) { namespace { int start_listen(HttpServer *sv, struct ev_loop *loop, Sessions *sessions, const Config *config) { - addrinfo hints; int r; bool ok = false; const char *addr = nullptr; @@ -1674,7 +1673,7 @@ int start_listen(HttpServer *sv, struct ev_loop *loop, Sessions *sessions, auto acceptor = std::make_shared(sv, sessions, config); auto service = util::utos(config->port); - memset(&hints, 0, sizeof(addrinfo)); + addrinfo hints{}; hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_PASSIVE; diff --git a/src/h2load.cc b/src/h2load.cc index 918e0f42..32763a0a 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -841,9 +841,8 @@ process_time_stats(const std::vector> &workers) { namespace { void resolve_host() { int rv; - addrinfo hints, *res; + addrinfo hints{}, *res; - memset(&hints, 0, sizeof(hints)); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_protocol = 0; @@ -906,8 +905,7 @@ std::vector parse_uris(Iterator first, Iterator last) { // First URI is treated specially. We use scheme, host and port of // this URI and ignore those in the remaining URIs if present. - http_parser_url u; - memset(&u, 0, sizeof(u)); + http_parser_url u{}; if (first == last) { std::cerr << "no URI available" << std::endl; @@ -935,8 +933,7 @@ std::vector parse_uris(Iterator first, Iterator last) { reqlines.push_back(get_reqline(uri, u)); for (; first != last; ++first) { - http_parser_url u; - memset(&u, 0, sizeof(u)); + http_parser_url u{}; auto uri = (*first).c_str(); @@ -1333,8 +1330,7 @@ int main(int argc, char **argv) { config.data_length = data_stat.st_size; } - struct sigaction act; - memset(&act, 0, sizeof(struct sigaction)); + struct sigaction act {}; act.sa_handler = SIG_IGN; sigaction(SIGPIPE, &act, nullptr); diff --git a/src/http2_test.cc b/src/http2_test.cc index 88339045..9a99a3a9 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -191,8 +191,7 @@ void check_rewrite_location_uri(const std::string &want, const std::string &uri, const std::string &match_host, const std::string &req_authority, const std::string &upstream_scheme) { - http_parser_url u; - memset(&u, 0, sizeof(u)); + http_parser_url u{}; CU_ASSERT(0 == http_parser_parse_url(uri.c_str(), uri.size(), 0, &u)); auto got = http2::rewrite_location_uri(uri, u, match_host, req_authority, upstream_scheme); diff --git a/src/nghttp.cc b/src/nghttp.cc index eb423c6d..8f85aaca 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -501,9 +501,8 @@ bool HttpClient::need_upgrade() const { int HttpClient::resolve_host(const std::string &host, uint16_t port) { int rv; - addrinfo hints; this->host = host; - memset(&hints, 0, sizeof(hints)); + addrinfo hints{}; hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_protocol = 0; @@ -1260,8 +1259,7 @@ bool HttpClient::add_request(const std::string &uri, const nghttp2_data_provider *data_prd, int64_t data_length, const nghttp2_priority_spec &pri_spec, int level) { - http_parser_url u; - memset(&u, 0, sizeof(u)); + http_parser_url u{}; if (http_parser_parse_url(uri.c_str(), uri.size(), 0, &u) != 0) { return false; } @@ -1485,8 +1483,7 @@ void update_html_parser(HttpClient *client, Request *req, const uint8_t *data, auto uri = strip_fragment(p.first.c_str()); auto res_type = p.second; - http_parser_url u; - memset(&u, 0, sizeof(u)); + http_parser_url u{}; if (http_parser_parse_url(uri.c_str(), uri.size(), 0, &u) == 0 && util::fieldeq(uri.c_str(), u, req->uri.c_str(), req->u, UF_SCHEMA) && util::fieldeq(uri.c_str(), u, req->uri.c_str(), req->u, UF_HOST) && @@ -1650,8 +1647,7 @@ int on_begin_headers_callback(nghttp2_session *session, } case NGHTTP2_PUSH_PROMISE: { auto stream_id = frame->push_promise.promised_stream_id; - http_parser_url u; - memset(&u, 0, sizeof(u)); + http_parser_url u{}; // TODO Set pri and level nghttp2_priority_spec pri_spec; @@ -1820,8 +1816,7 @@ int on_frame_recv_callback2(nghttp2_session *session, uri += "://"; uri += authority->value; uri += path->value; - http_parser_url u; - memset(&u, 0, sizeof(u)); + http_parser_url u{}; if (http_parser_parse_url(uri.c_str(), uri.size(), 0, &u) != 0) { nghttp2_submit_rst_stream(session, NGHTTP2_FLAG_NONE, frame->push_promise.promised_stream_id, @@ -2299,8 +2294,7 @@ int run(char **uris, int n) { std::vector> requests; for (int i = 0; i < n; ++i) { - http_parser_url u; - memset(&u, 0, sizeof(u)); + http_parser_url u{}; auto uri = strip_fragment(uris[i]); if (http_parser_parse_url(uri.c_str(), uri.size(), 0, &u) != 0) { std::cerr << "[ERROR] Could not parse URI " << uri << std::endl; @@ -2701,8 +2695,7 @@ int main(int argc, char **argv) { nghttp2_option_set_peer_max_concurrent_streams( config.http2_option, config.peer_max_concurrent_streams); - struct sigaction act; - memset(&act, 0, sizeof(struct sigaction)); + struct sigaction act {}; act.sa_handler = SIG_IGN; sigaction(SIGPIPE, &act, nullptr); reset_timer(); diff --git a/src/nghttpd.cc b/src/nghttpd.cc index e5d7c2e0..7f11efd5 100644 --- a/src/nghttpd.cc +++ b/src/nghttpd.cc @@ -371,8 +371,7 @@ int main(int argc, char **argv) { set_color_output(color || isatty(fileno(stdout))); - struct sigaction act; - memset(&act, 0, sizeof(struct sigaction)); + struct sigaction act {}; act.sa_handler = SIG_IGN; sigaction(SIGPIPE, &act, nullptr); diff --git a/src/shrpx.cc b/src/shrpx.cc index d3b1c9fb..b346aa99 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -119,12 +119,11 @@ const int GRACEFUL_SHUTDOWN_SIGNAL = SIGQUIT; namespace { int resolve_hostname(sockaddr_union *addr, size_t *addrlen, const char *hostname, uint16_t port, int family) { - addrinfo hints; int rv; auto service = util::utos(port); - memset(&hints, 0, sizeof(addrinfo)); + addrinfo hints{}; hints.ai_family = family; hints.ai_socktype = SOCK_STREAM; #ifdef AI_ADDRCONFIG @@ -279,12 +278,11 @@ std::unique_ptr create_acceptor(ConnectionHandler *handler, } } - addrinfo hints; int fd = -1; int rv; auto service = util::utos(get_config()->port); - memset(&hints, 0, sizeof(addrinfo)); + addrinfo hints{}; hints.ai_family = family; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_PASSIVE; @@ -849,7 +847,7 @@ int16_t DEFAULT_DOWNSTREAM_PORT = 80; namespace { void fill_default_config() { - memset(mod_config(), 0, sizeof(*mod_config())); + *mod_config() = {}; mod_config()->verbose = false; mod_config()->daemon = false; @@ -2359,8 +2357,7 @@ int main(int argc, char **argv) { reset_timer(); } - struct sigaction act; - memset(&act, 0, sizeof(struct sigaction)); + struct sigaction act {}; act.sa_handler = SIG_IGN; sigaction(SIGPIPE, &act, nullptr); diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 5486fda8..b51d44e2 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1575,8 +1575,7 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_BACKEND_HTTP_PROXY_URI: { // parse URI and get hostname, port and optionally userinfo. - http_parser_url u; - memset(&u, 0, sizeof(u)); + http_parser_url u{}; int rv = http_parser_parse_url(optarg, strlen(optarg), 0, &u); if (rv == 0) { std::string val; diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index cbe828b8..91b477d2 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -129,9 +129,8 @@ ConnectionHandler::~ConnectionHandler() { } void ConnectionHandler::worker_reopen_log_files() { - WorkerEvent wev; + WorkerEvent wev{}; - memset(&wev, 0, sizeof(wev)); wev.type = REOPEN_LOG; for (auto &worker : workers_) { @@ -141,9 +140,8 @@ void ConnectionHandler::worker_reopen_log_files() { void ConnectionHandler::worker_renew_ticket_keys( const std::shared_ptr &ticket_keys) { - WorkerEvent wev; + WorkerEvent wev{}; - memset(&wev, 0, sizeof(wev)); wev.type = RENEW_TICKET_KEYS; wev.ticket_keys = ticket_keys; @@ -216,8 +214,7 @@ void ConnectionHandler::graceful_shutdown_worker() { return; } - WorkerEvent wev; - memset(&wev, 0, sizeof(wev)); + WorkerEvent wev{}; wev.type = GRACEFUL_SHUTDOWN; if (LOG_ENABLED(INFO)) { @@ -266,8 +263,7 @@ int ConnectionHandler::handle_connection(int fd, sockaddr *addr, int addrlen) { LOG(INFO) << "Dispatch connection to worker #" << idx; } ++worker_round_robin_cnt_; - WorkerEvent wev; - memset(&wev, 0, sizeof(wev)); + WorkerEvent wev{}; wev.type = NEW_CONNECTION; wev.client_fd = fd; memcpy(&wev.client_addr, addr, addrlen); diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 87a2d082..bfd7aa4b 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -621,8 +621,7 @@ void Downstream::rewrite_location_response_header( if (!hd) { return; } - http_parser_url u; - memset(&u, 0, sizeof(u)); + http_parser_url u{}; int rv = http_parser_parse_url((*hd).value.c_str(), (*hd).value.size(), 0, &u); if (rv != 0) { diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 87c77d14..51146e8c 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1482,8 +1482,7 @@ int Http2Upstream::on_downstream_reset(bool no_retry) { int Http2Upstream::prepare_push_promise(Downstream *downstream) { int rv; - http_parser_url u; - memset(&u, 0, sizeof(u)); + http_parser_url u{}; rv = http_parser_parse_url(downstream->get_request_path().c_str(), downstream->get_request_path().size(), 0, &u); if (rv != 0) { @@ -1513,8 +1512,7 @@ int Http2Upstream::prepare_push_promise(Downstream *downstream) { const char *relq = nullptr; size_t relqlen = 0; - http_parser_url v; - memset(&v, 0, sizeof(v)); + http_parser_url v{}; rv = http_parser_parse_url(link_url, link_urllen, 0, &v); if (rv != 0) { assert(link_urllen); diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 5f124296..745cc0b5 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -452,8 +452,7 @@ SpdyUpstream::SpdyUpstream(uint16_t version, ClientHandler *handler) : 0, !get_config()->http2_proxy), handler_(handler), session_(nullptr) { - spdylay_session_callbacks callbacks; - memset(&callbacks, 0, sizeof(callbacks)); + spdylay_session_callbacks callbacks{}; callbacks.send_callback = send_callback; callbacks.recv_callback = recv_callback; callbacks.on_stream_close_callback = on_stream_close_callback; diff --git a/src/util.cc b/src/util.cc index 9586ab77..0f70f0da 100644 --- a/src/util.cc +++ b/src/util.cc @@ -348,8 +348,7 @@ std::string iso8601_date(int64_t ms) { } time_t parse_http_date(const std::string &s) { - tm tm; - memset(&tm, 0, sizeof(tm)); + tm tm{}; char *r = strptime(s.c_str(), "%a, %d %b %Y %H:%M:%S GMT", &tm); if (r == 0) { return 0; @@ -637,9 +636,8 @@ void write_uri_field(std::ostream &o, const char *uri, const http_parser_url &u, } bool numeric_host(const char *hostname) { - struct addrinfo hints; struct addrinfo *res; - memset(&hints, 0, sizeof(hints)); + struct addrinfo hints {}; hints.ai_family = AF_UNSPEC; hints.ai_flags = AI_NUMERICHOST; if (getaddrinfo(hostname, nullptr, &hints, &res)) { From b37716ab6af4dc69d483ebd4c81860ac8259481d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 19 Jul 2015 18:35:14 +0900 Subject: [PATCH 56/77] h2load: Workaround with clang-3.4 --- src/h2load.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/h2load.cc b/src/h2load.cc index 32763a0a..ba57e251 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -899,14 +899,12 @@ int client_select_next_proto_cb(SSL *ssl, unsigned char **out, } // namespace namespace { -template -std::vector parse_uris(Iterator first, Iterator last) { +// Use std::vector::iterator explicitly, without that, +// http_parser_url u{} fails with clang-3.4. +std::vector parse_uris(std::vector::iterator first, + std::vector::iterator last) { std::vector reqlines; - // First URI is treated specially. We use scheme, host and port of - // this URI and ignore those in the remaining URIs if present. - http_parser_url u{}; - if (first == last) { std::cerr << "no URI available" << std::endl; exit(EXIT_FAILURE); @@ -915,6 +913,9 @@ std::vector parse_uris(Iterator first, Iterator last) { auto uri = (*first).c_str(); ++first; + // First URI is treated specially. We use scheme, host and port of + // this URI and ignore those in the remaining URIs if present. + http_parser_url u{}; if (http_parser_parse_url(uri, strlen(uri), 0, &u) != 0 || !util::has_uri_field(u, UF_SCHEMA) || !util::has_uri_field(u, UF_HOST)) { std::cerr << "invalid URI: " << uri << std::endl; From fa7a74cfa8c616425244d3aba4d061be9956715f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 19 Jul 2015 18:37:41 +0900 Subject: [PATCH 57/77] h2load: Use std::string::size instead of strlen --- src/h2load.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/h2load.cc b/src/h2load.cc index ba57e251..efb49c27 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -911,17 +911,18 @@ std::vector parse_uris(std::vector::iterator first, } auto uri = (*first).c_str(); - ++first; // First URI is treated specially. We use scheme, host and port of // this URI and ignore those in the remaining URIs if present. http_parser_url u{}; - if (http_parser_parse_url(uri, strlen(uri), 0, &u) != 0 || + if (http_parser_parse_url(uri, (*first).size(), 0, &u) != 0 || !util::has_uri_field(u, UF_SCHEMA) || !util::has_uri_field(u, UF_HOST)) { std::cerr << "invalid URI: " << uri << std::endl; exit(EXIT_FAILURE); } + ++first; + config.scheme = util::get_uri_field(uri, u, UF_SCHEMA); config.host = util::get_uri_field(uri, u, UF_HOST); config.default_port = util::get_default_port(uri, u); @@ -938,7 +939,7 @@ std::vector parse_uris(std::vector::iterator first, auto uri = (*first).c_str(); - if (http_parser_parse_url(uri, strlen(uri), 0, &u) != 0) { + if (http_parser_parse_url(uri, (*first).size(), 0, &u) != 0) { std::cerr << "invalid URI: " << uri << std::endl; exit(EXIT_FAILURE); } From 5c96ecd77d18ae302d62a2d59841c76381b2a536 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 19 Jul 2015 19:16:09 +0900 Subject: [PATCH 58/77] Dump APIDOC removal failure errors to /dev/null --- doc/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Makefile.am b/doc/Makefile.am index 0eefa442..55042f65 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -223,7 +223,7 @@ $(APIDOC): apidoc.stamp fi clean-local: - -rm $(APIDOCS) + -rm $(APIDOCS) > /dev/null 2>&1 -rm -rf $(BUILDDIR)/* html-local: apiref.rst From ca3444c34cfce8ec16ddc50754058336ada6c503 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 19 Jul 2015 20:50:14 +0900 Subject: [PATCH 59/77] Fix compile error/warnings with gcc-4.7 --- src/shrpx.cc | 3 ++- src/shrpx_config.cc | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index b346aa99..b6f442d7 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -630,7 +630,8 @@ void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) { new_key.hmac = EVP_sha256(); new_key.hmac_keylen = EVP_MD_size(new_key.hmac); - assert(EVP_CIPHER_key_length(new_key.cipher) <= sizeof(new_key.data.enc_key)); + assert(static_cast(EVP_CIPHER_key_length(new_key.cipher)) <= + sizeof(new_key.data.enc_key)); assert(new_key.hmac_keylen <= sizeof(new_key.data.hmac_key)); if (LOG_ENABLED(INFO)) { diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index b51d44e2..4db9c060 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -171,7 +171,7 @@ read_tls_ticket_key_file(const std::vector &files, return nullptr; } - if (fst.st_size != expectedlen) { + if (static_cast(fst.st_size) != expectedlen) { LOG(ERROR) << "tls-ticket-key-file: the expected file size is " << expectedlen << ", the actual file size is " << fst.st_size; return nullptr; @@ -184,7 +184,7 @@ read_tls_ticket_key_file(const std::vector &files, } f.read(buf, expectedlen); - if (f.gcount() != expectedlen) { + if (static_cast(f.gcount()) != expectedlen) { LOG(ERROR) << "tls-ticket-key-file: want to read " << expectedlen << " bytes but only read " << f.gcount() << " bytes from " << file; @@ -1836,7 +1836,7 @@ int parse_config(const char *opt, const char *optarg, return -1; } - included_set.emplace(optarg); + included_set.insert(optarg); auto rv = load_config(optarg, included_set); included_set.erase(optarg); From dd8ce1e9d2522aa17593666ca4e2dad064cfc71a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 20 Jul 2015 21:37:23 +0900 Subject: [PATCH 60/77] nghttpx: Use std::unique_ptr instead of raw char pointer --- src/shrpx_config.cc | 74 ++++++++++++++++++------------------- src/shrpx_config.h | 39 ++++++++++--------- src/shrpx_config_test.cc | 27 ++++++-------- src/shrpx_https_upstream.cc | 5 ++- src/shrpx_ssl.cc | 31 ++++++++-------- src/shrpx_ssl.h | 8 ++-- 6 files changed, 90 insertions(+), 94 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 4db9c060..99f1983d 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -261,35 +261,37 @@ std::string read_passwd_from_file(const char *filename) { return line; } -std::vector parse_config_str_list(const char *s, char delim) { +std::vector> split_config_str_list(const char *s, + char delim) { size_t len = 1; - for (const char *first = s, *p = nullptr; (p = strchr(first, delim)); - ++len, first = p + 1) + auto last = s + strlen(s); + for (const char *first = s, *d = nullptr; + (d = std::find(first, last, delim)) != last; ++len, first = d + 1) ; - auto list = std::vector(len); - auto first = strdup(s); + + auto list = std::vector>(len); + len = 0; - for (;;) { - auto p = strchr(first, delim); - if (p == nullptr) { + for (auto first = s;; ++len) { + auto stop = std::find(first, last, delim); + list[len] = {first, stop}; + if (stop == last) { break; } - list[len++] = first; - *p = '\0'; - first = p + 1; + first = stop + 1; } - list[len++] = first; - return list; } -void clear_config_str_list(std::vector &list) { - if (list.empty()) { - return; +std::vector> parse_config_str_list(const char *s, + char delim) { + auto ranges = split_config_str_list(s, delim); + auto res = std::vector>(); + res.reserve(ranges.size()); + for (const auto &range : ranges) { + res.push_back(strcopy(range.first, range.second)); } - - free(list[0]); - list.clear(); + return res; } std::pair parse_header(const char *optarg) { @@ -590,22 +592,21 @@ namespace { void parse_mapping(const DownstreamAddr &addr, const char *src) { // This returns at least 1 element (it could be empty string). We // will append '/' to all patterns, so it becomes catch-all pattern. - auto mapping = parse_config_str_list(src, ':'); + auto mapping = split_config_str_list(src, ':'); assert(!mapping.empty()); - for (auto raw_pattern : mapping) { + for (const auto &raw_pattern : mapping) { auto done = false; std::string pattern; - auto slash = strchr(raw_pattern, '/'); - if (slash == nullptr) { + auto slash = std::find(raw_pattern.first, raw_pattern.second, '/'); + if (slash == raw_pattern.second) { // This effectively makes empty pattern to "/". - pattern = raw_pattern; + pattern.assign(raw_pattern.first, raw_pattern.second); util::inp_strlower(pattern); pattern += "/"; } else { - pattern.assign(raw_pattern, slash); + pattern.assign(raw_pattern.first, slash); util::inp_strlower(pattern); - pattern += - http2::normalize_path(slash, raw_pattern + strlen(raw_pattern)); + pattern += http2::normalize_path(slash, raw_pattern.second); } for (auto &g : mod_config()->downstream_addr_groups) { if (g.pattern == pattern) { @@ -621,7 +622,6 @@ void parse_mapping(const DownstreamAddr &addr, const char *src) { g.addrs.push_back(addr); mod_config()->downstream_addr_groups.push_back(std::move(g)); } - clear_config_str_list(mapping); } } // namespace @@ -1629,12 +1629,10 @@ int parse_config(const char *opt, const char *optarg, LOG(WARN) << opt << ": not implemented yet"; return parse_uint_with_unit(&mod_config()->worker_write_burst, opt, optarg); case SHRPX_OPTID_NPN_LIST: - clear_config_str_list(mod_config()->npn_list); mod_config()->npn_list = parse_config_str_list(optarg); return 0; case SHRPX_OPTID_TLS_PROTO_LIST: - clear_config_str_list(mod_config()->tls_proto_list); mod_config()->tls_proto_list = parse_config_str_list(optarg); return 0; @@ -1689,13 +1687,13 @@ int parse_config(const char *opt, const char *optarg, int port; - if (parse_uint(&port, opt, tokens[1]) != 0) { + if (parse_uint(&port, opt, tokens[1].get()) != 0) { return -1; } if (port < 1 || port > static_cast(std::numeric_limits::max())) { - LOG(ERROR) << opt << ": port is invalid: " << tokens[1]; + LOG(ERROR) << opt << ": port is invalid: " << tokens[1].get(); return -1; } @@ -1703,16 +1701,16 @@ int parse_config(const char *opt, const char *optarg, altsvc.port = port; - altsvc.protocol_id = tokens[0]; - altsvc.protocol_id_len = strlen(altsvc.protocol_id); + altsvc.protocol_id = std::move(tokens[0]); + altsvc.protocol_id_len = strlen(altsvc.protocol_id.get()); if (tokens.size() > 2) { - altsvc.host = tokens[2]; - altsvc.host_len = strlen(altsvc.host); + altsvc.host = std::move(tokens[2]); + altsvc.host_len = strlen(altsvc.host.get()); if (tokens.size() > 3) { - altsvc.origin = tokens[3]; - altsvc.origin_len = strlen(altsvc.origin); + altsvc.origin = std::move(tokens[3]); + altsvc.origin_len = strlen(altsvc.origin.get()); } } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 1fee15f4..68840c1d 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -184,13 +184,11 @@ union sockaddr_union { enum shrpx_proto { PROTO_HTTP2, PROTO_HTTP }; struct AltSvc { - AltSvc() - : protocol_id(nullptr), host(nullptr), origin(nullptr), - protocol_id_len(0), host_len(0), origin_len(0), port(0) {} + AltSvc() : protocol_id_len(0), host_len(0), origin_len(0), port(0) {} - char *protocol_id; - char *host; - char *origin; + std::unique_ptr protocol_id; + std::unique_ptr host; + std::unique_ptr origin; size_t protocol_id_len; size_t host_len; @@ -291,10 +289,10 @@ struct Config { // list of supported NPN/ALPN protocol strings in the order of // preference. The each element of this list is a NULL-terminated // string. - std::vector npn_list; + std::vector> npn_list; // list of supported SSL/TLS protocol strings. The each element of // this list is a NULL-terminated string. - std::vector tls_proto_list; + std::vector> tls_proto_list; // Path to file containing CA certificate solely used for client // certificate validation std::unique_ptr verify_client_cacert; @@ -410,19 +408,20 @@ int load_config(const char *filename, std::set &include_set); // Read passwd from |filename| std::string read_passwd_from_file(const char *filename); -// Parses delimited strings in |s| and returns the array of pointers, -// each element points to the each substring in |s|. The delimiter is -// given by |delim. The |s| must be comma delimited list of strings. -// The strings must be delimited by a single comma and any white -// spaces around it are treated as a part of protocol strings. This -// function copies |s| and first element in the return value points to -// it. It is caller's responsibility to deallocate its memory. -std::vector parse_config_str_list(const char *s, char delim = ','); +template using Range = std::pair; -// Clears all elements of |list|, which is returned by -// parse_config_str_list(). If list is not empty, list[0] is freed by -// free(2). After this call, list.empty() must be true. -void clear_config_str_list(std::vector &list); +// Parses delimited strings in |s| and returns the array of substring, +// delimited by |delim|. The any white spaces around substring are +// treated as a part of substring. +std::vector> parse_config_str_list(const char *s, + char delim = ','); + +// Parses delimited strings in |s| and returns the array of pointers, +// each element points to the beginning and one beyond last of +// substring in |s|. The delimiter is given by |delim|. The any +// white spaces around substring are treated as a part of substring. +std::vector> split_config_str_list(const char *s, + char delim); // Parses header field in |optarg|. We expect header field is formed // like "NAME: VALUE". We require that NAME is non empty string. ":" diff --git a/src/shrpx_config_test.cc b/src/shrpx_config_test.cc index d91a706b..decbb50c 100644 --- a/src/shrpx_config_test.cc +++ b/src/shrpx_config_test.cc @@ -39,34 +39,29 @@ namespace shrpx { void test_shrpx_config_parse_config_str_list(void) { auto res = parse_config_str_list("a"); CU_ASSERT(1 == res.size()); - CU_ASSERT(0 == strcmp("a", res[0])); - clear_config_str_list(res); + CU_ASSERT(0 == strcmp("a", res[0].get())); res = parse_config_str_list("a,"); CU_ASSERT(2 == res.size()); - CU_ASSERT(0 == strcmp("a", res[0])); - CU_ASSERT(0 == strcmp("", res[1])); - clear_config_str_list(res); + CU_ASSERT(0 == strcmp("a", res[0].get())); + CU_ASSERT(0 == strcmp("", res[1].get())); res = parse_config_str_list(":a::", ':'); CU_ASSERT(4 == res.size()); - CU_ASSERT(0 == strcmp("", res[0])); - CU_ASSERT(0 == strcmp("a", res[1])); - CU_ASSERT(0 == strcmp("", res[2])); - CU_ASSERT(0 == strcmp("", res[3])); - clear_config_str_list(res); + CU_ASSERT(0 == strcmp("", res[0].get())); + CU_ASSERT(0 == strcmp("a", res[1].get())); + CU_ASSERT(0 == strcmp("", res[2].get())); + CU_ASSERT(0 == strcmp("", res[3].get())); res = parse_config_str_list(""); CU_ASSERT(1 == res.size()); - CU_ASSERT(0 == strcmp("", res[0])); - clear_config_str_list(res); + CU_ASSERT(0 == strcmp("", res[0].get())); res = parse_config_str_list("alpha,bravo,charlie"); CU_ASSERT(3 == res.size()); - CU_ASSERT(0 == strcmp("alpha", res[0])); - CU_ASSERT(0 == strcmp("bravo", res[1])); - CU_ASSERT(0 == strcmp("charlie", res[2])); - clear_config_str_list(res); + CU_ASSERT(0 == strcmp("alpha", res[0].get())); + CU_ASSERT(0 == strcmp("bravo", res[1].get())); + CU_ASSERT(0 == strcmp("charlie", res[2].get())); } void test_shrpx_config_parse_header(void) { diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 22c048a3..ef2659f8 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -845,9 +845,10 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) { hdrs += "Alt-Svc: "; for (auto &altsvc : get_config()->altsvcs) { - hdrs += util::percent_encode_token(altsvc.protocol_id); + hdrs += util::percent_encode_token(altsvc.protocol_id.get()); hdrs += "=\""; - hdrs += util::quote_string(std::string(altsvc.host, altsvc.host_len)); + hdrs += + util::quote_string(std::string(altsvc.host.get(), altsvc.host_len)); hdrs += ":"; hdrs += util::utos(altsvc.port); hdrs += "\", "; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 121f4ce3..10981803 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -86,11 +86,12 @@ int verify_callback(int preverify_ok, X509_STORE_CTX *ctx) { } } // namespace -std::vector set_alpn_prefs(const std::vector &protos) { +std::vector +set_alpn_prefs(const std::vector> &protos) { size_t len = 0; - for (auto proto : protos) { - auto n = strlen(proto); + for (auto &proto : protos) { + auto n = strlen(proto.get()); if (n > 255) { LOG(FATAL) << "Too long ALPN identifier: " << n; @@ -108,11 +109,11 @@ std::vector set_alpn_prefs(const std::vector &protos) { auto out = std::vector(len); auto ptr = out.data(); - for (auto proto : protos) { - auto proto_len = strlen(proto); + for (auto &proto : protos) { + auto proto_len = strlen(proto.get()); *ptr++ = proto_len; - memcpy(ptr, proto, proto_len); + memcpy(ptr, proto.get(), proto_len); ptr += proto_len; } @@ -281,16 +282,15 @@ int alpn_select_proto_cb(SSL *ssl, const unsigned char **out, // We assume that get_config()->npn_list contains ALPN protocol // identifier sorted by preference order. So we just break when we // found the first overlap. - for (auto target_proto_id : get_config()->npn_list) { - auto target_proto_len = - strlen(reinterpret_cast(target_proto_id)); + for (auto &target_proto_id : get_config()->npn_list) { + auto target_proto_len = strlen(target_proto_id.get()); for (auto p = in, end = in + inlen; p < end;) { auto proto_id = p + 1; auto proto_len = *p; if (proto_id + proto_len <= end && target_proto_len == proto_len && - memcmp(target_proto_id, proto_id, proto_len) == 0) { + memcmp(target_proto_id.get(), proto_id, proto_len) == 0) { *out = reinterpret_cast(proto_id); *outlen = proto_len; @@ -314,13 +314,14 @@ constexpr long int tls_masks[] = {SSL_OP_NO_TLSv1_2, SSL_OP_NO_TLSv1_1, SSL_OP_NO_TLSv1}; } // namespace -long int create_tls_proto_mask(const std::vector &tls_proto_list) { +long int create_tls_proto_mask( + const std::vector> &tls_proto_list) { long int res = 0; for (size_t i = 0; i < tls_namelen; ++i) { size_t j; for (j = 0; j < tls_proto_list.size(); ++j) { - if (util::strieq(tls_names[i], tls_proto_list[j])) { + if (util::strieq(tls_names[i], tls_proto_list[j].get())) { break; } } @@ -949,10 +950,10 @@ int cert_lookup_tree_add_cert_from_file(CertLookupTree *lt, SSL_CTX *ssl_ctx, return 0; } -bool in_proto_list(const std::vector &protos, +bool in_proto_list(const std::vector> &protos, const unsigned char *needle, size_t len) { - for (auto proto : protos) { - if (strlen(proto) == len && memcmp(proto, needle, len) == 0) { + for (auto &proto : protos) { + if (strlen(proto.get()) == len && memcmp(proto.get(), needle, len) == 0) { return true; } } diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h index 168b5f95..85b3460d 100644 --- a/src/shrpx_ssl.h +++ b/src/shrpx_ssl.h @@ -140,7 +140,7 @@ int cert_lookup_tree_add_cert_from_file(CertLookupTree *lt, SSL_CTX *ssl_ctx, // Returns true if |needle| which has |len| bytes is included in the // protocol list |protos|. -bool in_proto_list(const std::vector &protos, +bool in_proto_list(const std::vector> &protos, const unsigned char *needle, size_t len); // Returns true if security requirement for HTTP/2 is fulfilled. @@ -149,9 +149,11 @@ bool check_http2_requirement(SSL *ssl); // Returns SSL/TLS option mask to disable SSL/TLS protocol version not // included in |tls_proto_list|. The returned mask can be directly // passed to SSL_CTX_set_options(). -long int create_tls_proto_mask(const std::vector &tls_proto_list); +long int create_tls_proto_mask( + const std::vector> &tls_proto_list); -std::vector set_alpn_prefs(const std::vector &protos); +std::vector +set_alpn_prefs(const std::vector> &protos); // Setups server side SSL_CTX. This function inspects get_config() // and if upstream_no_tls is true, returns nullptr. Otherwise From a8574fdef232df1ee654688593a6013d734af7b0 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 20 Jul 2015 22:37:26 +0900 Subject: [PATCH 61/77] nghttpx: Use Use std::string instead of std::unique_ptr for tls config --- src/shrpx_config.cc | 19 ++++++++---------- src/shrpx_config.h | 25 ++++++++--------------- src/shrpx_config_test.cc | 22 ++++++++++---------- src/shrpx_https_upstream.cc | 9 ++++----- src/shrpx_ssl.cc | 40 ++++++++++++++++--------------------- src/shrpx_ssl.h | 7 +++---- src/util.h | 4 ++++ 7 files changed, 55 insertions(+), 71 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 99f1983d..3fae2d29 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -283,13 +283,12 @@ std::vector> split_config_str_list(const char *s, return list; } -std::vector> parse_config_str_list(const char *s, - char delim) { +std::vector parse_config_str_list(const char *s, char delim) { auto ranges = split_config_str_list(s, delim); - auto res = std::vector>(); + auto res = std::vector(); res.reserve(ranges.size()); for (const auto &range : ranges) { - res.push_back(strcopy(range.first, range.second)); + res.emplace_back(range.first, range.second); } return res; } @@ -1687,30 +1686,28 @@ int parse_config(const char *opt, const char *optarg, int port; - if (parse_uint(&port, opt, tokens[1].get()) != 0) { + if (parse_uint(&port, opt, tokens[1].c_str()) != 0) { return -1; } if (port < 1 || port > static_cast(std::numeric_limits::max())) { - LOG(ERROR) << opt << ": port is invalid: " << tokens[1].get(); + LOG(ERROR) << opt << ": port is invalid: " << tokens[1]; return -1; } AltSvc altsvc; - altsvc.port = port; - altsvc.protocol_id = std::move(tokens[0]); - altsvc.protocol_id_len = strlen(altsvc.protocol_id.get()); + + altsvc.port = port; + altsvc.service = std::move(tokens[1]); if (tokens.size() > 2) { altsvc.host = std::move(tokens[2]); - altsvc.host_len = strlen(altsvc.host.get()); if (tokens.size() > 3) { altsvc.origin = std::move(tokens[3]); - altsvc.origin_len = strlen(altsvc.origin.get()); } } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 68840c1d..2ad0297e 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -184,15 +184,9 @@ union sockaddr_union { enum shrpx_proto { PROTO_HTTP2, PROTO_HTTP }; struct AltSvc { - AltSvc() : protocol_id_len(0), host_len(0), origin_len(0), port(0) {} + AltSvc() : port(0) {} - std::unique_ptr protocol_id; - std::unique_ptr host; - std::unique_ptr origin; - - size_t protocol_id_len; - size_t host_len; - size_t origin_len; + std::string protocol_id, host, origin, service; uint16_t port; }; @@ -251,6 +245,11 @@ struct Config { std::vector accesslog_format; std::vector downstream_addr_groups; std::vector tls_ticket_key_files; + // list of supported NPN/ALPN protocol strings in the order of + // preference. + std::vector npn_list; + // list of supported SSL/TLS protocol strings. + std::vector tls_proto_list; // binary form of http proxy host and port sockaddr_union downstream_http_proxy_addr; ev_tstamp http2_upstream_read_timeout; @@ -286,13 +285,6 @@ struct Config { // ev_token_bucket_cfg *rate_limit_cfg; // // Rate limit configuration per worker (thread) // ev_token_bucket_cfg *worker_rate_limit_cfg; - // list of supported NPN/ALPN protocol strings in the order of - // preference. The each element of this list is a NULL-terminated - // string. - std::vector> npn_list; - // list of supported SSL/TLS protocol strings. The each element of - // this list is a NULL-terminated string. - std::vector> tls_proto_list; // Path to file containing CA certificate solely used for client // certificate validation std::unique_ptr verify_client_cacert; @@ -413,8 +405,7 @@ template using Range = std::pair; // Parses delimited strings in |s| and returns the array of substring, // delimited by |delim|. The any white spaces around substring are // treated as a part of substring. -std::vector> parse_config_str_list(const char *s, - char delim = ','); +std::vector parse_config_str_list(const char *s, char delim = ','); // Parses delimited strings in |s| and returns the array of pointers, // each element points to the beginning and one beyond last of diff --git a/src/shrpx_config_test.cc b/src/shrpx_config_test.cc index decbb50c..8ccd458c 100644 --- a/src/shrpx_config_test.cc +++ b/src/shrpx_config_test.cc @@ -39,29 +39,29 @@ namespace shrpx { void test_shrpx_config_parse_config_str_list(void) { auto res = parse_config_str_list("a"); CU_ASSERT(1 == res.size()); - CU_ASSERT(0 == strcmp("a", res[0].get())); + CU_ASSERT("a" == res[0]); res = parse_config_str_list("a,"); CU_ASSERT(2 == res.size()); - CU_ASSERT(0 == strcmp("a", res[0].get())); - CU_ASSERT(0 == strcmp("", res[1].get())); + CU_ASSERT("a" == res[0]); + CU_ASSERT("" == res[1]); res = parse_config_str_list(":a::", ':'); CU_ASSERT(4 == res.size()); - CU_ASSERT(0 == strcmp("", res[0].get())); - CU_ASSERT(0 == strcmp("a", res[1].get())); - CU_ASSERT(0 == strcmp("", res[2].get())); - CU_ASSERT(0 == strcmp("", res[3].get())); + CU_ASSERT("" == res[0]); + CU_ASSERT("a" == res[1]); + CU_ASSERT("" == res[2]); + CU_ASSERT("" == res[3]); res = parse_config_str_list(""); CU_ASSERT(1 == res.size()); - CU_ASSERT(0 == strcmp("", res[0].get())); + CU_ASSERT("" == res[0]); res = parse_config_str_list("alpha,bravo,charlie"); CU_ASSERT(3 == res.size()); - CU_ASSERT(0 == strcmp("alpha", res[0].get())); - CU_ASSERT(0 == strcmp("bravo", res[1].get())); - CU_ASSERT(0 == strcmp("charlie", res[2].get())); + CU_ASSERT("alpha" == res[0]); + CU_ASSERT("bravo" == res[1]); + CU_ASSERT("charlie" == res[2]); } void test_shrpx_config_parse_header(void) { diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index ef2659f8..4893c9fd 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -844,13 +844,12 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) { if (!get_config()->altsvcs.empty()) { hdrs += "Alt-Svc: "; - for (auto &altsvc : get_config()->altsvcs) { - hdrs += util::percent_encode_token(altsvc.protocol_id.get()); + for (const auto &altsvc : get_config()->altsvcs) { + hdrs += util::percent_encode_token(altsvc.protocol_id); hdrs += "=\""; - hdrs += - util::quote_string(std::string(altsvc.host.get(), altsvc.host_len)); + hdrs += util::quote_string(altsvc.host); hdrs += ":"; - hdrs += util::utos(altsvc.port); + hdrs += altsvc.service; hdrs += "\", "; } diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 10981803..ea5a1cc1 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -87,18 +87,16 @@ int verify_callback(int preverify_ok, X509_STORE_CTX *ctx) { } // namespace std::vector -set_alpn_prefs(const std::vector> &protos) { +set_alpn_prefs(const std::vector &protos) { size_t len = 0; - for (auto &proto : protos) { - auto n = strlen(proto.get()); - - if (n > 255) { - LOG(FATAL) << "Too long ALPN identifier: " << n; + for (const auto &proto : protos) { + if (proto.size() > 255) { + LOG(FATAL) << "Too long ALPN identifier: " << proto.size(); DIE(); } - len += 1 + n; + len += 1 + proto.size(); } if (len > (1 << 16) - 1) { @@ -109,12 +107,10 @@ set_alpn_prefs(const std::vector> &protos) { auto out = std::vector(len); auto ptr = out.data(); - for (auto &proto : protos) { - auto proto_len = strlen(proto.get()); - - *ptr++ = proto_len; - memcpy(ptr, proto.get(), proto_len); - ptr += proto_len; + for (const auto &proto : protos) { + *ptr++ = proto.size(); + memcpy(ptr, proto.c_str(), proto.size()); + ptr += proto.size(); } return out; @@ -282,15 +278,14 @@ int alpn_select_proto_cb(SSL *ssl, const unsigned char **out, // We assume that get_config()->npn_list contains ALPN protocol // identifier sorted by preference order. So we just break when we // found the first overlap. - for (auto &target_proto_id : get_config()->npn_list) { - auto target_proto_len = strlen(target_proto_id.get()); - + for (const auto &target_proto_id : get_config()->npn_list) { for (auto p = in, end = in + inlen; p < end;) { auto proto_id = p + 1; auto proto_len = *p; - if (proto_id + proto_len <= end && target_proto_len == proto_len && - memcmp(target_proto_id.get(), proto_id, proto_len) == 0) { + if (proto_id + proto_len <= end && + util::streq(target_proto_id.c_str(), target_proto_id.size(), proto_id, + proto_len)) { *out = reinterpret_cast(proto_id); *outlen = proto_len; @@ -314,14 +309,13 @@ constexpr long int tls_masks[] = {SSL_OP_NO_TLSv1_2, SSL_OP_NO_TLSv1_1, SSL_OP_NO_TLSv1}; } // namespace -long int create_tls_proto_mask( - const std::vector> &tls_proto_list) { +long int create_tls_proto_mask(const std::vector &tls_proto_list) { long int res = 0; for (size_t i = 0; i < tls_namelen; ++i) { size_t j; for (j = 0; j < tls_proto_list.size(); ++j) { - if (util::strieq(tls_names[i], tls_proto_list[j].get())) { + if (util::strieq(tls_names[i], tls_proto_list[j])) { break; } } @@ -950,10 +944,10 @@ int cert_lookup_tree_add_cert_from_file(CertLookupTree *lt, SSL_CTX *ssl_ctx, return 0; } -bool in_proto_list(const std::vector> &protos, +bool in_proto_list(const std::vector &protos, const unsigned char *needle, size_t len) { for (auto &proto : protos) { - if (strlen(proto.get()) == len && memcmp(proto.get(), needle, len) == 0) { + if (util::streq(proto.c_str(), proto.size(), needle, len)) { return true; } } diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h index 85b3460d..de2509a8 100644 --- a/src/shrpx_ssl.h +++ b/src/shrpx_ssl.h @@ -140,7 +140,7 @@ int cert_lookup_tree_add_cert_from_file(CertLookupTree *lt, SSL_CTX *ssl_ctx, // Returns true if |needle| which has |len| bytes is included in the // protocol list |protos|. -bool in_proto_list(const std::vector> &protos, +bool in_proto_list(const std::vector &protos, const unsigned char *needle, size_t len); // Returns true if security requirement for HTTP/2 is fulfilled. @@ -149,11 +149,10 @@ bool check_http2_requirement(SSL *ssl); // Returns SSL/TLS option mask to disable SSL/TLS protocol version not // included in |tls_proto_list|. The returned mask can be directly // passed to SSL_CTX_set_options(). -long int create_tls_proto_mask( - const std::vector> &tls_proto_list); +long int create_tls_proto_mask(const std::vector &tls_proto_list); std::vector -set_alpn_prefs(const std::vector> &protos); +set_alpn_prefs(const std::vector &protos); // Setups server side SSL_CTX. This function inspects get_config() // and if upstream_no_tls is true, returns nullptr. Otherwise diff --git a/src/util.h b/src/util.h index 6c04fef0..1f84c37a 100644 --- a/src/util.h +++ b/src/util.h @@ -349,6 +349,10 @@ inline bool strieq(const std::string &a, const std::string &b) { bool strieq(const char *a, const char *b); +inline bool strieq(const char *a, const std::string &b) { + return strieq(a, b.c_str(), b.size()); +} + template bool strieq_l(const char (&a)[N], InputIt b, size_t blen) { return strieq(a, N - 1, b, blen); From 0ec1b98f27ccf777182180a3131ab139145fbc90 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 20 Jul 2015 22:42:48 +0900 Subject: [PATCH 62/77] nghttpx: Reorder config fields --- src/shrpx_config.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 2ad0297e..267decf9 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -269,7 +269,6 @@ struct Config { std::unique_ptr private_key_passwd; std::unique_ptr cert_file; std::unique_ptr dh_param_file; - const char *server_name; std::unique_ptr backend_tls_sni_name; std::unique_ptr pid_file; std::unique_ptr conf_path; @@ -293,6 +292,7 @@ struct Config { std::unique_ptr accesslog_file; std::unique_ptr errorlog_file; std::unique_ptr fetch_ocsp_response_file; + std::unique_ptr user; FILE *http2_upstream_dump_request_header; FILE *http2_upstream_dump_response_header; nghttp2_session_callbacks *http2_upstream_callbacks; @@ -300,6 +300,7 @@ struct Config { nghttp2_option *http2_option; nghttp2_option *http2_client_option; const EVP_CIPHER *tls_ticket_cipher; + const char *server_name; char **argv; char *cwd; size_t num_worker; @@ -338,7 +339,6 @@ struct Config { int syslog_facility; int backlog; int argc; - std::unique_ptr user; uid_t uid; gid_t gid; pid_t pid; From 921e393dcd2f171a69fbd7e19005021bef078aba Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 20 Jul 2015 23:50:05 +0900 Subject: [PATCH 63/77] src: Use instead of in usage text --- src/h2load.cc | 6 +++--- src/nghttp.cc | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/h2load.cc b/src/h2load.cc index efb49c27..638a5ee2 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -1022,10 +1022,10 @@ Options: -t, --threads= Number of native threads. Default: )" << config.nthreads << R"( - -i, --input-file= + -i, --input-file= Path of a file with multiple URIs are separated by EOLs. This option will disable URIs getting from command-line. - If '-' is given as , URIs will be read from stdin. + If '-' is given as , URIs will be read from stdin. URIs are used in this order for each client. All URIs are used, then first URI is used and then 2nd URI, and so on. The scheme, host and port in the subsequent @@ -1063,7 +1063,7 @@ Options: #endif // !HAVE_SPDYLAY out << NGHTTP2_CLEARTEXT_PROTO_VERSION_ID << R"( Default: )" << NGHTTP2_CLEARTEXT_PROTO_VERSION_ID << R"( - -d, --data= + -d, --data= Post FILE to server. The request method is changed to POST. -r, --rate= diff --git a/src/nghttp.cc b/src/nghttp.cc index 8f85aaca..5bd9c503 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -2394,7 +2394,7 @@ Options: must be in PEM format. --key= Use the client private key file. The file must be in PEM format. - -d, --data= + -d, --data= Post FILE to server. If '-' is given, data will be read from stdin. -m, --multiply= @@ -2418,8 +2418,8 @@ Options: -b, --padding= Add at most bytes to a frame payload as padding. Specify 0 to disable padding. - -r, --har= - Output HTTP transactions in HAR format. If '-' + -r, --har= + Output HTTP transactions in HAR format. If '-' is given, data is written to stdout. --color Force colored log output. --continuation From 26b7bee16fbf50b2466431b20fe282c8a3f30243 Mon Sep 17 00:00:00 2001 From: Alexis La Goutte Date: Mon, 20 Jul 2015 17:33:23 +0200 Subject: [PATCH 64/77] =?UTF-8?q?Fix=20rm:=20cannot=20remove=20=E2=80=98*.?= =?UTF-8?q?rst=E2=80=99:=20No=20such=20file=20or=20directory=20when=20"mak?= =?UTF-8?q?e=20clean"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When you don't generated documentation via make html --- doc/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Makefile.am b/doc/Makefile.am index 55042f65..5b6272c1 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -223,7 +223,7 @@ $(APIDOC): apidoc.stamp fi clean-local: - -rm $(APIDOCS) > /dev/null 2>&1 + -rm -f $(APIDOCS) -rm -rf $(BUILDDIR)/* html-local: apiref.rst From f3288092e8d74204f98c05a672b5dbbcfd4f52d7 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 22 Jul 2015 00:11:23 +0900 Subject: [PATCH 65/77] Strictly check occurrence of dynamic table size update RFC 7541 requires that dynamic table size update must occur at the beginning of the first header block, and is signaled as SETTINGS acknowledgement. This commit checks these conditions. If dynamic table size update appears other than the beginning of the first header block, it is treated as error. If SETTINGS ACK is received, and next HEADERS header block does not have dynamic table size update, it is treated as error. --- lib/includes/nghttp2/nghttp2.h | 11 ++++++++ lib/nghttp2_hd.c | 30 +++++++++++++++++++-- lib/nghttp2_hd.h | 2 ++ tests/main.c | 4 +++ tests/nghttp2_hd_test.c | 48 ++++++++++++++++++++++++++++++++++ tests/nghttp2_hd_test.h | 2 ++ 6 files changed, 95 insertions(+), 2 deletions(-) diff --git a/lib/includes/nghttp2/nghttp2.h b/lib/includes/nghttp2/nghttp2.h index d8b1cc4e..856bfb1f 100644 --- a/lib/includes/nghttp2/nghttp2.h +++ b/lib/includes/nghttp2/nghttp2.h @@ -3777,11 +3777,22 @@ NGHTTP2_EXTERN void nghttp2_hd_inflate_del(nghttp2_hd_inflater *inflater); * The |settings_hd_table_bufsize_max| should be the value transmitted * in SETTINGS_HEADER_TABLE_SIZE. * + * This function must not be called while header block is being + * inflated. In other words, this function must be called after + * initialization of |inflater|, but before calling + * `nghttp2_hd_inflate_hd()`, or after + * `nghttp2_hd_inflate_end_headers()`. Otherwise, + * `NGHTTP2_ERR_INVALID_STATE` was returned. + * * This function returns 0 if it succeeds, or one of the following * negative error codes: * * :enum:`NGHTTP2_ERR_NOMEM` * Out of memory. + * :enum:`NGHTTP2_ERR_INVALID_STATE` + * The function is called while header block is being inflated. + * Probably, application missed to call + * `nghttp2_hd_inflate_end_headers()`. */ NGHTTP2_EXTERN int nghttp2_hd_inflate_change_table_size(nghttp2_hd_inflater *inflater, diff --git a/lib/nghttp2_hd.c b/lib/nghttp2_hd.c index 83191211..07ce37e0 100644 --- a/lib/nghttp2_hd.c +++ b/lib/nghttp2_hd.c @@ -704,7 +704,7 @@ int nghttp2_hd_inflate_init(nghttp2_hd_inflater *inflater, nghttp2_mem *mem) { inflater->nv_keep = NULL; inflater->opcode = NGHTTP2_HD_OPCODE_NONE; - inflater->state = NGHTTP2_HD_STATE_OPCODE; + inflater->state = NGHTTP2_HD_STATE_INFLATE_START; rv = nghttp2_bufs_init3(&inflater->nvbufs, NGHTTP2_HD_MAX_NV / 8, 8, 1, 0, mem); @@ -1261,6 +1261,15 @@ int nghttp2_hd_deflate_change_table_size(nghttp2_hd_deflater *deflater, int nghttp2_hd_inflate_change_table_size(nghttp2_hd_inflater *inflater, size_t settings_hd_table_bufsize_max) { + switch (inflater->state) { + case NGHTTP2_HD_STATE_EXPECT_TABLE_SIZE: + case NGHTTP2_HD_STATE_INFLATE_START: + break; + default: + return NGHTTP2_ERR_INVALID_STATE; + } + + inflater->state = NGHTTP2_HD_STATE_EXPECT_TABLE_SIZE; inflater->settings_hd_table_bufsize_max = settings_hd_table_bufsize_max; inflater->ctx.hd_table_bufsize_max = settings_hd_table_bufsize_max; hd_context_shrink_table_size(&inflater->ctx); @@ -1951,9 +1960,25 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, for (; in != last || busy;) { busy = 0; switch (inflater->state) { + case NGHTTP2_HD_STATE_EXPECT_TABLE_SIZE: + if ((*in & 0xe0u) != 0x20u) { + DEBUGF(fprintf(stderr, "inflatehd: header table size change was " + "expected, but saw 0x%02x as first byte", + *in)); + rv = NGHTTP2_ERR_HEADER_COMP; + goto fail; + } + /* fall through */ + case NGHTTP2_HD_STATE_INFLATE_START: case NGHTTP2_HD_STATE_OPCODE: if ((*in & 0xe0u) == 0x20u) { DEBUGF(fprintf(stderr, "inflatehd: header table size change\n")); + if (inflater->state == NGHTTP2_HD_STATE_OPCODE) { + DEBUGF(fprintf(stderr, "inflatehd: header table size change must " + "appear at the head of header block\n")); + rv = NGHTTP2_ERR_HEADER_COMP; + goto fail; + } inflater->opcode = NGHTTP2_HD_OPCODE_INDEXED; inflater->state = NGHTTP2_HD_STATE_READ_TABLE_SIZE; } else if (*in & 0x80u) { @@ -1997,7 +2022,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, DEBUGF(fprintf(stderr, "inflatehd: table_size=%zu\n", inflater->left)); inflater->ctx.hd_table_bufsize_max = inflater->left; hd_context_shrink_table_size(&inflater->ctx); - inflater->state = NGHTTP2_HD_STATE_OPCODE; + inflater->state = NGHTTP2_HD_STATE_INFLATE_START; break; case NGHTTP2_HD_STATE_READ_INDEX: { size_t prefixlen; @@ -2282,6 +2307,7 @@ fail: int nghttp2_hd_inflate_end_headers(nghttp2_hd_inflater *inflater) { hd_inflate_keep_free(inflater); + inflater->state = NGHTTP2_HD_STATE_INFLATE_START; return 0; } diff --git a/lib/nghttp2_hd.h b/lib/nghttp2_hd.h index bf40ab05..12177e8d 100644 --- a/lib/nghttp2_hd.h +++ b/lib/nghttp2_hd.h @@ -151,6 +151,8 @@ typedef enum { } nghttp2_hd_opcode; typedef enum { + NGHTTP2_HD_STATE_EXPECT_TABLE_SIZE, + NGHTTP2_HD_STATE_INFLATE_START, NGHTTP2_HD_STATE_OPCODE, NGHTTP2_HD_STATE_READ_TABLE_SIZE, NGHTTP2_HD_STATE_READ_INDEX, diff --git a/tests/main.c b/tests/main.c index 7353ad74..39cb1327 100644 --- a/tests/main.c +++ b/tests/main.c @@ -326,6 +326,10 @@ int main(int argc _U_, char *argv[] _U_) { test_nghttp2_hd_inflate_clearall_inc) || !CU_add_test(pSuite, "hd_inflate_zero_length_huffman", test_nghttp2_hd_inflate_zero_length_huffman) || + !CU_add_test(pSuite, "hd_inflate_expect_table_size_update", + test_nghttp2_hd_inflate_expect_table_size_update) || + !CU_add_test(pSuite, "hd_inflate_unexpected_table_size_update", + test_nghttp2_hd_inflate_unexpected_table_size_update) || !CU_add_test(pSuite, "hd_ringbuf_reserve", test_nghttp2_hd_ringbuf_reserve) || !CU_add_test(pSuite, "hd_change_table_size", diff --git a/tests/nghttp2_hd_test.c b/tests/nghttp2_hd_test.c index ecd7bc11..db72e3fe 100644 --- a/tests/nghttp2_hd_test.c +++ b/tests/nghttp2_hd_test.c @@ -540,6 +540,54 @@ void test_nghttp2_hd_inflate_zero_length_huffman(void) { nghttp2_hd_inflate_free(&inflater); } +void test_nghttp2_hd_inflate_expect_table_size_update(void) { + nghttp2_hd_inflater inflater; + nghttp2_bufs bufs; + nghttp2_mem *mem; + /* Indexed Header: :method: GET */ + uint8_t data[] = {0x82}; + nva_out out; + + mem = nghttp2_mem_default(); + frame_pack_bufs_init(&bufs); + nva_out_init(&out); + + nghttp2_bufs_add(&bufs, data, sizeof(data)); + nghttp2_hd_inflate_init(&inflater, mem); + /* This will make inflater require table size update in the next + inflation. */ + nghttp2_hd_inflate_change_table_size(&inflater, 4096); + CU_ASSERT(NGHTTP2_ERR_HEADER_COMP == + inflate_hd(&inflater, &out, &bufs, 0, mem)); + + nva_out_reset(&out, mem); + nghttp2_bufs_free(&bufs); + nghttp2_hd_inflate_free(&inflater); +} + +void test_nghttp2_hd_inflate_unexpected_table_size_update(void) { + nghttp2_hd_inflater inflater; + nghttp2_bufs bufs; + nghttp2_mem *mem; + /* Indexed Header: :method: GET, followed by table size update. + This violates RFC 7541. */ + uint8_t data[] = {0x82, 0x20}; + nva_out out; + + mem = nghttp2_mem_default(); + frame_pack_bufs_init(&bufs); + nva_out_init(&out); + + nghttp2_bufs_add(&bufs, data, sizeof(data)); + nghttp2_hd_inflate_init(&inflater, mem); + CU_ASSERT(NGHTTP2_ERR_HEADER_COMP == + inflate_hd(&inflater, &out, &bufs, 0, mem)); + + nva_out_reset(&out, mem); + nghttp2_bufs_free(&bufs); + nghttp2_hd_inflate_free(&inflater); +} + void test_nghttp2_hd_ringbuf_reserve(void) { nghttp2_hd_deflater deflater; nghttp2_hd_inflater inflater; diff --git a/tests/nghttp2_hd_test.h b/tests/nghttp2_hd_test.h index e41fad73..158a98c5 100644 --- a/tests/nghttp2_hd_test.h +++ b/tests/nghttp2_hd_test.h @@ -35,6 +35,8 @@ void test_nghttp2_hd_inflate_newname_noinc(void); void test_nghttp2_hd_inflate_newname_inc(void); void test_nghttp2_hd_inflate_clearall_inc(void); void test_nghttp2_hd_inflate_zero_length_huffman(void); +void test_nghttp2_hd_inflate_expect_table_size_update(void); +void test_nghttp2_hd_inflate_unexpected_table_size_update(void); void test_nghttp2_hd_ringbuf_reserve(void); void test_nghttp2_hd_change_table_size(void); void test_nghttp2_hd_deflate_inflate(void); From 44cbc785fcd116c924dd086c7e78dceb6ad86513 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 22 Jul 2015 21:41:16 +0900 Subject: [PATCH 66/77] nghttpx: Don't reuse backend connection if it is not clean --- src/shrpx_downstream.cc | 6 ++++++ src/shrpx_downstream.h | 3 +++ src/shrpx_http2_upstream.cc | 25 +++++++------------------ src/shrpx_https_upstream.cc | 8 +++----- src/shrpx_spdy_upstream.cc | 22 ++++++---------------- 5 files changed, 25 insertions(+), 39 deletions(-) diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index bfd7aa4b..a37c7f2f 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -1207,4 +1207,10 @@ void Downstream::add_request_headers_sum(size_t amount) { request_headers_sum_ += amount; } +bool Downstream::can_detach_downstream_connection() const { + return dconn_ && response_state_ == Downstream::MSG_COMPLETE && + request_state_ == Downstream::MSG_COMPLETE && !upgraded_ && + !response_connection_close_; +} + } // namespace shrpx diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 3486621a..c64aba4f 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -332,6 +332,9 @@ public: void attach_blocked_link(BlockedLink *l); BlockedLink *detach_blocked_link(); + // Returns true if downstream_connection can be detached and reused. + bool can_detach_downstream_connection() const; + enum { EVENT_ERROR = 0x1, EVENT_TIMEOUT = 0x2, diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 51146e8c..db57b9db 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -74,22 +74,13 @@ int on_stream_close_callback(nghttp2_session *session, int32_t stream_id, return 0; } - downstream->set_request_state(Downstream::STREAM_CLOSED); - - if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { - // At this point, downstream response was read - if (!downstream->get_upgraded() && - !downstream->get_response_connection_close()) { - // Keep-alive - downstream->detach_downstream_connection(); - } - - upstream->remove_downstream(downstream); - // downstream was deleted - - return 0; + if (downstream->can_detach_downstream_connection()) { + // Keep-alive + downstream->detach_downstream_connection(); } + downstream->set_request_state(Downstream::STREAM_CLOSED); + // At this point, downstream read may be paused. // If shrpx_downstream::push_request_headers() failed, the @@ -915,10 +906,8 @@ int Http2Upstream::downstream_read(DownstreamConnection *dconn) { } return downstream_error(dconn, Downstream::EVENT_ERROR); } - // Detach downstream connection early so that it could be reused - // without hitting server's request timeout. - if (downstream->get_response_state() == Downstream::MSG_COMPLETE && - !downstream->get_response_connection_close()) { + + if (downstream->can_detach_downstream_connection()) { // Keep-alive downstream->detach_downstream_connection(); } diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 4893c9fd..40a29593 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -540,7 +540,8 @@ int HttpsUpstream::on_write() { // We need to postpone detachment until all data are sent so that // we can notify nghttp2 library all data consumed. if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { - if (downstream->get_response_connection_close()) { + if (downstream->get_response_connection_close() || + downstream->get_request_state() != Downstream::MSG_COMPLETE) { // Connection close downstream->pop_downstream_connection(); // dconn was deleted @@ -607,10 +608,7 @@ int HttpsUpstream::downstream_read(DownstreamConnection *dconn) { goto end; } - // Detach downstream connection early so that it could be reused - // without hitting server's request timeout. - if (downstream->get_response_state() == Downstream::MSG_COMPLETE && - !downstream->get_response_connection_close()) { + if (downstream->can_detach_downstream_connection()) { // Keep-alive downstream->detach_downstream_connection(); } diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 745cc0b5..89fcb678 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -109,20 +109,13 @@ void on_stream_close_callback(spdylay_session *session, int32_t stream_id, return; } - downstream->set_request_state(Downstream::STREAM_CLOSED); - if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { - // At this point, downstream response was read - if (!downstream->get_upgraded() && - !downstream->get_response_connection_close()) { - // Keep-alive - downstream->detach_downstream_connection(); - } - upstream->remove_downstream(downstream); - // downstrea was deleted - - return; + if (downstream->can_detach_downstream_connection()) { + // Keep-alive + downstream->detach_downstream_connection(); } + downstream->set_request_state(Downstream::STREAM_CLOSED); + // At this point, downstream read may be paused. // If shrpx_downstream::push_request_headers() failed, the @@ -589,10 +582,7 @@ int SpdyUpstream::downstream_read(DownstreamConnection *dconn) { } return downstream_error(dconn, Downstream::EVENT_ERROR); } - // Detach downstream connection early so that it could be reused - // without hitting server's request timeout. - if (downstream->get_response_state() == Downstream::MSG_COMPLETE && - !downstream->get_response_connection_close()) { + if (downstream->can_detach_downstream_connection()) { // Keep-alive downstream->detach_downstream_connection(); } From 7f71fed963e73c88770e29f4aa6419eb91d9148d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 23 Jul 2015 00:36:00 +0900 Subject: [PATCH 67/77] Allow multiple in-flight SETTINGS --- lib/includes/nghttp2/nghttp2.h | 4 -- lib/nghttp2_session.c | 116 +++++++++++++++++++++++---------- lib/nghttp2_session.h | 20 ++++-- tests/main.c | 2 + tests/nghttp2_session_test.c | 91 +++++++++++++++++++++++--- tests/nghttp2_session_test.h | 1 + 6 files changed, 179 insertions(+), 55 deletions(-) diff --git a/lib/includes/nghttp2/nghttp2.h b/lib/includes/nghttp2/nghttp2.h index 856bfb1f..6c3c2c43 100644 --- a/lib/includes/nghttp2/nghttp2.h +++ b/lib/includes/nghttp2/nghttp2.h @@ -3278,10 +3278,6 @@ NGHTTP2_EXTERN int nghttp2_submit_rst_stream(nghttp2_session *session, * :enum:`NGHTTP2_ERR_INVALID_ARGUMENT` * The |iv| contains invalid value (e.g., initial window size * strictly greater than (1 << 31) - 1. - * :enum:`NGHTTP2_ERR_TOO_MANY_INFLIGHT_SETTINGS` - * There is already another in-flight SETTINGS. Note that the - * current implementation only allows 1 in-flight SETTINGS frame - * without ACK flag set. * :enum:`NGHTTP2_ERR_NOMEM` * Out of memory. */ diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 6285443c..2f74ca4b 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -362,8 +362,6 @@ static int session_new(nghttp2_session **session_ptr, (*session_ptr)->local_last_stream_id = (1u << 31) - 1; (*session_ptr)->remote_last_stream_id = (1u << 31) - 1; - (*session_ptr)->inflight_niv = -1; - (*session_ptr)->pending_local_max_concurrent_stream = NGHTTP2_INITIAL_MAX_CONCURRENT_STREAMS; (*session_ptr)->pending_enable_push = 1; @@ -561,8 +559,42 @@ static void ob_q_free(nghttp2_outbound_queue *q, nghttp2_mem *mem) { } } +static int inflight_settings_new(nghttp2_inflight_settings **settings_ptr, + const nghttp2_settings_entry *iv, size_t niv, + nghttp2_mem *mem) { + *settings_ptr = nghttp2_mem_malloc(mem, sizeof(nghttp2_inflight_settings)); + if (!*settings_ptr) { + return NGHTTP2_ERR_NOMEM; + } + + if (niv > 0) { + (*settings_ptr)->iv = nghttp2_frame_iv_copy(iv, niv, mem); + if (!(*settings_ptr)->iv) { + return NGHTTP2_ERR_NOMEM; + } + } else { + (*settings_ptr)->iv = NULL; + } + + (*settings_ptr)->niv = niv; + (*settings_ptr)->next = NULL; + + return 0; +} + +static void inflight_settings_del(nghttp2_inflight_settings *settings, + nghttp2_mem *mem) { + if (!settings) { + return; + } + + nghttp2_mem_free(mem, settings->iv); + nghttp2_mem_free(mem, settings); +} + void nghttp2_session_del(nghttp2_session *session) { nghttp2_mem *mem; + nghttp2_inflight_settings *settings; if (session == NULL) { return; @@ -570,7 +602,11 @@ void nghttp2_session_del(nghttp2_session *session) { mem = &session->mem; - nghttp2_mem_free(mem, session->inflight_iv); + for (settings = session->inflight_settings_head; settings;) { + nghttp2_inflight_settings *next = settings->next; + inflight_settings_del(settings, mem); + settings = next; + } nghttp2_stream_roots_free(&session->roots); @@ -3939,10 +3975,6 @@ int nghttp2_session_update_local_settings(nghttp2_session *session, } } - session->pending_local_max_concurrent_stream = - NGHTTP2_INITIAL_MAX_CONCURRENT_STREAMS; - session->pending_enable_push = 1; - return 0; } @@ -3951,6 +3983,7 @@ int nghttp2_session_on_settings_received(nghttp2_session *session, int rv; size_t i; nghttp2_mem *mem; + nghttp2_inflight_settings *settings; mem = &session->mem; @@ -3964,15 +3997,21 @@ int nghttp2_session_on_settings_received(nghttp2_session *session, session, frame, NGHTTP2_ERR_FRAME_SIZE_ERROR, "SETTINGS: ACK and payload != 0"); } - if (session->inflight_niv == -1) { + + settings = session->inflight_settings_head; + + if (!settings) { return session_handle_invalid_connection( session, frame, NGHTTP2_ERR_PROTO, "SETTINGS: unexpected ACK"); } - rv = nghttp2_session_update_local_settings(session, session->inflight_iv, - session->inflight_niv); - nghttp2_mem_free(mem, session->inflight_iv); - session->inflight_iv = NULL; - session->inflight_niv = -1; + + rv = nghttp2_session_update_local_settings(session, settings->iv, + settings->niv); + + session->inflight_settings_head = settings->next; + + inflight_settings_del(settings, mem); + if (rv != 0) { if (nghttp2_is_fatal(rv)) { return rv; @@ -6091,6 +6130,22 @@ int nghttp2_session_add_window_update(nghttp2_session *session, uint8_t flags, return 0; } +static void +session_append_inflight_settings(nghttp2_session *session, + nghttp2_inflight_settings *settings) { + nghttp2_inflight_settings *i; + + if (!session->inflight_settings_head) { + session->inflight_settings_head = settings; + return; + } + + for (i = session->inflight_settings_head; i->next; i = i->next) + ; + + i->next = settings; +} + int nghttp2_session_add_settings(nghttp2_session *session, uint8_t flags, const nghttp2_settings_entry *iv, size_t niv) { nghttp2_outbound_item *item; @@ -6099,15 +6154,12 @@ int nghttp2_session_add_settings(nghttp2_session *session, uint8_t flags, size_t i; int rv; nghttp2_mem *mem; + nghttp2_inflight_settings *inflight_settings = NULL; mem = &session->mem; - if (flags & NGHTTP2_FLAG_ACK) { - if (niv != 0) { - return NGHTTP2_ERR_INVALID_ARGUMENT; - } - } else if (session->inflight_niv != -1) { - return NGHTTP2_ERR_TOO_MANY_INFLIGHT_SETTINGS; + if ((flags & NGHTTP2_FLAG_ACK) && niv != 0) { + return NGHTTP2_ERR_INVALID_ARGUMENT; } if (!nghttp2_iv_check(iv, niv)) { @@ -6130,19 +6182,13 @@ int nghttp2_session_add_settings(nghttp2_session *session, uint8_t flags, } if ((flags & NGHTTP2_FLAG_ACK) == 0) { - if (niv > 0) { - session->inflight_iv = nghttp2_frame_iv_copy(iv, niv, mem); - - if (session->inflight_iv == NULL) { - nghttp2_mem_free(mem, iv_copy); - nghttp2_mem_free(mem, item); - return NGHTTP2_ERR_NOMEM; - } - } else { - session->inflight_iv = NULL; + rv = inflight_settings_new(&inflight_settings, iv, niv, mem); + if (rv != 0) { + assert(nghttp2_is_fatal(rv)); + nghttp2_mem_free(mem, iv_copy); + nghttp2_mem_free(mem, item); + return rv; } - - session->inflight_niv = niv; } nghttp2_outbound_item_init(item); @@ -6155,11 +6201,7 @@ int nghttp2_session_add_settings(nghttp2_session *session, uint8_t flags, /* The only expected error is fatal one */ assert(nghttp2_is_fatal(rv)); - if ((flags & NGHTTP2_FLAG_ACK) == 0) { - nghttp2_mem_free(mem, session->inflight_iv); - session->inflight_iv = NULL; - session->inflight_niv = -1; - } + inflight_settings_del(inflight_settings, mem); nghttp2_frame_settings_free(&frame->settings, mem); nghttp2_mem_free(mem, item); @@ -6167,6 +6209,8 @@ int nghttp2_session_add_settings(nghttp2_session *session, uint8_t flags, return rv; } + session_append_inflight_settings(session, inflight_settings); + /* Extract NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS and ENABLE_PUSH here. We use it to refuse the incoming stream and PUSH_PROMISE with RST_STREAM. */ diff --git a/lib/nghttp2_session.h b/lib/nghttp2_session.h index 62c27965..87430bdb 100644 --- a/lib/nghttp2_session.h +++ b/lib/nghttp2_session.h @@ -140,6 +140,17 @@ typedef enum { NGHTTP2_GOAWAY_RECV = 0x8 } nghttp2_goaway_flag; +/* nghttp2_inflight_settings stores the SETTINGS entries which local + endpoint has sent to the remote endpoint, and has not received ACK + yet. */ +struct nghttp2_inflight_settings { + struct nghttp2_inflight_settings *next; + nghttp2_settings_entry *iv; + size_t niv; +}; + +typedef struct nghttp2_inflight_settings nghttp2_inflight_settings; + struct nghttp2_session { nghttp2_map /* */ streams; nghttp2_stream_roots roots; @@ -176,12 +187,9 @@ struct nghttp2_session { /* Points to the oldest idle stream. NULL if there is no idle stream. Only used when session is initialized as erver. */ nghttp2_stream *idle_stream_tail; - /* In-flight SETTINGS values. NULL does not necessarily mean there - is no in-flight SETTINGS. */ - nghttp2_settings_entry *inflight_iv; - /* The number of entries in |inflight_iv|. -1 if there is no - in-flight SETTINGS. */ - ssize_t inflight_niv; + /* Queue of In-flight SETTINGS values. SETTINGS bearing ACK is not + considered as in-flight. */ + nghttp2_inflight_settings *inflight_settings_head; /* The number of outgoing streams. This will be capped by remote_settings.max_concurrent_streams. */ size_t num_outgoing_streams; diff --git a/tests/main.c b/tests/main.c index 39cb1327..3bd70f7c 100644 --- a/tests/main.c +++ b/tests/main.c @@ -171,6 +171,8 @@ int main(int argc _U_, char *argv[] _U_) { test_nghttp2_submit_settings) || !CU_add_test(pSuite, "session_submit_settings_update_local_window_size", test_nghttp2_submit_settings_update_local_window_size) || + !CU_add_test(pSuite, "session_submit_settings_multiple_times", + test_nghttp2_submit_settings_multiple_times) || !CU_add_test(pSuite, "session_submit_push_promise", test_nghttp2_submit_push_promise) || !CU_add_test(pSuite, "submit_window_update", diff --git a/tests/nghttp2_session_test.c b/tests/nghttp2_session_test.c index 33f8ef54..457a4128 100644 --- a/tests/nghttp2_session_test.c +++ b/tests/nghttp2_session_test.c @@ -2300,18 +2300,11 @@ void test_nghttp2_session_on_settings_received(void) { nghttp2_session_server_new(&session, &callbacks, NULL); nghttp2_frame_settings_init(&frame.settings, NGHTTP2_FLAG_ACK, dup_iv(iv, 1), 1); - /* Specify inflight_iv deliberately */ - session->inflight_iv = frame.settings.iv; - session->inflight_niv = frame.settings.niv; - CU_ASSERT(0 == nghttp2_session_on_settings_received(session, &frame, 0)); item = nghttp2_session_get_next_ob_item(session); CU_ASSERT(item != NULL); CU_ASSERT(NGHTTP2_GOAWAY == item->frame.hd.type); - session->inflight_iv = NULL; - session->inflight_niv = -1; - nghttp2_frame_settings_free(&frame.settings, mem); nghttp2_session_del(session); @@ -4041,8 +4034,8 @@ void test_nghttp2_submit_settings(void) { CU_ASSERT(16 * 1024 == session->local_settings.initial_window_size); CU_ASSERT(0 == session->hd_inflater.ctx.hd_table_bufsize_max); CU_ASSERT(50 == session->local_settings.max_concurrent_streams); - CU_ASSERT(NGHTTP2_INITIAL_MAX_CONCURRENT_STREAMS == - session->pending_local_max_concurrent_stream); + /* We just keep the last seen value */ + CU_ASSERT(50 == session->pending_local_max_concurrent_stream); nghttp2_session_del(session); } @@ -4113,6 +4106,86 @@ void test_nghttp2_submit_settings_update_local_window_size(void) { nghttp2_frame_settings_free(&ack_frame.settings, mem); } +void test_nghttp2_submit_settings_multiple_times(void) { + nghttp2_session *session; + nghttp2_session_callbacks callbacks; + nghttp2_settings_entry iv[4]; + nghttp2_frame frame; + nghttp2_mem *mem; + nghttp2_inflight_settings *inflight_settings; + + mem = nghttp2_mem_default(); + + memset(&callbacks, 0, sizeof(callbacks)); + callbacks.send_callback = null_send_callback; + + nghttp2_session_client_new(&session, &callbacks, NULL); + + /* first SETTINGS */ + iv[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS; + iv[0].value = 100; + + iv[1].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH; + iv[1].value = 0; + + CU_ASSERT(0 == nghttp2_submit_settings(session, NGHTTP2_FLAG_NONE, iv, 2)); + + inflight_settings = session->inflight_settings_head; + + CU_ASSERT(NULL != inflight_settings); + CU_ASSERT(NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS == + inflight_settings->iv[0].settings_id); + CU_ASSERT(100 == inflight_settings->iv[0].value); + CU_ASSERT(2 == inflight_settings->niv); + CU_ASSERT(NULL == inflight_settings->next); + + CU_ASSERT(100 == session->pending_local_max_concurrent_stream); + CU_ASSERT(0 == session->pending_enable_push); + + /* second SETTINGS */ + iv[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS; + iv[0].value = 99; + + CU_ASSERT(0 == nghttp2_submit_settings(session, NGHTTP2_FLAG_NONE, iv, 1)); + + inflight_settings = session->inflight_settings_head->next; + + CU_ASSERT(NULL != inflight_settings); + CU_ASSERT(NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS == + inflight_settings->iv[0].settings_id); + CU_ASSERT(99 == inflight_settings->iv[0].value); + CU_ASSERT(1 == inflight_settings->niv); + CU_ASSERT(NULL == inflight_settings->next); + + CU_ASSERT(99 == session->pending_local_max_concurrent_stream); + CU_ASSERT(0 == session->pending_enable_push); + + nghttp2_frame_settings_init(&frame.settings, NGHTTP2_FLAG_ACK, NULL, 0); + + /* receive SETTINGS ACK */ + CU_ASSERT(0 == nghttp2_session_on_settings_received(session, &frame, 0)); + + inflight_settings = session->inflight_settings_head; + + /* first inflight SETTINGS was removed */ + CU_ASSERT(NULL != inflight_settings); + CU_ASSERT(NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS == + inflight_settings->iv[0].settings_id); + CU_ASSERT(99 == inflight_settings->iv[0].value); + CU_ASSERT(1 == inflight_settings->niv); + CU_ASSERT(NULL == inflight_settings->next); + + CU_ASSERT(100 == session->local_settings.max_concurrent_streams); + + /* receive SETTINGS ACK again */ + CU_ASSERT(0 == nghttp2_session_on_settings_received(session, &frame, 0)); + + CU_ASSERT(NULL == session->inflight_settings_head); + CU_ASSERT(99 == session->local_settings.max_concurrent_streams); + + nghttp2_session_del(session); +} + void test_nghttp2_submit_push_promise(void) { nghttp2_session *session; nghttp2_session_callbacks callbacks; diff --git a/tests/nghttp2_session_test.h b/tests/nghttp2_session_test.h index b93d848e..fc626ce0 100644 --- a/tests/nghttp2_session_test.h +++ b/tests/nghttp2_session_test.h @@ -79,6 +79,7 @@ void test_nghttp2_submit_headers_continuation(void); void test_nghttp2_submit_priority(void); void test_nghttp2_submit_settings(void); void test_nghttp2_submit_settings_update_local_window_size(void); +void test_nghttp2_submit_settings_multiple_times(void); void test_nghttp2_submit_push_promise(void); void test_nghttp2_submit_window_update(void); void test_nghttp2_submit_window_update_local_window_size(void); From f6a8c8d0786b112bab8a4200eb54f2923ac6a80c Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 23 Jul 2015 00:44:54 +0900 Subject: [PATCH 68/77] Remove unused variable --- tests/nghttp2_session_test.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/nghttp2_session_test.c b/tests/nghttp2_session_test.c index 457a4128..5b05b88e 100644 --- a/tests/nghttp2_session_test.c +++ b/tests/nghttp2_session_test.c @@ -4111,11 +4111,8 @@ void test_nghttp2_submit_settings_multiple_times(void) { nghttp2_session_callbacks callbacks; nghttp2_settings_entry iv[4]; nghttp2_frame frame; - nghttp2_mem *mem; nghttp2_inflight_settings *inflight_settings; - mem = nghttp2_mem_default(); - memset(&callbacks, 0, sizeof(callbacks)); callbacks.send_callback = null_send_callback; From aa012f7a58ae535c38ad6de8e359513df3501e93 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 23 Jul 2015 02:02:27 +0900 Subject: [PATCH 69/77] Use PROTOCOL_ERROR against DATA sent to idle stream --- lib/nghttp2_session.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 2f74ca4b..d9a74264 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -4672,7 +4672,7 @@ static int session_on_data_received_fail_fast(nghttp2_session *session) { if (!stream) { if (session_detect_idle_stream(session, stream_id)) { failure_reason = "DATA: stream in idle"; - error_code = NGHTTP2_STREAM_CLOSED; + error_code = NGHTTP2_PROTOCOL_ERROR; goto fail; } return NGHTTP2_ERR_IGN_PAYLOAD; From cd2c751f827bc0a3f76a6cfbb8b65e14bcdbabf1 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 23 Jul 2015 21:09:16 +0900 Subject: [PATCH 70/77] nghttpx: Generate new ticket key every 1hr and its life time is now 12hrs --- src/shrpx.cc | 104 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 71 insertions(+), 33 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index b6f442d7..8b171ae1 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -604,54 +604,91 @@ void graceful_shutdown_signal_cb(struct ev_loop *loop, ev_signal *w, } } // namespace +namespace { +int generate_ticket_key(TicketKey &ticket_key) { + ticket_key.cipher = get_config()->tls_ticket_cipher; + ticket_key.hmac = EVP_sha256(); + ticket_key.hmac_keylen = EVP_MD_size(ticket_key.hmac); + + assert(static_cast(EVP_CIPHER_key_length(ticket_key.cipher)) <= + sizeof(ticket_key.data.enc_key)); + assert(ticket_key.hmac_keylen <= sizeof(ticket_key.data.hmac_key)); + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "enc_keylen=" << EVP_CIPHER_key_length(ticket_key.cipher) + << ", hmac_keylen=" << ticket_key.hmac_keylen; + } + + if (RAND_bytes(reinterpret_cast(&ticket_key.data), + sizeof(ticket_key.data)) == 0) { + return -1; + } + + return 0; +} +} // namespace + namespace { void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) { auto conn_handler = static_cast(w->data); const auto &old_ticket_keys = conn_handler->get_ticket_keys(); auto ticket_keys = std::make_shared(); - LOG(NOTICE) << "Renew ticket keys: main"; + LOG(NOTICE) << "Renew new ticket keys"; - // We store at most 2 ticket keys + // If old_ticket_keys is not empty, it should contain at least 2 + // keys: one for encryption, and last one for the next encryption + // key but decryption only. The keys in between are old keys and + // decryption only. The next key is provided to ensure to mitigate + // possible problem when one worker encrypt new key, but one worker, + // which did not take the that key yet, and cannot decrypt it. + // + // We keep keys for 12 hours. Thus the maximum ticket vector size + // is 12 + 1. if (old_ticket_keys) { auto &old_keys = old_ticket_keys->keys; auto &new_keys = ticket_keys->keys; - assert(!old_keys.empty()); + assert(old_keys.size() >= 2); - new_keys.resize(2); - new_keys[1] = old_keys[0]; + new_keys.resize(std::min(13ul, old_keys.size() + 1)); + std::copy_n(std::begin(old_keys), new_keys.size() - 2, + std::begin(new_keys) + 1); + new_keys[0] = old_keys.back(); } else { - ticket_keys->keys.resize(1); - } - - auto &new_key = ticket_keys->keys[0]; - new_key.cipher = get_config()->tls_ticket_cipher; - new_key.hmac = EVP_sha256(); - new_key.hmac_keylen = EVP_MD_size(new_key.hmac); - - assert(static_cast(EVP_CIPHER_key_length(new_key.cipher)) <= - sizeof(new_key.data.enc_key)); - assert(new_key.hmac_keylen <= sizeof(new_key.data.hmac_key)); - - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "enc_keylen=" << EVP_CIPHER_key_length(new_key.cipher) - << ", hmac_keylen=" << new_key.hmac_keylen; - } - - if (RAND_bytes(reinterpret_cast(&new_key.data), - sizeof(new_key.data)) == 0) { - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "failed to renew ticket key"; + ticket_keys->keys.resize(2); + if (generate_ticket_key(ticket_keys->keys[0]) != 0) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "failed to generate ticket key"; + } + conn_handler->set_ticket_keys(nullptr); + conn_handler->worker_renew_ticket_keys(nullptr); + return; } + } + + auto &new_key = ticket_keys->keys.back(); + + if (generate_ticket_key(new_key) != 0) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "failed to generate ticket key"; + } + conn_handler->set_ticket_keys(nullptr); + conn_handler->worker_renew_ticket_keys(nullptr); return; } if (LOG_ENABLED(INFO)) { LOG(INFO) << "ticket keys generation done"; - for (auto &key : ticket_keys->keys) { - LOG(INFO) << "name: " << util::format_hex(key.data.name); + assert(ticket_keys->keys.size() >= 2); + LOG(INFO) << "enc+dec: " + << util::format_hex(ticket_keys->keys[0].data.name); + for (size_t i = 1; i < ticket_keys->keys.size() - 1; ++i) { + auto &key = ticket_keys->keys[i]; + LOG(INFO) << "dec: " << util::format_hex(key.data.name); } + LOG(INFO) << "dec, next enc: " + << util::format_hex(ticket_keys->keys.back().data.name); } conn_handler->set_ticket_keys(ticket_keys); @@ -740,8 +777,8 @@ int event_loop() { } } if (auto_tls_ticket_key) { - // Renew ticket key every 12hrs - ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 12_h); + // Generate new ticket key every 1hr. + ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 1_h); renew_ticket_key_timer.data = conn_handler.get(); ev_timer_again(loop, &renew_ticket_key_timer); @@ -1317,9 +1354,10 @@ SSL/TLS: opening or reading given file fails, all loaded keys are discarded and it is treated as if none of this option is given. If this option is not given or an error occurred - while opening or reading a file, key is generated - automatically and renewed every 12hrs. At most 2 keys - are stored in memory. + while opening or reading a file, key is generated every + 1 hour internally and they are valid for 12 hours. This + is recommended if ticket key sharing between nghttpx + instances is not required. --tls-ticket-cipher= Specify cipher to encrypt TLS session ticket. Specify either aes-128-cbc or aes-256-cbc. By default, From 04bd25d468aee33c03555146cf456505a0cfa9d2 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 23 Jul 2015 23:13:29 +0900 Subject: [PATCH 71/77] nghttpx: Simplify ticket handling between workers just using mutex --- src/shrpx.cc | 33 +++++++++++---------------------- src/shrpx_connection_handler.cc | 19 +++++++------------ src/shrpx_connection_handler.h | 3 ++- src/shrpx_ssl.cc | 2 +- src/shrpx_worker.cc | 10 +++------- src/shrpx_worker.h | 7 +++++-- 6 files changed, 29 insertions(+), 45 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 8b171ae1..aa97762e 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -649,50 +649,39 @@ void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) { auto &old_keys = old_ticket_keys->keys; auto &new_keys = ticket_keys->keys; - assert(old_keys.size() >= 2); + assert(!old_keys.empty()); - new_keys.resize(std::min(13ul, old_keys.size() + 1)); - std::copy_n(std::begin(old_keys), new_keys.size() - 2, + new_keys.resize(std::min(12ul, old_keys.size() + 1)); + std::copy_n(std::begin(old_keys), new_keys.size() - 1, std::begin(new_keys) + 1); - new_keys[0] = old_keys.back(); } else { - ticket_keys->keys.resize(2); - if (generate_ticket_key(ticket_keys->keys[0]) != 0) { - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "failed to generate ticket key"; - } - conn_handler->set_ticket_keys(nullptr); - conn_handler->worker_renew_ticket_keys(nullptr); - return; - } + ticket_keys->keys.resize(1); } - auto &new_key = ticket_keys->keys.back(); + auto &new_key = ticket_keys->keys[0]; if (generate_ticket_key(new_key) != 0) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "failed to generate ticket key"; } conn_handler->set_ticket_keys(nullptr); - conn_handler->worker_renew_ticket_keys(nullptr); + conn_handler->set_ticket_keys_to_worker(nullptr); return; } if (LOG_ENABLED(INFO)) { LOG(INFO) << "ticket keys generation done"; - assert(ticket_keys->keys.size() >= 2); - LOG(INFO) << "enc+dec: " + assert(ticket_keys->keys.size() >= 1); + LOG(INFO) << 0 << " enc+dec: " << util::format_hex(ticket_keys->keys[0].data.name); - for (size_t i = 1; i < ticket_keys->keys.size() - 1; ++i) { + for (size_t i = 1; i < ticket_keys->keys.size(); ++i) { auto &key = ticket_keys->keys[i]; - LOG(INFO) << "dec: " << util::format_hex(key.data.name); + LOG(INFO) << i << " dec: " << util::format_hex(key.data.name); } - LOG(INFO) << "dec, next enc: " - << util::format_hex(ticket_keys->keys.back().data.name); } conn_handler->set_ticket_keys(ticket_keys); - conn_handler->worker_renew_ticket_keys(ticket_keys); + conn_handler->set_ticket_keys_to_worker(ticket_keys); } } // namespace diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 91b477d2..64260e9d 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -128,6 +128,13 @@ ConnectionHandler::~ConnectionHandler() { } } +void ConnectionHandler::set_ticket_keys_to_worker( + const std::shared_ptr &ticket_keys) { + for (auto &worker : workers_) { + worker->set_ticket_keys(ticket_keys); + } +} + void ConnectionHandler::worker_reopen_log_files() { WorkerEvent wev{}; @@ -138,18 +145,6 @@ void ConnectionHandler::worker_reopen_log_files() { } } -void ConnectionHandler::worker_renew_ticket_keys( - const std::shared_ptr &ticket_keys) { - WorkerEvent wev{}; - - wev.type = RENEW_TICKET_KEYS; - wev.ticket_keys = ticket_keys; - - for (auto &worker : workers_) { - worker->send(wev); - } -} - void ConnectionHandler::create_single_worker() { auto cert_tree = ssl::create_cert_lookup_tree(); auto sv_ssl_ctx = ssl::setup_server_ssl_context(all_ssl_ctx_, cert_tree); diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h index 5b17fef2..c6488854 100644 --- a/src/shrpx_connection_handler.h +++ b/src/shrpx_connection_handler.h @@ -76,8 +76,9 @@ public: // Creates |num| Worker objects for multi threaded configuration. // The |num| must be strictly more than 1. void create_worker_thread(size_t num); + void + set_ticket_keys_to_worker(const std::shared_ptr &ticket_keys); void worker_reopen_log_files(); - void worker_renew_ticket_keys(const std::shared_ptr &ticket_keys); void set_ticket_keys(std::shared_ptr ticket_keys); const std::shared_ptr &get_ticket_keys() const; struct ev_loop *get_loop() const; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index ea5a1cc1..2d4635cd 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -188,7 +188,7 @@ int ticket_key_cb(SSL *ssl, unsigned char *key_name, unsigned char *iv, EVP_CIPHER_CTX *ctx, HMAC_CTX *hctx, int enc) { auto handler = static_cast(SSL_get_app_data(ssl)); auto worker = handler->get_worker(); - const auto &ticket_keys = worker->get_ticket_keys(); + auto ticket_keys = worker->get_ticket_keys(); if (!ticket_keys) { // No ticket keys available. diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index ecf55b51..449d4c48 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -172,12 +172,6 @@ void Worker::process_events() { break; } - case RENEW_TICKET_KEYS: - WLOG(NOTICE, this) << "Renew ticket keys: worker(" << this << ")"; - - ticket_keys_ = wev.ticket_keys; - - break; case REOPEN_LOG: WLOG(NOTICE, this) << "Reopening log files: worker(" << this << ")"; @@ -206,11 +200,13 @@ void Worker::process_events() { ssl::CertLookupTree *Worker::get_cert_lookup_tree() const { return cert_tree_; } -const std::shared_ptr &Worker::get_ticket_keys() const { +std::shared_ptr Worker::get_ticket_keys() { + std::lock_guard g(m_); return ticket_keys_; } void Worker::set_ticket_keys(std::shared_ptr ticket_keys) { + std::lock_guard g(m_); ticket_keys_ = std::move(ticket_keys); } diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index b5f820b4..1ad7e3b1 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -75,7 +75,6 @@ enum WorkerEventType { NEW_CONNECTION = 0x01, REOPEN_LOG = 0x02, GRACEFUL_SHUTDOWN = 0x03, - RENEW_TICKET_KEYS = 0x04, }; struct WorkerEvent { @@ -100,8 +99,12 @@ public: void send(const WorkerEvent &event); ssl::CertLookupTree *get_cert_lookup_tree() const; - const std::shared_ptr &get_ticket_keys() const; + + // These 2 functions make a lock m_ to get/set ticket keys + // atomically. + std::shared_ptr get_ticket_keys(); void set_ticket_keys(std::shared_ptr ticket_keys); + WorkerStat *get_worker_stat(); DownstreamConnectionPool *get_dconn_pool(); Http2Session *next_http2_session(size_t group); From cab6c7871cbc73be7f31710e3537e812eab1a902 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 23 Jul 2015 23:54:56 +0900 Subject: [PATCH 72/77] nghttpx: Don't rewrite host header field by default In reverse proxy usage, backend server most likely wants to see the original header field. So this commit turns off host header rewrite by default. --no-host-rewrite option is deprecated, and if it is used, warning message is displayed. --host-rewrite option is added to enable host rewrite. --- gennghttpxfun.py | 1 + src/shrpx.cc | 11 ++++++++--- src/shrpx_config.cc | 13 ++++++++++++- src/shrpx_config.h | 1 + 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 9bf1c012..2c04e7d5 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -92,6 +92,7 @@ OPTIONS = [ "max-header-fields", "include", "tls-ticket-cipher", + "host-rewrite", "conf", ] diff --git a/src/shrpx.cc b/src/shrpx.cc index aa97762e..4c59167f 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -995,7 +995,7 @@ void fill_default_config() { mod_config()->tls_proto_mask = 0; mod_config()->no_location_rewrite = false; - mod_config()->no_host_rewrite = false; + mod_config()->no_host_rewrite = true; mod_config()->argc = 0; mod_config()->argv = nullptr; mod_config()->downstream_connections_per_host = 8; @@ -1498,8 +1498,8 @@ HTTP: --client and default mode. For --http2-proxy and --client-proxy mode, location header field will not be altered regardless of this option. - --no-host-rewrite - Don't rewrite host and :authority header fields on + --host-rewrite + Rewrite host and :authority header fields on --http2-bridge, --client and default mode. For --http2-proxy and --client-proxy mode, these headers will not be altered regardless of this option. @@ -1718,6 +1718,7 @@ int main(int argc, char **argv) { {SHRPX_OPT_ADD_REQUEST_HEADER, required_argument, &flag, 82}, {SHRPX_OPT_INCLUDE, required_argument, &flag, 83}, {SHRPX_OPT_TLS_TICKET_CIPHER, required_argument, &flag, 84}, + {SHRPX_OPT_HOST_REWRITE, no_argument, &flag, 85}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2088,6 +2089,10 @@ int main(int argc, char **argv) { // --tls-ticket-cipher cmdcfgs.emplace_back(SHRPX_OPT_TLS_TICKET_CIPHER, optarg); break; + case 85: + // --host-rewrite + cmdcfgs.emplace_back(SHRPX_OPT_HOST_REWRITE, "yes"); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 3fae2d29..8029d0bb 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -674,6 +674,7 @@ enum { SHRPX_OPTID_FRONTEND_READ_TIMEOUT, SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT, SHRPX_OPTID_HEADER_FIELD_BUFFER, + SHRPX_OPTID_HOST_REWRITE, SHRPX_OPTID_HTTP2_BRIDGE, SHRPX_OPTID_HTTP2_MAX_CONCURRENT_STREAMS, SHRPX_OPTID_HTTP2_NO_COOKIE_CRUMBLING, @@ -881,6 +882,9 @@ int option_lookup_token(const char *name, size_t namelen) { } break; case 'e': + if (util::strieq_l("host-rewrit", name, 11)) { + return SHRPX_OPTID_HOST_REWRITE; + } if (util::strieq_l("http2-bridg", name, 11)) { return SHRPX_OPTID_HTTP2_BRIDGE; } @@ -1736,7 +1740,10 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_NO_HOST_REWRITE: - mod_config()->no_host_rewrite = util::strieq(optarg, "yes"); + LOG(WARN) << SHRPX_OPT_NO_HOST_REWRITE + << ": deprecated. :authority and host header fields are NOT " + "altered by default. To rewrite these headers, use " + "--host-rewrite option."; return 0; case SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST: { @@ -1853,6 +1860,10 @@ int parse_config(const char *opt, const char *optarg, } mod_config()->tls_ticket_cipher_given = true; + return 0; + case SHRPX_OPTID_HOST_REWRITE: + mod_config()->no_host_rewrite = !util::strieq(optarg, "yes"); + return 0; case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 267decf9..749c7399 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -172,6 +172,7 @@ constexpr char SHRPX_OPT_HEADER_FIELD_BUFFER[] = "header-field-buffer"; constexpr char SHRPX_OPT_MAX_HEADER_FIELDS[] = "max-header-fields"; constexpr char SHRPX_OPT_INCLUDE[] = "include"; constexpr char SHRPX_OPT_TLS_TICKET_CIPHER[] = "tls-ticket-cipher"; +constexpr char SHRPX_OPT_HOST_REWRITE[] = "host-rewrite"; union sockaddr_union { sockaddr_storage storage; From 9b63fc011ea648ab85d832b6cc49c43b02d297e1 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 24 Jul 2015 00:32:19 +0900 Subject: [PATCH 73/77] nghttpx: Open log files by default configuration --- src/shrpx.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 4c59167f..c17f604e 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1601,6 +1601,10 @@ int main(int argc, char **argv) { create_config(); fill_default_config(); + // First open log files with default configuration, so that we can + // log errors/warnings while reading configuration files. + reopen_log_files(); + // We have to copy argv, since getopt_long may change its content. mod_config()->argc = argc; mod_config()->argv = new char *[argc]; @@ -2117,8 +2121,7 @@ int main(int argc, char **argv) { cmdcfgs.emplace_back(SHRPX_OPT_CERTIFICATE_FILE, argv[optind++]); } - // First open default log files to deal with errors occurred while - // parsing option values. + // Reopen log files using configurations in file reopen_log_files(); { From abce7c7210306dae8cfebab17c572b2e8882e6c4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 24 Jul 2015 23:25:10 +0900 Subject: [PATCH 74/77] nghttpd: Fix the bug that 304 response has non-empty body --- src/HttpServer.cc | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/HttpServer.cc b/src/HttpServer.cc index 6936bc85..5241aec6 100644 --- a/src/HttpServer.cc +++ b/src/HttpServer.cc @@ -1102,7 +1102,7 @@ void prepare_response(Stream *stream, Http2Handler *hd, if (last_mod_found && static_cast(buf.st_mtime) <= last_mod) { close(file); - prepare_status_response(stream, hd, 304); + hd->submit_response("304", stream->stream_id, nullptr); return; } @@ -1111,7 +1111,7 @@ void prepare_response(Stream *stream, Http2Handler *hd, path, FileEntry(path, buf.st_size, buf.st_mtime, file)); } else if (last_mod_found && file_ent->mtime <= last_mod) { sessions->release_fd(file_ent->path); - prepare_status_response(stream, hd, 304); + hd->submit_response("304", stream->stream_id, nullptr); return; } @@ -1630,7 +1630,6 @@ FileEntry make_status_body(int status, uint16_t port) { enum { IDX_200, IDX_301, - IDX_304, IDX_400, IDX_404, }; @@ -1639,7 +1638,6 @@ HttpServer::HttpServer(const Config *config) : config_(config) { status_pages_ = std::vector{ {"200", make_status_body(200, config_->port)}, {"301", make_status_body(301, config_->port)}, - {"304", make_status_body(304, config_->port)}, {"400", make_status_body(400, config_->port)}, {"404", make_status_body(404, config_->port)}, }; @@ -1884,8 +1882,6 @@ const StatusPage *HttpServer::get_status_page(int status) const { return &status_pages_[IDX_200]; case 301: return &status_pages_[IDX_301]; - case 304: - return &status_pages_[IDX_304]; case 400: return &status_pages_[IDX_400]; case 404: From afbb99ecf7804413a0ec0347714a04d23de40c0f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 24 Jul 2015 23:40:27 +0900 Subject: [PATCH 75/77] nghttpx: Enable session resumption on HTTP/2 backend --- src/shrpx_connection.cc | 17 +++++++++++++---- src/shrpx_http2_session.cc | 13 ++++++++----- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/shrpx_connection.cc b/src/shrpx_connection.cc index 30c2ec98..1ba8dfdb 100644 --- a/src/shrpx_connection.cc +++ b/src/shrpx_connection.cc @@ -62,7 +62,13 @@ Connection::Connection(struct ev_loop *loop, int fd, SSL *ssl, tls.last_write_time = 0.; } -Connection::~Connection() { disconnect(); } +Connection::~Connection() { + disconnect(); + + if (tls.ssl) { + SSL_free(tls.ssl); + } +} void Connection::disconnect() { ev_timer_stop(loop, &rt); @@ -75,9 +81,12 @@ void Connection::disconnect() { SSL_set_app_data(tls.ssl, nullptr); SSL_set_shutdown(tls.ssl, SSL_RECEIVED_SHUTDOWN); ERR_clear_error(); - SSL_shutdown(tls.ssl); - SSL_free(tls.ssl); - tls.ssl = nullptr; + // To reuse SSL/TLS session, we have to shutdown, and don't free + // tls.ssl. + if (SSL_shutdown(tls.ssl) != 1) { + SSL_free(tls.ssl); + tls.ssl = nullptr; + } } if (fd != -1) { diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index e2b4e2ee..164a57da 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -320,12 +320,15 @@ int Http2Session::initiate_connection() { SSLOG(INFO, this) << "Connecting to downstream server"; } if (ssl_ctx_) { - // We are establishing TLS connection. - conn_.tls.ssl = SSL_new(ssl_ctx_); + // We are establishing TLS connection. If conn_.tls.ssl, we may + // reuse the previous session. if (!conn_.tls.ssl) { - SSLOG(ERROR, this) << "SSL_new() failed: " - << ERR_error_string(ERR_get_error(), NULL); - return -1; + conn_.tls.ssl = SSL_new(ssl_ctx_); + if (!conn_.tls.ssl) { + SSLOG(ERROR, this) << "SSL_new() failed: " + << ERR_error_string(ERR_get_error(), NULL); + return -1; + } } const char *sni_name = nullptr; From 6c8243f6d2e1766a80df6fe72c378dd8a461aa9d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 24 Jul 2015 23:40:44 +0900 Subject: [PATCH 76/77] nghttpd: Add verbose output when SSL/TLS session is reused --- src/HttpServer.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/HttpServer.cc b/src/HttpServer.cc index 5241aec6..01624d33 100644 --- a/src/HttpServer.cc +++ b/src/HttpServer.cc @@ -571,6 +571,12 @@ int Http2Handler::tls_handshake() { return -1; } + if (sessions_->get_config()->verbose) { + if (SSL_session_reused(ssl_)) { + std::cerr << "SSL/TLS session reused" << std::endl; + } + } + return 0; } From adec2c06bf72f44da109219102374bf25a6919e0 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 24 Jul 2015 23:59:19 +0900 Subject: [PATCH 77/77] nghttpx: Set SSL/TLS session timeout to 12 hours --- src/shrpx.cc | 11 ++++++++--- src/shrpx_config.h | 1 + src/shrpx_ssl.cc | 1 + 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index c17f604e..01d15205 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -643,15 +643,19 @@ void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) { // possible problem when one worker encrypt new key, but one worker, // which did not take the that key yet, and cannot decrypt it. // - // We keep keys for 12 hours. Thus the maximum ticket vector size - // is 12 + 1. + // We keep keys for get_config()->tls_session_timeout seconds. The + // default is 12 hours. Thus the maximum ticket vector size is 12. if (old_ticket_keys) { auto &old_keys = old_ticket_keys->keys; auto &new_keys = ticket_keys->keys; assert(!old_keys.empty()); - new_keys.resize(std::min(12ul, old_keys.size() + 1)); + auto max_tickets = + static_cast(std::chrono::duration_cast( + get_config()->tls_session_timeout).count()); + + new_keys.resize(std::min(max_tickets, old_keys.size() + 1)); std::copy_n(std::begin(old_keys), new_keys.size() - 1, std::begin(new_keys) + 1); } else { @@ -1016,6 +1020,7 @@ void fill_default_config() { mod_config()->downstream_addr_group_catch_all = 0; mod_config()->tls_ticket_cipher = EVP_aes_128_cbc(); mod_config()->tls_ticket_cipher_given = false; + mod_config()->tls_session_timeout = std::chrono::hours(12); } } // namespace diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 749c7399..9f0e52f8 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -253,6 +253,7 @@ struct Config { std::vector tls_proto_list; // binary form of http proxy host and port sockaddr_union downstream_http_proxy_addr; + std::chrono::seconds tls_session_timeout; ev_tstamp http2_upstream_read_timeout; ev_tstamp upstream_read_timeout; ev_tstamp upstream_write_timeout; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 2d4635cd..0681888e 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -346,6 +346,7 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const unsigned char sid_ctx[] = "shrpx"; SSL_CTX_set_session_id_context(ssl_ctx, sid_ctx, sizeof(sid_ctx) - 1); SSL_CTX_set_session_cache_mode(ssl_ctx, SSL_SESS_CACHE_SERVER); + SSL_CTX_set_timeout(ssl_ctx, get_config()->tls_session_timeout.count()); const char *ciphers; if (get_config()->ciphers) {