From 3119fc259cd29513c9aeef1371bdd6807a51b850 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 10 Jul 2015 02:52:11 +0900 Subject: [PATCH 01/44] Select backend based on request host and path by extending -b option -b option syntax is now ,[;[:...]]. The optional s specify the request host and path it is used for. The can contain path, host + path or host. The matching rule is closely designed to ServeMux in Go programming language. --- src/http2.h | 36 ++++ src/http2_test.cc | 27 +++ src/http2_test.h | 1 + src/shrpx-unittest.cc | 4 + src/shrpx.cc | 151 +++++++++++---- src/shrpx_client_handler.cc | 45 ++++- src/shrpx_client_handler.h | 3 +- src/shrpx_config.cc | 226 +++++++++++++++++++++-- src/shrpx_config.h | 38 +++- src/shrpx_config_test.cc | 66 ++++++- src/shrpx_config_test.h | 1 + src/shrpx_downstream_connection.h | 1 + src/shrpx_downstream_connection_pool.cc | 27 ++- src/shrpx_downstream_connection_pool.h | 6 +- src/shrpx_http2_downstream_connection.cc | 2 +- src/shrpx_http2_downstream_connection.h | 2 + src/shrpx_http2_session.cc | 22 ++- src/shrpx_http2_upstream.cc | 4 +- src/shrpx_http_downstream_connection.cc | 32 ++-- src/shrpx_http_downstream_connection.h | 4 +- src/shrpx_https_upstream.cc | 4 +- src/shrpx_spdy_upstream.cc | 4 +- src/shrpx_ssl.cc | 6 +- src/shrpx_ssl.h | 3 +- src/shrpx_worker.cc | 7 +- src/shrpx_worker.h | 8 +- src/util.h | 6 +- 27 files changed, 613 insertions(+), 123 deletions(-) diff --git a/src/http2.h b/src/http2.h index 9b7bf94b..be79caca 100644 --- a/src/http2.h +++ b/src/http2.h @@ -37,6 +37,8 @@ #include "http-parser/http_parser.h" +#include "util.h" + namespace nghttp2 { struct Header { @@ -298,6 +300,40 @@ int lookup_method_token(const std::string &name); const char *to_method_string(int method_token); +template +std::string normalize_path(InputIt first, InputIt last) { + // First, decode %XX for unreserved characters, then do + // http2::join_path + std::string result; + // We won't find %XX if length is less than 3. + if (last - first < 3) { + result.assign(first, last); + } else { + for (; first < last - 2;) { + if (*first == '%') { + if (util::isHexDigit(*(first + 1)) && util::isHexDigit(*(first + 2))) { + auto c = (util::hex_to_uint(*(first + 1)) << 4) + + util::hex_to_uint(*(first + 2)); + if (util::inRFC3986UnreservedChars(c)) { + result += c; + first += 3; + continue; + } + result += '%'; + result += util::upcase(*(first + 1)); + result += util::upcase(*(first + 2)); + first += 3; + continue; + } + } + result += *first++; + } + result.append(first, last); + } + return path_join(nullptr, 0, nullptr, 0, result.c_str(), result.size(), + nullptr, 0); +} + } // namespace http2 } // namespace nghttp2 diff --git a/src/http2_test.cc b/src/http2_test.cc index a11df674..1937bfaf 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -829,4 +829,31 @@ void test_http2_path_join(void) { } } +void test_http2_normalize_path(void) { + std::string src; + + src = "/alpha/bravo/../charlie"; + CU_ASSERT("/alpha/charlie" == + http2::normalize_path(std::begin(src), std::end(src))); + + src = "/a%6c%70%68%61"; + CU_ASSERT("/alpha" == http2::normalize_path(std::begin(src), std::end(src))); + + src = "/alpha%2f%3a"; + CU_ASSERT("/alpha%2F%3A" == + http2::normalize_path(std::begin(src), std::end(src))); + + src = "%2f"; + CU_ASSERT("/%2F" == http2::normalize_path(std::begin(src), std::end(src))); + + src = "%f"; + CU_ASSERT("/%f" == http2::normalize_path(std::begin(src), std::end(src))); + + src = "%"; + CU_ASSERT("/%" == http2::normalize_path(std::begin(src), std::end(src))); + + src = ""; + CU_ASSERT("/" == http2::normalize_path(std::begin(src), std::end(src))); +} + } // namespace shrpx diff --git a/src/http2_test.h b/src/http2_test.h index 36c5a59a..193f23e3 100644 --- a/src/http2_test.h +++ b/src/http2_test.h @@ -45,6 +45,7 @@ void test_http2_http2_header_allowed(void); void test_http2_mandatory_request_headers_presence(void); void test_http2_parse_link_header(void); void test_http2_path_join(void); +void test_http2_normalize_path(void); } // namespace shrpx diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index d4e332ae..7030aa3d 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -96,6 +96,8 @@ int main(int argc, char *argv[]) { !CU_add_test(pSuite, "http2_parse_link_header", shrpx::test_http2_parse_link_header) || !CU_add_test(pSuite, "http2_path_join", shrpx::test_http2_path_join) || + !CU_add_test(pSuite, "http2_normalize_path", + shrpx::test_http2_normalize_path) || !CU_add_test(pSuite, "downstream_index_request_headers", shrpx::test_downstream_index_request_headers) || !CU_add_test(pSuite, "downstream_index_response_headers", @@ -118,6 +120,8 @@ int main(int argc, char *argv[]) { shrpx::test_shrpx_config_parse_log_format) || !CU_add_test(pSuite, "config_read_tls_ticket_key_file", shrpx::test_shrpx_config_read_tls_ticket_key_file) || + !CU_add_test(pSuite, "config_match_downstream_addr_group", + shrpx::test_shrpx_config_match_downstream_addr_group) || !CU_add_test(pSuite, "util_streq", shrpx::test_util_streq) || !CU_add_test(pSuite, "util_strieq", shrpx::test_util_strieq) || !CU_add_test(pSuite, "util_inp_strlower", diff --git a/src/shrpx.cc b/src/shrpx.cc index 92046aed..8bfadbc3 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -966,6 +966,7 @@ void fill_default_config() { mod_config()->no_ocsp = false; mod_config()->header_field_buffer = 64_k; mod_config()->max_header_fields = 100; + mod_config()->downstream_addr_group_catch_all = 0; } } // namespace @@ -997,14 +998,45 @@ Options: The options are categorized into several groups. Connections: - -b, --backend= + -b, --backend=,[;[:...]] Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX domain socket can be specified by prefixing path name - with "unix:" (e.g., unix:/var/run/backend.sock) + with "unix:" (e.g., unix:/var/run/backend.sock). + Optionally, if s are given, the backend address + is only used if request matches the pattern. If -s, -p, + --client or --http2-bridge is used, s are + ignored. The pattern matching is closely designed to + ServeMux in net/http package of Go programming language. + consists of path, host + path or host. The + path must starts with "/". If it ends with "/", it + matches to the request path whose prefix is the path. + If it does not end with "/", it performs exact match + against the request path. If host is given, it performs + exact match against the request host. If host alone is + given, "/" is appended to it, so that it matches all + paths under the host (e.g., specifying "nghttp2.org" + equals to "nghttp2.org/"). Longer patterns take + precedence over shorter ones, breaking a tie by the + order of the appearance in the configuration. If + is omitted, "/" is used as pattern, which + matches all paths (catch-all pattern). For example, + -b'127.0.0.1,8080;nghttp2.org/httpbin/' matches the + request host "nghttp2.org" and the request path + "/httpbin/get", but does not match the request host + "nghttp2.org" and the request path "/index.html". The + multiple s can be specified, delimiting them by + ":". Specifying + -b'127.0.0.1,8080;nghttp2.org:www.nghttp2.org' has the + same effect to specify -b'127.0.0.1,8080;nghttp2.org' + and -b'127.0.0.1,8080:www.nghttp2.org'. The backend + addresses sharing same are grouped together + forming load balancing group. Since ";" and ":" are + used as delimiter, must not contain these + characters. Default: )" << DEFAULT_DOWNSTREAM_HOST << "," << DEFAULT_DOWNSTREAM_PORT << R"( - -f, --frontend= + -f, --frontend=, Set frontend host and port. If is '*', it assumes all addresses including both IPv4 and IPv6. UNIX domain socket can be specified by prefixing path @@ -2118,55 +2150,97 @@ int main(int argc, char **argv) { } } - if (get_config()->downstream_addrs.empty()) { + if (get_config()->downstream_addr_groups.empty()) { DownstreamAddr addr; addr.host = strcopy(DEFAULT_DOWNSTREAM_HOST); addr.port = DEFAULT_DOWNSTREAM_PORT; - mod_config()->downstream_addrs.push_back(std::move(addr)); + DownstreamAddrGroup g("/"); + g.addrs.push_back(std::move(addr)); + mod_config()->downstream_addr_groups.push_back(std::move(g)); + } else if (get_config()->downstream_proto == PROTO_HTTP2 || + get_config()->http2_proxy || get_config()->client_proxy) { + // We don't support host mapping in these cases. Move all + // non-catch-all patterns to catch-all pattern for HTTP/2 backend + DownstreamAddrGroup catch_all("/"); + for (auto &g : mod_config()->downstream_addr_groups) { + std::move(std::begin(g.addrs), std::end(g.addrs), + std::back_inserter(catch_all.addrs)); + } + std::vector().swap( + mod_config()->downstream_addr_groups); + mod_config()->downstream_addr_groups.push_back(std::move(catch_all)); } if (LOG_ENABLED(INFO)) { LOG(INFO) << "Resolving backend address"; } - for (auto &addr : mod_config()->downstream_addrs) { + ssize_t catch_all_group = -1; + for (size_t i = 0; i < mod_config()->downstream_addr_groups.size(); ++i) { + auto &g = mod_config()->downstream_addr_groups[i]; + if (g.pattern == "/") { + catch_all_group = i; + } + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Host-path pattern: group " << i << ": '" << g.pattern + << "'"; + for (auto &addr : g.addrs) { + LOG(INFO) << "group " << i << " -> " << addr.host.get() + << (addr.host_unix ? "" : ":" + util::utos(addr.port)); + } + } + } - if (addr.host_unix) { - // for AF_UNIX socket, we use "localhost" as host for backend - // hostport. This is used as Host header field to backend and - // not going to be passed to any syscalls. - addr.hostport = - strcopy(util::make_hostport("localhost", get_config()->port)); + if (catch_all_group == -1) { + LOG(FATAL) << "-b: No catch-all backend address is configured"; + exit(EXIT_FAILURE); + } + mod_config()->downstream_addr_group_catch_all = catch_all_group; - auto path = addr.host.get(); - auto pathlen = strlen(path); + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Catch-all pattern is group " << catch_all_group; + } - if (pathlen + 1 > sizeof(addr.addr.un.sun_path)) { - LOG(FATAL) << "UNIX domain socket path " << path << " is too long > " - << sizeof(addr.addr.un.sun_path); - exit(EXIT_FAILURE); + for (auto &g : mod_config()->downstream_addr_groups) { + for (auto &addr : g.addrs) { + + if (addr.host_unix) { + // for AF_UNIX socket, we use "localhost" as host for backend + // hostport. This is used as Host header field to backend and + // not going to be passed to any syscalls. + addr.hostport = + strcopy(util::make_hostport("localhost", get_config()->port)); + + auto path = addr.host.get(); + auto pathlen = strlen(path); + + if (pathlen + 1 > sizeof(addr.addr.un.sun_path)) { + LOG(FATAL) << "UNIX domain socket path " << path << " is too long > " + << sizeof(addr.addr.un.sun_path); + exit(EXIT_FAILURE); + } + + LOG(INFO) << "Use UNIX domain socket path " << path + << " for backend connection"; + + addr.addr.un.sun_family = AF_UNIX; + // copy path including terminal NULL + std::copy_n(path, pathlen + 1, addr.addr.un.sun_path); + addr.addrlen = sizeof(addr.addr.un); + + continue; } - LOG(INFO) << "Use UNIX domain socket path " << path - << " for backend connection"; + addr.hostport = strcopy(util::make_hostport(addr.host.get(), addr.port)); - addr.addr.un.sun_family = AF_UNIX; - // copy path including terminal NULL - std::copy_n(path, pathlen + 1, addr.addr.un.sun_path); - addr.addrlen = sizeof(addr.addr.un); - - continue; - } - - addr.hostport = strcopy(util::make_hostport(addr.host.get(), addr.port)); - - if (resolve_hostname( - &addr.addr, &addr.addrlen, addr.host.get(), addr.port, - get_config()->backend_ipv4 - ? AF_INET - : (get_config()->backend_ipv6 ? AF_INET6 : AF_UNSPEC)) == -1) { - exit(EXIT_FAILURE); + if (resolve_hostname( + &addr.addr, &addr.addrlen, addr.host.get(), addr.port, + get_config()->backend_ipv4 ? AF_INET : (get_config()->backend_ipv6 + ? AF_INET6 + : AF_UNSPEC)) == -1) { + exit(EXIT_FAILURE); + } } } @@ -2183,9 +2257,10 @@ int main(int argc, char **argv) { } } - if (get_config()->http2_downstream_connections_per_worker == 0) { + if (get_config()->downstream_proto == PROTO_HTTP2 && + get_config()->http2_downstream_connections_per_worker == 0) { mod_config()->http2_downstream_connections_per_worker = - get_config()->downstream_addrs.size(); + get_config()->downstream_addr_groups[0].addrs.size(); } if (get_config()->rlimit_nofile) { diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index d727fe39..644a8cdc 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -588,7 +588,8 @@ void ClientHandler::pool_downstream_connection( return; } if (LOG_ENABLED(INFO)) { - CLOG(INFO, this) << "Pooling downstream connection DCONN:" << dconn.get(); + CLOG(INFO, this) << "Pooling downstream connection DCONN:" << dconn.get() + << " in group " << dconn->get_group(); } dconn->set_client_handler(nullptr); auto dconn_pool = worker_->get_dconn_pool(); @@ -605,9 +606,44 @@ void ClientHandler::remove_downstream_connection(DownstreamConnection *dconn) { } std::unique_ptr -ClientHandler::get_downstream_connection() { +ClientHandler::get_downstream_connection(Downstream *downstream) { + size_t group; + auto &groups = get_config()->downstream_addr_groups; + auto catch_all = get_config()->downstream_addr_group_catch_all; + + // Fast path. If we have one group, it must be catch-all group. + // HTTP/2 and client proxy modes fall in this case. Currently, + // HTTP/2 backend does not perform host-path mapping. + if (groups.size() == 1 || get_config()->downstream_proto == PROTO_HTTP2) { + group = 0; + } else if (downstream->get_request_method() == HTTP_CONNECT) { + // We don't know how to treat CONNECT request in host-path + // mapping. It most likely appears in proxy scenario. Since we + // have dealt with proxy case already, just use catch-all group. + group = catch_all; + } else { + if (!downstream->get_request_http2_authority().empty()) { + group = match_downstream_addr_group( + downstream->get_request_http2_authority(), + downstream->get_request_path(), groups, catch_all); + } else { + auto h = downstream->get_request_header(http2::HD_HOST); + if (h) { + group = match_downstream_addr_group( + h->value, downstream->get_request_path(), groups, catch_all); + } else { + group = match_downstream_addr_group("", downstream->get_request_path(), + groups, catch_all); + } + } + } + + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) << "Downstream address group: " << group; + } + auto dconn_pool = worker_->get_dconn_pool(); - auto dconn = dconn_pool->pop_downstream_connection(); + auto dconn = dconn_pool->pop_downstream_connection(group); if (!dconn) { if (LOG_ENABLED(INFO)) { @@ -620,7 +656,8 @@ ClientHandler::get_downstream_connection() { if (http2session_) { dconn = make_unique(dconn_pool, http2session_); } else { - dconn = make_unique(dconn_pool, conn_.loop); + dconn = + make_unique(dconn_pool, group, conn_.loop); } dconn->set_client_handler(this); return dconn; diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index a2df98eb..4f21f513 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -92,7 +92,8 @@ public: void pool_downstream_connection(std::unique_ptr dconn); void remove_downstream_connection(DownstreamConnection *dconn); - std::unique_ptr get_downstream_connection(); + std::unique_ptr + get_downstream_connection(Downstream *downstream); MemchunkPool *get_mcpool(); SSL *get_ssl() const; ConnectBlocker *get_connect_blocker() const; diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index d59a1d0d..3fdbe425 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -57,6 +57,7 @@ #include "http2.h" #include "util.h" #include "template.h" +#include "base64.h" using namespace nghttp2; @@ -79,9 +80,29 @@ TicketKeys::~TicketKeys() { } } +DownstreamAddr::DownstreamAddr(const DownstreamAddr &other) + : addr(other.addr), host(other.host ? strcopy(other.host.get()) : nullptr), + hostport(other.hostport ? strcopy(other.hostport.get()) : nullptr), + addrlen(other.addrlen), port(other.port), host_unix(other.host_unix) {} + +DownstreamAddr &DownstreamAddr::operator=(const DownstreamAddr &other) { + if (this == &other) { + return *this; + } + + addr = other.addr; + host = (other.host ? strcopy(other.host.get()) : nullptr); + hostport = (other.hostport ? strcopy(other.hostport.get()) : nullptr); + addrlen = other.addrlen; + port = other.port; + host_unix = other.host_unix; + + return *this; +} + namespace { int split_host_port(char *host, size_t hostlen, uint16_t *port_ptr, - const char *hostport) { + const char *hostport, size_t hostportlen) { // host and port in |hostport| is separated by single ','. const char *p = strchr(hostport, ','); if (!p) { @@ -97,12 +118,13 @@ int split_host_port(char *host, size_t hostlen, uint16_t *port_ptr, host[len] = '\0'; errno = 0; - unsigned long d = strtoul(p + 1, nullptr, 10); - if (errno == 0 && 1 <= d && d <= std::numeric_limits::max()) { + auto portlen = hostportlen - len - 1; + auto d = util::parse_uint(reinterpret_cast(p + 1), portlen); + if (1 <= d && d <= std::numeric_limits::max()) { *port_ptr = d; return 0; } else { - LOG(ERROR) << "Port is invalid: " << p + 1; + LOG(ERROR) << "Port is invalid: " << std::string(p + 1, portlen); return -1; } } @@ -220,16 +242,16 @@ std::unique_ptr strcopy(const std::string &val) { return strcopy(val.c_str(), val.size()); } -std::vector parse_config_str_list(const char *s) { +std::vector parse_config_str_list(const char *s, char delim) { size_t len = 1; - for (const char *first = s, *p = nullptr; (p = strchr(first, ',')); + for (const char *first = s, *p = nullptr; (p = strchr(first, delim)); ++len, first = p + 1) ; auto list = std::vector(len); auto first = strdup(s); len = 0; for (;;) { - auto p = strchr(first, ','); + auto p = strchr(first, delim); if (p == nullptr) { break; } @@ -440,30 +462,82 @@ int parse_duration(ev_tstamp *dest, const char *opt, const char *optarg) { } } // namespace +namespace { +// Parses host-path mapping patterns in |src|, and stores mappings in +// config. We will store each host-path pattern found in |src| with +// |addr|. |addr| will be copied accordingly. Also we make a group +// based on the pattern. The "/" pattern is considered as catch-all. +void parse_mapping(DownstreamAddr addr, const char *src) { + // This returns at least 1 element (it could be empty string). We + // will append '/' to all patterns, so it becomes catch-all pattern. + auto mapping = parse_config_str_list(src, ':'); + assert(!mapping.empty()); + for (auto raw_pattern : mapping) { + auto done = false; + std::string pattern; + auto slash = strchr(raw_pattern, '/'); + if (slash == nullptr) { + // This effectively makes empty pattern to "/". + pattern = raw_pattern; + util::inp_strlower(pattern); + pattern += "/"; + } else { + pattern.assign(raw_pattern, slash); + util::inp_strlower(pattern); + pattern += + http2::normalize_path(slash, raw_pattern + strlen(raw_pattern)); + } + for (auto &g : mod_config()->downstream_addr_groups) { + if (g.pattern == pattern) { + g.addrs.push_back(addr); + done = true; + break; + } + } + if (done) { + continue; + } + DownstreamAddrGroup g(pattern); + g.addrs.push_back(addr); + mod_config()->downstream_addr_groups.push_back(std::move(g)); + } + clear_config_str_list(mapping); +} +} // namespace + int parse_config(const char *opt, const char *optarg) { char host[NI_MAXHOST]; uint16_t port; + if (util::strieq(opt, SHRPX_OPT_BACKEND)) { + auto optarglen = strlen(optarg); + auto pat_delim = strchr(optarg, ';'); + if (!pat_delim) { + pat_delim = optarg + optarglen; + } + DownstreamAddr addr; if (util::istartsWith(optarg, SHRPX_UNIX_PATH_PREFIX)) { - DownstreamAddr addr; auto path = optarg + str_size(SHRPX_UNIX_PATH_PREFIX); addr.host = strcopy(path); addr.host_unix = true; + } else { + if (split_host_port(host, sizeof(host), &port, optarg, + pat_delim - optarg) == -1) { + return -1; + } - mod_config()->downstream_addrs.push_back(std::move(addr)); - - return 0; + addr.host = strcopy(host); + addr.port = port; } - if (split_host_port(host, sizeof(host), &port, optarg) == -1) { + auto mapping = pat_delim < optarg + optarglen ? pat_delim + 1 : pat_delim; + // We may introduce new parameter after additional ';', so don't + // allow extra ';' in pattern for now. + if (strchr(mapping, ';') != nullptr) { + LOG(ERROR) << opt << ": ';' must not be used in pattern"; return -1; } - - DownstreamAddr addr; - addr.host = strcopy(host); - addr.port = port; - - mod_config()->downstream_addrs.push_back(std::move(addr)); + parse_mapping(std::move(addr), mapping); return 0; } @@ -478,7 +552,8 @@ int parse_config(const char *opt, const char *optarg) { return 0; } - if (split_host_port(host, sizeof(host), &port, optarg) == -1) { + if (split_host_port(host, sizeof(host), &port, optarg, strlen(optarg)) == + -1) { return -1; } @@ -1322,4 +1397,117 @@ int int_syslog_facility(const char *strfacility) { return -1; } +namespace { +bool path_match(const std::string &pattern, const std::string &path) { + if (pattern.back() != '/') { + return pattern == path; + } + return util::startsWith(path, pattern); +} +} // namespace + +namespace { +ssize_t match(const std::string &path, + const std::vector &groups) { + ssize_t res = -1; + size_t best = 0; + for (size_t i = 0; i < groups.size(); ++i) { + auto &g = groups[i]; + auto &pattern = g.pattern; + if (!path_match(pattern, path)) { + continue; + } + if (res == -1 || best < pattern.size()) { + best = pattern.size(); + res = i; + } + } + return res; +} +} // namespace + +namespace { +size_t match_downstream_addr_group_host( + const std::string &host, const std::string &raw_path, + const std::vector &groups, size_t catch_all) { + if (raw_path == "*") { + auto group = match(host + "/", groups); + if (group != -1) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Found pattern with query " << host + << ", matched pattern=" << groups[group].pattern; + } + return group; + } + return catch_all; + } + + // probably, not necessary most of the case, but just in case. + auto fragment = std::find(std::begin(raw_path), std::end(raw_path), '#'); + auto query = std::find(std::begin(raw_path), fragment, '?'); + auto path = http2::normalize_path(std::begin(raw_path), query); + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Perform mapping selection, using host=" << host + << ", path=" << path; + } + + auto group = match(host + path, groups); + if (group != -1) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Found pattern with query " << host + path + << ", matched pattern=" << groups[group].pattern; + } + return group; + } + + group = match(path, groups); + if (group != -1) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Found pattern with query " << path + << ", matched pattern=" << groups[group].pattern; + } + return group; + } + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "None match. Use catch-all pattern"; + } + return catch_all; +} +} // namespace + +size_t match_downstream_addr_group( + const std::string &hostport, const std::string &raw_path, + const std::vector &groups, size_t catch_all) { + if (hostport.empty() || + std::find(std::begin(hostport), std::end(hostport), '/') != + std::end(hostport)) { + // We use '/' specially, and if '/' is included in host, it breaks + // our code. Select catch-all case. + return catch_all; + } + std::string host; + if (hostport[0] == '[') { + // assume this is IPv6 numeric address + auto p = std::find(std::begin(hostport), std::end(hostport), ']'); + if (p == std::end(hostport)) { + return catch_all; + } + if (p + 1 < std::end(hostport) && *(p + 1) != ':') { + return catch_all; + } + host.assign(std::begin(hostport), p + 1); + } else { + auto p = std::find(std::begin(hostport), std::end(hostport), ':'); + if (p == std::begin(hostport)) { + return catch_all; + } + host.assign(std::begin(hostport), p); + } + + util::inp_strlower(host); + return match_downstream_addr_group_host(host, raw_path, groups, catch_all); +} + } // namespace shrpx diff --git a/src/shrpx_config.h b/src/shrpx_config.h index c50ab64b..3e5c869e 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -194,6 +194,11 @@ struct AltSvc { struct DownstreamAddr { DownstreamAddr() : addr{{0}}, addrlen(0), port(0), host_unix(false) {} + DownstreamAddr(const DownstreamAddr &other); + DownstreamAddr(DownstreamAddr &&) = default; + DownstreamAddr &operator=(const DownstreamAddr &other); + DownstreamAddr &operator=(DownstreamAddr &&other) = default; + sockaddr_union addr; // backend address. If |host_unix| is true, this is UNIX domain // socket path. @@ -206,6 +211,12 @@ struct DownstreamAddr { bool host_unix; }; +struct DownstreamAddrGroup { + DownstreamAddrGroup(std::string pattern) : pattern(std::move(pattern)) {} + std::string pattern; + std::vector addrs; +}; + struct TicketKey { uint8_t name[16]; uint8_t aes_key[16]; @@ -225,7 +236,7 @@ struct Config { std::vector> add_response_headers; std::vector alpn_prefs; std::vector accesslog_format; - std::vector downstream_addrs; + std::vector downstream_addr_groups; std::vector tls_ticket_key_files; // binary form of http proxy host and port sockaddr_union downstream_http_proxy_addr; @@ -311,6 +322,8 @@ struct Config { size_t downstream_response_buffer_size; size_t header_field_buffer; size_t max_header_fields; + // The index of catch-all group in downstream_addr_groups. + size_t downstream_addr_group_catch_all; // Bit mask to disable SSL/TLS protocol versions. This will be // passed to SSL_CTX_set_options(). long int tls_proto_mask; @@ -376,15 +389,14 @@ int load_config(const char *filename); // Read passwd from |filename| std::string read_passwd_from_file(const char *filename); -// Parses comma delimited strings in |s| and returns the array of -// pointers, each element points to the each substring in |s|. The -// |s| must be comma delimited list of strings. The strings must be -// delimited by a single comma and any white spaces around it are -// treated as a part of protocol strings. This function may modify -// |s| and the caller must leave it as is after this call. This +// Parses delimited strings in |s| and returns the array of pointers, +// each element points to the each substring in |s|. The delimiter is +// given by |delim. The |s| must be comma delimited list of strings. +// The strings must be delimited by a single comma and any white +// spaces around it are treated as a part of protocol strings. This // function copies |s| and first element in the return value points to // it. It is caller's responsibility to deallocate its memory. -std::vector parse_config_str_list(const char *s); +std::vector parse_config_str_list(const char *s, char delim = ','); // Clears all elements of |list|, which is returned by // parse_config_str_list(). If list is not empty, list[0] is freed by @@ -423,6 +435,16 @@ FILE *open_file_for_write(const char *filename); std::unique_ptr read_tls_ticket_key_file(const std::vector &files); +// Selects group based on request's |hostport| and |path|. |hostport| +// is the value taken from :authority or host header field, and may +// contain port. The |path| may contain query part. We require the +// catch-all pattern in place, so this function always selects one +// group. The catch-all group index is given in |catch_all|. All +// patterns are given in |groups|. +size_t match_downstream_addr_group( + const std::string &hostport, const std::string &path, + const std::vector &groups, size_t catch_all); + } // namespace shrpx #endif // SHRPX_CONFIG_H diff --git a/src/shrpx_config_test.cc b/src/shrpx_config_test.cc index ad570bbc..4f21fc48 100644 --- a/src/shrpx_config_test.cc +++ b/src/shrpx_config_test.cc @@ -48,7 +48,7 @@ void test_shrpx_config_parse_config_str_list(void) { CU_ASSERT(0 == strcmp("", res[1])); clear_config_str_list(res); - res = parse_config_str_list(",a,,"); + res = parse_config_str_list(":a::", ':'); CU_ASSERT(4 == res.size()); CU_ASSERT(0 == strcmp("", res[0])); CU_ASSERT(0 == strcmp("a", res[1])); @@ -172,4 +172,68 @@ void test_shrpx_config_read_tls_ticket_key_file(void) { memcmp("a..............b", key->hmac_key, sizeof(key->hmac_key))); } +void test_shrpx_config_match_downstream_addr_group(void) { + auto groups = std::vector{ + {"nghttp2.org/"}, + {"nghttp2.org/alpha/bravo/"}, + {"nghttp2.org/alpha/charlie"}, + {"nghttp2.org/delta%3A"}, + {"www.nghttp2.org/"}, + {"[::1]/"}, + }; + + CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "/", groups, 255)); + + // port is removed + CU_ASSERT(0 == + match_downstream_addr_group("nghttp2.org:8080", "/", groups, 255)); + + // host is case-insensitive + CU_ASSERT(4 == match_downstream_addr_group("WWW.nghttp2.org", "/alpha", + groups, 255)); + + // path part is case-sensitive + CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "/Alpha/bravo", + groups, 255)); + + // unreserved characters are decoded before matching + CU_ASSERT(1 == match_downstream_addr_group("nghttp2.org", "/alpha/%62ravo/", + groups, 255)); + + CU_ASSERT(1 == match_downstream_addr_group( + "nghttp2.org", "/alpha/%62ravo/charlie", groups, 255)); + + CU_ASSERT(2 == match_downstream_addr_group("nghttp2.org", "/alpha/charlie", + groups, 255)); + + // pattern which does not end with '/' must match its entirely. So + // this matches to group 0, not group 2. + CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "/alpha/charlie/", + groups, 255)); + + // percent-encoding is normalized to upper case hex digits. + CU_ASSERT(3 == match_downstream_addr_group("nghttp2.org", "/delta%3a", groups, + 255)); + + // path component is normalized before mathcing + CU_ASSERT(1 == match_downstream_addr_group( + "nghttp2.org", "/alpha/charlie/%2e././bravo/delta/..", + groups, 255)); + + CU_ASSERT(255 == + match_downstream_addr_group("example.org", "/", groups, 255)); + + CU_ASSERT(255 == match_downstream_addr_group("", "/", groups, 255)); + + CU_ASSERT(255 == match_downstream_addr_group("foo/bar", "/", groups, 255)); + + // If path is "*", only match with host + "/". + CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "*", groups, 255)); + + CU_ASSERT(5 == match_downstream_addr_group("[::1]", "/", groups, 255)); + CU_ASSERT(5 == match_downstream_addr_group("[::1]:8080", "/", groups, 255)); + CU_ASSERT(255 == match_downstream_addr_group("[::1", "/", groups, 255)); + CU_ASSERT(255 == match_downstream_addr_group("[::1]8000", "/", groups, 255)); +} + } // namespace shrpx diff --git a/src/shrpx_config_test.h b/src/shrpx_config_test.h index 6d6d0353..5f9c170a 100644 --- a/src/shrpx_config_test.h +++ b/src/shrpx_config_test.h @@ -35,6 +35,7 @@ void test_shrpx_config_parse_config_str_list(void); void test_shrpx_config_parse_header(void); void test_shrpx_config_parse_log_format(void); void test_shrpx_config_read_tls_ticket_key_file(void); +void test_shrpx_config_match_downstream_addr_group(void); } // namespace shrpx diff --git a/src/shrpx_downstream_connection.h b/src/shrpx_downstream_connection.h index 49ca1beb..8d409f61 100644 --- a/src/shrpx_downstream_connection.h +++ b/src/shrpx_downstream_connection.h @@ -57,6 +57,7 @@ public: virtual void on_upstream_change(Upstream *uptream) = 0; virtual int on_priority_change(int32_t pri) = 0; + virtual size_t get_group() const = 0; // true if this object is poolable. virtual bool poolable() const = 0; diff --git a/src/shrpx_downstream_connection_pool.cc b/src/shrpx_downstream_connection_pool.cc index d762b770..eb9b1c8c 100644 --- a/src/shrpx_downstream_connection_pool.cc +++ b/src/shrpx_downstream_connection_pool.cc @@ -27,33 +27,42 @@ namespace shrpx { -DownstreamConnectionPool::DownstreamConnectionPool() {} +DownstreamConnectionPool::DownstreamConnectionPool(size_t num_groups) + : gpool_(num_groups) {} DownstreamConnectionPool::~DownstreamConnectionPool() { - for (auto dconn : pool_) { - delete dconn; + for (auto &pool : gpool_) { + for (auto dconn : pool) { + delete dconn; + } } } void DownstreamConnectionPool::add_downstream_connection( std::unique_ptr dconn) { - pool_.insert(dconn.release()); + auto group = dconn->get_group(); + assert(gpool_.size() > group); + gpool_[group].insert(dconn.release()); } std::unique_ptr -DownstreamConnectionPool::pop_downstream_connection() { - if (pool_.empty()) { +DownstreamConnectionPool::pop_downstream_connection(size_t group) { + assert(gpool_.size() > group); + auto &pool = gpool_[group]; + if (pool.empty()) { return nullptr; } - auto dconn = std::unique_ptr(*std::begin(pool_)); - pool_.erase(std::begin(pool_)); + auto dconn = std::unique_ptr(*std::begin(pool)); + pool.erase(std::begin(pool)); return dconn; } void DownstreamConnectionPool::remove_downstream_connection( DownstreamConnection *dconn) { - pool_.erase(dconn); + auto group = dconn->get_group(); + assert(gpool_.size() > group); + gpool_[group].erase(dconn); delete dconn; } diff --git a/src/shrpx_downstream_connection_pool.h b/src/shrpx_downstream_connection_pool.h index c2edce45..1fb889bb 100644 --- a/src/shrpx_downstream_connection_pool.h +++ b/src/shrpx_downstream_connection_pool.h @@ -36,15 +36,15 @@ class DownstreamConnection; class DownstreamConnectionPool { public: - DownstreamConnectionPool(); + DownstreamConnectionPool(size_t num_groups); ~DownstreamConnectionPool(); void add_downstream_connection(std::unique_ptr dconn); - std::unique_ptr pop_downstream_connection(); + std::unique_ptr pop_downstream_connection(size_t group); void remove_downstream_connection(DownstreamConnection *dconn); private: - std::set pool_; + std::vector> gpool_; }; } // namespace shrpx diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index 8a83f012..f82b5230 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -264,7 +264,7 @@ int Http2DownstreamConnection::push_request_headers() { // addr_idx here. auto addr_idx = http2session_->get_addr_idx(); auto downstream_hostport = - get_config()->downstream_addrs[addr_idx].hostport.get(); + get_config()->downstream_addr_groups[0].addrs[addr_idx].hostport.get(); const char *authority = nullptr, *host = nullptr; if (!no_host_rewrite) { diff --git a/src/shrpx_http2_downstream_connection.h b/src/shrpx_http2_downstream_connection.h index a2a78a34..736f546f 100644 --- a/src/shrpx_http2_downstream_connection.h +++ b/src/shrpx_http2_downstream_connection.h @@ -61,6 +61,8 @@ public: virtual void on_upstream_change(Upstream *upstream) {} virtual int on_priority_change(int32_t pri); + // Currently, HTTP/2 backend does not perform host-path mapping. + virtual size_t get_group() const { return 0; } // This object is not poolable because we dont' have facility to // migrate to another Http2Session object. diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index c071a722..5ed2f3c3 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -233,11 +233,16 @@ int Http2Session::disconnect(bool hard) { return 0; } -int Http2Session::check_cert() { return ssl::check_cert(conn_.tls.ssl); } +int Http2Session::check_cert() { + return ssl::check_cert( + conn_.tls.ssl, &get_config()->downstream_addr_groups[0].addrs[addr_idx_]); +} int Http2Session::initiate_connection() { int rv = 0; + auto &addrs = get_config()->downstream_addr_groups[0].addrs; + if (state_ == DISCONNECTED) { if (connect_blocker_->blocked()) { if (LOG_ENABLED(INFO)) { @@ -248,17 +253,19 @@ int Http2Session::initiate_connection() { } auto worker_stat = worker_->get_worker_stat(); - addr_idx_ = worker_stat->next_downstream; - ++worker_stat->next_downstream; - worker_stat->next_downstream %= get_config()->downstream_addrs.size(); + auto &next_downstream = worker_stat->next_downstream[0]; + addr_idx_ = next_downstream; + if (++next_downstream >= addrs.size()) { + next_downstream = 0; + } if (LOG_ENABLED(INFO)) { SSLOG(INFO, this) << "Using downstream address idx=" << addr_idx_ - << " out of " << get_config()->downstream_addrs.size(); + << " out of " << addrs.size(); } } - auto &downstream_addr = get_config()->downstream_addrs[addr_idx_]; + auto &downstream_addr = addrs[addr_idx_]; if (get_config()->downstream_http_proxy_host && state_ == DISCONNECTED) { if (LOG_ENABLED(INFO)) { @@ -503,7 +510,8 @@ int Http2Session::downstream_connect_proxy() { if (LOG_ENABLED(INFO)) { SSLOG(INFO, this) << "Connected to the proxy"; } - auto &downstream_addr = get_config()->downstream_addrs[addr_idx_]; + auto &downstream_addr = + get_config()->downstream_addr_groups[0].addrs[addr_idx_]; std::string req = "CONNECT "; req += downstream_addr.hostport.get(); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index d8016ca6..59553292 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -334,7 +334,7 @@ void Http2Upstream::initiate_downstream(Downstream *downstream) { int rv; rv = downstream->attach_downstream_connection( - handler_->get_downstream_connection()); + handler_->get_downstream_connection(downstream)); if (rv != 0) { // downstream connection fails, send error page if (error_reply(downstream, 503) != 0) { @@ -1476,7 +1476,7 @@ int Http2Upstream::on_downstream_reset(bool no_retry) { // downstream connection. rv = downstream->attach_downstream_connection( - handler_->get_downstream_connection()); + handler_->get_downstream_connection(downstream)); if (rv != 0) { goto fail; } diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 34f06b56..d0181b7e 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -109,12 +109,12 @@ void connectcb(struct ev_loop *loop, ev_io *w, int revents) { } // namespace HttpDownstreamConnection::HttpDownstreamConnection( - DownstreamConnectionPool *dconn_pool, struct ev_loop *loop) + DownstreamConnectionPool *dconn_pool, size_t group, struct ev_loop *loop) : DownstreamConnection(dconn_pool), conn_(loop, -1, nullptr, get_config()->downstream_write_timeout, get_config()->downstream_read_timeout, 0, 0, 0, 0, connectcb, readcb, timeoutcb, this), - ioctrl_(&conn_.rlimit), response_htp_{0}, addr_idx_(0), + ioctrl_(&conn_.rlimit), response_htp_{0}, group_(group), addr_idx_(0), connected_(false) {} HttpDownstreamConnection::~HttpDownstreamConnection() { @@ -143,14 +143,17 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { auto worker = client_handler_->get_worker(); auto worker_stat = worker->get_worker_stat(); - auto end = worker_stat->next_downstream; + auto &next_downstream = worker_stat->next_downstream[group_]; + auto end = next_downstream; + auto &addrs = get_config()->downstream_addr_groups[group_].addrs; for (;;) { - auto i = worker_stat->next_downstream; - ++worker_stat->next_downstream; - worker_stat->next_downstream %= get_config()->downstream_addrs.size(); + auto &addr = addrs[next_downstream]; + auto i = next_downstream; + if (++next_downstream >= addrs.size()) { + next_downstream = 0; + } - conn_.fd = util::create_nonblock_socket( - get_config()->downstream_addrs[i].addr.storage.ss_family); + conn_.fd = util::create_nonblock_socket(addr.addr.storage.ss_family); if (conn_.fd == -1) { auto error = errno; @@ -162,8 +165,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { } int rv; - rv = connect(conn_.fd, &get_config()->downstream_addrs[i].addr.sa, - get_config()->downstream_addrs[i].addrlen); + rv = connect(conn_.fd, &addr.addr.sa, addr.addrlen); if (rv != 0 && errno != EINPROGRESS) { auto error = errno; DCLOG(WARN, this) << "connect() failed; errno=" << error; @@ -172,7 +174,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { close(conn_.fd); conn_.fd = -1; - if (end == worker_stat->next_downstream) { + if (end == next_downstream) { return SHRPX_ERR_NETWORK; } @@ -212,8 +214,10 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { int HttpDownstreamConnection::push_request_headers() { const char *authority = nullptr, *host = nullptr; - auto downstream_hostport = - get_config()->downstream_addrs[addr_idx_].hostport.get(); + auto downstream_hostport = get_config() + ->downstream_addr_groups[group_] + .addrs[addr_idx_] + .hostport.get(); auto connect_method = downstream_->get_request_method() == HTTP_CONNECT; if (!get_config()->no_host_rewrite && !get_config()->http2_proxy && @@ -877,4 +881,6 @@ void HttpDownstreamConnection::on_upstream_change(Upstream *upstream) {} void HttpDownstreamConnection::signal_write() { conn_.wlimit.startw(); } +size_t HttpDownstreamConnection::get_group() const { return group_; } + } // namespace shrpx diff --git a/src/shrpx_http_downstream_connection.h b/src/shrpx_http_downstream_connection.h index 02480d5b..430a5edb 100644 --- a/src/shrpx_http_downstream_connection.h +++ b/src/shrpx_http_downstream_connection.h @@ -39,7 +39,7 @@ class DownstreamConnectionPool; class HttpDownstreamConnection : public DownstreamConnection { public: - HttpDownstreamConnection(DownstreamConnectionPool *dconn_pool, + HttpDownstreamConnection(DownstreamConnectionPool *dconn_pool, size_t group, struct ev_loop *loop); virtual ~HttpDownstreamConnection(); virtual int attach_downstream(Downstream *downstream); @@ -58,6 +58,7 @@ public: virtual void on_upstream_change(Upstream *upstream); virtual int on_priority_change(int32_t pri) { return 0; } + virtual size_t get_group() const; virtual bool poolable() const { return true; } @@ -68,6 +69,7 @@ private: Connection conn_; IOControl ioctrl_; http_parser response_htp_; + size_t group_; // index of get_config()->downstream_addrs this object is using size_t addr_idx_; bool connected_; diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 261d0c38..0fef7e6f 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -297,7 +297,7 @@ int htp_hdrs_completecb(http_parser *htp) { } rv = downstream->attach_downstream_connection( - upstream->get_client_handler()->get_downstream_connection()); + upstream->get_client_handler()->get_downstream_connection(downstream)); if (rv != 0) { downstream->set_request_state(Downstream::CONNECT_FAIL); @@ -993,7 +993,7 @@ int HttpsUpstream::on_downstream_reset(bool no_retry) { } rv = downstream_->attach_downstream_connection( - handler_->get_downstream_connection()); + handler_->get_downstream_connection(downstream_.get())); if (rv != 0) { goto fail; } diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 535909e6..eb23e107 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -271,7 +271,7 @@ void SpdyUpstream::start_downstream(Downstream *downstream) { void SpdyUpstream::initiate_downstream(Downstream *downstream) { int rv = downstream->attach_downstream_connection( - handler_->get_downstream_connection()); + handler_->get_downstream_connection(downstream)); if (rv != 0) { // If downstream connection fails, issue RST_STREAM. rst_stream(downstream, SPDYLAY_INTERNAL_ERROR); @@ -1104,7 +1104,7 @@ int SpdyUpstream::on_downstream_reset(bool no_retry) { // downstream connection. rv = downstream->attach_downstream_connection( - handler_->get_downstream_connection()); + handler_->get_downstream_connection(downstream)); if (rv != 0) { goto fail; } diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 1c04fe6c..60526653 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -743,7 +743,7 @@ void get_altnames(X509 *cert, std::vector &dns_names, } } -int check_cert(SSL *ssl) { +int check_cert(SSL *ssl, const DownstreamAddr *addr) { auto cert = SSL_get_peer_certificate(ssl); if (!cert) { LOG(ERROR) << "No certificate found"; @@ -760,9 +760,7 @@ int check_cert(SSL *ssl) { std::vector dns_names; std::vector ip_addrs; get_altnames(cert, dns_names, ip_addrs, common_name); - if (verify_hostname(get_config()->downstream_addrs[0].host.get(), - &get_config()->downstream_addrs[0].addr, - get_config()->downstream_addrs[0].addrlen, dns_names, + if (verify_hostname(addr->host.get(), &addr->addr, addr->addrlen, dns_names, ip_addrs, common_name) != 0) { LOG(ERROR) << "Certificate verification failed: hostname does not match"; return -1; diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h index ae428eaa..168b5f95 100644 --- a/src/shrpx_ssl.h +++ b/src/shrpx_ssl.h @@ -40,6 +40,7 @@ namespace shrpx { class ClientHandler; class Worker; class DownstreamConnectionPool; +struct DownstreamAddr; namespace ssl { @@ -68,7 +69,7 @@ ClientHandler *accept_connection(Worker *worker, int fd, sockaddr *addr, // Check peer's certificate against first downstream address in // Config::downstream_addrs. We only consider first downstream since // we use this function for HTTP/2 downstream link only. -int check_cert(SSL *ssl); +int check_cert(SSL *ssl, const DownstreamAddr *addr); // Retrieves DNS and IP address in subjectAltNames and commonName from // the |cert|. diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 9074b91e..a756e7c0 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -61,8 +61,11 @@ void mcpool_clear_cb(struct ev_loop *loop, ev_timer *w, int revents) { Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, ssl::CertLookupTree *cert_tree, const std::shared_ptr &ticket_keys) - : next_http2session_(0), loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), - cl_ssl_ctx_(cl_ssl_ctx), cert_tree_(cert_tree), ticket_keys_(ticket_keys), + : next_http2session_(0), + dconn_pool_(get_config()->downstream_addr_groups.size()), + worker_stat_(get_config()->downstream_addr_groups.size()), loop_(loop), + sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), cert_tree_(cert_tree), + ticket_keys_(ticket_keys), connect_blocker_(make_unique(loop_)), graceful_shutdown_(false) { ev_async_init(&w_, eventcb); diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 7a7a4e9e..f9924647 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -55,13 +55,13 @@ class CertLookupTree; } // namespace ssl struct WorkerStat { - WorkerStat() : num_connections(0), next_downstream(0) {} + WorkerStat(size_t num_groups) + : num_connections(0), next_downstream(num_groups) {} size_t num_connections; - // Next downstream index in Config::downstream_addrs. For HTTP/2 - // downstream connections, this is always 0. For HTTP/1, this is + // Next downstream index in Config::downstream_addr_groups. This is // used as load balancing. - size_t next_downstream; + std::vector next_downstream; }; enum WorkerEventType { diff --git a/src/util.h b/src/util.h index 9ee2b115..d43d3c76 100644 --- a/src/util.h +++ b/src/util.h @@ -386,9 +386,13 @@ bool streq_l(const char (&a)[N], InputIt b, size_t blen) { bool strifind(const char *a, const char *b); +template void inp_strlower(InputIt first, InputIt last) { + std::transform(first, last, first, lowcase); +} + // Lowercase |s| in place. inline void inp_strlower(std::string &s) { - std::transform(std::begin(s), std::end(s), std::begin(s), lowcase); + inp_strlower(std::begin(s), std::end(s)); } // Returns string representation of |n| with 2 fractional digits. From 6d556755ee437768569b14561fe28b6167e4b010 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 02:08:16 +0900 Subject: [PATCH 02/44] Attemp to fix travis build error --- src/shrpx_config.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 3fdbe425..11f29bb0 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -511,7 +511,7 @@ int parse_config(const char *opt, const char *optarg) { if (util::strieq(opt, SHRPX_OPT_BACKEND)) { auto optarglen = strlen(optarg); - auto pat_delim = strchr(optarg, ';'); + const char *pat_delim = strchr(optarg, ';'); if (!pat_delim) { pat_delim = optarg + optarglen; } From d457f39b1ec42498674654ff82230c9fa8872819 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 02:41:33 +0900 Subject: [PATCH 03/44] nghttpx: Fix unix domain backend --- src/shrpx_config.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 11f29bb0..5ea100f4 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -518,7 +518,7 @@ int parse_config(const char *opt, const char *optarg) { DownstreamAddr addr; if (util::istartsWith(optarg, SHRPX_UNIX_PATH_PREFIX)) { auto path = optarg + str_size(SHRPX_UNIX_PATH_PREFIX); - addr.host = strcopy(path); + addr.host = strcopy(path, pat_delim - path); addr.host_unix = true; } else { if (split_host_port(host, sizeof(host), &port, optarg, From c2e4ed96242ebdf69e8e0142558e98e5138da3f5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 12:43:48 +0900 Subject: [PATCH 04/44] nghttpx: Deal with the path without trailing slash on pattern match If pattern ends with '/', and pattern and path matches without that slash, we consider they match to deal with request to the directory without trailing slash. That is if pattern is "/foo/" and path is "/foo", we consider they match. --- src/shrpx.cc | 66 +++++++++++++++++++++++++--------------- src/shrpx_config.cc | 12 +++++++- src/shrpx_config_test.cc | 7 +++++ 3 files changed, 60 insertions(+), 25 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 8bfadbc3..c9524e25 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1002,38 +1002,56 @@ Connections: Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX domain socket can be specified by prefixing path name - with "unix:" (e.g., unix:/var/run/backend.sock). + with "unix:" (e.g., unix:/var/run/backend.sock). + Optionally, if s are given, the backend address is only used if request matches the pattern. If -s, -p, --client or --http2-bridge is used, s are ignored. The pattern matching is closely designed to ServeMux in net/http package of Go programming language. - consists of path, host + path or host. The - path must starts with "/". If it ends with "/", it + consists of path, host + path or just host. + The path must starts with "/". If it ends with "/", it matches to the request path whose prefix is the path. - If it does not end with "/", it performs exact match - against the request path. If host is given, it performs - exact match against the request host. If host alone is - given, "/" is appended to it, so that it matches all - paths under the host (e.g., specifying "nghttp2.org" - equals to "nghttp2.org/"). Longer patterns take - precedence over shorter ones, breaking a tie by the - order of the appearance in the configuration. If - is omitted, "/" is used as pattern, which - matches all paths (catch-all pattern). For example, - -b'127.0.0.1,8080;nghttp2.org/httpbin/' matches the - request host "nghttp2.org" and the request path - "/httpbin/get", but does not match the request host - "nghttp2.org" and the request path "/index.html". The - multiple s can be specified, delimiting them by - ":". Specifying + To deal with the request to the directory without + trailing slash, pattern which ends with "/" also matches + the path if pattern == path + "/" (e.g., pattern "/foo/" + matches path "/foo"). If it does not end with "/", it + performs exact match against the request path. If host + is given, it performs exact match against the request + host. If host alone is given, "/" is appended to it, so + that it matches all paths under the host (e.g., + specifying "nghttp2.org" equals to "nghttp2.org/"). + + Longer patterns take precedence over shorter ones, + breaking a tie by the order of the appearance in the + configuration. + + If is omitted, "/" is used as pattern, which + matches all paths (catch-all pattern). The catch-all + backend must be given. + + When doing a match, nghttpx made some normalization to + pattern, request host and path. For host part, they are + converted to lower case. For path part, percent-encoded + unreserved characters defined in RFC 3986 are decoded, + and any dot-segments (".." and ".") are resolved and + removed. + + For example, -b'127.0.0.1,8080;nghttp2.org/httpbin/' + matches the request host "nghttp2.org" and the request + path "/httpbin/get", but does not match the request host + "nghttp2.org" and the request path "/index.html". + + The multiple s can be specified, delimiting + them by ":". Specifying -b'127.0.0.1,8080;nghttp2.org:www.nghttp2.org' has the same effect to specify -b'127.0.0.1,8080;nghttp2.org' - and -b'127.0.0.1,8080:www.nghttp2.org'. The backend - addresses sharing same are grouped together - forming load balancing group. Since ";" and ":" are - used as delimiter, must not contain these - characters. + and -b'127.0.0.1,8080:www.nghttp2.org'. + + The backend addresses sharing same are grouped + together forming load balancing group. Since ";" and + ":" are used as delimiter, must not contain + these characters. Default: )" << DEFAULT_DOWNSTREAM_HOST << "," << DEFAULT_DOWNSTREAM_PORT << R"( -f, --frontend=, diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 5ea100f4..66b10ccf 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1402,7 +1402,17 @@ bool path_match(const std::string &pattern, const std::string &path) { if (pattern.back() != '/') { return pattern == path; } - return util::startsWith(path, pattern); + + if (util::startsWith(path, pattern)) { + return true; + } + + // If pattern ends with '/', and pattern and path matches without + // that slash, we consider they match to deal with request to the + // directory without trailing slash. That is if pattern is "/foo/" + // and path is "/foo", we consider they match. + return util::streq(std::begin(path), path.size(), std::begin(pattern), + pattern.size() - 1); } } // namespace diff --git a/src/shrpx_config_test.cc b/src/shrpx_config_test.cc index 4f21fc48..9f458d43 100644 --- a/src/shrpx_config_test.cc +++ b/src/shrpx_config_test.cc @@ -192,6 +192,13 @@ void test_shrpx_config_match_downstream_addr_group(void) { CU_ASSERT(4 == match_downstream_addr_group("WWW.nghttp2.org", "/alpha", groups, 255)); + CU_ASSERT(1 == match_downstream_addr_group("nghttp2.org", "/alpha/bravo/", + groups, 255)); + + // /alpha/bravo also matches /alpha/bravo/ + CU_ASSERT(1 == match_downstream_addr_group("nghttp2.org", "/alpha/bravo", + groups, 255)); + // path part is case-sensitive CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "/Alpha/bravo", groups, 255)); From d62e4dbc5e00523fea46fc7c0197e744c17c0249 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 12:45:23 +0900 Subject: [PATCH 05/44] Update man pages --- doc/h2load.1 | 8 +++++- doc/h2load.1.rst | 5 ++++ doc/nghttp.1 | 8 +++++- doc/nghttp.1.rst | 5 ++++ doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 66 ++++++++++++++++++++++++++++++++++++++++++++--- doc/nghttpx.1.rst | 60 +++++++++++++++++++++++++++++++++++++++--- 7 files changed, 144 insertions(+), 10 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index fd3009c6..0b2924f3 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "June 27, 2015" "1.0.5" "nghttp2" +.TH "H2LOAD" "1" "July 11, 2015" "1.0.6-DEV" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . @@ -113,6 +113,12 @@ Add/Override a header to the requests. .UNINDENT .INDENT 0.0 .TP +.B \-\-ciphers= +Set allowed cipher list. The format of the string is +described in OpenSSL ciphers(1). +.UNINDENT +.INDENT 0.0 +.TP .B \-p, \-\-no\-tls\-proto= Specify ALPN identifier of the protocol to be used when accessing http URI without SSL/TLS. diff --git a/doc/h2load.1.rst b/doc/h2load.1.rst index 21e5a425..ba1a820a 100644 --- a/doc/h2load.1.rst +++ b/doc/h2load.1.rst @@ -84,6 +84,11 @@ OPTIONS Add/Override a header to the requests. +.. option:: --ciphers= + + Set allowed cipher list. The format of the string is + described in OpenSSL ciphers(1). + .. option:: -p, --no-tls-proto= Specify ALPN identifier of the protocol to be used when diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 79b42b0c..7fb39c33 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "June 27, 2015" "1.0.5" "nghttp2" +.TH "NGHTTP" "1" "July 11, 2015" "1.0.6-DEV" "nghttp2" .SH NAME nghttp \- HTTP/2 experimental client . @@ -205,6 +205,12 @@ Disable server push. .UNINDENT .INDENT 0.0 .TP +.B \-\-max\-concurrent\-streams= +The number of concurrent pushed streams this client +accepts. +.UNINDENT +.INDENT 0.0 +.TP .B \-\-version Display version information and exit. .UNINDENT diff --git a/doc/nghttp.1.rst b/doc/nghttp.1.rst index 7f2449b2..b22d0c22 100644 --- a/doc/nghttp.1.rst +++ b/doc/nghttp.1.rst @@ -158,6 +158,11 @@ OPTIONS Disable server push. +.. option:: --max-concurrent-streams= + + The number of concurrent pushed streams this client + accepts. + .. option:: --version Display version information and exit. diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index 138a4bbf..5a477567 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "June 27, 2015" "1.0.5" "nghttp2" +.TH "NGHTTPD" "1" "July 11, 2015" "1.0.6-DEV" "nghttp2" .SH NAME nghttpd \- HTTP/2 experimental server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index 4cd1c3d9..bf67f98f 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "June 27, 2015" "1.0.5" "nghttp2" +.TH "NGHTTPX" "1" "July 11, 2015" "1.0.6-DEV" "nghttp2" .SH NAME nghttpx \- HTTP/2 experimental proxy . @@ -55,17 +55,66 @@ The options are categorized into several groups. .SS Connections .INDENT 0.0 .TP -.B \-b, \-\-backend= +.B \-b, \-\-backend=,[;[:...]] Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX domain socket can be specified by prefixing path name -with "unix:" (e.g., unix:/var/run/backend.sock) +with "unix:" (e.g., unix:/var/run/backend.sock). +.sp +Optionally, if s are given, the backend address +is only used if request matches the pattern. If \fI\%\-s\fP, \fI\%\-p\fP, +\fI\%\-\-client\fP or \fI\%\-\-http2\-bridge\fP is used, s are +ignored. The pattern matching is closely designed to +ServeMux in net/http package of Go programming language. + consists of path, host + path or just host. +The path must starts with "\fI/\fP". If it ends with "\fI/\fP", it +matches to the request path whose prefix is the path. +To deal with the request to the directory without +trailing slash, pattern which ends with "\fI/\fP" also matches +the path if pattern == path + "\fI/\fP" (e.g., pattern "\fI/foo/\fP" +matches path "\fI/foo\fP"). If it does not end with "\fI/\fP", it +performs exact match against the request path. If host +is given, it performs exact match against the request +host. If host alone is given, "\fI/\fP" is appended to it, so +that it matches all paths under the host (e.g., +specifying "nghttp2.org" equals to "nghttp2.org/"). +.sp +Longer patterns take precedence over shorter ones, +breaking a tie by the order of the appearance in the +configuration. +.sp +If is omitted, "\fI/\fP" is used as pattern, which +matches all paths (catch\-all pattern). The catch\-all +backend must be given. +.sp +When doing a match, nghttpx made some normalization to +pattern, request host and path. For host part, they are +converted to lower case. For path part, percent\-encoded +unreserved characters defined in RFC 3986 are decoded, +and any dot\-segments (".." and ".") are resolved and +removed. +.sp +For example, \fI\%\-b\fP\(aq127.0.0.1,8080;nghttp2.org/httpbin/\(aq +matches the request host "nghttp2.org" and the request +path "\fI/httpbin/get\fP", but does not match the request host +"nghttp2.org" and the request path "\fI/index.html\fP". +.sp +The multiple s can be specified, delimiting +them by ":". Specifying +\fI\%\-b\fP\(aq127.0.0.1,8080;nghttp2.org:www.nghttp2.org\(aq has the +same effect to specify \fI\%\-b\fP\(aq127.0.0.1,8080;nghttp2.org\(aq +and \fI\%\-b\fP\(aq127.0.0.1,8080:www.nghttp2.org\(aq. +.sp +The backend addresses sharing same are grouped +together forming load balancing group. Since ";" and +":" are used as delimiter, must not contain +these characters. .sp Default: \fB127.0.0.1,80\fP .UNINDENT .INDENT 0.0 .TP -.B \-f, \-\-frontend= +.B \-f, \-\-frontend=, Set frontend host and port. If is \(aq*\(aq, it assumes all addresses including both IPv4 and IPv6. UNIX domain socket can be specified by prefixing path @@ -626,6 +675,15 @@ $pid: PID of the running process. $alpn: ALPN identifier of the protocol which generates the response. For HTTP/1, ALPN is always http/1.1, regardless of minor version. +.IP \(bu 2 +$ssl_cipher: cipher used for SSL/TLS connection. +.IP \(bu 2 +$ssl_protocol: protocol for SSL/TLS connection. +.IP \(bu 2 +$ssl_session_id: session ID for SSL/TLS connection. +.IP \(bu 2 +$ssl_session_reused: "r" if SSL/TLS session was +reused. Otherwise, "." .UNINDENT .sp Default: \fB$remote_addr \- \- [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"\fP diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index ac904fb9..3132dca0 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -37,16 +37,65 @@ The options are categorized into several groups. Connections ~~~~~~~~~~~ -.. option:: -b, --backend= +.. option:: -b, --backend=,[;[:...]] Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX domain socket can be specified by prefixing path name - with "unix:" (e.g., unix:/var/run/backend.sock) + with "unix:" (e.g., unix:/var/run/backend.sock). + + Optionally, if s are given, the backend address + is only used if request matches the pattern. If :option:`-s`\, :option:`-p`\, + :option:`--client` or :option:`\--http2-bridge` is used, s are + ignored. The pattern matching is closely designed to + ServeMux in net/http package of Go programming language. + consists of path, host + path or just host. + The path must starts with "*/*". If it ends with "*/*", it + matches to the request path whose prefix is the path. + To deal with the request to the directory without + trailing slash, pattern which ends with "*/*" also matches + the path if pattern == path + "*/*" (e.g., pattern "*/foo/*" + matches path "*/foo*"). If it does not end with "*/*", it + performs exact match against the request path. If host + is given, it performs exact match against the request + host. If host alone is given, "*/*" is appended to it, so + that it matches all paths under the host (e.g., + specifying "nghttp2.org" equals to "nghttp2.org/"). + + Longer patterns take precedence over shorter ones, + breaking a tie by the order of the appearance in the + configuration. + + If is omitted, "*/*" is used as pattern, which + matches all paths (catch-all pattern). The catch-all + backend must be given. + + When doing a match, nghttpx made some normalization to + pattern, request host and path. For host part, they are + converted to lower case. For path part, percent-encoded + unreserved characters defined in RFC 3986 are decoded, + and any dot-segments (".." and ".") are resolved and + removed. + + For example, :option:`-b`\'127.0.0.1,8080;nghttp2.org/httpbin/' + matches the request host "nghttp2.org" and the request + path "*/httpbin/get*", but does not match the request host + "nghttp2.org" and the request path "*/index.html*". + + The multiple s can be specified, delimiting + them by ":". Specifying + :option:`-b`\'127.0.0.1,8080;nghttp2.org:www.nghttp2.org' has the + same effect to specify :option:`-b`\'127.0.0.1,8080;nghttp2.org' + and :option:`-b`\'127.0.0.1,8080:www.nghttp2.org'. + + The backend addresses sharing same are grouped + together forming load balancing group. Since ";" and + ":" are used as delimiter, must not contain + these characters. Default: ``127.0.0.1,80`` -.. option:: -f, --frontend= +.. option:: -f, --frontend=, Set frontend host and port. If is '\*', it assumes all addresses including both IPv4 and IPv6. @@ -550,6 +599,11 @@ Logging * $alpn: ALPN identifier of the protocol which generates the response. For HTTP/1, ALPN is always http/1.1, regardless of minor version. + * $ssl_cipher: cipher used for SSL/TLS connection. + * $ssl_protocol: protocol for SSL/TLS connection. + * $ssl_session_id: session ID for SSL/TLS connection. + * $ssl_session_reused: "r" if SSL/TLS session was + reused. Otherwise, "." Default: ``$remote_addr - - [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"`` From 19e47a192237faf8619c883b60b24a1a23bf4ac4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 16:12:35 +0900 Subject: [PATCH 06/44] nghttpx: Normalize path when setting it to Downstream --- src/http2.h | 15 +++++++++++++++ src/http2_test.cc | 25 +++++++++++++++++++++++++ src/http2_test.h | 1 + src/shrpx-unittest.cc | 2 ++ src/shrpx_config.cc | 25 ++++++++++++++----------- src/shrpx_config_test.cc | 17 +++-------------- src/shrpx_http2_upstream.cc | 9 +++++++-- src/shrpx_https_upstream.cc | 6 +++++- src/shrpx_spdy_upstream.cc | 3 ++- 9 files changed, 74 insertions(+), 29 deletions(-) diff --git a/src/http2.h b/src/http2.h index be79caca..182c45d9 100644 --- a/src/http2.h +++ b/src/http2.h @@ -334,6 +334,21 @@ std::string normalize_path(InputIt first, InputIt last) { nullptr, 0); } +template +std::string rewrite_clean_path(InputIt first, InputIt last) { + if (first == last || *first != '/') { + return std::string(first, last); + } + // probably, not necessary most of the case, but just in case. + auto fragment = std::find(first, last, '#'); + auto query = std::find(first, fragment, '?'); + auto path = normalize_path(first, query); + if (query != fragment) { + path.append(query, fragment); + } + return path; +} + } // namespace http2 } // namespace nghttp2 diff --git a/src/http2_test.cc b/src/http2_test.cc index 1937bfaf..88339045 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -856,4 +856,29 @@ void test_http2_normalize_path(void) { CU_ASSERT("/" == http2::normalize_path(std::begin(src), std::end(src))); } +void test_http2_rewrite_clean_path(void) { + std::string src; + + // unreserved characters + src = "/alpha/%62ravo/"; + CU_ASSERT("/alpha/bravo/" == + http2::rewrite_clean_path(std::begin(src), std::end(src))); + + // percent-encoding is converted to upper case. + src = "/delta%3a"; + CU_ASSERT("/delta%3A" == + http2::rewrite_clean_path(std::begin(src), std::end(src))); + + // path component is normalized before mathcing + src = "/alpha/charlie/%2e././bravo/delta/.."; + CU_ASSERT("/alpha/bravo/" == + http2::rewrite_clean_path(std::begin(src), std::end(src))); + + src = "alpha%3a"; + CU_ASSERT(src == http2::rewrite_clean_path(std::begin(src), std::end(src))); + + src = ""; + CU_ASSERT(src == http2::rewrite_clean_path(std::begin(src), std::end(src))); +} + } // namespace shrpx diff --git a/src/http2_test.h b/src/http2_test.h index 193f23e3..bc65d453 100644 --- a/src/http2_test.h +++ b/src/http2_test.h @@ -46,6 +46,7 @@ void test_http2_mandatory_request_headers_presence(void); void test_http2_parse_link_header(void); void test_http2_path_join(void); void test_http2_normalize_path(void); +void test_http2_rewrite_clean_path(void); } // namespace shrpx diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index 7030aa3d..83695204 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -98,6 +98,8 @@ int main(int argc, char *argv[]) { !CU_add_test(pSuite, "http2_path_join", shrpx::test_http2_path_join) || !CU_add_test(pSuite, "http2_normalize_path", shrpx::test_http2_normalize_path) || + !CU_add_test(pSuite, "http2_rewrite_clean_path", + shrpx::test_http2_rewrite_clean_path) || !CU_add_test(pSuite, "downstream_index_request_headers", shrpx::test_downstream_index_request_headers) || !CU_add_test(pSuite, "downstream_index_response_headers", diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 66b10ccf..ca130e77 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1438,9 +1438,9 @@ ssize_t match(const std::string &path, namespace { size_t match_downstream_addr_group_host( - const std::string &host, const std::string &raw_path, + const std::string &host, const std::string &path, const std::vector &groups, size_t catch_all) { - if (raw_path == "*") { + if (path.empty() || path[0] != '/') { auto group = match(host + "/", groups); if (group != -1) { if (LOG_ENABLED(INFO)) { @@ -1452,11 +1452,6 @@ size_t match_downstream_addr_group_host( return catch_all; } - // probably, not necessary most of the case, but just in case. - auto fragment = std::find(std::begin(raw_path), std::end(raw_path), '#'); - auto query = std::find(std::begin(raw_path), fragment, '?'); - auto path = http2::normalize_path(std::begin(raw_path), query); - if (LOG_ENABLED(INFO)) { LOG(INFO) << "Perform mapping selection, using host=" << host << ", path=" << path; @@ -1490,13 +1485,21 @@ size_t match_downstream_addr_group_host( size_t match_downstream_addr_group( const std::string &hostport, const std::string &raw_path, const std::vector &groups, size_t catch_all) { - if (hostport.empty() || - std::find(std::begin(hostport), std::end(hostport), '/') != - std::end(hostport)) { + if (std::find(std::begin(hostport), std::end(hostport), '/') != + std::end(hostport)) { // We use '/' specially, and if '/' is included in host, it breaks // our code. Select catch-all case. return catch_all; } + + auto fragment = std::find(std::begin(raw_path), std::end(raw_path), '#'); + auto query = std::find(std::begin(raw_path), fragment, '?'); + auto path = std::string(std::begin(raw_path), query); + + if (hostport.empty()) { + return match_downstream_addr_group_host(hostport, path, groups, catch_all); + } + std::string host; if (hostport[0] == '[') { // assume this is IPv6 numeric address @@ -1517,7 +1520,7 @@ size_t match_downstream_addr_group( } util::inp_strlower(host); - return match_downstream_addr_group_host(host, raw_path, groups, catch_all); + return match_downstream_addr_group_host(host, path, groups, catch_all); } } // namespace shrpx diff --git a/src/shrpx_config_test.cc b/src/shrpx_config_test.cc index 9f458d43..ab6a0f27 100644 --- a/src/shrpx_config_test.cc +++ b/src/shrpx_config_test.cc @@ -203,12 +203,8 @@ void test_shrpx_config_match_downstream_addr_group(void) { CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "/Alpha/bravo", groups, 255)); - // unreserved characters are decoded before matching - CU_ASSERT(1 == match_downstream_addr_group("nghttp2.org", "/alpha/%62ravo/", - groups, 255)); - CU_ASSERT(1 == match_downstream_addr_group( - "nghttp2.org", "/alpha/%62ravo/charlie", groups, 255)); + "nghttp2.org", "/alpha/bravo/charlie", groups, 255)); CU_ASSERT(2 == match_downstream_addr_group("nghttp2.org", "/alpha/charlie", groups, 255)); @@ -218,20 +214,13 @@ void test_shrpx_config_match_downstream_addr_group(void) { CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "/alpha/charlie/", groups, 255)); - // percent-encoding is normalized to upper case hex digits. - CU_ASSERT(3 == match_downstream_addr_group("nghttp2.org", "/delta%3a", groups, - 255)); - - // path component is normalized before mathcing - CU_ASSERT(1 == match_downstream_addr_group( - "nghttp2.org", "/alpha/charlie/%2e././bravo/delta/..", - groups, 255)); - CU_ASSERT(255 == match_downstream_addr_group("example.org", "/", groups, 255)); CU_ASSERT(255 == match_downstream_addr_group("", "/", groups, 255)); + CU_ASSERT(255 == match_downstream_addr_group("", "alpha", groups, 255)); + CU_ASSERT(255 == match_downstream_addr_group("foo/bar", "/", groups, 255)); // If path is "*", only match with host + "/". diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 59553292..f0b3b213 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -300,7 +300,11 @@ int Http2Upstream::on_request_headers(Downstream *downstream, downstream->set_request_method(method_token); downstream->set_request_http2_scheme(http2::value_to_str(scheme)); downstream->set_request_http2_authority(http2::value_to_str(authority)); - downstream->set_request_path(http2::value_to_str(path)); + if (path) { + auto &value = path->value; + downstream->set_request_path( + http2::rewrite_clean_path(std::begin(value), std::end(value))); + } if (!(frame->hd.flags & NGHTTP2_FLAG_END_STREAM)) { downstream->set_request_http2_expect_body(true); @@ -541,7 +545,8 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame, {nv.value, nv.value + nv.valuelen}); break; case http2::HD__PATH: - downstream->set_request_path({nv.value, nv.value + nv.valuelen}); + downstream->set_request_path( + http2::rewrite_clean_path(nv.value, nv.value + nv.valuelen)); break; } downstream->add_request_header(nv.name, nv.namelen, nv.value, nv.valuelen, diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 0fef7e6f..d1732e7a 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -218,7 +218,8 @@ void rewrite_request_host_path_from_uri(Downstream *downstream, const char *uri, path += '?'; path.append(uri + fdata.off, fdata.len); } - downstream->set_request_path(std::move(path)); + downstream->set_request_path( + http2::rewrite_clean_path(std::begin(path), std::end(path))); std::string scheme; http2::copy_url_component(scheme, &u, UF_SCHEMA, uri); @@ -286,6 +287,9 @@ int htp_hdrs_completecb(http_parser *htp) { return -1; } + downstream->set_request_path( + http2::rewrite_clean_path(std::begin(uri), std::end(uri))); + if (upstream->get_client_handler()->get_ssl()) { downstream->set_request_http2_scheme("https"); } else { diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index eb23e107..d6ac4f6c 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -229,7 +229,8 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, } else { downstream->set_request_http2_scheme(scheme->value); downstream->set_request_http2_authority(host->value); - downstream->set_request_path(path->value); + downstream->set_request_path(http2::rewrite_clean_path( + std::begin(path->value), std::end(path->value))); } if (!(frame->syn_stream.hd.flags & SPDYLAY_CTRL_FLAG_FIN)) { From e7724914a9aad9b9fafebeda2c0c61878b0944d4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 16:42:23 +0900 Subject: [PATCH 07/44] nghttpx: Less copy when matching path --- src/shrpx_config.cc | 54 ++++++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index ca130e77..8508cb93 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1398,12 +1398,19 @@ int int_syslog_facility(const char *strfacility) { } namespace { -bool path_match(const std::string &pattern, const std::string &path) { +template +bool path_match(const std::string &pattern, const std::string &host, + InputIt path_first, InputIt path_last) { if (pattern.back() != '/') { - return pattern == path; + return pattern.size() == host.size() + (path_last - path_first) && + std::equal(std::begin(host), std::end(host), std::begin(pattern)) && + std::equal(path_first, path_last, std::begin(pattern) + host.size()); } - if (util::startsWith(path, pattern)) { + if (pattern.size() >= host.size() && + std::equal(std::begin(host), std::end(host), std::begin(pattern)) && + util::startsWith(path_first, path_last, std::begin(pattern) + host.size(), + std::end(pattern))) { return true; } @@ -1411,20 +1418,24 @@ bool path_match(const std::string &pattern, const std::string &path) { // that slash, we consider they match to deal with request to the // directory without trailing slash. That is if pattern is "/foo/" // and path is "/foo", we consider they match. - return util::streq(std::begin(path), path.size(), std::begin(pattern), - pattern.size() - 1); + + assert(!pattern.empty()); + return pattern.size() - 1 == host.size() + (path_last - path_first) && + std::equal(std::begin(host), std::end(host), std::begin(pattern)) && + std::equal(path_first, path_last, std::begin(pattern) + host.size()); } } // namespace namespace { -ssize_t match(const std::string &path, +template +ssize_t match(const std::string &host, InputIt path_first, InputIt path_last, const std::vector &groups) { ssize_t res = -1; size_t best = 0; for (size_t i = 0; i < groups.size(); ++i) { auto &g = groups[i]; auto &pattern = g.pattern; - if (!path_match(pattern, path)) { + if (!path_match(pattern, host, path_first, path_last)) { continue; } if (res == -1 || best < pattern.size()) { @@ -1437,11 +1448,13 @@ ssize_t match(const std::string &path, } // namespace namespace { +template size_t match_downstream_addr_group_host( - const std::string &host, const std::string &path, + const std::string &host, InputIt path_first, InputIt path_last, const std::vector &groups, size_t catch_all) { - if (path.empty() || path[0] != '/') { - auto group = match(host + "/", groups); + if (path_first == path_last || *path_first != '/') { + constexpr const char P[] = "/"; + auto group = match(host, P, P + 1, groups); if (group != -1) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Found pattern with query " << host @@ -1454,22 +1467,24 @@ size_t match_downstream_addr_group_host( if (LOG_ENABLED(INFO)) { LOG(INFO) << "Perform mapping selection, using host=" << host - << ", path=" << path; + << ", path=" << std::string(path_first, path_last); } - auto group = match(host + path, groups); + auto group = match(host, path_first, path_last, groups); if (group != -1) { if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Found pattern with query " << host + path + LOG(INFO) << "Found pattern with query " << host + << std::string(path_first, path_last) << ", matched pattern=" << groups[group].pattern; } return group; } - group = match(path, groups); + group = match("", path_first, path_last, groups); if (group != -1) { if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Found pattern with query " << path + LOG(INFO) << "Found pattern with query " + << std::string(path_first, path_last) << ", matched pattern=" << groups[group].pattern; } return group; @@ -1494,10 +1509,12 @@ size_t match_downstream_addr_group( auto fragment = std::find(std::begin(raw_path), std::end(raw_path), '#'); auto query = std::find(std::begin(raw_path), fragment, '?'); - auto path = std::string(std::begin(raw_path), query); + auto path_first = std::begin(raw_path); + auto path_last = query; if (hostport.empty()) { - return match_downstream_addr_group_host(hostport, path, groups, catch_all); + return match_downstream_addr_group_host(hostport, path_first, path_last, + groups, catch_all); } std::string host; @@ -1520,7 +1537,8 @@ size_t match_downstream_addr_group( } util::inp_strlower(host); - return match_downstream_addr_group_host(host, path, groups, catch_all); + return match_downstream_addr_group_host(host, path_first, path_last, groups, + catch_all); } } // namespace shrpx From 7c216c6df8d6776582d03a39094263df08498f24 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 16:45:07 +0900 Subject: [PATCH 08/44] nghttpx: Document that patterns with host take precedence --- src/shrpx.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index c9524e25..389bf7e5 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1022,9 +1022,10 @@ Connections: that it matches all paths under the host (e.g., specifying "nghttp2.org" equals to "nghttp2.org/"). - Longer patterns take precedence over shorter ones, - breaking a tie by the order of the appearance in the - configuration. + Patterns with host take precedence over path only + patterns. Then, longer patterns take precedence over + shorter ones, breaking a tie by the order of the + appearance in the configuration. If is omitted, "/" is used as pattern, which matches all paths (catch-all pattern). The catch-all From 1a63cd94aa690df48432aa8d8ad292094c67b023 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 17:30:38 +0900 Subject: [PATCH 09/44] nghttpx: Pass by reference, since it just get copied there --- src/shrpx_config.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 8508cb93..ebaf2b23 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -467,7 +467,7 @@ namespace { // config. We will store each host-path pattern found in |src| with // |addr|. |addr| will be copied accordingly. Also we make a group // based on the pattern. The "/" pattern is considered as catch-all. -void parse_mapping(DownstreamAddr addr, const char *src) { +void parse_mapping(const DownstreamAddr &addr, const char *src) { // This returns at least 1 element (it could be empty string). We // will append '/' to all patterns, so it becomes catch-all pattern. auto mapping = parse_config_str_list(src, ':'); @@ -537,7 +537,7 @@ int parse_config(const char *opt, const char *optarg) { LOG(ERROR) << opt << ": ';' must not be used in pattern"; return -1; } - parse_mapping(std::move(addr), mapping); + parse_mapping(addr, mapping); return 0; } From fa7069a27388c085e22a3901bce744d36aa72671 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 17:50:58 +0900 Subject: [PATCH 10/44] nghttpx: Don't rewrite path if http2 proxy or client proxy is enabled There are many requests which changes its meaning when we rewrite path. This is due to bad percent-encoding in URI; reserved characters are just used without percent encoding. It seems this is common in ad services, but I suspect more to come. For reverse proxying situation, sane service most likely encodes URI properly, so probably this is not an issue. --- src/shrpx_http2_upstream.cc | 10 +++++++--- src/shrpx_https_upstream.cc | 8 ++++++-- src/shrpx_spdy_upstream.cc | 8 ++++++-- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index f0b3b213..07ac4e3d 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -301,9 +301,13 @@ int Http2Upstream::on_request_headers(Downstream *downstream, downstream->set_request_http2_scheme(http2::value_to_str(scheme)); downstream->set_request_http2_authority(http2::value_to_str(authority)); if (path) { - auto &value = path->value; - downstream->set_request_path( - http2::rewrite_clean_path(std::begin(value), std::end(value))); + if (get_config()->http2_proxy || get_config()->client_proxy) { + downstream->set_request_path(http2::value_to_str(path)); + } else { + auto &value = path->value; + downstream->set_request_path( + http2::rewrite_clean_path(std::begin(value), std::end(value))); + } } if (!(frame->hd.flags & NGHTTP2_FLAG_END_STREAM)) { diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index d1732e7a..1a832e60 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -218,8 +218,12 @@ void rewrite_request_host_path_from_uri(Downstream *downstream, const char *uri, path += '?'; path.append(uri + fdata.off, fdata.len); } - downstream->set_request_path( - http2::rewrite_clean_path(std::begin(path), std::end(path))); + if (get_config()->http2_proxy || get_config()->client_proxy) { + downstream->set_request_path(std::move(path)); + } else { + downstream->set_request_path( + http2::rewrite_clean_path(std::begin(path), std::end(path))); + } std::string scheme; http2::copy_url_component(scheme, &u, UF_SCHEMA, uri); diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index d6ac4f6c..52f267da 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -229,8 +229,12 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, } else { downstream->set_request_http2_scheme(scheme->value); downstream->set_request_http2_authority(host->value); - downstream->set_request_path(http2::rewrite_clean_path( - std::begin(path->value), std::end(path->value))); + if (get_config()->http2_proxy || get_config()->client_proxy) { + downstream->set_request_path(path->value); + } else { + downstream->set_request_path(http2::rewrite_clean_path( + std::begin(path->value), std::end(path->value))); + } } if (!(frame->syn_stream.hd.flags & SPDYLAY_CTRL_FLAG_FIN)) { From 8a2543d7b7f13ca34701829ebdfe1ad0d8b86622 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 11 Jul 2015 19:37:04 +0900 Subject: [PATCH 11/44] asio: Fix custom OpenSSL build --- src/Makefile.am | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index b1336ae8..8dc53570 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -207,10 +207,10 @@ libnghttp2_asio_la_LDFLAGS = -no-undefined -version-info 1:0:0 libnghttp2_asio_la_LIBADD = \ $(top_builddir)/lib/libnghttp2.la \ $(top_builddir)/third-party/libhttp-parser.la \ + @OPENSSL_LIBS@ \ ${BOOST_LDFLAGS} \ ${BOOST_ASIO_LIB} \ ${BOOST_THREAD_LIB} \ - ${BOOST_SYSTEM_LIB} \ - @OPENSSL_LIBS@ + ${BOOST_SYSTEM_LIB} endif # ENABLE_ASIO_LIB From 6307f96fb35335ecdf2281faa2465d2c596d5cdf Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 12 Jul 2015 22:16:20 +0900 Subject: [PATCH 12/44] nghttpx: Enable host-path backend routing in HTTP/2 backend To achieve host-path backend routing, we changed behaviour of --backend-http2-connections-per-worker. It now sets the number of HTTP/2 physical connections per pattern group if pattern is used in -b option. Fixes GH-292 --- src/shrpx.cc | 54 ++++++++++++------------ src/shrpx_client_handler.cc | 12 +++--- src/shrpx_client_handler.h | 2 - src/shrpx_http2_downstream_connection.cc | 13 +++++- src/shrpx_http2_downstream_connection.h | 3 +- src/shrpx_http2_session.cc | 17 +++++--- src/shrpx_http2_session.h | 5 ++- src/shrpx_http2_upstream.cc | 1 + src/shrpx_http_downstream_connection.cc | 4 +- src/shrpx_https_upstream.cc | 1 + src/shrpx_ssl.cc | 1 + src/shrpx_worker.cc | 39 +++++++++++------ src/shrpx_worker.h | 24 +++++++---- 13 files changed, 106 insertions(+), 70 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 389bf7e5..7ce6aa2b 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1005,22 +1005,22 @@ Connections: with "unix:" (e.g., unix:/var/run/backend.sock). Optionally, if s are given, the backend address - is only used if request matches the pattern. If -s, -p, - --client or --http2-bridge is used, s are - ignored. The pattern matching is closely designed to - ServeMux in net/http package of Go programming language. - consists of path, host + path or just host. - The path must starts with "/". If it ends with "/", it - matches to the request path whose prefix is the path. - To deal with the request to the directory without - trailing slash, pattern which ends with "/" also matches - the path if pattern == path + "/" (e.g., pattern "/foo/" - matches path "/foo"). If it does not end with "/", it - performs exact match against the request path. If host - is given, it performs exact match against the request - host. If host alone is given, "/" is appended to it, so - that it matches all paths under the host (e.g., - specifying "nghttp2.org" equals to "nghttp2.org/"). + is only used if request matches the pattern. If -s or + -p is used, s are ignored. The pattern + matching is closely designed to ServeMux in net/http + package of Go programming language. consists + of path, host + path or just host. The path must starts + with "/". If it ends with "/", it matches to the + request path whose prefix is the path. To deal with the + request to the directory without trailing slash, pattern + which ends with "/" also matches the path if pattern == + path + "/" (e.g., pattern "/foo/" matches path "/foo"). + If it does not end with "/", it performs exact match + against the request path. If host is given, it performs + exact match against the request host. If host alone is + given, "/" is appended to it, so that it matches all + paths under the host (e.g., specifying "nghttp2.org" + equals to "nghttp2.org/"). Patterns with host take precedence over path only patterns. Then, longer patterns take precedence over @@ -1130,9 +1130,14 @@ Performance: accepts. Setting 0 means unlimited. Default: )" << get_config()->worker_frontend_connections << R"( --backend-http2-connections-per-worker= - Set maximum number of HTTP/2 connections per worker. - The default value is 0, which means the number of - backend addresses specified by -b option. + Set maximum number of backend HTTP/2 physical + connections per worker. If pattern is used in -b + option, this limit is applied to each pattern group (in + other words, each pattern group can have maximum + HTTP/2 connections). The default value is 0, which + means that the value is adjusted to the number of + backend addresses. If pattern is used, this adjustment + is done for each pattern group. --backend-http1-connections-per-host= Set maximum number of backend concurrent HTTP/1 connections per origin host. This option is meaningful @@ -2177,10 +2182,9 @@ int main(int argc, char **argv) { DownstreamAddrGroup g("/"); g.addrs.push_back(std::move(addr)); mod_config()->downstream_addr_groups.push_back(std::move(g)); - } else if (get_config()->downstream_proto == PROTO_HTTP2 || - get_config()->http2_proxy || get_config()->client_proxy) { + } else if (get_config()->http2_proxy || get_config()->client_proxy) { // We don't support host mapping in these cases. Move all - // non-catch-all patterns to catch-all pattern for HTTP/2 backend + // non-catch-all patterns to catch-all pattern. DownstreamAddrGroup catch_all("/"); for (auto &g : mod_config()->downstream_addr_groups) { std::move(std::begin(g.addrs), std::end(g.addrs), @@ -2276,12 +2280,6 @@ int main(int argc, char **argv) { } } - if (get_config()->downstream_proto == PROTO_HTTP2 && - get_config()->http2_downstream_connections_per_worker == 0) { - mod_config()->http2_downstream_connections_per_worker = - get_config()->downstream_addr_groups[0].addrs.size(); - } - if (get_config()->rlimit_nofile) { struct rlimit lim = {static_cast(get_config()->rlimit_nofile), static_cast(get_config()->rlimit_nofile)}; diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 644a8cdc..fcf8a235 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -39,6 +39,7 @@ #include "shrpx_worker.h" #include "shrpx_downstream_connection_pool.h" #include "shrpx_downstream.h" +#include "shrpx_http2_session.h" #ifdef HAVE_SPDYLAY #include "shrpx_spdy_upstream.h" #endif // HAVE_SPDYLAY @@ -361,7 +362,6 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl, get_config()->write_burst, get_config()->read_rate, get_config()->read_burst, writecb, readcb, timeoutcb, this), ipaddr_(ipaddr), port_(port), worker_(worker), - http2session_(worker_->next_http2_session()), left_connhd_len_(NGHTTP2_CLIENT_MAGIC_LEN), should_close_after_write_(false) { @@ -612,9 +612,8 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { auto catch_all = get_config()->downstream_addr_group_catch_all; // Fast path. If we have one group, it must be catch-all group. - // HTTP/2 and client proxy modes fall in this case. Currently, - // HTTP/2 backend does not perform host-path mapping. - if (groups.size() == 1 || get_config()->downstream_proto == PROTO_HTTP2) { + // HTTP/2 and client proxy modes fall in this case. + if (groups.size() == 1) { group = 0; } else if (downstream->get_request_method() == HTTP_CONNECT) { // We don't know how to treat CONNECT request in host-path @@ -653,8 +652,9 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { auto dconn_pool = worker_->get_dconn_pool(); - if (http2session_) { - dconn = make_unique(dconn_pool, http2session_); + if (get_config()->downstream_proto == PROTO_HTTP2) { + auto http2session = worker_->next_http2_session(group); + dconn = make_unique(dconn_pool, http2session); } else { dconn = make_unique(dconn_pool, group, conn_.loop); diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index 4f21f513..67dc0974 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -44,7 +44,6 @@ namespace shrpx { class Upstream; class DownstreamConnection; -class Http2Session; class HttpsUpstream; class ConnectBlocker; class DownstreamConnectionPool; @@ -142,7 +141,6 @@ private: std::function read_, write_; std::function on_read_, on_write_; Worker *worker_; - Http2Session *http2session_; // The number of bytes of HTTP/2 client connection header to read size_t left_connhd_len_; bool should_close_after_write_; diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index f82b5230..d6bc433f 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -263,8 +263,11 @@ int Http2DownstreamConnection::push_request_headers() { // http2session_ has already in CONNECTED state, so we can get // addr_idx here. auto addr_idx = http2session_->get_addr_idx(); - auto downstream_hostport = - get_config()->downstream_addr_groups[0].addrs[addr_idx].hostport.get(); + auto group = http2session_->get_group(); + auto downstream_hostport = get_config() + ->downstream_addr_groups[group] + .addrs[addr_idx] + .hostport.get(); const char *authority = nullptr, *host = nullptr; if (!no_host_rewrite) { @@ -557,4 +560,10 @@ int Http2DownstreamConnection::on_timeout() { return submit_rst_stream(downstream_, NGHTTP2_NO_ERROR); } +size_t Http2DownstreamConnection::get_group() const { + // HTTP/2 backend connections are managed by Http2Session object, + // and it stores group index. + return http2session_->get_group(); +} + } // namespace shrpx diff --git a/src/shrpx_http2_downstream_connection.h b/src/shrpx_http2_downstream_connection.h index 736f546f..da9f7ef5 100644 --- a/src/shrpx_http2_downstream_connection.h +++ b/src/shrpx_http2_downstream_connection.h @@ -61,8 +61,7 @@ public: virtual void on_upstream_change(Upstream *upstream) {} virtual int on_priority_change(int32_t pri); - // Currently, HTTP/2 backend does not perform host-path mapping. - virtual size_t get_group() const { return 0; } + virtual size_t get_group() const; // This object is not poolable because we dont' have facility to // migrate to another Http2Session object. diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 5ed2f3c3..64c9feb9 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -142,13 +142,14 @@ void writecb(struct ev_loop *loop, ev_io *w, int revents) { } // namespace Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, - ConnectBlocker *connect_blocker, Worker *worker) + ConnectBlocker *connect_blocker, Worker *worker, + size_t group) : conn_(loop, -1, nullptr, get_config()->downstream_write_timeout, get_config()->downstream_read_timeout, 0, 0, 0, 0, writecb, readcb, timeoutcb, this), worker_(worker), connect_blocker_(connect_blocker), ssl_ctx_(ssl_ctx), session_(nullptr), data_pending_(nullptr), data_pendinglen_(0), - addr_idx_(0), state_(DISCONNECTED), + addr_idx_(0), group_(group), state_(DISCONNECTED), connection_check_state_(CONNECTION_CHECK_NONE), flow_control_(false) { read_ = write_ = &Http2Session::noop; @@ -235,13 +236,14 @@ int Http2Session::disconnect(bool hard) { int Http2Session::check_cert() { return ssl::check_cert( - conn_.tls.ssl, &get_config()->downstream_addr_groups[0].addrs[addr_idx_]); + conn_.tls.ssl, + &get_config()->downstream_addr_groups[group_].addrs[addr_idx_]); } int Http2Session::initiate_connection() { int rv = 0; - auto &addrs = get_config()->downstream_addr_groups[0].addrs; + auto &addrs = get_config()->downstream_addr_groups[group_].addrs; if (state_ == DISCONNECTED) { if (connect_blocker_->blocked()) { @@ -252,8 +254,7 @@ int Http2Session::initiate_connection() { return -1; } - auto worker_stat = worker_->get_worker_stat(); - auto &next_downstream = worker_stat->next_downstream[0]; + auto &next_downstream = worker_->get_dgrp(group_)->next; addr_idx_ = next_downstream; if (++next_downstream >= addrs.size()) { next_downstream = 0; @@ -511,7 +512,7 @@ int Http2Session::downstream_connect_proxy() { SSLOG(INFO, this) << "Connected to the proxy"; } auto &downstream_addr = - get_config()->downstream_addr_groups[0].addrs[addr_idx_]; + get_config()->downstream_addr_groups[group_].addrs[addr_idx_]; std::string req = "CONNECT "; req += downstream_addr.hostport.get(); @@ -1752,4 +1753,6 @@ bool Http2Session::should_hard_fail() const { size_t Http2Session::get_addr_idx() const { return addr_idx_; } +size_t Http2Session::get_group() const { return group_; } + } // namespace shrpx diff --git a/src/shrpx_http2_session.h b/src/shrpx_http2_session.h index 6f642063..184252de 100644 --- a/src/shrpx_http2_session.h +++ b/src/shrpx_http2_session.h @@ -58,7 +58,7 @@ struct StreamData { class Http2Session { public: Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, - ConnectBlocker *connect_blocker, Worker *worker); + ConnectBlocker *connect_blocker, Worker *worker, size_t group); ~Http2Session(); int check_cert(); @@ -151,6 +151,8 @@ public: size_t get_addr_idx() const; + size_t get_group() const; + enum { // Disconnected DISCONNECTED, @@ -203,6 +205,7 @@ private: size_t data_pendinglen_; // index of get_config()->downstream_addrs this object uses size_t addr_idx_; + size_t group_; int state_; int connection_check_state_; bool flow_control_; diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 07ac4e3d..97ff206c 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -36,6 +36,7 @@ #include "shrpx_config.h" #include "shrpx_http.h" #include "shrpx_worker.h" +#include "shrpx_http2_session.h" #include "http2.h" #include "util.h" #include "base64.h" diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index d0181b7e..a69357a5 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -34,6 +34,7 @@ #include "shrpx_connect_blocker.h" #include "shrpx_downstream_connection_pool.h" #include "shrpx_worker.h" +#include "shrpx_http2_session.h" #include "http2.h" #include "util.h" @@ -142,8 +143,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { } auto worker = client_handler_->get_worker(); - auto worker_stat = worker->get_worker_stat(); - auto &next_downstream = worker_stat->next_downstream[group_]; + auto &next_downstream = worker->get_dgrp(group_)->next; auto end = next_downstream; auto &addrs = get_config()->downstream_addr_groups[group_].addrs; for (;;) { diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 1a832e60..22c048a3 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -36,6 +36,7 @@ #include "shrpx_error.h" #include "shrpx_log_config.h" #include "shrpx_worker.h" +#include "shrpx_http2_session.h" #include "http2.h" #include "util.h" #include "template.h" diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 60526653..9f95fa2f 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -53,6 +53,7 @@ #include "shrpx_config.h" #include "shrpx_worker.h" #include "shrpx_downstream_connection_pool.h" +#include "shrpx_http2_session.h" #include "util.h" #include "ssl.h" #include "template.h" diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index a756e7c0..e5f0f8d4 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -61,9 +61,9 @@ void mcpool_clear_cb(struct ev_loop *loop, ev_timer *w, int revents) { Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, ssl::CertLookupTree *cert_tree, const std::shared_ptr &ticket_keys) - : next_http2session_(0), - dconn_pool_(get_config()->downstream_addr_groups.size()), - worker_stat_(get_config()->downstream_addr_groups.size()), loop_(loop), + : dconn_pool_(get_config()->downstream_addr_groups.size()), + worker_stat_(get_config()->downstream_addr_groups.size()), + dgrps_(get_config()->downstream_addr_groups.size()), loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), cert_tree_(cert_tree), ticket_keys_(ticket_keys), connect_blocker_(make_unique(loop_)), @@ -77,9 +77,17 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, if (get_config()->downstream_proto == PROTO_HTTP2) { auto n = get_config()->http2_downstream_connections_per_worker; - for (; n > 0; --n) { - http2sessions_.push_back(make_unique( - loop_, cl_ssl_ctx, connect_blocker_.get(), this)); + size_t group = 0; + for (auto &dgrp : dgrps_) { + auto m = n; + if (m == 0) { + m = get_config()->downstream_addr_groups[group].addrs.size(); + } + for (; m; --m) { + dgrp.http2sessions.push_back(make_unique( + loop_, cl_ssl_ctx, connect_blocker_.get(), this, group)); + } + ++group; } } } @@ -210,15 +218,17 @@ WorkerStat *Worker::get_worker_stat() { return &worker_stat_; } DownstreamConnectionPool *Worker::get_dconn_pool() { return &dconn_pool_; } -Http2Session *Worker::next_http2_session() { - if (http2sessions_.empty()) { +Http2Session *Worker::next_http2_session(size_t group) { + auto &dgrp = dgrps_[group]; + auto &http2sessions = dgrp.http2sessions; + if (http2sessions.empty()) { return nullptr; } - auto res = http2sessions_[next_http2session_].get(); - ++next_http2session_; - if (next_http2session_ >= http2sessions_.size()) { - next_http2session_ = 0; + auto res = http2sessions[dgrp.next_http2session].get(); + ++dgrp.next_http2session; + if (dgrp.next_http2session >= http2sessions.size()) { + dgrp.next_http2session = 0; } return res; @@ -242,4 +252,9 @@ bool Worker::get_graceful_shutdown() const { return graceful_shutdown_; } MemchunkPool *Worker::get_mcpool() { return &mcpool_; } +DownstreamGroup *Worker::get_dgrp(size_t group) { + assert(group < dgrps_.size()); + return &dgrps_[group]; +} + } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index f9924647..b5f820b4 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -54,14 +54,21 @@ namespace ssl { class CertLookupTree; } // namespace ssl +struct DownstreamGroup { + DownstreamGroup() : next_http2session(0), next(0) {} + + std::vector> http2sessions; + // Next index in http2sessions. + size_t next_http2session; + // Next downstream address index corresponding to + // Config::downstream_addr_groups[]. + size_t next; +}; + struct WorkerStat { - WorkerStat(size_t num_groups) - : num_connections(0), next_downstream(num_groups) {} + WorkerStat(size_t num_groups) : num_connections(0) {} size_t num_connections; - // Next downstream index in Config::downstream_addr_groups. This is - // used as load balancing. - std::vector next_downstream; }; enum WorkerEventType { @@ -97,7 +104,7 @@ public: void set_ticket_keys(std::shared_ptr ticket_keys); WorkerStat *get_worker_stat(); DownstreamConnectionPool *get_dconn_pool(); - Http2Session *next_http2_session(); + Http2Session *next_http2_session(size_t group); ConnectBlocker *get_connect_blocker() const; struct ev_loop *get_loop() const; SSL_CTX *get_sv_ssl_ctx() const; @@ -109,9 +116,9 @@ public: MemchunkPool *get_mcpool(); void schedule_clear_mcpool(); + DownstreamGroup *get_dgrp(size_t group); + private: - std::vector> http2sessions_; - size_t next_http2session_; #ifndef NOTHREADS std::future fut_; #endif // NOTHREADS @@ -122,6 +129,7 @@ private: MemchunkPool mcpool_; DownstreamConnectionPool dconn_pool_; WorkerStat worker_stat_; + std::vector dgrps_; struct ev_loop *loop_; // Following fields are shared across threads if From 309754749140d0ac59d4f504c6f86b4026cfa72f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 12 Jul 2015 23:18:36 +0900 Subject: [PATCH 13/44] nghttpx: Add --include option to read additional configuration from given file --- src/shrpx.cc | 10 ++++++++++ src/shrpx_config.cc | 7 ++++++- src/shrpx_config.h | 1 + 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 7ce6aa2b..26326e2f 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1502,6 +1502,11 @@ Misc: --conf= Load configuration from . Default: )" << get_config()->conf_path.get() << R"( + --include= + Load additional configurations from . File + is read when configuration parser encountered this + option. This option can be used multiple times, or even + recursively. -v, --version Print version and exit. -h, --help Print this help and exit. @@ -1648,6 +1653,7 @@ int main(int argc, char **argv) { {SHRPX_OPT_HEADER_FIELD_BUFFER, required_argument, &flag, 80}, {SHRPX_OPT_MAX_HEADER_FIELDS, required_argument, &flag, 81}, {SHRPX_OPT_ADD_REQUEST_HEADER, required_argument, &flag, 82}, + {SHRPX_OPT_INCLUDE, required_argument, &flag, 83}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2010,6 +2016,10 @@ int main(int argc, char **argv) { // --add-request-header cmdcfgs.emplace_back(SHRPX_OPT_ADD_REQUEST_HEADER, optarg); break; + case 83: + // --include + cmdcfgs.emplace_back(SHRPX_OPT_INCLUDE, optarg); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index ebaf2b23..49ad49a8 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1219,6 +1219,10 @@ int parse_config(const char *opt, const char *optarg) { return parse_uint(&mod_config()->max_header_fields, opt, optarg); } + if (util::strieq(opt, SHRPX_OPT_INCLUDE)) { + return load_config(optarg); + } + if (util::strieq(opt, "conf")) { LOG(WARN) << "conf: ignored"; @@ -1248,7 +1252,8 @@ int load_config(const char *filename) { for (i = 0; i < size && line[i] != '='; ++i) ; if (i == size) { - LOG(ERROR) << "Bad configuration format at line " << linenum; + LOG(ERROR) << "Bad configuration format in " << filename << " at line " + << linenum; return -1; } line[i] = '\0'; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 3e5c869e..5f23c5b3 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -165,6 +165,7 @@ constexpr char SHRPX_OPT_OCSP_UPDATE_INTERVAL[] = "ocsp-update-interval"; constexpr char SHRPX_OPT_NO_OCSP[] = "no-ocsp"; constexpr char SHRPX_OPT_HEADER_FIELD_BUFFER[] = "header-field-buffer"; constexpr char SHRPX_OPT_MAX_HEADER_FIELDS[] = "max-header-fields"; +constexpr char SHRPX_OPT_INCLUDE[] = "include"; union sockaddr_union { sockaddr_storage storage; From 3db0badc354532509c11408b5f7b9bfa6bc1dc3d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 12 Jul 2015 23:26:45 +0900 Subject: [PATCH 14/44] Update man pages --- doc/h2load.1 | 2 +- doc/nghttp.1 | 2 +- doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 60 +++++++++++++++++++++++++++++------------------ doc/nghttpx.1.rst | 57 +++++++++++++++++++++++++++----------------- 5 files changed, 75 insertions(+), 48 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index 0b2924f3..b5b60925 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "July 11, 2015" "1.0.6-DEV" "nghttp2" +.TH "H2LOAD" "1" "July 12, 2015" "1.0.6-DEV" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 7fb39c33..9695b45a 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "July 11, 2015" "1.0.6-DEV" "nghttp2" +.TH "NGHTTP" "1" "July 12, 2015" "1.0.6-DEV" "nghttp2" .SH NAME nghttp \- HTTP/2 experimental client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index 5a477567..70a816bd 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "July 11, 2015" "1.0.6-DEV" "nghttp2" +.TH "NGHTTPD" "1" "July 12, 2015" "1.0.6-DEV" "nghttp2" .SH NAME nghttpd \- HTTP/2 experimental server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index bf67f98f..fa4e2cde 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "July 11, 2015" "1.0.6-DEV" "nghttp2" +.TH "NGHTTPX" "1" "July 12, 2015" "1.0.6-DEV" "nghttp2" .SH NAME nghttpx \- HTTP/2 experimental proxy . @@ -62,26 +62,27 @@ domain socket can be specified by prefixing path name with "unix:" (e.g., unix:/var/run/backend.sock). .sp Optionally, if s are given, the backend address -is only used if request matches the pattern. If \fI\%\-s\fP, \fI\%\-p\fP, -\fI\%\-\-client\fP or \fI\%\-\-http2\-bridge\fP is used, s are -ignored. The pattern matching is closely designed to -ServeMux in net/http package of Go programming language. - consists of path, host + path or just host. -The path must starts with "\fI/\fP". If it ends with "\fI/\fP", it -matches to the request path whose prefix is the path. -To deal with the request to the directory without -trailing slash, pattern which ends with "\fI/\fP" also matches -the path if pattern == path + "\fI/\fP" (e.g., pattern "\fI/foo/\fP" -matches path "\fI/foo\fP"). If it does not end with "\fI/\fP", it -performs exact match against the request path. If host -is given, it performs exact match against the request -host. If host alone is given, "\fI/\fP" is appended to it, so -that it matches all paths under the host (e.g., -specifying "nghttp2.org" equals to "nghttp2.org/"). +is only used if request matches the pattern. If \fI\%\-s\fP or +\fI\%\-p\fP is used, s are ignored. The pattern +matching is closely designed to ServeMux in net/http +package of Go programming language. consists +of path, host + path or just host. The path must starts +with "\fI/\fP". If it ends with "\fI/\fP", it matches to the +request path whose prefix is the path. To deal with the +request to the directory without trailing slash, pattern +which ends with "\fI/\fP" also matches the path if pattern == +path + "\fI/\fP" (e.g., pattern "\fI/foo/\fP" matches path "\fI/foo\fP"). +If it does not end with "\fI/\fP", it performs exact match +against the request path. If host is given, it performs +exact match against the request host. If host alone is +given, "\fI/\fP" is appended to it, so that it matches all +paths under the host (e.g., specifying "nghttp2.org" +equals to "nghttp2.org/"). .sp -Longer patterns take precedence over shorter ones, -breaking a tie by the order of the appearance in the -configuration. +Patterns with host take precedence over path only +patterns. Then, longer patterns take precedence over +shorter ones, breaking a tie by the order of the +appearance in the configuration. .sp If is omitted, "\fI/\fP" is used as pattern, which matches all paths (catch\-all pattern). The catch\-all @@ -244,9 +245,14 @@ Default: \fB0\fP .INDENT 0.0 .TP .B \-\-backend\-http2\-connections\-per\-worker= -Set maximum number of HTTP/2 connections per worker. -The default value is 0, which means the number of -backend addresses specified by \fI\%\-b\fP option. +Set maximum number of backend HTTP/2 physical +connections per worker. If pattern is used in \fI\%\-b\fP +option, this limit is applied to each pattern group (in +other words, each pattern group can have maximum +HTTP/2 connections). The default value is 0, which +means that the value is adjusted to the number of +backend addresses. If pattern is used, this adjustment +is done for each pattern group. .UNINDENT .INDENT 0.0 .TP @@ -844,6 +850,14 @@ Default: \fB/etc/nghttpx/nghttpx.conf\fP .UNINDENT .INDENT 0.0 .TP +.B \-\-include= +Load additional configurations from . File +is read when configuration parser encountered this +option. This option can be used multiple times, or even +recursively. +.UNINDENT +.INDENT 0.0 +.TP .B \-v, \-\-version Print version and exit. .UNINDENT diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index 3132dca0..7ff041c6 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -45,26 +45,27 @@ Connections with "unix:" (e.g., unix:/var/run/backend.sock). Optionally, if s are given, the backend address - is only used if request matches the pattern. If :option:`-s`\, :option:`-p`\, - :option:`--client` or :option:`\--http2-bridge` is used, s are - ignored. The pattern matching is closely designed to - ServeMux in net/http package of Go programming language. - consists of path, host + path or just host. - The path must starts with "*/*". If it ends with "*/*", it - matches to the request path whose prefix is the path. - To deal with the request to the directory without - trailing slash, pattern which ends with "*/*" also matches - the path if pattern == path + "*/*" (e.g., pattern "*/foo/*" - matches path "*/foo*"). If it does not end with "*/*", it - performs exact match against the request path. If host - is given, it performs exact match against the request - host. If host alone is given, "*/*" is appended to it, so - that it matches all paths under the host (e.g., - specifying "nghttp2.org" equals to "nghttp2.org/"). + is only used if request matches the pattern. If :option:`-s` or + :option:`-p` is used, s are ignored. The pattern + matching is closely designed to ServeMux in net/http + package of Go programming language. consists + of path, host + path or just host. The path must starts + with "*/*". If it ends with "*/*", it matches to the + request path whose prefix is the path. To deal with the + request to the directory without trailing slash, pattern + which ends with "*/*" also matches the path if pattern == + path + "*/*" (e.g., pattern "*/foo/*" matches path "*/foo*"). + If it does not end with "*/*", it performs exact match + against the request path. If host is given, it performs + exact match against the request host. If host alone is + given, "*/*" is appended to it, so that it matches all + paths under the host (e.g., specifying "nghttp2.org" + equals to "nghttp2.org/"). - Longer patterns take precedence over shorter ones, - breaking a tie by the order of the appearance in the - configuration. + Patterns with host take precedence over path only + patterns. Then, longer patterns take precedence over + shorter ones, breaking a tie by the order of the + appearance in the configuration. If is omitted, "*/*" is used as pattern, which matches all paths (catch-all pattern). The catch-all @@ -214,9 +215,14 @@ Performance .. option:: --backend-http2-connections-per-worker= - Set maximum number of HTTP/2 connections per worker. - The default value is 0, which means the number of - backend addresses specified by :option:`-b` option. + Set maximum number of backend HTTP/2 physical + connections per worker. If pattern is used in :option:`-b` + option, this limit is applied to each pattern group (in + other words, each pattern group can have maximum + HTTP/2 connections). The default value is 0, which + means that the value is adjusted to the number of + backend addresses. If pattern is used, this adjustment + is done for each pattern group. .. option:: --backend-http1-connections-per-host= @@ -754,6 +760,13 @@ Misc Default: ``/etc/nghttpx/nghttpx.conf`` +.. option:: --include= + + Load additional configurations from . File + is read when configuration parser encountered this + option. This option can be used multiple times, or even + recursively. + .. option:: -v, --version Print version and exit. From f96edbf987109d33b15573766be76e4c4c3c63c5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 13 Jul 2015 21:31:37 +0900 Subject: [PATCH 15/44] nghttpx: Pin frontend to specific HTTP/2 session object per group --- src/shrpx_client_handler.cc | 15 ++++++++++++++- src/shrpx_client_handler.h | 1 + src/shrpx_http2_session.cc | 6 ++++-- src/shrpx_http2_session.h | 8 +++++++- src/shrpx_worker.cc | 4 ++-- 5 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index fcf8a235..0e2c8ff9 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -361,6 +361,11 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl, get_config()->upstream_read_timeout, get_config()->write_rate, get_config()->write_burst, get_config()->read_rate, get_config()->read_burst, writecb, readcb, timeoutcb, this), + pinned_http2sessions_( + get_config()->downstream_proto == PROTO_HTTP2 + ? make_unique>( + get_config()->downstream_addr_groups.size(), -1) + : nullptr), ipaddr_(ipaddr), port_(port), worker_(worker), left_connhd_len_(NGHTTP2_CLIENT_MAGIC_LEN), should_close_after_write_(false) { @@ -653,7 +658,15 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { auto dconn_pool = worker_->get_dconn_pool(); if (get_config()->downstream_proto == PROTO_HTTP2) { - auto http2session = worker_->next_http2_session(group); + Http2Session *http2session; + auto &pinned = (*pinned_http2sessions_)[group]; + if (pinned == -1) { + http2session = worker_->next_http2_session(group); + pinned = http2session->get_index(); + } else { + auto dgrp = worker_->get_dgrp(group); + http2session = dgrp->http2sessions[pinned].get(); + } dconn = make_unique(dconn_pool, http2session); } else { dconn = diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index 67dc0974..4d4ccd9c 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -134,6 +134,7 @@ private: Connection conn_; ev_timer reneg_shutdown_timer_; std::unique_ptr upstream_; + std::unique_ptr> pinned_http2sessions_; std::string ipaddr_; std::string port_; // The ALPN identifier negotiated for this connection. diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 64c9feb9..e2b4e2ee 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -143,13 +143,13 @@ void writecb(struct ev_loop *loop, ev_io *w, int revents) { Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, ConnectBlocker *connect_blocker, Worker *worker, - size_t group) + size_t group, size_t idx) : conn_(loop, -1, nullptr, get_config()->downstream_write_timeout, get_config()->downstream_read_timeout, 0, 0, 0, 0, writecb, readcb, timeoutcb, this), worker_(worker), connect_blocker_(connect_blocker), ssl_ctx_(ssl_ctx), session_(nullptr), data_pending_(nullptr), data_pendinglen_(0), - addr_idx_(0), group_(group), state_(DISCONNECTED), + addr_idx_(0), group_(group), index_(idx), state_(DISCONNECTED), connection_check_state_(CONNECTION_CHECK_NONE), flow_control_(false) { read_ = write_ = &Http2Session::noop; @@ -1755,4 +1755,6 @@ size_t Http2Session::get_addr_idx() const { return addr_idx_; } size_t Http2Session::get_group() const { return group_; } +size_t Http2Session::get_index() const { return index_; } + } // namespace shrpx diff --git a/src/shrpx_http2_session.h b/src/shrpx_http2_session.h index 184252de..3ae3e784 100644 --- a/src/shrpx_http2_session.h +++ b/src/shrpx_http2_session.h @@ -58,7 +58,8 @@ struct StreamData { class Http2Session { public: Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, - ConnectBlocker *connect_blocker, Worker *worker, size_t group); + ConnectBlocker *connect_blocker, Worker *worker, size_t group, + size_t idx); ~Http2Session(); int check_cert(); @@ -153,6 +154,8 @@ public: size_t get_group() const; + size_t get_index() const; + enum { // Disconnected DISCONNECTED, @@ -206,6 +209,9 @@ private: // index of get_config()->downstream_addrs this object uses size_t addr_idx_; size_t group_; + // index inside group, this is used to pin frontend to certain + // HTTP/2 backend for better throughput. + size_t index_; int state_; int connection_check_state_; bool flow_control_; diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index e5f0f8d4..ecf55b51 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -83,9 +83,9 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, if (m == 0) { m = get_config()->downstream_addr_groups[group].addrs.size(); } - for (; m; --m) { + for (size_t idx = 0; idx < m; ++idx) { dgrp.http2sessions.push_back(make_unique( - loop_, cl_ssl_ctx, connect_blocker_.get(), this, group)); + loop_, cl_ssl_ctx, connect_blocker_.get(), this, group, idx)); } ++group; } From fb7775e382ef62189dfe98e93474fcefa116e0b4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 13 Jul 2015 21:44:06 +0900 Subject: [PATCH 16/44] nghttpx: Detect loop in --include paths --- src/shrpx.cc | 19 ++++++++++++++----- src/shrpx_config.cc | 22 ++++++++++++++++++---- src/shrpx_config.h | 12 ++++++++---- 3 files changed, 40 insertions(+), 13 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 26326e2f..7160bf6c 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2030,11 +2030,13 @@ int main(int argc, char **argv) { } if (conf_exists(get_config()->conf_path.get())) { - if (load_config(get_config()->conf_path.get()) == -1) { + std::set include_set; + if (load_config(get_config()->conf_path.get(), include_set) == -1) { LOG(FATAL) << "Failed to load configuration from " << get_config()->conf_path.get(); exit(EXIT_FAILURE); } + assert(include_set.empty()); } if (argc - optind >= 2) { @@ -2046,11 +2048,18 @@ int main(int argc, char **argv) { // parsing option values. reopen_log_files(); - for (size_t i = 0, len = cmdcfgs.size(); i < len; ++i) { - if (parse_config(cmdcfgs[i].first, cmdcfgs[i].second) == -1) { - LOG(FATAL) << "Failed to parse command-line argument."; - exit(EXIT_FAILURE); + { + std::set include_set; + + for (size_t i = 0, len = cmdcfgs.size(); i < len; ++i) { + if (parse_config(cmdcfgs[i].first, cmdcfgs[i].second, include_set) == + -1) { + LOG(FATAL) << "Failed to parse command-line argument."; + exit(EXIT_FAILURE); + } } + + assert(include_set.empty()); } if (get_config()->accesslog_syslog || get_config()->errorlog_syslog) { diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 49ad49a8..6a978b1c 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -505,7 +505,8 @@ void parse_mapping(const DownstreamAddr &addr, const char *src) { } } // namespace -int parse_config(const char *opt, const char *optarg) { +int parse_config(const char *opt, const char *optarg, + std::set &included_set) { char host[NI_MAXHOST]; uint16_t port; @@ -1220,7 +1221,20 @@ int parse_config(const char *opt, const char *optarg) { } if (util::strieq(opt, SHRPX_OPT_INCLUDE)) { - return load_config(optarg); + if (included_set.count(optarg)) { + LOG(ERROR) << opt << ": " << optarg << " has already been included"; + return -1; + } + + included_set.emplace(optarg); + auto rv = load_config(optarg, included_set); + included_set.erase(optarg); + + if (rv != 0) { + return -1; + } + + return 0; } if (util::strieq(opt, "conf")) { @@ -1234,7 +1248,7 @@ int parse_config(const char *opt, const char *optarg) { return -1; } -int load_config(const char *filename) { +int load_config(const char *filename, std::set &include_set) { std::ifstream in(filename, std::ios::binary); if (!in) { LOG(ERROR) << "Could not open config file " << filename; @@ -1258,7 +1272,7 @@ int load_config(const char *filename) { } line[i] = '\0'; auto s = line.c_str(); - if (parse_config(s, s + i + 1) == -1) { + if (parse_config(s, s + i + 1, include_set) == -1) { return -1; } } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 5f23c5b3..ea455f02 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -42,6 +42,7 @@ #include #include #include +#include #include @@ -379,13 +380,16 @@ void create_config(); // Parses option name |opt| and value |optarg|. The results are // stored into statically allocated Config object. This function -// returns 0 if it succeeds, or -1. -int parse_config(const char *opt, const char *optarg); +// returns 0 if it succeeds, or -1. The |included_set| contains the +// all paths already included while processing this configuration, to +// avoid loop in --include option. +int parse_config(const char *opt, const char *optarg, + std::set &included_set); // Loads configurations from |filename| and stores them in statically // allocated Config object. This function returns 0 if it succeeds, or -// -1. -int load_config(const char *filename); +// -1. See parse_config() for |include_set|. +int load_config(const char *filename, std::set &include_set); // Read passwd from |filename| std::string read_passwd_from_file(const char *filename); From 7c301defbd6f7f993bfc0448b697f2aee4d739a6 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Jul 2015 21:37:34 +0900 Subject: [PATCH 17/44] clang-format src/nghttp.cc --- src/nghttp.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/nghttp.cc b/src/nghttp.cc index 78cd5652..eb423c6d 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -1376,10 +1376,11 @@ void HttpClient::output_har(FILE *outfile) { entry, "startedDateTime", json_string(util::format_iso8601(request_time).c_str())); json_object_set_new(entry, "time", json_real(time_sum)); - + auto pushed = req->stream_id % 2 == 0; - json_object_set_new(entry, "comment", json_string(pushed ? "Pushed Object" : "")); + json_object_set_new(entry, "comment", + json_string(pushed ? "Pushed Object" : "")); auto request = json_object(); json_object_set_new(entry, "request", request); @@ -1463,7 +1464,8 @@ void HttpClient::output_har(FILE *outfile) { json_object_set_new(timings, "receive", json_real(receive_delta)); json_object_set_new(entry, "pageref", json_string(PAGE_ID)); - json_object_set_new(entry, "connection", json_string(util::utos(req->stream_id).c_str())); + json_object_set_new(entry, "connection", + json_string(util::utos(req->stream_id).c_str())); } json_dumpf(root, outfile, JSON_PRESERVE_ORDER | JSON_INDENT(2)); From 7f7b6d641d35083f4ed66c11d5c031dc7060a107 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Jul 2015 22:25:52 +0900 Subject: [PATCH 18/44] nghttpx: Allow log variable to be enclosed by curly braces --- src/shrpx.cc | 3 ++ src/shrpx_config.cc | 80 ++++++++++++++++++++++++---------------- src/shrpx_config_test.cc | 44 ++++++++++++++++++++-- 3 files changed, 93 insertions(+), 34 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 7160bf6c..d458ed68 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1407,6 +1407,9 @@ Logging: * $ssl_session_reused: "r" if SSL/TLS session was reused. Otherwise, "." + The variable can be enclosed by "{" and "}" for + disambiguation (e.g., ${remote_addr}). + Default: )" << DEFAULT_ACCESSLOG_FORMAT << R"( --errorlog-file= Set path to write error log. To reopen file, send USR1 diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 6a978b1c..c1501dff 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -370,52 +370,69 @@ std::vector parse_log_format(const char *optarg) { ++p; - for (; p != eop && var_token(*p); ++p) - ; + const char *var_name; + size_t var_namelen; + if (p != eop && *p == '{') { + var_name = ++p; + for (; p != eop && var_token(*p); ++p) + ; - auto varlen = p - var_start; + if (p == eop || *p != '}') { + LOG(WARN) << "Missing '}' after " << std::string(var_start, p); + continue; + } + + var_namelen = p - var_name; + ++p; + } else { + var_name = p; + for (; p != eop && var_token(*p); ++p) + ; + + var_namelen = p - var_name; + } auto type = SHRPX_LOGF_NONE; const char *value = nullptr; size_t valuelen = 0; - if (util::strieq_l("$remote_addr", var_start, varlen)) { + if (util::strieq_l("remote_addr", var_name, var_namelen)) { type = SHRPX_LOGF_REMOTE_ADDR; - } else if (util::strieq_l("$time_local", var_start, varlen)) { + } else if (util::strieq_l("time_local", var_name, var_namelen)) { type = SHRPX_LOGF_TIME_LOCAL; - } else if (util::strieq_l("$time_iso8601", var_start, varlen)) { + } else if (util::strieq_l("time_iso8601", var_name, var_namelen)) { type = SHRPX_LOGF_TIME_ISO8601; - } else if (util::strieq_l("$request", var_start, varlen)) { + } else if (util::strieq_l("request", var_name, var_namelen)) { type = SHRPX_LOGF_REQUEST; - } else if (util::strieq_l("$status", var_start, varlen)) { + } else if (util::strieq_l("status", var_name, var_namelen)) { type = SHRPX_LOGF_STATUS; - } else if (util::strieq_l("$body_bytes_sent", var_start, varlen)) { + } else if (util::strieq_l("body_bytes_sent", var_name, var_namelen)) { type = SHRPX_LOGF_BODY_BYTES_SENT; - } else if (util::istartsWith(var_start, varlen, "$http_")) { + } else if (util::istartsWith(var_name, var_namelen, "http_")) { type = SHRPX_LOGF_HTTP; - value = var_start + sizeof("$http_") - 1; - valuelen = varlen - (sizeof("$http_") - 1); - } else if (util::strieq_l("$remote_port", var_start, varlen)) { + value = var_name + sizeof("http_") - 1; + valuelen = var_namelen - (sizeof("http_") - 1); + } else if (util::strieq_l("remote_port", var_name, var_namelen)) { type = SHRPX_LOGF_REMOTE_PORT; - } else if (util::strieq_l("$server_port", var_start, varlen)) { + } else if (util::strieq_l("server_port", var_name, var_namelen)) { type = SHRPX_LOGF_SERVER_PORT; - } else if (util::strieq_l("$request_time", var_start, varlen)) { + } else if (util::strieq_l("request_time", var_name, var_namelen)) { type = SHRPX_LOGF_REQUEST_TIME; - } else if (util::strieq_l("$pid", var_start, varlen)) { + } else if (util::strieq_l("pid", var_name, var_namelen)) { type = SHRPX_LOGF_PID; - } else if (util::strieq_l("$alpn", var_start, varlen)) { + } else if (util::strieq_l("alpn", var_name, var_namelen)) { type = SHRPX_LOGF_ALPN; - } else if (util::strieq_l("$ssl_cipher", var_start, varlen)) { + } else if (util::strieq_l("ssl_cipher", var_name, var_namelen)) { type = SHRPX_LOGF_SSL_CIPHER; - } else if (util::strieq_l("$ssl_protocol", var_start, varlen)) { + } else if (util::strieq_l("ssl_protocol", var_name, var_namelen)) { type = SHRPX_LOGF_SSL_PROTOCOL; - } else if (util::strieq_l("$ssl_session_id", var_start, varlen)) { + } else if (util::strieq_l("ssl_session_id", var_name, var_namelen)) { type = SHRPX_LOGF_SSL_SESSION_ID; - } else if (util::strieq_l("$ssl_session_reused", var_start, varlen)) { + } else if (util::strieq_l("ssl_session_reused", var_name, var_namelen)) { type = SHRPX_LOGF_SSL_SESSION_REUSED; } else { LOG(WARN) << "Unrecognized log format variable: " - << std::string(var_start, varlen); + << std::string(var_name, var_namelen); continue; } @@ -425,19 +442,20 @@ std::vector parse_log_format(const char *optarg) { strcopy(literal_start, var_start - literal_start))); } + literal_start = p; + if (value == nullptr) { res.push_back(make_log_fragment(type)); - } else { - res.push_back(make_log_fragment(type, strcopy(value, valuelen))); - auto &v = res.back().value; - for (size_t i = 0; v[i]; ++i) { - if (v[i] == '_') { - v[i] = '-'; - } - } + continue; } - literal_start = var_start + varlen; + res.push_back(make_log_fragment(type, strcopy(value, valuelen))); + auto &v = res.back().value; + for (size_t i = 0; v[i]; ++i) { + if (v[i] == '_') { + v[i] = '-'; + } + } } if (literal_start != eop) { diff --git a/src/shrpx_config_test.cc b/src/shrpx_config_test.cc index ab6a0f27..c5116294 100644 --- a/src/shrpx_config_test.cc +++ b/src/shrpx_config_test.cc @@ -95,9 +95,9 @@ void test_shrpx_config_parse_header(void) { } void test_shrpx_config_parse_log_format(void) { - auto res = parse_log_format("$remote_addr - $remote_user [$time_local] " - "\"$request\" $status $body_bytes_sent " - "\"$http_referer\" \"$http_user_agent\""); + auto res = parse_log_format(R"($remote_addr - $remote_user [$time_local] )" + R"("$request" $status $body_bytes_sent )" + R"("${http_referer}" "$http_user_agent")"); CU_ASSERT(14 == res.size()); CU_ASSERT(SHRPX_LOGF_REMOTE_ADDR == res[0].type); @@ -136,6 +136,44 @@ void test_shrpx_config_parse_log_format(void) { CU_ASSERT(SHRPX_LOGF_LITERAL == res[13].type); CU_ASSERT(0 == strcmp("\"", res[13].value.get())); + + res = parse_log_format("$"); + + CU_ASSERT(1 == res.size()); + + CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type); + CU_ASSERT(0 == strcmp("$", res[0].value.get())); + + res = parse_log_format("${"); + + CU_ASSERT(1 == res.size()); + + CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type); + CU_ASSERT(0 == strcmp("${", res[0].value.get())); + + res = parse_log_format("${a"); + + CU_ASSERT(1 == res.size()); + + CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type); + CU_ASSERT(0 == strcmp("${a", res[0].value.get())); + + res = parse_log_format("${a "); + + CU_ASSERT(1 == res.size()); + + CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type); + CU_ASSERT(0 == strcmp("${a ", res[0].value.get())); + + res = parse_log_format("$$remote_addr"); + + CU_ASSERT(2 == res.size()); + + CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type); + CU_ASSERT(0 == strcmp("$", res[0].value.get())); + + CU_ASSERT(SHRPX_LOGF_REMOTE_ADDR == res[1].type); + CU_ASSERT(nullptr == res[1].value.get()); } void test_shrpx_config_read_tls_ticket_key_file(void) { From 0a6877d091e51ba18e971818f365643a2ba59be5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Jul 2015 22:40:33 +0900 Subject: [PATCH 19/44] nghttpx: Supply template version strcopy --- src/shrpx_config.cc | 33 +++++++-------------------------- src/shrpx_config.h | 24 ++++++++++++++++++------ 2 files changed, 25 insertions(+), 32 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index c1501dff..cb255cf3 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -59,8 +59,6 @@ #include "template.h" #include "base64.h" -using namespace nghttp2; - namespace shrpx { namespace { @@ -227,21 +225,6 @@ std::string read_passwd_from_file(const char *filename) { return line; } -std::unique_ptr strcopy(const char *val) { - return strcopy(val, strlen(val)); -} - -std::unique_ptr strcopy(const char *val, size_t len) { - auto res = make_unique(len + 1); - memcpy(res.get(), val, len); - res[len] = '\0'; - return res; -} - -std::unique_ptr strcopy(const std::string &val) { - return strcopy(val.c_str(), val.size()); -} - std::vector parse_config_str_list(const char *s, char delim) { size_t len = 1; for (const char *first = s, *p = nullptr; (p = strchr(first, delim)); @@ -394,7 +377,6 @@ std::vector parse_log_format(const char *optarg) { auto type = SHRPX_LOGF_NONE; const char *value = nullptr; - size_t valuelen = 0; if (util::strieq_l("remote_addr", var_name, var_namelen)) { type = SHRPX_LOGF_REMOTE_ADDR; @@ -411,7 +393,6 @@ std::vector parse_log_format(const char *optarg) { } else if (util::istartsWith(var_name, var_namelen, "http_")) { type = SHRPX_LOGF_HTTP; value = var_name + sizeof("http_") - 1; - valuelen = var_namelen - (sizeof("http_") - 1); } else if (util::strieq_l("remote_port", var_name, var_namelen)) { type = SHRPX_LOGF_REMOTE_PORT; } else if (util::strieq_l("server_port", var_name, var_namelen)) { @@ -437,9 +418,8 @@ std::vector parse_log_format(const char *optarg) { } if (literal_start < var_start) { - res.push_back( - make_log_fragment(SHRPX_LOGF_LITERAL, - strcopy(literal_start, var_start - literal_start))); + res.push_back(make_log_fragment(SHRPX_LOGF_LITERAL, + strcopy(literal_start, var_start))); } literal_start = p; @@ -449,7 +429,8 @@ std::vector parse_log_format(const char *optarg) { continue; } - res.push_back(make_log_fragment(type, strcopy(value, valuelen))); + res.push_back( + make_log_fragment(type, strcopy(value, var_name + var_namelen))); auto &v = res.back().value; for (size_t i = 0; v[i]; ++i) { if (v[i] == '_') { @@ -459,8 +440,8 @@ std::vector parse_log_format(const char *optarg) { } if (literal_start != eop) { - res.push_back(make_log_fragment( - SHRPX_LOGF_LITERAL, strcopy(literal_start, eop - literal_start))); + res.push_back( + make_log_fragment(SHRPX_LOGF_LITERAL, strcopy(literal_start, eop))); } return res; @@ -537,7 +518,7 @@ int parse_config(const char *opt, const char *optarg, DownstreamAddr addr; if (util::istartsWith(optarg, SHRPX_UNIX_PATH_PREFIX)) { auto path = optarg + str_size(SHRPX_UNIX_PATH_PREFIX); - addr.host = strcopy(path, pat_delim - path); + addr.host = strcopy(path, pat_delim); addr.host_unix = true; } else { if (split_host_port(host, sizeof(host), &port, optarg, diff --git a/src/shrpx_config.h b/src/shrpx_config.h index ea455f02..87af15ba 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -50,6 +50,10 @@ #include +#include "template.h" + +using namespace nghttp2; + namespace shrpx { struct LogFragment; @@ -416,15 +420,23 @@ std::pair parse_header(const char *optarg); std::vector parse_log_format(const char *optarg); -// Returns a copy of NULL-terminated string |val|. -std::unique_ptr strcopy(const char *val); +// Returns a copy of NULL-terminated string [first, last). +template +std::unique_ptr strcopy(InputIt first, InputIt last) { + auto res = make_unique(last - first + 1); + *std::copy(first, last, res.get()) = '\0'; + return res; +} -// Returns a copy of string |val| of length |n|. The returned string -// will be NULL-terminated. -std::unique_ptr strcopy(const char *val, size_t n); +// Returns a copy of NULL-terminated string |val|. +inline std::unique_ptr strcopy(const char *val) { + return strcopy(val, val + strlen(val)); +} // Returns a copy of val.c_str(). -std::unique_ptr strcopy(const std::string &val); +inline std::unique_ptr strcopy(const std::string &val) { + return strcopy(std::begin(val), std::end(val)); +} // Returns string for syslog |facility|. const char *str_syslog_facility(int facility); From 27da08ee68168eb9b548947881f6eace969c3331 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Jul 2015 22:43:02 +0900 Subject: [PATCH 20/44] nghttpx: Add inline LogFragment ctor --- src/shrpx_config.cc | 18 ++++-------------- src/shrpx_log.h | 2 ++ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index cb255cf3..37b9693b 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -323,13 +323,6 @@ int parse_int(T *dest, const char *opt, const char *optarg) { return 0; } -namespace { -LogFragment make_log_fragment(LogFragmentType type, - std::unique_ptr value = nullptr) { - return LogFragment{type, std::move(value)}; -} -} // namespace - namespace { bool var_token(char c) { return util::isAlpha(c) || util::isDigit(c) || c == '_'; @@ -418,19 +411,17 @@ std::vector parse_log_format(const char *optarg) { } if (literal_start < var_start) { - res.push_back(make_log_fragment(SHRPX_LOGF_LITERAL, - strcopy(literal_start, var_start))); + res.emplace_back(SHRPX_LOGF_LITERAL, strcopy(literal_start, var_start)); } literal_start = p; if (value == nullptr) { - res.push_back(make_log_fragment(type)); + res.emplace_back(type); continue; } - res.push_back( - make_log_fragment(type, strcopy(value, var_name + var_namelen))); + res.emplace_back(type, strcopy(value, var_name + var_namelen)); auto &v = res.back().value; for (size_t i = 0; v[i]; ++i) { if (v[i] == '_') { @@ -440,8 +431,7 @@ std::vector parse_log_format(const char *optarg) { } if (literal_start != eop) { - res.push_back( - make_log_fragment(SHRPX_LOGF_LITERAL, strcopy(literal_start, eop))); + res.emplace_back(SHRPX_LOGF_LITERAL, strcopy(literal_start, eop)); } return res; diff --git a/src/shrpx_log.h b/src/shrpx_log.h index 1d63e641..e13be41e 100644 --- a/src/shrpx_log.h +++ b/src/shrpx_log.h @@ -123,6 +123,8 @@ enum LogFragmentType { }; struct LogFragment { + LogFragment(LogFragmentType type, std::unique_ptr value = nullptr) + : type(type), value(std::move(value)) {} LogFragmentType type; std::unique_ptr value; }; From 8c1e8635230350fda69f5421b1e5b481cc2df21f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Jul 2015 23:21:38 +0900 Subject: [PATCH 21/44] nghttpx: Refactor option name lookup --- gennghttpxfun.py | 117 ++++++ gentokenlookup.py | 14 +- src/shrpx_config.cc | 978 +++++++++++++++++++++++++++++++++----------- 3 files changed, 854 insertions(+), 255 deletions(-) create mode 100755 gennghttpxfun.py diff --git a/gennghttpxfun.py b/gennghttpxfun.py new file mode 100755 index 00000000..81836fe9 --- /dev/null +++ b/gennghttpxfun.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python + +from gentokenlookup import gentokenlookup + +OPTIONS = [ + "private-key-file", + "private-key-passwd-file", + "certificate-file", + "dh-param-file", + "subcert", + "backend", + "frontend", + "workers", + "http2-max-concurrent-streams", + "log-level", + "daemon", + "http2-proxy", + "http2-bridge", + "client-proxy", + "add-x-forwarded-for", + "strip-incoming-x-forwarded-for", + "no-via", + "frontend-http2-read-timeout", + "frontend-read-timeout", + "frontend-write-timeout", + "backend-read-timeout", + "backend-write-timeout", + "stream-read-timeout", + "stream-write-timeout", + "accesslog-file", + "accesslog-syslog", + "accesslog-format", + "errorlog-file", + "errorlog-syslog", + "backend-keep-alive-timeout", + "frontend-http2-window-bits", + "backend-http2-window-bits", + "frontend-http2-connection-window-bits", + "backend-http2-connection-window-bits", + "frontend-no-tls", + "backend-no-tls", + "backend-tls-sni-field", + "pid-file", + "user", + "syslog-facility", + "backlog", + "ciphers", + "client", + "insecure", + "cacert", + "backend-ipv4", + "backend-ipv6", + "backend-http-proxy-uri", + "read-rate", + "read-burst", + "write-rate", + "write-burst", + "worker-read-rate", + "worker-read-burst", + "worker-write-rate", + "worker-write-burst", + "npn-list", + "tls-proto-list", + "verify-client", + "verify-client-cacert", + "client-private-key-file", + "client-cert-file", + "frontend-http2-dump-request-header", + "frontend-http2-dump-response-header", + "http2-no-cookie-crumbling", + "frontend-frame-debug", + "padding", + "altsvc", + "add-request-header", + "add-response-header", + "worker-frontend-connections", + "no-location-rewrite", + "no-host-rewrite", + "backend-http1-connections-per-host", + "backend-http1-connections-per-frontend", + "listener-disable-timeout", + "tls-ticket-key-file", + "rlimit-nofile", + "backend-request-buffer", + "backend-response-buffer", + "no-server-push", + "backend-http2-connections-per-worker", + "fetch-ocsp-response-file", + "ocsp-update-interval", + "no-ocsp", + "header-field-buffer", + "max-header-fields", + "include", + "conf", +] + +LOGVARS = [ + "remote_addr", + "time_local", + "time_iso8601", + "request", + "status", + "body_bytes_sent", + "remote_port", + "server_port", + "request_time", + "pid", + "alpn", + "ssl_cipher", + "ssl_protocol", + "ssl_session_id", + "ssl_session_reused", +] + +if __name__ == '__main__': + gentokenlookup(OPTIONS, 'SHRPX_OPTID', value_type='char', comp_fun='util::strieq_l') + gentokenlookup(LOGVARS, 'SHRPX_LOGF', value_type='char', comp_fun='util::strieq_l') diff --git a/gentokenlookup.py b/gentokenlookup.py index 254fd5d6..cf96bf68 100644 --- a/gentokenlookup.py +++ b/gentokenlookup.py @@ -33,10 +33,10 @@ enum {''' {}_MAXIDX, }};'''.format(prefix) -def gen_index_header(tokens, prefix): +def gen_index_header(tokens, prefix, value_type, comp_fun): print '''\ -int lookup_token(const uint8_t *name, size_t namelen) { - switch (namelen) {''' +int lookup_token(const {} *name, size_t namelen) {{ + switch (namelen) {{'''.format(value_type) b = build_header(tokens) for size in sorted(b.keys()): ents = b[size] @@ -50,9 +50,9 @@ int lookup_token(const uint8_t *name, size_t namelen) { case '{}':'''.format(c) for k in headers: print '''\ - if (util::streq_l("{}", name, {})) {{ + if ({}("{}", name, {})) {{ return {}; - }}'''.format(k[:-1], size - 1, to_enum_hd(k, prefix)) + }}'''.format(comp_fun, k[:-1], size - 1, to_enum_hd(k, prefix)) print '''\ break;''' print '''\ @@ -63,7 +63,7 @@ int lookup_token(const uint8_t *name, size_t namelen) { return -1; }''' -def gentokenlookup(tokens, prefix): +def gentokenlookup(tokens, prefix, value_type='uint8_t', comp_fun='util::streq_l'): gen_enum(tokens, prefix) print '' - gen_index_header(tokens, prefix) + gen_index_header(tokens, prefix, value_type, comp_fun) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 37b9693b..3064b5ea 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -494,12 +494,649 @@ void parse_mapping(const DownstreamAddr &addr, const char *src) { } } // namespace +enum { + SHRPX_OPTID_ACCESSLOG_FILE, + SHRPX_OPTID_ACCESSLOG_FORMAT, + SHRPX_OPTID_ACCESSLOG_SYSLOG, + SHRPX_OPTID_ADD_REQUEST_HEADER, + SHRPX_OPTID_ADD_RESPONSE_HEADER, + SHRPX_OPTID_ADD_X_FORWARDED_FOR, + SHRPX_OPTID_ALTSVC, + SHRPX_OPTID_BACKEND, + SHRPX_OPTID_BACKEND_HTTP_PROXY_URI, + SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND, + SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST, + SHRPX_OPTID_BACKEND_HTTP2_CONNECTION_WINDOW_BITS, + SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER, + SHRPX_OPTID_BACKEND_HTTP2_WINDOW_BITS, + SHRPX_OPTID_BACKEND_IPV4, + SHRPX_OPTID_BACKEND_IPV6, + SHRPX_OPTID_BACKEND_KEEP_ALIVE_TIMEOUT, + SHRPX_OPTID_BACKEND_NO_TLS, + SHRPX_OPTID_BACKEND_READ_TIMEOUT, + SHRPX_OPTID_BACKEND_REQUEST_BUFFER, + SHRPX_OPTID_BACKEND_RESPONSE_BUFFER, + SHRPX_OPTID_BACKEND_TLS_SNI_FIELD, + SHRPX_OPTID_BACKEND_WRITE_TIMEOUT, + SHRPX_OPTID_BACKLOG, + SHRPX_OPTID_CACERT, + SHRPX_OPTID_CERTIFICATE_FILE, + SHRPX_OPTID_CIPHERS, + SHRPX_OPTID_CLIENT, + SHRPX_OPTID_CLIENT_CERT_FILE, + SHRPX_OPTID_CLIENT_PRIVATE_KEY_FILE, + SHRPX_OPTID_CLIENT_PROXY, + SHRPX_OPTID_CONF, + SHRPX_OPTID_DAEMON, + SHRPX_OPTID_DH_PARAM_FILE, + SHRPX_OPTID_ERRORLOG_FILE, + SHRPX_OPTID_ERRORLOG_SYSLOG, + SHRPX_OPTID_FETCH_OCSP_RESPONSE_FILE, + SHRPX_OPTID_FRONTEND, + SHRPX_OPTID_FRONTEND_FRAME_DEBUG, + SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS, + SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER, + SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER, + SHRPX_OPTID_FRONTEND_HTTP2_READ_TIMEOUT, + SHRPX_OPTID_FRONTEND_HTTP2_WINDOW_BITS, + SHRPX_OPTID_FRONTEND_NO_TLS, + SHRPX_OPTID_FRONTEND_READ_TIMEOUT, + SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT, + SHRPX_OPTID_HEADER_FIELD_BUFFER, + SHRPX_OPTID_HTTP2_BRIDGE, + SHRPX_OPTID_HTTP2_MAX_CONCURRENT_STREAMS, + SHRPX_OPTID_HTTP2_NO_COOKIE_CRUMBLING, + SHRPX_OPTID_HTTP2_PROXY, + SHRPX_OPTID_INCLUDE, + SHRPX_OPTID_INSECURE, + SHRPX_OPTID_LISTENER_DISABLE_TIMEOUT, + SHRPX_OPTID_LOG_LEVEL, + SHRPX_OPTID_MAX_HEADER_FIELDS, + SHRPX_OPTID_NO_HOST_REWRITE, + SHRPX_OPTID_NO_LOCATION_REWRITE, + SHRPX_OPTID_NO_OCSP, + SHRPX_OPTID_NO_SERVER_PUSH, + SHRPX_OPTID_NO_VIA, + SHRPX_OPTID_NPN_LIST, + SHRPX_OPTID_OCSP_UPDATE_INTERVAL, + SHRPX_OPTID_PADDING, + SHRPX_OPTID_PID_FILE, + SHRPX_OPTID_PRIVATE_KEY_FILE, + SHRPX_OPTID_PRIVATE_KEY_PASSWD_FILE, + SHRPX_OPTID_READ_BURST, + SHRPX_OPTID_READ_RATE, + SHRPX_OPTID_RLIMIT_NOFILE, + SHRPX_OPTID_STREAM_READ_TIMEOUT, + SHRPX_OPTID_STREAM_WRITE_TIMEOUT, + SHRPX_OPTID_STRIP_INCOMING_X_FORWARDED_FOR, + SHRPX_OPTID_SUBCERT, + SHRPX_OPTID_SYSLOG_FACILITY, + SHRPX_OPTID_TLS_PROTO_LIST, + SHRPX_OPTID_TLS_TICKET_KEY_FILE, + SHRPX_OPTID_USER, + SHRPX_OPTID_VERIFY_CLIENT, + SHRPX_OPTID_VERIFY_CLIENT_CACERT, + SHRPX_OPTID_WORKER_FRONTEND_CONNECTIONS, + SHRPX_OPTID_WORKER_READ_BURST, + SHRPX_OPTID_WORKER_READ_RATE, + SHRPX_OPTID_WORKER_WRITE_BURST, + SHRPX_OPTID_WORKER_WRITE_RATE, + SHRPX_OPTID_WORKERS, + SHRPX_OPTID_WRITE_BURST, + SHRPX_OPTID_WRITE_RATE, + SHRPX_OPTID_MAXIDX, +}; + +namespace { +int option_lookup_token(const char *name, size_t namelen) { + switch (namelen) { + case 4: + switch (name[3]) { + case 'f': + if (util::strieq_l("con", name, 3)) { + return SHRPX_OPTID_CONF; + } + break; + case 'r': + if (util::strieq_l("use", name, 3)) { + return SHRPX_OPTID_USER; + } + break; + } + break; + case 6: + switch (name[5]) { + case 'a': + if (util::strieq_l("no-vi", name, 5)) { + return SHRPX_OPTID_NO_VIA; + } + break; + case 'c': + if (util::strieq_l("altsv", name, 5)) { + return SHRPX_OPTID_ALTSVC; + } + break; + case 'n': + if (util::strieq_l("daemo", name, 5)) { + return SHRPX_OPTID_DAEMON; + } + break; + case 't': + if (util::strieq_l("cacer", name, 5)) { + return SHRPX_OPTID_CACERT; + } + if (util::strieq_l("clien", name, 5)) { + return SHRPX_OPTID_CLIENT; + } + break; + } + break; + case 7: + switch (name[6]) { + case 'd': + if (util::strieq_l("backen", name, 6)) { + return SHRPX_OPTID_BACKEND; + } + break; + case 'e': + if (util::strieq_l("includ", name, 6)) { + return SHRPX_OPTID_INCLUDE; + } + break; + case 'g': + if (util::strieq_l("backlo", name, 6)) { + return SHRPX_OPTID_BACKLOG; + } + if (util::strieq_l("paddin", name, 6)) { + return SHRPX_OPTID_PADDING; + } + break; + case 'p': + if (util::strieq_l("no-ocs", name, 6)) { + return SHRPX_OPTID_NO_OCSP; + } + break; + case 's': + if (util::strieq_l("cipher", name, 6)) { + return SHRPX_OPTID_CIPHERS; + } + if (util::strieq_l("worker", name, 6)) { + return SHRPX_OPTID_WORKERS; + } + break; + case 't': + if (util::strieq_l("subcer", name, 6)) { + return SHRPX_OPTID_SUBCERT; + } + break; + } + break; + case 8: + switch (name[7]) { + case 'd': + if (util::strieq_l("fronten", name, 7)) { + return SHRPX_OPTID_FRONTEND; + } + break; + case 'e': + if (util::strieq_l("insecur", name, 7)) { + return SHRPX_OPTID_INSECURE; + } + if (util::strieq_l("pid-fil", name, 7)) { + return SHRPX_OPTID_PID_FILE; + } + break; + case 't': + if (util::strieq_l("npn-lis", name, 7)) { + return SHRPX_OPTID_NPN_LIST; + } + break; + } + break; + case 9: + switch (name[8]) { + case 'e': + if (util::strieq_l("read-rat", name, 8)) { + return SHRPX_OPTID_READ_RATE; + } + break; + case 'l': + if (util::strieq_l("log-leve", name, 8)) { + return SHRPX_OPTID_LOG_LEVEL; + } + break; + } + break; + case 10: + switch (name[9]) { + case 'e': + if (util::strieq_l("write-rat", name, 9)) { + return SHRPX_OPTID_WRITE_RATE; + } + break; + case 't': + if (util::strieq_l("read-burs", name, 9)) { + return SHRPX_OPTID_READ_BURST; + } + break; + } + break; + case 11: + switch (name[10]) { + case 't': + if (util::strieq_l("write-burs", name, 10)) { + return SHRPX_OPTID_WRITE_BURST; + } + break; + case 'y': + if (util::strieq_l("http2-prox", name, 10)) { + return SHRPX_OPTID_HTTP2_PROXY; + } + break; + } + break; + case 12: + switch (name[11]) { + case '4': + if (util::strieq_l("backend-ipv", name, 11)) { + return SHRPX_OPTID_BACKEND_IPV4; + } + break; + case '6': + if (util::strieq_l("backend-ipv", name, 11)) { + return SHRPX_OPTID_BACKEND_IPV6; + } + break; + case 'e': + if (util::strieq_l("http2-bridg", name, 11)) { + return SHRPX_OPTID_HTTP2_BRIDGE; + } + break; + case 'y': + if (util::strieq_l("client-prox", name, 11)) { + return SHRPX_OPTID_CLIENT_PROXY; + } + break; + } + break; + case 13: + switch (name[12]) { + case 'e': + if (util::strieq_l("dh-param-fil", name, 12)) { + return SHRPX_OPTID_DH_PARAM_FILE; + } + if (util::strieq_l("errorlog-fil", name, 12)) { + return SHRPX_OPTID_ERRORLOG_FILE; + } + if (util::strieq_l("rlimit-nofil", name, 12)) { + return SHRPX_OPTID_RLIMIT_NOFILE; + } + break; + case 't': + if (util::strieq_l("verify-clien", name, 12)) { + return SHRPX_OPTID_VERIFY_CLIENT; + } + break; + } + break; + case 14: + switch (name[13]) { + case 'e': + if (util::strieq_l("accesslog-fil", name, 13)) { + return SHRPX_OPTID_ACCESSLOG_FILE; + } + break; + case 'h': + if (util::strieq_l("no-server-pus", name, 13)) { + return SHRPX_OPTID_NO_SERVER_PUSH; + } + break; + case 's': + if (util::strieq_l("backend-no-tl", name, 13)) { + return SHRPX_OPTID_BACKEND_NO_TLS; + } + break; + case 't': + if (util::strieq_l("tls-proto-lis", name, 13)) { + return SHRPX_OPTID_TLS_PROTO_LIST; + } + break; + } + break; + case 15: + switch (name[14]) { + case 'e': + if (util::strieq_l("no-host-rewrit", name, 14)) { + return SHRPX_OPTID_NO_HOST_REWRITE; + } + break; + case 'g': + if (util::strieq_l("errorlog-syslo", name, 14)) { + return SHRPX_OPTID_ERRORLOG_SYSLOG; + } + break; + case 's': + if (util::strieq_l("frontend-no-tl", name, 14)) { + return SHRPX_OPTID_FRONTEND_NO_TLS; + } + break; + case 'y': + if (util::strieq_l("syslog-facilit", name, 14)) { + return SHRPX_OPTID_SYSLOG_FACILITY; + } + break; + } + break; + case 16: + switch (name[15]) { + case 'e': + if (util::strieq_l("certificate-fil", name, 15)) { + return SHRPX_OPTID_CERTIFICATE_FILE; + } + if (util::strieq_l("client-cert-fil", name, 15)) { + return SHRPX_OPTID_CLIENT_CERT_FILE; + } + if (util::strieq_l("private-key-fil", name, 15)) { + return SHRPX_OPTID_PRIVATE_KEY_FILE; + } + if (util::strieq_l("worker-read-rat", name, 15)) { + return SHRPX_OPTID_WORKER_READ_RATE; + } + break; + case 'g': + if (util::strieq_l("accesslog-syslo", name, 15)) { + return SHRPX_OPTID_ACCESSLOG_SYSLOG; + } + break; + case 't': + if (util::strieq_l("accesslog-forma", name, 15)) { + return SHRPX_OPTID_ACCESSLOG_FORMAT; + } + break; + } + break; + case 17: + switch (name[16]) { + case 'e': + if (util::strieq_l("worker-write-rat", name, 16)) { + return SHRPX_OPTID_WORKER_WRITE_RATE; + } + break; + case 's': + if (util::strieq_l("max-header-field", name, 16)) { + return SHRPX_OPTID_MAX_HEADER_FIELDS; + } + break; + case 't': + if (util::strieq_l("worker-read-burs", name, 16)) { + return SHRPX_OPTID_WORKER_READ_BURST; + } + break; + } + break; + case 18: + switch (name[17]) { + case 'r': + if (util::strieq_l("add-request-heade", name, 17)) { + return SHRPX_OPTID_ADD_REQUEST_HEADER; + } + break; + case 't': + if (util::strieq_l("worker-write-burs", name, 17)) { + return SHRPX_OPTID_WORKER_WRITE_BURST; + } + break; + } + break; + case 19: + switch (name[18]) { + case 'e': + if (util::strieq_l("no-location-rewrit", name, 18)) { + return SHRPX_OPTID_NO_LOCATION_REWRITE; + } + if (util::strieq_l("tls-ticket-key-fil", name, 18)) { + return SHRPX_OPTID_TLS_TICKET_KEY_FILE; + } + break; + case 'r': + if (util::strieq_l("add-response-heade", name, 18)) { + return SHRPX_OPTID_ADD_RESPONSE_HEADER; + } + if (util::strieq_l("add-x-forwarded-fo", name, 18)) { + return SHRPX_OPTID_ADD_X_FORWARDED_FOR; + } + if (util::strieq_l("header-field-buffe", name, 18)) { + return SHRPX_OPTID_HEADER_FIELD_BUFFER; + } + break; + case 't': + if (util::strieq_l("stream-read-timeou", name, 18)) { + return SHRPX_OPTID_STREAM_READ_TIMEOUT; + } + break; + } + break; + case 20: + switch (name[19]) { + case 'g': + if (util::strieq_l("frontend-frame-debu", name, 19)) { + return SHRPX_OPTID_FRONTEND_FRAME_DEBUG; + } + break; + case 'l': + if (util::strieq_l("ocsp-update-interva", name, 19)) { + return SHRPX_OPTID_OCSP_UPDATE_INTERVAL; + } + break; + case 't': + if (util::strieq_l("backend-read-timeou", name, 19)) { + return SHRPX_OPTID_BACKEND_READ_TIMEOUT; + } + if (util::strieq_l("stream-write-timeou", name, 19)) { + return SHRPX_OPTID_STREAM_WRITE_TIMEOUT; + } + if (util::strieq_l("verify-client-cacer", name, 19)) { + return SHRPX_OPTID_VERIFY_CLIENT_CACERT; + } + break; + } + break; + case 21: + switch (name[20]) { + case 'd': + if (util::strieq_l("backend-tls-sni-fiel", name, 20)) { + return SHRPX_OPTID_BACKEND_TLS_SNI_FIELD; + } + break; + case 't': + if (util::strieq_l("backend-write-timeou", name, 20)) { + return SHRPX_OPTID_BACKEND_WRITE_TIMEOUT; + } + if (util::strieq_l("frontend-read-timeou", name, 20)) { + return SHRPX_OPTID_FRONTEND_READ_TIMEOUT; + } + break; + } + break; + case 22: + switch (name[21]) { + case 'i': + if (util::strieq_l("backend-http-proxy-ur", name, 21)) { + return SHRPX_OPTID_BACKEND_HTTP_PROXY_URI; + } + break; + case 'r': + if (util::strieq_l("backend-request-buffe", name, 21)) { + return SHRPX_OPTID_BACKEND_REQUEST_BUFFER; + } + break; + case 't': + if (util::strieq_l("frontend-write-timeou", name, 21)) { + return SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT; + } + break; + } + break; + case 23: + switch (name[22]) { + case 'e': + if (util::strieq_l("client-private-key-fil", name, 22)) { + return SHRPX_OPTID_CLIENT_PRIVATE_KEY_FILE; + } + if (util::strieq_l("private-key-passwd-fil", name, 22)) { + return SHRPX_OPTID_PRIVATE_KEY_PASSWD_FILE; + } + break; + case 'r': + if (util::strieq_l("backend-response-buffe", name, 22)) { + return SHRPX_OPTID_BACKEND_RESPONSE_BUFFER; + } + break; + } + break; + case 24: + switch (name[23]) { + case 'e': + if (util::strieq_l("fetch-ocsp-response-fil", name, 23)) { + return SHRPX_OPTID_FETCH_OCSP_RESPONSE_FILE; + } + break; + case 't': + if (util::strieq_l("listener-disable-timeou", name, 23)) { + return SHRPX_OPTID_LISTENER_DISABLE_TIMEOUT; + } + break; + } + break; + case 25: + switch (name[24]) { + case 'g': + if (util::strieq_l("http2-no-cookie-crumblin", name, 24)) { + return SHRPX_OPTID_HTTP2_NO_COOKIE_CRUMBLING; + } + break; + case 's': + if (util::strieq_l("backend-http2-window-bit", name, 24)) { + return SHRPX_OPTID_BACKEND_HTTP2_WINDOW_BITS; + } + break; + } + break; + case 26: + switch (name[25]) { + case 's': + if (util::strieq_l("frontend-http2-window-bit", name, 25)) { + return SHRPX_OPTID_FRONTEND_HTTP2_WINDOW_BITS; + } + break; + case 't': + if (util::strieq_l("backend-keep-alive-timeou", name, 25)) { + return SHRPX_OPTID_BACKEND_KEEP_ALIVE_TIMEOUT; + } + break; + } + break; + case 27: + switch (name[26]) { + case 's': + if (util::strieq_l("worker-frontend-connection", name, 26)) { + return SHRPX_OPTID_WORKER_FRONTEND_CONNECTIONS; + } + break; + case 't': + if (util::strieq_l("frontend-http2-read-timeou", name, 26)) { + return SHRPX_OPTID_FRONTEND_HTTP2_READ_TIMEOUT; + } + break; + } + break; + case 28: + switch (name[27]) { + case 's': + if (util::strieq_l("http2-max-concurrent-stream", name, 27)) { + return SHRPX_OPTID_HTTP2_MAX_CONCURRENT_STREAMS; + } + break; + } + break; + case 30: + switch (name[29]) { + case 'r': + if (util::strieq_l("strip-incoming-x-forwarded-fo", name, 29)) { + return SHRPX_OPTID_STRIP_INCOMING_X_FORWARDED_FOR; + } + break; + } + break; + case 34: + switch (name[33]) { + case 'r': + if (util::strieq_l("frontend-http2-dump-request-heade", name, 33)) { + return SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER; + } + break; + case 't': + if (util::strieq_l("backend-http1-connections-per-hos", name, 33)) { + return SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST; + } + break; + } + break; + case 35: + switch (name[34]) { + case 'r': + if (util::strieq_l("frontend-http2-dump-response-heade", name, 34)) { + return SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER; + } + break; + } + break; + case 36: + switch (name[35]) { + case 'r': + if (util::strieq_l("backend-http2-connections-per-worke", name, 35)) { + return SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER; + } + break; + case 's': + if (util::strieq_l("backend-http2-connection-window-bit", name, 35)) { + return SHRPX_OPTID_BACKEND_HTTP2_CONNECTION_WINDOW_BITS; + } + break; + } + break; + case 37: + switch (name[36]) { + case 's': + if (util::strieq_l("frontend-http2-connection-window-bit", name, 36)) { + return SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS; + } + break; + } + break; + case 38: + switch (name[37]) { + case 'd': + if (util::strieq_l("backend-http1-connections-per-fronten", name, 37)) { + return SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND; + } + break; + } + break; + } + return -1; +} +} // namespace + int parse_config(const char *opt, const char *optarg, std::set &included_set) { char host[NI_MAXHOST]; uint16_t port; - if (util::strieq(opt, SHRPX_OPT_BACKEND)) { + auto optid = option_lookup_token(opt, strlen(opt)); + + switch (optid) { + case SHRPX_OPTID_BACKEND: { auto optarglen = strlen(optarg); const char *pat_delim = strchr(optarg, ';'); if (!pat_delim) { @@ -531,8 +1168,7 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND)) { + case SHRPX_OPTID_FRONTEND: { if (util::istartsWith(optarg, SHRPX_UNIX_PATH_PREFIX)) { auto path = optarg + str_size(SHRPX_UNIX_PATH_PREFIX); mod_config()->host = strcopy(path); @@ -553,136 +1189,88 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_WORKERS)) { + case SHRPX_OPTID_WORKERS: return parse_uint(&mod_config()->num_worker, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_HTTP2_MAX_CONCURRENT_STREAMS)) { + case SHRPX_OPTID_HTTP2_MAX_CONCURRENT_STREAMS: return parse_uint(&mod_config()->http2_max_concurrent_streams, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_LOG_LEVEL)) { + case SHRPX_OPTID_LOG_LEVEL: if (Log::set_severity_level_by_name(optarg) == -1) { LOG(ERROR) << opt << ": Invalid severity level: " << optarg; return -1; } return 0; - } - - if (util::strieq(opt, SHRPX_OPT_DAEMON)) { + case SHRPX_OPTID_DAEMON: mod_config()->daemon = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_HTTP2_PROXY)) { + case SHRPX_OPTID_HTTP2_PROXY: mod_config()->http2_proxy = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_HTTP2_BRIDGE)) { + case SHRPX_OPTID_HTTP2_BRIDGE: mod_config()->http2_bridge = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_CLIENT_PROXY)) { + case SHRPX_OPTID_CLIENT_PROXY: mod_config()->client_proxy = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_ADD_X_FORWARDED_FOR)) { + case SHRPX_OPTID_ADD_X_FORWARDED_FOR: mod_config()->add_x_forwarded_for = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_STRIP_INCOMING_X_FORWARDED_FOR)) { + case SHRPX_OPTID_STRIP_INCOMING_X_FORWARDED_FOR: mod_config()->strip_incoming_x_forwarded_for = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_NO_VIA)) { + case SHRPX_OPTID_NO_VIA: mod_config()->no_via = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND_HTTP2_READ_TIMEOUT)) { + case SHRPX_OPTID_FRONTEND_HTTP2_READ_TIMEOUT: return parse_duration(&mod_config()->http2_upstream_read_timeout, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND_READ_TIMEOUT)) { + case SHRPX_OPTID_FRONTEND_READ_TIMEOUT: return parse_duration(&mod_config()->upstream_read_timeout, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND_WRITE_TIMEOUT)) { + case SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT: return parse_duration(&mod_config()->upstream_write_timeout, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_READ_TIMEOUT)) { + case SHRPX_OPTID_BACKEND_READ_TIMEOUT: return parse_duration(&mod_config()->downstream_read_timeout, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_WRITE_TIMEOUT)) { + case SHRPX_OPTID_BACKEND_WRITE_TIMEOUT: return parse_duration(&mod_config()->downstream_write_timeout, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_STREAM_READ_TIMEOUT)) { + case SHRPX_OPTID_STREAM_READ_TIMEOUT: return parse_duration(&mod_config()->stream_read_timeout, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_STREAM_WRITE_TIMEOUT)) { + case SHRPX_OPTID_STREAM_WRITE_TIMEOUT: return parse_duration(&mod_config()->stream_write_timeout, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_ACCESSLOG_FILE)) { + case SHRPX_OPTID_ACCESSLOG_FILE: mod_config()->accesslog_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_ACCESSLOG_SYSLOG)) { + case SHRPX_OPTID_ACCESSLOG_SYSLOG: mod_config()->accesslog_syslog = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_ACCESSLOG_FORMAT)) { + case SHRPX_OPTID_ACCESSLOG_FORMAT: mod_config()->accesslog_format = parse_log_format(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_ERRORLOG_FILE)) { + case SHRPX_OPTID_ERRORLOG_FILE: mod_config()->errorlog_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_ERRORLOG_SYSLOG)) { + case SHRPX_OPTID_ERRORLOG_SYSLOG: mod_config()->errorlog_syslog = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_KEEP_ALIVE_TIMEOUT)) { + case SHRPX_OPTID_BACKEND_KEEP_ALIVE_TIMEOUT: return parse_duration(&mod_config()->downstream_idle_read_timeout, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND_HTTP2_WINDOW_BITS) || - util::strieq(opt, SHRPX_OPT_BACKEND_HTTP2_WINDOW_BITS)) { - + case SHRPX_OPTID_FRONTEND_HTTP2_WINDOW_BITS: + case SHRPX_OPTID_BACKEND_HTTP2_WINDOW_BITS: { size_t *resp; - if (util::strieq(opt, SHRPX_OPT_FRONTEND_HTTP2_WINDOW_BITS)) { + if (optid == SHRPX_OPTID_FRONTEND_HTTP2_WINDOW_BITS) { resp = &mod_config()->http2_upstream_window_bits; } else { resp = &mod_config()->http2_downstream_window_bits; @@ -706,13 +1294,11 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS) || - util::strieq(opt, SHRPX_OPT_BACKEND_HTTP2_CONNECTION_WINDOW_BITS)) { - + case SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS: + case SHRPX_OPTID_BACKEND_HTTP2_CONNECTION_WINDOW_BITS: { size_t *resp; - if (util::strieq(opt, SHRPX_OPT_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS)) { + if (optid == SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS) { resp = &mod_config()->http2_upstream_connection_window_bits; } else { resp = &mod_config()->http2_downstream_connection_window_bits; @@ -736,32 +1322,23 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND_NO_TLS)) { + case SHRPX_OPTID_FRONTEND_NO_TLS: mod_config()->upstream_no_tls = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_NO_TLS)) { + case SHRPX_OPTID_BACKEND_NO_TLS: mod_config()->downstream_no_tls = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_TLS_SNI_FIELD)) { + case SHRPX_OPTID_BACKEND_TLS_SNI_FIELD: mod_config()->backend_tls_sni_name = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_PID_FILE)) { + case SHRPX_OPTID_PID_FILE: mod_config()->pid_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_USER)) { + case SHRPX_OPTID_USER: { auto pwd = getpwnam(optarg); if (!pwd) { LOG(ERROR) << opt << ": failed to get uid from " << optarg << ": " @@ -774,14 +1351,11 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_PRIVATE_KEY_FILE)) { + case SHRPX_OPTID_PRIVATE_KEY_FILE: mod_config()->private_key_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_PRIVATE_KEY_PASSWD_FILE)) { + case SHRPX_OPTID_PRIVATE_KEY_PASSWD_FILE: { auto passwd = read_passwd_from_file(optarg); if (passwd.empty()) { LOG(ERROR) << opt << ": Couldn't read key file's passwd from " << optarg; @@ -791,20 +1365,15 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_CERTIFICATE_FILE)) { + case SHRPX_OPTID_CERTIFICATE_FILE: mod_config()->cert_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_DH_PARAM_FILE)) { + case SHRPX_OPTID_DH_PARAM_FILE: mod_config()->dh_param_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_SUBCERT)) { + case SHRPX_OPTID_SUBCERT: { // Private Key file and certificate file separated by ':'. const char *sp = strchr(optarg, ':'); if (sp) { @@ -815,8 +1384,7 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_SYSLOG_FACILITY)) { + case SHRPX_OPTID_SYSLOG_FACILITY: { int facility = int_syslog_facility(optarg); if (facility == -1) { LOG(ERROR) << opt << ": Unknown syslog facility: " << optarg; @@ -826,8 +1394,7 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_BACKLOG)) { + case SHRPX_OPTID_BACKLOG: { int n; if (parse_int(&n, opt, optarg) != 0) { return -1; @@ -843,44 +1410,31 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_CIPHERS)) { + case SHRPX_OPTID_CIPHERS: mod_config()->ciphers = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_CLIENT)) { + case SHRPX_OPTID_CLIENT: mod_config()->client = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_INSECURE)) { + case SHRPX_OPTID_INSECURE: mod_config()->insecure = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_CACERT)) { + case SHRPX_OPTID_CACERT: mod_config()->cacert = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_IPV4)) { + case SHRPX_OPTID_BACKEND_IPV4: mod_config()->backend_ipv4 = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_IPV6)) { + case SHRPX_OPTID_BACKEND_IPV6: mod_config()->backend_ipv6 = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_HTTP_PROXY_URI)) { + case SHRPX_OPTID_BACKEND_HTTP_PROXY_URI: { // parse URI and get hostname, port and optionally userinfo. http_parser_url u; memset(&u, 0, sizeof(u)); @@ -916,112 +1470,71 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_READ_RATE)) { + case SHRPX_OPTID_READ_RATE: return parse_uint_with_unit(&mod_config()->read_rate, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_READ_BURST)) { + case SHRPX_OPTID_READ_BURST: return parse_uint_with_unit(&mod_config()->read_burst, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_WRITE_RATE)) { + case SHRPX_OPTID_WRITE_RATE: return parse_uint_with_unit(&mod_config()->write_rate, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_WRITE_BURST)) { + case SHRPX_OPTID_WRITE_BURST: return parse_uint_with_unit(&mod_config()->write_burst, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_WORKER_READ_RATE)) { + case SHRPX_OPTID_WORKER_READ_RATE: LOG(WARN) << opt << ": not implemented yet"; return parse_uint_with_unit(&mod_config()->worker_read_rate, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_WORKER_READ_BURST)) { + case SHRPX_OPTID_WORKER_READ_BURST: LOG(WARN) << opt << ": not implemented yet"; return parse_uint_with_unit(&mod_config()->worker_read_burst, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_WORKER_WRITE_RATE)) { + case SHRPX_OPTID_WORKER_WRITE_RATE: LOG(WARN) << opt << ": not implemented yet"; return parse_uint_with_unit(&mod_config()->worker_write_rate, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_WORKER_WRITE_BURST)) { + case SHRPX_OPTID_WORKER_WRITE_BURST: LOG(WARN) << opt << ": not implemented yet"; return parse_uint_with_unit(&mod_config()->worker_write_burst, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_NPN_LIST)) { + case SHRPX_OPTID_NPN_LIST: clear_config_str_list(mod_config()->npn_list); - mod_config()->npn_list = parse_config_str_list(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_TLS_PROTO_LIST)) { + case SHRPX_OPTID_TLS_PROTO_LIST: clear_config_str_list(mod_config()->tls_proto_list); - mod_config()->tls_proto_list = parse_config_str_list(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_VERIFY_CLIENT)) { + case SHRPX_OPTID_VERIFY_CLIENT: mod_config()->verify_client = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_VERIFY_CLIENT_CACERT)) { + case SHRPX_OPTID_VERIFY_CLIENT_CACERT: mod_config()->verify_client_cacert = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_CLIENT_PRIVATE_KEY_FILE)) { + case SHRPX_OPTID_CLIENT_PRIVATE_KEY_FILE: mod_config()->client_private_key_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_CLIENT_CERT_FILE)) { + case SHRPX_OPTID_CLIENT_CERT_FILE: mod_config()->client_cert_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND_HTTP2_DUMP_REQUEST_HEADER)) { + case SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER: mod_config()->http2_upstream_dump_request_header_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER)) { + case SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER: mod_config()->http2_upstream_dump_response_header_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_HTTP2_NO_COOKIE_CRUMBLING)) { + case SHRPX_OPTID_HTTP2_NO_COOKIE_CRUMBLING: mod_config()->http2_no_cookie_crumbling = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_FRONTEND_FRAME_DEBUG)) { + case SHRPX_OPTID_FRONTEND_FRAME_DEBUG: mod_config()->upstream_frame_debug = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_PADDING)) { + case SHRPX_OPTID_PADDING: return parse_uint(&mod_config()->padding, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_ALTSVC)) { + case SHRPX_OPTID_ALTSVC: { auto tokens = parse_config_str_list(optarg); if (tokens.size() < 2) { @@ -1069,39 +1582,31 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_ADD_REQUEST_HEADER) || - util::strieq(opt, SHRPX_OPT_ADD_RESPONSE_HEADER)) { + case SHRPX_OPTID_ADD_REQUEST_HEADER: + case SHRPX_OPTID_ADD_RESPONSE_HEADER: { auto p = parse_header(optarg); if (p.first.empty()) { LOG(ERROR) << opt << ": header field name is empty: " << optarg; return -1; } - if (util::strieq(opt, SHRPX_OPT_ADD_REQUEST_HEADER)) { + if (optid == SHRPX_OPTID_ADD_REQUEST_HEADER) { mod_config()->add_request_headers.push_back(std::move(p)); } else { mod_config()->add_response_headers.push_back(std::move(p)); } return 0; } - - if (util::strieq(opt, SHRPX_OPT_WORKER_FRONTEND_CONNECTIONS)) { + case SHRPX_OPTID_WORKER_FRONTEND_CONNECTIONS: return parse_uint(&mod_config()->worker_frontend_connections, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_NO_LOCATION_REWRITE)) { + case SHRPX_OPTID_NO_LOCATION_REWRITE: mod_config()->no_location_rewrite = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_NO_HOST_REWRITE)) { + case SHRPX_OPTID_NO_HOST_REWRITE: mod_config()->no_host_rewrite = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_HTTP1_CONNECTIONS_PER_HOST)) { + case SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST: { int n; if (parse_uint(&n, opt, optarg) != 0) { @@ -1118,22 +1623,15 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND)) { + case SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND: return parse_uint(&mod_config()->downstream_connections_per_frontend, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_LISTENER_DISABLE_TIMEOUT)) { + case SHRPX_OPTID_LISTENER_DISABLE_TIMEOUT: return parse_duration(&mod_config()->listener_disable_timeout, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_TLS_TICKET_KEY_FILE)) { + case SHRPX_OPTID_TLS_TICKET_KEY_FILE: mod_config()->tls_ticket_key_files.push_back(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_RLIMIT_NOFILE)) { + case SHRPX_OPTID_RLIMIT_NOFILE: { int n; if (parse_uint(&n, opt, optarg) != 0) { @@ -1150,9 +1648,8 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_REQUEST_BUFFER) || - util::strieq(opt, SHRPX_OPT_BACKEND_RESPONSE_BUFFER)) { + case SHRPX_OPTID_BACKEND_REQUEST_BUFFER: + case SHRPX_OPTID_BACKEND_RESPONSE_BUFFER: { size_t n; if (parse_uint_with_unit(&n, opt, optarg) != 0) { return -1; @@ -1164,7 +1661,7 @@ int parse_config(const char *opt, const char *optarg, return -1; } - if (util::strieq(opt, SHRPX_OPT_BACKEND_REQUEST_BUFFER)) { + if (optid == SHRPX_OPTID_BACKEND_REQUEST_BUFFER) { mod_config()->downstream_request_buffer_size = n; } else { mod_config()->downstream_response_buffer_size = n; @@ -1173,43 +1670,29 @@ int parse_config(const char *opt, const char *optarg, return 0; } - if (util::strieq(opt, SHRPX_OPT_NO_SERVER_PUSH)) { + case SHRPX_OPTID_NO_SERVER_PUSH: mod_config()->no_server_push = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_BACKEND_HTTP2_CONNECTIONS_PER_WORKER)) { + case SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER: return parse_uint(&mod_config()->http2_downstream_connections_per_worker, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_FETCH_OCSP_RESPONSE_FILE)) { + case SHRPX_OPTID_FETCH_OCSP_RESPONSE_FILE: mod_config()->fetch_ocsp_response_file = strcopy(optarg); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_OCSP_UPDATE_INTERVAL)) { + case SHRPX_OPTID_OCSP_UPDATE_INTERVAL: return parse_duration(&mod_config()->ocsp_update_interval, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_NO_OCSP)) { + case SHRPX_OPTID_NO_OCSP: mod_config()->no_ocsp = util::strieq(optarg, "yes"); return 0; - } - - if (util::strieq(opt, SHRPX_OPT_HEADER_FIELD_BUFFER)) { + case SHRPX_OPTID_HEADER_FIELD_BUFFER: return parse_uint_with_unit(&mod_config()->header_field_buffer, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_MAX_HEADER_FIELDS)) { + case SHRPX_OPTID_MAX_HEADER_FIELDS: return parse_uint(&mod_config()->max_header_fields, opt, optarg); - } - - if (util::strieq(opt, SHRPX_OPT_INCLUDE)) { + case SHRPX_OPTID_INCLUDE: { if (included_set.count(optarg)) { LOG(ERROR) << opt << ": " << optarg << " has already been included"; return -1; @@ -1225,8 +1708,7 @@ int parse_config(const char *opt, const char *optarg, return 0; } - - if (util::strieq(opt, "conf")) { + case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; return 0; From 4fed7a147611980003459330508e1902f645ea61 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Jul 2015 23:36:44 +0900 Subject: [PATCH 22/44] nghttpx: Refactor log format parsing --- gennghttpxfun.py | 2 +- gentokenlookup.py | 16 ++--- src/shrpx_config.cc | 170 ++++++++++++++++++++++++++++++++++---------- 3 files changed, 141 insertions(+), 47 deletions(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 81836fe9..6412ae2b 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -114,4 +114,4 @@ LOGVARS = [ if __name__ == '__main__': gentokenlookup(OPTIONS, 'SHRPX_OPTID', value_type='char', comp_fun='util::strieq_l') - gentokenlookup(LOGVARS, 'SHRPX_LOGF', value_type='char', comp_fun='util::strieq_l') + gentokenlookup(LOGVARS, 'SHRPX_LOGF', value_type='char', comp_fun='util::strieq_l', return_type='LogFragmentType', fail_value='SHRPX_LOGF_NONE') diff --git a/gentokenlookup.py b/gentokenlookup.py index cf96bf68..105eb8d2 100644 --- a/gentokenlookup.py +++ b/gentokenlookup.py @@ -33,10 +33,10 @@ enum {''' {}_MAXIDX, }};'''.format(prefix) -def gen_index_header(tokens, prefix, value_type, comp_fun): +def gen_index_header(tokens, prefix, value_type, comp_fun, return_type, fail_value): print '''\ -int lookup_token(const {} *name, size_t namelen) {{ - switch (namelen) {{'''.format(value_type) +{} lookup_token(const {} *name, size_t namelen) {{ + switch (namelen) {{'''.format(return_type, value_type) b = build_header(tokens) for size in sorted(b.keys()): ents = b[size] @@ -59,11 +59,11 @@ int lookup_token(const {} *name, size_t namelen) {{ } break;''' print '''\ - } - return -1; -}''' + }} + return {}; +}}'''.format(fail_value) -def gentokenlookup(tokens, prefix, value_type='uint8_t', comp_fun='util::streq_l'): +def gentokenlookup(tokens, prefix, value_type='uint8_t', comp_fun='util::streq_l', return_type='int', fail_value='-1'): gen_enum(tokens, prefix) print '' - gen_index_header(tokens, prefix, value_type, comp_fun) + gen_index_header(tokens, prefix, value_type, comp_fun, return_type, fail_value) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 3064b5ea..cc827f66 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -323,6 +323,127 @@ int parse_int(T *dest, const char *opt, const char *optarg) { return 0; } +namespace { +LogFragmentType log_var_lookup_token(const char *name, size_t namelen) { + switch (namelen) { + case 3: + switch (name[2]) { + case 'd': + if (util::strieq_l("pi", name, 2)) { + return SHRPX_LOGF_PID; + } + break; + } + break; + case 4: + switch (name[3]) { + case 'n': + if (util::strieq_l("alp", name, 3)) { + return SHRPX_LOGF_ALPN; + } + break; + } + break; + case 6: + switch (name[5]) { + case 's': + if (util::strieq_l("statu", name, 5)) { + return SHRPX_LOGF_STATUS; + } + break; + } + break; + case 7: + switch (name[6]) { + case 't': + if (util::strieq_l("reques", name, 6)) { + return SHRPX_LOGF_REQUEST; + } + break; + } + break; + case 10: + switch (name[9]) { + case 'l': + if (util::strieq_l("time_loca", name, 9)) { + return SHRPX_LOGF_TIME_LOCAL; + } + break; + case 'r': + if (util::strieq_l("ssl_ciphe", name, 9)) { + return SHRPX_LOGF_SSL_CIPHER; + } + break; + } + break; + case 11: + switch (name[10]) { + case 'r': + if (util::strieq_l("remote_add", name, 10)) { + return SHRPX_LOGF_REMOTE_ADDR; + } + break; + case 't': + if (util::strieq_l("remote_por", name, 10)) { + return SHRPX_LOGF_REMOTE_PORT; + } + if (util::strieq_l("server_por", name, 10)) { + return SHRPX_LOGF_SERVER_PORT; + } + break; + } + break; + case 12: + switch (name[11]) { + case '1': + if (util::strieq_l("time_iso860", name, 11)) { + return SHRPX_LOGF_TIME_ISO8601; + } + break; + case 'e': + if (util::strieq_l("request_tim", name, 11)) { + return SHRPX_LOGF_REQUEST_TIME; + } + break; + case 'l': + if (util::strieq_l("ssl_protoco", name, 11)) { + return SHRPX_LOGF_SSL_PROTOCOL; + } + break; + } + break; + case 14: + switch (name[13]) { + case 'd': + if (util::strieq_l("ssl_session_i", name, 13)) { + return SHRPX_LOGF_SSL_SESSION_ID; + } + break; + } + break; + case 15: + switch (name[14]) { + case 't': + if (util::strieq_l("body_bytes_sen", name, 14)) { + return SHRPX_LOGF_BODY_BYTES_SENT; + } + break; + } + break; + case 18: + switch (name[17]) { + case 'd': + if (util::strieq_l("ssl_session_reuse", name, 17)) { + return SHRPX_LOGF_SSL_SESSION_REUSED; + } + break; + } + break; + } + return SHRPX_LOGF_NONE; +} +} // namespace + namespace { bool var_token(char c) { return util::isAlpha(c) || util::isDigit(c) || c == '_'; @@ -368,46 +489,19 @@ std::vector parse_log_format(const char *optarg) { var_namelen = p - var_name; } - auto type = SHRPX_LOGF_NONE; const char *value = nullptr; - if (util::strieq_l("remote_addr", var_name, var_namelen)) { - type = SHRPX_LOGF_REMOTE_ADDR; - } else if (util::strieq_l("time_local", var_name, var_namelen)) { - type = SHRPX_LOGF_TIME_LOCAL; - } else if (util::strieq_l("time_iso8601", var_name, var_namelen)) { - type = SHRPX_LOGF_TIME_ISO8601; - } else if (util::strieq_l("request", var_name, var_namelen)) { - type = SHRPX_LOGF_REQUEST; - } else if (util::strieq_l("status", var_name, var_namelen)) { - type = SHRPX_LOGF_STATUS; - } else if (util::strieq_l("body_bytes_sent", var_name, var_namelen)) { - type = SHRPX_LOGF_BODY_BYTES_SENT; - } else if (util::istartsWith(var_name, var_namelen, "http_")) { - type = SHRPX_LOGF_HTTP; - value = var_name + sizeof("http_") - 1; - } else if (util::strieq_l("remote_port", var_name, var_namelen)) { - type = SHRPX_LOGF_REMOTE_PORT; - } else if (util::strieq_l("server_port", var_name, var_namelen)) { - type = SHRPX_LOGF_SERVER_PORT; - } else if (util::strieq_l("request_time", var_name, var_namelen)) { - type = SHRPX_LOGF_REQUEST_TIME; - } else if (util::strieq_l("pid", var_name, var_namelen)) { - type = SHRPX_LOGF_PID; - } else if (util::strieq_l("alpn", var_name, var_namelen)) { - type = SHRPX_LOGF_ALPN; - } else if (util::strieq_l("ssl_cipher", var_name, var_namelen)) { - type = SHRPX_LOGF_SSL_CIPHER; - } else if (util::strieq_l("ssl_protocol", var_name, var_namelen)) { - type = SHRPX_LOGF_SSL_PROTOCOL; - } else if (util::strieq_l("ssl_session_id", var_name, var_namelen)) { - type = SHRPX_LOGF_SSL_SESSION_ID; - } else if (util::strieq_l("ssl_session_reused", var_name, var_namelen)) { - type = SHRPX_LOGF_SSL_SESSION_REUSED; - } else { - LOG(WARN) << "Unrecognized log format variable: " - << std::string(var_name, var_namelen); - continue; + auto type = log_var_lookup_token(var_name, var_namelen); + + if (type == SHRPX_LOGF_NONE) { + if (util::istartsWith(var_name, var_namelen, "http_")) { + type = SHRPX_LOGF_HTTP; + value = var_name + str_size("http_"); + } else { + LOG(WARN) << "Unrecognized log format variable: " + << std::string(var_name, var_namelen); + continue; + } } if (literal_start < var_start) { From a2c78cfc69ecc8f6d1ecfa67c81306ea2bacce24 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Jul 2015 23:44:58 +0900 Subject: [PATCH 23/44] nghttpx: Update doc --- src/shrpx.cc | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index d458ed68..a5fa929f 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -998,7 +998,7 @@ Options: The options are categorized into several groups. Connections: - -b, --backend=,[;[:...]] + -b, --backend=(,|unix:)[;[:...]] Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX domain socket can be specified by prefixing path name @@ -1009,27 +1009,27 @@ Connections: -p is used, s are ignored. The pattern matching is closely designed to ServeMux in net/http package of Go programming language. consists - of path, host + path or just host. The path must starts - with "/". If it ends with "/", it matches to the - request path whose prefix is the path. To deal with the - request to the directory without trailing slash, pattern - which ends with "/" also matches the path if pattern == - path + "/" (e.g., pattern "/foo/" matches path "/foo"). - If it does not end with "/", it performs exact match - against the request path. If host is given, it performs - exact match against the request host. If host alone is - given, "/" is appended to it, so that it matches all - paths under the host (e.g., specifying "nghttp2.org" - equals to "nghttp2.org/"). + of path, host + path or just host. The path must start + with "/". If it ends with "/", it matches all request + path in its subtree. To deal with the request to the + directory without trailing slash, the path which ends + with "/" also matches the request path which only lacks + trailing '/' (e.g., path "/foo/" matches request path + "/foo"). If it does not end with "/", it performs exact + match against the request path. If host is given, it + performs exact match against the request host. If host + alone is given, "/" is appended to it, so that it + matches all request paths under the host (e.g., + specifying "nghttp2.org" equals to "nghttp2.org/"). - Patterns with host take precedence over path only - patterns. Then, longer patterns take precedence over + Patterns with host take precedence over patterns with + just path. Then, longer patterns take precedence over shorter ones, breaking a tie by the order of the appearance in the configuration. If is omitted, "/" is used as pattern, which - matches all paths (catch-all pattern). The catch-all - backend must be given. + matches all request paths (catch-all pattern). The + catch-all backend must be given. When doing a match, nghttpx made some normalization to pattern, request host and path. For host part, they are @@ -1047,15 +1047,18 @@ Connections: them by ":". Specifying -b'127.0.0.1,8080;nghttp2.org:www.nghttp2.org' has the same effect to specify -b'127.0.0.1,8080;nghttp2.org' - and -b'127.0.0.1,8080:www.nghttp2.org'. + and -b'127.0.0.1,8080;www.nghttp2.org'. The backend addresses sharing same are grouped - together forming load balancing group. Since ";" and - ":" are used as delimiter, must not contain - these characters. + together forming load balancing group. + + Since ";" and ":" are used as delimiter, must + not contain these characters. Since ";" has special + meaning in shell, the option value must be quoted. + Default: )" << DEFAULT_DOWNSTREAM_HOST << "," << DEFAULT_DOWNSTREAM_PORT << R"( - -f, --frontend=, + -f, --frontend=(,|unix:) Set frontend host and port. If is '*', it assumes all addresses including both IPv4 and IPv6. UNIX domain socket can be specified by prefixing path From 326ac31a23f1ecdf4a377f976eb4c9f0de93b324 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Jul 2015 23:45:50 +0900 Subject: [PATCH 24/44] nghttpx: Update doc --- src/shrpx_config.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index cc827f66..2b20fa93 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -324,6 +324,7 @@ int parse_int(T *dest, const char *opt, const char *optarg) { } namespace { +// generated by gennghttpxfun.py LogFragmentType log_var_lookup_token(const char *name, size_t namelen) { switch (namelen) { case 3: @@ -588,6 +589,7 @@ void parse_mapping(const DownstreamAddr &addr, const char *src) { } } // namespace +// generated by gennghttpxfun.py enum { SHRPX_OPTID_ACCESSLOG_FILE, SHRPX_OPTID_ACCESSLOG_FORMAT, @@ -682,6 +684,7 @@ enum { }; namespace { +// generated by gennghttpxfun.py int option_lookup_token(const char *name, size_t namelen) { switch (namelen) { case 4: From 860da8bf87e5d58379fa3c600c3edfe6f646b174 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 14 Jul 2015 23:54:29 +0900 Subject: [PATCH 25/44] Bump up version number to 1.0.6, LT revision to 14:6:0 --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index d9f5e454..73f7bf67 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.0.6-DEV], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.0.6], [t-tujikawa@users.sourceforge.net]) AC_USE_SYSTEM_EXTENSIONS LT_PREREQ([2.2.6]) @@ -48,7 +48,7 @@ AC_CONFIG_HEADERS([config.h]) dnl See versioning rule: dnl http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html AC_SUBST(LT_CURRENT, 14) -AC_SUBST(LT_REVISION, 5) +AC_SUBST(LT_REVISION, 6) AC_SUBST(LT_AGE, 0) major=`echo $PACKAGE_VERSION |cut -d. -f1 | sed -e "s/[^0-9]//g"` From 9c8bc1218d49c4a161c508c45fd6041daa764289 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 00:01:04 +0900 Subject: [PATCH 26/44] Update man pages --- doc/h2load.1 | 2 +- doc/nghttp.1 | 2 +- doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 51 ++++++++++++++++++++++++++--------------------- doc/nghttpx.1.rst | 50 ++++++++++++++++++++++++++-------------------- 5 files changed, 59 insertions(+), 48 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index b5b60925..dd6776f8 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "July 12, 2015" "1.0.6-DEV" "nghttp2" +.TH "H2LOAD" "1" "July 14, 2015" "1.0.6" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 9695b45a..96d367e3 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "July 12, 2015" "1.0.6-DEV" "nghttp2" +.TH "NGHTTP" "1" "July 14, 2015" "1.0.6" "nghttp2" .SH NAME nghttp \- HTTP/2 experimental client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index 70a816bd..f2eb2f42 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "July 12, 2015" "1.0.6-DEV" "nghttp2" +.TH "NGHTTPD" "1" "July 14, 2015" "1.0.6" "nghttp2" .SH NAME nghttpd \- HTTP/2 experimental server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index fa4e2cde..055c38fc 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "July 12, 2015" "1.0.6-DEV" "nghttp2" +.TH "NGHTTPX" "1" "July 14, 2015" "1.0.6" "nghttp2" .SH NAME nghttpx \- HTTP/2 experimental proxy . @@ -55,7 +55,7 @@ The options are categorized into several groups. .SS Connections .INDENT 0.0 .TP -.B \-b, \-\-backend=,[;[:...]] +.B \-b, \-\-backend=(,|unix:)[;[:...]] Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX domain socket can be specified by prefixing path name @@ -66,27 +66,27 @@ is only used if request matches the pattern. If \fI\%\-s\fP or \fI\%\-p\fP is used, s are ignored. The pattern matching is closely designed to ServeMux in net/http package of Go programming language. consists -of path, host + path or just host. The path must starts -with "\fI/\fP". If it ends with "\fI/\fP", it matches to the -request path whose prefix is the path. To deal with the -request to the directory without trailing slash, pattern -which ends with "\fI/\fP" also matches the path if pattern == -path + "\fI/\fP" (e.g., pattern "\fI/foo/\fP" matches path "\fI/foo\fP"). -If it does not end with "\fI/\fP", it performs exact match -against the request path. If host is given, it performs -exact match against the request host. If host alone is -given, "\fI/\fP" is appended to it, so that it matches all -paths under the host (e.g., specifying "nghttp2.org" -equals to "nghttp2.org/"). +of path, host + path or just host. The path must start +with "\fI/\fP". If it ends with "\fI/\fP", it matches all request +path in its subtree. To deal with the request to the +directory without trailing slash, the path which ends +with "\fI/\fP" also matches the request path which only lacks +trailing \(aq\fI/\fP\(aq (e.g., path "\fI/foo/\fP" matches request path +"\fI/foo\fP"). If it does not end with "\fI/\fP", it performs exact +match against the request path. If host is given, it +performs exact match against the request host. If host +alone is given, "\fI/\fP" is appended to it, so that it +matches all request paths under the host (e.g., +specifying "nghttp2.org" equals to "nghttp2.org/"). .sp -Patterns with host take precedence over path only -patterns. Then, longer patterns take precedence over +Patterns with host take precedence over patterns with +just path. Then, longer patterns take precedence over shorter ones, breaking a tie by the order of the appearance in the configuration. .sp If is omitted, "\fI/\fP" is used as pattern, which -matches all paths (catch\-all pattern). The catch\-all -backend must be given. +matches all request paths (catch\-all pattern). The +catch\-all backend must be given. .sp When doing a match, nghttpx made some normalization to pattern, request host and path. For host part, they are @@ -104,18 +104,20 @@ The multiple s can be specified, delimiting them by ":". Specifying \fI\%\-b\fP\(aq127.0.0.1,8080;nghttp2.org:www.nghttp2.org\(aq has the same effect to specify \fI\%\-b\fP\(aq127.0.0.1,8080;nghttp2.org\(aq -and \fI\%\-b\fP\(aq127.0.0.1,8080:www.nghttp2.org\(aq. +and \fI\%\-b\fP\(aq127.0.0.1,8080;www.nghttp2.org\(aq. .sp The backend addresses sharing same are grouped -together forming load balancing group. Since ";" and -":" are used as delimiter, must not contain -these characters. +together forming load balancing group. +.sp +Since ";" and ":" are used as delimiter, must +not contain these characters. Since ";" has special +meaning in shell, the option value must be quoted. .sp Default: \fB127.0.0.1,80\fP .UNINDENT .INDENT 0.0 .TP -.B \-f, \-\-frontend=, +.B \-f, \-\-frontend=(,|unix:) Set frontend host and port. If is \(aq*\(aq, it assumes all addresses including both IPv4 and IPv6. UNIX domain socket can be specified by prefixing path @@ -692,6 +694,9 @@ $ssl_session_reused: "r" if SSL/TLS session was reused. Otherwise, "." .UNINDENT .sp +The variable can be enclosed by "{" and "}" for +disambiguation (e.g., ${remote_addr}). +.sp Default: \fB$remote_addr \- \- [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"\fP .UNINDENT .INDENT 0.0 diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index 7ff041c6..f88cdf8d 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -37,7 +37,7 @@ The options are categorized into several groups. Connections ~~~~~~~~~~~ -.. option:: -b, --backend=,[;[:...]] +.. option:: -b, --backend=(,|unix:)[;[:...]] Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX @@ -49,27 +49,27 @@ Connections :option:`-p` is used, s are ignored. The pattern matching is closely designed to ServeMux in net/http package of Go programming language. consists - of path, host + path or just host. The path must starts - with "*/*". If it ends with "*/*", it matches to the - request path whose prefix is the path. To deal with the - request to the directory without trailing slash, pattern - which ends with "*/*" also matches the path if pattern == - path + "*/*" (e.g., pattern "*/foo/*" matches path "*/foo*"). - If it does not end with "*/*", it performs exact match - against the request path. If host is given, it performs - exact match against the request host. If host alone is - given, "*/*" is appended to it, so that it matches all - paths under the host (e.g., specifying "nghttp2.org" - equals to "nghttp2.org/"). + of path, host + path or just host. The path must start + with "*/*". If it ends with "*/*", it matches all request + path in its subtree. To deal with the request to the + directory without trailing slash, the path which ends + with "*/*" also matches the request path which only lacks + trailing '*/*' (e.g., path "*/foo/*" matches request path + "*/foo*"). If it does not end with "*/*", it performs exact + match against the request path. If host is given, it + performs exact match against the request host. If host + alone is given, "*/*" is appended to it, so that it + matches all request paths under the host (e.g., + specifying "nghttp2.org" equals to "nghttp2.org/"). - Patterns with host take precedence over path only - patterns. Then, longer patterns take precedence over + Patterns with host take precedence over patterns with + just path. Then, longer patterns take precedence over shorter ones, breaking a tie by the order of the appearance in the configuration. If is omitted, "*/*" is used as pattern, which - matches all paths (catch-all pattern). The catch-all - backend must be given. + matches all request paths (catch-all pattern). The + catch-all backend must be given. When doing a match, nghttpx made some normalization to pattern, request host and path. For host part, they are @@ -87,16 +87,19 @@ Connections them by ":". Specifying :option:`-b`\'127.0.0.1,8080;nghttp2.org:www.nghttp2.org' has the same effect to specify :option:`-b`\'127.0.0.1,8080;nghttp2.org' - and :option:`-b`\'127.0.0.1,8080:www.nghttp2.org'. + and :option:`-b`\'127.0.0.1,8080;www.nghttp2.org'. The backend addresses sharing same are grouped - together forming load balancing group. Since ";" and - ":" are used as delimiter, must not contain - these characters. + together forming load balancing group. + + Since ";" and ":" are used as delimiter, must + not contain these characters. Since ";" has special + meaning in shell, the option value must be quoted. + Default: ``127.0.0.1,80`` -.. option:: -f, --frontend=, +.. option:: -f, --frontend=(,|unix:) Set frontend host and port. If is '\*', it assumes all addresses including both IPv4 and IPv6. @@ -611,6 +614,9 @@ Logging * $ssl_session_reused: "r" if SSL/TLS session was reused. Otherwise, "." + The variable can be enclosed by "{" and "}" for + disambiguation (e.g., ${remote_addr}). + Default: ``$remote_addr - - [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"`` From 1c7431293d29cb4b348f907a6a9bf16426389960 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 00:01:22 +0900 Subject: [PATCH 27/44] Update bash_completion --- doc/bash_completion/h2load | 2 +- doc/bash_completion/nghttp | 2 +- doc/bash_completion/nghttpx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/bash_completion/h2load b/doc/bash_completion/h2load index 2a4bf98d..c8438893 100644 --- a/doc/bash_completion/h2load +++ b/doc/bash_completion/h2load @@ -8,7 +8,7 @@ _h2load() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--threads --connection-window-bits --input-file --help --requests --data --verbose --version --window-bits --clients --no-tls-proto --header --max-concurrent-streams ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--threads --connection-window-bits --input-file --help --requests --data --verbose --ciphers --window-bits --clients --no-tls-proto --version --header --max-concurrent-streams ' -- "$cur" ) ) ;; *) _filedir diff --git a/doc/bash_completion/nghttp b/doc/bash_completion/nghttp index 6bb7a3e4..7d7da704 100644 --- a/doc/bash_completion/nghttp +++ b/doc/bash_completion/nghttp @@ -8,7 +8,7 @@ _nghttp() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--no-push --verbose --no-dep --get-assets --har --header-table-size --multiply --padding --hexdump --continuation --connection-window-bits --peer-max-concurrent-streams --timeout --data --no-content-length --version --color --cert --upgrade --remote-name --trailer --weight --help --key --null-out --window-bits --stat --header ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--no-push --verbose --no-dep --get-assets --har --header-table-size --multiply --padding --hexdump --max-concurrent-streams --continuation --connection-window-bits --peer-max-concurrent-streams --timeout --data --no-content-length --version --color --cert --upgrade --remote-name --trailer --weight --help --key --null-out --window-bits --stat --header ' -- "$cur" ) ) ;; *) _filedir diff --git a/doc/bash_completion/nghttpx b/doc/bash_completion/nghttpx index 8855b81a..64b6645e 100644 --- a/doc/bash_completion/nghttpx +++ b/doc/bash_completion/nghttpx @@ -8,7 +8,7 @@ _nghttpx() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--worker-read-rate --frontend-no-tls --frontend-http2-dump-response-header --backend-http1-connections-per-frontend --tls-ticket-key-file --verify-client-cacert --backend-request-buffer --backend-http2-connection-window-bits --conf --worker-write-burst --npn-list --fetch-ocsp-response-file --stream-read-timeout --accesslog-syslog --frontend-http2-read-timeout --listener-disable-timeout --frontend-http2-connection-window-bits --ciphers --strip-incoming-x-forwarded-for --daemon --backend-keep-alive-timeout --backend-http-proxy-uri --backend-http1-connections-per-host --rlimit-nofile --no-via --ocsp-update-interval --backend-write-timeout --client --http2-no-cookie-crumbling --worker-read-burst --client-proxy --http2-bridge --accesslog-format --errorlog-syslog --errorlog-file --http2-max-concurrent-streams --frontend-write-timeout --read-burst --backend-ipv4 --backend-ipv6 --backend --insecure --log-level --tls-proto-list --backend-http2-connections-per-worker --dh-param-file --worker-frontend-connections --header-field-buffer --no-server-push --no-location-rewrite --no-ocsp --backend-response-buffer --workers --frontend-http2-window-bits --no-host-rewrite --worker-write-rate --add-request-header --backend-tls-sni-field --subcert --help --frontend-frame-debug --pid-file --frontend-http2-dump-request-header --private-key-passwd-file --write-rate --altsvc --user --add-x-forwarded-for --syslog-facility --frontend-read-timeout --backlog --write-burst --backend-http2-window-bits --padding --stream-write-timeout --cacert --version --verify-client --backend-read-timeout --frontend --accesslog-file --http2-proxy --max-header-fields --backend-no-tls --client-private-key-file --client-cert-file --add-response-header --read-rate ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--worker-read-rate --frontend-no-tls --frontend-http2-dump-response-header --backend-http1-connections-per-frontend --tls-ticket-key-file --verify-client-cacert --include --backend-request-buffer --backend-http2-connection-window-bits --conf --worker-write-burst --npn-list --fetch-ocsp-response-file --stream-read-timeout --accesslog-syslog --frontend-http2-read-timeout --listener-disable-timeout --frontend-http2-connection-window-bits --ciphers --strip-incoming-x-forwarded-for --daemon --backend-keep-alive-timeout --backend-http-proxy-uri --backend-http1-connections-per-host --rlimit-nofile --no-via --ocsp-update-interval --backend-write-timeout --client --http2-no-cookie-crumbling --worker-read-burst --client-proxy --http2-bridge --accesslog-format --errorlog-syslog --errorlog-file --http2-max-concurrent-streams --frontend-write-timeout --read-burst --backend-ipv4 --backend-ipv6 --backend --insecure --log-level --tls-proto-list --backend-http2-connections-per-worker --dh-param-file --worker-frontend-connections --header-field-buffer --no-server-push --no-location-rewrite --no-ocsp --backend-response-buffer --workers --frontend-http2-window-bits --no-host-rewrite --worker-write-rate --add-request-header --backend-tls-sni-field --subcert --help --frontend-frame-debug --pid-file --frontend-http2-dump-request-header --private-key-passwd-file --write-rate --altsvc --user --add-x-forwarded-for --syslog-facility --frontend-read-timeout --backlog --write-burst --backend-http2-window-bits --padding --stream-write-timeout --cacert --version --verify-client --backend-read-timeout --frontend --accesslog-file --http2-proxy --max-header-fields --backend-no-tls --client-private-key-file --client-cert-file --add-response-header --read-rate ' -- "$cur" ) ) ;; *) _filedir From 5e966d000a76f7f0f89f0c97972c8b3397fc51b9 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 00:11:34 +0900 Subject: [PATCH 28/44] python: Fix out-of-tree builds failure --- python/setup.py.in | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/python/setup.py.in b/python/setup.py.in index 3abaddea..a2b4851d 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -22,8 +22,7 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. from setuptools import setup, Extension -from Cython.Build import cythonize - + LIBS = ['nghttp2'] setup( @@ -34,13 +33,13 @@ setup( author_email = 'tatsuhiro.t@gmail.com', url = 'https://nghttp2.org/', keywords = [], - ext_modules = cythonize([Extension("nghttp2", - ["nghttp2.pyx"], + ext_modules = [Extension("nghttp2", + ["nghttp2.c"], include_dirs=['@top_srcdir@/lib', '@top_srcdir@/lib/includes', '@top_builddir@/lib/includes'], library_dirs=['@top_builddir@/lib/.libs', '@top_builddir@'], - libraries=LIBS)]), + libraries=LIBS)], long_description='TBD' ) From 3db03a3f517747256e4d33df369a99c19d3a072a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 01:16:56 +0900 Subject: [PATCH 29/44] Exclude python bindings for `make distcheck` setuptools does now allow us to install custom location without hack. Also it does not provide uninstall feature, and `make uninstall` leaves several files we cannot easily remove (e.g., easy-install.pth). Therefore, it is better just exclude python bindings from `make distcheck`. --- Makefile.am | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile.am b/Makefile.am index 831b317d..d1a7dd16 100644 --- a/Makefile.am +++ b/Makefile.am @@ -23,6 +23,11 @@ SUBDIRS = lib third-party src examples python tests integration-tests \ doc contrib script +# Now with python setuptools, make uninstall will leave many files we +# cannot easily remove (e.g., easy-install.pth). Disable it for +# distcheck rule. +AM_DISTCHECK_CONFIGURE_FLAGS = --disable-python-bindings + ACLOCAL_AMFLAGS = -I m4 dist_doc_DATA = README.rst From a06c16c99a722c3727893243178710eb9e4a0e6b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 01:23:18 +0900 Subject: [PATCH 30/44] Bump up version number to 1.1.0 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 73f7bf67..e893f19e 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.0.6], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.1.0], [t-tujikawa@users.sourceforge.net]) AC_USE_SYSTEM_EXTENSIONS LT_PREREQ([2.2.6]) From da5138992729e4bf95e04141403541c8b352a852 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 01:25:27 +0900 Subject: [PATCH 31/44] Update man pages --- doc/h2load.1 | 2 +- doc/nghttp.1 | 2 +- doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index dd6776f8..95369ed9 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "July 14, 2015" "1.0.6" "nghttp2" +.TH "H2LOAD" "1" "July 15, 2015" "1.1.0" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 96d367e3..42e22100 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "July 14, 2015" "1.0.6" "nghttp2" +.TH "NGHTTP" "1" "July 15, 2015" "1.1.0" "nghttp2" .SH NAME nghttp \- HTTP/2 experimental client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index f2eb2f42..096b5e68 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "July 14, 2015" "1.0.6" "nghttp2" +.TH "NGHTTPD" "1" "July 15, 2015" "1.1.0" "nghttp2" .SH NAME nghttpd \- HTTP/2 experimental server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index 055c38fc..eb747b30 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "July 14, 2015" "1.0.6" "nghttp2" +.TH "NGHTTPX" "1" "July 15, 2015" "1.1.0" "nghttp2" .SH NAME nghttpx \- HTTP/2 experimental proxy . From f54745fe4c9a0628a28e15fdd01ea71d0fd8fb3a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 01:29:57 +0900 Subject: [PATCH 32/44] Bump up version number to 1.1.1-DEV --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index e893f19e..0e4f9788 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.1.0], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.1.1-DEV], [t-tujikawa@users.sourceforge.net]) AC_USE_SYSTEM_EXTENSIONS LT_PREREQ([2.2.6]) From 6d1079930191ea53cf1637ec624fecdc5fecddba Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 19:46:48 +0900 Subject: [PATCH 33/44] nghttpx: Don't pool failed HTTP/1 backend connection --- src/shrpx_http_downstream_connection.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index a69357a5..af16ba24 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -864,6 +864,8 @@ int HttpDownstreamConnection::on_connect() { DLOG(INFO, this) << "downstream connect failed"; } + downstream_->set_request_state(Downstream::CONNECT_FAIL); + return -1; } From 031fb31248edcfef4a45d10c2a015589ac2059a5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 19:47:15 +0900 Subject: [PATCH 34/44] nghttpx: Fix memory leak, and blocked stream dispatch --- src/shrpx_downstream_queue.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/shrpx_downstream_queue.cc b/src/shrpx_downstream_queue.cc index c70eb064..fa2ba517 100644 --- a/src/shrpx_downstream_queue.cc +++ b/src/shrpx_downstream_queue.cc @@ -124,17 +124,14 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream) { // Delete downstream when this function returns. auto delptr = std::unique_ptr(downstream); - if (downstream->get_dispatch_state() != Downstream::DISPATCH_ACTIVE) { - assert(downstream->get_dispatch_state() != Downstream::DISPATCH_NONE); - downstreams_.remove(downstream); - return nullptr; - } - downstreams_.remove(downstream); auto &host = make_host_key(downstream); auto &ent = find_host_entry(host); - --ent.num_active; + + if (downstream->get_dispatch_state() == Downstream::DISPATCH_ACTIVE) { + --ent.num_active; + } if (remove_host_entry_if_empty(ent, host_entries_, host)) { return nullptr; @@ -147,7 +144,10 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream) { for (auto link = ent.blocked.head; link;) { auto next = link->dlnext; if (!link->downstream) { + // If non-active (e.g., pending) Downstream got deleted, + // link->downstream is nullptr. ent.blocked.remove(link); + delete link; link = next; continue; } From e63e775fea41264f6970271d6c11b08fa25ca0bd Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 20:44:44 +0900 Subject: [PATCH 35/44] nghttpx: Simplify BlockedLink management --- src/shrpx_downstream.cc | 12 +++--------- src/shrpx_downstream.h | 2 +- src/shrpx_downstream_queue.cc | 35 ++++++++++++++++++----------------- 3 files changed, 22 insertions(+), 27 deletions(-) diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index e97e2913..54c29fcc 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -152,10 +152,6 @@ Downstream::~Downstream() { DLOG(INFO, this) << "Deleting"; } - if (blocked_link_) { - detach_blocked_link(blocked_link_); - } - // check nullptr for unittest if (upstream_) { auto loop = upstream_->get_client_handler()->get_loop(); @@ -1200,12 +1196,10 @@ void Downstream::attach_blocked_link(BlockedLink *l) { blocked_link_ = l; } -void Downstream::detach_blocked_link(BlockedLink *l) { - assert(blocked_link_); - assert(l->downstream == this); - - l->downstream = nullptr; +BlockedLink *Downstream::detach_blocked_link() { + auto link = blocked_link_; blocked_link_ = nullptr; + return link; } void Downstream::add_request_headers_sum(size_t amount) { diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index f0edeb2c..1294c0bd 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -332,7 +332,7 @@ public: void set_dispatch_state(int s); void attach_blocked_link(BlockedLink *l); - void detach_blocked_link(BlockedLink *l); + BlockedLink *detach_blocked_link(); enum { EVENT_ERROR = 0x1, diff --git a/src/shrpx_downstream_queue.cc b/src/shrpx_downstream_queue.cc index fa2ba517..0462d2a7 100644 --- a/src/shrpx_downstream_queue.cc +++ b/src/shrpx_downstream_queue.cc @@ -131,6 +131,12 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream) { if (downstream->get_dispatch_state() == Downstream::DISPATCH_ACTIVE) { --ent.num_active; + } else { + auto link = downstream->detach_blocked_link(); + if (link) { + ent.blocked.remove(link); + delete link; + } } if (remove_host_entry_if_empty(ent, host_entries_, host)) { @@ -141,24 +147,19 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream) { return nullptr; } - for (auto link = ent.blocked.head; link;) { - auto next = link->dlnext; - if (!link->downstream) { - // If non-active (e.g., pending) Downstream got deleted, - // link->downstream is nullptr. - ent.blocked.remove(link); - delete link; - link = next; - continue; - } - auto next_downstream = link->downstream; - next_downstream->detach_blocked_link(link); - ent.blocked.remove(link); - delete link; - remove_host_entry_if_empty(ent, host_entries_, host); - return next_downstream; + auto link = ent.blocked.head; + + if (!link) { + return nullptr; } - return nullptr; + + auto next_downstream = link->downstream; + next_downstream->detach_blocked_link(); + ent.blocked.remove(link); + delete link; + remove_host_entry_if_empty(ent, host_entries_, host); + + return next_downstream; } Downstream *DownstreamQueue::get_downstreams() const { From 20e63151a5ebecfc264199252aef3b2285ed1b91 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 23:18:32 +0900 Subject: [PATCH 36/44] nghttpx: Fix bug that idle timer is used after reuse --- src/shrpx_http_downstream_connection.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index af16ba24..eeff9312 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -205,6 +205,8 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { ev_set_cb(&conn_.rev, readcb); conn_.rt.repeat = get_config()->downstream_read_timeout; + // we may set read timer cb to idle_timeoutcb. Reset again. + ev_set_cb(&conn_.rt, timeoutcb); ev_timer_again(conn_.loop, &conn_.rt); // TODO we should have timeout for connection establishment ev_timer_again(conn_.loop, &conn_.wt); From fc062976a1ec700abe0a50fd66779d65ec45e3d6 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 23:31:32 +0900 Subject: [PATCH 37/44] nghttpx: Delete DownstreamConnection from Downstream explicitly --- src/shrpx_downstream.cc | 6 ++++-- src/shrpx_downstream.h | 2 -- src/shrpx_http2_downstream_connection.cc | 6 +----- src/shrpx_http_downstream_connection.cc | 8 +------- 4 files changed, 6 insertions(+), 16 deletions(-) diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 54c29fcc..87a2d082 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -162,6 +162,10 @@ Downstream::~Downstream() { ev_timer_stop(loop, &downstream_wtimer_); } + // DownstreamConnection may refer to this object. Delete it now + // explicitly. + dconn_.reset(); + if (LOG_ENABLED(INFO)) { DLOG(INFO, this) << "Deleted"; } @@ -191,8 +195,6 @@ void Downstream::detach_downstream_connection() { std::unique_ptr(dconn_.release())); } -void Downstream::release_downstream_connection() { dconn_.release(); } - DownstreamConnection *Downstream::get_downstream_connection() { return dconn_.get(); } diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 1294c0bd..3486621a 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -69,8 +69,6 @@ public: int attach_downstream_connection(std::unique_ptr dconn); void detach_downstream_connection(); - // Releases dconn_, without freeing it. - void release_downstream_connection(); DownstreamConnection *get_downstream_connection(); // Returns dconn_ and nullifies dconn_. std::unique_ptr pop_downstream_connection(); diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index d6bc433f..f4377b44 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -86,11 +86,7 @@ Http2DownstreamConnection::~Http2DownstreamConnection() { } } http2session_->remove_downstream_connection(this); - // Downstream and DownstreamConnection may be deleted - // asynchronously. - if (downstream_) { - downstream_->release_downstream_connection(); - } + if (LOG_ENABLED(INFO)) { DCLOG(INFO, this) << "Deleted"; } diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index eeff9312..c3f4b89e 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -118,13 +118,7 @@ HttpDownstreamConnection::HttpDownstreamConnection( ioctrl_(&conn_.rlimit), response_htp_{0}, group_(group), addr_idx_(0), connected_(false) {} -HttpDownstreamConnection::~HttpDownstreamConnection() { - // Downstream and DownstreamConnection may be deleted - // asynchronously. - if (downstream_) { - downstream_->release_downstream_connection(); - } -} +HttpDownstreamConnection::~HttpDownstreamConnection() {} int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { if (LOG_ENABLED(INFO)) { From ba31b990a25d38e09321a7c847ecbcede2903a46 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 23:33:44 +0900 Subject: [PATCH 38/44] nghttpx: Remove dead handling of Downstream::STREAM_CLOSED --- src/shrpx_http2_upstream.cc | 25 ------------------------- src/shrpx_spdy_upstream.cc | 25 ------------------------- 2 files changed, 50 deletions(-) diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 97ff206c..87c77d14 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -884,16 +884,6 @@ ClientHandler *Http2Upstream::get_client_handler() const { return handler_; } int Http2Upstream::downstream_read(DownstreamConnection *dconn) { auto downstream = dconn->get_downstream(); - if (downstream->get_request_state() == Downstream::STREAM_CLOSED) { - // If upstream HTTP2 stream was closed, we just close downstream, - // because there is no consumer now. Downstream connection is also - // closed in this case. - remove_downstream(downstream); - // downstream was deleted - - return 0; - } - if (downstream->get_response_state() == Downstream::MSG_RESET) { // The downstream stream was reset (canceled). In this case, // RST_STREAM to the upstream and delete downstream connection @@ -959,14 +949,6 @@ int Http2Upstream::downstream_eof(DownstreamConnection *dconn) { if (LOG_ENABLED(INFO)) { DCLOG(INFO, dconn) << "EOF. stream_id=" << downstream->get_stream_id(); } - if (downstream->get_request_state() == Downstream::STREAM_CLOSED) { - // If stream was closed already, we don't need to send reply at - // the first place. We can delete downstream. - remove_downstream(downstream); - // downstream was deleted - - return 0; - } // Delete downstream connection. If we don't delete it here, it will // be pooled in on_stream_close_callback. @@ -1012,13 +994,6 @@ int Http2Upstream::downstream_error(DownstreamConnection *dconn, int events) { } } - if (downstream->get_request_state() == Downstream::STREAM_CLOSED) { - remove_downstream(downstream); - // downstream was deleted - - return 0; - } - // Delete downstream connection. If we don't delete it here, it will // be pooled in on_stream_close_callback. downstream->pop_downstream_connection(); diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 52f267da..5f124296 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -560,16 +560,6 @@ ClientHandler *SpdyUpstream::get_client_handler() const { return handler_; } int SpdyUpstream::downstream_read(DownstreamConnection *dconn) { auto downstream = dconn->get_downstream(); - if (downstream->get_request_state() == Downstream::STREAM_CLOSED) { - // If upstream SPDY stream was closed, we just close downstream, - // because there is no consumer now. Downstream connection is also - // closed in this case. - remove_downstream(downstream); - // downstrea was deleted - - return 0; - } - if (downstream->get_response_state() == Downstream::MSG_RESET) { // The downstream stream was reset (canceled). In this case, // RST_STREAM to the upstream and delete downstream connection @@ -633,14 +623,6 @@ int SpdyUpstream::downstream_eof(DownstreamConnection *dconn) { if (LOG_ENABLED(INFO)) { DCLOG(INFO, dconn) << "EOF. stream_id=" << downstream->get_stream_id(); } - if (downstream->get_request_state() == Downstream::STREAM_CLOSED) { - // If stream was closed already, we don't need to send reply at - // the first place. We can delete downstream. - remove_downstream(downstream); - // downstream was deleted - - return 0; - } // Delete downstream connection. If we don't delete it here, it will // be pooled in on_stream_close_callback. @@ -686,13 +668,6 @@ int SpdyUpstream::downstream_error(DownstreamConnection *dconn, int events) { } } - if (downstream->get_request_state() == Downstream::STREAM_CLOSED) { - remove_downstream(downstream); - // downstream was deleted - - return 0; - } - // Delete downstream connection. If we don't delete it here, it will // be pooled in on_stream_close_callback. downstream->pop_downstream_connection(); From 7a3012cc1b09b0bfed7e92774d017648881457f0 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 23:44:24 +0900 Subject: [PATCH 39/44] src: Rename timegm as nghttp2_timegm --- src/timegm.c | 2 +- src/timegm.h | 2 +- src/util.cc | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/timegm.c b/src/timegm.c index b80e0789..62b14430 100644 --- a/src/timegm.c +++ b/src/timegm.c @@ -34,7 +34,7 @@ static int count_leap_year(int y) { } /* Based on the algorithm of Python 2.7 calendar.timegm. */ -time_t timegm(struct tm *tm) { +time_t nghttp2_timegm(struct tm *tm) { int days; int num_leap_year; int64_t t; diff --git a/src/timegm.h b/src/timegm.h index 2624657a..4168fdd1 100644 --- a/src/timegm.h +++ b/src/timegm.h @@ -37,7 +37,7 @@ extern "C" { #include #endif // HAVE_TIME_H -time_t timegm(struct tm *tm); +time_t nghttp2_timegm(struct tm *tm); #ifdef __cplusplus } diff --git a/src/util.cc b/src/util.cc index 2d7c7734..191bd13f 100644 --- a/src/util.cc +++ b/src/util.cc @@ -278,7 +278,7 @@ std::string common_log_date(time_t t) { #ifdef HAVE_STRUCT_TM_TM_GMTOFF auto gmtoff = tms.tm_gmtoff; #else // !HAVE_STRUCT_TM_TM_GMTOFF - auto gmtoff = timegm(&tms) - t; + auto gmtoff = nghttp2_timegm(&tms) - t; #endif // !HAVE_STRUCT_TM_TM_GMTOFF if (gmtoff >= 0) { *p++ = '+'; @@ -326,7 +326,7 @@ std::string iso8601_date(int64_t ms) { #ifdef HAVE_STRUCT_TM_TM_GMTOFF auto gmtoff = tms.tm_gmtoff; #else // !HAVE_STRUCT_TM_TM_GMTOFF - auto gmtoff = timegm(&tms) - sec; + auto gmtoff = nghttp2_timegm(&tms) - sec; #endif // !HAVE_STRUCT_TM_TM_GMTOFF if (gmtoff == 0) { *p++ = 'Z'; @@ -354,7 +354,7 @@ time_t parse_http_date(const std::string &s) { if (r == 0) { return 0; } - return timegm(&tm); + return nghttp2_timegm(&tm); } namespace { From f821f6bd80437d0a79f9464e99e25e87fa84518b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 23:46:22 +0900 Subject: [PATCH 40/44] Bump up version number to 1.1.1 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 0e4f9788..458438fe 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.1.1-DEV], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.1.1], [t-tujikawa@users.sourceforge.net]) AC_USE_SYSTEM_EXTENSIONS LT_PREREQ([2.2.6]) From 3396c71c3c0c7da46f61a1f1fa38c77a6e08ac37 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 15 Jul 2015 23:48:41 +0900 Subject: [PATCH 41/44] Update man pages --- doc/h2load.1 | 2 +- doc/nghttp.1 | 2 +- doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index 95369ed9..d3f6b7a1 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "July 15, 2015" "1.1.0" "nghttp2" +.TH "H2LOAD" "1" "July 15, 2015" "1.1.1" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 42e22100..b1ee0180 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "July 15, 2015" "1.1.0" "nghttp2" +.TH "NGHTTP" "1" "July 15, 2015" "1.1.1" "nghttp2" .SH NAME nghttp \- HTTP/2 experimental client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index 096b5e68..ac31ae6b 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "July 15, 2015" "1.1.0" "nghttp2" +.TH "NGHTTPD" "1" "July 15, 2015" "1.1.1" "nghttp2" .SH NAME nghttpd \- HTTP/2 experimental server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index eb747b30..b7e2041d 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "July 15, 2015" "1.1.0" "nghttp2" +.TH "NGHTTPX" "1" "July 15, 2015" "1.1.1" "nghttp2" .SH NAME nghttpx \- HTTP/2 experimental proxy . From 2bac00ea5fe5cec6d1188e99346312b7866d4254 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 16 Jul 2015 00:01:26 +0900 Subject: [PATCH 42/44] nghttpx: Add additional assert just in case --- src/shrpx_downstream_queue.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/shrpx_downstream_queue.cc b/src/shrpx_downstream_queue.cc index 0462d2a7..28815fb4 100644 --- a/src/shrpx_downstream_queue.cc +++ b/src/shrpx_downstream_queue.cc @@ -154,7 +154,8 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream) { } auto next_downstream = link->downstream; - next_downstream->detach_blocked_link(); + auto link2 = next_downstream->detach_blocked_link(); + assert(link2 == link); ent.blocked.remove(link); delete link; remove_host_entry_if_empty(ent, host_entries_, host); From 8585599b4b252135304c9409892211cbe63cb7d1 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 16 Jul 2015 00:03:34 +0900 Subject: [PATCH 43/44] nghttpx: Add doc --- src/shrpx_downstream_queue.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/shrpx_downstream_queue.cc b/src/shrpx_downstream_queue.cc index 28815fb4..d1430e61 100644 --- a/src/shrpx_downstream_queue.cc +++ b/src/shrpx_downstream_queue.cc @@ -132,6 +132,7 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream) { if (downstream->get_dispatch_state() == Downstream::DISPATCH_ACTIVE) { --ent.num_active; } else { + // For those downstreams deleted while in blocked state auto link = downstream->detach_blocked_link(); if (link) { ent.blocked.remove(link); From 1ce3a000a1843a5266851b7b58a1856fd10be37a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 16 Jul 2015 00:36:10 +0900 Subject: [PATCH 44/44] Bump up version number to 1.1.2-DEV --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 458438fe..930c0f7c 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.1.1], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.1.2-DEV], [t-tujikawa@users.sourceforge.net]) AC_USE_SYSTEM_EXTENSIONS LT_PREREQ([2.2.6])