From 1ebb6810a1aca7ec641e602924fb2ecb90f59f51 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 1 Dec 2017 00:28:36 +0900 Subject: [PATCH] nghttpx: Faster configuration loading with lots of backends --- src/shrpx.cc | 7 +- src/shrpx_api_downstream_connection.cc | 5 +- src/shrpx_config.cc | 85 +++++++++++------------ src/shrpx_config.h | 14 ++-- src/shrpx_worker.cc | 93 ++++++++++++-------------- 5 files changed, 106 insertions(+), 98 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index c660b1f2..6ad1a434 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2768,10 +2768,12 @@ namespace { int process_options(Config *config, std::vector> &cmdcfgs) { std::array errbuf; + std::map pattern_addr_indexer; if (conf_exists(config->conf_path.c_str())) { LOG(NOTICE) << "Loading configuration from " << config->conf_path; std::set include_set; - if (load_config(config, config->conf_path.c_str(), include_set) == -1) { + if (load_config(config, config->conf_path.c_str(), include_set, + pattern_addr_indexer) == -1) { LOG(FATAL) << "Failed to load configuration from " << config->conf_path; return -1; } @@ -2785,7 +2787,8 @@ int process_options(Config *config, std::set include_set; for (auto &p : cmdcfgs) { - if (parse_config(config, p.first, p.second, include_set) == -1) { + if (parse_config(config, p.first, p.second, include_set, + pattern_addr_indexer) == -1) { LOG(FATAL) << "Failed to parse command-line argument."; return -1; } diff --git a/src/shrpx_api_downstream_connection.cc b/src/shrpx_api_downstream_connection.cc index 39db4a9a..102a6b88 100644 --- a/src/shrpx_api_downstream_connection.cc +++ b/src/shrpx_api_downstream_connection.cc @@ -23,7 +23,6 @@ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "shrpx_api_downstream_connection.h" - #include "shrpx_client_handler.h" #include "shrpx_upstream.h" #include "shrpx_downstream.h" @@ -343,6 +342,7 @@ int APIDownstreamConnection::handle_backendconfig() { downstreamconf->family = src->family; std::set include_set; + std::map pattern_addr_indexer; for (auto first = reinterpret_cast(iov[0].iov_base), last = first + iov[0].iov_len; @@ -376,7 +376,8 @@ int APIDownstreamConnection::handle_backendconfig() { continue; } - if (parse_config(&new_config, optid, opt, optval, include_set) != 0) { + if (parse_config(&new_config, optid, opt, optval, include_set, + pattern_addr_indexer) != 0) { send_reply(400, API_FAILURE); return 0; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 04d89c34..64773260 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -943,6 +943,7 @@ namespace { // // This function returns 0 if it succeeds, or -1. int parse_mapping(Config *config, DownstreamAddrConfig &addr, + std::map &pattern_addr_indexer, const StringRef &src_pattern, const StringRef &src_params) { // This returns at least 1 element (it could be empty string). We // will append '/' to all patterns, so it becomes catch-all pattern. @@ -983,7 +984,6 @@ int parse_mapping(Config *config, DownstreamAddrConfig &addr, auto &wildcard_patterns = routerconf.wildcard_patterns; for (const auto &raw_pattern : mapping) { - auto done = false; StringRef pattern; auto slash = std::find(std::begin(raw_pattern), std::end(raw_pattern), '/'); if (slash == std::end(raw_pattern)) { @@ -1010,47 +1010,43 @@ int parse_mapping(Config *config, DownstreamAddrConfig &addr, *p = '\0'; pattern = StringRef{iov.base, p}; } - for (auto &g : addr_groups) { - if (g.pattern == pattern) { - // Last value wins if we have multiple different affinity - // value under one group. - if (params.affinity.type != AFFINITY_NONE) { - if (g.affinity.type == AFFINITY_NONE) { - g.affinity.type = params.affinity.type; - if (params.affinity.type == AFFINITY_COOKIE) { - g.affinity.cookie.name = make_string_ref( - downstreamconf.balloc, params.affinity.cookie.name); - if (!params.affinity.cookie.path.empty()) { - g.affinity.cookie.path = make_string_ref( - downstreamconf.balloc, params.affinity.cookie.path); - } - g.affinity.cookie.secure = params.affinity.cookie.secure; + auto it = pattern_addr_indexer.find(pattern); + if (it != std::end(pattern_addr_indexer)) { + auto &g = addr_groups[(*it).second]; + // Last value wins if we have multiple different affinity + // value under one group. + if (params.affinity.type != AFFINITY_NONE) { + if (g.affinity.type == AFFINITY_NONE) { + g.affinity.type = params.affinity.type; + if (params.affinity.type == AFFINITY_COOKIE) { + g.affinity.cookie.name = make_string_ref( + downstreamconf.balloc, params.affinity.cookie.name); + if (!params.affinity.cookie.path.empty()) { + g.affinity.cookie.path = make_string_ref( + downstreamconf.balloc, params.affinity.cookie.path); } - } else if (g.affinity.type != params.affinity.type || - g.affinity.cookie.name != params.affinity.cookie.name || - g.affinity.cookie.path != params.affinity.cookie.path || - g.affinity.cookie.secure != - params.affinity.cookie.secure) { - LOG(ERROR) << "backend: affinity: multiple different affinity " - "configurations found in a single group"; - return -1; + g.affinity.cookie.secure = params.affinity.cookie.secure; } + } else if (g.affinity.type != params.affinity.type || + g.affinity.cookie.name != params.affinity.cookie.name || + g.affinity.cookie.path != params.affinity.cookie.path || + g.affinity.cookie.secure != params.affinity.cookie.secure) { + LOG(ERROR) << "backend: affinity: multiple different affinity " + "configurations found in a single group"; + return -1; } - // If at least one backend requires frontend TLS connection, - // enable it for all backends sharing the same pattern. - if (params.redirect_if_not_tls) { - g.redirect_if_not_tls = true; - } - g.addrs.push_back(addr); - done = true; - break; } - } - if (done) { + // If at least one backend requires frontend TLS connection, + // enable it for all backends sharing the same pattern. + if (params.redirect_if_not_tls) { + g.redirect_if_not_tls = true; + } + g.addrs.push_back(addr); continue; } auto idx = addr_groups.size(); + pattern_addr_indexer.emplace(pattern, idx); addr_groups.emplace_back(pattern); auto &g = addr_groups.back(); g.addrs.push_back(addr); @@ -2387,13 +2383,16 @@ int option_lookup_token(const char *name, size_t namelen) { } int parse_config(Config *config, const StringRef &opt, const StringRef &optarg, - std::set &included_set) { + std::set &included_set, + std::map &pattern_addr_indexer) { auto optid = option_lookup_token(opt.c_str(), opt.size()); - return parse_config(config, optid, opt, optarg, included_set); + return parse_config(config, optid, opt, optarg, included_set, + pattern_addr_indexer); } int parse_config(Config *config, int optid, const StringRef &opt, - const StringRef &optarg, std::set &included_set) { + const StringRef &optarg, std::set &included_set, + std::map &pattern_addr_indexer) { std::array errbuf; char host[NI_MAXHOST]; uint16_t port; @@ -2425,7 +2424,8 @@ int parse_config(Config *config, int optid, const StringRef &opt, auto params = mapping_end == std::end(optarg) ? mapping_end : mapping_end + 1; - if (parse_mapping(config, addr, StringRef{mapping, mapping_end}, + if (parse_mapping(config, addr, pattern_addr_indexer, + StringRef{mapping, mapping_end}, StringRef{params, std::end(optarg)}) != 0) { return -1; } @@ -3126,7 +3126,8 @@ int parse_config(Config *config, int optid, const StringRef &opt, } included_set.insert(optarg); - auto rv = load_config(config, optarg.c_str(), included_set); + auto rv = + load_config(config, optarg.c_str(), included_set, pattern_addr_indexer); included_set.erase(optarg); if (rv != 0) { @@ -3563,7 +3564,8 @@ int parse_config(Config *config, int optid, const StringRef &opt, } int load_config(Config *config, const char *filename, - std::set &include_set) { + std::set &include_set, + std::map &pattern_addr_indexer) { std::ifstream in(filename, std::ios::binary); if (!in) { LOG(ERROR) << "Could not open config file " << filename; @@ -3585,7 +3587,8 @@ int load_config(Config *config, const char *filename, *eq = '\0'; if (parse_config(config, StringRef{std::begin(line), eq}, - StringRef{eq + 1, std::end(line)}, include_set) != 0) { + StringRef{eq + 1, std::end(line)}, include_set, + pattern_addr_indexer) != 0) { return -1; } } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index a69c1ce1..c6527e48 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -1139,20 +1139,26 @@ int option_lookup_token(const char *name, size_t namelen); // stored into the object pointed by |config|. This function returns 0 // if it succeeds, or -1. The |included_set| contains the all paths // already included while processing this configuration, to avoid loop -// in --include option. +// in --include option. The |pattern_addr_indexer| contains a pair of +// pattern of backend, and its index in DownstreamConfig::addr_groups. +// It is introduced to speed up loading configuration file with lots +// of backends. int parse_config(Config *config, const StringRef &opt, const StringRef &optarg, - std::set &included_set); + std::set &included_set, + std::map &pattern_addr_indexer); // Similar to parse_config() above, but additional |optid| which // should be the return value of option_lookup_token(opt). int parse_config(Config *config, int optid, const StringRef &opt, - const StringRef &optarg, std::set &included_set); + const StringRef &optarg, std::set &included_set, + std::map &pattern_addr_indexer); // Loads configurations from |filename| and stores them in |config|. // This function returns 0 if it succeeds, or -1. See parse_config() // for |include_set|. int load_config(Config *config, const char *filename, - std::set &include_set); + std::set &include_set, + std::map &pattern_addr_indexer); // Parses header field in |optarg|. We expect header field is formed // like "NAME: VALUE". We require that NAME is non empty string. ":" diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 3c6b270c..1865e441 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -68,51 +68,44 @@ void proc_wev_cb(struct ev_loop *loop, ev_timer *w, int revents) { } } // namespace +// DownstreamKey is used to index SharedDownstreamAddr in order to +// find the same configuration. +using DownstreamKey = + std::tuple>, + bool, int, StringRef, StringRef, int>; + namespace { -bool match_shared_downstream_addr( - const std::shared_ptr &lhs, - const std::shared_ptr &rhs) { - if (lhs->addrs.size() != rhs->addrs.size()) { - return false; +DownstreamKey create_downstream_key( + const std::shared_ptr &shared_addr) { + DownstreamKey dkey; + + auto &addrs = std::get<0>(dkey); + addrs.resize(shared_addr->addrs.size()); + auto p = std::begin(addrs); + for (auto &a : shared_addr->addrs) { + std::get<0>(*p) = a.host; + std::get<1>(*p) = a.sni; + std::get<2>(*p) = a.fall; + std::get<3>(*p) = a.rise; + std::get<4>(*p) = a.proto; + std::get<5>(*p) = a.port; + std::get<6>(*p) = a.host_unix; + std::get<7>(*p) = a.tls; + std::get<8>(*p) = a.dns; + ++p; } + std::sort(std::begin(addrs), std::end(addrs)); - if (lhs->affinity.type != rhs->affinity.type || - lhs->redirect_if_not_tls != rhs->redirect_if_not_tls) { - return false; - } + std::get<1>(dkey) = shared_addr->redirect_if_not_tls; - if (lhs->affinity.type == AFFINITY_COOKIE && - (lhs->affinity.cookie.name != rhs->affinity.cookie.name || - lhs->affinity.cookie.path != rhs->affinity.cookie.path || - lhs->affinity.cookie.secure != rhs->affinity.cookie.secure)) { - return false; - } + auto &affinity = shared_addr->affinity; + std::get<2>(dkey) = affinity.type; + std::get<3>(dkey) = affinity.cookie.name; + std::get<4>(dkey) = affinity.cookie.path; + std::get<5>(dkey) = affinity.cookie.secure; - auto used = std::vector(lhs->addrs.size()); - - for (auto &a : lhs->addrs) { - size_t i; - for (i = 0; i < rhs->addrs.size(); ++i) { - if (used[i]) { - continue; - } - - auto &b = rhs->addrs[i]; - if (a.host == b.host && a.port == b.port && a.host_unix == b.host_unix && - a.proto == b.proto && a.tls == b.tls && a.sni == b.sni && - a.fall == b.fall && a.rise == b.rise && a.dns == b.dns) { - break; - } - } - - if (i == rhs->addrs.size()) { - return false; - } - - used[i] = true; - } - - return true; + return dkey; } } // namespace @@ -182,6 +175,8 @@ void Worker::replace_downstream_config( downstream_addr_groups_ = std::vector>(groups.size()); + std::map addr_groups_indexer; + for (size_t i = 0; i < groups.size(); ++i) { auto &src = groups[i]; auto &dst = downstream_addr_groups_[i]; @@ -268,14 +263,11 @@ void Worker::replace_downstream_config( // share the connection if patterns have the same set of backend // addresses. - auto end = std::begin(downstream_addr_groups_) + i; - auto it = std::find_if( - std::begin(downstream_addr_groups_), end, - [&shared_addr](const std::shared_ptr &group) { - return match_shared_downstream_addr(group->shared_addr, shared_addr); - }); - if (it == end) { + auto dkey = create_downstream_key(shared_addr); + auto it = addr_groups_indexer.find(dkey); + + if (it == std::end(addr_groups_indexer)) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "number of http/1.1 backend: " << num_http1 << ", number of h2 backend: " << num_http2; @@ -291,12 +283,15 @@ void Worker::replace_downstream_config( } dst->shared_addr = shared_addr; + + addr_groups_indexer.emplace(std::move(dkey), i); } else { + auto &g = *(std::begin(downstream_addr_groups_) + (*it).second); if (LOG_ENABLED(INFO)) { LOG(INFO) << dst->pattern << " shares the same backend group with " - << (*it)->pattern; + << g->pattern; } - dst->shared_addr = (*it)->shared_addr; + dst->shared_addr = g->shared_addr; } } }