nghttpx: Faster configuration loading with lots of backends

src/shrpx.cc

@@ -2768,10 +2768,12 @@ namespace {
 int process_options(Config *config,
                     std::vector<std::pair<StringRef, StringRef>> &cmdcfgs) {
   std::array<char, STRERROR_BUFSIZE> errbuf;
+  std::map<StringRef, size_t> pattern_addr_indexer;
   if (conf_exists(config->conf_path.c_str())) {
     LOG(NOTICE) << "Loading configuration from " << config->conf_path;
     std::set<StringRef> include_set;
-    if (load_config(config, config->conf_path.c_str(), include_set) == -1) {
+    if (load_config(config, config->conf_path.c_str(), include_set,
+                    pattern_addr_indexer) == -1) {
       LOG(FATAL) << "Failed to load configuration from " << config->conf_path;
       return -1;
     }
@@ -2785,7 +2787,8 @@ int process_options(Config *config,
     std::set<StringRef> include_set;
 
     for (auto &p : cmdcfgs) {
-      if (parse_config(config, p.first, p.second, include_set) == -1) {
+      if (parse_config(config, p.first, p.second, include_set,
+                       pattern_addr_indexer) == -1) {
         LOG(FATAL) << "Failed to parse command-line argument.";
         return -1;
       }

src/shrpx_api_downstream_connection.cc

@@ -23,7 +23,6 @@
  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 #include "shrpx_api_downstream_connection.h"
-
 #include "shrpx_client_handler.h"
 #include "shrpx_upstream.h"
 #include "shrpx_downstream.h"
@@ -343,6 +342,7 @@ int APIDownstreamConnection::handle_backendconfig() {
   downstreamconf->family = src->family;
 
   std::set<StringRef> include_set;
+  std::map<StringRef, size_t> pattern_addr_indexer;
 
   for (auto first = reinterpret_cast<const uint8_t *>(iov[0].iov_base),
             last = first + iov[0].iov_len;
@@ -376,7 +376,8 @@ int APIDownstreamConnection::handle_backendconfig() {
       continue;
     }
 
-    if (parse_config(&new_config, optid, opt, optval, include_set) != 0) {
+    if (parse_config(&new_config, optid, opt, optval, include_set,
+                     pattern_addr_indexer) != 0) {
       send_reply(400, API_FAILURE);
       return 0;
     }

src/shrpx_config.cc

@@ -943,6 +943,7 @@ namespace {
 //
 // This function returns 0 if it succeeds, or -1.
 int parse_mapping(Config *config, DownstreamAddrConfig &addr,
+                  std::map<StringRef, size_t> &pattern_addr_indexer,
                   const StringRef &src_pattern, const StringRef &src_params) {
   // This returns at least 1 element (it could be empty string).  We
   // will append '/' to all patterns, so it becomes catch-all pattern.
@@ -983,7 +984,6 @@ int parse_mapping(Config *config, DownstreamAddrConfig &addr,
   auto &wildcard_patterns = routerconf.wildcard_patterns;
 
   for (const auto &raw_pattern : mapping) {
-    auto done = false;
     StringRef pattern;
     auto slash = std::find(std::begin(raw_pattern), std::end(raw_pattern), '/');
     if (slash == std::end(raw_pattern)) {
@@ -1010,8 +1010,9 @@ int parse_mapping(Config *config, DownstreamAddrConfig &addr,
       *p = '\0';
       pattern = StringRef{iov.base, p};
     }
-    for (auto &g : addr_groups) {
+    auto it = pattern_addr_indexer.find(pattern);
-      if (g.pattern == pattern) {
+    if (it != std::end(pattern_addr_indexer)) {
+      auto &g = addr_groups[(*it).second];
       // Last value wins if we have multiple different affinity
       // value under one group.
       if (params.affinity.type != AFFINITY_NONE) {
@@ -1029,8 +1030,7 @@ int parse_mapping(Config *config, DownstreamAddrConfig &addr,
         } else if (g.affinity.type != params.affinity.type ||
                    g.affinity.cookie.name != params.affinity.cookie.name ||
                    g.affinity.cookie.path != params.affinity.cookie.path ||
-                     g.affinity.cookie.secure !=
+                   g.affinity.cookie.secure != params.affinity.cookie.secure) {
-                         params.affinity.cookie.secure) {
           LOG(ERROR) << "backend: affinity: multiple different affinity "
                         "configurations found in a single group";
           return -1;
@@ -1042,15 +1042,11 @@ int parse_mapping(Config *config, DownstreamAddrConfig &addr,
         g.redirect_if_not_tls = true;
       }
       g.addrs.push_back(addr);
-        done = true;
-        break;
-      }
-    }
-    if (done) {
       continue;
     }
 
     auto idx = addr_groups.size();
+    pattern_addr_indexer.emplace(pattern, idx);
     addr_groups.emplace_back(pattern);
     auto &g = addr_groups.back();
     g.addrs.push_back(addr);
@@ -2387,13 +2383,16 @@ int option_lookup_token(const char *name, size_t namelen) {
 }
 
 int parse_config(Config *config, const StringRef &opt, const StringRef &optarg,
-                 std::set<StringRef> &included_set) {
+                 std::set<StringRef> &included_set,
+                 std::map<StringRef, size_t> &pattern_addr_indexer) {
   auto optid = option_lookup_token(opt.c_str(), opt.size());
-  return parse_config(config, optid, opt, optarg, included_set);
+  return parse_config(config, optid, opt, optarg, included_set,
+                      pattern_addr_indexer);
 }
 
 int parse_config(Config *config, int optid, const StringRef &opt,
-                 const StringRef &optarg, std::set<StringRef> &included_set) {
+                 const StringRef &optarg, std::set<StringRef> &included_set,
+                 std::map<StringRef, size_t> &pattern_addr_indexer) {
   std::array<char, STRERROR_BUFSIZE> errbuf;
   char host[NI_MAXHOST];
   uint16_t port;
@@ -2425,7 +2424,8 @@ int parse_config(Config *config, int optid, const StringRef &opt,
     auto params =
         mapping_end == std::end(optarg) ? mapping_end : mapping_end + 1;
 
-    if (parse_mapping(config, addr, StringRef{mapping, mapping_end},
+    if (parse_mapping(config, addr, pattern_addr_indexer,
+                      StringRef{mapping, mapping_end},
                       StringRef{params, std::end(optarg)}) != 0) {
       return -1;
     }
@@ -3126,7 +3126,8 @@ int parse_config(Config *config, int optid, const StringRef &opt,
     }
 
     included_set.insert(optarg);
-    auto rv = load_config(config, optarg.c_str(), included_set);
+    auto rv =
+        load_config(config, optarg.c_str(), included_set, pattern_addr_indexer);
     included_set.erase(optarg);
 
     if (rv != 0) {
@@ -3563,7 +3564,8 @@ int parse_config(Config *config, int optid, const StringRef &opt,
 }
 
 int load_config(Config *config, const char *filename,
-                std::set<StringRef> &include_set) {
+                std::set<StringRef> &include_set,
+                std::map<StringRef, size_t> &pattern_addr_indexer) {
   std::ifstream in(filename, std::ios::binary);
   if (!in) {
     LOG(ERROR) << "Could not open config file " << filename;
@@ -3585,7 +3587,8 @@ int load_config(Config *config, const char *filename,
     *eq = '\0';
 
     if (parse_config(config, StringRef{std::begin(line), eq},
-                     StringRef{eq + 1, std::end(line)}, include_set) != 0) {
+                     StringRef{eq + 1, std::end(line)}, include_set,
+                     pattern_addr_indexer) != 0) {
       return -1;
     }
   }

src/shrpx_config.h

@@ -1139,20 +1139,26 @@ int option_lookup_token(const char *name, size_t namelen);
 // stored into the object pointed by |config|. This function returns 0
 // if it succeeds, or -1.  The |included_set| contains the all paths
 // already included while processing this configuration, to avoid loop
-// in --include option.
+// in --include option.  The |pattern_addr_indexer| contains a pair of
+// pattern of backend, and its index in DownstreamConfig::addr_groups.
+// It is introduced to speed up loading configuration file with lots
+// of backends.
 int parse_config(Config *config, const StringRef &opt, const StringRef &optarg,
-                 std::set<StringRef> &included_set);
+                 std::set<StringRef> &included_set,
+                 std::map<StringRef, size_t> &pattern_addr_indexer);
 
 // Similar to parse_config() above, but additional |optid| which
 // should be the return value of option_lookup_token(opt).
 int parse_config(Config *config, int optid, const StringRef &opt,
-                 const StringRef &optarg, std::set<StringRef> &included_set);
+                 const StringRef &optarg, std::set<StringRef> &included_set,
+                 std::map<StringRef, size_t> &pattern_addr_indexer);
 
 // Loads configurations from |filename| and stores them in |config|.
 // This function returns 0 if it succeeds, or -1.  See parse_config()
 // for |include_set|.
 int load_config(Config *config, const char *filename,
-                std::set<StringRef> &include_set);
+                std::set<StringRef> &include_set,
+                std::map<StringRef, size_t> &pattern_addr_indexer);
 
 // Parses header field in |optarg|.  We expect header field is formed
 // like "NAME: VALUE".  We require that NAME is non empty string.  ":"

src/shrpx_worker.cc

@@ -68,51 +68,44 @@ void proc_wev_cb(struct ev_loop *loop, ev_timer *w, int revents) {
 }
 } // namespace
 
+// DownstreamKey is used to index SharedDownstreamAddr in order to
+// find the same configuration.
+using DownstreamKey =
+    std::tuple<std::vector<std::tuple<StringRef, StringRef, size_t, size_t,
+                                      shrpx_proto, uint16_t, bool, bool, bool>>,
+               bool, int, StringRef, StringRef, int>;
+
 namespace {
-bool match_shared_downstream_addr(
+DownstreamKey create_downstream_key(
-    const std::shared_ptr<SharedDownstreamAddr> &lhs,
+    const std::shared_ptr<SharedDownstreamAddr> &shared_addr) {
-    const std::shared_ptr<SharedDownstreamAddr> &rhs) {
+  DownstreamKey dkey;
-  if (lhs->addrs.size() != rhs->addrs.size()) {
-    return false;
-  }
 
-  if (lhs->affinity.type != rhs->affinity.type ||
+  auto &addrs = std::get<0>(dkey);
-      lhs->redirect_if_not_tls != rhs->redirect_if_not_tls) {
+  addrs.resize(shared_addr->addrs.size());
-    return false;
+  auto p = std::begin(addrs);
+  for (auto &a : shared_addr->addrs) {
+    std::get<0>(*p) = a.host;
+    std::get<1>(*p) = a.sni;
+    std::get<2>(*p) = a.fall;
+    std::get<3>(*p) = a.rise;
+    std::get<4>(*p) = a.proto;
+    std::get<5>(*p) = a.port;
+    std::get<6>(*p) = a.host_unix;
+    std::get<7>(*p) = a.tls;
+    std::get<8>(*p) = a.dns;
+    ++p;
   }
+  std::sort(std::begin(addrs), std::end(addrs));
 
-  if (lhs->affinity.type == AFFINITY_COOKIE &&
+  std::get<1>(dkey) = shared_addr->redirect_if_not_tls;
-      (lhs->affinity.cookie.name != rhs->affinity.cookie.name ||
-       lhs->affinity.cookie.path != rhs->affinity.cookie.path ||
-       lhs->affinity.cookie.secure != rhs->affinity.cookie.secure)) {
-    return false;
-  }
 
-  auto used = std::vector<bool>(lhs->addrs.size());
+  auto &affinity = shared_addr->affinity;
+  std::get<2>(dkey) = affinity.type;
+  std::get<3>(dkey) = affinity.cookie.name;
+  std::get<4>(dkey) = affinity.cookie.path;
+  std::get<5>(dkey) = affinity.cookie.secure;
 
-  for (auto &a : lhs->addrs) {
+  return dkey;
-    size_t i;
-    for (i = 0; i < rhs->addrs.size(); ++i) {
-      if (used[i]) {
-        continue;
-      }
-
-      auto &b = rhs->addrs[i];
-      if (a.host == b.host && a.port == b.port && a.host_unix == b.host_unix &&
-          a.proto == b.proto && a.tls == b.tls && a.sni == b.sni &&
-          a.fall == b.fall && a.rise == b.rise && a.dns == b.dns) {
-        break;
-      }
-    }
-
-    if (i == rhs->addrs.size()) {
-      return false;
-    }
-
-    used[i] = true;
-  }
-
-  return true;
 }
 } // namespace
 
@@ -182,6 +175,8 @@ void Worker::replace_downstream_config(
   downstream_addr_groups_ =
       std::vector<std::shared_ptr<DownstreamAddrGroup>>(groups.size());
 
+  std::map<DownstreamKey, size_t> addr_groups_indexer;
+
   for (size_t i = 0; i < groups.size(); ++i) {
     auto &src = groups[i];
     auto &dst = downstream_addr_groups_[i];
@@ -268,14 +263,11 @@ void Worker::replace_downstream_config(
 
     // share the connection if patterns have the same set of backend
     // addresses.
-    auto end = std::begin(downstream_addr_groups_) + i;
-    auto it = std::find_if(
-        std::begin(downstream_addr_groups_), end,
-        [&shared_addr](const std::shared_ptr<DownstreamAddrGroup> &group) {
-          return match_shared_downstream_addr(group->shared_addr, shared_addr);
-        });
 
-    if (it == end) {
+    auto dkey = create_downstream_key(shared_addr);
+    auto it = addr_groups_indexer.find(dkey);
+
+    if (it == std::end(addr_groups_indexer)) {
       if (LOG_ENABLED(INFO)) {
         LOG(INFO) << "number of http/1.1 backend: " << num_http1
                   << ", number of h2 backend: " << num_http2;
@@ -291,12 +283,15 @@ void Worker::replace_downstream_config(
       }
 
       dst->shared_addr = shared_addr;
+
+      addr_groups_indexer.emplace(std::move(dkey), i);
     } else {
+      auto &g = *(std::begin(downstream_addr_groups_) + (*it).second);
       if (LOG_ENABLED(INFO)) {
         LOG(INFO) << dst->pattern << " shares the same backend group with "
-                  << (*it)->pattern;
+                  << g->pattern;
       }
-      dst->shared_addr = (*it)->shared_addr;
+      dst->shared_addr = g->shared_addr;
     }
   }
 }