nghttpx: Support multiple HTTP/1 backend address

For HTTP/1 backend, -b option can be used several times to specify
multiple backend address.  HTTP/2 backend does not support multiple
addresses and only uses first address even if multiple addresses are
specified.

src/shrpx.cc

@@ -653,6 +653,11 @@ const char *DEFAULT_ACCESSLOG_FORMAT = "$remote_addr - - [$time_local] "
                                        "\"$http_referer\" \"$http_user_agent\"";
 } // namespace
 
+namespace {
+auto DEFAULT_DOWNSTREAM_HOST = "127.0.0.1";
+int16_t DEFAULT_DOWNSTREAM_PORT = 80;
+} // namespace;
+
 namespace {
 void fill_default_config() {
   memset(mod_config(), 0, sizeof(*mod_config()));
@@ -703,11 +708,6 @@ void fill_default_config() {
   mod_config()->upstream_no_tls = false;
   mod_config()->downstream_no_tls = false;
 
-  mod_config()->downstream_host = strcopy("127.0.0.1");
-  mod_config()->downstream_port = 80;
-  mod_config()->downstream_hostport = nullptr;
-  mod_config()->downstream_addrlen = 0;
-
   mod_config()->num_worker = 1;
   mod_config()->http2_max_concurrent_streams = 100;
   mod_config()->add_x_forwarded_for = false;
@@ -819,9 +819,13 @@ Options:
 
 Connections:
   -b, --backend=<HOST,PORT>
-                     Set backend host and port.
+                     Set backend  host and port.  For  HTTP/1 backend,
-                     Default: ')" << get_config()->downstream_host.get() << ","
+                     multiple  backend   addresses  are   accepted  by
-      << get_config()->downstream_port << R"('
+                     repeating this  option.  HTTP/2 backend  does not
+                     support multiple backend  addresses and the first
+                     occurrence of this option is used.
+                     Default: ')" << DEFAULT_DOWNSTREAM_HOST << ","
+      << DEFAULT_DOWNSTREAM_PORT << R"('
   -f, --frontend=<HOST,PORT>
                      Set frontend host and port.  If <HOST> is '*', it
                      assumes  all addresses  including  both IPv4  and
@@ -1772,38 +1776,44 @@ int main(int argc, char **argv) {
     }
   }
 
-  bool downstream_ipv6_addr =
+  if (get_config()->downstream_addrs.empty()) {
-      is_ipv6_numeric_addr(get_config()->downstream_host.get());
+    DownstreamAddr addr;
+    addr.host = strcopy(DEFAULT_DOWNSTREAM_HOST);
+    addr.port = DEFAULT_DOWNSTREAM_PORT;
 
-  {
+    mod_config()->downstream_addrs.push_back(std::move(addr));
-    std::string hostport;
-
-    if (downstream_ipv6_addr) {
-      hostport += "[";
-    }
-
-    hostport += get_config()->downstream_host.get();
-
-    if (downstream_ipv6_addr) {
-      hostport += "]";
-    }
-
-    hostport += ":";
-    hostport += util::utos(get_config()->downstream_port);
-
-    mod_config()->downstream_hostport = strcopy(hostport);
   }
 
   if (LOG_ENABLED(INFO)) {
     LOG(INFO) << "Resolving backend address";
   }
-  if (resolve_hostname(
+
-          &mod_config()->downstream_addr, &mod_config()->downstream_addrlen,
+  for (auto &addr : mod_config()->downstream_addrs) {
-          get_config()->downstream_host.get(), get_config()->downstream_port,
+    auto ipv6 = is_ipv6_numeric_addr(addr.host.get());
-          get_config()->backend_ipv4
+    std::string hostport;
-              ? AF_INET
+
-              : (get_config()->backend_ipv6 ? AF_INET6 : AF_UNSPEC)) == -1) {
+    if (ipv6) {
-    exit(EXIT_FAILURE);
+      hostport += "[";
+    }
+
+    hostport += addr.host.get();
+
+    if (ipv6) {
+      hostport += "]";
+    }
+
+    hostport += ":";
+    hostport += util::utos(addr.port);
+
+    addr.hostport = strcopy(hostport);
+
+    if (resolve_hostname(
+            &addr.addr, &addr.addrlen, addr.host.get(), addr.port,
+            get_config()->backend_ipv4
+                ? AF_INET
+                : (get_config()->backend_ipv6 ? AF_INET6 : AF_UNSPEC)) == -1) {
+      exit(EXIT_FAILURE);
+    }
   }
 
   if (get_config()->downstream_http_proxy_host) {

src/shrpx_client_handler.cc

@@ -717,4 +717,6 @@ void ClientHandler::write_accesslog(int major, int minor, unsigned int status,
   upstream_accesslog(get_config()->accesslog_format, &lgsp);
 }
 
+WorkerStat *ClientHandler::get_worker_stat() const { return worker_stat_; }
+
 } // namespace shrpx

src/shrpx_client_handler.h

@@ -114,6 +114,7 @@ public:
   // corresponding Downstream object is not available.
   void write_accesslog(int major, int minor, unsigned int status,
                        int64_t body_bytes_sent);
+  WorkerStat *get_worker_stat() const;
 
 private:
   std::unique_ptr<Upstream> upstream_;

src/shrpx_config.cc

@@ -443,8 +443,11 @@ int parse_config(const char *opt, const char *optarg) {
       return -1;
     }
 
-    mod_config()->downstream_host = strcopy(host);
+    DownstreamAddr addr;
-    mod_config()->downstream_port = port;
+    addr.host = strcopy(host);
+    addr.port = port;
+
+    mod_config()->downstream_addrs.push_back(std::move(addr));
 
     return 0;
   }

src/shrpx_config.h

@@ -151,6 +151,15 @@ struct AltSvc {
   uint16_t port;
 };
 
+struct DownstreamAddr {
+  DownstreamAddr() : addr{{0}}, addrlen(0), port(0) {}
+  sockaddr_union addr;
+  std::unique_ptr<char[]> host;
+  std::unique_ptr<char[]> hostport;
+  size_t addrlen;
+  uint16_t port;
+};
+
 struct Config {
   // The list of (private key file, certificate file) pair
   std::vector<std::pair<std::string, std::string>> subcerts;
@@ -159,7 +168,7 @@ struct Config {
   std::vector<unsigned char> alpn_prefs;
   std::vector<LogFragment> accesslog_format;
   std::shared_ptr<std::string> cached_time;
-  sockaddr_union downstream_addr;
+  std::vector<DownstreamAddr> downstream_addrs;
   // binary form of http proxy host and port
   sockaddr_union downstream_http_proxy_addr;
   timeval http2_upstream_read_timeout;
@@ -179,8 +188,6 @@ struct Config {
   SSL_CTX *default_ssl_ctx;
   ssl::CertLookupTree *cert_tree;
   const char *server_name;
-  std::unique_ptr<char[]> downstream_host;
-  std::unique_ptr<char[]> downstream_hostport;
   std::unique_ptr<char[]> backend_tls_sni_name;
   std::unique_ptr<char[]> pid_file;
   std::unique_ptr<char[]> conf_path;
@@ -215,7 +222,6 @@ struct Config {
   nghttp2_option *http2_option;
   char **argv;
   char *cwd;
-  size_t downstream_addrlen;
   size_t num_worker;
   size_t http2_max_concurrent_streams;
   size_t http2_upstream_window_bits;
@@ -247,7 +253,6 @@ struct Config {
   gid_t gid;
   pid_t pid;
   uint16_t port;
-  uint16_t downstream_port;
   // port in http proxy URI
   uint16_t downstream_http_proxy_port;
   bool verbose;

src/shrpx_http2_session.cc

@@ -319,9 +319,9 @@ void proxy_eventcb(bufferevent *bev, short events, void *ptr) {
       SSLOG(INFO, http2session) << "Connected to the proxy";
     }
     std::string req = "CONNECT ";
-    req += get_config()->downstream_hostport.get();
+    req += get_config()->downstream_addrs[0].hostport.get();
     req += " HTTP/1.1\r\nHost: ";
-    req += get_config()->downstream_host.get();
+    req += get_config()->downstream_addrs[0].host.get();
     req += "\r\n";
     if (get_config()->downstream_http_proxy_userinfo) {
       req += "Proxy-Authorization: Basic ";
@@ -431,7 +431,7 @@ int Http2Session::initiate_connection() {
       if (get_config()->backend_tls_sni_name) {
         sni_name = get_config()->backend_tls_sni_name.get();
       } else {
-        sni_name = get_config()->downstream_host.get();
+        sni_name = get_config()->downstream_addrs[0].host.get();
       }
 
       if (sni_name && !util::numeric_host(sni_name)) {
@@ -445,7 +445,7 @@ int Http2Session::initiate_connection() {
       if (state_ == DISCONNECTED) {
         assert(fd_ == -1);
 
-        fd_ = socket(get_config()->downstream_addr.storage.ss_family,
+        fd_ = socket(get_config()->downstream_addrs[0].addr.storage.ss_family,
                      SOCK_STREAM | SOCK_CLOEXEC, 0);
       }
 
@@ -460,8 +460,8 @@ int Http2Session::initiate_connection() {
         rv = bufferevent_socket_connect(
             bev_,
             // TODO maybe not thread-safe?
-            const_cast<sockaddr *>(&get_config()->downstream_addr.sa),
+            const_cast<sockaddr *>(&get_config()->downstream_addrs[0].addr.sa),
-            get_config()->downstream_addrlen);
+            get_config()->downstream_addrs[0].addrlen);
       } else {
         rv = 0;
       }
@@ -470,7 +470,7 @@ int Http2Session::initiate_connection() {
         // Without TLS and proxy.
         assert(fd_ == -1);
 
-        fd_ = socket(get_config()->downstream_addr.storage.ss_family,
+        fd_ = socket(get_config()->downstream_addrs[0].addr.storage.ss_family,
                      SOCK_STREAM | SOCK_CLOEXEC, 0);
 
         if (fd_ == -1) {
@@ -486,8 +486,9 @@ int Http2Session::initiate_connection() {
 
       if (state_ == DISCONNECTED) {
         rv = bufferevent_socket_connect(
-            bev_, const_cast<sockaddr *>(&get_config()->downstream_addr.sa),
+            bev_,
-            get_config()->downstream_addrlen);
+            const_cast<sockaddr *>(&get_config()->downstream_addrs[0].addr.sa),
+            get_config()->downstream_addrs[0].addrlen);
       } else {
         // Without TLS but with proxy.
 

src/shrpx_http_downstream_connection.cc

@@ -33,6 +33,7 @@
 #include "shrpx_worker_config.h"
 #include "shrpx_connect_blocker.h"
 #include "shrpx_downstream_connection_pool.h"
+#include "shrpx_worker.h"
 #include "http2.h"
 #include "util.h"
 #include "libevent_util.h"
@@ -75,43 +76,64 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
     }
 
     auto evbase = client_handler_->get_evbase();
+    auto worker_stat = client_handler_->get_worker_stat();
+    for (;;) {
+      auto i = worker_stat->next_downstream;
+      ++worker_stat->next_downstream;
+      worker_stat->next_downstream %= get_config()->downstream_addrs.size();
 
-    auto fd = socket(get_config()->downstream_addr.storage.ss_family,
+      auto fd = socket(get_config()->downstream_addrs[i].addr.storage.ss_family,
-                     SOCK_STREAM | SOCK_CLOEXEC, 0);
+                       SOCK_STREAM | SOCK_CLOEXEC, 0);
 
-    if (fd == -1) {
+      if (fd == -1) {
-      connect_blocker->on_failure();
+        auto error = errno;
+        DCLOG(WARN, this) << "socket() failed; errno=" << error;
 
-      return SHRPX_ERR_NETWORK;
+        connect_blocker->on_failure();
-    }
 
-    bev_ = bufferevent_socket_new(evbase, fd, BEV_OPT_CLOSE_ON_FREE |
+        return SHRPX_ERR_NETWORK;
-                                                  BEV_OPT_DEFER_CALLBACKS);
+      }
-    if (!bev_) {
-      connect_blocker->on_failure();
 
-      DCLOG(INFO, this) << "bufferevent_socket_new() failed";
+      bev_ = bufferevent_socket_new(evbase, fd, BEV_OPT_CLOSE_ON_FREE |
-      close(fd);
+                                                    BEV_OPT_DEFER_CALLBACKS);
+      if (!bev_) {
+        auto error = errno;
+        DCLOG(WARN, this) << "bufferevent_socket_new() failed; errno=" << error;
 
-      return SHRPX_ERR_NETWORK;
+        connect_blocker->on_failure();
-    }
+        close(fd);
-    int rv = bufferevent_socket_connect(
-        bev_,
-        // TODO maybe not thread-safe?
-        const_cast<sockaddr *>(&get_config()->downstream_addr.sa),
-        get_config()->downstream_addrlen);
-    if (rv != 0) {
-      connect_blocker->on_failure();
 
-      bufferevent_free(bev_);
+        return SHRPX_ERR_NETWORK;
-      bev_ = nullptr;
+      }
-      return SHRPX_ERR_NETWORK;
+      int rv = bufferevent_socket_connect(
-    }
+          bev_,
+          // TODO maybe not thread-safe?
+          const_cast<sockaddr *>(&get_config()->downstream_addrs[i].addr.sa),
+          get_config()->downstream_addrs[i].addrlen);
+      if (rv != 0) {
+        auto error = errno;
+        DCLOG(WARN, this) << "bufferevent_socket_connect() failed; errno="
+                          << error;
 
-    connect_blocker->on_success();
+        connect_blocker->on_failure();
+        bufferevent_free(bev_);
+        bev_ = nullptr;
 
-    if (LOG_ENABLED(INFO)) {
+        if (i == worker_stat->next_downstream) {
-      DCLOG(INFO, this) << "Connecting to downstream server";
+          return SHRPX_ERR_NETWORK;
+        }
+
+        // Try again with the next downstream server
+        continue;
+      }
+
+      connect_blocker->on_success();
+
+      if (LOG_ENABLED(INFO)) {
+        DCLOG(INFO, this) << "Connecting to downstream server";
+      }
+
+      break;
     }
   }
 

src/shrpx_ssl.cc

@@ -657,10 +657,10 @@ int check_cert(SSL *ssl) {
   std::vector<std::string> dns_names;
   std::vector<std::string> ip_addrs;
   get_altnames(cert, dns_names, ip_addrs, common_name);
-  if (verify_hostname(get_config()->downstream_host.get(),
+  if (verify_hostname(get_config()->downstream_addrs[0].host.get(),
-                      &get_config()->downstream_addr,
+                      &get_config()->downstream_addrs[0].addr,
-                      get_config()->downstream_addrlen, dns_names, ip_addrs,
+                      get_config()->downstream_addrs[0].addrlen, dns_names,
-                      common_name) != 0) {
+                      ip_addrs, common_name) != 0) {
     LOG(ERROR) << "Certificate verification failed: hostname does not match";
     return -1;
   }

src/shrpx_ssl.h

@@ -54,6 +54,9 @@ ClientHandler *accept_connection(event_base *evbase,
                                  WorkerStat *worker_stat,
                                  DownstreamConnectionPool *dconn_pool);
 
+// Check peer's certificate against first downstream address in
+// Config::downstream_addrs.  We only consider first downstream since
+// we use this function for HTTP/2 downstream link only.
 int check_cert(SSL *ssl);
 
 // Retrieves DNS and IP address in subjectAltNames and commonName from

src/shrpx_worker.h

@@ -35,9 +35,13 @@
 namespace shrpx {
 
 struct WorkerStat {
-  WorkerStat() : num_connections(0) {}
+  WorkerStat() : num_connections(0), next_downstream(0) {}
 
   size_t num_connections;
+  // Next downstream index in Config::downstream_addrs.  For HTTP/2
+  // downstream connections, this is always 0.  For HTTP/1, this is
+  // used as load balancing.
+  size_t next_downstream;
 };
 
 class Worker {