diff --git a/src/shrpx.cc b/src/shrpx.cc index 0876b336..70234262 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -653,6 +653,11 @@ const char *DEFAULT_ACCESSLOG_FORMAT = "$remote_addr - - [$time_local] " "\"$http_referer\" \"$http_user_agent\""; } // namespace +namespace { +auto DEFAULT_DOWNSTREAM_HOST = "127.0.0.1"; +int16_t DEFAULT_DOWNSTREAM_PORT = 80; +} // namespace; + namespace { void fill_default_config() { memset(mod_config(), 0, sizeof(*mod_config())); @@ -703,11 +708,6 @@ void fill_default_config() { mod_config()->upstream_no_tls = false; mod_config()->downstream_no_tls = false; - mod_config()->downstream_host = strcopy("127.0.0.1"); - mod_config()->downstream_port = 80; - mod_config()->downstream_hostport = nullptr; - mod_config()->downstream_addrlen = 0; - mod_config()->num_worker = 1; mod_config()->http2_max_concurrent_streams = 100; mod_config()->add_x_forwarded_for = false; @@ -819,9 +819,13 @@ Options: Connections: -b, --backend= - Set backend host and port. - Default: ')" << get_config()->downstream_host.get() << "," - << get_config()->downstream_port << R"(' + Set backend host and port. For HTTP/1 backend, + multiple backend addresses are accepted by + repeating this option. HTTP/2 backend does not + support multiple backend addresses and the first + occurrence of this option is used. + Default: ')" << DEFAULT_DOWNSTREAM_HOST << "," + << DEFAULT_DOWNSTREAM_PORT << R"(' -f, --frontend= Set frontend host and port. If is '*', it assumes all addresses including both IPv4 and @@ -1772,38 +1776,44 @@ int main(int argc, char **argv) { } } - bool downstream_ipv6_addr = - is_ipv6_numeric_addr(get_config()->downstream_host.get()); + if (get_config()->downstream_addrs.empty()) { + DownstreamAddr addr; + addr.host = strcopy(DEFAULT_DOWNSTREAM_HOST); + addr.port = DEFAULT_DOWNSTREAM_PORT; - { - std::string hostport; - - if (downstream_ipv6_addr) { - hostport += "["; - } - - hostport += get_config()->downstream_host.get(); - - if (downstream_ipv6_addr) { - hostport += "]"; - } - - hostport += ":"; - hostport += util::utos(get_config()->downstream_port); - - mod_config()->downstream_hostport = strcopy(hostport); + mod_config()->downstream_addrs.push_back(std::move(addr)); } if (LOG_ENABLED(INFO)) { LOG(INFO) << "Resolving backend address"; } - if (resolve_hostname( - &mod_config()->downstream_addr, &mod_config()->downstream_addrlen, - get_config()->downstream_host.get(), get_config()->downstream_port, - get_config()->backend_ipv4 - ? AF_INET - : (get_config()->backend_ipv6 ? AF_INET6 : AF_UNSPEC)) == -1) { - exit(EXIT_FAILURE); + + for (auto &addr : mod_config()->downstream_addrs) { + auto ipv6 = is_ipv6_numeric_addr(addr.host.get()); + std::string hostport; + + if (ipv6) { + hostport += "["; + } + + hostport += addr.host.get(); + + if (ipv6) { + hostport += "]"; + } + + hostport += ":"; + hostport += util::utos(addr.port); + + addr.hostport = strcopy(hostport); + + if (resolve_hostname( + &addr.addr, &addr.addrlen, addr.host.get(), addr.port, + get_config()->backend_ipv4 + ? AF_INET + : (get_config()->backend_ipv6 ? AF_INET6 : AF_UNSPEC)) == -1) { + exit(EXIT_FAILURE); + } } if (get_config()->downstream_http_proxy_host) { diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index ec76efe0..4946d592 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -717,4 +717,6 @@ void ClientHandler::write_accesslog(int major, int minor, unsigned int status, upstream_accesslog(get_config()->accesslog_format, &lgsp); } +WorkerStat *ClientHandler::get_worker_stat() const { return worker_stat_; } + } // namespace shrpx diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index 7770d74f..87a4ed9c 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -114,6 +114,7 @@ public: // corresponding Downstream object is not available. void write_accesslog(int major, int minor, unsigned int status, int64_t body_bytes_sent); + WorkerStat *get_worker_stat() const; private: std::unique_ptr upstream_; diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 9b7a9322..64fba1a0 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -443,8 +443,11 @@ int parse_config(const char *opt, const char *optarg) { return -1; } - mod_config()->downstream_host = strcopy(host); - mod_config()->downstream_port = port; + DownstreamAddr addr; + addr.host = strcopy(host); + addr.port = port; + + mod_config()->downstream_addrs.push_back(std::move(addr)); return 0; } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 0db64d54..abe46695 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -151,6 +151,15 @@ struct AltSvc { uint16_t port; }; +struct DownstreamAddr { + DownstreamAddr() : addr{{0}}, addrlen(0), port(0) {} + sockaddr_union addr; + std::unique_ptr host; + std::unique_ptr hostport; + size_t addrlen; + uint16_t port; +}; + struct Config { // The list of (private key file, certificate file) pair std::vector> subcerts; @@ -159,7 +168,7 @@ struct Config { std::vector alpn_prefs; std::vector accesslog_format; std::shared_ptr cached_time; - sockaddr_union downstream_addr; + std::vector downstream_addrs; // binary form of http proxy host and port sockaddr_union downstream_http_proxy_addr; timeval http2_upstream_read_timeout; @@ -179,8 +188,6 @@ struct Config { SSL_CTX *default_ssl_ctx; ssl::CertLookupTree *cert_tree; const char *server_name; - std::unique_ptr downstream_host; - std::unique_ptr downstream_hostport; std::unique_ptr backend_tls_sni_name; std::unique_ptr pid_file; std::unique_ptr conf_path; @@ -215,7 +222,6 @@ struct Config { nghttp2_option *http2_option; char **argv; char *cwd; - size_t downstream_addrlen; size_t num_worker; size_t http2_max_concurrent_streams; size_t http2_upstream_window_bits; @@ -247,7 +253,6 @@ struct Config { gid_t gid; pid_t pid; uint16_t port; - uint16_t downstream_port; // port in http proxy URI uint16_t downstream_http_proxy_port; bool verbose; diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 9c147c1c..a59fd8bf 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -319,9 +319,9 @@ void proxy_eventcb(bufferevent *bev, short events, void *ptr) { SSLOG(INFO, http2session) << "Connected to the proxy"; } std::string req = "CONNECT "; - req += get_config()->downstream_hostport.get(); + req += get_config()->downstream_addrs[0].hostport.get(); req += " HTTP/1.1\r\nHost: "; - req += get_config()->downstream_host.get(); + req += get_config()->downstream_addrs[0].host.get(); req += "\r\n"; if (get_config()->downstream_http_proxy_userinfo) { req += "Proxy-Authorization: Basic "; @@ -431,7 +431,7 @@ int Http2Session::initiate_connection() { if (get_config()->backend_tls_sni_name) { sni_name = get_config()->backend_tls_sni_name.get(); } else { - sni_name = get_config()->downstream_host.get(); + sni_name = get_config()->downstream_addrs[0].host.get(); } if (sni_name && !util::numeric_host(sni_name)) { @@ -445,7 +445,7 @@ int Http2Session::initiate_connection() { if (state_ == DISCONNECTED) { assert(fd_ == -1); - fd_ = socket(get_config()->downstream_addr.storage.ss_family, + fd_ = socket(get_config()->downstream_addrs[0].addr.storage.ss_family, SOCK_STREAM | SOCK_CLOEXEC, 0); } @@ -460,8 +460,8 @@ int Http2Session::initiate_connection() { rv = bufferevent_socket_connect( bev_, // TODO maybe not thread-safe? - const_cast(&get_config()->downstream_addr.sa), - get_config()->downstream_addrlen); + const_cast(&get_config()->downstream_addrs[0].addr.sa), + get_config()->downstream_addrs[0].addrlen); } else { rv = 0; } @@ -470,7 +470,7 @@ int Http2Session::initiate_connection() { // Without TLS and proxy. assert(fd_ == -1); - fd_ = socket(get_config()->downstream_addr.storage.ss_family, + fd_ = socket(get_config()->downstream_addrs[0].addr.storage.ss_family, SOCK_STREAM | SOCK_CLOEXEC, 0); if (fd_ == -1) { @@ -486,8 +486,9 @@ int Http2Session::initiate_connection() { if (state_ == DISCONNECTED) { rv = bufferevent_socket_connect( - bev_, const_cast(&get_config()->downstream_addr.sa), - get_config()->downstream_addrlen); + bev_, + const_cast(&get_config()->downstream_addrs[0].addr.sa), + get_config()->downstream_addrs[0].addrlen); } else { // Without TLS but with proxy. diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index b816ca25..9c9816ff 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -33,6 +33,7 @@ #include "shrpx_worker_config.h" #include "shrpx_connect_blocker.h" #include "shrpx_downstream_connection_pool.h" +#include "shrpx_worker.h" #include "http2.h" #include "util.h" #include "libevent_util.h" @@ -75,43 +76,64 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { } auto evbase = client_handler_->get_evbase(); + auto worker_stat = client_handler_->get_worker_stat(); + for (;;) { + auto i = worker_stat->next_downstream; + ++worker_stat->next_downstream; + worker_stat->next_downstream %= get_config()->downstream_addrs.size(); - auto fd = socket(get_config()->downstream_addr.storage.ss_family, - SOCK_STREAM | SOCK_CLOEXEC, 0); + auto fd = socket(get_config()->downstream_addrs[i].addr.storage.ss_family, + SOCK_STREAM | SOCK_CLOEXEC, 0); - if (fd == -1) { - connect_blocker->on_failure(); + if (fd == -1) { + auto error = errno; + DCLOG(WARN, this) << "socket() failed; errno=" << error; - return SHRPX_ERR_NETWORK; - } + connect_blocker->on_failure(); - bev_ = bufferevent_socket_new(evbase, fd, BEV_OPT_CLOSE_ON_FREE | - BEV_OPT_DEFER_CALLBACKS); - if (!bev_) { - connect_blocker->on_failure(); + return SHRPX_ERR_NETWORK; + } - DCLOG(INFO, this) << "bufferevent_socket_new() failed"; - close(fd); + bev_ = bufferevent_socket_new(evbase, fd, BEV_OPT_CLOSE_ON_FREE | + BEV_OPT_DEFER_CALLBACKS); + if (!bev_) { + auto error = errno; + DCLOG(WARN, this) << "bufferevent_socket_new() failed; errno=" << error; - return SHRPX_ERR_NETWORK; - } - int rv = bufferevent_socket_connect( - bev_, - // TODO maybe not thread-safe? - const_cast(&get_config()->downstream_addr.sa), - get_config()->downstream_addrlen); - if (rv != 0) { - connect_blocker->on_failure(); + connect_blocker->on_failure(); + close(fd); - bufferevent_free(bev_); - bev_ = nullptr; - return SHRPX_ERR_NETWORK; - } + return SHRPX_ERR_NETWORK; + } + int rv = bufferevent_socket_connect( + bev_, + // TODO maybe not thread-safe? + const_cast(&get_config()->downstream_addrs[i].addr.sa), + get_config()->downstream_addrs[i].addrlen); + if (rv != 0) { + auto error = errno; + DCLOG(WARN, this) << "bufferevent_socket_connect() failed; errno=" + << error; - connect_blocker->on_success(); + connect_blocker->on_failure(); + bufferevent_free(bev_); + bev_ = nullptr; - if (LOG_ENABLED(INFO)) { - DCLOG(INFO, this) << "Connecting to downstream server"; + if (i == worker_stat->next_downstream) { + return SHRPX_ERR_NETWORK; + } + + // Try again with the next downstream server + continue; + } + + connect_blocker->on_success(); + + if (LOG_ENABLED(INFO)) { + DCLOG(INFO, this) << "Connecting to downstream server"; + } + + break; } } diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 16fb62ce..c76553de 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -657,10 +657,10 @@ int check_cert(SSL *ssl) { std::vector dns_names; std::vector ip_addrs; get_altnames(cert, dns_names, ip_addrs, common_name); - if (verify_hostname(get_config()->downstream_host.get(), - &get_config()->downstream_addr, - get_config()->downstream_addrlen, dns_names, ip_addrs, - common_name) != 0) { + if (verify_hostname(get_config()->downstream_addrs[0].host.get(), + &get_config()->downstream_addrs[0].addr, + get_config()->downstream_addrs[0].addrlen, dns_names, + ip_addrs, common_name) != 0) { LOG(ERROR) << "Certificate verification failed: hostname does not match"; return -1; } diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h index 35c77252..90c1159d 100644 --- a/src/shrpx_ssl.h +++ b/src/shrpx_ssl.h @@ -54,6 +54,9 @@ ClientHandler *accept_connection(event_base *evbase, WorkerStat *worker_stat, DownstreamConnectionPool *dconn_pool); +// Check peer's certificate against first downstream address in +// Config::downstream_addrs. We only consider first downstream since +// we use this function for HTTP/2 downstream link only. int check_cert(SSL *ssl); // Retrieves DNS and IP address in subjectAltNames and commonName from diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index aa0330aa..ebfff386 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -35,9 +35,13 @@ namespace shrpx { struct WorkerStat { - WorkerStat() : num_connections(0) {} + WorkerStat() : num_connections(0), next_downstream(0) {} size_t num_connections; + // Next downstream index in Config::downstream_addrs. For HTTP/2 + // downstream connections, this is always 0. For HTTP/1, this is + // used as load balancing. + size_t next_downstream; }; class Worker {