nghttpx: Support multiple HTTP/1 backend address

For HTTP/1 backend, -b option can be used several times to specify
multiple backend address.  HTTP/2 backend does not support multiple
addresses and only uses first address even if multiple addresses are
specified.
This commit is contained in:
Tatsuhiro Tsujikawa 2014-12-06 18:31:46 +09:00
parent b8dafbdf5e
commit b607a22076
10 changed files with 134 additions and 83 deletions

View File

@ -653,6 +653,11 @@ const char *DEFAULT_ACCESSLOG_FORMAT = "$remote_addr - - [$time_local] "
"\"$http_referer\" \"$http_user_agent\""; "\"$http_referer\" \"$http_user_agent\"";
} // namespace } // namespace
namespace {
auto DEFAULT_DOWNSTREAM_HOST = "127.0.0.1";
int16_t DEFAULT_DOWNSTREAM_PORT = 80;
} // namespace;
namespace { namespace {
void fill_default_config() { void fill_default_config() {
memset(mod_config(), 0, sizeof(*mod_config())); memset(mod_config(), 0, sizeof(*mod_config()));
@ -703,11 +708,6 @@ void fill_default_config() {
mod_config()->upstream_no_tls = false; mod_config()->upstream_no_tls = false;
mod_config()->downstream_no_tls = false; mod_config()->downstream_no_tls = false;
mod_config()->downstream_host = strcopy("127.0.0.1");
mod_config()->downstream_port = 80;
mod_config()->downstream_hostport = nullptr;
mod_config()->downstream_addrlen = 0;
mod_config()->num_worker = 1; mod_config()->num_worker = 1;
mod_config()->http2_max_concurrent_streams = 100; mod_config()->http2_max_concurrent_streams = 100;
mod_config()->add_x_forwarded_for = false; mod_config()->add_x_forwarded_for = false;
@ -819,9 +819,13 @@ Options:
Connections: Connections:
-b, --backend=<HOST,PORT> -b, --backend=<HOST,PORT>
Set backend host and port. Set backend host and port. For HTTP/1 backend,
Default: ')" << get_config()->downstream_host.get() << "," multiple backend addresses are accepted by
<< get_config()->downstream_port << R"(' repeating this option. HTTP/2 backend does not
support multiple backend addresses and the first
occurrence of this option is used.
Default: ')" << DEFAULT_DOWNSTREAM_HOST << ","
<< DEFAULT_DOWNSTREAM_PORT << R"('
-f, --frontend=<HOST,PORT> -f, --frontend=<HOST,PORT>
Set frontend host and port. If <HOST> is '*', it Set frontend host and port. If <HOST> is '*', it
assumes all addresses including both IPv4 and assumes all addresses including both IPv4 and
@ -1772,38 +1776,44 @@ int main(int argc, char **argv) {
} }
} }
bool downstream_ipv6_addr = if (get_config()->downstream_addrs.empty()) {
is_ipv6_numeric_addr(get_config()->downstream_host.get()); DownstreamAddr addr;
addr.host = strcopy(DEFAULT_DOWNSTREAM_HOST);
addr.port = DEFAULT_DOWNSTREAM_PORT;
{ mod_config()->downstream_addrs.push_back(std::move(addr));
std::string hostport;
if (downstream_ipv6_addr) {
hostport += "[";
}
hostport += get_config()->downstream_host.get();
if (downstream_ipv6_addr) {
hostport += "]";
}
hostport += ":";
hostport += util::utos(get_config()->downstream_port);
mod_config()->downstream_hostport = strcopy(hostport);
} }
if (LOG_ENABLED(INFO)) { if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Resolving backend address"; LOG(INFO) << "Resolving backend address";
} }
if (resolve_hostname(
&mod_config()->downstream_addr, &mod_config()->downstream_addrlen, for (auto &addr : mod_config()->downstream_addrs) {
get_config()->downstream_host.get(), get_config()->downstream_port, auto ipv6 = is_ipv6_numeric_addr(addr.host.get());
get_config()->backend_ipv4 std::string hostport;
? AF_INET
: (get_config()->backend_ipv6 ? AF_INET6 : AF_UNSPEC)) == -1) { if (ipv6) {
exit(EXIT_FAILURE); hostport += "[";
}
hostport += addr.host.get();
if (ipv6) {
hostport += "]";
}
hostport += ":";
hostport += util::utos(addr.port);
addr.hostport = strcopy(hostport);
if (resolve_hostname(
&addr.addr, &addr.addrlen, addr.host.get(), addr.port,
get_config()->backend_ipv4
? AF_INET
: (get_config()->backend_ipv6 ? AF_INET6 : AF_UNSPEC)) == -1) {
exit(EXIT_FAILURE);
}
} }
if (get_config()->downstream_http_proxy_host) { if (get_config()->downstream_http_proxy_host) {

View File

@ -717,4 +717,6 @@ void ClientHandler::write_accesslog(int major, int minor, unsigned int status,
upstream_accesslog(get_config()->accesslog_format, &lgsp); upstream_accesslog(get_config()->accesslog_format, &lgsp);
} }
WorkerStat *ClientHandler::get_worker_stat() const { return worker_stat_; }
} // namespace shrpx } // namespace shrpx

View File

@ -114,6 +114,7 @@ public:
// corresponding Downstream object is not available. // corresponding Downstream object is not available.
void write_accesslog(int major, int minor, unsigned int status, void write_accesslog(int major, int minor, unsigned int status,
int64_t body_bytes_sent); int64_t body_bytes_sent);
WorkerStat *get_worker_stat() const;
private: private:
std::unique_ptr<Upstream> upstream_; std::unique_ptr<Upstream> upstream_;

View File

@ -443,8 +443,11 @@ int parse_config(const char *opt, const char *optarg) {
return -1; return -1;
} }
mod_config()->downstream_host = strcopy(host); DownstreamAddr addr;
mod_config()->downstream_port = port; addr.host = strcopy(host);
addr.port = port;
mod_config()->downstream_addrs.push_back(std::move(addr));
return 0; return 0;
} }

View File

@ -151,6 +151,15 @@ struct AltSvc {
uint16_t port; uint16_t port;
}; };
struct DownstreamAddr {
DownstreamAddr() : addr{{0}}, addrlen(0), port(0) {}
sockaddr_union addr;
std::unique_ptr<char[]> host;
std::unique_ptr<char[]> hostport;
size_t addrlen;
uint16_t port;
};
struct Config { struct Config {
// The list of (private key file, certificate file) pair // The list of (private key file, certificate file) pair
std::vector<std::pair<std::string, std::string>> subcerts; std::vector<std::pair<std::string, std::string>> subcerts;
@ -159,7 +168,7 @@ struct Config {
std::vector<unsigned char> alpn_prefs; std::vector<unsigned char> alpn_prefs;
std::vector<LogFragment> accesslog_format; std::vector<LogFragment> accesslog_format;
std::shared_ptr<std::string> cached_time; std::shared_ptr<std::string> cached_time;
sockaddr_union downstream_addr; std::vector<DownstreamAddr> downstream_addrs;
// binary form of http proxy host and port // binary form of http proxy host and port
sockaddr_union downstream_http_proxy_addr; sockaddr_union downstream_http_proxy_addr;
timeval http2_upstream_read_timeout; timeval http2_upstream_read_timeout;
@ -179,8 +188,6 @@ struct Config {
SSL_CTX *default_ssl_ctx; SSL_CTX *default_ssl_ctx;
ssl::CertLookupTree *cert_tree; ssl::CertLookupTree *cert_tree;
const char *server_name; const char *server_name;
std::unique_ptr<char[]> downstream_host;
std::unique_ptr<char[]> downstream_hostport;
std::unique_ptr<char[]> backend_tls_sni_name; std::unique_ptr<char[]> backend_tls_sni_name;
std::unique_ptr<char[]> pid_file; std::unique_ptr<char[]> pid_file;
std::unique_ptr<char[]> conf_path; std::unique_ptr<char[]> conf_path;
@ -215,7 +222,6 @@ struct Config {
nghttp2_option *http2_option; nghttp2_option *http2_option;
char **argv; char **argv;
char *cwd; char *cwd;
size_t downstream_addrlen;
size_t num_worker; size_t num_worker;
size_t http2_max_concurrent_streams; size_t http2_max_concurrent_streams;
size_t http2_upstream_window_bits; size_t http2_upstream_window_bits;
@ -247,7 +253,6 @@ struct Config {
gid_t gid; gid_t gid;
pid_t pid; pid_t pid;
uint16_t port; uint16_t port;
uint16_t downstream_port;
// port in http proxy URI // port in http proxy URI
uint16_t downstream_http_proxy_port; uint16_t downstream_http_proxy_port;
bool verbose; bool verbose;

View File

@ -319,9 +319,9 @@ void proxy_eventcb(bufferevent *bev, short events, void *ptr) {
SSLOG(INFO, http2session) << "Connected to the proxy"; SSLOG(INFO, http2session) << "Connected to the proxy";
} }
std::string req = "CONNECT "; std::string req = "CONNECT ";
req += get_config()->downstream_hostport.get(); req += get_config()->downstream_addrs[0].hostport.get();
req += " HTTP/1.1\r\nHost: "; req += " HTTP/1.1\r\nHost: ";
req += get_config()->downstream_host.get(); req += get_config()->downstream_addrs[0].host.get();
req += "\r\n"; req += "\r\n";
if (get_config()->downstream_http_proxy_userinfo) { if (get_config()->downstream_http_proxy_userinfo) {
req += "Proxy-Authorization: Basic "; req += "Proxy-Authorization: Basic ";
@ -431,7 +431,7 @@ int Http2Session::initiate_connection() {
if (get_config()->backend_tls_sni_name) { if (get_config()->backend_tls_sni_name) {
sni_name = get_config()->backend_tls_sni_name.get(); sni_name = get_config()->backend_tls_sni_name.get();
} else { } else {
sni_name = get_config()->downstream_host.get(); sni_name = get_config()->downstream_addrs[0].host.get();
} }
if (sni_name && !util::numeric_host(sni_name)) { if (sni_name && !util::numeric_host(sni_name)) {
@ -445,7 +445,7 @@ int Http2Session::initiate_connection() {
if (state_ == DISCONNECTED) { if (state_ == DISCONNECTED) {
assert(fd_ == -1); assert(fd_ == -1);
fd_ = socket(get_config()->downstream_addr.storage.ss_family, fd_ = socket(get_config()->downstream_addrs[0].addr.storage.ss_family,
SOCK_STREAM | SOCK_CLOEXEC, 0); SOCK_STREAM | SOCK_CLOEXEC, 0);
} }
@ -460,8 +460,8 @@ int Http2Session::initiate_connection() {
rv = bufferevent_socket_connect( rv = bufferevent_socket_connect(
bev_, bev_,
// TODO maybe not thread-safe? // TODO maybe not thread-safe?
const_cast<sockaddr *>(&get_config()->downstream_addr.sa), const_cast<sockaddr *>(&get_config()->downstream_addrs[0].addr.sa),
get_config()->downstream_addrlen); get_config()->downstream_addrs[0].addrlen);
} else { } else {
rv = 0; rv = 0;
} }
@ -470,7 +470,7 @@ int Http2Session::initiate_connection() {
// Without TLS and proxy. // Without TLS and proxy.
assert(fd_ == -1); assert(fd_ == -1);
fd_ = socket(get_config()->downstream_addr.storage.ss_family, fd_ = socket(get_config()->downstream_addrs[0].addr.storage.ss_family,
SOCK_STREAM | SOCK_CLOEXEC, 0); SOCK_STREAM | SOCK_CLOEXEC, 0);
if (fd_ == -1) { if (fd_ == -1) {
@ -486,8 +486,9 @@ int Http2Session::initiate_connection() {
if (state_ == DISCONNECTED) { if (state_ == DISCONNECTED) {
rv = bufferevent_socket_connect( rv = bufferevent_socket_connect(
bev_, const_cast<sockaddr *>(&get_config()->downstream_addr.sa), bev_,
get_config()->downstream_addrlen); const_cast<sockaddr *>(&get_config()->downstream_addrs[0].addr.sa),
get_config()->downstream_addrs[0].addrlen);
} else { } else {
// Without TLS but with proxy. // Without TLS but with proxy.

View File

@ -33,6 +33,7 @@
#include "shrpx_worker_config.h" #include "shrpx_worker_config.h"
#include "shrpx_connect_blocker.h" #include "shrpx_connect_blocker.h"
#include "shrpx_downstream_connection_pool.h" #include "shrpx_downstream_connection_pool.h"
#include "shrpx_worker.h"
#include "http2.h" #include "http2.h"
#include "util.h" #include "util.h"
#include "libevent_util.h" #include "libevent_util.h"
@ -75,43 +76,64 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
} }
auto evbase = client_handler_->get_evbase(); auto evbase = client_handler_->get_evbase();
auto worker_stat = client_handler_->get_worker_stat();
for (;;) {
auto i = worker_stat->next_downstream;
++worker_stat->next_downstream;
worker_stat->next_downstream %= get_config()->downstream_addrs.size();
auto fd = socket(get_config()->downstream_addr.storage.ss_family, auto fd = socket(get_config()->downstream_addrs[i].addr.storage.ss_family,
SOCK_STREAM | SOCK_CLOEXEC, 0); SOCK_STREAM | SOCK_CLOEXEC, 0);
if (fd == -1) { if (fd == -1) {
connect_blocker->on_failure(); auto error = errno;
DCLOG(WARN, this) << "socket() failed; errno=" << error;
return SHRPX_ERR_NETWORK; connect_blocker->on_failure();
}
bev_ = bufferevent_socket_new(evbase, fd, BEV_OPT_CLOSE_ON_FREE | return SHRPX_ERR_NETWORK;
BEV_OPT_DEFER_CALLBACKS); }
if (!bev_) {
connect_blocker->on_failure();
DCLOG(INFO, this) << "bufferevent_socket_new() failed"; bev_ = bufferevent_socket_new(evbase, fd, BEV_OPT_CLOSE_ON_FREE |
close(fd); BEV_OPT_DEFER_CALLBACKS);
if (!bev_) {
auto error = errno;
DCLOG(WARN, this) << "bufferevent_socket_new() failed; errno=" << error;
return SHRPX_ERR_NETWORK; connect_blocker->on_failure();
} close(fd);
int rv = bufferevent_socket_connect(
bev_,
// TODO maybe not thread-safe?
const_cast<sockaddr *>(&get_config()->downstream_addr.sa),
get_config()->downstream_addrlen);
if (rv != 0) {
connect_blocker->on_failure();
bufferevent_free(bev_); return SHRPX_ERR_NETWORK;
bev_ = nullptr; }
return SHRPX_ERR_NETWORK; int rv = bufferevent_socket_connect(
} bev_,
// TODO maybe not thread-safe?
const_cast<sockaddr *>(&get_config()->downstream_addrs[i].addr.sa),
get_config()->downstream_addrs[i].addrlen);
if (rv != 0) {
auto error = errno;
DCLOG(WARN, this) << "bufferevent_socket_connect() failed; errno="
<< error;
connect_blocker->on_success(); connect_blocker->on_failure();
bufferevent_free(bev_);
bev_ = nullptr;
if (LOG_ENABLED(INFO)) { if (i == worker_stat->next_downstream) {
DCLOG(INFO, this) << "Connecting to downstream server"; return SHRPX_ERR_NETWORK;
}
// Try again with the next downstream server
continue;
}
connect_blocker->on_success();
if (LOG_ENABLED(INFO)) {
DCLOG(INFO, this) << "Connecting to downstream server";
}
break;
} }
} }

View File

@ -657,10 +657,10 @@ int check_cert(SSL *ssl) {
std::vector<std::string> dns_names; std::vector<std::string> dns_names;
std::vector<std::string> ip_addrs; std::vector<std::string> ip_addrs;
get_altnames(cert, dns_names, ip_addrs, common_name); get_altnames(cert, dns_names, ip_addrs, common_name);
if (verify_hostname(get_config()->downstream_host.get(), if (verify_hostname(get_config()->downstream_addrs[0].host.get(),
&get_config()->downstream_addr, &get_config()->downstream_addrs[0].addr,
get_config()->downstream_addrlen, dns_names, ip_addrs, get_config()->downstream_addrs[0].addrlen, dns_names,
common_name) != 0) { ip_addrs, common_name) != 0) {
LOG(ERROR) << "Certificate verification failed: hostname does not match"; LOG(ERROR) << "Certificate verification failed: hostname does not match";
return -1; return -1;
} }

View File

@ -54,6 +54,9 @@ ClientHandler *accept_connection(event_base *evbase,
WorkerStat *worker_stat, WorkerStat *worker_stat,
DownstreamConnectionPool *dconn_pool); DownstreamConnectionPool *dconn_pool);
// Check peer's certificate against first downstream address in
// Config::downstream_addrs. We only consider first downstream since
// we use this function for HTTP/2 downstream link only.
int check_cert(SSL *ssl); int check_cert(SSL *ssl);
// Retrieves DNS and IP address in subjectAltNames and commonName from // Retrieves DNS and IP address in subjectAltNames and commonName from

View File

@ -35,9 +35,13 @@
namespace shrpx { namespace shrpx {
struct WorkerStat { struct WorkerStat {
WorkerStat() : num_connections(0) {} WorkerStat() : num_connections(0), next_downstream(0) {}
size_t num_connections; size_t num_connections;
// Next downstream index in Config::downstream_addrs. For HTTP/2
// downstream connections, this is always 0. For HTTP/1, this is
// used as load balancing.
size_t next_downstream;
}; };
class Worker { class Worker {