walkero
/
nghttp2
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

nghttpx: Lookup backend host name dynamically 

We have added "dns" parameter to backend option.  If specified, name
lookup is done dynamically.  If not, name lookup is done at start up,
or configuration reloading.  nghttpx caches DNS result including error
case in 30 seconds in this commit.  Later commit makes this
configurable.

DNS resolution is done asynchronously using c-ares library.
This commit is contained in:
Tatsuhiro Tsujikawa 2016-12-04 23:43:41 +09:00
parent 0872f6babe
commit 38b5cad4e3
23 changed files with 1514 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
configure.ac
View File

@ -370,6 +370,13 @@ if test "x${have_openssl}" = "xno"; then
AC_MSG_NOTICE($OPENSSL_PKG_ERRORS) AC_MSG_NOTICE($OPENSSL_PKG_ERRORS)
fi fi
# c-ares (for src)
PKG_CHECK_MODULES([LIBCARES], [libcares >= 1.11.0], [have_libcares=yes],
[have_libcares=no])
if test "x${have_libcares}" = "xno"; then
AC_MSG_NOTICE($LIBCARES_PKG_ERRORS)
fi
# libevent_openssl (for examples) # libevent_openssl (for examples)
# 2.0.8 is required because we use evconnlistener_set_error_cb() # 2.0.8 is required because we use evconnlistener_set_error_cb()
PKG_CHECK_MODULES([LIBEVENT_OPENSSL], [libevent_openssl >= 2.0.8], PKG_CHECK_MODULES([LIBEVENT_OPENSSL], [libevent_openssl >= 2.0.8],
@ -479,13 +486,14 @@ if test "x${request_asio_lib}" = "xyes"; then
fi fi
fi fi
# The nghttp, nghttpd and nghttpx under src depend on zlib, OpenSSL # The nghttp, nghttpd and nghttpx under src depend on zlib, OpenSSL,
# and libev # libev, and libc-ares.
enable_app=no enable_app=no
if test "x${request_app}" != "xno" && if test "x${request_app}" != "xno" &&
test "x${have_zlib}" = "xyes" && test "x${have_zlib}" = "xyes" &&
test "x${have_openssl}" = "xyes" && test "x${have_openssl}" = "xyes" &&
test "x${have_libev}" = "xyes"; then test "x${have_libev}" = "xyes" &&
test "x${have_libcares}" = "xyes"; then
enable_app=yes enable_app=yes
fi fi
@ -640,6 +648,26 @@ AC_SYS_LARGEFILE
AC_CHECK_MEMBER([struct tm.tm_gmtoff], [have_struct_tm_tm_gmtoff=yes], AC_CHECK_MEMBER([struct tm.tm_gmtoff], [have_struct_tm_tm_gmtoff=yes],
[have_struct_tm_tm_gmtoff=no], [[#include <time.h>]]) [have_struct_tm_tm_gmtoff=no], [[#include <time.h>]])
AC_CHECK_MEMBER([struct sockaddr_in.sin_len],
[AC_DEFINE([HAVE_SOCKADDR_IN_SIN_LEN],[1],
[Define to 1 if struct sockaddr_in has sin_len member.])],
[],
[[
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
]])
AC_CHECK_MEMBER([struct sockaddr_in6.sin6_len],
[AC_DEFINE([HAVE_SOCKADDR_IN6_SIN6_LEN],[1],
[Define to 1 if struct sockaddr_in6 has sin6_len member.])],
[],
[[
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
]])
if test "x$have_struct_tm_tm_gmtoff" = "xyes"; then if test "x$have_struct_tm_tm_gmtoff" = "xyes"; then
AC_DEFINE([HAVE_STRUCT_TM_TM_GMTOFF], [1], AC_DEFINE([HAVE_STRUCT_TM_TM_GMTOFF], [1],
[Define to 1 if you have `struct tm.tm_gmtoff` member.]) [Define to 1 if you have `struct tm.tm_gmtoff` member.])
@ -890,6 +918,7 @@ AC_MSG_NOTICE([summary of build options:
OpenSSL: ${have_openssl} (CFLAGS='${OPENSSL_CFLAGS}' LIBS='${OPENSSL_LIBS}') OpenSSL: ${have_openssl} (CFLAGS='${OPENSSL_CFLAGS}' LIBS='${OPENSSL_LIBS}')
Libxml2: ${have_libxml2} (CFLAGS='${XML_CPPFLAGS}' LIBS='${XML_LIBS}') Libxml2: ${have_libxml2} (CFLAGS='${XML_CPPFLAGS}' LIBS='${XML_LIBS}')
Libev: ${have_libev} (CFLAGS='${LIBEV_CFLAGS}' LIBS='${LIBEV_LIBS}') Libev: ${have_libev} (CFLAGS='${LIBEV_CFLAGS}' LIBS='${LIBEV_LIBS}')
Libc-ares ${have_libcares} (CFLAGS='${LIBCARES_CFLAGS}' LIBS='${LIBCARES_LIBS}')
Libevent(SSL): ${have_libevent_openssl} (CFLAGS='${LIBEVENT_OPENSSL_CFLAGS}' LIBS='${LIBEVENT_OPENSSL_LIBS}') Libevent(SSL): ${have_libevent_openssl} (CFLAGS='${LIBEVENT_OPENSSL_CFLAGS}' LIBS='${LIBEVENT_OPENSSL_LIBS}')
Spdylay: ${have_spdylay} (CFLAGS='${LIBSPDYLAY_CFLAGS}' LIBS='${LIBSPDYLAY_LIBS}') Spdylay: ${have_spdylay} (CFLAGS='${LIBSPDYLAY_CFLAGS}' LIBS='${LIBSPDYLAY_LIBS}')
Jansson: ${have_jansson} (CFLAGS='${JANSSON_CFLAGS}' LIBS='${JANSSON_LIBS}') Jansson: ${have_jansson} (CFLAGS='${JANSSON_CFLAGS}' LIBS='${JANSSON_LIBS}')

5
src/Makefile.am
View File

@ -44,6 +44,7 @@ AM_CPPFLAGS = \
@XML_CPPFLAGS@ \ @XML_CPPFLAGS@ \
@LIBEV_CFLAGS@ \ @LIBEV_CFLAGS@ \
@OPENSSL_CFLAGS@ \ @OPENSSL_CFLAGS@ \
@LIBCARES_CFLAGS@ \
@JANSSON_CFLAGS@ \ @JANSSON_CFLAGS@ \
@ZLIB_CFLAGS@ \ @ZLIB_CFLAGS@ \
@DEFS@ @DEFS@
@ -55,6 +56,7 @@ LDADD = $(top_builddir)/lib/libnghttp2.la \
@XML_LIBS@ \ @XML_LIBS@ \
@LIBEV_LIBS@ \ @LIBEV_LIBS@ \
@OPENSSL_LIBS@ \ @OPENSSL_LIBS@ \
@LIBCARES_LIBS@ \
@JANSSON_LIBS@ \ @JANSSON_LIBS@ \
@ZLIB_LIBS@ \ @ZLIB_LIBS@ \
@APPLDFLAGS@ @APPLDFLAGS@
@ -139,6 +141,9 @@ NGHTTPX_SRCS = \
shrpx_health_monitor_downstream_connection.cc \ shrpx_health_monitor_downstream_connection.cc \
shrpx_health_monitor_downstream_connection.h \ shrpx_health_monitor_downstream_connection.h \
shrpx_exec.cc shrpx_exec.h \ shrpx_exec.cc shrpx_exec.h \
shrpx_dns_resolver.cc shrpx_dns_resolver.h \
shrpx_dual_dns_resolver.cc shrpx_dual_dns_resolver.h \
shrpx_dns_tracker.cc shrpx_dns_tracker.h \
buffer.h memchunk.h template.h allocator.h \ buffer.h memchunk.h template.h allocator.h \
xsi_strerror.c xsi_strerror.h xsi_strerror.c xsi_strerror.h

22
src/shrpx.cc
View File

@ -1587,13 +1587,13 @@ Connections:
Several parameters <PARAM> are accepted after <PATTERN>. Several parameters <PARAM> are accepted after <PATTERN>.
The parameters are delimited by ";". The available The parameters are delimited by ";". The available
parameters are: "proto=<PROTO>", "tls", parameters are: "proto=<PROTO>", "tls",
"sni=<SNI_HOST>", "fall=<N>", "rise=<N>", and "sni=<SNI_HOST>", "fall=<N>", "rise=<N>",
"affinity=<METHOD>". The parameter consists of keyword, "affinity=<METHOD>", and "dns". The parameter consists
and optionally followed by "=" and value. For example, of keyword, and optionally followed by "=" and value.
the parameter "proto=h2" consists of the keyword "proto" For example, the parameter "proto=h2" consists of the
and value "h2". The parameter "tls" consists of the keyword "proto" and value "h2". The parameter "tls"
keyword "tls" without value. Each parameter is consists of the keyword "tls" without value. Each
described as follows. parameter is described as follows.
The backend application protocol can be specified using The backend application protocol can be specified using
optional "proto" parameter, and in the form of optional "proto" parameter, and in the form of
@ -1642,6 +1642,14 @@ Connections:
break if one of the backend gets unreachable, or backend break if one of the backend gets unreachable, or backend
settings are reloaded or replaced by API. settings are reloaded or replaced by API.
By default, name resolution of backend host name is done
at start up, or reloading configuration. If "dns"
parameter is given, name resolution takes place
dynamically. This is useful if backend address changes
frequently. If "dns" is given, name resolution of
backend host name at start up, or reloading
configuration is skipped.
Since ";" and ":" are used as delimiter, <PATTERN> must Since ";" and ":" are used as delimiter, <PATTERN> must
not contain these characters. Since ";" has special not contain these characters. Since ";" has special
meaning in shell, the option value must be quoted. meaning in shell, the option value must be quoted.

45
src/shrpx_config.cc
View File

@ -728,6 +728,7 @@ struct DownstreamParams {
shrpx_proto proto; shrpx_proto proto;
shrpx_session_affinity affinity; shrpx_session_affinity affinity;
bool tls; bool tls;
bool dns;
}; };
namespace { namespace {
@ -801,6 +802,8 @@ int parse_downstream_params(DownstreamParams &out,
LOG(ERROR) << "backend: affinity: value must be either none or ip"; LOG(ERROR) << "backend: affinity: value must be either none or ip";
return -1; return -1;
} }
} else if (util::strieq_l("dns", param)) {
out.dns = true;
} else if (!param.empty()) { } else if (!param.empty()) {
LOG(ERROR) << "backend: " << param << ": unknown keyword"; LOG(ERROR) << "backend: " << param << ": unknown keyword";
return -1; return -1;
@ -841,11 +844,17 @@ int parse_mapping(Config *config, DownstreamAddrConfig &addr,
return -1; return -1;
} }
if (addr.host_unix && params.dns) {
LOG(ERROR) << "backend: dns: cannot be used for UNIX domain socket";
return -1;
}
addr.fall = params.fall; addr.fall = params.fall;
addr.rise = params.rise; addr.rise = params.rise;
addr.proto = params.proto; addr.proto = params.proto;
addr.tls = params.tls; addr.tls = params.tls;
addr.sni = make_string_ref(downstreamconf.balloc, params.sni); addr.sni = make_string_ref(downstreamconf.balloc, params.sni);
addr.dns = params.dns;
auto &routerconf = downstreamconf.router; auto &routerconf = downstreamconf.router;
auto &router = routerconf.router; auto &router = routerconf.router;
@ -3442,24 +3451,38 @@ int configure_downstream_group(Config *config, bool http2_proxy,
auto hostport = auto hostport =
util::make_hostport(downstreamconf.balloc, addr.host, addr.port); util::make_hostport(downstreamconf.balloc, addr.host, addr.port);
if (resolve_hostname(&addr.addr, addr.host.c_str(), addr.port, if (!addr.dns) {
downstreamconf.family, resolve_flags) == -1) { if (resolve_hostname(&addr.addr, addr.host.c_str(), addr.port,
LOG(FATAL) << "Resolving backend address failed: " << hostport; downstreamconf.family, resolve_flags) == -1) {
return -1; LOG(FATAL) << "Resolving backend address failed: " << hostport;
} return -1;
}
if (LOG_ENABLED(INFO)) { if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Resolved backend address: " << hostport << " -> " LOG(INFO) << "Resolved backend address: " << hostport << " -> "
<< util::to_numeric_addr(&addr.addr); << util::to_numeric_addr(&addr.addr);
}
} else {
LOG(INFO) << "Resolving backend address " << hostport
<< " takes place dynamically";
} }
} }
if (g.affinity == AFFINITY_IP) { if (g.affinity == AFFINITY_IP) {
size_t idx = 0; size_t idx = 0;
for (auto &addr : g.addrs) { for (auto &addr : g.addrs) {
auto p = reinterpret_cast<uint8_t *>(&addr.addr.su); StringRef key;
rv = compute_affinity_hash(g.affinity_hash, idx, if (addr.dns) {
StringRef{p, addr.addr.len}); if (addr.host_unix) {
key = addr.host;
} else {
key = addr.hostport;
}
} else {
auto p = reinterpret_cast<uint8_t *>(&addr.addr.su);
key = StringRef{p, addr.addr.len};
}
rv = compute_affinity_hash(g.affinity_hash, idx, key);
if (rv != 0) { if (rv != 0) {
return -1; return -1;
} }

3
src/shrpx_config.h
View File

@ -379,6 +379,7 @@ struct UpstreamAddr {
}; };
struct DownstreamAddrConfig { struct DownstreamAddrConfig {
// Resolved address if |dns| is false
Address addr; Address addr;
// backend address. If |host_unix| is true, this is UNIX domain // backend address. If |host_unix| is true, this is UNIX domain
// socket path. This must be NULL terminated string. // socket path. This must be NULL terminated string.
@ -397,6 +398,8 @@ struct DownstreamAddrConfig {
// true if |host| contains UNIX domain socket path. // true if |host| contains UNIX domain socket path.
bool host_unix; bool host_unix;
bool tls; bool tls;
// true if dynamic DNS is enabled
bool dns;
}; };
// Mapping hash to idx which is an index into // Mapping hash to idx which is an index into

334
src/shrpx_dns_resolver.cc Normal file
View File

@ -0,0 +1,334 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2016 Tatsuhiro Tsujikawa
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "shrpx_dns_resolver.h"
#include <cstring>
#include <sys/time.h>
#include "shrpx_log.h"
#include "shrpx_connection.h"
namespace shrpx {
namespace {
void sock_state_cb(void *data, int s, int read, int write) {
auto resolv = static_cast<DNSResolver *>(data);
if (resolv->get_status(nullptr) != DNS_STATUS_RUNNING) {
return;
}
if (read) {
resolv->start_rev(s);
} else {
resolv->stop_rev(s);
}
if (write) {
resolv->start_wev(s);
} else {
resolv->stop_wev(s);
}
}
} // namespace
namespace {
void host_cb(void *arg, int status, int timeouts, hostent *hostent) {
auto resolv = static_cast<DNSResolver *>(arg);
resolv->on_result(status, hostent);
}
} // namespace
namespace {
void process_result(DNSResolver *resolv) {
auto cb = resolv->get_complete_cb();
if (!cb) {
return;
}
Address result;
auto status = resolv->get_status(&result);
switch (status) {
case DNS_STATUS_OK:
case DNS_STATUS_ERROR:
cb(status, &result);
break;
}
// resolv may be deleted here.
}
} // namespace
namespace {
void readcb(struct ev_loop *loop, ev_io *w, int revents) {
auto resolv = static_cast<DNSResolver *>(w->data);
resolv->on_read(w->fd);
process_result(resolv);
}
} // namespace
namespace {
void writecb(struct ev_loop *loop, ev_io *w, int revents) {
auto resolv = static_cast<DNSResolver *>(w->data);
resolv->on_write(w->fd);
process_result(resolv);
}
} // namespace
namespace {
void timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) {
auto resolv = static_cast<DNSResolver *>(w->data);
resolv->on_timeout();
}
} // namespace
namespace {
void stop_ev(struct ev_loop *loop,
const std::vector<std::unique_ptr<ev_io>> &evs) {
for (auto &w : evs) {
ev_io_stop(loop, w.get());
}
}
} // namespace
DNSResolver::DNSResolver(struct ev_loop *loop)
: loop_(loop),
channel_(nullptr),
family_(AF_UNSPEC),
status_(DNS_STATUS_IDLE) {
ev_timer_init(&timer_, timeoutcb, 0., 0.);
timer_.data = this;
}
DNSResolver::~DNSResolver() {
if (channel_) {
ares_destroy(channel_);
}
stop_ev(loop_, revs_);
stop_ev(loop_, wevs_);
ev_timer_stop(loop_, &timer_);
}
int DNSResolver::resolve(const StringRef &name, int family) {
if (status_ != DNS_STATUS_IDLE) {
return -1;
}
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Start resolving host " << name << " in IPv"
<< (family == AF_INET ? "4" : "6");
}
name_ = name;
family_ = family;
int rv;
ares_options opts{};
opts.sock_state_cb = sock_state_cb;
opts.sock_state_cb_data = this;
auto optmask = ARES_OPT_SOCK_STATE_CB;
ares_channel chan;
rv = ares_init_options(&chan, &opts, optmask);
if (rv != ARES_SUCCESS) {
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "ares_init_options failed: " << ares_strerror(rv);
}
status_ = DNS_STATUS_ERROR;
return -1;
}
channel_ = chan;
status_ = DNS_STATUS_RUNNING;
ares_gethostbyname(channel_, name_.c_str(), family_, host_cb, this);
reset_timeout();
return 0;
}
int DNSResolver::on_read(int fd) { return handle_event(fd, ARES_SOCKET_BAD); }
int DNSResolver::on_write(int fd) { return handle_event(ARES_SOCKET_BAD, fd); }
int DNSResolver::on_timeout() {
return handle_event(ARES_SOCKET_BAD, ARES_SOCKET_BAD);
}
int DNSResolver::handle_event(int rfd, int wfd) {
if (status_ == DNS_STATUS_IDLE) {
return -1;
}
ares_process_fd(channel_, rfd, wfd);
switch (status_) {
case DNS_STATUS_RUNNING: {
reset_timeout();
return 0;
}
case DNS_STATUS_OK:
return 0;
case DNS_STATUS_ERROR:
return -1;
default:
// Unreachable
assert(0);
}
}
void DNSResolver::reset_timeout() {
if (status_ != DNS_STATUS_RUNNING) {
return;
}
timeval tvout;
auto tv = ares_timeout(channel_, nullptr, &tvout);
if (tv == nullptr) {
return;
}
timer_.repeat = tv->tv_sec + tv->tv_usec / 1000000.;
ev_timer_again(loop_, &timer_);
}
int DNSResolver::get_status(Address *result) const {
if (status_ != DNS_STATUS_OK) {
return status_;
}
if (result) {
memcpy(result, &result_, sizeof(result_));
}
return status_;
}
namespace {
void start_ev(std::vector<std::unique_ptr<ev_io>> &evs, struct ev_loop *loop,
int fd, int event, IOCb cb, void *data) {
for (auto &w : evs) {
if (w->fd == fd) {
return;
}
}
for (auto &w : evs) {
if (w->fd == -1) {
ev_io_set(w.get(), fd, event);
ev_io_start(loop, w.get());
return;
}
}
auto w = make_unique<ev_io>();
ev_io_init(w.get(), cb, fd, event);
w->data = data;
ev_io_start(loop, w.get());
evs.emplace_back(std::move(w));
}
} // namespace
namespace {
void stop_ev(std::vector<std::unique_ptr<ev_io>> &evs, struct ev_loop *loop,
int fd, int event) {
for (auto &w : evs) {
if (w->fd == fd) {
ev_io_stop(loop, w.get());
ev_io_set(w.get(), -1, event);
return;
}
}
}
} // namespace
void DNSResolver::start_rev(int fd) {
start_ev(revs_, loop_, fd, EV_READ, readcb, this);
}
void DNSResolver::stop_rev(int fd) { stop_ev(revs_, loop_, fd, EV_READ); }
void DNSResolver::start_wev(int fd) {
start_ev(wevs_, loop_, fd, EV_WRITE, writecb, this);
}
void DNSResolver::stop_wev(int fd) { stop_ev(wevs_, loop_, fd, EV_WRITE); }
void DNSResolver::on_result(int status, hostent *hostent) {
stop_ev(loop_, revs_);
stop_ev(loop_, wevs_);
ev_timer_stop(loop_, &timer_);
if (status != ARES_SUCCESS) {
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Address lookup for " << name_
<< " failed: " << ares_strerror(status);
}
status_ = DNS_STATUS_ERROR;
return;
}
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Address lookup for " << name_ << " succeeded";
}
status_ = DNS_STATUS_OK;
switch (hostent->h_addrtype) {
case AF_INET:
for (auto ap = hostent->h_addr_list; *ap; ++ap) {
result_.len = sizeof(result_.su.in);
result_.su.in = {};
result_.su.in.sin_family = AF_INET;
#ifdef HAVE_SOCKADDR_IN_SIN_LEN
result_.su.in.sin_len = sizeof(result_.su.in);
#endif // HAVE_SOCKADDR_IN_SIN_LEN
memcpy(&result_.su.in.sin_addr, *ap, sizeof(result_.su.in.sin_addr));
return;
}
break;
case AF_INET6:
for (auto ap = hostent->h_addr_list; *ap; ++ap) {
result_.len = sizeof(result_.su.in6);
result_.su.in6 = {};
result_.su.in6.sin6_family = AF_INET6;
#ifdef HAVE_SOCKADDR_IN6_SIN6_LEN
result_.su.in6.sin6_len = sizeof(result_.su.in6);
#endif // HAVE_SOCKADDR_IN6_SIN6_LEN
memcpy(&result_.su.in6.sin6_addr, *ap, sizeof(result_.su.in6.sin6_addr));
return;
}
break;
}
// Somehow we got unsupported address family
status_ = DNS_STATUS_ERROR;
}
void DNSResolver::set_complete_cb(CompleteCb cb) {
completeCb_ = std::move(cb);
}
CompleteCb DNSResolver::get_complete_cb() const { return completeCb_; }
} // namespace shrpx

116
src/shrpx_dns_resolver.h Normal file
View File

@ -0,0 +1,116 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2016 Tatsuhiro Tsujikawa
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef SHRPX_DNS_RESOLVER_H
#define SHRPX_DNS_RESOLVER_H
#include "shrpx.h"
#include <sys/socket.h>
#include <netinet/in.h>
#include <vector>
#include <ev.h>
#include <ares.h>
#include "template.h"
#include "network.h"
using namespace nghttp2;
namespace shrpx {
enum DNSResolverStatus {
// Resolver is in initial status
DNS_STATUS_IDLE,
// Resolver is currently resolving host name
DNS_STATUS_RUNNING,
// Resolver successfully resolved host name
DNS_STATUS_OK,
// Resolver failed to resolve host name
DNS_STATUS_ERROR,
};
// Callback function called when host name lookup is finished.
// |status| is either DNS_STATUS_OK, or DNS_STATUS_ERROR. If |status|
// is DNS_STATUS_OK, |result| points to the resolved address. Note
// that port portion of |result| is undefined, and must be initialized
// by application. This callback function is not called if name
// lookup finishes in DNSResolver::resolve() completely. In this
// case, application should call DNSResolver::get_status() to get
// current status and result. In other words, callback is called if
// get_status() returns DNS_STATUS_RUNNING.
using CompleteCb = std::function<void(int status, const Address *result)>;
// DNSResolver is asynchronous name resolver, backed by c-ares
// library.
class DNSResolver {
public:
DNSResolver(struct ev_loop *loop);
~DNSResolver();
// Starts resolving hostname |name|.
int resolve(const StringRef &name, int family);
// Returns status. If status_ is DNS_STATUS_SUCCESS && |result| is
// not nullptr, |*result| is filled.
int get_status(Address *result) const;
// Sets callback function when name lookup finishes. The callback
// function is called in a way that it can destroy this DNSResolver.
void set_complete_cb(CompleteCb cb);
CompleteCb get_complete_cb() const;
// Calls these functions when read/write event occurred respectively.
int on_read(int fd);
int on_write(int fd);
int on_timeout();
// Calls this function when DNS query finished.
void on_result(int staus, hostent *hostent);
void reset_timeout();
void start_rev(int fd);
void stop_rev(int fd);
void start_wev(int fd);
void stop_wev(int fd);
private:
int handle_event(int rfd, int wfd);
std::vector<std::unique_ptr<ev_io>> revs_, wevs_;
Address result_;
CompleteCb completeCb_;
ev_timer timer_;
StringRef name_;
struct ev_loop *loop_;
// ares_channel is pointer type
ares_channel channel_;
// AF_INET or AF_INET6. AF_INET for A record lookup, and AF_INET6
// for AAAA record lookup.
int family_;
int status_;
};
} // namespace shrpx
#endif // SHRPX_DNS_RESOLVER_H

262
src/shrpx_dns_tracker.cc Normal file
View File

@ -0,0 +1,262 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2016 Tatsuhiro Tsujikawa
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "shrpx_dns_tracker.h"
#include "util.h"
namespace shrpx {
DNSTracker::DNSTracker(struct ev_loop *loop) : loop_(loop) {}
DNSTracker::~DNSTracker() {
for (auto &p : ents_) {
auto &qlist = p.second.qlist;
while (!qlist.empty()) {
auto head = qlist.head;
qlist.remove(head);
head->status = DNS_STATUS_ERROR;
head->in_qlist = false;
// TODO Not sure we should call callback here, or it is even be
// safe to do that.
}
}
}
namespace {
constexpr auto DNS_TTL = 30_s;
} // namespace
ResolverEntry DNSTracker::make_entry(std::unique_ptr<DualDNSResolver> resolv,
ImmutableString host, int status,
const Address *result) {
auto ent = ResolverEntry{};
ent.resolv = std::move(resolv);
ent.host = std::move(host);
ent.status = status;
ent.expiry = ev_now(loop_) + DNS_TTL;
if (result) {
ent.result = *result;
}
return ent;
}
void DNSTracker::update_entry(ResolverEntry &ent,
std::unique_ptr<DualDNSResolver> resolv,
int status, const Address *result) {
ent.resolv = std::move(resolv);
ent.status = status;
if (result) {
ent.result = *result;
}
}
int DNSTracker::resolve(Address *result, DNSQuery *dnsq) {
int rv;
auto it = ents_.find(dnsq->host);
if (it == std::end(ents_)) {
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "DNS entry not found for " << dnsq->host;
}
auto resolv = make_unique<DualDNSResolver>(loop_);
auto host_copy =
ImmutableString{std::begin(dnsq->host), std::end(dnsq->host)};
auto host = StringRef{host_copy};
rv = resolv->resolve(host);
if (rv != 0) {
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Name lookup failed for " << host;
}
ents_.emplace(host, make_entry(nullptr, std::move(host_copy),
DNS_STATUS_ERROR, nullptr));
return DNS_STATUS_ERROR;
}
rv = resolv->get_status(result);
switch (rv) {
case DNS_STATUS_ERROR: {
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Name lookup failed for " << host;
}
ents_.emplace(host, make_entry(nullptr, std::move(host_copy),
DNS_STATUS_ERROR, nullptr));
return DNS_STATUS_ERROR;
}
case DNS_STATUS_OK: {
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Name lookup succeeded: " << host << " -> "
<< util::numeric_name(&result->su.sa, result->len);
}
ents_.emplace(host, make_entry(nullptr, std::move(host_copy),
DNS_STATUS_OK, result));
return DNS_STATUS_OK;
}
case DNS_STATUS_RUNNING: {
assert(rv == DNS_STATUS_RUNNING);
auto p = ents_.emplace(host,
make_entry(std::move(resolv), std::move(host_copy),
DNS_STATUS_RUNNING, nullptr));
auto &ent = (*p.first).second;
add_to_qlist(ent, dnsq);
return DNS_STATUS_RUNNING;
}
default:
assert(0);
}
}
auto &ent = (*it).second;
if (ent.status != DNS_STATUS_RUNNING && ent.expiry < ev_now(loop_)) {
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "DNS entry found for " << dnsq->host
<< ", but it has been expired";
}
auto resolv = make_unique<DualDNSResolver>(loop_);
auto host = StringRef{ent.host};
rv = resolv->resolve(host);
if (rv != 0) {
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Name lookup failed for " << host;
}
update_entry(ent, nullptr, DNS_STATUS_ERROR, nullptr);
return DNS_STATUS_ERROR;
}
rv = resolv->get_status(result);
switch (rv) {
case DNS_STATUS_ERROR: {
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Name lookup failed for " << host;
}
update_entry(ent, nullptr, DNS_STATUS_ERROR, nullptr);
return DNS_STATUS_ERROR;
}
case DNS_STATUS_OK: {
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Name lookup succeeded: " << host << " -> "
<< util::numeric_name(&result->su.sa, result->len);
}
update_entry(ent, nullptr, DNS_STATUS_OK, result);
return DNS_STATUS_OK;
}
case DNS_STATUS_RUNNING: {
update_entry(ent, std::move(resolv), DNS_STATUS_RUNNING, nullptr);
add_to_qlist(ent, dnsq);
return DNS_STATUS_RUNNING;
}
default:
assert(0);
}
}
switch (ent.status) {
case DNS_STATUS_RUNNING:
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Waiting for name lookup complete for " << dnsq->host;
}
ent.qlist.append(dnsq);
dnsq->in_qlist = true;
return DNS_STATUS_RUNNING;
case DNS_STATUS_ERROR:
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Name lookup failed for " << dnsq->host << " (cached)";
}
return DNS_STATUS_ERROR;
case DNS_STATUS_OK:
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Name lookup succeeded (cached): " << dnsq->host << " -> "
<< util::numeric_name(&ent.result.su.sa, ent.result.len);
}
if (result) {
memcpy(result, &ent.result, sizeof(*result));
}
return DNS_STATUS_OK;
default:
assert(0);
}
}
void DNSTracker::add_to_qlist(ResolverEntry &ent, DNSQuery *dnsq) {
auto loop = loop_;
ent.resolv->set_complete_cb([&ent, loop](int status, const Address *result) {
auto &qlist = ent.qlist;
while (!qlist.empty()) {
auto head = qlist.head;
qlist.remove(head);
head->status = status;
head->in_qlist = false;
auto cb = head->cb;
cb(status, result);
}
ent.resolv.reset();
ent.status = status;
ent.expiry = ev_now(loop) + DNS_TTL;
if (ent.status == DNS_STATUS_OK) {
ent.result = *result;
}
});
ent.qlist.append(dnsq);
dnsq->in_qlist = true;
}
void DNSTracker::cancel(DNSQuery *dnsq) {
if (!dnsq->in_qlist) {
return;
}
auto it = ents_.find(dnsq->host);
if (it == std::end(ents_)) {
return;
}
auto &ent = (*it).second;
ent.qlist.remove(dnsq);
dnsq->in_qlist = false;
}
} // namespace shrpx

108
src/shrpx_dns_tracker.h Normal file
View File

@ -0,0 +1,108 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2016 Tatsuhiro Tsujikawa
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef SHRPX_DNS_TRACKER_H
#define SHRPX_DNS_TRACKER_H
#include "shrpx.h"
#include <map>
#include "shrpx_dual_dns_resolver.h"
using namespace nghttp2;
namespace shrpx {
struct DNSQuery {
DNSQuery(StringRef host, CompleteCb cb)
: host(std::move(host)),
cb(std::move(cb)),
dlnext(nullptr),
dlprev(nullptr),
status(DNS_STATUS_IDLE),
in_qlist(false) {}
// Host name we lookup for.
StringRef host;
// Callback function called when name lookup finished. This
// callback is not called if name lookup finishes within
// DNSTracker::resolve().
CompleteCb cb;
DNSQuery *dlnext, *dlprev;
int status;
// true if this object is in linked list ResolverEntry::qlist.
bool in_qlist;
};
struct ResolverEntry {
// Host name this entry lookups for.
ImmutableString host;
// DNS resolver. Only non-nullptr if status is DNS_STATUS_RUNNING.
std::unique_ptr<DualDNSResolver> resolv;
// DNSQuery interested in this name lookup result. The result is
// notified to them all.
DList<DNSQuery> qlist;
// Use the same enum with DNSResolverStatus
int status;
// result and its expiry time
Address result;
// time point when cached result expires.
ev_tstamp expiry;
};
class DNSTracker {
public:
DNSTracker(struct ev_loop *loop);
~DNSTracker();
// Lookups host name described in |dnsq|. If name lookup finishes
// within this function (either it came from /etc/hosts, host name
// is numeric, lookup result is cached, etc), it returns
// DNS_STATUS_OK or DNS_STATUS_ERROR. If lookup is successful,
// DNS_STATUS_OK is returned, and |result| is filled. If lookup
// failed, DNS_STATUS_ERROR is returned. If name lookup is being
// done background, it returns DNS_STATUS_RUNNING. Its completion
// is notified by calling dnsq->cb.
int resolve(Address *result, DNSQuery *dnsq);
// Cancels name lookup requested by |dnsq|.
void cancel(DNSQuery *dnsq);
private:
ResolverEntry make_entry(std::unique_ptr<DualDNSResolver> resolv,
ImmutableString host, int status,
const Address *result);
void update_entry(ResolverEntry &ent, std::unique_ptr<DualDNSResolver> resolv,
int status, const Address *result);
void add_to_qlist(ResolverEntry &ent, DNSQuery *dnsq);
std::map<StringRef, ResolverEntry> ents_;
struct ev_loop *loop_;
};
} // namespace shrpx
#endif // SHRPX_DNS_TRACKER_H

87
src/shrpx_dual_dns_resolver.cc Normal file
View File

@ -0,0 +1,87 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2016 Tatsuhiro Tsujikawa
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#include "shrpx_dual_dns_resolver.h"
namespace shrpx {
DualDNSResolver::DualDNSResolver(struct ev_loop *loop)
: resolv4_(loop), resolv6_(loop) {
auto cb = [this](int, const Address *) {
int rv;
Address result;
rv = this->get_status(&result);
switch (rv) {
case DNS_STATUS_ERROR:
case DNS_STATUS_OK:
break;
default:
return;
}
auto cb = this->get_complete_cb();
cb(rv, &result);
};
resolv4_.set_complete_cb(cb);
resolv6_.set_complete_cb(cb);
}
int DualDNSResolver::resolve(const StringRef &host) {
int rv4, rv6;
rv4 = resolv4_.resolve(host, AF_INET);
rv6 = resolv6_.resolve(host, AF_INET6);
if (rv4 != 0 && rv6 != 0) {
return -1;
}
return 0;
}
CompleteCb DualDNSResolver::get_complete_cb() const { return complete_cb_; }
void DualDNSResolver::set_complete_cb(CompleteCb cb) { complete_cb_ = cb; }
int DualDNSResolver::get_status(Address *result) const {
int rv4, rv6;
rv6 = resolv6_.get_status(result);
if (rv6 == DNS_STATUS_OK) {
return DNS_STATUS_OK;
}
rv4 = resolv4_.get_status(result);
if (rv4 == DNS_STATUS_OK) {
return DNS_STATUS_OK;
}
if (rv4 == DNS_STATUS_RUNNING || rv6 == DNS_STATUS_RUNNING) {
return DNS_STATUS_RUNNING;
}
if (rv4 == DNS_STATUS_ERROR && rv6 == DNS_STATUS_ERROR) {
return DNS_STATUS_ERROR;
}
return DNS_STATUS_IDLE;
}
} // namespace shrpx

63
src/shrpx_dual_dns_resolver.h Normal file
View File

@ -0,0 +1,63 @@
/*
* nghttp2 - HTTP/2 C Library
*
* Copyright (c) 2016 Tatsuhiro Tsujikawa
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef SHRPX_DUAL_DNS_RESOLVER_H
#define SHRPX_DUAL_DNS_RESOLVER_H
#include "shrpx.h"
#include <ev.h>
#include "shrpx_dns_resolver.h"
using namespace nghttp2;
namespace shrpx {
// DualDNSResolver performs name resolution for both A and AAAA
// records at the same time. The first successful return (or if we
// have both successful results, prefer to AAAA) is chosen. This is
// wrapper around 2 DNSResolver inside. resolve(), get_status(), and
// how CompleteCb is called have the same semantics with DNSResolver.
class DualDNSResolver {
public:
DualDNSResolver(struct ev_loop *loop);
// Resolves |host|. |host| must be NULL-terminated string.
int resolve(const StringRef &host);
CompleteCb get_complete_cb() const;
void set_complete_cb(CompleteCb cb);
int get_status(Address *result) const;
private:
// For A record
DNSResolver resolv4_;
// For AAAA record
DNSResolver resolv6_;
CompleteCb complete_cb_;
};
} // namespace shrpx
#endif // SHRPX_DUAL_DNS_RESOLVER_H

206
src/shrpx_http2_session.cc
View File

@ -95,7 +95,7 @@ void settings_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents) {
SSLOG(INFO, http2session) << "SETTINGS timeout"; SSLOG(INFO, http2session) << "SETTINGS timeout";
} }
downstream_failure(http2session->get_addr()); downstream_failure(http2session->get_addr(), http2session->get_raddr());
if (http2session->terminate_session(NGHTTP2_SETTINGS_TIMEOUT) != 0) { if (http2session->terminate_session(NGHTTP2_SETTINGS_TIMEOUT) != 0) {
delete http2session; delete http2session;
@ -202,6 +202,7 @@ Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx,
group_(group), group_(group),
addr_(addr), addr_(addr),
session_(nullptr), session_(nullptr),
raddr_(nullptr),
state_(DISCONNECTED), state_(DISCONNECTED),
connection_check_state_(CONNECTION_CHECK_NONE), connection_check_state_(CONNECTION_CHECK_NONE),
freelist_zone_(FREELIST_ZONE_NONE) { freelist_zone_(FREELIST_ZONE_NONE) {
@ -244,6 +245,11 @@ int Http2Session::disconnect(bool hard) {
wb_.reset(); wb_.reset();
if (dns_query_) {
auto dns_tracker = worker_->get_dns_tracker();
dns_tracker->cancel(dns_query_.get());
}
conn_.rlimit.stopw(); conn_.rlimit.stopw();
conn_.wlimit.stopw(); conn_.wlimit.stopw();
@ -302,12 +308,47 @@ int Http2Session::disconnect(bool hard) {
return 0; return 0;
} }
int Http2Session::resolve_name() {
int rv;
auto dns_query = make_unique<DNSQuery>(
addr_->host, [this](int status, const Address *result) {
int rv;
if (status == DNS_STATUS_OK) {
*resolved_addr_ = *result;
util::set_port(*this->resolved_addr_, this->addr_->port);
}
rv = this->initiate_connection();
if (rv != 0) {
delete this;
}
});
resolved_addr_ = make_unique<Address>();
auto dns_tracker = worker_->get_dns_tracker();
rv = dns_tracker->resolve(resolved_addr_.get(), dns_query.get());
switch (rv) {
case DNS_STATUS_ERROR:
return -1;
case DNS_STATUS_RUNNING:
dns_query_ = std::move(dns_query);
state_ = RESOLVING_NAME;
return 0;
case DNS_STATUS_OK:
util::set_port(*resolved_addr_, addr_->port);
return 0;
default:
assert(0);
}
}
int Http2Session::initiate_connection() { int Http2Session::initiate_connection() {
int rv = 0; int rv = 0;
auto worker_blocker = worker_->get_connect_blocker(); auto worker_blocker = worker_->get_connect_blocker();
if (state_ == DISCONNECTED) { if (state_ == DISCONNECTED || state_ == RESOLVING_NAME) {
if (worker_blocker->blocked()) { if (worker_blocker->blocked()) {
if (LOG_ENABLED(INFO)) { if (LOG_ENABLED(INFO)) {
SSLOG(INFO, this) SSLOG(INFO, this)
@ -350,6 +391,8 @@ int Http2Session::initiate_connection() {
return -1; return -1;
} }
raddr_ = &proxy.addr;
worker_blocker->on_success(); worker_blocker->on_success();
ev_io_set(&conn_.rev, conn_.fd, EV_READ); ev_io_set(&conn_.rev, conn_.fd, EV_READ);
@ -374,36 +417,68 @@ int Http2Session::initiate_connection() {
return 0; return 0;
} }
if (state_ == DISCONNECTED || state_ == PROXY_CONNECTED) { if (state_ == DISCONNECTED || state_ == PROXY_CONNECTED ||
state_ == RESOLVING_NAME) {
if (LOG_ENABLED(INFO)) { if (LOG_ENABLED(INFO)) {
SSLOG(INFO, this) << "Connecting to downstream server"; if (state_ != RESOLVING_NAME) {
SSLOG(INFO, this) << "Connecting to downstream server";
}
} }
if (addr_->tls) { if (addr_->tls) {
assert(ssl_ctx_); assert(ssl_ctx_);
auto ssl = ssl::create_ssl(ssl_ctx_); if (state_ != RESOLVING_NAME) {
if (!ssl) { auto ssl = ssl::create_ssl(ssl_ctx_);
return -1; if (!ssl) {
return -1;
}
ssl::setup_downstream_http2_alpn(ssl);
conn_.set_ssl(ssl);
auto sni_name =
addr_->sni.empty() ? StringRef{addr_->host} : StringRef{addr_->sni};
if (!util::numeric_host(sni_name.c_str())) {
// TLS extensions: SNI. There is no documentation about the return
// code for this function (actually this is macro wrapping SSL_ctrl
// at the time of this writing).
SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str());
}
auto tls_session = ssl::reuse_tls_session(addr_->tls_session_cache);
if (tls_session) {
SSL_set_session(conn_.tls.ssl, tls_session);
SSL_SESSION_free(tls_session);
}
} }
ssl::setup_downstream_http2_alpn(ssl); if (state_ == DISCONNECTED) {
if (addr_->dns) {
conn_.set_ssl(ssl); rv = resolve_name();
if (rv != 0) {
auto sni_name = downstream_failure(addr_, nullptr);
addr_->sni.empty() ? StringRef{addr_->host} : StringRef{addr_->sni}; return -1;
}
if (!util::numeric_host(sni_name.c_str())) { if (state_ == RESOLVING_NAME) {
// TLS extensions: SNI. There is no documentation about the return return 0;
// code for this function (actually this is macro wrapping SSL_ctrl }
// at the time of this writing). raddr_ = resolved_addr_.get();
SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str()); } else {
raddr_ = &addr_->addr;
}
} }
auto tls_session = ssl::reuse_tls_session(addr_->tls_session_cache); if (state_ == RESOLVING_NAME) {
if (tls_session) { if (dns_query_->status == DNS_STATUS_ERROR) {
SSL_set_session(conn_.tls.ssl, tls_session); downstream_failure(addr_, nullptr);
SSL_SESSION_free(tls_session); return -1;
}
assert(dns_query_->status == DNS_STATUS_OK);
state_ = DISCONNECTED;
dns_query_.reset();
raddr_ = resolved_addr_.get();
} }
// If state_ == PROXY_CONNECTED, we has connected to the proxy // If state_ == PROXY_CONNECTED, we has connected to the proxy
@ -411,12 +486,11 @@ int Http2Session::initiate_connection() {
if (state_ == DISCONNECTED) { if (state_ == DISCONNECTED) {
assert(conn_.fd == -1); assert(conn_.fd == -1);
conn_.fd = conn_.fd = util::create_nonblock_socket(raddr_->su.storage.ss_family);
util::create_nonblock_socket(addr_->addr.su.storage.ss_family);
if (conn_.fd == -1) { if (conn_.fd == -1) {
auto error = errno; auto error = errno;
SSLOG(WARN, this) SSLOG(WARN, this)
<< "socket() failed; addr=" << util::to_numeric_addr(&addr_->addr) << "socket() failed; addr=" << util::to_numeric_addr(raddr_)
<< ", errno=" << error; << ", errno=" << error;
worker_blocker->on_failure(); worker_blocker->on_failure();
@ -427,15 +501,14 @@ int Http2Session::initiate_connection() {
rv = connect(conn_.fd, rv = connect(conn_.fd,
// TODO maybe not thread-safe? // TODO maybe not thread-safe?
const_cast<sockaddr *>(&addr_->addr.su.sa), const_cast<sockaddr *>(&raddr_->su.sa), raddr_->len);
addr_->addr.len);
if (rv != 0 && errno != EINPROGRESS) { if (rv != 0 && errno != EINPROGRESS) {
auto error = errno; auto error = errno;
SSLOG(WARN, this) << "connect() failed; addr=" SSLOG(WARN, this)
<< util::to_numeric_addr(&addr_->addr) << "connect() failed; addr=" << util::to_numeric_addr(raddr_)
<< ", errno=" << error; << ", errno=" << error;
downstream_failure(addr_); downstream_failure(addr_, raddr_);
return -1; return -1;
} }
@ -445,17 +518,44 @@ int Http2Session::initiate_connection() {
conn_.prepare_client_handshake(); conn_.prepare_client_handshake();
} else { } else {
if (state_ == DISCONNECTED) {
// Without TLS and proxy.
if (addr_->dns) {
rv = resolve_name();
if (rv != 0) {
downstream_failure(addr_, nullptr);
return -1;
}
if (state_ == RESOLVING_NAME) {
return 0;
}
raddr_ = resolved_addr_.get();
} else {
raddr_ = &addr_->addr;
}
}
if (state_ == RESOLVING_NAME) {
if (dns_query_->status == DNS_STATUS_ERROR) {
downstream_failure(addr_, nullptr);
return -1;
}
assert(dns_query_->status == DNS_STATUS_OK);
state_ = DISCONNECTED;
dns_query_.reset();
raddr_ = resolved_addr_.get();
}
if (state_ == DISCONNECTED) { if (state_ == DISCONNECTED) {
// Without TLS and proxy. // Without TLS and proxy.
assert(conn_.fd == -1); assert(conn_.fd == -1);
conn_.fd = conn_.fd = util::create_nonblock_socket(raddr_->su.storage.ss_family);
util::create_nonblock_socket(addr_->addr.su.storage.ss_family);
if (conn_.fd == -1) { if (conn_.fd == -1) {
auto error = errno; auto error = errno;
SSLOG(WARN, this) SSLOG(WARN, this)
<< "socket() failed; addr=" << util::to_numeric_addr(&addr_->addr) << "socket() failed; addr=" << util::to_numeric_addr(raddr_)
<< ", errno=" << error; << ", errno=" << error;
worker_blocker->on_failure(); worker_blocker->on_failure();
@ -464,15 +564,15 @@ int Http2Session::initiate_connection() {
worker_blocker->on_success(); worker_blocker->on_success();
rv = connect(conn_.fd, const_cast<sockaddr *>(&addr_->addr.su.sa), rv = connect(conn_.fd, const_cast<sockaddr *>(&raddr_->su.sa),
addr_->addr.len); raddr_->len);
if (rv != 0 && errno != EINPROGRESS) { if (rv != 0 && errno != EINPROGRESS) {
auto error = errno; auto error = errno;
SSLOG(WARN, this) << "connect() failed; addr=" SSLOG(WARN, this)
<< util::to_numeric_addr(&addr_->addr) << "connect() failed; addr=" << util::to_numeric_addr(raddr_)
<< ", errno=" << error; << ", errno=" << error;
downstream_failure(addr_); downstream_failure(addr_, raddr_);
return -1; return -1;
} }
@ -1544,7 +1644,7 @@ int Http2Session::connection_made() {
#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
if (!next_proto) { if (!next_proto) {
downstream_failure(addr_); downstream_failure(addr_, raddr_);
return -1; return -1;
} }
@ -1553,7 +1653,7 @@ int Http2Session::connection_made() {
SSLOG(INFO, this) << "Negotiated next protocol: " << proto; SSLOG(INFO, this) << "Negotiated next protocol: " << proto;
} }
if (!util::check_h2_is_selected(proto)) { if (!util::check_h2_is_selected(proto)) {
downstream_failure(addr_); downstream_failure(addr_, raddr_);
return -1; return -1;
} }
} }
@ -1842,10 +1942,10 @@ int Http2Session::connected() {
auto sock_error = util::get_socket_error(conn_.fd); auto sock_error = util::get_socket_error(conn_.fd);
if (sock_error != 0) { if (sock_error != 0) {
SSLOG(WARN, this) << "Backend connect failed; addr=" SSLOG(WARN, this) << "Backend connect failed; addr="
<< util::to_numeric_addr(&addr_->addr) << util::to_numeric_addr(raddr_)
<< ": errno=" << sock_error; << ": errno=" << sock_error;
downstream_failure(addr_); downstream_failure(addr_, raddr_);
return -1; return -1;
} }
@ -1955,7 +2055,7 @@ int Http2Session::tls_handshake() {
} }
if (rv < 0) { if (rv < 0) {
downstream_failure(addr_); downstream_failure(addr_, raddr_);
return rv; return rv;
} }
@ -1965,8 +2065,8 @@ int Http2Session::tls_handshake() {
} }
if (!get_config()->tls.insecure && if (!get_config()->tls.insecure &&
ssl::check_cert(conn_.tls.ssl, addr_) != 0) { ssl::check_cert(conn_.tls.ssl, addr_, raddr_) != 0) {
downstream_failure(addr_); downstream_failure(addr_, raddr_);
return -1; return -1;
} }
@ -1974,8 +2074,8 @@ int Http2Session::tls_handshake() {
if (!SSL_session_reused(conn_.tls.ssl)) { if (!SSL_session_reused(conn_.tls.ssl)) {
auto tls_session = SSL_get0_session(conn_.tls.ssl); auto tls_session = SSL_get0_session(conn_.tls.ssl);
if (tls_session) { if (tls_session) {
ssl::try_cache_tls_session(addr_->tls_session_cache, addr_->addr, ssl::try_cache_tls_session(addr_->tls_session_cache, *raddr_, tls_session,
tls_session, ev_now(conn_.loop)); ev_now(conn_.loop));
} }
} }
@ -2262,9 +2362,9 @@ void Http2Session::on_timeout() {
} }
case CONNECTING: { case CONNECTING: {
SSLOG(WARN, this) << "Connect time out; addr=" SSLOG(WARN, this) << "Connect time out; addr="
<< util::to_numeric_addr(&addr_->addr); << util::to_numeric_addr(raddr_);
downstream_failure(addr_); downstream_failure(addr_, raddr_);
break; break;
} }
} }
@ -2284,4 +2384,6 @@ void Http2Session::check_retire() {
signal_write(); signal_write();
} }
const Address *Http2Session::get_raddr() const { return raddr_; }
} // namespace shrpx } // namespace shrpx

14
src/shrpx_http2_session.h
View File

@ -50,6 +50,7 @@ class Http2DownstreamConnection;
class Worker; class Worker;
struct DownstreamAddrGroup; struct DownstreamAddrGroup;
struct DownstreamAddr; struct DownstreamAddr;
struct DNSQuery;
struct StreamData { struct StreamData {
StreamData *dlnext, *dlprev; StreamData *dlnext, *dlprev;
@ -81,6 +82,7 @@ public:
// associated ClientHandlers will be deleted. // associated ClientHandlers will be deleted.
int disconnect(bool hard = false); int disconnect(bool hard = false);
int initiate_connection(); int initiate_connection();
int resolve_name();
void add_downstream_connection(Http2DownstreamConnection *dconn); void add_downstream_connection(Http2DownstreamConnection *dconn);
void remove_downstream_connection(Http2DownstreamConnection *dconn); void remove_downstream_connection(Http2DownstreamConnection *dconn);
@ -203,6 +205,9 @@ public:
// shutdown the connection. // shutdown the connection.
void check_retire(); void check_retire();
// Returns address used to connect to backend. Could be nullptr.
const Address *get_raddr() const;
enum { enum {
// Disconnected // Disconnected
DISCONNECTED, DISCONNECTED,
@ -218,6 +223,8 @@ public:
CONNECTED, CONNECTED,
// Connection is started to fail // Connection is started to fail
CONNECT_FAILING, CONNECT_FAILING,
// Resolving host name
RESOLVING_NAME,
}; };
enum { enum {
@ -259,6 +266,13 @@ private:
// Address of remote endpoint // Address of remote endpoint
DownstreamAddr *addr_; DownstreamAddr *addr_;
nghttp2_session *session_; nghttp2_session *session_;
// Actual remote address used to contact backend. This is initially
// nullptr, and may point to either &addr_->addr,
// resolved_addr_.get(), or HTTP proxy's address structure.
const Address *raddr_;
// Resolved IP address if dns parameter is used
std::unique_ptr<Address> resolved_addr_;
std::unique_ptr<DNSQuery> dns_query_;
int state_; int state_;
int connection_check_state_; int connection_check_state_;
int freelist_zone_; int freelist_zone_;

178
src/shrpx_http_downstream_connection.cc
View File

@ -75,11 +75,12 @@ void connect_timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) {
auto conn = static_cast<Connection *>(w->data); auto conn = static_cast<Connection *>(w->data);
auto dconn = static_cast<HttpDownstreamConnection *>(conn->data); auto dconn = static_cast<HttpDownstreamConnection *>(conn->data);
auto addr = dconn->get_addr(); auto addr = dconn->get_addr();
auto raddr = dconn->get_raddr();
DCLOG(WARN, dconn) << "Connect time out; addr=" DCLOG(WARN, dconn) << "Connect time out; addr="
<< util::to_numeric_addr(&addr->addr); << util::to_numeric_addr(raddr);
downstream_failure(addr); downstream_failure(addr, raddr);
auto downstream = dconn->get_downstream(); auto downstream = dconn->get_downstream();
auto upstream = downstream->get_upstream(); auto upstream = downstream->get_upstream();
@ -182,13 +183,34 @@ HttpDownstreamConnection::~HttpDownstreamConnection() {
if (LOG_ENABLED(INFO)) { if (LOG_ENABLED(INFO)) {
DCLOG(INFO, this) << "Deleted"; DCLOG(INFO, this) << "Deleted";
} }
if (dns_query_) {
auto dns_tracker = worker_->get_dns_tracker();
dns_tracker->cancel(dns_query_.get());
}
} }
int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
int rv;
if (LOG_ENABLED(INFO)) { if (LOG_ENABLED(INFO)) {
DCLOG(INFO, this) << "Attaching to DOWNSTREAM:" << downstream; DCLOG(INFO, this) << "Attaching to DOWNSTREAM:" << downstream;
} }
downstream_ = downstream;
rv = initiate_connection();
if (rv != 0) {
downstream_ = nullptr;
return rv;
}
return 0;
}
int HttpDownstreamConnection::initiate_connection() {
int rv;
auto worker_blocker = worker_->get_connect_blocker(); auto worker_blocker = worker_->get_connect_blocker();
if (worker_blocker->blocked()) { if (worker_blocker->blocked()) {
if (LOG_ENABLED(INFO)) { if (LOG_ENABLED(INFO)) {
@ -212,42 +234,136 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
shared_addr->affinity == AFFINITY_NONE ? shared_addr->next : temp_idx; shared_addr->affinity == AFFINITY_NONE ? shared_addr->next : temp_idx;
auto end = next_downstream; auto end = next_downstream;
for (;;) { for (;;) {
auto &addr = addrs[next_downstream]; auto check_dns_result = dns_query_.get() != nullptr;
if (++next_downstream >= addrs.size()) { DownstreamAddr *addr;
next_downstream = 0; if (check_dns_result) {
} addr = addr_;
addr_ = nullptr;
assert(addr);
assert(addr->dns);
} else {
assert(addr_ == nullptr);
addr = &addrs[next_downstream];
if (addr.proto != PROTO_HTTP1) { if (++next_downstream >= addrs.size()) {
if (end == next_downstream) { next_downstream = 0;
return SHRPX_ERR_NETWORK;
} }
continue; if (addr->proto != PROTO_HTTP1) {
if (end == next_downstream) {
return SHRPX_ERR_NETWORK;
}
continue;
}
} }
auto &connect_blocker = addr.connect_blocker; auto &connect_blocker = addr->connect_blocker;
if (connect_blocker->blocked()) { if (connect_blocker->blocked()) {
if (LOG_ENABLED(INFO)) { if (LOG_ENABLED(INFO)) {
DCLOG(INFO, this) << "Backend server " DCLOG(INFO, this) << "Backend server " << addr->host << ":"
<< util::to_numeric_addr(&addr.addr) << addr->port << " was not available temporarily";
<< " was not available temporarily";
} }
if (end == next_downstream) { if (check_dns_result) {
dns_query_.reset();
} else if (end == next_downstream) {
return SHRPX_ERR_NETWORK; return SHRPX_ERR_NETWORK;
} }
continue; continue;
} }
conn_.fd = util::create_nonblock_socket(addr.addr.su.storage.ss_family); Address *raddr;
if (addr->dns) {
if (!check_dns_result) {
auto dns_query = make_unique<DNSQuery>(
addr->host, [this](int status, const Address *result) {
int rv;
if (status == DNS_STATUS_OK) {
*this->resolved_addr_ = *result;
}
rv = this->initiate_connection();
if (rv != 0) {
// This callback destroys |this|.
auto downstream = this->downstream_;
auto upstream = downstream->get_upstream();
auto handler = upstream->get_client_handler();
downstream->pop_downstream_connection();
auto ndconn = handler->get_downstream_connection(downstream);
if (ndconn) {
if (downstream->attach_downstream_connection(
std::move(ndconn)) == 0) {
return;
}
}
downstream->set_request_state(Downstream::CONNECT_FAIL);
if (upstream->on_downstream_abort_request(downstream, 503) !=
0) {
delete handler;
}
return;
}
});
auto dns_tracker = worker_->get_dns_tracker();
if (!resolved_addr_) {
resolved_addr_ = make_unique<Address>();
}
rv = dns_tracker->resolve(resolved_addr_.get(), dns_query.get());
switch (rv) {
case DNS_STATUS_ERROR:
downstream_failure(addr, nullptr);
if (end == next_downstream) {
return SHRPX_ERR_NETWORK;
}
continue;
case DNS_STATUS_RUNNING:
dns_query_ = std::move(dns_query);
// Remember current addr
addr_ = addr;
return 0;
case DNS_STATUS_OK:
break;
default:
assert(0);
}
} else {
switch (dns_query_->status) {
case DNS_STATUS_ERROR:
dns_query_.reset();
downstream_failure(addr, nullptr);
continue;
case DNS_STATUS_OK:
dns_query_.reset();
break;
default:
assert(0);
}
}
raddr = resolved_addr_.get();
util::set_port(*resolved_addr_, addr->port);
} else {
raddr = &addr->addr;
}
conn_.fd = util::create_nonblock_socket(raddr->su.storage.ss_family);
if (conn_.fd == -1) { if (conn_.fd == -1) {
auto error = errno; auto error = errno;
DCLOG(WARN, this) << "socket() failed; addr=" DCLOG(WARN, this) << "socket() failed; addr="
<< util::to_numeric_addr(&addr.addr) << util::to_numeric_addr(raddr)
<< ", errno=" << error; << ", errno=" << error;
worker_blocker->on_failure(); worker_blocker->on_failure();
@ -257,20 +373,19 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
worker_blocker->on_success(); worker_blocker->on_success();
int rv; rv = connect(conn_.fd, &raddr->su.sa, raddr->len);
rv = connect(conn_.fd, &addr.addr.su.sa, addr.addr.len);
if (rv != 0 && errno != EINPROGRESS) { if (rv != 0 && errno != EINPROGRESS) {
auto error = errno; auto error = errno;
DCLOG(WARN, this) << "connect() failed; addr=" DCLOG(WARN, this) << "connect() failed; addr="
<< util::to_numeric_addr(&addr.addr) << util::to_numeric_addr(raddr)
<< ", errno=" << error; << ", errno=" << error;
downstream_failure(&addr); downstream_failure(addr, raddr);
close(conn_.fd); close(conn_.fd);
conn_.fd = -1; conn_.fd = -1;
if (end == next_downstream) { if (!check_dns_result && end == next_downstream) {
return SHRPX_ERR_NETWORK; return SHRPX_ERR_NETWORK;
} }
@ -282,7 +397,8 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
DCLOG(INFO, this) << "Connecting to downstream server"; DCLOG(INFO, this) << "Connecting to downstream server";
} }
addr_ = &addr; addr_ = addr;
raddr_ = raddr;
if (addr_->tls) { if (addr_->tls) {
assert(ssl_ctx_); assert(ssl_ctx_);
@ -334,8 +450,6 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
ev_set_cb(&conn_.rev, readcb); ev_set_cb(&conn_.rev, readcb);
} }
downstream_ = downstream;
http_parser_init(&response_htp_, HTTP_RESPONSE); http_parser_init(&response_htp_, HTTP_RESPONSE);
response_htp_.data = downstream_; response_htp_.data = downstream_;
@ -1031,7 +1145,7 @@ int HttpDownstreamConnection::tls_handshake() {
} }
if (rv < 0) { if (rv < 0) {
downstream_failure(addr_); downstream_failure(addr_, raddr_);
return rv; return rv;
} }
@ -1041,8 +1155,8 @@ int HttpDownstreamConnection::tls_handshake() {
} }
if (!get_config()->tls.insecure && if (!get_config()->tls.insecure &&
ssl::check_cert(conn_.tls.ssl, addr_) != 0) { ssl::check_cert(conn_.tls.ssl, addr_, raddr_) != 0) {
downstream_failure(addr_); downstream_failure(addr_, raddr_);
return -1; return -1;
} }
@ -1050,7 +1164,7 @@ int HttpDownstreamConnection::tls_handshake() {
if (!SSL_session_reused(conn_.tls.ssl)) { if (!SSL_session_reused(conn_.tls.ssl)) {
auto session = SSL_get0_session(conn_.tls.ssl); auto session = SSL_get0_session(conn_.tls.ssl);
if (session) { if (session) {
ssl::try_cache_tls_session(addr_->tls_session_cache, addr_->addr, session, ssl::try_cache_tls_session(addr_->tls_session_cache, *raddr_, session,
ev_now(conn_.loop)); ev_now(conn_.loop));
} }
} }
@ -1215,10 +1329,10 @@ int HttpDownstreamConnection::connected() {
conn_.wlimit.stopw(); conn_.wlimit.stopw();
DCLOG(WARN, this) << "Backend connect failed; addr=" DCLOG(WARN, this) << "Backend connect failed; addr="
<< util::to_numeric_addr(&addr_->addr) << util::to_numeric_addr(raddr_)
<< ": errno=" << sock_error; << ": errno=" << sock_error;
downstream_failure(addr_); downstream_failure(addr_, raddr_);
return -1; return -1;
} }
@ -1282,4 +1396,6 @@ DownstreamAddr *HttpDownstreamConnection::get_addr() const { return addr_; }
bool HttpDownstreamConnection::poolable() const { return !group_->retired; } bool HttpDownstreamConnection::poolable() const { return !group_->retired; }
const Address *HttpDownstreamConnection::get_raddr() const { return raddr_; }
} // namespace shrpx } // namespace shrpx

13
src/shrpx_http_downstream_connection.h
View File

@ -39,6 +39,7 @@ class DownstreamConnectionPool;
class Worker; class Worker;
struct DownstreamAddrGroup; struct DownstreamAddrGroup;
struct DownstreamAddr; struct DownstreamAddr;
struct DNSQuery;
class HttpDownstreamConnection : public DownstreamConnection { class HttpDownstreamConnection : public DownstreamConnection {
public: public:
@ -68,6 +69,8 @@ public:
get_downstream_addr_group() const; get_downstream_addr_group() const;
virtual DownstreamAddr *get_addr() const; virtual DownstreamAddr *get_addr() const;
int initiate_connection();
int read_clear(); int read_clear();
int write_clear(); int write_clear();
int read_tls(); int read_tls();
@ -80,6 +83,9 @@ public:
void signal_write(); void signal_write();
int actual_signal_write(); int actual_signal_write();
// Returns address used to connect to backend. Could be nullptr.
const Address *get_raddr() const;
int noop(); int noop();
private: private:
@ -92,6 +98,13 @@ private:
std::shared_ptr<DownstreamAddrGroup> group_; std::shared_ptr<DownstreamAddrGroup> group_;
// Address of remote endpoint // Address of remote endpoint
DownstreamAddr *addr_; DownstreamAddr *addr_;
// Actual remote address used to contact backend. This is initially
// nullptr, and may point to either &addr_->addr, or
// resolved_addr_.get().
const Address *raddr_;
// Resolved IP address if dns parameter is used
std::unique_ptr<Address> resolved_addr_;
std::unique_ptr<DNSQuery> dns_query_;
IOControl ioctrl_; IOControl ioctrl_;
http_parser response_htp_; http_parser response_htp_;
ssize_t initial_addr_idx_; ssize_t initial_addr_idx_;

87
src/shrpx_live_check.cc
View File

@ -114,6 +114,7 @@ LiveCheck::LiveCheck(struct ev_loop *loop, SSL_CTX *ssl_ctx, Worker *worker,
ssl_ctx_(ssl_ctx), ssl_ctx_(ssl_ctx),
addr_(addr), addr_(addr),
session_(nullptr), session_(nullptr),
raddr_(nullptr),
success_count_(0), success_count_(0),
fail_count_(0), fail_count_(0),
settings_ack_received_(false), settings_ack_received_(false),
@ -134,6 +135,16 @@ LiveCheck::~LiveCheck() {
} }
void LiveCheck::disconnect() { void LiveCheck::disconnect() {
if (dns_query_) {
auto dns_tracker = worker_->get_dns_tracker();
dns_tracker->cancel(dns_query_.get());
}
dns_query_.reset();
// We can reuse resolved_addr_
raddr_ = nullptr;
conn_.rlimit.stopw(); conn_.rlimit.stopw();
conn_.wlimit.stopw(); conn_.wlimit.stopw();
@ -190,7 +201,7 @@ int LiveCheck::initiate_connection() {
return -1; return -1;
} }
if (addr_->tls) { if (!dns_query_ && addr_->tls) {
assert(ssl_ctx_); assert(ssl_ctx_);
auto ssl = ssl::create_ssl(ssl_ctx_); auto ssl = ssl::create_ssl(ssl_ctx_);
@ -212,20 +223,71 @@ int LiveCheck::initiate_connection() {
conn_.set_ssl(ssl); conn_.set_ssl(ssl);
} }
conn_.fd = util::create_nonblock_socket(addr_->addr.su.storage.ss_family); if (addr_->dns) {
if (!dns_query_) {
auto dns_query = make_unique<DNSQuery>(
addr_->host, [this](int status, const Address *result) {
int rv;
if (status == DNS_STATUS_OK) {
*this->resolved_addr_ = *result;
}
rv = this->initiate_connection();
if (rv != 0) {
this->on_failure();
}
});
auto dns_tracker = worker_->get_dns_tracker();
if (!resolved_addr_) {
resolved_addr_ = make_unique<Address>();
}
rv = dns_tracker->resolve(resolved_addr_.get(), dns_query.get());
switch (rv) {
case DNS_STATUS_ERROR:
return -1;
case DNS_STATUS_RUNNING:
dns_query_ = std::move(dns_query);
return 0;
case DNS_STATUS_OK:
break;
default:
assert(0);
}
} else {
switch (dns_query_->status) {
case DNS_STATUS_ERROR:
dns_query_.reset();
return -1;
case DNS_STATUS_OK:
dns_query_.reset();
break;
default:
assert(0);
}
}
util::set_port(*resolved_addr_, addr_->port);
raddr_ = resolved_addr_.get();
} else {
raddr_ = &addr_->addr;
}
conn_.fd = util::create_nonblock_socket(raddr_->su.storage.ss_family);
if (conn_.fd == -1) { if (conn_.fd == -1) {
auto error = errno; auto error = errno;
LOG(WARN) << "socket() failed; addr=" << util::to_numeric_addr(&addr_->addr) LOG(WARN) << "socket() failed; addr=" << util::to_numeric_addr(raddr_)
<< ", errno=" << error; << ", errno=" << error;
return -1; return -1;
} }
rv = connect(conn_.fd, &addr_->addr.su.sa, addr_->addr.len); rv = connect(conn_.fd, &raddr_->su.sa, raddr_->len);
if (rv != 0 && errno != EINPROGRESS) { if (rv != 0 && errno != EINPROGRESS) {
auto error = errno; auto error = errno;
LOG(WARN) << "connect() failed; addr=" LOG(WARN) << "connect() failed; addr=" << util::to_numeric_addr(raddr_)
<< util::to_numeric_addr(&addr_->addr) << ", errno=" << error; << ", errno=" << error;
close(conn_.fd); close(conn_.fd);
conn_.fd = -1; conn_.fd = -1;
@ -269,8 +331,7 @@ int LiveCheck::connected() {
if (sock_error != 0) { if (sock_error != 0) {
if (LOG_ENABLED(INFO)) { if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Backend connect failed; addr=" LOG(INFO) << "Backend connect failed; addr="
<< util::to_numeric_addr(&addr_->addr) << util::to_numeric_addr(raddr_) << ": errno=" << sock_error;
<< ": errno=" << sock_error;
} }
return -1; return -1;
@ -334,15 +395,15 @@ int LiveCheck::tls_handshake() {
} }
if (!get_config()->tls.insecure && if (!get_config()->tls.insecure &&
ssl::check_cert(conn_.tls.ssl, addr_) != 0) { ssl::check_cert(conn_.tls.ssl, addr_, raddr_) != 0) {
return -1; return -1;
} }
if (!SSL_session_reused(conn_.tls.ssl)) { if (!SSL_session_reused(conn_.tls.ssl)) {
auto tls_session = SSL_get0_session(conn_.tls.ssl); auto tls_session = SSL_get0_session(conn_.tls.ssl);
if (tls_session) { if (tls_session) {
ssl::try_cache_tls_session(addr_->tls_session_cache, addr_->addr, ssl::try_cache_tls_session(addr_->tls_session_cache, *raddr_, tls_session,
tls_session, ev_now(conn_.loop)); ev_now(conn_.loop));
} }
} }
@ -601,7 +662,7 @@ void LiveCheck::on_failure() {
++fail_count_; ++fail_count_;
if (LOG_ENABLED(INFO)) { if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Liveness check for " << util::to_numeric_addr(&addr_->addr) LOG(INFO) << "Liveness check for " << addr_->host << ":" << addr_->port
<< " failed " << fail_count_ << " time(s) in a row"; << " failed " << fail_count_ << " time(s) in a row";
} }
@ -615,7 +676,7 @@ void LiveCheck::on_success() {
fail_count_ = 0; fail_count_ = 0;
if (LOG_ENABLED(INFO)) { if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Liveness check for " << util::to_numeric_addr(&addr_->addr) LOG(INFO) << "Liveness check for " << addr_->host << ":" << addr_->port
<< " succeeded " << success_count_ << " time(s) in a row"; << " succeeded " << success_count_ << " time(s) in a row";
} }

8
src/shrpx_live_check.h
View File

@ -42,6 +42,7 @@ namespace shrpx {
class Worker; class Worker;
struct DownstreamAddr; struct DownstreamAddr;
struct DNSQuery;
class LiveCheck { class LiveCheck {
public: public:
@ -102,6 +103,13 @@ private:
// Address of remote endpoint // Address of remote endpoint
DownstreamAddr *addr_; DownstreamAddr *addr_;
nghttp2_session *session_; nghttp2_session *session_;
// Actual remote address used to contact backend. This is initially
// nullptr, and may point to either &addr_->addr, or
// resolved_addr_.get().
const Address *raddr_;
// Resolved IP address if dns parameter is used
std::unique_ptr<Address> resolved_addr_;
std::unique_ptr<DNSQuery> dns_query_;
// The number of successful connect attempt in a row. // The number of successful connect attempt in a row.
size_t success_count_; size_t success_count_;
// The number of unsuccessful connect attempt in a row. // The number of unsuccessful connect attempt in a row.

4
src/shrpx_ssl.cc
View File

@ -1171,10 +1171,10 @@ int check_cert(SSL *ssl, const Address *addr, const StringRef &host) {
return 0; return 0;
} }
int check_cert(SSL *ssl, const DownstreamAddr *addr) { int check_cert(SSL *ssl, const DownstreamAddr *addr, const Address *raddr) {
auto hostname = auto hostname =
addr->sni.empty() ? StringRef{addr->host} : StringRef{addr->sni}; addr->sni.empty() ? StringRef{addr->host} : StringRef{addr->sni};
return check_cert(ssl, &addr->addr, hostname); return check_cert(ssl, raddr, hostname);
} }
CertLookupTree::CertLookupTree() {} CertLookupTree::CertLookupTree() {}

5
src/shrpx_ssl.h
View File

@ -104,7 +104,10 @@ ClientHandler *accept_connection(Worker *worker, int fd, sockaddr *addr,
// Check peer's certificate against given |address| and |host|. // Check peer's certificate against given |address| and |host|.
int check_cert(SSL *ssl, const Address *addr, const StringRef &host); int check_cert(SSL *ssl, const Address *addr, const StringRef &host);
int check_cert(SSL *ssl, const DownstreamAddr *addr); // Check peer's certificate against given host name described in
// |addr| and numeric address in |raddr|. Note that |raddr| might not
// point to &addr->addr.
int check_cert(SSL *ssl, const DownstreamAddr *addr, const Address *raddr);
struct WildcardRevPrefix { struct WildcardRevPrefix {
WildcardRevPrefix(const StringRef &prefix, size_t idx) WildcardRevPrefix(const StringRef &prefix, size_t idx)

19
src/shrpx_worker.cc
View File

@ -92,7 +92,7 @@ bool match_shared_downstream_addr(
auto &b = rhs->addrs[i]; auto &b = rhs->addrs[i];
if (a.host == b.host && a.port == b.port && a.host_unix == b.host_unix && if (a.host == b.host && a.port == b.port && a.host_unix == b.host_unix &&
a.proto == b.proto && a.tls == b.tls && a.sni == b.sni && a.proto == b.proto && a.tls == b.tls && a.sni == b.sni &&
a.fall == b.fall && a.rise == b.rise) { a.fall == b.fall && a.rise == b.rise && a.dns == b.dns) {
break; break;
} }
} }
@ -120,6 +120,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
std::shared_ptr<DownstreamConfig> downstreamconf) std::shared_ptr<DownstreamConfig> downstreamconf)
: randgen_(rd()), : randgen_(rd()),
worker_stat_{}, worker_stat_{},
dns_tracker_(loop),
loop_(loop), loop_(loop),
sv_ssl_ctx_(sv_ssl_ctx), sv_ssl_ctx_(sv_ssl_ctx),
cl_ssl_ctx_(cl_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx),
@ -209,6 +210,7 @@ void Worker::replace_downstream_config(
dst_addr.sni = make_string_ref(shared_addr->balloc, src_addr.sni); dst_addr.sni = make_string_ref(shared_addr->balloc, src_addr.sni);
dst_addr.fall = src_addr.fall; dst_addr.fall = src_addr.fall;
dst_addr.rise = src_addr.rise; dst_addr.rise = src_addr.rise;
dst_addr.dns = src_addr.dns;
auto shared_addr_ptr = shared_addr.get(); auto shared_addr_ptr = shared_addr.get();
@ -490,6 +492,8 @@ ConnectionHandler *Worker::get_connection_handler() const {
return conn_handler_; return conn_handler_;
} }
DNSTracker *Worker::get_dns_tracker() { return &dns_tracker_; }
namespace { namespace {
size_t match_downstream_addr_group_host( size_t match_downstream_addr_group_host(
const RouterConfig &routerconf, const StringRef &host, const RouterConfig &routerconf, const StringRef &host,
@ -635,7 +639,7 @@ size_t match_downstream_addr_group(
catch_all, balloc); catch_all, balloc);
} }
void downstream_failure(DownstreamAddr *addr) { void downstream_failure(DownstreamAddr *addr, const Address *raddr) {
const auto &connect_blocker = addr->connect_blocker; const auto &connect_blocker = addr->connect_blocker;
if (connect_blocker->in_offline()) { if (connect_blocker->in_offline()) {
@ -651,8 +655,15 @@ void downstream_failure(DownstreamAddr *addr) {
auto fail_count = connect_blocker->get_fail_count(); auto fail_count = connect_blocker->get_fail_count();
if (fail_count >= addr->fall) { if (fail_count >= addr->fall) {
LOG(WARN) << "Could not connect to " << util::to_numeric_addr(&addr->addr) if (raddr) {
<< " " << fail_count << " times in a row; considered as offline"; LOG(WARN) << "Could not connect to " << util::to_numeric_addr(raddr)
<< " " << fail_count
<< " times in a row; considered as offline";
} else {
LOG(WARN) << "Could not connect to " << addr->host << ":" << addr->port
<< " " << fail_count
<< " times in a row; considered as offline";
}
connect_blocker->offline(); connect_blocker->offline();

11
src/shrpx_worker.h
View File

@ -48,6 +48,7 @@
#include "shrpx_ssl.h" #include "shrpx_ssl.h"
#include "shrpx_live_check.h" #include "shrpx_live_check.h"
#include "shrpx_connect_blocker.h" #include "shrpx_connect_blocker.h"
#include "shrpx_dns_tracker.h"
#include "allocator.h" #include "allocator.h"
using namespace nghttp2; using namespace nghttp2;
@ -112,6 +113,8 @@ struct DownstreamAddr {
shrpx_proto proto; shrpx_proto proto;
// true if TLS is used in this backend // true if TLS is used in this backend
bool tls; bool tls;
// true if dynamic DNS is enabled
bool dns;
}; };
// Simplified weighted fair queuing. Actually we don't use queue here // Simplified weighted fair queuing. Actually we don't use queue here
@ -263,6 +266,8 @@ public:
ConnectionHandler *get_connection_handler() const; ConnectionHandler *get_connection_handler() const;
DNSTracker *get_dns_tracker();
private: private:
#ifndef NOTHREADS #ifndef NOTHREADS
std::future<void> fut_; std::future<void> fut_;
@ -275,6 +280,7 @@ private:
ev_timer proc_wev_timer_; ev_timer proc_wev_timer_;
MemchunkPool mcpool_; MemchunkPool mcpool_;
WorkerStat worker_stat_; WorkerStat worker_stat_;
DNSTracker dns_tracker_;
std::shared_ptr<DownstreamConfig> downstreamconf_; std::shared_ptr<DownstreamConfig> downstreamconf_;
std::unique_ptr<MemcachedDispatcher> session_cache_memcached_dispatcher_; std::unique_ptr<MemcachedDispatcher> session_cache_memcached_dispatcher_;
@ -314,7 +320,10 @@ size_t match_downstream_addr_group(
const std::vector<std::shared_ptr<DownstreamAddrGroup>> &groups, const std::vector<std::shared_ptr<DownstreamAddrGroup>> &groups,
size_t catch_all, BlockAllocator &balloc); size_t catch_all, BlockAllocator &balloc);
void downstream_failure(DownstreamAddr *addr); // Calls this function if connecting to backend failed. |raddr| is
// the actual address used to connect to backend, and it could be
// nullptr. This function may schedule live check.
void downstream_failure(DownstreamAddr *addr, const Address *raddr);
} // namespace shrpx } // namespace shrpx

11
src/util.cc
View File

@ -666,6 +666,17 @@ std::string to_numeric_addr(const Address *addr) {
return s; return s;
} }
void set_port(Address &addr, uint16_t port) {
switch (addr.su.storage.ss_family) {
case AF_INET:
addr.su.in.sin_port = htons(port);
break;
case AF_INET6:
addr.su.in6.sin6_port = htons(port);
break;
}
}
static int STDERR_COPY = -1; static int STDERR_COPY = -1;
static int STDOUT_COPY = -1; static int STDOUT_COPY = -1;

3
src/util.h
View File

@ -478,6 +478,9 @@ std::string numeric_name(const struct sockaddr *sa, socklen_t salen);
// IPv6 address, address is enclosed by square brackets ([]). // IPv6 address, address is enclosed by square brackets ([]).
std::string to_numeric_addr(const Address *addr); std::string to_numeric_addr(const Address *addr);
// Sets |port| to |addr|.
void set_port(Address &addr, uint16_t port);
// Makes internal copy of stderr (and possibly stdout in the future), // Makes internal copy of stderr (and possibly stdout in the future),
// which is then used as pointer to /dev/stderr or /proc/self/fd/2 // which is then used as pointer to /dev/stderr or /proc/self/fd/2
void store_original_fds(); void store_original_fds();