From 38b5cad4e346240cb5c34c51c7adf6e59c2faeed Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 4 Dec 2016 23:43:41 +0900 Subject: [PATCH] nghttpx: Lookup backend host name dynamically We have added "dns" parameter to backend option. If specified, name lookup is done dynamically. If not, name lookup is done at start up, or configuration reloading. nghttpx caches DNS result including error case in 30 seconds in this commit. Later commit makes this configurable. DNS resolution is done asynchronously using c-ares library. --- configure.ac | 35 ++- src/Makefile.am | 5 + src/shrpx.cc | 22 +- src/shrpx_config.cc | 45 +++- src/shrpx_config.h | 3 + src/shrpx_dns_resolver.cc | 334 ++++++++++++++++++++++++ src/shrpx_dns_resolver.h | 116 ++++++++ src/shrpx_dns_tracker.cc | 262 +++++++++++++++++++ src/shrpx_dns_tracker.h | 108 ++++++++ src/shrpx_dual_dns_resolver.cc | 87 ++++++ src/shrpx_dual_dns_resolver.h | 63 +++++ src/shrpx_http2_session.cc | 206 +++++++++++---- src/shrpx_http2_session.h | 14 + src/shrpx_http_downstream_connection.cc | 178 ++++++++++--- src/shrpx_http_downstream_connection.h | 13 + src/shrpx_live_check.cc | 87 +++++- src/shrpx_live_check.h | 8 + src/shrpx_ssl.cc | 4 +- src/shrpx_ssl.h | 5 +- src/shrpx_worker.cc | 19 +- src/shrpx_worker.h | 11 +- src/util.cc | 11 + src/util.h | 3 + 23 files changed, 1514 insertions(+), 125 deletions(-) create mode 100644 src/shrpx_dns_resolver.cc create mode 100644 src/shrpx_dns_resolver.h create mode 100644 src/shrpx_dns_tracker.cc create mode 100644 src/shrpx_dns_tracker.h create mode 100644 src/shrpx_dual_dns_resolver.cc create mode 100644 src/shrpx_dual_dns_resolver.h diff --git a/configure.ac b/configure.ac index 35743647..e18efbef 100644 --- a/configure.ac +++ b/configure.ac @@ -370,6 +370,13 @@ if test "x${have_openssl}" = "xno"; then AC_MSG_NOTICE($OPENSSL_PKG_ERRORS) fi +# c-ares (for src) +PKG_CHECK_MODULES([LIBCARES], [libcares >= 1.11.0], [have_libcares=yes], + [have_libcares=no]) +if test "x${have_libcares}" = "xno"; then + AC_MSG_NOTICE($LIBCARES_PKG_ERRORS) +fi + # libevent_openssl (for examples) # 2.0.8 is required because we use evconnlistener_set_error_cb() PKG_CHECK_MODULES([LIBEVENT_OPENSSL], [libevent_openssl >= 2.0.8], @@ -479,13 +486,14 @@ if test "x${request_asio_lib}" = "xyes"; then fi fi -# The nghttp, nghttpd and nghttpx under src depend on zlib, OpenSSL -# and libev +# The nghttp, nghttpd and nghttpx under src depend on zlib, OpenSSL, +# libev, and libc-ares. enable_app=no if test "x${request_app}" != "xno" && test "x${have_zlib}" = "xyes" && test "x${have_openssl}" = "xyes" && - test "x${have_libev}" = "xyes"; then + test "x${have_libev}" = "xyes" && + test "x${have_libcares}" = "xyes"; then enable_app=yes fi @@ -640,6 +648,26 @@ AC_SYS_LARGEFILE AC_CHECK_MEMBER([struct tm.tm_gmtoff], [have_struct_tm_tm_gmtoff=yes], [have_struct_tm_tm_gmtoff=no], [[#include ]]) +AC_CHECK_MEMBER([struct sockaddr_in.sin_len], + [AC_DEFINE([HAVE_SOCKADDR_IN_SIN_LEN],[1], + [Define to 1 if struct sockaddr_in has sin_len member.])], + [], + [[ +#include +#include +#include +]]) + +AC_CHECK_MEMBER([struct sockaddr_in6.sin6_len], + [AC_DEFINE([HAVE_SOCKADDR_IN6_SIN6_LEN],[1], + [Define to 1 if struct sockaddr_in6 has sin6_len member.])], + [], + [[ +#include +#include +#include +]]) + if test "x$have_struct_tm_tm_gmtoff" = "xyes"; then AC_DEFINE([HAVE_STRUCT_TM_TM_GMTOFF], [1], [Define to 1 if you have `struct tm.tm_gmtoff` member.]) @@ -890,6 +918,7 @@ AC_MSG_NOTICE([summary of build options: OpenSSL: ${have_openssl} (CFLAGS='${OPENSSL_CFLAGS}' LIBS='${OPENSSL_LIBS}') Libxml2: ${have_libxml2} (CFLAGS='${XML_CPPFLAGS}' LIBS='${XML_LIBS}') Libev: ${have_libev} (CFLAGS='${LIBEV_CFLAGS}' LIBS='${LIBEV_LIBS}') + Libc-ares ${have_libcares} (CFLAGS='${LIBCARES_CFLAGS}' LIBS='${LIBCARES_LIBS}') Libevent(SSL): ${have_libevent_openssl} (CFLAGS='${LIBEVENT_OPENSSL_CFLAGS}' LIBS='${LIBEVENT_OPENSSL_LIBS}') Spdylay: ${have_spdylay} (CFLAGS='${LIBSPDYLAY_CFLAGS}' LIBS='${LIBSPDYLAY_LIBS}') Jansson: ${have_jansson} (CFLAGS='${JANSSON_CFLAGS}' LIBS='${JANSSON_LIBS}') diff --git a/src/Makefile.am b/src/Makefile.am index ceaa04a8..6f60faed 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -44,6 +44,7 @@ AM_CPPFLAGS = \ @XML_CPPFLAGS@ \ @LIBEV_CFLAGS@ \ @OPENSSL_CFLAGS@ \ + @LIBCARES_CFLAGS@ \ @JANSSON_CFLAGS@ \ @ZLIB_CFLAGS@ \ @DEFS@ @@ -55,6 +56,7 @@ LDADD = $(top_builddir)/lib/libnghttp2.la \ @XML_LIBS@ \ @LIBEV_LIBS@ \ @OPENSSL_LIBS@ \ + @LIBCARES_LIBS@ \ @JANSSON_LIBS@ \ @ZLIB_LIBS@ \ @APPLDFLAGS@ @@ -139,6 +141,9 @@ NGHTTPX_SRCS = \ shrpx_health_monitor_downstream_connection.cc \ shrpx_health_monitor_downstream_connection.h \ shrpx_exec.cc shrpx_exec.h \ + shrpx_dns_resolver.cc shrpx_dns_resolver.h \ + shrpx_dual_dns_resolver.cc shrpx_dual_dns_resolver.h \ + shrpx_dns_tracker.cc shrpx_dns_tracker.h \ buffer.h memchunk.h template.h allocator.h \ xsi_strerror.c xsi_strerror.h diff --git a/src/shrpx.cc b/src/shrpx.cc index 8e1a7273..4f6f28a3 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1587,13 +1587,13 @@ Connections: Several parameters are accepted after . The parameters are delimited by ";". The available parameters are: "proto=", "tls", - "sni=", "fall=", "rise=", and - "affinity=". The parameter consists of keyword, - and optionally followed by "=" and value. For example, - the parameter "proto=h2" consists of the keyword "proto" - and value "h2". The parameter "tls" consists of the - keyword "tls" without value. Each parameter is - described as follows. + "sni=", "fall=", "rise=", + "affinity=", and "dns". The parameter consists + of keyword, and optionally followed by "=" and value. + For example, the parameter "proto=h2" consists of the + keyword "proto" and value "h2". The parameter "tls" + consists of the keyword "tls" without value. Each + parameter is described as follows. The backend application protocol can be specified using optional "proto" parameter, and in the form of @@ -1642,6 +1642,14 @@ Connections: break if one of the backend gets unreachable, or backend settings are reloaded or replaced by API. + By default, name resolution of backend host name is done + at start up, or reloading configuration. If "dns" + parameter is given, name resolution takes place + dynamically. This is useful if backend address changes + frequently. If "dns" is given, name resolution of + backend host name at start up, or reloading + configuration is skipped. + Since ";" and ":" are used as delimiter, must not contain these characters. Since ";" has special meaning in shell, the option value must be quoted. diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 94cb1227..b4b1b8ad 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -728,6 +728,7 @@ struct DownstreamParams { shrpx_proto proto; shrpx_session_affinity affinity; bool tls; + bool dns; }; namespace { @@ -801,6 +802,8 @@ int parse_downstream_params(DownstreamParams &out, LOG(ERROR) << "backend: affinity: value must be either none or ip"; return -1; } + } else if (util::strieq_l("dns", param)) { + out.dns = true; } else if (!param.empty()) { LOG(ERROR) << "backend: " << param << ": unknown keyword"; return -1; @@ -841,11 +844,17 @@ int parse_mapping(Config *config, DownstreamAddrConfig &addr, return -1; } + if (addr.host_unix && params.dns) { + LOG(ERROR) << "backend: dns: cannot be used for UNIX domain socket"; + return -1; + } + addr.fall = params.fall; addr.rise = params.rise; addr.proto = params.proto; addr.tls = params.tls; addr.sni = make_string_ref(downstreamconf.balloc, params.sni); + addr.dns = params.dns; auto &routerconf = downstreamconf.router; auto &router = routerconf.router; @@ -3442,24 +3451,38 @@ int configure_downstream_group(Config *config, bool http2_proxy, auto hostport = util::make_hostport(downstreamconf.balloc, addr.host, addr.port); - if (resolve_hostname(&addr.addr, addr.host.c_str(), addr.port, - downstreamconf.family, resolve_flags) == -1) { - LOG(FATAL) << "Resolving backend address failed: " << hostport; - return -1; - } + if (!addr.dns) { + if (resolve_hostname(&addr.addr, addr.host.c_str(), addr.port, + downstreamconf.family, resolve_flags) == -1) { + LOG(FATAL) << "Resolving backend address failed: " << hostport; + return -1; + } - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Resolved backend address: " << hostport << " -> " - << util::to_numeric_addr(&addr.addr); + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Resolved backend address: " << hostport << " -> " + << util::to_numeric_addr(&addr.addr); + } + } else { + LOG(INFO) << "Resolving backend address " << hostport + << " takes place dynamically"; } } if (g.affinity == AFFINITY_IP) { size_t idx = 0; for (auto &addr : g.addrs) { - auto p = reinterpret_cast(&addr.addr.su); - rv = compute_affinity_hash(g.affinity_hash, idx, - StringRef{p, addr.addr.len}); + StringRef key; + if (addr.dns) { + if (addr.host_unix) { + key = addr.host; + } else { + key = addr.hostport; + } + } else { + auto p = reinterpret_cast(&addr.addr.su); + key = StringRef{p, addr.addr.len}; + } + rv = compute_affinity_hash(g.affinity_hash, idx, key); if (rv != 0) { return -1; } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 9020ef83..d6ed19d6 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -379,6 +379,7 @@ struct UpstreamAddr { }; struct DownstreamAddrConfig { + // Resolved address if |dns| is false Address addr; // backend address. If |host_unix| is true, this is UNIX domain // socket path. This must be NULL terminated string. @@ -397,6 +398,8 @@ struct DownstreamAddrConfig { // true if |host| contains UNIX domain socket path. bool host_unix; bool tls; + // true if dynamic DNS is enabled + bool dns; }; // Mapping hash to idx which is an index into diff --git a/src/shrpx_dns_resolver.cc b/src/shrpx_dns_resolver.cc new file mode 100644 index 00000000..a72b3364 --- /dev/null +++ b/src/shrpx_dns_resolver.cc @@ -0,0 +1,334 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "shrpx_dns_resolver.h" + +#include +#include + +#include "shrpx_log.h" +#include "shrpx_connection.h" + +namespace shrpx { + +namespace { +void sock_state_cb(void *data, int s, int read, int write) { + auto resolv = static_cast(data); + + if (resolv->get_status(nullptr) != DNS_STATUS_RUNNING) { + return; + } + + if (read) { + resolv->start_rev(s); + } else { + resolv->stop_rev(s); + } + if (write) { + resolv->start_wev(s); + } else { + resolv->stop_wev(s); + } +} +} // namespace + +namespace { +void host_cb(void *arg, int status, int timeouts, hostent *hostent) { + auto resolv = static_cast(arg); + resolv->on_result(status, hostent); +} +} // namespace + +namespace { +void process_result(DNSResolver *resolv) { + auto cb = resolv->get_complete_cb(); + if (!cb) { + return; + } + Address result; + auto status = resolv->get_status(&result); + switch (status) { + case DNS_STATUS_OK: + case DNS_STATUS_ERROR: + cb(status, &result); + break; + } + // resolv may be deleted here. +} +} // namespace + +namespace { +void readcb(struct ev_loop *loop, ev_io *w, int revents) { + auto resolv = static_cast(w->data); + resolv->on_read(w->fd); + process_result(resolv); +} +} // namespace + +namespace { +void writecb(struct ev_loop *loop, ev_io *w, int revents) { + auto resolv = static_cast(w->data); + resolv->on_write(w->fd); + process_result(resolv); +} +} // namespace + +namespace { +void timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) { + auto resolv = static_cast(w->data); + resolv->on_timeout(); +} +} // namespace + +namespace { +void stop_ev(struct ev_loop *loop, + const std::vector> &evs) { + for (auto &w : evs) { + ev_io_stop(loop, w.get()); + } +} +} // namespace + +DNSResolver::DNSResolver(struct ev_loop *loop) + : loop_(loop), + channel_(nullptr), + family_(AF_UNSPEC), + status_(DNS_STATUS_IDLE) { + ev_timer_init(&timer_, timeoutcb, 0., 0.); + timer_.data = this; +} + +DNSResolver::~DNSResolver() { + if (channel_) { + ares_destroy(channel_); + } + + stop_ev(loop_, revs_); + stop_ev(loop_, wevs_); + + ev_timer_stop(loop_, &timer_); +} + +int DNSResolver::resolve(const StringRef &name, int family) { + if (status_ != DNS_STATUS_IDLE) { + return -1; + } + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Start resolving host " << name << " in IPv" + << (family == AF_INET ? "4" : "6"); + } + + name_ = name; + family_ = family; + + int rv; + + ares_options opts{}; + opts.sock_state_cb = sock_state_cb; + opts.sock_state_cb_data = this; + + auto optmask = ARES_OPT_SOCK_STATE_CB; + + ares_channel chan; + rv = ares_init_options(&chan, &opts, optmask); + if (rv != ARES_SUCCESS) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "ares_init_options failed: " << ares_strerror(rv); + } + status_ = DNS_STATUS_ERROR; + return -1; + } + + channel_ = chan; + status_ = DNS_STATUS_RUNNING; + + ares_gethostbyname(channel_, name_.c_str(), family_, host_cb, this); + reset_timeout(); + + return 0; +} + +int DNSResolver::on_read(int fd) { return handle_event(fd, ARES_SOCKET_BAD); } + +int DNSResolver::on_write(int fd) { return handle_event(ARES_SOCKET_BAD, fd); } + +int DNSResolver::on_timeout() { + return handle_event(ARES_SOCKET_BAD, ARES_SOCKET_BAD); +} + +int DNSResolver::handle_event(int rfd, int wfd) { + if (status_ == DNS_STATUS_IDLE) { + return -1; + } + + ares_process_fd(channel_, rfd, wfd); + + switch (status_) { + case DNS_STATUS_RUNNING: { + reset_timeout(); + return 0; + } + case DNS_STATUS_OK: + return 0; + case DNS_STATUS_ERROR: + return -1; + default: + // Unreachable + assert(0); + } +} + +void DNSResolver::reset_timeout() { + if (status_ != DNS_STATUS_RUNNING) { + return; + } + timeval tvout; + auto tv = ares_timeout(channel_, nullptr, &tvout); + if (tv == nullptr) { + return; + } + timer_.repeat = tv->tv_sec + tv->tv_usec / 1000000.; + ev_timer_again(loop_, &timer_); +} + +int DNSResolver::get_status(Address *result) const { + if (status_ != DNS_STATUS_OK) { + return status_; + } + + if (result) { + memcpy(result, &result_, sizeof(result_)); + } + + return status_; +} + +namespace { +void start_ev(std::vector> &evs, struct ev_loop *loop, + int fd, int event, IOCb cb, void *data) { + for (auto &w : evs) { + if (w->fd == fd) { + return; + } + } + for (auto &w : evs) { + if (w->fd == -1) { + ev_io_set(w.get(), fd, event); + ev_io_start(loop, w.get()); + return; + } + } + + auto w = make_unique(); + ev_io_init(w.get(), cb, fd, event); + w->data = data; + ev_io_start(loop, w.get()); + evs.emplace_back(std::move(w)); +} +} // namespace + +namespace { +void stop_ev(std::vector> &evs, struct ev_loop *loop, + int fd, int event) { + for (auto &w : evs) { + if (w->fd == fd) { + ev_io_stop(loop, w.get()); + ev_io_set(w.get(), -1, event); + return; + } + } +} +} // namespace + +void DNSResolver::start_rev(int fd) { + start_ev(revs_, loop_, fd, EV_READ, readcb, this); +} + +void DNSResolver::stop_rev(int fd) { stop_ev(revs_, loop_, fd, EV_READ); } + +void DNSResolver::start_wev(int fd) { + start_ev(wevs_, loop_, fd, EV_WRITE, writecb, this); +} + +void DNSResolver::stop_wev(int fd) { stop_ev(wevs_, loop_, fd, EV_WRITE); } + +void DNSResolver::on_result(int status, hostent *hostent) { + stop_ev(loop_, revs_); + stop_ev(loop_, wevs_); + ev_timer_stop(loop_, &timer_); + + if (status != ARES_SUCCESS) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Address lookup for " << name_ + << " failed: " << ares_strerror(status); + } + status_ = DNS_STATUS_ERROR; + return; + } + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Address lookup for " << name_ << " succeeded"; + } + + status_ = DNS_STATUS_OK; + + switch (hostent->h_addrtype) { + case AF_INET: + for (auto ap = hostent->h_addr_list; *ap; ++ap) { + result_.len = sizeof(result_.su.in); + result_.su.in = {}; + result_.su.in.sin_family = AF_INET; +#ifdef HAVE_SOCKADDR_IN_SIN_LEN + result_.su.in.sin_len = sizeof(result_.su.in); +#endif // HAVE_SOCKADDR_IN_SIN_LEN + memcpy(&result_.su.in.sin_addr, *ap, sizeof(result_.su.in.sin_addr)); + return; + } + break; + case AF_INET6: + for (auto ap = hostent->h_addr_list; *ap; ++ap) { + result_.len = sizeof(result_.su.in6); + result_.su.in6 = {}; + result_.su.in6.sin6_family = AF_INET6; +#ifdef HAVE_SOCKADDR_IN6_SIN6_LEN + result_.su.in6.sin6_len = sizeof(result_.su.in6); +#endif // HAVE_SOCKADDR_IN6_SIN6_LEN + memcpy(&result_.su.in6.sin6_addr, *ap, sizeof(result_.su.in6.sin6_addr)); + return; + } + break; + } + + // Somehow we got unsupported address family + status_ = DNS_STATUS_ERROR; +} + +void DNSResolver::set_complete_cb(CompleteCb cb) { + completeCb_ = std::move(cb); +} + +CompleteCb DNSResolver::get_complete_cb() const { return completeCb_; } + +} // namespace shrpx diff --git a/src/shrpx_dns_resolver.h b/src/shrpx_dns_resolver.h new file mode 100644 index 00000000..268547e3 --- /dev/null +++ b/src/shrpx_dns_resolver.h @@ -0,0 +1,116 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef SHRPX_DNS_RESOLVER_H +#define SHRPX_DNS_RESOLVER_H + +#include "shrpx.h" + +#include +#include + +#include + +#include +#include + +#include "template.h" +#include "network.h" + +using namespace nghttp2; + +namespace shrpx { + +enum DNSResolverStatus { + // Resolver is in initial status + DNS_STATUS_IDLE, + // Resolver is currently resolving host name + DNS_STATUS_RUNNING, + // Resolver successfully resolved host name + DNS_STATUS_OK, + // Resolver failed to resolve host name + DNS_STATUS_ERROR, +}; + +// Callback function called when host name lookup is finished. +// |status| is either DNS_STATUS_OK, or DNS_STATUS_ERROR. If |status| +// is DNS_STATUS_OK, |result| points to the resolved address. Note +// that port portion of |result| is undefined, and must be initialized +// by application. This callback function is not called if name +// lookup finishes in DNSResolver::resolve() completely. In this +// case, application should call DNSResolver::get_status() to get +// current status and result. In other words, callback is called if +// get_status() returns DNS_STATUS_RUNNING. +using CompleteCb = std::function; + +// DNSResolver is asynchronous name resolver, backed by c-ares +// library. +class DNSResolver { +public: + DNSResolver(struct ev_loop *loop); + ~DNSResolver(); + + // Starts resolving hostname |name|. + int resolve(const StringRef &name, int family); + // Returns status. If status_ is DNS_STATUS_SUCCESS && |result| is + // not nullptr, |*result| is filled. + int get_status(Address *result) const; + // Sets callback function when name lookup finishes. The callback + // function is called in a way that it can destroy this DNSResolver. + void set_complete_cb(CompleteCb cb); + CompleteCb get_complete_cb() const; + + // Calls these functions when read/write event occurred respectively. + int on_read(int fd); + int on_write(int fd); + int on_timeout(); + // Calls this function when DNS query finished. + void on_result(int staus, hostent *hostent); + void reset_timeout(); + + void start_rev(int fd); + void stop_rev(int fd); + void start_wev(int fd); + void stop_wev(int fd); + +private: + int handle_event(int rfd, int wfd); + + std::vector> revs_, wevs_; + Address result_; + CompleteCb completeCb_; + ev_timer timer_; + StringRef name_; + struct ev_loop *loop_; + // ares_channel is pointer type + ares_channel channel_; + // AF_INET or AF_INET6. AF_INET for A record lookup, and AF_INET6 + // for AAAA record lookup. + int family_; + int status_; +}; + +} // namespace shrpx + +#endif // SHRPX_DNS_RESOLVER_H diff --git a/src/shrpx_dns_tracker.cc b/src/shrpx_dns_tracker.cc new file mode 100644 index 00000000..3acb100a --- /dev/null +++ b/src/shrpx_dns_tracker.cc @@ -0,0 +1,262 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "shrpx_dns_tracker.h" +#include "util.h" + +namespace shrpx { + +DNSTracker::DNSTracker(struct ev_loop *loop) : loop_(loop) {} + +DNSTracker::~DNSTracker() { + for (auto &p : ents_) { + auto &qlist = p.second.qlist; + while (!qlist.empty()) { + auto head = qlist.head; + qlist.remove(head); + head->status = DNS_STATUS_ERROR; + head->in_qlist = false; + // TODO Not sure we should call callback here, or it is even be + // safe to do that. + } + } +} + +namespace { +constexpr auto DNS_TTL = 30_s; +} // namespace + +ResolverEntry DNSTracker::make_entry(std::unique_ptr resolv, + ImmutableString host, int status, + const Address *result) { + auto ent = ResolverEntry{}; + ent.resolv = std::move(resolv); + ent.host = std::move(host); + ent.status = status; + ent.expiry = ev_now(loop_) + DNS_TTL; + if (result) { + ent.result = *result; + } + return ent; +} + +void DNSTracker::update_entry(ResolverEntry &ent, + std::unique_ptr resolv, + int status, const Address *result) { + ent.resolv = std::move(resolv); + ent.status = status; + if (result) { + ent.result = *result; + } +} + +int DNSTracker::resolve(Address *result, DNSQuery *dnsq) { + int rv; + + auto it = ents_.find(dnsq->host); + + if (it == std::end(ents_)) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "DNS entry not found for " << dnsq->host; + } + + auto resolv = make_unique(loop_); + auto host_copy = + ImmutableString{std::begin(dnsq->host), std::end(dnsq->host)}; + auto host = StringRef{host_copy}; + + rv = resolv->resolve(host); + if (rv != 0) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Name lookup failed for " << host; + } + + ents_.emplace(host, make_entry(nullptr, std::move(host_copy), + DNS_STATUS_ERROR, nullptr)); + + return DNS_STATUS_ERROR; + } + + rv = resolv->get_status(result); + switch (rv) { + case DNS_STATUS_ERROR: { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Name lookup failed for " << host; + } + + ents_.emplace(host, make_entry(nullptr, std::move(host_copy), + DNS_STATUS_ERROR, nullptr)); + + return DNS_STATUS_ERROR; + } + case DNS_STATUS_OK: { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Name lookup succeeded: " << host << " -> " + << util::numeric_name(&result->su.sa, result->len); + } + + ents_.emplace(host, make_entry(nullptr, std::move(host_copy), + DNS_STATUS_OK, result)); + + return DNS_STATUS_OK; + } + case DNS_STATUS_RUNNING: { + assert(rv == DNS_STATUS_RUNNING); + + auto p = ents_.emplace(host, + make_entry(std::move(resolv), std::move(host_copy), + DNS_STATUS_RUNNING, nullptr)); + + auto &ent = (*p.first).second; + + add_to_qlist(ent, dnsq); + + return DNS_STATUS_RUNNING; + } + default: + assert(0); + } + } + + auto &ent = (*it).second; + + if (ent.status != DNS_STATUS_RUNNING && ent.expiry < ev_now(loop_)) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "DNS entry found for " << dnsq->host + << ", but it has been expired"; + } + + auto resolv = make_unique(loop_); + auto host = StringRef{ent.host}; + + rv = resolv->resolve(host); + if (rv != 0) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Name lookup failed for " << host; + } + + update_entry(ent, nullptr, DNS_STATUS_ERROR, nullptr); + + return DNS_STATUS_ERROR; + } + + rv = resolv->get_status(result); + switch (rv) { + case DNS_STATUS_ERROR: { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Name lookup failed for " << host; + } + + update_entry(ent, nullptr, DNS_STATUS_ERROR, nullptr); + + return DNS_STATUS_ERROR; + } + case DNS_STATUS_OK: { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Name lookup succeeded: " << host << " -> " + << util::numeric_name(&result->su.sa, result->len); + } + + update_entry(ent, nullptr, DNS_STATUS_OK, result); + + return DNS_STATUS_OK; + } + case DNS_STATUS_RUNNING: { + update_entry(ent, std::move(resolv), DNS_STATUS_RUNNING, nullptr); + add_to_qlist(ent, dnsq); + + return DNS_STATUS_RUNNING; + } + default: + assert(0); + } + } + + switch (ent.status) { + case DNS_STATUS_RUNNING: + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Waiting for name lookup complete for " << dnsq->host; + } + ent.qlist.append(dnsq); + dnsq->in_qlist = true; + return DNS_STATUS_RUNNING; + case DNS_STATUS_ERROR: + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Name lookup failed for " << dnsq->host << " (cached)"; + } + return DNS_STATUS_ERROR; + case DNS_STATUS_OK: + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Name lookup succeeded (cached): " << dnsq->host << " -> " + << util::numeric_name(&ent.result.su.sa, ent.result.len); + } + if (result) { + memcpy(result, &ent.result, sizeof(*result)); + } + return DNS_STATUS_OK; + default: + assert(0); + } +} + +void DNSTracker::add_to_qlist(ResolverEntry &ent, DNSQuery *dnsq) { + auto loop = loop_; + ent.resolv->set_complete_cb([&ent, loop](int status, const Address *result) { + auto &qlist = ent.qlist; + while (!qlist.empty()) { + auto head = qlist.head; + qlist.remove(head); + head->status = status; + head->in_qlist = false; + auto cb = head->cb; + cb(status, result); + } + + ent.resolv.reset(); + ent.status = status; + ent.expiry = ev_now(loop) + DNS_TTL; + if (ent.status == DNS_STATUS_OK) { + ent.result = *result; + } + }); + ent.qlist.append(dnsq); + dnsq->in_qlist = true; +} + +void DNSTracker::cancel(DNSQuery *dnsq) { + if (!dnsq->in_qlist) { + return; + } + + auto it = ents_.find(dnsq->host); + if (it == std::end(ents_)) { + return; + } + + auto &ent = (*it).second; + ent.qlist.remove(dnsq); + dnsq->in_qlist = false; +} + +} // namespace shrpx diff --git a/src/shrpx_dns_tracker.h b/src/shrpx_dns_tracker.h new file mode 100644 index 00000000..455026ec --- /dev/null +++ b/src/shrpx_dns_tracker.h @@ -0,0 +1,108 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef SHRPX_DNS_TRACKER_H +#define SHRPX_DNS_TRACKER_H + +#include "shrpx.h" + +#include + +#include "shrpx_dual_dns_resolver.h" + +using namespace nghttp2; + +namespace shrpx { + +struct DNSQuery { + DNSQuery(StringRef host, CompleteCb cb) + : host(std::move(host)), + cb(std::move(cb)), + dlnext(nullptr), + dlprev(nullptr), + status(DNS_STATUS_IDLE), + in_qlist(false) {} + + // Host name we lookup for. + StringRef host; + // Callback function called when name lookup finished. This + // callback is not called if name lookup finishes within + // DNSTracker::resolve(). + CompleteCb cb; + DNSQuery *dlnext, *dlprev; + int status; + // true if this object is in linked list ResolverEntry::qlist. + bool in_qlist; +}; + +struct ResolverEntry { + // Host name this entry lookups for. + ImmutableString host; + // DNS resolver. Only non-nullptr if status is DNS_STATUS_RUNNING. + std::unique_ptr resolv; + // DNSQuery interested in this name lookup result. The result is + // notified to them all. + DList qlist; + // Use the same enum with DNSResolverStatus + int status; + // result and its expiry time + Address result; + // time point when cached result expires. + ev_tstamp expiry; +}; + +class DNSTracker { +public: + DNSTracker(struct ev_loop *loop); + ~DNSTracker(); + + // Lookups host name described in |dnsq|. If name lookup finishes + // within this function (either it came from /etc/hosts, host name + // is numeric, lookup result is cached, etc), it returns + // DNS_STATUS_OK or DNS_STATUS_ERROR. If lookup is successful, + // DNS_STATUS_OK is returned, and |result| is filled. If lookup + // failed, DNS_STATUS_ERROR is returned. If name lookup is being + // done background, it returns DNS_STATUS_RUNNING. Its completion + // is notified by calling dnsq->cb. + int resolve(Address *result, DNSQuery *dnsq); + // Cancels name lookup requested by |dnsq|. + void cancel(DNSQuery *dnsq); + +private: + ResolverEntry make_entry(std::unique_ptr resolv, + ImmutableString host, int status, + const Address *result); + + void update_entry(ResolverEntry &ent, std::unique_ptr resolv, + int status, const Address *result); + + void add_to_qlist(ResolverEntry &ent, DNSQuery *dnsq); + + std::map ents_; + struct ev_loop *loop_; +}; + +} // namespace shrpx + +#endif // SHRPX_DNS_TRACKER_H diff --git a/src/shrpx_dual_dns_resolver.cc b/src/shrpx_dual_dns_resolver.cc new file mode 100644 index 00000000..82145628 --- /dev/null +++ b/src/shrpx_dual_dns_resolver.cc @@ -0,0 +1,87 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "shrpx_dual_dns_resolver.h" + +namespace shrpx { + +DualDNSResolver::DualDNSResolver(struct ev_loop *loop) + : resolv4_(loop), resolv6_(loop) { + auto cb = [this](int, const Address *) { + int rv; + Address result; + + rv = this->get_status(&result); + switch (rv) { + case DNS_STATUS_ERROR: + case DNS_STATUS_OK: + break; + default: + return; + } + + auto cb = this->get_complete_cb(); + cb(rv, &result); + }; + + resolv4_.set_complete_cb(cb); + resolv6_.set_complete_cb(cb); +} + +int DualDNSResolver::resolve(const StringRef &host) { + int rv4, rv6; + rv4 = resolv4_.resolve(host, AF_INET); + rv6 = resolv6_.resolve(host, AF_INET6); + + if (rv4 != 0 && rv6 != 0) { + return -1; + } + + return 0; +} + +CompleteCb DualDNSResolver::get_complete_cb() const { return complete_cb_; } + +void DualDNSResolver::set_complete_cb(CompleteCb cb) { complete_cb_ = cb; } + +int DualDNSResolver::get_status(Address *result) const { + int rv4, rv6; + rv6 = resolv6_.get_status(result); + if (rv6 == DNS_STATUS_OK) { + return DNS_STATUS_OK; + } + rv4 = resolv4_.get_status(result); + if (rv4 == DNS_STATUS_OK) { + return DNS_STATUS_OK; + } + if (rv4 == DNS_STATUS_RUNNING || rv6 == DNS_STATUS_RUNNING) { + return DNS_STATUS_RUNNING; + } + if (rv4 == DNS_STATUS_ERROR && rv6 == DNS_STATUS_ERROR) { + return DNS_STATUS_ERROR; + } + return DNS_STATUS_IDLE; +} + +} // namespace shrpx diff --git a/src/shrpx_dual_dns_resolver.h b/src/shrpx_dual_dns_resolver.h new file mode 100644 index 00000000..a8e08264 --- /dev/null +++ b/src/shrpx_dual_dns_resolver.h @@ -0,0 +1,63 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef SHRPX_DUAL_DNS_RESOLVER_H +#define SHRPX_DUAL_DNS_RESOLVER_H + +#include "shrpx.h" + +#include + +#include "shrpx_dns_resolver.h" + +using namespace nghttp2; + +namespace shrpx { + +// DualDNSResolver performs name resolution for both A and AAAA +// records at the same time. The first successful return (or if we +// have both successful results, prefer to AAAA) is chosen. This is +// wrapper around 2 DNSResolver inside. resolve(), get_status(), and +// how CompleteCb is called have the same semantics with DNSResolver. +class DualDNSResolver { +public: + DualDNSResolver(struct ev_loop *loop); + + // Resolves |host|. |host| must be NULL-terminated string. + int resolve(const StringRef &host); + CompleteCb get_complete_cb() const; + void set_complete_cb(CompleteCb cb); + int get_status(Address *result) const; + +private: + // For A record + DNSResolver resolv4_; + // For AAAA record + DNSResolver resolv6_; + CompleteCb complete_cb_; +}; + +} // namespace shrpx + +#endif // SHRPX_DUAL_DNS_RESOLVER_H diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 111e5290..e2b4608c 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -95,7 +95,7 @@ void settings_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents) { SSLOG(INFO, http2session) << "SETTINGS timeout"; } - downstream_failure(http2session->get_addr()); + downstream_failure(http2session->get_addr(), http2session->get_raddr()); if (http2session->terminate_session(NGHTTP2_SETTINGS_TIMEOUT) != 0) { delete http2session; @@ -202,6 +202,7 @@ Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, group_(group), addr_(addr), session_(nullptr), + raddr_(nullptr), state_(DISCONNECTED), connection_check_state_(CONNECTION_CHECK_NONE), freelist_zone_(FREELIST_ZONE_NONE) { @@ -244,6 +245,11 @@ int Http2Session::disconnect(bool hard) { wb_.reset(); + if (dns_query_) { + auto dns_tracker = worker_->get_dns_tracker(); + dns_tracker->cancel(dns_query_.get()); + } + conn_.rlimit.stopw(); conn_.wlimit.stopw(); @@ -302,12 +308,47 @@ int Http2Session::disconnect(bool hard) { return 0; } +int Http2Session::resolve_name() { + int rv; + + auto dns_query = make_unique( + addr_->host, [this](int status, const Address *result) { + int rv; + + if (status == DNS_STATUS_OK) { + *resolved_addr_ = *result; + util::set_port(*this->resolved_addr_, this->addr_->port); + } + + rv = this->initiate_connection(); + if (rv != 0) { + delete this; + } + }); + resolved_addr_ = make_unique
(); + auto dns_tracker = worker_->get_dns_tracker(); + rv = dns_tracker->resolve(resolved_addr_.get(), dns_query.get()); + switch (rv) { + case DNS_STATUS_ERROR: + return -1; + case DNS_STATUS_RUNNING: + dns_query_ = std::move(dns_query); + state_ = RESOLVING_NAME; + return 0; + case DNS_STATUS_OK: + util::set_port(*resolved_addr_, addr_->port); + return 0; + default: + assert(0); + } +} + int Http2Session::initiate_connection() { int rv = 0; auto worker_blocker = worker_->get_connect_blocker(); - if (state_ == DISCONNECTED) { + if (state_ == DISCONNECTED || state_ == RESOLVING_NAME) { if (worker_blocker->blocked()) { if (LOG_ENABLED(INFO)) { SSLOG(INFO, this) @@ -350,6 +391,8 @@ int Http2Session::initiate_connection() { return -1; } + raddr_ = &proxy.addr; + worker_blocker->on_success(); ev_io_set(&conn_.rev, conn_.fd, EV_READ); @@ -374,36 +417,68 @@ int Http2Session::initiate_connection() { return 0; } - if (state_ == DISCONNECTED || state_ == PROXY_CONNECTED) { + if (state_ == DISCONNECTED || state_ == PROXY_CONNECTED || + state_ == RESOLVING_NAME) { if (LOG_ENABLED(INFO)) { - SSLOG(INFO, this) << "Connecting to downstream server"; + if (state_ != RESOLVING_NAME) { + SSLOG(INFO, this) << "Connecting to downstream server"; + } } if (addr_->tls) { assert(ssl_ctx_); - auto ssl = ssl::create_ssl(ssl_ctx_); - if (!ssl) { - return -1; + if (state_ != RESOLVING_NAME) { + auto ssl = ssl::create_ssl(ssl_ctx_); + if (!ssl) { + return -1; + } + + ssl::setup_downstream_http2_alpn(ssl); + + conn_.set_ssl(ssl); + + auto sni_name = + addr_->sni.empty() ? StringRef{addr_->host} : StringRef{addr_->sni}; + + if (!util::numeric_host(sni_name.c_str())) { + // TLS extensions: SNI. There is no documentation about the return + // code for this function (actually this is macro wrapping SSL_ctrl + // at the time of this writing). + SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str()); + } + + auto tls_session = ssl::reuse_tls_session(addr_->tls_session_cache); + if (tls_session) { + SSL_set_session(conn_.tls.ssl, tls_session); + SSL_SESSION_free(tls_session); + } } - ssl::setup_downstream_http2_alpn(ssl); - - conn_.set_ssl(ssl); - - auto sni_name = - addr_->sni.empty() ? StringRef{addr_->host} : StringRef{addr_->sni}; - - if (!util::numeric_host(sni_name.c_str())) { - // TLS extensions: SNI. There is no documentation about the return - // code for this function (actually this is macro wrapping SSL_ctrl - // at the time of this writing). - SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str()); + if (state_ == DISCONNECTED) { + if (addr_->dns) { + rv = resolve_name(); + if (rv != 0) { + downstream_failure(addr_, nullptr); + return -1; + } + if (state_ == RESOLVING_NAME) { + return 0; + } + raddr_ = resolved_addr_.get(); + } else { + raddr_ = &addr_->addr; + } } - auto tls_session = ssl::reuse_tls_session(addr_->tls_session_cache); - if (tls_session) { - SSL_set_session(conn_.tls.ssl, tls_session); - SSL_SESSION_free(tls_session); + if (state_ == RESOLVING_NAME) { + if (dns_query_->status == DNS_STATUS_ERROR) { + downstream_failure(addr_, nullptr); + return -1; + } + assert(dns_query_->status == DNS_STATUS_OK); + state_ = DISCONNECTED; + dns_query_.reset(); + raddr_ = resolved_addr_.get(); } // If state_ == PROXY_CONNECTED, we has connected to the proxy @@ -411,12 +486,11 @@ int Http2Session::initiate_connection() { if (state_ == DISCONNECTED) { assert(conn_.fd == -1); - conn_.fd = - util::create_nonblock_socket(addr_->addr.su.storage.ss_family); + conn_.fd = util::create_nonblock_socket(raddr_->su.storage.ss_family); if (conn_.fd == -1) { auto error = errno; SSLOG(WARN, this) - << "socket() failed; addr=" << util::to_numeric_addr(&addr_->addr) + << "socket() failed; addr=" << util::to_numeric_addr(raddr_) << ", errno=" << error; worker_blocker->on_failure(); @@ -427,15 +501,14 @@ int Http2Session::initiate_connection() { rv = connect(conn_.fd, // TODO maybe not thread-safe? - const_cast(&addr_->addr.su.sa), - addr_->addr.len); + const_cast(&raddr_->su.sa), raddr_->len); if (rv != 0 && errno != EINPROGRESS) { auto error = errno; - SSLOG(WARN, this) << "connect() failed; addr=" - << util::to_numeric_addr(&addr_->addr) - << ", errno=" << error; + SSLOG(WARN, this) + << "connect() failed; addr=" << util::to_numeric_addr(raddr_) + << ", errno=" << error; - downstream_failure(addr_); + downstream_failure(addr_, raddr_); return -1; } @@ -445,17 +518,44 @@ int Http2Session::initiate_connection() { conn_.prepare_client_handshake(); } else { + if (state_ == DISCONNECTED) { + // Without TLS and proxy. + if (addr_->dns) { + rv = resolve_name(); + if (rv != 0) { + downstream_failure(addr_, nullptr); + return -1; + } + if (state_ == RESOLVING_NAME) { + return 0; + } + raddr_ = resolved_addr_.get(); + } else { + raddr_ = &addr_->addr; + } + } + + if (state_ == RESOLVING_NAME) { + if (dns_query_->status == DNS_STATUS_ERROR) { + downstream_failure(addr_, nullptr); + return -1; + } + assert(dns_query_->status == DNS_STATUS_OK); + state_ = DISCONNECTED; + dns_query_.reset(); + raddr_ = resolved_addr_.get(); + } + if (state_ == DISCONNECTED) { // Without TLS and proxy. assert(conn_.fd == -1); - conn_.fd = - util::create_nonblock_socket(addr_->addr.su.storage.ss_family); + conn_.fd = util::create_nonblock_socket(raddr_->su.storage.ss_family); if (conn_.fd == -1) { auto error = errno; SSLOG(WARN, this) - << "socket() failed; addr=" << util::to_numeric_addr(&addr_->addr) + << "socket() failed; addr=" << util::to_numeric_addr(raddr_) << ", errno=" << error; worker_blocker->on_failure(); @@ -464,15 +564,15 @@ int Http2Session::initiate_connection() { worker_blocker->on_success(); - rv = connect(conn_.fd, const_cast(&addr_->addr.su.sa), - addr_->addr.len); + rv = connect(conn_.fd, const_cast(&raddr_->su.sa), + raddr_->len); if (rv != 0 && errno != EINPROGRESS) { auto error = errno; - SSLOG(WARN, this) << "connect() failed; addr=" - << util::to_numeric_addr(&addr_->addr) - << ", errno=" << error; + SSLOG(WARN, this) + << "connect() failed; addr=" << util::to_numeric_addr(raddr_) + << ", errno=" << error; - downstream_failure(addr_); + downstream_failure(addr_, raddr_); return -1; } @@ -1544,7 +1644,7 @@ int Http2Session::connection_made() { #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L if (!next_proto) { - downstream_failure(addr_); + downstream_failure(addr_, raddr_); return -1; } @@ -1553,7 +1653,7 @@ int Http2Session::connection_made() { SSLOG(INFO, this) << "Negotiated next protocol: " << proto; } if (!util::check_h2_is_selected(proto)) { - downstream_failure(addr_); + downstream_failure(addr_, raddr_); return -1; } } @@ -1842,10 +1942,10 @@ int Http2Session::connected() { auto sock_error = util::get_socket_error(conn_.fd); if (sock_error != 0) { SSLOG(WARN, this) << "Backend connect failed; addr=" - << util::to_numeric_addr(&addr_->addr) + << util::to_numeric_addr(raddr_) << ": errno=" << sock_error; - downstream_failure(addr_); + downstream_failure(addr_, raddr_); return -1; } @@ -1955,7 +2055,7 @@ int Http2Session::tls_handshake() { } if (rv < 0) { - downstream_failure(addr_); + downstream_failure(addr_, raddr_); return rv; } @@ -1965,8 +2065,8 @@ int Http2Session::tls_handshake() { } if (!get_config()->tls.insecure && - ssl::check_cert(conn_.tls.ssl, addr_) != 0) { - downstream_failure(addr_); + ssl::check_cert(conn_.tls.ssl, addr_, raddr_) != 0) { + downstream_failure(addr_, raddr_); return -1; } @@ -1974,8 +2074,8 @@ int Http2Session::tls_handshake() { if (!SSL_session_reused(conn_.tls.ssl)) { auto tls_session = SSL_get0_session(conn_.tls.ssl); if (tls_session) { - ssl::try_cache_tls_session(addr_->tls_session_cache, addr_->addr, - tls_session, ev_now(conn_.loop)); + ssl::try_cache_tls_session(addr_->tls_session_cache, *raddr_, tls_session, + ev_now(conn_.loop)); } } @@ -2262,9 +2362,9 @@ void Http2Session::on_timeout() { } case CONNECTING: { SSLOG(WARN, this) << "Connect time out; addr=" - << util::to_numeric_addr(&addr_->addr); + << util::to_numeric_addr(raddr_); - downstream_failure(addr_); + downstream_failure(addr_, raddr_); break; } } @@ -2284,4 +2384,6 @@ void Http2Session::check_retire() { signal_write(); } +const Address *Http2Session::get_raddr() const { return raddr_; } + } // namespace shrpx diff --git a/src/shrpx_http2_session.h b/src/shrpx_http2_session.h index 33ff5cf9..3dce8084 100644 --- a/src/shrpx_http2_session.h +++ b/src/shrpx_http2_session.h @@ -50,6 +50,7 @@ class Http2DownstreamConnection; class Worker; struct DownstreamAddrGroup; struct DownstreamAddr; +struct DNSQuery; struct StreamData { StreamData *dlnext, *dlprev; @@ -81,6 +82,7 @@ public: // associated ClientHandlers will be deleted. int disconnect(bool hard = false); int initiate_connection(); + int resolve_name(); void add_downstream_connection(Http2DownstreamConnection *dconn); void remove_downstream_connection(Http2DownstreamConnection *dconn); @@ -203,6 +205,9 @@ public: // shutdown the connection. void check_retire(); + // Returns address used to connect to backend. Could be nullptr. + const Address *get_raddr() const; + enum { // Disconnected DISCONNECTED, @@ -218,6 +223,8 @@ public: CONNECTED, // Connection is started to fail CONNECT_FAILING, + // Resolving host name + RESOLVING_NAME, }; enum { @@ -259,6 +266,13 @@ private: // Address of remote endpoint DownstreamAddr *addr_; nghttp2_session *session_; + // Actual remote address used to contact backend. This is initially + // nullptr, and may point to either &addr_->addr, + // resolved_addr_.get(), or HTTP proxy's address structure. + const Address *raddr_; + // Resolved IP address if dns parameter is used + std::unique_ptr
resolved_addr_; + std::unique_ptr dns_query_; int state_; int connection_check_state_; int freelist_zone_; diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 16435a84..d0fbe82b 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -75,11 +75,12 @@ void connect_timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) { auto conn = static_cast(w->data); auto dconn = static_cast(conn->data); auto addr = dconn->get_addr(); + auto raddr = dconn->get_raddr(); DCLOG(WARN, dconn) << "Connect time out; addr=" - << util::to_numeric_addr(&addr->addr); + << util::to_numeric_addr(raddr); - downstream_failure(addr); + downstream_failure(addr, raddr); auto downstream = dconn->get_downstream(); auto upstream = downstream->get_upstream(); @@ -182,13 +183,34 @@ HttpDownstreamConnection::~HttpDownstreamConnection() { if (LOG_ENABLED(INFO)) { DCLOG(INFO, this) << "Deleted"; } + + if (dns_query_) { + auto dns_tracker = worker_->get_dns_tracker(); + dns_tracker->cancel(dns_query_.get()); + } } int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { + int rv; + if (LOG_ENABLED(INFO)) { DCLOG(INFO, this) << "Attaching to DOWNSTREAM:" << downstream; } + downstream_ = downstream; + + rv = initiate_connection(); + if (rv != 0) { + downstream_ = nullptr; + return rv; + } + + return 0; +} + +int HttpDownstreamConnection::initiate_connection() { + int rv; + auto worker_blocker = worker_->get_connect_blocker(); if (worker_blocker->blocked()) { if (LOG_ENABLED(INFO)) { @@ -212,42 +234,136 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { shared_addr->affinity == AFFINITY_NONE ? shared_addr->next : temp_idx; auto end = next_downstream; for (;;) { - auto &addr = addrs[next_downstream]; + auto check_dns_result = dns_query_.get() != nullptr; - if (++next_downstream >= addrs.size()) { - next_downstream = 0; - } + DownstreamAddr *addr; + if (check_dns_result) { + addr = addr_; + addr_ = nullptr; + assert(addr); + assert(addr->dns); + } else { + assert(addr_ == nullptr); + addr = &addrs[next_downstream]; - if (addr.proto != PROTO_HTTP1) { - if (end == next_downstream) { - return SHRPX_ERR_NETWORK; + if (++next_downstream >= addrs.size()) { + next_downstream = 0; } - continue; + if (addr->proto != PROTO_HTTP1) { + if (end == next_downstream) { + return SHRPX_ERR_NETWORK; + } + + continue; + } } - auto &connect_blocker = addr.connect_blocker; + auto &connect_blocker = addr->connect_blocker; if (connect_blocker->blocked()) { if (LOG_ENABLED(INFO)) { - DCLOG(INFO, this) << "Backend server " - << util::to_numeric_addr(&addr.addr) - << " was not available temporarily"; + DCLOG(INFO, this) << "Backend server " << addr->host << ":" + << addr->port << " was not available temporarily"; } - if (end == next_downstream) { + if (check_dns_result) { + dns_query_.reset(); + } else if (end == next_downstream) { return SHRPX_ERR_NETWORK; } continue; } - conn_.fd = util::create_nonblock_socket(addr.addr.su.storage.ss_family); + Address *raddr; + + if (addr->dns) { + if (!check_dns_result) { + auto dns_query = make_unique( + addr->host, [this](int status, const Address *result) { + int rv; + + if (status == DNS_STATUS_OK) { + *this->resolved_addr_ = *result; + } + + rv = this->initiate_connection(); + if (rv != 0) { + // This callback destroys |this|. + auto downstream = this->downstream_; + auto upstream = downstream->get_upstream(); + auto handler = upstream->get_client_handler(); + + downstream->pop_downstream_connection(); + + auto ndconn = handler->get_downstream_connection(downstream); + if (ndconn) { + if (downstream->attach_downstream_connection( + std::move(ndconn)) == 0) { + return; + } + } + + downstream->set_request_state(Downstream::CONNECT_FAIL); + + if (upstream->on_downstream_abort_request(downstream, 503) != + 0) { + delete handler; + } + return; + } + }); + + auto dns_tracker = worker_->get_dns_tracker(); + + if (!resolved_addr_) { + resolved_addr_ = make_unique
(); + } + rv = dns_tracker->resolve(resolved_addr_.get(), dns_query.get()); + switch (rv) { + case DNS_STATUS_ERROR: + downstream_failure(addr, nullptr); + if (end == next_downstream) { + return SHRPX_ERR_NETWORK; + } + continue; + case DNS_STATUS_RUNNING: + dns_query_ = std::move(dns_query); + // Remember current addr + addr_ = addr; + return 0; + case DNS_STATUS_OK: + break; + default: + assert(0); + } + } else { + switch (dns_query_->status) { + case DNS_STATUS_ERROR: + dns_query_.reset(); + downstream_failure(addr, nullptr); + continue; + case DNS_STATUS_OK: + dns_query_.reset(); + break; + default: + assert(0); + } + } + + raddr = resolved_addr_.get(); + util::set_port(*resolved_addr_, addr->port); + } else { + raddr = &addr->addr; + } + + conn_.fd = util::create_nonblock_socket(raddr->su.storage.ss_family); if (conn_.fd == -1) { auto error = errno; DCLOG(WARN, this) << "socket() failed; addr=" - << util::to_numeric_addr(&addr.addr) + << util::to_numeric_addr(raddr) << ", errno=" << error; worker_blocker->on_failure(); @@ -257,20 +373,19 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { worker_blocker->on_success(); - int rv; - rv = connect(conn_.fd, &addr.addr.su.sa, addr.addr.len); + rv = connect(conn_.fd, &raddr->su.sa, raddr->len); if (rv != 0 && errno != EINPROGRESS) { auto error = errno; DCLOG(WARN, this) << "connect() failed; addr=" - << util::to_numeric_addr(&addr.addr) + << util::to_numeric_addr(raddr) << ", errno=" << error; - downstream_failure(&addr); + downstream_failure(addr, raddr); close(conn_.fd); conn_.fd = -1; - if (end == next_downstream) { + if (!check_dns_result && end == next_downstream) { return SHRPX_ERR_NETWORK; } @@ -282,7 +397,8 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { DCLOG(INFO, this) << "Connecting to downstream server"; } - addr_ = &addr; + addr_ = addr; + raddr_ = raddr; if (addr_->tls) { assert(ssl_ctx_); @@ -334,8 +450,6 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { ev_set_cb(&conn_.rev, readcb); } - downstream_ = downstream; - http_parser_init(&response_htp_, HTTP_RESPONSE); response_htp_.data = downstream_; @@ -1031,7 +1145,7 @@ int HttpDownstreamConnection::tls_handshake() { } if (rv < 0) { - downstream_failure(addr_); + downstream_failure(addr_, raddr_); return rv; } @@ -1041,8 +1155,8 @@ int HttpDownstreamConnection::tls_handshake() { } if (!get_config()->tls.insecure && - ssl::check_cert(conn_.tls.ssl, addr_) != 0) { - downstream_failure(addr_); + ssl::check_cert(conn_.tls.ssl, addr_, raddr_) != 0) { + downstream_failure(addr_, raddr_); return -1; } @@ -1050,7 +1164,7 @@ int HttpDownstreamConnection::tls_handshake() { if (!SSL_session_reused(conn_.tls.ssl)) { auto session = SSL_get0_session(conn_.tls.ssl); if (session) { - ssl::try_cache_tls_session(addr_->tls_session_cache, addr_->addr, session, + ssl::try_cache_tls_session(addr_->tls_session_cache, *raddr_, session, ev_now(conn_.loop)); } } @@ -1215,10 +1329,10 @@ int HttpDownstreamConnection::connected() { conn_.wlimit.stopw(); DCLOG(WARN, this) << "Backend connect failed; addr=" - << util::to_numeric_addr(&addr_->addr) + << util::to_numeric_addr(raddr_) << ": errno=" << sock_error; - downstream_failure(addr_); + downstream_failure(addr_, raddr_); return -1; } @@ -1282,4 +1396,6 @@ DownstreamAddr *HttpDownstreamConnection::get_addr() const { return addr_; } bool HttpDownstreamConnection::poolable() const { return !group_->retired; } +const Address *HttpDownstreamConnection::get_raddr() const { return raddr_; } + } // namespace shrpx diff --git a/src/shrpx_http_downstream_connection.h b/src/shrpx_http_downstream_connection.h index 03050d8a..3001c4ea 100644 --- a/src/shrpx_http_downstream_connection.h +++ b/src/shrpx_http_downstream_connection.h @@ -39,6 +39,7 @@ class DownstreamConnectionPool; class Worker; struct DownstreamAddrGroup; struct DownstreamAddr; +struct DNSQuery; class HttpDownstreamConnection : public DownstreamConnection { public: @@ -68,6 +69,8 @@ public: get_downstream_addr_group() const; virtual DownstreamAddr *get_addr() const; + int initiate_connection(); + int read_clear(); int write_clear(); int read_tls(); @@ -80,6 +83,9 @@ public: void signal_write(); int actual_signal_write(); + // Returns address used to connect to backend. Could be nullptr. + const Address *get_raddr() const; + int noop(); private: @@ -92,6 +98,13 @@ private: std::shared_ptr group_; // Address of remote endpoint DownstreamAddr *addr_; + // Actual remote address used to contact backend. This is initially + // nullptr, and may point to either &addr_->addr, or + // resolved_addr_.get(). + const Address *raddr_; + // Resolved IP address if dns parameter is used + std::unique_ptr
resolved_addr_; + std::unique_ptr dns_query_; IOControl ioctrl_; http_parser response_htp_; ssize_t initial_addr_idx_; diff --git a/src/shrpx_live_check.cc b/src/shrpx_live_check.cc index 4065ff7c..1af88c4c 100644 --- a/src/shrpx_live_check.cc +++ b/src/shrpx_live_check.cc @@ -114,6 +114,7 @@ LiveCheck::LiveCheck(struct ev_loop *loop, SSL_CTX *ssl_ctx, Worker *worker, ssl_ctx_(ssl_ctx), addr_(addr), session_(nullptr), + raddr_(nullptr), success_count_(0), fail_count_(0), settings_ack_received_(false), @@ -134,6 +135,16 @@ LiveCheck::~LiveCheck() { } void LiveCheck::disconnect() { + if (dns_query_) { + auto dns_tracker = worker_->get_dns_tracker(); + + dns_tracker->cancel(dns_query_.get()); + } + + dns_query_.reset(); + // We can reuse resolved_addr_ + raddr_ = nullptr; + conn_.rlimit.stopw(); conn_.wlimit.stopw(); @@ -190,7 +201,7 @@ int LiveCheck::initiate_connection() { return -1; } - if (addr_->tls) { + if (!dns_query_ && addr_->tls) { assert(ssl_ctx_); auto ssl = ssl::create_ssl(ssl_ctx_); @@ -212,20 +223,71 @@ int LiveCheck::initiate_connection() { conn_.set_ssl(ssl); } - conn_.fd = util::create_nonblock_socket(addr_->addr.su.storage.ss_family); + if (addr_->dns) { + if (!dns_query_) { + auto dns_query = make_unique( + addr_->host, [this](int status, const Address *result) { + int rv; + + if (status == DNS_STATUS_OK) { + *this->resolved_addr_ = *result; + } + rv = this->initiate_connection(); + if (rv != 0) { + this->on_failure(); + } + }); + auto dns_tracker = worker_->get_dns_tracker(); + + if (!resolved_addr_) { + resolved_addr_ = make_unique
(); + } + + rv = dns_tracker->resolve(resolved_addr_.get(), dns_query.get()); + switch (rv) { + case DNS_STATUS_ERROR: + return -1; + case DNS_STATUS_RUNNING: + dns_query_ = std::move(dns_query); + return 0; + case DNS_STATUS_OK: + break; + default: + assert(0); + } + } else { + switch (dns_query_->status) { + case DNS_STATUS_ERROR: + dns_query_.reset(); + return -1; + case DNS_STATUS_OK: + dns_query_.reset(); + break; + default: + assert(0); + } + } + + util::set_port(*resolved_addr_, addr_->port); + raddr_ = resolved_addr_.get(); + } else { + raddr_ = &addr_->addr; + } + + conn_.fd = util::create_nonblock_socket(raddr_->su.storage.ss_family); if (conn_.fd == -1) { auto error = errno; - LOG(WARN) << "socket() failed; addr=" << util::to_numeric_addr(&addr_->addr) + LOG(WARN) << "socket() failed; addr=" << util::to_numeric_addr(raddr_) << ", errno=" << error; return -1; } - rv = connect(conn_.fd, &addr_->addr.su.sa, addr_->addr.len); + rv = connect(conn_.fd, &raddr_->su.sa, raddr_->len); if (rv != 0 && errno != EINPROGRESS) { auto error = errno; - LOG(WARN) << "connect() failed; addr=" - << util::to_numeric_addr(&addr_->addr) << ", errno=" << error; + LOG(WARN) << "connect() failed; addr=" << util::to_numeric_addr(raddr_) + << ", errno=" << error; close(conn_.fd); conn_.fd = -1; @@ -269,8 +331,7 @@ int LiveCheck::connected() { if (sock_error != 0) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Backend connect failed; addr=" - << util::to_numeric_addr(&addr_->addr) - << ": errno=" << sock_error; + << util::to_numeric_addr(raddr_) << ": errno=" << sock_error; } return -1; @@ -334,15 +395,15 @@ int LiveCheck::tls_handshake() { } if (!get_config()->tls.insecure && - ssl::check_cert(conn_.tls.ssl, addr_) != 0) { + ssl::check_cert(conn_.tls.ssl, addr_, raddr_) != 0) { return -1; } if (!SSL_session_reused(conn_.tls.ssl)) { auto tls_session = SSL_get0_session(conn_.tls.ssl); if (tls_session) { - ssl::try_cache_tls_session(addr_->tls_session_cache, addr_->addr, - tls_session, ev_now(conn_.loop)); + ssl::try_cache_tls_session(addr_->tls_session_cache, *raddr_, tls_session, + ev_now(conn_.loop)); } } @@ -601,7 +662,7 @@ void LiveCheck::on_failure() { ++fail_count_; if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Liveness check for " << util::to_numeric_addr(&addr_->addr) + LOG(INFO) << "Liveness check for " << addr_->host << ":" << addr_->port << " failed " << fail_count_ << " time(s) in a row"; } @@ -615,7 +676,7 @@ void LiveCheck::on_success() { fail_count_ = 0; if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Liveness check for " << util::to_numeric_addr(&addr_->addr) + LOG(INFO) << "Liveness check for " << addr_->host << ":" << addr_->port << " succeeded " << success_count_ << " time(s) in a row"; } diff --git a/src/shrpx_live_check.h b/src/shrpx_live_check.h index 3cd83a55..b65ecdcd 100644 --- a/src/shrpx_live_check.h +++ b/src/shrpx_live_check.h @@ -42,6 +42,7 @@ namespace shrpx { class Worker; struct DownstreamAddr; +struct DNSQuery; class LiveCheck { public: @@ -102,6 +103,13 @@ private: // Address of remote endpoint DownstreamAddr *addr_; nghttp2_session *session_; + // Actual remote address used to contact backend. This is initially + // nullptr, and may point to either &addr_->addr, or + // resolved_addr_.get(). + const Address *raddr_; + // Resolved IP address if dns parameter is used + std::unique_ptr
resolved_addr_; + std::unique_ptr dns_query_; // The number of successful connect attempt in a row. size_t success_count_; // The number of unsuccessful connect attempt in a row. diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 5c9f72bd..fa2f39e2 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -1171,10 +1171,10 @@ int check_cert(SSL *ssl, const Address *addr, const StringRef &host) { return 0; } -int check_cert(SSL *ssl, const DownstreamAddr *addr) { +int check_cert(SSL *ssl, const DownstreamAddr *addr, const Address *raddr) { auto hostname = addr->sni.empty() ? StringRef{addr->host} : StringRef{addr->sni}; - return check_cert(ssl, &addr->addr, hostname); + return check_cert(ssl, raddr, hostname); } CertLookupTree::CertLookupTree() {} diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h index 6f44a058..08f07a24 100644 --- a/src/shrpx_ssl.h +++ b/src/shrpx_ssl.h @@ -104,7 +104,10 @@ ClientHandler *accept_connection(Worker *worker, int fd, sockaddr *addr, // Check peer's certificate against given |address| and |host|. int check_cert(SSL *ssl, const Address *addr, const StringRef &host); -int check_cert(SSL *ssl, const DownstreamAddr *addr); +// Check peer's certificate against given host name described in +// |addr| and numeric address in |raddr|. Note that |raddr| might not +// point to &addr->addr. +int check_cert(SSL *ssl, const DownstreamAddr *addr, const Address *raddr); struct WildcardRevPrefix { WildcardRevPrefix(const StringRef &prefix, size_t idx) diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 2b9ae1b2..30db93a7 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -92,7 +92,7 @@ bool match_shared_downstream_addr( auto &b = rhs->addrs[i]; if (a.host == b.host && a.port == b.port && a.host_unix == b.host_unix && a.proto == b.proto && a.tls == b.tls && a.sni == b.sni && - a.fall == b.fall && a.rise == b.rise) { + a.fall == b.fall && a.rise == b.rise && a.dns == b.dns) { break; } } @@ -120,6 +120,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, std::shared_ptr downstreamconf) : randgen_(rd()), worker_stat_{}, + dns_tracker_(loop), loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), @@ -209,6 +210,7 @@ void Worker::replace_downstream_config( dst_addr.sni = make_string_ref(shared_addr->balloc, src_addr.sni); dst_addr.fall = src_addr.fall; dst_addr.rise = src_addr.rise; + dst_addr.dns = src_addr.dns; auto shared_addr_ptr = shared_addr.get(); @@ -490,6 +492,8 @@ ConnectionHandler *Worker::get_connection_handler() const { return conn_handler_; } +DNSTracker *Worker::get_dns_tracker() { return &dns_tracker_; } + namespace { size_t match_downstream_addr_group_host( const RouterConfig &routerconf, const StringRef &host, @@ -635,7 +639,7 @@ size_t match_downstream_addr_group( catch_all, balloc); } -void downstream_failure(DownstreamAddr *addr) { +void downstream_failure(DownstreamAddr *addr, const Address *raddr) { const auto &connect_blocker = addr->connect_blocker; if (connect_blocker->in_offline()) { @@ -651,8 +655,15 @@ void downstream_failure(DownstreamAddr *addr) { auto fail_count = connect_blocker->get_fail_count(); if (fail_count >= addr->fall) { - LOG(WARN) << "Could not connect to " << util::to_numeric_addr(&addr->addr) - << " " << fail_count << " times in a row; considered as offline"; + if (raddr) { + LOG(WARN) << "Could not connect to " << util::to_numeric_addr(raddr) + << " " << fail_count + << " times in a row; considered as offline"; + } else { + LOG(WARN) << "Could not connect to " << addr->host << ":" << addr->port + << " " << fail_count + << " times in a row; considered as offline"; + } connect_blocker->offline(); diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index c68c2485..44a29f09 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -48,6 +48,7 @@ #include "shrpx_ssl.h" #include "shrpx_live_check.h" #include "shrpx_connect_blocker.h" +#include "shrpx_dns_tracker.h" #include "allocator.h" using namespace nghttp2; @@ -112,6 +113,8 @@ struct DownstreamAddr { shrpx_proto proto; // true if TLS is used in this backend bool tls; + // true if dynamic DNS is enabled + bool dns; }; // Simplified weighted fair queuing. Actually we don't use queue here @@ -263,6 +266,8 @@ public: ConnectionHandler *get_connection_handler() const; + DNSTracker *get_dns_tracker(); + private: #ifndef NOTHREADS std::future fut_; @@ -275,6 +280,7 @@ private: ev_timer proc_wev_timer_; MemchunkPool mcpool_; WorkerStat worker_stat_; + DNSTracker dns_tracker_; std::shared_ptr downstreamconf_; std::unique_ptr session_cache_memcached_dispatcher_; @@ -314,7 +320,10 @@ size_t match_downstream_addr_group( const std::vector> &groups, size_t catch_all, BlockAllocator &balloc); -void downstream_failure(DownstreamAddr *addr); +// Calls this function if connecting to backend failed. |raddr| is +// the actual address used to connect to backend, and it could be +// nullptr. This function may schedule live check. +void downstream_failure(DownstreamAddr *addr, const Address *raddr); } // namespace shrpx diff --git a/src/util.cc b/src/util.cc index 3153f34b..62442cb4 100644 --- a/src/util.cc +++ b/src/util.cc @@ -666,6 +666,17 @@ std::string to_numeric_addr(const Address *addr) { return s; } +void set_port(Address &addr, uint16_t port) { + switch (addr.su.storage.ss_family) { + case AF_INET: + addr.su.in.sin_port = htons(port); + break; + case AF_INET6: + addr.su.in6.sin6_port = htons(port); + break; + } +} + static int STDERR_COPY = -1; static int STDOUT_COPY = -1; diff --git a/src/util.h b/src/util.h index 8a767a0a..9bb63ef7 100644 --- a/src/util.h +++ b/src/util.h @@ -478,6 +478,9 @@ std::string numeric_name(const struct sockaddr *sa, socklen_t salen); // IPv6 address, address is enclosed by square brackets ([]). std::string to_numeric_addr(const Address *addr); +// Sets |port| to |addr|. +void set_port(Address &addr, uint16_t port); + // Makes internal copy of stderr (and possibly stdout in the future), // which is then used as pointer to /dev/stderr or /proc/self/fd/2 void store_original_fds();