diff --git a/gennghttpxfun.py b/gennghttpxfun.py index ab9e7e44..f6d0e11c 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -133,6 +133,7 @@ OPTIONS = [ "frontend-http2-settings-timeout", "backend-http2-settings-timeout", "api-max-request-body", + "backend-max-backoff", ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index 716f393a..25bcd4e4 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1176,6 +1176,7 @@ void fill_default_config() { timeoutconf.write = 30_s; // Timeout for pooled (idle) connections timeoutconf.idle_read = 2_s; + timeoutconf.max_backoff = 120_s; } downstreamconf.connections_per_host = 8; @@ -1549,6 +1550,18 @@ Timeout: Default: )" << util::duration_str(get_config()->http2.downstream.timeout.settings) << R"( + --backend-max-backoff= + Specify maximum backoff interval. This is used when + doing health check against offline backend (see "fail" + parameter in --backend option). It is also used to + limit the maximum interval to temporarily disable + backend when nghttpx failed to connect to it. These + intervals are calculated using exponential backoff, and + consecutive failed attempts increase the interval. This + option caps its maximum value. + Default: )" + << util::duration_str(get_config()->conn.downstream->timeout.max_backoff) + << R"( SSL/TLS: --ciphers= @@ -2488,6 +2501,7 @@ int main(int argc, char **argv) { {SHRPX_OPT_BACKEND_HTTP2_SETTINGS_TIMEOUT.c_str(), required_argument, &flag, 125}, {SHRPX_OPT_API_MAX_REQUEST_BODY.c_str(), required_argument, &flag, 126}, + {SHRPX_OPT_BACKEND_MAX_BACKOFF.c_str(), required_argument, &flag, 127}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -3081,6 +3095,10 @@ int main(int argc, char **argv) { // --api-max-request-body cmdcfgs.emplace_back(SHRPX_OPT_API_MAX_REQUEST_BODY, StringRef{optarg}); break; + case 127: + // --backend-max-backoff + cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_MAX_BACKOFF, StringRef{optarg}); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index c36e5693..e059b276 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1290,6 +1290,11 @@ int option_lookup_token(const char *name, size_t namelen) { return SHRPX_OPTID_TLS_TICKET_KEY_FILE; } break; + case 'f': + if (util::strieq_l("backend-max-backof", name, 18)) { + return SHRPX_OPTID_BACKEND_MAX_BACKOFF; + } + break; case 'r': if (util::strieq_l("add-response-heade", name, 18)) { return SHRPX_OPTID_ADD_RESPONSE_HEADER; @@ -2629,6 +2634,9 @@ int parse_config(Config *config, int optid, const StringRef &opt, optarg); case SHRPX_OPTID_API_MAX_REQUEST_BODY: return parse_uint_with_unit(&config->api.max_request_body, opt, optarg); + case SHRPX_OPTID_BACKEND_MAX_BACKOFF: + return parse_duration(&config->conn.downstream->timeout.max_backoff, opt, + optarg); case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index b04f4d7f..25a34dac 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -282,6 +282,8 @@ constexpr auto SHRPX_OPT_BACKEND_HTTP2_SETTINGS_TIMEOUT = StringRef::from_lit("backend-http2-settings-timeout"); constexpr auto SHRPX_OPT_API_MAX_REQUEST_BODY = StringRef::from_lit("api-max-request-body"); +constexpr auto SHRPX_OPT_BACKEND_MAX_BACKOFF = + StringRef::from_lit("backend-max-backoff"); constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -635,6 +637,10 @@ struct DownstreamConfig { ev_tstamp read; ev_tstamp write; ev_tstamp idle_read; + // The maximum backoff while checking health check for offline + // backend or while detaching failed backend from load balancing + // group temporarily. + ev_tstamp max_backoff; } timeout; RouterConfig router; std::vector addr_groups; @@ -748,6 +754,7 @@ enum { SHRPX_OPTID_BACKEND_IPV4, SHRPX_OPTID_BACKEND_IPV6, SHRPX_OPTID_BACKEND_KEEP_ALIVE_TIMEOUT, + SHRPX_OPTID_BACKEND_MAX_BACKOFF, SHRPX_OPTID_BACKEND_NO_TLS, SHRPX_OPTID_BACKEND_READ_TIMEOUT, SHRPX_OPTID_BACKEND_REQUEST_BUFFER, diff --git a/src/shrpx_connect_blocker.cc b/src/shrpx_connect_blocker.cc index 29ab178d..03396ad8 100644 --- a/src/shrpx_connect_blocker.cc +++ b/src/shrpx_connect_blocker.cc @@ -23,6 +23,7 @@ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "shrpx_connect_blocker.h" +#include "shrpx_config.h" namespace shrpx { @@ -82,7 +83,11 @@ void ConnectBlocker::on_failure() { auto base_backoff = pow(MULTIPLIER, std::min(MAX_BACKOFF_EXP, fail_count_)); auto dist = std::uniform_real_distribution<>(-JITTER * base_backoff, JITTER * base_backoff); - auto backoff = base_backoff + dist(gen_); + + auto &downstreamconf = *get_config()->conn.downstream; + + auto backoff = + std::min(downstreamconf.timeout.max_backoff, base_backoff + dist(gen_)); LOG(WARN) << "Could not connect " << fail_count_ << " times in a row; sleep for " << backoff << " seconds"; diff --git a/src/shrpx_live_check.cc b/src/shrpx_live_check.cc index eefa8783..1fe28c6f 100644 --- a/src/shrpx_live_check.cc +++ b/src/shrpx_live_check.cc @@ -160,7 +160,11 @@ void LiveCheck::schedule() { auto base_backoff = pow(MULTIPLIER, std::min(fail_count_, MAX_BACKOFF_EXP)); auto dist = std::uniform_real_distribution<>(-JITTER * base_backoff, JITTER * base_backoff); - auto backoff = base_backoff + dist(gen_); + + auto &downstreamconf = *get_config()->conn.downstream; + + auto backoff = + std::min(downstreamconf.timeout.max_backoff, base_backoff + dist(gen_)); ev_timer_set(&backoff_timer_, backoff, 0.); ev_timer_start(conn_.loop, &backoff_timer_);