nghttpx: Add --backend-max-backoff option

gennghttpxfun.py

@@ -133,6 +133,7 @@ OPTIONS = [
     "frontend-http2-settings-timeout",
     "backend-http2-settings-timeout",
     "api-max-request-body",
+    "backend-max-backoff",
 ]
 
 LOGVARS = [

src/shrpx.cc

@@ -1176,6 +1176,7 @@ void fill_default_config() {
       timeoutconf.write = 30_s;
       // Timeout for pooled (idle) connections
       timeoutconf.idle_read = 2_s;
+      timeoutconf.max_backoff = 120_s;
     }
 
     downstreamconf.connections_per_host = 8;
@@ -1549,6 +1550,18 @@ Timeout:
               Default: )"
       << util::duration_str(get_config()->http2.downstream.timeout.settings)
       << R"(
+  --backend-max-backoff=<DURATION>
+              Specify  maximum backoff  interval.  This  is used  when
+              doing health  check against offline backend  (see "fail"
+              parameter  in --backend  option).   It is  also used  to
+              limit  the  maximum   interval  to  temporarily  disable
+              backend  when nghttpx  failed to  connect to  it.  These
+              intervals are calculated  using exponential backoff, and
+              consecutive failed attempts increase the interval.  This
+              option caps its maximum value.
+              Default: )"
+      << util::duration_str(get_config()->conn.downstream->timeout.max_backoff)
+      << R"(
 
 SSL/TLS:
   --ciphers=<SUITE>
@@ -2488,6 +2501,7 @@ int main(int argc, char **argv) {
         {SHRPX_OPT_BACKEND_HTTP2_SETTINGS_TIMEOUT.c_str(), required_argument,
          &flag, 125},
         {SHRPX_OPT_API_MAX_REQUEST_BODY.c_str(), required_argument, &flag, 126},
+        {SHRPX_OPT_BACKEND_MAX_BACKOFF.c_str(), required_argument, &flag, 127},
         {nullptr, 0, nullptr, 0}};
 
     int option_index = 0;
@@ -3081,6 +3095,10 @@ int main(int argc, char **argv) {
         // --api-max-request-body
         cmdcfgs.emplace_back(SHRPX_OPT_API_MAX_REQUEST_BODY, StringRef{optarg});
         break;
+      case 127:
+        // --backend-max-backoff
+        cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_MAX_BACKOFF, StringRef{optarg});
+        break;
       default:
         break;
       }

src/shrpx_config.cc

@@ -1290,6 +1290,11 @@ int option_lookup_token(const char *name, size_t namelen) {
         return SHRPX_OPTID_TLS_TICKET_KEY_FILE;
       }
       break;
+    case 'f':
+      if (util::strieq_l("backend-max-backof", name, 18)) {
+        return SHRPX_OPTID_BACKEND_MAX_BACKOFF;
+      }
+      break;
     case 'r':
       if (util::strieq_l("add-response-heade", name, 18)) {
         return SHRPX_OPTID_ADD_RESPONSE_HEADER;
@@ -2629,6 +2634,9 @@ int parse_config(Config *config, int optid, const StringRef &opt,
                           optarg);
   case SHRPX_OPTID_API_MAX_REQUEST_BODY:
     return parse_uint_with_unit(&config->api.max_request_body, opt, optarg);
+  case SHRPX_OPTID_BACKEND_MAX_BACKOFF:
+    return parse_duration(&config->conn.downstream->timeout.max_backoff, opt,
+                          optarg);
   case SHRPX_OPTID_CONF:
     LOG(WARN) << "conf: ignored";
 

src/shrpx_config.h

@@ -282,6 +282,8 @@ constexpr auto SHRPX_OPT_BACKEND_HTTP2_SETTINGS_TIMEOUT =
     StringRef::from_lit("backend-http2-settings-timeout");
 constexpr auto SHRPX_OPT_API_MAX_REQUEST_BODY =
     StringRef::from_lit("api-max-request-body");
+constexpr auto SHRPX_OPT_BACKEND_MAX_BACKOFF =
+    StringRef::from_lit("backend-max-backoff");
 
 constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8;
 
@@ -635,6 +637,10 @@ struct DownstreamConfig {
     ev_tstamp read;
     ev_tstamp write;
     ev_tstamp idle_read;
+    // The maximum backoff while checking health check for offline
+    // backend or while detaching failed backend from load balancing
+    // group temporarily.
+    ev_tstamp max_backoff;
   } timeout;
   RouterConfig router;
   std::vector<DownstreamAddrGroupConfig> addr_groups;
@@ -748,6 +754,7 @@ enum {
   SHRPX_OPTID_BACKEND_IPV4,
   SHRPX_OPTID_BACKEND_IPV6,
   SHRPX_OPTID_BACKEND_KEEP_ALIVE_TIMEOUT,
+  SHRPX_OPTID_BACKEND_MAX_BACKOFF,
   SHRPX_OPTID_BACKEND_NO_TLS,
   SHRPX_OPTID_BACKEND_READ_TIMEOUT,
   SHRPX_OPTID_BACKEND_REQUEST_BUFFER,

src/shrpx_connect_blocker.cc

@@ -23,6 +23,7 @@
  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 #include "shrpx_connect_blocker.h"
+#include "shrpx_config.h"
 
 namespace shrpx {
 
@@ -82,7 +83,11 @@ void ConnectBlocker::on_failure() {
   auto base_backoff = pow(MULTIPLIER, std::min(MAX_BACKOFF_EXP, fail_count_));
   auto dist = std::uniform_real_distribution<>(-JITTER * base_backoff,
                                                JITTER * base_backoff);
-  auto backoff = base_backoff + dist(gen_);
+
+  auto &downstreamconf = *get_config()->conn.downstream;
+
+  auto backoff =
+      std::min(downstreamconf.timeout.max_backoff, base_backoff + dist(gen_));
 
   LOG(WARN) << "Could not connect " << fail_count_
             << " times in a row; sleep for " << backoff << " seconds";

src/shrpx_live_check.cc

@@ -160,7 +160,11 @@ void LiveCheck::schedule() {
   auto base_backoff = pow(MULTIPLIER, std::min(fail_count_, MAX_BACKOFF_EXP));
   auto dist = std::uniform_real_distribution<>(-JITTER * base_backoff,
                                                JITTER * base_backoff);
-  auto backoff = base_backoff + dist(gen_);
+
+  auto &downstreamconf = *get_config()->conn.downstream;
+
+  auto backoff =
+      std::min(downstreamconf.timeout.max_backoff, base_backoff + dist(gen_));
 
   ev_timer_set(&backoff_timer_, backoff, 0.);
   ev_timer_start(conn_.loop, &backoff_timer_);