nghttpx: Add affinity-cookie-stickiness backend parameter

This commit is contained in:
Tatsuhiro Tsujikawa 2022-03-16 17:39:33 +09:00
parent 3ec588bb54
commit 5ded01e288
8 changed files with 153 additions and 13 deletions

View File

@ -427,7 +427,11 @@ parameter. Optionally, a Path attribute can be specified in
backend=127.0.0.1,3000;;affinity=cookie;affinity-cookie-name=nghttpxlb;affinity-cookie-path=/ backend=127.0.0.1,3000;;affinity=cookie;affinity-cookie-name=nghttpxlb;affinity-cookie-path=/
Secure attribute of cookie is set if client connection is protected by Secure attribute of cookie is set if client connection is protected by
TLS. TLS. ``affinity-cookie-stickiness`` specifies the stickiness of this
affinity. If ``loose`` is given, which is the default, removing or
adding a backend server might break affinity. While ``strict`` is
given, removing the designated backend server breaks affinity, but
adding new backend server does not cause breakage.
PSK cipher suites PSK cipher suites
----------------- -----------------

View File

@ -2249,7 +2249,18 @@ Connections:
If a request scheme is "https", then Secure attribute is If a request scheme is "https", then Secure attribute is
set. Otherwise, it is not set. If <SECURE> is "yes", set. Otherwise, it is not set. If <SECURE> is "yes",
the Secure attribute is always set. If <SECURE> is the Secure attribute is always set. If <SECURE> is
"no", the Secure attribute is always omitted. "no", the Secure attribute is always omitted.
"affinity-cookie-stickiness=<STICKINESS>" controls
stickiness of this affinity. If <STICKINESS> is
"loose", removing or adding a backend server might break
the affinity and the request might be forwarded to a
different backend server. If <STICKINESS> is "strict",
removing the designated backend server breaks affinity,
but adding new backend server does not cause breakage.
If the designated backend server becomes unavailable,
new backend server is chosen as if the request does not
have an affinity cookie. <STICKINESS> defaults to
"loose".
By default, name resolution of backend host name is done By default, name resolution of backend host name is done
at start up, or reloading configuration. If "dns" at start up, or reloading configuration. If "dns"

View File

@ -847,6 +847,12 @@ DownstreamAddr *ClientHandler::get_downstream_addr(int &err,
hash = affinity_hash_; hash = affinity_hash_;
break; break;
case SessionAffinity::COOKIE: case SessionAffinity::COOKIE:
if (shared_addr->affinity.cookie.stickiness ==
SessionAffinityCookieStickiness::STRICT) {
return get_downstream_addr_strict_affinity(err, shared_addr,
downstream);
}
hash = get_affinity_cookie(downstream, shared_addr->affinity.cookie.name); hash = get_affinity_cookie(downstream, shared_addr->affinity.cookie.name);
break; break;
default: default:
@ -923,6 +929,69 @@ DownstreamAddr *ClientHandler::get_downstream_addr(int &err,
} }
} }
DownstreamAddr *ClientHandler::get_downstream_addr_strict_affinity(
int &err, const std::shared_ptr<SharedDownstreamAddr> &shared_addr,
Downstream *downstream) {
const auto &affinity_hash = shared_addr->affinity_hash;
auto h = downstream->find_affinity_cookie(shared_addr->affinity.cookie.name);
if (h) {
auto it = shared_addr->affinity_hash_map.find(h);
if (it != std::end(shared_addr->affinity_hash_map)) {
auto addr = &shared_addr->addrs[(*it).second];
if (!addr->connect_blocker->blocked()) {
return addr;
}
}
} else {
auto d = std::uniform_int_distribution<uint32_t>(
1, std::numeric_limits<uint32_t>::max());
auto rh = d(worker_->get_randgen());
h = util::hash32(StringRef{reinterpret_cast<uint8_t *>(&rh),
reinterpret_cast<uint8_t *>(&rh) + sizeof(rh)});
}
// Client is not bound to a particular backend, or the bound backend
// is not found, or is blocked. Find new backend using h. Using
// existing h allows us to find new server in a deterministic way.
// It is preferable because multiple concurrent requests with the
// stale cookie might be in-flight.
auto it = std::lower_bound(
std::begin(affinity_hash), std::end(affinity_hash), h,
[](const AffinityHash &lhs, uint32_t rhs) { return lhs.hash < rhs; });
if (it == std::end(affinity_hash)) {
it = std::begin(affinity_hash);
}
auto aff_idx =
static_cast<size_t>(std::distance(std::begin(affinity_hash), it));
auto idx = (*it).idx;
auto addr = &shared_addr->addrs[idx];
if (addr->connect_blocker->blocked()) {
size_t i;
for (i = aff_idx + 1; i != aff_idx; ++i) {
if (i == shared_addr->affinity_hash.size()) {
i = 0;
}
addr = &shared_addr->addrs[shared_addr->affinity_hash[i].idx];
if (addr->connect_blocker->blocked()) {
continue;
}
break;
}
if (i == aff_idx) {
err = -1;
return nullptr;
}
}
downstream->renew_affinity_cookie(addr->affinity_hash);
return addr;
}
std::unique_ptr<DownstreamConnection> std::unique_ptr<DownstreamConnection>
ClientHandler::get_downstream_connection(int &err, Downstream *downstream) { ClientHandler::get_downstream_connection(int &err, Downstream *downstream) {
size_t group_idx; size_t group_idx;

View File

@ -52,6 +52,7 @@ class Worker;
class Downstream; class Downstream;
struct WorkerStat; struct WorkerStat;
struct DownstreamAddrGroup; struct DownstreamAddrGroup;
struct SharedDownstreamAddr;
struct DownstreamAddr; struct DownstreamAddr;
#ifdef ENABLE_HTTP3 #ifdef ENABLE_HTTP3
class Http3Upstream; class Http3Upstream;
@ -170,6 +171,10 @@ public:
uint32_t get_affinity_cookie(Downstream *downstream, uint32_t get_affinity_cookie(Downstream *downstream,
const StringRef &cookie_name); const StringRef &cookie_name);
DownstreamAddr *get_downstream_addr_strict_affinity(
int &err, const std::shared_ptr<SharedDownstreamAddr> &shared_addr,
Downstream *downstream);
const UpstreamAddr *get_upstream_addr() const; const UpstreamAddr *get_upstream_addr() const;
void repeat_read_timer(); void repeat_read_timer();

View File

@ -1098,6 +1098,19 @@ int parse_downstream_params(DownstreamParams &out,
"auto, yes, and no"; "auto, yes, and no";
return -1; return -1;
} }
} else if (util::istarts_with_l(param, "affinity-cookie-stickiness=")) {
auto valstr =
StringRef{first + str_size("affinity-cookie-stickiness="), end};
if (util::strieq_l("loose", valstr)) {
out.affinity.cookie.stickiness = SessionAffinityCookieStickiness::LOOSE;
} else if (util::strieq_l("strict", valstr)) {
out.affinity.cookie.stickiness =
SessionAffinityCookieStickiness::STRICT;
} else {
LOG(ERROR) << "backend: affinity-cookie-stickiness: value must be "
"either loose or strict";
return -1;
}
} else if (util::strieq_l("dns", param)) { } else if (util::strieq_l("dns", param)) {
out.dns = true; out.dns = true;
} else if (util::strieq_l("redirect-if-not-tls", param)) { } else if (util::strieq_l("redirect-if-not-tls", param)) {
@ -1276,7 +1289,9 @@ int parse_mapping(Config *config, DownstreamAddrConfig &addr,
} else if (g.affinity.type != params.affinity.type || } else if (g.affinity.type != params.affinity.type ||
g.affinity.cookie.name != params.affinity.cookie.name || g.affinity.cookie.name != params.affinity.cookie.name ||
g.affinity.cookie.path != params.affinity.cookie.path || g.affinity.cookie.path != params.affinity.cookie.path ||
g.affinity.cookie.secure != params.affinity.cookie.secure) { g.affinity.cookie.secure != params.affinity.cookie.secure ||
g.affinity.cookie.stickiness !=
params.affinity.cookie.stickiness) {
LOG(ERROR) << "backend: affinity: multiple different affinity " LOG(ERROR) << "backend: affinity: multiple different affinity "
"configurations found in a single group"; "configurations found in a single group";
return -1; return -1;
@ -1350,6 +1365,7 @@ int parse_mapping(Config *config, DownstreamAddrConfig &addr,
make_string_ref(downstreamconf.balloc, params.affinity.cookie.path); make_string_ref(downstreamconf.balloc, params.affinity.cookie.path);
} }
g.affinity.cookie.secure = params.affinity.cookie.secure; g.affinity.cookie.secure = params.affinity.cookie.secure;
g.affinity.cookie.stickiness = params.affinity.cookie.stickiness;
} }
g.redirect_if_not_tls = params.redirect_if_not_tls; g.redirect_if_not_tls = params.redirect_if_not_tls;
g.mruby_file = make_string_ref(downstreamconf.balloc, params.mruby); g.mruby_file = make_string_ref(downstreamconf.balloc, params.mruby);
@ -4602,6 +4618,12 @@ int configure_downstream_group(Config *config, bool http2_proxy,
return -1; return -1;
} }
if (g.affinity.cookie.stickiness ==
SessionAffinityCookieStickiness::STRICT) {
addr.affinity_hash = util::hash32(key);
g.affinity_hash_map.emplace(addr.affinity_hash, idx);
}
++idx; ++idx;
} }

View File

@ -43,6 +43,7 @@
#include <vector> #include <vector>
#include <memory> #include <memory>
#include <set> #include <set>
#include <unordered_map>
#include <openssl/ssl.h> #include <openssl/ssl.h>
@ -436,6 +437,16 @@ enum class SessionAffinityCookieSecure {
NO, NO,
}; };
enum class SessionAffinityCookieStickiness {
// Backend server might be changed when an existing backend server
// is removed, or new backend server is added.
LOOSE,
// Backend server might be changed when a designated backend server
// is removed, but adding new backend server does not cause
// breakage.
STRICT,
};
struct AffinityConfig { struct AffinityConfig {
// Type of session affinity. // Type of session affinity.
SessionAffinity type; SessionAffinity type;
@ -446,6 +457,8 @@ struct AffinityConfig {
StringRef path; StringRef path;
// Secure attribute // Secure attribute
SessionAffinityCookieSecure secure; SessionAffinityCookieSecure secure;
// Affinity Stickiness
SessionAffinityCookieStickiness stickiness;
} cookie; } cookie;
}; };
@ -528,6 +541,9 @@ struct DownstreamAddrConfig {
uint32_t weight; uint32_t weight;
// weight of the weight group. Its range is [1, 256], inclusive. // weight of the weight group. Its range is [1, 256], inclusive.
uint32_t group_weight; uint32_t group_weight;
// affinity hash for this address. It is assigned when strict
// stickiness is enabled.
uint32_t affinity_hash;
// Application protocol used in this group // Application protocol used in this group
Proto proto; Proto proto;
// backend port. 0 if |host_unix| is true. // backend port. 0 if |host_unix| is true.
@ -568,6 +584,9 @@ struct DownstreamAddrGroupConfig {
// Bunch of session affinity hash. Only used if affinity == // Bunch of session affinity hash. Only used if affinity ==
// SessionAffinity::IP. // SessionAffinity::IP.
std::vector<AffinityHash> affinity_hash; std::vector<AffinityHash> affinity_hash;
// Maps affinity hash of each DownstreamAddrConfig to its index in
// addrs. It is only assigned when strict stickiness is enabled.
std::unordered_map<uint32_t, size_t> affinity_hash_map;
// Cookie based session affinity configuration. // Cookie based session affinity configuration.
AffinityConfig affinity; AffinityConfig affinity;
// true if this group requires that client connection must be TLS, // true if this group requires that client connection must be TLS,

View File

@ -90,12 +90,12 @@ DownstreamAddrGroup::~DownstreamAddrGroup() {}
// DownstreamKey is used to index SharedDownstreamAddr in order to // DownstreamKey is used to index SharedDownstreamAddr in order to
// find the same configuration. // find the same configuration.
using DownstreamKey = using DownstreamKey = std::tuple<
std::tuple<std::vector<std::tuple<StringRef, StringRef, StringRef, size_t, std::vector<
size_t, Proto, uint32_t, uint32_t, std::tuple<StringRef, StringRef, StringRef, size_t, size_t, Proto,
uint32_t, bool, bool, bool, bool>>, uint32_t, uint32_t, uint32_t, bool, bool, bool, bool>>,
bool, SessionAffinity, StringRef, StringRef, bool, SessionAffinity, StringRef, StringRef, SessionAffinityCookieSecure,
SessionAffinityCookieSecure, int64_t, int64_t, StringRef, bool>; SessionAffinityCookieStickiness, int64_t, int64_t, StringRef, bool>;
namespace { namespace {
DownstreamKey DownstreamKey
@ -131,11 +131,12 @@ create_downstream_key(const std::shared_ptr<SharedDownstreamAddr> &shared_addr,
std::get<3>(dkey) = affinity.cookie.name; std::get<3>(dkey) = affinity.cookie.name;
std::get<4>(dkey) = affinity.cookie.path; std::get<4>(dkey) = affinity.cookie.path;
std::get<5>(dkey) = affinity.cookie.secure; std::get<5>(dkey) = affinity.cookie.secure;
std::get<6>(dkey) = affinity.cookie.stickiness;
auto &timeout = shared_addr->timeout; auto &timeout = shared_addr->timeout;
std::get<6>(dkey) = timeout.read; std::get<7>(dkey) = timeout.read;
std::get<7>(dkey) = timeout.write; std::get<8>(dkey) = timeout.write;
std::get<8>(dkey) = mruby_file; std::get<9>(dkey) = mruby_file;
std::get<9>(dkey) = shared_addr->dnf; std::get<10>(dkey) = shared_addr->dnf;
return dkey; return dkey;
} }
@ -286,8 +287,10 @@ void Worker::replace_downstream_config(
make_string_ref(shared_addr->balloc, src.affinity.cookie.path); make_string_ref(shared_addr->balloc, src.affinity.cookie.path);
} }
shared_addr->affinity.cookie.secure = src.affinity.cookie.secure; shared_addr->affinity.cookie.secure = src.affinity.cookie.secure;
shared_addr->affinity.cookie.stickiness = src.affinity.cookie.stickiness;
} }
shared_addr->affinity_hash = src.affinity_hash; shared_addr->affinity_hash = src.affinity_hash;
shared_addr->affinity_hash_map = src.affinity_hash_map;
shared_addr->redirect_if_not_tls = src.redirect_if_not_tls; shared_addr->redirect_if_not_tls = src.redirect_if_not_tls;
shared_addr->dnf = src.dnf; shared_addr->dnf = src.dnf;
shared_addr->timeout.read = src.timeout.read; shared_addr->timeout.read = src.timeout.read;
@ -306,6 +309,7 @@ void Worker::replace_downstream_config(
dst_addr.weight = src_addr.weight; dst_addr.weight = src_addr.weight;
dst_addr.group = make_string_ref(shared_addr->balloc, src_addr.group); dst_addr.group = make_string_ref(shared_addr->balloc, src_addr.group);
dst_addr.group_weight = src_addr.group_weight; dst_addr.group_weight = src_addr.group_weight;
dst_addr.affinity_hash = src_addr.affinity_hash;
dst_addr.proto = src_addr.proto; dst_addr.proto = src_addr.proto;
dst_addr.tls = src_addr.tls; dst_addr.tls = src_addr.tls;
dst_addr.sni = make_string_ref(shared_addr->balloc, src_addr.sni); dst_addr.sni = make_string_ref(shared_addr->balloc, src_addr.sni);

View File

@ -133,6 +133,9 @@ struct DownstreamAddr {
// Weight of the weight group which this address belongs to. Its // Weight of the weight group which this address belongs to. Its
// range is [1, 256], inclusive. // range is [1, 256], inclusive.
uint32_t group_weight; uint32_t group_weight;
// affinity hash for this address. It is assigned when strict
// stickiness is enabled.
uint32_t affinity_hash;
// true if TLS is used in this backend // true if TLS is used in this backend
bool tls; bool tls;
// true if dynamic DNS is enabled // true if dynamic DNS is enabled
@ -215,6 +218,9 @@ struct SharedDownstreamAddr {
// Bunch of session affinity hash. Only used if affinity == // Bunch of session affinity hash. Only used if affinity ==
// SessionAffinity::IP. // SessionAffinity::IP.
std::vector<AffinityHash> affinity_hash; std::vector<AffinityHash> affinity_hash;
// Maps affinity hash of each DownstreamAddr to its index in addrs.
// It is only assigned when strict stickiness is enabled.
std::unordered_map<uint32_t, size_t> affinity_hash_map;
#ifdef HAVE_MRUBY #ifdef HAVE_MRUBY
std::shared_ptr<mruby::MRubyContext> mruby_ctx; std::shared_ptr<mruby::MRubyContext> mruby_ctx;
#endif // HAVE_MRUBY #endif // HAVE_MRUBY