From 6307f96fb35335ecdf2281faa2465d2c596d5cdf Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 12 Jul 2015 22:16:20 +0900 Subject: [PATCH] nghttpx: Enable host-path backend routing in HTTP/2 backend To achieve host-path backend routing, we changed behaviour of --backend-http2-connections-per-worker. It now sets the number of HTTP/2 physical connections per pattern group if pattern is used in -b option. Fixes GH-292 --- src/shrpx.cc | 54 ++++++++++++------------ src/shrpx_client_handler.cc | 12 +++--- src/shrpx_client_handler.h | 2 - src/shrpx_http2_downstream_connection.cc | 13 +++++- src/shrpx_http2_downstream_connection.h | 3 +- src/shrpx_http2_session.cc | 17 +++++--- src/shrpx_http2_session.h | 5 ++- src/shrpx_http2_upstream.cc | 1 + src/shrpx_http_downstream_connection.cc | 4 +- src/shrpx_https_upstream.cc | 1 + src/shrpx_ssl.cc | 1 + src/shrpx_worker.cc | 39 +++++++++++------ src/shrpx_worker.h | 24 +++++++---- 13 files changed, 106 insertions(+), 70 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 389bf7e5..7ce6aa2b 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1005,22 +1005,22 @@ Connections: with "unix:" (e.g., unix:/var/run/backend.sock). Optionally, if s are given, the backend address - is only used if request matches the pattern. If -s, -p, - --client or --http2-bridge is used, s are - ignored. The pattern matching is closely designed to - ServeMux in net/http package of Go programming language. - consists of path, host + path or just host. - The path must starts with "/". If it ends with "/", it - matches to the request path whose prefix is the path. - To deal with the request to the directory without - trailing slash, pattern which ends with "/" also matches - the path if pattern == path + "/" (e.g., pattern "/foo/" - matches path "/foo"). If it does not end with "/", it - performs exact match against the request path. If host - is given, it performs exact match against the request - host. If host alone is given, "/" is appended to it, so - that it matches all paths under the host (e.g., - specifying "nghttp2.org" equals to "nghttp2.org/"). + is only used if request matches the pattern. If -s or + -p is used, s are ignored. The pattern + matching is closely designed to ServeMux in net/http + package of Go programming language. consists + of path, host + path or just host. The path must starts + with "/". If it ends with "/", it matches to the + request path whose prefix is the path. To deal with the + request to the directory without trailing slash, pattern + which ends with "/" also matches the path if pattern == + path + "/" (e.g., pattern "/foo/" matches path "/foo"). + If it does not end with "/", it performs exact match + against the request path. If host is given, it performs + exact match against the request host. If host alone is + given, "/" is appended to it, so that it matches all + paths under the host (e.g., specifying "nghttp2.org" + equals to "nghttp2.org/"). Patterns with host take precedence over path only patterns. Then, longer patterns take precedence over @@ -1130,9 +1130,14 @@ Performance: accepts. Setting 0 means unlimited. Default: )" << get_config()->worker_frontend_connections << R"( --backend-http2-connections-per-worker= - Set maximum number of HTTP/2 connections per worker. - The default value is 0, which means the number of - backend addresses specified by -b option. + Set maximum number of backend HTTP/2 physical + connections per worker. If pattern is used in -b + option, this limit is applied to each pattern group (in + other words, each pattern group can have maximum + HTTP/2 connections). The default value is 0, which + means that the value is adjusted to the number of + backend addresses. If pattern is used, this adjustment + is done for each pattern group. --backend-http1-connections-per-host= Set maximum number of backend concurrent HTTP/1 connections per origin host. This option is meaningful @@ -2177,10 +2182,9 @@ int main(int argc, char **argv) { DownstreamAddrGroup g("/"); g.addrs.push_back(std::move(addr)); mod_config()->downstream_addr_groups.push_back(std::move(g)); - } else if (get_config()->downstream_proto == PROTO_HTTP2 || - get_config()->http2_proxy || get_config()->client_proxy) { + } else if (get_config()->http2_proxy || get_config()->client_proxy) { // We don't support host mapping in these cases. Move all - // non-catch-all patterns to catch-all pattern for HTTP/2 backend + // non-catch-all patterns to catch-all pattern. DownstreamAddrGroup catch_all("/"); for (auto &g : mod_config()->downstream_addr_groups) { std::move(std::begin(g.addrs), std::end(g.addrs), @@ -2276,12 +2280,6 @@ int main(int argc, char **argv) { } } - if (get_config()->downstream_proto == PROTO_HTTP2 && - get_config()->http2_downstream_connections_per_worker == 0) { - mod_config()->http2_downstream_connections_per_worker = - get_config()->downstream_addr_groups[0].addrs.size(); - } - if (get_config()->rlimit_nofile) { struct rlimit lim = {static_cast(get_config()->rlimit_nofile), static_cast(get_config()->rlimit_nofile)}; diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 644a8cdc..fcf8a235 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -39,6 +39,7 @@ #include "shrpx_worker.h" #include "shrpx_downstream_connection_pool.h" #include "shrpx_downstream.h" +#include "shrpx_http2_session.h" #ifdef HAVE_SPDYLAY #include "shrpx_spdy_upstream.h" #endif // HAVE_SPDYLAY @@ -361,7 +362,6 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl, get_config()->write_burst, get_config()->read_rate, get_config()->read_burst, writecb, readcb, timeoutcb, this), ipaddr_(ipaddr), port_(port), worker_(worker), - http2session_(worker_->next_http2_session()), left_connhd_len_(NGHTTP2_CLIENT_MAGIC_LEN), should_close_after_write_(false) { @@ -612,9 +612,8 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { auto catch_all = get_config()->downstream_addr_group_catch_all; // Fast path. If we have one group, it must be catch-all group. - // HTTP/2 and client proxy modes fall in this case. Currently, - // HTTP/2 backend does not perform host-path mapping. - if (groups.size() == 1 || get_config()->downstream_proto == PROTO_HTTP2) { + // HTTP/2 and client proxy modes fall in this case. + if (groups.size() == 1) { group = 0; } else if (downstream->get_request_method() == HTTP_CONNECT) { // We don't know how to treat CONNECT request in host-path @@ -653,8 +652,9 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { auto dconn_pool = worker_->get_dconn_pool(); - if (http2session_) { - dconn = make_unique(dconn_pool, http2session_); + if (get_config()->downstream_proto == PROTO_HTTP2) { + auto http2session = worker_->next_http2_session(group); + dconn = make_unique(dconn_pool, http2session); } else { dconn = make_unique(dconn_pool, group, conn_.loop); diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index 4f21f513..67dc0974 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -44,7 +44,6 @@ namespace shrpx { class Upstream; class DownstreamConnection; -class Http2Session; class HttpsUpstream; class ConnectBlocker; class DownstreamConnectionPool; @@ -142,7 +141,6 @@ private: std::function read_, write_; std::function on_read_, on_write_; Worker *worker_; - Http2Session *http2session_; // The number of bytes of HTTP/2 client connection header to read size_t left_connhd_len_; bool should_close_after_write_; diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index f82b5230..d6bc433f 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -263,8 +263,11 @@ int Http2DownstreamConnection::push_request_headers() { // http2session_ has already in CONNECTED state, so we can get // addr_idx here. auto addr_idx = http2session_->get_addr_idx(); - auto downstream_hostport = - get_config()->downstream_addr_groups[0].addrs[addr_idx].hostport.get(); + auto group = http2session_->get_group(); + auto downstream_hostport = get_config() + ->downstream_addr_groups[group] + .addrs[addr_idx] + .hostport.get(); const char *authority = nullptr, *host = nullptr; if (!no_host_rewrite) { @@ -557,4 +560,10 @@ int Http2DownstreamConnection::on_timeout() { return submit_rst_stream(downstream_, NGHTTP2_NO_ERROR); } +size_t Http2DownstreamConnection::get_group() const { + // HTTP/2 backend connections are managed by Http2Session object, + // and it stores group index. + return http2session_->get_group(); +} + } // namespace shrpx diff --git a/src/shrpx_http2_downstream_connection.h b/src/shrpx_http2_downstream_connection.h index 736f546f..da9f7ef5 100644 --- a/src/shrpx_http2_downstream_connection.h +++ b/src/shrpx_http2_downstream_connection.h @@ -61,8 +61,7 @@ public: virtual void on_upstream_change(Upstream *upstream) {} virtual int on_priority_change(int32_t pri); - // Currently, HTTP/2 backend does not perform host-path mapping. - virtual size_t get_group() const { return 0; } + virtual size_t get_group() const; // This object is not poolable because we dont' have facility to // migrate to another Http2Session object. diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 5ed2f3c3..64c9feb9 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -142,13 +142,14 @@ void writecb(struct ev_loop *loop, ev_io *w, int revents) { } // namespace Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, - ConnectBlocker *connect_blocker, Worker *worker) + ConnectBlocker *connect_blocker, Worker *worker, + size_t group) : conn_(loop, -1, nullptr, get_config()->downstream_write_timeout, get_config()->downstream_read_timeout, 0, 0, 0, 0, writecb, readcb, timeoutcb, this), worker_(worker), connect_blocker_(connect_blocker), ssl_ctx_(ssl_ctx), session_(nullptr), data_pending_(nullptr), data_pendinglen_(0), - addr_idx_(0), state_(DISCONNECTED), + addr_idx_(0), group_(group), state_(DISCONNECTED), connection_check_state_(CONNECTION_CHECK_NONE), flow_control_(false) { read_ = write_ = &Http2Session::noop; @@ -235,13 +236,14 @@ int Http2Session::disconnect(bool hard) { int Http2Session::check_cert() { return ssl::check_cert( - conn_.tls.ssl, &get_config()->downstream_addr_groups[0].addrs[addr_idx_]); + conn_.tls.ssl, + &get_config()->downstream_addr_groups[group_].addrs[addr_idx_]); } int Http2Session::initiate_connection() { int rv = 0; - auto &addrs = get_config()->downstream_addr_groups[0].addrs; + auto &addrs = get_config()->downstream_addr_groups[group_].addrs; if (state_ == DISCONNECTED) { if (connect_blocker_->blocked()) { @@ -252,8 +254,7 @@ int Http2Session::initiate_connection() { return -1; } - auto worker_stat = worker_->get_worker_stat(); - auto &next_downstream = worker_stat->next_downstream[0]; + auto &next_downstream = worker_->get_dgrp(group_)->next; addr_idx_ = next_downstream; if (++next_downstream >= addrs.size()) { next_downstream = 0; @@ -511,7 +512,7 @@ int Http2Session::downstream_connect_proxy() { SSLOG(INFO, this) << "Connected to the proxy"; } auto &downstream_addr = - get_config()->downstream_addr_groups[0].addrs[addr_idx_]; + get_config()->downstream_addr_groups[group_].addrs[addr_idx_]; std::string req = "CONNECT "; req += downstream_addr.hostport.get(); @@ -1752,4 +1753,6 @@ bool Http2Session::should_hard_fail() const { size_t Http2Session::get_addr_idx() const { return addr_idx_; } +size_t Http2Session::get_group() const { return group_; } + } // namespace shrpx diff --git a/src/shrpx_http2_session.h b/src/shrpx_http2_session.h index 6f642063..184252de 100644 --- a/src/shrpx_http2_session.h +++ b/src/shrpx_http2_session.h @@ -58,7 +58,7 @@ struct StreamData { class Http2Session { public: Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, - ConnectBlocker *connect_blocker, Worker *worker); + ConnectBlocker *connect_blocker, Worker *worker, size_t group); ~Http2Session(); int check_cert(); @@ -151,6 +151,8 @@ public: size_t get_addr_idx() const; + size_t get_group() const; + enum { // Disconnected DISCONNECTED, @@ -203,6 +205,7 @@ private: size_t data_pendinglen_; // index of get_config()->downstream_addrs this object uses size_t addr_idx_; + size_t group_; int state_; int connection_check_state_; bool flow_control_; diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 07ac4e3d..97ff206c 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -36,6 +36,7 @@ #include "shrpx_config.h" #include "shrpx_http.h" #include "shrpx_worker.h" +#include "shrpx_http2_session.h" #include "http2.h" #include "util.h" #include "base64.h" diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index d0181b7e..a69357a5 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -34,6 +34,7 @@ #include "shrpx_connect_blocker.h" #include "shrpx_downstream_connection_pool.h" #include "shrpx_worker.h" +#include "shrpx_http2_session.h" #include "http2.h" #include "util.h" @@ -142,8 +143,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { } auto worker = client_handler_->get_worker(); - auto worker_stat = worker->get_worker_stat(); - auto &next_downstream = worker_stat->next_downstream[group_]; + auto &next_downstream = worker->get_dgrp(group_)->next; auto end = next_downstream; auto &addrs = get_config()->downstream_addr_groups[group_].addrs; for (;;) { diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 1a832e60..22c048a3 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -36,6 +36,7 @@ #include "shrpx_error.h" #include "shrpx_log_config.h" #include "shrpx_worker.h" +#include "shrpx_http2_session.h" #include "http2.h" #include "util.h" #include "template.h" diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 60526653..9f95fa2f 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -53,6 +53,7 @@ #include "shrpx_config.h" #include "shrpx_worker.h" #include "shrpx_downstream_connection_pool.h" +#include "shrpx_http2_session.h" #include "util.h" #include "ssl.h" #include "template.h" diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index a756e7c0..e5f0f8d4 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -61,9 +61,9 @@ void mcpool_clear_cb(struct ev_loop *loop, ev_timer *w, int revents) { Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, ssl::CertLookupTree *cert_tree, const std::shared_ptr &ticket_keys) - : next_http2session_(0), - dconn_pool_(get_config()->downstream_addr_groups.size()), - worker_stat_(get_config()->downstream_addr_groups.size()), loop_(loop), + : dconn_pool_(get_config()->downstream_addr_groups.size()), + worker_stat_(get_config()->downstream_addr_groups.size()), + dgrps_(get_config()->downstream_addr_groups.size()), loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), cert_tree_(cert_tree), ticket_keys_(ticket_keys), connect_blocker_(make_unique(loop_)), @@ -77,9 +77,17 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, if (get_config()->downstream_proto == PROTO_HTTP2) { auto n = get_config()->http2_downstream_connections_per_worker; - for (; n > 0; --n) { - http2sessions_.push_back(make_unique( - loop_, cl_ssl_ctx, connect_blocker_.get(), this)); + size_t group = 0; + for (auto &dgrp : dgrps_) { + auto m = n; + if (m == 0) { + m = get_config()->downstream_addr_groups[group].addrs.size(); + } + for (; m; --m) { + dgrp.http2sessions.push_back(make_unique( + loop_, cl_ssl_ctx, connect_blocker_.get(), this, group)); + } + ++group; } } } @@ -210,15 +218,17 @@ WorkerStat *Worker::get_worker_stat() { return &worker_stat_; } DownstreamConnectionPool *Worker::get_dconn_pool() { return &dconn_pool_; } -Http2Session *Worker::next_http2_session() { - if (http2sessions_.empty()) { +Http2Session *Worker::next_http2_session(size_t group) { + auto &dgrp = dgrps_[group]; + auto &http2sessions = dgrp.http2sessions; + if (http2sessions.empty()) { return nullptr; } - auto res = http2sessions_[next_http2session_].get(); - ++next_http2session_; - if (next_http2session_ >= http2sessions_.size()) { - next_http2session_ = 0; + auto res = http2sessions[dgrp.next_http2session].get(); + ++dgrp.next_http2session; + if (dgrp.next_http2session >= http2sessions.size()) { + dgrp.next_http2session = 0; } return res; @@ -242,4 +252,9 @@ bool Worker::get_graceful_shutdown() const { return graceful_shutdown_; } MemchunkPool *Worker::get_mcpool() { return &mcpool_; } +DownstreamGroup *Worker::get_dgrp(size_t group) { + assert(group < dgrps_.size()); + return &dgrps_[group]; +} + } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index f9924647..b5f820b4 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -54,14 +54,21 @@ namespace ssl { class CertLookupTree; } // namespace ssl +struct DownstreamGroup { + DownstreamGroup() : next_http2session(0), next(0) {} + + std::vector> http2sessions; + // Next index in http2sessions. + size_t next_http2session; + // Next downstream address index corresponding to + // Config::downstream_addr_groups[]. + size_t next; +}; + struct WorkerStat { - WorkerStat(size_t num_groups) - : num_connections(0), next_downstream(num_groups) {} + WorkerStat(size_t num_groups) : num_connections(0) {} size_t num_connections; - // Next downstream index in Config::downstream_addr_groups. This is - // used as load balancing. - std::vector next_downstream; }; enum WorkerEventType { @@ -97,7 +104,7 @@ public: void set_ticket_keys(std::shared_ptr ticket_keys); WorkerStat *get_worker_stat(); DownstreamConnectionPool *get_dconn_pool(); - Http2Session *next_http2_session(); + Http2Session *next_http2_session(size_t group); ConnectBlocker *get_connect_blocker() const; struct ev_loop *get_loop() const; SSL_CTX *get_sv_ssl_ctx() const; @@ -109,9 +116,9 @@ public: MemchunkPool *get_mcpool(); void schedule_clear_mcpool(); + DownstreamGroup *get_dgrp(size_t group); + private: - std::vector> http2sessions_; - size_t next_http2session_; #ifndef NOTHREADS std::future fut_; #endif // NOTHREADS @@ -122,6 +129,7 @@ private: MemchunkPool mcpool_; DownstreamConnectionPool dconn_pool_; WorkerStat worker_stat_; + std::vector dgrps_; struct ev_loop *loop_; // Following fields are shared across threads if