diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index 4800613f..03500013 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -14,9 +14,10 @@ If nghttpx is invoked without any ``-s``, ``-p`` and ``--client``, it operates in default mode. In this mode, nghttpx frontend listens for HTTP/2 requests and translates them to HTTP/1 requests. Thus it works as reverse proxy (gateway) for HTTP/2 clients to HTTP/1 web server. -HTTP/1 requests are also supported in frontend as a fallback. If -nghttpx is linked with spdylay library and frontend connection is -SSL/TLS, the frontend also supports SPDY protocol. +This is also known as "HTTP/2 router". HTTP/1 requests are also +supported in frontend as a fallback. If nghttpx is linked with +spdylay library and frontend connection is SSL/TLS, the frontend also +supports SPDY protocol. By default, this mode's frontend connection is encrypted using SSL/TLS. So server's private key and certificate must be supplied to @@ -30,6 +31,10 @@ available on the frontend and a HTTP/1 connection can be upgraded to HTTP/2 using HTTP Upgrade. Starting HTTP/2 connection by sending HTTP/2 connection preface is also supported. +By default, backend HTTP/1 connections are not encrypted. To enable +TLS on HTTP/1 backend connections, use ``--backend-http1-tls`` option. +This applies to all mode whose backend connections are HTTP/1. + The backend is supposed to be HTTP/1 Web server. For example, to make nghttpx listen to encrypted HTTP/2 requests at port 8443, and a backend HTTP/1 web server is configured to listen to HTTP/1 request at diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 627549aa..f75add0b 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -112,7 +112,9 @@ OPTIONS = [ "max-request-header-fields", "header-field-buffer", "max-header-fields", - "no-http2-cipher-black-list" + "no-http2-cipher-black-list", + "backend-http1-tls", + "backend-tls-session-cache-per-worker" ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index 40c28650..5ccbc17e 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1046,8 +1046,6 @@ void fill_default_config() { auto &tlsconf = mod_config()->tls; { auto &ticketconf = tlsconf.ticket; - ticketconf.cipher = EVP_aes_128_cbc(); - { auto &memcachedconf = ticketconf.memcached; memcachedconf.max_retry = 3; @@ -1055,19 +1053,26 @@ void fill_default_config() { memcachedconf.interval = 10_min; } + ticketconf.cipher = EVP_aes_128_cbc(); + } + + { auto &ocspconf = tlsconf.ocsp; // ocsp update interval = 14400 secs = 4 hours, borrowed from h2o ocspconf.update_interval = 4_h; ocspconf.fetch_ocsp_response_file = strcopy(PKGDATADIR "/fetch-ocsp-response"); + } + { auto &dyn_recconf = tlsconf.dyn_rec; dyn_recconf.warmup_threshold = 1_m; dyn_recconf.idle_timeout = 1_s; - - tlsconf.session_timeout = std::chrono::hours(12); } + tlsconf.session_timeout = std::chrono::hours(12); + tlsconf.downstream_session_cache_per_worker = 10000; + auto &httpconf = mod_config()->http; httpconf.server_name = "nghttpx nghttp2/" NGHTTP2_VERSION; httpconf.no_host_rewrite = true; @@ -1277,6 +1282,16 @@ Connections: --backend-write-timeout options. --accept-proxy-protocol Accept PROXY protocol version 1 on frontend connection. + --backend-no-tls + Disable SSL/TLS on backend connections. For HTTP/2 + backend connections, TLS is enabled by default. For + HTTP/1 backend connections, TLS is disabled by default, + and can be enabled by --backend-http1-tls option. If + both --backend-no-tls and --backend-http1-tls options + are used, --backend-no-tls has the precedence. + --backend-http1-tls + Enable SSL/TLS on backend HTTP/1 connections. See also + --backend-no-tls option. Performance: -n, --workers= @@ -1426,16 +1441,14 @@ SSL/TLS: Set allowed cipher list. The format of the string is described in OpenSSL ciphers(1). -k, --insecure - Don't verify backend server's certificate if -p, - --client or --http2-bridge are given and - --backend-no-tls is not given. + Don't verify backend server's certificate if TLS is + enabled for backend connections. --cacert= - Set path to trusted CA certificate file if -p, --client - or --http2-bridge are given and --backend-no-tls is not - given. The file must be in PEM format. It can contain - multiple certificates. If the linked OpenSSL is - configured to load system wide certificates, they are - loaded at startup regardless of this option. + Set path to trusted CA certificate file used in backend + TLS connections. The file must be in PEM format. It + can contain multiple certificates. If the linked + OpenSSL is configured to load system wide certificates, + they are loaded at startup regardless of this option. --private-key-passwd-file= Path to file that contains password for the server's private key. If none is given and the private key is @@ -1575,6 +1588,11 @@ SSL/TLS: Allow black listed cipher suite on HTTP/2 connection. See https://tools.ietf.org/html/rfc7540#appendix-A for the complete HTTP/2 cipher suites black list. + --backend-tls-session-cache-per-worker= + Set the maximum number of backend TLS session cache + stored per worker. + Default: )" + << get_config()->tls.downstream_session_cache_per_worker << R"( HTTP/2 and SPDY: -c, --http2-max-concurrent-streams= @@ -1603,8 +1621,6 @@ HTTP/2 and SPDY: connection to 2**-1. Default: )" << get_config()->http2.downstream.connection_window_bits << R"( - --backend-no-tls - Disable SSL/TLS on backend connections. --http2-no-cookie-crumbling Don't crumble cookie header field. --padding= @@ -2029,6 +2045,10 @@ void process_options( downstreamconf.proto = PROTO_HTTP; } + if (downstreamconf.proto == PROTO_HTTP && !downstreamconf.http1_tls) { + downstreamconf.no_tls = true; + } + if (!upstreamconf.no_tls && (!tlsconf.private_key_file || !tlsconf.cert_file)) { print_usage(std::cerr); @@ -2377,6 +2397,9 @@ int main(int argc, char **argv) { {SHRPX_OPT_NO_HTTP2_CIPHER_BLACK_LIST, no_argument, &flag, 103}, {SHRPX_OPT_REQUEST_HEADER_FIELD_BUFFER, required_argument, &flag, 104}, {SHRPX_OPT_MAX_REQUEST_HEADER_FIELDS, required_argument, &flag, 105}, + {SHRPX_OPT_BACKEND_HTTP1_TLS, no_argument, &flag, 106}, + {SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER, required_argument, + &flag, 107}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2826,6 +2849,15 @@ int main(int argc, char **argv) { // --max-request-header-fields cmdcfgs.emplace_back(SHRPX_OPT_MAX_REQUEST_HEADER_FIELDS, optarg); break; + case 106: + // --backend-http1-tls + cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_HTTP1_TLS, "yes"); + break; + case 107: + // --backend-tls-session-cache-per-worker + cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER, + optarg); + break; default: break; } diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index aa0f6195..30f2fc4e 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -722,8 +722,8 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { } dconn = make_unique(dconn_pool, http2session); } else { - dconn = - make_unique(dconn_pool, group, conn_.loop); + dconn = make_unique(dconn_pool, group, + conn_.loop, worker_); } dconn->set_client_handler(this); return dconn; diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index d90c12d3..ed61378f 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -676,6 +676,7 @@ enum { SHRPX_OPTID_BACKEND_HTTP_PROXY_URI, SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND, SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST, + SHRPX_OPTID_BACKEND_HTTP1_TLS, SHRPX_OPTID_BACKEND_HTTP2_CONNECTION_WINDOW_BITS, SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER, SHRPX_OPTID_BACKEND_HTTP2_WINDOW_BITS, @@ -686,6 +687,7 @@ enum { SHRPX_OPTID_BACKEND_READ_TIMEOUT, SHRPX_OPTID_BACKEND_REQUEST_BUFFER, SHRPX_OPTID_BACKEND_RESPONSE_BUFFER, + SHRPX_OPTID_BACKEND_TLS_SESSION_CACHE_PER_WORKER, SHRPX_OPTID_BACKEND_TLS_SNI_FIELD, SHRPX_OPTID_BACKEND_WRITE_TIMEOUT, SHRPX_OPTID_BACKLOG, @@ -1077,6 +1079,9 @@ int option_lookup_token(const char *name, size_t namelen) { } break; case 's': + if (util::strieq_l("backend-http1-tl", name, 16)) { + return SHRPX_OPTID_BACKEND_HTTP1_TLS; + } if (util::strieq_l("max-header-field", name, 16)) { return SHRPX_OPTID_MAX_HEADER_FIELDS; } @@ -1378,6 +1383,9 @@ int option_lookup_token(const char *name, size_t namelen) { if (util::strieq_l("backend-http2-connections-per-worke", name, 35)) { return SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER; } + if (util::strieq_l("backend-tls-session-cache-per-worke", name, 35)) { + return SHRPX_OPTID_BACKEND_TLS_SESSION_CACHE_PER_WORKER; + } break; case 's': if (util::strieq_l("backend-http2-connection-window-bit", name, 35)) { @@ -2214,6 +2222,13 @@ int parse_config(const char *opt, const char *optarg, mod_config()->tls.no_http2_cipher_black_list = util::strieq(optarg, "yes"); return 0; + case SHRPX_OPTID_BACKEND_HTTP1_TLS: + mod_config()->conn.downstream.http1_tls = util::strieq(optarg, "yes"); + + return 0; + case SHRPX_OPTID_BACKEND_TLS_SESSION_CACHE_PER_WORKER: + return parse_uint(&mod_config()->tls.downstream_session_cache_per_worker, + opt, optarg); case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 02f027d1..614e7a25 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -206,6 +206,9 @@ constexpr char SHRPX_OPT_MAX_RESPONSE_HEADER_FIELDS[] = "max-response-header-fields"; constexpr char SHRPX_OPT_NO_HTTP2_CIPHER_BLACK_LIST[] = "no-http2-cipher-black-list"; +constexpr char SHRPX_OPT_BACKEND_HTTP1_TLS[] = "backend-http1-tls"; +constexpr char SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER[] = + "backend-tls-session-cache-per-worker"; constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -390,6 +393,7 @@ struct TLSConfig { std::vector npn_list; // list of supported SSL/TLS protocol strings. std::vector tls_proto_list; + size_t downstream_session_cache_per_worker; // Bit mask to disable SSL/TLS protocol versions. This will be // passed to SSL_CTX_set_options(). long int tls_proto_mask; @@ -534,6 +538,7 @@ struct ConnectionConfig { // downstream protocol; this will be determined by given options. shrpx_proto proto; bool no_tls; + bool http1_tls; // true if IPv4 only; ipv4 and ipv6 are mutually exclusive; and // (ipv4 && ipv6) must be false. bool ipv4; diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 2c39fe50..3ee4d3f1 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -35,6 +35,7 @@ #include "shrpx_downstream_connection_pool.h" #include "shrpx_worker.h" #include "shrpx_http2_session.h" +#include "shrpx_ssl.h" #include "http2.h" #include "util.h" @@ -100,7 +101,7 @@ void connectcb(struct ev_loop *loop, ev_io *w, int revents) { auto downstream = dconn->get_downstream(); auto upstream = downstream->get_upstream(); auto handler = upstream->get_client_handler(); - if (dconn->on_connect() != 0) { + if (dconn->connected() != 0) { if (upstream->on_downstream_abort_request(downstream, 503) != 0) { delete handler; } @@ -111,20 +112,34 @@ void connectcb(struct ev_loop *loop, ev_io *w, int revents) { } // namespace HttpDownstreamConnection::HttpDownstreamConnection( - DownstreamConnectionPool *dconn_pool, size_t group, struct ev_loop *loop) + DownstreamConnectionPool *dconn_pool, size_t group, struct ev_loop *loop, + Worker *worker) : DownstreamConnection(dconn_pool), - conn_(loop, -1, nullptr, nullptr, + conn_(loop, -1, nullptr, worker->get_mcpool(), get_config()->conn.downstream.timeout.write, get_config()->conn.downstream.timeout.read, {}, {}, connectcb, readcb, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold, get_config()->tls.dyn_rec.idle_timeout), + do_read_(&HttpDownstreamConnection::noop), + do_write_(&HttpDownstreamConnection::noop), + worker_(worker), + ssl_ctx_(worker->get_cl_ssl_ctx()), ioctrl_(&conn_.rlimit), response_htp_{0}, group_(group), - addr_idx_(0), - connected_(false) {} + addr_idx_(0) {} -HttpDownstreamConnection::~HttpDownstreamConnection() {} +HttpDownstreamConnection::~HttpDownstreamConnection() { + if (conn_.tls.ssl) { + auto session = SSL_get1_session(conn_.tls.ssl); + if (session) { + auto &downstreamconf = get_config()->conn.downstream; + auto &addr = downstreamconf.addr_groups[group_].addrs[addr_idx_]; + + worker_->cache_downstream_tls_session(&addr, session); + } + } +} int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { if (LOG_ENABLED(INFO)) { @@ -144,8 +159,16 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { return -1; } - auto worker = client_handler_->get_worker(); - auto &next_downstream = worker->get_dgrp(group_)->next; + if (ssl_ctx_) { + auto ssl = ssl::create_ssl(ssl_ctx_); + if (!ssl) { + return -1; + } + + conn_.set_ssl(ssl); + } + + auto &next_downstream = worker_->get_dgrp(group_)->next; auto end = next_downstream; auto &addrs = downstreamconf.addr_groups[group_].addrs; for (;;) { @@ -190,6 +213,23 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { addr_idx_ = i; + if (ssl_ctx_) { + auto sni_name = !get_config()->tls.backend_sni_name.empty() + ? StringRef(get_config()->tls.backend_sni_name) + : StringRef(addrs[i].host); + if (!util::numeric_host(sni_name.c_str())) { + SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str()); + } + + auto session = worker_->reuse_downstream_tls_session(&addrs[i]); + if (session) { + SSL_set_session(conn_.tls.ssl, session); + SSL_SESSION_free(session); + } + + conn_.prepare_client_handshake(); + } + ev_io_set(&conn_.wev, conn_.fd, EV_WRITE); ev_io_set(&conn_.rev, conn_.fd, EV_READ); @@ -748,44 +788,13 @@ http_parser_settings htp_hooks = { }; } // namespace -int HttpDownstreamConnection::on_read() { - if (!connected_) { - return 0; - } - +int HttpDownstreamConnection::read_clear() { ev_timer_again(conn_.loop, &conn_.rt); std::array buf; int rv; - if (downstream_->get_upgraded()) { - // For upgraded connection, just pass data to the upstream. - for (;;) { - auto nread = conn_.read_clear(buf.data(), buf.size()); - - if (nread == 0) { - return 0; - } - - if (nread < 0) { - return nread; - } - - rv = downstream_->get_upstream()->on_downstream_body( - downstream_, buf.data(), nread, true); - if (rv != 0) { - return rv; - } - - if (downstream_->response_buf_full()) { - downstream_->pause_read(SHRPX_NO_BUFFER); - return 0; - } - } - } - for (;;) { auto nread = conn_.read_clear(buf.data(), buf.size()); - if (nread == 0) { return 0; } @@ -794,59 +803,18 @@ int HttpDownstreamConnection::on_read() { return nread; } - auto nproc = - http_parser_execute(&response_htp_, &htp_hooks, - reinterpret_cast(buf.data()), nread); - - auto htperr = HTTP_PARSER_ERRNO(&response_htp_); - - if (htperr != HPE_OK) { - // Handling early return (in other words, response was hijacked - // by mruby scripting). - if (downstream_->get_response_state() == Downstream::MSG_COMPLETE) { - return SHRPX_ERR_DCONN_CANCELED; - } - - if (LOG_ENABLED(INFO)) { - DCLOG(INFO, this) << "HTTP parser failure: " - << "(" << http_errno_name(htperr) << ") " - << http_errno_description(htperr); - } - - return -1; + rv = process_input(buf.data(), nread); + if (rv != 0) { + return rv; } - if (downstream_->get_upgraded()) { - if (nproc < static_cast(nread)) { - // Data from buf.data() + nproc are for upgraded protocol. - rv = downstream_->get_upstream()->on_downstream_body( - downstream_, buf.data() + nproc, nread - nproc, true); - if (rv != 0) { - return rv; - } - - if (downstream_->response_buf_full()) { - downstream_->pause_read(SHRPX_NO_BUFFER); - return 0; - } - } - // call on_read(), so that we can process data left in buffer as - // upgrade. - return on_read(); - } - - if (downstream_->response_buf_full()) { - downstream_->pause_read(SHRPX_NO_BUFFER); + if (!ev_is_active(&conn_.rev)) { return 0; } } } -int HttpDownstreamConnection::on_write() { - if (!connected_) { - return 0; - } - +int HttpDownstreamConnection::write_clear() { ev_timer_again(conn_.loop, &conn_.rt); auto upstream = downstream_->get_upstream(); @@ -883,7 +851,172 @@ int HttpDownstreamConnection::on_write() { return 0; } -int HttpDownstreamConnection::on_connect() { +int HttpDownstreamConnection::tls_handshake() { + ERR_clear_error(); + + ev_timer_again(conn_.loop, &conn_.rt); + + auto rv = conn_.tls_handshake(); + if (rv == SHRPX_ERR_INPROGRESS) { + return 0; + } + + if (rv < 0) { + return rv; + } + + if (LOG_ENABLED(INFO)) { + DCLOG(INFO, this) << "SSL/TLS handshake completed"; + } + + if (!get_config()->tls.insecure && + ssl::check_cert(conn_.tls.ssl, &get_config() + ->conn.downstream.addr_groups[group_] + .addrs[addr_idx_]) != 0) { + return -1; + } + + do_read_ = &HttpDownstreamConnection::read_tls; + do_write_ = &HttpDownstreamConnection::write_tls; + + // TODO Check negotiated ALPN + + return on_write(); +} + +int HttpDownstreamConnection::read_tls() { + ERR_clear_error(); + + ev_timer_again(conn_.loop, &conn_.rt); + std::array buf; + int rv; + + for (;;) { + auto nread = conn_.read_tls(buf.data(), buf.size()); + if (nread == 0) { + return 0; + } + + if (nread < 0) { + return nread; + } + + rv = process_input(buf.data(), nread); + if (rv != 0) { + return rv; + } + + if (!ev_is_active(&conn_.rev)) { + return 0; + } + } +} + +int HttpDownstreamConnection::write_tls() { + ERR_clear_error(); + + ev_timer_again(conn_.loop, &conn_.rt); + + auto upstream = downstream_->get_upstream(); + auto input = downstream_->get_request_buf(); + + struct iovec iov; + + while (input->rleft() > 0) { + auto iovcnt = input->riovec(&iov, 1); + assert(iovcnt == 1); + auto nwrite = conn_.write_tls(iov.iov_base, iov.iov_len); + + if (nwrite == 0) { + return 0; + } + + if (nwrite < 0) { + return nwrite; + } + + input->drain(nwrite); + } + + conn_.wlimit.stopw(); + ev_timer_stop(conn_.loop, &conn_.wt); + + if (input->rleft() == 0) { + auto &req = downstream_->request(); + + upstream->resume_read(SHRPX_NO_BUFFER, downstream_, + req.unconsumed_body_length); + } + + return 0; +} + +int HttpDownstreamConnection::process_input(const uint8_t *data, + size_t datalen) { + int rv; + + if (downstream_->get_upgraded()) { + // For upgraded connection, just pass data to the upstream. + rv = downstream_->get_upstream()->on_downstream_body(downstream_, data, + datalen, true); + if (rv != 0) { + return rv; + } + + if (downstream_->response_buf_full()) { + downstream_->pause_read(SHRPX_NO_BUFFER); + return 0; + } + } + + auto nproc = + http_parser_execute(&response_htp_, &htp_hooks, + reinterpret_cast(data), datalen); + + auto htperr = HTTP_PARSER_ERRNO(&response_htp_); + + if (htperr != HPE_OK) { + // Handling early return (in other words, response was hijacked by + // mruby scripting). + if (downstream_->get_response_state() == Downstream::MSG_COMPLETE) { + return SHRPX_ERR_DCONN_CANCELED; + } + + if (LOG_ENABLED(INFO)) { + DCLOG(INFO, this) << "HTTP parser failure: " + << "(" << http_errno_name(htperr) << ") " + << http_errno_description(htperr); + } + + return -1; + } + + if (downstream_->get_upgraded()) { + if (nproc < datalen) { + // Data from data + nproc are for upgraded protocol. + rv = downstream_->get_upstream()->on_downstream_body( + downstream_, data + nproc, datalen - nproc, true); + if (rv != 0) { + return rv; + } + + if (downstream_->response_buf_full()) { + downstream_->pause_read(SHRPX_NO_BUFFER); + return 0; + } + } + return 0; + } + + if (downstream_->response_buf_full()) { + downstream_->pause_read(SHRPX_NO_BUFFER); + return 0; + } + + return 0; +} + +int HttpDownstreamConnection::connected() { auto connect_blocker = client_handler_->get_connect_blocker(); if (!util::check_socket_connected(conn_.fd)) { @@ -898,18 +1031,33 @@ int HttpDownstreamConnection::on_connect() { return -1; } - connected_ = true; + if (LOG_ENABLED(INFO)) { + DLOG(INFO, this) << "Connected to downstream host"; + } connect_blocker->on_success(); conn_.rlimit.startw(); - ev_timer_again(conn_.loop, &conn_.rt); ev_set_cb(&conn_.wev, writecb); + if (conn_.tls.ssl) { + do_read_ = &HttpDownstreamConnection::tls_handshake; + do_write_ = &HttpDownstreamConnection::tls_handshake; + + return 0; + } + + do_read_ = &HttpDownstreamConnection::read_clear; + do_write_ = &HttpDownstreamConnection::write_clear; + return 0; } +int HttpDownstreamConnection::on_read() { return do_read_(*this); } + +int HttpDownstreamConnection::on_write() { return do_write_(*this); } + void HttpDownstreamConnection::on_upstream_change(Upstream *upstream) {} void HttpDownstreamConnection::signal_write() { @@ -918,4 +1066,6 @@ void HttpDownstreamConnection::signal_write() { size_t HttpDownstreamConnection::get_group() const { return group_; } +int HttpDownstreamConnection::noop() { return 0; } + } // namespace shrpx diff --git a/src/shrpx_http_downstream_connection.h b/src/shrpx_http_downstream_connection.h index 7c82be49..a905a9fc 100644 --- a/src/shrpx_http_downstream_connection.h +++ b/src/shrpx_http_downstream_connection.h @@ -36,11 +36,12 @@ namespace shrpx { class DownstreamConnectionPool; +class Worker; class HttpDownstreamConnection : public DownstreamConnection { public: HttpDownstreamConnection(DownstreamConnectionPool *dconn_pool, size_t group, - struct ev_loop *loop); + struct ev_loop *loop, Worker *worker); virtual ~HttpDownstreamConnection(); virtual int attach_downstream(Downstream *downstream); virtual void detach_downstream(Downstream *downstream); @@ -61,17 +62,30 @@ public: virtual bool poolable() const { return true; } - int on_connect(); + int read_clear(); + int write_clear(); + int read_tls(); + int write_tls(); + + int process_input(const uint8_t *data, size_t datalen); + int tls_handshake(); + + int connected(); void signal_write(); + int noop(); + private: Connection conn_; + std::function do_read_, do_write_; + Worker *worker_; + // nullptr if TLS is not used. + SSL_CTX *ssl_ctx_; IOControl ioctrl_; http_parser response_htp_; size_t group_; // index of get_config()->downstream_addrs this object is using size_t addr_idx_; - bool connected_; }; } // namespace shrpx diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 6b7883df..115fbf91 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -628,9 +628,9 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file } namespace { -int select_next_proto_cb(SSL *ssl, unsigned char **out, unsigned char *outlen, - const unsigned char *in, unsigned int inlen, - void *arg) { +int select_h2_next_proto_cb(SSL *ssl, unsigned char **out, + unsigned char *outlen, const unsigned char *in, + unsigned int inlen, void *arg) { if (!util::select_h2(const_cast(out), outlen, in, inlen)) { return SSL_TLSEXT_ERR_NOACK; @@ -640,6 +640,24 @@ int select_next_proto_cb(SSL *ssl, unsigned char **out, unsigned char *outlen, } } // namespace +namespace { +int select_h1_next_proto_cb(SSL *ssl, unsigned char **out, + unsigned char *outlen, const unsigned char *in, + unsigned int inlen, void *arg) { + auto end = in + inlen; + for (; in < end;) { + if (util::streq_l(NGHTTP2_H1_1_ALPN, in, in[0] + 1)) { + *out = const_cast(in) + 1; + *outlen = in[0]; + return SSL_TLSEXT_ERR_OK; + } + in += in[0] + 1; + } + + return SSL_TLSEXT_ERR_NOACK; +} +} // namespace + SSL_CTX *create_ssl_client_context( #ifdef HAVE_NEVERBLEED neverbleed_t *nb @@ -722,15 +740,29 @@ SSL_CTX *create_ssl_client_context( DIE(); } } - // NPN selection callback - SSL_CTX_set_next_proto_select_cb(ssl_ctx, select_next_proto_cb, nullptr); + + auto &downstreamconf = get_config()->conn.downstream; + + if (downstreamconf.proto == PROTO_HTTP2) { + // NPN selection callback + SSL_CTX_set_next_proto_select_cb(ssl_ctx, select_h2_next_proto_cb, nullptr); #if OPENSSL_VERSION_NUMBER >= 0x10002000L - // ALPN advertisement; We only advertise HTTP/2 - auto proto_list = util::get_default_alpn(); + // ALPN advertisement; We only advertise HTTP/2 + auto proto_list = util::get_default_alpn(); - SSL_CTX_set_alpn_protos(ssl_ctx, proto_list.data(), proto_list.size()); + SSL_CTX_set_alpn_protos(ssl_ctx, proto_list.data(), proto_list.size()); #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L + } else { + // NPN selection callback + SSL_CTX_set_next_proto_select_cb(ssl_ctx, select_h1_next_proto_cb, nullptr); + +#if OPENSSL_VERSION_NUMBER >= 0x10002000L + SSL_CTX_set_alpn_protos( + ssl_ctx, reinterpret_cast(NGHTTP2_H1_1_ALPN), + str_size(NGHTTP2_H1_1_ALPN)); +#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L + } return ssl_ctx; } @@ -1270,15 +1302,7 @@ SSL_CTX *setup_server_ssl_context(std::vector &all_ssl_ctx, return ssl_ctx; } -bool downstream_tls_enabled() { - auto no_tls = get_config()->conn.downstream.no_tls; - - if (get_config()->client_mode) { - return !no_tls; - } - - return get_config()->http2_bridge && !no_tls; -} +bool downstream_tls_enabled() { return !get_config()->conn.downstream.no_tls; } SSL_CTX *setup_client_ssl_context( #ifdef HAVE_NEVERBLEED diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 255d23aa..2204b9af 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -73,6 +73,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, dconn_pool_(get_config()->conn.downstream.addr_groups.size()), worker_stat_(get_config()->conn.downstream.addr_groups.size()), dgrps_(get_config()->conn.downstream.addr_groups.size()), + downstream_tls_session_cache_size_(0), loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), @@ -116,6 +117,12 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, Worker::~Worker() { ev_async_stop(loop_, &w_); ev_timer_stop(loop_, &mcpool_clear_timer_); + + for (auto &p : downstream_tls_session_cache_) { + for (auto session : p.second) { + SSL_SESSION_free(session); + } + } } void Worker::schedule_clear_mcpool() { @@ -300,4 +307,57 @@ mruby::MRubyContext *Worker::get_mruby_context() const { } #endif // HAVE_MRUBY +void Worker::cache_downstream_tls_session(const DownstreamAddr *addr, + SSL_SESSION *session) { + auto &tlsconf = get_config()->tls; + + auto max = tlsconf.downstream_session_cache_per_worker; + if (max == 0) { + return; + } + + if (downstream_tls_session_cache_size_ >= max) { + // It is implementation dependent which item is returned from + // std::begin(). Probably, this depends on hash algorithm. If it + // is random fashion, then we are mostly OK. + auto it = std::begin(downstream_tls_session_cache_); + assert(it != std::end(downstream_tls_session_cache_)); + auto &v = (*it).second; + assert(!v.empty()); + auto sess = v.front(); + v.pop_front(); + SSL_SESSION_free(sess); + if (v.empty()) { + downstream_tls_session_cache_.erase(it); + } + } + + auto it = downstream_tls_session_cache_.find(addr); + if (it == std::end(downstream_tls_session_cache_)) { + std::tie(it, std::ignore) = downstream_tls_session_cache_.emplace( + addr, std::deque()); + } + (*it).second.push_back(session); + ++downstream_tls_session_cache_size_; +} + +SSL_SESSION *Worker::reuse_downstream_tls_session(const DownstreamAddr *addr) { + auto it = downstream_tls_session_cache_.find(addr); + if (it == std::end(downstream_tls_session_cache_)) { + return nullptr; + } + + auto &v = (*it).second; + assert(!v.empty()); + auto session = v.back(); + v.pop_back(); + --downstream_tls_session_cache_size_; + + if (v.empty()) { + downstream_tls_session_cache_.erase(it); + } + + return session; +} + } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 367416f6..5158a55f 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -30,6 +30,8 @@ #include #include #include +#include +#include #include #ifndef NOTHREADS #include @@ -143,6 +145,17 @@ public: mruby::MRubyContext *get_mruby_context() const; #endif // HAVE_MRUBY + // Caches |session| which is associated to downstream address + // |addr|. The caller is responsible to increment the reference + // count of |session|, since this function does not do so. + void cache_downstream_tls_session(const DownstreamAddr *addr, + SSL_SESSION *session); + // Returns cached session associated |addr|. If non-nullptr value + // is returned, its cache entry was successfully removed from cache. + // If no cache entry is found associated to |addr|, nullptr will be + // returned. + SSL_SESSION *reuse_downstream_tls_session(const DownstreamAddr *addr); + private: #ifndef NOTHREADS std::future fut_; @@ -156,6 +169,16 @@ private: DownstreamConnectionPool dconn_pool_; WorkerStat worker_stat_; std::vector dgrps_; + + // Cache for SSL_SESSION for downstream connections. SSL_SESSION is + // associated to downstream address. One address has multiple + // SSL_SESSION objects. New SSL_SESSION is appended to the deque. + // When doing eviction due to storage limitation, the SSL_SESSION + // which sits at the front of deque is removed. + std::unordered_map> + downstream_tls_session_cache_; + size_t downstream_tls_session_cache_size_; + std::unique_ptr session_cache_memcached_dispatcher_; #ifdef HAVE_MRUBY std::unique_ptr mruby_ctx_;