Merge branch 'nghttpx-backend-h1-tls'

doc/sources/nghttpx-howto.rst

@@ -14,9 +14,10 @@ If nghttpx is invoked without any ``-s``, ``-p`` and ``--client``, it
 operates in default mode.  In this mode, nghttpx frontend listens for
 HTTP/2 requests and translates them to HTTP/1 requests.  Thus it works
 as reverse proxy (gateway) for HTTP/2 clients to HTTP/1 web server.
-HTTP/1 requests are also supported in frontend as a fallback.  If
+This is also known as "HTTP/2 router".  HTTP/1 requests are also
-nghttpx is linked with spdylay library and frontend connection is
+supported in frontend as a fallback.  If nghttpx is linked with
-SSL/TLS, the frontend also supports SPDY protocol.
+spdylay library and frontend connection is SSL/TLS, the frontend also
+supports SPDY protocol.
 
 By default, this mode's frontend connection is encrypted using
 SSL/TLS.  So server's private key and certificate must be supplied to
@@ -30,6 +31,10 @@ available on the frontend and a HTTP/1 connection can be upgraded to
 HTTP/2 using HTTP Upgrade.  Starting HTTP/2 connection by sending
 HTTP/2 connection preface is also supported.
 
+By default, backend HTTP/1 connections are not encrypted.  To enable
+TLS on HTTP/1 backend connections, use ``--backend-http1-tls`` option.
+This applies to all mode whose backend connections are HTTP/1.
+
 The backend is supposed to be HTTP/1 Web server.  For example, to make
 nghttpx listen to encrypted HTTP/2 requests at port 8443, and a
 backend HTTP/1 web server is configured to listen to HTTP/1 request at

gennghttpxfun.py

@@ -112,7 +112,9 @@ OPTIONS = [
     "max-request-header-fields",
     "header-field-buffer",
     "max-header-fields",
-    "no-http2-cipher-black-list"
+    "no-http2-cipher-black-list",
+    "backend-http1-tls",
+    "backend-tls-session-cache-per-worker"
 ]
 
 LOGVARS = [

src/shrpx.cc

@@ -1046,8 +1046,6 @@ void fill_default_config() {
   auto &tlsconf = mod_config()->tls;
   {
     auto &ticketconf = tlsconf.ticket;
-    ticketconf.cipher = EVP_aes_128_cbc();
-
     {
       auto &memcachedconf = ticketconf.memcached;
       memcachedconf.max_retry = 3;
@@ -1055,19 +1053,26 @@ void fill_default_config() {
       memcachedconf.interval = 10_min;
     }
 
+    ticketconf.cipher = EVP_aes_128_cbc();
+  }
+
+  {
     auto &ocspconf = tlsconf.ocsp;
     // ocsp update interval = 14400 secs = 4 hours, borrowed from h2o
     ocspconf.update_interval = 4_h;
     ocspconf.fetch_ocsp_response_file =
         strcopy(PKGDATADIR "/fetch-ocsp-response");
+  }
 
+  {
     auto &dyn_recconf = tlsconf.dyn_rec;
     dyn_recconf.warmup_threshold = 1_m;
     dyn_recconf.idle_timeout = 1_s;
-
-    tlsconf.session_timeout = std::chrono::hours(12);
   }
 
+  tlsconf.session_timeout = std::chrono::hours(12);
+  tlsconf.downstream_session_cache_per_worker = 10000;
+
   auto &httpconf = mod_config()->http;
   httpconf.server_name = "nghttpx nghttp2/" NGHTTP2_VERSION;
   httpconf.no_host_rewrite = true;
@@ -1277,6 +1282,16 @@ Connections:
               --backend-write-timeout options.
   --accept-proxy-protocol
               Accept PROXY protocol version 1 on frontend connection.
+  --backend-no-tls
+              Disable  SSL/TLS  on  backend connections.   For  HTTP/2
+              backend  connections, TLS  is enabled  by default.   For
+              HTTP/1 backend connections, TLS  is disabled by default,
+              and can  be enabled  by --backend-http1-tls  option.  If
+              both  --backend-no-tls  and --backend-http1-tls  options
+              are used, --backend-no-tls has the precedence.
+  --backend-http1-tls
+              Enable SSL/TLS on backend  HTTP/1 connections.  See also
+              --backend-no-tls option.
 
 Performance:
   -n, --workers=<N>
@@ -1426,16 +1441,14 @@ SSL/TLS:
               Set allowed  cipher list.  The  format of the  string is
               described in OpenSSL ciphers(1).
   -k, --insecure
-              Don't  verify   backend  server's  certificate   if  -p,
+              Don't  verify backend  server's  certificate  if TLS  is
-              --client    or    --http2-bridge     are    given    and
+              enabled for backend connections.
-              --backend-no-tls is not given.
   --cacert=<PATH>
-              Set path to trusted CA  certificate file if -p, --client
+              Set path to trusted CA  certificate file used in backend
-              or --http2-bridge are given  and --backend-no-tls is not
+              TLS connections.   The file must  be in PEM  format.  It
-              given.  The file must be  in PEM format.  It can contain
+              can  contain  multiple   certificates.   If  the  linked
-              multiple  certificates.    If  the  linked   OpenSSL  is
+              OpenSSL is configured to  load system wide certificates,
-              configured to  load system  wide certificates,  they are
+              they are loaded at startup regardless of this option.
-              loaded at startup regardless of this option.
   --private-key-passwd-file=<PATH>
               Path  to file  that contains  password for  the server's
               private key.   If none is  given and the private  key is
@@ -1575,6 +1588,11 @@ SSL/TLS:
               Allow black  listed cipher  suite on  HTTP/2 connection.
               See  https://tools.ietf.org/html/rfc7540#appendix-A  for
               the complete HTTP/2 cipher suites black list.
+  --backend-tls-session-cache-per-worker=<N>
+              Set  the maximum  number  of backend  TLS session  cache
+              stored per worker.
+              Default: )"
+      << get_config()->tls.downstream_session_cache_per_worker << R"(
 
 HTTP/2 and SPDY:
   -c, --http2-max-concurrent-streams=<N>
@@ -1603,8 +1621,6 @@ HTTP/2 and SPDY:
               connection to 2**<N>-1.
               Default: )"
       << get_config()->http2.downstream.connection_window_bits << R"(
-  --backend-no-tls
-              Disable SSL/TLS on backend connections.
   --http2-no-cookie-crumbling
               Don't crumble cookie header field.
   --padding=<N>
@@ -2029,6 +2045,10 @@ void process_options(
     downstreamconf.proto = PROTO_HTTP;
   }
 
+  if (downstreamconf.proto == PROTO_HTTP && !downstreamconf.http1_tls) {
+    downstreamconf.no_tls = true;
+  }
+
   if (!upstreamconf.no_tls &&
       (!tlsconf.private_key_file || !tlsconf.cert_file)) {
     print_usage(std::cerr);
@@ -2377,6 +2397,9 @@ int main(int argc, char **argv) {
         {SHRPX_OPT_NO_HTTP2_CIPHER_BLACK_LIST, no_argument, &flag, 103},
         {SHRPX_OPT_REQUEST_HEADER_FIELD_BUFFER, required_argument, &flag, 104},
         {SHRPX_OPT_MAX_REQUEST_HEADER_FIELDS, required_argument, &flag, 105},
+        {SHRPX_OPT_BACKEND_HTTP1_TLS, no_argument, &flag, 106},
+        {SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER, required_argument,
+         &flag, 107},
         {nullptr, 0, nullptr, 0}};
 
     int option_index = 0;
@@ -2826,6 +2849,15 @@ int main(int argc, char **argv) {
         // --max-request-header-fields
         cmdcfgs.emplace_back(SHRPX_OPT_MAX_REQUEST_HEADER_FIELDS, optarg);
         break;
+      case 106:
+        // --backend-http1-tls
+        cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_HTTP1_TLS, "yes");
+        break;
+      case 107:
+        // --backend-tls-session-cache-per-worker
+        cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER,
+                             optarg);
+        break;
       default:
         break;
       }

src/shrpx_client_handler.cc

@@ -722,8 +722,8 @@ ClientHandler::get_downstream_connection(Downstream *downstream) {
       }
       dconn = make_unique<Http2DownstreamConnection>(dconn_pool, http2session);
     } else {
-      dconn =
+      dconn = make_unique<HttpDownstreamConnection>(dconn_pool, group,
-          make_unique<HttpDownstreamConnection>(dconn_pool, group, conn_.loop);
+                                                    conn_.loop, worker_);
     }
     dconn->set_client_handler(this);
     return dconn;

src/shrpx_config.cc

@@ -676,6 +676,7 @@ enum {
   SHRPX_OPTID_BACKEND_HTTP_PROXY_URI,
   SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND,
   SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST,
+  SHRPX_OPTID_BACKEND_HTTP1_TLS,
   SHRPX_OPTID_BACKEND_HTTP2_CONNECTION_WINDOW_BITS,
   SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER,
   SHRPX_OPTID_BACKEND_HTTP2_WINDOW_BITS,
@@ -686,6 +687,7 @@ enum {
   SHRPX_OPTID_BACKEND_READ_TIMEOUT,
   SHRPX_OPTID_BACKEND_REQUEST_BUFFER,
   SHRPX_OPTID_BACKEND_RESPONSE_BUFFER,
+  SHRPX_OPTID_BACKEND_TLS_SESSION_CACHE_PER_WORKER,
   SHRPX_OPTID_BACKEND_TLS_SNI_FIELD,
   SHRPX_OPTID_BACKEND_WRITE_TIMEOUT,
   SHRPX_OPTID_BACKLOG,
@@ -1077,6 +1079,9 @@ int option_lookup_token(const char *name, size_t namelen) {
       }
       break;
     case 's':
+      if (util::strieq_l("backend-http1-tl", name, 16)) {
+        return SHRPX_OPTID_BACKEND_HTTP1_TLS;
+      }
       if (util::strieq_l("max-header-field", name, 16)) {
         return SHRPX_OPTID_MAX_HEADER_FIELDS;
       }
@@ -1378,6 +1383,9 @@ int option_lookup_token(const char *name, size_t namelen) {
       if (util::strieq_l("backend-http2-connections-per-worke", name, 35)) {
         return SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER;
       }
+      if (util::strieq_l("backend-tls-session-cache-per-worke", name, 35)) {
+        return SHRPX_OPTID_BACKEND_TLS_SESSION_CACHE_PER_WORKER;
+      }
       break;
     case 's':
       if (util::strieq_l("backend-http2-connection-window-bit", name, 35)) {
@@ -2214,6 +2222,13 @@ int parse_config(const char *opt, const char *optarg,
     mod_config()->tls.no_http2_cipher_black_list = util::strieq(optarg, "yes");
 
     return 0;
+  case SHRPX_OPTID_BACKEND_HTTP1_TLS:
+    mod_config()->conn.downstream.http1_tls = util::strieq(optarg, "yes");
+
+    return 0;
+  case SHRPX_OPTID_BACKEND_TLS_SESSION_CACHE_PER_WORKER:
+    return parse_uint(&mod_config()->tls.downstream_session_cache_per_worker,
+                      opt, optarg);
   case SHRPX_OPTID_CONF:
     LOG(WARN) << "conf: ignored";
 

src/shrpx_config.h

@@ -206,6 +206,9 @@ constexpr char SHRPX_OPT_MAX_RESPONSE_HEADER_FIELDS[] =
     "max-response-header-fields";
 constexpr char SHRPX_OPT_NO_HTTP2_CIPHER_BLACK_LIST[] =
     "no-http2-cipher-black-list";
+constexpr char SHRPX_OPT_BACKEND_HTTP1_TLS[] = "backend-http1-tls";
+constexpr char SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER[] =
+    "backend-tls-session-cache-per-worker";
 
 constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8;
 
@@ -390,6 +393,7 @@ struct TLSConfig {
   std::vector<std::string> npn_list;
   // list of supported SSL/TLS protocol strings.
   std::vector<std::string> tls_proto_list;
+  size_t downstream_session_cache_per_worker;
   // Bit mask to disable SSL/TLS protocol versions.  This will be
   // passed to SSL_CTX_set_options().
   long int tls_proto_mask;
@@ -534,6 +538,7 @@ struct ConnectionConfig {
     // downstream protocol; this will be determined by given options.
     shrpx_proto proto;
     bool no_tls;
+    bool http1_tls;
     // true if IPv4 only; ipv4 and ipv6 are mutually exclusive; and
     // (ipv4 && ipv6) must be false.
     bool ipv4;

src/shrpx_http_downstream_connection.cc

@@ -35,6 +35,7 @@
 #include "shrpx_downstream_connection_pool.h"
 #include "shrpx_worker.h"
 #include "shrpx_http2_session.h"
+#include "shrpx_ssl.h"
 #include "http2.h"
 #include "util.h"
 
@@ -100,7 +101,7 @@ void connectcb(struct ev_loop *loop, ev_io *w, int revents) {
   auto downstream = dconn->get_downstream();
   auto upstream = downstream->get_upstream();
   auto handler = upstream->get_client_handler();
-  if (dconn->on_connect() != 0) {
+  if (dconn->connected() != 0) {
     if (upstream->on_downstream_abort_request(downstream, 503) != 0) {
       delete handler;
     }
@@ -111,20 +112,34 @@ void connectcb(struct ev_loop *loop, ev_io *w, int revents) {
 } // namespace
 
 HttpDownstreamConnection::HttpDownstreamConnection(
-    DownstreamConnectionPool *dconn_pool, size_t group, struct ev_loop *loop)
+    DownstreamConnectionPool *dconn_pool, size_t group, struct ev_loop *loop,
+    Worker *worker)
     : DownstreamConnection(dconn_pool),
-      conn_(loop, -1, nullptr, nullptr,
+      conn_(loop, -1, nullptr, worker->get_mcpool(),
             get_config()->conn.downstream.timeout.write,
             get_config()->conn.downstream.timeout.read, {}, {}, connectcb,
             readcb, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold,
             get_config()->tls.dyn_rec.idle_timeout),
+      do_read_(&HttpDownstreamConnection::noop),
+      do_write_(&HttpDownstreamConnection::noop),
+      worker_(worker),
+      ssl_ctx_(worker->get_cl_ssl_ctx()),
       ioctrl_(&conn_.rlimit),
       response_htp_{0},
       group_(group),
-      addr_idx_(0),
+      addr_idx_(0) {}
-      connected_(false) {}
 
-HttpDownstreamConnection::~HttpDownstreamConnection() {}
+HttpDownstreamConnection::~HttpDownstreamConnection() {
+  if (conn_.tls.ssl) {
+    auto session = SSL_get1_session(conn_.tls.ssl);
+    if (session) {
+      auto &downstreamconf = get_config()->conn.downstream;
+      auto &addr = downstreamconf.addr_groups[group_].addrs[addr_idx_];
+
+      worker_->cache_downstream_tls_session(&addr, session);
+    }
+  }
+}
 
 int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
   if (LOG_ENABLED(INFO)) {
@@ -144,8 +159,16 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
       return -1;
     }
 
-    auto worker = client_handler_->get_worker();
+    if (ssl_ctx_) {
-    auto &next_downstream = worker->get_dgrp(group_)->next;
+      auto ssl = ssl::create_ssl(ssl_ctx_);
+      if (!ssl) {
+        return -1;
+      }
+
+      conn_.set_ssl(ssl);
+    }
+
+    auto &next_downstream = worker_->get_dgrp(group_)->next;
     auto end = next_downstream;
     auto &addrs = downstreamconf.addr_groups[group_].addrs;
     for (;;) {
@@ -190,6 +213,23 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
 
       addr_idx_ = i;
 
+      if (ssl_ctx_) {
+        auto sni_name = !get_config()->tls.backend_sni_name.empty()
+                            ? StringRef(get_config()->tls.backend_sni_name)
+                            : StringRef(addrs[i].host);
+        if (!util::numeric_host(sni_name.c_str())) {
+          SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str());
+        }
+
+        auto session = worker_->reuse_downstream_tls_session(&addrs[i]);
+        if (session) {
+          SSL_set_session(conn_.tls.ssl, session);
+          SSL_SESSION_free(session);
+        }
+
+        conn_.prepare_client_handshake();
+      }
+
       ev_io_set(&conn_.wev, conn_.fd, EV_WRITE);
       ev_io_set(&conn_.rev, conn_.fd, EV_READ);
 
@@ -748,44 +788,13 @@ http_parser_settings htp_hooks = {
 };
 } // namespace
 
-int HttpDownstreamConnection::on_read() {
+int HttpDownstreamConnection::read_clear() {
-  if (!connected_) {
-    return 0;
-  }
-
   ev_timer_again(conn_.loop, &conn_.rt);
   std::array<uint8_t, 8_k> buf;
   int rv;
 
-  if (downstream_->get_upgraded()) {
-    // For upgraded connection, just pass data to the upstream.
-    for (;;) {
-      auto nread = conn_.read_clear(buf.data(), buf.size());
-
-      if (nread == 0) {
-        return 0;
-      }
-
-      if (nread < 0) {
-        return nread;
-      }
-
-      rv = downstream_->get_upstream()->on_downstream_body(
-          downstream_, buf.data(), nread, true);
-      if (rv != 0) {
-        return rv;
-      }
-
-      if (downstream_->response_buf_full()) {
-        downstream_->pause_read(SHRPX_NO_BUFFER);
-        return 0;
-      }
-    }
-  }
-
   for (;;) {
     auto nread = conn_.read_clear(buf.data(), buf.size());
-
     if (nread == 0) {
       return 0;
     }
@@ -794,59 +803,18 @@ int HttpDownstreamConnection::on_read() {
       return nread;
     }
 
-    auto nproc =
+    rv = process_input(buf.data(), nread);
-        http_parser_execute(&response_htp_, &htp_hooks,
+    if (rv != 0) {
-                            reinterpret_cast<char *>(buf.data()), nread);
+      return rv;
-
-    auto htperr = HTTP_PARSER_ERRNO(&response_htp_);
-
-    if (htperr != HPE_OK) {
-      // Handling early return (in other words, response was hijacked
-      // by mruby scripting).
-      if (downstream_->get_response_state() == Downstream::MSG_COMPLETE) {
-        return SHRPX_ERR_DCONN_CANCELED;
-      }
-
-      if (LOG_ENABLED(INFO)) {
-        DCLOG(INFO, this) << "HTTP parser failure: "
-                          << "(" << http_errno_name(htperr) << ") "
-                          << http_errno_description(htperr);
-      }
-
-      return -1;
     }
 
-    if (downstream_->get_upgraded()) {
+    if (!ev_is_active(&conn_.rev)) {
-      if (nproc < static_cast<size_t>(nread)) {
-        // Data from buf.data() + nproc are for upgraded protocol.
-        rv = downstream_->get_upstream()->on_downstream_body(
-            downstream_, buf.data() + nproc, nread - nproc, true);
-        if (rv != 0) {
-          return rv;
-        }
-
-        if (downstream_->response_buf_full()) {
-          downstream_->pause_read(SHRPX_NO_BUFFER);
-          return 0;
-        }
-      }
-      // call on_read(), so that we can process data left in buffer as
-      // upgrade.
-      return on_read();
-    }
-
-    if (downstream_->response_buf_full()) {
-      downstream_->pause_read(SHRPX_NO_BUFFER);
       return 0;
     }
   }
 }
 
-int HttpDownstreamConnection::on_write() {
+int HttpDownstreamConnection::write_clear() {
-  if (!connected_) {
-    return 0;
-  }
-
   ev_timer_again(conn_.loop, &conn_.rt);
 
   auto upstream = downstream_->get_upstream();
@@ -883,7 +851,172 @@ int HttpDownstreamConnection::on_write() {
   return 0;
 }
 
-int HttpDownstreamConnection::on_connect() {
+int HttpDownstreamConnection::tls_handshake() {
+  ERR_clear_error();
+
+  ev_timer_again(conn_.loop, &conn_.rt);
+
+  auto rv = conn_.tls_handshake();
+  if (rv == SHRPX_ERR_INPROGRESS) {
+    return 0;
+  }
+
+  if (rv < 0) {
+    return rv;
+  }
+
+  if (LOG_ENABLED(INFO)) {
+    DCLOG(INFO, this) << "SSL/TLS handshake completed";
+  }
+
+  if (!get_config()->tls.insecure &&
+      ssl::check_cert(conn_.tls.ssl, &get_config()
+                                          ->conn.downstream.addr_groups[group_]
+                                          .addrs[addr_idx_]) != 0) {
+    return -1;
+  }
+
+  do_read_ = &HttpDownstreamConnection::read_tls;
+  do_write_ = &HttpDownstreamConnection::write_tls;
+
+  // TODO Check negotiated ALPN
+
+  return on_write();
+}
+
+int HttpDownstreamConnection::read_tls() {
+  ERR_clear_error();
+
+  ev_timer_again(conn_.loop, &conn_.rt);
+  std::array<uint8_t, 8_k> buf;
+  int rv;
+
+  for (;;) {
+    auto nread = conn_.read_tls(buf.data(), buf.size());
+    if (nread == 0) {
+      return 0;
+    }
+
+    if (nread < 0) {
+      return nread;
+    }
+
+    rv = process_input(buf.data(), nread);
+    if (rv != 0) {
+      return rv;
+    }
+
+    if (!ev_is_active(&conn_.rev)) {
+      return 0;
+    }
+  }
+}
+
+int HttpDownstreamConnection::write_tls() {
+  ERR_clear_error();
+
+  ev_timer_again(conn_.loop, &conn_.rt);
+
+  auto upstream = downstream_->get_upstream();
+  auto input = downstream_->get_request_buf();
+
+  struct iovec iov;
+
+  while (input->rleft() > 0) {
+    auto iovcnt = input->riovec(&iov, 1);
+    assert(iovcnt == 1);
+    auto nwrite = conn_.write_tls(iov.iov_base, iov.iov_len);
+
+    if (nwrite == 0) {
+      return 0;
+    }
+
+    if (nwrite < 0) {
+      return nwrite;
+    }
+
+    input->drain(nwrite);
+  }
+
+  conn_.wlimit.stopw();
+  ev_timer_stop(conn_.loop, &conn_.wt);
+
+  if (input->rleft() == 0) {
+    auto &req = downstream_->request();
+
+    upstream->resume_read(SHRPX_NO_BUFFER, downstream_,
+                          req.unconsumed_body_length);
+  }
+
+  return 0;
+}
+
+int HttpDownstreamConnection::process_input(const uint8_t *data,
+                                            size_t datalen) {
+  int rv;
+
+  if (downstream_->get_upgraded()) {
+    // For upgraded connection, just pass data to the upstream.
+    rv = downstream_->get_upstream()->on_downstream_body(downstream_, data,
+                                                         datalen, true);
+    if (rv != 0) {
+      return rv;
+    }
+
+    if (downstream_->response_buf_full()) {
+      downstream_->pause_read(SHRPX_NO_BUFFER);
+      return 0;
+    }
+  }
+
+  auto nproc =
+      http_parser_execute(&response_htp_, &htp_hooks,
+                          reinterpret_cast<const char *>(data), datalen);
+
+  auto htperr = HTTP_PARSER_ERRNO(&response_htp_);
+
+  if (htperr != HPE_OK) {
+    // Handling early return (in other words, response was hijacked by
+    // mruby scripting).
+    if (downstream_->get_response_state() == Downstream::MSG_COMPLETE) {
+      return SHRPX_ERR_DCONN_CANCELED;
+    }
+
+    if (LOG_ENABLED(INFO)) {
+      DCLOG(INFO, this) << "HTTP parser failure: "
+                        << "(" << http_errno_name(htperr) << ") "
+                        << http_errno_description(htperr);
+    }
+
+    return -1;
+  }
+
+  if (downstream_->get_upgraded()) {
+    if (nproc < datalen) {
+      // Data from data + nproc are for upgraded protocol.
+      rv = downstream_->get_upstream()->on_downstream_body(
+          downstream_, data + nproc, datalen - nproc, true);
+      if (rv != 0) {
+        return rv;
+      }
+
+      if (downstream_->response_buf_full()) {
+        downstream_->pause_read(SHRPX_NO_BUFFER);
+        return 0;
+      }
+    }
+    return 0;
+  }
+
+  if (downstream_->response_buf_full()) {
+    downstream_->pause_read(SHRPX_NO_BUFFER);
+    return 0;
+  }
+
+  return 0;
+}
+
+int HttpDownstreamConnection::connected() {
   auto connect_blocker = client_handler_->get_connect_blocker();
 
   if (!util::check_socket_connected(conn_.fd)) {
@@ -898,18 +1031,33 @@ int HttpDownstreamConnection::on_connect() {
     return -1;
   }
 
-  connected_ = true;
+  if (LOG_ENABLED(INFO)) {
+    DLOG(INFO, this) << "Connected to downstream host";
+  }
 
   connect_blocker->on_success();
 
   conn_.rlimit.startw();
-  ev_timer_again(conn_.loop, &conn_.rt);
 
   ev_set_cb(&conn_.wev, writecb);
 
+  if (conn_.tls.ssl) {
+    do_read_ = &HttpDownstreamConnection::tls_handshake;
+    do_write_ = &HttpDownstreamConnection::tls_handshake;
+
+    return 0;
+  }
+
+  do_read_ = &HttpDownstreamConnection::read_clear;
+  do_write_ = &HttpDownstreamConnection::write_clear;
+
   return 0;
 }
 
+int HttpDownstreamConnection::on_read() { return do_read_(*this); }
+
+int HttpDownstreamConnection::on_write() { return do_write_(*this); }
+
 void HttpDownstreamConnection::on_upstream_change(Upstream *upstream) {}
 
 void HttpDownstreamConnection::signal_write() {
@@ -918,4 +1066,6 @@ void HttpDownstreamConnection::signal_write() {
 
 size_t HttpDownstreamConnection::get_group() const { return group_; }
 
+int HttpDownstreamConnection::noop() { return 0; }
+
 } // namespace shrpx

src/shrpx_http_downstream_connection.h

@@ -36,11 +36,12 @@
 namespace shrpx {
 
 class DownstreamConnectionPool;
+class Worker;
 
 class HttpDownstreamConnection : public DownstreamConnection {
 public:
   HttpDownstreamConnection(DownstreamConnectionPool *dconn_pool, size_t group,
-                           struct ev_loop *loop);
+                           struct ev_loop *loop, Worker *worker);
   virtual ~HttpDownstreamConnection();
   virtual int attach_downstream(Downstream *downstream);
   virtual void detach_downstream(Downstream *downstream);
@@ -61,17 +62,30 @@ public:
 
   virtual bool poolable() const { return true; }
 
-  int on_connect();
+  int read_clear();
+  int write_clear();
+  int read_tls();
+  int write_tls();
+
+  int process_input(const uint8_t *data, size_t datalen);
+  int tls_handshake();
+
+  int connected();
   void signal_write();
 
+  int noop();
+
 private:
   Connection conn_;
+  std::function<int(HttpDownstreamConnection &)> do_read_, do_write_;
+  Worker *worker_;
+  // nullptr if TLS is not used.
+  SSL_CTX *ssl_ctx_;
   IOControl ioctrl_;
   http_parser response_htp_;
   size_t group_;
   // index of get_config()->downstream_addrs this object is using
   size_t addr_idx_;
-  bool connected_;
 };
 
 } // namespace shrpx

src/shrpx_ssl.cc

@@ -628,9 +628,9 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file
 }
 
 namespace {
-int select_next_proto_cb(SSL *ssl, unsigned char **out, unsigned char *outlen,
+int select_h2_next_proto_cb(SSL *ssl, unsigned char **out,
-                         const unsigned char *in, unsigned int inlen,
+                            unsigned char *outlen, const unsigned char *in,
-                         void *arg) {
+                            unsigned int inlen, void *arg) {
   if (!util::select_h2(const_cast<const unsigned char **>(out), outlen, in,
                        inlen)) {
     return SSL_TLSEXT_ERR_NOACK;
@@ -640,6 +640,24 @@ int select_next_proto_cb(SSL *ssl, unsigned char **out, unsigned char *outlen,
 }
 } // namespace
 
+namespace {
+int select_h1_next_proto_cb(SSL *ssl, unsigned char **out,
+                            unsigned char *outlen, const unsigned char *in,
+                            unsigned int inlen, void *arg) {
+  auto end = in + inlen;
+  for (; in < end;) {
+    if (util::streq_l(NGHTTP2_H1_1_ALPN, in, in[0] + 1)) {
+      *out = const_cast<unsigned char *>(in) + 1;
+      *outlen = in[0];
+      return SSL_TLSEXT_ERR_OK;
+    }
+    in += in[0] + 1;
+  }
+
+  return SSL_TLSEXT_ERR_NOACK;
+}
+} // namespace
+
 SSL_CTX *create_ssl_client_context(
 #ifdef HAVE_NEVERBLEED
     neverbleed_t *nb
@@ -722,15 +740,29 @@ SSL_CTX *create_ssl_client_context(
       DIE();
     }
   }
-  // NPN selection callback
+
-  SSL_CTX_set_next_proto_select_cb(ssl_ctx, select_next_proto_cb, nullptr);
+  auto &downstreamconf = get_config()->conn.downstream;
+
+  if (downstreamconf.proto == PROTO_HTTP2) {
+    // NPN selection callback
+    SSL_CTX_set_next_proto_select_cb(ssl_ctx, select_h2_next_proto_cb, nullptr);
 
 #if OPENSSL_VERSION_NUMBER >= 0x10002000L
-  // ALPN advertisement; We only advertise HTTP/2
+    // ALPN advertisement; We only advertise HTTP/2
-  auto proto_list = util::get_default_alpn();
+    auto proto_list = util::get_default_alpn();
 
-  SSL_CTX_set_alpn_protos(ssl_ctx, proto_list.data(), proto_list.size());
+    SSL_CTX_set_alpn_protos(ssl_ctx, proto_list.data(), proto_list.size());
 #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
+  } else {
+    // NPN selection callback
+    SSL_CTX_set_next_proto_select_cb(ssl_ctx, select_h1_next_proto_cb, nullptr);
+
+#if OPENSSL_VERSION_NUMBER >= 0x10002000L
+    SSL_CTX_set_alpn_protos(
+        ssl_ctx, reinterpret_cast<const unsigned char *>(NGHTTP2_H1_1_ALPN),
+        str_size(NGHTTP2_H1_1_ALPN));
+#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
+  }
 
   return ssl_ctx;
 }
@@ -1270,15 +1302,7 @@ SSL_CTX *setup_server_ssl_context(std::vector<SSL_CTX *> &all_ssl_ctx,
   return ssl_ctx;
 }
 
-bool downstream_tls_enabled() {
+bool downstream_tls_enabled() { return !get_config()->conn.downstream.no_tls; }
-  auto no_tls = get_config()->conn.downstream.no_tls;
-
-  if (get_config()->client_mode) {
-    return !no_tls;
-  }
-
-  return get_config()->http2_bridge && !no_tls;
-}
 
 SSL_CTX *setup_client_ssl_context(
 #ifdef HAVE_NEVERBLEED

src/shrpx_worker.cc

@@ -73,6 +73,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
       dconn_pool_(get_config()->conn.downstream.addr_groups.size()),
       worker_stat_(get_config()->conn.downstream.addr_groups.size()),
       dgrps_(get_config()->conn.downstream.addr_groups.size()),
+      downstream_tls_session_cache_size_(0),
       loop_(loop),
       sv_ssl_ctx_(sv_ssl_ctx),
       cl_ssl_ctx_(cl_ssl_ctx),
@@ -116,6 +117,12 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
 Worker::~Worker() {
   ev_async_stop(loop_, &w_);
   ev_timer_stop(loop_, &mcpool_clear_timer_);
+
+  for (auto &p : downstream_tls_session_cache_) {
+    for (auto session : p.second) {
+      SSL_SESSION_free(session);
+    }
+  }
 }
 
 void Worker::schedule_clear_mcpool() {
@@ -300,4 +307,57 @@ mruby::MRubyContext *Worker::get_mruby_context() const {
 }
 #endif // HAVE_MRUBY
 
+void Worker::cache_downstream_tls_session(const DownstreamAddr *addr,
+                                          SSL_SESSION *session) {
+  auto &tlsconf = get_config()->tls;
+
+  auto max = tlsconf.downstream_session_cache_per_worker;
+  if (max == 0) {
+    return;
+  }
+
+  if (downstream_tls_session_cache_size_ >= max) {
+    // It is implementation dependent which item is returned from
+    // std::begin().  Probably, this depends on hash algorithm.  If it
+    // is random fashion, then we are mostly OK.
+    auto it = std::begin(downstream_tls_session_cache_);
+    assert(it != std::end(downstream_tls_session_cache_));
+    auto &v = (*it).second;
+    assert(!v.empty());
+    auto sess = v.front();
+    v.pop_front();
+    SSL_SESSION_free(sess);
+    if (v.empty()) {
+      downstream_tls_session_cache_.erase(it);
+    }
+  }
+
+  auto it = downstream_tls_session_cache_.find(addr);
+  if (it == std::end(downstream_tls_session_cache_)) {
+    std::tie(it, std::ignore) = downstream_tls_session_cache_.emplace(
+        addr, std::deque<SSL_SESSION *>());
+  }
+  (*it).second.push_back(session);
+  ++downstream_tls_session_cache_size_;
+}
+
+SSL_SESSION *Worker::reuse_downstream_tls_session(const DownstreamAddr *addr) {
+  auto it = downstream_tls_session_cache_.find(addr);
+  if (it == std::end(downstream_tls_session_cache_)) {
+    return nullptr;
+  }
+
+  auto &v = (*it).second;
+  assert(!v.empty());
+  auto session = v.back();
+  v.pop_back();
+  --downstream_tls_session_cache_size_;
+
+  if (v.empty()) {
+    downstream_tls_session_cache_.erase(it);
+  }
+
+  return session;
+}
+
 } // namespace shrpx

src/shrpx_worker.h

@@ -30,6 +30,8 @@
 #include <mutex>
 #include <vector>
 #include <random>
+#include <unordered_map>
+#include <deque>
 #include <thread>
 #ifndef NOTHREADS
 #include <future>
@@ -143,6 +145,17 @@ public:
   mruby::MRubyContext *get_mruby_context() const;
 #endif // HAVE_MRUBY
 
+  // Caches |session| which is associated to downstream address
+  // |addr|.  The caller is responsible to increment the reference
+  // count of |session|, since this function does not do so.
+  void cache_downstream_tls_session(const DownstreamAddr *addr,
+                                    SSL_SESSION *session);
+  // Returns cached session associated |addr|.  If non-nullptr value
+  // is returned, its cache entry was successfully removed from cache.
+  // If no cache entry is found associated to |addr|, nullptr will be
+  // returned.
+  SSL_SESSION *reuse_downstream_tls_session(const DownstreamAddr *addr);
+
 private:
 #ifndef NOTHREADS
   std::future<void> fut_;
@@ -156,6 +169,16 @@ private:
   DownstreamConnectionPool dconn_pool_;
   WorkerStat worker_stat_;
   std::vector<DownstreamGroup> dgrps_;
+
+  // Cache for SSL_SESSION for downstream connections.  SSL_SESSION is
+  // associated to downstream address.  One address has multiple
+  // SSL_SESSION objects.  New SSL_SESSION is appended to the deque.
+  // When doing eviction due to storage limitation, the SSL_SESSION
+  // which sits at the front of deque is removed.
+  std::unordered_map<const DownstreamAddr *, std::deque<SSL_SESSION *>>
+      downstream_tls_session_cache_;
+  size_t downstream_tls_session_cache_size_;
+
   std::unique_ptr<MemcachedDispatcher> session_cache_memcached_dispatcher_;
 #ifdef HAVE_MRUBY
   std::unique_ptr<mruby::MRubyContext> mruby_ctx_;