Merge branch 'tls13-early-data'

gennghttpxfun.py

@@ -170,6 +170,8 @@ OPTIONS = [
     "no-verify-ocsp",
     "verify-client-tolerate-expired",
     "ignore-per-pattern-mruby-error",
+    "tls-no-postpone-early-data",
+    "tls-max-early-data",
 ]
 
 LOGVARS = [

src/http2.cc

@@ -107,6 +107,9 @@ StringRef get_reason_phrase(unsigned int status_code) {
     return StringRef::from_lit("Expectation Failed");
   case 421:
     return StringRef::from_lit("Misdirected Request");
+  case 425:
+    // https://tools.ietf.org/html/draft-ietf-httpbis-replay-02
+    return StringRef::from_lit("Too Early");
   case 426:
     return StringRef::from_lit("Upgrade Required");
   case 428:

src/shrpx.cc

@@ -1465,6 +1465,7 @@ void fill_default_config(Config *config) {
       tls::proto_version_from_string(DEFAULT_TLS_MIN_PROTO_VERSION);
   tlsconf.max_proto_version =
       tls::proto_version_from_string(DEFAULT_TLS_MAX_PROTO_VERSION);
+  tlsconf.max_early_data = 16_k;
 #if OPENSSL_1_1_API || defined(OPENSSL_IS_BORINGSSL)
   tlsconf.ecdh_curves = StringRef::from_lit("X25519:P-256:P-384:P-521");
 #else  // !OPENSSL_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
@@ -2370,6 +2371,18 @@ SSL/TLS:
               HTTP/2.   To  use  those   cipher  suites  with  HTTP/2,
               consider   to  use   --client-no-http2-cipher-black-list
               option.  But be aware its implications.
+  --tls-no-postpone-early-data
+              By default,  nghttpx postpones forwarding  HTTP requests
+              sent in early data, including those sent in partially in
+              it, until TLS handshake finishes.  If all backend server
+              recognizes "Early-Data" header  field, using this option
+              makes nghttpx  not postpone  forwarding request  and get
+              full potential of 0-RTT data.
+  --tls-max-early-data=<SIZE>
+              Sets  the  maximum  amount  of 0-RTT  data  that  server
+              accepts.
+              Default: )"
+      << util::utos_unit(config->tls.max_early_data) << R"(
 
 HTTP/2:
   -c, --frontend-http2-max-concurrent-streams=<N>
@@ -3436,6 +3449,8 @@ int main(int argc, char **argv) {
          160},
         {SHRPX_OPT_IGNORE_PER_PATTERN_MRUBY_ERROR.c_str(), no_argument, &flag,
          161},
+        {SHRPX_OPT_TLS_NO_POSTPONE_EARLY_DATA.c_str(), no_argument, &flag, 162},
+        {SHRPX_OPT_TLS_MAX_EARLY_DATA.c_str(), required_argument, &flag, 163},
         {nullptr, 0, nullptr, 0}};
 
     int option_index = 0;
@@ -4207,6 +4222,15 @@ int main(int argc, char **argv) {
         cmdcfgs.emplace_back(SHRPX_OPT_IGNORE_PER_PATTERN_MRUBY_ERROR,
                              StringRef::from_lit("yes"));
         break;
+      case 162:
+        // --tls-no-postpone-early-data
+        cmdcfgs.emplace_back(SHRPX_OPT_TLS_NO_POSTPONE_EARLY_DATA,
+                             StringRef::from_lit("yes"));
+        break;
+      case 163:
+        // --tls-max-early-data
+        cmdcfgs.emplace_back(SHRPX_OPT_TLS_MAX_EARLY_DATA, StringRef{optarg});
+        break;
       default:
         break;
       }

src/shrpx_config.cc

@@ -1883,6 +1883,11 @@ int option_lookup_token(const char *name, size_t namelen) {
     break;
   case 18:
     switch (name[17]) {
+    case 'a':
+      if (util::strieq_l("tls-max-early-dat", name, 17)) {
+        return SHRPX_OPTID_TLS_MAX_EARLY_DATA;
+      }
+      break;
     case 'r':
       if (util::strieq_l("add-request-heade", name, 17)) {
         return SHRPX_OPTID_ADD_REQUEST_HEADER;
@@ -2114,6 +2119,11 @@ int option_lookup_token(const char *name, size_t namelen) {
     break;
   case 26:
     switch (name[25]) {
+    case 'a':
+      if (util::strieq_l("tls-no-postpone-early-dat", name, 25)) {
+        return SHRPX_OPTID_TLS_NO_POSTPONE_EARLY_DATA;
+      }
+      break;
     case 'e':
       if (util::strieq_l("frontend-http2-window-siz", name, 25)) {
         return SHRPX_OPTID_FRONTEND_HTTP2_WINDOW_SIZE;
@@ -3591,6 +3601,13 @@ int parse_config(Config *config, int optid, const StringRef &opt,
     config->ignore_per_pattern_mruby_error = util::strieq_l("yes", optarg);
 
     return 0;
+  case SHRPX_OPTID_TLS_NO_POSTPONE_EARLY_DATA:
+    config->tls.no_postpone_early_data = util::strieq_l("yes", optarg);
+
+    return 0;
+  case SHRPX_OPTID_TLS_MAX_EARLY_DATA: {
+    return parse_uint_with_unit(&config->tls.max_early_data, opt, optarg);
+  }
   case SHRPX_OPTID_CONF:
     LOG(WARN) << "conf: ignored";
 

src/shrpx_config.h

@@ -347,6 +347,10 @@ constexpr auto SHRPX_OPT_VERIFY_CLIENT_TOLERATE_EXPIRED =
     StringRef::from_lit("verify-client-tolerate-expired");
 constexpr auto SHRPX_OPT_IGNORE_PER_PATTERN_MRUBY_ERROR =
     StringRef::from_lit("ignore-per-pattern-mruby-error");
+constexpr auto SHRPX_OPT_TLS_NO_POSTPONE_EARLY_DATA =
+    StringRef::from_lit("tls-no-postpone-early-data");
+constexpr auto SHRPX_OPT_TLS_MAX_EARLY_DATA =
+    StringRef::from_lit("tls-max-early-data");
 
 constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8;
 
@@ -650,12 +654,17 @@ struct TLSConfig {
   StringRef ciphers;
   StringRef ecdh_curves;
   StringRef cacert;
+  // The maximum amount of 0-RTT data that server accepts.
+  uint32_t max_early_data;
   // The minimum and maximum TLS version.  These values are defined in
   // OpenSSL header file.
   int min_proto_version;
   int max_proto_version;
   bool insecure;
   bool no_http2_cipher_black_list;
+  // true if forwarding requests included in TLS early data should not
+  // be postponed until TLS handshake finishes.
+  bool no_postpone_early_data;
 };
 
 // custom error page
@@ -1114,8 +1123,10 @@ enum {
   SHRPX_OPTID_SYSLOG_FACILITY,
   SHRPX_OPTID_TLS_DYN_REC_IDLE_TIMEOUT,
   SHRPX_OPTID_TLS_DYN_REC_WARMUP_THRESHOLD,
+  SHRPX_OPTID_TLS_MAX_EARLY_DATA,
   SHRPX_OPTID_TLS_MAX_PROTO_VERSION,
   SHRPX_OPTID_TLS_MIN_PROTO_VERSION,
+  SHRPX_OPTID_TLS_NO_POSTPONE_EARLY_DATA,
   SHRPX_OPTID_TLS_PROTO_LIST,
   SHRPX_OPTID_TLS_SCT_DIR,
   SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED,

src/shrpx_connection.cc

@@ -60,7 +60,8 @@ Connection::Connection(struct ev_loop *loop, int fd, SSL *ssl,
                        IOCb readcb, TimerCb timeoutcb, void *data,
                        size_t tls_dyn_rec_warmup_threshold,
                        ev_tstamp tls_dyn_rec_idle_timeout, shrpx_proto proto)
-    : tls{DefaultMemchunks(mcpool), DefaultPeekMemchunks(mcpool)},
+    : tls{DefaultMemchunks(mcpool), DefaultPeekMemchunks(mcpool),
+          DefaultMemchunks(mcpool)},
       wlimit(loop, &wev, write_limit.rate, write_limit.burst),
       rlimit(loop, &rev, read_limit.rate, read_limit.burst, this),
       loop(loop),
@@ -120,10 +121,11 @@ void Connection::disconnect() {
     tls.warmup_writelen = 0;
     tls.last_writelen = 0;
     tls.last_readlen = 0;
-    tls.handshake_state = 0;
+    tls.handshake_state = TLS_CONN_NORMAL;
     tls.initial_handshake_done = false;
     tls.reneg_started = false;
     tls.sct_requested = false;
+    tls.early_data_finish = false;
   }
 
   if (fd != -1) {
@@ -141,7 +143,11 @@ void Connection::disconnect() {
   wlimit.stopw();
 }
 
-void Connection::prepare_client_handshake() { SSL_set_connect_state(tls.ssl); }
+void Connection::prepare_client_handshake() {
+  SSL_set_connect_state(tls.ssl);
+  // This prevents SSL_read_early_data from being called.
+  tls.early_data_finish = true;
+}
 
 void Connection::prepare_server_handshake() {
   SSL_set_accept_state(tls.ssl);
@@ -327,8 +333,9 @@ int Connection::tls_handshake() {
   wlimit.stopw();
   ev_timer_stop(loop, &wt);
 
+  std::array<uint8_t, 16_k> buf;
+
   if (ev_is_active(&rev)) {
-    std::array<uint8_t, 8_k> buf;
     auto nread = read_clear(buf.data(), buf.size());
     if (nread < 0) {
       if (LOG_ENABLED(INFO)) {
@@ -381,9 +388,65 @@ int Connection::tls_handshake() {
     break;
   }
 
+  int rv;
+
   ERR_clear_error();
 
-  auto rv = SSL_do_handshake(tls.ssl);
+#if OPENSSL_1_1_1_API
+  if (!tls.server_handshake || tls.early_data_finish) {
+    rv = SSL_do_handshake(tls.ssl);
+  } else {
+    auto &tlsconf = get_config()->tls;
+    for (;;) {
+      size_t nread;
+
+      rv = SSL_read_early_data(tls.ssl, buf.data(), buf.size(), &nread);
+      if (rv == SSL_READ_EARLY_DATA_ERROR) {
+        // If we have early data, and server sends ServerHello, assume
+        // that handshake is completed in server side, and start
+        // processing request.  If we don't exit handshake code here,
+        // server waits for EndOfEarlyData and Finished message from
+        // client, which voids the purpose of 0-RTT data.  The left
+        // over of handshake is done through write_tls or read_tls.
+        if (tlsconf.no_postpone_early_data &&
+            (tls.handshake_state == TLS_CONN_WRITE_STARTED ||
+             tls.wbuf.rleft()) &&
+            tls.earlybuf.rleft()) {
+          rv = 1;
+        }
+
+        break;
+      }
+
+      if (LOG_ENABLED(INFO)) {
+        LOG(INFO) << "tls: read early data " << nread << " bytes";
+      }
+
+      tls.earlybuf.append(buf.data(), nread);
+
+      if (rv == SSL_READ_EARLY_DATA_FINISH) {
+        if (LOG_ENABLED(INFO)) {
+          LOG(INFO) << "tls: read all early data; total "
+                    << tls.earlybuf.rleft() << " bytes";
+        }
+        tls.early_data_finish = true;
+        // The same reason stated above.
+        if (tlsconf.no_postpone_early_data &&
+            (tls.handshake_state == TLS_CONN_WRITE_STARTED ||
+             tls.wbuf.rleft()) &&
+            tls.earlybuf.rleft()) {
+          rv = 1;
+        } else {
+          ERR_clear_error();
+          rv = SSL_do_handshake(tls.ssl);
+        }
+        break;
+      }
+    }
+  }
+#else  // !OPENSSL_1_1_1_API
+  rv = SSL_do_handshake(tls.ssl);
+#endif // !OPENSSL_1_1_1_API
 
   if (rv <= 0) {
     auto err = SSL_get_error(tls.ssl, rv);
@@ -621,7 +684,21 @@ ssize_t Connection::write_tls(const void *data, size_t len) {
 
   ERR_clear_error();
 
+#if OPENSSL_1_1_1_API
+  int rv;
+  if (SSL_is_init_finished(tls.ssl)) {
+    rv = SSL_write(tls.ssl, data, len);
+  } else {
+    size_t nwrite;
+    rv = SSL_write_early_data(tls.ssl, data, len, &nwrite);
+    // Use the same semantics with SSL_write.
+    if (rv == 1) {
+      rv = nwrite;
+    }
+  }
+#else  // !OPENSSL_1_1_1_API
   auto rv = SSL_write(tls.ssl, data, len);
+#endif // !OPENSSL_1_1_1_API
 
   if (rv <= 0) {
     auto err = SSL_get_error(tls.ssl, rv);
@@ -656,6 +733,14 @@ ssize_t Connection::write_tls(const void *data, size_t len) {
 }
 
 ssize_t Connection::read_tls(void *data, size_t len) {
+  ERR_clear_error();
+
+#if OPENSSL_1_1_1_API
+  if (tls.earlybuf.rleft()) {
+    return tls.earlybuf.remove(data, len);
+  }
+#endif // OPENSSL_1_1_1_API
+
   // SSL_read requires the same arguments (buf pointer and its
   // length) on SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE.
   // rlimit_.avail() or rlimit_.avail() may return different length
@@ -673,7 +758,46 @@ ssize_t Connection::read_tls(void *data, size_t len) {
     tls.last_readlen = 0;
   }
 
-  ERR_clear_error();
+#if OPENSSL_1_1_1_API
+  if (!tls.early_data_finish) {
+    // TLSv1.3 handshake is still going on.
+    size_t nread;
+    auto rv = SSL_read_early_data(tls.ssl, data, len, &nread);
+    if (rv == SSL_READ_EARLY_DATA_ERROR) {
+      auto err = SSL_get_error(tls.ssl, rv);
+      switch (err) {
+      case SSL_ERROR_WANT_READ:
+        tls.last_readlen = len;
+        return 0;
+      case SSL_ERROR_SSL:
+        if (LOG_ENABLED(INFO)) {
+          LOG(INFO) << "SSL_read: "
+                    << ERR_error_string(ERR_get_error(), nullptr);
+        }
+        return SHRPX_ERR_NETWORK;
+      default:
+        if (LOG_ENABLED(INFO)) {
+          LOG(INFO) << "SSL_read: SSL_get_error returned " << err;
+        }
+        return SHRPX_ERR_NETWORK;
+      }
+    }
+
+    if (LOG_ENABLED(INFO)) {
+      LOG(INFO) << "tls: read early data " << nread << " bytes";
+    }
+
+    if (rv == SSL_READ_EARLY_DATA_FINISH) {
+      if (LOG_ENABLED(INFO)) {
+        LOG(INFO) << "tls: read all early data";
+      }
+      tls.early_data_finish = true;
+      // We may have stopped write watcher in write_tls.
+      wlimit.startw();
+    }
+    return nread;
+  }
+#endif // OPENSSL_1_1_1_API
 
   auto rv = SSL_read(tls.ssl, data, len);
 

src/shrpx_connection.h

@@ -56,6 +56,8 @@ enum {
 struct TLSConnection {
   DefaultMemchunks wbuf;
   DefaultPeekMemchunks rbuf;
+  // Stores TLSv1.3 early data.
+  DefaultMemchunks earlybuf;
   SSL *ssl;
   SSL_SESSION *cached_session;
   MemcachedRequest *cached_session_lookup_req;
@@ -74,6 +76,12 @@ struct TLSConnection {
   // true if ssl is initialized as server, and client requested
   // signed_certificate_timestamp extension.
   bool sct_requested;
+  // true if TLSv1.3 early data has been completely received.  Since
+  // SSL_read_early_data acts like SSL_do_handshake, this field may be
+  // true even if the negotiated TLS version is TLSv1.2 or earlier.
+  // This value is also true if this is client side connection for
+  // convenience.
+  bool early_data_finish;
 };
 
 struct TCPHint {

src/shrpx_http2_downstream_connection.cc

@@ -41,6 +41,7 @@
 #include "shrpx_log.h"
 #include "http2.h"
 #include "util.h"
+#include "ssl_compat.h"
 
 using namespace nghttp2;
 
@@ -271,7 +272,7 @@ int Http2DownstreamConnection::push_request_headers() {
     num_cookies = downstream_->count_crumble_request_cookie();
   }
 
-  // 9 means:
+  // 10 means:
   // 1. :method
   // 2. :scheme
   // 3. :path
@@ -281,8 +282,9 @@ int Http2DownstreamConnection::push_request_headers() {
   // 7. x-forwarded-proto (optional)
   // 8. te (optional)
   // 9. forwarded (optional)
+  // 10. early-data (optional)
   auto nva = std::vector<nghttp2_nv>();
-  nva.reserve(req.fs.headers().size() + 9 + num_cookies +
+  nva.reserve(req.fs.headers().size() + 10 + num_cookies +
               httpconf.add_request_headers.size());
 
   nva.push_back(
@@ -333,6 +335,14 @@ int Http2DownstreamConnection::push_request_headers() {
   auto upstream = downstream_->get_upstream();
   auto handler = upstream->get_client_handler();
 
+#if OPENSSL_1_1_1_API
+  auto conn = handler->get_connection();
+
+  if (!SSL_is_init_finished(conn->tls.ssl)) {
+    nva.push_back(http2::make_nv_ll("early-data", "1"));
+  }
+#endif // OPENSSL_1_1_1_API
+
   auto fwd =
       fwdconf.strip_incoming ? nullptr : req.fs.header(http2::HD_FORWARDED);
 

src/shrpx_http_downstream_connection.cc

@@ -39,6 +39,7 @@
 #include "shrpx_log.h"
 #include "http2.h"
 #include "util.h"
+#include "ssl_compat.h"
 
 using namespace nghttp2;
 
@@ -584,6 +585,14 @@ int HttpDownstreamConnection::push_request_headers() {
   auto upstream = downstream_->get_upstream();
   auto handler = upstream->get_client_handler();
 
+#if OPENSSL_1_1_1_API
+  auto conn = handler->get_connection();
+
+  if (!SSL_is_init_finished(conn->tls.ssl)) {
+    buf->append("Early-Data: 1\r\n");
+  }
+#endif // OPENSSL_1_1_1_API
+
   auto fwd =
       fwdconf.strip_incoming ? nullptr : req.fs.header(http2::HD_FORWARDED);
 

src/shrpx_rate_limit.cc

@@ -108,8 +108,9 @@ void RateLimit::stopw() {
 }
 
 void RateLimit::handle_tls_pending_read() {
-  if (!conn_ || !conn_->tls.ssl ||
+  if (!conn_ || !conn_->tls.ssl || !conn_->tls.initial_handshake_done ||
-      (SSL_pending(conn_->tls.ssl) == 0 && conn_->tls.rbuf.rleft() == 0)) {
+      (SSL_pending(conn_->tls.ssl) == 0 && conn_->tls.rbuf.rleft() == 0 &&
+       conn_->tls.earlybuf.rleft() == 0)) {
     return;
   }
 

src/shrpx_tls.cc

@@ -517,6 +517,13 @@ int ticket_key_cb(SSL *ssl, unsigned char *key_name, unsigned char *iv,
 
 namespace {
 void info_callback(const SSL *ssl, int where, int ret) {
+#ifdef TLS1_3_VERSION
+  // TLSv1.3 has no renegotiation.
+  if (SSL_version(ssl) == TLS1_3_VERSION) {
+    return;
+  }
+#endif // TLS1_3_VERSION
+
   // To mitigate possible DOS attack using lots of renegotiations, we
   // disable renegotiation. Since OpenSSL does not provide an easy way
   // to disable it, we check that renegotiation is started in this
@@ -763,7 +770,17 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file,
       (SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS) | SSL_OP_NO_SSLv2 |
       SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION |
       SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION | SSL_OP_SINGLE_ECDH_USE |
-      SSL_OP_SINGLE_DH_USE | SSL_OP_CIPHER_SERVER_PREFERENCE;
+      SSL_OP_SINGLE_DH_USE |
+      SSL_OP_CIPHER_SERVER_PREFERENCE
+#if OPENSSL_1_1_1_API
+      // The reason for disabling built-in anti-replay in OpenSSL is
+      // that it only works if client gets back to the same server.
+      // The freshness check described in
+      // https://tools.ietf.org/html/rfc8446#section-8.3 is still
+      // performed.
+      | SSL_OP_NO_ANTI_REPLAY
+#endif // OPENSSL_1_1_1_API
+      ;
 
   auto config = mod_config();
   auto &tlsconf = config->tls;
@@ -966,6 +983,14 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file,
   }
 #endif // !LIBRESSL_IN_USE && OPENSSL_VERSION_NUMBER >= 0x10002000L
 
+#if OPENSSL_1_1_1_API
+  if (SSL_CTX_set_max_early_data(ssl_ctx, tlsconf.max_early_data) != 1) {
+    LOG(FATAL) << "SSL_CTX_set_max_early_data failed: "
+               << ERR_error_string(ERR_get_error(), nullptr);
+    DIE();
+  }
+#endif // OPENSSL_1_1_1_API
+
 #ifndef OPENSSL_NO_PSK
   SSL_CTX_set_psk_server_callback(ssl_ctx, psk_server_cb);
 #endif // !LIBRESSL_NO_PSK