From 09344eb1ad13bcfe25920107a9d4c0999a9d0f3f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 7 May 2022 10:52:07 +0900 Subject: [PATCH] nghttp, nghttpd, nghttpx: Add ktls support --- .github/workflows/build.yml | 2 +- gennghttpxfun.py | 1 + src/HttpServer.cc | 9 +- src/HttpServer.h | 1 + src/nghttp.cc | 15 +++- src/nghttp.h | 1 + src/nghttpd.cc | 6 ++ src/shrpx.cc | 6 ++ src/shrpx_config.cc | 8 ++ src/shrpx_config.h | 3 + src/shrpx_connection.cc | 166 ++++++++++++++++++++++++++++++++++-- src/shrpx_connection.h | 1 + src/shrpx_tls.cc | 43 ++++++---- 13 files changed, 237 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0569ad14..401de66e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -110,7 +110,7 @@ jobs: git clone --depth 1 -b openssl-3.0.3+quic https://github.com/quictls/openssl cd openssl - ./config enable-tls1_3 --prefix=$PWD/build --libdir=$PWD/build/lib + ./config enable-tls1_3 enable-ktls --prefix=$PWD/build --libdir=$PWD/build/lib make -j$(nproc) make install_sw - name: Build nghttp3 diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 97ec0f69..ffd253f0 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -199,6 +199,7 @@ OPTIONS = [ "worker-process-grace-shutdown-period", "frontend-quic-initial-rtt", "require-http-scheme", + "tls-ktls", ] LOGVARS = [ diff --git a/src/HttpServer.cc b/src/HttpServer.cc index e82310e9..6598e6f0 100644 --- a/src/HttpServer.cc +++ b/src/HttpServer.cc @@ -113,7 +113,8 @@ Config::Config() early_response(false), hexdump(false), echo_upload(false), - no_content_length(false) {} + no_content_length(false), + ktls(false) {} Config::~Config() {} @@ -2122,6 +2123,12 @@ int HttpServer::run() { SSL_OP_SINGLE_ECDH_USE | SSL_OP_NO_TICKET | SSL_OP_CIPHER_SERVER_PREFERENCE; +#ifdef SSL_OP_ENABLE_KTLS + if (config_->ktls) { + ssl_opts |= SSL_OP_ENABLE_KTLS; + } +#endif // SSL_OP_ENABLE_KTLS + SSL_CTX_set_options(ssl_ctx, ssl_opts); SSL_CTX_set_mode(ssl_ctx, SSL_MODE_AUTO_RETRY); SSL_CTX_set_mode(ssl_ctx, SSL_MODE_RELEASE_BUFFERS); diff --git a/src/HttpServer.h b/src/HttpServer.h index 0de90e42..7776ab6e 100644 --- a/src/HttpServer.h +++ b/src/HttpServer.h @@ -82,6 +82,7 @@ struct Config { bool hexdump; bool echo_upload; bool no_content_length; + bool ktls; Config(); ~Config(); }; diff --git a/src/nghttp.cc b/src/nghttp.cc index 5d62baef..46c777ea 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -122,7 +122,8 @@ Config::Config() hexdump(false), no_push(false), expect_continue(false), - verify_peer(true) { + verify_peer(true), + ktls(false) { nghttp2_option_new(&http2_option); nghttp2_option_set_peer_max_concurrent_streams(http2_option, peer_max_concurrent_streams); @@ -2280,6 +2281,12 @@ int communicate( SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION | SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION; +#ifdef SSL_OP_ENABLE_KTLS + if (config.ktls) { + ssl_opts |= SSL_OP_ENABLE_KTLS; + } +#endif // SSL_OP_ENABLE_KTLS + SSL_CTX_set_options(ssl_ctx, ssl_opts); SSL_CTX_set_mode(ssl_ctx, SSL_MODE_AUTO_RETRY); SSL_CTX_set_mode(ssl_ctx, SSL_MODE_RELEASE_BUFFERS); @@ -2748,6 +2755,7 @@ Options: -y, --no-verify-peer Suppress warning on server certificate verification failure. + --ktls Enable ktls. --version Display version information and exit. -h, --help Display this help and exit. @@ -2803,6 +2811,7 @@ int main(int argc, char **argv) { {"max-concurrent-streams", required_argument, &flag, 12}, {"expect-continue", no_argument, &flag, 13}, {"encoder-header-table-size", required_argument, &flag, 14}, + {"ktls", no_argument, &flag, 15}, {nullptr, 0, nullptr, 0}}; int option_index = 0; int c = @@ -3030,6 +3039,10 @@ int main(int argc, char **argv) { config.encoder_header_table_size = n; break; } + case 15: + // ktls option + config.ktls = true; + break; } break; default: diff --git a/src/nghttp.h b/src/nghttp.h index bec8a657..807cc0ef 100644 --- a/src/nghttp.h +++ b/src/nghttp.h @@ -97,6 +97,7 @@ struct Config { bool no_push; bool expect_continue; bool verify_peer; + bool ktls; }; enum class RequestState { INITIAL, ON_REQUEST, ON_RESPONSE, ON_COMPLETE }; diff --git a/src/nghttpd.cc b/src/nghttpd.cc index b930d47a..6de8cfa4 100644 --- a/src/nghttpd.cc +++ b/src/nghttpd.cc @@ -178,6 +178,7 @@ Options: << config.mime_types_file << R"( --no-content-length Don't send content-length header field. + --ktls Enable ktls. --version Display version information and exit. -h, --help Display this help and exit. @@ -228,6 +229,7 @@ int main(int argc, char **argv) { {"mime-types-file", required_argument, &flag, 9}, {"no-content-length", no_argument, &flag, 10}, {"encoder-header-table-size", required_argument, &flag, 11}, + {"ktls", no_argument, &flag, 12}, {nullptr, 0, nullptr, 0}}; int option_index = 0; int c = getopt_long(argc, argv, "DVb:c:d:ehm:n:p:va:w:W:", long_options, @@ -407,6 +409,10 @@ int main(int argc, char **argv) { config.encoder_header_table_size = n; break; } + case 12: + // tls option + config.ktls = true; + break; } break; default: diff --git a/src/shrpx.cc b/src/shrpx.cc index f9c4e843..e94c9390 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2921,6 +2921,7 @@ SSL/TLS: accepts. Default: )" << util::utos_unit(config->tls.max_early_data) << R"( + --tls-ktls Enable ktls. HTTP/2: -c, --frontend-http2-max-concurrent-streams= @@ -4263,6 +4264,7 @@ int main(int argc, char **argv) { {SHRPX_OPT_FRONTEND_QUIC_INITIAL_RTT.c_str(), required_argument, &flag, 190}, {SHRPX_OPT_REQUIRE_HTTP_SCHEME.c_str(), no_argument, &flag, 191}, + {SHRPX_OPT_TLS_KTLS.c_str(), no_argument, &flag, 192}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -5172,6 +5174,10 @@ int main(int argc, char **argv) { cmdcfgs.emplace_back(SHRPX_OPT_REQUIRE_HTTP_SCHEME, StringRef::from_lit("yes")); break; + case 192: + // --tls-ktls + cmdcfgs.emplace_back(SHRPX_OPT_TLS_KTLS, StringRef::from_lit("yes")); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index a4676e2f..8af7f52e 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1903,6 +1903,11 @@ int option_lookup_token(const char *name, size_t namelen) { return SHRPX_OPTID_FASTOPEN; } break; + case 's': + if (util::strieq_l("tls-ktl", name, 7)) { + return SHRPX_OPTID_TLS_KTLS; + } + break; case 't': if (util::strieq_l("npn-lis", name, 7)) { return SHRPX_OPTID_NPN_LIST; @@ -4188,6 +4193,9 @@ int parse_config(Config *config, int optid, const StringRef &opt, case SHRPX_OPTID_REQUIRE_HTTP_SCHEME: config->http.require_http_scheme = util::strieq_l("yes", optarg); return 0; + case SHRPX_OPTID_TLS_KTLS: + config->tls.ktls = util::strieq_l("yes", optarg); + return 0; case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 38becc61..0c43a3c2 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -404,6 +404,7 @@ constexpr auto SHRPX_OPT_FRONTEND_QUIC_INITIAL_RTT = StringRef::from_lit("frontend-quic-initial-rtt"); constexpr auto SHRPX_OPT_REQUIRE_HTTP_SCHEME = StringRef::from_lit("require-http-scheme"); +constexpr auto SHRPX_OPT_TLS_KTLS = StringRef::from_lit("tls-ktls"); constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -783,6 +784,7 @@ struct TLSConfig { // true if forwarding requests included in TLS early data should not // be postponed until TLS handshake finishes. bool no_postpone_early_data; + bool ktls; }; #ifdef ENABLE_HTTP3 @@ -1332,6 +1334,7 @@ enum { SHRPX_OPTID_SYSLOG_FACILITY, SHRPX_OPTID_TLS_DYN_REC_IDLE_TIMEOUT, SHRPX_OPTID_TLS_DYN_REC_WARMUP_THRESHOLD, + SHRPX_OPTID_TLS_KTLS, SHRPX_OPTID_TLS_MAX_EARLY_DATA, SHRPX_OPTID_TLS_MAX_PROTO_VERSION, SHRPX_OPTID_TLS_MIN_PROTO_VERSION, diff --git a/src/shrpx_connection.cc b/src/shrpx_connection.cc index b4211adc..f0669638 100644 --- a/src/shrpx_connection.cc +++ b/src/shrpx_connection.cc @@ -312,8 +312,8 @@ BIO_METHOD *create_bio_method() { void Connection::set_ssl(SSL *ssl) { tls.ssl = ssl; - if (proto != Proto::HTTP3) { - auto &tlsconf = get_config()->tls; + auto &tlsconf = get_config()->tls; + if (proto != Proto::HTTP3 && !tlsconf.session_cache.memcached.host.empty()) { auto bio = BIO_new(tlsconf.bio_method); BIO_set_data(bio, this); SSL_set_bio(tls.ssl, bio, bio); @@ -336,6 +336,12 @@ int Connection::tls_handshake() { wlimit.stopw(); ev_timer_stop(loop, &wt); + auto &tlsconf = get_config()->tls; + + if (tlsconf.session_cache.memcached.host.empty()) { + return tls_handshake_simple(); + } + std::array buf; if (ev_is_active(&rev)) { @@ -397,10 +403,6 @@ int Connection::tls_handshake() { ERR_clear_error(); -#if OPENSSL_1_1_1_API || defined(OPENSSL_IS_BORINGSSL) - auto &tlsconf = get_config()->tls; -#endif // OPENSSL_1_1_1_API || defined(OPENSSL_IS_BORINGSSL) - #if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) if (!tls.server_handshake || tls.early_data_finish) { rv = SSL_do_handshake(tls.ssl); @@ -592,6 +594,158 @@ int Connection::tls_handshake() { return write_tls_pending_handshake(); } +int Connection::tls_handshake_simple() { + wlimit.stopw(); + ev_timer_stop(loop, &wt); + + if (tls.initial_handshake_done) { + return write_tls_pending_handshake(); + } + + if (SSL_get_fd(tls.ssl) == -1) { + SSL_set_fd(tls.ssl, fd); + } + + int rv; +#if OPENSSL_1_1_1_API || defined(OPENSSL_IS_BORINGSSL) + auto &tlsconf = get_config()->tls; + std::array buf; +#endif // OPENSSL_1_1_1_API || defined(OPENSSL_IS_BORINGSSL) + + ERR_clear_error(); + +#if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) + if (!tls.server_handshake || tls.early_data_finish) { + rv = SSL_do_handshake(tls.ssl); + } else { + for (;;) { + size_t nread; + + rv = SSL_read_early_data(tls.ssl, buf.data(), buf.size(), &nread); + if (rv == SSL_READ_EARLY_DATA_ERROR) { + // If we have early data, and server sends ServerHello, assume + // that handshake is completed in server side, and start + // processing request. If we don't exit handshake code here, + // server waits for EndOfEarlyData and Finished message from + // client, which voids the purpose of 0-RTT data. The left + // over of handshake is done through write_tls or read_tls. + if (tlsconf.no_postpone_early_data && tls.earlybuf.rleft()) { + rv = 1; + } + + break; + } + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "tls: read early data " << nread << " bytes"; + } + + tls.earlybuf.append(buf.data(), nread); + + if (rv == SSL_READ_EARLY_DATA_FINISH) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "tls: read all early data; total " + << tls.earlybuf.rleft() << " bytes"; + } + tls.early_data_finish = true; + // The same reason stated above. + if (tlsconf.no_postpone_early_data && tls.earlybuf.rleft()) { + rv = 1; + } else { + ERR_clear_error(); + rv = SSL_do_handshake(tls.ssl); + } + break; + } + } + } +#else // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) + rv = SSL_do_handshake(tls.ssl); +#endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)) + + if (rv <= 0) { + auto err = SSL_get_error(tls.ssl, rv); + switch (err) { + case SSL_ERROR_WANT_READ: + if (read_buffer_full(tls.rbuf)) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "tls: handshake message is too large"; + } + return -1; + } + break; + case SSL_ERROR_WANT_WRITE: + wlimit.startw(); + ev_timer_again(loop, &wt); + break; + case SSL_ERROR_SSL: { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "tls: handshake libssl error: " + << ERR_error_string(ERR_get_error(), nullptr); + } + return SHRPX_ERR_NETWORK; + } + default: + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "tls: handshake libssl error " << err; + } + return SHRPX_ERR_NETWORK; + } + } + + if (rv != 1) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "tls: handshake is still in progress"; + } + return SHRPX_ERR_INPROGRESS; + } + +#ifdef OPENSSL_IS_BORINGSSL + if (!tlsconf.no_postpone_early_data && SSL_in_early_data(tls.ssl) && + SSL_in_init(tls.ssl)) { + auto nread = SSL_read(tls.ssl, buf.data(), buf.size()); + if (nread <= 0) { + auto err = SSL_get_error(tls.ssl, nread); + switch (err) { + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + break; + case SSL_ERROR_ZERO_RETURN: + return SHRPX_ERR_EOF; + case SSL_ERROR_SSL: + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "SSL_read: " + << ERR_error_string(ERR_get_error(), nullptr); + } + return SHRPX_ERR_NETWORK; + default: + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "SSL_read: SSL_get_error returned " << err; + } + return SHRPX_ERR_NETWORK; + } + } else { + tls.earlybuf.append(buf.data(), nread); + } + + if (SSL_in_init(tls.ssl)) { + return SHRPX_ERR_INPROGRESS; + } + } +#endif // OPENSSL_IS_BORINGSSL + + // Handshake was done + + rv = check_http2_requirement(); + if (rv != 0) { + return -1; + } + + tls.initial_handshake_done = true; + + return write_tls_pending_handshake(); +} + int Connection::write_tls_pending_handshake() { // Send handshake data left in the buffer while (tls.wbuf.rleft()) { diff --git a/src/shrpx_connection.h b/src/shrpx_connection.h index 5d0d79a3..9aed671f 100644 --- a/src/shrpx_connection.h +++ b/src/shrpx_connection.h @@ -109,6 +109,7 @@ struct Connection { void prepare_server_handshake(); int tls_handshake(); + int tls_handshake_simple(); int write_tls_pending_handshake(); int check_http2_requirement(); diff --git a/src/shrpx_tls.cc b/src/shrpx_tls.cc index 7913d86a..7a53f8d9 100644 --- a/src/shrpx_tls.cc +++ b/src/shrpx_tls.cc @@ -933,25 +933,31 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file, DIE(); } - constexpr auto ssl_opts = - (SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS) | SSL_OP_NO_SSLv2 | - SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION | - SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION | SSL_OP_SINGLE_ECDH_USE | - SSL_OP_SINGLE_DH_USE | - SSL_OP_CIPHER_SERVER_PREFERENCE + auto ssl_opts = (SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS) | + SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION | + SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION | + SSL_OP_SINGLE_ECDH_USE | SSL_OP_SINGLE_DH_USE | + SSL_OP_CIPHER_SERVER_PREFERENCE #if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) - // The reason for disabling built-in anti-replay in OpenSSL is - // that it only works if client gets back to the same server. - // The freshness check described in - // https://tools.ietf.org/html/rfc8446#section-8.3 is still - // performed. - | SSL_OP_NO_ANTI_REPLAY + // The reason for disabling built-in anti-replay in + // OpenSSL is that it only works if client gets back + // to the same server. The freshness check + // described in + // https://tools.ietf.org/html/rfc8446#section-8.3 + // is still performed. + | SSL_OP_NO_ANTI_REPLAY #endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL) ; auto config = mod_config(); auto &tlsconf = config->tls; +#ifdef SSL_OP_ENABLE_KTLS + if (tlsconf.ktls) { + ssl_opts |= SSL_OP_ENABLE_KTLS; + } +#endif // SSL_OP_ENABLE_KTLS + SSL_CTX_set_options(ssl_ctx, ssl_opts | tlsconf.tls_proto_mask); if (nghttp2::tls::ssl_ctx_set_proto_versions( @@ -1700,13 +1706,18 @@ SSL_CTX *create_ssl_client_context( DIE(); } - constexpr auto ssl_opts = (SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS) | - SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | - SSL_OP_NO_COMPRESSION | - SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION; + auto ssl_opts = (SSL_OP_ALL & ~SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS) | + SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION | + SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION; auto &tlsconf = get_config()->tls; +#ifdef SSL_OP_ENABLE_KTLS + if (tlsconf.ktls) { + ssl_opts |= SSL_OP_ENABLE_KTLS; + } +#endif // SSL_OP_ENABLE_KTLS + SSL_CTX_set_options(ssl_ctx, ssl_opts | tlsconf.tls_proto_mask); SSL_CTX_set_session_cache_mode(ssl_ctx, SSL_SESS_CACHE_CLIENT |