nghttpx: Read quic packet

This commit is contained in:
Tatsuhiro Tsujikawa 2021-08-16 15:11:18 +09:00
parent ef53db201e
commit 940fdd5573
15 changed files with 531 additions and 35 deletions

View File

@ -51,6 +51,7 @@
#include "shrpx_api_downstream_connection.h" #include "shrpx_api_downstream_connection.h"
#include "shrpx_health_monitor_downstream_connection.h" #include "shrpx_health_monitor_downstream_connection.h"
#include "shrpx_null_downstream_connection.h" #include "shrpx_null_downstream_connection.h"
#include "shrpx_http3_upstream.h"
#include "shrpx_log.h" #include "shrpx_log.h"
#include "util.h" #include "util.h"
#include "template.h" #include "template.h"
@ -286,6 +287,15 @@ int ClientHandler::write_tls() {
} }
} }
int ClientHandler::read_quic(const UpstreamAddr *faddr,
const Address &remote_addr,
const Address &local_addr, const uint8_t *data,
size_t datalen) {
auto upstream = static_cast<Http3Upstream *>(upstream_.get());
return upstream->on_read(faddr, remote_addr, local_addr, data, datalen);
}
int ClientHandler::upstream_noop() { return 0; } int ClientHandler::upstream_noop() { return 0; }
int ClientHandler::upstream_read() { int ClientHandler::upstream_read() {
@ -402,7 +412,8 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl,
get_config()->conn.upstream.ratelimit.write, get_config()->conn.upstream.ratelimit.write,
get_config()->conn.upstream.ratelimit.read, writecb, readcb, get_config()->conn.upstream.ratelimit.read, writecb, readcb,
timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold,
get_config()->tls.dyn_rec.idle_timeout, Proto::NONE), get_config()->tls.dyn_rec.idle_timeout,
faddr->quic ? Proto::HTTP3 : Proto::NONE),
ipaddr_(make_string_ref(balloc_, ipaddr)), ipaddr_(make_string_ref(balloc_, ipaddr)),
port_(make_string_ref(balloc_, port)), port_(make_string_ref(balloc_, port)),
faddr_(faddr), faddr_(faddr),
@ -423,14 +434,16 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl,
auto config = get_config(); auto config = get_config();
if (faddr_->accept_proxy_protocol || if (!faddr->quic) {
config->conn.upstream.accept_proxy_protocol) { if (faddr_->accept_proxy_protocol ||
read_ = &ClientHandler::read_clear; config->conn.upstream.accept_proxy_protocol) {
write_ = &ClientHandler::noop; read_ = &ClientHandler::read_clear;
on_read_ = &ClientHandler::proxy_protocol_read; write_ = &ClientHandler::noop;
on_write_ = &ClientHandler::upstream_noop; on_read_ = &ClientHandler::proxy_protocol_read;
} else { on_write_ = &ClientHandler::upstream_noop;
setup_upstream_io_callback(); } else {
setup_upstream_io_callback();
}
} }
auto &fwdconf = config->http.forwarded; auto &fwdconf = config->http.forwarded;
@ -492,6 +505,12 @@ void ClientHandler::setup_upstream_io_callback() {
} }
} }
void ClientHandler::setup_http3_upstream(
std::unique_ptr<Http3Upstream> &&upstream) {
upstream_ = std::move(upstream);
alpn_ = StringRef::from_lit("h3");
}
ClientHandler::~ClientHandler() { ClientHandler::~ClientHandler() {
if (LOG_ENABLED(INFO)) { if (LOG_ENABLED(INFO)) {
CLOG(INFO, this) << "Deleting"; CLOG(INFO, this) << "Deleting";

View File

@ -53,6 +53,7 @@ class Downstream;
struct WorkerStat; struct WorkerStat;
struct DownstreamAddrGroup; struct DownstreamAddrGroup;
struct DownstreamAddr; struct DownstreamAddr;
class Http3Upstream;
class ClientHandler { class ClientHandler {
public: public:
@ -70,6 +71,9 @@ public:
int read_tls(); int read_tls();
int write_tls(); int write_tls();
int read_quic(const UpstreamAddr *faddr, const Address &remote_addr,
const Address &local_addr, const uint8_t *data, size_t datalen);
int upstream_noop(); int upstream_noop();
int upstream_read(); int upstream_read();
int upstream_http2_connhd_read(); int upstream_http2_connhd_read();
@ -143,6 +147,8 @@ public:
void setup_upstream_io_callback(); void setup_upstream_io_callback();
void setup_http3_upstream(std::unique_ptr<Http3Upstream> &&upstream);
// Returns string suitable for use in "by" parameter of Forwarded // Returns string suitable for use in "by" parameter of Forwarded
// header field. // header field.
StringRef get_forwarded_by() const; StringRef get_forwarded_by() const;

View File

@ -2668,6 +2668,7 @@ int parse_config(Config *config, int optid, const StringRef &opt,
addr.sni_fwd = params.sni_fwd; addr.sni_fwd = params.sni_fwd;
addr.alt_mode = params.alt_mode; addr.alt_mode = params.alt_mode;
addr.accept_proxy_protocol = params.proxyproto; addr.accept_proxy_protocol = params.proxyproto;
addr.quic = params.quic;
if (addr.alt_mode == UpstreamAltMode::API) { if (addr.alt_mode == UpstreamAltMode::API) {
apiconf.enabled = true; apiconf.enabled = true;
@ -3990,6 +3991,8 @@ StringRef strproto(Proto proto) {
return StringRef::from_lit("http/1.1"); return StringRef::from_lit("http/1.1");
case Proto::HTTP2: case Proto::HTTP2:
return StringRef::from_lit("h2"); return StringRef::from_lit("h2");
case Proto::HTTP3:
return StringRef::from_lit("h3");
case Proto::MEMCACHED: case Proto::MEMCACHED:
return StringRef::from_lit("memcached"); return StringRef::from_lit("memcached");
} }

View File

@ -370,6 +370,7 @@ enum class Proto {
NONE, NONE,
HTTP1, HTTP1,
HTTP2, HTTP2,
HTTP3,
MEMCACHED, MEMCACHED,
}; };
@ -458,6 +459,7 @@ struct UpstreamAddr {
bool sni_fwd; bool sni_fwd;
// true if client is supposed to send PROXY protocol v1 header. // true if client is supposed to send PROXY protocol v1 header.
bool accept_proxy_protocol; bool accept_proxy_protocol;
bool quic;
int fd; int fd;
}; };
@ -702,6 +704,12 @@ struct TLSConfig {
bool no_postpone_early_data; bool no_postpone_early_data;
}; };
struct QUICConfig {
struct {
std::array<uint8_t, 32> secret;
} stateless_reset;
};
// custom error page // custom error page
struct ErrorPage { struct ErrorPage {
// not NULL-terminated // not NULL-terminated
@ -954,6 +962,7 @@ struct Config {
http{}, http{},
http2{}, http2{},
tls{}, tls{},
quic{},
logging{}, logging{},
conn{}, conn{},
api{}, api{},
@ -987,6 +996,7 @@ struct Config {
HttpConfig http; HttpConfig http;
Http2Config http2; Http2Config http2;
TLSConfig tls; TLSConfig tls;
QUICConfig quic;
LoggingConfig logging; LoggingConfig logging;
ConnectionConfig conn; ConnectionConfig conn;
APIConfig api; APIConfig api;

View File

@ -312,10 +312,13 @@ BIO_METHOD *create_bio_method() {
void Connection::set_ssl(SSL *ssl) { void Connection::set_ssl(SSL *ssl) {
tls.ssl = ssl; tls.ssl = ssl;
auto &tlsconf = get_config()->tls; if (proto != Proto::HTTP3) {
auto bio = BIO_new(tlsconf.bio_method); auto &tlsconf = get_config()->tls;
BIO_set_data(bio, this); auto bio = BIO_new(tlsconf.bio_method);
SSL_set_bio(tls.ssl, bio, bio); BIO_set_data(bio, this);
SSL_set_bio(tls.ssl, bio, bio);
}
SSL_set_app_data(tls.ssl, this); SSL_set_app_data(tls.ssl, this);
} }

View File

@ -222,11 +222,11 @@ int ConnectionHandler::create_single_worker() {
); );
quic_cert_tree_ = tls::create_cert_lookup_tree(); quic_cert_tree_ = tls::create_cert_lookup_tree();
tls::setup_quic_server_ssl_context(quic_all_ssl_ctx_, quic_indexed_ssl_ctx_, auto quic_sv_ssl_ctx = tls::setup_quic_server_ssl_context(
quic_cert_tree_.get() quic_all_ssl_ctx_, quic_indexed_ssl_ctx_, quic_cert_tree_.get()
#ifdef HAVE_NEVERBLEED #ifdef HAVE_NEVERBLEED
, ,
nb_ nb_
#endif // HAVE_NEVERBLEED #endif // HAVE_NEVERBLEED
); );
@ -261,7 +261,8 @@ int ConnectionHandler::create_single_worker() {
single_worker_ = std::make_unique<Worker>( single_worker_ = std::make_unique<Worker>(
loop_, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(), loop_, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(),
ticket_keys_, this, config->conn.downstream); quic_sv_ssl_ctx, quic_cert_tree_.get(), ticket_keys_, this,
config->conn.downstream);
#ifdef HAVE_MRUBY #ifdef HAVE_MRUBY
if (single_worker_->create_mruby_context() != 0) { if (single_worker_->create_mruby_context() != 0) {
return -1; return -1;
@ -288,11 +289,11 @@ int ConnectionHandler::create_worker_thread(size_t num) {
); );
quic_cert_tree_ = tls::create_cert_lookup_tree(); quic_cert_tree_ = tls::create_cert_lookup_tree();
tls::setup_quic_server_ssl_context(quic_all_ssl_ctx_, quic_indexed_ssl_ctx_, auto quic_sv_ssl_ctx = tls::setup_quic_server_ssl_context(
quic_cert_tree_.get() quic_all_ssl_ctx_, quic_indexed_ssl_ctx_, quic_cert_tree_.get()
# ifdef HAVE_NEVERBLEED # ifdef HAVE_NEVERBLEED
, ,
nb_ nb_
# endif // HAVE_NEVERBLEED # endif // HAVE_NEVERBLEED
); );
@ -337,7 +338,8 @@ int ConnectionHandler::create_worker_thread(size_t num) {
auto worker = std::make_unique<Worker>( auto worker = std::make_unique<Worker>(
loop, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(), loop, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, cert_tree_.get(),
ticket_keys_, this, config->conn.downstream); quic_sv_ssl_ctx, quic_cert_tree_.get(), ticket_keys_, this,
config->conn.downstream);
# ifdef HAVE_MRUBY # ifdef HAVE_MRUBY
if (worker->create_mruby_context() != 0) { if (worker->create_mruby_context() != 0) {
return -1; return -1;

View File

@ -23,17 +23,201 @@
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/ */
#include "shrpx_http3_upstream.h" #include "shrpx_http3_upstream.h"
#include <stdio.h>
#include <ngtcp2/ngtcp2_crypto.h>
#include "shrpx_client_handler.h" #include "shrpx_client_handler.h"
#include "shrpx_downstream.h" #include "shrpx_downstream.h"
#include "shrpx_downstream_connection.h" #include "shrpx_downstream_connection.h"
#include "shrpx_log.h" #include "shrpx_log.h"
#include "shrpx_quic.h"
#include "shrpx_worker.h"
#include "util.h"
namespace shrpx { namespace shrpx {
Http3Upstream::Http3Upstream(ClientHandler *handler) Http3Upstream::Http3Upstream(ClientHandler *handler)
: handler_{handler}, tls_alert_{0} {} : handler_{handler},
conn_{nullptr},
last_error_{QUICErrorType::Transport, 0},
tls_alert_{0} {}
Http3Upstream::~Http3Upstream() {} Http3Upstream::~Http3Upstream() {
if (conn_) {
auto worker = handler_->get_worker();
auto quic_client_handler = worker->get_quic_connection_handler();
quic_client_handler->remove_connection_id(&initial_client_dcid_);
std::vector<ngtcp2_cid> scids(ngtcp2_conn_get_num_scid(conn_));
ngtcp2_conn_get_scid(conn_, scids.data());
for (auto &cid : scids) {
quic_client_handler->remove_connection_id(&cid);
}
ngtcp2_conn_del(conn_);
}
}
namespace {
void log_printf(void *user_data, const char *fmt, ...) {
va_list ap;
std::array<char, 4096> buf;
va_start(ap, fmt);
auto nwrite = vsnprintf(buf.data(), buf.size(), fmt, ap);
va_end(ap);
if (nwrite >= buf.size()) {
nwrite = buf.size() - 1;
}
buf[nwrite++] = '\n';
write(fileno(stderr), buf.data(), nwrite);
}
} // namespace
namespace {
void rand(uint8_t *dest, size_t destlen, const ngtcp2_rand_ctx *rand_ctx) {
util::random_bytes(dest, dest + destlen,
*static_cast<std::mt19937 *>(rand_ctx->native_handle));
}
} // namespace
namespace {
int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token,
size_t cidlen, void *user_data) {
if (generate_quic_connection_id(cid, cidlen) != 0) {
return NGTCP2_ERR_CALLBACK_FAILURE;
}
auto config = get_config();
auto &quicconf = config->quic;
auto &secret = quicconf.stateless_reset.secret;
if (generate_quic_stateless_reset_token(token, cid, secret.data(),
secret.size()) != 0) {
return NGTCP2_ERR_CALLBACK_FAILURE;
}
return 0;
}
} // namespace
namespace {
int remove_connection_id(ngtcp2_conn *conn, const ngtcp2_cid *cid,
void *user_data) {
auto upstream = static_cast<Http3Upstream *>(user_data);
auto handler = upstream->get_client_handler();
auto worker = handler->get_worker();
auto quic_conn_handler = worker->get_quic_connection_handler();
quic_conn_handler->remove_connection_id(cid);
return 0;
}
} // namespace
int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
const Address &local_addr,
const ngtcp2_pkt_hd &initial_hd) {
int rv;
auto worker = handler_->get_worker();
auto callbacks = ngtcp2_callbacks{
nullptr, // client_initial
ngtcp2_crypto_recv_client_initial_cb,
ngtcp2_crypto_recv_crypto_data_cb,
nullptr, // handshake_completed
nullptr, // recv_version_negotiation
ngtcp2_crypto_encrypt_cb,
ngtcp2_crypto_decrypt_cb,
ngtcp2_crypto_hp_mask_cb,
nullptr, // recv_stream_data
nullptr, // acked_crypto_offset
nullptr, // acked_stream_data_offset
nullptr, // stream_open
nullptr, // stream_close
nullptr, // recv_stateless_reset
nullptr, // recv_retry
nullptr, // extend_max_local_streams_bidi
nullptr, // extend_max_local_streams_uni
rand,
get_new_connection_id,
remove_connection_id,
ngtcp2_crypto_update_key_cb,
nullptr, // path_validation
nullptr, // select_preferred_addr
nullptr, // stream_reset
nullptr, // extend_max_remote_streams_bidi
nullptr, // extend_max_remote_streams_uni
nullptr, // extend_max_stream_data
nullptr, // dcid_status
nullptr, // handshake_confirmed
nullptr, // recv_new_token
ngtcp2_crypto_delete_crypto_aead_ctx_cb,
ngtcp2_crypto_delete_crypto_cipher_ctx_cb,
nullptr, // recv_datagram
nullptr, // ack_datagram
nullptr, // lost_datagram
ngtcp2_crypto_get_path_challenge_data_cb,
nullptr, // stream_stop_sending
};
initial_client_dcid_ = initial_hd.dcid;
ngtcp2_cid scid;
if (generate_quic_connection_id(&scid, SHRPX_QUIC_SCIDLEN) != 0) {
return -1;
}
ngtcp2_settings settings;
ngtcp2_settings_default(&settings);
settings.log_printf = log_printf;
settings.initial_ts = quic_timestamp();
settings.cc_algo = NGTCP2_CC_ALGO_BBR;
settings.max_window = 6_m;
settings.max_stream_window = 6_m;
settings.max_udp_payload_size = SHRPX_MAX_UDP_PAYLOAD_SIZE;
settings.rand_ctx = {&worker->get_randgen()};
ngtcp2_transport_params params;
ngtcp2_transport_params_default(&params);
params.initial_max_data = 1_m;
params.initial_max_stream_data_bidi_remote = 256_k;
params.initial_max_stream_data_uni = 256_k;
params.max_idle_timeout = 30 * NGTCP2_SECONDS;
params.original_dcid = initial_hd.dcid;
auto path = ngtcp2_path{
{local_addr.len, const_cast<sockaddr *>(&local_addr.su.sa)},
{remote_addr.len, const_cast<sockaddr *>(&remote_addr.su.sa)},
const_cast<UpstreamAddr *>(faddr),
};
rv = ngtcp2_conn_server_new(&conn_, &initial_hd.scid, &scid, &path,
initial_hd.version, &callbacks, &settings,
&params, nullptr, this);
if (rv != 0) {
LOG(ERROR) << "ngtcp2_conn_server_new: " << ngtcp2_strerror(rv);
return -1;
}
ngtcp2_conn_set_tls_native_handle(conn_, handler_->get_ssl());
auto quic_connection_handler = worker->get_quic_connection_handler();
quic_connection_handler->add_connection_id(&initial_client_dcid_, handler_);
quic_connection_handler->add_connection_id(&scid, handler_);
return 0;
}
int Http3Upstream::on_read() { return 0; } int Http3Upstream::on_read() { return 0; }
@ -127,16 +311,76 @@ int Http3Upstream::on_read(const UpstreamAddr *faddr,
const Address &remote_addr, const Address &remote_addr,
const Address &local_addr, const uint8_t *data, const Address &local_addr, const uint8_t *data,
size_t datalen) { size_t datalen) {
int rv;
ngtcp2_pkt_info pi{};
auto path = ngtcp2_path{
{
local_addr.len,
const_cast<sockaddr *>(&local_addr.su.sa),
},
{
remote_addr.len,
const_cast<sockaddr *>(&remote_addr.su.sa),
},
const_cast<UpstreamAddr *>(faddr),
};
rv = ngtcp2_conn_read_pkt(conn_, &path, &pi, data, datalen, quic_timestamp());
if (rv != 0) {
LOG(ERROR) << "ngtcp2_conn_read_pkt: " << ngtcp2_strerror(rv);
switch (rv) {
case NGTCP2_ERR_DRAINING:
// TODO Start drain period
return -1;
case NGTCP2_ERR_RETRY:
// TODO Send Retry packet
return -1;
case NGTCP2_ERR_REQUIRED_TRANSPORT_PARAM:
case NGTCP2_ERR_MALFORMED_TRANSPORT_PARAM:
case NGTCP2_ERR_TRANSPORT_PARAM:
// If rv indicates transport_parameters related error, we should
// send TRANSPORT_PARAMETER_ERROR even if last_error_.code is
// already set. This is because OpenSSL might set Alert.
last_error_ = quic_err_transport(rv);
break;
case NGTCP2_ERR_DROP_CONN:
return -1;
default:
if (!last_error_.code) {
last_error_ = quic_err_transport(rv);
}
}
// TODO Send connection close
return handle_error();
}
return 0; return 0;
} }
int Http3Upstream::handle_error() { return -1; }
int Http3Upstream::on_rx_secret(ngtcp2_crypto_level level, int Http3Upstream::on_rx_secret(ngtcp2_crypto_level level,
const uint8_t *secret, size_t secretlen) { const uint8_t *secret, size_t secretlen) {
if (ngtcp2_crypto_derive_and_install_rx_key(conn_, nullptr, nullptr, nullptr,
level, secret, secretlen) != 0) {
LOG(ERROR) << "ngtcp2_crypto_derive_and_install_rx_key failed";
return -1;
}
return 0; return 0;
} }
int Http3Upstream::on_tx_secret(ngtcp2_crypto_level level, int Http3Upstream::on_tx_secret(ngtcp2_crypto_level level,
const uint8_t *secret, size_t secretlen) { const uint8_t *secret, size_t secretlen) {
if (ngtcp2_crypto_derive_and_install_tx_key(conn_, nullptr, nullptr, nullptr,
level, secret, secretlen) != 0) {
LOG(ERROR) << "ngtcp2_crypto_derive_and_install_tx_key failed";
return -1;
}
return 0; return 0;
} }

View File

@ -30,6 +30,7 @@
#include <ngtcp2/ngtcp2.h> #include <ngtcp2/ngtcp2.h>
#include "shrpx_upstream.h" #include "shrpx_upstream.h"
#include "shrpx_quic.h"
#include "network.h" #include "network.h"
using namespace nghttp2; using namespace nghttp2;
@ -84,6 +85,9 @@ public:
virtual bool push_enabled() const; virtual bool push_enabled() const;
virtual void cancel_premature_downstream(Downstream *promised_downstream); virtual void cancel_premature_downstream(Downstream *promised_downstream);
int init(const UpstreamAddr *faddr, const Address &remote_addr,
const Address &local_addr, const ngtcp2_pkt_hd &initial_hd);
int on_read(const UpstreamAddr *faddr, const Address &remote_addr, int on_read(const UpstreamAddr *faddr, const Address &remote_addr,
const Address &local_addr, const uint8_t *data, size_t datalen); const Address &local_addr, const uint8_t *data, size_t datalen);
@ -97,8 +101,13 @@ public:
void set_tls_alert(uint8_t alert); void set_tls_alert(uint8_t alert);
int handle_error();
private: private:
ClientHandler *handler_; ClientHandler *handler_;
ngtcp2_cid initial_client_dcid_;
ngtcp2_conn *conn_;
QUICError last_error_;
uint8_t tls_alert_; uint8_t tls_alert_;
}; };

View File

@ -29,6 +29,13 @@
#include <netdb.h> #include <netdb.h>
#include <array> #include <array>
#include <chrono>
#include <ngtcp2/ngtcp2_crypto.h>
#include <nghttp3/nghttp3.h>
#include <openssl/rand.h>
#include "shrpx_config.h" #include "shrpx_config.h"
#include "shrpx_log.h" #include "shrpx_log.h"
@ -39,6 +46,34 @@ using namespace nghttp2;
namespace shrpx { namespace shrpx {
QUICError quic_err_transport(int liberr) {
if (liberr == NGTCP2_ERR_RECV_VERSION_NEGOTIATION) {
return {QUICErrorType::TransportVersionNegotiation, 0};
}
return {QUICErrorType::Transport,
ngtcp2_err_infer_quic_transport_error_code(liberr)};
}
QUICError quic_err_idle_timeout() {
return {QUICErrorType::TransportIdleTimeout, 0};
}
QUICError quic_err_tls(int alert) {
return {QUICErrorType::Transport,
static_cast<uint64_t>(NGTCP2_CRYPTO_ERROR | alert)};
}
QUICError quic_err_app(int liberr) {
return {QUICErrorType::Application,
nghttp3_err_infer_quic_app_error_code(liberr)};
}
ngtcp2_tstamp quic_timestamp() {
return std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::steady_clock::now().time_since_epoch())
.count();
}
int create_quic_server_socket(UpstreamAddr &faddr) { int create_quic_server_socket(UpstreamAddr &faddr) {
std::array<char, STRERROR_BUFSIZE> errbuf; std::array<char, STRERROR_BUFSIZE> errbuf;
int fd = -1; int fd = -1;
@ -192,4 +227,28 @@ int quic_send_packet(const UpstreamAddr *addr, const sockaddr *remote_sa,
return 0; return 0;
} }
int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen) {
if (RAND_bytes(cid->data, cidlen) != 1) {
return -1;
}
cid->datalen = cidlen;
return 0;
}
int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid,
const uint8_t *secret,
size_t secretlen) {
ngtcp2_crypto_md md;
ngtcp2_crypto_md_init(&md, const_cast<EVP_MD *>(EVP_sha256()));
if (ngtcp2_crypto_generate_stateless_reset_token(token, &md, secret,
secretlen, cid) != 0) {
return -1;
}
return 0;
}
} // namespace shrpx } // namespace shrpx

View File

@ -29,11 +29,38 @@
#include <stdint.h> #include <stdint.h>
#include <ngtcp2/ngtcp2.h>
namespace shrpx { namespace shrpx {
struct UpstreamAddr; struct UpstreamAddr;
constexpr size_t SHRPX_QUIC_SCIDLEN = 20; constexpr size_t SHRPX_QUIC_SCIDLEN = 20;
constexpr size_t SHRPX_MAX_UDP_PAYLOAD_SIZE = 1280;
enum class QUICErrorType {
Application,
Transport,
TransportVersionNegotiation,
TransportIdleTimeout,
};
struct QUICError {
QUICError(QUICErrorType type, uint64_t code) : type{type}, code{code} {}
QUICErrorType type;
uint64_t code;
};
QUICError quic_err_transport(int liberr);
QUICError quic_err_idle_timeout();
QUICError quic_err_tls(int alert);
QUICError quic_err_app(int liberr);
ngtcp2_tstamp quic_timestamp();
int create_quic_server_socket(UpstreamAddr &addr); int create_quic_server_socket(UpstreamAddr &addr);
@ -42,6 +69,12 @@ int quic_send_packet(const UpstreamAddr *addr, const sockaddr *remote_sa,
size_t local_salen, const uint8_t *data, size_t datalen, size_t local_salen, const uint8_t *data, size_t datalen,
size_t gso_size); size_t gso_size);
int generate_quic_connection_id(ngtcp2_cid *cid, size_t cidlen);
int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid,
const uint8_t *secret,
size_t secretlen);
} // namespace shrpx } // namespace shrpx
#endif // SHRPX_QUIC_H #endif // SHRPX_QUIC_H

View File

@ -45,6 +45,12 @@ std::string make_cid_key(const uint8_t *dcid, size_t dcidlen) {
} }
} // namespace } // namespace
namespace {
std::string make_cid_key(const ngtcp2_cid *cid) {
return make_cid_key(cid->data, cid->datalen);
}
} // namespace
int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr, int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
const Address &remote_addr, const Address &remote_addr,
const Address &local_addr, const Address &local_addr,
@ -67,6 +73,8 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
auto dcid_key = make_cid_key(dcid, dcidlen); auto dcid_key = make_cid_key(dcid, dcidlen);
ClientHandler *handler;
auto it = connections_.find(dcid_key); auto it = connections_.find(dcid_key);
if (it == std::end(connections_)) { if (it == std::end(connections_)) {
// new connection // new connection
@ -87,18 +95,66 @@ int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
return 0; return 0;
} }
return 0; handler = handle_new_connection(faddr, remote_addr, local_addr, hd);
if (handler == nullptr) {
return 0;
}
} else {
handler = (*it).second;
} }
auto h = (*it).second.get(); if (handler->read_quic(faddr, remote_addr, local_addr, data, datalen) != 0) {
auto upstream = static_cast<Http3Upstream *>(h->get_upstream()); delete handler;
}
// TODO Delete h on failure
upstream->on_read(faddr, remote_addr, local_addr, data, datalen);
return 0; return 0;
} }
ClientHandler *QUICConnectionHandler::handle_new_connection(
const UpstreamAddr *faddr, const Address &remote_addr,
const Address &local_addr, const ngtcp2_pkt_hd &hd) {
std::array<char, NI_MAXHOST> host;
std::array<char, NI_MAXSERV> service;
int rv;
rv = getnameinfo(&remote_addr.su.sa, remote_addr.len, host.data(),
host.size(), service.data(), service.size(),
NI_NUMERICHOST | NI_NUMERICSERV);
if (rv != 0) {
LOG(ERROR) << "getnameinfo() failed: " << gai_strerror(rv);
return nullptr;
}
auto ssl_ctx = worker_->get_quic_sv_ssl_ctx();
assert(ssl_ctx);
auto ssl = tls::create_ssl(ssl_ctx);
if (ssl == nullptr) {
return nullptr;
}
// Disable TLS session ticket if we don't have working ticket
// keys.
if (!worker_->get_ticket_keys()) {
SSL_set_options(ssl, SSL_OP_NO_TICKET);
}
auto handler = std::make_unique<ClientHandler>(
worker_, faddr->fd, ssl, StringRef{host.data()},
StringRef{service.data()}, remote_addr.su.sa.sa_family, faddr);
auto upstream = std::make_unique<Http3Upstream>(handler.get());
if (upstream->init(faddr, remote_addr, local_addr, hd) != 0) {
return nullptr;
}
handler->setup_http3_upstream(std::move(upstream));
return handler.release();
}
namespace { namespace {
uint32_t generate_reserved_version(const Address &addr, uint32_t version) { uint32_t generate_reserved_version(const Address &addr, uint32_t version) {
uint32_t h = 0x811C9DC5u; uint32_t h = 0x811C9DC5u;
@ -154,4 +210,15 @@ int QUICConnectionHandler::send_version_negotiation(
0); 0);
} }
void QUICConnectionHandler::add_connection_id(const ngtcp2_cid *cid,
ClientHandler *handler) {
auto key = make_cid_key(cid);
connections_.emplace(key, handler);
}
void QUICConnectionHandler::remove_connection_id(const ngtcp2_cid *cid) {
auto key = make_cid_key(cid);
connections_.erase(key);
}
} // namespace shrpx } // namespace shrpx

View File

@ -31,6 +31,8 @@
#include <unordered_map> #include <unordered_map>
#include <string> #include <string>
#include <ngtcp2/ngtcp2.h>
#include "network.h" #include "network.h"
using namespace nghttp2; using namespace nghttp2;
@ -53,10 +55,16 @@ public:
const uint8_t *scid, size_t scidlen, const uint8_t *scid, size_t scidlen,
const Address &remote_addr, const Address &remote_addr,
const Address &local_addr); const Address &local_addr);
ClientHandler *handle_new_connection(const UpstreamAddr *faddr,
const Address &remote_addr,
const Address &local_addr,
const ngtcp2_pkt_hd &hd);
void add_connection_id(const ngtcp2_cid *cid, ClientHandler *handler);
void remove_connection_id(const ngtcp2_cid *cid);
private: private:
Worker *worker_; Worker *worker_;
std::unordered_map<std::string, std::shared_ptr<ClientHandler>> connections_; std::unordered_map<std::string, ClientHandler *> connections_;
}; };
} // namespace shrpx } // namespace shrpx

View File

@ -37,7 +37,7 @@ void readcb(struct ev_loop *loop, ev_io *w, int revent) {
} // namespace } // namespace
QUICListener::QUICListener(const UpstreamAddr *faddr, Worker *worker) QUICListener::QUICListener(const UpstreamAddr *faddr, Worker *worker)
: faddr_(faddr), worker_(worker) { : faddr_{faddr}, worker_{worker} {
ev_io_init(&rev_, readcb, faddr_->fd, EV_READ); ev_io_init(&rev_, readcb, faddr_->fd, EV_READ);
rev_.data = this; rev_.data = this;
ev_io_start(worker_->get_loop(), &rev_); ev_io_start(worker_->get_loop(), &rev_);
@ -62,6 +62,8 @@ void QUICListener::on_read() {
uint8_t msg_ctrl[CMSG_SPACE(sizeof(in6_pktinfo))]; uint8_t msg_ctrl[CMSG_SPACE(sizeof(in6_pktinfo))];
msg.msg_control = msg_ctrl; msg.msg_control = msg_ctrl;
auto quic_conn_handler = worker_->get_quic_connection_handler();
for (; pktcnt < 10;) { for (; pktcnt < 10;) {
msg.msg_namelen = sizeof(su); msg.msg_namelen = sizeof(su);
msg.msg_controllen = sizeof(msg_ctrl); msg.msg_controllen = sizeof(msg_ctrl);
@ -91,6 +93,13 @@ void QUICListener::on_read() {
if (nread == 0) { if (nread == 0) {
continue; continue;
} }
Address remote_addr;
remote_addr.su = su;
remote_addr.len = msg.msg_namelen;
quic_conn_handler->handle_packet(faddr_, remote_addr, local_addr,
buf.data(), nread);
} }
} }

View File

@ -132,7 +132,8 @@ create_downstream_key(const std::shared_ptr<SharedDownstreamAddr> &shared_addr,
Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
SSL_CTX *tls_session_cache_memcached_ssl_ctx, SSL_CTX *tls_session_cache_memcached_ssl_ctx,
tls::CertLookupTree *cert_tree, tls::CertLookupTree *cert_tree, SSL_CTX *quic_sv_ssl_ctx,
tls::CertLookupTree *quic_cert_tree,
const std::shared_ptr<TicketKeys> &ticket_keys, const std::shared_ptr<TicketKeys> &ticket_keys,
ConnectionHandler *conn_handler, ConnectionHandler *conn_handler,
std::shared_ptr<DownstreamConfig> downstreamconf) std::shared_ptr<DownstreamConfig> downstreamconf)
@ -145,6 +146,9 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
cl_ssl_ctx_(cl_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx),
cert_tree_(cert_tree), cert_tree_(cert_tree),
conn_handler_(conn_handler), conn_handler_(conn_handler),
quic_sv_ssl_ctx_{quic_sv_ssl_ctx},
quic_cert_tree_{quic_cert_tree},
quic_conn_handler_{this},
ticket_keys_(ticket_keys), ticket_keys_(ticket_keys),
connect_blocker_( connect_blocker_(
std::make_unique<ConnectBlocker>(randgen_, loop_, nullptr, nullptr)), std::make_unique<ConnectBlocker>(randgen_, loop_, nullptr, nullptr)),
@ -507,6 +511,10 @@ void Worker::process_events() {
tls::CertLookupTree *Worker::get_cert_lookup_tree() const { return cert_tree_; } tls::CertLookupTree *Worker::get_cert_lookup_tree() const { return cert_tree_; }
tls::CertLookupTree *Worker::get_quic_cert_lookup_tree() const {
return quic_cert_tree_;
}
std::shared_ptr<TicketKeys> Worker::get_ticket_keys() { std::shared_ptr<TicketKeys> Worker::get_ticket_keys() {
#ifdef HAVE_ATOMIC_STD_SHARED_PTR #ifdef HAVE_ATOMIC_STD_SHARED_PTR
return std::atomic_load_explicit(&ticket_keys_, std::memory_order_acquire); return std::atomic_load_explicit(&ticket_keys_, std::memory_order_acquire);
@ -537,6 +545,8 @@ SSL_CTX *Worker::get_sv_ssl_ctx() const { return sv_ssl_ctx_; }
SSL_CTX *Worker::get_cl_ssl_ctx() const { return cl_ssl_ctx_; } SSL_CTX *Worker::get_cl_ssl_ctx() const { return cl_ssl_ctx_; }
SSL_CTX *Worker::get_quic_sv_ssl_ctx() const { return quic_sv_ssl_ctx_; }
void Worker::set_graceful_shutdown(bool f) { graceful_shutdown_ = f; } void Worker::set_graceful_shutdown(bool f) { graceful_shutdown_ = f; }
bool Worker::get_graceful_shutdown() const { return graceful_shutdown_; } bool Worker::get_graceful_shutdown() const { return graceful_shutdown_; }
@ -581,6 +591,10 @@ ConnectionHandler *Worker::get_connection_handler() const {
return conn_handler_; return conn_handler_;
} }
QUICConnectionHandler *Worker::get_quic_connection_handler() {
return &quic_conn_handler_;
}
DNSTracker *Worker::get_dns_tracker() { return &dns_tracker_; } DNSTracker *Worker::get_dns_tracker() { return &dns_tracker_; }
int Worker::setup_quic_server_socket() { int Worker::setup_quic_server_socket() {

View File

@ -50,6 +50,7 @@
#include "shrpx_live_check.h" #include "shrpx_live_check.h"
#include "shrpx_connect_blocker.h" #include "shrpx_connect_blocker.h"
#include "shrpx_dns_tracker.h" #include "shrpx_dns_tracker.h"
#include "shrpx_quic_connection_handler.h"
#include "allocator.h" #include "allocator.h"
using namespace nghttp2; using namespace nghttp2;
@ -270,7 +271,8 @@ class Worker {
public: public:
Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
SSL_CTX *tls_session_cache_memcached_ssl_ctx, SSL_CTX *tls_session_cache_memcached_ssl_ctx,
tls::CertLookupTree *cert_tree, tls::CertLookupTree *cert_tree, SSL_CTX *quic_sv_ssl_ctx,
tls::CertLookupTree *quic_cert_tree,
const std::shared_ptr<TicketKeys> &ticket_keys, const std::shared_ptr<TicketKeys> &ticket_keys,
ConnectionHandler *conn_handler, ConnectionHandler *conn_handler,
std::shared_ptr<DownstreamConfig> downstreamconf); std::shared_ptr<DownstreamConfig> downstreamconf);
@ -281,6 +283,7 @@ public:
void send(const WorkerEvent &event); void send(const WorkerEvent &event);
tls::CertLookupTree *get_cert_lookup_tree() const; tls::CertLookupTree *get_cert_lookup_tree() const;
tls::CertLookupTree *get_quic_cert_lookup_tree() const;
// These 2 functions make a lock m_ to get/set ticket keys // These 2 functions make a lock m_ to get/set ticket keys
// atomically. // atomically.
@ -291,6 +294,7 @@ public:
struct ev_loop *get_loop() const; struct ev_loop *get_loop() const;
SSL_CTX *get_sv_ssl_ctx() const; SSL_CTX *get_sv_ssl_ctx() const;
SSL_CTX *get_cl_ssl_ctx() const; SSL_CTX *get_cl_ssl_ctx() const;
SSL_CTX *get_quic_sv_ssl_ctx() const;
void set_graceful_shutdown(bool f); void set_graceful_shutdown(bool f);
bool get_graceful_shutdown() const; bool get_graceful_shutdown() const;
@ -320,6 +324,8 @@ public:
ConnectionHandler *get_connection_handler() const; ConnectionHandler *get_connection_handler() const;
QUICConnectionHandler *get_quic_connection_handler();
DNSTracker *get_dns_tracker(); DNSTracker *get_dns_tracker();
int setup_quic_server_socket(); int setup_quic_server_socket();
@ -354,6 +360,10 @@ private:
SSL_CTX *cl_ssl_ctx_; SSL_CTX *cl_ssl_ctx_;
tls::CertLookupTree *cert_tree_; tls::CertLookupTree *cert_tree_;
ConnectionHandler *conn_handler_; ConnectionHandler *conn_handler_;
SSL_CTX *quic_sv_ssl_ctx_;
tls::CertLookupTree *quic_cert_tree_;
QUICConnectionHandler quic_conn_handler_;
#ifndef HAVE_ATOMIC_STD_SHARED_PTR #ifndef HAVE_ATOMIC_STD_SHARED_PTR
std::mutex ticket_keys_m_; std::mutex ticket_keys_m_;