walkero
/
nghttp2
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

nghttpx: Add experimental TCP optimization for h2 frontend

This commit is contained in:
Tatsuhiro Tsujikawa 2016-09-08 22:49:36 +09:00
parent b14375ec63
commit 27b250ac8e
10 changed files with 231 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
gennghttpxfun.py
View File

@ -136,6 +136,8 @@ OPTIONS = [
"backend-max-backoff",
"server-name",
"no-server-rewrite",
"frontend-http2-optimize-write-buffer-size",
"frontend-http2-optimize-window-size",
]
LOGVARS = [

36
src/shrpx.cc
View File

@ -2032,6 +2032,27 @@ HTTP/2 and SPDY:
backend session is relayed to frontend, and server push
via Link header field is also supported. SPDY frontend
does not support server push.
--frontend-http2-optimize-write-buffer-size
(Experimental) Enable write buffer size optimization in
frontend HTTP/2 TLS connection. This optimization aims
to reduce write buffer size so that it only contains
bytes which can send immediately. This makes server
more responsive to prioritized HTTP/2 stream because the
buffering of lower priority stream is reduced. This
option is only effective on recent Linux platform.
--frontend-http2-optimize-window-size
(Experimental) Automatically tune connection level
window size of frontend HTTP/2 TLS connection. If this
feature is enabled, connection window size starts with
the default window size, 65535 bytes. nghttpx
automatically adjusts connection window size based on
TCP receiving window size. The maximum window size is
capped by the value specified by
--frontend-http2-connection-window-bits. Since the
stream is subject to stream level window size, it should
be adjusted using --frontend-http2-window-bits option as
well. This option is only effective on recent Linux
platform.
Mode:
(default mode)
@ -2842,6 +2863,10 @@ int main(int argc, char **argv) {
{SHRPX_OPT_BACKEND_MAX_BACKOFF.c_str(), required_argument, &flag, 127},
{SHRPX_OPT_SERVER_NAME.c_str(), required_argument, &flag, 128},
{SHRPX_OPT_NO_SERVER_REWRITE.c_str(), no_argument, &flag, 129},
{SHRPX_OPT_FRONTEND_HTTP2_OPTIMIZE_WRITE_BUFFER_SIZE.c_str(),
no_argument, &flag, 130},
{SHRPX_OPT_FRONTEND_HTTP2_OPTIMIZE_WINDOW_SIZE.c_str(), no_argument,
&flag, 131},
{nullptr, 0, nullptr, 0}};
int option_index = 0;
@ -3448,6 +3473,17 @@ int main(int argc, char **argv) {
cmdcfgs.emplace_back(SHRPX_OPT_NO_SERVER_REWRITE,
StringRef::from_lit("yes"));
break;
case 130:
// --frontend-http2-optimize-write-buffer-size
cmdcfgs.emplace_back(
SHRPX_OPT_FRONTEND_HTTP2_OPTIMIZE_WRITE_BUFFER_SIZE,
StringRef::from_lit("yes"));
break;
case 131:
// --frontend-http2-optimize-window-size
cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_HTTP2_OPTIMIZE_WINDOW_SIZE,
StringRef::from_lit("yes"));
break;
default:
break;
}

36
src/shrpx_client_handler.cc
View File

@ -244,17 +244,21 @@ int ClientHandler::write_tls() {
ERR_clear_error();
if (on_write() != 0) {
return -1;
}
auto iovcnt = upstream_->response_riovec(&iov, 1);
if (iovcnt == 0) {
conn_.start_tls_write_idle();
conn_.wlimit.stopw();
ev_timer_stop(conn_.loop, &conn_.wt);
return 0;
}
for (;;) {
if (on_write() != 0) {
return -1;
}
auto iovcnt = upstream_->response_riovec(&iov, 1);
if (iovcnt == 0) {
conn_.start_tls_write_idle();
break;
}
auto nwrite = conn_.write_tls(iov.iov_base, iov.iov_len);
if (nwrite < 0) {
return -1;
@ -265,12 +269,12 @@ int ClientHandler::write_tls() {
}
upstream_->response_drain(nwrite);
iovcnt = upstream_->response_riovec(&iov, 1);
if (iovcnt == 0) {
return 0;
}
}
conn_.wlimit.stopw();
ev_timer_stop(conn_.loop, &conn_.wt);
return 0;
}
int ClientHandler::upstream_noop() { return 0; }
@ -1445,4 +1449,6 @@ StringRef ClientHandler::get_forwarded_for() const {
const UpstreamAddr *ClientHandler::get_upstream_addr() const { return faddr_; }
Connection *ClientHandler::get_connection() { return &conn_; };
} // namespace shrpx

2
src/shrpx_client_handler.h
View File

@ -154,6 +154,8 @@ public:
void repeat_read_timer();
void stop_read_timer();
Connection *get_connection();
private:
Connection conn_;
ev_timer reneg_shutdown_timer_;

18
src/shrpx_config.cc
View File

@ -1643,6 +1643,11 @@ int option_lookup_token(const char *name, size_t namelen) {
break;
case 35:
switch (name[34]) {
case 'e':
if (util::strieq_l("frontend-http2-optimize-window-siz", name, 34)) {
return SHRPX_OPTID_FRONTEND_HTTP2_OPTIMIZE_WINDOW_SIZE;
}
break;
case 'r':
if (util::strieq_l("frontend-http2-dump-response-heade", name, 34)) {
return SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER;
@ -1705,6 +1710,10 @@ int option_lookup_token(const char *name, size_t namelen) {
case 41:
switch (name[40]) {
case 'e':
if (util::strieq_l("frontend-http2-optimize-write-buffer-siz", name,
40)) {
return SHRPX_OPTID_FRONTEND_HTTP2_OPTIMIZE_WRITE_BUFFER_SIZE;
}
if (util::strieq_l("tls-ticket-key-memcached-private-key-fil", name,
40)) {
return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE;
@ -2689,6 +2698,15 @@ int parse_config(Config *config, int optid, const StringRef &opt,
case SHRPX_OPTID_NO_SERVER_REWRITE:
config->http.no_server_rewrite = util::strieq_l("yes", optarg);
return 0;
case SHRPX_OPTID_FRONTEND_HTTP2_OPTIMIZE_WRITE_BUFFER_SIZE:
config->http2.upstream.optimize_write_buffer_size =
util::strieq_l("yes", optarg);
return 0;
case SHRPX_OPTID_FRONTEND_HTTP2_OPTIMIZE_WINDOW_SIZE:
config->http2.upstream.optimize_window_size = util::strieq_l("yes", optarg);
return 0;
case SHRPX_OPTID_CONF:
LOG(WARN) << "conf: ignored";

8
src/shrpx_config.h
View File

@ -287,6 +287,10 @@ constexpr auto SHRPX_OPT_BACKEND_MAX_BACKOFF =
constexpr auto SHRPX_OPT_SERVER_NAME = StringRef::from_lit("server-name");
constexpr auto SHRPX_OPT_NO_SERVER_REWRITE =
StringRef::from_lit("no-server-rewrite");
constexpr auto SHRPX_OPT_FRONTEND_HTTP2_OPTIMIZE_WRITE_BUFFER_SIZE =
StringRef::from_lit("frontend-http2-optimize-write-buffer-size");
constexpr auto SHRPX_OPT_FRONTEND_HTTP2_OPTIMIZE_WINDOW_SIZE =
StringRef::from_lit("frontend-http2-optimize-window-size");
constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8;
@ -586,6 +590,8 @@ struct Http2Config {
size_t window_bits;
size_t connection_window_bits;
size_t max_concurrent_streams;
bool optimize_write_buffer_size;
bool optimize_window_size;
} upstream;
struct {
struct {
@ -818,6 +824,8 @@ enum {
SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER,
SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER,
SHRPX_OPTID_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS,
SHRPX_OPTID_FRONTEND_HTTP2_OPTIMIZE_WINDOW_SIZE,
SHRPX_OPTID_FRONTEND_HTTP2_OPTIMIZE_WRITE_BUFFER_SIZE,
SHRPX_OPTID_FRONTEND_HTTP2_READ_TIMEOUT,
SHRPX_OPTID_FRONTEND_HTTP2_SETTINGS_TIMEOUT,
SHRPX_OPTID_FRONTEND_HTTP2_WINDOW_BITS,

52
src/shrpx_connection.cc
View File

@ -27,6 +27,7 @@
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif // HAVE_UNISTD_H
#include <netinet/tcp.h>
#include <limits>
@ -757,4 +758,55 @@ void Connection::handle_tls_pending_read() {
rlimit.handle_tls_pending_read();
}
int Connection::get_tcp_hint(TCPHint *hint) const {
#if defined(TCP_INFO) && defined(TCP_NOTSENT_LOWAT)
struct tcp_info tcp_info;
socklen_t tcp_info_len = sizeof(tcp_info);
int rv;
rv = getsockopt(fd, IPPROTO_TCP, TCP_INFO, &tcp_info, &tcp_info_len);
if (rv != 0) {
return -1;
}
auto avail_packets = tcp_info.tcpi_snd_cwnd > tcp_info.tcpi_unacked
? tcp_info.tcpi_snd_cwnd - tcp_info.tcpi_unacked
: 0;
// http://www.slideshare.net/kazuho/programming-tcp-for-responsiveness
//
// TODO 29 (5 + 8 + 16) is TLS overhead for AES-GCM. For
// CHACHA20_POLY1305, it is 21 since it does not need 8 bytes
// explicit nonce.
auto writable_size = (avail_packets + 2) * (tcp_info.tcpi_snd_mss - 29);
if (writable_size > 16_k) {
writable_size = writable_size & ~(16_k - 1);
} else {
if (writable_size < 536) {
LOG(INFO) << "writable_size is too small: " << writable_size;
}
// TODO is this required?
writable_size = std::max(writable_size, static_cast<uint32_t>(536 * 2));
}
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "snd_cwnd=" << tcp_info.tcpi_snd_cwnd
<< ", unacked=" << tcp_info.tcpi_unacked
<< ", snd_mss=" << tcp_info.tcpi_snd_mss
<< ", rtt=" << tcp_info.tcpi_rtt << "us"
<< ", rcv_space=" << tcp_info.tcpi_rcv_space
<< ", writable=" << writable_size;
}
hint->write_buffer_size = writable_size;
// TODO tcpi_rcv_space is considered as rwin, is that correct?
hint->rwin = tcp_info.tcpi_rcv_space;
return 0;
#else // !defined(TCP_INFO) || !defined(TCP_NOTSENT_LOWAT)
return -1;
#endif // !defined(TCP_INFO) || !defined(TCP_NOTSENT_LOWAT)
}
} // namespace shrpx

7
src/shrpx_connection.h
View File

@ -66,6 +66,11 @@ struct TLSConnection {
bool reneg_started;
};
struct TCPHint {
size_t write_buffer_size;
uint32_t rwin;
};
template <typename T> using EVCb = void (*)(struct ev_loop *, T *, int);
using IOCb = EVCb<ev_io>;
@ -118,6 +123,8 @@ struct Connection {
void set_ssl(SSL *ssl);
int get_tcp_hint(TCPHint *hint) const;
TLSConnection tls;
ev_io wev;
ev_io rev;

86
src/shrpx_http2_upstream.cc
View File

@ -776,7 +776,9 @@ int send_data_callback(nghttp2_session *session, nghttp2_frame *frame,
// data transferred.
downstream->response_sent_body_length += length;
return wb->rleft() >= MAX_BUFFER_SIZE ? NGHTTP2_ERR_PAUSE : 0;
auto max_buffer_size = upstream->get_max_buffer_size();
return wb->rleft() >= max_buffer_size ? NGHTTP2_ERR_PAUSE : 0;
}
} // namespace
@ -919,7 +921,8 @@ Http2Upstream::Http2Upstream(ClientHandler *handler)
downstream_queue_(downstream_queue_size(handler->get_worker()),
!get_config()->http2_proxy),
handler_(handler),
session_(nullptr) {
session_(nullptr),
max_buffer_size_(MAX_BUFFER_SIZE) {
int rv;
auto &http2conf = get_config()->http2;
@ -955,7 +958,9 @@ Http2Upstream::Http2Upstream(ClientHandler *handler)
}
int32_t window_bits =
faddr->alt_mode ? 31 : http2conf.upstream.connection_window_bits;
faddr->alt_mode ? 31 : http2conf.upstream.optimize_window_size
? 16
: http2conf.upstream.connection_window_bits;
if (window_bits != 16) {
int32_t window_size = (1u << window_bits) - 1;
@ -983,6 +988,25 @@ Http2Upstream::Http2Upstream(ClientHandler *handler)
prep_.data = this;
ev_prepare_start(handler_->get_loop(), &prep_);
#if defined(TCP_INFO) && defined(TCP_NOTSENT_LOWAT)
if (http2conf.upstream.optimize_write_buffer_size) {
auto conn = handler_->get_connection();
conn->tls_dyn_rec_warmup_threshold = 0;
uint32_t pollout_thres = 1;
rv = setsockopt(conn->fd, IPPROTO_TCP, TCP_NOTSENT_LOWAT, &pollout_thres,
static_cast<socklen_t>(sizeof(pollout_thres)));
if (rv != 0) {
if (LOG_ENABLED(INFO)) {
auto error = errno;
LOG(INFO) << "setsockopt(TCP_NOTSENT_LOWAT, " << pollout_thres
<< ") failed: errno=" << error;
}
}
}
#endif // defined(TCP_INFO) && defined(TCP_NOTSENT_LOWAT)
handler_->reset_upstream_read_timeout(
get_config()->conn.upstream.timeout.http2_read);
@ -1032,8 +1056,44 @@ int Http2Upstream::on_read() {
// After this function call, downstream may be deleted.
int Http2Upstream::on_write() {
int rv;
auto &http2conf = get_config()->http2;
if ((http2conf.upstream.optimize_write_buffer_size ||
http2conf.upstream.optimize_window_size) &&
handler_->get_ssl()) {
auto conn = handler_->get_connection();
TCPHint hint;
rv = conn->get_tcp_hint(&hint);
if (rv == 0) {
if (http2conf.upstream.optimize_write_buffer_size) {
max_buffer_size_ = std::min(MAX_BUFFER_SIZE, hint.write_buffer_size);
}
if (http2conf.upstream.optimize_window_size) {
auto faddr = handler_->get_upstream_addr();
if (!faddr->alt_mode) {
int32_t window_size =
(1u << http2conf.upstream.connection_window_bits) - 1;
window_size =
std::min(static_cast<uint32_t>(window_size), hint.rwin * 2);
rv = nghttp2_session_set_local_window_size(
session_, NGHTTP2_FLAG_NONE, 0, window_size);
if (rv != 0) {
if (LOG_ENABLED(INFO)) {
ULOG(INFO, this)
<< "nghttp2_session_set_local_window_size() with window_size="
<< window_size << " failed: " << nghttp2_strerror(rv);
}
}
}
}
}
}
for (;;) {
if (wb_.rleft() >= MAX_BUFFER_SIZE) {
if (wb_.rleft() >= max_buffer_size_) {
return 0;
}
@ -1253,10 +1313,26 @@ ssize_t downstream_data_read_callback(nghttp2_session *session,
auto downstream = static_cast<Downstream *>(source->ptr);
auto body = downstream->get_response_buf();
assert(body);
auto upstream = static_cast<Http2Upstream *>(user_data);
const auto &resp = downstream->response();
auto nread = std::min(body->rleft(), length);
auto max_buffer_size = upstream->get_max_buffer_size();
auto buffer = upstream->get_response_buf();
if (max_buffer_size <
std::min(nread, static_cast<size_t>(256)) + 9 + buffer->rleft()) {
if (LOG_ENABLED(INFO)) {
ULOG(INFO, upstream) << "Buffer is almost full. Skip write DATA";
}
return NGHTTP2_ERR_PAUSE;
}
nread = std::min(nread, max_buffer_size - 9 - buffer->rleft());
auto body_empty = body->rleft() == nread;
*data_flags |= NGHTTP2_DATA_FLAG_NO_COPY;
@ -2027,4 +2103,6 @@ void Http2Upstream::cancel_premature_downstream(
downstream_queue_.remove_and_get_blocked(promised_downstream, false);
}
size_t Http2Upstream::get_max_buffer_size() const { return max_buffer_size_; }
} // namespace shrpx

3
src/shrpx_http2_upstream.h
View File

@ -118,6 +118,8 @@ public:
DefaultMemchunks *get_response_buf();
size_t get_max_buffer_size() const;
private:
DefaultMemchunks wb_;
std::unique_ptr<HttpsUpstream> pre_upstream_;
@ -127,6 +129,7 @@ private:
ev_prepare prep_;
ClientHandler *handler_;
nghttp2_session *session_;
size_t max_buffer_size_;
bool flow_control_;
};