From 3785cf07ba654d3f1b02a538d87b621b78fb65e4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 10 Oct 2015 22:27:48 +0900 Subject: [PATCH 001/147] Add simple HTTP/2 extension framework Application can utilize this framework to send/receive user defined extension frames. These frames are expected not to change existing protocol behaviour. --- doc/Makefile.am | 4 + lib/includes/nghttp2/nghttp2.h | 154 +++++++++++++++++++++++++++++++ lib/nghttp2_callbacks.c | 18 ++++ lib/nghttp2_callbacks.h | 3 + lib/nghttp2_frame.c | 9 ++ lib/nghttp2_frame.h | 6 ++ lib/nghttp2_helper.c | 2 + lib/nghttp2_outbound_item.c | 3 + lib/nghttp2_session.c | 159 ++++++++++++++++++++++++++++++++- lib/nghttp2_session.h | 3 +- lib/nghttp2_submit.c | 37 ++++++++ src/app_helper.cc | 12 ++- 12 files changed, 402 insertions(+), 8 deletions(-) diff --git a/doc/Makefile.am b/doc/Makefile.am index ca094fb0..d1e41e4a 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -62,16 +62,19 @@ APIDOCS= \ nghttp2_session_callbacks_set_on_begin_frame_callback.rst \ nghttp2_session_callbacks_set_on_begin_headers_callback.rst \ nghttp2_session_callbacks_set_on_data_chunk_recv_callback.rst \ + nghttp2_session_callbacks_set_on_extension_chunk_recv_callback.rst \ nghttp2_session_callbacks_set_on_frame_not_send_callback.rst \ nghttp2_session_callbacks_set_on_frame_recv_callback.rst \ nghttp2_session_callbacks_set_on_frame_send_callback.rst \ nghttp2_session_callbacks_set_on_header_callback.rst \ nghttp2_session_callbacks_set_on_invalid_frame_recv_callback.rst \ nghttp2_session_callbacks_set_on_stream_close_callback.rst \ + nghttp2_session_callbacks_set_pack_extension_callback.rst \ nghttp2_session_callbacks_set_recv_callback.rst \ nghttp2_session_callbacks_set_select_padding_callback.rst \ nghttp2_session_callbacks_set_send_callback.rst \ nghttp2_session_callbacks_set_send_data_callback.rst \ + nghttp2_session_callbacks_set_unpack_extension_callback.rst \ nghttp2_session_client_new.rst \ nghttp2_session_client_new2.rst \ nghttp2_session_client_new3.rst \ @@ -118,6 +121,7 @@ APIDOCS= \ nghttp2_stream_get_weight.rst \ nghttp2_strerror.rst \ nghttp2_submit_data.rst \ + nghttp2_submit_extension.rst \ nghttp2_submit_goaway.rst \ nghttp2_submit_headers.rst \ nghttp2_submit_ping.rst \ diff --git a/lib/includes/nghttp2/nghttp2.h b/lib/includes/nghttp2/nghttp2.h index 037793ed..91030f28 100644 --- a/lib/includes/nghttp2/nghttp2.h +++ b/lib/includes/nghttp2/nghttp2.h @@ -378,6 +378,10 @@ typedef enum { * Unexpected internal error, but recovered. */ NGHTTP2_ERR_INTERNAL = -534, + /** + * Indicates that a processing was canceled. + */ + NGHTTP2_ERR_CANCEL = -535, /** * The errors < :enum:`NGHTTP2_ERR_FATAL` mean that the library is * under unexpected condition and processing was terminated (e.g., @@ -1657,6 +1661,94 @@ typedef int (*nghttp2_on_begin_frame_callback)(nghttp2_session *session, const nghttp2_frame_hd *hd, void *user_data); +/** + * @functypedef + * + * Callback function invoked when chunk of extension frame payload is + * received. The |hd| points to frame header. The received + * chunk is |data| of length |len|. + * + * The implementation of this function must return 0 if it succeeds. + * + * To abort processing this extension frame, return + * :enum:`NGHTTP2_ERR_CANCEL`. + * + * If fatal error occurred, application should return + * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. In this case, + * `nghttp2_session_recv()` and `nghttp2_session_mem_recv()` functions + * immediately return :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. If the + * other values are returned, currently they are treated as + * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. + */ +typedef int (*nghttp2_on_extension_chunk_recv_callback)( + nghttp2_session *session, const nghttp2_frame_hd *hd, const uint8_t *data, + size_t len, void *user_data); + +/** + * @functypedef + * + * Callback function invoked when library asks the application to + * unpack extension payload from its wire format. The extension + * payload has been passed to the application using + * :type:`nghttp2_on_extension_chunk_recv_callback`. The frame header + * is already unpacked by the library and provided as |hd|. + * + * The implementation of this function may store the pointer to the + * created object as a result of unpacking in |*payload|, and returns + * 0. The pointer stored in |*payload| is opaque to the library, and + * the library does not own its pointer. |*payload| is initialled as + * `NULL`. + * + * To abort processing this extension frame, return + * :enum:`NGHTTP2_ERR_CANCEL`. + * + * If fatal error occurred, application should return + * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. In this case, + * `nghttp2_session_recv()` and `nghttp2_session_mem_recv()` functions + * immediately return :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. If the + * other values are returned, currently they are treated as + * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. + */ +typedef int (*nghttp2_unpack_extension_callback)(nghttp2_session *session, + void **payload, + const nghttp2_frame_hd *hd, + void *user_data); + +/** + * @functypedef + * + * Callbck function invoked when library asks the application to pack + * extension payload in its wire format. The frame header will be + * packed by library. Application must pack payload only. + * frame->ext.payload is the object passed to + * `nghttp2_submit_extension()` as payload parameter. The |*flags| is + * initialized to the flags parameter passed in + * `nghttp2_submit_extension()`. Application can modify flags value + * if it wants. Application must pack extension payload to the |buf| + * of its capacity |len| bytes. + * + * The implementation of this function returns the number of bytes + * written into |buf| when it succeeds. + * + * To abort processing this extension frame, return + * :enum:`NGHTTP2_ERR_CANCEL`, and + * :type:`nghttp2_on_frame_not_send_callback` will be invoked. + * + * If fatal error occurred, application should return + * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. In this case, + * `nghttp2_session_send()` and `nghttp2_session_mem_send()` functions + * immediately return :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. If the + * other values are returned, currently they are treated as + * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. If the return value is + * strictly larger than |len|, it is treated as + * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. + */ +typedef ssize_t (*nghttp2_pack_extension_callback)(nghttp2_session *session, + uint8_t *flags, uint8_t *buf, + size_t len, + const nghttp2_frame *frame, + void *user_data); + struct nghttp2_session_callbacks; /** @@ -1849,6 +1941,37 @@ NGHTTP2_EXTERN void nghttp2_session_callbacks_set_send_data_callback( nghttp2_session_callbacks *cbs, nghttp2_send_data_callback send_data_callback); +/** + * @function + * + * Sets callback function invoked when the library asks the + * application to pack extension frame payload in wire format. + */ +NGHTTP2_EXTERN void nghttp2_session_callbacks_set_pack_extension_callback( + nghttp2_session_callbacks *cbs, + nghttp2_pack_extension_callback pack_extension_callback); + +/** + * @function + * + * Sets callback function invoked when the library asks the + * application to unpack extension frame payload from wire format. + */ +NGHTTP2_EXTERN void nghttp2_session_callbacks_set_unpack_extension_callback( + nghttp2_session_callbacks *cbs, + nghttp2_unpack_extension_callback unpack_extension_callback); + +/** + * @function + * + * Sets callback function invoked when chunk of extension frame + * payload is received. + */ +NGHTTP2_EXTERN void +nghttp2_session_callbacks_set_on_extension_chunk_recv_callback( + nghttp2_session_callbacks *cbs, + nghttp2_on_extension_chunk_recv_callback on_extension_chunk_recv_callback); + /** * @functypedef * @@ -3515,6 +3638,37 @@ NGHTTP2_EXTERN int nghttp2_submit_window_update(nghttp2_session *session, int32_t stream_id, int32_t window_size_increment); +/** + * @function + * + * Submits extension frame. + * + * Application can pass arbitrary frame flags and stream ID to |flags| + * and |stream_id| respectively. The |payload| is opaque pointer, and + * it can be accessible though frame->ext.payload in + * :type:`nghttp2_pack_extension_callback`. The library will not own + * passed |payload| pointer. + * + * The standard HTTP/2 frame cannot be sent with this function, so + * |type| must be strictly grater than 0x9. Otherwise, this function + * will fail with error code :enum:`NGHTTP2_ERR_INVALID_ARGUMENT`. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * :enum:`NGHTTP2_ERR_INVALID_STATE` + * If :type:`nghttp2_pack_extension_callback` is not set. + * :enum:`NGHTTP2_ERR_INVALID_ARGUMENT` + * If |type| specifies standard HTTP/2 frame type. The frame + * types in the rage [0x0, 0x9], both inclusive, are standard + * HTTP/2 frame type, and cannot be sent using this function. + * :enum:`NGHTTP2_ERR_NOMEM` + * Out of memory + */ +NGHTTP2_EXTERN int nghttp2_submit_extension(nghttp2_session *session, + uint8_t type, uint8_t flags, + int32_t stream_id, void *payload); + /** * @function * diff --git a/lib/nghttp2_callbacks.c b/lib/nghttp2_callbacks.c index 3c4be9a0..3b0369c1 100644 --- a/lib/nghttp2_callbacks.c +++ b/lib/nghttp2_callbacks.c @@ -127,3 +127,21 @@ void nghttp2_session_callbacks_set_send_data_callback( nghttp2_send_data_callback send_data_callback) { cbs->send_data_callback = send_data_callback; } + +void nghttp2_session_callbacks_set_pack_extension_callback( + nghttp2_session_callbacks *cbs, + nghttp2_pack_extension_callback pack_extension_callback) { + cbs->pack_extension_callback = pack_extension_callback; +} + +void nghttp2_session_callbacks_set_unpack_extension_callback( + nghttp2_session_callbacks *cbs, + nghttp2_unpack_extension_callback unpack_extension_callback) { + cbs->unpack_extension_callback = unpack_extension_callback; +} + +void nghttp2_session_callbacks_set_on_extension_chunk_recv_callback( + nghttp2_session_callbacks *cbs, + nghttp2_on_extension_chunk_recv_callback on_extension_chunk_recv_callback) { + cbs->on_extension_chunk_recv_callback = on_extension_chunk_recv_callback; +} diff --git a/lib/nghttp2_callbacks.h b/lib/nghttp2_callbacks.h index 37958ea9..664bf35b 100644 --- a/lib/nghttp2_callbacks.h +++ b/lib/nghttp2_callbacks.h @@ -107,6 +107,9 @@ struct nghttp2_session_callbacks { */ nghttp2_on_begin_frame_callback on_begin_frame_callback; nghttp2_send_data_callback send_data_callback; + nghttp2_pack_extension_callback pack_extension_callback; + nghttp2_unpack_extension_callback unpack_extension_callback; + nghttp2_on_extension_chunk_recv_callback on_extension_chunk_recv_callback; }; #endif /* NGHTTP2_CALLBACKS_H */ diff --git a/lib/nghttp2_frame.c b/lib/nghttp2_frame.c index e324b9c9..5677cea4 100644 --- a/lib/nghttp2_frame.c +++ b/lib/nghttp2_frame.c @@ -184,6 +184,15 @@ void nghttp2_frame_data_init(nghttp2_data *frame, uint8_t flags, void nghttp2_frame_data_free(nghttp2_data *frame _U_) {} +void nghttp2_frame_extension_init(nghttp2_extension *frame, uint8_t type, + uint8_t flags, int32_t stream_id, + void *payload) { + nghttp2_frame_hd_init(&frame->hd, 0, type, flags, stream_id); + frame->payload = payload; +} + +void nghttp2_frame_extension_free(nghttp2_extension *frame _U_) {} + size_t nghttp2_frame_priority_len(uint8_t flags) { if (flags & NGHTTP2_FLAG_PRIORITY) { return NGHTTP2_PRIORITY_SPECLEN; diff --git a/lib/nghttp2_frame.h b/lib/nghttp2_frame.h index b0b02ee1..fa0eb452 100644 --- a/lib/nghttp2_frame.h +++ b/lib/nghttp2_frame.h @@ -439,6 +439,12 @@ void nghttp2_frame_window_update_init(nghttp2_window_update *frame, void nghttp2_frame_window_update_free(nghttp2_window_update *frame); +void nghttp2_frame_extension_init(nghttp2_extension *frame, uint8_t type, + uint8_t flags, int32_t stream_id, + void *payload); + +void nghttp2_frame_extension_free(nghttp2_extension *frame); + /* * Returns the number of padding bytes after payload. The total * padding length is given in the |padlen|. The returned value does diff --git a/lib/nghttp2_helper.c b/lib/nghttp2_helper.c index 884abc68..bdba797c 100644 --- a/lib/nghttp2_helper.c +++ b/lib/nghttp2_helper.c @@ -288,6 +288,8 @@ const char *nghttp2_strerror(int error_code) { return "Stream was refused"; case NGHTTP2_ERR_INTERNAL: return "Internal error"; + case NGHTTP2_ERR_CANCEL: + return "Cancel"; case NGHTTP2_ERR_NOMEM: return "Out of memory"; case NGHTTP2_ERR_CALLBACK_FAILURE: diff --git a/lib/nghttp2_outbound_item.c b/lib/nghttp2_outbound_item.c index c4ecab90..886f330c 100644 --- a/lib/nghttp2_outbound_item.c +++ b/lib/nghttp2_outbound_item.c @@ -72,6 +72,9 @@ void nghttp2_outbound_item_free(nghttp2_outbound_item *item, nghttp2_mem *mem) { case NGHTTP2_WINDOW_UPDATE: nghttp2_frame_window_update_free(&frame->window_update); break; + default: + nghttp2_frame_extension_free(&frame->ext); + break; } } diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 7712ac86..787bc311 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -1718,6 +1718,44 @@ static size_t session_estimate_headers_payload(nghttp2_session *session, additional; } +static int session_pack_extension(nghttp2_session *session, nghttp2_bufs *bufs, + nghttp2_frame *frame) { + ssize_t rv; + uint8_t flags; + nghttp2_buf *buf; + size_t buflen; + size_t framelen; + + assert(session->callbacks.pack_extension_callback); + + flags = frame->hd.flags; + buf = &bufs->head->buf; + buflen = nghttp2_min(nghttp2_buf_avail(buf), NGHTTP2_MAX_PAYLOADLEN); + + rv = session->callbacks.pack_extension_callback( + session, &flags, buf->last, buflen, frame, session->user_data); + if (rv == NGHTTP2_ERR_CANCEL) { + return (int)rv; + } + + if (rv < 0 || (size_t)rv > buflen) { + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + + framelen = (size_t)rv; + + frame->hd.flags = flags; + frame->hd.length = framelen; + + assert(buf->pos == buf->last); + buf->last += framelen; + buf->pos -= NGHTTP2_FRAME_HDLEN; + + nghttp2_frame_pack_frame_hd(buf->pos, &frame->hd); + + return 0; +} + /* * This function serializes frame for transmission. * @@ -1956,8 +1994,18 @@ static int session_prep_frame(nghttp2_session *session, nghttp2_frame_pack_window_update(&session->aob.framebufs, &frame->window_update); break; + case NGHTTP2_CONTINUATION: + /* We never handle CONTINUATION here. */ + assert(0); + break; default: - return NGHTTP2_ERR_INVALID_ARGUMENT; + /* extension frame */ + rv = session_pack_extension(session, &session->aob.framebufs, frame); + if (rv != 0) { + return rv; + } + + break; } return 0; } else { @@ -3016,6 +3064,47 @@ static int session_call_on_header(nghttp2_session *session, return 0; } +static int +session_call_on_extension_chunk_recv_callback(nghttp2_session *session, + const uint8_t *data, size_t len) { + int rv; + nghttp2_inbound_frame *iframe = &session->iframe; + nghttp2_frame *frame = &iframe->frame; + + if (session->callbacks.on_extension_chunk_recv_callback) { + rv = session->callbacks.on_extension_chunk_recv_callback( + session, &frame->hd, data, len, session->user_data); + if (rv == NGHTTP2_ERR_CANCEL) { + return rv; + } + if (rv != 0) { + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + } + + return 0; +} + +static int session_call_unpack_extension_callback(nghttp2_session *session) { + int rv; + nghttp2_inbound_frame *iframe = &session->iframe; + nghttp2_frame *frame = &iframe->frame; + void *payload = NULL; + + rv = session->callbacks.unpack_extension_callback( + session, &payload, &frame->hd, session->user_data); + if (rv == NGHTTP2_ERR_CANCEL) { + return rv; + } + if (rv != 0) { + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + + frame->ext.payload = payload; + + return 0; +} + /* * Handles frame size error. * @@ -4383,6 +4472,24 @@ static int session_process_window_update_frame(nghttp2_session *session) { return nghttp2_session_on_window_update_received(session, frame); } +static int session_process_extension_frame(nghttp2_session *session) { + int rv; + nghttp2_inbound_frame *iframe = &session->iframe; + nghttp2_frame *frame = &iframe->frame; + + rv = session_call_unpack_extension_callback(session); + if (nghttp2_is_fatal(rv)) { + return rv; + } + + /* This handles the case where rv == NGHTTP2_ERR_CANCEL as well */ + if (rv != 0) { + return 0; + } + + return session_call_on_frame_received(session, frame); +} + int nghttp2_session_on_data_received(nghttp2_session *session, nghttp2_frame *frame) { int rv = 0; @@ -5184,11 +5291,19 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session, const uint8_t *in, default: DEBUGF(fprintf(stderr, "recv: unknown frame\n")); - /* Silently ignore unknown frame type. */ + if (!session->callbacks.unpack_extension_callback) { + /* Silently ignore unknown frame type. */ + + busy = 1; + + iframe->state = NGHTTP2_IB_IGN_PAYLOAD; + + break; + } busy = 1; - iframe->state = NGHTTP2_IB_IGN_PAYLOAD; + iframe->state = NGHTTP2_IB_READ_EXTENSION_PAYLOAD; break; } @@ -5886,6 +6001,44 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session, const uint8_t *in, break; case NGHTTP2_IB_IGN_ALL: return (ssize_t)inlen; + case NGHTTP2_IB_READ_EXTENSION_PAYLOAD: + DEBUGF(fprintf(stderr, "recv: [IB_READ_EXTENSION_PAYLOAD]\n")); + + readlen = inbound_frame_payload_readlen(iframe, in, last); + iframe->payloadleft -= readlen; + in += readlen; + + DEBUGF(fprintf(stderr, "recv: readlen=%zu, payloadleft=%zu\n", readlen, + iframe->payloadleft)); + + if (readlen > 0) { + rv = session_call_on_extension_chunk_recv_callback( + session, in - readlen, readlen); + if (nghttp2_is_fatal(rv)) { + return rv; + } + + if (rv != 0) { + busy = 1; + + iframe->state = NGHTTP2_IB_IGN_PAYLOAD; + + break; + } + } + + if (iframe->payloadleft > 0) { + break; + } + + rv = session_process_extension_frame(session); + if (nghttp2_is_fatal(rv)) { + return rv; + } + + session_inbound_frame_reset(session); + + break; } if (!busy && in == last) { diff --git a/lib/nghttp2_session.h b/lib/nghttp2_session.h index 67860b20..a996f8fd 100644 --- a/lib/nghttp2_session.h +++ b/lib/nghttp2_session.h @@ -102,7 +102,8 @@ typedef enum { NGHTTP2_IB_READ_PAD_DATA, NGHTTP2_IB_READ_DATA, NGHTTP2_IB_IGN_DATA, - NGHTTP2_IB_IGN_ALL + NGHTTP2_IB_IGN_ALL, + NGHTTP2_IB_READ_EXTENSION_PAYLOAD } nghttp2_inbound_state; #define NGHTTP2_INBOUND_NUM_IV 7 diff --git a/lib/nghttp2_submit.c b/lib/nghttp2_submit.c index 763b4038..7dc29ed8 100644 --- a/lib/nghttp2_submit.c +++ b/lib/nghttp2_submit.c @@ -479,3 +479,40 @@ ssize_t nghttp2_pack_settings_payload(uint8_t *buf, size_t buflen, return (ssize_t)nghttp2_frame_pack_settings_payload(buf, iv, niv); } + +int nghttp2_submit_extension(nghttp2_session *session, uint8_t type, + uint8_t flags, int32_t stream_id, void *payload) { + int rv; + nghttp2_outbound_item *item; + nghttp2_frame *frame; + nghttp2_mem *mem; + + mem = &session->mem; + + if (type <= NGHTTP2_CONTINUATION) { + return NGHTTP2_ERR_INVALID_ARGUMENT; + } + + if (!session->callbacks.pack_extension_callback) { + return NGHTTP2_ERR_INVALID_STATE; + } + + item = nghttp2_mem_malloc(mem, sizeof(nghttp2_outbound_item)); + if (item == NULL) { + return NGHTTP2_ERR_NOMEM; + } + + nghttp2_outbound_item_init(item); + + frame = &item->frame; + nghttp2_frame_extension_init(&frame->ext, type, flags, stream_id, payload); + + rv = nghttp2_session_add_item(session, item); + if (rv != 0) { + nghttp2_frame_extension_free(&frame->ext); + nghttp2_mem_free(mem, item); + return rv; + } + + return 0; +} diff --git a/src/app_helper.cc b/src/app_helper.cc index 2ce51af7..c360a7de 100644 --- a/src/app_helper.cc +++ b/src/app_helper.cc @@ -121,7 +121,7 @@ const char *strsettingsid(int32_t id) { } // namespace namespace { -const char *strframetype(uint8_t type) { +std::string strframetype(uint8_t type) { switch (type) { case NGHTTP2_DATA: return "DATA"; @@ -141,9 +141,13 @@ const char *strframetype(uint8_t type) { return "GOAWAY"; case NGHTTP2_WINDOW_UPDATE: return "WINDOW_UPDATE"; - default: - return "UNKNOWN"; } + + std::string s = "UNKNOWN(0x"; + s += util::format_hex(&type, 1); + s += ")"; + + return s; }; } // namespace @@ -280,7 +284,7 @@ const char *frame_name_ansi_esc(print_type ptype) { namespace { void print_frame(print_type ptype, const nghttp2_frame *frame) { fprintf(outfile, "%s%s%s frame ", frame_name_ansi_esc(ptype), - strframetype(frame->hd.type), ansi_escend()); + strframetype(frame->hd.type).c_str(), ansi_escend()); print_frame_hd(frame->hd); if (frame->hd.flags) { print_frame_attr_indent(); From d9893d014c4e1ed061472bbac63c46324582a213 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 11 Oct 2015 17:23:01 +0900 Subject: [PATCH 002/147] Add tests --- tests/main.c | 3 + tests/nghttp2_session_test.c | 189 +++++++++++++++++++++++++++++++++++ tests/nghttp2_session_test.h | 2 + 3 files changed, 194 insertions(+) diff --git a/tests/main.c b/tests/main.c index 64710066..afcf5390 100644 --- a/tests/main.c +++ b/tests/main.c @@ -97,6 +97,8 @@ int main(int argc _U_, char *argv[] _U_) { test_nghttp2_session_recv_settings_header_table_size) || !CU_add_test(pSuite, "session_recv_too_large_frame_length", test_nghttp2_session_recv_too_large_frame_length) || + !CU_add_test(pSuite, "session_recv_extension", + test_nghttp2_session_recv_extension) || !CU_add_test(pSuite, "session_continue", test_nghttp2_session_continue) || !CU_add_test(pSuite, "session_add_frame", test_nghttp2_session_add_frame) || @@ -186,6 +188,7 @@ int main(int argc _U_, char *argv[] _U_) { test_nghttp2_submit_shutdown_notice) || !CU_add_test(pSuite, "submit_invalid_nv", test_nghttp2_submit_invalid_nv) || + !CU_add_test(pSuite, "submit_extension", test_nghttp2_submit_extension) || !CU_add_test(pSuite, "session_open_stream", test_nghttp2_session_open_stream) || !CU_add_test(pSuite, "session_open_stream_with_idle_stream_dep", diff --git a/tests/nghttp2_session_test.c b/tests/nghttp2_session_test.c index c2376b30..d34b1569 100644 --- a/tests/nghttp2_session_test.c +++ b/tests/nghttp2_session_test.c @@ -58,6 +58,7 @@ typedef struct { scripted_data_feed *df; int frame_recv_cb_called, invalid_frame_recv_cb_called; uint8_t recv_frame_type; + nghttp2_frame_hd recv_frame_hd; int frame_send_cb_called; uint8_t sent_frame_type; int frame_not_send_cb_called; @@ -77,6 +78,7 @@ typedef struct { size_t data_chunk_len; size_t padlen; int begin_frame_cb_called; + nghttp2_buf scratchbuf; } my_user_data; static const nghttp2_nv reqnv[] = { @@ -179,6 +181,8 @@ static int on_frame_recv_callback(nghttp2_session *session _U_, my_user_data *ud = (my_user_data *)user_data; ++ud->frame_recv_cb_called; ud->recv_frame_type = frame->hd.type; + ud->recv_frame_hd = frame->hd; + return 0; } @@ -430,6 +434,55 @@ static int on_stream_close_callback(nghttp2_session *session _U_, return 0; } +static ssize_t pack_extension_callback(nghttp2_session *session _U_, + uint8_t *flags _U_, uint8_t *buf, + size_t len _U_, + const nghttp2_frame *frame, + void *user_data _U_) { + nghttp2_buf *p = frame->ext.payload; + + memcpy(buf, p->pos, nghttp2_buf_len(p)); + + return (ssize_t)nghttp2_buf_len(p); +} + +static int on_extension_chunk_recv_callback(nghttp2_session *session _U_, + const nghttp2_frame_hd *hd _U_, + const uint8_t *data, size_t len, + void *user_data) { + my_user_data *my_data = (my_user_data *)user_data; + nghttp2_buf *buf = &my_data->scratchbuf; + + buf->last = nghttp2_cpymem(buf->last, data, len); + + return 0; +} + +static int cancel_on_extension_chunk_recv_callback( + nghttp2_session *session _U_, const nghttp2_frame_hd *hd _U_, + const uint8_t *data _U_, size_t len _U_, void *user_data _U_) { + return NGHTTP2_ERR_CANCEL; +} + +static int unpack_extension_callback(nghttp2_session *session _U_, + void **payload, + const nghttp2_frame_hd *hd _U_, + void *user_data) { + my_user_data *my_data = (my_user_data *)user_data; + nghttp2_buf *buf = &my_data->scratchbuf; + + *payload = buf; + + return 0; +} + +static int cancel_unpack_extension_callback(nghttp2_session *session _U_, + void **payload _U_, + const nghttp2_frame_hd *hd _U_, + void *user_data _U_) { + return NGHTTP2_ERR_CANCEL; +} + static nghttp2_settings_entry *dup_iv(const nghttp2_settings_entry *iv, size_t niv) { return nghttp2_frame_iv_copy(iv, niv, nghttp2_mem_default()); @@ -1707,6 +1760,81 @@ void test_nghttp2_session_recv_too_large_frame_length(void) { nghttp2_session_del(session); } +void test_nghttp2_session_recv_extension(void) { + nghttp2_session *session; + nghttp2_session_callbacks callbacks; + my_user_data ud; + nghttp2_buf buf; + nghttp2_frame_hd hd; + nghttp2_mem *mem; + const char data[] = "Hello World!"; + ssize_t rv; + + mem = nghttp2_mem_default(); + + memset(&callbacks, 0, sizeof(nghttp2_session_callbacks)); + + callbacks.on_extension_chunk_recv_callback = on_extension_chunk_recv_callback; + callbacks.unpack_extension_callback = unpack_extension_callback; + callbacks.on_frame_recv_callback = on_frame_recv_callback; + + nghttp2_buf_init2(&ud.scratchbuf, 4096, mem); + nghttp2_buf_init2(&buf, 4096, mem); + + nghttp2_frame_hd_init(&hd, sizeof(data), 111, 0xab, 1000000007); + nghttp2_frame_pack_frame_hd(buf.last, &hd); + buf.last += NGHTTP2_FRAME_HDLEN; + buf.last = nghttp2_cpymem(buf.last, data, sizeof(data)); + + nghttp2_session_client_new(&session, &callbacks, &ud); + + nghttp2_frame_hd_init(&ud.recv_frame_hd, 0, 0, 0, 0); + rv = nghttp2_session_mem_recv(session, buf.pos, nghttp2_buf_len(&buf)); + + CU_ASSERT(NGHTTP2_FRAME_HDLEN + hd.length == (size_t)rv); + CU_ASSERT(111 == ud.recv_frame_hd.type); + CU_ASSERT(0xab == ud.recv_frame_hd.flags); + CU_ASSERT(1000000007 == ud.recv_frame_hd.stream_id); + CU_ASSERT(0 == memcmp(data, ud.scratchbuf.pos, sizeof(data))); + + nghttp2_session_del(session); + + /* cancel in on_extension_chunk_recv_callback */ + nghttp2_buf_reset(&ud.scratchbuf); + + callbacks.on_extension_chunk_recv_callback = + cancel_on_extension_chunk_recv_callback; + + nghttp2_session_server_new(&session, &callbacks, &ud); + + ud.frame_recv_cb_called = 0; + rv = nghttp2_session_mem_recv(session, buf.pos, nghttp2_buf_len(&buf)); + + CU_ASSERT(NGHTTP2_FRAME_HDLEN + hd.length == (size_t)rv); + CU_ASSERT(0 == ud.frame_recv_cb_called); + + nghttp2_session_del(session); + + /* cancel in unpack_extension_callback */ + nghttp2_buf_reset(&ud.scratchbuf); + + callbacks.on_extension_chunk_recv_callback = on_extension_chunk_recv_callback; + callbacks.unpack_extension_callback = cancel_unpack_extension_callback; + + nghttp2_session_server_new(&session, &callbacks, &ud); + + ud.frame_recv_cb_called = 0; + rv = nghttp2_session_mem_recv(session, buf.pos, nghttp2_buf_len(&buf)); + + CU_ASSERT(NGHTTP2_FRAME_HDLEN + hd.length == (size_t)rv); + CU_ASSERT(0 == ud.frame_recv_cb_called); + + nghttp2_session_del(session); + + nghttp2_buf_free(&buf, mem); + nghttp2_buf_free(&ud.scratchbuf, mem); +} + void test_nghttp2_session_continue(void) { nghttp2_session *session; nghttp2_session_callbacks callbacks; @@ -4691,6 +4819,67 @@ void test_nghttp2_submit_invalid_nv(void) { nghttp2_session_del(session); } +void test_nghttp2_submit_extension(void) { + nghttp2_session *session; + nghttp2_session_callbacks callbacks; + my_user_data ud; + accumulator acc; + nghttp2_mem *mem; + const char data[] = "Hello World!"; + size_t len; + int32_t stream_id; + int rv; + + mem = nghttp2_mem_default(); + + memset(&callbacks, 0, sizeof(nghttp2_session_callbacks)); + + callbacks.pack_extension_callback = pack_extension_callback; + callbacks.send_callback = accumulator_send_callback; + + nghttp2_buf_init2(&ud.scratchbuf, 4096, mem); + + nghttp2_session_client_new(&session, &callbacks, &ud); + + ud.scratchbuf.last = nghttp2_cpymem(ud.scratchbuf.last, data, sizeof(data)); + ud.acc = &acc; + + rv = nghttp2_submit_extension(session, 211, 0x01, 3, &ud.scratchbuf); + + CU_ASSERT(0 == rv); + + acc.length = 0; + + rv = nghttp2_session_send(session); + + CU_ASSERT(0 == rv); + CU_ASSERT(NGHTTP2_FRAME_HDLEN + sizeof(data) == acc.length); + + len = nghttp2_get_uint32(acc.buf) >> 8; + + CU_ASSERT(sizeof(data) == len); + CU_ASSERT(211 == acc.buf[3]); + CU_ASSERT(0x01 == acc.buf[4]); + + stream_id = (int32_t)nghttp2_get_uint32(acc.buf + 5); + + CU_ASSERT(3 == stream_id); + CU_ASSERT(0 == memcmp(data, &acc.buf[NGHTTP2_FRAME_HDLEN], sizeof(data))); + + nghttp2_session_del(session); + + /* submitting standard HTTP/2 frame is error */ + nghttp2_session_server_new(&session, &callbacks, &ud); + + rv = nghttp2_submit_extension(session, NGHTTP2_GOAWAY, NGHTTP2_FLAG_NONE, 0, + NULL); + + CU_ASSERT(NGHTTP2_ERR_INVALID_ARGUMENT == rv); + + nghttp2_session_del(session); + nghttp2_buf_free(&ud.scratchbuf, mem); +} + void test_nghttp2_session_open_stream(void) { nghttp2_session *session; nghttp2_session_callbacks callbacks; diff --git a/tests/nghttp2_session_test.h b/tests/nghttp2_session_test.h index ce08b5f9..80f0cfc9 100644 --- a/tests/nghttp2_session_test.h +++ b/tests/nghttp2_session_test.h @@ -38,6 +38,7 @@ void test_nghttp2_session_recv_unknown_frame(void); void test_nghttp2_session_recv_unexpected_continuation(void); void test_nghttp2_session_recv_settings_header_table_size(void); void test_nghttp2_session_recv_too_large_frame_length(void); +void test_nghttp2_session_recv_extension(void); void test_nghttp2_session_continue(void); void test_nghttp2_session_add_frame(void); void test_nghttp2_session_on_request_headers_received(void); @@ -85,6 +86,7 @@ void test_nghttp2_submit_window_update(void); void test_nghttp2_submit_window_update_local_window_size(void); void test_nghttp2_submit_shutdown_notice(void); void test_nghttp2_submit_invalid_nv(void); +void test_nghttp2_submit_extension(void); void test_nghttp2_session_open_stream(void); void test_nghttp2_session_open_stream_with_idle_stream_dep(void); void test_nghttp2_session_get_next_ob_item(void); From 061a557839207028593015c5f0bd3ae4b81018bd Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 15 Oct 2015 00:17:07 +0900 Subject: [PATCH 003/147] Add nghttp2_option_set_user_recv_extension_type to opt-in incoming extension type --- lib/includes/nghttp2/nghttp2.h | 15 +++++++++++++++ lib/nghttp2_option.c | 10 ++++++++++ lib/nghttp2_option.h | 7 ++++++- lib/nghttp2_session.c | 9 ++++++++- lib/nghttp2_session.h | 6 ++++++ tests/nghttp2_session_test.c | 12 +++++++++--- 6 files changed, 54 insertions(+), 5 deletions(-) diff --git a/lib/includes/nghttp2/nghttp2.h b/lib/includes/nghttp2/nghttp2.h index 91030f28..e098bcce 100644 --- a/lib/includes/nghttp2/nghttp2.h +++ b/lib/includes/nghttp2/nghttp2.h @@ -2186,6 +2186,21 @@ NGHTTP2_EXTERN void nghttp2_option_set_max_reserved_remote_streams(nghttp2_option *option, uint32_t val); +/** + * @function + * + * Set extension frame type the application is willing to handle with + * user defined callbacks (see + * :type:`nghttp2_on_extension_chunk_recv_callback` and + * :type:`nghttp2_unpack_extension_callback`). The |type| is + * extension frame type, and must be strictly greater than 0x9. + * Otherwise, this function does nothing. The application does not + * have to call this function if it just sends extension frames. + */ +NGHTTP2_EXTERN void +nghttp2_option_set_user_recv_extension_type(nghttp2_option *option, + uint8_t type); + /** * @function * diff --git a/lib/nghttp2_option.c b/lib/nghttp2_option.c index 04dbbc6a..81ae6ba7 100644 --- a/lib/nghttp2_option.c +++ b/lib/nghttp2_option.c @@ -62,3 +62,13 @@ void nghttp2_option_set_max_reserved_remote_streams(nghttp2_option *option, option->opt_set_mask |= NGHTTP2_OPT_MAX_RESERVED_REMOTE_STREAMS; option->max_reserved_remote_streams = val; } + +void nghttp2_option_set_user_recv_extension_type(nghttp2_option *option, + uint8_t type) { + if (type < 10) { + return; + } + + option->opt_set_mask |= NGHTTP2_OPT_USER_RECV_EXT_TYPES; + option->user_recv_ext_types[type / 8] |= 1 << (7 - (type & 0x7)); +} diff --git a/lib/nghttp2_option.h b/lib/nghttp2_option.h index ebf416ac..a2d090fb 100644 --- a/lib/nghttp2_option.h +++ b/lib/nghttp2_option.h @@ -59,7 +59,8 @@ typedef enum { NGHTTP2_OPT_PEER_MAX_CONCURRENT_STREAMS = 1 << 1, NGHTTP2_OPT_NO_RECV_CLIENT_MAGIC = 1 << 2, NGHTTP2_OPT_NO_HTTP_MESSAGING = 1 << 3, - NGHTTP2_OPT_MAX_RESERVED_REMOTE_STREAMS = 1 << 4 + NGHTTP2_OPT_MAX_RESERVED_REMOTE_STREAMS = 1 << 4, + NGHTTP2_OPT_USER_RECV_EXT_TYPES = 1 << 5 } nghttp2_option_flag; /** @@ -91,6 +92,10 @@ struct nghttp2_option { * NGHTTP2_OPT_NO_HTTP_MESSAGING */ int no_http_messaging; + /** + * NGHTTP2_OPT_USER_RECV_EXT_TYPES + */ + uint8_t user_recv_ext_types[32]; }; #endif /* NGHTTP2_OPTION_H */ diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 787bc311..1e8e81e5 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -402,6 +402,11 @@ static int session_new(nghttp2_session **session_ptr, (*session_ptr)->opt_flags |= NGHTTP2_OPTMASK_NO_HTTP_MESSAGING; } + + if (option->opt_set_mask & NGHTTP2_OPT_USER_RECV_EXT_TYPES) { + memcpy((*session_ptr)->user_recv_ext_types, option->user_recv_ext_types, + sizeof((*session_ptr)->user_recv_ext_types)); + } } (*session_ptr)->callbacks = *callbacks; @@ -5291,7 +5296,9 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session, const uint8_t *in, default: DEBUGF(fprintf(stderr, "recv: unknown frame\n")); - if (!session->callbacks.unpack_extension_callback) { + if (!session->callbacks.unpack_extension_callback || + (session->user_recv_ext_types[iframe->frame.hd.type / 8] & + (1 << (7 - (iframe->frame.hd.type & 0x7)))) == 0) { /* Silently ignore unknown frame type. */ busy = 1; diff --git a/lib/nghttp2_session.h b/lib/nghttp2_session.h index a996f8fd..73c10cdf 100644 --- a/lib/nghttp2_session.h +++ b/lib/nghttp2_session.h @@ -296,6 +296,12 @@ struct nghttp2_session { this session. The nonzero does not necessarily mean WINDOW_UPDATE is not queued. */ uint8_t window_update_queued; + /* Bitfield of extension frame types that application is willing to + receive. First most significant 10 bits are standard frame types + and not used. If bit is set, it indicates that incoming frame + with that type is passed to user defined callbacks, otherwise + they are ignored. */ + uint8_t user_recv_ext_types[32]; }; /* Struct used when updating initial window size of each active diff --git a/tests/nghttp2_session_test.c b/tests/nghttp2_session_test.c index d34b1569..7ed6af65 100644 --- a/tests/nghttp2_session_test.c +++ b/tests/nghttp2_session_test.c @@ -1769,6 +1769,7 @@ void test_nghttp2_session_recv_extension(void) { nghttp2_mem *mem; const char data[] = "Hello World!"; ssize_t rv; + nghttp2_option *option; mem = nghttp2_mem_default(); @@ -1778,6 +1779,9 @@ void test_nghttp2_session_recv_extension(void) { callbacks.unpack_extension_callback = unpack_extension_callback; callbacks.on_frame_recv_callback = on_frame_recv_callback; + nghttp2_option_new(&option); + nghttp2_option_set_user_recv_extension_type(option, 111); + nghttp2_buf_init2(&ud.scratchbuf, 4096, mem); nghttp2_buf_init2(&buf, 4096, mem); @@ -1786,7 +1790,7 @@ void test_nghttp2_session_recv_extension(void) { buf.last += NGHTTP2_FRAME_HDLEN; buf.last = nghttp2_cpymem(buf.last, data, sizeof(data)); - nghttp2_session_client_new(&session, &callbacks, &ud); + nghttp2_session_client_new2(&session, &callbacks, &ud, option); nghttp2_frame_hd_init(&ud.recv_frame_hd, 0, 0, 0, 0); rv = nghttp2_session_mem_recv(session, buf.pos, nghttp2_buf_len(&buf)); @@ -1805,7 +1809,7 @@ void test_nghttp2_session_recv_extension(void) { callbacks.on_extension_chunk_recv_callback = cancel_on_extension_chunk_recv_callback; - nghttp2_session_server_new(&session, &callbacks, &ud); + nghttp2_session_server_new2(&session, &callbacks, &ud, option); ud.frame_recv_cb_called = 0; rv = nghttp2_session_mem_recv(session, buf.pos, nghttp2_buf_len(&buf)); @@ -1821,7 +1825,7 @@ void test_nghttp2_session_recv_extension(void) { callbacks.on_extension_chunk_recv_callback = on_extension_chunk_recv_callback; callbacks.unpack_extension_callback = cancel_unpack_extension_callback; - nghttp2_session_server_new(&session, &callbacks, &ud); + nghttp2_session_server_new2(&session, &callbacks, &ud, option); ud.frame_recv_cb_called = 0; rv = nghttp2_session_mem_recv(session, buf.pos, nghttp2_buf_len(&buf)); @@ -1833,6 +1837,8 @@ void test_nghttp2_session_recv_extension(void) { nghttp2_buf_free(&buf, mem); nghttp2_buf_free(&ud.scratchbuf, mem); + + nghttp2_option_del(option); } void test_nghttp2_session_continue(void) { From 837e71630600df6854f9d9fffc075787602a2a6e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 15 Oct 2015 00:30:42 +0900 Subject: [PATCH 004/147] Fix compile error with gcc --- lib/nghttp2_option.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/nghttp2_option.c b/lib/nghttp2_option.c index 81ae6ba7..2201ec6e 100644 --- a/lib/nghttp2_option.c +++ b/lib/nghttp2_option.c @@ -70,5 +70,6 @@ void nghttp2_option_set_user_recv_extension_type(nghttp2_option *option, } option->opt_set_mask |= NGHTTP2_OPT_USER_RECV_EXT_TYPES; - option->user_recv_ext_types[type / 8] |= 1 << (7 - (type & 0x7)); + option->user_recv_ext_types[type / 8] = (uint8_t)( + option->user_recv_ext_types[type / 8] | (1 << (7 - (type & 0x7)))); } From 83cc2511e33ef3bd486261454030c038b10852cc Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 17 Nov 2015 21:29:21 +0900 Subject: [PATCH 005/147] Remove flags parameter from nghttp2_pack_extension_callback It has no usecase at the moment. It is most likely that applications know the flags when it submitted extension frame, no need to modify it later. Possibly feature bloat. --- lib/includes/nghttp2/nghttp2.h | 11 ++++------- lib/nghttp2_session.c | 7 ++----- tests/nghttp2_session_test.c | 3 +-- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/lib/includes/nghttp2/nghttp2.h b/lib/includes/nghttp2/nghttp2.h index 926d730d..09e2dba5 100644 --- a/lib/includes/nghttp2/nghttp2.h +++ b/lib/includes/nghttp2/nghttp2.h @@ -1747,11 +1747,9 @@ typedef int (*nghttp2_unpack_extension_callback)(nghttp2_session *session, * extension payload in its wire format. The frame header will be * packed by library. Application must pack payload only. * frame->ext.payload is the object passed to - * `nghttp2_submit_extension()` as payload parameter. The |*flags| is - * initialized to the flags parameter passed in - * `nghttp2_submit_extension()`. Application can modify flags value - * if it wants. Application must pack extension payload to the |buf| - * of its capacity |len| bytes. + * `nghttp2_submit_extension()` as payload parameter. Application + * must pack extension payload to the |buf| of its capacity |len| + * bytes. * * The implementation of this function returns the number of bytes * written into |buf| when it succeeds. @@ -1770,8 +1768,7 @@ typedef int (*nghttp2_unpack_extension_callback)(nghttp2_session *session, * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. */ typedef ssize_t (*nghttp2_pack_extension_callback)(nghttp2_session *session, - uint8_t *flags, uint8_t *buf, - size_t len, + uint8_t *buf, size_t len, const nghttp2_frame *frame, void *user_data); diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 9c6d34ac..84f90a5b 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -1738,19 +1738,17 @@ static size_t session_estimate_headers_payload(nghttp2_session *session, static int session_pack_extension(nghttp2_session *session, nghttp2_bufs *bufs, nghttp2_frame *frame) { ssize_t rv; - uint8_t flags; nghttp2_buf *buf; size_t buflen; size_t framelen; assert(session->callbacks.pack_extension_callback); - flags = frame->hd.flags; buf = &bufs->head->buf; buflen = nghttp2_min(nghttp2_buf_avail(buf), NGHTTP2_MAX_PAYLOADLEN); - rv = session->callbacks.pack_extension_callback( - session, &flags, buf->last, buflen, frame, session->user_data); + rv = session->callbacks.pack_extension_callback(session, buf->last, buflen, + frame, session->user_data); if (rv == NGHTTP2_ERR_CANCEL) { return (int)rv; } @@ -1761,7 +1759,6 @@ static int session_pack_extension(nghttp2_session *session, nghttp2_bufs *bufs, framelen = (size_t)rv; - frame->hd.flags = flags; frame->hd.length = framelen; assert(buf->pos == buf->last); diff --git a/tests/nghttp2_session_test.c b/tests/nghttp2_session_test.c index dcde6d93..86c9e711 100644 --- a/tests/nghttp2_session_test.c +++ b/tests/nghttp2_session_test.c @@ -435,8 +435,7 @@ static int on_stream_close_callback(nghttp2_session *session _U_, } static ssize_t pack_extension_callback(nghttp2_session *session _U_, - uint8_t *flags _U_, uint8_t *buf, - size_t len _U_, + uint8_t *buf, size_t len _U_, const nghttp2_frame *frame, void *user_data _U_) { nghttp2_buf *p = frame->ext.payload; From 0248d979fe7a71d35d7938944c04e270f76421ae Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 10 Jan 2016 17:08:03 +0900 Subject: [PATCH 006/147] Add missing nghttp2_option_set_user_recv_extension_type.rst --- doc/Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/Makefile.am b/doc/Makefile.am index 065b4c24..a0de3cb0 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -58,6 +58,7 @@ APIDOCS= \ nghttp2_option_set_no_http_messaging.rst \ nghttp2_option_set_no_recv_client_magic.rst \ nghttp2_option_set_peer_max_concurrent_streams.rst \ + nghttp2_option_set_user_recv_extension_type.rst \ nghttp2_pack_settings_payload.rst \ nghttp2_priority_spec_check_default.rst \ nghttp2_priority_spec_default_init.rst \ From 304ff6a6f98c7437a5a7f31d14f6db4991290f7d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 7 Feb 2016 21:12:36 +0900 Subject: [PATCH 007/147] Don't send extension frame in closing state --- lib/nghttp2_session.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 35be7cf5..4a905ef5 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -2034,6 +2034,10 @@ static int session_prep_frame(nghttp2_session *session, break; default: /* extension frame */ + if (session_is_closing(session)) { + return NGHTTP2_ERR_SESSION_CLOSING; + } + rv = session_pack_extension(session, &session->aob.framebufs, frame); if (rv != 0) { return rv; From fe74600a5f4dfcce2f6979d2cd62f388f97426fa Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 17:40:58 +0900 Subject: [PATCH 008/147] List all contributors in AUTHORS --- AUTHORS | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- COPYING | 1 + 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 95bc9542..8c4a70e8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1 +1,80 @@ -Tatsuhiro Tsujikawa +nghttp2 project was started as a fork of spdylay project [1]. Both +projects were started by Tatsuhiro Tsujikawa, who is still the main +author of these projects. Meanwhile, we have many contributions, and +we are not here without them. We sincerely thank you to all who made +a contribution. Here is the all individuals/organizations who +contributed to nghttp2 and spdylay project at which we forked. These +names are retrieved from git commit log. If you have made a +contribution, but you are missing in the list, please let us know via +github issues [2]. + +[1] https://github.com/tatsuhiro-t/spdylay +[2] https://github.com/tatsuhiro-t/nghttp2/issues + +-------- + +187j3x1 +Alek Storm +Alex Nalivko +Alexis La Goutte +Anders Bakken +Andreas Pohl +Andy Davies +Ant Bryan +Bernard Spil +Brian Card +Daniel Stenberg +Dave Reisner +David Beitey +David Weekly +Etienne Cimon +Fabian Möller +Fabian Wiesel +Gabi Davar +Janusz Dziemidowicz +Jay Satiro +Jim Morrison +José F. Calcerrada +Kamil Dudka +Kazuho Oku +Kenny (kang-yen) Peng +Kenny Peng +Kit Chan +Kyle Schomp +Lucas Pardue +MATSUMOTO Ryosuke +Mike Frysinger +Nicholas Hurley +Nora Shoemaker +Peeyush Aggarwal +Peter Wu +Piotr Sikora +Raul Gutierrez Segales +Remo E +Reza Tavakoli +Ross Smith II +Scott Mitchell +Stefan Eissing +Stephen Ludin +Sunpoet Po-Chuan Hsieh +Svante Signell +Syohei YOSHIDA +Tatsuhiko Kubo +Tatsuhiro Tsujikawa +Tom Harwood +Tomasz Buchert +Vernon Tang +Viacheslav Biriukov +Viktor Szépe +Xiaoguang Sun +Zhuoyun Wei +acesso +ayanamist +bxshi +es +fangdingjun +kumagi +mod-h2-dev +moparisthebest +snnn +yuuki-kodama diff --git a/COPYING b/COPYING index 6e079b80..80201792 100644 --- a/COPYING +++ b/COPYING @@ -1,6 +1,7 @@ The MIT License Copyright (c) 2012, 2014, 2015, 2016 Tatsuhiro Tsujikawa +Copyright (c) 2012, 2014, 2015, 2016 nghttp2 contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the From eebed206c91690032bfc7102144aadc295a59f19 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 18:23:28 +0900 Subject: [PATCH 009/147] Add Architecture doc --- doc/programmers-guide.rst | 56 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/doc/programmers-guide.rst b/doc/programmers-guide.rst index 94c72e98..253c9fd3 100644 --- a/doc/programmers-guide.rst +++ b/doc/programmers-guide.rst @@ -1,6 +1,62 @@ Programmers' Guide ================== +Architecture +------------ + +The most notable point in nghttp2 library architecture is it does not +perform any I/O. nghttp2 only performs HTTP/2 protocol stuff based on +input byte strings. It will calls callback functions set by +applications while processing input. The output of nghttp2 is just +byte string. An application is responsible to send these output to +the remote peer. The callback functions may be called while producing +output. + +Not doing I/O makes embedding nghttp2 library in the existing code +base very easy. Usually, the existing applications have its own I/O +event loops. It is very hard to use nghttp2 in that situation if +nghttp2 does its own I/O. It also makes light weight language wrapper +for nghttp2 easy with the same reason. The down side is that an +application author has to write more code to write complete +application using nghttp2. This is especially true for simple "toy" +application. For the real applications, however, this is not the +case. This is because you probably want to support HTTP/1 which +nghttp2 does not provide, and to do that, you will need to write your +own HTTP/1 stack or use existing third-party library, and bind them +together with nghttp2 and I/O event loop. In this point, not +performing I/O in nghttp2 has more point than doing it. + +The primary object that an application uses is :type:`nghttp2_session` +object, which is opaque struct and its details are hidden in order to +ensure the upgrading its internal architecture without breaking the +backward compatibility. An application can set callbacks to +:type:`nghttp2_session` object through the dedicated object and +functions, and it also interacts with it via many API function calls. + +An application can create as many :type:`nghttp2_session` object as it +wants. But single :type:`nghttp2_session` object must be used by a +single thread at the same time. This is not so hard to enforce since +most event-based architecture applicatons use is single thread per +core, and handling one connection I/O is done by single thread. + +To feed input to :type:`nghttp2_session` object, one can use +`nghttp2_session_recv()` or `nghttp2_session_mem_recv()` functions. +They behave similarly, and the difference is that +`nghttp2_session_recv()` will use :type:`nghttp2_read_callback` to get +input. On the other hand, `nghttp2_session_mem_recv()` will take +input as its parameter. If in doubt, use `nghttp2_session_mem_recv()` +since it is simpler, and could be faster since it avoids calling +callback function. + +To get output from :type:`nghttp2_session` object, one can use +`nghttp2_session_send()` or `nghttp2_session_mem_send()`. The +difference between them is that the former uses +:type:`nghttp2_send_callback` to pass output to an application. On +the other hand, the latter returns the output to the caller. If in +doubt, use `nghttp2_session_mem_send()` since it is simpler. But +`nghttp2_session_send()` might be easier to use if the output buffer +an application has is fixed sized. + Includes -------- From 1bd98dcf4febfc43544c4c193d2f114a266f737f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 18:31:08 +0900 Subject: [PATCH 010/147] nghttpx: Remove user defined ctor/assignment op from DownstreamAddr --- src/shrpx.cc | 2 +- src/shrpx_config.cc | 23 +---------------------- src/shrpx_config.h | 6 ------ 3 files changed, 2 insertions(+), 29 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 17a6bb5a..87bd9c28 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2105,7 +2105,7 @@ void process_options( auto &addr_groups = downstreamconf.addr_groups; if (addr_groups.empty()) { - DownstreamAddr addr; + DownstreamAddr addr{}; addr.host = ImmutableString::from_lit(DEFAULT_DOWNSTREAM_HOST); addr.port = DEFAULT_DOWNSTREAM_PORT; diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 104c97b6..489885cf 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -78,27 +78,6 @@ TicketKeys::~TicketKeys() { } } -DownstreamAddr::DownstreamAddr(const DownstreamAddr &other) - : addr(other.addr), - host(other.host), - hostport(other.hostport), - port(other.port), - host_unix(other.host_unix) {} - -DownstreamAddr &DownstreamAddr::operator=(const DownstreamAddr &other) { - if (this == &other) { - return *this; - } - - addr = other.addr; - host = other.host; - hostport = other.hostport; - port = other.port; - host_unix = other.host_unix; - - return *this; -} - DownstreamAddrGroup::DownstreamAddrGroup(const DownstreamAddrGroup &other) : pattern(strcopy(other.pattern)), addrs(other.addrs) {} @@ -1523,7 +1502,7 @@ int parse_config(const char *opt, const char *optarg, if (!pat_delim) { pat_delim = optarg + optarglen; } - DownstreamAddr addr; + DownstreamAddr addr{}; if (util::istarts_with(optarg, SHRPX_UNIX_PATH_PREFIX)) { auto path = optarg + str_size(SHRPX_UNIX_PATH_PREFIX); addr.host = ImmutableString(path, pat_delim); diff --git a/src/shrpx_config.h b/src/shrpx_config.h index ebc97f8a..c460cbc8 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -289,12 +289,6 @@ struct UpstreamAddr { }; struct DownstreamAddr { - DownstreamAddr() : addr{}, port(0), host_unix(false) {} - DownstreamAddr(const DownstreamAddr &other); - DownstreamAddr(DownstreamAddr &&) = default; - DownstreamAddr &operator=(const DownstreamAddr &other); - DownstreamAddr &operator=(DownstreamAddr &&other) = default; - Address addr; // backend address. If |host_unix| is true, this is UNIX domain // socket path. From a53f0f0a17ecf86dbb09f6e7a0d397f5cfda2431 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 18:47:24 +0900 Subject: [PATCH 011/147] nghttpx: Refactor DownstreamAddrGroup and router API --- src/shrpx.cc | 12 ++++++------ src/shrpx_config.cc | 27 ++++++--------------------- src/shrpx_config.h | 9 +++------ src/shrpx_config_test.cc | 2 +- 4 files changed, 16 insertions(+), 34 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 87bd9c28..24290891 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2109,14 +2109,14 @@ void process_options( addr.host = ImmutableString::from_lit(DEFAULT_DOWNSTREAM_HOST); addr.port = DEFAULT_DOWNSTREAM_PORT; - DownstreamAddrGroup g("/"); + DownstreamAddrGroup g(StringRef::from_lit("/")); g.addrs.push_back(std::move(addr)); - mod_config()->router.add_route(g.pattern.get(), 1, addr_groups.size()); + mod_config()->router.add_route(g.pattern.c_str(), 1, addr_groups.size()); addr_groups.push_back(std::move(g)); } else if (get_config()->http2_proxy || get_config()->client_proxy) { // We don't support host mapping in these cases. Move all // non-catch-all patterns to catch-all pattern. - DownstreamAddrGroup catch_all("/"); + DownstreamAddrGroup catch_all(StringRef::from_lit("/")); for (auto &g : addr_groups) { std::move(std::begin(g.addrs), std::end(g.addrs), std::back_inserter(catch_all.addrs)); @@ -2124,7 +2124,7 @@ void process_options( std::vector().swap(addr_groups); // maybe not necessary? mod_config()->router = Router(); - mod_config()->router.add_route(catch_all.pattern.get(), 1, + mod_config()->router.add_route(catch_all.pattern.c_str(), 1, addr_groups.size()); addr_groups.push_back(std::move(catch_all)); } @@ -2136,11 +2136,11 @@ void process_options( ssize_t catch_all_group = -1; for (size_t i = 0; i < addr_groups.size(); ++i) { auto &g = addr_groups[i]; - if (util::streq(g.pattern.get(), "/")) { + if (g.pattern == "/") { catch_all_group = i; } if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Host-path pattern: group " << i << ": '" << g.pattern.get() + LOG(INFO) << "Host-path pattern: group " << i << ": '" << g.pattern << "'"; for (auto &addr : g.addrs) { LOG(INFO) << "group " << i << " -> " << addr.host.c_str() diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 489885cf..23b480b3 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -78,21 +78,6 @@ TicketKeys::~TicketKeys() { } } -DownstreamAddrGroup::DownstreamAddrGroup(const DownstreamAddrGroup &other) - : pattern(strcopy(other.pattern)), addrs(other.addrs) {} - -DownstreamAddrGroup &DownstreamAddrGroup:: -operator=(const DownstreamAddrGroup &other) { - if (this == &other) { - return *this; - } - - pattern = strcopy(other.pattern); - addrs = other.addrs; - - return *this; -} - namespace { int split_host_port(char *host, size_t hostlen, uint16_t *port_ptr, const char *hostport, size_t hostportlen) { @@ -612,7 +597,7 @@ void parse_mapping(const DownstreamAddr &addr, const char *src) { pattern += http2::normalize_path(slash, raw_pattern.second); } for (auto &g : addr_groups) { - if (g.pattern.get() == pattern) { + if (g.pattern == pattern) { g.addrs.push_back(addr); done = true; break; @@ -621,10 +606,10 @@ void parse_mapping(const DownstreamAddr &addr, const char *src) { if (done) { continue; } - DownstreamAddrGroup g(pattern); + DownstreamAddrGroup g(StringRef{pattern}); g.addrs.push_back(addr); - mod_config()->router.add_route(g.pattern.get(), strlen(g.pattern.get()), + mod_config()->router.add_route(g.pattern.c_str(), g.pattern.size(), addr_groups.size()); addr_groups.push_back(std::move(g)); @@ -2529,7 +2514,7 @@ match_downstream_addr_group_host(const Router &router, const std::string &host, if (group != -1) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Found pattern with query " << host - << ", matched pattern=" << groups[group].pattern.get(); + << ", matched pattern=" << groups[group].pattern; } return group; } @@ -2546,7 +2531,7 @@ match_downstream_addr_group_host(const Router &router, const std::string &host, if (LOG_ENABLED(INFO)) { LOG(INFO) << "Found pattern with query " << host << std::string(path, pathlen) - << ", matched pattern=" << groups[group].pattern.get(); + << ", matched pattern=" << groups[group].pattern; } return group; } @@ -2555,7 +2540,7 @@ match_downstream_addr_group_host(const Router &router, const std::string &host, if (group != -1) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Found pattern with query " << std::string(path, pathlen) - << ", matched pattern=" << groups[group].pattern.get(); + << ", matched pattern=" << groups[group].pattern; } return group; } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index c460cbc8..e024317a 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -301,13 +301,10 @@ struct DownstreamAddr { }; struct DownstreamAddrGroup { - DownstreamAddrGroup(const std::string &pattern) : pattern(strcopy(pattern)) {} - DownstreamAddrGroup(const DownstreamAddrGroup &other); - DownstreamAddrGroup(DownstreamAddrGroup &&) = default; - DownstreamAddrGroup &operator=(const DownstreamAddrGroup &other); - DownstreamAddrGroup &operator=(DownstreamAddrGroup &&) = default; + DownstreamAddrGroup(const StringRef &pattern) + : pattern(pattern.c_str(), pattern.size()) {} - std::unique_ptr pattern; + ImmutableString pattern; std::vector addrs; }; diff --git a/src/shrpx_config_test.cc b/src/shrpx_config_test.cc index eda67e85..fa8c1391 100644 --- a/src/shrpx_config_test.cc +++ b/src/shrpx_config_test.cc @@ -256,7 +256,7 @@ void test_shrpx_config_match_downstream_addr_group(void) { for (size_t i = 0; i < groups.size(); ++i) { auto &g = groups[i]; - router.add_route(g.pattern.get(), strlen(g.pattern.get()), i); + router.add_route(g.pattern.c_str(), g.pattern.size(), i); } CU_ASSERT(0 == match_downstream_addr_group(router, "nghttp2.org", "/", groups, From 2d273f82375b527b081bf1e3b2fd60a6024c30de Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 18:55:53 +0900 Subject: [PATCH 012/147] nghttpx: Use StringRef for pattern paramter in Router::add_route --- src/shrpx.cc | 4 ++-- src/shrpx_config.cc | 3 +-- src/shrpx_config_test.cc | 2 +- src/shrpx_router.cc | 12 ++++++------ src/shrpx_router.h | 4 ++-- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 24290891..ee05263b 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2111,7 +2111,7 @@ void process_options( DownstreamAddrGroup g(StringRef::from_lit("/")); g.addrs.push_back(std::move(addr)); - mod_config()->router.add_route(g.pattern.c_str(), 1, addr_groups.size()); + mod_config()->router.add_route(StringRef{g.pattern}, addr_groups.size()); addr_groups.push_back(std::move(g)); } else if (get_config()->http2_proxy || get_config()->client_proxy) { // We don't support host mapping in these cases. Move all @@ -2124,7 +2124,7 @@ void process_options( std::vector().swap(addr_groups); // maybe not necessary? mod_config()->router = Router(); - mod_config()->router.add_route(catch_all.pattern.c_str(), 1, + mod_config()->router.add_route(StringRef{catch_all.pattern}, addr_groups.size()); addr_groups.push_back(std::move(catch_all)); } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 23b480b3..9dd5c444 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -609,8 +609,7 @@ void parse_mapping(const DownstreamAddr &addr, const char *src) { DownstreamAddrGroup g(StringRef{pattern}); g.addrs.push_back(addr); - mod_config()->router.add_route(g.pattern.c_str(), g.pattern.size(), - addr_groups.size()); + mod_config()->router.add_route(StringRef{g.pattern}, addr_groups.size()); addr_groups.push_back(std::move(g)); } diff --git a/src/shrpx_config_test.cc b/src/shrpx_config_test.cc index fa8c1391..53b1bc30 100644 --- a/src/shrpx_config_test.cc +++ b/src/shrpx_config_test.cc @@ -256,7 +256,7 @@ void test_shrpx_config_match_downstream_addr_group(void) { for (size_t i = 0; i < groups.size(); ++i) { auto &g = groups[i]; - router.add_route(g.pattern.c_str(), g.pattern.size(), i); + router.add_route(StringRef{g.pattern}, i); } CU_ASSERT(0 == match_downstream_addr_group(router, "nghttp2.org", "/", groups, diff --git a/src/shrpx_router.cc b/src/shrpx_router.cc index e0404252..4324748a 100644 --- a/src/shrpx_router.cc +++ b/src/shrpx_router.cc @@ -66,21 +66,21 @@ void Router::add_node(RNode *node, const char *pattern, size_t patlen, add_next_node(node, std::move(new_node)); } -bool Router::add_route(const char *pattern, size_t patlen, size_t index) { +bool Router::add_route(const StringRef &pattern, size_t index) { auto node = &root_; size_t i = 0; for (;;) { auto next_node = find_next_node(node, pattern[i]); if (next_node == nullptr) { - add_node(node, pattern + i, patlen - i, index); + add_node(node, pattern.c_str() + i, pattern.size() - i, index); return true; } node = next_node; - auto slen = patlen - i; - auto s = pattern + i; + auto slen = pattern.size() - i; + auto s = pattern.c_str() + i; auto n = std::min(node->len, slen); size_t j; for (j = 0; j < n && node->s[j] == s[j]; ++j) @@ -125,8 +125,8 @@ bool Router::add_route(const char *pattern, size_t patlen, size_t index) { i += j; - assert(patlen > i); - add_node(node, pattern + i, patlen - i, index); + assert(pattern.size() > i); + add_node(node, pattern.c_str() + i, pattern.size() - i, index); return true; } diff --git a/src/shrpx_router.h b/src/shrpx_router.h index 07aadf69..e4d9a465 100644 --- a/src/shrpx_router.h +++ b/src/shrpx_router.h @@ -55,8 +55,8 @@ struct RNode { class Router { public: Router(); - // Adds route |pattern| of size |patlen| with its |index|. - bool add_route(const char *pattern, size_t patlen, size_t index); + // Adds route |pattern| with its |index|. + bool add_route(const StringRef &pattern, size_t index); // Returns the matched index of pattern. -1 if there is no match. ssize_t match(const std::string &host, const char *path, size_t pathlen) const; From 93eabc642b9732a5172b5b2eaf104320d46fa68d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 19:07:22 +0900 Subject: [PATCH 013/147] nghttpx: Use StringRef for parameter in Router::match --- src/shrpx_config.cc | 32 ++++++++++++++------------------ src/shrpx_router.cc | 5 ++--- src/shrpx_router.h | 3 +-- src/template.h | 3 +++ 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 9dd5c444..a5d9f534 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2503,13 +2503,11 @@ int int_syslog_facility(const char *strfacility) { } namespace { -size_t -match_downstream_addr_group_host(const Router &router, const std::string &host, - const char *path, size_t pathlen, - const std::vector &groups, - size_t catch_all) { - if (pathlen == 0 || *path != '/') { - auto group = router.match(host, "/", 1); +size_t match_downstream_addr_group_host( + const Router &router, const std::string &host, const StringRef &path, + const std::vector &groups, size_t catch_all) { + if (path.empty() || path[0] != '/') { + auto group = router.match(host, StringRef::from_lit("/")); if (group != -1) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Found pattern with query " << host @@ -2522,23 +2520,22 @@ match_downstream_addr_group_host(const Router &router, const std::string &host, if (LOG_ENABLED(INFO)) { LOG(INFO) << "Perform mapping selection, using host=" << host - << ", path=" << std::string(path, pathlen); + << ", path=" << path; } - auto group = router.match(host, path, pathlen); + auto group = router.match(host, path); if (group != -1) { if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Found pattern with query " << host - << std::string(path, pathlen) + LOG(INFO) << "Found pattern with query " << host << path << ", matched pattern=" << groups[group].pattern; } return group; } - group = router.match("", path, pathlen); + group = router.match("", path); if (group != -1) { if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Found pattern with query " << std::string(path, pathlen) + LOG(INFO) << "Found pattern with query " << path << ", matched pattern=" << groups[group].pattern; } return group; @@ -2565,12 +2562,11 @@ match_downstream_addr_group(const Router &router, const std::string &hostport, auto fragment = std::find(std::begin(raw_path), std::end(raw_path), '#'); auto query = std::find(std::begin(raw_path), fragment, '?'); - auto path = raw_path.c_str(); - auto pathlen = query - std::begin(raw_path); + auto path = StringRef{std::begin(raw_path), query}; if (hostport.empty()) { - return match_downstream_addr_group_host(router, hostport, path, pathlen, - groups, catch_all); + return match_downstream_addr_group_host(router, hostport, path, groups, + catch_all); } std::string host; @@ -2593,7 +2589,7 @@ match_downstream_addr_group(const Router &router, const std::string &hostport, } util::inp_strlower(host); - return match_downstream_addr_group_host(router, host, path, pathlen, groups, + return match_downstream_addr_group_host(router, host, path, groups, catch_all); } diff --git a/src/shrpx_router.cc b/src/shrpx_router.cc index 4324748a..2975c3e9 100644 --- a/src/shrpx_router.cc +++ b/src/shrpx_router.cc @@ -259,8 +259,7 @@ const RNode *match_partial(const RNode *node, size_t offset, const char *first, } } // namespace -ssize_t Router::match(const std::string &host, const char *path, - size_t pathlen) const { +ssize_t Router::match(const std::string &host, const StringRef &path) const { const RNode *node; size_t offset; @@ -270,7 +269,7 @@ ssize_t Router::match(const std::string &host, const char *path, return -1; } - node = match_partial(node, offset, path, path + pathlen); + node = match_partial(node, offset, std::begin(path), std::end(path)); if (node == nullptr || node == &root_) { return -1; } diff --git a/src/shrpx_router.h b/src/shrpx_router.h index e4d9a465..5f70b212 100644 --- a/src/shrpx_router.h +++ b/src/shrpx_router.h @@ -58,8 +58,7 @@ public: // Adds route |pattern| with its |index|. bool add_route(const StringRef &pattern, size_t index); // Returns the matched index of pattern. -1 if there is no match. - ssize_t match(const std::string &host, const char *path, - size_t pathlen) const; + ssize_t match(const std::string &host, const StringRef &path) const; void add_node(RNode *node, const char *pattern, size_t patlen, size_t index); diff --git a/src/template.h b/src/template.h index 3061ad52..253ea215 100644 --- a/src/template.h +++ b/src/template.h @@ -397,6 +397,9 @@ public: : base(reinterpret_cast(s)), len(n) {} template StringRef(InputIt first, InputIt last) + : base(&*first), len(std::distance(first, last)) {} + template + StringRef(InputIt *first, InputIt *last) : base(first), len(std::distance(first, last)) {} template constexpr static StringRef from_lit(const CharT(&s)[N]) { From 49fa914db5cbf74708e025f9aeae74f9a79f2ab1 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 20:48:06 +0900 Subject: [PATCH 014/147] nghttpx: Use StringRef for string parameters in match_downstream_addr_group --- src/shrpx_client_handler.cc | 15 +++++++++------ src/shrpx_config.cc | 12 +++++------- src/shrpx_config.h | 2 +- src/shrpx_router.cc | 5 ++--- src/shrpx_router.h | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index fa814dd9..4d063647 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -681,16 +681,19 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { } else { auto &router = get_config()->router; if (!req.authority.empty()) { - group = match_downstream_addr_group(router, req.authority, req.path, - groups, catch_all); + group = + match_downstream_addr_group(router, StringRef{req.authority}, + StringRef{req.path}, groups, catch_all); } else { auto h = req.fs.header(http2::HD_HOST); if (h) { - group = match_downstream_addr_group(router, h->value, req.path, groups, - catch_all); + group = + match_downstream_addr_group(router, StringRef{h->value}, + StringRef{req.path}, groups, catch_all); } else { - group = match_downstream_addr_group(router, "", req.path, groups, - catch_all); + group = + match_downstream_addr_group(router, StringRef::from_lit(""), + StringRef{req.path}, groups, catch_all); } } } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index a5d9f534..5e6236c3 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2504,7 +2504,7 @@ int int_syslog_facility(const char *strfacility) { namespace { size_t match_downstream_addr_group_host( - const Router &router, const std::string &host, const StringRef &path, + const Router &router, const StringRef &host, const StringRef &path, const std::vector &groups, size_t catch_all) { if (path.empty() || path[0] != '/') { auto group = router.match(host, StringRef::from_lit("/")); @@ -2548,11 +2548,9 @@ size_t match_downstream_addr_group_host( } } // namespace -size_t -match_downstream_addr_group(const Router &router, const std::string &hostport, - const std::string &raw_path, - const std::vector &groups, - size_t catch_all) { +size_t match_downstream_addr_group( + const Router &router, const StringRef &hostport, const StringRef &raw_path, + const std::vector &groups, size_t catch_all) { if (std::find(std::begin(hostport), std::end(hostport), '/') != std::end(hostport)) { // We use '/' specially, and if '/' is included in host, it breaks @@ -2589,7 +2587,7 @@ match_downstream_addr_group(const Router &router, const std::string &hostport, } util::inp_strlower(host); - return match_downstream_addr_group_host(router, host, path, groups, + return match_downstream_addr_group_host(router, StringRef{host}, path, groups, catch_all); } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index e024317a..b81f8e1d 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -652,7 +652,7 @@ read_tls_ticket_key_file(const std::vector &files, // group. The catch-all group index is given in |catch_all|. All // patterns are given in |groups|. size_t match_downstream_addr_group( - const Router &router, const std::string &hostport, const std::string &path, + const Router &router, const StringRef &hostport, const StringRef &path, const std::vector &groups, size_t catch_all); } // namespace shrpx diff --git a/src/shrpx_router.cc b/src/shrpx_router.cc index 2975c3e9..4c88283f 100644 --- a/src/shrpx_router.cc +++ b/src/shrpx_router.cc @@ -259,12 +259,11 @@ const RNode *match_partial(const RNode *node, size_t offset, const char *first, } } // namespace -ssize_t Router::match(const std::string &host, const StringRef &path) const { +ssize_t Router::match(const StringRef &host, const StringRef &path) const { const RNode *node; size_t offset; - node = - match_complete(&offset, &root_, host.c_str(), host.c_str() + host.size()); + node = match_complete(&offset, &root_, std::begin(host), std::end(host)); if (node == nullptr) { return -1; } diff --git a/src/shrpx_router.h b/src/shrpx_router.h index 5f70b212..ece0e276 100644 --- a/src/shrpx_router.h +++ b/src/shrpx_router.h @@ -58,7 +58,7 @@ public: // Adds route |pattern| with its |index|. bool add_route(const StringRef &pattern, size_t index); // Returns the matched index of pattern. -1 if there is no match. - ssize_t match(const std::string &host, const StringRef &path) const; + ssize_t match(const StringRef &host, const StringRef &path) const; void add_node(RNode *node, const char *pattern, size_t patlen, size_t index); From bfc26e82997aa86460eedd182abc1a3814d5d107 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 20:59:10 +0900 Subject: [PATCH 015/147] nghttpx: Use ImmutableString to store memcached server host --- src/shrpx.cc | 16 ++++++++-------- src/shrpx_config.cc | 4 ++-- src/shrpx_config.h | 8 ++++++-- src/shrpx_ssl.cc | 2 +- src/shrpx_worker.cc | 4 ++-- src/shrpx_worker_process.cc | 4 ++-- 6 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index ee05263b..33272f97 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2220,10 +2220,10 @@ void process_options( { auto &memcachedconf = tlsconf.session_cache.memcached; - if (memcachedconf.host) { - auto hostport = - util::make_hostport(memcachedconf.host.get(), memcachedconf.port); - if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.get(), + if (!memcachedconf.host.empty()) { + auto hostport = util::make_hostport(StringRef{memcachedconf.host}, + memcachedconf.port); + if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.c_str(), memcachedconf.port, memcachedconf.family) == -1) { LOG(FATAL) << "Resolving memcached address for TLS session cache failed: " @@ -2238,10 +2238,10 @@ void process_options( { auto &memcachedconf = tlsconf.ticket.memcached; - if (memcachedconf.host) { - auto hostport = - util::make_hostport(memcachedconf.host.get(), memcachedconf.port); - if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.get(), + if (!memcachedconf.host.empty()) { + auto hostport = util::make_hostport(StringRef{memcachedconf.host}, + memcachedconf.port); + if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.c_str(), memcachedconf.port, memcachedconf.family) == -1) { LOG(FATAL) << "Resolving memcached address for TLS ticket key failed: " << hostport; diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 5e6236c3..20d30c86 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2154,7 +2154,7 @@ int parse_config(const char *opt, const char *optarg, } auto &memcachedconf = mod_config()->tls.session_cache.memcached; - memcachedconf.host = strcopy(host); + memcachedconf.host = host; memcachedconf.port = port; return 0; @@ -2166,7 +2166,7 @@ int parse_config(const char *opt, const char *optarg, } auto &memcachedconf = mod_config()->tls.ticket.memcached; - memcachedconf.host = strcopy(host); + memcachedconf.host = host; memcachedconf.port = port; return 0; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index b81f8e1d..e5a21241 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -343,7 +343,9 @@ struct TLSConfig { struct { Address addr; uint16_t port; - std::unique_ptr host; + // Hostname of memcached server. This is also used as SNI field + // if TLS is enabled. + ImmutableString host; // Client private key and certificate for authentication ImmutableString private_key_file; ImmutableString cert_file; @@ -370,7 +372,9 @@ struct TLSConfig { struct { Address addr; uint16_t port; - std::unique_ptr host; + // Hostname of memcached server. This is also used as SNI field + // if TLS is enabled. + ImmutableString host; // Client private key and certificate for authentication ImmutableString private_key_file; ImmutableString cert_file; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 5d8a2360..afbcbb15 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -485,7 +485,7 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file SSL_CTX_set_session_id_context(ssl_ctx, sid_ctx, sizeof(sid_ctx) - 1); SSL_CTX_set_session_cache_mode(ssl_ctx, SSL_SESS_CACHE_SERVER); - if (tlsconf.session_cache.memcached.host) { + if (!tlsconf.session_cache.memcached.host.empty()) { SSL_CTX_sess_set_new_cb(ssl_ctx, tls_session_new_cb); SSL_CTX_sess_set_get_cb(ssl_ctx, tls_session_get_cb); } diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index e663eace..83cfff9b 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -91,11 +91,11 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, auto &session_cacheconf = get_config()->tls.session_cache; - if (session_cacheconf.memcached.host) { + if (!session_cacheconf.memcached.host.empty()) { session_cache_memcached_dispatcher_ = make_unique( &session_cacheconf.memcached.addr, loop, tls_session_cache_memcached_ssl_ctx, - session_cacheconf.memcached.host.get(), &mcpool_); + StringRef{session_cacheconf.memcached.host}, &mcpool_); } auto &downstreamconf = get_config()->conn.downstream; diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc index 26863d82..f7dacc7c 100644 --- a/src/shrpx_worker_process.cc +++ b/src/shrpx_worker_process.cc @@ -427,7 +427,7 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { auto &ticketconf = get_config()->tls.ticket; auto &memcachedconf = ticketconf.memcached; - if (ticketconf.memcached.host) { + if (!memcachedconf.host.empty()) { SSL_CTX *ssl_ctx = nullptr; if (memcachedconf.tls) { @@ -437,7 +437,7 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { conn_handler.set_tls_ticket_key_memcached_dispatcher( make_unique( &ticketconf.memcached.addr, loop, ssl_ctx, - StringRef(memcachedconf.host.get()), &mcpool)); + StringRef{memcachedconf.host}, &mcpool)); ev_timer_init(&renew_ticket_key_timer, memcached_get_ticket_key_cb, 0., 0.); From 660bc389e6f8276fe0b508f890f3e3ce4567fbd5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:01:54 +0900 Subject: [PATCH 016/147] nghttpx: Use ImmutableString for fetch_ocsp_response_file --- src/shrpx.cc | 11 +++++------ src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- src/shrpx_connection_handler.cc | 3 ++- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 33272f97..44c4094e 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1067,8 +1067,7 @@ void fill_default_config() { auto &ocspconf = tlsconf.ocsp; // ocsp update interval = 14400 secs = 4 hours, borrowed from h2o ocspconf.update_interval = 4_h; - ocspconf.fetch_ocsp_response_file = - strcopy(PKGDATADIR "/fetch-ocsp-response"); + ocspconf.fetch_ocsp_response_file = PKGDATADIR "/fetch-ocsp-response"; } { @@ -1579,8 +1578,8 @@ SSL/TLS: --fetch-ocsp-response-file= Path to fetch-ocsp-response script file. It should be absolute path. - Default: )" - << get_config()->tls.ocsp.fetch_ocsp_response_file.get() << R"( + Default: )" << get_config()->tls.ocsp.fetch_ocsp_response_file + << R"( --ocsp-update-interval= Set interval to update OCSP response cache. Default: )" @@ -2094,10 +2093,10 @@ void process_options( if (!upstreamconf.no_tls && !tlsconf.ocsp.disabled) { struct stat buf; - if (stat(tlsconf.ocsp.fetch_ocsp_response_file.get(), &buf) != 0) { + if (stat(tlsconf.ocsp.fetch_ocsp_response_file.c_str(), &buf) != 0) { tlsconf.ocsp.disabled = true; LOG(WARN) << "--fetch-ocsp-response-file: " - << tlsconf.ocsp.fetch_ocsp_response_file.get() + << tlsconf.ocsp.fetch_ocsp_response_file << " not found. OCSP stapling has been disabled."; } } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 20d30c86..6afcc47b 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2086,7 +2086,7 @@ int parse_config(const char *opt, const char *optarg, return parse_uint(&mod_config()->http2.downstream.connections_per_worker, opt, optarg); case SHRPX_OPTID_FETCH_OCSP_RESPONSE_FILE: - mod_config()->tls.ocsp.fetch_ocsp_response_file = strcopy(optarg); + mod_config()->tls.ocsp.fetch_ocsp_response_file = optarg; return 0; case SHRPX_OPTID_OCSP_UPDATE_INTERVAL: diff --git a/src/shrpx_config.h b/src/shrpx_config.h index e5a21241..d290a37a 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -394,7 +394,7 @@ struct TLSConfig { // OCSP realted configurations struct { ev_tstamp update_interval; - std::unique_ptr fetch_ocsp_response_file; + ImmutableString fetch_ocsp_response_file; bool disabled; } ocsp; diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 588e814b..8dea4f38 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -463,7 +463,8 @@ int ConnectionHandler::start_ocsp_update(const char *cert_file) { assert(!ev_is_active(&ocsp_.chldev)); char *const argv[] = { - const_cast(get_config()->tls.ocsp.fetch_ocsp_response_file.get()), + const_cast( + get_config()->tls.ocsp.fetch_ocsp_response_file.c_str()), const_cast(cert_file), nullptr}; char *const envp[] = {nullptr}; From 52f641781321f15453bd0d890a3db25e5bf96c3e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:06:16 +0900 Subject: [PATCH 017/147] nghttpx: Use ImmutableString for tls.cacert --- src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- src/shrpx_connection_handler.cc | 15 ++++++--------- src/shrpx_ssl.cc | 2 +- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 6afcc47b..fc3cdce8 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1816,7 +1816,7 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_CACERT: - mod_config()->tls.cacert = strcopy(optarg); + mod_config()->tls.cacert = optarg; return 0; case SHRPX_OPTID_BACKEND_IPV4: diff --git a/src/shrpx_config.h b/src/shrpx_config.h index d290a37a..8326cbf3 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -431,7 +431,7 @@ struct TLSConfig { std::unique_ptr cert_file; std::unique_ptr dh_param_file; std::unique_ptr ciphers; - std::unique_ptr cacert; + ImmutableString cacert; bool insecure; bool no_http2_cipher_black_list; }; diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 8dea4f38..008f1289 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -202,9 +202,8 @@ int ConnectionHandler::create_single_worker() { #ifdef HAVE_NEVERBLEED nb_.get(), #endif // HAVE_NEVERBLEED - StringRef::from_maybe_nullptr(tlsconf.cacert.get()), - StringRef(memcachedconf.cert_file), - StringRef(memcachedconf.private_key_file), StringRef(), nullptr); + StringRef{tlsconf.cacert}, StringRef{memcachedconf.cert_file}, + StringRef{memcachedconf.private_key_file}, StringRef(), nullptr); all_ssl_ctx_.push_back(session_cache_ssl_ctx); } @@ -253,9 +252,8 @@ int ConnectionHandler::create_worker_thread(size_t num) { #ifdef HAVE_NEVERBLEED nb_.get(), #endif // HAVE_NEVERBLEED - StringRef::from_maybe_nullptr(tlsconf.cacert.get()), - StringRef(memcachedconf.cert_file), - StringRef(memcachedconf.private_key_file), StringRef(), nullptr); + StringRef{tlsconf.cacert}, StringRef{memcachedconf.cert_file}, + StringRef{memcachedconf.private_key_file}, StringRef{}, nullptr); all_ssl_ctx_.push_back(session_cache_ssl_ctx); } auto worker = @@ -768,9 +766,8 @@ SSL_CTX *ConnectionHandler::create_tls_ticket_key_memcached_ssl_ctx() { #ifdef HAVE_NEVERBLEED nb_.get(), #endif // HAVE_NEVERBLEED - StringRef::from_maybe_nullptr(tlsconf.cacert.get()), - StringRef(memcachedconf.cert_file), - StringRef(memcachedconf.private_key_file), StringRef(), nullptr); + StringRef{tlsconf.cacert}, StringRef{memcachedconf.cert_file}, + StringRef{memcachedconf.private_key_file}, StringRef{}, nullptr); all_ssl_ctx_.push_back(ssl_ctx); diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index afbcbb15..bfba5a57 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -1323,7 +1323,7 @@ SSL_CTX *setup_downstream_client_ssl_context( #ifdef HAVE_NEVERBLEED nb, #endif // HAVE_NEVERBLEED - StringRef::from_maybe_nullptr(tlsconf.cacert.get()), + StringRef{tlsconf.cacert}, StringRef::from_maybe_nullptr(tlsconf.client.cert_file.get()), StringRef::from_maybe_nullptr(tlsconf.client.private_key_file.get()), alpn, next_proto_select_cb); From 529a59d309f40c9e0570a6c2be6332b182b53826 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:09:15 +0900 Subject: [PATCH 018/147] nghttpx: Use ImmutableString for tls.client_verify.cacert --- src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- src/shrpx_ssl.cc | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index fc3cdce8..506391a7 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1907,7 +1907,7 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_VERIFY_CLIENT_CACERT: - mod_config()->tls.client_verify.cacert = strcopy(optarg); + mod_config()->tls.client_verify.cacert = optarg; return 0; case SHRPX_OPTID_CLIENT_PRIVATE_KEY_FILE: diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 8326cbf3..5ea8cc62 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -402,7 +402,7 @@ struct TLSConfig { struct { // Path to file containing CA certificate solely used for client // certificate validation - std::unique_ptr cacert; + ImmutableString cacert; bool enabled; } client_verify; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index bfba5a57..d6fa26a7 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -579,12 +579,12 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file DIE(); } if (tlsconf.client_verify.enabled) { - if (tlsconf.client_verify.cacert) { + if (!tlsconf.client_verify.cacert.empty()) { if (SSL_CTX_load_verify_locations( - ssl_ctx, tlsconf.client_verify.cacert.get(), nullptr) != 1) { + ssl_ctx, tlsconf.client_verify.cacert.c_str(), nullptr) != 1) { LOG(FATAL) << "Could not load trusted ca certificates from " - << tlsconf.client_verify.cacert.get() << ": " + << tlsconf.client_verify.cacert << ": " << ERR_error_string(ERR_get_error(), nullptr); DIE(); } @@ -592,10 +592,10 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file // error even though it returns success. See // http://forum.nginx.org/read.php?29,242540 ERR_clear_error(); - auto list = SSL_load_client_CA_file(tlsconf.client_verify.cacert.get()); + auto list = SSL_load_client_CA_file(tlsconf.client_verify.cacert.c_str()); if (!list) { LOG(FATAL) << "Could not load ca certificates from " - << tlsconf.client_verify.cacert.get() << ": " + << tlsconf.client_verify.cacert << ": " << ERR_error_string(ERR_get_error(), nullptr); DIE(); } From c999987baf2b4c0a53bbab4ce50ed00f2cb3e66f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:13:46 +0900 Subject: [PATCH 019/147] nghttpx: Use ImmutableString for private_key_file --- src/shrpx.cc | 2 +- src/shrpx_config.cc | 4 ++-- src/shrpx_config.h | 4 ++-- src/shrpx_ssl.cc | 5 ++--- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 44c4094e..369c1a78 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2085,7 +2085,7 @@ void process_options( } if (!upstreamconf.no_tls && - (!tlsconf.private_key_file || !tlsconf.cert_file)) { + (tlsconf.private_key_file.empty() || !tlsconf.cert_file)) { print_usage(std::cerr); LOG(FATAL) << "Too few arguments"; exit(EXIT_FAILURE); diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 506391a7..10d9d97b 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1745,7 +1745,7 @@ int parse_config(const char *opt, const char *optarg, return 0; } case SHRPX_OPTID_PRIVATE_KEY_FILE: - mod_config()->tls.private_key_file = strcopy(optarg); + mod_config()->tls.private_key_file = optarg; return 0; case SHRPX_OPTID_PRIVATE_KEY_PASSWD_FILE: { @@ -1911,7 +1911,7 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_CLIENT_PRIVATE_KEY_FILE: - mod_config()->tls.client.private_key_file = strcopy(optarg); + mod_config()->tls.client.private_key_file = optarg; return 0; case SHRPX_OPTID_CLIENT_CERT_FILE: diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 5ea8cc62..4b6b514d 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -408,7 +408,7 @@ struct TLSConfig { // Client private key and certificate used in backend connections. struct { - std::unique_ptr private_key_file; + ImmutableString private_key_file; std::unique_ptr cert_file; } client; @@ -426,7 +426,7 @@ struct TLSConfig { long int tls_proto_mask; std::string backend_sni_name; std::chrono::seconds session_timeout; - std::unique_ptr private_key_file; + ImmutableString private_key_file; std::unique_ptr private_key_passwd; std::unique_ptr cert_file; std::unique_ptr dh_param_file; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index d6fa26a7..c5e8b79f 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -1245,7 +1245,7 @@ SSL_CTX *setup_server_ssl_context(std::vector &all_ssl_ctx, auto &tlsconf = get_config()->tls; - auto ssl_ctx = ssl::create_ssl_context(tlsconf.private_key_file.get(), + auto ssl_ctx = ssl::create_ssl_context(tlsconf.private_key_file.c_str(), tlsconf.cert_file.get() #ifdef HAVE_NEVERBLEED , @@ -1325,8 +1325,7 @@ SSL_CTX *setup_downstream_client_ssl_context( #endif // HAVE_NEVERBLEED StringRef{tlsconf.cacert}, StringRef::from_maybe_nullptr(tlsconf.client.cert_file.get()), - StringRef::from_maybe_nullptr(tlsconf.client.private_key_file.get()), - alpn, next_proto_select_cb); + StringRef{tlsconf.client.private_key_file}, alpn, next_proto_select_cb); } CertLookupTree *create_cert_lookup_tree() { From ac81003669deb40ede9b5559b9c832b43a787d71 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:17:25 +0900 Subject: [PATCH 020/147] nghttpx: Use ImmutableString for cert_file --- src/shrpx.cc | 2 +- src/shrpx_config.cc | 4 ++-- src/shrpx_config.h | 4 ++-- src/shrpx_ssl.cc | 9 ++++----- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 369c1a78..17adf31d 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2085,7 +2085,7 @@ void process_options( } if (!upstreamconf.no_tls && - (tlsconf.private_key_file.empty() || !tlsconf.cert_file)) { + (tlsconf.private_key_file.empty() || tlsconf.cert_file.empty())) { print_usage(std::cerr); LOG(FATAL) << "Too few arguments"; exit(EXIT_FAILURE); diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 10d9d97b..12e01978 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1759,7 +1759,7 @@ int parse_config(const char *opt, const char *optarg, return 0; } case SHRPX_OPTID_CERTIFICATE_FILE: - mod_config()->tls.cert_file = strcopy(optarg); + mod_config()->tls.cert_file = optarg; return 0; case SHRPX_OPTID_DH_PARAM_FILE: @@ -1915,7 +1915,7 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_CLIENT_CERT_FILE: - mod_config()->tls.client.cert_file = strcopy(optarg); + mod_config()->tls.client.cert_file = optarg; return 0; case SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER: diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 4b6b514d..52f12ca1 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -409,7 +409,7 @@ struct TLSConfig { // Client private key and certificate used in backend connections. struct { ImmutableString private_key_file; - std::unique_ptr cert_file; + ImmutableString cert_file; } client; // The list of (private key file, certificate file) pair @@ -428,7 +428,7 @@ struct TLSConfig { std::chrono::seconds session_timeout; ImmutableString private_key_file; std::unique_ptr private_key_passwd; - std::unique_ptr cert_file; + ImmutableString cert_file; std::unique_ptr dh_param_file; std::unique_ptr ciphers; ImmutableString cacert; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index c5e8b79f..adb80e47 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -1246,7 +1246,7 @@ SSL_CTX *setup_server_ssl_context(std::vector &all_ssl_ctx, auto &tlsconf = get_config()->tls; auto ssl_ctx = ssl::create_ssl_context(tlsconf.private_key_file.c_str(), - tlsconf.cert_file.get() + tlsconf.cert_file.c_str() #ifdef HAVE_NEVERBLEED , nb @@ -1281,8 +1281,8 @@ SSL_CTX *setup_server_ssl_context(std::vector &all_ssl_ctx, } } - if (ssl::cert_lookup_tree_add_cert_from_file(cert_tree, ssl_ctx, - tlsconf.cert_file.get()) == -1) { + if (ssl::cert_lookup_tree_add_cert_from_file( + cert_tree, ssl_ctx, tlsconf.cert_file.c_str()) == -1) { LOG(FATAL) << "Failed to add default certificate."; DIE(); } @@ -1323,8 +1323,7 @@ SSL_CTX *setup_downstream_client_ssl_context( #ifdef HAVE_NEVERBLEED nb, #endif // HAVE_NEVERBLEED - StringRef{tlsconf.cacert}, - StringRef::from_maybe_nullptr(tlsconf.client.cert_file.get()), + StringRef{tlsconf.cacert}, StringRef{tlsconf.client.cert_file}, StringRef{tlsconf.client.private_key_file}, alpn, next_proto_select_cb); } From 35ebdd35bc6bb9ccc98686fb8b432db181d2ae84 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:20:40 +0900 Subject: [PATCH 021/147] nghttpx: Use ImmutableString for private_key_file --- src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- src/shrpx_ssl.cc | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 12e01978..0682149f 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1754,7 +1754,7 @@ int parse_config(const char *opt, const char *optarg, LOG(ERROR) << opt << ": Couldn't read key file's passwd from " << optarg; return -1; } - mod_config()->tls.private_key_passwd = strcopy(passwd); + mod_config()->tls.private_key_passwd = passwd; return 0; } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 52f12ca1..3f5a8565 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -427,7 +427,7 @@ struct TLSConfig { std::string backend_sni_name; std::chrono::seconds session_timeout; ImmutableString private_key_file; - std::unique_ptr private_key_passwd; + ImmutableString private_key_passwd; ImmutableString cert_file; std::unique_ptr dh_param_file; std::unique_ptr ciphers; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index adb80e47..f5d2b594 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -124,13 +124,13 @@ set_alpn_prefs(const std::vector &protos) { namespace { int ssl_pem_passwd_cb(char *buf, int size, int rwflag, void *user_data) { auto config = static_cast(user_data); - int len = (int)strlen(config->tls.private_key_passwd.get()); + auto len = static_cast(config->tls.private_key_passwd.size()); if (size < len + 1) { LOG(ERROR) << "ssl_pem_passwd_cb: buf is too small " << size; return 0; } // Copy string including last '\0'. - memcpy(buf, config->tls.private_key_passwd.get(), len + 1); + memcpy(buf, config->tls.private_key_passwd.c_str(), len + 1); return len; } } // namespace @@ -548,7 +548,7 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file SSL_CTX_set_mode(ssl_ctx, SSL_MODE_AUTO_RETRY); SSL_CTX_set_mode(ssl_ctx, SSL_MODE_RELEASE_BUFFERS); - if (tlsconf.private_key_passwd) { + if (!tlsconf.private_key_passwd.empty()) { SSL_CTX_set_default_passwd_cb(ssl_ctx, ssl_pem_passwd_cb); SSL_CTX_set_default_passwd_cb_userdata(ssl_ctx, (void *)get_config()); } From 2344932b45e5b22b37651ab2b432f0bb2504ba06 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:22:28 +0900 Subject: [PATCH 022/147] nghttpx: Use ImmutableString for dh_param_file --- src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- src/shrpx_ssl.cc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 0682149f..24e89d05 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1763,7 +1763,7 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_DH_PARAM_FILE: - mod_config()->tls.dh_param_file = strcopy(optarg); + mod_config()->tls.dh_param_file = optarg; return 0; case SHRPX_OPTID_SUBCERT: { diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 3f5a8565..dbe8c9a5 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -429,7 +429,7 @@ struct TLSConfig { ImmutableString private_key_file; ImmutableString private_key_passwd; ImmutableString cert_file; - std::unique_ptr dh_param_file; + ImmutableString dh_param_file; std::unique_ptr ciphers; ImmutableString cacert; bool insecure; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index f5d2b594..a7e00086 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -527,9 +527,9 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file #endif // OPENSSL_NO_EC - if (tlsconf.dh_param_file) { + if (!tlsconf.dh_param_file.empty()) { // Read DH parameters from file - auto bio = BIO_new_file(tlsconf.dh_param_file.get(), "r"); + auto bio = BIO_new_file(tlsconf.dh_param_file.c_str(), "r"); if (bio == nullptr) { LOG(FATAL) << "BIO_new_file() failed: " << ERR_error_string(ERR_get_error(), nullptr); From 67804cfc8c4818953dbcaf5ed5d17d49692d5ff3 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:25:24 +0900 Subject: [PATCH 023/147] nghttpx: Use ImmutableString for ciphers --- src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- src/shrpx_ssl.cc | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 24e89d05..d99ca91a 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1804,7 +1804,7 @@ int parse_config(const char *opt, const char *optarg, return 0; } case SHRPX_OPTID_CIPHERS: - mod_config()->tls.ciphers = strcopy(optarg); + mod_config()->tls.ciphers = optarg; return 0; case SHRPX_OPTID_CLIENT: diff --git a/src/shrpx_config.h b/src/shrpx_config.h index dbe8c9a5..d8c63fd4 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -430,7 +430,7 @@ struct TLSConfig { ImmutableString private_key_passwd; ImmutableString cert_file; ImmutableString dh_param_file; - std::unique_ptr ciphers; + ImmutableString ciphers; ImmutableString cacert; bool insecure; bool no_http2_cipher_black_list; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index a7e00086..aff475e6 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -493,8 +493,8 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file SSL_CTX_set_timeout(ssl_ctx, tlsconf.session_timeout.count()); const char *ciphers; - if (tlsconf.ciphers) { - ciphers = tlsconf.ciphers.get(); + if (!tlsconf.ciphers.empty()) { + ciphers = tlsconf.ciphers.c_str(); } else { ciphers = nghttp2::ssl::DEFAULT_CIPHER_LIST; } @@ -683,8 +683,8 @@ SSL_CTX *create_ssl_client_context( SSL_CTX_set_options(ssl_ctx, ssl_opts | tlsconf.tls_proto_mask); const char *ciphers; - if (tlsconf.ciphers) { - ciphers = tlsconf.ciphers.get(); + if (!tlsconf.ciphers.empty()) { + ciphers = tlsconf.ciphers.c_str(); } else { ciphers = nghttp2::ssl::DEFAULT_CIPHER_LIST; } From 9055323b67590adb4cbb941adddc05fddefd1dd2 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:28:47 +0900 Subject: [PATCH 024/147] nghttpx: Use ImmutableString for request_header_file and response_header_file --- src/shrpx.cc | 8 ++++---- src/shrpx_config.cc | 6 ++---- src/shrpx_config.h | 4 ++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 17adf31d..71af4447 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1986,8 +1986,8 @@ void process_options( { auto &dumpconf = http2conf.upstream.debug.dump; - if (dumpconf.request_header_file) { - auto path = dumpconf.request_header_file.get(); + if (!dumpconf.request_header_file.empty()) { + auto path = dumpconf.request_header_file.c_str(); auto f = open_file_for_write(path); if (f == nullptr) { @@ -2007,8 +2007,8 @@ void process_options( } } - if (dumpconf.response_header_file) { - auto path = dumpconf.response_header_file.get(); + if (!dumpconf.response_header_file.empty()) { + auto path = dumpconf.response_header_file.c_str(); auto f = open_file_for_write(path); if (f == nullptr) { diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index d99ca91a..c16a6132 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1919,13 +1919,11 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER: - mod_config()->http2.upstream.debug.dump.request_header_file = - strcopy(optarg); + mod_config()->http2.upstream.debug.dump.request_header_file = optarg; return 0; case SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER: - mod_config()->http2.upstream.debug.dump.response_header_file = - strcopy(optarg); + mod_config()->http2.upstream.debug.dump.response_header_file = optarg; return 0; case SHRPX_OPTID_HTTP2_NO_COOKIE_CRUMBLING: diff --git a/src/shrpx_config.h b/src/shrpx_config.h index d8c63fd4..93d55ade 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -473,8 +473,8 @@ struct Http2Config { struct { struct { struct { - std::unique_ptr request_header_file; - std::unique_ptr response_header_file; + ImmutableString request_header_file; + ImmutableString response_header_file; FILE *request_header; FILE *response_header; } dump; From 2b707bff27dc377ef8f0a8d60ffc4c09edadf815 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:32:27 +0900 Subject: [PATCH 025/147] nghttpx: Use ImmutableString for log file --- src/shrpx.cc | 4 ++-- src/shrpx_config.cc | 4 ++-- src/shrpx_config.h | 4 ++-- src/shrpx_log.cc | 14 +++++++------- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 71af4447..a914a656 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1121,7 +1121,7 @@ void fill_default_config() { accessconf.format = parse_log_format(DEFAULT_ACCESSLOG_FORMAT); auto &errorconf = loggingconf.error; - errorconf.file = strcopy("/dev/stderr"); + errorconf.file = "/dev/stderr"; } loggingconf.syslog_facility = LOG_DAEMON; @@ -1752,7 +1752,7 @@ Logging: Set path to write error log. To reopen file, send USR1 signal to nghttpx. stderr will be redirected to the error log file unless --errorlog-syslog is used. - Default: )" << get_config()->logging.error.file.get() << R"( + Default: )" << get_config()->logging.error.file << R"( --errorlog-syslog Send error log to syslog. If this option is used, --errorlog-file option is ignored. diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index c16a6132..2c092148 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1622,7 +1622,7 @@ int parse_config(const char *opt, const char *optarg, return parse_duration(&mod_config()->http2.timeout.stream_write, opt, optarg); case SHRPX_OPTID_ACCESSLOG_FILE: - mod_config()->logging.access.file = strcopy(optarg); + mod_config()->logging.access.file = optarg; return 0; case SHRPX_OPTID_ACCESSLOG_SYSLOG: @@ -1634,7 +1634,7 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_ERRORLOG_FILE: - mod_config()->logging.error.file = strcopy(optarg); + mod_config()->logging.error.file = optarg; return 0; case SHRPX_OPTID_ERRORLOG_SYSLOG: diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 93d55ade..04faed86 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -504,12 +504,12 @@ struct Http2Config { struct LoggingConfig { struct { std::vector format; - std::unique_ptr file; + ImmutableString file; // Send accesslog to syslog, ignoring accesslog_file. bool syslog; } access; struct { - std::unique_ptr file; + ImmutableString file; // Send errorlog to syslog, ignoring errorlog_file. bool syslog; } error; diff --git a/src/shrpx_log.cc b/src/shrpx_log.cc index 261b02e5..ae699e8b 100644 --- a/src/shrpx_log.cc +++ b/src/shrpx_log.cc @@ -393,23 +393,23 @@ int reopen_log_files() { auto &accessconf = get_config()->logging.access; auto &errorconf = get_config()->logging.error; - if (!accessconf.syslog && accessconf.file) { - new_accesslog_fd = util::open_log_file(accessconf.file.get()); + if (!accessconf.syslog && !accessconf.file.empty()) { + new_accesslog_fd = util::open_log_file(accessconf.file.c_str()); if (new_accesslog_fd == -1) { - LOG(ERROR) << "Failed to open accesslog file " << accessconf.file.get(); + LOG(ERROR) << "Failed to open accesslog file " << accessconf.file; res = -1; } } - if (!errorconf.syslog && errorconf.file) { - new_errorlog_fd = util::open_log_file(errorconf.file.get()); + if (!errorconf.syslog && !errorconf.file.empty()) { + new_errorlog_fd = util::open_log_file(errorconf.file.c_str()); if (new_errorlog_fd == -1) { if (lgconf->errorlog_fd != -1) { - LOG(ERROR) << "Failed to open errorlog file " << errorconf.file.get(); + LOG(ERROR) << "Failed to open errorlog file " << errorconf.file; } else { - std::cerr << "Failed to open errorlog file " << errorconf.file.get() + std::cerr << "Failed to open errorlog file " << errorconf.file << std::endl; } From 76a425226ffcfdd88d8b0009b2cd847fc16e65e2 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:34:33 +0900 Subject: [PATCH 026/147] nghttpx: Use ImmutableString for pid_file --- src/shrpx.cc | 10 +++++----- src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index a914a656..8508da4d 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -199,18 +199,18 @@ int chown_to_running_user(const char *path) { namespace { void save_pid() { - std::ofstream out(get_config()->pid_file.get(), std::ios::binary); + std::ofstream out(get_config()->pid_file.c_str(), std::ios::binary); out << get_config()->pid << "\n"; out.close(); if (!out) { - LOG(ERROR) << "Could not save PID to file " << get_config()->pid_file.get(); + LOG(ERROR) << "Could not save PID to file " << get_config()->pid_file; exit(EXIT_FAILURE); } if (get_config()->uid != 0) { - if (chown_to_running_user(get_config()->pid_file.get()) == -1) { + if (chown_to_running_user(get_config()->pid_file.c_str()) == -1) { auto error = errno; - LOG(WARN) << "Changing owner of pid file " << get_config()->pid_file.get() + LOG(WARN) << "Changing owner of pid file " << get_config()->pid_file << " failed: " << strerror(error); } } @@ -946,7 +946,7 @@ int event_loop() { redirect_stderr_to_errorlog(); } - if (get_config()->pid_file) { + if (!get_config()->pid_file.empty()) { save_pid(); } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 2c092148..8d23b9ac 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1728,7 +1728,7 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_PID_FILE: - mod_config()->pid_file = strcopy(optarg); + mod_config()->pid_file = optarg; return 0; case SHRPX_OPTID_USER: { diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 04faed86..3caa7aeb 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -581,7 +581,7 @@ struct Config { TLSConfig tls; LoggingConfig logging; ConnectionConfig conn; - std::unique_ptr pid_file; + ImmutableString pid_file; std::unique_ptr conf_path; std::unique_ptr user; std::unique_ptr mruby_file; From 466e4b7a1e171b7b183922b7e8a27a61ca3a293d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 22:20:10 +0900 Subject: [PATCH 027/147] nghttpx: Use ImmutableString for conf_path --- src/shrpx.cc | 12 ++++++------ src/shrpx_config.h | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 8508da4d..0c1fb3f7 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1040,7 +1040,7 @@ void fill_default_config() { *mod_config() = {}; mod_config()->num_worker = 1; - mod_config()->conf_path = strcopy("/etc/nghttpx/nghttpx.conf"); + mod_config()->conf_path = "/etc/nghttpx/nghttpx.conf"; mod_config()->pid = getpid(); auto &tlsconf = mod_config()->tls; @@ -1893,7 +1893,7 @@ Scripting: Misc: --conf= Load configuration from . - Default: )" << get_config()->conf_path.get() << R"( + Default: )" << get_config()->conf_path << R"( --include= Load additional configurations from . File is read when configuration parser encountered this @@ -1919,11 +1919,11 @@ namespace { void process_options( int argc, char **argv, std::vector> &cmdcfgs) { - if (conf_exists(get_config()->conf_path.get())) { + if (conf_exists(get_config()->conf_path.c_str())) { std::set include_set; - if (load_config(get_config()->conf_path.get(), include_set) == -1) { + if (load_config(get_config()->conf_path.c_str(), include_set) == -1) { LOG(FATAL) << "Failed to load configuration from " - << get_config()->conf_path.get(); + << get_config()->conf_path; exit(EXIT_FAILURE); } assert(include_set.empty()); @@ -2563,7 +2563,7 @@ int main(int argc, char **argv) { break; case 12: // --conf - mod_config()->conf_path = strcopy(optarg); + mod_config()->conf_path = optarg; break; case 14: // --syslog-facility diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 3caa7aeb..c7a3ef59 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -582,7 +582,7 @@ struct Config { LoggingConfig logging; ConnectionConfig conn; ImmutableString pid_file; - std::unique_ptr conf_path; + ImmutableString conf_path; std::unique_ptr user; std::unique_ptr mruby_file; char **original_argv; From 7aabc6b125279fea2a60c23f329651fe5a323a58 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 22:21:55 +0900 Subject: [PATCH 028/147] nghttpx: Use ImmutableString for user --- src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- src/shrpx_worker_process.cc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 8d23b9ac..1e023d18 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1738,7 +1738,7 @@ int parse_config(const char *opt, const char *optarg, << strerror(errno); return -1; } - mod_config()->user = strcopy(pwd->pw_name); + mod_config()->user = pwd->pw_name; mod_config()->uid = pwd->pw_uid; mod_config()->gid = pwd->pw_gid; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index c7a3ef59..03b3700b 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -583,7 +583,7 @@ struct Config { ConnectionConfig conn; ImmutableString pid_file; ImmutableString conf_path; - std::unique_ptr user; + ImmutableString user; std::unique_ptr mruby_file; char **original_argv; char **argv; diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc index f7dacc7c..8d09b5d2 100644 --- a/src/shrpx_worker_process.cc +++ b/src/shrpx_worker_process.cc @@ -64,7 +64,7 @@ void drop_privileges( #endif // HAVE_NEVERBLEED ) { if (getuid() == 0 && get_config()->uid != 0) { - if (initgroups(get_config()->user.get(), get_config()->gid) != 0) { + if (initgroups(get_config()->user.c_str(), get_config()->gid) != 0) { auto error = errno; LOG(FATAL) << "Could not change supplementary groups: " << strerror(error); @@ -86,7 +86,7 @@ void drop_privileges( } #ifdef HAVE_NEVERBLEED if (nb) { - neverbleed_setuidgid(nb, get_config()->user.get(), 1); + neverbleed_setuidgid(nb, get_config()->user.c_str(), 1); } #endif // HAVE_NEVERBLEED } From aa3373a107a2e9a96afe10929db4773a88745f90 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 22:27:59 +0900 Subject: [PATCH 029/147] nghttpx: Use ImmutableString for mruby_file --- src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- src/shrpx_mruby.cc | 11 +++++------ src/shrpx_mruby.h | 6 ++++-- src/shrpx_worker.cc | 3 +-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 1e023d18..4d95ffe2 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2205,7 +2205,7 @@ int parse_config(const char *opt, const char *optarg, case SHRPX_OPTID_MRUBY_FILE: #ifdef HAVE_MRUBY - mod_config()->mruby_file = strcopy(optarg); + mod_config()->mruby_file = optarg; #else // !HAVE_MRUBY LOG(WARN) << opt << ": ignored because mruby support is disabled at build time."; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 03b3700b..80f0e06e 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -584,7 +584,7 @@ struct Config { ImmutableString pid_file; ImmutableString conf_path; ImmutableString user; - std::unique_ptr mruby_file; + ImmutableString mruby_file; char **original_argv; char **argv; char *cwd; diff --git a/src/shrpx_mruby.cc b/src/shrpx_mruby.cc index 5b1b3967..bd57c146 100644 --- a/src/shrpx_mruby.cc +++ b/src/shrpx_mruby.cc @@ -31,7 +31,6 @@ #include "shrpx_config.h" #include "shrpx_mruby_module.h" #include "shrpx_downstream_connection.h" -#include "template.h" namespace shrpx { @@ -146,12 +145,12 @@ mrb_value instantiate_app(mrb_state *mrb, RProc *proc) { // very hard to write these kind of code because mruby has almost no // documentation aobut compiling or generating code, at least at the // time of this writing. -RProc *compile(mrb_state *mrb, const char *filename) { - if (filename == nullptr) { +RProc *compile(mrb_state *mrb, const StringRef &filename) { + if (filename.empty()) { return nullptr; } - auto infile = fopen(filename, "rb"); + auto infile = fopen(filename.c_str(), "rb"); if (infile == nullptr) { return nullptr; } @@ -185,8 +184,8 @@ RProc *compile(mrb_state *mrb, const char *filename) { return proc; } -std::unique_ptr create_mruby_context(const char *filename) { - if (!filename) { +std::unique_ptr create_mruby_context(const StringRef &filename) { + if (filename.empty()) { return make_unique(nullptr, mrb_nil_value(), mrb_nil_value()); } diff --git a/src/shrpx_mruby.h b/src/shrpx_mruby.h index 95f0430c..55a18351 100644 --- a/src/shrpx_mruby.h +++ b/src/shrpx_mruby.h @@ -32,6 +32,8 @@ #include #include +#include "template.h" + using namespace nghttp2; namespace shrpx { @@ -69,9 +71,9 @@ struct MRubyAssocData { bool response_headers_dirty; }; -RProc *compile(mrb_state *mrb, const char *filename); +RProc *compile(mrb_state *mrb, const StringRef &filename); -std::unique_ptr create_mruby_context(const char *filename); +std::unique_ptr create_mruby_context(const StringRef &filename); // Return interned |ptr|. mrb_sym intern_ptr(mrb_state *mrb, void *ptr); diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 83cfff9b..604f9058 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -290,8 +290,7 @@ std::mt19937 &Worker::get_randgen() { return randgen_; } #ifdef HAVE_MRUBY int Worker::create_mruby_context() { - auto mruby_file = get_config()->mruby_file.get(); - mruby_ctx_ = mruby::create_mruby_context(mruby_file); + mruby_ctx_ = mruby::create_mruby_context(StringRef{get_config()->mruby_file}); if (!mruby_ctx_) { return -1; } From 63e43bff9983ed95fa3953950aeeff6e8e627457 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Mon, 15 Feb 2016 10:20:13 +0100 Subject: [PATCH 030/147] tests: remove unused macros Since v0.6.2-7-g1d138ac ("Unify DATA and other frames in nghttp2_outbound_item and save malloc()"), the macros are unused and the builds fails on -Werror=unused-macros. --- tests/nghttp2_session_test.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/nghttp2_session_test.c b/tests/nghttp2_session_test.c index bd07b822..0cdec93f 100644 --- a/tests/nghttp2_session_test.c +++ b/tests/nghttp2_session_test.c @@ -36,10 +36,6 @@ #include "nghttp2_test_helper.h" #include "nghttp2_priority_spec.h" -#define OB_CTRL(ITEM) nghttp2_outbound_item_get_ctrl_frame(ITEM) -#define OB_CTRL_TYPE(ITEM) nghttp2_outbound_item_get_ctrl_frame_type(ITEM) -#define OB_DATA(ITEM) nghttp2_outbound_item_get_data_frame(ITEM) - typedef struct { uint8_t buf[65535]; size_t length; From f0b5a8db8c50e03bde30be96ed6172e220533411 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 17 Feb 2016 00:26:34 +0900 Subject: [PATCH 031/147] Add unistd.h to test initgroups() declaration --- configure.ac | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 985925d7..844695c5 100644 --- a/configure.ac +++ b/configure.ac @@ -654,8 +654,13 @@ AC_CHECK_FUNC([timerfd_create], # For cygwin: we can link initgroups, so AC_CHECK_FUNCS succeeds, but # cygwin disables initgroups due to feature test macro magic with our -# configuration. -AC_CHECK_DECLS([initgroups], [], [], [[#include ]]) +# configuration. FreeBSD declares initgroups() in unistd.h. +AC_CHECK_DECLS([initgroups], [], [], [[ + #ifdef HAVE_UNISTD_H + # include + #endif + #include +]]) # Checks for epoll availability, primarily for examples/tiny-nghttpd AX_HAVE_EPOLL([have_epoll=yes], [have_epoll=no]) From b7159f80b20432d510d43d9c8b57cf3f2b39159b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 18 Feb 2016 23:56:29 +0900 Subject: [PATCH 032/147] Eliminate the possibility of nghttp2_stream.cycle overflow --- lib/nghttp2_stream.c | 42 +++++++++++++++++++++++++++++++----------- lib/nghttp2_stream.h | 4 ++-- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/lib/nghttp2_stream.c b/lib/nghttp2_stream.c index 3e848d8e..70325bc3 100644 --- a/lib/nghttp2_stream.c +++ b/lib/nghttp2_stream.c @@ -30,14 +30,32 @@ #include "nghttp2_session.h" #include "nghttp2_helper.h" +/* Maximum distance between any two stream's cycle in the same + prirority queue. Imagine stream A's cycle is A, and stream B's + cycle is B, and A < B. The cycle is unsigned 32 bit integer, it + may get overflow. Because of how we calculate the next cycle + value, if B - A is less than or equals to + NGHTTP2_MAX_CYCLE_DISTANCE, A and B are in the same scale, in other + words, B is really greater than or equal to A. Otherwise, A is a + result of overflow, and it is actually A > B if we consider that + fact. */ +#define NGHTTP2_MAX_CYCLE_DISTANCE (16384 * 256 + 255) + static int stream_less(const void *lhsx, const void *rhsx) { const nghttp2_stream *lhs, *rhs; lhs = nghttp2_struct_of(lhsx, nghttp2_stream, pq_entry); rhs = nghttp2_struct_of(rhsx, nghttp2_stream, pq_entry); - return lhs->cycle < rhs->cycle || - (lhs->cycle == rhs->cycle && lhs->seq < rhs->seq); + if (lhs->cycle == rhs->cycle) { + return lhs->seq < rhs->seq; + } + + if (lhs->cycle < rhs->cycle) { + return rhs->cycle - lhs->cycle <= NGHTTP2_MAX_CYCLE_DISTANCE; + } + + return lhs->cycle - rhs->cycle > NGHTTP2_MAX_CYCLE_DISTANCE; } void nghttp2_stream_init(nghttp2_stream *stream, int32_t stream_id, @@ -116,14 +134,14 @@ static int stream_subtree_active(nghttp2_stream *stream) { /* * Returns next cycle for |stream|. */ -static void stream_next_cycle(nghttp2_stream *stream, uint64_t last_cycle) { - size_t penalty; +static void stream_next_cycle(nghttp2_stream *stream, uint32_t last_cycle) { + uint32_t penalty; - penalty = - stream->last_writelen * NGHTTP2_MAX_WEIGHT + stream->pending_penalty; + penalty = (uint32_t)stream->last_writelen * NGHTTP2_MAX_WEIGHT + + stream->pending_penalty; stream->cycle = last_cycle + penalty / (uint32_t)stream->weight; - stream->pending_penalty = (uint32_t)(penalty % (uint32_t)stream->weight); + stream->pending_penalty = penalty % (uint32_t)stream->weight; } static int stream_obq_push(nghttp2_stream *dep_stream, nghttp2_stream *stream) { @@ -229,9 +247,9 @@ void nghttp2_stream_reschedule(nghttp2_stream *stream) { void nghttp2_stream_change_weight(nghttp2_stream *stream, int32_t weight) { nghttp2_stream *dep_stream; - uint64_t last_cycle; + uint32_t last_cycle; int32_t old_weight; - size_t wlen_penalty; + uint32_t wlen_penalty; if (stream->weight == weight) { return; @@ -254,7 +272,7 @@ void nghttp2_stream_change_weight(nghttp2_stream *stream, int32_t weight) { nghttp2_pq_remove(&dep_stream->obq, &stream->pq_entry); - wlen_penalty = stream->last_writelen * NGHTTP2_MAX_WEIGHT; + wlen_penalty = (uint32_t)stream->last_writelen * NGHTTP2_MAX_WEIGHT; /* Compute old stream->pending_penalty we used to calculate stream->cycle */ @@ -270,7 +288,9 @@ void nghttp2_stream_change_weight(nghttp2_stream *stream, int32_t weight) { place */ stream_next_cycle(stream, last_cycle); - if (stream->cycle < dep_stream->descendant_last_cycle) { + if (stream->cycle < dep_stream->descendant_last_cycle && + (dep_stream->descendant_last_cycle - stream->cycle) <= + NGHTTP2_MAX_CYCLE_DISTANCE) { stream->cycle = dep_stream->descendant_last_cycle; } diff --git a/lib/nghttp2_stream.h b/lib/nghttp2_stream.h index b5d07592..da0e5d53 100644 --- a/lib/nghttp2_stream.h +++ b/lib/nghttp2_stream.h @@ -147,9 +147,9 @@ struct nghttp2_stream { /* Received body so far */ int64_t recv_content_length; /* Base last_cycle for direct descendent streams. */ - uint64_t descendant_last_cycle; + uint32_t descendant_last_cycle; /* Next scheduled time to sent item */ - uint64_t cycle; + uint32_t cycle; /* Next seq used for direct descendant streams */ uint64_t descendant_next_seq; /* Secondary key for prioritization to break a tie for cycle. This From 7921029e33b6270b488e391fc5e860b5cd36194b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 20 Feb 2016 00:48:18 +0900 Subject: [PATCH 033/147] Tokenize extra HTTP header fields The extra HTTP header fields are compiled from https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers, https://en.wikipedia.org/wiki/List_of_HTTP_header_fields, and https://www.owasp.org/index.php/List_of_useful_HTTP_headers. --- genlibtokenlookup.py | 60 ++++++++- lib/nghttp2_hd.c | 306 ++++++++++++++++++++++++++++++++++++++++++- lib/nghttp2_hd.h | 60 ++++++++- 3 files changed, 417 insertions(+), 9 deletions(-) diff --git a/genlibtokenlookup.py b/genlibtokenlookup.py index d7ac7f47..625e62d1 100755 --- a/genlibtokenlookup.py +++ b/genlibtokenlookup.py @@ -62,11 +62,67 @@ HEADERS = [ ('vary', 58), ('via', 59), ('www-authenticate', 60), - ('te', None), + ('accept-ch', None), + ('accept-datetime', None), + ('accept-features', None), + ('accept-patch', None), + ('access-control-allow-credentials', None), + ('access-control-allow-headers', None), + ('access-control-allow-methods', None), + ('access-control-expose-headers', None), + ('access-control-max-age', None), + ('access-control-request-headers', None), + ('access-control-request-method', None), + ('alt-svc', None), + ('alternates', None), ('connection', None), - ('keep-alive',None), + ('content-md5', None), + ('content-security-policy', None), + ('content-security-policy-report-only', None), + ('dnt', None), + ('forwarded', None), + ('front-end-https', None), + ('keep-alive', None), + ('last-event-id', None), + ('negotiate', None), + ('origin', None), + ('p3p', None), + ('pragma', None), ('proxy-connection', None), + ('public-key-pins', None), + ('sec-websocket-extensions', None), + ('sec-websocket-key', None), + ('sec-websocket-origin', None), + ('sec-websocket-protocol', None), + ('sec-websocket-version', None), + ('set-cookie2', None), + ('status', None), + ('tcn', None), + ('te', None), + ('trailer', None), + ('tsv', None), ('upgrade', None), + ('upgrade-insecure-requests', None), + ('variant-vary', None), + ('warning', None), + ('x-api-version', None), + ('x-att-deviceid', None), + ('x-cache', None), + ('x-cache-lookup', None), + ('x-content-duration', None), + ('x-content-security-policy', None), + ('x-content-type-options', None), + ('x-dnsprefetch-control', None), + ('x-forwarded-for', None), + ('x-forwarded-host', None), + ('x-forwarded-proto', None), + ('x-frame-options', None), + ('x-powered-by', None), + ('x-requested-with', None), + ('x-ua-compatible', None), + ('x-wap-profile', None), + ('x-webkit-csp', None), + ('x-xss-protection', None), ] def to_enum_hd(k): diff --git a/lib/nghttp2_hd.c b/lib/nghttp2_hd.c index 5a02c0f4..65a09a5b 100644 --- a/lib/nghttp2_hd.c +++ b/lib/nghttp2_hd.c @@ -137,6 +137,26 @@ static int lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_AGE; } break; + case 'n': + if (lstreq("tc", name, 2)) { + return NGHTTP2_TOKEN_TCN; + } + break; + case 'p': + if (lstreq("p3", name, 2)) { + return NGHTTP2_TOKEN_P3P; + } + break; + case 't': + if (lstreq("dn", name, 2)) { + return NGHTTP2_TOKEN_DNT; + } + break; + case 'v': + if (lstreq("ts", name, 2)) { + return NGHTTP2_TOKEN_TSV; + } + break; } break; case 4: @@ -197,16 +217,31 @@ static int lookup_token(const uint8_t *name, size_t namelen) { break; case 6: switch (name[5]) { + case 'a': + if (lstreq("pragm", name, 5)) { + return NGHTTP2_TOKEN_PRAGMA; + } + break; case 'e': if (lstreq("cooki", name, 5)) { return NGHTTP2_TOKEN_COOKIE; } break; + case 'n': + if (lstreq("origi", name, 5)) { + return NGHTTP2_TOKEN_ORIGIN; + } + break; case 'r': if (lstreq("serve", name, 5)) { return NGHTTP2_TOKEN_SERVER; } break; + case 's': + if (lstreq("statu", name, 5)) { + return NGHTTP2_TOKEN_STATUS; + } + break; case 't': if (lstreq("accep", name, 5)) { return NGHTTP2_TOKEN_ACCEPT; @@ -219,6 +254,11 @@ static int lookup_token(const uint8_t *name, size_t namelen) { break; case 7: switch (name[6]) { + case 'c': + if (lstreq("alt-sv", name, 6)) { + return NGHTTP2_TOKEN_ALT_SVC; + } + break; case 'd': if (lstreq(":metho", name, 6)) { return NGHTTP2_TOKEN__METHOD; @@ -237,6 +277,14 @@ static int lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("upgrad", name, 6)) { return NGHTTP2_TOKEN_UPGRADE; } + if (lstreq("x-cach", name, 6)) { + return NGHTTP2_TOKEN_X_CACHE; + } + break; + case 'g': + if (lstreq("warnin", name, 6)) { + return NGHTTP2_TOKEN_WARNING; + } break; case 'h': if (lstreq("refres", name, 6)) { @@ -247,6 +295,9 @@ static int lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("refere", name, 6)) { return NGHTTP2_TOKEN_REFERER; } + if (lstreq("traile", name, 6)) { + return NGHTTP2_TOKEN_TRAILER; + } break; case 's': if (lstreq(":statu", name, 6)) { @@ -295,6 +346,25 @@ static int lookup_token(const uint8_t *name, size_t namelen) { break; } break; + case 9: + switch (name[8]) { + case 'd': + if (lstreq("forwarde", name, 8)) { + return NGHTTP2_TOKEN_FORWARDED; + } + break; + case 'e': + if (lstreq("negotiat", name, 8)) { + return NGHTTP2_TOKEN_NEGOTIATE; + } + break; + case 'h': + if (lstreq("accept-c", name, 8)) { + return NGHTTP2_TOKEN_ACCEPT_CH; + } + break; + } + break; case 10: switch (name[9]) { case 'e': @@ -310,6 +380,11 @@ static int lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_CONNECTION; } break; + case 's': + if (lstreq("alternate", name, 9)) { + return NGHTTP2_TOKEN_ALTERNATES; + } + break; case 't': if (lstreq("user-agen", name, 9)) { return NGHTTP2_TOKEN_USER_AGENT; @@ -324,6 +399,16 @@ static int lookup_token(const uint8_t *name, size_t namelen) { break; case 11: switch (name[10]) { + case '2': + if (lstreq("set-cookie", name, 10)) { + return NGHTTP2_TOKEN_SET_COOKIE2; + } + break; + case '5': + if (lstreq("content-md", name, 10)) { + return NGHTTP2_TOKEN_CONTENT_MD5; + } + break; case 'r': if (lstreq("retry-afte", name, 10)) { return NGHTTP2_TOKEN_RETRY_AFTER; @@ -338,16 +423,37 @@ static int lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_CONTENT_TYPE; } break; + case 'h': + if (lstreq("accept-patc", name, 11)) { + return NGHTTP2_TOKEN_ACCEPT_PATCH; + } + break; + case 'p': + if (lstreq("x-webkit-cs", name, 11)) { + return NGHTTP2_TOKEN_X_WEBKIT_CSP; + } + break; case 's': if (lstreq("max-forward", name, 11)) { return NGHTTP2_TOKEN_MAX_FORWARDS; } break; + case 'y': + if (lstreq("variant-var", name, 11)) { + return NGHTTP2_TOKEN_VARIANT_VARY; + } + if (lstreq("x-powered-b", name, 11)) { + return NGHTTP2_TOKEN_X_POWERED_BY; + } + break; } break; case 13: switch (name[12]) { case 'd': + if (lstreq("last-event-i", name, 12)) { + return NGHTTP2_TOKEN_LAST_EVENT_ID; + } if (lstreq("last-modifie", name, 12)) { return NGHTTP2_TOKEN_LAST_MODIFIED; } @@ -356,6 +462,9 @@ static int lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("content-rang", name, 12)) { return NGHTTP2_TOKEN_CONTENT_RANGE; } + if (lstreq("x-wap-profil", name, 12)) { + return NGHTTP2_TOKEN_X_WAP_PROFILE; + } break; case 'h': if (lstreq("if-none-matc", name, 12)) { @@ -371,6 +480,9 @@ static int lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("authorizatio", name, 12)) { return NGHTTP2_TOKEN_AUTHORIZATION; } + if (lstreq("x-api-versio", name, 12)) { + return NGHTTP2_TOKEN_X_API_VERSION; + } break; case 's': if (lstreq("accept-range", name, 12)) { @@ -381,11 +493,21 @@ static int lookup_token(const uint8_t *name, size_t namelen) { break; case 14: switch (name[13]) { + case 'd': + if (lstreq("x-att-devicei", name, 13)) { + return NGHTTP2_TOKEN_X_ATT_DEVICEID; + } + break; case 'h': if (lstreq("content-lengt", name, 13)) { return NGHTTP2_TOKEN_CONTENT_LENGTH; } break; + case 'p': + if (lstreq("x-cache-looku", name, 13)) { + return NGHTTP2_TOKEN_X_CACHE_LOOKUP; + } + break; case 't': if (lstreq("accept-charse", name, 13)) { return NGHTTP2_TOKEN_ACCEPT_CHARSET; @@ -396,15 +518,40 @@ static int lookup_token(const uint8_t *name, size_t namelen) { case 15: switch (name[14]) { case 'e': + if (lstreq("accept-datetim", name, 14)) { + return NGHTTP2_TOKEN_ACCEPT_DATETIME; + } if (lstreq("accept-languag", name, 14)) { return NGHTTP2_TOKEN_ACCEPT_LANGUAGE; } + if (lstreq("x-ua-compatibl", name, 14)) { + return NGHTTP2_TOKEN_X_UA_COMPATIBLE; + } break; case 'g': if (lstreq("accept-encodin", name, 14)) { return NGHTTP2_TOKEN_ACCEPT_ENCODING; } break; + case 'r': + if (lstreq("x-forwarded-fo", name, 14)) { + return NGHTTP2_TOKEN_X_FORWARDED_FOR; + } + break; + case 's': + if (lstreq("accept-feature", name, 14)) { + return NGHTTP2_TOKEN_ACCEPT_FEATURES; + } + if (lstreq("front-end-http", name, 14)) { + return NGHTTP2_TOKEN_FRONT_END_HTTPS; + } + if (lstreq("public-key-pin", name, 14)) { + return NGHTTP2_TOKEN_PUBLIC_KEY_PINS; + } + if (lstreq("x-frame-option", name, 14)) { + return NGHTTP2_TOKEN_X_FRAME_OPTIONS; + } + break; } break; case 16: @@ -422,6 +569,11 @@ static int lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_CONTENT_ENCODING; } break; + case 'h': + if (lstreq("x-requested-wit", name, 15)) { + return NGHTTP2_TOKEN_X_REQUESTED_WITH; + } + break; case 'n': if (lstreq("content-locatio", name, 15)) { return NGHTTP2_TOKEN_CONTENT_LOCATION; @@ -429,6 +581,14 @@ static int lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("proxy-connectio", name, 15)) { return NGHTTP2_TOKEN_PROXY_CONNECTION; } + if (lstreq("x-xss-protectio", name, 15)) { + return NGHTTP2_TOKEN_X_XSS_PROTECTION; + } + break; + case 't': + if (lstreq("x-forwarded-hos", name, 15)) { + return NGHTTP2_TOKEN_X_FORWARDED_HOST; + } break; } break; @@ -444,6 +604,16 @@ static int lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_TRANSFER_ENCODING; } break; + case 'o': + if (lstreq("x-forwarded-prot", name, 16)) { + return NGHTTP2_TOKEN_X_FORWARDED_PROTO; + } + break; + case 'y': + if (lstreq("sec-websocket-ke", name, 16)) { + return NGHTTP2_TOKEN_SEC_WEBSOCKET_KEY; + } + break; } break; case 18: @@ -453,6 +623,11 @@ static int lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_PROXY_AUTHENTICATE; } break; + case 'n': + if (lstreq("x-content-duratio", name, 17)) { + return NGHTTP2_TOKEN_X_CONTENT_DURATION; + } + break; } break; case 19: @@ -472,12 +647,80 @@ static int lookup_token(const uint8_t *name, size_t namelen) { break; } break; + case 20: + switch (name[19]) { + case 'n': + if (lstreq("sec-websocket-origi", name, 19)) { + return NGHTTP2_TOKEN_SEC_WEBSOCKET_ORIGIN; + } + break; + } + break; + case 21: + switch (name[20]) { + case 'l': + if (lstreq("x-dnsprefetch-contro", name, 20)) { + return NGHTTP2_TOKEN_X_DNSPREFETCH_CONTROL; + } + break; + case 'n': + if (lstreq("sec-websocket-versio", name, 20)) { + return NGHTTP2_TOKEN_SEC_WEBSOCKET_VERSION; + } + break; + } + break; + case 22: + switch (name[21]) { + case 'e': + if (lstreq("access-control-max-ag", name, 21)) { + return NGHTTP2_TOKEN_ACCESS_CONTROL_MAX_AGE; + } + break; + case 'l': + if (lstreq("sec-websocket-protoco", name, 21)) { + return NGHTTP2_TOKEN_SEC_WEBSOCKET_PROTOCOL; + } + break; + case 's': + if (lstreq("x-content-type-option", name, 21)) { + return NGHTTP2_TOKEN_X_CONTENT_TYPE_OPTIONS; + } + break; + } + break; + case 23: + switch (name[22]) { + case 'y': + if (lstreq("content-security-polic", name, 22)) { + return NGHTTP2_TOKEN_CONTENT_SECURITY_POLICY; + } + break; + } + break; + case 24: + switch (name[23]) { + case 's': + if (lstreq("sec-websocket-extension", name, 23)) { + return NGHTTP2_TOKEN_SEC_WEBSOCKET_EXTENSIONS; + } + break; + } + break; case 25: switch (name[24]) { + case 's': + if (lstreq("upgrade-insecure-request", name, 24)) { + return NGHTTP2_TOKEN_UPGRADE_INSECURE_REQUESTS; + } + break; case 'y': if (lstreq("strict-transport-securit", name, 24)) { return NGHTTP2_TOKEN_STRICT_TRANSPORT_SECURITY; } + if (lstreq("x-content-security-polic", name, 24)) { + return NGHTTP2_TOKEN_X_CONTENT_SECURITY_POLICY; + } break; } break; @@ -490,6 +733,59 @@ static int lookup_token(const uint8_t *name, size_t namelen) { break; } break; + case 28: + switch (name[27]) { + case 's': + if (lstreq("access-control-allow-header", name, 27)) { + return NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_HEADERS; + } + if (lstreq("access-control-allow-method", name, 27)) { + return NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_METHODS; + } + break; + } + break; + case 29: + switch (name[28]) { + case 'd': + if (lstreq("access-control-request-metho", name, 28)) { + return NGHTTP2_TOKEN_ACCESS_CONTROL_REQUEST_METHOD; + } + break; + case 's': + if (lstreq("access-control-expose-header", name, 28)) { + return NGHTTP2_TOKEN_ACCESS_CONTROL_EXPOSE_HEADERS; + } + break; + } + break; + case 30: + switch (name[29]) { + case 's': + if (lstreq("access-control-request-header", name, 29)) { + return NGHTTP2_TOKEN_ACCESS_CONTROL_REQUEST_HEADERS; + } + break; + } + break; + case 32: + switch (name[31]) { + case 's': + if (lstreq("access-control-allow-credential", name, 31)) { + return NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_CREDENTIALS; + } + break; + } + break; + case 35: + switch (name[34]) { + case 'y': + if (lstreq("content-security-policy-report-onl", name, 34)) { + return NGHTTP2_TOKEN_CONTENT_SECURITY_POLICY_REPORT_ONLY; + } + break; + } + break; } return -1; } @@ -617,8 +913,8 @@ static nghttp2_hd_entry *hd_map_find(nghttp2_hd_map *map, int *exact_match, *exact_match = 0; for (p = map->table[hash & (HD_MAP_SIZE - 1)]; p; p = p->next) { - if (hash != p->hash || token != p->token || - (token == -1 && !name_eq(&p->nv, nv))) { + if (token != p->token || + (token == -1 && (hash != p->hash || !name_eq(&p->nv, nv)))) { continue; } if (!res) { @@ -1444,7 +1740,7 @@ static int deflate_nv(nghttp2_hd_deflater *deflater, nghttp2_bufs *bufs, int indexing_mode; int token; nghttp2_mem *mem; - uint32_t hash; + uint32_t hash = 0; DEBUGF(fprintf(stderr, "deflatehd: deflating %.*s: %.*s\n", (int)nv->namelen, nv->name, (int)nv->valuelen, nv->value)); @@ -1452,9 +1748,9 @@ static int deflate_nv(nghttp2_hd_deflater *deflater, nghttp2_bufs *bufs, mem = deflater->ctx.mem; token = lookup_token(nv->name, nv->namelen); - if (token == -1 || token > NGHTTP2_TOKEN_WWW_AUTHENTICATE) { + if (token == -1) { hash = name_hash(nv); - } else { + } else if (token <= NGHTTP2_TOKEN_WWW_AUTHENTICATE) { hash = static_table[token].hash; } diff --git a/lib/nghttp2_hd.h b/lib/nghttp2_hd.h index c667888e..adf227be 100644 --- a/lib/nghttp2_hd.h +++ b/lib/nghttp2_hd.h @@ -105,11 +105,67 @@ typedef enum { NGHTTP2_TOKEN_VARY = 58, NGHTTP2_TOKEN_VIA = 59, NGHTTP2_TOKEN_WWW_AUTHENTICATE = 60, - NGHTTP2_TOKEN_TE, + NGHTTP2_TOKEN_ACCEPT_CH, + NGHTTP2_TOKEN_ACCEPT_DATETIME, + NGHTTP2_TOKEN_ACCEPT_FEATURES, + NGHTTP2_TOKEN_ACCEPT_PATCH, + NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_CREDENTIALS, + NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_HEADERS, + NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_METHODS, + NGHTTP2_TOKEN_ACCESS_CONTROL_EXPOSE_HEADERS, + NGHTTP2_TOKEN_ACCESS_CONTROL_MAX_AGE, + NGHTTP2_TOKEN_ACCESS_CONTROL_REQUEST_HEADERS, + NGHTTP2_TOKEN_ACCESS_CONTROL_REQUEST_METHOD, + NGHTTP2_TOKEN_ALT_SVC, + NGHTTP2_TOKEN_ALTERNATES, NGHTTP2_TOKEN_CONNECTION, + NGHTTP2_TOKEN_CONTENT_MD5, + NGHTTP2_TOKEN_CONTENT_SECURITY_POLICY, + NGHTTP2_TOKEN_CONTENT_SECURITY_POLICY_REPORT_ONLY, + NGHTTP2_TOKEN_DNT, + NGHTTP2_TOKEN_FORWARDED, + NGHTTP2_TOKEN_FRONT_END_HTTPS, NGHTTP2_TOKEN_KEEP_ALIVE, + NGHTTP2_TOKEN_LAST_EVENT_ID, + NGHTTP2_TOKEN_NEGOTIATE, + NGHTTP2_TOKEN_ORIGIN, + NGHTTP2_TOKEN_P3P, + NGHTTP2_TOKEN_PRAGMA, NGHTTP2_TOKEN_PROXY_CONNECTION, - NGHTTP2_TOKEN_UPGRADE + NGHTTP2_TOKEN_PUBLIC_KEY_PINS, + NGHTTP2_TOKEN_SEC_WEBSOCKET_EXTENSIONS, + NGHTTP2_TOKEN_SEC_WEBSOCKET_KEY, + NGHTTP2_TOKEN_SEC_WEBSOCKET_ORIGIN, + NGHTTP2_TOKEN_SEC_WEBSOCKET_PROTOCOL, + NGHTTP2_TOKEN_SEC_WEBSOCKET_VERSION, + NGHTTP2_TOKEN_SET_COOKIE2, + NGHTTP2_TOKEN_STATUS, + NGHTTP2_TOKEN_TCN, + NGHTTP2_TOKEN_TE, + NGHTTP2_TOKEN_TRAILER, + NGHTTP2_TOKEN_TSV, + NGHTTP2_TOKEN_UPGRADE, + NGHTTP2_TOKEN_UPGRADE_INSECURE_REQUESTS, + NGHTTP2_TOKEN_VARIANT_VARY, + NGHTTP2_TOKEN_WARNING, + NGHTTP2_TOKEN_X_API_VERSION, + NGHTTP2_TOKEN_X_ATT_DEVICEID, + NGHTTP2_TOKEN_X_CACHE, + NGHTTP2_TOKEN_X_CACHE_LOOKUP, + NGHTTP2_TOKEN_X_CONTENT_DURATION, + NGHTTP2_TOKEN_X_CONTENT_SECURITY_POLICY, + NGHTTP2_TOKEN_X_CONTENT_TYPE_OPTIONS, + NGHTTP2_TOKEN_X_DNSPREFETCH_CONTROL, + NGHTTP2_TOKEN_X_FORWARDED_FOR, + NGHTTP2_TOKEN_X_FORWARDED_HOST, + NGHTTP2_TOKEN_X_FORWARDED_PROTO, + NGHTTP2_TOKEN_X_FRAME_OPTIONS, + NGHTTP2_TOKEN_X_POWERED_BY, + NGHTTP2_TOKEN_X_REQUESTED_WITH, + NGHTTP2_TOKEN_X_UA_COMPATIBLE, + NGHTTP2_TOKEN_X_WAP_PROFILE, + NGHTTP2_TOKEN_X_WEBKIT_CSP, + NGHTTP2_TOKEN_X_XSS_PROTECTION, } nghttp2_token; typedef enum { From 17032010845f257a830ec6ba07961c5e75527d5b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 20 Feb 2016 19:02:19 +0900 Subject: [PATCH 034/147] nghttpx: Get rid of hdidx --- src/shrpx_downstream.cc | 81 +++++++++++++------------ src/shrpx_downstream.h | 6 +- src/shrpx_http_downstream_connection.cc | 4 +- src/shrpx_https_upstream.cc | 4 +- src/shrpx_spdy_upstream.cc | 5 +- 5 files changed, 51 insertions(+), 49 deletions(-) diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 8f8165b7..71a0a2b6 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -237,15 +237,15 @@ void Downstream::force_resume_read() { } namespace { -const Headers::value_type *search_header_linear(const Headers &headers, - const StringRef &name) { - const Headers::value_type *res = nullptr; - for (auto &kv : headers) { +const Headers::value_type * +search_header_linear_backwards(const Headers &headers, const StringRef &name) { + for (auto it = headers.rbegin(); it != headers.rend(); ++it) { + auto &kv = *it; if (kv.name == name) { - res = &kv; + return &kv; } } - return res; + return nullptr; } } // namespace @@ -330,10 +330,10 @@ void Downstream::crumble_request_cookie(std::vector &nva) { namespace { void add_header(bool &key_prev, size_t &sum, Headers &headers, std::string name, - std::string value) { + std::string value, int16_t token = -1) { key_prev = true; sum += name.size() + value.size(); - headers.emplace_back(std::move(name), std::move(value)); + headers.emplace_back(std::move(name), std::move(value), false, token); } } // namespace @@ -356,6 +356,8 @@ void append_last_header_key(bool &key_prev, size_t &sum, Headers &headers, sum += len; auto &item = headers.back(); item.name.append(data, len); + util::inp_strlower(item.name); + item.token = http2::lookup_token(item.name); } } // namespace @@ -370,56 +372,58 @@ void append_last_header_value(bool &key_prev, size_t &sum, Headers &headers, } // namespace int FieldStore::index_headers() { - http2::init_hdidx(hdidx_); content_length = -1; - for (size_t i = 0; i < headers_.size(); ++i) { - auto &kv = headers_[i]; - util::inp_strlower(kv.name); - - auto token = http2::lookup_token( - reinterpret_cast(kv.name.c_str()), kv.name.size()); - if (token < 0) { + for (auto &kv : headers_) { + if (kv.token != http2::HD_CONTENT_LENGTH) { continue; } - kv.token = token; - http2::index_header(hdidx_, token, i); - - if (token == http2::HD_CONTENT_LENGTH) { - auto len = util::parse_uint(kv.value); - if (len == -1) { - return -1; - } - if (content_length != -1) { - return -1; - } - content_length = len; + auto len = util::parse_uint(kv.value); + if (len == -1) { + return -1; } + if (content_length != -1) { + return -1; + } + content_length = len; } return 0; } const Headers::value_type *FieldStore::header(int16_t token) const { - return http2::get_header(hdidx_, token, headers_); + for (auto it = headers_.rbegin(); it != headers_.rend(); ++it) { + auto &kv = *it; + if (kv.token == token) { + return &kv; + } + } + return nullptr; } Headers::value_type *FieldStore::header(int16_t token) { - return http2::get_header(hdidx_, token, headers_); + for (auto it = headers_.rbegin(); it != headers_.rend(); ++it) { + auto &kv = *it; + if (kv.token == token) { + return &kv; + } + } + return nullptr; } const Headers::value_type *FieldStore::header(const StringRef &name) const { - return search_header_linear(headers_, name); + return search_header_linear_backwards(headers_, name); } -void FieldStore::add_header(std::string name, std::string value) { +void FieldStore::add_header_lower(std::string name, std::string value) { + util::inp_strlower(name); + auto token = http2::lookup_token(name); shrpx::add_header(header_key_prev_, buffer_size_, headers_, std::move(name), - std::move(value)); + std::move(value), token); } void FieldStore::add_header(std::string name, std::string value, int16_t token) { - http2::index_header(hdidx_, token, headers_.size()); buffer_size_ += name.size() + value.size(); headers_.emplace_back(std::move(name), std::move(value), false, token); } @@ -427,7 +431,6 @@ void FieldStore::add_header(std::string name, std::string value, void FieldStore::add_header(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, int16_t token) { - http2::index_header(hdidx_, token, headers_.size()); shrpx::add_header(buffer_size_, headers_, name, namelen, value, valuelen, no_index, token); } @@ -442,10 +445,7 @@ void FieldStore::append_last_header_value(const char *data, size_t len) { data, len); } -void FieldStore::clear_headers() { - headers_.clear(); - http2::init_hdidx(hdidx_); -} +void FieldStore::clear_headers() { headers_.clear(); } void FieldStore::add_trailer(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, @@ -456,7 +456,8 @@ void FieldStore::add_trailer(const uint8_t *name, size_t namelen, no_index, -1); } -void FieldStore::add_trailer(std::string name, std::string value) { +void FieldStore::add_trailer_lower(std::string name, std::string value) { + util::inp_strlower(name); shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, std::move(name), std::move(value)); } diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index a1f32760..4132e905 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -56,7 +56,6 @@ public: buffer_size_(0), header_key_prev_(false), trailer_key_prev_(false) { - http2::init_hdidx(hdidx_); headers_.reserve(headers_initial_capacity); } @@ -80,7 +79,7 @@ public: // such header is found, returns nullptr. const Headers::value_type *header(const StringRef &name) const; - void add_header(std::string name, std::string value); + void add_header_lower(std::string name, std::string value); void add_header(std::string name, std::string value, int16_t token); void add_header(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, int16_t token); @@ -100,7 +99,7 @@ public: void add_trailer(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, int16_t token); - void add_trailer(std::string name, std::string value); + void add_trailer_lower(std::string name, std::string value); void append_last_trailer_key(const char *data, size_t len); void append_last_trailer_value(const char *data, size_t len); @@ -115,7 +114,6 @@ private: // trailer fields. For HTTP/1.1, trailer fields are only included // with chunked encoding. For HTTP/2, there is no such limit. Headers trailers_; - http2::HeaderIndex hdidx_; // Sum of the length of name and value in headers_ and trailers_. // This could also be increased by add_extra_buffer_size() to take // into account for request URI in case of HTTP/1.x request. diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 8e2796a6..8b96e314 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -691,7 +691,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { if (ensure_max_header_fields(downstream, httpconf) != 0) { return -1; } - resp.fs.add_header(std::string(data, len), ""); + resp.fs.add_header_lower(std::string(data, len), ""); } } else { // trailer part @@ -704,7 +704,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { // wrong place or crash if trailer fields are currently empty. return -1; } - resp.fs.add_trailer(std::string(data, len), ""); + resp.fs.add_trailer_lower(std::string(data, len), ""); } } return 0; diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index c807e753..5875ac1b 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -142,7 +142,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { Downstream::HTTP1_REQUEST_HEADER_TOO_LARGE); return -1; } - req.fs.add_header(std::string(data, len), ""); + req.fs.add_header_lower(std::string(data, len), ""); } } else { // trailer part @@ -156,7 +156,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { } return -1; } - req.fs.add_trailer(std::string(data, len), ""); + req.fs.add_trailer_lower(std::string(data, len), ""); } } return 0; diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index ee80a144..27d768c2 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -188,7 +188,10 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, } for (size_t i = 0; nv[i]; i += 2) { - req.fs.add_header(nv[i], nv[i + 1]); + auto name = std::string(nv[i]); + auto value = std::string(nv[i + 1]); + auto token = http2::lookup_token(name); + req.fs.add_header(std::move(name), std::move(value), token); } if (req.fs.index_headers() != 0) { From 6f1347fc8b19b6419d19a55587651330fe02116b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 20 Feb 2016 19:13:39 +0900 Subject: [PATCH 035/147] nghttpx: Tokenize trailer field as well so that we can ditch prohibited headers in HTTP/2 --- src/shrpx_downstream.cc | 31 ++++++++++++++----------- src/shrpx_downstream.h | 4 ++-- src/shrpx_http2_session.cc | 6 ++--- src/shrpx_http2_upstream.cc | 6 ++--- src/shrpx_http_downstream_connection.cc | 4 ++-- src/shrpx_https_upstream.cc | 4 ++-- 6 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 71a0a2b6..bd76592e 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -330,7 +330,7 @@ void Downstream::crumble_request_cookie(std::vector &nva) { namespace { void add_header(bool &key_prev, size_t &sum, Headers &headers, std::string name, - std::string value, int16_t token = -1) { + std::string value, int16_t token) { key_prev = true; sum += name.size() + value.size(); headers.emplace_back(std::move(name), std::move(value), false, token); @@ -415,11 +415,13 @@ const Headers::value_type *FieldStore::header(const StringRef &name) const { return search_header_linear_backwards(headers_, name); } -void FieldStore::add_header_lower(std::string name, std::string value) { - util::inp_strlower(name); - auto token = http2::lookup_token(name); - shrpx::add_header(header_key_prev_, buffer_size_, headers_, std::move(name), - std::move(value), token); +void FieldStore::add_header_lower(const StringRef &name, + const StringRef &value) { + auto low_name = name.str(); + util::inp_strlower(low_name); + auto token = http2::lookup_token(low_name); + shrpx::add_header(header_key_prev_, buffer_size_, headers_, + std::move(low_name), value.str(), token); } void FieldStore::add_header(std::string name, std::string value, @@ -450,16 +452,19 @@ void FieldStore::clear_headers() { headers_.clear(); } void FieldStore::add_trailer(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, int16_t token) { - // we never index trailer fields. Header size limit should be - // applied to all header and trailer fields combined. + // Header size limit should be applied to all header and trailer + // fields combined. shrpx::add_header(buffer_size_, trailers_, name, namelen, value, valuelen, - no_index, -1); + no_index, token); } -void FieldStore::add_trailer_lower(std::string name, std::string value) { - util::inp_strlower(name); - shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, std::move(name), - std::move(value)); +void FieldStore::add_trailer_lower(const StringRef &name, + const StringRef &value) { + auto low_name = name.str(); + util::inp_strlower(low_name); + auto token = http2::lookup_token(low_name); + shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, + std::move(low_name), value.str(), token); } void FieldStore::append_last_trailer_key(const char *data, size_t len) { diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 4132e905..03286b83 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -79,7 +79,7 @@ public: // such header is found, returns nullptr. const Headers::value_type *header(const StringRef &name) const; - void add_header_lower(std::string name, std::string value); + void add_header_lower(const StringRef &name, const StringRef &value); void add_header(std::string name, std::string value, int16_t token); void add_header(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, int16_t token); @@ -99,7 +99,7 @@ public: void add_trailer(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, int16_t token); - void add_trailer_lower(std::string name, std::string value); + void add_trailer_lower(const StringRef &name, const StringRef &value); void append_last_trailer_key(const char *data, size_t len); void append_last_trailer_value(const char *data, size_t len); diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index f4004b9a..24283172 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -735,15 +735,15 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; } + auto token = http2::lookup_token(name, namelen); + if (trailer) { // just store header fields for trailer part resp.fs.add_trailer(name, namelen, value, valuelen, - flags & NGHTTP2_NV_FLAG_NO_INDEX, -1); + flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } - auto token = http2::lookup_token(name, namelen); - resp.fs.add_header(name, namelen, value, valuelen, flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 7e909a25..f13ea759 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -201,15 +201,15 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, return 0; } + auto token = http2::lookup_token(name, namelen); + if (frame->headers.cat == NGHTTP2_HCAT_HEADERS) { // just store header fields for trailer part req.fs.add_trailer(name, namelen, value, valuelen, - flags & NGHTTP2_NV_FLAG_NO_INDEX, -1); + flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } - auto token = http2::lookup_token(name, namelen); - req.fs.add_header(name, namelen, value, valuelen, flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 8b96e314..0af56ce1 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -691,7 +691,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { if (ensure_max_header_fields(downstream, httpconf) != 0) { return -1; } - resp.fs.add_header_lower(std::string(data, len), ""); + resp.fs.add_header_lower(StringRef{data, len}, StringRef{}); } } else { // trailer part @@ -704,7 +704,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { // wrong place or crash if trailer fields are currently empty. return -1; } - resp.fs.add_trailer_lower(std::string(data, len), ""); + resp.fs.add_trailer_lower(StringRef(data, len), StringRef{}); } } return 0; diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 5875ac1b..7f47b681 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -142,7 +142,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { Downstream::HTTP1_REQUEST_HEADER_TOO_LARGE); return -1; } - req.fs.add_header_lower(std::string(data, len), ""); + req.fs.add_header_lower(StringRef{data, len}, StringRef{}); } } else { // trailer part @@ -156,7 +156,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { } return -1; } - req.fs.add_trailer_lower(std::string(data, len), ""); + req.fs.add_trailer_lower(StringRef{data, len}, StringRef{}); } } return 0; From 3ff148811b54380f56a14d97d69cac2f2b722471 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 20 Feb 2016 20:53:19 +0900 Subject: [PATCH 036/147] nghttpx: Use StringRef for add_hedeader --- src/shrpx_downstream.cc | 24 ++++++++---------------- src/shrpx_downstream.h | 8 ++++---- src/shrpx_http2_session.cc | 7 ++++--- src/shrpx_http2_upstream.cc | 7 ++++--- 4 files changed, 20 insertions(+), 26 deletions(-) diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index bd76592e..0d3b2618 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -338,14 +338,10 @@ void add_header(bool &key_prev, size_t &sum, Headers &headers, std::string name, } // namespace namespace { -void add_header(size_t &sum, Headers &headers, const uint8_t *name, - size_t namelen, const uint8_t *value, size_t valuelen, - bool no_index, int16_t token) { - sum += namelen + valuelen; - headers.emplace_back( - std::string(reinterpret_cast(name), namelen), - std::string(reinterpret_cast(value), valuelen), no_index, - token); +void add_header(size_t &sum, Headers &headers, const StringRef &name, + const StringRef &value, bool no_index, int16_t token) { + sum += name.size() + value.size(); + headers.emplace_back(name.str(), value.str(), no_index, token); } } // namespace @@ -430,11 +426,9 @@ void FieldStore::add_header(std::string name, std::string value, headers_.emplace_back(std::move(name), std::move(value), false, token); } -void FieldStore::add_header(const uint8_t *name, size_t namelen, - const uint8_t *value, size_t valuelen, +void FieldStore::add_header(const StringRef &name, const StringRef &value, bool no_index, int16_t token) { - shrpx::add_header(buffer_size_, headers_, name, namelen, value, valuelen, - no_index, token); + shrpx::add_header(buffer_size_, headers_, name, value, no_index, token); } void FieldStore::append_last_header_key(const char *data, size_t len) { @@ -449,13 +443,11 @@ void FieldStore::append_last_header_value(const char *data, size_t len) { void FieldStore::clear_headers() { headers_.clear(); } -void FieldStore::add_trailer(const uint8_t *name, size_t namelen, - const uint8_t *value, size_t valuelen, +void FieldStore::add_trailer(const StringRef &name, const StringRef &value, bool no_index, int16_t token) { // Header size limit should be applied to all header and trailer // fields combined. - shrpx::add_header(buffer_size_, trailers_, name, namelen, value, valuelen, - no_index, token); + shrpx::add_header(buffer_size_, trailers_, name, value, no_index, token); } void FieldStore::add_trailer_lower(const StringRef &name, diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 03286b83..53778e49 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -81,8 +81,8 @@ public: void add_header_lower(const StringRef &name, const StringRef &value); void add_header(std::string name, std::string value, int16_t token); - void add_header(const uint8_t *name, size_t namelen, const uint8_t *value, - size_t valuelen, bool no_index, int16_t token); + void add_header(const StringRef &name, const StringRef &value, bool no_index, + int16_t token); void append_last_header_key(const char *data, size_t len); void append_last_header_value(const char *data, size_t len); @@ -97,8 +97,8 @@ public: // Empties headers. void clear_headers(); - void add_trailer(const uint8_t *name, size_t namelen, const uint8_t *value, - size_t valuelen, bool no_index, int16_t token); + void add_trailer(const StringRef &name, const StringRef &value, bool no_index, + int16_t token); void add_trailer_lower(const StringRef &name, const StringRef &value); void append_last_trailer_key(const char *data, size_t len); diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 24283172..2c96a579 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -739,12 +739,12 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, if (trailer) { // just store header fields for trailer part - resp.fs.add_trailer(name, namelen, value, valuelen, + resp.fs.add_trailer(StringRef{name, namelen}, StringRef{value, valuelen}, flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } - resp.fs.add_header(name, namelen, value, valuelen, + resp.fs.add_header(StringRef{name, namelen}, StringRef{value, valuelen}, flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } @@ -778,7 +778,8 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, } auto token = http2::lookup_token(name, namelen); - promised_req.fs.add_header(name, namelen, value, valuelen, + promised_req.fs.add_header(StringRef{name, namelen}, + StringRef{value, valuelen}, flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index f13ea759..4506081c 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -205,12 +205,12 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, if (frame->headers.cat == NGHTTP2_HCAT_HEADERS) { // just store header fields for trailer part - req.fs.add_trailer(name, namelen, value, valuelen, + req.fs.add_trailer(StringRef{name, namelen}, StringRef{value, valuelen}, flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } - req.fs.add_header(name, namelen, value, valuelen, + req.fs.add_header(StringRef{name, namelen}, StringRef{value, valuelen}, flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } @@ -593,7 +593,8 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame, req.path = http2::rewrite_clean_path(nv.value, nv.value + nv.valuelen); break; } - req.fs.add_header(nv.name, nv.namelen, nv.value, nv.valuelen, + req.fs.add_header(StringRef{nv.name, nv.namelen}, + StringRef{nv.value, nv.valuelen}, nv.flags & NGHTTP2_NV_FLAG_NO_INDEX, token); } From 23ecfd412d5f4143be50243fbbad81c865d0c302 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 20 Feb 2016 21:41:23 +0900 Subject: [PATCH 037/147] nghttpx: Fix mruby compile error, clean up add_header interface --- src/shrpx-unittest.cc | 4 +- src/shrpx_downstream.cc | 39 +++++------- src/shrpx_downstream.h | 15 +++-- src/shrpx_downstream_test.cc | 85 ++++++++++++++++--------- src/shrpx_downstream_test.h | 2 +- src/shrpx_http2_session.cc | 20 +++--- src/shrpx_http2_upstream.cc | 15 +++-- src/shrpx_http_downstream_connection.cc | 4 +- src/shrpx_https_upstream.cc | 4 +- src/shrpx_mruby.cc | 8 --- src/shrpx_mruby.h | 2 - src/shrpx_mruby_module_request.cc | 25 +++++--- src/shrpx_mruby_module_response.cc | 39 ++++++------ src/shrpx_spdy_upstream.cc | 8 +-- 14 files changed, 146 insertions(+), 124 deletions(-) diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index 4c6b39c8..0e7b17a1 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -106,8 +106,8 @@ int main(int argc, char *argv[]) { shrpx::test_http2_get_pure_path_component) || !CU_add_test(pSuite, "http2_construct_push_component", shrpx::test_http2_construct_push_component) || - !CU_add_test(pSuite, "downstream_field_store_index_headers", - shrpx::test_downstream_field_store_index_headers) || + !CU_add_test(pSuite, "downstream_field_store_add_header_lower", + shrpx::test_downstream_field_store_add_header_lower) || !CU_add_test(pSuite, "downstream_field_store_header", shrpx::test_downstream_field_store_header) || !CU_add_test(pSuite, "downstream_crumble_request_cookie", diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 0d3b2618..11ec9756 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -330,10 +330,10 @@ void Downstream::crumble_request_cookie(std::vector &nva) { namespace { void add_header(bool &key_prev, size_t &sum, Headers &headers, std::string name, - std::string value, int16_t token) { + std::string value, bool no_index, int16_t token) { key_prev = true; sum += name.size() + value.size(); - headers.emplace_back(std::move(name), std::move(value), false, token); + headers.emplace_back(std::move(name), std::move(value), no_index, token); } } // namespace @@ -411,23 +411,17 @@ const Headers::value_type *FieldStore::header(const StringRef &name) const { return search_header_linear_backwards(headers_, name); } -void FieldStore::add_header_lower(const StringRef &name, - const StringRef &value) { +void FieldStore::add_header_lower(const StringRef &name, const StringRef &value, + bool no_index) { auto low_name = name.str(); util::inp_strlower(low_name); auto token = http2::lookup_token(low_name); shrpx::add_header(header_key_prev_, buffer_size_, headers_, - std::move(low_name), value.str(), token); + std::move(low_name), value.str(), no_index, token); } -void FieldStore::add_header(std::string name, std::string value, - int16_t token) { - buffer_size_ += name.size() + value.size(); - headers_.emplace_back(std::move(name), std::move(value), false, token); -} - -void FieldStore::add_header(const StringRef &name, const StringRef &value, - bool no_index, int16_t token) { +void FieldStore::add_header_token(const StringRef &name, const StringRef &value, + bool no_index, int16_t token) { shrpx::add_header(buffer_size_, headers_, name, value, no_index, token); } @@ -443,20 +437,21 @@ void FieldStore::append_last_header_value(const char *data, size_t len) { void FieldStore::clear_headers() { headers_.clear(); } -void FieldStore::add_trailer(const StringRef &name, const StringRef &value, - bool no_index, int16_t token) { - // Header size limit should be applied to all header and trailer - // fields combined. - shrpx::add_header(buffer_size_, trailers_, name, value, no_index, token); -} - void FieldStore::add_trailer_lower(const StringRef &name, - const StringRef &value) { + const StringRef &value, bool no_index) { auto low_name = name.str(); util::inp_strlower(low_name); auto token = http2::lookup_token(low_name); shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, - std::move(low_name), value.str(), token); + std::move(low_name), value.str(), no_index, token); +} + +void FieldStore::add_trailer_token(const StringRef &name, + const StringRef &value, bool no_index, + int16_t token) { + // Header size limit should be applied to all header and trailer + // fields combined. + shrpx::add_header(buffer_size_, trailers_, name, value, no_index, token); } void FieldStore::append_last_trailer_key(const char *data, size_t len) { diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 53778e49..8a855b1c 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -79,10 +79,10 @@ public: // such header is found, returns nullptr. const Headers::value_type *header(const StringRef &name) const; - void add_header_lower(const StringRef &name, const StringRef &value); - void add_header(std::string name, std::string value, int16_t token); - void add_header(const StringRef &name, const StringRef &value, bool no_index, - int16_t token); + void add_header_lower(const StringRef &name, const StringRef &value, + bool no_index); + void add_header_token(const StringRef &name, const StringRef &value, + bool no_index, int16_t token); void append_last_header_key(const char *data, size_t len); void append_last_header_value(const char *data, size_t len); @@ -97,9 +97,10 @@ public: // Empties headers. void clear_headers(); - void add_trailer(const StringRef &name, const StringRef &value, bool no_index, - int16_t token); - void add_trailer_lower(const StringRef &name, const StringRef &value); + void add_trailer_lower(const StringRef &name, const StringRef &value, + bool no_index); + void add_trailer_token(const StringRef &name, const StringRef &value, + bool no_index, int16_t token); void append_last_trailer_key(const char *data, size_t len); void append_last_trailer_value(const char *data, size_t len); diff --git a/src/shrpx_downstream_test.cc b/src/shrpx_downstream_test.cc index c7b559a4..5b496f9b 100644 --- a/src/shrpx_downstream_test.cc +++ b/src/shrpx_downstream_test.cc @@ -32,17 +32,24 @@ namespace shrpx { -void test_downstream_field_store_index_headers(void) { +void test_downstream_field_store_add_header_lower(void) { FieldStore fs(0); - fs.add_header("1", "0"); - fs.add_header("2", "1"); - fs.add_header("Charlie", "2"); - fs.add_header("Alpha", "3"); - fs.add_header("Delta", "4"); - fs.add_header("BravO", "5"); - fs.add_header(":method", "6"); - fs.add_header(":authority", "7"); - fs.index_headers(); + fs.add_header_lower(StringRef::from_lit("1"), StringRef::from_lit("0"), + false); + fs.add_header_lower(StringRef::from_lit("2"), StringRef::from_lit("1"), + false); + fs.add_header_lower(StringRef::from_lit("Charlie"), StringRef::from_lit("2"), + false); + fs.add_header_lower(StringRef::from_lit("Alpha"), StringRef::from_lit("3"), + false); + fs.add_header_lower(StringRef::from_lit("Delta"), StringRef::from_lit("4"), + false); + fs.add_header_lower(StringRef::from_lit("BravO"), StringRef::from_lit("5"), + false); + fs.add_header_lower(StringRef::from_lit(":method"), StringRef::from_lit("6"), + false); + fs.add_header_lower(StringRef::from_lit(":authority"), + StringRef::from_lit("7"), false); auto ans = Headers{{"1", "0"}, {"2", "1"}, @@ -57,10 +64,13 @@ void test_downstream_field_store_index_headers(void) { void test_downstream_field_store_header(void) { FieldStore fs(0); - fs.add_header("alpha", "0"); - fs.add_header(":authority", "1"); - fs.add_header("content-length", "2"); - fs.index_headers(); + fs.add_header_token(StringRef::from_lit("alpha"), StringRef::from_lit("0"), + false, -1); + fs.add_header_token(StringRef::from_lit(":authority"), + StringRef::from_lit("1"), false, http2::HD__AUTHORITY); + fs.add_header_token(StringRef::from_lit("content-length"), + StringRef::from_lit("2"), false, + http2::HD_CONTENT_LENGTH); // By token CU_ASSERT(Header(":authority", "1") == *fs.header(http2::HD__AUTHORITY)); @@ -74,14 +84,18 @@ void test_downstream_field_store_header(void) { void test_downstream_crumble_request_cookie(void) { Downstream d(nullptr, nullptr, 0); auto &req = d.request(); - req.fs.add_header(":method", "get"); - req.fs.add_header(":path", "/"); - auto val = "alpha; bravo; ; ;; charlie;;"; - req.fs.add_header( - reinterpret_cast("cookie"), sizeof("cookie") - 1, - reinterpret_cast(val), strlen(val), true, -1); - req.fs.add_header("cookie", ";delta"); - req.fs.add_header("cookie", "echo"); + req.fs.add_header_token(StringRef::from_lit(":method"), + StringRef::from_lit("get"), false, -1); + req.fs.add_header_token(StringRef::from_lit(":path"), + StringRef::from_lit("/"), false, -1); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit("alpha; bravo; ; ;; charlie;;"), + true, http2::HD_COOKIE); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit(";delta"), false, + http2::HD_COOKIE); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit("echo"), false, http2::HD_COOKIE); std::vector nva; d.crumble_request_cookie(nva); @@ -114,12 +128,22 @@ void test_downstream_crumble_request_cookie(void) { void test_downstream_assemble_request_cookie(void) { Downstream d(nullptr, nullptr, 0); auto &req = d.request(); - req.fs.add_header(":method", "get"); - req.fs.add_header(":path", "/"); - req.fs.add_header("cookie", "alpha"); - req.fs.add_header("cookie", "bravo;"); - req.fs.add_header("cookie", "charlie; "); - req.fs.add_header("cookie", "delta;;"); + req.fs.add_header_token(StringRef::from_lit(":method"), + StringRef::from_lit("get"), false, -1); + req.fs.add_header_token(StringRef::from_lit(":path"), + StringRef::from_lit("/"), false, -1); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit("alpha"), false, + http2::HD_COOKIE); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit("bravo;"), false, + http2::HD_COOKIE); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit("charlie; "), false, + http2::HD_COOKIE); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit("delta;;"), false, + http2::HD_COOKIE); CU_ASSERT("alpha; bravo; charlie; delta" == d.assemble_request_cookie()); } @@ -129,8 +153,9 @@ void test_downstream_rewrite_location_response_header(void) { auto &resp = d.response(); d.set_request_downstream_host("localhost2"); req.authority = "localhost:8443"; - resp.fs.add_header("location", "http://localhost2:3000/"); - resp.fs.index_headers(); + resp.fs.add_header_token(StringRef::from_lit("location"), + StringRef::from_lit("http://localhost2:3000/"), + false, http2::HD_LOCATION); d.rewrite_location_response_header("https"); auto location = resp.fs.header(http2::HD_LOCATION); CU_ASSERT("https://localhost:8443/" == (*location).value); diff --git a/src/shrpx_downstream_test.h b/src/shrpx_downstream_test.h index 13b33744..bcae1887 100644 --- a/src/shrpx_downstream_test.h +++ b/src/shrpx_downstream_test.h @@ -31,7 +31,7 @@ namespace shrpx { -void test_downstream_field_store_index_headers(void); +void test_downstream_field_store_add_header_lower(void); void test_downstream_field_store_header(void); void test_downstream_crumble_request_cookie(void); void test_downstream_assemble_request_cookie(void); diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 2c96a579..d11e3fd7 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -736,16 +736,17 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, } auto token = http2::lookup_token(name, namelen); + auto no_index = flags & NGHTTP2_NV_FLAG_NO_INDEX; if (trailer) { // just store header fields for trailer part - resp.fs.add_trailer(StringRef{name, namelen}, StringRef{value, valuelen}, - flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + resp.fs.add_trailer_token(StringRef{name, namelen}, + StringRef{value, valuelen}, no_index, token); return 0; } - resp.fs.add_header(StringRef{name, namelen}, StringRef{value, valuelen}, - flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + resp.fs.add_header_token(StringRef{name, namelen}, + StringRef{value, valuelen}, no_index, token); return 0; } case NGHTTP2_PUSH_PROMISE: { @@ -778,9 +779,9 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, } auto token = http2::lookup_token(name, namelen); - promised_req.fs.add_header(StringRef{name, namelen}, - StringRef{value, valuelen}, - flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + promised_req.fs.add_header_token(StringRef{name, namelen}, + StringRef{value, valuelen}, + flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } } @@ -927,8 +928,9 @@ int on_response_headers(Http2Session *http2session, Downstream *downstream, // Otherwise, use chunked encoding to keep upstream connection // open. In HTTP2, we are supporsed not to receive // transfer-encoding. - resp.fs.add_header("transfer-encoding", "chunked", - http2::HD_TRANSFER_ENCODING); + resp.fs.add_header_token(StringRef::from_lit("transfer-encoding"), + StringRef::from_lit("chunked"), false, + http2::HD_TRANSFER_ENCODING); downstream->set_chunked_response(true); } } diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 4506081c..0c8795dc 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -202,16 +202,17 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, } auto token = http2::lookup_token(name, namelen); + auto no_index = flags & NGHTTP2_NV_FLAG_NO_INDEX; if (frame->headers.cat == NGHTTP2_HCAT_HEADERS) { // just store header fields for trailer part - req.fs.add_trailer(StringRef{name, namelen}, StringRef{value, valuelen}, - flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + req.fs.add_trailer_token(StringRef{name, namelen}, + StringRef{value, valuelen}, no_index, token); return 0; } - req.fs.add_header(StringRef{name, namelen}, StringRef{value, valuelen}, - flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + req.fs.add_header_token(StringRef{name, namelen}, StringRef{value, valuelen}, + no_index, token); return 0; } } // namespace @@ -593,9 +594,9 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame, req.path = http2::rewrite_clean_path(nv.value, nv.value + nv.valuelen); break; } - req.fs.add_header(StringRef{nv.name, nv.namelen}, - StringRef{nv.value, nv.valuelen}, - nv.flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + req.fs.add_header_token(StringRef{nv.name, nv.namelen}, + StringRef{nv.value, nv.valuelen}, + nv.flags & NGHTTP2_NV_FLAG_NO_INDEX, token); } promised_downstream->inspect_http2_request(); diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 0af56ce1..c5b3da09 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -691,7 +691,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { if (ensure_max_header_fields(downstream, httpconf) != 0) { return -1; } - resp.fs.add_header_lower(StringRef{data, len}, StringRef{}); + resp.fs.add_header_lower(StringRef{data, len}, StringRef{}, false); } } else { // trailer part @@ -704,7 +704,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { // wrong place or crash if trailer fields are currently empty. return -1; } - resp.fs.add_trailer_lower(StringRef(data, len), StringRef{}); + resp.fs.add_trailer_lower(StringRef(data, len), StringRef{}, false); } } return 0; diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 7f47b681..c84af459 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -142,7 +142,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { Downstream::HTTP1_REQUEST_HEADER_TOO_LARGE); return -1; } - req.fs.add_header_lower(StringRef{data, len}, StringRef{}); + req.fs.add_header_lower(StringRef{data, len}, StringRef{}, false); } } else { // trailer part @@ -156,7 +156,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { } return -1; } - req.fs.add_trailer_lower(StringRef{data, len}, StringRef{}); + req.fs.add_trailer_lower(StringRef{data, len}, StringRef{}, false); } } return 0; diff --git a/src/shrpx_mruby.cc b/src/shrpx_mruby.cc index bd57c146..507d13b9 100644 --- a/src/shrpx_mruby.cc +++ b/src/shrpx_mruby.cc @@ -94,14 +94,6 @@ int MRubyContext::run_app(Downstream *downstream, int phase) { mrb_->ud = nullptr; - if (data.request_headers_dirty) { - downstream->request().fs.index_headers(); - } - - if (data.response_headers_dirty) { - downstream->response().fs.index_headers(); - } - return rv; } diff --git a/src/shrpx_mruby.h b/src/shrpx_mruby.h index 55a18351..4466cf6f 100644 --- a/src/shrpx_mruby.h +++ b/src/shrpx_mruby.h @@ -67,8 +67,6 @@ enum { struct MRubyAssocData { Downstream *downstream; int phase; - bool request_headers_dirty; - bool response_headers_dirty; }; RProc *compile(mrb_state *mrb, const StringRef &filename); diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index 0b1ddb15..f585d730 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -216,17 +216,20 @@ mrb_value request_mod_header(mrb_state *mrb, mrb_value self, bool repl) { key = mrb_funcall(mrb, key, "downcase", 0); + auto keyref = + StringRef{RSTRING_PTR(key), static_cast(RSTRING_LEN(key))}; + auto token = http2::lookup_token(keyref.byte(), keyref.size()); + if (repl) { size_t p = 0; auto &headers = req.fs.headers(); for (size_t i = 0; i < headers.size(); ++i) { - auto &hd = headers[i]; - if (util::streq(std::begin(hd.name), hd.name.size(), RSTRING_PTR(key), - RSTRING_LEN(key))) { + auto &kv = headers[i]; + if (kv.name == keyref) { continue; } if (i != p) { - headers[p++] = std::move(hd); + headers[p++] = std::move(kv); } } headers.resize(p); @@ -236,16 +239,18 @@ mrb_value request_mod_header(mrb_state *mrb, mrb_value self, bool repl) { auto n = mrb_ary_len(mrb, values); for (int i = 0; i < n; ++i) { auto value = mrb_ary_entry(values, i); - req.fs.add_header(std::string(RSTRING_PTR(key), RSTRING_LEN(key)), - std::string(RSTRING_PTR(value), RSTRING_LEN(value))); + req.fs.add_header_token( + keyref, StringRef{RSTRING_PTR(value), + static_cast(RSTRING_LEN(value))}, + false, token); } } else if (!mrb_nil_p(values)) { - req.fs.add_header(std::string(RSTRING_PTR(key), RSTRING_LEN(key)), - std::string(RSTRING_PTR(values), RSTRING_LEN(values))); + req.fs.add_header_token(keyref, + StringRef{RSTRING_PTR(values), + static_cast(RSTRING_LEN(values))}, + false, token); } - data->request_headers_dirty = true; - return mrb_nil_value(); } } // namespace diff --git a/src/shrpx_mruby_module_response.cc b/src/shrpx_mruby_module_response.cc index 5b88f879..73dacccc 100644 --- a/src/shrpx_mruby_module_response.cc +++ b/src/shrpx_mruby_module_response.cc @@ -117,17 +117,20 @@ mrb_value response_mod_header(mrb_state *mrb, mrb_value self, bool repl) { key = mrb_funcall(mrb, key, "downcase", 0); + auto keyref = + StringRef{RSTRING_PTR(key), static_cast(RSTRING_LEN(key))}; + auto token = http2::lookup_token(keyref.byte(), keyref.size()); + if (repl) { size_t p = 0; auto &headers = resp.fs.headers(); for (size_t i = 0; i < headers.size(); ++i) { - auto &hd = headers[i]; - if (util::streq(std::begin(hd.name), hd.name.size(), RSTRING_PTR(key), - RSTRING_LEN(key))) { + auto &kv = headers[i]; + if (kv.name == keyref) { continue; } if (i != p) { - headers[p++] = std::move(hd); + headers[p++] = std::move(kv); } } headers.resize(p); @@ -137,16 +140,18 @@ mrb_value response_mod_header(mrb_state *mrb, mrb_value self, bool repl) { auto n = mrb_ary_len(mrb, values); for (int i = 0; i < n; ++i) { auto value = mrb_ary_entry(values, i); - resp.fs.add_header(std::string(RSTRING_PTR(key), RSTRING_LEN(key)), - std::string(RSTRING_PTR(value), RSTRING_LEN(value))); + resp.fs.add_header_token( + keyref, StringRef{RSTRING_PTR(value), + static_cast(RSTRING_LEN(value))}, + false, token); } } else if (!mrb_nil_p(values)) { - resp.fs.add_header(std::string(RSTRING_PTR(key), RSTRING_LEN(key)), - std::string(RSTRING_PTR(values), RSTRING_LEN(values))); + resp.fs.add_header_token( + keyref, StringRef{RSTRING_PTR(values), + static_cast(RSTRING_LEN(values))}, + false, token); } - data->response_headers_dirty = true; - return mrb_nil_value(); } } // namespace @@ -197,11 +202,6 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { resp.http_status = 200; } - if (data->response_headers_dirty) { - resp.fs.index_headers(); - data->response_headers_dirty = false; - } - if (downstream->expect_response_body() && vallen > 0) { body = reinterpret_cast(val); bodylen = vallen; @@ -211,8 +211,9 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { if (cl) { cl->value = util::utos(bodylen); } else { - resp.fs.add_header("content-length", util::utos(bodylen), - http2::HD_CONTENT_LENGTH); + resp.fs.add_header_token(StringRef::from_lit("content-length"), + StringRef{util::utos(bodylen)}, false, + http2::HD_CONTENT_LENGTH); } resp.fs.content_length = bodylen; @@ -220,7 +221,9 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { if (!date) { auto lgconf = log_config(); lgconf->update_tstamp(std::chrono::system_clock::now()); - resp.fs.add_header("date", lgconf->time_http_str, http2::HD_DATE); + resp.fs.add_header_token(StringRef::from_lit("date"), + StringRef{lgconf->time_http_str}, false, + http2::HD_DATE); } auto upstream = downstream->get_upstream(); diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 27d768c2..dc4a5db1 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -188,10 +188,10 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, } for (size_t i = 0; nv[i]; i += 2) { - auto name = std::string(nv[i]); - auto value = std::string(nv[i + 1]); - auto token = http2::lookup_token(name); - req.fs.add_header(std::move(name), std::move(value), token); + auto name = StringRef{nv[i]}; + auto value = StringRef{nv[i + 1]}; + auto token = http2::lookup_token(name.byte(), name.size()); + req.fs.add_header_token(name, value, false, token); } if (req.fs.index_headers() != 0) { From 9678daa46aa72f7abc698822ea6d930665b0d612 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 20 Feb 2016 21:44:08 +0900 Subject: [PATCH 038/147] nghttpx: Rename index_headers() as parse_content_length() --- src/shrpx_downstream.cc | 2 +- src/shrpx_downstream.h | 7 +++---- src/shrpx_http_downstream_connection.cc | 2 +- src/shrpx_https_upstream.cc | 2 +- src/shrpx_spdy_upstream.cc | 2 +- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 11ec9756..9cfc2cc7 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -367,7 +367,7 @@ void append_last_header_value(bool &key_prev, size_t &sum, Headers &headers, } } // namespace -int FieldStore::index_headers() { +int FieldStore::parse_content_length() { content_length = -1; for (auto &kv : headers_) { diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 8a855b1c..4306940d 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -89,10 +89,9 @@ public: bool header_key_prev() const { return header_key_prev_; } - // Lower the header field names and indexes header fields. If there - // is any invalid headers (e.g., multiple Content-Length having - // different values), returns -1. - int index_headers(); + // Parses content-length, and records it in the field. If there are + // multiple Content-Length, returns -1. + int parse_content_length(); // Empties headers. void clear_headers(); diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index c5b3da09..9e55f39b 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -572,7 +572,7 @@ int htp_hdrs_completecb(http_parser *htp) { resp.http_major = htp->http_major; resp.http_minor = htp->http_minor; - if (resp.fs.index_headers() != 0) { + if (resp.fs.parse_content_length() != 0) { downstream->set_response_state(Downstream::MSG_BAD_HEADER); return -1; } diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index c84af459..9eaa5202 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -270,7 +270,7 @@ int htp_hdrs_completecb(http_parser *htp) { ULOG(INFO, upstream) << "HTTP request headers\n" << ss.str(); } - if (req.fs.index_headers() != 0) { + if (req.fs.parse_content_length() != 0) { return -1; } diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index dc4a5db1..d9c22209 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -194,7 +194,7 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, req.fs.add_header_token(name, value, false, token); } - if (req.fs.index_headers() != 0) { + if (req.fs.parse_content_length() != 0) { if (upstream->error_reply(downstream, 400) != 0) { ULOG(FATAL, upstream) << "error_reply failed"; } From 61579ad20ff965eb7df80b55a412282766e0110e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 20 Feb 2016 22:56:55 +0900 Subject: [PATCH 039/147] nghttpx: Use StringRef for shrpx::add_header --- src/shrpx_downstream.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 9cfc2cc7..32b7c1f2 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -329,11 +329,12 @@ void Downstream::crumble_request_cookie(std::vector &nva) { } namespace { -void add_header(bool &key_prev, size_t &sum, Headers &headers, std::string name, - std::string value, bool no_index, int16_t token) { +void add_header(bool &key_prev, size_t &sum, Headers &headers, + const StringRef &name, const StringRef &value, bool no_index, + int16_t token) { key_prev = true; sum += name.size() + value.size(); - headers.emplace_back(std::move(name), std::move(value), no_index, token); + headers.emplace_back(name.str(), value.str(), no_index, token); } } // namespace @@ -417,7 +418,7 @@ void FieldStore::add_header_lower(const StringRef &name, const StringRef &value, util::inp_strlower(low_name); auto token = http2::lookup_token(low_name); shrpx::add_header(header_key_prev_, buffer_size_, headers_, - std::move(low_name), value.str(), no_index, token); + StringRef{low_name}, value, no_index, token); } void FieldStore::add_header_token(const StringRef &name, const StringRef &value, @@ -443,7 +444,7 @@ void FieldStore::add_trailer_lower(const StringRef &name, util::inp_strlower(low_name); auto token = http2::lookup_token(low_name); shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, - std::move(low_name), value.str(), no_index, token); + StringRef{low_name}, value, no_index, token); } void FieldStore::add_trailer_token(const StringRef &name, From c9a4f293a11be14c1fcc4b7ce58ee617b53da947 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 14:53:06 +0900 Subject: [PATCH 040/147] nghttpx: ConnectBlocker per backend address --- src/shrpx_client_handler.cc | 8 +-- src/shrpx_client_handler.h | 1 - src/shrpx_config.h | 2 + src/shrpx_connect_blocker.cc | 27 +++++---- src/shrpx_connect_blocker.h | 14 +++-- src/shrpx_http2_downstream_connection.cc | 3 + src/shrpx_http2_session.cc | 73 +++++++++++++++--------- src/shrpx_http2_session.h | 7 +-- src/shrpx_http_downstream_connection.cc | 33 +++++++---- src/shrpx_worker.cc | 26 ++++++--- src/shrpx_worker.h | 5 +- 11 files changed, 125 insertions(+), 74 deletions(-) diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 4d063647..6e89a5c4 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -389,7 +389,7 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl, pinned_http2sessions_( get_config()->conn.downstream.proto == PROTO_HTTP2 ? make_unique>( - get_config()->conn.downstream.addr_groups.size(), -1) + worker->get_downstream_addr_groups().size(), -1) : nullptr), ipaddr_(ipaddr), port_(port), @@ -664,8 +664,8 @@ std::unique_ptr ClientHandler::get_downstream_connection(Downstream *downstream) { size_t group; auto &downstreamconf = get_config()->conn.downstream; - auto &groups = downstreamconf.addr_groups; auto catch_all = downstreamconf.addr_group_catch_all; + auto &groups = worker_->get_downstream_addr_groups(); const auto &req = downstream->request(); @@ -746,10 +746,6 @@ MemchunkPool *ClientHandler::get_mcpool() { return worker_->get_mcpool(); } SSL *ClientHandler::get_ssl() const { return conn_.tls.ssl; } -ConnectBlocker *ClientHandler::get_connect_blocker() const { - return worker_->get_connect_blocker(); -} - void ClientHandler::direct_http2_upgrade() { upstream_ = make_unique(this); alpn_ = NGHTTP2_CLEARTEXT_PROTO_VERSION_ID; diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index 4830e93e..9e932ae2 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -99,7 +99,6 @@ public: get_downstream_connection(Downstream *downstream); MemchunkPool *get_mcpool(); SSL *get_ssl() const; - ConnectBlocker *get_connect_blocker() const; // Call this function when HTTP/2 connection header is received at // the start of the connection. void direct_http2_upgrade(); diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 80f0e06e..7913fa19 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -59,6 +59,7 @@ using namespace nghttp2; namespace shrpx { struct LogFragment; +class ConnectBlocker; namespace ssl { @@ -294,6 +295,7 @@ struct DownstreamAddr { // socket path. ImmutableString host; ImmutableString hostport; + ConnectBlocker *connect_blocker; // backend port. 0 if |host_unix| is true. uint16_t port; // true if |host| contains UNIX domain socket path. diff --git a/src/shrpx_connect_blocker.cc b/src/shrpx_connect_blocker.cc index 3eb6de16..cdab59a9 100644 --- a/src/shrpx_connect_blocker.cc +++ b/src/shrpx_connect_blocker.cc @@ -26,20 +26,16 @@ namespace shrpx { -namespace { -const ev_tstamp INITIAL_SLEEP = 2.; -} // namespace - namespace { void connect_blocker_cb(struct ev_loop *loop, ev_timer *w, int revents) { if (LOG_ENABLED(INFO)) { - LOG(INFO) << "unblock downstream connection"; + LOG(INFO) << "Unblock"; } } } // namespace -ConnectBlocker::ConnectBlocker(struct ev_loop *loop) - : loop_(loop), sleep_(INITIAL_SLEEP) { +ConnectBlocker::ConnectBlocker(std::mt19937 &gen, struct ev_loop *loop) + : gen_(gen), loop_(loop), fail_count_(0) { ev_timer_init(&timer_, connect_blocker_cb, 0., 0.); } @@ -47,18 +43,27 @@ ConnectBlocker::~ConnectBlocker() { ev_timer_stop(loop_, &timer_); } bool ConnectBlocker::blocked() const { return ev_is_active(&timer_); } -void ConnectBlocker::on_success() { sleep_ = INITIAL_SLEEP; } +void ConnectBlocker::on_success() { fail_count_ = 0; } + +namespace { +constexpr size_t MAX_BACKOFF_EXP = 10; +} // namespace void ConnectBlocker::on_failure() { if (ev_is_active(&timer_)) { return; } - sleep_ = std::min(128., sleep_ * 2); + ++fail_count_; - LOG(WARN) << "connect failure, start sleeping " << sleep_; + auto max_backoff = (1 << std::min(MAX_BACKOFF_EXP, fail_count_)) - 1; + auto dist = std::uniform_int_distribution<>(0, max_backoff); + auto backoff = dist(gen_); - ev_timer_set(&timer_, sleep_, 0.); + LOG(WARN) << "Could not connect " << fail_count_ + << " times in a row; sleep for " << backoff << " seconds"; + + ev_timer_set(&timer_, backoff, 0.); ev_timer_start(loop_, &timer_); } diff --git a/src/shrpx_connect_blocker.h b/src/shrpx_connect_blocker.h index af445644..63a1e3f9 100644 --- a/src/shrpx_connect_blocker.h +++ b/src/shrpx_connect_blocker.h @@ -27,13 +27,15 @@ #include "shrpx.h" +#include + #include namespace shrpx { class ConnectBlocker { public: - ConnectBlocker(struct ev_loop *loop); + ConnectBlocker(std::mt19937 &gen, struct ev_loop *loop); ~ConnectBlocker(); // Returns true if making connection is not allowed. @@ -41,14 +43,18 @@ public: // Call this function if connect operation succeeded. This will // reset sleep_ to minimum value. void on_success(); - // Call this function if connect operation failed. This will start - // timer and blocks connection establishment for sleep_ seconds. + // Call this function if connect operations failed. This will start + // timer and blocks connection establishment with exponential + // backoff. void on_failure(); private: + std::mt19937 gen_; ev_timer timer_; struct ev_loop *loop_; - ev_tstamp sleep_; + // The number of consecutive connection failure. Reset to 0 on + // success. + size_t fail_count_; }; } // namespace diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index 02982b8a..da2d953d 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -98,6 +98,9 @@ int Http2DownstreamConnection::attach_downstream(Downstream *downstream) { http2session_->add_downstream_connection(this); if (http2session_->get_state() == Http2Session::DISCONNECTED) { http2session_->signal_write(); + if (http2session_->get_state() == Http2Session::DISCONNECTED) { + return -1; + } } downstream_ = downstream; diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index d11e3fd7..1f660649 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -147,8 +147,7 @@ void writecb(struct ev_loop *loop, ev_io *w, int revents) { } // namespace Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, - ConnectBlocker *connect_blocker, Worker *worker, - size_t group, size_t idx) + Worker *worker, size_t group, size_t idx) : conn_(loop, -1, nullptr, worker->get_mcpool(), get_config()->conn.downstream.timeout.write, get_config()->conn.downstream.timeout.read, {}, {}, writecb, readcb, @@ -156,7 +155,6 @@ Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, get_config()->tls.dyn_rec.idle_timeout), wb_(worker->get_mcpool()), worker_(worker), - connect_blocker_(connect_blocker), ssl_ctx_(ssl_ctx), addr_(nullptr), session_(nullptr), @@ -254,30 +252,49 @@ int Http2Session::disconnect(bool hard) { int Http2Session::initiate_connection() { int rv = 0; - auto &addrs = get_config()->conn.downstream.addr_groups[group_].addrs; + auto &groups = worker_->get_downstream_addr_groups(); + auto &addrs = groups[group_].addrs; if (state_ == DISCONNECTED) { - if (connect_blocker_->blocked()) { - if (LOG_ENABLED(INFO)) { - DCLOG(INFO, this) - << "Downstream connection was blocked by connect_blocker"; - } - return -1; - } - auto &next_downstream = worker_->get_dgrp(group_)->next; - addr_ = &addrs[next_downstream]; + auto end = next_downstream; - if (LOG_ENABLED(INFO)) { - SSLOG(INFO, this) << "Using downstream address idx=" << next_downstream - << " out of " << addrs.size(); - } + for (;;) { + auto &addr = addrs[next_downstream]; - if (++next_downstream >= addrs.size()) { - next_downstream = 0; + if (++next_downstream >= addrs.size()) { + next_downstream = 0; + } + + auto &connect_blocker = addr.connect_blocker; + + if (connect_blocker->blocked()) { + if (LOG_ENABLED(INFO)) { + DCLOG(INFO, this) << "Backend server " + << (addr.host_unix ? addr.host : addr.hostport) + << " was not available temporarily"; + } + + if (end == next_downstream) { + return -1; + } + + continue; + } + + if (LOG_ENABLED(INFO)) { + SSLOG(INFO, this) << "Using downstream address idx=" << next_downstream + << " out of " << addrs.size(); + } + + addr_ = &addr; + + break; } } + auto &connect_blocker = addr_->connect_blocker; + const auto &proxy = get_config()->downstream_http_proxy; if (!proxy.host.empty() && state_ == DISCONNECTED) { if (LOG_ENABLED(INFO)) { @@ -288,7 +305,7 @@ int Http2Session::initiate_connection() { conn_.fd = util::create_nonblock_socket(proxy.addr.su.storage.ss_family); if (conn_.fd == -1) { - connect_blocker_->on_failure(); + connect_blocker->on_failure(); return -1; } @@ -296,7 +313,7 @@ int Http2Session::initiate_connection() { if (rv != 0 && errno != EINPROGRESS) { SSLOG(ERROR, this) << "Failed to connect to the proxy " << proxy.host << ":" << proxy.port; - connect_blocker_->on_failure(); + connect_blocker->on_failure(); return -1; } @@ -356,7 +373,7 @@ int Http2Session::initiate_connection() { conn_.fd = util::create_nonblock_socket(addr_->addr.su.storage.ss_family); if (conn_.fd == -1) { - connect_blocker_->on_failure(); + connect_blocker->on_failure(); return -1; } @@ -365,7 +382,7 @@ int Http2Session::initiate_connection() { const_cast(&addr_->addr.su.sa), addr_->addr.len); if (rv != 0 && errno != EINPROGRESS) { - connect_blocker_->on_failure(); + connect_blocker->on_failure(); return -1; } @@ -383,14 +400,14 @@ int Http2Session::initiate_connection() { util::create_nonblock_socket(addr_->addr.su.storage.ss_family); if (conn_.fd == -1) { - connect_blocker_->on_failure(); + connect_blocker->on_failure(); return -1; } rv = connect(conn_.fd, const_cast(&addr_->addr.su.sa), addr_->addr.len); if (rv != 0 && errno != EINPROGRESS) { - connect_blocker_->on_failure(); + connect_blocker->on_failure(); return -1; } @@ -1615,11 +1632,15 @@ int Http2Session::read_noop(const uint8_t *data, size_t datalen) { return 0; } int Http2Session::write_noop() { return 0; } int Http2Session::connected() { + auto &connect_blocker = addr_->connect_blocker; + if (!util::check_socket_connected(conn_.fd)) { + connect_blocker->on_failure(); + return -1; } - connect_blocker_->on_success(); + connect_blocker->on_success(); if (LOG_ENABLED(INFO)) { SSLOG(INFO, this) << "Connection established"; diff --git a/src/shrpx_http2_session.h b/src/shrpx_http2_session.h index a858b1bc..62e1235c 100644 --- a/src/shrpx_http2_session.h +++ b/src/shrpx_http2_session.h @@ -48,7 +48,6 @@ namespace shrpx { class Http2DownstreamConnection; class Worker; -class ConnectBlocker; struct StreamData { StreamData *dlnext, *dlprev; @@ -57,9 +56,8 @@ struct StreamData { class Http2Session { public: - Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, - ConnectBlocker *connect_blocker, Worker *worker, size_t group, - size_t idx); + Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, Worker *worker, + size_t group, size_t idx); ~Http2Session(); // If hard is true, all pending requests are abandoned and @@ -203,7 +201,6 @@ private: // Used to parse the response from HTTP proxy std::unique_ptr proxy_htp_; Worker *worker_; - ConnectBlocker *connect_blocker_; // NULL if no TLS is configured SSL_CTX *ssl_ctx_; // Address of remote endpoint diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 9e55f39b..151bd45b 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -147,16 +147,6 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { auto &downstreamconf = get_config()->conn.downstream; if (conn_.fd == -1) { - auto connect_blocker = client_handler_->get_connect_blocker(); - - if (connect_blocker->blocked()) { - if (LOG_ENABLED(INFO)) { - DCLOG(INFO, this) - << "Downstream connection was blocked by connect_blocker"; - } - return -1; - } - if (ssl_ctx_) { auto ssl = ssl::create_ssl(ssl_ctx_); if (!ssl) { @@ -168,7 +158,8 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { auto &next_downstream = worker_->get_dgrp(group_)->next; auto end = next_downstream; - auto &addrs = downstreamconf.addr_groups[group_].addrs; + auto &groups = worker_->get_downstream_addr_groups(); + auto &addrs = groups[group_].addrs; for (;;) { auto &addr = addrs[next_downstream]; @@ -176,6 +167,22 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { next_downstream = 0; } + auto &connect_blocker = addr.connect_blocker; + + if (connect_blocker->blocked()) { + if (LOG_ENABLED(INFO)) { + DCLOG(INFO, this) << "Backend server " + << (addr.host_unix ? addr.host : addr.hostport) + << " was not available temporarily"; + } + + if (end == next_downstream) { + return SHRPX_ERR_NETWORK; + } + + continue; + } + conn_.fd = util::create_nonblock_socket(addr.addr.su.storage.ss_family); if (conn_.fd == -1) { @@ -1012,7 +1019,7 @@ int HttpDownstreamConnection::process_input(const uint8_t *data, } int HttpDownstreamConnection::connected() { - auto connect_blocker = client_handler_->get_connect_blocker(); + auto connect_blocker = addr_->connect_blocker; if (!util::check_socket_connected(conn_.fd)) { conn_.wlimit.stopw(); @@ -1021,6 +1028,8 @@ int HttpDownstreamConnection::connected() { DLOG(INFO, this) << "downstream connect failed"; } + connect_blocker->on_failure(); + downstream_->set_request_state(Downstream::CONNECT_FAIL); return -1; diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 604f9058..566ac334 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -80,7 +80,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, cl_ssl_ctx_(cl_ssl_ctx), cert_tree_(cert_tree), ticket_keys_(ticket_keys), - connect_blocker_(make_unique(loop_)), + downstream_addr_groups_(get_config()->conn.downstream.addr_groups), graceful_shutdown_(false) { ev_async_init(&w_, eventcb); w_.data = this; @@ -109,17 +109,29 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, m = downstreamconf.addr_groups[group].addrs.size(); } for (size_t idx = 0; idx < m; ++idx) { - dgrp.http2sessions.push_back(make_unique( - loop_, cl_ssl_ctx, connect_blocker_.get(), this, group, idx)); + dgrp.http2sessions.push_back( + make_unique(loop_, cl_ssl_ctx, this, group, idx)); } ++group; } } + + for (auto &group : downstream_addr_groups_) { + for (auto &addr : group.addrs) { + addr.connect_blocker = new ConnectBlocker(randgen_, loop_); + } + } } Worker::~Worker() { ev_async_stop(loop_, &w_); ev_timer_stop(loop_, &mcpool_clear_timer_); + + for (auto &group : downstream_addr_groups_) { + for (auto &addr : group.addrs) { + delete addr.connect_blocker; + } + } } void Worker::schedule_clear_mcpool() { @@ -259,10 +271,6 @@ Http2Session *Worker::next_http2_session(size_t group) { return res; } -ConnectBlocker *Worker::get_connect_blocker() const { - return connect_blocker_.get(); -} - struct ev_loop *Worker::get_loop() const { return loop_; } @@ -361,4 +369,8 @@ SSL_SESSION *Worker::reuse_client_tls_session(const Address *addr) { return d2i_SSL_SESSION(nullptr, &p, ent.session_data.size()); } +std::vector &Worker::get_downstream_addr_groups() { + return downstream_addr_groups_; +} + } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 8d352429..e56595c1 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -131,7 +131,6 @@ public: WorkerStat *get_worker_stat(); DownstreamConnectionPool *get_dconn_pool(); Http2Session *next_http2_session(size_t group); - ConnectBlocker *get_connect_blocker() const; struct ev_loop *get_loop() const; SSL_CTX *get_sv_ssl_ctx() const; SSL_CTX *get_cl_ssl_ctx() const; @@ -164,6 +163,8 @@ public: // found associated to |addr|, nullptr will be returned. SSL_SESSION *reuse_client_tls_session(const Address *addr); + std::vector &get_downstream_addr_groups(); + private: #ifndef NOTHREADS std::future fut_; @@ -196,7 +197,7 @@ private: ssl::CertLookupTree *cert_tree_; std::shared_ptr ticket_keys_; - std::unique_ptr connect_blocker_; + std::vector downstream_addr_groups_; bool graceful_shutdown_; }; From 11c8803b92504eaff42550a174532f9a5459e756 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 15:27:19 +0900 Subject: [PATCH 041/147] nghttpx: Worker wide blocker which is used when socket(2) is failed --- src/shrpx_http2_session.cc | 19 ++++++++++++++++--- src/shrpx_http_downstream_connection.cc | 11 ++++++++++- src/shrpx_worker.cc | 5 +++++ src/shrpx_worker.h | 5 +++++ 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 1f660649..4cb22ce1 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -254,8 +254,15 @@ int Http2Session::initiate_connection() { auto &groups = worker_->get_downstream_addr_groups(); auto &addrs = groups[group_].addrs; + auto worker_blocker = worker_->get_connect_blocker(); if (state_ == DISCONNECTED) { + if (worker_blocker->blocked()) { + DLOG(INFO, this) + << "Worker wide backend connection was blocked temporarily"; + return -1; + } + auto &next_downstream = worker_->get_dgrp(group_)->next; auto end = next_downstream; @@ -305,10 +312,12 @@ int Http2Session::initiate_connection() { conn_.fd = util::create_nonblock_socket(proxy.addr.su.storage.ss_family); if (conn_.fd == -1) { - connect_blocker->on_failure(); + worker_blocker->on_failure(); return -1; } + worker_blocker->on_success(); + rv = connect(conn_.fd, &proxy.addr.su.sa, proxy.addr.len); if (rv != 0 && errno != EINPROGRESS) { SSLOG(ERROR, this) << "Failed to connect to the proxy " << proxy.host @@ -373,10 +382,12 @@ int Http2Session::initiate_connection() { conn_.fd = util::create_nonblock_socket(addr_->addr.su.storage.ss_family); if (conn_.fd == -1) { - connect_blocker->on_failure(); + worker_blocker->on_failure(); return -1; } + worker_blocker->on_success(); + rv = connect(conn_.fd, // TODO maybe not thread-safe? const_cast(&addr_->addr.su.sa), @@ -400,10 +411,12 @@ int Http2Session::initiate_connection() { util::create_nonblock_socket(addr_->addr.su.storage.ss_family); if (conn_.fd == -1) { - connect_blocker->on_failure(); + worker_blocker->on_failure(); return -1; } + worker_blocker->on_success(); + rv = connect(conn_.fd, const_cast(&addr_->addr.su.sa), addr_->addr.len); if (rv != 0 && errno != EINPROGRESS) { diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 151bd45b..81acfcd0 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -144,6 +144,13 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { DCLOG(INFO, this) << "Attaching to DOWNSTREAM:" << downstream; } + auto worker_blocker = worker_->get_connect_blocker(); + if (worker_blocker->blocked()) { + DLOG(INFO, this) + << "Worker wide backend connection was blocked temporarily"; + return SHRPX_ERR_NETWORK; + } + auto &downstreamconf = get_config()->conn.downstream; if (conn_.fd == -1) { @@ -189,11 +196,13 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { auto error = errno; DCLOG(WARN, this) << "socket() failed; errno=" << error; - connect_blocker->on_failure(); + worker_blocker->on_failure(); return SHRPX_ERR_NETWORK; } + worker_blocker->on_success(); + int rv; rv = connect(conn_.fd, &addr.addr.su.sa, addr.addr.len); if (rv != 0 && errno != EINPROGRESS) { diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 566ac334..997138db 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -81,6 +81,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, cert_tree_(cert_tree), ticket_keys_(ticket_keys), downstream_addr_groups_(get_config()->conn.downstream.addr_groups), + connect_blocker_(make_unique(randgen_, loop_)), graceful_shutdown_(false) { ev_async_init(&w_, eventcb); w_.data = this; @@ -373,4 +374,8 @@ std::vector &Worker::get_downstream_addr_groups() { return downstream_addr_groups_; } +ConnectBlocker *Worker::get_connect_blocker() const { + return connect_blocker_.get(); +} + } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index e56595c1..9cd8b5ff 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -165,6 +165,8 @@ public: std::vector &get_downstream_addr_groups(); + ConnectBlocker *get_connect_blocker() const; + private: #ifndef NOTHREADS std::future fut_; @@ -198,6 +200,9 @@ private: std::shared_ptr ticket_keys_; std::vector downstream_addr_groups_; + // Worker level blocker for downstream connection. For example, + // this is used when file decriptor is exhausted. + std::unique_ptr connect_blocker_; bool graceful_shutdown_; }; From dfc02843b6342d9c6617bcd2bb1ebe424683aedb Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 15:28:11 +0900 Subject: [PATCH 042/147] src: Rename and rewrite numeric_hostport as to_numeric_addr and support AF_UNIX path --- src/Makefile.am | 2 +- src/network.h | 61 ++++++++++++++++++++++++++++++++ src/shrpx.cc | 10 +++--- src/shrpx_config.h | 14 +------- src/shrpx_memcached_connection.h | 3 +- src/shrpx_memcached_dispatcher.h | 2 +- src/shrpx_ssl.h | 3 +- src/shrpx_worker.cc | 13 +++---- src/util.cc | 14 ++++---- src/util.h | 10 +++--- 10 files changed, 91 insertions(+), 41 deletions(-) create mode 100644 src/network.h diff --git a/src/Makefile.am b/src/Makefile.am index 9c2cfa21..2c873893 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -62,7 +62,7 @@ HELPER_OBJECTS = util.cc \ http2.cc timegm.c app_helper.cc nghttp2_gzip.c HELPER_HFILES = util.h \ http2.h timegm.h app_helper.h nghttp2_config.h \ - nghttp2_gzip.h + nghttp2_gzip.h network.h HTML_PARSER_OBJECTS = HTML_PARSER_HFILES = HtmlParser.h diff --git a/src/network.h b/src/network.h new file mode 100644 index 00000000..3fb765d0 --- /dev/null +++ b/src/network.h @@ -0,0 +1,61 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef NETWORK_H +#define NETWORK_H + +#ifdef HAVE_CONFIG_H +#include +#endif // HAVE_CONFIG_H + +#include +#ifdef HAVE_SYS_SOCKET_H +#include +#endif // HAVE_SYS_SOCKET_H +#include +#ifdef HAVE_NETINET_IN_H +#include +#endif // HAVE_NETINET_IN_H +#ifdef HAVE_ARPA_INET_H +#include +#endif // HAVE_ARPA_INET_H + +namespace nghttp2 { + +union sockaddr_union { + sockaddr_storage storage; + sockaddr sa; + sockaddr_in6 in6; + sockaddr_in in; + sockaddr_un un; +}; + +struct Address { + size_t len; + union sockaddr_union su; +}; + +} // namespace nghttp2 + +#endif // NETWORK_H diff --git a/src/shrpx.cc b/src/shrpx.cc index 0c1fb3f7..f84f10ea 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2201,7 +2201,7 @@ void process_options( exit(EXIT_FAILURE); } LOG(NOTICE) << "Resolved backend address: " << hostport << " -> " - << util::numeric_hostport(&addr.addr.su.sa, addr.addr.len); + << util::to_numeric_addr(&addr.addr); } } @@ -2214,7 +2214,7 @@ void process_options( exit(EXIT_FAILURE); } LOG(NOTICE) << "Backend HTTP proxy address: " << hostport << " -> " - << util::numeric_hostport(&proxy.addr.su.sa, proxy.addr.len); + << util::to_numeric_addr(&proxy.addr); } { @@ -2230,8 +2230,7 @@ void process_options( exit(EXIT_FAILURE); } LOG(NOTICE) << "Memcached address for TLS session cache: " << hostport - << " -> " << util::numeric_hostport(&memcachedconf.addr.su.sa, - memcachedconf.addr.len); + << " -> " << util::to_numeric_addr(&memcachedconf.addr); } } @@ -2247,8 +2246,7 @@ void process_options( exit(EXIT_FAILURE); } LOG(NOTICE) << "Memcached address for TLS ticket key: " << hostport - << " -> " << util::numeric_hostport(&memcachedconf.addr.su.sa, - memcachedconf.addr.len); + << " -> " << util::to_numeric_addr(&memcachedconf.addr); } } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 7913fa19..944099d4 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -53,6 +53,7 @@ #include "shrpx_router.h" #include "template.h" #include "http2.h" +#include "network.h" using namespace nghttp2; @@ -231,19 +232,6 @@ constexpr char SHRPX_OPT_BACKEND_ADDRESS_FAMILY[] = "backend-address-family"; constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; -union sockaddr_union { - sockaddr_storage storage; - sockaddr sa; - sockaddr_in6 in6; - sockaddr_in in; - sockaddr_un un; -}; - -struct Address { - size_t len; - union sockaddr_union su; -}; - enum shrpx_proto { PROTO_HTTP2, PROTO_HTTP }; enum shrpx_forwarded_param { diff --git a/src/shrpx_memcached_connection.h b/src/shrpx_memcached_connection.h index c9552198..a093589a 100644 --- a/src/shrpx_memcached_connection.h +++ b/src/shrpx_memcached_connection.h @@ -35,12 +35,13 @@ #include "shrpx_connection.h" #include "buffer.h" +#include "network.h" + using namespace nghttp2; namespace shrpx { struct MemcachedRequest; -struct Address; enum { MEMCACHED_PARSE_HEADER24, diff --git a/src/shrpx_memcached_dispatcher.h b/src/shrpx_memcached_dispatcher.h index 64021c68..7cca3bcb 100644 --- a/src/shrpx_memcached_dispatcher.h +++ b/src/shrpx_memcached_dispatcher.h @@ -34,12 +34,12 @@ #include #include "memchunk.h" +#include "network.h" namespace shrpx { struct MemcachedRequest; class MemcachedConnection; -struct Address; class MemcachedDispatcher { public: diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h index ca93da14..0def8875 100644 --- a/src/shrpx_ssl.h +++ b/src/shrpx_ssl.h @@ -39,6 +39,8 @@ #include #endif // HAVE_NEVERBLEED +#include "network.h" + namespace shrpx { class ClientHandler; @@ -46,7 +48,6 @@ class Worker; class DownstreamConnectionPool; struct DownstreamAddr; struct UpstreamAddr; -struct Address; namespace ssl { diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 997138db..4b0fd452 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -329,9 +329,8 @@ void Worker::cache_client_tls_session(const Address *addr, SSL_SESSION *session, if (it == std::end(client_tls_session_cache_)) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Create cache entry for SSL_SESSION=" << session - << ", addr=" << util::numeric_hostport(&addr->su.sa, addr->len) - << "(" << addr << "), timestamp=" << std::fixed - << std::setprecision(6) << t; + << ", addr=" << util::to_numeric_addr(addr) << "(" << addr + << "), timestamp=" << std::fixed << std::setprecision(6) << t; } client_tls_session_cache_.emplace( addr, SessionCacheEntry{serialize_ssl_session(session), t}); @@ -341,8 +340,7 @@ void Worker::cache_client_tls_session(const Address *addr, SSL_SESSION *session, auto &ent = (*it).second; if (ent.last_updated + 1_min > t) { if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Cache for addr=" - << util::numeric_hostport(&addr->su.sa, addr->len) << "(" + LOG(INFO) << "Cache for addr=" << util::to_numeric_addr(addr) << "(" << addr << ") is still host. Not updating."; } return; @@ -350,9 +348,8 @@ void Worker::cache_client_tls_session(const Address *addr, SSL_SESSION *session, if (LOG_ENABLED(INFO)) { LOG(INFO) << "Update cache entry for SSL_SESSION=" << session - << ", addr=" << util::numeric_hostport(&addr->su.sa, addr->len) - << "(" << addr << "), timestamp=" << std::fixed - << std::setprecision(6) << t; + << ", addr=" << util::to_numeric_addr(addr) << "(" << addr + << "), timestamp=" << std::fixed << std::setprecision(6) << t; } ent.session_data = serialize_ssl_session(session); diff --git a/src/util.cc b/src/util.cc index 0897a0f7..12b90594 100644 --- a/src/util.cc +++ b/src/util.cc @@ -650,15 +650,17 @@ std::string numeric_name(const struct sockaddr *sa, socklen_t salen) { return host.data(); } -std::string numeric_hostport(const struct sockaddr *sa, socklen_t salen) { - if (sa->sa_family == AF_UNIX) { - return "localhost"; +std::string to_numeric_addr(const Address *addr) { + auto family = addr->su.storage.ss_family; + if (family == AF_UNIX) { + return addr->su.un.sun_path; } std::array host; std::array serv; - auto rv = getnameinfo(sa, salen, host.data(), host.size(), serv.data(), - serv.size(), NI_NUMERICHOST | NI_NUMERICSERV); + auto rv = + getnameinfo(&addr->su.sa, addr->len, host.data(), host.size(), + serv.data(), serv.size(), NI_NUMERICHOST | NI_NUMERICSERV); if (rv != 0) { return "unknown"; } @@ -668,7 +670,7 @@ std::string numeric_hostport(const struct sockaddr *sa, socklen_t salen) { std::string s; char *p; - if (sa->sa_family == AF_INET6) { + if (family == AF_INET6) { s.resize(hostlen + servlen + 2 + 1); p = &s[0]; *p++ = '['; diff --git a/src/util.h b/src/util.h index b1f64d77..f4b1c2a0 100644 --- a/src/util.h +++ b/src/util.h @@ -50,6 +50,7 @@ #include "http-parser/http_parser.h" #include "template.h" +#include "network.h" namespace nghttp2 { @@ -461,10 +462,11 @@ bool numeric_host(const char *hostname, int family); // failed, "unknown" is returned. std::string numeric_name(const struct sockaddr *sa, socklen_t salen); -// Returns string representation of numeric address and port of |addr| -// of length |salen|. The format is like :. For IPv6 -// address, address is enclosed by square brackets ([]). -std::string numeric_hostport(const struct sockaddr *sa, socklen_t salen); +// Returns string representation of numeric address and port of +// |addr|. If address family is AF_UNIX, this return path to UNIX +// domain socket. Otherwise, the format is like :. For +// IPv6 address, address is enclosed by square brackets ([]). +std::string to_numeric_addr(const Address *addr); // Makes internal copy of stderr (and possibly stdout in the future), // which is then used as pointer to /dev/stderr or /proc/self/fd/2 From 177d0a513f3f7e10ec08444404aceb124afede66 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 16:11:50 +0900 Subject: [PATCH 043/147] nghttpx: More logging for backend connection initiation --- src/shrpx_http2_session.cc | 47 +++++++++++++++++++++---- src/shrpx_http_downstream_connection.cc | 21 +++++++---- 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 4cb22ce1..f7ac9a82 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -258,8 +258,10 @@ int Http2Session::initiate_connection() { if (state_ == DISCONNECTED) { if (worker_blocker->blocked()) { - DLOG(INFO, this) - << "Worker wide backend connection was blocked temporarily"; + if (LOG_ENABLED(INFO)) { + SSLOG(INFO, this) + << "Worker wide backend connection was blocked temporarily"; + } return -1; } @@ -277,8 +279,8 @@ int Http2Session::initiate_connection() { if (connect_blocker->blocked()) { if (LOG_ENABLED(INFO)) { - DCLOG(INFO, this) << "Backend server " - << (addr.host_unix ? addr.host : addr.hostport) + SSLOG(INFO, this) << "Backend server " + << util::to_numeric_addr(&addr.addr) << " was not available temporarily"; } @@ -312,6 +314,11 @@ int Http2Session::initiate_connection() { conn_.fd = util::create_nonblock_socket(proxy.addr.su.storage.ss_family); if (conn_.fd == -1) { + auto error = errno; + SSLOG(WARN, this) << "Backend proxy socket() failed; addr=" + << util::to_numeric_addr(&proxy.addr) + << ", errno=" << error; + worker_blocker->on_failure(); return -1; } @@ -320,8 +327,11 @@ int Http2Session::initiate_connection() { rv = connect(conn_.fd, &proxy.addr.su.sa, proxy.addr.len); if (rv != 0 && errno != EINPROGRESS) { - SSLOG(ERROR, this) << "Failed to connect to the proxy " << proxy.host - << ":" << proxy.port; + auto error = errno; + SSLOG(WARN, this) << "Backend proxy connect() failed; addr=" + << util::to_numeric_addr(&proxy.addr) + << ", errno=" << error; + connect_blocker->on_failure(); return -1; } @@ -382,6 +392,11 @@ int Http2Session::initiate_connection() { conn_.fd = util::create_nonblock_socket(addr_->addr.su.storage.ss_family); if (conn_.fd == -1) { + auto error = errno; + SSLOG(WARN, this) + << "socket() failed; addr=" << util::to_numeric_addr(&addr_->addr) + << ", errno=" << error; + worker_blocker->on_failure(); return -1; } @@ -393,6 +408,11 @@ int Http2Session::initiate_connection() { const_cast(&addr_->addr.su.sa), addr_->addr.len); if (rv != 0 && errno != EINPROGRESS) { + auto error = errno; + SSLOG(WARN, this) << "connect() failed; addr=" + << util::to_numeric_addr(&addr_->addr) + << ", errno=" << error; + connect_blocker->on_failure(); return -1; } @@ -411,6 +431,11 @@ int Http2Session::initiate_connection() { util::create_nonblock_socket(addr_->addr.su.storage.ss_family); if (conn_.fd == -1) { + auto error = errno; + SSLOG(WARN, this) + << "socket() failed; addr=" << util::to_numeric_addr(&addr_->addr) + << ", errno=" << error; + worker_blocker->on_failure(); return -1; } @@ -420,6 +445,11 @@ int Http2Session::initiate_connection() { rv = connect(conn_.fd, const_cast(&addr_->addr.su.sa), addr_->addr.len); if (rv != 0 && errno != EINPROGRESS) { + auto error = errno; + SSLOG(WARN, this) << "connect() failed; addr=" + << util::to_numeric_addr(&addr_->addr) + << ", errno=" << error; + connect_blocker->on_failure(); return -1; } @@ -1648,6 +1678,11 @@ int Http2Session::connected() { auto &connect_blocker = addr_->connect_blocker; if (!util::check_socket_connected(conn_.fd)) { + if (LOG_ENABLED(INFO)) { + SSLOG(INFO, this) << "Backend connect failed; addr=" + << util::to_numeric_addr(&addr_->addr); + } + connect_blocker->on_failure(); return -1; diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 81acfcd0..86b9119b 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -146,8 +146,10 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { auto worker_blocker = worker_->get_connect_blocker(); if (worker_blocker->blocked()) { - DLOG(INFO, this) - << "Worker wide backend connection was blocked temporarily"; + if (LOG_ENABLED(INFO)) { + DCLOG(INFO, this) + << "Worker wide backend connection was blocked temporarily"; + } return SHRPX_ERR_NETWORK; } @@ -179,7 +181,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { if (connect_blocker->blocked()) { if (LOG_ENABLED(INFO)) { DCLOG(INFO, this) << "Backend server " - << (addr.host_unix ? addr.host : addr.hostport) + << util::to_numeric_addr(&addr.addr) << " was not available temporarily"; } @@ -194,7 +196,9 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { if (conn_.fd == -1) { auto error = errno; - DCLOG(WARN, this) << "socket() failed; errno=" << error; + DCLOG(WARN, this) << "socket() failed; addr=" + << util::to_numeric_addr(&addr.addr) + << ", errno=" << error; worker_blocker->on_failure(); @@ -207,7 +211,9 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { rv = connect(conn_.fd, &addr.addr.su.sa, addr.addr.len); if (rv != 0 && errno != EINPROGRESS) { auto error = errno; - DCLOG(WARN, this) << "connect() failed; errno=" << error; + DCLOG(WARN, this) << "connect() failed; addr=" + << util::to_numeric_addr(&addr.addr) + << ", errno=" << error; connect_blocker->on_failure(); close(conn_.fd); @@ -1034,7 +1040,8 @@ int HttpDownstreamConnection::connected() { conn_.wlimit.stopw(); if (LOG_ENABLED(INFO)) { - DLOG(INFO, this) << "downstream connect failed"; + DCLOG(INFO, this) << "Backend connect failed; addr=" + << util::to_numeric_addr(&addr_->addr); } connect_blocker->on_failure(); @@ -1045,7 +1052,7 @@ int HttpDownstreamConnection::connected() { } if (LOG_ENABLED(INFO)) { - DLOG(INFO, this) << "Connected to downstream host"; + DCLOG(INFO, this) << "Connected to downstream host"; } connect_blocker->on_success(); From f2a72757001e56d9dfc3a1fb85ca7aeeac4818a1 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 16:35:43 +0900 Subject: [PATCH 044/147] nghttpx: Cache TLS session inside DownstreamAddr object --- src/shrpx_config.h | 10 +++++ src/shrpx_http_downstream_connection.cc | 19 ++++----- src/shrpx_http_downstream_connection.h | 2 +- src/shrpx_ssl.cc | 45 ++++++++++++++++++++ src/shrpx_ssl.h | 11 +++++ src/shrpx_worker.cc | 56 ------------------------- src/shrpx_worker.h | 23 ---------- 7 files changed, 76 insertions(+), 90 deletions(-) diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 944099d4..1a760d9e 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -277,6 +277,14 @@ struct UpstreamAddr { int fd; }; +struct TLSSessionCache { + // ASN1 representation of SSL_SESSION object. See + // i2d_SSL_SESSION(3SSL). + std::vector session_data; + // The last time stamp when this cache entry is created or updated. + ev_tstamp last_updated; +}; + struct DownstreamAddr { Address addr; // backend address. If |host_unix| is true, this is UNIX domain @@ -284,6 +292,8 @@ struct DownstreamAddr { ImmutableString host; ImmutableString hostport; ConnectBlocker *connect_blocker; + // Client side TLS session cache + TLSSessionCache tls_session_cache; // backend port. 0 if |host_unix| is true. uint16_t port; // true if |host| contains UNIX domain socket path. diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 86b9119b..f0b6fd3c 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -129,15 +129,7 @@ HttpDownstreamConnection::HttpDownstreamConnection( response_htp_{0}, group_(group) {} -HttpDownstreamConnection::~HttpDownstreamConnection() { - if (conn_.tls.ssl && conn_.tls.initial_handshake_done) { - auto session = SSL_get0_session(conn_.tls.ssl); - if (session) { - worker_->cache_client_tls_session(&addr_->addr, session, - ev_now(conn_.loop)); - } - } -} +HttpDownstreamConnection::~HttpDownstreamConnection() {} int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { if (LOG_ENABLED(INFO)) { @@ -241,7 +233,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str()); } - auto session = worker_->reuse_client_tls_session(&addr_->addr); + auto session = ssl::reuse_tls_session(addr_); if (session) { SSL_set_session(conn_.tls.ssl, session); SSL_SESSION_free(session); @@ -891,6 +883,13 @@ int HttpDownstreamConnection::tls_handshake() { return -1; } + if (!SSL_session_reused(conn_.tls.ssl)) { + auto session = SSL_get0_session(conn_.tls.ssl); + if (session) { + ssl::try_cache_tls_session(addr_, session, ev_now(conn_.loop)); + } + } + do_read_ = &HttpDownstreamConnection::read_tls; do_write_ = &HttpDownstreamConnection::write_tls; diff --git a/src/shrpx_http_downstream_connection.h b/src/shrpx_http_downstream_connection.h index b65db9ac..9689283d 100644 --- a/src/shrpx_http_downstream_connection.h +++ b/src/shrpx_http_downstream_connection.h @@ -82,7 +82,7 @@ private: // nullptr if TLS is not used. SSL_CTX *ssl_ctx_; // Address of remote endpoint - const DownstreamAddr *addr_; + DownstreamAddr *addr_; IOControl ioctrl_; http_parser response_htp_; size_t group_; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index aff475e6..2f9d200a 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -36,6 +36,7 @@ #include #include +#include #include #include @@ -1335,6 +1336,50 @@ CertLookupTree *create_cert_lookup_tree() { return new ssl::CertLookupTree(); } +namespace { +std::vector serialize_ssl_session(SSL_SESSION *session) { + auto len = i2d_SSL_SESSION(session, nullptr); + auto buf = std::vector(len); + auto p = buf.data(); + i2d_SSL_SESSION(session, &p); + + return buf; +} +} // namespace + +void try_cache_tls_session(DownstreamAddr *addr, SSL_SESSION *session, + ev_tstamp t) { + auto &cache = addr->tls_session_cache; + + if (cache.last_updated + 1_min > t) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Cache for addr=" << util::to_numeric_addr(&addr->addr) + << " is still host. Not updating."; + } + return; + } + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Update cache entry for SSL_SESSION=" << session + << ", addr=" << util::to_numeric_addr(&addr->addr) + << ", timestamp=" << std::fixed << std::setprecision(6) << t; + } + + cache.session_data = serialize_ssl_session(session); + cache.last_updated = t; +} + +SSL_SESSION *reuse_tls_session(const DownstreamAddr *addr) { + auto &cache = addr->tls_session_cache; + + if (cache.session_data.empty()) { + return nullptr; + } + + auto p = cache.session_data.data(); + return d2i_SSL_SESSION(nullptr, &p, cache.session_data.size()); +} + } // namespace ssl } // namespace shrpx diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h index 0def8875..c913e5b9 100644 --- a/src/shrpx_ssl.h +++ b/src/shrpx_ssl.h @@ -217,6 +217,17 @@ bool downstream_tls_enabled(); bool tls_hostname_match(const char *pattern, size_t plen, const char *hostname, size_t hlen); +// Caches |session| which is associated to remote address |addr|. +// |session| is serialized into ASN1 representation, and stored. |t| +// is used as a time stamp. Depending on the existing cache's time +// stamp, |session| might not be cached. +void try_cache_tls_session(DownstreamAddr *addr, SSL_SESSION *session, + ev_tstamp t); + +// Returns cached session associated |addr|. If no cache entry is +// found associated to |addr|, nullptr will be returned. +SSL_SESSION *reuse_tls_session(const DownstreamAddr *addr); + } // namespace ssl } // namespace shrpx diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 4b0fd452..2cb71acc 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -29,7 +29,6 @@ #endif // HAVE_UNISTD_H #include -#include #include "shrpx_ssl.h" #include "shrpx_log.h" @@ -312,61 +311,6 @@ mruby::MRubyContext *Worker::get_mruby_context() const { } #endif // HAVE_MRUBY -namespace { -std::vector serialize_ssl_session(SSL_SESSION *session) { - auto len = i2d_SSL_SESSION(session, nullptr); - auto buf = std::vector(len); - auto p = buf.data(); - i2d_SSL_SESSION(session, &p); - - return buf; -} -} // namespace - -void Worker::cache_client_tls_session(const Address *addr, SSL_SESSION *session, - ev_tstamp t) { - auto it = client_tls_session_cache_.find(addr); - if (it == std::end(client_tls_session_cache_)) { - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Create cache entry for SSL_SESSION=" << session - << ", addr=" << util::to_numeric_addr(addr) << "(" << addr - << "), timestamp=" << std::fixed << std::setprecision(6) << t; - } - client_tls_session_cache_.emplace( - addr, SessionCacheEntry{serialize_ssl_session(session), t}); - return; - } - - auto &ent = (*it).second; - if (ent.last_updated + 1_min > t) { - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Cache for addr=" << util::to_numeric_addr(addr) << "(" - << addr << ") is still host. Not updating."; - } - return; - } - - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Update cache entry for SSL_SESSION=" << session - << ", addr=" << util::to_numeric_addr(addr) << "(" << addr - << "), timestamp=" << std::fixed << std::setprecision(6) << t; - } - - ent.session_data = serialize_ssl_session(session); - ent.last_updated = t; -} - -SSL_SESSION *Worker::reuse_client_tls_session(const Address *addr) { - auto it = client_tls_session_cache_.find(addr); - if (it == std::end(client_tls_session_cache_)) { - return nullptr; - } - - const auto &ent = (*it).second; - auto p = ent.session_data.data(); - return d2i_SSL_SESSION(nullptr, &p, ent.session_data.size()); -} - std::vector &Worker::get_downstream_addr_groups() { return downstream_addr_groups_; } diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 9cd8b5ff..aa1531e6 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -101,14 +101,6 @@ struct WorkerEvent { std::shared_ptr ticket_keys; }; -struct SessionCacheEntry { - // ASN1 representation of SSL_SESSION object. See - // i2d_SSL_SESSION(3SSL). - std::vector session_data; - // The last time stamp when this cache entry is created or updated. - ev_tstamp last_updated; -}; - class Worker { public: Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, @@ -153,16 +145,6 @@ public: mruby::MRubyContext *get_mruby_context() const; #endif // HAVE_MRUBY - // Caches |session| which is associated to remote address |addr|. - // |session| is serialized into ASN1 representation, and stored. - // |t| is used as a time stamp. Depending on the existing cache's - // time stamp, |session| might not be cached. - void cache_client_tls_session(const Address *addr, SSL_SESSION *session, - ev_tstamp t); - // Returns cached session associated |addr|. If no cache entry is - // found associated to |addr|, nullptr will be returned. - SSL_SESSION *reuse_client_tls_session(const Address *addr); - std::vector &get_downstream_addr_groups(); ConnectBlocker *get_connect_blocker() const; @@ -181,11 +163,6 @@ private: WorkerStat worker_stat_; std::vector dgrps_; - // Client side SSL_SESSION cache. SSL_SESSION is associated to - // remote address. - std::unordered_map - client_tls_session_cache_; - std::unique_ptr session_cache_memcached_dispatcher_; #ifdef HAVE_MRUBY std::unique_ptr mruby_ctx_; From b68be1e1fb5b3f29fc531626279d2a64601918a1 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 16:44:00 +0900 Subject: [PATCH 045/147] src: Make token of type int32_t; we have no reason to use int16_t --- src/http2.cc | 16 ++++++++-------- src/http2.h | 20 ++++++++++---------- src/nghttp.cc | 31 ++----------------------------- src/nghttp.h | 7 ++----- src/shrpx_downstream.cc | 12 ++++++------ src/shrpx_downstream.h | 8 ++++---- 6 files changed, 32 insertions(+), 62 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index eba7aa54..6f347cc2 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -271,7 +271,7 @@ void copy_url_component(std::string &dest, const http_parser_url *u, int field, Headers::value_type to_header(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, - bool no_index, int16_t token) { + bool no_index, int32_t token) { return Header(std::string(reinterpret_cast(name), namelen), std::string(reinterpret_cast(value), valuelen), no_index, token); @@ -279,7 +279,7 @@ Headers::value_type to_header(const uint8_t *name, size_t namelen, void add_header(Headers &nva, const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, - int16_t token) { + int32_t token) { if (valuelen > 0) { size_t i, j; for (i = 0; i < valuelen && (value[i] == ' ' || value[i] == '\t'); ++i) @@ -760,7 +760,7 @@ void init_hdidx(HeaderIndex &hdidx) { std::fill(std::begin(hdidx), std::end(hdidx), -1); } -void index_header(HeaderIndex &hdidx, int16_t token, size_t idx) { +void index_header(HeaderIndex &hdidx, int32_t token, size_t idx) { if (token == -1) { return; } @@ -769,7 +769,7 @@ void index_header(HeaderIndex &hdidx, int16_t token, size_t idx) { } bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, - int16_t token) { + int32_t token) { switch (token) { case HD__AUTHORITY: case HD__METHOD: @@ -782,7 +782,7 @@ bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, } bool check_http2_response_pseudo_header(const HeaderIndex &hdidx, - int16_t token) { + int32_t token) { switch (token) { case HD__STATUS: return hdidx[token] == -1; @@ -791,7 +791,7 @@ bool check_http2_response_pseudo_header(const HeaderIndex &hdidx, } } -bool http2_header_allowed(int16_t token) { +bool http2_header_allowed(int32_t token) { switch (token) { case HD_CONNECTION: case HD_KEEP_ALIVE: @@ -813,7 +813,7 @@ bool http2_mandatory_request_headers_presence(const HeaderIndex &hdidx) { return true; } -const Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, +const Headers::value_type *get_header(const HeaderIndex &hdidx, int32_t token, const Headers &nva) { auto i = hdidx[token]; if (i == -1) { @@ -822,7 +822,7 @@ const Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, return &nva[i]; } -Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, +Headers::value_type *get_header(const HeaderIndex &hdidx, int32_t token, Headers &nva) { auto i = hdidx[token]; if (i == -1) { diff --git a/src/http2.h b/src/http2.h index 8eaeff56..58a4cc00 100644 --- a/src/http2.h +++ b/src/http2.h @@ -44,7 +44,7 @@ namespace nghttp2 { struct Header { Header(std::string name, std::string value, bool no_index = false, - int16_t token = -1) + int32_t token = -1) : name(std::move(name)), value(std::move(value)), token(token), @@ -62,7 +62,7 @@ struct Header { std::string name; std::string value; - int16_t token; + int32_t token; bool no_index; }; @@ -89,14 +89,14 @@ void copy_url_component(std::string &dest, const http_parser_url *u, int field, Headers::value_type to_header(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, - bool no_index, int16_t token); + bool no_index, int32_t token); // Add name/value pairs to |nva|. If |no_index| is true, this // name/value pair won't be indexed when it is forwarded to the next // hop. This function strips white spaces around |value|. void add_header(Headers &nva, const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, - int16_t token); + int32_t token); // Returns pointer to the entry in |nva| which has name |name|. If // more than one entries which have the name |name|, last occurrence @@ -277,30 +277,30 @@ int lookup_token(const std::string &name); // array containing at least HD_MAXIDX elements. void init_hdidx(HeaderIndex &hdidx); // Indexes header |token| using index |idx|. -void index_header(HeaderIndex &hdidx, int16_t token, size_t idx); +void index_header(HeaderIndex &hdidx, int32_t token, size_t idx); // Returns true if HTTP/2 request pseudo header |token| is not indexed // yet and not -1. -bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, int16_t token); +bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, int32_t token); // Returns true if HTTP/2 response pseudo header |token| is not // indexed yet and not -1. bool check_http2_response_pseudo_header(const HeaderIndex &hdidx, - int16_t token); + int32_t token); // Returns true if header field denoted by |token| is allowed for // HTTP/2. -bool http2_header_allowed(int16_t token); +bool http2_header_allowed(int32_t token); // Returns true that |hdidx| contains mandatory HTTP/2 request // headers. bool http2_mandatory_request_headers_presence(const HeaderIndex &hdidx); // Returns header denoted by |token| using index |hdidx|. -const Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, +const Headers::value_type *get_header(const HeaderIndex &hdidx, int32_t token, const Headers &nva); -Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, +Headers::value_type *get_header(const HeaderIndex &hdidx, int32_t token, Headers &nva); struct LinkHeader { diff --git a/src/nghttp.cc b/src/nghttp.cc index e3d4e110..a1705124 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -273,34 +273,7 @@ bool Request::is_ipv6_literal_addr() const { } } -bool Request::response_pseudo_header_allowed(int16_t token) const { - if (!res_nva.empty() && res_nva.back().name.c_str()[0] != ':') { - return false; - } - switch (token) { - case http2::HD__STATUS: - return res_hdidx[token] == -1; - default: - return false; - } -} - -bool Request::push_request_pseudo_header_allowed(int16_t token) const { - if (!req_nva.empty() && req_nva.back().name.c_str()[0] != ':') { - return false; - } - switch (token) { - case http2::HD__AUTHORITY: - case http2::HD__METHOD: - case http2::HD__PATH: - case http2::HD__SCHEME: - return req_hdidx[token] == -1; - default: - return false; - } -} - -Headers::value_type *Request::get_res_header(int16_t token) { +Headers::value_type *Request::get_res_header(int32_t token) { auto idx = res_hdidx[token]; if (idx == -1) { return nullptr; @@ -308,7 +281,7 @@ Headers::value_type *Request::get_res_header(int16_t token) { return &res_nva[idx]; } -Headers::value_type *Request::get_req_header(int16_t token) { +Headers::value_type *Request::get_req_header(int32_t token) { auto idx = req_hdidx[token]; if (idx == -1) { return nullptr; diff --git a/src/nghttp.h b/src/nghttp.h index 715a110e..df4ca685 100644 --- a/src/nghttp.h +++ b/src/nghttp.h @@ -125,11 +125,8 @@ struct Request { bool is_ipv6_literal_addr() const; - bool response_pseudo_header_allowed(int16_t token) const; - bool push_request_pseudo_header_allowed(int16_t token) const; - - Headers::value_type *get_res_header(int16_t token); - Headers::value_type *get_req_header(int16_t token); + Headers::value_type *get_res_header(int32_t token); + Headers::value_type *get_req_header(int32_t token); void record_request_start_time(); void record_response_start_time(); diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 32b7c1f2..0a8da837 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -331,7 +331,7 @@ void Downstream::crumble_request_cookie(std::vector &nva) { namespace { void add_header(bool &key_prev, size_t &sum, Headers &headers, const StringRef &name, const StringRef &value, bool no_index, - int16_t token) { + int32_t token) { key_prev = true; sum += name.size() + value.size(); headers.emplace_back(name.str(), value.str(), no_index, token); @@ -340,7 +340,7 @@ void add_header(bool &key_prev, size_t &sum, Headers &headers, namespace { void add_header(size_t &sum, Headers &headers, const StringRef &name, - const StringRef &value, bool no_index, int16_t token) { + const StringRef &value, bool no_index, int32_t token) { sum += name.size() + value.size(); headers.emplace_back(name.str(), value.str(), no_index, token); } @@ -388,7 +388,7 @@ int FieldStore::parse_content_length() { return 0; } -const Headers::value_type *FieldStore::header(int16_t token) const { +const Headers::value_type *FieldStore::header(int32_t token) const { for (auto it = headers_.rbegin(); it != headers_.rend(); ++it) { auto &kv = *it; if (kv.token == token) { @@ -398,7 +398,7 @@ const Headers::value_type *FieldStore::header(int16_t token) const { return nullptr; } -Headers::value_type *FieldStore::header(int16_t token) { +Headers::value_type *FieldStore::header(int32_t token) { for (auto it = headers_.rbegin(); it != headers_.rend(); ++it) { auto &kv = *it; if (kv.token == token) { @@ -422,7 +422,7 @@ void FieldStore::add_header_lower(const StringRef &name, const StringRef &value, } void FieldStore::add_header_token(const StringRef &name, const StringRef &value, - bool no_index, int16_t token) { + bool no_index, int32_t token) { shrpx::add_header(buffer_size_, headers_, name, value, no_index, token); } @@ -449,7 +449,7 @@ void FieldStore::add_trailer_lower(const StringRef &name, void FieldStore::add_trailer_token(const StringRef &name, const StringRef &value, bool no_index, - int16_t token) { + int32_t token) { // Header size limit should be applied to all header and trailer // fields combined. shrpx::add_header(buffer_size_, trailers_, name, value, no_index, token); diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 4306940d..3864674f 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -73,8 +73,8 @@ public: // multiple header have |name| as name, return last occurrence from // the beginning. If no such header is found, returns nullptr. // This function must be called after headers are indexed - const Headers::value_type *header(int16_t token) const; - Headers::value_type *header(int16_t token); + const Headers::value_type *header(int32_t token) const; + Headers::value_type *header(int32_t token); // Returns pointer to the header field with the name |name|. If no // such header is found, returns nullptr. const Headers::value_type *header(const StringRef &name) const; @@ -82,7 +82,7 @@ public: void add_header_lower(const StringRef &name, const StringRef &value, bool no_index); void add_header_token(const StringRef &name, const StringRef &value, - bool no_index, int16_t token); + bool no_index, int32_t token); void append_last_header_key(const char *data, size_t len); void append_last_header_value(const char *data, size_t len); @@ -99,7 +99,7 @@ public: void add_trailer_lower(const StringRef &name, const StringRef &value, bool no_index); void add_trailer_token(const StringRef &name, const StringRef &value, - bool no_index, int16_t token); + bool no_index, int32_t token); void append_last_trailer_key(const char *data, size_t len); void append_last_trailer_value(const char *data, size_t len); From 9672bc322f5fe744bdd0102a92bec7b34ed71490 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 16:47:06 +0900 Subject: [PATCH 046/147] src: Remove unused functions --- src/http2.cc | 45 ----------------------------------------- src/http2.h | 17 ---------------- src/http2_test.cc | 47 ------------------------------------------- src/http2_test.h | 3 --- src/shrpx-unittest.cc | 6 ------ 5 files changed, 118 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index 6f347cc2..f563596e 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -768,51 +768,6 @@ void index_header(HeaderIndex &hdidx, int32_t token, size_t idx) { hdidx[token] = idx; } -bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, - int32_t token) { - switch (token) { - case HD__AUTHORITY: - case HD__METHOD: - case HD__PATH: - case HD__SCHEME: - return hdidx[token] == -1; - default: - return false; - } -} - -bool check_http2_response_pseudo_header(const HeaderIndex &hdidx, - int32_t token) { - switch (token) { - case HD__STATUS: - return hdidx[token] == -1; - default: - return false; - } -} - -bool http2_header_allowed(int32_t token) { - switch (token) { - case HD_CONNECTION: - case HD_KEEP_ALIVE: - case HD_PROXY_CONNECTION: - case HD_TRANSFER_ENCODING: - case HD_UPGRADE: - return false; - default: - return true; - } -} - -bool http2_mandatory_request_headers_presence(const HeaderIndex &hdidx) { - if (hdidx[HD__METHOD] == -1 || hdidx[HD__PATH] == -1 || - hdidx[HD__SCHEME] == -1 || - (hdidx[HD__AUTHORITY] == -1 && hdidx[HD_HOST] == -1)) { - return false; - } - return true; -} - const Headers::value_type *get_header(const HeaderIndex &hdidx, int32_t token, const Headers &nva) { auto i = hdidx[token]; diff --git a/src/http2.h b/src/http2.h index 58a4cc00..785089c6 100644 --- a/src/http2.h +++ b/src/http2.h @@ -279,23 +279,6 @@ void init_hdidx(HeaderIndex &hdidx); // Indexes header |token| using index |idx|. void index_header(HeaderIndex &hdidx, int32_t token, size_t idx); -// Returns true if HTTP/2 request pseudo header |token| is not indexed -// yet and not -1. -bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, int32_t token); - -// Returns true if HTTP/2 response pseudo header |token| is not -// indexed yet and not -1. -bool check_http2_response_pseudo_header(const HeaderIndex &hdidx, - int32_t token); - -// Returns true if header field denoted by |token| is allowed for -// HTTP/2. -bool http2_header_allowed(int32_t token); - -// Returns true that |hdidx| contains mandatory HTTP/2 request -// headers. -bool http2_mandatory_request_headers_presence(const HeaderIndex &hdidx); - // Returns header denoted by |token| using index |hdidx|. const Headers::value_type *get_header(const HeaderIndex &hdidx, int32_t token, const Headers &nva); diff --git a/src/http2_test.cc b/src/http2_test.cc index 8f3aab35..34f96461 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -271,53 +271,6 @@ void test_http2_lookup_token(void) { CU_ASSERT(http2::HD_EXPECT == http2::lookup_token("expect")); } -void test_http2_check_http2_pseudo_header(void) { - http2::HeaderIndex hdidx; - http2::init_hdidx(hdidx); - - CU_ASSERT(http2::check_http2_request_pseudo_header(hdidx, http2::HD__METHOD)); - hdidx[http2::HD__PATH] = 0; - CU_ASSERT(http2::check_http2_request_pseudo_header(hdidx, http2::HD__METHOD)); - hdidx[http2::HD__METHOD] = 1; - CU_ASSERT( - !http2::check_http2_request_pseudo_header(hdidx, http2::HD__METHOD)); - CU_ASSERT(!http2::check_http2_request_pseudo_header(hdidx, http2::HD_VIA)); - - http2::init_hdidx(hdidx); - - CU_ASSERT( - http2::check_http2_response_pseudo_header(hdidx, http2::HD__STATUS)); - hdidx[http2::HD__STATUS] = 0; - CU_ASSERT( - !http2::check_http2_response_pseudo_header(hdidx, http2::HD__STATUS)); - CU_ASSERT(!http2::check_http2_response_pseudo_header(hdidx, http2::HD_VIA)); -} - -void test_http2_http2_header_allowed(void) { - CU_ASSERT(http2::http2_header_allowed(http2::HD__PATH)); - CU_ASSERT(http2::http2_header_allowed(http2::HD_CONTENT_LENGTH)); - CU_ASSERT(!http2::http2_header_allowed(http2::HD_CONNECTION)); -} - -void test_http2_mandatory_request_headers_presence(void) { - http2::HeaderIndex hdidx; - http2::init_hdidx(hdidx); - - CU_ASSERT(!http2::http2_mandatory_request_headers_presence(hdidx)); - hdidx[http2::HD__AUTHORITY] = 0; - CU_ASSERT(!http2::http2_mandatory_request_headers_presence(hdidx)); - hdidx[http2::HD__METHOD] = 1; - CU_ASSERT(!http2::http2_mandatory_request_headers_presence(hdidx)); - hdidx[http2::HD__PATH] = 2; - CU_ASSERT(!http2::http2_mandatory_request_headers_presence(hdidx)); - hdidx[http2::HD__SCHEME] = 3; - CU_ASSERT(http2::http2_mandatory_request_headers_presence(hdidx)); - - hdidx[http2::HD__AUTHORITY] = -1; - hdidx[http2::HD_HOST] = 0; - CU_ASSERT(http2::http2_mandatory_request_headers_presence(hdidx)); -} - void test_http2_parse_link_header(void) { { // only URI appears; we don't extract URI unless it bears rel=preload diff --git a/src/http2_test.h b/src/http2_test.h index 80f14cd8..029e8e6f 100644 --- a/src/http2_test.h +++ b/src/http2_test.h @@ -40,9 +40,6 @@ void test_http2_rewrite_location_uri(void); void test_http2_parse_http_status_code(void); void test_http2_index_header(void); void test_http2_lookup_token(void); -void test_http2_check_http2_pseudo_header(void); -void test_http2_http2_header_allowed(void); -void test_http2_mandatory_request_headers_presence(void); void test_http2_parse_link_header(void); void test_http2_path_join(void); void test_http2_normalize_path(void); diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index 0e7b17a1..8eba2c65 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -89,12 +89,6 @@ int main(int argc, char *argv[]) { shrpx::test_http2_index_header) || !CU_add_test(pSuite, "http2_lookup_token", shrpx::test_http2_lookup_token) || - !CU_add_test(pSuite, "http2_check_http2_pseudo_header", - shrpx::test_http2_check_http2_pseudo_header) || - !CU_add_test(pSuite, "http2_http2_header_allowed", - shrpx::test_http2_http2_header_allowed) || - !CU_add_test(pSuite, "http2_mandatory_request_headers_presence", - shrpx::test_http2_mandatory_request_headers_presence) || !CU_add_test(pSuite, "http2_parse_link_header", shrpx::test_http2_parse_link_header) || !CU_add_test(pSuite, "http2_path_join", shrpx::test_http2_path_join) || From 216ae0a328b4e9a51b022ff42aaf10a3c72fdc2d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 17:36:10 +0900 Subject: [PATCH 047/147] Update nghttpx documentation --- doc/nghttpx.1.rst | 6 + doc/sources/nghttpx-howto.rst | 212 +++++++++++++++++++++++----------- 2 files changed, 151 insertions(+), 67 deletions(-) diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index 42aa4642..49bb5e27 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -1474,6 +1474,12 @@ addresses: App.new +NOTES +----- + +1. nghttpx - HTTP/2 proxy - HOW-TO + https://nghttp2.org/documentation/nghttpx-howto.html + SEE ALSO -------- diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index 03500013..e38af2c4 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -1,3 +1,5 @@ +.. program:: nghttpx + nghttpx - HTTP/2 proxy - HOW-TO =============================== @@ -10,21 +12,22 @@ also covers some useful options later. Default mode ------------ -If nghttpx is invoked without any ``-s``, ``-p`` and ``--client``, it -operates in default mode. In this mode, nghttpx frontend listens for -HTTP/2 requests and translates them to HTTP/1 requests. Thus it works -as reverse proxy (gateway) for HTTP/2 clients to HTTP/1 web server. -This is also known as "HTTP/2 router". HTTP/1 requests are also -supported in frontend as a fallback. If nghttpx is linked with -spdylay library and frontend connection is SSL/TLS, the frontend also -supports SPDY protocol. +If nghttpx is invoked without any :option:`--http2-proxy`, +:option:`--client`, and :option:`--client-proxy`, it operates in +default mode. In this mode, nghttpx frontend listens for HTTP/2 +requests and translates them to HTTP/1 requests. Thus it works as +reverse proxy (gateway) for HTTP/2 clients to HTTP/1 web server. This +is also known as "HTTP/2 router". HTTP/1 requests are also supported +in frontend as a fallback. If nghttpx is linked with spdylay library +and frontend connection is SSL/TLS, the frontend also supports SPDY +protocol. By default, this mode's frontend connection is encrypted using SSL/TLS. So server's private key and certificate must be supplied to the command line (or through configuration file). In this case, the frontend protocol selection will be done via ALPN or NPN. -With ``--frontend-no-tls`` option, user can turn off SSL/TLS in +With :option:`--frontend-no-tls` option, user can turn off SSL/TLS in frontend connection. In this case, SPDY protocol is not available even if spdylay library is liked to nghttpx. HTTP/2 and HTTP/1 are available on the frontend and a HTTP/1 connection can be upgraded to @@ -32,8 +35,9 @@ HTTP/2 using HTTP Upgrade. Starting HTTP/2 connection by sending HTTP/2 connection preface is also supported. By default, backend HTTP/1 connections are not encrypted. To enable -TLS on HTTP/1 backend connections, use ``--backend-http1-tls`` option. -This applies to all mode whose backend connections are HTTP/1. +TLS on HTTP/1 backend connections, use :option:`--backend-http1-tls` +option. This applies to all mode whose backend connections are +HTTP/1. The backend is supposed to be HTTP/1 Web server. For example, to make nghttpx listen to encrypted HTTP/2 requests at port 8443, and a @@ -50,19 +54,19 @@ example, you can send GET request to the server using nghttp:: HTTP/2 proxy mode ----------------- -If nghttpx is invoked with ``-s`` option, it operates in HTTP/2 proxy -mode. The supported protocols in frontend and backend connections are -the same in `default mode`_. The difference is that this mode acts -like forward proxy and assumes the backend is HTTP/1 proxy server -(e.g., squid, traffic server). So HTTP/1 request must include -absolute URI in request line. +If nghttpx is invoked with :option:`--http2-proxy` (or its shorthand +:option:`-s`) option, it operates in HTTP/2 proxy mode. The supported +protocols in frontend and backend connections are the same in `default +mode`_. The difference is that this mode acts like forward proxy and +assumes the backend is HTTP/1 proxy server (e.g., squid, traffic +server). So HTTP/1 request must include absolute URI in request line. By default, frontend connection is encrypted. So this mode is also called secure proxy. If nghttpx is linked with spdylay, it supports SPDY protocols and it works as so called SPDY proxy. -With ``--frontend-no-tls`` option, SSL/TLS is turned off in frontend -connection, so the connection gets insecure. +With :option:`--frontend-no-tls` option, SSL/TLS is turned off in +frontend connection, so the connection gets insecure. The backend must be HTTP/1 proxy server. nghttpx supports multiple backend server addresses. It translates incoming requests to HTTP/1 @@ -96,7 +100,9 @@ Chromium require valid certificate for secure proxy. For Firefox, open Preference window and select Advanced then click Network tab. Clicking Connection Settings button will show the dialog. Select "Automatic proxy configuration URL" and enter the path -to proxy.pac file, something like this:: +to proxy.pac file, something like this: + +.. code-block:: text file:///path/to/proxy.pac @@ -112,25 +118,27 @@ configuration items to edit:: CONFIG proxy.config.url_remap.remap_required INT 0 Consult Traffic server `documentation -`_ +`_ to know how to configure traffic server as forward proxy and its security implications. Client mode ----------- -If nghttpx is invoked with ``--client`` option, it operates in client -mode. In this mode, nghttpx listens for plain, unencrypted HTTP/2 and -HTTP/1 requests and translates them to encrypted HTTP/2 requests to -the backend. User cannot enable SSL/TLS in frontend connection. +If nghttpx is invoked with :option:`--client` option, it operates in +client mode. In this mode, nghttpx listens for plain, unencrypted +HTTP/2 and HTTP/1 requests and translates them to encrypted HTTP/2 +requests to the backend. User cannot enable SSL/TLS in frontend +connection. HTTP/1 frontend connection can be upgraded to HTTP/2 using HTTP Upgrade. To disable SSL/TLS in backend connection, use -``--backend-no-tls`` option. +:option:`--backend-no-tls` option. By default, the number of backend HTTP/2 connections per worker -(thread) is determined by number of ``-b`` option. To adjust this -value, use ``--backend-http2-connections-per-worker`` option. +(thread) is determined by number of :option:`--backend` option. To +adjust this value, use +:option:`--backend-http2-connections-per-worker` option. The backend server is supporsed to be a HTTP/2 web server (e.g., nghttpd). The one use-case of this mode is utilize existing HTTP/1 @@ -142,9 +150,10 @@ mode to access to that web server:: .. note:: - You may need ``-k`` option if HTTP/2 server enables SSL/TLS and - its certificate is self-signed. But please note that it is - insecure. + You may need :option:`--insecure` (or its shorthand :option:`-k`) + option if HTTP/2 server enables SSL/TLS and its certificate is + self-signed. But please note that it is insecure, and you should + know what you are doing. Then you can use curl to access HTTP/2 server via nghttpx:: @@ -153,18 +162,19 @@ Then you can use curl to access HTTP/2 server via nghttpx:: Client proxy mode ----------------- -If nghttpx is invoked with ``-p`` option, it operates in client proxy -mode. This mode behaves like `client mode`_, but it works like -forward proxy. So HTTP/1 request must include absolute URI in request -line. +If nghttpx is invoked with :option:`--client-proxy` (or its shorthand +:option:`-p`) option, it operates in client proxy mode. This mode +behaves like `client mode`_, but it works like forward proxy. So +HTTP/1 request must include absolute URI in request line. HTTP/1 frontend connection can be upgraded to HTTP/2 using HTTP Upgrade. To disable SSL/TLS in backend connection, use -``--backend-no-tls`` option. +:option:`--backend-no-tls` option. By default, the number of backend HTTP/2 connections per worker -(thread) is determined by number of ``-b`` option. To adjust this -value, use ``--backend-http2-connections-per-worker`` option. +(thread) is determined by number of :option:`--backend` option. To +adjust this value, use +:option:`--backend-http2-connections-per-worker` option. The backend server must be a HTTP/2 proxy. You can use nghttpx in `HTTP/2 proxy mode`_ as backend server. The one use-case of this mode @@ -182,8 +192,9 @@ that server, invoke nghttpx like this:: .. note:: - You may need ``-k`` option if HTTP/2 server's certificate is - self-signed. But please note that it is insecure. + You may need :option:`--insecure` (or its shorthand :option:`-k`) + option if HTTP/2 server's certificate is self-signed. But please + note that it is insecure, and you should know what you are doing. Then you can use curl to issue HTTP request via HTTP/2 proxy:: @@ -195,23 +206,24 @@ proxy. HTTP/2 bridge mode ------------------ -If nghttpx is invoked with ``--http2-bridge`` option, it operates in -HTTP/2 bridge mode. The supported protocols in frontend connections -are the same in `default mode`_. The protocol in backend is HTTP/2 -only. +If nghttpx is invoked with :option:`--http2-bridge` option, it +operates in HTTP/2 bridge mode. The supported protocols in frontend +connections are the same in `default mode`_. The protocol in backend +is HTTP/2 only. -With ``--frontend-no-tls`` option, SSL/TLS is turned off in frontend -connection, so the connection gets insecure. To disable SSL/TLS in -backend connection, use ``--backend-no-tls`` option. +With :option:`--frontend-no-tls` option, SSL/TLS is turned off in +frontend connection, so the connection gets insecure. To disable +SSL/TLS in backend connection, use :option:`--backend-no-tls` option. By default, the number of backend HTTP/2 connections per worker -(thread) is determined by number of ``-b`` option. To adjust this -value, use ``--backend-http2-connections-per-worker`` option. +(thread) is determined by number of :option:`--backend` option. To +adjust this value, use +:option:`--backend-http2-connections-per-worker` option. The backend server is supporsed to be a HTTP/2 web server or HTTP/2 proxy. If backend server is HTTP/2 proxy, use -``--no-location-rewrite`` and ``--no-host-rewrite`` options to disable -rewriting location, host and :authority header field. +:option:`--no-location-rewrite` option to disable rewriting +``Location`` header field. The use-case of this mode is aggregate the incoming connections to one HTTP/2 connection. One backend HTTP/2 connection is created per @@ -222,26 +234,42 @@ Disable SSL/TLS In `default mode`_, `HTTP/2 proxy mode`_ and `HTTP/2 bridge mode`_, frontend connections are encrypted with SSL/TLS by default. To turn -off SSL/TLS, use ``--frontend-no-tls`` option. If this option is -used, the private key and certificate are not required to run nghttpx. +off SSL/TLS, use :option:`--frontend-no-tls` option. If this option +is used, the private key and certificate are not required to run +nghttpx. In `client mode`_, `client proxy mode`_ and `HTTP/2 bridge mode`_, backend connections are encrypted with SSL/TLS by default. To turn -off SSL/TLS, use ``--backend-no-tls`` option. +off SSL/TLS, use :option:`--backend-no-tls` option. + +Enable SSL/TLS on HTTP/1 backend +-------------------------------- + +In all modes which use HTTP/1 as backend protocol, backend HTTP/1 +connection is not encrypted by default. To enable encryption, use +:option:`--backend-http1-tls` option. + +Enable SSL/TLS on memcached connection +-------------------------------------- + +By default, memcached connection is not encrypted. To enable +encryption, use :option:`--tls-ticket-key-memcached-tls` for TLS +ticket key, and use :option:`--tls-session-cache-memcached-tls` for +TLS session cache. Specifying additional CA certificate ------------------------------------ By default, nghttpx tries to read CA certificate from system. But depending on the system you use, this may fail or is not supported. -To specify CA certificate manually, use ``--cacert`` option. The -specified file must be PEM format and can contain multiple +To specify CA certificate manually, use :option:`--cacert` option. +The specified file must be PEM format and can contain multiple certificates. By default, nghttpx validates server's certificate. If you want to turn off this validation, knowing this is really insecure and what you -are doing, you can use ``-k`` option to disable certificate -validation. +are doing, you can use :option:`--insecure` option to disable +certificate validation. Read/write rate limit --------------------- @@ -250,9 +278,9 @@ nghttpx supports transfer rate limiting on frontend connections. You can do rate limit per frontend connection for reading and writing individually. -To perform rate limit for reading, use ``--read-rate`` and -``--read-burst`` options. For writing, use ``--write-rate`` and -``--write-burst``. +To perform rate limit for reading, use :option:`--read-rate` and +:option:`--read-burst` options. For writing, use +:option:`--write-rate` and :option:`--write-burst`. Please note that rate limit is performed on top of TCP and nothing to do with HTTP/2 flow control. @@ -294,14 +322,64 @@ Re-opening log files When rotating log files, it is desirable to re-open log files after log rotation daemon renamed existing log files. To tell nghttpx to re-open log files, send USR1 signal to nghttpx process. It will -re-open files specified by ``--accesslog-file`` and -``--errorlog-file`` options. +re-open files specified by :option:`--accesslog-file` and +:option:`--errorlog-file` options. Multiple backend addresses -------------------------- nghttpx supports multiple backend addresses. To specify them, just -use ``-b`` option repeatedly. For example, to use backend1:8080 and -backend2:8080, use command-line like this: ``-bbackend1,8080 --bbackend2,8080``. For HTTP/2 backend, see also -``--backend-http2-connections-per-worker`` option. +use :option:`--backend` (or its shorthand :option:`-b`) option +repeatedly. For example, to use ``192.168.0.10:8080`` and +``192.168.0.11:8080``, use command-line like this: +``-b192.168.0.10,8080 -b192.168.0.11,8080``. In configuration file, +this looks like: + +.. code-block:: text + + backend=192.168.0.10,8080 + backend=192.168.0.11,8008 + +nghttpx can route request to different backend according to request +host and path. For example, to route request destined to host +``doc.example.com`` to backend server ``docserv:3000``, you can write +like so: + +.. code-block:: text + + backend=docserv,3000;doc.example.com/ + +When you write this option in command-line, you should enclose +argument with single or double quotes, since the character ``;`` has a +special meaning in shell. + +To route, request to request path whose prefix is ``/foo`` to backend +server ``[::1]:8080``, you can write like so: + +.. code-block:: text + + backend=::1,8080;/foo + +Of course, you can specify both host and request path at the same +time. + +One important thing you have to remember is that we have to specify +default routing pattern for so called "catch all" pattern. To write +"catch all" pattern, just specify backend server address, without +pattern. + +Usually, host is the value of ``Host`` header field. In HTTP/2, the +value of ``:authority`` pseudo header field is used. + +When you write multiple backend addresses sharing the same routing +pattern, they are used as load balancing. For example, to use 2 +servers ``serv1:3000`` and ``serv2:3000`` for request host +``example.com`` and path ``/myservice``, you can write like so: + +.. code-block:: text + + backend=serv1,3000;example.com/myservice + backend=serv2,3000;example.com/myservice + +For HTTP/2 backend, see also +:option:`--backend-http2-connections-per-worker` option. From 936d4aca1abf52cea9ce445da5dd4cbd734f9be7 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 17:46:48 +0900 Subject: [PATCH 048/147] Update h2load documentation --- doc/sources/h2load-howto.rst | 56 +++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/doc/sources/h2load-howto.rst b/doc/sources/h2load-howto.rst index f66c2c76..602c7ea2 100644 --- a/doc/sources/h2load-howto.rst +++ b/doc/sources/h2load-howto.rst @@ -1,18 +1,21 @@ +.. program:: h2load + h2load - HTTP/2 benchmarking tool - HOW-TO ========================================== -h2load is benchmarking tool for HTTP/2 and HTTP/1.1. If built with -spdylay (http://tatsuhiro-t.github.io/spdylay/) library, it also -supports SPDY protocol. It supports SSL/TLS and clear text for all -supported protocols. +:doc:`h2load.1` is benchmarking tool for HTTP/2 and HTTP/1.1. If +built with spdylay (http://tatsuhiro-t.github.io/spdylay/) library, it +also supports SPDY protocol. It supports SSL/TLS and clear text for +all supported protocols. Compiling from source --------------------- -``h2load`` is compiled alongside ``nghttp2`` and requires that the -``--enable-apps`` flag is passed to ``./configure`` and `required dependencies -`_ are available during -compilation. For details on compiling, see `nghttp2: Building from Git +h2load is compiled alongside nghttp2 and requires that the +``--enable-apps`` flag is passed to ``./configure`` and `required +dependencies `_ +are available during compilation. For details on compiling, see +`nghttp2: Building from Git `_. Basic Usage @@ -20,23 +23,21 @@ Basic Usage In order to set benchmark settings, specify following 3 options. -``-n`` +:option:`-n` The number of total requests. Default: 1 -``-c`` +:option:`-c` The number of concurrent clients. Default: 1 -``-m`` - The max concurrent streams to issue per client. - If ``auto`` is given, the number of given URIs is used. - Default: ``auto`` +:option:`-m` + The max concurrent streams to issue per client. Default: 1 For SSL/TLS connection, the protocol will be negotiated via ALPN/NPN. -You can set specific protocols in ``--npn-list`` option. For +You can set specific protocols in :option:`--npn-list` option. For cleartext connection, the default protocol is HTTP/2. To change the -protocol in cleartext connection, use ``--no-tls-proto`` option. For -convenience, ``--h1`` option forces HTTP/1.1 for both cleartext and -SSL/TLS connections. +protocol in cleartext connection, use :option:`--no-tls-proto` option. +For convenience, :option:`--h1` option forces HTTP/1.1 for both +cleartext and SSL/TLS connections. Here is a command-line to perform benchmark to URI \https://localhost using total 100000 requests, 100 concurrent clients and 10 max @@ -71,11 +72,11 @@ benchmarking results. By default, h2load uses large enough flow control window, which effectively disables flow control. To adjust receiver flow control window size, there are following options: -``-w`` +:option:`-w` Sets the stream level initial window size to (2**)-1. For SPDY, 2** is used instead. -``-W`` +:option:`-W` Sets the connection level initial window size to (2**)-1. For SPDY, if is strictly less than 16, this option is ignored. Otherwise @@ -85,17 +86,17 @@ Multi-Threading --------------- Sometimes benchmarking client itself becomes a bottleneck. To remedy -this situation, use ``-t`` option to specify the number of native +this situation, use :option:`-t` option to specify the number of native thread to use. -``-t`` +:option:`-t` The number of native threads. Default: 1 Selecting protocol for clear text --------------------------------- By default, if \http:// URI is given, HTTP/2 protocol is used. To -change the protocol to use for clear text, use ``-p`` option. +change the protocol to use for clear text, use :option:`-p` option. Multiple URIs ------------- @@ -106,3 +107,12 @@ If multiple URIs are specified, they are used in round robin manner. Please note that h2load uses scheme, host and port in the first URI and ignores those parts in the rest of the URIs. + +UNIX domain socket +------------------ + +To request against UNIX domain socket, use :option:`--base-uri`, and +specify ``unix:`` followed by the path to UNIX domain socket. For +example, if UNIX domain socket is ``/tmp/nghttpx.sock``, use +``--base-uri=unix:/tmp/nghttpx.sock``. h2load uses scheme, host and +port in the first URI in command-line or input file. From f3a415f6234818e66053dcfe621708d121101aaf Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 17:47:27 +0900 Subject: [PATCH 049/147] Update nghttpx documentation --- doc/sources/nghttpx-howto.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index e38af2c4..b387d220 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -3,11 +3,11 @@ nghttpx - HTTP/2 proxy - HOW-TO =============================== -nghttpx is a proxy translating protocols between HTTP/2 and other -protocols (e.g., HTTP/1, SPDY). It operates in several modes and each -mode may require additional programs to work with. This article -describes each operation mode and explains the intended use-cases. It -also covers some useful options later. +:doc:`nghttpx.1` is a proxy translating protocols between HTTP/2 and +other protocols (e.g., HTTP/1, SPDY). It operates in several modes +and each mode may require additional programs to work with. This +article describes each operation mode and explains the intended +use-cases. It also covers some useful options later. Default mode ------------ From 95ffb4565f7c8b56812f110f6a8dde6e131755cf Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 18:20:53 +0900 Subject: [PATCH 050/147] Update nghttpx documentation --- doc/sources/nghttpx-howto.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index b387d220..ce142503 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -257,6 +257,12 @@ encryption, use :option:`--tls-ticket-key-memcached-tls` for TLS ticket key, and use :option:`--tls-session-cache-memcached-tls` for TLS session cache. +Specifying additional server certificates +----------------------------------------- + +nghttpx accepts additional server private key and certificate pairs +using :option:`--subcert` option. It can be used multiple times. + Specifying additional CA certificate ------------------------------------ From f4bb8776d0880650a8e8187790cd040630e090c7 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 23 Feb 2016 01:06:23 +0900 Subject: [PATCH 051/147] mruby: Clean up mrbgems as well --- third-party/Makefile.am | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/third-party/Makefile.am b/third-party/Makefile.am index 6e9bcf27..c1955eea 100644 --- a/third-party/Makefile.am +++ b/third-party/Makefile.am @@ -55,7 +55,9 @@ mruby: all-local: mruby clean-local: - -rm -rf "${abs_builddir}/mruby/build" + MRUBY_CONFIG="${srcdir}/build_config.rb" \ + BUILD_DIR="${abs_builddir}/mruby/build" \ + "${srcdir}/mruby/minirake" -f "${srcdir}/mruby/Rakefile" clean endif # HAVE_MRUBY From 3e72711e23cdaecdb3d82fa7efbb508e8ef1ead0 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 23 Feb 2016 01:09:45 +0900 Subject: [PATCH 052/147] Cap 100 limit for remembering idle streams --- lib/nghttp2_session.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 69f43d16..944a4790 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -1216,11 +1216,12 @@ int nghttp2_session_adjust_idle_stream(nghttp2_session *session) { size_t max; int rv; - /* Make minimum number of idle streams 16, which is arbitrary chosen - number. */ - max = nghttp2_max(16, - nghttp2_min(session->local_settings.max_concurrent_streams, - session->pending_local_max_concurrent_stream)); + /* Make minimum number of idle streams 16, and maximum 100, which + are arbitrary chosen numbers. */ + max = nghttp2_min( + 100, nghttp2_max( + 16, nghttp2_min(session->local_settings.max_concurrent_streams, + session->pending_local_max_concurrent_stream))); DEBUGF(fprintf(stderr, "stream: adjusting kept idle streams " "num_idle_streams=%zu, max=%zu\n", From 9d15f9b00d3c36709d287a9a17fa6f3351839733 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 23 Feb 2016 01:18:07 +0900 Subject: [PATCH 053/147] nghttpd: Start SETTINGS timer after it is written to output buffer --- src/HttpServer.cc | 15 +++++++++++++-- src/HttpServer.h | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/HttpServer.cc b/src/HttpServer.cc index e5406cdc..15650136 100644 --- a/src/HttpServer.cc +++ b/src/HttpServer.cc @@ -577,6 +577,10 @@ Http2Handler::WriteBuf *Http2Handler::get_wb() { return &wb_; } int Http2Handler::setup_bev() { return 0; } +void Http2Handler::start_settings_timer() { + ev_timer_start(sessions_->get_loop(), &settings_timerev_); +} + int Http2Handler::fill_wb() { if (data_pending_) { auto n = std::min(wb_.wleft(), data_pendinglen_); @@ -870,8 +874,6 @@ int Http2Handler::connection_made() { } } - ev_timer_start(sessions_->get_loop(), &settings_timerev_); - if (ssl_ && !nghttp2::ssl::check_http2_requirement(ssl_)) { terminate_session(NGHTTP2_INADEQUATE_SECURITY); } @@ -1538,6 +1540,15 @@ int hd_on_frame_send_callback(nghttp2_session *session, break; } + case NGHTTP2_SETTINGS: { + if (frame->hd.flags & NGHTTP2_FLAG_ACK) { + return 0; + } + + hd->start_settings_timer(); + + break; + } case NGHTTP2_PUSH_PROMISE: { auto promised_stream_id = frame->push_promise.promised_stream_id; auto promised_stream = hd->get_stream(promised_stream_id); diff --git a/src/HttpServer.h b/src/HttpServer.h index 99623397..27b5b2c9 100644 --- a/src/HttpServer.h +++ b/src/HttpServer.h @@ -138,6 +138,7 @@ public: void remove_self(); int setup_bev(); + void start_settings_timer(); int on_read(); int on_write(); int connection_made(); From 2782ef67de9ba332bcc8454d3afd5b0914a6f94c Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 23 Feb 2016 01:18:52 +0900 Subject: [PATCH 054/147] nghttpd: Remove unused function --- src/HttpServer.cc | 3 --- src/HttpServer.h | 1 - 2 files changed, 4 deletions(-) diff --git a/src/HttpServer.cc b/src/HttpServer.cc index 15650136..03140c3b 100644 --- a/src/HttpServer.cc +++ b/src/HttpServer.cc @@ -308,7 +308,6 @@ public: } auto handler = make_unique(this, fd, ssl, get_next_session_id()); - handler->setup_bev(); if (!ssl) { if (handler->connection_made() != 0) { return; @@ -575,8 +574,6 @@ struct ev_loop *Http2Handler::get_loop() const { Http2Handler::WriteBuf *Http2Handler::get_wb() { return &wb_; } -int Http2Handler::setup_bev() { return 0; } - void Http2Handler::start_settings_timer() { ev_timer_start(sessions_->get_loop(), &settings_timerev_); } diff --git a/src/HttpServer.h b/src/HttpServer.h index 27b5b2c9..0690307e 100644 --- a/src/HttpServer.h +++ b/src/HttpServer.h @@ -137,7 +137,6 @@ public: ~Http2Handler(); void remove_self(); - int setup_bev(); void start_settings_timer(); int on_read(); int on_write(); From 9aee43f7d8378895db32f34951a86586baac1793 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 24 Feb 2016 23:51:00 +0900 Subject: [PATCH 055/147] Update doc for extension frames --- lib/includes/nghttp2/nghttp2.h | 45 +++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/lib/includes/nghttp2/nghttp2.h b/lib/includes/nghttp2/nghttp2.h index 6a2a5c13..9b753a71 100644 --- a/lib/includes/nghttp2/nghttp2.h +++ b/lib/includes/nghttp2/nghttp2.h @@ -1736,11 +1736,19 @@ typedef int (*nghttp2_on_extension_chunk_recv_callback)( * :type:`nghttp2_on_extension_chunk_recv_callback`. The frame header * is already unpacked by the library and provided as |hd|. * + * To receive extension frames, the application must tell desired + * extension frame type to the library using + * `nghttp2_option_set_user_recv_extension_type()`. + * * The implementation of this function may store the pointer to the * created object as a result of unpacking in |*payload|, and returns * 0. The pointer stored in |*payload| is opaque to the library, and - * the library does not own its pointer. |*payload| is initialled as - * `NULL`. + * the library does not own its pointer. |*payload| is initialized as + * `NULL`. The |*payload| is available as ``frame->ext.payload`` in + * :type:`nghttp2_on_frame_recv_callback`. Therefore if application + * can free that memory inside :type:`nghttp2_on_frame_recv_callback` + * callback. Of course, application has a liberty not ot use + * |*payload|, and do its own mechanism to process extension frames. * * To abort processing this extension frame, return * :enum:`NGHTTP2_ERR_CANCEL`. @@ -1760,16 +1768,16 @@ typedef int (*nghttp2_unpack_extension_callback)(nghttp2_session *session, /** * @functypedef * - * Callbck function invoked when library asks the application to pack + * Callback function invoked when library asks the application to pack * extension payload in its wire format. The frame header will be * packed by library. Application must pack payload only. - * frame->ext.payload is the object passed to + * ``frame->ext.payload`` is the object passed to * `nghttp2_submit_extension()` as payload parameter. Application * must pack extension payload to the |buf| of its capacity |len| - * bytes. + * bytes. The |len| is at least 16KiB. * - * The implementation of this function returns the number of bytes - * written into |buf| when it succeeds. + * The implementation of this function should return the number of + * bytes written into |buf| when it succeeds. * * To abort processing this extension frame, return * :enum:`NGHTTP2_ERR_CANCEL`, and @@ -2229,13 +2237,15 @@ nghttp2_option_set_max_reserved_remote_streams(nghttp2_option *option, /** * @function * - * Set extension frame type the application is willing to handle with + * Sets extension frame type the application is willing to handle with * user defined callbacks (see * :type:`nghttp2_on_extension_chunk_recv_callback` and * :type:`nghttp2_unpack_extension_callback`). The |type| is * extension frame type, and must be strictly greater than 0x9. - * Otherwise, this function does nothing. The application does not - * have to call this function if it just sends extension frames. + * Otherwise, this function does nothing. The application can call + * this function multiple times to set more than one frame type to + * receive. The application does not have to call this function if it + * just sends extension frames. */ NGHTTP2_EXTERN void nghttp2_option_set_user_recv_extension_type(nghttp2_option *option, @@ -3916,12 +3926,23 @@ NGHTTP2_EXTERN int nghttp2_submit_window_update(nghttp2_session *session, * * Submits extension frame. * - * Application can pass arbitrary frame flags and stream ID to |flags| + * Application can pass arbitrary frame flags and stream ID in |flags| * and |stream_id| respectively. The |payload| is opaque pointer, and - * it can be accessible though frame->ext.payload in + * it can be accessible though ``frame->ext.payload`` in * :type:`nghttp2_pack_extension_callback`. The library will not own * passed |payload| pointer. * + * The application must set :type:`nghttp2_pack_extension_callback` + * using `nghttp2_session_callbacks_set_pack_extension_callback()`. + * + * The application should retain the memory pointed by |payload| until + * the transmission of extension frame is done (which is indicated by + * :type:`nghttp2_on_frame_send_callback`), or transmission fails + * (which is indicated by :type:`nghttp2_on_frame_not_send_callback`). + * If application does not touch this memory region after packing it + * into a wire format, application can free it inside + * :type:`nghttp2_pack_extension_callback`. + * * The standard HTTP/2 frame cannot be sent with this function, so * |type| must be strictly grater than 0x9. Otherwise, this function * will fail with error code :enum:`NGHTTP2_ERR_INVALID_ARGUMENT`. From 827abb57e98ad624abccbb8f05e22217119e81e8 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 24 Feb 2016 23:59:01 +0900 Subject: [PATCH 056/147] Simplified bitfield calculation of extension frame --- lib/nghttp2_option.c | 4 ++-- lib/nghttp2_session.c | 2 +- lib/nghttp2_session.h | 9 +++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/nghttp2_option.c b/lib/nghttp2_option.c index 2201ec6e..fd665112 100644 --- a/lib/nghttp2_option.c +++ b/lib/nghttp2_option.c @@ -70,6 +70,6 @@ void nghttp2_option_set_user_recv_extension_type(nghttp2_option *option, } option->opt_set_mask |= NGHTTP2_OPT_USER_RECV_EXT_TYPES; - option->user_recv_ext_types[type / 8] = (uint8_t)( - option->user_recv_ext_types[type / 8] | (1 << (7 - (type & 0x7)))); + option->user_recv_ext_types[type / 8] = + (uint8_t)(option->user_recv_ext_types[type / 8] | (1 << (type & 0x7))); } diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 6be3c631..6f3843c8 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -5345,7 +5345,7 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session, const uint8_t *in, if (!session->callbacks.unpack_extension_callback || (session->user_recv_ext_types[iframe->frame.hd.type / 8] & - (1 << (7 - (iframe->frame.hd.type & 0x7)))) == 0) { + (1 << (iframe->frame.hd.type & 0x7))) == 0) { /* Silently ignore unknown frame type. */ busy = 1; diff --git a/lib/nghttp2_session.h b/lib/nghttp2_session.h index 0bf12db5..c319c15e 100644 --- a/lib/nghttp2_session.h +++ b/lib/nghttp2_session.h @@ -306,10 +306,11 @@ struct nghttp2_session { WINDOW_UPDATE is not queued. */ uint8_t window_update_queued; /* Bitfield of extension frame types that application is willing to - receive. First most significant 10 bits are standard frame types - and not used. If bit is set, it indicates that incoming frame - with that type is passed to user defined callbacks, otherwise - they are ignored. */ + receive. To designate the bit of given frame type i, use + user_recv_ext_types[i / 8] & (1 << (i & 0x7)). First 10 frame + types are standard frame types and not used in this bitfield. If + bit is set, it indicates that incoming frame with that type is + passed to user defined callbacks, otherwise they are ignored. */ uint8_t user_recv_ext_types[32]; }; From ebfae904abb47cea04b357b0a7570164b4483c45 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 25 Feb 2016 00:32:17 +0900 Subject: [PATCH 057/147] Fix typo --- lib/includes/nghttp2/nghttp2.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/includes/nghttp2/nghttp2.h b/lib/includes/nghttp2/nghttp2.h index 9b753a71..329a605d 100644 --- a/lib/includes/nghttp2/nghttp2.h +++ b/lib/includes/nghttp2/nghttp2.h @@ -1744,7 +1744,7 @@ typedef int (*nghttp2_on_extension_chunk_recv_callback)( * created object as a result of unpacking in |*payload|, and returns * 0. The pointer stored in |*payload| is opaque to the library, and * the library does not own its pointer. |*payload| is initialized as - * `NULL`. The |*payload| is available as ``frame->ext.payload`` in + * ``NULL``. The |*payload| is available as ``frame->ext.payload`` in * :type:`nghttp2_on_frame_recv_callback`. Therefore if application * can free that memory inside :type:`nghttp2_on_frame_recv_callback` * callback. Of course, application has a liberty not ot use From dbffb8995b3624484e3567bcd639d4af7daf7f34 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 25 Feb 2016 00:45:24 +0900 Subject: [PATCH 058/147] Handle extension frame in session_inbound_frame_reset --- lib/nghttp2_session.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 6f3843c8..fffdf45d 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -231,6 +231,8 @@ static void session_inbound_frame_reset(nghttp2_session *session) { nghttp2_session_new(), we rely on the fact that iframe->frame.hd.type is 0, so that no free is performed. */ switch (iframe->frame.hd.type) { + case NGHTTP2_DATA: + break; case NGHTTP2_HEADERS: nghttp2_frame_headers_free(&iframe->frame.headers, mem); break; @@ -255,6 +257,10 @@ static void session_inbound_frame_reset(nghttp2_session *session) { case NGHTTP2_WINDOW_UPDATE: nghttp2_frame_window_update_free(&iframe->frame.window_update); break; + default: + /* extension frame */ + nghttp2_frame_extension_free(&iframe->frame.ext); + break; } memset(&iframe->frame, 0, sizeof(nghttp2_frame)); From 56bdfd1df20116ecd756a070f13fba2d0336889f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 25 Feb 2016 00:58:24 +0900 Subject: [PATCH 059/147] Revert "Handle extension frame in session_inbound_frame_reset" This reverts commit dbffb8995b3624484e3567bcd639d4af7daf7f34. --- lib/nghttp2_session.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index fffdf45d..6f3843c8 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -231,8 +231,6 @@ static void session_inbound_frame_reset(nghttp2_session *session) { nghttp2_session_new(), we rely on the fact that iframe->frame.hd.type is 0, so that no free is performed. */ switch (iframe->frame.hd.type) { - case NGHTTP2_DATA: - break; case NGHTTP2_HEADERS: nghttp2_frame_headers_free(&iframe->frame.headers, mem); break; @@ -257,10 +255,6 @@ static void session_inbound_frame_reset(nghttp2_session *session) { case NGHTTP2_WINDOW_UPDATE: nghttp2_frame_window_update_free(&iframe->frame.window_update); break; - default: - /* extension frame */ - nghttp2_frame_extension_free(&iframe->frame.ext); - break; } memset(&iframe->frame, 0, sizeof(nghttp2_frame)); From 8aac5d6af2e081644baa74f3eb16ed234a990e17 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 25 Feb 2016 00:58:50 +0900 Subject: [PATCH 060/147] Update doc --- lib/nghttp2_frame.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/nghttp2_frame.c b/lib/nghttp2_frame.c index 3d057847..ebf2e5e6 100644 --- a/lib/nghttp2_frame.c +++ b/lib/nghttp2_frame.c @@ -191,7 +191,9 @@ void nghttp2_frame_extension_init(nghttp2_extension *frame, uint8_t type, frame->payload = payload; } -void nghttp2_frame_extension_free(nghttp2_extension *frame _U_) {} +void nghttp2_frame_extension_free(nghttp2_extension *frame _U_) { + /* should be noop for performance reason */ +} size_t nghttp2_frame_priority_len(uint8_t flags) { if (flags & NGHTTP2_FLAG_PRIORITY) { From bc933e998154a1f8b536a444016ce8fbda149a54 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 25 Feb 2016 21:18:09 +0900 Subject: [PATCH 061/147] src: Use lowercase to show it is not the name of frame --- src/app_helper.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app_helper.cc b/src/app_helper.cc index c360a7de..ecbe0b2f 100644 --- a/src/app_helper.cc +++ b/src/app_helper.cc @@ -143,9 +143,9 @@ std::string strframetype(uint8_t type) { return "WINDOW_UPDATE"; } - std::string s = "UNKNOWN(0x"; + std::string s = "extension(0x"; s += util::format_hex(&type, 1); - s += ")"; + s += ')'; return s; }; From 40c1b29f36aa653ffb8be1bd0f88ada3c05d89f7 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 25 Feb 2016 00:45:24 +0900 Subject: [PATCH 062/147] Handle extension frame in session_inbound_frame_reset --- lib/nghttp2_session.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 6f3843c8..fffdf45d 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -231,6 +231,8 @@ static void session_inbound_frame_reset(nghttp2_session *session) { nghttp2_session_new(), we rely on the fact that iframe->frame.hd.type is 0, so that no free is performed. */ switch (iframe->frame.hd.type) { + case NGHTTP2_DATA: + break; case NGHTTP2_HEADERS: nghttp2_frame_headers_free(&iframe->frame.headers, mem); break; @@ -255,6 +257,10 @@ static void session_inbound_frame_reset(nghttp2_session *session) { case NGHTTP2_WINDOW_UPDATE: nghttp2_frame_window_update_free(&iframe->frame.window_update); break; + default: + /* extension frame */ + nghttp2_frame_extension_free(&iframe->frame.ext); + break; } memset(&iframe->frame, 0, sizeof(nghttp2_frame)); From 32446a519798657746b0ed12eef6c1c8b35bc9d5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 25 Feb 2016 21:19:38 +0900 Subject: [PATCH 063/147] Revert "Update doc" This reverts commit 8aac5d6af2e081644baa74f3eb16ed234a990e17. --- lib/nghttp2_frame.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/nghttp2_frame.c b/lib/nghttp2_frame.c index ebf2e5e6..3d057847 100644 --- a/lib/nghttp2_frame.c +++ b/lib/nghttp2_frame.c @@ -191,9 +191,7 @@ void nghttp2_frame_extension_init(nghttp2_extension *frame, uint8_t type, frame->payload = payload; } -void nghttp2_frame_extension_free(nghttp2_extension *frame _U_) { - /* should be noop for performance reason */ -} +void nghttp2_frame_extension_free(nghttp2_extension *frame _U_) {} size_t nghttp2_frame_priority_len(uint8_t flags) { if (flags & NGHTTP2_FLAG_PRIORITY) { From c098b4ac70a14f9d35c1a0f2bb76fbe63f25fb4f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 25 Feb 2016 22:46:25 +0900 Subject: [PATCH 064/147] nghttpx: Remove --backend-tls-session-cache-per-worker option --- gennghttpxfun.py | 1 - src/shrpx.cc | 13 ------------- src/shrpx_config.cc | 7 ------- src/shrpx_config.h | 3 --- 4 files changed, 24 deletions(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 043e9426..1d9dc846 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -115,7 +115,6 @@ OPTIONS = [ "max-header-fields", "no-http2-cipher-black-list", "backend-http1-tls", - "backend-tls-session-cache-per-worker", "tls-session-cache-memcached-cert-file", "tls-session-cache-memcached-private-key-file", "tls-session-cache-memcached-address-family", diff --git a/src/shrpx.cc b/src/shrpx.cc index f84f10ea..cd00e635 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1077,7 +1077,6 @@ void fill_default_config() { } tlsconf.session_timeout = std::chrono::hours(12); - tlsconf.downstream_session_cache_per_worker = 10000; auto &httpconf = mod_config()->http; httpconf.server_name = "nghttpx nghttp2/" NGHTTP2_VERSION; @@ -1629,11 +1628,6 @@ SSL/TLS: Allow black listed cipher suite on HTTP/2 connection. See https://tools.ietf.org/html/rfc7540#appendix-A for the complete HTTP/2 cipher suites black list. - --backend-tls-session-cache-per-worker= - Set the maximum number of backend TLS session cache - stored per worker. - Default: )" - << get_config()->tls.downstream_session_cache_per_worker << R"( HTTP/2 and SPDY: -c, --http2-max-concurrent-streams= @@ -2451,8 +2445,6 @@ int main(int argc, char **argv) { {SHRPX_OPT_REQUEST_HEADER_FIELD_BUFFER, required_argument, &flag, 104}, {SHRPX_OPT_MAX_REQUEST_HEADER_FIELDS, required_argument, &flag, 105}, {SHRPX_OPT_BACKEND_HTTP1_TLS, no_argument, &flag, 106}, - {SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER, required_argument, - &flag, 107}, {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_TLS, no_argument, &flag, 108}, {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE, required_argument, &flag, 109}, @@ -2921,11 +2913,6 @@ int main(int argc, char **argv) { // --backend-http1-tls cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_HTTP1_TLS, "yes"); break; - case 107: - // --backend-tls-session-cache-per-worker - cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER, - optarg); - break; case 108: // --tls-session-cache-memcached-tls cmdcfgs.emplace_back(SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_TLS, "yes"); diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 4d95ffe2..f61788b7 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -668,7 +668,6 @@ enum { SHRPX_OPTID_BACKEND_READ_TIMEOUT, SHRPX_OPTID_BACKEND_REQUEST_BUFFER, SHRPX_OPTID_BACKEND_RESPONSE_BUFFER, - SHRPX_OPTID_BACKEND_TLS_SESSION_CACHE_PER_WORKER, SHRPX_OPTID_BACKEND_TLS_SNI_FIELD, SHRPX_OPTID_BACKEND_WRITE_TIMEOUT, SHRPX_OPTID_BACKLOG, @@ -1394,9 +1393,6 @@ int option_lookup_token(const char *name, size_t namelen) { if (util::strieq_l("backend-http2-connections-per-worke", name, 35)) { return SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER; } - if (util::strieq_l("backend-tls-session-cache-per-worke", name, 35)) { - return SHRPX_OPTID_BACKEND_TLS_SESSION_CACHE_PER_WORKER; - } break; case 's': if (util::strieq_l("backend-http2-connection-window-bit", name, 35)) { @@ -2285,9 +2281,6 @@ int parse_config(const char *opt, const char *optarg, mod_config()->conn.downstream.http1_tls = util::strieq(optarg, "yes"); return 0; - case SHRPX_OPTID_BACKEND_TLS_SESSION_CACHE_PER_WORKER: - return parse_uint(&mod_config()->tls.downstream_session_cache_per_worker, - opt, optarg); case SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_TLS: mod_config()->tls.session_cache.memcached.tls = util::strieq(optarg, "yes"); diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 1a760d9e..aba62253 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -210,8 +210,6 @@ constexpr char SHRPX_OPT_MAX_RESPONSE_HEADER_FIELDS[] = constexpr char SHRPX_OPT_NO_HTTP2_CIPHER_BLACK_LIST[] = "no-http2-cipher-black-list"; constexpr char SHRPX_OPT_BACKEND_HTTP1_TLS[] = "backend-http1-tls"; -constexpr char SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER[] = - "backend-tls-session-cache-per-worker"; constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_TLS[] = "tls-session-cache-memcached-tls"; constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE[] = @@ -420,7 +418,6 @@ struct TLSConfig { std::vector npn_list; // list of supported SSL/TLS protocol strings. std::vector tls_proto_list; - size_t downstream_session_cache_per_worker; // Bit mask to disable SSL/TLS protocol versions. This will be // passed to SSL_CTX_set_options(). long int tls_proto_mask; From 31d4077638078cc8b65fa1c09fa43b1755a2f340 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 25 Feb 2016 23:18:21 +0900 Subject: [PATCH 065/147] Remove files copied in the rule for apiref.rst target --- doc/Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/Makefile.am b/doc/Makefile.am index dc4948a1..e0316df0 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -243,6 +243,7 @@ apiref.rst: \ $(APIDOCS): apiref.rst clean-local: + [ $(srcdir) = $(builddir) ] || for i in $(RST_FILES); do [ -e $(builddir)/$$i ] && rm $(builddir)/$$i; done -rm -f apiref.rst -rm -f $(APIDOCS) -rm -rf $(BUILDDIR)/* From 36e931e0d72209459fdb8c03299567c797d5741d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 25 Feb 2016 23:33:16 +0900 Subject: [PATCH 066/147] Bump up version number to 1.8.0, LT revision to 19:0:5 --- configure.ac | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 844695c5..6361ce0e 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.8.0-DEV], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.8.0], [t-tujikawa@users.sourceforge.net]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_HEADERS([config.h]) @@ -46,9 +46,9 @@ m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) dnl See versioning rule: dnl http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html -AC_SUBST(LT_CURRENT, 18) -AC_SUBST(LT_REVISION, 1) -AC_SUBST(LT_AGE, 4) +AC_SUBST(LT_CURRENT, 19) +AC_SUBST(LT_REVISION, 0) +AC_SUBST(LT_AGE, 5) major=`echo $PACKAGE_VERSION |cut -d. -f1 | sed -e "s/[^0-9]//g"` minor=`echo $PACKAGE_VERSION |cut -d. -f2 | sed -e "s/[^0-9]//g"` From 1c8e625045b7cf5fb2d7753ab1b73d34e8b18676 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 25 Feb 2016 23:35:23 +0900 Subject: [PATCH 067/147] Update man pages --- doc/h2load.1 | 2 +- doc/nghttp.1 | 2 +- doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 10 +--------- doc/nghttpx.1.rst | 13 ------------- 5 files changed, 4 insertions(+), 25 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index a6db9aea..c01af0ee 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "February 14, 2016" "1.8.0-DEV" "nghttp2" +.TH "H2LOAD" "1" "February 25, 2016" "1.8.0" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 24beec68..66165b1d 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "February 14, 2016" "1.8.0-DEV" "nghttp2" +.TH "NGHTTP" "1" "February 25, 2016" "1.8.0" "nghttp2" .SH NAME nghttp \- HTTP/2 client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index a3b7bc9f..52eb071e 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "February 14, 2016" "1.8.0-DEV" "nghttp2" +.TH "NGHTTPD" "1" "February 25, 2016" "1.8.0" "nghttp2" .SH NAME nghttpd \- HTTP/2 server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index a89f545f..a8b465af 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "February 14, 2016" "1.8.0-DEV" "nghttp2" +.TH "NGHTTPX" "1" "February 25, 2016" "1.8.0" "nghttp2" .SH NAME nghttpx \- HTTP/2 proxy . @@ -697,14 +697,6 @@ Allow black listed cipher suite on HTTP/2 connection. See \fI\%https://tools.ietf.org/html/rfc7540#appendix\-A\fP for the complete HTTP/2 cipher suites black list. .UNINDENT -.INDENT 0.0 -.TP -.B \-\-backend\-tls\-session\-cache\-per\-worker= -Set the maximum number of backend TLS session cache -stored per worker. -.sp -Default: \fB10000\fP -.UNINDENT .SS HTTP/2 and SPDY .INDENT 0.0 .TP diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index 49bb5e27..22f98ff0 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -622,13 +622,6 @@ SSL/TLS See https://tools.ietf.org/html/rfc7540#appendix-A for the complete HTTP/2 cipher suites black list. -.. option:: --backend-tls-session-cache-per-worker= - - Set the maximum number of backend TLS session cache - stored per worker. - - Default: ``10000`` - HTTP/2 and SPDY ~~~~~~~~~~~~~~~ @@ -1474,12 +1467,6 @@ addresses: App.new -NOTES ------ - -1. nghttpx - HTTP/2 proxy - HOW-TO - https://nghttp2.org/documentation/nghttpx-howto.html - SEE ALSO -------- From 887d4d2a4149e41155053e78f5e782b315c02c1c Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 25 Feb 2016 23:35:44 +0900 Subject: [PATCH 068/147] Update bash_completion --- doc/bash_completion/nghttpx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/bash_completion/nghttpx b/doc/bash_completion/nghttpx index e127d178..7e56320f 100644 --- a/doc/bash_completion/nghttpx +++ b/doc/bash_completion/nghttpx @@ -8,7 +8,7 @@ _nghttpx() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--worker-read-rate --frontend-no-tls --frontend-http2-dump-response-header --backend-http1-connections-per-frontend --tls-ticket-key-file --verify-client-cacert --include --max-response-header-fields --backend-request-buffer --max-request-header-fields --backend-http2-connection-window-bits --backend-tls-session-cache-per-worker --conf --worker-write-burst --npn-list --fetch-ocsp-response-file --no-http2-cipher-black-list --mruby-file --stream-read-timeout --tls-ticket-key-memcached --forwarded-for --accesslog-syslog --frontend-http2-read-timeout --listener-disable-timeout --frontend-http2-connection-window-bits --ciphers --strip-incoming-x-forwarded-for --private-key-passwd-file --backend-keep-alive-timeout --backend-http-proxy-uri --backend-http1-connections-per-host --rlimit-nofile --tls-dyn-rec-warmup-threshold --no-via --ocsp-update-interval --backend-write-timeout --client --tls-ticket-key-memcached-max-retry --http2-no-cookie-crumbling --worker-read-burst --client-proxy --http2-bridge --accesslog-format --errorlog-syslog --request-header-field-buffer --errorlog-file --http2-max-concurrent-streams --frontend-write-timeout --tls-ticket-key-cipher --read-burst --backend-ipv4 --backend-ipv6 --backend --insecure --log-level --host-rewrite --tls-proto-list --backend-http2-connections-per-worker --tls-ticket-key-memcached-interval --dh-param-file --worker-frontend-connections --backend-http1-tls --syslog-facility --fastopen --no-location-rewrite --tls-session-cache-memcached --no-ocsp --backend-response-buffer --workers --add-forwarded --frontend-http2-window-bits --worker-write-rate --add-request-header --backend-tls-sni-field --subcert --help --frontend-frame-debug --pid-file --frontend-http2-dump-request-header --daemon --write-rate --altsvc --user --add-x-forwarded-for --frontend-read-timeout --tls-ticket-key-memcached-max-fail --backlog --write-burst --no-server-push --backend-http2-window-bits --response-header-field-buffer --padding --stream-write-timeout --cacert --forwarded-by --version --add-response-header --backend-read-timeout --frontend --accesslog-file --http2-proxy --backend-no-tls --client-private-key-file --client-cert-file --accept-proxy-protocol --tls-dyn-rec-idle-timeout --verify-client --read-rate --strip-incoming-forwarded ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--worker-read-rate --frontend-no-tls --frontend-http2-dump-response-header --backend-http1-connections-per-frontend --tls-ticket-key-file --verify-client-cacert --include --max-response-header-fields --backend-request-buffer --max-request-header-fields --backend-http2-connection-window-bits --conf --worker-write-burst --npn-list --fetch-ocsp-response-file --no-via --tls-session-cache-memcached-cert-file --no-http2-cipher-black-list --mruby-file --stream-read-timeout --tls-ticket-key-memcached --forwarded-for --accesslog-syslog --frontend-http2-read-timeout --listener-disable-timeout --frontend-http2-connection-window-bits --ciphers --strip-incoming-x-forwarded-for --private-key-passwd-file --backend-keep-alive-timeout --backend-http-proxy-uri --backend-http1-connections-per-host --rlimit-nofile --tls-dyn-rec-warmup-threshold --tls-ticket-key-memcached-cert-file --ocsp-update-interval --forwarded-by --tls-session-cache-memcached-private-key-file --backend-write-timeout --client --tls-ticket-key-memcached-max-retry --http2-no-cookie-crumbling --worker-read-burst --client-proxy --http2-bridge --accesslog-format --errorlog-syslog --request-header-field-buffer --errorlog-file --http2-max-concurrent-streams --frontend-write-timeout --tls-ticket-key-cipher --read-burst --backend --insecure --log-level --host-rewrite --tls-session-cache-memcached-tls --tls-proto-list --backend-http2-connections-per-worker --tls-ticket-key-memcached-interval --dh-param-file --worker-frontend-connections --backend-http1-tls --syslog-facility --fastopen --no-location-rewrite --tls-ticket-key-memcached-tls --tls-session-cache-memcached --no-ocsp --backend-response-buffer --workers --add-forwarded --frontend-http2-window-bits --worker-write-rate --add-request-header --backend-tls-sni-field --subcert --help --frontend-frame-debug --pid-file --frontend-http2-dump-request-header --daemon --write-rate --altsvc --user --add-x-forwarded-for --frontend-read-timeout --tls-ticket-key-memcached-max-fail --backlog --write-burst --no-server-push --backend-http2-window-bits --response-header-field-buffer --tls-ticket-key-memcached-address-family --padding --tls-session-cache-memcached-address-family --stream-write-timeout --cacert --tls-ticket-key-memcached-private-key-file --backend-address-family --version --add-response-header --backend-read-timeout --frontend --accesslog-file --http2-proxy --backend-no-tls --client-private-key-file --client-cert-file --accept-proxy-protocol --tls-dyn-rec-idle-timeout --verify-client --read-rate --strip-incoming-forwarded ' -- "$cur" ) ) ;; *) _filedir From 2415a227571671636c7d678ca47c5d74ccf7f9ef Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 26 Feb 2016 00:00:24 +0900 Subject: [PATCH 069/147] h2load: Fix uninitialized fields --- src/h2load.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/h2load.cc b/src/h2load.cc index aeedbe7f..61048123 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -96,7 +96,9 @@ Config::Config() port(0), default_port(0), verbose(false), - timing_script(false) {} + timing_script(false), + base_uri_unix(false), + unix_addr{} {} Config::~Config() { if (base_uri_unix) { From d1f9f9a3aa2c7208ddc3b4e4b0964099c4b57a3e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 26 Feb 2016 00:00:52 +0900 Subject: [PATCH 070/147] makerelease.sh: Add autoreconf --- makerelease.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/makerelease.sh b/makerelease.sh index dd22feb3..cfd32baf 100755 --- a/makerelease.sh +++ b/makerelease.sh @@ -9,6 +9,8 @@ git checkout refs/tags/$TAG git log --pretty=fuller --date=short refs/tags/$PREV_TAG..HEAD > ChangeLog git submodule update --init + +autoreconf -i ./configure --with-mruby && \ make dist-bzip2 && make dist-gzip && make dist-xz || echo "error" make distclean From e6ff7e1e4ad0df7c1b929ae77ffe0204bd8e2d01 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 26 Feb 2016 00:53:42 +0900 Subject: [PATCH 071/147] Bump up version number to 1.9.0-DEV --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 6361ce0e..493995ba 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.8.0], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.9.0-DEV], [t-tujikawa@users.sourceforge.net]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_HEADERS([config.h]) From 9eeac27966ed6c01ff131314e2a292018f61b930 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 26 Feb 2016 00:57:34 +0900 Subject: [PATCH 072/147] makerelease.sh: Remove duplicate `git module update --init` --- makerelease.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/makerelease.sh b/makerelease.sh index cfd32baf..3356d0b0 100755 --- a/makerelease.sh +++ b/makerelease.sh @@ -3,8 +3,6 @@ TAG=$1 PREV_TAG=$2 -git submodule update --init - git checkout refs/tags/$TAG git log --pretty=fuller --date=short refs/tags/$PREV_TAG..HEAD > ChangeLog From 21007da3924f1daff0891d7c94054e96e80cd28f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 27 Feb 2016 19:39:03 +0900 Subject: [PATCH 073/147] nghttpx: Rewrite backend HTTP/2 connection coalesce strategy Previously, we use one Http2Session object per DownstreamAddrGroup. This is not flexible, and we have to provision how many HTTP/2 connection is required in advance. The new strategy is we add Http2Session object on demand. We measure the number of attached downstream connection object and server advertised concurrency limit. As long as former is smaller than the latter, we attach new downstream connection to it. Once the limit is reached, we create new Http2Session object. If the number lowers the limit, we start to share Http2Session object again. --- src/shrpx_client_handler.cc | 36 +++++++---- src/shrpx_client_handler.h | 1 - src/shrpx_config.h | 11 ++++ src/shrpx_http2_session.cc | 79 +++++++++++++++++++++---- src/shrpx_http2_session.h | 25 ++++++-- src/shrpx_http_downstream_connection.cc | 4 +- src/shrpx_worker.cc | 40 ------------- src/shrpx_worker.h | 15 ----- src/template.h | 22 +++++-- 9 files changed, 141 insertions(+), 92 deletions(-) diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 6e89a5c4..d67c472e 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -386,11 +386,6 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl, get_config()->conn.upstream.ratelimit.read, writecb, readcb, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold, get_config()->tls.dyn_rec.idle_timeout), - pinned_http2sessions_( - get_config()->conn.downstream.proto == PROTO_HTTP2 - ? make_unique>( - worker->get_downstream_addr_groups().size(), -1) - : nullptr), ipaddr_(ipaddr), port_(port), faddr_(faddr), @@ -714,15 +709,30 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { auto dconn_pool = worker_->get_dconn_pool(); if (downstreamconf.proto == PROTO_HTTP2) { - Http2Session *http2session; - auto &pinned = (*pinned_http2sessions_)[group]; - if (pinned == -1) { - http2session = worker_->next_http2_session(group); - pinned = http2session->get_index(); - } else { - auto dgrp = worker_->get_dgrp(group); - http2session = dgrp->http2sessions[pinned].get(); + auto &addr_group = worker_->get_downstream_addr_groups()[group]; + if (addr_group.http2_freelist.empty()) { + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) + << "http2_freelist is empty; create new Http2Session"; + } + auto session = make_unique( + conn_.loop, worker_->get_cl_ssl_ctx(), worker_, group); + addr_group.http2_freelist.append(session.release()); } + + auto http2session = addr_group.http2_freelist.head; + + // TODO max_concurrent_streams option must be independent from + // frontend and backend. + if (http2session->max_concurrency_reached(1)) { + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) << "Maximum streams are reached for Http2Session(" + << http2session + << "). Remove Http2Session from http2_freelist"; + } + addr_group.http2_freelist.remove(http2session); + } + dconn = make_unique(dconn_pool, http2session); } else { dconn = make_unique(dconn_pool, group, diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index 9e932ae2..42b03c26 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -144,7 +144,6 @@ private: Connection conn_; ev_timer reneg_shutdown_timer_; std::unique_ptr upstream_; - std::unique_ptr> pinned_http2sessions_; // IP address of client. If UNIX domain socket is used, this is // "localhost". std::string ipaddr_; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index aba62253..1081f123 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -61,6 +61,7 @@ namespace shrpx { struct LogFragment; class ConnectBlocker; +class Http2Session; namespace ssl { @@ -304,6 +305,16 @@ struct DownstreamAddrGroup { ImmutableString pattern; std::vector addrs; + // List of Http2Session which is not fully utilized (i.e., the + // server advertized maximum concurrency is not reached). We will + // coalesce as much stream as possible in one Http2Session to fully + // utilize TCP connection. + // + // TODO Verify that this approach performs better in performance + // wise. + DList http2_freelist; + // Next downstream address index in addrs. + size_t next; }; struct TicketKey { diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index f7ac9a82..7ed816a9 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -74,6 +74,10 @@ void connchk_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents) { SSLOG(INFO, http2session) << "ping timeout"; } http2session->disconnect(); + + if (http2session->get_num_dconns() == 0) { + delete http2session; + } return; default: if (LOG_ENABLED(INFO)) { @@ -92,6 +96,9 @@ void settings_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents) { SSLOG(INFO, http2session) << "SETTINGS timeout"; if (http2session->terminate_session(NGHTTP2_SETTINGS_TIMEOUT) != 0) { http2session->disconnect(); + if (http2session->get_num_dconns() == 0) { + delete http2session; + } return; } http2session->signal_write(); @@ -109,6 +116,9 @@ void timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) { http2session->disconnect(http2session->get_state() == Http2Session::CONNECTING); + if (http2session->get_num_dconns() == 0) { + delete http2session; + } } } // namespace @@ -120,6 +130,9 @@ void readcb(struct ev_loop *loop, ev_io *w, int revents) { rv = http2session->do_read(); if (rv != 0) { http2session->disconnect(http2session->should_hard_fail()); + if (http2session->get_num_dconns() == 0) { + delete http2session; + } return; } http2session->connection_alive(); @@ -127,6 +140,9 @@ void readcb(struct ev_loop *loop, ev_io *w, int revents) { rv = http2session->do_write(); if (rv != 0) { http2session->disconnect(http2session->should_hard_fail()); + if (http2session->get_num_dconns() == 0) { + delete http2session; + } return; } } @@ -140,6 +156,9 @@ void writecb(struct ev_loop *loop, ev_io *w, int revents) { rv = http2session->do_write(); if (rv != 0) { http2session->disconnect(http2session->should_hard_fail()); + if (http2session->get_num_dconns() == 0) { + delete http2session; + } return; } http2session->reset_connection_check_timer_if_not_checking(); @@ -147,8 +166,10 @@ void writecb(struct ev_loop *loop, ev_io *w, int revents) { } // namespace Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, - Worker *worker, size_t group, size_t idx) - : conn_(loop, -1, nullptr, worker->get_mcpool(), + Worker *worker, size_t group) + : dlnext(nullptr), + dlprev(nullptr), + conn_(loop, -1, nullptr, worker->get_mcpool(), get_config()->conn.downstream.timeout.write, get_config()->conn.downstream.timeout.read, {}, {}, writecb, readcb, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold, @@ -159,11 +180,9 @@ Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, addr_(nullptr), session_(nullptr), group_(group), - index_(idx), state_(DISCONNECTED), connection_check_state_(CONNECTION_CHECK_NONE), flow_control_(false) { - read_ = write_ = &Http2Session::noop; on_read_ = &Http2Session::read_noop; @@ -182,7 +201,17 @@ Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, settings_timer_.data = this; } -Http2Session::~Http2Session() { disconnect(); } +Http2Session::~Http2Session() { + disconnect(); + + if (in_freelist()) { + if (LOG_ENABLED(INFO)) { + SSLOG(INFO, this) << "Removed from http2_freelist"; + } + auto &addr_group = worker_->get_downstream_addr_groups()[group_]; + addr_group.http2_freelist.remove(this); + } +} int Http2Session::disconnect(bool hard) { if (LOG_ENABLED(INFO)) { @@ -252,8 +281,8 @@ int Http2Session::disconnect(bool hard) { int Http2Session::initiate_connection() { int rv = 0; - auto &groups = worker_->get_downstream_addr_groups(); - auto &addrs = groups[group_].addrs; + auto &addr_group = worker_->get_downstream_addr_groups()[group_]; + auto &addrs = addr_group.addrs; auto worker_blocker = worker_->get_connect_blocker(); if (state_ == DISCONNECTED) { @@ -265,7 +294,7 @@ int Http2Session::initiate_connection() { return -1; } - auto &next_downstream = worker_->get_dgrp(group_)->next; + auto &next_downstream = addr_group.next; auto end = next_downstream; for (;;) { @@ -598,6 +627,18 @@ void Http2Session::remove_downstream_connection( Http2DownstreamConnection *dconn) { dconns_.remove(dconn); dconn->detach_stream_data(); + + if (LOG_ENABLED(INFO)) { + SSLOG(INFO, this) << "Remove downstream"; + } + + if (!in_freelist() && !max_concurrency_reached()) { + if (LOG_ENABLED(INFO)) { + SSLOG(INFO, this) << "Append to Http2Session freelist"; + } + auto &addr_group = worker_->get_downstream_addr_groups()[group_]; + addr_group.http2_freelist.append(this); + } } void Http2Session::remove_stream_data(StreamData *sd) { @@ -1892,8 +1933,6 @@ const DownstreamAddr *Http2Session::get_addr() const { return addr_; } size_t Http2Session::get_group() const { return group_; } -size_t Http2Session::get_index() const { return index_; } - int Http2Session::handle_downstream_push_promise(Downstream *downstream, int32_t promised_stream_id) { auto upstream = downstream->get_upstream(); @@ -1986,4 +2025,24 @@ int Http2Session::handle_downstream_push_promise_complete( return 0; } +size_t Http2Session::get_num_dconns() const { return dconns_.size(); } + +bool Http2Session::in_freelist() const { + auto &addr_group = worker_->get_downstream_addr_groups()[group_]; + + return dlnext != nullptr || dlprev != nullptr || + addr_group.http2_freelist.head == this || + addr_group.http2_freelist.tail == this; +} + +bool Http2Session::max_concurrency_reached(size_t extra) const { + if (!session_) { + return dconns_.size() + extra >= 100; + } + + return dconns_.size() + extra >= + nghttp2_session_get_remote_settings( + session_, NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); +} + } // namespace shrpx diff --git a/src/shrpx_http2_session.h b/src/shrpx_http2_session.h index 62e1235c..bc81f420 100644 --- a/src/shrpx_http2_session.h +++ b/src/shrpx_http2_session.h @@ -57,7 +57,7 @@ struct StreamData { class Http2Session { public: Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, Worker *worker, - size_t group, size_t idx); + size_t group); ~Http2Session(); // If hard is true, all pending requests are abandoned and @@ -149,13 +149,27 @@ public: size_t get_group() const; - size_t get_index() const; - int handle_downstream_push_promise(Downstream *downstream, int32_t promised_stream_id); int handle_downstream_push_promise_complete(Downstream *downstream, Downstream *promised_downstream); + // Returns number of downstream connections, including pushed + // streams. + size_t get_num_dconns() const; + + // Returns true if this object is included in freelist. See + // DownstreamAddrGroup object. + bool in_freelist() const; + + // Returns true if the maximum concurrency is reached. In other + // words, the number of currently participated streams in this + // session is equal or greater than the max concurrent streams limit + // advertised by server. If |extra| is nonzero, it is added to the + // number of current concurrent streams when comparing against + // server initiated concurrency limit. + bool max_concurrency_reached(size_t extra = 0) const; + enum { // Disconnected DISCONNECTED, @@ -184,6 +198,8 @@ public: using ReadBuf = Buffer<8_k>; + Http2Session *dlnext, *dlprev; + private: Connection conn_; DefaultMemchunks wb_; @@ -207,9 +223,6 @@ private: const DownstreamAddr *addr_; nghttp2_session *session_; size_t group_; - // index inside group, this is used to pin frontend to certain - // HTTP/2 backend for better throughput. - size_t index_; int state_; int connection_check_state_; bool flow_control_; diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index f0b6fd3c..e8f0e298 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -157,10 +157,10 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { conn_.set_ssl(ssl); } - auto &next_downstream = worker_->get_dgrp(group_)->next; - auto end = next_downstream; auto &groups = worker_->get_downstream_addr_groups(); auto &addrs = groups[group_].addrs; + auto &next_downstream = groups[group_].next; + auto end = next_downstream; for (;;) { auto &addr = addrs[next_downstream]; diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 2cb71acc..51ee9ad3 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -73,7 +73,6 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, : randgen_(rd()), dconn_pool_(get_config()->conn.downstream.addr_groups.size()), worker_stat_(get_config()->conn.downstream.addr_groups.size()), - dgrps_(get_config()->conn.downstream.addr_groups.size()), loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), @@ -98,24 +97,6 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, StringRef{session_cacheconf.memcached.host}, &mcpool_); } - auto &downstreamconf = get_config()->conn.downstream; - - if (downstreamconf.proto == PROTO_HTTP2) { - auto n = get_config()->http2.downstream.connections_per_worker; - size_t group = 0; - for (auto &dgrp : dgrps_) { - auto m = n; - if (m == 0) { - m = downstreamconf.addr_groups[group].addrs.size(); - } - for (size_t idx = 0; idx < m; ++idx) { - dgrp.http2sessions.push_back( - make_unique(loop_, cl_ssl_ctx, this, group, idx)); - } - ++group; - } - } - for (auto &group : downstream_addr_groups_) { for (auto &addr : group.addrs) { addr.connect_blocker = new ConnectBlocker(randgen_, loop_); @@ -255,22 +236,6 @@ WorkerStat *Worker::get_worker_stat() { return &worker_stat_; } DownstreamConnectionPool *Worker::get_dconn_pool() { return &dconn_pool_; } -Http2Session *Worker::next_http2_session(size_t group) { - auto &dgrp = dgrps_[group]; - auto &http2sessions = dgrp.http2sessions; - if (http2sessions.empty()) { - return nullptr; - } - - auto res = http2sessions[dgrp.next_http2session].get(); - ++dgrp.next_http2session; - if (dgrp.next_http2session >= http2sessions.size()) { - dgrp.next_http2session = 0; - } - - return res; -} - struct ev_loop *Worker::get_loop() const { return loop_; } @@ -285,11 +250,6 @@ bool Worker::get_graceful_shutdown() const { return graceful_shutdown_; } MemchunkPool *Worker::get_mcpool() { return &mcpool_; } -DownstreamGroup *Worker::get_dgrp(size_t group) { - assert(group < dgrps_.size()); - return &dgrps_[group]; -} - MemcachedDispatcher *Worker::get_session_cache_memcached_dispatcher() { return session_cache_memcached_dispatcher_.get(); } diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index aa1531e6..35917ec9 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -67,17 +67,6 @@ namespace ssl { class CertLookupTree; } // namespace ssl -struct DownstreamGroup { - DownstreamGroup() : next_http2session(0), next(0) {} - - std::vector> http2sessions; - // Next index in http2sessions. - size_t next_http2session; - // Next downstream address index corresponding to - // Config::downstream_addr_groups[]. - size_t next; -}; - struct WorkerStat { WorkerStat(size_t num_groups) : num_connections(0) {} @@ -122,7 +111,6 @@ public: WorkerStat *get_worker_stat(); DownstreamConnectionPool *get_dconn_pool(); - Http2Session *next_http2_session(size_t group); struct ev_loop *get_loop() const; SSL_CTX *get_sv_ssl_ctx() const; SSL_CTX *get_cl_ssl_ctx() const; @@ -133,8 +121,6 @@ public: MemchunkPool *get_mcpool(); void schedule_clear_mcpool(); - DownstreamGroup *get_dgrp(size_t group); - MemcachedDispatcher *get_session_cache_memcached_dispatcher(); std::mt19937 &get_randgen(); @@ -161,7 +147,6 @@ private: MemchunkPool mcpool_; DownstreamConnectionPool dconn_pool_; WorkerStat worker_stat_; - std::vector dgrps_; std::unique_ptr session_cache_memcached_dispatcher_; #ifdef HAVE_MRUBY diff --git a/src/template.h b/src/template.h index 253ea215..3b1727fc 100644 --- a/src/template.h +++ b/src/template.h @@ -99,14 +99,17 @@ template bool test_flags(T t, F flags) { // T *dlnext, which point to previous element and next element in the // list respectively. template struct DList { - DList() : head(nullptr), tail(nullptr) {} + DList() : head(nullptr), tail(nullptr), n(0) {} - DList(const DList &) = delete; + // We should delete these copy ctor and assignment operator. We + // need to them where copy is required before we add item to it. If + // you doubt, make them delete and try to compile. + DList(const DList &) = default; + DList &operator=(const DList &) = default; - DList &operator=(const DList &) = delete; - - DList(DList &&other) : head(other.head), tail(other.tail) { + DList(DList &&other) : head(other.head), tail(other.tail), n(other.n) { other.head = other.tail = nullptr; + other.n = 0; } DList &operator=(DList &&other) { @@ -115,11 +118,16 @@ template struct DList { } head = other.head; tail = other.tail; + n = other.n; + other.head = other.tail = nullptr; + other.n = 0; + return *this; } void append(T *t) { + ++n; if (tail) { tail->dlnext = t; t->dlprev = tail; @@ -130,6 +138,7 @@ template struct DList { } void remove(T *t) { + --n; auto p = t->dlprev; auto n = t->dlnext; if (p) { @@ -149,7 +158,10 @@ template struct DList { bool empty() const { return head == nullptr; } + size_t size() const { return n; } + T *head, *tail; + size_t n; }; template void dlist_delete_all(DList &dl) { From 8ca3e5f6ba762e1862dffd557f2c50d4a6d12d1b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 27 Feb 2016 23:24:14 +0900 Subject: [PATCH 074/147] nghttpx: Separate Downstream address group from config to runtime --- src/shrpx.cc | 8 +- src/shrpx_client_handler.cc | 55 +++++----- src/shrpx_config.cc | 95 +---------------- src/shrpx_config.h | 33 +----- src/shrpx_downstream_connection.cc | 9 +- src/shrpx_downstream_connection.h | 9 +- src/shrpx_downstream_connection_pool.cc | 29 ++---- src/shrpx_downstream_connection_pool.h | 6 +- src/shrpx_http2_downstream_connection.cc | 14 ++- src/shrpx_http2_downstream_connection.h | 6 +- src/shrpx_http2_session.cc | 33 +++--- src/shrpx_http2_session.h | 12 ++- src/shrpx_http_downstream_connection.cc | 35 ++++--- src/shrpx_http_downstream_connection.h | 11 +- src/shrpx_worker.cc | 125 ++++++++++++++++++++--- src/shrpx_worker.h | 46 ++++++++- src/template.h | 7 +- 17 files changed, 271 insertions(+), 262 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index cd00e635..7da056a3 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2098,23 +2098,23 @@ void process_options( auto &addr_groups = downstreamconf.addr_groups; if (addr_groups.empty()) { - DownstreamAddr addr{}; + DownstreamAddrConfig addr{}; addr.host = ImmutableString::from_lit(DEFAULT_DOWNSTREAM_HOST); addr.port = DEFAULT_DOWNSTREAM_PORT; - DownstreamAddrGroup g(StringRef::from_lit("/")); + DownstreamAddrGroupConfig g(StringRef::from_lit("/")); g.addrs.push_back(std::move(addr)); mod_config()->router.add_route(StringRef{g.pattern}, addr_groups.size()); addr_groups.push_back(std::move(g)); } else if (get_config()->http2_proxy || get_config()->client_proxy) { // We don't support host mapping in these cases. Move all // non-catch-all patterns to catch-all pattern. - DownstreamAddrGroup catch_all(StringRef::from_lit("/")); + DownstreamAddrGroupConfig catch_all(StringRef::from_lit("/")); for (auto &g : addr_groups) { std::move(std::begin(g.addrs), std::end(g.addrs), std::back_inserter(catch_all.addrs)); } - std::vector().swap(addr_groups); + std::vector().swap(addr_groups); // maybe not necessary? mod_config()->router = Router(); mod_config()->router.add_route(StringRef{catch_all.pattern}, diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index d67c472e..7032471d 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -637,13 +637,18 @@ void ClientHandler::pool_downstream_connection( if (!dconn->poolable()) { return; } + + dconn->set_client_handler(nullptr); + + auto group = dconn->get_downstream_addr_group(); + if (LOG_ENABLED(INFO)) { CLOG(INFO, this) << "Pooling downstream connection DCONN:" << dconn.get() - << " in group " << dconn->get_group(); + << " in group " << group; } - dconn->set_client_handler(nullptr); - auto dconn_pool = worker_->get_dconn_pool(); - dconn_pool->add_downstream_connection(std::move(dconn)); + + auto &dconn_pool = group->dconn_pool; + dconn_pool.add_downstream_connection(std::move(dconn)); } void ClientHandler::remove_downstream_connection(DownstreamConnection *dconn) { @@ -651,13 +656,13 @@ void ClientHandler::remove_downstream_connection(DownstreamConnection *dconn) { CLOG(INFO, this) << "Removing downstream connection DCONN:" << dconn << " from pool"; } - auto dconn_pool = worker_->get_dconn_pool(); - dconn_pool->remove_downstream_connection(dconn); + auto &dconn_pool = dconn->get_downstream_addr_group()->dconn_pool; + dconn_pool.remove_downstream_connection(dconn); } std::unique_ptr ClientHandler::get_downstream_connection(Downstream *downstream) { - size_t group; + size_t group_idx; auto &downstreamconf = get_config()->conn.downstream; auto catch_all = downstreamconf.addr_group_catch_all; auto &groups = worker_->get_downstream_addr_groups(); @@ -667,26 +672,26 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { // Fast path. If we have one group, it must be catch-all group. // HTTP/2 and client proxy modes fall in this case. if (groups.size() == 1) { - group = 0; + group_idx = 0; } else if (req.method == HTTP_CONNECT) { // We don't know how to treat CONNECT request in host-path // mapping. It most likely appears in proxy scenario. Since we // have dealt with proxy case already, just use catch-all group. - group = catch_all; + group_idx = catch_all; } else { auto &router = get_config()->router; if (!req.authority.empty()) { - group = + group_idx = match_downstream_addr_group(router, StringRef{req.authority}, StringRef{req.path}, groups, catch_all); } else { auto h = req.fs.header(http2::HD_HOST); if (h) { - group = + group_idx = match_downstream_addr_group(router, StringRef{h->value}, StringRef{req.path}, groups, catch_all); } else { - group = + group_idx = match_downstream_addr_group(router, StringRef::from_lit(""), StringRef{req.path}, groups, catch_all); } @@ -694,11 +699,12 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { } if (LOG_ENABLED(INFO)) { - CLOG(INFO, this) << "Downstream address group: " << group; + CLOG(INFO, this) << "Downstream address group_idx: " << group_idx; } - auto dconn_pool = worker_->get_dconn_pool(); - auto dconn = dconn_pool->pop_downstream_connection(group); + auto &group = worker_->get_downstream_addr_groups()[group_idx]; + auto &dconn_pool = group.dconn_pool; + auto dconn = dconn_pool.pop_downstream_connection(); if (!dconn) { if (LOG_ENABLED(INFO)) { @@ -706,21 +712,18 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { << " Create new one"; } - auto dconn_pool = worker_->get_dconn_pool(); - if (downstreamconf.proto == PROTO_HTTP2) { - auto &addr_group = worker_->get_downstream_addr_groups()[group]; - if (addr_group.http2_freelist.empty()) { + if (group.http2_freelist.empty()) { if (LOG_ENABLED(INFO)) { CLOG(INFO, this) << "http2_freelist is empty; create new Http2Session"; } auto session = make_unique( - conn_.loop, worker_->get_cl_ssl_ctx(), worker_, group); - addr_group.http2_freelist.append(session.release()); + conn_.loop, worker_->get_cl_ssl_ctx(), worker_, &group); + group.http2_freelist.append(session.release()); } - auto http2session = addr_group.http2_freelist.head; + auto http2session = group.http2_freelist.head; // TODO max_concurrent_streams option must be independent from // frontend and backend. @@ -730,13 +733,13 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { << http2session << "). Remove Http2Session from http2_freelist"; } - addr_group.http2_freelist.remove(http2session); + group.http2_freelist.remove(http2session); } - dconn = make_unique(dconn_pool, http2session); + dconn = make_unique(http2session); } else { - dconn = make_unique(dconn_pool, group, - conn_.loop, worker_); + dconn = + make_unique(&group, conn_.loop, worker_); } dconn->set_client_handler(this); return dconn; diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index f61788b7..8b5cc537 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -575,7 +575,7 @@ namespace { // config. We will store each host-path pattern found in |src| with // |addr|. |addr| will be copied accordingly. Also we make a group // based on the pattern. The "/" pattern is considered as catch-all. -void parse_mapping(const DownstreamAddr &addr, const char *src) { +void parse_mapping(const DownstreamAddrConfig &addr, const char *src) { // This returns at least 1 element (it could be empty string). We // will append '/' to all patterns, so it becomes catch-all pattern. auto mapping = util::split_config_str_list(src, ':'); @@ -606,7 +606,7 @@ void parse_mapping(const DownstreamAddr &addr, const char *src) { if (done) { continue; } - DownstreamAddrGroup g(StringRef{pattern}); + DownstreamAddrGroupConfig g(StringRef{pattern}); g.addrs.push_back(addr); mod_config()->router.add_route(StringRef{g.pattern}, addr_groups.size()); @@ -1482,7 +1482,7 @@ int parse_config(const char *opt, const char *optarg, if (!pat_delim) { pat_delim = optarg + optarglen; } - DownstreamAddr addr{}; + DownstreamAddrConfig addr{}; if (util::istarts_with(optarg, SHRPX_UNIX_PATH_PREFIX)) { auto path = optarg + str_size(SHRPX_UNIX_PATH_PREFIX); addr.host = ImmutableString(path, pat_delim); @@ -2493,93 +2493,4 @@ int int_syslog_facility(const char *strfacility) { return -1; } -namespace { -size_t match_downstream_addr_group_host( - const Router &router, const StringRef &host, const StringRef &path, - const std::vector &groups, size_t catch_all) { - if (path.empty() || path[0] != '/') { - auto group = router.match(host, StringRef::from_lit("/")); - if (group != -1) { - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Found pattern with query " << host - << ", matched pattern=" << groups[group].pattern; - } - return group; - } - return catch_all; - } - - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Perform mapping selection, using host=" << host - << ", path=" << path; - } - - auto group = router.match(host, path); - if (group != -1) { - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Found pattern with query " << host << path - << ", matched pattern=" << groups[group].pattern; - } - return group; - } - - group = router.match("", path); - if (group != -1) { - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Found pattern with query " << path - << ", matched pattern=" << groups[group].pattern; - } - return group; - } - - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "None match. Use catch-all pattern"; - } - return catch_all; -} -} // namespace - -size_t match_downstream_addr_group( - const Router &router, const StringRef &hostport, const StringRef &raw_path, - const std::vector &groups, size_t catch_all) { - if (std::find(std::begin(hostport), std::end(hostport), '/') != - std::end(hostport)) { - // We use '/' specially, and if '/' is included in host, it breaks - // our code. Select catch-all case. - return catch_all; - } - - auto fragment = std::find(std::begin(raw_path), std::end(raw_path), '#'); - auto query = std::find(std::begin(raw_path), fragment, '?'); - auto path = StringRef{std::begin(raw_path), query}; - - if (hostport.empty()) { - return match_downstream_addr_group_host(router, hostport, path, groups, - catch_all); - } - - std::string host; - if (hostport[0] == '[') { - // assume this is IPv6 numeric address - auto p = std::find(std::begin(hostport), std::end(hostport), ']'); - if (p == std::end(hostport)) { - return catch_all; - } - if (p + 1 < std::end(hostport) && *(p + 1) != ':') { - return catch_all; - } - host.assign(std::begin(hostport), p + 1); - } else { - auto p = std::find(std::begin(hostport), std::end(hostport), ':'); - if (p == std::begin(hostport)) { - return catch_all; - } - host.assign(std::begin(hostport), p); - } - - util::inp_strlower(host); - return match_downstream_addr_group_host(router, StringRef{host}, path, groups, - catch_all); -} - } // namespace shrpx diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 1081f123..10742685 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -284,37 +284,24 @@ struct TLSSessionCache { ev_tstamp last_updated; }; -struct DownstreamAddr { +struct DownstreamAddrConfig { Address addr; // backend address. If |host_unix| is true, this is UNIX domain // socket path. ImmutableString host; ImmutableString hostport; - ConnectBlocker *connect_blocker; - // Client side TLS session cache - TLSSessionCache tls_session_cache; // backend port. 0 if |host_unix| is true. uint16_t port; // true if |host| contains UNIX domain socket path. bool host_unix; }; -struct DownstreamAddrGroup { - DownstreamAddrGroup(const StringRef &pattern) +struct DownstreamAddrGroupConfig { + DownstreamAddrGroupConfig(const StringRef &pattern) : pattern(pattern.c_str(), pattern.size()) {} ImmutableString pattern; - std::vector addrs; - // List of Http2Session which is not fully utilized (i.e., the - // server advertized maximum concurrency is not reached). We will - // coalesce as much stream as possible in one Http2Session to fully - // utilize TCP connection. - // - // TODO Verify that this approach performs better in performance - // wise. - DList http2_freelist; - // Next downstream address index in addrs. - size_t next; + std::vector addrs; }; struct TicketKey { @@ -563,7 +550,7 @@ struct ConnectionConfig { ev_tstamp write; ev_tstamp idle_read; } timeout; - std::vector addr_groups; + std::vector addr_groups; // The index of catch-all group in downstream_addr_groups. size_t addr_group_catch_all; size_t connections_per_host; @@ -657,16 +644,6 @@ std::unique_ptr read_tls_ticket_key_file(const std::vector &files, const EVP_CIPHER *cipher, const EVP_MD *hmac); -// Selects group based on request's |hostport| and |path|. |hostport| -// is the value taken from :authority or host header field, and may -// contain port. The |path| may contain query part. We require the -// catch-all pattern in place, so this function always selects one -// group. The catch-all group index is given in |catch_all|. All -// patterns are given in |groups|. -size_t match_downstream_addr_group( - const Router &router, const StringRef &hostport, const StringRef &path, - const std::vector &groups, size_t catch_all); - } // namespace shrpx #endif // SHRPX_CONFIG_H diff --git a/src/shrpx_downstream_connection.cc b/src/shrpx_downstream_connection.cc index 77dcd442..2c6d2c27 100644 --- a/src/shrpx_downstream_connection.cc +++ b/src/shrpx_downstream_connection.cc @@ -26,12 +26,11 @@ #include "shrpx_client_handler.h" #include "shrpx_downstream.h" -#include "shrpx_downstream_connection_pool.h" namespace shrpx { -DownstreamConnection::DownstreamConnection(DownstreamConnectionPool *dconn_pool) - : dconn_pool_(dconn_pool), client_handler_(nullptr), downstream_(nullptr) {} +DownstreamConnection::DownstreamConnection() + : client_handler_(nullptr), downstream_(nullptr) {} DownstreamConnection::~DownstreamConnection() {} @@ -45,8 +44,4 @@ ClientHandler *DownstreamConnection::get_client_handler() { Downstream *DownstreamConnection::get_downstream() { return downstream_; } -DownstreamConnectionPool *DownstreamConnection::get_dconn_pool() const { - return dconn_pool_; -} - } // namespace shrpx diff --git a/src/shrpx_downstream_connection.h b/src/shrpx_downstream_connection.h index 3d7aba0a..7935099a 100644 --- a/src/shrpx_downstream_connection.h +++ b/src/shrpx_downstream_connection.h @@ -34,11 +34,11 @@ namespace shrpx { class ClientHandler; class Upstream; class Downstream; -class DownstreamConnectionPool; +struct DownstreamAddrGroup; class DownstreamConnection { public: - DownstreamConnection(DownstreamConnectionPool *dconn_pool); + DownstreamConnection(); virtual ~DownstreamConnection(); virtual int attach_downstream(Downstream *downstream) = 0; virtual void detach_downstream(Downstream *downstream) = 0; @@ -56,18 +56,17 @@ public: virtual int on_timeout() { return 0; } virtual void on_upstream_change(Upstream *uptream) = 0; - virtual size_t get_group() const = 0; // true if this object is poolable. virtual bool poolable() const = 0; + virtual DownstreamAddrGroup *get_downstream_addr_group() const = 0; + void set_client_handler(ClientHandler *client_handler); ClientHandler *get_client_handler(); Downstream *get_downstream(); - DownstreamConnectionPool *get_dconn_pool() const; protected: - DownstreamConnectionPool *dconn_pool_; ClientHandler *client_handler_; Downstream *downstream_; }; diff --git a/src/shrpx_downstream_connection_pool.cc b/src/shrpx_downstream_connection_pool.cc index eb9b1c8c..c58ed81a 100644 --- a/src/shrpx_downstream_connection_pool.cc +++ b/src/shrpx_downstream_connection_pool.cc @@ -27,42 +27,35 @@ namespace shrpx { -DownstreamConnectionPool::DownstreamConnectionPool(size_t num_groups) - : gpool_(num_groups) {} +DownstreamConnectionPool::DownstreamConnectionPool() {} DownstreamConnectionPool::~DownstreamConnectionPool() { - for (auto &pool : gpool_) { - for (auto dconn : pool) { - delete dconn; - } + for (auto dconn : pool_) { + delete dconn; } } void DownstreamConnectionPool::add_downstream_connection( std::unique_ptr dconn) { - auto group = dconn->get_group(); - assert(gpool_.size() > group); - gpool_[group].insert(dconn.release()); + pool_.insert(dconn.release()); } std::unique_ptr -DownstreamConnectionPool::pop_downstream_connection(size_t group) { - assert(gpool_.size() > group); - auto &pool = gpool_[group]; - if (pool.empty()) { +DownstreamConnectionPool::pop_downstream_connection() { + if (pool_.empty()) { return nullptr; } - auto dconn = std::unique_ptr(*std::begin(pool)); - pool.erase(std::begin(pool)); + auto it = std::begin(pool_); + auto dconn = std::unique_ptr(*it); + pool_.erase(it); + return dconn; } void DownstreamConnectionPool::remove_downstream_connection( DownstreamConnection *dconn) { - auto group = dconn->get_group(); - assert(gpool_.size() > group); - gpool_[group].erase(dconn); + pool_.erase(dconn); delete dconn; } diff --git a/src/shrpx_downstream_connection_pool.h b/src/shrpx_downstream_connection_pool.h index 1fb889bb..c2edce45 100644 --- a/src/shrpx_downstream_connection_pool.h +++ b/src/shrpx_downstream_connection_pool.h @@ -36,15 +36,15 @@ class DownstreamConnection; class DownstreamConnectionPool { public: - DownstreamConnectionPool(size_t num_groups); + DownstreamConnectionPool(); ~DownstreamConnectionPool(); void add_downstream_connection(std::unique_ptr dconn); - std::unique_ptr pop_downstream_connection(size_t group); + std::unique_ptr pop_downstream_connection(); void remove_downstream_connection(DownstreamConnection *dconn); private: - std::vector> gpool_; + std::set pool_; }; } // namespace shrpx diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index da2d953d..aba79df4 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -37,6 +37,7 @@ #include "shrpx_error.h" #include "shrpx_http.h" #include "shrpx_http2_session.h" +#include "shrpx_worker.h" #include "http2.h" #include "util.h" @@ -44,10 +45,8 @@ using namespace nghttp2; namespace shrpx { -Http2DownstreamConnection::Http2DownstreamConnection( - DownstreamConnectionPool *dconn_pool, Http2Session *http2session) - : DownstreamConnection(dconn_pool), - dlnext(nullptr), +Http2DownstreamConnection::Http2DownstreamConnection(Http2Session *http2session) + : dlnext(nullptr), dlprev(nullptr), http2session_(http2session), sd_(nullptr) {} @@ -557,10 +556,9 @@ int Http2DownstreamConnection::on_timeout() { return submit_rst_stream(downstream_, NGHTTP2_NO_ERROR); } -size_t Http2DownstreamConnection::get_group() const { - // HTTP/2 backend connections are managed by Http2Session object, - // and it stores group index. - return http2session_->get_group(); +DownstreamAddrGroup * +Http2DownstreamConnection::get_downstream_addr_group() const { + return http2session_->get_downstream_addr_group(); } } // namespace shrpx diff --git a/src/shrpx_http2_downstream_connection.h b/src/shrpx_http2_downstream_connection.h index 3561ac7b..6f72a884 100644 --- a/src/shrpx_http2_downstream_connection.h +++ b/src/shrpx_http2_downstream_connection.h @@ -41,8 +41,7 @@ class DownstreamConnectionPool; class Http2DownstreamConnection : public DownstreamConnection { public: - Http2DownstreamConnection(DownstreamConnectionPool *dconn_pool, - Http2Session *http2session); + Http2DownstreamConnection(Http2Session *http2session); virtual ~Http2DownstreamConnection(); virtual int attach_downstream(Downstream *downstream); virtual void detach_downstream(Downstream *downstream); @@ -60,12 +59,13 @@ public: virtual int on_timeout(); virtual void on_upstream_change(Upstream *upstream) {} - virtual size_t get_group() const; // This object is not poolable because we dont' have facility to // migrate to another Http2Session object. virtual bool poolable() const { return false; } + virtual DownstreamAddrGroup *get_downstream_addr_group() const; + int send(); void attach_stream_data(StreamData *sd); diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 7ed816a9..8fcbb34d 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -166,7 +166,7 @@ void writecb(struct ev_loop *loop, ev_io *w, int revents) { } // namespace Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, - Worker *worker, size_t group) + Worker *worker, DownstreamAddrGroup *group) : dlnext(nullptr), dlprev(nullptr), conn_(loop, -1, nullptr, worker->get_mcpool(), @@ -177,9 +177,9 @@ Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, wb_(worker->get_mcpool()), worker_(worker), ssl_ctx_(ssl_ctx), + group_(group), addr_(nullptr), session_(nullptr), - group_(group), state_(DISCONNECTED), connection_check_state_(CONNECTION_CHECK_NONE), flow_control_(false) { @@ -208,8 +208,7 @@ Http2Session::~Http2Session() { if (LOG_ENABLED(INFO)) { SSLOG(INFO, this) << "Removed from http2_freelist"; } - auto &addr_group = worker_->get_downstream_addr_groups()[group_]; - addr_group.http2_freelist.remove(this); + group_->http2_freelist.remove(this); } } @@ -281,8 +280,7 @@ int Http2Session::disconnect(bool hard) { int Http2Session::initiate_connection() { int rv = 0; - auto &addr_group = worker_->get_downstream_addr_groups()[group_]; - auto &addrs = addr_group.addrs; + auto &addrs = group_->addrs; auto worker_blocker = worker_->get_connect_blocker(); if (state_ == DISCONNECTED) { @@ -294,7 +292,7 @@ int Http2Session::initiate_connection() { return -1; } - auto &next_downstream = addr_group.next; + auto &next_downstream = group_->next; auto end = next_downstream; for (;;) { @@ -636,8 +634,7 @@ void Http2Session::remove_downstream_connection( if (LOG_ENABLED(INFO)) { SSLOG(INFO, this) << "Append to Http2Session freelist"; } - auto &addr_group = worker_->get_downstream_addr_groups()[group_]; - addr_group.http2_freelist.append(this); + group_->http2_freelist.append(this); } } @@ -1929,9 +1926,7 @@ bool Http2Session::should_hard_fail() const { } } -const DownstreamAddr *Http2Session::get_addr() const { return addr_; } - -size_t Http2Session::get_group() const { return group_; } +DownstreamAddr *Http2Session::get_addr() const { return addr_; } int Http2Session::handle_downstream_push_promise(Downstream *downstream, int32_t promised_stream_id) { @@ -1950,10 +1945,8 @@ int Http2Session::handle_downstream_push_promise(Downstream *downstream, // promised_downstream->get_stream() still returns 0. auto handler = upstream->get_client_handler(); - auto worker = handler->get_worker(); - auto promised_dconn = - make_unique(worker->get_dconn_pool(), this); + auto promised_dconn = make_unique(this); promised_dconn->set_client_handler(handler); auto ptr = promised_dconn.get(); @@ -2028,11 +2021,9 @@ int Http2Session::handle_downstream_push_promise_complete( size_t Http2Session::get_num_dconns() const { return dconns_.size(); } bool Http2Session::in_freelist() const { - auto &addr_group = worker_->get_downstream_addr_groups()[group_]; - return dlnext != nullptr || dlprev != nullptr || - addr_group.http2_freelist.head == this || - addr_group.http2_freelist.tail == this; + group_->http2_freelist.head == this || + group_->http2_freelist.tail == this; } bool Http2Session::max_concurrency_reached(size_t extra) const { @@ -2045,4 +2036,8 @@ bool Http2Session::max_concurrency_reached(size_t extra) const { session_, NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); } +DownstreamAddrGroup *Http2Session::get_downstream_addr_group() const { + return group_; +} + } // namespace shrpx diff --git a/src/shrpx_http2_session.h b/src/shrpx_http2_session.h index bc81f420..ecea8886 100644 --- a/src/shrpx_http2_session.h +++ b/src/shrpx_http2_session.h @@ -48,6 +48,8 @@ namespace shrpx { class Http2DownstreamConnection; class Worker; +struct DownstreamAddrGroup; +struct DownstreamAddr; struct StreamData { StreamData *dlnext, *dlprev; @@ -57,7 +59,7 @@ struct StreamData { class Http2Session { public: Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, Worker *worker, - size_t group); + DownstreamAddrGroup *group); ~Http2Session(); // If hard is true, all pending requests are abandoned and @@ -145,9 +147,9 @@ public: void submit_pending_requests(); - const DownstreamAddr *get_addr() const; + DownstreamAddr *get_addr() const; - size_t get_group() const; + DownstreamAddrGroup *get_downstream_addr_group() const; int handle_downstream_push_promise(Downstream *downstream, int32_t promised_stream_id); @@ -219,10 +221,10 @@ private: Worker *worker_; // NULL if no TLS is configured SSL_CTX *ssl_ctx_; + DownstreamAddrGroup *group_; // Address of remote endpoint - const DownstreamAddr *addr_; + DownstreamAddr *addr_; nghttp2_session *session_; - size_t group_; int state_; int connection_check_state_; bool flow_control_; diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index e8f0e298..cecc9961 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -111,11 +111,10 @@ void connectcb(struct ev_loop *loop, ev_io *w, int revents) { } } // namespace -HttpDownstreamConnection::HttpDownstreamConnection( - DownstreamConnectionPool *dconn_pool, size_t group, struct ev_loop *loop, - Worker *worker) - : DownstreamConnection(dconn_pool), - conn_(loop, -1, nullptr, worker->get_mcpool(), +HttpDownstreamConnection::HttpDownstreamConnection(DownstreamAddrGroup *group, + struct ev_loop *loop, + Worker *worker) + : conn_(loop, -1, nullptr, worker->get_mcpool(), get_config()->conn.downstream.timeout.write, get_config()->conn.downstream.timeout.read, {}, {}, connectcb, readcb, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold, @@ -124,10 +123,10 @@ HttpDownstreamConnection::HttpDownstreamConnection( do_write_(&HttpDownstreamConnection::noop), worker_(worker), ssl_ctx_(worker->get_cl_ssl_ctx()), + group_(group), addr_(nullptr), ioctrl_(&conn_.rlimit), - response_htp_{0}, - group_(group) {} + response_htp_{0} {} HttpDownstreamConnection::~HttpDownstreamConnection() {} @@ -157,9 +156,8 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { conn_.set_ssl(ssl); } - auto &groups = worker_->get_downstream_addr_groups(); - auto &addrs = groups[group_].addrs; - auto &next_downstream = groups[group_].next; + auto &addrs = group_->addrs; + auto &next_downstream = group_->next; auto end = next_downstream; for (;;) { auto &addr = addrs[next_downstream]; @@ -508,8 +506,8 @@ void idle_readcb(struct ev_loop *loop, ev_io *w, int revents) { if (LOG_ENABLED(INFO)) { DCLOG(INFO, dconn) << "Idle connection EOF"; } - auto dconn_pool = dconn->get_dconn_pool(); - dconn_pool->remove_downstream_connection(dconn); + auto &dconn_pool = dconn->get_downstream_addr_group()->dconn_pool; + dconn_pool.remove_downstream_connection(dconn); // dconn was deleted } } // namespace @@ -521,8 +519,8 @@ void idle_timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) { if (LOG_ENABLED(INFO)) { DCLOG(INFO, dconn) << "Idle connection timeout"; } - auto dconn_pool = dconn->get_dconn_pool(); - dconn_pool->remove_downstream_connection(dconn); + auto &dconn_pool = dconn->get_downstream_addr_group()->dconn_pool; + dconn_pool.remove_downstream_connection(dconn); // dconn was deleted } } // namespace @@ -1033,7 +1031,7 @@ int HttpDownstreamConnection::process_input(const uint8_t *data, } int HttpDownstreamConnection::connected() { - auto connect_blocker = addr_->connect_blocker; + auto &connect_blocker = addr_->connect_blocker; if (!util::check_socket_connected(conn_.fd)) { conn_.wlimit.stopw(); @@ -1083,8 +1081,11 @@ void HttpDownstreamConnection::signal_write() { ev_feed_event(conn_.loop, &conn_.wev, EV_WRITE); } -size_t HttpDownstreamConnection::get_group() const { return group_; } - int HttpDownstreamConnection::noop() { return 0; } +DownstreamAddrGroup * +HttpDownstreamConnection::get_downstream_addr_group() const { + return group_; +} + } // namespace shrpx diff --git a/src/shrpx_http_downstream_connection.h b/src/shrpx_http_downstream_connection.h index 9689283d..32c41702 100644 --- a/src/shrpx_http_downstream_connection.h +++ b/src/shrpx_http_downstream_connection.h @@ -37,11 +37,13 @@ namespace shrpx { class DownstreamConnectionPool; class Worker; +struct DownstreamAddrGroup; +struct DownstreamAddr; class HttpDownstreamConnection : public DownstreamConnection { public: - HttpDownstreamConnection(DownstreamConnectionPool *dconn_pool, size_t group, - struct ev_loop *loop, Worker *worker); + HttpDownstreamConnection(DownstreamAddrGroup *group, struct ev_loop *loop, + Worker *worker); virtual ~HttpDownstreamConnection(); virtual int attach_downstream(Downstream *downstream); virtual void detach_downstream(Downstream *downstream); @@ -58,10 +60,11 @@ public: virtual int on_write(); virtual void on_upstream_change(Upstream *upstream); - virtual size_t get_group() const; virtual bool poolable() const { return true; } + virtual DownstreamAddrGroup *get_downstream_addr_group() const; + int read_clear(); int write_clear(); int read_tls(); @@ -81,11 +84,11 @@ private: Worker *worker_; // nullptr if TLS is not used. SSL_CTX *ssl_ctx_; + DownstreamAddrGroup *group_; // Address of remote endpoint DownstreamAddr *addr_; IOControl ioctrl_; http_parser response_htp_; - size_t group_; }; } // namespace shrpx diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 51ee9ad3..d2115d25 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -71,14 +71,13 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, ssl::CertLookupTree *cert_tree, const std::shared_ptr &ticket_keys) : randgen_(rd()), - dconn_pool_(get_config()->conn.downstream.addr_groups.size()), - worker_stat_(get_config()->conn.downstream.addr_groups.size()), + worker_stat_{}, loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), cert_tree_(cert_tree), ticket_keys_(ticket_keys), - downstream_addr_groups_(get_config()->conn.downstream.addr_groups), + downstream_addr_groups_(get_config()->conn.downstream.addr_groups.size()), connect_blocker_(make_unique(randgen_, loop_)), graceful_shutdown_(false) { ev_async_init(&w_, eventcb); @@ -97,9 +96,26 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, StringRef{session_cacheconf.memcached.host}, &mcpool_); } - for (auto &group : downstream_addr_groups_) { - for (auto &addr : group.addrs) { - addr.connect_blocker = new ConnectBlocker(randgen_, loop_); + auto &downstreamconf = get_config()->conn.downstream; + + for (size_t i = 0; i < downstreamconf.addr_groups.size(); ++i) { + auto &src = downstreamconf.addr_groups[i]; + auto &dst = downstream_addr_groups_[i]; + + dst.pattern = src.pattern; + dst.addrs.resize(src.addrs.size()); + + for (size_t j = 0; j < src.addrs.size(); ++j) { + auto &src_addr = src.addrs[j]; + auto &dst_addr = dst.addrs[j]; + + dst_addr.addr = src_addr.addr; + dst_addr.host = src_addr.host; + dst_addr.hostport = src_addr.hostport; + dst_addr.port = src_addr.port; + dst_addr.host_unix = src_addr.host_unix; + + dst_addr.connect_blocker = make_unique(randgen_, loop_); } } } @@ -107,12 +123,6 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, Worker::~Worker() { ev_async_stop(loop_, &w_); ev_timer_stop(loop_, &mcpool_clear_timer_); - - for (auto &group : downstream_addr_groups_) { - for (auto &addr : group.addrs) { - delete addr.connect_blocker; - } - } } void Worker::schedule_clear_mcpool() { @@ -234,8 +244,6 @@ void Worker::set_ticket_keys(std::shared_ptr ticket_keys) { WorkerStat *Worker::get_worker_stat() { return &worker_stat_; } -DownstreamConnectionPool *Worker::get_dconn_pool() { return &dconn_pool_; } - struct ev_loop *Worker::get_loop() const { return loop_; } @@ -279,4 +287,93 @@ ConnectBlocker *Worker::get_connect_blocker() const { return connect_blocker_.get(); } +namespace { +size_t match_downstream_addr_group_host( + const Router &router, const StringRef &host, const StringRef &path, + const std::vector &groups, size_t catch_all) { + if (path.empty() || path[0] != '/') { + auto group = router.match(host, StringRef::from_lit("/")); + if (group != -1) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Found pattern with query " << host + << ", matched pattern=" << groups[group].pattern; + } + return group; + } + return catch_all; + } + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Perform mapping selection, using host=" << host + << ", path=" << path; + } + + auto group = router.match(host, path); + if (group != -1) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Found pattern with query " << host << path + << ", matched pattern=" << groups[group].pattern; + } + return group; + } + + group = router.match("", path); + if (group != -1) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Found pattern with query " << path + << ", matched pattern=" << groups[group].pattern; + } + return group; + } + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "None match. Use catch-all pattern"; + } + return catch_all; +} +} // namespace + +size_t match_downstream_addr_group( + const Router &router, const StringRef &hostport, const StringRef &raw_path, + const std::vector &groups, size_t catch_all) { + if (std::find(std::begin(hostport), std::end(hostport), '/') != + std::end(hostport)) { + // We use '/' specially, and if '/' is included in host, it breaks + // our code. Select catch-all case. + return catch_all; + } + + auto fragment = std::find(std::begin(raw_path), std::end(raw_path), '#'); + auto query = std::find(std::begin(raw_path), fragment, '?'); + auto path = StringRef{std::begin(raw_path), query}; + + if (hostport.empty()) { + return match_downstream_addr_group_host(router, hostport, path, groups, + catch_all); + } + + std::string host; + if (hostport[0] == '[') { + // assume this is IPv6 numeric address + auto p = std::find(std::begin(hostport), std::end(hostport), ']'); + if (p == std::end(hostport)) { + return catch_all; + } + if (p + 1 < std::end(hostport) && *(p + 1) != ':') { + return catch_all; + } + host.assign(std::begin(hostport), p + 1); + } else { + auto p = std::find(std::begin(hostport), std::end(hostport), ':'); + if (p == std::begin(hostport)) { + return catch_all; + } + host.assign(std::begin(hostport), p); + } + + util::inp_strlower(host); + return match_downstream_addr_group_host(router, StringRef{host}, path, groups, + catch_all); +} + } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 35917ec9..1f0c7d78 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -67,9 +67,39 @@ namespace ssl { class CertLookupTree; } // namespace ssl -struct WorkerStat { - WorkerStat(size_t num_groups) : num_connections(0) {} +struct DownstreamAddr { + Address addr; + // backend address. If |host_unix| is true, this is UNIX domain + // socket path. + ImmutableString host; + ImmutableString hostport; + // backend port. 0 if |host_unix| is true. + uint16_t port; + // true if |host| contains UNIX domain socket path. + bool host_unix; + std::unique_ptr connect_blocker; + // Client side TLS session cache + TLSSessionCache tls_session_cache; +}; + +struct DownstreamAddrGroup { + ImmutableString pattern; + std::vector addrs; + // List of Http2Session which is not fully utilized (i.e., the + // server advertized maximum concurrency is not reached). We will + // coalesce as much stream as possible in one Http2Session to fully + // utilize TCP connection. + // + // TODO Verify that this approach performs better in performance + // wise. + DList http2_freelist; + DownstreamConnectionPool dconn_pool; + // Next downstream address index in addrs. + size_t next; +}; + +struct WorkerStat { size_t num_connections; }; @@ -110,7 +140,6 @@ public: void set_ticket_keys(std::shared_ptr ticket_keys); WorkerStat *get_worker_stat(); - DownstreamConnectionPool *get_dconn_pool(); struct ev_loop *get_loop() const; SSL_CTX *get_sv_ssl_ctx() const; SSL_CTX *get_cl_ssl_ctx() const; @@ -145,7 +174,6 @@ private: ev_async w_; ev_timer mcpool_clear_timer_; MemchunkPool mcpool_; - DownstreamConnectionPool dconn_pool_; WorkerStat worker_stat_; std::unique_ptr session_cache_memcached_dispatcher_; @@ -169,6 +197,16 @@ private: bool graceful_shutdown_; }; +// Selects group based on request's |hostport| and |path|. |hostport| +// is the value taken from :authority or host header field, and may +// contain port. The |path| may contain query part. We require the +// catch-all pattern in place, so this function always selects one +// group. The catch-all group index is given in |catch_all|. All +// patterns are given in |groups|. +size_t match_downstream_addr_group( + const Router &router, const StringRef &hostport, const StringRef &path, + const std::vector &groups, size_t catch_all); + } // namespace shrpx #endif // SHRPX_WORKER_H diff --git a/src/template.h b/src/template.h index 3b1727fc..7c9dc6f1 100644 --- a/src/template.h +++ b/src/template.h @@ -101,11 +101,8 @@ template bool test_flags(T t, F flags) { template struct DList { DList() : head(nullptr), tail(nullptr), n(0) {} - // We should delete these copy ctor and assignment operator. We - // need to them where copy is required before we add item to it. If - // you doubt, make them delete and try to compile. - DList(const DList &) = default; - DList &operator=(const DList &) = default; + DList(const DList &) = delete; + DList &operator=(const DList &) = delete; DList(DList &&other) : head(other.head), tail(other.tail), n(other.n) { other.head = other.tail = nullptr; From aa892e4d37ad5bef237e8796bb6ea3be4ee199f9 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 27 Feb 2016 23:40:04 +0900 Subject: [PATCH 075/147] nghttpx: Share TLS session cache between HTTP/2 and HTTP/1 backend --- src/shrpx_http2_session.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 8fcbb34d..79920fe8 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -411,6 +411,13 @@ int Http2Session::initiate_connection() { // at the time of this writing). SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str()); } + + auto tls_session = ssl::reuse_tls_session(addr_); + if (tls_session) { + SSL_set_session(conn_.tls.ssl, tls_session); + SSL_SESSION_free(tls_session); + } + // If state_ == PROXY_CONNECTED, we has connected to the proxy // using conn_.fd and tunnel has been established. if (state_ == DISCONNECTED) { @@ -1838,6 +1845,13 @@ int Http2Session::tls_handshake() { return -1; } + if (!SSL_session_reused(conn_.tls.ssl)) { + auto tls_session = SSL_get0_session(conn_.tls.ssl); + if (tls_session) { + ssl::try_cache_tls_session(addr_, tls_session, ev_now(conn_.loop)); + } + } + read_ = &Http2Session::read_tls; write_ = &Http2Session::write_tls; From c731d1fea576d9bba5b2d955006457d0cbede972 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 27 Feb 2016 23:44:35 +0900 Subject: [PATCH 076/147] nghttpx: Deprecate --backend-http2-connections-per-worker option --- src/shrpx.cc | 9 --------- src/shrpx_config.cc | 4 ++-- src/shrpx_config.h | 1 - 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 7da056a3..e0e38688 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1352,15 +1352,6 @@ Performance: accepts. Setting 0 means unlimited. Default: )" << get_config()->conn.upstream.worker_connections << R"( - --backend-http2-connections-per-worker= - Set maximum number of backend HTTP/2 physical - connections per worker. If pattern is used in -b - option, this limit is applied to each pattern group (in - other words, each pattern group can have maximum - HTTP/2 connections). The default value is 0, which - means that the value is adjusted to the number of - backend addresses. If pattern is used, this adjustment - is done for each pattern group. --backend-http1-connections-per-host= Set maximum number of backend concurrent HTTP/1 connections per origin host. This option is meaningful diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 8b5cc537..7cb19d9f 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2077,8 +2077,8 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER: - return parse_uint(&mod_config()->http2.downstream.connections_per_worker, - opt, optarg); + LOG(WARN) << opt << ": deprecated."; + return 0; case SHRPX_OPTID_FETCH_OCSP_RESPONSE_FILE: mod_config()->tls.ocsp.fetch_ocsp_response_file = optarg; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 10742685..69d807f4 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -485,7 +485,6 @@ struct Http2Config { nghttp2_session_callbacks *callbacks; size_t window_bits; size_t connection_window_bits; - size_t connections_per_worker; } downstream; struct { ev_tstamp stream_read; From aafcc55006fcf6fa3185b86f7d19e259089c0be1 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 28 Feb 2016 00:06:40 +0900 Subject: [PATCH 077/147] nghttpx: Deprecate --http2-max-concurrent-streams option We added 2 new option instead: --frontend-http2-max-concurrent-streams and --backend-http2-max-concurrent-streams. --- gennghttpxfun.py | 4 +++- src/shrpx.cc | 32 +++++++++++++++++++++++++++----- src/shrpx_config.cc | 30 ++++++++++++++++++++++++++++-- src/shrpx_config.h | 7 ++++++- src/shrpx_http2_session.cc | 2 +- src/shrpx_http2_upstream.cc | 2 +- src/shrpx_spdy_upstream.cc | 2 +- 7 files changed, 67 insertions(+), 12 deletions(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 1d9dc846..8ff8cf83 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -122,7 +122,9 @@ OPTIONS = [ "tls-ticket-key-memcached-cert-file", "tls-ticket-key-memcached-private-key-file", "tls-ticket-key-memcached-address-family", - "backend-address-family" + "backend-address-family", + "frontend-http2-max-concurrent-streams", + "backend-http2-max-concurrent-streams" ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index e0e38688..975f390c 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1096,6 +1096,7 @@ void fill_default_config() { // HTTP/2 SPDY/3.1 has connection-level flow control. The default // window size for HTTP/2 is 64KiB - 1. SPDY/3's default is 64KiB upstreamconf.connection_window_bits = 16; + upstreamconf.max_concurrent_streams = 100; nghttp2_option_new(&upstreamconf.option); nghttp2_option_set_no_auto_window_update(upstreamconf.option, 1); @@ -1106,14 +1107,13 @@ void fill_default_config() { auto &downstreamconf = http2conf.downstream; downstreamconf.window_bits = 16; downstreamconf.connection_window_bits = 16; + downstreamconf.max_concurrent_streams = 100; nghttp2_option_new(&downstreamconf.option); nghttp2_option_set_no_auto_window_update(downstreamconf.option, 1); nghttp2_option_set_peer_max_concurrent_streams(downstreamconf.option, 100); } - http2conf.max_concurrent_streams = 100; - auto &loggingconf = mod_config()->logging; { auto &accessconf = loggingconf.access; @@ -1621,10 +1621,18 @@ SSL/TLS: the complete HTTP/2 cipher suites black list. HTTP/2 and SPDY: - -c, --http2-max-concurrent-streams= + -c, --frontend-http2-max-concurrent-streams= Set the maximum number of the concurrent streams in one - HTTP/2 and SPDY session. - Default: )" << get_config()->http2.max_concurrent_streams << R"( + frontend HTTP/2 and SPDY session. + Default: )" + << get_config()->http2.upstream.max_concurrent_streams << R"( + --backend-http2-max-concurrent-streams= + Set the maximum number of the concurrent streams in one + backend HTTP/2 session. This sets maximum number of + concurrent opened pushed streams. The maximum number of + concurrent requests are set by a remote server. + Default: )" + << get_config()->http2.downstream.max_concurrent_streams << R"( --frontend-http2-window-bits= Sets the per-stream initial window size of HTTP/2 SPDY frontend connection. For HTTP/2, the size is 2**-1. @@ -2451,6 +2459,10 @@ int main(int argc, char **argv) { {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY, required_argument, &flag, 115}, {SHRPX_OPT_BACKEND_ADDRESS_FAMILY, required_argument, &flag, 116}, + {SHRPX_OPT_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS, required_argument, + &flag, 117}, + {SHRPX_OPT_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS, required_argument, + &flag, 118}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2946,6 +2958,16 @@ int main(int argc, char **argv) { // --backend-address-family cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_ADDRESS_FAMILY, optarg); break; + case 117: + // --frontend-http2-max-concurrent-streams + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS, + optarg); + break; + case 118: + // --backend-http2-max-concurrent-streams + cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS, + optarg); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 7cb19d9f..85c36e68 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -660,6 +660,7 @@ enum { SHRPX_OPTID_BACKEND_HTTP1_TLS, SHRPX_OPTID_BACKEND_HTTP2_CONNECTION_WINDOW_BITS, SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER, + SHRPX_OPTID_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS, SHRPX_OPTID_BACKEND_HTTP2_WINDOW_BITS, SHRPX_OPTID_BACKEND_IPV4, SHRPX_OPTID_BACKEND_IPV6, @@ -692,6 +693,7 @@ enum { SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS, SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER, SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER, + SHRPX_OPTID_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS, SHRPX_OPTID_FRONTEND_HTTP2_READ_TIMEOUT, SHRPX_OPTID_FRONTEND_HTTP2_WINDOW_BITS, SHRPX_OPTID_FRONTEND_NO_TLS, @@ -1398,6 +1400,9 @@ int option_lookup_token(const char *name, size_t namelen) { if (util::strieq_l("backend-http2-connection-window-bit", name, 35)) { return SHRPX_OPTID_BACKEND_HTTP2_CONNECTION_WINDOW_BITS; } + if (util::strieq_l("backend-http2-max-concurrent-stream", name, 35)) { + return SHRPX_OPTID_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS; + } break; } break; @@ -1412,6 +1417,9 @@ int option_lookup_token(const char *name, size_t namelen) { if (util::strieq_l("frontend-http2-connection-window-bit", name, 36)) { return SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS; } + if (util::strieq_l("frontend-http2-max-concurrent-stream", name, 36)) { + return SHRPX_OPTID_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS; + } break; } break; @@ -1559,8 +1567,20 @@ int parse_config(const char *opt, const char *optarg, #else // !NOTHREADS return parse_uint(&mod_config()->num_worker, opt, optarg); #endif // !NOTHREADS - case SHRPX_OPTID_HTTP2_MAX_CONCURRENT_STREAMS: - return parse_uint(&mod_config()->http2.max_concurrent_streams, opt, optarg); + case SHRPX_OPTID_HTTP2_MAX_CONCURRENT_STREAMS: { + LOG(WARN) << opt << ": deprecated. Use " + << SHRPX_OPT_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS << " and " + << SHRPX_OPT_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS << " instead."; + size_t n; + if (parse_uint(&n, opt, optarg) != 0) { + return -1; + } + auto &http2conf = mod_config()->http2; + http2conf.upstream.max_concurrent_streams = n; + http2conf.downstream.max_concurrent_streams = n; + + return 0; + } case SHRPX_OPTID_LOG_LEVEL: if (Log::set_severity_level_by_name(optarg) == -1) { LOG(ERROR) << opt << ": Invalid severity level: " << optarg; @@ -2314,6 +2334,12 @@ int parse_config(const char *opt, const char *optarg, case SHRPX_OPTID_BACKEND_ADDRESS_FAMILY: return parse_address_family(&mod_config()->conn.downstream.family, opt, optarg); + case SHRPX_OPTID_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS: + return parse_uint(&mod_config()->http2.upstream.max_concurrent_streams, opt, + optarg); + case SHRPX_OPTID_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS: + return parse_uint(&mod_config()->http2.downstream.max_concurrent_streams, + opt, optarg); case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 69d807f4..039d2a9d 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -228,6 +228,10 @@ constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE[] = constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY[] = "tls-ticket-key-memcached-address-family"; constexpr char SHRPX_OPT_BACKEND_ADDRESS_FAMILY[] = "backend-address-family"; +constexpr char SHRPX_OPT_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS[] = + "frontend-http2-max-concurrent-streams"; +constexpr char SHRPX_OPT_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS[] = + "backend-http2-max-concurrent-streams"; constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -479,18 +483,19 @@ struct Http2Config { nghttp2_session_callbacks *callbacks; size_t window_bits; size_t connection_window_bits; + size_t max_concurrent_streams; } upstream; struct { nghttp2_option *option; nghttp2_session_callbacks *callbacks; size_t window_bits; size_t connection_window_bits; + size_t max_concurrent_streams; } downstream; struct { ev_tstamp stream_read; ev_tstamp stream_write; } timeout; - size_t max_concurrent_streams; bool no_cookie_crumbling; bool no_server_push; }; diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 79920fe8..b988d2e6 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -1453,7 +1453,7 @@ int Http2Session::connection_made() { std::array entry; size_t nentry = 2; entry[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS; - entry[0].value = http2conf.max_concurrent_streams; + entry[0].value = http2conf.downstream.max_concurrent_streams; entry[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE; entry[1].value = (1 << http2conf.downstream.window_bits) - 1; diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 0c8795dc..0803433c 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -846,7 +846,7 @@ Http2Upstream::Http2Upstream(ClientHandler *handler) // TODO Maybe call from outside? std::array entry; entry[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS; - entry[0].value = http2conf.max_concurrent_streams; + entry[0].value = http2conf.upstream.max_concurrent_streams; entry[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE; entry[1].value = (1 << http2conf.upstream.window_bits) - 1; diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index d9c22209..70d4096c 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -547,7 +547,7 @@ SpdyUpstream::SpdyUpstream(uint16_t version, ClientHandler *handler) // TODO Maybe call from outside? std::array entry; entry[0].settings_id = SPDYLAY_SETTINGS_MAX_CONCURRENT_STREAMS; - entry[0].value = http2conf.max_concurrent_streams; + entry[0].value = http2conf.upstream.max_concurrent_streams; entry[0].flags = SPDYLAY_ID_FLAG_SETTINGS_NONE; entry[1].settings_id = SPDYLAY_SETTINGS_INITIAL_WINDOW_SIZE; From 1e1752266fbcb327915e2eb7053edcd295df43d4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 28 Feb 2016 00:10:38 +0900 Subject: [PATCH 078/147] Update doc --- doc/sources/nghttpx-howto.rst | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index ce142503..e7dec217 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -135,10 +135,8 @@ HTTP/1 frontend connection can be upgraded to HTTP/2 using HTTP Upgrade. To disable SSL/TLS in backend connection, use :option:`--backend-no-tls` option. -By default, the number of backend HTTP/2 connections per worker -(thread) is determined by number of :option:`--backend` option. To -adjust this value, use -:option:`--backend-http2-connections-per-worker` option. +A single HTTP/2 backend connection is shared among multiple frontend +connections. The backend server is supporsed to be a HTTP/2 web server (e.g., nghttpd). The one use-case of this mode is utilize existing HTTP/1 @@ -171,10 +169,8 @@ HTTP/1 frontend connection can be upgraded to HTTP/2 using HTTP Upgrade. To disable SSL/TLS in backend connection, use :option:`--backend-no-tls` option. -By default, the number of backend HTTP/2 connections per worker -(thread) is determined by number of :option:`--backend` option. To -adjust this value, use -:option:`--backend-http2-connections-per-worker` option. +A single HTTP/2 backend connection is shared among multiple frontend +connections. The backend server must be a HTTP/2 proxy. You can use nghttpx in `HTTP/2 proxy mode`_ as backend server. The one use-case of this mode @@ -215,10 +211,8 @@ With :option:`--frontend-no-tls` option, SSL/TLS is turned off in frontend connection, so the connection gets insecure. To disable SSL/TLS in backend connection, use :option:`--backend-no-tls` option. -By default, the number of backend HTTP/2 connections per worker -(thread) is determined by number of :option:`--backend` option. To -adjust this value, use -:option:`--backend-http2-connections-per-worker` option. +A single HTTP/2 backend connection is shared among multiple frontend +connections. The backend server is supporsed to be a HTTP/2 web server or HTTP/2 proxy. If backend server is HTTP/2 proxy, use @@ -386,6 +380,3 @@ servers ``serv1:3000`` and ``serv2:3000`` for request host backend=serv1,3000;example.com/myservice backend=serv2,3000;example.com/myservice - -For HTTP/2 backend, see also -:option:`--backend-http2-connections-per-worker` option. From 36f6a009b8311cd9dd886ceace3fc77275c24560 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 28 Feb 2016 00:11:55 +0900 Subject: [PATCH 079/147] nghttpx: Effectively disable backend HTTP/2 connection flow control This is required to avoid session stall because of too slow frontend connection. --- src/shrpx.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 975f390c..db01ba57 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1106,7 +1106,7 @@ void fill_default_config() { { auto &downstreamconf = http2conf.downstream; downstreamconf.window_bits = 16; - downstreamconf.connection_window_bits = 16; + downstreamconf.connection_window_bits = 30; downstreamconf.max_concurrent_streams = 100; nghttp2_option_new(&downstreamconf.option); From e7601cde8ae69f3cabd6b6c2121a6e17fbd35e0a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 28 Feb 2016 12:41:34 +0900 Subject: [PATCH 080/147] nghttpx: Don't share session which is already in draining state --- src/shrpx_http2_session.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index b988d2e6..8bb6ab22 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -2045,9 +2045,12 @@ bool Http2Session::max_concurrency_reached(size_t extra) const { return dconns_.size() + extra >= 100; } - return dconns_.size() + extra >= - nghttp2_session_get_remote_settings( - session_, NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); + // If session does not allow further requests, it effectively means + // that maximum concurrency is reached. + return !nghttp2_session_check_request_allowed(session_) || + dconns_.size() + extra >= + nghttp2_session_get_remote_settings( + session_, NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); } DownstreamAddrGroup *Http2Session::get_downstream_addr_group() const { From 1832f78684e1a1cf0f7535eab4f5336eacebc747 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 28 Feb 2016 16:56:14 +0900 Subject: [PATCH 081/147] nghttpx: Move downstream proto to DownstreamAddrGroup --- src/shrpx.cc | 16 +++++-- src/shrpx_client_handler.cc | 4 +- src/shrpx_config.h | 6 +-- src/shrpx_connection.cc | 5 ++- src/shrpx_connection.h | 6 ++- src/shrpx_connection_handler.cc | 6 +-- src/shrpx_http2_session.cc | 4 +- src/shrpx_http2_upstream.cc | 4 +- src/shrpx_http_downstream_connection.cc | 4 +- src/shrpx_memcached_connection.cc | 2 +- src/shrpx_spdy_upstream.cc | 4 +- src/shrpx_ssl.cc | 60 +++++++++++++++---------- src/shrpx_ssl.h | 10 ++++- src/shrpx_worker.cc | 1 + src/shrpx_worker.h | 2 + 15 files changed, 85 insertions(+), 49 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index db01ba57..c52a3c99 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2067,13 +2067,15 @@ void process_options( upstreamconf.no_tls = true; } + shrpx_proto default_proto; if (get_config()->client_mode || get_config()->http2_bridge) { - downstreamconf.proto = PROTO_HTTP2; + default_proto = PROTO_HTTP2; } else { - downstreamconf.proto = PROTO_HTTP; + default_proto = PROTO_HTTP1; } - if (downstreamconf.proto == PROTO_HTTP && !downstreamconf.http1_tls) { + if (!get_config()->client_mode && !get_config()->http2_bridge && + !downstreamconf.http1_tls) { downstreamconf.no_tls = true; } @@ -2102,6 +2104,7 @@ void process_options( addr.port = DEFAULT_DOWNSTREAM_PORT; DownstreamAddrGroupConfig g(StringRef::from_lit("/")); + g.proto = default_proto; g.addrs.push_back(std::move(addr)); mod_config()->router.add_route(StringRef{g.pattern}, addr_groups.size()); addr_groups.push_back(std::move(g)); @@ -2113,12 +2116,19 @@ void process_options( std::move(std::begin(g.addrs), std::end(g.addrs), std::back_inserter(catch_all.addrs)); } + catch_all.proto = default_proto; std::vector().swap(addr_groups); // maybe not necessary? mod_config()->router = Router(); mod_config()->router.add_route(StringRef{catch_all.pattern}, addr_groups.size()); addr_groups.push_back(std::move(catch_all)); + } else { + for (auto &g : addr_groups) { + if (g.proto == PROTO_NONE) { + g.proto = default_proto; + } + } } if (LOG_ENABLED(INFO)) { diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 7032471d..9dbd6453 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -385,7 +385,7 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl, get_config()->conn.upstream.ratelimit.write, get_config()->conn.upstream.ratelimit.read, writecb, readcb, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold, - get_config()->tls.dyn_rec.idle_timeout), + get_config()->tls.dyn_rec.idle_timeout, PROTO_NONE), ipaddr_(ipaddr), port_(port), faddr_(faddr), @@ -712,7 +712,7 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { << " Create new one"; } - if (downstreamconf.proto == PROTO_HTTP2) { + if (group.proto == PROTO_HTTP2) { if (group.http2_freelist.empty()) { if (LOG_ENABLED(INFO)) { CLOG(INFO, this) diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 039d2a9d..b54951dd 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -235,7 +235,7 @@ constexpr char SHRPX_OPT_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS[] = constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; -enum shrpx_proto { PROTO_HTTP2, PROTO_HTTP }; +enum shrpx_proto { PROTO_NONE, PROTO_HTTP1, PROTO_HTTP2, PROTO_MEMCACHED }; enum shrpx_forwarded_param { FORWARDED_NONE = 0, @@ -306,6 +306,8 @@ struct DownstreamAddrGroupConfig { ImmutableString pattern; std::vector addrs; + // Application protocol used in this group + shrpx_proto proto; }; struct TicketKey { @@ -561,8 +563,6 @@ struct ConnectionConfig { size_t connections_per_frontend; size_t request_buffer_size; size_t response_buffer_size; - // downstream protocol; this will be determined by given options. - shrpx_proto proto; // Address family of backend connection. One of either AF_INET, // AF_INET6 or AF_UNSPEC. This is ignored if backend connection // is made via Unix domain socket. diff --git a/src/shrpx_connection.cc b/src/shrpx_connection.cc index 6bbaab3b..418afb12 100644 --- a/src/shrpx_connection.cc +++ b/src/shrpx_connection.cc @@ -47,7 +47,7 @@ Connection::Connection(struct ev_loop *loop, int fd, SSL *ssl, const RateLimitConfig &read_limit, IOCb writecb, IOCb readcb, TimerCb timeoutcb, void *data, size_t tls_dyn_rec_warmup_threshold, - ev_tstamp tls_dyn_rec_idle_timeout) + ev_tstamp tls_dyn_rec_idle_timeout, shrpx_proto proto) : tls{DefaultMemchunks(mcpool), DefaultPeekMemchunks(mcpool)}, wlimit(loop, &wev, write_limit.rate, write_limit.burst), rlimit(loop, &rev, read_limit.rate, read_limit.burst, this), @@ -58,7 +58,8 @@ Connection::Connection(struct ev_loop *loop, int fd, SSL *ssl, data(data), fd(fd), tls_dyn_rec_warmup_threshold(tls_dyn_rec_warmup_threshold), - tls_dyn_rec_idle_timeout(tls_dyn_rec_idle_timeout) { + tls_dyn_rec_idle_timeout(tls_dyn_rec_idle_timeout), + proto(proto) { ev_io_init(&wev, writecb, fd, EV_WRITE); ev_io_init(&rev, readcb, fd, EV_READ); diff --git a/src/shrpx_connection.h b/src/shrpx_connection.h index 30ff448d..71ba33f4 100644 --- a/src/shrpx_connection.h +++ b/src/shrpx_connection.h @@ -77,7 +77,7 @@ struct Connection { const RateLimitConfig &write_limit, const RateLimitConfig &read_limit, IOCb writecb, IOCb readcb, TimerCb timeoutcb, void *data, size_t tls_dyn_rec_warmup_threshold, - ev_tstamp tls_dyn_rec_idle_timeout); + ev_tstamp tls_dyn_rec_idle_timeout, shrpx_proto proto); ~Connection(); void disconnect(); @@ -133,6 +133,10 @@ struct Connection { int fd; size_t tls_dyn_rec_warmup_threshold; ev_tstamp tls_dyn_rec_idle_timeout; + // Application protocol used over the connection. This field is not + // used in this object at the moment. The rest of the program may + // use this value when it is useful. + shrpx_proto proto; }; } // namespace shrpx diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 008f1289..4da62790 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -203,7 +203,7 @@ int ConnectionHandler::create_single_worker() { nb_.get(), #endif // HAVE_NEVERBLEED StringRef{tlsconf.cacert}, StringRef{memcachedconf.cert_file}, - StringRef{memcachedconf.private_key_file}, StringRef(), nullptr); + StringRef{memcachedconf.private_key_file}, nullptr); all_ssl_ctx_.push_back(session_cache_ssl_ctx); } @@ -253,7 +253,7 @@ int ConnectionHandler::create_worker_thread(size_t num) { nb_.get(), #endif // HAVE_NEVERBLEED StringRef{tlsconf.cacert}, StringRef{memcachedconf.cert_file}, - StringRef{memcachedconf.private_key_file}, StringRef{}, nullptr); + StringRef{memcachedconf.private_key_file}, nullptr); all_ssl_ctx_.push_back(session_cache_ssl_ctx); } auto worker = @@ -767,7 +767,7 @@ SSL_CTX *ConnectionHandler::create_tls_ticket_key_memcached_ssl_ctx() { nb_.get(), #endif // HAVE_NEVERBLEED StringRef{tlsconf.cacert}, StringRef{memcachedconf.cert_file}, - StringRef{memcachedconf.private_key_file}, StringRef{}, nullptr); + StringRef{memcachedconf.private_key_file}, nullptr); all_ssl_ctx_.push_back(ssl_ctx); diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 8bb6ab22..dd8cd5e4 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -173,7 +173,7 @@ Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, get_config()->conn.downstream.timeout.write, get_config()->conn.downstream.timeout.read, {}, {}, writecb, readcb, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold, - get_config()->tls.dyn_rec.idle_timeout), + get_config()->tls.dyn_rec.idle_timeout, PROTO_HTTP2), wb_(worker->get_mcpool()), worker_(worker), ssl_ctx_(ssl_ctx), @@ -398,6 +398,8 @@ int Http2Session::initiate_connection() { return -1; } + ssl::setup_downstream_http2_alpn(ssl); + conn_.set_ssl(ssl); } diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 0803433c..aecff732 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -824,9 +824,7 @@ Http2Upstream::Http2Upstream(ClientHandler *handler) downstream_queue_( get_config()->http2_proxy ? get_config()->conn.downstream.connections_per_host - : get_config()->conn.downstream.proto == PROTO_HTTP - ? get_config()->conn.downstream.connections_per_frontend - : 0, + : get_config()->conn.downstream.connections_per_frontend, !get_config()->http2_proxy), handler_(handler), session_(nullptr), diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index cecc9961..b68bb27e 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -118,7 +118,7 @@ HttpDownstreamConnection::HttpDownstreamConnection(DownstreamAddrGroup *group, get_config()->conn.downstream.timeout.write, get_config()->conn.downstream.timeout.read, {}, {}, connectcb, readcb, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold, - get_config()->tls.dyn_rec.idle_timeout), + get_config()->tls.dyn_rec.idle_timeout, PROTO_HTTP1), do_read_(&HttpDownstreamConnection::noop), do_write_(&HttpDownstreamConnection::noop), worker_(worker), @@ -153,6 +153,8 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { return -1; } + ssl::setup_downstream_http1_alpn(ssl); + conn_.set_ssl(ssl); } diff --git a/src/shrpx_memcached_connection.cc b/src/shrpx_memcached_connection.cc index 4df031d8..0b494ba8 100644 --- a/src/shrpx_memcached_connection.cc +++ b/src/shrpx_memcached_connection.cc @@ -96,7 +96,7 @@ MemcachedConnection::MemcachedConnection(const Address *addr, const StringRef &sni_name, MemchunkPool *mcpool) : conn_(loop, -1, nullptr, mcpool, write_timeout, read_timeout, {}, {}, - connectcb, readcb, timeoutcb, this, 0, 0.), + connectcb, readcb, timeoutcb, this, 0, 0., PROTO_MEMCACHED), do_read_(&MemcachedConnection::noop), do_write_(&MemcachedConnection::noop), sni_name_(sni_name.str()), diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 70d4096c..97e56f10 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -503,9 +503,7 @@ SpdyUpstream::SpdyUpstream(uint16_t version, ClientHandler *handler) downstream_queue_( get_config()->http2_proxy ? get_config()->conn.downstream.connections_per_host - : get_config()->conn.downstream.proto == PROTO_HTTP - ? get_config()->conn.downstream.connections_per_frontend - : 0, + : get_config()->conn.downstream.connections_per_frontend, !get_config()->http2_proxy), handler_(handler), session_(nullptr) { diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 2f9d200a..208a7342 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -659,12 +659,28 @@ int select_h1_next_proto_cb(SSL *ssl, unsigned char **out, } } // namespace +namespace { +int select_next_proto_cb(SSL *ssl, unsigned char **out, unsigned char *outlen, + const unsigned char *in, unsigned int inlen, + void *arg) { + auto conn = static_cast(SSL_get_app_data(ssl)); + switch (conn->proto) { + case PROTO_HTTP1: + return select_h1_next_proto_cb(ssl, out, outlen, in, inlen, arg); + case PROTO_HTTP2: + return select_h2_next_proto_cb(ssl, out, outlen, in, inlen, arg); + default: + return SSL_TLSEXT_ERR_NOACK; + } +} +} // namespace + SSL_CTX *create_ssl_client_context( #ifdef HAVE_NEVERBLEED neverbleed_t *nb, #endif // HAVE_NEVERBLEED const StringRef &cacert, const StringRef &cert_file, - const StringRef &private_key_file, const StringRef &alpn, + const StringRef &private_key_file, int (*next_proto_select_cb)(SSL *s, unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, void *arg)) { @@ -742,14 +758,10 @@ SSL_CTX *create_ssl_client_context( #endif // HAVE_NEVERBLEED } - // NPN selection callback + // NPN selection callback. This is required to set SSL_CTX because + // OpenSSL does not offer SSL_set_next_proto_select_cb. SSL_CTX_set_next_proto_select_cb(ssl_ctx, next_proto_select_cb, nullptr); -#if OPENSSL_VERSION_NUMBER >= 0x10002000L - // ALPN advertisement - SSL_CTX_set_alpn_protos(ssl_ctx, alpn.byte(), alpn.size()); -#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L - return ssl_ctx; } @@ -1303,29 +1315,29 @@ SSL_CTX *setup_downstream_client_ssl_context( } auto &tlsconf = get_config()->tls; - auto &downstreamconf = get_config()->conn.downstream; - - std::vector h2alpn; - StringRef alpn; - int (*next_proto_select_cb)(SSL *s, unsigned char **out, - unsigned char *outlen, const unsigned char *in, - unsigned int inlen, void *arg); - - if (downstreamconf.proto == PROTO_HTTP2) { - h2alpn = util::get_default_alpn(); - alpn = StringRef(h2alpn.data(), h2alpn.size()); - next_proto_select_cb = select_h2_next_proto_cb; - } else { - alpn = StringRef::from_lit(NGHTTP2_H1_1_ALPN); - next_proto_select_cb = select_h1_next_proto_cb; - } return ssl::create_ssl_client_context( #ifdef HAVE_NEVERBLEED nb, #endif // HAVE_NEVERBLEED StringRef{tlsconf.cacert}, StringRef{tlsconf.client.cert_file}, - StringRef{tlsconf.client.private_key_file}, alpn, next_proto_select_cb); + StringRef{tlsconf.client.private_key_file}, select_next_proto_cb); +} + +void setup_downstream_http2_alpn(SSL *ssl) { + auto alpn = util::get_default_alpn(); +#if OPENSSL_VERSION_NUMBER >= 0x10002000L + // ALPN advertisement + SSL_set_alpn_protos(ssl, alpn.data(), alpn.size()); +#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L +} + +void setup_downstream_http1_alpn(SSL *ssl) { + auto alpn = StringRef::from_lit(NGHTTP2_H1_1_ALPN); +#if OPENSSL_VERSION_NUMBER >= 0x10002000L + // ALPN advertisement + SSL_set_alpn_protos(ssl, alpn.byte(), alpn.size()); +#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L } CertLookupTree *create_cert_lookup_tree() { diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h index c913e5b9..900d9516 100644 --- a/src/shrpx_ssl.h +++ b/src/shrpx_ssl.h @@ -71,13 +71,14 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file #endif // HAVE_NEVERBLEED ); -// Create client side SSL_CTX +// Create client side SSL_CTX. This does not configure ALPN settings. +// |next_proto_select_cb| is for NPN. SSL_CTX *create_ssl_client_context( #ifdef HAVE_NEVERBLEED neverbleed_t *nb, #endif // HAVE_NEVERBLEED const StringRef &cacert, const StringRef &cert_file, - const StringRef &private_key_file, const StringRef &alpn, + const StringRef &private_key_file, int (*next_proto_select_cb)(SSL *s, unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, void *arg)); @@ -201,6 +202,11 @@ SSL_CTX *setup_downstream_client_ssl_context( #endif // HAVE_NEVERBLEED ); +// Sets ALPN settings in |SSL| suitable for HTTP/2 use. +void setup_downstream_http2_alpn(SSL *ssl); +// Sets ALPN settings in |SSL| suitable for HTTP/1.1 use. +void setup_downstream_http1_alpn(SSL *ssl); + // Creates CertLookupTree. If frontend is configured not to use TLS, // this function returns nullptr. CertLookupTree *create_cert_lookup_tree(); diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index d2115d25..65d547b6 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -104,6 +104,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, dst.pattern = src.pattern; dst.addrs.resize(src.addrs.size()); + dst.proto = src.proto; for (size_t j = 0; j < src.addrs.size(); ++j) { auto &src_addr = src.addrs[j]; diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 1f0c7d78..0602e988 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -86,6 +86,8 @@ struct DownstreamAddr { struct DownstreamAddrGroup { ImmutableString pattern; std::vector addrs; + // Application protocol used in this group + shrpx_proto proto; // List of Http2Session which is not fully utilized (i.e., the // server advertized maximum concurrency is not reached). We will // coalesce as much stream as possible in one Http2Session to fully From 44d38017606dace1de10917428119ad49ac67852 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 28 Feb 2016 17:11:12 +0900 Subject: [PATCH 082/147] nghttpx: Deprecate backend-http1-connections-per-frontend in favor of backend-connections-per-frontend --- gennghttpxfun.py | 3 ++- src/shrpx.cc | 20 ++++++++++++++------ src/shrpx_config.cc | 14 ++++++++++++++ src/shrpx_config.h | 2 ++ 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 8ff8cf83..6609a910 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -124,7 +124,8 @@ OPTIONS = [ "tls-ticket-key-memcached-address-family", "backend-address-family", "frontend-http2-max-concurrent-streams", - "backend-http2-max-concurrent-streams" + "backend-http2-max-concurrent-streams", + "backend-connections-per-frontend" ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index c52a3c99..76e6f3f0 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1362,12 +1362,13 @@ Performance: --backend-http1-connections-per-frontend. Default: )" << get_config()->conn.downstream.connections_per_host << R"( - --backend-http1-connections-per-frontend= - Set maximum number of backend concurrent HTTP/1 - connections per frontend. This option is only used for - default mode. 0 means unlimited. To limit the number - of connections per host for HTTP/2 or SPDY proxy mode - (-s option), use --backend-http1-connections-per-host. + --backend-connections-per-frontend= + Set maximum number of backend concurrent connections (or + streams in case of HTTP/2) per frontend. This option is + only used for default mode. 0 means unlimited. To + limit the number of connections per host for HTTP/2 or + SPDY proxy mode (-s option), use + --backend-http1-connections-per-host. Default: )" << get_config()->conn.downstream.connections_per_frontend << R"( --rlimit-nofile= @@ -2473,6 +2474,8 @@ int main(int argc, char **argv) { &flag, 117}, {SHRPX_OPT_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS, required_argument, &flag, 118}, + {SHRPX_OPT_BACKEND_CONNECTIONS_PER_FRONTEND, required_argument, &flag, + 119}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2978,6 +2981,11 @@ int main(int argc, char **argv) { cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS, optarg); break; + case 119: + // --backend-connections-per-frontend + cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_CONNECTIONS_PER_FRONTEND, + optarg); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 85c36e68..2c8a0958 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -654,6 +654,7 @@ enum { SHRPX_OPTID_ALTSVC, SHRPX_OPTID_BACKEND, SHRPX_OPTID_BACKEND_ADDRESS_FAMILY, + SHRPX_OPTID_BACKEND_CONNECTIONS_PER_FRONTEND, SHRPX_OPTID_BACKEND_HTTP_PROXY_URI, SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND, SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST, @@ -1344,6 +1345,15 @@ int option_lookup_token(const char *name, size_t namelen) { break; } break; + case 32: + switch (name[31]) { + case 'd': + if (util::strieq_l("backend-connections-per-fronten", name, 31)) { + return SHRPX_OPTID_BACKEND_CONNECTIONS_PER_FRONTEND; + } + break; + } + break; case 33: switch (name[32]) { case 'l': @@ -2045,6 +2055,10 @@ int parse_config(const char *opt, const char *optarg, return 0; } case SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND: + LOG(WARN) << opt << ": deprecated. Use " + << SHRPX_OPT_BACKEND_CONNECTIONS_PER_FRONTEND << " instead."; + // fall through + case SHRPX_OPTID_BACKEND_CONNECTIONS_PER_FRONTEND: return parse_uint(&mod_config()->conn.downstream.connections_per_frontend, opt, optarg); case SHRPX_OPTID_LISTENER_DISABLE_TIMEOUT: diff --git a/src/shrpx_config.h b/src/shrpx_config.h index b54951dd..611e40ab 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -232,6 +232,8 @@ constexpr char SHRPX_OPT_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS[] = "frontend-http2-max-concurrent-streams"; constexpr char SHRPX_OPT_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS[] = "backend-http2-max-concurrent-streams"; +constexpr char SHRPX_OPT_BACKEND_CONNECTIONS_PER_FRONTEND[] = + "backend-connections-per-frontend"; constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; From 06921f35f35d550e7dd7a94cc14c197f524bcaa4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 28 Feb 2016 21:35:26 +0900 Subject: [PATCH 083/147] nghttpx: Restructure mode settings It is very hard to support multiple protocols in backend while retaining multiple mode settings. Therefore, we dropped modes except for default and HTTP/2 proxy mode. The other removed modes can be emulated using combinations of options. Now the backend connection is not encrypted by default. To enable encryption on backend connection, use --backend-tls option. --- gennghttpxfun.py | 3 +- src/http2.cc | 4 +- src/http2.h | 6 +- src/shrpx.cc | 196 ++++++++++------------- src/shrpx_client_handler.cc | 4 +- src/shrpx_config.cc | 138 +++++++++++----- src/shrpx_config.h | 9 +- src/shrpx_http2_downstream_connection.cc | 14 +- src/shrpx_http2_session.cc | 3 +- src/shrpx_http2_upstream.cc | 14 +- src/shrpx_http_downstream_connection.cc | 15 +- src/shrpx_https_upstream.cc | 9 +- src/shrpx_mruby_module_request.cc | 2 +- src/shrpx_spdy_upstream.cc | 7 +- src/shrpx_worker.cc | 2 +- src/template.h | 2 +- src/util.cc | 21 +++ src/util.h | 10 ++ 18 files changed, 257 insertions(+), 202 deletions(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 6609a910..5ffb960d 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -125,7 +125,8 @@ OPTIONS = [ "backend-address-family", "frontend-http2-max-concurrent-streams", "backend-http2-max-concurrent-streams", - "backend-connections-per-frontend" + "backend-connections-per-frontend", + "backend-tls" ] LOGVARS = [ diff --git a/src/http2.cc b/src/http2.cc index f563596e..9acf1862 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -1416,9 +1416,9 @@ int lookup_method_token(const uint8_t *name, size_t namelen) { return -1; } -const char *to_method_string(int method_token) { +StringRef to_method_string(int method_token) { // we happened to use same value for method with http-parser. - return http_method_str(static_cast(method_token)); + return StringRef{http_method_str(static_cast(method_token))}; } int get_pure_path_component(const char **base, size_t *baselen, diff --git a/src/http2.h b/src/http2.h index 785089c6..bb879fdd 100644 --- a/src/http2.h +++ b/src/http2.h @@ -322,7 +322,11 @@ bool expect_response_body(int status_code); int lookup_method_token(const uint8_t *name, size_t namelen); int lookup_method_token(const std::string &name); -const char *to_method_string(int method_token); +// Returns string representation of |method_token|. This is wrapper +// function over http_method_str from http-parser. If |method_token| +// is not known to http-parser, "" is returned. The returned +// StringRef is guaranteed to be NULL-terminated. +StringRef to_method_string(int method_token); template std::string normalize_path(InputIt first, InputIt last) { diff --git a/src/shrpx.cc b/src/shrpx.cc index 76e6f3f0..7a9616f7 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -656,7 +656,7 @@ int create_tcp_server_socket(UpstreamAddr &faddr, } faddr.fd = fd; - faddr.hostport = util::make_http_hostport(host.data(), faddr.port); + faddr.hostport = util::make_http_hostport(StringRef{host.data()}, faddr.port); LOG(NOTICE) << "Listening on " << faddr.hostport; @@ -1079,7 +1079,8 @@ void fill_default_config() { tlsconf.session_timeout = std::chrono::hours(12); auto &httpconf = mod_config()->http; - httpconf.server_name = "nghttpx nghttp2/" NGHTTP2_VERSION; + httpconf.server_name = + StringRef::from_lit("nghttpx nghttp2/" NGHTTP2_VERSION); httpconf.no_host_rewrite = true; httpconf.request_header_field_buffer = 64_k; httpconf.max_request_header_fields = 100; @@ -1165,6 +1166,7 @@ void fill_default_config() { downstreamconf.request_buffer_size = 16_k; downstreamconf.response_buffer_size = 128_k; downstreamconf.family = AF_UNSPEC; + downstreamconf.no_tls = true; } } @@ -1188,48 +1190,49 @@ void print_help(std::ostream &out) { print_usage(out); out << R"( - Set path to server's private key. Required unless -p, - --client or --frontend-no-tls are given. - Set path to server's certificate. Required unless -p, - --client or --frontend-no-tls are given. To make OCSP - stapling work, this must be absolute path. + Set path to server's private key. Required unless + --frontend-no-tls are given. + Set path to server's certificate. Required unless + --frontend-no-tls are given. To make OCSP stapling + work, this must be an absolute path. Options: The options are categorized into several groups. Connections: - -b, --backend=(,|unix:)[;[:...]] + -b, --backend=(,|unix:)[;[[:...]][;proto=]] Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX domain socket can be specified by prefixing path name with "unix:" (e.g., unix:/var/run/backend.sock). Optionally, if s are given, the backend address - is only used if request matches the pattern. If -s or - -p is used, s are ignored. The pattern - matching is closely designed to ServeMux in net/http - package of Go programming language. consists - of path, host + path or just host. The path must start - with "/". If it ends with "/", it matches all request - path in its subtree. To deal with the request to the - directory without trailing slash, the path which ends - with "/" also matches the request path which only lacks - trailing '/' (e.g., path "/foo/" matches request path - "/foo"). If it does not end with "/", it performs exact - match against the request path. If host is given, it - performs exact match against the request host. If host - alone is given, "/" is appended to it, so that it - matches all request paths under the host (e.g., - specifying "nghttp2.org" equals to "nghttp2.org/"). + is only used if request matches the pattern. If + --http2-proxy is used, s are ignored. The + pattern matching is closely designed to ServeMux in + net/http package of Go programming language. + consists of path, host + path or just host. The path + must start with "/". If it ends with "/", it matches + all request path in its subtree. To deal with the + request to the directory without trailing slash, the + path which ends with "/" also matches the request path + which only lacks trailing '/' (e.g., path "/foo/" + matches request path "/foo"). If it does not end with + "/", it performs exact match against the request path. + If host is given, it performs exact match against the + request host. If host alone is given, "/" is appended + to it, so that it matches all request paths under the + host (e.g., specifying "nghttp2.org" equals to + "nghttp2.org/"). Patterns with host take precedence over patterns with just path. Then, longer patterns take precedence over shorter ones, breaking a tie by the order of the appearance in the configuration. - If is omitted, "/" is used as pattern, which - matches all request paths (catch-all pattern). The - catch-all backend must be given. + If is omitted or empty string, "/" is used as + pattern, which matches all request paths (catch-all + pattern). The catch-all backend must be given. When doing a match, nghttpx made some normalization to pattern, request host and path. For host part, they are @@ -1252,6 +1255,15 @@ Connections: The backend addresses sharing same are grouped together forming load balancing group. + Optionally, backend application protocol can be + specified in . All that share the same + must have the same value if it is given. + should be one of the following list without + quotes: "h2", "http/1.1". The default value of + is "http/1.1". Note that usually "h2" refers to HTTP/2 + over TLS. But in this option, it may mean HTTP/2 over + cleartext TCP unless --backend-tls is used. + Since ";" and ":" are used as delimiter, must not contain these characters. Since ";" has special meaning in shell, the option value must be quoted. @@ -1290,16 +1302,8 @@ Connections: --backend-write-timeout options. --accept-proxy-protocol Accept PROXY protocol version 1 on frontend connection. - --backend-no-tls - Disable SSL/TLS on backend connections. For HTTP/2 - backend connections, TLS is enabled by default. For - HTTP/1 backend connections, TLS is disabled by default, - and can be enabled by --backend-http1-tls option. If - both --backend-no-tls and --backend-http1-tls options - are used, --backend-no-tls has the precedence. - --backend-http1-tls - Enable SSL/TLS on backend HTTP/1 connections. See also - --backend-no-tls option. + --backend-tls + Enable SSL/TLS on backend connections. Performance: -n, --workers= @@ -1355,19 +1359,19 @@ Performance: --backend-http1-connections-per-host= Set maximum number of backend concurrent HTTP/1 connections per origin host. This option is meaningful - when -s option is used. The origin host is determined - by authority portion of request URI (or :authority - header field for HTTP/2). To limit the number of - connections per frontend for default mode, use - --backend-http1-connections-per-frontend. + when --http2-proxy option is used. The origin host is + determined by authority portion of request URI (or + :authority header field for HTTP/2). To limit the + number of connections per frontend for default mode, use + --backend-connections-per-frontend. Default: )" << get_config()->conn.downstream.connections_per_host << R"( --backend-connections-per-frontend= Set maximum number of backend concurrent connections (or streams in case of HTTP/2) per frontend. This option is only used for default mode. 0 means unlimited. To - limit the number of connections per host for HTTP/2 or - SPDY proxy mode (-s option), use + limit the number of connections per host with + --http2-proxy option, use --backend-http1-connections-per-host. Default: )" << get_config()->conn.downstream.connections_per_frontend << R"( @@ -1667,37 +1671,20 @@ HTTP/2 and SPDY: Disable HTTP/2 server push. Server push is supported by default mode and HTTP/2 frontend via Link header field. It is also supported if both frontend and backend are - HTTP/2 (which implies --http2-bridge or --client mode). - In this case, server push from backend session is - relayed to frontend, and server push via Link header - field is also supported. HTTP SPDY frontend does not - support server push. + HTTP/2. In this case, server push from backend session + is relayed to frontend, and server push via Link header + field is also supported. SPDY frontend does not support + server push. Mode: (default mode) Accept HTTP/2, SPDY and HTTP/1.1 over SSL/TLS. If --frontend-no-tls is used, accept HTTP/2 and HTTP/1.1. The incoming HTTP/1.1 connection can be upgraded to - HTTP/2 through HTTP Upgrade. The protocol to the - backend is HTTP/1.1. + HTTP/2 through HTTP Upgrade. -s, --http2-proxy - Like default mode, but enable secure proxy mode. - --http2-bridge - Like default mode, but communicate with the backend in - HTTP/2 over SSL/TLS. Thus the incoming all connections - are converted to HTTP/2 connection and relayed to the - backend. See --backend-http-proxy-uri option if you are - behind the proxy and want to connect to the outside - HTTP/2 proxy. - --client Accept HTTP/2 and HTTP/1.1 without SSL/TLS. The - incoming HTTP/1.1 connection can be upgraded to HTTP/2 - connection through HTTP Upgrade. The protocol to the - backend is HTTP/2. To use nghttpx as a forward proxy, - use -p option instead. - -p, --client-proxy - Like --client option, but it also requires the request - path from frontend must be an absolute URI, suitable for - use as a forward proxy. + Like default mode, but enable forward proxy. This is so + called HTTP/2 proxy mode. Logging: -L, --log-level= @@ -1798,15 +1785,13 @@ HTTP: --no-via Don't append to Via header field. If Via header field is received, it is left unaltered. --no-location-rewrite - Don't rewrite location header field on --http2-bridge, - --client and default mode. For --http2-proxy and - --client-proxy mode, location header field will not be - altered regardless of this option. + Don't rewrite location header field in default mode. + When --http2-proxy is used, location header field will + not be altered regardless of this option. --host-rewrite - Rewrite host and :authority header fields on - --http2-bridge, --client and default mode. For - --http2-proxy and --client-proxy mode, these headers - will not be altered regardless of this option. + Rewrite host and :authority header fields in default + mode. When --http2-proxy is used, these headers will + not be altered regardless of this option. --altsvc= Specify protocol ID, port, host and origin of alternative service. and are optional. @@ -2055,31 +2040,6 @@ void process_options( upstreamconf.worker_connections = std::numeric_limits::max(); } - if (get_config()->http2_proxy + get_config()->http2_bridge + - get_config()->client_proxy + get_config()->client > - 1) { - LOG(FATAL) << "--http2-proxy, --http2-bridge, --client-proxy and --client " - << "cannot be used at the same time."; - exit(EXIT_FAILURE); - } - - if (get_config()->client || get_config()->client_proxy) { - mod_config()->client_mode = true; - upstreamconf.no_tls = true; - } - - shrpx_proto default_proto; - if (get_config()->client_mode || get_config()->http2_bridge) { - default_proto = PROTO_HTTP2; - } else { - default_proto = PROTO_HTTP1; - } - - if (!get_config()->client_mode && !get_config()->http2_bridge && - !downstreamconf.http1_tls) { - downstreamconf.no_tls = true; - } - if (!upstreamconf.no_tls && (tlsconf.private_key_file.empty() || tlsconf.cert_file.empty())) { print_usage(std::cerr); @@ -2105,31 +2065,34 @@ void process_options( addr.port = DEFAULT_DOWNSTREAM_PORT; DownstreamAddrGroupConfig g(StringRef::from_lit("/")); - g.proto = default_proto; + g.proto = PROTO_HTTP1; g.addrs.push_back(std::move(addr)); mod_config()->router.add_route(StringRef{g.pattern}, addr_groups.size()); addr_groups.push_back(std::move(g)); - } else if (get_config()->http2_proxy || get_config()->client_proxy) { + } else if (get_config()->http2_proxy) { // We don't support host mapping in these cases. Move all // non-catch-all patterns to catch-all pattern. DownstreamAddrGroupConfig catch_all(StringRef::from_lit("/")); + auto proto = PROTO_NONE; for (auto &g : addr_groups) { + if (proto == PROTO_NONE) { + proto = g.proto; + } else if (proto != g.proto) { + LOG(ERROR) << SHRPX_OPT_BACKEND << ": was ignored with " + "--http2-proxy, and protocol must " + "be the same for all backends."; + exit(EXIT_FAILURE); + } std::move(std::begin(g.addrs), std::end(g.addrs), std::back_inserter(catch_all.addrs)); } - catch_all.proto = default_proto; + catch_all.proto = proto; std::vector().swap(addr_groups); // maybe not necessary? mod_config()->router = Router(); mod_config()->router.add_route(StringRef{catch_all.pattern}, addr_groups.size()); addr_groups.push_back(std::move(catch_all)); - } else { - for (auto &g : addr_groups) { - if (g.proto == PROTO_NONE) { - g.proto = default_proto; - } - } } if (LOG_ENABLED(INFO)) { @@ -2144,7 +2107,7 @@ void process_options( } if (LOG_ENABLED(INFO)) { LOG(INFO) << "Host-path pattern: group " << i << ": '" << g.pattern - << "'"; + << "', proto=" << strproto(g.proto); for (auto &addr : g.addrs) { LOG(INFO) << "group " << i << " -> " << addr.host.c_str() << (addr.host_unix ? "" : ":" + util::utos(addr.port)); @@ -2153,7 +2116,7 @@ void process_options( } if (catch_all_group == -1) { - LOG(FATAL) << "-b: No catch-all backend address is configured"; + LOG(FATAL) << "backend: No catch-all backend address is configured"; exit(EXIT_FAILURE); } @@ -2197,7 +2160,7 @@ void process_options( addr.hostport = ImmutableString( util::make_http_hostport(StringRef(addr.host), addr.port)); - auto hostport = util::make_hostport(addr.host.c_str(), addr.port); + auto hostport = util::make_hostport(StringRef{addr.host}, addr.port); if (resolve_hostname(&addr.addr, addr.host.c_str(), addr.port, downstreamconf.family) == -1) { @@ -2211,7 +2174,7 @@ void process_options( auto &proxy = mod_config()->downstream_http_proxy; if (!proxy.host.empty()) { - auto hostport = util::make_hostport(proxy.host.c_str(), proxy.port); + auto hostport = util::make_hostport(StringRef{proxy.host}, proxy.port); if (resolve_hostname(&proxy.addr, proxy.host.c_str(), proxy.port, AF_UNSPEC) == -1) { LOG(FATAL) << "Resolving backend HTTP proxy address failed: " << hostport; @@ -2476,6 +2439,7 @@ int main(int argc, char **argv) { &flag, 118}, {SHRPX_OPT_BACKEND_CONNECTIONS_PER_FRONTEND, required_argument, &flag, 119}, + {SHRPX_OPT_BACKEND_TLS, no_argument, &flag, 120}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2986,6 +2950,10 @@ int main(int argc, char **argv) { cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_CONNECTIONS_PER_FRONTEND, optarg); break; + case 120: + // --backend-tls + cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_TLS, "yes"); + break; default: break; } diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 9dbd6453..7151dbf2 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -853,11 +853,11 @@ void ClientHandler::write_accesslog(Downstream *downstream) { upstream_accesslog( get_config()->logging.access.format, LogSpec{ - downstream, StringRef(ipaddr_), http2::to_method_string(req.method), + downstream, StringRef{ipaddr_}, http2::to_method_string(req.method), req.method == HTTP_CONNECT ? StringRef(req.authority) - : (get_config()->http2_proxy || get_config()->client_proxy) + : get_config()->http2_proxy ? StringRef(construct_absolute_request_uri(req)) : req.path.empty() ? req.method == HTTP_OPTIONS diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 2c8a0958..8a1d2103 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -571,33 +571,70 @@ int parse_duration(ev_tstamp *dest, const char *opt, const char *optarg) { } // namespace namespace { -// Parses host-path mapping patterns in |src|, and stores mappings in -// config. We will store each host-path pattern found in |src| with -// |addr|. |addr| will be copied accordingly. Also we make a group -// based on the pattern. The "/" pattern is considered as catch-all. -void parse_mapping(const DownstreamAddrConfig &addr, const char *src) { +// Parses host-path mapping patterns in |src_pattern|, and stores +// mappings in config. We will store each host-path pattern found in +// |src| with |addr|. |addr| will be copied accordingly. Also we +// make a group based on the pattern. The "/" pattern is considered +// as catch-all. We also parse protocol specified in |src_proto|. +// +// This function returns 0 if it succeeds, or -1. +int parse_mapping(const DownstreamAddrConfig &addr, + const StringRef &src_pattern, const StringRef &src_proto) { // This returns at least 1 element (it could be empty string). We // will append '/' to all patterns, so it becomes catch-all pattern. - auto mapping = util::split_config_str_list(src, ':'); + auto mapping = util::split_str(src_pattern, ':'); assert(!mapping.empty()); auto &addr_groups = mod_config()->conn.downstream.addr_groups; + auto proto = PROTO_HTTP1; + + if (!src_proto.empty()) { + if (!util::istarts_with_l(src_proto, "proto=")) { + LOG(ERROR) << "backend: proto keyword not found"; + return -1; + } + + auto protostr = StringRef{std::begin(src_proto) + str_size("proto="), + std::end(src_proto)}; + if (protostr.empty()) { + LOG(ERROR) << "backend: protocol is empty"; + return -1; + } + + if (util::streq_l("h2", std::begin(protostr), protostr.size())) { + proto = PROTO_HTTP2; + } else if (util::streq_l("http/1.1", std::begin(protostr), + protostr.size())) { + proto = PROTO_HTTP1; + } else { + LOG(ERROR) << "backend: unknown protocol " << protostr; + return -1; + } + } + for (const auto &raw_pattern : mapping) { auto done = false; std::string pattern; - auto slash = std::find(raw_pattern.first, raw_pattern.second, '/'); - if (slash == raw_pattern.second) { + auto slash = std::find(std::begin(raw_pattern), std::end(raw_pattern), '/'); + if (slash == std::end(raw_pattern)) { // This effectively makes empty pattern to "/". - pattern.assign(raw_pattern.first, raw_pattern.second); + pattern.assign(std::begin(raw_pattern), std::end(raw_pattern)); util::inp_strlower(pattern); pattern += '/'; } else { - pattern.assign(raw_pattern.first, slash); + pattern.assign(std::begin(raw_pattern), slash); util::inp_strlower(pattern); - pattern += http2::normalize_path(slash, raw_pattern.second); + pattern += http2::normalize_path(slash, std::end(raw_pattern)); } for (auto &g : addr_groups) { if (g.pattern == pattern) { + if (g.proto != proto) { + LOG(ERROR) << "backend: protocol mismatch. We saw protocol " + << strproto(g.proto) << " for pattern " << g.pattern + << ", but another protocol " << strproto(proto); + return -1; + } + g.addrs.push_back(addr); done = true; break; @@ -608,11 +645,13 @@ void parse_mapping(const DownstreamAddrConfig &addr, const char *src) { } DownstreamAddrGroupConfig g(StringRef{pattern}); g.addrs.push_back(addr); + g.proto = proto; mod_config()->router.add_route(StringRef{g.pattern}, addr_groups.size()); addr_groups.push_back(std::move(g)); } + return 0; } } // namespace @@ -670,6 +709,7 @@ enum { SHRPX_OPTID_BACKEND_READ_TIMEOUT, SHRPX_OPTID_BACKEND_REQUEST_BUFFER, SHRPX_OPTID_BACKEND_RESPONSE_BUFFER, + SHRPX_OPTID_BACKEND_TLS, SHRPX_OPTID_BACKEND_TLS_SNI_FIELD, SHRPX_OPTID_BACKEND_WRITE_TIMEOUT, SHRPX_OPTID_BACKLOG, @@ -914,6 +954,11 @@ int option_lookup_token(const char *name, size_t namelen) { break; case 11: switch (name[10]) { + case 's': + if (util::strieq_l("backend-tl", name, 10)) { + return SHRPX_OPTID_BACKEND_TLS; + } + break; case 't': if (util::strieq_l("write-burs", name, 10)) { return SHRPX_OPTID_WRITE_BURST; @@ -1495,19 +1540,17 @@ int parse_config(const char *opt, const char *optarg, switch (optid) { case SHRPX_OPTID_BACKEND: { - auto optarglen = strlen(optarg); - const char *pat_delim = strchr(optarg, ';'); - if (!pat_delim) { - pat_delim = optarg + optarglen; - } + auto src = StringRef{optarg}; + auto addr_end = std::find(std::begin(src), std::end(src), ';'); + DownstreamAddrConfig addr{}; if (util::istarts_with(optarg, SHRPX_UNIX_PATH_PREFIX)) { - auto path = optarg + str_size(SHRPX_UNIX_PATH_PREFIX); - addr.host = ImmutableString(path, pat_delim); + auto path = std::begin(src) + str_size(SHRPX_UNIX_PATH_PREFIX); + addr.host = ImmutableString(path, addr_end); addr.host_unix = true; } else { - if (split_host_port(host, sizeof(host), &port, optarg, - pat_delim - optarg) == -1) { + if (split_host_port(host, sizeof(host), &port, &src[0], + addr_end - std::begin(src)) == -1) { return -1; } @@ -1515,14 +1558,16 @@ int parse_config(const char *opt, const char *optarg, addr.port = port; } - auto mapping = pat_delim < optarg + optarglen ? pat_delim + 1 : pat_delim; - // We may introduce new parameter after additional ';', so don't - // allow extra ';' in pattern for now. - if (strchr(mapping, ';') != nullptr) { - LOG(ERROR) << opt << ": ';' must not be used in pattern"; + auto mapping = addr_end == std::end(src) ? addr_end : addr_end + 1; + auto mapping_end = std::find(mapping, std::end(src), ';'); + + auto proto = mapping_end == std::end(src) ? mapping_end : mapping_end + 1; + auto proto_end = std::find(proto, std::end(src), ';'); + + if (parse_mapping(addr, StringRef{mapping, mapping_end}, + StringRef{proto, proto_end}) != 0) { return -1; } - parse_mapping(addr, mapping); return 0; } @@ -1607,13 +1652,13 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_HTTP2_BRIDGE: - mod_config()->http2_bridge = util::strieq(optarg, "yes"); - - return 0; + LOG(ERROR) << opt << ": deprecated. Use backend=,;;proto=h2 " + "and backend-tls"; + return -1; case SHRPX_OPTID_CLIENT_PROXY: - mod_config()->client_proxy = util::strieq(optarg, "yes"); - - return 0; + LOG(ERROR) << opt << ": deprecated. Use http2-proxy, frontend-no-tls, " + "backend=,;;proto=h2 and backend-tls"; + return -1; case SHRPX_OPTID_ADD_X_FORWARDED_FOR: mod_config()->http.xff.add = util::strieq(optarg, "yes"); @@ -1746,8 +1791,8 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_BACKEND_NO_TLS: - mod_config()->conn.downstream.no_tls = util::strieq(optarg, "yes"); - + LOG(WARN) << opt << ": deprecated. backend connection is not encrypted by " + "default. See also " << SHRPX_OPT_BACKEND_TLS; return 0; case SHRPX_OPTID_BACKEND_TLS_SNI_FIELD: mod_config()->tls.backend_sni_name = optarg; @@ -1834,9 +1879,9 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_CLIENT: - mod_config()->client = util::strieq(optarg, "yes"); - - return 0; + LOG(ERROR) << opt << ": deprecated. Use frontend-no-tls, " + "backend=,;;proto=h2 and backend-tls"; + return -1; case SHRPX_OPTID_INSECURE: mod_config()->tls.insecure = util::strieq(optarg, "yes"); @@ -2312,7 +2357,11 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_BACKEND_HTTP1_TLS: - mod_config()->conn.downstream.http1_tls = util::strieq(optarg, "yes"); + LOG(WARN) << opt << ": deprecated. Use " << SHRPX_OPT_BACKEND_TLS + << " instead."; + // fall through + case SHRPX_OPTID_BACKEND_TLS: + mod_config()->conn.downstream.no_tls = !util::strieq(optarg, "yes"); return 0; case SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_TLS: @@ -2533,4 +2582,17 @@ int int_syslog_facility(const char *strfacility) { return -1; } +StringRef strproto(shrpx_proto proto) { + switch (proto) { + case PROTO_NONE: + return StringRef::from_lit("none"); + case PROTO_HTTP1: + return StringRef::from_lit("http/1.1"); + case PROTO_HTTP2: + return StringRef::from_lit("h2"); + case PROTO_MEMCACHED: + return StringRef::from_lit("memcached"); + } +} + } // namespace shrpx diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 611e40ab..874dabe1 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -234,6 +234,7 @@ constexpr char SHRPX_OPT_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS[] = "backend-http2-max-concurrent-streams"; constexpr char SHRPX_OPT_BACKEND_CONNECTIONS_PER_FRONTEND[] = "backend-connections-per-frontend"; +constexpr char SHRPX_OPT_BACKEND_TLS[] = "backend-tls"; constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -599,11 +600,6 @@ struct Config { bool verbose; bool daemon; bool http2_proxy; - bool http2_bridge; - bool client_proxy; - bool client; - // true if --client or --client-proxy are enabled. - bool client_mode; }; const Config *get_config(); @@ -650,6 +646,9 @@ std::unique_ptr read_tls_ticket_key_file(const std::vector &files, const EVP_CIPHER *cipher, const EVP_MD *hmac); +// Returns string representation of |proto|. +StringRef strproto(shrpx_proto proto); + } // namespace shrpx #endif // SHRPX_CONFIG_H diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index aba79df4..6aa97d1f 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -264,9 +264,9 @@ int Http2DownstreamConnection::push_request_headers() { auto &httpconf = get_config()->http; auto &http2conf = get_config()->http2; - auto no_host_rewrite = - httpconf.no_host_rewrite || get_config()->http2_proxy || - get_config()->client_proxy || req.method == HTTP_CONNECT; + auto no_host_rewrite = httpconf.no_host_rewrite || + get_config()->http2_proxy || + req.method == HTTP_CONNECT; // http2session_ has already in CONNECTED state, so we can get // addr_idx here. @@ -302,7 +302,7 @@ int Http2DownstreamConnection::push_request_headers() { httpconf.add_request_headers.size()); nva.push_back( - http2::make_nv_lc_nocopy(":method", http2::to_method_string(req.method))); + http2::make_nv_ls_nocopy(":method", http2::to_method_string(req.method))); if (req.method != HTTP_CONNECT) { assert(!req.scheme.empty()); @@ -350,8 +350,7 @@ int Http2DownstreamConnection::push_request_headers() { if (fwdconf.params) { auto params = fwdconf.params; - if (get_config()->http2_proxy || get_config()->client_proxy || - req.method == HTTP_CONNECT) { + if (get_config()->http2_proxy || req.method == HTTP_CONNECT) { params &= ~FORWARDED_PROTO; } @@ -394,8 +393,7 @@ int Http2DownstreamConnection::push_request_headers() { nva.push_back(http2::make_nv_ls_nocopy("x-forwarded-for", (*xff).value)); } - if (!get_config()->http2_proxy && !get_config()->client_proxy && - req.method != HTTP_CONNECT) { + if (!get_config()->http2_proxy && req.method != HTTP_CONNECT) { // We use same protocol with :scheme header field nva.push_back(http2::make_nv_ls_nocopy("x-forwarded-proto", req.scheme)); } diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index dd8cd5e4..9a411ca1 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -1460,8 +1460,7 @@ int Http2Session::connection_made() { entry[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE; entry[1].value = (1 << http2conf.downstream.window_bits) - 1; - if (http2conf.no_server_push || get_config()->http2_proxy || - get_config()->client_proxy) { + if (http2conf.no_server_push || get_config()->http2_proxy) { entry[nentry].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH; entry[nentry].value = 0; ++nentry; diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index aecff732..5bf1c5c4 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -316,7 +316,7 @@ int Http2Upstream::on_request_headers(Downstream *downstream, if (path) { if (method_token == HTTP_OPTIONS && path->value == "*") { // Server-wide OPTIONS request. Path is empty. - } else if (get_config()->http2_proxy || get_config()->client_proxy) { + } else if (get_config()->http2_proxy) { req.path = http2::value_to_str(path); } else { const auto &value = path->value; @@ -1346,8 +1346,7 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { auto &httpconf = get_config()->http; - if (!get_config()->http2_proxy && !get_config()->client_proxy && - !httpconf.no_location_rewrite) { + if (!get_config()->http2_proxy && !httpconf.no_location_rewrite) { downstream->rewrite_location_response_header(req.scheme); } @@ -1416,7 +1415,7 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { http2::copy_headers_to_nva_nocopy(nva, resp.fs.headers()); - if (!get_config()->http2_proxy && !get_config()->client_proxy) { + if (!get_config()->http2_proxy) { nva.push_back(http2::make_nv_ls_nocopy("server", httpconf.server_name)); } else { auto server = resp.fs.header(http2::HD_SERVER); @@ -1489,9 +1488,8 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { if (!http2conf.no_server_push && nghttp2_session_get_remote_settings(session_, NGHTTP2_SETTINGS_ENABLE_PUSH) == 1 && - !get_config()->http2_proxy && !get_config()->client_proxy && - (downstream->get_stream_id() % 2) && resp.fs.header(http2::HD_LINK) && - resp.http_status == 200 && + !get_config()->http2_proxy && (downstream->get_stream_id() % 2) && + resp.fs.header(http2::HD_LINK) && resp.http_status == 200 && (req.method == HTTP_GET || req.method == HTTP_POST)) { if (prepare_push_promise(downstream) != 0) { @@ -1852,7 +1850,7 @@ bool Http2Upstream::push_enabled() const { return !(get_config()->http2.no_server_push || nghttp2_session_get_remote_settings( session_, NGHTTP2_SETTINGS_ENABLE_PUSH) == 0 || - get_config()->http2_proxy || get_config()->client_proxy); + get_config()->http2_proxy); } int Http2Upstream::initiate_push(Downstream *downstream, const char *uri, diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index b68bb27e..fcd9b8e6 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -279,9 +279,8 @@ int HttpDownstreamConnection::push_request_headers() { // For HTTP/1.0 request, there is no authority in request. In that // case, we use backend server's host nonetheless. auto authority = StringRef(downstream_hostport); - auto no_host_rewrite = httpconf.no_host_rewrite || - get_config()->http2_proxy || - get_config()->client_proxy || connect_method; + auto no_host_rewrite = + httpconf.no_host_rewrite || get_config()->http2_proxy || connect_method; if (no_host_rewrite && !req.authority.empty()) { authority = StringRef(req.authority); @@ -293,12 +292,12 @@ int HttpDownstreamConnection::push_request_headers() { // Assume that method and request path do not contain \r\n. auto meth = http2::to_method_string(req.method); - buf->append(meth, strlen(meth)); + buf->append(meth); buf->append(" "); if (connect_method) { buf->append(authority); - } else if (get_config()->http2_proxy || get_config()->client_proxy) { + } else if (get_config()->http2_proxy) { // Construct absolute-form request target because we are going to // send a request to a HTTP/1 proxy. assert(!req.scheme.empty()); @@ -363,8 +362,7 @@ int HttpDownstreamConnection::push_request_headers() { if (fwdconf.params) { auto params = fwdconf.params; - if (get_config()->http2_proxy || get_config()->client_proxy || - connect_method) { + if (get_config()->http2_proxy || connect_method) { params &= ~FORWARDED_PROTO; } @@ -407,8 +405,7 @@ int HttpDownstreamConnection::push_request_headers() { buf->append((*xff).value); buf->append("\r\n"); } - if (!get_config()->http2_proxy && !get_config()->client_proxy && - !connect_method) { + if (!get_config()->http2_proxy && !connect_method) { buf->append("X-Forwarded-Proto: "); assert(!req.scheme.empty()); buf->append(req.scheme); diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 9eaa5202..e089428d 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -232,7 +232,7 @@ void rewrite_request_host_path_from_uri(Request &req, const char *uri, path += '?'; path.append(uri + fdata.off, fdata.len); } - if (get_config()->http2_proxy || get_config()->client_proxy) { + if (get_config()->http2_proxy) { req.path = std::move(path); } else { req.path = http2::rewrite_clean_path(std::begin(path), std::end(path)); @@ -306,7 +306,7 @@ int htp_hdrs_completecb(http_parser *htp) { } // checking UF_HOST could be redundant, but just in case ... if (!(u.field_set & (1 << UF_SCHEMA)) || !(u.field_set & (1 << UF_HOST))) { - if (get_config()->http2_proxy || get_config()->client_proxy) { + if (get_config()->http2_proxy) { // Request URI should be absolute-form for client proxy mode return -1; } @@ -929,8 +929,7 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) { auto &httpconf = get_config()->http; - if (!get_config()->http2_proxy && !get_config()->client_proxy && - !httpconf.no_location_rewrite) { + if (!get_config()->http2_proxy && !httpconf.no_location_rewrite) { downstream->rewrite_location_response_header( get_client_handler()->get_upstream_scheme()); } @@ -999,7 +998,7 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) { } } - if (!get_config()->http2_proxy && !get_config()->client_proxy) { + if (!get_config()->http2_proxy) { buf->append("Server: "); buf->append(httpconf.server_name); buf->append("\r\n"); diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index f585d730..89bbb660 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -70,7 +70,7 @@ mrb_value request_get_method(mrb_state *mrb, mrb_value self) { const auto &req = downstream->request(); auto method = http2::to_method_string(req.method); - return mrb_str_new_cstr(mrb, method); + return mrb_str_new(mrb, method.c_str(), method.size()); } } // namespace diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 97e56f10..b7f07724 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -262,7 +262,7 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, } else { req.scheme = scheme->value; req.authority = host->value; - if (get_config()->http2_proxy || get_config()->client_proxy) { + if (get_config()->http2_proxy) { req.path = path->value; } else if (method_token == HTTP_OPTIONS && path->value == "*") { // Server-wide OPTIONS request. Path is empty. @@ -1009,8 +1009,7 @@ int SpdyUpstream::on_downstream_header_complete(Downstream *downstream) { auto &httpconf = get_config()->http; - if (!get_config()->http2_proxy && !get_config()->client_proxy && - !httpconf.no_location_rewrite) { + if (!get_config()->http2_proxy && !httpconf.no_location_rewrite) { downstream->rewrite_location_response_header(req.scheme); } @@ -1044,7 +1043,7 @@ int SpdyUpstream::on_downstream_header_complete(Downstream *downstream) { nv[hdidx++] = hd.value.c_str(); } - if (!get_config()->http2_proxy && !get_config()->client_proxy) { + if (!get_config()->http2_proxy) { nv[hdidx++] = "server"; nv[hdidx++] = httpconf.server_name.c_str(); } else { diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 65d547b6..7c8c6bb3 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -318,7 +318,7 @@ size_t match_downstream_addr_group_host( return group; } - group = router.match("", path); + group = router.match(StringRef::from_lit(""), path); if (group != -1) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Found pattern with query " << path diff --git a/src/template.h b/src/template.h index 7c9dc6f1..6adcf0d5 100644 --- a/src/template.h +++ b/src/template.h @@ -400,7 +400,7 @@ public: explicit StringRef(const std::string &s) : base(s.c_str()), len(s.size()) {} explicit StringRef(const ImmutableString &s) : base(s.c_str()), len(s.size()) {} - StringRef(const char *s) : base(s), len(strlen(s)) {} + explicit StringRef(const char *s) : base(s), len(strlen(s)) {} template constexpr StringRef(const CharT *s, size_t n) : base(reinterpret_cast(s)), len(n) {} diff --git a/src/util.cc b/src/util.cc index 12b90594..7805a957 100644 --- a/src/util.cc +++ b/src/util.cc @@ -857,6 +857,27 @@ std::vector get_default_alpn() { return res; } +std::vector split_str(const StringRef &s, char delim) { + size_t len = 1; + auto last = std::end(s); + for (auto first = std::begin(s), d = first; + (d = std::find(first, last, delim)) != last; ++len, first = d + 1) + ; + + auto list = std::vector(len); + + len = 0; + for (auto first = std::begin(s);; ++len) { + auto stop = std::find(first, last, delim); + list[len] = StringRef{first, stop}; + if (stop == last) { + break; + } + first = stop + 1; + } + return list; +} + std::vector> split_config_str_list(const char *s, char delim) { size_t len = 1; diff --git a/src/util.h b/src/util.h index f4b1c2a0..15894816 100644 --- a/src/util.h +++ b/src/util.h @@ -225,6 +225,11 @@ bool istarts_with_l(const std::string &a, const CharT(&b)[N]) { return istarts_with(std::begin(a), std::end(a), b, b + N - 1); } +template +bool istarts_with_l(const StringRef &a, const CharT(&b)[N]) { + return istarts_with(std::begin(a), std::end(a), b, b + N - 1); +} + template bool ends_with(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2) { @@ -543,6 +548,11 @@ std::vector parse_config_str_list(const char *s, char delim = ','); std::vector> split_config_str_list(const char *s, char delim); +// Parses delimited strings in |s| and returns Substrings in |s| +// delimited by |delim|. The any white spaces around substring are +// treated as a part of substring. +std::vector split_str(const StringRef &s, char delim); + // Returns given time |tp| in Common Log format (e.g., // 03/Jul/2014:00:19:38 +0900) // Expected type of |tp| is std::chrono::timepoint From 2326337d323cac84d658d543ab1f99cdd4d71877 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 28 Feb 2016 22:15:49 +0900 Subject: [PATCH 084/147] nghttpx: Deprecate backend-http1-connections-per-host in favor of backend-connections-per-host --- gennghttpxfun.py | 3 ++- src/shrpx.cc | 32 +++++++++++++++++++------------- src/shrpx_config.cc | 12 +++++++++++- src/shrpx_config.h | 2 ++ 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 5ffb960d..9df537f1 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -126,7 +126,8 @@ OPTIONS = [ "frontend-http2-max-concurrent-streams", "backend-http2-max-concurrent-streams", "backend-connections-per-frontend", - "backend-tls" + "backend-tls", + "backend-connections-per-host" ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index 7a9616f7..333dbc13 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1356,23 +1356,24 @@ Performance: accepts. Setting 0 means unlimited. Default: )" << get_config()->conn.upstream.worker_connections << R"( - --backend-http1-connections-per-host= - Set maximum number of backend concurrent HTTP/1 - connections per origin host. This option is meaningful - when --http2-proxy option is used. The origin host is - determined by authority portion of request URI (or - :authority header field for HTTP/2). To limit the - number of connections per frontend for default mode, use + --backend-connections-per-host= + Set maximum number of backend concurrent connections + (and/or streams in case of HTTP/2) per origin host. + This option is meaningful when --http2-proxy option is + used. The origin host is determined by authority + portion of request URI (or :authority header field for + HTTP/2). To limit the number of connections per + frontend for default mode, use --backend-connections-per-frontend. Default: )" << get_config()->conn.downstream.connections_per_host << R"( --backend-connections-per-frontend= - Set maximum number of backend concurrent connections (or - streams in case of HTTP/2) per frontend. This option is - only used for default mode. 0 means unlimited. To - limit the number of connections per host with - --http2-proxy option, use - --backend-http1-connections-per-host. + Set maximum number of backend concurrent connections + (and/or streams in case of HTTP/2) per frontend. This + option is only used for default mode. 0 means + unlimited. To limit the number of connections per host + with --http2-proxy option, use + --backend-connections-per-host. Default: )" << get_config()->conn.downstream.connections_per_frontend << R"( --rlimit-nofile= @@ -2440,6 +2441,7 @@ int main(int argc, char **argv) { {SHRPX_OPT_BACKEND_CONNECTIONS_PER_FRONTEND, required_argument, &flag, 119}, {SHRPX_OPT_BACKEND_TLS, no_argument, &flag, 120}, + {SHRPX_OPT_BACKEND_CONNECTIONS_PER_HOST, required_argument, &flag, 121}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2954,6 +2956,10 @@ int main(int argc, char **argv) { // --backend-tls cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_TLS, "yes"); break; + case 121: + // --backend-connections-per-host + cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_CONNECTIONS_PER_HOST, optarg); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 8a1d2103..5f2e51cb 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -694,6 +694,7 @@ enum { SHRPX_OPTID_BACKEND, SHRPX_OPTID_BACKEND_ADDRESS_FAMILY, SHRPX_OPTID_BACKEND_CONNECTIONS_PER_FRONTEND, + SHRPX_OPTID_BACKEND_CONNECTIONS_PER_HOST, SHRPX_OPTID_BACKEND_HTTP_PROXY_URI, SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND, SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST, @@ -1370,6 +1371,11 @@ int option_lookup_token(const char *name, size_t namelen) { return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_TLS; } break; + case 't': + if (util::strieq_l("backend-connections-per-hos", name, 27)) { + return SHRPX_OPTID_BACKEND_CONNECTIONS_PER_HOST; + } + break; } break; case 30: @@ -2082,7 +2088,11 @@ int parse_config(const char *opt, const char *optarg, "--host-rewrite option."; return 0; - case SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST: { + case SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST: + LOG(WARN) << opt + << ": deprecated. Use backend-connections-per-host instead."; + // fall through + case SHRPX_OPTID_BACKEND_CONNECTIONS_PER_HOST: { int n; if (parse_uint(&n, opt, optarg) != 0) { diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 874dabe1..72594fa8 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -235,6 +235,8 @@ constexpr char SHRPX_OPT_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS[] = constexpr char SHRPX_OPT_BACKEND_CONNECTIONS_PER_FRONTEND[] = "backend-connections-per-frontend"; constexpr char SHRPX_OPT_BACKEND_TLS[] = "backend-tls"; +constexpr char SHRPX_OPT_BACKEND_CONNECTIONS_PER_HOST[] = + "backend-connections-per-host"; constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; From df6466cfbda2586fc4e1c24103908fcc8fac0789 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 28 Feb 2016 23:12:57 +0900 Subject: [PATCH 085/147] nghttpx: Update doc --- README.rst | 84 ++++--------- doc/nghttpx.h2r | 10 +- doc/sources/nghttpx-howto.rst | 223 ++++++++++++---------------------- src/shrpx.cc | 8 +- 4 files changed, 112 insertions(+), 213 deletions(-) diff --git a/README.rst b/README.rst index 290c7dd4..11a4087b 100644 --- a/README.rst +++ b/README.rst @@ -655,46 +655,38 @@ HTTP/2. ``nghttpx`` also offers the functionality to share session cache and ticket keys among multiple ``nghttpx`` instances via memcached. -``nghttpx`` has several operational modes: +``nghttpx`` has 2 operation modes: -================== ============================ ============== ============= -Mode option Frontend Backend Note -================== ============================ ============== ============= -default mode HTTP/2, SPDY, HTTP/1.1 (TLS) HTTP/1.1 Reverse proxy -``--http2-proxy`` HTTP/2, SPDY, HTTP/1.1 (TLS) HTTP/1.1 SPDY proxy -``--http2-bridge`` HTTP/2, SPDY, HTTP/1.1 (TLS) HTTP/2 (TLS) -``--client`` HTTP/2, HTTP/1.1 HTTP/2 (TLS) -``--client-proxy`` HTTP/2, HTTP/1.1 HTTP/2 (TLS) Forward proxy -================== ============================ ============== ============= +================== ====================== =================== ============= +Mode option Frontend Backend Note +================== ====================== =================== ============= +default mode HTTP/2, SPDY, HTTP/1.1 HTTP/1.1, HTTP/2 Reverse proxy +``--http2-proxy`` HTTP/2, SPDY, HTTP/1.1 HTTP/1.1, or HTTP/2 Forward proxy +================== ====================== =================== ============= The interesting mode at the moment is the default mode. It works like a reverse proxy and listens for HTTP/2, SPDY and HTTP/1.1 and can be deployed as a SSL/TLS terminator for existing web server. -The default mode, ``--http2-proxy`` and ``--http2-bridge`` modes use -SSL/TLS in the frontend connection by default. To disable SSL/TLS, -use the ``--frontend-no-tls`` option. If that option is used, SPDY is -disabled in the frontend and incoming HTTP/1.1 connections can be -upgraded to HTTP/2 through HTTP Upgrade. In these modes, HTTP/1 -backend connections are cleartext by default. To enable TLS, use -``--backend-http1-tls`` opiton. - -The ``--http2-bridge``, ``--client`` and ``--client-proxy`` modes use -SSL/TLS in the backend connection by default. To disable SSL/TLS, use -the ``--backend-no-tls`` option. +In all modes, the frontend connections are encrypted by SSL/TLS by +default. To disable encryption, use the ``--frontend-no-tls`` option. +If encryption is disabled, SPDY is disabled in the frontend and +incoming HTTP/1.1 connections can be upgraded to HTTP/2 through HTTP +Upgrade. On the other hard, backend connections are not encrypted by +default. To encrypt backend connections, use ``--backend-tls`` +option. ``nghttpx`` supports a configuration file. See the ``--conf`` option and sample configuration file ``nghttpx.conf.sample``. -In the default mode, (without any of ``--http2-proxy``, -``--http2-bridge``, ``--client-proxy`` and ``--client`` options), -``nghttpx`` works as reverse proxy to the backend server:: +In the default mode, ``nghttpx`` works as reverse proxy to the backend +server:: - Client <-- (HTTP/2, SPDY, HTTP/1.1) --> nghttpx <-- (HTTP/1.1) --> Web Server + Client <-- (HTTP/2, SPDY, HTTP/1.1) --> nghttpx <-- (HTTP/1.1, HTTP/2) --> Web Server [reverse proxy] -With the ``--http2-proxy`` option, it works as a so called secure proxy (aka -SPDY proxy):: +With the ``--http2-proxy`` option, it works as forward proxy, and it +is so called secure HTTP/2 proxy (aka SPDY proxy):: Client <-- (HTTP/2, SPDY, HTTP/1.1) --> nghttpx <-- (HTTP/1.1) --> Proxy [secure proxy] (e.g., Squid, ATS) @@ -702,9 +694,9 @@ SPDY proxy):: The ``Client`` in the above example needs to be configured to use ``nghttpx`` as secure proxy. -At the time of this writing, Chrome is the only browser which supports -secure proxy. One way to configure Chrome to use a secure proxy is -to create a proxy.pac script like this: +At the time of this writing, both Chrome and Firefox support secure +HTTP/2 proxy. One way to configure Chrome to use a secure proxy is to +create a proxy.pac script like this: .. code-block:: javascript @@ -720,37 +712,9 @@ Then run Chrome with the following arguments:: $ google-chrome --proxy-pac-url=file:///path/to/proxy.pac --use-npn -With ``--http2-bridge``, it accepts HTTP/2, SPDY and HTTP/1.1 -connections and communicates with the backend in HTTP/2:: - - Client <-- (HTTP/2, SPDY, HTTP/1.1) --> nghttpx <-- (HTTP/2) --> Web or HTTP/2 Proxy etc - (e.g., nghttpx -s) - -With ``--client-proxy``, it works as a forward proxy and expects -that the backend is an HTTP/2 proxy:: - - Client <-- (HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/2) --> HTTP/2 Proxy - [forward proxy] (e.g., nghttpx -s) - -The ``Client`` needs to be configured to use nghttpx as a forward -proxy. The frontend HTTP/1.1 connection can be upgraded to HTTP/2 -through HTTP Upgrade. With the above configuration, one can use -HTTP/1.1 client to access and test their HTTP/2 servers. - -With ``--client``, it works as a reverse proxy and expects that -the backend is an HTTP/2 Web server:: - - Client <-- (HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/2) --> Web Server - [reverse proxy] - -The frontend HTTP/1.1 connection can be upgraded to HTTP/2 -through HTTP Upgrade. - -For the operation modes which talk to the backend in HTTP/2 over -SSL/TLS, the backend connections can be tunneled through an HTTP proxy. +The backend HTTP/2 connections can be tunneled through an HTTP proxy. The proxy is specified using ``--backend-http-proxy-uri``. The -following figure illustrates the example of the ``--http2-bridge`` and -``--backend-http-proxy-uri`` options to talk to the outside HTTP/2 +following figure illustrates how nghttpx talks to the outside HTTP/2 proxy through an HTTP proxy:: Client <-- (HTTP/2, SPDY, HTTP/1.1) --> nghttpx <-- (HTTP/2) -- diff --git a/doc/nghttpx.h2r b/doc/nghttpx.h2r index 14512bb0..a37812cb 100644 --- a/doc/nghttpx.h2r +++ b/doc/nghttpx.h2r @@ -98,12 +98,12 @@ Currently, the following restriction is applied for server push: This limitation may be loosened in the future release. nghttpx also supports server push if both frontend and backend are -HTTP/2 (which implies :option:`--http2-bridge` or :option:`--client`). -In this case, in addition to server push via Link header field, server -push from backend is relayed to frontend HTTP/2 session. +HTTP/2 in default mode. In this case, in addition to server push via +Link header field, server push from backend is forwarded to frontend +HTTP/2 session. -HTTP/2 server push will be disabled if :option:`--http2-proxy` or -:option:`--client-proxy` is used. +HTTP/2 server push will be disabled if :option:`--http2-proxy` is +used. UNIX DOMAIN SOCKET ------------------ diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index e7dec217..b9bbaa25 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -12,37 +12,38 @@ use-cases. It also covers some useful options later. Default mode ------------ -If nghttpx is invoked without any :option:`--http2-proxy`, -:option:`--client`, and :option:`--client-proxy`, it operates in -default mode. In this mode, nghttpx frontend listens for HTTP/2 -requests and translates them to HTTP/1 requests. Thus it works as -reverse proxy (gateway) for HTTP/2 clients to HTTP/1 web server. This -is also known as "HTTP/2 router". HTTP/1 requests are also supported -in frontend as a fallback. If nghttpx is linked with spdylay library -and frontend connection is SSL/TLS, the frontend also supports SPDY +If nghttpx is invoked without :option:`--http2-proxy`, it operates in +default mode. In this mode, it works as reverse proxy (gateway) for +both HTTP/2 and HTTP/1 clients to backend servers. This is also known +as "HTTP/2 router". If nghttpx is linked with spdylay library and +frontend connection is SSL/TLS, the frontend also supports SPDY protocol. -By default, this mode's frontend connection is encrypted using -SSL/TLS. So server's private key and certificate must be supplied to -the command line (or through configuration file). In this case, the -frontend protocol selection will be done via ALPN or NPN. +By default, frontend connection is encrypted using SSL/TLS. So +server's private key and certificate must be supplied to the command +line (or through configuration file). In this case, the frontend +protocol selection will be done via ALPN or NPN. With :option:`--frontend-no-tls` option, user can turn off SSL/TLS in frontend connection. In this case, SPDY protocol is not available even if spdylay library is liked to nghttpx. HTTP/2 and HTTP/1 are -available on the frontend and a HTTP/1 connection can be upgraded to +available on the frontend, and an HTTP/1 connection can be upgraded to HTTP/2 using HTTP Upgrade. Starting HTTP/2 connection by sending HTTP/2 connection preface is also supported. -By default, backend HTTP/1 connections are not encrypted. To enable -TLS on HTTP/1 backend connections, use :option:`--backend-http1-tls` -option. This applies to all mode whose backend connections are -HTTP/1. +By default, backend connections are not encrypted. To enable TLS +encryption on backend connections, use :option:`--backend-tls` option. +Using patterns and ``proto`` keyword in :option:`--backend` option, +backend application protocol can be specified per host/request path +pattern. It means that you can use both HTTP/2 and HTTP/1 in backend +connections at the same time. Note that default backend protocol is +HTTP/1.1. To use HTTP/2 in backend, you have to specify ``h2`` in +``proto`` keyword in :option:`--backend` explicitly. -The backend is supposed to be HTTP/1 Web server. For example, to make +The backend is supposed to be Web server. For example, to make nghttpx listen to encrypted HTTP/2 requests at port 8443, and a -backend HTTP/1 web server is configured to listen to HTTP/1 request at -port 8080 in the same host, run nghttpx command-line like this:: +backend Web server is configured to listen to HTTP request at port +8080 in the same host, run nghttpx command-line like this:: $ nghttpx -f0.0.0.0,8443 -b127.0.0.1,8080 /path/to/server.key /path/to/server.crt @@ -58,8 +59,8 @@ If nghttpx is invoked with :option:`--http2-proxy` (or its shorthand :option:`-s`) option, it operates in HTTP/2 proxy mode. The supported protocols in frontend and backend connections are the same in `default mode`_. The difference is that this mode acts like forward proxy and -assumes the backend is HTTP/1 proxy server (e.g., squid, traffic -server). So HTTP/1 request must include absolute URI in request line. +assumes the backend is HTTP proxy server (e.g., Squid, Apache Traffic +Server). HTTP/1 request must include absolute URI in request line. By default, frontend connection is encrypted. So this mode is also called secure proxy. If nghttpx is linked with spdylay, it supports @@ -68,16 +69,22 @@ SPDY protocols and it works as so called SPDY proxy. With :option:`--frontend-no-tls` option, SSL/TLS is turned off in frontend connection, so the connection gets insecure. -The backend must be HTTP/1 proxy server. nghttpx supports multiple -backend server addresses. It translates incoming requests to HTTP/1 +The backend must be HTTP proxy server. nghttpx supports multiple +backend server addresses. It translates incoming requests to HTTP request to backend server. The backend server performs real proxy work for each request, for example, dispatching requests to the origin server and caching contents. +The backend connection is not encrypted by default. To enable +encryption, use :option:`--backend-tls` option. The default backend +protocol is HTTP/1.1. To use HTTP/2 in backend connection, use +:option:`--backend` option, and specify ``h2`` in ``proto`` keyword +explicitly. + For example, to make nghttpx listen to encrypted HTTP/2 requests at -port 8443, and a backend HTTP/1 proxy server is configured to listen -to HTTP/1 request at port 8080 in the same host, run nghttpx -command-line like this:: +port 8443, and a backend HTTP proxy server is configured to listen to +HTTP/1 request at port 8080 in the same host, run nghttpx command-line +like this:: $ nghttpx -s -f'*,8443' -b127.0.0.1,8080 /path/to/server.key /path/to/server.crt @@ -122,126 +129,19 @@ Consult Traffic server `documentation to know how to configure traffic server as forward proxy and its security implications. -Client mode ------------ +Disable frontend SSL/TLS +------------------------ -If nghttpx is invoked with :option:`--client` option, it operates in -client mode. In this mode, nghttpx listens for plain, unencrypted -HTTP/2 and HTTP/1 requests and translates them to encrypted HTTP/2 -requests to the backend. User cannot enable SSL/TLS in frontend -connection. +The frontend connections are encrypted with SSL/TLS by default. To +turn off SSL/TLS, use :option:`--frontend-no-tls` option. If this +option is used, the private key and certificate are not required to +run nghttpx. -HTTP/1 frontend connection can be upgraded to HTTP/2 using HTTP -Upgrade. To disable SSL/TLS in backend connection, use -:option:`--backend-no-tls` option. +Enable backend SSL/TLS +---------------------- -A single HTTP/2 backend connection is shared among multiple frontend -connections. - -The backend server is supporsed to be a HTTP/2 web server (e.g., -nghttpd). The one use-case of this mode is utilize existing HTTP/1 -clients to test HTTP/2 deployment. Suppose that HTTP/2 web server -listens to port 80 without encryption. Then run nghttpx as client -mode to access to that web server:: - - $ nghttpx --client -f127.0.0.1,8080 -b127.0.0.1,80 --backend-no-tls - -.. note:: - - You may need :option:`--insecure` (or its shorthand :option:`-k`) - option if HTTP/2 server enables SSL/TLS and its certificate is - self-signed. But please note that it is insecure, and you should - know what you are doing. - -Then you can use curl to access HTTP/2 server via nghttpx:: - - $ curl http://localhost:8080/ - -Client proxy mode ------------------ - -If nghttpx is invoked with :option:`--client-proxy` (or its shorthand -:option:`-p`) option, it operates in client proxy mode. This mode -behaves like `client mode`_, but it works like forward proxy. So -HTTP/1 request must include absolute URI in request line. - -HTTP/1 frontend connection can be upgraded to HTTP/2 using HTTP -Upgrade. To disable SSL/TLS in backend connection, use -:option:`--backend-no-tls` option. - -A single HTTP/2 backend connection is shared among multiple frontend -connections. - -The backend server must be a HTTP/2 proxy. You can use nghttpx in -`HTTP/2 proxy mode`_ as backend server. The one use-case of this mode -is utilize existing HTTP/1 clients to test HTTP/2 connections between -2 proxies. The another use-case is use this mode to aggregate local -HTTP/1 connections to one HTTP/2 backend encrypted connection. This -makes HTTP/1 clients which does not support secure proxy can use -secure HTTP/2 proxy via nghttpx client mode. - -Suppose that HTTP/2 proxy listens to port 8443, just like we saw in -`HTTP/2 proxy mode`_. To run nghttpx in client proxy mode to access -that server, invoke nghttpx like this:: - - $ nghttpx -p -f127.0.0.1,8080 -b127.0.0.1,8443 - -.. note:: - - You may need :option:`--insecure` (or its shorthand :option:`-k`) - option if HTTP/2 server's certificate is self-signed. But please - note that it is insecure, and you should know what you are doing. - -Then you can use curl to issue HTTP request via HTTP/2 proxy:: - - $ curl --http-proxy=http://localhost:8080 http://www.google.com/ - -You can configure web browser to use localhost:8080 as forward -proxy. - -HTTP/2 bridge mode ------------------- - -If nghttpx is invoked with :option:`--http2-bridge` option, it -operates in HTTP/2 bridge mode. The supported protocols in frontend -connections are the same in `default mode`_. The protocol in backend -is HTTP/2 only. - -With :option:`--frontend-no-tls` option, SSL/TLS is turned off in -frontend connection, so the connection gets insecure. To disable -SSL/TLS in backend connection, use :option:`--backend-no-tls` option. - -A single HTTP/2 backend connection is shared among multiple frontend -connections. - -The backend server is supporsed to be a HTTP/2 web server or HTTP/2 -proxy. If backend server is HTTP/2 proxy, use -:option:`--no-location-rewrite` option to disable rewriting -``Location`` header field. - -The use-case of this mode is aggregate the incoming connections to one -HTTP/2 connection. One backend HTTP/2 connection is created per -worker (thread). - -Disable SSL/TLS ---------------- - -In `default mode`_, `HTTP/2 proxy mode`_ and `HTTP/2 bridge mode`_, -frontend connections are encrypted with SSL/TLS by default. To turn -off SSL/TLS, use :option:`--frontend-no-tls` option. If this option -is used, the private key and certificate are not required to run -nghttpx. - -In `client mode`_, `client proxy mode`_ and `HTTP/2 bridge mode`_, -backend connections are encrypted with SSL/TLS by default. To turn -off SSL/TLS, use :option:`--backend-no-tls` option. - -Enable SSL/TLS on HTTP/1 backend --------------------------------- - -In all modes which use HTTP/1 as backend protocol, backend HTTP/1 -connection is not encrypted by default. To enable encryption, use -:option:`--backend-http1-tls` option. +The backend connections are not encrypted by default. To enable +SSL/TLS encryption, :option:`--backend-tls` option. Enable SSL/TLS on memcached connection -------------------------------------- @@ -380,3 +280,38 @@ servers ``serv1:3000`` and ``serv2:3000`` for request host backend=serv1,3000;example.com/myservice backend=serv2,3000;example.com/myservice + +You can also specify backend application protocol in +:option:`--backend` option using ``proto`` keyword after pattern. +Utilizing this allows ngttpx to route certain request to HTTP/2, other +requests to HTTP/1. For example, to route requests to ``/ws/`` in +backend HTTP/1.1 connection, and use backend HTTP/2 for other +requests, do this: + +.. code-block:: text + + backend=serv1,3000;/;proto=h2 + backend=serv1,3000;/ws/;proto=http/1.1 + +Note that the backends share the same pattern must have the same +backend protocol. The default backend protocol is HTTP/1.1. + +Deprecated modes +---------------- + +As of nghttpx 1.9.0, ``--http2-bridge``, ``--client`` and +``--client-proxy`` options were removed. These functionality can be +used using combinations of options. + +* ``--http2-bridge``: Use + :option:`--backend`\='-b,;;proto=h2', and + :option:`--backend-tls`. + +* ``--client``: Use :option:`--frontend-no-tls`, + :option:`--backend`\='-b,;;proto=h2', and + :option:`--backend-tls`. + +* ``--client-proxy``: Use :option:`--http2-proxy`, + :option:`--frontend-no-tls`, + :option:`--backend`\='-b,;;proto=h2', and + :option:`--backend-tls`. diff --git a/src/shrpx.cc b/src/shrpx.cc index 333dbc13..3beb91be 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1672,10 +1672,10 @@ HTTP/2 and SPDY: Disable HTTP/2 server push. Server push is supported by default mode and HTTP/2 frontend via Link header field. It is also supported if both frontend and backend are - HTTP/2. In this case, server push from backend session - is relayed to frontend, and server push via Link header - field is also supported. SPDY frontend does not support - server push. + HTTP/2 in default mode. In this case, server push from + backend session is relayed to frontend, and server push + via Link header field is also supported. SPDY frontend + does not support server push. Mode: (default mode) From 5fbe4cc2257d95dfa51c5dd8969b3f5201e0f9db Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 28 Feb 2016 23:21:57 +0900 Subject: [PATCH 086/147] nghttpx: Clear upgrade_request flag when Downstream is attached to HTTP/2 backend --- src/shrpx_http2_downstream_connection.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index 6aa97d1f..b5accaa2 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -105,6 +105,11 @@ int Http2DownstreamConnection::attach_downstream(Downstream *downstream) { downstream_ = downstream; downstream_->reset_downstream_rtimer(); + auto &req = downstream_->request(); + + // HTTP/2 disables HTTP Upgrade. + req.upgrade_request = false; + return 0; } From 124d4c9fad736dbb16acb594b2988c21ba7f668b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 28 Feb 2016 23:47:23 +0900 Subject: [PATCH 087/147] src: Fix tests --- src/Makefile.am | 1 + src/shrpx-unittest.cc | 5 +- src/shrpx_config_test.cc | 116 ----------------------- src/shrpx_downstream_test.cc | 4 +- src/shrpx_http_test.cc | 25 ++--- src/shrpx_worker_test.cc | 179 +++++++++++++++++++++++++++++++++++ src/shrpx_worker_test.h | 38 ++++++++ src/util_test.cc | 15 ++- 8 files changed, 247 insertions(+), 136 deletions(-) create mode 100644 src/shrpx_worker_test.cc create mode 100644 src/shrpx_worker_test.h diff --git a/src/Makefile.am b/src/Makefile.am index 2c873893..cd75fdf4 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -169,6 +169,7 @@ nghttpx_unittest_SOURCES = shrpx-unittest.cc \ shrpx_ssl_test.cc shrpx_ssl_test.h \ shrpx_downstream_test.cc shrpx_downstream_test.h \ shrpx_config_test.cc shrpx_config_test.h \ + shrpx_worker_test.cc shrpx_worker_test.h \ shrpx_http_test.cc shrpx_http_test.h \ http2_test.cc http2_test.h \ util_test.cc util_test.h \ diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index 8eba2c65..bebc0f28 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -33,6 +33,7 @@ #include "shrpx_ssl_test.h" #include "shrpx_downstream_test.h" #include "shrpx_config_test.h" +#include "shrpx_worker_test.h" #include "http2_test.h" #include "util_test.h" #include "nghttp2_gzip_test.h" @@ -118,8 +119,8 @@ int main(int argc, char *argv[]) { shrpx::test_shrpx_config_read_tls_ticket_key_file) || !CU_add_test(pSuite, "config_read_tls_ticket_key_file_aes_256", shrpx::test_shrpx_config_read_tls_ticket_key_file_aes_256) || - !CU_add_test(pSuite, "config_match_downstream_addr_group", - shrpx::test_shrpx_config_match_downstream_addr_group) || + !CU_add_test(pSuite, "worker_match_downstream_addr_group", + shrpx::test_shrpx_worker_match_downstream_addr_group) || !CU_add_test(pSuite, "http_create_forwarded", shrpx::test_shrpx_http_create_forwarded) || !CU_add_test(pSuite, "util_streq", shrpx::test_util_streq) || diff --git a/src/shrpx_config_test.cc b/src/shrpx_config_test.cc index 53b1bc30..635c23d9 100644 --- a/src/shrpx_config_test.cc +++ b/src/shrpx_config_test.cc @@ -238,120 +238,4 @@ void test_shrpx_config_read_tls_ticket_key_file_aes_256(void) { "a..............................b")); } -void test_shrpx_config_match_downstream_addr_group(void) { - auto groups = std::vector{ - {"nghttp2.org/"}, - {"nghttp2.org/alpha/bravo/"}, - {"nghttp2.org/alpha/charlie"}, - {"nghttp2.org/delta%3A"}, - {"www.nghttp2.org/"}, - {"[::1]/"}, - {"nghttp2.org/alpha/bravo/delta"}, - // Check that match is done in the single node - {"example.com/alpha/bravo"}, - {"192.168.0.1/alpha/"}, - }; - - Router router; - - for (size_t i = 0; i < groups.size(); ++i) { - auto &g = groups[i]; - router.add_route(StringRef{g.pattern}, i); - } - - CU_ASSERT(0 == match_downstream_addr_group(router, "nghttp2.org", "/", groups, - 255)); - - // port is removed - CU_ASSERT(0 == match_downstream_addr_group(router, "nghttp2.org:8080", "/", - groups, 255)); - - // host is case-insensitive - CU_ASSERT(4 == match_downstream_addr_group(router, "WWW.nghttp2.org", - "/alpha", groups, 255)); - - CU_ASSERT(1 == match_downstream_addr_group(router, "nghttp2.org", - "/alpha/bravo/", groups, 255)); - - // /alpha/bravo also matches /alpha/bravo/ - CU_ASSERT(1 == match_downstream_addr_group(router, "nghttp2.org", - "/alpha/bravo", groups, 255)); - - // path part is case-sensitive - CU_ASSERT(0 == match_downstream_addr_group(router, "nghttp2.org", - "/Alpha/bravo", groups, 255)); - - CU_ASSERT(1 == match_downstream_addr_group(router, "nghttp2.org", - "/alpha/bravo/charlie", groups, - 255)); - - CU_ASSERT(2 == match_downstream_addr_group(router, "nghttp2.org", - "/alpha/charlie", groups, 255)); - - // pattern which does not end with '/' must match its entirely. So - // this matches to group 0, not group 2. - CU_ASSERT(0 == match_downstream_addr_group(router, "nghttp2.org", - "/alpha/charlie/", groups, 255)); - - CU_ASSERT(255 == match_downstream_addr_group(router, "example.org", "/", - groups, 255)); - - CU_ASSERT(255 == match_downstream_addr_group(router, "", "/", groups, 255)); - - CU_ASSERT(255 == - match_downstream_addr_group(router, "", "alpha", groups, 255)); - - CU_ASSERT(255 == - match_downstream_addr_group(router, "foo/bar", "/", groups, 255)); - - // If path is "*", only match with host + "/". - CU_ASSERT(0 == match_downstream_addr_group(router, "nghttp2.org", "*", groups, - 255)); - - CU_ASSERT(5 == - match_downstream_addr_group(router, "[::1]", "/", groups, 255)); - CU_ASSERT( - 5 == match_downstream_addr_group(router, "[::1]:8080", "/", groups, 255)); - CU_ASSERT(255 == - match_downstream_addr_group(router, "[::1", "/", groups, 255)); - CU_ASSERT(255 == - match_downstream_addr_group(router, "[::1]8000", "/", groups, 255)); - - // Check the case where adding route extends tree - CU_ASSERT(6 == match_downstream_addr_group( - router, "nghttp2.org", "/alpha/bravo/delta", groups, 255)); - - CU_ASSERT(1 == match_downstream_addr_group(router, "nghttp2.org", - "/alpha/bravo/delta/", groups, - 255)); - - // Check the case where query is done in a single node - CU_ASSERT(7 == match_downstream_addr_group(router, "example.com", - "/alpha/bravo", groups, 255)); - - CU_ASSERT(255 == match_downstream_addr_group(router, "example.com", - "/alpha/bravo/", groups, 255)); - - CU_ASSERT(255 == match_downstream_addr_group(router, "example.com", "/alpha", - groups, 255)); - - // Check the case where quey is done in a single node - CU_ASSERT(8 == match_downstream_addr_group(router, "192.168.0.1", "/alpha", - groups, 255)); - - CU_ASSERT(8 == match_downstream_addr_group(router, "192.168.0.1", "/alpha/", - groups, 255)); - - CU_ASSERT(8 == match_downstream_addr_group(router, "192.168.0.1", - "/alpha/bravo", groups, 255)); - - CU_ASSERT(255 == match_downstream_addr_group(router, "192.168.0.1", "/alph", - groups, 255)); - - CU_ASSERT(255 == match_downstream_addr_group(router, "192.168.0.1", "/", - groups, 255)); - - router.dump(); -} - } // namespace shrpx diff --git a/src/shrpx_downstream_test.cc b/src/shrpx_downstream_test.cc index 5b496f9b..d9989a72 100644 --- a/src/shrpx_downstream_test.cc +++ b/src/shrpx_downstream_test.cc @@ -77,8 +77,8 @@ void test_downstream_field_store_header(void) { CU_ASSERT(nullptr == fs.header(http2::HD__METHOD)); // By name - CU_ASSERT(Header("alpha", "0") == *fs.header("alpha")); - CU_ASSERT(nullptr == fs.header("bravo")); + CU_ASSERT(Header("alpha", "0") == *fs.header(StringRef::from_lit("alpha"))); + CU_ASSERT(nullptr == fs.header(StringRef::from_lit("bravo"))); } void test_downstream_crumble_request_cookie(void) { diff --git a/src/shrpx_http_test.cc b/src/shrpx_http_test.cc index 721b53f5..03510550 100644 --- a/src/shrpx_http_test.cc +++ b/src/shrpx_http_test.cc @@ -40,25 +40,28 @@ namespace shrpx { void test_shrpx_http_create_forwarded(void) { CU_ASSERT("by=\"example.com:3000\";for=\"[::1]\";host=\"www.example.com\";" "proto=https" == - http::create_forwarded( - FORWARDED_BY | FORWARDED_FOR | FORWARDED_HOST | FORWARDED_PROTO, - "example.com:3000", "[::1]", "www.example.com", "https")); + http::create_forwarded(FORWARDED_BY | FORWARDED_FOR | + FORWARDED_HOST | FORWARDED_PROTO, + StringRef::from_lit("example.com:3000"), + "[::1]", "www.example.com", "https")); - CU_ASSERT("for=192.168.0.1" == http::create_forwarded(FORWARDED_FOR, "alpha", - "192.168.0.1", "bravo", - "charlie")); + CU_ASSERT("for=192.168.0.1" == + http::create_forwarded(FORWARDED_FOR, StringRef::from_lit("alpha"), + "192.168.0.1", "bravo", "charlie")); CU_ASSERT("by=_hidden;for=\"[::1]\"" == - http::create_forwarded(FORWARDED_BY | FORWARDED_FOR, "_hidden", - "[::1]", "", "")); + http::create_forwarded(FORWARDED_BY | FORWARDED_FOR, + StringRef::from_lit("_hidden"), "[::1]", "", + "")); CU_ASSERT("by=\"[::1]\";for=_hidden" == - http::create_forwarded(FORWARDED_BY | FORWARDED_FOR, "[::1]", - "_hidden", "", "")); + http::create_forwarded(FORWARDED_BY | FORWARDED_FOR, + StringRef::from_lit("[::1]"), "_hidden", "", + "")); CU_ASSERT("" == http::create_forwarded(FORWARDED_BY | FORWARDED_FOR | FORWARDED_HOST | FORWARDED_PROTO, - "", "", "", "")); + StringRef::from_lit(""), "", "", "")); } } // namespace shrpx diff --git a/src/shrpx_worker_test.cc b/src/shrpx_worker_test.cc new file mode 100644 index 00000000..11a15e01 --- /dev/null +++ b/src/shrpx_worker_test.cc @@ -0,0 +1,179 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "shrpx_worker_test.h" + +#ifdef HAVE_UNISTD_H +#include +#endif // HAVE_UNISTD_H + +#include + +#include + +#include "shrpx_worker.h" +#include "shrpx_connect_blocker.h" + +namespace shrpx { + +void test_shrpx_worker_match_downstream_addr_group(void) { + auto groups = std::vector(); + for (auto &s : {"nghttp2.org/", "nghttp2.org/alpha/bravo/", + "nghttp2.org/alpha/charlie", "nghttp2.org/delta%3A", + "www.nghttp2.org/", "[::1]/", "nghttp2.org/alpha/bravo/delta", + // Check that match is done in the single node + "example.com/alpha/bravo", "192.168.0.1/alpha/"}) { + groups.push_back(DownstreamAddrGroup{ImmutableString(s)}); + } + + Router router; + + for (size_t i = 0; i < groups.size(); ++i) { + auto &g = groups[i]; + router.add_route(StringRef{g.pattern}, i); + } + + CU_ASSERT(0 == match_downstream_addr_group( + router, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/"), groups, 255)); + + // port is removed + CU_ASSERT(0 == match_downstream_addr_group( + router, StringRef::from_lit("nghttp2.org:8080"), + StringRef::from_lit("/"), groups, 255)); + + // host is case-insensitive + CU_ASSERT(4 == match_downstream_addr_group( + router, StringRef::from_lit("WWW.nghttp2.org"), + StringRef::from_lit("/alpha"), groups, 255)); + + CU_ASSERT(1 == match_downstream_addr_group( + router, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/alpha/bravo/"), groups, 255)); + + // /alpha/bravo also matches /alpha/bravo/ + CU_ASSERT(1 == match_downstream_addr_group( + router, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/alpha/bravo"), groups, 255)); + + // path part is case-sensitive + CU_ASSERT(0 == match_downstream_addr_group( + router, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/Alpha/bravo"), groups, 255)); + + CU_ASSERT(1 == match_downstream_addr_group( + router, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/alpha/bravo/charlie"), groups, 255)); + + CU_ASSERT(2 == match_downstream_addr_group( + router, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/alpha/charlie"), groups, 255)); + + // pattern which does not end with '/' must match its entirely. So + // this matches to group 0, not group 2. + CU_ASSERT(0 == match_downstream_addr_group( + router, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/alpha/charlie/"), groups, 255)); + + CU_ASSERT(255 == match_downstream_addr_group( + router, StringRef::from_lit("example.org"), + StringRef::from_lit("/"), groups, 255)); + + CU_ASSERT(255 == match_downstream_addr_group(router, StringRef::from_lit(""), + StringRef::from_lit("/"), groups, + 255)); + + CU_ASSERT(255 == match_downstream_addr_group(router, StringRef::from_lit(""), + StringRef::from_lit("alpha"), + groups, 255)); + + CU_ASSERT(255 == + match_downstream_addr_group(router, StringRef::from_lit("foo/bar"), + StringRef::from_lit("/"), groups, 255)); + + // If path is StringRef::from_lit("*", only match with host + "/"). + CU_ASSERT(0 == match_downstream_addr_group( + router, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("*"), groups, 255)); + + CU_ASSERT(5 == + match_downstream_addr_group(router, StringRef::from_lit("[::1]"), + StringRef::from_lit("/"), groups, 255)); + CU_ASSERT(5 == match_downstream_addr_group( + router, StringRef::from_lit("[::1]:8080"), + StringRef::from_lit("/"), groups, 255)); + CU_ASSERT(255 == + match_downstream_addr_group(router, StringRef::from_lit("[::1"), + StringRef::from_lit("/"), groups, 255)); + CU_ASSERT(255 == match_downstream_addr_group( + router, StringRef::from_lit("[::1]8000"), + StringRef::from_lit("/"), groups, 255)); + + // Check the case where adding route extends tree + CU_ASSERT(6 == match_downstream_addr_group( + router, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/alpha/bravo/delta"), groups, 255)); + + CU_ASSERT(1 == match_downstream_addr_group( + router, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/alpha/bravo/delta/"), groups, 255)); + + // Check the case where query is done in a single node + CU_ASSERT(7 == match_downstream_addr_group( + router, StringRef::from_lit("example.com"), + StringRef::from_lit("/alpha/bravo"), groups, 255)); + + CU_ASSERT(255 == match_downstream_addr_group( + router, StringRef::from_lit("example.com"), + StringRef::from_lit("/alpha/bravo/"), groups, 255)); + + CU_ASSERT(255 == match_downstream_addr_group( + router, StringRef::from_lit("example.com"), + StringRef::from_lit("/alpha"), groups, 255)); + + // Check the case where quey is done in a single node + CU_ASSERT(8 == match_downstream_addr_group( + router, StringRef::from_lit("192.168.0.1"), + StringRef::from_lit("/alpha"), groups, 255)); + + CU_ASSERT(8 == match_downstream_addr_group( + router, StringRef::from_lit("192.168.0.1"), + StringRef::from_lit("/alpha/"), groups, 255)); + + CU_ASSERT(8 == match_downstream_addr_group( + router, StringRef::from_lit("192.168.0.1"), + StringRef::from_lit("/alpha/bravo"), groups, 255)); + + CU_ASSERT(255 == match_downstream_addr_group( + router, StringRef::from_lit("192.168.0.1"), + StringRef::from_lit("/alph"), groups, 255)); + + CU_ASSERT(255 == match_downstream_addr_group( + router, StringRef::from_lit("192.168.0.1"), + StringRef::from_lit("/"), groups, 255)); + + router.dump(); +} + +} // namespace shrpx diff --git a/src/shrpx_worker_test.h b/src/shrpx_worker_test.h new file mode 100644 index 00000000..c9bdfe0d --- /dev/null +++ b/src/shrpx_worker_test.h @@ -0,0 +1,38 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef SHRPX_WORKER_TEST_H +#define SHRPX_WORKER_TEST_H + +#ifdef HAVE_CONFIG_H +#include +#endif // HAVE_CONFIG_H + +namespace shrpx { + +void test_shrpx_worker_match_downstream_addr_group(void); + +} // namespace shrpx + +#endif // SHRPX_WORKER_TEST_H diff --git a/src/util_test.cc b/src/util_test.cc index 6f605c5b..cd8e2b02 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -456,14 +456,19 @@ void test_util_parse_config_str_list(void) { } void test_util_make_http_hostport(void) { - CU_ASSERT("localhost" == util::make_http_hostport("localhost", 80)); - CU_ASSERT("[::1]" == util::make_http_hostport("::1", 443)); - CU_ASSERT("localhost:3000" == util::make_http_hostport("localhost", 3000)); + CU_ASSERT("localhost" == + util::make_http_hostport(StringRef::from_lit("localhost"), 80)); + CU_ASSERT("[::1]" == + util::make_http_hostport(StringRef::from_lit("::1"), 443)); + CU_ASSERT("localhost:3000" == + util::make_http_hostport(StringRef::from_lit("localhost"), 3000)); } void test_util_make_hostport(void) { - CU_ASSERT("localhost:80" == util::make_hostport("localhost", 80)); - CU_ASSERT("[::1]:443" == util::make_hostport("::1", 443)); + CU_ASSERT("localhost:80" == + util::make_hostport(StringRef::from_lit("localhost"), 80)); + CU_ASSERT("[::1]:443" == + util::make_hostport(StringRef::from_lit("::1"), 443)); } } // namespace shrpx From 284691253ffce0b03fb4161f0bfaa32ed2c5839f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 28 Feb 2016 23:54:02 +0900 Subject: [PATCH 088/147] nghttpx: Use StringRef for http::create_forwarded parameter --- src/shrpx_client_handler.cc | 8 +++---- src/shrpx_client_handler.h | 4 ++-- src/shrpx_http.cc | 5 ++-- src/shrpx_http.h | 4 ++-- src/shrpx_http2_downstream_connection.cc | 6 ++--- src/shrpx_http_downstream_connection.cc | 6 ++--- src/shrpx_http_test.cc | 30 +++++++++++++++--------- 7 files changed, 35 insertions(+), 28 deletions(-) diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 7151dbf2..46540d42 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -1138,18 +1138,18 @@ int ClientHandler::proxy_protocol_read() { return on_proxy_protocol_finish(); } -StringRef ClientHandler::get_forwarded_by() { +StringRef ClientHandler::get_forwarded_by() const { auto &fwdconf = get_config()->http.forwarded; if (fwdconf.by_node_type == FORWARDED_NODE_OBFUSCATED) { return StringRef(fwdconf.by_obfuscated); } - return StringRef(faddr_->hostport); + return StringRef{faddr_->hostport}; } -const std::string &ClientHandler::get_forwarded_for() const { - return forwarded_for_; +StringRef ClientHandler::get_forwarded_for() const { + return StringRef{forwarded_for_}; } } // namespace shrpx diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index 42b03c26..7d6ace81 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -135,10 +135,10 @@ public: // Returns string suitable for use in "by" parameter of Forwarded // header field. - StringRef get_forwarded_by(); + StringRef get_forwarded_by() const; // Returns string suitable for use in "for" parameter of Forwarded // header field. - const std::string &get_forwarded_for() const; + StringRef get_forwarded_for() const; private: Connection conn_; diff --git a/src/shrpx_http.cc b/src/shrpx_http.cc index 86870435..7bca56dd 100644 --- a/src/shrpx_http.cc +++ b/src/shrpx_http.cc @@ -62,9 +62,8 @@ std::string create_via_header_value(int major, int minor) { } std::string create_forwarded(int params, const StringRef &node_by, - const std::string &node_for, - const std::string &host, - const std::string &proto) { + const StringRef &node_for, const StringRef &host, + const StringRef &proto) { std::string res; if ((params & FORWARDED_BY) && !node_by.empty()) { // This must be quoted-string unless it is obfuscated version diff --git a/src/shrpx_http.h b/src/shrpx_http.h index bd29df02..56705ecd 100644 --- a/src/shrpx_http.h +++ b/src/shrpx_http.h @@ -43,8 +43,8 @@ std::string create_via_header_value(int major, int minor); // |params| is bitwise-OR of zero or more of shrpx_forwarded_param // defined in shrpx_config.h. std::string create_forwarded(int params, const StringRef &node_by, - const std::string &node_for, - const std::string &host, const std::string &proto); + const StringRef &node_for, const StringRef &host, + const StringRef &proto); // Adds ANSI color codes to HTTP headers |hdrs|. std::string colorizeHeaders(const char *hdrs); diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index b5accaa2..028b6e41 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -359,9 +359,9 @@ int Http2DownstreamConnection::push_request_headers() { params &= ~FORWARDED_PROTO; } - auto value = http::create_forwarded(params, handler->get_forwarded_by(), - handler->get_forwarded_for(), - req.authority, req.scheme); + auto value = http::create_forwarded( + params, handler->get_forwarded_by(), handler->get_forwarded_for(), + StringRef{req.authority}, StringRef{req.scheme}); if (fwd || !value.empty()) { if (fwd) { forwarded_value = fwd->value; diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index fcd9b8e6..a6c238a1 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -366,9 +366,9 @@ int HttpDownstreamConnection::push_request_headers() { params &= ~FORWARDED_PROTO; } - auto value = http::create_forwarded(params, handler->get_forwarded_by(), - handler->get_forwarded_for(), - req.authority, req.scheme); + auto value = http::create_forwarded( + params, handler->get_forwarded_by(), handler->get_forwarded_for(), + StringRef{req.authority}, StringRef{req.scheme}); if (fwd || !value.empty()) { buf->append("Forwarded: "); if (fwd) { diff --git a/src/shrpx_http_test.cc b/src/shrpx_http_test.cc index 03510550..87fbb9c5 100644 --- a/src/shrpx_http_test.cc +++ b/src/shrpx_http_test.cc @@ -43,25 +43,33 @@ void test_shrpx_http_create_forwarded(void) { http::create_forwarded(FORWARDED_BY | FORWARDED_FOR | FORWARDED_HOST | FORWARDED_PROTO, StringRef::from_lit("example.com:3000"), - "[::1]", "www.example.com", "https")); + StringRef::from_lit("[::1]"), + StringRef::from_lit("www.example.com"), + StringRef::from_lit("https"))); CU_ASSERT("for=192.168.0.1" == http::create_forwarded(FORWARDED_FOR, StringRef::from_lit("alpha"), - "192.168.0.1", "bravo", "charlie")); + StringRef::from_lit("192.168.0.1"), + StringRef::from_lit("bravo"), + StringRef::from_lit("charlie"))); CU_ASSERT("by=_hidden;for=\"[::1]\"" == - http::create_forwarded(FORWARDED_BY | FORWARDED_FOR, - StringRef::from_lit("_hidden"), "[::1]", "", - "")); + http::create_forwarded( + FORWARDED_BY | FORWARDED_FOR, StringRef::from_lit("_hidden"), + StringRef::from_lit("[::1]"), StringRef::from_lit(""), + StringRef::from_lit(""))); CU_ASSERT("by=\"[::1]\";for=_hidden" == - http::create_forwarded(FORWARDED_BY | FORWARDED_FOR, - StringRef::from_lit("[::1]"), "_hidden", "", - "")); + http::create_forwarded( + FORWARDED_BY | FORWARDED_FOR, StringRef::from_lit("[::1]"), + StringRef::from_lit("_hidden"), StringRef::from_lit(""), + StringRef::from_lit(""))); - CU_ASSERT("" == http::create_forwarded(FORWARDED_BY | FORWARDED_FOR | - FORWARDED_HOST | FORWARDED_PROTO, - StringRef::from_lit(""), "", "", "")); + CU_ASSERT("" == + http::create_forwarded( + FORWARDED_BY | FORWARDED_FOR | FORWARDED_HOST | FORWARDED_PROTO, + StringRef::from_lit(""), StringRef::from_lit(""), + StringRef::from_lit(""), StringRef::from_lit(""))); } } // namespace shrpx From c2ec73302dfd7121cd6d6d97f2c99e5257aeeb89 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 29 Feb 2016 00:37:51 +0900 Subject: [PATCH 089/147] integration: Fix tests + `gofmt` --- integration-tests/server_tester.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/integration-tests/server_tester.go b/integration-tests/server_tester.go index 8226c899..67bbec32 100644 --- a/integration-tests/server_tester.go +++ b/integration-tests/server_tester.go @@ -6,10 +6,10 @@ import ( "crypto/tls" "errors" "fmt" - "golang.org/x/net/http2" - "golang.org/x/net/http2/hpack" "github.com/tatsuhiro-t/go-nghttp2" "github.com/tatsuhiro-t/spdy" + "golang.org/x/net/http2" + "golang.org/x/net/http2/hpack" "golang.org/x/net/websocket" "io" "io/ioutil" @@ -30,7 +30,7 @@ const ( serverBin = buildDir + "/src/nghttpx" serverPort = 3009 testDir = sourceDir + "/integration-tests" - logDir = buildDir + "/integration-tests" + logDir = buildDir + "/integration-tests" ) func pair(name, value string) hpack.HeaderField { @@ -86,14 +86,18 @@ func newServerTesterTLSConfig(args []string, t *testing.T, handler http.HandlerF // newServerTesterInternal creates test context. If frontendTLS is // true, set up TLS frontend connection. -func newServerTesterInternal(args []string, t *testing.T, handler http.Handler, frontendTLS bool, clientConfig *tls.Config) *serverTester { +func newServerTesterInternal(src_args []string, t *testing.T, handler http.Handler, frontendTLS bool, clientConfig *tls.Config) *serverTester { ts := httptest.NewUnstartedServer(handler) + args := []string{} + backendTLS := false - for _, k := range args { + for _, k := range src_args { switch k { case "--http2-bridge": backendTLS = true + default: + args = append(args, k) } } if backendTLS { @@ -102,9 +106,9 @@ func newServerTesterInternal(args []string, t *testing.T, handler http.Handler, // NextProtos separately for ts.TLS. NextProtos set // in nghttp2.ConfigureServer is effectively ignored. ts.TLS = new(tls.Config) - ts.TLS.NextProtos = append(ts.TLS.NextProtos, "h2-14") + ts.TLS.NextProtos = append(ts.TLS.NextProtos, "h2") ts.StartTLS() - args = append(args, "-k") + args = append(args, "-k", "--backend-tls") } else { ts.Start() } @@ -124,6 +128,9 @@ func newServerTesterInternal(args []string, t *testing.T, handler http.Handler, // URL.Host looks like "127.0.0.1:8080", but we want // "127.0.0.1,8080" b := "-b" + strings.Replace(backendURL.Host, ":", ",", -1) + if backendTLS { + b += ";;proto=h2" + } args = append(args, fmt.Sprintf("-f127.0.0.1,%v", serverPort), b, "--errorlog-file="+logDir+"/log.txt", "-LINFO") From 1b9b03b68f614b7609bae5c3256a218e82e3be17 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 29 Feb 2016 00:55:16 +0900 Subject: [PATCH 090/147] Update man pages --- doc/h2load.1 | 2 +- doc/nghttp.1 | 2 +- doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 200 ++++++++++++++++++++-------------------------- doc/nghttpx.1.rst | 192 +++++++++++++++++++------------------------- 5 files changed, 171 insertions(+), 227 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index c01af0ee..623fa633 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "February 25, 2016" "1.8.0" "nghttp2" +.TH "H2LOAD" "1" "February 29, 2016" "1.9.0-DEV" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 66165b1d..c825350a 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "February 25, 2016" "1.8.0" "nghttp2" +.TH "NGHTTP" "1" "February 29, 2016" "1.9.0-DEV" "nghttp2" .SH NAME nghttp \- HTTP/2 client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index 52eb071e..fbb76af3 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "February 25, 2016" "1.8.0" "nghttp2" +.TH "NGHTTPD" "1" "February 29, 2016" "1.9.0-DEV" "nghttp2" .SH NAME nghttpd \- HTTP/2 server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index a8b465af..c615bd69 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "February 25, 2016" "1.8.0" "nghttp2" +.TH "NGHTTPX" "1" "February 29, 2016" "1.9.0-DEV" "nghttp2" .SH NAME nghttpx \- HTTP/2 proxy . @@ -39,15 +39,15 @@ A reverse proxy for HTTP/2, HTTP/1 and SPDY. .INDENT 0.0 .TP .B -Set path to server\(aqs private key. Required unless \fI\%\-p\fP, -\fI\%\-\-client\fP or \fI\%\-\-frontend\-no\-tls\fP are given. +Set path to server\(aqs private key. Required unless +\fI\%\-\-frontend\-no\-tls\fP are given. .UNINDENT .INDENT 0.0 .TP .B -Set path to server\(aqs certificate. Required unless \fI\%\-p\fP, -\fI\%\-\-client\fP or \fI\%\-\-frontend\-no\-tls\fP are given. To make OCSP -stapling work, this must be absolute path. +Set path to server\(aqs certificate. Required unless +\fI\%\-\-frontend\-no\-tls\fP are given. To make OCSP stapling +work, this must be an absolute path. .UNINDENT .SH OPTIONS .sp @@ -55,38 +55,39 @@ The options are categorized into several groups. .SS Connections .INDENT 0.0 .TP -.B \-b, \-\-backend=(,|unix:)[;[:...]] +.B \-b, \-\-backend=(,|unix:)[;[[:...]][;proto=]] Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX domain socket can be specified by prefixing path name with "unix:" (e.g., unix:/var/run/backend.sock). .sp Optionally, if s are given, the backend address -is only used if request matches the pattern. If \fI\%\-s\fP or -\fI\%\-p\fP is used, s are ignored. The pattern -matching is closely designed to ServeMux in net/http -package of Go programming language. consists -of path, host + path or just host. The path must start -with "\fI/\fP". If it ends with "\fI/\fP", it matches all request -path in its subtree. To deal with the request to the -directory without trailing slash, the path which ends -with "\fI/\fP" also matches the request path which only lacks -trailing \(aq\fI/\fP\(aq (e.g., path "\fI/foo/\fP" matches request path -"\fI/foo\fP"). If it does not end with "\fI/\fP", it performs exact -match against the request path. If host is given, it -performs exact match against the request host. If host -alone is given, "\fI/\fP" is appended to it, so that it -matches all request paths under the host (e.g., -specifying "nghttp2.org" equals to "nghttp2.org/"). +is only used if request matches the pattern. If +\fI\%\-\-http2\-proxy\fP is used, s are ignored. The +pattern matching is closely designed to ServeMux in +net/http package of Go programming language. +consists of path, host + path or just host. The path +must start with "\fI/\fP". If it ends with "\fI/\fP", it matches +all request path in its subtree. To deal with the +request to the directory without trailing slash, the +path which ends with "\fI/\fP" also matches the request path +which only lacks trailing \(aq\fI/\fP\(aq (e.g., path "\fI/foo/\fP" +matches request path "\fI/foo\fP"). If it does not end with +"\fI/\fP", it performs exact match against the request path. +If host is given, it performs exact match against the +request host. If host alone is given, "\fI/\fP" is appended +to it, so that it matches all request paths under the +host (e.g., specifying "nghttp2.org" equals to +"nghttp2.org/"). .sp Patterns with host take precedence over patterns with just path. Then, longer patterns take precedence over shorter ones, breaking a tie by the order of the appearance in the configuration. .sp -If is omitted, "\fI/\fP" is used as pattern, which -matches all request paths (catch\-all pattern). The -catch\-all backend must be given. +If is omitted or empty string, "\fI/\fP" is used as +pattern, which matches all request paths (catch\-all +pattern). The catch\-all backend must be given. .sp When doing a match, nghttpx made some normalization to pattern, request host and path. For host part, they are @@ -109,6 +110,15 @@ and \fI\%\-b\fP\(aq127.0.0.1,8080;www.nghttp2.org\(aq. The backend addresses sharing same are grouped together forming load balancing group. .sp +Optionally, backend application protocol can be +specified in . All that share the same +must have the same value if it is given. + should be one of the following list without +quotes: "h2", "http/1.1". The default value of +is "http/1.1". Note that usually "h2" refers to HTTP/2 +over TLS. But in this option, it may mean HTTP/2 over +cleartext TCP unless \fI\%\-\-backend\-tls\fP is used. +.sp Since ";" and ":" are used as delimiter, must not contain these characters. Since ";" has special meaning in shell, the option value must be quoted. @@ -167,19 +177,8 @@ Accept PROXY protocol version 1 on frontend connection. .UNINDENT .INDENT 0.0 .TP -.B \-\-backend\-no\-tls -Disable SSL/TLS on backend connections. For HTTP/2 -backend connections, TLS is enabled by default. For -HTTP/1 backend connections, TLS is disabled by default, -and can be enabled by \fI\%\-\-backend\-http1\-tls\fP option. If -both \fI\%\-\-backend\-no\-tls\fP and \fI\%\-\-backend\-http1\-tls\fP options -are used, \fI\%\-\-backend\-no\-tls\fP has the precedence. -.UNINDENT -.INDENT 0.0 -.TP -.B \-\-backend\-http1\-tls -Enable SSL/TLS on backend HTTP/1 connections. See also -\fI\%\-\-backend\-no\-tls\fP option. +.B \-\-backend\-tls +Enable SSL/TLS on backend connections. .UNINDENT .SS Performance .INDENT 0.0 @@ -269,37 +268,27 @@ Default: \fB0\fP .UNINDENT .INDENT 0.0 .TP -.B \-\-backend\-http2\-connections\-per\-worker= -Set maximum number of backend HTTP/2 physical -connections per worker. If pattern is used in \fI\%\-b\fP -option, this limit is applied to each pattern group (in -other words, each pattern group can have maximum -HTTP/2 connections). The default value is 0, which -means that the value is adjusted to the number of -backend addresses. If pattern is used, this adjustment -is done for each pattern group. -.UNINDENT -.INDENT 0.0 -.TP -.B \-\-backend\-http1\-connections\-per\-host= -Set maximum number of backend concurrent HTTP/1 -connections per origin host. This option is meaningful -when \fI\%\-s\fP option is used. The origin host is determined -by authority portion of request URI (or :authority -header field for HTTP/2). To limit the number of -connections per frontend for default mode, use -\fI\%\-\-backend\-http1\-connections\-per\-frontend\fP\&. +.B \-\-backend\-connections\-per\-host= +Set maximum number of backend concurrent connections +(and/or streams in case of HTTP/2) per origin host. +This option is meaningful when \fI\%\-\-http2\-proxy\fP option is +used. The origin host is determined by authority +portion of request URI (or :authority header field for +HTTP/2). To limit the number of connections per +frontend for default mode, use +\fI\%\-\-backend\-connections\-per\-frontend\fP\&. .sp Default: \fB8\fP .UNINDENT .INDENT 0.0 .TP -.B \-\-backend\-http1\-connections\-per\-frontend= -Set maximum number of backend concurrent HTTP/1 -connections per frontend. This option is only used for -default mode. 0 means unlimited. To limit the number -of connections per host for HTTP/2 or SPDY proxy mode -(\-s option), use \fI\%\-\-backend\-http1\-connections\-per\-host\fP\&. +.B \-\-backend\-connections\-per\-frontend= +Set maximum number of backend concurrent connections +(and/or streams in case of HTTP/2) per frontend. This +option is only used for default mode. 0 means +unlimited. To limit the number of connections per host +with \fI\%\-\-http2\-proxy\fP option, use +\fI\%\-\-backend\-connections\-per\-host\fP\&. .sp Default: \fB0\fP .UNINDENT @@ -700,9 +689,19 @@ the complete HTTP/2 cipher suites black list. .SS HTTP/2 and SPDY .INDENT 0.0 .TP -.B \-c, \-\-http2\-max\-concurrent\-streams= +.B \-c, \-\-frontend\-http2\-max\-concurrent\-streams= Set the maximum number of the concurrent streams in one -HTTP/2 and SPDY session. +frontend HTTP/2 and SPDY session. +.sp +Default: \(ga\(ga 100\(ga\(ga +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-backend\-http2\-max\-concurrent\-streams= +Set the maximum number of the concurrent streams in one +backend HTTP/2 session. This sets maximum number of +concurrent opened pushed streams. The maximum number of +concurrent requests are set by a remote server. .sp Default: \fB100\fP .UNINDENT @@ -743,7 +742,7 @@ Default: \fB16\fP Sets the per\-connection window size of HTTP/2 backend connection to 2**\-1. .sp -Default: \fB16\fP +Default: \fB30\fP .UNINDENT .INDENT 0.0 .TP @@ -764,11 +763,10 @@ protocol security. Disable HTTP/2 server push. Server push is supported by default mode and HTTP/2 frontend via Link header field. It is also supported if both frontend and backend are -HTTP/2 (which implies \fI\%\-\-http2\-bridge\fP or \fI\%\-\-client\fP mode). -In this case, server push from backend session is -relayed to frontend, and server push via Link header -field is also supported. HTTP SPDY frontend does not -support server push. +HTTP/2 in default mode. In this case, server push from +backend session is relayed to frontend, and server push +via Link header field is also supported. SPDY frontend +does not support server push. .UNINDENT .SS Mode .INDENT 0.0 @@ -777,39 +775,13 @@ support server push. Accept HTTP/2, SPDY and HTTP/1.1 over SSL/TLS. If \fI\%\-\-frontend\-no\-tls\fP is used, accept HTTP/2 and HTTP/1.1. The incoming HTTP/1.1 connection can be upgraded to -HTTP/2 through HTTP Upgrade. The protocol to the -backend is HTTP/1.1. +HTTP/2 through HTTP Upgrade. .UNINDENT .INDENT 0.0 .TP .B \-s, \-\-http2\-proxy -Like default mode, but enable secure proxy mode. -.UNINDENT -.INDENT 0.0 -.TP -.B \-\-http2\-bridge -Like default mode, but communicate with the backend in -HTTP/2 over SSL/TLS. Thus the incoming all connections -are converted to HTTP/2 connection and relayed to the -backend. See \fI\%\-\-backend\-http\-proxy\-uri\fP option if you are -behind the proxy and want to connect to the outside -HTTP/2 proxy. -.UNINDENT -.INDENT 0.0 -.TP -.B \-\-client -Accept HTTP/2 and HTTP/1.1 without SSL/TLS. The -incoming HTTP/1.1 connection can be upgraded to HTTP/2 -connection through HTTP Upgrade. The protocol to the -backend is HTTP/2. To use nghttpx as a forward proxy, -use \fI\%\-p\fP option instead. -.UNINDENT -.INDENT 0.0 -.TP -.B \-p, \-\-client\-proxy -Like \fI\%\-\-client\fP option, but it also requires the request -path from frontend must be an absolute URI, suitable for -use as a forward proxy. +Like default mode, but enable forward proxy. This is so +called HTTP/2 proxy mode. .UNINDENT .SS Logging .INDENT 0.0 @@ -975,18 +947,16 @@ is received, it is left unaltered. .INDENT 0.0 .TP .B \-\-no\-location\-rewrite -Don\(aqt rewrite location header field on \fI\%\-\-http2\-bridge\fP, -\fI\%\-\-client\fP and default mode. For \fI\%\-\-http2\-proxy\fP and -\fI\%\-\-client\-proxy\fP mode, location header field will not be -altered regardless of this option. +Don\(aqt rewrite location header field in default mode. +When \fI\%\-\-http2\-proxy\fP is used, location header field will +not be altered regardless of this option. .UNINDENT .INDENT 0.0 .TP .B \-\-host\-rewrite -Rewrite host and :authority header fields on -\fI\%\-\-http2\-bridge\fP, \fI\%\-\-client\fP and default mode. For -\fI\%\-\-http2\-proxy\fP and \fI\%\-\-client\-proxy\fP mode, these headers -will not be altered regardless of this option. +Rewrite host and :authority header fields in default +mode. When \fI\%\-\-http2\-proxy\fP is used, these headers will +not be altered regardless of this option. .UNINDENT .INDENT 0.0 .TP @@ -1258,12 +1228,12 @@ associated stream\(aqs status code must be 200. This limitation may be loosened in the future release. .sp nghttpx also supports server push if both frontend and backend are -HTTP/2 (which implies \fI\%\-\-http2\-bridge\fP or \fI\%\-\-client\fP). -In this case, in addition to server push via Link header field, server -push from backend is relayed to frontend HTTP/2 session. +HTTP/2 in default mode. In this case, in addition to server push via +Link header field, server push from backend is forwarded to frontend +HTTP/2 session. .sp -HTTP/2 server push will be disabled if \fI\%\-\-http2\-proxy\fP or -\fI\%\-\-client\-proxy\fP is used. +HTTP/2 server push will be disabled if \fI\%\-\-http2\-proxy\fP is +used. .SH UNIX DOMAIN SOCKET .sp nghttpx supports UNIX domain socket with a filename for both frontend diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index 22f98ff0..afccc2e9 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -19,14 +19,14 @@ A reverse proxy for HTTP/2, HTTP/1 and SPDY. .. describe:: - Set path to server's private key. Required unless :option:`-p`\, - :option:`--client` or :option:`\--frontend-no-tls` are given. + Set path to server's private key. Required unless + :option:`--frontend-no-tls` are given. .. describe:: - Set path to server's certificate. Required unless :option:`-p`\, - :option:`--client` or :option:`\--frontend-no-tls` are given. To make OCSP - stapling work, this must be absolute path. + Set path to server's certificate. Required unless + :option:`--frontend-no-tls` are given. To make OCSP stapling + work, this must be an absolute path. OPTIONS @@ -37,7 +37,7 @@ The options are categorized into several groups. Connections ~~~~~~~~~~~ -.. option:: -b, --backend=(,|unix:)[;[:...]] +.. option:: -b, --backend=(,|unix:)[;[[:...]][;proto=]] Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX @@ -45,31 +45,32 @@ Connections with "unix:" (e.g., unix:/var/run/backend.sock). Optionally, if s are given, the backend address - is only used if request matches the pattern. If :option:`-s` or - :option:`-p` is used, s are ignored. The pattern - matching is closely designed to ServeMux in net/http - package of Go programming language. consists - of path, host + path or just host. The path must start - with "*/*". If it ends with "*/*", it matches all request - path in its subtree. To deal with the request to the - directory without trailing slash, the path which ends - with "*/*" also matches the request path which only lacks - trailing '*/*' (e.g., path "*/foo/*" matches request path - "*/foo*"). If it does not end with "*/*", it performs exact - match against the request path. If host is given, it - performs exact match against the request host. If host - alone is given, "*/*" is appended to it, so that it - matches all request paths under the host (e.g., - specifying "nghttp2.org" equals to "nghttp2.org/"). + is only used if request matches the pattern. If + :option:`--http2-proxy` is used, s are ignored. The + pattern matching is closely designed to ServeMux in + net/http package of Go programming language. + consists of path, host + path or just host. The path + must start with "*/*". If it ends with "*/*", it matches + all request path in its subtree. To deal with the + request to the directory without trailing slash, the + path which ends with "*/*" also matches the request path + which only lacks trailing '*/*' (e.g., path "*/foo/*" + matches request path "*/foo*"). If it does not end with + "*/*", it performs exact match against the request path. + If host is given, it performs exact match against the + request host. If host alone is given, "*/*" is appended + to it, so that it matches all request paths under the + host (e.g., specifying "nghttp2.org" equals to + "nghttp2.org/"). Patterns with host take precedence over patterns with just path. Then, longer patterns take precedence over shorter ones, breaking a tie by the order of the appearance in the configuration. - If is omitted, "*/*" is used as pattern, which - matches all request paths (catch-all pattern). The - catch-all backend must be given. + If is omitted or empty string, "*/*" is used as + pattern, which matches all request paths (catch-all + pattern). The catch-all backend must be given. When doing a match, nghttpx made some normalization to pattern, request host and path. For host part, they are @@ -92,6 +93,15 @@ Connections The backend addresses sharing same are grouped together forming load balancing group. + Optionally, backend application protocol can be + specified in . All that share the same + must have the same value if it is given. + should be one of the following list without + quotes: "h2", "http/1.1". The default value of + is "http/1.1". Note that usually "h2" refers to HTTP/2 + over TLS. But in this option, it may mean HTTP/2 over + cleartext TCP unless :option:`--backend-tls` is used. + Since ";" and ":" are used as delimiter, must not contain these characters. Since ";" has special meaning in shell, the option value must be quoted. @@ -144,19 +154,9 @@ Connections Accept PROXY protocol version 1 on frontend connection. -.. option:: --backend-no-tls +.. option:: --backend-tls - Disable SSL/TLS on backend connections. For HTTP/2 - backend connections, TLS is enabled by default. For - HTTP/1 backend connections, TLS is disabled by default, - and can be enabled by :option:`--backend-http1-tls` option. If - both :option:`--backend-no-tls` and :option:`\--backend-http1-tls` options - are used, :option:`--backend-no-tls` has the precedence. - -.. option:: --backend-http1-tls - - Enable SSL/TLS on backend HTTP/1 connections. See also - :option:`--backend-no-tls` option. + Enable SSL/TLS on backend connections. Performance @@ -237,36 +237,27 @@ Performance Default: ``0`` -.. option:: --backend-http2-connections-per-worker= +.. option:: --backend-connections-per-host= - Set maximum number of backend HTTP/2 physical - connections per worker. If pattern is used in :option:`-b` - option, this limit is applied to each pattern group (in - other words, each pattern group can have maximum - HTTP/2 connections). The default value is 0, which - means that the value is adjusted to the number of - backend addresses. If pattern is used, this adjustment - is done for each pattern group. - -.. option:: --backend-http1-connections-per-host= - - Set maximum number of backend concurrent HTTP/1 - connections per origin host. This option is meaningful - when :option:`-s` option is used. The origin host is determined - by authority portion of request URI (or :authority - header field for HTTP/2). To limit the number of - connections per frontend for default mode, use - :option:`--backend-http1-connections-per-frontend`\. + Set maximum number of backend concurrent connections + (and/or streams in case of HTTP/2) per origin host. + This option is meaningful when :option:`--http2-proxy` option is + used. The origin host is determined by authority + portion of request URI (or :authority header field for + HTTP/2). To limit the number of connections per + frontend for default mode, use + :option:`--backend-connections-per-frontend`\. Default: ``8`` -.. option:: --backend-http1-connections-per-frontend= +.. option:: --backend-connections-per-frontend= - Set maximum number of backend concurrent HTTP/1 - connections per frontend. This option is only used for - default mode. 0 means unlimited. To limit the number - of connections per host for HTTP/2 or SPDY proxy mode - (-s option), use :option:`--backend-http1-connections-per-host`\. + Set maximum number of backend concurrent connections + (and/or streams in case of HTTP/2) per frontend. This + option is only used for default mode. 0 means + unlimited. To limit the number of connections per host + with :option:`--http2-proxy` option, use + :option:`--backend-connections-per-host`\. Default: ``0`` @@ -626,10 +617,19 @@ SSL/TLS HTTP/2 and SPDY ~~~~~~~~~~~~~~~ -.. option:: -c, --http2-max-concurrent-streams= +.. option:: -c, --frontend-http2-max-concurrent-streams= Set the maximum number of the concurrent streams in one - HTTP/2 and SPDY session. + frontend HTTP/2 and SPDY session. + + Default: `` 100`` + +.. option:: --backend-http2-max-concurrent-streams= + + Set the maximum number of the concurrent streams in one + backend HTTP/2 session. This sets maximum number of + concurrent opened pushed streams. The maximum number of + concurrent requests are set by a remote server. Default: ``100`` @@ -665,7 +665,7 @@ HTTP/2 and SPDY Sets the per-connection window size of HTTP/2 backend connection to 2\*\*-1. - Default: ``16`` + Default: ``30`` .. option:: --http2-no-cookie-crumbling @@ -683,11 +683,10 @@ HTTP/2 and SPDY Disable HTTP/2 server push. Server push is supported by default mode and HTTP/2 frontend via Link header field. It is also supported if both frontend and backend are - HTTP/2 (which implies :option:`--http2-bridge` or :option:`\--client` mode). - In this case, server push from backend session is - relayed to frontend, and server push via Link header - field is also supported. HTTP SPDY frontend does not - support server push. + HTTP/2 in default mode. In this case, server push from + backend session is relayed to frontend, and server push + via Link header field is also supported. SPDY frontend + does not support server push. Mode @@ -699,35 +698,12 @@ Mode Accept HTTP/2, SPDY and HTTP/1.1 over SSL/TLS. If :option:`--frontend-no-tls` is used, accept HTTP/2 and HTTP/1.1. The incoming HTTP/1.1 connection can be upgraded to - HTTP/2 through HTTP Upgrade. The protocol to the - backend is HTTP/1.1. + HTTP/2 through HTTP Upgrade. .. option:: -s, --http2-proxy - Like default mode, but enable secure proxy mode. - -.. option:: --http2-bridge - - Like default mode, but communicate with the backend in - HTTP/2 over SSL/TLS. Thus the incoming all connections - are converted to HTTP/2 connection and relayed to the - backend. See :option:`--backend-http-proxy-uri` option if you are - behind the proxy and want to connect to the outside - HTTP/2 proxy. - -.. option:: --client - - Accept HTTP/2 and HTTP/1.1 without SSL/TLS. The - incoming HTTP/1.1 connection can be upgraded to HTTP/2 - connection through HTTP Upgrade. The protocol to the - backend is HTTP/2. To use nghttpx as a forward proxy, - use :option:`-p` option instead. - -.. option:: -p, --client-proxy - - Like :option:`--client` option, but it also requires the request - path from frontend must be an absolute URI, suitable for - use as a forward proxy. + Like default mode, but enable forward proxy. This is so + called HTTP/2 proxy mode. Logging @@ -868,17 +844,15 @@ HTTP .. option:: --no-location-rewrite - Don't rewrite location header field on :option:`--http2-bridge`\, - :option:`--client` and default mode. For :option:`\--http2-proxy` and - :option:`--client-proxy` mode, location header field will not be - altered regardless of this option. + Don't rewrite location header field in default mode. + When :option:`--http2-proxy` is used, location header field will + not be altered regardless of this option. .. option:: --host-rewrite - Rewrite host and :authority header fields on - :option:`--http2-bridge`\, :option:`--client` and default mode. For - :option:`--http2-proxy` and :option:`\--client-proxy` mode, these headers - will not be altered regardless of this option. + Rewrite host and :authority header fields in default + mode. When :option:`--http2-proxy` is used, these headers will + not be altered regardless of this option. .. option:: --altsvc= @@ -1126,12 +1100,12 @@ Currently, the following restriction is applied for server push: This limitation may be loosened in the future release. nghttpx also supports server push if both frontend and backend are -HTTP/2 (which implies :option:`--http2-bridge` or :option:`--client`). -In this case, in addition to server push via Link header field, server -push from backend is relayed to frontend HTTP/2 session. +HTTP/2 in default mode. In this case, in addition to server push via +Link header field, server push from backend is forwarded to frontend +HTTP/2 session. -HTTP/2 server push will be disabled if :option:`--http2-proxy` or -:option:`--client-proxy` is used. +HTTP/2 server push will be disabled if :option:`--http2-proxy` is +used. UNIX DOMAIN SOCKET ------------------ From bffc0ec87a65c2e22eb69b859ffa2e08b431fe8a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 29 Feb 2016 01:00:03 +0900 Subject: [PATCH 091/147] Update doc --- doc/sources/nghttpx-howto.rst | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index b9bbaa25..6a27bfe3 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -303,15 +303,11 @@ As of nghttpx 1.9.0, ``--http2-bridge``, ``--client`` and ``--client-proxy`` options were removed. These functionality can be used using combinations of options. -* ``--http2-bridge``: Use - :option:`--backend`\='-b,;;proto=h2', and - :option:`--backend-tls`. +* ``--http2-bridge``: Use ``--backend=',;;proto=h2'``, and + ``--backend-tls``. -* ``--client``: Use :option:`--frontend-no-tls`, - :option:`--backend`\='-b,;;proto=h2', and - :option:`--backend-tls`. +* ``--client``: Use ``--frontend-no-tls``, + ``--backend=',;;proto=h2'``, and ``--backend-tls``. -* ``--client-proxy``: Use :option:`--http2-proxy`, - :option:`--frontend-no-tls`, - :option:`--backend`\='-b,;;proto=h2', and - :option:`--backend-tls`. +* ``--client-proxy``: Use ``--http2-proxy``, ``--frontend-no-tls``, + ``--backend=',;;proto=h2'``, and ``--backend-tls``. From 478fde5fef3bc183dfa50ac77a263633d10a4366 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 29 Feb 2016 01:16:45 +0900 Subject: [PATCH 092/147] nghttpx: Fix compile error --- src/shrpx_config.cc | 3 +++ src/shrpx_ssl.cc | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 5f2e51cb..1046fdf0 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2603,6 +2603,9 @@ StringRef strproto(shrpx_proto proto) { case PROTO_MEMCACHED: return StringRef::from_lit("memcached"); } + + // gcc needs this. + assert(0); } } // namespace shrpx diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 208a7342..a36b20be 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -1325,17 +1325,17 @@ SSL_CTX *setup_downstream_client_ssl_context( } void setup_downstream_http2_alpn(SSL *ssl) { - auto alpn = util::get_default_alpn(); #if OPENSSL_VERSION_NUMBER >= 0x10002000L // ALPN advertisement + auto alpn = util::get_default_alpn(); SSL_set_alpn_protos(ssl, alpn.data(), alpn.size()); #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L } void setup_downstream_http1_alpn(SSL *ssl) { - auto alpn = StringRef::from_lit(NGHTTP2_H1_1_ALPN); #if OPENSSL_VERSION_NUMBER >= 0x10002000L // ALPN advertisement + auto alpn = StringRef::from_lit(NGHTTP2_H1_1_ALPN); SSL_set_alpn_protos(ssl, alpn.byte(), alpn.size()); #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L } From a21c87d11cde50a0979dfa52824e167a5bc1a667 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 29 Feb 2016 23:09:15 +0900 Subject: [PATCH 093/147] Add nghttp2_http2_strerror() to return HTTP/2 error code string --- doc/Makefile.am | 1 + lib/includes/nghttp2/nghttp2.h | 10 ++++++++ lib/nghttp2_helper.c | 35 ++++++++++++++++++++++++++++ src/app_helper.cc | 42 +++------------------------------- 4 files changed, 49 insertions(+), 39 deletions(-) diff --git a/doc/Makefile.am b/doc/Makefile.am index e0316df0..1052f957 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -49,6 +49,7 @@ APIDOCS= \ nghttp2_hd_inflate_hd.rst \ nghttp2_hd_inflate_new.rst \ nghttp2_hd_inflate_new2.rst \ + nghttp2_http2_strerror.rst \ nghttp2_is_fatal.rst \ nghttp2_nv_compare_name.rst \ nghttp2_option_del.rst \ diff --git a/lib/includes/nghttp2/nghttp2.h b/lib/includes/nghttp2/nghttp2.h index 329a605d..f4c53fa6 100644 --- a/lib/includes/nghttp2/nghttp2.h +++ b/lib/includes/nghttp2/nghttp2.h @@ -3224,6 +3224,16 @@ nghttp2_pack_settings_payload(uint8_t *buf, size_t buflen, */ NGHTTP2_EXTERN const char *nghttp2_strerror(int lib_error_code); +/** + * @function + * + * Returns string representation of HTTP/2 error code |error_code| + * (e.g., ``PROTOCOL_ERROR`` is returned if ``error_code == + * NGHTTP2_PROTOCOL_ERROR``). If string representation is unknown for + * given |error_code|, this function returns string ``unknown``. + */ +NGHTTP2_EXTERN const char *nghttp2_http2_strerror(uint32_t error_code); + /** * @function * diff --git a/lib/nghttp2_helper.c b/lib/nghttp2_helper.c index bdba797c..9f208519 100644 --- a/lib/nghttp2_helper.c +++ b/lib/nghttp2_helper.c @@ -451,3 +451,38 @@ uint8_t *nghttp2_cpymem(uint8_t *dest, const void *src, size_t len) { return dest + len; } + +const char *nghttp2_http2_strerror(uint32_t error_code) { + switch (error_code) { + case NGHTTP2_NO_ERROR: + return "NO_ERROR"; + case NGHTTP2_PROTOCOL_ERROR: + return "PROTOCOL_ERROR"; + case NGHTTP2_INTERNAL_ERROR: + return "INTERNAL_ERROR"; + case NGHTTP2_FLOW_CONTROL_ERROR: + return "FLOW_CONTROL_ERROR"; + case NGHTTP2_SETTINGS_TIMEOUT: + return "SETTINGS_TIMEOUT"; + case NGHTTP2_STREAM_CLOSED: + return "STREAM_CLOSED"; + case NGHTTP2_FRAME_SIZE_ERROR: + return "FRAME_SIZE_ERROR"; + case NGHTTP2_REFUSED_STREAM: + return "REFUSED_STREAM"; + case NGHTTP2_CANCEL: + return "CANCEL"; + case NGHTTP2_COMPRESSION_ERROR: + return "COMPRESSION_ERROR"; + case NGHTTP2_CONNECT_ERROR: + return "CONNECT_ERROR"; + case NGHTTP2_ENHANCE_YOUR_CALM: + return "ENHANCE_YOUR_CALM"; + case NGHTTP2_INADEQUATE_SECURITY: + return "INADEQUATE_SECURITY"; + case NGHTTP2_HTTP_1_1_REQUIRED: + return "HTTP_1_1_REQUIRED"; + default: + return "unknown"; + } +} diff --git a/src/app_helper.cc b/src/app_helper.cc index ecbe0b2f..ba84052b 100644 --- a/src/app_helper.cc +++ b/src/app_helper.cc @@ -62,43 +62,6 @@ namespace nghttp2 { -namespace { -const char *strstatus(uint32_t error_code) { - switch (error_code) { - case NGHTTP2_NO_ERROR: - return "NO_ERROR"; - case NGHTTP2_PROTOCOL_ERROR: - return "PROTOCOL_ERROR"; - case NGHTTP2_INTERNAL_ERROR: - return "INTERNAL_ERROR"; - case NGHTTP2_FLOW_CONTROL_ERROR: - return "FLOW_CONTROL_ERROR"; - case NGHTTP2_SETTINGS_TIMEOUT: - return "SETTINGS_TIMEOUT"; - case NGHTTP2_STREAM_CLOSED: - return "STREAM_CLOSED"; - case NGHTTP2_FRAME_SIZE_ERROR: - return "FRAME_SIZE_ERROR"; - case NGHTTP2_REFUSED_STREAM: - return "REFUSED_STREAM"; - case NGHTTP2_CANCEL: - return "CANCEL"; - case NGHTTP2_COMPRESSION_ERROR: - return "COMPRESSION_ERROR"; - case NGHTTP2_CONNECT_ERROR: - return "CONNECT_ERROR"; - case NGHTTP2_ENHANCE_YOUR_CALM: - return "ENHANCE_YOUR_CALM"; - case NGHTTP2_INADEQUATE_SECURITY: - return "INADEQUATE_SECURITY"; - case NGHTTP2_HTTP_1_1_REQUIRED: - return "HTTP_1_1_REQUIRED"; - default: - return "UNKNOWN"; - } -} -} // namespace - namespace { const char *strsettingsid(int32_t id) { switch (id) { @@ -335,7 +298,7 @@ void print_frame(print_type ptype, const nghttp2_frame *frame) { case NGHTTP2_RST_STREAM: print_frame_attr_indent(); fprintf(outfile, "(error_code=%s(0x%02x))\n", - strstatus(frame->rst_stream.error_code), + nghttp2_http2_strerror(frame->rst_stream.error_code), frame->rst_stream.error_code); break; case NGHTTP2_SETTINGS: @@ -364,7 +327,8 @@ void print_frame(print_type ptype, const nghttp2_frame *frame) { print_frame_attr_indent(); fprintf(outfile, "(last_stream_id=%d, error_code=%s(0x%02x), " "opaque_data(%u)=[%s])\n", - frame->goaway.last_stream_id, strstatus(frame->goaway.error_code), + frame->goaway.last_stream_id, + nghttp2_http2_strerror(frame->goaway.error_code), frame->goaway.error_code, static_cast(frame->goaway.opaque_data_len), util::ascii_dump(frame->goaway.opaque_data, From e4537596372b2a7a9582a18676b0df5cc48953ae Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 29 Feb 2016 23:39:50 +0900 Subject: [PATCH 094/147] Add nghttp2_option_set_no_auto_ping_ack() option This option prevents the nghttp2 library from sending PING frame with ACK flag set in the reply to incoming PING frame. To allow the application to send PING with ACK flag set, nghttp2_submit_ping() now recognizes NGHTTP2_FLAG_PING in its flags parameter. --- doc/Makefile.am | 1 + lib/includes/nghttp2/nghttp2.h | 22 ++++++++++++++++++++-- lib/nghttp2_option.c | 5 +++++ lib/nghttp2_option.h | 7 ++++++- lib/nghttp2_session.c | 8 +++++++- lib/nghttp2_session.h | 3 ++- lib/nghttp2_submit.c | 3 ++- tests/nghttp2_session_test.c | 27 +++++++++++++++++++++++++++ 8 files changed, 70 insertions(+), 6 deletions(-) diff --git a/doc/Makefile.am b/doc/Makefile.am index 1052f957..35f1390b 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -55,6 +55,7 @@ APIDOCS= \ nghttp2_option_del.rst \ nghttp2_option_new.rst \ nghttp2_option_set_max_reserved_remote_streams.rst \ + nghttp2_option_set_no_auto_ping_ack.rst \ nghttp2_option_set_no_auto_window_update.rst \ nghttp2_option_set_no_http_messaging.rst \ nghttp2_option_set_no_recv_client_magic.rst \ diff --git a/lib/includes/nghttp2/nghttp2.h b/lib/includes/nghttp2/nghttp2.h index f4c53fa6..f6133055 100644 --- a/lib/includes/nghttp2/nghttp2.h +++ b/lib/includes/nghttp2/nghttp2.h @@ -2251,6 +2251,20 @@ NGHTTP2_EXTERN void nghttp2_option_set_user_recv_extension_type(nghttp2_option *option, uint8_t type); +/** + * @function + * + * This option prevents the library from sending PING frame with ACK + * flag set automatically when PING frame without ACK flag set is + * received. If this option is set to nonzero, the library won't send + * PING frame with ACK flag set in the response for incoming PING + * frame. The application can send PING frame with ACK flag set using + * `nghttp2_submit_ping()` with :enum:`NGHTTP2_FLAG_ACK` as flags + * parameter. + */ +NGHTTP2_EXTERN void nghttp2_option_set_no_auto_ping_ack(nghttp2_option *option, + int val); + /** * @function * @@ -3783,8 +3797,12 @@ nghttp2_submit_push_promise(nghttp2_session *session, uint8_t flags, * received PING frame. The library automatically submits PING frame * in this case. * - * The |flags| is currently ignored and should be - * :enum:`NGHTTP2_FLAG_NONE`. + * The |flags| is bitwise OR of 0 or more of the following value. + * + * * :enum:`NGHTTP2_FLAG_ACK` + * + * Unless `nghttp2_option_set_no_auto_ping_ack()` is used, the |flags| + * should be :enum:`NGHTTP2_FLAG_NONE`. * * If the |opaque_data| is non ``NULL``, then it should point to the 8 * bytes array of memory to specify opaque data to send with PING diff --git a/lib/nghttp2_option.c b/lib/nghttp2_option.c index fd665112..e3e8717b 100644 --- a/lib/nghttp2_option.c +++ b/lib/nghttp2_option.c @@ -73,3 +73,8 @@ void nghttp2_option_set_user_recv_extension_type(nghttp2_option *option, option->user_recv_ext_types[type / 8] = (uint8_t)(option->user_recv_ext_types[type / 8] | (1 << (type & 0x7))); } + +void nghttp2_option_set_no_auto_ping_ack(nghttp2_option *option, int val) { + option->opt_set_mask |= NGHTTP2_OPT_NO_AUTO_PING_ACK; + option->no_auto_ping_ack = val; +} diff --git a/lib/nghttp2_option.h b/lib/nghttp2_option.h index a2d090fb..f859b4eb 100644 --- a/lib/nghttp2_option.h +++ b/lib/nghttp2_option.h @@ -60,7 +60,8 @@ typedef enum { NGHTTP2_OPT_NO_RECV_CLIENT_MAGIC = 1 << 2, NGHTTP2_OPT_NO_HTTP_MESSAGING = 1 << 3, NGHTTP2_OPT_MAX_RESERVED_REMOTE_STREAMS = 1 << 4, - NGHTTP2_OPT_USER_RECV_EXT_TYPES = 1 << 5 + NGHTTP2_OPT_USER_RECV_EXT_TYPES = 1 << 5, + NGHTTP2_OPT_NO_AUTO_PING_ACK = 1 << 6 } nghttp2_option_flag; /** @@ -92,6 +93,10 @@ struct nghttp2_option { * NGHTTP2_OPT_NO_HTTP_MESSAGING */ int no_http_messaging; + /** + * NGHTTP2_OPT_NO_AUTO_PING_ACK + */ + int no_auto_ping_ack; /** * NGHTTP2_OPT_USER_RECV_EXT_TYPES */ diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index fffdf45d..cbc1bac6 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -416,6 +416,11 @@ static int session_new(nghttp2_session **session_ptr, memcpy((*session_ptr)->user_recv_ext_types, option->user_recv_ext_types, sizeof((*session_ptr)->user_recv_ext_types)); } + + if ((option->opt_set_mask & NGHTTP2_OPT_NO_AUTO_PING_ACK) && + option->no_auto_ping_ack) { + (*session_ptr)->opt_flags |= NGHTTP2_OPTMASK_NO_AUTO_PING_ACK; + } } (*session_ptr)->callbacks = *callbacks; @@ -4364,7 +4369,8 @@ int nghttp2_session_on_ping_received(nghttp2_session *session, return session_handle_invalid_connection(session, frame, NGHTTP2_ERR_PROTO, "PING: stream_id != 0"); } - if ((frame->hd.flags & NGHTTP2_FLAG_ACK) == 0 && + if ((session->opt_flags & NGHTTP2_OPTMASK_NO_AUTO_PING_ACK) == 0 && + (frame->hd.flags & NGHTTP2_FLAG_ACK) == 0 && !session_is_closing(session)) { /* Peer sent ping, so ping it back */ rv = nghttp2_session_add_ping(session, NGHTTP2_FLAG_ACK, diff --git a/lib/nghttp2_session.h b/lib/nghttp2_session.h index c319c15e..3c3764de 100644 --- a/lib/nghttp2_session.h +++ b/lib/nghttp2_session.h @@ -50,7 +50,8 @@ extern int nghttp2_enable_strict_preface; typedef enum { NGHTTP2_OPTMASK_NO_AUTO_WINDOW_UPDATE = 1 << 0, NGHTTP2_OPTMASK_NO_RECV_CLIENT_MAGIC = 1 << 1, - NGHTTP2_OPTMASK_NO_HTTP_MESSAGING = 1 << 2 + NGHTTP2_OPTMASK_NO_HTTP_MESSAGING = 1 << 2, + NGHTTP2_OPTMASK_NO_AUTO_PING_ACK = 1 << 3 } nghttp2_optmask; typedef enum { diff --git a/lib/nghttp2_submit.c b/lib/nghttp2_submit.c index 19f2a316..9c11e210 100644 --- a/lib/nghttp2_submit.c +++ b/lib/nghttp2_submit.c @@ -211,8 +211,9 @@ int32_t nghttp2_submit_headers(nghttp2_session *session, uint8_t flags, nvlen, NULL, stream_user_data); } -int nghttp2_submit_ping(nghttp2_session *session, uint8_t flags _U_, +int nghttp2_submit_ping(nghttp2_session *session, uint8_t flags, const uint8_t *opaque_data) { + flags &= NGHTTP2_FLAG_ACK; return nghttp2_session_add_ping(session, NGHTTP2_FLAG_NONE, opaque_data); } diff --git a/tests/nghttp2_session_test.c b/tests/nghttp2_session_test.c index 54da8670..2e7dafee 100644 --- a/tests/nghttp2_session_test.c +++ b/tests/nghttp2_session_test.c @@ -3095,6 +3095,7 @@ void test_nghttp2_session_on_ping_received(void) { nghttp2_frame frame; nghttp2_outbound_item *top; const uint8_t opaque_data[] = "01234567"; + nghttp2_option *option; user_data.frame_recv_cb_called = 0; user_data.invalid_frame_recv_cb_called = 0; @@ -3125,6 +3126,23 @@ void test_nghttp2_session_on_ping_received(void) { nghttp2_frame_ping_free(&frame.ping); nghttp2_session_del(session); + + /* Use nghttp2_option_set_no_auto_ping_ack() */ + nghttp2_option_new(&option); + nghttp2_option_set_no_auto_ping_ack(option, 1); + + nghttp2_session_server_new2(&session, &callbacks, &user_data, option); + nghttp2_frame_ping_init(&frame.ping, NGHTTP2_FLAG_NONE, NULL); + + user_data.frame_recv_cb_called = 0; + + CU_ASSERT(0 == nghttp2_session_on_ping_received(session, &frame)); + CU_ASSERT(1 == user_data.frame_recv_cb_called); + CU_ASSERT(NULL == nghttp2_outbound_queue_top(&session->ob_urgent)); + + nghttp2_frame_ping_free(&frame.ping); + nghttp2_session_del(session); + nghttp2_option_del(option); } void test_nghttp2_session_on_goaway_received(void) { @@ -6153,6 +6171,15 @@ void test_nghttp2_session_set_option(void) { CU_ASSERT(99 == session->max_incoming_reserved_streams); nghttp2_session_del(session); + /* Test for nghttp2_option_set_no_auto_ping_ack */ + nghttp2_option_set_no_auto_ping_ack(option, 1); + + nghttp2_session_client_new2(&session, &callbacks, NULL, option); + + CU_ASSERT(session->opt_flags & NGHTTP2_OPTMASK_NO_AUTO_PING_ACK); + + nghttp2_session_del(session); + nghttp2_option_del(option); } From ce61f62644d80d8790f76b363cc65615220fad90 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 2 Mar 2016 21:18:42 +0900 Subject: [PATCH 095/147] h2load: Fix bug that initial max concurrent streams was too large --- src/h2load.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/h2load.cc b/src/h2load.cc index 61048123..24276f6e 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -84,7 +84,7 @@ Config::Config() nreqs(1), nclients(1), nthreads(1), - max_concurrent_streams(-1), + max_concurrent_streams(1), window_bits(30), connection_window_bits(30), rate(0), From 5da38b22c0a9290ccc6d98c94c65ed9c30ec53fa Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 2 Mar 2016 22:24:14 +0900 Subject: [PATCH 096/147] h2load: Fix bug that it did not try to connect to server again --- src/h2load.cc | 16 +++++++++++++--- src/h2load.h | 6 ++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/h2load.cc b/src/h2load.cc index 24276f6e..dc164b28 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -202,7 +202,9 @@ void readcb(struct ev_loop *loop, ev_io *w, int revents) { auto client = static_cast(w->data); client->restart_timeout(); if (client->do_read() != 0) { - client->fail(); + if (client->try_again_or_fail() == 0) { + return; + } delete client; return; } @@ -453,7 +455,7 @@ void Client::restart_timeout() { } } -void Client::fail() { +int Client::try_again_or_fail() { disconnect(); if (new_connection_requested) { @@ -471,13 +473,21 @@ void Client::fail() { // Keep using current address if (connect() == 0) { - return; + return 0; } std::cerr << "client could not connect to host" << std::endl; } } process_abandoned_streams(); + + return -1; +} + +void Client::fail() { + disconnect(); + + process_abandoned_streams(); } void Client::disconnect() { diff --git a/src/h2load.h b/src/h2load.h index c053b9c8..de11b4b4 100644 --- a/src/h2load.h +++ b/src/h2load.h @@ -307,6 +307,12 @@ struct Client { int connect(); void disconnect(); void fail(); + // Call this function when do_read() returns -1. This function + // tries to connect to the remote host again if it is requested. If + // so, this function returns 0, and this object should be retained. + // Otherwise, this function returns -1, and this object should be + // deleted. + int try_again_or_fail(); void timeout(); void restart_timeout(); int submit_request(); From 9afc017532775642fe4294bf75144b1311050afb Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 3 Mar 2016 23:20:32 +0900 Subject: [PATCH 097/147] src: Don't push if Link header field includes nopush --- src/http2.cc | 199 +++++++++++++++++++++++++++------------------- src/http2_test.cc | 32 ++++++++ 2 files changed, 148 insertions(+), 83 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index 9acf1862..68587216 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -864,6 +864,29 @@ bool check_link_param_empty(const char *first, const char *last, } } // namespace +namespace { +// Returns true if link-param consists of only parmname, and it +// matches string [pat, pat + patlen). +bool check_link_param_without_value(const char *first, const char *last, + const char *pat, size_t patlen) { + if (first + patlen > last) { + return false; + } + + if (first + patlen == last) { + return std::equal(pat, pat + patlen, first, util::CaseCmp()); + } + + switch (*(first + patlen)) { + case ';': + case ',': + return std::equal(pat, pat + patlen, first, util::CaseCmp()); + } + + return false; +} +} // namespace + namespace { std::pair parse_next_link_header_once(const char *first, const char *last) { @@ -900,99 +923,109 @@ parse_next_link_header_once(const char *first, const char *last) { } // we expect link-param - // rel can take several relations using quoted form. - static constexpr char PLP[] = "rel=\""; - static constexpr size_t PLPLEN = sizeof(PLP) - 1; + if (!ign) { + // rel can take several relations using quoted form. + static constexpr char PLP[] = "rel=\""; + static constexpr size_t PLPLEN = sizeof(PLP) - 1; - static constexpr char PLT[] = "preload"; - static constexpr size_t PLTLEN = sizeof(PLT) - 1; - if (first + PLPLEN < last && *(first + PLPLEN - 1) == '"' && - std::equal(PLP, PLP + PLPLEN, first, util::CaseCmp())) { - // we have to search preload in whitespace separated list: - // rel="preload something http://example.org/foo" - first += PLPLEN; - auto start = first; - for (; first != last;) { - if (*first != ' ' && *first != '"') { + static constexpr char PLT[] = "preload"; + static constexpr size_t PLTLEN = sizeof(PLT) - 1; + if (first + PLPLEN < last && *(first + PLPLEN - 1) == '"' && + std::equal(PLP, PLP + PLPLEN, first, util::CaseCmp())) { + // we have to search preload in whitespace separated list: + // rel="preload something http://example.org/foo" + first += PLPLEN; + auto start = first; + for (; first != last;) { + if (*first != ' ' && *first != '"') { + ++first; + continue; + } + + if (start == first) { + return {{{nullptr, nullptr}}, last}; + } + + if (!ok && start + PLTLEN == first && + std::equal(PLT, PLT + PLTLEN, start, util::CaseCmp())) { + ok = true; + } + + if (*first == '"') { + break; + } + first = skip_lws(first, last); + start = first; + } + if (first == last) { + return {{{nullptr, nullptr}}, first}; + } + assert(*first == '"'); + ++first; + if (first == last || *first == ',') { + goto almost_done; + } + if (*first == ';') { ++first; + // parse next link-param continue; } - - if (start == first) { - return {{{nullptr, nullptr}}, last}; + return {{{nullptr, nullptr}}, last}; + } + // we are only interested in rel=preload parameter. Others are + // simply skipped. + static constexpr char PL[] = "rel=preload"; + static constexpr size_t PLLEN = sizeof(PL) - 1; + if (first + PLLEN == last) { + if (std::equal(PL, PL + PLLEN, first, util::CaseCmp())) { + // ok = true; + // this is the end of sequence + return {{{url_first, url_last}}, last}; } - - if (!ok && start + PLTLEN == first && - std::equal(PLT, PLT + PLTLEN, start, util::CaseCmp())) { + } else if (first + PLLEN + 1 <= last) { + switch (*(first + PLLEN)) { + case ',': + if (!std::equal(PL, PL + PLLEN, first, util::CaseCmp())) { + break; + } + // ok = true; + // skip including ',' + first += PLLEN + 1; + return {{{url_first, url_last}}, first}; + case ';': + if (!std::equal(PL, PL + PLLEN, first, util::CaseCmp())) { + break; + } ok = true; + // skip including ';' + first += PLLEN + 1; + // continue parse next link-param + continue; } + } + // we have to reject URI if we have nonempty anchor parameter. + static constexpr char ANCHOR[] = "anchor="; + static constexpr size_t ANCHORLEN = sizeof(ANCHOR) - 1; + if (!ign && !check_link_param_empty(first, last, ANCHOR, ANCHORLEN)) { + ign = true; + } - if (*first == '"') { - break; - } - first = skip_lws(first, last); - start = first; + // reject URI if we have non-empty loadpolicy. This could be + // tightened up to just pick up "next" or "insert". + static constexpr char LOADPOLICY[] = "loadpolicy="; + static constexpr size_t LOADPOLICYLEN = sizeof(LOADPOLICY) - 1; + if (!ign && + !check_link_param_empty(first, last, LOADPOLICY, LOADPOLICYLEN)) { + ign = true; } - if (first == last) { - return {{{nullptr, nullptr}}, first}; - } - assert(*first == '"'); - ++first; - if (first == last || *first == ',') { - goto almost_done; - } - if (*first == ';') { - ++first; - // parse next link-param - continue; - } - return {{{nullptr, nullptr}}, last}; - } - // we are only interested in rel=preload parameter. Others are - // simply skipped. - static constexpr char PL[] = "rel=preload"; - static constexpr size_t PLLEN = sizeof(PL) - 1; - if (first + PLLEN == last) { - if (std::equal(PL, PL + PLLEN, first, util::CaseCmp())) { - // ok = true; - // this is the end of sequence - return {{{url_first, url_last}}, last}; - } - } else if (first + PLLEN + 1 <= last) { - switch (*(first + PLLEN)) { - case ',': - if (!std::equal(PL, PL + PLLEN, first, util::CaseCmp())) { - break; - } - // ok = true; - // skip including ',' - first += PLLEN + 1; - return {{{url_first, url_last}}, first}; - case ';': - if (!std::equal(PL, PL + PLLEN, first, util::CaseCmp())) { - break; - } - ok = true; - // skip including ';' - first += PLLEN + 1; - // continue parse next link-param - continue; - } - } - // we have to reject URI if we have nonempty anchor parameter. - static constexpr char ANCHOR[] = "anchor="; - static constexpr size_t ANCHORLEN = sizeof(ANCHOR) - 1; - if (!ign && !check_link_param_empty(first, last, ANCHOR, ANCHORLEN)) { - ign = true; - } - // reject URI if we have non-empty loadpolicy. This could be - // tightened up to just pick up "next" or "insert". - static constexpr char LOADPOLICY[] = "loadpolicy="; - static constexpr size_t LOADPOLICYLEN = sizeof(LOADPOLICY) - 1; - if (!ign && - !check_link_param_empty(first, last, LOADPOLICY, LOADPOLICYLEN)) { - ign = true; + // reject URI if we have nopush attribute. + static constexpr char NOPUSH[] = "nopush"; + static constexpr size_t NOPUSHLEN = str_size(NOPUSH); + if (!ign && + check_link_param_without_value(first, last, NOPUSH, NOPUSHLEN)) { + ign = true; + } } auto param_first = first; diff --git a/src/http2_test.cc b/src/http2_test.cc index 34f96461..46653686 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -625,6 +625,38 @@ void test_http2_parse_link_header(void) { CU_ASSERT(std::make_pair(&s[36], &s[39]) == res[0].uri); CU_ASSERT(std::make_pair(&s[42 + 14], &s[42 + 17]) == res[1].uri); } + { + // nopush at the end of input + constexpr char s[] = "; rel=preload; nopush"; + auto res = http2::parse_link_header(s, str_size(s)); + CU_ASSERT(0 == res.size()); + } + { + // nopush followed by ';' + constexpr char s[] = "; rel=preload; nopush; foo"; + auto res = http2::parse_link_header(s, str_size(s)); + CU_ASSERT(0 == res.size()); + } + { + // nopush followed by ',' + constexpr char s[] = "; nopush; rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); + CU_ASSERT(0 == res.size()); + } + { + // string whose prefix is nopush + constexpr char s[] = "; nopushyes; rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); + CU_ASSERT(1 == res.size()); + CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + } + { + // rel=preload twice + constexpr char s[] = "; rel=preload; rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); + CU_ASSERT(1 == res.size()); + CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + } } void test_http2_path_join(void) { From aaf017731875943f206e26ec83b570b1eee6bfd9 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 3 Mar 2016 23:23:51 +0900 Subject: [PATCH 098/147] src: Use str_size --- src/http2.cc | 10 +-- src/http2_test.cc | 211 +++++++++++++++++++++++----------------------- 2 files changed, 111 insertions(+), 110 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index 68587216..8fcbd43c 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -926,10 +926,10 @@ parse_next_link_header_once(const char *first, const char *last) { if (!ign) { // rel can take several relations using quoted form. static constexpr char PLP[] = "rel=\""; - static constexpr size_t PLPLEN = sizeof(PLP) - 1; + static constexpr size_t PLPLEN = str_size(PLP); static constexpr char PLT[] = "preload"; - static constexpr size_t PLTLEN = sizeof(PLT) - 1; + static constexpr size_t PLTLEN = str_size(PLT); if (first + PLPLEN < last && *(first + PLPLEN - 1) == '"' && std::equal(PLP, PLP + PLPLEN, first, util::CaseCmp())) { // we have to search preload in whitespace separated list: @@ -975,7 +975,7 @@ parse_next_link_header_once(const char *first, const char *last) { // we are only interested in rel=preload parameter. Others are // simply skipped. static constexpr char PL[] = "rel=preload"; - static constexpr size_t PLLEN = sizeof(PL) - 1; + static constexpr size_t PLLEN = str_size(PL); if (first + PLLEN == last) { if (std::equal(PL, PL + PLLEN, first, util::CaseCmp())) { // ok = true; @@ -1005,7 +1005,7 @@ parse_next_link_header_once(const char *first, const char *last) { } // we have to reject URI if we have nonempty anchor parameter. static constexpr char ANCHOR[] = "anchor="; - static constexpr size_t ANCHORLEN = sizeof(ANCHOR) - 1; + static constexpr size_t ANCHORLEN = str_size(ANCHOR); if (!ign && !check_link_param_empty(first, last, ANCHOR, ANCHORLEN)) { ign = true; } @@ -1013,7 +1013,7 @@ parse_next_link_header_once(const char *first, const char *last) { // reject URI if we have non-empty loadpolicy. This could be // tightened up to just pick up "next" or "insert". static constexpr char LOADPOLICY[] = "loadpolicy="; - static constexpr size_t LOADPOLICYLEN = sizeof(LOADPOLICY) - 1; + static constexpr size_t LOADPOLICYLEN = str_size(LOADPOLICY); if (!ign && !check_link_param_empty(first, last, LOADPOLICY, LOADPOLICYLEN)) { ign = true; diff --git a/src/http2_test.cc b/src/http2_test.cc index 46653686..88bf89e3 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -274,353 +274,354 @@ void test_http2_lookup_token(void) { void test_http2_parse_link_header(void) { { // only URI appears; we don't extract URI unless it bears rel=preload - const char s[] = ""; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = ""; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // URI url should be extracted - const char s[] = "; rel=preload"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = "; rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); } { // With extra link-param. URI url should be extracted - const char s[] = "; rel=preload; as=file"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = "; rel=preload; as=file"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); } { // With extra link-param. URI url should be extracted - const char s[] = "; as=file; rel=preload"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = "; as=file; rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); } { // With extra link-param and quote-string. URI url should be // extracted - const char s[] = R"(; rel=preload; title="foo,bar")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel=preload; title="foo,bar")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); } { // With extra link-param and quote-string. URI url should be // extracted - const char s[] = R"(; title="foo,bar"; rel=preload)"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; title="foo,bar"; rel=preload)"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); } { // ',' after quote-string - const char s[] = R"(; title="foo,bar", ; rel=preload)"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; title="foo,bar", ; rel=preload)"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[25], &s[28]) == res[0].uri); } { // Only first URI should be extracted. - const char s[] = "; rel=preload, "; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = "; rel=preload, "; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); } { // Both have rel=preload, so both urls should be extracted - const char s[] = "; rel=preload, ; rel=preload"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = "; rel=preload, ; rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(2 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); CU_ASSERT(std::make_pair(&s[21], &s[24]) == res[1].uri); } { // Second URI uri should be extracted. - const char s[] = ", ;rel=preload"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = ", ;rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[8], &s[11]) == res[0].uri); } { // Error if input ends with ';' - const char s[] = ";rel=preload;"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = ";rel=preload;"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // Error if link header ends with ';' - const char s[] = ";rel=preload;, "; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = ";rel=preload;, "; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // OK if input ends with ',' - const char s[] = ";rel=preload,"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = ";rel=preload,"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); } { // Multiple repeated ','s between fields is OK - const char s[] = ",,,;rel=preload"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = ",,,;rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[9], &s[12]) == res[0].uri); } { // Error if url is not enclosed by <> - const char s[] = "url>;rel=preload"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = "url>;rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // Error if url is not enclosed by <> - const char s[] = "' is not followed by ';' - const char s[] = " rel=preload"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = " rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // Starting with whitespace is no problem. - const char s[] = " ; rel=preload"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = " ; rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[3], &s[6]) == res[0].uri); } { // preload is a prefix of bogus rel parameter value - const char s[] = "; rel=preloadx"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = "; rel=preloadx"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // preload in relation-types list - const char s[] = R"(; rel="preload")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="preload")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); } { // preload in relation-types list followed by another parameter - const char s[] = R"(; rel="preload foo")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="preload foo")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); } { // preload in relation-types list following another parameter - const char s[] = R"(; rel="foo preload")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="foo preload")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); } { // preload in relation-types list between other parameters - const char s[] = R"(; rel="foo preload bar")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="foo preload bar")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); } { // preload in relation-types list between other parameters - const char s[] = R"(; rel="foo preload bar")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="foo preload bar")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); } { // no preload in relation-types list - const char s[] = R"(; rel="foo")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="foo")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // no preload in relation-types list, multiple unrelated elements. - const char s[] = R"(; rel="foo bar")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="foo bar")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // preload in relation-types list, followed by another link-value. - const char s[] = R"(; rel="preload", )"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="preload", )"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); } { // preload in relation-types list, following another link-value. - const char s[] = R"(, ; rel="preload")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(, ; rel="preload")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[8], &s[11]) == res[0].uri); } { // preload in relation-types list, followed by another link-param. - const char s[] = R"(; rel="preload"; as="font")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="preload"; as="font")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); } { // preload in relation-types list, followed by character other // than ';' or ',' - const char s[] = R"(; rel="preload".)"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="preload".)"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // preload in relation-types list, followed by ';' but it // terminates input - const char s[] = R"(; rel="preload";)"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="preload";)"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // preload in relation-types list, followed by ',' but it // terminates input - const char s[] = R"(; rel="preload",)"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="preload",)"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); } { // preload in relation-types list but there is preceding white // space. - const char s[] = R"(; rel=" preload")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel=" preload")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // preload in relation-types list but there is trailing white // space. - const char s[] = R"(; rel="preload ")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="preload ")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // backslash escaped characters in quoted-string - const char s[] = R"(; rel=preload; title="foo\"baz\"bar")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel=preload; title="foo\"baz\"bar")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); } { // anchor="" is acceptable - const char s[] = R"(; rel=preload; anchor="")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel=preload; anchor="")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); } { // With anchor="#foo", url should be ignored - const char s[] = R"(; rel=preload; anchor="#foo")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel=preload; anchor="#foo")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // With anchor=f, url should be ignored - const char s[] = "; rel=preload; anchor=f"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = "; rel=preload; anchor=f"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // First url is ignored With anchor="#foo", but url should be // accepted. - const char s[] = R"(; rel=preload; anchor="#foo", ; rel=preload)"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = + R"(; rel=preload; anchor="#foo", ; rel=preload)"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[36], &s[39]) == res[0].uri); } { // With loadpolicy="next", url should be ignored - const char s[] = R"(; rel=preload; loadpolicy="next")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel=preload; loadpolicy="next")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // url should be picked up if empty loadpolicy is specified - const char s[] = R"(; rel=preload; loadpolicy="")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel=preload; loadpolicy="")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); } { // case-insensitive match - const char s[] = R"(; rel=preload; ANCHOR="#foo", ; )" - R"(REL=PRELOAD, ; REL="foo PRELOAD bar")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel=preload; ANCHOR="#foo", ; )" + R"(REL=PRELOAD, ; REL="foo PRELOAD bar")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(2 == res.size()); CU_ASSERT(std::make_pair(&s[36], &s[39]) == res[0].uri); CU_ASSERT(std::make_pair(&s[42 + 14], &s[42 + 17]) == res[1].uri); From 1e8bea15e5db8f826c4c31db55e40598044e429f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 3 Mar 2016 23:31:44 +0900 Subject: [PATCH 099/147] src: Use StringRef inside LinkHeader --- src/http2.cc | 37 +++++++++++++++++++------------------ src/http2.h | 4 ++-- src/shrpx_http2_upstream.cc | 6 ++---- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index 8fcbd43c..3bad70cc 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -892,26 +892,26 @@ std::pair parse_next_link_header_once(const char *first, const char *last) { first = skip_to_next_field(first, last); if (first == last || *first != '<') { - return {{{nullptr, nullptr}}, last}; + return {{StringRef{}}, last}; } auto url_first = ++first; first = std::find(first, last, '>'); if (first == last) { - return {{{nullptr, nullptr}}, first}; + return {{StringRef{}}, first}; } auto url_last = first++; if (first == last) { - return {{{nullptr, nullptr}}, first}; + return {{StringRef{}}, first}; } // we expect ';' or ',' here switch (*first) { case ',': - return {{{nullptr, nullptr}}, ++first}; + return {{StringRef{}}, ++first}; case ';': ++first; break; default: - return {{{nullptr, nullptr}}, last}; + return {{StringRef{}}, last}; } auto ok = false; @@ -919,7 +919,7 @@ parse_next_link_header_once(const char *first, const char *last) { for (;;) { first = skip_lws(first, last); if (first == last) { - return {{{nullptr, nullptr}}, first}; + return {{StringRef{}}, first}; } // we expect link-param @@ -943,7 +943,7 @@ parse_next_link_header_once(const char *first, const char *last) { } if (start == first) { - return {{{nullptr, nullptr}}, last}; + return {{StringRef{}}, last}; } if (!ok && start + PLTLEN == first && @@ -958,7 +958,7 @@ parse_next_link_header_once(const char *first, const char *last) { start = first; } if (first == last) { - return {{{nullptr, nullptr}}, first}; + return {{StringRef{}}, first}; } assert(*first == '"'); ++first; @@ -970,7 +970,7 @@ parse_next_link_header_once(const char *first, const char *last) { // parse next link-param continue; } - return {{{nullptr, nullptr}}, last}; + return {{StringRef{}}, last}; } // we are only interested in rel=preload parameter. Others are // simply skipped. @@ -1045,11 +1045,11 @@ parse_next_link_header_once(const char *first, const char *last) { if (*first == '=' || *first == ';' || *first == ',') { break; } - return {{{nullptr, nullptr}}, last}; + return {{StringRef{}}, last}; } if (param_first == first) { // empty parmname - return {{{nullptr, nullptr}}, last}; + return {{StringRef{}}, last}; } // link-param without value is acceptable (see link-extension) if // it is not followed by '=' @@ -1066,13 +1066,13 @@ parse_next_link_header_once(const char *first, const char *last) { ++first; if (first == last) { // empty value is not acceptable - return {{{nullptr, nullptr}}, first}; + return {{StringRef{}}, first}; } if (*first == '"') { // quoted-string first = skip_to_right_dquote(first + 1, last); if (first == last) { - return {{{nullptr, nullptr}}, first}; + return {{StringRef{}}, first}; } ++first; if (first == last || *first == ',') { @@ -1083,12 +1083,12 @@ parse_next_link_header_once(const char *first, const char *last) { // parse next link-param continue; } - return {{{nullptr, nullptr}}, last}; + return {{StringRef{}}, last}; } // not quoted-string, skip to next ',' or ';' if (*first == ',' || *first == ';') { // empty value - return {{{nullptr, nullptr}}, last}; + return {{StringRef{}}, last}; } for (; first != last; ++first) { if (*first == ',' || *first == ';') { @@ -1112,7 +1112,7 @@ almost_done: if (ok && !ign) { return {{{url_first, url_last}}, first}; } - return {{{nullptr, nullptr}}, first}; + return {{StringRef{}}, first}; } } // namespace @@ -1123,8 +1123,9 @@ std::vector parse_link_header(const char *src, size_t len) { for (; first != last;) { auto rv = parse_next_link_header_once(first, last); first = rv.second; - if (rv.first.uri.first != nullptr && rv.first.uri.second != nullptr) { - res.push_back(rv.first); + auto &link = rv.first; + if (!link.uri.empty()) { + res.push_back(link); } } return res; diff --git a/src/http2.h b/src/http2.h index bb879fdd..2bc93f03 100644 --- a/src/http2.h +++ b/src/http2.h @@ -287,8 +287,8 @@ Headers::value_type *get_header(const HeaderIndex &hdidx, int32_t token, Headers &nva); struct LinkHeader { - // The region of URI is [uri.first, uri.second). - std::pair uri; + // The region of URI. This might not be NULL-terminated. + StringRef uri; }; // Returns next URI-reference in Link header field value |src| of diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 5bf1c5c4..37edafcc 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1752,14 +1752,12 @@ int Http2Upstream::prepare_push_promise(Downstream *downstream) { for (auto &link : http2::parse_link_header(kv.value.c_str(), kv.value.size())) { - auto uri = link.uri.first; - auto len = link.uri.second - link.uri.first; - const std::string *scheme_ptr, *authority_ptr; std::string scheme, authority, path; rv = http2::construct_push_component(scheme, authority, path, base, - baselen, uri, len); + baselen, link.uri.c_str(), + link.uri.size()); if (rv != 0) { continue; } From acbf38fd3c4febb75a9f86813d8a09405222b04e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 4 Mar 2016 00:26:59 +0900 Subject: [PATCH 100/147] src: Refactor using StringRef, simplify function parameters --- src/asio_server_serve_mux.cc | 4 +- src/http2.cc | 67 ++++---- src/http2.h | 28 ++-- src/http2_test.cc | 315 ++++++++++++++++------------------- src/shrpx_http2_upstream.cc | 9 +- 5 files changed, 191 insertions(+), 232 deletions(-) diff --git a/src/asio_server_serve_mux.cc b/src/asio_server_serve_mux.cc index b635ee4d..7765adf0 100644 --- a/src/asio_server_serve_mux.cc +++ b/src/asio_server_serve_mux.cc @@ -77,8 +77,8 @@ bool serve_mux::handle(std::string pattern, request_cb cb) { request_cb serve_mux::handler(request_impl &req) const { auto &path = req.uri().path; if (req.method() != "CONNECT") { - auto clean_path = ::nghttp2::http2::path_join( - nullptr, 0, nullptr, 0, path.c_str(), path.size(), nullptr, 0); + auto clean_path = ::nghttp2::http2::path_join(StringRef{}, StringRef{}, + StringRef{path}, StringRef{}); if (clean_path != path) { auto new_uri = util::percent_encode_path(clean_path); auto &uref = req.uri(); diff --git a/src/http2.cc b/src/http2.cc index 3bad70cc..6bc898b9 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -1185,39 +1185,37 @@ void eat_dir(std::string &path) { } } // namespace -std::string path_join(const char *base_path, size_t base_pathlen, - const char *base_query, size_t base_querylen, - const char *rel_path, size_t rel_pathlen, - const char *rel_query, size_t rel_querylen) { +std::string path_join(const StringRef &base_path, const StringRef &base_query, + const StringRef &rel_path, const StringRef &rel_query) { std::string res; - if (rel_pathlen == 0) { - if (base_pathlen == 0) { + if (rel_path.empty()) { + if (base_path.empty()) { res = "/"; } else { - res.assign(base_path, base_pathlen); + res.assign(std::begin(base_path), std::end(base_path)); } - if (rel_querylen == 0) { - if (base_querylen) { + if (rel_query.empty()) { + if (!base_query.empty()) { res += '?'; - res.append(base_query, base_querylen); + res.append(std::begin(base_query), std::end(base_query)); } return res; } res += '?'; - res.append(rel_query, rel_querylen); + res.append(std::begin(rel_query), std::end(rel_query)); return res; } - auto first = rel_path; - auto last = rel_path + rel_pathlen; + auto first = std::begin(rel_path); + auto last = std::end(rel_path); if (rel_path[0] == '/') { res = "/"; ++first; - } else if (base_pathlen == 0) { + } else if (base_path.empty()) { res = "/"; } else { - res.assign(base_path, base_pathlen); + res.assign(std::begin(base_path), std::end(base_path)); } for (; first != last;) { @@ -1254,9 +1252,9 @@ std::string path_join(const char *base_path, size_t base_pathlen, for (; first != last && *first == '/'; ++first) ; } - if (rel_querylen) { + if (!rel_query.empty()) { res += '?'; - res.append(rel_query, rel_querylen); + res.append(std::begin(rel_query), std::end(rel_query)); } return res; } @@ -1480,15 +1478,14 @@ int get_pure_path_component(const char **base, size_t *baselen, } int construct_push_component(std::string &scheme, std::string &authority, - std::string &path, const char *base, - size_t baselen, const char *uri, size_t len) { + std::string &path, const StringRef &base, + const StringRef &uri) { int rv; - const char *rel, *relq = nullptr; - size_t rellen, relqlen = 0; + StringRef rel, relq; http_parser_url u{}; - rv = http_parser_parse_url(uri, len, 0, &u); + rv = http_parser_parse_url(uri.c_str(), uri.size(), 0, &u); if (rv != 0) { if (uri[0] == '/') { @@ -1496,22 +1493,20 @@ int construct_push_component(std::string &scheme, std::string &authority, } // treat link_url as relative URI. - auto end = std::find(uri, uri + len, '#'); - auto q = std::find(uri, end, '?'); + auto end = std::find(std::begin(uri), std::end(uri), '#'); + auto q = std::find(std::begin(uri), end, '?'); - rel = uri; - rellen = q - uri; + rel = StringRef{std::begin(uri), q}; if (q != end) { - relq = q + 1; - relqlen = end - relq; + relq = StringRef{q + 1, std::end(uri)}; } } else { if (u.field_set & (1 << UF_SCHEMA)) { - http2::copy_url_component(scheme, &u, UF_SCHEMA, uri); + http2::copy_url_component(scheme, &u, UF_SCHEMA, uri.c_str()); } if (u.field_set & (1 << UF_HOST)) { - http2::copy_url_component(authority, &u, UF_HOST, uri); + http2::copy_url_component(authority, &u, UF_HOST, uri.c_str()); if (u.field_set & (1 << UF_PORT)) { authority += ':'; authority += util::utos(u.port); @@ -1520,22 +1515,18 @@ int construct_push_component(std::string &scheme, std::string &authority, if (u.field_set & (1 << UF_PATH)) { auto &f = u.field_data[UF_PATH]; - rel = uri + f.off; - rellen = f.len; + rel = StringRef{uri.c_str() + f.off, f.len}; } else { - rel = "/"; - rellen = 1; + rel = StringRef::from_lit("/"); } if (u.field_set & (1 << UF_QUERY)) { auto &f = u.field_data[UF_QUERY]; - relq = uri + f.off; - relqlen = f.len; + relq = StringRef{uri.c_str() + f.off, f.len}; } } - path = - http2::path_join(base, baselen, nullptr, 0, rel, rellen, relq, relqlen); + path = http2::path_join(base, StringRef{}, rel, relq); return 0; } diff --git a/src/http2.h b/src/http2.h index 2bc93f03..23707b45 100644 --- a/src/http2.h +++ b/src/http2.h @@ -297,16 +297,13 @@ struct LinkHeader { // is ignored during parsing. std::vector parse_link_header(const char *src, size_t len); -// Constructs path by combining base path |base_path| of length -// |base_pathlen| with another path |rel_path| of length -// |rel_pathlen|. The base path and another path can have optional +// Constructs path by combining base path |base_path| with another +// path |rel_path|. The base path and another path can have optional // query component. This function assumes |base_path| is normalized. // In other words, it does not contain ".." or "." path components // and starts with "/" if it is not empty. -std::string path_join(const char *base_path, size_t base_pathlen, - const char *base_query, size_t base_querylen, - const char *rel_path, size_t rel_pathlen, - const char *rel_query, size_t rel_querylen); +std::string path_join(const StringRef &base, const StringRef &base_query, + const StringRef &rel_path, const StringRef &rel_query); // true if response has body, taking into account the request method // and status code. @@ -359,8 +356,7 @@ std::string normalize_path(InputIt first, InputIt last) { } result.append(first, last); } - return path_join(nullptr, 0, nullptr, 0, result.c_str(), result.size(), - nullptr, 0); + return path_join(StringRef{}, StringRef{}, StringRef{result}, StringRef{}); } template @@ -384,14 +380,14 @@ std::string rewrite_clean_path(InputIt first, InputIt last) { int get_pure_path_component(const char **base, size_t *baselen, const std::string &uri); -// Deduces scheme, authority and path from given |uri| of length -// |len|, and stores them in |scheme|, |authority|, and |path| -// respectively. If |uri| is relative path, path resolution is taken -// palce using path given in |base| of length |baselen|. This -// function returns 0 if it succeeds, or -1. +// Deduces scheme, authority and path from given |uri|, and stores +// them in |scheme|, |authority|, and |path| respectively. If |uri| +// is relative path, path resolution takes place using path given in +// |base| of length |baselen|. This function returns 0 if it +// succeeds, or -1. int construct_push_component(std::string &scheme, std::string &authority, - std::string &path, const char *base, - size_t baselen, const char *uri, size_t len); + std::string &path, const StringRef &base, + const StringRef &uri); } // namespace http2 diff --git a/src/http2_test.cc b/src/http2_test.cc index 88bf89e3..e3b19693 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -283,21 +283,21 @@ void test_http2_parse_link_header(void) { constexpr char s[] = "; rel=preload"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // With extra link-param. URI url should be extracted constexpr char s[] = "; rel=preload; as=file"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // With extra link-param. URI url should be extracted constexpr char s[] = "; as=file; rel=preload"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // With extra link-param and quote-string. URI url should be @@ -305,7 +305,7 @@ void test_http2_parse_link_header(void) { constexpr char s[] = R"(; rel=preload; title="foo,bar")"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // With extra link-param and quote-string. URI url should be @@ -313,36 +313,37 @@ void test_http2_parse_link_header(void) { constexpr char s[] = R"(; title="foo,bar"; rel=preload)"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // ',' after quote-string - constexpr char s[] = R"(; title="foo,bar", ; rel=preload)"; + constexpr char s[] = R"(; title="foo,bar", ; rel=preload)"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[25], &s[28]) == res[0].uri); + CU_ASSERT("url2" == res[0].uri); + CU_ASSERT(&s[25] == &res[0].uri[0]); } { // Only first URI should be extracted. - constexpr char s[] = "; rel=preload, "; + constexpr char s[] = "; rel=preload, "; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // Both have rel=preload, so both urls should be extracted - constexpr char s[] = "; rel=preload, ; rel=preload"; + constexpr char s[] = "; rel=preload, ; rel=preload"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(2 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); - CU_ASSERT(std::make_pair(&s[21], &s[24]) == res[1].uri); + CU_ASSERT("url" == res[0].uri); + CU_ASSERT("url2" == res[1].uri); } { // Second URI uri should be extracted. - constexpr char s[] = ", ;rel=preload"; + constexpr char s[] = ", ;rel=preload"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[8], &s[11]) == res[0].uri); + CU_ASSERT("url2" == res[0].uri); } { // Error if input ends with ';' @@ -361,14 +362,14 @@ void test_http2_parse_link_header(void) { constexpr char s[] = ";rel=preload,"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // Multiple repeated ','s between fields is OK - constexpr char s[] = ",,,;rel=preload"; + constexpr char s[] = ",,,;rel=preload"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[9], &s[12]) == res[0].uri); + CU_ASSERT("url2" == res[0].uri); } { // Error if url is not enclosed by <> @@ -408,25 +409,25 @@ void test_http2_parse_link_header(void) { } { // Without whitespaces - constexpr char s[] = ";as=file;rel=preload,;rel=preload"; + constexpr char s[] = ";as=file;rel=preload,;rel=preload"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(2 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); - CU_ASSERT(std::make_pair(&s[27], &s[30]) == res[1].uri); + CU_ASSERT("url" == res[0].uri); + CU_ASSERT("url2" == res[1].uri); } { // link-extension may have no value constexpr char s[] = "; as; rel=preload"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // ext-name-star constexpr char s[] = "; foo*=bar; rel=preload"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // '*' is not allowed expect for trailing one @@ -457,7 +458,7 @@ void test_http2_parse_link_header(void) { constexpr char s[] = " ; rel=preload"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[3], &s[6]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // preload is a prefix of bogus rel parameter value @@ -470,35 +471,35 @@ void test_http2_parse_link_header(void) { constexpr char s[] = R"(; rel="preload")"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // preload in relation-types list followed by another parameter constexpr char s[] = R"(; rel="preload foo")"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // preload in relation-types list following another parameter constexpr char s[] = R"(; rel="foo preload")"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // preload in relation-types list between other parameters constexpr char s[] = R"(; rel="foo preload bar")"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // preload in relation-types list between other parameters constexpr char s[] = R"(; rel="foo preload bar")"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // no preload in relation-types list @@ -514,24 +515,24 @@ void test_http2_parse_link_header(void) { } { // preload in relation-types list, followed by another link-value. - constexpr char s[] = R"(; rel="preload", )"; + constexpr char s[] = R"(; rel="preload", )"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // preload in relation-types list, following another link-value. - constexpr char s[] = R"(, ; rel="preload")"; + constexpr char s[] = R"(, ; rel="preload")"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[8], &s[11]) == res[0].uri); + CU_ASSERT("url2" == res[0].uri); } { // preload in relation-types list, followed by another link-param. constexpr char s[] = R"(; rel="preload"; as="font")"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // preload in relation-types list, followed by character other @@ -553,7 +554,7 @@ void test_http2_parse_link_header(void) { constexpr char s[] = R"(; rel="preload",)"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // preload in relation-types list but there is preceding white @@ -574,14 +575,14 @@ void test_http2_parse_link_header(void) { constexpr char s[] = R"(; rel=preload; title="foo\"baz\"bar")"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // anchor="" is acceptable constexpr char s[] = R"(; rel=preload; anchor="")"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // With anchor="#foo", url should be ignored @@ -599,10 +600,10 @@ void test_http2_parse_link_header(void) { // First url is ignored With anchor="#foo", but url should be // accepted. constexpr char s[] = - R"(; rel=preload; anchor="#foo", ; rel=preload)"; + R"(; rel=preload; anchor="#foo", ; rel=preload)"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[36], &s[39]) == res[0].uri); + CU_ASSERT("url2" == res[0].uri); } { // With loadpolicy="next", url should be ignored @@ -615,16 +616,16 @@ void test_http2_parse_link_header(void) { constexpr char s[] = R"(; rel=preload; loadpolicy="")"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // case-insensitive match - constexpr char s[] = R"(; rel=preload; ANCHOR="#foo", ; )" - R"(REL=PRELOAD, ; REL="foo PRELOAD bar")"; + constexpr char s[] = R"(; rel=preload; ANCHOR="#foo", ; )" + R"(REL=PRELOAD, ; REL="foo PRELOAD bar")"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(2 == res.size()); - CU_ASSERT(std::make_pair(&s[36], &s[39]) == res[0].uri); - CU_ASSERT(std::make_pair(&s[42 + 14], &s[42 + 17]) == res[1].uri); + CU_ASSERT("url2" == res[0].uri); + CU_ASSERT("url3" == res[1].uri); } { // nopush at the end of input @@ -649,186 +650,166 @@ void test_http2_parse_link_header(void) { constexpr char s[] = "; nopushyes; rel=preload"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // rel=preload twice constexpr char s[] = "; rel=preload; rel=preload"; auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } } void test_http2_path_join(void) { { - const char base[] = "/"; - const char rel[] = "/"; - CU_ASSERT("/" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/"); + auto rel = StringRef::from_lit("/"); + CU_ASSERT("/" == http2::path_join(base, StringRef{}, rel, StringRef{})); } { - const char base[] = "/"; - const char rel[] = "/alpha"; - CU_ASSERT("/alpha" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, - rel, sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/"); + auto rel = StringRef::from_lit("/alpha"); + CU_ASSERT("/alpha" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // rel ends with trailing '/' - const char base[] = "/"; - const char rel[] = "/alpha/"; - CU_ASSERT("/alpha/" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, - rel, sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/"); + auto rel = StringRef::from_lit("/alpha/"); + CU_ASSERT("/alpha/" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // rel contains multiple components - const char base[] = "/"; - const char rel[] = "/alpha/bravo"; - CU_ASSERT("/alpha/bravo" == http2::path_join(base, sizeof(base) - 1, - nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/"); + auto rel = StringRef::from_lit("/alpha/bravo"); + CU_ASSERT("/alpha/bravo" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // rel is relative - const char base[] = "/"; - const char rel[] = "alpha/bravo"; - CU_ASSERT("/alpha/bravo" == http2::path_join(base, sizeof(base) - 1, - nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/"); + auto rel = StringRef::from_lit("alpha/bravo"); + CU_ASSERT("/alpha/bravo" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // rel is relative and base ends without /, which means it refers // to file. - const char base[] = "/alpha"; - const char rel[] = "bravo/charlie"; + auto base = StringRef::from_lit("/alpha"); + auto rel = StringRef::from_lit("bravo/charlie"); CU_ASSERT("/bravo/charlie" == - http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // rel contains repeated '/'s - const char base[] = "/"; - const char rel[] = "/alpha/////bravo/////"; - CU_ASSERT("/alpha/bravo/" == http2::path_join(base, sizeof(base) - 1, - nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/"); + auto rel = StringRef::from_lit("/alpha/////bravo/////"); + CU_ASSERT("/alpha/bravo/" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // base ends with '/', so '..' eats 'bravo' - const char base[] = "/alpha/bravo/"; - const char rel[] = "../charlie/delta"; + auto base = StringRef::from_lit("/alpha/bravo/"); + auto rel = StringRef::from_lit("../charlie/delta"); CU_ASSERT("/alpha/charlie/delta" == - http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // base does not end with '/', so '..' eats 'alpha/bravo' - const char base[] = "/alpha/bravo"; - const char rel[] = "../charlie"; - CU_ASSERT("/charlie" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, - rel, sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/alpha/bravo"); + auto rel = StringRef::from_lit("../charlie"); + CU_ASSERT("/charlie" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // 'charlie' is eaten by following '..' - const char base[] = "/alpha/bravo/"; - const char rel[] = "../charlie/../delta"; - CU_ASSERT("/alpha/delta" == http2::path_join(base, sizeof(base) - 1, - nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/alpha/bravo/"); + auto rel = StringRef::from_lit("../charlie/../delta"); + CU_ASSERT("/alpha/delta" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // excessive '..' results in '/' - const char base[] = "/alpha/bravo/"; - const char rel[] = "../../../"; - CU_ASSERT("/" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/alpha/bravo/"); + auto rel = StringRef::from_lit("../../../"); + CU_ASSERT("/" == http2::path_join(base, StringRef{}, rel, StringRef{})); } { // excessive '..' and path component - const char base[] = "/alpha/bravo/"; - const char rel[] = "../../../charlie"; - CU_ASSERT("/charlie" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, - rel, sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/alpha/bravo/"); + auto rel = StringRef::from_lit("../../../charlie"); + CU_ASSERT("/charlie" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // rel ends with '..' - const char base[] = "/alpha/bravo/"; - const char rel[] = "charlie/.."; - CU_ASSERT("/alpha/bravo/" == http2::path_join(base, sizeof(base) - 1, - nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/alpha/bravo/"); + auto rel = StringRef::from_lit("charlie/.."); + CU_ASSERT("/alpha/bravo/" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // base empty and rel contains '..' - const char base[] = ""; - const char rel[] = "charlie/.."; - CU_ASSERT("/" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef{}; + auto rel = StringRef::from_lit("charlie/.."); + CU_ASSERT("/" == http2::path_join(base, StringRef{}, rel, StringRef{})); } { // '.' is ignored - const char base[] = "/"; - const char rel[] = "charlie/././././delta"; + auto base = StringRef::from_lit("/"); + auto rel = StringRef::from_lit("charlie/././././delta"); CU_ASSERT("/charlie/delta" == - http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // trailing '.' is ignored - const char base[] = "/"; - const char rel[] = "charlie/."; - CU_ASSERT("/charlie/" == http2::path_join(base, sizeof(base) - 1, nullptr, - 0, rel, sizeof(rel) - 1, nullptr, - 0)); + auto base = StringRef::from_lit("/"); + auto rel = StringRef::from_lit("charlie/."); + CU_ASSERT("/charlie/" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // query - const char base[] = "/"; - const char rel[] = "/"; - const char relq[] = "q"; - CU_ASSERT("/?q" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, - sizeof(rel) - 1, relq, - sizeof(relq) - 1)); + auto base = StringRef::from_lit("/"); + auto rel = StringRef::from_lit("/"); + auto relq = StringRef::from_lit("q"); + CU_ASSERT("/?q" == http2::path_join(base, StringRef{}, rel, relq)); } { // empty rel and query - const char base[] = "/alpha"; - const char rel[] = ""; - const char relq[] = "q"; - CU_ASSERT("/alpha?q" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, - rel, sizeof(rel) - 1, relq, - sizeof(relq) - 1)); + auto base = StringRef::from_lit("/alpha"); + auto rel = StringRef{}; + auto relq = StringRef::from_lit("q"); + CU_ASSERT("/alpha?q" == http2::path_join(base, StringRef{}, rel, relq)); } { // both rel and query are empty - const char base[] = "/alpha"; - const char baseq[] = "r"; - const char rel[] = ""; - const char relq[] = ""; - CU_ASSERT("/alpha?r" == - http2::path_join(base, sizeof(base) - 1, baseq, sizeof(baseq) - 1, - rel, sizeof(rel) - 1, relq, sizeof(relq) - 1)); + auto base = StringRef::from_lit("/alpha"); + auto baseq = StringRef::from_lit("r"); + auto rel = StringRef{}; + auto relq = StringRef{}; + CU_ASSERT("/alpha?r" == http2::path_join(base, baseq, rel, relq)); } { // empty base - const char base[] = ""; - const char rel[] = "/alpha"; - CU_ASSERT("/alpha" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, - rel, sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef{}; + auto rel = StringRef::from_lit("/alpha"); + CU_ASSERT("/alpha" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // everything is empty - CU_ASSERT("/" == - http2::path_join(nullptr, 0, nullptr, 0, nullptr, 0, nullptr, 0)); + CU_ASSERT("/" == http2::path_join(StringRef{}, StringRef{}, StringRef{}, + StringRef{})); } { // only baseq is not empty - const char base[] = ""; - const char baseq[] = "r"; - const char rel[] = ""; - CU_ASSERT("/?r" == http2::path_join(base, sizeof(base) - 1, baseq, - sizeof(baseq) - 1, rel, sizeof(rel) - 1, - nullptr, 0)); + auto base = StringRef{}; + auto baseq = StringRef::from_lit("r"); + auto rel = StringRef{}; + CU_ASSERT("/?r" == http2::path_join(base, baseq, rel, StringRef{})); } } @@ -914,19 +895,15 @@ void test_http2_get_pure_path_component(void) { } void test_http2_construct_push_component(void) { - const char *base; - size_t baselen; - std::string uri; + StringRef base, uri; std::string scheme, authority, path; - base = "/b/"; - baselen = 3; + base = StringRef::from_lit("/b/"); - uri = "https://example.org/foo"; + uri = StringRef::from_lit("https://example.org/foo"); - CU_ASSERT(0 == http2::construct_push_component(scheme, authority, path, base, - baselen, uri.c_str(), - uri.size())); + CU_ASSERT( + 0 == http2::construct_push_component(scheme, authority, path, base, uri)); CU_ASSERT("https" == scheme); CU_ASSERT("example.org" == authority); CU_ASSERT("/foo" == path); @@ -935,11 +912,10 @@ void test_http2_construct_push_component(void) { authority.clear(); path.clear(); - uri = "/foo/bar?q=a"; + uri = StringRef::from_lit("/foo/bar?q=a"); - CU_ASSERT(0 == http2::construct_push_component(scheme, authority, path, base, - baselen, uri.c_str(), - uri.size())); + CU_ASSERT( + 0 == http2::construct_push_component(scheme, authority, path, base, uri)); CU_ASSERT("" == scheme); CU_ASSERT("" == authority); CU_ASSERT("/foo/bar?q=a" == path); @@ -948,11 +924,10 @@ void test_http2_construct_push_component(void) { authority.clear(); path.clear(); - uri = "foo/../bar?q=a"; + uri = StringRef::from_lit("foo/../bar?q=a"); - CU_ASSERT(0 == http2::construct_push_component(scheme, authority, path, base, - baselen, uri.c_str(), - uri.size())); + CU_ASSERT( + 0 == http2::construct_push_component(scheme, authority, path, base, uri)); CU_ASSERT("" == scheme); CU_ASSERT("" == authority); CU_ASSERT("/b/bar?q=a" == path); @@ -961,11 +936,10 @@ void test_http2_construct_push_component(void) { authority.clear(); path.clear(); - uri = ""; + uri = StringRef{}; - CU_ASSERT(0 == http2::construct_push_component(scheme, authority, path, base, - baselen, uri.c_str(), - uri.size())); + CU_ASSERT( + 0 == http2::construct_push_component(scheme, authority, path, base, uri)); CU_ASSERT("" == scheme); CU_ASSERT("" == authority); CU_ASSERT("/" == path); @@ -974,11 +948,10 @@ void test_http2_construct_push_component(void) { authority.clear(); path.clear(); - uri = "?q=a"; + uri = StringRef::from_lit("?q=a"); - CU_ASSERT(0 == http2::construct_push_component(scheme, authority, path, base, - baselen, uri.c_str(), - uri.size())); + CU_ASSERT( + 0 == http2::construct_push_component(scheme, authority, path, base, uri)); CU_ASSERT("" == scheme); CU_ASSERT("" == authority); CU_ASSERT("/b/?q=a" == path); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 37edafcc..048cc50b 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1755,9 +1755,8 @@ int Http2Upstream::prepare_push_promise(Downstream *downstream) { const std::string *scheme_ptr, *authority_ptr; std::string scheme, authority, path; - rv = http2::construct_push_component(scheme, authority, path, base, - baselen, link.uri.c_str(), - link.uri.size()); + rv = http2::construct_push_component(scheme, authority, path, + StringRef{base, baselen}, link.uri); if (rv != 0) { continue; } @@ -1872,8 +1871,8 @@ int Http2Upstream::initiate_push(Downstream *downstream, const char *uri, const std::string *scheme_ptr, *authority_ptr; std::string scheme, authority, path; - rv = http2::construct_push_component(scheme, authority, path, base, baselen, - uri, len); + rv = http2::construct_push_component( + scheme, authority, path, StringRef{base, baselen}, StringRef{uri, len}); if (rv != 0) { return -1; } From 3f2b54cfc4d252b236709f53d79c7b32bb277f18 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 4 Mar 2016 00:33:35 +0900 Subject: [PATCH 101/147] src: Refactor using StringRef --- src/http2.cc | 15 ++++----------- src/http2.h | 9 ++++----- src/http2_test.cc | 19 ++++++------------- src/shrpx_http2_upstream.cc | 21 ++++++++------------- 4 files changed, 22 insertions(+), 42 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index 6bc898b9..ebc2a8a5 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -1453,28 +1453,21 @@ StringRef to_method_string(int method_token) { return StringRef{http_method_str(static_cast(method_token))}; } -int get_pure_path_component(const char **base, size_t *baselen, - const std::string &uri) { +StringRef get_pure_path_component(const std::string &uri) { int rv; http_parser_url u{}; rv = http_parser_parse_url(uri.c_str(), uri.size(), 0, &u); if (rv != 0) { - return -1; + return StringRef{}; } if (u.field_set & (1 << UF_PATH)) { auto &f = u.field_data[UF_PATH]; - *base = uri.c_str() + f.off; - *baselen = f.len; - - return 0; + return StringRef{uri.c_str() + f.off, f.len}; } - *base = "/"; - *baselen = 1; - - return 0; + return StringRef::from_lit("/"); } int construct_push_component(std::string &scheme, std::string &authority, diff --git a/src/http2.h b/src/http2.h index 23707b45..7a05dd45 100644 --- a/src/http2.h +++ b/src/http2.h @@ -374,11 +374,10 @@ std::string rewrite_clean_path(InputIt first, InputIt last) { return path; } -// Stores path component of |uri| in *base. Its extracted length is -// stored in *baselen. The extracted path does not include query -// component. This function returns 0 if it succeeds, or -1. -int get_pure_path_component(const char **base, size_t *baselen, - const std::string &uri); +// Returns path component of |uri|. The returned path does not +// include query component. This function returns empty string if it +// fails. +StringRef get_pure_path_component(const std::string &uri); // Deduces scheme, authority and path from given |uri|, and stores // them in |scheme|, |authority|, and |path| respectively. If |uri| diff --git a/src/http2_test.cc b/src/http2_test.cc index e3b19693..4d4c7669 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -866,32 +866,25 @@ void test_http2_rewrite_clean_path(void) { } void test_http2_get_pure_path_component(void) { - const char *base; - size_t len; std::string path; path = "/"; - CU_ASSERT(0 == http2::get_pure_path_component(&base, &len, path)); - CU_ASSERT(util::streq_l("/", base, len)); + CU_ASSERT("/" == http2::get_pure_path_component(path)); path = "/foo"; - CU_ASSERT(0 == http2::get_pure_path_component(&base, &len, path)); - CU_ASSERT(util::streq_l("/foo", base, len)); + CU_ASSERT("/foo" == http2::get_pure_path_component(path)); path = "https://example.org/bar"; - CU_ASSERT(0 == http2::get_pure_path_component(&base, &len, path)); - CU_ASSERT(util::streq_l("/bar", base, len)); + CU_ASSERT("/bar" == http2::get_pure_path_component(path)); path = "https://example.org/alpha?q=a"; - CU_ASSERT(0 == http2::get_pure_path_component(&base, &len, path)); - CU_ASSERT(util::streq_l("/alpha", base, len)); + CU_ASSERT("/alpha" == http2::get_pure_path_component(path)); path = "https://example.org/bravo?q=a#fragment"; - CU_ASSERT(0 == http2::get_pure_path_component(&base, &len, path)); - CU_ASSERT(util::streq_l("/bravo", base, len)); + CU_ASSERT("/bravo" == http2::get_pure_path_component(path)); path = "\x01\x02"; - CU_ASSERT(-1 == http2::get_pure_path_component(&base, &len, path)); + CU_ASSERT("" == http2::get_pure_path_component(path)); } void test_http2_construct_push_component(void) { diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 048cc50b..cb4ef196 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1734,14 +1734,12 @@ int Http2Upstream::on_downstream_reset(bool no_retry) { int Http2Upstream::prepare_push_promise(Downstream *downstream) { int rv; - const char *base; - size_t baselen; const auto &req = downstream->request(); const auto &resp = downstream->response(); - rv = http2::get_pure_path_component(&base, &baselen, req.path); - if (rv != 0) { + auto base = http2::get_pure_path_component(req.path); + if (base.empty()) { return 0; } @@ -1755,8 +1753,8 @@ int Http2Upstream::prepare_push_promise(Downstream *downstream) { const std::string *scheme_ptr, *authority_ptr; std::string scheme, authority, path; - rv = http2::construct_push_component(scheme, authority, path, - StringRef{base, baselen}, link.uri); + rv = http2::construct_push_component(scheme, authority, path, base, + link.uri); if (rv != 0) { continue; } @@ -1858,21 +1856,18 @@ int Http2Upstream::initiate_push(Downstream *downstream, const char *uri, return 0; } - const char *base; - size_t baselen; - const auto &req = downstream->request(); - rv = http2::get_pure_path_component(&base, &baselen, req.path); - if (rv != 0) { + auto base = http2::get_pure_path_component(req.path); + if (base.empty()) { return -1; } const std::string *scheme_ptr, *authority_ptr; std::string scheme, authority, path; - rv = http2::construct_push_component( - scheme, authority, path, StringRef{base, baselen}, StringRef{uri, len}); + rv = http2::construct_push_component(scheme, authority, path, base, + StringRef{uri, len}); if (rv != 0) { return -1; } From e302cc9c16a6fb029b9cde55f4b9b01d5233a357 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 4 Mar 2016 00:38:43 +0900 Subject: [PATCH 102/147] src: Simplify function parameter using StringRef --- src/shrpx_http2_upstream.cc | 8 +++----- src/shrpx_http2_upstream.h | 3 +-- src/shrpx_https_upstream.cc | 3 +-- src/shrpx_https_upstream.h | 3 +-- src/shrpx_mruby_module_request.cc | 2 +- src/shrpx_spdy_upstream.cc | 3 +-- src/shrpx_spdy_upstream.h | 3 +-- src/shrpx_upstream.h | 3 +-- 8 files changed, 10 insertions(+), 18 deletions(-) diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index cb4ef196..dfeb2217 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1848,11 +1848,10 @@ bool Http2Upstream::push_enabled() const { get_config()->http2_proxy); } -int Http2Upstream::initiate_push(Downstream *downstream, const char *uri, - size_t len) { +int Http2Upstream::initiate_push(Downstream *downstream, const StringRef &uri) { int rv; - if (len == 0 || !push_enabled() || (downstream->get_stream_id() % 2)) { + if (uri.empty() || !push_enabled() || (downstream->get_stream_id() % 2)) { return 0; } @@ -1866,8 +1865,7 @@ int Http2Upstream::initiate_push(Downstream *downstream, const char *uri, const std::string *scheme_ptr, *authority_ptr; std::string scheme, authority, path; - rv = http2::construct_push_component(scheme, authority, path, base, - StringRef{uri, len}); + rv = http2::construct_push_component(scheme, authority, path, base, uri); if (rv != 0) { return -1; } diff --git a/src/shrpx_http2_upstream.h b/src/shrpx_http2_upstream.h index d090f149..bf1e1c28 100644 --- a/src/shrpx_http2_upstream.h +++ b/src/shrpx_http2_upstream.h @@ -81,8 +81,7 @@ public: virtual int on_downstream_reset(bool no_retry); virtual int send_reply(Downstream *downstream, const uint8_t *body, size_t bodylen); - virtual int initiate_push(Downstream *downstream, const char *uri, - size_t len); + virtual int initiate_push(Downstream *downstream, const StringRef &uri); virtual int response_riovec(struct iovec *iov, int iovcnt) const; virtual void response_drain(size_t n); virtual bool response_empty() const; diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index e089428d..0af9a97e 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -1153,8 +1153,7 @@ fail: return 0; } -int HttpsUpstream::initiate_push(Downstream *downstream, const char *uri, - size_t len) { +int HttpsUpstream::initiate_push(Downstream *downstream, const StringRef &uri) { return 0; } diff --git a/src/shrpx_https_upstream.h b/src/shrpx_https_upstream.h index 3ab698bc..0f9b8dd0 100644 --- a/src/shrpx_https_upstream.h +++ b/src/shrpx_https_upstream.h @@ -76,8 +76,7 @@ public: virtual int on_downstream_reset(bool no_retry); virtual int send_reply(Downstream *downstream, const uint8_t *body, size_t bodylen); - virtual int initiate_push(Downstream *downstream, const char *uri, - size_t len); + virtual int initiate_push(Downstream *downstream, const StringRef &uri); virtual int response_riovec(struct iovec *iov, int iovcnt) const; virtual void response_drain(size_t n); virtual bool response_empty() const; diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index 89bbb660..74201ff2 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -291,7 +291,7 @@ mrb_value request_push(mrb_state *mrb, mrb_value self) { mrb_int len; mrb_get_args(mrb, "s", &uri, &len); - upstream->initiate_push(downstream, uri, len); + upstream->initiate_push(downstream, StringRef{uri, static_cast(len)}); return mrb_nil_value(); } diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index b7f07724..1a2341a8 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -1252,8 +1252,7 @@ int SpdyUpstream::on_downstream_reset(bool no_retry) { return 0; } -int SpdyUpstream::initiate_push(Downstream *downstream, const char *uri, - size_t len) { +int SpdyUpstream::initiate_push(Downstream *downstream, const StringRef &uri) { return 0; } diff --git a/src/shrpx_spdy_upstream.h b/src/shrpx_spdy_upstream.h index 403c3e5a..c84d3f7e 100644 --- a/src/shrpx_spdy_upstream.h +++ b/src/shrpx_spdy_upstream.h @@ -76,8 +76,7 @@ public: virtual int send_reply(Downstream *downstream, const uint8_t *body, size_t bodylen); - virtual int initiate_push(Downstream *downstream, const char *uri, - size_t len); + virtual int initiate_push(Downstream *downstream, const StringRef &uri); virtual int response_riovec(struct iovec *iov, int iovcnt) const; virtual void response_drain(size_t n); virtual bool response_empty() const; diff --git a/src/shrpx_upstream.h b/src/shrpx_upstream.h index b9639e72..df96830e 100644 --- a/src/shrpx_upstream.h +++ b/src/shrpx_upstream.h @@ -68,8 +68,7 @@ public: virtual int send_reply(Downstream *downstream, const uint8_t *body, size_t bodylen) = 0; - virtual int initiate_push(Downstream *downstream, const char *uri, - size_t len) = 0; + virtual int initiate_push(Downstream *downstream, const StringRef &uri) = 0; // Fills response data in |iov| whose capacity is |iovcnt|. Returns // the number of iovs filled. From 2d6211c45552252864f7afa71d7634cbf1e2f9d0 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 4 Mar 2016 22:43:36 +0900 Subject: [PATCH 103/147] asio: Fix bug that server event loop breaks with exception This can happen when we call throwing version of basic_stream_socket::remote_endpoint() call while client disconnected. --- src/asio_server_connection.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/asio_server_connection.h b/src/asio_server_connection.h index 4498497d..6be79406 100644 --- a/src/asio_server_connection.h +++ b/src/asio_server_connection.h @@ -79,8 +79,10 @@ public: /// Start the first asynchronous operation for the connection. void start() { + boost::system::error_code ec; + handler_ = std::make_shared( - socket_.get_io_service(), socket_.lowest_layer().remote_endpoint(), + socket_.get_io_service(), socket_.lowest_layer().remote_endpoint(ec), [this]() { do_write(); }, mux_); if (handler_->start() != 0) { stop(); From 10ec00126c55813641fd89836ab3dff5b57c5bcd Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 4 Mar 2016 23:00:33 +0900 Subject: [PATCH 104/147] src: Don't process rel=preload again once we found it --- src/http2.cc | 84 +++++++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index ebc2a8a5..bfad624e 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -924,53 +924,55 @@ parse_next_link_header_once(const char *first, const char *last) { // we expect link-param if (!ign) { - // rel can take several relations using quoted form. - static constexpr char PLP[] = "rel=\""; - static constexpr size_t PLPLEN = str_size(PLP); + if (!ok) { + // rel can take several relations using quoted form. + static constexpr char PLP[] = "rel=\""; + static constexpr size_t PLPLEN = str_size(PLP); - static constexpr char PLT[] = "preload"; - static constexpr size_t PLTLEN = str_size(PLT); - if (first + PLPLEN < last && *(first + PLPLEN - 1) == '"' && - std::equal(PLP, PLP + PLPLEN, first, util::CaseCmp())) { - // we have to search preload in whitespace separated list: - // rel="preload something http://example.org/foo" - first += PLPLEN; - auto start = first; - for (; first != last;) { - if (*first != ' ' && *first != '"') { - ++first; - continue; + static constexpr char PLT[] = "preload"; + static constexpr size_t PLTLEN = str_size(PLT); + if (first + PLPLEN < last && *(first + PLPLEN - 1) == '"' && + std::equal(PLP, PLP + PLPLEN, first, util::CaseCmp())) { + // we have to search preload in whitespace separated list: + // rel="preload something http://example.org/foo" + first += PLPLEN; + auto start = first; + for (; first != last;) { + if (*first != ' ' && *first != '"') { + ++first; + continue; + } + + if (start == first) { + return {{StringRef{}}, last}; + } + + if (!ok && start + PLTLEN == first && + std::equal(PLT, PLT + PLTLEN, start, util::CaseCmp())) { + ok = true; + } + + if (*first == '"') { + break; + } + first = skip_lws(first, last); + start = first; } - - if (start == first) { + if (first == last) { return {{StringRef{}}, last}; } - - if (!ok && start + PLTLEN == first && - std::equal(PLT, PLT + PLTLEN, start, util::CaseCmp())) { - ok = true; - } - - if (*first == '"') { - break; - } - first = skip_lws(first, last); - start = first; - } - if (first == last) { - return {{StringRef{}}, first}; - } - assert(*first == '"'); - ++first; - if (first == last || *first == ',') { - goto almost_done; - } - if (*first == ';') { + assert(*first == '"'); ++first; - // parse next link-param - continue; + if (first == last || *first == ',') { + goto almost_done; + } + if (*first == ';') { + ++first; + // parse next link-param + continue; + } + return {{StringRef{}}, last}; } - return {{StringRef{}}, last}; } // we are only interested in rel=preload parameter. Others are // simply skipped. From fe6ccd16da67f0ec0b00a6aa16ff419f78208f82 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Mar 2016 19:11:36 +0900 Subject: [PATCH 105/147] nghttpx: Change read timeout reset timing --- src/shrpx_client_handler.cc | 29 ++++++++++++++++++++++++----- src/shrpx_client_handler.h | 2 ++ src/shrpx_http2_upstream.cc | 3 +++ src/shrpx_https_upstream.cc | 14 +++++++++++--- src/shrpx_spdy_upstream.cc | 9 ++++++++- 5 files changed, 48 insertions(+), 9 deletions(-) diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 46540d42..3326e885 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -114,8 +114,6 @@ void writecb(struct ev_loop *loop, ev_io *w, int revents) { int ClientHandler::noop() { return 0; } int ClientHandler::read_clear() { - ev_timer_again(conn_.loop, &conn_.rt); - for (;;) { if (rb_.rleft() && on_read() != 0) { return -1; @@ -124,6 +122,10 @@ int ClientHandler::read_clear() { rb_.reset(); } else if (rb_.wleft() == 0) { conn_.rlimit.stopw(); + if (reset_conn_rtimer_required_) { + reset_conn_rtimer_required_ = false; + ev_timer_again(conn_.loop, &conn_.rt); + } return 0; } @@ -134,6 +136,10 @@ int ClientHandler::read_clear() { auto nread = conn_.read_clear(rb_.last, rb_.wleft()); if (nread == 0) { + if (reset_conn_rtimer_required_) { + reset_conn_rtimer_required_ = false; + ev_timer_again(conn_.loop, &conn_.rt); + } return 0; } @@ -208,8 +214,6 @@ int ClientHandler::tls_handshake() { } int ClientHandler::read_tls() { - ev_timer_again(conn_.loop, &conn_.rt); - ERR_clear_error(); for (;;) { @@ -221,6 +225,11 @@ int ClientHandler::read_tls() { rb_.reset(); } else if (rb_.wleft() == 0) { conn_.rlimit.stopw(); + if (reset_conn_rtimer_required_) { + reset_conn_rtimer_required_ = false; + ev_timer_again(conn_.loop, &conn_.rt); + } + return 0; } @@ -231,6 +240,11 @@ int ClientHandler::read_tls() { auto nread = conn_.read_tls(rb_.last, rb_.wleft()); if (nread == 0) { + if (reset_conn_rtimer_required_) { + reset_conn_rtimer_required_ = false; + ev_timer_again(conn_.loop, &conn_.rt); + } + return 0; } @@ -391,7 +405,8 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl, faddr_(faddr), worker_(worker), left_connhd_len_(NGHTTP2_CLIENT_MAGIC_LEN), - should_close_after_write_(false) { + should_close_after_write_(false), + reset_conn_rtimer_required_(false) { ++worker_->get_worker_stat()->num_connections; @@ -497,6 +512,10 @@ void ClientHandler::reset_upstream_write_timeout(ev_tstamp t) { } } +void ClientHandler::signal_reset_upstream_conn_rtimer() { + reset_conn_rtimer_required_ = true; +} + int ClientHandler::validate_next_proto() { const unsigned char *next_proto = nullptr; unsigned int next_proto_len; diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index 7d6ace81..ec353576 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -86,6 +86,7 @@ public: struct ev_loop *get_loop() const; void reset_upstream_read_timeout(ev_tstamp t); void reset_upstream_write_timeout(ev_tstamp t); + void signal_reset_upstream_conn_rtimer(); int validate_next_proto(); const std::string &get_ipaddr() const; const std::string &get_port() const; @@ -161,6 +162,7 @@ private: // The number of bytes of HTTP/2 client connection header to read size_t left_connhd_len_; bool should_close_after_write_; + bool reset_conn_rtimer_required_; ReadBuf rb_; }; diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index dfeb2217..9d42fe3a 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -411,6 +411,9 @@ int on_frame_recv_callback(nghttp2_session *session, const nghttp2_frame *frame, verbose_on_frame_recv_callback(session, frame, user_data); } auto upstream = static_cast(user_data); + auto handler = upstream->get_client_handler(); + + handler->signal_reset_upstream_conn_rtimer(); switch (frame->hd.type) { case NGHTTP2_DATA: { diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 0af9a97e..49c9848e 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -247,6 +247,11 @@ int htp_hdrs_completecb(http_parser *htp) { if (LOG_ENABLED(INFO)) { ULOG(INFO, upstream) << "HTTP request headers completed"; } + + auto handler = upstream->get_client_handler(); + + handler->signal_reset_upstream_conn_rtimer(); + auto downstream = upstream->get_downstream(); auto &req = downstream->request(); @@ -323,7 +328,7 @@ int htp_hdrs_completecb(http_parser *htp) { req.authority = host->value; } - if (upstream->get_client_handler()->get_ssl()) { + if (handler->get_ssl()) { req.scheme = "https"; } else { req.scheme = "http"; @@ -336,7 +341,6 @@ int htp_hdrs_completecb(http_parser *htp) { downstream->set_request_state(Downstream::HEADER_COMPLETE); #ifdef HAVE_MRUBY - auto handler = upstream->get_client_handler(); auto worker = handler->get_worker(); auto mruby_ctx = worker->get_mruby_context(); @@ -355,7 +359,7 @@ int htp_hdrs_completecb(http_parser *htp) { } rv = downstream->attach_downstream_connection( - upstream->get_client_handler()->get_downstream_connection(downstream)); + handler->get_downstream_connection(downstream)); if (rv != 0) { downstream->set_request_state(Downstream::CONNECT_FAIL); @@ -377,6 +381,10 @@ namespace { int htp_bodycb(http_parser *htp, const char *data, size_t len) { int rv; auto upstream = static_cast(htp->data); + auto handler = upstream->get_client_handler(); + + handler->signal_reset_upstream_conn_rtimer(); + auto downstream = upstream->get_downstream(); rv = downstream->push_upload_data_chunk( reinterpret_cast(data), len); diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 1a2341a8..e15444a4 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -143,6 +143,10 @@ namespace { void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, spdylay_frame *frame, void *user_data) { auto upstream = static_cast(user_data); + auto handler = upstream->get_client_handler(); + + handler->signal_reset_upstream_conn_rtimer(); + switch (type) { case SPDYLAY_SYN_STREAM: { if (LOG_ENABLED(INFO)) { @@ -281,7 +285,6 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, downstream->set_request_state(Downstream::HEADER_COMPLETE); #ifdef HAVE_MRUBY - auto handler = upstream->get_client_handler(); auto worker = handler->get_worker(); auto mruby_ctx = worker->get_mruby_context(); @@ -419,6 +422,10 @@ void on_data_recv_callback(spdylay_session *session, uint8_t flags, auto upstream = static_cast(user_data); auto downstream = static_cast( spdylay_session_get_stream_user_data(session, stream_id)); + auto handler = upstream->get_client_handler(); + + handler->signal_reset_upstream_conn_rtimer(); + if (downstream && (flags & SPDYLAY_DATA_FLAG_FIN)) { if (!downstream->validate_request_recv_body_length()) { upstream->rst_stream(downstream, SPDYLAY_PROTOCOL_ERROR); From 907eeeda8a1f5a43e00e29badabcd32a8d19415e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 8 Mar 2016 21:38:15 +0900 Subject: [PATCH 106/147] src: Return StringRef from util::get_uri_field --- src/h2load.cc | 6 +++--- src/nghttp.cc | 24 ++++++++++++------------ src/util.cc | 12 ++++++------ src/util.h | 4 ++-- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/h2load.cc b/src/h2load.cc index dc164b28..14b1fca4 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -1396,7 +1396,7 @@ std::string get_reqline(const char *uri, const http_parser_url &u) { std::string reqline; if (util::has_uri_field(u, UF_PATH)) { - reqline = util::get_uri_field(uri, u, UF_PATH); + reqline = util::get_uri_field(uri, u, UF_PATH).str(); } else { reqline = "/"; } @@ -1437,8 +1437,8 @@ bool parse_base_uri(std::string base_uri) { return false; } - config.scheme = util::get_uri_field(base_uri.c_str(), u, UF_SCHEMA); - config.host = util::get_uri_field(base_uri.c_str(), u, UF_HOST); + config.scheme = util::get_uri_field(base_uri.c_str(), u, UF_SCHEMA).str(); + config.host = util::get_uri_field(base_uri.c_str(), u, UF_HOST).str(); config.default_port = util::get_default_port(base_uri.c_str(), u); if (util::has_uri_field(u, UF_PORT)) { config.port = u.port; diff --git a/src/nghttp.cc b/src/nghttp.cc index a1705124..af349519 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -187,7 +187,7 @@ int Request::update_html_parser(const uint8_t *data, size_t len, int fin) { std::string Request::make_reqpath() const { std::string path = util::has_uri_field(u, UF_PATH) - ? util::get_uri_field(uri.c_str(), u, UF_PATH) + ? util::get_uri_field(uri.c_str(), u, UF_PATH).str() : "/"; if (util::has_uri_field(u, UF_QUERY)) { path += '?'; @@ -200,21 +200,20 @@ std::string Request::make_reqpath() const { namespace { // Perform special handling |host| if it is IPv6 literal and includes // zone ID per RFC 6874. -std::string decode_host(std::string host) { +std::string decode_host(const StringRef &host) { auto zone_start = std::find(std::begin(host), std::end(host), '%'); if (zone_start == std::end(host) || !util::ipv6_numeric_addr( std::string(std::begin(host), zone_start).c_str())) { - return host; + return host.str(); } // case: ::1% if (zone_start + 1 == std::end(host)) { - host.pop_back(); - return host; + return StringRef{host.c_str(), host.size() - 1}.str(); } // case: ::1%12 or ::1%1 if (zone_start + 3 >= std::end(host)) { - return host; + return host.str(); } // If we see "%25", followed by more characters, then decode %25 as // '%'. @@ -222,9 +221,9 @@ std::string decode_host(std::string host) { ? zone_start + 3 : zone_start + 1; auto zone_id = util::percent_decode(zone_id_src, std::end(host)); - host.erase(zone_start + 1, std::end(host)); - host += zone_id; - return host; + auto res = std::string(std::begin(host), zone_start + 1); + res += zone_id; + return res; } } // namespace @@ -349,7 +348,7 @@ int submit_request(HttpClient *client, const Headers &headers, Request *req) { auto scheme = util::get_uri_field(req->uri.c_str(), req->u, UF_SCHEMA); auto build_headers = Headers{{":method", req->data_prd ? "POST" : "GET"}, {":path", path}, - {":scheme", scheme}, + {":scheme", scheme.str()}, {":authority", client->hostport}, {"accept", "*/*"}, {"accept-encoding", "gzip, deflate"}, @@ -1255,7 +1254,8 @@ void HttpClient::update_hostport() { if (reqvec.empty()) { return; } - scheme = util::get_uri_field(reqvec[0]->uri.c_str(), reqvec[0]->u, UF_SCHEMA); + scheme = util::get_uri_field(reqvec[0]->uri.c_str(), reqvec[0]->u, UF_SCHEMA) + .str(); std::stringstream ss; if (reqvec[0]->is_ipv6_literal_addr()) { // we may have zone ID, which must start with "%25", or "%". RFC @@ -2381,7 +2381,7 @@ int run(char **uris, int n) { } requests.clear(); } - prev_scheme = util::get_uri_field(uri.c_str(), u, UF_SCHEMA); + prev_scheme = util::get_uri_field(uri.c_str(), u, UF_SCHEMA).str(); prev_host = std::move(host); prev_port = port; } diff --git a/src/util.cc b/src/util.cc index 7805a957..784806be 100644 --- a/src/util.cc +++ b/src/util.cc @@ -591,13 +591,13 @@ bool fieldeq(const char *uri, const http_parser_url &u, return i == len && !t[i]; } -std::string get_uri_field(const char *uri, const http_parser_url &u, - http_parser_url_fields field) { - if (util::has_uri_field(u, field)) { - return std::string(uri + u.field_data[field].off, u.field_data[field].len); - } else { - return ""; +StringRef get_uri_field(const char *uri, const http_parser_url &u, + http_parser_url_fields field) { + if (!util::has_uri_field(u, field)) { + return StringRef{}; } + + return StringRef{uri + u.field_data[field].off, u.field_data[field].len}; } uint16_t get_default_port(const char *uri, const http_parser_url &u) { diff --git a/src/util.h b/src/util.h index 15894816..d03ffd8e 100644 --- a/src/util.h +++ b/src/util.h @@ -448,8 +448,8 @@ bool fieldeq(const char *uri1, const http_parser_url &u1, const char *uri2, bool fieldeq(const char *uri, const http_parser_url &u, http_parser_url_fields field, const char *t); -std::string get_uri_field(const char *uri, const http_parser_url &u, - http_parser_url_fields field); +StringRef get_uri_field(const char *uri, const http_parser_url &u, + http_parser_url_fields field); uint16_t get_default_port(const char *uri, const http_parser_url &u); From bae37e3e4aec20f730560d9c0fb968824f6db6e7 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 9 Mar 2016 21:15:32 +0900 Subject: [PATCH 107/147] nghttpx: Add custom memory allocator mainly for header related objects --- src/allocator.h | 108 +++++++++++++++++ src/http2.cc | 63 ++++++++-- src/http2.h | 51 ++++++-- src/nghttp.cc | 2 +- src/shrpx_downstream.cc | 142 +++++++++++++---------- src/shrpx_downstream.h | 35 +++--- src/shrpx_http2_downstream_connection.cc | 8 +- src/shrpx_http2_upstream.cc | 4 +- src/shrpx_https_upstream.cc | 2 +- src/shrpx_mruby_module.cc | 2 +- src/shrpx_mruby_module.h | 2 +- src/shrpx_mruby_module_response.cc | 3 +- src/shrpx_spdy_upstream.cc | 10 +- src/template.h | 10 ++ src/util.cc | 4 + src/util.h | 11 ++ 16 files changed, 346 insertions(+), 111 deletions(-) create mode 100644 src/allocator.h diff --git a/src/allocator.h b/src/allocator.h new file mode 100644 index 00000000..2f48f38b --- /dev/null +++ b/src/allocator.h @@ -0,0 +1,108 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef ALLOCATOR_H +#define ALLOCATOR_H + +#include "nghttp2_config.h" + +#include "template.h" + +namespace nghttp2 { + +struct MemBlock { + MemBlock *next; + uint8_t *begin, *last, *end; +}; + +template struct BlockAllocator { + BlockAllocator() : retain(nullptr), head(nullptr) {} + + ~BlockAllocator() { + for (auto mb = retain; mb;) { + auto next = mb->next; + delete[] reinterpret_cast(mb); + mb = next; + } + } + + MemBlock *alloc_mem_block(size_t size) { + auto block = new uint8_t[sizeof(MemBlock) + size]; + auto mb = reinterpret_cast(block); + + mb->next = retain; + mb->begin = mb->last = block + sizeof(MemBlock); + mb->end = mb->begin + size; + retain = mb; + return mb; + } + + void *alloc(size_t size) { + if (size >= BLOCK_SIZE) { + auto mb = alloc_mem_block(size); + mb->last = mb->end; + return mb->begin; + } + + if (!head || head->end - head->last < static_cast(size)) { + head = alloc_mem_block(BLOCK_SIZE); + } + + auto res = head->last; + + head->last = reinterpret_cast( + (reinterpret_cast(head->last + size) + 0xf) & ~0xf); + + return res; + } + + MemBlock *retain; + MemBlock *head; +}; + +template +StringRef make_string_ref(BlockAllocator &alloc, const StringRef &src) { + auto dst = static_cast(alloc.alloc(src.size() + 1)); + auto p = dst; + p = std::copy(std::begin(src), std::end(src), p); + *p = '\0'; + return StringRef{dst, src.size()}; +} + +template +StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, + const StringRef &b) { + auto dst = static_cast(alloc.alloc(a.size() + b.size() + 1)); + auto p = dst; + p = std::copy(std::begin(a), std::end(a), p); + p = std::copy(std::begin(b), std::end(b), p); + *p = '\0'; + return StringRef{dst, a.size() + b.size()}; +} + +using DefaultBlockAllocator = BlockAllocator<1024>; + +} // namespace aria2 + +#endif // ALLOCATOR_H diff --git a/src/http2.cc b/src/http2.cc index bfad624e..17de9b05 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -238,7 +238,7 @@ const char *stringify_status(unsigned int status_code) { } } -void capitalize(DefaultMemchunks *buf, const std::string &s) { +void capitalize(DefaultMemchunks *buf, const StringRef &s) { buf->append(util::upcase(s[0])); for (size_t i = 1; i < s.size(); ++i) { if (s[i - 1] == '-') { @@ -302,14 +302,14 @@ const Headers::value_type *get_header(const Headers &nva, const char *name) { return res; } -std::string value_to_str(const Headers::value_type *nv) { +std::string value_to_str(const HeaderRefs::value_type *nv) { if (nv) { - return nv->value; + return nv->value.str(); } return ""; } -bool non_empty_value(const Headers::value_type *nv) { +bool non_empty_value(const HeaderRefs::value_type *nv) { return nv && !nv->value.empty(); } @@ -326,11 +326,29 @@ nghttp2_nv make_nv_internal(const std::string &name, const std::string &value, } } // namespace +namespace { +nghttp2_nv make_nv_internal(const StringRef &name, const StringRef &value, + bool no_index, uint8_t nv_flags) { + uint8_t flags; + + flags = + nv_flags | (no_index ? NGHTTP2_NV_FLAG_NO_INDEX : NGHTTP2_NV_FLAG_NONE); + + return {(uint8_t *)name.c_str(), (uint8_t *)value.c_str(), name.size(), + value.size(), flags}; +} +} // namespace + nghttp2_nv make_nv(const std::string &name, const std::string &value, bool no_index) { return make_nv_internal(name, value, no_index, NGHTTP2_NV_FLAG_NONE); } +nghttp2_nv make_nv(const StringRef &name, const StringRef &value, + bool no_index) { + return make_nv_internal(name, value, no_index, NGHTTP2_NV_FLAG_NONE); +} + nghttp2_nv make_nv_nocopy(const std::string &name, const std::string &value, bool no_index) { return make_nv_internal(name, value, no_index, @@ -338,9 +356,16 @@ nghttp2_nv make_nv_nocopy(const std::string &name, const std::string &value, NGHTTP2_NV_FLAG_NO_COPY_VALUE); } +nghttp2_nv make_nv_nocopy(const StringRef &name, const StringRef &value, + bool no_index) { + return make_nv_internal(name, value, no_index, + NGHTTP2_NV_FLAG_NO_COPY_NAME | + NGHTTP2_NV_FLAG_NO_COPY_VALUE); +} + namespace { void copy_headers_to_nva_internal(std::vector &nva, - const Headers &headers, uint8_t nv_flags) { + const HeaderRefs &headers, uint8_t nv_flags) { for (auto &kv : headers) { if (kv.name.empty() || kv.name[0] == ':') { continue; @@ -367,18 +392,19 @@ void copy_headers_to_nva_internal(std::vector &nva, } } // namespace -void copy_headers_to_nva(std::vector &nva, const Headers &headers) { +void copy_headers_to_nva(std::vector &nva, + const HeaderRefs &headers) { copy_headers_to_nva_internal(nva, headers, NGHTTP2_NV_FLAG_NONE); } void copy_headers_to_nva_nocopy(std::vector &nva, - const Headers &headers) { + const HeaderRefs &headers) { copy_headers_to_nva_internal(nva, headers, NGHTTP2_NV_FLAG_NO_COPY_NAME | NGHTTP2_NV_FLAG_NO_COPY_VALUE); } void build_http1_headers_from_headers(DefaultMemchunks *buf, - const Headers &headers) { + const HeaderRefs &headers) { for (auto &kv : headers) { if (kv.name.empty() || kv.name[0] == ':') { continue; @@ -450,8 +476,15 @@ void dump_nv(FILE *out, const Headers &nva) { fflush(out); } -std::string rewrite_location_uri(const std::string &uri, - const http_parser_url &u, +void dump_nv(FILE *out, const HeaderRefs &nva) { + for (auto &nv : nva) { + fprintf(out, "%s: %s\n", nv.name.c_str(), nv.value.c_str()); + } + fputc('\n', out); + fflush(out); +} + +std::string rewrite_location_uri(const StringRef &uri, const http_parser_url &u, const std::string &match_host, const std::string &request_authority, const std::string &upstream_scheme) { @@ -499,7 +532,7 @@ int check_nv(const uint8_t *name, size_t namelen, const uint8_t *value, return 1; } -int parse_http_status_code(const std::string &src) { +int parse_http_status_code(const StringRef &src) { if (src.size() != 3) { return -1; } @@ -525,6 +558,10 @@ int lookup_token(const std::string &name) { name.size()); } +int lookup_token(const StringRef &name) { + return lookup_token(name.byte(), name.size()); +} + // This function was generated by genheaderfunc.py. Inspired by h2o // header lookup. https://github.com/h2o/h2o int lookup_token(const uint8_t *name, size_t namelen) { @@ -1278,6 +1315,10 @@ int lookup_method_token(const std::string &name) { name.size()); } +int lookup_method_token(const StringRef &name) { + return lookup_method_token(name.byte(), name.size()); +} + // This function was generated by genmethodfunc.py. int lookup_method_token(const uint8_t *name, size_t namelen) { switch (namelen) { diff --git a/src/http2.h b/src/http2.h index 7a05dd45..b4e84e8b 100644 --- a/src/http2.h +++ b/src/http2.h @@ -39,6 +39,7 @@ #include "util.h" #include "memchunk.h" +#include "template.h" namespace nghttp2 { @@ -66,7 +67,29 @@ struct Header { bool no_index; }; +struct HeaderRef { + HeaderRef(const StringRef &name, const StringRef &value, + bool no_index = false, int32_t token = -1) + : name(name), value(value), token(token), no_index(no_index) {} + + HeaderRef() : token(-1), no_index(false) {} + + bool operator==(const HeaderRef &other) const { + return name == other.name && value == other.value; + } + + bool operator<(const HeaderRef &rhs) const { + return name < rhs.name || (name == rhs.name && value < rhs.value); + } + + StringRef name; + StringRef value; + int32_t token; + bool no_index; +}; + using Headers = std::vector
; +using HeaderRefs = std::vector; namespace http2 { @@ -76,7 +99,7 @@ std::string get_status_string(unsigned int status_code); // only predefined status code. Otherwise, returns nullptr. const char *stringify_status(unsigned int status_code); -void capitalize(DefaultMemchunks *buf, const std::string &s); +void capitalize(DefaultMemchunks *buf, const StringRef &s); // Returns true if |value| is LWS bool lws(const char *value); @@ -104,10 +127,10 @@ void add_header(Headers &nva, const uint8_t *name, size_t namelen, const Headers::value_type *get_header(const Headers &nva, const char *name); // Returns nv->second if nv is not nullptr. Otherwise, returns "". -std::string value_to_str(const Headers::value_type *nv); +std::string value_to_str(const HeaderRefs::value_type *nv); // Returns true if the value of |nv| is not empty. -bool non_empty_value(const Headers::value_type *nv); +bool non_empty_value(const HeaderRefs::value_type *nv); // Creates nghttp2_nv using |name| and |value| and returns it. The // returned value only references the data pointer to name.c_str() and @@ -116,9 +139,15 @@ bool non_empty_value(const Headers::value_type *nv); nghttp2_nv make_nv(const std::string &name, const std::string &value, bool no_index = false); +nghttp2_nv make_nv(const StringRef &name, const StringRef &value, + bool no_index = false); + nghttp2_nv make_nv_nocopy(const std::string &name, const std::string &value, bool no_index = false); +nghttp2_nv make_nv_nocopy(const StringRef &name, const StringRef &value, + bool no_index = false); + // Create nghttp2_nv from string literal |name| and |value|. template constexpr nghttp2_nv make_nv_ll(const char(&name)[N], const char(&value)[M]) { @@ -163,19 +192,20 @@ nghttp2_nv make_nv_ls_nocopy(const char(&name)[N], const StringRef &value) { // before this call (its element's token field is assigned). Certain // headers, including disallowed headers in HTTP/2 spec and headers // which require special handling (i.e. via), are not copied. -void copy_headers_to_nva(std::vector &nva, const Headers &headers); +void copy_headers_to_nva(std::vector &nva, + const HeaderRefs &headers); // Just like copy_headers_to_nva(), but this adds // NGHTTP2_NV_FLAG_NO_COPY_NAME and NGHTTP2_NV_FLAG_NO_COPY_VALUE. void copy_headers_to_nva_nocopy(std::vector &nva, - const Headers &headers); + const HeaderRefs &headers); // Appends HTTP/1.1 style header lines to |buf| from headers in // |headers|. |headers| must be indexed before this call (its // element's token field is assigned). Certain headers, which // requires special handling (i.e. via and cookie), are not appended. void build_http1_headers_from_headers(DefaultMemchunks *buf, - const Headers &headers); + const HeaderRefs &headers); // Return positive window_size_increment if WINDOW_UPDATE should be // sent for the stream |stream_id|. If |stream_id| == 0, this function @@ -196,6 +226,8 @@ void dump_nv(FILE *out, const nghttp2_nv *nva, size_t nvlen); // Dumps name/value pairs in |nva| to |out|. void dump_nv(FILE *out, const Headers &nva); +void dump_nv(FILE *out, const HeaderRefs &nva); + // Rewrites redirection URI which usually appears in location header // field. The |uri| is the URI in the location header field. The |u| // stores the result of parsed |uri|. The |request_authority| is the @@ -209,8 +241,7 @@ void dump_nv(FILE *out, const Headers &nva); // This function returns the new rewritten URI on success. If the // location URI is not subject to the rewrite, this function returns // emtpy string. -std::string rewrite_location_uri(const std::string &uri, - const http_parser_url &u, +std::string rewrite_location_uri(const StringRef &uri, const http_parser_url &u, const std::string &match_host, const std::string &request_authority, const std::string &upstream_scheme); @@ -222,7 +253,7 @@ int check_nv(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen); // Returns parsed HTTP status code. Returns -1 on failure. -int parse_http_status_code(const std::string &src); +int parse_http_status_code(const StringRef &src); // Header fields to be indexed, except HD_MAXIDX which is convenient // member to get maximum value. @@ -272,6 +303,7 @@ using HeaderIndex = std::array; // cannot be tokenized, returns -1. int lookup_token(const uint8_t *name, size_t namelen); int lookup_token(const std::string &name); +int lookup_token(const StringRef &name); // Initializes |hdidx|, header index. The |hdidx| must point to the // array containing at least HD_MAXIDX elements. @@ -318,6 +350,7 @@ bool expect_response_body(int status_code); // tokenized. If method name cannot be tokenized, returns -1. int lookup_method_token(const uint8_t *name, size_t namelen); int lookup_method_token(const std::string &name); +int lookup_method_token(const StringRef &name); // Returns string representation of |method_token|. This is wrapper // function over http_method_str from http-parser. If |method_token| diff --git a/src/nghttp.cc b/src/nghttp.cc index af349519..d0fc8ece 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -1600,7 +1600,7 @@ void check_response_header(nghttp2_session *session, Request *req) { return; } - auto status = http2::parse_http_status_code(status_hd->value); + auto status = http2::parse_http_status_code(StringRef{status_hd->value}); if (status == -1) { nghttp2_submit_rst_stream(session, NGHTTP2_FLAG_NONE, req->stream_id, NGHTTP2_PROTOCOL_ERROR); diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 0a8da837..11462d7c 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -116,6 +116,8 @@ Downstream::Downstream(Upstream *upstream, MemchunkPool *mcpool, : dlnext(nullptr), dlprev(nullptr), response_sent_body_length(0), + req_(balloc_), + resp_(balloc_), request_start_time_(std::chrono::high_resolution_clock::now()), request_buf_(mcpool), response_buf_(mcpool), @@ -237,8 +239,9 @@ void Downstream::force_resume_read() { } namespace { -const Headers::value_type * -search_header_linear_backwards(const Headers &headers, const StringRef &name) { +const HeaderRefs::value_type * +search_header_linear_backwards(const HeaderRefs &headers, + const StringRef &name) { for (auto it = headers.rbegin(); it != headers.rend(); ++it) { auto &kv = *it; if (kv.name == name) { @@ -258,11 +261,25 @@ std::string Downstream::assemble_request_cookie() const { continue; } - auto end = kv.value.find_last_not_of(" ;"); - if (end == std::string::npos) { + if (kv.value.empty()) { + continue; + } + + auto end = std::end(kv.value); + for (auto it = std::begin(kv.value) + kv.value.size(); + it != std::begin(kv.value); --it) { + auto c = *(it - 1); + if (c == ' ' || c == ';') { + continue; + } + end = it + 1; + break; + } + + if (end == std::end(kv.value)) { cookie += kv.value; } else { - cookie.append(std::begin(kv.value), std::begin(kv.value) + end + 1); + cookie.append(std::begin(kv.value), end); } cookie += "; "; } @@ -280,18 +297,14 @@ size_t Downstream::count_crumble_request_cookie() { !util::streq_l("cooki", kv.name.c_str(), 5)) { continue; } - size_t last = kv.value.size(); - for (size_t j = 0; j < last;) { - j = kv.value.find_first_not_of("\t ;", j); - if (j == std::string::npos) { - break; + for (auto it = std::begin(kv.value); it != std::end(kv.value);) { + if (*it == '\t' || *it == ' ' || *it == ';') { + ++it; + continue; } - j = kv.value.find(';', j); - if (j == std::string::npos) { - j = last; - } + it = std::find(it, std::end(kv.value), ';'); ++n; } @@ -305,22 +318,19 @@ void Downstream::crumble_request_cookie(std::vector &nva) { !util::streq_l("cooki", kv.name.c_str(), 5)) { continue; } - size_t last = kv.value.size(); - for (size_t j = 0; j < last;) { - j = kv.value.find_first_not_of("\t ;", j); - if (j == std::string::npos) { - break; - } - auto first = j; - - j = kv.value.find(';', j); - if (j == std::string::npos) { - j = last; + for (auto it = std::begin(kv.value); it != std::end(kv.value);) { + if (*it == '\t' || *it == ' ' || *it == ';') { + ++it; + continue; } - nva.push_back({(uint8_t *)"cookie", (uint8_t *)kv.value.c_str() + first, - str_size("cookie"), j - first, + auto first = it; + + it = std::find(it, std::end(kv.value), ';'); + + nva.push_back({(uint8_t *)"cookie", (uint8_t *)first, str_size("cookie"), + (size_t)(it - first), (uint8_t)(NGHTTP2_NV_FLAG_NO_COPY_NAME | NGHTTP2_NV_FLAG_NO_COPY_VALUE | (kv.no_index ? NGHTTP2_NV_FLAG_NO_INDEX : 0))}); @@ -329,42 +339,45 @@ void Downstream::crumble_request_cookie(std::vector &nva) { } namespace { -void add_header(bool &key_prev, size_t &sum, Headers &headers, +void add_header(bool &key_prev, size_t &sum, HeaderRefs &headers, const StringRef &name, const StringRef &value, bool no_index, int32_t token) { key_prev = true; sum += name.size() + value.size(); - headers.emplace_back(name.str(), value.str(), no_index, token); + headers.emplace_back(name, value, no_index, token); } } // namespace namespace { -void add_header(size_t &sum, Headers &headers, const StringRef &name, +void add_header(size_t &sum, HeaderRefs &headers, const StringRef &name, const StringRef &value, bool no_index, int32_t token) { sum += name.size() + value.size(); - headers.emplace_back(name.str(), value.str(), no_index, token); + headers.emplace_back(name, value, no_index, token); } } // namespace namespace { -void append_last_header_key(bool &key_prev, size_t &sum, Headers &headers, - const char *data, size_t len) { +void append_last_header_key(DefaultBlockAllocator &balloc, bool &key_prev, + size_t &sum, HeaderRefs &headers, const char *data, + size_t len) { assert(key_prev); sum += len; auto &item = headers.back(); - item.name.append(data, len); - util::inp_strlower(item.name); + item.name = concat_string_ref(balloc, item.name, StringRef{data, len}); + util::inp_strlower((uint8_t *)item.name.c_str(), + (uint8_t *)item.name.c_str() + item.name.size()); item.token = http2::lookup_token(item.name); } } // namespace namespace { -void append_last_header_value(bool &key_prev, size_t &sum, Headers &headers, +void append_last_header_value(DefaultBlockAllocator &balloc, bool &key_prev, + size_t &sum, HeaderRefs &headers, const char *data, size_t len) { key_prev = false; sum += len; auto &item = headers.back(); - item.value.append(data, len); + item.value = concat_string_ref(balloc, item.value, StringRef{data, len}); } } // namespace @@ -388,7 +401,7 @@ int FieldStore::parse_content_length() { return 0; } -const Headers::value_type *FieldStore::header(int32_t token) const { +const HeaderRefs::value_type *FieldStore::header(int32_t token) const { for (auto it = headers_.rbegin(); it != headers_.rend(); ++it) { auto &kv = *it; if (kv.token == token) { @@ -398,7 +411,7 @@ const Headers::value_type *FieldStore::header(int32_t token) const { return nullptr; } -Headers::value_type *FieldStore::header(int32_t token) { +HeaderRefs::value_type *FieldStore::header(int32_t token) { for (auto it = headers_.rbegin(); it != headers_.rend(); ++it) { auto &kv = *it; if (kv.token == token) { @@ -408,43 +421,47 @@ Headers::value_type *FieldStore::header(int32_t token) { return nullptr; } -const Headers::value_type *FieldStore::header(const StringRef &name) const { +const HeaderRefs::value_type *FieldStore::header(const StringRef &name) const { return search_header_linear_backwards(headers_, name); } void FieldStore::add_header_lower(const StringRef &name, const StringRef &value, bool no_index) { - auto low_name = name.str(); - util::inp_strlower(low_name); + auto low_name = make_string_ref(balloc_, name); + util::inp_strlower((uint8_t *)low_name.c_str(), + (uint8_t *)low_name.c_str() + low_name.size()); auto token = http2::lookup_token(low_name); - shrpx::add_header(header_key_prev_, buffer_size_, headers_, - StringRef{low_name}, value, no_index, token); + shrpx::add_header(header_key_prev_, buffer_size_, headers_, low_name, + make_string_ref(balloc_, value), no_index, token); } void FieldStore::add_header_token(const StringRef &name, const StringRef &value, bool no_index, int32_t token) { - shrpx::add_header(buffer_size_, headers_, name, value, no_index, token); + shrpx::add_header(buffer_size_, headers_, make_string_ref(balloc_, name), + make_string_ref(balloc_, value), no_index, token); + make_string_ref(balloc_, name); } void FieldStore::append_last_header_key(const char *data, size_t len) { - shrpx::append_last_header_key(header_key_prev_, buffer_size_, headers_, data, - len); + shrpx::append_last_header_key(balloc_, header_key_prev_, buffer_size_, + headers_, data, len); } void FieldStore::append_last_header_value(const char *data, size_t len) { - shrpx::append_last_header_value(header_key_prev_, buffer_size_, headers_, - data, len); + shrpx::append_last_header_value(balloc_, header_key_prev_, buffer_size_, + headers_, data, len); } void FieldStore::clear_headers() { headers_.clear(); } void FieldStore::add_trailer_lower(const StringRef &name, const StringRef &value, bool no_index) { - auto low_name = name.str(); - util::inp_strlower(low_name); + auto low_name = make_string_ref(balloc_, name); + util::inp_strlower((uint8_t *)low_name.c_str(), + (uint8_t *)low_name.c_str() + low_name.size()); auto token = http2::lookup_token(low_name); - shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, - StringRef{low_name}, value, no_index, token); + shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, low_name, + make_string_ref(balloc_, value), no_index, token); } void FieldStore::add_trailer_token(const StringRef &name, @@ -452,17 +469,18 @@ void FieldStore::add_trailer_token(const StringRef &name, int32_t token) { // Header size limit should be applied to all header and trailer // fields combined. - shrpx::add_header(buffer_size_, trailers_, name, value, no_index, token); + shrpx::add_header(buffer_size_, trailers_, make_string_ref(balloc_, name), + make_string_ref(balloc_, value), no_index, token); } void FieldStore::append_last_trailer_key(const char *data, size_t len) { - shrpx::append_last_header_key(trailer_key_prev_, buffer_size_, trailers_, - data, len); + shrpx::append_last_header_key(balloc_, trailer_key_prev_, buffer_size_, + trailers_, data, len); } void FieldStore::append_last_trailer_value(const char *data, size_t len) { - shrpx::append_last_header_value(trailer_key_prev_, buffer_size_, trailers_, - data, len); + shrpx::append_last_header_value(balloc_, trailer_key_prev_, buffer_size_, + trailers_, data, len); } void Downstream::set_request_start_time( @@ -566,7 +584,7 @@ void Downstream::rewrite_location_response_header( return; } - hd->value = std::move(new_uri); + hd->value = make_string_ref(balloc_, StringRef{new_uri}); } bool Downstream::get_chunked_response() const { return chunked_response_; } @@ -703,10 +721,10 @@ bool Downstream::get_http2_upgrade_request() const { response_state_ == INITIAL; } -const std::string &Downstream::get_http2_settings() const { +StringRef Downstream::get_http2_settings() const { auto http2_settings = req_.fs.header(http2::HD_HTTP2_SETTINGS); if (!http2_settings) { - return EMPTY_STRING; + return StringRef{}; } return http2_settings->value; } @@ -908,4 +926,6 @@ void Downstream::set_assoc_stream_id(int32_t stream_id) { int32_t Downstream::get_assoc_stream_id() const { return assoc_stream_id_; } +DefaultBlockAllocator &Downstream::get_block_allocator() { return balloc_; } + } // namespace shrpx diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 3864674f..7b9dfa83 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -40,6 +40,7 @@ #include "shrpx_io_control.h" #include "http2.h" #include "memchunk.h" +#include "allocator.h" using namespace nghttp2; @@ -51,18 +52,19 @@ struct BlockedLink; class FieldStore { public: - FieldStore(size_t headers_initial_capacity) + FieldStore(DefaultBlockAllocator &balloc, size_t headers_initial_capacity) : content_length(-1), + balloc_(balloc), buffer_size_(0), header_key_prev_(false), trailer_key_prev_(false) { headers_.reserve(headers_initial_capacity); } - const Headers &headers() const { return headers_; } - const Headers &trailers() const { return trailers_; } + const HeaderRefs &headers() const { return headers_; } + const HeaderRefs &trailers() const { return trailers_; } - Headers &headers() { return headers_; } + HeaderRefs &headers() { return headers_; } const void add_extra_buffer_size(size_t n) { buffer_size_ += n; } size_t buffer_size() const { return buffer_size_; } @@ -73,11 +75,11 @@ public: // multiple header have |name| as name, return last occurrence from // the beginning. If no such header is found, returns nullptr. // This function must be called after headers are indexed - const Headers::value_type *header(int32_t token) const; - Headers::value_type *header(int32_t token); + const HeaderRefs::value_type *header(int32_t token) const; + HeaderRefs::value_type *header(int32_t token); // Returns pointer to the header field with the name |name|. If no // such header is found, returns nullptr. - const Headers::value_type *header(const StringRef &name) const; + const HeaderRefs::value_type *header(const StringRef &name) const; void add_header_lower(const StringRef &name, const StringRef &value, bool no_index); @@ -110,10 +112,11 @@ public: int64_t content_length; private: - Headers headers_; + DefaultBlockAllocator &balloc_; + HeaderRefs headers_; // trailer fields. For HTTP/1.1, trailer fields are only included // with chunked encoding. For HTTP/2, there is no such limit. - Headers trailers_; + HeaderRefs trailers_; // Sum of the length of name and value in headers_ and trailers_. // This could also be increased by add_extra_buffer_size() to take // into account for request URI in case of HTTP/1.x request. @@ -123,8 +126,8 @@ private: }; struct Request { - Request() - : fs(16), + Request(DefaultBlockAllocator &balloc) + : fs(balloc, 16), recv_body_length(0), unconsumed_body_length(0), method(-1), @@ -179,8 +182,8 @@ struct Request { }; struct Response { - Response() - : fs(32), + Response(DefaultBlockAllocator &balloc) + : fs(balloc, 32), recv_body_length(0), unconsumed_body_length(0), http_status(0), @@ -245,7 +248,7 @@ public: // Returns true if the request is HTTP Upgrade for HTTP/2 bool get_http2_upgrade_request() const; // Returns the value of HTTP2-Settings request header field. - const std::string &get_http2_settings() const; + StringRef get_http2_settings() const; // downstream request API const Request &request() const { return req_; } @@ -373,6 +376,8 @@ public: DefaultMemchunks pop_response_buf(); + DefaultBlockAllocator &get_block_allocator(); + enum { EVENT_ERROR = 0x1, EVENT_TIMEOUT = 0x2, @@ -392,6 +397,8 @@ public: int64_t response_sent_body_length; private: + DefaultBlockAllocator balloc_; + Request req_; Response resp_; diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index 028b6e41..357bd664 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -364,7 +364,7 @@ int Http2DownstreamConnection::push_request_headers() { StringRef{req.authority}, StringRef{req.scheme}); if (fwd || !value.empty()) { if (fwd) { - forwarded_value = fwd->value; + forwarded_value = fwd->value.str(); if (!value.empty()) { forwarded_value += ", "; @@ -377,7 +377,7 @@ int Http2DownstreamConnection::push_request_headers() { } } else if (fwd) { nva.push_back(http2::make_nv_ls_nocopy("forwarded", fwd->value)); - forwarded_value = fwd->value; + forwarded_value = fwd->value.str(); } auto &xffconf = httpconf.xff; @@ -389,7 +389,7 @@ int Http2DownstreamConnection::push_request_headers() { if (xffconf.add) { if (xff) { - xff_value = (*xff).value; + xff_value = (*xff).value.str(); xff_value += ", "; } xff_value += upstream->get_client_handler()->get_ipaddr(); @@ -411,7 +411,7 @@ int Http2DownstreamConnection::push_request_headers() { } } else { if (via) { - via_value = (*via).value; + via_value = (*via).value.str(); via_value += ", "; } via_value += http::create_via_header_value(req.http_major, req.http_minor); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 9d42fe3a..cf89bf3e 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -108,7 +108,7 @@ int on_stream_close_callback(nghttp2_session *session, int32_t stream_id, int Http2Upstream::upgrade_upstream(HttpsUpstream *http) { int rv; - auto http2_settings = http->get_downstream()->get_http2_settings(); + auto http2_settings = http->get_downstream()->get_http2_settings().str(); util::to_base64(http2_settings); auto settings_payload = @@ -1434,7 +1434,7 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { } } else { if (via) { - via_value = (*via).value; + via_value = (*via).value.str(); via_value += ", "; } via_value += diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 49c9848e..92994207 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -325,7 +325,7 @@ int htp_hdrs_completecb(http_parser *htp) { } if (host) { - req.authority = host->value; + req.authority = host->value.str(); } if (handler->get_ssl()) { diff --git a/src/shrpx_mruby_module.cc b/src/shrpx_mruby_module.cc index e2f17fb1..27b7769c 100644 --- a/src/shrpx_mruby_module.cc +++ b/src/shrpx_mruby_module.cc @@ -85,7 +85,7 @@ mrb_value init_module(mrb_state *mrb) { return create_env(mrb); } -mrb_value create_headers_hash(mrb_state *mrb, const Headers &headers) { +mrb_value create_headers_hash(mrb_state *mrb, const HeaderRefs &headers) { auto hash = mrb_hash_new(mrb); for (auto &hd : headers) { diff --git a/src/shrpx_mruby_module.h b/src/shrpx_mruby_module.h index 3e417f88..a426bead 100644 --- a/src/shrpx_mruby_module.h +++ b/src/shrpx_mruby_module.h @@ -43,7 +43,7 @@ mrb_value init_module(mrb_state *mrb); void delete_downstream_from_module(mrb_state *mrb, Downstream *downstream); -mrb_value create_headers_hash(mrb_state *mrb, const Headers &headers); +mrb_value create_headers_hash(mrb_state *mrb, const HeaderRefs &headers); } // namespace mruby diff --git a/src/shrpx_mruby_module_response.cc b/src/shrpx_mruby_module_response.cc index 73dacccc..c50391e1 100644 --- a/src/shrpx_mruby_module_response.cc +++ b/src/shrpx_mruby_module_response.cc @@ -209,7 +209,8 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { auto cl = resp.fs.header(http2::HD_CONTENT_LENGTH); if (cl) { - cl->value = util::utos(bodylen); + cl->value = make_string_ref(downstream->get_block_allocator(), + StringRef{util::utos(bodylen)}); } else { resp.fs.add_header_token(StringRef::from_lit("content-length"), StringRef{util::utos(bodylen)}, false, diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index e15444a4..e5e1a096 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -262,12 +262,12 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, req.method = method_token; if (is_connect) { - req.authority = path->value; + req.authority = path->value.str(); } else { - req.scheme = scheme->value; - req.authority = host->value; + req.scheme = scheme->value.str(); + req.authority = host->value.str(); if (get_config()->http2_proxy) { - req.path = path->value; + req.path = path->value.str(); } else if (method_token == HTTP_OPTIONS && path->value == "*") { // Server-wide OPTIONS request. Path is empty. } else { @@ -1069,7 +1069,7 @@ int SpdyUpstream::on_downstream_header_complete(Downstream *downstream) { } } else { if (via) { - via_value = via->value; + via_value = via->value.str(); via_value += ", "; } via_value += diff --git a/src/template.h b/src/template.h index 6adcf0d5..9c52387f 100644 --- a/src/template.h +++ b/src/template.h @@ -443,6 +443,11 @@ private: size_type len; }; +inline bool operator==(const StringRef &lhs, const StringRef &rhs) { + return lhs.size() == rhs.size() && + std::equal(std::begin(lhs), std::end(lhs), std::begin(rhs)); +} + inline bool operator==(const StringRef &lhs, const std::string &rhs) { return lhs.size() == rhs.size() && std::equal(std::begin(lhs), std::end(lhs), std::begin(rhs)); @@ -477,6 +482,11 @@ inline bool operator!=(const char *lhs, const StringRef &rhs) { return !(rhs == lhs); } +inline bool operator<(const StringRef &lhs, const StringRef &rhs) { + return std::lexicographical_compare(std::begin(lhs), std::end(lhs), + std::begin(rhs), std::end(rhs)); +} + inline std::ostream &operator<<(std::ostream &o, const StringRef &s) { return o.write(s.c_str(), s.size()); } diff --git a/src/util.cc b/src/util.cc index 784806be..90304315 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1056,6 +1056,10 @@ int64_t parse_uint(const std::string &s) { return parse_uint(reinterpret_cast(s.c_str()), s.size()); } +int64_t parse_uint(const StringRef &s) { + return parse_uint(s.byte(), s.size()); +} + int64_t parse_uint(const uint8_t *s, size_t len) { int64_t n; size_t i; diff --git a/src/util.h b/src/util.h index d03ffd8e..ba3e1277 100644 --- a/src/util.h +++ b/src/util.h @@ -261,6 +261,11 @@ bool iends_with_l(const std::string &a, const CharT(&b)[N]) { return iends_with(std::begin(a), std::end(a), b, b + N - 1); } +template +bool iends_with_l(const StringRef &a, const CharT(&b)[N]) { + return iends_with(std::begin(a), std::end(a), b, b + N - 1); +} + int strcompare(const char *a, const uint8_t *b, size_t n); template bool strieq(const char *a, InputIt b, size_t bn) { @@ -310,6 +315,11 @@ bool strieq_l(const CharT(&a)[N], const std::string &b) { return strieq(a, N - 1, std::begin(b), b.size()); } +template +bool strieq_l(const CharT(&a)[N], const StringRef &b) { + return strieq(a, N - 1, std::begin(b), b.size()); +} + template bool streq(const char *a, InputIt b, size_t bn) { if (!a) { return false; @@ -609,6 +619,7 @@ int64_t parse_uint_with_unit(const char *s); int64_t parse_uint(const char *s); int64_t parse_uint(const uint8_t *s, size_t len); int64_t parse_uint(const std::string &s); +int64_t parse_uint(const StringRef &s); // Parses NULL terminated string |s| as unsigned integer and returns // the parsed integer casted to double. If |s| ends with "s", the From fa601e5ba3fb6af598e77923dbc21f340ffdfda9 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 9 Mar 2016 21:25:11 +0900 Subject: [PATCH 108/147] Add isolation_threshold, use field to store block size rather than template parameter --- src/allocator.h | 19 +++++++++++++------ src/shrpx_downstream.cc | 10 +++++----- src/shrpx_downstream.h | 12 ++++++------ 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/allocator.h b/src/allocator.h index 2f48f38b..998653be 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -36,8 +36,12 @@ struct MemBlock { uint8_t *begin, *last, *end; }; -template struct BlockAllocator { - BlockAllocator() : retain(nullptr), head(nullptr) {} +struct BlockAllocator { + BlockAllocator(size_t block_size, size_t isolation_threshold) + : retain(nullptr), + head(nullptr), + block_size(block_size), + isolation_threshold(std::min(block_size, isolation_threshold)) {} ~BlockAllocator() { for (auto mb = retain; mb;) { @@ -59,14 +63,14 @@ template struct BlockAllocator { } void *alloc(size_t size) { - if (size >= BLOCK_SIZE) { + if (size >= isolation_threshold) { auto mb = alloc_mem_block(size); mb->last = mb->end; return mb->begin; } if (!head || head->end - head->last < static_cast(size)) { - head = alloc_mem_block(BLOCK_SIZE); + head = alloc_mem_block(block_size); } auto res = head->last; @@ -79,6 +83,11 @@ template struct BlockAllocator { MemBlock *retain; MemBlock *head; + // size of single memory block + size_t block_size; + // if allocation greater or equal to isolation_threshold bytes is + // requested, allocate dedicated block. + size_t isolation_threshold; }; template @@ -101,8 +110,6 @@ StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, return StringRef{dst, a.size() + b.size()}; } -using DefaultBlockAllocator = BlockAllocator<1024>; - } // namespace aria2 #endif // ALLOCATOR_H diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 11462d7c..ed1edf2e 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -116,6 +116,7 @@ Downstream::Downstream(Upstream *upstream, MemchunkPool *mcpool, : dlnext(nullptr), dlprev(nullptr), response_sent_body_length(0), + balloc_(1024, 1024), req_(balloc_), resp_(balloc_), request_start_time_(std::chrono::high_resolution_clock::now()), @@ -357,9 +358,8 @@ void add_header(size_t &sum, HeaderRefs &headers, const StringRef &name, } // namespace namespace { -void append_last_header_key(DefaultBlockAllocator &balloc, bool &key_prev, - size_t &sum, HeaderRefs &headers, const char *data, - size_t len) { +void append_last_header_key(BlockAllocator &balloc, bool &key_prev, size_t &sum, + HeaderRefs &headers, const char *data, size_t len) { assert(key_prev); sum += len; auto &item = headers.back(); @@ -371,7 +371,7 @@ void append_last_header_key(DefaultBlockAllocator &balloc, bool &key_prev, } // namespace namespace { -void append_last_header_value(DefaultBlockAllocator &balloc, bool &key_prev, +void append_last_header_value(BlockAllocator &balloc, bool &key_prev, size_t &sum, HeaderRefs &headers, const char *data, size_t len) { key_prev = false; @@ -926,6 +926,6 @@ void Downstream::set_assoc_stream_id(int32_t stream_id) { int32_t Downstream::get_assoc_stream_id() const { return assoc_stream_id_; } -DefaultBlockAllocator &Downstream::get_block_allocator() { return balloc_; } +BlockAllocator &Downstream::get_block_allocator() { return balloc_; } } // namespace shrpx diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 7b9dfa83..2ea0bf2e 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -52,7 +52,7 @@ struct BlockedLink; class FieldStore { public: - FieldStore(DefaultBlockAllocator &balloc, size_t headers_initial_capacity) + FieldStore(BlockAllocator &balloc, size_t headers_initial_capacity) : content_length(-1), balloc_(balloc), buffer_size_(0), @@ -112,7 +112,7 @@ public: int64_t content_length; private: - DefaultBlockAllocator &balloc_; + BlockAllocator &balloc_; HeaderRefs headers_; // trailer fields. For HTTP/1.1, trailer fields are only included // with chunked encoding. For HTTP/2, there is no such limit. @@ -126,7 +126,7 @@ private: }; struct Request { - Request(DefaultBlockAllocator &balloc) + Request(BlockAllocator &balloc) : fs(balloc, 16), recv_body_length(0), unconsumed_body_length(0), @@ -182,7 +182,7 @@ struct Request { }; struct Response { - Response(DefaultBlockAllocator &balloc) + Response(BlockAllocator &balloc) : fs(balloc, 32), recv_body_length(0), unconsumed_body_length(0), @@ -376,7 +376,7 @@ public: DefaultMemchunks pop_response_buf(); - DefaultBlockAllocator &get_block_allocator(); + BlockAllocator &get_block_allocator(); enum { EVENT_ERROR = 0x1, @@ -397,7 +397,7 @@ public: int64_t response_sent_body_length; private: - DefaultBlockAllocator balloc_; + BlockAllocator balloc_; Request req_; Response resp_; From b1b57cc740e25d989f562d3dba5bab8cef3a7196 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 10 Mar 2016 22:42:07 +0900 Subject: [PATCH 109/147] nghttpx: Use StringRef for authority, scheme and path --- src/allocator.h | 13 ++ src/http2.cc | 272 ++++++++++++++++++++--- src/http2.h | 27 ++- src/shrpx_client_handler.cc | 37 ++- src/shrpx_client_handler.h | 2 +- src/shrpx_downstream.cc | 13 +- src/shrpx_downstream.h | 12 +- src/shrpx_downstream_queue.cc | 13 +- src/shrpx_downstream_queue.h | 6 +- src/shrpx_http2_downstream_connection.cc | 10 +- src/shrpx_http2_session.cc | 13 +- src/shrpx_http2_upstream.cc | 76 ++++--- src/shrpx_http2_upstream.h | 5 +- src/shrpx_http_downstream_connection.cc | 10 +- src/shrpx_https_upstream.cc | 99 ++++++--- src/shrpx_mruby_module_request.cc | 15 +- src/shrpx_mruby_module_response.cc | 18 +- src/shrpx_spdy_upstream.cc | 13 +- src/template.h | 3 +- src/util.h | 22 ++ 20 files changed, 511 insertions(+), 168 deletions(-) diff --git a/src/allocator.h b/src/allocator.h index 998653be..00acdbcf 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -27,6 +27,8 @@ #include "nghttp2_config.h" +#include + #include "template.h" namespace nghttp2 { @@ -110,6 +112,17 @@ StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, return StringRef{dst, a.size() + b.size()}; } +struct ByteRef { + uint8_t *base; + size_t len; +}; + +template +ByteRef make_byte_ref(BlockAllocator &alloc, size_t size) { + auto dst = static_cast(alloc.alloc(size)); + return {dst, size}; +} + } // namespace aria2 #endif // ALLOCATOR_H diff --git a/src/http2.cc b/src/http2.cc index 17de9b05..28314134 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -484,41 +484,69 @@ void dump_nv(FILE *out, const HeaderRefs &nva) { fflush(out); } -std::string rewrite_location_uri(const StringRef &uri, const http_parser_url &u, - const std::string &match_host, - const std::string &request_authority, - const std::string &upstream_scheme) { +StringRef rewrite_location_uri(BlockAllocator &balloc, const StringRef &uri, + const http_parser_url &u, + const StringRef &match_host, + const StringRef &request_authority, + const StringRef &upstream_scheme) { // We just rewrite scheme and authority. if ((u.field_set & (1 << UF_HOST)) == 0) { - return ""; + return StringRef{}; } auto field = &u.field_data[UF_HOST]; if (!util::starts_with(std::begin(match_host), std::end(match_host), &uri[field->off], &uri[field->off] + field->len) || (match_host.size() != field->len && match_host[field->len] != ':')) { - return ""; + return StringRef{}; } - std::string res; + + auto len = 0; if (!request_authority.empty()) { - res += upstream_scheme; - res += "://"; - res += request_authority; + len += upstream_scheme.size() + str_size("://") + request_authority.size(); + } + + if (u.field_set & (1 << UF_PATH)) { + field = &u.field_data[UF_PATH]; + len += field->len; + } + + if (u.field_set & (1 << UF_QUERY)) { + field = &u.field_data[UF_QUERY]; + len += 1 + field->len; + } + + if (u.field_set & (1 << UF_FRAGMENT)) { + field = &u.field_data[UF_FRAGMENT]; + len += 1 + field->len; + } + + auto iov = make_byte_ref(balloc, len + 1); + auto p = iov.base; + + if (!request_authority.empty()) { + p = std::copy(std::begin(upstream_scheme), std::end(upstream_scheme), p); + p = util::copy_lit(p, "://"); + p = std::copy(std::begin(request_authority), std::end(request_authority), + p); } if (u.field_set & (1 << UF_PATH)) { field = &u.field_data[UF_PATH]; - res.append(&uri[field->off], field->len); + p = std::copy_n(&uri[field->off], field->len, p); } if (u.field_set & (1 << UF_QUERY)) { field = &u.field_data[UF_QUERY]; - res += '?'; - res.append(&uri[field->off], field->len); + *p++ = '?'; + p = std::copy_n(&uri[field->off], field->len, p); } if (u.field_set & (1 << UF_FRAGMENT)) { field = &u.field_data[UF_FRAGMENT]; - res += '#'; - res.append(&uri[field->off], field->len); + *p++ = '#'; + p = std::copy_n(&uri[field->off], field->len, p); } - return res; + + *p = '\0'; + + return StringRef{iov.base, p}; } int check_nv(const uint8_t *name, size_t namelen, const uint8_t *value, @@ -1496,7 +1524,7 @@ StringRef to_method_string(int method_token) { return StringRef{http_method_str(static_cast(method_token))}; } -StringRef get_pure_path_component(const std::string &uri) { +StringRef get_pure_path_component(const StringRef &uri) { int rv; http_parser_url u{}; @@ -1513,9 +1541,9 @@ StringRef get_pure_path_component(const std::string &uri) { return StringRef::from_lit("/"); } -int construct_push_component(std::string &scheme, std::string &authority, - std::string &path, const StringRef &base, - const StringRef &uri) { +int construct_push_component(BlockAllocator &balloc, StringRef &scheme, + StringRef &authority, StringRef &path, + const StringRef &base, const StringRef &uri) { int rv; StringRef rel, relq; @@ -1538,15 +1566,26 @@ int construct_push_component(std::string &scheme, std::string &authority, } } else { if (u.field_set & (1 << UF_SCHEMA)) { - http2::copy_url_component(scheme, &u, UF_SCHEMA, uri.c_str()); + scheme = util::get_uri_field(uri.c_str(), u, UF_SCHEMA); } if (u.field_set & (1 << UF_HOST)) { - http2::copy_url_component(authority, &u, UF_HOST, uri.c_str()); - if (u.field_set & (1 << UF_PORT)) { - authority += ':'; - authority += util::utos(u.port); + auto auth = util::get_uri_field(uri.c_str(), u, UF_HOST); + auto len = auth.size(); + auto port_exists = u.field_set & (1 << UF_PORT); + if (port_exists) { + len += 1 + str_size("65535"); } + auto iov = make_byte_ref(balloc, len + 1); + auto p = iov.base; + p = std::copy(std::begin(auth), std::end(auth), p); + if (port_exists) { + *p++ = ':'; + p = util::utos(p, u.port); + } + *p = '\0'; + + authority = StringRef{iov.base, p}; } if (u.field_set & (1 << UF_PATH)) { @@ -1562,11 +1601,192 @@ int construct_push_component(std::string &scheme, std::string &authority, } } - path = http2::path_join(base, StringRef{}, rel, relq); + path = http2::path_join(balloc, base, StringRef{}, rel, relq); return 0; } +namespace { +template InputIt eat_file(InputIt first, InputIt last) { + if (first == last) { + *first++ = '/'; + return first; + } + + if (*(last - 1) == '/') { + return last; + } + + auto p = last; + for (; p != first && *(p - 1) != '/'; --p) + ; + if (p == first) { + // this should not happend in normal case, where we expect path + // starts with '/' + *first++ = '/'; + return first; + } + + return p; +} +} // namespace + +namespace { +template InputIt eat_dir(InputIt first, InputIt last) { + auto p = eat_file(first, last); + + --p; + + assert(*p == '/'); + + return eat_file(first, p); +} +} // namespace + +StringRef path_join(BlockAllocator &balloc, const StringRef &base_path, + const StringRef &base_query, const StringRef &rel_path, + const StringRef &rel_query) { + auto res = make_byte_ref( + balloc, std::max(static_cast(1), base_path.size()) + + rel_path.size() + 1 + + std::max(base_query.size(), rel_query.size()) + 1); + auto p = res.base; + + if (rel_path.empty()) { + if (base_path.empty()) { + *p++ = '/'; + } else { + p = std::copy(std::begin(base_path), std::end(base_path), p); + } + if (rel_query.empty()) { + if (!base_query.empty()) { + *p++ = '?'; + p = std::copy(std::begin(base_query), std::end(base_query), p); + } + *p = '\0'; + return StringRef{res.base, p}; + } + *p++ = '?'; + p = std::copy(std::begin(rel_query), std::end(rel_query), p); + *p = '\0'; + return StringRef{res.base, p}; + } + + auto first = std::begin(rel_path); + auto last = std::end(rel_path); + + if (rel_path[0] == '/') { + *p++ = '/'; + ++first; + } else if (base_path.empty()) { + *p++ = '/'; + } else { + p = std::copy(std::begin(base_path), std::end(base_path), p); + } + + for (; first != last;) { + if (*first == '.') { + if (first + 1 == last) { + break; + } + if (*(first + 1) == '/') { + first += 2; + continue; + } + if (*(first + 1) == '.') { + if (first + 2 == last) { + p = eat_dir(res.base, p); + break; + } + if (*(first + 2) == '/') { + p = eat_dir(res.base, p); + first += 3; + continue; + } + } + } + if (*(p - 1) != '/') { + p = eat_file(res.base, p); + } + auto slash = std::find(first, last, '/'); + if (slash == last) { + p = std::copy(first, last, p); + break; + } + p = std::copy(first, slash + 1, p); + first = slash + 1; + for (; first != last && *first == '/'; ++first) + ; + } + if (!rel_query.empty()) { + *p++ = '?'; + p = std::copy(std::begin(rel_query), std::end(rel_query), p); + } + *p = '\0'; + return StringRef{res.base, p}; +} + +StringRef normalize_path(BlockAllocator &balloc, const StringRef &path, + const StringRef &query) { + // First, decode %XX for unreserved characters, then do + // http2::join_path + + // We won't find %XX if length is less than 3. + if (path.size() < 3 || + std::find(std::begin(path), std::end(path), '%') == std::end(path)) { + return path_join(balloc, StringRef{}, StringRef{}, path, query); + } + + // includes last terminal NULL. + auto result = make_byte_ref(balloc, path.size() + 1); + auto p = result.base; + + auto it = std::begin(path); + for (; it + 2 != std::end(path);) { + if (*it == '%') { + if (util::is_hex_digit(*(it + 1)) && util::is_hex_digit(*(it + 2))) { + auto c = + (util::hex_to_uint(*(it + 1)) << 4) + util::hex_to_uint(*(it + 2)); + if (util::in_rfc3986_unreserved_chars(c)) { + *p++ = c; + + it += 3; + + continue; + } + *p++ = '%'; + *p++ = util::upcase(*(it + 1)); + *p++ = util::upcase(*(it + 2)); + + it += 3; + + continue; + } + } + *p++ = *it++; + } + + p = std::copy(it, std::end(path), p); + *p = '\0'; + + return path_join(balloc, StringRef{}, StringRef{}, StringRef{result.base, p}, + query); +} + +StringRef rewrite_clean_path(BlockAllocator &balloc, const StringRef &src) { + if (src.empty() || src[0] != '/') { + return src; + } + // probably, not necessary most of the case, but just in case. + auto fragment = std::find(std::begin(src), std::end(src), '#'); + auto query = std::find(std::begin(src), fragment, '?'); + if (query != fragment) { + ++query; + } + return normalize_path(balloc, StringRef{std::begin(src), query}, + StringRef{query, fragment}); +} + } // namespace http2 } // namespace nghttp2 diff --git a/src/http2.h b/src/http2.h index b4e84e8b..9b8b0bbb 100644 --- a/src/http2.h +++ b/src/http2.h @@ -40,6 +40,7 @@ #include "util.h" #include "memchunk.h" #include "template.h" +#include "allocator.h" namespace nghttp2 { @@ -241,10 +242,11 @@ void dump_nv(FILE *out, const HeaderRefs &nva); // This function returns the new rewritten URI on success. If the // location URI is not subject to the rewrite, this function returns // emtpy string. -std::string rewrite_location_uri(const StringRef &uri, const http_parser_url &u, - const std::string &match_host, - const std::string &request_authority, - const std::string &upstream_scheme); +StringRef rewrite_location_uri(BlockAllocator &balloc, const StringRef &uri, + const http_parser_url &u, + const StringRef &match_host, + const StringRef &request_authority, + const StringRef &upstream_scheme); // Checks the header name/value pair using nghttp2_check_header_name() // and nghttp2_check_header_value(). If both function returns nonzero, @@ -337,6 +339,10 @@ std::vector parse_link_header(const char *src, size_t len); std::string path_join(const StringRef &base, const StringRef &base_query, const StringRef &rel_path, const StringRef &rel_query); +StringRef path_join(BlockAllocator &balloc, const StringRef &base_path, + const StringRef &base_query, const StringRef &rel_path, + const StringRef &rel_query); + // true if response has body, taking into account the request method // and status code. bool expect_response_body(const std::string &method, int status_code); @@ -407,19 +413,24 @@ std::string rewrite_clean_path(InputIt first, InputIt last) { return path; } +StringRef normalize_path(BlockAllocator &balloc, const StringRef &path, + const StringRef &query); + +StringRef rewrite_clean_path(BlockAllocator &balloc, const StringRef &src); + // Returns path component of |uri|. The returned path does not // include query component. This function returns empty string if it // fails. -StringRef get_pure_path_component(const std::string &uri); +StringRef get_pure_path_component(const StringRef &uri); // Deduces scheme, authority and path from given |uri|, and stores // them in |scheme|, |authority|, and |path| respectively. If |uri| // is relative path, path resolution takes place using path given in // |base| of length |baselen|. This function returns 0 if it // succeeds, or -1. -int construct_push_component(std::string &scheme, std::string &authority, - std::string &path, const StringRef &base, - const StringRef &uri); +int construct_push_component(BlockAllocator &balloc, StringRef &scheme, + StringRef &authority, StringRef &path, + const StringRef &base, const StringRef &uri); } // namespace http2 diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 3326e885..d7366b04 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -825,11 +825,11 @@ int ClientHandler::perform_http2_upgrade(HttpsUpstream *http) { bool ClientHandler::get_http2_upgrade_allowed() const { return !conn_.tls.ssl; } -std::string ClientHandler::get_upstream_scheme() const { +StringRef ClientHandler::get_upstream_scheme() const { if (conn_.tls.ssl) { - return "https"; + return StringRef::from_lit("https"); } else { - return "http"; + return StringRef::from_lit("http"); } } @@ -843,24 +843,35 @@ namespace { // is mostly same routine found in // HttpDownstreamConnection::push_request_headers(), but vastly // simplified since we only care about absolute URI. -std::string construct_absolute_request_uri(const Request &req) { +StringRef construct_absolute_request_uri(BlockAllocator &balloc, + const Request &req) { if (req.authority.empty()) { return req.path; } - std::string uri; + auto len = req.authority.size() + req.path.size(); + if (req.scheme.empty()) { + len += str_size("http://"); + } else { + len += req.scheme.size() + str_size("://"); + } + + auto iov = make_byte_ref(balloc, len + 1); + auto p = iov.base; + if (req.scheme.empty()) { // We may have to log the request which lacks scheme (e.g., // http/1.1 with origin form). - uri += "http://"; + p = util::copy_lit(p, "http://"); } else { - uri += req.scheme; - uri += "://"; + p = std::copy(std::begin(req.scheme), std::end(req.scheme), p); + p = util::copy_lit(p, "://"); } - uri += req.authority; - uri += req.path; + p = std::copy(std::begin(req.authority), std::end(req.authority), p); + p = std::copy(std::begin(req.path), std::end(req.path), p); + *p = '\0'; - return uri; + return StringRef{iov.base, p}; } } // namespace @@ -869,6 +880,8 @@ void ClientHandler::write_accesslog(Downstream *downstream) { const auto &req = downstream->request(); const auto &resp = downstream->response(); + auto &balloc = downstream->get_block_allocator(); + upstream_accesslog( get_config()->logging.access.format, LogSpec{ @@ -877,7 +890,7 @@ void ClientHandler::write_accesslog(Downstream *downstream) { req.method == HTTP_CONNECT ? StringRef(req.authority) : get_config()->http2_proxy - ? StringRef(construct_absolute_request_uri(req)) + ? StringRef(construct_absolute_request_uri(balloc, req)) : req.path.empty() ? req.method == HTTP_OPTIONS ? StringRef::from_lit("*") diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index ec353576..8d1bc7c0 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -109,7 +109,7 @@ public: int perform_http2_upgrade(HttpsUpstream *http); bool get_http2_upgrade_allowed() const; // Returns upstream scheme, either "http" or "https" - std::string get_upstream_scheme() const; + StringRef get_upstream_scheme() const; void start_immediate_shutdown(); // Writes upstream accesslog using |downstream|. The |downstream| diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index ed1edf2e..f5e4f084 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -561,7 +561,7 @@ int Downstream::end_upload_data() { } void Downstream::rewrite_location_response_header( - const std::string &upstream_scheme) { + const StringRef &upstream_scheme) { auto hd = resp_.fs.header(http2::HD_LOCATION); if (!hd) { return; @@ -577,14 +577,15 @@ void Downstream::rewrite_location_response_header( return; } - auto new_uri = http2::rewrite_location_uri( - hd->value, u, request_downstream_host_, req_.authority, upstream_scheme); + auto new_uri = http2::rewrite_location_uri(balloc_, hd->value, u, + request_downstream_host_, + req_.authority, upstream_scheme); if (new_uri.empty()) { return; } - hd->value = make_string_ref(balloc_, StringRef{new_uri}); + hd->value = new_uri; } bool Downstream::get_chunked_response() const { return chunked_response_; } @@ -879,8 +880,8 @@ void Downstream::add_retry() { ++num_retry_; } bool Downstream::no_more_retry() const { return num_retry_ > 5; } -void Downstream::set_request_downstream_host(std::string host) { - request_downstream_host_ = std::move(host); +void Downstream::set_request_downstream_host(const StringRef &host) { + request_downstream_host_ = host; } void Downstream::set_request_pending(bool f) { request_pending_ = f; } diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 2ea0bf2e..cea17d86 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -147,17 +147,17 @@ struct Request { FieldStore fs; // Request scheme. For HTTP/2, this is :scheme header field value. // For HTTP/1.1, this is deduced from URI or connection. - std::string scheme; + StringRef scheme; // Request authority. This is HTTP/2 :authority header field value // or host header field value. We may deduce it from absolute-form // HTTP/1 request. We also store authority-form HTTP/1 request. // This could be empty if request comes from HTTP/1.0 without Host // header field and origin-form. - std::string authority; + StringRef authority; // Request path, including query component. For HTTP/1.1, this is // request-target. For HTTP/2, this is :path header field value. // For CONNECT request, this is empty. - std::string path; + StringRef path; // the length of request body received so far int64_t recv_body_length; // The number of bytes not consumed by the application yet. @@ -275,7 +275,7 @@ public: // Validates that received request body length and content-length // matches. bool validate_request_recv_body_length() const; - void set_request_downstream_host(std::string host); + void set_request_downstream_host(const StringRef &host); bool expect_response_body() const; enum { INITIAL, @@ -306,7 +306,7 @@ public: Response &response() { return resp_; } // Rewrites the location response header field. - void rewrite_location_response_header(const std::string &upstream_scheme); + void rewrite_location_response_header(const StringRef &upstream_scheme); bool get_chunked_response() const; void set_chunked_response(bool f); @@ -407,7 +407,7 @@ private: // host we requested to downstream. This is used to rewrite // location header field to decide the location should be rewritten // or not. - std::string request_downstream_host_; + StringRef request_downstream_host_; DefaultMemchunks request_buf_; DefaultMemchunks response_buf_; diff --git a/src/shrpx_downstream_queue.cc b/src/shrpx_downstream_queue.cc index 893effb9..91e88cff 100644 --- a/src/shrpx_downstream_queue.cc +++ b/src/shrpx_downstream_queue.cc @@ -71,14 +71,11 @@ DownstreamQueue::find_host_entry(const std::string &host) { return (*itr).second; } -const std::string & -DownstreamQueue::make_host_key(const std::string &host) const { - static std::string empty_key; - return unified_host_ ? empty_key : host; +std::string DownstreamQueue::make_host_key(const StringRef &host) const { + return unified_host_ ? "" : host.str(); } -const std::string & -DownstreamQueue::make_host_key(Downstream *downstream) const { +std::string DownstreamQueue::make_host_key(Downstream *downstream) const { return make_host_key(downstream->request().authority); } @@ -99,7 +96,7 @@ void DownstreamQueue::mark_blocked(Downstream *downstream) { ent.blocked.append(link); } -bool DownstreamQueue::can_activate(const std::string &host) const { +bool DownstreamQueue::can_activate(const StringRef &host) const { auto itr = host_entries_.find(make_host_key(host)); if (itr == std::end(host_entries_)) { return true; @@ -127,7 +124,7 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream, downstreams_.remove(downstream); - auto &host = make_host_key(downstream); + auto host = make_host_key(downstream); auto &ent = find_host_entry(host); if (downstream->get_dispatch_state() == Downstream::DISPATCH_ACTIVE) { diff --git a/src/shrpx_downstream_queue.h b/src/shrpx_downstream_queue.h index 755af10d..47e8555a 100644 --- a/src/shrpx_downstream_queue.h +++ b/src/shrpx_downstream_queue.h @@ -77,7 +77,7 @@ public: void mark_blocked(Downstream *downstream); // Returns true if we can make downstream connection to given // |host|. - bool can_activate(const std::string &host) const; + bool can_activate(const StringRef &host) const; // Removes and frees |downstream| object. If |downstream| is in // Downstream::DISPATCH_ACTIVE, and |next_blocked| is true, this // function may return Downstream object with the same target host @@ -87,8 +87,8 @@ public: bool next_blocked = true); Downstream *get_downstreams() const; HostEntry &find_host_entry(const std::string &host); - const std::string &make_host_key(const std::string &host) const; - const std::string &make_host_key(Downstream *downstream) const; + std::string make_host_key(const StringRef &host) const; + std::string make_host_key(Downstream *downstream) const; private: // Per target host structure to keep track of the number of diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index 357bd664..5e2d5129 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -282,10 +282,10 @@ int Http2DownstreamConnection::push_request_headers() { auto authority = StringRef(downstream_hostport); if (no_host_rewrite && !req.authority.empty()) { - authority = StringRef(req.authority); + authority = req.authority; } - downstream_->set_request_downstream_host(authority.str()); + downstream_->set_request_downstream_host(authority); size_t num_cookies = 0; if (!http2conf.no_cookie_crumbling) { @@ -359,9 +359,9 @@ int Http2DownstreamConnection::push_request_headers() { params &= ~FORWARDED_PROTO; } - auto value = http::create_forwarded( - params, handler->get_forwarded_by(), handler->get_forwarded_for(), - StringRef{req.authority}, StringRef{req.scheme}); + auto value = http::create_forwarded(params, handler->get_forwarded_by(), + handler->get_forwarded_for(), + req.authority, req.scheme); if (fwd || !value.empty()) { if (fwd) { forwarded_value = fwd->value.str(); diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 9a411ca1..828d888c 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -1986,6 +1986,8 @@ int Http2Session::handle_downstream_push_promise_complete( Downstream *downstream, Downstream *promised_downstream) { auto &promised_req = promised_downstream->request(); + auto &promised_balloc = promised_downstream->get_block_allocator(); + auto authority = promised_req.fs.header(http2::HD__AUTHORITY); auto path = promised_req.fs.header(http2::HD__PATH); auto method = promised_req.fs.header(http2::HD__METHOD); @@ -2007,16 +2009,19 @@ int Http2Session::handle_downstream_push_promise_complete( // TODO Rewrite authority if we enabled rewrite host. But we // really don't know how to rewrite host. Should we use the same // host in associated stream? - promised_req.authority = http2::value_to_str(authority); + if (authority) { + promised_req.authority = authority->value; + } promised_req.method = method_token; // libnghttp2 ensures that we don't have CONNECT method in // PUSH_PROMISE, and guarantees that :scheme exists. - promised_req.scheme = http2::value_to_str(scheme); + if (scheme) { + promised_req.scheme = scheme->value; + } // For server-wide OPTIONS request, path is empty. if (method_token != HTTP_OPTIONS || path->value != "*") { - promised_req.path = http2::rewrite_clean_path(std::begin(path->value), - std::end(path->value)); + promised_req.path = http2::rewrite_clean_path(promised_balloc, path->value); } promised_downstream->inspect_http2_request(); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index cf89bf3e..5bfdb826 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -303,7 +303,9 @@ int Http2Upstream::on_request_headers(Downstream *downstream, } req.method = method_token; - req.scheme = http2::value_to_str(scheme); + if (scheme) { + req.scheme = scheme->value; + } // nghttp2 library guarantees either :authority or host exist if (!authority) { @@ -311,16 +313,18 @@ int Http2Upstream::on_request_headers(Downstream *downstream, authority = req.fs.header(http2::HD_HOST); } - req.authority = http2::value_to_str(authority); + if (authority) { + req.authority = authority->value; + } if (path) { if (method_token == HTTP_OPTIONS && path->value == "*") { // Server-wide OPTIONS request. Path is empty. } else if (get_config()->http2_proxy) { - req.path = http2::value_to_str(path); + req.path = path->value; } else { - const auto &value = path->value; - req.path = http2::rewrite_clean_path(std::begin(value), std::end(value)); + req.path = http2::rewrite_clean_path(downstream->get_block_allocator(), + path->value); } } @@ -580,6 +584,8 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame, req.http_major = 2; req.http_minor = 0; + auto &promised_balloc = promised_downstream->get_block_allocator(); + for (size_t i = 0; i < frame->push_promise.nvlen; ++i) { auto &nv = frame->push_promise.nva[i]; auto token = http2::lookup_token(nv.name, nv.namelen); @@ -588,13 +594,16 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame, req.method = http2::lookup_method_token(nv.value, nv.valuelen); break; case http2::HD__SCHEME: - req.scheme.assign(nv.value, nv.value + nv.valuelen); + req.scheme = + make_string_ref(promised_balloc, StringRef{nv.value, nv.valuelen}); break; case http2::HD__AUTHORITY: - req.authority.assign(nv.value, nv.value + nv.valuelen); + req.authority = + make_string_ref(promised_balloc, StringRef{nv.value, nv.valuelen}); break; case http2::HD__PATH: - req.path = http2::rewrite_clean_path(nv.value, nv.value + nv.valuelen); + req.path = http2::rewrite_clean_path(promised_balloc, + StringRef{nv.value, nv.valuelen}); break; } req.fs.add_header_token(StringRef{nv.name, nv.namelen}, @@ -1746,6 +1755,8 @@ int Http2Upstream::prepare_push_promise(Downstream *downstream) { return 0; } + auto &balloc = downstream->get_block_allocator(); + for (auto &kv : resp.fs.headers()) { if (kv.token != http2::HD_LINK) { continue; @@ -1753,28 +1764,23 @@ int Http2Upstream::prepare_push_promise(Downstream *downstream) { for (auto &link : http2::parse_link_header(kv.value.c_str(), kv.value.size())) { - const std::string *scheme_ptr, *authority_ptr; - std::string scheme, authority, path; + StringRef scheme, authority, path; - rv = http2::construct_push_component(scheme, authority, path, base, - link.uri); + rv = http2::construct_push_component(balloc, scheme, authority, path, + base, link.uri); if (rv != 0) { continue; } if (scheme.empty()) { - scheme_ptr = &req.scheme; - } else { - scheme_ptr = &scheme; + scheme = req.scheme; } if (authority.empty()) { - authority_ptr = &req.authority; - } else { - authority_ptr = &authority; + authority = req.authority; } - rv = submit_push_promise(*scheme_ptr, *authority_ptr, path, downstream); + rv = submit_push_promise(scheme, authority, path, downstream); if (rv != 0) { return -1; } @@ -1783,9 +1789,9 @@ int Http2Upstream::prepare_push_promise(Downstream *downstream) { return 0; } -int Http2Upstream::submit_push_promise(const std::string &scheme, - const std::string &authority, - const std::string &path, +int Http2Upstream::submit_push_promise(const StringRef &scheme, + const StringRef &authority, + const StringRef &path, Downstream *downstream) { const auto &req = downstream->request(); @@ -1795,9 +1801,9 @@ int Http2Upstream::submit_push_promise(const std::string &scheme, // juse use "GET" for now nva.push_back(http2::make_nv_ll(":method", "GET")); - nva.push_back(http2::make_nv_ls(":scheme", scheme)); - nva.push_back(http2::make_nv_ls(":path", path)); - nva.push_back(http2::make_nv_ls(":authority", authority)); + nva.push_back(http2::make_nv_ls_nocopy(":scheme", scheme)); + nva.push_back(http2::make_nv_ls_nocopy(":path", path)); + nva.push_back(http2::make_nv_ls_nocopy(":authority", authority)); for (auto &kv : req.fs.headers()) { switch (kv.token) { @@ -1865,27 +1871,25 @@ int Http2Upstream::initiate_push(Downstream *downstream, const StringRef &uri) { return -1; } - const std::string *scheme_ptr, *authority_ptr; - std::string scheme, authority, path; + auto &balloc = downstream->get_block_allocator(); - rv = http2::construct_push_component(scheme, authority, path, base, uri); + StringRef scheme, authority, path; + + rv = http2::construct_push_component(balloc, scheme, authority, path, base, + uri); if (rv != 0) { return -1; } if (scheme.empty()) { - scheme_ptr = &req.scheme; - } else { - scheme_ptr = &scheme; + scheme = req.scheme; } if (authority.empty()) { - authority_ptr = &req.authority; - } else { - authority_ptr = &authority; + authority = req.authority; } - rv = submit_push_promise(*scheme_ptr, *authority_ptr, path, downstream); + rv = submit_push_promise(scheme, authority, path, downstream); if (rv != 0) { return -1; @@ -1943,7 +1947,7 @@ int Http2Upstream::on_downstream_push_promise_complete( nva.reserve(headers.size()); for (auto &kv : headers) { - nva.push_back(http2::make_nv_nocopy(kv.name, kv.value, kv.no_index)); + nva.push_back(http2::make_nv(kv.name, kv.value, kv.no_index)); } auto promised_stream_id = nghttp2_submit_push_promise( diff --git a/src/shrpx_http2_upstream.h b/src/shrpx_http2_upstream.h index bf1e1c28..1908a504 100644 --- a/src/shrpx_http2_upstream.h +++ b/src/shrpx_http2_upstream.h @@ -111,9 +111,8 @@ public: void check_shutdown(); int prepare_push_promise(Downstream *downstream); - int submit_push_promise(const std::string &scheme, - const std::string &authority, const std::string &path, - Downstream *downstream); + int submit_push_promise(const StringRef &scheme, const StringRef &authority, + const StringRef &path, Downstream *downstream); int on_request_headers(Downstream *downstream, const nghttp2_frame *frame); diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index a6c238a1..d7f5492b 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -283,10 +283,10 @@ int HttpDownstreamConnection::push_request_headers() { httpconf.no_host_rewrite || get_config()->http2_proxy || connect_method; if (no_host_rewrite && !req.authority.empty()) { - authority = StringRef(req.authority); + authority = req.authority; } - downstream_->set_request_downstream_host(authority.str()); + downstream_->set_request_downstream_host(authority); auto buf = downstream_->get_request_buf(); @@ -366,9 +366,9 @@ int HttpDownstreamConnection::push_request_headers() { params &= ~FORWARDED_PROTO; } - auto value = http::create_forwarded( - params, handler->get_forwarded_by(), handler->get_forwarded_for(), - StringRef{req.authority}, StringRef{req.scheme}); + auto value = http::create_forwarded(params, handler->get_forwarded_by(), + handler->get_forwarded_for(), + req.authority, req.scheme); if (fwd || !value.empty()) { buf->append("Forwarded: "); if (fwd) { diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 92994207..73e91a67 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -86,6 +86,8 @@ int htp_uricb(http_parser *htp, const char *data, size_t len) { auto downstream = upstream->get_downstream(); auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + // We happen to have the same value for method token. req.method = htp->method; @@ -103,9 +105,10 @@ int htp_uricb(http_parser *htp, const char *data, size_t len) { req.fs.add_extra_buffer_size(len); if (req.method == HTTP_CONNECT) { - req.authority.append(data, len); + req.authority = + concat_string_ref(balloc, req.authority, StringRef{data, len}); } else { - req.path.append(data, len); + req.path = concat_string_ref(balloc, req.path, StringRef{data, len}); } return 0; @@ -190,30 +193,51 @@ int htp_hdr_valcb(http_parser *htp, const char *data, size_t len) { } // namespace namespace { -void rewrite_request_host_path_from_uri(Request &req, const char *uri, +void rewrite_request_host_path_from_uri(BlockAllocator &balloc, Request &req, + const StringRef &uri, http_parser_url &u) { assert(u.field_set & (1 << UF_HOST)); - auto &authority = req.authority; - authority.clear(); // As per https://tools.ietf.org/html/rfc7230#section-5.4, we // rewrite host header field with authority component. - http2::copy_url_component(authority, &u, UF_HOST, uri); + auto authority = util::get_uri_field(uri.c_str(), u, UF_HOST); // TODO properly check IPv6 numeric address - if (authority.find(':') != std::string::npos) { - authority = '[' + authority; - authority += ']'; + auto ipv6 = std::find(std::begin(authority), std::end(authority), ':') != + std::end(authority); + auto authoritylen = authority.size(); + if (ipv6) { + authoritylen += 2; } if (u.field_set & (1 << UF_PORT)) { - authority += ':'; - authority += util::utos(u.port); + authoritylen += 1 + str_size("65535"); + } + if (authoritylen > authority.size()) { + auto iovec = make_byte_ref(balloc, authoritylen + 1); + auto p = iovec.base; + if (ipv6) { + *p++ = '['; + } + p = std::copy(std::begin(authority), std::end(authority), p); + if (ipv6) { + *p++ = ']'; + } + + if (u.field_set & (1 << UF_PORT)) { + *p++ = ':'; + p = util::utos(p, u.port); + } + *p = '\0'; + + req.authority = StringRef{iovec.base, p}; + } else { + req.authority = authority; } - http2::copy_url_component(req.scheme, &u, UF_SCHEMA, uri); + req.scheme = util::get_uri_field(uri.c_str(), u, UF_SCHEMA); - std::string path; + StringRef path; if (u.field_set & (1 << UF_PATH)) { - http2::copy_url_component(path, &u, UF_PATH, uri); + path = util::get_uri_field(uri.c_str(), u, UF_PATH); } else if (req.method == HTTP_OPTIONS) { // Server-wide OPTIONS takes following form in proxy request: // @@ -221,21 +245,35 @@ void rewrite_request_host_path_from_uri(Request &req, const char *uri, // // Notice that no slash after authority. See // http://tools.ietf.org/html/rfc7230#section-5.3.4 - req.path = ""; + req.path = StringRef::from_lit(""); // we ignore query component here return; } else { - path = "/"; + path = StringRef::from_lit("/"); } + if (u.field_set & (1 << UF_QUERY)) { auto &fdata = u.field_data[UF_QUERY]; - path += '?'; - path.append(uri + fdata.off, fdata.len); + + if (u.field_set & (1 << UF_PATH)) { + auto q = util::get_uri_field(uri.c_str(), u, UF_QUERY); + path = StringRef{std::begin(path), std::end(q)}; + } else { + auto iov = make_byte_ref(balloc, path.size() + 1 + fdata.len + 1); + auto p = iov.base; + + p = std::copy(std::begin(path), std::end(path), p); + *p++ = '?'; + p = std::copy_n(&uri[fdata.off], fdata.len, p); + *p = '\0'; + path = StringRef{iov.base, p}; + } } + if (get_config()->http2_proxy) { - req.path = std::move(path); + req.path = path; } else { - req.path = http2::rewrite_clean_path(std::begin(path), std::end(path)); + req.path = http2::rewrite_clean_path(balloc, path); } } } // namespace @@ -299,12 +337,11 @@ int htp_hdrs_completecb(http_parser *htp) { downstream->inspect_http1_request(); + auto &balloc = downstream->get_block_allocator(); + if (method != HTTP_CONNECT) { http_parser_url u{}; - // make a copy of request path, since we may set request path - // while we are refering to original request path. - auto path = req.path; - rv = http_parser_parse_url(path.c_str(), path.size(), 0, &u); + rv = http_parser_parse_url(req.path.c_str(), req.path.size(), 0, &u); if (rv != 0) { // Expect to respond with 400 bad request return -1; @@ -318,23 +355,23 @@ int htp_hdrs_completecb(http_parser *htp) { req.no_authority = true; - if (method == HTTP_OPTIONS && path == "*") { - req.path = ""; + if (method == HTTP_OPTIONS && req.path == "*") { + req.path = StringRef{}; } else { - req.path = http2::rewrite_clean_path(std::begin(path), std::end(path)); + req.path = http2::rewrite_clean_path(balloc, req.path); } if (host) { - req.authority = host->value.str(); + req.authority = host->value; } if (handler->get_ssl()) { - req.scheme = "https"; + req.scheme = StringRef::from_lit("https"); } else { - req.scheme = "http"; + req.scheme = StringRef::from_lit("http"); } } else { - rewrite_request_host_path_from_uri(req, path.c_str(), u); + rewrite_request_host_path_from_uri(balloc, req, req.path, u); } } diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index 74201ff2..bf83eba2 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -116,6 +116,8 @@ mrb_value request_set_authority(mrb_state *mrb, mrb_value self) { auto downstream = data->downstream; auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + check_phase(mrb, data->phase, PHASE_REQUEST); const char *authority; @@ -125,7 +127,8 @@ mrb_value request_set_authority(mrb_state *mrb, mrb_value self) { mrb_raise(mrb, E_RUNTIME_ERROR, "authority must not be empty string"); } - req.authority.assign(authority, n); + req.authority = + make_string_ref(balloc, StringRef{authority, static_cast(n)}); return self; } @@ -147,6 +150,8 @@ mrb_value request_set_scheme(mrb_state *mrb, mrb_value self) { auto downstream = data->downstream; auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + check_phase(mrb, data->phase, PHASE_REQUEST); const char *scheme; @@ -156,7 +161,8 @@ mrb_value request_set_scheme(mrb_state *mrb, mrb_value self) { mrb_raise(mrb, E_RUNTIME_ERROR, "scheme must not be empty string"); } - req.scheme.assign(scheme, n); + req.scheme = + make_string_ref(balloc, StringRef{scheme, static_cast(n)}); return self; } @@ -178,13 +184,16 @@ mrb_value request_set_path(mrb_state *mrb, mrb_value self) { auto downstream = data->downstream; auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + check_phase(mrb, data->phase, PHASE_REQUEST); const char *path; mrb_int pathlen; mrb_get_args(mrb, "s", &path, &pathlen); - req.path.assign(path, pathlen); + req.path = + make_string_ref(balloc, StringRef{path, static_cast(pathlen)}); return self; } diff --git a/src/shrpx_mruby_module_response.cc b/src/shrpx_mruby_module_response.cc index c50391e1..ada76b7c 100644 --- a/src/shrpx_mruby_module_response.cc +++ b/src/shrpx_mruby_module_response.cc @@ -187,6 +187,8 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { auto &resp = downstream->response(); int rv; + auto &balloc = downstream->get_block_allocator(); + if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { mrb_raise(mrb, E_RUNTIME_ERROR, "response has already been committed"); } @@ -207,14 +209,22 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { bodylen = vallen; } + StringRef content_length; + { + auto iov = make_byte_ref(balloc, str_size("18446744073709551615") + 1); + auto p = iov.base; + p = util::utos(p, bodylen); + *p = '\0'; + content_length = StringRef{iov.base, p}; + } + auto cl = resp.fs.header(http2::HD_CONTENT_LENGTH); if (cl) { - cl->value = make_string_ref(downstream->get_block_allocator(), - StringRef{util::utos(bodylen)}); + cl->value = content_length; } else { + // TODO we don't have to make a copy here. resp.fs.add_header_token(StringRef::from_lit("content-length"), - StringRef{util::utos(bodylen)}, false, - http2::HD_CONTENT_LENGTH); + content_length, false, http2::HD_CONTENT_LENGTH); } resp.fs.content_length = bodylen; diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index e5e1a096..89508ba8 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -159,6 +159,8 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + downstream->reset_upstream_rtimer(); auto nv = frame->syn_stream.nv; @@ -262,17 +264,16 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, req.method = method_token; if (is_connect) { - req.authority = path->value.str(); + req.authority = path->value; } else { - req.scheme = scheme->value.str(); - req.authority = host->value.str(); + req.scheme = scheme->value; + req.authority = host->value; if (get_config()->http2_proxy) { - req.path = path->value.str(); + req.path = path->value; } else if (method_token == HTTP_OPTIONS && path->value == "*") { // Server-wide OPTIONS request. Path is empty. } else { - req.path = http2::rewrite_clean_path(std::begin(path->value), - std::end(path->value)); + req.path = http2::rewrite_clean_path(balloc, path->value); } } diff --git a/src/template.h b/src/template.h index 9c52387f..b15efa02 100644 --- a/src/template.h +++ b/src/template.h @@ -409,7 +409,8 @@ public: : base(&*first), len(std::distance(first, last)) {} template StringRef(InputIt *first, InputIt *last) - : base(first), len(std::distance(first, last)) {} + : base(reinterpret_cast(first)), + len(std::distance(first, last)) {} template constexpr static StringRef from_lit(const CharT(&s)[N]) { return StringRef(s, N - 1); diff --git a/src/util.h b/src/util.h index ba3e1277..f24f8140 100644 --- a/src/util.h +++ b/src/util.h @@ -387,6 +387,23 @@ template std::string utos(T n) { return res; } +template OutputIt utos(OutputIt dst, T n) { + if (n == 0) { + *dst++ = '0'; + return dst; + } + int i = 0; + T t = n; + for (; t; t /= 10, ++i) + ; + --i; + auto p = dst + i; + for (; n; --i, n /= 10) { + *p-- = (n % 10) + '0'; + } + return dst + i + 1; +} + template std::string utos_unit(T n) { char u = 0; if (n >= (1 << 30)) { @@ -695,6 +712,11 @@ std::string random_alpha_digit(Generator &gen, size_t len) { return res; } +template +OutputIterator copy_lit(OutputIterator it, CharT(&s)[N]) { + return std::copy_n(s, N - 1, it); +} + } // namespace util } // namespace nghttp2 From 7a412df9a540dac5e3f9fbb9dad22b0ac36e6bd7 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 10 Mar 2016 23:50:04 +0900 Subject: [PATCH 110/147] nghttpx: Fix tests --- src/http2.cc | 8 +- src/http2.h | 51 +--------- src/http2_test.cc | 188 ++++++++++++++++++----------------- src/shrpx-unittest.cc | 1 + src/shrpx_config.cc | 3 +- src/shrpx_downstream.cc | 5 +- src/shrpx_downstream_test.cc | 55 +++++----- src/util.h | 3 +- src/util_test.cc | 9 ++ src/util_test.h | 1 + 10 files changed, 153 insertions(+), 171 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index 28314134..3a4474b8 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -1742,7 +1742,7 @@ StringRef normalize_path(BlockAllocator &balloc, const StringRef &path, auto p = result.base; auto it = std::begin(path); - for (; it + 2 != std::end(path);) { + for (; it + 2 < std::end(path);) { if (*it == '%') { if (util::is_hex_digit(*(it + 1)) && util::is_hex_digit(*(it + 2))) { auto c = @@ -1773,6 +1773,12 @@ StringRef normalize_path(BlockAllocator &balloc, const StringRef &path, query); } +std::string normalize_path(const StringRef &path, const StringRef &query) { + BlockAllocator balloc(1024, 1024); + + return normalize_path(balloc, path, query).str(); +} + StringRef rewrite_clean_path(BlockAllocator &balloc, const StringRef &src) { if (src.empty() || src[0] != '/') { return src; diff --git a/src/http2.h b/src/http2.h index 9b8b0bbb..cc295079 100644 --- a/src/http2.h +++ b/src/http2.h @@ -364,58 +364,11 @@ int lookup_method_token(const StringRef &name); // StringRef is guaranteed to be NULL-terminated. StringRef to_method_string(int method_token); -template -std::string normalize_path(InputIt first, InputIt last) { - // First, decode %XX for unreserved characters, then do - // http2::join_path - std::string result; - // We won't find %XX if length is less than 3. - if (last - first < 3) { - result.assign(first, last); - } else { - for (; first < last - 2;) { - if (*first == '%') { - if (util::is_hex_digit(*(first + 1)) && - util::is_hex_digit(*(first + 2))) { - auto c = (util::hex_to_uint(*(first + 1)) << 4) + - util::hex_to_uint(*(first + 2)); - if (util::in_rfc3986_unreserved_chars(c)) { - result += c; - first += 3; - continue; - } - result += '%'; - result += util::upcase(*(first + 1)); - result += util::upcase(*(first + 2)); - first += 3; - continue; - } - } - result += *first++; - } - result.append(first, last); - } - return path_join(StringRef{}, StringRef{}, StringRef{result}, StringRef{}); -} - -template -std::string rewrite_clean_path(InputIt first, InputIt last) { - if (first == last || *first != '/') { - return std::string(first, last); - } - // probably, not necessary most of the case, but just in case. - auto fragment = std::find(first, last, '#'); - auto query = std::find(first, fragment, '?'); - auto path = normalize_path(first, query); - if (query != fragment) { - path.append(query, fragment); - } - return path; -} - StringRef normalize_path(BlockAllocator &balloc, const StringRef &path, const StringRef &query); +std::string normalize_path(const StringRef &path, const StringRef &query); + StringRef rewrite_clean_path(BlockAllocator &balloc, const StringRef &src); // Returns path component of |uri|. The returned path does not diff --git a/src/http2_test.cc b/src/http2_test.cc index 4d4c7669..5ac726aa 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -46,7 +46,7 @@ using namespace nghttp2; namespace shrpx { namespace { -void check_nv(const Header &a, const nghttp2_nv *b) { +void check_nv(const HeaderRef &a, const nghttp2_nv *b) { CU_ASSERT(a.name.size() == b->namelen); CU_ASSERT(a.value.size() == b->valuelen); CU_ASSERT(memcmp(a.name.c_str(), b->name, b->namelen) == 0); @@ -134,20 +134,24 @@ void test_http2_get_header(void) { } namespace { -auto headers = - Headers{{"alpha", "0", true}, - {"bravo", "1"}, - {"connection", "2", false, http2::HD_CONNECTION}, - {"connection", "3", false, http2::HD_CONNECTION}, - {"delta", "4"}, - {"expect", "5"}, - {"foxtrot", "6"}, - {"tango", "7"}, - {"te", "8", false, http2::HD_TE}, - {"te", "9", false, http2::HD_TE}, - {"x-forwarded-proto", "10", false, http2::HD_X_FORWARDED_FOR}, - {"x-forwarded-proto", "11", false, http2::HD_X_FORWARDED_FOR}, - {"zulu", "12"}}; +auto headers = HeaderRefs{ + {StringRef::from_lit("alpha"), StringRef::from_lit("0"), true}, + {StringRef::from_lit("bravo"), StringRef::from_lit("1")}, + {StringRef::from_lit("connection"), StringRef::from_lit("2"), false, + http2::HD_CONNECTION}, + {StringRef::from_lit("connection"), StringRef::from_lit("3"), false, + http2::HD_CONNECTION}, + {StringRef::from_lit("delta"), StringRef::from_lit("4")}, + {StringRef::from_lit("expect"), StringRef::from_lit("5")}, + {StringRef::from_lit("foxtrot"), StringRef::from_lit("6")}, + {StringRef::from_lit("tango"), StringRef::from_lit("7")}, + {StringRef::from_lit("te"), StringRef::from_lit("8"), false, http2::HD_TE}, + {StringRef::from_lit("te"), StringRef::from_lit("9"), false, http2::HD_TE}, + {StringRef::from_lit("x-forwarded-proto"), StringRef::from_lit("10"), false, + http2::HD_X_FORWARDED_FOR}, + {StringRef::from_lit("x-forwarded-proto"), StringRef::from_lit("11"), false, + http2::HD_X_FORWARDED_FOR}, + {StringRef::from_lit("zulu"), StringRef::from_lit("12")}}; } // namespace void test_http2_copy_headers_to_nva(void) { @@ -209,10 +213,12 @@ void check_rewrite_location_uri(const std::string &want, const std::string &uri, const std::string &match_host, const std::string &req_authority, const std::string &upstream_scheme) { + BlockAllocator balloc(4096, 4096); http_parser_url u{}; CU_ASSERT(0 == http_parser_parse_url(uri.c_str(), uri.size(), 0, &u)); - auto got = http2::rewrite_location_uri(uri, u, match_host, req_authority, - upstream_scheme); + auto got = http2::rewrite_location_uri( + balloc, StringRef{uri}, u, StringRef{match_host}, + StringRef{req_authority}, StringRef{upstream_scheme}); CU_ASSERT(want == got); } } // namespace @@ -245,13 +251,13 @@ void test_http2_rewrite_location_uri(void) { } void test_http2_parse_http_status_code(void) { - CU_ASSERT(200 == http2::parse_http_status_code("200")); - CU_ASSERT(102 == http2::parse_http_status_code("102")); - CU_ASSERT(-1 == http2::parse_http_status_code("099")); - CU_ASSERT(-1 == http2::parse_http_status_code("99")); - CU_ASSERT(-1 == http2::parse_http_status_code("-1")); - CU_ASSERT(-1 == http2::parse_http_status_code("20a")); - CU_ASSERT(-1 == http2::parse_http_status_code("")); + CU_ASSERT(200 == http2::parse_http_status_code(StringRef::from_lit("200"))); + CU_ASSERT(102 == http2::parse_http_status_code(StringRef::from_lit("102"))); + CU_ASSERT(-1 == http2::parse_http_status_code(StringRef::from_lit("099"))); + CU_ASSERT(-1 == http2::parse_http_status_code(StringRef::from_lit("99"))); + CU_ASSERT(-1 == http2::parse_http_status_code(StringRef::from_lit("-1"))); + CU_ASSERT(-1 == http2::parse_http_status_code(StringRef::from_lit("20a"))); + CU_ASSERT(-1 == http2::parse_http_status_code(StringRef{})); } void test_http2_index_header(void) { @@ -814,137 +820,135 @@ void test_http2_path_join(void) { } void test_http2_normalize_path(void) { - std::string src; - - src = "/alpha/bravo/../charlie"; CU_ASSERT("/alpha/charlie" == - http2::normalize_path(std::begin(src), std::end(src))); + http2::normalize_path( + StringRef::from_lit("/alpha/bravo/../charlie"), StringRef{})); - src = "/a%6c%70%68%61"; - CU_ASSERT("/alpha" == http2::normalize_path(std::begin(src), std::end(src))); + CU_ASSERT("/alpha" == + http2::normalize_path(StringRef::from_lit("/a%6c%70%68%61"), + StringRef{})); - src = "/alpha%2f%3a"; - CU_ASSERT("/alpha%2F%3A" == - http2::normalize_path(std::begin(src), std::end(src))); + CU_ASSERT( + "/alpha%2F%3A" == + http2::normalize_path(StringRef::from_lit("/alpha%2f%3a"), StringRef{})); - src = "%2f"; - CU_ASSERT("/%2F" == http2::normalize_path(std::begin(src), std::end(src))); + CU_ASSERT("/%2F" == + http2::normalize_path(StringRef::from_lit("%2f"), StringRef{})); - src = "%f"; - CU_ASSERT("/%f" == http2::normalize_path(std::begin(src), std::end(src))); + CU_ASSERT("/%f" == + http2::normalize_path(StringRef::from_lit("%f"), StringRef{})); - src = "%"; - CU_ASSERT("/%" == http2::normalize_path(std::begin(src), std::end(src))); + CU_ASSERT("/%" == + http2::normalize_path(StringRef::from_lit("%"), StringRef{})); - src = ""; - CU_ASSERT("/" == http2::normalize_path(std::begin(src), std::end(src))); + CU_ASSERT("/" == http2::normalize_path(StringRef{}, StringRef{})); + + CU_ASSERT("/alpha?bravo" == + http2::normalize_path(StringRef::from_lit("/alpha"), + StringRef::from_lit("bravo"))); } void test_http2_rewrite_clean_path(void) { - std::string src; + BlockAllocator balloc(4096, 4096); // unreserved characters - src = "/alpha/%62ravo/"; CU_ASSERT("/alpha/bravo/" == - http2::rewrite_clean_path(std::begin(src), std::end(src))); + http2::rewrite_clean_path(balloc, + StringRef::from_lit("/alpha/%62ravo/"))); // percent-encoding is converted to upper case. - src = "/delta%3a"; - CU_ASSERT("/delta%3A" == - http2::rewrite_clean_path(std::begin(src), std::end(src))); + CU_ASSERT("/delta%3A" == http2::rewrite_clean_path( + balloc, StringRef::from_lit("/delta%3a"))); // path component is normalized before mathcing - src = "/alpha/charlie/%2e././bravo/delta/.."; - CU_ASSERT("/alpha/bravo/" == - http2::rewrite_clean_path(std::begin(src), std::end(src))); + CU_ASSERT( + "/alpha/bravo/" == + http2::rewrite_clean_path( + balloc, StringRef::from_lit("/alpha/charlie/%2e././bravo/delta/.."))); - src = "alpha%3a"; - CU_ASSERT(src == http2::rewrite_clean_path(std::begin(src), std::end(src))); + CU_ASSERT("alpha%3a" == + http2::rewrite_clean_path(balloc, StringRef::from_lit("alpha%3a"))); - src = ""; - CU_ASSERT(src == http2::rewrite_clean_path(std::begin(src), std::end(src))); + CU_ASSERT("" == http2::rewrite_clean_path(balloc, StringRef{})); } void test_http2_get_pure_path_component(void) { - std::string path; + CU_ASSERT("/" == http2::get_pure_path_component(StringRef::from_lit("/"))); - path = "/"; - CU_ASSERT("/" == http2::get_pure_path_component(path)); + CU_ASSERT("/foo" == + http2::get_pure_path_component(StringRef::from_lit("/foo"))); - path = "/foo"; - CU_ASSERT("/foo" == http2::get_pure_path_component(path)); + CU_ASSERT("/bar" == http2::get_pure_path_component( + StringRef::from_lit("https://example.org/bar"))); - path = "https://example.org/bar"; - CU_ASSERT("/bar" == http2::get_pure_path_component(path)); + CU_ASSERT("/alpha" == http2::get_pure_path_component(StringRef::from_lit( + "https://example.org/alpha?q=a"))); - path = "https://example.org/alpha?q=a"; - CU_ASSERT("/alpha" == http2::get_pure_path_component(path)); + CU_ASSERT("/bravo" == http2::get_pure_path_component(StringRef::from_lit( + "https://example.org/bravo?q=a#fragment"))); - path = "https://example.org/bravo?q=a#fragment"; - CU_ASSERT("/bravo" == http2::get_pure_path_component(path)); - - path = "\x01\x02"; - CU_ASSERT("" == http2::get_pure_path_component(path)); + CU_ASSERT("" == + http2::get_pure_path_component(StringRef::from_lit("\x01\x02"))); } void test_http2_construct_push_component(void) { + BlockAllocator balloc(4096, 4096); StringRef base, uri; - std::string scheme, authority, path; + StringRef scheme, authority, path; base = StringRef::from_lit("/b/"); - uri = StringRef::from_lit("https://example.org/foo"); - CU_ASSERT( - 0 == http2::construct_push_component(scheme, authority, path, base, uri)); + CU_ASSERT(0 == http2::construct_push_component(balloc, scheme, authority, + path, base, uri)); CU_ASSERT("https" == scheme); CU_ASSERT("example.org" == authority); CU_ASSERT("/foo" == path); - scheme.clear(); - authority.clear(); - path.clear(); + scheme = StringRef{}; + authority = StringRef{}; + path = StringRef{}; uri = StringRef::from_lit("/foo/bar?q=a"); - CU_ASSERT( - 0 == http2::construct_push_component(scheme, authority, path, base, uri)); + CU_ASSERT(0 == http2::construct_push_component(balloc, scheme, authority, + path, base, uri)); CU_ASSERT("" == scheme); CU_ASSERT("" == authority); CU_ASSERT("/foo/bar?q=a" == path); - scheme.clear(); - authority.clear(); - path.clear(); + scheme = StringRef{}; + authority = StringRef{}; + path = StringRef{}; uri = StringRef::from_lit("foo/../bar?q=a"); - CU_ASSERT( - 0 == http2::construct_push_component(scheme, authority, path, base, uri)); + CU_ASSERT(0 == http2::construct_push_component(balloc, scheme, authority, + path, base, uri)); CU_ASSERT("" == scheme); CU_ASSERT("" == authority); CU_ASSERT("/b/bar?q=a" == path); - scheme.clear(); - authority.clear(); - path.clear(); + scheme = StringRef{}; + authority = StringRef{}; + path = StringRef{}; uri = StringRef{}; - CU_ASSERT( - 0 == http2::construct_push_component(scheme, authority, path, base, uri)); + CU_ASSERT(0 == http2::construct_push_component(balloc, scheme, authority, + path, base, uri)); CU_ASSERT("" == scheme); CU_ASSERT("" == authority); CU_ASSERT("/" == path); - scheme.clear(); - authority.clear(); - path.clear(); + scheme = StringRef{}; + authority = StringRef{}; + path = StringRef{}; uri = StringRef::from_lit("?q=a"); - CU_ASSERT( - 0 == http2::construct_push_component(scheme, authority, path, base, uri)); + CU_ASSERT(0 == http2::construct_push_component(balloc, scheme, authority, + path, base, uri)); CU_ASSERT("" == scheme); CU_ASSERT("" == authority); CU_ASSERT("/b/?q=a" == path); diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index bebc0f28..8cd75690 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -142,6 +142,7 @@ int main(int argc, char *argv[]) { !CU_add_test(pSuite, "util_select_h2", shrpx::test_util_select_h2) || !CU_add_test(pSuite, "util_ipv6_numeric_addr", shrpx::test_util_ipv6_numeric_addr) || + !CU_add_test(pSuite, "util_utos", shrpx::test_util_utos) || !CU_add_test(pSuite, "util_utos_unit", shrpx::test_util_utos_unit) || !CU_add_test(pSuite, "util_utos_funit", shrpx::test_util_utos_funit) || !CU_add_test(pSuite, "util_parse_uint_with_unit", diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 1046fdf0..a2ad7c54 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -624,7 +624,8 @@ int parse_mapping(const DownstreamAddrConfig &addr, } else { pattern.assign(std::begin(raw_pattern), slash); util::inp_strlower(pattern); - pattern += http2::normalize_path(slash, std::end(raw_pattern)); + pattern += http2::normalize_path(StringRef{slash, std::end(raw_pattern)}, + StringRef{}); } for (auto &g : addr_groups) { if (g.pattern == pattern) { diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index f5e4f084..eadfed39 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -257,8 +257,7 @@ std::string Downstream::assemble_request_cookie() const { std::string cookie; cookie = ""; for (auto &kv : req_.fs.headers()) { - if (kv.name.size() != 6 || kv.name[5] != 'e' || - !util::streq_l("cooki", kv.name.c_str(), 5)) { + if (kv.token != http2::HD_COOKIE) { continue; } @@ -273,7 +272,7 @@ std::string Downstream::assemble_request_cookie() const { if (c == ' ' || c == ';') { continue; } - end = it + 1; + end = it; break; } diff --git a/src/shrpx_downstream_test.cc b/src/shrpx_downstream_test.cc index d9989a72..5bfcda67 100644 --- a/src/shrpx_downstream_test.cc +++ b/src/shrpx_downstream_test.cc @@ -33,7 +33,8 @@ namespace shrpx { void test_downstream_field_store_add_header_lower(void) { - FieldStore fs(0); + BlockAllocator balloc(4096, 4096); + FieldStore fs(balloc, 0); fs.add_header_lower(StringRef::from_lit("1"), StringRef::from_lit("0"), false); fs.add_header_lower(StringRef::from_lit("2"), StringRef::from_lit("1"), @@ -51,19 +52,21 @@ void test_downstream_field_store_add_header_lower(void) { fs.add_header_lower(StringRef::from_lit(":authority"), StringRef::from_lit("7"), false); - auto ans = Headers{{"1", "0"}, - {"2", "1"}, - {"charlie", "2"}, - {"alpha", "3"}, - {"delta", "4"}, - {"bravo", "5"}, - {":method", "6"}, - {":authority", "7"}}; + auto ans = + HeaderRefs{{StringRef::from_lit("1"), StringRef::from_lit("0")}, + {StringRef::from_lit("2"), StringRef::from_lit("1")}, + {StringRef::from_lit("charlie"), StringRef::from_lit("2")}, + {StringRef::from_lit("alpha"), StringRef::from_lit("3")}, + {StringRef::from_lit("delta"), StringRef::from_lit("4")}, + {StringRef::from_lit("bravo"), StringRef::from_lit("5")}, + {StringRef::from_lit(":method"), StringRef::from_lit("6")}, + {StringRef::from_lit(":authority"), StringRef::from_lit("7")}}; CU_ASSERT(ans == fs.headers()); } void test_downstream_field_store_header(void) { - FieldStore fs(0); + BlockAllocator balloc(4096, 4096); + FieldStore fs(balloc, 0); fs.add_header_token(StringRef::from_lit("alpha"), StringRef::from_lit("0"), false, -1); fs.add_header_token(StringRef::from_lit(":authority"), @@ -73,11 +76,13 @@ void test_downstream_field_store_header(void) { http2::HD_CONTENT_LENGTH); // By token - CU_ASSERT(Header(":authority", "1") == *fs.header(http2::HD__AUTHORITY)); + CU_ASSERT(HeaderRef(StringRef{":authority"}, StringRef{"1"}) == + *fs.header(http2::HD__AUTHORITY)); CU_ASSERT(nullptr == fs.header(http2::HD__METHOD)); // By name - CU_ASSERT(Header("alpha", "0") == *fs.header(StringRef::from_lit("alpha"))); + CU_ASSERT(HeaderRef(StringRef{"alpha"}, StringRef{"0"}) == + *fs.header(StringRef::from_lit("alpha"))); CU_ASSERT(nullptr == fs.header(StringRef::from_lit("bravo"))); } @@ -105,19 +110,20 @@ void test_downstream_crumble_request_cookie(void) { CU_ASSERT(5 == nva.size()); CU_ASSERT(5 == num_cookies); - Headers cookies; + HeaderRefs cookies; std::transform(std::begin(nva), std::end(nva), std::back_inserter(cookies), [](const nghttp2_nv &nv) { - return Header(std::string(nv.name, nv.name + nv.namelen), - std::string(nv.value, nv.value + nv.valuelen), - nv.flags & NGHTTP2_NV_FLAG_NO_INDEX); + return HeaderRef(StringRef{nv.name, nv.namelen}, + StringRef{nv.value, nv.valuelen}, + nv.flags & NGHTTP2_NV_FLAG_NO_INDEX); }); - Headers ans = {{"cookie", "alpha"}, - {"cookie", "bravo"}, - {"cookie", "charlie"}, - {"cookie", "delta"}, - {"cookie", "echo"}}; + HeaderRefs ans = { + {StringRef::from_lit("cookie"), StringRef::from_lit("alpha")}, + {StringRef::from_lit("cookie"), StringRef::from_lit("bravo")}, + {StringRef::from_lit("cookie"), StringRef::from_lit("charlie")}, + {StringRef::from_lit("cookie"), StringRef::from_lit("delta")}, + {StringRef::from_lit("cookie"), StringRef::from_lit("echo")}}; CU_ASSERT(ans == cookies); CU_ASSERT(cookies[0].no_index); @@ -128,6 +134,7 @@ void test_downstream_crumble_request_cookie(void) { void test_downstream_assemble_request_cookie(void) { Downstream d(nullptr, nullptr, 0); auto &req = d.request(); + req.fs.add_header_token(StringRef::from_lit(":method"), StringRef::from_lit("get"), false, -1); req.fs.add_header_token(StringRef::from_lit(":path"), @@ -151,12 +158,12 @@ void test_downstream_rewrite_location_response_header(void) { Downstream d(nullptr, nullptr, 0); auto &req = d.request(); auto &resp = d.response(); - d.set_request_downstream_host("localhost2"); - req.authority = "localhost:8443"; + d.set_request_downstream_host(StringRef::from_lit("localhost2")); + req.authority = StringRef::from_lit("localhost:8443"); resp.fs.add_header_token(StringRef::from_lit("location"), StringRef::from_lit("http://localhost2:3000/"), false, http2::HD_LOCATION); - d.rewrite_location_response_header("https"); + d.rewrite_location_response_header(StringRef::from_lit("https")); auto location = resp.fs.header(http2::HD_LOCATION); CU_ASSERT("https://localhost:8443/" == (*location).value); } diff --git a/src/util.h b/src/util.h index f24f8140..90f03f1f 100644 --- a/src/util.h +++ b/src/util.h @@ -398,10 +398,11 @@ template OutputIt utos(OutputIt dst, T n) { ; --i; auto p = dst + i; + auto res = p + 1; for (; n; --i, n /= 10) { *p-- = (n % 10) + '0'; } - return dst + i + 1; + return res; } template std::string utos_unit(T n) { diff --git a/src/util_test.cc b/src/util_test.cc index cd8e2b02..d99c3a96 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -234,6 +234,15 @@ void test_util_ipv6_numeric_addr(void) { CU_ASSERT(!util::ipv6_numeric_addr("localhost")); } +void test_util_utos(void) { + uint8_t buf[32]; + + CU_ASSERT(("0" == StringRef{buf, util::utos(buf, 0)})); + CU_ASSERT(("123" == StringRef{buf, util::utos(buf, 123)})); + CU_ASSERT(("9223372036854775808" == + StringRef{buf, util::utos(buf, 9223372036854775808ULL)})); +} + void test_util_utos_unit(void) { CU_ASSERT("0" == util::utos_unit(0)); CU_ASSERT("1023" == util::utos_unit(1023)); diff --git a/src/util_test.h b/src/util_test.h index 1b79bbae..f51692aa 100644 --- a/src/util_test.h +++ b/src/util_test.h @@ -44,6 +44,7 @@ void test_util_utox(void); void test_util_http_date(void); void test_util_select_h2(void); void test_util_ipv6_numeric_addr(void); +void test_util_utos(void); void test_util_utos_unit(void); void test_util_utos_funit(void); void test_util_parse_uint_with_unit(void); From eb393985b78c7eca4f948c1ad0ebf76930680cbc Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 11 Mar 2016 00:50:27 +0900 Subject: [PATCH 111/147] nghttpx: Make a copy before adding header to Downstream --- src/http2.cc | 9 +++++ src/http2.h | 3 ++ src/shrpx-unittest.cc | 4 +- src/shrpx_downstream.cc | 49 +++++++------------------ src/shrpx_downstream.h | 4 -- src/shrpx_downstream_test.cc | 39 +++++++------------- src/shrpx_downstream_test.h | 2 +- src/shrpx_http2_session.cc | 20 ++++++---- src/shrpx_http2_upstream.cc | 32 +++++++++------- src/shrpx_http_downstream_connection.cc | 9 ++++- src/shrpx_https_upstream.cc | 9 ++++- src/shrpx_mruby_module_request.cc | 20 ++++++---- src/shrpx_mruby_module_response.cc | 24 +++++++----- src/shrpx_spdy_upstream.cc | 4 +- 14 files changed, 118 insertions(+), 110 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index 3a4474b8..8385cafe 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -1793,6 +1793,15 @@ StringRef rewrite_clean_path(BlockAllocator &balloc, const StringRef &src) { StringRef{query, fragment}); } +StringRef copy_lower(BlockAllocator &balloc, const StringRef &src) { + auto iov = make_byte_ref(balloc, src.size() + 1); + auto p = iov.base; + p = std::copy(std::begin(src), std::end(src), p); + *p = '\0'; + util::inp_strlower(iov.base, p); + return StringRef{iov.base, p}; +} + } // namespace http2 } // namespace nghttp2 diff --git a/src/http2.h b/src/http2.h index cc295079..48d4916d 100644 --- a/src/http2.h +++ b/src/http2.h @@ -385,6 +385,9 @@ int construct_push_component(BlockAllocator &balloc, StringRef &scheme, StringRef &authority, StringRef &path, const StringRef &base, const StringRef &uri); +// Copies |src| and return its lower-cased version. +StringRef copy_lower(BlockAllocator &balloc, const StringRef &src); + } // namespace http2 } // namespace nghttp2 diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index 8cd75690..0f2e2fd2 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -101,8 +101,8 @@ int main(int argc, char *argv[]) { shrpx::test_http2_get_pure_path_component) || !CU_add_test(pSuite, "http2_construct_push_component", shrpx::test_http2_construct_push_component) || - !CU_add_test(pSuite, "downstream_field_store_add_header_lower", - shrpx::test_downstream_field_store_add_header_lower) || + !CU_add_test(pSuite, "downstream_field_store_append_last_header", + shrpx::test_downstream_field_store_append_last_header) || !CU_add_test(pSuite, "downstream_field_store_header", shrpx::test_downstream_field_store_header) || !CU_add_test(pSuite, "downstream_crumble_request_cookie", diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index eadfed39..51931aad 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -348,23 +348,21 @@ void add_header(bool &key_prev, size_t &sum, HeaderRefs &headers, } } // namespace -namespace { -void add_header(size_t &sum, HeaderRefs &headers, const StringRef &name, - const StringRef &value, bool no_index, int32_t token) { - sum += name.size() + value.size(); - headers.emplace_back(name, value, no_index, token); -} -} // namespace - namespace { void append_last_header_key(BlockAllocator &balloc, bool &key_prev, size_t &sum, HeaderRefs &headers, const char *data, size_t len) { assert(key_prev); sum += len; auto &item = headers.back(); - item.name = concat_string_ref(balloc, item.name, StringRef{data, len}); - util::inp_strlower((uint8_t *)item.name.c_str(), - (uint8_t *)item.name.c_str() + item.name.size()); + auto iov = make_byte_ref(balloc, item.name.size() + len + 1); + auto p = iov.base; + p = std::copy(std::begin(item.name), std::end(item.name), p); + p = std::copy_n(data, len, p); + util::inp_strlower(p - len, p); + *p = '\0'; + + item.name = StringRef{iov.base, p}; + item.token = http2::lookup_token(item.name); } } // namespace @@ -424,21 +422,10 @@ const HeaderRefs::value_type *FieldStore::header(const StringRef &name) const { return search_header_linear_backwards(headers_, name); } -void FieldStore::add_header_lower(const StringRef &name, const StringRef &value, - bool no_index) { - auto low_name = make_string_ref(balloc_, name); - util::inp_strlower((uint8_t *)low_name.c_str(), - (uint8_t *)low_name.c_str() + low_name.size()); - auto token = http2::lookup_token(low_name); - shrpx::add_header(header_key_prev_, buffer_size_, headers_, low_name, - make_string_ref(balloc_, value), no_index, token); -} - void FieldStore::add_header_token(const StringRef &name, const StringRef &value, bool no_index, int32_t token) { - shrpx::add_header(buffer_size_, headers_, make_string_ref(balloc_, name), - make_string_ref(balloc_, value), no_index, token); - make_string_ref(balloc_, name); + shrpx::add_header(header_key_prev_, buffer_size_, headers_, name, value, + no_index, token); } void FieldStore::append_last_header_key(const char *data, size_t len) { @@ -453,23 +440,13 @@ void FieldStore::append_last_header_value(const char *data, size_t len) { void FieldStore::clear_headers() { headers_.clear(); } -void FieldStore::add_trailer_lower(const StringRef &name, - const StringRef &value, bool no_index) { - auto low_name = make_string_ref(balloc_, name); - util::inp_strlower((uint8_t *)low_name.c_str(), - (uint8_t *)low_name.c_str() + low_name.size()); - auto token = http2::lookup_token(low_name); - shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, low_name, - make_string_ref(balloc_, value), no_index, token); -} - void FieldStore::add_trailer_token(const StringRef &name, const StringRef &value, bool no_index, int32_t token) { // Header size limit should be applied to all header and trailer // fields combined. - shrpx::add_header(buffer_size_, trailers_, make_string_ref(balloc_, name), - make_string_ref(balloc_, value), no_index, token); + shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, name, value, + no_index, token); } void FieldStore::append_last_trailer_key(const char *data, size_t len) { diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index cea17d86..a5024169 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -81,8 +81,6 @@ public: // such header is found, returns nullptr. const HeaderRefs::value_type *header(const StringRef &name) const; - void add_header_lower(const StringRef &name, const StringRef &value, - bool no_index); void add_header_token(const StringRef &name, const StringRef &value, bool no_index, int32_t token); @@ -98,8 +96,6 @@ public: // Empties headers. void clear_headers(); - void add_trailer_lower(const StringRef &name, const StringRef &value, - bool no_index); void add_trailer_token(const StringRef &name, const StringRef &value, bool no_index, int32_t token); diff --git a/src/shrpx_downstream_test.cc b/src/shrpx_downstream_test.cc index 5bfcda67..786e9046 100644 --- a/src/shrpx_downstream_test.cc +++ b/src/shrpx_downstream_test.cc @@ -32,35 +32,22 @@ namespace shrpx { -void test_downstream_field_store_add_header_lower(void) { +void test_downstream_field_store_append_last_header(void) { BlockAllocator balloc(4096, 4096); FieldStore fs(balloc, 0); - fs.add_header_lower(StringRef::from_lit("1"), StringRef::from_lit("0"), - false); - fs.add_header_lower(StringRef::from_lit("2"), StringRef::from_lit("1"), - false); - fs.add_header_lower(StringRef::from_lit("Charlie"), StringRef::from_lit("2"), - false); - fs.add_header_lower(StringRef::from_lit("Alpha"), StringRef::from_lit("3"), - false); - fs.add_header_lower(StringRef::from_lit("Delta"), StringRef::from_lit("4"), - false); - fs.add_header_lower(StringRef::from_lit("BravO"), StringRef::from_lit("5"), - false); - fs.add_header_lower(StringRef::from_lit(":method"), StringRef::from_lit("6"), - false); - fs.add_header_lower(StringRef::from_lit(":authority"), - StringRef::from_lit("7"), false); + fs.add_header_token(StringRef::from_lit("alpha"), StringRef{}, false, -1); + auto bravo = StringRef::from_lit("BRAVO"); + fs.append_last_header_key(bravo.c_str(), bravo.size()); + auto charlie = StringRef::from_lit("Charlie"); + fs.append_last_header_value(charlie.c_str(), charlie.size()); + auto delta = StringRef::from_lit("deltA"); + fs.append_last_header_value(delta.c_str(), delta.size()); + fs.add_header_token(StringRef::from_lit("echo"), + StringRef::from_lit("foxtrot"), false, -1); - auto ans = - HeaderRefs{{StringRef::from_lit("1"), StringRef::from_lit("0")}, - {StringRef::from_lit("2"), StringRef::from_lit("1")}, - {StringRef::from_lit("charlie"), StringRef::from_lit("2")}, - {StringRef::from_lit("alpha"), StringRef::from_lit("3")}, - {StringRef::from_lit("delta"), StringRef::from_lit("4")}, - {StringRef::from_lit("bravo"), StringRef::from_lit("5")}, - {StringRef::from_lit(":method"), StringRef::from_lit("6")}, - {StringRef::from_lit(":authority"), StringRef::from_lit("7")}}; + auto ans = HeaderRefs{ + {StringRef::from_lit("alphabravo"), StringRef::from_lit("CharliedeltA")}, + {StringRef::from_lit("echo"), StringRef::from_lit("foxtrot")}}; CU_ASSERT(ans == fs.headers()); } diff --git a/src/shrpx_downstream_test.h b/src/shrpx_downstream_test.h index bcae1887..f7b1e603 100644 --- a/src/shrpx_downstream_test.h +++ b/src/shrpx_downstream_test.h @@ -31,7 +31,7 @@ namespace shrpx { -void test_downstream_field_store_add_header_lower(void); +void test_downstream_field_store_append_last_header(void); void test_downstream_field_store_header(void); void test_downstream_crumble_request_cookie(void); void test_downstream_assemble_request_cookie(void); diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 828d888c..25a7f996 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -818,6 +818,7 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, auto &resp = downstream->response(); auto &httpconf = get_config()->http; + auto &balloc = downstream->get_block_allocator(); switch (frame->hd.type) { case NGHTTP2_HEADERS: { @@ -847,13 +848,15 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, if (trailer) { // just store header fields for trailer part - resp.fs.add_trailer_token(StringRef{name, namelen}, - StringRef{value, valuelen}, no_index, token); + resp.fs.add_trailer_token( + make_string_ref(balloc, StringRef{name, namelen}), + make_string_ref(balloc, StringRef{value, valuelen}), no_index, token); return 0; } - resp.fs.add_header_token(StringRef{name, namelen}, - StringRef{value, valuelen}, no_index, token); + resp.fs.add_header_token( + make_string_ref(balloc, StringRef{name, namelen}), + make_string_ref(balloc, StringRef{value, valuelen}), no_index, token); return 0; } case NGHTTP2_PUSH_PROMISE: { @@ -870,6 +873,7 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, assert(promised_downstream); auto &promised_req = promised_downstream->request(); + auto &promised_balloc = promised_downstream->get_block_allocator(); // We use request header limit for PUSH_PROMISE if (promised_req.fs.buffer_size() + namelen + valuelen > @@ -886,9 +890,11 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, } auto token = http2::lookup_token(name, namelen); - promised_req.fs.add_header_token(StringRef{name, namelen}, - StringRef{value, valuelen}, - flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + promised_req.fs.add_header_token( + make_string_ref(promised_balloc, StringRef{name, namelen}), + make_string_ref(promised_balloc, StringRef{value, valuelen}), + flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + return 0; } } diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 5bfdb826..a3c78572 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -176,6 +176,8 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, auto &httpconf = get_config()->http; + auto &balloc = downstream->get_block_allocator(); + if (req.fs.buffer_size() + namelen + valuelen > httpconf.request_header_field_buffer || req.fs.num_fields() >= httpconf.max_request_header_fields) { @@ -206,12 +208,14 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, if (frame->headers.cat == NGHTTP2_HCAT_HEADERS) { // just store header fields for trailer part - req.fs.add_trailer_token(StringRef{name, namelen}, - StringRef{value, valuelen}, no_index, token); + req.fs.add_trailer_token( + make_string_ref(balloc, StringRef{name, namelen}), + make_string_ref(balloc, StringRef{value, valuelen}), no_index, token); return 0; } - req.fs.add_header_token(StringRef{name, namelen}, StringRef{value, valuelen}, + req.fs.add_header_token(make_string_ref(balloc, StringRef{name, namelen}), + make_string_ref(balloc, StringRef{value, valuelen}), no_index, token); return 0; } @@ -588,27 +592,29 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame, for (size_t i = 0; i < frame->push_promise.nvlen; ++i) { auto &nv = frame->push_promise.nva[i]; + + auto name = + make_string_ref(promised_balloc, StringRef{nv.name, nv.namelen}); + auto value = + make_string_ref(promised_balloc, StringRef{nv.value, nv.valuelen}); + auto token = http2::lookup_token(nv.name, nv.namelen); switch (token) { case http2::HD__METHOD: - req.method = http2::lookup_method_token(nv.value, nv.valuelen); + req.method = http2::lookup_method_token(value); break; case http2::HD__SCHEME: - req.scheme = - make_string_ref(promised_balloc, StringRef{nv.value, nv.valuelen}); + req.scheme = value; break; case http2::HD__AUTHORITY: - req.authority = - make_string_ref(promised_balloc, StringRef{nv.value, nv.valuelen}); + req.authority = value; break; case http2::HD__PATH: - req.path = http2::rewrite_clean_path(promised_balloc, - StringRef{nv.value, nv.valuelen}); + req.path = http2::rewrite_clean_path(promised_balloc, value); break; } - req.fs.add_header_token(StringRef{nv.name, nv.namelen}, - StringRef{nv.value, nv.valuelen}, - nv.flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + req.fs.add_header_token(name, value, nv.flags & NGHTTP2_NV_FLAG_NO_INDEX, + token); } promised_downstream->inspect_http2_request(); diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index d7f5492b..5f4f9468 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -690,6 +690,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { auto downstream = static_cast(htp->data); auto &resp = downstream->response(); auto &httpconf = get_config()->http; + auto &balloc = downstream->get_block_allocator(); if (ensure_header_field_buffer(downstream, httpconf, len) != 0) { return -1; @@ -702,7 +703,9 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { if (ensure_max_header_fields(downstream, httpconf) != 0) { return -1; } - resp.fs.add_header_lower(StringRef{data, len}, StringRef{}, false); + auto name = http2::copy_lower(balloc, StringRef{data, len}); + auto token = http2::lookup_token(name); + resp.fs.add_header_token(name, StringRef{}, false, token); } } else { // trailer part @@ -715,7 +718,9 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { // wrong place or crash if trailer fields are currently empty. return -1; } - resp.fs.add_trailer_lower(StringRef(data, len), StringRef{}, false); + auto name = http2::copy_lower(balloc, StringRef{data, len}); + auto token = http2::lookup_token(name); + resp.fs.add_trailer_token(name, StringRef{}, false, token); } } return 0; diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 73e91a67..dd028190 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -121,6 +121,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { auto downstream = upstream->get_downstream(); auto &req = downstream->request(); auto &httpconf = get_config()->http; + auto &balloc = downstream->get_block_allocator(); if (req.fs.buffer_size() + len > httpconf.request_header_field_buffer) { if (LOG_ENABLED(INFO)) { @@ -145,7 +146,9 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { Downstream::HTTP1_REQUEST_HEADER_TOO_LARGE); return -1; } - req.fs.add_header_lower(StringRef{data, len}, StringRef{}, false); + auto name = http2::copy_lower(balloc, StringRef{data, len}); + auto token = http2::lookup_token(name); + req.fs.add_header_token(name, StringRef{}, false, token); } } else { // trailer part @@ -159,7 +162,9 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { } return -1; } - req.fs.add_trailer_lower(StringRef{data, len}, StringRef{}, false); + auto name = http2::copy_lower(balloc, StringRef{data, len}); + auto token = http2::lookup_token(name); + req.fs.add_trailer_token(name, StringRef{}, false, token); } } return 0; diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index bf83eba2..f98442a4 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -213,6 +213,7 @@ mrb_value request_mod_header(mrb_state *mrb, mrb_value self, bool repl) { auto data = static_cast(mrb->ud); auto downstream = data->downstream; auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); check_phase(mrb, data->phase, PHASE_REQUEST); @@ -226,7 +227,8 @@ mrb_value request_mod_header(mrb_state *mrb, mrb_value self, bool repl) { key = mrb_funcall(mrb, key, "downcase", 0); auto keyref = - StringRef{RSTRING_PTR(key), static_cast(RSTRING_LEN(key))}; + make_string_ref(balloc, StringRef{RSTRING_PTR(key), + static_cast(RSTRING_LEN(key))}); auto token = http2::lookup_token(keyref.byte(), keyref.size()); if (repl) { @@ -249,15 +251,19 @@ mrb_value request_mod_header(mrb_state *mrb, mrb_value self, bool repl) { for (int i = 0; i < n; ++i) { auto value = mrb_ary_entry(values, i); req.fs.add_header_token( - keyref, StringRef{RSTRING_PTR(value), - static_cast(RSTRING_LEN(value))}, + keyref, + make_string_ref(balloc, + StringRef{RSTRING_PTR(value), + static_cast(RSTRING_LEN(value))}), false, token); } } else if (!mrb_nil_p(values)) { - req.fs.add_header_token(keyref, - StringRef{RSTRING_PTR(values), - static_cast(RSTRING_LEN(values))}, - false, token); + req.fs.add_header_token( + keyref, + make_string_ref(balloc, + StringRef{RSTRING_PTR(values), + static_cast(RSTRING_LEN(values))}), + false, token); } return mrb_nil_value(); diff --git a/src/shrpx_mruby_module_response.cc b/src/shrpx_mruby_module_response.cc index ada76b7c..65fec390 100644 --- a/src/shrpx_mruby_module_response.cc +++ b/src/shrpx_mruby_module_response.cc @@ -107,6 +107,7 @@ mrb_value response_mod_header(mrb_state *mrb, mrb_value self, bool repl) { auto data = static_cast(mrb->ud); auto downstream = data->downstream; auto &resp = downstream->response(); + auto &balloc = downstream->get_block_allocator(); mrb_value key, values; mrb_get_args(mrb, "oo", &key, &values); @@ -118,7 +119,8 @@ mrb_value response_mod_header(mrb_state *mrb, mrb_value self, bool repl) { key = mrb_funcall(mrb, key, "downcase", 0); auto keyref = - StringRef{RSTRING_PTR(key), static_cast(RSTRING_LEN(key))}; + make_string_ref(balloc, StringRef{RSTRING_PTR(key), + static_cast(RSTRING_LEN(key))}); auto token = http2::lookup_token(keyref.byte(), keyref.size()); if (repl) { @@ -141,14 +143,18 @@ mrb_value response_mod_header(mrb_state *mrb, mrb_value self, bool repl) { for (int i = 0; i < n; ++i) { auto value = mrb_ary_entry(values, i); resp.fs.add_header_token( - keyref, StringRef{RSTRING_PTR(value), - static_cast(RSTRING_LEN(value))}, + keyref, + make_string_ref(balloc, + StringRef{RSTRING_PTR(value), + static_cast(RSTRING_LEN(value))}), false, token); } } else if (!mrb_nil_p(values)) { resp.fs.add_header_token( - keyref, StringRef{RSTRING_PTR(values), - static_cast(RSTRING_LEN(values))}, + keyref, + make_string_ref(balloc, + StringRef{RSTRING_PTR(values), + static_cast(RSTRING_LEN(values))}), false, token); } @@ -222,7 +228,6 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { if (cl) { cl->value = content_length; } else { - // TODO we don't have to make a copy here. resp.fs.add_header_token(StringRef::from_lit("content-length"), content_length, false, http2::HD_CONTENT_LENGTH); } @@ -232,9 +237,10 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { if (!date) { auto lgconf = log_config(); lgconf->update_tstamp(std::chrono::system_clock::now()); - resp.fs.add_header_token(StringRef::from_lit("date"), - StringRef{lgconf->time_http_str}, false, - http2::HD_DATE); + resp.fs.add_header_token( + StringRef::from_lit("date"), + make_string_ref(balloc, StringRef{lgconf->time_http_str}), false, + http2::HD_DATE); } auto upstream = downstream->get_upstream(); diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 89508ba8..4b48c93f 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -197,7 +197,9 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, auto name = StringRef{nv[i]}; auto value = StringRef{nv[i + 1]}; auto token = http2::lookup_token(name.byte(), name.size()); - req.fs.add_header_token(name, value, false, token); + req.fs.add_header_token(make_string_ref(balloc, StringRef{name}), + make_string_ref(balloc, StringRef{value}), false, + token); } if (req.fs.parse_content_length() != 0) { From 8da20975f9049b00bc27f407637815c7ee494be8 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 11 Mar 2016 01:52:55 +0900 Subject: [PATCH 112/147] Always allocate buffer for name, and value --- lib/nghttp2_buf.h | 2 +- lib/nghttp2_hd.c | 278 +++++++++++---------------------------- lib/nghttp2_hd.h | 18 +-- lib/nghttp2_hd_huffman.c | 29 +--- tests/nghttp2_hd_test.c | 13 +- 5 files changed, 97 insertions(+), 243 deletions(-) diff --git a/lib/nghttp2_buf.h b/lib/nghttp2_buf.h index 8977f2ea..7c43606e 100644 --- a/lib/nghttp2_buf.h +++ b/lib/nghttp2_buf.h @@ -73,7 +73,7 @@ typedef struct { /* * Initializes the |buf|. No memory is allocated in this function. Use - * nghttp2_buf_reserve() or nghttp2_buf_reserve2() to allocate memory. + * nghttp2_buf_reserve() to allocate memory. */ void nghttp2_buf_init(nghttp2_buf *buf); diff --git a/lib/nghttp2_hd.c b/lib/nghttp2_hd.c index 65a09a5b..c6c5b5f8 100644 --- a/lib/nghttp2_hd.c +++ b/lib/nghttp2_hd.c @@ -1099,30 +1099,24 @@ int nghttp2_hd_inflate_init(nghttp2_hd_inflater *inflater, nghttp2_mem *mem) { inflater->min_hd_table_bufsize_max = UINT32_MAX; inflater->ent_keep = NULL; - inflater->nv_keep = NULL; + inflater->nv_name_keep = NULL; + inflater->nv_value_keep = NULL; inflater->opcode = NGHTTP2_HD_OPCODE_NONE; inflater->state = NGHTTP2_HD_STATE_INFLATE_START; - rv = nghttp2_bufs_init3(&inflater->nvbufs, NGHTTP2_HD_MAX_NV / 8, 8, 1, 0, - mem); - - if (rv != 0) { - goto nvbufs_fail; - } + nghttp2_buf_init(&inflater->namebuf); + nghttp2_buf_init(&inflater->valuebuf); inflater->huffman_encoded = 0; inflater->index = 0; inflater->left = 0; inflater->shift = 0; - inflater->newnamelen = 0; inflater->index_required = 0; inflater->no_index = 0; return 0; -nvbufs_fail: - hd_context_free(&inflater->ctx); fail: return rv; } @@ -1139,8 +1133,11 @@ static void hd_inflate_keep_free(nghttp2_hd_inflater *inflater) { inflater->ent_keep = NULL; } - nghttp2_mem_free(mem, inflater->nv_keep); - inflater->nv_keep = NULL; + nghttp2_mem_free(mem, inflater->nv_value_keep); + inflater->nv_value_keep = NULL; + + nghttp2_mem_free(mem, inflater->nv_name_keep); + inflater->nv_name_keep = NULL; } void nghttp2_hd_deflate_free(nghttp2_hd_deflater *deflater) { @@ -1148,8 +1145,13 @@ void nghttp2_hd_deflate_free(nghttp2_hd_deflater *deflater) { } void nghttp2_hd_inflate_free(nghttp2_hd_inflater *inflater) { + nghttp2_mem *mem; + + mem = inflater->ctx.mem; + hd_inflate_keep_free(inflater); - nghttp2_bufs_free(&inflater->nvbufs); + nghttp2_buf_free(&inflater->valuebuf, mem); + nghttp2_buf_free(&inflater->namebuf, mem); hd_context_free(&inflater->ctx); } @@ -2036,11 +2038,9 @@ static ssize_t hd_inflate_read_len(nghttp2_hd_inflater *inflater, int *rfin, * Out of memory * NGHTTP2_ERR_HEADER_COMP * Huffman decoding failed - * NGHTTP2_ERR_BUFFER_ERROR - * Out of buffer space. */ static ssize_t hd_inflate_read_huff(nghttp2_hd_inflater *inflater, - nghttp2_bufs *bufs, uint8_t *in, + nghttp2_buf *buf, uint8_t *in, uint8_t *last) { ssize_t readlen; int final = 0; @@ -2048,7 +2048,7 @@ static ssize_t hd_inflate_read_huff(nghttp2_hd_inflater *inflater, last = in + inflater->left; final = 1; } - readlen = nghttp2_hd_huff_decode(&inflater->huff_decode_ctx, bufs, in, + readlen = nghttp2_hd_huff_decode(&inflater->huff_decode_ctx, buf, in, (size_t)(last - in), final); if (readlen < 0) { @@ -2070,17 +2070,13 @@ static ssize_t hd_inflate_read_huff(nghttp2_hd_inflater *inflater, * Out of memory * NGHTTP2_ERR_HEADER_COMP * Header decompression failed - * NGHTTP2_ERR_BUFFER_ERROR - * Out of buffer space. */ -static ssize_t hd_inflate_read(nghttp2_hd_inflater *inflater, - nghttp2_bufs *bufs, uint8_t *in, uint8_t *last) { - int rv; +static ssize_t hd_inflate_read(nghttp2_hd_inflater *inflater, nghttp2_buf *buf, + uint8_t *in, uint8_t *last) { size_t len = nghttp2_min((size_t)(last - in), inflater->left); - rv = nghttp2_bufs_add(bufs, in, len); - if (rv != 0) { - return rv; - } + + buf->last = nghttp2_cpymem(buf->last, in, len); + inflater->left -= len; return (ssize_t)len; } @@ -2105,112 +2101,6 @@ static int hd_inflate_commit_indexed(nghttp2_hd_inflater *inflater, return 0; } -static int hd_inflate_remove_bufs(nghttp2_hd_inflater *inflater, nghttp2_nv *nv, - int value_only) { - ssize_t rv; - size_t buflen; - uint8_t *buf; - nghttp2_buf *pbuf; - - if (inflater->index_required || - inflater->nvbufs.head != inflater->nvbufs.cur) { - - rv = nghttp2_bufs_remove(&inflater->nvbufs, &buf); - - if (rv < 0) { - return NGHTTP2_ERR_NOMEM; - } - - nghttp2_bufs_reset(&inflater->nvbufs); - - buflen = (size_t)rv; - - if (value_only) { - /* we don't use this value, so no need to NULL-terminate */ - nv->name = NULL; - nv->namelen = 0; - - nv->value = buf; - nv->valuelen = buflen - 1; - } else { - nv->name = buf; - nv->namelen = inflater->newnamelen; - - nv->value = buf + nv->namelen + 1; - nv->valuelen = buflen - nv->namelen - 2; - } - - return 0; - } - - /* If we are not going to store header in header table and - name/value are in first chunk, we just refer them from nv, - instead of mallocing another memory. */ - - pbuf = &inflater->nvbufs.head->buf; - - if (value_only) { - /* we don't use this value, so no need to NULL-terminate */ - nv->name = NULL; - nv->namelen = 0; - - nv->value = pbuf->pos; - nv->valuelen = nghttp2_buf_len(pbuf) - 1; - } else { - nv->name = pbuf->pos; - nv->namelen = inflater->newnamelen; - - nv->value = pbuf->pos + nv->namelen + 1; - nv->valuelen = nghttp2_buf_len(pbuf) - nv->namelen - 2; - } - - /* Resetting does not change the content of first buffer */ - nghttp2_bufs_reset(&inflater->nvbufs); - - return 0; -} - -static int hd_inflate_remove_bufs_with_name(nghttp2_hd_inflater *inflater, - nghttp2_nv *nv, - nghttp2_hd_entry *ent_name) { -#ifndef NDEBUG - size_t rv; -#endif - size_t buflen; - uint8_t *buf; - nghttp2_mem *mem; - - mem = inflater->ctx.mem; - - /* Allocate buffer including name in ent_name, plus terminating - NULL. */ - buflen = ent_name->nv.namelen + 1 + nghttp2_bufs_len(&inflater->nvbufs); - - buf = nghttp2_mem_malloc(mem, buflen); - if (buf == NULL) { - return NGHTTP2_ERR_NOMEM; - } - - /* Copy including terminal NULL */ - memcpy(buf, ent_name->nv.name, ent_name->nv.namelen + 1); -#ifndef NDEBUG - rv = -#endif - nghttp2_bufs_remove_copy(&inflater->nvbufs, - buf + ent_name->nv.namelen + 1); - assert(ent_name->nv.namelen + 1 + rv == buflen); - - nghttp2_bufs_reset(&inflater->nvbufs); - - nv->name = buf; - nv->namelen = ent_name->nv.namelen; - - nv->value = buf + nv->namelen + 1; - nv->valuelen = buflen - nv->namelen - 2; - - return 0; -} - /* * Finalize literal header representation - new name- reception. If * header is emitted, |*nv_out| is filled with that value and 0 is @@ -2224,16 +2114,18 @@ static int hd_inflate_remove_bufs_with_name(nghttp2_hd_inflater *inflater, */ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, nghttp2_nv *nv_out, int *token_out) { - int rv; nghttp2_nv nv; nghttp2_mem *mem; mem = inflater->ctx.mem; - rv = hd_inflate_remove_bufs(inflater, &nv, 0 /* name and value */); - if (rv != 0) { - return NGHTTP2_ERR_NOMEM; - } + nv.name = inflater->namebuf.pos; + nv.namelen = nghttp2_buf_len(&inflater->namebuf); + nv.value = inflater->valuebuf.pos; + nv.valuelen = nghttp2_buf_len(&inflater->valuebuf); + + nghttp2_buf_init(&inflater->valuebuf); + nghttp2_buf_init(&inflater->namebuf); if (inflater->no_index) { nv.flags = NGHTTP2_NV_FLAG_NO_INDEX; @@ -2245,10 +2137,8 @@ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, nghttp2_hd_entry *new_ent; uint8_t ent_flags; - /* nv->value points to the middle of the buffer pointed by - nv->name. So we just need to keep track of nv->name for memory - management. */ - ent_flags = NGHTTP2_HD_FLAG_NAME_ALLOC | NGHTTP2_HD_FLAG_NAME_GIFT; + ent_flags = NGHTTP2_HD_FLAG_NAME_ALLOC | NGHTTP2_HD_FLAG_NAME_GIFT | + NGHTTP2_HD_FLAG_VALUE_ALLOC | NGHTTP2_HD_FLAG_VALUE_GIFT; new_ent = add_hd_table_incremental(&inflater->ctx, &nv, lookup_token(nv.name, nv.namelen), @@ -2261,6 +2151,7 @@ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, return 0; } + nghttp2_mem_free(mem, nv.value); nghttp2_mem_free(mem, nv.name); return NGHTTP2_ERR_NOMEM; @@ -2268,9 +2159,8 @@ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, emit_literal_header(nv_out, token_out, &nv); - if (nv.name != inflater->nvbufs.head->buf.pos) { - inflater->nv_keep = nv.name; - } + inflater->nv_name_keep = nv.name; + inflater->nv_value_keep = nv.value; return 0; } @@ -2288,7 +2178,6 @@ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, */ static int hd_inflate_commit_indname(nghttp2_hd_inflater *inflater, nghttp2_nv *nv_out, int *token_out) { - int rv; nghttp2_nv nv; nghttp2_hd_entry *ent_name; nghttp2_mem *mem; @@ -2307,25 +2196,21 @@ static int hd_inflate_commit_indname(nghttp2_hd_inflater *inflater, nghttp2_hd_entry *new_ent; uint8_t ent_flags; - if (inflater->index < NGHTTP2_STATIC_TABLE_LENGTH) { - /* We don't copy name in static table */ - rv = hd_inflate_remove_bufs(inflater, &nv, 1 /* value only */); - if (rv != 0) { - return NGHTTP2_ERR_NOMEM; - } - nv.name = ent_name->nv.name; - nv.namelen = ent_name->nv.namelen; + ent_flags = NGHTTP2_HD_FLAG_VALUE_ALLOC | NGHTTP2_HD_FLAG_VALUE_GIFT; - ent_flags = NGHTTP2_HD_FLAG_VALUE_ALLOC | NGHTTP2_HD_FLAG_VALUE_GIFT; - } else { - rv = hd_inflate_remove_bufs_with_name(inflater, &nv, ent_name); - if (rv != 0) { - return NGHTTP2_ERR_NOMEM; - } - /* nv->name and nv->value are in the same buffer. */ - ent_flags = NGHTTP2_HD_FLAG_NAME_ALLOC | NGHTTP2_HD_FLAG_NAME_GIFT; + if (inflater->index >= NGHTTP2_STATIC_TABLE_LENGTH) { + /* We don't copy name in static table */ + ent_flags |= NGHTTP2_HD_FLAG_NAME_ALLOC; } + nv.name = ent_name->nv.name; + nv.namelen = ent_name->nv.namelen; + + nv.value = inflater->valuebuf.pos; + nv.valuelen = nghttp2_buf_len(&inflater->valuebuf); + + nghttp2_buf_init(&inflater->valuebuf); + new_ent = add_hd_table_incremental(&inflater->ctx, &nv, ent_name->token, ent_flags, NULL, 0); @@ -2339,28 +2224,21 @@ static int hd_inflate_commit_indname(nghttp2_hd_inflater *inflater, return 0; } - if (inflater->index < NGHTTP2_STATIC_TABLE_LENGTH) { - nghttp2_mem_free(mem, nv.value); - } else { - nghttp2_mem_free(mem, nv.name); - } + nghttp2_mem_free(mem, nv.value); return NGHTTP2_ERR_NOMEM; } - rv = hd_inflate_remove_bufs(inflater, &nv, 1 /* value only */); - if (rv != 0) { - return NGHTTP2_ERR_NOMEM; - } - nv.name = ent_name->nv.name; nv.namelen = ent_name->nv.namelen; + nv.value = inflater->valuebuf.pos; + nv.valuelen = nghttp2_buf_len(&inflater->valuebuf); + + nghttp2_buf_init(&inflater->valuebuf); emit_literal_header(nv_out, token_out, &nv); - if (nv.value != inflater->nvbufs.head->buf.pos) { - inflater->nv_keep = nv.value; - } + inflater->nv_value_keep = nv.value; return 0; } @@ -2383,6 +2261,9 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, uint8_t *last = in + inlen; int rfin = 0; int busy = 0; + nghttp2_mem *mem; + + mem = inflater->ctx.mem; if (inflater->ctx.bad) { return NGHTTP2_ERR_HEADER_COMP; @@ -2541,12 +2422,20 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, nghttp2_hd_huff_decode_context_init(&inflater->huff_decode_ctx); inflater->state = NGHTTP2_HD_STATE_NEWNAME_READ_NAMEHUFF; + rv = nghttp2_buf_reserve(&inflater->namebuf, inflater->left * 2 + 1, + mem); } else { inflater->state = NGHTTP2_HD_STATE_NEWNAME_READ_NAME; + rv = nghttp2_buf_reserve(&inflater->namebuf, inflater->left + 1, mem); } + + if (rv != 0) { + goto fail; + } + break; case NGHTTP2_HD_STATE_NEWNAME_READ_NAMEHUFF: - rv = hd_inflate_read_huff(inflater, &inflater->nvbufs, in, last); + rv = hd_inflate_read_huff(inflater, &inflater->namebuf, in, last); if (rv < 0) { goto fail; } @@ -2562,18 +2451,13 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, goto almost_ok; } - inflater->newnamelen = nghttp2_bufs_len(&inflater->nvbufs); - - rv = nghttp2_bufs_addb(&inflater->nvbufs, '\0'); - if (rv != 0) { - goto fail; - } + *inflater->namebuf.last = '\0'; inflater->state = NGHTTP2_HD_STATE_CHECK_VALUELEN; break; case NGHTTP2_HD_STATE_NEWNAME_READ_NAME: - rv = hd_inflate_read(inflater, &inflater->nvbufs, in, last); + rv = hd_inflate_read(inflater, &inflater->namebuf, in, last); if (rv < 0) { goto fail; } @@ -2588,12 +2472,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, goto almost_ok; } - inflater->newnamelen = nghttp2_bufs_len(&inflater->nvbufs); - - rv = nghttp2_bufs_addb(&inflater->nvbufs, '\0'); - if (rv != 0) { - goto fail; - } + *inflater->namebuf.last = '\0'; inflater->state = NGHTTP2_HD_STATE_CHECK_VALUELEN; @@ -2625,15 +2504,24 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, nghttp2_hd_huff_decode_context_init(&inflater->huff_decode_ctx); inflater->state = NGHTTP2_HD_STATE_READ_VALUEHUFF; + + rv = nghttp2_buf_reserve(&inflater->valuebuf, inflater->left * 2 + 1, + mem); } else { inflater->state = NGHTTP2_HD_STATE_READ_VALUE; + + rv = nghttp2_buf_reserve(&inflater->valuebuf, inflater->left + 1, mem); + } + + if (rv != 0) { + goto fail; } busy = 1; break; case NGHTTP2_HD_STATE_READ_VALUEHUFF: - rv = hd_inflate_read_huff(inflater, &inflater->nvbufs, in, last); + rv = hd_inflate_read_huff(inflater, &inflater->valuebuf, in, last); if (rv < 0) { goto fail; } @@ -2649,10 +2537,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, goto almost_ok; } - rv = nghttp2_bufs_addb(&inflater->nvbufs, '\0'); - if (rv != 0) { - goto fail; - } + *inflater->valuebuf.last = '\0'; if (inflater->opcode == NGHTTP2_HD_OPCODE_NEWNAME) { rv = hd_inflate_commit_newname(inflater, nv_out, token_out); @@ -2669,7 +2554,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, return (ssize_t)(in - first); case NGHTTP2_HD_STATE_READ_VALUE: - rv = hd_inflate_read(inflater, &inflater->nvbufs, in, last); + rv = hd_inflate_read(inflater, &inflater->valuebuf, in, last); if (rv < 0) { DEBUGF(fprintf(stderr, "inflatehd: value read failure %zd: %s\n", rv, nghttp2_strerror((int)rv))); @@ -2686,10 +2571,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, goto almost_ok; } - rv = nghttp2_bufs_addb(&inflater->nvbufs, '\0'); - if (rv != 0) { - goto fail; - } + *inflater->valuebuf.last = '\0'; if (inflater->opcode == NGHTTP2_HD_OPCODE_NEWNAME) { rv = hd_inflate_commit_newname(inflater, nv_out, token_out); diff --git a/lib/nghttp2_hd.h b/lib/nghttp2_hd.h index adf227be..fff4c5a5 100644 --- a/lib/nghttp2_hd.h +++ b/lib/nghttp2_hd.h @@ -276,7 +276,7 @@ struct nghttp2_hd_deflater { struct nghttp2_hd_inflater { nghttp2_hd_context ctx; /* header buffer */ - nghttp2_bufs nvbufs; + nghttp2_buf namebuf, valuebuf; /* Stores current state of huffman decoding */ nghttp2_hd_huff_decode_context huff_decode_ctx; /* Pointer to the nghttp2_hd_entry which is used current header @@ -285,14 +285,11 @@ struct nghttp2_hd_inflater { nghttp2_hd_entry *ent_keep; /* Pointer to the name/value pair buffer which is used in the current header emission. */ - uint8_t *nv_keep; + uint8_t *nv_name_keep, *nv_value_keep; /* The number of bytes to read */ size_t left; /* The index in indexed repr or indexed name */ size_t index; - /* The length of new name encoded in literal. For huffman encoded - string, this is the length after it is decoded. */ - size_t newnamelen; /* The maximum header table size the inflater supports. This is the same value transmitted in SETTINGS_HEADER_TABLE_SIZE */ size_t settings_hd_table_bufsize_max; @@ -470,11 +467,10 @@ int nghttp2_hd_huff_encode(nghttp2_bufs *bufs, const uint8_t *src, void nghttp2_hd_huff_decode_context_init(nghttp2_hd_huff_decode_context *ctx); /* - * Decodes the given data |src| with length |srclen|. The |ctx| must + * Decodes the given data |src| with length |srclen|. The |ctx| must * be initialized by nghttp2_hd_huff_decode_context_init(). The result - * will be added to |dest|. This function may expand |dest| as - * needed. The caller is responsible to release the memory of |dest| - * by calling nghttp2_bufs_free(). + * will be written to |buf|. This function assumes that |buf| has the + * enough room to store the decoded byte string. * * The caller must set the |final| to nonzero if the given input is * the final block. @@ -486,13 +482,11 @@ void nghttp2_hd_huff_decode_context_init(nghttp2_hd_huff_decode_context *ctx); * * NGHTTP2_ERR_NOMEM * Out of memory. - * NGHTTP2_ERR_BUFFER_ERROR - * Maximum buffer capacity size exceeded. * NGHTTP2_ERR_HEADER_COMP * Decoding process has failed. */ ssize_t nghttp2_hd_huff_decode(nghttp2_hd_huff_decode_context *ctx, - nghttp2_bufs *bufs, const uint8_t *src, + nghttp2_buf *buf, const uint8_t *src, size_t srclen, int final); #endif /* NGHTTP2_HD_H */ diff --git a/lib/nghttp2_hd_huffman.c b/lib/nghttp2_hd_huffman.c index 48638b75..3fb0d886 100644 --- a/lib/nghttp2_hd_huffman.c +++ b/lib/nghttp2_hd_huffman.c @@ -166,31 +166,10 @@ void nghttp2_hd_huff_decode_context_init(nghttp2_hd_huff_decode_context *ctx) { ctx->accept = 1; } -/* Use macro to make the code simpler..., but error case is tricky. - We spent most of the CPU in decoding, so we are doing this - thing. */ -#define hd_huff_decode_sym_emit(bufs, sym, avail) \ - do { \ - if ((avail)) { \ - nghttp2_bufs_fast_addb((bufs), (sym)); \ - --(avail); \ - } else { \ - rv = nghttp2_bufs_addb((bufs), (sym)); \ - if (rv != 0) { \ - return rv; \ - } \ - (avail) = nghttp2_bufs_cur_avail((bufs)); \ - } \ - } while (0) - ssize_t nghttp2_hd_huff_decode(nghttp2_hd_huff_decode_context *ctx, - nghttp2_bufs *bufs, const uint8_t *src, + nghttp2_buf *buf, const uint8_t *src, size_t srclen, int final) { size_t i; - int rv; - size_t avail; - - avail = nghttp2_bufs_cur_avail(bufs); /* We use the decoding algorithm described in http://graphics.ics.uci.edu/pub/Prefix.pdf */ @@ -202,8 +181,7 @@ ssize_t nghttp2_hd_huff_decode(nghttp2_hd_huff_decode_context *ctx, return NGHTTP2_ERR_HEADER_COMP; } if (t->flags & NGHTTP2_HUFF_SYM) { - /* this is macro, and may return from this function on error */ - hd_huff_decode_sym_emit(bufs, t->sym, avail); + *buf->last++ = t->sym; } t = &huff_decode_table[t->state][src[i] & 0xf]; @@ -211,8 +189,7 @@ ssize_t nghttp2_hd_huff_decode(nghttp2_hd_huff_decode_context *ctx, return NGHTTP2_ERR_HEADER_COMP; } if (t->flags & NGHTTP2_HUFF_SYM) { - /* this is macro, and may return from this function on error */ - hd_huff_decode_sym_emit(bufs, t->sym, avail); + *buf->last++ = t->sym; } ctx->state = t->state; diff --git a/tests/nghttp2_hd_test.c b/tests/nghttp2_hd_test.c index 5a4d7c5f..ed954bde 100644 --- a/tests/nghttp2_hd_test.c +++ b/tests/nghttp2_hd_test.c @@ -1352,13 +1352,15 @@ void test_nghttp2_hd_decode_length(void) { void test_nghttp2_hd_huff_encode(void) { int rv; ssize_t len; - nghttp2_bufs bufs, outbufs; + nghttp2_buf outbuf; + nghttp2_bufs bufs; nghttp2_hd_huff_decode_context ctx; const uint8_t t1[] = {22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; + uint8_t b[256]; + nghttp2_buf_wrap_init(&outbuf, b, sizeof(b)); frame_pack_bufs_init(&bufs); - frame_pack_bufs_init(&outbufs); rv = nghttp2_hd_huff_encode(&bufs, t1, sizeof(t1)); @@ -1366,14 +1368,13 @@ void test_nghttp2_hd_huff_encode(void) { nghttp2_hd_huff_decode_context_init(&ctx); - len = nghttp2_hd_huff_decode(&ctx, &outbufs, bufs.cur->buf.pos, + len = nghttp2_hd_huff_decode(&ctx, &outbuf, bufs.cur->buf.pos, nghttp2_bufs_len(&bufs), 1); CU_ASSERT((ssize_t)nghttp2_bufs_len(&bufs) == len); - CU_ASSERT((ssize_t)sizeof(t1) == nghttp2_bufs_len(&outbufs)); + CU_ASSERT((ssize_t)sizeof(t1) == nghttp2_buf_len(&outbuf)); - CU_ASSERT(0 == memcmp(t1, outbufs.cur->buf.pos, sizeof(t1))); + CU_ASSERT(0 == memcmp(t1, outbuf.pos, sizeof(t1))); nghttp2_bufs_free(&bufs); - nghttp2_bufs_free(&outbufs); } From ff0d137fb33e4302c6be010ec10030218a743efb Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 13:23:12 +0900 Subject: [PATCH 113/147] Reference counted HPACK name/value pair --- genlibtokenlookup.py | 2 +- lib/Makefile.am | 6 +- lib/includes/nghttp2/nghttp2.h | 47 +++ lib/nghttp2_hd.c | 518 ++++++++++++++------------------- lib/nghttp2_hd.h | 106 ++++--- lib/nghttp2_http.c | 87 +++--- lib/nghttp2_http.h | 5 +- lib/nghttp2_mem.c | 4 + lib/nghttp2_mem.h | 1 + lib/nghttp2_rcbuf.c | 98 +++++++ lib/nghttp2_rcbuf.h | 80 +++++ lib/nghttp2_session.c | 21 +- tests/nghttp2_hd_test.c | 12 +- 13 files changed, 557 insertions(+), 430 deletions(-) create mode 100644 lib/nghttp2_rcbuf.c create mode 100644 lib/nghttp2_rcbuf.h diff --git a/genlibtokenlookup.py b/genlibtokenlookup.py index 625e62d1..360d2198 100755 --- a/genlibtokenlookup.py +++ b/genlibtokenlookup.py @@ -162,7 +162,7 @@ def gen_enum(): def gen_index_header(): print '''\ -static inline int lookup_token(const uint8_t *name, size_t namelen) { +static inline int32_t lookup_token(const uint8_t *name, size_t namelen) { switch (namelen) {''' b = build_header(HEADERS) for size in sorted(b.keys()): diff --git a/lib/Makefile.am b/lib/Makefile.am index 3ea5bed2..17e86bfc 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -47,7 +47,8 @@ OBJECTS = nghttp2_pq.c nghttp2_map.c nghttp2_queue.c \ nghttp2_option.c \ nghttp2_callbacks.c \ nghttp2_mem.c \ - nghttp2_http.c + nghttp2_http.c \ + nghttp2_rcbuf.c HFILES = nghttp2_pq.h nghttp2_int.h nghttp2_map.h nghttp2_queue.h \ nghttp2_frame.h \ @@ -61,7 +62,8 @@ HFILES = nghttp2_pq.h nghttp2_int.h nghttp2_map.h nghttp2_queue.h \ nghttp2_option.h \ nghttp2_callbacks.h \ nghttp2_mem.h \ - nghttp2_http.h + nghttp2_http.h \ + nghttp2_rcbuf.h libnghttp2_la_SOURCES = $(HFILES) $(OBJECTS) libnghttp2_la_LDFLAGS = -no-undefined \ diff --git a/lib/includes/nghttp2/nghttp2.h b/lib/includes/nghttp2/nghttp2.h index f6133055..2a3674fb 100644 --- a/lib/includes/nghttp2/nghttp2.h +++ b/lib/includes/nghttp2/nghttp2.h @@ -419,6 +419,53 @@ typedef enum { NGHTTP2_ERR_FLOODED = -904 } nghttp2_error; +/** + * @struct + * + * The object representing single contagious buffer. + */ +typedef struct { + /** + * The pointer to the buffer. + */ + uint8_t *base; + /** + * The length of the buffer. + */ + size_t len; +} nghttp2_vec; + +struct nghttp2_rcbuf; + +/** + * @struct + * + * The object representing reference counted buffer. The details of + * this structure are intentionally hidden from the public API. + */ +typedef struct nghttp2_rcbuf nghttp2_rcbuf; + +/** + * @function + * + * Increments the reference count of |rcbuf| by 1. + */ +void nghttp2_rcbuf_incref(nghttp2_rcbuf *rcbuf); + +/** + * @function + * + * Decrements the reference count of |rcbuf| by 1. If the reference + * count becomes zero, the object pointed by |rcbuf| will be freed. + * In this case, application must not use |rcbuf| again. + */ +void nghttp2_rcbuf_decref(nghttp2_rcbuf *rcbuf); + +/** + * Returns the underlying buffer managed by |rcbuf|. + */ +nghttp2_vec nghttp2_rcbuf_get_buf(nghttp2_rcbuf *rcbuf); + /** * @enum * diff --git a/lib/nghttp2_hd.c b/lib/nghttp2_hd.c index c6c5b5f8..8da40064 100644 --- a/lib/nghttp2_hd.c +++ b/lib/nghttp2_hd.c @@ -34,15 +34,17 @@ /* Make scalar initialization form of nghttp2_hd_entry */ #define MAKE_STATIC_ENT(N, V, T, H) \ { \ - { (uint8_t *)(N), (uint8_t *)(V), sizeof((N)) - 1, sizeof((V)) - 1, 0 } \ - , NULL, 0, (H), (T), 1, NGHTTP2_HD_FLAG_NONE \ + { NULL, NULL, (uint8_t *)(N), sizeof((N)) - 1, -1 } \ + , {NULL, NULL, (uint8_t *)(V), sizeof((V)) - 1, -1}, \ + {(uint8_t *)(N), (uint8_t *)(V), sizeof((N)) - 1, sizeof((V)) - 1, 0}, \ + T, H \ } /* Generated by mkstatictbl.py */ /* 3rd parameter is nghttp2_token value for header field name. We use first enum value if same header names are repeated (e.g., :status). */ -static nghttp2_hd_entry static_table[] = { +static nghttp2_hd_static_entry static_table[] = { MAKE_STATIC_ENT(":authority", "", 0, 3153725150u), MAKE_STATIC_ENT(":method", "GET", 1, 695666056u), MAKE_STATIC_ENT(":method", "POST", 1, 695666056u), @@ -114,7 +116,7 @@ static int memeq(const void *s1, const void *s2, size_t n) { * This function was generated by genlibtokenlookup.py. Inspired by * h2o header lookup. https://github.com/h2o/h2o */ -static int lookup_token(const uint8_t *name, size_t namelen) { +static int32_t lookup_token(const uint8_t *name, size_t namelen) { switch (namelen) { case 2: switch (name[1]) { @@ -790,86 +792,33 @@ static int lookup_token(const uint8_t *name, size_t namelen) { return -1; } -int nghttp2_hd_entry_init(nghttp2_hd_entry *ent, uint8_t flags, uint8_t *name, - size_t namelen, uint8_t *value, size_t valuelen, - int token, nghttp2_mem *mem) { - int rv = 0; - - /* Since nghttp2_hd_entry is used for indexing, ent->nv.flags always - NGHTTP2_NV_FLAG_NONE */ - ent->nv.flags = NGHTTP2_NV_FLAG_NONE; - - if ((flags & NGHTTP2_HD_FLAG_NAME_ALLOC) && - (flags & NGHTTP2_HD_FLAG_NAME_GIFT) == 0) { - if (namelen == 0) { - flags = (uint8_t)(flags & ~NGHTTP2_HD_FLAG_NAME_ALLOC); - ent->nv.name = (uint8_t *)""; - } else { - /* name may not be NULL terminated on compression. */ - ent->nv.name = nghttp2_mem_malloc(mem, namelen + 1); - if (ent->nv.name == NULL) { - rv = NGHTTP2_ERR_NOMEM; - goto fail; - } - memcpy(ent->nv.name, name, namelen); - ent->nv.name[namelen] = '\0'; - } - } else { - ent->nv.name = name; - } - if ((flags & NGHTTP2_HD_FLAG_VALUE_ALLOC) && - (flags & NGHTTP2_HD_FLAG_VALUE_GIFT) == 0) { - if (valuelen == 0) { - flags = (uint8_t)(flags & ~NGHTTP2_HD_FLAG_VALUE_ALLOC); - ent->nv.value = (uint8_t *)""; - } else { - /* value may not be NULL terminated on compression. */ - ent->nv.value = nghttp2_mem_malloc(mem, valuelen + 1); - if (ent->nv.value == NULL) { - rv = NGHTTP2_ERR_NOMEM; - goto fail2; - } - memcpy(ent->nv.value, value, valuelen); - ent->nv.value[valuelen] = '\0'; - } - } else { - ent->nv.value = value; - } - ent->nv.namelen = namelen; - ent->nv.valuelen = valuelen; - ent->token = token; - ent->ref = 1; - ent->flags = flags; +void nghttp2_hd_entry_init(nghttp2_hd_entry *ent, nghttp2_hd_nv *nv) { + ent->nv = *nv; + ent->cnv.name = nv->name->base; + ent->cnv.namelen = nv->name->len; + ent->cnv.value = nv->value->base; + ent->cnv.valuelen = nv->value->len; + ent->cnv.flags = nv->flags; ent->next = NULL; ent->hash = 0; - return 0; - -fail2: - if ((flags & NGHTTP2_HD_FLAG_NAME_ALLOC) && - (flags & NGHTTP2_HD_FLAG_NAME_GIFT) == 0) { - nghttp2_mem_free(mem, ent->nv.name); - } -fail: - return rv; + nghttp2_rcbuf_incref(ent->nv.name); + nghttp2_rcbuf_incref(ent->nv.value); } -void nghttp2_hd_entry_free(nghttp2_hd_entry *ent, nghttp2_mem *mem) { - assert(ent->ref == 0); - if (ent->flags & NGHTTP2_HD_FLAG_NAME_ALLOC) { - nghttp2_mem_free(mem, ent->nv.name); - } - if (ent->flags & NGHTTP2_HD_FLAG_VALUE_ALLOC) { - nghttp2_mem_free(mem, ent->nv.value); - } +void nghttp2_hd_entry_free(nghttp2_hd_entry *ent) { + nghttp2_rcbuf_decref(ent->nv.value); + nghttp2_rcbuf_decref(ent->nv.name); } -static int name_eq(const nghttp2_nv *a, const nghttp2_nv *b) { - return a->namelen == b->namelen && memeq(a->name, b->name, a->namelen); +static int name_eq(const nghttp2_hd_nv *a, const nghttp2_nv *b) { + return a->name->len == b->namelen && + memeq(a->name->base, b->name, b->namelen); } -static int value_eq(const nghttp2_nv *a, const nghttp2_nv *b) { - return a->valuelen == b->valuelen && memeq(a->value, b->value, a->valuelen); +static int value_eq(const nghttp2_hd_nv *a, const nghttp2_nv *b) { + return a->value->len == b->valuelen && + memeq(a->value->base, b->value, b->valuelen); } static uint32_t name_hash(const nghttp2_nv *nv) { @@ -905,7 +854,7 @@ static void hd_map_insert(nghttp2_hd_map *map, nghttp2_hd_entry *ent) { } static nghttp2_hd_entry *hd_map_find(nghttp2_hd_map *map, int *exact_match, - const nghttp2_nv *nv, int token, + const nghttp2_nv *nv, int32_t token, uint32_t hash) { nghttp2_hd_entry *p; nghttp2_hd_entry *res = NULL; @@ -913,7 +862,7 @@ static nghttp2_hd_entry *hd_map_find(nghttp2_hd_map *map, int *exact_match, *exact_match = 0; for (p = map->table[hash & (HD_MAP_SIZE - 1)]; p; p = p->next) { - if (token != p->token || + if (token != p->nv.token || (token == -1 && (hash != p->hash || !name_eq(&p->nv, nv)))) { continue; } @@ -1008,8 +957,8 @@ static void hd_ringbuf_free(nghttp2_hd_ringbuf *ringbuf, nghttp2_mem *mem) { } for (i = 0; i < ringbuf->len; ++i) { nghttp2_hd_entry *ent = hd_ringbuf_get(ringbuf, i); - --ent->ref; - nghttp2_hd_entry_free(ent, mem); + + nghttp2_hd_entry_free(ent); nghttp2_mem_free(mem, ent); } nghttp2_mem_free(mem, ringbuf->buffer); @@ -1098,7 +1047,6 @@ int nghttp2_hd_inflate_init(nghttp2_hd_inflater *inflater, nghttp2_mem *mem) { inflater->settings_hd_table_bufsize_max = NGHTTP2_HD_DEFAULT_MAX_BUFFER_SIZE; inflater->min_hd_table_bufsize_max = UINT32_MAX; - inflater->ent_keep = NULL; inflater->nv_name_keep = NULL; inflater->nv_value_keep = NULL; @@ -1108,6 +1056,9 @@ int nghttp2_hd_inflate_init(nghttp2_hd_inflater *inflater, nghttp2_mem *mem) { nghttp2_buf_init(&inflater->namebuf); nghttp2_buf_init(&inflater->valuebuf); + inflater->namercbuf = NULL; + inflater->valuercbuf = NULL; + inflater->huffman_encoded = 0; inflater->index = 0; inflater->left = 0; @@ -1122,21 +1073,10 @@ fail: } static void hd_inflate_keep_free(nghttp2_hd_inflater *inflater) { - nghttp2_mem *mem; + nghttp2_rcbuf_decref(inflater->nv_value_keep); + nghttp2_rcbuf_decref(inflater->nv_name_keep); - mem = inflater->ctx.mem; - if (inflater->ent_keep) { - if (inflater->ent_keep->ref == 0) { - nghttp2_hd_entry_free(inflater->ent_keep, mem); - nghttp2_mem_free(mem, inflater->ent_keep); - } - inflater->ent_keep = NULL; - } - - nghttp2_mem_free(mem, inflater->nv_value_keep); inflater->nv_value_keep = NULL; - - nghttp2_mem_free(mem, inflater->nv_name_keep); inflater->nv_name_keep = NULL; } @@ -1145,13 +1085,11 @@ void nghttp2_hd_deflate_free(nghttp2_hd_deflater *deflater) { } void nghttp2_hd_inflate_free(nghttp2_hd_inflater *inflater) { - nghttp2_mem *mem; - - mem = inflater->ctx.mem; - hd_inflate_keep_free(inflater); - nghttp2_buf_free(&inflater->valuebuf, mem); - nghttp2_buf_free(&inflater->namebuf, mem); + + nghttp2_rcbuf_decref(inflater->valuercbuf); + nghttp2_rcbuf_decref(inflater->namercbuf); + hd_context_free(&inflater->ctx); } @@ -1159,23 +1097,13 @@ static size_t entry_room(size_t namelen, size_t valuelen) { return NGHTTP2_HD_ENTRY_OVERHEAD + namelen + valuelen; } -static int emit_indexed_header(nghttp2_nv *nv_out, int *token_out, - nghttp2_hd_entry *ent) { - DEBUGF(fprintf(stderr, "inflatehd: header emission: %s: %s\n", ent->nv.name, - ent->nv.value)); +static int emit_header(nghttp2_hd_nv *nv_out, nghttp2_hd_nv *nv) { + DEBUGF(fprintf(stderr, "inflatehd: header emission: %s: %s\n", nv->name->base, + nv->value->base)); /* ent->ref may be 0. This happens if the encoder emits literal block larger than header table capacity with indexing. */ - *nv_out = ent->nv; - *token_out = ent->token; - return 0; -} - -static int emit_literal_header(nghttp2_nv *nv_out, int *token_out, - nghttp2_nv *nv) { - DEBUGF(fprintf(stderr, "inflatehd: header emission: %s: %s\n", nv->name, - nv->value)); *nv_out = *nv; - *token_out = lookup_token(nv->name, nv->namelen); + return 0; } @@ -1483,17 +1411,16 @@ static int emit_newname_block(nghttp2_bufs *bufs, const nghttp2_nv *nv, return 0; } -static nghttp2_hd_entry * -add_hd_table_incremental(nghttp2_hd_context *context, const nghttp2_nv *nv, - int token, uint8_t entry_flags, nghttp2_hd_map *map, - uint32_t hash) { +static int add_hd_table_incremental(nghttp2_hd_context *context, + nghttp2_hd_nv *nv, nghttp2_hd_map *map, + uint32_t hash) { int rv; nghttp2_hd_entry *new_ent; size_t room; nghttp2_mem *mem; mem = context->mem; - room = entry_room(nv->namelen, nv->valuelen); + room = entry_room(nv->name->len, nv->value->len); while (context->hd_table_bufsize + room > context->hd_table_bufsize_max && context->hd_table.len > 0) { @@ -1501,72 +1428,53 @@ add_hd_table_incremental(nghttp2_hd_context *context, const nghttp2_nv *nv, size_t idx = context->hd_table.len - 1; nghttp2_hd_entry *ent = hd_ringbuf_get(&context->hd_table, idx); - context->hd_table_bufsize -= entry_room(ent->nv.namelen, ent->nv.valuelen); + context->hd_table_bufsize -= + entry_room(ent->nv.name->len, ent->nv.value->len); DEBUGF(fprintf(stderr, "hpack: remove item from header table: %s: %s\n", - ent->nv.name, ent->nv.value)); + (char *)ent->nv.name->base, (char *)ent->nv.value->base)); hd_ringbuf_pop_back(&context->hd_table); if (map) { hd_map_remove(map, ent); } - if (--ent->ref == 0) { - nghttp2_hd_entry_free(ent, mem); - nghttp2_mem_free(mem, ent); - } - } - new_ent = nghttp2_mem_malloc(mem, sizeof(nghttp2_hd_entry)); - if (new_ent == NULL) { - return NULL; - } - - rv = nghttp2_hd_entry_init(new_ent, entry_flags, nv->name, nv->namelen, - nv->value, nv->valuelen, token, mem); - if (rv != 0) { - nghttp2_mem_free(mem, new_ent); - return NULL; + nghttp2_hd_entry_free(ent); + nghttp2_mem_free(mem, ent); } if (room > context->hd_table_bufsize_max) { /* The entry taking more than NGHTTP2_HD_MAX_BUFFER_SIZE is - immediately evicted. */ - --new_ent->ref; - } else { - rv = hd_ringbuf_push_front(&context->hd_table, new_ent, mem); - - if (rv != 0) { - --new_ent->ref; - - if ((entry_flags & NGHTTP2_HD_FLAG_NAME_ALLOC) && - (entry_flags & NGHTTP2_HD_FLAG_NAME_GIFT)) { - /* nv->name are managed by caller. */ - new_ent->nv.name = NULL; - new_ent->nv.namelen = 0; - } - if ((entry_flags & NGHTTP2_HD_FLAG_VALUE_ALLOC) && - (entry_flags & NGHTTP2_HD_FLAG_VALUE_GIFT)) { - /* nv->value are managed by caller. */ - new_ent->nv.value = NULL; - new_ent->nv.valuelen = 0; - } - - nghttp2_hd_entry_free(new_ent, mem); - nghttp2_mem_free(mem, new_ent); - - return NULL; - } - - new_ent->seq = context->next_seq++; - new_ent->hash = hash; - - if (map) { - hd_map_insert(map, new_ent); - } - - context->hd_table_bufsize += room; + immediately evicted. So we don't allocate memory for it. */ + return 0; } - return new_ent; + + new_ent = nghttp2_mem_malloc(mem, sizeof(nghttp2_hd_entry)); + if (new_ent == NULL) { + return NGHTTP2_ERR_NOMEM; + } + + nghttp2_hd_entry_init(new_ent, nv); + + rv = hd_ringbuf_push_front(&context->hd_table, new_ent, mem); + + if (rv != 0) { + nghttp2_hd_entry_free(new_ent); + nghttp2_mem_free(mem, new_ent); + + return rv; + } + + new_ent->seq = context->next_seq++; + new_ent->hash = hash; + + if (map) { + hd_map_insert(map, new_ent); + } + + context->hd_table_bufsize += room; + + return 0; } typedef struct { @@ -1575,10 +1483,11 @@ typedef struct { uint8_t name_value_match; } search_result; -static search_result search_static_table(const nghttp2_nv *nv, int token, +static search_result search_static_table(const nghttp2_nv *nv, int32_t token, int indexing_mode) { search_result res = {token, 0}; int i; + nghttp2_hd_static_entry *ent; if (indexing_mode == NGHTTP2_HD_NEVER_INDEXING) { return res; @@ -1587,7 +1496,9 @@ static search_result search_static_table(const nghttp2_nv *nv, int token, for (i = token; i <= NGHTTP2_TOKEN_WWW_AUTHENTICATE && static_table[i].token == token; ++i) { - if (value_eq(&static_table[i].nv, nv)) { + ent = &static_table[i]; + if (ent->value.len == nv->valuelen && + memcmp(ent->value.base, nv->value, nv->valuelen) == 0) { res.index = i; res.name_value_match = 1; return res; @@ -1597,7 +1508,7 @@ static search_result search_static_table(const nghttp2_nv *nv, int token, } static search_result search_hd_table(nghttp2_hd_context *context, - const nghttp2_nv *nv, int token, + const nghttp2_nv *nv, int32_t token, int indexing_mode, nghttp2_hd_map *map, uint32_t hash) { search_result res = {-1, 0}; @@ -1641,15 +1552,15 @@ static void hd_context_shrink_table_size(nghttp2_hd_context *context, context->hd_table.len > 0) { size_t idx = context->hd_table.len - 1; nghttp2_hd_entry *ent = hd_ringbuf_get(&context->hd_table, idx); - context->hd_table_bufsize -= entry_room(ent->nv.namelen, ent->nv.valuelen); + context->hd_table_bufsize -= + entry_room(ent->nv.name->len, ent->nv.value->len); hd_ringbuf_pop_back(&context->hd_table); if (map) { hd_map_remove(map, ent); } - if (--ent->ref == 0) { - nghttp2_hd_entry_free(ent, mem); - nghttp2_mem_free(mem, ent); - } + + nghttp2_hd_entry_free(ent); + nghttp2_mem_free(mem, ent); } } @@ -1708,19 +1619,33 @@ static size_t get_max_index(nghttp2_hd_context *context) { return context->hd_table.len + NGHTTP2_STATIC_TABLE_LENGTH - 1; } -nghttp2_hd_entry *nghttp2_hd_table_get(nghttp2_hd_context *context, - size_t idx) { +nghttp2_hd_nv nghttp2_hd_table_get(nghttp2_hd_context *context, size_t idx) { assert(INDEX_RANGE_VALID(context, idx)); if (idx >= NGHTTP2_STATIC_TABLE_LENGTH) { - return hd_ringbuf_get(&context->hd_table, - idx - NGHTTP2_STATIC_TABLE_LENGTH); + return hd_ringbuf_get(&context->hd_table, idx - NGHTTP2_STATIC_TABLE_LENGTH) + ->nv; } else { - return &static_table[idx]; + nghttp2_hd_static_entry *ent; + + ent = &static_table[idx]; + return (nghttp2_hd_nv){&ent->name, &ent->value, ent->token, + NGHTTP2_NV_FLAG_NONE}; } } +static const nghttp2_nv *nghttp2_hd_table_get2(nghttp2_hd_context *context, + size_t idx) { + assert(INDEX_RANGE_VALID(context, idx)); + if (idx >= NGHTTP2_STATIC_TABLE_LENGTH) { + return &hd_ringbuf_get(&context->hd_table, + idx - NGHTTP2_STATIC_TABLE_LENGTH)->cnv; + } + + return &static_table[idx].cnv; +} + static int hd_deflate_decide_indexing(nghttp2_hd_deflater *deflater, - const nghttp2_nv *nv, int token) { + const nghttp2_nv *nv, int32_t token) { if (token == NGHTTP2_TOKEN__PATH || token == NGHTTP2_TOKEN_AGE || token == NGHTTP2_TOKEN_CONTENT_LENGTH || token == NGHTTP2_TOKEN_ETAG || token == NGHTTP2_TOKEN_IF_MODIFIED_SINCE || @@ -1740,7 +1665,7 @@ static int deflate_nv(nghttp2_hd_deflater *deflater, nghttp2_bufs *bufs, search_result res; ssize_t idx; int indexing_mode; - int token; + int32_t token; nghttp2_mem *mem; uint32_t hash = 0; @@ -1789,28 +1714,36 @@ static int deflate_nv(nghttp2_hd_deflater *deflater, nghttp2_bufs *bufs, } if (indexing_mode == NGHTTP2_HD_WITH_INDEXING) { - nghttp2_hd_entry *new_ent; + nghttp2_hd_nv hd_nv; + if (idx != -1 && idx < (ssize_t)NGHTTP2_STATIC_TABLE_LENGTH) { - nghttp2_nv nv_indname; - nv_indname = *nv; - nv_indname.name = - nghttp2_hd_table_get(&deflater->ctx, (size_t)idx)->nv.name; - new_ent = add_hd_table_incremental(&deflater->ctx, &nv_indname, token, - NGHTTP2_HD_FLAG_VALUE_ALLOC, - &deflater->map, hash); + hd_nv.name = nghttp2_hd_table_get(&deflater->ctx, (size_t)idx).name; + nghttp2_rcbuf_incref(hd_nv.name); } else { - new_ent = add_hd_table_incremental(&deflater->ctx, nv, token, - NGHTTP2_HD_FLAG_NAME_ALLOC | - NGHTTP2_HD_FLAG_VALUE_ALLOC, - &deflater->map, hash); + rv = nghttp2_rcbuf_new2(&hd_nv.name, nv->name, nv->namelen, mem); + if (rv != 0) { + return rv; + } } - if (!new_ent) { + + rv = nghttp2_rcbuf_new2(&hd_nv.value, nv->value, nv->valuelen, mem); + + if (rv != 0) { + nghttp2_rcbuf_decref(hd_nv.name); + return rv; + } + + hd_nv.token = token; + hd_nv.flags = NGHTTP2_NV_FLAG_NONE; + + rv = add_hd_table_incremental(&deflater->ctx, &hd_nv, &deflater->map, hash); + + nghttp2_rcbuf_decref(hd_nv.value); + nghttp2_rcbuf_decref(hd_nv.name); + + if (rv != 0) { return NGHTTP2_ERR_HEADER_COMP; } - if (new_ent->ref == 0) { - nghttp2_hd_entry_free(new_ent, mem); - nghttp2_mem_free(mem, new_ent); - } } if (idx == -1) { rv = emit_newname_block(bufs, nv, indexing_mode); @@ -2093,10 +2026,10 @@ static ssize_t hd_inflate_read(nghttp2_hd_inflater *inflater, nghttp2_buf *buf, * Out of memory */ static int hd_inflate_commit_indexed(nghttp2_hd_inflater *inflater, - nghttp2_nv *nv_out, int *token_out) { - nghttp2_hd_entry *ent = nghttp2_hd_table_get(&inflater->ctx, inflater->index); + nghttp2_hd_nv *nv_out) { + nghttp2_hd_nv nv = nghttp2_hd_table_get(&inflater->ctx, inflater->index); - emit_indexed_header(nv_out, token_out, ent); + emit_header(nv_out, &nv); return 0; } @@ -2113,19 +2046,9 @@ static int hd_inflate_commit_indexed(nghttp2_hd_inflater *inflater, * Out of memory */ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, - nghttp2_nv *nv_out, int *token_out) { - nghttp2_nv nv; - nghttp2_mem *mem; - - mem = inflater->ctx.mem; - - nv.name = inflater->namebuf.pos; - nv.namelen = nghttp2_buf_len(&inflater->namebuf); - nv.value = inflater->valuebuf.pos; - nv.valuelen = nghttp2_buf_len(&inflater->valuebuf); - - nghttp2_buf_init(&inflater->valuebuf); - nghttp2_buf_init(&inflater->namebuf); + nghttp2_hd_nv *nv_out) { + nghttp2_hd_nv nv; + int rv; if (inflater->no_index) { nv.flags = NGHTTP2_NV_FLAG_NO_INDEX; @@ -2133,35 +2056,26 @@ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, nv.flags = NGHTTP2_NV_FLAG_NONE; } + nv.name = inflater->namercbuf; + nv.value = inflater->valuercbuf; + nv.token = lookup_token(inflater->namercbuf->base, inflater->namercbuf->len); + if (inflater->index_required) { - nghttp2_hd_entry *new_ent; - uint8_t ent_flags; + rv = add_hd_table_incremental(&inflater->ctx, &nv, NULL, 0); - ent_flags = NGHTTP2_HD_FLAG_NAME_ALLOC | NGHTTP2_HD_FLAG_NAME_GIFT | - NGHTTP2_HD_FLAG_VALUE_ALLOC | NGHTTP2_HD_FLAG_VALUE_GIFT; - - new_ent = add_hd_table_incremental(&inflater->ctx, &nv, - lookup_token(nv.name, nv.namelen), - ent_flags, NULL, 0); - - if (new_ent) { - emit_indexed_header(nv_out, token_out, new_ent); - inflater->ent_keep = new_ent; - - return 0; + if (rv != 0) { + return rv; } - - nghttp2_mem_free(mem, nv.value); - nghttp2_mem_free(mem, nv.name); - - return NGHTTP2_ERR_NOMEM; } - emit_literal_header(nv_out, token_out, &nv); + emit_header(nv_out, &nv); inflater->nv_name_keep = nv.name; inflater->nv_value_keep = nv.value; + inflater->namercbuf = NULL; + inflater->valuercbuf = NULL; + return 0; } @@ -2177,12 +2091,11 @@ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, * Out of memory */ static int hd_inflate_commit_indname(nghttp2_hd_inflater *inflater, - nghttp2_nv *nv_out, int *token_out) { - nghttp2_nv nv; - nghttp2_hd_entry *ent_name; - nghttp2_mem *mem; + nghttp2_hd_nv *nv_out) { + nghttp2_hd_nv nv; + int rv; - mem = inflater->ctx.mem; + nv = nghttp2_hd_table_get(&inflater->ctx, inflater->index); if (inflater->no_index) { nv.flags = NGHTTP2_NV_FLAG_NO_INDEX; @@ -2190,72 +2103,57 @@ static int hd_inflate_commit_indname(nghttp2_hd_inflater *inflater, nv.flags = NGHTTP2_NV_FLAG_NONE; } - ent_name = nghttp2_hd_table_get(&inflater->ctx, inflater->index); + nghttp2_rcbuf_incref(nv.name); + + nv.value = inflater->valuercbuf; if (inflater->index_required) { - nghttp2_hd_entry *new_ent; - uint8_t ent_flags; - - ent_flags = NGHTTP2_HD_FLAG_VALUE_ALLOC | NGHTTP2_HD_FLAG_VALUE_GIFT; - - if (inflater->index >= NGHTTP2_STATIC_TABLE_LENGTH) { - /* We don't copy name in static table */ - ent_flags |= NGHTTP2_HD_FLAG_NAME_ALLOC; + rv = add_hd_table_incremental(&inflater->ctx, &nv, NULL, 0); + if (rv != 0) { + nghttp2_rcbuf_decref(nv.name); + return NGHTTP2_ERR_NOMEM; } - - nv.name = ent_name->nv.name; - nv.namelen = ent_name->nv.namelen; - - nv.value = inflater->valuebuf.pos; - nv.valuelen = nghttp2_buf_len(&inflater->valuebuf); - - nghttp2_buf_init(&inflater->valuebuf); - - new_ent = add_hd_table_incremental(&inflater->ctx, &nv, ent_name->token, - ent_flags, NULL, 0); - - /* At this point, ent_name might be deleted. */ - - if (new_ent) { - emit_indexed_header(nv_out, token_out, new_ent); - - inflater->ent_keep = new_ent; - - return 0; - } - - nghttp2_mem_free(mem, nv.value); - - return NGHTTP2_ERR_NOMEM; } - nv.name = ent_name->nv.name; - nv.namelen = ent_name->nv.namelen; - nv.value = inflater->valuebuf.pos; - nv.valuelen = nghttp2_buf_len(&inflater->valuebuf); - - nghttp2_buf_init(&inflater->valuebuf); - - emit_literal_header(nv_out, token_out, &nv); + emit_header(nv_out, &nv); + inflater->nv_name_keep = nv.name; inflater->nv_value_keep = nv.value; + inflater->valuercbuf = NULL; + return 0; } ssize_t nghttp2_hd_inflate_hd(nghttp2_hd_inflater *inflater, nghttp2_nv *nv_out, int *inflate_flags, uint8_t *in, size_t inlen, int in_final) { - int token; + ssize_t rv; + nghttp2_hd_nv hd_nv; - return nghttp2_hd_inflate_hd2(inflater, nv_out, inflate_flags, &token, in, - inlen, in_final); + rv = nghttp2_hd_inflate_hd2(inflater, &hd_nv, inflate_flags, in, inlen, + in_final); + + if (rv < 0) { + return rv; + } + + if (*inflate_flags & NGHTTP2_HD_INFLATE_EMIT) { + nv_out->name = hd_nv.name->base; + nv_out->namelen = hd_nv.name->len; + + nv_out->value = hd_nv.value->base; + nv_out->valuelen = hd_nv.value->len; + + nv_out->flags = hd_nv.flags; + } + + return rv; } ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, - nghttp2_nv *nv_out, int *inflate_flags, - int *token_out, uint8_t *in, size_t inlen, - int in_final) { + nghttp2_hd_nv *nv_out, int *inflate_flags, + uint8_t *in, size_t inlen, int in_final) { ssize_t rv = 0; uint8_t *first = in; uint8_t *last = in + inlen; @@ -2271,7 +2169,6 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, DEBUGF(fprintf(stderr, "inflatehd: start state=%d\n", inflater->state)); hd_inflate_keep_free(inflater); - *token_out = -1; *inflate_flags = NGHTTP2_HD_INFLATE_NONE; for (; in != last || busy;) { busy = 0; @@ -2377,7 +2274,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, inflater->index = inflater->left; --inflater->index; - rv = hd_inflate_commit_indexed(inflater, nv_out, token_out); + rv = hd_inflate_commit_indexed(inflater, nv_out); if (rv < 0) { goto fail; } @@ -2422,17 +2319,21 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, nghttp2_hd_huff_decode_context_init(&inflater->huff_decode_ctx); inflater->state = NGHTTP2_HD_STATE_NEWNAME_READ_NAMEHUFF; - rv = nghttp2_buf_reserve(&inflater->namebuf, inflater->left * 2 + 1, - mem); + + rv = nghttp2_rcbuf_new(&inflater->namercbuf, inflater->left * 2 + 1, + mem); } else { inflater->state = NGHTTP2_HD_STATE_NEWNAME_READ_NAME; - rv = nghttp2_buf_reserve(&inflater->namebuf, inflater->left + 1, mem); + rv = nghttp2_rcbuf_new(&inflater->namercbuf, inflater->left + 1, mem); } if (rv != 0) { goto fail; } + nghttp2_buf_wrap_init(&inflater->namebuf, inflater->namercbuf->base, + inflater->namercbuf->len); + break; case NGHTTP2_HD_STATE_NEWNAME_READ_NAMEHUFF: rv = hd_inflate_read_huff(inflater, &inflater->namebuf, in, last); @@ -2452,6 +2353,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, } *inflater->namebuf.last = '\0'; + inflater->namercbuf->len = nghttp2_buf_len(&inflater->namebuf); inflater->state = NGHTTP2_HD_STATE_CHECK_VALUELEN; @@ -2473,6 +2375,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, } *inflater->namebuf.last = '\0'; + inflater->namercbuf->len = nghttp2_buf_len(&inflater->namebuf); inflater->state = NGHTTP2_HD_STATE_CHECK_VALUELEN; @@ -2505,18 +2408,21 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, inflater->state = NGHTTP2_HD_STATE_READ_VALUEHUFF; - rv = nghttp2_buf_reserve(&inflater->valuebuf, inflater->left * 2 + 1, - mem); + rv = nghttp2_rcbuf_new(&inflater->valuercbuf, inflater->left * 2 + 1, + mem); } else { inflater->state = NGHTTP2_HD_STATE_READ_VALUE; - rv = nghttp2_buf_reserve(&inflater->valuebuf, inflater->left + 1, mem); + rv = nghttp2_rcbuf_new(&inflater->valuercbuf, inflater->left + 1, mem); } if (rv != 0) { goto fail; } + nghttp2_buf_wrap_init(&inflater->valuebuf, inflater->valuercbuf->base, + inflater->valuercbuf->len); + busy = 1; break; @@ -2538,11 +2444,12 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, } *inflater->valuebuf.last = '\0'; + inflater->valuercbuf->len = nghttp2_buf_len(&inflater->valuebuf); if (inflater->opcode == NGHTTP2_HD_OPCODE_NEWNAME) { - rv = hd_inflate_commit_newname(inflater, nv_out, token_out); + rv = hd_inflate_commit_newname(inflater, nv_out); } else { - rv = hd_inflate_commit_indname(inflater, nv_out, token_out); + rv = hd_inflate_commit_indname(inflater, nv_out); } if (rv != 0) { @@ -2572,11 +2479,12 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, } *inflater->valuebuf.last = '\0'; + inflater->valuercbuf->len = nghttp2_buf_len(&inflater->valuebuf); if (inflater->opcode == NGHTTP2_HD_OPCODE_NEWNAME) { - rv = hd_inflate_commit_newname(inflater, nv_out, token_out); + rv = hd_inflate_commit_newname(inflater, nv_out); } else { - rv = hd_inflate_commit_indname(inflater, nv_out, token_out); + rv = hd_inflate_commit_indname(inflater, nv_out); } if (rv != 0) { @@ -2710,7 +2618,7 @@ static const nghttp2_nv *hd_get_table_entry(nghttp2_hd_context *context, return NULL; } - return &nghttp2_hd_table_get(context, idx)->nv; + return nghttp2_hd_table_get2(context, idx); } size_t nghttp2_hd_deflate_get_num_table_entries(nghttp2_hd_deflater *deflater) { diff --git a/lib/nghttp2_hd.h b/lib/nghttp2_hd.h index fff4c5a5..cacfcf3a 100644 --- a/lib/nghttp2_hd.h +++ b/lib/nghttp2_hd.h @@ -34,6 +34,7 @@ #include "nghttp2_hd_huffman.h" #include "nghttp2_buf.h" #include "nghttp2_mem.h" +#include "nghttp2_rcbuf.h" #define NGHTTP2_HD_DEFAULT_MAX_BUFFER_SIZE NGHTTP2_DEFAULT_HEADER_TABLE_SIZE #define NGHTTP2_HD_ENTRY_OVERHEAD 32 @@ -168,25 +169,29 @@ typedef enum { NGHTTP2_TOKEN_X_XSS_PROTECTION, } nghttp2_token; -typedef enum { - NGHTTP2_HD_FLAG_NONE = 0, - /* Indicates name was dynamically allocated and must be freed */ - NGHTTP2_HD_FLAG_NAME_ALLOC = 1, - /* Indicates value was dynamically allocated and must be freed */ - NGHTTP2_HD_FLAG_VALUE_ALLOC = 1 << 1, - /* Indicates that the name was gifted to the entry and no copying - necessary. */ - NGHTTP2_HD_FLAG_NAME_GIFT = 1 << 2, - /* Indicates that the value was gifted to the entry and no copying - necessary. */ - NGHTTP2_HD_FLAG_VALUE_GIFT = 1 << 3 -} nghttp2_hd_flags; - struct nghttp2_hd_entry; typedef struct nghttp2_hd_entry nghttp2_hd_entry; +typedef struct { + /* The buffer containing header field name. NULL-termination is + guaranteed. */ + nghttp2_rcbuf *name; + /* The buffer containing header field value. NULL-termination is + guaranteed. */ + nghttp2_rcbuf *value; + /* nghttp2_token value for name. It could be -1 if we have no token + for that header field name. */ + int32_t token; + /* Bitwise OR of one or more of nghttp2_nv_flag. */ + uint8_t flags; +} nghttp2_hd_nv; + struct nghttp2_hd_entry { - nghttp2_nv nv; + /* The header field name/value pair */ + nghttp2_hd_nv nv; + /* This is solely for nghttp2_hd_{deflate,inflate}_get_table_entry + APIs to keep backward compatibility. */ + nghttp2_nv cnv; /* The next entry which shares same bucket in hash table. */ nghttp2_hd_entry *next; /* The sequence number. We will increment it by one whenever we @@ -194,14 +199,17 @@ struct nghttp2_hd_entry { uint32_t seq; /* The hash value for header name (nv.name). */ uint32_t hash; - /* nghttp2_token value for nv.name. It could be -1 if we have no - token for that header field name. */ - int token; - /* Reference count */ - uint8_t ref; - uint8_t flags; }; +/* The entry used for static header table. */ +typedef struct { + nghttp2_rcbuf name; + nghttp2_rcbuf value; + nghttp2_nv cnv; + int32_t token; + uint32_t hash; +} nghttp2_hd_static_entry; + typedef struct { nghttp2_hd_entry **buffer; size_t mask; @@ -275,17 +283,14 @@ struct nghttp2_hd_deflater { struct nghttp2_hd_inflater { nghttp2_hd_context ctx; - /* header buffer */ - nghttp2_buf namebuf, valuebuf; /* Stores current state of huffman decoding */ nghttp2_hd_huff_decode_context huff_decode_ctx; - /* Pointer to the nghttp2_hd_entry which is used current header - emission. This is required because in some cases the - ent_keep->ref == 0 and we have to keep track of it. */ - nghttp2_hd_entry *ent_keep; - /* Pointer to the name/value pair buffer which is used in the - current header emission. */ - uint8_t *nv_name_keep, *nv_value_keep; + /* header buffer */ + nghttp2_buf namebuf, valuebuf; + nghttp2_rcbuf *namercbuf, *valuercbuf; + /* Pointer to the name/value pair which are used in the current + header emission. */ + nghttp2_rcbuf *nv_name_keep, *nv_value_keep; /* The number of bytes to read */ size_t left; /* The index in indexed repr or indexed name */ @@ -309,24 +314,16 @@ struct nghttp2_hd_inflater { }; /* - * Initializes the |ent| members. If NGHTTP2_HD_FLAG_NAME_ALLOC bit - * set in the |flags|, the content pointed by the |name| with length - * |namelen| is copied. Likewise, if NGHTTP2_HD_FLAG_VALUE_ALLOC bit - * set in the |flags|, the content pointed by the |value| with length - * |valuelen| is copied. The |token| is enum number looked up by - * |name|. It could be -1 if we don't have that enum value. - * - * This function returns 0 if it succeeds, or one of the following - * negative error codes: - * - * NGHTTP2_ERR_NOMEM - * Out of memory. + * Initializes the |ent| members. The reference counts of nv->name + * and nv->value are increased by one for each. */ -int nghttp2_hd_entry_init(nghttp2_hd_entry *ent, uint8_t flags, uint8_t *name, - size_t namelen, uint8_t *value, size_t valuelen, - int token, nghttp2_mem *mem); +void nghttp2_hd_entry_init(nghttp2_hd_entry *ent, nghttp2_hd_nv *nv); -void nghttp2_hd_entry_free(nghttp2_hd_entry *ent, nghttp2_mem *mem); +/* + * This function decreases the reference counts of nv->name and + * nv->value. + */ +void nghttp2_hd_entry_free(nghttp2_hd_entry *ent); /* * Initializes |deflater| for deflating name/values pairs. @@ -407,16 +404,14 @@ int nghttp2_hd_inflate_init(nghttp2_hd_inflater *inflater, nghttp2_mem *mem); void nghttp2_hd_inflate_free(nghttp2_hd_inflater *inflater); /* - * Similar to nghttp2_hd_inflate_hd(), but this takes additional - * output parameter |token|. On successful header emission, it - * contains nghttp2_token value for nv_out->name. It could be -1 if - * we don't have enum value for the name. Other than that return - * values and semantics are the same as nghttp2_hd_inflate_hd(). + * Similar to nghttp2_hd_inflate_hd(), but this takes nghttp2_hd_nv + * instead of nghttp2_nv as output parameter |nv_out|. Other than + * that return values and semantics are the same as + * nghttp2_hd_inflate_hd(). */ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, - nghttp2_nv *nv_out, int *inflate_flags, - int *token, uint8_t *in, size_t inlen, - int in_final); + nghttp2_hd_nv *nv_out, int *inflate_flags, + uint8_t *in, size_t inlen, int in_final); /* For unittesting purpose */ int nghttp2_hd_emit_indname_block(nghttp2_bufs *bufs, size_t index, @@ -430,8 +425,7 @@ int nghttp2_hd_emit_newname_block(nghttp2_bufs *bufs, nghttp2_nv *nv, int nghttp2_hd_emit_table_size(nghttp2_bufs *bufs, size_t table_size); /* For unittesting purpose */ -nghttp2_hd_entry *nghttp2_hd_table_get(nghttp2_hd_context *context, - size_t index); +nghttp2_hd_nv nghttp2_hd_table_get(nghttp2_hd_context *context, size_t index); /* For unittesting purpose */ ssize_t nghttp2_hd_decode_length(uint32_t *res, size_t *shift_ptr, int *final, diff --git a/lib/nghttp2_http.c b/lib/nghttp2_http.c index 6aee90f0..f993c167 100644 --- a/lib/nghttp2_http.c +++ b/lib/nghttp2_http.c @@ -82,12 +82,12 @@ static int lws(const uint8_t *s, size_t n) { return 1; } -static int check_pseudo_header(nghttp2_stream *stream, const nghttp2_nv *nv, +static int check_pseudo_header(nghttp2_stream *stream, const nghttp2_hd_nv *nv, int flag) { if (stream->http_flags & flag) { return 0; } - if (lws(nv->value, nv->valuelen)) { + if (lws(nv->value->base, nv->value->len)) { return 0; } stream->http_flags = (uint16_t)(stream->http_flags | flag); @@ -112,16 +112,16 @@ static int check_path(nghttp2_stream *stream) { (stream->http_flags & NGHTTP2_HTTP_FLAG_PATH_ASTERISK))); } -static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, - int token, int trailer) { - if (nv->name[0] == ':') { +static int http_request_on_header(nghttp2_stream *stream, nghttp2_hd_nv *nv, + int trailer) { + if (nv->name->base[0] == ':') { if (trailer || (stream->http_flags & NGHTTP2_HTTP_FLAG_PSEUDO_HEADER_DISALLOWED)) { return NGHTTP2_ERR_HTTP_HEADER; } } - switch (token) { + switch (nv->token) { case NGHTTP2_TOKEN__AUTHORITY: if (!check_pseudo_header(stream, nv, NGHTTP2_HTTP_FLAG__AUTHORITY)) { return NGHTTP2_ERR_HTTP_HEADER; @@ -131,16 +131,16 @@ static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, if (!check_pseudo_header(stream, nv, NGHTTP2_HTTP_FLAG__METHOD)) { return NGHTTP2_ERR_HTTP_HEADER; } - switch (nv->valuelen) { + switch (nv->value->len) { case 4: - if (lstreq("HEAD", nv->value, nv->valuelen)) { + if (lstreq("HEAD", nv->value->base, nv->value->len)) { stream->http_flags |= NGHTTP2_HTTP_FLAG_METH_HEAD; } break; case 7: - switch (nv->value[6]) { + switch (nv->value->base[6]) { case 'T': - if (lstreq("CONNECT", nv->value, nv->valuelen)) { + if (lstreq("CONNECT", nv->value->base, nv->value->len)) { if (stream->stream_id % 2 == 0) { /* we won't allow CONNECT for push */ return NGHTTP2_ERR_HTTP_HEADER; @@ -153,7 +153,7 @@ static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, } break; case 'S': - if (lstreq("OPTIONS", nv->value, nv->valuelen)) { + if (lstreq("OPTIONS", nv->value->base, nv->value->len)) { stream->http_flags |= NGHTTP2_HTTP_FLAG_METH_OPTIONS; } break; @@ -168,9 +168,9 @@ static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, if (!check_pseudo_header(stream, nv, NGHTTP2_HTTP_FLAG__PATH)) { return NGHTTP2_ERR_HTTP_HEADER; } - if (nv->value[0] == '/') { + if (nv->value->base[0] == '/') { stream->http_flags |= NGHTTP2_HTTP_FLAG_PATH_REGULAR; - } else if (nv->valuelen == 1 && nv->value[0] == '*') { + } else if (nv->value->len == 1 && nv->value->base[0] == '*') { stream->http_flags |= NGHTTP2_HTTP_FLAG_PATH_ASTERISK; } break; @@ -181,8 +181,8 @@ static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, if (!check_pseudo_header(stream, nv, NGHTTP2_HTTP_FLAG__SCHEME)) { return NGHTTP2_ERR_HTTP_HEADER; } - if ((nv->valuelen == 4 && memieq("http", nv->value, 4)) || - (nv->valuelen == 5 && memieq("https", nv->value, 5))) { + if ((nv->value->len == 4 && memieq("http", nv->value->base, 4)) || + (nv->value->len == 5 && memieq("https", nv->value->base, 5))) { stream->http_flags |= NGHTTP2_HTTP_FLAG_SCHEME_HTTP; } break; @@ -195,7 +195,7 @@ static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, if (stream->content_length != -1) { return NGHTTP2_ERR_HTTP_HEADER; } - stream->content_length = parse_uint(nv->value, nv->valuelen); + stream->content_length = parse_uint(nv->value->base, nv->value->len); if (stream->content_length == -1) { return NGHTTP2_ERR_HTTP_HEADER; } @@ -209,41 +209,41 @@ static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, case NGHTTP2_TOKEN_UPGRADE: return NGHTTP2_ERR_HTTP_HEADER; case NGHTTP2_TOKEN_TE: - if (!lstrieq("trailers", nv->value, nv->valuelen)) { + if (!lstrieq("trailers", nv->value->base, nv->value->len)) { return NGHTTP2_ERR_HTTP_HEADER; } break; default: - if (nv->name[0] == ':') { + if (nv->name->base[0] == ':') { return NGHTTP2_ERR_HTTP_HEADER; } } - if (nv->name[0] != ':') { + if (nv->name->base[0] != ':') { stream->http_flags |= NGHTTP2_HTTP_FLAG_PSEUDO_HEADER_DISALLOWED; } return 0; } -static int http_response_on_header(nghttp2_stream *stream, nghttp2_nv *nv, - int token, int trailer) { - if (nv->name[0] == ':') { +static int http_response_on_header(nghttp2_stream *stream, nghttp2_hd_nv *nv, + int trailer) { + if (nv->name->base[0] == ':') { if (trailer || (stream->http_flags & NGHTTP2_HTTP_FLAG_PSEUDO_HEADER_DISALLOWED)) { return NGHTTP2_ERR_HTTP_HEADER; } } - switch (token) { + switch (nv->token) { case NGHTTP2_TOKEN__STATUS: { if (!check_pseudo_header(stream, nv, NGHTTP2_HTTP_FLAG__STATUS)) { return NGHTTP2_ERR_HTTP_HEADER; } - if (nv->valuelen != 3) { + if (nv->value->len != 3) { return NGHTTP2_ERR_HTTP_HEADER; } - stream->status_code = (int16_t)parse_uint(nv->value, nv->valuelen); + stream->status_code = (int16_t)parse_uint(nv->value->base, nv->value->len); if (stream->status_code == -1) { return NGHTTP2_ERR_HTTP_HEADER; } @@ -253,7 +253,7 @@ static int http_response_on_header(nghttp2_stream *stream, nghttp2_nv *nv, if (stream->content_length != -1) { return NGHTTP2_ERR_HTTP_HEADER; } - stream->content_length = parse_uint(nv->value, nv->valuelen); + stream->content_length = parse_uint(nv->value->base, nv->value->len); if (stream->content_length == -1) { return NGHTTP2_ERR_HTTP_HEADER; } @@ -267,17 +267,17 @@ static int http_response_on_header(nghttp2_stream *stream, nghttp2_nv *nv, case NGHTTP2_TOKEN_UPGRADE: return NGHTTP2_ERR_HTTP_HEADER; case NGHTTP2_TOKEN_TE: - if (!lstrieq("trailers", nv->value, nv->valuelen)) { + if (!lstrieq("trailers", nv->value->base, nv->value->len)) { return NGHTTP2_ERR_HTTP_HEADER; } break; default: - if (nv->name[0] == ':') { + if (nv->name->base[0] == ':') { return NGHTTP2_ERR_HTTP_HEADER; } } - if (nv->name[0] != ':') { + if (nv->name->base[0] != ':') { stream->http_flags |= NGHTTP2_HTTP_FLAG_PSEUDO_HEADER_DISALLOWED; } @@ -375,7 +375,7 @@ static int check_scheme(const uint8_t *value, size_t len) { } int nghttp2_http_on_header(nghttp2_session *session, nghttp2_stream *stream, - nghttp2_frame *frame, nghttp2_nv *nv, int token, + nghttp2_frame *frame, nghttp2_hd_nv *nv, int trailer) { int rv; @@ -386,14 +386,14 @@ int nghttp2_http_on_header(nghttp2_session *session, nghttp2_stream *stream, this, we may disrupt many web sites and/or libraries. So we become conservative here, and just ignore those illegal regular headers. */ - if (!nghttp2_check_header_name(nv->name, nv->namelen)) { + if (!nghttp2_check_header_name(nv->name->base, nv->name->len)) { size_t i; - if (nv->namelen > 0 && nv->name[0] == ':') { + if (nv->name->len > 0 && nv->name->base[0] == ':') { return NGHTTP2_ERR_HTTP_HEADER; } /* header field name must be lower-cased without exception */ - for (i = 0; i < nv->namelen; ++i) { - uint8_t c = nv->name[i]; + for (i = 0; i < nv->name->len; ++i) { + uint8_t c = nv->name->base[i]; if ('A' <= c && c <= 'Z') { return NGHTTP2_ERR_HTTP_HEADER; } @@ -405,17 +405,18 @@ int nghttp2_http_on_header(nghttp2_session *session, nghttp2_stream *stream, return NGHTTP2_ERR_IGN_HTTP_HEADER; } - if (token == NGHTTP2_TOKEN__AUTHORITY || token == NGHTTP2_TOKEN_HOST) { - rv = check_authority(nv->value, nv->valuelen); - } else if (token == NGHTTP2_TOKEN__SCHEME) { - rv = check_scheme(nv->value, nv->valuelen); + if (nv->token == NGHTTP2_TOKEN__AUTHORITY || + nv->token == NGHTTP2_TOKEN_HOST) { + rv = check_authority(nv->value->base, nv->value->len); + } else if (nv->token == NGHTTP2_TOKEN__SCHEME) { + rv = check_scheme(nv->value->base, nv->value->len); } else { - rv = nghttp2_check_header_value(nv->value, nv->valuelen); + rv = nghttp2_check_header_value(nv->value->base, nv->value->len); } if (rv == 0) { - assert(nv->namelen > 0); - if (nv->name[0] == ':') { + assert(nv->name->len > 0); + if (nv->name->base[0] == ':') { return NGHTTP2_ERR_HTTP_HEADER; } /* When ignoring regular headers, we set this flag so that we @@ -426,10 +427,10 @@ int nghttp2_http_on_header(nghttp2_session *session, nghttp2_stream *stream, } if (session->server || frame->hd.type == NGHTTP2_PUSH_PROMISE) { - return http_request_on_header(stream, nv, token, trailer); + return http_request_on_header(stream, nv, trailer); } - return http_response_on_header(stream, nv, token, trailer); + return http_response_on_header(stream, nv, trailer); } int nghttp2_http_on_request_headers(nghttp2_stream *stream, diff --git a/lib/nghttp2_http.h b/lib/nghttp2_http.h index f782058f..ac684c4d 100644 --- a/lib/nghttp2_http.h +++ b/lib/nghttp2_http.h @@ -36,8 +36,7 @@ /* * This function is called when HTTP header field |nv| in |frame| is * received for |stream|. This function will validate |nv| against - * the current state of stream. The |token| is nghttp2_token value - * for nv->name, or -1 if we don't have enum value for the name. + * the current state of stream. * * This function returns 0 if it succeeds, or one of the following * negative error codes: @@ -49,7 +48,7 @@ * if it was not received because of compatibility reasons. */ int nghttp2_http_on_header(nghttp2_session *session, nghttp2_stream *stream, - nghttp2_frame *frame, nghttp2_nv *nv, int token, + nghttp2_frame *frame, nghttp2_hd_nv *nv, int trailer); /* diff --git a/lib/nghttp2_mem.c b/lib/nghttp2_mem.c index e7d5aae3..317363a7 100644 --- a/lib/nghttp2_mem.c +++ b/lib/nghttp2_mem.c @@ -52,6 +52,10 @@ void nghttp2_mem_free(nghttp2_mem *mem, void *ptr) { mem->free(ptr, mem->mem_user_data); } +void nghttp2_mem_free2(nghttp2_free free, void *ptr, void *mem_user_data) { + free(ptr, mem_user_data); +} + void *nghttp2_mem_calloc(nghttp2_mem *mem, size_t nmemb, size_t size) { return mem->calloc(nmemb, size, mem->mem_user_data); } diff --git a/lib/nghttp2_mem.h b/lib/nghttp2_mem.h index d1fded4f..7709e1c5 100644 --- a/lib/nghttp2_mem.h +++ b/lib/nghttp2_mem.h @@ -38,6 +38,7 @@ nghttp2_mem *nghttp2_mem_default(void); |mem|. */ void *nghttp2_mem_malloc(nghttp2_mem *mem, size_t size); void nghttp2_mem_free(nghttp2_mem *mem, void *ptr); +void nghttp2_mem_free2(nghttp2_free free, void *ptr, void *mem_user_data); void *nghttp2_mem_calloc(nghttp2_mem *mem, size_t nmemb, size_t size); void *nghttp2_mem_realloc(nghttp2_mem *mem, void *ptr, size_t size); diff --git a/lib/nghttp2_rcbuf.c b/lib/nghttp2_rcbuf.c new file mode 100644 index 00000000..a1af1da8 --- /dev/null +++ b/lib/nghttp2_rcbuf.c @@ -0,0 +1,98 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "nghttp2_rcbuf.h" + +#include +#include + +#include "nghttp2_mem.h" + +int nghttp2_rcbuf_new(nghttp2_rcbuf **rcbuf_ptr, size_t size, + nghttp2_mem *mem) { + uint8_t *p; + + p = nghttp2_mem_malloc(mem, sizeof(nghttp2_rcbuf) + size); + if (p == NULL) { + return NGHTTP2_ERR_NOMEM; + } + + *rcbuf_ptr = (void *)p; + + (*rcbuf_ptr)->mem_user_data = mem->mem_user_data; + (*rcbuf_ptr)->free = mem->free; + (*rcbuf_ptr)->base = p + sizeof(nghttp2_rcbuf); + (*rcbuf_ptr)->len = size; + (*rcbuf_ptr)->ref = 1; + + return 0; +} + +int nghttp2_rcbuf_new2(nghttp2_rcbuf **rcbuf_ptr, const uint8_t *src, + size_t srclen, nghttp2_mem *mem) { + int rv; + + rv = nghttp2_rcbuf_new(rcbuf_ptr, srclen + 1, mem); + if (rv != 0) { + return rv; + } + + memcpy((*rcbuf_ptr)->base, src, srclen); + + (*rcbuf_ptr)->len = srclen; + (*rcbuf_ptr)->base[srclen] = '\0'; + + return 0; +} + +/* + * Frees |rcbuf| itself, regardless of its reference cout. + */ +void nghttp2_rcbuf_del(nghttp2_rcbuf *rcbuf) { + nghttp2_mem_free2(rcbuf->free, rcbuf, rcbuf->mem_user_data); +} + +void nghttp2_rcbuf_incref(nghttp2_rcbuf *rcbuf) { + if (rcbuf->ref == -1) { + return; + } + + ++rcbuf->ref; +} + +void nghttp2_rcbuf_decref(nghttp2_rcbuf *rcbuf) { + if (rcbuf == NULL || rcbuf->ref == -1) { + return; + } + + assert(rcbuf->ref > 0); + + if (--rcbuf->ref == 0) { + nghttp2_rcbuf_del(rcbuf); + } +} + +nghttp2_vec nghttp2_rcbuf_get_buf(nghttp2_rcbuf *rcbuf) { + return (nghttp2_vec){rcbuf->base, rcbuf->len}; +} diff --git a/lib/nghttp2_rcbuf.h b/lib/nghttp2_rcbuf.h new file mode 100644 index 00000000..29d1543e --- /dev/null +++ b/lib/nghttp2_rcbuf.h @@ -0,0 +1,80 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef NGHTTP2_RCBUF_H +#define NGHTTP2_RCBUF_H + +#ifdef HAVE_CONFIG_H +#include +#endif /* HAVE_CONFIG_H */ + +#include + +struct nghttp2_rcbuf { + /* custom memory allocator belongs to the mem parameter when + creating this object. */ + void *mem_user_data; + nghttp2_free free; + /* The pointer to the underlying buffer */ + uint8_t *base; + /* Size of buffer pointed by |base|. */ + size_t len; + /* Reference count */ + int32_t ref; +}; + +/* + * Allocates nghttp2_rcbuf object with |size| as initial buffer size. + * When the function succeeds, the reference count becomes 1. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * NGHTTP2_ERR_NOMEM: + * Out of memory. + */ +int nghttp2_rcbuf_new(nghttp2_rcbuf **rcbuf_ptr, size_t size, nghttp2_mem *mem); + +/* + * Like nghttp2_rcbuf_new(), but initializes the buffer with |src| of + * length |srclen|. This function allocates additional byte at the + * end and puts '\0' into it, so that the resulting buffer could be + * used as NULL-terminated string. Still (*rcbuf_ptr)->len equals to + * |srclen|. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * NGHTTP2_ERR_NOMEM: + * Out of memory. + */ +int nghttp2_rcbuf_new2(nghttp2_rcbuf **rcbuf_ptr, const uint8_t *src, + size_t srclen, nghttp2_mem *mem); + +/* + * Frees |rcbuf| itself, regardless of its reference cout. + */ +void nghttp2_rcbuf_del(nghttp2_rcbuf *rcbuf); + +#endif /* NGHTTP2_RCBUF_H */ diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index cbc1bac6..92b1c482 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -3122,12 +3122,12 @@ static int session_call_on_begin_headers(nghttp2_session *session, static int session_call_on_header(nghttp2_session *session, const nghttp2_frame *frame, - const nghttp2_nv *nv) { + const nghttp2_hd_nv *nv) { int rv; if (session->callbacks.on_header_callback) { rv = session->callbacks.on_header_callback( - session, frame, nv->name, nv->namelen, nv->value, nv->valuelen, - nv->flags, session->user_data); + session, frame, nv->name->base, nv->name->len, nv->value->base, + nv->value->len, nv->flags, session->user_data); if (rv == NGHTTP2_ERR_PAUSE || rv == NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE) { return rv; @@ -3317,11 +3317,10 @@ static int inflate_header_block(nghttp2_session *session, nghttp2_frame *frame, ssize_t proclen; int rv; int inflate_flags; - nghttp2_nv nv; + nghttp2_hd_nv nv; nghttp2_stream *stream; nghttp2_stream *subject_stream; int trailer = 0; - int token; *readlen_ptr = 0; stream = nghttp2_session_get_stream(session, frame->hd.stream_id); @@ -3338,7 +3337,7 @@ static int inflate_header_block(nghttp2_session *session, nghttp2_frame *frame, for (;;) { inflate_flags = 0; proclen = nghttp2_hd_inflate_hd2(&session->hd_inflater, &nv, &inflate_flags, - &token, in, inlen, final); + in, inlen, final); if (nghttp2_is_fatal((int)proclen)) { return (int)proclen; } @@ -3373,13 +3372,13 @@ static int inflate_header_block(nghttp2_session *session, nghttp2_frame *frame, if (call_header_cb && (inflate_flags & NGHTTP2_HD_INFLATE_EMIT)) { rv = 0; if (subject_stream && session_enforce_http_messaging(session)) { - rv = nghttp2_http_on_header(session, subject_stream, frame, &nv, token, + rv = nghttp2_http_on_header(session, subject_stream, frame, &nv, trailer); if (rv == NGHTTP2_ERR_HTTP_HEADER) { DEBUGF(fprintf( stderr, "recv: HTTP error: type=%d, id=%d, header %.*s: %.*s\n", - frame->hd.type, subject_stream->stream_id, (int)nv.namelen, - nv.name, (int)nv.valuelen, nv.value)); + frame->hd.type, subject_stream->stream_id, (int)nv.name->len, + nv.name->base, (int)nv.value->len, nv.value->base)); rv = session_handle_invalid_stream2(session, subject_stream->stream_id, @@ -3394,8 +3393,8 @@ static int inflate_header_block(nghttp2_session *session, nghttp2_frame *frame, /* header is ignored */ DEBUGF(fprintf( stderr, "recv: HTTP ignored: type=%d, id=%d, header %.*s: %.*s\n", - frame->hd.type, subject_stream->stream_id, (int)nv.namelen, - nv.name, (int)nv.valuelen, nv.value)); + frame->hd.type, subject_stream->stream_id, (int)nv.name->len, + nv.name->base, (int)nv.value->len, nv.value->base)); } } if (rv == 0) { diff --git a/tests/nghttp2_hd_test.c b/tests/nghttp2_hd_test.c index ed954bde..3d8ab83b 100644 --- a/tests/nghttp2_hd_test.c +++ b/tests/nghttp2_hd_test.c @@ -295,11 +295,6 @@ void test_nghttp2_hd_inflate_indname_inc(void) { assert_nv_equal(&nv, out.nva, 1, mem); CU_ASSERT(1 == inflater.ctx.hd_table.len); CU_ASSERT(62 == nghttp2_hd_inflate_get_num_table_entries(&inflater)); - assert_nv_equal(&nv, - &nghttp2_hd_table_get(&inflater.ctx, - NGHTTP2_STATIC_TABLE_LENGTH + - inflater.ctx.hd_table.len - 1)->nv, - 1, mem); assert_nv_equal(&nv, nghttp2_hd_inflate_get_table_entry( &inflater, NGHTTP2_STATIC_TABLE_LENGTH + inflater.ctx.hd_table.len), @@ -429,10 +424,9 @@ void test_nghttp2_hd_inflate_newname_inc(void) { CU_ASSERT(1 == out.nvlen); assert_nv_equal(&nv, out.nva, 1, mem); CU_ASSERT(1 == inflater.ctx.hd_table.len); - assert_nv_equal(&nv, - &nghttp2_hd_table_get(&inflater.ctx, - NGHTTP2_STATIC_TABLE_LENGTH + - inflater.ctx.hd_table.len - 1)->nv, + assert_nv_equal(&nv, nghttp2_hd_inflate_get_table_entry( + &inflater, NGHTTP2_STATIC_TABLE_LENGTH + + inflater.ctx.hd_table.len), 1, mem); nva_out_reset(&out, mem); From 689d2a1afbc683f80e9241440ff458d61f1e09be Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 13:24:02 +0900 Subject: [PATCH 114/147] Fix compile error with --enable-debug --- lib/nghttp2_stream.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/nghttp2_stream.c b/lib/nghttp2_stream.c index 70325bc3..3221fbf8 100644 --- a/lib/nghttp2_stream.c +++ b/lib/nghttp2_stream.c @@ -152,7 +152,7 @@ static int stream_obq_push(nghttp2_stream *dep_stream, nghttp2_stream *stream) { stream_next_cycle(stream, dep_stream->descendant_last_cycle); stream->seq = dep_stream->descendant_next_seq++; - DEBUGF(fprintf(stderr, "stream: stream=%d obq push cycle=%ld\n", + DEBUGF(fprintf(stderr, "stream: stream=%d obq push cycle=%d\n", stream->stream_id, stream->cycle)); DEBUGF(fprintf(stderr, "stream: push stream %d to stream %d\n", @@ -238,7 +238,7 @@ void nghttp2_stream_reschedule(nghttp2_stream *stream) { nghttp2_pq_push(&dep_stream->obq, &stream->pq_entry); - DEBUGF(fprintf(stderr, "stream: stream=%d obq resched cycle=%ld\n", + DEBUGF(fprintf(stderr, "stream: stream=%d obq resched cycle=%d\n", stream->stream_id, stream->cycle)); dep_stream->last_writelen = stream->last_writelen; @@ -298,7 +298,7 @@ void nghttp2_stream_change_weight(nghttp2_stream *stream, int32_t weight) { nghttp2_pq_push(&dep_stream->obq, &stream->pq_entry); - DEBUGF(fprintf(stderr, "stream: stream=%d obq resched cycle=%ld\n", + DEBUGF(fprintf(stderr, "stream: stream=%d obq resched cycle=%d\n", stream->stream_id, stream->cycle)); } From 12dad328902bdccdfbde8d803b73bd82295b7e47 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 15:05:20 +0900 Subject: [PATCH 115/147] Add nghttp2_on_header_callback2 --- doc/Makefile.am | 4 +++ lib/includes/nghttp2/nghttp2.h | 49 ++++++++++++++++++++++++--- lib/nghttp2_callbacks.c | 6 ++++ lib/nghttp2_callbacks.h | 1 + lib/nghttp2_session.c | 22 +++++++----- src/shrpx_downstream.cc | 9 +++++ src/shrpx_downstream.h | 4 +++ src/shrpx_http2_session.cc | 61 +++++++++++++++++++--------------- src/shrpx_http2_upstream.cc | 41 ++++++++++++----------- 9 files changed, 139 insertions(+), 58 deletions(-) diff --git a/doc/Makefile.am b/doc/Makefile.am index 35f1390b..c450a029 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -65,6 +65,9 @@ APIDOCS= \ nghttp2_priority_spec_check_default.rst \ nghttp2_priority_spec_default_init.rst \ nghttp2_priority_spec_init.rst \ + nghttp2_rcbuf_decref.rst \ + nghttp2_rcbuf_get_buf.rst \ + nghttp2_rcbuf_incref.rst \ nghttp2_select_next_protocol.rst \ nghttp2_session_callbacks_del.rst \ nghttp2_session_callbacks_new.rst \ @@ -78,6 +81,7 @@ APIDOCS= \ nghttp2_session_callbacks_set_on_frame_recv_callback.rst \ nghttp2_session_callbacks_set_on_frame_send_callback.rst \ nghttp2_session_callbacks_set_on_header_callback.rst \ + nghttp2_session_callbacks_set_on_header_callback2.rst \ nghttp2_session_callbacks_set_on_invalid_frame_recv_callback.rst \ nghttp2_session_callbacks_set_on_stream_close_callback.rst \ nghttp2_session_callbacks_set_pack_extension_callback.rst \ diff --git a/lib/includes/nghttp2/nghttp2.h b/lib/includes/nghttp2/nghttp2.h index 2a3674fb..0e888fa7 100644 --- a/lib/includes/nghttp2/nghttp2.h +++ b/lib/includes/nghttp2/nghttp2.h @@ -450,7 +450,7 @@ typedef struct nghttp2_rcbuf nghttp2_rcbuf; * * Increments the reference count of |rcbuf| by 1. */ -void nghttp2_rcbuf_incref(nghttp2_rcbuf *rcbuf); +NGHTTP2_EXTERN void nghttp2_rcbuf_incref(nghttp2_rcbuf *rcbuf); /** * @function @@ -459,12 +459,14 @@ void nghttp2_rcbuf_incref(nghttp2_rcbuf *rcbuf); * count becomes zero, the object pointed by |rcbuf| will be freed. * In this case, application must not use |rcbuf| again. */ -void nghttp2_rcbuf_decref(nghttp2_rcbuf *rcbuf); +NGHTTP2_EXTERN void nghttp2_rcbuf_decref(nghttp2_rcbuf *rcbuf); /** + * @function + * * Returns the underlying buffer managed by |rcbuf|. */ -nghttp2_vec nghttp2_rcbuf_get_buf(nghttp2_rcbuf *rcbuf); +NGHTTP2_EXTERN nghttp2_vec nghttp2_rcbuf_get_buf(nghttp2_rcbuf *rcbuf); /** * @enum @@ -1674,6 +1676,32 @@ typedef int (*nghttp2_on_header_callback)(nghttp2_session *session, const uint8_t *value, size_t valuelen, uint8_t flags, void *user_data); +/** + * @functypedef + * + * Callback function invoked when a header name/value pair is received + * for the |frame|. The |name| is header name. The |value| is header + * value. The |flags| is bitwise OR of one or more of + * :type:`nghttp2_nv_flag`. + * + * This callback behaves like :type:`nghttp2_on_header_callback`, + * except that |name| and |value| are stored in reference counted + * buffer. If application wishes to keep these references without + * copying them, use `nghttp2_rcbuf_incref()` to increment their + * reference count. It is the application's responsibility to call + * `nghttp2_rcbuf_decref()` if they called `nghttp2_rcbuf_incref()` so + * as not to leak memory. If the |session| is created by + * `nghttp2_session_server_new3()` or `nghttp2_session_client_new3()`, + * the function to free memory is the one belongs to the mem + * parameter. As long as this free function alives, |name| and + * |value| can live after |session| was destroyed. + */ +typedef int (*nghttp2_on_header_callback2)(nghttp2_session *session, + const nghttp2_frame *frame, + nghttp2_rcbuf *name, + nghttp2_rcbuf *value, uint8_t flags, + void *user_data); + /** * @functypedef * @@ -1988,12 +2016,25 @@ NGHTTP2_EXTERN void nghttp2_session_callbacks_set_on_begin_headers_callback( * @function * * Sets callback function invoked when a header name/value pair is - * received. + * received. If both + * `nghttp2_session_callbacks_set_on_header_callback()` and + * `nghttp2_session_callbacks_set_on_header_callback2()` are used to + * set callbacks, the latter has the precedence. */ NGHTTP2_EXTERN void nghttp2_session_callbacks_set_on_header_callback( nghttp2_session_callbacks *cbs, nghttp2_on_header_callback on_header_callback); +/** + * @function + * + * Sets callback function invoked when a header name/value pair is + * received. + */ +NGHTTP2_EXTERN void nghttp2_session_callbacks_set_on_header_callback2( + nghttp2_session_callbacks *cbs, + nghttp2_on_header_callback2 on_header_callback2); + /** * @function * diff --git a/lib/nghttp2_callbacks.c b/lib/nghttp2_callbacks.c index 3b0369c1..4bf0e7a5 100644 --- a/lib/nghttp2_callbacks.c +++ b/lib/nghttp2_callbacks.c @@ -104,6 +104,12 @@ void nghttp2_session_callbacks_set_on_header_callback( cbs->on_header_callback = on_header_callback; } +void nghttp2_session_callbacks_set_on_header_callback2( + nghttp2_session_callbacks *cbs, + nghttp2_on_header_callback2 on_header_callback2) { + cbs->on_header_callback2 = on_header_callback2; +} + void nghttp2_session_callbacks_set_select_padding_callback( nghttp2_session_callbacks *cbs, nghttp2_select_padding_callback select_padding_callback) { diff --git a/lib/nghttp2_callbacks.h b/lib/nghttp2_callbacks.h index 664bf35b..80971c54 100644 --- a/lib/nghttp2_callbacks.h +++ b/lib/nghttp2_callbacks.h @@ -91,6 +91,7 @@ struct nghttp2_session_callbacks { * received. */ nghttp2_on_header_callback on_header_callback; + nghttp2_on_header_callback2 on_header_callback2; /** * Callback function invoked when the library asks application how * many padding bytes are required for the transmission of the given diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 92b1c482..44ed7203 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -3123,19 +3123,23 @@ static int session_call_on_begin_headers(nghttp2_session *session, static int session_call_on_header(nghttp2_session *session, const nghttp2_frame *frame, const nghttp2_hd_nv *nv) { - int rv; - if (session->callbacks.on_header_callback) { + int rv = 0; + if (session->callbacks.on_header_callback2) { + rv = session->callbacks.on_header_callback2( + session, frame, nv->name, nv->value, nv->flags, session->user_data); + } else if (session->callbacks.on_header_callback) { rv = session->callbacks.on_header_callback( session, frame, nv->name->base, nv->name->len, nv->value->base, nv->value->len, nv->flags, session->user_data); - if (rv == NGHTTP2_ERR_PAUSE || - rv == NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE) { - return rv; - } - if (rv != 0) { - return NGHTTP2_ERR_CALLBACK_FAILURE; - } } + + if (rv == NGHTTP2_ERR_PAUSE || rv == NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE) { + return rv; + } + if (rv != 0) { + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + return 0; } diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 51931aad..0cdd81b8 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -182,6 +182,10 @@ Downstream::~Downstream() { // explicitly. dconn_.reset(); + for (auto rcbuf : rcbufs_) { + nghttp2_rcbuf_decref(rcbuf); + } + if (LOG_ENABLED(INFO)) { DLOG(INFO, this) << "Deleted"; } @@ -905,4 +909,9 @@ int32_t Downstream::get_assoc_stream_id() const { return assoc_stream_id_; } BlockAllocator &Downstream::get_block_allocator() { return balloc_; } +void Downstream::add_rcbuf(nghttp2_rcbuf *rcbuf) { + nghttp2_rcbuf_incref(rcbuf); + rcbufs_.push_back(rcbuf); +} + } // namespace shrpx diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index a5024169..0d987517 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -374,6 +374,8 @@ public: BlockAllocator &get_block_allocator(); + void add_rcbuf(nghttp2_rcbuf *rcbuf); + enum { EVENT_ERROR = 0x1, EVENT_TIMEOUT = 0x2, @@ -395,6 +397,8 @@ public: private: BlockAllocator balloc_; + std::vector rcbufs_; + Request req_; Response resp_; diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 25a7f996..84258896 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -801,10 +801,9 @@ void Http2Session::stop_settings_timer() { } namespace { -int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, - const uint8_t *name, size_t namelen, - const uint8_t *value, size_t valuelen, uint8_t flags, - void *user_data) { +int on_header_callback2(nghttp2_session *session, const nghttp2_frame *frame, + nghttp2_rcbuf *name, nghttp2_rcbuf *value, + uint8_t flags, void *user_data) { auto http2session = static_cast(user_data); auto sd = static_cast( nghttp2_session_get_stream_user_data(session, frame->hd.stream_id)); @@ -816,22 +815,25 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, return 0; } + auto namebuf = nghttp2_rcbuf_get_buf(name); + auto valuebuf = nghttp2_rcbuf_get_buf(value); + auto &resp = downstream->response(); auto &httpconf = get_config()->http; - auto &balloc = downstream->get_block_allocator(); switch (frame->hd.type) { case NGHTTP2_HEADERS: { auto trailer = frame->headers.cat == NGHTTP2_HCAT_HEADERS && !downstream->get_expect_final_response(); - if (resp.fs.buffer_size() + namelen + valuelen > + if (resp.fs.buffer_size() + namebuf.len + valuebuf.len > httpconf.response_header_field_buffer || resp.fs.num_fields() >= httpconf.max_response_header_fields) { if (LOG_ENABLED(INFO)) { - DLOG(INFO, downstream) << "Too large or many header field size=" - << resp.fs.buffer_size() + namelen + valuelen - << ", num=" << resp.fs.num_fields() + 1; + DLOG(INFO, downstream) + << "Too large or many header field size=" + << resp.fs.buffer_size() + namebuf.len + valuebuf.len + << ", num=" << resp.fs.num_fields() + 1; } if (trailer) { @@ -843,20 +845,23 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; } - auto token = http2::lookup_token(name, namelen); + auto token = http2::lookup_token(namebuf.base, namebuf.len); auto no_index = flags & NGHTTP2_NV_FLAG_NO_INDEX; + downstream->add_rcbuf(name); + downstream->add_rcbuf(value); + if (trailer) { // just store header fields for trailer part - resp.fs.add_trailer_token( - make_string_ref(balloc, StringRef{name, namelen}), - make_string_ref(balloc, StringRef{value, valuelen}), no_index, token); + resp.fs.add_trailer_token(StringRef{namebuf.base, namebuf.len}, + StringRef{valuebuf.base, valuebuf.len}, + no_index, token); return 0; } - resp.fs.add_header_token( - make_string_ref(balloc, StringRef{name, namelen}), - make_string_ref(balloc, StringRef{value, valuelen}), no_index, token); + resp.fs.add_header_token(StringRef{namebuf.base, namebuf.len}, + StringRef{valuebuf.base, valuebuf.len}, no_index, + token); return 0; } case NGHTTP2_PUSH_PROMISE: { @@ -870,30 +875,34 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, auto promised_downstream = promised_sd->dconn->get_downstream(); + auto namebuf = nghttp2_rcbuf_get_buf(name); + auto valuebuf = nghttp2_rcbuf_get_buf(value); + assert(promised_downstream); auto &promised_req = promised_downstream->request(); - auto &promised_balloc = promised_downstream->get_block_allocator(); // We use request header limit for PUSH_PROMISE - if (promised_req.fs.buffer_size() + namelen + valuelen > + if (promised_req.fs.buffer_size() + namebuf.len + valuebuf.len > httpconf.request_header_field_buffer || promised_req.fs.num_fields() >= httpconf.max_request_header_fields) { if (LOG_ENABLED(INFO)) { DLOG(INFO, downstream) << "Too large or many header field size=" - << promised_req.fs.buffer_size() + namelen + valuelen + << promised_req.fs.buffer_size() + namebuf.len + valuebuf.len << ", num=" << promised_req.fs.num_fields() + 1; } return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; } - auto token = http2::lookup_token(name, namelen); - promised_req.fs.add_header_token( - make_string_ref(promised_balloc, StringRef{name, namelen}), - make_string_ref(promised_balloc, StringRef{value, valuelen}), - flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + promised_downstream->add_rcbuf(name); + promised_downstream->add_rcbuf(value); + + auto token = http2::lookup_token(namebuf.base, namebuf.len); + promised_req.fs.add_header_token(StringRef{namebuf.base, namebuf.len}, + StringRef{valuebuf.base, valuebuf.len}, + flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } @@ -1402,8 +1411,8 @@ nghttp2_session_callbacks *create_http2_downstream_callbacks() { nghttp2_session_callbacks_set_on_frame_not_send_callback( callbacks, on_frame_not_send_callback); - nghttp2_session_callbacks_set_on_header_callback(callbacks, - on_header_callback); + nghttp2_session_callbacks_set_on_header_callback2(callbacks, + on_header_callback2); nghttp2_session_callbacks_set_on_begin_headers_callback( callbacks, on_begin_headers_callback); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index a3c78572..79fac5de 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -154,13 +154,15 @@ void Http2Upstream::stop_settings_timer() { } namespace { -int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, - const uint8_t *name, size_t namelen, - const uint8_t *value, size_t valuelen, uint8_t flags, - void *user_data) { +int on_header_callback2(nghttp2_session *session, const nghttp2_frame *frame, + nghttp2_rcbuf *name, nghttp2_rcbuf *value, + uint8_t flags, void *user_data) { + auto namebuf = nghttp2_rcbuf_get_buf(name); + auto valuebuf = nghttp2_rcbuf_get_buf(value); + if (get_config()->http2.upstream.debug.frame_debug) { - verbose_on_header_callback(session, frame, name, namelen, value, valuelen, - flags, user_data); + verbose_on_header_callback(session, frame, namebuf.base, namebuf.len, + valuebuf.base, valuebuf.len, flags, user_data); } if (frame->hd.type != NGHTTP2_HEADERS) { return 0; @@ -176,9 +178,7 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, auto &httpconf = get_config()->http; - auto &balloc = downstream->get_block_allocator(); - - if (req.fs.buffer_size() + namelen + valuelen > + if (req.fs.buffer_size() + namebuf.len + valuebuf.len > httpconf.request_header_field_buffer || req.fs.num_fields() >= httpconf.max_request_header_fields) { if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { @@ -187,7 +187,7 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, if (LOG_ENABLED(INFO)) { ULOG(INFO, upstream) << "Too large or many header field size=" - << req.fs.buffer_size() + namelen + valuelen + << req.fs.buffer_size() + namebuf.len + valuebuf.len << ", num=" << req.fs.num_fields() + 1; } @@ -203,20 +203,23 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, return 0; } - auto token = http2::lookup_token(name, namelen); + auto token = http2::lookup_token(namebuf.base, namebuf.len); auto no_index = flags & NGHTTP2_NV_FLAG_NO_INDEX; + downstream->add_rcbuf(name); + downstream->add_rcbuf(value); + if (frame->headers.cat == NGHTTP2_HCAT_HEADERS) { // just store header fields for trailer part - req.fs.add_trailer_token( - make_string_ref(balloc, StringRef{name, namelen}), - make_string_ref(balloc, StringRef{value, valuelen}), no_index, token); + req.fs.add_trailer_token(StringRef{namebuf.base, namebuf.len}, + StringRef{valuebuf.base, valuebuf.len}, no_index, + token); return 0; } - req.fs.add_header_token(make_string_ref(balloc, StringRef{name, namelen}), - make_string_ref(balloc, StringRef{value, valuelen}), - no_index, token); + req.fs.add_header_token(StringRef{namebuf.base, namebuf.len}, + StringRef{valuebuf.base, valuebuf.len}, no_index, + token); return 0; } } // namespace @@ -820,8 +823,8 @@ nghttp2_session_callbacks *create_http2_upstream_callbacks() { nghttp2_session_callbacks_set_on_frame_not_send_callback( callbacks, on_frame_not_send_callback); - nghttp2_session_callbacks_set_on_header_callback(callbacks, - on_header_callback); + nghttp2_session_callbacks_set_on_header_callback2(callbacks, + on_header_callback2); nghttp2_session_callbacks_set_on_begin_headers_callback( callbacks, on_begin_headers_callback); From 3db9c2c7964872239c439010303fdd062921c813 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 15:54:48 +0900 Subject: [PATCH 116/147] Remove extra tokenization of header names; they are not effective at best --- genlibtokenlookup.py | 62 +-------- lib/nghttp2_hd.c | 296 ------------------------------------------- lib/nghttp2_hd.h | 62 +-------- 3 files changed, 6 insertions(+), 414 deletions(-) diff --git a/genlibtokenlookup.py b/genlibtokenlookup.py index 360d2198..e0726c59 100755 --- a/genlibtokenlookup.py +++ b/genlibtokenlookup.py @@ -62,67 +62,11 @@ HEADERS = [ ('vary', 58), ('via', 59), ('www-authenticate', 60), - ('accept-ch', None), - ('accept-datetime', None), - ('accept-features', None), - ('accept-patch', None), - ('access-control-allow-credentials', None), - ('access-control-allow-headers', None), - ('access-control-allow-methods', None), - ('access-control-expose-headers', None), - ('access-control-max-age', None), - ('access-control-request-headers', None), - ('access-control-request-method', None), - ('alt-svc', None), - ('alternates', None), - ('connection', None), - ('content-md5', None), - ('content-security-policy', None), - ('content-security-policy-report-only', None), - ('dnt', None), - ('forwarded', None), - ('front-end-https', None), - ('keep-alive', None), - ('last-event-id', None), - ('negotiate', None), - ('origin', None), - ('p3p', None), - ('pragma', None), - ('proxy-connection', None), - ('public-key-pins', None), - ('sec-websocket-extensions', None), - ('sec-websocket-key', None), - ('sec-websocket-origin', None), - ('sec-websocket-protocol', None), - ('sec-websocket-version', None), - ('set-cookie2', None), - ('status', None), - ('tcn', None), ('te', None), - ('trailer', None), - ('tsv', None), + ('connection', None), + ('keep-alive',None), + ('proxy-connection', None), ('upgrade', None), - ('upgrade-insecure-requests', None), - ('variant-vary', None), - ('warning', None), - ('x-api-version', None), - ('x-att-deviceid', None), - ('x-cache', None), - ('x-cache-lookup', None), - ('x-content-duration', None), - ('x-content-security-policy', None), - ('x-content-type-options', None), - ('x-dnsprefetch-control', None), - ('x-forwarded-for', None), - ('x-forwarded-host', None), - ('x-forwarded-proto', None), - ('x-frame-options', None), - ('x-powered-by', None), - ('x-requested-with', None), - ('x-ua-compatible', None), - ('x-wap-profile', None), - ('x-webkit-csp', None), - ('x-xss-protection', None), ] def to_enum_hd(k): diff --git a/lib/nghttp2_hd.c b/lib/nghttp2_hd.c index 8da40064..3aedc0fd 100644 --- a/lib/nghttp2_hd.c +++ b/lib/nghttp2_hd.c @@ -139,26 +139,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_AGE; } break; - case 'n': - if (lstreq("tc", name, 2)) { - return NGHTTP2_TOKEN_TCN; - } - break; - case 'p': - if (lstreq("p3", name, 2)) { - return NGHTTP2_TOKEN_P3P; - } - break; - case 't': - if (lstreq("dn", name, 2)) { - return NGHTTP2_TOKEN_DNT; - } - break; - case 'v': - if (lstreq("ts", name, 2)) { - return NGHTTP2_TOKEN_TSV; - } - break; } break; case 4: @@ -219,31 +199,16 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { break; case 6: switch (name[5]) { - case 'a': - if (lstreq("pragm", name, 5)) { - return NGHTTP2_TOKEN_PRAGMA; - } - break; case 'e': if (lstreq("cooki", name, 5)) { return NGHTTP2_TOKEN_COOKIE; } break; - case 'n': - if (lstreq("origi", name, 5)) { - return NGHTTP2_TOKEN_ORIGIN; - } - break; case 'r': if (lstreq("serve", name, 5)) { return NGHTTP2_TOKEN_SERVER; } break; - case 's': - if (lstreq("statu", name, 5)) { - return NGHTTP2_TOKEN_STATUS; - } - break; case 't': if (lstreq("accep", name, 5)) { return NGHTTP2_TOKEN_ACCEPT; @@ -256,11 +221,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { break; case 7: switch (name[6]) { - case 'c': - if (lstreq("alt-sv", name, 6)) { - return NGHTTP2_TOKEN_ALT_SVC; - } - break; case 'd': if (lstreq(":metho", name, 6)) { return NGHTTP2_TOKEN__METHOD; @@ -279,14 +239,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("upgrad", name, 6)) { return NGHTTP2_TOKEN_UPGRADE; } - if (lstreq("x-cach", name, 6)) { - return NGHTTP2_TOKEN_X_CACHE; - } - break; - case 'g': - if (lstreq("warnin", name, 6)) { - return NGHTTP2_TOKEN_WARNING; - } break; case 'h': if (lstreq("refres", name, 6)) { @@ -297,9 +249,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("refere", name, 6)) { return NGHTTP2_TOKEN_REFERER; } - if (lstreq("traile", name, 6)) { - return NGHTTP2_TOKEN_TRAILER; - } break; case 's': if (lstreq(":statu", name, 6)) { @@ -348,25 +297,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { break; } break; - case 9: - switch (name[8]) { - case 'd': - if (lstreq("forwarde", name, 8)) { - return NGHTTP2_TOKEN_FORWARDED; - } - break; - case 'e': - if (lstreq("negotiat", name, 8)) { - return NGHTTP2_TOKEN_NEGOTIATE; - } - break; - case 'h': - if (lstreq("accept-c", name, 8)) { - return NGHTTP2_TOKEN_ACCEPT_CH; - } - break; - } - break; case 10: switch (name[9]) { case 'e': @@ -382,11 +312,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_CONNECTION; } break; - case 's': - if (lstreq("alternate", name, 9)) { - return NGHTTP2_TOKEN_ALTERNATES; - } - break; case 't': if (lstreq("user-agen", name, 9)) { return NGHTTP2_TOKEN_USER_AGENT; @@ -401,16 +326,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { break; case 11: switch (name[10]) { - case '2': - if (lstreq("set-cookie", name, 10)) { - return NGHTTP2_TOKEN_SET_COOKIE2; - } - break; - case '5': - if (lstreq("content-md", name, 10)) { - return NGHTTP2_TOKEN_CONTENT_MD5; - } - break; case 'r': if (lstreq("retry-afte", name, 10)) { return NGHTTP2_TOKEN_RETRY_AFTER; @@ -425,37 +340,16 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_CONTENT_TYPE; } break; - case 'h': - if (lstreq("accept-patc", name, 11)) { - return NGHTTP2_TOKEN_ACCEPT_PATCH; - } - break; - case 'p': - if (lstreq("x-webkit-cs", name, 11)) { - return NGHTTP2_TOKEN_X_WEBKIT_CSP; - } - break; case 's': if (lstreq("max-forward", name, 11)) { return NGHTTP2_TOKEN_MAX_FORWARDS; } break; - case 'y': - if (lstreq("variant-var", name, 11)) { - return NGHTTP2_TOKEN_VARIANT_VARY; - } - if (lstreq("x-powered-b", name, 11)) { - return NGHTTP2_TOKEN_X_POWERED_BY; - } - break; } break; case 13: switch (name[12]) { case 'd': - if (lstreq("last-event-i", name, 12)) { - return NGHTTP2_TOKEN_LAST_EVENT_ID; - } if (lstreq("last-modifie", name, 12)) { return NGHTTP2_TOKEN_LAST_MODIFIED; } @@ -464,9 +358,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("content-rang", name, 12)) { return NGHTTP2_TOKEN_CONTENT_RANGE; } - if (lstreq("x-wap-profil", name, 12)) { - return NGHTTP2_TOKEN_X_WAP_PROFILE; - } break; case 'h': if (lstreq("if-none-matc", name, 12)) { @@ -482,9 +373,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("authorizatio", name, 12)) { return NGHTTP2_TOKEN_AUTHORIZATION; } - if (lstreq("x-api-versio", name, 12)) { - return NGHTTP2_TOKEN_X_API_VERSION; - } break; case 's': if (lstreq("accept-range", name, 12)) { @@ -495,21 +383,11 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { break; case 14: switch (name[13]) { - case 'd': - if (lstreq("x-att-devicei", name, 13)) { - return NGHTTP2_TOKEN_X_ATT_DEVICEID; - } - break; case 'h': if (lstreq("content-lengt", name, 13)) { return NGHTTP2_TOKEN_CONTENT_LENGTH; } break; - case 'p': - if (lstreq("x-cache-looku", name, 13)) { - return NGHTTP2_TOKEN_X_CACHE_LOOKUP; - } - break; case 't': if (lstreq("accept-charse", name, 13)) { return NGHTTP2_TOKEN_ACCEPT_CHARSET; @@ -520,40 +398,15 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { case 15: switch (name[14]) { case 'e': - if (lstreq("accept-datetim", name, 14)) { - return NGHTTP2_TOKEN_ACCEPT_DATETIME; - } if (lstreq("accept-languag", name, 14)) { return NGHTTP2_TOKEN_ACCEPT_LANGUAGE; } - if (lstreq("x-ua-compatibl", name, 14)) { - return NGHTTP2_TOKEN_X_UA_COMPATIBLE; - } break; case 'g': if (lstreq("accept-encodin", name, 14)) { return NGHTTP2_TOKEN_ACCEPT_ENCODING; } break; - case 'r': - if (lstreq("x-forwarded-fo", name, 14)) { - return NGHTTP2_TOKEN_X_FORWARDED_FOR; - } - break; - case 's': - if (lstreq("accept-feature", name, 14)) { - return NGHTTP2_TOKEN_ACCEPT_FEATURES; - } - if (lstreq("front-end-http", name, 14)) { - return NGHTTP2_TOKEN_FRONT_END_HTTPS; - } - if (lstreq("public-key-pin", name, 14)) { - return NGHTTP2_TOKEN_PUBLIC_KEY_PINS; - } - if (lstreq("x-frame-option", name, 14)) { - return NGHTTP2_TOKEN_X_FRAME_OPTIONS; - } - break; } break; case 16: @@ -571,11 +424,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_CONTENT_ENCODING; } break; - case 'h': - if (lstreq("x-requested-wit", name, 15)) { - return NGHTTP2_TOKEN_X_REQUESTED_WITH; - } - break; case 'n': if (lstreq("content-locatio", name, 15)) { return NGHTTP2_TOKEN_CONTENT_LOCATION; @@ -583,14 +431,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("proxy-connectio", name, 15)) { return NGHTTP2_TOKEN_PROXY_CONNECTION; } - if (lstreq("x-xss-protectio", name, 15)) { - return NGHTTP2_TOKEN_X_XSS_PROTECTION; - } - break; - case 't': - if (lstreq("x-forwarded-hos", name, 15)) { - return NGHTTP2_TOKEN_X_FORWARDED_HOST; - } break; } break; @@ -606,16 +446,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_TRANSFER_ENCODING; } break; - case 'o': - if (lstreq("x-forwarded-prot", name, 16)) { - return NGHTTP2_TOKEN_X_FORWARDED_PROTO; - } - break; - case 'y': - if (lstreq("sec-websocket-ke", name, 16)) { - return NGHTTP2_TOKEN_SEC_WEBSOCKET_KEY; - } - break; } break; case 18: @@ -625,11 +455,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_PROXY_AUTHENTICATE; } break; - case 'n': - if (lstreq("x-content-duratio", name, 17)) { - return NGHTTP2_TOKEN_X_CONTENT_DURATION; - } - break; } break; case 19: @@ -649,80 +474,12 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { break; } break; - case 20: - switch (name[19]) { - case 'n': - if (lstreq("sec-websocket-origi", name, 19)) { - return NGHTTP2_TOKEN_SEC_WEBSOCKET_ORIGIN; - } - break; - } - break; - case 21: - switch (name[20]) { - case 'l': - if (lstreq("x-dnsprefetch-contro", name, 20)) { - return NGHTTP2_TOKEN_X_DNSPREFETCH_CONTROL; - } - break; - case 'n': - if (lstreq("sec-websocket-versio", name, 20)) { - return NGHTTP2_TOKEN_SEC_WEBSOCKET_VERSION; - } - break; - } - break; - case 22: - switch (name[21]) { - case 'e': - if (lstreq("access-control-max-ag", name, 21)) { - return NGHTTP2_TOKEN_ACCESS_CONTROL_MAX_AGE; - } - break; - case 'l': - if (lstreq("sec-websocket-protoco", name, 21)) { - return NGHTTP2_TOKEN_SEC_WEBSOCKET_PROTOCOL; - } - break; - case 's': - if (lstreq("x-content-type-option", name, 21)) { - return NGHTTP2_TOKEN_X_CONTENT_TYPE_OPTIONS; - } - break; - } - break; - case 23: - switch (name[22]) { - case 'y': - if (lstreq("content-security-polic", name, 22)) { - return NGHTTP2_TOKEN_CONTENT_SECURITY_POLICY; - } - break; - } - break; - case 24: - switch (name[23]) { - case 's': - if (lstreq("sec-websocket-extension", name, 23)) { - return NGHTTP2_TOKEN_SEC_WEBSOCKET_EXTENSIONS; - } - break; - } - break; case 25: switch (name[24]) { - case 's': - if (lstreq("upgrade-insecure-request", name, 24)) { - return NGHTTP2_TOKEN_UPGRADE_INSECURE_REQUESTS; - } - break; case 'y': if (lstreq("strict-transport-securit", name, 24)) { return NGHTTP2_TOKEN_STRICT_TRANSPORT_SECURITY; } - if (lstreq("x-content-security-polic", name, 24)) { - return NGHTTP2_TOKEN_X_CONTENT_SECURITY_POLICY; - } break; } break; @@ -735,59 +492,6 @@ static int32_t lookup_token(const uint8_t *name, size_t namelen) { break; } break; - case 28: - switch (name[27]) { - case 's': - if (lstreq("access-control-allow-header", name, 27)) { - return NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_HEADERS; - } - if (lstreq("access-control-allow-method", name, 27)) { - return NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_METHODS; - } - break; - } - break; - case 29: - switch (name[28]) { - case 'd': - if (lstreq("access-control-request-metho", name, 28)) { - return NGHTTP2_TOKEN_ACCESS_CONTROL_REQUEST_METHOD; - } - break; - case 's': - if (lstreq("access-control-expose-header", name, 28)) { - return NGHTTP2_TOKEN_ACCESS_CONTROL_EXPOSE_HEADERS; - } - break; - } - break; - case 30: - switch (name[29]) { - case 's': - if (lstreq("access-control-request-header", name, 29)) { - return NGHTTP2_TOKEN_ACCESS_CONTROL_REQUEST_HEADERS; - } - break; - } - break; - case 32: - switch (name[31]) { - case 's': - if (lstreq("access-control-allow-credential", name, 31)) { - return NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_CREDENTIALS; - } - break; - } - break; - case 35: - switch (name[34]) { - case 'y': - if (lstreq("content-security-policy-report-onl", name, 34)) { - return NGHTTP2_TOKEN_CONTENT_SECURITY_POLICY_REPORT_ONLY; - } - break; - } - break; } return -1; } diff --git a/lib/nghttp2_hd.h b/lib/nghttp2_hd.h index cacfcf3a..b0e93852 100644 --- a/lib/nghttp2_hd.h +++ b/lib/nghttp2_hd.h @@ -106,67 +106,11 @@ typedef enum { NGHTTP2_TOKEN_VARY = 58, NGHTTP2_TOKEN_VIA = 59, NGHTTP2_TOKEN_WWW_AUTHENTICATE = 60, - NGHTTP2_TOKEN_ACCEPT_CH, - NGHTTP2_TOKEN_ACCEPT_DATETIME, - NGHTTP2_TOKEN_ACCEPT_FEATURES, - NGHTTP2_TOKEN_ACCEPT_PATCH, - NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_CREDENTIALS, - NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_HEADERS, - NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_METHODS, - NGHTTP2_TOKEN_ACCESS_CONTROL_EXPOSE_HEADERS, - NGHTTP2_TOKEN_ACCESS_CONTROL_MAX_AGE, - NGHTTP2_TOKEN_ACCESS_CONTROL_REQUEST_HEADERS, - NGHTTP2_TOKEN_ACCESS_CONTROL_REQUEST_METHOD, - NGHTTP2_TOKEN_ALT_SVC, - NGHTTP2_TOKEN_ALTERNATES, - NGHTTP2_TOKEN_CONNECTION, - NGHTTP2_TOKEN_CONTENT_MD5, - NGHTTP2_TOKEN_CONTENT_SECURITY_POLICY, - NGHTTP2_TOKEN_CONTENT_SECURITY_POLICY_REPORT_ONLY, - NGHTTP2_TOKEN_DNT, - NGHTTP2_TOKEN_FORWARDED, - NGHTTP2_TOKEN_FRONT_END_HTTPS, - NGHTTP2_TOKEN_KEEP_ALIVE, - NGHTTP2_TOKEN_LAST_EVENT_ID, - NGHTTP2_TOKEN_NEGOTIATE, - NGHTTP2_TOKEN_ORIGIN, - NGHTTP2_TOKEN_P3P, - NGHTTP2_TOKEN_PRAGMA, - NGHTTP2_TOKEN_PROXY_CONNECTION, - NGHTTP2_TOKEN_PUBLIC_KEY_PINS, - NGHTTP2_TOKEN_SEC_WEBSOCKET_EXTENSIONS, - NGHTTP2_TOKEN_SEC_WEBSOCKET_KEY, - NGHTTP2_TOKEN_SEC_WEBSOCKET_ORIGIN, - NGHTTP2_TOKEN_SEC_WEBSOCKET_PROTOCOL, - NGHTTP2_TOKEN_SEC_WEBSOCKET_VERSION, - NGHTTP2_TOKEN_SET_COOKIE2, - NGHTTP2_TOKEN_STATUS, - NGHTTP2_TOKEN_TCN, NGHTTP2_TOKEN_TE, - NGHTTP2_TOKEN_TRAILER, - NGHTTP2_TOKEN_TSV, + NGHTTP2_TOKEN_CONNECTION, + NGHTTP2_TOKEN_KEEP_ALIVE, + NGHTTP2_TOKEN_PROXY_CONNECTION, NGHTTP2_TOKEN_UPGRADE, - NGHTTP2_TOKEN_UPGRADE_INSECURE_REQUESTS, - NGHTTP2_TOKEN_VARIANT_VARY, - NGHTTP2_TOKEN_WARNING, - NGHTTP2_TOKEN_X_API_VERSION, - NGHTTP2_TOKEN_X_ATT_DEVICEID, - NGHTTP2_TOKEN_X_CACHE, - NGHTTP2_TOKEN_X_CACHE_LOOKUP, - NGHTTP2_TOKEN_X_CONTENT_DURATION, - NGHTTP2_TOKEN_X_CONTENT_SECURITY_POLICY, - NGHTTP2_TOKEN_X_CONTENT_TYPE_OPTIONS, - NGHTTP2_TOKEN_X_DNSPREFETCH_CONTROL, - NGHTTP2_TOKEN_X_FORWARDED_FOR, - NGHTTP2_TOKEN_X_FORWARDED_HOST, - NGHTTP2_TOKEN_X_FORWARDED_PROTO, - NGHTTP2_TOKEN_X_FRAME_OPTIONS, - NGHTTP2_TOKEN_X_POWERED_BY, - NGHTTP2_TOKEN_X_REQUESTED_WITH, - NGHTTP2_TOKEN_X_UA_COMPATIBLE, - NGHTTP2_TOKEN_X_WAP_PROFILE, - NGHTTP2_TOKEN_X_WEBKIT_CSP, - NGHTTP2_TOKEN_X_XSS_PROTECTION, } nghttp2_token; struct nghttp2_hd_entry; From 863a9441798b001001bade04cee6d5d311491d13 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 17:29:07 +0900 Subject: [PATCH 117/147] src: Add specialization for char to avoid reinterpret_cast in constexpr --- src/template.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/template.h b/src/template.h index b15efa02..1664714a 100644 --- a/src/template.h +++ b/src/template.h @@ -401,8 +401,9 @@ public: explicit StringRef(const ImmutableString &s) : base(s.c_str()), len(s.size()) {} explicit StringRef(const char *s) : base(s), len(strlen(s)) {} + constexpr StringRef(const char *s, size_t n) : base(s), len(n) {} template - constexpr StringRef(const CharT *s, size_t n) + StringRef(const CharT *s, size_t n) : base(reinterpret_cast(s)), len(n) {} template StringRef(InputIt first, InputIt last) @@ -413,7 +414,7 @@ public: len(std::distance(first, last)) {} template constexpr static StringRef from_lit(const CharT(&s)[N]) { - return StringRef(s, N - 1); + return StringRef{s, N - 1}; } static StringRef from_maybe_nullptr(const char *s) { if (s == nullptr) { From c897d5b2944a138b0649f188c224dfba72159d5d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 17:31:48 +0900 Subject: [PATCH 118/147] src: Use StringRef in parse_http_date --- src/HttpServer.cc | 2 +- src/util.cc | 2 +- src/util.h | 2 +- src/util_test.cc | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/HttpServer.cc b/src/HttpServer.cc index 03140c3b..5c65295d 100644 --- a/src/HttpServer.cc +++ b/src/HttpServer.cc @@ -1235,7 +1235,7 @@ void prepare_response(Stream *stream, Http2Handler *hd, bool last_mod_found = false; if (ims) { last_mod_found = true; - last_mod = util::parse_http_date(ims->value); + last_mod = util::parse_http_date(StringRef{ims->value}); } auto query_pos = reqpath.find("?"); std::string url; diff --git a/src/util.cc b/src/util.cc index 90304315..35fc8275 100644 --- a/src/util.cc +++ b/src/util.cc @@ -339,7 +339,7 @@ std::string iso8601_date(int64_t ms) { return res; } -time_t parse_http_date(const std::string &s) { +time_t parse_http_date(const StringRef &s) { tm tm{}; char *r = strptime(s.c_str(), "%a, %d %b %Y %H:%M:%S GMT", &tm); if (r == 0) { diff --git a/src/util.h b/src/util.h index 90f03f1f..4f926e2a 100644 --- a/src/util.h +++ b/src/util.h @@ -149,7 +149,7 @@ std::string common_log_date(time_t t); // 2014-11-15T12:58:24.741Z) std::string iso8601_date(int64_t ms); -time_t parse_http_date(const std::string &s); +time_t parse_http_date(const StringRef &s); char upcase(char c); diff --git a/src/util_test.cc b/src/util_test.cc index d99c3a96..0a17e255 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -392,8 +392,8 @@ void test_util_ends_with(void) { } void test_util_parse_http_date(void) { - CU_ASSERT(1001939696 == - util::parse_http_date("Mon, 1 Oct 2001 12:34:56 GMT")); + CU_ASSERT(1001939696 == util::parse_http_date(StringRef::from_lit( + "Mon, 1 Oct 2001 12:34:56 GMT"))); } void test_util_localtime_date(void) { From d64051fedcdafc2ca27e20c76c3da815b76bd071 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 18:00:50 +0900 Subject: [PATCH 119/147] src: Return StringRef from http2::stringify_status --- src/http2.cc | 102 ++++++++++++++--------------- src/http2.h | 2 +- src/shrpx-unittest.cc | 2 + src/shrpx_http2_upstream.cc | 54 +++++++-------- src/shrpx_mruby_module_response.cc | 9 +-- src/util.h | 10 +++ src/util_test.cc | 13 +++- src/util_test.h | 1 + 8 files changed, 105 insertions(+), 88 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index 8385cafe..445ebe44 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -134,107 +134,107 @@ std::string get_status_string(unsigned int status_code) { } } -const char *stringify_status(unsigned int status_code) { +StringRef stringify_status(unsigned int status_code) { switch (status_code) { case 100: - return "100"; + return StringRef::from_lit("100"); case 101: - return "101"; + return StringRef::from_lit("101"); case 200: - return "200"; + return StringRef::from_lit("200"); case 201: - return "201"; + return StringRef::from_lit("201"); case 202: - return "202"; + return StringRef::from_lit("202"); case 203: - return "203"; + return StringRef::from_lit("203"); case 204: - return "204"; + return StringRef::from_lit("204"); case 205: - return "205"; + return StringRef::from_lit("205"); case 206: - return "206"; + return StringRef::from_lit("206"); case 300: - return "300"; + return StringRef::from_lit("300"); case 301: - return "301"; + return StringRef::from_lit("301"); case 302: - return "302"; + return StringRef::from_lit("302"); case 303: - return "303"; + return StringRef::from_lit("303"); case 304: - return "304"; + return StringRef::from_lit("304"); case 305: - return "305"; - // case 306: return "306"; + return StringRef::from_lit("305"); + // case 306: return StringRef::from_lit("306"); case 307: - return "307"; + return StringRef::from_lit("307"); case 308: - return "308"; + return StringRef::from_lit("308"); case 400: - return "400"; + return StringRef::from_lit("400"); case 401: - return "401"; + return StringRef::from_lit("401"); case 402: - return "402"; + return StringRef::from_lit("402"); case 403: - return "403"; + return StringRef::from_lit("403"); case 404: - return "404"; + return StringRef::from_lit("404"); case 405: - return "405"; + return StringRef::from_lit("405"); case 406: - return "406"; + return StringRef::from_lit("406"); case 407: - return "407"; + return StringRef::from_lit("407"); case 408: - return "408"; + return StringRef::from_lit("408"); case 409: - return "409"; + return StringRef::from_lit("409"); case 410: - return "410"; + return StringRef::from_lit("410"); case 411: - return "411"; + return StringRef::from_lit("411"); case 412: - return "412"; + return StringRef::from_lit("412"); case 413: - return "413"; + return StringRef::from_lit("413"); case 414: - return "414"; + return StringRef::from_lit("414"); case 415: - return "415"; + return StringRef::from_lit("415"); case 416: - return "416"; + return StringRef::from_lit("416"); case 417: - return "417"; + return StringRef::from_lit("417"); case 421: - return "421"; + return StringRef::from_lit("421"); case 426: - return "426"; + return StringRef::from_lit("426"); case 428: - return "428"; + return StringRef::from_lit("428"); case 429: - return "429"; + return StringRef::from_lit("429"); case 431: - return "431"; + return StringRef::from_lit("431"); case 451: - return "451"; + return StringRef::from_lit("451"); case 500: - return "500"; + return StringRef::from_lit("500"); case 501: - return "501"; + return StringRef::from_lit("501"); case 502: - return "502"; + return StringRef::from_lit("502"); case 503: - return "503"; + return StringRef::from_lit("503"); case 504: - return "504"; + return StringRef::from_lit("504"); case 505: - return "505"; + return StringRef::from_lit("505"); case 511: - return "511"; + return StringRef::from_lit("511"); default: - return nullptr; + return StringRef{}; } } diff --git a/src/http2.h b/src/http2.h index 48d4916d..0ae9d3b3 100644 --- a/src/http2.h +++ b/src/http2.h @@ -98,7 +98,7 @@ std::string get_status_string(unsigned int status_code); // Returns string version of |status_code|. This function can handle // only predefined status code. Otherwise, returns nullptr. -const char *stringify_status(unsigned int status_code); +StringRef stringify_status(unsigned int status_code); void capitalize(DefaultMemchunks *buf, const StringRef &s); diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index 0f2e2fd2..725838f8 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -143,6 +143,8 @@ int main(int argc, char *argv[]) { !CU_add_test(pSuite, "util_ipv6_numeric_addr", shrpx::test_util_ipv6_numeric_addr) || !CU_add_test(pSuite, "util_utos", shrpx::test_util_utos) || + !CU_add_test(pSuite, "util_make_string_ref_uint", + shrpx::test_util_make_string_ref_uint) || !CU_add_test(pSuite, "util_utos_unit", shrpx::test_util_utos_unit) || !CU_add_test(pSuite, "util_utos_funit", shrpx::test_util_utos_funit) || !CU_add_test(pSuite, "util_parse_uint_with_unit", diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 79fac5de..e54b8e90 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1229,20 +1229,20 @@ int Http2Upstream::send_reply(Downstream *downstream, const uint8_t *body, const auto &resp = downstream->response(); auto &httpconf = get_config()->http; + auto &balloc = downstream->get_block_allocator(); + const auto &headers = resp.fs.headers(); auto nva = std::vector(); // 2 for :status and server nva.reserve(2 + headers.size() + httpconf.add_response_headers.size()); - std::string status_code_str; - auto response_status_const = http2::stringify_status(resp.http_status); - if (response_status_const) { - nva.push_back(http2::make_nv_lc_nocopy(":status", response_status_const)); - } else { - status_code_str = util::utos(resp.http_status); - nva.push_back(http2::make_nv_ls(":status", status_code_str)); + auto response_status = http2::stringify_status(resp.http_status); + if (response_status.empty()) { + response_status = util::make_string_ref_uint(balloc, resp.http_status); } + nva.push_back(http2::make_nv_ls_nocopy(":status", response_status)); + for (auto &kv : headers) { if (kv.name.empty() || kv.name[0] == ':') { continue; @@ -1290,6 +1290,8 @@ int Http2Upstream::error_reply(Downstream *downstream, int rv; auto &resp = downstream->response(); + auto &balloc = downstream->get_block_allocator(); + auto html = http::create_error_html(status_code); resp.http_status = status_code; auto body = downstream->get_response_buf(); @@ -1303,20 +1305,20 @@ int Http2Upstream::error_reply(Downstream *downstream, auto lgconf = log_config(); lgconf->update_tstamp(std::chrono::system_clock::now()); - auto response_status_const = http2::stringify_status(status_code); - auto content_length = util::utos(html.size()); + auto response_status = http2::stringify_status(status_code); + if (response_status.empty()) { + response_status = util::make_string_ref_uint(balloc, status_code); + } - std::string status_code_str; + auto content_length = util::make_string_ref_uint(balloc, html.size()); + auto date = make_string_ref(balloc, StringRef{lgconf->time_http_str}); - auto nva = make_array( - response_status_const - ? http2::make_nv_lc_nocopy(":status", response_status_const) - : http2::make_nv_ls(":status", - (status_code_str = util::utos(status_code))), - http2::make_nv_ll("content-type", "text/html; charset=UTF-8"), - http2::make_nv_ls_nocopy("server", get_config()->http.server_name), - http2::make_nv_ls("content-length", content_length), - http2::make_nv_ls("date", lgconf->time_http_str)); + auto nva = std::array{ + {http2::make_nv_ls_nocopy(":status", response_status), + http2::make_nv_ll("content-type", "text/html; charset=UTF-8"), + http2::make_nv_ls_nocopy("server", get_config()->http.server_name), + http2::make_nv_ls_nocopy("content-length", content_length), + http2::make_nv_ls_nocopy("date", date)}}; rv = nghttp2_submit_response(session_, downstream->get_stream_id(), nva.data(), nva.size(), &data_prd); @@ -1357,6 +1359,8 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { const auto &req = downstream->request(); auto &resp = downstream->response(); + auto &balloc = downstream->get_block_allocator(); + if (LOG_ENABLED(INFO)) { if (downstream->get_non_final_response()) { DLOG(INFO, downstream) << "HTTP non-final response header"; @@ -1396,16 +1400,14 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { nva.reserve(resp.fs.headers().size() + 4 + httpconf.add_response_headers.size()); std::string via_value; - std::string response_status; - auto response_status_const = http2::stringify_status(resp.http_status); - if (response_status_const) { - nva.push_back(http2::make_nv_lc_nocopy(":status", response_status_const)); - } else { - response_status = util::utos(resp.http_status); - nva.push_back(http2::make_nv_ls(":status", response_status)); + auto response_status = http2::stringify_status(resp.http_status); + if (response_status.empty()) { + response_status = util::make_string_ref_uint(balloc, resp.http_status); } + nva.push_back(http2::make_nv_ls_nocopy(":status", response_status)); + if (downstream->get_non_final_response()) { http2::copy_headers_to_nva(nva, resp.fs.headers()); diff --git a/src/shrpx_mruby_module_response.cc b/src/shrpx_mruby_module_response.cc index 65fec390..8a89a058 100644 --- a/src/shrpx_mruby_module_response.cc +++ b/src/shrpx_mruby_module_response.cc @@ -215,14 +215,7 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { bodylen = vallen; } - StringRef content_length; - { - auto iov = make_byte_ref(balloc, str_size("18446744073709551615") + 1); - auto p = iov.base; - p = util::utos(p, bodylen); - *p = '\0'; - content_length = StringRef{iov.base, p}; - } + auto content_length = util::make_string_ref_uint(balloc, bodylen); auto cl = resp.fs.header(http2::HD_CONTENT_LENGTH); if (cl) { diff --git a/src/util.h b/src/util.h index 4f926e2a..2d3208fc 100644 --- a/src/util.h +++ b/src/util.h @@ -51,6 +51,7 @@ #include "template.h" #include "network.h" +#include "allocator.h" namespace nghttp2 { @@ -405,6 +406,15 @@ template OutputIt utos(OutputIt dst, T n) { return res; } +template +StringRef make_string_ref_uint(BlockAllocator &balloc, T n) { + auto iov = make_byte_ref(balloc, str_size("18446744073709551615") + 1); + auto p = iov.base; + p = util::utos(p, n); + *p = '\0'; + return StringRef{iov.base, p}; +} + template std::string utos_unit(T n) { char u = 0; if (n >= (1 << 30)) { diff --git a/src/util_test.cc b/src/util_test.cc index 0a17e255..8a8ee181 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -239,8 +239,17 @@ void test_util_utos(void) { CU_ASSERT(("0" == StringRef{buf, util::utos(buf, 0)})); CU_ASSERT(("123" == StringRef{buf, util::utos(buf, 123)})); - CU_ASSERT(("9223372036854775808" == - StringRef{buf, util::utos(buf, 9223372036854775808ULL)})); + CU_ASSERT(("18446744073709551615" == + StringRef{buf, util::utos(buf, 18446744073709551615ULL)})); +} + +void test_util_make_string_ref_uint(void) { + BlockAllocator balloc(1024, 1024); + + CU_ASSERT("0" == util::make_string_ref_uint(balloc, 0)); + CU_ASSERT("123" == util::make_string_ref_uint(balloc, 123)); + CU_ASSERT("18446744073709551615" == + util::make_string_ref_uint(balloc, 18446744073709551615ULL)); } void test_util_utos_unit(void) { diff --git a/src/util_test.h b/src/util_test.h index f51692aa..b44d5b17 100644 --- a/src/util_test.h +++ b/src/util_test.h @@ -45,6 +45,7 @@ void test_util_http_date(void); void test_util_select_h2(void); void test_util_ipv6_numeric_addr(void); void test_util_utos(void); +void test_util_make_string_ref_uint(void); void test_util_utos_unit(void); void test_util_utos_funit(void); void test_util_parse_uint_with_unit(void); From 67569486d1be3d2b54f393416de89b7a7e2dad97 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 18:36:05 +0900 Subject: [PATCH 120/147] src: Rewrite http:create_via_header_value --- src/shrpx-unittest.cc | 2 ++ src/shrpx_http.cc | 11 ----------- src/shrpx_http.h | 12 +++++++++++- src/shrpx_http2_downstream_connection.cc | 21 ++++++++++++++++----- src/shrpx_http2_upstream.cc | 21 +++++++++++++++------ src/shrpx_http_downstream_connection.cc | 5 ++++- src/shrpx_http_test.cc | 14 ++++++++++++++ src/shrpx_http_test.h | 1 + src/shrpx_https_upstream.cc | 6 ++++-- src/shrpx_spdy_upstream.cc | 6 ++++-- 10 files changed, 71 insertions(+), 28 deletions(-) diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index 725838f8..6a015778 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -123,6 +123,8 @@ int main(int argc, char *argv[]) { shrpx::test_shrpx_worker_match_downstream_addr_group) || !CU_add_test(pSuite, "http_create_forwarded", shrpx::test_shrpx_http_create_forwarded) || + !CU_add_test(pSuite, "http_create_via_header_value", + shrpx::test_shrpx_http_create_via_header_value) || !CU_add_test(pSuite, "util_streq", shrpx::test_util_streq) || !CU_add_test(pSuite, "util_strieq", shrpx::test_util_strieq) || !CU_add_test(pSuite, "util_inp_strlower", diff --git a/src/shrpx_http.cc b/src/shrpx_http.cc index 7bca56dd..071f2841 100644 --- a/src/shrpx_http.cc +++ b/src/shrpx_http.cc @@ -50,17 +50,6 @@ std::string create_error_html(unsigned int status_code) { return res; } -std::string create_via_header_value(int major, int minor) { - std::string hdrs; - hdrs += static_cast(major + '0'); - if (major < 2) { - hdrs += '.'; - hdrs += static_cast(minor + '0'); - } - hdrs += " nghttpx"; - return hdrs; -} - std::string create_forwarded(int params, const StringRef &node_by, const StringRef &node_for, const StringRef &host, const StringRef &proto) { diff --git a/src/shrpx_http.h b/src/shrpx_http.h index 56705ecd..35275baf 100644 --- a/src/shrpx_http.h +++ b/src/shrpx_http.h @@ -31,13 +31,23 @@ #include +#include "util.h" + namespace shrpx { namespace http { std::string create_error_html(unsigned int status_code); -std::string create_via_header_value(int major, int minor); +template +OutputIt create_via_header_value(OutputIt dst, int major, int minor) { + *dst++ = static_cast(major + '0'); + if (major < 2) { + *dst++ = '.'; + *dst++ = static_cast(minor + '0'); + } + return util::copy_lit(dst, " nghttpx"); +} // Returns generated RFC 7239 Forwarded header field value. The // |params| is bitwise-OR of zero or more of shrpx_forwarded_param diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index 5e2d5129..2e5fb177 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -266,6 +266,8 @@ int Http2DownstreamConnection::push_request_headers() { const auto &req = downstream_->request(); + auto &balloc = downstream_->get_block_allocator(); + auto &httpconf = get_config()->http; auto &http2conf = get_config()->http2; @@ -403,19 +405,28 @@ int Http2DownstreamConnection::push_request_headers() { nva.push_back(http2::make_nv_ls_nocopy("x-forwarded-proto", req.scheme)); } - std::string via_value; auto via = req.fs.header(http2::HD_VIA); if (httpconf.no_via) { if (via) { nva.push_back(http2::make_nv_ls_nocopy("via", (*via).value)); } } else { + size_t vialen = 16; if (via) { - via_value = (*via).value.str(); - via_value += ", "; + vialen += via->value.size() + 2; } - via_value += http::create_via_header_value(req.http_major, req.http_minor); - nva.push_back(http2::make_nv_ls("via", via_value)); + + auto iov = make_byte_ref(balloc, vialen + 1); + auto p = iov.base; + + if (via) { + p = std::copy(std::begin(via->value), std::end(via->value), p); + p = util::copy_lit(p, ", "); + } + p = http::create_via_header_value(p, req.http_major, req.http_minor); + *p = '\0'; + + nva.push_back(http2::make_nv_ls_nocopy("via", StringRef{iov.base, p})); } auto te = req.fs.header(http2::HD_TE); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index e54b8e90..32905876 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1399,7 +1399,6 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { // field. nva.reserve(resp.fs.headers().size() + 4 + httpconf.add_response_headers.size()); - std::string via_value; auto response_status = http2::stringify_status(resp.http_status); if (response_status.empty()) { @@ -1453,13 +1452,23 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { nva.push_back(http2::make_nv_ls_nocopy("via", (*via).value)); } } else { + // we don't create more than 16 bytes in + // http::create_via_header_value. + size_t len = 16; if (via) { - via_value = (*via).value.str(); - via_value += ", "; + len += via->value.size() + 2; } - via_value += - http::create_via_header_value(resp.http_major, resp.http_minor); - nva.push_back(http2::make_nv_ls("via", via_value)); + + auto iov = make_byte_ref(balloc, len + 1); + auto p = iov.base; + if (via) { + p = std::copy(std::begin(via->value), std::end(via->value), p); + p = util::copy_lit(p, ", "); + } + p = http::create_via_header_value(p, resp.http_major, resp.http_minor); + *p = '\0'; + + nva.push_back(http2::make_nv_ls_nocopy("via", StringRef{iov.base, p})); } for (auto &p : httpconf.add_response_headers) { diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 5f4f9468..490fefe8 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -424,7 +424,10 @@ int HttpDownstreamConnection::push_request_headers() { buf->append((*via).value); buf->append(", "); } - buf->append(http::create_via_header_value(req.http_major, req.http_minor)); + std::array viabuf; + auto end = http::create_via_header_value(viabuf.data(), req.http_major, + req.http_minor); + buf->append(viabuf.data(), end - viabuf.data()); buf->append("\r\n"); } diff --git a/src/shrpx_http_test.cc b/src/shrpx_http_test.cc index 87fbb9c5..bee8d966 100644 --- a/src/shrpx_http_test.cc +++ b/src/shrpx_http_test.cc @@ -72,4 +72,18 @@ void test_shrpx_http_create_forwarded(void) { StringRef::from_lit(""), StringRef::from_lit(""))); } +void test_shrpx_http_create_via_header_value(void) { + std::array buf; + + auto end = http::create_via_header_value(std::begin(buf), 1, 1); + + CU_ASSERT(("1.1 nghttpx" == StringRef{std::begin(buf), end})); + + std::fill(std::begin(buf), std::end(buf), '\0'); + + end = http::create_via_header_value(std::begin(buf), 2, 0); + + CU_ASSERT(("2 nghttpx" == StringRef{std::begin(buf), end})); +} + } // namespace shrpx diff --git a/src/shrpx_http_test.h b/src/shrpx_http_test.h index d8e28a9f..7692c95a 100644 --- a/src/shrpx_http_test.h +++ b/src/shrpx_http_test.h @@ -32,6 +32,7 @@ namespace shrpx { void test_shrpx_http_create_forwarded(void); +void test_shrpx_http_create_via_header_value(void); } // namespace shrpx diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index dd028190..ac99c457 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -1074,8 +1074,10 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) { buf->append((*via).value); buf->append(", "); } - buf->append( - http::create_via_header_value(resp.http_major, resp.http_minor)); + std::array viabuf; + auto end = http::create_via_header_value(viabuf.data(), resp.http_major, + resp.http_minor); + buf->append(viabuf.data(), end - std::begin(viabuf)); buf->append("\r\n"); } diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 4b48c93f..21380afa 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -1075,8 +1075,10 @@ int SpdyUpstream::on_downstream_header_complete(Downstream *downstream) { via_value = via->value.str(); via_value += ", "; } - via_value += - http::create_via_header_value(resp.http_major, resp.http_minor); + std::array viabuf; + auto end = http::create_via_header_value(std::begin(viabuf), + resp.http_major, resp.http_minor); + via_value.append(std::begin(viabuf), end); nv[hdidx++] = "via"; nv[hdidx++] = via_value.c_str(); } From c1571a3209329798ca91eee19d2a6c20073ff60b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 18:46:58 +0900 Subject: [PATCH 121/147] src: Rewrite xff handling --- src/allocator.h | 18 ++++++++++++++++-- src/shrpx_http2_downstream_connection.cc | 15 ++++++++------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/allocator.h b/src/allocator.h index 00acdbcf..cd92d3fd 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -104,12 +104,26 @@ StringRef make_string_ref(BlockAllocator &alloc, const StringRef &src) { template StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, const StringRef &b) { - auto dst = static_cast(alloc.alloc(a.size() + b.size() + 1)); + auto len = a.size() + b.size(); + auto dst = static_cast(alloc.alloc(len + 1)); auto p = dst; p = std::copy(std::begin(a), std::end(a), p); p = std::copy(std::begin(b), std::end(b), p); *p = '\0'; - return StringRef{dst, a.size() + b.size()}; + return StringRef{dst, len}; +} + +template +StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, + const StringRef &b, const StringRef &c) { + auto len = a.size() + b.size() + c.size(); + auto dst = static_cast(alloc.alloc(len + 1)); + auto p = dst; + p = std::copy(std::begin(a), std::end(a), p); + p = std::copy(std::begin(b), std::end(b), p); + p = std::copy(std::begin(c), std::end(c), p); + *p = '\0'; + return StringRef{dst, len}; } struct ByteRef { diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index 2e5fb177..c9df0369 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -387,17 +387,18 @@ int Http2DownstreamConnection::push_request_headers() { auto xff = xffconf.strip_incoming ? nullptr : req.fs.header(http2::HD_X_FORWARDED_FOR); - std::string xff_value; - if (xffconf.add) { + StringRef xff_value; + auto addr = StringRef{upstream->get_client_handler()->get_ipaddr()}; if (xff) { - xff_value = (*xff).value.str(); - xff_value += ", "; + xff_value = concat_string_ref(balloc, xff->value, + StringRef::from_lit(", "), addr); + } else { + xff_value = addr; } - xff_value += upstream->get_client_handler()->get_ipaddr(); - nva.push_back(http2::make_nv_ls("x-forwarded-for", xff_value)); + nva.push_back(http2::make_nv_ls_nocopy("x-forwarded-for", xff_value)); } else if (xff) { - nva.push_back(http2::make_nv_ls_nocopy("x-forwarded-for", (*xff).value)); + nva.push_back(http2::make_nv_ls_nocopy("x-forwarded-for", xff->value)); } if (!get_config()->http2_proxy && req.method != HTTP_CONNECT) { From 3455cb35e4b29c6afabdde119a0613865b980bb3 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 19:07:48 +0900 Subject: [PATCH 122/147] nghttpx: Rewrite create_forwarded to use BlockAllocator --- src/shrpx_http.cc | 69 +++++++++++++++--------- src/shrpx_http.h | 7 +-- src/shrpx_http2_downstream_connection.cc | 23 ++++---- src/shrpx_http_downstream_connection.cc | 9 ++-- src/shrpx_http_test.cc | 36 +++++++------ 5 files changed, 82 insertions(+), 62 deletions(-) diff --git a/src/shrpx_http.cc b/src/shrpx_http.cc index 071f2841..c6a70e06 100644 --- a/src/shrpx_http.cc +++ b/src/shrpx_http.cc @@ -50,58 +50,75 @@ std::string create_error_html(unsigned int status_code) { return res; } -std::string create_forwarded(int params, const StringRef &node_by, - const StringRef &node_for, const StringRef &host, - const StringRef &proto) { - std::string res; +StringRef create_forwarded(BlockAllocator &balloc, int params, + const StringRef &node_by, const StringRef &node_for, + const StringRef &host, const StringRef &proto) { + size_t len = 0; + if ((params & FORWARDED_BY) && !node_by.empty()) { + len += str_size("by=\"") + node_by.size() + str_size("\";"); + } + if ((params & FORWARDED_FOR) && !node_for.empty()) { + len += str_size("for=\"") + node_for.size() + str_size("\";"); + } + if ((params & FORWARDED_HOST) && !host.empty()) { + len += str_size("host=\"") + host.size() + str_size("\";"); + } + if ((params & FORWARDED_PROTO) && !proto.empty()) { + len += str_size("proto=") + proto.size() + str_size(";"); + } + + auto iov = make_byte_ref(balloc, len + 1); + auto p = iov.base; + if ((params & FORWARDED_BY) && !node_by.empty()) { // This must be quoted-string unless it is obfuscated version // (which starts with "_") or some special value (e.g., // "localhost" for UNIX domain socket), since ':' is not allowed // in token. ':' is used to separate host and port. if (node_by[0] == '_' || node_by[0] == 'l') { - res += "by="; - res += node_by; - res += ";"; + p = util::copy_lit(p, "by="); + p = std::copy(std::begin(node_by), std::end(node_by), p); + p = util::copy_lit(p, ";"); } else { - res += "by=\""; - res += node_by; - res += "\";"; + p = util::copy_lit(p, "by=\""); + p = std::copy(std::begin(node_by), std::end(node_by), p); + p = util::copy_lit(p, "\";"); } } if ((params & FORWARDED_FOR) && !node_for.empty()) { // We only quote IPv6 literal address only, which starts with '['. if (node_for[0] == '[') { - res += "for=\""; - res += node_for; - res += "\";"; + p = util::copy_lit(p, "for=\""); + p = std::copy(std::begin(node_for), std::end(node_for), p); + p = util::copy_lit(p, "\";"); } else { - res += "for="; - res += node_for; - res += ";"; + p = util::copy_lit(p, "for="); + p = std::copy(std::begin(node_for), std::end(node_for), p); + p = util::copy_lit(p, ";"); } } if ((params & FORWARDED_HOST) && !host.empty()) { // Just be quoted to skip checking characters. - res += "host=\""; - res += host; - res += "\";"; + p = util::copy_lit(p, "host=\""); + p = std::copy(std::begin(host), std::end(host), p); + p = util::copy_lit(p, "\";"); } if ((params & FORWARDED_PROTO) && !proto.empty()) { // Scheme production rule only allow characters which are all in // token. - res += "proto="; - res += proto; - res += ";"; + p = util::copy_lit(p, "proto="); + p = std::copy(std::begin(proto), std::end(proto), p); + *p++ = ';'; } - if (res.empty()) { - return res; + if (iov.base == p) { + return StringRef{}; } - res.erase(res.size() - 1); + --p; + *p = '\0'; - return res; + return StringRef{iov.base, p}; } std::string colorizeHeaders(const char *hdrs) { diff --git a/src/shrpx_http.h b/src/shrpx_http.h index 35275baf..856f192a 100644 --- a/src/shrpx_http.h +++ b/src/shrpx_http.h @@ -32,6 +32,7 @@ #include #include "util.h" +#include "allocator.h" namespace shrpx { @@ -52,9 +53,9 @@ OutputIt create_via_header_value(OutputIt dst, int major, int minor) { // Returns generated RFC 7239 Forwarded header field value. The // |params| is bitwise-OR of zero or more of shrpx_forwarded_param // defined in shrpx_config.h. -std::string create_forwarded(int params, const StringRef &node_by, - const StringRef &node_for, const StringRef &host, - const StringRef &proto); +StringRef create_forwarded(BlockAllocator &balloc, int params, + const StringRef &node_by, const StringRef &node_for, + const StringRef &host, const StringRef &proto); // Adds ANSI color codes to HTTP headers |hdrs|. std::string colorizeHeaders(const char *hdrs); diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index c9df0369..73a43a8d 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -347,8 +347,6 @@ int Http2DownstreamConnection::push_request_headers() { auto upstream = downstream_->get_upstream(); auto handler = upstream->get_client_handler(); - std::string forwarded_value; - auto &fwdconf = httpconf.forwarded; auto fwd = @@ -361,25 +359,24 @@ int Http2DownstreamConnection::push_request_headers() { params &= ~FORWARDED_PROTO; } - auto value = http::create_forwarded(params, handler->get_forwarded_by(), - handler->get_forwarded_for(), - req.authority, req.scheme); + auto value = http::create_forwarded( + balloc, params, handler->get_forwarded_by(), + handler->get_forwarded_for(), req.authority, req.scheme); + if (fwd || !value.empty()) { if (fwd) { - forwarded_value = fwd->value.str(); - - if (!value.empty()) { - forwarded_value += ", "; + if (value.empty()) { + value = fwd->value; + } else { + value = concat_string_ref(balloc, fwd->value, + StringRef::from_lit(", "), value); } } - forwarded_value += value; - - nva.push_back(http2::make_nv_ls("forwarded", forwarded_value)); + nva.push_back(http2::make_nv_ls_nocopy("forwarded", value)); } } else if (fwd) { nva.push_back(http2::make_nv_ls_nocopy("forwarded", fwd->value)); - forwarded_value = fwd->value.str(); } auto &xffconf = httpconf.xff; diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 490fefe8..893c9efc 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -272,6 +272,8 @@ int HttpDownstreamConnection::push_request_headers() { const auto &downstream_hostport = addr_->hostport; const auto &req = downstream_->request(); + auto &balloc = downstream_->get_block_allocator(); + auto connect_method = req.method == HTTP_CONNECT; auto &httpconf = get_config()->http; @@ -366,9 +368,10 @@ int HttpDownstreamConnection::push_request_headers() { params &= ~FORWARDED_PROTO; } - auto value = http::create_forwarded(params, handler->get_forwarded_by(), - handler->get_forwarded_for(), - req.authority, req.scheme); + auto value = http::create_forwarded( + balloc, params, handler->get_forwarded_by(), + handler->get_forwarded_for(), req.authority, req.scheme); + if (fwd || !value.empty()) { buf->append("Forwarded: "); if (fwd) { diff --git a/src/shrpx_http_test.cc b/src/shrpx_http_test.cc index bee8d966..455397a6 100644 --- a/src/shrpx_http_test.cc +++ b/src/shrpx_http_test.cc @@ -38,38 +38,40 @@ namespace shrpx { void test_shrpx_http_create_forwarded(void) { + BlockAllocator balloc(1024, 1024); + CU_ASSERT("by=\"example.com:3000\";for=\"[::1]\";host=\"www.example.com\";" "proto=https" == - http::create_forwarded(FORWARDED_BY | FORWARDED_FOR | - FORWARDED_HOST | FORWARDED_PROTO, + http::create_forwarded(balloc, FORWARDED_BY | FORWARDED_FOR | + FORWARDED_HOST | FORWARDED_PROTO, StringRef::from_lit("example.com:3000"), StringRef::from_lit("[::1]"), StringRef::from_lit("www.example.com"), StringRef::from_lit("https"))); CU_ASSERT("for=192.168.0.1" == - http::create_forwarded(FORWARDED_FOR, StringRef::from_lit("alpha"), - StringRef::from_lit("192.168.0.1"), - StringRef::from_lit("bravo"), - StringRef::from_lit("charlie"))); + http::create_forwarded( + balloc, FORWARDED_FOR, StringRef::from_lit("alpha"), + StringRef::from_lit("192.168.0.1"), + StringRef::from_lit("bravo"), StringRef::from_lit("charlie"))); CU_ASSERT("by=_hidden;for=\"[::1]\"" == http::create_forwarded( - FORWARDED_BY | FORWARDED_FOR, StringRef::from_lit("_hidden"), - StringRef::from_lit("[::1]"), StringRef::from_lit(""), - StringRef::from_lit(""))); + balloc, FORWARDED_BY | FORWARDED_FOR, + StringRef::from_lit("_hidden"), StringRef::from_lit("[::1]"), + StringRef::from_lit(""), StringRef::from_lit(""))); CU_ASSERT("by=\"[::1]\";for=_hidden" == http::create_forwarded( - FORWARDED_BY | FORWARDED_FOR, StringRef::from_lit("[::1]"), - StringRef::from_lit("_hidden"), StringRef::from_lit(""), - StringRef::from_lit(""))); - - CU_ASSERT("" == - http::create_forwarded( - FORWARDED_BY | FORWARDED_FOR | FORWARDED_HOST | FORWARDED_PROTO, - StringRef::from_lit(""), StringRef::from_lit(""), + balloc, FORWARDED_BY | FORWARDED_FOR, + StringRef::from_lit("[::1]"), StringRef::from_lit("_hidden"), StringRef::from_lit(""), StringRef::from_lit(""))); + + CU_ASSERT("" == http::create_forwarded( + balloc, FORWARDED_BY | FORWARDED_FOR | FORWARDED_HOST | + FORWARDED_PROTO, + StringRef::from_lit(""), StringRef::from_lit(""), + StringRef::from_lit(""), StringRef::from_lit(""))); } void test_shrpx_http_create_via_header_value(void) { From 755b14de5d3e6d83f2cbeff4a67f93252545cfac Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 19:24:02 +0900 Subject: [PATCH 123/147] src: Unify path_join implementation --- src/http2.cc | 127 ++------------------------------------------------- 1 file changed, 3 insertions(+), 124 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index 445ebe44..092c4c8e 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -1198,132 +1198,11 @@ std::vector parse_link_header(const char *src, size_t len) { return res; } -namespace { -void eat_file(std::string &path) { - if (path.empty()) { - path = "/"; - return; - } - auto p = path.size() - 1; - if (path[p] == '/') { - return; - } - p = path.rfind('/', p); - if (p == std::string::npos) { - // this should not happend in normal case, where we expect path - // starts with '/' - path = "/"; - return; - } - path.erase(std::begin(path) + p + 1, std::end(path)); -} -} // namespace - -namespace { -void eat_dir(std::string &path) { - if (path.empty()) { - path = "/"; - return; - } - auto p = path.size() - 1; - if (path[p] != '/') { - p = path.rfind('/', p); - if (p == std::string::npos) { - // this should not happend in normal case, where we expect path - // starts with '/' - path = "/"; - return; - } - } - if (path[p] == '/') { - if (p == 0) { - return; - } - --p; - } - p = path.rfind('/', p); - if (p == std::string::npos) { - // this should not happend in normal case, where we expect path - // starts with '/' - path = "/"; - return; - } - path.erase(std::begin(path) + p + 1, std::end(path)); -} -} // namespace - std::string path_join(const StringRef &base_path, const StringRef &base_query, const StringRef &rel_path, const StringRef &rel_query) { - std::string res; - if (rel_path.empty()) { - if (base_path.empty()) { - res = "/"; - } else { - res.assign(std::begin(base_path), std::end(base_path)); - } - if (rel_query.empty()) { - if (!base_query.empty()) { - res += '?'; - res.append(std::begin(base_query), std::end(base_query)); - } - return res; - } - res += '?'; - res.append(std::begin(rel_query), std::end(rel_query)); - return res; - } + BlockAllocator balloc(1024, 1024); - auto first = std::begin(rel_path); - auto last = std::end(rel_path); - - if (rel_path[0] == '/') { - res = "/"; - ++first; - } else if (base_path.empty()) { - res = "/"; - } else { - res.assign(std::begin(base_path), std::end(base_path)); - } - - for (; first != last;) { - if (*first == '.') { - if (first + 1 == last) { - break; - } - if (*(first + 1) == '/') { - first += 2; - continue; - } - if (*(first + 1) == '.') { - if (first + 2 == last) { - eat_dir(res); - break; - } - if (*(first + 2) == '/') { - eat_dir(res); - first += 3; - continue; - } - } - } - if (res.back() != '/') { - eat_file(res); - } - auto slash = std::find(first, last, '/'); - if (slash == last) { - res.append(first, last); - break; - } - res.append(first, slash + 1); - first = slash + 1; - for (; first != last && *first == '/'; ++first) - ; - } - if (!rel_query.empty()) { - res += '?'; - res.append(std::begin(rel_query), std::end(rel_query)); - } - return res; + return path_join(balloc, base_path, base_query, rel_path, rel_query).str(); } bool expect_response_body(int status_code) { @@ -1729,7 +1608,7 @@ StringRef path_join(BlockAllocator &balloc, const StringRef &base_path, StringRef normalize_path(BlockAllocator &balloc, const StringRef &path, const StringRef &query) { // First, decode %XX for unreserved characters, then do - // http2::join_path + // http2::path_join // We won't find %XX if length is less than 3. if (path.size() < 3 || From 78fcb2143f51809a8858d2034866b3ec0b43e13f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 19:42:24 +0900 Subject: [PATCH 124/147] Update doc --- src/allocator.h | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/allocator.h b/src/allocator.h index cd92d3fd..28fe8eb0 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -34,10 +34,19 @@ namespace nghttp2 { struct MemBlock { + // The next MemBlock to chain them. This is for book keeping + // purpose to free them later. MemBlock *next; + // begin is the pointer to the beginning of buffer. last is the + // location of next write. end is the one beyond of the end of the + // buffer. uint8_t *begin, *last, *end; }; +// BlockAllocator allocates memory block with given size at once, and +// cuts the region from it when allocation is requested. If the +// requested size is larger than given threshold, it will be allocated +// in a distinct buffer on demand. struct BlockAllocator { BlockAllocator(size_t block_size, size_t isolation_threshold) : retain(nullptr), @@ -83,7 +92,9 @@ struct BlockAllocator { return res; } + // This holds live memory block to free them in dtor. MemBlock *retain; + // Current memory block to use. MemBlock *head; // size of single memory block size_t block_size; @@ -92,6 +103,8 @@ struct BlockAllocator { size_t isolation_threshold; }; +// Makes a copy of |src|. The resulting string will be +// NULL-terminated. template StringRef make_string_ref(BlockAllocator &alloc, const StringRef &src) { auto dst = static_cast(alloc.alloc(src.size() + 1)); @@ -101,6 +114,8 @@ StringRef make_string_ref(BlockAllocator &alloc, const StringRef &src) { return StringRef{dst, src.size()}; } +// Returns the string which is the concatenation of |a| and |b| in +// this order. The resulting string will be NULL-terminated. template StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, const StringRef &b) { @@ -113,6 +128,8 @@ StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, return StringRef{dst, len}; } +// Returns the string which is the concatenation of |a|, |b| and |c| +// in this order. The resulting string will be NULL-terminated. template StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, const StringRef &b, const StringRef &c) { @@ -127,10 +144,14 @@ StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, } struct ByteRef { + // The pointer to the beginning of the buffer. uint8_t *base; + // The length of the buffer. size_t len; }; +// Makes a buffer with given size. The resulting byte string might +// not be NULL-terminated. template ByteRef make_byte_ref(BlockAllocator &alloc, size_t size) { auto dst = static_cast(alloc.alloc(size)); From 0ee09320e00693b772f9d62156cce4a4bb88bebf Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 21:06:05 +0900 Subject: [PATCH 125/147] src: Remove unused value_to_str --- src/http2.cc | 7 ------- src/http2.h | 3 --- 2 files changed, 10 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index 092c4c8e..8b82dcca 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -302,13 +302,6 @@ const Headers::value_type *get_header(const Headers &nva, const char *name) { return res; } -std::string value_to_str(const HeaderRefs::value_type *nv) { - if (nv) { - return nv->value.str(); - } - return ""; -} - bool non_empty_value(const HeaderRefs::value_type *nv) { return nv && !nv->value.empty(); } diff --git a/src/http2.h b/src/http2.h index 0ae9d3b3..8ea2f91c 100644 --- a/src/http2.h +++ b/src/http2.h @@ -127,9 +127,6 @@ void add_header(Headers &nva, const uint8_t *name, size_t namelen, // in |nva| is returned. If no such entry exist, returns nullptr. const Headers::value_type *get_header(const Headers &nva, const char *name); -// Returns nv->second if nv is not nullptr. Otherwise, returns "". -std::string value_to_str(const HeaderRefs::value_type *nv); - // Returns true if the value of |nv| is not empty. bool non_empty_value(const HeaderRefs::value_type *nv); From 94f52d49dc907d57f4046558eb5c3f8a07595acd Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 12 Mar 2016 21:25:53 +0900 Subject: [PATCH 126/147] BlockAllocator: Delete copy ctor and assignment operator --- src/allocator.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allocator.h b/src/allocator.h index 28fe8eb0..64bdff49 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -62,6 +62,12 @@ struct BlockAllocator { } } + BlockAllocator(BlockAllocator &&) = default; + BlockAllocator &operator=(BlockAllocator &&) = default; + + BlockAllocator(const BlockAllocator &) = delete; + BlockAllocator &operator=(const BlockAllocator &) = delete; + MemBlock *alloc_mem_block(size_t size) { auto block = new uint8_t[sizeof(MemBlock) + size]; auto mb = reinterpret_cast(block); From 34d209b30b602c9b04e8e7ee8a748865d5fecf9d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 13 Mar 2016 00:59:25 +0900 Subject: [PATCH 127/147] nghttpx: Add wildcard host routing This change allows host pattern in --backend to include '*' to indicate wildcard match. The wildcard match is made in suffix match only. --- src/shrpx.cc | 26 ++++++++- src/shrpx_client_handler.cc | 16 ++--- src/shrpx_config.cc | 27 ++++++++- src/shrpx_config.h | 9 +++ src/shrpx_worker.cc | 47 +++++++++++---- src/shrpx_worker.h | 3 +- src/shrpx_worker_test.cc | 113 ++++++++++++++++++++++++------------ src/template.h | 31 ++++++++++ 8 files changed, 212 insertions(+), 60 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 3beb91be..f59c2b24 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1227,8 +1227,14 @@ Connections: Patterns with host take precedence over patterns with just path. Then, longer patterns take precedence over - shorter ones, breaking a tie by the order of the - appearance in the configuration. + shorter ones. + + Host can include "*" in the left most position to + indicate wildcard match (only suffix match is done). + For example, host pattern "*www.nghttp2.org" matches + against "www.nghttp2.org" and "1www.ngttp2.org", but + does not match against "nghttp2.org". The exact hosts + match takes precedence over the wildcard hosts match. If is omitted or empty string, "/" is used as pattern, which matches all request paths (catch-all @@ -2089,11 +2095,27 @@ void process_options( } catch_all.proto = proto; std::vector().swap(addr_groups); + std::vector().swap(mod_config()->wildcard_patterns); // maybe not necessary? mod_config()->router = Router(); mod_config()->router.add_route(StringRef{catch_all.pattern}, addr_groups.size()); addr_groups.push_back(std::move(catch_all)); + } else { + auto &wildcard_patterns = mod_config()->wildcard_patterns; + std::sort(std::begin(wildcard_patterns), std::end(wildcard_patterns), + [](const WildcardPattern &lhs, const WildcardPattern &rhs) { + return std::lexicographical_compare( + rhs.host.rbegin(), rhs.host.rend(), lhs.host.rbegin(), + lhs.host.rend()); + }); + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Reverse sorted wildcard hosts (compared from tail to head, " + "and sorted in reverse order):"; + for (auto &wp : mod_config()->wildcard_patterns) { + LOG(INFO) << wp.host; + } + } } if (LOG_ENABLED(INFO)) { diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index d7366b04..2588264f 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -689,7 +689,7 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { const auto &req = downstream->request(); // Fast path. If we have one group, it must be catch-all group. - // HTTP/2 and client proxy modes fall in this case. + // proxy mode falls in this case. if (groups.size() == 1) { group_idx = 0; } else if (req.method == HTTP_CONNECT) { @@ -699,20 +699,20 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { group_idx = catch_all; } else { auto &router = get_config()->router; + auto &wildcard_patterns = get_config()->wildcard_patterns; if (!req.authority.empty()) { group_idx = - match_downstream_addr_group(router, StringRef{req.authority}, - StringRef{req.path}, groups, catch_all); + match_downstream_addr_group(router, wildcard_patterns, req.authority, + req.path, groups, catch_all); } else { auto h = req.fs.header(http2::HD_HOST); if (h) { - group_idx = - match_downstream_addr_group(router, StringRef{h->value}, - StringRef{req.path}, groups, catch_all); + group_idx = match_downstream_addr_group( + router, wildcard_patterns, h->value, req.path, groups, catch_all); } else { group_idx = - match_downstream_addr_group(router, StringRef::from_lit(""), - StringRef{req.path}, groups, catch_all); + match_downstream_addr_group(router, wildcard_patterns, StringRef{}, + req.path, groups, catch_all); } } } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index a2ad7c54..2a08b690 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -648,7 +648,32 @@ int parse_mapping(const DownstreamAddrConfig &addr, g.addrs.push_back(addr); g.proto = proto; - mod_config()->router.add_route(StringRef{g.pattern}, addr_groups.size()); + if (pattern[0] == '*') { + // wildcard pattern + auto path_first = + std::find(std::begin(g.pattern), std::end(g.pattern), '/'); + + auto host = StringRef{std::begin(g.pattern) + 1, path_first}; + auto path = StringRef{path_first, std::end(g.pattern)}; + + auto &wildcard_patterns = mod_config()->wildcard_patterns; + + auto it = std::find_if( + std::begin(wildcard_patterns), std::end(wildcard_patterns), + [&host](const WildcardPattern &wp) { return wp.host == host; }); + + if (it == std::end(wildcard_patterns)) { + mod_config()->wildcard_patterns.push_back( + {ImmutableString{std::begin(host), std::end(host)}}); + + auto &router = mod_config()->wildcard_patterns.back().router; + router.add_route(path, addr_groups.size()); + } else { + (*it).router.add_route(path, addr_groups.size()); + } + } else { + mod_config()->router.add_route(StringRef{g.pattern}, addr_groups.size()); + } addr_groups.push_back(std::move(g)); } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 72594fa8..5a31e820 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -577,8 +577,17 @@ struct ConnectionConfig { } downstream; }; +// Wildcard host pattern routing. We strips left most '*' from host +// field. router includes all path pattern sharing same wildcard +// host. +struct WildcardPattern { + ImmutableString host; + Router router; +}; + struct Config { Router router; + std::vector wildcard_patterns; HttpProxy downstream_http_proxy; HttpConfig http; Http2Config http2; diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 7c8c6bb3..601c8143 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -290,7 +290,8 @@ ConnectBlocker *Worker::get_connect_blocker() const { namespace { size_t match_downstream_addr_group_host( - const Router &router, const StringRef &host, const StringRef &path, + const Router &router, const std::vector &wildcard_patterns, + const StringRef &host, const StringRef &path, const std::vector &groups, size_t catch_all) { if (path.empty() || path[0] != '/') { auto group = router.match(host, StringRef::from_lit("/")); @@ -318,6 +319,24 @@ size_t match_downstream_addr_group_host( return group; } + for (auto it = std::begin(wildcard_patterns); + it != std::end(wildcard_patterns); ++it) { + if (!util::ends_with(std::begin(host), std::end(host), + std::begin((*it).host), std::end((*it).host))) { + continue; + } + auto group = (*it).router.match(StringRef{}, path); + if (group != -1) { + // We sorted wildcard_patterns in a way that first match is the + // longest host pattern. + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Found wildcard pattern with query " << host << path + << ", matched pattern=" << groups[group].pattern; + } + return group; + } + } + group = router.match(StringRef::from_lit(""), path); if (group != -1) { if (LOG_ENABLED(INFO)) { @@ -335,7 +354,8 @@ size_t match_downstream_addr_group_host( } // namespace size_t match_downstream_addr_group( - const Router &router, const StringRef &hostport, const StringRef &raw_path, + const Router &router, const std::vector &wildcard_patterns, + const StringRef &hostport, const StringRef &raw_path, const std::vector &groups, size_t catch_all) { if (std::find(std::begin(hostport), std::end(hostport), '/') != std::end(hostport)) { @@ -349,11 +369,11 @@ size_t match_downstream_addr_group( auto path = StringRef{std::begin(raw_path), query}; if (hostport.empty()) { - return match_downstream_addr_group_host(router, hostport, path, groups, - catch_all); + return match_downstream_addr_group_host(router, wildcard_patterns, hostport, + path, groups, catch_all); } - std::string host; + StringRef host; if (hostport[0] == '[') { // assume this is IPv6 numeric address auto p = std::find(std::begin(hostport), std::end(hostport), ']'); @@ -363,18 +383,25 @@ size_t match_downstream_addr_group( if (p + 1 < std::end(hostport) && *(p + 1) != ':') { return catch_all; } - host.assign(std::begin(hostport), p + 1); + host = StringRef{std::begin(hostport), p + 1}; } else { auto p = std::find(std::begin(hostport), std::end(hostport), ':'); if (p == std::begin(hostport)) { return catch_all; } - host.assign(std::begin(hostport), p); + host = StringRef{std::begin(hostport), p}; } - util::inp_strlower(host); - return match_downstream_addr_group_host(router, StringRef{host}, path, groups, - catch_all); + std::string low_host; + if (std::find_if(std::begin(host), std::end(host), [](char c) { + return 'A' <= c || c <= 'Z'; + }) != std::end(host)) { + low_host = host.str(); + util::inp_strlower(low_host); + host = StringRef{low_host}; + } + return match_downstream_addr_group_host(router, wildcard_patterns, host, path, + groups, catch_all); } } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 0602e988..17a9fee9 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -206,7 +206,8 @@ private: // group. The catch-all group index is given in |catch_all|. All // patterns are given in |groups|. size_t match_downstream_addr_group( - const Router &router, const StringRef &hostport, const StringRef &path, + const Router &router, const std::vector &wildcard_patterns, + const StringRef &hostport, const StringRef &path, const std::vector &groups, size_t catch_all); } // namespace shrpx diff --git a/src/shrpx_worker_test.cc b/src/shrpx_worker_test.cc index 11a15e01..ae2f0e80 100644 --- a/src/shrpx_worker_test.cc +++ b/src/shrpx_worker_test.cc @@ -43,7 +43,7 @@ void test_shrpx_worker_match_downstream_addr_group(void) { "nghttp2.org/alpha/charlie", "nghttp2.org/delta%3A", "www.nghttp2.org/", "[::1]/", "nghttp2.org/alpha/bravo/delta", // Check that match is done in the single node - "example.com/alpha/bravo", "192.168.0.1/alpha/"}) { + "example.com/alpha/bravo", "192.168.0.1/alpha/", "/golf/"}) { groups.push_back(DownstreamAddrGroup{ImmutableString(s)}); } @@ -54,126 +54,163 @@ void test_shrpx_worker_match_downstream_addr_group(void) { router.add_route(StringRef{g.pattern}, i); } + std::vector wp; + CU_ASSERT(0 == match_downstream_addr_group( - router, StringRef::from_lit("nghttp2.org"), + router, wp, StringRef::from_lit("nghttp2.org"), StringRef::from_lit("/"), groups, 255)); // port is removed CU_ASSERT(0 == match_downstream_addr_group( - router, StringRef::from_lit("nghttp2.org:8080"), + router, wp, StringRef::from_lit("nghttp2.org:8080"), StringRef::from_lit("/"), groups, 255)); // host is case-insensitive CU_ASSERT(4 == match_downstream_addr_group( - router, StringRef::from_lit("WWW.nghttp2.org"), + router, wp, StringRef::from_lit("WWW.nghttp2.org"), StringRef::from_lit("/alpha"), groups, 255)); CU_ASSERT(1 == match_downstream_addr_group( - router, StringRef::from_lit("nghttp2.org"), + router, wp, StringRef::from_lit("nghttp2.org"), StringRef::from_lit("/alpha/bravo/"), groups, 255)); // /alpha/bravo also matches /alpha/bravo/ CU_ASSERT(1 == match_downstream_addr_group( - router, StringRef::from_lit("nghttp2.org"), + router, wp, StringRef::from_lit("nghttp2.org"), StringRef::from_lit("/alpha/bravo"), groups, 255)); // path part is case-sensitive CU_ASSERT(0 == match_downstream_addr_group( - router, StringRef::from_lit("nghttp2.org"), + router, wp, StringRef::from_lit("nghttp2.org"), StringRef::from_lit("/Alpha/bravo"), groups, 255)); CU_ASSERT(1 == match_downstream_addr_group( - router, StringRef::from_lit("nghttp2.org"), + router, wp, StringRef::from_lit("nghttp2.org"), StringRef::from_lit("/alpha/bravo/charlie"), groups, 255)); CU_ASSERT(2 == match_downstream_addr_group( - router, StringRef::from_lit("nghttp2.org"), + router, wp, StringRef::from_lit("nghttp2.org"), StringRef::from_lit("/alpha/charlie"), groups, 255)); // pattern which does not end with '/' must match its entirely. So // this matches to group 0, not group 2. CU_ASSERT(0 == match_downstream_addr_group( - router, StringRef::from_lit("nghttp2.org"), + router, wp, StringRef::from_lit("nghttp2.org"), StringRef::from_lit("/alpha/charlie/"), groups, 255)); CU_ASSERT(255 == match_downstream_addr_group( - router, StringRef::from_lit("example.org"), + router, wp, StringRef::from_lit("example.org"), StringRef::from_lit("/"), groups, 255)); - CU_ASSERT(255 == match_downstream_addr_group(router, StringRef::from_lit(""), - StringRef::from_lit("/"), groups, - 255)); - - CU_ASSERT(255 == match_downstream_addr_group(router, StringRef::from_lit(""), - StringRef::from_lit("alpha"), - groups, 255)); - CU_ASSERT(255 == - match_downstream_addr_group(router, StringRef::from_lit("foo/bar"), + match_downstream_addr_group(router, wp, StringRef::from_lit(""), StringRef::from_lit("/"), groups, 255)); + CU_ASSERT(255 == match_downstream_addr_group( + router, wp, StringRef::from_lit(""), + StringRef::from_lit("alpha"), groups, 255)); + + CU_ASSERT(255 == match_downstream_addr_group( + router, wp, StringRef::from_lit("foo/bar"), + StringRef::from_lit("/"), groups, 255)); + // If path is StringRef::from_lit("*", only match with host + "/"). CU_ASSERT(0 == match_downstream_addr_group( - router, StringRef::from_lit("nghttp2.org"), + router, wp, StringRef::from_lit("nghttp2.org"), StringRef::from_lit("*"), groups, 255)); - CU_ASSERT(5 == - match_downstream_addr_group(router, StringRef::from_lit("[::1]"), - StringRef::from_lit("/"), groups, 255)); + CU_ASSERT( + 5 == match_downstream_addr_group(router, wp, StringRef::from_lit("[::1]"), + StringRef::from_lit("/"), groups, 255)); CU_ASSERT(5 == match_downstream_addr_group( - router, StringRef::from_lit("[::1]:8080"), + router, wp, StringRef::from_lit("[::1]:8080"), StringRef::from_lit("/"), groups, 255)); CU_ASSERT(255 == - match_downstream_addr_group(router, StringRef::from_lit("[::1"), + match_downstream_addr_group(router, wp, StringRef::from_lit("[::1"), StringRef::from_lit("/"), groups, 255)); CU_ASSERT(255 == match_downstream_addr_group( - router, StringRef::from_lit("[::1]8000"), + router, wp, StringRef::from_lit("[::1]8000"), StringRef::from_lit("/"), groups, 255)); // Check the case where adding route extends tree CU_ASSERT(6 == match_downstream_addr_group( - router, StringRef::from_lit("nghttp2.org"), + router, wp, StringRef::from_lit("nghttp2.org"), StringRef::from_lit("/alpha/bravo/delta"), groups, 255)); CU_ASSERT(1 == match_downstream_addr_group( - router, StringRef::from_lit("nghttp2.org"), + router, wp, StringRef::from_lit("nghttp2.org"), StringRef::from_lit("/alpha/bravo/delta/"), groups, 255)); // Check the case where query is done in a single node CU_ASSERT(7 == match_downstream_addr_group( - router, StringRef::from_lit("example.com"), + router, wp, StringRef::from_lit("example.com"), StringRef::from_lit("/alpha/bravo"), groups, 255)); CU_ASSERT(255 == match_downstream_addr_group( - router, StringRef::from_lit("example.com"), + router, wp, StringRef::from_lit("example.com"), StringRef::from_lit("/alpha/bravo/"), groups, 255)); CU_ASSERT(255 == match_downstream_addr_group( - router, StringRef::from_lit("example.com"), + router, wp, StringRef::from_lit("example.com"), StringRef::from_lit("/alpha"), groups, 255)); // Check the case where quey is done in a single node CU_ASSERT(8 == match_downstream_addr_group( - router, StringRef::from_lit("192.168.0.1"), + router, wp, StringRef::from_lit("192.168.0.1"), StringRef::from_lit("/alpha"), groups, 255)); CU_ASSERT(8 == match_downstream_addr_group( - router, StringRef::from_lit("192.168.0.1"), + router, wp, StringRef::from_lit("192.168.0.1"), StringRef::from_lit("/alpha/"), groups, 255)); CU_ASSERT(8 == match_downstream_addr_group( - router, StringRef::from_lit("192.168.0.1"), + router, wp, StringRef::from_lit("192.168.0.1"), StringRef::from_lit("/alpha/bravo"), groups, 255)); CU_ASSERT(255 == match_downstream_addr_group( - router, StringRef::from_lit("192.168.0.1"), + router, wp, StringRef::from_lit("192.168.0.1"), StringRef::from_lit("/alph"), groups, 255)); CU_ASSERT(255 == match_downstream_addr_group( - router, StringRef::from_lit("192.168.0.1"), + router, wp, StringRef::from_lit("192.168.0.1"), StringRef::from_lit("/"), groups, 255)); - router.dump(); + // Test for wildcard hosts + groups.push_back( + DownstreamAddrGroup{ImmutableString::from_lit("git.nghttp2.org")}); + groups.push_back( + DownstreamAddrGroup{ImmutableString::from_lit(".nghttp2.org")}); + + wp.push_back({ImmutableString("git.nghttp2.org")}); + wp.back().router.add_route(StringRef::from_lit("/echo/"), 10); + + wp.push_back({ImmutableString(".nghttp2.org")}); + wp.back().router.add_route(StringRef::from_lit("/echo/"), 11); + wp.back().router.add_route(StringRef::from_lit("/echo/foxtrot"), 12); + + CU_ASSERT(10 == match_downstream_addr_group( + router, wp, StringRef::from_lit("git.nghttp2.org"), + StringRef::from_lit("/echo"), groups, 255)); + + CU_ASSERT(10 == match_downstream_addr_group( + router, wp, StringRef::from_lit("0git.nghttp2.org"), + StringRef::from_lit("/echo"), groups, 255)); + + CU_ASSERT(11 == match_downstream_addr_group( + router, wp, StringRef::from_lit("it.nghttp2.org"), + StringRef::from_lit("/echo"), groups, 255)); + + CU_ASSERT(12 == match_downstream_addr_group( + router, wp, StringRef::from_lit(".nghttp2.org"), + StringRef::from_lit("/echo/foxtrot"), groups, 255)); + + CU_ASSERT(9 == match_downstream_addr_group( + router, wp, StringRef::from_lit("alpha.nghttp2.org"), + StringRef::from_lit("/golf"), groups, 255)); + + CU_ASSERT(0 == match_downstream_addr_group( + router, wp, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/echo"), groups, 255)); } } // namespace shrpx diff --git a/src/template.h b/src/template.h index 1664714a..fdbc2aac 100644 --- a/src/template.h +++ b/src/template.h @@ -250,6 +250,7 @@ public: using const_reference = const value_type &; using const_pointer = const value_type *; using const_iterator = const_pointer; + using const_reverse_iterator = std::reverse_iterator; ImmutableString() : len(0), base("") {} ImmutableString(const char *s, size_t slen) @@ -308,6 +309,16 @@ public: const_iterator end() const { return base + len; }; const_iterator cend() const { return base + len; }; + const_reverse_iterator rbegin() const { + return const_reverse_iterator{base + len}; + } + const_reverse_iterator crbegin() const { + return const_reverse_iterator{base + len}; + } + + const_reverse_iterator rend() const { return const_reverse_iterator{base}; } + const_reverse_iterator crend() const { return const_reverse_iterator{base}; } + const char *c_str() const { return base; } size_type size() const { return len; } bool empty() const { return len == 0; } @@ -395,6 +406,7 @@ public: using const_reference = const value_type &; using const_pointer = const value_type *; using const_iterator = const_pointer; + using const_reverse_iterator = std::reverse_iterator; constexpr StringRef() : base(""), len(0) {} explicit StringRef(const std::string &s) : base(s.c_str()), len(s.size()) {} @@ -430,6 +442,16 @@ public: const_iterator end() const { return base + len; }; const_iterator cend() const { return base + len; }; + const_reverse_iterator rbegin() const { + return const_reverse_iterator{base + len}; + } + const_reverse_iterator crbegin() const { + return const_reverse_iterator{base + len}; + } + + const_reverse_iterator rend() const { return const_reverse_iterator{base}; } + const_reverse_iterator crend() const { return const_reverse_iterator{base}; } + const char *c_str() const { return base; } size_type size() const { return len; } bool empty() const { return len == 0; } @@ -464,6 +486,15 @@ inline bool operator==(const StringRef &lhs, const char *rhs) { std::equal(std::begin(lhs), std::end(lhs), rhs); } +inline bool operator==(const StringRef &lhs, const ImmutableString &rhs) { + return lhs.size() == rhs.size() && + std::equal(std::begin(lhs), std::end(lhs), std::begin(rhs)); +} + +inline bool operator==(const ImmutableString &lhs, const StringRef &rhs) { + return rhs == lhs; +} + inline bool operator==(const char *lhs, const StringRef &rhs) { return rhs == lhs; } From 27275a02aca9008c2c551d6228612f07e692c662 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 13 Mar 2016 13:59:06 +0900 Subject: [PATCH 128/147] nghttpx: Remove unused EMPTY_STRING --- src/shrpx_config.cc | 2 -- src/shrpx_config.h | 4 ---- 2 files changed, 6 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 2a08b690..8e3dec1c 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -69,8 +69,6 @@ Config *mod_config() { return config; } void create_config() { config = new Config(); } -std::string EMPTY_STRING; - TicketKeys::~TicketKeys() { /* Erase keys from memory */ for (auto &key : keys) { diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 5a31e820..f849fc39 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -255,10 +255,6 @@ enum shrpx_forwarded_node_type { FORWARDED_NODE_IP, }; -// Used inside function if it has to return const reference to empty -// string without defining empty string each time. -extern std::string EMPTY_STRING; - struct AltSvc { AltSvc() : port(0) {} From 3b2811bfc4b10c5a976fd6d3bef3cf8b99308a66 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 13 Mar 2016 14:01:59 +0900 Subject: [PATCH 129/147] nghttpx: Remove AltSvc ctor --- src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 8e3dec1c..73bdf71a 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2065,7 +2065,7 @@ int parse_config(const char *opt, const char *optarg, return -1; } - AltSvc altsvc; + AltSvc altsvc{}; altsvc.protocol_id = std::move(tokens[0]); diff --git a/src/shrpx_config.h b/src/shrpx_config.h index f849fc39..ea442556 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -256,8 +256,6 @@ enum shrpx_forwarded_node_type { }; struct AltSvc { - AltSvc() : port(0) {} - std::string protocol_id, host, origin, service; uint16_t port; From 2a606bae4c1b0cd358025e37fc5adabfd12ddda1 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 13 Mar 2016 18:21:07 +0900 Subject: [PATCH 130/147] Add nghttp2_error_callback to tell application human readable error message --- configure.ac | 3 ++ doc/Makefile.am | 1 + lib/includes/nghttp2/nghttp2.h | 34 +++++++++++++++++ lib/nghttp2_callbacks.c | 5 +++ lib/nghttp2_callbacks.h | 1 + lib/nghttp2_session.c | 67 ++++++++++++++++++++++++++++++++++ src/HttpServer.cc | 3 ++ src/app_helper.cc | 9 +++++ src/app_helper.h | 3 ++ src/nghttp.cc | 3 ++ src/shrpx_http2_upstream.cc | 5 +++ 11 files changed, 134 insertions(+) diff --git a/configure.ac b/configure.ac index 493995ba..cf9e60fd 100644 --- a/configure.ac +++ b/configure.ac @@ -723,6 +723,9 @@ if test "x$werror" != "xno"; then # Only work with Clang for the moment AX_CHECK_COMPILE_FLAG([-Wheader-guard], [CFLAGS="$CFLAGS -Wheader-guard"]) + # This is required because we pass format string as "const char*. + AX_CHECK_COMPILE_FLAG([-Wno-format-nonliteral], [CFLAGS="$CFLAGS -Wno-format-nonliteral"]) + # For C++ compiler AC_LANG_PUSH(C++) AX_CHECK_COMPILE_FLAG([-Wall], [CXXFLAGS="$CXXFLAGS -Wall"]) diff --git a/doc/Makefile.am b/doc/Makefile.am index c450a029..c1e9baac 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -73,6 +73,7 @@ APIDOCS= \ nghttp2_session_callbacks_new.rst \ nghttp2_session_callbacks_set_before_frame_send_callback.rst \ nghttp2_session_callbacks_set_data_source_read_length_callback.rst \ + nghttp2_session_callbacks_set_error_callback.rst \ nghttp2_session_callbacks_set_on_begin_frame_callback.rst \ nghttp2_session_callbacks_set_on_begin_headers_callback.rst \ nghttp2_session_callbacks_set_on_data_chunk_recv_callback.rst \ diff --git a/lib/includes/nghttp2/nghttp2.h b/lib/includes/nghttp2/nghttp2.h index 0e888fa7..8cd00ca6 100644 --- a/lib/includes/nghttp2/nghttp2.h +++ b/lib/includes/nghttp2/nghttp2.h @@ -1872,6 +1872,31 @@ typedef ssize_t (*nghttp2_pack_extension_callback)(nghttp2_session *session, const nghttp2_frame *frame, void *user_data); +/** + * @functypedef + * + * Callback function invoked when library provides the error message + * intended for human consumption. This callback is solely for + * debugging purpose. The |msg| is typically NULL-terminated string + * of length |len|. |len| does not include the sentinel NULL + * character. + * + * The format of error message may change between nghttp2 library + * versions. The application should not depend on the particular + * format. + * + * Normally, application should return 0 from this callback. If fatal + * error occurred while doing something in this callback, application + * should return :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. In this case, + * library will return immediately with return value + * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. Currently, if nonzero value + * is returned from this callback, they are treated as + * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`, but application should not + * rely on this details. + */ +typedef int (*nghttp2_error_callback)(nghttp2_session *session, const char *msg, + size_t len, void *user_data); + struct nghttp2_session_callbacks; /** @@ -2108,6 +2133,15 @@ nghttp2_session_callbacks_set_on_extension_chunk_recv_callback( nghttp2_session_callbacks *cbs, nghttp2_on_extension_chunk_recv_callback on_extension_chunk_recv_callback); +/** + * @function + * + * Sets callback function invoked when library tells error message to + * the application. + */ +NGHTTP2_EXTERN void nghttp2_session_callbacks_set_error_callback( + nghttp2_session_callbacks *cbs, nghttp2_error_callback error_callback); + /** * @functypedef * diff --git a/lib/nghttp2_callbacks.c b/lib/nghttp2_callbacks.c index 4bf0e7a5..4d5211a1 100644 --- a/lib/nghttp2_callbacks.c +++ b/lib/nghttp2_callbacks.c @@ -151,3 +151,8 @@ void nghttp2_session_callbacks_set_on_extension_chunk_recv_callback( nghttp2_on_extension_chunk_recv_callback on_extension_chunk_recv_callback) { cbs->on_extension_chunk_recv_callback = on_extension_chunk_recv_callback; } + +void nghttp2_session_callbacks_set_error_callback( + nghttp2_session_callbacks *cbs, nghttp2_error_callback error_callback) { + cbs->error_callback = error_callback; +} diff --git a/lib/nghttp2_callbacks.h b/lib/nghttp2_callbacks.h index 80971c54..5f08474a 100644 --- a/lib/nghttp2_callbacks.h +++ b/lib/nghttp2_callbacks.h @@ -111,6 +111,7 @@ struct nghttp2_session_callbacks { nghttp2_pack_extension_callback pack_extension_callback; nghttp2_unpack_extension_callback unpack_extension_callback; nghttp2_on_extension_chunk_recv_callback on_extension_chunk_recv_callback; + nghttp2_error_callback error_callback; }; #endif /* NGHTTP2_CALLBACKS_H */ diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 44ed7203..9d3bf1e6 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -28,6 +28,7 @@ #include #include #include +#include #include "nghttp2_helper.h" #include "nghttp2_net.h" @@ -141,6 +142,62 @@ static int session_detect_idle_stream(nghttp2_session *session, return 0; } +static int session_call_error_callback(nghttp2_session *session, + const char *fmt, ...) { + size_t bufsize; + va_list ap; + char *buf; + int rv; + nghttp2_mem *mem; + + if (!session->callbacks.error_callback) { + return 0; + } + + mem = &session->mem; + + va_start(ap, fmt); + rv = vsnprintf(NULL, 0, fmt, ap); + va_end(ap); + + if (rv < 0) { + return NGHTTP2_ERR_NOMEM; + } + + bufsize = (size_t)(rv + 1); + + buf = nghttp2_mem_malloc(mem, bufsize); + if (buf == NULL) { + return NGHTTP2_ERR_NOMEM; + } + + va_start(ap, fmt); + rv = vsnprintf(buf, bufsize, fmt, ap); + va_end(ap); + + if (rv < 0) { + nghttp2_mem_free(mem, buf); + /* vsnprintf may return error because of various things we can + imagine, but typically we don't want to drop session just for + debug callback. */ + DEBUGF(fprintf(stderr, + "error_callback: vsnprintf failed. The template was %s\n", + fmt)); + return 0; + } + + rv = session->callbacks.error_callback(session, buf, (size_t)rv, + session->user_data); + + nghttp2_mem_free(mem, buf); + + if (rv != 0) { + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + + return 0; +} + static int session_terminate_session(nghttp2_session *session, int32_t last_stream_id, uint32_t error_code, const char *reason) { @@ -3384,6 +3441,16 @@ static int inflate_header_block(nghttp2_session *session, nghttp2_frame *frame, frame->hd.type, subject_stream->stream_id, (int)nv.name->len, nv.name->base, (int)nv.value->len, nv.value->base)); + rv = session_call_error_callback( + session, "Invalid HTTP header field was received: frame type: " + "%u, stream: %d, name: [%.*s], value: [%.*s]", + frame->hd.type, frame->hd.stream_id, (int)nv.name->len, + nv.name->base, (int)nv.value->len, nv.value->base); + + if (nghttp2_is_fatal(rv)) { + return rv; + } + rv = session_handle_invalid_stream2(session, subject_stream->stream_id, frame, NGHTTP2_ERR_HTTP_HEADER); diff --git a/src/HttpServer.cc b/src/HttpServer.cc index 5c65295d..74e9a9b6 100644 --- a/src/HttpServer.cc +++ b/src/HttpServer.cc @@ -1689,6 +1689,9 @@ void fill_callback(nghttp2_session_callbacks *callbacks, const Config *config) { if (config->verbose) { nghttp2_session_callbacks_set_on_invalid_frame_recv_callback( callbacks, verbose_on_invalid_frame_recv_callback); + + nghttp2_session_callbacks_set_error_callback(callbacks, + verbose_error_callback); } nghttp2_session_callbacks_set_on_data_chunk_recv_callback( diff --git a/src/app_helper.cc b/src/app_helper.cc index ba84052b..603151cf 100644 --- a/src/app_helper.cc +++ b/src/app_helper.cc @@ -414,6 +414,15 @@ int verbose_on_data_chunk_recv_callback(nghttp2_session *session, uint8_t flags, return 0; } +int verbose_error_callback(nghttp2_session *session, const char *msg, + size_t len, void *user_data) { + print_timer(); + fprintf(outfile, " [ERROR] %.*s\n", (int)len, msg); + fflush(outfile); + + return 0; +} + namespace { std::chrono::steady_clock::time_point base_tv; } // namespace diff --git a/src/app_helper.h b/src/app_helper.h index 408adac2..263bb99c 100644 --- a/src/app_helper.h +++ b/src/app_helper.h @@ -60,6 +60,9 @@ int verbose_on_data_chunk_recv_callback(nghttp2_session *session, uint8_t flags, int32_t stream_id, const uint8_t *data, size_t len, void *user_data); +int verbose_error_callback(nghttp2_session *session, const char *msg, + size_t len, void *user_data); + // Returns difference between |a| and |b| in milliseconds, assuming // |a| is more recent than |b|. template diff --git a/src/nghttp.cc b/src/nghttp.cc index d0fc8ece..d219d1da 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -2258,6 +2258,9 @@ int run(char **uris, int n) { nghttp2_session_callbacks_set_on_invalid_frame_recv_callback( callbacks, verbose_on_invalid_frame_recv_callback); + + nghttp2_session_callbacks_set_error_callback(callbacks, + verbose_error_callback); } nghttp2_session_callbacks_set_on_data_chunk_recv_callback( diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 32905876..6ad4ac24 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -837,6 +837,11 @@ nghttp2_session_callbacks *create_http2_upstream_callbacks() { callbacks, http::select_padding_callback); } + if (get_config()->http2.upstream.debug.frame_debug) { + nghttp2_session_callbacks_set_error_callback(callbacks, + verbose_error_callback); + } + return callbacks; } From 37b09f6785359d070c59677a4dc6279d9fae7e60 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 13 Mar 2016 18:25:30 +0900 Subject: [PATCH 131/147] Use %u for uint8_t for formatting --- lib/nghttp2_session.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 9d3bf1e6..2c13b2b0 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -3437,7 +3437,7 @@ static int inflate_header_block(nghttp2_session *session, nghttp2_frame *frame, trailer); if (rv == NGHTTP2_ERR_HTTP_HEADER) { DEBUGF(fprintf( - stderr, "recv: HTTP error: type=%d, id=%d, header %.*s: %.*s\n", + stderr, "recv: HTTP error: type=%u, id=%d, header %.*s: %.*s\n", frame->hd.type, subject_stream->stream_id, (int)nv.name->len, nv.name->base, (int)nv.value->len, nv.value->base)); @@ -3463,7 +3463,7 @@ static int inflate_header_block(nghttp2_session *session, nghttp2_frame *frame, if (rv == NGHTTP2_ERR_IGN_HTTP_HEADER) { /* header is ignored */ DEBUGF(fprintf( - stderr, "recv: HTTP ignored: type=%d, id=%d, header %.*s: %.*s\n", + stderr, "recv: HTTP ignored: type=%u, id=%d, header %.*s: %.*s\n", frame->hd.type, subject_stream->stream_id, (int)nv.name->len, nv.name->base, (int)nv.value->len, nv.value->base)); } @@ -5899,7 +5899,7 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session, const uint8_t *in, if (cont_hd.type != NGHTTP2_CONTINUATION || cont_hd.stream_id != iframe->frame.hd.stream_id) { DEBUGF(fprintf(stderr, "recv: expected stream_id=%d, type=%d, but " - "got stream_id=%d, type=%d\n", + "got stream_id=%d, type=%u\n", iframe->frame.hd.stream_id, NGHTTP2_CONTINUATION, cont_hd.stream_id, cont_hd.type)); rv = nghttp2_session_terminate_session_with_reason( From e80977c8128b0201a15576c9792a943ba1caaf9b Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Sun, 13 Mar 2016 15:24:27 +0100 Subject: [PATCH 132/147] integration-tests: do not use recursive target Older automake (travis, *cough*) do not support the AM_EXTRA_RECURSIVE_TARGETS option, this results in invoking "it-local" directly in the travis script which relies on an implementation detail. Since the "it" target is only used by the integration-tests directory, just avoid the recursive targets. The README.rst suggests to enter the integration-tests directory anyway. --- .travis.yml | 4 ++-- configure.ac | 2 -- integration-tests/Makefile.am | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7239bd90..0191126e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,5 +48,5 @@ script: - make check - cd integration-tests - export GOPATH="$PWD/integration-tests/golang" - - make itprep-local - - make it-local + - make itprep + - make it diff --git a/configure.ac b/configure.ac index cf9e60fd..b8afe63d 100644 --- a/configure.ac +++ b/configure.ac @@ -40,8 +40,6 @@ AC_CANONICAL_TARGET AM_INIT_AUTOMAKE([subdir-objects]) -# AM_EXTRA_RECURSIVE_TARGETS requires automake 1.13 or higher -m4_ifdef([AM_EXTRA_RECURSIVE_TARGETS], [AM_EXTRA_RECURSIVE_TARGETS([it itprep])]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) dnl See versioning rule: diff --git a/integration-tests/Makefile.am b/integration-tests/Makefile.am index aab50096..a926c4f2 100644 --- a/integration-tests/Makefile.am +++ b/integration-tests/Makefile.am @@ -39,12 +39,12 @@ EXTRA_DIST = \ req-return.rb \ resp-return.rb -itprep-local: +itprep: go get -d -v golang.org/x/net/http2 go get -d -v github.com/tatsuhiro-t/go-nghttp2 go get -d -v github.com/tatsuhiro-t/spdy go get -d -v golang.org/x/net/websocket -it-local: +it: for i in $(GO_FILES); do [ -e $(builddir)/$$i ] || cp $(srcdir)/$$i $(builddir); done sh setenv go test -v From e58db4f8b0ba733c711250315c43db8240d90b64 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 14 Mar 2016 00:54:02 +0900 Subject: [PATCH 133/147] Attempt to fix compile error with msvc --- lib/nghttp2_hd.c | 9 ++++----- lib/nghttp2_rcbuf.c | 3 ++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/nghttp2_hd.c b/lib/nghttp2_hd.c index 3aedc0fd..49ca4ae7 100644 --- a/lib/nghttp2_hd.c +++ b/lib/nghttp2_hd.c @@ -1329,11 +1329,10 @@ nghttp2_hd_nv nghttp2_hd_table_get(nghttp2_hd_context *context, size_t idx) { return hd_ringbuf_get(&context->hd_table, idx - NGHTTP2_STATIC_TABLE_LENGTH) ->nv; } else { - nghttp2_hd_static_entry *ent; - - ent = &static_table[idx]; - return (nghttp2_hd_nv){&ent->name, &ent->value, ent->token, - NGHTTP2_NV_FLAG_NONE}; + nghttp2_hd_static_entry *ent = &static_table[idx]; + nghttp2_hd_nv nv = {&ent->name, &ent->value, ent->token, + NGHTTP2_NV_FLAG_NONE}; + return nv; } } diff --git a/lib/nghttp2_rcbuf.c b/lib/nghttp2_rcbuf.c index a1af1da8..053f0dfa 100644 --- a/lib/nghttp2_rcbuf.c +++ b/lib/nghttp2_rcbuf.c @@ -94,5 +94,6 @@ void nghttp2_rcbuf_decref(nghttp2_rcbuf *rcbuf) { } nghttp2_vec nghttp2_rcbuf_get_buf(nghttp2_rcbuf *rcbuf) { - return (nghttp2_vec){rcbuf->base, rcbuf->len}; + nghttp2_vec res = {rcbuf->base, rcbuf->len}; + return res; } From e02512032be39346e377688174c0f3677eaf8544 Mon Sep 17 00:00:00 2001 From: Jan-E Date: Sun, 13 Mar 2016 18:12:39 +0100 Subject: [PATCH 134/147] Fix missing external symbol nghttp2_rcbuf_new2 --- lib/Makefile.msvc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/Makefile.msvc b/lib/Makefile.msvc index c34b6995..391ae0e3 100644 --- a/lib/Makefile.msvc +++ b/lib/Makefile.msvc @@ -90,7 +90,8 @@ NGHTTP2_SRC := nghttp2_pq.c \ nghttp2_option.c \ nghttp2_callbacks.c \ nghttp2_mem.c \ - nghttp2_http.c + nghttp2_http.c \ + nghttp2_rcbuf.c NGHTTP2_OBJ_R := $(addprefix $(OBJ_DIR)/r_, $(notdir $(NGHTTP2_SRC:.c=.obj))) NGHTTP2_OBJ_D := $(addprefix $(OBJ_DIR)/d_, $(notdir $(NGHTTP2_SRC:.c=.obj))) From 4cde76c9c3474472460d7f73467dde723cdaee77 Mon Sep 17 00:00:00 2001 From: Jan-E Date: Sun, 13 Mar 2016 18:13:46 +0100 Subject: [PATCH 135/147] Do not use Cython by default --- lib/Makefile.msvc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Makefile.msvc b/lib/Makefile.msvc index 391ae0e3..4c2043af 100644 --- a/lib/Makefile.msvc +++ b/lib/Makefile.msvc @@ -12,8 +12,8 @@ # THIS_MAKEFILE := $(lastword $(MAKEFILE_LIST)) -USE_CYTHON := 1 -#USE_CYTHON := 0 +USE_CYTHON := 0 +#USE_CYTHON := 1 _VERSION := $(shell grep AC_INIT ../configure.ac | cut -d'[' -f3 | sed -r -e 's/(-DEV)?], //g') _VERSION := $(subst ., ,$(_VERSION)) From fa36537f8285c7a317427b46cb4627d4ee2f7b5d Mon Sep 17 00:00:00 2001 From: Jan-E Date: Sun, 13 Mar 2016 18:18:22 +0100 Subject: [PATCH 136/147] Detect version for releases en dev versions --- lib/Makefile.msvc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Makefile.msvc b/lib/Makefile.msvc index 4c2043af..5ab6a697 100644 --- a/lib/Makefile.msvc +++ b/lib/Makefile.msvc @@ -15,7 +15,7 @@ THIS_MAKEFILE := $(lastword $(MAKEFILE_LIST)) USE_CYTHON := 0 #USE_CYTHON := 1 -_VERSION := $(shell grep AC_INIT ../configure.ac | cut -d'[' -f3 | sed -r -e 's/(-DEV)?], //g') +_VERSION := $(shell grep AC_INIT ../configure.ac | cut -d'[' -f3 | sed -e 's/-DEV//g' -e 's/], //g') _VERSION := $(subst ., ,$(_VERSION)) VER_MAJOR := $(word 1,$(_VERSION)) VER_MINOR := $(word 2,$(_VERSION)) From 77852cf57258382956c812b5762d1a55d53183b6 Mon Sep 17 00:00:00 2001 From: Jan-E Date: Sun, 13 Mar 2016 18:24:32 +0100 Subject: [PATCH 137/147] Use string for NGHTTP2_VERSION_NUM, e.g 0x010800 --- lib/Makefile.msvc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Makefile.msvc b/lib/Makefile.msvc index 5ab6a697..411d3a47 100644 --- a/lib/Makefile.msvc +++ b/lib/Makefile.msvc @@ -21,7 +21,7 @@ VER_MAJOR := $(word 1,$(_VERSION)) VER_MINOR := $(word 2,$(_VERSION)) VER_MICRO := $(word 3,$(_VERSION)) VERSION := $(VER_MAJOR).$(VER_MINOR).$(VER_MICRO) -VERSION_NUM := ($(VER_MAJOR) << 16) + ($(VER_MINOR) << 8) + $(VER_MICRO) +VERSION_NUM := 0x$(shell if [ ${VER_MAJOR} -lt 10 ] ;then echo 0;fi)$(VER_MAJOR)$(shell if [ ${VER_MINOR} -lt 10 ] ;then echo 0;fi)$(VER_MINOR)$(shell if [ ${VER_MICRO} -lt 10 ] ; then echo 0;fi)$(VER_MICRO) GENERATED := 'Generated by $(realpath Makefile.MSVC)' From 78e5417ff2e630cc565873403cd8506f1af55473 Mon Sep 17 00:00:00 2001 From: Jan-E Date: Sun, 13 Mar 2016 18:31:14 +0100 Subject: [PATCH 138/147] Remove -nologo: not used in VC11/VC14, throws error in VC9 --- lib/Makefile.msvc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Makefile.msvc b/lib/Makefile.msvc index 411d3a47..3a867368 100644 --- a/lib/Makefile.msvc +++ b/lib/Makefile.msvc @@ -194,11 +194,11 @@ $(OBJ_DIR)/d_%.obj: %.c $(THIS_MAKEFILE) @echo $(OBJ_DIR)/r_nghttp2.res: $(OBJ_DIR)/nghttp2.rc $(THIS_MAKEFILE) - $(RC) -nologo -D_RELEASE -Fo $@ $< + $(RC) -D_RELEASE -Fo $@ $< @echo $(OBJ_DIR)/d_nghttp2.res: $(OBJ_DIR)/nghttp2.rc $(THIS_MAKEFILE) - $(RC) -nologo -D_DEBUG -Fo $@ $< + $(RC) -D_DEBUG -Fo $@ $< @echo includes/nghttp2/nghttp2ver.h: includes/nghttp2/nghttp2ver.h.in $(THIS_MAKEFILE) From 4e2972a5dc497cdf2e2e26c584021a16cb0b650c Mon Sep 17 00:00:00 2001 From: Jan-E Date: Sun, 13 Mar 2016 18:43:44 +0100 Subject: [PATCH 139/147] Use string for NGHTTP2_VERSION_NUM, e.g 0x010800: do not add parentheses --- lib/Makefile.msvc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Makefile.msvc b/lib/Makefile.msvc index 3a867368..21a93f9c 100644 --- a/lib/Makefile.msvc +++ b/lib/Makefile.msvc @@ -204,7 +204,7 @@ $(OBJ_DIR)/d_nghttp2.res: $(OBJ_DIR)/nghttp2.rc $(THIS_MAKEFILE) includes/nghttp2/nghttp2ver.h: includes/nghttp2/nghttp2ver.h.in $(THIS_MAKEFILE) sed < includes/nghttp2/nghttp2ver.h.in \ -e 's/@PACKAGE_VERSION@/$(VERSION)/g' \ - -e 's/@PACKAGE_VERSION_NUM@/($(VERSION_NUM))/g' > $@ + -e 's/@PACKAGE_VERSION_NUM@/$(VERSION_NUM)/g' > $@ touch --reference=includes/nghttp2/nghttp2ver.h.in $@ From 0f71fbce8dd711610e78b6b4658ea6f32cc8c4ca Mon Sep 17 00:00:00 2001 From: Jan-E Date: Sun, 13 Mar 2016 18:57:58 +0100 Subject: [PATCH 140/147] Generate nghttp2ver.h before compiling --- lib/Makefile.msvc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Makefile.msvc b/lib/Makefile.msvc index 21a93f9c..2a68b950 100644 --- a/lib/Makefile.msvc +++ b/lib/Makefile.msvc @@ -102,7 +102,7 @@ NGHTTP2_OBJ_D := $(addprefix $(OBJ_DIR)/d_, $(notdir $(NGHTTP2_SRC:.c=.obj))) clean_nghttp2_pyd_0 clean_nghttp2_pyd_1 -all: intro $(OBJ_DIR) $(TARGETS) build_nghttp2_pyd_$(USE_CYTHON) +all: intro includes/nghttp2/nghttp2ver.h $(OBJ_DIR) $(TARGETS) build_nghttp2_pyd_$(USE_CYTHON) @echo 'Welcome to NgHTTP2 (release + debug).' @echo 'Do a "make -f Makefile.MSVC install" at own risk!' From e8b62c620d62088fae8b0873feeb8ab335c47713 Mon Sep 17 00:00:00 2001 From: Jan-E Date: Mon, 14 Mar 2016 14:13:19 +0100 Subject: [PATCH 141/147] Revert change of NGHTTP2_VERSION_NUM, keep the parentheses around VERSION_NUM, not in the sed -e --- lib/Makefile.msvc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Makefile.msvc b/lib/Makefile.msvc index 2a68b950..2a930d11 100644 --- a/lib/Makefile.msvc +++ b/lib/Makefile.msvc @@ -21,7 +21,7 @@ VER_MAJOR := $(word 1,$(_VERSION)) VER_MINOR := $(word 2,$(_VERSION)) VER_MICRO := $(word 3,$(_VERSION)) VERSION := $(VER_MAJOR).$(VER_MINOR).$(VER_MICRO) -VERSION_NUM := 0x$(shell if [ ${VER_MAJOR} -lt 10 ] ;then echo 0;fi)$(VER_MAJOR)$(shell if [ ${VER_MINOR} -lt 10 ] ;then echo 0;fi)$(VER_MINOR)$(shell if [ ${VER_MICRO} -lt 10 ] ; then echo 0;fi)$(VER_MICRO) +VERSION_NUM := (($(VER_MAJOR) << 16) + ($(VER_MINOR) << 8) + $(VER_MICRO)) GENERATED := 'Generated by $(realpath Makefile.MSVC)' From 01408209d8095df0fd72afc1628a11daa44823c6 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 14 Mar 2016 22:20:00 +0900 Subject: [PATCH 142/147] nghttpx: Fix the bug that forwarded query contains duplicated '?' This change also fixes that bug that the multiple '/' at the start of request were not coalesced into one. --- src/http2.cc | 7 +++++-- src/http2_test.cc | 13 +++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index 8b82dcca..c808b5b4 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -1550,6 +1550,8 @@ StringRef path_join(BlockAllocator &balloc, const StringRef &base_path, if (rel_path[0] == '/') { *p++ = '/'; ++first; + for (; first != last && *first == '/'; ++first) + ; } else if (base_path.empty()) { *p++ = '/'; } else { @@ -1657,11 +1659,12 @@ StringRef rewrite_clean_path(BlockAllocator &balloc, const StringRef &src) { } // probably, not necessary most of the case, but just in case. auto fragment = std::find(std::begin(src), std::end(src), '#'); - auto query = std::find(std::begin(src), fragment, '?'); + auto raw_query = std::find(std::begin(src), fragment, '?'); + auto query = raw_query; if (query != fragment) { ++query; } - return normalize_path(balloc, StringRef{std::begin(src), query}, + return normalize_path(balloc, StringRef{std::begin(src), raw_query}, StringRef{query, fragment}); } diff --git a/src/http2_test.cc b/src/http2_test.cc index 5ac726aa..fa15e745 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -817,6 +817,15 @@ void test_http2_path_join(void) { auto rel = StringRef{}; CU_ASSERT("/?r" == http2::path_join(base, baseq, rel, StringRef{})); } + { + // path starts with multiple '/'s. + auto base = StringRef{}; + auto baseq = StringRef{}; + auto rel = StringRef::from_lit("//alpha//bravo"); + auto relq = StringRef::from_lit("charlie"); + CU_ASSERT("/alpha/bravo?charlie" == + http2::path_join(base, baseq, rel, relq)); + } } void test_http2_normalize_path(void) { @@ -870,6 +879,10 @@ void test_http2_rewrite_clean_path(void) { http2::rewrite_clean_path(balloc, StringRef::from_lit("alpha%3a"))); CU_ASSERT("" == http2::rewrite_clean_path(balloc, StringRef{})); + + CU_ASSERT( + "/alpha?bravo" == + http2::rewrite_clean_path(balloc, StringRef::from_lit("//alpha?bravo"))); } void test_http2_get_pure_path_component(void) { From 3de435d7b807b85eebc021d47f2d610a873630b8 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 14 Mar 2016 22:57:26 +0900 Subject: [PATCH 143/147] contrib: Use simple form in prerequisites --- contrib/Makefile.am | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/contrib/Makefile.am b/contrib/Makefile.am index 07829e25..b71cc531 100644 --- a/contrib/Makefile.am +++ b/contrib/Makefile.am @@ -27,13 +27,16 @@ EXTRA_DIST = $(configfiles:%=%.in) nghttpx-logrotate tlsticketupdate.go edit = sed -e 's|@bindir[@]|$(bindir)|g' -nghttpx-init: %: $(srcdir)/%.in +nghttpx-init: $(srcdir)/nghttpx-init.in rm -f $@ $@.tmp $(edit) $< > $@.tmp chmod +x $@.tmp mv $@.tmp $@ -nghttpx.service nghttpx-upstart.conf: %: $(srcdir)/%.in +nghttpx.service: $(srcdir)/nghttpx.service.in + $(edit) $< > $@ + +nghttpx-upstart.conf: $(srcdir)/nghttpx-upstart.conf.in $(edit) $< > $@ $(configfiles): Makefile From a909bb3eca222e4a65c001988a679b8f1ed51ef4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 14 Mar 2016 23:47:29 +0900 Subject: [PATCH 144/147] nghttpx: Initialize proto field --- src/shrpx_config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shrpx_config.h b/src/shrpx_config.h index ea442556..252d297d 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -301,7 +301,7 @@ struct DownstreamAddrConfig { struct DownstreamAddrGroupConfig { DownstreamAddrGroupConfig(const StringRef &pattern) - : pattern(pattern.c_str(), pattern.size()) {} + : pattern(pattern.c_str(), pattern.size()), proto(PROTO_HTTP1) {} ImmutableString pattern; std::vector addrs; From dd2bdf3e3148b61a4adaf51e54ad8e04e7fe2e8d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 14 Mar 2016 23:53:10 +0900 Subject: [PATCH 145/147] nghttpx: Update doc --- src/shrpx_config.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 252d297d..c4b0cde1 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -292,6 +292,8 @@ struct DownstreamAddrConfig { // backend address. If |host_unix| is true, this is UNIX domain // socket path. ImmutableString host; + // :. This does not treat 80 and 443 specially. If + // |host_unix| is true, this is "localhost". ImmutableString hostport; // backend port. 0 if |host_unix| is true. uint16_t port; From ab734f09b6d50b784c94e1bc1a545f0d53048cd2 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 15 Mar 2016 00:46:15 +0900 Subject: [PATCH 146/147] nghttpx: Fix possible crash in rare case --- src/shrpx_http2_upstream.cc | 5 +++++ src/shrpx_spdy_upstream.cc | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 6ad4ac24..c1bca902 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1728,6 +1728,11 @@ int Http2Upstream::on_downstream_reset(bool no_retry) { for (auto downstream = downstream_queue_.get_downstreams(); downstream; downstream = downstream->dlnext) { if (downstream->get_dispatch_state() != Downstream::DISPATCH_ACTIVE) { + // This is error condition when we failed push_request_headers() + // in initiate_downstream(). Otherwise, we have + // Downstream::DISPATCH_ACTIVE state, or we did not set + // DownstreamConnection. + downstream->pop_downstream_connection(); continue; } diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 21380afa..32cb7b27 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -1224,6 +1224,11 @@ int SpdyUpstream::on_downstream_reset(bool no_retry) { for (auto downstream = downstream_queue_.get_downstreams(); downstream; downstream = downstream->dlnext) { if (downstream->get_dispatch_state() != Downstream::DISPATCH_ACTIVE) { + // This is error condition when we failed push_request_headers() + // in initiate_downstream(). Otherwise, we have + // Downstream::DISPATCH_ACTIVE state, or we did not set + // DownstreamConnection. + downstream->pop_downstream_connection(); continue; } From 2ddb83206bcec1e355e83fd70ca36c539bf29e71 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Sun, 13 Mar 2016 16:19:23 +0100 Subject: [PATCH 147/147] cmake: sync with v1.8.0-63-g37b09f6 Adds missing source files and configure.ac changes since v1.7.0-93-g093eb51. --- CMakeLists.txt | 19 ++++++++++++++----- doc/CMakeLists.txt | 12 ++++++++++++ lib/CMakeLists.txt | 1 + src/CMakeLists.txt | 1 + 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cf6be263..c7a0f4c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,14 +23,14 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. cmake_minimum_required(VERSION 3.0) -# XXX using 1.7.90 instead of 1.8.0-DEV -project(nghttp2 VERSION 1.7.90) +# XXX using 1.8.90 instead of 1.9.0-DEV +project(nghttp2 VERSION 1.8.90) # See versioning rule: # http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html -set(LT_CURRENT 18) -set(LT_REVISION 1) -set(LT_AGE 4) +set(LT_CURRENT 19) +set(LT_REVISION 0) +set(LT_AGE 5) set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") include(Version) @@ -280,6 +280,13 @@ check_function_exists(accept4 HAVE_ACCEPT4) include(CheckSymbolExists) # XXX does this correctly detect initgroups (un)availability on cygwin? check_symbol_exists(initgroups grp.h HAVE_DECL_INITGROUPS) +if(NOT HAVE_DECL_INITGROUPS AND HAVE_UNISTD_H) + # FreeBSD declares initgroups() in unistd.h + check_symbol_exists(initgroups unistd.h HAVE_DECL_INITGROUPS2) + if(HAVE_DECL_INITGROUPS2) + set(HAVE_DECL_INITGROUPS 1) + endif() +endif() check_function_exists(timerfd_create HAVE_TIMERFD_CREATE) # Checks for epoll availability, primarily for examples/tiny-nghttpd @@ -341,6 +348,8 @@ if(NOT CMAKE_C_COMPILER_ID STREQUAL "MSVC") -Wredundant-decls # Only work with Clang for the moment -Wheader-guard + # This is required because we pass format string as "const char*. + -Wno-format-nonliteral ) extract_valid_cxx_flags(WARNCXXFLAGS diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index be8078a8..1fa04f4f 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -25,37 +25,48 @@ set(APIDOCS nghttp2_hd_inflate_hd.rst nghttp2_hd_inflate_new.rst nghttp2_hd_inflate_new2.rst + nghttp2_http2_strerror.rst nghttp2_is_fatal.rst nghttp2_nv_compare_name.rst nghttp2_option_del.rst nghttp2_option_new.rst nghttp2_option_set_max_reserved_remote_streams.rst + nghttp2_option_set_no_auto_ping_ack.rst nghttp2_option_set_no_auto_window_update.rst nghttp2_option_set_no_http_messaging.rst nghttp2_option_set_no_recv_client_magic.rst nghttp2_option_set_peer_max_concurrent_streams.rst + nghttp2_option_set_user_recv_extension_type.rst nghttp2_pack_settings_payload.rst nghttp2_priority_spec_check_default.rst nghttp2_priority_spec_default_init.rst nghttp2_priority_spec_init.rst + nghttp2_rcbuf_decref.rst + nghttp2_rcbuf_get_buf.rst + nghttp2_rcbuf_incref.rst nghttp2_select_next_protocol.rst nghttp2_session_callbacks_del.rst nghttp2_session_callbacks_new.rst nghttp2_session_callbacks_set_before_frame_send_callback.rst nghttp2_session_callbacks_set_data_source_read_length_callback.rst + nghttp2_session_callbacks_set_error_callback.rst nghttp2_session_callbacks_set_on_begin_frame_callback.rst nghttp2_session_callbacks_set_on_begin_headers_callback.rst nghttp2_session_callbacks_set_on_data_chunk_recv_callback.rst nghttp2_session_callbacks_set_on_frame_not_send_callback.rst nghttp2_session_callbacks_set_on_frame_recv_callback.rst + nghttp2_session_callbacks_set_on_extension_chunk_recv_callback.rst nghttp2_session_callbacks_set_on_frame_send_callback.rst nghttp2_session_callbacks_set_on_header_callback.rst + nghttp2_session_callbacks_set_on_header_callback2.rst nghttp2_session_callbacks_set_on_invalid_frame_recv_callback.rst nghttp2_session_callbacks_set_on_stream_close_callback.rst + nghttp2_session_callbacks_set_pack_extension_callback.rst nghttp2_session_callbacks_set_recv_callback.rst nghttp2_session_callbacks_set_select_padding_callback.rst nghttp2_session_callbacks_set_send_callback.rst nghttp2_session_callbacks_set_send_data_callback.rst + nghttp2_session_callbacks_set_unpack_extension_callback.rst nghttp2_session_client_new.rst nghttp2_session_client_new2.rst nghttp2_session_client_new3.rst @@ -107,6 +118,7 @@ set(APIDOCS nghttp2_stream_get_weight.rst nghttp2_strerror.rst nghttp2_submit_data.rst + nghttp2_submit_extension.rst nghttp2_submit_goaway.rst nghttp2_submit_headers.rst nghttp2_submit_ping.rst diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index ff512a45..17777ec1 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -22,6 +22,7 @@ set(NGHTTP2_SOURCES nghttp2_callbacks.c nghttp2_mem.c nghttp2_http.c + nghttp2_rcbuf.c ) # Public shared library diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5bf8f972..3caf0edc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -145,6 +145,7 @@ if(ENABLE_APP) shrpx_ssl_test.cc shrpx_downstream_test.cc shrpx_config_test.cc + shrpx_worker_test.cc shrpx_http_test.cc http2_test.cc util_test.cc