diff --git a/contrib/.gitignore b/contrib/.gitignore new file mode 100644 index 00000000..a08baf90 --- /dev/null +++ b/contrib/.gitignore @@ -0,0 +1 @@ +nghttpx-init diff --git a/lib/nghttp2_hd.c b/lib/nghttp2_hd.c index 86773b7e..ff882790 100644 --- a/lib/nghttp2_hd.c +++ b/lib/nghttp2_hd.c @@ -436,51 +436,41 @@ static size_t count_encoded_length(size_t n, size_t prefix) { size_t k = (1 << prefix) - 1; size_t len = 0; - if(n >= k) { - n -= k; - ++len; - } else { + + if(n < k) { return 1; } - do { - ++len; - if(n >= 128) { - n >>= 7; - } else { - break; - } - } while(n); - return len; + + n -= k; + ++len; + + for(; n >= 128; n >>= 7, ++len); + + return len + 1; } static size_t encode_length(uint8_t *buf, size_t n, size_t prefix) { size_t k = (1 << prefix) - 1; - size_t len = 0; + uint8_t *begin = buf; *buf &= ~k; if(n < k) { - *buf++ |= n; - + *buf |= n; return 1; } *buf++ |= k; n -= k; - ++len; - do { - ++len; - if(n >= 128) { - *buf++ = (1 << 7) | (n & 0x7f); - n >>= 7; - } else { - *buf++ = (uint8_t)n; - break; - } - } while(n); - return len; + for(; n >= 128; n >>= 7) { + *buf++ = (1 << 7) | (n & 0x7f); + } + + *buf++ = (uint8_t)n; + + return (size_t)(buf - begin); } /* diff --git a/src/h2load.cc b/src/h2load.cc index e7ad07b2..8c958fce 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -462,33 +462,42 @@ void eventcb(bufferevent *bev, short events, void *ptr) unsigned int next_proto_len; SSL_get0_next_proto_negotiated(client->ssl, &next_proto, &next_proto_len); + for(int i = 0; i < 2; ++i) { + if(next_proto) { + if(util::check_h2_is_selected(next_proto, next_proto_len)) { + client->session = util::make_unique(client); + } else { +#ifdef HAVE_SPDYLAY + auto spdy_version = spdylay_npn_get_version(next_proto, + next_proto_len); + if(spdy_version) { + client->session = util::make_unique(client, + spdy_version); + } else { + debug_nextproto_error(); + client->fail(); + return; + } +#else // !HAVE_SPDYLAY + debug_nextproto_error(); + client->fail(); + return; +#endif // !HAVE_SPDYLAY + } + } + +#if OPENSSL_VERSION_NUMBER >= 0x10002000L + SSL_get0_alpn_selected(client->ssl, &next_proto, &next_proto_len); +#else // OPENSSL_VERSION_NUMBER < 0x10002000L + break; +#endif // OPENSSL_VERSION_NUMBER < 0x10002000L + } if(!next_proto) { debug_nextproto_error(); client->fail(); return; } - - if(util::check_h2_is_selected(next_proto, next_proto_len)) { - client->session = util::make_unique(client); - } else { -#ifdef HAVE_SPDYLAY - auto spdy_version = spdylay_npn_get_version(next_proto, - next_proto_len); - if(spdy_version) { - client->session = util::make_unique(client, - spdy_version); - } else { - debug_nextproto_error(); - client->fail(); - return; - } -#else // !HAVE_SPDYLAY - debug_nextproto_error(); - client->fail(); - return; -#endif // !HAVE_SPDYLAY - } } else { switch(config.no_tls_proto) { case Config::PROTO_HTTP2: @@ -974,6 +983,16 @@ int main(int argc, char **argv) SSL_CTX_set_next_proto_select_cb(ssl_ctx, client_select_next_proto_cb, nullptr); +#if OPENSSL_VERSION_NUMBER >= 0x10002000L + auto proto_list = util::get_default_alpn(); +#ifdef HAVE_SPDYLAY + static const char spdy_proto_list[] = "\x8spdy/3.1\x6spdy/3\x6spdy/2"; + std::copy(spdy_proto_list, spdy_proto_list + sizeof(spdy_proto_list) - 1, + std::back_inserter(proto_list)); +#endif // HAVE_SPDYLAY + SSL_CTX_set_alpn_protos(ssl_ctx, proto_list.data(), proto_list.size()); +#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L + std::vector reqlines; if(config.ifile.empty()) { diff --git a/src/nghttp.cc b/src/nghttp.cc index b31d4429..ff7a781f 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -1057,6 +1057,7 @@ struct HttpClient { auto& req_stat = req->stat; auto request_time = (i == 0) ? stat.started_system_time : stat.started_system_time + + std::chrono::duration_cast (req_stat.on_request_time - stat.on_started_time); auto wait_delta = std::chrono::duration_cast diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 27091ca1..183049f5 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -1197,4 +1197,9 @@ void Downstream::disable_downstream_wtimer() disable_timer(downstream_wtimerev_); } +bool Downstream::accesslog_ready() const +{ + return response_http_status_ > 0; +} + } // namespace shrpx diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 514b8dc5..889bbdb9 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -279,6 +279,9 @@ public: void ensure_downstream_wtimer(); void disable_downstream_rtimer(); void disable_downstream_wtimer(); + + // Returns true if accesslog can be written for this downstream. + bool accesslog_ready() const; private: Headers request_headers_; Headers response_headers_; diff --git a/src/shrpx_downstream_queue.cc b/src/shrpx_downstream_queue.cc index 9535e1bf..da4005be 100644 --- a/src/shrpx_downstream_queue.cc +++ b/src/shrpx_downstream_queue.cc @@ -142,4 +142,10 @@ bool DownstreamQueue::pending_empty() const return pending_downstreams_.empty(); } +const std::map>& +DownstreamQueue::get_active_downstreams() const +{ + return active_downstreams_; +} + } // namespace shrpx diff --git a/src/shrpx_downstream_queue.h b/src/shrpx_downstream_queue.h index 1e3632fb..9232ab8f 100644 --- a/src/shrpx_downstream_queue.h +++ b/src/shrpx_downstream_queue.h @@ -61,6 +61,8 @@ public: // Returns first Downstream object in pending_downstreams_. This // does not pop the first one. If queue is empty, returns nullptr. Downstream* pending_top() const; + const std::map>& + get_active_downstreams() const; private: // Downstream objects, not processed yet std::map> pending_downstreams_; diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index e9665812..f633fc1e 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -67,7 +67,24 @@ Http2DownstreamConnection::~Http2DownstreamConnection() downstream_->disable_downstream_rtimer(); downstream_->disable_downstream_wtimer(); - if(submit_rst_stream(downstream_) == 0) { + uint32_t error_code; + if(downstream_->get_request_state() == Downstream::STREAM_CLOSED && + downstream_->get_upgraded()) { + // For upgraded connection, send NO_ERROR. Should we consider + // request states other than Downstream::STREAM_CLOSED ? + error_code = NGHTTP2_NO_ERROR; + } else { + error_code = NGHTTP2_INTERNAL_ERROR; + } + + if(LOG_ENABLED(INFO)) { + DCLOG(INFO, this) << "Submit RST_STREAM for DOWNSTREAM:" + << downstream_ << ", stream_id=" + << downstream_->get_downstream_stream_id() + << ", error_code=" << error_code; + } + + if(submit_rst_stream(downstream_, error_code) == 0) { http2session_->notify(); } @@ -196,64 +213,53 @@ ssize_t http2_data_read_callback(nghttp2_session *session, return NGHTTP2_ERR_DEFERRED; } auto body = dconn->get_request_body_buf(); - int nread = 0; - for(;;) { - nread = evbuffer_remove(body, buf, length); - if(nread == -1) { - DCLOG(FATAL, dconn) << "evbuffer_remove() failed"; + + auto nread = evbuffer_remove(body, buf, length); + if(nread == -1) { + DCLOG(FATAL, dconn) << "evbuffer_remove() failed"; + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + + if(nread > 0) { + // This is important because it will handle flow control + // stuff. + if(downstream->get_upstream()->resume_read + (SHRPX_NO_BUFFER, downstream, nread) != 0) { + // In this case, downstream may be deleted. return NGHTTP2_ERR_CALLBACK_FAILURE; } - if(nread > 0) { - // This is important because it will handle flow control - // stuff. - if(downstream->get_upstream()->resume_read - (SHRPX_NO_BUFFER, downstream, nread) != 0) { - // In this case, downstream may be deleted. - return NGHTTP2_ERR_CALLBACK_FAILURE; - } - - // Check dconn is still alive because Upstream::resume_read() - // may delete downstream which will delete dconn. - if(sd->dconn == nullptr) { - return NGHTTP2_ERR_DEFERRED; - } - - break; - } - - if(downstream->get_request_state() == Downstream::MSG_COMPLETE) { - if(!downstream->get_upgrade_request() || - (downstream->get_response_state() == Downstream::HEADER_COMPLETE && - !downstream->get_upgraded())) { - *data_flags |= NGHTTP2_DATA_FLAG_EOF; - } else { - downstream->disable_downstream_wtimer(); - - return NGHTTP2_ERR_DEFERRED; - } - break; - } else { - if(evbuffer_get_length(body) == 0) { - // Check get_request_state() == MSG_COMPLETE just in case - if(downstream->get_request_state() == Downstream::MSG_COMPLETE) { - *data_flags |= NGHTTP2_DATA_FLAG_EOF; - break; - } - - downstream->disable_downstream_wtimer(); - - return NGHTTP2_ERR_DEFERRED; - } + // Check dconn is still alive because Upstream::resume_read() + // may delete downstream which will delete dconn. + if(sd->dconn == nullptr) { + return NGHTTP2_ERR_DEFERRED; } } - if(evbuffer_get_length(body) > 0 && !downstream->get_output_buffer_full()) { + if(evbuffer_get_length(body) == 0 && + downstream->get_request_state() == Downstream::MSG_COMPLETE && + // If connection is upgraded, don't set EOF flag, since HTTP/1 + // will set MSG_COMPLETE to request state after upgrade response + // header is seen. + (!downstream->get_upgrade_request() || + (downstream->get_response_state() == Downstream::HEADER_COMPLETE && + !downstream->get_upgraded()))) { + + *data_flags |= NGHTTP2_DATA_FLAG_EOF; + } + + if(evbuffer_get_length(body) > 0) { downstream->reset_downstream_wtimer(); } else { downstream->disable_downstream_wtimer(); } + if(nread == 0 && (*data_flags & NGHTTP2_DATA_FLAG_EOF) == 0) { + downstream->disable_downstream_wtimer(); + + return NGHTTP2_ERR_DEFERRED; + } + return nread; } } // namespace diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index f0395ffe..c2a53c5a 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -781,11 +781,6 @@ int on_stream_close_callback // For tunneled connection, we have to submit RST_STREAM to // upstream *after* whole response body is sent. We just set // MSG_COMPLETE here. Upstream will take care of that. - if(LOG_ENABLED(INFO)) { - SSLOG(INFO, http2session) << "RST_STREAM against tunneled stream " - << "stream_id=" - << stream_id; - } downstream->get_upstream()->on_downstream_body_complete(downstream); downstream->set_response_state(Downstream::MSG_COMPLETE); } else if(error_code == NGHTTP2_NO_ERROR) { diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 586fcb90..ec02f131 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1102,7 +1102,9 @@ ssize_t downstream_data_read_callback(nghttp2_session *session, return NGHTTP2_ERR_CALLBACK_FAILURE; } - if(nread == 0 && + auto body_empty = evbuffer_get_length(body) == 0; + + if(body_empty && downstream->get_response_state() == Downstream::MSG_COMPLETE) { *data_flags |= NGHTTP2_DATA_FLAG_EOF; @@ -1122,10 +1124,10 @@ ssize_t downstream_data_read_callback(nghttp2_session *session, } } - if(evbuffer_get_length(body) > 0) { - downstream->reset_upstream_wtimer(); - } else { + if(body_empty) { downstream->disable_upstream_wtimer(); + } else { + downstream->reset_upstream_wtimer(); } if(nread > 0 && downstream->resume_read(SHRPX_NO_BUFFER, nread) != 0) { @@ -1206,7 +1208,7 @@ void Http2Upstream::add_pending_downstream void Http2Upstream::remove_downstream(Downstream *downstream) { - if(downstream->get_response_state() == Downstream::MSG_COMPLETE) { + if(downstream->accesslog_ready()) { handler_->write_accesslog(downstream); } @@ -1468,6 +1470,12 @@ void Http2Upstream::reset_timeouts() } void Http2Upstream::on_handler_delete() -{} +{ + for(auto& ent : downstream_queue_.get_active_downstreams()) { + if(ent.second->accesslog_ready()) { + handler_->write_accesslog(ent.second.get()); + } + } +} } // namespace shrpx diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 61befc7d..d820cd97 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -740,8 +740,7 @@ void HttpsUpstream::attach_downstream(std::unique_ptr downstream) void HttpsUpstream::delete_downstream() { - if(downstream_ && - downstream_->get_response_state() == Downstream::MSG_COMPLETE) { + if(downstream_ && downstream_->accesslog_ready()) { handler_->write_accesslog(downstream_.get()); } @@ -975,8 +974,7 @@ void HttpsUpstream::reset_timeouts() void HttpsUpstream::on_handler_delete() { - if(downstream_ && - downstream_->get_response_state() == Downstream::MSG_COMPLETE) { + if(downstream_ && downstream_->accesslog_ready()) { handler_->write_accesslog(downstream_.get()); } } diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index a7135614..97e23509 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -920,7 +920,7 @@ Downstream* SpdyUpstream::add_pending_downstream void SpdyUpstream::remove_downstream(Downstream *downstream) { - if(downstream->get_response_state() == Downstream::MSG_COMPLETE) { + if(downstream->accesslog_ready()) { handler_->write_accesslog(downstream); } @@ -1157,6 +1157,13 @@ void SpdyUpstream::reset_timeouts() } void SpdyUpstream::on_handler_delete() -{} +{ + for(auto& ent : downstream_queue_.get_active_downstreams()) { + if(ent.second->accesslog_ready()) { + handler_->write_accesslog(ent.second.get()); + } + } +} + } // namespace shrpx diff --git a/src/util.h b/src/util.h index 916fe84c..94f51f94 100644 --- a/src/util.h +++ b/src/util.h @@ -522,7 +522,8 @@ std::string format_iso8601(const T& tp) char buf[128]; auto nwrite = strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S", &tms); - snprintf(&buf[nwrite], sizeof(buf) - nwrite, ".%03ldZ", t.count() % 1000); + snprintf(&buf[nwrite], sizeof(buf) - nwrite, ".%03ldZ", + static_cast(t.count() % 1000)); return buf; } diff --git a/third-party/http-parser/README.md b/third-party/http-parser/README.md index a054dbe1..f9972ae5 100644 --- a/third-party/http-parser/README.md +++ b/third-party/http-parser/README.md @@ -75,7 +75,7 @@ if (parser->upgrade) { HTTP needs to know where the end of the stream is. For example, sometimes servers send responses without Content-Length and expect the client to consume input (for the body) until EOF. To tell http_parser about EOF, give -`0` as the forth parameter to `http_parser_execute()`. Callbacks and errors +`0` as the fourth parameter to `http_parser_execute()`. Callbacks and errors can still be encountered during an EOF, so one must still be prepared to receive them. @@ -110,7 +110,7 @@ followed by non-HTTP data. information the Web Socket protocol.) To support this, the parser will treat this as a normal HTTP message without a -body. Issuing both on_headers_complete and on_message_complete callbacks. However +body, issuing both on_headers_complete and on_message_complete callbacks. However http_parser_execute() will stop parsing at the end of the headers and return. The user is expected to check if `parser->upgrade` has been set to 1 after @@ -145,7 +145,7 @@ buffer to avoid copying memory around if this fits your application. Reading headers may be a tricky task if you read/parse headers partially. Basically, you need to remember whether last header callback was field or value -and apply following logic: +and apply the following logic: (on_header_field and on_header_value shortened to on_h_*) ------------------------ ------------ -------------------------------------------- diff --git a/third-party/http-parser/http_parser.c b/third-party/http-parser/http_parser.c index df1b696b..749d1bb6 100644 --- a/third-party/http-parser/http_parser.c +++ b/third-party/http-parser/http_parser.c @@ -925,7 +925,7 @@ size_t http_parser_execute (http_parser *parser, case 'G': parser->method = HTTP_GET; break; case 'H': parser->method = HTTP_HEAD; break; case 'L': parser->method = HTTP_LOCK; break; - case 'M': parser->method = HTTP_MKCOL; /* or MOVE, MKACTIVITY, MERGE, M-SEARCH */ break; + case 'M': parser->method = HTTP_MKCOL; /* or MOVE, MKACTIVITY, MERGE, M-SEARCH, MKCALENDAR */ break; case 'N': parser->method = HTTP_NOTIFY; break; case 'O': parser->method = HTTP_OPTIONS; break; case 'P': parser->method = HTTP_POST; @@ -977,6 +977,8 @@ size_t http_parser_execute (http_parser *parser, parser->method = HTTP_MSEARCH; } else if (parser->index == 2 && ch == 'A') { parser->method = HTTP_MKACTIVITY; + } else if (parser->index == 3 && ch == 'A') { + parser->method = HTTP_MKCALENDAR; } else { SET_ERRNO(HPE_INVALID_METHOD); goto error; @@ -1388,18 +1390,6 @@ size_t http_parser_execute (http_parser *parser, break; } - if (ch == CR) { - parser->state = s_header_almost_done; - CALLBACK_DATA(header_field); - break; - } - - if (ch == LF) { - parser->state = s_header_field_start; - CALLBACK_DATA(header_field); - break; - } - SET_ERRNO(HPE_INVALID_HEADER_TOKEN); goto error; } @@ -2142,7 +2132,7 @@ http_parser_parse_url(const char *buf, size_t buflen, int is_connect, u->port = u->field_set = 0; s = is_connect ? s_req_server_start : s_req_spaces_before_url; - uf = old_uf = UF_MAX; + old_uf = UF_MAX; for (p = buf; p < buf + buflen; p++) { s = parse_url_char(s, *p); diff --git a/third-party/http-parser/http_parser.h b/third-party/http-parser/http_parser.h index d150f26d..2f4ab9bd 100644 --- a/third-party/http-parser/http_parser.h +++ b/third-party/http-parser/http_parser.h @@ -76,7 +76,7 @@ typedef struct http_parser_settings http_parser_settings; * HEAD request which may contain 'Content-Length' or 'Transfer-Encoding: * chunked' headers that indicate the presence of a body. * - * http_data_cb does not return data chunks. It will be call arbitrarally + * http_data_cb does not return data chunks. It will be called arbitrarily * many times for each string. E.G. you might get 10 callbacks for "on_url" * each providing just a few characters more data. */ @@ -117,6 +117,8 @@ typedef int (*http_cb) (http_parser*); /* RFC-5789 */ \ XX(24, PATCH, PATCH) \ XX(25, PURGE, PURGE) \ + /* CalDAV */ \ + XX(26, MKCALENDAR, MKCALENDAR) \ enum http_method { @@ -278,13 +280,15 @@ struct http_parser_url { * unsigned major = (version >> 16) & 255; * unsigned minor = (version >> 8) & 255; * unsigned patch = version & 255; - * printf("http_parser v%u.%u.%u\n", major, minor, version); + * printf("http_parser v%u.%u.%u\n", major, minor, patch); */ unsigned long http_parser_version(void); void http_parser_init(http_parser *parser, enum http_parser_type type); +/* Executes the parser. Returns number of parsed bytes. Sets + * `parser->http_errno` on error. */ size_t http_parser_execute(http_parser *parser, const http_parser_settings *settings, const char *data, diff --git a/third-party/http-parser/test.c b/third-party/http-parser/test.c index 9799dc6d..f09e1063 100644 --- a/third-party/http-parser/test.c +++ b/third-party/http-parser/test.c @@ -2207,7 +2207,6 @@ print_error (const char *raw, size_t error_location) break; case '\n': - char_len = 2; fprintf(stderr, "\\n\n"); if (this_line) goto print; @@ -2910,15 +2909,11 @@ test_simple (const char *buf, enum http_errno err_expected) { parser_init(HTTP_REQUEST); - size_t parsed; - int pass; enum http_errno err; - parsed = parse(buf, strlen(buf)); - pass = (parsed == strlen(buf)); + parse(buf, strlen(buf)); err = HTTP_PARSER_ERRNO(parser); - parsed = parse(NULL, 0); - pass &= (parsed == 0); + parse(NULL, 0); parser_free(); @@ -3476,6 +3471,13 @@ main (void) test_simple(buf, HPE_INVALID_METHOD); } + // illegal header field name line folding + test_simple("GET / HTTP/1.1\r\n" + "name\r\n" + " : value\r\n" + "\r\n", + HPE_INVALID_HEADER_TOKEN); + const char *dumbfuck2 = "GET / HTTP/1.1\r\n" "X-SSL-Bullshit: -----BEGIN CERTIFICATE-----\r\n"