diff --git a/.travis.yml b/.travis.yml index 09c65c3c..9d8395da 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,6 +43,7 @@ before_script: - git clone https://github.com/tatsuhiro-t/spdylay.git - cd spdylay - autoreconf -i + # Don't use ASAN for spdylay since failmalloc does not work with it. - ./configure --disable-src --disable-examples - make check - export SPDYLAY_HOME=$PWD @@ -50,15 +51,13 @@ before_script: # Now build nghttp2 - if [ "$CI_BUILD" = "autotools" ]; then autoreconf -i; fi - git submodule update --init - - if [ "$CI_BUILD" = "autotools" ]; then ./configure --enable-werror --with-mruby --with-neverbleed LIBSPDYLAY_CFLAGS="-I$SPDYLAY_HOME/lib/includes" LIBSPDYLAY_LIBS="-L$SPDYLAY_HOME/lib/.libs -lspdylay"; fi + - if [ "$CI_BUILD" = "autotools" ]; then ./configure --enable-werror --with-mruby --with-neverbleed LIBSPDYLAY_CFLAGS="-I$SPDYLAY_HOME/lib/includes" LIBSPDYLAY_LIBS="-L$SPDYLAY_HOME/lib/.libs -lspdylay" CPPFLAGS=-fsanitize=address LDFLAGS=-fsanitize=address; fi - if [ "$CI_BUILD" = "cmake" ]; then cmake -DENABLE_WERROR=1 -DWITH_MRUBY=1 -DWITH_NEVERBLEED=1 -DSPDYLAY_INCLUDE_DIR="$SPDYLAY_HOME/lib/includes" -DSPDYLAY_LIBRARY="$SPDYLAY_HOME/lib/.libs/libspdylay.so"; fi script: - - make - make check - - cd integration-tests # As of April, 23, 2016, golang http2 build fails, probably because # the default go version is too old. - + # - cd integration-tests # - export GOPATH="$PWD/integration-tests/golang" # - make itprep # - make it diff --git a/README.rst b/README.rst index 6eedcf22..edcb702b 100644 --- a/README.rst +++ b/README.rst @@ -1370,7 +1370,7 @@ The extension module is called ``nghttp2``. determined by the ``configure`` script. If the detected Python version is not what you expect, specify a path to Python executable in a ``PYTHON`` variable as an argument to configure script (e.g., ``./configure -PYTHON=/usr/bin/python3.4``). +PYTHON=/usr/bin/python3.5``). The following example code illustrates basic usage of the HPACK compressor and decompressor in Python: diff --git a/doc/sources/python-apiref.rst b/doc/sources/python-apiref.rst index d0a0b331..d64b43cc 100644 --- a/doc/sources/python-apiref.rst +++ b/doc/sources/python-apiref.rst @@ -13,7 +13,7 @@ The extension module is called ``nghttp2``. determined by configure script. If the detected Python version is not what you expect, specify a path to Python executable in ``PYTHON`` variable as an argument to configure script (e.g., ``./configure -PYTHON=/usr/bin/python3.4``). +PYTHON=/usr/bin/python3.5``). HPACK API --------- @@ -136,15 +136,15 @@ HTTP/2 servers .. note:: - We use :py:mod:`asyncio` for HTTP/2 server classes. Therefore, - Python 3.4 or later is required to use these objects. To - explicitly configure nghttp2 build to use Python 3.4, specify the - ``PYTHON`` variable to the path to Python 3.4 executable when + We use :py:mod:`asyncio` for HTTP/2 server classes, and ALPN. + Therefore, Python 3.5 or later is required to use these objects. + To explicitly configure nghttp2 build to use Python 3.5, specify + the ``PYTHON`` variable to the path to Python 3.5 executable when invoking configure script like this: .. code-block:: text - $ ./configure PYTHON=/usr/bin/python3.4 + $ ./configure PYTHON=/usr/bin/python3.5 .. py:class:: HTTP2Server(address, RequestHandlerClass, ssl=None) diff --git a/lib/includes/nghttp2/nghttp2.h b/lib/includes/nghttp2/nghttp2.h index bc35e40a..59615fb4 100644 --- a/lib/includes/nghttp2/nghttp2.h +++ b/lib/includes/nghttp2/nghttp2.h @@ -860,8 +860,13 @@ typedef enum { * achieved by returning :enum:`NGHTTP2_ERR_DEFERRED` without reading * any data in this invocation. The library removes DATA frame from * the outgoing queue temporarily. To move back deferred DATA frame - * to outgoing queue, call `nghttp2_session_resume_data()`. In case - * of error, there are 2 choices. Returning + * to outgoing queue, call `nghttp2_session_resume_data()`. + * + * If the application just wants to return from + * `nghttp2_session_send()` or `nghttp2_session_mem_send()` without + * sending anything, return :enum:`NGHTTP2_ERR_PAUSE`. + * + * In case of error, there are 2 choices. Returning * :enum:`NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE` will close the stream * by issuing RST_STREAM with :enum:`NGHTTP2_INTERNAL_ERROR`. If a * different error code is desirable, use @@ -1742,9 +1747,11 @@ typedef int (*nghttp2_on_header_callback2)(nghttp2_session *session, * * With this callback, application inspects the incoming invalid * field, and it also can reset stream from this callback by returning - * :enum:`NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE`, or using - * `nghttp2_submit_rst_stream()` directly with the error code of - * choice. + * :enum:`NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE`. By default, the + * error code is :enum:`NGHTTP2_INTERNAL_ERROR`. To change the error + * code, call `nghttp2_submit_rst_stream()` with the error code of + * choice in addition to returning + * :enum:`NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE`. */ typedef int (*nghttp2_on_invalid_header_callback)( nghttp2_session *session, const nghttp2_frame *frame, const uint8_t *name, @@ -1772,9 +1779,11 @@ typedef int (*nghttp2_on_invalid_header_callback)( * * With this callback, application inspects the incoming invalid * field, and it also can reset stream from this callback by returning - * :enum:`NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE`, or using - * `nghttp2_submit_rst_stream()` directly with the error code of - * choice. + * :enum:`NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE`. By default, the + * error code is :enum:`NGHTTP2_INTERNAL_ERROR`. To change the error + * code, call `nghttp2_submit_rst_stream()` with the error code of + * choice in addition to returning + * :enum:`NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE`. */ typedef int (*nghttp2_on_invalid_header_callback2)( nghttp2_session *session, const nghttp2_frame *frame, nghttp2_rcbuf *name, diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 2442db39..cfa71f8a 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -2263,6 +2263,9 @@ static int session_prep_frame(nghttp2_session *session, rv = nghttp2_session_pack_data(session, &session->aob.framebufs, next_readmax, frame, &item->aux_data.data, stream); + if (rv == NGHTTP2_ERR_PAUSE) { + return rv; + } if (rv == NGHTTP2_ERR_DEFERRED) { rv = nghttp2_stream_defer_item(stream, NGHTTP2_STREAM_FLAG_DEFERRED_USER); @@ -2918,6 +2921,9 @@ static ssize_t nghttp2_session_mem_send_internal(nghttp2_session *session, } rv = session_prep_frame(session, item); + if (rv == NGHTTP2_ERR_PAUSE) { + return 0; + } if (rv == NGHTTP2_ERR_DEFERRED) { DEBUGF(fprintf(stderr, "send: frame transmission deferred\n")); break; @@ -7020,7 +7026,8 @@ int nghttp2_session_pack_data(nghttp2_session *session, nghttp2_bufs *bufs, &aux_data->data_prd.source, session->user_data); if (payloadlen == NGHTTP2_ERR_DEFERRED || - payloadlen == NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE) { + payloadlen == NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE || + payloadlen == NGHTTP2_ERR_PAUSE) { DEBUGF(fprintf(stderr, "send: DATA postponed due to %s\n", nghttp2_strerror((int)payloadlen))); diff --git a/python/nghttp2.pyx b/python/nghttp2.pyx index 5def868e..7821ff14 100644 --- a/python/nghttp2.pyx +++ b/python/nghttp2.pyx @@ -260,6 +260,7 @@ try: import email.utils import datetime import time + import ssl as tls from urllib.parse import urlparse except ImportError: asyncio = None @@ -294,6 +295,25 @@ def wrap_body(body): # string and flag. return body +def negotiated_protocol(ssl_obj): + protocol = ssl_obj.selected_alpn_protocol() + if protocol: + logging.info('alpn, protocol:%s', protocol) + return protocol + + protocol = ssl_obj.selected_npn_protocol() + if protocol: + logging.info('npn, protocol:%s', protocol) + return protocol + + return None + +def set_application_protocol(ssl_ctx): + app_protos = [cnghttp2.NGHTTP2_PROTO_VERSION_ID.decode('utf-8')] + ssl_ctx.set_npn_protocols(app_protos) + if tls.HAS_ALPN: + ssl_ctx.set_alpn_protocols(app_protos) + cdef _get_stream_user_data(cnghttp2.nghttp2_session *session, int32_t stream_id): cdef void *stream_user_data @@ -902,6 +922,8 @@ cdef class _HTTP2SessionCore(_HTTP2SessionCoreBase): return promised_handler def connection_lost(self): + self._stop_settings_timer() + for handler in self.handlers: handler.on_close(cnghttp2.NGHTTP2_INTERNAL_ERROR) self.handlers = set() @@ -1284,8 +1306,8 @@ if asyncio: logging.info('failed to set tcp-nodelay: %s', str(e)) ssl_ctx = self.transport.get_extra_info('sslcontext') if ssl_ctx: - protocol = sock.selected_npn_protocol() - logging.info('npn, protocol:%s', protocol) + ssl_obj = self.transport.get_extra_info('ssl_object') + protocol = negotiated_protocol(ssl_obj) if protocol is None or protocol.encode('utf-8') != \ cnghttp2.NGHTTP2_PROTO_VERSION_ID: self.transport.abort() @@ -1346,8 +1368,7 @@ if asyncio: self.loop = asyncio.get_event_loop() if ssl: - ssl.set_npn_protocols([cnghttp2.NGHTTP2_PROTO_VERSION_ID\ - .decode('utf-8')]) + set_application_protocol(ssl) coro = self.loop.create_server(session_factory, host=address[0], port=address[1], @@ -1516,8 +1537,8 @@ if asyncio: sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) ssl_ctx = self.transport.get_extra_info('sslcontext') if ssl_ctx: - protocol = sock.selected_npn_protocol() - logging.info('npn, protocol:%s', protocol) + ssl_obj = self.transport.get_extra_info('ssl_object') + protocol = negotiated_protocol(ssl_obj) if protocol is None or protocol.encode('utf-8') != \ cnghttp2.NGHTTP2_PROTO_VERSION_ID: self.transport.abort() @@ -1594,8 +1615,7 @@ if asyncio: return self.session if ssl: - ssl.set_npn_protocols([cnghttp2.NGHTTP2_PROTO_VERSION_ID\ - .decode('utf-8')]) + set_application_protocol(ssl) self.loop = loop if not self.loop: diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 624289bb..20c6ff1f 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -904,6 +904,47 @@ int on_header_callback2(nghttp2_session *session, const nghttp2_frame *frame, } } // namespace +namespace { +int on_invalid_header_callback2(nghttp2_session *session, + const nghttp2_frame *frame, nghttp2_rcbuf *name, + nghttp2_rcbuf *value, uint8_t flags, + void *user_data) { + auto http2session = static_cast(user_data); + auto sd = static_cast( + nghttp2_session_get_stream_user_data(session, frame->hd.stream_id)); + if (!sd || !sd->dconn) { + return 0; + } + auto downstream = sd->dconn->get_downstream(); + if (!downstream) { + return 0; + } + + int32_t stream_id; + + if (frame->hd.type == NGHTTP2_PUSH_PROMISE) { + stream_id = frame->push_promise.promised_stream_id; + } else { + stream_id = frame->hd.stream_id; + } + + if (LOG_ENABLED(INFO)) { + auto namebuf = nghttp2_rcbuf_get_buf(name); + auto valuebuf = nghttp2_rcbuf_get_buf(value); + + SSLOG(INFO, http2session) + << "Invalid header field for stream_id=" << stream_id + << " in frame type=" << static_cast(frame->hd.type) + << ": name=[" << StringRef{namebuf.base, namebuf.len} << "], value=[" + << StringRef{valuebuf.base, valuebuf.len} << "]"; + } + + http2session->submit_rst_stream(stream_id, NGHTTP2_PROTOCOL_ERROR); + + return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; +} +} // namespace + namespace { int on_begin_headers_callback(nghttp2_session *session, const nghttp2_frame *frame, void *user_data) { @@ -1486,6 +1527,9 @@ nghttp2_session_callbacks *create_http2_downstream_callbacks() { nghttp2_session_callbacks_set_on_header_callback2(callbacks, on_header_callback2); + nghttp2_session_callbacks_set_on_invalid_header_callback2( + callbacks, on_invalid_header_callback2); + nghttp2_session_callbacks_set_on_begin_headers_callback( callbacks, on_begin_headers_callback); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 202d46ae..42ec9561 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -227,6 +227,34 @@ int on_header_callback2(nghttp2_session *session, const nghttp2_frame *frame, } } // namespace +namespace { +int on_invalid_header_callback2(nghttp2_session *session, + const nghttp2_frame *frame, nghttp2_rcbuf *name, + nghttp2_rcbuf *value, uint8_t flags, + void *user_data) { + auto upstream = static_cast(user_data); + auto downstream = static_cast( + nghttp2_session_get_stream_user_data(session, frame->hd.stream_id)); + if (!downstream) { + return 0; + } + + if (LOG_ENABLED(INFO)) { + auto namebuf = nghttp2_rcbuf_get_buf(name); + auto valuebuf = nghttp2_rcbuf_get_buf(value); + + ULOG(INFO, upstream) << "Invalid header field for stream_id=" + << frame->hd.stream_id << ": name=[" + << StringRef{namebuf.base, namebuf.len} << "], value=[" + << StringRef{valuebuf.base, valuebuf.len} << "]"; + } + + upstream->rst_stream(downstream, NGHTTP2_PROTOCOL_ERROR); + + return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; +} +} // namespace + namespace { int on_begin_headers_callback(nghttp2_session *session, const nghttp2_frame *frame, void *user_data) { @@ -852,6 +880,9 @@ nghttp2_session_callbacks *create_http2_upstream_callbacks() { nghttp2_session_callbacks_set_on_header_callback2(callbacks, on_header_callback2); + nghttp2_session_callbacks_set_on_invalid_header_callback2( + callbacks, on_invalid_header_callback2); + nghttp2_session_callbacks_set_on_begin_headers_callback( callbacks, on_begin_headers_callback); diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 469338a3..80e1cfbd 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -72,6 +72,14 @@ namespace shrpx { namespace ssl { +#if !OPENSSL_101_API +namespace { +const unsigned char *ASN1_STRING_get0_data(ASN1_STRING *x) { + return ASN1_STRING_data(x); +} +} // namespace +#endif // !OPENSSL_101_API + namespace { int next_proto_cb(SSL *s, const unsigned char **data, unsigned int *len, void *arg) { @@ -1015,7 +1023,7 @@ int verify_hostname(X509 *cert, const StringRef &hostname, continue; } - auto name = reinterpret_cast(ASN1_STRING_data(altname->d.ia5)); + auto name = ASN1_STRING_get0_data(altname->d.ia5); if (!name) { continue; } @@ -1235,7 +1243,7 @@ int cert_lookup_tree_add_cert_from_x509(CertLookupTree *lt, size_t idx, continue; } - auto name = reinterpret_cast(ASN1_STRING_data(altname->d.ia5)); + auto name = ASN1_STRING_get0_data(altname->d.ia5); if (!name) { continue; } diff --git a/src/ssl_compat.h b/src/ssl_compat.h index 4b015624..c81b8c3b 100644 --- a/src/ssl_compat.h +++ b/src/ssl_compat.h @@ -24,7 +24,9 @@ */ #ifndef OPENSSL_COMPAT_H +#include + #define OPENSSL_101_API \ - (!defined(LIBRESSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER >= 0x10100005L) + (!defined(LIBRESSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER >= 0x1010000fL) #endif // OPENSSL_COMPAT_H diff --git a/tests/main.c b/tests/main.c index e4f4900f..20ff40ae 100644 --- a/tests/main.c +++ b/tests/main.c @@ -310,6 +310,8 @@ int main(int argc _U_, char *argv[] _U_) { test_nghttp2_session_cancel_from_before_frame_send) || !CU_add_test(pSuite, "session_removed_closed_stream", test_nghttp2_session_removed_closed_stream) || + !CU_add_test(pSuite, "session_pause_data", + test_nghttp2_session_pause_data) || !CU_add_test(pSuite, "http_mandatory_headers", test_nghttp2_http_mandatory_headers) || !CU_add_test(pSuite, "http_content_length", diff --git a/tests/nghttp2_session_test.c b/tests/nghttp2_session_test.c index 8934f7f2..31d47d1a 100644 --- a/tests/nghttp2_session_test.c +++ b/tests/nghttp2_session_test.c @@ -77,6 +77,7 @@ typedef struct { size_t padlen; int begin_frame_cb_called; nghttp2_buf scratchbuf; + size_t data_source_read_cb_paused; } my_user_data; static const nghttp2_nv reqnv[] = { @@ -10021,6 +10022,53 @@ void test_nghttp2_session_removed_closed_stream(void) { nghttp2_bufs_free(&bufs); } +static ssize_t pause_once_data_source_read_callback( + nghttp2_session *session, int32_t stream_id, uint8_t *buf, size_t len, + uint32_t *data_flags, nghttp2_data_source *source, void *user_data) { + my_user_data *ud = user_data; + if (ud->data_source_read_cb_paused == 0) { + ++ud->data_source_read_cb_paused; + return NGHTTP2_ERR_PAUSE; + } + + return fixed_length_data_source_read_callback(session, stream_id, buf, len, + data_flags, source, user_data); +} + +void test_nghttp2_session_pause_data(void) { + nghttp2_session *session; + nghttp2_session_callbacks callbacks; + nghttp2_data_provider data_prd; + my_user_data ud; + + memset(&callbacks, 0, sizeof(nghttp2_session_callbacks)); + callbacks.send_callback = null_send_callback; + callbacks.on_frame_send_callback = on_frame_send_callback; + + data_prd.read_callback = pause_once_data_source_read_callback; + ud.data_source_length = NGHTTP2_DATA_PAYLOADLEN; + + nghttp2_session_server_new(&session, &callbacks, &ud); + + open_recv_stream(session, 1); + + CU_ASSERT( + 0 == nghttp2_submit_data(session, NGHTTP2_FLAG_END_STREAM, 1, &data_prd)); + + ud.frame_send_cb_called = 0; + ud.data_source_read_cb_paused = 0; + + CU_ASSERT(0 == nghttp2_session_send(session)); + CU_ASSERT(0 == ud.frame_send_cb_called); + CU_ASSERT(NULL == session->aob.item); + CU_ASSERT(0 == nghttp2_session_send(session)); + CU_ASSERT(1 == ud.frame_send_cb_called); + CU_ASSERT(NGHTTP2_DATA == ud.sent_frame_type); + CU_ASSERT(NULL == nghttp2_session_get_next_ob_item(session)); + + nghttp2_session_del(session); +} + static void check_nghttp2_http_recv_headers_fail( nghttp2_session *session, nghttp2_hd_deflater *deflater, int32_t stream_id, int stream_state, const nghttp2_nv *nva, size_t nvlen) { diff --git a/tests/nghttp2_session_test.h b/tests/nghttp2_session_test.h index 5e81b215..b37713cb 100644 --- a/tests/nghttp2_session_test.h +++ b/tests/nghttp2_session_test.h @@ -153,6 +153,7 @@ void test_nghttp2_session_repeated_priority_submission(void); void test_nghttp2_session_set_local_window_size(void); void test_nghttp2_session_cancel_from_before_frame_send(void); void test_nghttp2_session_removed_closed_stream(void); +void test_nghttp2_session_pause_data(void); void test_nghttp2_http_mandatory_headers(void); void test_nghttp2_http_content_length(void); void test_nghttp2_http_content_length_mismatch(void);