Merge branch 'master' into http2-debug-state-api

.travis.yml

@@ -43,6 +43,7 @@ before_script:
   - git clone https://github.com/tatsuhiro-t/spdylay.git
   - cd spdylay
   - autoreconf -i
+  # Don't use ASAN for spdylay since failmalloc does not work with it.
   - ./configure --disable-src --disable-examples
   - make check
   - export SPDYLAY_HOME=$PWD
@@ -50,15 +51,13 @@ before_script:
   # Now build nghttp2
   - if [ "$CI_BUILD" = "autotools" ]; then autoreconf -i; fi
   - git submodule update --init
-  - if [ "$CI_BUILD" = "autotools" ]; then ./configure --enable-werror --with-mruby --with-neverbleed LIBSPDYLAY_CFLAGS="-I$SPDYLAY_HOME/lib/includes" LIBSPDYLAY_LIBS="-L$SPDYLAY_HOME/lib/.libs -lspdylay"; fi
+  - if [ "$CI_BUILD" = "autotools" ]; then ./configure --enable-werror --with-mruby --with-neverbleed LIBSPDYLAY_CFLAGS="-I$SPDYLAY_HOME/lib/includes" LIBSPDYLAY_LIBS="-L$SPDYLAY_HOME/lib/.libs -lspdylay" CPPFLAGS=-fsanitize=address LDFLAGS=-fsanitize=address; fi
   - if [ "$CI_BUILD" = "cmake" ]; then cmake -DENABLE_WERROR=1 -DWITH_MRUBY=1 -DWITH_NEVERBLEED=1 -DSPDYLAY_INCLUDE_DIR="$SPDYLAY_HOME/lib/includes" -DSPDYLAY_LIBRARY="$SPDYLAY_HOME/lib/.libs/libspdylay.so"; fi
 script:
-  - make
   - make check
-  - cd integration-tests
   # As of April, 23, 2016, golang http2 build fails, probably because
   # the default go version is too old.
-
+  # - cd integration-tests
   # - export GOPATH="$PWD/integration-tests/golang"
   # - make itprep
   # - make it

README.rst

@@ -1370,7 +1370,7 @@ The extension module is called ``nghttp2``.
 determined by the ``configure`` script.  If the detected Python version is not
 what you expect, specify a path to Python executable in a ``PYTHON``
 variable as an argument to configure script (e.g., ``./configure
-PYTHON=/usr/bin/python3.4``).
+PYTHON=/usr/bin/python3.5``).
 
 The following example code illustrates basic usage of the HPACK compressor
 and decompressor in Python:

doc/sources/python-apiref.rst

@@ -13,7 +13,7 @@ The extension module is called ``nghttp2``.
 determined by configure script.  If the detected Python version is not
 what you expect, specify a path to Python executable in ``PYTHON``
 variable as an argument to configure script (e.g., ``./configure
-PYTHON=/usr/bin/python3.4``).
+PYTHON=/usr/bin/python3.5``).
 
 HPACK API
 ---------
@@ -136,15 +136,15 @@ HTTP/2 servers
 
 .. note::
 
-   We use :py:mod:`asyncio` for HTTP/2 server classes.  Therefore,
+   We use :py:mod:`asyncio` for HTTP/2 server classes, and ALPN.
-   Python 3.4 or later is required to use these objects.  To
+   Therefore, Python 3.5 or later is required to use these objects.
-   explicitly configure nghttp2 build to use Python 3.4, specify the
+   To explicitly configure nghttp2 build to use Python 3.5, specify
-   ``PYTHON`` variable to the path to Python 3.4 executable when
+   the ``PYTHON`` variable to the path to Python 3.5 executable when
    invoking configure script like this:
 
    .. code-block:: text
 
-       $ ./configure PYTHON=/usr/bin/python3.4
+       $ ./configure PYTHON=/usr/bin/python3.5
 
 .. py:class:: HTTP2Server(address, RequestHandlerClass, ssl=None)
 

lib/includes/nghttp2/nghttp2.h

@@ -860,8 +860,13 @@ typedef enum {
  * achieved by returning :enum:`NGHTTP2_ERR_DEFERRED` without reading
  * any data in this invocation.  The library removes DATA frame from
  * the outgoing queue temporarily.  To move back deferred DATA frame
- * to outgoing queue, call `nghttp2_session_resume_data()`.  In case
+ * to outgoing queue, call `nghttp2_session_resume_data()`.
- * of error, there are 2 choices. Returning
+ *
+ * If the application just wants to return from
+ * `nghttp2_session_send()` or `nghttp2_session_mem_send()` without
+ * sending anything, return :enum:`NGHTTP2_ERR_PAUSE`.
+ *
+ * In case of error, there are 2 choices. Returning
  * :enum:`NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE` will close the stream
  * by issuing RST_STREAM with :enum:`NGHTTP2_INTERNAL_ERROR`.  If a
  * different error code is desirable, use
@@ -1742,9 +1747,11 @@ typedef int (*nghttp2_on_header_callback2)(nghttp2_session *session,
  *
  * With this callback, application inspects the incoming invalid
  * field, and it also can reset stream from this callback by returning
- * :enum:`NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE`, or using
+ * :enum:`NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE`.  By default, the
- * `nghttp2_submit_rst_stream()` directly with the error code of
+ * error code is :enum:`NGHTTP2_INTERNAL_ERROR`.  To change the error
- * choice.
+ * code, call `nghttp2_submit_rst_stream()` with the error code of
+ * choice in addition to returning
+ * :enum:`NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE`.
  */
 typedef int (*nghttp2_on_invalid_header_callback)(
     nghttp2_session *session, const nghttp2_frame *frame, const uint8_t *name,
@@ -1772,9 +1779,11 @@ typedef int (*nghttp2_on_invalid_header_callback)(
  *
  * With this callback, application inspects the incoming invalid
  * field, and it also can reset stream from this callback by returning
- * :enum:`NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE`, or using
+ * :enum:`NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE`.  By default, the
- * `nghttp2_submit_rst_stream()` directly with the error code of
+ * error code is :enum:`NGHTTP2_INTERNAL_ERROR`.  To change the error
- * choice.
+ * code, call `nghttp2_submit_rst_stream()` with the error code of
+ * choice in addition to returning
+ * :enum:`NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE`.
  */
 typedef int (*nghttp2_on_invalid_header_callback2)(
     nghttp2_session *session, const nghttp2_frame *frame, nghttp2_rcbuf *name,

lib/nghttp2_session.c

@@ -2263,6 +2263,9 @@ static int session_prep_frame(nghttp2_session *session,
     rv = nghttp2_session_pack_data(session, &session->aob.framebufs,
                                    next_readmax, frame, &item->aux_data.data,
                                    stream);
+    if (rv == NGHTTP2_ERR_PAUSE) {
+      return rv;
+    }
     if (rv == NGHTTP2_ERR_DEFERRED) {
       rv = nghttp2_stream_defer_item(stream, NGHTTP2_STREAM_FLAG_DEFERRED_USER);
 
@@ -2918,6 +2921,9 @@ static ssize_t nghttp2_session_mem_send_internal(nghttp2_session *session,
       }
 
       rv = session_prep_frame(session, item);
+      if (rv == NGHTTP2_ERR_PAUSE) {
+        return 0;
+      }
       if (rv == NGHTTP2_ERR_DEFERRED) {
         DEBUGF(fprintf(stderr, "send: frame transmission deferred\n"));
         break;
@@ -7020,7 +7026,8 @@ int nghttp2_session_pack_data(nghttp2_session *session, nghttp2_bufs *bufs,
       &aux_data->data_prd.source, session->user_data);
 
   if (payloadlen == NGHTTP2_ERR_DEFERRED ||
-      payloadlen == NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE) {
+      payloadlen == NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE ||
+      payloadlen == NGHTTP2_ERR_PAUSE) {
     DEBUGF(fprintf(stderr, "send: DATA postponed due to %s\n",
                    nghttp2_strerror((int)payloadlen)));
 

python/nghttp2.pyx

@@ -260,6 +260,7 @@ try:
     import email.utils
     import datetime
     import time
+    import ssl as tls
     from urllib.parse import urlparse
 except ImportError:
     asyncio = None
@@ -294,6 +295,25 @@ def wrap_body(body):
         # string and flag.
         return body
 
+def negotiated_protocol(ssl_obj):
+    protocol = ssl_obj.selected_alpn_protocol()
+    if protocol:
+        logging.info('alpn, protocol:%s', protocol)
+        return protocol
+
+    protocol = ssl_obj.selected_npn_protocol()
+    if protocol:
+        logging.info('npn, protocol:%s', protocol)
+        return protocol
+
+    return None
+
+def set_application_protocol(ssl_ctx):
+    app_protos = [cnghttp2.NGHTTP2_PROTO_VERSION_ID.decode('utf-8')]
+    ssl_ctx.set_npn_protocols(app_protos)
+    if tls.HAS_ALPN:
+        ssl_ctx.set_alpn_protocols(app_protos)
+
 cdef _get_stream_user_data(cnghttp2.nghttp2_session *session,
                            int32_t stream_id):
     cdef void *stream_user_data
@@ -902,6 +922,8 @@ cdef class _HTTP2SessionCore(_HTTP2SessionCoreBase):
         return promised_handler
 
     def connection_lost(self):
+        self._stop_settings_timer()
+
         for handler in self.handlers:
             handler.on_close(cnghttp2.NGHTTP2_INTERNAL_ERROR)
         self.handlers = set()
@@ -1284,8 +1306,8 @@ if asyncio:
                 logging.info('failed to set tcp-nodelay: %s', str(e))
             ssl_ctx = self.transport.get_extra_info('sslcontext')
             if ssl_ctx:
-                protocol = sock.selected_npn_protocol()
+                ssl_obj = self.transport.get_extra_info('ssl_object')
-                logging.info('npn, protocol:%s', protocol)
+                protocol = negotiated_protocol(ssl_obj)
                 if protocol is None or protocol.encode('utf-8') != \
                    cnghttp2.NGHTTP2_PROTO_VERSION_ID:
                     self.transport.abort()
@@ -1346,8 +1368,7 @@ if asyncio:
             self.loop = asyncio.get_event_loop()
 
             if ssl:
-                ssl.set_npn_protocols([cnghttp2.NGHTTP2_PROTO_VERSION_ID\
+                set_application_protocol(ssl)
-                                       .decode('utf-8')])
 
             coro = self.loop.create_server(session_factory,
                                            host=address[0], port=address[1],
@@ -1516,8 +1537,8 @@ if asyncio:
             sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
             ssl_ctx = self.transport.get_extra_info('sslcontext')
             if ssl_ctx:
-                protocol = sock.selected_npn_protocol()
+                ssl_obj = self.transport.get_extra_info('ssl_object')
-                logging.info('npn, protocol:%s', protocol)
+                protocol = negotiated_protocol(ssl_obj)
                 if protocol is None or protocol.encode('utf-8') != \
                    cnghttp2.NGHTTP2_PROTO_VERSION_ID:
                     self.transport.abort()
@@ -1594,8 +1615,7 @@ if asyncio:
                 return self.session
 
             if ssl:
-                ssl.set_npn_protocols([cnghttp2.NGHTTP2_PROTO_VERSION_ID\
+                set_application_protocol(ssl)
-                                       .decode('utf-8')])
 
             self.loop = loop
             if not self.loop:

src/shrpx_http2_session.cc

@@ -904,6 +904,47 @@ int on_header_callback2(nghttp2_session *session, const nghttp2_frame *frame,
 }
 } // namespace
 
+namespace {
+int on_invalid_header_callback2(nghttp2_session *session,
+                                const nghttp2_frame *frame, nghttp2_rcbuf *name,
+                                nghttp2_rcbuf *value, uint8_t flags,
+                                void *user_data) {
+  auto http2session = static_cast<Http2Session *>(user_data);
+  auto sd = static_cast<StreamData *>(
+      nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
+  if (!sd || !sd->dconn) {
+    return 0;
+  }
+  auto downstream = sd->dconn->get_downstream();
+  if (!downstream) {
+    return 0;
+  }
+
+  int32_t stream_id;
+
+  if (frame->hd.type == NGHTTP2_PUSH_PROMISE) {
+    stream_id = frame->push_promise.promised_stream_id;
+  } else {
+    stream_id = frame->hd.stream_id;
+  }
+
+  if (LOG_ENABLED(INFO)) {
+    auto namebuf = nghttp2_rcbuf_get_buf(name);
+    auto valuebuf = nghttp2_rcbuf_get_buf(value);
+
+    SSLOG(INFO, http2session)
+        << "Invalid header field for stream_id=" << stream_id
+        << " in frame type=" << static_cast<uint32_t>(frame->hd.type)
+        << ": name=[" << StringRef{namebuf.base, namebuf.len} << "], value=["
+        << StringRef{valuebuf.base, valuebuf.len} << "]";
+  }
+
+  http2session->submit_rst_stream(stream_id, NGHTTP2_PROTOCOL_ERROR);
+
+  return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
+}
+} // namespace
+
 namespace {
 int on_begin_headers_callback(nghttp2_session *session,
                               const nghttp2_frame *frame, void *user_data) {
@@ -1486,6 +1527,9 @@ nghttp2_session_callbacks *create_http2_downstream_callbacks() {
   nghttp2_session_callbacks_set_on_header_callback2(callbacks,
                                                     on_header_callback2);
 
+  nghttp2_session_callbacks_set_on_invalid_header_callback2(
+      callbacks, on_invalid_header_callback2);
+
   nghttp2_session_callbacks_set_on_begin_headers_callback(
       callbacks, on_begin_headers_callback);
 

src/shrpx_http2_upstream.cc

@@ -227,6 +227,34 @@ int on_header_callback2(nghttp2_session *session, const nghttp2_frame *frame,
 }
 } // namespace
 
+namespace {
+int on_invalid_header_callback2(nghttp2_session *session,
+                                const nghttp2_frame *frame, nghttp2_rcbuf *name,
+                                nghttp2_rcbuf *value, uint8_t flags,
+                                void *user_data) {
+  auto upstream = static_cast<Http2Upstream *>(user_data);
+  auto downstream = static_cast<Downstream *>(
+      nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
+  if (!downstream) {
+    return 0;
+  }
+
+  if (LOG_ENABLED(INFO)) {
+    auto namebuf = nghttp2_rcbuf_get_buf(name);
+    auto valuebuf = nghttp2_rcbuf_get_buf(value);
+
+    ULOG(INFO, upstream) << "Invalid header field for stream_id="
+                         << frame->hd.stream_id << ": name=["
+                         << StringRef{namebuf.base, namebuf.len} << "], value=["
+                         << StringRef{valuebuf.base, valuebuf.len} << "]";
+  }
+
+  upstream->rst_stream(downstream, NGHTTP2_PROTOCOL_ERROR);
+
+  return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
+}
+} // namespace
+
 namespace {
 int on_begin_headers_callback(nghttp2_session *session,
                               const nghttp2_frame *frame, void *user_data) {
@@ -852,6 +880,9 @@ nghttp2_session_callbacks *create_http2_upstream_callbacks() {
   nghttp2_session_callbacks_set_on_header_callback2(callbacks,
                                                     on_header_callback2);
 
+  nghttp2_session_callbacks_set_on_invalid_header_callback2(
+      callbacks, on_invalid_header_callback2);
+
   nghttp2_session_callbacks_set_on_begin_headers_callback(
       callbacks, on_begin_headers_callback);
 

src/shrpx_ssl.cc

@@ -72,6 +72,14 @@ namespace shrpx {
 
 namespace ssl {
 
+#if !OPENSSL_101_API
+namespace {
+const unsigned char *ASN1_STRING_get0_data(ASN1_STRING *x) {
+  return ASN1_STRING_data(x);
+}
+} // namespace
+#endif // !OPENSSL_101_API
+
 namespace {
 int next_proto_cb(SSL *s, const unsigned char **data, unsigned int *len,
                   void *arg) {
@@ -1015,7 +1023,7 @@ int verify_hostname(X509 *cert, const StringRef &hostname,
         continue;
       }
 
-      auto name = reinterpret_cast<char *>(ASN1_STRING_data(altname->d.ia5));
+      auto name = ASN1_STRING_get0_data(altname->d.ia5);
       if (!name) {
         continue;
       }
@@ -1235,7 +1243,7 @@ int cert_lookup_tree_add_cert_from_x509(CertLookupTree *lt, size_t idx,
         continue;
       }
 
-      auto name = reinterpret_cast<char *>(ASN1_STRING_data(altname->d.ia5));
+      auto name = ASN1_STRING_get0_data(altname->d.ia5);
       if (!name) {
         continue;
       }

src/ssl_compat.h

@@ -24,7 +24,9 @@
  */
 #ifndef OPENSSL_COMPAT_H
 
+#include <openssl/opensslv.h>
+
 #define OPENSSL_101_API                                                        \
-  (!defined(LIBRESSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER >= 0x10100005L)
+  (!defined(LIBRESSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER >= 0x1010000fL)
 
 #endif // OPENSSL_COMPAT_H

tests/main.c

@@ -310,6 +310,8 @@ int main(int argc _U_, char *argv[] _U_) {
                    test_nghttp2_session_cancel_from_before_frame_send) ||
       !CU_add_test(pSuite, "session_removed_closed_stream",
                    test_nghttp2_session_removed_closed_stream) ||
+      !CU_add_test(pSuite, "session_pause_data",
+                   test_nghttp2_session_pause_data) ||
       !CU_add_test(pSuite, "http_mandatory_headers",
                    test_nghttp2_http_mandatory_headers) ||
       !CU_add_test(pSuite, "http_content_length",

tests/nghttp2_session_test.c

@@ -77,6 +77,7 @@ typedef struct {
   size_t padlen;
   int begin_frame_cb_called;
   nghttp2_buf scratchbuf;
+  size_t data_source_read_cb_paused;
 } my_user_data;
 
 static const nghttp2_nv reqnv[] = {
@@ -10021,6 +10022,53 @@ void test_nghttp2_session_removed_closed_stream(void) {
   nghttp2_bufs_free(&bufs);
 }
 
+static ssize_t pause_once_data_source_read_callback(
+    nghttp2_session *session, int32_t stream_id, uint8_t *buf, size_t len,
+    uint32_t *data_flags, nghttp2_data_source *source, void *user_data) {
+  my_user_data *ud = user_data;
+  if (ud->data_source_read_cb_paused == 0) {
+    ++ud->data_source_read_cb_paused;
+    return NGHTTP2_ERR_PAUSE;
+  }
+
+  return fixed_length_data_source_read_callback(session, stream_id, buf, len,
+                                                data_flags, source, user_data);
+}
+
+void test_nghttp2_session_pause_data(void) {
+  nghttp2_session *session;
+  nghttp2_session_callbacks callbacks;
+  nghttp2_data_provider data_prd;
+  my_user_data ud;
+
+  memset(&callbacks, 0, sizeof(nghttp2_session_callbacks));
+  callbacks.send_callback = null_send_callback;
+  callbacks.on_frame_send_callback = on_frame_send_callback;
+
+  data_prd.read_callback = pause_once_data_source_read_callback;
+  ud.data_source_length = NGHTTP2_DATA_PAYLOADLEN;
+
+  nghttp2_session_server_new(&session, &callbacks, &ud);
+
+  open_recv_stream(session, 1);
+
+  CU_ASSERT(
+      0 == nghttp2_submit_data(session, NGHTTP2_FLAG_END_STREAM, 1, &data_prd));
+
+  ud.frame_send_cb_called = 0;
+  ud.data_source_read_cb_paused = 0;
+
+  CU_ASSERT(0 == nghttp2_session_send(session));
+  CU_ASSERT(0 == ud.frame_send_cb_called);
+  CU_ASSERT(NULL == session->aob.item);
+  CU_ASSERT(0 == nghttp2_session_send(session));
+  CU_ASSERT(1 == ud.frame_send_cb_called);
+  CU_ASSERT(NGHTTP2_DATA == ud.sent_frame_type);
+  CU_ASSERT(NULL == nghttp2_session_get_next_ob_item(session));
+
+  nghttp2_session_del(session);
+}
+
 static void check_nghttp2_http_recv_headers_fail(
     nghttp2_session *session, nghttp2_hd_deflater *deflater, int32_t stream_id,
     int stream_state, const nghttp2_nv *nva, size_t nvlen) {

tests/nghttp2_session_test.h

@@ -153,6 +153,7 @@ void test_nghttp2_session_repeated_priority_submission(void);
 void test_nghttp2_session_set_local_window_size(void);
 void test_nghttp2_session_cancel_from_before_frame_send(void);
 void test_nghttp2_session_removed_closed_stream(void);
+void test_nghttp2_session_pause_data(void);
 void test_nghttp2_http_mandatory_headers(void);
 void test_nghttp2_http_content_length(void);
 void test_nghttp2_http_content_length_mismatch(void);