Return fatal error if flooding is detected to close session immediately

This change adds new return error code from nghttp2_session_mem_recv
and nghttp2_session_recv functions, namely NGHTTP2_ERR_FLOODED.  It is
fatal error, and is returned when flooding was detected.

lib/includes/nghttp2/nghttp2.h

@@ -399,7 +399,16 @@ typedef enum {
    * Invalid client magic (see :macro:`NGHTTP2_CLIENT_MAGIC`) was
    * received and further processing is not possible.
    */
-  NGHTTP2_ERR_BAD_CLIENT_MAGIC = -903
+  NGHTTP2_ERR_BAD_CLIENT_MAGIC = -903,
+  /**
+   * Possible flooding by peer was detected in this HTTP/2 session.
+   * Flooding is measured by how many PING and SETTINGS frames with
+   * ACK flag set are queued for transmission.  These frames are
+   * response for the peer initiated frames, and peer can cause memory
+   * exhaustion on server side to send these frames forever and does
+   * not read network.
+   */
+  NGHTTP2_ERR_FLOODED = -904
 } nghttp2_error;
 
 /**
@@ -2366,6 +2375,9 @@ NGHTTP2_EXTERN ssize_t nghttp2_session_mem_send(nghttp2_session *session,
  *     when |session| was configured as server and
  *     `nghttp2_option_set_no_recv_client_magic()` is not used with
  *     nonzero value.
+ * :enum:`NGHTTP2_ERR_FLOODED`
+ *     Flooding was detected in this HTTP/2 session, and it must be
+ *     closed.  This is most likely caused by misbehaviour of peer.
  */
 NGHTTP2_EXTERN int nghttp2_session_recv(nghttp2_session *session);
 
@@ -2402,6 +2414,9 @@ NGHTTP2_EXTERN int nghttp2_session_recv(nghttp2_session *session);
  *     when |session| was configured as server and
  *     `nghttp2_option_set_no_recv_client_magic()` is not used with
  *     nonzero value.
+ * :enum:`NGHTTP2_ERR_FLOODED`
+ *     Flooding was detected in this HTTP/2 session, and it must be
+ *     closed.  This is most likely caused by misbehaviour of peer.
  */
 NGHTTP2_EXTERN ssize_t nghttp2_session_mem_recv(nghttp2_session *session,
                                                 const uint8_t *in,

lib/nghttp2_helper.c

@@ -294,6 +294,9 @@ const char *nghttp2_strerror(int error_code) {
     return "The user callback function failed";
   case NGHTTP2_ERR_BAD_CLIENT_MAGIC:
     return "Received bad client magic byte string";
+  case NGHTTP2_ERR_FLOODED:
+    return "Flooding was detected in this HTTP/2 session, and it must be "
+           "closed";
   default:
     return "Unknown error code";
   }

lib/nghttp2_int.h

@@ -52,8 +52,7 @@ typedef enum {
    * Invalid HTTP header field was received but it can be treated as
    * if it was not received because of compatibility reasons.
    */
-  NGHTTP2_ERR_IGN_HTTP_HEADER = -105,
+  NGHTTP2_ERR_IGN_HTTP_HEADER = -105
-  NGHTTP2_ERR_FLOODING_DETECTED = -106
 } nghttp2_internal_error;
 
 #endif /* NGHTTP2_INT_H */

lib/nghttp2_session.c

@@ -141,15 +141,6 @@ static int session_detect_idle_stream(nghttp2_session *session,
   return 0;
 }
 
-static void session_on_flooding_detected(nghttp2_session *session) {
-  /* If we found flooding, we might not send GOAWAY since peer might
-     not read at all.  So we just set these flags to pretend that
-     GOAWAY is sent, so that nghttp2_session_want_read() and
-     nghttp2_session_want_write() return 0. */
-  session->goaway_flags |= NGHTTP2_GOAWAY_TERM_ON_SEND |
-                           NGHTTP2_GOAWAY_TERM_SENT | NGHTTP2_GOAWAY_SENT;
-}
-
 static int session_terminate_session(nghttp2_session *session,
                                      int32_t last_stream_id,
                                      uint32_t error_code, const char *reason) {
@@ -5930,8 +5921,7 @@ int nghttp2_session_add_ping(nghttp2_session *session, uint8_t flags,
 
   if ((flags & NGHTTP2_FLAG_ACK) &&
       session->obq_flood_counter_ >= NGHTTP2_MAX_OBQ_FLOOD_ITEM) {
-    session_on_flooding_detected(session);
+    return NGHTTP2_ERR_FLOODED;
-    return NGHTTP2_ERR_FLOODING_DETECTED;
   }
 
   item = nghttp2_mem_malloc(mem, sizeof(nghttp2_outbound_item));
@@ -6081,8 +6071,7 @@ int nghttp2_session_add_settings(nghttp2_session *session, uint8_t flags,
     }
 
     if (session->obq_flood_counter_ >= NGHTTP2_MAX_OBQ_FLOOD_ITEM) {
-      session_on_flooding_detected(session);
+      return NGHTTP2_ERR_FLOODED;
-      return NGHTTP2_ERR_FLOODING_DETECTED;
     }
   }
 

lib/nghttp2_session.h

@@ -367,7 +367,7 @@ int nghttp2_session_add_rst_stream(nghttp2_session *session, int32_t stream_id,
  *
  * NGHTTP2_ERR_NOMEM
  *     Out of memory.
- * NGHTTP2_ERR_FLOODING_DETECTED
+ * NGHTTP2_ERR_FLOODED
  *     There are too many items in outbound queue; this only happens
  *     if NGHTTP2_FLAG_ACK is set in |flags|
  */
@@ -417,7 +417,7 @@ int nghttp2_session_add_window_update(nghttp2_session *session, uint8_t flags,
  *
  * NGHTTP2_ERR_NOMEM
  *     Out of memory.
- * NGHTTP2_ERR_FLOODING_DETECTED
+ * NGHTTP2_ERR_FLOODED
  *     There are too many items in outbound queue; this only happens
  *     if NGHTTP2_FLAG_ACK is set in |flags|
  */
@@ -643,6 +643,9 @@ int nghttp2_session_on_rst_stream_received(nghttp2_session *session,
  *     Out of memory
  * NGHTTP2_ERR_CALLBACK_FAILURE
  *     The read_callback failed
+ * NGHTTP2_ERR_FLOODED
+ *     There are too many items in outbound queue, and this is most
+ *     likely caused by misbehaviour of peer.
  */
 int nghttp2_session_on_settings_received(nghttp2_session *session,
                                          nghttp2_frame *frame, int noack);
@@ -676,6 +679,9 @@ int nghttp2_session_on_push_promise_received(nghttp2_session *session,
  *     Out of memory.
  * NGHTTP2_ERR_CALLBACK_FAILURE
  *   The callback function failed.
+ * NGHTTP2_ERR_FLOODED
+ *     There are too many items in outbound queue, and this is most
+ *     likely caused by misbehaviour of peer.
  */
 int nghttp2_session_on_ping_received(nghttp2_session *session,
                                      nghttp2_frame *frame);

tests/nghttp2_session_test.c

@@ -8097,6 +8097,7 @@ void test_nghttp2_session_flooding(void) {
   nghttp2_buf *buf;
   nghttp2_frame frame;
   nghttp2_mem *mem;
+  size_t i;
 
   mem = nghttp2_mem_default();
 
@@ -8113,21 +8114,15 @@ void test_nghttp2_session_flooding(void) {
 
   buf = &bufs.head->buf;
 
-  for (int i = 0; i < NGHTTP2_MAX_OBQ_FLOOD_ITEM; ++i) {
+  for (i = 0; i < NGHTTP2_MAX_OBQ_FLOOD_ITEM; ++i) {
     CU_ASSERT(
         (ssize_t)nghttp2_buf_len(buf) ==
         nghttp2_session_mem_recv(session, buf->pos, nghttp2_buf_len(buf)));
   }
 
-  CU_ASSERT(1 == nghttp2_session_want_read(session));
+  CU_ASSERT(NGHTTP2_ERR_FLOODED ==
-  CU_ASSERT(1 == nghttp2_session_want_write(session));
-
-  CU_ASSERT((ssize_t)nghttp2_buf_len(buf) ==
             nghttp2_session_mem_recv(session, buf->pos, nghttp2_buf_len(buf)));
 
-  CU_ASSERT(0 == nghttp2_session_want_read(session));
-  CU_ASSERT(0 == nghttp2_session_want_write(session));
-
   nghttp2_session_del(session);
 
   /* SETTINGS ACK */
@@ -8141,21 +8136,15 @@ void test_nghttp2_session_flooding(void) {
 
   buf = &bufs.head->buf;
 
-  for (int i = 0; i < NGHTTP2_MAX_OBQ_FLOOD_ITEM; ++i) {
+  for (i = 0; i < NGHTTP2_MAX_OBQ_FLOOD_ITEM; ++i) {
     CU_ASSERT(
         (ssize_t)nghttp2_buf_len(buf) ==
         nghttp2_session_mem_recv(session, buf->pos, nghttp2_buf_len(buf)));
   }
 
-  CU_ASSERT(1 == nghttp2_session_want_read(session));
+  CU_ASSERT(NGHTTP2_ERR_FLOODED ==
-  CU_ASSERT(1 == nghttp2_session_want_write(session));
-
-  CU_ASSERT((ssize_t)nghttp2_buf_len(buf) ==
             nghttp2_session_mem_recv(session, buf->pos, nghttp2_buf_len(buf)));
 
-  CU_ASSERT(0 == nghttp2_session_want_read(session));
-  CU_ASSERT(0 == nghttp2_session_want_write(session));
-
   nghttp2_session_del(session);
   nghttp2_bufs_free(&bufs);
 }