Cancel frame transmission from before_frame_send_callback

We define the behaviour when NGHTTP2_ERR_CANCEL is returned from
before_frame_send_callback.  That is to cancel the frame passed to the
callback.

lib/includes/nghttp2/nghttp2.h

@@ -1454,9 +1454,19 @@ typedef int (*nghttp2_on_data_chunk_recv_callback)(nghttp2_session *session,
  * `nghttp2_session_server_new()`.
  *
  * The implementation of this function must return 0 if it succeeds.
- * If nonzero is returned, it is treated as fatal error and
+ * It can also return :enum:`NGHTTP2_ERR_CANCEL` to cancel the
- * `nghttp2_session_send()` and `nghttp2_session_mem_send()` functions
+ * transmission of the given frame.
- * immediately return :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`.
+ *
+ * If there is a fatal error while executing this callback, the
+ * implementation should return :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`,
+ * which makes `nghttp2_session_send()` and
+ * `nghttp2_session_mem_send()` functions immediately return
+ * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`.
+ *
+ * If the other value is returned, it is treated as if
+ * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE` is returned.  But the
+ * implementation should not rely on this since the library may define
+ * new return value to extend its capability.
  *
  * To set this callback to :type:`nghttp2_session_callbacks`, use
  * `nghttp2_session_callbacks_set_before_frame_send_callback()`.

lib/nghttp2_session.c

@@ -2364,6 +2364,10 @@ static int session_call_before_frame_send(nghttp2_session *session,
   if (session->callbacks.before_frame_send_callback) {
     rv = session->callbacks.before_frame_send_callback(session, frame,
                                                        session->user_data);
+    if (rv == NGHTTP2_ERR_CANCEL) {
+      return rv;
+    }
+
     if (rv != 0) {
       return NGHTTP2_ERR_CALLBACK_FAILURE;
     }
@@ -3004,6 +3008,51 @@ static ssize_t nghttp2_session_mem_send_internal(nghttp2_session *session,
         if (nghttp2_is_fatal(rv)) {
           return rv;
         }
+
+        if (rv == NGHTTP2_ERR_CANCEL) {
+          int32_t opened_stream_id = 0;
+          uint32_t error_code = NGHTTP2_INTERNAL_ERROR;
+
+          if (session->callbacks.on_frame_not_send_callback) {
+            if (session->callbacks.on_frame_not_send_callback(
+                    session, frame, rv, session->user_data) != 0) {
+              return NGHTTP2_ERR_CALLBACK_FAILURE;
+            }
+          }
+
+          /* We have to close stream opened by canceled request
+             HEADERS or PUSH_PROMISE. */
+          switch (item->frame.hd.type) {
+          case NGHTTP2_HEADERS:
+            if (item->frame.headers.cat == NGHTTP2_HCAT_REQUEST) {
+              opened_stream_id = item->frame.hd.stream_id;
+              /* We don't have to check
+                 item->aux_data.headers.canceled since it has already
+                 been checked. */
+              /* Set error_code to REFUSED_STREAM so that application
+                 can send request again. */
+              error_code = NGHTTP2_REFUSED_STREAM;
+            }
+            break;
+          case NGHTTP2_PUSH_PROMISE:
+            opened_stream_id = item->frame.push_promise.promised_stream_id;
+            break;
+          }
+          if (opened_stream_id) {
+            /* careful not to override rv */
+            int rv2;
+            rv2 = nghttp2_session_close_stream(session, opened_stream_id,
+                                               error_code);
+
+            if (nghttp2_is_fatal(rv2)) {
+              return rv2;
+            }
+          }
+
+          active_outbound_item_reset(aob, mem);
+
+          break;
+        }
       } else {
         DEBUGF(fprintf(stderr, "send: next frame: DATA\n"));
 

tests/main.c

@@ -306,6 +306,8 @@ int main(int argc _U_, char *argv[] _U_) {
                    test_nghttp2_session_repeated_priority_submission) ||
       !CU_add_test(pSuite, "session_set_local_window_size",
                    test_nghttp2_session_set_local_window_size) ||
+      !CU_add_test(pSuite, "session_cancel_from_before_frame_send",
+                   test_nghttp2_session_cancel_from_before_frame_send) ||
       !CU_add_test(pSuite, "http_mandatory_headers",
                    test_nghttp2_http_mandatory_headers) ||
       !CU_add_test(pSuite, "http_content_length",

tests/nghttp2_session_test.c

@@ -57,6 +57,7 @@ typedef struct {
   nghttp2_frame_hd recv_frame_hd;
   int frame_send_cb_called;
   uint8_t sent_frame_type;
+  int before_frame_send_cb_called;
   int frame_not_send_cb_called;
   uint8_t not_sent_frame_type;
   int not_sent_error;
@@ -209,6 +210,14 @@ static int on_frame_not_send_callback(nghttp2_session *session _U_,
   return 0;
 }
 
+static int cancel_before_frame_send_callback(nghttp2_session *session _U_,
+                                             const nghttp2_frame *frame _U_,
+                                             void *user_data) {
+  my_user_data *ud = (my_user_data *)user_data;
+  ++ud->before_frame_send_cb_called;
+  return NGHTTP2_ERR_CANCEL;
+}
+
 static int on_data_chunk_recv_callback(nghttp2_session *session _U_,
                                        uint8_t flags _U_, int32_t stream_id _U_,
                                        const uint8_t *data _U_, size_t len,
@@ -9548,6 +9557,62 @@ void test_nghttp2_session_set_local_window_size(void) {
   nghttp2_session_del(session);
 }
 
+void test_nghttp2_session_cancel_from_before_frame_send(void) {
+  int rv;
+  nghttp2_session *session;
+  nghttp2_session_callbacks callbacks;
+  my_user_data ud;
+  nghttp2_settings_entry iv;
+  nghttp2_data_provider data_prd;
+
+  memset(&callbacks, 0, sizeof(callbacks));
+
+  callbacks.before_frame_send_callback = cancel_before_frame_send_callback;
+  callbacks.on_frame_not_send_callback = on_frame_not_send_callback;
+  callbacks.send_callback = null_send_callback;
+
+  nghttp2_session_client_new(&session, &callbacks, &ud);
+
+  iv.settings_id = 0;
+  iv.value = 1000000009;
+
+  rv = nghttp2_submit_settings(session, NGHTTP2_FLAG_NONE, &iv, 1);
+
+  CU_ASSERT(0 == rv);
+
+  ud.frame_send_cb_called = 0;
+  ud.before_frame_send_cb_called = 0;
+  ud.frame_not_send_cb_called = 0;
+
+  rv = nghttp2_session_send(session);
+
+  CU_ASSERT(0 == rv);
+  CU_ASSERT(0 == ud.frame_send_cb_called);
+  CU_ASSERT(1 == ud.before_frame_send_cb_called);
+  CU_ASSERT(1 == ud.frame_not_send_cb_called);
+
+  data_prd.source.ptr = NULL;
+  data_prd.read_callback = temporal_failure_data_source_read_callback;
+
+  rv = nghttp2_submit_request(session, NULL, reqnv, ARRLEN(reqnv), &data_prd,
+                              NULL);
+
+  CU_ASSERT(rv > 0);
+
+  ud.frame_send_cb_called = 0;
+  ud.before_frame_send_cb_called = 0;
+  ud.frame_not_send_cb_called = 0;
+
+  rv = nghttp2_session_send(session);
+
+  CU_ASSERT(0 == rv);
+  CU_ASSERT(0 == ud.frame_send_cb_called);
+  CU_ASSERT(1 == ud.before_frame_send_cb_called);
+  CU_ASSERT(1 == ud.frame_not_send_cb_called);
+
+  nghttp2_session_del(session);
+}
+
 static void check_nghttp2_http_recv_headers_fail(
     nghttp2_session *session, nghttp2_hd_deflater *deflater, int32_t stream_id,
     int stream_state, const nghttp2_nv *nva, size_t nvlen) {

tests/nghttp2_session_test.h

@@ -151,6 +151,7 @@ void test_nghttp2_session_create_idle_stream(void);
 void test_nghttp2_session_repeated_priority_change(void);
 void test_nghttp2_session_repeated_priority_submission(void);
 void test_nghttp2_session_set_local_window_size(void);
+void test_nghttp2_session_cancel_from_before_frame_send(void);
 void test_nghttp2_http_mandatory_headers(void);
 void test_nghttp2_http_content_length(void);
 void test_nghttp2_http_content_length_mismatch(void);