Implement compressed DATA

The library interface supports compressed DATA.  The library does not
deflate nor inflate data payload.  When sending data, an application
has to compress data and set NGHTTP2_DATA_FLAG_COMPRESSED to
data_flags parameter in nghttp2_data_source_read_callback.  On
receiving, flags parameter in nghttp2_on_data_chunk_recv_callback
includes NGHTTP2_FLAG_COMPRESSED.  An application should check the
flags and inflate data as necessary.  Since compression context is per
frame, when DATA is seen in nghttp2_on_frame_recv_callback, an
application should reset compression context.

lib/includes/nghttp2/nghttp2.h

@@ -488,7 +488,11 @@ typedef enum {
   /**
    * The PRIORITY flag.
    */
-  NGHTTP2_FLAG_PRIORITY = 0x20
+  NGHTTP2_FLAG_PRIORITY = 0x20,
+  /**
+   * THE COMPRESSED flag.
+   */
+  NGHTTP2_FLAG_COMPRESSED = 0x20
 } nghttp2_flag;
 
 /**
@@ -512,10 +516,14 @@ typedef enum {
    * SETTINGS_INITIAL_WINDOW_SIZE
    */
   NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE = 4,
+  /**
+   * SETTINGS_COMPRESS_DATA
+   */
+  NGHTTP2_SETTINGS_COMPRESS_DATA = 5,
   /**
    * Maximum ID of :type:`nghttp2_settings_id`.
    */
-  NGHTTP2_SETTINGS_MAX = 4
+  NGHTTP2_SETTINGS_MAX = 5
 } nghttp2_settings_id;
 
 /**
@@ -638,7 +646,11 @@ typedef enum {
   /**
    * Indicates EOF was sensed.
    */
-  NGHTTP2_DATA_FLAG_EOF = 0x01
+  NGHTTP2_DATA_FLAG_EOF = 0x01,
+  /**
+   * Indicates data was compressed.
+   */
+  NGHTTP2_DATA_FLAG_COMPRESSED = 0x02
 } nghttp2_data_flag;
 
 /**
@@ -649,7 +661,11 @@ typedef enum {
  * The implementation of this function must read at most |length|
  * bytes of data from |source| (or possibly other places) and store
  * them in |buf| and return number of data stored in |buf|.  If EOF is
- * reached, set :enum:`NGHTTP2_DATA_FLAG_EOF` flag in |*data_falgs|.
+ * reached, set :enum:`NGHTTP2_DATA_FLAG_EOF` flag in |*data_flags|.
+ *
+ * To send compressed data payload without affecting content-length,
+ * set :enum:`NGHTTP2_DATA_FLAG_COMPRESSED` flag in |*data_flags|.
+ *
  * If the application wants to postpone DATA frames (e.g.,
  * asynchronous I/O, or reading data blocks for long time), it is
  * achieved by returning :enum:`NGHTTP2_ERR_DEFERRED` without reading

lib/nghttp2_frame.c

@@ -938,6 +938,7 @@ int nghttp2_iv_check(const nghttp2_settings_entry *iv, size_t niv)
     case NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS:
       break;
     case NGHTTP2_SETTINGS_ENABLE_PUSH:
+    case NGHTTP2_SETTINGS_COMPRESS_DATA:
       if(iv[i].value != 0 && iv[i].value != 1) {
         return 0;
       }

lib/nghttp2_session.c

@@ -225,6 +225,7 @@ static void init_settings(uint32_t *settings)
     NGHTTP2_INITIAL_MAX_CONCURRENT_STREAMS;
   settings[NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE] =
     NGHTTP2_INITIAL_WINDOW_SIZE;
+  settings[NGHTTP2_SETTINGS_COMPRESS_DATA] = 0;
 }
 
 static void nghttp2_active_outbound_item_reset
@@ -3464,6 +3465,12 @@ int nghttp2_session_on_settings_received(nghttp2_session *session,
           (session, frame, NGHTTP2_FLOW_CONTROL_ERROR);
       }
       break;
+    case NGHTTP2_SETTINGS_COMPRESS_DATA:
+      if(entry->value != 0 && entry->value != 1) {
+        return nghttp2_session_handle_invalid_connection
+          (session, frame, NGHTTP2_PROTOCOL_ERROR);
+      }
+      break;
     }
     session->remote_settings[entry->settings_id] = entry->value;
   }
@@ -4024,7 +4031,12 @@ static int nghttp2_session_on_data_received_fail_fast(nghttp2_session *session)
 {
   int rv;
   nghttp2_stream *stream;
-  int32_t stream_id = session->iframe.frame.hd.stream_id;
+  nghttp2_inbound_frame *iframe;
+  int32_t stream_id;
+
+  iframe = &session->iframe;
+  stream_id = iframe->frame.hd.stream_id;
+
   if(stream_id == 0) {
     /* The spec says that if a DATA frame is received whose stream ID
        is 0, the recipient MUST respond with a connection error of
@@ -4041,6 +4053,12 @@ static int nghttp2_session_on_data_received_fail_fast(nghttp2_session *session)
   if(stream->shut_flags & NGHTTP2_SHUT_RD) {
     goto fail;
   }
+
+  if((iframe->frame.hd.flags & NGHTTP2_FLAG_COMPRESSED) &&
+     session->local_settings[NGHTTP2_SETTINGS_COMPRESS_DATA] == 0) {
+    goto fail;
+  }
+
   if(nghttp2_session_is_my_stream_id(session, stream_id)) {
     if(stream->state == NGHTTP2_STREAM_CLOSING) {
       return NGHTTP2_ERR_IGN_PAYLOAD;
@@ -4111,6 +4129,7 @@ static int inbound_frame_set_settings_entry(nghttp2_inbound_frame *iframe)
   case NGHTTP2_SETTINGS_ENABLE_PUSH:
   case NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS:
   case NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE:
+  case NGHTTP2_SETTINGS_COMPRESS_DATA:
     break;
   default:
     return -1;
@@ -4260,7 +4279,8 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session,
         iframe->frame.hd.flags &= (NGHTTP2_FLAG_END_STREAM |
                                    NGHTTP2_FLAG_END_SEGMENT |
                                    NGHTTP2_FLAG_PAD_LOW |
-                                   NGHTTP2_FLAG_PAD_HIGH);
+                                   NGHTTP2_FLAG_PAD_HIGH |
+                                   NGHTTP2_FLAG_COMPRESSED);
         /* Check stream is open. If it is not open or closing,
            ignore payload. */
         busy = 1;
@@ -5559,6 +5579,10 @@ int nghttp2_session_pack_data(nghttp2_session *session,
     }
   }
 
+  if(data_flags & NGHTTP2_DATA_FLAG_COMPRESSED) {
+    flags |= NGHTTP2_FLAG_COMPRESSED;
+  }
+
   /* The primary reason of data_frame is pass to the user callback */
   data_frame.hd.length = payloadlen;
   data_frame.hd.stream_id = frame->hd.stream_id;

src/app_helper.cc

@@ -234,6 +234,12 @@ void print_flags(const nghttp2_frame_hd& hd)
       }
       s += "PAD_HIGH";
     }
+    if(hd.flags & NGHTTP2_FLAG_COMPRESSED) {
+      if(!s.empty()) {
+        s += " | ";
+      }
+      s += "COMPRESSED";
+    }
     break;
   case NGHTTP2_HEADERS:
     if(hd.flags & NGHTTP2_FLAG_END_STREAM) {

tests/nghttp2_frame_test.c

@@ -632,6 +632,15 @@ void test_nghttp2_iv_check(void)
   iv[1].value = 3;
   CU_ASSERT(!nghttp2_iv_check(iv, 2));
 
+  /* COMPRESSED_DATA only allows 0 or 1 */
+  iv[1].settings_id = NGHTTP2_SETTINGS_COMPRESS_DATA;
+  iv[1].value = 0;
+  CU_ASSERT(nghttp2_iv_check(iv, 2));
+  iv[1].value = 1;
+  CU_ASSERT(nghttp2_iv_check(iv, 2));
+  iv[1].value = 3;
+  CU_ASSERT(!nghttp2_iv_check(iv, 2));
+
   /* Undefined SETTINGS ID */
   iv[1].settings_id = 1000000009;
   iv[1].value = 0;

tests/nghttp2_session_test.c

@@ -75,6 +75,7 @@ typedef struct {
   nghttp2_nv nv;
   size_t data_chunk_len;
   size_t padding_boundary;
+  int compress_data;
 } my_user_data;
 
 static void scripted_data_feed_init2(scripted_data_feed *df,
@@ -249,6 +250,24 @@ static ssize_t fixed_length_data_source_read_callback
   return wlen;
 }
 
+static ssize_t compressed_fixed_length_data_source_read_callback
+(nghttp2_session *session, int32_t stream_id,
+ uint8_t *buf, size_t len, uint32_t *data_flags,
+ nghttp2_data_source *source, void *user_data)
+{
+  my_user_data *ud = (my_user_data*)user_data;
+  size_t wlen;
+
+  wlen = fixed_length_data_source_read_callback
+    (session, stream_id, buf, len, data_flags, source, user_data);
+
+  if(ud->compress_data) {
+    *data_flags |= NGHTTP2_DATA_FLAG_COMPRESSED;
+  }
+
+  return wlen;
+}
+
 static ssize_t temporal_failure_data_source_read_callback
 (nghttp2_session *session, int32_t stream_id,
  uint8_t *buf, size_t len, uint32_t *data_flags,
@@ -703,6 +722,57 @@ void test_nghttp2_session_recv_data(void)
   CU_ASSERT(NGHTTP2_PROTOCOL_ERROR == OB_CTRL(item)->goaway.error_code);
 
   nghttp2_session_del(session);
+
+  /* Receiving compressed DATA while SETTINGS_COMPRESS_DATA == 0 is
+     subject to connection error */
+
+  nghttp2_session_client_new(&session, &callbacks, &ud);
+
+  hd.length = 4096;
+  hd.type = NGHTTP2_DATA;
+  hd.flags = NGHTTP2_FLAG_COMPRESSED;
+  hd.stream_id = 1;
+  nghttp2_frame_pack_frame_hd(data, &hd);
+
+  stream = nghttp2_session_open_stream(session, 1,
+                                       NGHTTP2_STREAM_FLAG_NONE,
+                                       &pri_spec_default,
+                                       NGHTTP2_STREAM_OPENED, NULL);
+
+  ud.data_chunk_recv_cb_called = 0;
+  ud.frame_recv_cb_called = 0;
+  rv = nghttp2_session_mem_recv(session, data, 8+4096);
+  CU_ASSERT(8+4096 == rv);
+
+  CU_ASSERT(0 == ud.data_chunk_recv_cb_called);
+  CU_ASSERT(0 == ud.frame_recv_cb_called);
+  item = nghttp2_session_get_next_ob_item(session);
+  CU_ASSERT(NGHTTP2_GOAWAY == OB_CTRL_TYPE(item));
+
+  nghttp2_session_del(session);
+
+  /* Receiving compressed DATA while SETTINGS_COMPRESS_DATA == 1 is
+     allowed */
+
+  nghttp2_session_client_new(&session, &callbacks, &ud);
+
+  session->local_settings[NGHTTP2_SETTINGS_COMPRESS_DATA] = 1;
+
+  stream = nghttp2_session_open_stream(session, 1,
+                                       NGHTTP2_STREAM_FLAG_NONE,
+                                       &pri_spec_default,
+                                       NGHTTP2_STREAM_OPENED, NULL);
+
+  ud.data_chunk_recv_cb_called = 0;
+  ud.frame_recv_cb_called = 0;
+  rv = nghttp2_session_mem_recv(session, data, 8+4096);
+  CU_ASSERT(8+4096 == rv);
+
+  CU_ASSERT(1 == ud.data_chunk_recv_cb_called);
+  CU_ASSERT(1 == ud.frame_recv_cb_called);
+  CU_ASSERT(NULL == nghttp2_session_get_next_ob_item(session));
+
+  nghttp2_session_del(session);
 }
 
 void test_nghttp2_session_recv_continuation(void)
@@ -2613,12 +2683,14 @@ void test_nghttp2_submit_data(void)
   memset(&callbacks, 0, sizeof(nghttp2_session_callbacks));
   callbacks.send_callback = block_count_send_callback;
 
-  data_prd.read_callback = fixed_length_data_source_read_callback;
+  data_prd.read_callback = compressed_fixed_length_data_source_read_callback;
   ud.data_source_length = NGHTTP2_DATA_PAYLOADLEN * 2;
   CU_ASSERT(0 == nghttp2_session_client_new(&session, &callbacks, &ud));
   aob = &session->aob;
   framebufs = &aob->framebufs;
 
+  session->remote_settings[NGHTTP2_SETTINGS_COMPRESS_DATA] = 1;
+
   nghttp2_session_open_stream(session, 1, NGHTTP2_STREAM_FLAG_NONE,
                               &pri_spec_default, NGHTTP2_STREAM_OPENING,
                               NULL);
@@ -2626,6 +2698,8 @@ void test_nghttp2_submit_data(void)
                                      NGHTTP2_FLAG_END_STREAM |
                                      NGHTTP2_FLAG_END_SEGMENT, 1, &data_prd));
 
+  /* First, no compression */
+  ud.compress_data = 0;
   ud.block_count = 0;
   CU_ASSERT(0 == nghttp2_session_send(session));
   data_frame = nghttp2_outbound_item_get_data_frame(aob->item);
@@ -2638,15 +2712,19 @@ void test_nghttp2_submit_data(void)
   CU_ASSERT((NGHTTP2_FLAG_END_STREAM | NGHTTP2_FLAG_END_SEGMENT) ==
             data_frame->hd.flags);
 
+  /* Now compression enabled */
+  ud.compress_data = 1;
   ud.block_count = 1;
   CU_ASSERT(0 == nghttp2_session_send(session));
   data_frame = nghttp2_outbound_item_get_data_frame(aob->item);
   nghttp2_frame_unpack_frame_hd(&hd, buf->pos);
 
   /* This is the last frame, so we must have following flags */
-  CU_ASSERT((NGHTTP2_FLAG_END_STREAM | NGHTTP2_FLAG_END_SEGMENT) == hd.flags);
+  CU_ASSERT((NGHTTP2_FLAG_END_STREAM | NGHTTP2_FLAG_END_SEGMENT |
+             NGHTTP2_FLAG_COMPRESSED) == hd.flags);
   /* frame->hd.flags has these flags */
-  CU_ASSERT((NGHTTP2_FLAG_END_STREAM | NGHTTP2_FLAG_END_SEGMENT) ==
+  CU_ASSERT((NGHTTP2_FLAG_END_STREAM | NGHTTP2_FLAG_END_SEGMENT |
+             NGHTTP2_FLAG_COMPRESSED) ==
             data_frame->hd.flags);
 
   nghttp2_session_del(session);