nghttpx, nghttpd: Support non-final response

src/HttpServer.cc

@@ -843,6 +843,16 @@ int Http2Handler::submit_response(const std::string& status,
                                  data_prd);
 }
 
+int Http2Handler::submit_non_final_response(const std::string& status,
+                                            int32_t stream_id)
+{
+  auto nva = std::vector<nghttp2_nv>{
+    http2::make_nv_ls(":status", status)
+  };
+  return nghttp2_submit_headers(session_, NGHTTP2_FLAG_NONE, stream_id,
+                                nullptr, nva.data(), nva.size(), nullptr);
+}
+
 int Http2Handler::submit_push_promise(Stream *stream,
                                       const std::string& push_path)
 {
@@ -1266,6 +1276,12 @@ int hd_on_frame_recv_callback
         return 0;
       }
 
+      auto expect100 = http2::get_header(stream->headers, "expect");
+
+      if(expect100 && util::strieq("100-continue", expect100->value.c_str())) {
+        hd->submit_non_final_response("100", frame->hd.stream_id);
+      }
+
       if(hd->get_config()->early_response) {
         prepare_response(stream, hd);
       }

src/HttpServer.h

@@ -126,6 +126,8 @@ public:
    const std::vector<std::pair<std::string, std::string>>& headers,
    nghttp2_data_provider *data_prd);
 
+  int submit_non_final_response(const std::string& status, int32_t stream_id);
+
   int submit_push_promise(Stream *stream, const std::string& push_path);
 
   int submit_rst_stream(Stream *stream, nghttp2_error_code error_code);

src/http2.cc

@@ -165,7 +165,6 @@ size_t DISALLOWED_HDLEN = sizeof(DISALLOWED_HD)/sizeof(DISALLOWED_HD[0]);
 namespace {
 const char *IGN_HD[] = {
   "connection",
-  "expect",
   "http2-settings",
   "keep-alive",
   "proxy-connection",
@@ -186,7 +185,6 @@ namespace {
 const char *HTTP1_IGN_HD[] = {
   "connection",
   "cookie",
-  "expect",
   "http2-settings",
   "keep-alive",
   "proxy-connection",

src/shrpx_downstream.cc

@@ -63,12 +63,12 @@ Downstream::Downstream(Upstream *upstream, int stream_id, int priority)
     http2_settings_seen_(false),
     chunked_request_(false),
     request_connection_close_(false),
-    request_expect_100_continue_(false),
     request_header_key_prev_(false),
     request_http2_expect_body_(false),
     chunked_response_(false),
     response_connection_close_(false),
-    response_header_key_prev_(false)
+    response_header_key_prev_(false),
+    expect_final_response_(false)
 {}
 
 Downstream::~Downstream()
@@ -424,11 +424,6 @@ void Downstream::set_request_http2_expect_body(bool f)
   request_http2_expect_body_ = f;
 }
 
-bool Downstream::get_expect_100_continue() const
-{
-  return request_expect_100_continue_;
-}
-
 bool Downstream::get_output_buffer_full()
 {
   if(dconn_) {
@@ -744,11 +739,6 @@ void Downstream::inspect_http1_request()
       if(util::strifind(hd.value.c_str(), "chunked")) {
         chunked_request_ = true;
       }
-    } else if(!request_expect_100_continue_ &&
-              util::strieq(hd.name.c_str(), "expect")) {
-      if(util::strifind(hd.value.c_str(), "100-continue")) {
-        request_expect_100_continue_ = true;
-      }
     }
   }
 }
@@ -765,6 +755,18 @@ void Downstream::inspect_http1_response()
   }
 }
 
+void Downstream::reset_response()
+{
+  response_http_status_ = 0;
+  response_major_ = 1;
+  response_minor_ = 1;
+}
+
+bool Downstream::get_non_final_response() const
+{
+  return response_http_status_ / 100 == 1;
+}
+
 bool Downstream::get_upgraded() const
 {
   return upgraded_;
@@ -806,4 +808,14 @@ void Downstream::set_response_rst_stream_error_code
   response_rst_stream_error_code_ = error_code;
 }
 
+void Downstream::set_expect_final_response(bool f)
+{
+  expect_final_response_ = f;
+}
+
+bool Downstream::get_expect_final_response() const
+{
+  return expect_final_response_;
+}
+
 } // namespace shrpx

src/shrpx_downstream.h

@@ -140,7 +140,6 @@ public:
   const std::string& get_request_user_agent() const;
   bool get_request_http2_expect_body() const;
   void set_request_http2_expect_body(bool f);
-  bool get_expect_100_continue() const;
   int push_upload_data_chunk(const uint8_t *data, size_t datalen);
   int end_upload_data();
   enum {
@@ -207,6 +206,11 @@ public:
   void set_response_rst_stream_error_code(nghttp2_error_code error_code);
   // Inspects HTTP/1 response.  This checks tranfer-encoding etc.
   void inspect_http1_response();
+  // Clears some of member variables for response.
+  void reset_response();
+  bool get_non_final_response() const;
+  void set_expect_final_response(bool f);
+  bool get_expect_final_response() const;
 
   // Call this method when there is incoming data in downstream
   // connection.
@@ -274,13 +278,13 @@ private:
 
   bool chunked_request_;
   bool request_connection_close_;
-  bool request_expect_100_continue_;
   bool request_header_key_prev_;
   bool request_http2_expect_body_;
 
   bool chunked_response_;
   bool response_connection_close_;
   bool response_header_key_prev_;
+  bool expect_final_response_;
 };
 
 } // namespace shrpx

src/shrpx_http2_session.cc

@@ -752,7 +752,6 @@ int on_stream_close_callback
 (nghttp2_session *session, int32_t stream_id, nghttp2_error_code error_code,
  void *user_data)
 {
-  int rv;
   auto http2session = static_cast<Http2Session*>(user_data);
   if(LOG_ENABLED(INFO)) {
     SSLOG(INFO, http2session) << "Stream stream_id=" << stream_id
@@ -769,11 +768,8 @@ int on_stream_close_callback
   if(dconn) {
     auto downstream = dconn->get_downstream();
     if(downstream && downstream->get_downstream_stream_id() == stream_id) {
-      auto upstream = downstream->get_upstream();
       if(error_code == NGHTTP2_NO_ERROR) {
-        downstream->set_response_state(Downstream::MSG_COMPLETE);
+        if(downstream->get_response_state() != Downstream::MSG_COMPLETE) {
-        rv = upstream->on_downstream_body_complete(downstream);
-        if(rv != 0) {
           downstream->set_response_state(Downstream::MSG_RESET);
         }
       } else {
@@ -841,10 +837,6 @@ int on_header_callback(nghttp2_session *session,
                        uint8_t flags,
                        void *user_data)
 {
-  if(frame->hd.type != NGHTTP2_HEADERS ||
-     frame->headers.cat != NGHTTP2_HCAT_RESPONSE) {
-    return 0;
-  }
   auto sd = static_cast<StreamData*>
     (nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
   if(!sd || !sd->dconn) {
@@ -854,6 +846,13 @@ int on_header_callback(nghttp2_session *session,
   if(!downstream) {
     return 0;
   }
+
+  if(frame->hd.type != NGHTTP2_HEADERS ||
+     (frame->headers.cat != NGHTTP2_HCAT_RESPONSE &&
+      !downstream->get_expect_final_response())) {
+    return 0;
+  }
+
   if(downstream->get_response_headers_sum() > Downstream::MAX_HEADERS_SUM) {
     if(LOG_ENABLED(INFO)) {
       DLOG(INFO, downstream) << "Too large header block size="
@@ -905,19 +904,14 @@ int on_begin_headers_callback(nghttp2_session *session,
 
 namespace {
 int on_response_headers(Http2Session *http2session,
+                        Downstream *downstream,
                         nghttp2_session *session,
                         const nghttp2_frame *frame)
 {
   int rv;
-  auto sd = static_cast<StreamData*>
+
-    (nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
+  auto upstream = downstream->get_upstream();
-  if(!sd || !sd->dconn) {
+
-    return 0;
-  }
-  auto downstream = sd->dconn->get_downstream();
-  if(!downstream) {
-    return 0;
-  }
   downstream->normalize_response_headers();
   auto& nva = downstream->get_response_headers();
 
@@ -935,13 +929,38 @@ int on_response_headers(Http2Session *http2session,
                                     NGHTTP2_PROTOCOL_ERROR);
     downstream->set_response_state(Downstream::MSG_RESET);
     call_downstream_readcb(http2session, downstream);
+
     return 0;
   }
+
   downstream->set_response_http_status(strtoul(status->value.c_str(),
                                                nullptr, 10));
   downstream->set_response_major(2);
   downstream->set_response_minor(0);
 
+  if(downstream->get_non_final_response()) {
+
+    if(LOG_ENABLED(INFO)) {
+      SSLOG(INFO, http2session) << "HTTP non-final response. stream_id="
+                                << frame->hd.stream_id;
+    }
+
+    downstream->set_expect_final_response(true);
+    rv = upstream->on_downstream_header_complete(downstream);
+
+    // Now Dowstream's response headers are erased.
+
+    if(rv != 0) {
+      http2session->submit_rst_stream(frame->hd.stream_id,
+                                      NGHTTP2_PROTOCOL_ERROR);
+      downstream->set_response_state(Downstream::MSG_RESET);
+    }
+
+    return 0;
+  }
+
+  downstream->set_expect_final_response(false);
+
   if(LOG_ENABLED(INFO)) {
     std::stringstream ss;
     for(auto& nv : nva) {
@@ -975,7 +994,6 @@ int on_response_headers(Http2Session *http2session,
     }
   }
 
-  auto upstream = downstream->get_upstream();
   downstream->set_response_state(Downstream::HEADER_COMPLETE);
   downstream->check_upgrade_fulfilled();
   if(downstream->get_upgraded()) {
@@ -1033,19 +1051,69 @@ int on_frame_recv_callback
       http2session->submit_rst_stream(frame->hd.stream_id,
                                       NGHTTP2_INTERNAL_ERROR);
       downstream->set_response_state(Downstream::MSG_RESET);
+
+    } else if(frame->hd.flags & NGHTTP2_FLAG_END_STREAM) {
+
+      if(downstream->get_response_state() != Downstream::MSG_RESET) {
+
+        downstream->set_response_state(Downstream::MSG_COMPLETE);
+
+        rv = upstream->on_downstream_body_complete(downstream);
+
+        if(rv != 0) {
+          downstream->set_response_state(Downstream::MSG_RESET);
         }
+      }
+    }
+
     call_downstream_readcb(http2session, downstream);
     break;
   }
-  case NGHTTP2_HEADERS:
+  case NGHTTP2_HEADERS: {
+    auto sd = static_cast<StreamData*>
+      (nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
+    if(!sd || !sd->dconn) {
+      break;
+    }
+    auto downstream = sd->dconn->get_downstream();
+
+    if(!downstream) {
+      return 0;
+    }
+
     if(frame->headers.cat == NGHTTP2_HCAT_RESPONSE) {
-      rv = on_response_headers(http2session, session, frame);
+      rv = on_response_headers(http2session, downstream, session, frame);
 
       if(rv != 0) {
         return rv;
       }
     }
+
+    if(frame->headers.cat == NGHTTP2_HCAT_HEADERS &&
+       downstream->get_expect_final_response()) {
+
+      rv = on_response_headers(http2session, downstream, session, frame);
+
+      if(rv != 0) {
+        return rv;
+      }
+    }
+
+    if(frame->hd.flags & NGHTTP2_FLAG_END_STREAM) {
+      if(downstream->get_response_state() != Downstream::MSG_RESET) {
+        downstream->set_response_state(Downstream::MSG_COMPLETE);
+
+        auto upstream = downstream->get_upstream();
+
+        rv = upstream->on_downstream_body_complete(downstream);
+
+        if(rv != 0) {
+          downstream->set_response_state(Downstream::MSG_RESET);
+        }
+      }
+    }
     break;
+  }
   case NGHTTP2_RST_STREAM: {
     auto sd = static_cast<StreamData*>
       (nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
@@ -1055,7 +1123,7 @@ int on_frame_recv_callback
          downstream->get_downstream_stream_id() == frame->hd.stream_id) {
         if(downstream->get_upgraded() &&
            downstream->get_response_state() == Downstream::HEADER_COMPLETE) {
-          // For tunneled connection, we has to submit RST_STREAM to
+          // For tunneled connection, we have to submit RST_STREAM to
           // upstream *after* whole response body is sent. We just set
           // MSG_COMPLETE here. Upstream will take care of that.
           if(LOG_ENABLED(INFO)) {
@@ -1122,6 +1190,14 @@ int on_data_chunk_recv_callback(nghttp2_session *session,
     return 0;
   }
 
+  // We don't want DATA after non-final response, which is illegal in
+  // HTTP.
+  if(downstream->get_non_final_response()) {
+    http2session->submit_rst_stream(stream_id, NGHTTP2_PROTOCOL_ERROR);
+    http2session->handle_ign_data_chunk(len);
+    return 0;
+  }
+
   downstream->add_response_bodylen(len);
 
   auto upstream = downstream->get_upstream();

src/shrpx_http2_upstream.cc

@@ -1153,9 +1153,16 @@ nghttp2_session* Http2Upstream::get_http2_session()
 // nghttp2_session_recv. These calls may delete downstream.
 int Http2Upstream::on_downstream_header_complete(Downstream *downstream)
 {
+  int rv;
+
   if(LOG_ENABLED(INFO)) {
+    if(downstream->get_non_final_response()) {
+      DLOG(INFO, downstream) << "HTTP non-final response header";
+    } else {
       DLOG(INFO, downstream) << "HTTP response header completed";
     }
+  }
+
   downstream->normalize_response_headers();
   if(!get_config()->http2_proxy && !get_config()->client_proxy) {
     downstream->rewrite_norm_location_response_header
@@ -1172,6 +1179,22 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream)
   nva.push_back(http2::make_nv_ls(":status", response_status));
 
   http2::copy_norm_headers_to_nva(nva, downstream->get_response_headers());
+
+  if(downstream->get_non_final_response()) {
+    rv = nghttp2_submit_headers(session_, NGHTTP2_FLAG_NONE,
+                                downstream->get_stream_id(), nullptr,
+                                nva.data(), nva.size(), nullptr);
+
+    downstream->clear_response_headers();
+
+    if(rv != 0) {
+      ULOG(FATAL, this) << "nghttp2_submit_headers() failed";
+      return -1;
+    }
+
+    return 0;
+  }
+
   auto via = downstream->get_norm_response_header("via");
   if(get_config()->no_via) {
     if(via != end_headers) {
@@ -1214,7 +1237,6 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream)
   data_prd.source.ptr = downstream;
   data_prd.read_callback = downstream_data_read_callback;
 
-  int rv;
   rv = nghttp2_submit_response(session_, downstream->get_stream_id(),
                                nva.data(), nva.size(), &data_prd);
   if(rv != 0) {

src/shrpx_http_downstream_connection.cc

@@ -408,9 +408,26 @@ namespace {
 int htp_hdrs_completecb(http_parser *htp)
 {
   auto downstream = static_cast<Downstream*>(htp->data);
+  auto upstream = downstream->get_upstream();
+  int rv;
+
   downstream->set_response_http_status(htp->status_code);
   downstream->set_response_major(htp->http_major);
   downstream->set_response_minor(htp->http_minor);
+
+  if(downstream->get_non_final_response()) {
+    // For non-final response code, we just call
+    // on_downstream_header_complete() without changing response
+    // state.
+    rv = upstream->on_downstream_header_complete(downstream);
+
+    if(rv != 0) {
+      return -1;
+    }
+
+    return 0;
+  }
+
   downstream->set_response_connection_close(!http_should_keep_alive(htp));
   downstream->set_response_state(Downstream::HEADER_COMPLETE);
   downstream->inspect_http1_response();
@@ -418,15 +435,13 @@ int htp_hdrs_completecb(http_parser *htp)
   if(downstream->get_upgraded()) {
     downstream->set_response_connection_close(true);
   }
-  if(downstream->get_upstream()->on_downstream_header_complete(downstream)
+  if(upstream->on_downstream_header_complete(downstream) != 0) {
-     != 0) {
     return -1;
   }
 
   if(downstream->get_upgraded()) {
     // Upgrade complete, read until EOF in both ends
-    if(downstream->get_upstream()->resume_read(SHRPX_MSG_BLOCK,
+    if(upstream->resume_read(SHRPX_MSG_BLOCK, downstream) != 0) {
-                                               downstream) != 0) {
       return -1;
     }
     downstream->set_request_state(Downstream::HEADER_COMPLETE);
@@ -443,6 +458,9 @@ int htp_hdrs_completecb(http_parser *htp)
   // 304 status code with nonzero Content-Length, but without response
   // body. See
   // http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-20#section-3.3
+
+  // TODO It seems that the cases other than HEAD are handled by
+  // http-parser.  Need test.
   return downstream->get_request_method() == "HEAD" ||
     (100 <= status && status <= 199) || status == 204 ||
     status == 304 ? 1 : 0;
@@ -505,6 +523,13 @@ namespace {
 int htp_msg_completecb(http_parser *htp)
 {
   auto downstream = static_cast<Downstream*>(htp->data);
+
+  if(downstream->get_non_final_response()) {
+    downstream->reset_response();
+
+    return 0;
+  }
+
   downstream->set_response_state(Downstream::MSG_COMPLETE);
   // Block reading another response message from (broken?)
   // server. This callback is not called if the connection is

src/shrpx_https_upstream.cc

@@ -190,16 +190,6 @@ int htp_hdrs_completecb(http_parser *htp)
 
   auto dconn = upstream->get_client_handler()->get_downstream_connection();
 
-  if(downstream->get_expect_100_continue()) {
-    static const char reply_100[] = "HTTP/1.1 100 Continue\r\n\r\n";
-    if(bufferevent_write(upstream->get_client_handler()->get_bev(),
-                         reply_100, sizeof(reply_100)-1) != 0) {
-      ULOG(FATAL, upstream) << "bufferevent_write() faild";
-      delete dconn;
-      return -1;
-    }
-  }
-
   rv =  dconn->attach_downstream(downstream);
 
   if(rv != 0) {
@@ -740,8 +730,12 @@ Downstream* HttpsUpstream::pop_downstream()
 int HttpsUpstream::on_downstream_header_complete(Downstream *downstream)
 {
   if(LOG_ENABLED(INFO)) {
+    if(downstream->get_non_final_response()) {
+      DLOG(INFO, downstream) << "HTTP non-final response header";
+    } else {
       DLOG(INFO, downstream) << "HTTP response header completed";
     }
+  }
 
   std::string hdrs = "HTTP/";
   hdrs += util::utos(downstream->get_request_major());
@@ -759,6 +753,20 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream)
   http2::build_http1_headers_from_norm_headers
     (hdrs, downstream->get_response_headers());
 
+  if(downstream->get_non_final_response()) {
+    hdrs += "\r\n";
+
+    auto output = bufferevent_get_output(handler_->get_bev());
+    if(evbuffer_add(output, hdrs.c_str(), hdrs.size()) != 0) {
+      ULOG(FATAL, this) << "evbuffer_add() failed";
+      return -1;
+    }
+
+    downstream->clear_response_headers();
+
+    return 0;
+  }
+
   // We check downstream->get_response_connection_close() in case when
   // the Content-Length is not available.
   if(!downstream->get_request_connection_close() &&

src/shrpx_spdy_upstream.cc

@@ -886,6 +886,15 @@ spdylay_session* SpdyUpstream::get_http2_session()
 // spdylay_session_recv. These calls may delete downstream.
 int SpdyUpstream::on_downstream_header_complete(Downstream *downstream)
 {
+  if(downstream->get_non_final_response()) {
+    // SPDY does not support non-final response.  We could send it
+    // with HEADERS and final response in SYN_REPLY, but it is not
+    // official way.
+    downstream->clear_response_headers();
+
+    return 0;
+  }
+
   if(LOG_ENABLED(INFO)) {
     DLOG(INFO, downstream) << "HTTP response header completed";
   }