src: Avoid copy in evbuffer_pullup()

Previously, we use evbuffer_pullup(buf, -1) to linearize the memory
region and it may cause buffer copy.  To avoid this, we use the return
value of evbuffer_get_contiguous_space() as 2nd parameter.  According
to the libevent manual, by doing so evbuffer_pullup() will not copy or
modify any data in evbuffer.

src/h2load_http2_session.cc

@@ -153,16 +153,33 @@ void Http2Session::submit_request()
 ssize_t Http2Session::on_read()
 {
   int rv;
-  auto input = bufferevent_get_input(client_->bev);
+  size_t nread = 0;
-  auto inputlen = evbuffer_get_length(input);
-  auto mem = evbuffer_pullup(input, -1);
 
-  rv = nghttp2_session_mem_recv(session_, mem, inputlen);
+  auto input = bufferevent_get_input(client_->bev);
-  if(rv < 0) {
+
-    return -1;
+  for(;;) {
+    auto inputlen = evbuffer_get_contiguous_space(input);
+
+    if(inputlen == 0) {
+      assert(evbuffer_get_length(input) == 0);
+
+      return nread;
+    }
+
+    auto mem = evbuffer_pullup(input, inputlen);
+
+    rv = nghttp2_session_mem_recv(session_, mem, inputlen);
+
+    if(rv < 0) {
+      return -1;
+    }
+
+    nread += rv;
+
+    if(evbuffer_drain(input, rv) != 0) {
+      return -1;
+    }
   }
-  evbuffer_drain(input, rv);
-  return rv;
 }
 
 int Http2Session::on_write()

src/h2load_spdy_session.cc

@@ -24,6 +24,8 @@
  */
 #include "h2load_spdy_session.h"
 
+#include <cassert>
+
 #include "h2load.h"
 
 namespace h2load {
@@ -163,16 +165,32 @@ void SpdySession::submit_request()
 ssize_t SpdySession::on_read()
 {
   int rv;
+  size_t nread = 0;
   auto input = bufferevent_get_input(client_->bev);
-  auto inputlen = evbuffer_get_length(input);
-  auto mem = evbuffer_pullup(input, -1);
 
-  rv = spdylay_session_mem_recv(session_, mem, inputlen);
+  for(;;) {
-  if(rv < 0) {
+    auto inputlen = evbuffer_get_contiguous_space(input);
-    return -1;
+
+    if(inputlen == 0) {
+      assert(evbuffer_get_length(input) == 0);
+
+      return nread;
+    }
+
+    auto mem = evbuffer_pullup(input, inputlen);
+
+    rv = spdylay_session_mem_recv(session_, mem, inputlen);
+
+    if(rv < 0) {
+      return -1;
+    }
+
+    nread += rv;
+
+    if(evbuffer_drain(input, rv) != 0) {
+      return -1;
+    }
   }
-  evbuffer_drain(input, rv);
-  return rv;
 }
 
 int SpdySession::on_write()

src/nghttp.cc

@@ -651,53 +651,75 @@ struct HttpClient {
   {
     int rv;
     auto input = bufferevent_get_input(bev);
-    auto inputlen = evbuffer_get_length(input);
-    if(inputlen == 0) {
-      return 0;
-    }
-    auto mem = evbuffer_pullup(input, -1);
-    auto nread = http_parser_execute(htp.get(), &htp_hooks,
-                                     reinterpret_cast<const char*>(mem),
-                                     inputlen);
-    if(config.verbose) {
-      std::cout.write(reinterpret_cast<const char*>(mem), nread);
-    }
-    evbuffer_drain(input, nread);
 
-    auto htperr = HTTP_PARSER_ERRNO(htp.get());
+    for(;;) {
-    if(htperr == HPE_OK) {
+      auto inputlen = evbuffer_get_contiguous_space(input);
+
+      if(inputlen == 0) {
+        assert(evbuffer_get_length(input) == 0);
+
+        return 0;
+      }
+
+      auto mem = evbuffer_pullup(input, inputlen);
+
+      auto nread = http_parser_execute(htp.get(), &htp_hooks,
+                                       reinterpret_cast<const char*>(mem),
+                                       inputlen);
+
+      if(config.verbose) {
+        std::cout.write(reinterpret_cast<const char*>(mem), nread);
+      }
+
+      if(evbuffer_drain(input, nread) != 0) {
+        return -1;
+      }
+
+      auto htperr = HTTP_PARSER_ERRNO(htp.get());
+
+      if(htperr != HPE_OK) {
+        std::cerr << "Failed to parse HTTP Upgrade response header: "
+                  << "(" << http_errno_name(htperr) << ") "
+                  << http_errno_description(htperr) << std::endl;
+        return -1;
+      }
+
       if(upgrade_response_complete) {
+
         if(config.verbose) {
           std::cout << std::endl;
         }
+
         if(upgrade_response_status_code == 101) {
           if(config.verbose) {
             print_timer();
             std::cout << " HTTP Upgrade success" << std::endl;
           }
+
           bufferevent_setcb(bev, readcb, writecb, eventcb, this);
+
           rv = on_connect();
+
           if(rv != 0) {
             return rv;
           }
+
           // Read remaining data in the buffer because it is not
           // notified callback anymore.
           rv = on_read();
+
           if(rv != 0) {
             return rv;
           }
-        } else {
+
-          std::cerr << "HTTP Upgrade failed" << std::endl;
+          return 0;
-          return -1;
         }
+
+        std::cerr << "HTTP Upgrade failed" << std::endl;
+
+        return -1;
       }
-    } else {
-      std::cerr << "Failed to parse HTTP Upgrade response header: "
-                << "(" << http_errno_name(htperr) << ") "
-                << http_errno_description(htperr) << std::endl;
-      return -1;
     }
-    return 0;
   }
 
   int on_connect()
@@ -776,18 +798,30 @@ struct HttpClient {
   {
     int rv;
     auto input = bufferevent_get_input(bev);
-    auto inputlen = evbuffer_get_length(input);
-    auto mem = evbuffer_pullup(input, -1);
 
-    rv = nghttp2_session_mem_recv(session, mem, inputlen);
+    for(;;) {
-    if(rv < 0) {
+      auto inputlen = evbuffer_get_contiguous_space(input);
-      std::cerr << "nghttp2_session_mem_recv() returned error: "
+
-                << nghttp2_strerror(rv) << std::endl;
+      if(inputlen == 0) {
-      return -1;
+        assert(evbuffer_get_length(input) == 0);
+
+        return on_write();
+      }
+
+      auto mem = evbuffer_pullup(input, inputlen);
+
+      rv = nghttp2_session_mem_recv(session, mem, inputlen);
+
+      if(rv < 0) {
+        std::cerr << "nghttp2_session_mem_recv() returned error: "
+                  << nghttp2_strerror(rv) << std::endl;
+        return -1;
+      }
+
+      if(evbuffer_drain(input, rv) != 0) {
+        return -1;
+      }
     }
-    evbuffer_drain(input, rv);
-
-    return on_write();
   }
 
   int on_write()

src/shrpx_http2_session.cc

@@ -541,21 +541,32 @@ http_parser_settings htp_hooks = {
 int Http2Session::on_read_proxy()
 {
   auto input = bufferevent_get_input(bev_);
-  auto mem = evbuffer_pullup(input, -1);
 
-  size_t nread = http_parser_execute(proxy_htp_.get(), &htp_hooks,
+  for(;;) {
-                                     reinterpret_cast<const char*>(mem),
+    auto inputlen = evbuffer_get_contiguous_space(input);
-                                     evbuffer_get_length(input));
 
-  if(evbuffer_drain(input, nread) != 0) {
+    if(inputlen == 0) {
-    SSLOG(FATAL, this) << "evbuffer_drain() failed";
+      assert(evbuffer_get_length(input) == 0);
-    return -1;
+
-  }
+      return 0;
-  auto htperr = HTTP_PARSER_ERRNO(proxy_htp_.get());
+    }
-  if(htperr == HPE_OK) {
+
-    return 0;
+    auto mem = evbuffer_pullup(input, inputlen);
-  } else {
+
-    return -1;
+    size_t nread = http_parser_execute(proxy_htp_.get(), &htp_hooks,
+                                       reinterpret_cast<const char*>(mem),
+                                       inputlen);
+
+    if(evbuffer_drain(input, nread) != 0) {
+      SSLOG(FATAL, this) << "evbuffer_drain() failed";
+      return -1;
+    }
+
+    auto htperr = HTTP_PARSER_ERRNO(proxy_htp_.get());
+
+    if(htperr != HPE_OK) {
+      return -1;
+    }
   }
 }
 
@@ -1250,17 +1261,31 @@ int Http2Session::on_read()
 {
   ssize_t rv = 0;
   auto input = bufferevent_get_input(bev_);
-  auto inputlen = evbuffer_get_length(input);
-  auto mem = evbuffer_pullup(input, -1);
 
-  rv = nghttp2_session_mem_recv(session_, mem, inputlen);
+  for(;;) {
-  if(rv < 0) {
+    auto inputlen = evbuffer_get_contiguous_space(input);
-    SSLOG(ERROR, this) << "nghttp2_session_recv() returned error: "
+
-                       << nghttp2_strerror(rv);
+    if(inputlen == 0) {
-    return -1;
+      assert(evbuffer_get_length(input) == 0);
+
+      return send();
+    }
+
+    auto mem = evbuffer_pullup(input, inputlen);
+
+    rv = nghttp2_session_mem_recv(session_, mem, inputlen);
+
+    if(rv < 0) {
+      SSLOG(ERROR, this) << "nghttp2_session_recv() returned error: "
+                         << nghttp2_strerror(rv);
+      return -1;
+    }
+
+    if(evbuffer_drain(input, rv) != 0) {
+      SSLOG(FATAL, this) << "evbuffer_drain() faild";
+      return -1;
+    }
   }
-  evbuffer_drain(input, rv);
-  return send();
 }
 
 int Http2Session::on_write()

src/shrpx_http2_upstream.cc

@@ -594,17 +594,30 @@ int Http2Upstream::on_read()
   ssize_t rv = 0;
   auto bev = handler_->get_bev();
   auto input = bufferevent_get_input(bev);
-  auto inputlen = evbuffer_get_length(input);
-  auto mem = evbuffer_pullup(input, -1);
 
-  rv = nghttp2_session_mem_recv(session_, mem, inputlen);
+  for(;;) {
-  if(rv < 0) {
+    auto inputlen = evbuffer_get_contiguous_space(input);
-    ULOG(ERROR, this) << "nghttp2_session_recv() returned error: "
+
-                      << nghttp2_strerror(rv);
+    if(inputlen == 0) {
-    return -1;
+      assert(evbuffer_get_length(input) == 0);
+
+      return send();
+    }
+
+    auto mem = evbuffer_pullup(input, inputlen);
+
+    rv = nghttp2_session_mem_recv(session_, mem, inputlen);
+    if(rv < 0) {
+      ULOG(ERROR, this) << "nghttp2_session_recv() returned error: "
+                        << nghttp2_strerror(rv);
+      return -1;
+    }
+
+    if(evbuffer_drain(input, rv) != 0) {
+      DCLOG(FATAL, this) << "evbuffer_drain() failed";
+      return -1;
+    }
   }
-  evbuffer_drain(input, rv);
-  return send();
 }
 
 int Http2Upstream::on_write()

src/shrpx_http_downstream_connection.cc

@@ -502,40 +502,64 @@ http_parser_settings htp_hooks = {
 int HttpDownstreamConnection::on_read()
 {
   auto input = bufferevent_get_input(bev_);
-  size_t inputlen = evbuffer_get_length(input);
+
-  auto mem = evbuffer_pullup(input, -1);
   if(downstream_->get_upgraded()) {
     // For upgraded connection, just pass data to the upstream.
-    int rv;
+    for(;;) {
-    rv = downstream_->get_upstream()->on_downstream_body
+      auto inputlen = evbuffer_get_contiguous_space(input);
-      (downstream_, reinterpret_cast<const uint8_t*>(mem), inputlen, true);
+
-    if(rv != 0) {
+      if(inputlen == 0) {
-      return rv;
+        assert(evbuffer_get_length(input) == 0);
+
+        return 0;
+      }
+
+      auto mem = evbuffer_pullup(input, inputlen);
+
+      int rv;
+      rv = downstream_->get_upstream()->on_downstream_body
+        (downstream_, reinterpret_cast<const uint8_t*>(mem), inputlen, true);
+      if(rv != 0) {
+        return rv;
+      }
+      if(evbuffer_drain(input, inputlen) != 0) {
+        DCLOG(FATAL, this) << "evbuffer_drain() failed";
+        return -1;
+      }
     }
-    if(evbuffer_drain(input, inputlen) != 0) {
-      DCLOG(FATAL, this) << "evbuffer_drain() failed";
-      return -1;
-    }
-    return 0;
   }
-  size_t nread = http_parser_execute(&response_htp_, &htp_hooks,
+
+
+  for(;;) {
+    auto inputlen = evbuffer_get_contiguous_space(input);
+
+    if(inputlen == 0) {
+      assert(evbuffer_get_length(input) == 0);
+      return 0;
+    }
+
+    auto mem = evbuffer_pullup(input, inputlen);
+
+    auto nread = http_parser_execute(&response_htp_, &htp_hooks,
                                      reinterpret_cast<const char*>(mem),
                                      inputlen);
 
-  if(evbuffer_drain(input, nread) != 0) {
+    if(evbuffer_drain(input, nread) != 0) {
-    DCLOG(FATAL, this) << "evbuffer_drain() failed";
+      DCLOG(FATAL, this) << "evbuffer_drain() failed";
-    return -1;
+      return -1;
-  }
+    }
-  auto htperr = HTTP_PARSER_ERRNO(&response_htp_);
+
-  if(htperr == HPE_OK) {
+    auto htperr = HTTP_PARSER_ERRNO(&response_htp_);
-    return 0;
+
-  } else {
+    if(htperr != HPE_OK) {
-    if(LOG_ENABLED(INFO)) {
+      if(LOG_ENABLED(INFO)) {
-      DCLOG(INFO, this) << "HTTP parser failure: "
+        DCLOG(INFO, this) << "HTTP parser failure: "
-                        << "(" << http_errno_name(htperr) << ") "
+                          << "(" << http_errno_name(htperr) << ") "
-                        << http_errno_description(htperr);
+                          << http_errno_description(htperr);
+      }
+
+      return SHRPX_ERR_HTTP_PARSE;
     }
-    return SHRPX_ERR_HTTP_PARSE;
   }
 }
 

src/shrpx_https_upstream.cc

@@ -264,97 +264,139 @@ int HttpsUpstream::on_read()
 {
   auto bev = handler_->get_bev();
   auto input = bufferevent_get_input(bev);
-  size_t inputlen = evbuffer_get_length(input);
-  auto mem = evbuffer_pullup(input, -1);
-
-  if(inputlen == 0) {
-    return 0;
-  }
   auto downstream = get_downstream();
+
   // downstream can be nullptr here, because it is initialized in the
   // callback chain called by http_parser_execute()
   if(downstream && downstream->get_upgraded()) {
-    int rv = downstream->push_upload_data_chunk
+    for(;;) {
-      (reinterpret_cast<const uint8_t*>(mem), inputlen);
+      auto inputlen = evbuffer_get_contiguous_space(input);
-    if(rv != 0) {
-      return -1;
-    }
-    if(evbuffer_drain(input, inputlen) != 0) {
-      ULOG(FATAL, this) << "evbuffer_drain() failed";
-      return -1;
-    }
-    if(downstream->get_output_buffer_full()) {
-      if(LOG_ENABLED(INFO)) {
-        ULOG(INFO, this) << "Downstream output buffer is full";
-      }
-      pause_read(SHRPX_NO_BUFFER);
-    }
-    return 0;
-  }
 
-  size_t nread = http_parser_execute(&htp_, &htp_hooks,
+      if(inputlen == 0) {
-                                     reinterpret_cast<const char*>(mem),
+        return 0;
-                                     inputlen);
+      }
-  if(evbuffer_drain(input, nread) != 0) {
+
-    ULOG(FATAL, this) << "evbuffer_drain() failed";
+      auto mem = evbuffer_pullup(input, inputlen);
-    return -1;
+
-  }
+      auto rv = downstream->push_upload_data_chunk
-  // Well, actually header length + some body bytes
+        (reinterpret_cast<const uint8_t*>(mem), inputlen);
-  current_header_length_ += nread;
+
-  // Get downstream again because it may be initialized in http parser
+      if(rv != 0) {
-  // execution
-  downstream = get_downstream();
-  auto handler = get_client_handler();
-  auto htperr = HTTP_PARSER_ERRNO(&htp_);
-  if(htperr == HPE_PAUSED) {
-    if(downstream->get_request_state() == Downstream::CONNECT_FAIL) {
-      handler->set_should_close_after_write(true);
-      // Following paues_read is needed to avoid reading next data.
-      pause_read(SHRPX_MSG_BLOCK);
-      if(error_reply(503) != 0) {
         return -1;
       }
-      // Downstream gets deleted after response body is read.
+
-    } else {
+      if(evbuffer_drain(input, inputlen) != 0) {
-      assert(downstream->get_request_state() == Downstream::MSG_COMPLETE);
+        ULOG(FATAL, this) << "evbuffer_drain() failed";
-      if(downstream->get_downstream_connection() == 0) {
+        return -1;
-        // Error response has already be sent
-        assert(downstream->get_response_state() == Downstream::MSG_COMPLETE);
-        delete_downstream();
-      } else {
-        if(handler->get_http2_upgrade_allowed() &&
-           downstream->http2_upgrade_request()) {
-          if(handler->perform_http2_upgrade(this) != 0) {
-            return -1;
-          }
-          return 0;
-        }
-        pause_read(SHRPX_MSG_BLOCK);
       }
-    }
+
-  } else if(htperr == HPE_OK) {
-    // downstream can be NULL here.
-    if(downstream) {
       if(downstream->get_output_buffer_full()) {
         if(LOG_ENABLED(INFO)) {
           ULOG(INFO, this) << "Downstream output buffer is full";
         }
         pause_read(SHRPX_NO_BUFFER);
+
+        return 0;
       }
     }
-  } else {
+  }
-    if(LOG_ENABLED(INFO)) {
+
-      ULOG(INFO, this) << "HTTP parse failure: "
+  for(;;) {
-                       << "(" << http_errno_name(htperr) << ") "
+    auto inputlen = evbuffer_get_contiguous_space(input);
-                       << http_errno_description(htperr);
+
+    if(inputlen == 0) {
+      return 0;
     }
-    handler->set_should_close_after_write(true);
+
-    pause_read(SHRPX_MSG_BLOCK);
+    auto mem = evbuffer_pullup(input, inputlen);
-    if(error_reply(400) != 0) {
+
+    auto nread = http_parser_execute(&htp_, &htp_hooks,
+                                     reinterpret_cast<const char*>(mem),
+                                     inputlen);
+
+    if(evbuffer_drain(input, nread) != 0) {
+      ULOG(FATAL, this) << "evbuffer_drain() failed";
       return -1;
     }
+
+    // Well, actually header length + some body bytes
+    current_header_length_ += nread;
+
+    // Get downstream again because it may be initialized in http parser
+    // execution
+    downstream = get_downstream();
+
+    auto handler = get_client_handler();
+    auto htperr = HTTP_PARSER_ERRNO(&htp_);
+
+    if(htperr == HPE_PAUSED) {
+
+      assert(downstream);
+
+      if(downstream->get_request_state() == Downstream::CONNECT_FAIL) {
+        handler->set_should_close_after_write(true);
+        // Following paues_read is needed to avoid reading next data.
+        pause_read(SHRPX_MSG_BLOCK);
+        if(error_reply(503) != 0) {
+          return -1;
+        }
+        // Downstream gets deleted after response body is read.
+        return 0;
+      }
+
+      assert(downstream->get_request_state() == Downstream::MSG_COMPLETE);
+
+      if(downstream->get_downstream_connection() == nullptr) {
+        // Error response has already be sent
+        assert(downstream->get_response_state() == Downstream::MSG_COMPLETE);
+        delete_downstream();
+
+        return 0;
+      }
+
+      if(handler->get_http2_upgrade_allowed() &&
+         downstream->http2_upgrade_request()) {
+
+        if(handler->perform_http2_upgrade(this) != 0) {
+          return -1;
+        }
+
+        return 0;
+      }
+
+      pause_read(SHRPX_MSG_BLOCK);
+
+      return 0;
+    }
+
+    if(htperr != HPE_OK) {
+      if(LOG_ENABLED(INFO)) {
+        ULOG(INFO, this) << "HTTP parse failure: "
+                         << "(" << http_errno_name(htperr) << ") "
+                         << http_errno_description(htperr);
+      }
+
+      handler->set_should_close_after_write(true);
+      pause_read(SHRPX_MSG_BLOCK);
+
+      if(error_reply(400) != 0) {
+        return -1;
+      }
+
+      return 0;
+    }
+
+    // downstream can be NULL here.
+    if(downstream && downstream->get_output_buffer_full()) {
+      if(LOG_ENABLED(INFO)) {
+        ULOG(INFO, this) << "Downstream output buffer is full";
+      }
+
+      pause_read(SHRPX_NO_BUFFER);
+
+      return 0;
+    }
   }
-  return 0;
 }
 
 int HttpsUpstream::on_write()