src: Avoid to call costly evbuffer_add() repeatedly

The profiler and benchmarking showed that calling evbuffer_add()
repeatedly is very costly. To avoid this, we buffer up small writes
into one large chunk and call evbuffer_add() less times.

src/HttpServer.cc

@@ -332,10 +332,12 @@ int Http2Handler::on_read()
 int Http2Handler::on_write()
 {
   int rv;
+  uint8_t buf[4096];
+  size_t buflen = 0;
   auto output = bufferevent_get_output(bev_);
 
   for(;;) {
-    if(evbuffer_get_length(output) >
+    if(evbuffer_get_length(output) + buflen >
        sessions_->get_config()->output_upper_thres) {
       break;
     }
@@ -351,11 +353,29 @@ int Http2Handler::on_write()
     if(datalen == 0) {
       break;
     }
-    rv = evbuffer_add(output, data, datalen);
-    if(rv != 0) {
-      std::cerr << "evbuffer_add() failed" << std::endl;
-      return -1;
+    if(buflen + datalen > sizeof(buf)) {
+      rv = evbuffer_add(output, buf, buflen);
+      if(rv != 0) {
+        std::cerr << "evbuffer_add() failed" << std::endl;
+        return -1;
+      }
+      buflen = 0;
+      if(datalen > static_cast<ssize_t>(sizeof(buf))) {
+        rv = evbuffer_add(output, data, datalen);
+        if(rv != 0) {
+          std::cerr << "evbuffer_add() failed" << std::endl;
+          return -1;
+        }
+        continue;
+      }
     }
+    memcpy(buf + buflen, data, datalen);
+    buflen += datalen;
+  }
+  rv = evbuffer_add(output, buf, buflen);
+  if(rv != 0) {
+    std::cerr << "evbuffer_add() failed" << std::endl;
+    return -1;
   }
   if(nghttp2_session_want_read(session_) == 0 &&
      nghttp2_session_want_write(session_) == 0 &&

src/h2load_http2_session.cc

@@ -162,6 +162,8 @@ ssize_t Http2Session::on_read()
 int Http2Session::on_write()
 {
   int rv;
+  uint8_t buf[4096];
+  size_t buflen = 0;
   auto output = bufferevent_get_output(client_->bev);
   for(;;) {
     const uint8_t *data;
@@ -173,10 +175,25 @@ int Http2Session::on_write()
     if(datalen == 0) {
       break;
     }
-    rv = evbuffer_add(output, data, datalen);
-    if(rv == -1) {
-      return -1;
+    if(buflen + datalen > sizeof(buf)) {
+      rv = evbuffer_add(output, buf, buflen);
+      if(rv == -1) {
+        return -1;
+      }
+      buflen = 0;
+      if(datalen > static_cast<ssize_t>(sizeof(buf))) {
+        rv = evbuffer_add(output, data, datalen);
+        if(rv == -1) {
+          return -1;
+        }
+      }
     }
+    memcpy(buf + buflen, data, datalen);
+    buflen += datalen;
+  }
+  rv = evbuffer_add(output, buf, buflen);
+  if(rv == -1) {
+    return -1;
   }
   if(nghttp2_session_want_read(session_) == 0 &&
      nghttp2_session_want_write(session_) == 0 &&

src/h2load_spdy_session.cc

@@ -101,8 +101,29 @@ ssize_t send_callback(spdylay_session *session,
                       void *user_data)
 {
   auto client = static_cast<Client*>(user_data);
+  auto spdy_session = static_cast<SpdySession*>(client->session.get());
   auto output = bufferevent_get_output(client->bev);
-  evbuffer_add(output, data, length);
+  int rv;
+
+  if(spdy_session->sendbuflen + length > spdy_session->sendbufmax) {
+    rv = evbuffer_add(output, spdy_session->sendbuf, spdy_session->sendbuflen);
+    if(rv == -1) {
+      return SPDYLAY_ERR_CALLBACK_FAILURE;
+    }
+    spdy_session->sendbuflen = 0;
+    if(length > spdy_session->sendbufmax) {
+      rv = evbuffer_add(output, data, length);
+      if(rv == -1) {
+        return SPDYLAY_ERR_CALLBACK_FAILURE;
+      }
+    } else {
+      memcpy(spdy_session->sendbuf + spdy_session->sendbuflen, data, length);
+      spdy_session->sendbuflen += length;
+    }
+  } else {
+    memcpy(spdy_session->sendbuf + spdy_session->sendbuflen, data, length);
+    spdy_session->sendbuflen += length;
+  }
   return length;
 }
 } //namespace
@@ -150,10 +171,22 @@ ssize_t SpdySession::on_read()
 int SpdySession::on_write()
 {
   int rv;
+  uint8_t buf[4096];
+
+  sendbuf = buf;
+  sendbuflen = 0;
+  sendbufmax = sizeof(buf);
+
   rv = spdylay_session_send(session_);
   if(rv != 0) {
     return -1;
   }
+
+  rv = bufferevent_write(client_->bev, sendbuf, sendbuflen);
+  if(rv == -1) {
+    return -1;
+  }
+
   if(spdylay_session_want_read(session_) == 0 &&
      spdylay_session_want_write(session_) == 0 &&
      evbuffer_get_length(bufferevent_get_output(client_->bev)) == 0) {

src/h2load_spdy_session.h

@@ -42,6 +42,10 @@ public:
   virtual ssize_t on_read();
   virtual int on_write();
   virtual void terminate();
+
+  uint8_t *sendbuf;
+  size_t sendbuflen;
+  size_t sendbufmax;
 private:
   Client *client_;
   spdylay_session *session_;

src/shrpx_http2_session.cc

@@ -1258,10 +1258,12 @@ int Http2Session::on_write()
 int Http2Session::send()
 {
   int rv;
+  uint8_t buf[4096];
+  size_t buflen = 0;
   auto output = bufferevent_get_output(bev_);
   for(;;) {
     // Check buffer length and return WOULDBLOCK if it is large enough.
-    if(evbuffer_get_length(output) > Http2Session::OUTBUF_MAX_THRES) {
+    if(evbuffer_get_length(output) + buflen > Http2Session::OUTBUF_MAX_THRES) {
       return NGHTTP2_ERR_WOULDBLOCK;
     }
 
@@ -1276,11 +1278,30 @@ int Http2Session::send()
     if(datalen == 0) {
       break;
     }
-    rv = evbuffer_add(output, data, datalen);
-    if(rv == -1) {
-      SSLOG(FATAL, this) << "evbuffer_add() failed";
-      return -1;
+    if(buflen + datalen > sizeof(buf)) {
+      rv = evbuffer_add(output, buf, buflen);
+      if(rv == -1) {
+        SSLOG(FATAL, this) << "evbuffer_add() failed";
+        return -1;
+      }
+      buflen = 0;
+      if(datalen > static_cast<ssize_t>(sizeof(buf))) {
+        rv = evbuffer_add(output, data, datalen);
+        if(rv == -1) {
+          SSLOG(FATAL, this) << "evbuffer_add() failed";
+          return -1;
+        }
+        continue;
+      }
     }
+    memcpy(buf + buflen, data, datalen);
+    buflen += datalen;
+  }
+
+  rv = evbuffer_add(output, buf, buflen);
+  if(rv == -1) {
+    SSLOG(FATAL, this) << "evbuffer_add() failed";
+    return -1;
   }
 
   if(nghttp2_session_want_read(session_) == 0 &&

src/shrpx_http2_upstream.cc

@@ -566,11 +566,13 @@ int Http2Upstream::on_write()
 int Http2Upstream::send()
 {
   int rv;
+  uint8_t buf[4096];
+  size_t buflen = 0;
   auto bev = handler_->get_bev();
   auto output = bufferevent_get_output(bev);
   for(;;) {
     // Check buffer length and return WOULDBLOCK if it is large enough.
-    if(handler_->get_outbuf_length() > OUTBUF_MAX_THRES) {
+    if(handler_->get_outbuf_length() + buflen > OUTBUF_MAX_THRES) {
       break;
     }
 
@@ -585,12 +587,32 @@ int Http2Upstream::send()
     if(datalen == 0) {
       break;
     }
-    rv = evbuffer_add(output, data, datalen);
-    if(rv == -1) {
-      ULOG(FATAL, this) << "evbuffer_add() failed";
-      return -1;
+    if(buflen + datalen > sizeof(buf)) {
+      rv = evbuffer_add(output, buf, buflen);
+      if(rv == -1) {
+        ULOG(FATAL, this) << "evbuffer_add() failed";
+        return -1;
+      }
+      buflen = 0;
+      if(datalen > static_cast<ssize_t>(sizeof(buf))) {
+        rv = evbuffer_add(output, data, datalen);
+        if(rv == -1) {
+          ULOG(FATAL, this) << "evbuffer_add() failed";
+          return -1;
+        }
+        continue;
+      }
     }
+    memcpy(buf + buflen, data, datalen);
+    buflen += datalen;
   }
+
+  rv = evbuffer_add(output, buf, buflen);
+  if(rv == -1) {
+    ULOG(FATAL, this) << "evbuffer_add() failed";
+    return -1;
+  }
+
   if(nghttp2_session_want_read(session_) == 0 &&
      nghttp2_session_want_write(session_) == 0 &&
      handler_->get_outbuf_length() == 0) {

src/shrpx_spdy_upstream.cc

@@ -58,18 +58,34 @@ ssize_t send_callback(spdylay_session *session,
   auto handler = upstream->get_client_handler();
   auto bev = handler->get_bev();
   auto output = bufferevent_get_output(bev);
+
   // Check buffer length and return WOULDBLOCK if it is large enough.
-  if(handler->get_outbuf_length() > OUTBUF_MAX_THRES) {
+  if(handler->get_outbuf_length() + upstream->sendbuflen > OUTBUF_MAX_THRES) {
     return SPDYLAY_ERR_WOULDBLOCK;
   }
 
-  rv = evbuffer_add(output, data, len);
-  if(rv == -1) {
-    ULOG(FATAL, upstream) << "evbuffer_add() failed";
-    return SPDYLAY_ERR_CALLBACK_FAILURE;
+  if(upstream->sendbuflen + len > upstream->sendbufmax) {
+    rv = evbuffer_add(output, upstream->sendbuf, upstream->sendbuflen);
+    if(rv == -1) {
+      ULOG(FATAL, upstream) << "evbuffer_add() failed";
+      return SPDYLAY_ERR_CALLBACK_FAILURE;
+    }
+    upstream->sendbuflen = 0;
+    if(len > upstream->sendbufmax) {
+      rv = evbuffer_add(output, data, len);
+      if(rv == -1) {
+        ULOG(FATAL, upstream) << "evbuffer_add() failed";
+        return SPDYLAY_ERR_CALLBACK_FAILURE;
+      }
+    } else {
+      memcpy(upstream->sendbuf + upstream->sendbuflen, data, len);
+      upstream->sendbuflen += len;
+    }
   } else {
-    return len;
+    memcpy(upstream->sendbuf + upstream->sendbuflen, data, len);
+    upstream->sendbuflen += len;
   }
+  return len;
 }
 } // namespace
 
@@ -439,26 +455,16 @@ SpdyUpstream::~SpdyUpstream()
 int SpdyUpstream::on_read()
 {
   int rv = 0;
-  if((rv = spdylay_session_recv(session_)) < 0) {
+
+  rv = spdylay_session_recv(session_);
+  if(rv < 0) {
     if(rv != SPDYLAY_ERR_EOF) {
       ULOG(ERROR, this) << "spdylay_session_recv() returned error: "
                         << spdylay_strerror(rv);
     }
-  } else if((rv = spdylay_session_send(session_)) < 0) {
-    ULOG(ERROR, this) << "spdylay_session_send() returned error: "
-                      << spdylay_strerror(rv);
+    return rv;
   }
-  if(rv == 0) {
-    if(spdylay_session_want_read(session_) == 0 &&
-       spdylay_session_want_write(session_) == 0 &&
-       handler_->get_outbuf_length() == 0) {
-      if(LOG_ENABLED(INFO)) {
-        ULOG(INFO, this) << "No more read/write for this SPDY session";
-      }
-      rv = -1;
-    }
-  }
-  return rv;
+  return send();
 }
 
 int SpdyUpstream::on_write()
@@ -470,21 +476,33 @@ int SpdyUpstream::on_write()
 int SpdyUpstream::send()
 {
   int rv = 0;
-  if((rv = spdylay_session_send(session_)) < 0) {
+  uint8_t buf[4096];
+
+  sendbuf = buf;
+  sendbuflen = 0;
+  sendbufmax = sizeof(buf);
+
+  rv = spdylay_session_send(session_);
+  if(rv != 0) {
     ULOG(ERROR, this) << "spdylay_session_send() returned error: "
                       << spdylay_strerror(rv);
+    return rv;
   }
-  if(rv == 0) {
-    if(spdylay_session_want_read(session_) == 0 &&
-       spdylay_session_want_write(session_) == 0 &&
-       handler_->get_outbuf_length() == 0) {
-      if(LOG_ENABLED(INFO)) {
-        ULOG(INFO, this) << "No more read/write for this SPDY session";
-      }
-      rv = -1;
+  rv = bufferevent_write(handler_->get_bev(), sendbuf, sendbuflen);
+  if(rv == -1) {
+    ULOG(FATAL, this) << "evbuffer_add() failed";
+    return -1;
+  }
+
+  if(spdylay_session_want_read(session_) == 0 &&
+     spdylay_session_want_write(session_) == 0 &&
+     handler_->get_outbuf_length() == 0) {
+    if(LOG_ENABLED(INFO)) {
+      ULOG(INFO, this) << "No more read/write for this SPDY session";
     }
+    return -1;
   }
-  return rv;
+  return 0;
 }
 
 int SpdyUpstream::on_event()

src/shrpx_spdy_upstream.h

@@ -67,6 +67,10 @@ public:
   virtual int on_downstream_body_complete(Downstream *downstream);
 
   bool get_flow_control() const;
+
+  uint8_t *sendbuf;
+  size_t sendbuflen;
+  size_t sendbufmax;
 private:
   DownstreamQueue downstream_queue_;
   ClientHandler *handler_;