Merge branch 'master' into v1.0.0

configure.ac

@@ -25,14 +25,14 @@ dnl Do not change user variables!
 dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html
 
 AC_PREREQ(2.61)
-AC_INIT([nghttp2], [0.7.10-DEV], [t-tujikawa@users.sourceforge.net])
+AC_INIT([nghttp2], [0.7.11-DEV], [t-tujikawa@users.sourceforge.net])
 LT_PREREQ([2.2.6])
 LT_INIT()
 dnl See versioning rule:
 dnl  http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
-AC_SUBST(LT_CURRENT, 12)
+AC_SUBST(LT_CURRENT, 13)
-AC_SUBST(LT_REVISION, 2)
+AC_SUBST(LT_REVISION, 0)
-AC_SUBST(LT_AGE, 7)
+AC_SUBST(LT_AGE, 8)
 
 major=`echo $PACKAGE_VERSION |cut -d. -f1 | sed -e "s/[^0-9]//g"`
 minor=`echo $PACKAGE_VERSION |cut -d. -f2 | sed -e "s/[^0-9]//g"`

doc/Makefile.am

@@ -76,6 +76,8 @@ APIDOCS= \
 	nghttp2_session_client_new2.rst \
 	nghttp2_session_client_new3.rst \
 	nghttp2_session_consume.rst \
+	nghttp2_session_consume_connection.rst \
+	nghttp2_session_consume_stream.rst \
 	nghttp2_session_del.rst \
 	nghttp2_session_get_effective_local_window_size.rst \
 	nghttp2_session_get_effective_recv_data_length.rst \
@@ -202,11 +204,7 @@ help:
 apiref.rst: $(top_builddir)/lib/includes/nghttp2/nghttp2ver.h \
 	$(top_builddir)/lib/includes/nghttp2/nghttp2.h
 	$(PYTHON) $(top_srcdir)/doc/mkapiref.py \
-	$@ \
+	$@ macros.rst enums.rst types.rst . $^
-	$(top_builddir)/doc/macros.rst \
-	$(top_builddir)/doc/enums.rst \
-	$(top_builddir)/doc/types.rst \
-	$(top_builddir)/doc/ $^
 
 clean-local:
 	-rm $(APIDOCS)

doc/bash_completion/h2load

@@ -8,7 +8,7 @@ _h2load()
     _get_comp_words_by_ref cur prev
     case $cur in
         -*)
-            COMPREPLY=( $( compgen -W '--threads --connection-window-bits --input-file --help --requests --verbose --version --window-bits --clients --no-tls-proto --header --max-concurrent-streams ' -- "$cur" ) )
+            COMPREPLY=( $( compgen -W '--threads --connection-window-bits --input-file --help --requests --data --verbose --version --window-bits --clients --no-tls-proto --header --max-concurrent-streams ' -- "$cur" ) )
             ;;
         *)
             _filedir

doc/bash_completion/nghttp

@@ -8,7 +8,7 @@ _nghttp()
     _get_comp_words_by_ref cur prev
     case $cur in
         -*)
-            COMPREPLY=( $( compgen -W '--verbose --no-dep --get-assets --har --header-table-size --multiply --padding --dep-idle --continuation --connection-window-bits --peer-max-concurrent-streams --timeout --data --no-content-length --version --color --cert --upgrade --remote-name --weight --help --key --null-out --window-bits --stat --header ' -- "$cur" ) )
+            COMPREPLY=( $( compgen -W '--verbose --no-dep --get-assets --har --header-table-size --multiply --padding --hexdump --dep-idle --continuation --connection-window-bits --peer-max-concurrent-streams --timeout --data --no-content-length --version --color --cert --upgrade --remote-name --trailer --weight --help --key --null-out --window-bits --stat --header ' -- "$cur" ) )
             ;;
         *)
             _filedir

doc/bash_completion/nghttpd

@@ -8,7 +8,7 @@ _nghttpd()
     _get_comp_words_by_ref cur prev
     case $cur in
         -*)
-            COMPREPLY=( $( compgen -W '--error-gzip --push --header-table-size --htdocs --padding --verbose --version --help --daemon --verify-client --workers --no-tls --color --early-response --dh-param-file ' -- "$cur" ) )
+            COMPREPLY=( $( compgen -W '--error-gzip --push --header-table-size --trailer --htdocs --address --padding --verbose --version --help --hexdump --daemon --verify-client --workers --no-tls --color --early-response --dh-param-file ' -- "$cur" ) )
             ;;
         *)
             _filedir

doc/bash_completion/nghttpx

@@ -8,7 +8,7 @@ _nghttpx()
     _get_comp_words_by_ref cur prev
     case $cur in
         -*)
-            COMPREPLY=( $( compgen -W '--frontend-http2-connection-window-bits --worker-read-rate --frontend-no-tls --frontend-http2-dump-request-header --daemon --write-rate --altsvc --frontend-http2-dump-response-header --backend-http1-connections-per-frontend --tls-ticket-key-file --ciphers --verify-client-cacert --backend-keep-alive-timeout --strip-incoming-x-forwarded-for --errorlog-file --private-key-passwd-file --version --backlog --backend-http-proxy-uri --add-response-header --backend-write-timeout --backend-request-buffer --add-x-forwarded-for --write-burst --backend-http2-connection-window-bits --insecure --rlimit-nofile --backend-http2-window-bits --tls-proto-list --no-location-rewrite --padding --accesslog-syslog --conf --http2-max-concurrent-streams --client-proxy --worker-frontend-connections --cacert --frontend-read-timeout --worker-write-burst --npn-list --syslog-facility --backend-http1-connections-per-host --no-server-push --client --http2-bridge --no-via --user --stream-write-timeout --backend-response-buffer --http2-no-cookie-crumbling --backend-read-timeout --stream-read-timeout --workers --worker-read-burst --tls-ctx-per-worker --dh-param-file --errorlog-syslog --frontend --accesslog-file --http2-proxy --read-burst --accesslog-format --frontend-http2-window-bits --backend-no-tls --client-private-key-file --pid-file --client-cert-file --no-host-rewrite --log-level --worker-write-rate --help --backend-tls-sni-field --subcert --frontend-frame-debug --frontend-write-timeout --verify-client --read-rate --frontend-http2-read-timeout --backend-ipv4 --listener-disable-timeout --backend-ipv6 --backend ' -- "$cur" ) )
+            COMPREPLY=( $( compgen -W '--frontend-http2-connection-window-bits --worker-read-rate --frontend-no-tls --frontend-http2-dump-request-header --daemon --write-rate --altsvc --frontend-http2-dump-response-header --backend-http1-connections-per-frontend --tls-ticket-key-file --ciphers --verify-client-cacert --backend-keep-alive-timeout --strip-incoming-x-forwarded-for --errorlog-file --private-key-passwd-file --version --backlog --backend-http-proxy-uri --add-response-header --backend-write-timeout --backend-request-buffer --add-x-forwarded-for --write-burst --backend-http2-connection-window-bits --insecure --rlimit-nofile --backend-http2-window-bits --tls-proto-list --no-location-rewrite --padding --conf --accesslog-syslog --backend-http2-connections-per-worker --http2-max-concurrent-streams --client-proxy --worker-frontend-connections --ocsp-update-interval --cacert --frontend-read-timeout --worker-write-burst --npn-list --syslog-facility --backend-http1-connections-per-host --no-server-push --client --http2-bridge --fetch-ocsp-response-file --no-via --user --stream-write-timeout --no-ocsp --backend-response-buffer --http2-no-cookie-crumbling --backend-read-timeout --stream-read-timeout --workers --worker-read-burst --dh-param-file --errorlog-syslog --frontend --accesslog-file --http2-proxy --frontend-http2-read-timeout --accesslog-format --frontend-http2-window-bits --backend-no-tls --client-private-key-file --pid-file --client-cert-file --no-host-rewrite --log-level --worker-write-rate --help --backend-tls-sni-field --subcert --frontend-frame-debug --frontend-write-timeout --verify-client --read-rate --read-burst --backend-ipv4 --listener-disable-timeout --backend-ipv6 --backend ' -- "$cur" ) )
             ;;
         *)
             _filedir

doc/h2load.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "H2LOAD" "1" "March 31, 2015" "0.7.10-DEV" "nghttp2"
+.TH "H2LOAD" "1" "April 08, 2015" "0.7.10" "nghttp2"
 .SH NAME
 h2load \- HTTP/2 benchmarking tool
 .

doc/nghttp.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "NGHTTP" "1" "March 31, 2015" "0.7.10-DEV" "nghttp2"
+.TH "NGHTTP" "1" "April 08, 2015" "0.7.10" "nghttp2"
 .SH NAME
 nghttp \- HTTP/2 experimental client
 .

doc/nghttpd.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "NGHTTPD" "1" "March 31, 2015" "0.7.10-DEV" "nghttp2"
+.TH "NGHTTPD" "1" "April 08, 2015" "0.7.10" "nghttp2"
 .SH NAME
 nghttpd \- HTTP/2 experimental server
 .

doc/nghttpx.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "NGHTTPX" "1" "March 31, 2015" "0.7.10-DEV" "nghttp2"
+.TH "NGHTTPX" "1" "April 08, 2015" "0.7.10" "nghttp2"
 .SH NAME
 nghttpx \- HTTP/2 experimental proxy
 .
@@ -435,17 +435,6 @@ are stored in memory.
 .UNINDENT
 .INDENT 0.0
 .TP
-.B \-\-tls\-ctx\-per\-worker
-Create OpenSSL\(aqs SSL_CTX per worker, so that no internal
-locking is required.  This  may improve scalability with
-multi  threaded   configuration.   If  this   option  is
-enabled, session ID is  no longer shared accross SSL_CTX
-objects, which means session  ID generated by one worker
-is not acceptable by another worker.  On the other hand,
-session ticket key is shared across all worker threads.
-.UNINDENT
-.INDENT 0.0
-.TP
 .B \-\-fetch\-ocsp\-response\-file=<PATH>
 Path to  fetch\-ocsp\-response script file.  It  should be
 absolute path.
@@ -643,7 +632,8 @@ Default: \fB$remote_addr \- \- [$time_local] "$request" $status $body_bytes_sent
 .TP
 .B \-\-errorlog\-file=<PATH>
 Set path to write error  log.  To reopen file, send USR1
-signal to nghttpx.
+signal  to nghttpx.   stderr will  be redirected  to the
+error log file unless \fI\%\-\-errorlog\-syslog\fP is used.
 .sp
 Default: \fB/dev/stderr\fP
 .UNINDENT
@@ -872,6 +862,15 @@ specified socket already exists in the file system, nghttpx first
 deletes it.  However, if SIGUSR2 is used to execute new binary and
 both old and new configurations use same filename, new binary does not
 delete the socket and continues to use it.
+.SH OCSP STAPLING
+.sp
+OCSP query is done using external perl script \fBfetch\-ocsp\-response\fP,
+which has been developed as part of h2o project
+(\fI\%https://github.com/h2o/h2o\fP).
+.sp
+The script file is usually installed under
+\fB$(prefix)/share/nghttp2/\fP directory.  The actual path to script can
+be customized using \fI\%\-\-fetch\-ocsp\-response\-file\fP option.
 .SH SEE ALSO
 .sp
 \fInghttp(1)\fP, \fInghttpd(1)\fP, \fIh2load(1)\fP

doc/nghttpx.1.rst

@@ -377,16 +377,6 @@ SSL/TLS
     automatically and  renewed every 12hrs.  At  most 2 keys
     are stored in memory.
 
-.. option:: --tls-ctx-per-worker
-
-    Create OpenSSL's SSL_CTX per worker, so that no internal
-    locking is required.  This  may improve scalability with
-    multi  threaded   configuration.   If  this   option  is
-    enabled, session ID is  no longer shared accross SSL_CTX
-    objects, which means session  ID generated by one worker
-    is not acceptable by another worker.  On the other hand,
-    session ticket key is shared across all worker threads.
-
 .. option:: --fetch-ocsp-response-file=<PATH>
 
     Path to  fetch-ocsp-response script file.  It  should be
@@ -561,7 +551,8 @@ Logging
 .. option:: --errorlog-file=<PATH>
 
     Set path to write error  log.  To reopen file, send USR1
-    signal to nghttpx.
+    signal  to nghttpx.   stderr will  be redirected  to the
+    error log file unless :option:`--errorlog-syslog` is used.
 
     Default: ``/dev/stderr``
 
@@ -784,6 +775,17 @@ deletes it.  However, if SIGUSR2 is used to execute new binary and
 both old and new configurations use same filename, new binary does not
 delete the socket and continues to use it.
 
+OCSP STAPLING
+-------------
+
+OCSP query is done using external perl script ``fetch-ocsp-response``,
+which has been developed as part of h2o project
+(https://github.com/h2o/h2o).
+
+The script file is usually installed under
+``$(prefix)/share/nghttp2/`` directory.  The actual path to script can
+be customized using :option:`--fetch-ocsp-response-file` option.
+
 SEE ALSO
 --------
 

integration-tests/nghttpx_http2_test.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"strings"
 	"syscall"
 	"testing"
 )
@@ -494,7 +495,9 @@ func TestH2H1SNI(t *testing.T) {
 func TestH2H1ServerPush(t *testing.T) {
 	st := newServerTester(nil, t, func(w http.ResponseWriter, r *http.Request) {
 		// only resources marked as rel=preload are pushed
-		w.Header().Add("Link", "</css/main.css>; rel=preload, </foo>, </css/theme.css>; rel=preload")
+		if !strings.HasPrefix(r.URL.Path, "/css/") {
+			w.Header().Add("Link", "</css/main.css>; rel=preload, </foo>, </css/theme.css>; rel=preload")
+		}
 	})
 	defer st.Close()
 

lib/includes/nghttp2/nghttp2.h

@@ -2673,7 +2673,11 @@ NGHTTP2_EXTERN uint32_t
  *
  * Tells the |session| that |size| bytes for a stream denoted by
  * |stream_id| were consumed by application and are ready to
- * WINDOW_UPDATE.  This function is intended to be used without
+ * WINDOW_UPDATE.  The consumed bytes are counted towards both
+ * connection and stream level WINDOW_UPDATE (see
+ * `nghttp2_session_consume_connection()` and
+ * `nghttp2_session_consume_stream()` to update consumption
+ * independently).  This function is intended to be used without
  * automatic window update (see
  * `nghttp2_option_set_no_auto_window_update()`).
  *
@@ -2690,6 +2694,47 @@ NGHTTP2_EXTERN uint32_t
 NGHTTP2_EXTERN int nghttp2_session_consume(nghttp2_session *session,
                                            int32_t stream_id, size_t size);
 
+/**
+ * @function
+ *
+ * Like `nghttp2_session_consume()`, but this only tells library that
+ * |size| bytes were consumed only for connection level.  Note that
+ * HTTP/2 maintains connection and stream level flow control windows
+ * independently.
+ *
+ * This function returns 0 if it succeeds, or one of the following
+ * negative error codes:
+ *
+ * :enum:`NGHTTP2_ERR_NOMEM`
+ *     Out of memory.
+ * :enum:`NGHTTP2_ERR_INVALID_STATE`
+ *     Automatic WINDOW_UPDATE is not disabled.
+ */
+NGHTTP2_EXTERN int nghttp2_session_consume_connection(nghttp2_session *session,
+                                                      size_t size);
+
+/**
+ * @function
+ *
+ * Like `nghttp2_session_consume()`, but this only tells library that
+ * |size| bytes were consumed only for stream denoted by |stream_id|.
+ * Note that HTTP/2 maintains connection and stream level flow control
+ * windows independently.
+ *
+ * This function returns 0 if it succeeds, or one of the following
+ * negative error codes:
+ *
+ * :enum:`NGHTTP2_ERR_NOMEM`
+ *     Out of memory.
+ * :enum:`NGHTTP2_ERR_INVALID_ARGUMENT`
+ *     The |stream_id| is 0.
+ * :enum:`NGHTTP2_ERR_INVALID_STATE`
+ *     Automatic WINDOW_UPDATE is not disabled.
+ */
+NGHTTP2_EXTERN int nghttp2_session_consume_stream(nghttp2_session *session,
+                                                  int32_t stream_id,
+                                                  size_t size);
+
 /**
  * @function
  *
@@ -3336,6 +3381,9 @@ NGHTTP2_EXTERN int32_t
  * The |flags| is currently ignored and should be
  * :enum:`NGHTTP2_FLAG_NONE`.
  *
+ * The |stream_id| is the stream ID to send this WINDOW_UPDATE.  To
+ * send connection level WINDOW_UPDATE, specify 0 to |stream_id|.
+ *
  * If the |window_size_increment| is positive, the WINDOW_UPDATE with
  * that value as window_size_increment is queued.  If the
  * |window_size_increment| is larger than the received bytes from the

lib/nghttp2_session.c

@@ -6612,12 +6612,58 @@ int nghttp2_session_consume(nghttp2_session *session, int32_t stream_id,
 
   stream = nghttp2_session_get_stream(session, stream_id);
 
-  if (stream) {
+  if (!stream) {
-    rv = session_update_stream_consumed_size(session, stream, size);
+    return 0;
+  }
 
-    if (nghttp2_is_fatal(rv)) {
+  rv = session_update_stream_consumed_size(session, stream, size);
-      return rv;
+
-    }
+  if (nghttp2_is_fatal(rv)) {
+    return rv;
+  }
+
+  return 0;
+}
+
+int nghttp2_session_consume_connection(nghttp2_session *session, size_t size) {
+  int rv;
+
+  if (!(session->opt_flags & NGHTTP2_OPTMASK_NO_AUTO_WINDOW_UPDATE)) {
+    return NGHTTP2_ERR_INVALID_STATE;
+  }
+
+  rv = session_update_connection_consumed_size(session, size);
+
+  if (nghttp2_is_fatal(rv)) {
+    return rv;
+  }
+
+  return 0;
+}
+
+int nghttp2_session_consume_stream(nghttp2_session *session, int32_t stream_id,
+                                   size_t size) {
+  int rv;
+  nghttp2_stream *stream;
+
+  if (stream_id == 0) {
+    return NGHTTP2_ERR_INVALID_ARGUMENT;
+  }
+
+  if (!(session->opt_flags & NGHTTP2_OPTMASK_NO_AUTO_WINDOW_UPDATE)) {
+    return NGHTTP2_ERR_INVALID_STATE;
+  }
+
+  stream = nghttp2_session_get_stream(session, stream_id);
+
+  if (!stream) {
+    return 0;
+  }
+
+  rv = session_update_stream_consumed_size(session, stream, size);
+
+  if (nghttp2_is_fatal(rv)) {
+    return rv;
   }
 
   return 0;

lib/nghttp2_stream.c

@@ -87,7 +87,8 @@ void nghttp2_stream_shutdown(nghttp2_stream *stream, nghttp2_shut_flag flag) {
 }
 
 static int stream_push_item(nghttp2_stream *stream, nghttp2_session *session) {
-  int rv;
+  /* This is required for Android NDK r10d */
+  int rv = 0;
   nghttp2_outbound_item *item;
 
   assert(stream->item);

src/memchunk.h

@@ -102,6 +102,11 @@ template <typename T> struct Pool {
     }
     freelist = m;
   }
+  void clear() {
+    freelist = nullptr;
+    pool = nullptr;
+    poolsize = 0;
+  }
   using value_type = T;
   std::unique_ptr<T> pool;
   T *freelist;

src/shrpx.cc

@@ -422,6 +422,7 @@ void reopen_log_signal_cb(struct ev_loop *loop, ev_signal *w, int revents) {
   }
 
   (void)reopen_log_files();
+  redirect_stderr_to_errorlog();
 
   if (get_config()->num_worker > 1) {
     conn_handler->worker_reopen_log_files();
@@ -636,6 +637,10 @@ int event_loop() {
 
     // We get new PID after successful daemon().
     mod_config()->pid = getpid();
+
+    // daemon redirects stderr file descriptor to /dev/null, so we
+    // need this.
+    redirect_stderr_to_errorlog();
   }
 
   if (get_config()->pid_file) {
@@ -1311,7 +1316,8 @@ Logging:
               Default: )" << DEFAULT_ACCESSLOG_FORMAT << R"(
   --errorlog-file=<PATH>
               Set path to write error  log.  To reopen file, send USR1
-              signal to nghttpx.
+              signal  to nghttpx.   stderr will  be redirected  to the
+              error log file unless --errorlog-syslog is used.
               Default: )" << get_config()->errorlog_file.get() << R"(
   --errorlog-syslog
               Send  error log  to  syslog.  If  this  option is  used,
@@ -1917,6 +1923,8 @@ int main(int argc, char **argv) {
     exit(EXIT_FAILURE);
   }
 
+  redirect_stderr_to_errorlog();
+
   if (get_config()->uid != 0) {
     if (log_config()->accesslog_fd != -1 &&
         fchown(log_config()->accesslog_fd, get_config()->uid,

src/shrpx_client_handler.cc

@@ -402,6 +402,10 @@ ClientHandler::~ClientHandler() {
   auto worker_stat = worker_->get_worker_stat();
   --worker_stat->num_connections;
 
+  if (worker_stat->num_connections == 0) {
+    worker_->schedule_clear_mcpool();
+  }
+
   ev_timer_stop(conn_.loop, &reneg_shutdown_timer_);
 
   // TODO If backend is http/2, and it is in CONNECTED state, signal
@@ -556,7 +560,14 @@ int ClientHandler::validate_next_proto() {
 int ClientHandler::do_read() { return read_(*this); }
 int ClientHandler::do_write() { return write_(*this); }
 
-int ClientHandler::on_read() { return on_read_(*this); }
+int ClientHandler::on_read() {
+  auto rv = on_read_(*this);
+  if (rv != 0) {
+    return rv;
+  }
+  conn_.handle_tls_pending_read();
+  return 0;
+}
 int ClientHandler::on_write() { return on_write_(*this); }
 
 const std::string &ClientHandler::get_ipaddr() const { return ipaddr_; }
@@ -623,6 +634,8 @@ ClientHandler::get_downstream_connection() {
   return dconn;
 }
 
+MemchunkPool *ClientHandler::get_mcpool() { return worker_->get_mcpool(); }
+
 SSL *ClientHandler::get_ssl() const { return conn_.tls.ssl; }
 
 ConnectBlocker *ClientHandler::get_connect_blocker() const {

src/shrpx_client_handler.h

@@ -36,6 +36,7 @@
 #include "shrpx_rate_limit.h"
 #include "shrpx_connection.h"
 #include "buffer.h"
+#include "memchunk.h"
 
 using namespace nghttp2;
 
@@ -92,6 +93,7 @@ public:
   void pool_downstream_connection(std::unique_ptr<DownstreamConnection> dconn);
   void remove_downstream_connection(DownstreamConnection *dconn);
   std::unique_ptr<DownstreamConnection> get_downstream_connection();
+  MemchunkPool *get_mcpool();
   SSL *get_ssl() const;
   ConnectBlocker *get_connect_blocker() const;
   // Call this function when HTTP/2 connection header is received at

src/shrpx_connection.cc

@@ -41,7 +41,7 @@ Connection::Connection(struct ev_loop *loop, int fd, SSL *ssl,
                        size_t read_burst, IOCb writecb, IOCb readcb,
                        TimerCb timeoutcb, void *data)
     : tls{ssl}, wlimit(loop, &wev, write_rate, write_burst),
-      rlimit(loop, &rev, read_rate, read_burst), writecb(writecb),
+      rlimit(loop, &rev, read_rate, read_burst, ssl), writecb(writecb),
       readcb(readcb), timeoutcb(timeoutcb), loop(loop), data(data), fd(fd) {
 
   ev_io_init(&wev, writecb, fd, EV_WRITE);
@@ -303,4 +303,11 @@ ssize_t Connection::read_clear(void *data, size_t len) {
   return nread;
 }
 
+void Connection::handle_tls_pending_read() {
+  if (!ev_is_active(&rev)) {
+    return;
+  }
+  rlimit.handle_tls_pending_read();
+}
+
 } // namespace shrpx

src/shrpx_connection.h

@@ -83,6 +83,8 @@ struct Connection {
   ssize_t writev_clear(struct iovec *iov, int iovcnt);
   ssize_t read_clear(void *data, size_t len);
 
+  void handle_tls_pending_read();
+
   TLSConnection tls;
   ev_io wev;
   ev_io rev;

src/shrpx_downstream.cc

@@ -106,12 +106,12 @@ void downstream_wtimeoutcb(struct ev_loop *loop, ev_timer *w, int revents) {
 } // namespace
 
 // upstream could be nullptr for unittests
-Downstream::Downstream(Upstream *upstream, int32_t stream_id, int32_t priority)
+Downstream::Downstream(Upstream *upstream, MemchunkPool *mcpool,
+                       int32_t stream_id, int32_t priority)
     : dlnext(nullptr), dlprev(nullptr),
       request_start_time_(std::chrono::high_resolution_clock::now()),
-      request_buf_(upstream ? upstream->get_mcpool() : nullptr),
+      request_buf_(mcpool), response_buf_(mcpool), request_bodylen_(0),
-      response_buf_(upstream ? upstream->get_mcpool() : nullptr),
+      response_bodylen_(0), response_sent_bodylen_(0),
-      request_bodylen_(0), response_bodylen_(0), response_sent_bodylen_(0),
       request_content_length_(-1), response_content_length_(-1),
       upstream_(upstream), blocked_link_(nullptr), request_headers_sum_(0),
       response_headers_sum_(0), request_datalen_(0), response_datalen_(0),

src/shrpx_downstream.h

@@ -52,7 +52,8 @@ struct BlockedLink;
 
 class Downstream {
 public:
-  Downstream(Upstream *upstream, int32_t stream_id, int32_t priority);
+  Downstream(Upstream *upstream, MemchunkPool *mcpool, int32_t stream_id,
+             int32_t priority);
   ~Downstream();
   void reset_upstream(Upstream *upstream);
   Upstream *get_upstream() const;

src/shrpx_downstream_test.cc

@@ -33,7 +33,7 @@
 namespace shrpx {
 
 void test_downstream_index_request_headers(void) {
-  Downstream d(nullptr, 0, 0);
+  Downstream d(nullptr, nullptr, 0, 0);
   d.add_request_header("1", "0");
   d.add_request_header("2", "1");
   d.add_request_header("Charlie", "2");
@@ -56,7 +56,7 @@ void test_downstream_index_request_headers(void) {
 }
 
 void test_downstream_index_response_headers(void) {
-  Downstream d(nullptr, 0, 0);
+  Downstream d(nullptr, nullptr, 0, 0);
   d.add_response_header("Charlie", "0");
   d.add_response_header("Alpha", "1");
   d.add_response_header("Delta", "2");
@@ -69,7 +69,7 @@ void test_downstream_index_response_headers(void) {
 }
 
 void test_downstream_get_request_header(void) {
-  Downstream d(nullptr, 0, 0);
+  Downstream d(nullptr, nullptr, 0, 0);
   d.add_request_header("alpha", "0");
   d.add_request_header(":authority", "1");
   d.add_request_header("content-length", "2");
@@ -86,7 +86,7 @@ void test_downstream_get_request_header(void) {
 }
 
 void test_downstream_get_response_header(void) {
-  Downstream d(nullptr, 0, 0);
+  Downstream d(nullptr, nullptr, 0, 0);
   d.add_response_header("alpha", "0");
   d.add_response_header(":status", "1");
   d.add_response_header("content-length", "2");
@@ -99,7 +99,7 @@ void test_downstream_get_response_header(void) {
 }
 
 void test_downstream_crumble_request_cookie(void) {
-  Downstream d(nullptr, 0, 0);
+  Downstream d(nullptr, nullptr, 0, 0);
   d.add_request_header(":method", "get");
   d.add_request_header(":path", "/");
   auto val = "alpha; bravo; ; ;; charlie;;";
@@ -122,7 +122,7 @@ void test_downstream_crumble_request_cookie(void) {
 }
 
 void test_downstream_assemble_request_cookie(void) {
-  Downstream d(nullptr, 0, 0);
+  Downstream d(nullptr, nullptr, 0, 0);
   d.add_request_header(":method", "get");
   d.add_request_header(":path", "/");
   d.add_request_header("cookie", "alpha");
@@ -135,7 +135,7 @@ void test_downstream_assemble_request_cookie(void) {
 
 void test_downstream_rewrite_location_response_header(void) {
   {
-    Downstream d(nullptr, 0, 0);
+    Downstream d(nullptr, nullptr, 0, 0);
     d.set_request_downstream_host("localhost:3000");
     d.add_request_header("host", "localhost");
     d.add_response_header("location", "http://localhost:3000/");
@@ -146,7 +146,7 @@ void test_downstream_rewrite_location_response_header(void) {
     CU_ASSERT("https://localhost/" == (*location).value);
   }
   {
-    Downstream d(nullptr, 0, 0);
+    Downstream d(nullptr, nullptr, 0, 0);
     d.set_request_downstream_host("localhost");
     d.set_request_http2_authority("localhost");
     d.add_response_header("location", "http://localhost:3000/");

src/shrpx_http2_upstream.cc

@@ -256,8 +256,11 @@ int on_begin_headers_callback(nghttp2_session *session,
                          << frame->hd.stream_id;
   }
 
+  auto handler = upstream->get_client_handler();
+
   // TODO Use priority 0 for now
-  auto downstream = make_unique<Downstream>(upstream, frame->hd.stream_id, 0);
+  auto downstream = make_unique<Downstream>(upstream, handler->get_mcpool(),
+                                            frame->hd.stream_id, 0);
   nghttp2_session_set_stream_user_data(session, frame->hd.stream_id,
                                        downstream.get());
 
@@ -484,16 +487,47 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame,
     verbose_on_frame_send_callback(session, frame, user_data);
   }
   auto upstream = static_cast<Http2Upstream *>(user_data);
+  auto handler = upstream->get_client_handler();
 
   switch (frame->hd.type) {
+  case NGHTTP2_DATA:
+  case NGHTTP2_HEADERS: {
+    if ((frame->hd.flags & NGHTTP2_FLAG_END_STREAM) == 0) {
+      return 0;
+    }
+    // RST_STREAM if request is still incomplete.
+    auto stream_id = frame->hd.stream_id;
+    auto downstream = static_cast<Downstream *>(
+        nghttp2_session_get_stream_user_data(session, stream_id));
+
+    // For tunneling, issue RST_STREAM to finish the stream.
+    if (downstream->get_upgraded() ||
+        nghttp2_session_get_stream_remote_close(session, stream_id) == 0) {
+      if (LOG_ENABLED(INFO)) {
+        ULOG(INFO, upstream)
+            << "Send RST_STREAM to "
+            << (downstream->get_upgraded() ? "tunneled " : "")
+            << "stream stream_id=" << downstream->get_stream_id()
+            << " to finish off incomplete request";
+      }
+
+      upstream->rst_stream(downstream, NGHTTP2_NO_ERROR);
+    }
+
+    return 0;
+  }
   case NGHTTP2_SETTINGS:
     if ((frame->hd.flags & NGHTTP2_FLAG_ACK) == 0) {
       upstream->start_settings_timer();
     }
     return 0;
   case NGHTTP2_PUSH_PROMISE: {
-    auto downstream = make_unique<Downstream>(
+    auto promised_stream_id = frame->push_promise.promised_stream_id;
-        upstream, frame->push_promise.promised_stream_id, 0);
+    auto downstream = make_unique<Downstream>(upstream, handler->get_mcpool(),
+                                              promised_stream_id, 0);
+
+    nghttp2_session_set_stream_user_data(session, promised_stream_id,
+                                         downstream.get());
 
     downstream->disable_upstream_rtimer();
 
@@ -1096,16 +1130,6 @@ ssize_t downstream_data_read_callback(nghttp2_session *session,
           }
         }
       }
-      if (nghttp2_session_get_stream_remote_close(session, stream_id) == 0) {
-        upstream->rst_stream(downstream, NGHTTP2_NO_ERROR);
-      }
-    } else {
-      // For tunneling, issue RST_STREAM to finish the stream.
-      if (LOG_ENABLED(INFO)) {
-        ULOG(INFO, upstream)
-            << "RST_STREAM to tunneled stream stream_id=" << stream_id;
-      }
-      upstream->rst_stream(downstream, NGHTTP2_NO_ERROR);
     }
   }
 
@@ -1480,8 +1504,6 @@ int Http2Upstream::on_downstream_reset(bool no_retry) {
   return 0;
 }
 
-MemchunkPool *Http2Upstream::get_mcpool() { return &mcpool_; }
-
 int Http2Upstream::prepare_push_promise(Downstream *downstream) {
   int rv;
   http_parser_url u;

src/shrpx_http2_upstream.h

@@ -79,8 +79,6 @@ public:
   virtual void on_handler_delete();
   virtual int on_downstream_reset(bool no_retry);
 
-  virtual MemchunkPool *get_mcpool();
-
   bool get_flow_control() const;
   // Perform HTTP/2 upgrade from |upstream|. On success, this object
   // takes ownership of the |upstream|. This function returns 0 if it
@@ -103,9 +101,7 @@ public:
   int on_request_headers(Downstream *downstream, const nghttp2_frame *frame);
 
 private:
-  // must be put before downstream_queue_
   std::unique_ptr<HttpsUpstream> pre_upstream_;
-  MemchunkPool mcpool_;
   DownstreamQueue downstream_queue_;
   ev_timer settings_timer_;
   ev_timer shutdown_timer_;

src/shrpx_http_downstream_connection.cc

@@ -720,13 +720,6 @@ int HttpDownstreamConnection::on_read() {
         http_parser_execute(&response_htp_, &htp_hooks,
                             reinterpret_cast<char *>(buf.data()), nread);
 
-    if (nproc != static_cast<size_t>(nread)) {
-      if (LOG_ENABLED(INFO)) {
-        DCLOG(INFO, this) << "nproc != nread";
-      }
-      return -1;
-    }
-
     auto htperr = HTTP_PARSER_ERRNO(&response_htp_);
 
     if (htperr != HPE_OK) {
@@ -739,6 +732,13 @@ int HttpDownstreamConnection::on_read() {
       return -1;
     }
 
+    if (nproc != static_cast<size_t>(nread)) {
+      if (LOG_ENABLED(INFO)) {
+        DCLOG(INFO, this) << "nproc != nread";
+      }
+      return -1;
+    }
+
     if (downstream_->response_buf_full()) {
       downstream_->pause_read(SHRPX_NO_BUFFER);
       return 0;

src/shrpx_https_upstream.cc

@@ -64,8 +64,12 @@ int htp_msg_begin(http_parser *htp) {
     ULOG(INFO, upstream) << "HTTP request started";
   }
   upstream->reset_current_header_length();
+
+  auto handler = upstream->get_client_handler();
+
   // TODO specify 0 as priority for now
-  upstream->attach_downstream(make_unique<Downstream>(upstream, 0, 0));
+  upstream->attach_downstream(
+      make_unique<Downstream>(upstream, handler->get_mcpool(), 0, 0));
   return 0;
 }
 } // namespace
@@ -636,7 +640,8 @@ void HttpsUpstream::error_reply(unsigned int status_code) {
   auto downstream = get_downstream();
 
   if (!downstream) {
-    attach_downstream(make_unique<Downstream>(this, 1, 1));
+    attach_downstream(
+        make_unique<Downstream>(this, handler_->get_mcpool(), 1, 1));
     downstream = get_downstream();
   }
 
@@ -926,6 +931,4 @@ fail:
   return 0;
 }
 
-MemchunkPool *HttpsUpstream::get_mcpool() { return &mcpool_; }
-
 } // namespace shrpx

src/shrpx_https_upstream.h

@@ -76,8 +76,6 @@ public:
   virtual void on_handler_delete();
   virtual int on_downstream_reset(bool no_retry);
 
-  virtual MemchunkPool *get_mcpool();
-
   void reset_current_header_length();
   void log_response_headers(const std::string &hdrs) const;
 
@@ -85,8 +83,6 @@ private:
   ClientHandler *handler_;
   http_parser htp_;
   size_t current_header_length_;
-  // must be put before downstream_
-  MemchunkPool mcpool_;
   std::unique_ptr<Downstream> downstream_;
   IOControl ioctrl_;
 };

src/shrpx_log.cc

@@ -324,4 +324,14 @@ int reopen_log_files() {
   return res;
 }
 
+void redirect_stderr_to_errorlog() {
+  auto lgconf = log_config();
+
+  if (get_config()->errorlog_syslog || lgconf->errorlog_fd == -1) {
+    return;
+  }
+
+  dup2(lgconf->errorlog_fd, STDERR_FILENO);
+}
+
 } // namespace shrpx

src/shrpx_log.h

@@ -143,6 +143,8 @@ void upstream_accesslog(const std::vector<LogFragment> &lf, LogSpec *lgsp);
 
 int reopen_log_files();
 
+void redirect_stderr_to_errorlog();
+
 } // namespace shrpx
 
 #endif // SHRPX_LOG_H

src/shrpx_rate_limit.cc

@@ -35,8 +35,9 @@ void regencb(struct ev_loop *loop, ev_timer *w, int revents) {
 }
 } // namespace
 
-RateLimit::RateLimit(struct ev_loop *loop, ev_io *w, size_t rate, size_t burst)
+RateLimit::RateLimit(struct ev_loop *loop, ev_io *w, size_t rate, size_t burst,
-    : w_(w), loop_(loop), rate_(rate), burst_(burst), avail_(burst),
+                     SSL *ssl)
+    : w_(w), loop_(loop), ssl_(ssl), rate_(rate), burst_(burst), avail_(burst),
       startw_req_(false) {
   ev_timer_init(&t_, regencb, 0., 1.);
   t_.data = this;
@@ -45,9 +46,7 @@ RateLimit::RateLimit(struct ev_loop *loop, ev_io *w, size_t rate, size_t burst)
   }
 }
 
-RateLimit::~RateLimit() {
+RateLimit::~RateLimit() { ev_timer_stop(loop_, &t_); }
-  ev_timer_stop(loop_, &t_);
-}
 
 size_t RateLimit::avail() const {
   if (rate_ == 0) {
@@ -79,6 +78,7 @@ void RateLimit::regen() {
 
   if (avail_ > 0 && startw_req_) {
     ev_io_start(loop_, w_);
+    handle_tls_pending_read();
   }
 }
 
@@ -86,6 +86,7 @@ void RateLimit::startw() {
   startw_req_ = true;
   if (rate_ == 0 || avail_ > 0) {
     ev_io_start(loop_, w_);
+    handle_tls_pending_read();
     return;
   }
 }
@@ -95,4 +96,14 @@ void RateLimit::stopw() {
   ev_io_stop(loop_, w_);
 }
 
+void RateLimit::handle_tls_pending_read() {
+  if (!ssl_ || SSL_pending(ssl_) == 0) {
+    return;
+  }
+
+  // Note that ev_feed_event works without starting watcher, but we
+  // only call this function if watcher is active.
+  ev_feed_event(loop_, w_, EV_READ);
+}
+
 } // namespace shrpx

src/shrpx_rate_limit.h

@@ -29,21 +29,31 @@
 
 #include <ev.h>
 
+#include <openssl/ssl.h>
+
 namespace shrpx {
 
 class RateLimit {
 public:
-  RateLimit(struct ev_loop *loop, ev_io *w, size_t rate, size_t burst);
+  // We need |ssl| object to check that it has unread decrypted bytes.
+  RateLimit(struct ev_loop *loop, ev_io *w, size_t rate, size_t burst,
+            SSL *ssl = nullptr);
   ~RateLimit();
   size_t avail() const;
   void drain(size_t n);
   void regen();
   void startw();
   void stopw();
+  // Feeds event if ssl_ object has unread decrypted bytes.  This is
+  // required since it is buffered in ssl_ object, io event is not
+  // generated unless new incoming data is received.
+  void handle_tls_pending_read();
+
 private:
-  ev_io *w_;
   ev_timer t_;
+  ev_io *w_;
   struct ev_loop *loop_;
+  SSL *ssl_;
   size_t rate_;
   size_t burst_;
   size_t avail_;

src/shrpx_spdy_upstream.cc

@@ -797,7 +797,8 @@ int SpdyUpstream::error_reply(Downstream *downstream,
 
 Downstream *SpdyUpstream::add_pending_downstream(int32_t stream_id,
                                                  int32_t priority) {
-  auto downstream = make_unique<Downstream>(this, stream_id, priority);
+  auto downstream = make_unique<Downstream>(this, handler_->get_mcpool(),
+                                            stream_id, priority);
   spdylay_session_set_stream_user_data(session_, stream_id, downstream.get());
   auto res = downstream.get();
 
@@ -1080,6 +1081,4 @@ int SpdyUpstream::on_downstream_reset(bool no_retry) {
   return 0;
 }
 
-MemchunkPool *SpdyUpstream::get_mcpool() { return &mcpool_; }
-
 } // namespace shrpx

src/shrpx_spdy_upstream.h

@@ -74,8 +74,6 @@ public:
   virtual void on_handler_delete();
   virtual int on_downstream_reset(bool no_retry);
 
-  virtual MemchunkPool *get_mcpool();
-
   bool get_flow_control() const;
 
   int consume(int32_t stream_id, size_t len);
@@ -84,8 +82,6 @@ public:
   void initiate_downstream(Downstream *downstream);
 
 private:
-  // must be put before downstream_queue_
-  MemchunkPool mcpool_;
   DownstreamQueue downstream_queue_;
   ClientHandler *handler_;
   spdylay_session *session_;

src/shrpx_upstream.h

@@ -27,9 +27,6 @@
 
 #include "shrpx.h"
 #include "shrpx_io_control.h"
-#include "memchunk.h"
-
-using namespace nghttp2;
 
 namespace shrpx {
 
@@ -65,8 +62,6 @@ public:
   virtual void pause_read(IOCtrlReason reason) = 0;
   virtual int resume_read(IOCtrlReason reason, Downstream *downstream,
                           size_t consumed) = 0;
-
-  virtual MemchunkPool *get_mcpool() = 0;
 };
 
 } // namespace shrpx

src/shrpx_worker.cc

@@ -37,8 +37,6 @@
 #include "util.h"
 #include "template.h"
 
-using namespace nghttp2;
-
 namespace shrpx {
 
 namespace {
@@ -48,6 +46,16 @@ void eventcb(struct ev_loop *loop, ev_async *w, int revents) {
 }
 } // namespace
 
+namespace {
+void mcpool_clear_cb(struct ev_loop *loop, ev_timer *w, int revents) {
+  auto worker = static_cast<Worker *>(w->data);
+  if (worker->get_worker_stat()->num_connections != 0) {
+    return;
+  }
+  worker->get_mcpool()->clear();
+}
+} // namespace
+
 Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
                ssl::CertLookupTree *cert_tree,
                const std::shared_ptr<TicketKeys> &ticket_keys)
@@ -59,6 +67,9 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
   w_.data = this;
   ev_async_start(loop_, &w_);
 
+  ev_timer_init(&mcpool_clear_timer_, mcpool_clear_cb, 0., 0.);
+  mcpool_clear_timer_.data = this;
+
   if (get_config()->downstream_proto == PROTO_HTTP2) {
     auto n = get_config()->http2_downstream_connections_per_worker;
     for (; n > 0; --n) {
@@ -68,7 +79,17 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
   }
 }
 
-Worker::~Worker() { ev_async_stop(loop_, &w_); }
+Worker::~Worker() {
+  ev_async_stop(loop_, &w_);
+  ev_timer_stop(loop_, &mcpool_clear_timer_);
+}
+
+void Worker::schedule_clear_mcpool() {
+  // libev manual says: "If the watcher is already active nothing will
+  // happen."  Since we don't change any timeout here, we don't have
+  // to worry about querying ev_is_active.
+  ev_timer_start(loop_, &mcpool_clear_timer_);
+}
 
 void Worker::wait() {
 #ifndef NOTHREADS
@@ -218,4 +239,6 @@ void Worker::set_graceful_shutdown(bool f) { graceful_shutdown_ = f; }
 
 bool Worker::get_graceful_shutdown() const { return graceful_shutdown_; }
 
+MemchunkPool *Worker::get_mcpool() { return &mcpool_; }
+
 } // namespace shrpx

src/shrpx_worker.h

@@ -42,6 +42,9 @@
 
 #include "shrpx_config.h"
 #include "shrpx_downstream_connection_pool.h"
+#include "memchunk.h"
+
+using namespace nghttp2;
 
 namespace shrpx {
 
@@ -104,6 +107,9 @@ public:
   void set_graceful_shutdown(bool f);
   bool get_graceful_shutdown() const;
 
+  MemchunkPool *get_mcpool();
+  void schedule_clear_mcpool();
+
 private:
   std::vector<std::unique_ptr<Http2Session>> http2sessions_;
   size_t next_http2session_;
@@ -113,6 +119,8 @@ private:
   std::mutex m_;
   std::deque<WorkerEvent> q_;
   ev_async w_;
+  ev_timer mcpool_clear_timer_;
+  MemchunkPool mcpool_;
   DownstreamConnectionPool dconn_pool_;
   WorkerStat worker_stat_;
   struct ev_loop *loop_;