Merge branch 'mt_master' into mt

Makefile.am

@@ -23,6 +23,11 @@
 SUBDIRS = lib third-party src examples python tests integration-tests \
 	doc contrib script
 
+# Now with python setuptools, make uninstall will leave many files we
+# cannot easily remove (e.g., easy-install.pth).  Disable it for
+# distcheck rule.
+AM_DISTCHECK_CONFIGURE_FLAGS = --disable-python-bindings
+
 ACLOCAL_AMFLAGS = -I m4
 
 dist_doc_DATA = README.rst

configure.ac

@@ -25,7 +25,7 @@ dnl Do not change user variables!
 dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html
 
 AC_PREREQ(2.61)
-AC_INIT([nghttp2], [1.0.6-DEV], [t-tujikawa@users.sourceforge.net])
+AC_INIT([nghttp2], [1.1.3-DEV], [t-tujikawa@users.sourceforge.net])
 AC_USE_SYSTEM_EXTENSIONS
 
 LT_PREREQ([2.2.6])
@@ -48,7 +48,7 @@ AC_CONFIG_HEADERS([config.h])
 dnl See versioning rule:
 dnl  http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
 AC_SUBST(LT_CURRENT, 14)
-AC_SUBST(LT_REVISION, 5)
+AC_SUBST(LT_REVISION, 6)
 AC_SUBST(LT_AGE, 0)
 
 major=`echo $PACKAGE_VERSION |cut -d. -f1 | sed -e "s/[^0-9]//g"`

doc/Makefile.am

@@ -223,7 +223,7 @@ $(APIDOC): apidoc.stamp
 	fi
 
 clean-local:
-	-rm $(APIDOCS)
+	-rm -f $(APIDOCS)
 	-rm -rf $(BUILDDIR)/*
 
 html-local: apiref.rst

doc/bash_completion/h2load

@@ -8,7 +8,7 @@ _h2load()
     _get_comp_words_by_ref cur prev
     case $cur in
         -*)
-            COMPREPLY=( $( compgen -W '--threads --connection-window-bits --input-file --help --requests --data --verbose --version --window-bits --clients --no-tls-proto --header --max-concurrent-streams ' -- "$cur" ) )
+            COMPREPLY=( $( compgen -W '--threads --connection-window-bits --input-file --help --requests --data --verbose --ciphers --window-bits --clients --no-tls-proto --version --header --max-concurrent-streams ' -- "$cur" ) )
             ;;
         *)
             _filedir

doc/bash_completion/nghttp

@@ -8,7 +8,7 @@ _nghttp()
     _get_comp_words_by_ref cur prev
     case $cur in
         -*)
-            COMPREPLY=( $( compgen -W '--no-push --verbose --no-dep --get-assets --har --header-table-size --multiply --padding --hexdump --continuation --connection-window-bits --peer-max-concurrent-streams --timeout --data --no-content-length --version --color --cert --upgrade --remote-name --trailer --weight --help --key --null-out --window-bits --stat --header ' -- "$cur" ) )
+            COMPREPLY=( $( compgen -W '--no-push --verbose --no-dep --get-assets --har --header-table-size --multiply --padding --hexdump --max-concurrent-streams --continuation --connection-window-bits --peer-max-concurrent-streams --timeout --data --no-content-length --version --color --cert --upgrade --remote-name --trailer --weight --help --key --null-out --window-bits --stat --header ' -- "$cur" ) )
             ;;
         *)
             _filedir

doc/bash_completion/nghttpx

@@ -8,7 +8,7 @@ _nghttpx()
     _get_comp_words_by_ref cur prev
     case $cur in
         -*)
-            COMPREPLY=( $( compgen -W '--worker-read-rate --frontend-no-tls --frontend-http2-dump-response-header --backend-http1-connections-per-frontend --tls-ticket-key-file --verify-client-cacert --backend-request-buffer --backend-http2-connection-window-bits --conf --worker-write-burst --npn-list --fetch-ocsp-response-file --stream-read-timeout --accesslog-syslog --frontend-http2-read-timeout --listener-disable-timeout --frontend-http2-connection-window-bits --ciphers --strip-incoming-x-forwarded-for --daemon --backend-keep-alive-timeout --backend-http-proxy-uri --backend-http1-connections-per-host --rlimit-nofile --no-via --ocsp-update-interval --backend-write-timeout --client --http2-no-cookie-crumbling --worker-read-burst --client-proxy --http2-bridge --accesslog-format --errorlog-syslog --errorlog-file --http2-max-concurrent-streams --frontend-write-timeout --read-burst --backend-ipv4 --backend-ipv6 --backend --insecure --log-level --tls-proto-list --backend-http2-connections-per-worker --dh-param-file --worker-frontend-connections --header-field-buffer --no-server-push --no-location-rewrite --no-ocsp --backend-response-buffer --workers --frontend-http2-window-bits --no-host-rewrite --worker-write-rate --add-request-header --backend-tls-sni-field --subcert --help --frontend-frame-debug --pid-file --frontend-http2-dump-request-header --private-key-passwd-file --write-rate --altsvc --user --add-x-forwarded-for --syslog-facility --frontend-read-timeout --backlog --write-burst --backend-http2-window-bits --padding --stream-write-timeout --cacert --version --verify-client --backend-read-timeout --frontend --accesslog-file --http2-proxy --max-header-fields --backend-no-tls --client-private-key-file --client-cert-file --add-response-header --read-rate ' -- "$cur" ) )
+            COMPREPLY=( $( compgen -W '--worker-read-rate --frontend-no-tls --frontend-http2-dump-response-header --backend-http1-connections-per-frontend --tls-ticket-key-file --verify-client-cacert --include --backend-request-buffer --backend-http2-connection-window-bits --conf --worker-write-burst --npn-list --fetch-ocsp-response-file --stream-read-timeout --accesslog-syslog --frontend-http2-read-timeout --listener-disable-timeout --frontend-http2-connection-window-bits --ciphers --strip-incoming-x-forwarded-for --daemon --backend-keep-alive-timeout --backend-http-proxy-uri --backend-http1-connections-per-host --rlimit-nofile --no-via --ocsp-update-interval --backend-write-timeout --client --http2-no-cookie-crumbling --worker-read-burst --client-proxy --http2-bridge --accesslog-format --errorlog-syslog --errorlog-file --http2-max-concurrent-streams --frontend-write-timeout --read-burst --backend-ipv4 --backend-ipv6 --backend --insecure --log-level --tls-proto-list --backend-http2-connections-per-worker --dh-param-file --worker-frontend-connections --header-field-buffer --no-server-push --no-location-rewrite --no-ocsp --backend-response-buffer --workers --frontend-http2-window-bits --no-host-rewrite --worker-write-rate --add-request-header --backend-tls-sni-field --subcert --help --frontend-frame-debug --pid-file --frontend-http2-dump-request-header --private-key-passwd-file --write-rate --altsvc --user --add-x-forwarded-for --syslog-facility --frontend-read-timeout --backlog --write-burst --backend-http2-window-bits --padding --stream-write-timeout --cacert --version --verify-client --backend-read-timeout --frontend --accesslog-file --http2-proxy --max-header-fields --backend-no-tls --client-private-key-file --client-cert-file --add-response-header --read-rate ' -- "$cur" ) )
             ;;
         *)
             _filedir

doc/h2load.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "H2LOAD" "1" "June 27, 2015" "1.0.5" "nghttp2"
+.TH "H2LOAD" "1" "July 18, 2015" "1.1.2" "nghttp2"
 .SH NAME
 h2load \- HTTP/2 benchmarking tool
 .
@@ -113,6 +113,12 @@ Add/Override a header to the requests.
 .UNINDENT
 .INDENT 0.0
 .TP
+.B \-\-ciphers=<SUITE>
+Set allowed  cipher list.  The  format of the  string is
+described in OpenSSL ciphers(1).
+.UNINDENT
+.INDENT 0.0
+.TP
 .B \-p, \-\-no\-tls\-proto=<PROTOID>
 Specify ALPN identifier of the  protocol to be used when
 accessing http URI without SSL/TLS.

doc/h2load.1.rst

@@ -84,6 +84,11 @@ OPTIONS
 
     Add/Override a header to the requests.
 
+.. option:: --ciphers=<SUITE>
+
+    Set allowed  cipher list.  The  format of the  string is
+    described in OpenSSL ciphers(1).
+
 .. option:: -p, --no-tls-proto=<PROTOID>
 
     Specify ALPN identifier of the  protocol to be used when

doc/nghttp.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "NGHTTP" "1" "June 27, 2015" "1.0.5" "nghttp2"
+.TH "NGHTTP" "1" "July 18, 2015" "1.1.2" "nghttp2"
 .SH NAME
 nghttp \- HTTP/2 experimental client
 .
@@ -205,6 +205,12 @@ Disable server push.
 .UNINDENT
 .INDENT 0.0
 .TP
+.B \-\-max\-concurrent\-streams=<N>
+The  number of  concurrent  pushed  streams this  client
+accepts.
+.UNINDENT
+.INDENT 0.0
+.TP
 .B \-\-version
 Display version information and exit.
 .UNINDENT

doc/nghttp.1.rst

@@ -158,6 +158,11 @@ OPTIONS
 
     Disable server push.
 
+.. option:: --max-concurrent-streams=<N>
+
+    The  number of  concurrent  pushed  streams this  client
+    accepts.
+
 .. option:: --version
 
     Display version information and exit.

doc/nghttpd.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "NGHTTPD" "1" "June 27, 2015" "1.0.5" "nghttp2"
+.TH "NGHTTPD" "1" "July 18, 2015" "1.1.2" "nghttp2"
 .SH NAME
 nghttpd \- HTTP/2 experimental server
 .

doc/nghttpx.1

@@ -1,6 +1,6 @@
 .\" Man page generated from reStructuredText.
 .
-.TH "NGHTTPX" "1" "June 27, 2015" "1.0.5" "nghttp2"
+.TH "NGHTTPX" "1" "July 18, 2015" "1.1.2" "nghttp2"
 .SH NAME
 nghttpx \- HTTP/2 experimental proxy
 .
@@ -55,17 +55,69 @@ The options are categorized into several groups.
 .SS Connections
 .INDENT 0.0
 .TP
-.B \-b, \-\-backend=<HOST,PORT>
+.B \-b, \-\-backend=(<HOST>,<PORT>|unix:<PATH>)[;<PATTERN>[:...]]
 Set  backend  host  and   port.   The  multiple  backend
 addresses are  accepted by repeating this  option.  UNIX
 domain socket  can be  specified by prefixing  path name
-with "unix:" (e.g., unix:/var/run/backend.sock)
+with "unix:" (e.g., unix:/var/run/backend.sock).
+.sp
+Optionally, if <PATTERN>s are given, the backend address
+is only used  if request matches the pattern.   If \fI\%\-s\fP or
+\fI\%\-p\fP  is  used,  <PATTERN>s   are  ignored.   The  pattern
+matching  is closely  designed to  ServeMux in  net/http
+package of Go  programming language.  <PATTERN> consists
+of path, host + path or  just host.  The path must start
+with "\fI/\fP".  If  it ends with "\fI/\fP", it  matches all request
+path in  its subtree.  To  deal with the request  to the
+directory without  trailing slash,  the path  which ends
+with "\fI/\fP" also matches the  request path which only lacks
+trailing \(aq\fI/\fP\(aq  (e.g., path  "\fI/foo/\fP" matches  request path
+"\fI/foo\fP").  If it does not end with "\fI/\fP", it performs exact
+match against  the request path.   If host is  given, it
+performs exact match against  the request host.  If host
+alone  is given,  "\fI/\fP"  is  appended to  it,  so that  it
+matches  all   request  paths  under  the   host  (e.g.,
+specifying "nghttp2.org" equals to "nghttp2.org/").
+.sp
+Patterns with  host take  precedence over  patterns with
+just path.   Then, longer patterns take  precedence over
+shorter  ones,  breaking  a  tie by  the  order  of  the
+appearance in the configuration.
+.sp
+If <PATTERN> is  omitted, "\fI/\fP" is used  as pattern, which
+matches  all  request  paths (catch\-all  pattern).   The
+catch\-all backend must be given.
+.sp
+When doing  a match, nghttpx made  some normalization to
+pattern, request host and path.  For host part, they are
+converted to lower case.  For path part, percent\-encoded
+unreserved characters  defined in RFC 3986  are decoded,
+and any  dot\-segments (".."  and ".")   are resolved and
+removed.
+.sp
+For   example,   \fI\%\-b\fP\(aq127.0.0.1,8080;nghttp2.org/httpbin/\(aq
+matches the  request host "nghttp2.org" and  the request
+path "\fI/httpbin/get\fP", but does not match the request host
+"nghttp2.org" and the request path "\fI/index.html\fP".
+.sp
+The  multiple <PATTERN>s  can  be specified,  delimiting
+them            by           ":".             Specifying
+\fI\%\-b\fP\(aq127.0.0.1,8080;nghttp2.org:www.nghttp2.org\(aq  has  the
+same  effect  to specify  \fI\%\-b\fP\(aq127.0.0.1,8080;nghttp2.org\(aq
+and \fI\%\-b\fP\(aq127.0.0.1,8080;www.nghttp2.org\(aq.
+.sp
+The backend addresses sharing same <PATTERN> are grouped
+together forming  load balancing  group.
+.sp
+Since ";" and ":" are  used as delimiter, <PATTERN> must
+not  contain these  characters.  Since  ";" has  special
+meaning in shell, the option value must be quoted.
 .sp
 Default: \fB127.0.0.1,80\fP
 .UNINDENT
 .INDENT 0.0
 .TP
-.B \-f, \-\-frontend=<HOST,PORT>
+.B \-f, \-\-frontend=(<HOST>,<PORT>|unix:<PATH>)
 Set  frontend  host and  port.   If  <HOST> is  \(aq*\(aq,  it
 assumes  all addresses  including  both  IPv4 and  IPv6.
 UNIX domain  socket can  be specified by  prefixing path
@@ -195,9 +247,14 @@ Default: \fB0\fP
 .INDENT 0.0
 .TP
 .B \-\-backend\-http2\-connections\-per\-worker=<N>
-Set  maximum number  of HTTP/2  connections per  worker.
-The  default  value is  0,  which  means the  number  of
-backend addresses specified by \fI\%\-b\fP option.
+Set   maximum   number   of  backend   HTTP/2   physical
+connections  per  worker.   If  pattern is  used  in  \fI\%\-b\fP
+option, this limit is applied  to each pattern group (in
+other  words, each  pattern group  can have  maximum <N>
+HTTP/2  connections).  The  default  value  is 0,  which
+means  that  the value  is  adjusted  to the  number  of
+backend addresses.  If pattern  is used, this adjustment
+is done for each pattern group.
 .UNINDENT
 .INDENT 0.0
 .TP
@@ -626,8 +683,20 @@ $pid: PID of the running process.
 $alpn: ALPN identifier of the protocol which generates
 the response.   For HTTP/1,  ALPN is  always http/1.1,
 regardless of minor version.
+.IP \(bu 2
+$ssl_cipher: cipher used for SSL/TLS connection.
+.IP \(bu 2
+$ssl_protocol: protocol for SSL/TLS connection.
+.IP \(bu 2
+$ssl_session_id: session ID for SSL/TLS connection.
+.IP \(bu 2
+$ssl_session_reused:  "r"   if  SSL/TLS   session  was
+reused.  Otherwise, "."
 .UNINDENT
 .sp
+The  variable  can  be  enclosed  by  "{"  and  "}"  for
+disambiguation (e.g., ${remote_addr}).
+.sp
 Default: \fB$remote_addr \- \- [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"\fP
 .UNINDENT
 .INDENT 0.0
@@ -786,6 +855,14 @@ Default: \fB/etc/nghttpx/nghttpx.conf\fP
 .UNINDENT
 .INDENT 0.0
 .TP
+.B \-\-include=<PATH>
+Load additional configurations from <PATH>.  File <PATH>
+is  read  when  configuration  parser  encountered  this
+option.  This option can be used multiple times, or even
+recursively.
+.UNINDENT
+.INDENT 0.0
+.TP
 .B \-v, \-\-version
 Print version and exit.
 .UNINDENT

doc/nghttpx.1.rst

@@ -37,16 +37,69 @@ The options are categorized into several groups.
 Connections
 ~~~~~~~~~~~
 
-.. option:: -b, --backend=<HOST,PORT>
+.. option:: -b, --backend=(<HOST>,<PORT>|unix:<PATH>)[;<PATTERN>[:...]]
 
     Set  backend  host  and   port.   The  multiple  backend
     addresses are  accepted by repeating this  option.  UNIX
     domain socket  can be  specified by prefixing  path name
-    with "unix:" (e.g., unix:/var/run/backend.sock)
+    with "unix:" (e.g., unix:/var/run/backend.sock).
+
+    Optionally, if <PATTERN>s are given, the backend address
+    is only used  if request matches the pattern.   If :option:`-s` or
+    :option:`-p`  is  used,  <PATTERN>s   are  ignored.   The  pattern
+    matching  is closely  designed to  ServeMux in  net/http
+    package of Go  programming language.  <PATTERN> consists
+    of path, host + path or  just host.  The path must start
+    with "*/*".  If  it ends with "*/*", it  matches all request
+    path in  its subtree.  To  deal with the request  to the
+    directory without  trailing slash,  the path  which ends
+    with "*/*" also matches the  request path which only lacks
+    trailing '*/*'  (e.g., path  "*/foo/*" matches  request path
+    "*/foo*").  If it does not end with "*/*", it performs exact
+    match against  the request path.   If host is  given, it
+    performs exact match against  the request host.  If host
+    alone  is given,  "*/*"  is  appended to  it,  so that  it
+    matches  all   request  paths  under  the   host  (e.g.,
+    specifying "nghttp2.org" equals to "nghttp2.org/").
+
+    Patterns with  host take  precedence over  patterns with
+    just path.   Then, longer patterns take  precedence over
+    shorter  ones,  breaking  a  tie by  the  order  of  the
+    appearance in the configuration.
+
+    If <PATTERN> is  omitted, "*/*" is used  as pattern, which
+    matches  all  request  paths (catch-all  pattern).   The
+    catch-all backend must be given.
+
+    When doing  a match, nghttpx made  some normalization to
+    pattern, request host and path.  For host part, they are
+    converted to lower case.  For path part, percent-encoded
+    unreserved characters  defined in RFC 3986  are decoded,
+    and any  dot-segments (".."  and ".")   are resolved and
+    removed.
+
+    For   example,   :option:`-b`\'127.0.0.1,8080;nghttp2.org/httpbin/'
+    matches the  request host "nghttp2.org" and  the request
+    path "*/httpbin/get*", but does not match the request host
+    "nghttp2.org" and the request path "*/index.html*".
+
+    The  multiple <PATTERN>s  can  be specified,  delimiting
+    them            by           ":".             Specifying
+    :option:`-b`\'127.0.0.1,8080;nghttp2.org:www.nghttp2.org'  has  the
+    same  effect  to specify  :option:`-b`\'127.0.0.1,8080;nghttp2.org'
+    and :option:`-b`\'127.0.0.1,8080;www.nghttp2.org'.
+
+    The backend addresses sharing same <PATTERN> are grouped
+    together forming  load balancing  group.
+
+    Since ";" and ":" are  used as delimiter, <PATTERN> must
+    not  contain these  characters.  Since  ";" has  special
+    meaning in shell, the option value must be quoted.
+
 
     Default: ``127.0.0.1,80``
 
-.. option:: -f, --frontend=<HOST,PORT>
+.. option:: -f, --frontend=(<HOST>,<PORT>|unix:<PATH>)
 
     Set  frontend  host and  port.   If  <HOST> is  '\*',  it
     assumes  all addresses  including  both  IPv4 and  IPv6.
@@ -165,9 +218,14 @@ Performance
 
 .. option:: --backend-http2-connections-per-worker=<N>
 
-    Set  maximum number  of HTTP/2  connections per  worker.
-    The  default  value is  0,  which  means the  number  of
-    backend addresses specified by :option:`-b` option.
+    Set   maximum   number   of  backend   HTTP/2   physical
+    connections  per  worker.   If  pattern is  used  in  :option:`-b`
+    option, this limit is applied  to each pattern group (in
+    other  words, each  pattern group  can have  maximum <N>
+    HTTP/2  connections).  The  default  value  is 0,  which
+    means  that  the value  is  adjusted  to the  number  of
+    backend addresses.  If pattern  is used, this adjustment
+    is done for each pattern group.
 
 .. option:: --backend-http1-connections-per-host=<N>
 
@@ -550,6 +608,14 @@ Logging
     * $alpn: ALPN identifier of the protocol which generates
       the response.   For HTTP/1,  ALPN is  always http/1.1,
       regardless of minor version.
+    * $ssl_cipher: cipher used for SSL/TLS connection.
+    * $ssl_protocol: protocol for SSL/TLS connection.
+    * $ssl_session_id: session ID for SSL/TLS connection.
+    * $ssl_session_reused:  "r"   if  SSL/TLS   session  was
+      reused.  Otherwise, "."
+
+    The  variable  can  be  enclosed  by  "{"  and  "}"  for
+    disambiguation (e.g., ${remote_addr}).
 
 
     Default: ``$remote_addr - - [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"``
@@ -700,6 +766,13 @@ Misc
 
     Default: ``/etc/nghttpx/nghttpx.conf``
 
+.. option:: --include=<PATH>
+
+    Load additional configurations from <PATH>.  File <PATH>
+    is  read  when  configuration  parser  encountered  this
+    option.  This option can be used multiple times, or even
+    recursively.
+
 .. option:: -v, --version
 
     Print version and exit.

gennghttpxfun.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+
+from gentokenlookup import gentokenlookup
+
+OPTIONS = [
+    "private-key-file",
+    "private-key-passwd-file",
+    "certificate-file",
+    "dh-param-file",
+    "subcert",
+    "backend",
+    "frontend",
+    "workers",
+    "http2-max-concurrent-streams",
+    "log-level",
+    "daemon",
+    "http2-proxy",
+    "http2-bridge",
+    "client-proxy",
+    "add-x-forwarded-for",
+    "strip-incoming-x-forwarded-for",
+    "no-via",
+    "frontend-http2-read-timeout",
+    "frontend-read-timeout",
+    "frontend-write-timeout",
+    "backend-read-timeout",
+    "backend-write-timeout",
+    "stream-read-timeout",
+    "stream-write-timeout",
+    "accesslog-file",
+    "accesslog-syslog",
+    "accesslog-format",
+    "errorlog-file",
+    "errorlog-syslog",
+    "backend-keep-alive-timeout",
+    "frontend-http2-window-bits",
+    "backend-http2-window-bits",
+    "frontend-http2-connection-window-bits",
+    "backend-http2-connection-window-bits",
+    "frontend-no-tls",
+    "backend-no-tls",
+    "backend-tls-sni-field",
+    "pid-file",
+    "user",
+    "syslog-facility",
+    "backlog",
+    "ciphers",
+    "client",
+    "insecure",
+    "cacert",
+    "backend-ipv4",
+    "backend-ipv6",
+    "backend-http-proxy-uri",
+    "read-rate",
+    "read-burst",
+    "write-rate",
+    "write-burst",
+    "worker-read-rate",
+    "worker-read-burst",
+    "worker-write-rate",
+    "worker-write-burst",
+    "npn-list",
+    "tls-proto-list",
+    "verify-client",
+    "verify-client-cacert",
+    "client-private-key-file",
+    "client-cert-file",
+    "frontend-http2-dump-request-header",
+    "frontend-http2-dump-response-header",
+    "http2-no-cookie-crumbling",
+    "frontend-frame-debug",
+    "padding",
+    "altsvc",
+    "add-request-header",
+    "add-response-header",
+    "worker-frontend-connections",
+    "no-location-rewrite",
+    "no-host-rewrite",
+    "backend-http1-connections-per-host",
+    "backend-http1-connections-per-frontend",
+    "listener-disable-timeout",
+    "tls-ticket-key-file",
+    "rlimit-nofile",
+    "backend-request-buffer",
+    "backend-response-buffer",
+    "no-server-push",
+    "backend-http2-connections-per-worker",
+    "fetch-ocsp-response-file",
+    "ocsp-update-interval",
+    "no-ocsp",
+    "header-field-buffer",
+    "max-header-fields",
+    "include",
+    "tls-ticket-cipher",
+    "host-rewrite",
+    "conf",
+]
+
+LOGVARS = [
+    "remote_addr",
+    "time_local",
+    "time_iso8601",
+    "request",
+    "status",
+    "body_bytes_sent",
+    "remote_port",
+    "server_port",
+    "request_time",
+    "pid",
+    "alpn",
+    "ssl_cipher",
+    "ssl_protocol",
+    "ssl_session_id",
+    "ssl_session_reused",
+]
+
+if __name__ == '__main__':
+    gentokenlookup(OPTIONS, 'SHRPX_OPTID', value_type='char', comp_fun='util::strieq_l')
+    gentokenlookup(LOGVARS, 'SHRPX_LOGF', value_type='char', comp_fun='util::strieq_l', return_type='LogFragmentType', fail_value='SHRPX_LOGF_NONE')

gentokenlookup.py

@@ -33,10 +33,10 @@ enum {'''
   {}_MAXIDX,
 }};'''.format(prefix)
 
-def gen_index_header(tokens, prefix):
+def gen_index_header(tokens, prefix, value_type, comp_fun, return_type, fail_value):
     print '''\
-int lookup_token(const uint8_t *name, size_t namelen) {
-  switch (namelen) {'''
+{} lookup_token(const {} *name, size_t namelen) {{
+  switch (namelen) {{'''.format(return_type, value_type)
     b = build_header(tokens)
     for size in sorted(b.keys()):
         ents = b[size]
@@ -50,20 +50,20 @@ int lookup_token(const uint8_t *name, size_t namelen) {
     case '{}':'''.format(c)
             for k in headers:
                 print '''\
-      if (util::streq_l("{}", name, {})) {{
+      if ({}("{}", name, {})) {{
         return {};
-      }}'''.format(k[:-1], size - 1, to_enum_hd(k, prefix))
+      }}'''.format(comp_fun, k[:-1], size - 1, to_enum_hd(k, prefix))
             print '''\
       break;'''
         print '''\
     }
     break;'''
     print '''\
-  }
-  return -1;
-}'''
+  }}
+  return {};
+}}'''.format(fail_value)
 
-def gentokenlookup(tokens, prefix):
+def gentokenlookup(tokens, prefix, value_type='uint8_t', comp_fun='util::streq_l', return_type='int', fail_value='-1'):
     gen_enum(tokens, prefix)
     print ''
-    gen_index_header(tokens, prefix)
+    gen_index_header(tokens, prefix, value_type, comp_fun, return_type, fail_value)

lib/includes/nghttp2/nghttp2.h

@@ -3278,10 +3278,6 @@ NGHTTP2_EXTERN int nghttp2_submit_rst_stream(nghttp2_session *session,
  * :enum:`NGHTTP2_ERR_INVALID_ARGUMENT`
  *     The |iv| contains invalid value (e.g., initial window size
  *     strictly greater than (1 << 31) - 1.
- * :enum:`NGHTTP2_ERR_TOO_MANY_INFLIGHT_SETTINGS`
- *     There is already another in-flight SETTINGS.  Note that the
- *     current implementation only allows 1 in-flight SETTINGS frame
- *     without ACK flag set.
  * :enum:`NGHTTP2_ERR_NOMEM`
  *     Out of memory.
  */
@@ -3777,11 +3773,22 @@ NGHTTP2_EXTERN void nghttp2_hd_inflate_del(nghttp2_hd_inflater *inflater);
  * The |settings_hd_table_bufsize_max| should be the value transmitted
  * in SETTINGS_HEADER_TABLE_SIZE.
  *
+ * This function must not be called while header block is being
+ * inflated.  In other words, this function must be called after
+ * initialization of |inflater|, but before calling
+ * `nghttp2_hd_inflate_hd()`, or after
+ * `nghttp2_hd_inflate_end_headers()`.  Otherwise,
+ * `NGHTTP2_ERR_INVALID_STATE` was returned.
+ *
  * This function returns 0 if it succeeds, or one of the following
  * negative error codes:
  *
  * :enum:`NGHTTP2_ERR_NOMEM`
  *     Out of memory.
+ * :enum:`NGHTTP2_ERR_INVALID_STATE`
+ *     The function is called while header block is being inflated.
+ *     Probably, application missed to call
+ *     `nghttp2_hd_inflate_end_headers()`.
  */
 NGHTTP2_EXTERN int
 nghttp2_hd_inflate_change_table_size(nghttp2_hd_inflater *inflater,

lib/nghttp2_hd.c

@@ -704,7 +704,7 @@ int nghttp2_hd_inflate_init(nghttp2_hd_inflater *inflater, nghttp2_mem *mem) {
   inflater->nv_keep = NULL;
 
   inflater->opcode = NGHTTP2_HD_OPCODE_NONE;
-  inflater->state = NGHTTP2_HD_STATE_OPCODE;
+  inflater->state = NGHTTP2_HD_STATE_INFLATE_START;
 
   rv = nghttp2_bufs_init3(&inflater->nvbufs, NGHTTP2_HD_MAX_NV / 8, 8, 1, 0,
                           mem);
@@ -1261,6 +1261,15 @@ int nghttp2_hd_deflate_change_table_size(nghttp2_hd_deflater *deflater,
 
 int nghttp2_hd_inflate_change_table_size(nghttp2_hd_inflater *inflater,
                                          size_t settings_hd_table_bufsize_max) {
+  switch (inflater->state) {
+  case NGHTTP2_HD_STATE_EXPECT_TABLE_SIZE:
+  case NGHTTP2_HD_STATE_INFLATE_START:
+    break;
+  default:
+    return NGHTTP2_ERR_INVALID_STATE;
+  }
+
+  inflater->state = NGHTTP2_HD_STATE_EXPECT_TABLE_SIZE;
   inflater->settings_hd_table_bufsize_max = settings_hd_table_bufsize_max;
   inflater->ctx.hd_table_bufsize_max = settings_hd_table_bufsize_max;
   hd_context_shrink_table_size(&inflater->ctx);
@@ -1951,9 +1960,25 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater,
   for (; in != last || busy;) {
     busy = 0;
     switch (inflater->state) {
+    case NGHTTP2_HD_STATE_EXPECT_TABLE_SIZE:
+      if ((*in & 0xe0u) != 0x20u) {
+        DEBUGF(fprintf(stderr, "inflatehd: header table size change was "
+                               "expected, but saw 0x%02x as first byte",
+                       *in));
+        rv = NGHTTP2_ERR_HEADER_COMP;
+        goto fail;
+      }
+    /* fall through */
+    case NGHTTP2_HD_STATE_INFLATE_START:
     case NGHTTP2_HD_STATE_OPCODE:
       if ((*in & 0xe0u) == 0x20u) {
         DEBUGF(fprintf(stderr, "inflatehd: header table size change\n"));
+        if (inflater->state == NGHTTP2_HD_STATE_OPCODE) {
+          DEBUGF(fprintf(stderr, "inflatehd: header table size change must "
+                                 "appear at the head of header block\n"));
+          rv = NGHTTP2_ERR_HEADER_COMP;
+          goto fail;
+        }
         inflater->opcode = NGHTTP2_HD_OPCODE_INDEXED;
         inflater->state = NGHTTP2_HD_STATE_READ_TABLE_SIZE;
       } else if (*in & 0x80u) {
@@ -1997,7 +2022,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater,
       DEBUGF(fprintf(stderr, "inflatehd: table_size=%zu\n", inflater->left));
       inflater->ctx.hd_table_bufsize_max = inflater->left;
       hd_context_shrink_table_size(&inflater->ctx);
-      inflater->state = NGHTTP2_HD_STATE_OPCODE;
+      inflater->state = NGHTTP2_HD_STATE_INFLATE_START;
       break;
     case NGHTTP2_HD_STATE_READ_INDEX: {
       size_t prefixlen;
@@ -2282,6 +2307,7 @@ fail:
 
 int nghttp2_hd_inflate_end_headers(nghttp2_hd_inflater *inflater) {
   hd_inflate_keep_free(inflater);
+  inflater->state = NGHTTP2_HD_STATE_INFLATE_START;
   return 0;
 }
 

lib/nghttp2_hd.h

@@ -151,6 +151,8 @@ typedef enum {
 } nghttp2_hd_opcode;
 
 typedef enum {
+  NGHTTP2_HD_STATE_EXPECT_TABLE_SIZE,
+  NGHTTP2_HD_STATE_INFLATE_START,
   NGHTTP2_HD_STATE_OPCODE,
   NGHTTP2_HD_STATE_READ_TABLE_SIZE,
   NGHTTP2_HD_STATE_READ_INDEX,

lib/nghttp2_session.c

@@ -362,8 +362,6 @@ static int session_new(nghttp2_session **session_ptr,
   (*session_ptr)->local_last_stream_id = (1u << 31) - 1;
   (*session_ptr)->remote_last_stream_id = (1u << 31) - 1;
 
-  (*session_ptr)->inflight_niv = -1;
-
   (*session_ptr)->pending_local_max_concurrent_stream =
       NGHTTP2_INITIAL_MAX_CONCURRENT_STREAMS;
   (*session_ptr)->pending_enable_push = 1;
@@ -561,8 +559,42 @@ static void ob_q_free(nghttp2_outbound_queue *q, nghttp2_mem *mem) {
   }
 }
 
+static int inflight_settings_new(nghttp2_inflight_settings **settings_ptr,
+                                 const nghttp2_settings_entry *iv, size_t niv,
+                                 nghttp2_mem *mem) {
+  *settings_ptr = nghttp2_mem_malloc(mem, sizeof(nghttp2_inflight_settings));
+  if (!*settings_ptr) {
+    return NGHTTP2_ERR_NOMEM;
+  }
+
+  if (niv > 0) {
+    (*settings_ptr)->iv = nghttp2_frame_iv_copy(iv, niv, mem);
+    if (!(*settings_ptr)->iv) {
+      return NGHTTP2_ERR_NOMEM;
+    }
+  } else {
+    (*settings_ptr)->iv = NULL;
+  }
+
+  (*settings_ptr)->niv = niv;
+  (*settings_ptr)->next = NULL;
+
+  return 0;
+}
+
+static void inflight_settings_del(nghttp2_inflight_settings *settings,
+                                  nghttp2_mem *mem) {
+  if (!settings) {
+    return;
+  }
+
+  nghttp2_mem_free(mem, settings->iv);
+  nghttp2_mem_free(mem, settings);
+}
+
 void nghttp2_session_del(nghttp2_session *session) {
   nghttp2_mem *mem;
+  nghttp2_inflight_settings *settings;
 
   if (session == NULL) {
     return;
@@ -570,7 +602,11 @@ void nghttp2_session_del(nghttp2_session *session) {
 
   mem = &session->mem;
 
-  nghttp2_mem_free(mem, session->inflight_iv);
+  for (settings = session->inflight_settings_head; settings;) {
+    nghttp2_inflight_settings *next = settings->next;
+    inflight_settings_del(settings, mem);
+    settings = next;
+  }
 
   nghttp2_stream_roots_free(&session->roots);
 
@@ -3939,10 +3975,6 @@ int nghttp2_session_update_local_settings(nghttp2_session *session,
     }
   }
 
-  session->pending_local_max_concurrent_stream =
-      NGHTTP2_INITIAL_MAX_CONCURRENT_STREAMS;
-  session->pending_enable_push = 1;
-
   return 0;
 }
 
@@ -3951,6 +3983,7 @@ int nghttp2_session_on_settings_received(nghttp2_session *session,
   int rv;
   size_t i;
   nghttp2_mem *mem;
+  nghttp2_inflight_settings *settings;
 
   mem = &session->mem;
 
@@ -3964,15 +3997,21 @@ int nghttp2_session_on_settings_received(nghttp2_session *session,
           session, frame, NGHTTP2_ERR_FRAME_SIZE_ERROR,
           "SETTINGS: ACK and payload != 0");
     }
-    if (session->inflight_niv == -1) {
+
+    settings = session->inflight_settings_head;
+
+    if (!settings) {
       return session_handle_invalid_connection(
           session, frame, NGHTTP2_ERR_PROTO, "SETTINGS: unexpected ACK");
     }
-    rv = nghttp2_session_update_local_settings(session, session->inflight_iv,
-                                               session->inflight_niv);
-    nghttp2_mem_free(mem, session->inflight_iv);
-    session->inflight_iv = NULL;
-    session->inflight_niv = -1;
+
+    rv = nghttp2_session_update_local_settings(session, settings->iv,
+                                               settings->niv);
+
+    session->inflight_settings_head = settings->next;
+
+    inflight_settings_del(settings, mem);
+
     if (rv != 0) {
       if (nghttp2_is_fatal(rv)) {
         return rv;
@@ -4633,7 +4672,7 @@ static int session_on_data_received_fail_fast(nghttp2_session *session) {
   if (!stream) {
     if (session_detect_idle_stream(session, stream_id)) {
       failure_reason = "DATA: stream in idle";
-      error_code = NGHTTP2_STREAM_CLOSED;
+      error_code = NGHTTP2_PROTOCOL_ERROR;
       goto fail;
     }
     return NGHTTP2_ERR_IGN_PAYLOAD;
@@ -6091,6 +6130,22 @@ int nghttp2_session_add_window_update(nghttp2_session *session, uint8_t flags,
   return 0;
 }
 
+static void
+session_append_inflight_settings(nghttp2_session *session,
+                                 nghttp2_inflight_settings *settings) {
+  nghttp2_inflight_settings *i;
+
+  if (!session->inflight_settings_head) {
+    session->inflight_settings_head = settings;
+    return;
+  }
+
+  for (i = session->inflight_settings_head; i->next; i = i->next)
+    ;
+
+  i->next = settings;
+}
+
 int nghttp2_session_add_settings(nghttp2_session *session, uint8_t flags,
                                  const nghttp2_settings_entry *iv, size_t niv) {
   nghttp2_outbound_item *item;
@@ -6099,16 +6154,13 @@ int nghttp2_session_add_settings(nghttp2_session *session, uint8_t flags,
   size_t i;
   int rv;
   nghttp2_mem *mem;
+  nghttp2_inflight_settings *inflight_settings = NULL;
 
   mem = &session->mem;
 
-  if (flags & NGHTTP2_FLAG_ACK) {
-    if (niv != 0) {
+  if ((flags & NGHTTP2_FLAG_ACK) && niv != 0) {
     return NGHTTP2_ERR_INVALID_ARGUMENT;
   }
-  } else if (session->inflight_niv != -1) {
-    return NGHTTP2_ERR_TOO_MANY_INFLIGHT_SETTINGS;
-  }
 
   if (!nghttp2_iv_check(iv, niv)) {
     return NGHTTP2_ERR_INVALID_ARGUMENT;
@@ -6130,19 +6182,13 @@ int nghttp2_session_add_settings(nghttp2_session *session, uint8_t flags,
   }
 
   if ((flags & NGHTTP2_FLAG_ACK) == 0) {
-    if (niv > 0) {
-      session->inflight_iv = nghttp2_frame_iv_copy(iv, niv, mem);
-
-      if (session->inflight_iv == NULL) {
+    rv = inflight_settings_new(&inflight_settings, iv, niv, mem);
+    if (rv != 0) {
+      assert(nghttp2_is_fatal(rv));
       nghttp2_mem_free(mem, iv_copy);
       nghttp2_mem_free(mem, item);
-        return NGHTTP2_ERR_NOMEM;
+      return rv;
     }
-    } else {
-      session->inflight_iv = NULL;
-    }
-
-    session->inflight_niv = niv;
   }
 
   nghttp2_outbound_item_init(item);
@@ -6155,11 +6201,7 @@ int nghttp2_session_add_settings(nghttp2_session *session, uint8_t flags,
     /* The only expected error is fatal one */
     assert(nghttp2_is_fatal(rv));
 
-    if ((flags & NGHTTP2_FLAG_ACK) == 0) {
-      nghttp2_mem_free(mem, session->inflight_iv);
-      session->inflight_iv = NULL;
-      session->inflight_niv = -1;
-    }
+    inflight_settings_del(inflight_settings, mem);
 
     nghttp2_frame_settings_free(&frame->settings, mem);
     nghttp2_mem_free(mem, item);
@@ -6167,6 +6209,8 @@ int nghttp2_session_add_settings(nghttp2_session *session, uint8_t flags,
     return rv;
   }
 
+  session_append_inflight_settings(session, inflight_settings);
+
   /* Extract NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS and ENABLE_PUSH
      here.  We use it to refuse the incoming stream and PUSH_PROMISE
      with RST_STREAM. */

lib/nghttp2_session.h

@@ -140,6 +140,17 @@ typedef enum {
   NGHTTP2_GOAWAY_RECV = 0x8
 } nghttp2_goaway_flag;
 
+/* nghttp2_inflight_settings stores the SETTINGS entries which local
+   endpoint has sent to the remote endpoint, and has not received ACK
+   yet. */
+struct nghttp2_inflight_settings {
+  struct nghttp2_inflight_settings *next;
+  nghttp2_settings_entry *iv;
+  size_t niv;
+};
+
+typedef struct nghttp2_inflight_settings nghttp2_inflight_settings;
+
 struct nghttp2_session {
   nghttp2_map /* <nghttp2_stream*> */ streams;
   nghttp2_stream_roots roots;
@@ -176,12 +187,9 @@ struct nghttp2_session {
   /* Points to the oldest idle stream.  NULL if there is no idle
      stream.  Only used when session is initialized as erver. */
   nghttp2_stream *idle_stream_tail;
-  /* In-flight SETTINGS values. NULL does not necessarily mean there
-     is no in-flight SETTINGS. */
-  nghttp2_settings_entry *inflight_iv;
-  /* The number of entries in |inflight_iv|. -1 if there is no
-     in-flight SETTINGS. */
-  ssize_t inflight_niv;
+  /* Queue of In-flight SETTINGS values.  SETTINGS bearing ACK is not
+     considered as in-flight. */
+  nghttp2_inflight_settings *inflight_settings_head;
   /* The number of outgoing streams. This will be capped by
      remote_settings.max_concurrent_streams. */
   size_t num_outgoing_streams;

python/Makefile.am

@@ -37,7 +37,7 @@ install-exec-local:
 
 uninstall-local:
 	rm -f $(DESTDIR)$(libdir)/python*/site-packages/nghttp2.so
-	rm -f $(DESTDIR)$(libdir)/python*/site-packages/python_nghttp2-*.egg-info
+	rm -f $(DESTDIR)$(libdir)/python*/site-packages/python_nghttp2-*.egg
 
 clean-local:
 	$(PYTHON) setup.py clean --all

python/setup.py.in

@@ -22,7 +22,6 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 from setuptools import setup, Extension
-from Cython.Build import cythonize
 
 LIBS = ['nghttp2']
 
@@ -34,13 +33,13 @@ setup(
     author_email = 'tatsuhiro.t@gmail.com',
     url = 'https://nghttp2.org/',
     keywords = [],
-    ext_modules = cythonize([Extension("nghttp2",
-                             ["nghttp2.pyx"],
+    ext_modules = [Extension("nghttp2",
+                             ["nghttp2.c"],
                              include_dirs=['@top_srcdir@/lib',
                                            '@top_srcdir@/lib/includes',
                                            '@top_builddir@/lib/includes'],
                              library_dirs=['@top_builddir@/lib/.libs',
                                            '@top_builddir@'],
-                             libraries=LIBS)]),
+                             libraries=LIBS)],
     long_description='TBD'
     )

src/HttpServer.cc

@@ -571,6 +571,12 @@ int Http2Handler::tls_handshake() {
     return -1;
   }
 
+  if (sessions_->get_config()->verbose) {
+    if (SSL_session_reused(ssl_)) {
+      std::cerr << "SSL/TLS session reused" << std::endl;
+    }
+  }
+
   return 0;
 }
 
@@ -1102,7 +1108,7 @@ void prepare_response(Stream *stream, Http2Handler *hd,
 
     if (last_mod_found && static_cast<time_t>(buf.st_mtime) <= last_mod) {
       close(file);
-      prepare_status_response(stream, hd, 304);
+      hd->submit_response("304", stream->stream_id, nullptr);
 
       return;
     }
@@ -1111,7 +1117,7 @@ void prepare_response(Stream *stream, Http2Handler *hd,
         path, FileEntry(path, buf.st_size, buf.st_mtime, file));
   } else if (last_mod_found && file_ent->mtime <= last_mod) {
     sessions->release_fd(file_ent->path);
-    prepare_status_response(stream, hd, 304);
+    hd->submit_response("304", stream->stream_id, nullptr);
 
     return;
   }
@@ -1630,7 +1636,6 @@ FileEntry make_status_body(int status, uint16_t port) {
 enum {
   IDX_200,
   IDX_301,
-  IDX_304,
   IDX_400,
   IDX_404,
 };
@@ -1639,7 +1644,6 @@ HttpServer::HttpServer(const Config *config) : config_(config) {
   status_pages_ = std::vector<StatusPage>{
       {"200", make_status_body(200, config_->port)},
       {"301", make_status_body(301, config_->port)},
-      {"304", make_status_body(304, config_->port)},
       {"400", make_status_body(400, config_->port)},
       {"404", make_status_body(404, config_->port)},
   };
@@ -1666,7 +1670,6 @@ int verify_callback(int preverify_ok, X509_STORE_CTX *ctx) {
 namespace {
 int start_listen(HttpServer *sv, struct ev_loop *loop, Sessions *sessions,
                  const Config *config) {
-  addrinfo hints;
   int r;
   bool ok = false;
   const char *addr = nullptr;
@@ -1674,7 +1677,7 @@ int start_listen(HttpServer *sv, struct ev_loop *loop, Sessions *sessions,
   auto acceptor = std::make_shared<AcceptHandler>(sv, sessions, config);
   auto service = util::utos(config->port);
 
-  memset(&hints, 0, sizeof(addrinfo));
+  addrinfo hints{};
   hints.ai_family = AF_UNSPEC;
   hints.ai_socktype = SOCK_STREAM;
   hints.ai_flags = AI_PASSIVE;
@@ -1885,8 +1888,6 @@ const StatusPage *HttpServer::get_status_page(int status) const {
     return &status_pages_[IDX_200];
   case 301:
     return &status_pages_[IDX_301];
-  case 304:
-    return &status_pages_[IDX_304];
   case 400:
     return &status_pages_[IDX_400];
   case 404:

src/Makefile.am

@@ -176,6 +176,7 @@ lib_LTLIBRARIES = libnghttp2_asio.la
 libnghttp2_asio_la_SOURCES = \
 	util.cc util.h http2.cc http2.h \
 	ssl.cc ssl.h \
+	timegm.c timegm.h \
 	asio_common.cc asio_common.h \
 	asio_io_service_pool.cc asio_io_service_pool.h \
 	asio_server_http2.cc \
@@ -207,10 +208,10 @@ libnghttp2_asio_la_LDFLAGS = -no-undefined -version-info 1:0:0
 libnghttp2_asio_la_LIBADD = \
 	$(top_builddir)/lib/libnghttp2.la \
 	$(top_builddir)/third-party/libhttp-parser.la \
+	@OPENSSL_LIBS@ \
 	${BOOST_LDFLAGS} \
 	${BOOST_ASIO_LIB} \
 	${BOOST_THREAD_LIB} \
-	${BOOST_SYSTEM_LIB} \
-	@OPENSSL_LIBS@
+	${BOOST_SYSTEM_LIB}
 
 endif # ENABLE_ASIO_LIB

src/h2load.cc

@@ -879,9 +879,8 @@ process_time_stats(const std::vector<std::unique_ptr<Worker>> &workers) {
 namespace {
 void resolve_host() {
   int rv;
-  addrinfo hints, *res;
+  addrinfo hints{}, *res;
 
-  memset(&hints, 0, sizeof(hints));
   hints.ai_family = AF_UNSPEC;
   hints.ai_socktype = SOCK_STREAM;
   hints.ai_protocol = 0;
@@ -938,29 +937,30 @@ int client_select_next_proto_cb(SSL *ssl, unsigned char **out,
 } // namespace
 
 namespace {
-template <typename Iterator>
-std::vector<std::string> parse_uris(Iterator first, Iterator last) {
+// Use std::vector<std::string>::iterator explicitly, without that,
+// http_parser_url u{} fails with clang-3.4.
+std::vector<std::string> parse_uris(std::vector<std::string>::iterator first,
+                                    std::vector<std::string>::iterator last) {
   std::vector<std::string> reqlines;
 
-  // First URI is treated specially.  We use scheme, host and port of
-  // this URI and ignore those in the remaining URIs if present.
-  http_parser_url u;
-  memset(&u, 0, sizeof(u));
-
   if (first == last) {
     std::cerr << "no URI available" << std::endl;
     exit(EXIT_FAILURE);
   }
 
   auto uri = (*first).c_str();
-  ++first;
 
-  if (http_parser_parse_url(uri, strlen(uri), 0, &u) != 0 ||
+  // First URI is treated specially.  We use scheme, host and port of
+  // this URI and ignore those in the remaining URIs if present.
+  http_parser_url u{};
+  if (http_parser_parse_url(uri, (*first).size(), 0, &u) != 0 ||
       !util::has_uri_field(u, UF_SCHEMA) || !util::has_uri_field(u, UF_HOST)) {
     std::cerr << "invalid URI: " << uri << std::endl;
     exit(EXIT_FAILURE);
   }
 
+  ++first;
+
   config.scheme = util::get_uri_field(uri, u, UF_SCHEMA);
   config.host = util::get_uri_field(uri, u, UF_HOST);
   config.default_port = util::get_default_port(uri, u);
@@ -973,12 +973,11 @@ std::vector<std::string> parse_uris(Iterator first, Iterator last) {
   reqlines.push_back(get_reqline(uri, u));
 
   for (; first != last; ++first) {
-    http_parser_url u;
-    memset(&u, 0, sizeof(u));
+    http_parser_url u{};
 
     auto uri = (*first).c_str();
 
-    if (http_parser_parse_url(uri, strlen(uri), 0, &u) != 0) {
+    if (http_parser_parse_url(uri, (*first).size(), 0, &u) != 0) {
       std::cerr << "invalid URI: " << uri << std::endl;
       exit(EXIT_FAILURE);
     }
@@ -1036,10 +1035,10 @@ Options:
   -t, --threads=<N>
               Number of native threads.
               Default: )" << config.nthreads << R"(
-  -i, --input-file=<FILE>
+  -i, --input-file=<PATH>
               Path of a file with multiple URIs are separated by EOLs.
               This option will disable URIs getting from command-line.
-              If '-' is given as <FILE>, URIs will be read from stdin.
+              If '-' is given as <PATH>, URIs will be read from stdin.
               URIs are used  in this order for each  client.  All URIs
               are used, then  first URI is used and then  2nd URI, and
               so  on.  The  scheme, host  and port  in the  subsequent
@@ -1076,9 +1075,8 @@ Options:
               Available protocol: )";
 #endif // !HAVE_SPDYLAY
   out << NGHTTP2_CLEARTEXT_PROTO_VERSION_ID << R"(
-              Default: )"
-      << NGHTTP2_CLEARTEXT_PROTO_VERSION_ID << R"(
-  -d, --data=<FILE>
+              Default: )" << NGHTTP2_CLEARTEXT_PROTO_VERSION_ID << R"(
+  -d, --data=<PATH>
               Post FILE to  server.  The request method  is changed to
               POST.
   -r, --rate=<N>
@@ -1089,21 +1087,20 @@ Options:
               normally does, creating connections at whatever variable
               rate it wants.  The default value for this option is 0.
   -C, --num-conns=<N>
-              Specifies the total number of connections to create. The
-              total number of connections must be a positive integer.
-              On each connection, '-m' requests are made. The test
-              stops once as soon as the N connections have either
+              Specifies  the total  number of  connections to  create.
+              The  total  number of  connections  must  be a  positive
+              integer.  On each connection, -m requests are made.  The
+              test stops once as soon as the N connections have either
               completed or failed.  When  the number of connections is
               0, the program will run as it normally does, creating as
-              many connections as it needs in order to make the '-n'
-              requests specified. The default value for this option is
-              0. The '-n' option is not required if the '-C' option
-              is being used.
+              many connections  as it  needs in order  to make  the -n
+              requests specified.   The default value for  this option
+              is 0.  The -n option is not required if the -C option is
+              being used.
   -v, --verbose
               Output debug information.
   --version   Display version information and exit.
-  -h, --help  Display this help and exit.)"
-      << std::endl;
+  -h, --help  Display this help and exit.)" << std::endl;
 }
 } // namespace
 
@@ -1355,8 +1352,7 @@ int main(int argc, char **argv) {
     config.data_length = data_stat.st_size;
   }
 
-  struct sigaction act;
-  memset(&act, 0, sizeof(struct sigaction));
+  struct sigaction act {};
   act.sa_handler = SIG_IGN;
   sigaction(SIGPIPE, &act, nullptr);
 

src/http2.h

@@ -37,6 +37,8 @@
 
 #include "http-parser/http_parser.h"
 
+#include "util.h"
+
 namespace nghttp2 {
 
 struct Header {
@@ -298,6 +300,55 @@ int lookup_method_token(const std::string &name);
 
 const char *to_method_string(int method_token);
 
+template <typename InputIt>
+std::string normalize_path(InputIt first, InputIt last) {
+  // First, decode %XX for unreserved characters, then do
+  // http2::join_path
+  std::string result;
+  // We won't find %XX if length is less than 3.
+  if (last - first < 3) {
+    result.assign(first, last);
+  } else {
+    for (; first < last - 2;) {
+      if (*first == '%') {
+        if (util::isHexDigit(*(first + 1)) && util::isHexDigit(*(first + 2))) {
+          auto c = (util::hex_to_uint(*(first + 1)) << 4) +
+                   util::hex_to_uint(*(first + 2));
+          if (util::inRFC3986UnreservedChars(c)) {
+            result += c;
+            first += 3;
+            continue;
+          }
+          result += '%';
+          result += util::upcase(*(first + 1));
+          result += util::upcase(*(first + 2));
+          first += 3;
+          continue;
+        }
+      }
+      result += *first++;
+    }
+    result.append(first, last);
+  }
+  return path_join(nullptr, 0, nullptr, 0, result.c_str(), result.size(),
+                   nullptr, 0);
+}
+
+template <typename InputIt>
+std::string rewrite_clean_path(InputIt first, InputIt last) {
+  if (first == last || *first != '/') {
+    return std::string(first, last);
+  }
+  // probably, not necessary most of the case, but just in case.
+  auto fragment = std::find(first, last, '#');
+  auto query = std::find(first, fragment, '?');
+  auto path = normalize_path(first, query);
+  if (query != fragment) {
+    path.append(query, fragment);
+  }
+  return path;
+}
+
 } // namespace http2
 
 } // namespace nghttp2

src/http2_test.cc

@@ -191,8 +191,7 @@ void check_rewrite_location_uri(const std::string &want, const std::string &uri,
                                 const std::string &match_host,
                                 const std::string &req_authority,
                                 const std::string &upstream_scheme) {
-  http_parser_url u;
-  memset(&u, 0, sizeof(u));
+  http_parser_url u{};
   CU_ASSERT(0 == http_parser_parse_url(uri.c_str(), uri.size(), 0, &u));
   auto got = http2::rewrite_location_uri(uri, u, match_host, req_authority,
                                          upstream_scheme);
@@ -829,4 +828,56 @@ void test_http2_path_join(void) {
   }
 }
 
+void test_http2_normalize_path(void) {
+  std::string src;
+
+  src = "/alpha/bravo/../charlie";
+  CU_ASSERT("/alpha/charlie" ==
+            http2::normalize_path(std::begin(src), std::end(src)));
+
+  src = "/a%6c%70%68%61";
+  CU_ASSERT("/alpha" == http2::normalize_path(std::begin(src), std::end(src)));
+
+  src = "/alpha%2f%3a";
+  CU_ASSERT("/alpha%2F%3A" ==
+            http2::normalize_path(std::begin(src), std::end(src)));
+
+  src = "%2f";
+  CU_ASSERT("/%2F" == http2::normalize_path(std::begin(src), std::end(src)));
+
+  src = "%f";
+  CU_ASSERT("/%f" == http2::normalize_path(std::begin(src), std::end(src)));
+
+  src = "%";
+  CU_ASSERT("/%" == http2::normalize_path(std::begin(src), std::end(src)));
+
+  src = "";
+  CU_ASSERT("/" == http2::normalize_path(std::begin(src), std::end(src)));
+}
+
+void test_http2_rewrite_clean_path(void) {
+  std::string src;
+
+  // unreserved characters
+  src = "/alpha/%62ravo/";
+  CU_ASSERT("/alpha/bravo/" ==
+            http2::rewrite_clean_path(std::begin(src), std::end(src)));
+
+  // percent-encoding is converted to upper case.
+  src = "/delta%3a";
+  CU_ASSERT("/delta%3A" ==
+            http2::rewrite_clean_path(std::begin(src), std::end(src)));
+
+  // path component is normalized before mathcing
+  src = "/alpha/charlie/%2e././bravo/delta/..";
+  CU_ASSERT("/alpha/bravo/" ==
+            http2::rewrite_clean_path(std::begin(src), std::end(src)));
+
+  src = "alpha%3a";
+  CU_ASSERT(src == http2::rewrite_clean_path(std::begin(src), std::end(src)));
+
+  src = "";
+  CU_ASSERT(src == http2::rewrite_clean_path(std::begin(src), std::end(src)));
+}
+
 } // namespace shrpx

src/http2_test.h

@@ -45,6 +45,8 @@ void test_http2_http2_header_allowed(void);
 void test_http2_mandatory_request_headers_presence(void);
 void test_http2_parse_link_header(void);
 void test_http2_path_join(void);
+void test_http2_normalize_path(void);
+void test_http2_rewrite_clean_path(void);
 
 } // namespace shrpx
 

src/nghttp.cc

@@ -501,9 +501,8 @@ bool HttpClient::need_upgrade() const {
 
 int HttpClient::resolve_host(const std::string &host, uint16_t port) {
   int rv;
-  addrinfo hints;
   this->host = host;
-  memset(&hints, 0, sizeof(hints));
+  addrinfo hints{};
   hints.ai_family = AF_UNSPEC;
   hints.ai_socktype = SOCK_STREAM;
   hints.ai_protocol = 0;
@@ -1260,8 +1259,7 @@ bool HttpClient::add_request(const std::string &uri,
                              const nghttp2_data_provider *data_prd,
                              int64_t data_length,
                              const nghttp2_priority_spec &pri_spec, int level) {
-  http_parser_url u;
-  memset(&u, 0, sizeof(u));
+  http_parser_url u{};
   if (http_parser_parse_url(uri.c_str(), uri.size(), 0, &u) != 0) {
     return false;
   }
@@ -1379,7 +1377,8 @@ void HttpClient::output_har(FILE *outfile) {
 
     auto pushed = req->stream_id % 2 == 0;
 
-    json_object_set_new(entry, "comment", json_string(pushed ? "Pushed Object" : ""));
+    json_object_set_new(entry, "comment",
+                        json_string(pushed ? "Pushed Object" : ""));
 
     auto request = json_object();
     json_object_set_new(entry, "request", request);
@@ -1463,7 +1462,8 @@ void HttpClient::output_har(FILE *outfile) {
     json_object_set_new(timings, "receive", json_real(receive_delta));
 
     json_object_set_new(entry, "pageref", json_string(PAGE_ID));
-    json_object_set_new(entry, "connection", json_string(util::utos(req->stream_id).c_str()));
+    json_object_set_new(entry, "connection",
+                        json_string(util::utos(req->stream_id).c_str()));
   }
 
   json_dumpf(root, outfile, JSON_PRESERVE_ORDER | JSON_INDENT(2));
@@ -1483,8 +1483,7 @@ void update_html_parser(HttpClient *client, Request *req, const uint8_t *data,
     auto uri = strip_fragment(p.first.c_str());
     auto res_type = p.second;
 
-    http_parser_url u;
-    memset(&u, 0, sizeof(u));
+    http_parser_url u{};
     if (http_parser_parse_url(uri.c_str(), uri.size(), 0, &u) == 0 &&
         util::fieldeq(uri.c_str(), u, req->uri.c_str(), req->u, UF_SCHEMA) &&
         util::fieldeq(uri.c_str(), u, req->uri.c_str(), req->u, UF_HOST) &&
@@ -1648,8 +1647,7 @@ int on_begin_headers_callback(nghttp2_session *session,
   }
   case NGHTTP2_PUSH_PROMISE: {
     auto stream_id = frame->push_promise.promised_stream_id;
-    http_parser_url u;
-    memset(&u, 0, sizeof(u));
+    http_parser_url u{};
     // TODO Set pri and level
     nghttp2_priority_spec pri_spec;
 
@@ -1818,8 +1816,7 @@ int on_frame_recv_callback2(nghttp2_session *session,
     uri += "://";
     uri += authority->value;
     uri += path->value;
-    http_parser_url u;
-    memset(&u, 0, sizeof(u));
+    http_parser_url u{};
     if (http_parser_parse_url(uri.c_str(), uri.size(), 0, &u) != 0) {
       nghttp2_submit_rst_stream(session, NGHTTP2_FLAG_NONE,
                                 frame->push_promise.promised_stream_id,
@@ -2297,8 +2294,7 @@ int run(char **uris, int n) {
   std::vector<std::tuple<std::string, nghttp2_data_provider *, int64_t>>
       requests;
   for (int i = 0; i < n; ++i) {
-    http_parser_url u;
-    memset(&u, 0, sizeof(u));
+    http_parser_url u{};
     auto uri = strip_fragment(uris[i]);
     if (http_parser_parse_url(uri.c_str(), uri.size(), 0, &u) != 0) {
       std::cerr << "[ERROR] Could not parse URI " << uri << std::endl;
@@ -2398,7 +2394,7 @@ Options:
               must be in PEM format.
   --key=<KEY> Use the  client private key  file.  The file must  be in
               PEM format.
-  -d, --data=<FILE>
+  -d, --data=<PATH>
               Post FILE to server. If '-'  is given, data will be read
               from stdin.
   -m, --multiply=<N>
@@ -2422,8 +2418,8 @@ Options:
   -b, --padding=<N>
               Add at  most <N>  bytes to a  frame payload  as padding.
               Specify 0 to disable padding.
-  -r, --har=<FILE>
-              Output HTTP  transactions <FILE> in HAR  format.  If '-'
+  -r, --har=<PATH>
+              Output HTTP  transactions <PATH> in HAR  format.  If '-'
               is given, data is written to stdout.
   --color     Force colored log output.
   --continuation
@@ -2699,8 +2695,7 @@ int main(int argc, char **argv) {
   nghttp2_option_set_peer_max_concurrent_streams(
       config.http2_option, config.peer_max_concurrent_streams);
 
-  struct sigaction act;
-  memset(&act, 0, sizeof(struct sigaction));
+  struct sigaction act {};
   act.sa_handler = SIG_IGN;
   sigaction(SIGPIPE, &act, nullptr);
   reset_timer();

src/nghttpd.cc

@@ -371,8 +371,7 @@ int main(int argc, char **argv) {
 
   set_color_output(color || isatty(fileno(stdout)));
 
-  struct sigaction act;
-  memset(&act, 0, sizeof(struct sigaction));
+  struct sigaction act {};
   act.sa_handler = SIG_IGN;
   sigaction(SIGPIPE, &act, nullptr);
 

src/shrpx-unittest.cc

@@ -96,6 +96,10 @@ int main(int argc, char *argv[]) {
       !CU_add_test(pSuite, "http2_parse_link_header",
                    shrpx::test_http2_parse_link_header) ||
       !CU_add_test(pSuite, "http2_path_join", shrpx::test_http2_path_join) ||
+      !CU_add_test(pSuite, "http2_normalize_path",
+                   shrpx::test_http2_normalize_path) ||
+      !CU_add_test(pSuite, "http2_rewrite_clean_path",
+                   shrpx::test_http2_rewrite_clean_path) ||
       !CU_add_test(pSuite, "downstream_index_request_headers",
                    shrpx::test_downstream_index_request_headers) ||
       !CU_add_test(pSuite, "downstream_index_response_headers",
@@ -118,6 +122,10 @@ int main(int argc, char *argv[]) {
                    shrpx::test_shrpx_config_parse_log_format) ||
       !CU_add_test(pSuite, "config_read_tls_ticket_key_file",
                    shrpx::test_shrpx_config_read_tls_ticket_key_file) ||
+      !CU_add_test(pSuite, "config_read_tls_ticket_key_file_aes_256",
+                   shrpx::test_shrpx_config_read_tls_ticket_key_file_aes_256) ||
+      !CU_add_test(pSuite, "config_match_downstream_addr_group",
+                   shrpx::test_shrpx_config_match_downstream_addr_group) ||
       !CU_add_test(pSuite, "util_streq", shrpx::test_util_streq) ||
       !CU_add_test(pSuite, "util_strieq", shrpx::test_util_strieq) ||
       !CU_add_test(pSuite, "util_inp_strlower",

src/shrpx.cc

@@ -119,12 +119,11 @@ const int GRACEFUL_SHUTDOWN_SIGNAL = SIGQUIT;
 namespace {
 int resolve_hostname(sockaddr_union *addr, size_t *addrlen,
                      const char *hostname, uint16_t port, int family) {
-  addrinfo hints;
   int rv;
 
   auto service = util::utos(port);
-  memset(&hints, 0, sizeof(addrinfo));
 
+  addrinfo hints{};
   hints.ai_family = family;
   hints.ai_socktype = SOCK_STREAM;
 #ifdef AI_ADDRCONFIG
@@ -279,12 +278,11 @@ std::unique_ptr<AcceptHandler> create_acceptor(ConnectionHandler *handler,
     }
   }
 
-  addrinfo hints;
   int fd = -1;
   int rv;
 
   auto service = util::utos(get_config()->port);
-  memset(&hints, 0, sizeof(addrinfo));
+  addrinfo hints{};
   hints.ai_family = family;
   hints.ai_socktype = SOCK_STREAM;
   hints.ai_flags = AI_PASSIVE;
@@ -606,44 +604,88 @@ void graceful_shutdown_signal_cb(struct ev_loop *loop, ev_signal *w,
 }
 } // namespace
 
+namespace {
+int generate_ticket_key(TicketKey &ticket_key) {
+  ticket_key.cipher = get_config()->tls_ticket_cipher;
+  ticket_key.hmac = EVP_sha256();
+  ticket_key.hmac_keylen = EVP_MD_size(ticket_key.hmac);
+
+  assert(static_cast<size_t>(EVP_CIPHER_key_length(ticket_key.cipher)) <=
+         sizeof(ticket_key.data.enc_key));
+  assert(ticket_key.hmac_keylen <= sizeof(ticket_key.data.hmac_key));
+
+  if (LOG_ENABLED(INFO)) {
+    LOG(INFO) << "enc_keylen=" << EVP_CIPHER_key_length(ticket_key.cipher)
+              << ", hmac_keylen=" << ticket_key.hmac_keylen;
+  }
+
+  if (RAND_bytes(reinterpret_cast<unsigned char *>(&ticket_key.data),
+                 sizeof(ticket_key.data)) == 0) {
+    return -1;
+  }
+
+  return 0;
+}
+} // namespace
+
 namespace {
 void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) {
   auto conn_handler = static_cast<ConnectionHandler *>(w->data);
   const auto &old_ticket_keys = conn_handler->get_ticket_keys();
 
   auto ticket_keys = std::make_shared<TicketKeys>();
-  LOG(NOTICE) << "Renew ticket keys: main";
+  LOG(NOTICE) << "Renew new ticket keys";
 
-  // We store at most 2 ticket keys
+  // If old_ticket_keys is not empty, it should contain at least 2
+  // keys: one for encryption, and last one for the next encryption
+  // key but decryption only.  The keys in between are old keys and
+  // decryption only.  The next key is provided to ensure to mitigate
+  // possible problem when one worker encrypt new key, but one worker,
+  // which did not take the that key yet, and cannot decrypt it.
+  //
+  // We keep keys for get_config()->tls_session_timeout seconds.  The
+  // default is 12 hours.  Thus the maximum ticket vector size is 12.
   if (old_ticket_keys) {
     auto &old_keys = old_ticket_keys->keys;
     auto &new_keys = ticket_keys->keys;
 
     assert(!old_keys.empty());
 
-    new_keys.resize(2);
-    new_keys[1] = old_keys[0];
+    auto max_tickets =
+        static_cast<size_t>(std::chrono::duration_cast<std::chrono::hours>(
+                                get_config()->tls_session_timeout).count());
+
+    new_keys.resize(std::min(max_tickets, old_keys.size() + 1));
+    std::copy_n(std::begin(old_keys), new_keys.size() - 1,
+                std::begin(new_keys) + 1);
   } else {
     ticket_keys->keys.resize(1);
   }
 
-  if (RAND_bytes(reinterpret_cast<unsigned char *>(&ticket_keys->keys[0]),
-                 sizeof(ticket_keys->keys[0])) == 0) {
+  auto &new_key = ticket_keys->keys[0];
+
+  if (generate_ticket_key(new_key) != 0) {
     if (LOG_ENABLED(INFO)) {
-      LOG(INFO) << "failed to renew ticket key";
+      LOG(INFO) << "failed to generate ticket key";
     }
+    conn_handler->set_ticket_keys(nullptr);
+    conn_handler->set_ticket_keys_to_worker(nullptr);
     return;
   }
 
   if (LOG_ENABLED(INFO)) {
     LOG(INFO) << "ticket keys generation done";
-    for (auto &key : ticket_keys->keys) {
-      LOG(INFO) << "name: " << util::format_hex(key.name, sizeof(key.name));
+    assert(ticket_keys->keys.size() >= 1);
+    LOG(INFO) << 0 << " enc+dec: "
+              << util::format_hex(ticket_keys->keys[0].data.name);
+    for (size_t i = 1; i < ticket_keys->keys.size(); ++i) {
+      auto &key = ticket_keys->keys[i];
+      LOG(INFO) << i << " dec: " << util::format_hex(key.data.name);
     }
   }
 
   conn_handler->set_ticket_keys(ticket_keys);
-  conn_handler->worker_renew_ticket_keys(ticket_keys);
+  conn_handler->set_ticket_keys_to_worker(ticket_keys);
 }
 } // namespace
 
@@ -709,8 +751,17 @@ int event_loop() {
   if (!get_config()->upstream_no_tls) {
     bool auto_tls_ticket_key = true;
     if (!get_config()->tls_ticket_key_files.empty()) {
-      auto ticket_keys =
-          read_tls_ticket_key_file(get_config()->tls_ticket_key_files);
+      if (!get_config()->tls_ticket_cipher_given) {
+        LOG(WARN) << "It is strongly recommended to specify "
+                     "--tls-ticket-cipher=aes-128-cbc (or "
+                     "tls-ticket-cipher=aes-128-cbc in configuration file) "
+                     "when --tls-ticket-key-file is used for the smooth "
+                     "transition when the default value of --tls-ticket-cipher "
+                     "becomes aes-256-cbc";
+      }
+      auto ticket_keys = read_tls_ticket_key_file(
+          get_config()->tls_ticket_key_files, get_config()->tls_ticket_cipher,
+          EVP_sha256());
       if (!ticket_keys) {
         LOG(WARN) << "Use internal session ticket key generator";
       } else {
@@ -719,8 +770,8 @@ int event_loop() {
       }
     }
     if (auto_tls_ticket_key) {
-      // Renew ticket key every 12hrs
-      ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 12_h);
+      // Generate new ticket key every 1hr.
+      ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 1_h);
       renew_ticket_key_timer.data = conn_handler.get();
       ev_timer_again(loop, &renew_ticket_key_timer);
 
@@ -827,7 +878,7 @@ int16_t DEFAULT_DOWNSTREAM_PORT = 80;
 
 namespace {
 void fill_default_config() {
-  memset(mod_config(), 0, sizeof(*mod_config()));
+  *mod_config() = {};
 
   mod_config()->verbose = false;
   mod_config()->daemon = false;
@@ -948,7 +999,7 @@ void fill_default_config() {
 
   mod_config()->tls_proto_mask = 0;
   mod_config()->no_location_rewrite = false;
-  mod_config()->no_host_rewrite = false;
+  mod_config()->no_host_rewrite = true;
   mod_config()->argc = 0;
   mod_config()->argv = nullptr;
   mod_config()->downstream_connections_per_host = 8;
@@ -966,6 +1017,10 @@ void fill_default_config() {
   mod_config()->no_ocsp = false;
   mod_config()->header_field_buffer = 64_k;
   mod_config()->max_header_fields = 100;
+  mod_config()->downstream_addr_group_catch_all = 0;
+  mod_config()->tls_ticket_cipher = EVP_aes_128_cbc();
+  mod_config()->tls_ticket_cipher_given = false;
+  mod_config()->tls_session_timeout = std::chrono::hours(12);
 }
 } // namespace
 
@@ -997,14 +1052,67 @@ Options:
   The options are categorized into several groups.
 
 Connections:
-  -b, --backend=<HOST,PORT>
+  -b, --backend=(<HOST>,<PORT>|unix:<PATH>)[;<PATTERN>[:...]]
               Set  backend  host  and   port.   The  multiple  backend
               addresses are  accepted by repeating this  option.  UNIX
               domain socket  can be  specified by prefixing  path name
-              with "unix:" (e.g., unix:/var/run/backend.sock)
+              with "unix:" (e.g., unix:/var/run/backend.sock).
+
+              Optionally, if <PATTERN>s are given, the backend address
+              is only used  if request matches the pattern.   If -s or
+              -p  is  used,  <PATTERN>s   are  ignored.   The  pattern
+              matching  is closely  designed to  ServeMux in  net/http
+              package of Go  programming language.  <PATTERN> consists
+              of path, host + path or  just host.  The path must start
+              with "/".  If  it ends with "/", it  matches all request
+              path in  its subtree.  To  deal with the request  to the
+              directory without  trailing slash,  the path  which ends
+              with "/" also matches the  request path which only lacks
+              trailing '/'  (e.g., path  "/foo/" matches  request path
+              "/foo").  If it does not end with "/", it performs exact
+              match against  the request path.   If host is  given, it
+              performs exact match against  the request host.  If host
+              alone  is given,  "/"  is  appended to  it,  so that  it
+              matches  all   request  paths  under  the   host  (e.g.,
+              specifying "nghttp2.org" equals to "nghttp2.org/").
+
+              Patterns with  host take  precedence over  patterns with
+              just path.   Then, longer patterns take  precedence over
+              shorter  ones,  breaking  a  tie by  the  order  of  the
+              appearance in the configuration.
+
+              If <PATTERN> is  omitted, "/" is used  as pattern, which
+              matches  all  request  paths (catch-all  pattern).   The
+              catch-all backend must be given.
+
+              When doing  a match, nghttpx made  some normalization to
+              pattern, request host and path.  For host part, they are
+              converted to lower case.  For path part, percent-encoded
+              unreserved characters  defined in RFC 3986  are decoded,
+              and any  dot-segments (".."  and ".")   are resolved and
+              removed.
+
+              For   example,   -b'127.0.0.1,8080;nghttp2.org/httpbin/'
+              matches the  request host "nghttp2.org" and  the request
+              path "/httpbin/get", but does not match the request host
+              "nghttp2.org" and the request path "/index.html".
+
+              The  multiple <PATTERN>s  can  be specified,  delimiting
+              them            by           ":".             Specifying
+              -b'127.0.0.1,8080;nghttp2.org:www.nghttp2.org'  has  the
+              same  effect  to specify  -b'127.0.0.1,8080;nghttp2.org'
+              and -b'127.0.0.1,8080;www.nghttp2.org'.
+
+              The backend addresses sharing same <PATTERN> are grouped
+              together forming  load balancing  group.
+
+              Since ";" and ":" are  used as delimiter, <PATTERN> must
+              not  contain these  characters.  Since  ";" has  special
+              meaning in shell, the option value must be quoted.
+
               Default: )" << DEFAULT_DOWNSTREAM_HOST << ","
       << DEFAULT_DOWNSTREAM_PORT << R"(
-  -f, --frontend=<HOST,PORT>
+  -f, --frontend=(<HOST>,<PORT>|unix:<PATH>)
               Set  frontend  host and  port.   If  <HOST> is  '*',  it
               assumes  all addresses  including  both  IPv4 and  IPv6.
               UNIX domain  socket can  be specified by  prefixing path
@@ -1079,9 +1187,14 @@ Performance:
               accepts.  Setting 0 means unlimited.
               Default: )" << get_config()->worker_frontend_connections << R"(
   --backend-http2-connections-per-worker=<N>
-              Set  maximum number  of HTTP/2  connections per  worker.
-              The  default  value is  0,  which  means the  number  of
-              backend addresses specified by -b option.
+              Set   maximum   number   of  backend   HTTP/2   physical
+              connections  per  worker.   If  pattern is  used  in  -b
+              option, this limit is applied  to each pattern group (in
+              other  words, each  pattern group  can have  maximum <N>
+              HTTP/2  connections).  The  default  value  is 0,  which
+              means  that  the value  is  adjusted  to the  number  of
+              backend addresses.  If pattern  is used, this adjustment
+              is done for each pattern group.
   --backend-http1-connections-per-host=<N>
               Set   maximum  number   of  backend   concurrent  HTTP/1
               connections per origin host.   This option is meaningful
@@ -1219,8 +1332,11 @@ SSL/TLS:
               are treated as a part of protocol string.
               Default: )" << DEFAULT_TLS_PROTO_LIST << R"(
   --tls-ticket-key-file=<PATH>
-              Path  to file  that  contains 48  bytes  random data  to
-              construct TLS  session ticket parameters.   This options
+              Path to file that contains  random data to construct TLS
+              session ticket  parameters.  If aes-128-cbc is  given in
+              --tls-ticket-cipher,  the file  must contain  exactly 48
+              bytes.  If aes-256-cbc  is given in --tls-ticket-cipher,
+              the file  must contain  exactly 80 bytes.   This options
               can  be  used  repeatedly  to  specify  multiple  ticket
               parameters.  If several files  are given, only the first
               key is used to encrypt  TLS session tickets.  Other keys
@@ -1232,9 +1348,14 @@ SSL/TLS:
               opening or reading given file fails, all loaded keys are
               discarded and it is treated as if none of this option is
               given.  If this option is not given or an error occurred
-              while  opening  or  reading  a file,  key  is  generated
-              automatically and  renewed every 12hrs.  At  most 2 keys
-              are stored in memory.
+              while opening or reading a  file, key is generated every
+              1 hour internally and they are valid for 12 hours.  This
+              is  recommended if  ticket key  sharing between  nghttpx
+              instances is not required.
+  --tls-ticket-cipher=<TICKET_CIPHER>
+              Specify cipher  to encrypt TLS session  ticket.  Specify
+              either   aes-128-cbc   or  aes-256-cbc.    By   default,
+              aes-128-cbc is used.
   --fetch-ocsp-response-file=<PATH>
               Path to  fetch-ocsp-response script file.  It  should be
               absolute path.
@@ -1351,6 +1472,9 @@ Logging:
               * $ssl_session_reused:  "r"   if  SSL/TLS   session  was
                 reused.  Otherwise, "."
 
+              The  variable  can  be  enclosed  by  "{"  and  "}"  for
+              disambiguation (e.g., ${remote_addr}).
+
               Default: )" << DEFAULT_ACCESSLOG_FORMAT << R"(
   --errorlog-file=<PATH>
               Set path to write error  log.  To reopen file, send USR1
@@ -1379,8 +1503,8 @@ HTTP:
               --client  and  default   mode.   For  --http2-proxy  and
               --client-proxy mode,  location header field will  not be
               altered regardless of this option.
-  --no-host-rewrite
-              Don't  rewrite  host  and :authority  header  fields  on
+  --host-rewrite
+              Rewrite   host   and   :authority   header   fields   on
               --http2-bridge,   --client   and  default   mode.    For
               --http2-proxy  and  --client-proxy mode,  these  headers
               will not be altered regardless of this option.
@@ -1446,6 +1570,11 @@ Misc:
   --conf=<PATH>
               Load configuration from <PATH>.
               Default: )" << get_config()->conf_path.get() << R"(
+  --include=<PATH>
+              Load additional configurations from <PATH>.  File <PATH>
+              is  read  when  configuration  parser  encountered  this
+              option.  This option can be used multiple times, or even
+              recursively.
   -v, --version
               Print version and exit.
   -h, --help  Print this help and exit.
@@ -1477,6 +1606,10 @@ int main(int argc, char **argv) {
   create_config();
   fill_default_config();
 
+  // First open log files with default configuration, so that we can
+  // log errors/warnings while reading configuration files.
+  reopen_log_files();
+
   // We have to copy argv, since getopt_long may change its content.
   mod_config()->argc = argc;
   mod_config()->argv = new char *[argc];
@@ -1592,6 +1725,9 @@ int main(int argc, char **argv) {
         {SHRPX_OPT_HEADER_FIELD_BUFFER, required_argument, &flag, 80},
         {SHRPX_OPT_MAX_HEADER_FIELDS, required_argument, &flag, 81},
         {SHRPX_OPT_ADD_REQUEST_HEADER, required_argument, &flag, 82},
+        {SHRPX_OPT_INCLUDE, required_argument, &flag, 83},
+        {SHRPX_OPT_TLS_TICKET_CIPHER, required_argument, &flag, 84},
+        {SHRPX_OPT_HOST_REWRITE, no_argument, &flag, 85},
         {nullptr, 0, nullptr, 0}};
 
     int option_index = 0;
@@ -1954,6 +2090,18 @@ int main(int argc, char **argv) {
         // --add-request-header
         cmdcfgs.emplace_back(SHRPX_OPT_ADD_REQUEST_HEADER, optarg);
         break;
+      case 83:
+        // --include
+        cmdcfgs.emplace_back(SHRPX_OPT_INCLUDE, optarg);
+        break;
+      case 84:
+        // --tls-ticket-cipher
+        cmdcfgs.emplace_back(SHRPX_OPT_TLS_TICKET_CIPHER, optarg);
+        break;
+      case 85:
+        // --host-rewrite
+        cmdcfgs.emplace_back(SHRPX_OPT_HOST_REWRITE, "yes");
+        break;
       default:
         break;
       }
@@ -1964,11 +2112,13 @@ int main(int argc, char **argv) {
   }
 
   if (conf_exists(get_config()->conf_path.get())) {
-    if (load_config(get_config()->conf_path.get()) == -1) {
+    std::set<std::string> include_set;
+    if (load_config(get_config()->conf_path.get(), include_set) == -1) {
       LOG(FATAL) << "Failed to load configuration from "
                  << get_config()->conf_path.get();
       exit(EXIT_FAILURE);
     }
+    assert(include_set.empty());
   }
 
   if (argc - optind >= 2) {
@@ -1976,17 +2126,23 @@ int main(int argc, char **argv) {
     cmdcfgs.emplace_back(SHRPX_OPT_CERTIFICATE_FILE, argv[optind++]);
   }
 
-  // First open default log files to deal with errors occurred while
-  // parsing option values.
+  // Reopen log files using configurations in file
   reopen_log_files();
 
+  {
+    std::set<std::string> include_set;
+
     for (size_t i = 0, len = cmdcfgs.size(); i < len; ++i) {
-    if (parse_config(cmdcfgs[i].first, cmdcfgs[i].second) == -1) {
+      if (parse_config(cmdcfgs[i].first, cmdcfgs[i].second, include_set) ==
+          -1) {
         LOG(FATAL) << "Failed to parse command-line argument.";
         exit(EXIT_FAILURE);
       }
     }
 
+    assert(include_set.empty());
+  }
+
   if (get_config()->accesslog_syslog || get_config()->errorlog_syslog) {
     openlog("nghttpx", LOG_NDELAY | LOG_NOWAIT | LOG_PID,
             get_config()->syslog_facility);
@@ -2118,19 +2274,59 @@ int main(int argc, char **argv) {
     }
   }
 
-  if (get_config()->downstream_addrs.empty()) {
+  if (get_config()->downstream_addr_groups.empty()) {
     DownstreamAddr addr;
     addr.host = strcopy(DEFAULT_DOWNSTREAM_HOST);
     addr.port = DEFAULT_DOWNSTREAM_PORT;
 
-    mod_config()->downstream_addrs.push_back(std::move(addr));
+    DownstreamAddrGroup g("/");
+    g.addrs.push_back(std::move(addr));
+    mod_config()->downstream_addr_groups.push_back(std::move(g));
+  } else if (get_config()->http2_proxy || get_config()->client_proxy) {
+    // We don't support host mapping in these cases.  Move all
+    // non-catch-all patterns to catch-all pattern.
+    DownstreamAddrGroup catch_all("/");
+    for (auto &g : mod_config()->downstream_addr_groups) {
+      std::move(std::begin(g.addrs), std::end(g.addrs),
+                std::back_inserter(catch_all.addrs));
+    }
+    std::vector<DownstreamAddrGroup>().swap(
+        mod_config()->downstream_addr_groups);
+    mod_config()->downstream_addr_groups.push_back(std::move(catch_all));
   }
 
   if (LOG_ENABLED(INFO)) {
     LOG(INFO) << "Resolving backend address";
   }
 
-  for (auto &addr : mod_config()->downstream_addrs) {
+  ssize_t catch_all_group = -1;
+  for (size_t i = 0; i < mod_config()->downstream_addr_groups.size(); ++i) {
+    auto &g = mod_config()->downstream_addr_groups[i];
+    if (g.pattern == "/") {
+      catch_all_group = i;
+    }
+    if (LOG_ENABLED(INFO)) {
+      LOG(INFO) << "Host-path pattern: group " << i << ": '" << g.pattern
+                << "'";
+      for (auto &addr : g.addrs) {
+        LOG(INFO) << "group " << i << " -> " << addr.host.get()
+                  << (addr.host_unix ? "" : ":" + util::utos(addr.port));
+      }
+    }
+  }
+
+  if (catch_all_group == -1) {
+    LOG(FATAL) << "-b: No catch-all backend address is configured";
+    exit(EXIT_FAILURE);
+  }
+  mod_config()->downstream_addr_group_catch_all = catch_all_group;
+
+  if (LOG_ENABLED(INFO)) {
+    LOG(INFO) << "Catch-all pattern is group " << catch_all_group;
+  }
+
+  for (auto &g : mod_config()->downstream_addr_groups) {
+    for (auto &addr : g.addrs) {
 
       if (addr.host_unix) {
         // for AF_UNIX socket, we use "localhost" as host for backend
@@ -2163,12 +2359,13 @@ int main(int argc, char **argv) {
 
       if (resolve_hostname(
               &addr.addr, &addr.addrlen, addr.host.get(), addr.port,
-            get_config()->backend_ipv4
-                ? AF_INET
-                : (get_config()->backend_ipv6 ? AF_INET6 : AF_UNSPEC)) == -1) {
+              get_config()->backend_ipv4 ? AF_INET : (get_config()->backend_ipv6
+                                                          ? AF_INET6
+                                                          : AF_UNSPEC)) == -1) {
         exit(EXIT_FAILURE);
       }
     }
+  }
 
   if (get_config()->downstream_http_proxy_host) {
     if (LOG_ENABLED(INFO)) {
@@ -2183,11 +2380,6 @@ int main(int argc, char **argv) {
     }
   }
 
-  if (get_config()->http2_downstream_connections_per_worker == 0) {
-    mod_config()->http2_downstream_connections_per_worker =
-        get_config()->downstream_addrs.size();
-  }
-
   if (get_config()->rlimit_nofile) {
     struct rlimit lim = {static_cast<rlim_t>(get_config()->rlimit_nofile),
                          static_cast<rlim_t>(get_config()->rlimit_nofile)};
@@ -2206,8 +2398,7 @@ int main(int argc, char **argv) {
     reset_timer();
   }
 
-  struct sigaction act;
-  memset(&act, 0, sizeof(struct sigaction));
+  struct sigaction act {};
   act.sa_handler = SIG_IGN;
   sigaction(SIGPIPE, &act, nullptr);
 

src/shrpx_client_handler.cc

@@ -39,6 +39,7 @@
 #include "shrpx_worker.h"
 #include "shrpx_downstream_connection_pool.h"
 #include "shrpx_downstream.h"
+#include "shrpx_http2_session.h"
 #ifdef HAVE_SPDYLAY
 #include "shrpx_spdy_upstream.h"
 #endif // HAVE_SPDYLAY
@@ -360,8 +361,12 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl,
             get_config()->upstream_read_timeout, get_config()->write_rate,
             get_config()->write_burst, get_config()->read_rate,
             get_config()->read_burst, writecb, readcb, timeoutcb, this),
+      pinned_http2sessions_(
+          get_config()->downstream_proto == PROTO_HTTP2
+              ? make_unique<std::vector<ssize_t>>(
+                    get_config()->downstream_addr_groups.size(), -1)
+              : nullptr),
       ipaddr_(ipaddr), port_(port), worker_(worker),
-      http2session_(worker_->next_http2_session()),
       left_connhd_len_(NGHTTP2_CLIENT_MAGIC_LEN),
       should_close_after_write_(false) {
 
@@ -588,7 +593,8 @@ void ClientHandler::pool_downstream_connection(
     return;
   }
   if (LOG_ENABLED(INFO)) {
-    CLOG(INFO, this) << "Pooling downstream connection DCONN:" << dconn.get();
+    CLOG(INFO, this) << "Pooling downstream connection DCONN:" << dconn.get()
+                     << " in group " << dconn->get_group();
   }
   dconn->set_client_handler(nullptr);
   auto dconn_pool = worker_->get_dconn_pool();
@@ -605,9 +611,43 @@ void ClientHandler::remove_downstream_connection(DownstreamConnection *dconn) {
 }
 
 std::unique_ptr<DownstreamConnection>
-ClientHandler::get_downstream_connection() {
+ClientHandler::get_downstream_connection(Downstream *downstream) {
+  size_t group;
+  auto &groups = get_config()->downstream_addr_groups;
+  auto catch_all = get_config()->downstream_addr_group_catch_all;
+
+  // Fast path.  If we have one group, it must be catch-all group.
+  // HTTP/2 and client proxy modes fall in this case.
+  if (groups.size() == 1) {
+    group = 0;
+  } else if (downstream->get_request_method() == HTTP_CONNECT) {
+    //  We don't know how to treat CONNECT request in host-path
+    //  mapping.  It most likely appears in proxy scenario.  Since we
+    //  have dealt with proxy case already, just use catch-all group.
+    group = catch_all;
+  } else {
+    if (!downstream->get_request_http2_authority().empty()) {
+      group = match_downstream_addr_group(
+          downstream->get_request_http2_authority(),
+          downstream->get_request_path(), groups, catch_all);
+    } else {
+      auto h = downstream->get_request_header(http2::HD_HOST);
+      if (h) {
+        group = match_downstream_addr_group(
+            h->value, downstream->get_request_path(), groups, catch_all);
+      } else {
+        group = match_downstream_addr_group("", downstream->get_request_path(),
+                                            groups, catch_all);
+      }
+    }
+  }
+
+  if (LOG_ENABLED(INFO)) {
+    CLOG(INFO, this) << "Downstream address group: " << group;
+  }
+
   auto dconn_pool = worker_->get_dconn_pool();
-  auto dconn = dconn_pool->pop_downstream_connection();
+  auto dconn = dconn_pool->pop_downstream_connection(group);
 
   if (!dconn) {
     if (LOG_ENABLED(INFO)) {
@@ -617,10 +657,20 @@ ClientHandler::get_downstream_connection() {
 
     auto dconn_pool = worker_->get_dconn_pool();
 
-    if (http2session_) {
-      dconn = make_unique<Http2DownstreamConnection>(dconn_pool, http2session_);
+    if (get_config()->downstream_proto == PROTO_HTTP2) {
+      Http2Session *http2session;
+      auto &pinned = (*pinned_http2sessions_)[group];
+      if (pinned == -1) {
+        http2session = worker_->next_http2_session(group);
+        pinned = http2session->get_index();
       } else {
-      dconn = make_unique<HttpDownstreamConnection>(dconn_pool, conn_.loop);
+        auto dgrp = worker_->get_dgrp(group);
+        http2session = dgrp->http2sessions[pinned].get();
+      }
+      dconn = make_unique<Http2DownstreamConnection>(dconn_pool, http2session);
+    } else {
+      dconn =
+          make_unique<HttpDownstreamConnection>(dconn_pool, group, conn_.loop);
     }
     dconn->set_client_handler(this);
     return dconn;

src/shrpx_client_handler.h

@@ -44,7 +44,6 @@ namespace shrpx {
 
 class Upstream;
 class DownstreamConnection;
-class Http2Session;
 class HttpsUpstream;
 class ConnectBlocker;
 class DownstreamConnectionPool;
@@ -92,7 +91,8 @@ public:
 
   void pool_downstream_connection(std::unique_ptr<DownstreamConnection> dconn);
   void remove_downstream_connection(DownstreamConnection *dconn);
-  std::unique_ptr<DownstreamConnection> get_downstream_connection();
+  std::unique_ptr<DownstreamConnection>
+  get_downstream_connection(Downstream *downstream);
   MemchunkPool *get_mcpool();
   SSL *get_ssl() const;
   ConnectBlocker *get_connect_blocker() const;
@@ -134,6 +134,7 @@ private:
   Connection conn_;
   ev_timer reneg_shutdown_timer_;
   std::unique_ptr<Upstream> upstream_;
+  std::unique_ptr<std::vector<ssize_t>> pinned_http2sessions_;
   std::string ipaddr_;
   std::string port_;
   // The ALPN identifier negotiated for this connection.
@@ -141,7 +142,6 @@ private:
   std::function<int(ClientHandler &)> read_, write_;
   std::function<int(ClientHandler &)> on_read_, on_write_;
   Worker *worker_;
-  Http2Session *http2session_;
   // The number of bytes of HTTP/2 client connection header to read
   size_t left_connhd_len_;
   bool should_close_after_write_;

src/shrpx_config.h

@@ -42,6 +42,7 @@
 #include <cstdio>
 #include <vector>
 #include <memory>
+#include <set>
 
 #include <openssl/ssl.h>
 
@@ -49,6 +50,10 @@
 
 #include <nghttp2/nghttp2.h>
 
+#include "template.h"
+
+using namespace nghttp2;
+
 namespace shrpx {
 
 struct LogFragment;
@@ -165,6 +170,9 @@ constexpr char SHRPX_OPT_OCSP_UPDATE_INTERVAL[] = "ocsp-update-interval";
 constexpr char SHRPX_OPT_NO_OCSP[] = "no-ocsp";
 constexpr char SHRPX_OPT_HEADER_FIELD_BUFFER[] = "header-field-buffer";
 constexpr char SHRPX_OPT_MAX_HEADER_FIELDS[] = "max-header-fields";
+constexpr char SHRPX_OPT_INCLUDE[] = "include";
+constexpr char SHRPX_OPT_TLS_TICKET_CIPHER[] = "tls-ticket-cipher";
+constexpr char SHRPX_OPT_HOST_REWRITE[] = "host-rewrite";
 
 union sockaddr_union {
   sockaddr_storage storage;
@@ -177,23 +185,20 @@ union sockaddr_union {
 enum shrpx_proto { PROTO_HTTP2, PROTO_HTTP };
 
 struct AltSvc {
-  AltSvc()
-      : protocol_id(nullptr), host(nullptr), origin(nullptr),
-        protocol_id_len(0), host_len(0), origin_len(0), port(0) {}
+  AltSvc() : port(0) {}
 
-  char *protocol_id;
-  char *host;
-  char *origin;
-
-  size_t protocol_id_len;
-  size_t host_len;
-  size_t origin_len;
+  std::string protocol_id, host, origin, service;
 
   uint16_t port;
 };
 
 struct DownstreamAddr {
   DownstreamAddr() : addr{{0}}, addrlen(0), port(0), host_unix(false) {}
+  DownstreamAddr(const DownstreamAddr &other);
+  DownstreamAddr(DownstreamAddr &&) = default;
+  DownstreamAddr &operator=(const DownstreamAddr &other);
+  DownstreamAddr &operator=(DownstreamAddr &&other) = default;
+
   sockaddr_union addr;
   // backend address.  If |host_unix| is true, this is UNIX domain
   // socket path.
@@ -206,10 +211,24 @@ struct DownstreamAddr {
   bool host_unix;
 };
 
+struct DownstreamAddrGroup {
+  DownstreamAddrGroup(std::string pattern) : pattern(std::move(pattern)) {}
+  std::string pattern;
+  std::vector<DownstreamAddr> addrs;
+};
+
 struct TicketKey {
+  const EVP_CIPHER *cipher;
+  const EVP_MD *hmac;
+  size_t hmac_keylen;
+  struct {
+    // name of this ticket configuration
     uint8_t name[16];
-  uint8_t aes_key[16];
-  uint8_t hmac_key[16];
+    // encryption key for |cipher|
+    uint8_t enc_key[32];
+    // hmac key for |hmac|
+    uint8_t hmac_key[32];
+  } data;
 };
 
 struct TicketKeys {
@@ -225,10 +244,16 @@ struct Config {
   std::vector<std::pair<std::string, std::string>> add_response_headers;
   std::vector<unsigned char> alpn_prefs;
   std::vector<LogFragment> accesslog_format;
-  std::vector<DownstreamAddr> downstream_addrs;
+  std::vector<DownstreamAddrGroup> downstream_addr_groups;
   std::vector<std::string> tls_ticket_key_files;
+  // list of supported NPN/ALPN protocol strings in the order of
+  // preference.
+  std::vector<std::string> npn_list;
+  // list of supported SSL/TLS protocol strings.
+  std::vector<std::string> tls_proto_list;
   // binary form of http proxy host and port
   sockaddr_union downstream_http_proxy_addr;
+  std::chrono::seconds tls_session_timeout;
   ev_tstamp http2_upstream_read_timeout;
   ev_tstamp upstream_read_timeout;
   ev_tstamp upstream_write_timeout;
@@ -246,7 +271,6 @@ struct Config {
   std::unique_ptr<char[]> private_key_passwd;
   std::unique_ptr<char[]> cert_file;
   std::unique_ptr<char[]> dh_param_file;
-  const char *server_name;
   std::unique_ptr<char[]> backend_tls_sni_name;
   std::unique_ptr<char[]> pid_file;
   std::unique_ptr<char[]> conf_path;
@@ -262,13 +286,6 @@ struct Config {
   // ev_token_bucket_cfg *rate_limit_cfg;
   // // Rate limit configuration per worker (thread)
   // ev_token_bucket_cfg *worker_rate_limit_cfg;
-  // list of supported NPN/ALPN protocol strings in the order of
-  // preference. The each element of this list is a NULL-terminated
-  // string.
-  std::vector<char *> npn_list;
-  // list of supported SSL/TLS protocol strings. The each element of
-  // this list is a NULL-terminated string.
-  std::vector<char *> tls_proto_list;
   // Path to file containing CA certificate solely used for client
   // certificate validation
   std::unique_ptr<char[]> verify_client_cacert;
@@ -277,12 +294,15 @@ struct Config {
   std::unique_ptr<char[]> accesslog_file;
   std::unique_ptr<char[]> errorlog_file;
   std::unique_ptr<char[]> fetch_ocsp_response_file;
+  std::unique_ptr<char[]> user;
   FILE *http2_upstream_dump_request_header;
   FILE *http2_upstream_dump_response_header;
   nghttp2_session_callbacks *http2_upstream_callbacks;
   nghttp2_session_callbacks *http2_downstream_callbacks;
   nghttp2_option *http2_option;
   nghttp2_option *http2_client_option;
+  const EVP_CIPHER *tls_ticket_cipher;
+  const char *server_name;
   char **argv;
   char *cwd;
   size_t num_worker;
@@ -311,6 +331,8 @@ struct Config {
   size_t downstream_response_buffer_size;
   size_t header_field_buffer;
   size_t max_header_fields;
+  // The index of catch-all group in downstream_addr_groups.
+  size_t downstream_addr_group_catch_all;
   // Bit mask to disable SSL/TLS protocol versions.  This will be
   // passed to SSL_CTX_set_options().
   long int tls_proto_mask;
@@ -319,7 +341,6 @@ struct Config {
   int syslog_facility;
   int backlog;
   int argc;
-  std::unique_ptr<char[]> user;
   uid_t uid;
   gid_t gid;
   pid_t pid;
@@ -357,6 +378,8 @@ struct Config {
   // true if host contains UNIX domain socket path
   bool host_unix;
   bool no_ocsp;
+  // true if --tls-ticket-cipher is used
+  bool tls_ticket_cipher_given;
 };
 
 const Config *get_config();
@@ -365,31 +388,33 @@ void create_config();
 
 // Parses option name |opt| and value |optarg|.  The results are
 // stored into statically allocated Config object. This function
-// returns 0 if it succeeds, or -1.
-int parse_config(const char *opt, const char *optarg);
+// returns 0 if it succeeds, or -1.  The |included_set| contains the
+// all paths already included while processing this configuration, to
+// avoid loop in --include option.
+int parse_config(const char *opt, const char *optarg,
+                 std::set<std::string> &included_set);
 
 // Loads configurations from |filename| and stores them in statically
 // allocated Config object. This function returns 0 if it succeeds, or
-// -1.
-int load_config(const char *filename);
+// -1.  See parse_config() for |include_set|.
+int load_config(const char *filename, std::set<std::string> &include_set);
 
 // Read passwd from |filename|
 std::string read_passwd_from_file(const char *filename);
 
-// Parses comma delimited strings in |s| and returns the array of
-// pointers, each element points to the each substring in |s|.  The
-// |s| must be comma delimited list of strings.  The strings must be
-// delimited by a single comma and any white spaces around it are
-// treated as a part of protocol strings.  This function may modify
-// |s| and the caller must leave it as is after this call.  This
-// function copies |s| and first element in the return value points to
-// it.  It is caller's responsibility to deallocate its memory.
-std::vector<char *> parse_config_str_list(const char *s);
+template <typename T> using Range = std::pair<T, T>;
 
-// Clears all elements of |list|, which is returned by
-// parse_config_str_list().  If list is not empty, list[0] is freed by
-// free(2).  After this call, list.empty() must be true.
-void clear_config_str_list(std::vector<char *> &list);
+// Parses delimited strings in |s| and returns the array of substring,
+// delimited by |delim|.  The any white spaces around substring are
+// treated as a part of substring.
+std::vector<std::string> parse_config_str_list(const char *s, char delim = ',');
+
+// Parses delimited strings in |s| and returns the array of pointers,
+// each element points to the beginning and one beyond last of
+// substring in |s|.  The delimiter is given by |delim|.  The any
+// white spaces around substring are treated as a part of substring.
+std::vector<Range<const char *>> split_config_str_list(const char *s,
+                                                       char delim);
 
 // Parses header field in |optarg|.  We expect header field is formed
 // like "NAME: VALUE".  We require that NAME is non empty string.  ":"
@@ -399,15 +424,23 @@ std::pair<std::string, std::string> parse_header(const char *optarg);
 
 std::vector<LogFragment> parse_log_format(const char *optarg);
 
-// Returns a copy of NULL-terminated string |val|.
-std::unique_ptr<char[]> strcopy(const char *val);
+// Returns a copy of NULL-terminated string [first, last).
+template <typename InputIt>
+std::unique_ptr<char[]> strcopy(InputIt first, InputIt last) {
+  auto res = make_unique<char[]>(last - first + 1);
+  *std::copy(first, last, res.get()) = '\0';
+  return res;
+}
 
-// Returns a copy of string |val| of length |n|.  The returned string
-// will be NULL-terminated.
-std::unique_ptr<char[]> strcopy(const char *val, size_t n);
+// Returns a copy of NULL-terminated string |val|.
+inline std::unique_ptr<char[]> strcopy(const char *val) {
+  return strcopy(val, val + strlen(val));
+}
 
 // Returns a copy of val.c_str().
-std::unique_ptr<char[]> strcopy(const std::string &val);
+inline std::unique_ptr<char[]> strcopy(const std::string &val) {
+  return strcopy(std::begin(val), std::end(val));
+}
 
 // Returns string for syslog |facility|.
 const char *str_syslog_facility(int facility);
@@ -418,10 +451,22 @@ int int_syslog_facility(const char *strfacility);
 FILE *open_file_for_write(const char *filename);
 
 // Reads TLS ticket key file in |files| and returns TicketKey which
-// stores read key data.  This function returns TicketKey if it
+// stores read key data.  The given |cipher| and |hmac| determine the
+// expected file size.  This function returns TicketKey if it
 // succeeds, or nullptr.
 std::unique_ptr<TicketKeys>
-read_tls_ticket_key_file(const std::vector<std::string> &files);
+read_tls_ticket_key_file(const std::vector<std::string> &files,
+                         const EVP_CIPHER *cipher, const EVP_MD *hmac);
+
+// Selects group based on request's |hostport| and |path|.  |hostport|
+// is the value taken from :authority or host header field, and may
+// contain port.  The |path| may contain query part.  We require the
+// catch-all pattern in place, so this function always selects one
+// group.  The catch-all group index is given in |catch_all|.  All
+// patterns are given in |groups|.
+size_t match_downstream_addr_group(
+    const std::string &hostport, const std::string &path,
+    const std::vector<DownstreamAddrGroup> &groups, size_t catch_all);
 
 } // namespace shrpx
 

src/shrpx_config_test.cc

@@ -39,34 +39,29 @@ namespace shrpx {
 void test_shrpx_config_parse_config_str_list(void) {
   auto res = parse_config_str_list("a");
   CU_ASSERT(1 == res.size());
-  CU_ASSERT(0 == strcmp("a", res[0]));
-  clear_config_str_list(res);
+  CU_ASSERT("a" == res[0]);
 
   res = parse_config_str_list("a,");
   CU_ASSERT(2 == res.size());
-  CU_ASSERT(0 == strcmp("a", res[0]));
-  CU_ASSERT(0 == strcmp("", res[1]));
-  clear_config_str_list(res);
+  CU_ASSERT("a" == res[0]);
+  CU_ASSERT("" == res[1]);
 
-  res = parse_config_str_list(",a,,");
+  res = parse_config_str_list(":a::", ':');
   CU_ASSERT(4 == res.size());
-  CU_ASSERT(0 == strcmp("", res[0]));
-  CU_ASSERT(0 == strcmp("a", res[1]));
-  CU_ASSERT(0 == strcmp("", res[2]));
-  CU_ASSERT(0 == strcmp("", res[3]));
-  clear_config_str_list(res);
+  CU_ASSERT("" == res[0]);
+  CU_ASSERT("a" == res[1]);
+  CU_ASSERT("" == res[2]);
+  CU_ASSERT("" == res[3]);
 
   res = parse_config_str_list("");
   CU_ASSERT(1 == res.size());
-  CU_ASSERT(0 == strcmp("", res[0]));
-  clear_config_str_list(res);
+  CU_ASSERT("" == res[0]);
 
   res = parse_config_str_list("alpha,bravo,charlie");
   CU_ASSERT(3 == res.size());
-  CU_ASSERT(0 == strcmp("alpha", res[0]));
-  CU_ASSERT(0 == strcmp("bravo", res[1]));
-  CU_ASSERT(0 == strcmp("charlie", res[2]));
-  clear_config_str_list(res);
+  CU_ASSERT("alpha" == res[0]);
+  CU_ASSERT("bravo" == res[1]);
+  CU_ASSERT("charlie" == res[2]);
 }
 
 void test_shrpx_config_parse_header(void) {
@@ -95,9 +90,9 @@ void test_shrpx_config_parse_header(void) {
 }
 
 void test_shrpx_config_parse_log_format(void) {
-  auto res = parse_log_format("$remote_addr - $remote_user [$time_local] "
-                              "\"$request\" $status $body_bytes_sent "
-                              "\"$http_referer\" \"$http_user_agent\"");
+  auto res = parse_log_format(R"($remote_addr - $remote_user [$time_local] )"
+                              R"("$request" $status $body_bytes_sent )"
+                              R"("${http_referer}" "$http_user_agent")");
   CU_ASSERT(14 == res.size());
 
   CU_ASSERT(SHRPX_LOGF_REMOTE_ADDR == res[0].type);
@@ -136,6 +131,44 @@ void test_shrpx_config_parse_log_format(void) {
 
   CU_ASSERT(SHRPX_LOGF_LITERAL == res[13].type);
   CU_ASSERT(0 == strcmp("\"", res[13].value.get()));
+
+  res = parse_log_format("$");
+
+  CU_ASSERT(1 == res.size());
+
+  CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type);
+  CU_ASSERT(0 == strcmp("$", res[0].value.get()));
+
+  res = parse_log_format("${");
+
+  CU_ASSERT(1 == res.size());
+
+  CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type);
+  CU_ASSERT(0 == strcmp("${", res[0].value.get()));
+
+  res = parse_log_format("${a");
+
+  CU_ASSERT(1 == res.size());
+
+  CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type);
+  CU_ASSERT(0 == strcmp("${a", res[0].value.get()));
+
+  res = parse_log_format("${a ");
+
+  CU_ASSERT(1 == res.size());
+
+  CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type);
+  CU_ASSERT(0 == strcmp("${a ", res[0].value.get()));
+
+  res = parse_log_format("$$remote_addr");
+
+  CU_ASSERT(2 == res.size());
+
+  CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type);
+  CU_ASSERT(0 == strcmp("$", res[0].value.get()));
+
+  CU_ASSERT(SHRPX_LOGF_REMOTE_ADDR == res[1].type);
+  CU_ASSERT(nullptr == res[1].value.get());
 }
 
 void test_shrpx_config_read_tls_ticket_key_file(void) {
@@ -152,24 +185,122 @@ void test_shrpx_config_read_tls_ticket_key_file(void) {
 
   close(fd1);
   close(fd2);
-  auto ticket_keys = read_tls_ticket_key_file({file1, file2});
+  auto ticket_keys =
+      read_tls_ticket_key_file({file1, file2}, EVP_aes_128_cbc(), EVP_sha256());
   unlink(file1);
   unlink(file2);
   CU_ASSERT(ticket_keys.get() != nullptr);
   CU_ASSERT(2 == ticket_keys->keys.size());
   auto key = &ticket_keys->keys[0];
-  CU_ASSERT(0 == memcmp("0..............1", key->name, sizeof(key->name)));
   CU_ASSERT(0 ==
-            memcmp("2..............3", key->aes_key, sizeof(key->aes_key)));
-  CU_ASSERT(0 ==
-            memcmp("4..............5", key->hmac_key, sizeof(key->hmac_key)));
+            memcmp("0..............1", key->data.name, sizeof(key->data.name)));
+  CU_ASSERT(0 == memcmp("2..............3", key->data.enc_key, 16));
+  CU_ASSERT(0 == memcmp("4..............5", key->data.hmac_key, 16));
 
   key = &ticket_keys->keys[1];
-  CU_ASSERT(0 == memcmp("6..............7", key->name, sizeof(key->name)));
   CU_ASSERT(0 ==
-            memcmp("8..............9", key->aes_key, sizeof(key->aes_key)));
+            memcmp("6..............7", key->data.name, sizeof(key->data.name)));
+  CU_ASSERT(0 == memcmp("8..............9", key->data.enc_key, 16));
+  CU_ASSERT(0 == memcmp("a..............b", key->data.hmac_key, 16));
+}
+
+void test_shrpx_config_read_tls_ticket_key_file_aes_256(void) {
+  char file1[] = "/tmp/nghttpx-unittest.XXXXXX";
+  auto fd1 = mkstemp(file1);
+  assert(fd1 != -1);
+  assert(80 == write(fd1, "0..............12..............................34..."
+                          "...........................5",
+                     80));
+  char file2[] = "/tmp/nghttpx-unittest.XXXXXX";
+  auto fd2 = mkstemp(file2);
+  assert(fd2 != -1);
+  assert(80 == write(fd2, "6..............78..............................9a..."
+                          "...........................b",
+                     80));
+
+  close(fd1);
+  close(fd2);
+  auto ticket_keys =
+      read_tls_ticket_key_file({file1, file2}, EVP_aes_256_cbc(), EVP_sha256());
+  unlink(file1);
+  unlink(file2);
+  CU_ASSERT(ticket_keys.get() != nullptr);
+  CU_ASSERT(2 == ticket_keys->keys.size());
+  auto key = &ticket_keys->keys[0];
   CU_ASSERT(0 ==
-            memcmp("a..............b", key->hmac_key, sizeof(key->hmac_key)));
+            memcmp("0..............1", key->data.name, sizeof(key->data.name)));
+  CU_ASSERT(0 ==
+            memcmp("2..............................3", key->data.enc_key, 32));
+  CU_ASSERT(0 ==
+            memcmp("4..............................5", key->data.hmac_key, 32));
+
+  key = &ticket_keys->keys[1];
+  CU_ASSERT(0 ==
+            memcmp("6..............7", key->data.name, sizeof(key->data.name)));
+  CU_ASSERT(0 ==
+            memcmp("8..............................9", key->data.enc_key, 32));
+  CU_ASSERT(0 ==
+            memcmp("a..............................b", key->data.hmac_key, 32));
+}
+
+void test_shrpx_config_match_downstream_addr_group(void) {
+  auto groups = std::vector<DownstreamAddrGroup>{
+      {"nghttp2.org/"},
+      {"nghttp2.org/alpha/bravo/"},
+      {"nghttp2.org/alpha/charlie"},
+      {"nghttp2.org/delta%3A"},
+      {"www.nghttp2.org/"},
+      {"[::1]/"},
+  };
+
+  CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "/", groups, 255));
+
+  // port is removed
+  CU_ASSERT(0 ==
+            match_downstream_addr_group("nghttp2.org:8080", "/", groups, 255));
+
+  // host is case-insensitive
+  CU_ASSERT(4 == match_downstream_addr_group("WWW.nghttp2.org", "/alpha",
+                                             groups, 255));
+
+  CU_ASSERT(1 == match_downstream_addr_group("nghttp2.org", "/alpha/bravo/",
+                                             groups, 255));
+
+  // /alpha/bravo also matches /alpha/bravo/
+  CU_ASSERT(1 == match_downstream_addr_group("nghttp2.org", "/alpha/bravo",
+                                             groups, 255));
+
+  // path part is case-sensitive
+  CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "/Alpha/bravo",
+                                             groups, 255));
+
+  CU_ASSERT(1 == match_downstream_addr_group(
+                     "nghttp2.org", "/alpha/bravo/charlie", groups, 255));
+
+  CU_ASSERT(2 == match_downstream_addr_group("nghttp2.org", "/alpha/charlie",
+                                             groups, 255));
+
+  // pattern which does not end with '/' must match its entirely.  So
+  // this matches to group 0, not group 2.
+  CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "/alpha/charlie/",
+                                             groups, 255));
+
+  CU_ASSERT(255 ==
+            match_downstream_addr_group("example.org", "/", groups, 255));
+
+  CU_ASSERT(255 == match_downstream_addr_group("", "/", groups, 255));
+
+  CU_ASSERT(255 == match_downstream_addr_group("", "alpha", groups, 255));
+
+  CU_ASSERT(255 == match_downstream_addr_group("foo/bar", "/", groups, 255));
+
+  // If path is "*", only match with host + "/".
+  CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "*", groups, 255));
+
+  CU_ASSERT(5 == match_downstream_addr_group("[::1]", "/", groups, 255));
+  CU_ASSERT(5 == match_downstream_addr_group("[::1]:8080", "/", groups, 255));
+  CU_ASSERT(255 == match_downstream_addr_group("[::1", "/", groups, 255));
+  CU_ASSERT(255 == match_downstream_addr_group("[::1]8000", "/", groups, 255));
 }
 
 } // namespace shrpx

src/shrpx_config_test.h

@@ -35,6 +35,8 @@ void test_shrpx_config_parse_config_str_list(void);
 void test_shrpx_config_parse_header(void);
 void test_shrpx_config_parse_log_format(void);
 void test_shrpx_config_read_tls_ticket_key_file(void);
+void test_shrpx_config_read_tls_ticket_key_file_aes_256(void);
+void test_shrpx_config_match_downstream_addr_group(void);
 
 } // namespace shrpx
 

src/shrpx_connection.cc

@@ -62,7 +62,13 @@ Connection::Connection(struct ev_loop *loop, int fd, SSL *ssl,
   tls.last_write_time = 0.;
 }
 
-Connection::~Connection() { disconnect(); }
+Connection::~Connection() {
+  disconnect();
+
+  if (tls.ssl) {
+    SSL_free(tls.ssl);
+  }
+}
 
 void Connection::disconnect() {
   ev_timer_stop(loop, &rt);
@@ -75,10 +81,13 @@ void Connection::disconnect() {
     SSL_set_app_data(tls.ssl, nullptr);
     SSL_set_shutdown(tls.ssl, SSL_RECEIVED_SHUTDOWN);
     ERR_clear_error();
-    SSL_shutdown(tls.ssl);
+    // To reuse SSL/TLS session, we have to shutdown, and don't free
+    // tls.ssl.
+    if (SSL_shutdown(tls.ssl) != 1) {
       SSL_free(tls.ssl);
       tls.ssl = nullptr;
     }
+  }
 
   if (fd != -1) {
     shutdown(fd, SHUT_WR);

src/shrpx_connection_handler.cc

@@ -128,24 +128,17 @@ ConnectionHandler::~ConnectionHandler() {
   }
 }
 
-void ConnectionHandler::worker_reopen_log_files() {
-  WorkerEvent wev;
-
-  memset(&wev, 0, sizeof(wev));
-  wev.type = REOPEN_LOG;
-
+void ConnectionHandler::set_ticket_keys_to_worker(
+    const std::shared_ptr<TicketKeys> &ticket_keys) {
   for (auto &worker : workers_) {
-    worker->send(wev);
+    worker->set_ticket_keys(ticket_keys);
   }
 }
 
-void ConnectionHandler::worker_renew_ticket_keys(
-    const std::shared_ptr<TicketKeys> &ticket_keys) {
-  WorkerEvent wev;
+void ConnectionHandler::worker_reopen_log_files() {
+  WorkerEvent wev{};
 
-  memset(&wev, 0, sizeof(wev));
-  wev.type = RENEW_TICKET_KEYS;
-  wev.ticket_keys = ticket_keys;
+  wev.type = REOPEN_LOG;
 
   for (auto &worker : workers_) {
     worker->send(wev);
@@ -216,8 +209,7 @@ void ConnectionHandler::graceful_shutdown_worker() {
     return;
   }
 
-  WorkerEvent wev;
-  memset(&wev, 0, sizeof(wev));
+  WorkerEvent wev{};
   wev.type = GRACEFUL_SHUTDOWN;
 
   if (LOG_ENABLED(INFO)) {
@@ -266,8 +258,7 @@ int ConnectionHandler::handle_connection(int fd, sockaddr *addr, int addrlen) {
     LOG(INFO) << "Dispatch connection to worker #" << idx;
   }
   ++worker_round_robin_cnt_;
-  WorkerEvent wev;
-  memset(&wev, 0, sizeof(wev));
+  WorkerEvent wev{};
   wev.type = NEW_CONNECTION;
   wev.client_fd = fd;
   memcpy(&wev.client_addr, addr, addrlen);

src/shrpx_connection_handler.h

@@ -76,8 +76,9 @@ public:
   // Creates |num| Worker objects for multi threaded configuration.
   // The |num| must be strictly more than 1.
   void create_worker_thread(size_t num);
+  void
+  set_ticket_keys_to_worker(const std::shared_ptr<TicketKeys> &ticket_keys);
   void worker_reopen_log_files();
-  void worker_renew_ticket_keys(const std::shared_ptr<TicketKeys> &ticket_keys);
   void set_ticket_keys(std::shared_ptr<TicketKeys> ticket_keys);
   const std::shared_ptr<TicketKeys> &get_ticket_keys() const;
   struct ev_loop *get_loop() const;

src/shrpx_downstream.cc

@@ -152,10 +152,6 @@ Downstream::~Downstream() {
     DLOG(INFO, this) << "Deleting";
   }
 
-  if (blocked_link_) {
-    detach_blocked_link(blocked_link_);
-  }
-
   // check nullptr for unittest
   if (upstream_) {
     auto loop = upstream_->get_client_handler()->get_loop();
@@ -166,6 +162,10 @@ Downstream::~Downstream() {
     ev_timer_stop(loop, &downstream_wtimer_);
   }
 
+  // DownstreamConnection may refer to this object.  Delete it now
+  // explicitly.
+  dconn_.reset();
+
   if (LOG_ENABLED(INFO)) {
     DLOG(INFO, this) << "Deleted";
   }
@@ -195,8 +195,6 @@ void Downstream::detach_downstream_connection() {
       std::unique_ptr<DownstreamConnection>(dconn_.release()));
 }
 
-void Downstream::release_downstream_connection() { dconn_.release(); }
-
 DownstreamConnection *Downstream::get_downstream_connection() {
   return dconn_.get();
 }
@@ -623,8 +621,7 @@ void Downstream::rewrite_location_response_header(
   if (!hd) {
     return;
   }
-  http_parser_url u;
-  memset(&u, 0, sizeof(u));
+  http_parser_url u{};
   int rv =
       http_parser_parse_url((*hd).value.c_str(), (*hd).value.size(), 0, &u);
   if (rv != 0) {
@@ -1200,16 +1197,20 @@ void Downstream::attach_blocked_link(BlockedLink *l) {
   blocked_link_ = l;
 }
 
-void Downstream::detach_blocked_link(BlockedLink *l) {
-  assert(blocked_link_);
-  assert(l->downstream == this);
-
-  l->downstream = nullptr;
+BlockedLink *Downstream::detach_blocked_link() {
+  auto link = blocked_link_;
   blocked_link_ = nullptr;
+  return link;
 }
 
 void Downstream::add_request_headers_sum(size_t amount) {
   request_headers_sum_ += amount;
 }
 
+bool Downstream::can_detach_downstream_connection() const {
+  return dconn_ && response_state_ == Downstream::MSG_COMPLETE &&
+         request_state_ == Downstream::MSG_COMPLETE && !upgraded_ &&
+         !response_connection_close_;
+}
+
 } // namespace shrpx

src/shrpx_downstream.h

@@ -69,8 +69,6 @@ public:
 
   int attach_downstream_connection(std::unique_ptr<DownstreamConnection> dconn);
   void detach_downstream_connection();
-  // Releases dconn_, without freeing it.
-  void release_downstream_connection();
   DownstreamConnection *get_downstream_connection();
   // Returns dconn_ and nullifies dconn_.
   std::unique_ptr<DownstreamConnection> pop_downstream_connection();
@@ -332,7 +330,10 @@ public:
   void set_dispatch_state(int s);
 
   void attach_blocked_link(BlockedLink *l);
-  void detach_blocked_link(BlockedLink *l);
+  BlockedLink *detach_blocked_link();
+
+  // Returns true if downstream_connection can be detached and reused.
+  bool can_detach_downstream_connection() const;
 
   enum {
     EVENT_ERROR = 0x1,

src/shrpx_downstream_connection.h

@@ -57,6 +57,7 @@ public:
 
   virtual void on_upstream_change(Upstream *uptream) = 0;
   virtual int on_priority_change(int32_t pri) = 0;
+  virtual size_t get_group() const = 0;
 
   // true if this object is poolable.
   virtual bool poolable() const = 0;

src/shrpx_downstream_connection_pool.cc

@@ -27,33 +27,42 @@
 
 namespace shrpx {
 
-DownstreamConnectionPool::DownstreamConnectionPool() {}
+DownstreamConnectionPool::DownstreamConnectionPool(size_t num_groups)
+    : gpool_(num_groups) {}
 
 DownstreamConnectionPool::~DownstreamConnectionPool() {
-  for (auto dconn : pool_) {
+  for (auto &pool : gpool_) {
+    for (auto dconn : pool) {
       delete dconn;
     }
+  }
 }
 
 void DownstreamConnectionPool::add_downstream_connection(
     std::unique_ptr<DownstreamConnection> dconn) {
-  pool_.insert(dconn.release());
+  auto group = dconn->get_group();
+  assert(gpool_.size() > group);
+  gpool_[group].insert(dconn.release());
 }
 
 std::unique_ptr<DownstreamConnection>
-DownstreamConnectionPool::pop_downstream_connection() {
-  if (pool_.empty()) {
+DownstreamConnectionPool::pop_downstream_connection(size_t group) {
+  assert(gpool_.size() > group);
+  auto &pool = gpool_[group];
+  if (pool.empty()) {
     return nullptr;
   }
 
-  auto dconn = std::unique_ptr<DownstreamConnection>(*std::begin(pool_));
-  pool_.erase(std::begin(pool_));
+  auto dconn = std::unique_ptr<DownstreamConnection>(*std::begin(pool));
+  pool.erase(std::begin(pool));
   return dconn;
 }
 
 void DownstreamConnectionPool::remove_downstream_connection(
     DownstreamConnection *dconn) {
-  pool_.erase(dconn);
+  auto group = dconn->get_group();
+  assert(gpool_.size() > group);
+  gpool_[group].erase(dconn);
   delete dconn;
 }
 

src/shrpx_downstream_connection_pool.h

@@ -36,15 +36,15 @@ class DownstreamConnection;
 
 class DownstreamConnectionPool {
 public:
-  DownstreamConnectionPool();
+  DownstreamConnectionPool(size_t num_groups);
   ~DownstreamConnectionPool();
 
   void add_downstream_connection(std::unique_ptr<DownstreamConnection> dconn);
-  std::unique_ptr<DownstreamConnection> pop_downstream_connection();
+  std::unique_ptr<DownstreamConnection> pop_downstream_connection(size_t group);
   void remove_downstream_connection(DownstreamConnection *dconn);
 
 private:
-  std::set<DownstreamConnection *> pool_;
+  std::vector<std::set<DownstreamConnection *>> gpool_;
 };
 
 } // namespace shrpx

src/shrpx_downstream_queue.cc

@@ -124,17 +124,21 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream) {
   // Delete downstream when this function returns.
   auto delptr = std::unique_ptr<Downstream>(downstream);
 
-  if (downstream->get_dispatch_state() != Downstream::DISPATCH_ACTIVE) {
-    assert(downstream->get_dispatch_state() != Downstream::DISPATCH_NONE);
-    downstreams_.remove(downstream);
-    return nullptr;
-  }
-
   downstreams_.remove(downstream);
 
   auto &host = make_host_key(downstream);
   auto &ent = find_host_entry(host);
+
+  if (downstream->get_dispatch_state() == Downstream::DISPATCH_ACTIVE) {
     --ent.num_active;
+  } else {
+    // For those downstreams deleted while in blocked state
+    auto link = downstream->detach_blocked_link();
+    if (link) {
+      ent.blocked.remove(link);
+      delete link;
+    }
+  }
 
   if (remove_host_entry_if_empty(ent, host_entries_, host)) {
     return nullptr;
@@ -144,21 +148,20 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream) {
     return nullptr;
   }
 
-  for (auto link = ent.blocked.head; link;) {
-    auto next = link->dlnext;
-    if (!link->downstream) {
-      ent.blocked.remove(link);
-      link = next;
-      continue;
+  auto link = ent.blocked.head;
+
+  if (!link) {
+    return nullptr;
   }
+
   auto next_downstream = link->downstream;
-    next_downstream->detach_blocked_link(link);
+  auto link2 = next_downstream->detach_blocked_link();
+  assert(link2 == link);
   ent.blocked.remove(link);
   delete link;
   remove_host_entry_if_empty(ent, host_entries_, host);
+
   return next_downstream;
-  }
-  return nullptr;
 }
 
 Downstream *DownstreamQueue::get_downstreams() const {

src/shrpx_http2_downstream_connection.cc

@@ -86,11 +86,7 @@ Http2DownstreamConnection::~Http2DownstreamConnection() {
     }
   }
   http2session_->remove_downstream_connection(this);
-  // Downstream and DownstreamConnection may be deleted
-  // asynchronously.
-  if (downstream_) {
-    downstream_->release_downstream_connection();
-  }
+
   if (LOG_ENABLED(INFO)) {
     DCLOG(INFO, this) << "Deleted";
   }
@@ -263,8 +259,11 @@ int Http2DownstreamConnection::push_request_headers() {
   // http2session_ has already in CONNECTED state, so we can get
   // addr_idx here.
   auto addr_idx = http2session_->get_addr_idx();
-  auto downstream_hostport =
-      get_config()->downstream_addrs[addr_idx].hostport.get();
+  auto group = http2session_->get_group();
+  auto downstream_hostport = get_config()
+                                 ->downstream_addr_groups[group]
+                                 .addrs[addr_idx]
+                                 .hostport.get();
 
   const char *authority = nullptr, *host = nullptr;
   if (!no_host_rewrite) {
@@ -557,4 +556,10 @@ int Http2DownstreamConnection::on_timeout() {
   return submit_rst_stream(downstream_, NGHTTP2_NO_ERROR);
 }
 
+size_t Http2DownstreamConnection::get_group() const {
+  // HTTP/2 backend connections are managed by Http2Session object,
+  // and it stores group index.
+  return http2session_->get_group();
+}
+
 } // namespace shrpx

src/shrpx_http2_downstream_connection.h

@@ -61,6 +61,7 @@ public:
 
   virtual void on_upstream_change(Upstream *upstream) {}
   virtual int on_priority_change(int32_t pri);
+  virtual size_t get_group() const;
 
   // This object is not poolable because we dont' have facility to
   // migrate to another Http2Session object.

src/shrpx_http2_session.cc

@@ -142,13 +142,14 @@ void writecb(struct ev_loop *loop, ev_io *w, int revents) {
 } // namespace
 
 Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx,
-                           ConnectBlocker *connect_blocker, Worker *worker)
+                           ConnectBlocker *connect_blocker, Worker *worker,
+                           size_t group, size_t idx)
     : conn_(loop, -1, nullptr, get_config()->downstream_write_timeout,
             get_config()->downstream_read_timeout, 0, 0, 0, 0, writecb, readcb,
             timeoutcb, this),
       worker_(worker), connect_blocker_(connect_blocker), ssl_ctx_(ssl_ctx),
       session_(nullptr), data_pending_(nullptr), data_pendinglen_(0),
-      addr_idx_(0), state_(DISCONNECTED),
+      addr_idx_(0), group_(group), index_(idx), state_(DISCONNECTED),
       connection_check_state_(CONNECTION_CHECK_NONE), flow_control_(false) {
 
   read_ = write_ = &Http2Session::noop;
@@ -233,11 +234,17 @@ int Http2Session::disconnect(bool hard) {
   return 0;
 }
 
-int Http2Session::check_cert() { return ssl::check_cert(conn_.tls.ssl); }
+int Http2Session::check_cert() {
+  return ssl::check_cert(
+      conn_.tls.ssl,
+      &get_config()->downstream_addr_groups[group_].addrs[addr_idx_]);
+}
 
 int Http2Session::initiate_connection() {
   int rv = 0;
 
+  auto &addrs = get_config()->downstream_addr_groups[group_].addrs;
+
   if (state_ == DISCONNECTED) {
     if (connect_blocker_->blocked()) {
       if (LOG_ENABLED(INFO)) {
@@ -247,18 +254,19 @@ int Http2Session::initiate_connection() {
       return -1;
     }
 
-    auto worker_stat = worker_->get_worker_stat();
-    addr_idx_ = worker_stat->next_downstream;
-    ++worker_stat->next_downstream;
-    worker_stat->next_downstream %= get_config()->downstream_addrs.size();
+    auto &next_downstream = worker_->get_dgrp(group_)->next;
+    addr_idx_ = next_downstream;
+    if (++next_downstream >= addrs.size()) {
+      next_downstream = 0;
+    }
 
     if (LOG_ENABLED(INFO)) {
       SSLOG(INFO, this) << "Using downstream address idx=" << addr_idx_
-                        << " out of " << get_config()->downstream_addrs.size();
+                        << " out of " << addrs.size();
     }
   }
 
-  auto &downstream_addr = get_config()->downstream_addrs[addr_idx_];
+  auto &downstream_addr = addrs[addr_idx_];
 
   if (get_config()->downstream_http_proxy_host && state_ == DISCONNECTED) {
     if (LOG_ENABLED(INFO)) {
@@ -312,13 +320,16 @@ int Http2Session::initiate_connection() {
       SSLOG(INFO, this) << "Connecting to downstream server";
     }
     if (ssl_ctx_) {
-      // We are establishing TLS connection.
+      // We are establishing TLS connection.  If conn_.tls.ssl, we may
+      // reuse the previous session.
+      if (!conn_.tls.ssl) {
         conn_.tls.ssl = SSL_new(ssl_ctx_);
         if (!conn_.tls.ssl) {
           SSLOG(ERROR, this) << "SSL_new() failed: "
                              << ERR_error_string(ERR_get_error(), NULL);
           return -1;
         }
+      }
 
       const char *sni_name = nullptr;
       if (get_config()->backend_tls_sni_name) {
@@ -503,7 +514,8 @@ int Http2Session::downstream_connect_proxy() {
   if (LOG_ENABLED(INFO)) {
     SSLOG(INFO, this) << "Connected to the proxy";
   }
-  auto &downstream_addr = get_config()->downstream_addrs[addr_idx_];
+  auto &downstream_addr =
+      get_config()->downstream_addr_groups[group_].addrs[addr_idx_];
 
   std::string req = "CONNECT ";
   req += downstream_addr.hostport.get();
@@ -1744,4 +1756,8 @@ bool Http2Session::should_hard_fail() const {
 
 size_t Http2Session::get_addr_idx() const { return addr_idx_; }
 
+size_t Http2Session::get_group() const { return group_; }
+
+size_t Http2Session::get_index() const { return index_; }
+
 } // namespace shrpx

src/shrpx_http2_session.h

@@ -58,7 +58,8 @@ struct StreamData {
 class Http2Session {
 public:
   Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx,
-               ConnectBlocker *connect_blocker, Worker *worker);
+               ConnectBlocker *connect_blocker, Worker *worker, size_t group,
+               size_t idx);
   ~Http2Session();
 
   int check_cert();
@@ -151,6 +152,10 @@ public:
 
   size_t get_addr_idx() const;
 
+  size_t get_group() const;
+
+  size_t get_index() const;
+
   enum {
     // Disconnected
     DISCONNECTED,
@@ -203,6 +208,10 @@ private:
   size_t data_pendinglen_;
   // index of get_config()->downstream_addrs this object uses
   size_t addr_idx_;
+  size_t group_;
+  // index inside group, this is used to pin frontend to certain
+  // HTTP/2 backend for better throughput.
+  size_t index_;
   int state_;
   int connection_check_state_;
   bool flow_control_;

src/shrpx_http2_upstream.cc

@@ -36,6 +36,7 @@
 #include "shrpx_config.h"
 #include "shrpx_http.h"
 #include "shrpx_worker.h"
+#include "shrpx_http2_session.h"
 #include "http2.h"
 #include "util.h"
 #include "base64.h"
@@ -73,21 +74,12 @@ int on_stream_close_callback(nghttp2_session *session, int32_t stream_id,
     return 0;
   }
 
-  downstream->set_request_state(Downstream::STREAM_CLOSED);
-
-  if (downstream->get_response_state() == Downstream::MSG_COMPLETE) {
-    // At this point, downstream response was read
-    if (!downstream->get_upgraded() &&
-        !downstream->get_response_connection_close()) {
+  if (downstream->can_detach_downstream_connection()) {
     // Keep-alive
     downstream->detach_downstream_connection();
   }
 
-    upstream->remove_downstream(downstream);
-    // downstream was deleted
-
-    return 0;
-  }
+  downstream->set_request_state(Downstream::STREAM_CLOSED);
 
   // At this point, downstream read may be paused.
 
@@ -300,7 +292,15 @@ int Http2Upstream::on_request_headers(Downstream *downstream,
   downstream->set_request_method(method_token);
   downstream->set_request_http2_scheme(http2::value_to_str(scheme));
   downstream->set_request_http2_authority(http2::value_to_str(authority));
+  if (path) {
+    if (get_config()->http2_proxy || get_config()->client_proxy) {
       downstream->set_request_path(http2::value_to_str(path));
+    } else {
+      auto &value = path->value;
+      downstream->set_request_path(
+          http2::rewrite_clean_path(std::begin(value), std::end(value)));
+    }
+  }
 
   if (!(frame->hd.flags & NGHTTP2_FLAG_END_STREAM)) {
     downstream->set_request_http2_expect_body(true);
@@ -334,7 +334,7 @@ void Http2Upstream::initiate_downstream(Downstream *downstream) {
   int rv;
 
   rv = downstream->attach_downstream_connection(
-      handler_->get_downstream_connection());
+      handler_->get_downstream_connection(downstream));
   if (rv != 0) {
     // downstream connection fails, send error page
     if (error_reply(downstream, 503) != 0) {
@@ -541,7 +541,8 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame,
             {nv.value, nv.value + nv.valuelen});
         break;
       case http2::HD__PATH:
-        downstream->set_request_path({nv.value, nv.value + nv.valuelen});
+        downstream->set_request_path(
+            http2::rewrite_clean_path(nv.value, nv.value + nv.valuelen));
         break;
       }
       downstream->add_request_header(nv.name, nv.namelen, nv.value, nv.valuelen,
@@ -874,16 +875,6 @@ ClientHandler *Http2Upstream::get_client_handler() const { return handler_; }
 int Http2Upstream::downstream_read(DownstreamConnection *dconn) {
   auto downstream = dconn->get_downstream();
 
-  if (downstream->get_request_state() == Downstream::STREAM_CLOSED) {
-    // If upstream HTTP2 stream was closed, we just close downstream,
-    // because there is no consumer now. Downstream connection is also
-    // closed in this case.
-    remove_downstream(downstream);
-    // downstream was deleted
-
-    return 0;
-  }
-
   if (downstream->get_response_state() == Downstream::MSG_RESET) {
     // The downstream stream was reset (canceled). In this case,
     // RST_STREAM to the upstream and delete downstream connection
@@ -915,10 +906,8 @@ int Http2Upstream::downstream_read(DownstreamConnection *dconn) {
       }
       return downstream_error(dconn, Downstream::EVENT_ERROR);
     }
-    // Detach downstream connection early so that it could be reused
-    // without hitting server's request timeout.
-    if (downstream->get_response_state() == Downstream::MSG_COMPLETE &&
-        !downstream->get_response_connection_close()) {
+
+    if (downstream->can_detach_downstream_connection()) {
       // Keep-alive
       downstream->detach_downstream_connection();
     }
@@ -949,14 +938,6 @@ int Http2Upstream::downstream_eof(DownstreamConnection *dconn) {
   if (LOG_ENABLED(INFO)) {
     DCLOG(INFO, dconn) << "EOF. stream_id=" << downstream->get_stream_id();
   }
-  if (downstream->get_request_state() == Downstream::STREAM_CLOSED) {
-    // If stream was closed already, we don't need to send reply at
-    // the first place. We can delete downstream.
-    remove_downstream(downstream);
-    // downstream was deleted
-
-    return 0;
-  }
 
   // Delete downstream connection. If we don't delete it here, it will
   // be pooled in on_stream_close_callback.
@@ -1002,13 +983,6 @@ int Http2Upstream::downstream_error(DownstreamConnection *dconn, int events) {
     }
   }
 
-  if (downstream->get_request_state() == Downstream::STREAM_CLOSED) {
-    remove_downstream(downstream);
-    // downstream was deleted
-
-    return 0;
-  }
-
   // Delete downstream connection. If we don't delete it here, it will
   // be pooled in on_stream_close_callback.
   downstream->pop_downstream_connection();
@@ -1476,7 +1450,7 @@ int Http2Upstream::on_downstream_reset(bool no_retry) {
     // downstream connection.
 
     rv = downstream->attach_downstream_connection(
-        handler_->get_downstream_connection());
+        handler_->get_downstream_connection(downstream));
     if (rv != 0) {
       goto fail;
     }
@@ -1497,8 +1471,7 @@ int Http2Upstream::on_downstream_reset(bool no_retry) {
 
 int Http2Upstream::prepare_push_promise(Downstream *downstream) {
   int rv;
-  http_parser_url u;
-  memset(&u, 0, sizeof(u));
+  http_parser_url u{};
   rv = http_parser_parse_url(downstream->get_request_path().c_str(),
                              downstream->get_request_path().size(), 0, &u);
   if (rv != 0) {
@@ -1528,8 +1501,7 @@ int Http2Upstream::prepare_push_promise(Downstream *downstream) {
       const char *relq = nullptr;
       size_t relqlen = 0;
 
-      http_parser_url v;
-      memset(&v, 0, sizeof(v));
+      http_parser_url v{};
       rv = http_parser_parse_url(link_url, link_urllen, 0, &v);
       if (rv != 0) {
         assert(link_urllen);

src/shrpx_http_downstream_connection.cc

@@ -34,6 +34,7 @@
 #include "shrpx_connect_blocker.h"
 #include "shrpx_downstream_connection_pool.h"
 #include "shrpx_worker.h"
+#include "shrpx_http2_session.h"
 #include "http2.h"
 #include "util.h"
 
@@ -109,21 +110,15 @@ void connectcb(struct ev_loop *loop, ev_io *w, int revents) {
 } // namespace
 
 HttpDownstreamConnection::HttpDownstreamConnection(
-    DownstreamConnectionPool *dconn_pool, struct ev_loop *loop)
+    DownstreamConnectionPool *dconn_pool, size_t group, struct ev_loop *loop)
     : DownstreamConnection(dconn_pool),
       conn_(loop, -1, nullptr, get_config()->downstream_write_timeout,
             get_config()->downstream_read_timeout, 0, 0, 0, 0, connectcb,
             readcb, timeoutcb, this),
-      ioctrl_(&conn_.rlimit), response_htp_{0}, addr_idx_(0),
+      ioctrl_(&conn_.rlimit), response_htp_{0}, group_(group), addr_idx_(0),
       connected_(false) {}
 
-HttpDownstreamConnection::~HttpDownstreamConnection() {
-  // Downstream and DownstreamConnection may be deleted
-  // asynchronously.
-  if (downstream_) {
-    downstream_->release_downstream_connection();
-  }
-}
+HttpDownstreamConnection::~HttpDownstreamConnection() {}
 
 int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
   if (LOG_ENABLED(INFO)) {
@@ -142,15 +137,17 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
     }
 
     auto worker = client_handler_->get_worker();
-    auto worker_stat = worker->get_worker_stat();
-    auto end = worker_stat->next_downstream;
+    auto &next_downstream = worker->get_dgrp(group_)->next;
+    auto end = next_downstream;
+    auto &addrs = get_config()->downstream_addr_groups[group_].addrs;
     for (;;) {
-      auto i = worker_stat->next_downstream;
-      ++worker_stat->next_downstream;
-      worker_stat->next_downstream %= get_config()->downstream_addrs.size();
+      auto &addr = addrs[next_downstream];
+      auto i = next_downstream;
+      if (++next_downstream >= addrs.size()) {
+        next_downstream = 0;
+      }
 
-      conn_.fd = util::create_nonblock_socket(
-          get_config()->downstream_addrs[i].addr.storage.ss_family);
+      conn_.fd = util::create_nonblock_socket(addr.addr.storage.ss_family);
 
       if (conn_.fd == -1) {
         auto error = errno;
@@ -162,8 +159,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
       }
 
       int rv;
-      rv = connect(conn_.fd, &get_config()->downstream_addrs[i].addr.sa,
-                   get_config()->downstream_addrs[i].addrlen);
+      rv = connect(conn_.fd, &addr.addr.sa, addr.addrlen);
       if (rv != 0 && errno != EINPROGRESS) {
         auto error = errno;
         DCLOG(WARN, this) << "connect() failed; errno=" << error;
@@ -172,7 +168,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
         close(conn_.fd);
         conn_.fd = -1;
 
-        if (end == worker_stat->next_downstream) {
+        if (end == next_downstream) {
           return SHRPX_ERR_NETWORK;
         }
 
@@ -203,6 +199,8 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
   ev_set_cb(&conn_.rev, readcb);
 
   conn_.rt.repeat = get_config()->downstream_read_timeout;
+  // we may set read timer cb to idle_timeoutcb.  Reset again.
+  ev_set_cb(&conn_.rt, timeoutcb);
   ev_timer_again(conn_.loop, &conn_.rt);
   // TODO we should have timeout for connection establishment
   ev_timer_again(conn_.loop, &conn_.wt);
@@ -212,8 +210,10 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
 
 int HttpDownstreamConnection::push_request_headers() {
   const char *authority = nullptr, *host = nullptr;
-  auto downstream_hostport =
-      get_config()->downstream_addrs[addr_idx_].hostport.get();
+  auto downstream_hostport = get_config()
+                                 ->downstream_addr_groups[group_]
+                                 .addrs[addr_idx_]
+                                 .hostport.get();
   auto connect_method = downstream_->get_request_method() == HTTP_CONNECT;
 
   if (!get_config()->no_host_rewrite && !get_config()->http2_proxy &&
@@ -860,6 +860,8 @@ int HttpDownstreamConnection::on_connect() {
       DLOG(INFO, this) << "downstream connect failed";
     }
 
+    downstream_->set_request_state(Downstream::CONNECT_FAIL);
+
     return -1;
   }
 
@@ -877,4 +879,6 @@ void HttpDownstreamConnection::on_upstream_change(Upstream *upstream) {}
 
 void HttpDownstreamConnection::signal_write() { conn_.wlimit.startw(); }
 
+size_t HttpDownstreamConnection::get_group() const { return group_; }
+
 } // namespace shrpx

src/shrpx_http_downstream_connection.h

@@ -39,7 +39,7 @@ class DownstreamConnectionPool;
 
 class HttpDownstreamConnection : public DownstreamConnection {
 public:
-  HttpDownstreamConnection(DownstreamConnectionPool *dconn_pool,
+  HttpDownstreamConnection(DownstreamConnectionPool *dconn_pool, size_t group,
                            struct ev_loop *loop);
   virtual ~HttpDownstreamConnection();
   virtual int attach_downstream(Downstream *downstream);
@@ -58,6 +58,7 @@ public:
 
   virtual void on_upstream_change(Upstream *upstream);
   virtual int on_priority_change(int32_t pri) { return 0; }
+  virtual size_t get_group() const;
 
   virtual bool poolable() const { return true; }
 
@@ -68,6 +69,7 @@ private:
   Connection conn_;
   IOControl ioctrl_;
   http_parser response_htp_;
+  size_t group_;
   // index of get_config()->downstream_addrs this object is using
   size_t addr_idx_;
   bool connected_;

src/shrpx_https_upstream.cc

@@ -36,6 +36,7 @@
 #include "shrpx_error.h"
 #include "shrpx_log_config.h"
 #include "shrpx_worker.h"
+#include "shrpx_http2_session.h"
 #include "http2.h"
 #include "util.h"
 #include "template.h"
@@ -218,7 +219,12 @@ void rewrite_request_host_path_from_uri(Downstream *downstream, const char *uri,
     path += '?';
     path.append(uri + fdata.off, fdata.len);
   }
+  if (get_config()->http2_proxy || get_config()->client_proxy) {
     downstream->set_request_path(std::move(path));
+  } else {
+    downstream->set_request_path(
+        http2::rewrite_clean_path(std::begin(path), std::end(path)));
+  }
 
   std::string scheme;
   http2::copy_url_component(scheme, &u, UF_SCHEMA, uri);
@@ -286,6 +292,9 @@ int htp_hdrs_completecb(http_parser *htp) {
         return -1;
       }
 
+      downstream->set_request_path(
+          http2::rewrite_clean_path(std::begin(uri), std::end(uri)));
+
       if (upstream->get_client_handler()->get_ssl()) {
         downstream->set_request_http2_scheme("https");
       } else {
@@ -297,7 +306,7 @@ int htp_hdrs_completecb(http_parser *htp) {
   }
 
   rv = downstream->attach_downstream_connection(
-      upstream->get_client_handler()->get_downstream_connection());
+      upstream->get_client_handler()->get_downstream_connection(downstream));
 
   if (rv != 0) {
     downstream->set_request_state(Downstream::CONNECT_FAIL);
@@ -531,7 +540,8 @@ int HttpsUpstream::on_write() {
   // We need to postpone detachment until all data are sent so that
   // we can notify nghttp2 library all data consumed.
   if (downstream->get_response_state() == Downstream::MSG_COMPLETE) {
-    if (downstream->get_response_connection_close()) {
+    if (downstream->get_response_connection_close() ||
+        downstream->get_request_state() != Downstream::MSG_COMPLETE) {
       // Connection close
       downstream->pop_downstream_connection();
       // dconn was deleted
@@ -598,10 +608,7 @@ int HttpsUpstream::downstream_read(DownstreamConnection *dconn) {
     goto end;
   }
 
-  // Detach downstream connection early so that it could be reused
-  // without hitting server's request timeout.
-  if (downstream->get_response_state() == Downstream::MSG_COMPLETE &&
-      !downstream->get_response_connection_close()) {
+  if (downstream->can_detach_downstream_connection()) {
     // Keep-alive
     downstream->detach_downstream_connection();
   }
@@ -835,12 +842,12 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) {
     if (!get_config()->altsvcs.empty()) {
       hdrs += "Alt-Svc: ";
 
-      for (auto &altsvc : get_config()->altsvcs) {
+      for (const auto &altsvc : get_config()->altsvcs) {
         hdrs += util::percent_encode_token(altsvc.protocol_id);
         hdrs += "=\"";
-        hdrs += util::quote_string(std::string(altsvc.host, altsvc.host_len));
+        hdrs += util::quote_string(altsvc.host);
         hdrs += ":";
-        hdrs += util::utos(altsvc.port);
+        hdrs += altsvc.service;
         hdrs += "\", ";
       }
 
@@ -993,7 +1000,7 @@ int HttpsUpstream::on_downstream_reset(bool no_retry) {
   }
 
   rv = downstream_->attach_downstream_connection(
-      handler_->get_downstream_connection());
+      handler_->get_downstream_connection(downstream_.get()));
   if (rv != 0) {
     goto fail;
   }

src/shrpx_log.h

@@ -123,6 +123,8 @@ enum LogFragmentType {
 };
 
 struct LogFragment {
+  LogFragment(LogFragmentType type, std::unique_ptr<char[]> value = nullptr)
+      : type(type), value(std::move(value)) {}
   LogFragmentType type;
   std::unique_ptr<char[]> value;
 };

src/shrpx_spdy_upstream.cc

@@ -109,19 +109,12 @@ void on_stream_close_callback(spdylay_session *session, int32_t stream_id,
     return;
   }
 
-  downstream->set_request_state(Downstream::STREAM_CLOSED);
-  if (downstream->get_response_state() == Downstream::MSG_COMPLETE) {
-    // At this point, downstream response was read
-    if (!downstream->get_upgraded() &&
-        !downstream->get_response_connection_close()) {
+  if (downstream->can_detach_downstream_connection()) {
     // Keep-alive
     downstream->detach_downstream_connection();
   }
-    upstream->remove_downstream(downstream);
-    // downstrea was deleted
 
-    return;
-  }
+  downstream->set_request_state(Downstream::STREAM_CLOSED);
 
   // At this point, downstream read may be paused.
 
@@ -229,7 +222,12 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type,
     } else {
       downstream->set_request_http2_scheme(scheme->value);
       downstream->set_request_http2_authority(host->value);
+      if (get_config()->http2_proxy || get_config()->client_proxy) {
         downstream->set_request_path(path->value);
+      } else {
+        downstream->set_request_path(http2::rewrite_clean_path(
+            std::begin(path->value), std::end(path->value)));
+      }
     }
 
     if (!(frame->syn_stream.hd.flags & SPDYLAY_CTRL_FLAG_FIN)) {
@@ -271,7 +269,7 @@ void SpdyUpstream::start_downstream(Downstream *downstream) {
 
 void SpdyUpstream::initiate_downstream(Downstream *downstream) {
   int rv = downstream->attach_downstream_connection(
-      handler_->get_downstream_connection());
+      handler_->get_downstream_connection(downstream));
   if (rv != 0) {
     // If downstream connection fails, issue RST_STREAM.
     rst_stream(downstream, SPDYLAY_INTERNAL_ERROR);
@@ -447,8 +445,7 @@ SpdyUpstream::SpdyUpstream(uint16_t version, ClientHandler *handler)
                     : 0,
           !get_config()->http2_proxy),
       handler_(handler), session_(nullptr) {
-  spdylay_session_callbacks callbacks;
-  memset(&callbacks, 0, sizeof(callbacks));
+  spdylay_session_callbacks callbacks{};
   callbacks.send_callback = send_callback;
   callbacks.recv_callback = recv_callback;
   callbacks.on_stream_close_callback = on_stream_close_callback;
@@ -555,16 +552,6 @@ ClientHandler *SpdyUpstream::get_client_handler() const { return handler_; }
 int SpdyUpstream::downstream_read(DownstreamConnection *dconn) {
   auto downstream = dconn->get_downstream();
 
-  if (downstream->get_request_state() == Downstream::STREAM_CLOSED) {
-    // If upstream SPDY stream was closed, we just close downstream,
-    // because there is no consumer now. Downstream connection is also
-    // closed in this case.
-    remove_downstream(downstream);
-    // downstrea was deleted
-
-    return 0;
-  }
-
   if (downstream->get_response_state() == Downstream::MSG_RESET) {
     // The downstream stream was reset (canceled). In this case,
     // RST_STREAM to the upstream and delete downstream connection
@@ -595,10 +582,7 @@ int SpdyUpstream::downstream_read(DownstreamConnection *dconn) {
       }
       return downstream_error(dconn, Downstream::EVENT_ERROR);
     }
-    // Detach downstream connection early so that it could be reused
-    // without hitting server's request timeout.
-    if (downstream->get_response_state() == Downstream::MSG_COMPLETE &&
-        !downstream->get_response_connection_close()) {
+    if (downstream->can_detach_downstream_connection()) {
       // Keep-alive
       downstream->detach_downstream_connection();
     }
@@ -628,14 +612,6 @@ int SpdyUpstream::downstream_eof(DownstreamConnection *dconn) {
   if (LOG_ENABLED(INFO)) {
     DCLOG(INFO, dconn) << "EOF. stream_id=" << downstream->get_stream_id();
   }
-  if (downstream->get_request_state() == Downstream::STREAM_CLOSED) {
-    // If stream was closed already, we don't need to send reply at
-    // the first place. We can delete downstream.
-    remove_downstream(downstream);
-    // downstream was deleted
-
-    return 0;
-  }
 
   // Delete downstream connection. If we don't delete it here, it will
   // be pooled in on_stream_close_callback.
@@ -681,13 +657,6 @@ int SpdyUpstream::downstream_error(DownstreamConnection *dconn, int events) {
     }
   }
 
-  if (downstream->get_request_state() == Downstream::STREAM_CLOSED) {
-    remove_downstream(downstream);
-    // downstream was deleted
-
-    return 0;
-  }
-
   // Delete downstream connection. If we don't delete it here, it will
   // be pooled in on_stream_close_callback.
   downstream->pop_downstream_connection();
@@ -1104,7 +1073,7 @@ int SpdyUpstream::on_downstream_reset(bool no_retry) {
     // downstream connection.
 
     rv = downstream->attach_downstream_connection(
-        handler_->get_downstream_connection());
+        handler_->get_downstream_connection(downstream));
     if (rv != 0) {
       goto fail;
     }

src/shrpx_ssl.cc

@@ -53,6 +53,7 @@
 #include "shrpx_config.h"
 #include "shrpx_worker.h"
 #include "shrpx_downstream_connection_pool.h"
+#include "shrpx_http2_session.h"
 #include "util.h"
 #include "ssl.h"
 #include "template.h"
@@ -85,18 +86,17 @@ int verify_callback(int preverify_ok, X509_STORE_CTX *ctx) {
 }
 } // namespace
 
-std::vector<unsigned char> set_alpn_prefs(const std::vector<char *> &protos) {
+std::vector<unsigned char>
+set_alpn_prefs(const std::vector<std::string> &protos) {
   size_t len = 0;
 
-  for (auto proto : protos) {
-    auto n = strlen(proto);
-
-    if (n > 255) {
-      LOG(FATAL) << "Too long ALPN identifier: " << n;
+  for (const auto &proto : protos) {
+    if (proto.size() > 255) {
+      LOG(FATAL) << "Too long ALPN identifier: " << proto.size();
       DIE();
     }
 
-    len += 1 + n;
+    len += 1 + proto.size();
   }
 
   if (len > (1 << 16) - 1) {
@@ -107,12 +107,10 @@ std::vector<unsigned char> set_alpn_prefs(const std::vector<char *> &protos) {
   auto out = std::vector<unsigned char>(len);
   auto ptr = out.data();
 
-  for (auto proto : protos) {
-    auto proto_len = strlen(proto);
-
-    *ptr++ = proto_len;
-    memcpy(ptr, proto, proto_len);
-    ptr += proto_len;
+  for (const auto &proto : protos) {
+    *ptr++ = proto.size();
+    memcpy(ptr, proto.c_str(), proto.size());
+    ptr += proto.size();
   }
 
   return out;
@@ -190,7 +188,7 @@ int ticket_key_cb(SSL *ssl, unsigned char *key_name, unsigned char *iv,
                   EVP_CIPHER_CTX *ctx, HMAC_CTX *hctx, int enc) {
   auto handler = static_cast<ClientHandler *>(SSL_get_app_data(ssl));
   auto worker = handler->get_worker();
-  const auto &ticket_keys = worker->get_ticket_keys();
+  auto ticket_keys = worker->get_ticket_keys();
 
   if (!ticket_keys) {
     // No ticket keys available.
@@ -212,21 +210,21 @@ int ticket_key_cb(SSL *ssl, unsigned char *key_name, unsigned char *iv,
 
     if (LOG_ENABLED(INFO)) {
       CLOG(INFO, handler) << "encrypt session ticket key: "
-                          << util::format_hex(key.name, 16);
+                          << util::format_hex(key.data.name);
     }
 
-    memcpy(key_name, key.name, sizeof(key.name));
+    memcpy(key_name, key.data.name, sizeof(key.data.name));
 
-    EVP_EncryptInit_ex(ctx, EVP_aes_128_cbc(), nullptr, key.aes_key, iv);
-    HMAC_Init_ex(hctx, key.hmac_key, sizeof(key.hmac_key), EVP_sha256(),
-                 nullptr);
+    EVP_EncryptInit_ex(ctx, get_config()->tls_ticket_cipher, nullptr,
+                       key.data.enc_key, iv);
+    HMAC_Init_ex(hctx, key.data.hmac_key, key.hmac_keylen, key.hmac, nullptr);
     return 1;
   }
 
   size_t i;
   for (i = 0; i < keys.size(); ++i) {
     auto &key = keys[0];
-    if (memcmp(key.name, key_name, sizeof(key.name)) == 0) {
+    if (memcmp(key_name, key.data.name, sizeof(key.data.name)) == 0) {
       break;
     }
   }
@@ -245,8 +243,8 @@ int ticket_key_cb(SSL *ssl, unsigned char *key_name, unsigned char *iv,
   }
 
   auto &key = keys[i];
-  HMAC_Init_ex(hctx, key.hmac_key, sizeof(key.hmac_key), EVP_sha256(), nullptr);
-  EVP_DecryptInit_ex(ctx, EVP_aes_128_cbc(), nullptr, key.aes_key, iv);
+  HMAC_Init_ex(hctx, key.data.hmac_key, key.hmac_keylen, key.hmac, nullptr);
+  EVP_DecryptInit_ex(ctx, key.cipher, nullptr, key.data.enc_key, iv);
 
   return i == 0 ? 1 : 2;
 }
@@ -280,16 +278,14 @@ int alpn_select_proto_cb(SSL *ssl, const unsigned char **out,
   // We assume that get_config()->npn_list contains ALPN protocol
   // identifier sorted by preference order.  So we just break when we
   // found the first overlap.
-  for (auto target_proto_id : get_config()->npn_list) {
-    auto target_proto_len =
-        strlen(reinterpret_cast<const char *>(target_proto_id));
-
+  for (const auto &target_proto_id : get_config()->npn_list) {
     for (auto p = in, end = in + inlen; p < end;) {
       auto proto_id = p + 1;
       auto proto_len = *p;
 
-      if (proto_id + proto_len <= end && target_proto_len == proto_len &&
-          memcmp(target_proto_id, proto_id, proto_len) == 0) {
+      if (proto_id + proto_len <= end &&
+          util::streq(target_proto_id.c_str(), target_proto_id.size(), proto_id,
+                      proto_len)) {
 
         *out = reinterpret_cast<const unsigned char *>(proto_id);
         *outlen = proto_len;
@@ -313,7 +309,7 @@ constexpr long int tls_masks[] = {SSL_OP_NO_TLSv1_2, SSL_OP_NO_TLSv1_1,
                                   SSL_OP_NO_TLSv1};
 } // namespace
 
-long int create_tls_proto_mask(const std::vector<char *> &tls_proto_list) {
+long int create_tls_proto_mask(const std::vector<std::string> &tls_proto_list) {
   long int res = 0;
 
   for (size_t i = 0; i < tls_namelen; ++i) {
@@ -350,6 +346,7 @@ SSL_CTX *create_ssl_context(const char *private_key_file,
   const unsigned char sid_ctx[] = "shrpx";
   SSL_CTX_set_session_id_context(ssl_ctx, sid_ctx, sizeof(sid_ctx) - 1);
   SSL_CTX_set_session_cache_mode(ssl_ctx, SSL_SESS_CACHE_SERVER);
+  SSL_CTX_set_timeout(ssl_ctx, get_config()->tls_session_timeout.count());
 
   const char *ciphers;
   if (get_config()->ciphers) {
@@ -743,7 +740,7 @@ void get_altnames(X509 *cert, std::vector<std::string> &dns_names,
   }
 }
 
-int check_cert(SSL *ssl) {
+int check_cert(SSL *ssl, const DownstreamAddr *addr) {
   auto cert = SSL_get_peer_certificate(ssl);
   if (!cert) {
     LOG(ERROR) << "No certificate found";
@@ -760,9 +757,7 @@ int check_cert(SSL *ssl) {
   std::vector<std::string> dns_names;
   std::vector<std::string> ip_addrs;
   get_altnames(cert, dns_names, ip_addrs, common_name);
-  if (verify_hostname(get_config()->downstream_addrs[0].host.get(),
-                      &get_config()->downstream_addrs[0].addr,
-                      get_config()->downstream_addrs[0].addrlen, dns_names,
+  if (verify_hostname(addr->host.get(), &addr->addr, addr->addrlen, dns_names,
                       ip_addrs, common_name) != 0) {
     LOG(ERROR) << "Certificate verification failed: hostname does not match";
     return -1;
@@ -950,10 +945,10 @@ int cert_lookup_tree_add_cert_from_file(CertLookupTree *lt, SSL_CTX *ssl_ctx,
   return 0;
 }
 
-bool in_proto_list(const std::vector<char *> &protos,
+bool in_proto_list(const std::vector<std::string> &protos,
                    const unsigned char *needle, size_t len) {
-  for (auto proto : protos) {
-    if (strlen(proto) == len && memcmp(proto, needle, len) == 0) {
+  for (auto &proto : protos) {
+    if (util::streq(proto.c_str(), proto.size(), needle, len)) {
       return true;
     }
   }

src/shrpx_ssl.h

@@ -40,6 +40,7 @@ namespace shrpx {
 class ClientHandler;
 class Worker;
 class DownstreamConnectionPool;
+struct DownstreamAddr;
 
 namespace ssl {
 
@@ -68,7 +69,7 @@ ClientHandler *accept_connection(Worker *worker, int fd, sockaddr *addr,
 // Check peer's certificate against first downstream address in
 // Config::downstream_addrs.  We only consider first downstream since
 // we use this function for HTTP/2 downstream link only.
-int check_cert(SSL *ssl);
+int check_cert(SSL *ssl, const DownstreamAddr *addr);
 
 // Retrieves DNS and IP address in subjectAltNames and commonName from
 // the |cert|.
@@ -139,7 +140,7 @@ int cert_lookup_tree_add_cert_from_file(CertLookupTree *lt, SSL_CTX *ssl_ctx,
 
 // Returns true if |needle| which has |len| bytes is included in the
 // protocol list |protos|.
-bool in_proto_list(const std::vector<char *> &protos,
+bool in_proto_list(const std::vector<std::string> &protos,
                    const unsigned char *needle, size_t len);
 
 // Returns true if security requirement for HTTP/2 is fulfilled.
@@ -148,9 +149,10 @@ bool check_http2_requirement(SSL *ssl);
 // Returns SSL/TLS option mask to disable SSL/TLS protocol version not
 // included in |tls_proto_list|.  The returned mask can be directly
 // passed to SSL_CTX_set_options().
-long int create_tls_proto_mask(const std::vector<char *> &tls_proto_list);
+long int create_tls_proto_mask(const std::vector<std::string> &tls_proto_list);
 
-std::vector<unsigned char> set_alpn_prefs(const std::vector<char *> &protos);
+std::vector<unsigned char>
+set_alpn_prefs(const std::vector<std::string> &protos);
 
 // Setups server side SSL_CTX.  This function inspects get_config()
 // and if upstream_no_tls is true, returns nullptr.  Otherwise

src/shrpx_worker.cc

@@ -61,8 +61,11 @@ void mcpool_clear_cb(struct ev_loop *loop, ev_timer *w, int revents) {
 Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
                ssl::CertLookupTree *cert_tree,
                const std::shared_ptr<TicketKeys> &ticket_keys)
-    : next_http2session_(0), loop_(loop), sv_ssl_ctx_(sv_ssl_ctx),
-      cl_ssl_ctx_(cl_ssl_ctx), cert_tree_(cert_tree), ticket_keys_(ticket_keys),
+    : dconn_pool_(get_config()->downstream_addr_groups.size()),
+      worker_stat_(get_config()->downstream_addr_groups.size()),
+      dgrps_(get_config()->downstream_addr_groups.size()), loop_(loop),
+      sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), cert_tree_(cert_tree),
+      ticket_keys_(ticket_keys),
       connect_blocker_(make_unique<ConnectBlocker>(loop_)),
       graceful_shutdown_(false) {
   ev_async_init(&w_, eventcb);
@@ -74,9 +77,17 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
 
   if (get_config()->downstream_proto == PROTO_HTTP2) {
     auto n = get_config()->http2_downstream_connections_per_worker;
-    for (; n > 0; --n) {
-      http2sessions_.push_back(make_unique<Http2Session>(
-          loop_, cl_ssl_ctx, connect_blocker_.get(), this));
+    size_t group = 0;
+    for (auto &dgrp : dgrps_) {
+      auto m = n;
+      if (m == 0) {
+        m = get_config()->downstream_addr_groups[group].addrs.size();
+      }
+      for (size_t idx = 0; idx < m; ++idx) {
+        dgrp.http2sessions.push_back(make_unique<Http2Session>(
+            loop_, cl_ssl_ctx, connect_blocker_.get(), this, group, idx));
+      }
+      ++group;
     }
   }
 }
@@ -161,12 +172,6 @@ void Worker::process_events() {
 
       break;
     }
-    case RENEW_TICKET_KEYS:
-      WLOG(NOTICE, this) << "Renew ticket keys: worker(" << this << ")";
-
-      ticket_keys_ = wev.ticket_keys;
-
-      break;
     case REOPEN_LOG:
       WLOG(NOTICE, this) << "Reopening log files: worker(" << this << ")";
 
@@ -195,11 +200,13 @@ void Worker::process_events() {
 
 ssl::CertLookupTree *Worker::get_cert_lookup_tree() const { return cert_tree_; }
 
-const std::shared_ptr<TicketKeys> &Worker::get_ticket_keys() const {
+std::shared_ptr<TicketKeys> Worker::get_ticket_keys() {
+  std::lock_guard<std::mutex> g(m_);
   return ticket_keys_;
 }
 
 void Worker::set_ticket_keys(std::shared_ptr<TicketKeys> ticket_keys) {
+  std::lock_guard<std::mutex> g(m_);
   ticket_keys_ = std::move(ticket_keys);
 }
 
@@ -207,15 +214,17 @@ WorkerStat *Worker::get_worker_stat() { return &worker_stat_; }
 
 DownstreamConnectionPool *Worker::get_dconn_pool() { return &dconn_pool_; }
 
-Http2Session *Worker::next_http2_session() {
-  if (http2sessions_.empty()) {
+Http2Session *Worker::next_http2_session(size_t group) {
+  auto &dgrp = dgrps_[group];
+  auto &http2sessions = dgrp.http2sessions;
+  if (http2sessions.empty()) {
     return nullptr;
   }
 
-  auto res = http2sessions_[next_http2session_].get();
-  ++next_http2session_;
-  if (next_http2session_ >= http2sessions_.size()) {
-    next_http2session_ = 0;
+  auto res = http2sessions[dgrp.next_http2session].get();
+  ++dgrp.next_http2session;
+  if (dgrp.next_http2session >= http2sessions.size()) {
+    dgrp.next_http2session = 0;
   }
 
   return res;
@@ -239,4 +248,9 @@ bool Worker::get_graceful_shutdown() const { return graceful_shutdown_; }
 
 MemchunkPool *Worker::get_mcpool() { return &mcpool_; }
 
+DownstreamGroup *Worker::get_dgrp(size_t group) {
+  assert(group < dgrps_.size());
+  return &dgrps_[group];
+}
+
 } // namespace shrpx

src/shrpx_worker.h

@@ -54,21 +54,27 @@ namespace ssl {
 class CertLookupTree;
 } // namespace ssl
 
+struct DownstreamGroup {
+  DownstreamGroup() : next_http2session(0), next(0) {}
+
+  std::vector<std::unique_ptr<Http2Session>> http2sessions;
+  // Next index in http2sessions.
+  size_t next_http2session;
+  // Next downstream address index corresponding to
+  // Config::downstream_addr_groups[].
+  size_t next;
+};
+
 struct WorkerStat {
-  WorkerStat() : num_connections(0), next_downstream(0) {}
+  WorkerStat(size_t num_groups) : num_connections(0) {}
 
   size_t num_connections;
-  // Next downstream index in Config::downstream_addrs.  For HTTP/2
-  // downstream connections, this is always 0.  For HTTP/1, this is
-  // used as load balancing.
-  size_t next_downstream;
 };
 
 enum WorkerEventType {
   NEW_CONNECTION = 0x01,
   REOPEN_LOG = 0x02,
   GRACEFUL_SHUTDOWN = 0x03,
-  RENEW_TICKET_KEYS = 0x04,
 };
 
 struct WorkerEvent {
@@ -93,11 +99,15 @@ public:
   void send(const WorkerEvent &event);
 
   ssl::CertLookupTree *get_cert_lookup_tree() const;
-  const std::shared_ptr<TicketKeys> &get_ticket_keys() const;
+
+  // These 2 functions make a lock m_ to get/set ticket keys
+  // atomically.
+  std::shared_ptr<TicketKeys> get_ticket_keys();
   void set_ticket_keys(std::shared_ptr<TicketKeys> ticket_keys);
+
   WorkerStat *get_worker_stat();
   DownstreamConnectionPool *get_dconn_pool();
-  Http2Session *next_http2_session();
+  Http2Session *next_http2_session(size_t group);
   ConnectBlocker *get_connect_blocker() const;
   struct ev_loop *get_loop() const;
   SSL_CTX *get_sv_ssl_ctx() const;
@@ -109,9 +119,9 @@ public:
   MemchunkPool *get_mcpool();
   void schedule_clear_mcpool();
 
+  DownstreamGroup *get_dgrp(size_t group);
+
 private:
-  std::vector<std::unique_ptr<Http2Session>> http2sessions_;
-  size_t next_http2session_;
 #ifndef NOTHREADS
   std::future<void> fut_;
 #endif // NOTHREADS
@@ -122,6 +132,7 @@ private:
   MemchunkPool mcpool_;
   DownstreamConnectionPool dconn_pool_;
   WorkerStat worker_stat_;
+  std::vector<DownstreamGroup> dgrps_;
   struct ev_loop *loop_;
 
   // Following fields are shared across threads if

src/timegm.c

@@ -34,7 +34,7 @@ static int count_leap_year(int y) {
 }
 
 /* Based on the algorithm of Python 2.7 calendar.timegm. */
-time_t timegm(struct tm *tm) {
+time_t nghttp2_timegm(struct tm *tm) {
   int days;
   int num_leap_year;
   int64_t t;
@@ -53,3 +53,36 @@ time_t timegm(struct tm *tm) {
 
   return (time_t)t;
 }
+
+/* Returns nonzero if the |y| is the leap year. The |y| is the year,
+   including century (e.g., 2012) */
+static int is_leap_year(int y) {
+  return y % 4 == 0 && (y % 100 != 0 || y % 400 == 0);
+}
+
+/* The number of days before ith month begins */
+static int daysum[] = {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334};
+
+time_t nghttp2_timegm_without_yday(struct tm *tm) {
+  int days;
+  int num_leap_year;
+  int64_t t;
+  if (tm->tm_mon > 11) {
+    return -1;
+  }
+  num_leap_year = count_leap_year(tm->tm_year + 1900) - count_leap_year(1970);
+  days = (tm->tm_year - 70) * 365 + num_leap_year + daysum[tm->tm_mon] +
+         tm->tm_mday - 1;
+  if (tm->tm_mon >= 2 && is_leap_year(tm->tm_year + 1900)) {
+    ++days;
+  }
+  t = ((int64_t)days * 24 + tm->tm_hour) * 3600 + tm->tm_min * 60 + tm->tm_sec;
+
+#if SIZEOF_TIME_T == 4
+  if (t < INT32_MIN || t > INT32_MAX) {
+    return -1;
+  }
+#endif /* SIZEOF_TIME_T == 4 */
+
+  return t;
+}

src/timegm.h

@@ -37,7 +37,12 @@ extern "C" {
 #include <time.h>
 #endif // HAVE_TIME_H
 
-time_t timegm(struct tm *tm);
+time_t nghttp2_timegm(struct tm *tm);
+
+/* Just like nghttp2_timegm, but without using tm->tm_yday.  This is
+   useful if we use tm from strptime, since some platforms do not
+   calculate tm_yday with that call. */
+time_t nghttp2_timegm_without_yday(struct tm *tm);
 
 #ifdef __cplusplus
 }

src/util.cc

@@ -278,7 +278,7 @@ std::string common_log_date(time_t t) {
 #ifdef HAVE_STRUCT_TM_TM_GMTOFF
   auto gmtoff = tms.tm_gmtoff;
 #else  // !HAVE_STRUCT_TM_TM_GMTOFF
-  auto gmtoff = timegm(&tms) - t;
+  auto gmtoff = nghttp2_timegm(&tms) - t;
 #endif // !HAVE_STRUCT_TM_TM_GMTOFF
   if (gmtoff >= 0) {
     *p++ = '+';
@@ -326,7 +326,7 @@ std::string iso8601_date(int64_t ms) {
 #ifdef HAVE_STRUCT_TM_TM_GMTOFF
   auto gmtoff = tms.tm_gmtoff;
 #else  // !HAVE_STRUCT_TM_TM_GMTOFF
-  auto gmtoff = timegm(&tms) - sec;
+  auto gmtoff = nghttp2_timegm(&tms) - sec;
 #endif // !HAVE_STRUCT_TM_TM_GMTOFF
   if (gmtoff == 0) {
     *p++ = 'Z';
@@ -348,13 +348,12 @@ std::string iso8601_date(int64_t ms) {
 }
 
 time_t parse_http_date(const std::string &s) {
-  tm tm;
-  memset(&tm, 0, sizeof(tm));
+  tm tm{};
   char *r = strptime(s.c_str(), "%a, %d %b %Y %H:%M:%S GMT", &tm);
   if (r == 0) {
     return 0;
   }
-  return timegm(&tm);
+  return nghttp2_timegm_without_yday(&tm);
 }
 
 namespace {
@@ -637,9 +636,8 @@ void write_uri_field(std::ostream &o, const char *uri, const http_parser_url &u,
 }
 
 bool numeric_host(const char *hostname) {
-  struct addrinfo hints;
   struct addrinfo *res;
-  memset(&hints, 0, sizeof(hints));
+  struct addrinfo hints {};
   hints.ai_family = AF_UNSPEC;
   hints.ai_flags = AI_NUMERICHOST;
   if (getaddrinfo(hostname, nullptr, &hints, &res)) {

src/util.h

@@ -212,6 +212,10 @@ std::string quote_string(const std::string &target);
 
 std::string format_hex(const unsigned char *s, size_t len);
 
+template <size_t N> std::string format_hex(const unsigned char (&s)[N]) {
+  return format_hex(s, N);
+}
+
 std::string http_date(time_t t);
 
 // Returns given time |t| from epoch in Common Log format (e.g.,
@@ -345,6 +349,10 @@ inline bool strieq(const std::string &a, const std::string &b) {
 
 bool strieq(const char *a, const char *b);
 
+inline bool strieq(const char *a, const std::string &b) {
+  return strieq(a, b.c_str(), b.size());
+}
+
 template <typename InputIt, size_t N>
 bool strieq_l(const char (&a)[N], InputIt b, size_t blen) {
   return strieq(a, N - 1, b, blen);
@@ -386,9 +394,13 @@ bool streq_l(const char (&a)[N], InputIt b, size_t blen) {
 
 bool strifind(const char *a, const char *b);
 
+template <typename InputIt> void inp_strlower(InputIt first, InputIt last) {
+  std::transform(first, last, first, lowcase);
+}
+
 // Lowercase |s| in place.
 inline void inp_strlower(std::string &s) {
-  std::transform(std::begin(s), std::end(s), std::begin(s), lowcase);
+  inp_strlower(std::begin(s), std::end(s));
 }
 
 // Returns string representation of |n| with 2 fractional digits.

tests/main.c

@@ -171,6 +171,8 @@ int main(int argc _U_, char *argv[] _U_) {
                    test_nghttp2_submit_settings) ||
       !CU_add_test(pSuite, "session_submit_settings_update_local_window_size",
                    test_nghttp2_submit_settings_update_local_window_size) ||
+      !CU_add_test(pSuite, "session_submit_settings_multiple_times",
+                   test_nghttp2_submit_settings_multiple_times) ||
       !CU_add_test(pSuite, "session_submit_push_promise",
                    test_nghttp2_submit_push_promise) ||
       !CU_add_test(pSuite, "submit_window_update",
@@ -326,6 +328,10 @@ int main(int argc _U_, char *argv[] _U_) {
                    test_nghttp2_hd_inflate_clearall_inc) ||
       !CU_add_test(pSuite, "hd_inflate_zero_length_huffman",
                    test_nghttp2_hd_inflate_zero_length_huffman) ||
+      !CU_add_test(pSuite, "hd_inflate_expect_table_size_update",
+                   test_nghttp2_hd_inflate_expect_table_size_update) ||
+      !CU_add_test(pSuite, "hd_inflate_unexpected_table_size_update",
+                   test_nghttp2_hd_inflate_unexpected_table_size_update) ||
       !CU_add_test(pSuite, "hd_ringbuf_reserve",
                    test_nghttp2_hd_ringbuf_reserve) ||
       !CU_add_test(pSuite, "hd_change_table_size",

tests/nghttp2_hd_test.c

@@ -540,6 +540,54 @@ void test_nghttp2_hd_inflate_zero_length_huffman(void) {
   nghttp2_hd_inflate_free(&inflater);
 }
 
+void test_nghttp2_hd_inflate_expect_table_size_update(void) {
+  nghttp2_hd_inflater inflater;
+  nghttp2_bufs bufs;
+  nghttp2_mem *mem;
+  /* Indexed Header: :method: GET */
+  uint8_t data[] = {0x82};
+  nva_out out;
+
+  mem = nghttp2_mem_default();
+  frame_pack_bufs_init(&bufs);
+  nva_out_init(&out);
+
+  nghttp2_bufs_add(&bufs, data, sizeof(data));
+  nghttp2_hd_inflate_init(&inflater, mem);
+  /* This will make inflater require table size update in the next
+     inflation. */
+  nghttp2_hd_inflate_change_table_size(&inflater, 4096);
+  CU_ASSERT(NGHTTP2_ERR_HEADER_COMP ==
+            inflate_hd(&inflater, &out, &bufs, 0, mem));
+
+  nva_out_reset(&out, mem);
+  nghttp2_bufs_free(&bufs);
+  nghttp2_hd_inflate_free(&inflater);
+}
+
+void test_nghttp2_hd_inflate_unexpected_table_size_update(void) {
+  nghttp2_hd_inflater inflater;
+  nghttp2_bufs bufs;
+  nghttp2_mem *mem;
+  /* Indexed Header: :method: GET, followed by table size update.
+     This violates RFC 7541. */
+  uint8_t data[] = {0x82, 0x20};
+  nva_out out;
+
+  mem = nghttp2_mem_default();
+  frame_pack_bufs_init(&bufs);
+  nva_out_init(&out);
+
+  nghttp2_bufs_add(&bufs, data, sizeof(data));
+  nghttp2_hd_inflate_init(&inflater, mem);
+  CU_ASSERT(NGHTTP2_ERR_HEADER_COMP ==
+            inflate_hd(&inflater, &out, &bufs, 0, mem));
+
+  nva_out_reset(&out, mem);
+  nghttp2_bufs_free(&bufs);
+  nghttp2_hd_inflate_free(&inflater);
+}
+
 void test_nghttp2_hd_ringbuf_reserve(void) {
   nghttp2_hd_deflater deflater;
   nghttp2_hd_inflater inflater;

tests/nghttp2_hd_test.h

@@ -35,6 +35,8 @@ void test_nghttp2_hd_inflate_newname_noinc(void);
 void test_nghttp2_hd_inflate_newname_inc(void);
 void test_nghttp2_hd_inflate_clearall_inc(void);
 void test_nghttp2_hd_inflate_zero_length_huffman(void);
+void test_nghttp2_hd_inflate_expect_table_size_update(void);
+void test_nghttp2_hd_inflate_unexpected_table_size_update(void);
 void test_nghttp2_hd_ringbuf_reserve(void);
 void test_nghttp2_hd_change_table_size(void);
 void test_nghttp2_hd_deflate_inflate(void);

tests/nghttp2_session_test.c

@@ -2300,18 +2300,11 @@ void test_nghttp2_session_on_settings_received(void) {
   nghttp2_session_server_new(&session, &callbacks, NULL);
   nghttp2_frame_settings_init(&frame.settings, NGHTTP2_FLAG_ACK, dup_iv(iv, 1),
                               1);
-  /* Specify inflight_iv deliberately */
-  session->inflight_iv = frame.settings.iv;
-  session->inflight_niv = frame.settings.niv;
-
   CU_ASSERT(0 == nghttp2_session_on_settings_received(session, &frame, 0));
   item = nghttp2_session_get_next_ob_item(session);
   CU_ASSERT(item != NULL);
   CU_ASSERT(NGHTTP2_GOAWAY == item->frame.hd.type);
 
-  session->inflight_iv = NULL;
-  session->inflight_niv = -1;
-
   nghttp2_frame_settings_free(&frame.settings, mem);
   nghttp2_session_del(session);
 
@@ -4041,8 +4034,8 @@ void test_nghttp2_submit_settings(void) {
   CU_ASSERT(16 * 1024 == session->local_settings.initial_window_size);
   CU_ASSERT(0 == session->hd_inflater.ctx.hd_table_bufsize_max);
   CU_ASSERT(50 == session->local_settings.max_concurrent_streams);
-  CU_ASSERT(NGHTTP2_INITIAL_MAX_CONCURRENT_STREAMS ==
-            session->pending_local_max_concurrent_stream);
+  /* We just keep the last seen value */
+  CU_ASSERT(50 == session->pending_local_max_concurrent_stream);
 
   nghttp2_session_del(session);
 }
@@ -4113,6 +4106,83 @@ void test_nghttp2_submit_settings_update_local_window_size(void) {
   nghttp2_frame_settings_free(&ack_frame.settings, mem);
 }
 
+void test_nghttp2_submit_settings_multiple_times(void) {
+  nghttp2_session *session;
+  nghttp2_session_callbacks callbacks;
+  nghttp2_settings_entry iv[4];
+  nghttp2_frame frame;
+  nghttp2_inflight_settings *inflight_settings;
+
+  memset(&callbacks, 0, sizeof(callbacks));
+  callbacks.send_callback = null_send_callback;
+
+  nghttp2_session_client_new(&session, &callbacks, NULL);
+
+  /* first SETTINGS */
+  iv[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS;
+  iv[0].value = 100;
+
+  iv[1].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH;
+  iv[1].value = 0;
+
+  CU_ASSERT(0 == nghttp2_submit_settings(session, NGHTTP2_FLAG_NONE, iv, 2));
+
+  inflight_settings = session->inflight_settings_head;
+
+  CU_ASSERT(NULL != inflight_settings);
+  CU_ASSERT(NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS ==
+            inflight_settings->iv[0].settings_id);
+  CU_ASSERT(100 == inflight_settings->iv[0].value);
+  CU_ASSERT(2 == inflight_settings->niv);
+  CU_ASSERT(NULL == inflight_settings->next);
+
+  CU_ASSERT(100 == session->pending_local_max_concurrent_stream);
+  CU_ASSERT(0 == session->pending_enable_push);
+
+  /* second SETTINGS */
+  iv[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS;
+  iv[0].value = 99;
+
+  CU_ASSERT(0 == nghttp2_submit_settings(session, NGHTTP2_FLAG_NONE, iv, 1));
+
+  inflight_settings = session->inflight_settings_head->next;
+
+  CU_ASSERT(NULL != inflight_settings);
+  CU_ASSERT(NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS ==
+            inflight_settings->iv[0].settings_id);
+  CU_ASSERT(99 == inflight_settings->iv[0].value);
+  CU_ASSERT(1 == inflight_settings->niv);
+  CU_ASSERT(NULL == inflight_settings->next);
+
+  CU_ASSERT(99 == session->pending_local_max_concurrent_stream);
+  CU_ASSERT(0 == session->pending_enable_push);
+
+  nghttp2_frame_settings_init(&frame.settings, NGHTTP2_FLAG_ACK, NULL, 0);
+
+  /* receive SETTINGS ACK */
+  CU_ASSERT(0 == nghttp2_session_on_settings_received(session, &frame, 0));
+
+  inflight_settings = session->inflight_settings_head;
+
+  /* first inflight SETTINGS was removed */
+  CU_ASSERT(NULL != inflight_settings);
+  CU_ASSERT(NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS ==
+            inflight_settings->iv[0].settings_id);
+  CU_ASSERT(99 == inflight_settings->iv[0].value);
+  CU_ASSERT(1 == inflight_settings->niv);
+  CU_ASSERT(NULL == inflight_settings->next);
+
+  CU_ASSERT(100 == session->local_settings.max_concurrent_streams);
+
+  /* receive SETTINGS ACK again */
+  CU_ASSERT(0 == nghttp2_session_on_settings_received(session, &frame, 0));
+
+  CU_ASSERT(NULL == session->inflight_settings_head);
+  CU_ASSERT(99 == session->local_settings.max_concurrent_streams);
+
+  nghttp2_session_del(session);
+}
+
 void test_nghttp2_submit_push_promise(void) {
   nghttp2_session *session;
   nghttp2_session_callbacks callbacks;

tests/nghttp2_session_test.h

@@ -79,6 +79,7 @@ void test_nghttp2_submit_headers_continuation(void);
 void test_nghttp2_submit_priority(void);
 void test_nghttp2_submit_settings(void);
 void test_nghttp2_submit_settings_update_local_window_size(void);
+void test_nghttp2_submit_settings_multiple_times(void);
 void test_nghttp2_submit_push_promise(void);
 void test_nghttp2_submit_window_update(void);
 void test_nghttp2_submit_window_update_local_window_size(void);