walkero
/
nghttp2
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Merge pull request #2 from tatsuhiro-t/master 

pulling changes from master
This commit is contained in:
Nora Shoemaker 2015-07-15 11:59:50 -07:00
parent 1940413eb3 1ce3a000a1
commit 6cf772da6a
52 changed files with 2214 additions and 650 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Makefile.am
View File

@ -23,6 +23,11 @@
SUBDIRS = lib third-party src examples python tests integration-tests \
doc contrib script
# Now with python setuptools, make uninstall will leave many files we
# cannot easily remove (e.g., easy-install.pth). Disable it for
# distcheck rule.
AM_DISTCHECK_CONFIGURE_FLAGS = --disable-python-bindings
ACLOCAL_AMFLAGS = -I m4
dist_doc_DATA = README.rst

4
configure.ac
View File

@ -25,7 +25,7 @@ dnl Do not change user variables!
dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html
AC_PREREQ(2.61)
AC_INIT([nghttp2], [1.0.6-DEV], [t-tujikawa@users.sourceforge.net])
AC_INIT([nghttp2], [1.1.2-DEV], [t-tujikawa@users.sourceforge.net])
AC_USE_SYSTEM_EXTENSIONS
LT_PREREQ([2.2.6])
@ -48,7 +48,7 @@ AC_CONFIG_HEADERS([config.h])
dnl See versioning rule:
dnl http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html
AC_SUBST(LT_CURRENT, 14)
AC_SUBST(LT_REVISION, 5)
AC_SUBST(LT_REVISION, 6)
AC_SUBST(LT_AGE, 0)
major=`echo $PACKAGE_VERSION |cut -d. -f1 | sed -e "s/[^0-9]//g"`

2
doc/bash_completion/h2load
View File

@ -8,7 +8,7 @@ _h2load()
_get_comp_words_by_ref cur prev
case $cur in
-*)
COMPREPLY=( $( compgen -W '--threads --connection-window-bits --input-file --help --requests --data --verbose --version --window-bits --clients --no-tls-proto --header --max-concurrent-streams ' -- "$cur" ) )
COMPREPLY=( $( compgen -W '--threads --connection-window-bits --input-file --help --requests --data --verbose --ciphers --window-bits --clients --no-tls-proto --version --header --max-concurrent-streams ' -- "$cur" ) )
;;
*)
_filedir

2
doc/bash_completion/nghttp
View File

@ -8,7 +8,7 @@ _nghttp()
_get_comp_words_by_ref cur prev
case $cur in
-*)
COMPREPLY=( $( compgen -W '--no-push --verbose --no-dep --get-assets --har --header-table-size --multiply --padding --hexdump --continuation --connection-window-bits --peer-max-concurrent-streams --timeout --data --no-content-length --version --color --cert --upgrade --remote-name --trailer --weight --help --key --null-out --window-bits --stat --header ' -- "$cur" ) )
COMPREPLY=( $( compgen -W '--no-push --verbose --no-dep --get-assets --har --header-table-size --multiply --padding --hexdump --max-concurrent-streams --continuation --connection-window-bits --peer-max-concurrent-streams --timeout --data --no-content-length --version --color --cert --upgrade --remote-name --trailer --weight --help --key --null-out --window-bits --stat --header ' -- "$cur" ) )
;;
*)
_filedir

2
doc/bash_completion/nghttpx
View File

@ -8,7 +8,7 @@ _nghttpx()
_get_comp_words_by_ref cur prev
case $cur in
-*)
COMPREPLY=( $( compgen -W '--worker-read-rate --frontend-no-tls --frontend-http2-dump-response-header --backend-http1-connections-per-frontend --tls-ticket-key-file --verify-client-cacert --backend-request-buffer --backend-http2-connection-window-bits --conf --worker-write-burst --npn-list --fetch-ocsp-response-file --stream-read-timeout --accesslog-syslog --frontend-http2-read-timeout --listener-disable-timeout --frontend-http2-connection-window-bits --ciphers --strip-incoming-x-forwarded-for --daemon --backend-keep-alive-timeout --backend-http-proxy-uri --backend-http1-connections-per-host --rlimit-nofile --no-via --ocsp-update-interval --backend-write-timeout --client --http2-no-cookie-crumbling --worker-read-burst --client-proxy --http2-bridge --accesslog-format --errorlog-syslog --errorlog-file --http2-max-concurrent-streams --frontend-write-timeout --read-burst --backend-ipv4 --backend-ipv6 --backend --insecure --log-level --tls-proto-list --backend-http2-connections-per-worker --dh-param-file --worker-frontend-connections --header-field-buffer --no-server-push --no-location-rewrite --no-ocsp --backend-response-buffer --workers --frontend-http2-window-bits --no-host-rewrite --worker-write-rate --add-request-header --backend-tls-sni-field --subcert --help --frontend-frame-debug --pid-file --frontend-http2-dump-request-header --private-key-passwd-file --write-rate --altsvc --user --add-x-forwarded-for --syslog-facility --frontend-read-timeout --backlog --write-burst --backend-http2-window-bits --padding --stream-write-timeout --cacert --version --verify-client --backend-read-timeout --frontend --accesslog-file --http2-proxy --max-header-fields --backend-no-tls --client-private-key-file --client-cert-file --add-response-header --read-rate ' -- "$cur" ) )
COMPREPLY=( $( compgen -W '--worker-read-rate --frontend-no-tls --frontend-http2-dump-response-header --backend-http1-connections-per-frontend --tls-ticket-key-file --verify-client-cacert --include --backend-request-buffer --backend-http2-connection-window-bits --conf --worker-write-burst --npn-list --fetch-ocsp-response-file --stream-read-timeout --accesslog-syslog --frontend-http2-read-timeout --listener-disable-timeout --frontend-http2-connection-window-bits --ciphers --strip-incoming-x-forwarded-for --daemon --backend-keep-alive-timeout --backend-http-proxy-uri --backend-http1-connections-per-host --rlimit-nofile --no-via --ocsp-update-interval --backend-write-timeout --client --http2-no-cookie-crumbling --worker-read-burst --client-proxy --http2-bridge --accesslog-format --errorlog-syslog --errorlog-file --http2-max-concurrent-streams --frontend-write-timeout --read-burst --backend-ipv4 --backend-ipv6 --backend --insecure --log-level --tls-proto-list --backend-http2-connections-per-worker --dh-param-file --worker-frontend-connections --header-field-buffer --no-server-push --no-location-rewrite --no-ocsp --backend-response-buffer --workers --frontend-http2-window-bits --no-host-rewrite --worker-write-rate --add-request-header --backend-tls-sni-field --subcert --help --frontend-frame-debug --pid-file --frontend-http2-dump-request-header --private-key-passwd-file --write-rate --altsvc --user --add-x-forwarded-for --syslog-facility --frontend-read-timeout --backlog --write-burst --backend-http2-window-bits --padding --stream-write-timeout --cacert --version --verify-client --backend-read-timeout --frontend --accesslog-file --http2-proxy --max-header-fields --backend-no-tls --client-private-key-file --client-cert-file --add-response-header --read-rate ' -- "$cur" ) )
;;
*)
_filedir

8
doc/h2load.1
View File

@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
.TH "H2LOAD" "1" "June 27, 2015" "1.0.5" "nghttp2"
.TH "H2LOAD" "1" "July 15, 2015" "1.1.1" "nghttp2"
.SH NAME
h2load \- HTTP/2 benchmarking tool
.
@ -113,6 +113,12 @@ Add/Override a header to the requests.
.UNINDENT
.INDENT 0.0
.TP
.B \-\-ciphers=<SUITE>
Set allowed cipher list. The format of the string is
described in OpenSSL ciphers(1).
.UNINDENT
.INDENT 0.0
.TP
.B \-p, \-\-no\-tls\-proto=<PROTOID>
Specify ALPN identifier of the protocol to be used when
accessing http URI without SSL/TLS.

5
doc/h2load.1.rst
View File

@ -84,6 +84,11 @@ OPTIONS
Add/Override a header to the requests.
.. option:: --ciphers=<SUITE>
Set allowed cipher list. The format of the string is
described in OpenSSL ciphers(1).
.. option:: -p, --no-tls-proto=<PROTOID>
Specify ALPN identifier of the protocol to be used when

8
doc/nghttp.1
View File

@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
.TH "NGHTTP" "1" "June 27, 2015" "1.0.5" "nghttp2"
.TH "NGHTTP" "1" "July 15, 2015" "1.1.1" "nghttp2"
.SH NAME
nghttp \- HTTP/2 experimental client
.
@ -205,6 +205,12 @@ Disable server push.
.UNINDENT
.INDENT 0.0
.TP
.B \-\-max\-concurrent\-streams=<N>
The number of concurrent pushed streams this client
accepts.
.UNINDENT
.INDENT 0.0
.TP
.B \-\-version
Display version information and exit.
.UNINDENT

5
doc/nghttp.1.rst
View File

@ -158,6 +158,11 @@ OPTIONS
Disable server push.
.. option:: --max-concurrent-streams=<N>
The number of concurrent pushed streams this client
accepts.
.. option:: --version
Display version information and exit.

2
doc/nghttpd.1
View File

@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
.TH "NGHTTPD" "1" "June 27, 2015" "1.0.5" "nghttp2"
.TH "NGHTTPD" "1" "July 15, 2015" "1.1.1" "nghttp2"
.SH NAME
nghttpd \- HTTP/2 experimental server
.

91
doc/nghttpx.1
View File

@ -1,6 +1,6 @@
.\" Man page generated from reStructuredText.
.
.TH "NGHTTPX" "1" "June 27, 2015" "1.0.5" "nghttp2"
.TH "NGHTTPX" "1" "July 15, 2015" "1.1.1" "nghttp2"
.SH NAME
nghttpx \- HTTP/2 experimental proxy
.
@ -55,17 +55,69 @@ The options are categorized into several groups.
.SS Connections
.INDENT 0.0
.TP
.B \-b, \-\-backend=<HOST,PORT>
.B \-b, \-\-backend=(<HOST>,<PORT>|unix:<PATH>)[;<PATTERN>[:...]]
Set backend host and port. The multiple backend
addresses are accepted by repeating this option. UNIX
domain socket can be specified by prefixing path name
with "unix:" (e.g., unix:/var/run/backend.sock)
with "unix:" (e.g., unix:/var/run/backend.sock).
.sp
Optionally, if <PATTERN>s are given, the backend address
is only used if request matches the pattern. If \fI\%\-s\fP or
\fI\%\-p\fP is used, <PATTERN>s are ignored. The pattern
matching is closely designed to ServeMux in net/http
package of Go programming language. <PATTERN> consists
of path, host + path or just host. The path must start
with "\fI/\fP". If it ends with "\fI/\fP", it matches all request
path in its subtree. To deal with the request to the
directory without trailing slash, the path which ends
with "\fI/\fP" also matches the request path which only lacks
trailing \(aq\fI/\fP\(aq (e.g., path "\fI/foo/\fP" matches request path
"\fI/foo\fP"). If it does not end with "\fI/\fP", it performs exact
match against the request path. If host is given, it
performs exact match against the request host. If host
alone is given, "\fI/\fP" is appended to it, so that it
matches all request paths under the host (e.g.,
specifying "nghttp2.org" equals to "nghttp2.org/").
.sp
Patterns with host take precedence over patterns with
just path. Then, longer patterns take precedence over
shorter ones, breaking a tie by the order of the
appearance in the configuration.
.sp
If <PATTERN> is omitted, "\fI/\fP" is used as pattern, which
matches all request paths (catch\-all pattern). The
catch\-all backend must be given.
.sp
When doing a match, nghttpx made some normalization to
pattern, request host and path. For host part, they are
converted to lower case. For path part, percent\-encoded
unreserved characters defined in RFC 3986 are decoded,
and any dot\-segments (".." and ".") are resolved and
removed.
.sp
For example, \fI\%\-b\fP\(aq127.0.0.1,8080;nghttp2.org/httpbin/\(aq
matches the request host "nghttp2.org" and the request
path "\fI/httpbin/get\fP", but does not match the request host
"nghttp2.org" and the request path "\fI/index.html\fP".
.sp
The multiple <PATTERN>s can be specified, delimiting
them by ":". Specifying
\fI\%\-b\fP\(aq127.0.0.1,8080;nghttp2.org:www.nghttp2.org\(aq has the
same effect to specify \fI\%\-b\fP\(aq127.0.0.1,8080;nghttp2.org\(aq
and \fI\%\-b\fP\(aq127.0.0.1,8080;www.nghttp2.org\(aq.
.sp
The backend addresses sharing same <PATTERN> are grouped
together forming load balancing group.
.sp
Since ";" and ":" are used as delimiter, <PATTERN> must
not contain these characters. Since ";" has special
meaning in shell, the option value must be quoted.
.sp
Default: \fB127.0.0.1,80\fP
.UNINDENT
.INDENT 0.0
.TP
.B \-f, \-\-frontend=<HOST,PORT>
.B \-f, \-\-frontend=(<HOST>,<PORT>|unix:<PATH>)
Set frontend host and port. If <HOST> is \(aq*\(aq, it
assumes all addresses including both IPv4 and IPv6.
UNIX domain socket can be specified by prefixing path
@ -195,9 +247,14 @@ Default: \fB0\fP
.INDENT 0.0
.TP
.B \-\-backend\-http2\-connections\-per\-worker=<N>
Set maximum number of HTTP/2 connections per worker.
The default value is 0, which means the number of
backend addresses specified by \fI\%\-b\fP option.
Set maximum number of backend HTTP/2 physical
connections per worker. If pattern is used in \fI\%\-b\fP
option, this limit is applied to each pattern group (in
other words, each pattern group can have maximum <N>
HTTP/2 connections). The default value is 0, which
means that the value is adjusted to the number of
backend addresses. If pattern is used, this adjustment
is done for each pattern group.
.UNINDENT
.INDENT 0.0
.TP
@ -626,8 +683,20 @@ $pid: PID of the running process.
$alpn: ALPN identifier of the protocol which generates
the response. For HTTP/1, ALPN is always http/1.1,
regardless of minor version.
.IP \(bu 2
$ssl_cipher: cipher used for SSL/TLS connection.
.IP \(bu 2
$ssl_protocol: protocol for SSL/TLS connection.
.IP \(bu 2
$ssl_session_id: session ID for SSL/TLS connection.
.IP \(bu 2
$ssl_session_reused: "r" if SSL/TLS session was
reused. Otherwise, "."
.UNINDENT
.sp
The variable can be enclosed by "{" and "}" for
disambiguation (e.g., ${remote_addr}).
.sp
Default: \fB$remote_addr \- \- [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"\fP
.UNINDENT
.INDENT 0.0
@ -786,6 +855,14 @@ Default: \fB/etc/nghttpx/nghttpx.conf\fP
.UNINDENT
.INDENT 0.0
.TP
.B \-\-include=<PATH>
Load additional configurations from <PATH>. File <PATH>
is read when configuration parser encountered this
option. This option can be used multiple times, or even
recursively.
.UNINDENT
.INDENT 0.0
.TP
.B \-v, \-\-version
Print version and exit.
.UNINDENT

85
doc/nghttpx.1.rst
View File

@ -37,16 +37,69 @@ The options are categorized into several groups.
Connections
~~~~~~~~~~~
.. option:: -b, --backend=<HOST,PORT>
.. option:: -b, --backend=(<HOST>,<PORT>|unix:<PATH>)[;<PATTERN>[:...]]
Set backend host and port. The multiple backend
addresses are accepted by repeating this option. UNIX
domain socket can be specified by prefixing path name
with "unix:" (e.g., unix:/var/run/backend.sock)
with "unix:" (e.g., unix:/var/run/backend.sock).
Optionally, if <PATTERN>s are given, the backend address
is only used if request matches the pattern. If :option:`-s` or
:option:`-p` is used, <PATTERN>s are ignored. The pattern
matching is closely designed to ServeMux in net/http
package of Go programming language. <PATTERN> consists
of path, host + path or just host. The path must start
with "*/*". If it ends with "*/*", it matches all request
path in its subtree. To deal with the request to the
directory without trailing slash, the path which ends
with "*/*" also matches the request path which only lacks
trailing '*/*' (e.g., path "*/foo/*" matches request path
"*/foo*"). If it does not end with "*/*", it performs exact
match against the request path. If host is given, it
performs exact match against the request host. If host
alone is given, "*/*" is appended to it, so that it
matches all request paths under the host (e.g.,
specifying "nghttp2.org" equals to "nghttp2.org/").
Patterns with host take precedence over patterns with
just path. Then, longer patterns take precedence over
shorter ones, breaking a tie by the order of the
appearance in the configuration.
If <PATTERN> is omitted, "*/*" is used as pattern, which
matches all request paths (catch-all pattern). The
catch-all backend must be given.
When doing a match, nghttpx made some normalization to
pattern, request host and path. For host part, they are
converted to lower case. For path part, percent-encoded
unreserved characters defined in RFC 3986 are decoded,
and any dot-segments (".." and ".") are resolved and
removed.
For example, :option:`-b`\'127.0.0.1,8080;nghttp2.org/httpbin/'
matches the request host "nghttp2.org" and the request
path "*/httpbin/get*", but does not match the request host
"nghttp2.org" and the request path "*/index.html*".
The multiple <PATTERN>s can be specified, delimiting
them by ":". Specifying
:option:`-b`\'127.0.0.1,8080;nghttp2.org:www.nghttp2.org' has the
same effect to specify :option:`-b`\'127.0.0.1,8080;nghttp2.org'
and :option:`-b`\'127.0.0.1,8080;www.nghttp2.org'.
The backend addresses sharing same <PATTERN> are grouped
together forming load balancing group.
Since ";" and ":" are used as delimiter, <PATTERN> must
not contain these characters. Since ";" has special
meaning in shell, the option value must be quoted.
Default: ``127.0.0.1,80``
.. option:: -f, --frontend=<HOST,PORT>
.. option:: -f, --frontend=(<HOST>,<PORT>|unix:<PATH>)
Set frontend host and port. If <HOST> is '\*', it
assumes all addresses including both IPv4 and IPv6.
@ -165,9 +218,14 @@ Performance
.. option:: --backend-http2-connections-per-worker=<N>
Set maximum number of HTTP/2 connections per worker.
The default value is 0, which means the number of
backend addresses specified by :option:`-b` option.
Set maximum number of backend HTTP/2 physical
connections per worker. If pattern is used in :option:`-b`
option, this limit is applied to each pattern group (in
other words, each pattern group can have maximum <N>
HTTP/2 connections). The default value is 0, which
means that the value is adjusted to the number of
backend addresses. If pattern is used, this adjustment
is done for each pattern group.
.. option:: --backend-http1-connections-per-host=<N>
@ -550,6 +608,14 @@ Logging
* $alpn: ALPN identifier of the protocol which generates
the response. For HTTP/1, ALPN is always http/1.1,
regardless of minor version.
* $ssl_cipher: cipher used for SSL/TLS connection.
* $ssl_protocol: protocol for SSL/TLS connection.
* $ssl_session_id: session ID for SSL/TLS connection.
* $ssl_session_reused: "r" if SSL/TLS session was
reused. Otherwise, "."
The variable can be enclosed by "{" and "}" for
disambiguation (e.g., ${remote_addr}).
Default: ``$remote_addr - - [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"``
@ -700,6 +766,13 @@ Misc
Default: ``/etc/nghttpx/nghttpx.conf``
.. option:: --include=<PATH>
Load additional configurations from <PATH>. File <PATH>
is read when configuration parser encountered this
option. This option can be used multiple times, or even
recursively.
.. option:: -v, --version
Print version and exit.

117
gennghttpxfun.py Executable file
View File

@ -0,0 +1,117 @@
#!/usr/bin/env python
from gentokenlookup import gentokenlookup
OPTIONS = [
"private-key-file",
"private-key-passwd-file",
"certificate-file",
"dh-param-file",
"subcert",
"backend",
"frontend",
"workers",
"http2-max-concurrent-streams",
"log-level",
"daemon",
"http2-proxy",
"http2-bridge",
"client-proxy",
"add-x-forwarded-for",
"strip-incoming-x-forwarded-for",
"no-via",
"frontend-http2-read-timeout",
"frontend-read-timeout",
"frontend-write-timeout",
"backend-read-timeout",
"backend-write-timeout",
"stream-read-timeout",
"stream-write-timeout",
"accesslog-file",
"accesslog-syslog",
"accesslog-format",
"errorlog-file",
"errorlog-syslog",
"backend-keep-alive-timeout",
"frontend-http2-window-bits",
"backend-http2-window-bits",
"frontend-http2-connection-window-bits",
"backend-http2-connection-window-bits",
"frontend-no-tls",
"backend-no-tls",
"backend-tls-sni-field",
"pid-file",
"user",
"syslog-facility",
"backlog",
"ciphers",
"client",
"insecure",
"cacert",
"backend-ipv4",
"backend-ipv6",
"backend-http-proxy-uri",
"read-rate",
"read-burst",
"write-rate",
"write-burst",
"worker-read-rate",
"worker-read-burst",
"worker-write-rate",
"worker-write-burst",
"npn-list",
"tls-proto-list",
"verify-client",
"verify-client-cacert",
"client-private-key-file",
"client-cert-file",
"frontend-http2-dump-request-header",
"frontend-http2-dump-response-header",
"http2-no-cookie-crumbling",
"frontend-frame-debug",
"padding",
"altsvc",
"add-request-header",
"add-response-header",
"worker-frontend-connections",
"no-location-rewrite",
"no-host-rewrite",
"backend-http1-connections-per-host",
"backend-http1-connections-per-frontend",
"listener-disable-timeout",
"tls-ticket-key-file",
"rlimit-nofile",
"backend-request-buffer",
"backend-response-buffer",
"no-server-push",
"backend-http2-connections-per-worker",
"fetch-ocsp-response-file",
"ocsp-update-interval",
"no-ocsp",
"header-field-buffer",
"max-header-fields",
"include",
"conf",
]
LOGVARS = [
"remote_addr",
"time_local",
"time_iso8601",
"request",
"status",
"body_bytes_sent",
"remote_port",
"server_port",
"request_time",
"pid",
"alpn",
"ssl_cipher",
"ssl_protocol",
"ssl_session_id",
"ssl_session_reused",
]
if __name__ == '__main__':
gentokenlookup(OPTIONS, 'SHRPX_OPTID', value_type='char', comp_fun='util::strieq_l')
gentokenlookup(LOGVARS, 'SHRPX_LOGF', value_type='char', comp_fun='util::strieq_l', return_type='LogFragmentType', fail_value='SHRPX_LOGF_NONE')

20
gentokenlookup.py
View File

@ -33,10 +33,10 @@ enum {'''
{}_MAXIDX,
}};'''.format(prefix)
def gen_index_header(tokens, prefix):
def gen_index_header(tokens, prefix, value_type, comp_fun, return_type, fail_value):
print '''\
int lookup_token(const uint8_t *name, size_t namelen) {
switch (namelen) {'''
{} lookup_token(const {} *name, size_t namelen) {{
switch (namelen) {{'''.format(return_type, value_type)
b = build_header(tokens)
for size in sorted(b.keys()):
ents = b[size]
@ -50,20 +50,20 @@ int lookup_token(const uint8_t *name, size_t namelen) {
case '{}':'''.format(c)
for k in headers:
print '''\
if (util::streq_l("{}", name, {})) {{
if ({}("{}", name, {})) {{
return {};
}}'''.format(k[:-1], size - 1, to_enum_hd(k, prefix))
}}'''.format(comp_fun, k[:-1], size - 1, to_enum_hd(k, prefix))
print '''\
break;'''
print '''\
}
break;'''
print '''\
}
return -1;
}'''
}}
return {};
}}'''.format(fail_value)
def gentokenlookup(tokens, prefix):
def gentokenlookup(tokens, prefix, value_type='uint8_t', comp_fun='util::streq_l', return_type='int', fail_value='-1'):
gen_enum(tokens, prefix)
print ''
gen_index_header(tokens, prefix)
gen_index_header(tokens, prefix, value_type, comp_fun, return_type, fail_value)

9
python/setup.py.in
View File

@ -22,8 +22,7 @@
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
from setuptools import setup, Extension
from Cython.Build import cythonize
LIBS = ['nghttp2']
setup(
@ -34,13 +33,13 @@ setup(
author_email = 'tatsuhiro.t@gmail.com',
url = 'https://nghttp2.org/',
keywords = [],
ext_modules = cythonize([Extension("nghttp2",
["nghttp2.pyx"],
ext_modules = [Extension("nghttp2",
["nghttp2.c"],
include_dirs=['@top_srcdir@/lib',
'@top_srcdir@/lib/includes',
'@top_builddir@/lib/includes'],
library_dirs=['@top_builddir@/lib/.libs',
'@top_builddir@'],
libraries=LIBS)]),
libraries=LIBS)],
long_description='TBD'
)

4
src/Makefile.am
View File

@ -207,10 +207,10 @@ libnghttp2_asio_la_LDFLAGS = -no-undefined -version-info 1:0:0
libnghttp2_asio_la_LIBADD = \
$(top_builddir)/lib/libnghttp2.la \
$(top_builddir)/third-party/libhttp-parser.la \
@OPENSSL_LIBS@ \
${BOOST_LDFLAGS} \
${BOOST_ASIO_LIB} \
${BOOST_THREAD_LIB} \
${BOOST_SYSTEM_LIB} \
@OPENSSL_LIBS@
${BOOST_SYSTEM_LIB}
endif # ENABLE_ASIO_LIB

51
src/http2.h
View File

@ -37,6 +37,8 @@
#include "http-parser/http_parser.h"
#include "util.h"
namespace nghttp2 {
struct Header {
@ -298,6 +300,55 @@ int lookup_method_token(const std::string &name);
const char *to_method_string(int method_token);
template <typename InputIt>
std::string normalize_path(InputIt first, InputIt last) {
// First, decode %XX for unreserved characters, then do
// http2::join_path
std::string result;
// We won't find %XX if length is less than 3.
if (last - first < 3) {
result.assign(first, last);
} else {
for (; first < last - 2;) {
if (*first == '%') {
if (util::isHexDigit(*(first + 1)) && util::isHexDigit(*(first + 2))) {
auto c = (util::hex_to_uint(*(first + 1)) << 4) +
util::hex_to_uint(*(first + 2));
if (util::inRFC3986UnreservedChars(c)) {
result += c;
first += 3;
continue;
}
result += '%';
result += util::upcase(*(first + 1));
result += util::upcase(*(first + 2));
first += 3;
continue;
}
}
result += *first++;
}
result.append(first, last);
}
return path_join(nullptr, 0, nullptr, 0, result.c_str(), result.size(),
nullptr, 0);
}
template <typename InputIt>
std::string rewrite_clean_path(InputIt first, InputIt last) {
if (first == last || *first != '/') {
return std::string(first, last);
}
// probably, not necessary most of the case, but just in case.
auto fragment = std::find(first, last, '#');
auto query = std::find(first, fragment, '?');
auto path = normalize_path(first, query);
if (query != fragment) {
path.append(query, fragment);
}
return path;
}
} // namespace http2
} // namespace nghttp2

52
src/http2_test.cc
View File

@ -829,4 +829,56 @@ void test_http2_path_join(void) {
}
}
void test_http2_normalize_path(void) {
std::string src;
src = "/alpha/bravo/../charlie";
CU_ASSERT("/alpha/charlie" ==
http2::normalize_path(std::begin(src), std::end(src)));
src = "/a%6c%70%68%61";
CU_ASSERT("/alpha" == http2::normalize_path(std::begin(src), std::end(src)));
src = "/alpha%2f%3a";
CU_ASSERT("/alpha%2F%3A" ==
http2::normalize_path(std::begin(src), std::end(src)));
src = "%2f";
CU_ASSERT("/%2F" == http2::normalize_path(std::begin(src), std::end(src)));
src = "%f";
CU_ASSERT("/%f" == http2::normalize_path(std::begin(src), std::end(src)));
src = "%";
CU_ASSERT("/%" == http2::normalize_path(std::begin(src), std::end(src)));
src = "";
CU_ASSERT("/" == http2::normalize_path(std::begin(src), std::end(src)));
}
void test_http2_rewrite_clean_path(void) {
std::string src;
// unreserved characters
src = "/alpha/%62ravo/";
CU_ASSERT("/alpha/bravo/" ==
http2::rewrite_clean_path(std::begin(src), std::end(src)));
// percent-encoding is converted to upper case.
src = "/delta%3a";
CU_ASSERT("/delta%3A" ==
http2::rewrite_clean_path(std::begin(src), std::end(src)));
// path component is normalized before mathcing
src = "/alpha/charlie/%2e././bravo/delta/..";
CU_ASSERT("/alpha/bravo/" ==
http2::rewrite_clean_path(std::begin(src), std::end(src)));
src = "alpha%3a";
CU_ASSERT(src == http2::rewrite_clean_path(std::begin(src), std::end(src)));
src = "";
CU_ASSERT(src == http2::rewrite_clean_path(std::begin(src), std::end(src)));
}
} // namespace shrpx

2
src/http2_test.h
View File

@ -45,6 +45,8 @@ void test_http2_http2_header_allowed(void);
void test_http2_mandatory_request_headers_presence(void);
void test_http2_parse_link_header(void);
void test_http2_path_join(void);
void test_http2_normalize_path(void);
void test_http2_rewrite_clean_path(void);
} // namespace shrpx

8
src/nghttp.cc
View File

@ -1376,10 +1376,11 @@ void HttpClient::output_har(FILE *outfile) {
entry, "startedDateTime",
json_string(util::format_iso8601(request_time).c_str()));
json_object_set_new(entry, "time", json_real(time_sum));
auto pushed = req->stream_id % 2 == 0;
json_object_set_new(entry, "comment", json_string(pushed ? "Pushed Object" : ""));
json_object_set_new(entry, "comment",
json_string(pushed ? "Pushed Object" : ""));
auto request = json_object();
json_object_set_new(entry, "request", request);
@ -1463,7 +1464,8 @@ void HttpClient::output_har(FILE *outfile) {
json_object_set_new(timings, "receive", json_real(receive_delta));
json_object_set_new(entry, "pageref", json_string(PAGE_ID));
json_object_set_new(entry, "connection", json_string(util::utos(req->stream_id).c_str()));
json_object_set_new(entry, "connection",
json_string(util::utos(req->stream_id).c_str()));
}
json_dumpf(root, outfile, JSON_PRESERVE_ORDER | JSON_INDENT(2));

6
src/shrpx-unittest.cc
View File

@ -96,6 +96,10 @@ int main(int argc, char *argv[]) {
!CU_add_test(pSuite, "http2_parse_link_header",
shrpx::test_http2_parse_link_header) ||
!CU_add_test(pSuite, "http2_path_join", shrpx::test_http2_path_join) ||
!CU_add_test(pSuite, "http2_normalize_path",
shrpx::test_http2_normalize_path) ||
!CU_add_test(pSuite, "http2_rewrite_clean_path",
shrpx::test_http2_rewrite_clean_path) ||
!CU_add_test(pSuite, "downstream_index_request_headers",
shrpx::test_downstream_index_request_headers) ||
!CU_add_test(pSuite, "downstream_index_response_headers",
@ -118,6 +122,8 @@ int main(int argc, char *argv[]) {
shrpx::test_shrpx_config_parse_log_format) ||
!CU_add_test(pSuite, "config_read_tls_ticket_key_file",
shrpx::test_shrpx_config_read_tls_ticket_key_file) ||
!CU_add_test(pSuite, "config_match_downstream_addr_group",
shrpx::test_shrpx_config_match_downstream_addr_group) ||
!CU_add_test(pSuite, "util_streq", shrpx::test_util_streq) ||
!CU_add_test(pSuite, "util_strieq", shrpx::test_util_strieq) ||
!CU_add_test(pSuite, "util_inp_strlower",

215
src/shrpx.cc
View File

@ -966,6 +966,7 @@ void fill_default_config() {
mod_config()->no_ocsp = false;
mod_config()->header_field_buffer = 64_k;
mod_config()->max_header_fields = 100;
mod_config()->downstream_addr_group_catch_all = 0;
}
} // namespace
@ -997,14 +998,67 @@ Options:
The options are categorized into several groups.
Connections:
-b, --backend=<HOST,PORT>
-b, --backend=(<HOST>,<PORT>|unix:<PATH>)[;<PATTERN>[:...]]
Set backend host and port. The multiple backend
addresses are accepted by repeating this option. UNIX
domain socket can be specified by prefixing path name
with "unix:" (e.g., unix:/var/run/backend.sock)
with "unix:" (e.g., unix:/var/run/backend.sock).
Optionally, if <PATTERN>s are given, the backend address
is only used if request matches the pattern. If -s or
-p is used, <PATTERN>s are ignored. The pattern
matching is closely designed to ServeMux in net/http
package of Go programming language. <PATTERN> consists
of path, host + path or just host. The path must start
with "/". If it ends with "/", it matches all request
path in its subtree. To deal with the request to the
directory without trailing slash, the path which ends
with "/" also matches the request path which only lacks
trailing '/' (e.g., path "/foo/" matches request path
"/foo"). If it does not end with "/", it performs exact
match against the request path. If host is given, it
performs exact match against the request host. If host
alone is given, "/" is appended to it, so that it
matches all request paths under the host (e.g.,
specifying "nghttp2.org" equals to "nghttp2.org/").
Patterns with host take precedence over patterns with
just path. Then, longer patterns take precedence over
shorter ones, breaking a tie by the order of the
appearance in the configuration.
If <PATTERN> is omitted, "/" is used as pattern, which
matches all request paths (catch-all pattern). The
catch-all backend must be given.
When doing a match, nghttpx made some normalization to
pattern, request host and path. For host part, they are
converted to lower case. For path part, percent-encoded
unreserved characters defined in RFC 3986 are decoded,
and any dot-segments (".." and ".") are resolved and
removed.
For example, -b'127.0.0.1,8080;nghttp2.org/httpbin/'
matches the request host "nghttp2.org" and the request
path "/httpbin/get", but does not match the request host
"nghttp2.org" and the request path "/index.html".
The multiple <PATTERN>s can be specified, delimiting
them by ":". Specifying
-b'127.0.0.1,8080;nghttp2.org:www.nghttp2.org' has the
same effect to specify -b'127.0.0.1,8080;nghttp2.org'
and -b'127.0.0.1,8080;www.nghttp2.org'.
The backend addresses sharing same <PATTERN> are grouped
together forming load balancing group.
Since ";" and ":" are used as delimiter, <PATTERN> must
not contain these characters. Since ";" has special
meaning in shell, the option value must be quoted.
Default: )" << DEFAULT_DOWNSTREAM_HOST << ","
<< DEFAULT_DOWNSTREAM_PORT << R"(
-f, --frontend=<HOST,PORT>
-f, --frontend=(<HOST>,<PORT>|unix:<PATH>)
Set frontend host and port. If <HOST> is '*', it
assumes all addresses including both IPv4 and IPv6.
UNIX domain socket can be specified by prefixing path
@ -1079,9 +1133,14 @@ Performance:
accepts. Setting 0 means unlimited.
Default: )" << get_config()->worker_frontend_connections << R"(
--backend-http2-connections-per-worker=<N>
Set maximum number of HTTP/2 connections per worker.
The default value is 0, which means the number of
backend addresses specified by -b option.
Set maximum number of backend HTTP/2 physical
connections per worker. If pattern is used in -b
option, this limit is applied to each pattern group (in
other words, each pattern group can have maximum <N>
HTTP/2 connections). The default value is 0, which
means that the value is adjusted to the number of
backend addresses. If pattern is used, this adjustment
is done for each pattern group.
--backend-http1-connections-per-host=<N>
Set maximum number of backend concurrent HTTP/1
connections per origin host. This option is meaningful
@ -1351,6 +1410,9 @@ Logging:
* $ssl_session_reused: "r" if SSL/TLS session was
reused. Otherwise, "."
The variable can be enclosed by "{" and "}" for
disambiguation (e.g., ${remote_addr}).
Default: )" << DEFAULT_ACCESSLOG_FORMAT << R"(
--errorlog-file=<PATH>
Set path to write error log. To reopen file, send USR1
@ -1446,6 +1508,11 @@ Misc:
--conf=<PATH>
Load configuration from <PATH>.
Default: )" << get_config()->conf_path.get() << R"(
--include=<PATH>
Load additional configurations from <PATH>. File <PATH>
is read when configuration parser encountered this
option. This option can be used multiple times, or even
recursively.
-v, --version
Print version and exit.
-h, --help Print this help and exit.
@ -1592,6 +1659,7 @@ int main(int argc, char **argv) {
{SHRPX_OPT_HEADER_FIELD_BUFFER, required_argument, &flag, 80},
{SHRPX_OPT_MAX_HEADER_FIELDS, required_argument, &flag, 81},
{SHRPX_OPT_ADD_REQUEST_HEADER, required_argument, &flag, 82},
{SHRPX_OPT_INCLUDE, required_argument, &flag, 83},
{nullptr, 0, nullptr, 0}};
int option_index = 0;
@ -1954,6 +2022,10 @@ int main(int argc, char **argv) {
// --add-request-header
cmdcfgs.emplace_back(SHRPX_OPT_ADD_REQUEST_HEADER, optarg);
break;
case 83:
// --include
cmdcfgs.emplace_back(SHRPX_OPT_INCLUDE, optarg);
break;
default:
break;
}
@ -1964,11 +2036,13 @@ int main(int argc, char **argv) {
}
if (conf_exists(get_config()->conf_path.get())) {
if (load_config(get_config()->conf_path.get()) == -1) {
std::set<std::string> include_set;
if (load_config(get_config()->conf_path.get(), include_set) == -1) {
LOG(FATAL) << "Failed to load configuration from "
<< get_config()->conf_path.get();
exit(EXIT_FAILURE);
}
assert(include_set.empty());
}
if (argc - optind >= 2) {
@ -1980,11 +2054,18 @@ int main(int argc, char **argv) {
// parsing option values.
reopen_log_files();
for (size_t i = 0, len = cmdcfgs.size(); i < len; ++i) {
if (parse_config(cmdcfgs[i].first, cmdcfgs[i].second) == -1) {
LOG(FATAL) << "Failed to parse command-line argument.";
exit(EXIT_FAILURE);
{
std::set<std::string> include_set;
for (size_t i = 0, len = cmdcfgs.size(); i < len; ++i) {
if (parse_config(cmdcfgs[i].first, cmdcfgs[i].second, include_set) ==
-1) {
LOG(FATAL) << "Failed to parse command-line argument.";
exit(EXIT_FAILURE);
}
}
assert(include_set.empty());
}
if (get_config()->accesslog_syslog || get_config()->errorlog_syslog) {
@ -2118,55 +2199,96 @@ int main(int argc, char **argv) {
}
}
if (get_config()->downstream_addrs.empty()) {
if (get_config()->downstream_addr_groups.empty()) {
DownstreamAddr addr;
addr.host = strcopy(DEFAULT_DOWNSTREAM_HOST);
addr.port = DEFAULT_DOWNSTREAM_PORT;
mod_config()->downstream_addrs.push_back(std::move(addr));
DownstreamAddrGroup g("/");
g.addrs.push_back(std::move(addr));
mod_config()->downstream_addr_groups.push_back(std::move(g));
} else if (get_config()->http2_proxy || get_config()->client_proxy) {
// We don't support host mapping in these cases. Move all
// non-catch-all patterns to catch-all pattern.
DownstreamAddrGroup catch_all("/");
for (auto &g : mod_config()->downstream_addr_groups) {
std::move(std::begin(g.addrs), std::end(g.addrs),
std::back_inserter(catch_all.addrs));
}
std::vector<DownstreamAddrGroup>().swap(
mod_config()->downstream_addr_groups);
mod_config()->downstream_addr_groups.push_back(std::move(catch_all));
}
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Resolving backend address";
}
for (auto &addr : mod_config()->downstream_addrs) {
ssize_t catch_all_group = -1;
for (size_t i = 0; i < mod_config()->downstream_addr_groups.size(); ++i) {
auto &g = mod_config()->downstream_addr_groups[i];
if (g.pattern == "/") {
catch_all_group = i;
}
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Host-path pattern: group " << i << ": '" << g.pattern
<< "'";
for (auto &addr : g.addrs) {
LOG(INFO) << "group " << i << " -> " << addr.host.get()
<< (addr.host_unix ? "" : ":" + util::utos(addr.port));
}
}
}
if (addr.host_unix) {
// for AF_UNIX socket, we use "localhost" as host for backend
// hostport. This is used as Host header field to backend and
// not going to be passed to any syscalls.
addr.hostport =
strcopy(util::make_hostport("localhost", get_config()->port));
if (catch_all_group == -1) {
LOG(FATAL) << "-b: No catch-all backend address is configured";
exit(EXIT_FAILURE);
}
mod_config()->downstream_addr_group_catch_all = catch_all_group;
auto path = addr.host.get();
auto pathlen = strlen(path);
if (LOG_ENABLED(INFO)) {
LOG(INFO) << "Catch-all pattern is group " << catch_all_group;
}
if (pathlen + 1 > sizeof(addr.addr.un.sun_path)) {
LOG(FATAL) << "UNIX domain socket path " << path << " is too long > "
<< sizeof(addr.addr.un.sun_path);
exit(EXIT_FAILURE);
for (auto &g : mod_config()->downstream_addr_groups) {
for (auto &addr : g.addrs) {
if (addr.host_unix) {
// for AF_UNIX socket, we use "localhost" as host for backend
// hostport. This is used as Host header field to backend and
// not going to be passed to any syscalls.
addr.hostport =
strcopy(util::make_hostport("localhost", get_config()->port));
auto path = addr.host.get();
auto pathlen = strlen(path);
if (pathlen + 1 > sizeof(addr.addr.un.sun_path)) {
LOG(FATAL) << "UNIX domain socket path " << path << " is too long > "
<< sizeof(addr.addr.un.sun_path);
exit(EXIT_FAILURE);
}
LOG(INFO) << "Use UNIX domain socket path " << path
<< " for backend connection";
addr.addr.un.sun_family = AF_UNIX;
// copy path including terminal NULL
std::copy_n(path, pathlen + 1, addr.addr.un.sun_path);
addr.addrlen = sizeof(addr.addr.un);
continue;
}
LOG(INFO) << "Use UNIX domain socket path " << path
<< " for backend connection";
addr.hostport = strcopy(util::make_hostport(addr.host.get(), addr.port));
addr.addr.un.sun_family = AF_UNIX;
// copy path including terminal NULL
std::copy_n(path, pathlen + 1, addr.addr.un.sun_path);
addr.addrlen = sizeof(addr.addr.un);
continue;
}
addr.hostport = strcopy(util::make_hostport(addr.host.get(), addr.port));
if (resolve_hostname(
&addr.addr, &addr.addrlen, addr.host.get(), addr.port,
get_config()->backend_ipv4
? AF_INET
: (get_config()->backend_ipv6 ? AF_INET6 : AF_UNSPEC)) == -1) {
exit(EXIT_FAILURE);
if (resolve_hostname(
&addr.addr, &addr.addrlen, addr.host.get(), addr.port,
get_config()->backend_ipv4 ? AF_INET : (get_config()->backend_ipv6
? AF_INET6
: AF_UNSPEC)) == -1) {
exit(EXIT_FAILURE);
}
}
}
@ -2183,11 +2305,6 @@ int main(int argc, char **argv) {
}
}
if (get_config()->http2_downstream_connections_per_worker == 0) {
mod_config()->http2_downstream_connections_per_worker =
get_config()->downstream_addrs.size();
}
if (get_config()->rlimit_nofile) {
struct rlimit lim = {static_cast<rlim_t>(get_config()->rlimit_nofile),
static_cast<rlim_t>(get_config()->rlimit_nofile)};

64
src/shrpx_client_handler.cc
View File

@ -39,6 +39,7 @@
#include "shrpx_worker.h"
#include "shrpx_downstream_connection_pool.h"
#include "shrpx_downstream.h"
#include "shrpx_http2_session.h"
#ifdef HAVE_SPDYLAY
#include "shrpx_spdy_upstream.h"
#endif // HAVE_SPDYLAY
@ -360,8 +361,12 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl,
get_config()->upstream_read_timeout, get_config()->write_rate,
get_config()->write_burst, get_config()->read_rate,
get_config()->read_burst, writecb, readcb, timeoutcb, this),
pinned_http2sessions_(
get_config()->downstream_proto == PROTO_HTTP2
? make_unique<std::vector<ssize_t>>(
get_config()->downstream_addr_groups.size(), -1)
: nullptr),
ipaddr_(ipaddr), port_(port), worker_(worker),
http2session_(worker_->next_http2_session()),
left_connhd_len_(NGHTTP2_CLIENT_MAGIC_LEN),
should_close_after_write_(false) {
@ -588,7 +593,8 @@ void ClientHandler::pool_downstream_connection(
return;
}
if (LOG_ENABLED(INFO)) {
CLOG(INFO, this) << "Pooling downstream connection DCONN:" << dconn.get();
CLOG(INFO, this) << "Pooling downstream connection DCONN:" << dconn.get()
<< " in group " << dconn->get_group();
}
dconn->set_client_handler(nullptr);
auto dconn_pool = worker_->get_dconn_pool();
@ -605,9 +611,43 @@ void ClientHandler::remove_downstream_connection(DownstreamConnection *dconn) {
}
std::unique_ptr<DownstreamConnection>
ClientHandler::get_downstream_connection() {
ClientHandler::get_downstream_connection(Downstream *downstream) {
size_t group;
auto &groups = get_config()->downstream_addr_groups;
auto catch_all = get_config()->downstream_addr_group_catch_all;
// Fast path. If we have one group, it must be catch-all group.
// HTTP/2 and client proxy modes fall in this case.
if (groups.size() == 1) {
group = 0;
} else if (downstream->get_request_method() == HTTP_CONNECT) {
// We don't know how to treat CONNECT request in host-path
// mapping. It most likely appears in proxy scenario. Since we
// have dealt with proxy case already, just use catch-all group.
group = catch_all;
} else {
if (!downstream->get_request_http2_authority().empty()) {
group = match_downstream_addr_group(
downstream->get_request_http2_authority(),
downstream->get_request_path(), groups, catch_all);
} else {
auto h = downstream->get_request_header(http2::HD_HOST);
if (h) {
group = match_downstream_addr_group(
h->value, downstream->get_request_path(), groups, catch_all);
} else {
group = match_downstream_addr_group("", downstream->get_request_path(),
groups, catch_all);
}
}
}
if (LOG_ENABLED(INFO)) {
CLOG(INFO, this) << "Downstream address group: " << group;
}
auto dconn_pool = worker_->get_dconn_pool();
auto dconn = dconn_pool->pop_downstream_connection();
auto dconn = dconn_pool->pop_downstream_connection(group);
if (!dconn) {
if (LOG_ENABLED(INFO)) {
@ -617,10 +657,20 @@ ClientHandler::get_downstream_connection() {
auto dconn_pool = worker_->get_dconn_pool();
if (http2session_) {
dconn = make_unique<Http2DownstreamConnection>(dconn_pool, http2session_);
if (get_config()->downstream_proto == PROTO_HTTP2) {
Http2Session *http2session;
auto &pinned = (*pinned_http2sessions_)[group];
if (pinned == -1) {
http2session = worker_->next_http2_session(group);
pinned = http2session->get_index();
} else {
auto dgrp = worker_->get_dgrp(group);
http2session = dgrp->http2sessions[pinned].get();
}
dconn = make_unique<Http2DownstreamConnection>(dconn_pool, http2session);
} else {
dconn = make_unique<HttpDownstreamConnection>(dconn_pool, conn_.loop);
dconn =
make_unique<HttpDownstreamConnection>(dconn_pool, group, conn_.loop);
}
dconn->set_client_handler(this);
return dconn;

6
src/shrpx_client_handler.h
View File

@ -44,7 +44,6 @@ namespace shrpx {
class Upstream;
class DownstreamConnection;
class Http2Session;
class HttpsUpstream;
class ConnectBlocker;
class DownstreamConnectionPool;
@ -92,7 +91,8 @@ public:
void pool_downstream_connection(std::unique_ptr<DownstreamConnection> dconn);
void remove_downstream_connection(DownstreamConnection *dconn);
std::unique_ptr<DownstreamConnection> get_downstream_connection();
std::unique_ptr<DownstreamConnection>
get_downstream_connection(Downstream *downstream);
MemchunkPool *get_mcpool();
SSL *get_ssl() const;
ConnectBlocker *get_connect_blocker() const;
@ -134,6 +134,7 @@ private:
Connection conn_;
ev_timer reneg_shutdown_timer_;
std::unique_ptr<Upstream> upstream_;
std::unique_ptr<std::vector<ssize_t>> pinned_http2sessions_;
std::string ipaddr_;
std::string port_;
// The ALPN identifier negotiated for this connection.
@ -141,7 +142,6 @@ private:
std::function<int(ClientHandler &)> read_, write_;
std::function<int(ClientHandler &)> on_read_, on_write_;
Worker *worker_;
Http2Session *http2session_;
// The number of bytes of HTTP/2 client connection header to read
size_t left_connhd_len_;
bool should_close_after_write_;

1508
src/shrpx_config.cc
View File

File diff suppressed because it is too large Load Diff

75
src/shrpx_config.h
View File

@ -42,6 +42,7 @@
#include <cstdio>
#include <vector>
#include <memory>
#include <set>
#include <openssl/ssl.h>
@ -49,6 +50,10 @@
#include <nghttp2/nghttp2.h>
#include "template.h"
using namespace nghttp2;
namespace shrpx {
struct LogFragment;
@ -165,6 +170,7 @@ constexpr char SHRPX_OPT_OCSP_UPDATE_INTERVAL[] = "ocsp-update-interval";
constexpr char SHRPX_OPT_NO_OCSP[] = "no-ocsp";
constexpr char SHRPX_OPT_HEADER_FIELD_BUFFER[] = "header-field-buffer";
constexpr char SHRPX_OPT_MAX_HEADER_FIELDS[] = "max-header-fields";
constexpr char SHRPX_OPT_INCLUDE[] = "include";
union sockaddr_union {
sockaddr_storage storage;
@ -194,6 +200,11 @@ struct AltSvc {
struct DownstreamAddr {
DownstreamAddr() : addr{{0}}, addrlen(0), port(0), host_unix(false) {}
DownstreamAddr(const DownstreamAddr &other);
DownstreamAddr(DownstreamAddr &&) = default;
DownstreamAddr &operator=(const DownstreamAddr &other);
DownstreamAddr &operator=(DownstreamAddr &&other) = default;
sockaddr_union addr;
// backend address. If |host_unix| is true, this is UNIX domain
// socket path.
@ -206,6 +217,12 @@ struct DownstreamAddr {
bool host_unix;
};
struct DownstreamAddrGroup {
DownstreamAddrGroup(std::string pattern) : pattern(std::move(pattern)) {}
std::string pattern;
std::vector<DownstreamAddr> addrs;
};
struct TicketKey {
uint8_t name[16];
uint8_t aes_key[16];
@ -225,7 +242,7 @@ struct Config {
std::vector<std::pair<std::string, std::string>> add_response_headers;
std::vector<unsigned char> alpn_prefs;
std::vector<LogFragment> accesslog_format;
std::vector<DownstreamAddr> downstream_addrs;
std::vector<DownstreamAddrGroup> downstream_addr_groups;
std::vector<std::string> tls_ticket_key_files;
// binary form of http proxy host and port
sockaddr_union downstream_http_proxy_addr;
@ -311,6 +328,8 @@ struct Config {
size_t downstream_response_buffer_size;
size_t header_field_buffer;
size_t max_header_fields;
// The index of catch-all group in downstream_addr_groups.
size_t downstream_addr_group_catch_all;
// Bit mask to disable SSL/TLS protocol versions. This will be
// passed to SSL_CTX_set_options().
long int tls_proto_mask;
@ -365,26 +384,28 @@ void create_config();
// Parses option name |opt| and value |optarg|. The results are
// stored into statically allocated Config object. This function
// returns 0 if it succeeds, or -1.
int parse_config(const char *opt, const char *optarg);
// returns 0 if it succeeds, or -1. The |included_set| contains the
// all paths already included while processing this configuration, to
// avoid loop in --include option.
int parse_config(const char *opt, const char *optarg,
std::set<std::string> &included_set);
// Loads configurations from |filename| and stores them in statically
// allocated Config object. This function returns 0 if it succeeds, or
// -1.
int load_config(const char *filename);
// -1. See parse_config() for |include_set|.
int load_config(const char *filename, std::set<std::string> &include_set);
// Read passwd from |filename|
std::string read_passwd_from_file(const char *filename);
// Parses comma delimited strings in |s| and returns the array of
// pointers, each element points to the each substring in |s|. The
// |s| must be comma delimited list of strings. The strings must be
// delimited by a single comma and any white spaces around it are
// treated as a part of protocol strings. This function may modify
// |s| and the caller must leave it as is after this call. This
// Parses delimited strings in |s| and returns the array of pointers,
// each element points to the each substring in |s|. The delimiter is
// given by |delim. The |s| must be comma delimited list of strings.
// The strings must be delimited by a single comma and any white
// spaces around it are treated as a part of protocol strings. This
// function copies |s| and first element in the return value points to
// it. It is caller's responsibility to deallocate its memory.
std::vector<char *> parse_config_str_list(const char *s);
std::vector<char *> parse_config_str_list(const char *s, char delim = ',');
// Clears all elements of |list|, which is returned by
// parse_config_str_list(). If list is not empty, list[0] is freed by
@ -399,15 +420,23 @@ std::pair<std::string, std::string> parse_header(const char *optarg);
std::vector<LogFragment> parse_log_format(const char *optarg);
// Returns a copy of NULL-terminated string |val|.
std::unique_ptr<char[]> strcopy(const char *val);
// Returns a copy of NULL-terminated string [first, last).
template <typename InputIt>
std::unique_ptr<char[]> strcopy(InputIt first, InputIt last) {
auto res = make_unique<char[]>(last - first + 1);
*std::copy(first, last, res.get()) = '\0';
return res;
}
// Returns a copy of string |val| of length |n|. The returned string
// will be NULL-terminated.
std::unique_ptr<char[]> strcopy(const char *val, size_t n);
// Returns a copy of NULL-terminated string |val|.
inline std::unique_ptr<char[]> strcopy(const char *val) {
return strcopy(val, val + strlen(val));
}
// Returns a copy of val.c_str().
std::unique_ptr<char[]> strcopy(const std::string &val);
inline std::unique_ptr<char[]> strcopy(const std::string &val) {
return strcopy(std::begin(val), std::end(val));
}
// Returns string for syslog |facility|.
const char *str_syslog_facility(int facility);
@ -423,6 +452,16 @@ FILE *open_file_for_write(const char *filename);
std::unique_ptr<TicketKeys>
read_tls_ticket_key_file(const std::vector<std::string> &files);
// Selects group based on request's |hostport| and |path|. |hostport|
// is the value taken from :authority or host header field, and may
// contain port. The |path| may contain query part. We require the
// catch-all pattern in place, so this function always selects one
// group. The catch-all group index is given in |catch_all|. All
// patterns are given in |groups|.
size_t match_downstream_addr_group(
const std::string &hostport, const std::string &path,
const std::vector<DownstreamAddrGroup> &groups, size_t catch_all);
} // namespace shrpx
#endif // SHRPX_CONFIG_H

106
src/shrpx_config_test.cc
View File

@ -48,7 +48,7 @@ void test_shrpx_config_parse_config_str_list(void) {
CU_ASSERT(0 == strcmp("", res[1]));
clear_config_str_list(res);
res = parse_config_str_list(",a,,");
res = parse_config_str_list(":a::", ':');
CU_ASSERT(4 == res.size());
CU_ASSERT(0 == strcmp("", res[0]));
CU_ASSERT(0 == strcmp("a", res[1]));
@ -95,9 +95,9 @@ void test_shrpx_config_parse_header(void) {
}
void test_shrpx_config_parse_log_format(void) {
auto res = parse_log_format("$remote_addr - $remote_user [$time_local] "
"\"$request\" $status $body_bytes_sent "
"\"$http_referer\" \"$http_user_agent\"");
auto res = parse_log_format(R"($remote_addr - $remote_user [$time_local] )"
R"("$request" $status $body_bytes_sent )"
R"("${http_referer}" "$http_user_agent")");
CU_ASSERT(14 == res.size());
CU_ASSERT(SHRPX_LOGF_REMOTE_ADDR == res[0].type);
@ -136,6 +136,44 @@ void test_shrpx_config_parse_log_format(void) {
CU_ASSERT(SHRPX_LOGF_LITERAL == res[13].type);
CU_ASSERT(0 == strcmp("\"", res[13].value.get()));
res = parse_log_format("$");
CU_ASSERT(1 == res.size());
CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type);
CU_ASSERT(0 == strcmp("$", res[0].value.get()));
res = parse_log_format("${");
CU_ASSERT(1 == res.size());
CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type);
CU_ASSERT(0 == strcmp("${", res[0].value.get()));
res = parse_log_format("${a");
CU_ASSERT(1 == res.size());
CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type);
CU_ASSERT(0 == strcmp("${a", res[0].value.get()));
res = parse_log_format("${a ");
CU_ASSERT(1 == res.size());
CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type);
CU_ASSERT(0 == strcmp("${a ", res[0].value.get()));
res = parse_log_format("$$remote_addr");
CU_ASSERT(2 == res.size());
CU_ASSERT(SHRPX_LOGF_LITERAL == res[0].type);
CU_ASSERT(0 == strcmp("$", res[0].value.get()));
CU_ASSERT(SHRPX_LOGF_REMOTE_ADDR == res[1].type);
CU_ASSERT(nullptr == res[1].value.get());
}
void test_shrpx_config_read_tls_ticket_key_file(void) {
@ -172,4 +210,64 @@ void test_shrpx_config_read_tls_ticket_key_file(void) {
memcmp("a..............b", key->hmac_key, sizeof(key->hmac_key)));
}
void test_shrpx_config_match_downstream_addr_group(void) {
auto groups = std::vector<DownstreamAddrGroup>{
{"nghttp2.org/"},
{"nghttp2.org/alpha/bravo/"},
{"nghttp2.org/alpha/charlie"},
{"nghttp2.org/delta%3A"},
{"www.nghttp2.org/"},
{"[::1]/"},
};
CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "/", groups, 255));
// port is removed
CU_ASSERT(0 ==
match_downstream_addr_group("nghttp2.org:8080", "/", groups, 255));
// host is case-insensitive
CU_ASSERT(4 == match_downstream_addr_group("WWW.nghttp2.org", "/alpha",
groups, 255));
CU_ASSERT(1 == match_downstream_addr_group("nghttp2.org", "/alpha/bravo/",
groups, 255));
// /alpha/bravo also matches /alpha/bravo/
CU_ASSERT(1 == match_downstream_addr_group("nghttp2.org", "/alpha/bravo",
groups, 255));
// path part is case-sensitive
CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "/Alpha/bravo",
groups, 255));
CU_ASSERT(1 == match_downstream_addr_group(
"nghttp2.org", "/alpha/bravo/charlie", groups, 255));
CU_ASSERT(2 == match_downstream_addr_group("nghttp2.org", "/alpha/charlie",
groups, 255));
// pattern which does not end with '/' must match its entirely. So
// this matches to group 0, not group 2.
CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "/alpha/charlie/",
groups, 255));
CU_ASSERT(255 ==
match_downstream_addr_group("example.org", "/", groups, 255));
CU_ASSERT(255 == match_downstream_addr_group("", "/", groups, 255));
CU_ASSERT(255 == match_downstream_addr_group("", "alpha", groups, 255));
CU_ASSERT(255 == match_downstream_addr_group("foo/bar", "/", groups, 255));
// If path is "*", only match with host + "/".
CU_ASSERT(0 == match_downstream_addr_group("nghttp2.org", "*", groups, 255));
CU_ASSERT(5 == match_downstream_addr_group("[::1]", "/", groups, 255));
CU_ASSERT(5 == match_downstream_addr_group("[::1]:8080", "/", groups, 255));
CU_ASSERT(255 == match_downstream_addr_group("[::1", "/", groups, 255));
CU_ASSERT(255 == match_downstream_addr_group("[::1]8000", "/", groups, 255));
}
} // namespace shrpx

1
src/shrpx_config_test.h
View File

@ -35,6 +35,7 @@ void test_shrpx_config_parse_config_str_list(void);
void test_shrpx_config_parse_header(void);
void test_shrpx_config_parse_log_format(void);
void test_shrpx_config_read_tls_ticket_key_file(void);
void test_shrpx_config_match_downstream_addr_group(void);
} // namespace shrpx

18
src/shrpx_downstream.cc
View File

@ -152,10 +152,6 @@ Downstream::~Downstream() {
DLOG(INFO, this) << "Deleting";
}
if (blocked_link_) {
detach_blocked_link(blocked_link_);
}
// check nullptr for unittest
if (upstream_) {
auto loop = upstream_->get_client_handler()->get_loop();
@ -166,6 +162,10 @@ Downstream::~Downstream() {
ev_timer_stop(loop, &downstream_wtimer_);
}
// DownstreamConnection may refer to this object. Delete it now
// explicitly.
dconn_.reset();
if (LOG_ENABLED(INFO)) {
DLOG(INFO, this) << "Deleted";
}
@ -195,8 +195,6 @@ void Downstream::detach_downstream_connection() {
std::unique_ptr<DownstreamConnection>(dconn_.release()));
}
void Downstream::release_downstream_connection() { dconn_.release(); }
DownstreamConnection *Downstream::get_downstream_connection() {
return dconn_.get();
}
@ -1200,12 +1198,10 @@ void Downstream::attach_blocked_link(BlockedLink *l) {
blocked_link_ = l;
}
void Downstream::detach_blocked_link(BlockedLink *l) {
assert(blocked_link_);
assert(l->downstream == this);
l->downstream = nullptr;
BlockedLink *Downstream::detach_blocked_link() {
auto link = blocked_link_;
blocked_link_ = nullptr;
return link;
}
void Downstream::add_request_headers_sum(size_t amount) {

4
src/shrpx_downstream.h
View File

@ -69,8 +69,6 @@ public:
int attach_downstream_connection(std::unique_ptr<DownstreamConnection> dconn);
void detach_downstream_connection();
// Releases dconn_, without freeing it.
void release_downstream_connection();
DownstreamConnection *get_downstream_connection();
// Returns dconn_ and nullifies dconn_.
std::unique_ptr<DownstreamConnection> pop_downstream_connection();
@ -332,7 +330,7 @@ public:
void set_dispatch_state(int s);
void attach_blocked_link(BlockedLink *l);
void detach_blocked_link(BlockedLink *l);
BlockedLink *detach_blocked_link();
enum {
EVENT_ERROR = 0x1,

1
src/shrpx_downstream_connection.h
View File

@ -57,6 +57,7 @@ public:
virtual void on_upstream_change(Upstream *uptream) = 0;
virtual int on_priority_change(int32_t pri) = 0;
virtual size_t get_group() const = 0;
// true if this object is poolable.
virtual bool poolable() const = 0;

27
src/shrpx_downstream_connection_pool.cc
View File

@ -27,33 +27,42 @@
namespace shrpx {
DownstreamConnectionPool::DownstreamConnectionPool() {}
DownstreamConnectionPool::DownstreamConnectionPool(size_t num_groups)
: gpool_(num_groups) {}
DownstreamConnectionPool::~DownstreamConnectionPool() {
for (auto dconn : pool_) {
delete dconn;
for (auto &pool : gpool_) {
for (auto dconn : pool) {
delete dconn;
}
}
}
void DownstreamConnectionPool::add_downstream_connection(
std::unique_ptr<DownstreamConnection> dconn) {
pool_.insert(dconn.release());
auto group = dconn->get_group();
assert(gpool_.size() > group);
gpool_[group].insert(dconn.release());
}
std::unique_ptr<DownstreamConnection>
DownstreamConnectionPool::pop_downstream_connection() {
if (pool_.empty()) {
DownstreamConnectionPool::pop_downstream_connection(size_t group) {
assert(gpool_.size() > group);
auto &pool = gpool_[group];
if (pool.empty()) {
return nullptr;
}
auto dconn = std::unique_ptr<DownstreamConnection>(*std::begin(pool_));
pool_.erase(std::begin(pool_));
auto dconn = std::unique_ptr<DownstreamConnection>(*std::begin(pool));
pool.erase(std::begin(pool));
return dconn;
}
void DownstreamConnectionPool::remove_downstream_connection(
DownstreamConnection *dconn) {
pool_.erase(dconn);
auto group = dconn->get_group();
assert(gpool_.size() > group);
gpool_[group].erase(dconn);
delete dconn;
}

6
src/shrpx_downstream_connection_pool.h
View File

@ -36,15 +36,15 @@ class DownstreamConnection;
class DownstreamConnectionPool {
public:
DownstreamConnectionPool();
DownstreamConnectionPool(size_t num_groups);
~DownstreamConnectionPool();
void add_downstream_connection(std::unique_ptr<DownstreamConnection> dconn);
std::unique_ptr<DownstreamConnection> pop_downstream_connection();
std::unique_ptr<DownstreamConnection> pop_downstream_connection(size_t group);
void remove_downstream_connection(DownstreamConnection *dconn);
private:
std::set<DownstreamConnection *> pool_;
std::vector<std::set<DownstreamConnection *>> gpool_;
};
} // namespace shrpx

45
src/shrpx_downstream_queue.cc
View File

@ -124,17 +124,21 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream) {
// Delete downstream when this function returns.
auto delptr = std::unique_ptr<Downstream>(downstream);
if (downstream->get_dispatch_state() != Downstream::DISPATCH_ACTIVE) {
assert(downstream->get_dispatch_state() != Downstream::DISPATCH_NONE);
downstreams_.remove(downstream);
return nullptr;
}
downstreams_.remove(downstream);
auto &host = make_host_key(downstream);
auto &ent = find_host_entry(host);
--ent.num_active;
if (downstream->get_dispatch_state() == Downstream::DISPATCH_ACTIVE) {
--ent.num_active;
} else {
// For those downstreams deleted while in blocked state
auto link = downstream->detach_blocked_link();
if (link) {
ent.blocked.remove(link);
delete link;
}
}
if (remove_host_entry_if_empty(ent, host_entries_, host)) {
return nullptr;
@ -144,21 +148,20 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream) {
return nullptr;
}
for (auto link = ent.blocked.head; link;) {
auto next = link->dlnext;
if (!link->downstream) {
ent.blocked.remove(link);
link = next;
continue;
}
auto next_downstream = link->downstream;
next_downstream->detach_blocked_link(link);
ent.blocked.remove(link);
delete link;
remove_host_entry_if_empty(ent, host_entries_, host);
return next_downstream;
auto link = ent.blocked.head;
if (!link) {
return nullptr;
}
return nullptr;
auto next_downstream = link->downstream;
auto link2 = next_downstream->detach_blocked_link();
assert(link2 == link);
ent.blocked.remove(link);
delete link;
remove_host_entry_if_empty(ent, host_entries_, host);
return next_downstream;
}
Downstream *DownstreamQueue::get_downstreams() const {

19
src/shrpx_http2_downstream_connection.cc
View File

@ -86,11 +86,7 @@ Http2DownstreamConnection::~Http2DownstreamConnection() {
}
}
http2session_->remove_downstream_connection(this);
// Downstream and DownstreamConnection may be deleted
// asynchronously.
if (downstream_) {
downstream_->release_downstream_connection();
}
if (LOG_ENABLED(INFO)) {
DCLOG(INFO, this) << "Deleted";
}
@ -263,8 +259,11 @@ int Http2DownstreamConnection::push_request_headers() {
// http2session_ has already in CONNECTED state, so we can get
// addr_idx here.
auto addr_idx = http2session_->get_addr_idx();
auto downstream_hostport =
get_config()->downstream_addrs[addr_idx].hostport.get();
auto group = http2session_->get_group();
auto downstream_hostport = get_config()
->downstream_addr_groups[group]
.addrs[addr_idx]
.hostport.get();
const char *authority = nullptr, *host = nullptr;
if (!no_host_rewrite) {
@ -557,4 +556,10 @@ int Http2DownstreamConnection::on_timeout() {
return submit_rst_stream(downstream_, NGHTTP2_NO_ERROR);
}
size_t Http2DownstreamConnection::get_group() const {
// HTTP/2 backend connections are managed by Http2Session object,
// and it stores group index.
return http2session_->get_group();
}
} // namespace shrpx

1
src/shrpx_http2_downstream_connection.h
View File

@ -61,6 +61,7 @@ public:
virtual void on_upstream_change(Upstream *upstream) {}
virtual int on_priority_change(int32_t pri);
virtual size_t get_group() const;
// This object is not poolable because we dont' have facility to
// migrate to another Http2Session object.

33
src/shrpx_http2_session.cc
View File

@ -142,13 +142,14 @@ void writecb(struct ev_loop *loop, ev_io *w, int revents) {
} // namespace
Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx,
ConnectBlocker *connect_blocker, Worker *worker)
ConnectBlocker *connect_blocker, Worker *worker,
size_t group, size_t idx)
: conn_(loop, -1, nullptr, get_config()->downstream_write_timeout,
get_config()->downstream_read_timeout, 0, 0, 0, 0, writecb, readcb,
timeoutcb, this),
worker_(worker), connect_blocker_(connect_blocker), ssl_ctx_(ssl_ctx),
session_(nullptr), data_pending_(nullptr), data_pendinglen_(0),
addr_idx_(0), state_(DISCONNECTED),
addr_idx_(0), group_(group), index_(idx), state_(DISCONNECTED),
connection_check_state_(CONNECTION_CHECK_NONE), flow_control_(false) {
read_ = write_ = &Http2Session::noop;
@ -233,11 +234,17 @@ int Http2Session::disconnect(bool hard) {
return 0;
}
int Http2Session::check_cert() { return ssl::check_cert(conn_.tls.ssl); }
int Http2Session::check_cert() {
return ssl::check_cert(
conn_.tls.ssl,
&get_config()->downstream_addr_groups[group_].addrs[addr_idx_]);
}
int Http2Session::initiate_connection() {
int rv = 0;
auto &addrs = get_config()->downstream_addr_groups[group_].addrs;
if (state_ == DISCONNECTED) {
if (connect_blocker_->blocked()) {
if (LOG_ENABLED(INFO)) {
@ -247,18 +254,19 @@ int Http2Session::initiate_connection() {
return -1;
}
auto worker_stat = worker_->get_worker_stat();
addr_idx_ = worker_stat->next_downstream;
++worker_stat->next_downstream;
worker_stat->next_downstream %= get_config()->downstream_addrs.size();
auto &next_downstream = worker_->get_dgrp(group_)->next;
addr_idx_ = next_downstream;
if (++next_downstream >= addrs.size()) {
next_downstream = 0;
}
if (LOG_ENABLED(INFO)) {
SSLOG(INFO, this) << "Using downstream address idx=" << addr_idx_
<< " out of " << get_config()->downstream_addrs.size();
<< " out of " << addrs.size();
}
}
auto &downstream_addr = get_config()->downstream_addrs[addr_idx_];
auto &downstream_addr = addrs[addr_idx_];
if (get_config()->downstream_http_proxy_host && state_ == DISCONNECTED) {
if (LOG_ENABLED(INFO)) {
@ -503,7 +511,8 @@ int Http2Session::downstream_connect_proxy() {
if (LOG_ENABLED(INFO)) {
SSLOG(INFO, this) << "Connected to the proxy";
}
auto &downstream_addr = get_config()->downstream_addrs[addr_idx_];
auto &downstream_addr =
get_config()->downstream_addr_groups[group_].addrs[addr_idx_];
std::string req = "CONNECT ";
req += downstream_addr.hostport.get();
@ -1744,4 +1753,8 @@ bool Http2Session::should_hard_fail() const {
size_t Http2Session::get_addr_idx() const { return addr_idx_; }
size_t Http2Session::get_group() const { return group_; }
size_t Http2Session::get_index() const { return index_; }
} // namespace shrpx

11
src/shrpx_http2_session.h
View File

@ -58,7 +58,8 @@ struct StreamData {
class Http2Session {
public:
Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx,
ConnectBlocker *connect_blocker, Worker *worker);
ConnectBlocker *connect_blocker, Worker *worker, size_t group,
size_t idx);
~Http2Session();
int check_cert();
@ -151,6 +152,10 @@ public:
size_t get_addr_idx() const;
size_t get_group() const;
size_t get_index() const;
enum {
// Disconnected
DISCONNECTED,
@ -203,6 +208,10 @@ private:
size_t data_pendinglen_;
// index of get_config()->downstream_addrs this object uses
size_t addr_idx_;
size_t group_;
// index inside group, this is used to pin frontend to certain
// HTTP/2 backend for better throughput.
size_t index_;
int state_;
int connection_check_state_;
bool flow_control_;

43
src/shrpx_http2_upstream.cc
View File

@ -36,6 +36,7 @@
#include "shrpx_config.h"
#include "shrpx_http.h"
#include "shrpx_worker.h"
#include "shrpx_http2_session.h"
#include "http2.h"
#include "util.h"
#include "base64.h"
@ -300,7 +301,15 @@ int Http2Upstream::on_request_headers(Downstream *downstream,
downstream->set_request_method(method_token);
downstream->set_request_http2_scheme(http2::value_to_str(scheme));
downstream->set_request_http2_authority(http2::value_to_str(authority));
downstream->set_request_path(http2::value_to_str(path));
if (path) {
if (get_config()->http2_proxy || get_config()->client_proxy) {
downstream->set_request_path(http2::value_to_str(path));
} else {
auto &value = path->value;
downstream->set_request_path(
http2::rewrite_clean_path(std::begin(value), std::end(value)));
}
}
if (!(frame->hd.flags & NGHTTP2_FLAG_END_STREAM)) {
downstream->set_request_http2_expect_body(true);
@ -334,7 +343,7 @@ void Http2Upstream::initiate_downstream(Downstream *downstream) {
int rv;
rv = downstream->attach_downstream_connection(
handler_->get_downstream_connection());
handler_->get_downstream_connection(downstream));
if (rv != 0) {
// downstream connection fails, send error page
if (error_reply(downstream, 503) != 0) {
@ -541,7 +550,8 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame,
{nv.value, nv.value + nv.valuelen});
break;
case http2::HD__PATH:
downstream->set_request_path({nv.value, nv.value + nv.valuelen});
downstream->set_request_path(
http2::rewrite_clean_path(nv.value, nv.value + nv.valuelen));
break;
}
downstream->add_request_header(nv.name, nv.namelen, nv.value, nv.valuelen,
@ -874,16 +884,6 @@ ClientHandler *Http2Upstream::get_client_handler() const { return handler_; }
int Http2Upstream::downstream_read(DownstreamConnection *dconn) {
auto downstream = dconn->get_downstream();
if (downstream->get_request_state() == Downstream::STREAM_CLOSED) {
// If upstream HTTP2 stream was closed, we just close downstream,
// because there is no consumer now. Downstream connection is also
// closed in this case.
remove_downstream(downstream);
// downstream was deleted
return 0;
}
if (downstream->get_response_state() == Downstream::MSG_RESET) {
// The downstream stream was reset (canceled). In this case,
// RST_STREAM to the upstream and delete downstream connection
@ -949,14 +949,6 @@ int Http2Upstream::downstream_eof(DownstreamConnection *dconn) {
if (LOG_ENABLED(INFO)) {
DCLOG(INFO, dconn) << "EOF. stream_id=" << downstream->get_stream_id();
}
if (downstream->get_request_state() == Downstream::STREAM_CLOSED) {
// If stream was closed already, we don't need to send reply at
// the first place. We can delete downstream.
remove_downstream(downstream);
// downstream was deleted
return 0;
}
// Delete downstream connection. If we don't delete it here, it will
// be pooled in on_stream_close_callback.
@ -1002,13 +994,6 @@ int Http2Upstream::downstream_error(DownstreamConnection *dconn, int events) {
}
}
if (downstream->get_request_state() == Downstream::STREAM_CLOSED) {
remove_downstream(downstream);
// downstream was deleted
return 0;
}
// Delete downstream connection. If we don't delete it here, it will
// be pooled in on_stream_close_callback.
downstream->pop_downstream_connection();
@ -1476,7 +1461,7 @@ int Http2Upstream::on_downstream_reset(bool no_retry) {
// downstream connection.
rv = downstream->attach_downstream_connection(
handler_->get_downstream_connection());
handler_->get_downstream_connection(downstream));
if (rv != 0) {
goto fail;
}

46
src/shrpx_http_downstream_connection.cc
View File

@ -34,6 +34,7 @@
#include "shrpx_connect_blocker.h"
#include "shrpx_downstream_connection_pool.h"
#include "shrpx_worker.h"
#include "shrpx_http2_session.h"
#include "http2.h"
#include "util.h"
@ -109,21 +110,15 @@ void connectcb(struct ev_loop *loop, ev_io *w, int revents) {
} // namespace
HttpDownstreamConnection::HttpDownstreamConnection(
DownstreamConnectionPool *dconn_pool, struct ev_loop *loop)
DownstreamConnectionPool *dconn_pool, size_t group, struct ev_loop *loop)
: DownstreamConnection(dconn_pool),
conn_(loop, -1, nullptr, get_config()->downstream_write_timeout,
get_config()->downstream_read_timeout, 0, 0, 0, 0, connectcb,
readcb, timeoutcb, this),
ioctrl_(&conn_.rlimit), response_htp_{0}, addr_idx_(0),
ioctrl_(&conn_.rlimit), response_htp_{0}, group_(group), addr_idx_(0),
connected_(false) {}
HttpDownstreamConnection::~HttpDownstreamConnection() {
// Downstream and DownstreamConnection may be deleted
// asynchronously.
if (downstream_) {
downstream_->release_downstream_connection();
}
}
HttpDownstreamConnection::~HttpDownstreamConnection() {}
int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
if (LOG_ENABLED(INFO)) {
@ -142,15 +137,17 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
}
auto worker = client_handler_->get_worker();
auto worker_stat = worker->get_worker_stat();
auto end = worker_stat->next_downstream;
auto &next_downstream = worker->get_dgrp(group_)->next;
auto end = next_downstream;
auto &addrs = get_config()->downstream_addr_groups[group_].addrs;
for (;;) {
auto i = worker_stat->next_downstream;
++worker_stat->next_downstream;
worker_stat->next_downstream %= get_config()->downstream_addrs.size();
auto &addr = addrs[next_downstream];
auto i = next_downstream;
if (++next_downstream >= addrs.size()) {
next_downstream = 0;
}
conn_.fd = util::create_nonblock_socket(
get_config()->downstream_addrs[i].addr.storage.ss_family);
conn_.fd = util::create_nonblock_socket(addr.addr.storage.ss_family);
if (conn_.fd == -1) {
auto error = errno;
@ -162,8 +159,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
}
int rv;
rv = connect(conn_.fd, &get_config()->downstream_addrs[i].addr.sa,
get_config()->downstream_addrs[i].addrlen);
rv = connect(conn_.fd, &addr.addr.sa, addr.addrlen);
if (rv != 0 && errno != EINPROGRESS) {
auto error = errno;
DCLOG(WARN, this) << "connect() failed; errno=" << error;
@ -172,7 +168,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
close(conn_.fd);
conn_.fd = -1;
if (end == worker_stat->next_downstream) {
if (end == next_downstream) {
return SHRPX_ERR_NETWORK;
}
@ -203,6 +199,8 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
ev_set_cb(&conn_.rev, readcb);
conn_.rt.repeat = get_config()->downstream_read_timeout;
// we may set read timer cb to idle_timeoutcb. Reset again.
ev_set_cb(&conn_.rt, timeoutcb);
ev_timer_again(conn_.loop, &conn_.rt);
// TODO we should have timeout for connection establishment
ev_timer_again(conn_.loop, &conn_.wt);
@ -212,8 +210,10 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
int HttpDownstreamConnection::push_request_headers() {
const char *authority = nullptr, *host = nullptr;
auto downstream_hostport =
get_config()->downstream_addrs[addr_idx_].hostport.get();
auto downstream_hostport = get_config()
->downstream_addr_groups[group_]
.addrs[addr_idx_]
.hostport.get();
auto connect_method = downstream_->get_request_method() == HTTP_CONNECT;
if (!get_config()->no_host_rewrite && !get_config()->http2_proxy &&
@ -860,6 +860,8 @@ int HttpDownstreamConnection::on_connect() {
DLOG(INFO, this) << "downstream connect failed";
}
downstream_->set_request_state(Downstream::CONNECT_FAIL);
return -1;
}
@ -877,4 +879,6 @@ void HttpDownstreamConnection::on_upstream_change(Upstream *upstream) {}
void HttpDownstreamConnection::signal_write() { conn_.wlimit.startw(); }
size_t HttpDownstreamConnection::get_group() const { return group_; }
} // namespace shrpx

4
src/shrpx_http_downstream_connection.h
View File

@ -39,7 +39,7 @@ class DownstreamConnectionPool;
class HttpDownstreamConnection : public DownstreamConnection {
public:
HttpDownstreamConnection(DownstreamConnectionPool *dconn_pool,
HttpDownstreamConnection(DownstreamConnectionPool *dconn_pool, size_t group,
struct ev_loop *loop);
virtual ~HttpDownstreamConnection();
virtual int attach_downstream(Downstream *downstream);
@ -58,6 +58,7 @@ public:
virtual void on_upstream_change(Upstream *upstream);
virtual int on_priority_change(int32_t pri) { return 0; }
virtual size_t get_group() const;
virtual bool poolable() const { return true; }
@ -68,6 +69,7 @@ private:
Connection conn_;
IOControl ioctrl_;
http_parser response_htp_;
size_t group_;
// index of get_config()->downstream_addrs this object is using
size_t addr_idx_;
bool connected_;

15
src/shrpx_https_upstream.cc
View File

@ -36,6 +36,7 @@
#include "shrpx_error.h"
#include "shrpx_log_config.h"
#include "shrpx_worker.h"
#include "shrpx_http2_session.h"
#include "http2.h"
#include "util.h"
#include "template.h"
@ -218,7 +219,12 @@ void rewrite_request_host_path_from_uri(Downstream *downstream, const char *uri,
path += '?';
path.append(uri + fdata.off, fdata.len);
}
downstream->set_request_path(std::move(path));
if (get_config()->http2_proxy || get_config()->client_proxy) {
downstream->set_request_path(std::move(path));
} else {
downstream->set_request_path(
http2::rewrite_clean_path(std::begin(path), std::end(path)));
}
std::string scheme;
http2::copy_url_component(scheme, &u, UF_SCHEMA, uri);
@ -286,6 +292,9 @@ int htp_hdrs_completecb(http_parser *htp) {
return -1;
}
downstream->set_request_path(
http2::rewrite_clean_path(std::begin(uri), std::end(uri)));
if (upstream->get_client_handler()->get_ssl()) {
downstream->set_request_http2_scheme("https");
} else {
@ -297,7 +306,7 @@ int htp_hdrs_completecb(http_parser *htp) {
}
rv = downstream->attach_downstream_connection(
upstream->get_client_handler()->get_downstream_connection());
upstream->get_client_handler()->get_downstream_connection(downstream));
if (rv != 0) {
downstream->set_request_state(Downstream::CONNECT_FAIL);
@ -993,7 +1002,7 @@ int HttpsUpstream::on_downstream_reset(bool no_retry) {
}
rv = downstream_->attach_downstream_connection(
handler_->get_downstream_connection());
handler_->get_downstream_connection(downstream_.get()));
if (rv != 0) {
goto fail;
}

2
src/shrpx_log.h
View File

@ -123,6 +123,8 @@ enum LogFragmentType {
};
struct LogFragment {
LogFragment(LogFragmentType type, std::unique_ptr<char[]> value = nullptr)
: type(type), value(std::move(value)) {}
LogFragmentType type;
std::unique_ptr<char[]> value;
};

36
src/shrpx_spdy_upstream.cc
View File

@ -229,7 +229,12 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type,
} else {
downstream->set_request_http2_scheme(scheme->value);
downstream->set_request_http2_authority(host->value);
downstream->set_request_path(path->value);
if (get_config()->http2_proxy || get_config()->client_proxy) {
downstream->set_request_path(path->value);
} else {
downstream->set_request_path(http2::rewrite_clean_path(
std::begin(path->value), std::end(path->value)));
}
}
if (!(frame->syn_stream.hd.flags & SPDYLAY_CTRL_FLAG_FIN)) {
@ -271,7 +276,7 @@ void SpdyUpstream::start_downstream(Downstream *downstream) {
void SpdyUpstream::initiate_downstream(Downstream *downstream) {
int rv = downstream->attach_downstream_connection(
handler_->get_downstream_connection());
handler_->get_downstream_connection(downstream));
if (rv != 0) {
// If downstream connection fails, issue RST_STREAM.
rst_stream(downstream, SPDYLAY_INTERNAL_ERROR);
@ -555,16 +560,6 @@ ClientHandler *SpdyUpstream::get_client_handler() const { return handler_; }
int SpdyUpstream::downstream_read(DownstreamConnection *dconn) {
auto downstream = dconn->get_downstream();
if (downstream->get_request_state() == Downstream::STREAM_CLOSED) {
// If upstream SPDY stream was closed, we just close downstream,
// because there is no consumer now. Downstream connection is also
// closed in this case.
remove_downstream(downstream);
// downstrea was deleted
return 0;
}
if (downstream->get_response_state() == Downstream::MSG_RESET) {
// The downstream stream was reset (canceled). In this case,
// RST_STREAM to the upstream and delete downstream connection
@ -628,14 +623,6 @@ int SpdyUpstream::downstream_eof(DownstreamConnection *dconn) {
if (LOG_ENABLED(INFO)) {
DCLOG(INFO, dconn) << "EOF. stream_id=" << downstream->get_stream_id();
}
if (downstream->get_request_state() == Downstream::STREAM_CLOSED) {
// If stream was closed already, we don't need to send reply at
// the first place. We can delete downstream.
remove_downstream(downstream);
// downstream was deleted
return 0;
}
// Delete downstream connection. If we don't delete it here, it will
// be pooled in on_stream_close_callback.
@ -681,13 +668,6 @@ int SpdyUpstream::downstream_error(DownstreamConnection *dconn, int events) {
}
}
if (downstream->get_request_state() == Downstream::STREAM_CLOSED) {
remove_downstream(downstream);
// downstream was deleted
return 0;
}
// Delete downstream connection. If we don't delete it here, it will
// be pooled in on_stream_close_callback.
downstream->pop_downstream_connection();
@ -1104,7 +1084,7 @@ int SpdyUpstream::on_downstream_reset(bool no_retry) {
// downstream connection.
rv = downstream->attach_downstream_connection(
handler_->get_downstream_connection());
handler_->get_downstream_connection(downstream));
if (rv != 0) {
goto fail;
}

7
src/shrpx_ssl.cc
View File

@ -53,6 +53,7 @@
#include "shrpx_config.h"
#include "shrpx_worker.h"
#include "shrpx_downstream_connection_pool.h"
#include "shrpx_http2_session.h"
#include "util.h"
#include "ssl.h"
#include "template.h"
@ -743,7 +744,7 @@ void get_altnames(X509 *cert, std::vector<std::string> &dns_names,
}
}
int check_cert(SSL *ssl) {
int check_cert(SSL *ssl, const DownstreamAddr *addr) {
auto cert = SSL_get_peer_certificate(ssl);
if (!cert) {
LOG(ERROR) << "No certificate found";
@ -760,9 +761,7 @@ int check_cert(SSL *ssl) {
std::vector<std::string> dns_names;
std::vector<std::string> ip_addrs;
get_altnames(cert, dns_names, ip_addrs, common_name);
if (verify_hostname(get_config()->downstream_addrs[0].host.get(),
&get_config()->downstream_addrs[0].addr,
get_config()->downstream_addrs[0].addrlen, dns_names,
if (verify_hostname(addr->host.get(), &addr->addr, addr->addrlen, dns_names,
ip_addrs, common_name) != 0) {
LOG(ERROR) << "Certificate verification failed: hostname does not match";
return -1;

3
src/shrpx_ssl.h
View File

@ -40,6 +40,7 @@ namespace shrpx {
class ClientHandler;
class Worker;
class DownstreamConnectionPool;
struct DownstreamAddr;
namespace ssl {
@ -68,7 +69,7 @@ ClientHandler *accept_connection(Worker *worker, int fd, sockaddr *addr,
// Check peer's certificate against first downstream address in
// Config::downstream_addrs. We only consider first downstream since
// we use this function for HTTP/2 downstream link only.
int check_cert(SSL *ssl);
int check_cert(SSL *ssl, const DownstreamAddr *addr);
// Retrieves DNS and IP address in subjectAltNames and commonName from
// the |cert|.

40
src/shrpx_worker.cc
View File

@ -61,8 +61,11 @@ void mcpool_clear_cb(struct ev_loop *loop, ev_timer *w, int revents) {
Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
ssl::CertLookupTree *cert_tree,
const std::shared_ptr<TicketKeys> &ticket_keys)
: next_http2session_(0), loop_(loop), sv_ssl_ctx_(sv_ssl_ctx),
cl_ssl_ctx_(cl_ssl_ctx), cert_tree_(cert_tree), ticket_keys_(ticket_keys),
: dconn_pool_(get_config()->downstream_addr_groups.size()),
worker_stat_(get_config()->downstream_addr_groups.size()),
dgrps_(get_config()->downstream_addr_groups.size()), loop_(loop),
sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), cert_tree_(cert_tree),
ticket_keys_(ticket_keys),
connect_blocker_(make_unique<ConnectBlocker>(loop_)),
graceful_shutdown_(false) {
ev_async_init(&w_, eventcb);
@ -74,9 +77,17 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
if (get_config()->downstream_proto == PROTO_HTTP2) {
auto n = get_config()->http2_downstream_connections_per_worker;
for (; n > 0; --n) {
http2sessions_.push_back(make_unique<Http2Session>(
loop_, cl_ssl_ctx, connect_blocker_.get(), this));
size_t group = 0;
for (auto &dgrp : dgrps_) {
auto m = n;
if (m == 0) {
m = get_config()->downstream_addr_groups[group].addrs.size();
}
for (size_t idx = 0; idx < m; ++idx) {
dgrp.http2sessions.push_back(make_unique<Http2Session>(
loop_, cl_ssl_ctx, connect_blocker_.get(), this, group, idx));
}
++group;
}
}
}
@ -207,15 +218,17 @@ WorkerStat *Worker::get_worker_stat() { return &worker_stat_; }
DownstreamConnectionPool *Worker::get_dconn_pool() { return &dconn_pool_; }
Http2Session *Worker::next_http2_session() {
if (http2sessions_.empty()) {
Http2Session *Worker::next_http2_session(size_t group) {
auto &dgrp = dgrps_[group];
auto &http2sessions = dgrp.http2sessions;
if (http2sessions.empty()) {
return nullptr;
}
auto res = http2sessions_[next_http2session_].get();
++next_http2session_;
if (next_http2session_ >= http2sessions_.size()) {
next_http2session_ = 0;
auto res = http2sessions[dgrp.next_http2session].get();
++dgrp.next_http2session;
if (dgrp.next_http2session >= http2sessions.size()) {
dgrp.next_http2session = 0;
}
return res;
@ -239,4 +252,9 @@ bool Worker::get_graceful_shutdown() const { return graceful_shutdown_; }
MemchunkPool *Worker::get_mcpool() { return &mcpool_; }
DownstreamGroup *Worker::get_dgrp(size_t group) {
assert(group < dgrps_.size());
return &dgrps_[group];
}
} // namespace shrpx

24
src/shrpx_worker.h
View File

@ -54,14 +54,21 @@ namespace ssl {
class CertLookupTree;
} // namespace ssl
struct DownstreamGroup {
DownstreamGroup() : next_http2session(0), next(0) {}
std::vector<std::unique_ptr<Http2Session>> http2sessions;
// Next index in http2sessions.
size_t next_http2session;
// Next downstream address index corresponding to
// Config::downstream_addr_groups[].
size_t next;
};
struct WorkerStat {
WorkerStat() : num_connections(0), next_downstream(0) {}
WorkerStat(size_t num_groups) : num_connections(0) {}
size_t num_connections;
// Next downstream index in Config::downstream_addrs. For HTTP/2
// downstream connections, this is always 0. For HTTP/1, this is
// used as load balancing.
size_t next_downstream;
};
enum WorkerEventType {
@ -97,7 +104,7 @@ public:
void set_ticket_keys(std::shared_ptr<TicketKeys> ticket_keys);
WorkerStat *get_worker_stat();
DownstreamConnectionPool *get_dconn_pool();
Http2Session *next_http2_session();
Http2Session *next_http2_session(size_t group);
ConnectBlocker *get_connect_blocker() const;
struct ev_loop *get_loop() const;
SSL_CTX *get_sv_ssl_ctx() const;
@ -109,9 +116,9 @@ public:
MemchunkPool *get_mcpool();
void schedule_clear_mcpool();
DownstreamGroup *get_dgrp(size_t group);
private:
std::vector<std::unique_ptr<Http2Session>> http2sessions_;
size_t next_http2session_;
#ifndef NOTHREADS
std::future<void> fut_;
#endif // NOTHREADS
@ -122,6 +129,7 @@ private:
MemchunkPool mcpool_;
DownstreamConnectionPool dconn_pool_;
WorkerStat worker_stat_;
std::vector<DownstreamGroup> dgrps_;
struct ev_loop *loop_;
// Following fields are shared across threads if

2
src/timegm.c
View File

@ -34,7 +34,7 @@ static int count_leap_year(int y) {
}
/* Based on the algorithm of Python 2.7 calendar.timegm. */
time_t timegm(struct tm *tm) {
time_t nghttp2_timegm(struct tm *tm) {
int days;
int num_leap_year;
int64_t t;

2
src/timegm.h
View File

@ -37,7 +37,7 @@ extern "C" {
#include <time.h>
#endif // HAVE_TIME_H
time_t timegm(struct tm *tm);
time_t nghttp2_timegm(struct tm *tm);
#ifdef __cplusplus
}

6
src/util.cc
View File

@ -278,7 +278,7 @@ std::string common_log_date(time_t t) {
#ifdef HAVE_STRUCT_TM_TM_GMTOFF
auto gmtoff = tms.tm_gmtoff;
#else // !HAVE_STRUCT_TM_TM_GMTOFF
auto gmtoff = timegm(&tms) - t;
auto gmtoff = nghttp2_timegm(&tms) - t;
#endif // !HAVE_STRUCT_TM_TM_GMTOFF
if (gmtoff >= 0) {
*p++ = '+';
@ -326,7 +326,7 @@ std::string iso8601_date(int64_t ms) {
#ifdef HAVE_STRUCT_TM_TM_GMTOFF
auto gmtoff = tms.tm_gmtoff;
#else // !HAVE_STRUCT_TM_TM_GMTOFF
auto gmtoff = timegm(&tms) - sec;
auto gmtoff = nghttp2_timegm(&tms) - sec;
#endif // !HAVE_STRUCT_TM_TM_GMTOFF
if (gmtoff == 0) {
*p++ = 'Z';
@ -354,7 +354,7 @@ time_t parse_http_date(const std::string &s) {
if (r == 0) {
return 0;
}
return timegm(&tm);
return nghttp2_timegm(&tm);
}
namespace {

6
src/util.h
View File

@ -386,9 +386,13 @@ bool streq_l(const char (&a)[N], InputIt b, size_t blen) {
bool strifind(const char *a, const char *b);
template <typename InputIt> void inp_strlower(InputIt first, InputIt last) {
std::transform(first, last, first, lowcase);
}
// Lowercase |s| in place.
inline void inp_strlower(std::string &s) {
std::transform(std::begin(s), std::end(s), std::begin(s), lowcase);
inp_strlower(std::begin(s), std::end(s));
}
// Returns string representation of |n| with 2 fractional digits.