From 64ffc1fc732f0aaa6de2490dff85365aba148b16 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 7 Feb 2016 21:20:05 +0900 Subject: [PATCH 01/85] Update README.rst --- README.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 33a5cfdd..2f70edf1 100644 --- a/README.rst +++ b/README.rst @@ -639,7 +639,9 @@ push. `_ in TLS, such as session IDs, session tickets (with automatic key rotation), OCSP stapling, dynamic record sizing, ALPN/NPN, forward secrecy and SPDY & -HTTP/2. +HTTP/2. ``nghttpx`` also offers the functionality to share session +cache and ticket keys among multiple ``nghttpx`` instances via +memcached. ``nghttpx`` has several operational modes: @@ -661,7 +663,9 @@ The default mode, ``--http2-proxy`` and ``--http2-bridge`` modes use SSL/TLS in the frontend connection by default. To disable SSL/TLS, use the ``--frontend-no-tls`` option. If that option is used, SPDY is disabled in the frontend and incoming HTTP/1.1 connections can be -upgraded to HTTP/2 through HTTP Upgrade. +upgraded to HTTP/2 through HTTP Upgrade. In these modes, HTTP/1 +backend connections are cleartext by default. To enable TLS, use +``--backend-http1-tls`` opiton. The ``--http2-bridge``, ``--client`` and ``--client-proxy`` modes use SSL/TLS in the backend connection by default. To disable SSL/TLS, use From 92e66fc167abac7bf003cf95012fe711edaedee4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 7 Feb 2016 21:20:44 +0900 Subject: [PATCH 02/85] Bump up version number to 1.8.0-DEV --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 4a4defe9..c4078790 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.7.1-DEV], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.8.0-DEV], [t-tujikawa@users.sourceforge.net]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_HEADERS([config.h]) From c8b6a79225eb98676e84148aa47008580d4fc352 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 7 Feb 2016 21:24:11 +0900 Subject: [PATCH 03/85] Update man pages --- doc/h2load.1 | 2 +- doc/nghttp.1 | 2 +- doc/nghttpd.1 | 13 +++++- doc/nghttpd.1.rst | 9 ++++ doc/nghttpx.1 | 102 ++++++++++++++++++++++++++++++++++------------ doc/nghttpx.1.rst | 93 ++++++++++++++++++++++++++++++------------ 6 files changed, 166 insertions(+), 55 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index 18b49f3d..f9dab89e 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "January 25, 2016" "1.7.0" "nghttp2" +.TH "H2LOAD" "1" "February 07, 2016" "1.8.0-DEV" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index cc302ba7..ad205088 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "January 25, 2016" "1.7.0" "nghttp2" +.TH "NGHTTP" "1" "February 07, 2016" "1.8.0-DEV" "nghttp2" .SH NAME nghttp \- HTTP/2 client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index dceb0a0a..e93ccc8a 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "January 25, 2016" "1.7.0" "nghttp2" +.TH "NGHTTPD" "1" "February 07, 2016" "1.8.0-DEV" "nghttp2" .SH NAME nghttpd \- HTTP/2 server . @@ -139,6 +139,17 @@ Make error response gzipped. .UNINDENT .INDENT 0.0 .TP +.B \-w, \-\-window\-bits= +Sets the stream level initial window size to 2**\-1. +.UNINDENT +.INDENT 0.0 +.TP +.B \-W, \-\-connection\-window\-bits= +Sets the connection level initial window size to +2**\-1. +.UNINDENT +.INDENT 0.0 +.TP .B \-\-dh\-param\-file= Path to file that contains DH parameters in PEM format. Without this option, DHE cipher suites are not diff --git a/doc/nghttpd.1.rst b/doc/nghttpd.1.rst index cdb80c31..a059bdea 100644 --- a/doc/nghttpd.1.rst +++ b/doc/nghttpd.1.rst @@ -104,6 +104,15 @@ OPTIONS Make error response gzipped. +.. option:: -w, --window-bits= + + Sets the stream level initial window size to 2\*\*-1. + +.. option:: -W, --connection-window-bits= + + Sets the connection level initial window size to + 2\*\*-1. + .. option:: --dh-param-file= Path to file that contains DH parameters in PEM format. diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index 9c26395c..90ee5a1f 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "January 25, 2016" "1.7.0" "nghttp2" +.TH "NGHTTPX" "1" "February 07, 2016" "1.8.0-DEV" "nghttp2" .SH NAME nghttpx \- HTTP/2 proxy . @@ -121,7 +121,9 @@ Default: \fB127.0.0.1,80\fP Set frontend host and port. If is \(aq*\(aq, it assumes all addresses including both IPv4 and IPv6. UNIX domain socket can be specified by prefixing path -name with "unix:" (e.g., unix:/var/run/nghttpx.sock) +name with "unix:" (e.g., unix:/var/run/nghttpx.sock). +This option can be used multiple times to listen to +multiple addresses. .sp Default: \fB*,3000\fP .UNINDENT @@ -163,6 +165,22 @@ be specified by \fI\%\-\-backend\-read\-timeout\fP and .B \-\-accept\-proxy\-protocol Accept PROXY protocol version 1 on frontend connection. .UNINDENT +.INDENT 0.0 +.TP +.B \-\-backend\-no\-tls +Disable SSL/TLS on backend connections. For HTTP/2 +backend connections, TLS is enabled by default. For +HTTP/1 backend connections, TLS is disabled by default, +and can be enabled by \fI\%\-\-backend\-http1\-tls\fP option. If +both \fI\%\-\-backend\-no\-tls\fP and \fI\%\-\-backend\-http1\-tls\fP options +are used, \fI\%\-\-backend\-no\-tls\fP has the precedence. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-backend\-http1\-tls +Enable SSL/TLS on backend HTTP/1 connections. See also +\fI\%\-\-backend\-no\-tls\fP option. +.UNINDENT .SS Performance .INDENT 0.0 .TP @@ -396,19 +414,17 @@ described in OpenSSL ciphers(1). .INDENT 0.0 .TP .B \-k, \-\-insecure -Don\(aqt verify backend server\(aqs certificate if \fI\%\-p\fP, -\fI\%\-\-client\fP or \fI\%\-\-http2\-bridge\fP are given and -\fI\%\-\-backend\-no\-tls\fP is not given. +Don\(aqt verify backend server\(aqs certificate if TLS is +enabled for backend connections. .UNINDENT .INDENT 0.0 .TP .B \-\-cacert= -Set path to trusted CA certificate file if \fI\%\-p\fP, \fI\%\-\-client\fP -or \fI\%\-\-http2\-bridge\fP are given and \fI\%\-\-backend\-no\-tls\fP is not -given. The file must be in PEM format. It can contain -multiple certificates. If the linked OpenSSL is -configured to load system wide certificates, they are -loaded at startup regardless of this option. +Set path to trusted CA certificate file used in backend +TLS connections. The file must be in PEM format. It +can contain multiple certificates. If the linked +OpenSSL is configured to load system wide certificates, +they are loaded at startup regardless of this option. .UNINDENT .INDENT 0.0 .TP @@ -616,6 +632,21 @@ TLS HTTP/2 backends. .sp Default: \fB1s\fP .UNINDENT +.INDENT 0.0 +.TP +.B \-\-no\-http2\-cipher\-black\-list +Allow black listed cipher suite on HTTP/2 connection. +See \fI\%https://tools.ietf.org/html/rfc7540#appendix\-A\fP for +the complete HTTP/2 cipher suites black list. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-backend\-tls\-session\-cache\-per\-worker= +Set the maximum number of backend TLS session cache +stored per worker. +.sp +Default: \fB10000\fP +.UNINDENT .SS HTTP/2 and SPDY .INDENT 0.0 .TP @@ -666,11 +697,6 @@ Default: \fB16\fP .UNINDENT .INDENT 0.0 .TP -.B \-\-backend\-no\-tls -Disable SSL/TLS on backend connections. -.UNINDENT -.INDENT 0.0 -.TP .B \-\-http2\-no\-cookie\-crumbling Don\(aqt crumble cookie header field. .UNINDENT @@ -868,11 +894,12 @@ Specify the parameter value sent out with "by" parameter of Forwarded header field. If "obfuscated" is given, the string is randomly generated at startup. If "ip" is given, the interface address of the connection, -including port number, is sent with "by" parameter. -User can also specify the static obfuscated string. The -limitation is that it must start with "_", and only -consists of character set [A\-Za\-z0\-9._\-], as described -in RFC 7239. +including port number, is sent with "by" parameter. In +case of UNIX domain socket, "localhost" is used instead +of address and port. User can also specify the static +obfuscated string. The limitation is that it must start +with "_", and only consists of character set +[A\-Za\-z0\-9._\-], as described in RFC 7239. .sp Default: \fBobfuscated\fP .UNINDENT @@ -884,7 +911,8 @@ parameter of Forwarded header field. If "obfuscated" is given, the string is randomly generated for each client connection. If "ip" is given, the remote client address of the connection, without port number, is sent with -"for" parameter. +"for" parameter. In case of UNIX domain socket, +"localhost" is used instead of address. .sp Default: \fBobfuscated\fP .UNINDENT @@ -940,22 +968,42 @@ Example: \fI\%\-\-add\-response\-header\fP="foo: bar" .UNINDENT .INDENT 0.0 .TP -.B \-\-header\-field\-buffer= +.B \-\-request\-header\-field\-buffer= Set maximum buffer size for incoming HTTP request header field list. This is the sum of header name and value in -bytes. +bytes. If trailer fields exist, they are counted +towards this number. .sp Default: \fB64K\fP .UNINDENT .INDENT 0.0 .TP -.B \-\-max\-header\-fields= +.B \-\-max\-request\-header\-fields= Set maximum number of incoming HTTP request header -fields, which appear in one request or response header -field list. +fields. If trailer fields exist, they are counted +towards this number. .sp Default: \fB100\fP .UNINDENT +.INDENT 0.0 +.TP +.B \-\-response\-header\-field\-buffer= +Set maximum buffer size for incoming HTTP response +header field list. This is the sum of header name and +value in bytes. If trailer fields exist, they are +counted towards this number. +.sp +Default: \fB64K\fP +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-max\-response\-header\-fields= +Set maximum number of incoming HTTP response header +fields. If trailer fields exist, they are counted +towards this number. +.sp +Default: \fB500\fP +.UNINDENT .SS Debug .INDENT 0.0 .TP diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index ba5ac4ed..ffafa2bf 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -104,7 +104,9 @@ Connections Set frontend host and port. If is '\*', it assumes all addresses including both IPv4 and IPv6. UNIX domain socket can be specified by prefixing path - name with "unix:" (e.g., unix:/var/run/nghttpx.sock) + name with "unix:" (e.g., unix:/var/run/nghttpx.sock). + This option can be used multiple times to listen to + multiple addresses. Default: ``*,3000`` @@ -141,6 +143,20 @@ Connections Accept PROXY protocol version 1 on frontend connection. +.. option:: --backend-no-tls + + Disable SSL/TLS on backend connections. For HTTP/2 + backend connections, TLS is enabled by default. For + HTTP/1 backend connections, TLS is disabled by default, + and can be enabled by :option:`--backend-http1-tls` option. If + both :option:`--backend-no-tls` and :option:`\--backend-http1-tls` options + are used, :option:`--backend-no-tls` has the precedence. + +.. option:: --backend-http1-tls + + Enable SSL/TLS on backend HTTP/1 connections. See also + :option:`--backend-no-tls` option. + Performance ~~~~~~~~~~~ @@ -354,18 +370,16 @@ SSL/TLS .. option:: -k, --insecure - Don't verify backend server's certificate if :option:`-p`\, - :option:`--client` or :option:`\--http2-bridge` are given and - :option:`--backend-no-tls` is not given. + Don't verify backend server's certificate if TLS is + enabled for backend connections. .. option:: --cacert= - Set path to trusted CA certificate file if :option:`-p`\, :option:`--client` - or :option:`--http2-bridge` are given and :option:`\--backend-no-tls` is not - given. The file must be in PEM format. It can contain - multiple certificates. If the linked OpenSSL is - configured to load system wide certificates, they are - loaded at startup regardless of this option. + Set path to trusted CA certificate file used in backend + TLS connections. The file must be in PEM format. It + can contain multiple certificates. If the linked + OpenSSL is configured to load system wide certificates, + they are loaded at startup regardless of this option. .. option:: --private-key-passwd-file= @@ -551,6 +565,19 @@ SSL/TLS Default: ``1s`` +.. option:: --no-http2-cipher-black-list + + Allow black listed cipher suite on HTTP/2 connection. + See https://tools.ietf.org/html/rfc7540#appendix-A for + the complete HTTP/2 cipher suites black list. + +.. option:: --backend-tls-session-cache-per-worker= + + Set the maximum number of backend TLS session cache + stored per worker. + + Default: ``10000`` + HTTP/2 and SPDY ~~~~~~~~~~~~~~~ @@ -596,10 +623,6 @@ HTTP/2 and SPDY Default: ``16`` -.. option:: --backend-no-tls - - Disable SSL/TLS on backend connections. - .. option:: --http2-no-cookie-crumbling Don't crumble cookie header field. @@ -773,11 +796,12 @@ HTTP of Forwarded header field. If "obfuscated" is given, the string is randomly generated at startup. If "ip" is given, the interface address of the connection, - including port number, is sent with "by" parameter. - User can also specify the static obfuscated string. The - limitation is that it must start with "_", and only - consists of character set [A-Za-z0-9._-], as described - in RFC 7239. + including port number, is sent with "by" parameter. In + case of UNIX domain socket, "localhost" is used instead + of address and port. User can also specify the static + obfuscated string. The limitation is that it must start + with "_", and only consists of character set + [A-Za-z0-9._-], as described in RFC 7239. Default: ``obfuscated`` @@ -788,7 +812,8 @@ HTTP given, the string is randomly generated for each client connection. If "ip" is given, the remote client address of the connection, without port number, is sent with - "for" parameter. + "for" parameter. In case of UNIX domain socket, + "localhost" is used instead of address. Default: ``obfuscated`` @@ -836,22 +861,40 @@ HTTP used several times to specify multiple header fields. Example: :option:`--add-response-header`\="foo: bar" -.. option:: --header-field-buffer= +.. option:: --request-header-field-buffer= Set maximum buffer size for incoming HTTP request header field list. This is the sum of header name and value in - bytes. + bytes. If trailer fields exist, they are counted + towards this number. Default: ``64K`` -.. option:: --max-header-fields= +.. option:: --max-request-header-fields= Set maximum number of incoming HTTP request header - fields, which appear in one request or response header - field list. + fields. If trailer fields exist, they are counted + towards this number. Default: ``100`` +.. option:: --response-header-field-buffer= + + Set maximum buffer size for incoming HTTP response + header field list. This is the sum of header name and + value in bytes. If trailer fields exist, they are + counted towards this number. + + Default: ``64K`` + +.. option:: --max-response-header-fields= + + Set maximum number of incoming HTTP response header + fields. If trailer fields exist, they are counted + towards this number. + + Default: ``500`` + Debug ~~~~~ From c3a5fe71856d4f5aa75c7efbfa5ce789f4c294a3 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 7 Feb 2016 21:24:29 +0900 Subject: [PATCH 04/85] Update bash_completion --- doc/bash_completion/nghttpd | 2 +- doc/bash_completion/nghttpx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/bash_completion/nghttpd b/doc/bash_completion/nghttpd index c2087b19..e68514cf 100644 --- a/doc/bash_completion/nghttpd +++ b/doc/bash_completion/nghttpd @@ -8,7 +8,7 @@ _nghttpd() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--htdocs --verbose --daemon --echo-upload --error-gzip --push --header-table-size --padding --hexdump --max-concurrent-streams --no-tls --mime-types-file --no-content-length --workers --version --color --early-response --dh-param-file --trailer --address --verify-client --help ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--htdocs --verbose --daemon --echo-upload --error-gzip --push --header-table-size --padding --hexdump --max-concurrent-streams --no-tls --connection-window-bits --mime-types-file --no-content-length --workers --version --color --early-response --dh-param-file --trailer --address --window-bits --verify-client --help ' -- "$cur" ) ) ;; *) _filedir diff --git a/doc/bash_completion/nghttpx b/doc/bash_completion/nghttpx index 82e34d57..e127d178 100644 --- a/doc/bash_completion/nghttpx +++ b/doc/bash_completion/nghttpx @@ -8,7 +8,7 @@ _nghttpx() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--worker-read-rate --frontend-no-tls --frontend-http2-dump-response-header --backend-http1-connections-per-frontend --tls-ticket-key-file --verify-client-cacert --include --backend-request-buffer --backend-http2-connection-window-bits --conf --worker-write-burst --npn-list --fetch-ocsp-response-file --mruby-file --stream-read-timeout --tls-ticket-key-memcached --forwarded-for --accesslog-syslog --frontend-http2-read-timeout --listener-disable-timeout --frontend-http2-connection-window-bits --ciphers --strip-incoming-x-forwarded-for --private-key-passwd-file --backend-keep-alive-timeout --backend-http-proxy-uri --backend-http1-connections-per-host --rlimit-nofile --tls-dyn-rec-warmup-threshold --no-via --ocsp-update-interval --backend-write-timeout --client --tls-ticket-key-memcached-max-retry --http2-no-cookie-crumbling --worker-read-burst --client-proxy --http2-bridge --accesslog-format --errorlog-syslog --errorlog-file --http2-max-concurrent-streams --frontend-write-timeout --tls-ticket-key-cipher --read-burst --backend-ipv4 --backend-ipv6 --backend --insecure --log-level --host-rewrite --tls-proto-list --backend-http2-connections-per-worker --tls-ticket-key-memcached-interval --dh-param-file --worker-frontend-connections --syslog-facility --fastopen --no-location-rewrite --tls-session-cache-memcached --no-ocsp --backend-response-buffer --workers --add-forwarded --frontend-http2-window-bits --worker-write-rate --add-request-header --backend-tls-sni-field --subcert --help --frontend-frame-debug --pid-file --frontend-http2-dump-request-header --daemon --write-rate --altsvc --user --add-x-forwarded-for --header-field-buffer --frontend-read-timeout --tls-ticket-key-memcached-max-fail --backlog --write-burst --no-server-push --backend-http2-window-bits --padding --stream-write-timeout --cacert --forwarded-by --version --add-response-header --backend-read-timeout --frontend --accesslog-file --http2-proxy --max-header-fields --backend-no-tls --client-private-key-file --client-cert-file --accept-proxy-protocol --tls-dyn-rec-idle-timeout --verify-client --read-rate --strip-incoming-forwarded ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--worker-read-rate --frontend-no-tls --frontend-http2-dump-response-header --backend-http1-connections-per-frontend --tls-ticket-key-file --verify-client-cacert --include --max-response-header-fields --backend-request-buffer --max-request-header-fields --backend-http2-connection-window-bits --backend-tls-session-cache-per-worker --conf --worker-write-burst --npn-list --fetch-ocsp-response-file --no-http2-cipher-black-list --mruby-file --stream-read-timeout --tls-ticket-key-memcached --forwarded-for --accesslog-syslog --frontend-http2-read-timeout --listener-disable-timeout --frontend-http2-connection-window-bits --ciphers --strip-incoming-x-forwarded-for --private-key-passwd-file --backend-keep-alive-timeout --backend-http-proxy-uri --backend-http1-connections-per-host --rlimit-nofile --tls-dyn-rec-warmup-threshold --no-via --ocsp-update-interval --backend-write-timeout --client --tls-ticket-key-memcached-max-retry --http2-no-cookie-crumbling --worker-read-burst --client-proxy --http2-bridge --accesslog-format --errorlog-syslog --request-header-field-buffer --errorlog-file --http2-max-concurrent-streams --frontend-write-timeout --tls-ticket-key-cipher --read-burst --backend-ipv4 --backend-ipv6 --backend --insecure --log-level --host-rewrite --tls-proto-list --backend-http2-connections-per-worker --tls-ticket-key-memcached-interval --dh-param-file --worker-frontend-connections --backend-http1-tls --syslog-facility --fastopen --no-location-rewrite --tls-session-cache-memcached --no-ocsp --backend-response-buffer --workers --add-forwarded --frontend-http2-window-bits --worker-write-rate --add-request-header --backend-tls-sni-field --subcert --help --frontend-frame-debug --pid-file --frontend-http2-dump-request-header --daemon --write-rate --altsvc --user --add-x-forwarded-for --frontend-read-timeout --tls-ticket-key-memcached-max-fail --backlog --write-burst --no-server-push --backend-http2-window-bits --response-header-field-buffer --padding --stream-write-timeout --cacert --forwarded-by --version --add-response-header --backend-read-timeout --frontend --accesslog-file --http2-proxy --backend-no-tls --client-private-key-file --client-cert-file --accept-proxy-protocol --tls-dyn-rec-idle-timeout --verify-client --read-rate --strip-incoming-forwarded ' -- "$cur" ) ) ;; *) _filedir From b8717208c70fc2ba1be0382d653e676e05c9aeb6 Mon Sep 17 00:00:00 2001 From: David Beitey Date: Tue, 9 Feb 2016 16:57:11 +1000 Subject: [PATCH 05/85] Document compiling apps and include h2load in configure --- README.rst | 12 +++++++++--- configure.ac | 2 +- doc/sources/h2load-howto.rst | 9 +++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 2f70edf1..974fc4f0 100644 --- a/README.rst +++ b/README.rst @@ -58,9 +58,9 @@ To build the documentation, you need to install: * sphinx (http://sphinx-doc.org/) -To build and run the application programs (``nghttp``, ``nghttpd`` and -``nghttpx``) in the ``src`` directory, the following packages are -required: +To build and run the application programs (``nghttp``, ``nghttpd``, +``nghttpx`` and ``h2load``) in the ``src`` directory, the following packages +are required: * OpenSSL >= 1.0.1 * libev >= 4.15 @@ -160,6 +160,12 @@ To compile the source code, gcc >= 4.8.3 or clang >= 3.4 is required. them from crashing. A patch is welcome to make multi threading work on Mac OS X platform. +.. note:: + + To compile the associated applications (nghttp, nghttpd, nghttpx and + h2load), you must use the ``--enable-app`` configure option and ensure + that the specified requirements above are met. + Notes for building on Windows (Mingw/Cygwin) -------------------------------------------- diff --git a/configure.ac b/configure.ac index c4078790..a1ed90c3 100644 --- a/configure.ac +++ b/configure.ac @@ -76,7 +76,7 @@ AC_ARG_ENABLE([threads], AC_ARG_ENABLE([app], [AS_HELP_STRING([--enable-app], - [Build applications (nghttp, nghttpd and nghttpx) [default=check]])], + [Build applications (nghttp, nghttpd, nghttpx and h2load) [default=check]])], [request_app=$enableval], [request_app=check]) AC_ARG_ENABLE([hpack-tools], diff --git a/doc/sources/h2load-howto.rst b/doc/sources/h2load-howto.rst index af3c42e3..f66c2c76 100644 --- a/doc/sources/h2load-howto.rst +++ b/doc/sources/h2load-howto.rst @@ -6,6 +6,15 @@ spdylay (http://tatsuhiro-t.github.io/spdylay/) library, it also supports SPDY protocol. It supports SSL/TLS and clear text for all supported protocols. +Compiling from source +--------------------- + +``h2load`` is compiled alongside ``nghttp2`` and requires that the +``--enable-apps`` flag is passed to ``./configure`` and `required dependencies +`_ are available during +compilation. For details on compiling, see `nghttp2: Building from Git +`_. + Basic Usage ----------- From 396dde134762c5598c8145e817614d46a5539427 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 10 Feb 2016 21:42:32 +0900 Subject: [PATCH 06/85] Mention libspdylay-dev package availability --- README.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 974fc4f0..9f7f3ce3 100644 --- a/README.rst +++ b/README.rst @@ -110,8 +110,9 @@ If you are using Ubuntu 14.04 LTS (trusty) or Debian 7.0 (wheezy) and above run zlib1g-dev libcunit1-dev libssl-dev libxml2-dev libev-dev libevent-dev libjansson-dev \ libjemalloc-dev cython python3-dev python-setuptools -spdylay is not packaged in Ubuntu, so you need to build it yourself: -http://tatsuhiro-t.github.io/spdylay/ +From Ubuntu 15.10, spdylay has been available as a package named +`libspdylay-dev`. For the earlier Ubuntu release, you need to build +it yourself: http://tatsuhiro-t.github.io/spdylay/ To enable mruby support for nghttpx, `mruby `_ is required. We need to build From 00175eac3369819ff3ff77ecb4ca9412c1fc8929 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 11 Feb 2016 12:40:15 +0900 Subject: [PATCH 07/85] nghttpx: Use Address* as a key for client side session cache --- src/shrpx_http_downstream_connection.cc | 4 ++-- src/shrpx_worker.cc | 4 ++-- src/shrpx_worker.h | 23 +++++++++++------------ 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 9cb5eee0..f5f17dec 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -133,7 +133,7 @@ HttpDownstreamConnection::~HttpDownstreamConnection() { if (conn_.tls.ssl) { auto session = SSL_get1_session(conn_.tls.ssl); if (session) { - worker_->cache_downstream_tls_session(addr_, session); + worker_->cache_downstream_tls_session(&addr_->addr, session); } } } @@ -218,7 +218,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str()); } - auto session = worker_->reuse_downstream_tls_session(addr_); + auto session = worker_->reuse_downstream_tls_session(&addr_->addr); if (session) { SSL_set_session(conn_.tls.ssl, session); SSL_SESSION_free(session); diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 2204b9af..15083347 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -307,7 +307,7 @@ mruby::MRubyContext *Worker::get_mruby_context() const { } #endif // HAVE_MRUBY -void Worker::cache_downstream_tls_session(const DownstreamAddr *addr, +void Worker::cache_downstream_tls_session(const Address *addr, SSL_SESSION *session) { auto &tlsconf = get_config()->tls; @@ -341,7 +341,7 @@ void Worker::cache_downstream_tls_session(const DownstreamAddr *addr, ++downstream_tls_session_cache_size_; } -SSL_SESSION *Worker::reuse_downstream_tls_session(const DownstreamAddr *addr) { +SSL_SESSION *Worker::reuse_downstream_tls_session(const Address *addr) { auto it = downstream_tls_session_cache_.find(addr); if (it == std::end(downstream_tls_session_cache_)) { return nullptr; diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 3939b4c8..a967caf6 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -145,16 +145,15 @@ public: mruby::MRubyContext *get_mruby_context() const; #endif // HAVE_MRUBY - // Caches |session| which is associated to downstream address - // |addr|. The caller is responsible to increment the reference - // count of |session|, since this function does not do so. - void cache_downstream_tls_session(const DownstreamAddr *addr, - SSL_SESSION *session); + // Caches |session| which is associated to remote address |addr|. + // The caller is responsible to increment the reference count of + // |session|, since this function does not do so. + void cache_downstream_tls_session(const Address *addr, SSL_SESSION *session); // Returns cached session associated |addr|. If non-nullptr value // is returned, its cache entry was successfully removed from cache. // If no cache entry is found associated to |addr|, nullptr will be // returned. - SSL_SESSION *reuse_downstream_tls_session(const DownstreamAddr *addr); + SSL_SESSION *reuse_downstream_tls_session(const Address *addr); private: #ifndef NOTHREADS @@ -170,12 +169,12 @@ private: WorkerStat worker_stat_; std::vector dgrps_; - // Cache for SSL_SESSION for downstream connections. SSL_SESSION is - // associated to downstream address. One address has multiple - // SSL_SESSION objects. New SSL_SESSION is appended to the deque. - // When doing eviction due to storage limitation, the SSL_SESSION - // which sits at the front of deque is removed. - std::unordered_map> + // Client side SSL_SESSION cache. SSL_SESSION is associated to + // remote address. One address has multiple SSL_SESSION objects. + // New SSL_SESSION is appended to the deque. When doing eviction + // due to storage limitation, the SSL_SESSION which sits at the + // front of deque is removed. + std::unordered_map> downstream_tls_session_cache_; size_t downstream_tls_session_cache_size_; From ba4c268172bd679cccc99fe43d921990e1bb6738 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 11 Feb 2016 17:07:48 +0900 Subject: [PATCH 08/85] nghttpx: Single SSL_SESSION cache entry for each address --- src/shrpx_http_downstream_connection.cc | 5 +- src/shrpx_worker.cc | 83 ++++++++++++------------- src/shrpx_worker.h | 30 +++++---- src/util.cc | 35 +++++++++++ src/util.h | 5 ++ 5 files changed, 101 insertions(+), 57 deletions(-) diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index f5f17dec..166269ea 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -131,9 +131,10 @@ HttpDownstreamConnection::HttpDownstreamConnection( HttpDownstreamConnection::~HttpDownstreamConnection() { if (conn_.tls.ssl) { - auto session = SSL_get1_session(conn_.tls.ssl); + auto session = SSL_get0_session(conn_.tls.ssl); if (session) { - worker_->cache_downstream_tls_session(&addr_->addr, session); + worker_->cache_downstream_tls_session(&addr_->addr, session, + ev_now(conn_.loop)); } } } diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 15083347..9521e5ac 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -29,6 +29,7 @@ #endif // HAVE_UNISTD_H #include +#include #include "shrpx_ssl.h" #include "shrpx_log.h" @@ -73,7 +74,6 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, dconn_pool_(get_config()->conn.downstream.addr_groups.size()), worker_stat_(get_config()->conn.downstream.addr_groups.size()), dgrps_(get_config()->conn.downstream.addr_groups.size()), - downstream_tls_session_cache_size_(0), loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), @@ -117,12 +117,6 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, Worker::~Worker() { ev_async_stop(loop_, &w_); ev_timer_stop(loop_, &mcpool_clear_timer_); - - for (auto &p : downstream_tls_session_cache_) { - for (auto session : p.second) { - SSL_SESSION_free(session); - } - } } void Worker::schedule_clear_mcpool() { @@ -307,38 +301,51 @@ mruby::MRubyContext *Worker::get_mruby_context() const { } #endif // HAVE_MRUBY -void Worker::cache_downstream_tls_session(const Address *addr, - SSL_SESSION *session) { - auto &tlsconf = get_config()->tls; +namespace { +std::vector serialize_ssl_session(SSL_SESSION *session) { + auto len = i2d_SSL_SESSION(session, nullptr); + auto buf = std::vector(len); + auto p = buf.data(); + i2d_SSL_SESSION(session, &p); - auto max = tlsconf.downstream_session_cache_per_worker; - if (max == 0) { + return buf; +} +} // namespace + +void Worker::cache_downstream_tls_session(const Address *addr, + SSL_SESSION *session, ev_tstamp t) { + auto it = downstream_tls_session_cache_.find(addr); + if (it == std::end(downstream_tls_session_cache_)) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Create cache entry for SSL_SESSION=" << session + << ", addr=" << util::numeric_hostport(&addr->su.sa, addr->len) + << "(" << addr << "), timestamp=" << std::fixed + << std::setprecision(6) << t; + } + downstream_tls_session_cache_.emplace( + addr, SessionCacheEntry{serialize_ssl_session(session), t}); return; } - if (downstream_tls_session_cache_size_ >= max) { - // It is implementation dependent which item is returned from - // std::begin(). Probably, this depends on hash algorithm. If it - // is random fashion, then we are mostly OK. - auto it = std::begin(downstream_tls_session_cache_); - assert(it != std::end(downstream_tls_session_cache_)); - auto &v = (*it).second; - assert(!v.empty()); - auto sess = v.front(); - v.pop_front(); - SSL_SESSION_free(sess); - if (v.empty()) { - downstream_tls_session_cache_.erase(it); + auto &ent = (*it).second; + if (ent.last_updated + 1_min > t) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Cache for addr=" + << util::numeric_hostport(&addr->su.sa, addr->len) << "(" + << addr << ") is still host. Not updating."; } + return; } - auto it = downstream_tls_session_cache_.find(addr); - if (it == std::end(downstream_tls_session_cache_)) { - std::tie(it, std::ignore) = downstream_tls_session_cache_.emplace( - addr, std::deque()); + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Update cache entry for SSL_SESSION=" << session + << ", addr=" << util::numeric_hostport(&addr->su.sa, addr->len) + << "(" << addr << "), timestamp=" << std::fixed + << std::setprecision(6) << t; } - (*it).second.push_back(session); - ++downstream_tls_session_cache_size_; + + ent.session_data = serialize_ssl_session(session); + ent.last_updated = t; } SSL_SESSION *Worker::reuse_downstream_tls_session(const Address *addr) { @@ -347,17 +354,9 @@ SSL_SESSION *Worker::reuse_downstream_tls_session(const Address *addr) { return nullptr; } - auto &v = (*it).second; - assert(!v.empty()); - auto session = v.back(); - v.pop_back(); - --downstream_tls_session_cache_size_; - - if (v.empty()) { - downstream_tls_session_cache_.erase(it); - } - - return session; + const auto &ent = (*it).second; + auto p = ent.session_data.data(); + return d2i_SSL_SESSION(nullptr, &p, ent.session_data.size()); } } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index a967caf6..e4861c03 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -101,6 +101,14 @@ struct WorkerEvent { std::shared_ptr ticket_keys; }; +struct SessionCacheEntry { + // ASN1 representation of SSL_SESSION object. See + // i2d_SSL_SESSION(3SSL). + std::vector session_data; + // The last time stamp when this cache entry is created or updated. + ev_tstamp last_updated; +}; + class Worker { public: Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, @@ -146,13 +154,13 @@ public: #endif // HAVE_MRUBY // Caches |session| which is associated to remote address |addr|. - // The caller is responsible to increment the reference count of - // |session|, since this function does not do so. - void cache_downstream_tls_session(const Address *addr, SSL_SESSION *session); - // Returns cached session associated |addr|. If non-nullptr value - // is returned, its cache entry was successfully removed from cache. - // If no cache entry is found associated to |addr|, nullptr will be - // returned. + // |session| is serialized into ASN1 representation, and stored. + // |t| is used as a time stamp. Depending on the existing cache's + // time stamp, |session| might not be cached. + void cache_downstream_tls_session(const Address *addr, SSL_SESSION *session, + ev_tstamp t); + // Returns cached session associated |addr|. If no cache entry is + // found associated to |addr|, nullptr will be returned. SSL_SESSION *reuse_downstream_tls_session(const Address *addr); private: @@ -170,13 +178,9 @@ private: std::vector dgrps_; // Client side SSL_SESSION cache. SSL_SESSION is associated to - // remote address. One address has multiple SSL_SESSION objects. - // New SSL_SESSION is appended to the deque. When doing eviction - // due to storage limitation, the SSL_SESSION which sits at the - // front of deque is removed. - std::unordered_map> + // remote address. + std::unordered_map downstream_tls_session_cache_; - size_t downstream_tls_session_cache_size_; std::unique_ptr session_cache_memcached_dispatcher_; #ifdef HAVE_MRUBY diff --git a/src/util.cc b/src/util.cc index b2d0ca1c..2f120887 100644 --- a/src/util.cc +++ b/src/util.cc @@ -651,6 +651,41 @@ std::string numeric_name(const struct sockaddr *sa, socklen_t salen) { return host.data(); } +std::string numeric_hostport(const struct sockaddr *sa, socklen_t salen) { + if (sa->sa_family == AF_UNIX) { + return "localhost"; + } + + std::array host; + std::array serv; + auto rv = getnameinfo(sa, salen, host.data(), host.size(), serv.data(), + serv.size(), NI_NUMERICHOST | NI_NUMERICSERV); + if (rv != 0) { + return "unknown"; + } + + auto hostlen = strlen(host.data()); + auto servlen = strlen(serv.data()); + + std::string s; + char *p; + if (sa->sa_family == AF_INET6) { + s.resize(hostlen + servlen + 2 + 1); + p = &s[0]; + *p++ = '['; + p = std::copy_n(host.data(), hostlen, p); + *p++ = ']'; + } else { + s.resize(hostlen + servlen + 1); + p = &s[0]; + p = std::copy_n(host.data(), hostlen, p); + } + *p++ = ':'; + std::copy_n(serv.data(), servlen, p); + + return s; +} + static int STDERR_COPY = -1; static int STDOUT_COPY = -1; diff --git a/src/util.h b/src/util.h index 52b6eb02..ad3a60dc 100644 --- a/src/util.h +++ b/src/util.h @@ -459,6 +459,11 @@ bool numeric_host(const char *hostname, int family); // failed, "unknown" is returned. std::string numeric_name(const struct sockaddr *sa, socklen_t salen); +// Returns string representation of numeric address and port of |addr| +// of length |salen|. The format is like :. For IPv6 +// address, address is enclosed by square brackets ([]). +std::string numeric_hostport(const struct sockaddr *sa, socklen_t salen); + // Makes internal copy of stderr (and possibly stdout in the future), // which is then used as pointer to /dev/stderr or /proc/self/fd/2 void store_original_fds(); From b624ca6dcdfa0edc5f14e5d35d752c33bde8b88d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 11 Feb 2016 17:12:57 +0900 Subject: [PATCH 09/85] nghttpx: Rename client TLS session cache field --- src/shrpx_http_downstream_connection.cc | 6 +++--- src/shrpx_worker.cc | 16 ++++++++-------- src/shrpx_worker.h | 8 ++++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 166269ea..8ba6be48 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -133,8 +133,8 @@ HttpDownstreamConnection::~HttpDownstreamConnection() { if (conn_.tls.ssl) { auto session = SSL_get0_session(conn_.tls.ssl); if (session) { - worker_->cache_downstream_tls_session(&addr_->addr, session, - ev_now(conn_.loop)); + worker_->cache_client_tls_session(&addr_->addr, session, + ev_now(conn_.loop)); } } } @@ -219,7 +219,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str()); } - auto session = worker_->reuse_downstream_tls_session(&addr_->addr); + auto session = worker_->reuse_client_tls_session(&addr_->addr); if (session) { SSL_set_session(conn_.tls.ssl, session); SSL_SESSION_free(session); diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 9521e5ac..5e46e398 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -312,17 +312,17 @@ std::vector serialize_ssl_session(SSL_SESSION *session) { } } // namespace -void Worker::cache_downstream_tls_session(const Address *addr, - SSL_SESSION *session, ev_tstamp t) { - auto it = downstream_tls_session_cache_.find(addr); - if (it == std::end(downstream_tls_session_cache_)) { +void Worker::cache_client_tls_session(const Address *addr, SSL_SESSION *session, + ev_tstamp t) { + auto it = client_tls_session_cache_.find(addr); + if (it == std::end(client_tls_session_cache_)) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Create cache entry for SSL_SESSION=" << session << ", addr=" << util::numeric_hostport(&addr->su.sa, addr->len) << "(" << addr << "), timestamp=" << std::fixed << std::setprecision(6) << t; } - downstream_tls_session_cache_.emplace( + client_tls_session_cache_.emplace( addr, SessionCacheEntry{serialize_ssl_session(session), t}); return; } @@ -348,9 +348,9 @@ void Worker::cache_downstream_tls_session(const Address *addr, ent.last_updated = t; } -SSL_SESSION *Worker::reuse_downstream_tls_session(const Address *addr) { - auto it = downstream_tls_session_cache_.find(addr); - if (it == std::end(downstream_tls_session_cache_)) { +SSL_SESSION *Worker::reuse_client_tls_session(const Address *addr) { + auto it = client_tls_session_cache_.find(addr); + if (it == std::end(client_tls_session_cache_)) { return nullptr; } diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index e4861c03..c578d9fb 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -157,11 +157,11 @@ public: // |session| is serialized into ASN1 representation, and stored. // |t| is used as a time stamp. Depending on the existing cache's // time stamp, |session| might not be cached. - void cache_downstream_tls_session(const Address *addr, SSL_SESSION *session, - ev_tstamp t); + void cache_client_tls_session(const Address *addr, SSL_SESSION *session, + ev_tstamp t); // Returns cached session associated |addr|. If no cache entry is // found associated to |addr|, nullptr will be returned. - SSL_SESSION *reuse_downstream_tls_session(const Address *addr); + SSL_SESSION *reuse_client_tls_session(const Address *addr); private: #ifndef NOTHREADS @@ -180,7 +180,7 @@ private: // Client side SSL_SESSION cache. SSL_SESSION is associated to // remote address. std::unordered_map - downstream_tls_session_cache_; + client_tls_session_cache_; std::unique_ptr session_cache_memcached_dispatcher_; #ifdef HAVE_MRUBY From e4a727f86c0e1ef3f8d766b740bbf7bcdfb837d4 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 11 Feb 2016 17:55:56 +0900 Subject: [PATCH 10/85] nghttpx: Cache TLS client session after initial handshake was done --- src/shrpx_http_downstream_connection.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 8ba6be48..40c5753d 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -130,7 +130,7 @@ HttpDownstreamConnection::HttpDownstreamConnection( group_(group) {} HttpDownstreamConnection::~HttpDownstreamConnection() { - if (conn_.tls.ssl) { + if (conn_.tls.ssl && conn_.tls.initial_handshake_done) { auto session = SSL_get0_session(conn_.tls.ssl); if (session) { worker_->cache_client_tls_session(&addr_->addr, session, From 82f942c3a382f35bebdbf2134459e3935f54460c Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 11 Feb 2016 18:34:31 +0900 Subject: [PATCH 11/85] nghttpx: Parameterize configuration values for client side TLS context --- src/shrpx_connection_handler.cc | 4 +- src/shrpx_ssl.cc | 98 +++++++++++++++++---------------- src/shrpx_ssl.h | 10 +++- src/template.h | 10 +++- 4 files changed, 68 insertions(+), 54 deletions(-) diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 6f2b585d..c9066b69 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -183,7 +183,7 @@ int ConnectionHandler::create_single_worker() { nb_.get() #endif // HAVE_NEVERBLEED ); - auto cl_ssl_ctx = ssl::setup_client_ssl_context( + auto cl_ssl_ctx = ssl::setup_downstream_client_ssl_context( #ifdef HAVE_NEVERBLEED nb_.get() #endif // HAVE_NEVERBLEED @@ -215,7 +215,7 @@ int ConnectionHandler::create_worker_thread(size_t num) { nb_.get() #endif // HAVE_NEVERBLEED ); - auto cl_ssl_ctx = ssl::setup_client_ssl_context( + auto cl_ssl_ctx = ssl::setup_downstream_client_ssl_context( #ifdef HAVE_NEVERBLEED nb_.get() #endif // HAVE_NEVERBLEED diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 88b3e95d..66c703ad 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -660,9 +660,13 @@ int select_h1_next_proto_cb(SSL *ssl, unsigned char **out, SSL_CTX *create_ssl_client_context( #ifdef HAVE_NEVERBLEED - neverbleed_t *nb + neverbleed_t *nb, #endif // HAVE_NEVERBLEED - ) { + const char *cacert, const char *cert_file, const char *private_key_file, + const StringRef &alpn, + int (*next_proto_select_cb)(SSL *s, unsigned char **out, + unsigned char *outlen, const unsigned char *in, + unsigned int inlen, void *arg)) { auto ssl_ctx = SSL_CTX_new(SSLv23_client_method()); if (!ssl_ctx) { LOG(FATAL) << ERR_error_string(ERR_get_error(), nullptr); @@ -698,71 +702,52 @@ SSL_CTX *create_ssl_client_context( << ERR_error_string(ERR_get_error(), nullptr); } - if (tlsconf.cacert) { - if (SSL_CTX_load_verify_locations(ssl_ctx, tlsconf.cacert.get(), nullptr) != - 1) { + if (cacert) { + if (SSL_CTX_load_verify_locations(ssl_ctx, cacert, nullptr) != 1) { - LOG(FATAL) << "Could not load trusted ca certificates from " - << tlsconf.cacert.get() << ": " - << ERR_error_string(ERR_get_error(), nullptr); + LOG(FATAL) << "Could not load trusted ca certificates from " << cacert + << ": " << ERR_error_string(ERR_get_error(), nullptr); DIE(); } } - if (tlsconf.client.private_key_file) { + if (cert_file) { + if (SSL_CTX_use_certificate_chain_file(ssl_ctx, cert_file) != 1) { + + LOG(FATAL) << "Could not load client certificate from " << cert_file + << ": " << ERR_error_string(ERR_get_error(), nullptr); + DIE(); + } + } + + if (private_key_file) { #ifndef HAVE_NEVERBLEED - if (SSL_CTX_use_PrivateKey_file(ssl_ctx, - tlsconf.client.private_key_file.get(), + if (SSL_CTX_use_PrivateKey_file(ssl_ctx, private_key_file, SSL_FILETYPE_PEM) != 1) { LOG(FATAL) << "Could not load client private key from " - << tlsconf.client.private_key_file.get() << ": " + << private_key_file << ": " << ERR_error_string(ERR_get_error(), nullptr); DIE(); } #else // HAVE_NEVERBLEED std::array errbuf; - if (neverbleed_load_private_key_file(nb, ssl_ctx, - tlsconf.client.private_key_file.get(), + if (neverbleed_load_private_key_file(nb, ssl_ctx, private_key_file, errbuf.data()) != 1) { - LOG(FATAL) << "neverbleed_load_private_key_file failed: " + LOG(FATAL) << "neverbleed_load_private_key_file: could not load client " + "private key from " << private_key_file << ": " << errbuf.data(); DIE(); } #endif // HAVE_NEVERBLEED } - if (tlsconf.client.cert_file) { - if (SSL_CTX_use_certificate_chain_file( - ssl_ctx, tlsconf.client.cert_file.get()) != 1) { - LOG(FATAL) << "Could not load client certificate from " - << tlsconf.client.cert_file.get() << ": " - << ERR_error_string(ERR_get_error(), nullptr); - DIE(); - } - } - - auto &downstreamconf = get_config()->conn.downstream; - - if (downstreamconf.proto == PROTO_HTTP2) { - // NPN selection callback - SSL_CTX_set_next_proto_select_cb(ssl_ctx, select_h2_next_proto_cb, nullptr); + // NPN selection callback + SSL_CTX_set_next_proto_select_cb(ssl_ctx, next_proto_select_cb, nullptr); #if OPENSSL_VERSION_NUMBER >= 0x10002000L - // ALPN advertisement; We only advertise HTTP/2 - auto proto_list = util::get_default_alpn(); - - SSL_CTX_set_alpn_protos(ssl_ctx, proto_list.data(), proto_list.size()); + // ALPN advertisement + SSL_CTX_set_alpn_protos(ssl_ctx, alpn.byte(), alpn.size()); #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L - } else { - // NPN selection callback - SSL_CTX_set_next_proto_select_cb(ssl_ctx, select_h1_next_proto_cb, nullptr); - -#if OPENSSL_VERSION_NUMBER >= 0x10002000L - SSL_CTX_set_alpn_protos( - ssl_ctx, reinterpret_cast(NGHTTP2_H1_1_ALPN), - str_size(NGHTTP2_H1_1_ALPN)); -#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L - } return ssl_ctx; } @@ -1304,7 +1289,7 @@ SSL_CTX *setup_server_ssl_context(std::vector &all_ssl_ctx, bool downstream_tls_enabled() { return !get_config()->conn.downstream.no_tls; } -SSL_CTX *setup_client_ssl_context( +SSL_CTX *setup_downstream_client_ssl_context( #ifdef HAVE_NEVERBLEED neverbleed_t *nb #endif // HAVE_NEVERBLEED @@ -1313,11 +1298,30 @@ SSL_CTX *setup_client_ssl_context( return nullptr; } + auto &tlsconf = get_config()->tls; + auto &downstreamconf = get_config()->conn.downstream; + + std::vector h2alpn; + StringRef alpn; + int (*next_proto_select_cb)(SSL *s, unsigned char **out, + unsigned char *outlen, const unsigned char *in, + unsigned int inlen, void *arg); + + if (downstreamconf.proto == PROTO_HTTP2) { + h2alpn = util::get_default_alpn(); + alpn = StringRef(h2alpn.data(), h2alpn.size()); + next_proto_select_cb = select_h2_next_proto_cb; + } else { + alpn = StringRef::from_lit(NGHTTP2_H1_1_ALPN); + next_proto_select_cb = select_h1_next_proto_cb; + } + return ssl::create_ssl_client_context( #ifdef HAVE_NEVERBLEED - nb + nb, #endif // HAVE_NEVERBLEED - ); + tlsconf.cacert.get(), tlsconf.client.cert_file.get(), + tlsconf.client.private_key_file.get(), alpn, next_proto_select_cb); } CertLookupTree *create_cert_lookup_tree() { diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h index 41c41f00..da7c1667 100644 --- a/src/shrpx_ssl.h +++ b/src/shrpx_ssl.h @@ -72,9 +72,13 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file // Create client side SSL_CTX SSL_CTX *create_ssl_client_context( #ifdef HAVE_NEVERBLEED - neverbleed_t *nb + neverbleed_t *nb, #endif // HAVE_NEVERBLEED - ); + const char *cacert, const char *cert_file, const char *private_key_file, + const StringRef &alpn, + int (*next_proto_select_cb)(SSL *s, unsigned char **out, + unsigned char *outlen, const unsigned char *in, + unsigned int inlen, void *arg)); ClientHandler *accept_connection(Worker *worker, int fd, sockaddr *addr, int addrlen, const UpstreamAddr *faddr); @@ -190,7 +194,7 @@ SSL_CTX *setup_server_ssl_context(std::vector &all_ssl_ctx, // Setups client side SSL_CTX. This function inspects get_config() // and if downstream_no_tls is true, returns nullptr. Otherwise, only // construct SSL_CTX if either client_mode or http2_bridge is true. -SSL_CTX *setup_client_ssl_context( +SSL_CTX *setup_downstream_client_ssl_context( #ifdef HAVE_NEVERBLEED neverbleed_t *nb #endif // HAVE_NEVERBLEED diff --git a/src/template.h b/src/template.h index 8657b213..0d4c4ba2 100644 --- a/src/template.h +++ b/src/template.h @@ -392,11 +392,14 @@ public: explicit StringRef(const ImmutableString &s) : base(s.c_str()), len(s.size()) {} StringRef(const char *s) : base(s), len(strlen(s)) {} - StringRef(const char *s, size_t n) : base(s), len(n) {} + template + StringRef(const CharT *s, size_t n) + : base(reinterpret_cast(s)), len(n) {} template StringRef(InputIt first, InputIt last) : base(first), len(std::distance(first, last)) {} - template static StringRef from_lit(const char(&s)[N]) { + template + static StringRef from_lit(const CharT(&s)[N]) { return StringRef(s, N - 1); } @@ -412,6 +415,9 @@ public: const_reference operator[](size_type pos) const { return *(base + pos); } std::string str() const { return std::string(base, len); } + const uint8_t *byte() const { + return reinterpret_cast(base); + } private: const char *base; From 28b643e531628e02149dc1f6085a220eda9ebfc5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 11 Feb 2016 23:05:16 +0900 Subject: [PATCH 12/85] Fix configure script for non-gcc, clang build --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index a1ed90c3..d7f6505f 100644 --- a/configure.ac +++ b/configure.ac @@ -185,8 +185,8 @@ if test "x$GCC" = "xyes" -o "x$CC" = "xclang" ; then AC_DEFINE([_U_], [__attribute__((unused))], [Hint to the compiler that a function parameters is not used]) AC_DEFINE([NGHTTP2_NORETURN], [__attribute__((noreturn))], [Hint to the compiler that a function never return]) else - AC_DEFINE([_U_], , [Hint to the compiler that a function parameters is not use AC_DEFINE([NGHTTP2_NORETURN], , [Hint to the compiler that a function never return]) -d]) + AC_DEFINE([_U_], , [Hint to the compiler that a function parameter is not used]) + AC_DEFINE([NGHTTP2_NORETURN], , [Hint to the compiler that a function never return]) fi save_CXXFLAGS="$CXXFLAGS" From b0227d40517cc3eddfd4f585297f2f7aa1f500fd Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 4 Feb 2016 22:51:06 +0900 Subject: [PATCH 13/85] nghttpd: Limit request header buffer --- src/HttpServer.cc | 8 ++++++++ src/HttpServer.h | 3 +++ 2 files changed, 11 insertions(+) diff --git a/src/HttpServer.cc b/src/HttpServer.cc index 1e5e679c..e5406cdc 100644 --- a/src/HttpServer.cc +++ b/src/HttpServer.cc @@ -447,6 +447,7 @@ Stream::Stream(Http2Handler *handler, int32_t stream_id) file_ent(nullptr), body_length(0), body_offset(0), + header_buffer_size(0), stream_id(stream_id), echo_upload(false) { auto config = handler->get_config(); @@ -1389,6 +1390,13 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, return 0; } + if (stream->header_buffer_size + namelen + valuelen > 64_k) { + hd->submit_rst_stream(stream, NGHTTP2_INTERNAL_ERROR); + return 0; + } + + stream->header_buffer_size += namelen + valuelen; + auto token = http2::lookup_token(name, namelen); http2::index_header(stream->hdidx, token, stream->headers.size()); diff --git a/src/HttpServer.h b/src/HttpServer.h index 151bae71..99623397 100644 --- a/src/HttpServer.h +++ b/src/HttpServer.h @@ -119,6 +119,9 @@ struct Stream { ev_timer wtimer; int64_t body_length; int64_t body_offset; + // Total amount of bytes (sum of name and value length) used in + // headers. + size_t header_buffer_size; int32_t stream_id; http2::HeaderIndex hdidx; bool echo_upload; From b2264ad57e0e46e872d9213fae7a68b755745af6 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 4 Feb 2016 23:05:05 +0900 Subject: [PATCH 14/85] asio: server: Limit incoming request header field buffer size --- src/asio_server_http2_handler.cc | 7 +++++++ src/asio_server_request_impl.cc | 8 +++++++- src/asio_server_request_impl.h | 4 ++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/asio_server_http2_handler.cc b/src/asio_server_http2_handler.cc index b08f2021..4be689a3 100644 --- a/src/asio_server_http2_handler.cc +++ b/src/asio_server_http2_handler.cc @@ -105,6 +105,13 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, } // fall through default: + if (req.header_buffer_size() + namelen + valuelen > 64_k) { + nghttp2_submit_rst_stream(session, NGHTTP2_FLAG_NONE, frame->hd.stream_id, + NGHTTP2_INTERNAL_ERROR); + break; + } + req.update_header_buffer_size(namelen + valuelen); + req.header().emplace(std::string(name, name + namelen), header_value{std::string(value, value + valuelen), (flags & NGHTTP2_NV_FLAG_NO_INDEX) != 0}); diff --git a/src/asio_server_request_impl.cc b/src/asio_server_request_impl.cc index 64866fa2..8442ad05 100644 --- a/src/asio_server_request_impl.cc +++ b/src/asio_server_request_impl.cc @@ -28,7 +28,7 @@ namespace nghttp2 { namespace asio_http2 { namespace server { -request_impl::request_impl() : strm_(nullptr) {} +request_impl::request_impl() : strm_(nullptr), header_buffer_size_(0) {} const header_map &request_impl::header() const { return header_; } @@ -62,6 +62,12 @@ void request_impl::remote_endpoint(boost::asio::ip::tcp::endpoint ep) { remote_ep_ = std::move(ep); } +size_t request_impl::header_buffer_size() const { return header_buffer_size_; } + +void request_impl::update_header_buffer_size(size_t len) { + header_buffer_size_ += len; +} + } // namespace server } // namespace asio_http2 } // namespace nghttp2 diff --git a/src/asio_server_request_impl.h b/src/asio_server_request_impl.h index b4a37ff1..05de98a8 100644 --- a/src/asio_server_request_impl.h +++ b/src/asio_server_request_impl.h @@ -58,6 +58,9 @@ public: const boost::asio::ip::tcp::endpoint &remote_endpoint() const; void remote_endpoint(boost::asio::ip::tcp::endpoint ep); + size_t header_buffer_size() const; + void update_header_buffer_size(size_t len); + private: class stream *strm_; header_map header_; @@ -65,6 +68,7 @@ private: uri_ref uri_; data_cb on_data_cb_; boost::asio::ip::tcp::endpoint remote_ep_; + size_t header_buffer_size_; }; } // namespace server From ff22862b9dcdcf94a66bd7e5a521d357ac3d43ab Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 4 Feb 2016 23:12:41 +0900 Subject: [PATCH 15/85] nghttp: Limit incoming header field buffer --- src/nghttp.cc | 22 ++++++++++++++++++++++ src/nghttp.h | 1 + 2 files changed, 23 insertions(+) diff --git a/src/nghttp.cc b/src/nghttp.cc index c7dd4d48..e3d4e110 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -155,6 +155,7 @@ Request::Request(const std::string &uri, const http_parser_url &u, inflater(nullptr), html_parser(nullptr), data_prd(data_prd), + header_buffer_size(0), stream_id(-1), status(0), level(level), @@ -1736,6 +1737,14 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, break; } + if (req->header_buffer_size + namelen + valuelen > 64_k) { + nghttp2_submit_rst_stream(session, NGHTTP2_FLAG_NONE, frame->hd.stream_id, + NGHTTP2_INTERNAL_ERROR); + return 0; + } + + req->header_buffer_size += namelen + valuelen; + auto token = http2::lookup_token(name, namelen); http2::index_header(req->res_hdidx, token, req->res_nva.size()); @@ -1751,6 +1760,15 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, break; } + if (req->header_buffer_size + namelen + valuelen > 64_k) { + nghttp2_submit_rst_stream(session, NGHTTP2_FLAG_NONE, + frame->push_promise.promised_stream_id, + NGHTTP2_INTERNAL_ERROR); + return 0; + } + + req->header_buffer_size += namelen + valuelen; + auto token = http2::lookup_token(name, namelen); http2::index_header(req->req_hdidx, token, req->req_nva.size()); @@ -1838,6 +1856,10 @@ int on_frame_recv_callback2(nghttp2_session *session, if (!req) { break; } + + // Reset for response header field reception + req->header_buffer_size = 0; + auto scheme = req->get_req_header(http2::HD__SCHEME); auto authority = req->get_req_header(http2::HD__AUTHORITY); auto path = req->get_req_header(http2::HD__PATH); diff --git a/src/nghttp.h b/src/nghttp.h index ffc07007..715a110e 100644 --- a/src/nghttp.h +++ b/src/nghttp.h @@ -150,6 +150,7 @@ struct Request { nghttp2_gzip *inflater; HtmlParser *html_parser; const nghttp2_data_provider *data_prd; + size_t header_buffer_size; int32_t stream_id; int status; // Recursion level: 0: first entity, 1: entity linked from first entity From 00e722f02cfaf2d5dd6759730c96614ced89da9f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 4 Feb 2016 23:27:22 +0900 Subject: [PATCH 16/85] Add warning --- lib/includes/nghttp2/nghttp2.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/includes/nghttp2/nghttp2.h b/lib/includes/nghttp2/nghttp2.h index 5f389a6e..59fc8674 100644 --- a/lib/includes/nghttp2/nghttp2.h +++ b/lib/includes/nghttp2/nghttp2.h @@ -1608,6 +1608,14 @@ typedef int (*nghttp2_on_begin_headers_callback)(nghttp2_session *session, * * To set this callback to :type:`nghttp2_session_callbacks`, use * `nghttp2_session_callbacks_set_on_header_callback()`. + * + * .. warning:: + * + * Application should properly limit the total buffer size to store + * incoming header fields. Without it, peer may send large number + * of header fields or large header fields to cause out of memory in + * local endpoint. Due to how HPACK works, peer can do this + * effectively without using much memory on their own. */ typedef int (*nghttp2_on_header_callback)(nghttp2_session *session, const nghttp2_frame *frame, From 0a1beea13a8609ee7013f4200283664aac10c138 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 10 Feb 2016 22:35:21 +0900 Subject: [PATCH 17/85] asio: client: Limit incoming response header field buffer size --- src/asio_client_request_impl.cc | 8 +++++++- src/asio_client_request_impl.h | 4 ++++ src/asio_client_response_impl.cc | 9 ++++++++- src/asio_client_response_impl.h | 4 ++++ src/asio_client_session_impl.cc | 13 +++++++++++++ 5 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/asio_client_request_impl.cc b/src/asio_client_request_impl.cc index 5512c967..1e4e2dd1 100644 --- a/src/asio_client_request_impl.cc +++ b/src/asio_client_request_impl.cc @@ -32,7 +32,7 @@ namespace nghttp2 { namespace asio_http2 { namespace client { -request_impl::request_impl() : strm_(nullptr) {} +request_impl::request_impl() : strm_(nullptr), header_buffer_size_(0) {} void request_impl::write_trailer(header_map h) { auto sess = strm_->session(); @@ -105,6 +105,12 @@ void request_impl::method(std::string s) { method_ = std::move(s); } const std::string &request_impl::method() const { return method_; } +size_t request_impl::header_buffer_size() const { return header_buffer_size_; } + +void request_impl::update_header_buffer_size(size_t len) { + header_buffer_size_ += len; +} + } // namespace client } // namespace asio_http2 } // namespace nghttp2 diff --git a/src/asio_client_request_impl.h b/src/asio_client_request_impl.h index 802a83d1..e0d43d2c 100644 --- a/src/asio_client_request_impl.h +++ b/src/asio_client_request_impl.h @@ -75,6 +75,9 @@ public: void method(std::string s); const std::string &method() const; + size_t header_buffer_size() const; + void update_header_buffer_size(size_t len); + private: header_map header_; response_cb response_cb_; @@ -84,6 +87,7 @@ private: class stream *strm_; uri_ref uri_; std::string method_; + size_t header_buffer_size_; }; } // namespace client diff --git a/src/asio_client_response_impl.cc b/src/asio_client_response_impl.cc index fce25a4f..bd2cdf5f 100644 --- a/src/asio_client_response_impl.cc +++ b/src/asio_client_response_impl.cc @@ -30,7 +30,8 @@ namespace nghttp2 { namespace asio_http2 { namespace client { -response_impl::response_impl() : content_length_(-1), status_code_(0) {} +response_impl::response_impl() + : content_length_(-1), header_buffer_size_(0), status_code_(0) {} void response_impl::on_data(data_cb cb) { data_cb_ = std::move(cb); } @@ -52,6 +53,12 @@ header_map &response_impl::header() { return header_; } const header_map &response_impl::header() const { return header_; } +size_t response_impl::header_buffer_size() const { return header_buffer_size_; } + +void response_impl::update_header_buffer_size(size_t len) { + header_buffer_size_ += len; +} + } // namespace client } // namespace asio_http2 } // namespace nghttp2 diff --git a/src/asio_client_response_impl.h b/src/asio_client_response_impl.h index e2c22862..524d7285 100644 --- a/src/asio_client_response_impl.h +++ b/src/asio_client_response_impl.h @@ -53,12 +53,16 @@ public: header_map &header(); const header_map &header() const; + size_t header_buffer_size() const; + void update_header_buffer_size(size_t len); + private: data_cb data_cb_; header_map header_; int64_t content_length_; + size_t header_buffer_size_; int status_code_; }; diff --git a/src/asio_client_session_impl.cc b/src/asio_client_session_impl.cc index 2ea65cd1..fade97bd 100644 --- a/src/asio_client_session_impl.cc +++ b/src/asio_client_session_impl.cc @@ -183,6 +183,12 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, if (token == http2::HD__STATUS) { res.status_code(util::parse_uint(value, valuelen)); } else { + if (res.header_buffer_size() + namelen + valuelen > 64_k) { + nghttp2_submit_rst_stream(session, NGHTTP2_FLAG_NONE, + frame->hd.stream_id, NGHTTP2_INTERNAL_ERROR); + break; + } + res.update_header_buffer_size(namelen + valuelen); if (token == http2::HD_CONTENT_LENGTH) { res.content_length(util::parse_uint(value, valuelen)); @@ -223,6 +229,13 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, } // fall through default: + if (req.header_buffer_size() + namelen + valuelen > 64_k) { + nghttp2_submit_rst_stream(session, NGHTTP2_FLAG_NONE, + frame->hd.stream_id, NGHTTP2_INTERNAL_ERROR); + break; + } + req.update_header_buffer_size(namelen + valuelen); + req.header().emplace( std::string(name, name + namelen), header_value{std::string(value, value + valuelen), From 61dda40b44c5cf010fb933138a5e1c7aefb61611 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 12 Feb 2016 22:31:47 +0900 Subject: [PATCH 18/85] Don't pass NULL to memcpy --- lib/nghttp2_session.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 344c9e71..69f43d16 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -5640,10 +5640,12 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session, const uint8_t *in, readlen = inbound_frame_payload_readlen(iframe, in, last); - iframe->lbuf.last = nghttp2_cpymem(iframe->lbuf.last, in, readlen); + if (readlen > 0) { + iframe->lbuf.last = nghttp2_cpymem(iframe->lbuf.last, in, readlen); - iframe->payloadleft -= readlen; - in += readlen; + iframe->payloadleft -= readlen; + in += readlen; + } DEBUGF(fprintf(stderr, "recv: readlen=%zu, payloadleft=%zu\n", readlen, iframe->payloadleft)); From ca371e3ba9a6cd0617207a0fc912d0635c1fbbdf Mon Sep 17 00:00:00 2001 From: Jay Satiro Date: Fri, 12 Feb 2016 21:46:29 -0500 Subject: [PATCH 19/85] nghttpx: Refactor blacklisted cipher suite check --- src/ssl.cc | 578 ++--------------------------------------------------- 1 file changed, 16 insertions(+), 562 deletions(-) diff --git a/src/ssl.cc b/src/ssl.cc index 078c5ec4..389a25de 100644 --- a/src/ssl.cc +++ b/src/ssl.cc @@ -117,571 +117,25 @@ TLSSessionInfo *get_tls_session_info(TLSSessionInfo *tls_info, SSL *ssl) { return tls_info; } -// The black listed cipher suites for HTTP/2 described in RFC 7540. -enum { - TLS_NULL_WITH_NULL_NULL = 0x0000u, - TLS_RSA_WITH_NULL_MD5 = 0x0001u, - TLS_RSA_WITH_NULL_SHA = 0x0002u, - TLS_RSA_EXPORT_WITH_RC4_40_MD5 = 0x0003u, - TLS_RSA_WITH_RC4_128_MD5 = 0x0004u, - TLS_RSA_WITH_RC4_128_SHA = 0x0005u, - TLS_RSA_EXPORT_WITH_RC2_CBC_40_MD5 = 0x0006u, - TLS_RSA_WITH_IDEA_CBC_SHA = 0x0007u, - TLS_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x0008u, - TLS_RSA_WITH_DES_CBC_SHA = 0x0009u, - TLS_RSA_WITH_3DES_EDE_CBC_SHA = 0x000Au, - TLS_DH_DSS_EXPORT_WITH_DES40_CBC_SHA = 0x000Bu, - TLS_DH_DSS_WITH_DES_CBC_SHA = 0x000Cu, - TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA = 0x000Du, - TLS_DH_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x000Eu, - TLS_DH_RSA_WITH_DES_CBC_SHA = 0x000Fu, - TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA = 0x0010u, - TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA = 0x0011u, - TLS_DHE_DSS_WITH_DES_CBC_SHA = 0x0012u, - TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA = 0x0013u, - TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x0014u, - TLS_DHE_RSA_WITH_DES_CBC_SHA = 0x0015u, - TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA = 0x0016u, - TLS_DH_anon_EXPORT_WITH_RC4_40_MD5 = 0x0017u, - TLS_DH_anon_WITH_RC4_128_MD5 = 0x0018u, - TLS_DH_anon_EXPORT_WITH_DES40_CBC_SHA = 0x0019u, - TLS_DH_anon_WITH_DES_CBC_SHA = 0x001Au, - TLS_DH_anon_WITH_3DES_EDE_CBC_SHA = 0x001Bu, - TLS_KRB5_WITH_DES_CBC_SHA = 0x001Eu, - TLS_KRB5_WITH_3DES_EDE_CBC_SHA = 0x001Fu, - TLS_KRB5_WITH_RC4_128_SHA = 0x0020u, - TLS_KRB5_WITH_IDEA_CBC_SHA = 0x0021u, - TLS_KRB5_WITH_DES_CBC_MD5 = 0x0022u, - TLS_KRB5_WITH_3DES_EDE_CBC_MD5 = 0x0023u, - TLS_KRB5_WITH_RC4_128_MD5 = 0x0024u, - TLS_KRB5_WITH_IDEA_CBC_MD5 = 0x0025u, - TLS_KRB5_EXPORT_WITH_DES_CBC_40_SHA = 0x0026u, - TLS_KRB5_EXPORT_WITH_RC2_CBC_40_SHA = 0x0027u, - TLS_KRB5_EXPORT_WITH_RC4_40_SHA = 0x0028u, - TLS_KRB5_EXPORT_WITH_DES_CBC_40_MD5 = 0x0029u, - TLS_KRB5_EXPORT_WITH_RC2_CBC_40_MD5 = 0x002Au, - TLS_KRB5_EXPORT_WITH_RC4_40_MD5 = 0x002Bu, - TLS_PSK_WITH_NULL_SHA = 0x002Cu, - TLS_DHE_PSK_WITH_NULL_SHA = 0x002Du, - TLS_RSA_PSK_WITH_NULL_SHA = 0x002Eu, - TLS_RSA_WITH_AES_128_CBC_SHA = 0x002Fu, - TLS_DH_DSS_WITH_AES_128_CBC_SHA = 0x0030u, - TLS_DH_RSA_WITH_AES_128_CBC_SHA = 0x0031u, - TLS_DHE_DSS_WITH_AES_128_CBC_SHA = 0x0032u, - TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x0033u, - TLS_DH_anon_WITH_AES_128_CBC_SHA = 0x0034u, - TLS_RSA_WITH_AES_256_CBC_SHA = 0x0035u, - TLS_DH_DSS_WITH_AES_256_CBC_SHA = 0x0036u, - TLS_DH_RSA_WITH_AES_256_CBC_SHA = 0x0037u, - TLS_DHE_DSS_WITH_AES_256_CBC_SHA = 0x0038u, - TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x0039u, - TLS_DH_anon_WITH_AES_256_CBC_SHA = 0x003Au, - TLS_RSA_WITH_NULL_SHA256 = 0x003Bu, - TLS_RSA_WITH_AES_128_CBC_SHA256 = 0x003Cu, - TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x003Du, - TLS_DH_DSS_WITH_AES_128_CBC_SHA256 = 0x003Eu, - TLS_DH_RSA_WITH_AES_128_CBC_SHA256 = 0x003Fu, - TLS_DHE_DSS_WITH_AES_128_CBC_SHA256 = 0x0040u, - TLS_RSA_WITH_CAMELLIA_128_CBC_SHA = 0x0041u, - TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA = 0x0042u, - TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA = 0x0043u, - TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA = 0x0044u, - TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA = 0x0045u, - TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA = 0x0046u, - TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 = 0x0067u, - TLS_DH_DSS_WITH_AES_256_CBC_SHA256 = 0x0068u, - TLS_DH_RSA_WITH_AES_256_CBC_SHA256 = 0x0069u, - TLS_DHE_DSS_WITH_AES_256_CBC_SHA256 = 0x006Au, - TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 = 0x006Bu, - TLS_DH_anon_WITH_AES_128_CBC_SHA256 = 0x006Cu, - TLS_DH_anon_WITH_AES_256_CBC_SHA256 = 0x006Du, - TLS_RSA_WITH_CAMELLIA_256_CBC_SHA = 0x0084u, - TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA = 0x0085u, - TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA = 0x0086u, - TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA = 0x0087u, - TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA = 0x0088u, - TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA = 0x0089u, - TLS_PSK_WITH_RC4_128_SHA = 0x008Au, - TLS_PSK_WITH_3DES_EDE_CBC_SHA = 0x008Bu, - TLS_PSK_WITH_AES_128_CBC_SHA = 0x008Cu, - TLS_PSK_WITH_AES_256_CBC_SHA = 0x008Du, - TLS_DHE_PSK_WITH_RC4_128_SHA = 0x008Eu, - TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA = 0x008Fu, - TLS_DHE_PSK_WITH_AES_128_CBC_SHA = 0x0090u, - TLS_DHE_PSK_WITH_AES_256_CBC_SHA = 0x0091u, - TLS_RSA_PSK_WITH_RC4_128_SHA = 0x0092u, - TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA = 0x0093u, - TLS_RSA_PSK_WITH_AES_128_CBC_SHA = 0x0094u, - TLS_RSA_PSK_WITH_AES_256_CBC_SHA = 0x0095u, - TLS_RSA_WITH_SEED_CBC_SHA = 0x0096u, - TLS_DH_DSS_WITH_SEED_CBC_SHA = 0x0097u, - TLS_DH_RSA_WITH_SEED_CBC_SHA = 0x0098u, - TLS_DHE_DSS_WITH_SEED_CBC_SHA = 0x0099u, - TLS_DHE_RSA_WITH_SEED_CBC_SHA = 0x009Au, - TLS_DH_anon_WITH_SEED_CBC_SHA = 0x009Bu, - TLS_RSA_WITH_AES_128_GCM_SHA256 = 0x009Cu, - TLS_RSA_WITH_AES_256_GCM_SHA384 = 0x009Du, - TLS_DH_RSA_WITH_AES_128_GCM_SHA256 = 0x00A0u, - TLS_DH_RSA_WITH_AES_256_GCM_SHA384 = 0x00A1u, - TLS_DH_DSS_WITH_AES_128_GCM_SHA256 = 0x00A4u, - TLS_DH_DSS_WITH_AES_256_GCM_SHA384 = 0x00A5u, - TLS_DH_anon_WITH_AES_128_GCM_SHA256 = 0x00A6u, - TLS_DH_anon_WITH_AES_256_GCM_SHA384 = 0x00A7u, - TLS_PSK_WITH_AES_128_GCM_SHA256 = 0x00A8u, - TLS_PSK_WITH_AES_256_GCM_SHA384 = 0x00A9u, - TLS_RSA_PSK_WITH_AES_128_GCM_SHA256 = 0x00ACu, - TLS_RSA_PSK_WITH_AES_256_GCM_SHA384 = 0x00ADu, - TLS_PSK_WITH_AES_128_CBC_SHA256 = 0x00AEu, - TLS_PSK_WITH_AES_256_CBC_SHA384 = 0x00AFu, - TLS_PSK_WITH_NULL_SHA256 = 0x00B0u, - TLS_PSK_WITH_NULL_SHA384 = 0x00B1u, - TLS_DHE_PSK_WITH_AES_128_CBC_SHA256 = 0x00B2u, - TLS_DHE_PSK_WITH_AES_256_CBC_SHA384 = 0x00B3u, - TLS_DHE_PSK_WITH_NULL_SHA256 = 0x00B4u, - TLS_DHE_PSK_WITH_NULL_SHA384 = 0x00B5u, - TLS_RSA_PSK_WITH_AES_128_CBC_SHA256 = 0x00B6u, - TLS_RSA_PSK_WITH_AES_256_CBC_SHA384 = 0x00B7u, - TLS_RSA_PSK_WITH_NULL_SHA256 = 0x00B8u, - TLS_RSA_PSK_WITH_NULL_SHA384 = 0x00B9u, - TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BAu, - TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BBu, - TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BCu, - TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BDu, - TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BEu, - TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BFu, - TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C0u, - TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C1u, - TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C2u, - TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C3u, - TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C4u, - TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C5u, - TLS_EMPTY_RENEGOTIATION_INFO_SCSV = 0x00FFu, - TLS_ECDH_ECDSA_WITH_NULL_SHA = 0xC001u, - TLS_ECDH_ECDSA_WITH_RC4_128_SHA = 0xC002u, - TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA = 0xC003u, - TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA = 0xC004u, - TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA = 0xC005u, - TLS_ECDHE_ECDSA_WITH_NULL_SHA = 0xC006u, - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA = 0xC007u, - TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA = 0xC008u, - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA = 0xC009u, - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA = 0xC00Au, - TLS_ECDH_RSA_WITH_NULL_SHA = 0xC00Bu, - TLS_ECDH_RSA_WITH_RC4_128_SHA = 0xC00Cu, - TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA = 0xC00Du, - TLS_ECDH_RSA_WITH_AES_128_CBC_SHA = 0xC00Eu, - TLS_ECDH_RSA_WITH_AES_256_CBC_SHA = 0xC00Fu, - TLS_ECDHE_RSA_WITH_NULL_SHA = 0xC010u, - TLS_ECDHE_RSA_WITH_RC4_128_SHA = 0xC011u, - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA = 0xC012u, - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA = 0xC013u, - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA = 0xC014u, - TLS_ECDH_anon_WITH_NULL_SHA = 0xC015u, - TLS_ECDH_anon_WITH_RC4_128_SHA = 0xC016u, - TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA = 0xC017u, - TLS_ECDH_anon_WITH_AES_128_CBC_SHA = 0xC018u, - TLS_ECDH_anon_WITH_AES_256_CBC_SHA = 0xC019u, - TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA = 0xC01Au, - TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA = 0xC01Bu, - TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA = 0xC01Cu, - TLS_SRP_SHA_WITH_AES_128_CBC_SHA = 0xC01Du, - TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA = 0xC01Eu, - TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA = 0xC01Fu, - TLS_SRP_SHA_WITH_AES_256_CBC_SHA = 0xC020u, - TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA = 0xC021u, - TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA = 0xC022u, - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC023u, - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC024u, - TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC025u, - TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC026u, - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 = 0xC027u, - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xC028u, - TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256 = 0xC029u, - TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384 = 0xC02Au, - TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02Du, - TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02Eu, - TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256 = 0xC031u, - TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384 = 0xC032u, - TLS_ECDHE_PSK_WITH_RC4_128_SHA = 0xC033u, - TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA = 0xC034u, - TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA = 0xC035u, - TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA = 0xC036u, - TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256 = 0xC037u, - TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384 = 0xC038u, - TLS_ECDHE_PSK_WITH_NULL_SHA = 0xC039u, - TLS_ECDHE_PSK_WITH_NULL_SHA256 = 0xC03Au, - TLS_ECDHE_PSK_WITH_NULL_SHA384 = 0xC03Bu, - TLS_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC03Cu, - TLS_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC03Du, - TLS_DH_DSS_WITH_ARIA_128_CBC_SHA256 = 0xC03Eu, - TLS_DH_DSS_WITH_ARIA_256_CBC_SHA384 = 0xC03Fu, - TLS_DH_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC040u, - TLS_DH_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC041u, - TLS_DHE_DSS_WITH_ARIA_128_CBC_SHA256 = 0xC042u, - TLS_DHE_DSS_WITH_ARIA_256_CBC_SHA384 = 0xC043u, - TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC044u, - TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC045u, - TLS_DH_anon_WITH_ARIA_128_CBC_SHA256 = 0xC046u, - TLS_DH_anon_WITH_ARIA_256_CBC_SHA384 = 0xC047u, - TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256 = 0xC048u, - TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384 = 0xC049u, - TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256 = 0xC04Au, - TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384 = 0xC04Bu, - TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC04Cu, - TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC04Du, - TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC04Eu, - TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC04Fu, - TLS_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC050u, - TLS_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC051u, - TLS_DH_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC054u, - TLS_DH_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC055u, - TLS_DH_DSS_WITH_ARIA_128_GCM_SHA256 = 0xC058u, - TLS_DH_DSS_WITH_ARIA_256_GCM_SHA384 = 0xC059u, - TLS_DH_anon_WITH_ARIA_128_GCM_SHA256 = 0xC05Au, - TLS_DH_anon_WITH_ARIA_256_GCM_SHA384 = 0xC05Bu, - TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256 = 0xC05Eu, - TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384 = 0xC05Fu, - TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC062u, - TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC063u, - TLS_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC064u, - TLS_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC065u, - TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC066u, - TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC067u, - TLS_RSA_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC068u, - TLS_RSA_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC069u, - TLS_PSK_WITH_ARIA_128_GCM_SHA256 = 0xC06Au, - TLS_PSK_WITH_ARIA_256_GCM_SHA384 = 0xC06Bu, - TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256 = 0xC06Eu, - TLS_RSA_PSK_WITH_ARIA_256_GCM_SHA384 = 0xC06Fu, - TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC070u, - TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC071u, - TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC072u, - TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC073u, - TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC074u, - TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC075u, - TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC076u, - TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC077u, - TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC078u, - TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC079u, - TLS_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC07Au, - TLS_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC07Bu, - TLS_DH_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC07Eu, - TLS_DH_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC07Fu, - TLS_DH_DSS_WITH_CAMELLIA_128_GCM_SHA256 = 0xC082u, - TLS_DH_DSS_WITH_CAMELLIA_256_GCM_SHA384 = 0xC083u, - TLS_DH_anon_WITH_CAMELLIA_128_GCM_SHA256 = 0xC084u, - TLS_DH_anon_WITH_CAMELLIA_256_GCM_SHA384 = 0xC085u, - TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC088u, - TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC089u, - TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC08Cu, - TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC08Du, - TLS_PSK_WITH_CAMELLIA_128_GCM_SHA256 = 0xC08Eu, - TLS_PSK_WITH_CAMELLIA_256_GCM_SHA384 = 0xC08Fu, - TLS_RSA_PSK_WITH_CAMELLIA_128_GCM_SHA256 = 0xC092u, - TLS_RSA_PSK_WITH_CAMELLIA_256_GCM_SHA384 = 0xC093u, - TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC094u, - TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC095u, - TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC096u, - TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC097u, - TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC098u, - TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC099u, - TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC09Au, - TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC09Bu, - TLS_RSA_WITH_AES_128_CCM = 0xC09Cu, - TLS_RSA_WITH_AES_256_CCM = 0xC09Du, - TLS_RSA_WITH_AES_128_CCM_8 = 0xC0A0u, - TLS_RSA_WITH_AES_256_CCM_8 = 0xC0A1u, - TLS_PSK_WITH_AES_128_CCM = 0xC0A4u, - TLS_PSK_WITH_AES_256_CCM = 0xC0A5u, - TLS_PSK_WITH_AES_128_CCM_8 = 0xC0A8u, - TLS_PSK_WITH_AES_256_CCM_8 = 0xC0A9u, -}; +/* Conditional logic w/ lookup tables to check if id is one of the + the black listed cipher suites for HTTP/2 described in RFC 7540. + https://github.com/jay/http2_blacklisted_ciphers +*/ +#define IS_CIPHER_BANNED_METHOD2(id) ( \ + (0x0000 <= id && id <= 0x00FF && \ + "\xFF\xFF\xFF\xCF\xFF\xFF\xFF\xFF\x7F\x00\x00\x00\x80\x3F\x00\x00" \ + "\xF0\xFF\xFF\x3F\xF3\xF3\xFF\xFF\x3F\x00\x00\x00\x00\x00\x00\x80" \ + [(id & 0xFF) / 8] & (1 << (id % 8))) || \ + (0xC000 <= id && id <= 0xC0FF && \ + "\xFE\xFF\xFF\xFF\xFF\x67\xFE\xFF\xFF\xFF\x33\xCF\xFC\xCF\xFF\xCF" \ + "\x3C\xF3\xFC\x3F\x33\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" \ + [(id & 0xFF) / 8] & (1 << (id % 8))) \ +) bool check_http2_cipher_black_list(SSL *ssl) { - auto cipher = SSL_get_current_cipher(ssl); + int id = SSL_CIPHER_get_id(SSL_get_current_cipher(ssl)) & 0xFFFFFF; - // Cipher suites in RFC 7540 black list are not allowed in HTTP/2. - switch (SSL_CIPHER_get_id(cipher) & 0xffffu) { - case TLS_NULL_WITH_NULL_NULL: - case TLS_RSA_WITH_NULL_MD5: - case TLS_RSA_WITH_NULL_SHA: - case TLS_RSA_EXPORT_WITH_RC4_40_MD5: - case TLS_RSA_WITH_RC4_128_MD5: - case TLS_RSA_WITH_RC4_128_SHA: - case TLS_RSA_EXPORT_WITH_RC2_CBC_40_MD5: - case TLS_RSA_WITH_IDEA_CBC_SHA: - case TLS_RSA_EXPORT_WITH_DES40_CBC_SHA: - case TLS_RSA_WITH_DES_CBC_SHA: - case TLS_RSA_WITH_3DES_EDE_CBC_SHA: - case TLS_DH_DSS_EXPORT_WITH_DES40_CBC_SHA: - case TLS_DH_DSS_WITH_DES_CBC_SHA: - case TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA: - case TLS_DH_RSA_EXPORT_WITH_DES40_CBC_SHA: - case TLS_DH_RSA_WITH_DES_CBC_SHA: - case TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA: - case TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA: - case TLS_DHE_DSS_WITH_DES_CBC_SHA: - case TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA: - case TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA: - case TLS_DHE_RSA_WITH_DES_CBC_SHA: - case TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA: - case TLS_DH_anon_EXPORT_WITH_RC4_40_MD5: - case TLS_DH_anon_WITH_RC4_128_MD5: - case TLS_DH_anon_EXPORT_WITH_DES40_CBC_SHA: - case TLS_DH_anon_WITH_DES_CBC_SHA: - case TLS_DH_anon_WITH_3DES_EDE_CBC_SHA: - case TLS_KRB5_WITH_DES_CBC_SHA: - case TLS_KRB5_WITH_3DES_EDE_CBC_SHA: - case TLS_KRB5_WITH_RC4_128_SHA: - case TLS_KRB5_WITH_IDEA_CBC_SHA: - case TLS_KRB5_WITH_DES_CBC_MD5: - case TLS_KRB5_WITH_3DES_EDE_CBC_MD5: - case TLS_KRB5_WITH_RC4_128_MD5: - case TLS_KRB5_WITH_IDEA_CBC_MD5: - case TLS_KRB5_EXPORT_WITH_DES_CBC_40_SHA: - case TLS_KRB5_EXPORT_WITH_RC2_CBC_40_SHA: - case TLS_KRB5_EXPORT_WITH_RC4_40_SHA: - case TLS_KRB5_EXPORT_WITH_DES_CBC_40_MD5: - case TLS_KRB5_EXPORT_WITH_RC2_CBC_40_MD5: - case TLS_KRB5_EXPORT_WITH_RC4_40_MD5: - case TLS_PSK_WITH_NULL_SHA: - case TLS_DHE_PSK_WITH_NULL_SHA: - case TLS_RSA_PSK_WITH_NULL_SHA: - case TLS_RSA_WITH_AES_128_CBC_SHA: - case TLS_DH_DSS_WITH_AES_128_CBC_SHA: - case TLS_DH_RSA_WITH_AES_128_CBC_SHA: - case TLS_DHE_DSS_WITH_AES_128_CBC_SHA: - case TLS_DHE_RSA_WITH_AES_128_CBC_SHA: - case TLS_DH_anon_WITH_AES_128_CBC_SHA: - case TLS_RSA_WITH_AES_256_CBC_SHA: - case TLS_DH_DSS_WITH_AES_256_CBC_SHA: - case TLS_DH_RSA_WITH_AES_256_CBC_SHA: - case TLS_DHE_DSS_WITH_AES_256_CBC_SHA: - case TLS_DHE_RSA_WITH_AES_256_CBC_SHA: - case TLS_DH_anon_WITH_AES_256_CBC_SHA: - case TLS_RSA_WITH_NULL_SHA256: - case TLS_RSA_WITH_AES_128_CBC_SHA256: - case TLS_RSA_WITH_AES_256_CBC_SHA256: - case TLS_DH_DSS_WITH_AES_128_CBC_SHA256: - case TLS_DH_RSA_WITH_AES_128_CBC_SHA256: - case TLS_DHE_DSS_WITH_AES_128_CBC_SHA256: - case TLS_RSA_WITH_CAMELLIA_128_CBC_SHA: - case TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA: - case TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA: - case TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA: - case TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA: - case TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA: - case TLS_DHE_RSA_WITH_AES_128_CBC_SHA256: - case TLS_DH_DSS_WITH_AES_256_CBC_SHA256: - case TLS_DH_RSA_WITH_AES_256_CBC_SHA256: - case TLS_DHE_DSS_WITH_AES_256_CBC_SHA256: - case TLS_DHE_RSA_WITH_AES_256_CBC_SHA256: - case TLS_DH_anon_WITH_AES_128_CBC_SHA256: - case TLS_DH_anon_WITH_AES_256_CBC_SHA256: - case TLS_RSA_WITH_CAMELLIA_256_CBC_SHA: - case TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA: - case TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA: - case TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA: - case TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA: - case TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA: - case TLS_PSK_WITH_RC4_128_SHA: - case TLS_PSK_WITH_3DES_EDE_CBC_SHA: - case TLS_PSK_WITH_AES_128_CBC_SHA: - case TLS_PSK_WITH_AES_256_CBC_SHA: - case TLS_DHE_PSK_WITH_RC4_128_SHA: - case TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA: - case TLS_DHE_PSK_WITH_AES_128_CBC_SHA: - case TLS_DHE_PSK_WITH_AES_256_CBC_SHA: - case TLS_RSA_PSK_WITH_RC4_128_SHA: - case TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA: - case TLS_RSA_PSK_WITH_AES_128_CBC_SHA: - case TLS_RSA_PSK_WITH_AES_256_CBC_SHA: - case TLS_RSA_WITH_SEED_CBC_SHA: - case TLS_DH_DSS_WITH_SEED_CBC_SHA: - case TLS_DH_RSA_WITH_SEED_CBC_SHA: - case TLS_DHE_DSS_WITH_SEED_CBC_SHA: - case TLS_DHE_RSA_WITH_SEED_CBC_SHA: - case TLS_DH_anon_WITH_SEED_CBC_SHA: - case TLS_RSA_WITH_AES_128_GCM_SHA256: - case TLS_RSA_WITH_AES_256_GCM_SHA384: - case TLS_DH_RSA_WITH_AES_128_GCM_SHA256: - case TLS_DH_RSA_WITH_AES_256_GCM_SHA384: - case TLS_DH_DSS_WITH_AES_128_GCM_SHA256: - case TLS_DH_DSS_WITH_AES_256_GCM_SHA384: - case TLS_DH_anon_WITH_AES_128_GCM_SHA256: - case TLS_DH_anon_WITH_AES_256_GCM_SHA384: - case TLS_PSK_WITH_AES_128_GCM_SHA256: - case TLS_PSK_WITH_AES_256_GCM_SHA384: - case TLS_RSA_PSK_WITH_AES_128_GCM_SHA256: - case TLS_RSA_PSK_WITH_AES_256_GCM_SHA384: - case TLS_PSK_WITH_AES_128_CBC_SHA256: - case TLS_PSK_WITH_AES_256_CBC_SHA384: - case TLS_PSK_WITH_NULL_SHA256: - case TLS_PSK_WITH_NULL_SHA384: - case TLS_DHE_PSK_WITH_AES_128_CBC_SHA256: - case TLS_DHE_PSK_WITH_AES_256_CBC_SHA384: - case TLS_DHE_PSK_WITH_NULL_SHA256: - case TLS_DHE_PSK_WITH_NULL_SHA384: - case TLS_RSA_PSK_WITH_AES_128_CBC_SHA256: - case TLS_RSA_PSK_WITH_AES_256_CBC_SHA384: - case TLS_RSA_PSK_WITH_NULL_SHA256: - case TLS_RSA_PSK_WITH_NULL_SHA384: - case TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256: - case TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA256: - case TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA256: - case TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256: - case TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256: - case TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA256: - case TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256: - case TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA256: - case TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA256: - case TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256: - case TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256: - case TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA256: - case TLS_EMPTY_RENEGOTIATION_INFO_SCSV: - case TLS_ECDH_ECDSA_WITH_NULL_SHA: - case TLS_ECDH_ECDSA_WITH_RC4_128_SHA: - case TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA: - case TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA: - case TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA: - case TLS_ECDHE_ECDSA_WITH_NULL_SHA: - case TLS_ECDHE_ECDSA_WITH_RC4_128_SHA: - case TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA: - case TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA: - case TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA: - case TLS_ECDH_RSA_WITH_NULL_SHA: - case TLS_ECDH_RSA_WITH_RC4_128_SHA: - case TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA: - case TLS_ECDH_RSA_WITH_AES_128_CBC_SHA: - case TLS_ECDH_RSA_WITH_AES_256_CBC_SHA: - case TLS_ECDHE_RSA_WITH_NULL_SHA: - case TLS_ECDHE_RSA_WITH_RC4_128_SHA: - case TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA: - case TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA: - case TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA: - case TLS_ECDH_anon_WITH_NULL_SHA: - case TLS_ECDH_anon_WITH_RC4_128_SHA: - case TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA: - case TLS_ECDH_anon_WITH_AES_128_CBC_SHA: - case TLS_ECDH_anon_WITH_AES_256_CBC_SHA: - case TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA: - case TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA: - case TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA: - case TLS_SRP_SHA_WITH_AES_128_CBC_SHA: - case TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA: - case TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA: - case TLS_SRP_SHA_WITH_AES_256_CBC_SHA: - case TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA: - case TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA: - case TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256: - case TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384: - case TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256: - case TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384: - case TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256: - case TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384: - case TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256: - case TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384: - case TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256: - case TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384: - case TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256: - case TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384: - case TLS_ECDHE_PSK_WITH_RC4_128_SHA: - case TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA: - case TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA: - case TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA: - case TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256: - case TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384: - case TLS_ECDHE_PSK_WITH_NULL_SHA: - case TLS_ECDHE_PSK_WITH_NULL_SHA256: - case TLS_ECDHE_PSK_WITH_NULL_SHA384: - case TLS_RSA_WITH_ARIA_128_CBC_SHA256: - case TLS_RSA_WITH_ARIA_256_CBC_SHA384: - case TLS_DH_DSS_WITH_ARIA_128_CBC_SHA256: - case TLS_DH_DSS_WITH_ARIA_256_CBC_SHA384: - case TLS_DH_RSA_WITH_ARIA_128_CBC_SHA256: - case TLS_DH_RSA_WITH_ARIA_256_CBC_SHA384: - case TLS_DHE_DSS_WITH_ARIA_128_CBC_SHA256: - case TLS_DHE_DSS_WITH_ARIA_256_CBC_SHA384: - case TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256: - case TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384: - case TLS_DH_anon_WITH_ARIA_128_CBC_SHA256: - case TLS_DH_anon_WITH_ARIA_256_CBC_SHA384: - case TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256: - case TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384: - case TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256: - case TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384: - case TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256: - case TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384: - case TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256: - case TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384: - case TLS_RSA_WITH_ARIA_128_GCM_SHA256: - case TLS_RSA_WITH_ARIA_256_GCM_SHA384: - case TLS_DH_RSA_WITH_ARIA_128_GCM_SHA256: - case TLS_DH_RSA_WITH_ARIA_256_GCM_SHA384: - case TLS_DH_DSS_WITH_ARIA_128_GCM_SHA256: - case TLS_DH_DSS_WITH_ARIA_256_GCM_SHA384: - case TLS_DH_anon_WITH_ARIA_128_GCM_SHA256: - case TLS_DH_anon_WITH_ARIA_256_GCM_SHA384: - case TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256: - case TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384: - case TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256: - case TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384: - case TLS_PSK_WITH_ARIA_128_CBC_SHA256: - case TLS_PSK_WITH_ARIA_256_CBC_SHA384: - case TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256: - case TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384: - case TLS_RSA_PSK_WITH_ARIA_128_CBC_SHA256: - case TLS_RSA_PSK_WITH_ARIA_256_CBC_SHA384: - case TLS_PSK_WITH_ARIA_128_GCM_SHA256: - case TLS_PSK_WITH_ARIA_256_GCM_SHA384: - case TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256: - case TLS_RSA_PSK_WITH_ARIA_256_GCM_SHA384: - case TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256: - case TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384: - case TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256: - case TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384: - case TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256: - case TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384: - case TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256: - case TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384: - case TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256: - case TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384: - case TLS_RSA_WITH_CAMELLIA_128_GCM_SHA256: - case TLS_RSA_WITH_CAMELLIA_256_GCM_SHA384: - case TLS_DH_RSA_WITH_CAMELLIA_128_GCM_SHA256: - case TLS_DH_RSA_WITH_CAMELLIA_256_GCM_SHA384: - case TLS_DH_DSS_WITH_CAMELLIA_128_GCM_SHA256: - case TLS_DH_DSS_WITH_CAMELLIA_256_GCM_SHA384: - case TLS_DH_anon_WITH_CAMELLIA_128_GCM_SHA256: - case TLS_DH_anon_WITH_CAMELLIA_256_GCM_SHA384: - case TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256: - case TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384: - case TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256: - case TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384: - case TLS_PSK_WITH_CAMELLIA_128_GCM_SHA256: - case TLS_PSK_WITH_CAMELLIA_256_GCM_SHA384: - case TLS_RSA_PSK_WITH_CAMELLIA_128_GCM_SHA256: - case TLS_RSA_PSK_WITH_CAMELLIA_256_GCM_SHA384: - case TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256: - case TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384: - case TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256: - case TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384: - case TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256: - case TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384: - case TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256: - case TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384: - case TLS_RSA_WITH_AES_128_CCM: - case TLS_RSA_WITH_AES_256_CCM: - case TLS_RSA_WITH_AES_128_CCM_8: - case TLS_RSA_WITH_AES_256_CCM_8: - case TLS_PSK_WITH_AES_128_CCM: - case TLS_PSK_WITH_AES_256_CCM: - case TLS_PSK_WITH_AES_128_CCM_8: - case TLS_PSK_WITH_AES_256_CCM_8: - return true; - } - - return false; + return IS_CIPHER_BANNED_METHOD2(id); } bool check_http2_tls_version(SSL *ssl) { From f1580f95d43552091576394be6536dc09197cd82 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 11 Feb 2016 22:56:45 +0900 Subject: [PATCH 20/85] nghttpx: Add TLS support for session cache memcached connection --- gennghttpxfun.py | 1 + src/shrpx.cc | 8 ++ src/shrpx_config.cc | 14 +++ src/shrpx_config.h | 3 + src/shrpx_connection_handler.cc | 33 ++++- src/shrpx_memcached_connection.cc | 199 +++++++++++++++++++++++++----- src/shrpx_memcached_connection.h | 23 +++- src/shrpx_memcached_dispatcher.cc | 8 +- src/shrpx_memcached_dispatcher.h | 8 +- src/shrpx_ssl.cc | 17 +-- src/shrpx_ssl.h | 6 +- src/shrpx_worker.cc | 5 +- src/shrpx_worker.h | 1 + src/shrpx_worker_process.cc | 3 +- 14 files changed, 276 insertions(+), 53 deletions(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index f75add0b..500d47a6 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -92,6 +92,7 @@ OPTIONS = [ "tls-ticket-key-cipher", "host-rewrite", "tls-session-cache-memcached", + "tls-session-cache-memcached-tls", "tls-ticket-key-memcached", "tls-ticket-key-memcached-interval", "tls-ticket-key-memcached-max-retry", diff --git a/src/shrpx.cc b/src/shrpx.cc index e93c5d67..04e85664 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1564,6 +1564,9 @@ SSL/TLS: Specify address of memcached server to store session cache. This enables shared session cache between multiple nghttpx instances. + --tls-session-cache-memcached-tls + Enable SSL/TLS on memcached connections to store session + cache. --tls-dyn-rec-warmup-threshold= Specify the threshold size for TLS dynamic record size behaviour. During a TLS session, after the threshold @@ -2400,6 +2403,7 @@ int main(int argc, char **argv) { {SHRPX_OPT_BACKEND_HTTP1_TLS, no_argument, &flag, 106}, {SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER, required_argument, &flag, 107}, + {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_TLS, no_argument, &flag, 108}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2858,6 +2862,10 @@ int main(int argc, char **argv) { cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER, optarg); break; + case 108: + // --tls-session-cache-memcached-tls + cmdcfgs.emplace_back(SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_TLS, "yes"); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 102319e5..68e4da5b 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -758,6 +758,7 @@ enum { SHRPX_OPTID_TLS_DYN_REC_WARMUP_THRESHOLD, SHRPX_OPTID_TLS_PROTO_LIST, SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED, + SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_TLS, SHRPX_OPTID_TLS_TICKET_KEY_CIPHER, SHRPX_OPTID_TLS_TICKET_KEY_FILE, SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED, @@ -1337,6 +1338,15 @@ int option_lookup_token(const char *name, size_t namelen) { break; } break; + case 31: + switch (name[30]) { + case 's': + if (util::strieq_l("tls-session-cache-memcached-tl", name, 30)) { + return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_TLS; + } + break; + } + break; case 33: switch (name[32]) { case 'l': @@ -2229,6 +2239,10 @@ int parse_config(const char *opt, const char *optarg, case SHRPX_OPTID_BACKEND_TLS_SESSION_CACHE_PER_WORKER: return parse_uint(&mod_config()->tls.downstream_session_cache_per_worker, opt, optarg); + case SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_TLS: + mod_config()->tls.session_cache.memcached.tls = util::strieq(optarg, "yes"); + + return 0; case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 2d511fda..5a67d06e 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -209,6 +209,8 @@ constexpr char SHRPX_OPT_NO_HTTP2_CIPHER_BLACK_LIST[] = constexpr char SHRPX_OPT_BACKEND_HTTP1_TLS[] = "backend-http1-tls"; constexpr char SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER[] = "backend-tls-session-cache-per-worker"; +constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_TLS[] = + "tls-session-cache-memcached-tls"; constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -355,6 +357,7 @@ struct TLSConfig { Address addr; uint16_t port; std::unique_ptr host; + bool tls; } memcached; } session_cache; diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index c9066b69..1ff4c7ef 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -193,8 +193,21 @@ int ConnectionHandler::create_single_worker() { all_ssl_ctx_.push_back(cl_ssl_ctx); } - single_worker_ = make_unique(loop_, sv_ssl_ctx, cl_ssl_ctx, cert_tree, - ticket_keys_); + auto &session_cacheconf = get_config()->tls.session_cache; + + SSL_CTX *session_cache_ssl_ctx = nullptr; + if (session_cacheconf.memcached.tls) { + session_cache_ssl_ctx = ssl::create_ssl_client_context( +#ifdef HAVE_NEVERBLEED + nb_.get(), +#endif // HAVE_NEVERBLEED + nullptr, nullptr, nullptr, StringRef(), nullptr); + all_ssl_ctx_.push_back(session_cache_ssl_ctx); + } + + single_worker_ = + make_unique(loop_, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, + cert_tree, ticket_keys_); #ifdef HAVE_MRUBY if (single_worker_->create_mruby_context() != 0) { return -1; @@ -225,11 +238,23 @@ int ConnectionHandler::create_worker_thread(size_t num) { all_ssl_ctx_.push_back(cl_ssl_ctx); } + auto &session_cacheconf = get_config()->tls.session_cache; + for (size_t i = 0; i < num; ++i) { auto loop = ev_loop_new(0); - auto worker = make_unique(loop, sv_ssl_ctx, cl_ssl_ctx, cert_tree, - ticket_keys_); + SSL_CTX *session_cache_ssl_ctx = nullptr; + if (session_cacheconf.memcached.tls) { + session_cache_ssl_ctx = ssl::create_ssl_client_context( +#ifdef HAVE_NEVERBLEED + nb_.get(), +#endif // HAVE_NEVERBLEED + nullptr, nullptr, nullptr, StringRef(), nullptr); + all_ssl_ctx_.push_back(session_cache_ssl_ctx); + } + auto worker = + make_unique(loop, sv_ssl_ctx, cl_ssl_ctx, session_cache_ssl_ctx, + cert_tree, ticket_keys_); #ifdef HAVE_MRUBY if (worker->create_mruby_context() != 0) { return -1; diff --git a/src/shrpx_memcached_connection.cc b/src/shrpx_memcached_connection.cc index 8c4d1cc2..4df031d8 100644 --- a/src/shrpx_memcached_connection.cc +++ b/src/shrpx_memcached_connection.cc @@ -32,6 +32,7 @@ #include "shrpx_memcached_request.h" #include "shrpx_memcached_result.h" #include "shrpx_config.h" +#include "shrpx_ssl.h" #include "util.h" namespace shrpx { @@ -78,7 +79,7 @@ void connectcb(struct ev_loop *loop, ev_io *w, int revents) { auto conn = static_cast(w->data); auto mconn = static_cast(conn->data); - if (mconn->on_connect() != 0) { + if (mconn->connected() != 0) { mconn->disconnect(); return; } @@ -91,11 +92,17 @@ constexpr ev_tstamp write_timeout = 10.; constexpr ev_tstamp read_timeout = 10.; MemcachedConnection::MemcachedConnection(const Address *addr, - struct ev_loop *loop) - : conn_(loop, -1, nullptr, nullptr, write_timeout, read_timeout, {}, {}, + struct ev_loop *loop, SSL_CTX *ssl_ctx, + const StringRef &sni_name, + MemchunkPool *mcpool) + : conn_(loop, -1, nullptr, mcpool, write_timeout, read_timeout, {}, {}, connectcb, readcb, timeoutcb, this, 0, 0.), + do_read_(&MemcachedConnection::noop), + do_write_(&MemcachedConnection::noop), + sni_name_(sni_name.str()), parse_state_{}, addr_(addr), + ssl_ctx_(ssl_ctx), sendsum_(0), connected_(false) {} @@ -127,11 +134,21 @@ void MemcachedConnection::disconnect() { assert(recvbuf_.rleft() == 0); recvbuf_.reset(); + + do_read_ = do_write_ = &MemcachedConnection::noop; } int MemcachedConnection::initiate_connection() { assert(conn_.fd == -1); + if (ssl_ctx_ && !conn_.tls.ssl) { + auto ssl = ssl::create_ssl(ssl_ctx_); + if (!ssl) { + return -1; + } + conn_.set_ssl(ssl); + } + conn_.fd = util::create_nonblock_socket(addr_->su.storage.ss_family); if (conn_.fd == -1) { @@ -153,6 +170,14 @@ int MemcachedConnection::initiate_connection() { return -1; } + if (ssl_ctx_) { + if (!util::numeric_host(sni_name_.c_str())) { + SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name_.c_str()); + } + + conn_.prepare_client_handshake(); + } + if (LOG_ENABLED(INFO)) { MCLOG(INFO, this) << "Connecting to memcached server"; } @@ -168,7 +193,7 @@ int MemcachedConnection::initiate_connection() { return 0; } -int MemcachedConnection::on_connect() { +int MemcachedConnection::connected() { if (!util::check_socket_connected(conn_.fd)) { conn_.wlimit.stopw(); @@ -185,15 +210,59 @@ int MemcachedConnection::on_connect() { connected_ = true; - ev_set_cb(&conn_.wev, writecb); - conn_.rlimit.startw(); ev_timer_again(conn_.loop, &conn_.rt); + ev_set_cb(&conn_.wev, writecb); + + if (conn_.tls.ssl) { + do_read_ = &MemcachedConnection::tls_handshake; + do_write_ = &MemcachedConnection::tls_handshake; + + return 0; + } + + do_read_ = &MemcachedConnection::read_clear; + do_write_ = &MemcachedConnection::write_clear; + return 0; } -int MemcachedConnection::on_write() { +int MemcachedConnection::on_write() { return do_write_(*this); } +int MemcachedConnection::on_read() { return do_read_(*this); } + +int MemcachedConnection::tls_handshake() { + ERR_clear_error(); + + ev_timer_again(conn_.loop, &conn_.rt); + + auto rv = conn_.tls_handshake(); + if (rv == SHRPX_ERR_INPROGRESS) { + return 0; + } + + if (rv < 0) { + return rv; + } + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "SSL/TLS handshake completed"; + } + + auto &tlsconf = get_config()->tls; + + if (!tlsconf.insecure && + ssl::check_cert(conn_.tls.ssl, addr_, StringRef(sni_name_)) != 0) { + return -1; + } + + do_read_ = &MemcachedConnection::read_tls; + do_write_ = &MemcachedConnection::write_tls; + + return on_write(); +} + +int MemcachedConnection::write_tls() { if (!connected_) { return 0; } @@ -207,19 +276,30 @@ int MemcachedConnection::on_write() { return 0; } - int rv; + std::array iov; + std::array buf; for (; !sendq_.empty();) { - rv = send_request(); + auto iovcnt = fill_request_buffer(iov.data(), iov.size()); + auto p = std::begin(buf); + for (size_t i = 0; i < iovcnt; ++i) { + auto &v = iov[i]; + auto n = std::min(static_cast(std::end(buf) - p), v.iov_len); + p = std::copy_n(static_cast(v.iov_base), n, p); + if (p == std::end(buf)) { + break; + } + } - if (rv < 0) { + auto nwrite = conn_.write_tls(buf.data(), p - std::begin(buf)); + if (nwrite < 0) { return -1; } - - if (rv == 1) { - // blocked + if (nwrite == 0) { return 0; } + + drain_send_queue(nwrite); } conn_.wlimit.stopw(); @@ -228,7 +308,70 @@ int MemcachedConnection::on_write() { return 0; } -int MemcachedConnection::on_read() { +int MemcachedConnection::read_tls() { + if (!connected_) { + return 0; + } + + ev_timer_again(conn_.loop, &conn_.rt); + + for (;;) { + auto nread = conn_.read_tls(recvbuf_.last, recvbuf_.wleft()); + + if (nread == 0) { + return 0; + } + + if (nread < 0) { + return -1; + } + + recvbuf_.write(nread); + + if (parse_packet() != 0) { + return -1; + } + } + + return 0; +} + +int MemcachedConnection::write_clear() { + if (!connected_) { + return 0; + } + + ev_timer_again(conn_.loop, &conn_.rt); + + if (sendq_.empty()) { + conn_.wlimit.stopw(); + ev_timer_stop(conn_.loop, &conn_.wt); + + return 0; + } + + std::array iov; + + for (; !sendq_.empty();) { + auto iovcnt = fill_request_buffer(iov.data(), iov.size()); + auto nwrite = conn_.writev_clear(iov.data(), iovcnt); + if (nwrite < 0) { + return -1; + } + if (nwrite == 0) { + return 0; + } + + drain_send_queue(nwrite); + } + + conn_.wlimit.stopw(); + ev_timer_stop(conn_.loop, &conn_.wt); + + return 0; +} + +int MemcachedConnection::read_clear() { if (!connected_) { return 0; } @@ -415,9 +558,8 @@ int MemcachedConnection::parse_packet() { #define MAX_WR_IOVCNT DEFAULT_WR_IOVCNT #endif // !defined(IOV_MAX) || IOV_MAX >= DEFAULT_WR_IOVCNT -int MemcachedConnection::send_request() { - ssize_t nwrite; - +size_t MemcachedConnection::fill_request_buffer(struct iovec *iov, + size_t iovlen) { if (sendsum_ == 0) { for (auto &req : sendq_) { if (req->canceled) { @@ -438,32 +580,27 @@ int MemcachedConnection::send_request() { } } - std::array iov; - size_t iovlen = 0; + size_t iovcnt = 0; for (auto &buf : sendbufv_) { - if (iovlen + 2 > iov.size()) { + if (iovcnt + 2 > iovlen) { break; } auto req = buf.req; if (buf.headbuf.rleft()) { - iov[iovlen++] = {buf.headbuf.pos, buf.headbuf.rleft()}; + iov[iovcnt++] = {buf.headbuf.pos, buf.headbuf.rleft()}; } if (buf.send_value_left) { - iov[iovlen++] = {req->value.data() + req->value.size() - + iov[iovcnt++] = {req->value.data() + req->value.size() - buf.send_value_left, buf.send_value_left}; } } - nwrite = conn_.writev_clear(iov.data(), iovlen); - if (nwrite < 0) { - return -1; - } - if (nwrite == 0) { - return 1; - } + return iovcnt; +} +void MemcachedConnection::drain_send_queue(size_t nwrite) { sendsum_ -= nwrite; while (nwrite > 0) { @@ -488,8 +625,6 @@ int MemcachedConnection::send_request() { recvq_.push_back(std::move(sendq_.front())); sendq_.pop_front(); } - - return 0; } size_t MemcachedConnection::serialized_size(MemcachedRequest *req) { @@ -549,4 +684,6 @@ int MemcachedConnection::add_request(std::unique_ptr req) { // TODO should we start write timer too? void MemcachedConnection::signal_write() { conn_.wlimit.startw(); } +int MemcachedConnection::noop() { return 0; } + } // namespace shrpx diff --git a/src/shrpx_memcached_connection.h b/src/shrpx_memcached_connection.h index 43d27c46..c9552198 100644 --- a/src/shrpx_memcached_connection.h +++ b/src/shrpx_memcached_connection.h @@ -93,7 +93,9 @@ constexpr uint8_t MEMCACHED_RES_MAGIC = 0x81; // https://code.google.com/p/memcached/wiki/MemcacheBinaryProtocol class MemcachedConnection { public: - MemcachedConnection(const Address *addr, struct ev_loop *loop); + MemcachedConnection(const Address *addr, struct ev_loop *loop, + SSL_CTX *ssl_ctx, const StringRef &sni_name, + MemchunkPool *mcpool); ~MemcachedConnection(); void disconnect(); @@ -101,23 +103,38 @@ public: int add_request(std::unique_ptr req); int initiate_connection(); - int on_connect(); + int connected(); int on_write(); int on_read(); - int send_request(); + + int write_clear(); + int read_clear(); + + int tls_handshake(); + int write_tls(); + int read_tls(); + + size_t fill_request_buffer(struct iovec *iov, size_t iovlen); + void drain_send_queue(size_t nwrite); + void make_request(MemcachedSendbuf *sendbuf, MemcachedRequest *req); int parse_packet(); size_t serialized_size(MemcachedRequest *req); void signal_write(); + int noop(); + private: Connection conn_; std::deque> recvq_; std::deque> sendq_; std::deque sendbufv_; + std::function do_read_, do_write_; + std::string sni_name_; MemcachedParseState parse_state_; const Address *addr_; + SSL_CTX *ssl_ctx_; // Sum of the bytes to be transmitted in sendbufv_. size_t sendsum_; bool connected_; diff --git a/src/shrpx_memcached_dispatcher.cc b/src/shrpx_memcached_dispatcher.cc index 28c00f13..796fef8e 100644 --- a/src/shrpx_memcached_dispatcher.cc +++ b/src/shrpx_memcached_dispatcher.cc @@ -31,8 +31,12 @@ namespace shrpx { MemcachedDispatcher::MemcachedDispatcher(const Address *addr, - struct ev_loop *loop) - : loop_(loop), mconn_(make_unique(addr, loop_)) {} + struct ev_loop *loop, SSL_CTX *ssl_ctx, + const StringRef &sni_name, + MemchunkPool *mcpool) + : loop_(loop), + mconn_(make_unique(addr, loop_, ssl_ctx, sni_name, + mcpool)) {} MemcachedDispatcher::~MemcachedDispatcher() {} diff --git a/src/shrpx_memcached_dispatcher.h b/src/shrpx_memcached_dispatcher.h index b3ea4866..64021c68 100644 --- a/src/shrpx_memcached_dispatcher.h +++ b/src/shrpx_memcached_dispatcher.h @@ -31,6 +31,10 @@ #include +#include + +#include "memchunk.h" + namespace shrpx { struct MemcachedRequest; @@ -39,7 +43,9 @@ struct Address; class MemcachedDispatcher { public: - MemcachedDispatcher(const Address *addr, struct ev_loop *loop); + MemcachedDispatcher(const Address *addr, struct ev_loop *loop, + SSL_CTX *ssl_ctx, const StringRef &sni_name, + MemchunkPool *mcpool); ~MemcachedDispatcher(); int add_request(std::unique_ptr req); diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 66c703ad..ee82022d 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -982,7 +982,7 @@ int verify_hostname(X509 *cert, const char *hostname, size_t hlen, } } // namespace -int check_cert(SSL *ssl, const DownstreamAddr *addr) { +int check_cert(SSL *ssl, const Address *addr, const StringRef &host) { auto cert = SSL_get_peer_certificate(ssl); if (!cert) { LOG(ERROR) << "No certificate found"; @@ -996,18 +996,21 @@ int check_cert(SSL *ssl, const DownstreamAddr *addr) { return -1; } - auto &backend_sni_name = get_config()->tls.backend_sni_name; - - auto hostname = !backend_sni_name.empty() ? StringRef(backend_sni_name) - : StringRef(addr->host); - if (verify_hostname(cert, hostname.c_str(), hostname.size(), &addr->addr) != - 0) { + if (verify_hostname(cert, host.c_str(), host.size(), addr) != 0) { LOG(ERROR) << "Certificate verification failed: hostname does not match"; return -1; } return 0; } +int check_cert(SSL *ssl, const DownstreamAddr *addr) { + auto &backend_sni_name = get_config()->tls.backend_sni_name; + + auto hostname = !backend_sni_name.empty() ? StringRef(backend_sni_name) + : StringRef(addr->host); + return check_cert(ssl, &addr->addr, hostname); +} + CertLookupTree::CertLookupTree() { root_.ssl_ctx = nullptr; root_.str = nullptr; diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h index da7c1667..4152a8ed 100644 --- a/src/shrpx_ssl.h +++ b/src/shrpx_ssl.h @@ -46,6 +46,7 @@ class Worker; class DownstreamConnectionPool; struct DownstreamAddr; struct UpstreamAddr; +struct Address; namespace ssl { @@ -83,9 +84,8 @@ SSL_CTX *create_ssl_client_context( ClientHandler *accept_connection(Worker *worker, int fd, sockaddr *addr, int addrlen, const UpstreamAddr *faddr); -// Check peer's certificate against first downstream address in -// Config::downstream_addrs. We only consider first downstream since -// we use this function for HTTP/2 downstream link only. +// Check peer's certificate against given |address| and |host|. +int check_cert(SSL *ssl, const Address *addr, const StringRef &host); int check_cert(SSL *ssl, const DownstreamAddr *addr); // Retrieves DNS and IP address in subjectAltNames and commonName from diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 5e46e398..e663eace 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -68,6 +68,7 @@ std::random_device rd; } // namespace Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, + SSL_CTX *tls_session_cache_memcached_ssl_ctx, ssl::CertLookupTree *cert_tree, const std::shared_ptr &ticket_keys) : randgen_(rd()), @@ -92,7 +93,9 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, if (session_cacheconf.memcached.host) { session_cache_memcached_dispatcher_ = make_unique( - &session_cacheconf.memcached.addr, loop); + &session_cacheconf.memcached.addr, loop, + tls_session_cache_memcached_ssl_ctx, + session_cacheconf.memcached.host.get(), &mcpool_); } auto &downstreamconf = get_config()->conn.downstream; diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index c578d9fb..8d352429 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -112,6 +112,7 @@ struct SessionCacheEntry { class Worker { public: Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, + SSL_CTX *tls_session_cache_memcached_ssl_ctx, ssl::CertLookupTree *cert_tree, const std::shared_ptr &ticket_keys); ~Worker(); diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc index 0e05e19f..e7a30311 100644 --- a/src/shrpx_worker_process.cc +++ b/src/shrpx_worker_process.cc @@ -426,7 +426,8 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { if (ticketconf.memcached.host) { conn_handler.set_tls_ticket_key_memcached_dispatcher( - make_unique(&ticketconf.memcached.addr, loop)); + make_unique(&ticketconf.memcached.addr, loop, + nullptr, "", nullptr)); ev_timer_init(&renew_ticket_key_timer, memcached_get_ticket_key_cb, 0., 0.); From 3297a303bfc0320a24da137616ff17213119b891 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 13 Feb 2016 00:20:38 +0900 Subject: [PATCH 21/85] nghttpx: Add client auth options for session cache memcached TLS connection --- gennghttpxfun.py | 4 +++- src/shrpx.cc | 20 ++++++++++++++++++++ src/shrpx_config.cc | 25 +++++++++++++++++++++++++ src/shrpx_config.h | 7 +++++++ src/shrpx_connection_handler.cc | 18 ++++++++++++------ src/shrpx_ssl.cc | 24 +++++++++++++----------- src/shrpx_ssl.h | 4 ++-- src/template.h | 7 +++++++ 8 files changed, 89 insertions(+), 20 deletions(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 500d47a6..0df2c3be 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -115,7 +115,9 @@ OPTIONS = [ "max-header-fields", "no-http2-cipher-black-list", "backend-http1-tls", - "backend-tls-session-cache-per-worker" + "backend-tls-session-cache-per-worker", + "tls-session-cache-memcached-cert-file", + "tls-session-cache-memcached-private-key-file" ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index 04e85664..7f365ed4 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1567,6 +1567,12 @@ SSL/TLS: --tls-session-cache-memcached-tls Enable SSL/TLS on memcached connections to store session cache. + --tls-session-cache-memcached-cert-file= + Path to client certificate for memcached connections to + store session cache. + --tls-session-cache-memcached-private-key-file= + Path to client private key for memcached connections to + store session cache. --tls-dyn-rec-warmup-threshold= Specify the threshold size for TLS dynamic record size behaviour. During a TLS session, after the threshold @@ -2404,6 +2410,10 @@ int main(int argc, char **argv) { {SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER, required_argument, &flag, 107}, {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_TLS, no_argument, &flag, 108}, + {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE, required_argument, + &flag, 109}, + {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE, + required_argument, &flag, 110}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2866,6 +2876,16 @@ int main(int argc, char **argv) { // --tls-session-cache-memcached-tls cmdcfgs.emplace_back(SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_TLS, "yes"); break; + case 109: + // --tls-session-cache-memcached-cert-file + cmdcfgs.emplace_back(SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE, + optarg); + break; + case 110: + // --tls-session-cache-memcached-private-key-file + cmdcfgs.emplace_back( + SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE, optarg); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 68e4da5b..0b0ff5ff 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -758,6 +758,8 @@ enum { SHRPX_OPTID_TLS_DYN_REC_WARMUP_THRESHOLD, SHRPX_OPTID_TLS_PROTO_LIST, SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED, + SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE, + SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE, SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_TLS, SHRPX_OPTID_TLS_TICKET_KEY_CIPHER, SHRPX_OPTID_TLS_TICKET_KEY_FILE, @@ -1406,6 +1408,11 @@ int option_lookup_token(const char *name, size_t namelen) { break; case 37: switch (name[36]) { + case 'e': + if (util::strieq_l("tls-session-cache-memcached-cert-fil", name, 36)) { + return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE; + } + break; case 's': if (util::strieq_l("frontend-http2-connection-window-bit", name, 36)) { return SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS; @@ -1422,6 +1429,16 @@ int option_lookup_token(const char *name, size_t namelen) { break; } break; + case 44: + switch (name[43]) { + case 'e': + if (util::strieq_l("tls-session-cache-memcached-private-key-fil", name, + 43)) { + return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE; + } + break; + } + break; } return -1; } @@ -2242,6 +2259,14 @@ int parse_config(const char *opt, const char *optarg, case SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_TLS: mod_config()->tls.session_cache.memcached.tls = util::strieq(optarg, "yes"); + return 0; + case SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE: + mod_config()->tls.session_cache.memcached.cert_file = optarg; + + return 0; + case SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE: + mod_config()->tls.session_cache.memcached.private_key_file = optarg; + return 0; case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 5a67d06e..2bf6aef7 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -211,6 +211,10 @@ constexpr char SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER[] = "backend-tls-session-cache-per-worker"; constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_TLS[] = "tls-session-cache-memcached-tls"; +constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE[] = + "tls-session-cache-memcached-cert-file"; +constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE[] = + "tls-session-cache-memcached-private-key-file"; constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -357,6 +361,9 @@ struct TLSConfig { Address addr; uint16_t port; std::unique_ptr host; + // Client private key and certificate for authentication + ImmutableString private_key_file; + ImmutableString cert_file; bool tls; } memcached; } session_cache; diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 1ff4c7ef..da023d9c 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -193,15 +193,18 @@ int ConnectionHandler::create_single_worker() { all_ssl_ctx_.push_back(cl_ssl_ctx); } - auto &session_cacheconf = get_config()->tls.session_cache; + auto &tlsconf = get_config()->tls; + auto &memcachedconf = get_config()->tls.session_cache.memcached; SSL_CTX *session_cache_ssl_ctx = nullptr; - if (session_cacheconf.memcached.tls) { + if (memcachedconf.tls) { session_cache_ssl_ctx = ssl::create_ssl_client_context( #ifdef HAVE_NEVERBLEED nb_.get(), #endif // HAVE_NEVERBLEED - nullptr, nullptr, nullptr, StringRef(), nullptr); + StringRef::from_maybe_nullptr(tlsconf.cacert.get()), + StringRef(memcachedconf.cert_file), + StringRef(memcachedconf.private_key_file), StringRef(), nullptr); all_ssl_ctx_.push_back(session_cache_ssl_ctx); } @@ -238,18 +241,21 @@ int ConnectionHandler::create_worker_thread(size_t num) { all_ssl_ctx_.push_back(cl_ssl_ctx); } - auto &session_cacheconf = get_config()->tls.session_cache; + auto &tlsconf = get_config()->tls; + auto &memcachedconf = get_config()->tls.session_cache.memcached; for (size_t i = 0; i < num; ++i) { auto loop = ev_loop_new(0); SSL_CTX *session_cache_ssl_ctx = nullptr; - if (session_cacheconf.memcached.tls) { + if (memcachedconf.tls) { session_cache_ssl_ctx = ssl::create_ssl_client_context( #ifdef HAVE_NEVERBLEED nb_.get(), #endif // HAVE_NEVERBLEED - nullptr, nullptr, nullptr, StringRef(), nullptr); + StringRef::from_maybe_nullptr(tlsconf.cacert.get()), + StringRef(memcachedconf.cert_file), + StringRef(memcachedconf.private_key_file), StringRef(), nullptr); all_ssl_ctx_.push_back(session_cache_ssl_ctx); } auto worker = diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index ee82022d..5d8a2360 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -662,8 +662,8 @@ SSL_CTX *create_ssl_client_context( #ifdef HAVE_NEVERBLEED neverbleed_t *nb, #endif // HAVE_NEVERBLEED - const char *cacert, const char *cert_file, const char *private_key_file, - const StringRef &alpn, + const StringRef &cacert, const StringRef &cert_file, + const StringRef &private_key_file, const StringRef &alpn, int (*next_proto_select_cb)(SSL *s, unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, void *arg)) { @@ -702,8 +702,8 @@ SSL_CTX *create_ssl_client_context( << ERR_error_string(ERR_get_error(), nullptr); } - if (cacert) { - if (SSL_CTX_load_verify_locations(ssl_ctx, cacert, nullptr) != 1) { + if (!cacert.empty()) { + if (SSL_CTX_load_verify_locations(ssl_ctx, cacert.c_str(), nullptr) != 1) { LOG(FATAL) << "Could not load trusted ca certificates from " << cacert << ": " << ERR_error_string(ERR_get_error(), nullptr); @@ -711,8 +711,8 @@ SSL_CTX *create_ssl_client_context( } } - if (cert_file) { - if (SSL_CTX_use_certificate_chain_file(ssl_ctx, cert_file) != 1) { + if (!cert_file.empty()) { + if (SSL_CTX_use_certificate_chain_file(ssl_ctx, cert_file.c_str()) != 1) { LOG(FATAL) << "Could not load client certificate from " << cert_file << ": " << ERR_error_string(ERR_get_error(), nullptr); @@ -720,9 +720,9 @@ SSL_CTX *create_ssl_client_context( } } - if (private_key_file) { + if (!private_key_file.empty()) { #ifndef HAVE_NEVERBLEED - if (SSL_CTX_use_PrivateKey_file(ssl_ctx, private_key_file, + if (SSL_CTX_use_PrivateKey_file(ssl_ctx, private_key_file.c_str(), SSL_FILETYPE_PEM) != 1) { LOG(FATAL) << "Could not load client private key from " << private_key_file << ": " @@ -731,7 +731,7 @@ SSL_CTX *create_ssl_client_context( } #else // HAVE_NEVERBLEED std::array errbuf; - if (neverbleed_load_private_key_file(nb, ssl_ctx, private_key_file, + if (neverbleed_load_private_key_file(nb, ssl_ctx, private_key_file.c_str(), errbuf.data()) != 1) { LOG(FATAL) << "neverbleed_load_private_key_file: could not load client " "private key from " << private_key_file << ": " @@ -1323,8 +1323,10 @@ SSL_CTX *setup_downstream_client_ssl_context( #ifdef HAVE_NEVERBLEED nb, #endif // HAVE_NEVERBLEED - tlsconf.cacert.get(), tlsconf.client.cert_file.get(), - tlsconf.client.private_key_file.get(), alpn, next_proto_select_cb); + StringRef::from_maybe_nullptr(tlsconf.cacert.get()), + StringRef::from_maybe_nullptr(tlsconf.client.cert_file.get()), + StringRef::from_maybe_nullptr(tlsconf.client.private_key_file.get()), + alpn, next_proto_select_cb); } CertLookupTree *create_cert_lookup_tree() { diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h index 4152a8ed..ca93da14 100644 --- a/src/shrpx_ssl.h +++ b/src/shrpx_ssl.h @@ -75,8 +75,8 @@ SSL_CTX *create_ssl_client_context( #ifdef HAVE_NEVERBLEED neverbleed_t *nb, #endif // HAVE_NEVERBLEED - const char *cacert, const char *cert_file, const char *private_key_file, - const StringRef &alpn, + const StringRef &cacert, const StringRef &cert_file, + const StringRef &private_key_file, const StringRef &alpn, int (*next_proto_select_cb)(SSL *s, unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, void *arg)); diff --git a/src/template.h b/src/template.h index 0d4c4ba2..1a48d787 100644 --- a/src/template.h +++ b/src/template.h @@ -402,6 +402,13 @@ public: static StringRef from_lit(const CharT(&s)[N]) { return StringRef(s, N - 1); } + static StringRef from_maybe_nullptr(const char *s) { + if (s == nullptr) { + return StringRef(); + } + + return StringRef(s); + } const_iterator begin() const { return base; }; const_iterator cbegin() const { return base; }; From 3a41e4dd1a029ff3254fde29d58d17f90459344a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 13 Feb 2016 18:17:11 +0900 Subject: [PATCH 22/85] nghttpx: Add encryption support for TLS ticket key retrieval --- gennghttpxfun.py | 5 +++- src/shrpx.cc | 48 ++++++++++++++++++++++++++------- src/shrpx_config.cc | 33 +++++++++++++++++++++++ src/shrpx_config.h | 10 +++++++ src/shrpx_connection_handler.cc | 17 ++++++++++++ src/shrpx_connection_handler.h | 1 + src/shrpx_worker_process.cc | 14 ++++++++-- 7 files changed, 115 insertions(+), 13 deletions(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 0df2c3be..b424fe17 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -117,7 +117,10 @@ OPTIONS = [ "backend-http1-tls", "backend-tls-session-cache-per-worker", "tls-session-cache-memcached-cert-file", - "tls-session-cache-memcached-private-key-file" + "tls-session-cache-memcached-private-key-file", + "tls-ticket-key-memcached-tls", + "tls-ticket-key-memcached-cert-file", + "tls-ticket-key-memcached-private-key-file" ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index 7f365ed4..72e91bac 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1520,16 +1520,16 @@ SSL/TLS: ticket key sharing between nghttpx instances is not required. --tls-ticket-key-memcached=, - Specify address of memcached server to store session - cache. This enables shared TLS ticket key between - multiple nghttpx instances. nghttpx does not set TLS - ticket key to memcached. The external ticket key - generator is required. nghttpx just gets TLS ticket - keys from memcached, and use them, possibly replacing - current set of keys. It is up to extern TLS ticket key - generator to rotate keys frequently. See "TLS SESSION - TICKET RESUMPTION" section in manual page to know the - data format in memcached entry. + Specify address of memcached server to get TLS ticket + keys for session resumption. This enables shared TLS + ticket key between multiple nghttpx instances. nghttpx + does not set TLS ticket key to memcached. The external + ticket key generator is required. nghttpx just gets TLS + ticket keys from memcached, and use them, possibly + replacing current set of keys. It is up to extern TLS + ticket key generator to rotate keys frequently. See + "TLS SESSION TICKET RESUMPTION" section in manual page + to know the data format in memcached entry. --tls-ticket-key-memcached-interval= Set interval to get TLS ticket keys from memcached. Default: )" @@ -1550,6 +1550,15 @@ SSL/TLS: Specify cipher to encrypt TLS session ticket. Specify either aes-128-cbc or aes-256-cbc. By default, aes-128-cbc is used. + --tls-ticket-key-memcached-tls + Enable SSL/TLS on memcached connections to get TLS + ticket keys. + --tls-ticket-key-memcached-cert-file= + Path to client certificate for memcached connections to + get TLS ticket keys. + --tls-ticket-key-memcached-private-key-file= + Path to client private key for memcached connections to + get TLS ticket keys. --fetch-ocsp-response-file= Path to fetch-ocsp-response script file. It should be absolute path. @@ -2414,6 +2423,11 @@ int main(int argc, char **argv) { &flag, 109}, {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE, required_argument, &flag, 110}, + {SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_TLS, no_argument, &flag, 111}, + {SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_CERT_FILE, required_argument, &flag, + 112}, + {SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE, required_argument, + &flag, 113}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2886,6 +2900,20 @@ int main(int argc, char **argv) { cmdcfgs.emplace_back( SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE, optarg); break; + case 111: + // --tls-ticket-key-memcached-tls + cmdcfgs.emplace_back(SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_TLS, "yes"); + break; + case 112: + // --tls-ticket-key-memcached-cert-file + cmdcfgs.emplace_back(SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_CERT_FILE, + optarg); + break; + case 113: + // --tls-ticket-key-memcached-private-key-file + cmdcfgs.emplace_back( + SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE, optarg); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 0b0ff5ff..ad7e4f65 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -764,9 +764,12 @@ enum { SHRPX_OPTID_TLS_TICKET_KEY_CIPHER, SHRPX_OPTID_TLS_TICKET_KEY_FILE, SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED, + SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_CERT_FILE, SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_INTERVAL, SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_MAX_FAIL, SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_MAX_RETRY, + SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE, + SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_TLS, SHRPX_OPTID_USER, SHRPX_OPTID_VERIFY_CLIENT, SHRPX_OPTID_VERIFY_CLIENT_CACERT, @@ -1328,6 +1331,9 @@ int option_lookup_token(const char *name, size_t namelen) { if (util::strieq_l("http2-max-concurrent-stream", name, 27)) { return SHRPX_OPTID_HTTP2_MAX_CONCURRENT_STREAMS; } + if (util::strieq_l("tls-ticket-key-memcached-tl", name, 27)) { + return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_TLS; + } break; } break; @@ -1363,6 +1369,11 @@ int option_lookup_token(const char *name, size_t namelen) { break; case 34: switch (name[33]) { + case 'e': + if (util::strieq_l("tls-ticket-key-memcached-cert-fil", name, 33)) { + return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_CERT_FILE; + } + break; case 'r': if (util::strieq_l("frontend-http2-dump-request-heade", name, 33)) { return SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER; @@ -1429,6 +1440,16 @@ int option_lookup_token(const char *name, size_t namelen) { break; } break; + case 41: + switch (name[40]) { + case 'e': + if (util::strieq_l("tls-ticket-key-memcached-private-key-fil", name, + 40)) { + return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE; + } + break; + } + break; case 44: switch (name[43]) { case 'e': @@ -2267,6 +2288,18 @@ int parse_config(const char *opt, const char *optarg, case SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE: mod_config()->tls.session_cache.memcached.private_key_file = optarg; + return 0; + case SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_TLS: + mod_config()->tls.ticket.memcached.tls = util::strieq(optarg, "yes"); + + return 0; + case SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_CERT_FILE: + mod_config()->tls.ticket.memcached.cert_file = optarg; + + return 0; + case SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE: + mod_config()->tls.ticket.memcached.private_key_file = optarg; + return 0; case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 2bf6aef7..19f17cb2 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -215,6 +215,12 @@ constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE[] = "tls-session-cache-memcached-cert-file"; constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE[] = "tls-session-cache-memcached-private-key-file"; +constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_TLS[] = + "tls-ticket-key-memcached-tls"; +constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_CERT_FILE[] = + "tls-ticket-key-memcached-cert-file"; +constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE[] = + "tls-ticket-key-memcached-private-key-file"; constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -341,6 +347,9 @@ struct TLSConfig { Address addr; uint16_t port; std::unique_ptr host; + // Client private key and certificate for authentication + ImmutableString private_key_file; + ImmutableString cert_file; ev_tstamp interval; // Maximum number of retries when getting TLS ticket key from // mamcached, due to network error. @@ -348,6 +357,7 @@ struct TLSConfig { // Maximum number of consecutive error from memcached, when this // limit reached, TLS ticket is disabled. size_t max_fail; + bool tls; } memcached; std::vector files; const EVP_CIPHER *cipher; diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index da023d9c..588e814b 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -759,6 +759,23 @@ void ConnectionHandler::schedule_next_tls_ticket_key_memcached_get( ev_timer_start(loop_, w); } +SSL_CTX *ConnectionHandler::create_tls_ticket_key_memcached_ssl_ctx() { + auto &tlsconf = get_config()->tls; + auto &memcachedconf = get_config()->tls.ticket.memcached; + + auto ssl_ctx = ssl::create_ssl_client_context( +#ifdef HAVE_NEVERBLEED + nb_.get(), +#endif // HAVE_NEVERBLEED + StringRef::from_maybe_nullptr(tlsconf.cacert.get()), + StringRef(memcachedconf.cert_file), + StringRef(memcachedconf.private_key_file), StringRef(), nullptr); + + all_ssl_ctx_.push_back(ssl_ctx); + + return ssl_ctx; +} + #ifdef HAVE_NEVERBLEED void ConnectionHandler::set_neverbleed(std::unique_ptr nb) { nb_ = std::move(nb); diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h index 81d872f5..8602b359 100644 --- a/src/shrpx_connection_handler.h +++ b/src/shrpx_connection_handler.h @@ -129,6 +129,7 @@ public: on_tls_ticket_key_get_success(const std::shared_ptr &ticket_keys, ev_timer *w); void schedule_next_tls_ticket_key_memcached_get(ev_timer *w); + SSL_CTX *create_tls_ticket_key_memcached_ssl_ctx(); #ifdef HAVE_NEVERBLEED void set_neverbleed(std::unique_ptr nb); diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc index e7a30311..26863d82 100644 --- a/src/shrpx_worker_process.cc +++ b/src/shrpx_worker_process.cc @@ -420,14 +420,24 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { #endif // HAVE_NEVERBLEED + MemchunkPool mcpool; + ev_timer renew_ticket_key_timer; if (!upstreamconf.no_tls) { auto &ticketconf = get_config()->tls.ticket; + auto &memcachedconf = ticketconf.memcached; if (ticketconf.memcached.host) { + SSL_CTX *ssl_ctx = nullptr; + + if (memcachedconf.tls) { + ssl_ctx = conn_handler.create_tls_ticket_key_memcached_ssl_ctx(); + } + conn_handler.set_tls_ticket_key_memcached_dispatcher( - make_unique(&ticketconf.memcached.addr, loop, - nullptr, "", nullptr)); + make_unique( + &ticketconf.memcached.addr, loop, ssl_ctx, + StringRef(memcachedconf.host.get()), &mcpool)); ev_timer_init(&renew_ticket_key_timer, memcached_get_ticket_key_cb, 0., 0.); From c0078ab45a3b1889672eef94ec2d03e0d5531825 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 13 Feb 2016 18:45:23 +0900 Subject: [PATCH 23/85] nghttpx: Add options to specify address family of memcached connections --- gennghttpxfun.py | 4 +++- src/shrpx.cc | 39 +++++++++++++++++++++++++++++++++++-- src/shrpx_config.cc | 47 +++++++++++++++++++++++++++++++++++++++++++++ src/shrpx_config.h | 10 ++++++++++ 4 files changed, 97 insertions(+), 3 deletions(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index b424fe17..cda5bb5c 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -118,9 +118,11 @@ OPTIONS = [ "backend-tls-session-cache-per-worker", "tls-session-cache-memcached-cert-file", "tls-session-cache-memcached-private-key-file", + "tls-session-cache-memcached-address-family", "tls-ticket-key-memcached-tls", "tls-ticket-key-memcached-cert-file", - "tls-ticket-key-memcached-private-key-file" + "tls-ticket-key-memcached-private-key-file", + "tls-ticket-key-memcached-address-family", ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index 72e91bac..4561f833 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1051,6 +1051,13 @@ void fill_default_config() { memcachedconf.max_retry = 3; memcachedconf.max_fail = 2; memcachedconf.interval = 10_min; + memcachedconf.family = AF_UNSPEC; + } + + auto &session_cacheconf = tlsconf.session_cache; + { + auto &memcachedconf = session_cacheconf.memcached; + memcachedconf.family = AF_UNSPEC; } ticketconf.cipher = EVP_aes_128_cbc(); @@ -1530,6 +1537,13 @@ SSL/TLS: ticket key generator to rotate keys frequently. See "TLS SESSION TICKET RESUMPTION" section in manual page to know the data format in memcached entry. + --tls-ticket-key-memcached-address-family=(auto|IPv4|IPv6) + Specify address family of memcached connections to get + TLS ticket keys. If "auto" is given, both IPv4 and IPv6 + are considered. If "IPv4" is given, only IPv4 address + is considered. If "IPv6" is given, only IPv6 address is + considered. + Default: auto --tls-ticket-key-memcached-interval= Set interval to get TLS ticket keys from memcached. Default: )" @@ -1573,6 +1587,13 @@ SSL/TLS: Specify address of memcached server to store session cache. This enables shared session cache between multiple nghttpx instances. + --tls-session-cache-memcached-address-family=(auto|IPv4|IPv6) + Specify address family of memcached connections to store + session cache. If "auto" is given, both IPv4 and IPv6 + are considered. If "IPv4" is given, only IPv4 address + is considered. If "IPv6" is given, only IPv6 address is + considered. + Default: auto --tls-session-cache-memcached-tls Enable SSL/TLS on memcached connections to store session cache. @@ -2199,7 +2220,7 @@ void process_options( auto &memcachedconf = tlsconf.session_cache.memcached; if (memcachedconf.host) { if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.get(), - memcachedconf.port, AF_UNSPEC) == -1) { + memcachedconf.port, memcachedconf.family) == -1) { exit(EXIT_FAILURE); } } @@ -2209,7 +2230,7 @@ void process_options( auto &memcachedconf = tlsconf.ticket.memcached; if (memcachedconf.host) { if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.get(), - memcachedconf.port, AF_UNSPEC) == -1) { + memcachedconf.port, memcachedconf.family) == -1) { exit(EXIT_FAILURE); } } @@ -2428,6 +2449,10 @@ int main(int argc, char **argv) { 112}, {SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE, required_argument, &flag, 113}, + {SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY, required_argument, + &flag, 114}, + {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY, + required_argument, &flag, 115}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2914,6 +2939,16 @@ int main(int argc, char **argv) { cmdcfgs.emplace_back( SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE, optarg); break; + case 114: + // --tls-ticket-key-memcached-address-family + cmdcfgs.emplace_back(SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY, + optarg); + break; + case 115: + // --tls-session-cache-memcached-address-family + cmdcfgs.emplace_back( + SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY, optarg); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index ad7e4f65..b7409358 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -575,6 +575,26 @@ std::vector parse_log_format(const char *optarg) { return res; } +namespace { +int parse_address_family(int *dest, const char *opt, const char *optarg) { + if (util::strieq("auto", optarg)) { + *dest = AF_UNSPEC; + return 0; + } + if (util::strieq("IPv4", optarg)) { + *dest = AF_INET; + return 0; + } + if (util::strieq("IPv6", optarg)) { + *dest = AF_INET6; + return 0; + } + + LOG(ERROR) << opt << ": bad value: '" << optarg << "'"; + return -1; +} +} // namespace + namespace { int parse_duration(ev_tstamp *dest, const char *opt, const char *optarg) { auto t = util::parse_duration_with_unit(optarg); @@ -758,12 +778,14 @@ enum { SHRPX_OPTID_TLS_DYN_REC_WARMUP_THRESHOLD, SHRPX_OPTID_TLS_PROTO_LIST, SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED, + SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY, SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE, SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE, SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_TLS, SHRPX_OPTID_TLS_TICKET_KEY_CIPHER, SHRPX_OPTID_TLS_TICKET_KEY_FILE, SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED, + SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY, SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_CERT_FILE, SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_INTERVAL, SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_MAX_FAIL, @@ -1440,6 +1462,15 @@ int option_lookup_token(const char *name, size_t namelen) { break; } break; + case 39: + switch (name[38]) { + case 'y': + if (util::strieq_l("tls-ticket-key-memcached-address-famil", name, 38)) { + return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY; + } + break; + } + break; case 41: switch (name[40]) { case 'e': @@ -1450,6 +1481,16 @@ int option_lookup_token(const char *name, size_t namelen) { break; } break; + case 42: + switch (name[41]) { + case 'y': + if (util::strieq_l("tls-session-cache-memcached-address-famil", name, + 41)) { + return SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY; + } + break; + } + break; case 44: switch (name[43]) { case 'e': @@ -2301,6 +2342,12 @@ int parse_config(const char *opt, const char *optarg, mod_config()->tls.ticket.memcached.private_key_file = optarg; return 0; + case SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY: + return parse_address_family(&mod_config()->tls.ticket.memcached.family, opt, + optarg); + case SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY: + return parse_address_family( + &mod_config()->tls.session_cache.memcached.family, opt, optarg); case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 19f17cb2..fd2564b9 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -215,12 +215,16 @@ constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE[] = "tls-session-cache-memcached-cert-file"; constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_PRIVATE_KEY_FILE[] = "tls-session-cache-memcached-private-key-file"; +constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY[] = + "tls-session-cache-memcached-address-family"; constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_TLS[] = "tls-ticket-key-memcached-tls"; constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_CERT_FILE[] = "tls-ticket-key-memcached-cert-file"; constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE[] = "tls-ticket-key-memcached-private-key-file"; +constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY[] = + "tls-ticket-key-memcached-address-family"; constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -357,6 +361,9 @@ struct TLSConfig { // Maximum number of consecutive error from memcached, when this // limit reached, TLS ticket is disabled. size_t max_fail; + // Address family of memcached connection. One of either + // AF_INET, AF_INET6 or AF_UNSPEC. + int family; bool tls; } memcached; std::vector files; @@ -374,6 +381,9 @@ struct TLSConfig { // Client private key and certificate for authentication ImmutableString private_key_file; ImmutableString cert_file; + // Address family of memcached connection. One of either + // AF_INET, AF_INET6 or AF_UNSPEC. + int family; bool tls; } memcached; } session_cache; From 84499584258e1f82e03117c4edf261389cc6e604 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 13 Feb 2016 18:50:15 +0900 Subject: [PATCH 24/85] doc: Mention encryption for memcached connections --- doc/nghttpx.h2r | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/nghttpx.h2r b/doc/nghttpx.h2r index 2a36dd70..14512bb0 100644 --- a/doc/nghttpx.h2r +++ b/doc/nghttpx.h2r @@ -150,6 +150,10 @@ insert serialized session data to memcached with as a memcached entry key, with expiry time 12 hours. Session timeout is set to 12 hours. +By default, connections to memcached server are not encrypted. To +enable encryption, use :option:`--tls-session-cache-memcached-tls` +option. + TLS SESSION TICKET RESUMPTION ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -189,6 +193,10 @@ used, LEN must be 48. If keys. The key appeared first is used as encryption key. All the remaining keys are used as decryption only. +By default, connections to memcached server are not encrypted. To +enable encryption, use :option:`--tls-ticket-key-memcached-tls` +option. + If :option:`--tls-ticket-key-file` is given, encryption key is read from the given file. In this case, nghttpx does not rotate key automatically. To rotate key, one has to restart nghttpx (see From 72877379ec2d81a8f2eec2440732c22bd3a39ecb Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 13 Feb 2016 19:09:39 +0900 Subject: [PATCH 25/85] nghttpx: Deprecate --backend-ipv4 and --backend-ipv6 in favor of --backend-address-family --- gennghttpxfun.py | 1 + src/shrpx.cc | 29 ++++++++++++++--------------- src/shrpx_config.cc | 19 +++++++++++++++++-- src/shrpx_config.h | 10 +++++----- 4 files changed, 37 insertions(+), 22 deletions(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index cda5bb5c..043e9426 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -123,6 +123,7 @@ OPTIONS = [ "tls-ticket-key-memcached-cert-file", "tls-ticket-key-memcached-private-key-file", "tls-ticket-key-memcached-address-family", + "backend-address-family" ] LOGVARS = [ diff --git a/src/shrpx.cc b/src/shrpx.cc index 4561f833..9fb64589 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1166,6 +1166,7 @@ void fill_default_config() { downstreamconf.connections_per_host = 8; downstreamconf.request_buffer_size = 16_k; downstreamconf.response_buffer_size = 128_k; + downstreamconf.family = AF_UNSPEC; } } @@ -1270,10 +1271,12 @@ Connections: --backlog= Set listen backlog size. Default: )" << get_config()->conn.listener.backlog << R"( - --backend-ipv4 - Resolve backend hostname to IPv4 address only. - --backend-ipv6 - Resolve backend hostname to IPv6 address only. + --backend-address-family=(auto|IPv4|IPv6) + Specify address family of backend connections. If + "auto" is given, both IPv4 and IPv6 are considered. If + "IPv4" is given, only IPv4 address is considered. If + "IPv6" is given, only IPv6 address is considered. + Default: auto --backend-http-proxy-uri= Specify proxy URI in the form http://[:@]:. If a proxy @@ -2055,12 +2058,6 @@ void process_options( listenerconf.addrs.push_back(std::move(addr)); } - if (downstreamconf.ipv4 && downstreamconf.ipv6) { - LOG(FATAL) << "--backend-ipv4 and --backend-ipv6 cannot be used at the " - << "same time."; - exit(EXIT_FAILURE); - } - if (upstreamconf.worker_connections == 0) { upstreamconf.worker_connections = std::numeric_limits::max(); } @@ -2195,11 +2192,8 @@ void process_options( addr.hostport = ImmutableString(util::make_hostport(addr.host.c_str(), addr.port)); - if (resolve_hostname( - &addr.addr, addr.host.c_str(), addr.port, - downstreamconf.ipv4 - ? AF_INET - : (downstreamconf.ipv6 ? AF_INET6 : AF_UNSPEC)) == -1) { + if (resolve_hostname(&addr.addr, addr.host.c_str(), addr.port, + downstreamconf.family) == -1) { exit(EXIT_FAILURE); } } @@ -2453,6 +2447,7 @@ int main(int argc, char **argv) { &flag, 114}, {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY, required_argument, &flag, 115}, + {SHRPX_OPT_BACKEND_ADDRESS_FAMILY, required_argument, &flag, 116}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2949,6 +2944,10 @@ int main(int argc, char **argv) { cmdcfgs.emplace_back( SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY, optarg); break; + case 116: + // --backend-address-family + cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_ADDRESS_FAMILY, optarg); + break; default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index b7409358..a5d89d8d 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -693,6 +693,7 @@ enum { SHRPX_OPTID_ADD_X_FORWARDED_FOR, SHRPX_OPTID_ALTSVC, SHRPX_OPTID_BACKEND, + SHRPX_OPTID_BACKEND_ADDRESS_FAMILY, SHRPX_OPTID_BACKEND_HTTP_PROXY_URI, SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND, SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST, @@ -1232,6 +1233,11 @@ int option_lookup_token(const char *name, size_t namelen) { return SHRPX_OPTID_FRONTEND_WRITE_TIMEOUT; } break; + case 'y': + if (util::strieq_l("backend-address-famil", name, 21)) { + return SHRPX_OPTID_BACKEND_ADDRESS_FAMILY; + } + break; } break; case 23: @@ -1854,11 +1860,17 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_BACKEND_IPV4: - mod_config()->conn.downstream.ipv4 = util::strieq(optarg, "yes"); + LOG(WARN) << opt + << ": deprecated. Use backend-address-family=IPv4 instead."; + + mod_config()->conn.downstream.family = AF_INET; return 0; case SHRPX_OPTID_BACKEND_IPV6: - mod_config()->conn.downstream.ipv6 = util::strieq(optarg, "yes"); + LOG(WARN) << opt + << ": deprecated. Use backend-address-family=IPv6 instead."; + + mod_config()->conn.downstream.family = AF_INET6; return 0; case SHRPX_OPTID_BACKEND_HTTP_PROXY_URI: { @@ -2348,6 +2360,9 @@ int parse_config(const char *opt, const char *optarg, case SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY: return parse_address_family( &mod_config()->tls.session_cache.memcached.family, opt, optarg); + case SHRPX_OPTID_BACKEND_ADDRESS_FAMILY: + return parse_address_family(&mod_config()->conn.downstream.family, opt, + optarg); case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index fd2564b9..d7b62513 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -225,6 +225,7 @@ constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE[] = "tls-ticket-key-memcached-private-key-file"; constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY[] = "tls-ticket-key-memcached-address-family"; +constexpr char SHRPX_OPT_BACKEND_ADDRESS_FAMILY[] = "backend-address-family"; constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; @@ -567,13 +568,12 @@ struct ConnectionConfig { size_t response_buffer_size; // downstream protocol; this will be determined by given options. shrpx_proto proto; + // Address family of backend connection. One of either AF_INET, + // AF_INET6 or AF_UNSPEC. This is ignored if backend connection + // is made via Unix domain socket. + int family; bool no_tls; bool http1_tls; - // true if IPv4 only; ipv4 and ipv6 are mutually exclusive; and - // (ipv4 && ipv6) must be false. - bool ipv4; - // true if IPv6 only - bool ipv6; } downstream; }; From 63a13ccb183cf34cc0909d4425f7ab3441456781 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 13 Feb 2016 19:15:14 +0900 Subject: [PATCH 26/85] src: Add constexpr to StringRef ctors --- src/template.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/template.h b/src/template.h index 1a48d787..3061ad52 100644 --- a/src/template.h +++ b/src/template.h @@ -387,19 +387,19 @@ public: using const_pointer = const value_type *; using const_iterator = const_pointer; - StringRef() : base(""), len(0) {} + constexpr StringRef() : base(""), len(0) {} explicit StringRef(const std::string &s) : base(s.c_str()), len(s.size()) {} explicit StringRef(const ImmutableString &s) : base(s.c_str()), len(s.size()) {} StringRef(const char *s) : base(s), len(strlen(s)) {} template - StringRef(const CharT *s, size_t n) + constexpr StringRef(const CharT *s, size_t n) : base(reinterpret_cast(s)), len(n) {} template StringRef(InputIt first, InputIt last) : base(first), len(std::distance(first, last)) {} template - static StringRef from_lit(const CharT(&s)[N]) { + constexpr static StringRef from_lit(const CharT(&s)[N]) { return StringRef(s, N - 1); } static StringRef from_maybe_nullptr(const char *s) { From b440f585bcf6774ae32e643904b937c377b4cc37 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 13 Feb 2016 22:19:05 +0900 Subject: [PATCH 27/85] nghttpx: Use Header to store custom request/response header fields --- src/shrpx_config.cc | 19 ++++++++----------- src/shrpx_config.h | 7 ++++--- src/shrpx_config_test.cc | 24 ++++++++++++------------ src/shrpx_http2_downstream_connection.cc | 2 +- src/shrpx_http2_upstream.cc | 2 +- src/shrpx_http_downstream_connection.cc | 4 ++-- src/shrpx_https_upstream.cc | 4 ++-- src/shrpx_spdy_upstream.cc | 4 ++-- 8 files changed, 32 insertions(+), 34 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index a5d89d8d..104c97b6 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -54,9 +54,7 @@ #include "shrpx_log.h" #include "shrpx_ssl.h" #include "shrpx_http.h" -#include "http2.h" #include "util.h" -#include "template.h" #include "base64.h" namespace shrpx { @@ -279,7 +277,7 @@ std::string read_passwd_from_file(const char *filename) { return line; } -std::pair parse_header(const char *optarg) { +Headers::value_type parse_header(const char *optarg) { const auto *colon = strchr(optarg, ':'); if (colon == nullptr || colon == optarg) { @@ -290,16 +288,15 @@ std::pair parse_header(const char *optarg) { for (; *value == '\t' || *value == ' '; ++value) ; - auto p = std::make_pair(std::string(optarg, colon), - std::string(value, strlen(value))); - util::inp_strlower(p.first); + auto p = + Header(std::string(optarg, colon), std::string(value, strlen(value))); + util::inp_strlower(p.name); if (!nghttp2_check_header_name( - reinterpret_cast(p.first.c_str()), p.first.size()) || + reinterpret_cast(p.name.c_str()), p.name.size()) || !nghttp2_check_header_value( - reinterpret_cast(p.second.c_str()), - p.second.size())) { - return {"", ""}; + reinterpret_cast(p.value.c_str()), p.value.size())) { + return Header(); } return p; @@ -2028,7 +2025,7 @@ int parse_config(const char *opt, const char *optarg, case SHRPX_OPTID_ADD_REQUEST_HEADER: case SHRPX_OPTID_ADD_RESPONSE_HEADER: { auto p = parse_header(optarg); - if (p.first.empty()) { + if (p.name.empty()) { LOG(ERROR) << opt << ": invalid header field: " << optarg; return -1; } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index d7b62513..ebc97f8a 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -52,6 +52,7 @@ #include "shrpx_router.h" #include "template.h" +#include "http2.h" using namespace nghttp2; @@ -461,8 +462,8 @@ struct HttpConfig { bool strip_incoming; } xff; std::vector altsvcs; - std::vector> add_request_headers; - std::vector> add_response_headers; + Headers add_request_headers; + Headers add_response_headers; StringRef server_name; size_t request_header_field_buffer; size_t max_request_header_fields; @@ -633,7 +634,7 @@ std::string read_passwd_from_file(const char *filename); // like "NAME: VALUE". We require that NAME is non empty string. ":" // is allowed at the start of the NAME, but NAME == ":" is not // allowed. This function returns pair of NAME and VALUE. -std::pair parse_header(const char *optarg); +Headers::value_type parse_header(const char *optarg); std::vector parse_log_format(const char *optarg); diff --git a/src/shrpx_config_test.cc b/src/shrpx_config_test.cc index 1ac82589..eda67e85 100644 --- a/src/shrpx_config_test.cc +++ b/src/shrpx_config_test.cc @@ -38,32 +38,32 @@ namespace shrpx { void test_shrpx_config_parse_header(void) { auto p = parse_header("a: b"); - CU_ASSERT("a" == p.first); - CU_ASSERT("b" == p.second); + CU_ASSERT("a" == p.name); + CU_ASSERT("b" == p.value); p = parse_header("a: b"); - CU_ASSERT("a" == p.first); - CU_ASSERT("b" == p.second); + CU_ASSERT("a" == p.name); + CU_ASSERT("b" == p.value); p = parse_header(":a: b"); - CU_ASSERT(p.first.empty()); + CU_ASSERT(p.name.empty()); p = parse_header("a: :b"); - CU_ASSERT("a" == p.first); - CU_ASSERT(":b" == p.second); + CU_ASSERT("a" == p.name); + CU_ASSERT(":b" == p.value); p = parse_header(": b"); - CU_ASSERT(p.first.empty()); + CU_ASSERT(p.name.empty()); p = parse_header("alpha: bravo charlie"); - CU_ASSERT("alpha" == p.first); - CU_ASSERT("bravo charlie" == p.second); + CU_ASSERT("alpha" == p.name); + CU_ASSERT("bravo charlie" == p.value); p = parse_header("a,: b"); - CU_ASSERT(p.first.empty()); + CU_ASSERT(p.name.empty()); p = parse_header("a: b\x0a"); - CU_ASSERT(p.first.empty()); + CU_ASSERT(p.name.empty()); } void test_shrpx_config_parse_log_format(void) { diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index 663599af..02982b8a 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -422,7 +422,7 @@ int Http2DownstreamConnection::push_request_headers() { } for (auto &p : httpconf.add_request_headers) { - nva.push_back(http2::make_nv_nocopy(p.first, p.second)); + nva.push_back(http2::make_nv_nocopy(p.name, p.value)); } if (LOG_ENABLED(INFO)) { diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index a08fc588..434873ac 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1436,7 +1436,7 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { } for (auto &p : httpconf.add_response_headers) { - nva.push_back(http2::make_nv_nocopy(p.first, p.second)); + nva.push_back(http2::make_nv_nocopy(p.name, p.value)); } if (downstream->get_stream_id() % 2 == 0) { diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 40c5753d..8e2796a6 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -418,9 +418,9 @@ int HttpDownstreamConnection::push_request_headers() { } for (auto &p : httpconf.add_request_headers) { - buf->append(p.first); + buf->append(p.name); buf->append(": "); - buf->append(p.second); + buf->append(p.value); buf->append("\r\n"); } diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 1794b94c..cc72e8f5 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -1022,9 +1022,9 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) { } for (auto &p : httpconf.add_response_headers) { - buf->append(p.first); + buf->append(p.name); buf->append(": "); - buf->append(p.second); + buf->append(p.value); buf->append("\r\n"); } diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 2a9ed64d..a44f2a46 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -1064,8 +1064,8 @@ int SpdyUpstream::on_downstream_header_complete(Downstream *downstream) { } for (auto &p : httpconf.add_response_headers) { - nv[hdidx++] = p.first.c_str(); - nv[hdidx++] = p.second.c_str(); + nv[hdidx++] = p.name.c_str(); + nv[hdidx++] = p.value.c_str(); } nv[hdidx++] = 0; From 17758126fa1d0e086427f364041e60155706f776 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 13 Feb 2016 22:31:38 +0900 Subject: [PATCH 28/85] nghttpx: Add headers given in add-response-headers for mruby response --- src/shrpx_http2_upstream.cc | 7 ++++++- src/shrpx_https_upstream.cc | 9 +++++++++ src/shrpx_spdy_upstream.cc | 12 ++++++++++-- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 434873ac..470585af 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1206,11 +1206,12 @@ int Http2Upstream::send_reply(Downstream *downstream, const uint8_t *body, } const auto &resp = downstream->response(); + auto &httpconf = get_config()->http; const auto &headers = resp.fs.headers(); auto nva = std::vector(); // 2 for :status and server - nva.reserve(2 + headers.size()); + nva.reserve(2 + headers.size() + httpconf.add_response_headers.size()); std::string status_code_str; auto response_status_const = http2::stringify_status(resp.http_status); @@ -1242,6 +1243,10 @@ int Http2Upstream::send_reply(Downstream *downstream, const uint8_t *body, http2::make_nv_ls_nocopy("server", get_config()->http.server_name)); } + for (auto &p : httpconf.add_response_headers) { + nva.push_back(http2::make_nv_nocopy(p.name, p.value)); + } + rv = nghttp2_submit_response(session_, downstream->get_stream_id(), nva.data(), nva.size(), data_prd_ptr); if (nghttp2_is_fatal(rv)) { diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index cc72e8f5..c807e753 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -798,6 +798,15 @@ int HttpsUpstream::send_reply(Downstream *downstream, const uint8_t *body, output->append("\r\n"); } + auto &httpconf = get_config()->http; + + for (auto &p : httpconf.add_response_headers) { + output->append(p.name); + output->append(": "); + output->append(p.value); + output->append("\r\n"); + } + output->append("\r\n"); output->append(body, bodylen); diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index a44f2a46..ee80a144 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -850,9 +850,12 @@ int SpdyUpstream::send_reply(Downstream *downstream, const uint8_t *body, const auto &headers = resp.fs.headers(); + auto &httpconf = get_config()->http; + auto nva = std::vector(); - // 3 for :status, :version and server - nva.reserve(3 + headers.size()); + // 6 for :status, :version and server. 1 for last terminal nullptr. + nva.reserve(6 + headers.size() * 2 + + httpconf.add_response_headers.size() * 2 + 1); nva.push_back(":status"); nva.push_back(status_string.c_str()); @@ -879,6 +882,11 @@ int SpdyUpstream::send_reply(Downstream *downstream, const uint8_t *body, nva.push_back(get_config()->http.server_name.c_str()); } + for (auto &p : httpconf.add_response_headers) { + nva.push_back(p.name.c_str()); + nva.push_back(p.value.c_str()); + } + nva.push_back(nullptr); rv = spdylay_submit_response(session_, downstream->get_stream_id(), From 7adfa5dea7e0444635354b35eb47fcc4c58f2da1 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 13 Feb 2016 22:39:24 +0900 Subject: [PATCH 29/85] Add note about --enable-app automatic behaviour --- README.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 9f7f3ce3..290c7dd4 100644 --- a/README.rst +++ b/README.rst @@ -163,9 +163,14 @@ To compile the source code, gcc >= 4.8.3 or clang >= 3.4 is required. .. note:: - To compile the associated applications (nghttp, nghttpd, nghttpx and - h2load), you must use the ``--enable-app`` configure option and ensure - that the specified requirements above are met. + To compile the associated applications (nghttp, nghttpd, nghttpx + and h2load), you must use the ``--enable-app`` configure option and + ensure that the specified requirements above are met. Normally, + configure script checks required dependencies to build these + applications, and enable ``--enable-app`` automatically, so you + don't have to use it explicitly. But if you found that + applications were not built, then using ``--enable-app`` may find + that cause, such as the missing dependency. Notes for building on Windows (Mingw/Cygwin) -------------------------------------------- From eb0c82d91f3962d2a57a8ac5462f618808f520a3 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 13 Feb 2016 23:21:32 +0900 Subject: [PATCH 30/85] nghttpx: More log output when resolving addresses for better debugging --- src/shrpx-unittest.cc | 4 +++ src/shrpx.cc | 39 +++++++++++++++++++++------ src/util.cc | 61 +++++++++++++++++++++++++++++++------------ src/util.h | 8 +++++- src/util_test.cc | 11 ++++++++ src/util_test.h | 2 ++ 6 files changed, 99 insertions(+), 26 deletions(-) diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index 17549331..4c6b39c8 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -167,6 +167,10 @@ int main(int argc, char *argv[]) { !CU_add_test(pSuite, "util_get_uint64", shrpx::test_util_get_uint64) || !CU_add_test(pSuite, "util_parse_config_str_list", shrpx::test_util_parse_config_str_list) || + !CU_add_test(pSuite, "util_make_http_hostport", + shrpx::test_util_make_http_hostport) || + !CU_add_test(pSuite, "util_make_hostport", + shrpx::test_util_make_hostport) || !CU_add_test(pSuite, "gzip_inflate", test_nghttp2_gzip_inflate) || !CU_add_test(pSuite, "buffer_write", nghttp2::test_buffer_write) || !CU_add_test(pSuite, "pool_recycle", nghttp2::test_pool_recycle) || diff --git a/src/shrpx.cc b/src/shrpx.cc index 9fb64589..17a6bb5a 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -656,7 +656,7 @@ int create_tcp_server_socket(UpstreamAddr &faddr, } faddr.fd = fd; - faddr.hostport = util::make_hostport(host.data(), faddr.port); + faddr.hostport = util::make_http_hostport(host.data(), faddr.port); LOG(NOTICE) << "Listening on " << faddr.hostport; @@ -2178,8 +2178,10 @@ void process_options( exit(EXIT_FAILURE); } - LOG(INFO) << "Use UNIX domain socket path " << path - << " for backend connection"; + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Use UNIX domain socket path " << path + << " for backend connection"; + } addr.addr.su.un.sun_family = AF_UNIX; // copy path including terminal NULL @@ -2189,44 +2191,65 @@ void process_options( continue; } - addr.hostport = - ImmutableString(util::make_hostport(addr.host.c_str(), addr.port)); + addr.hostport = ImmutableString( + util::make_http_hostport(StringRef(addr.host), addr.port)); + + auto hostport = util::make_hostport(addr.host.c_str(), addr.port); if (resolve_hostname(&addr.addr, addr.host.c_str(), addr.port, downstreamconf.family) == -1) { + LOG(FATAL) << "Resolving backend address failed: " << hostport; exit(EXIT_FAILURE); } + LOG(NOTICE) << "Resolved backend address: " << hostport << " -> " + << util::numeric_hostport(&addr.addr.su.sa, addr.addr.len); } } auto &proxy = mod_config()->downstream_http_proxy; if (!proxy.host.empty()) { - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Resolving backend http proxy address"; - } + auto hostport = util::make_hostport(proxy.host.c_str(), proxy.port); if (resolve_hostname(&proxy.addr, proxy.host.c_str(), proxy.port, AF_UNSPEC) == -1) { + LOG(FATAL) << "Resolving backend HTTP proxy address failed: " << hostport; exit(EXIT_FAILURE); } + LOG(NOTICE) << "Backend HTTP proxy address: " << hostport << " -> " + << util::numeric_hostport(&proxy.addr.su.sa, proxy.addr.len); } { auto &memcachedconf = tlsconf.session_cache.memcached; if (memcachedconf.host) { + auto hostport = + util::make_hostport(memcachedconf.host.get(), memcachedconf.port); if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.get(), memcachedconf.port, memcachedconf.family) == -1) { + LOG(FATAL) + << "Resolving memcached address for TLS session cache failed: " + << hostport; exit(EXIT_FAILURE); } + LOG(NOTICE) << "Memcached address for TLS session cache: " << hostport + << " -> " << util::numeric_hostport(&memcachedconf.addr.su.sa, + memcachedconf.addr.len); } } { auto &memcachedconf = tlsconf.ticket.memcached; if (memcachedconf.host) { + auto hostport = + util::make_hostport(memcachedconf.host.get(), memcachedconf.port); if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.get(), memcachedconf.port, memcachedconf.family) == -1) { + LOG(FATAL) << "Resolving memcached address for TLS ticket key failed: " + << hostport; exit(EXIT_FAILURE); } + LOG(NOTICE) << "Memcached address for TLS ticket key: " << hostport + << " -> " << util::numeric_hostport(&memcachedconf.addr.su.sa, + memcachedconf.addr.len); } } diff --git a/src/util.cc b/src/util.cc index 2f120887..0897a0f7 100644 --- a/src/util.cc +++ b/src/util.cc @@ -57,7 +57,6 @@ #include #include "timegm.h" -#include "template.h" namespace nghttp2 { @@ -1149,25 +1148,53 @@ std::string dtos(double n) { return utos(static_cast(n)) + "." + (f.size() == 1 ? "0" : "") + f; } -std::string make_hostport(const char *host, uint16_t port) { - auto ipv6 = ipv6_numeric_addr(host); - std::string hostport; - - if (ipv6) { - hostport += '['; - } - - hostport += host; - - if (ipv6) { - hostport += ']'; - } - +std::string make_http_hostport(const StringRef &host, uint16_t port) { if (port != 80 && port != 443) { - hostport += ':'; - hostport += utos(port); + return make_hostport(host, port); } + auto ipv6 = ipv6_numeric_addr(host.c_str()); + + std::string hostport; + hostport.resize(host.size() + (ipv6 ? 2 : 0)); + + auto p = &hostport[0]; + + if (ipv6) { + *p++ = '['; + } + + p = std::copy_n(host.c_str(), host.size(), p); + + if (ipv6) { + *p++ = ']'; + } + + return hostport; +} + +std::string make_hostport(const StringRef &host, uint16_t port) { + auto ipv6 = ipv6_numeric_addr(host.c_str()); + auto serv = utos(port); + + std::string hostport; + hostport.resize(host.size() + (ipv6 ? 2 : 0) + 1 + serv.size()); + + auto p = &hostport[0]; + + if (ipv6) { + *p++ = '['; + } + + p = std::copy_n(host.c_str(), host.size(), p); + + if (ipv6) { + *p++ = ']'; + } + + *p++ = ':'; + std::copy_n(serv.c_str(), serv.size(), p); + return hostport; } diff --git a/src/util.h b/src/util.h index ad3a60dc..b1f64d77 100644 --- a/src/util.h +++ b/src/util.h @@ -49,6 +49,8 @@ #include "http-parser/http_parser.h" +#include "template.h" + namespace nghttp2 { // The additional HTTP/2 protocol ALPN protocol identifier we also @@ -623,7 +625,11 @@ std::string format_duration(double t); // Creates "host:port" string using given |host| and |port|. If // |host| is numeric IPv6 address (e.g., ::1), it is enclosed by "[" // and "]". If |port| is 80 or 443, port part is omitted. -std::string make_hostport(const char *host, uint16_t port); +std::string make_http_hostport(const StringRef &host, uint16_t port); + +// Just like make_http_hostport(), but doesn't treat 80 and 443 +// specially. +std::string make_hostport(const StringRef &host, uint16_t port); // Dumps |src| of length |len| in the format similar to `hexdump -C`. void hexdump(FILE *out, const uint8_t *src, size_t len); diff --git a/src/util_test.cc b/src/util_test.cc index 440674b0..6f605c5b 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -455,4 +455,15 @@ void test_util_parse_config_str_list(void) { CU_ASSERT("charlie" == res[2]); } +void test_util_make_http_hostport(void) { + CU_ASSERT("localhost" == util::make_http_hostport("localhost", 80)); + CU_ASSERT("[::1]" == util::make_http_hostport("::1", 443)); + CU_ASSERT("localhost:3000" == util::make_http_hostport("localhost", 3000)); +} + +void test_util_make_hostport(void) { + CU_ASSERT("localhost:80" == util::make_hostport("localhost", 80)); + CU_ASSERT("[::1]:443" == util::make_hostport("::1", 443)); +} + } // namespace shrpx diff --git a/src/util_test.h b/src/util_test.h index 45d654a1..1b79bbae 100644 --- a/src/util_test.h +++ b/src/util_test.h @@ -57,6 +57,8 @@ void test_util_parse_http_date(void); void test_util_localtime_date(void); void test_util_get_uint64(void); void test_util_parse_config_str_list(void); +void test_util_make_http_hostport(void); +void test_util_make_hostport(void); } // namespace shrpx From 6b12f17f44c6c879cf7bb18040f91de885ffcd2f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 00:05:12 +0900 Subject: [PATCH 31/85] Wrap AM_PATH_XML2 by m4_ifdef to handle the case when AM_PATH_XML2 is not found --- configure.ac | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index d7f6505f..985925d7 100644 --- a/configure.ac +++ b/configure.ac @@ -352,7 +352,10 @@ fi # libxml2 (for src/nghttp) have_libxml2=no if test "x${request_libxml2}" != "xno"; then - AM_PATH_XML2(2.7.7, [have_libxml2=yes], [have_libxml2=no]) + m4_ifdef([AM_PATH_XML2], + [AM_PATH_XML2(2.7.7, [have_libxml2=yes], [have_libxml2=no])], + [AC_MSG_WARN([configure was created without libxml2 detection macro; libxml2 detection is disabled])]) + if test "x${have_libxml2}" = "xyes"; then AC_DEFINE([HAVE_LIBXML2], [1], [Define to 1 if you have `libxml2` library.]) fi From d8c8a4631d045e6a8c3b8e33f840fdf105c2696b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 00:28:08 +0900 Subject: [PATCH 32/85] nghttpx: Interleave text/html pushed resources with associated resource --- src/shrpx_http2_upstream.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 470585af..7e909a25 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1558,7 +1558,9 @@ int Http2Upstream::adjust_pushed_stream_priority(Downstream *downstream) { } if (!util::istarts_with_l(ct->value, "application/javascript") && - !util::istarts_with_l(ct->value, "text/css")) { + !util::istarts_with_l(ct->value, "text/css") && + // for polymer... + !util::istarts_with_l(ct->value, "text/html")) { return 0; } From 5f1866fd6be9b32e58c8c6d59e51ac83a06819fc Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 00:29:07 +0900 Subject: [PATCH 33/85] Update man pages --- doc/h2load.1 | 2 +- doc/nghttp.1 | 2 +- doc/nghttpd.1 | 2 +- doc/nghttpx.1 | 102 ++++++++++++++++++++++++++++++++++++++-------- doc/nghttpx.1.rst | 89 +++++++++++++++++++++++++++++++++------- 5 files changed, 161 insertions(+), 36 deletions(-) diff --git a/doc/h2load.1 b/doc/h2load.1 index f9dab89e..a6db9aea 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "February 07, 2016" "1.8.0-DEV" "nghttp2" +.TH "H2LOAD" "1" "February 14, 2016" "1.8.0-DEV" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index ad205088..24beec68 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "February 07, 2016" "1.8.0-DEV" "nghttp2" +.TH "NGHTTP" "1" "February 14, 2016" "1.8.0-DEV" "nghttp2" .SH NAME nghttp \- HTTP/2 client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index e93ccc8a..a3b7bc9f 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "February 07, 2016" "1.8.0-DEV" "nghttp2" +.TH "NGHTTPD" "1" "February 14, 2016" "1.8.0-DEV" "nghttp2" .SH NAME nghttpd \- HTTP/2 server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index 90ee5a1f..a89f545f 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "February 07, 2016" "1.8.0-DEV" "nghttp2" +.TH "NGHTTPX" "1" "February 14, 2016" "1.8.0-DEV" "nghttp2" .SH NAME nghttpx \- HTTP/2 proxy . @@ -136,13 +136,13 @@ Default: \fB512\fP .UNINDENT .INDENT 0.0 .TP -.B \-\-backend\-ipv4 -Resolve backend hostname to IPv4 address only. -.UNINDENT -.INDENT 0.0 -.TP -.B \-\-backend\-ipv6 -Resolve backend hostname to IPv6 address only. +.B \-\-backend\-address\-family=(auto|IPv4|IPv6) +Specify address family of backend connections. If +"auto" is given, both IPv4 and IPv6 are considered. If +"IPv4" is given, only IPv4 address is considered. If +"IPv6" is given, only IPv6 address is considered. +.sp +Default: \fBauto\fP .UNINDENT .INDENT 0.0 .TP @@ -534,16 +534,27 @@ required. .INDENT 0.0 .TP .B \-\-tls\-ticket\-key\-memcached=, -Specify address of memcached server to store session -cache. This enables shared TLS ticket key between -multiple nghttpx instances. nghttpx does not set TLS -ticket key to memcached. The external ticket key -generator is required. nghttpx just gets TLS ticket -keys from memcached, and use them, possibly replacing -current set of keys. It is up to extern TLS ticket key -generator to rotate keys frequently. See "TLS SESSION -TICKET RESUMPTION" section in manual page to know the -data format in memcached entry. +Specify address of memcached server to get TLS ticket +keys for session resumption. This enables shared TLS +ticket key between multiple nghttpx instances. nghttpx +does not set TLS ticket key to memcached. The external +ticket key generator is required. nghttpx just gets TLS +ticket keys from memcached, and use them, possibly +replacing current set of keys. It is up to extern TLS +ticket key generator to rotate keys frequently. See +"TLS SESSION TICKET RESUMPTION" section in manual page +to know the data format in memcached entry. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-tls\-ticket\-key\-memcached\-address\-family=(auto|IPv4|IPv6) +Specify address family of memcached connections to get +TLS ticket keys. If "auto" is given, both IPv4 and IPv6 +are considered. If "IPv4" is given, only IPv4 address +is considered. If "IPv6" is given, only IPv6 address is +considered. +.sp +Default: \fBauto\fP .UNINDENT .INDENT 0.0 .TP @@ -581,6 +592,24 @@ aes\-128\-cbc is used. .UNINDENT .INDENT 0.0 .TP +.B \-\-tls\-ticket\-key\-memcached\-tls +Enable SSL/TLS on memcached connections to get TLS +ticket keys. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-tls\-ticket\-key\-memcached\-cert\-file= +Path to client certificate for memcached connections to +get TLS ticket keys. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-tls\-ticket\-key\-memcached\-private\-key\-file= +Path to client private key for memcached connections to +get TLS ticket keys. +.UNINDENT +.INDENT 0.0 +.TP .B \-\-fetch\-ocsp\-response\-file= Path to fetch\-ocsp\-response script file. It should be absolute path. @@ -608,6 +637,35 @@ multiple nghttpx instances. .UNINDENT .INDENT 0.0 .TP +.B \-\-tls\-session\-cache\-memcached\-address\-family=(auto|IPv4|IPv6) +Specify address family of memcached connections to store +session cache. If "auto" is given, both IPv4 and IPv6 +are considered. If "IPv4" is given, only IPv4 address +is considered. If "IPv6" is given, only IPv6 address is +considered. +.sp +Default: \fBauto\fP +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-tls\-session\-cache\-memcached\-tls +Enable SSL/TLS on memcached connections to store session +cache. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-tls\-session\-cache\-memcached\-cert\-file= +Path to client certificate for memcached connections to +store session cache. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-tls\-session\-cache\-memcached\-private\-key\-file= +Path to client private key for memcached connections to +store session cache. +.UNINDENT +.INDENT 0.0 +.TP .B \-\-tls\-dyn\-rec\-warmup\-threshold= Specify the threshold size for TLS dynamic record size behaviour. During a TLS session, after the threshold @@ -1251,6 +1309,10 @@ insert serialized session data to memcached with \fBnghttpx:tls\-session\-cache:\fP + lowercased hex string of session ID as a memcached entry key, with expiry time 12 hours. Session timeout is set to 12 hours. +.sp +By default, connections to memcached server are not encrypted. To +enable encryption, use \fI\%\-\-tls\-session\-cache\-memcached\-tls\fP +option. .SS TLS SESSION TICKET RESUMPTION .sp By default, session ticket is shared by all worker threads. The @@ -1295,6 +1357,10 @@ used, LEN must be 48. If keys. The key appeared first is used as encryption key. All the remaining keys are used as decryption only. .sp +By default, connections to memcached server are not encrypted. To +enable encryption, use \fI\%\-\-tls\-ticket\-key\-memcached\-tls\fP +option. +.sp If \fI\%\-\-tls\-ticket\-key\-file\fP is given, encryption key is read from the given file. In this case, nghttpx does not rotate key automatically. To rotate key, one has to restart nghttpx (see diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index ffafa2bf..42aa4642 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -116,13 +116,14 @@ Connections Default: ``512`` -.. option:: --backend-ipv4 +.. option:: --backend-address-family=(auto|IPv4|IPv6) - Resolve backend hostname to IPv4 address only. + Specify address family of backend connections. If + "auto" is given, both IPv4 and IPv6 are considered. If + "IPv4" is given, only IPv4 address is considered. If + "IPv6" is given, only IPv6 address is considered. -.. option:: --backend-ipv6 - - Resolve backend hostname to IPv6 address only. + Default: ``auto`` .. option:: --backend-http-proxy-uri= @@ -477,16 +478,26 @@ SSL/TLS .. option:: --tls-ticket-key-memcached=, - Specify address of memcached server to store session - cache. This enables shared TLS ticket key between - multiple nghttpx instances. nghttpx does not set TLS - ticket key to memcached. The external ticket key - generator is required. nghttpx just gets TLS ticket - keys from memcached, and use them, possibly replacing - current set of keys. It is up to extern TLS ticket key - generator to rotate keys frequently. See "TLS SESSION - TICKET RESUMPTION" section in manual page to know the - data format in memcached entry. + Specify address of memcached server to get TLS ticket + keys for session resumption. This enables shared TLS + ticket key between multiple nghttpx instances. nghttpx + does not set TLS ticket key to memcached. The external + ticket key generator is required. nghttpx just gets TLS + ticket keys from memcached, and use them, possibly + replacing current set of keys. It is up to extern TLS + ticket key generator to rotate keys frequently. See + "TLS SESSION TICKET RESUMPTION" section in manual page + to know the data format in memcached entry. + +.. option:: --tls-ticket-key-memcached-address-family=(auto|IPv4|IPv6) + + Specify address family of memcached connections to get + TLS ticket keys. If "auto" is given, both IPv4 and IPv6 + are considered. If "IPv4" is given, only IPv4 address + is considered. If "IPv6" is given, only IPv6 address is + considered. + + Default: ``auto`` .. option:: --tls-ticket-key-memcached-interval= @@ -518,6 +529,21 @@ SSL/TLS either aes-128-cbc or aes-256-cbc. By default, aes-128-cbc is used. +.. option:: --tls-ticket-key-memcached-tls + + Enable SSL/TLS on memcached connections to get TLS + ticket keys. + +.. option:: --tls-ticket-key-memcached-cert-file= + + Path to client certificate for memcached connections to + get TLS ticket keys. + +.. option:: --tls-ticket-key-memcached-private-key-file= + + Path to client private key for memcached connections to + get TLS ticket keys. + .. option:: --fetch-ocsp-response-file= Path to fetch-ocsp-response script file. It should be @@ -541,6 +567,31 @@ SSL/TLS cache. This enables shared session cache between multiple nghttpx instances. +.. option:: --tls-session-cache-memcached-address-family=(auto|IPv4|IPv6) + + Specify address family of memcached connections to store + session cache. If "auto" is given, both IPv4 and IPv6 + are considered. If "IPv4" is given, only IPv4 address + is considered. If "IPv6" is given, only IPv6 address is + considered. + + Default: ``auto`` + +.. option:: --tls-session-cache-memcached-tls + + Enable SSL/TLS on memcached connections to store session + cache. + +.. option:: --tls-session-cache-memcached-cert-file= + + Path to client certificate for memcached connections to + store session cache. + +.. option:: --tls-session-cache-memcached-private-key-file= + + Path to client private key for memcached connections to + store session cache. + .. option:: --tls-dyn-rec-warmup-threshold= Specify the threshold size for TLS dynamic record size @@ -1134,6 +1185,10 @@ insert serialized session data to memcached with as a memcached entry key, with expiry time 12 hours. Session timeout is set to 12 hours. +By default, connections to memcached server are not encrypted. To +enable encryption, use :option:`--tls-session-cache-memcached-tls` +option. + TLS SESSION TICKET RESUMPTION ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1173,6 +1228,10 @@ used, LEN must be 48. If keys. The key appeared first is used as encryption key. All the remaining keys are used as decryption only. +By default, connections to memcached server are not encrypted. To +enable encryption, use :option:`--tls-ticket-key-memcached-tls` +option. + If :option:`--tls-ticket-key-file` is given, encryption key is read from the given file. In this case, nghttpx does not rotate key automatically. To rotate key, one has to restart nghttpx (see From 093eb51f8c71e077583d20ab6dcdcfc539ec7b82 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 00:44:50 +0900 Subject: [PATCH 34/85] Update default cipher list --- src/ssl.cc | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/ssl.cc b/src/ssl.cc index 389a25de..f144fa94 100644 --- a/src/ssl.cc +++ b/src/ssl.cc @@ -36,19 +36,21 @@ namespace nghttp2 { namespace ssl { -// Recommended general purpose "Non-Backward Compatible" cipher by -// mozilla. +// Recommended general purpose "Intermediate compatibility" cipher +// suites by mozilla. // // https://wiki.mozilla.org/Security/Server_Side_TLS const char *const DEFAULT_CIPHER_LIST = - "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-" - "AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:" - "DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-" - "AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-" - "AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-" - "AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:" - "DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:!aNULL:!eNULL:" - "!EXPORT:!DES:!RC4:!3DES:!MD5:!PSK"; + "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-" + "AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-" + "SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-" + "AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-" + "ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-" + "AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-" + "SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-" + "ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-" + "SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-" + "SHA:DES-CBC3-SHA:!DSS"; namespace { std::vector ssl_global_locks; From 0e469ed221deff669155011acd5d8e954715b1d2 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Thu, 11 Feb 2016 16:41:11 +0100 Subject: [PATCH 35/85] Fix typo in HAVE_CONFIG_H name Only used by lib/nghttp2_npn.c where the presence of config.h does not seem to make a difference though. --- lib/nghttp2_npn.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/nghttp2_npn.h b/lib/nghttp2_npn.h index c4bdedb3..a481bde3 100644 --- a/lib/nghttp2_npn.h +++ b/lib/nghttp2_npn.h @@ -25,9 +25,9 @@ #ifndef NGHTTP2_NPN_H #define NGHTTP2_NPN_H -#ifdef HAVE_CONFIG +#ifdef HAVE_CONFIG_H #include -#endif /* HAVE_CONFIG */ +#endif /* HAVE_CONFIG_H */ #include From 17215002a16dafe48c4f8536c046839d56eabc86 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Fri, 12 Feb 2016 14:23:16 +0100 Subject: [PATCH 36/85] examples: fix compile warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes the following two warnings: examples/client.c:292:0: error: macro "MAX_OUTLEN" is not used [-Werror=unused-macros] examples/tiny-nghttpd.c:298:13: error: function declaration isn’t a prototype [-Werror=strict-prototypes] Caught using cmake as the autoconf check fails due to unused macros (HAVE_xxx in conftest.c) and a main function without parameters respectively. --- examples/client.c | 2 -- examples/tiny-nghttpd.c | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/client.c b/examples/client.c index 4280dd52..1976fa41 100644 --- a/examples/client.c +++ b/examples/client.c @@ -289,8 +289,6 @@ static int on_stream_close_callback(nghttp2_session *session, int32_t stream_id, return 0; } -#define MAX_OUTLEN 4096 - /* * The implementation of nghttp2_on_data_chunk_recv_callback type. We * use this function to print the received response body. diff --git a/examples/tiny-nghttpd.c b/examples/tiny-nghttpd.c index 6cb95694..0141663d 100644 --- a/examples/tiny-nghttpd.c +++ b/examples/tiny-nghttpd.c @@ -295,7 +295,7 @@ static size_t http_date(char *buf, time_t t) { static char date[29]; static size_t datelen; -static void update_date() { datelen = http_date(date, time(NULL)); } +static void update_date(void) { datelen = http_date(date, time(NULL)); } static size_t utos(char *buf, size_t len, uint64_t n) { size_t nwrite = 0; From a6effb4d23137be7204bbaeff159344f51a04649 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Sat, 13 Feb 2016 14:36:12 +0100 Subject: [PATCH 37/85] doc: fix out-of-tree doc builds Fixes multiple errors while making docs: Could not import extension sphinxcontrib.rubydomain (exception: No module named 'sphinxcontrib') and ../../doc/sources/index.rst:15: WARNING: toctree contains reference to nonexisting document 'nghttp.1' ../../doc/sources/index.rst:15: WARNING: toctree contains reference to nonexisting document 'nghttpd.1' ../../doc/sources/index.rst:15: WARNING: toctree contains reference to nonexisting document 'nghttpx.1' ../../doc/sources/index.rst:15: WARNING: toctree contains reference to nonexisting document 'h2load.1' ../../doc/sources/index.rst:15: WARNING: toctree contains reference to nonexisting document 'programmers-guide' --- doc/Makefile.am | 14 +++++++++----- doc/conf.py.in | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/doc/Makefile.am b/doc/Makefile.am index 0e6a9abc..a9ea4349 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -145,15 +145,18 @@ APIDOCS= \ nghttp2_submit_window_update.rst \ nghttp2_version.rst -EXTRA_DIST = \ - mkapiref.py \ +RST_FILES = \ README.rst \ programmers-guide.rst \ - $(APIDOCS) \ nghttp.1.rst \ nghttpd.1.rst \ nghttpx.1.rst \ - h2load.1.rst \ + h2load.1.rst + +EXTRA_DIST = \ + mkapiref.py \ + $(RST_FILES) \ + $(APIDOCS) \ sources/index.rst \ sources/tutorial-client.rst \ sources/tutorial-server.rst \ @@ -227,7 +230,8 @@ help: apiref.rst: \ $(top_builddir)/lib/includes/nghttp2/nghttp2ver.h \ - $(top_builddir)/lib/includes/nghttp2/nghttp2.h + $(top_srcdir)/lib/includes/nghttp2/nghttp2.h + for i in $(RST_FILES); do [ -e $(builddir)/$$i ] || cp $(srcdir)/$$i $(builddir); done $(PYTHON) $(top_srcdir)/doc/mkapiref.py \ apiref.rst macros.rst enums.rst types.rst . $^ diff --git a/doc/conf.py.in b/doc/conf.py.in index 9c192827..0cc5a5a9 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -41,7 +41,7 @@ import sys, os # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) -sys.path.append(os.path.abspath('_exts')) +sys.path.append(os.path.abspath('@top_srcdir@/doc/_exts')) # -- General configuration ----------------------------------------------------- From 25930360537a201674fa6aa5b3ff93a948f11237 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Sat, 13 Feb 2016 20:03:00 +0100 Subject: [PATCH 38/85] integration-tests: support out-of-tree tests `go test` requires both config.go and the test files in the same directory. For out-of-tree builds, config.go is normally not placed next to the source files, so copy the tests to the build directory as a workaround. --- integration-tests/Makefile.am | 8 ++++++-- integration-tests/config.go.in | 1 + integration-tests/server_tester.go | 5 +++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/integration-tests/Makefile.am b/integration-tests/Makefile.am index 8f2cfabd..aab50096 100644 --- a/integration-tests/Makefile.am +++ b/integration-tests/Makefile.am @@ -21,11 +21,14 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -EXTRA_DIST = \ +GO_FILES = \ nghttpx_http1_test.go \ nghttpx_http2_test.go \ nghttpx_spdy_test.go \ - server_tester.go \ + server_tester.go + +EXTRA_DIST = \ + $(GO_FILES) \ server.key \ server.crt \ alt-server.key \ @@ -43,4 +46,5 @@ itprep-local: go get -d -v golang.org/x/net/websocket it-local: + for i in $(GO_FILES); do [ -e $(builddir)/$$i ] || cp $(srcdir)/$$i $(builddir); done sh setenv go test -v diff --git a/integration-tests/config.go.in b/integration-tests/config.go.in index 0a6fd6b7..3cc4766c 100644 --- a/integration-tests/config.go.in +++ b/integration-tests/config.go.in @@ -2,4 +2,5 @@ package nghttp2 const ( buildDir = "@top_builddir@" + sourceDir = "@top_srcdir@" ) diff --git a/integration-tests/server_tester.go b/integration-tests/server_tester.go index 0a544eb8..8226c899 100644 --- a/integration-tests/server_tester.go +++ b/integration-tests/server_tester.go @@ -29,7 +29,8 @@ import ( const ( serverBin = buildDir + "/src/nghttpx" serverPort = 3009 - testDir = buildDir + "/integration-tests" + testDir = sourceDir + "/integration-tests" + logDir = buildDir + "/integration-tests" ) func pair(name, value string) hpack.HeaderField { @@ -124,7 +125,7 @@ func newServerTesterInternal(args []string, t *testing.T, handler http.Handler, // "127.0.0.1,8080" b := "-b" + strings.Replace(backendURL.Host, ":", ",", -1) args = append(args, fmt.Sprintf("-f127.0.0.1,%v", serverPort), b, - "--errorlog-file="+testDir+"/log.txt", "-LINFO") + "--errorlog-file="+logDir+"/log.txt", "-LINFO") authority := fmt.Sprintf("127.0.0.1:%v", serverPort) From fe74600a5f4dfcce2f6979d2cd62f388f97426fa Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 17:40:58 +0900 Subject: [PATCH 39/85] List all contributors in AUTHORS --- AUTHORS | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- COPYING | 1 + 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 95bc9542..8c4a70e8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1 +1,80 @@ -Tatsuhiro Tsujikawa +nghttp2 project was started as a fork of spdylay project [1]. Both +projects were started by Tatsuhiro Tsujikawa, who is still the main +author of these projects. Meanwhile, we have many contributions, and +we are not here without them. We sincerely thank you to all who made +a contribution. Here is the all individuals/organizations who +contributed to nghttp2 and spdylay project at which we forked. These +names are retrieved from git commit log. If you have made a +contribution, but you are missing in the list, please let us know via +github issues [2]. + +[1] https://github.com/tatsuhiro-t/spdylay +[2] https://github.com/tatsuhiro-t/nghttp2/issues + +-------- + +187j3x1 +Alek Storm +Alex Nalivko +Alexis La Goutte +Anders Bakken +Andreas Pohl +Andy Davies +Ant Bryan +Bernard Spil +Brian Card +Daniel Stenberg +Dave Reisner +David Beitey +David Weekly +Etienne Cimon +Fabian Möller +Fabian Wiesel +Gabi Davar +Janusz Dziemidowicz +Jay Satiro +Jim Morrison +José F. Calcerrada +Kamil Dudka +Kazuho Oku +Kenny (kang-yen) Peng +Kenny Peng +Kit Chan +Kyle Schomp +Lucas Pardue +MATSUMOTO Ryosuke +Mike Frysinger +Nicholas Hurley +Nora Shoemaker +Peeyush Aggarwal +Peter Wu +Piotr Sikora +Raul Gutierrez Segales +Remo E +Reza Tavakoli +Ross Smith II +Scott Mitchell +Stefan Eissing +Stephen Ludin +Sunpoet Po-Chuan Hsieh +Svante Signell +Syohei YOSHIDA +Tatsuhiko Kubo +Tatsuhiro Tsujikawa +Tom Harwood +Tomasz Buchert +Vernon Tang +Viacheslav Biriukov +Viktor Szépe +Xiaoguang Sun +Zhuoyun Wei +acesso +ayanamist +bxshi +es +fangdingjun +kumagi +mod-h2-dev +moparisthebest +snnn +yuuki-kodama diff --git a/COPYING b/COPYING index 6e079b80..80201792 100644 --- a/COPYING +++ b/COPYING @@ -1,6 +1,7 @@ The MIT License Copyright (c) 2012, 2014, 2015, 2016 Tatsuhiro Tsujikawa +Copyright (c) 2012, 2014, 2015, 2016 nghttp2 contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the From eebed206c91690032bfc7102144aadc295a59f19 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 18:23:28 +0900 Subject: [PATCH 40/85] Add Architecture doc --- doc/programmers-guide.rst | 56 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/doc/programmers-guide.rst b/doc/programmers-guide.rst index 94c72e98..253c9fd3 100644 --- a/doc/programmers-guide.rst +++ b/doc/programmers-guide.rst @@ -1,6 +1,62 @@ Programmers' Guide ================== +Architecture +------------ + +The most notable point in nghttp2 library architecture is it does not +perform any I/O. nghttp2 only performs HTTP/2 protocol stuff based on +input byte strings. It will calls callback functions set by +applications while processing input. The output of nghttp2 is just +byte string. An application is responsible to send these output to +the remote peer. The callback functions may be called while producing +output. + +Not doing I/O makes embedding nghttp2 library in the existing code +base very easy. Usually, the existing applications have its own I/O +event loops. It is very hard to use nghttp2 in that situation if +nghttp2 does its own I/O. It also makes light weight language wrapper +for nghttp2 easy with the same reason. The down side is that an +application author has to write more code to write complete +application using nghttp2. This is especially true for simple "toy" +application. For the real applications, however, this is not the +case. This is because you probably want to support HTTP/1 which +nghttp2 does not provide, and to do that, you will need to write your +own HTTP/1 stack or use existing third-party library, and bind them +together with nghttp2 and I/O event loop. In this point, not +performing I/O in nghttp2 has more point than doing it. + +The primary object that an application uses is :type:`nghttp2_session` +object, which is opaque struct and its details are hidden in order to +ensure the upgrading its internal architecture without breaking the +backward compatibility. An application can set callbacks to +:type:`nghttp2_session` object through the dedicated object and +functions, and it also interacts with it via many API function calls. + +An application can create as many :type:`nghttp2_session` object as it +wants. But single :type:`nghttp2_session` object must be used by a +single thread at the same time. This is not so hard to enforce since +most event-based architecture applicatons use is single thread per +core, and handling one connection I/O is done by single thread. + +To feed input to :type:`nghttp2_session` object, one can use +`nghttp2_session_recv()` or `nghttp2_session_mem_recv()` functions. +They behave similarly, and the difference is that +`nghttp2_session_recv()` will use :type:`nghttp2_read_callback` to get +input. On the other hand, `nghttp2_session_mem_recv()` will take +input as its parameter. If in doubt, use `nghttp2_session_mem_recv()` +since it is simpler, and could be faster since it avoids calling +callback function. + +To get output from :type:`nghttp2_session` object, one can use +`nghttp2_session_send()` or `nghttp2_session_mem_send()`. The +difference between them is that the former uses +:type:`nghttp2_send_callback` to pass output to an application. On +the other hand, the latter returns the output to the caller. If in +doubt, use `nghttp2_session_mem_send()` since it is simpler. But +`nghttp2_session_send()` might be easier to use if the output buffer +an application has is fixed sized. + Includes -------- From 1bd98dcf4febfc43544c4c193d2f114a266f737f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 18:31:08 +0900 Subject: [PATCH 41/85] nghttpx: Remove user defined ctor/assignment op from DownstreamAddr --- src/shrpx.cc | 2 +- src/shrpx_config.cc | 23 +---------------------- src/shrpx_config.h | 6 ------ 3 files changed, 2 insertions(+), 29 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 17a6bb5a..87bd9c28 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2105,7 +2105,7 @@ void process_options( auto &addr_groups = downstreamconf.addr_groups; if (addr_groups.empty()) { - DownstreamAddr addr; + DownstreamAddr addr{}; addr.host = ImmutableString::from_lit(DEFAULT_DOWNSTREAM_HOST); addr.port = DEFAULT_DOWNSTREAM_PORT; diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 104c97b6..489885cf 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -78,27 +78,6 @@ TicketKeys::~TicketKeys() { } } -DownstreamAddr::DownstreamAddr(const DownstreamAddr &other) - : addr(other.addr), - host(other.host), - hostport(other.hostport), - port(other.port), - host_unix(other.host_unix) {} - -DownstreamAddr &DownstreamAddr::operator=(const DownstreamAddr &other) { - if (this == &other) { - return *this; - } - - addr = other.addr; - host = other.host; - hostport = other.hostport; - port = other.port; - host_unix = other.host_unix; - - return *this; -} - DownstreamAddrGroup::DownstreamAddrGroup(const DownstreamAddrGroup &other) : pattern(strcopy(other.pattern)), addrs(other.addrs) {} @@ -1523,7 +1502,7 @@ int parse_config(const char *opt, const char *optarg, if (!pat_delim) { pat_delim = optarg + optarglen; } - DownstreamAddr addr; + DownstreamAddr addr{}; if (util::istarts_with(optarg, SHRPX_UNIX_PATH_PREFIX)) { auto path = optarg + str_size(SHRPX_UNIX_PATH_PREFIX); addr.host = ImmutableString(path, pat_delim); diff --git a/src/shrpx_config.h b/src/shrpx_config.h index ebc97f8a..c460cbc8 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -289,12 +289,6 @@ struct UpstreamAddr { }; struct DownstreamAddr { - DownstreamAddr() : addr{}, port(0), host_unix(false) {} - DownstreamAddr(const DownstreamAddr &other); - DownstreamAddr(DownstreamAddr &&) = default; - DownstreamAddr &operator=(const DownstreamAddr &other); - DownstreamAddr &operator=(DownstreamAddr &&other) = default; - Address addr; // backend address. If |host_unix| is true, this is UNIX domain // socket path. From a53f0f0a17ecf86dbb09f6e7a0d397f5cfda2431 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 18:47:24 +0900 Subject: [PATCH 42/85] nghttpx: Refactor DownstreamAddrGroup and router API --- src/shrpx.cc | 12 ++++++------ src/shrpx_config.cc | 27 ++++++--------------------- src/shrpx_config.h | 9 +++------ src/shrpx_config_test.cc | 2 +- 4 files changed, 16 insertions(+), 34 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 87bd9c28..24290891 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2109,14 +2109,14 @@ void process_options( addr.host = ImmutableString::from_lit(DEFAULT_DOWNSTREAM_HOST); addr.port = DEFAULT_DOWNSTREAM_PORT; - DownstreamAddrGroup g("/"); + DownstreamAddrGroup g(StringRef::from_lit("/")); g.addrs.push_back(std::move(addr)); - mod_config()->router.add_route(g.pattern.get(), 1, addr_groups.size()); + mod_config()->router.add_route(g.pattern.c_str(), 1, addr_groups.size()); addr_groups.push_back(std::move(g)); } else if (get_config()->http2_proxy || get_config()->client_proxy) { // We don't support host mapping in these cases. Move all // non-catch-all patterns to catch-all pattern. - DownstreamAddrGroup catch_all("/"); + DownstreamAddrGroup catch_all(StringRef::from_lit("/")); for (auto &g : addr_groups) { std::move(std::begin(g.addrs), std::end(g.addrs), std::back_inserter(catch_all.addrs)); @@ -2124,7 +2124,7 @@ void process_options( std::vector().swap(addr_groups); // maybe not necessary? mod_config()->router = Router(); - mod_config()->router.add_route(catch_all.pattern.get(), 1, + mod_config()->router.add_route(catch_all.pattern.c_str(), 1, addr_groups.size()); addr_groups.push_back(std::move(catch_all)); } @@ -2136,11 +2136,11 @@ void process_options( ssize_t catch_all_group = -1; for (size_t i = 0; i < addr_groups.size(); ++i) { auto &g = addr_groups[i]; - if (util::streq(g.pattern.get(), "/")) { + if (g.pattern == "/") { catch_all_group = i; } if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Host-path pattern: group " << i << ": '" << g.pattern.get() + LOG(INFO) << "Host-path pattern: group " << i << ": '" << g.pattern << "'"; for (auto &addr : g.addrs) { LOG(INFO) << "group " << i << " -> " << addr.host.c_str() diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 489885cf..23b480b3 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -78,21 +78,6 @@ TicketKeys::~TicketKeys() { } } -DownstreamAddrGroup::DownstreamAddrGroup(const DownstreamAddrGroup &other) - : pattern(strcopy(other.pattern)), addrs(other.addrs) {} - -DownstreamAddrGroup &DownstreamAddrGroup:: -operator=(const DownstreamAddrGroup &other) { - if (this == &other) { - return *this; - } - - pattern = strcopy(other.pattern); - addrs = other.addrs; - - return *this; -} - namespace { int split_host_port(char *host, size_t hostlen, uint16_t *port_ptr, const char *hostport, size_t hostportlen) { @@ -612,7 +597,7 @@ void parse_mapping(const DownstreamAddr &addr, const char *src) { pattern += http2::normalize_path(slash, raw_pattern.second); } for (auto &g : addr_groups) { - if (g.pattern.get() == pattern) { + if (g.pattern == pattern) { g.addrs.push_back(addr); done = true; break; @@ -621,10 +606,10 @@ void parse_mapping(const DownstreamAddr &addr, const char *src) { if (done) { continue; } - DownstreamAddrGroup g(pattern); + DownstreamAddrGroup g(StringRef{pattern}); g.addrs.push_back(addr); - mod_config()->router.add_route(g.pattern.get(), strlen(g.pattern.get()), + mod_config()->router.add_route(g.pattern.c_str(), g.pattern.size(), addr_groups.size()); addr_groups.push_back(std::move(g)); @@ -2529,7 +2514,7 @@ match_downstream_addr_group_host(const Router &router, const std::string &host, if (group != -1) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Found pattern with query " << host - << ", matched pattern=" << groups[group].pattern.get(); + << ", matched pattern=" << groups[group].pattern; } return group; } @@ -2546,7 +2531,7 @@ match_downstream_addr_group_host(const Router &router, const std::string &host, if (LOG_ENABLED(INFO)) { LOG(INFO) << "Found pattern with query " << host << std::string(path, pathlen) - << ", matched pattern=" << groups[group].pattern.get(); + << ", matched pattern=" << groups[group].pattern; } return group; } @@ -2555,7 +2540,7 @@ match_downstream_addr_group_host(const Router &router, const std::string &host, if (group != -1) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Found pattern with query " << std::string(path, pathlen) - << ", matched pattern=" << groups[group].pattern.get(); + << ", matched pattern=" << groups[group].pattern; } return group; } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index c460cbc8..e024317a 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -301,13 +301,10 @@ struct DownstreamAddr { }; struct DownstreamAddrGroup { - DownstreamAddrGroup(const std::string &pattern) : pattern(strcopy(pattern)) {} - DownstreamAddrGroup(const DownstreamAddrGroup &other); - DownstreamAddrGroup(DownstreamAddrGroup &&) = default; - DownstreamAddrGroup &operator=(const DownstreamAddrGroup &other); - DownstreamAddrGroup &operator=(DownstreamAddrGroup &&) = default; + DownstreamAddrGroup(const StringRef &pattern) + : pattern(pattern.c_str(), pattern.size()) {} - std::unique_ptr pattern; + ImmutableString pattern; std::vector addrs; }; diff --git a/src/shrpx_config_test.cc b/src/shrpx_config_test.cc index eda67e85..fa8c1391 100644 --- a/src/shrpx_config_test.cc +++ b/src/shrpx_config_test.cc @@ -256,7 +256,7 @@ void test_shrpx_config_match_downstream_addr_group(void) { for (size_t i = 0; i < groups.size(); ++i) { auto &g = groups[i]; - router.add_route(g.pattern.get(), strlen(g.pattern.get()), i); + router.add_route(g.pattern.c_str(), g.pattern.size(), i); } CU_ASSERT(0 == match_downstream_addr_group(router, "nghttp2.org", "/", groups, From 2d273f82375b527b081bf1e3b2fd60a6024c30de Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 18:55:53 +0900 Subject: [PATCH 43/85] nghttpx: Use StringRef for pattern paramter in Router::add_route --- src/shrpx.cc | 4 ++-- src/shrpx_config.cc | 3 +-- src/shrpx_config_test.cc | 2 +- src/shrpx_router.cc | 12 ++++++------ src/shrpx_router.h | 4 ++-- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 24290891..ee05263b 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2111,7 +2111,7 @@ void process_options( DownstreamAddrGroup g(StringRef::from_lit("/")); g.addrs.push_back(std::move(addr)); - mod_config()->router.add_route(g.pattern.c_str(), 1, addr_groups.size()); + mod_config()->router.add_route(StringRef{g.pattern}, addr_groups.size()); addr_groups.push_back(std::move(g)); } else if (get_config()->http2_proxy || get_config()->client_proxy) { // We don't support host mapping in these cases. Move all @@ -2124,7 +2124,7 @@ void process_options( std::vector().swap(addr_groups); // maybe not necessary? mod_config()->router = Router(); - mod_config()->router.add_route(catch_all.pattern.c_str(), 1, + mod_config()->router.add_route(StringRef{catch_all.pattern}, addr_groups.size()); addr_groups.push_back(std::move(catch_all)); } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 23b480b3..9dd5c444 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -609,8 +609,7 @@ void parse_mapping(const DownstreamAddr &addr, const char *src) { DownstreamAddrGroup g(StringRef{pattern}); g.addrs.push_back(addr); - mod_config()->router.add_route(g.pattern.c_str(), g.pattern.size(), - addr_groups.size()); + mod_config()->router.add_route(StringRef{g.pattern}, addr_groups.size()); addr_groups.push_back(std::move(g)); } diff --git a/src/shrpx_config_test.cc b/src/shrpx_config_test.cc index fa8c1391..53b1bc30 100644 --- a/src/shrpx_config_test.cc +++ b/src/shrpx_config_test.cc @@ -256,7 +256,7 @@ void test_shrpx_config_match_downstream_addr_group(void) { for (size_t i = 0; i < groups.size(); ++i) { auto &g = groups[i]; - router.add_route(g.pattern.c_str(), g.pattern.size(), i); + router.add_route(StringRef{g.pattern}, i); } CU_ASSERT(0 == match_downstream_addr_group(router, "nghttp2.org", "/", groups, diff --git a/src/shrpx_router.cc b/src/shrpx_router.cc index e0404252..4324748a 100644 --- a/src/shrpx_router.cc +++ b/src/shrpx_router.cc @@ -66,21 +66,21 @@ void Router::add_node(RNode *node, const char *pattern, size_t patlen, add_next_node(node, std::move(new_node)); } -bool Router::add_route(const char *pattern, size_t patlen, size_t index) { +bool Router::add_route(const StringRef &pattern, size_t index) { auto node = &root_; size_t i = 0; for (;;) { auto next_node = find_next_node(node, pattern[i]); if (next_node == nullptr) { - add_node(node, pattern + i, patlen - i, index); + add_node(node, pattern.c_str() + i, pattern.size() - i, index); return true; } node = next_node; - auto slen = patlen - i; - auto s = pattern + i; + auto slen = pattern.size() - i; + auto s = pattern.c_str() + i; auto n = std::min(node->len, slen); size_t j; for (j = 0; j < n && node->s[j] == s[j]; ++j) @@ -125,8 +125,8 @@ bool Router::add_route(const char *pattern, size_t patlen, size_t index) { i += j; - assert(patlen > i); - add_node(node, pattern + i, patlen - i, index); + assert(pattern.size() > i); + add_node(node, pattern.c_str() + i, pattern.size() - i, index); return true; } diff --git a/src/shrpx_router.h b/src/shrpx_router.h index 07aadf69..e4d9a465 100644 --- a/src/shrpx_router.h +++ b/src/shrpx_router.h @@ -55,8 +55,8 @@ struct RNode { class Router { public: Router(); - // Adds route |pattern| of size |patlen| with its |index|. - bool add_route(const char *pattern, size_t patlen, size_t index); + // Adds route |pattern| with its |index|. + bool add_route(const StringRef &pattern, size_t index); // Returns the matched index of pattern. -1 if there is no match. ssize_t match(const std::string &host, const char *path, size_t pathlen) const; From 93eabc642b9732a5172b5b2eaf104320d46fa68d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 19:07:22 +0900 Subject: [PATCH 44/85] nghttpx: Use StringRef for parameter in Router::match --- src/shrpx_config.cc | 32 ++++++++++++++------------------ src/shrpx_router.cc | 5 ++--- src/shrpx_router.h | 3 +-- src/template.h | 3 +++ 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 9dd5c444..a5d9f534 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2503,13 +2503,11 @@ int int_syslog_facility(const char *strfacility) { } namespace { -size_t -match_downstream_addr_group_host(const Router &router, const std::string &host, - const char *path, size_t pathlen, - const std::vector &groups, - size_t catch_all) { - if (pathlen == 0 || *path != '/') { - auto group = router.match(host, "/", 1); +size_t match_downstream_addr_group_host( + const Router &router, const std::string &host, const StringRef &path, + const std::vector &groups, size_t catch_all) { + if (path.empty() || path[0] != '/') { + auto group = router.match(host, StringRef::from_lit("/")); if (group != -1) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Found pattern with query " << host @@ -2522,23 +2520,22 @@ match_downstream_addr_group_host(const Router &router, const std::string &host, if (LOG_ENABLED(INFO)) { LOG(INFO) << "Perform mapping selection, using host=" << host - << ", path=" << std::string(path, pathlen); + << ", path=" << path; } - auto group = router.match(host, path, pathlen); + auto group = router.match(host, path); if (group != -1) { if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Found pattern with query " << host - << std::string(path, pathlen) + LOG(INFO) << "Found pattern with query " << host << path << ", matched pattern=" << groups[group].pattern; } return group; } - group = router.match("", path, pathlen); + group = router.match("", path); if (group != -1) { if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Found pattern with query " << std::string(path, pathlen) + LOG(INFO) << "Found pattern with query " << path << ", matched pattern=" << groups[group].pattern; } return group; @@ -2565,12 +2562,11 @@ match_downstream_addr_group(const Router &router, const std::string &hostport, auto fragment = std::find(std::begin(raw_path), std::end(raw_path), '#'); auto query = std::find(std::begin(raw_path), fragment, '?'); - auto path = raw_path.c_str(); - auto pathlen = query - std::begin(raw_path); + auto path = StringRef{std::begin(raw_path), query}; if (hostport.empty()) { - return match_downstream_addr_group_host(router, hostport, path, pathlen, - groups, catch_all); + return match_downstream_addr_group_host(router, hostport, path, groups, + catch_all); } std::string host; @@ -2593,7 +2589,7 @@ match_downstream_addr_group(const Router &router, const std::string &hostport, } util::inp_strlower(host); - return match_downstream_addr_group_host(router, host, path, pathlen, groups, + return match_downstream_addr_group_host(router, host, path, groups, catch_all); } diff --git a/src/shrpx_router.cc b/src/shrpx_router.cc index 4324748a..2975c3e9 100644 --- a/src/shrpx_router.cc +++ b/src/shrpx_router.cc @@ -259,8 +259,7 @@ const RNode *match_partial(const RNode *node, size_t offset, const char *first, } } // namespace -ssize_t Router::match(const std::string &host, const char *path, - size_t pathlen) const { +ssize_t Router::match(const std::string &host, const StringRef &path) const { const RNode *node; size_t offset; @@ -270,7 +269,7 @@ ssize_t Router::match(const std::string &host, const char *path, return -1; } - node = match_partial(node, offset, path, path + pathlen); + node = match_partial(node, offset, std::begin(path), std::end(path)); if (node == nullptr || node == &root_) { return -1; } diff --git a/src/shrpx_router.h b/src/shrpx_router.h index e4d9a465..5f70b212 100644 --- a/src/shrpx_router.h +++ b/src/shrpx_router.h @@ -58,8 +58,7 @@ public: // Adds route |pattern| with its |index|. bool add_route(const StringRef &pattern, size_t index); // Returns the matched index of pattern. -1 if there is no match. - ssize_t match(const std::string &host, const char *path, - size_t pathlen) const; + ssize_t match(const std::string &host, const StringRef &path) const; void add_node(RNode *node, const char *pattern, size_t patlen, size_t index); diff --git a/src/template.h b/src/template.h index 3061ad52..253ea215 100644 --- a/src/template.h +++ b/src/template.h @@ -397,6 +397,9 @@ public: : base(reinterpret_cast(s)), len(n) {} template StringRef(InputIt first, InputIt last) + : base(&*first), len(std::distance(first, last)) {} + template + StringRef(InputIt *first, InputIt *last) : base(first), len(std::distance(first, last)) {} template constexpr static StringRef from_lit(const CharT(&s)[N]) { From 49fa914db5cbf74708e025f9aeae74f9a79f2ab1 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 20:48:06 +0900 Subject: [PATCH 45/85] nghttpx: Use StringRef for string parameters in match_downstream_addr_group --- src/shrpx_client_handler.cc | 15 +++++++++------ src/shrpx_config.cc | 12 +++++------- src/shrpx_config.h | 2 +- src/shrpx_router.cc | 5 ++--- src/shrpx_router.h | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index fa814dd9..4d063647 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -681,16 +681,19 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { } else { auto &router = get_config()->router; if (!req.authority.empty()) { - group = match_downstream_addr_group(router, req.authority, req.path, - groups, catch_all); + group = + match_downstream_addr_group(router, StringRef{req.authority}, + StringRef{req.path}, groups, catch_all); } else { auto h = req.fs.header(http2::HD_HOST); if (h) { - group = match_downstream_addr_group(router, h->value, req.path, groups, - catch_all); + group = + match_downstream_addr_group(router, StringRef{h->value}, + StringRef{req.path}, groups, catch_all); } else { - group = match_downstream_addr_group(router, "", req.path, groups, - catch_all); + group = + match_downstream_addr_group(router, StringRef::from_lit(""), + StringRef{req.path}, groups, catch_all); } } } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index a5d9f534..5e6236c3 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2504,7 +2504,7 @@ int int_syslog_facility(const char *strfacility) { namespace { size_t match_downstream_addr_group_host( - const Router &router, const std::string &host, const StringRef &path, + const Router &router, const StringRef &host, const StringRef &path, const std::vector &groups, size_t catch_all) { if (path.empty() || path[0] != '/') { auto group = router.match(host, StringRef::from_lit("/")); @@ -2548,11 +2548,9 @@ size_t match_downstream_addr_group_host( } } // namespace -size_t -match_downstream_addr_group(const Router &router, const std::string &hostport, - const std::string &raw_path, - const std::vector &groups, - size_t catch_all) { +size_t match_downstream_addr_group( + const Router &router, const StringRef &hostport, const StringRef &raw_path, + const std::vector &groups, size_t catch_all) { if (std::find(std::begin(hostport), std::end(hostport), '/') != std::end(hostport)) { // We use '/' specially, and if '/' is included in host, it breaks @@ -2589,7 +2587,7 @@ match_downstream_addr_group(const Router &router, const std::string &hostport, } util::inp_strlower(host); - return match_downstream_addr_group_host(router, host, path, groups, + return match_downstream_addr_group_host(router, StringRef{host}, path, groups, catch_all); } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index e024317a..b81f8e1d 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -652,7 +652,7 @@ read_tls_ticket_key_file(const std::vector &files, // group. The catch-all group index is given in |catch_all|. All // patterns are given in |groups|. size_t match_downstream_addr_group( - const Router &router, const std::string &hostport, const std::string &path, + const Router &router, const StringRef &hostport, const StringRef &path, const std::vector &groups, size_t catch_all); } // namespace shrpx diff --git a/src/shrpx_router.cc b/src/shrpx_router.cc index 2975c3e9..4c88283f 100644 --- a/src/shrpx_router.cc +++ b/src/shrpx_router.cc @@ -259,12 +259,11 @@ const RNode *match_partial(const RNode *node, size_t offset, const char *first, } } // namespace -ssize_t Router::match(const std::string &host, const StringRef &path) const { +ssize_t Router::match(const StringRef &host, const StringRef &path) const { const RNode *node; size_t offset; - node = - match_complete(&offset, &root_, host.c_str(), host.c_str() + host.size()); + node = match_complete(&offset, &root_, std::begin(host), std::end(host)); if (node == nullptr) { return -1; } diff --git a/src/shrpx_router.h b/src/shrpx_router.h index 5f70b212..ece0e276 100644 --- a/src/shrpx_router.h +++ b/src/shrpx_router.h @@ -58,7 +58,7 @@ public: // Adds route |pattern| with its |index|. bool add_route(const StringRef &pattern, size_t index); // Returns the matched index of pattern. -1 if there is no match. - ssize_t match(const std::string &host, const StringRef &path) const; + ssize_t match(const StringRef &host, const StringRef &path) const; void add_node(RNode *node, const char *pattern, size_t patlen, size_t index); From bfc26e82997aa86460eedd182abc1a3814d5d107 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 20:59:10 +0900 Subject: [PATCH 46/85] nghttpx: Use ImmutableString to store memcached server host --- src/shrpx.cc | 16 ++++++++-------- src/shrpx_config.cc | 4 ++-- src/shrpx_config.h | 8 ++++++-- src/shrpx_ssl.cc | 2 +- src/shrpx_worker.cc | 4 ++-- src/shrpx_worker_process.cc | 4 ++-- 6 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index ee05263b..33272f97 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2220,10 +2220,10 @@ void process_options( { auto &memcachedconf = tlsconf.session_cache.memcached; - if (memcachedconf.host) { - auto hostport = - util::make_hostport(memcachedconf.host.get(), memcachedconf.port); - if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.get(), + if (!memcachedconf.host.empty()) { + auto hostport = util::make_hostport(StringRef{memcachedconf.host}, + memcachedconf.port); + if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.c_str(), memcachedconf.port, memcachedconf.family) == -1) { LOG(FATAL) << "Resolving memcached address for TLS session cache failed: " @@ -2238,10 +2238,10 @@ void process_options( { auto &memcachedconf = tlsconf.ticket.memcached; - if (memcachedconf.host) { - auto hostport = - util::make_hostport(memcachedconf.host.get(), memcachedconf.port); - if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.get(), + if (!memcachedconf.host.empty()) { + auto hostport = util::make_hostport(StringRef{memcachedconf.host}, + memcachedconf.port); + if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.c_str(), memcachedconf.port, memcachedconf.family) == -1) { LOG(FATAL) << "Resolving memcached address for TLS ticket key failed: " << hostport; diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 5e6236c3..20d30c86 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2154,7 +2154,7 @@ int parse_config(const char *opt, const char *optarg, } auto &memcachedconf = mod_config()->tls.session_cache.memcached; - memcachedconf.host = strcopy(host); + memcachedconf.host = host; memcachedconf.port = port; return 0; @@ -2166,7 +2166,7 @@ int parse_config(const char *opt, const char *optarg, } auto &memcachedconf = mod_config()->tls.ticket.memcached; - memcachedconf.host = strcopy(host); + memcachedconf.host = host; memcachedconf.port = port; return 0; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index b81f8e1d..e5a21241 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -343,7 +343,9 @@ struct TLSConfig { struct { Address addr; uint16_t port; - std::unique_ptr host; + // Hostname of memcached server. This is also used as SNI field + // if TLS is enabled. + ImmutableString host; // Client private key and certificate for authentication ImmutableString private_key_file; ImmutableString cert_file; @@ -370,7 +372,9 @@ struct TLSConfig { struct { Address addr; uint16_t port; - std::unique_ptr host; + // Hostname of memcached server. This is also used as SNI field + // if TLS is enabled. + ImmutableString host; // Client private key and certificate for authentication ImmutableString private_key_file; ImmutableString cert_file; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 5d8a2360..afbcbb15 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -485,7 +485,7 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file SSL_CTX_set_session_id_context(ssl_ctx, sid_ctx, sizeof(sid_ctx) - 1); SSL_CTX_set_session_cache_mode(ssl_ctx, SSL_SESS_CACHE_SERVER); - if (tlsconf.session_cache.memcached.host) { + if (!tlsconf.session_cache.memcached.host.empty()) { SSL_CTX_sess_set_new_cb(ssl_ctx, tls_session_new_cb); SSL_CTX_sess_set_get_cb(ssl_ctx, tls_session_get_cb); } diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index e663eace..83cfff9b 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -91,11 +91,11 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, auto &session_cacheconf = get_config()->tls.session_cache; - if (session_cacheconf.memcached.host) { + if (!session_cacheconf.memcached.host.empty()) { session_cache_memcached_dispatcher_ = make_unique( &session_cacheconf.memcached.addr, loop, tls_session_cache_memcached_ssl_ctx, - session_cacheconf.memcached.host.get(), &mcpool_); + StringRef{session_cacheconf.memcached.host}, &mcpool_); } auto &downstreamconf = get_config()->conn.downstream; diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc index 26863d82..f7dacc7c 100644 --- a/src/shrpx_worker_process.cc +++ b/src/shrpx_worker_process.cc @@ -427,7 +427,7 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { auto &ticketconf = get_config()->tls.ticket; auto &memcachedconf = ticketconf.memcached; - if (ticketconf.memcached.host) { + if (!memcachedconf.host.empty()) { SSL_CTX *ssl_ctx = nullptr; if (memcachedconf.tls) { @@ -437,7 +437,7 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { conn_handler.set_tls_ticket_key_memcached_dispatcher( make_unique( &ticketconf.memcached.addr, loop, ssl_ctx, - StringRef(memcachedconf.host.get()), &mcpool)); + StringRef{memcachedconf.host}, &mcpool)); ev_timer_init(&renew_ticket_key_timer, memcached_get_ticket_key_cb, 0., 0.); From 660bc389e6f8276fe0b508f890f3e3ce4567fbd5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:01:54 +0900 Subject: [PATCH 47/85] nghttpx: Use ImmutableString for fetch_ocsp_response_file --- src/shrpx.cc | 11 +++++------ src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- src/shrpx_connection_handler.cc | 3 ++- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 33272f97..44c4094e 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1067,8 +1067,7 @@ void fill_default_config() { auto &ocspconf = tlsconf.ocsp; // ocsp update interval = 14400 secs = 4 hours, borrowed from h2o ocspconf.update_interval = 4_h; - ocspconf.fetch_ocsp_response_file = - strcopy(PKGDATADIR "/fetch-ocsp-response"); + ocspconf.fetch_ocsp_response_file = PKGDATADIR "/fetch-ocsp-response"; } { @@ -1579,8 +1578,8 @@ SSL/TLS: --fetch-ocsp-response-file= Path to fetch-ocsp-response script file. It should be absolute path. - Default: )" - << get_config()->tls.ocsp.fetch_ocsp_response_file.get() << R"( + Default: )" << get_config()->tls.ocsp.fetch_ocsp_response_file + << R"( --ocsp-update-interval= Set interval to update OCSP response cache. Default: )" @@ -2094,10 +2093,10 @@ void process_options( if (!upstreamconf.no_tls && !tlsconf.ocsp.disabled) { struct stat buf; - if (stat(tlsconf.ocsp.fetch_ocsp_response_file.get(), &buf) != 0) { + if (stat(tlsconf.ocsp.fetch_ocsp_response_file.c_str(), &buf) != 0) { tlsconf.ocsp.disabled = true; LOG(WARN) << "--fetch-ocsp-response-file: " - << tlsconf.ocsp.fetch_ocsp_response_file.get() + << tlsconf.ocsp.fetch_ocsp_response_file << " not found. OCSP stapling has been disabled."; } } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 20d30c86..6afcc47b 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2086,7 +2086,7 @@ int parse_config(const char *opt, const char *optarg, return parse_uint(&mod_config()->http2.downstream.connections_per_worker, opt, optarg); case SHRPX_OPTID_FETCH_OCSP_RESPONSE_FILE: - mod_config()->tls.ocsp.fetch_ocsp_response_file = strcopy(optarg); + mod_config()->tls.ocsp.fetch_ocsp_response_file = optarg; return 0; case SHRPX_OPTID_OCSP_UPDATE_INTERVAL: diff --git a/src/shrpx_config.h b/src/shrpx_config.h index e5a21241..d290a37a 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -394,7 +394,7 @@ struct TLSConfig { // OCSP realted configurations struct { ev_tstamp update_interval; - std::unique_ptr fetch_ocsp_response_file; + ImmutableString fetch_ocsp_response_file; bool disabled; } ocsp; diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 588e814b..8dea4f38 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -463,7 +463,8 @@ int ConnectionHandler::start_ocsp_update(const char *cert_file) { assert(!ev_is_active(&ocsp_.chldev)); char *const argv[] = { - const_cast(get_config()->tls.ocsp.fetch_ocsp_response_file.get()), + const_cast( + get_config()->tls.ocsp.fetch_ocsp_response_file.c_str()), const_cast(cert_file), nullptr}; char *const envp[] = {nullptr}; From 52f641781321f15453bd0d890a3db25e5bf96c3e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:06:16 +0900 Subject: [PATCH 48/85] nghttpx: Use ImmutableString for tls.cacert --- src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- src/shrpx_connection_handler.cc | 15 ++++++--------- src/shrpx_ssl.cc | 2 +- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 6afcc47b..fc3cdce8 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1816,7 +1816,7 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_CACERT: - mod_config()->tls.cacert = strcopy(optarg); + mod_config()->tls.cacert = optarg; return 0; case SHRPX_OPTID_BACKEND_IPV4: diff --git a/src/shrpx_config.h b/src/shrpx_config.h index d290a37a..8326cbf3 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -431,7 +431,7 @@ struct TLSConfig { std::unique_ptr cert_file; std::unique_ptr dh_param_file; std::unique_ptr ciphers; - std::unique_ptr cacert; + ImmutableString cacert; bool insecure; bool no_http2_cipher_black_list; }; diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 8dea4f38..008f1289 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -202,9 +202,8 @@ int ConnectionHandler::create_single_worker() { #ifdef HAVE_NEVERBLEED nb_.get(), #endif // HAVE_NEVERBLEED - StringRef::from_maybe_nullptr(tlsconf.cacert.get()), - StringRef(memcachedconf.cert_file), - StringRef(memcachedconf.private_key_file), StringRef(), nullptr); + StringRef{tlsconf.cacert}, StringRef{memcachedconf.cert_file}, + StringRef{memcachedconf.private_key_file}, StringRef(), nullptr); all_ssl_ctx_.push_back(session_cache_ssl_ctx); } @@ -253,9 +252,8 @@ int ConnectionHandler::create_worker_thread(size_t num) { #ifdef HAVE_NEVERBLEED nb_.get(), #endif // HAVE_NEVERBLEED - StringRef::from_maybe_nullptr(tlsconf.cacert.get()), - StringRef(memcachedconf.cert_file), - StringRef(memcachedconf.private_key_file), StringRef(), nullptr); + StringRef{tlsconf.cacert}, StringRef{memcachedconf.cert_file}, + StringRef{memcachedconf.private_key_file}, StringRef{}, nullptr); all_ssl_ctx_.push_back(session_cache_ssl_ctx); } auto worker = @@ -768,9 +766,8 @@ SSL_CTX *ConnectionHandler::create_tls_ticket_key_memcached_ssl_ctx() { #ifdef HAVE_NEVERBLEED nb_.get(), #endif // HAVE_NEVERBLEED - StringRef::from_maybe_nullptr(tlsconf.cacert.get()), - StringRef(memcachedconf.cert_file), - StringRef(memcachedconf.private_key_file), StringRef(), nullptr); + StringRef{tlsconf.cacert}, StringRef{memcachedconf.cert_file}, + StringRef{memcachedconf.private_key_file}, StringRef{}, nullptr); all_ssl_ctx_.push_back(ssl_ctx); diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index afbcbb15..bfba5a57 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -1323,7 +1323,7 @@ SSL_CTX *setup_downstream_client_ssl_context( #ifdef HAVE_NEVERBLEED nb, #endif // HAVE_NEVERBLEED - StringRef::from_maybe_nullptr(tlsconf.cacert.get()), + StringRef{tlsconf.cacert}, StringRef::from_maybe_nullptr(tlsconf.client.cert_file.get()), StringRef::from_maybe_nullptr(tlsconf.client.private_key_file.get()), alpn, next_proto_select_cb); From 529a59d309f40c9e0570a6c2be6332b182b53826 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:09:15 +0900 Subject: [PATCH 49/85] nghttpx: Use ImmutableString for tls.client_verify.cacert --- src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- src/shrpx_ssl.cc | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index fc3cdce8..506391a7 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1907,7 +1907,7 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_VERIFY_CLIENT_CACERT: - mod_config()->tls.client_verify.cacert = strcopy(optarg); + mod_config()->tls.client_verify.cacert = optarg; return 0; case SHRPX_OPTID_CLIENT_PRIVATE_KEY_FILE: diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 8326cbf3..5ea8cc62 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -402,7 +402,7 @@ struct TLSConfig { struct { // Path to file containing CA certificate solely used for client // certificate validation - std::unique_ptr cacert; + ImmutableString cacert; bool enabled; } client_verify; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index bfba5a57..d6fa26a7 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -579,12 +579,12 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file DIE(); } if (tlsconf.client_verify.enabled) { - if (tlsconf.client_verify.cacert) { + if (!tlsconf.client_verify.cacert.empty()) { if (SSL_CTX_load_verify_locations( - ssl_ctx, tlsconf.client_verify.cacert.get(), nullptr) != 1) { + ssl_ctx, tlsconf.client_verify.cacert.c_str(), nullptr) != 1) { LOG(FATAL) << "Could not load trusted ca certificates from " - << tlsconf.client_verify.cacert.get() << ": " + << tlsconf.client_verify.cacert << ": " << ERR_error_string(ERR_get_error(), nullptr); DIE(); } @@ -592,10 +592,10 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file // error even though it returns success. See // http://forum.nginx.org/read.php?29,242540 ERR_clear_error(); - auto list = SSL_load_client_CA_file(tlsconf.client_verify.cacert.get()); + auto list = SSL_load_client_CA_file(tlsconf.client_verify.cacert.c_str()); if (!list) { LOG(FATAL) << "Could not load ca certificates from " - << tlsconf.client_verify.cacert.get() << ": " + << tlsconf.client_verify.cacert << ": " << ERR_error_string(ERR_get_error(), nullptr); DIE(); } From c999987baf2b4c0a53bbab4ce50ed00f2cb3e66f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:13:46 +0900 Subject: [PATCH 50/85] nghttpx: Use ImmutableString for private_key_file --- src/shrpx.cc | 2 +- src/shrpx_config.cc | 4 ++-- src/shrpx_config.h | 4 ++-- src/shrpx_ssl.cc | 5 ++--- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 44c4094e..369c1a78 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2085,7 +2085,7 @@ void process_options( } if (!upstreamconf.no_tls && - (!tlsconf.private_key_file || !tlsconf.cert_file)) { + (tlsconf.private_key_file.empty() || !tlsconf.cert_file)) { print_usage(std::cerr); LOG(FATAL) << "Too few arguments"; exit(EXIT_FAILURE); diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 506391a7..10d9d97b 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1745,7 +1745,7 @@ int parse_config(const char *opt, const char *optarg, return 0; } case SHRPX_OPTID_PRIVATE_KEY_FILE: - mod_config()->tls.private_key_file = strcopy(optarg); + mod_config()->tls.private_key_file = optarg; return 0; case SHRPX_OPTID_PRIVATE_KEY_PASSWD_FILE: { @@ -1911,7 +1911,7 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_CLIENT_PRIVATE_KEY_FILE: - mod_config()->tls.client.private_key_file = strcopy(optarg); + mod_config()->tls.client.private_key_file = optarg; return 0; case SHRPX_OPTID_CLIENT_CERT_FILE: diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 5ea8cc62..4b6b514d 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -408,7 +408,7 @@ struct TLSConfig { // Client private key and certificate used in backend connections. struct { - std::unique_ptr private_key_file; + ImmutableString private_key_file; std::unique_ptr cert_file; } client; @@ -426,7 +426,7 @@ struct TLSConfig { long int tls_proto_mask; std::string backend_sni_name; std::chrono::seconds session_timeout; - std::unique_ptr private_key_file; + ImmutableString private_key_file; std::unique_ptr private_key_passwd; std::unique_ptr cert_file; std::unique_ptr dh_param_file; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index d6fa26a7..c5e8b79f 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -1245,7 +1245,7 @@ SSL_CTX *setup_server_ssl_context(std::vector &all_ssl_ctx, auto &tlsconf = get_config()->tls; - auto ssl_ctx = ssl::create_ssl_context(tlsconf.private_key_file.get(), + auto ssl_ctx = ssl::create_ssl_context(tlsconf.private_key_file.c_str(), tlsconf.cert_file.get() #ifdef HAVE_NEVERBLEED , @@ -1325,8 +1325,7 @@ SSL_CTX *setup_downstream_client_ssl_context( #endif // HAVE_NEVERBLEED StringRef{tlsconf.cacert}, StringRef::from_maybe_nullptr(tlsconf.client.cert_file.get()), - StringRef::from_maybe_nullptr(tlsconf.client.private_key_file.get()), - alpn, next_proto_select_cb); + StringRef{tlsconf.client.private_key_file}, alpn, next_proto_select_cb); } CertLookupTree *create_cert_lookup_tree() { From ac81003669deb40ede9b5559b9c832b43a787d71 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:17:25 +0900 Subject: [PATCH 51/85] nghttpx: Use ImmutableString for cert_file --- src/shrpx.cc | 2 +- src/shrpx_config.cc | 4 ++-- src/shrpx_config.h | 4 ++-- src/shrpx_ssl.cc | 9 ++++----- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 369c1a78..17adf31d 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2085,7 +2085,7 @@ void process_options( } if (!upstreamconf.no_tls && - (tlsconf.private_key_file.empty() || !tlsconf.cert_file)) { + (tlsconf.private_key_file.empty() || tlsconf.cert_file.empty())) { print_usage(std::cerr); LOG(FATAL) << "Too few arguments"; exit(EXIT_FAILURE); diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 10d9d97b..12e01978 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1759,7 +1759,7 @@ int parse_config(const char *opt, const char *optarg, return 0; } case SHRPX_OPTID_CERTIFICATE_FILE: - mod_config()->tls.cert_file = strcopy(optarg); + mod_config()->tls.cert_file = optarg; return 0; case SHRPX_OPTID_DH_PARAM_FILE: @@ -1915,7 +1915,7 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_CLIENT_CERT_FILE: - mod_config()->tls.client.cert_file = strcopy(optarg); + mod_config()->tls.client.cert_file = optarg; return 0; case SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER: diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 4b6b514d..52f12ca1 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -409,7 +409,7 @@ struct TLSConfig { // Client private key and certificate used in backend connections. struct { ImmutableString private_key_file; - std::unique_ptr cert_file; + ImmutableString cert_file; } client; // The list of (private key file, certificate file) pair @@ -428,7 +428,7 @@ struct TLSConfig { std::chrono::seconds session_timeout; ImmutableString private_key_file; std::unique_ptr private_key_passwd; - std::unique_ptr cert_file; + ImmutableString cert_file; std::unique_ptr dh_param_file; std::unique_ptr ciphers; ImmutableString cacert; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index c5e8b79f..adb80e47 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -1246,7 +1246,7 @@ SSL_CTX *setup_server_ssl_context(std::vector &all_ssl_ctx, auto &tlsconf = get_config()->tls; auto ssl_ctx = ssl::create_ssl_context(tlsconf.private_key_file.c_str(), - tlsconf.cert_file.get() + tlsconf.cert_file.c_str() #ifdef HAVE_NEVERBLEED , nb @@ -1281,8 +1281,8 @@ SSL_CTX *setup_server_ssl_context(std::vector &all_ssl_ctx, } } - if (ssl::cert_lookup_tree_add_cert_from_file(cert_tree, ssl_ctx, - tlsconf.cert_file.get()) == -1) { + if (ssl::cert_lookup_tree_add_cert_from_file( + cert_tree, ssl_ctx, tlsconf.cert_file.c_str()) == -1) { LOG(FATAL) << "Failed to add default certificate."; DIE(); } @@ -1323,8 +1323,7 @@ SSL_CTX *setup_downstream_client_ssl_context( #ifdef HAVE_NEVERBLEED nb, #endif // HAVE_NEVERBLEED - StringRef{tlsconf.cacert}, - StringRef::from_maybe_nullptr(tlsconf.client.cert_file.get()), + StringRef{tlsconf.cacert}, StringRef{tlsconf.client.cert_file}, StringRef{tlsconf.client.private_key_file}, alpn, next_proto_select_cb); } From 35ebdd35bc6bb9ccc98686fb8b432db181d2ae84 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:20:40 +0900 Subject: [PATCH 52/85] nghttpx: Use ImmutableString for private_key_file --- src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- src/shrpx_ssl.cc | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 12e01978..0682149f 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1754,7 +1754,7 @@ int parse_config(const char *opt, const char *optarg, LOG(ERROR) << opt << ": Couldn't read key file's passwd from " << optarg; return -1; } - mod_config()->tls.private_key_passwd = strcopy(passwd); + mod_config()->tls.private_key_passwd = passwd; return 0; } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 52f12ca1..3f5a8565 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -427,7 +427,7 @@ struct TLSConfig { std::string backend_sni_name; std::chrono::seconds session_timeout; ImmutableString private_key_file; - std::unique_ptr private_key_passwd; + ImmutableString private_key_passwd; ImmutableString cert_file; std::unique_ptr dh_param_file; std::unique_ptr ciphers; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index adb80e47..f5d2b594 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -124,13 +124,13 @@ set_alpn_prefs(const std::vector &protos) { namespace { int ssl_pem_passwd_cb(char *buf, int size, int rwflag, void *user_data) { auto config = static_cast(user_data); - int len = (int)strlen(config->tls.private_key_passwd.get()); + auto len = static_cast(config->tls.private_key_passwd.size()); if (size < len + 1) { LOG(ERROR) << "ssl_pem_passwd_cb: buf is too small " << size; return 0; } // Copy string including last '\0'. - memcpy(buf, config->tls.private_key_passwd.get(), len + 1); + memcpy(buf, config->tls.private_key_passwd.c_str(), len + 1); return len; } } // namespace @@ -548,7 +548,7 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file SSL_CTX_set_mode(ssl_ctx, SSL_MODE_AUTO_RETRY); SSL_CTX_set_mode(ssl_ctx, SSL_MODE_RELEASE_BUFFERS); - if (tlsconf.private_key_passwd) { + if (!tlsconf.private_key_passwd.empty()) { SSL_CTX_set_default_passwd_cb(ssl_ctx, ssl_pem_passwd_cb); SSL_CTX_set_default_passwd_cb_userdata(ssl_ctx, (void *)get_config()); } From 2344932b45e5b22b37651ab2b432f0bb2504ba06 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:22:28 +0900 Subject: [PATCH 53/85] nghttpx: Use ImmutableString for dh_param_file --- src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- src/shrpx_ssl.cc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 0682149f..24e89d05 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1763,7 +1763,7 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_DH_PARAM_FILE: - mod_config()->tls.dh_param_file = strcopy(optarg); + mod_config()->tls.dh_param_file = optarg; return 0; case SHRPX_OPTID_SUBCERT: { diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 3f5a8565..dbe8c9a5 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -429,7 +429,7 @@ struct TLSConfig { ImmutableString private_key_file; ImmutableString private_key_passwd; ImmutableString cert_file; - std::unique_ptr dh_param_file; + ImmutableString dh_param_file; std::unique_ptr ciphers; ImmutableString cacert; bool insecure; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index f5d2b594..a7e00086 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -527,9 +527,9 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file #endif // OPENSSL_NO_EC - if (tlsconf.dh_param_file) { + if (!tlsconf.dh_param_file.empty()) { // Read DH parameters from file - auto bio = BIO_new_file(tlsconf.dh_param_file.get(), "r"); + auto bio = BIO_new_file(tlsconf.dh_param_file.c_str(), "r"); if (bio == nullptr) { LOG(FATAL) << "BIO_new_file() failed: " << ERR_error_string(ERR_get_error(), nullptr); From 67804cfc8c4818953dbcaf5ed5d17d49692d5ff3 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:25:24 +0900 Subject: [PATCH 54/85] nghttpx: Use ImmutableString for ciphers --- src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- src/shrpx_ssl.cc | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 24e89d05..d99ca91a 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1804,7 +1804,7 @@ int parse_config(const char *opt, const char *optarg, return 0; } case SHRPX_OPTID_CIPHERS: - mod_config()->tls.ciphers = strcopy(optarg); + mod_config()->tls.ciphers = optarg; return 0; case SHRPX_OPTID_CLIENT: diff --git a/src/shrpx_config.h b/src/shrpx_config.h index dbe8c9a5..d8c63fd4 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -430,7 +430,7 @@ struct TLSConfig { ImmutableString private_key_passwd; ImmutableString cert_file; ImmutableString dh_param_file; - std::unique_ptr ciphers; + ImmutableString ciphers; ImmutableString cacert; bool insecure; bool no_http2_cipher_black_list; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index a7e00086..aff475e6 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -493,8 +493,8 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file SSL_CTX_set_timeout(ssl_ctx, tlsconf.session_timeout.count()); const char *ciphers; - if (tlsconf.ciphers) { - ciphers = tlsconf.ciphers.get(); + if (!tlsconf.ciphers.empty()) { + ciphers = tlsconf.ciphers.c_str(); } else { ciphers = nghttp2::ssl::DEFAULT_CIPHER_LIST; } @@ -683,8 +683,8 @@ SSL_CTX *create_ssl_client_context( SSL_CTX_set_options(ssl_ctx, ssl_opts | tlsconf.tls_proto_mask); const char *ciphers; - if (tlsconf.ciphers) { - ciphers = tlsconf.ciphers.get(); + if (!tlsconf.ciphers.empty()) { + ciphers = tlsconf.ciphers.c_str(); } else { ciphers = nghttp2::ssl::DEFAULT_CIPHER_LIST; } From 9055323b67590adb4cbb941adddc05fddefd1dd2 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:28:47 +0900 Subject: [PATCH 55/85] nghttpx: Use ImmutableString for request_header_file and response_header_file --- src/shrpx.cc | 8 ++++---- src/shrpx_config.cc | 6 ++---- src/shrpx_config.h | 4 ++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 17adf31d..71af4447 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1986,8 +1986,8 @@ void process_options( { auto &dumpconf = http2conf.upstream.debug.dump; - if (dumpconf.request_header_file) { - auto path = dumpconf.request_header_file.get(); + if (!dumpconf.request_header_file.empty()) { + auto path = dumpconf.request_header_file.c_str(); auto f = open_file_for_write(path); if (f == nullptr) { @@ -2007,8 +2007,8 @@ void process_options( } } - if (dumpconf.response_header_file) { - auto path = dumpconf.response_header_file.get(); + if (!dumpconf.response_header_file.empty()) { + auto path = dumpconf.response_header_file.c_str(); auto f = open_file_for_write(path); if (f == nullptr) { diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index d99ca91a..c16a6132 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1919,13 +1919,11 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER: - mod_config()->http2.upstream.debug.dump.request_header_file = - strcopy(optarg); + mod_config()->http2.upstream.debug.dump.request_header_file = optarg; return 0; case SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER: - mod_config()->http2.upstream.debug.dump.response_header_file = - strcopy(optarg); + mod_config()->http2.upstream.debug.dump.response_header_file = optarg; return 0; case SHRPX_OPTID_HTTP2_NO_COOKIE_CRUMBLING: diff --git a/src/shrpx_config.h b/src/shrpx_config.h index d8c63fd4..93d55ade 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -473,8 +473,8 @@ struct Http2Config { struct { struct { struct { - std::unique_ptr request_header_file; - std::unique_ptr response_header_file; + ImmutableString request_header_file; + ImmutableString response_header_file; FILE *request_header; FILE *response_header; } dump; From 2b707bff27dc377ef8f0a8d60ffc4c09edadf815 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:32:27 +0900 Subject: [PATCH 56/85] nghttpx: Use ImmutableString for log file --- src/shrpx.cc | 4 ++-- src/shrpx_config.cc | 4 ++-- src/shrpx_config.h | 4 ++-- src/shrpx_log.cc | 14 +++++++------- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 71af4447..a914a656 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1121,7 +1121,7 @@ void fill_default_config() { accessconf.format = parse_log_format(DEFAULT_ACCESSLOG_FORMAT); auto &errorconf = loggingconf.error; - errorconf.file = strcopy("/dev/stderr"); + errorconf.file = "/dev/stderr"; } loggingconf.syslog_facility = LOG_DAEMON; @@ -1752,7 +1752,7 @@ Logging: Set path to write error log. To reopen file, send USR1 signal to nghttpx. stderr will be redirected to the error log file unless --errorlog-syslog is used. - Default: )" << get_config()->logging.error.file.get() << R"( + Default: )" << get_config()->logging.error.file << R"( --errorlog-syslog Send error log to syslog. If this option is used, --errorlog-file option is ignored. diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index c16a6132..2c092148 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1622,7 +1622,7 @@ int parse_config(const char *opt, const char *optarg, return parse_duration(&mod_config()->http2.timeout.stream_write, opt, optarg); case SHRPX_OPTID_ACCESSLOG_FILE: - mod_config()->logging.access.file = strcopy(optarg); + mod_config()->logging.access.file = optarg; return 0; case SHRPX_OPTID_ACCESSLOG_SYSLOG: @@ -1634,7 +1634,7 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_ERRORLOG_FILE: - mod_config()->logging.error.file = strcopy(optarg); + mod_config()->logging.error.file = optarg; return 0; case SHRPX_OPTID_ERRORLOG_SYSLOG: diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 93d55ade..04faed86 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -504,12 +504,12 @@ struct Http2Config { struct LoggingConfig { struct { std::vector format; - std::unique_ptr file; + ImmutableString file; // Send accesslog to syslog, ignoring accesslog_file. bool syslog; } access; struct { - std::unique_ptr file; + ImmutableString file; // Send errorlog to syslog, ignoring errorlog_file. bool syslog; } error; diff --git a/src/shrpx_log.cc b/src/shrpx_log.cc index 261b02e5..ae699e8b 100644 --- a/src/shrpx_log.cc +++ b/src/shrpx_log.cc @@ -393,23 +393,23 @@ int reopen_log_files() { auto &accessconf = get_config()->logging.access; auto &errorconf = get_config()->logging.error; - if (!accessconf.syslog && accessconf.file) { - new_accesslog_fd = util::open_log_file(accessconf.file.get()); + if (!accessconf.syslog && !accessconf.file.empty()) { + new_accesslog_fd = util::open_log_file(accessconf.file.c_str()); if (new_accesslog_fd == -1) { - LOG(ERROR) << "Failed to open accesslog file " << accessconf.file.get(); + LOG(ERROR) << "Failed to open accesslog file " << accessconf.file; res = -1; } } - if (!errorconf.syslog && errorconf.file) { - new_errorlog_fd = util::open_log_file(errorconf.file.get()); + if (!errorconf.syslog && !errorconf.file.empty()) { + new_errorlog_fd = util::open_log_file(errorconf.file.c_str()); if (new_errorlog_fd == -1) { if (lgconf->errorlog_fd != -1) { - LOG(ERROR) << "Failed to open errorlog file " << errorconf.file.get(); + LOG(ERROR) << "Failed to open errorlog file " << errorconf.file; } else { - std::cerr << "Failed to open errorlog file " << errorconf.file.get() + std::cerr << "Failed to open errorlog file " << errorconf.file << std::endl; } From 76a425226ffcfdd88d8b0009b2cd847fc16e65e2 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 21:34:33 +0900 Subject: [PATCH 57/85] nghttpx: Use ImmutableString for pid_file --- src/shrpx.cc | 10 +++++----- src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index a914a656..8508da4d 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -199,18 +199,18 @@ int chown_to_running_user(const char *path) { namespace { void save_pid() { - std::ofstream out(get_config()->pid_file.get(), std::ios::binary); + std::ofstream out(get_config()->pid_file.c_str(), std::ios::binary); out << get_config()->pid << "\n"; out.close(); if (!out) { - LOG(ERROR) << "Could not save PID to file " << get_config()->pid_file.get(); + LOG(ERROR) << "Could not save PID to file " << get_config()->pid_file; exit(EXIT_FAILURE); } if (get_config()->uid != 0) { - if (chown_to_running_user(get_config()->pid_file.get()) == -1) { + if (chown_to_running_user(get_config()->pid_file.c_str()) == -1) { auto error = errno; - LOG(WARN) << "Changing owner of pid file " << get_config()->pid_file.get() + LOG(WARN) << "Changing owner of pid file " << get_config()->pid_file << " failed: " << strerror(error); } } @@ -946,7 +946,7 @@ int event_loop() { redirect_stderr_to_errorlog(); } - if (get_config()->pid_file) { + if (!get_config()->pid_file.empty()) { save_pid(); } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 2c092148..8d23b9ac 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1728,7 +1728,7 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_PID_FILE: - mod_config()->pid_file = strcopy(optarg); + mod_config()->pid_file = optarg; return 0; case SHRPX_OPTID_USER: { diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 04faed86..3caa7aeb 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -581,7 +581,7 @@ struct Config { TLSConfig tls; LoggingConfig logging; ConnectionConfig conn; - std::unique_ptr pid_file; + ImmutableString pid_file; std::unique_ptr conf_path; std::unique_ptr user; std::unique_ptr mruby_file; From 466e4b7a1e171b7b183922b7e8a27a61ca3a293d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 22:20:10 +0900 Subject: [PATCH 58/85] nghttpx: Use ImmutableString for conf_path --- src/shrpx.cc | 12 ++++++------ src/shrpx_config.h | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/shrpx.cc b/src/shrpx.cc index 8508da4d..0c1fb3f7 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1040,7 +1040,7 @@ void fill_default_config() { *mod_config() = {}; mod_config()->num_worker = 1; - mod_config()->conf_path = strcopy("/etc/nghttpx/nghttpx.conf"); + mod_config()->conf_path = "/etc/nghttpx/nghttpx.conf"; mod_config()->pid = getpid(); auto &tlsconf = mod_config()->tls; @@ -1893,7 +1893,7 @@ Scripting: Misc: --conf= Load configuration from . - Default: )" << get_config()->conf_path.get() << R"( + Default: )" << get_config()->conf_path << R"( --include= Load additional configurations from . File is read when configuration parser encountered this @@ -1919,11 +1919,11 @@ namespace { void process_options( int argc, char **argv, std::vector> &cmdcfgs) { - if (conf_exists(get_config()->conf_path.get())) { + if (conf_exists(get_config()->conf_path.c_str())) { std::set include_set; - if (load_config(get_config()->conf_path.get(), include_set) == -1) { + if (load_config(get_config()->conf_path.c_str(), include_set) == -1) { LOG(FATAL) << "Failed to load configuration from " - << get_config()->conf_path.get(); + << get_config()->conf_path; exit(EXIT_FAILURE); } assert(include_set.empty()); @@ -2563,7 +2563,7 @@ int main(int argc, char **argv) { break; case 12: // --conf - mod_config()->conf_path = strcopy(optarg); + mod_config()->conf_path = optarg; break; case 14: // --syslog-facility diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 3caa7aeb..c7a3ef59 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -582,7 +582,7 @@ struct Config { LoggingConfig logging; ConnectionConfig conn; ImmutableString pid_file; - std::unique_ptr conf_path; + ImmutableString conf_path; std::unique_ptr user; std::unique_ptr mruby_file; char **original_argv; From 7aabc6b125279fea2a60c23f329651fe5a323a58 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 22:21:55 +0900 Subject: [PATCH 59/85] nghttpx: Use ImmutableString for user --- src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- src/shrpx_worker_process.cc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 8d23b9ac..1e023d18 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -1738,7 +1738,7 @@ int parse_config(const char *opt, const char *optarg, << strerror(errno); return -1; } - mod_config()->user = strcopy(pwd->pw_name); + mod_config()->user = pwd->pw_name; mod_config()->uid = pwd->pw_uid; mod_config()->gid = pwd->pw_gid; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index c7a3ef59..03b3700b 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -583,7 +583,7 @@ struct Config { ConnectionConfig conn; ImmutableString pid_file; ImmutableString conf_path; - std::unique_ptr user; + ImmutableString user; std::unique_ptr mruby_file; char **original_argv; char **argv; diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc index f7dacc7c..8d09b5d2 100644 --- a/src/shrpx_worker_process.cc +++ b/src/shrpx_worker_process.cc @@ -64,7 +64,7 @@ void drop_privileges( #endif // HAVE_NEVERBLEED ) { if (getuid() == 0 && get_config()->uid != 0) { - if (initgroups(get_config()->user.get(), get_config()->gid) != 0) { + if (initgroups(get_config()->user.c_str(), get_config()->gid) != 0) { auto error = errno; LOG(FATAL) << "Could not change supplementary groups: " << strerror(error); @@ -86,7 +86,7 @@ void drop_privileges( } #ifdef HAVE_NEVERBLEED if (nb) { - neverbleed_setuidgid(nb, get_config()->user.get(), 1); + neverbleed_setuidgid(nb, get_config()->user.c_str(), 1); } #endif // HAVE_NEVERBLEED } From aa3373a107a2e9a96afe10929db4773a88745f90 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 14 Feb 2016 22:27:59 +0900 Subject: [PATCH 60/85] nghttpx: Use ImmutableString for mruby_file --- src/shrpx_config.cc | 2 +- src/shrpx_config.h | 2 +- src/shrpx_mruby.cc | 11 +++++------ src/shrpx_mruby.h | 6 ++++-- src/shrpx_worker.cc | 3 +-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 1e023d18..4d95ffe2 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -2205,7 +2205,7 @@ int parse_config(const char *opt, const char *optarg, case SHRPX_OPTID_MRUBY_FILE: #ifdef HAVE_MRUBY - mod_config()->mruby_file = strcopy(optarg); + mod_config()->mruby_file = optarg; #else // !HAVE_MRUBY LOG(WARN) << opt << ": ignored because mruby support is disabled at build time."; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 03b3700b..80f0e06e 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -584,7 +584,7 @@ struct Config { ImmutableString pid_file; ImmutableString conf_path; ImmutableString user; - std::unique_ptr mruby_file; + ImmutableString mruby_file; char **original_argv; char **argv; char *cwd; diff --git a/src/shrpx_mruby.cc b/src/shrpx_mruby.cc index 5b1b3967..bd57c146 100644 --- a/src/shrpx_mruby.cc +++ b/src/shrpx_mruby.cc @@ -31,7 +31,6 @@ #include "shrpx_config.h" #include "shrpx_mruby_module.h" #include "shrpx_downstream_connection.h" -#include "template.h" namespace shrpx { @@ -146,12 +145,12 @@ mrb_value instantiate_app(mrb_state *mrb, RProc *proc) { // very hard to write these kind of code because mruby has almost no // documentation aobut compiling or generating code, at least at the // time of this writing. -RProc *compile(mrb_state *mrb, const char *filename) { - if (filename == nullptr) { +RProc *compile(mrb_state *mrb, const StringRef &filename) { + if (filename.empty()) { return nullptr; } - auto infile = fopen(filename, "rb"); + auto infile = fopen(filename.c_str(), "rb"); if (infile == nullptr) { return nullptr; } @@ -185,8 +184,8 @@ RProc *compile(mrb_state *mrb, const char *filename) { return proc; } -std::unique_ptr create_mruby_context(const char *filename) { - if (!filename) { +std::unique_ptr create_mruby_context(const StringRef &filename) { + if (filename.empty()) { return make_unique(nullptr, mrb_nil_value(), mrb_nil_value()); } diff --git a/src/shrpx_mruby.h b/src/shrpx_mruby.h index 95f0430c..55a18351 100644 --- a/src/shrpx_mruby.h +++ b/src/shrpx_mruby.h @@ -32,6 +32,8 @@ #include #include +#include "template.h" + using namespace nghttp2; namespace shrpx { @@ -69,9 +71,9 @@ struct MRubyAssocData { bool response_headers_dirty; }; -RProc *compile(mrb_state *mrb, const char *filename); +RProc *compile(mrb_state *mrb, const StringRef &filename); -std::unique_ptr create_mruby_context(const char *filename); +std::unique_ptr create_mruby_context(const StringRef &filename); // Return interned |ptr|. mrb_sym intern_ptr(mrb_state *mrb, void *ptr); diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 83cfff9b..604f9058 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -290,8 +290,7 @@ std::mt19937 &Worker::get_randgen() { return randgen_; } #ifdef HAVE_MRUBY int Worker::create_mruby_context() { - auto mruby_file = get_config()->mruby_file.get(); - mruby_ctx_ = mruby::create_mruby_context(mruby_file); + mruby_ctx_ = mruby::create_mruby_context(StringRef{get_config()->mruby_file}); if (!mruby_ctx_) { return -1; } From 63e43bff9983ed95fa3953950aeeff6e8e627457 Mon Sep 17 00:00:00 2001 From: Peter Wu Date: Mon, 15 Feb 2016 10:20:13 +0100 Subject: [PATCH 61/85] tests: remove unused macros Since v0.6.2-7-g1d138ac ("Unify DATA and other frames in nghttp2_outbound_item and save malloc()"), the macros are unused and the builds fails on -Werror=unused-macros. --- tests/nghttp2_session_test.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/nghttp2_session_test.c b/tests/nghttp2_session_test.c index bd07b822..0cdec93f 100644 --- a/tests/nghttp2_session_test.c +++ b/tests/nghttp2_session_test.c @@ -36,10 +36,6 @@ #include "nghttp2_test_helper.h" #include "nghttp2_priority_spec.h" -#define OB_CTRL(ITEM) nghttp2_outbound_item_get_ctrl_frame(ITEM) -#define OB_CTRL_TYPE(ITEM) nghttp2_outbound_item_get_ctrl_frame_type(ITEM) -#define OB_DATA(ITEM) nghttp2_outbound_item_get_data_frame(ITEM) - typedef struct { uint8_t buf[65535]; size_t length; From f0b5a8db8c50e03bde30be96ed6172e220533411 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 17 Feb 2016 00:26:34 +0900 Subject: [PATCH 62/85] Add unistd.h to test initgroups() declaration --- configure.ac | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 985925d7..844695c5 100644 --- a/configure.ac +++ b/configure.ac @@ -654,8 +654,13 @@ AC_CHECK_FUNC([timerfd_create], # For cygwin: we can link initgroups, so AC_CHECK_FUNCS succeeds, but # cygwin disables initgroups due to feature test macro magic with our -# configuration. -AC_CHECK_DECLS([initgroups], [], [], [[#include ]]) +# configuration. FreeBSD declares initgroups() in unistd.h. +AC_CHECK_DECLS([initgroups], [], [], [[ + #ifdef HAVE_UNISTD_H + # include + #endif + #include +]]) # Checks for epoll availability, primarily for examples/tiny-nghttpd AX_HAVE_EPOLL([have_epoll=yes], [have_epoll=no]) From b7159f80b20432d510d43d9c8b57cf3f2b39159b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 18 Feb 2016 23:56:29 +0900 Subject: [PATCH 63/85] Eliminate the possibility of nghttp2_stream.cycle overflow --- lib/nghttp2_stream.c | 42 +++++++++++++++++++++++++++++++----------- lib/nghttp2_stream.h | 4 ++-- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/lib/nghttp2_stream.c b/lib/nghttp2_stream.c index 3e848d8e..70325bc3 100644 --- a/lib/nghttp2_stream.c +++ b/lib/nghttp2_stream.c @@ -30,14 +30,32 @@ #include "nghttp2_session.h" #include "nghttp2_helper.h" +/* Maximum distance between any two stream's cycle in the same + prirority queue. Imagine stream A's cycle is A, and stream B's + cycle is B, and A < B. The cycle is unsigned 32 bit integer, it + may get overflow. Because of how we calculate the next cycle + value, if B - A is less than or equals to + NGHTTP2_MAX_CYCLE_DISTANCE, A and B are in the same scale, in other + words, B is really greater than or equal to A. Otherwise, A is a + result of overflow, and it is actually A > B if we consider that + fact. */ +#define NGHTTP2_MAX_CYCLE_DISTANCE (16384 * 256 + 255) + static int stream_less(const void *lhsx, const void *rhsx) { const nghttp2_stream *lhs, *rhs; lhs = nghttp2_struct_of(lhsx, nghttp2_stream, pq_entry); rhs = nghttp2_struct_of(rhsx, nghttp2_stream, pq_entry); - return lhs->cycle < rhs->cycle || - (lhs->cycle == rhs->cycle && lhs->seq < rhs->seq); + if (lhs->cycle == rhs->cycle) { + return lhs->seq < rhs->seq; + } + + if (lhs->cycle < rhs->cycle) { + return rhs->cycle - lhs->cycle <= NGHTTP2_MAX_CYCLE_DISTANCE; + } + + return lhs->cycle - rhs->cycle > NGHTTP2_MAX_CYCLE_DISTANCE; } void nghttp2_stream_init(nghttp2_stream *stream, int32_t stream_id, @@ -116,14 +134,14 @@ static int stream_subtree_active(nghttp2_stream *stream) { /* * Returns next cycle for |stream|. */ -static void stream_next_cycle(nghttp2_stream *stream, uint64_t last_cycle) { - size_t penalty; +static void stream_next_cycle(nghttp2_stream *stream, uint32_t last_cycle) { + uint32_t penalty; - penalty = - stream->last_writelen * NGHTTP2_MAX_WEIGHT + stream->pending_penalty; + penalty = (uint32_t)stream->last_writelen * NGHTTP2_MAX_WEIGHT + + stream->pending_penalty; stream->cycle = last_cycle + penalty / (uint32_t)stream->weight; - stream->pending_penalty = (uint32_t)(penalty % (uint32_t)stream->weight); + stream->pending_penalty = penalty % (uint32_t)stream->weight; } static int stream_obq_push(nghttp2_stream *dep_stream, nghttp2_stream *stream) { @@ -229,9 +247,9 @@ void nghttp2_stream_reschedule(nghttp2_stream *stream) { void nghttp2_stream_change_weight(nghttp2_stream *stream, int32_t weight) { nghttp2_stream *dep_stream; - uint64_t last_cycle; + uint32_t last_cycle; int32_t old_weight; - size_t wlen_penalty; + uint32_t wlen_penalty; if (stream->weight == weight) { return; @@ -254,7 +272,7 @@ void nghttp2_stream_change_weight(nghttp2_stream *stream, int32_t weight) { nghttp2_pq_remove(&dep_stream->obq, &stream->pq_entry); - wlen_penalty = stream->last_writelen * NGHTTP2_MAX_WEIGHT; + wlen_penalty = (uint32_t)stream->last_writelen * NGHTTP2_MAX_WEIGHT; /* Compute old stream->pending_penalty we used to calculate stream->cycle */ @@ -270,7 +288,9 @@ void nghttp2_stream_change_weight(nghttp2_stream *stream, int32_t weight) { place */ stream_next_cycle(stream, last_cycle); - if (stream->cycle < dep_stream->descendant_last_cycle) { + if (stream->cycle < dep_stream->descendant_last_cycle && + (dep_stream->descendant_last_cycle - stream->cycle) <= + NGHTTP2_MAX_CYCLE_DISTANCE) { stream->cycle = dep_stream->descendant_last_cycle; } diff --git a/lib/nghttp2_stream.h b/lib/nghttp2_stream.h index b5d07592..da0e5d53 100644 --- a/lib/nghttp2_stream.h +++ b/lib/nghttp2_stream.h @@ -147,9 +147,9 @@ struct nghttp2_stream { /* Received body so far */ int64_t recv_content_length; /* Base last_cycle for direct descendent streams. */ - uint64_t descendant_last_cycle; + uint32_t descendant_last_cycle; /* Next scheduled time to sent item */ - uint64_t cycle; + uint32_t cycle; /* Next seq used for direct descendant streams */ uint64_t descendant_next_seq; /* Secondary key for prioritization to break a tie for cycle. This From 7921029e33b6270b488e391fc5e860b5cd36194b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 20 Feb 2016 00:48:18 +0900 Subject: [PATCH 64/85] Tokenize extra HTTP header fields The extra HTTP header fields are compiled from https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers, https://en.wikipedia.org/wiki/List_of_HTTP_header_fields, and https://www.owasp.org/index.php/List_of_useful_HTTP_headers. --- genlibtokenlookup.py | 60 ++++++++- lib/nghttp2_hd.c | 306 ++++++++++++++++++++++++++++++++++++++++++- lib/nghttp2_hd.h | 60 ++++++++- 3 files changed, 417 insertions(+), 9 deletions(-) diff --git a/genlibtokenlookup.py b/genlibtokenlookup.py index d7ac7f47..625e62d1 100755 --- a/genlibtokenlookup.py +++ b/genlibtokenlookup.py @@ -62,11 +62,67 @@ HEADERS = [ ('vary', 58), ('via', 59), ('www-authenticate', 60), - ('te', None), + ('accept-ch', None), + ('accept-datetime', None), + ('accept-features', None), + ('accept-patch', None), + ('access-control-allow-credentials', None), + ('access-control-allow-headers', None), + ('access-control-allow-methods', None), + ('access-control-expose-headers', None), + ('access-control-max-age', None), + ('access-control-request-headers', None), + ('access-control-request-method', None), + ('alt-svc', None), + ('alternates', None), ('connection', None), - ('keep-alive',None), + ('content-md5', None), + ('content-security-policy', None), + ('content-security-policy-report-only', None), + ('dnt', None), + ('forwarded', None), + ('front-end-https', None), + ('keep-alive', None), + ('last-event-id', None), + ('negotiate', None), + ('origin', None), + ('p3p', None), + ('pragma', None), ('proxy-connection', None), + ('public-key-pins', None), + ('sec-websocket-extensions', None), + ('sec-websocket-key', None), + ('sec-websocket-origin', None), + ('sec-websocket-protocol', None), + ('sec-websocket-version', None), + ('set-cookie2', None), + ('status', None), + ('tcn', None), + ('te', None), + ('trailer', None), + ('tsv', None), ('upgrade', None), + ('upgrade-insecure-requests', None), + ('variant-vary', None), + ('warning', None), + ('x-api-version', None), + ('x-att-deviceid', None), + ('x-cache', None), + ('x-cache-lookup', None), + ('x-content-duration', None), + ('x-content-security-policy', None), + ('x-content-type-options', None), + ('x-dnsprefetch-control', None), + ('x-forwarded-for', None), + ('x-forwarded-host', None), + ('x-forwarded-proto', None), + ('x-frame-options', None), + ('x-powered-by', None), + ('x-requested-with', None), + ('x-ua-compatible', None), + ('x-wap-profile', None), + ('x-webkit-csp', None), + ('x-xss-protection', None), ] def to_enum_hd(k): diff --git a/lib/nghttp2_hd.c b/lib/nghttp2_hd.c index 5a02c0f4..65a09a5b 100644 --- a/lib/nghttp2_hd.c +++ b/lib/nghttp2_hd.c @@ -137,6 +137,26 @@ static int lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_AGE; } break; + case 'n': + if (lstreq("tc", name, 2)) { + return NGHTTP2_TOKEN_TCN; + } + break; + case 'p': + if (lstreq("p3", name, 2)) { + return NGHTTP2_TOKEN_P3P; + } + break; + case 't': + if (lstreq("dn", name, 2)) { + return NGHTTP2_TOKEN_DNT; + } + break; + case 'v': + if (lstreq("ts", name, 2)) { + return NGHTTP2_TOKEN_TSV; + } + break; } break; case 4: @@ -197,16 +217,31 @@ static int lookup_token(const uint8_t *name, size_t namelen) { break; case 6: switch (name[5]) { + case 'a': + if (lstreq("pragm", name, 5)) { + return NGHTTP2_TOKEN_PRAGMA; + } + break; case 'e': if (lstreq("cooki", name, 5)) { return NGHTTP2_TOKEN_COOKIE; } break; + case 'n': + if (lstreq("origi", name, 5)) { + return NGHTTP2_TOKEN_ORIGIN; + } + break; case 'r': if (lstreq("serve", name, 5)) { return NGHTTP2_TOKEN_SERVER; } break; + case 's': + if (lstreq("statu", name, 5)) { + return NGHTTP2_TOKEN_STATUS; + } + break; case 't': if (lstreq("accep", name, 5)) { return NGHTTP2_TOKEN_ACCEPT; @@ -219,6 +254,11 @@ static int lookup_token(const uint8_t *name, size_t namelen) { break; case 7: switch (name[6]) { + case 'c': + if (lstreq("alt-sv", name, 6)) { + return NGHTTP2_TOKEN_ALT_SVC; + } + break; case 'd': if (lstreq(":metho", name, 6)) { return NGHTTP2_TOKEN__METHOD; @@ -237,6 +277,14 @@ static int lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("upgrad", name, 6)) { return NGHTTP2_TOKEN_UPGRADE; } + if (lstreq("x-cach", name, 6)) { + return NGHTTP2_TOKEN_X_CACHE; + } + break; + case 'g': + if (lstreq("warnin", name, 6)) { + return NGHTTP2_TOKEN_WARNING; + } break; case 'h': if (lstreq("refres", name, 6)) { @@ -247,6 +295,9 @@ static int lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("refere", name, 6)) { return NGHTTP2_TOKEN_REFERER; } + if (lstreq("traile", name, 6)) { + return NGHTTP2_TOKEN_TRAILER; + } break; case 's': if (lstreq(":statu", name, 6)) { @@ -295,6 +346,25 @@ static int lookup_token(const uint8_t *name, size_t namelen) { break; } break; + case 9: + switch (name[8]) { + case 'd': + if (lstreq("forwarde", name, 8)) { + return NGHTTP2_TOKEN_FORWARDED; + } + break; + case 'e': + if (lstreq("negotiat", name, 8)) { + return NGHTTP2_TOKEN_NEGOTIATE; + } + break; + case 'h': + if (lstreq("accept-c", name, 8)) { + return NGHTTP2_TOKEN_ACCEPT_CH; + } + break; + } + break; case 10: switch (name[9]) { case 'e': @@ -310,6 +380,11 @@ static int lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_CONNECTION; } break; + case 's': + if (lstreq("alternate", name, 9)) { + return NGHTTP2_TOKEN_ALTERNATES; + } + break; case 't': if (lstreq("user-agen", name, 9)) { return NGHTTP2_TOKEN_USER_AGENT; @@ -324,6 +399,16 @@ static int lookup_token(const uint8_t *name, size_t namelen) { break; case 11: switch (name[10]) { + case '2': + if (lstreq("set-cookie", name, 10)) { + return NGHTTP2_TOKEN_SET_COOKIE2; + } + break; + case '5': + if (lstreq("content-md", name, 10)) { + return NGHTTP2_TOKEN_CONTENT_MD5; + } + break; case 'r': if (lstreq("retry-afte", name, 10)) { return NGHTTP2_TOKEN_RETRY_AFTER; @@ -338,16 +423,37 @@ static int lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_CONTENT_TYPE; } break; + case 'h': + if (lstreq("accept-patc", name, 11)) { + return NGHTTP2_TOKEN_ACCEPT_PATCH; + } + break; + case 'p': + if (lstreq("x-webkit-cs", name, 11)) { + return NGHTTP2_TOKEN_X_WEBKIT_CSP; + } + break; case 's': if (lstreq("max-forward", name, 11)) { return NGHTTP2_TOKEN_MAX_FORWARDS; } break; + case 'y': + if (lstreq("variant-var", name, 11)) { + return NGHTTP2_TOKEN_VARIANT_VARY; + } + if (lstreq("x-powered-b", name, 11)) { + return NGHTTP2_TOKEN_X_POWERED_BY; + } + break; } break; case 13: switch (name[12]) { case 'd': + if (lstreq("last-event-i", name, 12)) { + return NGHTTP2_TOKEN_LAST_EVENT_ID; + } if (lstreq("last-modifie", name, 12)) { return NGHTTP2_TOKEN_LAST_MODIFIED; } @@ -356,6 +462,9 @@ static int lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("content-rang", name, 12)) { return NGHTTP2_TOKEN_CONTENT_RANGE; } + if (lstreq("x-wap-profil", name, 12)) { + return NGHTTP2_TOKEN_X_WAP_PROFILE; + } break; case 'h': if (lstreq("if-none-matc", name, 12)) { @@ -371,6 +480,9 @@ static int lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("authorizatio", name, 12)) { return NGHTTP2_TOKEN_AUTHORIZATION; } + if (lstreq("x-api-versio", name, 12)) { + return NGHTTP2_TOKEN_X_API_VERSION; + } break; case 's': if (lstreq("accept-range", name, 12)) { @@ -381,11 +493,21 @@ static int lookup_token(const uint8_t *name, size_t namelen) { break; case 14: switch (name[13]) { + case 'd': + if (lstreq("x-att-devicei", name, 13)) { + return NGHTTP2_TOKEN_X_ATT_DEVICEID; + } + break; case 'h': if (lstreq("content-lengt", name, 13)) { return NGHTTP2_TOKEN_CONTENT_LENGTH; } break; + case 'p': + if (lstreq("x-cache-looku", name, 13)) { + return NGHTTP2_TOKEN_X_CACHE_LOOKUP; + } + break; case 't': if (lstreq("accept-charse", name, 13)) { return NGHTTP2_TOKEN_ACCEPT_CHARSET; @@ -396,15 +518,40 @@ static int lookup_token(const uint8_t *name, size_t namelen) { case 15: switch (name[14]) { case 'e': + if (lstreq("accept-datetim", name, 14)) { + return NGHTTP2_TOKEN_ACCEPT_DATETIME; + } if (lstreq("accept-languag", name, 14)) { return NGHTTP2_TOKEN_ACCEPT_LANGUAGE; } + if (lstreq("x-ua-compatibl", name, 14)) { + return NGHTTP2_TOKEN_X_UA_COMPATIBLE; + } break; case 'g': if (lstreq("accept-encodin", name, 14)) { return NGHTTP2_TOKEN_ACCEPT_ENCODING; } break; + case 'r': + if (lstreq("x-forwarded-fo", name, 14)) { + return NGHTTP2_TOKEN_X_FORWARDED_FOR; + } + break; + case 's': + if (lstreq("accept-feature", name, 14)) { + return NGHTTP2_TOKEN_ACCEPT_FEATURES; + } + if (lstreq("front-end-http", name, 14)) { + return NGHTTP2_TOKEN_FRONT_END_HTTPS; + } + if (lstreq("public-key-pin", name, 14)) { + return NGHTTP2_TOKEN_PUBLIC_KEY_PINS; + } + if (lstreq("x-frame-option", name, 14)) { + return NGHTTP2_TOKEN_X_FRAME_OPTIONS; + } + break; } break; case 16: @@ -422,6 +569,11 @@ static int lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_CONTENT_ENCODING; } break; + case 'h': + if (lstreq("x-requested-wit", name, 15)) { + return NGHTTP2_TOKEN_X_REQUESTED_WITH; + } + break; case 'n': if (lstreq("content-locatio", name, 15)) { return NGHTTP2_TOKEN_CONTENT_LOCATION; @@ -429,6 +581,14 @@ static int lookup_token(const uint8_t *name, size_t namelen) { if (lstreq("proxy-connectio", name, 15)) { return NGHTTP2_TOKEN_PROXY_CONNECTION; } + if (lstreq("x-xss-protectio", name, 15)) { + return NGHTTP2_TOKEN_X_XSS_PROTECTION; + } + break; + case 't': + if (lstreq("x-forwarded-hos", name, 15)) { + return NGHTTP2_TOKEN_X_FORWARDED_HOST; + } break; } break; @@ -444,6 +604,16 @@ static int lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_TRANSFER_ENCODING; } break; + case 'o': + if (lstreq("x-forwarded-prot", name, 16)) { + return NGHTTP2_TOKEN_X_FORWARDED_PROTO; + } + break; + case 'y': + if (lstreq("sec-websocket-ke", name, 16)) { + return NGHTTP2_TOKEN_SEC_WEBSOCKET_KEY; + } + break; } break; case 18: @@ -453,6 +623,11 @@ static int lookup_token(const uint8_t *name, size_t namelen) { return NGHTTP2_TOKEN_PROXY_AUTHENTICATE; } break; + case 'n': + if (lstreq("x-content-duratio", name, 17)) { + return NGHTTP2_TOKEN_X_CONTENT_DURATION; + } + break; } break; case 19: @@ -472,12 +647,80 @@ static int lookup_token(const uint8_t *name, size_t namelen) { break; } break; + case 20: + switch (name[19]) { + case 'n': + if (lstreq("sec-websocket-origi", name, 19)) { + return NGHTTP2_TOKEN_SEC_WEBSOCKET_ORIGIN; + } + break; + } + break; + case 21: + switch (name[20]) { + case 'l': + if (lstreq("x-dnsprefetch-contro", name, 20)) { + return NGHTTP2_TOKEN_X_DNSPREFETCH_CONTROL; + } + break; + case 'n': + if (lstreq("sec-websocket-versio", name, 20)) { + return NGHTTP2_TOKEN_SEC_WEBSOCKET_VERSION; + } + break; + } + break; + case 22: + switch (name[21]) { + case 'e': + if (lstreq("access-control-max-ag", name, 21)) { + return NGHTTP2_TOKEN_ACCESS_CONTROL_MAX_AGE; + } + break; + case 'l': + if (lstreq("sec-websocket-protoco", name, 21)) { + return NGHTTP2_TOKEN_SEC_WEBSOCKET_PROTOCOL; + } + break; + case 's': + if (lstreq("x-content-type-option", name, 21)) { + return NGHTTP2_TOKEN_X_CONTENT_TYPE_OPTIONS; + } + break; + } + break; + case 23: + switch (name[22]) { + case 'y': + if (lstreq("content-security-polic", name, 22)) { + return NGHTTP2_TOKEN_CONTENT_SECURITY_POLICY; + } + break; + } + break; + case 24: + switch (name[23]) { + case 's': + if (lstreq("sec-websocket-extension", name, 23)) { + return NGHTTP2_TOKEN_SEC_WEBSOCKET_EXTENSIONS; + } + break; + } + break; case 25: switch (name[24]) { + case 's': + if (lstreq("upgrade-insecure-request", name, 24)) { + return NGHTTP2_TOKEN_UPGRADE_INSECURE_REQUESTS; + } + break; case 'y': if (lstreq("strict-transport-securit", name, 24)) { return NGHTTP2_TOKEN_STRICT_TRANSPORT_SECURITY; } + if (lstreq("x-content-security-polic", name, 24)) { + return NGHTTP2_TOKEN_X_CONTENT_SECURITY_POLICY; + } break; } break; @@ -490,6 +733,59 @@ static int lookup_token(const uint8_t *name, size_t namelen) { break; } break; + case 28: + switch (name[27]) { + case 's': + if (lstreq("access-control-allow-header", name, 27)) { + return NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_HEADERS; + } + if (lstreq("access-control-allow-method", name, 27)) { + return NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_METHODS; + } + break; + } + break; + case 29: + switch (name[28]) { + case 'd': + if (lstreq("access-control-request-metho", name, 28)) { + return NGHTTP2_TOKEN_ACCESS_CONTROL_REQUEST_METHOD; + } + break; + case 's': + if (lstreq("access-control-expose-header", name, 28)) { + return NGHTTP2_TOKEN_ACCESS_CONTROL_EXPOSE_HEADERS; + } + break; + } + break; + case 30: + switch (name[29]) { + case 's': + if (lstreq("access-control-request-header", name, 29)) { + return NGHTTP2_TOKEN_ACCESS_CONTROL_REQUEST_HEADERS; + } + break; + } + break; + case 32: + switch (name[31]) { + case 's': + if (lstreq("access-control-allow-credential", name, 31)) { + return NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_CREDENTIALS; + } + break; + } + break; + case 35: + switch (name[34]) { + case 'y': + if (lstreq("content-security-policy-report-onl", name, 34)) { + return NGHTTP2_TOKEN_CONTENT_SECURITY_POLICY_REPORT_ONLY; + } + break; + } + break; } return -1; } @@ -617,8 +913,8 @@ static nghttp2_hd_entry *hd_map_find(nghttp2_hd_map *map, int *exact_match, *exact_match = 0; for (p = map->table[hash & (HD_MAP_SIZE - 1)]; p; p = p->next) { - if (hash != p->hash || token != p->token || - (token == -1 && !name_eq(&p->nv, nv))) { + if (token != p->token || + (token == -1 && (hash != p->hash || !name_eq(&p->nv, nv)))) { continue; } if (!res) { @@ -1444,7 +1740,7 @@ static int deflate_nv(nghttp2_hd_deflater *deflater, nghttp2_bufs *bufs, int indexing_mode; int token; nghttp2_mem *mem; - uint32_t hash; + uint32_t hash = 0; DEBUGF(fprintf(stderr, "deflatehd: deflating %.*s: %.*s\n", (int)nv->namelen, nv->name, (int)nv->valuelen, nv->value)); @@ -1452,9 +1748,9 @@ static int deflate_nv(nghttp2_hd_deflater *deflater, nghttp2_bufs *bufs, mem = deflater->ctx.mem; token = lookup_token(nv->name, nv->namelen); - if (token == -1 || token > NGHTTP2_TOKEN_WWW_AUTHENTICATE) { + if (token == -1) { hash = name_hash(nv); - } else { + } else if (token <= NGHTTP2_TOKEN_WWW_AUTHENTICATE) { hash = static_table[token].hash; } diff --git a/lib/nghttp2_hd.h b/lib/nghttp2_hd.h index c667888e..adf227be 100644 --- a/lib/nghttp2_hd.h +++ b/lib/nghttp2_hd.h @@ -105,11 +105,67 @@ typedef enum { NGHTTP2_TOKEN_VARY = 58, NGHTTP2_TOKEN_VIA = 59, NGHTTP2_TOKEN_WWW_AUTHENTICATE = 60, - NGHTTP2_TOKEN_TE, + NGHTTP2_TOKEN_ACCEPT_CH, + NGHTTP2_TOKEN_ACCEPT_DATETIME, + NGHTTP2_TOKEN_ACCEPT_FEATURES, + NGHTTP2_TOKEN_ACCEPT_PATCH, + NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_CREDENTIALS, + NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_HEADERS, + NGHTTP2_TOKEN_ACCESS_CONTROL_ALLOW_METHODS, + NGHTTP2_TOKEN_ACCESS_CONTROL_EXPOSE_HEADERS, + NGHTTP2_TOKEN_ACCESS_CONTROL_MAX_AGE, + NGHTTP2_TOKEN_ACCESS_CONTROL_REQUEST_HEADERS, + NGHTTP2_TOKEN_ACCESS_CONTROL_REQUEST_METHOD, + NGHTTP2_TOKEN_ALT_SVC, + NGHTTP2_TOKEN_ALTERNATES, NGHTTP2_TOKEN_CONNECTION, + NGHTTP2_TOKEN_CONTENT_MD5, + NGHTTP2_TOKEN_CONTENT_SECURITY_POLICY, + NGHTTP2_TOKEN_CONTENT_SECURITY_POLICY_REPORT_ONLY, + NGHTTP2_TOKEN_DNT, + NGHTTP2_TOKEN_FORWARDED, + NGHTTP2_TOKEN_FRONT_END_HTTPS, NGHTTP2_TOKEN_KEEP_ALIVE, + NGHTTP2_TOKEN_LAST_EVENT_ID, + NGHTTP2_TOKEN_NEGOTIATE, + NGHTTP2_TOKEN_ORIGIN, + NGHTTP2_TOKEN_P3P, + NGHTTP2_TOKEN_PRAGMA, NGHTTP2_TOKEN_PROXY_CONNECTION, - NGHTTP2_TOKEN_UPGRADE + NGHTTP2_TOKEN_PUBLIC_KEY_PINS, + NGHTTP2_TOKEN_SEC_WEBSOCKET_EXTENSIONS, + NGHTTP2_TOKEN_SEC_WEBSOCKET_KEY, + NGHTTP2_TOKEN_SEC_WEBSOCKET_ORIGIN, + NGHTTP2_TOKEN_SEC_WEBSOCKET_PROTOCOL, + NGHTTP2_TOKEN_SEC_WEBSOCKET_VERSION, + NGHTTP2_TOKEN_SET_COOKIE2, + NGHTTP2_TOKEN_STATUS, + NGHTTP2_TOKEN_TCN, + NGHTTP2_TOKEN_TE, + NGHTTP2_TOKEN_TRAILER, + NGHTTP2_TOKEN_TSV, + NGHTTP2_TOKEN_UPGRADE, + NGHTTP2_TOKEN_UPGRADE_INSECURE_REQUESTS, + NGHTTP2_TOKEN_VARIANT_VARY, + NGHTTP2_TOKEN_WARNING, + NGHTTP2_TOKEN_X_API_VERSION, + NGHTTP2_TOKEN_X_ATT_DEVICEID, + NGHTTP2_TOKEN_X_CACHE, + NGHTTP2_TOKEN_X_CACHE_LOOKUP, + NGHTTP2_TOKEN_X_CONTENT_DURATION, + NGHTTP2_TOKEN_X_CONTENT_SECURITY_POLICY, + NGHTTP2_TOKEN_X_CONTENT_TYPE_OPTIONS, + NGHTTP2_TOKEN_X_DNSPREFETCH_CONTROL, + NGHTTP2_TOKEN_X_FORWARDED_FOR, + NGHTTP2_TOKEN_X_FORWARDED_HOST, + NGHTTP2_TOKEN_X_FORWARDED_PROTO, + NGHTTP2_TOKEN_X_FRAME_OPTIONS, + NGHTTP2_TOKEN_X_POWERED_BY, + NGHTTP2_TOKEN_X_REQUESTED_WITH, + NGHTTP2_TOKEN_X_UA_COMPATIBLE, + NGHTTP2_TOKEN_X_WAP_PROFILE, + NGHTTP2_TOKEN_X_WEBKIT_CSP, + NGHTTP2_TOKEN_X_XSS_PROTECTION, } nghttp2_token; typedef enum { From 17032010845f257a830ec6ba07961c5e75527d5b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 20 Feb 2016 19:02:19 +0900 Subject: [PATCH 65/85] nghttpx: Get rid of hdidx --- src/shrpx_downstream.cc | 81 +++++++++++++------------ src/shrpx_downstream.h | 6 +- src/shrpx_http_downstream_connection.cc | 4 +- src/shrpx_https_upstream.cc | 4 +- src/shrpx_spdy_upstream.cc | 5 +- 5 files changed, 51 insertions(+), 49 deletions(-) diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 8f8165b7..71a0a2b6 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -237,15 +237,15 @@ void Downstream::force_resume_read() { } namespace { -const Headers::value_type *search_header_linear(const Headers &headers, - const StringRef &name) { - const Headers::value_type *res = nullptr; - for (auto &kv : headers) { +const Headers::value_type * +search_header_linear_backwards(const Headers &headers, const StringRef &name) { + for (auto it = headers.rbegin(); it != headers.rend(); ++it) { + auto &kv = *it; if (kv.name == name) { - res = &kv; + return &kv; } } - return res; + return nullptr; } } // namespace @@ -330,10 +330,10 @@ void Downstream::crumble_request_cookie(std::vector &nva) { namespace { void add_header(bool &key_prev, size_t &sum, Headers &headers, std::string name, - std::string value) { + std::string value, int16_t token = -1) { key_prev = true; sum += name.size() + value.size(); - headers.emplace_back(std::move(name), std::move(value)); + headers.emplace_back(std::move(name), std::move(value), false, token); } } // namespace @@ -356,6 +356,8 @@ void append_last_header_key(bool &key_prev, size_t &sum, Headers &headers, sum += len; auto &item = headers.back(); item.name.append(data, len); + util::inp_strlower(item.name); + item.token = http2::lookup_token(item.name); } } // namespace @@ -370,56 +372,58 @@ void append_last_header_value(bool &key_prev, size_t &sum, Headers &headers, } // namespace int FieldStore::index_headers() { - http2::init_hdidx(hdidx_); content_length = -1; - for (size_t i = 0; i < headers_.size(); ++i) { - auto &kv = headers_[i]; - util::inp_strlower(kv.name); - - auto token = http2::lookup_token( - reinterpret_cast(kv.name.c_str()), kv.name.size()); - if (token < 0) { + for (auto &kv : headers_) { + if (kv.token != http2::HD_CONTENT_LENGTH) { continue; } - kv.token = token; - http2::index_header(hdidx_, token, i); - - if (token == http2::HD_CONTENT_LENGTH) { - auto len = util::parse_uint(kv.value); - if (len == -1) { - return -1; - } - if (content_length != -1) { - return -1; - } - content_length = len; + auto len = util::parse_uint(kv.value); + if (len == -1) { + return -1; } + if (content_length != -1) { + return -1; + } + content_length = len; } return 0; } const Headers::value_type *FieldStore::header(int16_t token) const { - return http2::get_header(hdidx_, token, headers_); + for (auto it = headers_.rbegin(); it != headers_.rend(); ++it) { + auto &kv = *it; + if (kv.token == token) { + return &kv; + } + } + return nullptr; } Headers::value_type *FieldStore::header(int16_t token) { - return http2::get_header(hdidx_, token, headers_); + for (auto it = headers_.rbegin(); it != headers_.rend(); ++it) { + auto &kv = *it; + if (kv.token == token) { + return &kv; + } + } + return nullptr; } const Headers::value_type *FieldStore::header(const StringRef &name) const { - return search_header_linear(headers_, name); + return search_header_linear_backwards(headers_, name); } -void FieldStore::add_header(std::string name, std::string value) { +void FieldStore::add_header_lower(std::string name, std::string value) { + util::inp_strlower(name); + auto token = http2::lookup_token(name); shrpx::add_header(header_key_prev_, buffer_size_, headers_, std::move(name), - std::move(value)); + std::move(value), token); } void FieldStore::add_header(std::string name, std::string value, int16_t token) { - http2::index_header(hdidx_, token, headers_.size()); buffer_size_ += name.size() + value.size(); headers_.emplace_back(std::move(name), std::move(value), false, token); } @@ -427,7 +431,6 @@ void FieldStore::add_header(std::string name, std::string value, void FieldStore::add_header(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, int16_t token) { - http2::index_header(hdidx_, token, headers_.size()); shrpx::add_header(buffer_size_, headers_, name, namelen, value, valuelen, no_index, token); } @@ -442,10 +445,7 @@ void FieldStore::append_last_header_value(const char *data, size_t len) { data, len); } -void FieldStore::clear_headers() { - headers_.clear(); - http2::init_hdidx(hdidx_); -} +void FieldStore::clear_headers() { headers_.clear(); } void FieldStore::add_trailer(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, @@ -456,7 +456,8 @@ void FieldStore::add_trailer(const uint8_t *name, size_t namelen, no_index, -1); } -void FieldStore::add_trailer(std::string name, std::string value) { +void FieldStore::add_trailer_lower(std::string name, std::string value) { + util::inp_strlower(name); shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, std::move(name), std::move(value)); } diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index a1f32760..4132e905 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -56,7 +56,6 @@ public: buffer_size_(0), header_key_prev_(false), trailer_key_prev_(false) { - http2::init_hdidx(hdidx_); headers_.reserve(headers_initial_capacity); } @@ -80,7 +79,7 @@ public: // such header is found, returns nullptr. const Headers::value_type *header(const StringRef &name) const; - void add_header(std::string name, std::string value); + void add_header_lower(std::string name, std::string value); void add_header(std::string name, std::string value, int16_t token); void add_header(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, int16_t token); @@ -100,7 +99,7 @@ public: void add_trailer(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, int16_t token); - void add_trailer(std::string name, std::string value); + void add_trailer_lower(std::string name, std::string value); void append_last_trailer_key(const char *data, size_t len); void append_last_trailer_value(const char *data, size_t len); @@ -115,7 +114,6 @@ private: // trailer fields. For HTTP/1.1, trailer fields are only included // with chunked encoding. For HTTP/2, there is no such limit. Headers trailers_; - http2::HeaderIndex hdidx_; // Sum of the length of name and value in headers_ and trailers_. // This could also be increased by add_extra_buffer_size() to take // into account for request URI in case of HTTP/1.x request. diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 8e2796a6..8b96e314 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -691,7 +691,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { if (ensure_max_header_fields(downstream, httpconf) != 0) { return -1; } - resp.fs.add_header(std::string(data, len), ""); + resp.fs.add_header_lower(std::string(data, len), ""); } } else { // trailer part @@ -704,7 +704,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { // wrong place or crash if trailer fields are currently empty. return -1; } - resp.fs.add_trailer(std::string(data, len), ""); + resp.fs.add_trailer_lower(std::string(data, len), ""); } } return 0; diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index c807e753..5875ac1b 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -142,7 +142,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { Downstream::HTTP1_REQUEST_HEADER_TOO_LARGE); return -1; } - req.fs.add_header(std::string(data, len), ""); + req.fs.add_header_lower(std::string(data, len), ""); } } else { // trailer part @@ -156,7 +156,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { } return -1; } - req.fs.add_trailer(std::string(data, len), ""); + req.fs.add_trailer_lower(std::string(data, len), ""); } } return 0; diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index ee80a144..27d768c2 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -188,7 +188,10 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, } for (size_t i = 0; nv[i]; i += 2) { - req.fs.add_header(nv[i], nv[i + 1]); + auto name = std::string(nv[i]); + auto value = std::string(nv[i + 1]); + auto token = http2::lookup_token(name); + req.fs.add_header(std::move(name), std::move(value), token); } if (req.fs.index_headers() != 0) { From 6f1347fc8b19b6419d19a55587651330fe02116b Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 20 Feb 2016 19:13:39 +0900 Subject: [PATCH 66/85] nghttpx: Tokenize trailer field as well so that we can ditch prohibited headers in HTTP/2 --- src/shrpx_downstream.cc | 31 ++++++++++++++----------- src/shrpx_downstream.h | 4 ++-- src/shrpx_http2_session.cc | 6 ++--- src/shrpx_http2_upstream.cc | 6 ++--- src/shrpx_http_downstream_connection.cc | 4 ++-- src/shrpx_https_upstream.cc | 4 ++-- 6 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 71a0a2b6..bd76592e 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -330,7 +330,7 @@ void Downstream::crumble_request_cookie(std::vector &nva) { namespace { void add_header(bool &key_prev, size_t &sum, Headers &headers, std::string name, - std::string value, int16_t token = -1) { + std::string value, int16_t token) { key_prev = true; sum += name.size() + value.size(); headers.emplace_back(std::move(name), std::move(value), false, token); @@ -415,11 +415,13 @@ const Headers::value_type *FieldStore::header(const StringRef &name) const { return search_header_linear_backwards(headers_, name); } -void FieldStore::add_header_lower(std::string name, std::string value) { - util::inp_strlower(name); - auto token = http2::lookup_token(name); - shrpx::add_header(header_key_prev_, buffer_size_, headers_, std::move(name), - std::move(value), token); +void FieldStore::add_header_lower(const StringRef &name, + const StringRef &value) { + auto low_name = name.str(); + util::inp_strlower(low_name); + auto token = http2::lookup_token(low_name); + shrpx::add_header(header_key_prev_, buffer_size_, headers_, + std::move(low_name), value.str(), token); } void FieldStore::add_header(std::string name, std::string value, @@ -450,16 +452,19 @@ void FieldStore::clear_headers() { headers_.clear(); } void FieldStore::add_trailer(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, int16_t token) { - // we never index trailer fields. Header size limit should be - // applied to all header and trailer fields combined. + // Header size limit should be applied to all header and trailer + // fields combined. shrpx::add_header(buffer_size_, trailers_, name, namelen, value, valuelen, - no_index, -1); + no_index, token); } -void FieldStore::add_trailer_lower(std::string name, std::string value) { - util::inp_strlower(name); - shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, std::move(name), - std::move(value)); +void FieldStore::add_trailer_lower(const StringRef &name, + const StringRef &value) { + auto low_name = name.str(); + util::inp_strlower(low_name); + auto token = http2::lookup_token(low_name); + shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, + std::move(low_name), value.str(), token); } void FieldStore::append_last_trailer_key(const char *data, size_t len) { diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 4132e905..03286b83 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -79,7 +79,7 @@ public: // such header is found, returns nullptr. const Headers::value_type *header(const StringRef &name) const; - void add_header_lower(std::string name, std::string value); + void add_header_lower(const StringRef &name, const StringRef &value); void add_header(std::string name, std::string value, int16_t token); void add_header(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, int16_t token); @@ -99,7 +99,7 @@ public: void add_trailer(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, int16_t token); - void add_trailer_lower(std::string name, std::string value); + void add_trailer_lower(const StringRef &name, const StringRef &value); void append_last_trailer_key(const char *data, size_t len); void append_last_trailer_value(const char *data, size_t len); diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index f4004b9a..24283172 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -735,15 +735,15 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; } + auto token = http2::lookup_token(name, namelen); + if (trailer) { // just store header fields for trailer part resp.fs.add_trailer(name, namelen, value, valuelen, - flags & NGHTTP2_NV_FLAG_NO_INDEX, -1); + flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } - auto token = http2::lookup_token(name, namelen); - resp.fs.add_header(name, namelen, value, valuelen, flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 7e909a25..f13ea759 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -201,15 +201,15 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, return 0; } + auto token = http2::lookup_token(name, namelen); + if (frame->headers.cat == NGHTTP2_HCAT_HEADERS) { // just store header fields for trailer part req.fs.add_trailer(name, namelen, value, valuelen, - flags & NGHTTP2_NV_FLAG_NO_INDEX, -1); + flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } - auto token = http2::lookup_token(name, namelen); - req.fs.add_header(name, namelen, value, valuelen, flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 8b96e314..0af56ce1 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -691,7 +691,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { if (ensure_max_header_fields(downstream, httpconf) != 0) { return -1; } - resp.fs.add_header_lower(std::string(data, len), ""); + resp.fs.add_header_lower(StringRef{data, len}, StringRef{}); } } else { // trailer part @@ -704,7 +704,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { // wrong place or crash if trailer fields are currently empty. return -1; } - resp.fs.add_trailer_lower(std::string(data, len), ""); + resp.fs.add_trailer_lower(StringRef(data, len), StringRef{}); } } return 0; diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 5875ac1b..7f47b681 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -142,7 +142,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { Downstream::HTTP1_REQUEST_HEADER_TOO_LARGE); return -1; } - req.fs.add_header_lower(std::string(data, len), ""); + req.fs.add_header_lower(StringRef{data, len}, StringRef{}); } } else { // trailer part @@ -156,7 +156,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { } return -1; } - req.fs.add_trailer_lower(std::string(data, len), ""); + req.fs.add_trailer_lower(StringRef{data, len}, StringRef{}); } } return 0; From 3ff148811b54380f56a14d97d69cac2f2b722471 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 20 Feb 2016 20:53:19 +0900 Subject: [PATCH 67/85] nghttpx: Use StringRef for add_hedeader --- src/shrpx_downstream.cc | 24 ++++++++---------------- src/shrpx_downstream.h | 8 ++++---- src/shrpx_http2_session.cc | 7 ++++--- src/shrpx_http2_upstream.cc | 7 ++++--- 4 files changed, 20 insertions(+), 26 deletions(-) diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index bd76592e..0d3b2618 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -338,14 +338,10 @@ void add_header(bool &key_prev, size_t &sum, Headers &headers, std::string name, } // namespace namespace { -void add_header(size_t &sum, Headers &headers, const uint8_t *name, - size_t namelen, const uint8_t *value, size_t valuelen, - bool no_index, int16_t token) { - sum += namelen + valuelen; - headers.emplace_back( - std::string(reinterpret_cast(name), namelen), - std::string(reinterpret_cast(value), valuelen), no_index, - token); +void add_header(size_t &sum, Headers &headers, const StringRef &name, + const StringRef &value, bool no_index, int16_t token) { + sum += name.size() + value.size(); + headers.emplace_back(name.str(), value.str(), no_index, token); } } // namespace @@ -430,11 +426,9 @@ void FieldStore::add_header(std::string name, std::string value, headers_.emplace_back(std::move(name), std::move(value), false, token); } -void FieldStore::add_header(const uint8_t *name, size_t namelen, - const uint8_t *value, size_t valuelen, +void FieldStore::add_header(const StringRef &name, const StringRef &value, bool no_index, int16_t token) { - shrpx::add_header(buffer_size_, headers_, name, namelen, value, valuelen, - no_index, token); + shrpx::add_header(buffer_size_, headers_, name, value, no_index, token); } void FieldStore::append_last_header_key(const char *data, size_t len) { @@ -449,13 +443,11 @@ void FieldStore::append_last_header_value(const char *data, size_t len) { void FieldStore::clear_headers() { headers_.clear(); } -void FieldStore::add_trailer(const uint8_t *name, size_t namelen, - const uint8_t *value, size_t valuelen, +void FieldStore::add_trailer(const StringRef &name, const StringRef &value, bool no_index, int16_t token) { // Header size limit should be applied to all header and trailer // fields combined. - shrpx::add_header(buffer_size_, trailers_, name, namelen, value, valuelen, - no_index, token); + shrpx::add_header(buffer_size_, trailers_, name, value, no_index, token); } void FieldStore::add_trailer_lower(const StringRef &name, diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 03286b83..53778e49 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -81,8 +81,8 @@ public: void add_header_lower(const StringRef &name, const StringRef &value); void add_header(std::string name, std::string value, int16_t token); - void add_header(const uint8_t *name, size_t namelen, const uint8_t *value, - size_t valuelen, bool no_index, int16_t token); + void add_header(const StringRef &name, const StringRef &value, bool no_index, + int16_t token); void append_last_header_key(const char *data, size_t len); void append_last_header_value(const char *data, size_t len); @@ -97,8 +97,8 @@ public: // Empties headers. void clear_headers(); - void add_trailer(const uint8_t *name, size_t namelen, const uint8_t *value, - size_t valuelen, bool no_index, int16_t token); + void add_trailer(const StringRef &name, const StringRef &value, bool no_index, + int16_t token); void add_trailer_lower(const StringRef &name, const StringRef &value); void append_last_trailer_key(const char *data, size_t len); diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 24283172..2c96a579 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -739,12 +739,12 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, if (trailer) { // just store header fields for trailer part - resp.fs.add_trailer(name, namelen, value, valuelen, + resp.fs.add_trailer(StringRef{name, namelen}, StringRef{value, valuelen}, flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } - resp.fs.add_header(name, namelen, value, valuelen, + resp.fs.add_header(StringRef{name, namelen}, StringRef{value, valuelen}, flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } @@ -778,7 +778,8 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, } auto token = http2::lookup_token(name, namelen); - promised_req.fs.add_header(name, namelen, value, valuelen, + promised_req.fs.add_header(StringRef{name, namelen}, + StringRef{value, valuelen}, flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index f13ea759..4506081c 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -205,12 +205,12 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, if (frame->headers.cat == NGHTTP2_HCAT_HEADERS) { // just store header fields for trailer part - req.fs.add_trailer(name, namelen, value, valuelen, + req.fs.add_trailer(StringRef{name, namelen}, StringRef{value, valuelen}, flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } - req.fs.add_header(name, namelen, value, valuelen, + req.fs.add_header(StringRef{name, namelen}, StringRef{value, valuelen}, flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } @@ -593,7 +593,8 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame, req.path = http2::rewrite_clean_path(nv.value, nv.value + nv.valuelen); break; } - req.fs.add_header(nv.name, nv.namelen, nv.value, nv.valuelen, + req.fs.add_header(StringRef{nv.name, nv.namelen}, + StringRef{nv.value, nv.valuelen}, nv.flags & NGHTTP2_NV_FLAG_NO_INDEX, token); } From 23ecfd412d5f4143be50243fbbad81c865d0c302 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 20 Feb 2016 21:41:23 +0900 Subject: [PATCH 68/85] nghttpx: Fix mruby compile error, clean up add_header interface --- src/shrpx-unittest.cc | 4 +- src/shrpx_downstream.cc | 39 +++++------- src/shrpx_downstream.h | 15 +++-- src/shrpx_downstream_test.cc | 85 ++++++++++++++++--------- src/shrpx_downstream_test.h | 2 +- src/shrpx_http2_session.cc | 20 +++--- src/shrpx_http2_upstream.cc | 15 +++-- src/shrpx_http_downstream_connection.cc | 4 +- src/shrpx_https_upstream.cc | 4 +- src/shrpx_mruby.cc | 8 --- src/shrpx_mruby.h | 2 - src/shrpx_mruby_module_request.cc | 25 +++++--- src/shrpx_mruby_module_response.cc | 39 ++++++------ src/shrpx_spdy_upstream.cc | 8 +-- 14 files changed, 146 insertions(+), 124 deletions(-) diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index 4c6b39c8..0e7b17a1 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -106,8 +106,8 @@ int main(int argc, char *argv[]) { shrpx::test_http2_get_pure_path_component) || !CU_add_test(pSuite, "http2_construct_push_component", shrpx::test_http2_construct_push_component) || - !CU_add_test(pSuite, "downstream_field_store_index_headers", - shrpx::test_downstream_field_store_index_headers) || + !CU_add_test(pSuite, "downstream_field_store_add_header_lower", + shrpx::test_downstream_field_store_add_header_lower) || !CU_add_test(pSuite, "downstream_field_store_header", shrpx::test_downstream_field_store_header) || !CU_add_test(pSuite, "downstream_crumble_request_cookie", diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 0d3b2618..11ec9756 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -330,10 +330,10 @@ void Downstream::crumble_request_cookie(std::vector &nva) { namespace { void add_header(bool &key_prev, size_t &sum, Headers &headers, std::string name, - std::string value, int16_t token) { + std::string value, bool no_index, int16_t token) { key_prev = true; sum += name.size() + value.size(); - headers.emplace_back(std::move(name), std::move(value), false, token); + headers.emplace_back(std::move(name), std::move(value), no_index, token); } } // namespace @@ -411,23 +411,17 @@ const Headers::value_type *FieldStore::header(const StringRef &name) const { return search_header_linear_backwards(headers_, name); } -void FieldStore::add_header_lower(const StringRef &name, - const StringRef &value) { +void FieldStore::add_header_lower(const StringRef &name, const StringRef &value, + bool no_index) { auto low_name = name.str(); util::inp_strlower(low_name); auto token = http2::lookup_token(low_name); shrpx::add_header(header_key_prev_, buffer_size_, headers_, - std::move(low_name), value.str(), token); + std::move(low_name), value.str(), no_index, token); } -void FieldStore::add_header(std::string name, std::string value, - int16_t token) { - buffer_size_ += name.size() + value.size(); - headers_.emplace_back(std::move(name), std::move(value), false, token); -} - -void FieldStore::add_header(const StringRef &name, const StringRef &value, - bool no_index, int16_t token) { +void FieldStore::add_header_token(const StringRef &name, const StringRef &value, + bool no_index, int16_t token) { shrpx::add_header(buffer_size_, headers_, name, value, no_index, token); } @@ -443,20 +437,21 @@ void FieldStore::append_last_header_value(const char *data, size_t len) { void FieldStore::clear_headers() { headers_.clear(); } -void FieldStore::add_trailer(const StringRef &name, const StringRef &value, - bool no_index, int16_t token) { - // Header size limit should be applied to all header and trailer - // fields combined. - shrpx::add_header(buffer_size_, trailers_, name, value, no_index, token); -} - void FieldStore::add_trailer_lower(const StringRef &name, - const StringRef &value) { + const StringRef &value, bool no_index) { auto low_name = name.str(); util::inp_strlower(low_name); auto token = http2::lookup_token(low_name); shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, - std::move(low_name), value.str(), token); + std::move(low_name), value.str(), no_index, token); +} + +void FieldStore::add_trailer_token(const StringRef &name, + const StringRef &value, bool no_index, + int16_t token) { + // Header size limit should be applied to all header and trailer + // fields combined. + shrpx::add_header(buffer_size_, trailers_, name, value, no_index, token); } void FieldStore::append_last_trailer_key(const char *data, size_t len) { diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 53778e49..8a855b1c 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -79,10 +79,10 @@ public: // such header is found, returns nullptr. const Headers::value_type *header(const StringRef &name) const; - void add_header_lower(const StringRef &name, const StringRef &value); - void add_header(std::string name, std::string value, int16_t token); - void add_header(const StringRef &name, const StringRef &value, bool no_index, - int16_t token); + void add_header_lower(const StringRef &name, const StringRef &value, + bool no_index); + void add_header_token(const StringRef &name, const StringRef &value, + bool no_index, int16_t token); void append_last_header_key(const char *data, size_t len); void append_last_header_value(const char *data, size_t len); @@ -97,9 +97,10 @@ public: // Empties headers. void clear_headers(); - void add_trailer(const StringRef &name, const StringRef &value, bool no_index, - int16_t token); - void add_trailer_lower(const StringRef &name, const StringRef &value); + void add_trailer_lower(const StringRef &name, const StringRef &value, + bool no_index); + void add_trailer_token(const StringRef &name, const StringRef &value, + bool no_index, int16_t token); void append_last_trailer_key(const char *data, size_t len); void append_last_trailer_value(const char *data, size_t len); diff --git a/src/shrpx_downstream_test.cc b/src/shrpx_downstream_test.cc index c7b559a4..5b496f9b 100644 --- a/src/shrpx_downstream_test.cc +++ b/src/shrpx_downstream_test.cc @@ -32,17 +32,24 @@ namespace shrpx { -void test_downstream_field_store_index_headers(void) { +void test_downstream_field_store_add_header_lower(void) { FieldStore fs(0); - fs.add_header("1", "0"); - fs.add_header("2", "1"); - fs.add_header("Charlie", "2"); - fs.add_header("Alpha", "3"); - fs.add_header("Delta", "4"); - fs.add_header("BravO", "5"); - fs.add_header(":method", "6"); - fs.add_header(":authority", "7"); - fs.index_headers(); + fs.add_header_lower(StringRef::from_lit("1"), StringRef::from_lit("0"), + false); + fs.add_header_lower(StringRef::from_lit("2"), StringRef::from_lit("1"), + false); + fs.add_header_lower(StringRef::from_lit("Charlie"), StringRef::from_lit("2"), + false); + fs.add_header_lower(StringRef::from_lit("Alpha"), StringRef::from_lit("3"), + false); + fs.add_header_lower(StringRef::from_lit("Delta"), StringRef::from_lit("4"), + false); + fs.add_header_lower(StringRef::from_lit("BravO"), StringRef::from_lit("5"), + false); + fs.add_header_lower(StringRef::from_lit(":method"), StringRef::from_lit("6"), + false); + fs.add_header_lower(StringRef::from_lit(":authority"), + StringRef::from_lit("7"), false); auto ans = Headers{{"1", "0"}, {"2", "1"}, @@ -57,10 +64,13 @@ void test_downstream_field_store_index_headers(void) { void test_downstream_field_store_header(void) { FieldStore fs(0); - fs.add_header("alpha", "0"); - fs.add_header(":authority", "1"); - fs.add_header("content-length", "2"); - fs.index_headers(); + fs.add_header_token(StringRef::from_lit("alpha"), StringRef::from_lit("0"), + false, -1); + fs.add_header_token(StringRef::from_lit(":authority"), + StringRef::from_lit("1"), false, http2::HD__AUTHORITY); + fs.add_header_token(StringRef::from_lit("content-length"), + StringRef::from_lit("2"), false, + http2::HD_CONTENT_LENGTH); // By token CU_ASSERT(Header(":authority", "1") == *fs.header(http2::HD__AUTHORITY)); @@ -74,14 +84,18 @@ void test_downstream_field_store_header(void) { void test_downstream_crumble_request_cookie(void) { Downstream d(nullptr, nullptr, 0); auto &req = d.request(); - req.fs.add_header(":method", "get"); - req.fs.add_header(":path", "/"); - auto val = "alpha; bravo; ; ;; charlie;;"; - req.fs.add_header( - reinterpret_cast("cookie"), sizeof("cookie") - 1, - reinterpret_cast(val), strlen(val), true, -1); - req.fs.add_header("cookie", ";delta"); - req.fs.add_header("cookie", "echo"); + req.fs.add_header_token(StringRef::from_lit(":method"), + StringRef::from_lit("get"), false, -1); + req.fs.add_header_token(StringRef::from_lit(":path"), + StringRef::from_lit("/"), false, -1); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit("alpha; bravo; ; ;; charlie;;"), + true, http2::HD_COOKIE); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit(";delta"), false, + http2::HD_COOKIE); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit("echo"), false, http2::HD_COOKIE); std::vector nva; d.crumble_request_cookie(nva); @@ -114,12 +128,22 @@ void test_downstream_crumble_request_cookie(void) { void test_downstream_assemble_request_cookie(void) { Downstream d(nullptr, nullptr, 0); auto &req = d.request(); - req.fs.add_header(":method", "get"); - req.fs.add_header(":path", "/"); - req.fs.add_header("cookie", "alpha"); - req.fs.add_header("cookie", "bravo;"); - req.fs.add_header("cookie", "charlie; "); - req.fs.add_header("cookie", "delta;;"); + req.fs.add_header_token(StringRef::from_lit(":method"), + StringRef::from_lit("get"), false, -1); + req.fs.add_header_token(StringRef::from_lit(":path"), + StringRef::from_lit("/"), false, -1); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit("alpha"), false, + http2::HD_COOKIE); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit("bravo;"), false, + http2::HD_COOKIE); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit("charlie; "), false, + http2::HD_COOKIE); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit("delta;;"), false, + http2::HD_COOKIE); CU_ASSERT("alpha; bravo; charlie; delta" == d.assemble_request_cookie()); } @@ -129,8 +153,9 @@ void test_downstream_rewrite_location_response_header(void) { auto &resp = d.response(); d.set_request_downstream_host("localhost2"); req.authority = "localhost:8443"; - resp.fs.add_header("location", "http://localhost2:3000/"); - resp.fs.index_headers(); + resp.fs.add_header_token(StringRef::from_lit("location"), + StringRef::from_lit("http://localhost2:3000/"), + false, http2::HD_LOCATION); d.rewrite_location_response_header("https"); auto location = resp.fs.header(http2::HD_LOCATION); CU_ASSERT("https://localhost:8443/" == (*location).value); diff --git a/src/shrpx_downstream_test.h b/src/shrpx_downstream_test.h index 13b33744..bcae1887 100644 --- a/src/shrpx_downstream_test.h +++ b/src/shrpx_downstream_test.h @@ -31,7 +31,7 @@ namespace shrpx { -void test_downstream_field_store_index_headers(void); +void test_downstream_field_store_add_header_lower(void); void test_downstream_field_store_header(void); void test_downstream_crumble_request_cookie(void); void test_downstream_assemble_request_cookie(void); diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 2c96a579..d11e3fd7 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -736,16 +736,17 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, } auto token = http2::lookup_token(name, namelen); + auto no_index = flags & NGHTTP2_NV_FLAG_NO_INDEX; if (trailer) { // just store header fields for trailer part - resp.fs.add_trailer(StringRef{name, namelen}, StringRef{value, valuelen}, - flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + resp.fs.add_trailer_token(StringRef{name, namelen}, + StringRef{value, valuelen}, no_index, token); return 0; } - resp.fs.add_header(StringRef{name, namelen}, StringRef{value, valuelen}, - flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + resp.fs.add_header_token(StringRef{name, namelen}, + StringRef{value, valuelen}, no_index, token); return 0; } case NGHTTP2_PUSH_PROMISE: { @@ -778,9 +779,9 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, } auto token = http2::lookup_token(name, namelen); - promised_req.fs.add_header(StringRef{name, namelen}, - StringRef{value, valuelen}, - flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + promised_req.fs.add_header_token(StringRef{name, namelen}, + StringRef{value, valuelen}, + flags & NGHTTP2_NV_FLAG_NO_INDEX, token); return 0; } } @@ -927,8 +928,9 @@ int on_response_headers(Http2Session *http2session, Downstream *downstream, // Otherwise, use chunked encoding to keep upstream connection // open. In HTTP2, we are supporsed not to receive // transfer-encoding. - resp.fs.add_header("transfer-encoding", "chunked", - http2::HD_TRANSFER_ENCODING); + resp.fs.add_header_token(StringRef::from_lit("transfer-encoding"), + StringRef::from_lit("chunked"), false, + http2::HD_TRANSFER_ENCODING); downstream->set_chunked_response(true); } } diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 4506081c..0c8795dc 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -202,16 +202,17 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, } auto token = http2::lookup_token(name, namelen); + auto no_index = flags & NGHTTP2_NV_FLAG_NO_INDEX; if (frame->headers.cat == NGHTTP2_HCAT_HEADERS) { // just store header fields for trailer part - req.fs.add_trailer(StringRef{name, namelen}, StringRef{value, valuelen}, - flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + req.fs.add_trailer_token(StringRef{name, namelen}, + StringRef{value, valuelen}, no_index, token); return 0; } - req.fs.add_header(StringRef{name, namelen}, StringRef{value, valuelen}, - flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + req.fs.add_header_token(StringRef{name, namelen}, StringRef{value, valuelen}, + no_index, token); return 0; } } // namespace @@ -593,9 +594,9 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame, req.path = http2::rewrite_clean_path(nv.value, nv.value + nv.valuelen); break; } - req.fs.add_header(StringRef{nv.name, nv.namelen}, - StringRef{nv.value, nv.valuelen}, - nv.flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + req.fs.add_header_token(StringRef{nv.name, nv.namelen}, + StringRef{nv.value, nv.valuelen}, + nv.flags & NGHTTP2_NV_FLAG_NO_INDEX, token); } promised_downstream->inspect_http2_request(); diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 0af56ce1..c5b3da09 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -691,7 +691,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { if (ensure_max_header_fields(downstream, httpconf) != 0) { return -1; } - resp.fs.add_header_lower(StringRef{data, len}, StringRef{}); + resp.fs.add_header_lower(StringRef{data, len}, StringRef{}, false); } } else { // trailer part @@ -704,7 +704,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { // wrong place or crash if trailer fields are currently empty. return -1; } - resp.fs.add_trailer_lower(StringRef(data, len), StringRef{}); + resp.fs.add_trailer_lower(StringRef(data, len), StringRef{}, false); } } return 0; diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 7f47b681..c84af459 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -142,7 +142,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { Downstream::HTTP1_REQUEST_HEADER_TOO_LARGE); return -1; } - req.fs.add_header_lower(StringRef{data, len}, StringRef{}); + req.fs.add_header_lower(StringRef{data, len}, StringRef{}, false); } } else { // trailer part @@ -156,7 +156,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { } return -1; } - req.fs.add_trailer_lower(StringRef{data, len}, StringRef{}); + req.fs.add_trailer_lower(StringRef{data, len}, StringRef{}, false); } } return 0; diff --git a/src/shrpx_mruby.cc b/src/shrpx_mruby.cc index bd57c146..507d13b9 100644 --- a/src/shrpx_mruby.cc +++ b/src/shrpx_mruby.cc @@ -94,14 +94,6 @@ int MRubyContext::run_app(Downstream *downstream, int phase) { mrb_->ud = nullptr; - if (data.request_headers_dirty) { - downstream->request().fs.index_headers(); - } - - if (data.response_headers_dirty) { - downstream->response().fs.index_headers(); - } - return rv; } diff --git a/src/shrpx_mruby.h b/src/shrpx_mruby.h index 55a18351..4466cf6f 100644 --- a/src/shrpx_mruby.h +++ b/src/shrpx_mruby.h @@ -67,8 +67,6 @@ enum { struct MRubyAssocData { Downstream *downstream; int phase; - bool request_headers_dirty; - bool response_headers_dirty; }; RProc *compile(mrb_state *mrb, const StringRef &filename); diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index 0b1ddb15..f585d730 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -216,17 +216,20 @@ mrb_value request_mod_header(mrb_state *mrb, mrb_value self, bool repl) { key = mrb_funcall(mrb, key, "downcase", 0); + auto keyref = + StringRef{RSTRING_PTR(key), static_cast(RSTRING_LEN(key))}; + auto token = http2::lookup_token(keyref.byte(), keyref.size()); + if (repl) { size_t p = 0; auto &headers = req.fs.headers(); for (size_t i = 0; i < headers.size(); ++i) { - auto &hd = headers[i]; - if (util::streq(std::begin(hd.name), hd.name.size(), RSTRING_PTR(key), - RSTRING_LEN(key))) { + auto &kv = headers[i]; + if (kv.name == keyref) { continue; } if (i != p) { - headers[p++] = std::move(hd); + headers[p++] = std::move(kv); } } headers.resize(p); @@ -236,16 +239,18 @@ mrb_value request_mod_header(mrb_state *mrb, mrb_value self, bool repl) { auto n = mrb_ary_len(mrb, values); for (int i = 0; i < n; ++i) { auto value = mrb_ary_entry(values, i); - req.fs.add_header(std::string(RSTRING_PTR(key), RSTRING_LEN(key)), - std::string(RSTRING_PTR(value), RSTRING_LEN(value))); + req.fs.add_header_token( + keyref, StringRef{RSTRING_PTR(value), + static_cast(RSTRING_LEN(value))}, + false, token); } } else if (!mrb_nil_p(values)) { - req.fs.add_header(std::string(RSTRING_PTR(key), RSTRING_LEN(key)), - std::string(RSTRING_PTR(values), RSTRING_LEN(values))); + req.fs.add_header_token(keyref, + StringRef{RSTRING_PTR(values), + static_cast(RSTRING_LEN(values))}, + false, token); } - data->request_headers_dirty = true; - return mrb_nil_value(); } } // namespace diff --git a/src/shrpx_mruby_module_response.cc b/src/shrpx_mruby_module_response.cc index 5b88f879..73dacccc 100644 --- a/src/shrpx_mruby_module_response.cc +++ b/src/shrpx_mruby_module_response.cc @@ -117,17 +117,20 @@ mrb_value response_mod_header(mrb_state *mrb, mrb_value self, bool repl) { key = mrb_funcall(mrb, key, "downcase", 0); + auto keyref = + StringRef{RSTRING_PTR(key), static_cast(RSTRING_LEN(key))}; + auto token = http2::lookup_token(keyref.byte(), keyref.size()); + if (repl) { size_t p = 0; auto &headers = resp.fs.headers(); for (size_t i = 0; i < headers.size(); ++i) { - auto &hd = headers[i]; - if (util::streq(std::begin(hd.name), hd.name.size(), RSTRING_PTR(key), - RSTRING_LEN(key))) { + auto &kv = headers[i]; + if (kv.name == keyref) { continue; } if (i != p) { - headers[p++] = std::move(hd); + headers[p++] = std::move(kv); } } headers.resize(p); @@ -137,16 +140,18 @@ mrb_value response_mod_header(mrb_state *mrb, mrb_value self, bool repl) { auto n = mrb_ary_len(mrb, values); for (int i = 0; i < n; ++i) { auto value = mrb_ary_entry(values, i); - resp.fs.add_header(std::string(RSTRING_PTR(key), RSTRING_LEN(key)), - std::string(RSTRING_PTR(value), RSTRING_LEN(value))); + resp.fs.add_header_token( + keyref, StringRef{RSTRING_PTR(value), + static_cast(RSTRING_LEN(value))}, + false, token); } } else if (!mrb_nil_p(values)) { - resp.fs.add_header(std::string(RSTRING_PTR(key), RSTRING_LEN(key)), - std::string(RSTRING_PTR(values), RSTRING_LEN(values))); + resp.fs.add_header_token( + keyref, StringRef{RSTRING_PTR(values), + static_cast(RSTRING_LEN(values))}, + false, token); } - data->response_headers_dirty = true; - return mrb_nil_value(); } } // namespace @@ -197,11 +202,6 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { resp.http_status = 200; } - if (data->response_headers_dirty) { - resp.fs.index_headers(); - data->response_headers_dirty = false; - } - if (downstream->expect_response_body() && vallen > 0) { body = reinterpret_cast(val); bodylen = vallen; @@ -211,8 +211,9 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { if (cl) { cl->value = util::utos(bodylen); } else { - resp.fs.add_header("content-length", util::utos(bodylen), - http2::HD_CONTENT_LENGTH); + resp.fs.add_header_token(StringRef::from_lit("content-length"), + StringRef{util::utos(bodylen)}, false, + http2::HD_CONTENT_LENGTH); } resp.fs.content_length = bodylen; @@ -220,7 +221,9 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { if (!date) { auto lgconf = log_config(); lgconf->update_tstamp(std::chrono::system_clock::now()); - resp.fs.add_header("date", lgconf->time_http_str, http2::HD_DATE); + resp.fs.add_header_token(StringRef::from_lit("date"), + StringRef{lgconf->time_http_str}, false, + http2::HD_DATE); } auto upstream = downstream->get_upstream(); diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 27d768c2..dc4a5db1 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -188,10 +188,10 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, } for (size_t i = 0; nv[i]; i += 2) { - auto name = std::string(nv[i]); - auto value = std::string(nv[i + 1]); - auto token = http2::lookup_token(name); - req.fs.add_header(std::move(name), std::move(value), token); + auto name = StringRef{nv[i]}; + auto value = StringRef{nv[i + 1]}; + auto token = http2::lookup_token(name.byte(), name.size()); + req.fs.add_header_token(name, value, false, token); } if (req.fs.index_headers() != 0) { From 9678daa46aa72f7abc698822ea6d930665b0d612 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 20 Feb 2016 21:44:08 +0900 Subject: [PATCH 69/85] nghttpx: Rename index_headers() as parse_content_length() --- src/shrpx_downstream.cc | 2 +- src/shrpx_downstream.h | 7 +++---- src/shrpx_http_downstream_connection.cc | 2 +- src/shrpx_https_upstream.cc | 2 +- src/shrpx_spdy_upstream.cc | 2 +- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 11ec9756..9cfc2cc7 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -367,7 +367,7 @@ void append_last_header_value(bool &key_prev, size_t &sum, Headers &headers, } } // namespace -int FieldStore::index_headers() { +int FieldStore::parse_content_length() { content_length = -1; for (auto &kv : headers_) { diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 8a855b1c..4306940d 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -89,10 +89,9 @@ public: bool header_key_prev() const { return header_key_prev_; } - // Lower the header field names and indexes header fields. If there - // is any invalid headers (e.g., multiple Content-Length having - // different values), returns -1. - int index_headers(); + // Parses content-length, and records it in the field. If there are + // multiple Content-Length, returns -1. + int parse_content_length(); // Empties headers. void clear_headers(); diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index c5b3da09..9e55f39b 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -572,7 +572,7 @@ int htp_hdrs_completecb(http_parser *htp) { resp.http_major = htp->http_major; resp.http_minor = htp->http_minor; - if (resp.fs.index_headers() != 0) { + if (resp.fs.parse_content_length() != 0) { downstream->set_response_state(Downstream::MSG_BAD_HEADER); return -1; } diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index c84af459..9eaa5202 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -270,7 +270,7 @@ int htp_hdrs_completecb(http_parser *htp) { ULOG(INFO, upstream) << "HTTP request headers\n" << ss.str(); } - if (req.fs.index_headers() != 0) { + if (req.fs.parse_content_length() != 0) { return -1; } diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index dc4a5db1..d9c22209 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -194,7 +194,7 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, req.fs.add_header_token(name, value, false, token); } - if (req.fs.index_headers() != 0) { + if (req.fs.parse_content_length() != 0) { if (upstream->error_reply(downstream, 400) != 0) { ULOG(FATAL, upstream) << "error_reply failed"; } From 61579ad20ff965eb7df80b55a412282766e0110e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 20 Feb 2016 22:56:55 +0900 Subject: [PATCH 70/85] nghttpx: Use StringRef for shrpx::add_header --- src/shrpx_downstream.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 9cfc2cc7..32b7c1f2 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -329,11 +329,12 @@ void Downstream::crumble_request_cookie(std::vector &nva) { } namespace { -void add_header(bool &key_prev, size_t &sum, Headers &headers, std::string name, - std::string value, bool no_index, int16_t token) { +void add_header(bool &key_prev, size_t &sum, Headers &headers, + const StringRef &name, const StringRef &value, bool no_index, + int16_t token) { key_prev = true; sum += name.size() + value.size(); - headers.emplace_back(std::move(name), std::move(value), no_index, token); + headers.emplace_back(name.str(), value.str(), no_index, token); } } // namespace @@ -417,7 +418,7 @@ void FieldStore::add_header_lower(const StringRef &name, const StringRef &value, util::inp_strlower(low_name); auto token = http2::lookup_token(low_name); shrpx::add_header(header_key_prev_, buffer_size_, headers_, - std::move(low_name), value.str(), no_index, token); + StringRef{low_name}, value, no_index, token); } void FieldStore::add_header_token(const StringRef &name, const StringRef &value, @@ -443,7 +444,7 @@ void FieldStore::add_trailer_lower(const StringRef &name, util::inp_strlower(low_name); auto token = http2::lookup_token(low_name); shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, - std::move(low_name), value.str(), no_index, token); + StringRef{low_name}, value, no_index, token); } void FieldStore::add_trailer_token(const StringRef &name, From c9a4f293a11be14c1fcc4b7ce58ee617b53da947 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 14:53:06 +0900 Subject: [PATCH 71/85] nghttpx: ConnectBlocker per backend address --- src/shrpx_client_handler.cc | 8 +-- src/shrpx_client_handler.h | 1 - src/shrpx_config.h | 2 + src/shrpx_connect_blocker.cc | 27 +++++---- src/shrpx_connect_blocker.h | 14 +++-- src/shrpx_http2_downstream_connection.cc | 3 + src/shrpx_http2_session.cc | 73 +++++++++++++++--------- src/shrpx_http2_session.h | 7 +-- src/shrpx_http_downstream_connection.cc | 33 +++++++---- src/shrpx_worker.cc | 26 ++++++--- src/shrpx_worker.h | 5 +- 11 files changed, 125 insertions(+), 74 deletions(-) diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 4d063647..6e89a5c4 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -389,7 +389,7 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl, pinned_http2sessions_( get_config()->conn.downstream.proto == PROTO_HTTP2 ? make_unique>( - get_config()->conn.downstream.addr_groups.size(), -1) + worker->get_downstream_addr_groups().size(), -1) : nullptr), ipaddr_(ipaddr), port_(port), @@ -664,8 +664,8 @@ std::unique_ptr ClientHandler::get_downstream_connection(Downstream *downstream) { size_t group; auto &downstreamconf = get_config()->conn.downstream; - auto &groups = downstreamconf.addr_groups; auto catch_all = downstreamconf.addr_group_catch_all; + auto &groups = worker_->get_downstream_addr_groups(); const auto &req = downstream->request(); @@ -746,10 +746,6 @@ MemchunkPool *ClientHandler::get_mcpool() { return worker_->get_mcpool(); } SSL *ClientHandler::get_ssl() const { return conn_.tls.ssl; } -ConnectBlocker *ClientHandler::get_connect_blocker() const { - return worker_->get_connect_blocker(); -} - void ClientHandler::direct_http2_upgrade() { upstream_ = make_unique(this); alpn_ = NGHTTP2_CLEARTEXT_PROTO_VERSION_ID; diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index 4830e93e..9e932ae2 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -99,7 +99,6 @@ public: get_downstream_connection(Downstream *downstream); MemchunkPool *get_mcpool(); SSL *get_ssl() const; - ConnectBlocker *get_connect_blocker() const; // Call this function when HTTP/2 connection header is received at // the start of the connection. void direct_http2_upgrade(); diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 80f0e06e..7913fa19 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -59,6 +59,7 @@ using namespace nghttp2; namespace shrpx { struct LogFragment; +class ConnectBlocker; namespace ssl { @@ -294,6 +295,7 @@ struct DownstreamAddr { // socket path. ImmutableString host; ImmutableString hostport; + ConnectBlocker *connect_blocker; // backend port. 0 if |host_unix| is true. uint16_t port; // true if |host| contains UNIX domain socket path. diff --git a/src/shrpx_connect_blocker.cc b/src/shrpx_connect_blocker.cc index 3eb6de16..cdab59a9 100644 --- a/src/shrpx_connect_blocker.cc +++ b/src/shrpx_connect_blocker.cc @@ -26,20 +26,16 @@ namespace shrpx { -namespace { -const ev_tstamp INITIAL_SLEEP = 2.; -} // namespace - namespace { void connect_blocker_cb(struct ev_loop *loop, ev_timer *w, int revents) { if (LOG_ENABLED(INFO)) { - LOG(INFO) << "unblock downstream connection"; + LOG(INFO) << "Unblock"; } } } // namespace -ConnectBlocker::ConnectBlocker(struct ev_loop *loop) - : loop_(loop), sleep_(INITIAL_SLEEP) { +ConnectBlocker::ConnectBlocker(std::mt19937 &gen, struct ev_loop *loop) + : gen_(gen), loop_(loop), fail_count_(0) { ev_timer_init(&timer_, connect_blocker_cb, 0., 0.); } @@ -47,18 +43,27 @@ ConnectBlocker::~ConnectBlocker() { ev_timer_stop(loop_, &timer_); } bool ConnectBlocker::blocked() const { return ev_is_active(&timer_); } -void ConnectBlocker::on_success() { sleep_ = INITIAL_SLEEP; } +void ConnectBlocker::on_success() { fail_count_ = 0; } + +namespace { +constexpr size_t MAX_BACKOFF_EXP = 10; +} // namespace void ConnectBlocker::on_failure() { if (ev_is_active(&timer_)) { return; } - sleep_ = std::min(128., sleep_ * 2); + ++fail_count_; - LOG(WARN) << "connect failure, start sleeping " << sleep_; + auto max_backoff = (1 << std::min(MAX_BACKOFF_EXP, fail_count_)) - 1; + auto dist = std::uniform_int_distribution<>(0, max_backoff); + auto backoff = dist(gen_); - ev_timer_set(&timer_, sleep_, 0.); + LOG(WARN) << "Could not connect " << fail_count_ + << " times in a row; sleep for " << backoff << " seconds"; + + ev_timer_set(&timer_, backoff, 0.); ev_timer_start(loop_, &timer_); } diff --git a/src/shrpx_connect_blocker.h b/src/shrpx_connect_blocker.h index af445644..63a1e3f9 100644 --- a/src/shrpx_connect_blocker.h +++ b/src/shrpx_connect_blocker.h @@ -27,13 +27,15 @@ #include "shrpx.h" +#include + #include namespace shrpx { class ConnectBlocker { public: - ConnectBlocker(struct ev_loop *loop); + ConnectBlocker(std::mt19937 &gen, struct ev_loop *loop); ~ConnectBlocker(); // Returns true if making connection is not allowed. @@ -41,14 +43,18 @@ public: // Call this function if connect operation succeeded. This will // reset sleep_ to minimum value. void on_success(); - // Call this function if connect operation failed. This will start - // timer and blocks connection establishment for sleep_ seconds. + // Call this function if connect operations failed. This will start + // timer and blocks connection establishment with exponential + // backoff. void on_failure(); private: + std::mt19937 gen_; ev_timer timer_; struct ev_loop *loop_; - ev_tstamp sleep_; + // The number of consecutive connection failure. Reset to 0 on + // success. + size_t fail_count_; }; } // namespace diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index 02982b8a..da2d953d 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -98,6 +98,9 @@ int Http2DownstreamConnection::attach_downstream(Downstream *downstream) { http2session_->add_downstream_connection(this); if (http2session_->get_state() == Http2Session::DISCONNECTED) { http2session_->signal_write(); + if (http2session_->get_state() == Http2Session::DISCONNECTED) { + return -1; + } } downstream_ = downstream; diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index d11e3fd7..1f660649 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -147,8 +147,7 @@ void writecb(struct ev_loop *loop, ev_io *w, int revents) { } // namespace Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, - ConnectBlocker *connect_blocker, Worker *worker, - size_t group, size_t idx) + Worker *worker, size_t group, size_t idx) : conn_(loop, -1, nullptr, worker->get_mcpool(), get_config()->conn.downstream.timeout.write, get_config()->conn.downstream.timeout.read, {}, {}, writecb, readcb, @@ -156,7 +155,6 @@ Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, get_config()->tls.dyn_rec.idle_timeout), wb_(worker->get_mcpool()), worker_(worker), - connect_blocker_(connect_blocker), ssl_ctx_(ssl_ctx), addr_(nullptr), session_(nullptr), @@ -254,30 +252,49 @@ int Http2Session::disconnect(bool hard) { int Http2Session::initiate_connection() { int rv = 0; - auto &addrs = get_config()->conn.downstream.addr_groups[group_].addrs; + auto &groups = worker_->get_downstream_addr_groups(); + auto &addrs = groups[group_].addrs; if (state_ == DISCONNECTED) { - if (connect_blocker_->blocked()) { - if (LOG_ENABLED(INFO)) { - DCLOG(INFO, this) - << "Downstream connection was blocked by connect_blocker"; - } - return -1; - } - auto &next_downstream = worker_->get_dgrp(group_)->next; - addr_ = &addrs[next_downstream]; + auto end = next_downstream; - if (LOG_ENABLED(INFO)) { - SSLOG(INFO, this) << "Using downstream address idx=" << next_downstream - << " out of " << addrs.size(); - } + for (;;) { + auto &addr = addrs[next_downstream]; - if (++next_downstream >= addrs.size()) { - next_downstream = 0; + if (++next_downstream >= addrs.size()) { + next_downstream = 0; + } + + auto &connect_blocker = addr.connect_blocker; + + if (connect_blocker->blocked()) { + if (LOG_ENABLED(INFO)) { + DCLOG(INFO, this) << "Backend server " + << (addr.host_unix ? addr.host : addr.hostport) + << " was not available temporarily"; + } + + if (end == next_downstream) { + return -1; + } + + continue; + } + + if (LOG_ENABLED(INFO)) { + SSLOG(INFO, this) << "Using downstream address idx=" << next_downstream + << " out of " << addrs.size(); + } + + addr_ = &addr; + + break; } } + auto &connect_blocker = addr_->connect_blocker; + const auto &proxy = get_config()->downstream_http_proxy; if (!proxy.host.empty() && state_ == DISCONNECTED) { if (LOG_ENABLED(INFO)) { @@ -288,7 +305,7 @@ int Http2Session::initiate_connection() { conn_.fd = util::create_nonblock_socket(proxy.addr.su.storage.ss_family); if (conn_.fd == -1) { - connect_blocker_->on_failure(); + connect_blocker->on_failure(); return -1; } @@ -296,7 +313,7 @@ int Http2Session::initiate_connection() { if (rv != 0 && errno != EINPROGRESS) { SSLOG(ERROR, this) << "Failed to connect to the proxy " << proxy.host << ":" << proxy.port; - connect_blocker_->on_failure(); + connect_blocker->on_failure(); return -1; } @@ -356,7 +373,7 @@ int Http2Session::initiate_connection() { conn_.fd = util::create_nonblock_socket(addr_->addr.su.storage.ss_family); if (conn_.fd == -1) { - connect_blocker_->on_failure(); + connect_blocker->on_failure(); return -1; } @@ -365,7 +382,7 @@ int Http2Session::initiate_connection() { const_cast(&addr_->addr.su.sa), addr_->addr.len); if (rv != 0 && errno != EINPROGRESS) { - connect_blocker_->on_failure(); + connect_blocker->on_failure(); return -1; } @@ -383,14 +400,14 @@ int Http2Session::initiate_connection() { util::create_nonblock_socket(addr_->addr.su.storage.ss_family); if (conn_.fd == -1) { - connect_blocker_->on_failure(); + connect_blocker->on_failure(); return -1; } rv = connect(conn_.fd, const_cast(&addr_->addr.su.sa), addr_->addr.len); if (rv != 0 && errno != EINPROGRESS) { - connect_blocker_->on_failure(); + connect_blocker->on_failure(); return -1; } @@ -1615,11 +1632,15 @@ int Http2Session::read_noop(const uint8_t *data, size_t datalen) { return 0; } int Http2Session::write_noop() { return 0; } int Http2Session::connected() { + auto &connect_blocker = addr_->connect_blocker; + if (!util::check_socket_connected(conn_.fd)) { + connect_blocker->on_failure(); + return -1; } - connect_blocker_->on_success(); + connect_blocker->on_success(); if (LOG_ENABLED(INFO)) { SSLOG(INFO, this) << "Connection established"; diff --git a/src/shrpx_http2_session.h b/src/shrpx_http2_session.h index a858b1bc..62e1235c 100644 --- a/src/shrpx_http2_session.h +++ b/src/shrpx_http2_session.h @@ -48,7 +48,6 @@ namespace shrpx { class Http2DownstreamConnection; class Worker; -class ConnectBlocker; struct StreamData { StreamData *dlnext, *dlprev; @@ -57,9 +56,8 @@ struct StreamData { class Http2Session { public: - Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, - ConnectBlocker *connect_blocker, Worker *worker, size_t group, - size_t idx); + Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, Worker *worker, + size_t group, size_t idx); ~Http2Session(); // If hard is true, all pending requests are abandoned and @@ -203,7 +201,6 @@ private: // Used to parse the response from HTTP proxy std::unique_ptr proxy_htp_; Worker *worker_; - ConnectBlocker *connect_blocker_; // NULL if no TLS is configured SSL_CTX *ssl_ctx_; // Address of remote endpoint diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 9e55f39b..151bd45b 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -147,16 +147,6 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { auto &downstreamconf = get_config()->conn.downstream; if (conn_.fd == -1) { - auto connect_blocker = client_handler_->get_connect_blocker(); - - if (connect_blocker->blocked()) { - if (LOG_ENABLED(INFO)) { - DCLOG(INFO, this) - << "Downstream connection was blocked by connect_blocker"; - } - return -1; - } - if (ssl_ctx_) { auto ssl = ssl::create_ssl(ssl_ctx_); if (!ssl) { @@ -168,7 +158,8 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { auto &next_downstream = worker_->get_dgrp(group_)->next; auto end = next_downstream; - auto &addrs = downstreamconf.addr_groups[group_].addrs; + auto &groups = worker_->get_downstream_addr_groups(); + auto &addrs = groups[group_].addrs; for (;;) { auto &addr = addrs[next_downstream]; @@ -176,6 +167,22 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { next_downstream = 0; } + auto &connect_blocker = addr.connect_blocker; + + if (connect_blocker->blocked()) { + if (LOG_ENABLED(INFO)) { + DCLOG(INFO, this) << "Backend server " + << (addr.host_unix ? addr.host : addr.hostport) + << " was not available temporarily"; + } + + if (end == next_downstream) { + return SHRPX_ERR_NETWORK; + } + + continue; + } + conn_.fd = util::create_nonblock_socket(addr.addr.su.storage.ss_family); if (conn_.fd == -1) { @@ -1012,7 +1019,7 @@ int HttpDownstreamConnection::process_input(const uint8_t *data, } int HttpDownstreamConnection::connected() { - auto connect_blocker = client_handler_->get_connect_blocker(); + auto connect_blocker = addr_->connect_blocker; if (!util::check_socket_connected(conn_.fd)) { conn_.wlimit.stopw(); @@ -1021,6 +1028,8 @@ int HttpDownstreamConnection::connected() { DLOG(INFO, this) << "downstream connect failed"; } + connect_blocker->on_failure(); + downstream_->set_request_state(Downstream::CONNECT_FAIL); return -1; diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 604f9058..566ac334 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -80,7 +80,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, cl_ssl_ctx_(cl_ssl_ctx), cert_tree_(cert_tree), ticket_keys_(ticket_keys), - connect_blocker_(make_unique(loop_)), + downstream_addr_groups_(get_config()->conn.downstream.addr_groups), graceful_shutdown_(false) { ev_async_init(&w_, eventcb); w_.data = this; @@ -109,17 +109,29 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, m = downstreamconf.addr_groups[group].addrs.size(); } for (size_t idx = 0; idx < m; ++idx) { - dgrp.http2sessions.push_back(make_unique( - loop_, cl_ssl_ctx, connect_blocker_.get(), this, group, idx)); + dgrp.http2sessions.push_back( + make_unique(loop_, cl_ssl_ctx, this, group, idx)); } ++group; } } + + for (auto &group : downstream_addr_groups_) { + for (auto &addr : group.addrs) { + addr.connect_blocker = new ConnectBlocker(randgen_, loop_); + } + } } Worker::~Worker() { ev_async_stop(loop_, &w_); ev_timer_stop(loop_, &mcpool_clear_timer_); + + for (auto &group : downstream_addr_groups_) { + for (auto &addr : group.addrs) { + delete addr.connect_blocker; + } + } } void Worker::schedule_clear_mcpool() { @@ -259,10 +271,6 @@ Http2Session *Worker::next_http2_session(size_t group) { return res; } -ConnectBlocker *Worker::get_connect_blocker() const { - return connect_blocker_.get(); -} - struct ev_loop *Worker::get_loop() const { return loop_; } @@ -361,4 +369,8 @@ SSL_SESSION *Worker::reuse_client_tls_session(const Address *addr) { return d2i_SSL_SESSION(nullptr, &p, ent.session_data.size()); } +std::vector &Worker::get_downstream_addr_groups() { + return downstream_addr_groups_; +} + } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 8d352429..e56595c1 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -131,7 +131,6 @@ public: WorkerStat *get_worker_stat(); DownstreamConnectionPool *get_dconn_pool(); Http2Session *next_http2_session(size_t group); - ConnectBlocker *get_connect_blocker() const; struct ev_loop *get_loop() const; SSL_CTX *get_sv_ssl_ctx() const; SSL_CTX *get_cl_ssl_ctx() const; @@ -164,6 +163,8 @@ public: // found associated to |addr|, nullptr will be returned. SSL_SESSION *reuse_client_tls_session(const Address *addr); + std::vector &get_downstream_addr_groups(); + private: #ifndef NOTHREADS std::future fut_; @@ -196,7 +197,7 @@ private: ssl::CertLookupTree *cert_tree_; std::shared_ptr ticket_keys_; - std::unique_ptr connect_blocker_; + std::vector downstream_addr_groups_; bool graceful_shutdown_; }; From 11c8803b92504eaff42550a174532f9a5459e756 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 15:27:19 +0900 Subject: [PATCH 72/85] nghttpx: Worker wide blocker which is used when socket(2) is failed --- src/shrpx_http2_session.cc | 19 ++++++++++++++++--- src/shrpx_http_downstream_connection.cc | 11 ++++++++++- src/shrpx_worker.cc | 5 +++++ src/shrpx_worker.h | 5 +++++ 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 1f660649..4cb22ce1 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -254,8 +254,15 @@ int Http2Session::initiate_connection() { auto &groups = worker_->get_downstream_addr_groups(); auto &addrs = groups[group_].addrs; + auto worker_blocker = worker_->get_connect_blocker(); if (state_ == DISCONNECTED) { + if (worker_blocker->blocked()) { + DLOG(INFO, this) + << "Worker wide backend connection was blocked temporarily"; + return -1; + } + auto &next_downstream = worker_->get_dgrp(group_)->next; auto end = next_downstream; @@ -305,10 +312,12 @@ int Http2Session::initiate_connection() { conn_.fd = util::create_nonblock_socket(proxy.addr.su.storage.ss_family); if (conn_.fd == -1) { - connect_blocker->on_failure(); + worker_blocker->on_failure(); return -1; } + worker_blocker->on_success(); + rv = connect(conn_.fd, &proxy.addr.su.sa, proxy.addr.len); if (rv != 0 && errno != EINPROGRESS) { SSLOG(ERROR, this) << "Failed to connect to the proxy " << proxy.host @@ -373,10 +382,12 @@ int Http2Session::initiate_connection() { conn_.fd = util::create_nonblock_socket(addr_->addr.su.storage.ss_family); if (conn_.fd == -1) { - connect_blocker->on_failure(); + worker_blocker->on_failure(); return -1; } + worker_blocker->on_success(); + rv = connect(conn_.fd, // TODO maybe not thread-safe? const_cast(&addr_->addr.su.sa), @@ -400,10 +411,12 @@ int Http2Session::initiate_connection() { util::create_nonblock_socket(addr_->addr.su.storage.ss_family); if (conn_.fd == -1) { - connect_blocker->on_failure(); + worker_blocker->on_failure(); return -1; } + worker_blocker->on_success(); + rv = connect(conn_.fd, const_cast(&addr_->addr.su.sa), addr_->addr.len); if (rv != 0 && errno != EINPROGRESS) { diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 151bd45b..81acfcd0 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -144,6 +144,13 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { DCLOG(INFO, this) << "Attaching to DOWNSTREAM:" << downstream; } + auto worker_blocker = worker_->get_connect_blocker(); + if (worker_blocker->blocked()) { + DLOG(INFO, this) + << "Worker wide backend connection was blocked temporarily"; + return SHRPX_ERR_NETWORK; + } + auto &downstreamconf = get_config()->conn.downstream; if (conn_.fd == -1) { @@ -189,11 +196,13 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { auto error = errno; DCLOG(WARN, this) << "socket() failed; errno=" << error; - connect_blocker->on_failure(); + worker_blocker->on_failure(); return SHRPX_ERR_NETWORK; } + worker_blocker->on_success(); + int rv; rv = connect(conn_.fd, &addr.addr.su.sa, addr.addr.len); if (rv != 0 && errno != EINPROGRESS) { diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 566ac334..997138db 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -81,6 +81,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, cert_tree_(cert_tree), ticket_keys_(ticket_keys), downstream_addr_groups_(get_config()->conn.downstream.addr_groups), + connect_blocker_(make_unique(randgen_, loop_)), graceful_shutdown_(false) { ev_async_init(&w_, eventcb); w_.data = this; @@ -373,4 +374,8 @@ std::vector &Worker::get_downstream_addr_groups() { return downstream_addr_groups_; } +ConnectBlocker *Worker::get_connect_blocker() const { + return connect_blocker_.get(); +} + } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index e56595c1..9cd8b5ff 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -165,6 +165,8 @@ public: std::vector &get_downstream_addr_groups(); + ConnectBlocker *get_connect_blocker() const; + private: #ifndef NOTHREADS std::future fut_; @@ -198,6 +200,9 @@ private: std::shared_ptr ticket_keys_; std::vector downstream_addr_groups_; + // Worker level blocker for downstream connection. For example, + // this is used when file decriptor is exhausted. + std::unique_ptr connect_blocker_; bool graceful_shutdown_; }; From dfc02843b6342d9c6617bcd2bb1ebe424683aedb Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 15:28:11 +0900 Subject: [PATCH 73/85] src: Rename and rewrite numeric_hostport as to_numeric_addr and support AF_UNIX path --- src/Makefile.am | 2 +- src/network.h | 61 ++++++++++++++++++++++++++++++++ src/shrpx.cc | 10 +++--- src/shrpx_config.h | 14 +------- src/shrpx_memcached_connection.h | 3 +- src/shrpx_memcached_dispatcher.h | 2 +- src/shrpx_ssl.h | 3 +- src/shrpx_worker.cc | 13 +++---- src/util.cc | 14 ++++---- src/util.h | 10 +++--- 10 files changed, 91 insertions(+), 41 deletions(-) create mode 100644 src/network.h diff --git a/src/Makefile.am b/src/Makefile.am index 9c2cfa21..2c873893 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -62,7 +62,7 @@ HELPER_OBJECTS = util.cc \ http2.cc timegm.c app_helper.cc nghttp2_gzip.c HELPER_HFILES = util.h \ http2.h timegm.h app_helper.h nghttp2_config.h \ - nghttp2_gzip.h + nghttp2_gzip.h network.h HTML_PARSER_OBJECTS = HTML_PARSER_HFILES = HtmlParser.h diff --git a/src/network.h b/src/network.h new file mode 100644 index 00000000..3fb765d0 --- /dev/null +++ b/src/network.h @@ -0,0 +1,61 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef NETWORK_H +#define NETWORK_H + +#ifdef HAVE_CONFIG_H +#include +#endif // HAVE_CONFIG_H + +#include +#ifdef HAVE_SYS_SOCKET_H +#include +#endif // HAVE_SYS_SOCKET_H +#include +#ifdef HAVE_NETINET_IN_H +#include +#endif // HAVE_NETINET_IN_H +#ifdef HAVE_ARPA_INET_H +#include +#endif // HAVE_ARPA_INET_H + +namespace nghttp2 { + +union sockaddr_union { + sockaddr_storage storage; + sockaddr sa; + sockaddr_in6 in6; + sockaddr_in in; + sockaddr_un un; +}; + +struct Address { + size_t len; + union sockaddr_union su; +}; + +} // namespace nghttp2 + +#endif // NETWORK_H diff --git a/src/shrpx.cc b/src/shrpx.cc index 0c1fb3f7..f84f10ea 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -2201,7 +2201,7 @@ void process_options( exit(EXIT_FAILURE); } LOG(NOTICE) << "Resolved backend address: " << hostport << " -> " - << util::numeric_hostport(&addr.addr.su.sa, addr.addr.len); + << util::to_numeric_addr(&addr.addr); } } @@ -2214,7 +2214,7 @@ void process_options( exit(EXIT_FAILURE); } LOG(NOTICE) << "Backend HTTP proxy address: " << hostport << " -> " - << util::numeric_hostport(&proxy.addr.su.sa, proxy.addr.len); + << util::to_numeric_addr(&proxy.addr); } { @@ -2230,8 +2230,7 @@ void process_options( exit(EXIT_FAILURE); } LOG(NOTICE) << "Memcached address for TLS session cache: " << hostport - << " -> " << util::numeric_hostport(&memcachedconf.addr.su.sa, - memcachedconf.addr.len); + << " -> " << util::to_numeric_addr(&memcachedconf.addr); } } @@ -2247,8 +2246,7 @@ void process_options( exit(EXIT_FAILURE); } LOG(NOTICE) << "Memcached address for TLS ticket key: " << hostport - << " -> " << util::numeric_hostport(&memcachedconf.addr.su.sa, - memcachedconf.addr.len); + << " -> " << util::to_numeric_addr(&memcachedconf.addr); } } diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 7913fa19..944099d4 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -53,6 +53,7 @@ #include "shrpx_router.h" #include "template.h" #include "http2.h" +#include "network.h" using namespace nghttp2; @@ -231,19 +232,6 @@ constexpr char SHRPX_OPT_BACKEND_ADDRESS_FAMILY[] = "backend-address-family"; constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; -union sockaddr_union { - sockaddr_storage storage; - sockaddr sa; - sockaddr_in6 in6; - sockaddr_in in; - sockaddr_un un; -}; - -struct Address { - size_t len; - union sockaddr_union su; -}; - enum shrpx_proto { PROTO_HTTP2, PROTO_HTTP }; enum shrpx_forwarded_param { diff --git a/src/shrpx_memcached_connection.h b/src/shrpx_memcached_connection.h index c9552198..a093589a 100644 --- a/src/shrpx_memcached_connection.h +++ b/src/shrpx_memcached_connection.h @@ -35,12 +35,13 @@ #include "shrpx_connection.h" #include "buffer.h" +#include "network.h" + using namespace nghttp2; namespace shrpx { struct MemcachedRequest; -struct Address; enum { MEMCACHED_PARSE_HEADER24, diff --git a/src/shrpx_memcached_dispatcher.h b/src/shrpx_memcached_dispatcher.h index 64021c68..7cca3bcb 100644 --- a/src/shrpx_memcached_dispatcher.h +++ b/src/shrpx_memcached_dispatcher.h @@ -34,12 +34,12 @@ #include #include "memchunk.h" +#include "network.h" namespace shrpx { struct MemcachedRequest; class MemcachedConnection; -struct Address; class MemcachedDispatcher { public: diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h index ca93da14..0def8875 100644 --- a/src/shrpx_ssl.h +++ b/src/shrpx_ssl.h @@ -39,6 +39,8 @@ #include #endif // HAVE_NEVERBLEED +#include "network.h" + namespace shrpx { class ClientHandler; @@ -46,7 +48,6 @@ class Worker; class DownstreamConnectionPool; struct DownstreamAddr; struct UpstreamAddr; -struct Address; namespace ssl { diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 997138db..4b0fd452 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -329,9 +329,8 @@ void Worker::cache_client_tls_session(const Address *addr, SSL_SESSION *session, if (it == std::end(client_tls_session_cache_)) { if (LOG_ENABLED(INFO)) { LOG(INFO) << "Create cache entry for SSL_SESSION=" << session - << ", addr=" << util::numeric_hostport(&addr->su.sa, addr->len) - << "(" << addr << "), timestamp=" << std::fixed - << std::setprecision(6) << t; + << ", addr=" << util::to_numeric_addr(addr) << "(" << addr + << "), timestamp=" << std::fixed << std::setprecision(6) << t; } client_tls_session_cache_.emplace( addr, SessionCacheEntry{serialize_ssl_session(session), t}); @@ -341,8 +340,7 @@ void Worker::cache_client_tls_session(const Address *addr, SSL_SESSION *session, auto &ent = (*it).second; if (ent.last_updated + 1_min > t) { if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Cache for addr=" - << util::numeric_hostport(&addr->su.sa, addr->len) << "(" + LOG(INFO) << "Cache for addr=" << util::to_numeric_addr(addr) << "(" << addr << ") is still host. Not updating."; } return; @@ -350,9 +348,8 @@ void Worker::cache_client_tls_session(const Address *addr, SSL_SESSION *session, if (LOG_ENABLED(INFO)) { LOG(INFO) << "Update cache entry for SSL_SESSION=" << session - << ", addr=" << util::numeric_hostport(&addr->su.sa, addr->len) - << "(" << addr << "), timestamp=" << std::fixed - << std::setprecision(6) << t; + << ", addr=" << util::to_numeric_addr(addr) << "(" << addr + << "), timestamp=" << std::fixed << std::setprecision(6) << t; } ent.session_data = serialize_ssl_session(session); diff --git a/src/util.cc b/src/util.cc index 0897a0f7..12b90594 100644 --- a/src/util.cc +++ b/src/util.cc @@ -650,15 +650,17 @@ std::string numeric_name(const struct sockaddr *sa, socklen_t salen) { return host.data(); } -std::string numeric_hostport(const struct sockaddr *sa, socklen_t salen) { - if (sa->sa_family == AF_UNIX) { - return "localhost"; +std::string to_numeric_addr(const Address *addr) { + auto family = addr->su.storage.ss_family; + if (family == AF_UNIX) { + return addr->su.un.sun_path; } std::array host; std::array serv; - auto rv = getnameinfo(sa, salen, host.data(), host.size(), serv.data(), - serv.size(), NI_NUMERICHOST | NI_NUMERICSERV); + auto rv = + getnameinfo(&addr->su.sa, addr->len, host.data(), host.size(), + serv.data(), serv.size(), NI_NUMERICHOST | NI_NUMERICSERV); if (rv != 0) { return "unknown"; } @@ -668,7 +670,7 @@ std::string numeric_hostport(const struct sockaddr *sa, socklen_t salen) { std::string s; char *p; - if (sa->sa_family == AF_INET6) { + if (family == AF_INET6) { s.resize(hostlen + servlen + 2 + 1); p = &s[0]; *p++ = '['; diff --git a/src/util.h b/src/util.h index b1f64d77..f4b1c2a0 100644 --- a/src/util.h +++ b/src/util.h @@ -50,6 +50,7 @@ #include "http-parser/http_parser.h" #include "template.h" +#include "network.h" namespace nghttp2 { @@ -461,10 +462,11 @@ bool numeric_host(const char *hostname, int family); // failed, "unknown" is returned. std::string numeric_name(const struct sockaddr *sa, socklen_t salen); -// Returns string representation of numeric address and port of |addr| -// of length |salen|. The format is like :. For IPv6 -// address, address is enclosed by square brackets ([]). -std::string numeric_hostport(const struct sockaddr *sa, socklen_t salen); +// Returns string representation of numeric address and port of +// |addr|. If address family is AF_UNIX, this return path to UNIX +// domain socket. Otherwise, the format is like :. For +// IPv6 address, address is enclosed by square brackets ([]). +std::string to_numeric_addr(const Address *addr); // Makes internal copy of stderr (and possibly stdout in the future), // which is then used as pointer to /dev/stderr or /proc/self/fd/2 From 177d0a513f3f7e10ec08444404aceb124afede66 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 16:11:50 +0900 Subject: [PATCH 74/85] nghttpx: More logging for backend connection initiation --- src/shrpx_http2_session.cc | 47 +++++++++++++++++++++---- src/shrpx_http_downstream_connection.cc | 21 +++++++---- 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index 4cb22ce1..f7ac9a82 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -258,8 +258,10 @@ int Http2Session::initiate_connection() { if (state_ == DISCONNECTED) { if (worker_blocker->blocked()) { - DLOG(INFO, this) - << "Worker wide backend connection was blocked temporarily"; + if (LOG_ENABLED(INFO)) { + SSLOG(INFO, this) + << "Worker wide backend connection was blocked temporarily"; + } return -1; } @@ -277,8 +279,8 @@ int Http2Session::initiate_connection() { if (connect_blocker->blocked()) { if (LOG_ENABLED(INFO)) { - DCLOG(INFO, this) << "Backend server " - << (addr.host_unix ? addr.host : addr.hostport) + SSLOG(INFO, this) << "Backend server " + << util::to_numeric_addr(&addr.addr) << " was not available temporarily"; } @@ -312,6 +314,11 @@ int Http2Session::initiate_connection() { conn_.fd = util::create_nonblock_socket(proxy.addr.su.storage.ss_family); if (conn_.fd == -1) { + auto error = errno; + SSLOG(WARN, this) << "Backend proxy socket() failed; addr=" + << util::to_numeric_addr(&proxy.addr) + << ", errno=" << error; + worker_blocker->on_failure(); return -1; } @@ -320,8 +327,11 @@ int Http2Session::initiate_connection() { rv = connect(conn_.fd, &proxy.addr.su.sa, proxy.addr.len); if (rv != 0 && errno != EINPROGRESS) { - SSLOG(ERROR, this) << "Failed to connect to the proxy " << proxy.host - << ":" << proxy.port; + auto error = errno; + SSLOG(WARN, this) << "Backend proxy connect() failed; addr=" + << util::to_numeric_addr(&proxy.addr) + << ", errno=" << error; + connect_blocker->on_failure(); return -1; } @@ -382,6 +392,11 @@ int Http2Session::initiate_connection() { conn_.fd = util::create_nonblock_socket(addr_->addr.su.storage.ss_family); if (conn_.fd == -1) { + auto error = errno; + SSLOG(WARN, this) + << "socket() failed; addr=" << util::to_numeric_addr(&addr_->addr) + << ", errno=" << error; + worker_blocker->on_failure(); return -1; } @@ -393,6 +408,11 @@ int Http2Session::initiate_connection() { const_cast(&addr_->addr.su.sa), addr_->addr.len); if (rv != 0 && errno != EINPROGRESS) { + auto error = errno; + SSLOG(WARN, this) << "connect() failed; addr=" + << util::to_numeric_addr(&addr_->addr) + << ", errno=" << error; + connect_blocker->on_failure(); return -1; } @@ -411,6 +431,11 @@ int Http2Session::initiate_connection() { util::create_nonblock_socket(addr_->addr.su.storage.ss_family); if (conn_.fd == -1) { + auto error = errno; + SSLOG(WARN, this) + << "socket() failed; addr=" << util::to_numeric_addr(&addr_->addr) + << ", errno=" << error; + worker_blocker->on_failure(); return -1; } @@ -420,6 +445,11 @@ int Http2Session::initiate_connection() { rv = connect(conn_.fd, const_cast(&addr_->addr.su.sa), addr_->addr.len); if (rv != 0 && errno != EINPROGRESS) { + auto error = errno; + SSLOG(WARN, this) << "connect() failed; addr=" + << util::to_numeric_addr(&addr_->addr) + << ", errno=" << error; + connect_blocker->on_failure(); return -1; } @@ -1648,6 +1678,11 @@ int Http2Session::connected() { auto &connect_blocker = addr_->connect_blocker; if (!util::check_socket_connected(conn_.fd)) { + if (LOG_ENABLED(INFO)) { + SSLOG(INFO, this) << "Backend connect failed; addr=" + << util::to_numeric_addr(&addr_->addr); + } + connect_blocker->on_failure(); return -1; diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 81acfcd0..86b9119b 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -146,8 +146,10 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { auto worker_blocker = worker_->get_connect_blocker(); if (worker_blocker->blocked()) { - DLOG(INFO, this) - << "Worker wide backend connection was blocked temporarily"; + if (LOG_ENABLED(INFO)) { + DCLOG(INFO, this) + << "Worker wide backend connection was blocked temporarily"; + } return SHRPX_ERR_NETWORK; } @@ -179,7 +181,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { if (connect_blocker->blocked()) { if (LOG_ENABLED(INFO)) { DCLOG(INFO, this) << "Backend server " - << (addr.host_unix ? addr.host : addr.hostport) + << util::to_numeric_addr(&addr.addr) << " was not available temporarily"; } @@ -194,7 +196,9 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { if (conn_.fd == -1) { auto error = errno; - DCLOG(WARN, this) << "socket() failed; errno=" << error; + DCLOG(WARN, this) << "socket() failed; addr=" + << util::to_numeric_addr(&addr.addr) + << ", errno=" << error; worker_blocker->on_failure(); @@ -207,7 +211,9 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { rv = connect(conn_.fd, &addr.addr.su.sa, addr.addr.len); if (rv != 0 && errno != EINPROGRESS) { auto error = errno; - DCLOG(WARN, this) << "connect() failed; errno=" << error; + DCLOG(WARN, this) << "connect() failed; addr=" + << util::to_numeric_addr(&addr.addr) + << ", errno=" << error; connect_blocker->on_failure(); close(conn_.fd); @@ -1034,7 +1040,8 @@ int HttpDownstreamConnection::connected() { conn_.wlimit.stopw(); if (LOG_ENABLED(INFO)) { - DLOG(INFO, this) << "downstream connect failed"; + DCLOG(INFO, this) << "Backend connect failed; addr=" + << util::to_numeric_addr(&addr_->addr); } connect_blocker->on_failure(); @@ -1045,7 +1052,7 @@ int HttpDownstreamConnection::connected() { } if (LOG_ENABLED(INFO)) { - DLOG(INFO, this) << "Connected to downstream host"; + DCLOG(INFO, this) << "Connected to downstream host"; } connect_blocker->on_success(); From f2a72757001e56d9dfc3a1fb85ca7aeeac4818a1 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 16:35:43 +0900 Subject: [PATCH 75/85] nghttpx: Cache TLS session inside DownstreamAddr object --- src/shrpx_config.h | 10 +++++ src/shrpx_http_downstream_connection.cc | 19 ++++----- src/shrpx_http_downstream_connection.h | 2 +- src/shrpx_ssl.cc | 45 ++++++++++++++++++++ src/shrpx_ssl.h | 11 +++++ src/shrpx_worker.cc | 56 ------------------------- src/shrpx_worker.h | 23 ---------- 7 files changed, 76 insertions(+), 90 deletions(-) diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 944099d4..1a760d9e 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -277,6 +277,14 @@ struct UpstreamAddr { int fd; }; +struct TLSSessionCache { + // ASN1 representation of SSL_SESSION object. See + // i2d_SSL_SESSION(3SSL). + std::vector session_data; + // The last time stamp when this cache entry is created or updated. + ev_tstamp last_updated; +}; + struct DownstreamAddr { Address addr; // backend address. If |host_unix| is true, this is UNIX domain @@ -284,6 +292,8 @@ struct DownstreamAddr { ImmutableString host; ImmutableString hostport; ConnectBlocker *connect_blocker; + // Client side TLS session cache + TLSSessionCache tls_session_cache; // backend port. 0 if |host_unix| is true. uint16_t port; // true if |host| contains UNIX domain socket path. diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 86b9119b..f0b6fd3c 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -129,15 +129,7 @@ HttpDownstreamConnection::HttpDownstreamConnection( response_htp_{0}, group_(group) {} -HttpDownstreamConnection::~HttpDownstreamConnection() { - if (conn_.tls.ssl && conn_.tls.initial_handshake_done) { - auto session = SSL_get0_session(conn_.tls.ssl); - if (session) { - worker_->cache_client_tls_session(&addr_->addr, session, - ev_now(conn_.loop)); - } - } -} +HttpDownstreamConnection::~HttpDownstreamConnection() {} int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { if (LOG_ENABLED(INFO)) { @@ -241,7 +233,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str()); } - auto session = worker_->reuse_client_tls_session(&addr_->addr); + auto session = ssl::reuse_tls_session(addr_); if (session) { SSL_set_session(conn_.tls.ssl, session); SSL_SESSION_free(session); @@ -891,6 +883,13 @@ int HttpDownstreamConnection::tls_handshake() { return -1; } + if (!SSL_session_reused(conn_.tls.ssl)) { + auto session = SSL_get0_session(conn_.tls.ssl); + if (session) { + ssl::try_cache_tls_session(addr_, session, ev_now(conn_.loop)); + } + } + do_read_ = &HttpDownstreamConnection::read_tls; do_write_ = &HttpDownstreamConnection::write_tls; diff --git a/src/shrpx_http_downstream_connection.h b/src/shrpx_http_downstream_connection.h index b65db9ac..9689283d 100644 --- a/src/shrpx_http_downstream_connection.h +++ b/src/shrpx_http_downstream_connection.h @@ -82,7 +82,7 @@ private: // nullptr if TLS is not used. SSL_CTX *ssl_ctx_; // Address of remote endpoint - const DownstreamAddr *addr_; + DownstreamAddr *addr_; IOControl ioctrl_; http_parser response_htp_; size_t group_; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index aff475e6..2f9d200a 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -36,6 +36,7 @@ #include #include +#include #include #include @@ -1335,6 +1336,50 @@ CertLookupTree *create_cert_lookup_tree() { return new ssl::CertLookupTree(); } +namespace { +std::vector serialize_ssl_session(SSL_SESSION *session) { + auto len = i2d_SSL_SESSION(session, nullptr); + auto buf = std::vector(len); + auto p = buf.data(); + i2d_SSL_SESSION(session, &p); + + return buf; +} +} // namespace + +void try_cache_tls_session(DownstreamAddr *addr, SSL_SESSION *session, + ev_tstamp t) { + auto &cache = addr->tls_session_cache; + + if (cache.last_updated + 1_min > t) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Cache for addr=" << util::to_numeric_addr(&addr->addr) + << " is still host. Not updating."; + } + return; + } + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Update cache entry for SSL_SESSION=" << session + << ", addr=" << util::to_numeric_addr(&addr->addr) + << ", timestamp=" << std::fixed << std::setprecision(6) << t; + } + + cache.session_data = serialize_ssl_session(session); + cache.last_updated = t; +} + +SSL_SESSION *reuse_tls_session(const DownstreamAddr *addr) { + auto &cache = addr->tls_session_cache; + + if (cache.session_data.empty()) { + return nullptr; + } + + auto p = cache.session_data.data(); + return d2i_SSL_SESSION(nullptr, &p, cache.session_data.size()); +} + } // namespace ssl } // namespace shrpx diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h index 0def8875..c913e5b9 100644 --- a/src/shrpx_ssl.h +++ b/src/shrpx_ssl.h @@ -217,6 +217,17 @@ bool downstream_tls_enabled(); bool tls_hostname_match(const char *pattern, size_t plen, const char *hostname, size_t hlen); +// Caches |session| which is associated to remote address |addr|. +// |session| is serialized into ASN1 representation, and stored. |t| +// is used as a time stamp. Depending on the existing cache's time +// stamp, |session| might not be cached. +void try_cache_tls_session(DownstreamAddr *addr, SSL_SESSION *session, + ev_tstamp t); + +// Returns cached session associated |addr|. If no cache entry is +// found associated to |addr|, nullptr will be returned. +SSL_SESSION *reuse_tls_session(const DownstreamAddr *addr); + } // namespace ssl } // namespace shrpx diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 4b0fd452..2cb71acc 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -29,7 +29,6 @@ #endif // HAVE_UNISTD_H #include -#include #include "shrpx_ssl.h" #include "shrpx_log.h" @@ -312,61 +311,6 @@ mruby::MRubyContext *Worker::get_mruby_context() const { } #endif // HAVE_MRUBY -namespace { -std::vector serialize_ssl_session(SSL_SESSION *session) { - auto len = i2d_SSL_SESSION(session, nullptr); - auto buf = std::vector(len); - auto p = buf.data(); - i2d_SSL_SESSION(session, &p); - - return buf; -} -} // namespace - -void Worker::cache_client_tls_session(const Address *addr, SSL_SESSION *session, - ev_tstamp t) { - auto it = client_tls_session_cache_.find(addr); - if (it == std::end(client_tls_session_cache_)) { - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Create cache entry for SSL_SESSION=" << session - << ", addr=" << util::to_numeric_addr(addr) << "(" << addr - << "), timestamp=" << std::fixed << std::setprecision(6) << t; - } - client_tls_session_cache_.emplace( - addr, SessionCacheEntry{serialize_ssl_session(session), t}); - return; - } - - auto &ent = (*it).second; - if (ent.last_updated + 1_min > t) { - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Cache for addr=" << util::to_numeric_addr(addr) << "(" - << addr << ") is still host. Not updating."; - } - return; - } - - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Update cache entry for SSL_SESSION=" << session - << ", addr=" << util::to_numeric_addr(addr) << "(" << addr - << "), timestamp=" << std::fixed << std::setprecision(6) << t; - } - - ent.session_data = serialize_ssl_session(session); - ent.last_updated = t; -} - -SSL_SESSION *Worker::reuse_client_tls_session(const Address *addr) { - auto it = client_tls_session_cache_.find(addr); - if (it == std::end(client_tls_session_cache_)) { - return nullptr; - } - - const auto &ent = (*it).second; - auto p = ent.session_data.data(); - return d2i_SSL_SESSION(nullptr, &p, ent.session_data.size()); -} - std::vector &Worker::get_downstream_addr_groups() { return downstream_addr_groups_; } diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 9cd8b5ff..aa1531e6 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -101,14 +101,6 @@ struct WorkerEvent { std::shared_ptr ticket_keys; }; -struct SessionCacheEntry { - // ASN1 representation of SSL_SESSION object. See - // i2d_SSL_SESSION(3SSL). - std::vector session_data; - // The last time stamp when this cache entry is created or updated. - ev_tstamp last_updated; -}; - class Worker { public: Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, @@ -153,16 +145,6 @@ public: mruby::MRubyContext *get_mruby_context() const; #endif // HAVE_MRUBY - // Caches |session| which is associated to remote address |addr|. - // |session| is serialized into ASN1 representation, and stored. - // |t| is used as a time stamp. Depending on the existing cache's - // time stamp, |session| might not be cached. - void cache_client_tls_session(const Address *addr, SSL_SESSION *session, - ev_tstamp t); - // Returns cached session associated |addr|. If no cache entry is - // found associated to |addr|, nullptr will be returned. - SSL_SESSION *reuse_client_tls_session(const Address *addr); - std::vector &get_downstream_addr_groups(); ConnectBlocker *get_connect_blocker() const; @@ -181,11 +163,6 @@ private: WorkerStat worker_stat_; std::vector dgrps_; - // Client side SSL_SESSION cache. SSL_SESSION is associated to - // remote address. - std::unordered_map - client_tls_session_cache_; - std::unique_ptr session_cache_memcached_dispatcher_; #ifdef HAVE_MRUBY std::unique_ptr mruby_ctx_; From b68be1e1fb5b3f29fc531626279d2a64601918a1 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 16:44:00 +0900 Subject: [PATCH 76/85] src: Make token of type int32_t; we have no reason to use int16_t --- src/http2.cc | 16 ++++++++-------- src/http2.h | 20 ++++++++++---------- src/nghttp.cc | 31 ++----------------------------- src/nghttp.h | 7 ++----- src/shrpx_downstream.cc | 12 ++++++------ src/shrpx_downstream.h | 8 ++++---- 6 files changed, 32 insertions(+), 62 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index eba7aa54..6f347cc2 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -271,7 +271,7 @@ void copy_url_component(std::string &dest, const http_parser_url *u, int field, Headers::value_type to_header(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, - bool no_index, int16_t token) { + bool no_index, int32_t token) { return Header(std::string(reinterpret_cast(name), namelen), std::string(reinterpret_cast(value), valuelen), no_index, token); @@ -279,7 +279,7 @@ Headers::value_type to_header(const uint8_t *name, size_t namelen, void add_header(Headers &nva, const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, - int16_t token) { + int32_t token) { if (valuelen > 0) { size_t i, j; for (i = 0; i < valuelen && (value[i] == ' ' || value[i] == '\t'); ++i) @@ -760,7 +760,7 @@ void init_hdidx(HeaderIndex &hdidx) { std::fill(std::begin(hdidx), std::end(hdidx), -1); } -void index_header(HeaderIndex &hdidx, int16_t token, size_t idx) { +void index_header(HeaderIndex &hdidx, int32_t token, size_t idx) { if (token == -1) { return; } @@ -769,7 +769,7 @@ void index_header(HeaderIndex &hdidx, int16_t token, size_t idx) { } bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, - int16_t token) { + int32_t token) { switch (token) { case HD__AUTHORITY: case HD__METHOD: @@ -782,7 +782,7 @@ bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, } bool check_http2_response_pseudo_header(const HeaderIndex &hdidx, - int16_t token) { + int32_t token) { switch (token) { case HD__STATUS: return hdidx[token] == -1; @@ -791,7 +791,7 @@ bool check_http2_response_pseudo_header(const HeaderIndex &hdidx, } } -bool http2_header_allowed(int16_t token) { +bool http2_header_allowed(int32_t token) { switch (token) { case HD_CONNECTION: case HD_KEEP_ALIVE: @@ -813,7 +813,7 @@ bool http2_mandatory_request_headers_presence(const HeaderIndex &hdidx) { return true; } -const Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, +const Headers::value_type *get_header(const HeaderIndex &hdidx, int32_t token, const Headers &nva) { auto i = hdidx[token]; if (i == -1) { @@ -822,7 +822,7 @@ const Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, return &nva[i]; } -Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, +Headers::value_type *get_header(const HeaderIndex &hdidx, int32_t token, Headers &nva) { auto i = hdidx[token]; if (i == -1) { diff --git a/src/http2.h b/src/http2.h index 8eaeff56..58a4cc00 100644 --- a/src/http2.h +++ b/src/http2.h @@ -44,7 +44,7 @@ namespace nghttp2 { struct Header { Header(std::string name, std::string value, bool no_index = false, - int16_t token = -1) + int32_t token = -1) : name(std::move(name)), value(std::move(value)), token(token), @@ -62,7 +62,7 @@ struct Header { std::string name; std::string value; - int16_t token; + int32_t token; bool no_index; }; @@ -89,14 +89,14 @@ void copy_url_component(std::string &dest, const http_parser_url *u, int field, Headers::value_type to_header(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, - bool no_index, int16_t token); + bool no_index, int32_t token); // Add name/value pairs to |nva|. If |no_index| is true, this // name/value pair won't be indexed when it is forwarded to the next // hop. This function strips white spaces around |value|. void add_header(Headers &nva, const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, - int16_t token); + int32_t token); // Returns pointer to the entry in |nva| which has name |name|. If // more than one entries which have the name |name|, last occurrence @@ -277,30 +277,30 @@ int lookup_token(const std::string &name); // array containing at least HD_MAXIDX elements. void init_hdidx(HeaderIndex &hdidx); // Indexes header |token| using index |idx|. -void index_header(HeaderIndex &hdidx, int16_t token, size_t idx); +void index_header(HeaderIndex &hdidx, int32_t token, size_t idx); // Returns true if HTTP/2 request pseudo header |token| is not indexed // yet and not -1. -bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, int16_t token); +bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, int32_t token); // Returns true if HTTP/2 response pseudo header |token| is not // indexed yet and not -1. bool check_http2_response_pseudo_header(const HeaderIndex &hdidx, - int16_t token); + int32_t token); // Returns true if header field denoted by |token| is allowed for // HTTP/2. -bool http2_header_allowed(int16_t token); +bool http2_header_allowed(int32_t token); // Returns true that |hdidx| contains mandatory HTTP/2 request // headers. bool http2_mandatory_request_headers_presence(const HeaderIndex &hdidx); // Returns header denoted by |token| using index |hdidx|. -const Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, +const Headers::value_type *get_header(const HeaderIndex &hdidx, int32_t token, const Headers &nva); -Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, +Headers::value_type *get_header(const HeaderIndex &hdidx, int32_t token, Headers &nva); struct LinkHeader { diff --git a/src/nghttp.cc b/src/nghttp.cc index e3d4e110..a1705124 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -273,34 +273,7 @@ bool Request::is_ipv6_literal_addr() const { } } -bool Request::response_pseudo_header_allowed(int16_t token) const { - if (!res_nva.empty() && res_nva.back().name.c_str()[0] != ':') { - return false; - } - switch (token) { - case http2::HD__STATUS: - return res_hdidx[token] == -1; - default: - return false; - } -} - -bool Request::push_request_pseudo_header_allowed(int16_t token) const { - if (!req_nva.empty() && req_nva.back().name.c_str()[0] != ':') { - return false; - } - switch (token) { - case http2::HD__AUTHORITY: - case http2::HD__METHOD: - case http2::HD__PATH: - case http2::HD__SCHEME: - return req_hdidx[token] == -1; - default: - return false; - } -} - -Headers::value_type *Request::get_res_header(int16_t token) { +Headers::value_type *Request::get_res_header(int32_t token) { auto idx = res_hdidx[token]; if (idx == -1) { return nullptr; @@ -308,7 +281,7 @@ Headers::value_type *Request::get_res_header(int16_t token) { return &res_nva[idx]; } -Headers::value_type *Request::get_req_header(int16_t token) { +Headers::value_type *Request::get_req_header(int32_t token) { auto idx = req_hdidx[token]; if (idx == -1) { return nullptr; diff --git a/src/nghttp.h b/src/nghttp.h index 715a110e..df4ca685 100644 --- a/src/nghttp.h +++ b/src/nghttp.h @@ -125,11 +125,8 @@ struct Request { bool is_ipv6_literal_addr() const; - bool response_pseudo_header_allowed(int16_t token) const; - bool push_request_pseudo_header_allowed(int16_t token) const; - - Headers::value_type *get_res_header(int16_t token); - Headers::value_type *get_req_header(int16_t token); + Headers::value_type *get_res_header(int32_t token); + Headers::value_type *get_req_header(int32_t token); void record_request_start_time(); void record_response_start_time(); diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 32b7c1f2..0a8da837 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -331,7 +331,7 @@ void Downstream::crumble_request_cookie(std::vector &nva) { namespace { void add_header(bool &key_prev, size_t &sum, Headers &headers, const StringRef &name, const StringRef &value, bool no_index, - int16_t token) { + int32_t token) { key_prev = true; sum += name.size() + value.size(); headers.emplace_back(name.str(), value.str(), no_index, token); @@ -340,7 +340,7 @@ void add_header(bool &key_prev, size_t &sum, Headers &headers, namespace { void add_header(size_t &sum, Headers &headers, const StringRef &name, - const StringRef &value, bool no_index, int16_t token) { + const StringRef &value, bool no_index, int32_t token) { sum += name.size() + value.size(); headers.emplace_back(name.str(), value.str(), no_index, token); } @@ -388,7 +388,7 @@ int FieldStore::parse_content_length() { return 0; } -const Headers::value_type *FieldStore::header(int16_t token) const { +const Headers::value_type *FieldStore::header(int32_t token) const { for (auto it = headers_.rbegin(); it != headers_.rend(); ++it) { auto &kv = *it; if (kv.token == token) { @@ -398,7 +398,7 @@ const Headers::value_type *FieldStore::header(int16_t token) const { return nullptr; } -Headers::value_type *FieldStore::header(int16_t token) { +Headers::value_type *FieldStore::header(int32_t token) { for (auto it = headers_.rbegin(); it != headers_.rend(); ++it) { auto &kv = *it; if (kv.token == token) { @@ -422,7 +422,7 @@ void FieldStore::add_header_lower(const StringRef &name, const StringRef &value, } void FieldStore::add_header_token(const StringRef &name, const StringRef &value, - bool no_index, int16_t token) { + bool no_index, int32_t token) { shrpx::add_header(buffer_size_, headers_, name, value, no_index, token); } @@ -449,7 +449,7 @@ void FieldStore::add_trailer_lower(const StringRef &name, void FieldStore::add_trailer_token(const StringRef &name, const StringRef &value, bool no_index, - int16_t token) { + int32_t token) { // Header size limit should be applied to all header and trailer // fields combined. shrpx::add_header(buffer_size_, trailers_, name, value, no_index, token); diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 4306940d..3864674f 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -73,8 +73,8 @@ public: // multiple header have |name| as name, return last occurrence from // the beginning. If no such header is found, returns nullptr. // This function must be called after headers are indexed - const Headers::value_type *header(int16_t token) const; - Headers::value_type *header(int16_t token); + const Headers::value_type *header(int32_t token) const; + Headers::value_type *header(int32_t token); // Returns pointer to the header field with the name |name|. If no // such header is found, returns nullptr. const Headers::value_type *header(const StringRef &name) const; @@ -82,7 +82,7 @@ public: void add_header_lower(const StringRef &name, const StringRef &value, bool no_index); void add_header_token(const StringRef &name, const StringRef &value, - bool no_index, int16_t token); + bool no_index, int32_t token); void append_last_header_key(const char *data, size_t len); void append_last_header_value(const char *data, size_t len); @@ -99,7 +99,7 @@ public: void add_trailer_lower(const StringRef &name, const StringRef &value, bool no_index); void add_trailer_token(const StringRef &name, const StringRef &value, - bool no_index, int16_t token); + bool no_index, int32_t token); void append_last_trailer_key(const char *data, size_t len); void append_last_trailer_value(const char *data, size_t len); From 9672bc322f5fe744bdd0102a92bec7b34ed71490 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 16:47:06 +0900 Subject: [PATCH 77/85] src: Remove unused functions --- src/http2.cc | 45 ----------------------------------------- src/http2.h | 17 ---------------- src/http2_test.cc | 47 ------------------------------------------- src/http2_test.h | 3 --- src/shrpx-unittest.cc | 6 ------ 5 files changed, 118 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index 6f347cc2..f563596e 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -768,51 +768,6 @@ void index_header(HeaderIndex &hdidx, int32_t token, size_t idx) { hdidx[token] = idx; } -bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, - int32_t token) { - switch (token) { - case HD__AUTHORITY: - case HD__METHOD: - case HD__PATH: - case HD__SCHEME: - return hdidx[token] == -1; - default: - return false; - } -} - -bool check_http2_response_pseudo_header(const HeaderIndex &hdidx, - int32_t token) { - switch (token) { - case HD__STATUS: - return hdidx[token] == -1; - default: - return false; - } -} - -bool http2_header_allowed(int32_t token) { - switch (token) { - case HD_CONNECTION: - case HD_KEEP_ALIVE: - case HD_PROXY_CONNECTION: - case HD_TRANSFER_ENCODING: - case HD_UPGRADE: - return false; - default: - return true; - } -} - -bool http2_mandatory_request_headers_presence(const HeaderIndex &hdidx) { - if (hdidx[HD__METHOD] == -1 || hdidx[HD__PATH] == -1 || - hdidx[HD__SCHEME] == -1 || - (hdidx[HD__AUTHORITY] == -1 && hdidx[HD_HOST] == -1)) { - return false; - } - return true; -} - const Headers::value_type *get_header(const HeaderIndex &hdidx, int32_t token, const Headers &nva) { auto i = hdidx[token]; diff --git a/src/http2.h b/src/http2.h index 58a4cc00..785089c6 100644 --- a/src/http2.h +++ b/src/http2.h @@ -279,23 +279,6 @@ void init_hdidx(HeaderIndex &hdidx); // Indexes header |token| using index |idx|. void index_header(HeaderIndex &hdidx, int32_t token, size_t idx); -// Returns true if HTTP/2 request pseudo header |token| is not indexed -// yet and not -1. -bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, int32_t token); - -// Returns true if HTTP/2 response pseudo header |token| is not -// indexed yet and not -1. -bool check_http2_response_pseudo_header(const HeaderIndex &hdidx, - int32_t token); - -// Returns true if header field denoted by |token| is allowed for -// HTTP/2. -bool http2_header_allowed(int32_t token); - -// Returns true that |hdidx| contains mandatory HTTP/2 request -// headers. -bool http2_mandatory_request_headers_presence(const HeaderIndex &hdidx); - // Returns header denoted by |token| using index |hdidx|. const Headers::value_type *get_header(const HeaderIndex &hdidx, int32_t token, const Headers &nva); diff --git a/src/http2_test.cc b/src/http2_test.cc index 8f3aab35..34f96461 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -271,53 +271,6 @@ void test_http2_lookup_token(void) { CU_ASSERT(http2::HD_EXPECT == http2::lookup_token("expect")); } -void test_http2_check_http2_pseudo_header(void) { - http2::HeaderIndex hdidx; - http2::init_hdidx(hdidx); - - CU_ASSERT(http2::check_http2_request_pseudo_header(hdidx, http2::HD__METHOD)); - hdidx[http2::HD__PATH] = 0; - CU_ASSERT(http2::check_http2_request_pseudo_header(hdidx, http2::HD__METHOD)); - hdidx[http2::HD__METHOD] = 1; - CU_ASSERT( - !http2::check_http2_request_pseudo_header(hdidx, http2::HD__METHOD)); - CU_ASSERT(!http2::check_http2_request_pseudo_header(hdidx, http2::HD_VIA)); - - http2::init_hdidx(hdidx); - - CU_ASSERT( - http2::check_http2_response_pseudo_header(hdidx, http2::HD__STATUS)); - hdidx[http2::HD__STATUS] = 0; - CU_ASSERT( - !http2::check_http2_response_pseudo_header(hdidx, http2::HD__STATUS)); - CU_ASSERT(!http2::check_http2_response_pseudo_header(hdidx, http2::HD_VIA)); -} - -void test_http2_http2_header_allowed(void) { - CU_ASSERT(http2::http2_header_allowed(http2::HD__PATH)); - CU_ASSERT(http2::http2_header_allowed(http2::HD_CONTENT_LENGTH)); - CU_ASSERT(!http2::http2_header_allowed(http2::HD_CONNECTION)); -} - -void test_http2_mandatory_request_headers_presence(void) { - http2::HeaderIndex hdidx; - http2::init_hdidx(hdidx); - - CU_ASSERT(!http2::http2_mandatory_request_headers_presence(hdidx)); - hdidx[http2::HD__AUTHORITY] = 0; - CU_ASSERT(!http2::http2_mandatory_request_headers_presence(hdidx)); - hdidx[http2::HD__METHOD] = 1; - CU_ASSERT(!http2::http2_mandatory_request_headers_presence(hdidx)); - hdidx[http2::HD__PATH] = 2; - CU_ASSERT(!http2::http2_mandatory_request_headers_presence(hdidx)); - hdidx[http2::HD__SCHEME] = 3; - CU_ASSERT(http2::http2_mandatory_request_headers_presence(hdidx)); - - hdidx[http2::HD__AUTHORITY] = -1; - hdidx[http2::HD_HOST] = 0; - CU_ASSERT(http2::http2_mandatory_request_headers_presence(hdidx)); -} - void test_http2_parse_link_header(void) { { // only URI appears; we don't extract URI unless it bears rel=preload diff --git a/src/http2_test.h b/src/http2_test.h index 80f14cd8..029e8e6f 100644 --- a/src/http2_test.h +++ b/src/http2_test.h @@ -40,9 +40,6 @@ void test_http2_rewrite_location_uri(void); void test_http2_parse_http_status_code(void); void test_http2_index_header(void); void test_http2_lookup_token(void); -void test_http2_check_http2_pseudo_header(void); -void test_http2_http2_header_allowed(void); -void test_http2_mandatory_request_headers_presence(void); void test_http2_parse_link_header(void); void test_http2_path_join(void); void test_http2_normalize_path(void); diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index 0e7b17a1..8eba2c65 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -89,12 +89,6 @@ int main(int argc, char *argv[]) { shrpx::test_http2_index_header) || !CU_add_test(pSuite, "http2_lookup_token", shrpx::test_http2_lookup_token) || - !CU_add_test(pSuite, "http2_check_http2_pseudo_header", - shrpx::test_http2_check_http2_pseudo_header) || - !CU_add_test(pSuite, "http2_http2_header_allowed", - shrpx::test_http2_http2_header_allowed) || - !CU_add_test(pSuite, "http2_mandatory_request_headers_presence", - shrpx::test_http2_mandatory_request_headers_presence) || !CU_add_test(pSuite, "http2_parse_link_header", shrpx::test_http2_parse_link_header) || !CU_add_test(pSuite, "http2_path_join", shrpx::test_http2_path_join) || From 216ae0a328b4e9a51b022ff42aaf10a3c72fdc2d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 17:36:10 +0900 Subject: [PATCH 78/85] Update nghttpx documentation --- doc/nghttpx.1.rst | 6 + doc/sources/nghttpx-howto.rst | 212 +++++++++++++++++++++++----------- 2 files changed, 151 insertions(+), 67 deletions(-) diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index 42aa4642..49bb5e27 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -1474,6 +1474,12 @@ addresses: App.new +NOTES +----- + +1. nghttpx - HTTP/2 proxy - HOW-TO + https://nghttp2.org/documentation/nghttpx-howto.html + SEE ALSO -------- diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index 03500013..e38af2c4 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -1,3 +1,5 @@ +.. program:: nghttpx + nghttpx - HTTP/2 proxy - HOW-TO =============================== @@ -10,21 +12,22 @@ also covers some useful options later. Default mode ------------ -If nghttpx is invoked without any ``-s``, ``-p`` and ``--client``, it -operates in default mode. In this mode, nghttpx frontend listens for -HTTP/2 requests and translates them to HTTP/1 requests. Thus it works -as reverse proxy (gateway) for HTTP/2 clients to HTTP/1 web server. -This is also known as "HTTP/2 router". HTTP/1 requests are also -supported in frontend as a fallback. If nghttpx is linked with -spdylay library and frontend connection is SSL/TLS, the frontend also -supports SPDY protocol. +If nghttpx is invoked without any :option:`--http2-proxy`, +:option:`--client`, and :option:`--client-proxy`, it operates in +default mode. In this mode, nghttpx frontend listens for HTTP/2 +requests and translates them to HTTP/1 requests. Thus it works as +reverse proxy (gateway) for HTTP/2 clients to HTTP/1 web server. This +is also known as "HTTP/2 router". HTTP/1 requests are also supported +in frontend as a fallback. If nghttpx is linked with spdylay library +and frontend connection is SSL/TLS, the frontend also supports SPDY +protocol. By default, this mode's frontend connection is encrypted using SSL/TLS. So server's private key and certificate must be supplied to the command line (or through configuration file). In this case, the frontend protocol selection will be done via ALPN or NPN. -With ``--frontend-no-tls`` option, user can turn off SSL/TLS in +With :option:`--frontend-no-tls` option, user can turn off SSL/TLS in frontend connection. In this case, SPDY protocol is not available even if spdylay library is liked to nghttpx. HTTP/2 and HTTP/1 are available on the frontend and a HTTP/1 connection can be upgraded to @@ -32,8 +35,9 @@ HTTP/2 using HTTP Upgrade. Starting HTTP/2 connection by sending HTTP/2 connection preface is also supported. By default, backend HTTP/1 connections are not encrypted. To enable -TLS on HTTP/1 backend connections, use ``--backend-http1-tls`` option. -This applies to all mode whose backend connections are HTTP/1. +TLS on HTTP/1 backend connections, use :option:`--backend-http1-tls` +option. This applies to all mode whose backend connections are +HTTP/1. The backend is supposed to be HTTP/1 Web server. For example, to make nghttpx listen to encrypted HTTP/2 requests at port 8443, and a @@ -50,19 +54,19 @@ example, you can send GET request to the server using nghttp:: HTTP/2 proxy mode ----------------- -If nghttpx is invoked with ``-s`` option, it operates in HTTP/2 proxy -mode. The supported protocols in frontend and backend connections are -the same in `default mode`_. The difference is that this mode acts -like forward proxy and assumes the backend is HTTP/1 proxy server -(e.g., squid, traffic server). So HTTP/1 request must include -absolute URI in request line. +If nghttpx is invoked with :option:`--http2-proxy` (or its shorthand +:option:`-s`) option, it operates in HTTP/2 proxy mode. The supported +protocols in frontend and backend connections are the same in `default +mode`_. The difference is that this mode acts like forward proxy and +assumes the backend is HTTP/1 proxy server (e.g., squid, traffic +server). So HTTP/1 request must include absolute URI in request line. By default, frontend connection is encrypted. So this mode is also called secure proxy. If nghttpx is linked with spdylay, it supports SPDY protocols and it works as so called SPDY proxy. -With ``--frontend-no-tls`` option, SSL/TLS is turned off in frontend -connection, so the connection gets insecure. +With :option:`--frontend-no-tls` option, SSL/TLS is turned off in +frontend connection, so the connection gets insecure. The backend must be HTTP/1 proxy server. nghttpx supports multiple backend server addresses. It translates incoming requests to HTTP/1 @@ -96,7 +100,9 @@ Chromium require valid certificate for secure proxy. For Firefox, open Preference window and select Advanced then click Network tab. Clicking Connection Settings button will show the dialog. Select "Automatic proxy configuration URL" and enter the path -to proxy.pac file, something like this:: +to proxy.pac file, something like this: + +.. code-block:: text file:///path/to/proxy.pac @@ -112,25 +118,27 @@ configuration items to edit:: CONFIG proxy.config.url_remap.remap_required INT 0 Consult Traffic server `documentation -`_ +`_ to know how to configure traffic server as forward proxy and its security implications. Client mode ----------- -If nghttpx is invoked with ``--client`` option, it operates in client -mode. In this mode, nghttpx listens for plain, unencrypted HTTP/2 and -HTTP/1 requests and translates them to encrypted HTTP/2 requests to -the backend. User cannot enable SSL/TLS in frontend connection. +If nghttpx is invoked with :option:`--client` option, it operates in +client mode. In this mode, nghttpx listens for plain, unencrypted +HTTP/2 and HTTP/1 requests and translates them to encrypted HTTP/2 +requests to the backend. User cannot enable SSL/TLS in frontend +connection. HTTP/1 frontend connection can be upgraded to HTTP/2 using HTTP Upgrade. To disable SSL/TLS in backend connection, use -``--backend-no-tls`` option. +:option:`--backend-no-tls` option. By default, the number of backend HTTP/2 connections per worker -(thread) is determined by number of ``-b`` option. To adjust this -value, use ``--backend-http2-connections-per-worker`` option. +(thread) is determined by number of :option:`--backend` option. To +adjust this value, use +:option:`--backend-http2-connections-per-worker` option. The backend server is supporsed to be a HTTP/2 web server (e.g., nghttpd). The one use-case of this mode is utilize existing HTTP/1 @@ -142,9 +150,10 @@ mode to access to that web server:: .. note:: - You may need ``-k`` option if HTTP/2 server enables SSL/TLS and - its certificate is self-signed. But please note that it is - insecure. + You may need :option:`--insecure` (or its shorthand :option:`-k`) + option if HTTP/2 server enables SSL/TLS and its certificate is + self-signed. But please note that it is insecure, and you should + know what you are doing. Then you can use curl to access HTTP/2 server via nghttpx:: @@ -153,18 +162,19 @@ Then you can use curl to access HTTP/2 server via nghttpx:: Client proxy mode ----------------- -If nghttpx is invoked with ``-p`` option, it operates in client proxy -mode. This mode behaves like `client mode`_, but it works like -forward proxy. So HTTP/1 request must include absolute URI in request -line. +If nghttpx is invoked with :option:`--client-proxy` (or its shorthand +:option:`-p`) option, it operates in client proxy mode. This mode +behaves like `client mode`_, but it works like forward proxy. So +HTTP/1 request must include absolute URI in request line. HTTP/1 frontend connection can be upgraded to HTTP/2 using HTTP Upgrade. To disable SSL/TLS in backend connection, use -``--backend-no-tls`` option. +:option:`--backend-no-tls` option. By default, the number of backend HTTP/2 connections per worker -(thread) is determined by number of ``-b`` option. To adjust this -value, use ``--backend-http2-connections-per-worker`` option. +(thread) is determined by number of :option:`--backend` option. To +adjust this value, use +:option:`--backend-http2-connections-per-worker` option. The backend server must be a HTTP/2 proxy. You can use nghttpx in `HTTP/2 proxy mode`_ as backend server. The one use-case of this mode @@ -182,8 +192,9 @@ that server, invoke nghttpx like this:: .. note:: - You may need ``-k`` option if HTTP/2 server's certificate is - self-signed. But please note that it is insecure. + You may need :option:`--insecure` (or its shorthand :option:`-k`) + option if HTTP/2 server's certificate is self-signed. But please + note that it is insecure, and you should know what you are doing. Then you can use curl to issue HTTP request via HTTP/2 proxy:: @@ -195,23 +206,24 @@ proxy. HTTP/2 bridge mode ------------------ -If nghttpx is invoked with ``--http2-bridge`` option, it operates in -HTTP/2 bridge mode. The supported protocols in frontend connections -are the same in `default mode`_. The protocol in backend is HTTP/2 -only. +If nghttpx is invoked with :option:`--http2-bridge` option, it +operates in HTTP/2 bridge mode. The supported protocols in frontend +connections are the same in `default mode`_. The protocol in backend +is HTTP/2 only. -With ``--frontend-no-tls`` option, SSL/TLS is turned off in frontend -connection, so the connection gets insecure. To disable SSL/TLS in -backend connection, use ``--backend-no-tls`` option. +With :option:`--frontend-no-tls` option, SSL/TLS is turned off in +frontend connection, so the connection gets insecure. To disable +SSL/TLS in backend connection, use :option:`--backend-no-tls` option. By default, the number of backend HTTP/2 connections per worker -(thread) is determined by number of ``-b`` option. To adjust this -value, use ``--backend-http2-connections-per-worker`` option. +(thread) is determined by number of :option:`--backend` option. To +adjust this value, use +:option:`--backend-http2-connections-per-worker` option. The backend server is supporsed to be a HTTP/2 web server or HTTP/2 proxy. If backend server is HTTP/2 proxy, use -``--no-location-rewrite`` and ``--no-host-rewrite`` options to disable -rewriting location, host and :authority header field. +:option:`--no-location-rewrite` option to disable rewriting +``Location`` header field. The use-case of this mode is aggregate the incoming connections to one HTTP/2 connection. One backend HTTP/2 connection is created per @@ -222,26 +234,42 @@ Disable SSL/TLS In `default mode`_, `HTTP/2 proxy mode`_ and `HTTP/2 bridge mode`_, frontend connections are encrypted with SSL/TLS by default. To turn -off SSL/TLS, use ``--frontend-no-tls`` option. If this option is -used, the private key and certificate are not required to run nghttpx. +off SSL/TLS, use :option:`--frontend-no-tls` option. If this option +is used, the private key and certificate are not required to run +nghttpx. In `client mode`_, `client proxy mode`_ and `HTTP/2 bridge mode`_, backend connections are encrypted with SSL/TLS by default. To turn -off SSL/TLS, use ``--backend-no-tls`` option. +off SSL/TLS, use :option:`--backend-no-tls` option. + +Enable SSL/TLS on HTTP/1 backend +-------------------------------- + +In all modes which use HTTP/1 as backend protocol, backend HTTP/1 +connection is not encrypted by default. To enable encryption, use +:option:`--backend-http1-tls` option. + +Enable SSL/TLS on memcached connection +-------------------------------------- + +By default, memcached connection is not encrypted. To enable +encryption, use :option:`--tls-ticket-key-memcached-tls` for TLS +ticket key, and use :option:`--tls-session-cache-memcached-tls` for +TLS session cache. Specifying additional CA certificate ------------------------------------ By default, nghttpx tries to read CA certificate from system. But depending on the system you use, this may fail or is not supported. -To specify CA certificate manually, use ``--cacert`` option. The -specified file must be PEM format and can contain multiple +To specify CA certificate manually, use :option:`--cacert` option. +The specified file must be PEM format and can contain multiple certificates. By default, nghttpx validates server's certificate. If you want to turn off this validation, knowing this is really insecure and what you -are doing, you can use ``-k`` option to disable certificate -validation. +are doing, you can use :option:`--insecure` option to disable +certificate validation. Read/write rate limit --------------------- @@ -250,9 +278,9 @@ nghttpx supports transfer rate limiting on frontend connections. You can do rate limit per frontend connection for reading and writing individually. -To perform rate limit for reading, use ``--read-rate`` and -``--read-burst`` options. For writing, use ``--write-rate`` and -``--write-burst``. +To perform rate limit for reading, use :option:`--read-rate` and +:option:`--read-burst` options. For writing, use +:option:`--write-rate` and :option:`--write-burst`. Please note that rate limit is performed on top of TCP and nothing to do with HTTP/2 flow control. @@ -294,14 +322,64 @@ Re-opening log files When rotating log files, it is desirable to re-open log files after log rotation daemon renamed existing log files. To tell nghttpx to re-open log files, send USR1 signal to nghttpx process. It will -re-open files specified by ``--accesslog-file`` and -``--errorlog-file`` options. +re-open files specified by :option:`--accesslog-file` and +:option:`--errorlog-file` options. Multiple backend addresses -------------------------- nghttpx supports multiple backend addresses. To specify them, just -use ``-b`` option repeatedly. For example, to use backend1:8080 and -backend2:8080, use command-line like this: ``-bbackend1,8080 --bbackend2,8080``. For HTTP/2 backend, see also -``--backend-http2-connections-per-worker`` option. +use :option:`--backend` (or its shorthand :option:`-b`) option +repeatedly. For example, to use ``192.168.0.10:8080`` and +``192.168.0.11:8080``, use command-line like this: +``-b192.168.0.10,8080 -b192.168.0.11,8080``. In configuration file, +this looks like: + +.. code-block:: text + + backend=192.168.0.10,8080 + backend=192.168.0.11,8008 + +nghttpx can route request to different backend according to request +host and path. For example, to route request destined to host +``doc.example.com`` to backend server ``docserv:3000``, you can write +like so: + +.. code-block:: text + + backend=docserv,3000;doc.example.com/ + +When you write this option in command-line, you should enclose +argument with single or double quotes, since the character ``;`` has a +special meaning in shell. + +To route, request to request path whose prefix is ``/foo`` to backend +server ``[::1]:8080``, you can write like so: + +.. code-block:: text + + backend=::1,8080;/foo + +Of course, you can specify both host and request path at the same +time. + +One important thing you have to remember is that we have to specify +default routing pattern for so called "catch all" pattern. To write +"catch all" pattern, just specify backend server address, without +pattern. + +Usually, host is the value of ``Host`` header field. In HTTP/2, the +value of ``:authority`` pseudo header field is used. + +When you write multiple backend addresses sharing the same routing +pattern, they are used as load balancing. For example, to use 2 +servers ``serv1:3000`` and ``serv2:3000`` for request host +``example.com`` and path ``/myservice``, you can write like so: + +.. code-block:: text + + backend=serv1,3000;example.com/myservice + backend=serv2,3000;example.com/myservice + +For HTTP/2 backend, see also +:option:`--backend-http2-connections-per-worker` option. From 936d4aca1abf52cea9ce445da5dd4cbd734f9be7 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 17:46:48 +0900 Subject: [PATCH 79/85] Update h2load documentation --- doc/sources/h2load-howto.rst | 56 +++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/doc/sources/h2load-howto.rst b/doc/sources/h2load-howto.rst index f66c2c76..602c7ea2 100644 --- a/doc/sources/h2load-howto.rst +++ b/doc/sources/h2load-howto.rst @@ -1,18 +1,21 @@ +.. program:: h2load + h2load - HTTP/2 benchmarking tool - HOW-TO ========================================== -h2load is benchmarking tool for HTTP/2 and HTTP/1.1. If built with -spdylay (http://tatsuhiro-t.github.io/spdylay/) library, it also -supports SPDY protocol. It supports SSL/TLS and clear text for all -supported protocols. +:doc:`h2load.1` is benchmarking tool for HTTP/2 and HTTP/1.1. If +built with spdylay (http://tatsuhiro-t.github.io/spdylay/) library, it +also supports SPDY protocol. It supports SSL/TLS and clear text for +all supported protocols. Compiling from source --------------------- -``h2load`` is compiled alongside ``nghttp2`` and requires that the -``--enable-apps`` flag is passed to ``./configure`` and `required dependencies -`_ are available during -compilation. For details on compiling, see `nghttp2: Building from Git +h2load is compiled alongside nghttp2 and requires that the +``--enable-apps`` flag is passed to ``./configure`` and `required +dependencies `_ +are available during compilation. For details on compiling, see +`nghttp2: Building from Git `_. Basic Usage @@ -20,23 +23,21 @@ Basic Usage In order to set benchmark settings, specify following 3 options. -``-n`` +:option:`-n` The number of total requests. Default: 1 -``-c`` +:option:`-c` The number of concurrent clients. Default: 1 -``-m`` - The max concurrent streams to issue per client. - If ``auto`` is given, the number of given URIs is used. - Default: ``auto`` +:option:`-m` + The max concurrent streams to issue per client. Default: 1 For SSL/TLS connection, the protocol will be negotiated via ALPN/NPN. -You can set specific protocols in ``--npn-list`` option. For +You can set specific protocols in :option:`--npn-list` option. For cleartext connection, the default protocol is HTTP/2. To change the -protocol in cleartext connection, use ``--no-tls-proto`` option. For -convenience, ``--h1`` option forces HTTP/1.1 for both cleartext and -SSL/TLS connections. +protocol in cleartext connection, use :option:`--no-tls-proto` option. +For convenience, :option:`--h1` option forces HTTP/1.1 for both +cleartext and SSL/TLS connections. Here is a command-line to perform benchmark to URI \https://localhost using total 100000 requests, 100 concurrent clients and 10 max @@ -71,11 +72,11 @@ benchmarking results. By default, h2load uses large enough flow control window, which effectively disables flow control. To adjust receiver flow control window size, there are following options: -``-w`` +:option:`-w` Sets the stream level initial window size to (2**)-1. For SPDY, 2** is used instead. -``-W`` +:option:`-W` Sets the connection level initial window size to (2**)-1. For SPDY, if is strictly less than 16, this option is ignored. Otherwise @@ -85,17 +86,17 @@ Multi-Threading --------------- Sometimes benchmarking client itself becomes a bottleneck. To remedy -this situation, use ``-t`` option to specify the number of native +this situation, use :option:`-t` option to specify the number of native thread to use. -``-t`` +:option:`-t` The number of native threads. Default: 1 Selecting protocol for clear text --------------------------------- By default, if \http:// URI is given, HTTP/2 protocol is used. To -change the protocol to use for clear text, use ``-p`` option. +change the protocol to use for clear text, use :option:`-p` option. Multiple URIs ------------- @@ -106,3 +107,12 @@ If multiple URIs are specified, they are used in round robin manner. Please note that h2load uses scheme, host and port in the first URI and ignores those parts in the rest of the URIs. + +UNIX domain socket +------------------ + +To request against UNIX domain socket, use :option:`--base-uri`, and +specify ``unix:`` followed by the path to UNIX domain socket. For +example, if UNIX domain socket is ``/tmp/nghttpx.sock``, use +``--base-uri=unix:/tmp/nghttpx.sock``. h2load uses scheme, host and +port in the first URI in command-line or input file. From f3a415f6234818e66053dcfe621708d121101aaf Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 17:47:27 +0900 Subject: [PATCH 80/85] Update nghttpx documentation --- doc/sources/nghttpx-howto.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index e38af2c4..b387d220 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -3,11 +3,11 @@ nghttpx - HTTP/2 proxy - HOW-TO =============================== -nghttpx is a proxy translating protocols between HTTP/2 and other -protocols (e.g., HTTP/1, SPDY). It operates in several modes and each -mode may require additional programs to work with. This article -describes each operation mode and explains the intended use-cases. It -also covers some useful options later. +:doc:`nghttpx.1` is a proxy translating protocols between HTTP/2 and +other protocols (e.g., HTTP/1, SPDY). It operates in several modes +and each mode may require additional programs to work with. This +article describes each operation mode and explains the intended +use-cases. It also covers some useful options later. Default mode ------------ From 95ffb4565f7c8b56812f110f6a8dde6e131755cf Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 21 Feb 2016 18:20:53 +0900 Subject: [PATCH 81/85] Update nghttpx documentation --- doc/sources/nghttpx-howto.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index b387d220..ce142503 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -257,6 +257,12 @@ encryption, use :option:`--tls-ticket-key-memcached-tls` for TLS ticket key, and use :option:`--tls-session-cache-memcached-tls` for TLS session cache. +Specifying additional server certificates +----------------------------------------- + +nghttpx accepts additional server private key and certificate pairs +using :option:`--subcert` option. It can be used multiple times. + Specifying additional CA certificate ------------------------------------ From f4bb8776d0880650a8e8187790cd040630e090c7 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 23 Feb 2016 01:06:23 +0900 Subject: [PATCH 82/85] mruby: Clean up mrbgems as well --- third-party/Makefile.am | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/third-party/Makefile.am b/third-party/Makefile.am index 6e9bcf27..c1955eea 100644 --- a/third-party/Makefile.am +++ b/third-party/Makefile.am @@ -55,7 +55,9 @@ mruby: all-local: mruby clean-local: - -rm -rf "${abs_builddir}/mruby/build" + MRUBY_CONFIG="${srcdir}/build_config.rb" \ + BUILD_DIR="${abs_builddir}/mruby/build" \ + "${srcdir}/mruby/minirake" -f "${srcdir}/mruby/Rakefile" clean endif # HAVE_MRUBY From 3e72711e23cdaecdb3d82fa7efbb508e8ef1ead0 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 23 Feb 2016 01:09:45 +0900 Subject: [PATCH 83/85] Cap 100 limit for remembering idle streams --- lib/nghttp2_session.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 69f43d16..944a4790 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -1216,11 +1216,12 @@ int nghttp2_session_adjust_idle_stream(nghttp2_session *session) { size_t max; int rv; - /* Make minimum number of idle streams 16, which is arbitrary chosen - number. */ - max = nghttp2_max(16, - nghttp2_min(session->local_settings.max_concurrent_streams, - session->pending_local_max_concurrent_stream)); + /* Make minimum number of idle streams 16, and maximum 100, which + are arbitrary chosen numbers. */ + max = nghttp2_min( + 100, nghttp2_max( + 16, nghttp2_min(session->local_settings.max_concurrent_streams, + session->pending_local_max_concurrent_stream))); DEBUGF(fprintf(stderr, "stream: adjusting kept idle streams " "num_idle_streams=%zu, max=%zu\n", From 9d15f9b00d3c36709d287a9a17fa6f3351839733 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 23 Feb 2016 01:18:07 +0900 Subject: [PATCH 84/85] nghttpd: Start SETTINGS timer after it is written to output buffer --- src/HttpServer.cc | 15 +++++++++++++-- src/HttpServer.h | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/HttpServer.cc b/src/HttpServer.cc index e5406cdc..15650136 100644 --- a/src/HttpServer.cc +++ b/src/HttpServer.cc @@ -577,6 +577,10 @@ Http2Handler::WriteBuf *Http2Handler::get_wb() { return &wb_; } int Http2Handler::setup_bev() { return 0; } +void Http2Handler::start_settings_timer() { + ev_timer_start(sessions_->get_loop(), &settings_timerev_); +} + int Http2Handler::fill_wb() { if (data_pending_) { auto n = std::min(wb_.wleft(), data_pendinglen_); @@ -870,8 +874,6 @@ int Http2Handler::connection_made() { } } - ev_timer_start(sessions_->get_loop(), &settings_timerev_); - if (ssl_ && !nghttp2::ssl::check_http2_requirement(ssl_)) { terminate_session(NGHTTP2_INADEQUATE_SECURITY); } @@ -1538,6 +1540,15 @@ int hd_on_frame_send_callback(nghttp2_session *session, break; } + case NGHTTP2_SETTINGS: { + if (frame->hd.flags & NGHTTP2_FLAG_ACK) { + return 0; + } + + hd->start_settings_timer(); + + break; + } case NGHTTP2_PUSH_PROMISE: { auto promised_stream_id = frame->push_promise.promised_stream_id; auto promised_stream = hd->get_stream(promised_stream_id); diff --git a/src/HttpServer.h b/src/HttpServer.h index 99623397..27b5b2c9 100644 --- a/src/HttpServer.h +++ b/src/HttpServer.h @@ -138,6 +138,7 @@ public: void remove_self(); int setup_bev(); + void start_settings_timer(); int on_read(); int on_write(); int connection_made(); From 2782ef67de9ba332bcc8454d3afd5b0914a6f94c Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 23 Feb 2016 01:18:52 +0900 Subject: [PATCH 85/85] nghttpd: Remove unused function --- src/HttpServer.cc | 3 --- src/HttpServer.h | 1 - 2 files changed, 4 deletions(-) diff --git a/src/HttpServer.cc b/src/HttpServer.cc index 15650136..03140c3b 100644 --- a/src/HttpServer.cc +++ b/src/HttpServer.cc @@ -308,7 +308,6 @@ public: } auto handler = make_unique(this, fd, ssl, get_next_session_id()); - handler->setup_bev(); if (!ssl) { if (handler->connection_made() != 0) { return; @@ -575,8 +574,6 @@ struct ev_loop *Http2Handler::get_loop() const { Http2Handler::WriteBuf *Http2Handler::get_wb() { return &wb_; } -int Http2Handler::setup_bev() { return 0; } - void Http2Handler::start_settings_timer() { ev_timer_start(sessions_->get_loop(), &settings_timerev_); } diff --git a/src/HttpServer.h b/src/HttpServer.h index 27b5b2c9..0690307e 100644 --- a/src/HttpServer.h +++ b/src/HttpServer.h @@ -137,7 +137,6 @@ public: ~Http2Handler(); void remove_self(); - int setup_bev(); void start_settings_timer(); int on_read(); int on_write();