diff --git a/.travis.yml b/.travis.yml index 7239bd90..0191126e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,5 +48,5 @@ script: - make check - cd integration-tests - export GOPATH="$PWD/integration-tests/golang" - - make itprep-local - - make it-local + - make itprep + - make it diff --git a/AUTHORS b/AUTHORS index 95bc9542..8c4a70e8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1 +1,80 @@ -Tatsuhiro Tsujikawa +nghttp2 project was started as a fork of spdylay project [1]. Both +projects were started by Tatsuhiro Tsujikawa, who is still the main +author of these projects. Meanwhile, we have many contributions, and +we are not here without them. We sincerely thank you to all who made +a contribution. Here is the all individuals/organizations who +contributed to nghttp2 and spdylay project at which we forked. These +names are retrieved from git commit log. If you have made a +contribution, but you are missing in the list, please let us know via +github issues [2]. + +[1] https://github.com/tatsuhiro-t/spdylay +[2] https://github.com/tatsuhiro-t/nghttp2/issues + +-------- + +187j3x1 +Alek Storm +Alex Nalivko +Alexis La Goutte +Anders Bakken +Andreas Pohl +Andy Davies +Ant Bryan +Bernard Spil +Brian Card +Daniel Stenberg +Dave Reisner +David Beitey +David Weekly +Etienne Cimon +Fabian Möller +Fabian Wiesel +Gabi Davar +Janusz Dziemidowicz +Jay Satiro +Jim Morrison +José F. Calcerrada +Kamil Dudka +Kazuho Oku +Kenny (kang-yen) Peng +Kenny Peng +Kit Chan +Kyle Schomp +Lucas Pardue +MATSUMOTO Ryosuke +Mike Frysinger +Nicholas Hurley +Nora Shoemaker +Peeyush Aggarwal +Peter Wu +Piotr Sikora +Raul Gutierrez Segales +Remo E +Reza Tavakoli +Ross Smith II +Scott Mitchell +Stefan Eissing +Stephen Ludin +Sunpoet Po-Chuan Hsieh +Svante Signell +Syohei YOSHIDA +Tatsuhiko Kubo +Tatsuhiro Tsujikawa +Tom Harwood +Tomasz Buchert +Vernon Tang +Viacheslav Biriukov +Viktor Szépe +Xiaoguang Sun +Zhuoyun Wei +acesso +ayanamist +bxshi +es +fangdingjun +kumagi +mod-h2-dev +moparisthebest +snnn +yuuki-kodama diff --git a/CMakeLists.txt b/CMakeLists.txt index cf6be263..c7a0f4c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,14 +23,14 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. cmake_minimum_required(VERSION 3.0) -# XXX using 1.7.90 instead of 1.8.0-DEV -project(nghttp2 VERSION 1.7.90) +# XXX using 1.8.90 instead of 1.9.0-DEV +project(nghttp2 VERSION 1.8.90) # See versioning rule: # http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html -set(LT_CURRENT 18) -set(LT_REVISION 1) -set(LT_AGE 4) +set(LT_CURRENT 19) +set(LT_REVISION 0) +set(LT_AGE 5) set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") include(Version) @@ -280,6 +280,13 @@ check_function_exists(accept4 HAVE_ACCEPT4) include(CheckSymbolExists) # XXX does this correctly detect initgroups (un)availability on cygwin? check_symbol_exists(initgroups grp.h HAVE_DECL_INITGROUPS) +if(NOT HAVE_DECL_INITGROUPS AND HAVE_UNISTD_H) + # FreeBSD declares initgroups() in unistd.h + check_symbol_exists(initgroups unistd.h HAVE_DECL_INITGROUPS2) + if(HAVE_DECL_INITGROUPS2) + set(HAVE_DECL_INITGROUPS 1) + endif() +endif() check_function_exists(timerfd_create HAVE_TIMERFD_CREATE) # Checks for epoll availability, primarily for examples/tiny-nghttpd @@ -341,6 +348,8 @@ if(NOT CMAKE_C_COMPILER_ID STREQUAL "MSVC") -Wredundant-decls # Only work with Clang for the moment -Wheader-guard + # This is required because we pass format string as "const char*. + -Wno-format-nonliteral ) extract_valid_cxx_flags(WARNCXXFLAGS diff --git a/COPYING b/COPYING index 6e079b80..80201792 100644 --- a/COPYING +++ b/COPYING @@ -1,6 +1,7 @@ The MIT License Copyright (c) 2012, 2014, 2015, 2016 Tatsuhiro Tsujikawa +Copyright (c) 2012, 2014, 2015, 2016 nghttp2 contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/README.rst b/README.rst index 0f05d206..ef09fb1b 100644 --- a/README.rst +++ b/README.rst @@ -655,46 +655,38 @@ HTTP/2. ``nghttpx`` also offers the functionality to share session cache and ticket keys among multiple ``nghttpx`` instances via memcached. -``nghttpx`` has several operational modes: +``nghttpx`` has 2 operation modes: -================== ============================ ============== ============= -Mode option Frontend Backend Note -================== ============================ ============== ============= -default mode HTTP/2, SPDY, HTTP/1.1 (TLS) HTTP/1.1 Reverse proxy -``--http2-proxy`` HTTP/2, SPDY, HTTP/1.1 (TLS) HTTP/1.1 SPDY proxy -``--http2-bridge`` HTTP/2, SPDY, HTTP/1.1 (TLS) HTTP/2 (TLS) -``--client`` HTTP/2, HTTP/1.1 HTTP/2 (TLS) -``--client-proxy`` HTTP/2, HTTP/1.1 HTTP/2 (TLS) Forward proxy -================== ============================ ============== ============= +================== ====================== =================== ============= +Mode option Frontend Backend Note +================== ====================== =================== ============= +default mode HTTP/2, SPDY, HTTP/1.1 HTTP/1.1, HTTP/2 Reverse proxy +``--http2-proxy`` HTTP/2, SPDY, HTTP/1.1 HTTP/1.1, or HTTP/2 Forward proxy +================== ====================== =================== ============= The interesting mode at the moment is the default mode. It works like a reverse proxy and listens for HTTP/2, SPDY and HTTP/1.1 and can be deployed as a SSL/TLS terminator for existing web server. -The default mode, ``--http2-proxy`` and ``--http2-bridge`` modes use -SSL/TLS in the frontend connection by default. To disable SSL/TLS, -use the ``--frontend-no-tls`` option. If that option is used, SPDY is -disabled in the frontend and incoming HTTP/1.1 connections can be -upgraded to HTTP/2 through HTTP Upgrade. In these modes, HTTP/1 -backend connections are cleartext by default. To enable TLS, use -``--backend-http1-tls`` opiton. - -The ``--http2-bridge``, ``--client`` and ``--client-proxy`` modes use -SSL/TLS in the backend connection by default. To disable SSL/TLS, use -the ``--backend-no-tls`` option. +In all modes, the frontend connections are encrypted by SSL/TLS by +default. To disable encryption, use the ``--frontend-no-tls`` option. +If encryption is disabled, SPDY is disabled in the frontend and +incoming HTTP/1.1 connections can be upgraded to HTTP/2 through HTTP +Upgrade. On the other hard, backend connections are not encrypted by +default. To encrypt backend connections, use ``--backend-tls`` +option. ``nghttpx`` supports a configuration file. See the ``--conf`` option and sample configuration file ``nghttpx.conf.sample``. -In the default mode, (without any of ``--http2-proxy``, -``--http2-bridge``, ``--client-proxy`` and ``--client`` options), -``nghttpx`` works as reverse proxy to the backend server:: +In the default mode, ``nghttpx`` works as reverse proxy to the backend +server:: - Client <-- (HTTP/2, SPDY, HTTP/1.1) --> nghttpx <-- (HTTP/1.1) --> Web Server + Client <-- (HTTP/2, SPDY, HTTP/1.1) --> nghttpx <-- (HTTP/1.1, HTTP/2) --> Web Server [reverse proxy] -With the ``--http2-proxy`` option, it works as a so called secure proxy (aka -SPDY proxy):: +With the ``--http2-proxy`` option, it works as forward proxy, and it +is so called secure HTTP/2 proxy (aka SPDY proxy):: Client <-- (HTTP/2, SPDY, HTTP/1.1) --> nghttpx <-- (HTTP/1.1) --> Proxy [secure proxy] (e.g., Squid, ATS) @@ -702,9 +694,9 @@ SPDY proxy):: The ``Client`` in the above example needs to be configured to use ``nghttpx`` as secure proxy. -At the time of this writing, Chrome is the only browser which supports -secure proxy. One way to configure Chrome to use a secure proxy is -to create a proxy.pac script like this: +At the time of this writing, both Chrome and Firefox support secure +HTTP/2 proxy. One way to configure Chrome to use a secure proxy is to +create a proxy.pac script like this: .. code-block:: javascript @@ -720,37 +712,9 @@ Then run Chrome with the following arguments:: $ google-chrome --proxy-pac-url=file:///path/to/proxy.pac --use-npn -With ``--http2-bridge``, it accepts HTTP/2, SPDY and HTTP/1.1 -connections and communicates with the backend in HTTP/2:: - - Client <-- (HTTP/2, SPDY, HTTP/1.1) --> nghttpx <-- (HTTP/2) --> Web or HTTP/2 Proxy etc - (e.g., nghttpx -s) - -With ``--client-proxy``, it works as a forward proxy and expects -that the backend is an HTTP/2 proxy:: - - Client <-- (HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/2) --> HTTP/2 Proxy - [forward proxy] (e.g., nghttpx -s) - -The ``Client`` needs to be configured to use nghttpx as a forward -proxy. The frontend HTTP/1.1 connection can be upgraded to HTTP/2 -through HTTP Upgrade. With the above configuration, one can use -HTTP/1.1 client to access and test their HTTP/2 servers. - -With ``--client``, it works as a reverse proxy and expects that -the backend is an HTTP/2 Web server:: - - Client <-- (HTTP/2, HTTP/1.1) --> nghttpx <-- (HTTP/2) --> Web Server - [reverse proxy] - -The frontend HTTP/1.1 connection can be upgraded to HTTP/2 -through HTTP Upgrade. - -For the operation modes which talk to the backend in HTTP/2 over -SSL/TLS, the backend connections can be tunneled through an HTTP proxy. +The backend HTTP/2 connections can be tunneled through an HTTP proxy. The proxy is specified using ``--backend-http-proxy-uri``. The -following figure illustrates the example of the ``--http2-bridge`` and -``--backend-http-proxy-uri`` options to talk to the outside HTTP/2 +following figure illustrates how nghttpx talks to the outside HTTP/2 proxy through an HTTP proxy:: Client <-- (HTTP/2, SPDY, HTTP/1.1) --> nghttpx <-- (HTTP/2) -- diff --git a/configure.ac b/configure.ac index 985925d7..b8afe63d 100644 --- a/configure.ac +++ b/configure.ac @@ -25,7 +25,7 @@ dnl Do not change user variables! dnl http://www.gnu.org/software/automake/manual/html_node/Flag-Variables-Ordering.html AC_PREREQ(2.61) -AC_INIT([nghttp2], [1.8.0-DEV], [t-tujikawa@users.sourceforge.net]) +AC_INIT([nghttp2], [1.9.0-DEV], [t-tujikawa@users.sourceforge.net]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_HEADERS([config.h]) @@ -40,15 +40,13 @@ AC_CANONICAL_TARGET AM_INIT_AUTOMAKE([subdir-objects]) -# AM_EXTRA_RECURSIVE_TARGETS requires automake 1.13 or higher -m4_ifdef([AM_EXTRA_RECURSIVE_TARGETS], [AM_EXTRA_RECURSIVE_TARGETS([it itprep])]) m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) dnl See versioning rule: dnl http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.html -AC_SUBST(LT_CURRENT, 18) -AC_SUBST(LT_REVISION, 1) -AC_SUBST(LT_AGE, 4) +AC_SUBST(LT_CURRENT, 19) +AC_SUBST(LT_REVISION, 0) +AC_SUBST(LT_AGE, 5) major=`echo $PACKAGE_VERSION |cut -d. -f1 | sed -e "s/[^0-9]//g"` minor=`echo $PACKAGE_VERSION |cut -d. -f2 | sed -e "s/[^0-9]//g"` @@ -654,8 +652,13 @@ AC_CHECK_FUNC([timerfd_create], # For cygwin: we can link initgroups, so AC_CHECK_FUNCS succeeds, but # cygwin disables initgroups due to feature test macro magic with our -# configuration. -AC_CHECK_DECLS([initgroups], [], [], [[#include ]]) +# configuration. FreeBSD declares initgroups() in unistd.h. +AC_CHECK_DECLS([initgroups], [], [], [[ + #ifdef HAVE_UNISTD_H + # include + #endif + #include +]]) # Checks for epoll availability, primarily for examples/tiny-nghttpd AX_HAVE_EPOLL([have_epoll=yes], [have_epoll=no]) @@ -718,6 +721,9 @@ if test "x$werror" != "xno"; then # Only work with Clang for the moment AX_CHECK_COMPILE_FLAG([-Wheader-guard], [CFLAGS="$CFLAGS -Wheader-guard"]) + # This is required because we pass format string as "const char*. + AX_CHECK_COMPILE_FLAG([-Wno-format-nonliteral], [CFLAGS="$CFLAGS -Wno-format-nonliteral"]) + # For C++ compiler AC_LANG_PUSH(C++) AX_CHECK_COMPILE_FLAG([-Wall], [CXXFLAGS="$CXXFLAGS -Wall"]) diff --git a/contrib/Makefile.am b/contrib/Makefile.am index 089a79b0..5a02e2f0 100644 --- a/contrib/Makefile.am +++ b/contrib/Makefile.am @@ -31,13 +31,16 @@ EXTRA_DIST = \ edit = sed -e 's|@bindir[@]|$(bindir)|g' -nghttpx-init: %: $(srcdir)/%.in +nghttpx-init: $(srcdir)/nghttpx-init.in rm -f $@ $@.tmp $(edit) $< > $@.tmp chmod +x $@.tmp mv $@.tmp $@ -nghttpx.service nghttpx-upstart.conf: %: $(srcdir)/%.in +nghttpx.service: $(srcdir)/nghttpx.service.in + $(edit) $< > $@ + +nghttpx-upstart.conf: $(srcdir)/nghttpx-upstart.conf.in $(edit) $< > $@ $(configfiles): Makefile diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index be8078a8..1fa04f4f 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -25,37 +25,48 @@ set(APIDOCS nghttp2_hd_inflate_hd.rst nghttp2_hd_inflate_new.rst nghttp2_hd_inflate_new2.rst + nghttp2_http2_strerror.rst nghttp2_is_fatal.rst nghttp2_nv_compare_name.rst nghttp2_option_del.rst nghttp2_option_new.rst nghttp2_option_set_max_reserved_remote_streams.rst + nghttp2_option_set_no_auto_ping_ack.rst nghttp2_option_set_no_auto_window_update.rst nghttp2_option_set_no_http_messaging.rst nghttp2_option_set_no_recv_client_magic.rst nghttp2_option_set_peer_max_concurrent_streams.rst + nghttp2_option_set_user_recv_extension_type.rst nghttp2_pack_settings_payload.rst nghttp2_priority_spec_check_default.rst nghttp2_priority_spec_default_init.rst nghttp2_priority_spec_init.rst + nghttp2_rcbuf_decref.rst + nghttp2_rcbuf_get_buf.rst + nghttp2_rcbuf_incref.rst nghttp2_select_next_protocol.rst nghttp2_session_callbacks_del.rst nghttp2_session_callbacks_new.rst nghttp2_session_callbacks_set_before_frame_send_callback.rst nghttp2_session_callbacks_set_data_source_read_length_callback.rst + nghttp2_session_callbacks_set_error_callback.rst nghttp2_session_callbacks_set_on_begin_frame_callback.rst nghttp2_session_callbacks_set_on_begin_headers_callback.rst nghttp2_session_callbacks_set_on_data_chunk_recv_callback.rst nghttp2_session_callbacks_set_on_frame_not_send_callback.rst nghttp2_session_callbacks_set_on_frame_recv_callback.rst + nghttp2_session_callbacks_set_on_extension_chunk_recv_callback.rst nghttp2_session_callbacks_set_on_frame_send_callback.rst nghttp2_session_callbacks_set_on_header_callback.rst + nghttp2_session_callbacks_set_on_header_callback2.rst nghttp2_session_callbacks_set_on_invalid_frame_recv_callback.rst nghttp2_session_callbacks_set_on_stream_close_callback.rst + nghttp2_session_callbacks_set_pack_extension_callback.rst nghttp2_session_callbacks_set_recv_callback.rst nghttp2_session_callbacks_set_select_padding_callback.rst nghttp2_session_callbacks_set_send_callback.rst nghttp2_session_callbacks_set_send_data_callback.rst + nghttp2_session_callbacks_set_unpack_extension_callback.rst nghttp2_session_client_new.rst nghttp2_session_client_new2.rst nghttp2_session_client_new3.rst @@ -107,6 +118,7 @@ set(APIDOCS nghttp2_stream_get_weight.rst nghttp2_strerror.rst nghttp2_submit_data.rst + nghttp2_submit_extension.rst nghttp2_submit_goaway.rst nghttp2_submit_headers.rst nghttp2_submit_ping.rst diff --git a/doc/Makefile.am b/doc/Makefile.am index 777fe226..8badad47 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -49,37 +49,48 @@ APIDOCS= \ nghttp2_hd_inflate_hd.rst \ nghttp2_hd_inflate_new.rst \ nghttp2_hd_inflate_new2.rst \ + nghttp2_http2_strerror.rst \ nghttp2_is_fatal.rst \ nghttp2_nv_compare_name.rst \ nghttp2_option_del.rst \ nghttp2_option_new.rst \ nghttp2_option_set_max_reserved_remote_streams.rst \ + nghttp2_option_set_no_auto_ping_ack.rst \ nghttp2_option_set_no_auto_window_update.rst \ nghttp2_option_set_no_http_messaging.rst \ nghttp2_option_set_no_recv_client_magic.rst \ nghttp2_option_set_peer_max_concurrent_streams.rst \ + nghttp2_option_set_user_recv_extension_type.rst \ nghttp2_pack_settings_payload.rst \ nghttp2_priority_spec_check_default.rst \ nghttp2_priority_spec_default_init.rst \ nghttp2_priority_spec_init.rst \ + nghttp2_rcbuf_decref.rst \ + nghttp2_rcbuf_get_buf.rst \ + nghttp2_rcbuf_incref.rst \ nghttp2_select_next_protocol.rst \ nghttp2_session_callbacks_del.rst \ nghttp2_session_callbacks_new.rst \ nghttp2_session_callbacks_set_before_frame_send_callback.rst \ nghttp2_session_callbacks_set_data_source_read_length_callback.rst \ + nghttp2_session_callbacks_set_error_callback.rst \ nghttp2_session_callbacks_set_on_begin_frame_callback.rst \ nghttp2_session_callbacks_set_on_begin_headers_callback.rst \ nghttp2_session_callbacks_set_on_data_chunk_recv_callback.rst \ + nghttp2_session_callbacks_set_on_extension_chunk_recv_callback.rst \ nghttp2_session_callbacks_set_on_frame_not_send_callback.rst \ nghttp2_session_callbacks_set_on_frame_recv_callback.rst \ nghttp2_session_callbacks_set_on_frame_send_callback.rst \ nghttp2_session_callbacks_set_on_header_callback.rst \ + nghttp2_session_callbacks_set_on_header_callback2.rst \ nghttp2_session_callbacks_set_on_invalid_frame_recv_callback.rst \ nghttp2_session_callbacks_set_on_stream_close_callback.rst \ + nghttp2_session_callbacks_set_pack_extension_callback.rst \ nghttp2_session_callbacks_set_recv_callback.rst \ nghttp2_session_callbacks_set_select_padding_callback.rst \ nghttp2_session_callbacks_set_send_callback.rst \ nghttp2_session_callbacks_set_send_data_callback.rst \ + nghttp2_session_callbacks_set_unpack_extension_callback.rst \ nghttp2_session_client_new.rst \ nghttp2_session_client_new2.rst \ nghttp2_session_client_new3.rst \ @@ -131,6 +142,7 @@ APIDOCS= \ nghttp2_stream_get_weight.rst \ nghttp2_strerror.rst \ nghttp2_submit_data.rst \ + nghttp2_submit_extension.rst \ nghttp2_submit_goaway.rst \ nghttp2_submit_headers.rst \ nghttp2_submit_ping.rst \ @@ -239,6 +251,7 @@ apiref.rst: \ $(APIDOCS): apiref.rst clean-local: + [ $(srcdir) = $(builddir) ] || for i in $(RST_FILES); do [ -e $(builddir)/$$i ] && rm $(builddir)/$$i; done -rm -f apiref.rst -rm -f $(APIDOCS) -rm -rf $(BUILDDIR)/* diff --git a/doc/bash_completion/nghttpx b/doc/bash_completion/nghttpx index e127d178..7e56320f 100644 --- a/doc/bash_completion/nghttpx +++ b/doc/bash_completion/nghttpx @@ -8,7 +8,7 @@ _nghttpx() _get_comp_words_by_ref cur prev case $cur in -*) - COMPREPLY=( $( compgen -W '--worker-read-rate --frontend-no-tls --frontend-http2-dump-response-header --backend-http1-connections-per-frontend --tls-ticket-key-file --verify-client-cacert --include --max-response-header-fields --backend-request-buffer --max-request-header-fields --backend-http2-connection-window-bits --backend-tls-session-cache-per-worker --conf --worker-write-burst --npn-list --fetch-ocsp-response-file --no-http2-cipher-black-list --mruby-file --stream-read-timeout --tls-ticket-key-memcached --forwarded-for --accesslog-syslog --frontend-http2-read-timeout --listener-disable-timeout --frontend-http2-connection-window-bits --ciphers --strip-incoming-x-forwarded-for --private-key-passwd-file --backend-keep-alive-timeout --backend-http-proxy-uri --backend-http1-connections-per-host --rlimit-nofile --tls-dyn-rec-warmup-threshold --no-via --ocsp-update-interval --backend-write-timeout --client --tls-ticket-key-memcached-max-retry --http2-no-cookie-crumbling --worker-read-burst --client-proxy --http2-bridge --accesslog-format --errorlog-syslog --request-header-field-buffer --errorlog-file --http2-max-concurrent-streams --frontend-write-timeout --tls-ticket-key-cipher --read-burst --backend-ipv4 --backend-ipv6 --backend --insecure --log-level --host-rewrite --tls-proto-list --backend-http2-connections-per-worker --tls-ticket-key-memcached-interval --dh-param-file --worker-frontend-connections --backend-http1-tls --syslog-facility --fastopen --no-location-rewrite --tls-session-cache-memcached --no-ocsp --backend-response-buffer --workers --add-forwarded --frontend-http2-window-bits --worker-write-rate --add-request-header --backend-tls-sni-field --subcert --help --frontend-frame-debug --pid-file --frontend-http2-dump-request-header --daemon --write-rate --altsvc --user --add-x-forwarded-for --frontend-read-timeout --tls-ticket-key-memcached-max-fail --backlog --write-burst --no-server-push --backend-http2-window-bits --response-header-field-buffer --padding --stream-write-timeout --cacert --forwarded-by --version --add-response-header --backend-read-timeout --frontend --accesslog-file --http2-proxy --backend-no-tls --client-private-key-file --client-cert-file --accept-proxy-protocol --tls-dyn-rec-idle-timeout --verify-client --read-rate --strip-incoming-forwarded ' -- "$cur" ) ) + COMPREPLY=( $( compgen -W '--worker-read-rate --frontend-no-tls --frontend-http2-dump-response-header --backend-http1-connections-per-frontend --tls-ticket-key-file --verify-client-cacert --include --max-response-header-fields --backend-request-buffer --max-request-header-fields --backend-http2-connection-window-bits --conf --worker-write-burst --npn-list --fetch-ocsp-response-file --no-via --tls-session-cache-memcached-cert-file --no-http2-cipher-black-list --mruby-file --stream-read-timeout --tls-ticket-key-memcached --forwarded-for --accesslog-syslog --frontend-http2-read-timeout --listener-disable-timeout --frontend-http2-connection-window-bits --ciphers --strip-incoming-x-forwarded-for --private-key-passwd-file --backend-keep-alive-timeout --backend-http-proxy-uri --backend-http1-connections-per-host --rlimit-nofile --tls-dyn-rec-warmup-threshold --tls-ticket-key-memcached-cert-file --ocsp-update-interval --forwarded-by --tls-session-cache-memcached-private-key-file --backend-write-timeout --client --tls-ticket-key-memcached-max-retry --http2-no-cookie-crumbling --worker-read-burst --client-proxy --http2-bridge --accesslog-format --errorlog-syslog --request-header-field-buffer --errorlog-file --http2-max-concurrent-streams --frontend-write-timeout --tls-ticket-key-cipher --read-burst --backend --insecure --log-level --host-rewrite --tls-session-cache-memcached-tls --tls-proto-list --backend-http2-connections-per-worker --tls-ticket-key-memcached-interval --dh-param-file --worker-frontend-connections --backend-http1-tls --syslog-facility --fastopen --no-location-rewrite --tls-ticket-key-memcached-tls --tls-session-cache-memcached --no-ocsp --backend-response-buffer --workers --add-forwarded --frontend-http2-window-bits --worker-write-rate --add-request-header --backend-tls-sni-field --subcert --help --frontend-frame-debug --pid-file --frontend-http2-dump-request-header --daemon --write-rate --altsvc --user --add-x-forwarded-for --frontend-read-timeout --tls-ticket-key-memcached-max-fail --backlog --write-burst --no-server-push --backend-http2-window-bits --response-header-field-buffer --tls-ticket-key-memcached-address-family --padding --tls-session-cache-memcached-address-family --stream-write-timeout --cacert --tls-ticket-key-memcached-private-key-file --backend-address-family --version --add-response-header --backend-read-timeout --frontend --accesslog-file --http2-proxy --backend-no-tls --client-private-key-file --client-cert-file --accept-proxy-protocol --tls-dyn-rec-idle-timeout --verify-client --read-rate --strip-incoming-forwarded ' -- "$cur" ) ) ;; *) _filedir diff --git a/doc/h2load.1 b/doc/h2load.1 index a6db9aea..623fa633 100644 --- a/doc/h2load.1 +++ b/doc/h2load.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "H2LOAD" "1" "February 14, 2016" "1.8.0-DEV" "nghttp2" +.TH "H2LOAD" "1" "February 29, 2016" "1.9.0-DEV" "nghttp2" .SH NAME h2load \- HTTP/2 benchmarking tool . diff --git a/doc/nghttp.1 b/doc/nghttp.1 index 24beec68..c825350a 100644 --- a/doc/nghttp.1 +++ b/doc/nghttp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTP" "1" "February 14, 2016" "1.8.0-DEV" "nghttp2" +.TH "NGHTTP" "1" "February 29, 2016" "1.9.0-DEV" "nghttp2" .SH NAME nghttp \- HTTP/2 client . diff --git a/doc/nghttpd.1 b/doc/nghttpd.1 index a3b7bc9f..fbb76af3 100644 --- a/doc/nghttpd.1 +++ b/doc/nghttpd.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPD" "1" "February 14, 2016" "1.8.0-DEV" "nghttp2" +.TH "NGHTTPD" "1" "February 29, 2016" "1.9.0-DEV" "nghttp2" .SH NAME nghttpd \- HTTP/2 server . diff --git a/doc/nghttpx.1 b/doc/nghttpx.1 index a89f545f..c615bd69 100644 --- a/doc/nghttpx.1 +++ b/doc/nghttpx.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "NGHTTPX" "1" "February 14, 2016" "1.8.0-DEV" "nghttp2" +.TH "NGHTTPX" "1" "February 29, 2016" "1.9.0-DEV" "nghttp2" .SH NAME nghttpx \- HTTP/2 proxy . @@ -39,15 +39,15 @@ A reverse proxy for HTTP/2, HTTP/1 and SPDY. .INDENT 0.0 .TP .B -Set path to server\(aqs private key. Required unless \fI\%\-p\fP, -\fI\%\-\-client\fP or \fI\%\-\-frontend\-no\-tls\fP are given. +Set path to server\(aqs private key. Required unless +\fI\%\-\-frontend\-no\-tls\fP are given. .UNINDENT .INDENT 0.0 .TP .B -Set path to server\(aqs certificate. Required unless \fI\%\-p\fP, -\fI\%\-\-client\fP or \fI\%\-\-frontend\-no\-tls\fP are given. To make OCSP -stapling work, this must be absolute path. +Set path to server\(aqs certificate. Required unless +\fI\%\-\-frontend\-no\-tls\fP are given. To make OCSP stapling +work, this must be an absolute path. .UNINDENT .SH OPTIONS .sp @@ -55,38 +55,39 @@ The options are categorized into several groups. .SS Connections .INDENT 0.0 .TP -.B \-b, \-\-backend=(,|unix:)[;[:...]] +.B \-b, \-\-backend=(,|unix:)[;[[:...]][;proto=]] Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX domain socket can be specified by prefixing path name with "unix:" (e.g., unix:/var/run/backend.sock). .sp Optionally, if s are given, the backend address -is only used if request matches the pattern. If \fI\%\-s\fP or -\fI\%\-p\fP is used, s are ignored. The pattern -matching is closely designed to ServeMux in net/http -package of Go programming language. consists -of path, host + path or just host. The path must start -with "\fI/\fP". If it ends with "\fI/\fP", it matches all request -path in its subtree. To deal with the request to the -directory without trailing slash, the path which ends -with "\fI/\fP" also matches the request path which only lacks -trailing \(aq\fI/\fP\(aq (e.g., path "\fI/foo/\fP" matches request path -"\fI/foo\fP"). If it does not end with "\fI/\fP", it performs exact -match against the request path. If host is given, it -performs exact match against the request host. If host -alone is given, "\fI/\fP" is appended to it, so that it -matches all request paths under the host (e.g., -specifying "nghttp2.org" equals to "nghttp2.org/"). +is only used if request matches the pattern. If +\fI\%\-\-http2\-proxy\fP is used, s are ignored. The +pattern matching is closely designed to ServeMux in +net/http package of Go programming language. +consists of path, host + path or just host. The path +must start with "\fI/\fP". If it ends with "\fI/\fP", it matches +all request path in its subtree. To deal with the +request to the directory without trailing slash, the +path which ends with "\fI/\fP" also matches the request path +which only lacks trailing \(aq\fI/\fP\(aq (e.g., path "\fI/foo/\fP" +matches request path "\fI/foo\fP"). If it does not end with +"\fI/\fP", it performs exact match against the request path. +If host is given, it performs exact match against the +request host. If host alone is given, "\fI/\fP" is appended +to it, so that it matches all request paths under the +host (e.g., specifying "nghttp2.org" equals to +"nghttp2.org/"). .sp Patterns with host take precedence over patterns with just path. Then, longer patterns take precedence over shorter ones, breaking a tie by the order of the appearance in the configuration. .sp -If is omitted, "\fI/\fP" is used as pattern, which -matches all request paths (catch\-all pattern). The -catch\-all backend must be given. +If is omitted or empty string, "\fI/\fP" is used as +pattern, which matches all request paths (catch\-all +pattern). The catch\-all backend must be given. .sp When doing a match, nghttpx made some normalization to pattern, request host and path. For host part, they are @@ -109,6 +110,15 @@ and \fI\%\-b\fP\(aq127.0.0.1,8080;www.nghttp2.org\(aq. The backend addresses sharing same are grouped together forming load balancing group. .sp +Optionally, backend application protocol can be +specified in . All that share the same +must have the same value if it is given. + should be one of the following list without +quotes: "h2", "http/1.1". The default value of +is "http/1.1". Note that usually "h2" refers to HTTP/2 +over TLS. But in this option, it may mean HTTP/2 over +cleartext TCP unless \fI\%\-\-backend\-tls\fP is used. +.sp Since ";" and ":" are used as delimiter, must not contain these characters. Since ";" has special meaning in shell, the option value must be quoted. @@ -167,19 +177,8 @@ Accept PROXY protocol version 1 on frontend connection. .UNINDENT .INDENT 0.0 .TP -.B \-\-backend\-no\-tls -Disable SSL/TLS on backend connections. For HTTP/2 -backend connections, TLS is enabled by default. For -HTTP/1 backend connections, TLS is disabled by default, -and can be enabled by \fI\%\-\-backend\-http1\-tls\fP option. If -both \fI\%\-\-backend\-no\-tls\fP and \fI\%\-\-backend\-http1\-tls\fP options -are used, \fI\%\-\-backend\-no\-tls\fP has the precedence. -.UNINDENT -.INDENT 0.0 -.TP -.B \-\-backend\-http1\-tls -Enable SSL/TLS on backend HTTP/1 connections. See also -\fI\%\-\-backend\-no\-tls\fP option. +.B \-\-backend\-tls +Enable SSL/TLS on backend connections. .UNINDENT .SS Performance .INDENT 0.0 @@ -269,37 +268,27 @@ Default: \fB0\fP .UNINDENT .INDENT 0.0 .TP -.B \-\-backend\-http2\-connections\-per\-worker= -Set maximum number of backend HTTP/2 physical -connections per worker. If pattern is used in \fI\%\-b\fP -option, this limit is applied to each pattern group (in -other words, each pattern group can have maximum -HTTP/2 connections). The default value is 0, which -means that the value is adjusted to the number of -backend addresses. If pattern is used, this adjustment -is done for each pattern group. -.UNINDENT -.INDENT 0.0 -.TP -.B \-\-backend\-http1\-connections\-per\-host= -Set maximum number of backend concurrent HTTP/1 -connections per origin host. This option is meaningful -when \fI\%\-s\fP option is used. The origin host is determined -by authority portion of request URI (or :authority -header field for HTTP/2). To limit the number of -connections per frontend for default mode, use -\fI\%\-\-backend\-http1\-connections\-per\-frontend\fP\&. +.B \-\-backend\-connections\-per\-host= +Set maximum number of backend concurrent connections +(and/or streams in case of HTTP/2) per origin host. +This option is meaningful when \fI\%\-\-http2\-proxy\fP option is +used. The origin host is determined by authority +portion of request URI (or :authority header field for +HTTP/2). To limit the number of connections per +frontend for default mode, use +\fI\%\-\-backend\-connections\-per\-frontend\fP\&. .sp Default: \fB8\fP .UNINDENT .INDENT 0.0 .TP -.B \-\-backend\-http1\-connections\-per\-frontend= -Set maximum number of backend concurrent HTTP/1 -connections per frontend. This option is only used for -default mode. 0 means unlimited. To limit the number -of connections per host for HTTP/2 or SPDY proxy mode -(\-s option), use \fI\%\-\-backend\-http1\-connections\-per\-host\fP\&. +.B \-\-backend\-connections\-per\-frontend= +Set maximum number of backend concurrent connections +(and/or streams in case of HTTP/2) per frontend. This +option is only used for default mode. 0 means +unlimited. To limit the number of connections per host +with \fI\%\-\-http2\-proxy\fP option, use +\fI\%\-\-backend\-connections\-per\-host\fP\&. .sp Default: \fB0\fP .UNINDENT @@ -697,20 +686,22 @@ Allow black listed cipher suite on HTTP/2 connection. See \fI\%https://tools.ietf.org/html/rfc7540#appendix\-A\fP for the complete HTTP/2 cipher suites black list. .UNINDENT -.INDENT 0.0 -.TP -.B \-\-backend\-tls\-session\-cache\-per\-worker= -Set the maximum number of backend TLS session cache -stored per worker. -.sp -Default: \fB10000\fP -.UNINDENT .SS HTTP/2 and SPDY .INDENT 0.0 .TP -.B \-c, \-\-http2\-max\-concurrent\-streams= +.B \-c, \-\-frontend\-http2\-max\-concurrent\-streams= Set the maximum number of the concurrent streams in one -HTTP/2 and SPDY session. +frontend HTTP/2 and SPDY session. +.sp +Default: \(ga\(ga 100\(ga\(ga +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-backend\-http2\-max\-concurrent\-streams= +Set the maximum number of the concurrent streams in one +backend HTTP/2 session. This sets maximum number of +concurrent opened pushed streams. The maximum number of +concurrent requests are set by a remote server. .sp Default: \fB100\fP .UNINDENT @@ -751,7 +742,7 @@ Default: \fB16\fP Sets the per\-connection window size of HTTP/2 backend connection to 2**\-1. .sp -Default: \fB16\fP +Default: \fB30\fP .UNINDENT .INDENT 0.0 .TP @@ -772,11 +763,10 @@ protocol security. Disable HTTP/2 server push. Server push is supported by default mode and HTTP/2 frontend via Link header field. It is also supported if both frontend and backend are -HTTP/2 (which implies \fI\%\-\-http2\-bridge\fP or \fI\%\-\-client\fP mode). -In this case, server push from backend session is -relayed to frontend, and server push via Link header -field is also supported. HTTP SPDY frontend does not -support server push. +HTTP/2 in default mode. In this case, server push from +backend session is relayed to frontend, and server push +via Link header field is also supported. SPDY frontend +does not support server push. .UNINDENT .SS Mode .INDENT 0.0 @@ -785,39 +775,13 @@ support server push. Accept HTTP/2, SPDY and HTTP/1.1 over SSL/TLS. If \fI\%\-\-frontend\-no\-tls\fP is used, accept HTTP/2 and HTTP/1.1. The incoming HTTP/1.1 connection can be upgraded to -HTTP/2 through HTTP Upgrade. The protocol to the -backend is HTTP/1.1. +HTTP/2 through HTTP Upgrade. .UNINDENT .INDENT 0.0 .TP .B \-s, \-\-http2\-proxy -Like default mode, but enable secure proxy mode. -.UNINDENT -.INDENT 0.0 -.TP -.B \-\-http2\-bridge -Like default mode, but communicate with the backend in -HTTP/2 over SSL/TLS. Thus the incoming all connections -are converted to HTTP/2 connection and relayed to the -backend. See \fI\%\-\-backend\-http\-proxy\-uri\fP option if you are -behind the proxy and want to connect to the outside -HTTP/2 proxy. -.UNINDENT -.INDENT 0.0 -.TP -.B \-\-client -Accept HTTP/2 and HTTP/1.1 without SSL/TLS. The -incoming HTTP/1.1 connection can be upgraded to HTTP/2 -connection through HTTP Upgrade. The protocol to the -backend is HTTP/2. To use nghttpx as a forward proxy, -use \fI\%\-p\fP option instead. -.UNINDENT -.INDENT 0.0 -.TP -.B \-p, \-\-client\-proxy -Like \fI\%\-\-client\fP option, but it also requires the request -path from frontend must be an absolute URI, suitable for -use as a forward proxy. +Like default mode, but enable forward proxy. This is so +called HTTP/2 proxy mode. .UNINDENT .SS Logging .INDENT 0.0 @@ -983,18 +947,16 @@ is received, it is left unaltered. .INDENT 0.0 .TP .B \-\-no\-location\-rewrite -Don\(aqt rewrite location header field on \fI\%\-\-http2\-bridge\fP, -\fI\%\-\-client\fP and default mode. For \fI\%\-\-http2\-proxy\fP and -\fI\%\-\-client\-proxy\fP mode, location header field will not be -altered regardless of this option. +Don\(aqt rewrite location header field in default mode. +When \fI\%\-\-http2\-proxy\fP is used, location header field will +not be altered regardless of this option. .UNINDENT .INDENT 0.0 .TP .B \-\-host\-rewrite -Rewrite host and :authority header fields on -\fI\%\-\-http2\-bridge\fP, \fI\%\-\-client\fP and default mode. For -\fI\%\-\-http2\-proxy\fP and \fI\%\-\-client\-proxy\fP mode, these headers -will not be altered regardless of this option. +Rewrite host and :authority header fields in default +mode. When \fI\%\-\-http2\-proxy\fP is used, these headers will +not be altered regardless of this option. .UNINDENT .INDENT 0.0 .TP @@ -1266,12 +1228,12 @@ associated stream\(aqs status code must be 200. This limitation may be loosened in the future release. .sp nghttpx also supports server push if both frontend and backend are -HTTP/2 (which implies \fI\%\-\-http2\-bridge\fP or \fI\%\-\-client\fP). -In this case, in addition to server push via Link header field, server -push from backend is relayed to frontend HTTP/2 session. +HTTP/2 in default mode. In this case, in addition to server push via +Link header field, server push from backend is forwarded to frontend +HTTP/2 session. .sp -HTTP/2 server push will be disabled if \fI\%\-\-http2\-proxy\fP or -\fI\%\-\-client\-proxy\fP is used. +HTTP/2 server push will be disabled if \fI\%\-\-http2\-proxy\fP is +used. .SH UNIX DOMAIN SOCKET .sp nghttpx supports UNIX domain socket with a filename for both frontend diff --git a/doc/nghttpx.1.rst b/doc/nghttpx.1.rst index 42aa4642..afccc2e9 100644 --- a/doc/nghttpx.1.rst +++ b/doc/nghttpx.1.rst @@ -19,14 +19,14 @@ A reverse proxy for HTTP/2, HTTP/1 and SPDY. .. describe:: - Set path to server's private key. Required unless :option:`-p`\, - :option:`--client` or :option:`\--frontend-no-tls` are given. + Set path to server's private key. Required unless + :option:`--frontend-no-tls` are given. .. describe:: - Set path to server's certificate. Required unless :option:`-p`\, - :option:`--client` or :option:`\--frontend-no-tls` are given. To make OCSP - stapling work, this must be absolute path. + Set path to server's certificate. Required unless + :option:`--frontend-no-tls` are given. To make OCSP stapling + work, this must be an absolute path. OPTIONS @@ -37,7 +37,7 @@ The options are categorized into several groups. Connections ~~~~~~~~~~~ -.. option:: -b, --backend=(,|unix:)[;[:...]] +.. option:: -b, --backend=(,|unix:)[;[[:...]][;proto=]] Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX @@ -45,31 +45,32 @@ Connections with "unix:" (e.g., unix:/var/run/backend.sock). Optionally, if s are given, the backend address - is only used if request matches the pattern. If :option:`-s` or - :option:`-p` is used, s are ignored. The pattern - matching is closely designed to ServeMux in net/http - package of Go programming language. consists - of path, host + path or just host. The path must start - with "*/*". If it ends with "*/*", it matches all request - path in its subtree. To deal with the request to the - directory without trailing slash, the path which ends - with "*/*" also matches the request path which only lacks - trailing '*/*' (e.g., path "*/foo/*" matches request path - "*/foo*"). If it does not end with "*/*", it performs exact - match against the request path. If host is given, it - performs exact match against the request host. If host - alone is given, "*/*" is appended to it, so that it - matches all request paths under the host (e.g., - specifying "nghttp2.org" equals to "nghttp2.org/"). + is only used if request matches the pattern. If + :option:`--http2-proxy` is used, s are ignored. The + pattern matching is closely designed to ServeMux in + net/http package of Go programming language. + consists of path, host + path or just host. The path + must start with "*/*". If it ends with "*/*", it matches + all request path in its subtree. To deal with the + request to the directory without trailing slash, the + path which ends with "*/*" also matches the request path + which only lacks trailing '*/*' (e.g., path "*/foo/*" + matches request path "*/foo*"). If it does not end with + "*/*", it performs exact match against the request path. + If host is given, it performs exact match against the + request host. If host alone is given, "*/*" is appended + to it, so that it matches all request paths under the + host (e.g., specifying "nghttp2.org" equals to + "nghttp2.org/"). Patterns with host take precedence over patterns with just path. Then, longer patterns take precedence over shorter ones, breaking a tie by the order of the appearance in the configuration. - If is omitted, "*/*" is used as pattern, which - matches all request paths (catch-all pattern). The - catch-all backend must be given. + If is omitted or empty string, "*/*" is used as + pattern, which matches all request paths (catch-all + pattern). The catch-all backend must be given. When doing a match, nghttpx made some normalization to pattern, request host and path. For host part, they are @@ -92,6 +93,15 @@ Connections The backend addresses sharing same are grouped together forming load balancing group. + Optionally, backend application protocol can be + specified in . All that share the same + must have the same value if it is given. + should be one of the following list without + quotes: "h2", "http/1.1". The default value of + is "http/1.1". Note that usually "h2" refers to HTTP/2 + over TLS. But in this option, it may mean HTTP/2 over + cleartext TCP unless :option:`--backend-tls` is used. + Since ";" and ":" are used as delimiter, must not contain these characters. Since ";" has special meaning in shell, the option value must be quoted. @@ -144,19 +154,9 @@ Connections Accept PROXY protocol version 1 on frontend connection. -.. option:: --backend-no-tls +.. option:: --backend-tls - Disable SSL/TLS on backend connections. For HTTP/2 - backend connections, TLS is enabled by default. For - HTTP/1 backend connections, TLS is disabled by default, - and can be enabled by :option:`--backend-http1-tls` option. If - both :option:`--backend-no-tls` and :option:`\--backend-http1-tls` options - are used, :option:`--backend-no-tls` has the precedence. - -.. option:: --backend-http1-tls - - Enable SSL/TLS on backend HTTP/1 connections. See also - :option:`--backend-no-tls` option. + Enable SSL/TLS on backend connections. Performance @@ -237,36 +237,27 @@ Performance Default: ``0`` -.. option:: --backend-http2-connections-per-worker= +.. option:: --backend-connections-per-host= - Set maximum number of backend HTTP/2 physical - connections per worker. If pattern is used in :option:`-b` - option, this limit is applied to each pattern group (in - other words, each pattern group can have maximum - HTTP/2 connections). The default value is 0, which - means that the value is adjusted to the number of - backend addresses. If pattern is used, this adjustment - is done for each pattern group. - -.. option:: --backend-http1-connections-per-host= - - Set maximum number of backend concurrent HTTP/1 - connections per origin host. This option is meaningful - when :option:`-s` option is used. The origin host is determined - by authority portion of request URI (or :authority - header field for HTTP/2). To limit the number of - connections per frontend for default mode, use - :option:`--backend-http1-connections-per-frontend`\. + Set maximum number of backend concurrent connections + (and/or streams in case of HTTP/2) per origin host. + This option is meaningful when :option:`--http2-proxy` option is + used. The origin host is determined by authority + portion of request URI (or :authority header field for + HTTP/2). To limit the number of connections per + frontend for default mode, use + :option:`--backend-connections-per-frontend`\. Default: ``8`` -.. option:: --backend-http1-connections-per-frontend= +.. option:: --backend-connections-per-frontend= - Set maximum number of backend concurrent HTTP/1 - connections per frontend. This option is only used for - default mode. 0 means unlimited. To limit the number - of connections per host for HTTP/2 or SPDY proxy mode - (-s option), use :option:`--backend-http1-connections-per-host`\. + Set maximum number of backend concurrent connections + (and/or streams in case of HTTP/2) per frontend. This + option is only used for default mode. 0 means + unlimited. To limit the number of connections per host + with :option:`--http2-proxy` option, use + :option:`--backend-connections-per-host`\. Default: ``0`` @@ -622,21 +613,23 @@ SSL/TLS See https://tools.ietf.org/html/rfc7540#appendix-A for the complete HTTP/2 cipher suites black list. -.. option:: --backend-tls-session-cache-per-worker= - - Set the maximum number of backend TLS session cache - stored per worker. - - Default: ``10000`` - HTTP/2 and SPDY ~~~~~~~~~~~~~~~ -.. option:: -c, --http2-max-concurrent-streams= +.. option:: -c, --frontend-http2-max-concurrent-streams= Set the maximum number of the concurrent streams in one - HTTP/2 and SPDY session. + frontend HTTP/2 and SPDY session. + + Default: `` 100`` + +.. option:: --backend-http2-max-concurrent-streams= + + Set the maximum number of the concurrent streams in one + backend HTTP/2 session. This sets maximum number of + concurrent opened pushed streams. The maximum number of + concurrent requests are set by a remote server. Default: ``100`` @@ -672,7 +665,7 @@ HTTP/2 and SPDY Sets the per-connection window size of HTTP/2 backend connection to 2\*\*-1. - Default: ``16`` + Default: ``30`` .. option:: --http2-no-cookie-crumbling @@ -690,11 +683,10 @@ HTTP/2 and SPDY Disable HTTP/2 server push. Server push is supported by default mode and HTTP/2 frontend via Link header field. It is also supported if both frontend and backend are - HTTP/2 (which implies :option:`--http2-bridge` or :option:`\--client` mode). - In this case, server push from backend session is - relayed to frontend, and server push via Link header - field is also supported. HTTP SPDY frontend does not - support server push. + HTTP/2 in default mode. In this case, server push from + backend session is relayed to frontend, and server push + via Link header field is also supported. SPDY frontend + does not support server push. Mode @@ -706,35 +698,12 @@ Mode Accept HTTP/2, SPDY and HTTP/1.1 over SSL/TLS. If :option:`--frontend-no-tls` is used, accept HTTP/2 and HTTP/1.1. The incoming HTTP/1.1 connection can be upgraded to - HTTP/2 through HTTP Upgrade. The protocol to the - backend is HTTP/1.1. + HTTP/2 through HTTP Upgrade. .. option:: -s, --http2-proxy - Like default mode, but enable secure proxy mode. - -.. option:: --http2-bridge - - Like default mode, but communicate with the backend in - HTTP/2 over SSL/TLS. Thus the incoming all connections - are converted to HTTP/2 connection and relayed to the - backend. See :option:`--backend-http-proxy-uri` option if you are - behind the proxy and want to connect to the outside - HTTP/2 proxy. - -.. option:: --client - - Accept HTTP/2 and HTTP/1.1 without SSL/TLS. The - incoming HTTP/1.1 connection can be upgraded to HTTP/2 - connection through HTTP Upgrade. The protocol to the - backend is HTTP/2. To use nghttpx as a forward proxy, - use :option:`-p` option instead. - -.. option:: -p, --client-proxy - - Like :option:`--client` option, but it also requires the request - path from frontend must be an absolute URI, suitable for - use as a forward proxy. + Like default mode, but enable forward proxy. This is so + called HTTP/2 proxy mode. Logging @@ -875,17 +844,15 @@ HTTP .. option:: --no-location-rewrite - Don't rewrite location header field on :option:`--http2-bridge`\, - :option:`--client` and default mode. For :option:`\--http2-proxy` and - :option:`--client-proxy` mode, location header field will not be - altered regardless of this option. + Don't rewrite location header field in default mode. + When :option:`--http2-proxy` is used, location header field will + not be altered regardless of this option. .. option:: --host-rewrite - Rewrite host and :authority header fields on - :option:`--http2-bridge`\, :option:`--client` and default mode. For - :option:`--http2-proxy` and :option:`\--client-proxy` mode, these headers - will not be altered regardless of this option. + Rewrite host and :authority header fields in default + mode. When :option:`--http2-proxy` is used, these headers will + not be altered regardless of this option. .. option:: --altsvc= @@ -1133,12 +1100,12 @@ Currently, the following restriction is applied for server push: This limitation may be loosened in the future release. nghttpx also supports server push if both frontend and backend are -HTTP/2 (which implies :option:`--http2-bridge` or :option:`--client`). -In this case, in addition to server push via Link header field, server -push from backend is relayed to frontend HTTP/2 session. +HTTP/2 in default mode. In this case, in addition to server push via +Link header field, server push from backend is forwarded to frontend +HTTP/2 session. -HTTP/2 server push will be disabled if :option:`--http2-proxy` or -:option:`--client-proxy` is used. +HTTP/2 server push will be disabled if :option:`--http2-proxy` is +used. UNIX DOMAIN SOCKET ------------------ diff --git a/doc/nghttpx.h2r b/doc/nghttpx.h2r index 14512bb0..a37812cb 100644 --- a/doc/nghttpx.h2r +++ b/doc/nghttpx.h2r @@ -98,12 +98,12 @@ Currently, the following restriction is applied for server push: This limitation may be loosened in the future release. nghttpx also supports server push if both frontend and backend are -HTTP/2 (which implies :option:`--http2-bridge` or :option:`--client`). -In this case, in addition to server push via Link header field, server -push from backend is relayed to frontend HTTP/2 session. +HTTP/2 in default mode. In this case, in addition to server push via +Link header field, server push from backend is forwarded to frontend +HTTP/2 session. -HTTP/2 server push will be disabled if :option:`--http2-proxy` or -:option:`--client-proxy` is used. +HTTP/2 server push will be disabled if :option:`--http2-proxy` is +used. UNIX DOMAIN SOCKET ------------------ diff --git a/doc/programmers-guide.rst b/doc/programmers-guide.rst index 94c72e98..253c9fd3 100644 --- a/doc/programmers-guide.rst +++ b/doc/programmers-guide.rst @@ -1,6 +1,62 @@ Programmers' Guide ================== +Architecture +------------ + +The most notable point in nghttp2 library architecture is it does not +perform any I/O. nghttp2 only performs HTTP/2 protocol stuff based on +input byte strings. It will calls callback functions set by +applications while processing input. The output of nghttp2 is just +byte string. An application is responsible to send these output to +the remote peer. The callback functions may be called while producing +output. + +Not doing I/O makes embedding nghttp2 library in the existing code +base very easy. Usually, the existing applications have its own I/O +event loops. It is very hard to use nghttp2 in that situation if +nghttp2 does its own I/O. It also makes light weight language wrapper +for nghttp2 easy with the same reason. The down side is that an +application author has to write more code to write complete +application using nghttp2. This is especially true for simple "toy" +application. For the real applications, however, this is not the +case. This is because you probably want to support HTTP/1 which +nghttp2 does not provide, and to do that, you will need to write your +own HTTP/1 stack or use existing third-party library, and bind them +together with nghttp2 and I/O event loop. In this point, not +performing I/O in nghttp2 has more point than doing it. + +The primary object that an application uses is :type:`nghttp2_session` +object, which is opaque struct and its details are hidden in order to +ensure the upgrading its internal architecture without breaking the +backward compatibility. An application can set callbacks to +:type:`nghttp2_session` object through the dedicated object and +functions, and it also interacts with it via many API function calls. + +An application can create as many :type:`nghttp2_session` object as it +wants. But single :type:`nghttp2_session` object must be used by a +single thread at the same time. This is not so hard to enforce since +most event-based architecture applicatons use is single thread per +core, and handling one connection I/O is done by single thread. + +To feed input to :type:`nghttp2_session` object, one can use +`nghttp2_session_recv()` or `nghttp2_session_mem_recv()` functions. +They behave similarly, and the difference is that +`nghttp2_session_recv()` will use :type:`nghttp2_read_callback` to get +input. On the other hand, `nghttp2_session_mem_recv()` will take +input as its parameter. If in doubt, use `nghttp2_session_mem_recv()` +since it is simpler, and could be faster since it avoids calling +callback function. + +To get output from :type:`nghttp2_session` object, one can use +`nghttp2_session_send()` or `nghttp2_session_mem_send()`. The +difference between them is that the former uses +:type:`nghttp2_send_callback` to pass output to an application. On +the other hand, the latter returns the output to the caller. If in +doubt, use `nghttp2_session_mem_send()` since it is simpler. But +`nghttp2_session_send()` might be easier to use if the output buffer +an application has is fixed sized. + Includes -------- diff --git a/doc/sources/h2load-howto.rst b/doc/sources/h2load-howto.rst index f66c2c76..602c7ea2 100644 --- a/doc/sources/h2load-howto.rst +++ b/doc/sources/h2load-howto.rst @@ -1,18 +1,21 @@ +.. program:: h2load + h2load - HTTP/2 benchmarking tool - HOW-TO ========================================== -h2load is benchmarking tool for HTTP/2 and HTTP/1.1. If built with -spdylay (http://tatsuhiro-t.github.io/spdylay/) library, it also -supports SPDY protocol. It supports SSL/TLS and clear text for all -supported protocols. +:doc:`h2load.1` is benchmarking tool for HTTP/2 and HTTP/1.1. If +built with spdylay (http://tatsuhiro-t.github.io/spdylay/) library, it +also supports SPDY protocol. It supports SSL/TLS and clear text for +all supported protocols. Compiling from source --------------------- -``h2load`` is compiled alongside ``nghttp2`` and requires that the -``--enable-apps`` flag is passed to ``./configure`` and `required dependencies -`_ are available during -compilation. For details on compiling, see `nghttp2: Building from Git +h2load is compiled alongside nghttp2 and requires that the +``--enable-apps`` flag is passed to ``./configure`` and `required +dependencies `_ +are available during compilation. For details on compiling, see +`nghttp2: Building from Git `_. Basic Usage @@ -20,23 +23,21 @@ Basic Usage In order to set benchmark settings, specify following 3 options. -``-n`` +:option:`-n` The number of total requests. Default: 1 -``-c`` +:option:`-c` The number of concurrent clients. Default: 1 -``-m`` - The max concurrent streams to issue per client. - If ``auto`` is given, the number of given URIs is used. - Default: ``auto`` +:option:`-m` + The max concurrent streams to issue per client. Default: 1 For SSL/TLS connection, the protocol will be negotiated via ALPN/NPN. -You can set specific protocols in ``--npn-list`` option. For +You can set specific protocols in :option:`--npn-list` option. For cleartext connection, the default protocol is HTTP/2. To change the -protocol in cleartext connection, use ``--no-tls-proto`` option. For -convenience, ``--h1`` option forces HTTP/1.1 for both cleartext and -SSL/TLS connections. +protocol in cleartext connection, use :option:`--no-tls-proto` option. +For convenience, :option:`--h1` option forces HTTP/1.1 for both +cleartext and SSL/TLS connections. Here is a command-line to perform benchmark to URI \https://localhost using total 100000 requests, 100 concurrent clients and 10 max @@ -71,11 +72,11 @@ benchmarking results. By default, h2load uses large enough flow control window, which effectively disables flow control. To adjust receiver flow control window size, there are following options: -``-w`` +:option:`-w` Sets the stream level initial window size to (2**)-1. For SPDY, 2** is used instead. -``-W`` +:option:`-W` Sets the connection level initial window size to (2**)-1. For SPDY, if is strictly less than 16, this option is ignored. Otherwise @@ -85,17 +86,17 @@ Multi-Threading --------------- Sometimes benchmarking client itself becomes a bottleneck. To remedy -this situation, use ``-t`` option to specify the number of native +this situation, use :option:`-t` option to specify the number of native thread to use. -``-t`` +:option:`-t` The number of native threads. Default: 1 Selecting protocol for clear text --------------------------------- By default, if \http:// URI is given, HTTP/2 protocol is used. To -change the protocol to use for clear text, use ``-p`` option. +change the protocol to use for clear text, use :option:`-p` option. Multiple URIs ------------- @@ -106,3 +107,12 @@ If multiple URIs are specified, they are used in round robin manner. Please note that h2load uses scheme, host and port in the first URI and ignores those parts in the rest of the URIs. + +UNIX domain socket +------------------ + +To request against UNIX domain socket, use :option:`--base-uri`, and +specify ``unix:`` followed by the path to UNIX domain socket. For +example, if UNIX domain socket is ``/tmp/nghttpx.sock``, use +``--base-uri=unix:/tmp/nghttpx.sock``. h2load uses scheme, host and +port in the first URI in command-line or input file. diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index 03500013..6a27bfe3 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -1,44 +1,49 @@ +.. program:: nghttpx + nghttpx - HTTP/2 proxy - HOW-TO =============================== -nghttpx is a proxy translating protocols between HTTP/2 and other -protocols (e.g., HTTP/1, SPDY). It operates in several modes and each -mode may require additional programs to work with. This article -describes each operation mode and explains the intended use-cases. It -also covers some useful options later. +:doc:`nghttpx.1` is a proxy translating protocols between HTTP/2 and +other protocols (e.g., HTTP/1, SPDY). It operates in several modes +and each mode may require additional programs to work with. This +article describes each operation mode and explains the intended +use-cases. It also covers some useful options later. Default mode ------------ -If nghttpx is invoked without any ``-s``, ``-p`` and ``--client``, it -operates in default mode. In this mode, nghttpx frontend listens for -HTTP/2 requests and translates them to HTTP/1 requests. Thus it works -as reverse proxy (gateway) for HTTP/2 clients to HTTP/1 web server. -This is also known as "HTTP/2 router". HTTP/1 requests are also -supported in frontend as a fallback. If nghttpx is linked with -spdylay library and frontend connection is SSL/TLS, the frontend also -supports SPDY protocol. +If nghttpx is invoked without :option:`--http2-proxy`, it operates in +default mode. In this mode, it works as reverse proxy (gateway) for +both HTTP/2 and HTTP/1 clients to backend servers. This is also known +as "HTTP/2 router". If nghttpx is linked with spdylay library and +frontend connection is SSL/TLS, the frontend also supports SPDY +protocol. -By default, this mode's frontend connection is encrypted using -SSL/TLS. So server's private key and certificate must be supplied to -the command line (or through configuration file). In this case, the -frontend protocol selection will be done via ALPN or NPN. +By default, frontend connection is encrypted using SSL/TLS. So +server's private key and certificate must be supplied to the command +line (or through configuration file). In this case, the frontend +protocol selection will be done via ALPN or NPN. -With ``--frontend-no-tls`` option, user can turn off SSL/TLS in +With :option:`--frontend-no-tls` option, user can turn off SSL/TLS in frontend connection. In this case, SPDY protocol is not available even if spdylay library is liked to nghttpx. HTTP/2 and HTTP/1 are -available on the frontend and a HTTP/1 connection can be upgraded to +available on the frontend, and an HTTP/1 connection can be upgraded to HTTP/2 using HTTP Upgrade. Starting HTTP/2 connection by sending HTTP/2 connection preface is also supported. -By default, backend HTTP/1 connections are not encrypted. To enable -TLS on HTTP/1 backend connections, use ``--backend-http1-tls`` option. -This applies to all mode whose backend connections are HTTP/1. +By default, backend connections are not encrypted. To enable TLS +encryption on backend connections, use :option:`--backend-tls` option. +Using patterns and ``proto`` keyword in :option:`--backend` option, +backend application protocol can be specified per host/request path +pattern. It means that you can use both HTTP/2 and HTTP/1 in backend +connections at the same time. Note that default backend protocol is +HTTP/1.1. To use HTTP/2 in backend, you have to specify ``h2`` in +``proto`` keyword in :option:`--backend` explicitly. -The backend is supposed to be HTTP/1 Web server. For example, to make +The backend is supposed to be Web server. For example, to make nghttpx listen to encrypted HTTP/2 requests at port 8443, and a -backend HTTP/1 web server is configured to listen to HTTP/1 request at -port 8080 in the same host, run nghttpx command-line like this:: +backend Web server is configured to listen to HTTP request at port +8080 in the same host, run nghttpx command-line like this:: $ nghttpx -f0.0.0.0,8443 -b127.0.0.1,8080 /path/to/server.key /path/to/server.crt @@ -50,30 +55,36 @@ example, you can send GET request to the server using nghttp:: HTTP/2 proxy mode ----------------- -If nghttpx is invoked with ``-s`` option, it operates in HTTP/2 proxy -mode. The supported protocols in frontend and backend connections are -the same in `default mode`_. The difference is that this mode acts -like forward proxy and assumes the backend is HTTP/1 proxy server -(e.g., squid, traffic server). So HTTP/1 request must include -absolute URI in request line. +If nghttpx is invoked with :option:`--http2-proxy` (or its shorthand +:option:`-s`) option, it operates in HTTP/2 proxy mode. The supported +protocols in frontend and backend connections are the same in `default +mode`_. The difference is that this mode acts like forward proxy and +assumes the backend is HTTP proxy server (e.g., Squid, Apache Traffic +Server). HTTP/1 request must include absolute URI in request line. By default, frontend connection is encrypted. So this mode is also called secure proxy. If nghttpx is linked with spdylay, it supports SPDY protocols and it works as so called SPDY proxy. -With ``--frontend-no-tls`` option, SSL/TLS is turned off in frontend -connection, so the connection gets insecure. +With :option:`--frontend-no-tls` option, SSL/TLS is turned off in +frontend connection, so the connection gets insecure. -The backend must be HTTP/1 proxy server. nghttpx supports multiple -backend server addresses. It translates incoming requests to HTTP/1 +The backend must be HTTP proxy server. nghttpx supports multiple +backend server addresses. It translates incoming requests to HTTP request to backend server. The backend server performs real proxy work for each request, for example, dispatching requests to the origin server and caching contents. +The backend connection is not encrypted by default. To enable +encryption, use :option:`--backend-tls` option. The default backend +protocol is HTTP/1.1. To use HTTP/2 in backend connection, use +:option:`--backend` option, and specify ``h2`` in ``proto`` keyword +explicitly. + For example, to make nghttpx listen to encrypted HTTP/2 requests at -port 8443, and a backend HTTP/1 proxy server is configured to listen -to HTTP/1 request at port 8080 in the same host, run nghttpx -command-line like this:: +port 8443, and a backend HTTP proxy server is configured to listen to +HTTP/1 request at port 8080 in the same host, run nghttpx command-line +like this:: $ nghttpx -s -f'*,8443' -b127.0.0.1,8080 /path/to/server.key /path/to/server.crt @@ -96,7 +107,9 @@ Chromium require valid certificate for secure proxy. For Firefox, open Preference window and select Advanced then click Network tab. Clicking Connection Settings button will show the dialog. Select "Automatic proxy configuration URL" and enter the path -to proxy.pac file, something like this:: +to proxy.pac file, something like this: + +.. code-block:: text file:///path/to/proxy.pac @@ -112,136 +125,51 @@ configuration items to edit:: CONFIG proxy.config.url_remap.remap_required INT 0 Consult Traffic server `documentation -`_ +`_ to know how to configure traffic server as forward proxy and its security implications. -Client mode ------------ +Disable frontend SSL/TLS +------------------------ -If nghttpx is invoked with ``--client`` option, it operates in client -mode. In this mode, nghttpx listens for plain, unencrypted HTTP/2 and -HTTP/1 requests and translates them to encrypted HTTP/2 requests to -the backend. User cannot enable SSL/TLS in frontend connection. +The frontend connections are encrypted with SSL/TLS by default. To +turn off SSL/TLS, use :option:`--frontend-no-tls` option. If this +option is used, the private key and certificate are not required to +run nghttpx. -HTTP/1 frontend connection can be upgraded to HTTP/2 using HTTP -Upgrade. To disable SSL/TLS in backend connection, use -``--backend-no-tls`` option. +Enable backend SSL/TLS +---------------------- -By default, the number of backend HTTP/2 connections per worker -(thread) is determined by number of ``-b`` option. To adjust this -value, use ``--backend-http2-connections-per-worker`` option. +The backend connections are not encrypted by default. To enable +SSL/TLS encryption, :option:`--backend-tls` option. -The backend server is supporsed to be a HTTP/2 web server (e.g., -nghttpd). The one use-case of this mode is utilize existing HTTP/1 -clients to test HTTP/2 deployment. Suppose that HTTP/2 web server -listens to port 80 without encryption. Then run nghttpx as client -mode to access to that web server:: +Enable SSL/TLS on memcached connection +-------------------------------------- - $ nghttpx --client -f127.0.0.1,8080 -b127.0.0.1,80 --backend-no-tls +By default, memcached connection is not encrypted. To enable +encryption, use :option:`--tls-ticket-key-memcached-tls` for TLS +ticket key, and use :option:`--tls-session-cache-memcached-tls` for +TLS session cache. -.. note:: +Specifying additional server certificates +----------------------------------------- - You may need ``-k`` option if HTTP/2 server enables SSL/TLS and - its certificate is self-signed. But please note that it is - insecure. - -Then you can use curl to access HTTP/2 server via nghttpx:: - - $ curl http://localhost:8080/ - -Client proxy mode ------------------ - -If nghttpx is invoked with ``-p`` option, it operates in client proxy -mode. This mode behaves like `client mode`_, but it works like -forward proxy. So HTTP/1 request must include absolute URI in request -line. - -HTTP/1 frontend connection can be upgraded to HTTP/2 using HTTP -Upgrade. To disable SSL/TLS in backend connection, use -``--backend-no-tls`` option. - -By default, the number of backend HTTP/2 connections per worker -(thread) is determined by number of ``-b`` option. To adjust this -value, use ``--backend-http2-connections-per-worker`` option. - -The backend server must be a HTTP/2 proxy. You can use nghttpx in -`HTTP/2 proxy mode`_ as backend server. The one use-case of this mode -is utilize existing HTTP/1 clients to test HTTP/2 connections between -2 proxies. The another use-case is use this mode to aggregate local -HTTP/1 connections to one HTTP/2 backend encrypted connection. This -makes HTTP/1 clients which does not support secure proxy can use -secure HTTP/2 proxy via nghttpx client mode. - -Suppose that HTTP/2 proxy listens to port 8443, just like we saw in -`HTTP/2 proxy mode`_. To run nghttpx in client proxy mode to access -that server, invoke nghttpx like this:: - - $ nghttpx -p -f127.0.0.1,8080 -b127.0.0.1,8443 - -.. note:: - - You may need ``-k`` option if HTTP/2 server's certificate is - self-signed. But please note that it is insecure. - -Then you can use curl to issue HTTP request via HTTP/2 proxy:: - - $ curl --http-proxy=http://localhost:8080 http://www.google.com/ - -You can configure web browser to use localhost:8080 as forward -proxy. - -HTTP/2 bridge mode ------------------- - -If nghttpx is invoked with ``--http2-bridge`` option, it operates in -HTTP/2 bridge mode. The supported protocols in frontend connections -are the same in `default mode`_. The protocol in backend is HTTP/2 -only. - -With ``--frontend-no-tls`` option, SSL/TLS is turned off in frontend -connection, so the connection gets insecure. To disable SSL/TLS in -backend connection, use ``--backend-no-tls`` option. - -By default, the number of backend HTTP/2 connections per worker -(thread) is determined by number of ``-b`` option. To adjust this -value, use ``--backend-http2-connections-per-worker`` option. - -The backend server is supporsed to be a HTTP/2 web server or HTTP/2 -proxy. If backend server is HTTP/2 proxy, use -``--no-location-rewrite`` and ``--no-host-rewrite`` options to disable -rewriting location, host and :authority header field. - -The use-case of this mode is aggregate the incoming connections to one -HTTP/2 connection. One backend HTTP/2 connection is created per -worker (thread). - -Disable SSL/TLS ---------------- - -In `default mode`_, `HTTP/2 proxy mode`_ and `HTTP/2 bridge mode`_, -frontend connections are encrypted with SSL/TLS by default. To turn -off SSL/TLS, use ``--frontend-no-tls`` option. If this option is -used, the private key and certificate are not required to run nghttpx. - -In `client mode`_, `client proxy mode`_ and `HTTP/2 bridge mode`_, -backend connections are encrypted with SSL/TLS by default. To turn -off SSL/TLS, use ``--backend-no-tls`` option. +nghttpx accepts additional server private key and certificate pairs +using :option:`--subcert` option. It can be used multiple times. Specifying additional CA certificate ------------------------------------ By default, nghttpx tries to read CA certificate from system. But depending on the system you use, this may fail or is not supported. -To specify CA certificate manually, use ``--cacert`` option. The -specified file must be PEM format and can contain multiple +To specify CA certificate manually, use :option:`--cacert` option. +The specified file must be PEM format and can contain multiple certificates. By default, nghttpx validates server's certificate. If you want to turn off this validation, knowing this is really insecure and what you -are doing, you can use ``-k`` option to disable certificate -validation. +are doing, you can use :option:`--insecure` option to disable +certificate validation. Read/write rate limit --------------------- @@ -250,9 +178,9 @@ nghttpx supports transfer rate limiting on frontend connections. You can do rate limit per frontend connection for reading and writing individually. -To perform rate limit for reading, use ``--read-rate`` and -``--read-burst`` options. For writing, use ``--write-rate`` and -``--write-burst``. +To perform rate limit for reading, use :option:`--read-rate` and +:option:`--read-burst` options. For writing, use +:option:`--write-rate` and :option:`--write-burst`. Please note that rate limit is performed on top of TCP and nothing to do with HTTP/2 flow control. @@ -294,14 +222,92 @@ Re-opening log files When rotating log files, it is desirable to re-open log files after log rotation daemon renamed existing log files. To tell nghttpx to re-open log files, send USR1 signal to nghttpx process. It will -re-open files specified by ``--accesslog-file`` and -``--errorlog-file`` options. +re-open files specified by :option:`--accesslog-file` and +:option:`--errorlog-file` options. Multiple backend addresses -------------------------- nghttpx supports multiple backend addresses. To specify them, just -use ``-b`` option repeatedly. For example, to use backend1:8080 and -backend2:8080, use command-line like this: ``-bbackend1,8080 --bbackend2,8080``. For HTTP/2 backend, see also -``--backend-http2-connections-per-worker`` option. +use :option:`--backend` (or its shorthand :option:`-b`) option +repeatedly. For example, to use ``192.168.0.10:8080`` and +``192.168.0.11:8080``, use command-line like this: +``-b192.168.0.10,8080 -b192.168.0.11,8080``. In configuration file, +this looks like: + +.. code-block:: text + + backend=192.168.0.10,8080 + backend=192.168.0.11,8008 + +nghttpx can route request to different backend according to request +host and path. For example, to route request destined to host +``doc.example.com`` to backend server ``docserv:3000``, you can write +like so: + +.. code-block:: text + + backend=docserv,3000;doc.example.com/ + +When you write this option in command-line, you should enclose +argument with single or double quotes, since the character ``;`` has a +special meaning in shell. + +To route, request to request path whose prefix is ``/foo`` to backend +server ``[::1]:8080``, you can write like so: + +.. code-block:: text + + backend=::1,8080;/foo + +Of course, you can specify both host and request path at the same +time. + +One important thing you have to remember is that we have to specify +default routing pattern for so called "catch all" pattern. To write +"catch all" pattern, just specify backend server address, without +pattern. + +Usually, host is the value of ``Host`` header field. In HTTP/2, the +value of ``:authority`` pseudo header field is used. + +When you write multiple backend addresses sharing the same routing +pattern, they are used as load balancing. For example, to use 2 +servers ``serv1:3000`` and ``serv2:3000`` for request host +``example.com`` and path ``/myservice``, you can write like so: + +.. code-block:: text + + backend=serv1,3000;example.com/myservice + backend=serv2,3000;example.com/myservice + +You can also specify backend application protocol in +:option:`--backend` option using ``proto`` keyword after pattern. +Utilizing this allows ngttpx to route certain request to HTTP/2, other +requests to HTTP/1. For example, to route requests to ``/ws/`` in +backend HTTP/1.1 connection, and use backend HTTP/2 for other +requests, do this: + +.. code-block:: text + + backend=serv1,3000;/;proto=h2 + backend=serv1,3000;/ws/;proto=http/1.1 + +Note that the backends share the same pattern must have the same +backend protocol. The default backend protocol is HTTP/1.1. + +Deprecated modes +---------------- + +As of nghttpx 1.9.0, ``--http2-bridge``, ``--client`` and +``--client-proxy`` options were removed. These functionality can be +used using combinations of options. + +* ``--http2-bridge``: Use ``--backend=',;;proto=h2'``, and + ``--backend-tls``. + +* ``--client``: Use ``--frontend-no-tls``, + ``--backend=',;;proto=h2'``, and ``--backend-tls``. + +* ``--client-proxy``: Use ``--http2-proxy``, ``--frontend-no-tls``, + ``--backend=',;;proto=h2'``, and ``--backend-tls``. diff --git a/genlibtokenlookup.py b/genlibtokenlookup.py index d7ac7f47..e0726c59 100755 --- a/genlibtokenlookup.py +++ b/genlibtokenlookup.py @@ -106,7 +106,7 @@ def gen_enum(): def gen_index_header(): print '''\ -static inline int lookup_token(const uint8_t *name, size_t namelen) { +static inline int32_t lookup_token(const uint8_t *name, size_t namelen) { switch (namelen) {''' b = build_header(HEADERS) for size in sorted(b.keys()): diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 043e9426..9df537f1 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -115,7 +115,6 @@ OPTIONS = [ "max-header-fields", "no-http2-cipher-black-list", "backend-http1-tls", - "backend-tls-session-cache-per-worker", "tls-session-cache-memcached-cert-file", "tls-session-cache-memcached-private-key-file", "tls-session-cache-memcached-address-family", @@ -123,7 +122,12 @@ OPTIONS = [ "tls-ticket-key-memcached-cert-file", "tls-ticket-key-memcached-private-key-file", "tls-ticket-key-memcached-address-family", - "backend-address-family" + "backend-address-family", + "frontend-http2-max-concurrent-streams", + "backend-http2-max-concurrent-streams", + "backend-connections-per-frontend", + "backend-tls", + "backend-connections-per-host" ] LOGVARS = [ diff --git a/integration-tests/Makefile.am b/integration-tests/Makefile.am index 479bfa14..53931da8 100644 --- a/integration-tests/Makefile.am +++ b/integration-tests/Makefile.am @@ -40,12 +40,12 @@ EXTRA_DIST = \ req-return.rb \ resp-return.rb -itprep-local: +itprep: go get -d -v golang.org/x/net/http2 go get -d -v github.com/tatsuhiro-t/go-nghttp2 go get -d -v github.com/tatsuhiro-t/spdy go get -d -v golang.org/x/net/websocket -it-local: +it: for i in $(GO_FILES); do [ -e $(builddir)/$$i ] || cp $(srcdir)/$$i $(builddir); done sh setenv go test -v diff --git a/integration-tests/server_tester.go b/integration-tests/server_tester.go index 8226c899..67bbec32 100644 --- a/integration-tests/server_tester.go +++ b/integration-tests/server_tester.go @@ -6,10 +6,10 @@ import ( "crypto/tls" "errors" "fmt" - "golang.org/x/net/http2" - "golang.org/x/net/http2/hpack" "github.com/tatsuhiro-t/go-nghttp2" "github.com/tatsuhiro-t/spdy" + "golang.org/x/net/http2" + "golang.org/x/net/http2/hpack" "golang.org/x/net/websocket" "io" "io/ioutil" @@ -30,7 +30,7 @@ const ( serverBin = buildDir + "/src/nghttpx" serverPort = 3009 testDir = sourceDir + "/integration-tests" - logDir = buildDir + "/integration-tests" + logDir = buildDir + "/integration-tests" ) func pair(name, value string) hpack.HeaderField { @@ -86,14 +86,18 @@ func newServerTesterTLSConfig(args []string, t *testing.T, handler http.HandlerF // newServerTesterInternal creates test context. If frontendTLS is // true, set up TLS frontend connection. -func newServerTesterInternal(args []string, t *testing.T, handler http.Handler, frontendTLS bool, clientConfig *tls.Config) *serverTester { +func newServerTesterInternal(src_args []string, t *testing.T, handler http.Handler, frontendTLS bool, clientConfig *tls.Config) *serverTester { ts := httptest.NewUnstartedServer(handler) + args := []string{} + backendTLS := false - for _, k := range args { + for _, k := range src_args { switch k { case "--http2-bridge": backendTLS = true + default: + args = append(args, k) } } if backendTLS { @@ -102,9 +106,9 @@ func newServerTesterInternal(args []string, t *testing.T, handler http.Handler, // NextProtos separately for ts.TLS. NextProtos set // in nghttp2.ConfigureServer is effectively ignored. ts.TLS = new(tls.Config) - ts.TLS.NextProtos = append(ts.TLS.NextProtos, "h2-14") + ts.TLS.NextProtos = append(ts.TLS.NextProtos, "h2") ts.StartTLS() - args = append(args, "-k") + args = append(args, "-k", "--backend-tls") } else { ts.Start() } @@ -124,6 +128,9 @@ func newServerTesterInternal(args []string, t *testing.T, handler http.Handler, // URL.Host looks like "127.0.0.1:8080", but we want // "127.0.0.1,8080" b := "-b" + strings.Replace(backendURL.Host, ":", ",", -1) + if backendTLS { + b += ";;proto=h2" + } args = append(args, fmt.Sprintf("-f127.0.0.1,%v", serverPort), b, "--errorlog-file="+logDir+"/log.txt", "-LINFO") diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index ff512a45..17777ec1 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -22,6 +22,7 @@ set(NGHTTP2_SOURCES nghttp2_callbacks.c nghttp2_mem.c nghttp2_http.c + nghttp2_rcbuf.c ) # Public shared library diff --git a/lib/Makefile.am b/lib/Makefile.am index 017e44a7..6387936a 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -47,7 +47,8 @@ OBJECTS = nghttp2_pq.c nghttp2_map.c nghttp2_queue.c \ nghttp2_option.c \ nghttp2_callbacks.c \ nghttp2_mem.c \ - nghttp2_http.c + nghttp2_http.c \ + nghttp2_rcbuf.c HFILES = nghttp2_pq.h nghttp2_int.h nghttp2_map.h nghttp2_queue.h \ nghttp2_frame.h \ @@ -61,7 +62,8 @@ HFILES = nghttp2_pq.h nghttp2_int.h nghttp2_map.h nghttp2_queue.h \ nghttp2_option.h \ nghttp2_callbacks.h \ nghttp2_mem.h \ - nghttp2_http.h + nghttp2_http.h \ + nghttp2_rcbuf.h libnghttp2_la_SOURCES = $(HFILES) $(OBJECTS) libnghttp2_la_LDFLAGS = -no-undefined \ diff --git a/lib/Makefile.msvc b/lib/Makefile.msvc index c34b6995..2a930d11 100644 --- a/lib/Makefile.msvc +++ b/lib/Makefile.msvc @@ -12,16 +12,16 @@ # THIS_MAKEFILE := $(lastword $(MAKEFILE_LIST)) -USE_CYTHON := 1 -#USE_CYTHON := 0 +USE_CYTHON := 0 +#USE_CYTHON := 1 -_VERSION := $(shell grep AC_INIT ../configure.ac | cut -d'[' -f3 | sed -r -e 's/(-DEV)?], //g') +_VERSION := $(shell grep AC_INIT ../configure.ac | cut -d'[' -f3 | sed -e 's/-DEV//g' -e 's/], //g') _VERSION := $(subst ., ,$(_VERSION)) VER_MAJOR := $(word 1,$(_VERSION)) VER_MINOR := $(word 2,$(_VERSION)) VER_MICRO := $(word 3,$(_VERSION)) VERSION := $(VER_MAJOR).$(VER_MINOR).$(VER_MICRO) -VERSION_NUM := ($(VER_MAJOR) << 16) + ($(VER_MINOR) << 8) + $(VER_MICRO) +VERSION_NUM := (($(VER_MAJOR) << 16) + ($(VER_MINOR) << 8) + $(VER_MICRO)) GENERATED := 'Generated by $(realpath Makefile.MSVC)' @@ -90,7 +90,8 @@ NGHTTP2_SRC := nghttp2_pq.c \ nghttp2_option.c \ nghttp2_callbacks.c \ nghttp2_mem.c \ - nghttp2_http.c + nghttp2_http.c \ + nghttp2_rcbuf.c NGHTTP2_OBJ_R := $(addprefix $(OBJ_DIR)/r_, $(notdir $(NGHTTP2_SRC:.c=.obj))) NGHTTP2_OBJ_D := $(addprefix $(OBJ_DIR)/d_, $(notdir $(NGHTTP2_SRC:.c=.obj))) @@ -101,7 +102,7 @@ NGHTTP2_OBJ_D := $(addprefix $(OBJ_DIR)/d_, $(notdir $(NGHTTP2_SRC:.c=.obj))) clean_nghttp2_pyd_0 clean_nghttp2_pyd_1 -all: intro $(OBJ_DIR) $(TARGETS) build_nghttp2_pyd_$(USE_CYTHON) +all: intro includes/nghttp2/nghttp2ver.h $(OBJ_DIR) $(TARGETS) build_nghttp2_pyd_$(USE_CYTHON) @echo 'Welcome to NgHTTP2 (release + debug).' @echo 'Do a "make -f Makefile.MSVC install" at own risk!' @@ -193,17 +194,17 @@ $(OBJ_DIR)/d_%.obj: %.c $(THIS_MAKEFILE) @echo $(OBJ_DIR)/r_nghttp2.res: $(OBJ_DIR)/nghttp2.rc $(THIS_MAKEFILE) - $(RC) -nologo -D_RELEASE -Fo $@ $< + $(RC) -D_RELEASE -Fo $@ $< @echo $(OBJ_DIR)/d_nghttp2.res: $(OBJ_DIR)/nghttp2.rc $(THIS_MAKEFILE) - $(RC) -nologo -D_DEBUG -Fo $@ $< + $(RC) -D_DEBUG -Fo $@ $< @echo includes/nghttp2/nghttp2ver.h: includes/nghttp2/nghttp2ver.h.in $(THIS_MAKEFILE) sed < includes/nghttp2/nghttp2ver.h.in \ -e 's/@PACKAGE_VERSION@/$(VERSION)/g' \ - -e 's/@PACKAGE_VERSION_NUM@/($(VERSION_NUM))/g' > $@ + -e 's/@PACKAGE_VERSION_NUM@/$(VERSION_NUM)/g' > $@ touch --reference=includes/nghttp2/nghttp2ver.h.in $@ diff --git a/lib/includes/nghttp2/nghttp2.h b/lib/includes/nghttp2/nghttp2.h index 59fc8674..8cd00ca6 100644 --- a/lib/includes/nghttp2/nghttp2.h +++ b/lib/includes/nghttp2/nghttp2.h @@ -382,6 +382,10 @@ typedef enum { * Unexpected internal error, but recovered. */ NGHTTP2_ERR_INTERNAL = -534, + /** + * Indicates that a processing was canceled. + */ + NGHTTP2_ERR_CANCEL = -535, /** * The errors < :enum:`NGHTTP2_ERR_FATAL` mean that the library is * under unexpected condition and processing was terminated (e.g., @@ -415,6 +419,55 @@ typedef enum { NGHTTP2_ERR_FLOODED = -904 } nghttp2_error; +/** + * @struct + * + * The object representing single contagious buffer. + */ +typedef struct { + /** + * The pointer to the buffer. + */ + uint8_t *base; + /** + * The length of the buffer. + */ + size_t len; +} nghttp2_vec; + +struct nghttp2_rcbuf; + +/** + * @struct + * + * The object representing reference counted buffer. The details of + * this structure are intentionally hidden from the public API. + */ +typedef struct nghttp2_rcbuf nghttp2_rcbuf; + +/** + * @function + * + * Increments the reference count of |rcbuf| by 1. + */ +NGHTTP2_EXTERN void nghttp2_rcbuf_incref(nghttp2_rcbuf *rcbuf); + +/** + * @function + * + * Decrements the reference count of |rcbuf| by 1. If the reference + * count becomes zero, the object pointed by |rcbuf| will be freed. + * In this case, application must not use |rcbuf| again. + */ +NGHTTP2_EXTERN void nghttp2_rcbuf_decref(nghttp2_rcbuf *rcbuf); + +/** + * @function + * + * Returns the underlying buffer managed by |rcbuf|. + */ +NGHTTP2_EXTERN nghttp2_vec nghttp2_rcbuf_get_buf(nghttp2_rcbuf *rcbuf); + /** * @enum * @@ -1623,6 +1676,32 @@ typedef int (*nghttp2_on_header_callback)(nghttp2_session *session, const uint8_t *value, size_t valuelen, uint8_t flags, void *user_data); +/** + * @functypedef + * + * Callback function invoked when a header name/value pair is received + * for the |frame|. The |name| is header name. The |value| is header + * value. The |flags| is bitwise OR of one or more of + * :type:`nghttp2_nv_flag`. + * + * This callback behaves like :type:`nghttp2_on_header_callback`, + * except that |name| and |value| are stored in reference counted + * buffer. If application wishes to keep these references without + * copying them, use `nghttp2_rcbuf_incref()` to increment their + * reference count. It is the application's responsibility to call + * `nghttp2_rcbuf_decref()` if they called `nghttp2_rcbuf_incref()` so + * as not to leak memory. If the |session| is created by + * `nghttp2_session_server_new3()` or `nghttp2_session_client_new3()`, + * the function to free memory is the one belongs to the mem + * parameter. As long as this free function alives, |name| and + * |value| can live after |session| was destroyed. + */ +typedef int (*nghttp2_on_header_callback2)(nghttp2_session *session, + const nghttp2_frame *frame, + nghttp2_rcbuf *name, + nghttp2_rcbuf *value, uint8_t flags, + void *user_data); + /** * @functypedef * @@ -1700,6 +1779,124 @@ typedef int (*nghttp2_on_begin_frame_callback)(nghttp2_session *session, const nghttp2_frame_hd *hd, void *user_data); +/** + * @functypedef + * + * Callback function invoked when chunk of extension frame payload is + * received. The |hd| points to frame header. The received + * chunk is |data| of length |len|. + * + * The implementation of this function must return 0 if it succeeds. + * + * To abort processing this extension frame, return + * :enum:`NGHTTP2_ERR_CANCEL`. + * + * If fatal error occurred, application should return + * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. In this case, + * `nghttp2_session_recv()` and `nghttp2_session_mem_recv()` functions + * immediately return :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. If the + * other values are returned, currently they are treated as + * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. + */ +typedef int (*nghttp2_on_extension_chunk_recv_callback)( + nghttp2_session *session, const nghttp2_frame_hd *hd, const uint8_t *data, + size_t len, void *user_data); + +/** + * @functypedef + * + * Callback function invoked when library asks the application to + * unpack extension payload from its wire format. The extension + * payload has been passed to the application using + * :type:`nghttp2_on_extension_chunk_recv_callback`. The frame header + * is already unpacked by the library and provided as |hd|. + * + * To receive extension frames, the application must tell desired + * extension frame type to the library using + * `nghttp2_option_set_user_recv_extension_type()`. + * + * The implementation of this function may store the pointer to the + * created object as a result of unpacking in |*payload|, and returns + * 0. The pointer stored in |*payload| is opaque to the library, and + * the library does not own its pointer. |*payload| is initialized as + * ``NULL``. The |*payload| is available as ``frame->ext.payload`` in + * :type:`nghttp2_on_frame_recv_callback`. Therefore if application + * can free that memory inside :type:`nghttp2_on_frame_recv_callback` + * callback. Of course, application has a liberty not ot use + * |*payload|, and do its own mechanism to process extension frames. + * + * To abort processing this extension frame, return + * :enum:`NGHTTP2_ERR_CANCEL`. + * + * If fatal error occurred, application should return + * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. In this case, + * `nghttp2_session_recv()` and `nghttp2_session_mem_recv()` functions + * immediately return :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. If the + * other values are returned, currently they are treated as + * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. + */ +typedef int (*nghttp2_unpack_extension_callback)(nghttp2_session *session, + void **payload, + const nghttp2_frame_hd *hd, + void *user_data); + +/** + * @functypedef + * + * Callback function invoked when library asks the application to pack + * extension payload in its wire format. The frame header will be + * packed by library. Application must pack payload only. + * ``frame->ext.payload`` is the object passed to + * `nghttp2_submit_extension()` as payload parameter. Application + * must pack extension payload to the |buf| of its capacity |len| + * bytes. The |len| is at least 16KiB. + * + * The implementation of this function should return the number of + * bytes written into |buf| when it succeeds. + * + * To abort processing this extension frame, return + * :enum:`NGHTTP2_ERR_CANCEL`, and + * :type:`nghttp2_on_frame_not_send_callback` will be invoked. + * + * If fatal error occurred, application should return + * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. In this case, + * `nghttp2_session_send()` and `nghttp2_session_mem_send()` functions + * immediately return :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. If the + * other values are returned, currently they are treated as + * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. If the return value is + * strictly larger than |len|, it is treated as + * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. + */ +typedef ssize_t (*nghttp2_pack_extension_callback)(nghttp2_session *session, + uint8_t *buf, size_t len, + const nghttp2_frame *frame, + void *user_data); + +/** + * @functypedef + * + * Callback function invoked when library provides the error message + * intended for human consumption. This callback is solely for + * debugging purpose. The |msg| is typically NULL-terminated string + * of length |len|. |len| does not include the sentinel NULL + * character. + * + * The format of error message may change between nghttp2 library + * versions. The application should not depend on the particular + * format. + * + * Normally, application should return 0 from this callback. If fatal + * error occurred while doing something in this callback, application + * should return :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. In this case, + * library will return immediately with return value + * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`. Currently, if nonzero value + * is returned from this callback, they are treated as + * :enum:`NGHTTP2_ERR_CALLBACK_FAILURE`, but application should not + * rely on this details. + */ +typedef int (*nghttp2_error_callback)(nghttp2_session *session, const char *msg, + size_t len, void *user_data); + struct nghttp2_session_callbacks; /** @@ -1844,12 +2041,25 @@ NGHTTP2_EXTERN void nghttp2_session_callbacks_set_on_begin_headers_callback( * @function * * Sets callback function invoked when a header name/value pair is - * received. + * received. If both + * `nghttp2_session_callbacks_set_on_header_callback()` and + * `nghttp2_session_callbacks_set_on_header_callback2()` are used to + * set callbacks, the latter has the precedence. */ NGHTTP2_EXTERN void nghttp2_session_callbacks_set_on_header_callback( nghttp2_session_callbacks *cbs, nghttp2_on_header_callback on_header_callback); +/** + * @function + * + * Sets callback function invoked when a header name/value pair is + * received. + */ +NGHTTP2_EXTERN void nghttp2_session_callbacks_set_on_header_callback2( + nghttp2_session_callbacks *cbs, + nghttp2_on_header_callback2 on_header_callback2); + /** * @function * @@ -1892,6 +2102,46 @@ NGHTTP2_EXTERN void nghttp2_session_callbacks_set_send_data_callback( nghttp2_session_callbacks *cbs, nghttp2_send_data_callback send_data_callback); +/** + * @function + * + * Sets callback function invoked when the library asks the + * application to pack extension frame payload in wire format. + */ +NGHTTP2_EXTERN void nghttp2_session_callbacks_set_pack_extension_callback( + nghttp2_session_callbacks *cbs, + nghttp2_pack_extension_callback pack_extension_callback); + +/** + * @function + * + * Sets callback function invoked when the library asks the + * application to unpack extension frame payload from wire format. + */ +NGHTTP2_EXTERN void nghttp2_session_callbacks_set_unpack_extension_callback( + nghttp2_session_callbacks *cbs, + nghttp2_unpack_extension_callback unpack_extension_callback); + +/** + * @function + * + * Sets callback function invoked when chunk of extension frame + * payload is received. + */ +NGHTTP2_EXTERN void +nghttp2_session_callbacks_set_on_extension_chunk_recv_callback( + nghttp2_session_callbacks *cbs, + nghttp2_on_extension_chunk_recv_callback on_extension_chunk_recv_callback); + +/** + * @function + * + * Sets callback function invoked when library tells error message to + * the application. + */ +NGHTTP2_EXTERN void nghttp2_session_callbacks_set_error_callback( + nghttp2_session_callbacks *cbs, nghttp2_error_callback error_callback); + /** * @functypedef * @@ -2106,6 +2356,37 @@ NGHTTP2_EXTERN void nghttp2_option_set_max_reserved_remote_streams(nghttp2_option *option, uint32_t val); +/** + * @function + * + * Sets extension frame type the application is willing to handle with + * user defined callbacks (see + * :type:`nghttp2_on_extension_chunk_recv_callback` and + * :type:`nghttp2_unpack_extension_callback`). The |type| is + * extension frame type, and must be strictly greater than 0x9. + * Otherwise, this function does nothing. The application can call + * this function multiple times to set more than one frame type to + * receive. The application does not have to call this function if it + * just sends extension frames. + */ +NGHTTP2_EXTERN void +nghttp2_option_set_user_recv_extension_type(nghttp2_option *option, + uint8_t type); + +/** + * @function + * + * This option prevents the library from sending PING frame with ACK + * flag set automatically when PING frame without ACK flag set is + * received. If this option is set to nonzero, the library won't send + * PING frame with ACK flag set in the response for incoming PING + * frame. The application can send PING frame with ACK flag set using + * `nghttp2_submit_ping()` with :enum:`NGHTTP2_FLAG_ACK` as flags + * parameter. + */ +NGHTTP2_EXTERN void nghttp2_option_set_no_auto_ping_ack(nghttp2_option *option, + int val); + /** * @function * @@ -3079,6 +3360,16 @@ nghttp2_pack_settings_payload(uint8_t *buf, size_t buflen, */ NGHTTP2_EXTERN const char *nghttp2_strerror(int lib_error_code); +/** + * @function + * + * Returns string representation of HTTP/2 error code |error_code| + * (e.g., ``PROTOCOL_ERROR`` is returned if ``error_code == + * NGHTTP2_PROTOCOL_ERROR``). If string representation is unknown for + * given |error_code|, this function returns string ``unknown``. + */ +NGHTTP2_EXTERN const char *nghttp2_http2_strerror(uint32_t error_code); + /** * @function * @@ -3628,8 +3919,12 @@ nghttp2_submit_push_promise(nghttp2_session *session, uint8_t flags, * received PING frame. The library automatically submits PING frame * in this case. * - * The |flags| is currently ignored and should be - * :enum:`NGHTTP2_FLAG_NONE`. + * The |flags| is bitwise OR of 0 or more of the following value. + * + * * :enum:`NGHTTP2_FLAG_ACK` + * + * Unless `nghttp2_option_set_no_auto_ping_ack()` is used, the |flags| + * should be :enum:`NGHTTP2_FLAG_NONE`. * * If the |opaque_data| is non ``NULL``, then it should point to the 8 * bytes array of memory to specify opaque data to send with PING @@ -3776,6 +4071,48 @@ NGHTTP2_EXTERN int nghttp2_submit_window_update(nghttp2_session *session, int32_t stream_id, int32_t window_size_increment); +/** + * @function + * + * Submits extension frame. + * + * Application can pass arbitrary frame flags and stream ID in |flags| + * and |stream_id| respectively. The |payload| is opaque pointer, and + * it can be accessible though ``frame->ext.payload`` in + * :type:`nghttp2_pack_extension_callback`. The library will not own + * passed |payload| pointer. + * + * The application must set :type:`nghttp2_pack_extension_callback` + * using `nghttp2_session_callbacks_set_pack_extension_callback()`. + * + * The application should retain the memory pointed by |payload| until + * the transmission of extension frame is done (which is indicated by + * :type:`nghttp2_on_frame_send_callback`), or transmission fails + * (which is indicated by :type:`nghttp2_on_frame_not_send_callback`). + * If application does not touch this memory region after packing it + * into a wire format, application can free it inside + * :type:`nghttp2_pack_extension_callback`. + * + * The standard HTTP/2 frame cannot be sent with this function, so + * |type| must be strictly grater than 0x9. Otherwise, this function + * will fail with error code :enum:`NGHTTP2_ERR_INVALID_ARGUMENT`. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * :enum:`NGHTTP2_ERR_INVALID_STATE` + * If :type:`nghttp2_pack_extension_callback` is not set. + * :enum:`NGHTTP2_ERR_INVALID_ARGUMENT` + * If |type| specifies standard HTTP/2 frame type. The frame + * types in the rage [0x0, 0x9], both inclusive, are standard + * HTTP/2 frame type, and cannot be sent using this function. + * :enum:`NGHTTP2_ERR_NOMEM` + * Out of memory + */ +NGHTTP2_EXTERN int nghttp2_submit_extension(nghttp2_session *session, + uint8_t type, uint8_t flags, + int32_t stream_id, void *payload); + /** * @function * diff --git a/lib/nghttp2_buf.h b/lib/nghttp2_buf.h index 8977f2ea..7c43606e 100644 --- a/lib/nghttp2_buf.h +++ b/lib/nghttp2_buf.h @@ -73,7 +73,7 @@ typedef struct { /* * Initializes the |buf|. No memory is allocated in this function. Use - * nghttp2_buf_reserve() or nghttp2_buf_reserve2() to allocate memory. + * nghttp2_buf_reserve() to allocate memory. */ void nghttp2_buf_init(nghttp2_buf *buf); diff --git a/lib/nghttp2_callbacks.c b/lib/nghttp2_callbacks.c index 3c4be9a0..4d5211a1 100644 --- a/lib/nghttp2_callbacks.c +++ b/lib/nghttp2_callbacks.c @@ -104,6 +104,12 @@ void nghttp2_session_callbacks_set_on_header_callback( cbs->on_header_callback = on_header_callback; } +void nghttp2_session_callbacks_set_on_header_callback2( + nghttp2_session_callbacks *cbs, + nghttp2_on_header_callback2 on_header_callback2) { + cbs->on_header_callback2 = on_header_callback2; +} + void nghttp2_session_callbacks_set_select_padding_callback( nghttp2_session_callbacks *cbs, nghttp2_select_padding_callback select_padding_callback) { @@ -127,3 +133,26 @@ void nghttp2_session_callbacks_set_send_data_callback( nghttp2_send_data_callback send_data_callback) { cbs->send_data_callback = send_data_callback; } + +void nghttp2_session_callbacks_set_pack_extension_callback( + nghttp2_session_callbacks *cbs, + nghttp2_pack_extension_callback pack_extension_callback) { + cbs->pack_extension_callback = pack_extension_callback; +} + +void nghttp2_session_callbacks_set_unpack_extension_callback( + nghttp2_session_callbacks *cbs, + nghttp2_unpack_extension_callback unpack_extension_callback) { + cbs->unpack_extension_callback = unpack_extension_callback; +} + +void nghttp2_session_callbacks_set_on_extension_chunk_recv_callback( + nghttp2_session_callbacks *cbs, + nghttp2_on_extension_chunk_recv_callback on_extension_chunk_recv_callback) { + cbs->on_extension_chunk_recv_callback = on_extension_chunk_recv_callback; +} + +void nghttp2_session_callbacks_set_error_callback( + nghttp2_session_callbacks *cbs, nghttp2_error_callback error_callback) { + cbs->error_callback = error_callback; +} diff --git a/lib/nghttp2_callbacks.h b/lib/nghttp2_callbacks.h index 37958ea9..5f08474a 100644 --- a/lib/nghttp2_callbacks.h +++ b/lib/nghttp2_callbacks.h @@ -91,6 +91,7 @@ struct nghttp2_session_callbacks { * received. */ nghttp2_on_header_callback on_header_callback; + nghttp2_on_header_callback2 on_header_callback2; /** * Callback function invoked when the library asks application how * many padding bytes are required for the transmission of the given @@ -107,6 +108,10 @@ struct nghttp2_session_callbacks { */ nghttp2_on_begin_frame_callback on_begin_frame_callback; nghttp2_send_data_callback send_data_callback; + nghttp2_pack_extension_callback pack_extension_callback; + nghttp2_unpack_extension_callback unpack_extension_callback; + nghttp2_on_extension_chunk_recv_callback on_extension_chunk_recv_callback; + nghttp2_error_callback error_callback; }; #endif /* NGHTTP2_CALLBACKS_H */ diff --git a/lib/nghttp2_frame.c b/lib/nghttp2_frame.c index 78fdf073..3d057847 100644 --- a/lib/nghttp2_frame.c +++ b/lib/nghttp2_frame.c @@ -184,6 +184,15 @@ void nghttp2_frame_data_init(nghttp2_data *frame, uint8_t flags, void nghttp2_frame_data_free(nghttp2_data *frame _U_) {} +void nghttp2_frame_extension_init(nghttp2_extension *frame, uint8_t type, + uint8_t flags, int32_t stream_id, + void *payload) { + nghttp2_frame_hd_init(&frame->hd, 0, type, flags, stream_id); + frame->payload = payload; +} + +void nghttp2_frame_extension_free(nghttp2_extension *frame _U_) {} + size_t nghttp2_frame_priority_len(uint8_t flags) { if (flags & NGHTTP2_FLAG_PRIORITY) { return NGHTTP2_PRIORITY_SPECLEN; diff --git a/lib/nghttp2_frame.h b/lib/nghttp2_frame.h index b0b02ee1..fa0eb452 100644 --- a/lib/nghttp2_frame.h +++ b/lib/nghttp2_frame.h @@ -439,6 +439,12 @@ void nghttp2_frame_window_update_init(nghttp2_window_update *frame, void nghttp2_frame_window_update_free(nghttp2_window_update *frame); +void nghttp2_frame_extension_init(nghttp2_extension *frame, uint8_t type, + uint8_t flags, int32_t stream_id, + void *payload); + +void nghttp2_frame_extension_free(nghttp2_extension *frame); + /* * Returns the number of padding bytes after payload. The total * padding length is given in the |padlen|. The returned value does diff --git a/lib/nghttp2_hd.c b/lib/nghttp2_hd.c index 5a02c0f4..49ca4ae7 100644 --- a/lib/nghttp2_hd.c +++ b/lib/nghttp2_hd.c @@ -34,15 +34,17 @@ /* Make scalar initialization form of nghttp2_hd_entry */ #define MAKE_STATIC_ENT(N, V, T, H) \ { \ - { (uint8_t *)(N), (uint8_t *)(V), sizeof((N)) - 1, sizeof((V)) - 1, 0 } \ - , NULL, 0, (H), (T), 1, NGHTTP2_HD_FLAG_NONE \ + { NULL, NULL, (uint8_t *)(N), sizeof((N)) - 1, -1 } \ + , {NULL, NULL, (uint8_t *)(V), sizeof((V)) - 1, -1}, \ + {(uint8_t *)(N), (uint8_t *)(V), sizeof((N)) - 1, sizeof((V)) - 1, 0}, \ + T, H \ } /* Generated by mkstatictbl.py */ /* 3rd parameter is nghttp2_token value for header field name. We use first enum value if same header names are repeated (e.g., :status). */ -static nghttp2_hd_entry static_table[] = { +static nghttp2_hd_static_entry static_table[] = { MAKE_STATIC_ENT(":authority", "", 0, 3153725150u), MAKE_STATIC_ENT(":method", "GET", 1, 695666056u), MAKE_STATIC_ENT(":method", "POST", 1, 695666056u), @@ -114,7 +116,7 @@ static int memeq(const void *s1, const void *s2, size_t n) { * This function was generated by genlibtokenlookup.py. Inspired by * h2o header lookup. https://github.com/h2o/h2o */ -static int lookup_token(const uint8_t *name, size_t namelen) { +static int32_t lookup_token(const uint8_t *name, size_t namelen) { switch (namelen) { case 2: switch (name[1]) { @@ -494,86 +496,33 @@ static int lookup_token(const uint8_t *name, size_t namelen) { return -1; } -int nghttp2_hd_entry_init(nghttp2_hd_entry *ent, uint8_t flags, uint8_t *name, - size_t namelen, uint8_t *value, size_t valuelen, - int token, nghttp2_mem *mem) { - int rv = 0; - - /* Since nghttp2_hd_entry is used for indexing, ent->nv.flags always - NGHTTP2_NV_FLAG_NONE */ - ent->nv.flags = NGHTTP2_NV_FLAG_NONE; - - if ((flags & NGHTTP2_HD_FLAG_NAME_ALLOC) && - (flags & NGHTTP2_HD_FLAG_NAME_GIFT) == 0) { - if (namelen == 0) { - flags = (uint8_t)(flags & ~NGHTTP2_HD_FLAG_NAME_ALLOC); - ent->nv.name = (uint8_t *)""; - } else { - /* name may not be NULL terminated on compression. */ - ent->nv.name = nghttp2_mem_malloc(mem, namelen + 1); - if (ent->nv.name == NULL) { - rv = NGHTTP2_ERR_NOMEM; - goto fail; - } - memcpy(ent->nv.name, name, namelen); - ent->nv.name[namelen] = '\0'; - } - } else { - ent->nv.name = name; - } - if ((flags & NGHTTP2_HD_FLAG_VALUE_ALLOC) && - (flags & NGHTTP2_HD_FLAG_VALUE_GIFT) == 0) { - if (valuelen == 0) { - flags = (uint8_t)(flags & ~NGHTTP2_HD_FLAG_VALUE_ALLOC); - ent->nv.value = (uint8_t *)""; - } else { - /* value may not be NULL terminated on compression. */ - ent->nv.value = nghttp2_mem_malloc(mem, valuelen + 1); - if (ent->nv.value == NULL) { - rv = NGHTTP2_ERR_NOMEM; - goto fail2; - } - memcpy(ent->nv.value, value, valuelen); - ent->nv.value[valuelen] = '\0'; - } - } else { - ent->nv.value = value; - } - ent->nv.namelen = namelen; - ent->nv.valuelen = valuelen; - ent->token = token; - ent->ref = 1; - ent->flags = flags; +void nghttp2_hd_entry_init(nghttp2_hd_entry *ent, nghttp2_hd_nv *nv) { + ent->nv = *nv; + ent->cnv.name = nv->name->base; + ent->cnv.namelen = nv->name->len; + ent->cnv.value = nv->value->base; + ent->cnv.valuelen = nv->value->len; + ent->cnv.flags = nv->flags; ent->next = NULL; ent->hash = 0; - return 0; - -fail2: - if ((flags & NGHTTP2_HD_FLAG_NAME_ALLOC) && - (flags & NGHTTP2_HD_FLAG_NAME_GIFT) == 0) { - nghttp2_mem_free(mem, ent->nv.name); - } -fail: - return rv; + nghttp2_rcbuf_incref(ent->nv.name); + nghttp2_rcbuf_incref(ent->nv.value); } -void nghttp2_hd_entry_free(nghttp2_hd_entry *ent, nghttp2_mem *mem) { - assert(ent->ref == 0); - if (ent->flags & NGHTTP2_HD_FLAG_NAME_ALLOC) { - nghttp2_mem_free(mem, ent->nv.name); - } - if (ent->flags & NGHTTP2_HD_FLAG_VALUE_ALLOC) { - nghttp2_mem_free(mem, ent->nv.value); - } +void nghttp2_hd_entry_free(nghttp2_hd_entry *ent) { + nghttp2_rcbuf_decref(ent->nv.value); + nghttp2_rcbuf_decref(ent->nv.name); } -static int name_eq(const nghttp2_nv *a, const nghttp2_nv *b) { - return a->namelen == b->namelen && memeq(a->name, b->name, a->namelen); +static int name_eq(const nghttp2_hd_nv *a, const nghttp2_nv *b) { + return a->name->len == b->namelen && + memeq(a->name->base, b->name, b->namelen); } -static int value_eq(const nghttp2_nv *a, const nghttp2_nv *b) { - return a->valuelen == b->valuelen && memeq(a->value, b->value, a->valuelen); +static int value_eq(const nghttp2_hd_nv *a, const nghttp2_nv *b) { + return a->value->len == b->valuelen && + memeq(a->value->base, b->value, b->valuelen); } static uint32_t name_hash(const nghttp2_nv *nv) { @@ -609,7 +558,7 @@ static void hd_map_insert(nghttp2_hd_map *map, nghttp2_hd_entry *ent) { } static nghttp2_hd_entry *hd_map_find(nghttp2_hd_map *map, int *exact_match, - const nghttp2_nv *nv, int token, + const nghttp2_nv *nv, int32_t token, uint32_t hash) { nghttp2_hd_entry *p; nghttp2_hd_entry *res = NULL; @@ -617,8 +566,8 @@ static nghttp2_hd_entry *hd_map_find(nghttp2_hd_map *map, int *exact_match, *exact_match = 0; for (p = map->table[hash & (HD_MAP_SIZE - 1)]; p; p = p->next) { - if (hash != p->hash || token != p->token || - (token == -1 && !name_eq(&p->nv, nv))) { + if (token != p->nv.token || + (token == -1 && (hash != p->hash || !name_eq(&p->nv, nv)))) { continue; } if (!res) { @@ -712,8 +661,8 @@ static void hd_ringbuf_free(nghttp2_hd_ringbuf *ringbuf, nghttp2_mem *mem) { } for (i = 0; i < ringbuf->len; ++i) { nghttp2_hd_entry *ent = hd_ringbuf_get(ringbuf, i); - --ent->ref; - nghttp2_hd_entry_free(ent, mem); + + nghttp2_hd_entry_free(ent); nghttp2_mem_free(mem, ent); } nghttp2_mem_free(mem, ringbuf->buffer); @@ -802,49 +751,37 @@ int nghttp2_hd_inflate_init(nghttp2_hd_inflater *inflater, nghttp2_mem *mem) { inflater->settings_hd_table_bufsize_max = NGHTTP2_HD_DEFAULT_MAX_BUFFER_SIZE; inflater->min_hd_table_bufsize_max = UINT32_MAX; - inflater->ent_keep = NULL; - inflater->nv_keep = NULL; + inflater->nv_name_keep = NULL; + inflater->nv_value_keep = NULL; inflater->opcode = NGHTTP2_HD_OPCODE_NONE; inflater->state = NGHTTP2_HD_STATE_INFLATE_START; - rv = nghttp2_bufs_init3(&inflater->nvbufs, NGHTTP2_HD_MAX_NV / 8, 8, 1, 0, - mem); + nghttp2_buf_init(&inflater->namebuf); + nghttp2_buf_init(&inflater->valuebuf); - if (rv != 0) { - goto nvbufs_fail; - } + inflater->namercbuf = NULL; + inflater->valuercbuf = NULL; inflater->huffman_encoded = 0; inflater->index = 0; inflater->left = 0; inflater->shift = 0; - inflater->newnamelen = 0; inflater->index_required = 0; inflater->no_index = 0; return 0; -nvbufs_fail: - hd_context_free(&inflater->ctx); fail: return rv; } static void hd_inflate_keep_free(nghttp2_hd_inflater *inflater) { - nghttp2_mem *mem; + nghttp2_rcbuf_decref(inflater->nv_value_keep); + nghttp2_rcbuf_decref(inflater->nv_name_keep); - mem = inflater->ctx.mem; - if (inflater->ent_keep) { - if (inflater->ent_keep->ref == 0) { - nghttp2_hd_entry_free(inflater->ent_keep, mem); - nghttp2_mem_free(mem, inflater->ent_keep); - } - inflater->ent_keep = NULL; - } - - nghttp2_mem_free(mem, inflater->nv_keep); - inflater->nv_keep = NULL; + inflater->nv_value_keep = NULL; + inflater->nv_name_keep = NULL; } void nghttp2_hd_deflate_free(nghttp2_hd_deflater *deflater) { @@ -853,7 +790,10 @@ void nghttp2_hd_deflate_free(nghttp2_hd_deflater *deflater) { void nghttp2_hd_inflate_free(nghttp2_hd_inflater *inflater) { hd_inflate_keep_free(inflater); - nghttp2_bufs_free(&inflater->nvbufs); + + nghttp2_rcbuf_decref(inflater->valuercbuf); + nghttp2_rcbuf_decref(inflater->namercbuf); + hd_context_free(&inflater->ctx); } @@ -861,23 +801,13 @@ static size_t entry_room(size_t namelen, size_t valuelen) { return NGHTTP2_HD_ENTRY_OVERHEAD + namelen + valuelen; } -static int emit_indexed_header(nghttp2_nv *nv_out, int *token_out, - nghttp2_hd_entry *ent) { - DEBUGF(fprintf(stderr, "inflatehd: header emission: %s: %s\n", ent->nv.name, - ent->nv.value)); +static int emit_header(nghttp2_hd_nv *nv_out, nghttp2_hd_nv *nv) { + DEBUGF(fprintf(stderr, "inflatehd: header emission: %s: %s\n", nv->name->base, + nv->value->base)); /* ent->ref may be 0. This happens if the encoder emits literal block larger than header table capacity with indexing. */ - *nv_out = ent->nv; - *token_out = ent->token; - return 0; -} - -static int emit_literal_header(nghttp2_nv *nv_out, int *token_out, - nghttp2_nv *nv) { - DEBUGF(fprintf(stderr, "inflatehd: header emission: %s: %s\n", nv->name, - nv->value)); *nv_out = *nv; - *token_out = lookup_token(nv->name, nv->namelen); + return 0; } @@ -1185,17 +1115,16 @@ static int emit_newname_block(nghttp2_bufs *bufs, const nghttp2_nv *nv, return 0; } -static nghttp2_hd_entry * -add_hd_table_incremental(nghttp2_hd_context *context, const nghttp2_nv *nv, - int token, uint8_t entry_flags, nghttp2_hd_map *map, - uint32_t hash) { +static int add_hd_table_incremental(nghttp2_hd_context *context, + nghttp2_hd_nv *nv, nghttp2_hd_map *map, + uint32_t hash) { int rv; nghttp2_hd_entry *new_ent; size_t room; nghttp2_mem *mem; mem = context->mem; - room = entry_room(nv->namelen, nv->valuelen); + room = entry_room(nv->name->len, nv->value->len); while (context->hd_table_bufsize + room > context->hd_table_bufsize_max && context->hd_table.len > 0) { @@ -1203,72 +1132,53 @@ add_hd_table_incremental(nghttp2_hd_context *context, const nghttp2_nv *nv, size_t idx = context->hd_table.len - 1; nghttp2_hd_entry *ent = hd_ringbuf_get(&context->hd_table, idx); - context->hd_table_bufsize -= entry_room(ent->nv.namelen, ent->nv.valuelen); + context->hd_table_bufsize -= + entry_room(ent->nv.name->len, ent->nv.value->len); DEBUGF(fprintf(stderr, "hpack: remove item from header table: %s: %s\n", - ent->nv.name, ent->nv.value)); + (char *)ent->nv.name->base, (char *)ent->nv.value->base)); hd_ringbuf_pop_back(&context->hd_table); if (map) { hd_map_remove(map, ent); } - if (--ent->ref == 0) { - nghttp2_hd_entry_free(ent, mem); - nghttp2_mem_free(mem, ent); - } - } - new_ent = nghttp2_mem_malloc(mem, sizeof(nghttp2_hd_entry)); - if (new_ent == NULL) { - return NULL; - } - - rv = nghttp2_hd_entry_init(new_ent, entry_flags, nv->name, nv->namelen, - nv->value, nv->valuelen, token, mem); - if (rv != 0) { - nghttp2_mem_free(mem, new_ent); - return NULL; + nghttp2_hd_entry_free(ent); + nghttp2_mem_free(mem, ent); } if (room > context->hd_table_bufsize_max) { /* The entry taking more than NGHTTP2_HD_MAX_BUFFER_SIZE is - immediately evicted. */ - --new_ent->ref; - } else { - rv = hd_ringbuf_push_front(&context->hd_table, new_ent, mem); - - if (rv != 0) { - --new_ent->ref; - - if ((entry_flags & NGHTTP2_HD_FLAG_NAME_ALLOC) && - (entry_flags & NGHTTP2_HD_FLAG_NAME_GIFT)) { - /* nv->name are managed by caller. */ - new_ent->nv.name = NULL; - new_ent->nv.namelen = 0; - } - if ((entry_flags & NGHTTP2_HD_FLAG_VALUE_ALLOC) && - (entry_flags & NGHTTP2_HD_FLAG_VALUE_GIFT)) { - /* nv->value are managed by caller. */ - new_ent->nv.value = NULL; - new_ent->nv.valuelen = 0; - } - - nghttp2_hd_entry_free(new_ent, mem); - nghttp2_mem_free(mem, new_ent); - - return NULL; - } - - new_ent->seq = context->next_seq++; - new_ent->hash = hash; - - if (map) { - hd_map_insert(map, new_ent); - } - - context->hd_table_bufsize += room; + immediately evicted. So we don't allocate memory for it. */ + return 0; } - return new_ent; + + new_ent = nghttp2_mem_malloc(mem, sizeof(nghttp2_hd_entry)); + if (new_ent == NULL) { + return NGHTTP2_ERR_NOMEM; + } + + nghttp2_hd_entry_init(new_ent, nv); + + rv = hd_ringbuf_push_front(&context->hd_table, new_ent, mem); + + if (rv != 0) { + nghttp2_hd_entry_free(new_ent); + nghttp2_mem_free(mem, new_ent); + + return rv; + } + + new_ent->seq = context->next_seq++; + new_ent->hash = hash; + + if (map) { + hd_map_insert(map, new_ent); + } + + context->hd_table_bufsize += room; + + return 0; } typedef struct { @@ -1277,10 +1187,11 @@ typedef struct { uint8_t name_value_match; } search_result; -static search_result search_static_table(const nghttp2_nv *nv, int token, +static search_result search_static_table(const nghttp2_nv *nv, int32_t token, int indexing_mode) { search_result res = {token, 0}; int i; + nghttp2_hd_static_entry *ent; if (indexing_mode == NGHTTP2_HD_NEVER_INDEXING) { return res; @@ -1289,7 +1200,9 @@ static search_result search_static_table(const nghttp2_nv *nv, int token, for (i = token; i <= NGHTTP2_TOKEN_WWW_AUTHENTICATE && static_table[i].token == token; ++i) { - if (value_eq(&static_table[i].nv, nv)) { + ent = &static_table[i]; + if (ent->value.len == nv->valuelen && + memcmp(ent->value.base, nv->value, nv->valuelen) == 0) { res.index = i; res.name_value_match = 1; return res; @@ -1299,7 +1212,7 @@ static search_result search_static_table(const nghttp2_nv *nv, int token, } static search_result search_hd_table(nghttp2_hd_context *context, - const nghttp2_nv *nv, int token, + const nghttp2_nv *nv, int32_t token, int indexing_mode, nghttp2_hd_map *map, uint32_t hash) { search_result res = {-1, 0}; @@ -1343,15 +1256,15 @@ static void hd_context_shrink_table_size(nghttp2_hd_context *context, context->hd_table.len > 0) { size_t idx = context->hd_table.len - 1; nghttp2_hd_entry *ent = hd_ringbuf_get(&context->hd_table, idx); - context->hd_table_bufsize -= entry_room(ent->nv.namelen, ent->nv.valuelen); + context->hd_table_bufsize -= + entry_room(ent->nv.name->len, ent->nv.value->len); hd_ringbuf_pop_back(&context->hd_table); if (map) { hd_map_remove(map, ent); } - if (--ent->ref == 0) { - nghttp2_hd_entry_free(ent, mem); - nghttp2_mem_free(mem, ent); - } + + nghttp2_hd_entry_free(ent); + nghttp2_mem_free(mem, ent); } } @@ -1410,19 +1323,32 @@ static size_t get_max_index(nghttp2_hd_context *context) { return context->hd_table.len + NGHTTP2_STATIC_TABLE_LENGTH - 1; } -nghttp2_hd_entry *nghttp2_hd_table_get(nghttp2_hd_context *context, - size_t idx) { +nghttp2_hd_nv nghttp2_hd_table_get(nghttp2_hd_context *context, size_t idx) { assert(INDEX_RANGE_VALID(context, idx)); if (idx >= NGHTTP2_STATIC_TABLE_LENGTH) { - return hd_ringbuf_get(&context->hd_table, - idx - NGHTTP2_STATIC_TABLE_LENGTH); + return hd_ringbuf_get(&context->hd_table, idx - NGHTTP2_STATIC_TABLE_LENGTH) + ->nv; } else { - return &static_table[idx]; + nghttp2_hd_static_entry *ent = &static_table[idx]; + nghttp2_hd_nv nv = {&ent->name, &ent->value, ent->token, + NGHTTP2_NV_FLAG_NONE}; + return nv; } } +static const nghttp2_nv *nghttp2_hd_table_get2(nghttp2_hd_context *context, + size_t idx) { + assert(INDEX_RANGE_VALID(context, idx)); + if (idx >= NGHTTP2_STATIC_TABLE_LENGTH) { + return &hd_ringbuf_get(&context->hd_table, + idx - NGHTTP2_STATIC_TABLE_LENGTH)->cnv; + } + + return &static_table[idx].cnv; +} + static int hd_deflate_decide_indexing(nghttp2_hd_deflater *deflater, - const nghttp2_nv *nv, int token) { + const nghttp2_nv *nv, int32_t token) { if (token == NGHTTP2_TOKEN__PATH || token == NGHTTP2_TOKEN_AGE || token == NGHTTP2_TOKEN_CONTENT_LENGTH || token == NGHTTP2_TOKEN_ETAG || token == NGHTTP2_TOKEN_IF_MODIFIED_SINCE || @@ -1442,9 +1368,9 @@ static int deflate_nv(nghttp2_hd_deflater *deflater, nghttp2_bufs *bufs, search_result res; ssize_t idx; int indexing_mode; - int token; + int32_t token; nghttp2_mem *mem; - uint32_t hash; + uint32_t hash = 0; DEBUGF(fprintf(stderr, "deflatehd: deflating %.*s: %.*s\n", (int)nv->namelen, nv->name, (int)nv->valuelen, nv->value)); @@ -1452,9 +1378,9 @@ static int deflate_nv(nghttp2_hd_deflater *deflater, nghttp2_bufs *bufs, mem = deflater->ctx.mem; token = lookup_token(nv->name, nv->namelen); - if (token == -1 || token > NGHTTP2_TOKEN_WWW_AUTHENTICATE) { + if (token == -1) { hash = name_hash(nv); - } else { + } else if (token <= NGHTTP2_TOKEN_WWW_AUTHENTICATE) { hash = static_table[token].hash; } @@ -1491,28 +1417,36 @@ static int deflate_nv(nghttp2_hd_deflater *deflater, nghttp2_bufs *bufs, } if (indexing_mode == NGHTTP2_HD_WITH_INDEXING) { - nghttp2_hd_entry *new_ent; + nghttp2_hd_nv hd_nv; + if (idx != -1 && idx < (ssize_t)NGHTTP2_STATIC_TABLE_LENGTH) { - nghttp2_nv nv_indname; - nv_indname = *nv; - nv_indname.name = - nghttp2_hd_table_get(&deflater->ctx, (size_t)idx)->nv.name; - new_ent = add_hd_table_incremental(&deflater->ctx, &nv_indname, token, - NGHTTP2_HD_FLAG_VALUE_ALLOC, - &deflater->map, hash); + hd_nv.name = nghttp2_hd_table_get(&deflater->ctx, (size_t)idx).name; + nghttp2_rcbuf_incref(hd_nv.name); } else { - new_ent = add_hd_table_incremental(&deflater->ctx, nv, token, - NGHTTP2_HD_FLAG_NAME_ALLOC | - NGHTTP2_HD_FLAG_VALUE_ALLOC, - &deflater->map, hash); + rv = nghttp2_rcbuf_new2(&hd_nv.name, nv->name, nv->namelen, mem); + if (rv != 0) { + return rv; + } } - if (!new_ent) { + + rv = nghttp2_rcbuf_new2(&hd_nv.value, nv->value, nv->valuelen, mem); + + if (rv != 0) { + nghttp2_rcbuf_decref(hd_nv.name); + return rv; + } + + hd_nv.token = token; + hd_nv.flags = NGHTTP2_NV_FLAG_NONE; + + rv = add_hd_table_incremental(&deflater->ctx, &hd_nv, &deflater->map, hash); + + nghttp2_rcbuf_decref(hd_nv.value); + nghttp2_rcbuf_decref(hd_nv.name); + + if (rv != 0) { return NGHTTP2_ERR_HEADER_COMP; } - if (new_ent->ref == 0) { - nghttp2_hd_entry_free(new_ent, mem); - nghttp2_mem_free(mem, new_ent); - } } if (idx == -1) { rv = emit_newname_block(bufs, nv, indexing_mode); @@ -1740,11 +1674,9 @@ static ssize_t hd_inflate_read_len(nghttp2_hd_inflater *inflater, int *rfin, * Out of memory * NGHTTP2_ERR_HEADER_COMP * Huffman decoding failed - * NGHTTP2_ERR_BUFFER_ERROR - * Out of buffer space. */ static ssize_t hd_inflate_read_huff(nghttp2_hd_inflater *inflater, - nghttp2_bufs *bufs, uint8_t *in, + nghttp2_buf *buf, uint8_t *in, uint8_t *last) { ssize_t readlen; int final = 0; @@ -1752,7 +1684,7 @@ static ssize_t hd_inflate_read_huff(nghttp2_hd_inflater *inflater, last = in + inflater->left; final = 1; } - readlen = nghttp2_hd_huff_decode(&inflater->huff_decode_ctx, bufs, in, + readlen = nghttp2_hd_huff_decode(&inflater->huff_decode_ctx, buf, in, (size_t)(last - in), final); if (readlen < 0) { @@ -1774,17 +1706,13 @@ static ssize_t hd_inflate_read_huff(nghttp2_hd_inflater *inflater, * Out of memory * NGHTTP2_ERR_HEADER_COMP * Header decompression failed - * NGHTTP2_ERR_BUFFER_ERROR - * Out of buffer space. */ -static ssize_t hd_inflate_read(nghttp2_hd_inflater *inflater, - nghttp2_bufs *bufs, uint8_t *in, uint8_t *last) { - int rv; +static ssize_t hd_inflate_read(nghttp2_hd_inflater *inflater, nghttp2_buf *buf, + uint8_t *in, uint8_t *last) { size_t len = nghttp2_min((size_t)(last - in), inflater->left); - rv = nghttp2_bufs_add(bufs, in, len); - if (rv != 0) { - return rv; - } + + buf->last = nghttp2_cpymem(buf->last, in, len); + inflater->left -= len; return (ssize_t)len; } @@ -1801,116 +1729,10 @@ static ssize_t hd_inflate_read(nghttp2_hd_inflater *inflater, * Out of memory */ static int hd_inflate_commit_indexed(nghttp2_hd_inflater *inflater, - nghttp2_nv *nv_out, int *token_out) { - nghttp2_hd_entry *ent = nghttp2_hd_table_get(&inflater->ctx, inflater->index); + nghttp2_hd_nv *nv_out) { + nghttp2_hd_nv nv = nghttp2_hd_table_get(&inflater->ctx, inflater->index); - emit_indexed_header(nv_out, token_out, ent); - - return 0; -} - -static int hd_inflate_remove_bufs(nghttp2_hd_inflater *inflater, nghttp2_nv *nv, - int value_only) { - ssize_t rv; - size_t buflen; - uint8_t *buf; - nghttp2_buf *pbuf; - - if (inflater->index_required || - inflater->nvbufs.head != inflater->nvbufs.cur) { - - rv = nghttp2_bufs_remove(&inflater->nvbufs, &buf); - - if (rv < 0) { - return NGHTTP2_ERR_NOMEM; - } - - nghttp2_bufs_reset(&inflater->nvbufs); - - buflen = (size_t)rv; - - if (value_only) { - /* we don't use this value, so no need to NULL-terminate */ - nv->name = NULL; - nv->namelen = 0; - - nv->value = buf; - nv->valuelen = buflen - 1; - } else { - nv->name = buf; - nv->namelen = inflater->newnamelen; - - nv->value = buf + nv->namelen + 1; - nv->valuelen = buflen - nv->namelen - 2; - } - - return 0; - } - - /* If we are not going to store header in header table and - name/value are in first chunk, we just refer them from nv, - instead of mallocing another memory. */ - - pbuf = &inflater->nvbufs.head->buf; - - if (value_only) { - /* we don't use this value, so no need to NULL-terminate */ - nv->name = NULL; - nv->namelen = 0; - - nv->value = pbuf->pos; - nv->valuelen = nghttp2_buf_len(pbuf) - 1; - } else { - nv->name = pbuf->pos; - nv->namelen = inflater->newnamelen; - - nv->value = pbuf->pos + nv->namelen + 1; - nv->valuelen = nghttp2_buf_len(pbuf) - nv->namelen - 2; - } - - /* Resetting does not change the content of first buffer */ - nghttp2_bufs_reset(&inflater->nvbufs); - - return 0; -} - -static int hd_inflate_remove_bufs_with_name(nghttp2_hd_inflater *inflater, - nghttp2_nv *nv, - nghttp2_hd_entry *ent_name) { -#ifndef NDEBUG - size_t rv; -#endif - size_t buflen; - uint8_t *buf; - nghttp2_mem *mem; - - mem = inflater->ctx.mem; - - /* Allocate buffer including name in ent_name, plus terminating - NULL. */ - buflen = ent_name->nv.namelen + 1 + nghttp2_bufs_len(&inflater->nvbufs); - - buf = nghttp2_mem_malloc(mem, buflen); - if (buf == NULL) { - return NGHTTP2_ERR_NOMEM; - } - - /* Copy including terminal NULL */ - memcpy(buf, ent_name->nv.name, ent_name->nv.namelen + 1); -#ifndef NDEBUG - rv = -#endif - nghttp2_bufs_remove_copy(&inflater->nvbufs, - buf + ent_name->nv.namelen + 1); - assert(ent_name->nv.namelen + 1 + rv == buflen); - - nghttp2_bufs_reset(&inflater->nvbufs); - - nv->name = buf; - nv->namelen = ent_name->nv.namelen; - - nv->value = buf + nv->namelen + 1; - nv->valuelen = buflen - nv->namelen - 2; + emit_header(nv_out, &nv); return 0; } @@ -1927,17 +1749,9 @@ static int hd_inflate_remove_bufs_with_name(nghttp2_hd_inflater *inflater, * Out of memory */ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, - nghttp2_nv *nv_out, int *token_out) { + nghttp2_hd_nv *nv_out) { + nghttp2_hd_nv nv; int rv; - nghttp2_nv nv; - nghttp2_mem *mem; - - mem = inflater->ctx.mem; - - rv = hd_inflate_remove_bufs(inflater, &nv, 0 /* name and value */); - if (rv != 0) { - return NGHTTP2_ERR_NOMEM; - } if (inflater->no_index) { nv.flags = NGHTTP2_NV_FLAG_NO_INDEX; @@ -1945,36 +1759,25 @@ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, nv.flags = NGHTTP2_NV_FLAG_NONE; } + nv.name = inflater->namercbuf; + nv.value = inflater->valuercbuf; + nv.token = lookup_token(inflater->namercbuf->base, inflater->namercbuf->len); + if (inflater->index_required) { - nghttp2_hd_entry *new_ent; - uint8_t ent_flags; + rv = add_hd_table_incremental(&inflater->ctx, &nv, NULL, 0); - /* nv->value points to the middle of the buffer pointed by - nv->name. So we just need to keep track of nv->name for memory - management. */ - ent_flags = NGHTTP2_HD_FLAG_NAME_ALLOC | NGHTTP2_HD_FLAG_NAME_GIFT; - - new_ent = add_hd_table_incremental(&inflater->ctx, &nv, - lookup_token(nv.name, nv.namelen), - ent_flags, NULL, 0); - - if (new_ent) { - emit_indexed_header(nv_out, token_out, new_ent); - inflater->ent_keep = new_ent; - - return 0; + if (rv != 0) { + return rv; } - - nghttp2_mem_free(mem, nv.name); - - return NGHTTP2_ERR_NOMEM; } - emit_literal_header(nv_out, token_out, &nv); + emit_header(nv_out, &nv); - if (nv.name != inflater->nvbufs.head->buf.pos) { - inflater->nv_keep = nv.name; - } + inflater->nv_name_keep = nv.name; + inflater->nv_value_keep = nv.value; + + inflater->namercbuf = NULL; + inflater->valuercbuf = NULL; return 0; } @@ -1991,13 +1794,11 @@ static int hd_inflate_commit_newname(nghttp2_hd_inflater *inflater, * Out of memory */ static int hd_inflate_commit_indname(nghttp2_hd_inflater *inflater, - nghttp2_nv *nv_out, int *token_out) { + nghttp2_hd_nv *nv_out) { + nghttp2_hd_nv nv; int rv; - nghttp2_nv nv; - nghttp2_hd_entry *ent_name; - nghttp2_mem *mem; - mem = inflater->ctx.mem; + nv = nghttp2_hd_table_get(&inflater->ctx, inflater->index); if (inflater->no_index) { nv.flags = NGHTTP2_NV_FLAG_NO_INDEX; @@ -2005,66 +1806,24 @@ static int hd_inflate_commit_indname(nghttp2_hd_inflater *inflater, nv.flags = NGHTTP2_NV_FLAG_NONE; } - ent_name = nghttp2_hd_table_get(&inflater->ctx, inflater->index); + nghttp2_rcbuf_incref(nv.name); + + nv.value = inflater->valuercbuf; if (inflater->index_required) { - nghttp2_hd_entry *new_ent; - uint8_t ent_flags; - - if (inflater->index < NGHTTP2_STATIC_TABLE_LENGTH) { - /* We don't copy name in static table */ - rv = hd_inflate_remove_bufs(inflater, &nv, 1 /* value only */); - if (rv != 0) { - return NGHTTP2_ERR_NOMEM; - } - nv.name = ent_name->nv.name; - nv.namelen = ent_name->nv.namelen; - - ent_flags = NGHTTP2_HD_FLAG_VALUE_ALLOC | NGHTTP2_HD_FLAG_VALUE_GIFT; - } else { - rv = hd_inflate_remove_bufs_with_name(inflater, &nv, ent_name); - if (rv != 0) { - return NGHTTP2_ERR_NOMEM; - } - /* nv->name and nv->value are in the same buffer. */ - ent_flags = NGHTTP2_HD_FLAG_NAME_ALLOC | NGHTTP2_HD_FLAG_NAME_GIFT; + rv = add_hd_table_incremental(&inflater->ctx, &nv, NULL, 0); + if (rv != 0) { + nghttp2_rcbuf_decref(nv.name); + return NGHTTP2_ERR_NOMEM; } - - new_ent = add_hd_table_incremental(&inflater->ctx, &nv, ent_name->token, - ent_flags, NULL, 0); - - /* At this point, ent_name might be deleted. */ - - if (new_ent) { - emit_indexed_header(nv_out, token_out, new_ent); - - inflater->ent_keep = new_ent; - - return 0; - } - - if (inflater->index < NGHTTP2_STATIC_TABLE_LENGTH) { - nghttp2_mem_free(mem, nv.value); - } else { - nghttp2_mem_free(mem, nv.name); - } - - return NGHTTP2_ERR_NOMEM; } - rv = hd_inflate_remove_bufs(inflater, &nv, 1 /* value only */); - if (rv != 0) { - return NGHTTP2_ERR_NOMEM; - } + emit_header(nv_out, &nv); - nv.name = ent_name->nv.name; - nv.namelen = ent_name->nv.namelen; + inflater->nv_name_keep = nv.name; + inflater->nv_value_keep = nv.value; - emit_literal_header(nv_out, token_out, &nv); - - if (nv.value != inflater->nvbufs.head->buf.pos) { - inflater->nv_keep = nv.value; - } + inflater->valuercbuf = NULL; return 0; } @@ -2072,21 +1831,40 @@ static int hd_inflate_commit_indname(nghttp2_hd_inflater *inflater, ssize_t nghttp2_hd_inflate_hd(nghttp2_hd_inflater *inflater, nghttp2_nv *nv_out, int *inflate_flags, uint8_t *in, size_t inlen, int in_final) { - int token; + ssize_t rv; + nghttp2_hd_nv hd_nv; - return nghttp2_hd_inflate_hd2(inflater, nv_out, inflate_flags, &token, in, - inlen, in_final); + rv = nghttp2_hd_inflate_hd2(inflater, &hd_nv, inflate_flags, in, inlen, + in_final); + + if (rv < 0) { + return rv; + } + + if (*inflate_flags & NGHTTP2_HD_INFLATE_EMIT) { + nv_out->name = hd_nv.name->base; + nv_out->namelen = hd_nv.name->len; + + nv_out->value = hd_nv.value->base; + nv_out->valuelen = hd_nv.value->len; + + nv_out->flags = hd_nv.flags; + } + + return rv; } ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, - nghttp2_nv *nv_out, int *inflate_flags, - int *token_out, uint8_t *in, size_t inlen, - int in_final) { + nghttp2_hd_nv *nv_out, int *inflate_flags, + uint8_t *in, size_t inlen, int in_final) { ssize_t rv = 0; uint8_t *first = in; uint8_t *last = in + inlen; int rfin = 0; int busy = 0; + nghttp2_mem *mem; + + mem = inflater->ctx.mem; if (inflater->ctx.bad) { return NGHTTP2_ERR_HEADER_COMP; @@ -2094,7 +1872,6 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, DEBUGF(fprintf(stderr, "inflatehd: start state=%d\n", inflater->state)); hd_inflate_keep_free(inflater); - *token_out = -1; *inflate_flags = NGHTTP2_HD_INFLATE_NONE; for (; in != last || busy;) { busy = 0; @@ -2200,7 +1977,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, inflater->index = inflater->left; --inflater->index; - rv = hd_inflate_commit_indexed(inflater, nv_out, token_out); + rv = hd_inflate_commit_indexed(inflater, nv_out); if (rv < 0) { goto fail; } @@ -2245,12 +2022,24 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, nghttp2_hd_huff_decode_context_init(&inflater->huff_decode_ctx); inflater->state = NGHTTP2_HD_STATE_NEWNAME_READ_NAMEHUFF; + + rv = nghttp2_rcbuf_new(&inflater->namercbuf, inflater->left * 2 + 1, + mem); } else { inflater->state = NGHTTP2_HD_STATE_NEWNAME_READ_NAME; + rv = nghttp2_rcbuf_new(&inflater->namercbuf, inflater->left + 1, mem); } + + if (rv != 0) { + goto fail; + } + + nghttp2_buf_wrap_init(&inflater->namebuf, inflater->namercbuf->base, + inflater->namercbuf->len); + break; case NGHTTP2_HD_STATE_NEWNAME_READ_NAMEHUFF: - rv = hd_inflate_read_huff(inflater, &inflater->nvbufs, in, last); + rv = hd_inflate_read_huff(inflater, &inflater->namebuf, in, last); if (rv < 0) { goto fail; } @@ -2266,18 +2055,14 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, goto almost_ok; } - inflater->newnamelen = nghttp2_bufs_len(&inflater->nvbufs); - - rv = nghttp2_bufs_addb(&inflater->nvbufs, '\0'); - if (rv != 0) { - goto fail; - } + *inflater->namebuf.last = '\0'; + inflater->namercbuf->len = nghttp2_buf_len(&inflater->namebuf); inflater->state = NGHTTP2_HD_STATE_CHECK_VALUELEN; break; case NGHTTP2_HD_STATE_NEWNAME_READ_NAME: - rv = hd_inflate_read(inflater, &inflater->nvbufs, in, last); + rv = hd_inflate_read(inflater, &inflater->namebuf, in, last); if (rv < 0) { goto fail; } @@ -2292,12 +2077,8 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, goto almost_ok; } - inflater->newnamelen = nghttp2_bufs_len(&inflater->nvbufs); - - rv = nghttp2_bufs_addb(&inflater->nvbufs, '\0'); - if (rv != 0) { - goto fail; - } + *inflater->namebuf.last = '\0'; + inflater->namercbuf->len = nghttp2_buf_len(&inflater->namebuf); inflater->state = NGHTTP2_HD_STATE_CHECK_VALUELEN; @@ -2329,15 +2110,27 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, nghttp2_hd_huff_decode_context_init(&inflater->huff_decode_ctx); inflater->state = NGHTTP2_HD_STATE_READ_VALUEHUFF; + + rv = nghttp2_rcbuf_new(&inflater->valuercbuf, inflater->left * 2 + 1, + mem); } else { inflater->state = NGHTTP2_HD_STATE_READ_VALUE; + + rv = nghttp2_rcbuf_new(&inflater->valuercbuf, inflater->left + 1, mem); } + if (rv != 0) { + goto fail; + } + + nghttp2_buf_wrap_init(&inflater->valuebuf, inflater->valuercbuf->base, + inflater->valuercbuf->len); + busy = 1; break; case NGHTTP2_HD_STATE_READ_VALUEHUFF: - rv = hd_inflate_read_huff(inflater, &inflater->nvbufs, in, last); + rv = hd_inflate_read_huff(inflater, &inflater->valuebuf, in, last); if (rv < 0) { goto fail; } @@ -2353,15 +2146,13 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, goto almost_ok; } - rv = nghttp2_bufs_addb(&inflater->nvbufs, '\0'); - if (rv != 0) { - goto fail; - } + *inflater->valuebuf.last = '\0'; + inflater->valuercbuf->len = nghttp2_buf_len(&inflater->valuebuf); if (inflater->opcode == NGHTTP2_HD_OPCODE_NEWNAME) { - rv = hd_inflate_commit_newname(inflater, nv_out, token_out); + rv = hd_inflate_commit_newname(inflater, nv_out); } else { - rv = hd_inflate_commit_indname(inflater, nv_out, token_out); + rv = hd_inflate_commit_indname(inflater, nv_out); } if (rv != 0) { @@ -2373,7 +2164,7 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, return (ssize_t)(in - first); case NGHTTP2_HD_STATE_READ_VALUE: - rv = hd_inflate_read(inflater, &inflater->nvbufs, in, last); + rv = hd_inflate_read(inflater, &inflater->valuebuf, in, last); if (rv < 0) { DEBUGF(fprintf(stderr, "inflatehd: value read failure %zd: %s\n", rv, nghttp2_strerror((int)rv))); @@ -2390,15 +2181,13 @@ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, goto almost_ok; } - rv = nghttp2_bufs_addb(&inflater->nvbufs, '\0'); - if (rv != 0) { - goto fail; - } + *inflater->valuebuf.last = '\0'; + inflater->valuercbuf->len = nghttp2_buf_len(&inflater->valuebuf); if (inflater->opcode == NGHTTP2_HD_OPCODE_NEWNAME) { - rv = hd_inflate_commit_newname(inflater, nv_out, token_out); + rv = hd_inflate_commit_newname(inflater, nv_out); } else { - rv = hd_inflate_commit_indname(inflater, nv_out, token_out); + rv = hd_inflate_commit_indname(inflater, nv_out); } if (rv != 0) { @@ -2532,7 +2321,7 @@ static const nghttp2_nv *hd_get_table_entry(nghttp2_hd_context *context, return NULL; } - return &nghttp2_hd_table_get(context, idx)->nv; + return nghttp2_hd_table_get2(context, idx); } size_t nghttp2_hd_deflate_get_num_table_entries(nghttp2_hd_deflater *deflater) { diff --git a/lib/nghttp2_hd.h b/lib/nghttp2_hd.h index c667888e..b0e93852 100644 --- a/lib/nghttp2_hd.h +++ b/lib/nghttp2_hd.h @@ -34,6 +34,7 @@ #include "nghttp2_hd_huffman.h" #include "nghttp2_buf.h" #include "nghttp2_mem.h" +#include "nghttp2_rcbuf.h" #define NGHTTP2_HD_DEFAULT_MAX_BUFFER_SIZE NGHTTP2_DEFAULT_HEADER_TABLE_SIZE #define NGHTTP2_HD_ENTRY_OVERHEAD 32 @@ -109,28 +110,32 @@ typedef enum { NGHTTP2_TOKEN_CONNECTION, NGHTTP2_TOKEN_KEEP_ALIVE, NGHTTP2_TOKEN_PROXY_CONNECTION, - NGHTTP2_TOKEN_UPGRADE + NGHTTP2_TOKEN_UPGRADE, } nghttp2_token; -typedef enum { - NGHTTP2_HD_FLAG_NONE = 0, - /* Indicates name was dynamically allocated and must be freed */ - NGHTTP2_HD_FLAG_NAME_ALLOC = 1, - /* Indicates value was dynamically allocated and must be freed */ - NGHTTP2_HD_FLAG_VALUE_ALLOC = 1 << 1, - /* Indicates that the name was gifted to the entry and no copying - necessary. */ - NGHTTP2_HD_FLAG_NAME_GIFT = 1 << 2, - /* Indicates that the value was gifted to the entry and no copying - necessary. */ - NGHTTP2_HD_FLAG_VALUE_GIFT = 1 << 3 -} nghttp2_hd_flags; - struct nghttp2_hd_entry; typedef struct nghttp2_hd_entry nghttp2_hd_entry; +typedef struct { + /* The buffer containing header field name. NULL-termination is + guaranteed. */ + nghttp2_rcbuf *name; + /* The buffer containing header field value. NULL-termination is + guaranteed. */ + nghttp2_rcbuf *value; + /* nghttp2_token value for name. It could be -1 if we have no token + for that header field name. */ + int32_t token; + /* Bitwise OR of one or more of nghttp2_nv_flag. */ + uint8_t flags; +} nghttp2_hd_nv; + struct nghttp2_hd_entry { - nghttp2_nv nv; + /* The header field name/value pair */ + nghttp2_hd_nv nv; + /* This is solely for nghttp2_hd_{deflate,inflate}_get_table_entry + APIs to keep backward compatibility. */ + nghttp2_nv cnv; /* The next entry which shares same bucket in hash table. */ nghttp2_hd_entry *next; /* The sequence number. We will increment it by one whenever we @@ -138,14 +143,17 @@ struct nghttp2_hd_entry { uint32_t seq; /* The hash value for header name (nv.name). */ uint32_t hash; - /* nghttp2_token value for nv.name. It could be -1 if we have no - token for that header field name. */ - int token; - /* Reference count */ - uint8_t ref; - uint8_t flags; }; +/* The entry used for static header table. */ +typedef struct { + nghttp2_rcbuf name; + nghttp2_rcbuf value; + nghttp2_nv cnv; + int32_t token; + uint32_t hash; +} nghttp2_hd_static_entry; + typedef struct { nghttp2_hd_entry **buffer; size_t mask; @@ -219,24 +227,18 @@ struct nghttp2_hd_deflater { struct nghttp2_hd_inflater { nghttp2_hd_context ctx; - /* header buffer */ - nghttp2_bufs nvbufs; /* Stores current state of huffman decoding */ nghttp2_hd_huff_decode_context huff_decode_ctx; - /* Pointer to the nghttp2_hd_entry which is used current header - emission. This is required because in some cases the - ent_keep->ref == 0 and we have to keep track of it. */ - nghttp2_hd_entry *ent_keep; - /* Pointer to the name/value pair buffer which is used in the - current header emission. */ - uint8_t *nv_keep; + /* header buffer */ + nghttp2_buf namebuf, valuebuf; + nghttp2_rcbuf *namercbuf, *valuercbuf; + /* Pointer to the name/value pair which are used in the current + header emission. */ + nghttp2_rcbuf *nv_name_keep, *nv_value_keep; /* The number of bytes to read */ size_t left; /* The index in indexed repr or indexed name */ size_t index; - /* The length of new name encoded in literal. For huffman encoded - string, this is the length after it is decoded. */ - size_t newnamelen; /* The maximum header table size the inflater supports. This is the same value transmitted in SETTINGS_HEADER_TABLE_SIZE */ size_t settings_hd_table_bufsize_max; @@ -256,24 +258,16 @@ struct nghttp2_hd_inflater { }; /* - * Initializes the |ent| members. If NGHTTP2_HD_FLAG_NAME_ALLOC bit - * set in the |flags|, the content pointed by the |name| with length - * |namelen| is copied. Likewise, if NGHTTP2_HD_FLAG_VALUE_ALLOC bit - * set in the |flags|, the content pointed by the |value| with length - * |valuelen| is copied. The |token| is enum number looked up by - * |name|. It could be -1 if we don't have that enum value. - * - * This function returns 0 if it succeeds, or one of the following - * negative error codes: - * - * NGHTTP2_ERR_NOMEM - * Out of memory. + * Initializes the |ent| members. The reference counts of nv->name + * and nv->value are increased by one for each. */ -int nghttp2_hd_entry_init(nghttp2_hd_entry *ent, uint8_t flags, uint8_t *name, - size_t namelen, uint8_t *value, size_t valuelen, - int token, nghttp2_mem *mem); +void nghttp2_hd_entry_init(nghttp2_hd_entry *ent, nghttp2_hd_nv *nv); -void nghttp2_hd_entry_free(nghttp2_hd_entry *ent, nghttp2_mem *mem); +/* + * This function decreases the reference counts of nv->name and + * nv->value. + */ +void nghttp2_hd_entry_free(nghttp2_hd_entry *ent); /* * Initializes |deflater| for deflating name/values pairs. @@ -354,16 +348,14 @@ int nghttp2_hd_inflate_init(nghttp2_hd_inflater *inflater, nghttp2_mem *mem); void nghttp2_hd_inflate_free(nghttp2_hd_inflater *inflater); /* - * Similar to nghttp2_hd_inflate_hd(), but this takes additional - * output parameter |token|. On successful header emission, it - * contains nghttp2_token value for nv_out->name. It could be -1 if - * we don't have enum value for the name. Other than that return - * values and semantics are the same as nghttp2_hd_inflate_hd(). + * Similar to nghttp2_hd_inflate_hd(), but this takes nghttp2_hd_nv + * instead of nghttp2_nv as output parameter |nv_out|. Other than + * that return values and semantics are the same as + * nghttp2_hd_inflate_hd(). */ ssize_t nghttp2_hd_inflate_hd2(nghttp2_hd_inflater *inflater, - nghttp2_nv *nv_out, int *inflate_flags, - int *token, uint8_t *in, size_t inlen, - int in_final); + nghttp2_hd_nv *nv_out, int *inflate_flags, + uint8_t *in, size_t inlen, int in_final); /* For unittesting purpose */ int nghttp2_hd_emit_indname_block(nghttp2_bufs *bufs, size_t index, @@ -377,8 +369,7 @@ int nghttp2_hd_emit_newname_block(nghttp2_bufs *bufs, nghttp2_nv *nv, int nghttp2_hd_emit_table_size(nghttp2_bufs *bufs, size_t table_size); /* For unittesting purpose */ -nghttp2_hd_entry *nghttp2_hd_table_get(nghttp2_hd_context *context, - size_t index); +nghttp2_hd_nv nghttp2_hd_table_get(nghttp2_hd_context *context, size_t index); /* For unittesting purpose */ ssize_t nghttp2_hd_decode_length(uint32_t *res, size_t *shift_ptr, int *final, @@ -414,11 +405,10 @@ int nghttp2_hd_huff_encode(nghttp2_bufs *bufs, const uint8_t *src, void nghttp2_hd_huff_decode_context_init(nghttp2_hd_huff_decode_context *ctx); /* - * Decodes the given data |src| with length |srclen|. The |ctx| must + * Decodes the given data |src| with length |srclen|. The |ctx| must * be initialized by nghttp2_hd_huff_decode_context_init(). The result - * will be added to |dest|. This function may expand |dest| as - * needed. The caller is responsible to release the memory of |dest| - * by calling nghttp2_bufs_free(). + * will be written to |buf|. This function assumes that |buf| has the + * enough room to store the decoded byte string. * * The caller must set the |final| to nonzero if the given input is * the final block. @@ -430,13 +420,11 @@ void nghttp2_hd_huff_decode_context_init(nghttp2_hd_huff_decode_context *ctx); * * NGHTTP2_ERR_NOMEM * Out of memory. - * NGHTTP2_ERR_BUFFER_ERROR - * Maximum buffer capacity size exceeded. * NGHTTP2_ERR_HEADER_COMP * Decoding process has failed. */ ssize_t nghttp2_hd_huff_decode(nghttp2_hd_huff_decode_context *ctx, - nghttp2_bufs *bufs, const uint8_t *src, + nghttp2_buf *buf, const uint8_t *src, size_t srclen, int final); #endif /* NGHTTP2_HD_H */ diff --git a/lib/nghttp2_hd_huffman.c b/lib/nghttp2_hd_huffman.c index 48638b75..3fb0d886 100644 --- a/lib/nghttp2_hd_huffman.c +++ b/lib/nghttp2_hd_huffman.c @@ -166,31 +166,10 @@ void nghttp2_hd_huff_decode_context_init(nghttp2_hd_huff_decode_context *ctx) { ctx->accept = 1; } -/* Use macro to make the code simpler..., but error case is tricky. - We spent most of the CPU in decoding, so we are doing this - thing. */ -#define hd_huff_decode_sym_emit(bufs, sym, avail) \ - do { \ - if ((avail)) { \ - nghttp2_bufs_fast_addb((bufs), (sym)); \ - --(avail); \ - } else { \ - rv = nghttp2_bufs_addb((bufs), (sym)); \ - if (rv != 0) { \ - return rv; \ - } \ - (avail) = nghttp2_bufs_cur_avail((bufs)); \ - } \ - } while (0) - ssize_t nghttp2_hd_huff_decode(nghttp2_hd_huff_decode_context *ctx, - nghttp2_bufs *bufs, const uint8_t *src, + nghttp2_buf *buf, const uint8_t *src, size_t srclen, int final) { size_t i; - int rv; - size_t avail; - - avail = nghttp2_bufs_cur_avail(bufs); /* We use the decoding algorithm described in http://graphics.ics.uci.edu/pub/Prefix.pdf */ @@ -202,8 +181,7 @@ ssize_t nghttp2_hd_huff_decode(nghttp2_hd_huff_decode_context *ctx, return NGHTTP2_ERR_HEADER_COMP; } if (t->flags & NGHTTP2_HUFF_SYM) { - /* this is macro, and may return from this function on error */ - hd_huff_decode_sym_emit(bufs, t->sym, avail); + *buf->last++ = t->sym; } t = &huff_decode_table[t->state][src[i] & 0xf]; @@ -211,8 +189,7 @@ ssize_t nghttp2_hd_huff_decode(nghttp2_hd_huff_decode_context *ctx, return NGHTTP2_ERR_HEADER_COMP; } if (t->flags & NGHTTP2_HUFF_SYM) { - /* this is macro, and may return from this function on error */ - hd_huff_decode_sym_emit(bufs, t->sym, avail); + *buf->last++ = t->sym; } ctx->state = t->state; diff --git a/lib/nghttp2_helper.c b/lib/nghttp2_helper.c index 884abc68..9f208519 100644 --- a/lib/nghttp2_helper.c +++ b/lib/nghttp2_helper.c @@ -288,6 +288,8 @@ const char *nghttp2_strerror(int error_code) { return "Stream was refused"; case NGHTTP2_ERR_INTERNAL: return "Internal error"; + case NGHTTP2_ERR_CANCEL: + return "Cancel"; case NGHTTP2_ERR_NOMEM: return "Out of memory"; case NGHTTP2_ERR_CALLBACK_FAILURE: @@ -449,3 +451,38 @@ uint8_t *nghttp2_cpymem(uint8_t *dest, const void *src, size_t len) { return dest + len; } + +const char *nghttp2_http2_strerror(uint32_t error_code) { + switch (error_code) { + case NGHTTP2_NO_ERROR: + return "NO_ERROR"; + case NGHTTP2_PROTOCOL_ERROR: + return "PROTOCOL_ERROR"; + case NGHTTP2_INTERNAL_ERROR: + return "INTERNAL_ERROR"; + case NGHTTP2_FLOW_CONTROL_ERROR: + return "FLOW_CONTROL_ERROR"; + case NGHTTP2_SETTINGS_TIMEOUT: + return "SETTINGS_TIMEOUT"; + case NGHTTP2_STREAM_CLOSED: + return "STREAM_CLOSED"; + case NGHTTP2_FRAME_SIZE_ERROR: + return "FRAME_SIZE_ERROR"; + case NGHTTP2_REFUSED_STREAM: + return "REFUSED_STREAM"; + case NGHTTP2_CANCEL: + return "CANCEL"; + case NGHTTP2_COMPRESSION_ERROR: + return "COMPRESSION_ERROR"; + case NGHTTP2_CONNECT_ERROR: + return "CONNECT_ERROR"; + case NGHTTP2_ENHANCE_YOUR_CALM: + return "ENHANCE_YOUR_CALM"; + case NGHTTP2_INADEQUATE_SECURITY: + return "INADEQUATE_SECURITY"; + case NGHTTP2_HTTP_1_1_REQUIRED: + return "HTTP_1_1_REQUIRED"; + default: + return "unknown"; + } +} diff --git a/lib/nghttp2_http.c b/lib/nghttp2_http.c index 6aee90f0..f993c167 100644 --- a/lib/nghttp2_http.c +++ b/lib/nghttp2_http.c @@ -82,12 +82,12 @@ static int lws(const uint8_t *s, size_t n) { return 1; } -static int check_pseudo_header(nghttp2_stream *stream, const nghttp2_nv *nv, +static int check_pseudo_header(nghttp2_stream *stream, const nghttp2_hd_nv *nv, int flag) { if (stream->http_flags & flag) { return 0; } - if (lws(nv->value, nv->valuelen)) { + if (lws(nv->value->base, nv->value->len)) { return 0; } stream->http_flags = (uint16_t)(stream->http_flags | flag); @@ -112,16 +112,16 @@ static int check_path(nghttp2_stream *stream) { (stream->http_flags & NGHTTP2_HTTP_FLAG_PATH_ASTERISK))); } -static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, - int token, int trailer) { - if (nv->name[0] == ':') { +static int http_request_on_header(nghttp2_stream *stream, nghttp2_hd_nv *nv, + int trailer) { + if (nv->name->base[0] == ':') { if (trailer || (stream->http_flags & NGHTTP2_HTTP_FLAG_PSEUDO_HEADER_DISALLOWED)) { return NGHTTP2_ERR_HTTP_HEADER; } } - switch (token) { + switch (nv->token) { case NGHTTP2_TOKEN__AUTHORITY: if (!check_pseudo_header(stream, nv, NGHTTP2_HTTP_FLAG__AUTHORITY)) { return NGHTTP2_ERR_HTTP_HEADER; @@ -131,16 +131,16 @@ static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, if (!check_pseudo_header(stream, nv, NGHTTP2_HTTP_FLAG__METHOD)) { return NGHTTP2_ERR_HTTP_HEADER; } - switch (nv->valuelen) { + switch (nv->value->len) { case 4: - if (lstreq("HEAD", nv->value, nv->valuelen)) { + if (lstreq("HEAD", nv->value->base, nv->value->len)) { stream->http_flags |= NGHTTP2_HTTP_FLAG_METH_HEAD; } break; case 7: - switch (nv->value[6]) { + switch (nv->value->base[6]) { case 'T': - if (lstreq("CONNECT", nv->value, nv->valuelen)) { + if (lstreq("CONNECT", nv->value->base, nv->value->len)) { if (stream->stream_id % 2 == 0) { /* we won't allow CONNECT for push */ return NGHTTP2_ERR_HTTP_HEADER; @@ -153,7 +153,7 @@ static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, } break; case 'S': - if (lstreq("OPTIONS", nv->value, nv->valuelen)) { + if (lstreq("OPTIONS", nv->value->base, nv->value->len)) { stream->http_flags |= NGHTTP2_HTTP_FLAG_METH_OPTIONS; } break; @@ -168,9 +168,9 @@ static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, if (!check_pseudo_header(stream, nv, NGHTTP2_HTTP_FLAG__PATH)) { return NGHTTP2_ERR_HTTP_HEADER; } - if (nv->value[0] == '/') { + if (nv->value->base[0] == '/') { stream->http_flags |= NGHTTP2_HTTP_FLAG_PATH_REGULAR; - } else if (nv->valuelen == 1 && nv->value[0] == '*') { + } else if (nv->value->len == 1 && nv->value->base[0] == '*') { stream->http_flags |= NGHTTP2_HTTP_FLAG_PATH_ASTERISK; } break; @@ -181,8 +181,8 @@ static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, if (!check_pseudo_header(stream, nv, NGHTTP2_HTTP_FLAG__SCHEME)) { return NGHTTP2_ERR_HTTP_HEADER; } - if ((nv->valuelen == 4 && memieq("http", nv->value, 4)) || - (nv->valuelen == 5 && memieq("https", nv->value, 5))) { + if ((nv->value->len == 4 && memieq("http", nv->value->base, 4)) || + (nv->value->len == 5 && memieq("https", nv->value->base, 5))) { stream->http_flags |= NGHTTP2_HTTP_FLAG_SCHEME_HTTP; } break; @@ -195,7 +195,7 @@ static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, if (stream->content_length != -1) { return NGHTTP2_ERR_HTTP_HEADER; } - stream->content_length = parse_uint(nv->value, nv->valuelen); + stream->content_length = parse_uint(nv->value->base, nv->value->len); if (stream->content_length == -1) { return NGHTTP2_ERR_HTTP_HEADER; } @@ -209,41 +209,41 @@ static int http_request_on_header(nghttp2_stream *stream, nghttp2_nv *nv, case NGHTTP2_TOKEN_UPGRADE: return NGHTTP2_ERR_HTTP_HEADER; case NGHTTP2_TOKEN_TE: - if (!lstrieq("trailers", nv->value, nv->valuelen)) { + if (!lstrieq("trailers", nv->value->base, nv->value->len)) { return NGHTTP2_ERR_HTTP_HEADER; } break; default: - if (nv->name[0] == ':') { + if (nv->name->base[0] == ':') { return NGHTTP2_ERR_HTTP_HEADER; } } - if (nv->name[0] != ':') { + if (nv->name->base[0] != ':') { stream->http_flags |= NGHTTP2_HTTP_FLAG_PSEUDO_HEADER_DISALLOWED; } return 0; } -static int http_response_on_header(nghttp2_stream *stream, nghttp2_nv *nv, - int token, int trailer) { - if (nv->name[0] == ':') { +static int http_response_on_header(nghttp2_stream *stream, nghttp2_hd_nv *nv, + int trailer) { + if (nv->name->base[0] == ':') { if (trailer || (stream->http_flags & NGHTTP2_HTTP_FLAG_PSEUDO_HEADER_DISALLOWED)) { return NGHTTP2_ERR_HTTP_HEADER; } } - switch (token) { + switch (nv->token) { case NGHTTP2_TOKEN__STATUS: { if (!check_pseudo_header(stream, nv, NGHTTP2_HTTP_FLAG__STATUS)) { return NGHTTP2_ERR_HTTP_HEADER; } - if (nv->valuelen != 3) { + if (nv->value->len != 3) { return NGHTTP2_ERR_HTTP_HEADER; } - stream->status_code = (int16_t)parse_uint(nv->value, nv->valuelen); + stream->status_code = (int16_t)parse_uint(nv->value->base, nv->value->len); if (stream->status_code == -1) { return NGHTTP2_ERR_HTTP_HEADER; } @@ -253,7 +253,7 @@ static int http_response_on_header(nghttp2_stream *stream, nghttp2_nv *nv, if (stream->content_length != -1) { return NGHTTP2_ERR_HTTP_HEADER; } - stream->content_length = parse_uint(nv->value, nv->valuelen); + stream->content_length = parse_uint(nv->value->base, nv->value->len); if (stream->content_length == -1) { return NGHTTP2_ERR_HTTP_HEADER; } @@ -267,17 +267,17 @@ static int http_response_on_header(nghttp2_stream *stream, nghttp2_nv *nv, case NGHTTP2_TOKEN_UPGRADE: return NGHTTP2_ERR_HTTP_HEADER; case NGHTTP2_TOKEN_TE: - if (!lstrieq("trailers", nv->value, nv->valuelen)) { + if (!lstrieq("trailers", nv->value->base, nv->value->len)) { return NGHTTP2_ERR_HTTP_HEADER; } break; default: - if (nv->name[0] == ':') { + if (nv->name->base[0] == ':') { return NGHTTP2_ERR_HTTP_HEADER; } } - if (nv->name[0] != ':') { + if (nv->name->base[0] != ':') { stream->http_flags |= NGHTTP2_HTTP_FLAG_PSEUDO_HEADER_DISALLOWED; } @@ -375,7 +375,7 @@ static int check_scheme(const uint8_t *value, size_t len) { } int nghttp2_http_on_header(nghttp2_session *session, nghttp2_stream *stream, - nghttp2_frame *frame, nghttp2_nv *nv, int token, + nghttp2_frame *frame, nghttp2_hd_nv *nv, int trailer) { int rv; @@ -386,14 +386,14 @@ int nghttp2_http_on_header(nghttp2_session *session, nghttp2_stream *stream, this, we may disrupt many web sites and/or libraries. So we become conservative here, and just ignore those illegal regular headers. */ - if (!nghttp2_check_header_name(nv->name, nv->namelen)) { + if (!nghttp2_check_header_name(nv->name->base, nv->name->len)) { size_t i; - if (nv->namelen > 0 && nv->name[0] == ':') { + if (nv->name->len > 0 && nv->name->base[0] == ':') { return NGHTTP2_ERR_HTTP_HEADER; } /* header field name must be lower-cased without exception */ - for (i = 0; i < nv->namelen; ++i) { - uint8_t c = nv->name[i]; + for (i = 0; i < nv->name->len; ++i) { + uint8_t c = nv->name->base[i]; if ('A' <= c && c <= 'Z') { return NGHTTP2_ERR_HTTP_HEADER; } @@ -405,17 +405,18 @@ int nghttp2_http_on_header(nghttp2_session *session, nghttp2_stream *stream, return NGHTTP2_ERR_IGN_HTTP_HEADER; } - if (token == NGHTTP2_TOKEN__AUTHORITY || token == NGHTTP2_TOKEN_HOST) { - rv = check_authority(nv->value, nv->valuelen); - } else if (token == NGHTTP2_TOKEN__SCHEME) { - rv = check_scheme(nv->value, nv->valuelen); + if (nv->token == NGHTTP2_TOKEN__AUTHORITY || + nv->token == NGHTTP2_TOKEN_HOST) { + rv = check_authority(nv->value->base, nv->value->len); + } else if (nv->token == NGHTTP2_TOKEN__SCHEME) { + rv = check_scheme(nv->value->base, nv->value->len); } else { - rv = nghttp2_check_header_value(nv->value, nv->valuelen); + rv = nghttp2_check_header_value(nv->value->base, nv->value->len); } if (rv == 0) { - assert(nv->namelen > 0); - if (nv->name[0] == ':') { + assert(nv->name->len > 0); + if (nv->name->base[0] == ':') { return NGHTTP2_ERR_HTTP_HEADER; } /* When ignoring regular headers, we set this flag so that we @@ -426,10 +427,10 @@ int nghttp2_http_on_header(nghttp2_session *session, nghttp2_stream *stream, } if (session->server || frame->hd.type == NGHTTP2_PUSH_PROMISE) { - return http_request_on_header(stream, nv, token, trailer); + return http_request_on_header(stream, nv, trailer); } - return http_response_on_header(stream, nv, token, trailer); + return http_response_on_header(stream, nv, trailer); } int nghttp2_http_on_request_headers(nghttp2_stream *stream, diff --git a/lib/nghttp2_http.h b/lib/nghttp2_http.h index f782058f..ac684c4d 100644 --- a/lib/nghttp2_http.h +++ b/lib/nghttp2_http.h @@ -36,8 +36,7 @@ /* * This function is called when HTTP header field |nv| in |frame| is * received for |stream|. This function will validate |nv| against - * the current state of stream. The |token| is nghttp2_token value - * for nv->name, or -1 if we don't have enum value for the name. + * the current state of stream. * * This function returns 0 if it succeeds, or one of the following * negative error codes: @@ -49,7 +48,7 @@ * if it was not received because of compatibility reasons. */ int nghttp2_http_on_header(nghttp2_session *session, nghttp2_stream *stream, - nghttp2_frame *frame, nghttp2_nv *nv, int token, + nghttp2_frame *frame, nghttp2_hd_nv *nv, int trailer); /* diff --git a/lib/nghttp2_mem.c b/lib/nghttp2_mem.c index e7d5aae3..317363a7 100644 --- a/lib/nghttp2_mem.c +++ b/lib/nghttp2_mem.c @@ -52,6 +52,10 @@ void nghttp2_mem_free(nghttp2_mem *mem, void *ptr) { mem->free(ptr, mem->mem_user_data); } +void nghttp2_mem_free2(nghttp2_free free, void *ptr, void *mem_user_data) { + free(ptr, mem_user_data); +} + void *nghttp2_mem_calloc(nghttp2_mem *mem, size_t nmemb, size_t size) { return mem->calloc(nmemb, size, mem->mem_user_data); } diff --git a/lib/nghttp2_mem.h b/lib/nghttp2_mem.h index d1fded4f..7709e1c5 100644 --- a/lib/nghttp2_mem.h +++ b/lib/nghttp2_mem.h @@ -38,6 +38,7 @@ nghttp2_mem *nghttp2_mem_default(void); |mem|. */ void *nghttp2_mem_malloc(nghttp2_mem *mem, size_t size); void nghttp2_mem_free(nghttp2_mem *mem, void *ptr); +void nghttp2_mem_free2(nghttp2_free free, void *ptr, void *mem_user_data); void *nghttp2_mem_calloc(nghttp2_mem *mem, size_t nmemb, size_t size); void *nghttp2_mem_realloc(nghttp2_mem *mem, void *ptr, size_t size); diff --git a/lib/nghttp2_option.c b/lib/nghttp2_option.c index 04dbbc6a..e3e8717b 100644 --- a/lib/nghttp2_option.c +++ b/lib/nghttp2_option.c @@ -62,3 +62,19 @@ void nghttp2_option_set_max_reserved_remote_streams(nghttp2_option *option, option->opt_set_mask |= NGHTTP2_OPT_MAX_RESERVED_REMOTE_STREAMS; option->max_reserved_remote_streams = val; } + +void nghttp2_option_set_user_recv_extension_type(nghttp2_option *option, + uint8_t type) { + if (type < 10) { + return; + } + + option->opt_set_mask |= NGHTTP2_OPT_USER_RECV_EXT_TYPES; + option->user_recv_ext_types[type / 8] = + (uint8_t)(option->user_recv_ext_types[type / 8] | (1 << (type & 0x7))); +} + +void nghttp2_option_set_no_auto_ping_ack(nghttp2_option *option, int val) { + option->opt_set_mask |= NGHTTP2_OPT_NO_AUTO_PING_ACK; + option->no_auto_ping_ack = val; +} diff --git a/lib/nghttp2_option.h b/lib/nghttp2_option.h index ebf416ac..f859b4eb 100644 --- a/lib/nghttp2_option.h +++ b/lib/nghttp2_option.h @@ -59,7 +59,9 @@ typedef enum { NGHTTP2_OPT_PEER_MAX_CONCURRENT_STREAMS = 1 << 1, NGHTTP2_OPT_NO_RECV_CLIENT_MAGIC = 1 << 2, NGHTTP2_OPT_NO_HTTP_MESSAGING = 1 << 3, - NGHTTP2_OPT_MAX_RESERVED_REMOTE_STREAMS = 1 << 4 + NGHTTP2_OPT_MAX_RESERVED_REMOTE_STREAMS = 1 << 4, + NGHTTP2_OPT_USER_RECV_EXT_TYPES = 1 << 5, + NGHTTP2_OPT_NO_AUTO_PING_ACK = 1 << 6 } nghttp2_option_flag; /** @@ -91,6 +93,14 @@ struct nghttp2_option { * NGHTTP2_OPT_NO_HTTP_MESSAGING */ int no_http_messaging; + /** + * NGHTTP2_OPT_NO_AUTO_PING_ACK + */ + int no_auto_ping_ack; + /** + * NGHTTP2_OPT_USER_RECV_EXT_TYPES + */ + uint8_t user_recv_ext_types[32]; }; #endif /* NGHTTP2_OPTION_H */ diff --git a/lib/nghttp2_outbound_item.c b/lib/nghttp2_outbound_item.c index c4ecab90..886f330c 100644 --- a/lib/nghttp2_outbound_item.c +++ b/lib/nghttp2_outbound_item.c @@ -72,6 +72,9 @@ void nghttp2_outbound_item_free(nghttp2_outbound_item *item, nghttp2_mem *mem) { case NGHTTP2_WINDOW_UPDATE: nghttp2_frame_window_update_free(&frame->window_update); break; + default: + nghttp2_frame_extension_free(&frame->ext); + break; } } diff --git a/lib/nghttp2_rcbuf.c b/lib/nghttp2_rcbuf.c new file mode 100644 index 00000000..053f0dfa --- /dev/null +++ b/lib/nghttp2_rcbuf.c @@ -0,0 +1,99 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "nghttp2_rcbuf.h" + +#include +#include + +#include "nghttp2_mem.h" + +int nghttp2_rcbuf_new(nghttp2_rcbuf **rcbuf_ptr, size_t size, + nghttp2_mem *mem) { + uint8_t *p; + + p = nghttp2_mem_malloc(mem, sizeof(nghttp2_rcbuf) + size); + if (p == NULL) { + return NGHTTP2_ERR_NOMEM; + } + + *rcbuf_ptr = (void *)p; + + (*rcbuf_ptr)->mem_user_data = mem->mem_user_data; + (*rcbuf_ptr)->free = mem->free; + (*rcbuf_ptr)->base = p + sizeof(nghttp2_rcbuf); + (*rcbuf_ptr)->len = size; + (*rcbuf_ptr)->ref = 1; + + return 0; +} + +int nghttp2_rcbuf_new2(nghttp2_rcbuf **rcbuf_ptr, const uint8_t *src, + size_t srclen, nghttp2_mem *mem) { + int rv; + + rv = nghttp2_rcbuf_new(rcbuf_ptr, srclen + 1, mem); + if (rv != 0) { + return rv; + } + + memcpy((*rcbuf_ptr)->base, src, srclen); + + (*rcbuf_ptr)->len = srclen; + (*rcbuf_ptr)->base[srclen] = '\0'; + + return 0; +} + +/* + * Frees |rcbuf| itself, regardless of its reference cout. + */ +void nghttp2_rcbuf_del(nghttp2_rcbuf *rcbuf) { + nghttp2_mem_free2(rcbuf->free, rcbuf, rcbuf->mem_user_data); +} + +void nghttp2_rcbuf_incref(nghttp2_rcbuf *rcbuf) { + if (rcbuf->ref == -1) { + return; + } + + ++rcbuf->ref; +} + +void nghttp2_rcbuf_decref(nghttp2_rcbuf *rcbuf) { + if (rcbuf == NULL || rcbuf->ref == -1) { + return; + } + + assert(rcbuf->ref > 0); + + if (--rcbuf->ref == 0) { + nghttp2_rcbuf_del(rcbuf); + } +} + +nghttp2_vec nghttp2_rcbuf_get_buf(nghttp2_rcbuf *rcbuf) { + nghttp2_vec res = {rcbuf->base, rcbuf->len}; + return res; +} diff --git a/lib/nghttp2_rcbuf.h b/lib/nghttp2_rcbuf.h new file mode 100644 index 00000000..29d1543e --- /dev/null +++ b/lib/nghttp2_rcbuf.h @@ -0,0 +1,80 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef NGHTTP2_RCBUF_H +#define NGHTTP2_RCBUF_H + +#ifdef HAVE_CONFIG_H +#include +#endif /* HAVE_CONFIG_H */ + +#include + +struct nghttp2_rcbuf { + /* custom memory allocator belongs to the mem parameter when + creating this object. */ + void *mem_user_data; + nghttp2_free free; + /* The pointer to the underlying buffer */ + uint8_t *base; + /* Size of buffer pointed by |base|. */ + size_t len; + /* Reference count */ + int32_t ref; +}; + +/* + * Allocates nghttp2_rcbuf object with |size| as initial buffer size. + * When the function succeeds, the reference count becomes 1. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * NGHTTP2_ERR_NOMEM: + * Out of memory. + */ +int nghttp2_rcbuf_new(nghttp2_rcbuf **rcbuf_ptr, size_t size, nghttp2_mem *mem); + +/* + * Like nghttp2_rcbuf_new(), but initializes the buffer with |src| of + * length |srclen|. This function allocates additional byte at the + * end and puts '\0' into it, so that the resulting buffer could be + * used as NULL-terminated string. Still (*rcbuf_ptr)->len equals to + * |srclen|. + * + * This function returns 0 if it succeeds, or one of the following + * negative error codes: + * + * NGHTTP2_ERR_NOMEM: + * Out of memory. + */ +int nghttp2_rcbuf_new2(nghttp2_rcbuf **rcbuf_ptr, const uint8_t *src, + size_t srclen, nghttp2_mem *mem); + +/* + * Frees |rcbuf| itself, regardless of its reference cout. + */ +void nghttp2_rcbuf_del(nghttp2_rcbuf *rcbuf); + +#endif /* NGHTTP2_RCBUF_H */ diff --git a/lib/nghttp2_session.c b/lib/nghttp2_session.c index 69f43d16..2c13b2b0 100644 --- a/lib/nghttp2_session.c +++ b/lib/nghttp2_session.c @@ -28,6 +28,7 @@ #include #include #include +#include #include "nghttp2_helper.h" #include "nghttp2_net.h" @@ -141,6 +142,62 @@ static int session_detect_idle_stream(nghttp2_session *session, return 0; } +static int session_call_error_callback(nghttp2_session *session, + const char *fmt, ...) { + size_t bufsize; + va_list ap; + char *buf; + int rv; + nghttp2_mem *mem; + + if (!session->callbacks.error_callback) { + return 0; + } + + mem = &session->mem; + + va_start(ap, fmt); + rv = vsnprintf(NULL, 0, fmt, ap); + va_end(ap); + + if (rv < 0) { + return NGHTTP2_ERR_NOMEM; + } + + bufsize = (size_t)(rv + 1); + + buf = nghttp2_mem_malloc(mem, bufsize); + if (buf == NULL) { + return NGHTTP2_ERR_NOMEM; + } + + va_start(ap, fmt); + rv = vsnprintf(buf, bufsize, fmt, ap); + va_end(ap); + + if (rv < 0) { + nghttp2_mem_free(mem, buf); + /* vsnprintf may return error because of various things we can + imagine, but typically we don't want to drop session just for + debug callback. */ + DEBUGF(fprintf(stderr, + "error_callback: vsnprintf failed. The template was %s\n", + fmt)); + return 0; + } + + rv = session->callbacks.error_callback(session, buf, (size_t)rv, + session->user_data); + + nghttp2_mem_free(mem, buf); + + if (rv != 0) { + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + + return 0; +} + static int session_terminate_session(nghttp2_session *session, int32_t last_stream_id, uint32_t error_code, const char *reason) { @@ -231,6 +288,8 @@ static void session_inbound_frame_reset(nghttp2_session *session) { nghttp2_session_new(), we rely on the fact that iframe->frame.hd.type is 0, so that no free is performed. */ switch (iframe->frame.hd.type) { + case NGHTTP2_DATA: + break; case NGHTTP2_HEADERS: nghttp2_frame_headers_free(&iframe->frame.headers, mem); break; @@ -255,6 +314,10 @@ static void session_inbound_frame_reset(nghttp2_session *session) { case NGHTTP2_WINDOW_UPDATE: nghttp2_frame_window_update_free(&iframe->frame.window_update); break; + default: + /* extension frame */ + nghttp2_frame_extension_free(&iframe->frame.ext); + break; } memset(&iframe->frame, 0, sizeof(nghttp2_frame)); @@ -405,6 +468,16 @@ static int session_new(nghttp2_session **session_ptr, (*session_ptr)->opt_flags |= NGHTTP2_OPTMASK_NO_HTTP_MESSAGING; } + + if (option->opt_set_mask & NGHTTP2_OPT_USER_RECV_EXT_TYPES) { + memcpy((*session_ptr)->user_recv_ext_types, option->user_recv_ext_types, + sizeof((*session_ptr)->user_recv_ext_types)); + } + + if ((option->opt_set_mask & NGHTTP2_OPT_NO_AUTO_PING_ACK) && + option->no_auto_ping_ack) { + (*session_ptr)->opt_flags |= NGHTTP2_OPTMASK_NO_AUTO_PING_ACK; + } } (*session_ptr)->callbacks = *callbacks; @@ -1216,11 +1289,12 @@ int nghttp2_session_adjust_idle_stream(nghttp2_session *session) { size_t max; int rv; - /* Make minimum number of idle streams 16, which is arbitrary chosen - number. */ - max = nghttp2_max(16, - nghttp2_min(session->local_settings.max_concurrent_streams, - session->pending_local_max_concurrent_stream)); + /* Make minimum number of idle streams 16, and maximum 100, which + are arbitrary chosen numbers. */ + max = nghttp2_min( + 100, nghttp2_max( + 16, nghttp2_min(session->local_settings.max_concurrent_streams, + session->pending_local_max_concurrent_stream))); DEBUGF(fprintf(stderr, "stream: adjusting kept idle streams " "num_idle_streams=%zu, max=%zu\n", @@ -1748,6 +1822,41 @@ static size_t session_estimate_headers_payload(nghttp2_session *session, additional; } +static int session_pack_extension(nghttp2_session *session, nghttp2_bufs *bufs, + nghttp2_frame *frame) { + ssize_t rv; + nghttp2_buf *buf; + size_t buflen; + size_t framelen; + + assert(session->callbacks.pack_extension_callback); + + buf = &bufs->head->buf; + buflen = nghttp2_min(nghttp2_buf_avail(buf), NGHTTP2_MAX_PAYLOADLEN); + + rv = session->callbacks.pack_extension_callback(session, buf->last, buflen, + frame, session->user_data); + if (rv == NGHTTP2_ERR_CANCEL) { + return (int)rv; + } + + if (rv < 0 || (size_t)rv > buflen) { + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + + framelen = (size_t)rv; + + frame->hd.length = framelen; + + assert(buf->pos == buf->last); + buf->last += framelen; + buf->pos -= NGHTTP2_FRAME_HDLEN; + + nghttp2_frame_pack_frame_hd(buf->pos, &frame->hd); + + return 0; +} + /* * This function serializes frame for transmission. * @@ -1988,8 +2097,22 @@ static int session_prep_frame(nghttp2_session *session, nghttp2_frame_pack_window_update(&session->aob.framebufs, &frame->window_update); break; + case NGHTTP2_CONTINUATION: + /* We never handle CONTINUATION here. */ + assert(0); + break; default: - return NGHTTP2_ERR_INVALID_ARGUMENT; + /* extension frame */ + if (session_is_closing(session)) { + return NGHTTP2_ERR_SESSION_CLOSING; + } + + rv = session_pack_extension(session, &session->aob.framebufs, frame); + if (rv != 0) { + return rv; + } + + break; } return 0; } else { @@ -3056,20 +3179,65 @@ static int session_call_on_begin_headers(nghttp2_session *session, static int session_call_on_header(nghttp2_session *session, const nghttp2_frame *frame, - const nghttp2_nv *nv) { - int rv; - if (session->callbacks.on_header_callback) { + const nghttp2_hd_nv *nv) { + int rv = 0; + if (session->callbacks.on_header_callback2) { + rv = session->callbacks.on_header_callback2( + session, frame, nv->name, nv->value, nv->flags, session->user_data); + } else if (session->callbacks.on_header_callback) { rv = session->callbacks.on_header_callback( - session, frame, nv->name, nv->namelen, nv->value, nv->valuelen, - nv->flags, session->user_data); - if (rv == NGHTTP2_ERR_PAUSE || - rv == NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE) { + session, frame, nv->name->base, nv->name->len, nv->value->base, + nv->value->len, nv->flags, session->user_data); + } + + if (rv == NGHTTP2_ERR_PAUSE || rv == NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE) { + return rv; + } + if (rv != 0) { + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + + return 0; +} + +static int +session_call_on_extension_chunk_recv_callback(nghttp2_session *session, + const uint8_t *data, size_t len) { + int rv; + nghttp2_inbound_frame *iframe = &session->iframe; + nghttp2_frame *frame = &iframe->frame; + + if (session->callbacks.on_extension_chunk_recv_callback) { + rv = session->callbacks.on_extension_chunk_recv_callback( + session, &frame->hd, data, len, session->user_data); + if (rv == NGHTTP2_ERR_CANCEL) { return rv; } if (rv != 0) { return NGHTTP2_ERR_CALLBACK_FAILURE; } } + + return 0; +} + +static int session_call_unpack_extension_callback(nghttp2_session *session) { + int rv; + nghttp2_inbound_frame *iframe = &session->iframe; + nghttp2_frame *frame = &iframe->frame; + void *payload = NULL; + + rv = session->callbacks.unpack_extension_callback( + session, &payload, &frame->hd, session->user_data); + if (rv == NGHTTP2_ERR_CANCEL) { + return rv; + } + if (rv != 0) { + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + + frame->ext.payload = payload; + return 0; } @@ -3210,11 +3378,10 @@ static int inflate_header_block(nghttp2_session *session, nghttp2_frame *frame, ssize_t proclen; int rv; int inflate_flags; - nghttp2_nv nv; + nghttp2_hd_nv nv; nghttp2_stream *stream; nghttp2_stream *subject_stream; int trailer = 0; - int token; *readlen_ptr = 0; stream = nghttp2_session_get_stream(session, frame->hd.stream_id); @@ -3231,7 +3398,7 @@ static int inflate_header_block(nghttp2_session *session, nghttp2_frame *frame, for (;;) { inflate_flags = 0; proclen = nghttp2_hd_inflate_hd2(&session->hd_inflater, &nv, &inflate_flags, - &token, in, inlen, final); + in, inlen, final); if (nghttp2_is_fatal((int)proclen)) { return (int)proclen; } @@ -3266,13 +3433,23 @@ static int inflate_header_block(nghttp2_session *session, nghttp2_frame *frame, if (call_header_cb && (inflate_flags & NGHTTP2_HD_INFLATE_EMIT)) { rv = 0; if (subject_stream && session_enforce_http_messaging(session)) { - rv = nghttp2_http_on_header(session, subject_stream, frame, &nv, token, + rv = nghttp2_http_on_header(session, subject_stream, frame, &nv, trailer); if (rv == NGHTTP2_ERR_HTTP_HEADER) { DEBUGF(fprintf( - stderr, "recv: HTTP error: type=%d, id=%d, header %.*s: %.*s\n", - frame->hd.type, subject_stream->stream_id, (int)nv.namelen, - nv.name, (int)nv.valuelen, nv.value)); + stderr, "recv: HTTP error: type=%u, id=%d, header %.*s: %.*s\n", + frame->hd.type, subject_stream->stream_id, (int)nv.name->len, + nv.name->base, (int)nv.value->len, nv.value->base)); + + rv = session_call_error_callback( + session, "Invalid HTTP header field was received: frame type: " + "%u, stream: %d, name: [%.*s], value: [%.*s]", + frame->hd.type, frame->hd.stream_id, (int)nv.name->len, + nv.name->base, (int)nv.value->len, nv.value->base); + + if (nghttp2_is_fatal(rv)) { + return rv; + } rv = session_handle_invalid_stream2(session, subject_stream->stream_id, @@ -3286,9 +3463,9 @@ static int inflate_header_block(nghttp2_session *session, nghttp2_frame *frame, if (rv == NGHTTP2_ERR_IGN_HTTP_HEADER) { /* header is ignored */ DEBUGF(fprintf( - stderr, "recv: HTTP ignored: type=%d, id=%d, header %.*s: %.*s\n", - frame->hd.type, subject_stream->stream_id, (int)nv.namelen, - nv.name, (int)nv.valuelen, nv.value)); + stderr, "recv: HTTP ignored: type=%u, id=%d, header %.*s: %.*s\n", + frame->hd.type, subject_stream->stream_id, (int)nv.name->len, + nv.name->base, (int)nv.value->len, nv.value->base)); } } if (rv == 0) { @@ -4262,7 +4439,8 @@ int nghttp2_session_on_ping_received(nghttp2_session *session, return session_handle_invalid_connection(session, frame, NGHTTP2_ERR_PROTO, "PING: stream_id != 0"); } - if ((frame->hd.flags & NGHTTP2_FLAG_ACK) == 0 && + if ((session->opt_flags & NGHTTP2_OPTMASK_NO_AUTO_PING_ACK) == 0 && + (frame->hd.flags & NGHTTP2_FLAG_ACK) == 0 && !session_is_closing(session)) { /* Peer sent ping, so ping it back */ rv = nghttp2_session_add_ping(session, NGHTTP2_FLAG_ACK, @@ -4411,6 +4589,24 @@ static int session_process_window_update_frame(nghttp2_session *session) { return nghttp2_session_on_window_update_received(session, frame); } +static int session_process_extension_frame(nghttp2_session *session) { + int rv; + nghttp2_inbound_frame *iframe = &session->iframe; + nghttp2_frame *frame = &iframe->frame; + + rv = session_call_unpack_extension_callback(session); + if (nghttp2_is_fatal(rv)) { + return rv; + } + + /* This handles the case where rv == NGHTTP2_ERR_CANCEL as well */ + if (rv != 0) { + return 0; + } + + return session_call_on_frame_received(session, frame); +} + int nghttp2_session_on_data_received(nghttp2_session *session, nghttp2_frame *frame) { int rv = 0; @@ -5229,11 +5425,21 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session, const uint8_t *in, default: DEBUGF(fprintf(stderr, "recv: unknown frame\n")); - /* Silently ignore unknown frame type. */ + if (!session->callbacks.unpack_extension_callback || + (session->user_recv_ext_types[iframe->frame.hd.type / 8] & + (1 << (iframe->frame.hd.type & 0x7))) == 0) { + /* Silently ignore unknown frame type. */ + + busy = 1; + + iframe->state = NGHTTP2_IB_IGN_PAYLOAD; + + break; + } busy = 1; - iframe->state = NGHTTP2_IB_IGN_PAYLOAD; + iframe->state = NGHTTP2_IB_READ_EXTENSION_PAYLOAD; break; } @@ -5693,7 +5899,7 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session, const uint8_t *in, if (cont_hd.type != NGHTTP2_CONTINUATION || cont_hd.stream_id != iframe->frame.hd.stream_id) { DEBUGF(fprintf(stderr, "recv: expected stream_id=%d, type=%d, but " - "got stream_id=%d, type=%d\n", + "got stream_id=%d, type=%u\n", iframe->frame.hd.stream_id, NGHTTP2_CONTINUATION, cont_hd.stream_id, cont_hd.type)); rv = nghttp2_session_terminate_session_with_reason( @@ -5933,6 +6139,44 @@ ssize_t nghttp2_session_mem_recv(nghttp2_session *session, const uint8_t *in, break; case NGHTTP2_IB_IGN_ALL: return (ssize_t)inlen; + case NGHTTP2_IB_READ_EXTENSION_PAYLOAD: + DEBUGF(fprintf(stderr, "recv: [IB_READ_EXTENSION_PAYLOAD]\n")); + + readlen = inbound_frame_payload_readlen(iframe, in, last); + iframe->payloadleft -= readlen; + in += readlen; + + DEBUGF(fprintf(stderr, "recv: readlen=%zu, payloadleft=%zu\n", readlen, + iframe->payloadleft)); + + if (readlen > 0) { + rv = session_call_on_extension_chunk_recv_callback( + session, in - readlen, readlen); + if (nghttp2_is_fatal(rv)) { + return rv; + } + + if (rv != 0) { + busy = 1; + + iframe->state = NGHTTP2_IB_IGN_PAYLOAD; + + break; + } + } + + if (iframe->payloadleft > 0) { + break; + } + + rv = session_process_extension_frame(session); + if (nghttp2_is_fatal(rv)) { + return rv; + } + + session_inbound_frame_reset(session); + + break; } if (!busy && in == last) { diff --git a/lib/nghttp2_session.h b/lib/nghttp2_session.h index 204aea91..3c3764de 100644 --- a/lib/nghttp2_session.h +++ b/lib/nghttp2_session.h @@ -50,7 +50,8 @@ extern int nghttp2_enable_strict_preface; typedef enum { NGHTTP2_OPTMASK_NO_AUTO_WINDOW_UPDATE = 1 << 0, NGHTTP2_OPTMASK_NO_RECV_CLIENT_MAGIC = 1 << 1, - NGHTTP2_OPTMASK_NO_HTTP_MESSAGING = 1 << 2 + NGHTTP2_OPTMASK_NO_HTTP_MESSAGING = 1 << 2, + NGHTTP2_OPTMASK_NO_AUTO_PING_ACK = 1 << 3 } nghttp2_optmask; typedef enum { @@ -105,7 +106,8 @@ typedef enum { NGHTTP2_IB_READ_PAD_DATA, NGHTTP2_IB_READ_DATA, NGHTTP2_IB_IGN_DATA, - NGHTTP2_IB_IGN_ALL + NGHTTP2_IB_IGN_ALL, + NGHTTP2_IB_READ_EXTENSION_PAYLOAD } nghttp2_inbound_state; #define NGHTTP2_INBOUND_NUM_IV 7 @@ -304,6 +306,13 @@ struct nghttp2_session { this session. The nonzero does not necessarily mean WINDOW_UPDATE is not queued. */ uint8_t window_update_queued; + /* Bitfield of extension frame types that application is willing to + receive. To designate the bit of given frame type i, use + user_recv_ext_types[i / 8] & (1 << (i & 0x7)). First 10 frame + types are standard frame types and not used in this bitfield. If + bit is set, it indicates that incoming frame with that type is + passed to user defined callbacks, otherwise they are ignored. */ + uint8_t user_recv_ext_types[32]; }; /* Struct used when updating initial window size of each active diff --git a/lib/nghttp2_stream.c b/lib/nghttp2_stream.c index 3e848d8e..3221fbf8 100644 --- a/lib/nghttp2_stream.c +++ b/lib/nghttp2_stream.c @@ -30,14 +30,32 @@ #include "nghttp2_session.h" #include "nghttp2_helper.h" +/* Maximum distance between any two stream's cycle in the same + prirority queue. Imagine stream A's cycle is A, and stream B's + cycle is B, and A < B. The cycle is unsigned 32 bit integer, it + may get overflow. Because of how we calculate the next cycle + value, if B - A is less than or equals to + NGHTTP2_MAX_CYCLE_DISTANCE, A and B are in the same scale, in other + words, B is really greater than or equal to A. Otherwise, A is a + result of overflow, and it is actually A > B if we consider that + fact. */ +#define NGHTTP2_MAX_CYCLE_DISTANCE (16384 * 256 + 255) + static int stream_less(const void *lhsx, const void *rhsx) { const nghttp2_stream *lhs, *rhs; lhs = nghttp2_struct_of(lhsx, nghttp2_stream, pq_entry); rhs = nghttp2_struct_of(rhsx, nghttp2_stream, pq_entry); - return lhs->cycle < rhs->cycle || - (lhs->cycle == rhs->cycle && lhs->seq < rhs->seq); + if (lhs->cycle == rhs->cycle) { + return lhs->seq < rhs->seq; + } + + if (lhs->cycle < rhs->cycle) { + return rhs->cycle - lhs->cycle <= NGHTTP2_MAX_CYCLE_DISTANCE; + } + + return lhs->cycle - rhs->cycle > NGHTTP2_MAX_CYCLE_DISTANCE; } void nghttp2_stream_init(nghttp2_stream *stream, int32_t stream_id, @@ -116,14 +134,14 @@ static int stream_subtree_active(nghttp2_stream *stream) { /* * Returns next cycle for |stream|. */ -static void stream_next_cycle(nghttp2_stream *stream, uint64_t last_cycle) { - size_t penalty; +static void stream_next_cycle(nghttp2_stream *stream, uint32_t last_cycle) { + uint32_t penalty; - penalty = - stream->last_writelen * NGHTTP2_MAX_WEIGHT + stream->pending_penalty; + penalty = (uint32_t)stream->last_writelen * NGHTTP2_MAX_WEIGHT + + stream->pending_penalty; stream->cycle = last_cycle + penalty / (uint32_t)stream->weight; - stream->pending_penalty = (uint32_t)(penalty % (uint32_t)stream->weight); + stream->pending_penalty = penalty % (uint32_t)stream->weight; } static int stream_obq_push(nghttp2_stream *dep_stream, nghttp2_stream *stream) { @@ -134,7 +152,7 @@ static int stream_obq_push(nghttp2_stream *dep_stream, nghttp2_stream *stream) { stream_next_cycle(stream, dep_stream->descendant_last_cycle); stream->seq = dep_stream->descendant_next_seq++; - DEBUGF(fprintf(stderr, "stream: stream=%d obq push cycle=%ld\n", + DEBUGF(fprintf(stderr, "stream: stream=%d obq push cycle=%d\n", stream->stream_id, stream->cycle)); DEBUGF(fprintf(stderr, "stream: push stream %d to stream %d\n", @@ -220,7 +238,7 @@ void nghttp2_stream_reschedule(nghttp2_stream *stream) { nghttp2_pq_push(&dep_stream->obq, &stream->pq_entry); - DEBUGF(fprintf(stderr, "stream: stream=%d obq resched cycle=%ld\n", + DEBUGF(fprintf(stderr, "stream: stream=%d obq resched cycle=%d\n", stream->stream_id, stream->cycle)); dep_stream->last_writelen = stream->last_writelen; @@ -229,9 +247,9 @@ void nghttp2_stream_reschedule(nghttp2_stream *stream) { void nghttp2_stream_change_weight(nghttp2_stream *stream, int32_t weight) { nghttp2_stream *dep_stream; - uint64_t last_cycle; + uint32_t last_cycle; int32_t old_weight; - size_t wlen_penalty; + uint32_t wlen_penalty; if (stream->weight == weight) { return; @@ -254,7 +272,7 @@ void nghttp2_stream_change_weight(nghttp2_stream *stream, int32_t weight) { nghttp2_pq_remove(&dep_stream->obq, &stream->pq_entry); - wlen_penalty = stream->last_writelen * NGHTTP2_MAX_WEIGHT; + wlen_penalty = (uint32_t)stream->last_writelen * NGHTTP2_MAX_WEIGHT; /* Compute old stream->pending_penalty we used to calculate stream->cycle */ @@ -270,7 +288,9 @@ void nghttp2_stream_change_weight(nghttp2_stream *stream, int32_t weight) { place */ stream_next_cycle(stream, last_cycle); - if (stream->cycle < dep_stream->descendant_last_cycle) { + if (stream->cycle < dep_stream->descendant_last_cycle && + (dep_stream->descendant_last_cycle - stream->cycle) <= + NGHTTP2_MAX_CYCLE_DISTANCE) { stream->cycle = dep_stream->descendant_last_cycle; } @@ -278,7 +298,7 @@ void nghttp2_stream_change_weight(nghttp2_stream *stream, int32_t weight) { nghttp2_pq_push(&dep_stream->obq, &stream->pq_entry); - DEBUGF(fprintf(stderr, "stream: stream=%d obq resched cycle=%ld\n", + DEBUGF(fprintf(stderr, "stream: stream=%d obq resched cycle=%d\n", stream->stream_id, stream->cycle)); } diff --git a/lib/nghttp2_stream.h b/lib/nghttp2_stream.h index b5d07592..da0e5d53 100644 --- a/lib/nghttp2_stream.h +++ b/lib/nghttp2_stream.h @@ -147,9 +147,9 @@ struct nghttp2_stream { /* Received body so far */ int64_t recv_content_length; /* Base last_cycle for direct descendent streams. */ - uint64_t descendant_last_cycle; + uint32_t descendant_last_cycle; /* Next scheduled time to sent item */ - uint64_t cycle; + uint32_t cycle; /* Next seq used for direct descendant streams */ uint64_t descendant_next_seq; /* Secondary key for prioritization to break a tie for cycle. This diff --git a/lib/nghttp2_submit.c b/lib/nghttp2_submit.c index 70271b7b..9c11e210 100644 --- a/lib/nghttp2_submit.c +++ b/lib/nghttp2_submit.c @@ -211,8 +211,9 @@ int32_t nghttp2_submit_headers(nghttp2_session *session, uint8_t flags, nvlen, NULL, stream_user_data); } -int nghttp2_submit_ping(nghttp2_session *session, uint8_t flags _U_, +int nghttp2_submit_ping(nghttp2_session *session, uint8_t flags, const uint8_t *opaque_data) { + flags &= NGHTTP2_FLAG_ACK; return nghttp2_session_add_ping(session, NGHTTP2_FLAG_NONE, opaque_data); } @@ -530,3 +531,40 @@ ssize_t nghttp2_pack_settings_payload(uint8_t *buf, size_t buflen, return (ssize_t)nghttp2_frame_pack_settings_payload(buf, iv, niv); } + +int nghttp2_submit_extension(nghttp2_session *session, uint8_t type, + uint8_t flags, int32_t stream_id, void *payload) { + int rv; + nghttp2_outbound_item *item; + nghttp2_frame *frame; + nghttp2_mem *mem; + + mem = &session->mem; + + if (type <= NGHTTP2_CONTINUATION) { + return NGHTTP2_ERR_INVALID_ARGUMENT; + } + + if (!session->callbacks.pack_extension_callback) { + return NGHTTP2_ERR_INVALID_STATE; + } + + item = nghttp2_mem_malloc(mem, sizeof(nghttp2_outbound_item)); + if (item == NULL) { + return NGHTTP2_ERR_NOMEM; + } + + nghttp2_outbound_item_init(item); + + frame = &item->frame; + nghttp2_frame_extension_init(&frame->ext, type, flags, stream_id, payload); + + rv = nghttp2_session_add_item(session, item); + if (rv != 0) { + nghttp2_frame_extension_free(&frame->ext); + nghttp2_mem_free(mem, item); + return rv; + } + + return 0; +} diff --git a/makerelease.sh b/makerelease.sh index dd22feb3..3356d0b0 100755 --- a/makerelease.sh +++ b/makerelease.sh @@ -3,12 +3,12 @@ TAG=$1 PREV_TAG=$2 -git submodule update --init - git checkout refs/tags/$TAG git log --pretty=fuller --date=short refs/tags/$PREV_TAG..HEAD > ChangeLog git submodule update --init + +autoreconf -i ./configure --with-mruby && \ make dist-bzip2 && make dist-gzip && make dist-xz || echo "error" make distclean diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5bf8f972..3caf0edc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -145,6 +145,7 @@ if(ENABLE_APP) shrpx_ssl_test.cc shrpx_downstream_test.cc shrpx_config_test.cc + shrpx_worker_test.cc shrpx_http_test.cc http2_test.cc util_test.cc diff --git a/src/HttpServer.cc b/src/HttpServer.cc index e5406cdc..74e9a9b6 100644 --- a/src/HttpServer.cc +++ b/src/HttpServer.cc @@ -308,7 +308,6 @@ public: } auto handler = make_unique(this, fd, ssl, get_next_session_id()); - handler->setup_bev(); if (!ssl) { if (handler->connection_made() != 0) { return; @@ -575,7 +574,9 @@ struct ev_loop *Http2Handler::get_loop() const { Http2Handler::WriteBuf *Http2Handler::get_wb() { return &wb_; } -int Http2Handler::setup_bev() { return 0; } +void Http2Handler::start_settings_timer() { + ev_timer_start(sessions_->get_loop(), &settings_timerev_); +} int Http2Handler::fill_wb() { if (data_pending_) { @@ -870,8 +871,6 @@ int Http2Handler::connection_made() { } } - ev_timer_start(sessions_->get_loop(), &settings_timerev_); - if (ssl_ && !nghttp2::ssl::check_http2_requirement(ssl_)) { terminate_session(NGHTTP2_INADEQUATE_SECURITY); } @@ -1236,7 +1235,7 @@ void prepare_response(Stream *stream, Http2Handler *hd, bool last_mod_found = false; if (ims) { last_mod_found = true; - last_mod = util::parse_http_date(ims->value); + last_mod = util::parse_http_date(StringRef{ims->value}); } auto query_pos = reqpath.find("?"); std::string url; @@ -1538,6 +1537,15 @@ int hd_on_frame_send_callback(nghttp2_session *session, break; } + case NGHTTP2_SETTINGS: { + if (frame->hd.flags & NGHTTP2_FLAG_ACK) { + return 0; + } + + hd->start_settings_timer(); + + break; + } case NGHTTP2_PUSH_PROMISE: { auto promised_stream_id = frame->push_promise.promised_stream_id; auto promised_stream = hd->get_stream(promised_stream_id); @@ -1681,6 +1689,9 @@ void fill_callback(nghttp2_session_callbacks *callbacks, const Config *config) { if (config->verbose) { nghttp2_session_callbacks_set_on_invalid_frame_recv_callback( callbacks, verbose_on_invalid_frame_recv_callback); + + nghttp2_session_callbacks_set_error_callback(callbacks, + verbose_error_callback); } nghttp2_session_callbacks_set_on_data_chunk_recv_callback( diff --git a/src/HttpServer.h b/src/HttpServer.h index 99623397..0690307e 100644 --- a/src/HttpServer.h +++ b/src/HttpServer.h @@ -137,7 +137,7 @@ public: ~Http2Handler(); void remove_self(); - int setup_bev(); + void start_settings_timer(); int on_read(); int on_write(); int connection_made(); diff --git a/src/Makefile.am b/src/Makefile.am index 5b21aee7..0f4656d8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -64,7 +64,7 @@ HELPER_OBJECTS = util.cc \ http2.cc timegm.c app_helper.cc nghttp2_gzip.c HELPER_HFILES = util.h \ http2.h timegm.h app_helper.h nghttp2_config.h \ - nghttp2_gzip.h + nghttp2_gzip.h network.h HTML_PARSER_OBJECTS = HTML_PARSER_HFILES = HtmlParser.h @@ -171,6 +171,7 @@ nghttpx_unittest_SOURCES = shrpx-unittest.cc \ shrpx_ssl_test.cc shrpx_ssl_test.h \ shrpx_downstream_test.cc shrpx_downstream_test.h \ shrpx_config_test.cc shrpx_config_test.h \ + shrpx_worker_test.cc shrpx_worker_test.h \ shrpx_http_test.cc shrpx_http_test.h \ http2_test.cc http2_test.h \ util_test.cc util_test.h \ diff --git a/src/allocator.h b/src/allocator.h new file mode 100644 index 00000000..64bdff49 --- /dev/null +++ b/src/allocator.h @@ -0,0 +1,169 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef ALLOCATOR_H +#define ALLOCATOR_H + +#include "nghttp2_config.h" + +#include + +#include "template.h" + +namespace nghttp2 { + +struct MemBlock { + // The next MemBlock to chain them. This is for book keeping + // purpose to free them later. + MemBlock *next; + // begin is the pointer to the beginning of buffer. last is the + // location of next write. end is the one beyond of the end of the + // buffer. + uint8_t *begin, *last, *end; +}; + +// BlockAllocator allocates memory block with given size at once, and +// cuts the region from it when allocation is requested. If the +// requested size is larger than given threshold, it will be allocated +// in a distinct buffer on demand. +struct BlockAllocator { + BlockAllocator(size_t block_size, size_t isolation_threshold) + : retain(nullptr), + head(nullptr), + block_size(block_size), + isolation_threshold(std::min(block_size, isolation_threshold)) {} + + ~BlockAllocator() { + for (auto mb = retain; mb;) { + auto next = mb->next; + delete[] reinterpret_cast(mb); + mb = next; + } + } + + BlockAllocator(BlockAllocator &&) = default; + BlockAllocator &operator=(BlockAllocator &&) = default; + + BlockAllocator(const BlockAllocator &) = delete; + BlockAllocator &operator=(const BlockAllocator &) = delete; + + MemBlock *alloc_mem_block(size_t size) { + auto block = new uint8_t[sizeof(MemBlock) + size]; + auto mb = reinterpret_cast(block); + + mb->next = retain; + mb->begin = mb->last = block + sizeof(MemBlock); + mb->end = mb->begin + size; + retain = mb; + return mb; + } + + void *alloc(size_t size) { + if (size >= isolation_threshold) { + auto mb = alloc_mem_block(size); + mb->last = mb->end; + return mb->begin; + } + + if (!head || head->end - head->last < static_cast(size)) { + head = alloc_mem_block(block_size); + } + + auto res = head->last; + + head->last = reinterpret_cast( + (reinterpret_cast(head->last + size) + 0xf) & ~0xf); + + return res; + } + + // This holds live memory block to free them in dtor. + MemBlock *retain; + // Current memory block to use. + MemBlock *head; + // size of single memory block + size_t block_size; + // if allocation greater or equal to isolation_threshold bytes is + // requested, allocate dedicated block. + size_t isolation_threshold; +}; + +// Makes a copy of |src|. The resulting string will be +// NULL-terminated. +template +StringRef make_string_ref(BlockAllocator &alloc, const StringRef &src) { + auto dst = static_cast(alloc.alloc(src.size() + 1)); + auto p = dst; + p = std::copy(std::begin(src), std::end(src), p); + *p = '\0'; + return StringRef{dst, src.size()}; +} + +// Returns the string which is the concatenation of |a| and |b| in +// this order. The resulting string will be NULL-terminated. +template +StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, + const StringRef &b) { + auto len = a.size() + b.size(); + auto dst = static_cast(alloc.alloc(len + 1)); + auto p = dst; + p = std::copy(std::begin(a), std::end(a), p); + p = std::copy(std::begin(b), std::end(b), p); + *p = '\0'; + return StringRef{dst, len}; +} + +// Returns the string which is the concatenation of |a|, |b| and |c| +// in this order. The resulting string will be NULL-terminated. +template +StringRef concat_string_ref(BlockAllocator &alloc, const StringRef &a, + const StringRef &b, const StringRef &c) { + auto len = a.size() + b.size() + c.size(); + auto dst = static_cast(alloc.alloc(len + 1)); + auto p = dst; + p = std::copy(std::begin(a), std::end(a), p); + p = std::copy(std::begin(b), std::end(b), p); + p = std::copy(std::begin(c), std::end(c), p); + *p = '\0'; + return StringRef{dst, len}; +} + +struct ByteRef { + // The pointer to the beginning of the buffer. + uint8_t *base; + // The length of the buffer. + size_t len; +}; + +// Makes a buffer with given size. The resulting byte string might +// not be NULL-terminated. +template +ByteRef make_byte_ref(BlockAllocator &alloc, size_t size) { + auto dst = static_cast(alloc.alloc(size)); + return {dst, size}; +} + +} // namespace aria2 + +#endif // ALLOCATOR_H diff --git a/src/app_helper.cc b/src/app_helper.cc index 2ce51af7..603151cf 100644 --- a/src/app_helper.cc +++ b/src/app_helper.cc @@ -62,43 +62,6 @@ namespace nghttp2 { -namespace { -const char *strstatus(uint32_t error_code) { - switch (error_code) { - case NGHTTP2_NO_ERROR: - return "NO_ERROR"; - case NGHTTP2_PROTOCOL_ERROR: - return "PROTOCOL_ERROR"; - case NGHTTP2_INTERNAL_ERROR: - return "INTERNAL_ERROR"; - case NGHTTP2_FLOW_CONTROL_ERROR: - return "FLOW_CONTROL_ERROR"; - case NGHTTP2_SETTINGS_TIMEOUT: - return "SETTINGS_TIMEOUT"; - case NGHTTP2_STREAM_CLOSED: - return "STREAM_CLOSED"; - case NGHTTP2_FRAME_SIZE_ERROR: - return "FRAME_SIZE_ERROR"; - case NGHTTP2_REFUSED_STREAM: - return "REFUSED_STREAM"; - case NGHTTP2_CANCEL: - return "CANCEL"; - case NGHTTP2_COMPRESSION_ERROR: - return "COMPRESSION_ERROR"; - case NGHTTP2_CONNECT_ERROR: - return "CONNECT_ERROR"; - case NGHTTP2_ENHANCE_YOUR_CALM: - return "ENHANCE_YOUR_CALM"; - case NGHTTP2_INADEQUATE_SECURITY: - return "INADEQUATE_SECURITY"; - case NGHTTP2_HTTP_1_1_REQUIRED: - return "HTTP_1_1_REQUIRED"; - default: - return "UNKNOWN"; - } -} -} // namespace - namespace { const char *strsettingsid(int32_t id) { switch (id) { @@ -121,7 +84,7 @@ const char *strsettingsid(int32_t id) { } // namespace namespace { -const char *strframetype(uint8_t type) { +std::string strframetype(uint8_t type) { switch (type) { case NGHTTP2_DATA: return "DATA"; @@ -141,9 +104,13 @@ const char *strframetype(uint8_t type) { return "GOAWAY"; case NGHTTP2_WINDOW_UPDATE: return "WINDOW_UPDATE"; - default: - return "UNKNOWN"; } + + std::string s = "extension(0x"; + s += util::format_hex(&type, 1); + s += ')'; + + return s; }; } // namespace @@ -280,7 +247,7 @@ const char *frame_name_ansi_esc(print_type ptype) { namespace { void print_frame(print_type ptype, const nghttp2_frame *frame) { fprintf(outfile, "%s%s%s frame ", frame_name_ansi_esc(ptype), - strframetype(frame->hd.type), ansi_escend()); + strframetype(frame->hd.type).c_str(), ansi_escend()); print_frame_hd(frame->hd); if (frame->hd.flags) { print_frame_attr_indent(); @@ -331,7 +298,7 @@ void print_frame(print_type ptype, const nghttp2_frame *frame) { case NGHTTP2_RST_STREAM: print_frame_attr_indent(); fprintf(outfile, "(error_code=%s(0x%02x))\n", - strstatus(frame->rst_stream.error_code), + nghttp2_http2_strerror(frame->rst_stream.error_code), frame->rst_stream.error_code); break; case NGHTTP2_SETTINGS: @@ -360,7 +327,8 @@ void print_frame(print_type ptype, const nghttp2_frame *frame) { print_frame_attr_indent(); fprintf(outfile, "(last_stream_id=%d, error_code=%s(0x%02x), " "opaque_data(%u)=[%s])\n", - frame->goaway.last_stream_id, strstatus(frame->goaway.error_code), + frame->goaway.last_stream_id, + nghttp2_http2_strerror(frame->goaway.error_code), frame->goaway.error_code, static_cast(frame->goaway.opaque_data_len), util::ascii_dump(frame->goaway.opaque_data, @@ -446,6 +414,15 @@ int verbose_on_data_chunk_recv_callback(nghttp2_session *session, uint8_t flags, return 0; } +int verbose_error_callback(nghttp2_session *session, const char *msg, + size_t len, void *user_data) { + print_timer(); + fprintf(outfile, " [ERROR] %.*s\n", (int)len, msg); + fflush(outfile); + + return 0; +} + namespace { std::chrono::steady_clock::time_point base_tv; } // namespace diff --git a/src/app_helper.h b/src/app_helper.h index 408adac2..263bb99c 100644 --- a/src/app_helper.h +++ b/src/app_helper.h @@ -60,6 +60,9 @@ int verbose_on_data_chunk_recv_callback(nghttp2_session *session, uint8_t flags, int32_t stream_id, const uint8_t *data, size_t len, void *user_data); +int verbose_error_callback(nghttp2_session *session, const char *msg, + size_t len, void *user_data); + // Returns difference between |a| and |b| in milliseconds, assuming // |a| is more recent than |b|. template diff --git a/src/asio_server_connection.h b/src/asio_server_connection.h index 4498497d..6be79406 100644 --- a/src/asio_server_connection.h +++ b/src/asio_server_connection.h @@ -79,8 +79,10 @@ public: /// Start the first asynchronous operation for the connection. void start() { + boost::system::error_code ec; + handler_ = std::make_shared( - socket_.get_io_service(), socket_.lowest_layer().remote_endpoint(), + socket_.get_io_service(), socket_.lowest_layer().remote_endpoint(ec), [this]() { do_write(); }, mux_); if (handler_->start() != 0) { stop(); diff --git a/src/asio_server_serve_mux.cc b/src/asio_server_serve_mux.cc index b635ee4d..7765adf0 100644 --- a/src/asio_server_serve_mux.cc +++ b/src/asio_server_serve_mux.cc @@ -77,8 +77,8 @@ bool serve_mux::handle(std::string pattern, request_cb cb) { request_cb serve_mux::handler(request_impl &req) const { auto &path = req.uri().path; if (req.method() != "CONNECT") { - auto clean_path = ::nghttp2::http2::path_join( - nullptr, 0, nullptr, 0, path.c_str(), path.size(), nullptr, 0); + auto clean_path = ::nghttp2::http2::path_join(StringRef{}, StringRef{}, + StringRef{path}, StringRef{}); if (clean_path != path) { auto new_uri = util::percent_encode_path(clean_path); auto &uref = req.uri(); diff --git a/src/h2load.cc b/src/h2load.cc index aeedbe7f..14b1fca4 100644 --- a/src/h2load.cc +++ b/src/h2load.cc @@ -84,7 +84,7 @@ Config::Config() nreqs(1), nclients(1), nthreads(1), - max_concurrent_streams(-1), + max_concurrent_streams(1), window_bits(30), connection_window_bits(30), rate(0), @@ -96,7 +96,9 @@ Config::Config() port(0), default_port(0), verbose(false), - timing_script(false) {} + timing_script(false), + base_uri_unix(false), + unix_addr{} {} Config::~Config() { if (base_uri_unix) { @@ -200,7 +202,9 @@ void readcb(struct ev_loop *loop, ev_io *w, int revents) { auto client = static_cast(w->data); client->restart_timeout(); if (client->do_read() != 0) { - client->fail(); + if (client->try_again_or_fail() == 0) { + return; + } delete client; return; } @@ -451,7 +455,7 @@ void Client::restart_timeout() { } } -void Client::fail() { +int Client::try_again_or_fail() { disconnect(); if (new_connection_requested) { @@ -469,13 +473,21 @@ void Client::fail() { // Keep using current address if (connect() == 0) { - return; + return 0; } std::cerr << "client could not connect to host" << std::endl; } } process_abandoned_streams(); + + return -1; +} + +void Client::fail() { + disconnect(); + + process_abandoned_streams(); } void Client::disconnect() { @@ -1384,7 +1396,7 @@ std::string get_reqline(const char *uri, const http_parser_url &u) { std::string reqline; if (util::has_uri_field(u, UF_PATH)) { - reqline = util::get_uri_field(uri, u, UF_PATH); + reqline = util::get_uri_field(uri, u, UF_PATH).str(); } else { reqline = "/"; } @@ -1425,8 +1437,8 @@ bool parse_base_uri(std::string base_uri) { return false; } - config.scheme = util::get_uri_field(base_uri.c_str(), u, UF_SCHEMA); - config.host = util::get_uri_field(base_uri.c_str(), u, UF_HOST); + config.scheme = util::get_uri_field(base_uri.c_str(), u, UF_SCHEMA).str(); + config.host = util::get_uri_field(base_uri.c_str(), u, UF_HOST).str(); config.default_port = util::get_default_port(base_uri.c_str(), u); if (util::has_uri_field(u, UF_PORT)) { config.port = u.port; diff --git a/src/h2load.h b/src/h2load.h index c053b9c8..de11b4b4 100644 --- a/src/h2load.h +++ b/src/h2load.h @@ -307,6 +307,12 @@ struct Client { int connect(); void disconnect(); void fail(); + // Call this function when do_read() returns -1. This function + // tries to connect to the remote host again if it is requested. If + // so, this function returns 0, and this object should be retained. + // Otherwise, this function returns -1, and this object should be + // deleted. + int try_again_or_fail(); void timeout(); void restart_timeout(); int submit_request(); diff --git a/src/http2.cc b/src/http2.cc index eba7aa54..c808b5b4 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -134,111 +134,111 @@ std::string get_status_string(unsigned int status_code) { } } -const char *stringify_status(unsigned int status_code) { +StringRef stringify_status(unsigned int status_code) { switch (status_code) { case 100: - return "100"; + return StringRef::from_lit("100"); case 101: - return "101"; + return StringRef::from_lit("101"); case 200: - return "200"; + return StringRef::from_lit("200"); case 201: - return "201"; + return StringRef::from_lit("201"); case 202: - return "202"; + return StringRef::from_lit("202"); case 203: - return "203"; + return StringRef::from_lit("203"); case 204: - return "204"; + return StringRef::from_lit("204"); case 205: - return "205"; + return StringRef::from_lit("205"); case 206: - return "206"; + return StringRef::from_lit("206"); case 300: - return "300"; + return StringRef::from_lit("300"); case 301: - return "301"; + return StringRef::from_lit("301"); case 302: - return "302"; + return StringRef::from_lit("302"); case 303: - return "303"; + return StringRef::from_lit("303"); case 304: - return "304"; + return StringRef::from_lit("304"); case 305: - return "305"; - // case 306: return "306"; + return StringRef::from_lit("305"); + // case 306: return StringRef::from_lit("306"); case 307: - return "307"; + return StringRef::from_lit("307"); case 308: - return "308"; + return StringRef::from_lit("308"); case 400: - return "400"; + return StringRef::from_lit("400"); case 401: - return "401"; + return StringRef::from_lit("401"); case 402: - return "402"; + return StringRef::from_lit("402"); case 403: - return "403"; + return StringRef::from_lit("403"); case 404: - return "404"; + return StringRef::from_lit("404"); case 405: - return "405"; + return StringRef::from_lit("405"); case 406: - return "406"; + return StringRef::from_lit("406"); case 407: - return "407"; + return StringRef::from_lit("407"); case 408: - return "408"; + return StringRef::from_lit("408"); case 409: - return "409"; + return StringRef::from_lit("409"); case 410: - return "410"; + return StringRef::from_lit("410"); case 411: - return "411"; + return StringRef::from_lit("411"); case 412: - return "412"; + return StringRef::from_lit("412"); case 413: - return "413"; + return StringRef::from_lit("413"); case 414: - return "414"; + return StringRef::from_lit("414"); case 415: - return "415"; + return StringRef::from_lit("415"); case 416: - return "416"; + return StringRef::from_lit("416"); case 417: - return "417"; + return StringRef::from_lit("417"); case 421: - return "421"; + return StringRef::from_lit("421"); case 426: - return "426"; + return StringRef::from_lit("426"); case 428: - return "428"; + return StringRef::from_lit("428"); case 429: - return "429"; + return StringRef::from_lit("429"); case 431: - return "431"; + return StringRef::from_lit("431"); case 451: - return "451"; + return StringRef::from_lit("451"); case 500: - return "500"; + return StringRef::from_lit("500"); case 501: - return "501"; + return StringRef::from_lit("501"); case 502: - return "502"; + return StringRef::from_lit("502"); case 503: - return "503"; + return StringRef::from_lit("503"); case 504: - return "504"; + return StringRef::from_lit("504"); case 505: - return "505"; + return StringRef::from_lit("505"); case 511: - return "511"; + return StringRef::from_lit("511"); default: - return nullptr; + return StringRef{}; } } -void capitalize(DefaultMemchunks *buf, const std::string &s) { +void capitalize(DefaultMemchunks *buf, const StringRef &s) { buf->append(util::upcase(s[0])); for (size_t i = 1; i < s.size(); ++i) { if (s[i - 1] == '-') { @@ -271,7 +271,7 @@ void copy_url_component(std::string &dest, const http_parser_url *u, int field, Headers::value_type to_header(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, - bool no_index, int16_t token) { + bool no_index, int32_t token) { return Header(std::string(reinterpret_cast(name), namelen), std::string(reinterpret_cast(value), valuelen), no_index, token); @@ -279,7 +279,7 @@ Headers::value_type to_header(const uint8_t *name, size_t namelen, void add_header(Headers &nva, const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, - int16_t token) { + int32_t token) { if (valuelen > 0) { size_t i, j; for (i = 0; i < valuelen && (value[i] == ' ' || value[i] == '\t'); ++i) @@ -302,14 +302,7 @@ const Headers::value_type *get_header(const Headers &nva, const char *name) { return res; } -std::string value_to_str(const Headers::value_type *nv) { - if (nv) { - return nv->value; - } - return ""; -} - -bool non_empty_value(const Headers::value_type *nv) { +bool non_empty_value(const HeaderRefs::value_type *nv) { return nv && !nv->value.empty(); } @@ -326,11 +319,29 @@ nghttp2_nv make_nv_internal(const std::string &name, const std::string &value, } } // namespace +namespace { +nghttp2_nv make_nv_internal(const StringRef &name, const StringRef &value, + bool no_index, uint8_t nv_flags) { + uint8_t flags; + + flags = + nv_flags | (no_index ? NGHTTP2_NV_FLAG_NO_INDEX : NGHTTP2_NV_FLAG_NONE); + + return {(uint8_t *)name.c_str(), (uint8_t *)value.c_str(), name.size(), + value.size(), flags}; +} +} // namespace + nghttp2_nv make_nv(const std::string &name, const std::string &value, bool no_index) { return make_nv_internal(name, value, no_index, NGHTTP2_NV_FLAG_NONE); } +nghttp2_nv make_nv(const StringRef &name, const StringRef &value, + bool no_index) { + return make_nv_internal(name, value, no_index, NGHTTP2_NV_FLAG_NONE); +} + nghttp2_nv make_nv_nocopy(const std::string &name, const std::string &value, bool no_index) { return make_nv_internal(name, value, no_index, @@ -338,9 +349,16 @@ nghttp2_nv make_nv_nocopy(const std::string &name, const std::string &value, NGHTTP2_NV_FLAG_NO_COPY_VALUE); } +nghttp2_nv make_nv_nocopy(const StringRef &name, const StringRef &value, + bool no_index) { + return make_nv_internal(name, value, no_index, + NGHTTP2_NV_FLAG_NO_COPY_NAME | + NGHTTP2_NV_FLAG_NO_COPY_VALUE); +} + namespace { void copy_headers_to_nva_internal(std::vector &nva, - const Headers &headers, uint8_t nv_flags) { + const HeaderRefs &headers, uint8_t nv_flags) { for (auto &kv : headers) { if (kv.name.empty() || kv.name[0] == ':') { continue; @@ -367,18 +385,19 @@ void copy_headers_to_nva_internal(std::vector &nva, } } // namespace -void copy_headers_to_nva(std::vector &nva, const Headers &headers) { +void copy_headers_to_nva(std::vector &nva, + const HeaderRefs &headers) { copy_headers_to_nva_internal(nva, headers, NGHTTP2_NV_FLAG_NONE); } void copy_headers_to_nva_nocopy(std::vector &nva, - const Headers &headers) { + const HeaderRefs &headers) { copy_headers_to_nva_internal(nva, headers, NGHTTP2_NV_FLAG_NO_COPY_NAME | NGHTTP2_NV_FLAG_NO_COPY_VALUE); } void build_http1_headers_from_headers(DefaultMemchunks *buf, - const Headers &headers) { + const HeaderRefs &headers) { for (auto &kv : headers) { if (kv.name.empty() || kv.name[0] == ':') { continue; @@ -450,42 +469,77 @@ void dump_nv(FILE *out, const Headers &nva) { fflush(out); } -std::string rewrite_location_uri(const std::string &uri, - const http_parser_url &u, - const std::string &match_host, - const std::string &request_authority, - const std::string &upstream_scheme) { +void dump_nv(FILE *out, const HeaderRefs &nva) { + for (auto &nv : nva) { + fprintf(out, "%s: %s\n", nv.name.c_str(), nv.value.c_str()); + } + fputc('\n', out); + fflush(out); +} + +StringRef rewrite_location_uri(BlockAllocator &balloc, const StringRef &uri, + const http_parser_url &u, + const StringRef &match_host, + const StringRef &request_authority, + const StringRef &upstream_scheme) { // We just rewrite scheme and authority. if ((u.field_set & (1 << UF_HOST)) == 0) { - return ""; + return StringRef{}; } auto field = &u.field_data[UF_HOST]; if (!util::starts_with(std::begin(match_host), std::end(match_host), &uri[field->off], &uri[field->off] + field->len) || (match_host.size() != field->len && match_host[field->len] != ':')) { - return ""; + return StringRef{}; } - std::string res; + + auto len = 0; if (!request_authority.empty()) { - res += upstream_scheme; - res += "://"; - res += request_authority; + len += upstream_scheme.size() + str_size("://") + request_authority.size(); + } + + if (u.field_set & (1 << UF_PATH)) { + field = &u.field_data[UF_PATH]; + len += field->len; + } + + if (u.field_set & (1 << UF_QUERY)) { + field = &u.field_data[UF_QUERY]; + len += 1 + field->len; + } + + if (u.field_set & (1 << UF_FRAGMENT)) { + field = &u.field_data[UF_FRAGMENT]; + len += 1 + field->len; + } + + auto iov = make_byte_ref(balloc, len + 1); + auto p = iov.base; + + if (!request_authority.empty()) { + p = std::copy(std::begin(upstream_scheme), std::end(upstream_scheme), p); + p = util::copy_lit(p, "://"); + p = std::copy(std::begin(request_authority), std::end(request_authority), + p); } if (u.field_set & (1 << UF_PATH)) { field = &u.field_data[UF_PATH]; - res.append(&uri[field->off], field->len); + p = std::copy_n(&uri[field->off], field->len, p); } if (u.field_set & (1 << UF_QUERY)) { field = &u.field_data[UF_QUERY]; - res += '?'; - res.append(&uri[field->off], field->len); + *p++ = '?'; + p = std::copy_n(&uri[field->off], field->len, p); } if (u.field_set & (1 << UF_FRAGMENT)) { field = &u.field_data[UF_FRAGMENT]; - res += '#'; - res.append(&uri[field->off], field->len); + *p++ = '#'; + p = std::copy_n(&uri[field->off], field->len, p); } - return res; + + *p = '\0'; + + return StringRef{iov.base, p}; } int check_nv(const uint8_t *name, size_t namelen, const uint8_t *value, @@ -499,7 +553,7 @@ int check_nv(const uint8_t *name, size_t namelen, const uint8_t *value, return 1; } -int parse_http_status_code(const std::string &src) { +int parse_http_status_code(const StringRef &src) { if (src.size() != 3) { return -1; } @@ -525,6 +579,10 @@ int lookup_token(const std::string &name) { name.size()); } +int lookup_token(const StringRef &name) { + return lookup_token(name.byte(), name.size()); +} + // This function was generated by genheaderfunc.py. Inspired by h2o // header lookup. https://github.com/h2o/h2o int lookup_token(const uint8_t *name, size_t namelen) { @@ -760,7 +818,7 @@ void init_hdidx(HeaderIndex &hdidx) { std::fill(std::begin(hdidx), std::end(hdidx), -1); } -void index_header(HeaderIndex &hdidx, int16_t token, size_t idx) { +void index_header(HeaderIndex &hdidx, int32_t token, size_t idx) { if (token == -1) { return; } @@ -768,52 +826,7 @@ void index_header(HeaderIndex &hdidx, int16_t token, size_t idx) { hdidx[token] = idx; } -bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, - int16_t token) { - switch (token) { - case HD__AUTHORITY: - case HD__METHOD: - case HD__PATH: - case HD__SCHEME: - return hdidx[token] == -1; - default: - return false; - } -} - -bool check_http2_response_pseudo_header(const HeaderIndex &hdidx, - int16_t token) { - switch (token) { - case HD__STATUS: - return hdidx[token] == -1; - default: - return false; - } -} - -bool http2_header_allowed(int16_t token) { - switch (token) { - case HD_CONNECTION: - case HD_KEEP_ALIVE: - case HD_PROXY_CONNECTION: - case HD_TRANSFER_ENCODING: - case HD_UPGRADE: - return false; - default: - return true; - } -} - -bool http2_mandatory_request_headers_presence(const HeaderIndex &hdidx) { - if (hdidx[HD__METHOD] == -1 || hdidx[HD__PATH] == -1 || - hdidx[HD__SCHEME] == -1 || - (hdidx[HD__AUTHORITY] == -1 && hdidx[HD_HOST] == -1)) { - return false; - } - return true; -} - -const Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, +const Headers::value_type *get_header(const HeaderIndex &hdidx, int32_t token, const Headers &nva) { auto i = hdidx[token]; if (i == -1) { @@ -822,7 +835,7 @@ const Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, return &nva[i]; } -Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, +Headers::value_type *get_header(const HeaderIndex &hdidx, int32_t token, Headers &nva) { auto i = hdidx[token]; if (i == -1) { @@ -909,31 +922,54 @@ bool check_link_param_empty(const char *first, const char *last, } } // namespace +namespace { +// Returns true if link-param consists of only parmname, and it +// matches string [pat, pat + patlen). +bool check_link_param_without_value(const char *first, const char *last, + const char *pat, size_t patlen) { + if (first + patlen > last) { + return false; + } + + if (first + patlen == last) { + return std::equal(pat, pat + patlen, first, util::CaseCmp()); + } + + switch (*(first + patlen)) { + case ';': + case ',': + return std::equal(pat, pat + patlen, first, util::CaseCmp()); + } + + return false; +} +} // namespace + namespace { std::pair parse_next_link_header_once(const char *first, const char *last) { first = skip_to_next_field(first, last); if (first == last || *first != '<') { - return {{{nullptr, nullptr}}, last}; + return {{StringRef{}}, last}; } auto url_first = ++first; first = std::find(first, last, '>'); if (first == last) { - return {{{nullptr, nullptr}}, first}; + return {{StringRef{}}, first}; } auto url_last = first++; if (first == last) { - return {{{nullptr, nullptr}}, first}; + return {{StringRef{}}, first}; } // we expect ';' or ',' here switch (*first) { case ',': - return {{{nullptr, nullptr}}, ++first}; + return {{StringRef{}}, ++first}; case ';': ++first; break; default: - return {{{nullptr, nullptr}}, last}; + return {{StringRef{}}, last}; } auto ok = false; @@ -941,103 +977,115 @@ parse_next_link_header_once(const char *first, const char *last) { for (;;) { first = skip_lws(first, last); if (first == last) { - return {{{nullptr, nullptr}}, first}; + return {{StringRef{}}, first}; } // we expect link-param - // rel can take several relations using quoted form. - static constexpr char PLP[] = "rel=\""; - static constexpr size_t PLPLEN = sizeof(PLP) - 1; + if (!ign) { + if (!ok) { + // rel can take several relations using quoted form. + static constexpr char PLP[] = "rel=\""; + static constexpr size_t PLPLEN = str_size(PLP); - static constexpr char PLT[] = "preload"; - static constexpr size_t PLTLEN = sizeof(PLT) - 1; - if (first + PLPLEN < last && *(first + PLPLEN - 1) == '"' && - std::equal(PLP, PLP + PLPLEN, first, util::CaseCmp())) { - // we have to search preload in whitespace separated list: - // rel="preload something http://example.org/foo" - first += PLPLEN; - auto start = first; - for (; first != last;) { - if (*first != ' ' && *first != '"') { + static constexpr char PLT[] = "preload"; + static constexpr size_t PLTLEN = str_size(PLT); + if (first + PLPLEN < last && *(first + PLPLEN - 1) == '"' && + std::equal(PLP, PLP + PLPLEN, first, util::CaseCmp())) { + // we have to search preload in whitespace separated list: + // rel="preload something http://example.org/foo" + first += PLPLEN; + auto start = first; + for (; first != last;) { + if (*first != ' ' && *first != '"') { + ++first; + continue; + } + + if (start == first) { + return {{StringRef{}}, last}; + } + + if (!ok && start + PLTLEN == first && + std::equal(PLT, PLT + PLTLEN, start, util::CaseCmp())) { + ok = true; + } + + if (*first == '"') { + break; + } + first = skip_lws(first, last); + start = first; + } + if (first == last) { + return {{StringRef{}}, last}; + } + assert(*first == '"'); ++first; + if (first == last || *first == ',') { + goto almost_done; + } + if (*first == ';') { + ++first; + // parse next link-param + continue; + } + return {{StringRef{}}, last}; + } + } + // we are only interested in rel=preload parameter. Others are + // simply skipped. + static constexpr char PL[] = "rel=preload"; + static constexpr size_t PLLEN = str_size(PL); + if (first + PLLEN == last) { + if (std::equal(PL, PL + PLLEN, first, util::CaseCmp())) { + // ok = true; + // this is the end of sequence + return {{{url_first, url_last}}, last}; + } + } else if (first + PLLEN + 1 <= last) { + switch (*(first + PLLEN)) { + case ',': + if (!std::equal(PL, PL + PLLEN, first, util::CaseCmp())) { + break; + } + // ok = true; + // skip including ',' + first += PLLEN + 1; + return {{{url_first, url_last}}, first}; + case ';': + if (!std::equal(PL, PL + PLLEN, first, util::CaseCmp())) { + break; + } + ok = true; + // skip including ';' + first += PLLEN + 1; + // continue parse next link-param continue; } + } + // we have to reject URI if we have nonempty anchor parameter. + static constexpr char ANCHOR[] = "anchor="; + static constexpr size_t ANCHORLEN = str_size(ANCHOR); + if (!ign && !check_link_param_empty(first, last, ANCHOR, ANCHORLEN)) { + ign = true; + } - if (start == first) { - return {{{nullptr, nullptr}}, last}; - } + // reject URI if we have non-empty loadpolicy. This could be + // tightened up to just pick up "next" or "insert". + static constexpr char LOADPOLICY[] = "loadpolicy="; + static constexpr size_t LOADPOLICYLEN = str_size(LOADPOLICY); + if (!ign && + !check_link_param_empty(first, last, LOADPOLICY, LOADPOLICYLEN)) { + ign = true; + } - if (!ok && start + PLTLEN == first && - std::equal(PLT, PLT + PLTLEN, start, util::CaseCmp())) { - ok = true; - } - - if (*first == '"') { - break; - } - first = skip_lws(first, last); - start = first; + // reject URI if we have nopush attribute. + static constexpr char NOPUSH[] = "nopush"; + static constexpr size_t NOPUSHLEN = str_size(NOPUSH); + if (!ign && + check_link_param_without_value(first, last, NOPUSH, NOPUSHLEN)) { + ign = true; } - if (first == last) { - return {{{nullptr, nullptr}}, first}; - } - assert(*first == '"'); - ++first; - if (first == last || *first == ',') { - goto almost_done; - } - if (*first == ';') { - ++first; - // parse next link-param - continue; - } - return {{{nullptr, nullptr}}, last}; - } - // we are only interested in rel=preload parameter. Others are - // simply skipped. - static constexpr char PL[] = "rel=preload"; - static constexpr size_t PLLEN = sizeof(PL) - 1; - if (first + PLLEN == last) { - if (std::equal(PL, PL + PLLEN, first, util::CaseCmp())) { - // ok = true; - // this is the end of sequence - return {{{url_first, url_last}}, last}; - } - } else if (first + PLLEN + 1 <= last) { - switch (*(first + PLLEN)) { - case ',': - if (!std::equal(PL, PL + PLLEN, first, util::CaseCmp())) { - break; - } - // ok = true; - // skip including ',' - first += PLLEN + 1; - return {{{url_first, url_last}}, first}; - case ';': - if (!std::equal(PL, PL + PLLEN, first, util::CaseCmp())) { - break; - } - ok = true; - // skip including ';' - first += PLLEN + 1; - // continue parse next link-param - continue; - } - } - // we have to reject URI if we have nonempty anchor parameter. - static constexpr char ANCHOR[] = "anchor="; - static constexpr size_t ANCHORLEN = sizeof(ANCHOR) - 1; - if (!ign && !check_link_param_empty(first, last, ANCHOR, ANCHORLEN)) { - ign = true; - } - - // reject URI if we have non-empty loadpolicy. This could be - // tightened up to just pick up "next" or "insert". - static constexpr char LOADPOLICY[] = "loadpolicy="; - static constexpr size_t LOADPOLICYLEN = sizeof(LOADPOLICY) - 1; - if (!ign && - !check_link_param_empty(first, last, LOADPOLICY, LOADPOLICYLEN)) { - ign = true; } auto param_first = first; @@ -1057,11 +1105,11 @@ parse_next_link_header_once(const char *first, const char *last) { if (*first == '=' || *first == ';' || *first == ',') { break; } - return {{{nullptr, nullptr}}, last}; + return {{StringRef{}}, last}; } if (param_first == first) { // empty parmname - return {{{nullptr, nullptr}}, last}; + return {{StringRef{}}, last}; } // link-param without value is acceptable (see link-extension) if // it is not followed by '=' @@ -1078,13 +1126,13 @@ parse_next_link_header_once(const char *first, const char *last) { ++first; if (first == last) { // empty value is not acceptable - return {{{nullptr, nullptr}}, first}; + return {{StringRef{}}, first}; } if (*first == '"') { // quoted-string first = skip_to_right_dquote(first + 1, last); if (first == last) { - return {{{nullptr, nullptr}}, first}; + return {{StringRef{}}, first}; } ++first; if (first == last || *first == ',') { @@ -1095,12 +1143,12 @@ parse_next_link_header_once(const char *first, const char *last) { // parse next link-param continue; } - return {{{nullptr, nullptr}}, last}; + return {{StringRef{}}, last}; } // not quoted-string, skip to next ',' or ';' if (*first == ',' || *first == ';') { // empty value - return {{{nullptr, nullptr}}, last}; + return {{StringRef{}}, last}; } for (; first != last; ++first) { if (*first == ',' || *first == ';') { @@ -1124,7 +1172,7 @@ almost_done: if (ok && !ign) { return {{{url_first, url_last}}, first}; } - return {{{nullptr, nullptr}}, first}; + return {{StringRef{}}, first}; } } // namespace @@ -1135,141 +1183,19 @@ std::vector parse_link_header(const char *src, size_t len) { for (; first != last;) { auto rv = parse_next_link_header_once(first, last); first = rv.second; - if (rv.first.uri.first != nullptr && rv.first.uri.second != nullptr) { - res.push_back(rv.first); + auto &link = rv.first; + if (!link.uri.empty()) { + res.push_back(link); } } return res; } -namespace { -void eat_file(std::string &path) { - if (path.empty()) { - path = "/"; - return; - } - auto p = path.size() - 1; - if (path[p] == '/') { - return; - } - p = path.rfind('/', p); - if (p == std::string::npos) { - // this should not happend in normal case, where we expect path - // starts with '/' - path = "/"; - return; - } - path.erase(std::begin(path) + p + 1, std::end(path)); -} -} // namespace +std::string path_join(const StringRef &base_path, const StringRef &base_query, + const StringRef &rel_path, const StringRef &rel_query) { + BlockAllocator balloc(1024, 1024); -namespace { -void eat_dir(std::string &path) { - if (path.empty()) { - path = "/"; - return; - } - auto p = path.size() - 1; - if (path[p] != '/') { - p = path.rfind('/', p); - if (p == std::string::npos) { - // this should not happend in normal case, where we expect path - // starts with '/' - path = "/"; - return; - } - } - if (path[p] == '/') { - if (p == 0) { - return; - } - --p; - } - p = path.rfind('/', p); - if (p == std::string::npos) { - // this should not happend in normal case, where we expect path - // starts with '/' - path = "/"; - return; - } - path.erase(std::begin(path) + p + 1, std::end(path)); -} -} // namespace - -std::string path_join(const char *base_path, size_t base_pathlen, - const char *base_query, size_t base_querylen, - const char *rel_path, size_t rel_pathlen, - const char *rel_query, size_t rel_querylen) { - std::string res; - if (rel_pathlen == 0) { - if (base_pathlen == 0) { - res = "/"; - } else { - res.assign(base_path, base_pathlen); - } - if (rel_querylen == 0) { - if (base_querylen) { - res += '?'; - res.append(base_query, base_querylen); - } - return res; - } - res += '?'; - res.append(rel_query, rel_querylen); - return res; - } - - auto first = rel_path; - auto last = rel_path + rel_pathlen; - - if (rel_path[0] == '/') { - res = "/"; - ++first; - } else if (base_pathlen == 0) { - res = "/"; - } else { - res.assign(base_path, base_pathlen); - } - - for (; first != last;) { - if (*first == '.') { - if (first + 1 == last) { - break; - } - if (*(first + 1) == '/') { - first += 2; - continue; - } - if (*(first + 1) == '.') { - if (first + 2 == last) { - eat_dir(res); - break; - } - if (*(first + 2) == '/') { - eat_dir(res); - first += 3; - continue; - } - } - } - if (res.back() != '/') { - eat_file(res); - } - auto slash = std::find(first, last, '/'); - if (slash == last) { - res.append(first, last); - break; - } - res.append(first, slash + 1); - first = slash + 1; - for (; first != last && *first == '/'; ++first) - ; - } - if (rel_querylen) { - res += '?'; - res.append(rel_query, rel_querylen); - } - return res; + return path_join(balloc, base_path, base_query, rel_path, rel_query).str(); } bool expect_response_body(int status_code) { @@ -1289,6 +1215,10 @@ int lookup_method_token(const std::string &name) { name.size()); } +int lookup_method_token(const StringRef &name) { + return lookup_method_token(name.byte(), name.size()); +} + // This function was generated by genmethodfunc.py. int lookup_method_token(const uint8_t *name, size_t namelen) { switch (namelen) { @@ -1461,45 +1391,37 @@ int lookup_method_token(const uint8_t *name, size_t namelen) { return -1; } -const char *to_method_string(int method_token) { +StringRef to_method_string(int method_token) { // we happened to use same value for method with http-parser. - return http_method_str(static_cast(method_token)); + return StringRef{http_method_str(static_cast(method_token))}; } -int get_pure_path_component(const char **base, size_t *baselen, - const std::string &uri) { +StringRef get_pure_path_component(const StringRef &uri) { int rv; http_parser_url u{}; rv = http_parser_parse_url(uri.c_str(), uri.size(), 0, &u); if (rv != 0) { - return -1; + return StringRef{}; } if (u.field_set & (1 << UF_PATH)) { auto &f = u.field_data[UF_PATH]; - *base = uri.c_str() + f.off; - *baselen = f.len; - - return 0; + return StringRef{uri.c_str() + f.off, f.len}; } - *base = "/"; - *baselen = 1; - - return 0; + return StringRef::from_lit("/"); } -int construct_push_component(std::string &scheme, std::string &authority, - std::string &path, const char *base, - size_t baselen, const char *uri, size_t len) { +int construct_push_component(BlockAllocator &balloc, StringRef &scheme, + StringRef &authority, StringRef &path, + const StringRef &base, const StringRef &uri) { int rv; - const char *rel, *relq = nullptr; - size_t rellen, relqlen = 0; + StringRef rel, relq; http_parser_url u{}; - rv = http_parser_parse_url(uri, len, 0, &u); + rv = http_parser_parse_url(uri.c_str(), uri.size(), 0, &u); if (rv != 0) { if (uri[0] == '/') { @@ -1507,50 +1429,254 @@ int construct_push_component(std::string &scheme, std::string &authority, } // treat link_url as relative URI. - auto end = std::find(uri, uri + len, '#'); - auto q = std::find(uri, end, '?'); + auto end = std::find(std::begin(uri), std::end(uri), '#'); + auto q = std::find(std::begin(uri), end, '?'); - rel = uri; - rellen = q - uri; + rel = StringRef{std::begin(uri), q}; if (q != end) { - relq = q + 1; - relqlen = end - relq; + relq = StringRef{q + 1, std::end(uri)}; } } else { if (u.field_set & (1 << UF_SCHEMA)) { - http2::copy_url_component(scheme, &u, UF_SCHEMA, uri); + scheme = util::get_uri_field(uri.c_str(), u, UF_SCHEMA); } if (u.field_set & (1 << UF_HOST)) { - http2::copy_url_component(authority, &u, UF_HOST, uri); - if (u.field_set & (1 << UF_PORT)) { - authority += ':'; - authority += util::utos(u.port); + auto auth = util::get_uri_field(uri.c_str(), u, UF_HOST); + auto len = auth.size(); + auto port_exists = u.field_set & (1 << UF_PORT); + if (port_exists) { + len += 1 + str_size("65535"); } + auto iov = make_byte_ref(balloc, len + 1); + auto p = iov.base; + p = std::copy(std::begin(auth), std::end(auth), p); + if (port_exists) { + *p++ = ':'; + p = util::utos(p, u.port); + } + *p = '\0'; + + authority = StringRef{iov.base, p}; } if (u.field_set & (1 << UF_PATH)) { auto &f = u.field_data[UF_PATH]; - rel = uri + f.off; - rellen = f.len; + rel = StringRef{uri.c_str() + f.off, f.len}; } else { - rel = "/"; - rellen = 1; + rel = StringRef::from_lit("/"); } if (u.field_set & (1 << UF_QUERY)) { auto &f = u.field_data[UF_QUERY]; - relq = uri + f.off; - relqlen = f.len; + relq = StringRef{uri.c_str() + f.off, f.len}; } } - path = - http2::path_join(base, baselen, nullptr, 0, rel, rellen, relq, relqlen); + path = http2::path_join(balloc, base, StringRef{}, rel, relq); return 0; } +namespace { +template InputIt eat_file(InputIt first, InputIt last) { + if (first == last) { + *first++ = '/'; + return first; + } + + if (*(last - 1) == '/') { + return last; + } + + auto p = last; + for (; p != first && *(p - 1) != '/'; --p) + ; + if (p == first) { + // this should not happend in normal case, where we expect path + // starts with '/' + *first++ = '/'; + return first; + } + + return p; +} +} // namespace + +namespace { +template InputIt eat_dir(InputIt first, InputIt last) { + auto p = eat_file(first, last); + + --p; + + assert(*p == '/'); + + return eat_file(first, p); +} +} // namespace + +StringRef path_join(BlockAllocator &balloc, const StringRef &base_path, + const StringRef &base_query, const StringRef &rel_path, + const StringRef &rel_query) { + auto res = make_byte_ref( + balloc, std::max(static_cast(1), base_path.size()) + + rel_path.size() + 1 + + std::max(base_query.size(), rel_query.size()) + 1); + auto p = res.base; + + if (rel_path.empty()) { + if (base_path.empty()) { + *p++ = '/'; + } else { + p = std::copy(std::begin(base_path), std::end(base_path), p); + } + if (rel_query.empty()) { + if (!base_query.empty()) { + *p++ = '?'; + p = std::copy(std::begin(base_query), std::end(base_query), p); + } + *p = '\0'; + return StringRef{res.base, p}; + } + *p++ = '?'; + p = std::copy(std::begin(rel_query), std::end(rel_query), p); + *p = '\0'; + return StringRef{res.base, p}; + } + + auto first = std::begin(rel_path); + auto last = std::end(rel_path); + + if (rel_path[0] == '/') { + *p++ = '/'; + ++first; + for (; first != last && *first == '/'; ++first) + ; + } else if (base_path.empty()) { + *p++ = '/'; + } else { + p = std::copy(std::begin(base_path), std::end(base_path), p); + } + + for (; first != last;) { + if (*first == '.') { + if (first + 1 == last) { + break; + } + if (*(first + 1) == '/') { + first += 2; + continue; + } + if (*(first + 1) == '.') { + if (first + 2 == last) { + p = eat_dir(res.base, p); + break; + } + if (*(first + 2) == '/') { + p = eat_dir(res.base, p); + first += 3; + continue; + } + } + } + if (*(p - 1) != '/') { + p = eat_file(res.base, p); + } + auto slash = std::find(first, last, '/'); + if (slash == last) { + p = std::copy(first, last, p); + break; + } + p = std::copy(first, slash + 1, p); + first = slash + 1; + for (; first != last && *first == '/'; ++first) + ; + } + if (!rel_query.empty()) { + *p++ = '?'; + p = std::copy(std::begin(rel_query), std::end(rel_query), p); + } + *p = '\0'; + return StringRef{res.base, p}; +} + +StringRef normalize_path(BlockAllocator &balloc, const StringRef &path, + const StringRef &query) { + // First, decode %XX for unreserved characters, then do + // http2::path_join + + // We won't find %XX if length is less than 3. + if (path.size() < 3 || + std::find(std::begin(path), std::end(path), '%') == std::end(path)) { + return path_join(balloc, StringRef{}, StringRef{}, path, query); + } + + // includes last terminal NULL. + auto result = make_byte_ref(balloc, path.size() + 1); + auto p = result.base; + + auto it = std::begin(path); + for (; it + 2 < std::end(path);) { + if (*it == '%') { + if (util::is_hex_digit(*(it + 1)) && util::is_hex_digit(*(it + 2))) { + auto c = + (util::hex_to_uint(*(it + 1)) << 4) + util::hex_to_uint(*(it + 2)); + if (util::in_rfc3986_unreserved_chars(c)) { + *p++ = c; + + it += 3; + + continue; + } + *p++ = '%'; + *p++ = util::upcase(*(it + 1)); + *p++ = util::upcase(*(it + 2)); + + it += 3; + + continue; + } + } + *p++ = *it++; + } + + p = std::copy(it, std::end(path), p); + *p = '\0'; + + return path_join(balloc, StringRef{}, StringRef{}, StringRef{result.base, p}, + query); +} + +std::string normalize_path(const StringRef &path, const StringRef &query) { + BlockAllocator balloc(1024, 1024); + + return normalize_path(balloc, path, query).str(); +} + +StringRef rewrite_clean_path(BlockAllocator &balloc, const StringRef &src) { + if (src.empty() || src[0] != '/') { + return src; + } + // probably, not necessary most of the case, but just in case. + auto fragment = std::find(std::begin(src), std::end(src), '#'); + auto raw_query = std::find(std::begin(src), fragment, '?'); + auto query = raw_query; + if (query != fragment) { + ++query; + } + return normalize_path(balloc, StringRef{std::begin(src), raw_query}, + StringRef{query, fragment}); +} + +StringRef copy_lower(BlockAllocator &balloc, const StringRef &src) { + auto iov = make_byte_ref(balloc, src.size() + 1); + auto p = iov.base; + p = std::copy(std::begin(src), std::end(src), p); + *p = '\0'; + util::inp_strlower(iov.base, p); + return StringRef{iov.base, p}; +} + } // namespace http2 } // namespace nghttp2 diff --git a/src/http2.h b/src/http2.h index 8eaeff56..8ea2f91c 100644 --- a/src/http2.h +++ b/src/http2.h @@ -39,12 +39,14 @@ #include "util.h" #include "memchunk.h" +#include "template.h" +#include "allocator.h" namespace nghttp2 { struct Header { Header(std::string name, std::string value, bool no_index = false, - int16_t token = -1) + int32_t token = -1) : name(std::move(name)), value(std::move(value)), token(token), @@ -62,11 +64,33 @@ struct Header { std::string name; std::string value; - int16_t token; + int32_t token; + bool no_index; +}; + +struct HeaderRef { + HeaderRef(const StringRef &name, const StringRef &value, + bool no_index = false, int32_t token = -1) + : name(name), value(value), token(token), no_index(no_index) {} + + HeaderRef() : token(-1), no_index(false) {} + + bool operator==(const HeaderRef &other) const { + return name == other.name && value == other.value; + } + + bool operator<(const HeaderRef &rhs) const { + return name < rhs.name || (name == rhs.name && value < rhs.value); + } + + StringRef name; + StringRef value; + int32_t token; bool no_index; }; using Headers = std::vector
; +using HeaderRefs = std::vector; namespace http2 { @@ -74,9 +98,9 @@ std::string get_status_string(unsigned int status_code); // Returns string version of |status_code|. This function can handle // only predefined status code. Otherwise, returns nullptr. -const char *stringify_status(unsigned int status_code); +StringRef stringify_status(unsigned int status_code); -void capitalize(DefaultMemchunks *buf, const std::string &s); +void capitalize(DefaultMemchunks *buf, const StringRef &s); // Returns true if |value| is LWS bool lws(const char *value); @@ -89,25 +113,22 @@ void copy_url_component(std::string &dest, const http_parser_url *u, int field, Headers::value_type to_header(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, - bool no_index, int16_t token); + bool no_index, int32_t token); // Add name/value pairs to |nva|. If |no_index| is true, this // name/value pair won't be indexed when it is forwarded to the next // hop. This function strips white spaces around |value|. void add_header(Headers &nva, const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen, bool no_index, - int16_t token); + int32_t token); // Returns pointer to the entry in |nva| which has name |name|. If // more than one entries which have the name |name|, last occurrence // in |nva| is returned. If no such entry exist, returns nullptr. const Headers::value_type *get_header(const Headers &nva, const char *name); -// Returns nv->second if nv is not nullptr. Otherwise, returns "". -std::string value_to_str(const Headers::value_type *nv); - // Returns true if the value of |nv| is not empty. -bool non_empty_value(const Headers::value_type *nv); +bool non_empty_value(const HeaderRefs::value_type *nv); // Creates nghttp2_nv using |name| and |value| and returns it. The // returned value only references the data pointer to name.c_str() and @@ -116,9 +137,15 @@ bool non_empty_value(const Headers::value_type *nv); nghttp2_nv make_nv(const std::string &name, const std::string &value, bool no_index = false); +nghttp2_nv make_nv(const StringRef &name, const StringRef &value, + bool no_index = false); + nghttp2_nv make_nv_nocopy(const std::string &name, const std::string &value, bool no_index = false); +nghttp2_nv make_nv_nocopy(const StringRef &name, const StringRef &value, + bool no_index = false); + // Create nghttp2_nv from string literal |name| and |value|. template constexpr nghttp2_nv make_nv_ll(const char(&name)[N], const char(&value)[M]) { @@ -163,19 +190,20 @@ nghttp2_nv make_nv_ls_nocopy(const char(&name)[N], const StringRef &value) { // before this call (its element's token field is assigned). Certain // headers, including disallowed headers in HTTP/2 spec and headers // which require special handling (i.e. via), are not copied. -void copy_headers_to_nva(std::vector &nva, const Headers &headers); +void copy_headers_to_nva(std::vector &nva, + const HeaderRefs &headers); // Just like copy_headers_to_nva(), but this adds // NGHTTP2_NV_FLAG_NO_COPY_NAME and NGHTTP2_NV_FLAG_NO_COPY_VALUE. void copy_headers_to_nva_nocopy(std::vector &nva, - const Headers &headers); + const HeaderRefs &headers); // Appends HTTP/1.1 style header lines to |buf| from headers in // |headers|. |headers| must be indexed before this call (its // element's token field is assigned). Certain headers, which // requires special handling (i.e. via and cookie), are not appended. void build_http1_headers_from_headers(DefaultMemchunks *buf, - const Headers &headers); + const HeaderRefs &headers); // Return positive window_size_increment if WINDOW_UPDATE should be // sent for the stream |stream_id|. If |stream_id| == 0, this function @@ -196,6 +224,8 @@ void dump_nv(FILE *out, const nghttp2_nv *nva, size_t nvlen); // Dumps name/value pairs in |nva| to |out|. void dump_nv(FILE *out, const Headers &nva); +void dump_nv(FILE *out, const HeaderRefs &nva); + // Rewrites redirection URI which usually appears in location header // field. The |uri| is the URI in the location header field. The |u| // stores the result of parsed |uri|. The |request_authority| is the @@ -209,11 +239,11 @@ void dump_nv(FILE *out, const Headers &nva); // This function returns the new rewritten URI on success. If the // location URI is not subject to the rewrite, this function returns // emtpy string. -std::string rewrite_location_uri(const std::string &uri, - const http_parser_url &u, - const std::string &match_host, - const std::string &request_authority, - const std::string &upstream_scheme); +StringRef rewrite_location_uri(BlockAllocator &balloc, const StringRef &uri, + const http_parser_url &u, + const StringRef &match_host, + const StringRef &request_authority, + const StringRef &upstream_scheme); // Checks the header name/value pair using nghttp2_check_header_name() // and nghttp2_check_header_value(). If both function returns nonzero, @@ -222,7 +252,7 @@ int check_nv(const uint8_t *name, size_t namelen, const uint8_t *value, size_t valuelen); // Returns parsed HTTP status code. Returns -1 on failure. -int parse_http_status_code(const std::string &src); +int parse_http_status_code(const StringRef &src); // Header fields to be indexed, except HD_MAXIDX which is convenient // member to get maximum value. @@ -272,40 +302,24 @@ using HeaderIndex = std::array; // cannot be tokenized, returns -1. int lookup_token(const uint8_t *name, size_t namelen); int lookup_token(const std::string &name); +int lookup_token(const StringRef &name); // Initializes |hdidx|, header index. The |hdidx| must point to the // array containing at least HD_MAXIDX elements. void init_hdidx(HeaderIndex &hdidx); // Indexes header |token| using index |idx|. -void index_header(HeaderIndex &hdidx, int16_t token, size_t idx); - -// Returns true if HTTP/2 request pseudo header |token| is not indexed -// yet and not -1. -bool check_http2_request_pseudo_header(const HeaderIndex &hdidx, int16_t token); - -// Returns true if HTTP/2 response pseudo header |token| is not -// indexed yet and not -1. -bool check_http2_response_pseudo_header(const HeaderIndex &hdidx, - int16_t token); - -// Returns true if header field denoted by |token| is allowed for -// HTTP/2. -bool http2_header_allowed(int16_t token); - -// Returns true that |hdidx| contains mandatory HTTP/2 request -// headers. -bool http2_mandatory_request_headers_presence(const HeaderIndex &hdidx); +void index_header(HeaderIndex &hdidx, int32_t token, size_t idx); // Returns header denoted by |token| using index |hdidx|. -const Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, +const Headers::value_type *get_header(const HeaderIndex &hdidx, int32_t token, const Headers &nva); -Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, +Headers::value_type *get_header(const HeaderIndex &hdidx, int32_t token, Headers &nva); struct LinkHeader { - // The region of URI is [uri.first, uri.second). - std::pair uri; + // The region of URI. This might not be NULL-terminated. + StringRef uri; }; // Returns next URI-reference in Link header field value |src| of @@ -314,16 +328,17 @@ struct LinkHeader { // is ignored during parsing. std::vector parse_link_header(const char *src, size_t len); -// Constructs path by combining base path |base_path| of length -// |base_pathlen| with another path |rel_path| of length -// |rel_pathlen|. The base path and another path can have optional +// Constructs path by combining base path |base_path| with another +// path |rel_path|. The base path and another path can have optional // query component. This function assumes |base_path| is normalized. // In other words, it does not contain ".." or "." path components // and starts with "/" if it is not empty. -std::string path_join(const char *base_path, size_t base_pathlen, - const char *base_query, size_t base_querylen, - const char *rel_path, size_t rel_pathlen, - const char *rel_query, size_t rel_querylen); +std::string path_join(const StringRef &base, const StringRef &base_query, + const StringRef &rel_path, const StringRef &rel_query); + +StringRef path_join(BlockAllocator &balloc, const StringRef &base_path, + const StringRef &base_query, const StringRef &rel_path, + const StringRef &rel_query); // true if response has body, taking into account the request method // and status code. @@ -338,73 +353,37 @@ bool expect_response_body(int status_code); // tokenized. If method name cannot be tokenized, returns -1. int lookup_method_token(const uint8_t *name, size_t namelen); int lookup_method_token(const std::string &name); +int lookup_method_token(const StringRef &name); -const char *to_method_string(int method_token); +// Returns string representation of |method_token|. This is wrapper +// function over http_method_str from http-parser. If |method_token| +// is not known to http-parser, "" is returned. The returned +// StringRef is guaranteed to be NULL-terminated. +StringRef to_method_string(int method_token); -template -std::string normalize_path(InputIt first, InputIt last) { - // First, decode %XX for unreserved characters, then do - // http2::join_path - std::string result; - // We won't find %XX if length is less than 3. - if (last - first < 3) { - result.assign(first, last); - } else { - for (; first < last - 2;) { - if (*first == '%') { - if (util::is_hex_digit(*(first + 1)) && - util::is_hex_digit(*(first + 2))) { - auto c = (util::hex_to_uint(*(first + 1)) << 4) + - util::hex_to_uint(*(first + 2)); - if (util::in_rfc3986_unreserved_chars(c)) { - result += c; - first += 3; - continue; - } - result += '%'; - result += util::upcase(*(first + 1)); - result += util::upcase(*(first + 2)); - first += 3; - continue; - } - } - result += *first++; - } - result.append(first, last); - } - return path_join(nullptr, 0, nullptr, 0, result.c_str(), result.size(), - nullptr, 0); -} +StringRef normalize_path(BlockAllocator &balloc, const StringRef &path, + const StringRef &query); -template -std::string rewrite_clean_path(InputIt first, InputIt last) { - if (first == last || *first != '/') { - return std::string(first, last); - } - // probably, not necessary most of the case, but just in case. - auto fragment = std::find(first, last, '#'); - auto query = std::find(first, fragment, '?'); - auto path = normalize_path(first, query); - if (query != fragment) { - path.append(query, fragment); - } - return path; -} +std::string normalize_path(const StringRef &path, const StringRef &query); -// Stores path component of |uri| in *base. Its extracted length is -// stored in *baselen. The extracted path does not include query -// component. This function returns 0 if it succeeds, or -1. -int get_pure_path_component(const char **base, size_t *baselen, - const std::string &uri); +StringRef rewrite_clean_path(BlockAllocator &balloc, const StringRef &src); -// Deduces scheme, authority and path from given |uri| of length -// |len|, and stores them in |scheme|, |authority|, and |path| -// respectively. If |uri| is relative path, path resolution is taken -// palce using path given in |base| of length |baselen|. This -// function returns 0 if it succeeds, or -1. -int construct_push_component(std::string &scheme, std::string &authority, - std::string &path, const char *base, - size_t baselen, const char *uri, size_t len); +// Returns path component of |uri|. The returned path does not +// include query component. This function returns empty string if it +// fails. +StringRef get_pure_path_component(const StringRef &uri); + +// Deduces scheme, authority and path from given |uri|, and stores +// them in |scheme|, |authority|, and |path| respectively. If |uri| +// is relative path, path resolution takes place using path given in +// |base| of length |baselen|. This function returns 0 if it +// succeeds, or -1. +int construct_push_component(BlockAllocator &balloc, StringRef &scheme, + StringRef &authority, StringRef &path, + const StringRef &base, const StringRef &uri); + +// Copies |src| and return its lower-cased version. +StringRef copy_lower(BlockAllocator &balloc, const StringRef &src); } // namespace http2 diff --git a/src/http2_test.cc b/src/http2_test.cc index 8f3aab35..fa15e745 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -46,7 +46,7 @@ using namespace nghttp2; namespace shrpx { namespace { -void check_nv(const Header &a, const nghttp2_nv *b) { +void check_nv(const HeaderRef &a, const nghttp2_nv *b) { CU_ASSERT(a.name.size() == b->namelen); CU_ASSERT(a.value.size() == b->valuelen); CU_ASSERT(memcmp(a.name.c_str(), b->name, b->namelen) == 0); @@ -134,20 +134,24 @@ void test_http2_get_header(void) { } namespace { -auto headers = - Headers{{"alpha", "0", true}, - {"bravo", "1"}, - {"connection", "2", false, http2::HD_CONNECTION}, - {"connection", "3", false, http2::HD_CONNECTION}, - {"delta", "4"}, - {"expect", "5"}, - {"foxtrot", "6"}, - {"tango", "7"}, - {"te", "8", false, http2::HD_TE}, - {"te", "9", false, http2::HD_TE}, - {"x-forwarded-proto", "10", false, http2::HD_X_FORWARDED_FOR}, - {"x-forwarded-proto", "11", false, http2::HD_X_FORWARDED_FOR}, - {"zulu", "12"}}; +auto headers = HeaderRefs{ + {StringRef::from_lit("alpha"), StringRef::from_lit("0"), true}, + {StringRef::from_lit("bravo"), StringRef::from_lit("1")}, + {StringRef::from_lit("connection"), StringRef::from_lit("2"), false, + http2::HD_CONNECTION}, + {StringRef::from_lit("connection"), StringRef::from_lit("3"), false, + http2::HD_CONNECTION}, + {StringRef::from_lit("delta"), StringRef::from_lit("4")}, + {StringRef::from_lit("expect"), StringRef::from_lit("5")}, + {StringRef::from_lit("foxtrot"), StringRef::from_lit("6")}, + {StringRef::from_lit("tango"), StringRef::from_lit("7")}, + {StringRef::from_lit("te"), StringRef::from_lit("8"), false, http2::HD_TE}, + {StringRef::from_lit("te"), StringRef::from_lit("9"), false, http2::HD_TE}, + {StringRef::from_lit("x-forwarded-proto"), StringRef::from_lit("10"), false, + http2::HD_X_FORWARDED_FOR}, + {StringRef::from_lit("x-forwarded-proto"), StringRef::from_lit("11"), false, + http2::HD_X_FORWARDED_FOR}, + {StringRef::from_lit("zulu"), StringRef::from_lit("12")}}; } // namespace void test_http2_copy_headers_to_nva(void) { @@ -209,10 +213,12 @@ void check_rewrite_location_uri(const std::string &want, const std::string &uri, const std::string &match_host, const std::string &req_authority, const std::string &upstream_scheme) { + BlockAllocator balloc(4096, 4096); http_parser_url u{}; CU_ASSERT(0 == http_parser_parse_url(uri.c_str(), uri.size(), 0, &u)); - auto got = http2::rewrite_location_uri(uri, u, match_host, req_authority, - upstream_scheme); + auto got = http2::rewrite_location_uri( + balloc, StringRef{uri}, u, StringRef{match_host}, + StringRef{req_authority}, StringRef{upstream_scheme}); CU_ASSERT(want == got); } } // namespace @@ -245,13 +251,13 @@ void test_http2_rewrite_location_uri(void) { } void test_http2_parse_http_status_code(void) { - CU_ASSERT(200 == http2::parse_http_status_code("200")); - CU_ASSERT(102 == http2::parse_http_status_code("102")); - CU_ASSERT(-1 == http2::parse_http_status_code("099")); - CU_ASSERT(-1 == http2::parse_http_status_code("99")); - CU_ASSERT(-1 == http2::parse_http_status_code("-1")); - CU_ASSERT(-1 == http2::parse_http_status_code("20a")); - CU_ASSERT(-1 == http2::parse_http_status_code("")); + CU_ASSERT(200 == http2::parse_http_status_code(StringRef::from_lit("200"))); + CU_ASSERT(102 == http2::parse_http_status_code(StringRef::from_lit("102"))); + CU_ASSERT(-1 == http2::parse_http_status_code(StringRef::from_lit("099"))); + CU_ASSERT(-1 == http2::parse_http_status_code(StringRef::from_lit("99"))); + CU_ASSERT(-1 == http2::parse_http_status_code(StringRef::from_lit("-1"))); + CU_ASSERT(-1 == http2::parse_http_status_code(StringRef::from_lit("20a"))); + CU_ASSERT(-1 == http2::parse_http_status_code(StringRef{})); } void test_http2_index_header(void) { @@ -271,728 +277,691 @@ void test_http2_lookup_token(void) { CU_ASSERT(http2::HD_EXPECT == http2::lookup_token("expect")); } -void test_http2_check_http2_pseudo_header(void) { - http2::HeaderIndex hdidx; - http2::init_hdidx(hdidx); - - CU_ASSERT(http2::check_http2_request_pseudo_header(hdidx, http2::HD__METHOD)); - hdidx[http2::HD__PATH] = 0; - CU_ASSERT(http2::check_http2_request_pseudo_header(hdidx, http2::HD__METHOD)); - hdidx[http2::HD__METHOD] = 1; - CU_ASSERT( - !http2::check_http2_request_pseudo_header(hdidx, http2::HD__METHOD)); - CU_ASSERT(!http2::check_http2_request_pseudo_header(hdidx, http2::HD_VIA)); - - http2::init_hdidx(hdidx); - - CU_ASSERT( - http2::check_http2_response_pseudo_header(hdidx, http2::HD__STATUS)); - hdidx[http2::HD__STATUS] = 0; - CU_ASSERT( - !http2::check_http2_response_pseudo_header(hdidx, http2::HD__STATUS)); - CU_ASSERT(!http2::check_http2_response_pseudo_header(hdidx, http2::HD_VIA)); -} - -void test_http2_http2_header_allowed(void) { - CU_ASSERT(http2::http2_header_allowed(http2::HD__PATH)); - CU_ASSERT(http2::http2_header_allowed(http2::HD_CONTENT_LENGTH)); - CU_ASSERT(!http2::http2_header_allowed(http2::HD_CONNECTION)); -} - -void test_http2_mandatory_request_headers_presence(void) { - http2::HeaderIndex hdidx; - http2::init_hdidx(hdidx); - - CU_ASSERT(!http2::http2_mandatory_request_headers_presence(hdidx)); - hdidx[http2::HD__AUTHORITY] = 0; - CU_ASSERT(!http2::http2_mandatory_request_headers_presence(hdidx)); - hdidx[http2::HD__METHOD] = 1; - CU_ASSERT(!http2::http2_mandatory_request_headers_presence(hdidx)); - hdidx[http2::HD__PATH] = 2; - CU_ASSERT(!http2::http2_mandatory_request_headers_presence(hdidx)); - hdidx[http2::HD__SCHEME] = 3; - CU_ASSERT(http2::http2_mandatory_request_headers_presence(hdidx)); - - hdidx[http2::HD__AUTHORITY] = -1; - hdidx[http2::HD_HOST] = 0; - CU_ASSERT(http2::http2_mandatory_request_headers_presence(hdidx)); -} - void test_http2_parse_link_header(void) { { // only URI appears; we don't extract URI unless it bears rel=preload - const char s[] = ""; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = ""; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // URI url should be extracted - const char s[] = "; rel=preload"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = "; rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // With extra link-param. URI url should be extracted - const char s[] = "; rel=preload; as=file"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = "; rel=preload; as=file"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // With extra link-param. URI url should be extracted - const char s[] = "; as=file; rel=preload"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = "; as=file; rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // With extra link-param and quote-string. URI url should be // extracted - const char s[] = R"(; rel=preload; title="foo,bar")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel=preload; title="foo,bar")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // With extra link-param and quote-string. URI url should be // extracted - const char s[] = R"(; title="foo,bar"; rel=preload)"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; title="foo,bar"; rel=preload)"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // ',' after quote-string - const char s[] = R"(; title="foo,bar", ; rel=preload)"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; title="foo,bar", ; rel=preload)"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[25], &s[28]) == res[0].uri); + CU_ASSERT("url2" == res[0].uri); + CU_ASSERT(&s[25] == &res[0].uri[0]); } { // Only first URI should be extracted. - const char s[] = "; rel=preload, "; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = "; rel=preload, "; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // Both have rel=preload, so both urls should be extracted - const char s[] = "; rel=preload, ; rel=preload"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = "; rel=preload, ; rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(2 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); - CU_ASSERT(std::make_pair(&s[21], &s[24]) == res[1].uri); + CU_ASSERT("url" == res[0].uri); + CU_ASSERT("url2" == res[1].uri); } { // Second URI uri should be extracted. - const char s[] = ", ;rel=preload"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = ", ;rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[8], &s[11]) == res[0].uri); + CU_ASSERT("url2" == res[0].uri); } { // Error if input ends with ';' - const char s[] = ";rel=preload;"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = ";rel=preload;"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // Error if link header ends with ';' - const char s[] = ";rel=preload;, "; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = ";rel=preload;, "; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // OK if input ends with ',' - const char s[] = ";rel=preload,"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = ";rel=preload,"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // Multiple repeated ','s between fields is OK - const char s[] = ",,,;rel=preload"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = ",,,;rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[9], &s[12]) == res[0].uri); + CU_ASSERT("url2" == res[0].uri); } { // Error if url is not enclosed by <> - const char s[] = "url>;rel=preload"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = "url>;rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // Error if url is not enclosed by <> - const char s[] = "' is not followed by ';' - const char s[] = " rel=preload"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = " rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // Starting with whitespace is no problem. - const char s[] = " ; rel=preload"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = " ; rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[3], &s[6]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // preload is a prefix of bogus rel parameter value - const char s[] = "; rel=preloadx"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = "; rel=preloadx"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // preload in relation-types list - const char s[] = R"(; rel="preload")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="preload")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // preload in relation-types list followed by another parameter - const char s[] = R"(; rel="preload foo")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="preload foo")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // preload in relation-types list following another parameter - const char s[] = R"(; rel="foo preload")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="foo preload")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // preload in relation-types list between other parameters - const char s[] = R"(; rel="foo preload bar")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="foo preload bar")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // preload in relation-types list between other parameters - const char s[] = R"(; rel="foo preload bar")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="foo preload bar")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // no preload in relation-types list - const char s[] = R"(; rel="foo")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="foo")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // no preload in relation-types list, multiple unrelated elements. - const char s[] = R"(; rel="foo bar")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="foo bar")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // preload in relation-types list, followed by another link-value. - const char s[] = R"(; rel="preload", )"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="preload", )"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // preload in relation-types list, following another link-value. - const char s[] = R"(, ; rel="preload")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(, ; rel="preload")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[8], &s[11]) == res[0].uri); + CU_ASSERT("url2" == res[0].uri); } { // preload in relation-types list, followed by another link-param. - const char s[] = R"(; rel="preload"; as="font")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="preload"; as="font")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // preload in relation-types list, followed by character other // than ';' or ',' - const char s[] = R"(; rel="preload".)"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="preload".)"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // preload in relation-types list, followed by ';' but it // terminates input - const char s[] = R"(; rel="preload";)"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="preload";)"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // preload in relation-types list, followed by ',' but it // terminates input - const char s[] = R"(; rel="preload",)"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="preload",)"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // preload in relation-types list but there is preceding white // space. - const char s[] = R"(; rel=" preload")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel=" preload")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // preload in relation-types list but there is trailing white // space. - const char s[] = R"(; rel="preload ")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel="preload ")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // backslash escaped characters in quoted-string - const char s[] = R"(; rel=preload; title="foo\"baz\"bar")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel=preload; title="foo\"baz\"bar")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // anchor="" is acceptable - const char s[] = R"(; rel=preload; anchor="")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel=preload; anchor="")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // With anchor="#foo", url should be ignored - const char s[] = R"(; rel=preload; anchor="#foo")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel=preload; anchor="#foo")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // With anchor=f, url should be ignored - const char s[] = "; rel=preload; anchor=f"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = "; rel=preload; anchor=f"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // First url is ignored With anchor="#foo", but url should be // accepted. - const char s[] = R"(; rel=preload; anchor="#foo", ; rel=preload)"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = + R"(; rel=preload; anchor="#foo", ; rel=preload)"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[36], &s[39]) == res[0].uri); + CU_ASSERT("url2" == res[0].uri); } { // With loadpolicy="next", url should be ignored - const char s[] = R"(; rel=preload; loadpolicy="next")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel=preload; loadpolicy="next")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(0 == res.size()); } { // url should be picked up if empty loadpolicy is specified - const char s[] = R"(; rel=preload; loadpolicy="")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel=preload; loadpolicy="")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(1 == res.size()); - CU_ASSERT(std::make_pair(&s[1], &s[4]) == res[0].uri); + CU_ASSERT("url" == res[0].uri); } { // case-insensitive match - const char s[] = R"(; rel=preload; ANCHOR="#foo", ; )" - R"(REL=PRELOAD, ; REL="foo PRELOAD bar")"; - auto res = http2::parse_link_header(s, sizeof(s) - 1); + constexpr char s[] = R"(; rel=preload; ANCHOR="#foo", ; )" + R"(REL=PRELOAD, ; REL="foo PRELOAD bar")"; + auto res = http2::parse_link_header(s, str_size(s)); CU_ASSERT(2 == res.size()); - CU_ASSERT(std::make_pair(&s[36], &s[39]) == res[0].uri); - CU_ASSERT(std::make_pair(&s[42 + 14], &s[42 + 17]) == res[1].uri); + CU_ASSERT("url2" == res[0].uri); + CU_ASSERT("url3" == res[1].uri); + } + { + // nopush at the end of input + constexpr char s[] = "; rel=preload; nopush"; + auto res = http2::parse_link_header(s, str_size(s)); + CU_ASSERT(0 == res.size()); + } + { + // nopush followed by ';' + constexpr char s[] = "; rel=preload; nopush; foo"; + auto res = http2::parse_link_header(s, str_size(s)); + CU_ASSERT(0 == res.size()); + } + { + // nopush followed by ',' + constexpr char s[] = "; nopush; rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); + CU_ASSERT(0 == res.size()); + } + { + // string whose prefix is nopush + constexpr char s[] = "; nopushyes; rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); + CU_ASSERT(1 == res.size()); + CU_ASSERT("url" == res[0].uri); + } + { + // rel=preload twice + constexpr char s[] = "; rel=preload; rel=preload"; + auto res = http2::parse_link_header(s, str_size(s)); + CU_ASSERT(1 == res.size()); + CU_ASSERT("url" == res[0].uri); } } void test_http2_path_join(void) { { - const char base[] = "/"; - const char rel[] = "/"; - CU_ASSERT("/" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/"); + auto rel = StringRef::from_lit("/"); + CU_ASSERT("/" == http2::path_join(base, StringRef{}, rel, StringRef{})); } { - const char base[] = "/"; - const char rel[] = "/alpha"; - CU_ASSERT("/alpha" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, - rel, sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/"); + auto rel = StringRef::from_lit("/alpha"); + CU_ASSERT("/alpha" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // rel ends with trailing '/' - const char base[] = "/"; - const char rel[] = "/alpha/"; - CU_ASSERT("/alpha/" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, - rel, sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/"); + auto rel = StringRef::from_lit("/alpha/"); + CU_ASSERT("/alpha/" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // rel contains multiple components - const char base[] = "/"; - const char rel[] = "/alpha/bravo"; - CU_ASSERT("/alpha/bravo" == http2::path_join(base, sizeof(base) - 1, - nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/"); + auto rel = StringRef::from_lit("/alpha/bravo"); + CU_ASSERT("/alpha/bravo" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // rel is relative - const char base[] = "/"; - const char rel[] = "alpha/bravo"; - CU_ASSERT("/alpha/bravo" == http2::path_join(base, sizeof(base) - 1, - nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/"); + auto rel = StringRef::from_lit("alpha/bravo"); + CU_ASSERT("/alpha/bravo" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // rel is relative and base ends without /, which means it refers // to file. - const char base[] = "/alpha"; - const char rel[] = "bravo/charlie"; + auto base = StringRef::from_lit("/alpha"); + auto rel = StringRef::from_lit("bravo/charlie"); CU_ASSERT("/bravo/charlie" == - http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // rel contains repeated '/'s - const char base[] = "/"; - const char rel[] = "/alpha/////bravo/////"; - CU_ASSERT("/alpha/bravo/" == http2::path_join(base, sizeof(base) - 1, - nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/"); + auto rel = StringRef::from_lit("/alpha/////bravo/////"); + CU_ASSERT("/alpha/bravo/" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // base ends with '/', so '..' eats 'bravo' - const char base[] = "/alpha/bravo/"; - const char rel[] = "../charlie/delta"; + auto base = StringRef::from_lit("/alpha/bravo/"); + auto rel = StringRef::from_lit("../charlie/delta"); CU_ASSERT("/alpha/charlie/delta" == - http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // base does not end with '/', so '..' eats 'alpha/bravo' - const char base[] = "/alpha/bravo"; - const char rel[] = "../charlie"; - CU_ASSERT("/charlie" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, - rel, sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/alpha/bravo"); + auto rel = StringRef::from_lit("../charlie"); + CU_ASSERT("/charlie" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // 'charlie' is eaten by following '..' - const char base[] = "/alpha/bravo/"; - const char rel[] = "../charlie/../delta"; - CU_ASSERT("/alpha/delta" == http2::path_join(base, sizeof(base) - 1, - nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/alpha/bravo/"); + auto rel = StringRef::from_lit("../charlie/../delta"); + CU_ASSERT("/alpha/delta" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // excessive '..' results in '/' - const char base[] = "/alpha/bravo/"; - const char rel[] = "../../../"; - CU_ASSERT("/" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/alpha/bravo/"); + auto rel = StringRef::from_lit("../../../"); + CU_ASSERT("/" == http2::path_join(base, StringRef{}, rel, StringRef{})); } { // excessive '..' and path component - const char base[] = "/alpha/bravo/"; - const char rel[] = "../../../charlie"; - CU_ASSERT("/charlie" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, - rel, sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/alpha/bravo/"); + auto rel = StringRef::from_lit("../../../charlie"); + CU_ASSERT("/charlie" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // rel ends with '..' - const char base[] = "/alpha/bravo/"; - const char rel[] = "charlie/.."; - CU_ASSERT("/alpha/bravo/" == http2::path_join(base, sizeof(base) - 1, - nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef::from_lit("/alpha/bravo/"); + auto rel = StringRef::from_lit("charlie/.."); + CU_ASSERT("/alpha/bravo/" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // base empty and rel contains '..' - const char base[] = ""; - const char rel[] = "charlie/.."; - CU_ASSERT("/" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef{}; + auto rel = StringRef::from_lit("charlie/.."); + CU_ASSERT("/" == http2::path_join(base, StringRef{}, rel, StringRef{})); } { // '.' is ignored - const char base[] = "/"; - const char rel[] = "charlie/././././delta"; + auto base = StringRef::from_lit("/"); + auto rel = StringRef::from_lit("charlie/././././delta"); CU_ASSERT("/charlie/delta" == - http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, - sizeof(rel) - 1, nullptr, 0)); + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // trailing '.' is ignored - const char base[] = "/"; - const char rel[] = "charlie/."; - CU_ASSERT("/charlie/" == http2::path_join(base, sizeof(base) - 1, nullptr, - 0, rel, sizeof(rel) - 1, nullptr, - 0)); + auto base = StringRef::from_lit("/"); + auto rel = StringRef::from_lit("charlie/."); + CU_ASSERT("/charlie/" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // query - const char base[] = "/"; - const char rel[] = "/"; - const char relq[] = "q"; - CU_ASSERT("/?q" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, rel, - sizeof(rel) - 1, relq, - sizeof(relq) - 1)); + auto base = StringRef::from_lit("/"); + auto rel = StringRef::from_lit("/"); + auto relq = StringRef::from_lit("q"); + CU_ASSERT("/?q" == http2::path_join(base, StringRef{}, rel, relq)); } { // empty rel and query - const char base[] = "/alpha"; - const char rel[] = ""; - const char relq[] = "q"; - CU_ASSERT("/alpha?q" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, - rel, sizeof(rel) - 1, relq, - sizeof(relq) - 1)); + auto base = StringRef::from_lit("/alpha"); + auto rel = StringRef{}; + auto relq = StringRef::from_lit("q"); + CU_ASSERT("/alpha?q" == http2::path_join(base, StringRef{}, rel, relq)); } { // both rel and query are empty - const char base[] = "/alpha"; - const char baseq[] = "r"; - const char rel[] = ""; - const char relq[] = ""; - CU_ASSERT("/alpha?r" == - http2::path_join(base, sizeof(base) - 1, baseq, sizeof(baseq) - 1, - rel, sizeof(rel) - 1, relq, sizeof(relq) - 1)); + auto base = StringRef::from_lit("/alpha"); + auto baseq = StringRef::from_lit("r"); + auto rel = StringRef{}; + auto relq = StringRef{}; + CU_ASSERT("/alpha?r" == http2::path_join(base, baseq, rel, relq)); } { // empty base - const char base[] = ""; - const char rel[] = "/alpha"; - CU_ASSERT("/alpha" == http2::path_join(base, sizeof(base) - 1, nullptr, 0, - rel, sizeof(rel) - 1, nullptr, 0)); + auto base = StringRef{}; + auto rel = StringRef::from_lit("/alpha"); + CU_ASSERT("/alpha" == + http2::path_join(base, StringRef{}, rel, StringRef{})); } { // everything is empty - CU_ASSERT("/" == - http2::path_join(nullptr, 0, nullptr, 0, nullptr, 0, nullptr, 0)); + CU_ASSERT("/" == http2::path_join(StringRef{}, StringRef{}, StringRef{}, + StringRef{})); } { // only baseq is not empty - const char base[] = ""; - const char baseq[] = "r"; - const char rel[] = ""; - CU_ASSERT("/?r" == http2::path_join(base, sizeof(base) - 1, baseq, - sizeof(baseq) - 1, rel, sizeof(rel) - 1, - nullptr, 0)); + auto base = StringRef{}; + auto baseq = StringRef::from_lit("r"); + auto rel = StringRef{}; + CU_ASSERT("/?r" == http2::path_join(base, baseq, rel, StringRef{})); + } + { + // path starts with multiple '/'s. + auto base = StringRef{}; + auto baseq = StringRef{}; + auto rel = StringRef::from_lit("//alpha//bravo"); + auto relq = StringRef::from_lit("charlie"); + CU_ASSERT("/alpha/bravo?charlie" == + http2::path_join(base, baseq, rel, relq)); } } void test_http2_normalize_path(void) { - std::string src; - - src = "/alpha/bravo/../charlie"; CU_ASSERT("/alpha/charlie" == - http2::normalize_path(std::begin(src), std::end(src))); + http2::normalize_path( + StringRef::from_lit("/alpha/bravo/../charlie"), StringRef{})); - src = "/a%6c%70%68%61"; - CU_ASSERT("/alpha" == http2::normalize_path(std::begin(src), std::end(src))); + CU_ASSERT("/alpha" == + http2::normalize_path(StringRef::from_lit("/a%6c%70%68%61"), + StringRef{})); - src = "/alpha%2f%3a"; - CU_ASSERT("/alpha%2F%3A" == - http2::normalize_path(std::begin(src), std::end(src))); + CU_ASSERT( + "/alpha%2F%3A" == + http2::normalize_path(StringRef::from_lit("/alpha%2f%3a"), StringRef{})); - src = "%2f"; - CU_ASSERT("/%2F" == http2::normalize_path(std::begin(src), std::end(src))); + CU_ASSERT("/%2F" == + http2::normalize_path(StringRef::from_lit("%2f"), StringRef{})); - src = "%f"; - CU_ASSERT("/%f" == http2::normalize_path(std::begin(src), std::end(src))); + CU_ASSERT("/%f" == + http2::normalize_path(StringRef::from_lit("%f"), StringRef{})); - src = "%"; - CU_ASSERT("/%" == http2::normalize_path(std::begin(src), std::end(src))); + CU_ASSERT("/%" == + http2::normalize_path(StringRef::from_lit("%"), StringRef{})); - src = ""; - CU_ASSERT("/" == http2::normalize_path(std::begin(src), std::end(src))); + CU_ASSERT("/" == http2::normalize_path(StringRef{}, StringRef{})); + + CU_ASSERT("/alpha?bravo" == + http2::normalize_path(StringRef::from_lit("/alpha"), + StringRef::from_lit("bravo"))); } void test_http2_rewrite_clean_path(void) { - std::string src; + BlockAllocator balloc(4096, 4096); // unreserved characters - src = "/alpha/%62ravo/"; CU_ASSERT("/alpha/bravo/" == - http2::rewrite_clean_path(std::begin(src), std::end(src))); + http2::rewrite_clean_path(balloc, + StringRef::from_lit("/alpha/%62ravo/"))); // percent-encoding is converted to upper case. - src = "/delta%3a"; - CU_ASSERT("/delta%3A" == - http2::rewrite_clean_path(std::begin(src), std::end(src))); + CU_ASSERT("/delta%3A" == http2::rewrite_clean_path( + balloc, StringRef::from_lit("/delta%3a"))); // path component is normalized before mathcing - src = "/alpha/charlie/%2e././bravo/delta/.."; - CU_ASSERT("/alpha/bravo/" == - http2::rewrite_clean_path(std::begin(src), std::end(src))); + CU_ASSERT( + "/alpha/bravo/" == + http2::rewrite_clean_path( + balloc, StringRef::from_lit("/alpha/charlie/%2e././bravo/delta/.."))); - src = "alpha%3a"; - CU_ASSERT(src == http2::rewrite_clean_path(std::begin(src), std::end(src))); + CU_ASSERT("alpha%3a" == + http2::rewrite_clean_path(balloc, StringRef::from_lit("alpha%3a"))); - src = ""; - CU_ASSERT(src == http2::rewrite_clean_path(std::begin(src), std::end(src))); + CU_ASSERT("" == http2::rewrite_clean_path(balloc, StringRef{})); + + CU_ASSERT( + "/alpha?bravo" == + http2::rewrite_clean_path(balloc, StringRef::from_lit("//alpha?bravo"))); } void test_http2_get_pure_path_component(void) { - const char *base; - size_t len; - std::string path; + CU_ASSERT("/" == http2::get_pure_path_component(StringRef::from_lit("/"))); - path = "/"; - CU_ASSERT(0 == http2::get_pure_path_component(&base, &len, path)); - CU_ASSERT(util::streq_l("/", base, len)); + CU_ASSERT("/foo" == + http2::get_pure_path_component(StringRef::from_lit("/foo"))); - path = "/foo"; - CU_ASSERT(0 == http2::get_pure_path_component(&base, &len, path)); - CU_ASSERT(util::streq_l("/foo", base, len)); + CU_ASSERT("/bar" == http2::get_pure_path_component( + StringRef::from_lit("https://example.org/bar"))); - path = "https://example.org/bar"; - CU_ASSERT(0 == http2::get_pure_path_component(&base, &len, path)); - CU_ASSERT(util::streq_l("/bar", base, len)); + CU_ASSERT("/alpha" == http2::get_pure_path_component(StringRef::from_lit( + "https://example.org/alpha?q=a"))); - path = "https://example.org/alpha?q=a"; - CU_ASSERT(0 == http2::get_pure_path_component(&base, &len, path)); - CU_ASSERT(util::streq_l("/alpha", base, len)); + CU_ASSERT("/bravo" == http2::get_pure_path_component(StringRef::from_lit( + "https://example.org/bravo?q=a#fragment"))); - path = "https://example.org/bravo?q=a#fragment"; - CU_ASSERT(0 == http2::get_pure_path_component(&base, &len, path)); - CU_ASSERT(util::streq_l("/bravo", base, len)); - - path = "\x01\x02"; - CU_ASSERT(-1 == http2::get_pure_path_component(&base, &len, path)); + CU_ASSERT("" == + http2::get_pure_path_component(StringRef::from_lit("\x01\x02"))); } void test_http2_construct_push_component(void) { - const char *base; - size_t baselen; - std::string uri; - std::string scheme, authority, path; + BlockAllocator balloc(4096, 4096); + StringRef base, uri; + StringRef scheme, authority, path; - base = "/b/"; - baselen = 3; + base = StringRef::from_lit("/b/"); + uri = StringRef::from_lit("https://example.org/foo"); - uri = "https://example.org/foo"; - - CU_ASSERT(0 == http2::construct_push_component(scheme, authority, path, base, - baselen, uri.c_str(), - uri.size())); + CU_ASSERT(0 == http2::construct_push_component(balloc, scheme, authority, + path, base, uri)); CU_ASSERT("https" == scheme); CU_ASSERT("example.org" == authority); CU_ASSERT("/foo" == path); - scheme.clear(); - authority.clear(); - path.clear(); + scheme = StringRef{}; + authority = StringRef{}; + path = StringRef{}; - uri = "/foo/bar?q=a"; + uri = StringRef::from_lit("/foo/bar?q=a"); - CU_ASSERT(0 == http2::construct_push_component(scheme, authority, path, base, - baselen, uri.c_str(), - uri.size())); + CU_ASSERT(0 == http2::construct_push_component(balloc, scheme, authority, + path, base, uri)); CU_ASSERT("" == scheme); CU_ASSERT("" == authority); CU_ASSERT("/foo/bar?q=a" == path); - scheme.clear(); - authority.clear(); - path.clear(); + scheme = StringRef{}; + authority = StringRef{}; + path = StringRef{}; - uri = "foo/../bar?q=a"; + uri = StringRef::from_lit("foo/../bar?q=a"); - CU_ASSERT(0 == http2::construct_push_component(scheme, authority, path, base, - baselen, uri.c_str(), - uri.size())); + CU_ASSERT(0 == http2::construct_push_component(balloc, scheme, authority, + path, base, uri)); CU_ASSERT("" == scheme); CU_ASSERT("" == authority); CU_ASSERT("/b/bar?q=a" == path); - scheme.clear(); - authority.clear(); - path.clear(); + scheme = StringRef{}; + authority = StringRef{}; + path = StringRef{}; - uri = ""; + uri = StringRef{}; - CU_ASSERT(0 == http2::construct_push_component(scheme, authority, path, base, - baselen, uri.c_str(), - uri.size())); + CU_ASSERT(0 == http2::construct_push_component(balloc, scheme, authority, + path, base, uri)); CU_ASSERT("" == scheme); CU_ASSERT("" == authority); CU_ASSERT("/" == path); - scheme.clear(); - authority.clear(); - path.clear(); + scheme = StringRef{}; + authority = StringRef{}; + path = StringRef{}; - uri = "?q=a"; + uri = StringRef::from_lit("?q=a"); - CU_ASSERT(0 == http2::construct_push_component(scheme, authority, path, base, - baselen, uri.c_str(), - uri.size())); + CU_ASSERT(0 == http2::construct_push_component(balloc, scheme, authority, + path, base, uri)); CU_ASSERT("" == scheme); CU_ASSERT("" == authority); CU_ASSERT("/b/?q=a" == path); diff --git a/src/http2_test.h b/src/http2_test.h index 80f14cd8..029e8e6f 100644 --- a/src/http2_test.h +++ b/src/http2_test.h @@ -40,9 +40,6 @@ void test_http2_rewrite_location_uri(void); void test_http2_parse_http_status_code(void); void test_http2_index_header(void); void test_http2_lookup_token(void); -void test_http2_check_http2_pseudo_header(void); -void test_http2_http2_header_allowed(void); -void test_http2_mandatory_request_headers_presence(void); void test_http2_parse_link_header(void); void test_http2_path_join(void); void test_http2_normalize_path(void); diff --git a/src/network.h b/src/network.h new file mode 100644 index 00000000..3fb765d0 --- /dev/null +++ b/src/network.h @@ -0,0 +1,61 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef NETWORK_H +#define NETWORK_H + +#ifdef HAVE_CONFIG_H +#include +#endif // HAVE_CONFIG_H + +#include +#ifdef HAVE_SYS_SOCKET_H +#include +#endif // HAVE_SYS_SOCKET_H +#include +#ifdef HAVE_NETINET_IN_H +#include +#endif // HAVE_NETINET_IN_H +#ifdef HAVE_ARPA_INET_H +#include +#endif // HAVE_ARPA_INET_H + +namespace nghttp2 { + +union sockaddr_union { + sockaddr_storage storage; + sockaddr sa; + sockaddr_in6 in6; + sockaddr_in in; + sockaddr_un un; +}; + +struct Address { + size_t len; + union sockaddr_union su; +}; + +} // namespace nghttp2 + +#endif // NETWORK_H diff --git a/src/nghttp.cc b/src/nghttp.cc index e3d4e110..d219d1da 100644 --- a/src/nghttp.cc +++ b/src/nghttp.cc @@ -187,7 +187,7 @@ int Request::update_html_parser(const uint8_t *data, size_t len, int fin) { std::string Request::make_reqpath() const { std::string path = util::has_uri_field(u, UF_PATH) - ? util::get_uri_field(uri.c_str(), u, UF_PATH) + ? util::get_uri_field(uri.c_str(), u, UF_PATH).str() : "/"; if (util::has_uri_field(u, UF_QUERY)) { path += '?'; @@ -200,21 +200,20 @@ std::string Request::make_reqpath() const { namespace { // Perform special handling |host| if it is IPv6 literal and includes // zone ID per RFC 6874. -std::string decode_host(std::string host) { +std::string decode_host(const StringRef &host) { auto zone_start = std::find(std::begin(host), std::end(host), '%'); if (zone_start == std::end(host) || !util::ipv6_numeric_addr( std::string(std::begin(host), zone_start).c_str())) { - return host; + return host.str(); } // case: ::1% if (zone_start + 1 == std::end(host)) { - host.pop_back(); - return host; + return StringRef{host.c_str(), host.size() - 1}.str(); } // case: ::1%12 or ::1%1 if (zone_start + 3 >= std::end(host)) { - return host; + return host.str(); } // If we see "%25", followed by more characters, then decode %25 as // '%'. @@ -222,9 +221,9 @@ std::string decode_host(std::string host) { ? zone_start + 3 : zone_start + 1; auto zone_id = util::percent_decode(zone_id_src, std::end(host)); - host.erase(zone_start + 1, std::end(host)); - host += zone_id; - return host; + auto res = std::string(std::begin(host), zone_start + 1); + res += zone_id; + return res; } } // namespace @@ -273,34 +272,7 @@ bool Request::is_ipv6_literal_addr() const { } } -bool Request::response_pseudo_header_allowed(int16_t token) const { - if (!res_nva.empty() && res_nva.back().name.c_str()[0] != ':') { - return false; - } - switch (token) { - case http2::HD__STATUS: - return res_hdidx[token] == -1; - default: - return false; - } -} - -bool Request::push_request_pseudo_header_allowed(int16_t token) const { - if (!req_nva.empty() && req_nva.back().name.c_str()[0] != ':') { - return false; - } - switch (token) { - case http2::HD__AUTHORITY: - case http2::HD__METHOD: - case http2::HD__PATH: - case http2::HD__SCHEME: - return req_hdidx[token] == -1; - default: - return false; - } -} - -Headers::value_type *Request::get_res_header(int16_t token) { +Headers::value_type *Request::get_res_header(int32_t token) { auto idx = res_hdidx[token]; if (idx == -1) { return nullptr; @@ -308,7 +280,7 @@ Headers::value_type *Request::get_res_header(int16_t token) { return &res_nva[idx]; } -Headers::value_type *Request::get_req_header(int16_t token) { +Headers::value_type *Request::get_req_header(int32_t token) { auto idx = req_hdidx[token]; if (idx == -1) { return nullptr; @@ -376,7 +348,7 @@ int submit_request(HttpClient *client, const Headers &headers, Request *req) { auto scheme = util::get_uri_field(req->uri.c_str(), req->u, UF_SCHEMA); auto build_headers = Headers{{":method", req->data_prd ? "POST" : "GET"}, {":path", path}, - {":scheme", scheme}, + {":scheme", scheme.str()}, {":authority", client->hostport}, {"accept", "*/*"}, {"accept-encoding", "gzip, deflate"}, @@ -1282,7 +1254,8 @@ void HttpClient::update_hostport() { if (reqvec.empty()) { return; } - scheme = util::get_uri_field(reqvec[0]->uri.c_str(), reqvec[0]->u, UF_SCHEMA); + scheme = util::get_uri_field(reqvec[0]->uri.c_str(), reqvec[0]->u, UF_SCHEMA) + .str(); std::stringstream ss; if (reqvec[0]->is_ipv6_literal_addr()) { // we may have zone ID, which must start with "%25", or "%". RFC @@ -1627,7 +1600,7 @@ void check_response_header(nghttp2_session *session, Request *req) { return; } - auto status = http2::parse_http_status_code(status_hd->value); + auto status = http2::parse_http_status_code(StringRef{status_hd->value}); if (status == -1) { nghttp2_submit_rst_stream(session, NGHTTP2_FLAG_NONE, req->stream_id, NGHTTP2_PROTOCOL_ERROR); @@ -2285,6 +2258,9 @@ int run(char **uris, int n) { nghttp2_session_callbacks_set_on_invalid_frame_recv_callback( callbacks, verbose_on_invalid_frame_recv_callback); + + nghttp2_session_callbacks_set_error_callback(callbacks, + verbose_error_callback); } nghttp2_session_callbacks_set_on_data_chunk_recv_callback( @@ -2408,7 +2384,7 @@ int run(char **uris, int n) { } requests.clear(); } - prev_scheme = util::get_uri_field(uri.c_str(), u, UF_SCHEMA); + prev_scheme = util::get_uri_field(uri.c_str(), u, UF_SCHEMA).str(); prev_host = std::move(host); prev_port = port; } diff --git a/src/nghttp.h b/src/nghttp.h index 715a110e..df4ca685 100644 --- a/src/nghttp.h +++ b/src/nghttp.h @@ -125,11 +125,8 @@ struct Request { bool is_ipv6_literal_addr() const; - bool response_pseudo_header_allowed(int16_t token) const; - bool push_request_pseudo_header_allowed(int16_t token) const; - - Headers::value_type *get_res_header(int16_t token); - Headers::value_type *get_req_header(int16_t token); + Headers::value_type *get_res_header(int32_t token); + Headers::value_type *get_req_header(int32_t token); void record_request_start_time(); void record_response_start_time(); diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index 4c6b39c8..6a015778 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -33,6 +33,7 @@ #include "shrpx_ssl_test.h" #include "shrpx_downstream_test.h" #include "shrpx_config_test.h" +#include "shrpx_worker_test.h" #include "http2_test.h" #include "util_test.h" #include "nghttp2_gzip_test.h" @@ -89,12 +90,6 @@ int main(int argc, char *argv[]) { shrpx::test_http2_index_header) || !CU_add_test(pSuite, "http2_lookup_token", shrpx::test_http2_lookup_token) || - !CU_add_test(pSuite, "http2_check_http2_pseudo_header", - shrpx::test_http2_check_http2_pseudo_header) || - !CU_add_test(pSuite, "http2_http2_header_allowed", - shrpx::test_http2_http2_header_allowed) || - !CU_add_test(pSuite, "http2_mandatory_request_headers_presence", - shrpx::test_http2_mandatory_request_headers_presence) || !CU_add_test(pSuite, "http2_parse_link_header", shrpx::test_http2_parse_link_header) || !CU_add_test(pSuite, "http2_path_join", shrpx::test_http2_path_join) || @@ -106,8 +101,8 @@ int main(int argc, char *argv[]) { shrpx::test_http2_get_pure_path_component) || !CU_add_test(pSuite, "http2_construct_push_component", shrpx::test_http2_construct_push_component) || - !CU_add_test(pSuite, "downstream_field_store_index_headers", - shrpx::test_downstream_field_store_index_headers) || + !CU_add_test(pSuite, "downstream_field_store_append_last_header", + shrpx::test_downstream_field_store_append_last_header) || !CU_add_test(pSuite, "downstream_field_store_header", shrpx::test_downstream_field_store_header) || !CU_add_test(pSuite, "downstream_crumble_request_cookie", @@ -124,10 +119,12 @@ int main(int argc, char *argv[]) { shrpx::test_shrpx_config_read_tls_ticket_key_file) || !CU_add_test(pSuite, "config_read_tls_ticket_key_file_aes_256", shrpx::test_shrpx_config_read_tls_ticket_key_file_aes_256) || - !CU_add_test(pSuite, "config_match_downstream_addr_group", - shrpx::test_shrpx_config_match_downstream_addr_group) || + !CU_add_test(pSuite, "worker_match_downstream_addr_group", + shrpx::test_shrpx_worker_match_downstream_addr_group) || !CU_add_test(pSuite, "http_create_forwarded", shrpx::test_shrpx_http_create_forwarded) || + !CU_add_test(pSuite, "http_create_via_header_value", + shrpx::test_shrpx_http_create_via_header_value) || !CU_add_test(pSuite, "util_streq", shrpx::test_util_streq) || !CU_add_test(pSuite, "util_strieq", shrpx::test_util_strieq) || !CU_add_test(pSuite, "util_inp_strlower", @@ -147,6 +144,9 @@ int main(int argc, char *argv[]) { !CU_add_test(pSuite, "util_select_h2", shrpx::test_util_select_h2) || !CU_add_test(pSuite, "util_ipv6_numeric_addr", shrpx::test_util_ipv6_numeric_addr) || + !CU_add_test(pSuite, "util_utos", shrpx::test_util_utos) || + !CU_add_test(pSuite, "util_make_string_ref_uint", + shrpx::test_util_make_string_ref_uint) || !CU_add_test(pSuite, "util_utos_unit", shrpx::test_util_utos_unit) || !CU_add_test(pSuite, "util_utos_funit", shrpx::test_util_utos_funit) || !CU_add_test(pSuite, "util_parse_uint_with_unit", diff --git a/src/shrpx.cc b/src/shrpx.cc index 17a6bb5a..f59c2b24 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -199,18 +199,18 @@ int chown_to_running_user(const char *path) { namespace { void save_pid() { - std::ofstream out(get_config()->pid_file.get(), std::ios::binary); + std::ofstream out(get_config()->pid_file.c_str(), std::ios::binary); out << get_config()->pid << "\n"; out.close(); if (!out) { - LOG(ERROR) << "Could not save PID to file " << get_config()->pid_file.get(); + LOG(ERROR) << "Could not save PID to file " << get_config()->pid_file; exit(EXIT_FAILURE); } if (get_config()->uid != 0) { - if (chown_to_running_user(get_config()->pid_file.get()) == -1) { + if (chown_to_running_user(get_config()->pid_file.c_str()) == -1) { auto error = errno; - LOG(WARN) << "Changing owner of pid file " << get_config()->pid_file.get() + LOG(WARN) << "Changing owner of pid file " << get_config()->pid_file << " failed: " << strerror(error); } } @@ -656,7 +656,7 @@ int create_tcp_server_socket(UpstreamAddr &faddr, } faddr.fd = fd; - faddr.hostport = util::make_http_hostport(host.data(), faddr.port); + faddr.hostport = util::make_http_hostport(StringRef{host.data()}, faddr.port); LOG(NOTICE) << "Listening on " << faddr.hostport; @@ -946,7 +946,7 @@ int event_loop() { redirect_stderr_to_errorlog(); } - if (get_config()->pid_file) { + if (!get_config()->pid_file.empty()) { save_pid(); } @@ -1040,7 +1040,7 @@ void fill_default_config() { *mod_config() = {}; mod_config()->num_worker = 1; - mod_config()->conf_path = strcopy("/etc/nghttpx/nghttpx.conf"); + mod_config()->conf_path = "/etc/nghttpx/nghttpx.conf"; mod_config()->pid = getpid(); auto &tlsconf = mod_config()->tls; @@ -1067,8 +1067,7 @@ void fill_default_config() { auto &ocspconf = tlsconf.ocsp; // ocsp update interval = 14400 secs = 4 hours, borrowed from h2o ocspconf.update_interval = 4_h; - ocspconf.fetch_ocsp_response_file = - strcopy(PKGDATADIR "/fetch-ocsp-response"); + ocspconf.fetch_ocsp_response_file = PKGDATADIR "/fetch-ocsp-response"; } { @@ -1078,10 +1077,10 @@ void fill_default_config() { } tlsconf.session_timeout = std::chrono::hours(12); - tlsconf.downstream_session_cache_per_worker = 10000; auto &httpconf = mod_config()->http; - httpconf.server_name = "nghttpx nghttp2/" NGHTTP2_VERSION; + httpconf.server_name = + StringRef::from_lit("nghttpx nghttp2/" NGHTTP2_VERSION); httpconf.no_host_rewrite = true; httpconf.request_header_field_buffer = 64_k; httpconf.max_request_header_fields = 100; @@ -1098,6 +1097,7 @@ void fill_default_config() { // HTTP/2 SPDY/3.1 has connection-level flow control. The default // window size for HTTP/2 is 64KiB - 1. SPDY/3's default is 64KiB upstreamconf.connection_window_bits = 16; + upstreamconf.max_concurrent_streams = 100; nghttp2_option_new(&upstreamconf.option); nghttp2_option_set_no_auto_window_update(upstreamconf.option, 1); @@ -1107,22 +1107,21 @@ void fill_default_config() { { auto &downstreamconf = http2conf.downstream; downstreamconf.window_bits = 16; - downstreamconf.connection_window_bits = 16; + downstreamconf.connection_window_bits = 30; + downstreamconf.max_concurrent_streams = 100; nghttp2_option_new(&downstreamconf.option); nghttp2_option_set_no_auto_window_update(downstreamconf.option, 1); nghttp2_option_set_peer_max_concurrent_streams(downstreamconf.option, 100); } - http2conf.max_concurrent_streams = 100; - auto &loggingconf = mod_config()->logging; { auto &accessconf = loggingconf.access; accessconf.format = parse_log_format(DEFAULT_ACCESSLOG_FORMAT); auto &errorconf = loggingconf.error; - errorconf.file = strcopy("/dev/stderr"); + errorconf.file = "/dev/stderr"; } loggingconf.syslog_facility = LOG_DAEMON; @@ -1167,6 +1166,7 @@ void fill_default_config() { downstreamconf.request_buffer_size = 16_k; downstreamconf.response_buffer_size = 128_k; downstreamconf.family = AF_UNSPEC; + downstreamconf.no_tls = true; } } @@ -1190,48 +1190,55 @@ void print_help(std::ostream &out) { print_usage(out); out << R"( - Set path to server's private key. Required unless -p, - --client or --frontend-no-tls are given. - Set path to server's certificate. Required unless -p, - --client or --frontend-no-tls are given. To make OCSP - stapling work, this must be absolute path. + Set path to server's private key. Required unless + --frontend-no-tls are given. + Set path to server's certificate. Required unless + --frontend-no-tls are given. To make OCSP stapling + work, this must be an absolute path. Options: The options are categorized into several groups. Connections: - -b, --backend=(,|unix:)[;[:...]] + -b, --backend=(,|unix:)[;[[:...]][;proto=]] Set backend host and port. The multiple backend addresses are accepted by repeating this option. UNIX domain socket can be specified by prefixing path name with "unix:" (e.g., unix:/var/run/backend.sock). Optionally, if s are given, the backend address - is only used if request matches the pattern. If -s or - -p is used, s are ignored. The pattern - matching is closely designed to ServeMux in net/http - package of Go programming language. consists - of path, host + path or just host. The path must start - with "/". If it ends with "/", it matches all request - path in its subtree. To deal with the request to the - directory without trailing slash, the path which ends - with "/" also matches the request path which only lacks - trailing '/' (e.g., path "/foo/" matches request path - "/foo"). If it does not end with "/", it performs exact - match against the request path. If host is given, it - performs exact match against the request host. If host - alone is given, "/" is appended to it, so that it - matches all request paths under the host (e.g., - specifying "nghttp2.org" equals to "nghttp2.org/"). + is only used if request matches the pattern. If + --http2-proxy is used, s are ignored. The + pattern matching is closely designed to ServeMux in + net/http package of Go programming language. + consists of path, host + path or just host. The path + must start with "/". If it ends with "/", it matches + all request path in its subtree. To deal with the + request to the directory without trailing slash, the + path which ends with "/" also matches the request path + which only lacks trailing '/' (e.g., path "/foo/" + matches request path "/foo"). If it does not end with + "/", it performs exact match against the request path. + If host is given, it performs exact match against the + request host. If host alone is given, "/" is appended + to it, so that it matches all request paths under the + host (e.g., specifying "nghttp2.org" equals to + "nghttp2.org/"). Patterns with host take precedence over patterns with just path. Then, longer patterns take precedence over - shorter ones, breaking a tie by the order of the - appearance in the configuration. + shorter ones. - If is omitted, "/" is used as pattern, which - matches all request paths (catch-all pattern). The - catch-all backend must be given. + Host can include "*" in the left most position to + indicate wildcard match (only suffix match is done). + For example, host pattern "*www.nghttp2.org" matches + against "www.nghttp2.org" and "1www.ngttp2.org", but + does not match against "nghttp2.org". The exact hosts + match takes precedence over the wildcard hosts match. + + If is omitted or empty string, "/" is used as + pattern, which matches all request paths (catch-all + pattern). The catch-all backend must be given. When doing a match, nghttpx made some normalization to pattern, request host and path. For host part, they are @@ -1254,6 +1261,15 @@ Connections: The backend addresses sharing same are grouped together forming load balancing group. + Optionally, backend application protocol can be + specified in . All that share the same + must have the same value if it is given. + should be one of the following list without + quotes: "h2", "http/1.1". The default value of + is "http/1.1". Note that usually "h2" refers to HTTP/2 + over TLS. But in this option, it may mean HTTP/2 over + cleartext TCP unless --backend-tls is used. + Since ";" and ":" are used as delimiter, must not contain these characters. Since ";" has special meaning in shell, the option value must be quoted. @@ -1292,16 +1308,8 @@ Connections: --backend-write-timeout options. --accept-proxy-protocol Accept PROXY protocol version 1 on frontend connection. - --backend-no-tls - Disable SSL/TLS on backend connections. For HTTP/2 - backend connections, TLS is enabled by default. For - HTTP/1 backend connections, TLS is disabled by default, - and can be enabled by --backend-http1-tls option. If - both --backend-no-tls and --backend-http1-tls options - are used, --backend-no-tls has the precedence. - --backend-http1-tls - Enable SSL/TLS on backend HTTP/1 connections. See also - --backend-no-tls option. + --backend-tls + Enable SSL/TLS on backend connections. Performance: -n, --workers= @@ -1354,31 +1362,24 @@ Performance: accepts. Setting 0 means unlimited. Default: )" << get_config()->conn.upstream.worker_connections << R"( - --backend-http2-connections-per-worker= - Set maximum number of backend HTTP/2 physical - connections per worker. If pattern is used in -b - option, this limit is applied to each pattern group (in - other words, each pattern group can have maximum - HTTP/2 connections). The default value is 0, which - means that the value is adjusted to the number of - backend addresses. If pattern is used, this adjustment - is done for each pattern group. - --backend-http1-connections-per-host= - Set maximum number of backend concurrent HTTP/1 - connections per origin host. This option is meaningful - when -s option is used. The origin host is determined - by authority portion of request URI (or :authority - header field for HTTP/2). To limit the number of - connections per frontend for default mode, use - --backend-http1-connections-per-frontend. + --backend-connections-per-host= + Set maximum number of backend concurrent connections + (and/or streams in case of HTTP/2) per origin host. + This option is meaningful when --http2-proxy option is + used. The origin host is determined by authority + portion of request URI (or :authority header field for + HTTP/2). To limit the number of connections per + frontend for default mode, use + --backend-connections-per-frontend. Default: )" << get_config()->conn.downstream.connections_per_host << R"( - --backend-http1-connections-per-frontend= - Set maximum number of backend concurrent HTTP/1 - connections per frontend. This option is only used for - default mode. 0 means unlimited. To limit the number - of connections per host for HTTP/2 or SPDY proxy mode - (-s option), use --backend-http1-connections-per-host. + --backend-connections-per-frontend= + Set maximum number of backend concurrent connections + (and/or streams in case of HTTP/2) per frontend. This + option is only used for default mode. 0 means + unlimited. To limit the number of connections per host + with --http2-proxy option, use + --backend-connections-per-host. Default: )" << get_config()->conn.downstream.connections_per_frontend << R"( --rlimit-nofile= @@ -1579,8 +1580,8 @@ SSL/TLS: --fetch-ocsp-response-file= Path to fetch-ocsp-response script file. It should be absolute path. - Default: )" - << get_config()->tls.ocsp.fetch_ocsp_response_file.get() << R"( + Default: )" << get_config()->tls.ocsp.fetch_ocsp_response_file + << R"( --ocsp-update-interval= Set interval to update OCSP response cache. Default: )" @@ -1630,17 +1631,20 @@ SSL/TLS: Allow black listed cipher suite on HTTP/2 connection. See https://tools.ietf.org/html/rfc7540#appendix-A for the complete HTTP/2 cipher suites black list. - --backend-tls-session-cache-per-worker= - Set the maximum number of backend TLS session cache - stored per worker. - Default: )" - << get_config()->tls.downstream_session_cache_per_worker << R"( HTTP/2 and SPDY: - -c, --http2-max-concurrent-streams= + -c, --frontend-http2-max-concurrent-streams= Set the maximum number of the concurrent streams in one - HTTP/2 and SPDY session. - Default: )" << get_config()->http2.max_concurrent_streams << R"( + frontend HTTP/2 and SPDY session. + Default: )" + << get_config()->http2.upstream.max_concurrent_streams << R"( + --backend-http2-max-concurrent-streams= + Set the maximum number of the concurrent streams in one + backend HTTP/2 session. This sets maximum number of + concurrent opened pushed streams. The maximum number of + concurrent requests are set by a remote server. + Default: )" + << get_config()->http2.downstream.max_concurrent_streams << R"( --frontend-http2-window-bits= Sets the per-stream initial window size of HTTP/2 SPDY frontend connection. For HTTP/2, the size is 2**-1. @@ -1674,37 +1678,20 @@ HTTP/2 and SPDY: Disable HTTP/2 server push. Server push is supported by default mode and HTTP/2 frontend via Link header field. It is also supported if both frontend and backend are - HTTP/2 (which implies --http2-bridge or --client mode). - In this case, server push from backend session is - relayed to frontend, and server push via Link header - field is also supported. HTTP SPDY frontend does not - support server push. + HTTP/2 in default mode. In this case, server push from + backend session is relayed to frontend, and server push + via Link header field is also supported. SPDY frontend + does not support server push. Mode: (default mode) Accept HTTP/2, SPDY and HTTP/1.1 over SSL/TLS. If --frontend-no-tls is used, accept HTTP/2 and HTTP/1.1. The incoming HTTP/1.1 connection can be upgraded to - HTTP/2 through HTTP Upgrade. The protocol to the - backend is HTTP/1.1. + HTTP/2 through HTTP Upgrade. -s, --http2-proxy - Like default mode, but enable secure proxy mode. - --http2-bridge - Like default mode, but communicate with the backend in - HTTP/2 over SSL/TLS. Thus the incoming all connections - are converted to HTTP/2 connection and relayed to the - backend. See --backend-http-proxy-uri option if you are - behind the proxy and want to connect to the outside - HTTP/2 proxy. - --client Accept HTTP/2 and HTTP/1.1 without SSL/TLS. The - incoming HTTP/1.1 connection can be upgraded to HTTP/2 - connection through HTTP Upgrade. The protocol to the - backend is HTTP/2. To use nghttpx as a forward proxy, - use -p option instead. - -p, --client-proxy - Like --client option, but it also requires the request - path from frontend must be an absolute URI, suitable for - use as a forward proxy. + Like default mode, but enable forward proxy. This is so + called HTTP/2 proxy mode. Logging: -L, --log-level= @@ -1753,7 +1740,7 @@ Logging: Set path to write error log. To reopen file, send USR1 signal to nghttpx. stderr will be redirected to the error log file unless --errorlog-syslog is used. - Default: )" << get_config()->logging.error.file.get() << R"( + Default: )" << get_config()->logging.error.file << R"( --errorlog-syslog Send error log to syslog. If this option is used, --errorlog-file option is ignored. @@ -1805,15 +1792,13 @@ HTTP: --no-via Don't append to Via header field. If Via header field is received, it is left unaltered. --no-location-rewrite - Don't rewrite location header field on --http2-bridge, - --client and default mode. For --http2-proxy and - --client-proxy mode, location header field will not be - altered regardless of this option. + Don't rewrite location header field in default mode. + When --http2-proxy is used, location header field will + not be altered regardless of this option. --host-rewrite - Rewrite host and :authority header fields on - --http2-bridge, --client and default mode. For - --http2-proxy and --client-proxy mode, these headers - will not be altered regardless of this option. + Rewrite host and :authority header fields in default + mode. When --http2-proxy is used, these headers will + not be altered regardless of this option. --altsvc= Specify protocol ID, port, host and origin of alternative service. and are optional. @@ -1894,7 +1879,7 @@ Scripting: Misc: --conf= Load configuration from . - Default: )" << get_config()->conf_path.get() << R"( + Default: )" << get_config()->conf_path << R"( --include= Load additional configurations from . File is read when configuration parser encountered this @@ -1920,11 +1905,11 @@ namespace { void process_options( int argc, char **argv, std::vector> &cmdcfgs) { - if (conf_exists(get_config()->conf_path.get())) { + if (conf_exists(get_config()->conf_path.c_str())) { std::set include_set; - if (load_config(get_config()->conf_path.get(), include_set) == -1) { + if (load_config(get_config()->conf_path.c_str(), include_set) == -1) { LOG(FATAL) << "Failed to load configuration from " - << get_config()->conf_path.get(); + << get_config()->conf_path; exit(EXIT_FAILURE); } assert(include_set.empty()); @@ -1987,8 +1972,8 @@ void process_options( { auto &dumpconf = http2conf.upstream.debug.dump; - if (dumpconf.request_header_file) { - auto path = dumpconf.request_header_file.get(); + if (!dumpconf.request_header_file.empty()) { + auto path = dumpconf.request_header_file.c_str(); auto f = open_file_for_write(path); if (f == nullptr) { @@ -2008,8 +1993,8 @@ void process_options( } } - if (dumpconf.response_header_file) { - auto path = dumpconf.response_header_file.get(); + if (!dumpconf.response_header_file.empty()) { + auto path = dumpconf.response_header_file.c_str(); auto f = open_file_for_write(path); if (f == nullptr) { @@ -2062,31 +2047,8 @@ void process_options( upstreamconf.worker_connections = std::numeric_limits::max(); } - if (get_config()->http2_proxy + get_config()->http2_bridge + - get_config()->client_proxy + get_config()->client > - 1) { - LOG(FATAL) << "--http2-proxy, --http2-bridge, --client-proxy and --client " - << "cannot be used at the same time."; - exit(EXIT_FAILURE); - } - - if (get_config()->client || get_config()->client_proxy) { - mod_config()->client_mode = true; - upstreamconf.no_tls = true; - } - - if (get_config()->client_mode || get_config()->http2_bridge) { - downstreamconf.proto = PROTO_HTTP2; - } else { - downstreamconf.proto = PROTO_HTTP; - } - - if (downstreamconf.proto == PROTO_HTTP && !downstreamconf.http1_tls) { - downstreamconf.no_tls = true; - } - if (!upstreamconf.no_tls && - (!tlsconf.private_key_file || !tlsconf.cert_file)) { + (tlsconf.private_key_file.empty() || tlsconf.cert_file.empty())) { print_usage(std::cerr); LOG(FATAL) << "Too few arguments"; exit(EXIT_FAILURE); @@ -2094,10 +2056,10 @@ void process_options( if (!upstreamconf.no_tls && !tlsconf.ocsp.disabled) { struct stat buf; - if (stat(tlsconf.ocsp.fetch_ocsp_response_file.get(), &buf) != 0) { + if (stat(tlsconf.ocsp.fetch_ocsp_response_file.c_str(), &buf) != 0) { tlsconf.ocsp.disabled = true; LOG(WARN) << "--fetch-ocsp-response-file: " - << tlsconf.ocsp.fetch_ocsp_response_file.get() + << tlsconf.ocsp.fetch_ocsp_response_file << " not found. OCSP stapling has been disabled."; } } @@ -2105,28 +2067,55 @@ void process_options( auto &addr_groups = downstreamconf.addr_groups; if (addr_groups.empty()) { - DownstreamAddr addr; + DownstreamAddrConfig addr{}; addr.host = ImmutableString::from_lit(DEFAULT_DOWNSTREAM_HOST); addr.port = DEFAULT_DOWNSTREAM_PORT; - DownstreamAddrGroup g("/"); + DownstreamAddrGroupConfig g(StringRef::from_lit("/")); + g.proto = PROTO_HTTP1; g.addrs.push_back(std::move(addr)); - mod_config()->router.add_route(g.pattern.get(), 1, addr_groups.size()); + mod_config()->router.add_route(StringRef{g.pattern}, addr_groups.size()); addr_groups.push_back(std::move(g)); - } else if (get_config()->http2_proxy || get_config()->client_proxy) { + } else if (get_config()->http2_proxy) { // We don't support host mapping in these cases. Move all // non-catch-all patterns to catch-all pattern. - DownstreamAddrGroup catch_all("/"); + DownstreamAddrGroupConfig catch_all(StringRef::from_lit("/")); + auto proto = PROTO_NONE; for (auto &g : addr_groups) { + if (proto == PROTO_NONE) { + proto = g.proto; + } else if (proto != g.proto) { + LOG(ERROR) << SHRPX_OPT_BACKEND << ": was ignored with " + "--http2-proxy, and protocol must " + "be the same for all backends."; + exit(EXIT_FAILURE); + } std::move(std::begin(g.addrs), std::end(g.addrs), std::back_inserter(catch_all.addrs)); } - std::vector().swap(addr_groups); + catch_all.proto = proto; + std::vector().swap(addr_groups); + std::vector().swap(mod_config()->wildcard_patterns); // maybe not necessary? mod_config()->router = Router(); - mod_config()->router.add_route(catch_all.pattern.get(), 1, + mod_config()->router.add_route(StringRef{catch_all.pattern}, addr_groups.size()); addr_groups.push_back(std::move(catch_all)); + } else { + auto &wildcard_patterns = mod_config()->wildcard_patterns; + std::sort(std::begin(wildcard_patterns), std::end(wildcard_patterns), + [](const WildcardPattern &lhs, const WildcardPattern &rhs) { + return std::lexicographical_compare( + rhs.host.rbegin(), rhs.host.rend(), lhs.host.rbegin(), + lhs.host.rend()); + }); + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Reverse sorted wildcard hosts (compared from tail to head, " + "and sorted in reverse order):"; + for (auto &wp : mod_config()->wildcard_patterns) { + LOG(INFO) << wp.host; + } + } } if (LOG_ENABLED(INFO)) { @@ -2136,12 +2125,12 @@ void process_options( ssize_t catch_all_group = -1; for (size_t i = 0; i < addr_groups.size(); ++i) { auto &g = addr_groups[i]; - if (util::streq(g.pattern.get(), "/")) { + if (g.pattern == "/") { catch_all_group = i; } if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Host-path pattern: group " << i << ": '" << g.pattern.get() - << "'"; + LOG(INFO) << "Host-path pattern: group " << i << ": '" << g.pattern + << "', proto=" << strproto(g.proto); for (auto &addr : g.addrs) { LOG(INFO) << "group " << i << " -> " << addr.host.c_str() << (addr.host_unix ? "" : ":" + util::utos(addr.port)); @@ -2150,7 +2139,7 @@ void process_options( } if (catch_all_group == -1) { - LOG(FATAL) << "-b: No catch-all backend address is configured"; + LOG(FATAL) << "backend: No catch-all backend address is configured"; exit(EXIT_FAILURE); } @@ -2194,7 +2183,7 @@ void process_options( addr.hostport = ImmutableString( util::make_http_hostport(StringRef(addr.host), addr.port)); - auto hostport = util::make_hostport(addr.host.c_str(), addr.port); + auto hostport = util::make_hostport(StringRef{addr.host}, addr.port); if (resolve_hostname(&addr.addr, addr.host.c_str(), addr.port, downstreamconf.family) == -1) { @@ -2202,28 +2191,28 @@ void process_options( exit(EXIT_FAILURE); } LOG(NOTICE) << "Resolved backend address: " << hostport << " -> " - << util::numeric_hostport(&addr.addr.su.sa, addr.addr.len); + << util::to_numeric_addr(&addr.addr); } } auto &proxy = mod_config()->downstream_http_proxy; if (!proxy.host.empty()) { - auto hostport = util::make_hostport(proxy.host.c_str(), proxy.port); + auto hostport = util::make_hostport(StringRef{proxy.host}, proxy.port); if (resolve_hostname(&proxy.addr, proxy.host.c_str(), proxy.port, AF_UNSPEC) == -1) { LOG(FATAL) << "Resolving backend HTTP proxy address failed: " << hostport; exit(EXIT_FAILURE); } LOG(NOTICE) << "Backend HTTP proxy address: " << hostport << " -> " - << util::numeric_hostport(&proxy.addr.su.sa, proxy.addr.len); + << util::to_numeric_addr(&proxy.addr); } { auto &memcachedconf = tlsconf.session_cache.memcached; - if (memcachedconf.host) { - auto hostport = - util::make_hostport(memcachedconf.host.get(), memcachedconf.port); - if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.get(), + if (!memcachedconf.host.empty()) { + auto hostport = util::make_hostport(StringRef{memcachedconf.host}, + memcachedconf.port); + if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.c_str(), memcachedconf.port, memcachedconf.family) == -1) { LOG(FATAL) << "Resolving memcached address for TLS session cache failed: " @@ -2231,25 +2220,23 @@ void process_options( exit(EXIT_FAILURE); } LOG(NOTICE) << "Memcached address for TLS session cache: " << hostport - << " -> " << util::numeric_hostport(&memcachedconf.addr.su.sa, - memcachedconf.addr.len); + << " -> " << util::to_numeric_addr(&memcachedconf.addr); } } { auto &memcachedconf = tlsconf.ticket.memcached; - if (memcachedconf.host) { - auto hostport = - util::make_hostport(memcachedconf.host.get(), memcachedconf.port); - if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.get(), + if (!memcachedconf.host.empty()) { + auto hostport = util::make_hostport(StringRef{memcachedconf.host}, + memcachedconf.port); + if (resolve_hostname(&memcachedconf.addr, memcachedconf.host.c_str(), memcachedconf.port, memcachedconf.family) == -1) { LOG(FATAL) << "Resolving memcached address for TLS ticket key failed: " << hostport; exit(EXIT_FAILURE); } LOG(NOTICE) << "Memcached address for TLS ticket key: " << hostport - << " -> " << util::numeric_hostport(&memcachedconf.addr.su.sa, - memcachedconf.addr.len); + << " -> " << util::to_numeric_addr(&memcachedconf.addr); } } @@ -2454,8 +2441,6 @@ int main(int argc, char **argv) { {SHRPX_OPT_REQUEST_HEADER_FIELD_BUFFER, required_argument, &flag, 104}, {SHRPX_OPT_MAX_REQUEST_HEADER_FIELDS, required_argument, &flag, 105}, {SHRPX_OPT_BACKEND_HTTP1_TLS, no_argument, &flag, 106}, - {SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER, required_argument, - &flag, 107}, {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_TLS, no_argument, &flag, 108}, {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE, required_argument, &flag, 109}, @@ -2471,6 +2456,14 @@ int main(int argc, char **argv) { {SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_ADDRESS_FAMILY, required_argument, &flag, 115}, {SHRPX_OPT_BACKEND_ADDRESS_FAMILY, required_argument, &flag, 116}, + {SHRPX_OPT_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS, required_argument, + &flag, 117}, + {SHRPX_OPT_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS, required_argument, + &flag, 118}, + {SHRPX_OPT_BACKEND_CONNECTIONS_PER_FRONTEND, required_argument, &flag, + 119}, + {SHRPX_OPT_BACKEND_TLS, no_argument, &flag, 120}, + {SHRPX_OPT_BACKEND_CONNECTIONS_PER_HOST, required_argument, &flag, 121}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2564,7 +2557,7 @@ int main(int argc, char **argv) { break; case 12: // --conf - mod_config()->conf_path = strcopy(optarg); + mod_config()->conf_path = optarg; break; case 14: // --syslog-facility @@ -2924,11 +2917,6 @@ int main(int argc, char **argv) { // --backend-http1-tls cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_HTTP1_TLS, "yes"); break; - case 107: - // --backend-tls-session-cache-per-worker - cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER, - optarg); - break; case 108: // --tls-session-cache-memcached-tls cmdcfgs.emplace_back(SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_TLS, "yes"); @@ -2971,6 +2959,29 @@ int main(int argc, char **argv) { // --backend-address-family cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_ADDRESS_FAMILY, optarg); break; + case 117: + // --frontend-http2-max-concurrent-streams + cmdcfgs.emplace_back(SHRPX_OPT_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS, + optarg); + break; + case 118: + // --backend-http2-max-concurrent-streams + cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS, + optarg); + break; + case 119: + // --backend-connections-per-frontend + cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_CONNECTIONS_PER_FRONTEND, + optarg); + break; + case 120: + // --backend-tls + cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_TLS, "yes"); + break; + case 121: + // --backend-connections-per-host + cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_CONNECTIONS_PER_HOST, optarg); + break; default: break; } diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index fa814dd9..2588264f 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -114,8 +114,6 @@ void writecb(struct ev_loop *loop, ev_io *w, int revents) { int ClientHandler::noop() { return 0; } int ClientHandler::read_clear() { - ev_timer_again(conn_.loop, &conn_.rt); - for (;;) { if (rb_.rleft() && on_read() != 0) { return -1; @@ -124,6 +122,10 @@ int ClientHandler::read_clear() { rb_.reset(); } else if (rb_.wleft() == 0) { conn_.rlimit.stopw(); + if (reset_conn_rtimer_required_) { + reset_conn_rtimer_required_ = false; + ev_timer_again(conn_.loop, &conn_.rt); + } return 0; } @@ -134,6 +136,10 @@ int ClientHandler::read_clear() { auto nread = conn_.read_clear(rb_.last, rb_.wleft()); if (nread == 0) { + if (reset_conn_rtimer_required_) { + reset_conn_rtimer_required_ = false; + ev_timer_again(conn_.loop, &conn_.rt); + } return 0; } @@ -208,8 +214,6 @@ int ClientHandler::tls_handshake() { } int ClientHandler::read_tls() { - ev_timer_again(conn_.loop, &conn_.rt); - ERR_clear_error(); for (;;) { @@ -221,6 +225,11 @@ int ClientHandler::read_tls() { rb_.reset(); } else if (rb_.wleft() == 0) { conn_.rlimit.stopw(); + if (reset_conn_rtimer_required_) { + reset_conn_rtimer_required_ = false; + ev_timer_again(conn_.loop, &conn_.rt); + } + return 0; } @@ -231,6 +240,11 @@ int ClientHandler::read_tls() { auto nread = conn_.read_tls(rb_.last, rb_.wleft()); if (nread == 0) { + if (reset_conn_rtimer_required_) { + reset_conn_rtimer_required_ = false; + ev_timer_again(conn_.loop, &conn_.rt); + } + return 0; } @@ -385,18 +399,14 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl, get_config()->conn.upstream.ratelimit.write, get_config()->conn.upstream.ratelimit.read, writecb, readcb, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold, - get_config()->tls.dyn_rec.idle_timeout), - pinned_http2sessions_( - get_config()->conn.downstream.proto == PROTO_HTTP2 - ? make_unique>( - get_config()->conn.downstream.addr_groups.size(), -1) - : nullptr), + get_config()->tls.dyn_rec.idle_timeout, PROTO_NONE), ipaddr_(ipaddr), port_(port), faddr_(faddr), worker_(worker), left_connhd_len_(NGHTTP2_CLIENT_MAGIC_LEN), - should_close_after_write_(false) { + should_close_after_write_(false), + reset_conn_rtimer_required_(false) { ++worker_->get_worker_stat()->num_connections; @@ -502,6 +512,10 @@ void ClientHandler::reset_upstream_write_timeout(ev_tstamp t) { } } +void ClientHandler::signal_reset_upstream_conn_rtimer() { + reset_conn_rtimer_required_ = true; +} + int ClientHandler::validate_next_proto() { const unsigned char *next_proto = nullptr; unsigned int next_proto_len; @@ -642,13 +656,18 @@ void ClientHandler::pool_downstream_connection( if (!dconn->poolable()) { return; } + + dconn->set_client_handler(nullptr); + + auto group = dconn->get_downstream_addr_group(); + if (LOG_ENABLED(INFO)) { CLOG(INFO, this) << "Pooling downstream connection DCONN:" << dconn.get() - << " in group " << dconn->get_group(); + << " in group " << group; } - dconn->set_client_handler(nullptr); - auto dconn_pool = worker_->get_dconn_pool(); - dconn_pool->add_downstream_connection(std::move(dconn)); + + auto &dconn_pool = group->dconn_pool; + dconn_pool.add_downstream_connection(std::move(dconn)); } void ClientHandler::remove_downstream_connection(DownstreamConnection *dconn) { @@ -656,51 +675,55 @@ void ClientHandler::remove_downstream_connection(DownstreamConnection *dconn) { CLOG(INFO, this) << "Removing downstream connection DCONN:" << dconn << " from pool"; } - auto dconn_pool = worker_->get_dconn_pool(); - dconn_pool->remove_downstream_connection(dconn); + auto &dconn_pool = dconn->get_downstream_addr_group()->dconn_pool; + dconn_pool.remove_downstream_connection(dconn); } std::unique_ptr ClientHandler::get_downstream_connection(Downstream *downstream) { - size_t group; + size_t group_idx; auto &downstreamconf = get_config()->conn.downstream; - auto &groups = downstreamconf.addr_groups; auto catch_all = downstreamconf.addr_group_catch_all; + auto &groups = worker_->get_downstream_addr_groups(); const auto &req = downstream->request(); // Fast path. If we have one group, it must be catch-all group. - // HTTP/2 and client proxy modes fall in this case. + // proxy mode falls in this case. if (groups.size() == 1) { - group = 0; + group_idx = 0; } else if (req.method == HTTP_CONNECT) { // We don't know how to treat CONNECT request in host-path // mapping. It most likely appears in proxy scenario. Since we // have dealt with proxy case already, just use catch-all group. - group = catch_all; + group_idx = catch_all; } else { auto &router = get_config()->router; + auto &wildcard_patterns = get_config()->wildcard_patterns; if (!req.authority.empty()) { - group = match_downstream_addr_group(router, req.authority, req.path, - groups, catch_all); + group_idx = + match_downstream_addr_group(router, wildcard_patterns, req.authority, + req.path, groups, catch_all); } else { auto h = req.fs.header(http2::HD_HOST); if (h) { - group = match_downstream_addr_group(router, h->value, req.path, groups, - catch_all); + group_idx = match_downstream_addr_group( + router, wildcard_patterns, h->value, req.path, groups, catch_all); } else { - group = match_downstream_addr_group(router, "", req.path, groups, - catch_all); + group_idx = + match_downstream_addr_group(router, wildcard_patterns, StringRef{}, + req.path, groups, catch_all); } } } if (LOG_ENABLED(INFO)) { - CLOG(INFO, this) << "Downstream address group: " << group; + CLOG(INFO, this) << "Downstream address group_idx: " << group_idx; } - auto dconn_pool = worker_->get_dconn_pool(); - auto dconn = dconn_pool->pop_downstream_connection(group); + auto &group = worker_->get_downstream_addr_groups()[group_idx]; + auto &dconn_pool = group.dconn_pool; + auto dconn = dconn_pool.pop_downstream_connection(); if (!dconn) { if (LOG_ENABLED(INFO)) { @@ -708,22 +731,34 @@ ClientHandler::get_downstream_connection(Downstream *downstream) { << " Create new one"; } - auto dconn_pool = worker_->get_dconn_pool(); - - if (downstreamconf.proto == PROTO_HTTP2) { - Http2Session *http2session; - auto &pinned = (*pinned_http2sessions_)[group]; - if (pinned == -1) { - http2session = worker_->next_http2_session(group); - pinned = http2session->get_index(); - } else { - auto dgrp = worker_->get_dgrp(group); - http2session = dgrp->http2sessions[pinned].get(); + if (group.proto == PROTO_HTTP2) { + if (group.http2_freelist.empty()) { + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) + << "http2_freelist is empty; create new Http2Session"; + } + auto session = make_unique( + conn_.loop, worker_->get_cl_ssl_ctx(), worker_, &group); + group.http2_freelist.append(session.release()); } - dconn = make_unique(dconn_pool, http2session); + + auto http2session = group.http2_freelist.head; + + // TODO max_concurrent_streams option must be independent from + // frontend and backend. + if (http2session->max_concurrency_reached(1)) { + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) << "Maximum streams are reached for Http2Session(" + << http2session + << "). Remove Http2Session from http2_freelist"; + } + group.http2_freelist.remove(http2session); + } + + dconn = make_unique(http2session); } else { - dconn = make_unique(dconn_pool, group, - conn_.loop, worker_); + dconn = + make_unique(&group, conn_.loop, worker_); } dconn->set_client_handler(this); return dconn; @@ -743,10 +778,6 @@ MemchunkPool *ClientHandler::get_mcpool() { return worker_->get_mcpool(); } SSL *ClientHandler::get_ssl() const { return conn_.tls.ssl; } -ConnectBlocker *ClientHandler::get_connect_blocker() const { - return worker_->get_connect_blocker(); -} - void ClientHandler::direct_http2_upgrade() { upstream_ = make_unique(this); alpn_ = NGHTTP2_CLEARTEXT_PROTO_VERSION_ID; @@ -794,11 +825,11 @@ int ClientHandler::perform_http2_upgrade(HttpsUpstream *http) { bool ClientHandler::get_http2_upgrade_allowed() const { return !conn_.tls.ssl; } -std::string ClientHandler::get_upstream_scheme() const { +StringRef ClientHandler::get_upstream_scheme() const { if (conn_.tls.ssl) { - return "https"; + return StringRef::from_lit("https"); } else { - return "http"; + return StringRef::from_lit("http"); } } @@ -812,24 +843,35 @@ namespace { // is mostly same routine found in // HttpDownstreamConnection::push_request_headers(), but vastly // simplified since we only care about absolute URI. -std::string construct_absolute_request_uri(const Request &req) { +StringRef construct_absolute_request_uri(BlockAllocator &balloc, + const Request &req) { if (req.authority.empty()) { return req.path; } - std::string uri; + auto len = req.authority.size() + req.path.size(); + if (req.scheme.empty()) { + len += str_size("http://"); + } else { + len += req.scheme.size() + str_size("://"); + } + + auto iov = make_byte_ref(balloc, len + 1); + auto p = iov.base; + if (req.scheme.empty()) { // We may have to log the request which lacks scheme (e.g., // http/1.1 with origin form). - uri += "http://"; + p = util::copy_lit(p, "http://"); } else { - uri += req.scheme; - uri += "://"; + p = std::copy(std::begin(req.scheme), std::end(req.scheme), p); + p = util::copy_lit(p, "://"); } - uri += req.authority; - uri += req.path; + p = std::copy(std::begin(req.authority), std::end(req.authority), p); + p = std::copy(std::begin(req.path), std::end(req.path), p); + *p = '\0'; - return uri; + return StringRef{iov.base, p}; } } // namespace @@ -838,15 +880,17 @@ void ClientHandler::write_accesslog(Downstream *downstream) { const auto &req = downstream->request(); const auto &resp = downstream->response(); + auto &balloc = downstream->get_block_allocator(); + upstream_accesslog( get_config()->logging.access.format, LogSpec{ - downstream, StringRef(ipaddr_), http2::to_method_string(req.method), + downstream, StringRef{ipaddr_}, http2::to_method_string(req.method), req.method == HTTP_CONNECT ? StringRef(req.authority) - : (get_config()->http2_proxy || get_config()->client_proxy) - ? StringRef(construct_absolute_request_uri(req)) + : get_config()->http2_proxy + ? StringRef(construct_absolute_request_uri(balloc, req)) : req.path.empty() ? req.method == HTTP_OPTIONS ? StringRef::from_lit("*") @@ -1126,18 +1170,18 @@ int ClientHandler::proxy_protocol_read() { return on_proxy_protocol_finish(); } -StringRef ClientHandler::get_forwarded_by() { +StringRef ClientHandler::get_forwarded_by() const { auto &fwdconf = get_config()->http.forwarded; if (fwdconf.by_node_type == FORWARDED_NODE_OBFUSCATED) { return StringRef(fwdconf.by_obfuscated); } - return StringRef(faddr_->hostport); + return StringRef{faddr_->hostport}; } -const std::string &ClientHandler::get_forwarded_for() const { - return forwarded_for_; +StringRef ClientHandler::get_forwarded_for() const { + return StringRef{forwarded_for_}; } } // namespace shrpx diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index 4830e93e..8d1bc7c0 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -86,6 +86,7 @@ public: struct ev_loop *get_loop() const; void reset_upstream_read_timeout(ev_tstamp t); void reset_upstream_write_timeout(ev_tstamp t); + void signal_reset_upstream_conn_rtimer(); int validate_next_proto(); const std::string &get_ipaddr() const; const std::string &get_port() const; @@ -99,7 +100,6 @@ public: get_downstream_connection(Downstream *downstream); MemchunkPool *get_mcpool(); SSL *get_ssl() const; - ConnectBlocker *get_connect_blocker() const; // Call this function when HTTP/2 connection header is received at // the start of the connection. void direct_http2_upgrade(); @@ -109,7 +109,7 @@ public: int perform_http2_upgrade(HttpsUpstream *http); bool get_http2_upgrade_allowed() const; // Returns upstream scheme, either "http" or "https" - std::string get_upstream_scheme() const; + StringRef get_upstream_scheme() const; void start_immediate_shutdown(); // Writes upstream accesslog using |downstream|. The |downstream| @@ -136,16 +136,15 @@ public: // Returns string suitable for use in "by" parameter of Forwarded // header field. - StringRef get_forwarded_by(); + StringRef get_forwarded_by() const; // Returns string suitable for use in "for" parameter of Forwarded // header field. - const std::string &get_forwarded_for() const; + StringRef get_forwarded_for() const; private: Connection conn_; ev_timer reneg_shutdown_timer_; std::unique_ptr upstream_; - std::unique_ptr> pinned_http2sessions_; // IP address of client. If UNIX domain socket is used, this is // "localhost". std::string ipaddr_; @@ -163,6 +162,7 @@ private: // The number of bytes of HTTP/2 client connection header to read size_t left_connhd_len_; bool should_close_after_write_; + bool reset_conn_rtimer_required_; ReadBuf rb_; }; diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 104c97b6..73bdf71a 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -69,8 +69,6 @@ Config *mod_config() { return config; } void create_config() { config = new Config(); } -std::string EMPTY_STRING; - TicketKeys::~TicketKeys() { /* Erase keys from memory */ for (auto &key : keys) { @@ -78,42 +76,6 @@ TicketKeys::~TicketKeys() { } } -DownstreamAddr::DownstreamAddr(const DownstreamAddr &other) - : addr(other.addr), - host(other.host), - hostport(other.hostport), - port(other.port), - host_unix(other.host_unix) {} - -DownstreamAddr &DownstreamAddr::operator=(const DownstreamAddr &other) { - if (this == &other) { - return *this; - } - - addr = other.addr; - host = other.host; - hostport = other.hostport; - port = other.port; - host_unix = other.host_unix; - - return *this; -} - -DownstreamAddrGroup::DownstreamAddrGroup(const DownstreamAddrGroup &other) - : pattern(strcopy(other.pattern)), addrs(other.addrs) {} - -DownstreamAddrGroup &DownstreamAddrGroup:: -operator=(const DownstreamAddrGroup &other) { - if (this == &other) { - return *this; - } - - pattern = strcopy(other.pattern); - addrs = other.addrs; - - return *this; -} - namespace { int split_host_port(char *host, size_t hostlen, uint16_t *port_ptr, const char *hostport, size_t hostportlen) { @@ -607,33 +569,71 @@ int parse_duration(ev_tstamp *dest, const char *opt, const char *optarg) { } // namespace namespace { -// Parses host-path mapping patterns in |src|, and stores mappings in -// config. We will store each host-path pattern found in |src| with -// |addr|. |addr| will be copied accordingly. Also we make a group -// based on the pattern. The "/" pattern is considered as catch-all. -void parse_mapping(const DownstreamAddr &addr, const char *src) { +// Parses host-path mapping patterns in |src_pattern|, and stores +// mappings in config. We will store each host-path pattern found in +// |src| with |addr|. |addr| will be copied accordingly. Also we +// make a group based on the pattern. The "/" pattern is considered +// as catch-all. We also parse protocol specified in |src_proto|. +// +// This function returns 0 if it succeeds, or -1. +int parse_mapping(const DownstreamAddrConfig &addr, + const StringRef &src_pattern, const StringRef &src_proto) { // This returns at least 1 element (it could be empty string). We // will append '/' to all patterns, so it becomes catch-all pattern. - auto mapping = util::split_config_str_list(src, ':'); + auto mapping = util::split_str(src_pattern, ':'); assert(!mapping.empty()); auto &addr_groups = mod_config()->conn.downstream.addr_groups; + auto proto = PROTO_HTTP1; + + if (!src_proto.empty()) { + if (!util::istarts_with_l(src_proto, "proto=")) { + LOG(ERROR) << "backend: proto keyword not found"; + return -1; + } + + auto protostr = StringRef{std::begin(src_proto) + str_size("proto="), + std::end(src_proto)}; + if (protostr.empty()) { + LOG(ERROR) << "backend: protocol is empty"; + return -1; + } + + if (util::streq_l("h2", std::begin(protostr), protostr.size())) { + proto = PROTO_HTTP2; + } else if (util::streq_l("http/1.1", std::begin(protostr), + protostr.size())) { + proto = PROTO_HTTP1; + } else { + LOG(ERROR) << "backend: unknown protocol " << protostr; + return -1; + } + } + for (const auto &raw_pattern : mapping) { auto done = false; std::string pattern; - auto slash = std::find(raw_pattern.first, raw_pattern.second, '/'); - if (slash == raw_pattern.second) { + auto slash = std::find(std::begin(raw_pattern), std::end(raw_pattern), '/'); + if (slash == std::end(raw_pattern)) { // This effectively makes empty pattern to "/". - pattern.assign(raw_pattern.first, raw_pattern.second); + pattern.assign(std::begin(raw_pattern), std::end(raw_pattern)); util::inp_strlower(pattern); pattern += '/'; } else { - pattern.assign(raw_pattern.first, slash); + pattern.assign(std::begin(raw_pattern), slash); util::inp_strlower(pattern); - pattern += http2::normalize_path(slash, raw_pattern.second); + pattern += http2::normalize_path(StringRef{slash, std::end(raw_pattern)}, + StringRef{}); } for (auto &g : addr_groups) { - if (g.pattern.get() == pattern) { + if (g.pattern == pattern) { + if (g.proto != proto) { + LOG(ERROR) << "backend: protocol mismatch. We saw protocol " + << strproto(g.proto) << " for pattern " << g.pattern + << ", but another protocol " << strproto(proto); + return -1; + } + g.addrs.push_back(addr); done = true; break; @@ -642,14 +642,40 @@ void parse_mapping(const DownstreamAddr &addr, const char *src) { if (done) { continue; } - DownstreamAddrGroup g(pattern); + DownstreamAddrGroupConfig g(StringRef{pattern}); g.addrs.push_back(addr); + g.proto = proto; - mod_config()->router.add_route(g.pattern.get(), strlen(g.pattern.get()), - addr_groups.size()); + if (pattern[0] == '*') { + // wildcard pattern + auto path_first = + std::find(std::begin(g.pattern), std::end(g.pattern), '/'); + + auto host = StringRef{std::begin(g.pattern) + 1, path_first}; + auto path = StringRef{path_first, std::end(g.pattern)}; + + auto &wildcard_patterns = mod_config()->wildcard_patterns; + + auto it = std::find_if( + std::begin(wildcard_patterns), std::end(wildcard_patterns), + [&host](const WildcardPattern &wp) { return wp.host == host; }); + + if (it == std::end(wildcard_patterns)) { + mod_config()->wildcard_patterns.push_back( + {ImmutableString{std::begin(host), std::end(host)}}); + + auto &router = mod_config()->wildcard_patterns.back().router; + router.add_route(path, addr_groups.size()); + } else { + (*it).router.add_route(path, addr_groups.size()); + } + } else { + mod_config()->router.add_route(StringRef{g.pattern}, addr_groups.size()); + } addr_groups.push_back(std::move(g)); } + return 0; } } // namespace @@ -691,12 +717,15 @@ enum { SHRPX_OPTID_ALTSVC, SHRPX_OPTID_BACKEND, SHRPX_OPTID_BACKEND_ADDRESS_FAMILY, + SHRPX_OPTID_BACKEND_CONNECTIONS_PER_FRONTEND, + SHRPX_OPTID_BACKEND_CONNECTIONS_PER_HOST, SHRPX_OPTID_BACKEND_HTTP_PROXY_URI, SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND, SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST, SHRPX_OPTID_BACKEND_HTTP1_TLS, SHRPX_OPTID_BACKEND_HTTP2_CONNECTION_WINDOW_BITS, SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER, + SHRPX_OPTID_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS, SHRPX_OPTID_BACKEND_HTTP2_WINDOW_BITS, SHRPX_OPTID_BACKEND_IPV4, SHRPX_OPTID_BACKEND_IPV6, @@ -705,7 +734,7 @@ enum { SHRPX_OPTID_BACKEND_READ_TIMEOUT, SHRPX_OPTID_BACKEND_REQUEST_BUFFER, SHRPX_OPTID_BACKEND_RESPONSE_BUFFER, - SHRPX_OPTID_BACKEND_TLS_SESSION_CACHE_PER_WORKER, + SHRPX_OPTID_BACKEND_TLS, SHRPX_OPTID_BACKEND_TLS_SNI_FIELD, SHRPX_OPTID_BACKEND_WRITE_TIMEOUT, SHRPX_OPTID_BACKLOG, @@ -730,6 +759,7 @@ enum { SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS, SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER, SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER, + SHRPX_OPTID_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS, SHRPX_OPTID_FRONTEND_HTTP2_READ_TIMEOUT, SHRPX_OPTID_FRONTEND_HTTP2_WINDOW_BITS, SHRPX_OPTID_FRONTEND_NO_TLS, @@ -949,6 +979,11 @@ int option_lookup_token(const char *name, size_t namelen) { break; case 11: switch (name[10]) { + case 's': + if (util::strieq_l("backend-tl", name, 10)) { + return SHRPX_OPTID_BACKEND_TLS; + } + break; case 't': if (util::strieq_l("write-burs", name, 10)) { return SHRPX_OPTID_WRITE_BURST; @@ -1360,6 +1395,11 @@ int option_lookup_token(const char *name, size_t namelen) { return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_TLS; } break; + case 't': + if (util::strieq_l("backend-connections-per-hos", name, 27)) { + return SHRPX_OPTID_BACKEND_CONNECTIONS_PER_HOST; + } + break; } break; case 30: @@ -1380,6 +1420,15 @@ int option_lookup_token(const char *name, size_t namelen) { break; } break; + case 32: + switch (name[31]) { + case 'd': + if (util::strieq_l("backend-connections-per-fronten", name, 31)) { + return SHRPX_OPTID_BACKEND_CONNECTIONS_PER_FRONTEND; + } + break; + } + break; case 33: switch (name[32]) { case 'l': @@ -1431,14 +1480,14 @@ int option_lookup_token(const char *name, size_t namelen) { if (util::strieq_l("backend-http2-connections-per-worke", name, 35)) { return SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER; } - if (util::strieq_l("backend-tls-session-cache-per-worke", name, 35)) { - return SHRPX_OPTID_BACKEND_TLS_SESSION_CACHE_PER_WORKER; - } break; case 's': if (util::strieq_l("backend-http2-connection-window-bit", name, 35)) { return SHRPX_OPTID_BACKEND_HTTP2_CONNECTION_WINDOW_BITS; } + if (util::strieq_l("backend-http2-max-concurrent-stream", name, 35)) { + return SHRPX_OPTID_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS; + } break; } break; @@ -1453,6 +1502,9 @@ int option_lookup_token(const char *name, size_t namelen) { if (util::strieq_l("frontend-http2-connection-window-bit", name, 36)) { return SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS; } + if (util::strieq_l("frontend-http2-max-concurrent-stream", name, 36)) { + return SHRPX_OPTID_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS; + } break; } break; @@ -1518,19 +1570,17 @@ int parse_config(const char *opt, const char *optarg, switch (optid) { case SHRPX_OPTID_BACKEND: { - auto optarglen = strlen(optarg); - const char *pat_delim = strchr(optarg, ';'); - if (!pat_delim) { - pat_delim = optarg + optarglen; - } - DownstreamAddr addr; + auto src = StringRef{optarg}; + auto addr_end = std::find(std::begin(src), std::end(src), ';'); + + DownstreamAddrConfig addr{}; if (util::istarts_with(optarg, SHRPX_UNIX_PATH_PREFIX)) { - auto path = optarg + str_size(SHRPX_UNIX_PATH_PREFIX); - addr.host = ImmutableString(path, pat_delim); + auto path = std::begin(src) + str_size(SHRPX_UNIX_PATH_PREFIX); + addr.host = ImmutableString(path, addr_end); addr.host_unix = true; } else { - if (split_host_port(host, sizeof(host), &port, optarg, - pat_delim - optarg) == -1) { + if (split_host_port(host, sizeof(host), &port, &src[0], + addr_end - std::begin(src)) == -1) { return -1; } @@ -1538,14 +1588,16 @@ int parse_config(const char *opt, const char *optarg, addr.port = port; } - auto mapping = pat_delim < optarg + optarglen ? pat_delim + 1 : pat_delim; - // We may introduce new parameter after additional ';', so don't - // allow extra ';' in pattern for now. - if (strchr(mapping, ';') != nullptr) { - LOG(ERROR) << opt << ": ';' must not be used in pattern"; + auto mapping = addr_end == std::end(src) ? addr_end : addr_end + 1; + auto mapping_end = std::find(mapping, std::end(src), ';'); + + auto proto = mapping_end == std::end(src) ? mapping_end : mapping_end + 1; + auto proto_end = std::find(proto, std::end(src), ';'); + + if (parse_mapping(addr, StringRef{mapping, mapping_end}, + StringRef{proto, proto_end}) != 0) { return -1; } - parse_mapping(addr, mapping); return 0; } @@ -1600,8 +1652,20 @@ int parse_config(const char *opt, const char *optarg, #else // !NOTHREADS return parse_uint(&mod_config()->num_worker, opt, optarg); #endif // !NOTHREADS - case SHRPX_OPTID_HTTP2_MAX_CONCURRENT_STREAMS: - return parse_uint(&mod_config()->http2.max_concurrent_streams, opt, optarg); + case SHRPX_OPTID_HTTP2_MAX_CONCURRENT_STREAMS: { + LOG(WARN) << opt << ": deprecated. Use " + << SHRPX_OPT_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS << " and " + << SHRPX_OPT_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS << " instead."; + size_t n; + if (parse_uint(&n, opt, optarg) != 0) { + return -1; + } + auto &http2conf = mod_config()->http2; + http2conf.upstream.max_concurrent_streams = n; + http2conf.downstream.max_concurrent_streams = n; + + return 0; + } case SHRPX_OPTID_LOG_LEVEL: if (Log::set_severity_level_by_name(optarg) == -1) { LOG(ERROR) << opt << ": Invalid severity level: " << optarg; @@ -1618,13 +1682,13 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_HTTP2_BRIDGE: - mod_config()->http2_bridge = util::strieq(optarg, "yes"); - - return 0; + LOG(ERROR) << opt << ": deprecated. Use backend=,;;proto=h2 " + "and backend-tls"; + return -1; case SHRPX_OPTID_CLIENT_PROXY: - mod_config()->client_proxy = util::strieq(optarg, "yes"); - - return 0; + LOG(ERROR) << opt << ": deprecated. Use http2-proxy, frontend-no-tls, " + "backend=,;;proto=h2 and backend-tls"; + return -1; case SHRPX_OPTID_ADD_X_FORWARDED_FOR: mod_config()->http.xff.add = util::strieq(optarg, "yes"); @@ -1659,7 +1723,7 @@ int parse_config(const char *opt, const char *optarg, return parse_duration(&mod_config()->http2.timeout.stream_write, opt, optarg); case SHRPX_OPTID_ACCESSLOG_FILE: - mod_config()->logging.access.file = strcopy(optarg); + mod_config()->logging.access.file = optarg; return 0; case SHRPX_OPTID_ACCESSLOG_SYSLOG: @@ -1671,7 +1735,7 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_ERRORLOG_FILE: - mod_config()->logging.error.file = strcopy(optarg); + mod_config()->logging.error.file = optarg; return 0; case SHRPX_OPTID_ERRORLOG_SYSLOG: @@ -1757,15 +1821,15 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_BACKEND_NO_TLS: - mod_config()->conn.downstream.no_tls = util::strieq(optarg, "yes"); - + LOG(WARN) << opt << ": deprecated. backend connection is not encrypted by " + "default. See also " << SHRPX_OPT_BACKEND_TLS; return 0; case SHRPX_OPTID_BACKEND_TLS_SNI_FIELD: mod_config()->tls.backend_sni_name = optarg; return 0; case SHRPX_OPTID_PID_FILE: - mod_config()->pid_file = strcopy(optarg); + mod_config()->pid_file = optarg; return 0; case SHRPX_OPTID_USER: { @@ -1775,14 +1839,14 @@ int parse_config(const char *opt, const char *optarg, << strerror(errno); return -1; } - mod_config()->user = strcopy(pwd->pw_name); + mod_config()->user = pwd->pw_name; mod_config()->uid = pwd->pw_uid; mod_config()->gid = pwd->pw_gid; return 0; } case SHRPX_OPTID_PRIVATE_KEY_FILE: - mod_config()->tls.private_key_file = strcopy(optarg); + mod_config()->tls.private_key_file = optarg; return 0; case SHRPX_OPTID_PRIVATE_KEY_PASSWD_FILE: { @@ -1791,16 +1855,16 @@ int parse_config(const char *opt, const char *optarg, LOG(ERROR) << opt << ": Couldn't read key file's passwd from " << optarg; return -1; } - mod_config()->tls.private_key_passwd = strcopy(passwd); + mod_config()->tls.private_key_passwd = passwd; return 0; } case SHRPX_OPTID_CERTIFICATE_FILE: - mod_config()->tls.cert_file = strcopy(optarg); + mod_config()->tls.cert_file = optarg; return 0; case SHRPX_OPTID_DH_PARAM_FILE: - mod_config()->tls.dh_param_file = strcopy(optarg); + mod_config()->tls.dh_param_file = optarg; return 0; case SHRPX_OPTID_SUBCERT: { @@ -1841,19 +1905,19 @@ int parse_config(const char *opt, const char *optarg, return 0; } case SHRPX_OPTID_CIPHERS: - mod_config()->tls.ciphers = strcopy(optarg); + mod_config()->tls.ciphers = optarg; return 0; case SHRPX_OPTID_CLIENT: - mod_config()->client = util::strieq(optarg, "yes"); - - return 0; + LOG(ERROR) << opt << ": deprecated. Use frontend-no-tls, " + "backend=,;;proto=h2 and backend-tls"; + return -1; case SHRPX_OPTID_INSECURE: mod_config()->tls.insecure = util::strieq(optarg, "yes"); return 0; case SHRPX_OPTID_CACERT: - mod_config()->tls.cacert = strcopy(optarg); + mod_config()->tls.cacert = optarg; return 0; case SHRPX_OPTID_BACKEND_IPV4: @@ -1944,25 +2008,23 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_VERIFY_CLIENT_CACERT: - mod_config()->tls.client_verify.cacert = strcopy(optarg); + mod_config()->tls.client_verify.cacert = optarg; return 0; case SHRPX_OPTID_CLIENT_PRIVATE_KEY_FILE: - mod_config()->tls.client.private_key_file = strcopy(optarg); + mod_config()->tls.client.private_key_file = optarg; return 0; case SHRPX_OPTID_CLIENT_CERT_FILE: - mod_config()->tls.client.cert_file = strcopy(optarg); + mod_config()->tls.client.cert_file = optarg; return 0; case SHRPX_OPTID_FRONTEND_HTTP2_DUMP_REQUEST_HEADER: - mod_config()->http2.upstream.debug.dump.request_header_file = - strcopy(optarg); + mod_config()->http2.upstream.debug.dump.request_header_file = optarg; return 0; case SHRPX_OPTID_FRONTEND_HTTP2_DUMP_RESPONSE_HEADER: - mod_config()->http2.upstream.debug.dump.response_header_file = - strcopy(optarg); + mod_config()->http2.upstream.debug.dump.response_header_file = optarg; return 0; case SHRPX_OPTID_HTTP2_NO_COOKIE_CRUMBLING: @@ -2003,7 +2065,7 @@ int parse_config(const char *opt, const char *optarg, return -1; } - AltSvc altsvc; + AltSvc altsvc{}; altsvc.protocol_id = std::move(tokens[0]); @@ -2050,7 +2112,11 @@ int parse_config(const char *opt, const char *optarg, "--host-rewrite option."; return 0; - case SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST: { + case SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_HOST: + LOG(WARN) << opt + << ": deprecated. Use backend-connections-per-host instead."; + // fall through + case SHRPX_OPTID_BACKEND_CONNECTIONS_PER_HOST: { int n; if (parse_uint(&n, opt, optarg) != 0) { @@ -2068,6 +2134,10 @@ int parse_config(const char *opt, const char *optarg, return 0; } case SHRPX_OPTID_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND: + LOG(WARN) << opt << ": deprecated. Use " + << SHRPX_OPT_BACKEND_CONNECTIONS_PER_FRONTEND << " instead."; + // fall through + case SHRPX_OPTID_BACKEND_CONNECTIONS_PER_FRONTEND: return parse_uint(&mod_config()->conn.downstream.connections_per_frontend, opt, optarg); case SHRPX_OPTID_LISTENER_DISABLE_TIMEOUT: @@ -2120,10 +2190,10 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_BACKEND_HTTP2_CONNECTIONS_PER_WORKER: - return parse_uint(&mod_config()->http2.downstream.connections_per_worker, - opt, optarg); + LOG(WARN) << opt << ": deprecated."; + return 0; case SHRPX_OPTID_FETCH_OCSP_RESPONSE_FILE: - mod_config()->tls.ocsp.fetch_ocsp_response_file = strcopy(optarg); + mod_config()->tls.ocsp.fetch_ocsp_response_file = optarg; return 0; case SHRPX_OPTID_OCSP_UPDATE_INTERVAL: @@ -2191,7 +2261,7 @@ int parse_config(const char *opt, const char *optarg, } auto &memcachedconf = mod_config()->tls.session_cache.memcached; - memcachedconf.host = strcopy(host); + memcachedconf.host = host; memcachedconf.port = port; return 0; @@ -2203,7 +2273,7 @@ int parse_config(const char *opt, const char *optarg, } auto &memcachedconf = mod_config()->tls.ticket.memcached; - memcachedconf.host = strcopy(host); + memcachedconf.host = host; memcachedconf.port = port; return 0; @@ -2244,7 +2314,7 @@ int parse_config(const char *opt, const char *optarg, case SHRPX_OPTID_MRUBY_FILE: #ifdef HAVE_MRUBY - mod_config()->mruby_file = strcopy(optarg); + mod_config()->mruby_file = optarg; #else // !HAVE_MRUBY LOG(WARN) << opt << ": ignored because mruby support is disabled at build time."; @@ -2321,12 +2391,13 @@ int parse_config(const char *opt, const char *optarg, return 0; case SHRPX_OPTID_BACKEND_HTTP1_TLS: - mod_config()->conn.downstream.http1_tls = util::strieq(optarg, "yes"); + LOG(WARN) << opt << ": deprecated. Use " << SHRPX_OPT_BACKEND_TLS + << " instead."; + // fall through + case SHRPX_OPTID_BACKEND_TLS: + mod_config()->conn.downstream.no_tls = !util::strieq(optarg, "yes"); return 0; - case SHRPX_OPTID_BACKEND_TLS_SESSION_CACHE_PER_WORKER: - return parse_uint(&mod_config()->tls.downstream_session_cache_per_worker, - opt, optarg); case SHRPX_OPTID_TLS_SESSION_CACHE_MEMCACHED_TLS: mod_config()->tls.session_cache.memcached.tls = util::strieq(optarg, "yes"); @@ -2360,6 +2431,12 @@ int parse_config(const char *opt, const char *optarg, case SHRPX_OPTID_BACKEND_ADDRESS_FAMILY: return parse_address_family(&mod_config()->conn.downstream.family, opt, optarg); + case SHRPX_OPTID_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS: + return parse_uint(&mod_config()->http2.upstream.max_concurrent_streams, opt, + optarg); + case SHRPX_OPTID_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS: + return parse_uint(&mod_config()->http2.downstream.max_concurrent_streams, + opt, optarg); case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; @@ -2539,99 +2616,20 @@ int int_syslog_facility(const char *strfacility) { return -1; } -namespace { -size_t -match_downstream_addr_group_host(const Router &router, const std::string &host, - const char *path, size_t pathlen, - const std::vector &groups, - size_t catch_all) { - if (pathlen == 0 || *path != '/') { - auto group = router.match(host, "/", 1); - if (group != -1) { - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Found pattern with query " << host - << ", matched pattern=" << groups[group].pattern.get(); - } - return group; - } - return catch_all; +StringRef strproto(shrpx_proto proto) { + switch (proto) { + case PROTO_NONE: + return StringRef::from_lit("none"); + case PROTO_HTTP1: + return StringRef::from_lit("http/1.1"); + case PROTO_HTTP2: + return StringRef::from_lit("h2"); + case PROTO_MEMCACHED: + return StringRef::from_lit("memcached"); } - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Perform mapping selection, using host=" << host - << ", path=" << std::string(path, pathlen); - } - - auto group = router.match(host, path, pathlen); - if (group != -1) { - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Found pattern with query " << host - << std::string(path, pathlen) - << ", matched pattern=" << groups[group].pattern.get(); - } - return group; - } - - group = router.match("", path, pathlen); - if (group != -1) { - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Found pattern with query " << std::string(path, pathlen) - << ", matched pattern=" << groups[group].pattern.get(); - } - return group; - } - - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "None match. Use catch-all pattern"; - } - return catch_all; -} -} // namespace - -size_t -match_downstream_addr_group(const Router &router, const std::string &hostport, - const std::string &raw_path, - const std::vector &groups, - size_t catch_all) { - if (std::find(std::begin(hostport), std::end(hostport), '/') != - std::end(hostport)) { - // We use '/' specially, and if '/' is included in host, it breaks - // our code. Select catch-all case. - return catch_all; - } - - auto fragment = std::find(std::begin(raw_path), std::end(raw_path), '#'); - auto query = std::find(std::begin(raw_path), fragment, '?'); - auto path = raw_path.c_str(); - auto pathlen = query - std::begin(raw_path); - - if (hostport.empty()) { - return match_downstream_addr_group_host(router, hostport, path, pathlen, - groups, catch_all); - } - - std::string host; - if (hostport[0] == '[') { - // assume this is IPv6 numeric address - auto p = std::find(std::begin(hostport), std::end(hostport), ']'); - if (p == std::end(hostport)) { - return catch_all; - } - if (p + 1 < std::end(hostport) && *(p + 1) != ':') { - return catch_all; - } - host.assign(std::begin(hostport), p + 1); - } else { - auto p = std::find(std::begin(hostport), std::end(hostport), ':'); - if (p == std::begin(hostport)) { - return catch_all; - } - host.assign(std::begin(hostport), p); - } - - util::inp_strlower(host); - return match_downstream_addr_group_host(router, host, path, pathlen, groups, - catch_all); + // gcc needs this. + assert(0); } } // namespace shrpx diff --git a/src/shrpx_config.h b/src/shrpx_config.h index ebc97f8a..c4b0cde1 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -53,12 +53,15 @@ #include "shrpx_router.h" #include "template.h" #include "http2.h" +#include "network.h" using namespace nghttp2; namespace shrpx { struct LogFragment; +class ConnectBlocker; +class Http2Session; namespace ssl { @@ -208,8 +211,6 @@ constexpr char SHRPX_OPT_MAX_RESPONSE_HEADER_FIELDS[] = constexpr char SHRPX_OPT_NO_HTTP2_CIPHER_BLACK_LIST[] = "no-http2-cipher-black-list"; constexpr char SHRPX_OPT_BACKEND_HTTP1_TLS[] = "backend-http1-tls"; -constexpr char SHRPX_OPT_BACKEND_TLS_SESSION_CACHE_PER_WORKER[] = - "backend-tls-session-cache-per-worker"; constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_TLS[] = "tls-session-cache-memcached-tls"; constexpr char SHRPX_OPT_TLS_SESSION_CACHE_MEMCACHED_CERT_FILE[] = @@ -227,23 +228,19 @@ constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_PRIVATE_KEY_FILE[] = constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_ADDRESS_FAMILY[] = "tls-ticket-key-memcached-address-family"; constexpr char SHRPX_OPT_BACKEND_ADDRESS_FAMILY[] = "backend-address-family"; +constexpr char SHRPX_OPT_FRONTEND_HTTP2_MAX_CONCURRENT_STREAMS[] = + "frontend-http2-max-concurrent-streams"; +constexpr char SHRPX_OPT_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS[] = + "backend-http2-max-concurrent-streams"; +constexpr char SHRPX_OPT_BACKEND_CONNECTIONS_PER_FRONTEND[] = + "backend-connections-per-frontend"; +constexpr char SHRPX_OPT_BACKEND_TLS[] = "backend-tls"; +constexpr char SHRPX_OPT_BACKEND_CONNECTIONS_PER_HOST[] = + "backend-connections-per-host"; constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; -union sockaddr_union { - sockaddr_storage storage; - sockaddr sa; - sockaddr_in6 in6; - sockaddr_in in; - sockaddr_un un; -}; - -struct Address { - size_t len; - union sockaddr_union su; -}; - -enum shrpx_proto { PROTO_HTTP2, PROTO_HTTP }; +enum shrpx_proto { PROTO_NONE, PROTO_HTTP1, PROTO_HTTP2, PROTO_MEMCACHED }; enum shrpx_forwarded_param { FORWARDED_NONE = 0, @@ -258,13 +255,7 @@ enum shrpx_forwarded_node_type { FORWARDED_NODE_IP, }; -// Used inside function if it has to return const reference to empty -// string without defining empty string each time. -extern std::string EMPTY_STRING; - struct AltSvc { - AltSvc() : port(0) {} - std::string protocol_id, host, origin, service; uint16_t port; @@ -288,17 +279,21 @@ struct UpstreamAddr { int fd; }; -struct DownstreamAddr { - DownstreamAddr() : addr{}, port(0), host_unix(false) {} - DownstreamAddr(const DownstreamAddr &other); - DownstreamAddr(DownstreamAddr &&) = default; - DownstreamAddr &operator=(const DownstreamAddr &other); - DownstreamAddr &operator=(DownstreamAddr &&other) = default; +struct TLSSessionCache { + // ASN1 representation of SSL_SESSION object. See + // i2d_SSL_SESSION(3SSL). + std::vector session_data; + // The last time stamp when this cache entry is created or updated. + ev_tstamp last_updated; +}; +struct DownstreamAddrConfig { Address addr; // backend address. If |host_unix| is true, this is UNIX domain // socket path. ImmutableString host; + // :. This does not treat 80 and 443 specially. If + // |host_unix| is true, this is "localhost". ImmutableString hostport; // backend port. 0 if |host_unix| is true. uint16_t port; @@ -306,15 +301,14 @@ struct DownstreamAddr { bool host_unix; }; -struct DownstreamAddrGroup { - DownstreamAddrGroup(const std::string &pattern) : pattern(strcopy(pattern)) {} - DownstreamAddrGroup(const DownstreamAddrGroup &other); - DownstreamAddrGroup(DownstreamAddrGroup &&) = default; - DownstreamAddrGroup &operator=(const DownstreamAddrGroup &other); - DownstreamAddrGroup &operator=(DownstreamAddrGroup &&) = default; +struct DownstreamAddrGroupConfig { + DownstreamAddrGroupConfig(const StringRef &pattern) + : pattern(pattern.c_str(), pattern.size()), proto(PROTO_HTTP1) {} - std::unique_ptr pattern; - std::vector addrs; + ImmutableString pattern; + std::vector addrs; + // Application protocol used in this group + shrpx_proto proto; }; struct TicketKey { @@ -352,7 +346,9 @@ struct TLSConfig { struct { Address addr; uint16_t port; - std::unique_ptr host; + // Hostname of memcached server. This is also used as SNI field + // if TLS is enabled. + ImmutableString host; // Client private key and certificate for authentication ImmutableString private_key_file; ImmutableString cert_file; @@ -379,7 +375,9 @@ struct TLSConfig { struct { Address addr; uint16_t port; - std::unique_ptr host; + // Hostname of memcached server. This is also used as SNI field + // if TLS is enabled. + ImmutableString host; // Client private key and certificate for authentication ImmutableString private_key_file; ImmutableString cert_file; @@ -399,7 +397,7 @@ struct TLSConfig { // OCSP realted configurations struct { ev_tstamp update_interval; - std::unique_ptr fetch_ocsp_response_file; + ImmutableString fetch_ocsp_response_file; bool disabled; } ocsp; @@ -407,14 +405,14 @@ struct TLSConfig { struct { // Path to file containing CA certificate solely used for client // certificate validation - std::unique_ptr cacert; + ImmutableString cacert; bool enabled; } client_verify; // Client private key and certificate used in backend connections. struct { - std::unique_ptr private_key_file; - std::unique_ptr cert_file; + ImmutableString private_key_file; + ImmutableString cert_file; } client; // The list of (private key file, certificate file) pair @@ -425,18 +423,17 @@ struct TLSConfig { std::vector npn_list; // list of supported SSL/TLS protocol strings. std::vector tls_proto_list; - size_t downstream_session_cache_per_worker; // Bit mask to disable SSL/TLS protocol versions. This will be // passed to SSL_CTX_set_options(). long int tls_proto_mask; std::string backend_sni_name; std::chrono::seconds session_timeout; - std::unique_ptr private_key_file; - std::unique_ptr private_key_passwd; - std::unique_ptr cert_file; - std::unique_ptr dh_param_file; - std::unique_ptr ciphers; - std::unique_ptr cacert; + ImmutableString private_key_file; + ImmutableString private_key_passwd; + ImmutableString cert_file; + ImmutableString dh_param_file; + ImmutableString ciphers; + ImmutableString cacert; bool insecure; bool no_http2_cipher_black_list; }; @@ -478,8 +475,8 @@ struct Http2Config { struct { struct { struct { - std::unique_ptr request_header_file; - std::unique_ptr response_header_file; + ImmutableString request_header_file; + ImmutableString response_header_file; FILE *request_header; FILE *response_header; } dump; @@ -489,19 +486,19 @@ struct Http2Config { nghttp2_session_callbacks *callbacks; size_t window_bits; size_t connection_window_bits; + size_t max_concurrent_streams; } upstream; struct { nghttp2_option *option; nghttp2_session_callbacks *callbacks; size_t window_bits; size_t connection_window_bits; - size_t connections_per_worker; + size_t max_concurrent_streams; } downstream; struct { ev_tstamp stream_read; ev_tstamp stream_write; } timeout; - size_t max_concurrent_streams; bool no_cookie_crumbling; bool no_server_push; }; @@ -509,12 +506,12 @@ struct Http2Config { struct LoggingConfig { struct { std::vector format; - std::unique_ptr file; + ImmutableString file; // Send accesslog to syslog, ignoring accesslog_file. bool syslog; } access; struct { - std::unique_ptr file; + ImmutableString file; // Send errorlog to syslog, ignoring errorlog_file. bool syslog; } error; @@ -560,15 +557,13 @@ struct ConnectionConfig { ev_tstamp write; ev_tstamp idle_read; } timeout; - std::vector addr_groups; + std::vector addr_groups; // The index of catch-all group in downstream_addr_groups. size_t addr_group_catch_all; size_t connections_per_host; size_t connections_per_frontend; size_t request_buffer_size; size_t response_buffer_size; - // downstream protocol; this will be determined by given options. - shrpx_proto proto; // Address family of backend connection. One of either AF_INET, // AF_INET6 or AF_UNSPEC. This is ignored if backend connection // is made via Unix domain socket. @@ -578,18 +573,27 @@ struct ConnectionConfig { } downstream; }; +// Wildcard host pattern routing. We strips left most '*' from host +// field. router includes all path pattern sharing same wildcard +// host. +struct WildcardPattern { + ImmutableString host; + Router router; +}; + struct Config { Router router; + std::vector wildcard_patterns; HttpProxy downstream_http_proxy; HttpConfig http; Http2Config http2; TLSConfig tls; LoggingConfig logging; ConnectionConfig conn; - std::unique_ptr pid_file; - std::unique_ptr conf_path; - std::unique_ptr user; - std::unique_ptr mruby_file; + ImmutableString pid_file; + ImmutableString conf_path; + ImmutableString user; + ImmutableString mruby_file; char **original_argv; char **argv; char *cwd; @@ -603,11 +607,6 @@ struct Config { bool verbose; bool daemon; bool http2_proxy; - bool http2_bridge; - bool client_proxy; - bool client; - // true if --client or --client-proxy are enabled. - bool client_mode; }; const Config *get_config(); @@ -654,15 +653,8 @@ std::unique_ptr read_tls_ticket_key_file(const std::vector &files, const EVP_CIPHER *cipher, const EVP_MD *hmac); -// Selects group based on request's |hostport| and |path|. |hostport| -// is the value taken from :authority or host header field, and may -// contain port. The |path| may contain query part. We require the -// catch-all pattern in place, so this function always selects one -// group. The catch-all group index is given in |catch_all|. All -// patterns are given in |groups|. -size_t match_downstream_addr_group( - const Router &router, const std::string &hostport, const std::string &path, - const std::vector &groups, size_t catch_all); +// Returns string representation of |proto|. +StringRef strproto(shrpx_proto proto); } // namespace shrpx diff --git a/src/shrpx_config_test.cc b/src/shrpx_config_test.cc index eda67e85..635c23d9 100644 --- a/src/shrpx_config_test.cc +++ b/src/shrpx_config_test.cc @@ -238,120 +238,4 @@ void test_shrpx_config_read_tls_ticket_key_file_aes_256(void) { "a..............................b")); } -void test_shrpx_config_match_downstream_addr_group(void) { - auto groups = std::vector{ - {"nghttp2.org/"}, - {"nghttp2.org/alpha/bravo/"}, - {"nghttp2.org/alpha/charlie"}, - {"nghttp2.org/delta%3A"}, - {"www.nghttp2.org/"}, - {"[::1]/"}, - {"nghttp2.org/alpha/bravo/delta"}, - // Check that match is done in the single node - {"example.com/alpha/bravo"}, - {"192.168.0.1/alpha/"}, - }; - - Router router; - - for (size_t i = 0; i < groups.size(); ++i) { - auto &g = groups[i]; - router.add_route(g.pattern.get(), strlen(g.pattern.get()), i); - } - - CU_ASSERT(0 == match_downstream_addr_group(router, "nghttp2.org", "/", groups, - 255)); - - // port is removed - CU_ASSERT(0 == match_downstream_addr_group(router, "nghttp2.org:8080", "/", - groups, 255)); - - // host is case-insensitive - CU_ASSERT(4 == match_downstream_addr_group(router, "WWW.nghttp2.org", - "/alpha", groups, 255)); - - CU_ASSERT(1 == match_downstream_addr_group(router, "nghttp2.org", - "/alpha/bravo/", groups, 255)); - - // /alpha/bravo also matches /alpha/bravo/ - CU_ASSERT(1 == match_downstream_addr_group(router, "nghttp2.org", - "/alpha/bravo", groups, 255)); - - // path part is case-sensitive - CU_ASSERT(0 == match_downstream_addr_group(router, "nghttp2.org", - "/Alpha/bravo", groups, 255)); - - CU_ASSERT(1 == match_downstream_addr_group(router, "nghttp2.org", - "/alpha/bravo/charlie", groups, - 255)); - - CU_ASSERT(2 == match_downstream_addr_group(router, "nghttp2.org", - "/alpha/charlie", groups, 255)); - - // pattern which does not end with '/' must match its entirely. So - // this matches to group 0, not group 2. - CU_ASSERT(0 == match_downstream_addr_group(router, "nghttp2.org", - "/alpha/charlie/", groups, 255)); - - CU_ASSERT(255 == match_downstream_addr_group(router, "example.org", "/", - groups, 255)); - - CU_ASSERT(255 == match_downstream_addr_group(router, "", "/", groups, 255)); - - CU_ASSERT(255 == - match_downstream_addr_group(router, "", "alpha", groups, 255)); - - CU_ASSERT(255 == - match_downstream_addr_group(router, "foo/bar", "/", groups, 255)); - - // If path is "*", only match with host + "/". - CU_ASSERT(0 == match_downstream_addr_group(router, "nghttp2.org", "*", groups, - 255)); - - CU_ASSERT(5 == - match_downstream_addr_group(router, "[::1]", "/", groups, 255)); - CU_ASSERT( - 5 == match_downstream_addr_group(router, "[::1]:8080", "/", groups, 255)); - CU_ASSERT(255 == - match_downstream_addr_group(router, "[::1", "/", groups, 255)); - CU_ASSERT(255 == - match_downstream_addr_group(router, "[::1]8000", "/", groups, 255)); - - // Check the case where adding route extends tree - CU_ASSERT(6 == match_downstream_addr_group( - router, "nghttp2.org", "/alpha/bravo/delta", groups, 255)); - - CU_ASSERT(1 == match_downstream_addr_group(router, "nghttp2.org", - "/alpha/bravo/delta/", groups, - 255)); - - // Check the case where query is done in a single node - CU_ASSERT(7 == match_downstream_addr_group(router, "example.com", - "/alpha/bravo", groups, 255)); - - CU_ASSERT(255 == match_downstream_addr_group(router, "example.com", - "/alpha/bravo/", groups, 255)); - - CU_ASSERT(255 == match_downstream_addr_group(router, "example.com", "/alpha", - groups, 255)); - - // Check the case where quey is done in a single node - CU_ASSERT(8 == match_downstream_addr_group(router, "192.168.0.1", "/alpha", - groups, 255)); - - CU_ASSERT(8 == match_downstream_addr_group(router, "192.168.0.1", "/alpha/", - groups, 255)); - - CU_ASSERT(8 == match_downstream_addr_group(router, "192.168.0.1", - "/alpha/bravo", groups, 255)); - - CU_ASSERT(255 == match_downstream_addr_group(router, "192.168.0.1", "/alph", - groups, 255)); - - CU_ASSERT(255 == match_downstream_addr_group(router, "192.168.0.1", "/", - groups, 255)); - - router.dump(); -} - } // namespace shrpx diff --git a/src/shrpx_connect_blocker.cc b/src/shrpx_connect_blocker.cc index 3eb6de16..cdab59a9 100644 --- a/src/shrpx_connect_blocker.cc +++ b/src/shrpx_connect_blocker.cc @@ -26,20 +26,16 @@ namespace shrpx { -namespace { -const ev_tstamp INITIAL_SLEEP = 2.; -} // namespace - namespace { void connect_blocker_cb(struct ev_loop *loop, ev_timer *w, int revents) { if (LOG_ENABLED(INFO)) { - LOG(INFO) << "unblock downstream connection"; + LOG(INFO) << "Unblock"; } } } // namespace -ConnectBlocker::ConnectBlocker(struct ev_loop *loop) - : loop_(loop), sleep_(INITIAL_SLEEP) { +ConnectBlocker::ConnectBlocker(std::mt19937 &gen, struct ev_loop *loop) + : gen_(gen), loop_(loop), fail_count_(0) { ev_timer_init(&timer_, connect_blocker_cb, 0., 0.); } @@ -47,18 +43,27 @@ ConnectBlocker::~ConnectBlocker() { ev_timer_stop(loop_, &timer_); } bool ConnectBlocker::blocked() const { return ev_is_active(&timer_); } -void ConnectBlocker::on_success() { sleep_ = INITIAL_SLEEP; } +void ConnectBlocker::on_success() { fail_count_ = 0; } + +namespace { +constexpr size_t MAX_BACKOFF_EXP = 10; +} // namespace void ConnectBlocker::on_failure() { if (ev_is_active(&timer_)) { return; } - sleep_ = std::min(128., sleep_ * 2); + ++fail_count_; - LOG(WARN) << "connect failure, start sleeping " << sleep_; + auto max_backoff = (1 << std::min(MAX_BACKOFF_EXP, fail_count_)) - 1; + auto dist = std::uniform_int_distribution<>(0, max_backoff); + auto backoff = dist(gen_); - ev_timer_set(&timer_, sleep_, 0.); + LOG(WARN) << "Could not connect " << fail_count_ + << " times in a row; sleep for " << backoff << " seconds"; + + ev_timer_set(&timer_, backoff, 0.); ev_timer_start(loop_, &timer_); } diff --git a/src/shrpx_connect_blocker.h b/src/shrpx_connect_blocker.h index af445644..63a1e3f9 100644 --- a/src/shrpx_connect_blocker.h +++ b/src/shrpx_connect_blocker.h @@ -27,13 +27,15 @@ #include "shrpx.h" +#include + #include namespace shrpx { class ConnectBlocker { public: - ConnectBlocker(struct ev_loop *loop); + ConnectBlocker(std::mt19937 &gen, struct ev_loop *loop); ~ConnectBlocker(); // Returns true if making connection is not allowed. @@ -41,14 +43,18 @@ public: // Call this function if connect operation succeeded. This will // reset sleep_ to minimum value. void on_success(); - // Call this function if connect operation failed. This will start - // timer and blocks connection establishment for sleep_ seconds. + // Call this function if connect operations failed. This will start + // timer and blocks connection establishment with exponential + // backoff. void on_failure(); private: + std::mt19937 gen_; ev_timer timer_; struct ev_loop *loop_; - ev_tstamp sleep_; + // The number of consecutive connection failure. Reset to 0 on + // success. + size_t fail_count_; }; } // namespace diff --git a/src/shrpx_connection.cc b/src/shrpx_connection.cc index 6bbaab3b..418afb12 100644 --- a/src/shrpx_connection.cc +++ b/src/shrpx_connection.cc @@ -47,7 +47,7 @@ Connection::Connection(struct ev_loop *loop, int fd, SSL *ssl, const RateLimitConfig &read_limit, IOCb writecb, IOCb readcb, TimerCb timeoutcb, void *data, size_t tls_dyn_rec_warmup_threshold, - ev_tstamp tls_dyn_rec_idle_timeout) + ev_tstamp tls_dyn_rec_idle_timeout, shrpx_proto proto) : tls{DefaultMemchunks(mcpool), DefaultPeekMemchunks(mcpool)}, wlimit(loop, &wev, write_limit.rate, write_limit.burst), rlimit(loop, &rev, read_limit.rate, read_limit.burst, this), @@ -58,7 +58,8 @@ Connection::Connection(struct ev_loop *loop, int fd, SSL *ssl, data(data), fd(fd), tls_dyn_rec_warmup_threshold(tls_dyn_rec_warmup_threshold), - tls_dyn_rec_idle_timeout(tls_dyn_rec_idle_timeout) { + tls_dyn_rec_idle_timeout(tls_dyn_rec_idle_timeout), + proto(proto) { ev_io_init(&wev, writecb, fd, EV_WRITE); ev_io_init(&rev, readcb, fd, EV_READ); diff --git a/src/shrpx_connection.h b/src/shrpx_connection.h index 30ff448d..71ba33f4 100644 --- a/src/shrpx_connection.h +++ b/src/shrpx_connection.h @@ -77,7 +77,7 @@ struct Connection { const RateLimitConfig &write_limit, const RateLimitConfig &read_limit, IOCb writecb, IOCb readcb, TimerCb timeoutcb, void *data, size_t tls_dyn_rec_warmup_threshold, - ev_tstamp tls_dyn_rec_idle_timeout); + ev_tstamp tls_dyn_rec_idle_timeout, shrpx_proto proto); ~Connection(); void disconnect(); @@ -133,6 +133,10 @@ struct Connection { int fd; size_t tls_dyn_rec_warmup_threshold; ev_tstamp tls_dyn_rec_idle_timeout; + // Application protocol used over the connection. This field is not + // used in this object at the moment. The rest of the program may + // use this value when it is useful. + shrpx_proto proto; }; } // namespace shrpx diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 588e814b..4da62790 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -202,9 +202,8 @@ int ConnectionHandler::create_single_worker() { #ifdef HAVE_NEVERBLEED nb_.get(), #endif // HAVE_NEVERBLEED - StringRef::from_maybe_nullptr(tlsconf.cacert.get()), - StringRef(memcachedconf.cert_file), - StringRef(memcachedconf.private_key_file), StringRef(), nullptr); + StringRef{tlsconf.cacert}, StringRef{memcachedconf.cert_file}, + StringRef{memcachedconf.private_key_file}, nullptr); all_ssl_ctx_.push_back(session_cache_ssl_ctx); } @@ -253,9 +252,8 @@ int ConnectionHandler::create_worker_thread(size_t num) { #ifdef HAVE_NEVERBLEED nb_.get(), #endif // HAVE_NEVERBLEED - StringRef::from_maybe_nullptr(tlsconf.cacert.get()), - StringRef(memcachedconf.cert_file), - StringRef(memcachedconf.private_key_file), StringRef(), nullptr); + StringRef{tlsconf.cacert}, StringRef{memcachedconf.cert_file}, + StringRef{memcachedconf.private_key_file}, nullptr); all_ssl_ctx_.push_back(session_cache_ssl_ctx); } auto worker = @@ -463,7 +461,8 @@ int ConnectionHandler::start_ocsp_update(const char *cert_file) { assert(!ev_is_active(&ocsp_.chldev)); char *const argv[] = { - const_cast(get_config()->tls.ocsp.fetch_ocsp_response_file.get()), + const_cast( + get_config()->tls.ocsp.fetch_ocsp_response_file.c_str()), const_cast(cert_file), nullptr}; char *const envp[] = {nullptr}; @@ -767,9 +766,8 @@ SSL_CTX *ConnectionHandler::create_tls_ticket_key_memcached_ssl_ctx() { #ifdef HAVE_NEVERBLEED nb_.get(), #endif // HAVE_NEVERBLEED - StringRef::from_maybe_nullptr(tlsconf.cacert.get()), - StringRef(memcachedconf.cert_file), - StringRef(memcachedconf.private_key_file), StringRef(), nullptr); + StringRef{tlsconf.cacert}, StringRef{memcachedconf.cert_file}, + StringRef{memcachedconf.private_key_file}, nullptr); all_ssl_ctx_.push_back(ssl_ctx); diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 8f8165b7..0cdd81b8 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -116,6 +116,9 @@ Downstream::Downstream(Upstream *upstream, MemchunkPool *mcpool, : dlnext(nullptr), dlprev(nullptr), response_sent_body_length(0), + balloc_(1024, 1024), + req_(balloc_), + resp_(balloc_), request_start_time_(std::chrono::high_resolution_clock::now()), request_buf_(mcpool), response_buf_(mcpool), @@ -179,6 +182,10 @@ Downstream::~Downstream() { // explicitly. dconn_.reset(); + for (auto rcbuf : rcbufs_) { + nghttp2_rcbuf_decref(rcbuf); + } + if (LOG_ENABLED(INFO)) { DLOG(INFO, this) << "Deleted"; } @@ -237,15 +244,16 @@ void Downstream::force_resume_read() { } namespace { -const Headers::value_type *search_header_linear(const Headers &headers, - const StringRef &name) { - const Headers::value_type *res = nullptr; - for (auto &kv : headers) { +const HeaderRefs::value_type * +search_header_linear_backwards(const HeaderRefs &headers, + const StringRef &name) { + for (auto it = headers.rbegin(); it != headers.rend(); ++it) { + auto &kv = *it; if (kv.name == name) { - res = &kv; + return &kv; } } - return res; + return nullptr; } } // namespace @@ -253,16 +261,29 @@ std::string Downstream::assemble_request_cookie() const { std::string cookie; cookie = ""; for (auto &kv : req_.fs.headers()) { - if (kv.name.size() != 6 || kv.name[5] != 'e' || - !util::streq_l("cooki", kv.name.c_str(), 5)) { + if (kv.token != http2::HD_COOKIE) { continue; } - auto end = kv.value.find_last_not_of(" ;"); - if (end == std::string::npos) { + if (kv.value.empty()) { + continue; + } + + auto end = std::end(kv.value); + for (auto it = std::begin(kv.value) + kv.value.size(); + it != std::begin(kv.value); --it) { + auto c = *(it - 1); + if (c == ' ' || c == ';') { + continue; + } + end = it; + break; + } + + if (end == std::end(kv.value)) { cookie += kv.value; } else { - cookie.append(std::begin(kv.value), std::begin(kv.value) + end + 1); + cookie.append(std::begin(kv.value), end); } cookie += "; "; } @@ -280,18 +301,14 @@ size_t Downstream::count_crumble_request_cookie() { !util::streq_l("cooki", kv.name.c_str(), 5)) { continue; } - size_t last = kv.value.size(); - for (size_t j = 0; j < last;) { - j = kv.value.find_first_not_of("\t ;", j); - if (j == std::string::npos) { - break; + for (auto it = std::begin(kv.value); it != std::end(kv.value);) { + if (*it == '\t' || *it == ' ' || *it == ';') { + ++it; + continue; } - j = kv.value.find(';', j); - if (j == std::string::npos) { - j = last; - } + it = std::find(it, std::end(kv.value), ';'); ++n; } @@ -305,22 +322,19 @@ void Downstream::crumble_request_cookie(std::vector &nva) { !util::streq_l("cooki", kv.name.c_str(), 5)) { continue; } - size_t last = kv.value.size(); - for (size_t j = 0; j < last;) { - j = kv.value.find_first_not_of("\t ;", j); - if (j == std::string::npos) { - break; - } - auto first = j; - - j = kv.value.find(';', j); - if (j == std::string::npos) { - j = last; + for (auto it = std::begin(kv.value); it != std::end(kv.value);) { + if (*it == '\t' || *it == ' ' || *it == ';') { + ++it; + continue; } - nva.push_back({(uint8_t *)"cookie", (uint8_t *)kv.value.c_str() + first, - str_size("cookie"), j - first, + auto first = it; + + it = std::find(it, std::end(kv.value), ';'); + + nva.push_back({(uint8_t *)"cookie", (uint8_t *)first, str_size("cookie"), + (size_t)(it - first), (uint8_t)(NGHTTP2_NV_FLAG_NO_COPY_NAME | NGHTTP2_NV_FLAG_NO_COPY_VALUE | (kv.no_index ? NGHTTP2_NV_FLAG_NO_INDEX : 0))}); @@ -329,146 +343,124 @@ void Downstream::crumble_request_cookie(std::vector &nva) { } namespace { -void add_header(bool &key_prev, size_t &sum, Headers &headers, std::string name, - std::string value) { +void add_header(bool &key_prev, size_t &sum, HeaderRefs &headers, + const StringRef &name, const StringRef &value, bool no_index, + int32_t token) { key_prev = true; sum += name.size() + value.size(); - headers.emplace_back(std::move(name), std::move(value)); + headers.emplace_back(name, value, no_index, token); } } // namespace namespace { -void add_header(size_t &sum, Headers &headers, const uint8_t *name, - size_t namelen, const uint8_t *value, size_t valuelen, - bool no_index, int16_t token) { - sum += namelen + valuelen; - headers.emplace_back( - std::string(reinterpret_cast(name), namelen), - std::string(reinterpret_cast(value), valuelen), no_index, - token); -} -} // namespace - -namespace { -void append_last_header_key(bool &key_prev, size_t &sum, Headers &headers, - const char *data, size_t len) { +void append_last_header_key(BlockAllocator &balloc, bool &key_prev, size_t &sum, + HeaderRefs &headers, const char *data, size_t len) { assert(key_prev); sum += len; auto &item = headers.back(); - item.name.append(data, len); + auto iov = make_byte_ref(balloc, item.name.size() + len + 1); + auto p = iov.base; + p = std::copy(std::begin(item.name), std::end(item.name), p); + p = std::copy_n(data, len, p); + util::inp_strlower(p - len, p); + *p = '\0'; + + item.name = StringRef{iov.base, p}; + + item.token = http2::lookup_token(item.name); } } // namespace namespace { -void append_last_header_value(bool &key_prev, size_t &sum, Headers &headers, +void append_last_header_value(BlockAllocator &balloc, bool &key_prev, + size_t &sum, HeaderRefs &headers, const char *data, size_t len) { key_prev = false; sum += len; auto &item = headers.back(); - item.value.append(data, len); + item.value = concat_string_ref(balloc, item.value, StringRef{data, len}); } } // namespace -int FieldStore::index_headers() { - http2::init_hdidx(hdidx_); +int FieldStore::parse_content_length() { content_length = -1; - for (size_t i = 0; i < headers_.size(); ++i) { - auto &kv = headers_[i]; - util::inp_strlower(kv.name); - - auto token = http2::lookup_token( - reinterpret_cast(kv.name.c_str()), kv.name.size()); - if (token < 0) { + for (auto &kv : headers_) { + if (kv.token != http2::HD_CONTENT_LENGTH) { continue; } - kv.token = token; - http2::index_header(hdidx_, token, i); - - if (token == http2::HD_CONTENT_LENGTH) { - auto len = util::parse_uint(kv.value); - if (len == -1) { - return -1; - } - if (content_length != -1) { - return -1; - } - content_length = len; + auto len = util::parse_uint(kv.value); + if (len == -1) { + return -1; } + if (content_length != -1) { + return -1; + } + content_length = len; } return 0; } -const Headers::value_type *FieldStore::header(int16_t token) const { - return http2::get_header(hdidx_, token, headers_); +const HeaderRefs::value_type *FieldStore::header(int32_t token) const { + for (auto it = headers_.rbegin(); it != headers_.rend(); ++it) { + auto &kv = *it; + if (kv.token == token) { + return &kv; + } + } + return nullptr; } -Headers::value_type *FieldStore::header(int16_t token) { - return http2::get_header(hdidx_, token, headers_); +HeaderRefs::value_type *FieldStore::header(int32_t token) { + for (auto it = headers_.rbegin(); it != headers_.rend(); ++it) { + auto &kv = *it; + if (kv.token == token) { + return &kv; + } + } + return nullptr; } -const Headers::value_type *FieldStore::header(const StringRef &name) const { - return search_header_linear(headers_, name); +const HeaderRefs::value_type *FieldStore::header(const StringRef &name) const { + return search_header_linear_backwards(headers_, name); } -void FieldStore::add_header(std::string name, std::string value) { - shrpx::add_header(header_key_prev_, buffer_size_, headers_, std::move(name), - std::move(value)); -} - -void FieldStore::add_header(std::string name, std::string value, - int16_t token) { - http2::index_header(hdidx_, token, headers_.size()); - buffer_size_ += name.size() + value.size(); - headers_.emplace_back(std::move(name), std::move(value), false, token); -} - -void FieldStore::add_header(const uint8_t *name, size_t namelen, - const uint8_t *value, size_t valuelen, - bool no_index, int16_t token) { - http2::index_header(hdidx_, token, headers_.size()); - shrpx::add_header(buffer_size_, headers_, name, namelen, value, valuelen, +void FieldStore::add_header_token(const StringRef &name, const StringRef &value, + bool no_index, int32_t token) { + shrpx::add_header(header_key_prev_, buffer_size_, headers_, name, value, no_index, token); } void FieldStore::append_last_header_key(const char *data, size_t len) { - shrpx::append_last_header_key(header_key_prev_, buffer_size_, headers_, data, - len); + shrpx::append_last_header_key(balloc_, header_key_prev_, buffer_size_, + headers_, data, len); } void FieldStore::append_last_header_value(const char *data, size_t len) { - shrpx::append_last_header_value(header_key_prev_, buffer_size_, headers_, - data, len); + shrpx::append_last_header_value(balloc_, header_key_prev_, buffer_size_, + headers_, data, len); } -void FieldStore::clear_headers() { - headers_.clear(); - http2::init_hdidx(hdidx_); -} +void FieldStore::clear_headers() { headers_.clear(); } -void FieldStore::add_trailer(const uint8_t *name, size_t namelen, - const uint8_t *value, size_t valuelen, - bool no_index, int16_t token) { - // we never index trailer fields. Header size limit should be - // applied to all header and trailer fields combined. - shrpx::add_header(buffer_size_, trailers_, name, namelen, value, valuelen, - no_index, -1); -} - -void FieldStore::add_trailer(std::string name, std::string value) { - shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, std::move(name), - std::move(value)); +void FieldStore::add_trailer_token(const StringRef &name, + const StringRef &value, bool no_index, + int32_t token) { + // Header size limit should be applied to all header and trailer + // fields combined. + shrpx::add_header(trailer_key_prev_, buffer_size_, trailers_, name, value, + no_index, token); } void FieldStore::append_last_trailer_key(const char *data, size_t len) { - shrpx::append_last_header_key(trailer_key_prev_, buffer_size_, trailers_, - data, len); + shrpx::append_last_header_key(balloc_, trailer_key_prev_, buffer_size_, + trailers_, data, len); } void FieldStore::append_last_trailer_value(const char *data, size_t len) { - shrpx::append_last_header_value(trailer_key_prev_, buffer_size_, trailers_, - data, len); + shrpx::append_last_header_value(balloc_, trailer_key_prev_, buffer_size_, + trailers_, data, len); } void Downstream::set_request_start_time( @@ -549,7 +541,7 @@ int Downstream::end_upload_data() { } void Downstream::rewrite_location_response_header( - const std::string &upstream_scheme) { + const StringRef &upstream_scheme) { auto hd = resp_.fs.header(http2::HD_LOCATION); if (!hd) { return; @@ -565,14 +557,15 @@ void Downstream::rewrite_location_response_header( return; } - auto new_uri = http2::rewrite_location_uri( - hd->value, u, request_downstream_host_, req_.authority, upstream_scheme); + auto new_uri = http2::rewrite_location_uri(balloc_, hd->value, u, + request_downstream_host_, + req_.authority, upstream_scheme); if (new_uri.empty()) { return; } - hd->value = std::move(new_uri); + hd->value = new_uri; } bool Downstream::get_chunked_response() const { return chunked_response_; } @@ -709,10 +702,10 @@ bool Downstream::get_http2_upgrade_request() const { response_state_ == INITIAL; } -const std::string &Downstream::get_http2_settings() const { +StringRef Downstream::get_http2_settings() const { auto http2_settings = req_.fs.header(http2::HD_HTTP2_SETTINGS); if (!http2_settings) { - return EMPTY_STRING; + return StringRef{}; } return http2_settings->value; } @@ -867,8 +860,8 @@ void Downstream::add_retry() { ++num_retry_; } bool Downstream::no_more_retry() const { return num_retry_ > 5; } -void Downstream::set_request_downstream_host(std::string host) { - request_downstream_host_ = std::move(host); +void Downstream::set_request_downstream_host(const StringRef &host) { + request_downstream_host_ = host; } void Downstream::set_request_pending(bool f) { request_pending_ = f; } @@ -914,4 +907,11 @@ void Downstream::set_assoc_stream_id(int32_t stream_id) { int32_t Downstream::get_assoc_stream_id() const { return assoc_stream_id_; } +BlockAllocator &Downstream::get_block_allocator() { return balloc_; } + +void Downstream::add_rcbuf(nghttp2_rcbuf *rcbuf) { + nghttp2_rcbuf_incref(rcbuf); + rcbufs_.push_back(rcbuf); +} + } // namespace shrpx diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index a1f32760..0d987517 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -40,6 +40,7 @@ #include "shrpx_io_control.h" #include "http2.h" #include "memchunk.h" +#include "allocator.h" using namespace nghttp2; @@ -51,19 +52,19 @@ struct BlockedLink; class FieldStore { public: - FieldStore(size_t headers_initial_capacity) + FieldStore(BlockAllocator &balloc, size_t headers_initial_capacity) : content_length(-1), + balloc_(balloc), buffer_size_(0), header_key_prev_(false), trailer_key_prev_(false) { - http2::init_hdidx(hdidx_); headers_.reserve(headers_initial_capacity); } - const Headers &headers() const { return headers_; } - const Headers &trailers() const { return trailers_; } + const HeaderRefs &headers() const { return headers_; } + const HeaderRefs &trailers() const { return trailers_; } - Headers &headers() { return headers_; } + HeaderRefs &headers() { return headers_; } const void add_extra_buffer_size(size_t n) { buffer_size_ += n; } size_t buffer_size() const { return buffer_size_; } @@ -74,33 +75,29 @@ public: // multiple header have |name| as name, return last occurrence from // the beginning. If no such header is found, returns nullptr. // This function must be called after headers are indexed - const Headers::value_type *header(int16_t token) const; - Headers::value_type *header(int16_t token); + const HeaderRefs::value_type *header(int32_t token) const; + HeaderRefs::value_type *header(int32_t token); // Returns pointer to the header field with the name |name|. If no // such header is found, returns nullptr. - const Headers::value_type *header(const StringRef &name) const; + const HeaderRefs::value_type *header(const StringRef &name) const; - void add_header(std::string name, std::string value); - void add_header(std::string name, std::string value, int16_t token); - void add_header(const uint8_t *name, size_t namelen, const uint8_t *value, - size_t valuelen, bool no_index, int16_t token); + void add_header_token(const StringRef &name, const StringRef &value, + bool no_index, int32_t token); void append_last_header_key(const char *data, size_t len); void append_last_header_value(const char *data, size_t len); bool header_key_prev() const { return header_key_prev_; } - // Lower the header field names and indexes header fields. If there - // is any invalid headers (e.g., multiple Content-Length having - // different values), returns -1. - int index_headers(); + // Parses content-length, and records it in the field. If there are + // multiple Content-Length, returns -1. + int parse_content_length(); // Empties headers. void clear_headers(); - void add_trailer(const uint8_t *name, size_t namelen, const uint8_t *value, - size_t valuelen, bool no_index, int16_t token); - void add_trailer(std::string name, std::string value); + void add_trailer_token(const StringRef &name, const StringRef &value, + bool no_index, int32_t token); void append_last_trailer_key(const char *data, size_t len); void append_last_trailer_value(const char *data, size_t len); @@ -111,11 +108,11 @@ public: int64_t content_length; private: - Headers headers_; + BlockAllocator &balloc_; + HeaderRefs headers_; // trailer fields. For HTTP/1.1, trailer fields are only included // with chunked encoding. For HTTP/2, there is no such limit. - Headers trailers_; - http2::HeaderIndex hdidx_; + HeaderRefs trailers_; // Sum of the length of name and value in headers_ and trailers_. // This could also be increased by add_extra_buffer_size() to take // into account for request URI in case of HTTP/1.x request. @@ -125,8 +122,8 @@ private: }; struct Request { - Request() - : fs(16), + Request(BlockAllocator &balloc) + : fs(balloc, 16), recv_body_length(0), unconsumed_body_length(0), method(-1), @@ -146,17 +143,17 @@ struct Request { FieldStore fs; // Request scheme. For HTTP/2, this is :scheme header field value. // For HTTP/1.1, this is deduced from URI or connection. - std::string scheme; + StringRef scheme; // Request authority. This is HTTP/2 :authority header field value // or host header field value. We may deduce it from absolute-form // HTTP/1 request. We also store authority-form HTTP/1 request. // This could be empty if request comes from HTTP/1.0 without Host // header field and origin-form. - std::string authority; + StringRef authority; // Request path, including query component. For HTTP/1.1, this is // request-target. For HTTP/2, this is :path header field value. // For CONNECT request, this is empty. - std::string path; + StringRef path; // the length of request body received so far int64_t recv_body_length; // The number of bytes not consumed by the application yet. @@ -181,8 +178,8 @@ struct Request { }; struct Response { - Response() - : fs(32), + Response(BlockAllocator &balloc) + : fs(balloc, 32), recv_body_length(0), unconsumed_body_length(0), http_status(0), @@ -247,7 +244,7 @@ public: // Returns true if the request is HTTP Upgrade for HTTP/2 bool get_http2_upgrade_request() const; // Returns the value of HTTP2-Settings request header field. - const std::string &get_http2_settings() const; + StringRef get_http2_settings() const; // downstream request API const Request &request() const { return req_; } @@ -274,7 +271,7 @@ public: // Validates that received request body length and content-length // matches. bool validate_request_recv_body_length() const; - void set_request_downstream_host(std::string host); + void set_request_downstream_host(const StringRef &host); bool expect_response_body() const; enum { INITIAL, @@ -305,7 +302,7 @@ public: Response &response() { return resp_; } // Rewrites the location response header field. - void rewrite_location_response_header(const std::string &upstream_scheme); + void rewrite_location_response_header(const StringRef &upstream_scheme); bool get_chunked_response() const; void set_chunked_response(bool f); @@ -375,6 +372,10 @@ public: DefaultMemchunks pop_response_buf(); + BlockAllocator &get_block_allocator(); + + void add_rcbuf(nghttp2_rcbuf *rcbuf); + enum { EVENT_ERROR = 0x1, EVENT_TIMEOUT = 0x2, @@ -394,6 +395,10 @@ public: int64_t response_sent_body_length; private: + BlockAllocator balloc_; + + std::vector rcbufs_; + Request req_; Response resp_; @@ -402,7 +407,7 @@ private: // host we requested to downstream. This is used to rewrite // location header field to decide the location should be rewritten // or not. - std::string request_downstream_host_; + StringRef request_downstream_host_; DefaultMemchunks request_buf_; DefaultMemchunks response_buf_; diff --git a/src/shrpx_downstream_connection.cc b/src/shrpx_downstream_connection.cc index 77dcd442..2c6d2c27 100644 --- a/src/shrpx_downstream_connection.cc +++ b/src/shrpx_downstream_connection.cc @@ -26,12 +26,11 @@ #include "shrpx_client_handler.h" #include "shrpx_downstream.h" -#include "shrpx_downstream_connection_pool.h" namespace shrpx { -DownstreamConnection::DownstreamConnection(DownstreamConnectionPool *dconn_pool) - : dconn_pool_(dconn_pool), client_handler_(nullptr), downstream_(nullptr) {} +DownstreamConnection::DownstreamConnection() + : client_handler_(nullptr), downstream_(nullptr) {} DownstreamConnection::~DownstreamConnection() {} @@ -45,8 +44,4 @@ ClientHandler *DownstreamConnection::get_client_handler() { Downstream *DownstreamConnection::get_downstream() { return downstream_; } -DownstreamConnectionPool *DownstreamConnection::get_dconn_pool() const { - return dconn_pool_; -} - } // namespace shrpx diff --git a/src/shrpx_downstream_connection.h b/src/shrpx_downstream_connection.h index 3d7aba0a..7935099a 100644 --- a/src/shrpx_downstream_connection.h +++ b/src/shrpx_downstream_connection.h @@ -34,11 +34,11 @@ namespace shrpx { class ClientHandler; class Upstream; class Downstream; -class DownstreamConnectionPool; +struct DownstreamAddrGroup; class DownstreamConnection { public: - DownstreamConnection(DownstreamConnectionPool *dconn_pool); + DownstreamConnection(); virtual ~DownstreamConnection(); virtual int attach_downstream(Downstream *downstream) = 0; virtual void detach_downstream(Downstream *downstream) = 0; @@ -56,18 +56,17 @@ public: virtual int on_timeout() { return 0; } virtual void on_upstream_change(Upstream *uptream) = 0; - virtual size_t get_group() const = 0; // true if this object is poolable. virtual bool poolable() const = 0; + virtual DownstreamAddrGroup *get_downstream_addr_group() const = 0; + void set_client_handler(ClientHandler *client_handler); ClientHandler *get_client_handler(); Downstream *get_downstream(); - DownstreamConnectionPool *get_dconn_pool() const; protected: - DownstreamConnectionPool *dconn_pool_; ClientHandler *client_handler_; Downstream *downstream_; }; diff --git a/src/shrpx_downstream_connection_pool.cc b/src/shrpx_downstream_connection_pool.cc index eb9b1c8c..c58ed81a 100644 --- a/src/shrpx_downstream_connection_pool.cc +++ b/src/shrpx_downstream_connection_pool.cc @@ -27,42 +27,35 @@ namespace shrpx { -DownstreamConnectionPool::DownstreamConnectionPool(size_t num_groups) - : gpool_(num_groups) {} +DownstreamConnectionPool::DownstreamConnectionPool() {} DownstreamConnectionPool::~DownstreamConnectionPool() { - for (auto &pool : gpool_) { - for (auto dconn : pool) { - delete dconn; - } + for (auto dconn : pool_) { + delete dconn; } } void DownstreamConnectionPool::add_downstream_connection( std::unique_ptr dconn) { - auto group = dconn->get_group(); - assert(gpool_.size() > group); - gpool_[group].insert(dconn.release()); + pool_.insert(dconn.release()); } std::unique_ptr -DownstreamConnectionPool::pop_downstream_connection(size_t group) { - assert(gpool_.size() > group); - auto &pool = gpool_[group]; - if (pool.empty()) { +DownstreamConnectionPool::pop_downstream_connection() { + if (pool_.empty()) { return nullptr; } - auto dconn = std::unique_ptr(*std::begin(pool)); - pool.erase(std::begin(pool)); + auto it = std::begin(pool_); + auto dconn = std::unique_ptr(*it); + pool_.erase(it); + return dconn; } void DownstreamConnectionPool::remove_downstream_connection( DownstreamConnection *dconn) { - auto group = dconn->get_group(); - assert(gpool_.size() > group); - gpool_[group].erase(dconn); + pool_.erase(dconn); delete dconn; } diff --git a/src/shrpx_downstream_connection_pool.h b/src/shrpx_downstream_connection_pool.h index 1fb889bb..c2edce45 100644 --- a/src/shrpx_downstream_connection_pool.h +++ b/src/shrpx_downstream_connection_pool.h @@ -36,15 +36,15 @@ class DownstreamConnection; class DownstreamConnectionPool { public: - DownstreamConnectionPool(size_t num_groups); + DownstreamConnectionPool(); ~DownstreamConnectionPool(); void add_downstream_connection(std::unique_ptr dconn); - std::unique_ptr pop_downstream_connection(size_t group); + std::unique_ptr pop_downstream_connection(); void remove_downstream_connection(DownstreamConnection *dconn); private: - std::vector> gpool_; + std::set pool_; }; } // namespace shrpx diff --git a/src/shrpx_downstream_queue.cc b/src/shrpx_downstream_queue.cc index 893effb9..91e88cff 100644 --- a/src/shrpx_downstream_queue.cc +++ b/src/shrpx_downstream_queue.cc @@ -71,14 +71,11 @@ DownstreamQueue::find_host_entry(const std::string &host) { return (*itr).second; } -const std::string & -DownstreamQueue::make_host_key(const std::string &host) const { - static std::string empty_key; - return unified_host_ ? empty_key : host; +std::string DownstreamQueue::make_host_key(const StringRef &host) const { + return unified_host_ ? "" : host.str(); } -const std::string & -DownstreamQueue::make_host_key(Downstream *downstream) const { +std::string DownstreamQueue::make_host_key(Downstream *downstream) const { return make_host_key(downstream->request().authority); } @@ -99,7 +96,7 @@ void DownstreamQueue::mark_blocked(Downstream *downstream) { ent.blocked.append(link); } -bool DownstreamQueue::can_activate(const std::string &host) const { +bool DownstreamQueue::can_activate(const StringRef &host) const { auto itr = host_entries_.find(make_host_key(host)); if (itr == std::end(host_entries_)) { return true; @@ -127,7 +124,7 @@ Downstream *DownstreamQueue::remove_and_get_blocked(Downstream *downstream, downstreams_.remove(downstream); - auto &host = make_host_key(downstream); + auto host = make_host_key(downstream); auto &ent = find_host_entry(host); if (downstream->get_dispatch_state() == Downstream::DISPATCH_ACTIVE) { diff --git a/src/shrpx_downstream_queue.h b/src/shrpx_downstream_queue.h index 755af10d..47e8555a 100644 --- a/src/shrpx_downstream_queue.h +++ b/src/shrpx_downstream_queue.h @@ -77,7 +77,7 @@ public: void mark_blocked(Downstream *downstream); // Returns true if we can make downstream connection to given // |host|. - bool can_activate(const std::string &host) const; + bool can_activate(const StringRef &host) const; // Removes and frees |downstream| object. If |downstream| is in // Downstream::DISPATCH_ACTIVE, and |next_blocked| is true, this // function may return Downstream object with the same target host @@ -87,8 +87,8 @@ public: bool next_blocked = true); Downstream *get_downstreams() const; HostEntry &find_host_entry(const std::string &host); - const std::string &make_host_key(const std::string &host) const; - const std::string &make_host_key(Downstream *downstream) const; + std::string make_host_key(const StringRef &host) const; + std::string make_host_key(Downstream *downstream) const; private: // Per target host structure to keep track of the number of diff --git a/src/shrpx_downstream_test.cc b/src/shrpx_downstream_test.cc index c7b559a4..786e9046 100644 --- a/src/shrpx_downstream_test.cc +++ b/src/shrpx_downstream_test.cc @@ -32,56 +32,62 @@ namespace shrpx { -void test_downstream_field_store_index_headers(void) { - FieldStore fs(0); - fs.add_header("1", "0"); - fs.add_header("2", "1"); - fs.add_header("Charlie", "2"); - fs.add_header("Alpha", "3"); - fs.add_header("Delta", "4"); - fs.add_header("BravO", "5"); - fs.add_header(":method", "6"); - fs.add_header(":authority", "7"); - fs.index_headers(); +void test_downstream_field_store_append_last_header(void) { + BlockAllocator balloc(4096, 4096); + FieldStore fs(balloc, 0); + fs.add_header_token(StringRef::from_lit("alpha"), StringRef{}, false, -1); + auto bravo = StringRef::from_lit("BRAVO"); + fs.append_last_header_key(bravo.c_str(), bravo.size()); + auto charlie = StringRef::from_lit("Charlie"); + fs.append_last_header_value(charlie.c_str(), charlie.size()); + auto delta = StringRef::from_lit("deltA"); + fs.append_last_header_value(delta.c_str(), delta.size()); + fs.add_header_token(StringRef::from_lit("echo"), + StringRef::from_lit("foxtrot"), false, -1); - auto ans = Headers{{"1", "0"}, - {"2", "1"}, - {"charlie", "2"}, - {"alpha", "3"}, - {"delta", "4"}, - {"bravo", "5"}, - {":method", "6"}, - {":authority", "7"}}; + auto ans = HeaderRefs{ + {StringRef::from_lit("alphabravo"), StringRef::from_lit("CharliedeltA")}, + {StringRef::from_lit("echo"), StringRef::from_lit("foxtrot")}}; CU_ASSERT(ans == fs.headers()); } void test_downstream_field_store_header(void) { - FieldStore fs(0); - fs.add_header("alpha", "0"); - fs.add_header(":authority", "1"); - fs.add_header("content-length", "2"); - fs.index_headers(); + BlockAllocator balloc(4096, 4096); + FieldStore fs(balloc, 0); + fs.add_header_token(StringRef::from_lit("alpha"), StringRef::from_lit("0"), + false, -1); + fs.add_header_token(StringRef::from_lit(":authority"), + StringRef::from_lit("1"), false, http2::HD__AUTHORITY); + fs.add_header_token(StringRef::from_lit("content-length"), + StringRef::from_lit("2"), false, + http2::HD_CONTENT_LENGTH); // By token - CU_ASSERT(Header(":authority", "1") == *fs.header(http2::HD__AUTHORITY)); + CU_ASSERT(HeaderRef(StringRef{":authority"}, StringRef{"1"}) == + *fs.header(http2::HD__AUTHORITY)); CU_ASSERT(nullptr == fs.header(http2::HD__METHOD)); // By name - CU_ASSERT(Header("alpha", "0") == *fs.header("alpha")); - CU_ASSERT(nullptr == fs.header("bravo")); + CU_ASSERT(HeaderRef(StringRef{"alpha"}, StringRef{"0"}) == + *fs.header(StringRef::from_lit("alpha"))); + CU_ASSERT(nullptr == fs.header(StringRef::from_lit("bravo"))); } void test_downstream_crumble_request_cookie(void) { Downstream d(nullptr, nullptr, 0); auto &req = d.request(); - req.fs.add_header(":method", "get"); - req.fs.add_header(":path", "/"); - auto val = "alpha; bravo; ; ;; charlie;;"; - req.fs.add_header( - reinterpret_cast("cookie"), sizeof("cookie") - 1, - reinterpret_cast(val), strlen(val), true, -1); - req.fs.add_header("cookie", ";delta"); - req.fs.add_header("cookie", "echo"); + req.fs.add_header_token(StringRef::from_lit(":method"), + StringRef::from_lit("get"), false, -1); + req.fs.add_header_token(StringRef::from_lit(":path"), + StringRef::from_lit("/"), false, -1); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit("alpha; bravo; ; ;; charlie;;"), + true, http2::HD_COOKIE); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit(";delta"), false, + http2::HD_COOKIE); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit("echo"), false, http2::HD_COOKIE); std::vector nva; d.crumble_request_cookie(nva); @@ -91,19 +97,20 @@ void test_downstream_crumble_request_cookie(void) { CU_ASSERT(5 == nva.size()); CU_ASSERT(5 == num_cookies); - Headers cookies; + HeaderRefs cookies; std::transform(std::begin(nva), std::end(nva), std::back_inserter(cookies), [](const nghttp2_nv &nv) { - return Header(std::string(nv.name, nv.name + nv.namelen), - std::string(nv.value, nv.value + nv.valuelen), - nv.flags & NGHTTP2_NV_FLAG_NO_INDEX); + return HeaderRef(StringRef{nv.name, nv.namelen}, + StringRef{nv.value, nv.valuelen}, + nv.flags & NGHTTP2_NV_FLAG_NO_INDEX); }); - Headers ans = {{"cookie", "alpha"}, - {"cookie", "bravo"}, - {"cookie", "charlie"}, - {"cookie", "delta"}, - {"cookie", "echo"}}; + HeaderRefs ans = { + {StringRef::from_lit("cookie"), StringRef::from_lit("alpha")}, + {StringRef::from_lit("cookie"), StringRef::from_lit("bravo")}, + {StringRef::from_lit("cookie"), StringRef::from_lit("charlie")}, + {StringRef::from_lit("cookie"), StringRef::from_lit("delta")}, + {StringRef::from_lit("cookie"), StringRef::from_lit("echo")}}; CU_ASSERT(ans == cookies); CU_ASSERT(cookies[0].no_index); @@ -114,12 +121,23 @@ void test_downstream_crumble_request_cookie(void) { void test_downstream_assemble_request_cookie(void) { Downstream d(nullptr, nullptr, 0); auto &req = d.request(); - req.fs.add_header(":method", "get"); - req.fs.add_header(":path", "/"); - req.fs.add_header("cookie", "alpha"); - req.fs.add_header("cookie", "bravo;"); - req.fs.add_header("cookie", "charlie; "); - req.fs.add_header("cookie", "delta;;"); + + req.fs.add_header_token(StringRef::from_lit(":method"), + StringRef::from_lit("get"), false, -1); + req.fs.add_header_token(StringRef::from_lit(":path"), + StringRef::from_lit("/"), false, -1); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit("alpha"), false, + http2::HD_COOKIE); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit("bravo;"), false, + http2::HD_COOKIE); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit("charlie; "), false, + http2::HD_COOKIE); + req.fs.add_header_token(StringRef::from_lit("cookie"), + StringRef::from_lit("delta;;"), false, + http2::HD_COOKIE); CU_ASSERT("alpha; bravo; charlie; delta" == d.assemble_request_cookie()); } @@ -127,11 +145,12 @@ void test_downstream_rewrite_location_response_header(void) { Downstream d(nullptr, nullptr, 0); auto &req = d.request(); auto &resp = d.response(); - d.set_request_downstream_host("localhost2"); - req.authority = "localhost:8443"; - resp.fs.add_header("location", "http://localhost2:3000/"); - resp.fs.index_headers(); - d.rewrite_location_response_header("https"); + d.set_request_downstream_host(StringRef::from_lit("localhost2")); + req.authority = StringRef::from_lit("localhost:8443"); + resp.fs.add_header_token(StringRef::from_lit("location"), + StringRef::from_lit("http://localhost2:3000/"), + false, http2::HD_LOCATION); + d.rewrite_location_response_header(StringRef::from_lit("https")); auto location = resp.fs.header(http2::HD_LOCATION); CU_ASSERT("https://localhost:8443/" == (*location).value); } diff --git a/src/shrpx_downstream_test.h b/src/shrpx_downstream_test.h index 13b33744..f7b1e603 100644 --- a/src/shrpx_downstream_test.h +++ b/src/shrpx_downstream_test.h @@ -31,7 +31,7 @@ namespace shrpx { -void test_downstream_field_store_index_headers(void); +void test_downstream_field_store_append_last_header(void); void test_downstream_field_store_header(void); void test_downstream_crumble_request_cookie(void); void test_downstream_assemble_request_cookie(void); diff --git a/src/shrpx_http.cc b/src/shrpx_http.cc index 86870435..c6a70e06 100644 --- a/src/shrpx_http.cc +++ b/src/shrpx_http.cc @@ -50,70 +50,75 @@ std::string create_error_html(unsigned int status_code) { return res; } -std::string create_via_header_value(int major, int minor) { - std::string hdrs; - hdrs += static_cast(major + '0'); - if (major < 2) { - hdrs += '.'; - hdrs += static_cast(minor + '0'); +StringRef create_forwarded(BlockAllocator &balloc, int params, + const StringRef &node_by, const StringRef &node_for, + const StringRef &host, const StringRef &proto) { + size_t len = 0; + if ((params & FORWARDED_BY) && !node_by.empty()) { + len += str_size("by=\"") + node_by.size() + str_size("\";"); + } + if ((params & FORWARDED_FOR) && !node_for.empty()) { + len += str_size("for=\"") + node_for.size() + str_size("\";"); + } + if ((params & FORWARDED_HOST) && !host.empty()) { + len += str_size("host=\"") + host.size() + str_size("\";"); + } + if ((params & FORWARDED_PROTO) && !proto.empty()) { + len += str_size("proto=") + proto.size() + str_size(";"); } - hdrs += " nghttpx"; - return hdrs; -} -std::string create_forwarded(int params, const StringRef &node_by, - const std::string &node_for, - const std::string &host, - const std::string &proto) { - std::string res; + auto iov = make_byte_ref(balloc, len + 1); + auto p = iov.base; + if ((params & FORWARDED_BY) && !node_by.empty()) { // This must be quoted-string unless it is obfuscated version // (which starts with "_") or some special value (e.g., // "localhost" for UNIX domain socket), since ':' is not allowed // in token. ':' is used to separate host and port. if (node_by[0] == '_' || node_by[0] == 'l') { - res += "by="; - res += node_by; - res += ";"; + p = util::copy_lit(p, "by="); + p = std::copy(std::begin(node_by), std::end(node_by), p); + p = util::copy_lit(p, ";"); } else { - res += "by=\""; - res += node_by; - res += "\";"; + p = util::copy_lit(p, "by=\""); + p = std::copy(std::begin(node_by), std::end(node_by), p); + p = util::copy_lit(p, "\";"); } } if ((params & FORWARDED_FOR) && !node_for.empty()) { // We only quote IPv6 literal address only, which starts with '['. if (node_for[0] == '[') { - res += "for=\""; - res += node_for; - res += "\";"; + p = util::copy_lit(p, "for=\""); + p = std::copy(std::begin(node_for), std::end(node_for), p); + p = util::copy_lit(p, "\";"); } else { - res += "for="; - res += node_for; - res += ";"; + p = util::copy_lit(p, "for="); + p = std::copy(std::begin(node_for), std::end(node_for), p); + p = util::copy_lit(p, ";"); } } if ((params & FORWARDED_HOST) && !host.empty()) { // Just be quoted to skip checking characters. - res += "host=\""; - res += host; - res += "\";"; + p = util::copy_lit(p, "host=\""); + p = std::copy(std::begin(host), std::end(host), p); + p = util::copy_lit(p, "\";"); } if ((params & FORWARDED_PROTO) && !proto.empty()) { // Scheme production rule only allow characters which are all in // token. - res += "proto="; - res += proto; - res += ";"; + p = util::copy_lit(p, "proto="); + p = std::copy(std::begin(proto), std::end(proto), p); + *p++ = ';'; } - if (res.empty()) { - return res; + if (iov.base == p) { + return StringRef{}; } - res.erase(res.size() - 1); + --p; + *p = '\0'; - return res; + return StringRef{iov.base, p}; } std::string colorizeHeaders(const char *hdrs) { diff --git a/src/shrpx_http.h b/src/shrpx_http.h index bd29df02..856f192a 100644 --- a/src/shrpx_http.h +++ b/src/shrpx_http.h @@ -31,20 +31,31 @@ #include +#include "util.h" +#include "allocator.h" + namespace shrpx { namespace http { std::string create_error_html(unsigned int status_code); -std::string create_via_header_value(int major, int minor); +template +OutputIt create_via_header_value(OutputIt dst, int major, int minor) { + *dst++ = static_cast(major + '0'); + if (major < 2) { + *dst++ = '.'; + *dst++ = static_cast(minor + '0'); + } + return util::copy_lit(dst, " nghttpx"); +} // Returns generated RFC 7239 Forwarded header field value. The // |params| is bitwise-OR of zero or more of shrpx_forwarded_param // defined in shrpx_config.h. -std::string create_forwarded(int params, const StringRef &node_by, - const std::string &node_for, - const std::string &host, const std::string &proto); +StringRef create_forwarded(BlockAllocator &balloc, int params, + const StringRef &node_by, const StringRef &node_for, + const StringRef &host, const StringRef &proto); // Adds ANSI color codes to HTTP headers |hdrs|. std::string colorizeHeaders(const char *hdrs); diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index 02982b8a..73a43a8d 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -37,6 +37,7 @@ #include "shrpx_error.h" #include "shrpx_http.h" #include "shrpx_http2_session.h" +#include "shrpx_worker.h" #include "http2.h" #include "util.h" @@ -44,10 +45,8 @@ using namespace nghttp2; namespace shrpx { -Http2DownstreamConnection::Http2DownstreamConnection( - DownstreamConnectionPool *dconn_pool, Http2Session *http2session) - : DownstreamConnection(dconn_pool), - dlnext(nullptr), +Http2DownstreamConnection::Http2DownstreamConnection(Http2Session *http2session) + : dlnext(nullptr), dlprev(nullptr), http2session_(http2session), sd_(nullptr) {} @@ -98,11 +97,19 @@ int Http2DownstreamConnection::attach_downstream(Downstream *downstream) { http2session_->add_downstream_connection(this); if (http2session_->get_state() == Http2Session::DISCONNECTED) { http2session_->signal_write(); + if (http2session_->get_state() == Http2Session::DISCONNECTED) { + return -1; + } } downstream_ = downstream; downstream_->reset_downstream_rtimer(); + auto &req = downstream_->request(); + + // HTTP/2 disables HTTP Upgrade. + req.upgrade_request = false; + return 0; } @@ -259,12 +266,14 @@ int Http2DownstreamConnection::push_request_headers() { const auto &req = downstream_->request(); + auto &balloc = downstream_->get_block_allocator(); + auto &httpconf = get_config()->http; auto &http2conf = get_config()->http2; - auto no_host_rewrite = - httpconf.no_host_rewrite || get_config()->http2_proxy || - get_config()->client_proxy || req.method == HTTP_CONNECT; + auto no_host_rewrite = httpconf.no_host_rewrite || + get_config()->http2_proxy || + req.method == HTTP_CONNECT; // http2session_ has already in CONNECTED state, so we can get // addr_idx here. @@ -275,10 +284,10 @@ int Http2DownstreamConnection::push_request_headers() { auto authority = StringRef(downstream_hostport); if (no_host_rewrite && !req.authority.empty()) { - authority = StringRef(req.authority); + authority = req.authority; } - downstream_->set_request_downstream_host(authority.str()); + downstream_->set_request_downstream_host(authority); size_t num_cookies = 0; if (!http2conf.no_cookie_crumbling) { @@ -300,7 +309,7 @@ int Http2DownstreamConnection::push_request_headers() { httpconf.add_request_headers.size()); nva.push_back( - http2::make_nv_lc_nocopy(":method", http2::to_method_string(req.method))); + http2::make_nv_ls_nocopy(":method", http2::to_method_string(req.method))); if (req.method != HTTP_CONNECT) { assert(!req.scheme.empty()); @@ -338,8 +347,6 @@ int Http2DownstreamConnection::push_request_headers() { auto upstream = downstream_->get_upstream(); auto handler = upstream->get_client_handler(); - std::string forwarded_value; - auto &fwdconf = httpconf.forwarded; auto fwd = @@ -348,30 +355,28 @@ int Http2DownstreamConnection::push_request_headers() { if (fwdconf.params) { auto params = fwdconf.params; - if (get_config()->http2_proxy || get_config()->client_proxy || - req.method == HTTP_CONNECT) { + if (get_config()->http2_proxy || req.method == HTTP_CONNECT) { params &= ~FORWARDED_PROTO; } - auto value = http::create_forwarded(params, handler->get_forwarded_by(), - handler->get_forwarded_for(), - req.authority, req.scheme); + auto value = http::create_forwarded( + balloc, params, handler->get_forwarded_by(), + handler->get_forwarded_for(), req.authority, req.scheme); + if (fwd || !value.empty()) { if (fwd) { - forwarded_value = fwd->value; - - if (!value.empty()) { - forwarded_value += ", "; + if (value.empty()) { + value = fwd->value; + } else { + value = concat_string_ref(balloc, fwd->value, + StringRef::from_lit(", "), value); } } - forwarded_value += value; - - nva.push_back(http2::make_nv_ls("forwarded", forwarded_value)); + nva.push_back(http2::make_nv_ls_nocopy("forwarded", value)); } } else if (fwd) { nva.push_back(http2::make_nv_ls_nocopy("forwarded", fwd->value)); - forwarded_value = fwd->value; } auto &xffconf = httpconf.xff; @@ -379,38 +384,47 @@ int Http2DownstreamConnection::push_request_headers() { auto xff = xffconf.strip_incoming ? nullptr : req.fs.header(http2::HD_X_FORWARDED_FOR); - std::string xff_value; - if (xffconf.add) { + StringRef xff_value; + auto addr = StringRef{upstream->get_client_handler()->get_ipaddr()}; if (xff) { - xff_value = (*xff).value; - xff_value += ", "; + xff_value = concat_string_ref(balloc, xff->value, + StringRef::from_lit(", "), addr); + } else { + xff_value = addr; } - xff_value += upstream->get_client_handler()->get_ipaddr(); - nva.push_back(http2::make_nv_ls("x-forwarded-for", xff_value)); + nva.push_back(http2::make_nv_ls_nocopy("x-forwarded-for", xff_value)); } else if (xff) { - nva.push_back(http2::make_nv_ls_nocopy("x-forwarded-for", (*xff).value)); + nva.push_back(http2::make_nv_ls_nocopy("x-forwarded-for", xff->value)); } - if (!get_config()->http2_proxy && !get_config()->client_proxy && - req.method != HTTP_CONNECT) { + if (!get_config()->http2_proxy && req.method != HTTP_CONNECT) { // We use same protocol with :scheme header field nva.push_back(http2::make_nv_ls_nocopy("x-forwarded-proto", req.scheme)); } - std::string via_value; auto via = req.fs.header(http2::HD_VIA); if (httpconf.no_via) { if (via) { nva.push_back(http2::make_nv_ls_nocopy("via", (*via).value)); } } else { + size_t vialen = 16; if (via) { - via_value = (*via).value; - via_value += ", "; + vialen += via->value.size() + 2; } - via_value += http::create_via_header_value(req.http_major, req.http_minor); - nva.push_back(http2::make_nv_ls("via", via_value)); + + auto iov = make_byte_ref(balloc, vialen + 1); + auto p = iov.base; + + if (via) { + p = std::copy(std::begin(via->value), std::end(via->value), p); + p = util::copy_lit(p, ", "); + } + p = http::create_via_header_value(p, req.http_major, req.http_minor); + *p = '\0'; + + nva.push_back(http2::make_nv_ls_nocopy("via", StringRef{iov.base, p})); } auto te = req.fs.header(http2::HD_TE); @@ -554,10 +568,9 @@ int Http2DownstreamConnection::on_timeout() { return submit_rst_stream(downstream_, NGHTTP2_NO_ERROR); } -size_t Http2DownstreamConnection::get_group() const { - // HTTP/2 backend connections are managed by Http2Session object, - // and it stores group index. - return http2session_->get_group(); +DownstreamAddrGroup * +Http2DownstreamConnection::get_downstream_addr_group() const { + return http2session_->get_downstream_addr_group(); } } // namespace shrpx diff --git a/src/shrpx_http2_downstream_connection.h b/src/shrpx_http2_downstream_connection.h index 3561ac7b..6f72a884 100644 --- a/src/shrpx_http2_downstream_connection.h +++ b/src/shrpx_http2_downstream_connection.h @@ -41,8 +41,7 @@ class DownstreamConnectionPool; class Http2DownstreamConnection : public DownstreamConnection { public: - Http2DownstreamConnection(DownstreamConnectionPool *dconn_pool, - Http2Session *http2session); + Http2DownstreamConnection(Http2Session *http2session); virtual ~Http2DownstreamConnection(); virtual int attach_downstream(Downstream *downstream); virtual void detach_downstream(Downstream *downstream); @@ -60,12 +59,13 @@ public: virtual int on_timeout(); virtual void on_upstream_change(Upstream *upstream) {} - virtual size_t get_group() const; // This object is not poolable because we dont' have facility to // migrate to another Http2Session object. virtual bool poolable() const { return false; } + virtual DownstreamAddrGroup *get_downstream_addr_group() const; + int send(); void attach_stream_data(StreamData *sd); diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index f4004b9a..84258896 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -74,6 +74,10 @@ void connchk_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents) { SSLOG(INFO, http2session) << "ping timeout"; } http2session->disconnect(); + + if (http2session->get_num_dconns() == 0) { + delete http2session; + } return; default: if (LOG_ENABLED(INFO)) { @@ -92,6 +96,9 @@ void settings_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents) { SSLOG(INFO, http2session) << "SETTINGS timeout"; if (http2session->terminate_session(NGHTTP2_SETTINGS_TIMEOUT) != 0) { http2session->disconnect(); + if (http2session->get_num_dconns() == 0) { + delete http2session; + } return; } http2session->signal_write(); @@ -109,6 +116,9 @@ void timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) { http2session->disconnect(http2session->get_state() == Http2Session::CONNECTING); + if (http2session->get_num_dconns() == 0) { + delete http2session; + } } } // namespace @@ -120,6 +130,9 @@ void readcb(struct ev_loop *loop, ev_io *w, int revents) { rv = http2session->do_read(); if (rv != 0) { http2session->disconnect(http2session->should_hard_fail()); + if (http2session->get_num_dconns() == 0) { + delete http2session; + } return; } http2session->connection_alive(); @@ -127,6 +140,9 @@ void readcb(struct ev_loop *loop, ev_io *w, int revents) { rv = http2session->do_write(); if (rv != 0) { http2session->disconnect(http2session->should_hard_fail()); + if (http2session->get_num_dconns() == 0) { + delete http2session; + } return; } } @@ -140,6 +156,9 @@ void writecb(struct ev_loop *loop, ev_io *w, int revents) { rv = http2session->do_write(); if (rv != 0) { http2session->disconnect(http2session->should_hard_fail()); + if (http2session->get_num_dconns() == 0) { + delete http2session; + } return; } http2session->reset_connection_check_timer_if_not_checking(); @@ -147,25 +166,23 @@ void writecb(struct ev_loop *loop, ev_io *w, int revents) { } // namespace Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, - ConnectBlocker *connect_blocker, Worker *worker, - size_t group, size_t idx) - : conn_(loop, -1, nullptr, worker->get_mcpool(), + Worker *worker, DownstreamAddrGroup *group) + : dlnext(nullptr), + dlprev(nullptr), + conn_(loop, -1, nullptr, worker->get_mcpool(), get_config()->conn.downstream.timeout.write, get_config()->conn.downstream.timeout.read, {}, {}, writecb, readcb, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold, - get_config()->tls.dyn_rec.idle_timeout), + get_config()->tls.dyn_rec.idle_timeout, PROTO_HTTP2), wb_(worker->get_mcpool()), worker_(worker), - connect_blocker_(connect_blocker), ssl_ctx_(ssl_ctx), + group_(group), addr_(nullptr), session_(nullptr), - group_(group), - index_(idx), state_(DISCONNECTED), connection_check_state_(CONNECTION_CHECK_NONE), flow_control_(false) { - read_ = write_ = &Http2Session::noop; on_read_ = &Http2Session::read_noop; @@ -184,7 +201,16 @@ Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, settings_timer_.data = this; } -Http2Session::~Http2Session() { disconnect(); } +Http2Session::~Http2Session() { + disconnect(); + + if (in_freelist()) { + if (LOG_ENABLED(INFO)) { + SSLOG(INFO, this) << "Removed from http2_freelist"; + } + group_->http2_freelist.remove(this); + } +} int Http2Session::disconnect(bool hard) { if (LOG_ENABLED(INFO)) { @@ -254,30 +280,57 @@ int Http2Session::disconnect(bool hard) { int Http2Session::initiate_connection() { int rv = 0; - auto &addrs = get_config()->conn.downstream.addr_groups[group_].addrs; + auto &addrs = group_->addrs; + auto worker_blocker = worker_->get_connect_blocker(); if (state_ == DISCONNECTED) { - if (connect_blocker_->blocked()) { + if (worker_blocker->blocked()) { if (LOG_ENABLED(INFO)) { - DCLOG(INFO, this) - << "Downstream connection was blocked by connect_blocker"; + SSLOG(INFO, this) + << "Worker wide backend connection was blocked temporarily"; } return -1; } - auto &next_downstream = worker_->get_dgrp(group_)->next; - addr_ = &addrs[next_downstream]; + auto &next_downstream = group_->next; + auto end = next_downstream; - if (LOG_ENABLED(INFO)) { - SSLOG(INFO, this) << "Using downstream address idx=" << next_downstream - << " out of " << addrs.size(); - } + for (;;) { + auto &addr = addrs[next_downstream]; - if (++next_downstream >= addrs.size()) { - next_downstream = 0; + if (++next_downstream >= addrs.size()) { + next_downstream = 0; + } + + auto &connect_blocker = addr.connect_blocker; + + if (connect_blocker->blocked()) { + if (LOG_ENABLED(INFO)) { + SSLOG(INFO, this) << "Backend server " + << util::to_numeric_addr(&addr.addr) + << " was not available temporarily"; + } + + if (end == next_downstream) { + return -1; + } + + continue; + } + + if (LOG_ENABLED(INFO)) { + SSLOG(INFO, this) << "Using downstream address idx=" << next_downstream + << " out of " << addrs.size(); + } + + addr_ = &addr; + + break; } } + auto &connect_blocker = addr_->connect_blocker; + const auto &proxy = get_config()->downstream_http_proxy; if (!proxy.host.empty() && state_ == DISCONNECTED) { if (LOG_ENABLED(INFO)) { @@ -288,15 +341,25 @@ int Http2Session::initiate_connection() { conn_.fd = util::create_nonblock_socket(proxy.addr.su.storage.ss_family); if (conn_.fd == -1) { - connect_blocker_->on_failure(); + auto error = errno; + SSLOG(WARN, this) << "Backend proxy socket() failed; addr=" + << util::to_numeric_addr(&proxy.addr) + << ", errno=" << error; + + worker_blocker->on_failure(); return -1; } + worker_blocker->on_success(); + rv = connect(conn_.fd, &proxy.addr.su.sa, proxy.addr.len); if (rv != 0 && errno != EINPROGRESS) { - SSLOG(ERROR, this) << "Failed to connect to the proxy " << proxy.host - << ":" << proxy.port; - connect_blocker_->on_failure(); + auto error = errno; + SSLOG(WARN, this) << "Backend proxy connect() failed; addr=" + << util::to_numeric_addr(&proxy.addr) + << ", errno=" << error; + + connect_blocker->on_failure(); return -1; } @@ -335,6 +398,8 @@ int Http2Session::initiate_connection() { return -1; } + ssl::setup_downstream_http2_alpn(ssl); + conn_.set_ssl(ssl); } @@ -348,6 +413,13 @@ int Http2Session::initiate_connection() { // at the time of this writing). SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str()); } + + auto tls_session = ssl::reuse_tls_session(addr_); + if (tls_session) { + SSL_set_session(conn_.tls.ssl, tls_session); + SSL_SESSION_free(tls_session); + } + // If state_ == PROXY_CONNECTED, we has connected to the proxy // using conn_.fd and tunnel has been established. if (state_ == DISCONNECTED) { @@ -356,16 +428,28 @@ int Http2Session::initiate_connection() { conn_.fd = util::create_nonblock_socket(addr_->addr.su.storage.ss_family); if (conn_.fd == -1) { - connect_blocker_->on_failure(); + auto error = errno; + SSLOG(WARN, this) + << "socket() failed; addr=" << util::to_numeric_addr(&addr_->addr) + << ", errno=" << error; + + worker_blocker->on_failure(); return -1; } + worker_blocker->on_success(); + rv = connect(conn_.fd, // TODO maybe not thread-safe? const_cast(&addr_->addr.su.sa), addr_->addr.len); if (rv != 0 && errno != EINPROGRESS) { - connect_blocker_->on_failure(); + auto error = errno; + SSLOG(WARN, this) << "connect() failed; addr=" + << util::to_numeric_addr(&addr_->addr) + << ", errno=" << error; + + connect_blocker->on_failure(); return -1; } @@ -383,14 +467,26 @@ int Http2Session::initiate_connection() { util::create_nonblock_socket(addr_->addr.su.storage.ss_family); if (conn_.fd == -1) { - connect_blocker_->on_failure(); + auto error = errno; + SSLOG(WARN, this) + << "socket() failed; addr=" << util::to_numeric_addr(&addr_->addr) + << ", errno=" << error; + + worker_blocker->on_failure(); return -1; } + worker_blocker->on_success(); + rv = connect(conn_.fd, const_cast(&addr_->addr.su.sa), addr_->addr.len); if (rv != 0 && errno != EINPROGRESS) { - connect_blocker_->on_failure(); + auto error = errno; + SSLOG(WARN, this) << "connect() failed; addr=" + << util::to_numeric_addr(&addr_->addr) + << ", errno=" << error; + + connect_blocker->on_failure(); return -1; } @@ -538,6 +634,17 @@ void Http2Session::remove_downstream_connection( Http2DownstreamConnection *dconn) { dconns_.remove(dconn); dconn->detach_stream_data(); + + if (LOG_ENABLED(INFO)) { + SSLOG(INFO, this) << "Remove downstream"; + } + + if (!in_freelist() && !max_concurrency_reached()) { + if (LOG_ENABLED(INFO)) { + SSLOG(INFO, this) << "Append to Http2Session freelist"; + } + group_->http2_freelist.append(this); + } } void Http2Session::remove_stream_data(StreamData *sd) { @@ -694,10 +801,9 @@ void Http2Session::stop_settings_timer() { } namespace { -int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, - const uint8_t *name, size_t namelen, - const uint8_t *value, size_t valuelen, uint8_t flags, - void *user_data) { +int on_header_callback2(nghttp2_session *session, const nghttp2_frame *frame, + nghttp2_rcbuf *name, nghttp2_rcbuf *value, + uint8_t flags, void *user_data) { auto http2session = static_cast(user_data); auto sd = static_cast( nghttp2_session_get_stream_user_data(session, frame->hd.stream_id)); @@ -709,6 +815,9 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, return 0; } + auto namebuf = nghttp2_rcbuf_get_buf(name); + auto valuebuf = nghttp2_rcbuf_get_buf(value); + auto &resp = downstream->response(); auto &httpconf = get_config()->http; @@ -717,13 +826,14 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, auto trailer = frame->headers.cat == NGHTTP2_HCAT_HEADERS && !downstream->get_expect_final_response(); - if (resp.fs.buffer_size() + namelen + valuelen > + if (resp.fs.buffer_size() + namebuf.len + valuebuf.len > httpconf.response_header_field_buffer || resp.fs.num_fields() >= httpconf.max_response_header_fields) { if (LOG_ENABLED(INFO)) { - DLOG(INFO, downstream) << "Too large or many header field size=" - << resp.fs.buffer_size() + namelen + valuelen - << ", num=" << resp.fs.num_fields() + 1; + DLOG(INFO, downstream) + << "Too large or many header field size=" + << resp.fs.buffer_size() + namebuf.len + valuebuf.len + << ", num=" << resp.fs.num_fields() + 1; } if (trailer) { @@ -735,17 +845,23 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; } + auto token = http2::lookup_token(namebuf.base, namebuf.len); + auto no_index = flags & NGHTTP2_NV_FLAG_NO_INDEX; + + downstream->add_rcbuf(name); + downstream->add_rcbuf(value); + if (trailer) { // just store header fields for trailer part - resp.fs.add_trailer(name, namelen, value, valuelen, - flags & NGHTTP2_NV_FLAG_NO_INDEX, -1); + resp.fs.add_trailer_token(StringRef{namebuf.base, namebuf.len}, + StringRef{valuebuf.base, valuebuf.len}, + no_index, token); return 0; } - auto token = http2::lookup_token(name, namelen); - - resp.fs.add_header(name, namelen, value, valuelen, - flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + resp.fs.add_header_token(StringRef{namebuf.base, namebuf.len}, + StringRef{valuebuf.base, valuebuf.len}, no_index, + token); return 0; } case NGHTTP2_PUSH_PROMISE: { @@ -759,27 +875,35 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, auto promised_downstream = promised_sd->dconn->get_downstream(); + auto namebuf = nghttp2_rcbuf_get_buf(name); + auto valuebuf = nghttp2_rcbuf_get_buf(value); + assert(promised_downstream); auto &promised_req = promised_downstream->request(); // We use request header limit for PUSH_PROMISE - if (promised_req.fs.buffer_size() + namelen + valuelen > + if (promised_req.fs.buffer_size() + namebuf.len + valuebuf.len > httpconf.request_header_field_buffer || promised_req.fs.num_fields() >= httpconf.max_request_header_fields) { if (LOG_ENABLED(INFO)) { DLOG(INFO, downstream) << "Too large or many header field size=" - << promised_req.fs.buffer_size() + namelen + valuelen + << promised_req.fs.buffer_size() + namebuf.len + valuebuf.len << ", num=" << promised_req.fs.num_fields() + 1; } return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; } - auto token = http2::lookup_token(name, namelen); - promised_req.fs.add_header(name, namelen, value, valuelen, - flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + promised_downstream->add_rcbuf(name); + promised_downstream->add_rcbuf(value); + + auto token = http2::lookup_token(namebuf.base, namebuf.len); + promised_req.fs.add_header_token(StringRef{namebuf.base, namebuf.len}, + StringRef{valuebuf.base, valuebuf.len}, + flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + return 0; } } @@ -926,8 +1050,9 @@ int on_response_headers(Http2Session *http2session, Downstream *downstream, // Otherwise, use chunked encoding to keep upstream connection // open. In HTTP2, we are supporsed not to receive // transfer-encoding. - resp.fs.add_header("transfer-encoding", "chunked", - http2::HD_TRANSFER_ENCODING); + resp.fs.add_header_token(StringRef::from_lit("transfer-encoding"), + StringRef::from_lit("chunked"), false, + http2::HD_TRANSFER_ENCODING); downstream->set_chunked_response(true); } } @@ -1286,8 +1411,8 @@ nghttp2_session_callbacks *create_http2_downstream_callbacks() { nghttp2_session_callbacks_set_on_frame_not_send_callback( callbacks, on_frame_not_send_callback); - nghttp2_session_callbacks_set_on_header_callback(callbacks, - on_header_callback); + nghttp2_session_callbacks_set_on_header_callback2(callbacks, + on_header_callback2); nghttp2_session_callbacks_set_on_begin_headers_callback( callbacks, on_begin_headers_callback); @@ -1345,13 +1470,12 @@ int Http2Session::connection_made() { std::array entry; size_t nentry = 2; entry[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS; - entry[0].value = http2conf.max_concurrent_streams; + entry[0].value = http2conf.downstream.max_concurrent_streams; entry[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE; entry[1].value = (1 << http2conf.downstream.window_bits) - 1; - if (http2conf.no_server_push || get_config()->http2_proxy || - get_config()->client_proxy) { + if (http2conf.no_server_push || get_config()->http2_proxy) { entry[nentry].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH; entry[nentry].value = 0; ++nentry; @@ -1612,11 +1736,20 @@ int Http2Session::read_noop(const uint8_t *data, size_t datalen) { return 0; } int Http2Session::write_noop() { return 0; } int Http2Session::connected() { + auto &connect_blocker = addr_->connect_blocker; + if (!util::check_socket_connected(conn_.fd)) { + if (LOG_ENABLED(INFO)) { + SSLOG(INFO, this) << "Backend connect failed; addr=" + << util::to_numeric_addr(&addr_->addr); + } + + connect_blocker->on_failure(); + return -1; } - connect_blocker_->on_success(); + connect_blocker->on_success(); if (LOG_ENABLED(INFO)) { SSLOG(INFO, this) << "Connection established"; @@ -1728,6 +1861,13 @@ int Http2Session::tls_handshake() { return -1; } + if (!SSL_session_reused(conn_.tls.ssl)) { + auto tls_session = SSL_get0_session(conn_.tls.ssl); + if (tls_session) { + ssl::try_cache_tls_session(addr_, tls_session, ev_now(conn_.loop)); + } + } + read_ = &Http2Session::read_tls; write_ = &Http2Session::write_tls; @@ -1816,11 +1956,7 @@ bool Http2Session::should_hard_fail() const { } } -const DownstreamAddr *Http2Session::get_addr() const { return addr_; } - -size_t Http2Session::get_group() const { return group_; } - -size_t Http2Session::get_index() const { return index_; } +DownstreamAddr *Http2Session::get_addr() const { return addr_; } int Http2Session::handle_downstream_push_promise(Downstream *downstream, int32_t promised_stream_id) { @@ -1839,10 +1975,8 @@ int Http2Session::handle_downstream_push_promise(Downstream *downstream, // promised_downstream->get_stream() still returns 0. auto handler = upstream->get_client_handler(); - auto worker = handler->get_worker(); - auto promised_dconn = - make_unique(worker->get_dconn_pool(), this); + auto promised_dconn = make_unique(this); promised_dconn->set_client_handler(handler); auto ptr = promised_dconn.get(); @@ -1867,6 +2001,8 @@ int Http2Session::handle_downstream_push_promise_complete( Downstream *downstream, Downstream *promised_downstream) { auto &promised_req = promised_downstream->request(); + auto &promised_balloc = promised_downstream->get_block_allocator(); + auto authority = promised_req.fs.header(http2::HD__AUTHORITY); auto path = promised_req.fs.header(http2::HD__PATH); auto method = promised_req.fs.header(http2::HD__METHOD); @@ -1888,16 +2024,19 @@ int Http2Session::handle_downstream_push_promise_complete( // TODO Rewrite authority if we enabled rewrite host. But we // really don't know how to rewrite host. Should we use the same // host in associated stream? - promised_req.authority = http2::value_to_str(authority); + if (authority) { + promised_req.authority = authority->value; + } promised_req.method = method_token; // libnghttp2 ensures that we don't have CONNECT method in // PUSH_PROMISE, and guarantees that :scheme exists. - promised_req.scheme = http2::value_to_str(scheme); + if (scheme) { + promised_req.scheme = scheme->value; + } // For server-wide OPTIONS request, path is empty. if (method_token != HTTP_OPTIONS || path->value != "*") { - promised_req.path = http2::rewrite_clean_path(std::begin(path->value), - std::end(path->value)); + promised_req.path = http2::rewrite_clean_path(promised_balloc, path->value); } promised_downstream->inspect_http2_request(); @@ -1914,4 +2053,29 @@ int Http2Session::handle_downstream_push_promise_complete( return 0; } +size_t Http2Session::get_num_dconns() const { return dconns_.size(); } + +bool Http2Session::in_freelist() const { + return dlnext != nullptr || dlprev != nullptr || + group_->http2_freelist.head == this || + group_->http2_freelist.tail == this; +} + +bool Http2Session::max_concurrency_reached(size_t extra) const { + if (!session_) { + return dconns_.size() + extra >= 100; + } + + // If session does not allow further requests, it effectively means + // that maximum concurrency is reached. + return !nghttp2_session_check_request_allowed(session_) || + dconns_.size() + extra >= + nghttp2_session_get_remote_settings( + session_, NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS); +} + +DownstreamAddrGroup *Http2Session::get_downstream_addr_group() const { + return group_; +} + } // namespace shrpx diff --git a/src/shrpx_http2_session.h b/src/shrpx_http2_session.h index a858b1bc..ecea8886 100644 --- a/src/shrpx_http2_session.h +++ b/src/shrpx_http2_session.h @@ -48,7 +48,8 @@ namespace shrpx { class Http2DownstreamConnection; class Worker; -class ConnectBlocker; +struct DownstreamAddrGroup; +struct DownstreamAddr; struct StreamData { StreamData *dlnext, *dlprev; @@ -57,9 +58,8 @@ struct StreamData { class Http2Session { public: - Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, - ConnectBlocker *connect_blocker, Worker *worker, size_t group, - size_t idx); + Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx, Worker *worker, + DownstreamAddrGroup *group); ~Http2Session(); // If hard is true, all pending requests are abandoned and @@ -147,17 +147,31 @@ public: void submit_pending_requests(); - const DownstreamAddr *get_addr() const; + DownstreamAddr *get_addr() const; - size_t get_group() const; - - size_t get_index() const; + DownstreamAddrGroup *get_downstream_addr_group() const; int handle_downstream_push_promise(Downstream *downstream, int32_t promised_stream_id); int handle_downstream_push_promise_complete(Downstream *downstream, Downstream *promised_downstream); + // Returns number of downstream connections, including pushed + // streams. + size_t get_num_dconns() const; + + // Returns true if this object is included in freelist. See + // DownstreamAddrGroup object. + bool in_freelist() const; + + // Returns true if the maximum concurrency is reached. In other + // words, the number of currently participated streams in this + // session is equal or greater than the max concurrent streams limit + // advertised by server. If |extra| is nonzero, it is added to the + // number of current concurrent streams when comparing against + // server initiated concurrency limit. + bool max_concurrency_reached(size_t extra = 0) const; + enum { // Disconnected DISCONNECTED, @@ -186,6 +200,8 @@ public: using ReadBuf = Buffer<8_k>; + Http2Session *dlnext, *dlprev; + private: Connection conn_; DefaultMemchunks wb_; @@ -203,16 +219,12 @@ private: // Used to parse the response from HTTP proxy std::unique_ptr proxy_htp_; Worker *worker_; - ConnectBlocker *connect_blocker_; // NULL if no TLS is configured SSL_CTX *ssl_ctx_; + DownstreamAddrGroup *group_; // Address of remote endpoint - const DownstreamAddr *addr_; + DownstreamAddr *addr_; nghttp2_session *session_; - size_t group_; - // index inside group, this is used to pin frontend to certain - // HTTP/2 backend for better throughput. - size_t index_; int state_; int connection_check_state_; bool flow_control_; diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 7e909a25..c1bca902 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -108,7 +108,7 @@ int on_stream_close_callback(nghttp2_session *session, int32_t stream_id, int Http2Upstream::upgrade_upstream(HttpsUpstream *http) { int rv; - auto http2_settings = http->get_downstream()->get_http2_settings(); + auto http2_settings = http->get_downstream()->get_http2_settings().str(); util::to_base64(http2_settings); auto settings_payload = @@ -154,13 +154,15 @@ void Http2Upstream::stop_settings_timer() { } namespace { -int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, - const uint8_t *name, size_t namelen, - const uint8_t *value, size_t valuelen, uint8_t flags, - void *user_data) { +int on_header_callback2(nghttp2_session *session, const nghttp2_frame *frame, + nghttp2_rcbuf *name, nghttp2_rcbuf *value, + uint8_t flags, void *user_data) { + auto namebuf = nghttp2_rcbuf_get_buf(name); + auto valuebuf = nghttp2_rcbuf_get_buf(value); + if (get_config()->http2.upstream.debug.frame_debug) { - verbose_on_header_callback(session, frame, name, namelen, value, valuelen, - flags, user_data); + verbose_on_header_callback(session, frame, namebuf.base, namebuf.len, + valuebuf.base, valuebuf.len, flags, user_data); } if (frame->hd.type != NGHTTP2_HEADERS) { return 0; @@ -176,7 +178,7 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, auto &httpconf = get_config()->http; - if (req.fs.buffer_size() + namelen + valuelen > + if (req.fs.buffer_size() + namebuf.len + valuebuf.len > httpconf.request_header_field_buffer || req.fs.num_fields() >= httpconf.max_request_header_fields) { if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { @@ -185,7 +187,7 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, if (LOG_ENABLED(INFO)) { ULOG(INFO, upstream) << "Too large or many header field size=" - << req.fs.buffer_size() + namelen + valuelen + << req.fs.buffer_size() + namebuf.len + valuebuf.len << ", num=" << req.fs.num_fields() + 1; } @@ -201,17 +203,23 @@ int on_header_callback(nghttp2_session *session, const nghttp2_frame *frame, return 0; } + auto token = http2::lookup_token(namebuf.base, namebuf.len); + auto no_index = flags & NGHTTP2_NV_FLAG_NO_INDEX; + + downstream->add_rcbuf(name); + downstream->add_rcbuf(value); + if (frame->headers.cat == NGHTTP2_HCAT_HEADERS) { // just store header fields for trailer part - req.fs.add_trailer(name, namelen, value, valuelen, - flags & NGHTTP2_NV_FLAG_NO_INDEX, -1); + req.fs.add_trailer_token(StringRef{namebuf.base, namebuf.len}, + StringRef{valuebuf.base, valuebuf.len}, no_index, + token); return 0; } - auto token = http2::lookup_token(name, namelen); - - req.fs.add_header(name, namelen, value, valuelen, - flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + req.fs.add_header_token(StringRef{namebuf.base, namebuf.len}, + StringRef{valuebuf.base, valuebuf.len}, no_index, + token); return 0; } } // namespace @@ -302,7 +310,9 @@ int Http2Upstream::on_request_headers(Downstream *downstream, } req.method = method_token; - req.scheme = http2::value_to_str(scheme); + if (scheme) { + req.scheme = scheme->value; + } // nghttp2 library guarantees either :authority or host exist if (!authority) { @@ -310,16 +320,18 @@ int Http2Upstream::on_request_headers(Downstream *downstream, authority = req.fs.header(http2::HD_HOST); } - req.authority = http2::value_to_str(authority); + if (authority) { + req.authority = authority->value; + } if (path) { if (method_token == HTTP_OPTIONS && path->value == "*") { // Server-wide OPTIONS request. Path is empty. - } else if (get_config()->http2_proxy || get_config()->client_proxy) { - req.path = http2::value_to_str(path); + } else if (get_config()->http2_proxy) { + req.path = path->value; } else { - const auto &value = path->value; - req.path = http2::rewrite_clean_path(std::begin(value), std::end(value)); + req.path = http2::rewrite_clean_path(downstream->get_block_allocator(), + path->value); } } @@ -410,6 +422,9 @@ int on_frame_recv_callback(nghttp2_session *session, const nghttp2_frame *frame, verbose_on_frame_recv_callback(session, frame, user_data); } auto upstream = static_cast(user_data); + auto handler = upstream->get_client_handler(); + + handler->signal_reset_upstream_conn_rtimer(); switch (frame->hd.type) { case NGHTTP2_DATA: { @@ -576,25 +591,33 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame, req.http_major = 2; req.http_minor = 0; + auto &promised_balloc = promised_downstream->get_block_allocator(); + for (size_t i = 0; i < frame->push_promise.nvlen; ++i) { auto &nv = frame->push_promise.nva[i]; + + auto name = + make_string_ref(promised_balloc, StringRef{nv.name, nv.namelen}); + auto value = + make_string_ref(promised_balloc, StringRef{nv.value, nv.valuelen}); + auto token = http2::lookup_token(nv.name, nv.namelen); switch (token) { case http2::HD__METHOD: - req.method = http2::lookup_method_token(nv.value, nv.valuelen); + req.method = http2::lookup_method_token(value); break; case http2::HD__SCHEME: - req.scheme.assign(nv.value, nv.value + nv.valuelen); + req.scheme = value; break; case http2::HD__AUTHORITY: - req.authority.assign(nv.value, nv.value + nv.valuelen); + req.authority = value; break; case http2::HD__PATH: - req.path = http2::rewrite_clean_path(nv.value, nv.value + nv.valuelen); + req.path = http2::rewrite_clean_path(promised_balloc, value); break; } - req.fs.add_header(nv.name, nv.namelen, nv.value, nv.valuelen, - nv.flags & NGHTTP2_NV_FLAG_NO_INDEX, token); + req.fs.add_header_token(name, value, nv.flags & NGHTTP2_NV_FLAG_NO_INDEX, + token); } promised_downstream->inspect_http2_request(); @@ -800,8 +823,8 @@ nghttp2_session_callbacks *create_http2_upstream_callbacks() { nghttp2_session_callbacks_set_on_frame_not_send_callback( callbacks, on_frame_not_send_callback); - nghttp2_session_callbacks_set_on_header_callback(callbacks, - on_header_callback); + nghttp2_session_callbacks_set_on_header_callback2(callbacks, + on_header_callback2); nghttp2_session_callbacks_set_on_begin_headers_callback( callbacks, on_begin_headers_callback); @@ -814,6 +837,11 @@ nghttp2_session_callbacks *create_http2_upstream_callbacks() { callbacks, http::select_padding_callback); } + if (get_config()->http2.upstream.debug.frame_debug) { + nghttp2_session_callbacks_set_error_callback(callbacks, + verbose_error_callback); + } + return callbacks; } @@ -822,9 +850,7 @@ Http2Upstream::Http2Upstream(ClientHandler *handler) downstream_queue_( get_config()->http2_proxy ? get_config()->conn.downstream.connections_per_host - : get_config()->conn.downstream.proto == PROTO_HTTP - ? get_config()->conn.downstream.connections_per_frontend - : 0, + : get_config()->conn.downstream.connections_per_frontend, !get_config()->http2_proxy), handler_(handler), session_(nullptr), @@ -844,7 +870,7 @@ Http2Upstream::Http2Upstream(ClientHandler *handler) // TODO Maybe call from outside? std::array entry; entry[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS; - entry[0].value = http2conf.max_concurrent_streams; + entry[0].value = http2conf.upstream.max_concurrent_streams; entry[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE; entry[1].value = (1 << http2conf.upstream.window_bits) - 1; @@ -1208,20 +1234,20 @@ int Http2Upstream::send_reply(Downstream *downstream, const uint8_t *body, const auto &resp = downstream->response(); auto &httpconf = get_config()->http; + auto &balloc = downstream->get_block_allocator(); + const auto &headers = resp.fs.headers(); auto nva = std::vector(); // 2 for :status and server nva.reserve(2 + headers.size() + httpconf.add_response_headers.size()); - std::string status_code_str; - auto response_status_const = http2::stringify_status(resp.http_status); - if (response_status_const) { - nva.push_back(http2::make_nv_lc_nocopy(":status", response_status_const)); - } else { - status_code_str = util::utos(resp.http_status); - nva.push_back(http2::make_nv_ls(":status", status_code_str)); + auto response_status = http2::stringify_status(resp.http_status); + if (response_status.empty()) { + response_status = util::make_string_ref_uint(balloc, resp.http_status); } + nva.push_back(http2::make_nv_ls_nocopy(":status", response_status)); + for (auto &kv : headers) { if (kv.name.empty() || kv.name[0] == ':') { continue; @@ -1269,6 +1295,8 @@ int Http2Upstream::error_reply(Downstream *downstream, int rv; auto &resp = downstream->response(); + auto &balloc = downstream->get_block_allocator(); + auto html = http::create_error_html(status_code); resp.http_status = status_code; auto body = downstream->get_response_buf(); @@ -1282,20 +1310,20 @@ int Http2Upstream::error_reply(Downstream *downstream, auto lgconf = log_config(); lgconf->update_tstamp(std::chrono::system_clock::now()); - auto response_status_const = http2::stringify_status(status_code); - auto content_length = util::utos(html.size()); + auto response_status = http2::stringify_status(status_code); + if (response_status.empty()) { + response_status = util::make_string_ref_uint(balloc, status_code); + } - std::string status_code_str; + auto content_length = util::make_string_ref_uint(balloc, html.size()); + auto date = make_string_ref(balloc, StringRef{lgconf->time_http_str}); - auto nva = make_array( - response_status_const - ? http2::make_nv_lc_nocopy(":status", response_status_const) - : http2::make_nv_ls(":status", - (status_code_str = util::utos(status_code))), - http2::make_nv_ll("content-type", "text/html; charset=UTF-8"), - http2::make_nv_ls_nocopy("server", get_config()->http.server_name), - http2::make_nv_ls("content-length", content_length), - http2::make_nv_ls("date", lgconf->time_http_str)); + auto nva = std::array{ + {http2::make_nv_ls_nocopy(":status", response_status), + http2::make_nv_ll("content-type", "text/html; charset=UTF-8"), + http2::make_nv_ls_nocopy("server", get_config()->http.server_name), + http2::make_nv_ls_nocopy("content-length", content_length), + http2::make_nv_ls_nocopy("date", date)}}; rv = nghttp2_submit_response(session_, downstream->get_stream_id(), nva.data(), nva.size(), &data_prd); @@ -1336,6 +1364,8 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { const auto &req = downstream->request(); auto &resp = downstream->response(); + auto &balloc = downstream->get_block_allocator(); + if (LOG_ENABLED(INFO)) { if (downstream->get_non_final_response()) { DLOG(INFO, downstream) << "HTTP non-final response header"; @@ -1346,8 +1376,7 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { auto &httpconf = get_config()->http; - if (!get_config()->http2_proxy && !get_config()->client_proxy && - !httpconf.no_location_rewrite) { + if (!get_config()->http2_proxy && !httpconf.no_location_rewrite) { downstream->rewrite_location_response_header(req.scheme); } @@ -1375,17 +1404,14 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { // field. nva.reserve(resp.fs.headers().size() + 4 + httpconf.add_response_headers.size()); - std::string via_value; - std::string response_status; - auto response_status_const = http2::stringify_status(resp.http_status); - if (response_status_const) { - nva.push_back(http2::make_nv_lc_nocopy(":status", response_status_const)); - } else { - response_status = util::utos(resp.http_status); - nva.push_back(http2::make_nv_ls(":status", response_status)); + auto response_status = http2::stringify_status(resp.http_status); + if (response_status.empty()) { + response_status = util::make_string_ref_uint(balloc, resp.http_status); } + nva.push_back(http2::make_nv_ls_nocopy(":status", response_status)); + if (downstream->get_non_final_response()) { http2::copy_headers_to_nva(nva, resp.fs.headers()); @@ -1416,7 +1442,7 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { http2::copy_headers_to_nva_nocopy(nva, resp.fs.headers()); - if (!get_config()->http2_proxy && !get_config()->client_proxy) { + if (!get_config()->http2_proxy) { nva.push_back(http2::make_nv_ls_nocopy("server", httpconf.server_name)); } else { auto server = resp.fs.header(http2::HD_SERVER); @@ -1431,13 +1457,23 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { nva.push_back(http2::make_nv_ls_nocopy("via", (*via).value)); } } else { + // we don't create more than 16 bytes in + // http::create_via_header_value. + size_t len = 16; if (via) { - via_value = (*via).value; - via_value += ", "; + len += via->value.size() + 2; } - via_value += - http::create_via_header_value(resp.http_major, resp.http_minor); - nva.push_back(http2::make_nv_ls("via", via_value)); + + auto iov = make_byte_ref(balloc, len + 1); + auto p = iov.base; + if (via) { + p = std::copy(std::begin(via->value), std::end(via->value), p); + p = util::copy_lit(p, ", "); + } + p = http::create_via_header_value(p, resp.http_major, resp.http_minor); + *p = '\0'; + + nva.push_back(http2::make_nv_ls_nocopy("via", StringRef{iov.base, p})); } for (auto &p : httpconf.add_response_headers) { @@ -1489,9 +1525,8 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { if (!http2conf.no_server_push && nghttp2_session_get_remote_settings(session_, NGHTTP2_SETTINGS_ENABLE_PUSH) == 1 && - !get_config()->http2_proxy && !get_config()->client_proxy && - (downstream->get_stream_id() % 2) && resp.fs.header(http2::HD_LINK) && - resp.http_status == 200 && + !get_config()->http2_proxy && (downstream->get_stream_id() % 2) && + resp.fs.header(http2::HD_LINK) && resp.http_status == 200 && (req.method == HTTP_GET || req.method == HTTP_POST)) { if (prepare_push_promise(downstream) != 0) { @@ -1693,6 +1728,11 @@ int Http2Upstream::on_downstream_reset(bool no_retry) { for (auto downstream = downstream_queue_.get_downstreams(); downstream; downstream = downstream->dlnext) { if (downstream->get_dispatch_state() != Downstream::DISPATCH_ACTIVE) { + // This is error condition when we failed push_request_headers() + // in initiate_downstream(). Otherwise, we have + // Downstream::DISPATCH_ACTIVE state, or we did not set + // DownstreamConnection. + downstream->pop_downstream_connection(); continue; } @@ -1736,17 +1776,17 @@ int Http2Upstream::on_downstream_reset(bool no_retry) { int Http2Upstream::prepare_push_promise(Downstream *downstream) { int rv; - const char *base; - size_t baselen; const auto &req = downstream->request(); const auto &resp = downstream->response(); - rv = http2::get_pure_path_component(&base, &baselen, req.path); - if (rv != 0) { + auto base = http2::get_pure_path_component(req.path); + if (base.empty()) { return 0; } + auto &balloc = downstream->get_block_allocator(); + for (auto &kv : resp.fs.headers()) { if (kv.token != http2::HD_LINK) { continue; @@ -1754,31 +1794,23 @@ int Http2Upstream::prepare_push_promise(Downstream *downstream) { for (auto &link : http2::parse_link_header(kv.value.c_str(), kv.value.size())) { - auto uri = link.uri.first; - auto len = link.uri.second - link.uri.first; + StringRef scheme, authority, path; - const std::string *scheme_ptr, *authority_ptr; - std::string scheme, authority, path; - - rv = http2::construct_push_component(scheme, authority, path, base, - baselen, uri, len); + rv = http2::construct_push_component(balloc, scheme, authority, path, + base, link.uri); if (rv != 0) { continue; } if (scheme.empty()) { - scheme_ptr = &req.scheme; - } else { - scheme_ptr = &scheme; + scheme = req.scheme; } if (authority.empty()) { - authority_ptr = &req.authority; - } else { - authority_ptr = &authority; + authority = req.authority; } - rv = submit_push_promise(*scheme_ptr, *authority_ptr, path, downstream); + rv = submit_push_promise(scheme, authority, path, downstream); if (rv != 0) { return -1; } @@ -1787,9 +1819,9 @@ int Http2Upstream::prepare_push_promise(Downstream *downstream) { return 0; } -int Http2Upstream::submit_push_promise(const std::string &scheme, - const std::string &authority, - const std::string &path, +int Http2Upstream::submit_push_promise(const StringRef &scheme, + const StringRef &authority, + const StringRef &path, Downstream *downstream) { const auto &req = downstream->request(); @@ -1799,9 +1831,9 @@ int Http2Upstream::submit_push_promise(const std::string &scheme, // juse use "GET" for now nva.push_back(http2::make_nv_ll(":method", "GET")); - nva.push_back(http2::make_nv_ls(":scheme", scheme)); - nva.push_back(http2::make_nv_ls(":path", path)); - nva.push_back(http2::make_nv_ls(":authority", authority)); + nva.push_back(http2::make_nv_ls_nocopy(":scheme", scheme)); + nva.push_back(http2::make_nv_ls_nocopy(":path", path)); + nva.push_back(http2::make_nv_ls_nocopy(":authority", authority)); for (auto &kv : req.fs.headers()) { switch (kv.token) { @@ -1852,49 +1884,42 @@ bool Http2Upstream::push_enabled() const { return !(get_config()->http2.no_server_push || nghttp2_session_get_remote_settings( session_, NGHTTP2_SETTINGS_ENABLE_PUSH) == 0 || - get_config()->http2_proxy || get_config()->client_proxy); + get_config()->http2_proxy); } -int Http2Upstream::initiate_push(Downstream *downstream, const char *uri, - size_t len) { +int Http2Upstream::initiate_push(Downstream *downstream, const StringRef &uri) { int rv; - if (len == 0 || !push_enabled() || (downstream->get_stream_id() % 2)) { + if (uri.empty() || !push_enabled() || (downstream->get_stream_id() % 2)) { return 0; } - const char *base; - size_t baselen; - const auto &req = downstream->request(); - rv = http2::get_pure_path_component(&base, &baselen, req.path); - if (rv != 0) { + auto base = http2::get_pure_path_component(req.path); + if (base.empty()) { return -1; } - const std::string *scheme_ptr, *authority_ptr; - std::string scheme, authority, path; + auto &balloc = downstream->get_block_allocator(); - rv = http2::construct_push_component(scheme, authority, path, base, baselen, - uri, len); + StringRef scheme, authority, path; + + rv = http2::construct_push_component(balloc, scheme, authority, path, base, + uri); if (rv != 0) { return -1; } if (scheme.empty()) { - scheme_ptr = &req.scheme; - } else { - scheme_ptr = &scheme; + scheme = req.scheme; } if (authority.empty()) { - authority_ptr = &req.authority; - } else { - authority_ptr = &authority; + authority = req.authority; } - rv = submit_push_promise(*scheme_ptr, *authority_ptr, path, downstream); + rv = submit_push_promise(scheme, authority, path, downstream); if (rv != 0) { return -1; @@ -1952,7 +1977,7 @@ int Http2Upstream::on_downstream_push_promise_complete( nva.reserve(headers.size()); for (auto &kv : headers) { - nva.push_back(http2::make_nv_nocopy(kv.name, kv.value, kv.no_index)); + nva.push_back(http2::make_nv(kv.name, kv.value, kv.no_index)); } auto promised_stream_id = nghttp2_submit_push_promise( diff --git a/src/shrpx_http2_upstream.h b/src/shrpx_http2_upstream.h index d090f149..1908a504 100644 --- a/src/shrpx_http2_upstream.h +++ b/src/shrpx_http2_upstream.h @@ -81,8 +81,7 @@ public: virtual int on_downstream_reset(bool no_retry); virtual int send_reply(Downstream *downstream, const uint8_t *body, size_t bodylen); - virtual int initiate_push(Downstream *downstream, const char *uri, - size_t len); + virtual int initiate_push(Downstream *downstream, const StringRef &uri); virtual int response_riovec(struct iovec *iov, int iovcnt) const; virtual void response_drain(size_t n); virtual bool response_empty() const; @@ -112,9 +111,8 @@ public: void check_shutdown(); int prepare_push_promise(Downstream *downstream); - int submit_push_promise(const std::string &scheme, - const std::string &authority, const std::string &path, - Downstream *downstream); + int submit_push_promise(const StringRef &scheme, const StringRef &authority, + const StringRef &path, Downstream *downstream); int on_request_headers(Downstream *downstream, const nghttp2_frame *frame); diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 8e2796a6..893c9efc 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -111,64 +111,56 @@ void connectcb(struct ev_loop *loop, ev_io *w, int revents) { } } // namespace -HttpDownstreamConnection::HttpDownstreamConnection( - DownstreamConnectionPool *dconn_pool, size_t group, struct ev_loop *loop, - Worker *worker) - : DownstreamConnection(dconn_pool), - conn_(loop, -1, nullptr, worker->get_mcpool(), +HttpDownstreamConnection::HttpDownstreamConnection(DownstreamAddrGroup *group, + struct ev_loop *loop, + Worker *worker) + : conn_(loop, -1, nullptr, worker->get_mcpool(), get_config()->conn.downstream.timeout.write, get_config()->conn.downstream.timeout.read, {}, {}, connectcb, readcb, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold, - get_config()->tls.dyn_rec.idle_timeout), + get_config()->tls.dyn_rec.idle_timeout, PROTO_HTTP1), do_read_(&HttpDownstreamConnection::noop), do_write_(&HttpDownstreamConnection::noop), worker_(worker), ssl_ctx_(worker->get_cl_ssl_ctx()), + group_(group), addr_(nullptr), ioctrl_(&conn_.rlimit), - response_htp_{0}, - group_(group) {} + response_htp_{0} {} -HttpDownstreamConnection::~HttpDownstreamConnection() { - if (conn_.tls.ssl && conn_.tls.initial_handshake_done) { - auto session = SSL_get0_session(conn_.tls.ssl); - if (session) { - worker_->cache_client_tls_session(&addr_->addr, session, - ev_now(conn_.loop)); - } - } -} +HttpDownstreamConnection::~HttpDownstreamConnection() {} int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { if (LOG_ENABLED(INFO)) { DCLOG(INFO, this) << "Attaching to DOWNSTREAM:" << downstream; } + auto worker_blocker = worker_->get_connect_blocker(); + if (worker_blocker->blocked()) { + if (LOG_ENABLED(INFO)) { + DCLOG(INFO, this) + << "Worker wide backend connection was blocked temporarily"; + } + return SHRPX_ERR_NETWORK; + } + auto &downstreamconf = get_config()->conn.downstream; if (conn_.fd == -1) { - auto connect_blocker = client_handler_->get_connect_blocker(); - - if (connect_blocker->blocked()) { - if (LOG_ENABLED(INFO)) { - DCLOG(INFO, this) - << "Downstream connection was blocked by connect_blocker"; - } - return -1; - } - if (ssl_ctx_) { auto ssl = ssl::create_ssl(ssl_ctx_); if (!ssl) { return -1; } + ssl::setup_downstream_http1_alpn(ssl); + conn_.set_ssl(ssl); } - auto &next_downstream = worker_->get_dgrp(group_)->next; + auto &addrs = group_->addrs; + auto &next_downstream = group_->next; auto end = next_downstream; - auto &addrs = downstreamconf.addr_groups[group_].addrs; for (;;) { auto &addr = addrs[next_downstream]; @@ -176,22 +168,44 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { next_downstream = 0; } + auto &connect_blocker = addr.connect_blocker; + + if (connect_blocker->blocked()) { + if (LOG_ENABLED(INFO)) { + DCLOG(INFO, this) << "Backend server " + << util::to_numeric_addr(&addr.addr) + << " was not available temporarily"; + } + + if (end == next_downstream) { + return SHRPX_ERR_NETWORK; + } + + continue; + } + conn_.fd = util::create_nonblock_socket(addr.addr.su.storage.ss_family); if (conn_.fd == -1) { auto error = errno; - DCLOG(WARN, this) << "socket() failed; errno=" << error; + DCLOG(WARN, this) << "socket() failed; addr=" + << util::to_numeric_addr(&addr.addr) + << ", errno=" << error; - connect_blocker->on_failure(); + worker_blocker->on_failure(); return SHRPX_ERR_NETWORK; } + worker_blocker->on_success(); + int rv; rv = connect(conn_.fd, &addr.addr.su.sa, addr.addr.len); if (rv != 0 && errno != EINPROGRESS) { auto error = errno; - DCLOG(WARN, this) << "connect() failed; errno=" << error; + DCLOG(WARN, this) << "connect() failed; addr=" + << util::to_numeric_addr(&addr.addr) + << ", errno=" << error; connect_blocker->on_failure(); close(conn_.fd); @@ -219,7 +233,7 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str()); } - auto session = worker_->reuse_client_tls_session(&addr_->addr); + auto session = ssl::reuse_tls_session(addr_); if (session) { SSL_set_session(conn_.tls.ssl, session); SSL_SESSION_free(session); @@ -258,6 +272,8 @@ int HttpDownstreamConnection::push_request_headers() { const auto &downstream_hostport = addr_->hostport; const auto &req = downstream_->request(); + auto &balloc = downstream_->get_block_allocator(); + auto connect_method = req.method == HTTP_CONNECT; auto &httpconf = get_config()->http; @@ -265,26 +281,25 @@ int HttpDownstreamConnection::push_request_headers() { // For HTTP/1.0 request, there is no authority in request. In that // case, we use backend server's host nonetheless. auto authority = StringRef(downstream_hostport); - auto no_host_rewrite = httpconf.no_host_rewrite || - get_config()->http2_proxy || - get_config()->client_proxy || connect_method; + auto no_host_rewrite = + httpconf.no_host_rewrite || get_config()->http2_proxy || connect_method; if (no_host_rewrite && !req.authority.empty()) { - authority = StringRef(req.authority); + authority = req.authority; } - downstream_->set_request_downstream_host(authority.str()); + downstream_->set_request_downstream_host(authority); auto buf = downstream_->get_request_buf(); // Assume that method and request path do not contain \r\n. auto meth = http2::to_method_string(req.method); - buf->append(meth, strlen(meth)); + buf->append(meth); buf->append(" "); if (connect_method) { buf->append(authority); - } else if (get_config()->http2_proxy || get_config()->client_proxy) { + } else if (get_config()->http2_proxy) { // Construct absolute-form request target because we are going to // send a request to a HTTP/1 proxy. assert(!req.scheme.empty()); @@ -349,14 +364,14 @@ int HttpDownstreamConnection::push_request_headers() { if (fwdconf.params) { auto params = fwdconf.params; - if (get_config()->http2_proxy || get_config()->client_proxy || - connect_method) { + if (get_config()->http2_proxy || connect_method) { params &= ~FORWARDED_PROTO; } - auto value = http::create_forwarded(params, handler->get_forwarded_by(), - handler->get_forwarded_for(), - req.authority, req.scheme); + auto value = http::create_forwarded( + balloc, params, handler->get_forwarded_by(), + handler->get_forwarded_for(), req.authority, req.scheme); + if (fwd || !value.empty()) { buf->append("Forwarded: "); if (fwd) { @@ -393,8 +408,7 @@ int HttpDownstreamConnection::push_request_headers() { buf->append((*xff).value); buf->append("\r\n"); } - if (!get_config()->http2_proxy && !get_config()->client_proxy && - !connect_method) { + if (!get_config()->http2_proxy && !connect_method) { buf->append("X-Forwarded-Proto: "); assert(!req.scheme.empty()); buf->append(req.scheme); @@ -413,7 +427,10 @@ int HttpDownstreamConnection::push_request_headers() { buf->append((*via).value); buf->append(", "); } - buf->append(http::create_via_header_value(req.http_major, req.http_minor)); + std::array viabuf; + auto end = http::create_via_header_value(viabuf.data(), req.http_major, + req.http_minor); + buf->append(viabuf.data(), end - viabuf.data()); buf->append("\r\n"); } @@ -494,8 +511,8 @@ void idle_readcb(struct ev_loop *loop, ev_io *w, int revents) { if (LOG_ENABLED(INFO)) { DCLOG(INFO, dconn) << "Idle connection EOF"; } - auto dconn_pool = dconn->get_dconn_pool(); - dconn_pool->remove_downstream_connection(dconn); + auto &dconn_pool = dconn->get_downstream_addr_group()->dconn_pool; + dconn_pool.remove_downstream_connection(dconn); // dconn was deleted } } // namespace @@ -507,8 +524,8 @@ void idle_timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) { if (LOG_ENABLED(INFO)) { DCLOG(INFO, dconn) << "Idle connection timeout"; } - auto dconn_pool = dconn->get_dconn_pool(); - dconn_pool->remove_downstream_connection(dconn); + auto &dconn_pool = dconn->get_downstream_addr_group()->dconn_pool; + dconn_pool.remove_downstream_connection(dconn); // dconn was deleted } } // namespace @@ -572,7 +589,7 @@ int htp_hdrs_completecb(http_parser *htp) { resp.http_major = htp->http_major; resp.http_minor = htp->http_minor; - if (resp.fs.index_headers() != 0) { + if (resp.fs.parse_content_length() != 0) { downstream->set_response_state(Downstream::MSG_BAD_HEADER); return -1; } @@ -679,6 +696,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { auto downstream = static_cast(htp->data); auto &resp = downstream->response(); auto &httpconf = get_config()->http; + auto &balloc = downstream->get_block_allocator(); if (ensure_header_field_buffer(downstream, httpconf, len) != 0) { return -1; @@ -691,7 +709,9 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { if (ensure_max_header_fields(downstream, httpconf) != 0) { return -1; } - resp.fs.add_header(std::string(data, len), ""); + auto name = http2::copy_lower(balloc, StringRef{data, len}); + auto token = http2::lookup_token(name); + resp.fs.add_header_token(name, StringRef{}, false, token); } } else { // trailer part @@ -704,7 +724,9 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { // wrong place or crash if trailer fields are currently empty. return -1; } - resp.fs.add_trailer(std::string(data, len), ""); + auto name = http2::copy_lower(balloc, StringRef{data, len}); + auto token = http2::lookup_token(name); + resp.fs.add_trailer_token(name, StringRef{}, false, token); } } return 0; @@ -869,6 +891,13 @@ int HttpDownstreamConnection::tls_handshake() { return -1; } + if (!SSL_session_reused(conn_.tls.ssl)) { + auto session = SSL_get0_session(conn_.tls.ssl); + if (session) { + ssl::try_cache_tls_session(addr_, session, ev_now(conn_.loop)); + } + } + do_read_ = &HttpDownstreamConnection::read_tls; do_write_ = &HttpDownstreamConnection::write_tls; @@ -1012,22 +1041,25 @@ int HttpDownstreamConnection::process_input(const uint8_t *data, } int HttpDownstreamConnection::connected() { - auto connect_blocker = client_handler_->get_connect_blocker(); + auto &connect_blocker = addr_->connect_blocker; if (!util::check_socket_connected(conn_.fd)) { conn_.wlimit.stopw(); if (LOG_ENABLED(INFO)) { - DLOG(INFO, this) << "downstream connect failed"; + DCLOG(INFO, this) << "Backend connect failed; addr=" + << util::to_numeric_addr(&addr_->addr); } + connect_blocker->on_failure(); + downstream_->set_request_state(Downstream::CONNECT_FAIL); return -1; } if (LOG_ENABLED(INFO)) { - DLOG(INFO, this) << "Connected to downstream host"; + DCLOG(INFO, this) << "Connected to downstream host"; } connect_blocker->on_success(); @@ -1059,8 +1091,11 @@ void HttpDownstreamConnection::signal_write() { ev_feed_event(conn_.loop, &conn_.wev, EV_WRITE); } -size_t HttpDownstreamConnection::get_group() const { return group_; } - int HttpDownstreamConnection::noop() { return 0; } +DownstreamAddrGroup * +HttpDownstreamConnection::get_downstream_addr_group() const { + return group_; +} + } // namespace shrpx diff --git a/src/shrpx_http_downstream_connection.h b/src/shrpx_http_downstream_connection.h index b65db9ac..32c41702 100644 --- a/src/shrpx_http_downstream_connection.h +++ b/src/shrpx_http_downstream_connection.h @@ -37,11 +37,13 @@ namespace shrpx { class DownstreamConnectionPool; class Worker; +struct DownstreamAddrGroup; +struct DownstreamAddr; class HttpDownstreamConnection : public DownstreamConnection { public: - HttpDownstreamConnection(DownstreamConnectionPool *dconn_pool, size_t group, - struct ev_loop *loop, Worker *worker); + HttpDownstreamConnection(DownstreamAddrGroup *group, struct ev_loop *loop, + Worker *worker); virtual ~HttpDownstreamConnection(); virtual int attach_downstream(Downstream *downstream); virtual void detach_downstream(Downstream *downstream); @@ -58,10 +60,11 @@ public: virtual int on_write(); virtual void on_upstream_change(Upstream *upstream); - virtual size_t get_group() const; virtual bool poolable() const { return true; } + virtual DownstreamAddrGroup *get_downstream_addr_group() const; + int read_clear(); int write_clear(); int read_tls(); @@ -81,11 +84,11 @@ private: Worker *worker_; // nullptr if TLS is not used. SSL_CTX *ssl_ctx_; + DownstreamAddrGroup *group_; // Address of remote endpoint - const DownstreamAddr *addr_; + DownstreamAddr *addr_; IOControl ioctrl_; http_parser response_htp_; - size_t group_; }; } // namespace shrpx diff --git a/src/shrpx_http_test.cc b/src/shrpx_http_test.cc index 721b53f5..455397a6 100644 --- a/src/shrpx_http_test.cc +++ b/src/shrpx_http_test.cc @@ -38,27 +38,54 @@ namespace shrpx { void test_shrpx_http_create_forwarded(void) { + BlockAllocator balloc(1024, 1024); + CU_ASSERT("by=\"example.com:3000\";for=\"[::1]\";host=\"www.example.com\";" "proto=https" == - http::create_forwarded( - FORWARDED_BY | FORWARDED_FOR | FORWARDED_HOST | FORWARDED_PROTO, - "example.com:3000", "[::1]", "www.example.com", "https")); + http::create_forwarded(balloc, FORWARDED_BY | FORWARDED_FOR | + FORWARDED_HOST | FORWARDED_PROTO, + StringRef::from_lit("example.com:3000"), + StringRef::from_lit("[::1]"), + StringRef::from_lit("www.example.com"), + StringRef::from_lit("https"))); - CU_ASSERT("for=192.168.0.1" == http::create_forwarded(FORWARDED_FOR, "alpha", - "192.168.0.1", "bravo", - "charlie")); + CU_ASSERT("for=192.168.0.1" == + http::create_forwarded( + balloc, FORWARDED_FOR, StringRef::from_lit("alpha"), + StringRef::from_lit("192.168.0.1"), + StringRef::from_lit("bravo"), StringRef::from_lit("charlie"))); CU_ASSERT("by=_hidden;for=\"[::1]\"" == - http::create_forwarded(FORWARDED_BY | FORWARDED_FOR, "_hidden", - "[::1]", "", "")); + http::create_forwarded( + balloc, FORWARDED_BY | FORWARDED_FOR, + StringRef::from_lit("_hidden"), StringRef::from_lit("[::1]"), + StringRef::from_lit(""), StringRef::from_lit(""))); CU_ASSERT("by=\"[::1]\";for=_hidden" == - http::create_forwarded(FORWARDED_BY | FORWARDED_FOR, "[::1]", - "_hidden", "", "")); + http::create_forwarded( + balloc, FORWARDED_BY | FORWARDED_FOR, + StringRef::from_lit("[::1]"), StringRef::from_lit("_hidden"), + StringRef::from_lit(""), StringRef::from_lit(""))); - CU_ASSERT("" == http::create_forwarded(FORWARDED_BY | FORWARDED_FOR | - FORWARDED_HOST | FORWARDED_PROTO, - "", "", "", "")); + CU_ASSERT("" == http::create_forwarded( + balloc, FORWARDED_BY | FORWARDED_FOR | FORWARDED_HOST | + FORWARDED_PROTO, + StringRef::from_lit(""), StringRef::from_lit(""), + StringRef::from_lit(""), StringRef::from_lit(""))); +} + +void test_shrpx_http_create_via_header_value(void) { + std::array buf; + + auto end = http::create_via_header_value(std::begin(buf), 1, 1); + + CU_ASSERT(("1.1 nghttpx" == StringRef{std::begin(buf), end})); + + std::fill(std::begin(buf), std::end(buf), '\0'); + + end = http::create_via_header_value(std::begin(buf), 2, 0); + + CU_ASSERT(("2 nghttpx" == StringRef{std::begin(buf), end})); } } // namespace shrpx diff --git a/src/shrpx_http_test.h b/src/shrpx_http_test.h index d8e28a9f..7692c95a 100644 --- a/src/shrpx_http_test.h +++ b/src/shrpx_http_test.h @@ -32,6 +32,7 @@ namespace shrpx { void test_shrpx_http_create_forwarded(void); +void test_shrpx_http_create_via_header_value(void); } // namespace shrpx diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index c807e753..ac99c457 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -86,6 +86,8 @@ int htp_uricb(http_parser *htp, const char *data, size_t len) { auto downstream = upstream->get_downstream(); auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + // We happen to have the same value for method token. req.method = htp->method; @@ -103,9 +105,10 @@ int htp_uricb(http_parser *htp, const char *data, size_t len) { req.fs.add_extra_buffer_size(len); if (req.method == HTTP_CONNECT) { - req.authority.append(data, len); + req.authority = + concat_string_ref(balloc, req.authority, StringRef{data, len}); } else { - req.path.append(data, len); + req.path = concat_string_ref(balloc, req.path, StringRef{data, len}); } return 0; @@ -118,6 +121,7 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { auto downstream = upstream->get_downstream(); auto &req = downstream->request(); auto &httpconf = get_config()->http; + auto &balloc = downstream->get_block_allocator(); if (req.fs.buffer_size() + len > httpconf.request_header_field_buffer) { if (LOG_ENABLED(INFO)) { @@ -142,7 +146,9 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { Downstream::HTTP1_REQUEST_HEADER_TOO_LARGE); return -1; } - req.fs.add_header(std::string(data, len), ""); + auto name = http2::copy_lower(balloc, StringRef{data, len}); + auto token = http2::lookup_token(name); + req.fs.add_header_token(name, StringRef{}, false, token); } } else { // trailer part @@ -156,7 +162,9 @@ int htp_hdr_keycb(http_parser *htp, const char *data, size_t len) { } return -1; } - req.fs.add_trailer(std::string(data, len), ""); + auto name = http2::copy_lower(balloc, StringRef{data, len}); + auto token = http2::lookup_token(name); + req.fs.add_trailer_token(name, StringRef{}, false, token); } } return 0; @@ -190,30 +198,51 @@ int htp_hdr_valcb(http_parser *htp, const char *data, size_t len) { } // namespace namespace { -void rewrite_request_host_path_from_uri(Request &req, const char *uri, +void rewrite_request_host_path_from_uri(BlockAllocator &balloc, Request &req, + const StringRef &uri, http_parser_url &u) { assert(u.field_set & (1 << UF_HOST)); - auto &authority = req.authority; - authority.clear(); // As per https://tools.ietf.org/html/rfc7230#section-5.4, we // rewrite host header field with authority component. - http2::copy_url_component(authority, &u, UF_HOST, uri); + auto authority = util::get_uri_field(uri.c_str(), u, UF_HOST); // TODO properly check IPv6 numeric address - if (authority.find(':') != std::string::npos) { - authority = '[' + authority; - authority += ']'; + auto ipv6 = std::find(std::begin(authority), std::end(authority), ':') != + std::end(authority); + auto authoritylen = authority.size(); + if (ipv6) { + authoritylen += 2; } if (u.field_set & (1 << UF_PORT)) { - authority += ':'; - authority += util::utos(u.port); + authoritylen += 1 + str_size("65535"); + } + if (authoritylen > authority.size()) { + auto iovec = make_byte_ref(balloc, authoritylen + 1); + auto p = iovec.base; + if (ipv6) { + *p++ = '['; + } + p = std::copy(std::begin(authority), std::end(authority), p); + if (ipv6) { + *p++ = ']'; + } + + if (u.field_set & (1 << UF_PORT)) { + *p++ = ':'; + p = util::utos(p, u.port); + } + *p = '\0'; + + req.authority = StringRef{iovec.base, p}; + } else { + req.authority = authority; } - http2::copy_url_component(req.scheme, &u, UF_SCHEMA, uri); + req.scheme = util::get_uri_field(uri.c_str(), u, UF_SCHEMA); - std::string path; + StringRef path; if (u.field_set & (1 << UF_PATH)) { - http2::copy_url_component(path, &u, UF_PATH, uri); + path = util::get_uri_field(uri.c_str(), u, UF_PATH); } else if (req.method == HTTP_OPTIONS) { // Server-wide OPTIONS takes following form in proxy request: // @@ -221,21 +250,35 @@ void rewrite_request_host_path_from_uri(Request &req, const char *uri, // // Notice that no slash after authority. See // http://tools.ietf.org/html/rfc7230#section-5.3.4 - req.path = ""; + req.path = StringRef::from_lit(""); // we ignore query component here return; } else { - path = "/"; + path = StringRef::from_lit("/"); } + if (u.field_set & (1 << UF_QUERY)) { auto &fdata = u.field_data[UF_QUERY]; - path += '?'; - path.append(uri + fdata.off, fdata.len); + + if (u.field_set & (1 << UF_PATH)) { + auto q = util::get_uri_field(uri.c_str(), u, UF_QUERY); + path = StringRef{std::begin(path), std::end(q)}; + } else { + auto iov = make_byte_ref(balloc, path.size() + 1 + fdata.len + 1); + auto p = iov.base; + + p = std::copy(std::begin(path), std::end(path), p); + *p++ = '?'; + p = std::copy_n(&uri[fdata.off], fdata.len, p); + *p = '\0'; + path = StringRef{iov.base, p}; + } } - if (get_config()->http2_proxy || get_config()->client_proxy) { - req.path = std::move(path); + + if (get_config()->http2_proxy) { + req.path = path; } else { - req.path = http2::rewrite_clean_path(std::begin(path), std::end(path)); + req.path = http2::rewrite_clean_path(balloc, path); } } } // namespace @@ -247,6 +290,11 @@ int htp_hdrs_completecb(http_parser *htp) { if (LOG_ENABLED(INFO)) { ULOG(INFO, upstream) << "HTTP request headers completed"; } + + auto handler = upstream->get_client_handler(); + + handler->signal_reset_upstream_conn_rtimer(); + auto downstream = upstream->get_downstream(); auto &req = downstream->request(); @@ -270,7 +318,7 @@ int htp_hdrs_completecb(http_parser *htp) { ULOG(INFO, upstream) << "HTTP request headers\n" << ss.str(); } - if (req.fs.index_headers() != 0) { + if (req.fs.parse_content_length() != 0) { return -1; } @@ -294,49 +342,47 @@ int htp_hdrs_completecb(http_parser *htp) { downstream->inspect_http1_request(); + auto &balloc = downstream->get_block_allocator(); + if (method != HTTP_CONNECT) { http_parser_url u{}; - // make a copy of request path, since we may set request path - // while we are refering to original request path. - auto path = req.path; - rv = http_parser_parse_url(path.c_str(), path.size(), 0, &u); + rv = http_parser_parse_url(req.path.c_str(), req.path.size(), 0, &u); if (rv != 0) { // Expect to respond with 400 bad request return -1; } // checking UF_HOST could be redundant, but just in case ... if (!(u.field_set & (1 << UF_SCHEMA)) || !(u.field_set & (1 << UF_HOST))) { - if (get_config()->http2_proxy || get_config()->client_proxy) { + if (get_config()->http2_proxy) { // Request URI should be absolute-form for client proxy mode return -1; } req.no_authority = true; - if (method == HTTP_OPTIONS && path == "*") { - req.path = ""; + if (method == HTTP_OPTIONS && req.path == "*") { + req.path = StringRef{}; } else { - req.path = http2::rewrite_clean_path(std::begin(path), std::end(path)); + req.path = http2::rewrite_clean_path(balloc, req.path); } if (host) { req.authority = host->value; } - if (upstream->get_client_handler()->get_ssl()) { - req.scheme = "https"; + if (handler->get_ssl()) { + req.scheme = StringRef::from_lit("https"); } else { - req.scheme = "http"; + req.scheme = StringRef::from_lit("http"); } } else { - rewrite_request_host_path_from_uri(req, path.c_str(), u); + rewrite_request_host_path_from_uri(balloc, req, req.path, u); } } downstream->set_request_state(Downstream::HEADER_COMPLETE); #ifdef HAVE_MRUBY - auto handler = upstream->get_client_handler(); auto worker = handler->get_worker(); auto mruby_ctx = worker->get_mruby_context(); @@ -355,7 +401,7 @@ int htp_hdrs_completecb(http_parser *htp) { } rv = downstream->attach_downstream_connection( - upstream->get_client_handler()->get_downstream_connection(downstream)); + handler->get_downstream_connection(downstream)); if (rv != 0) { downstream->set_request_state(Downstream::CONNECT_FAIL); @@ -377,6 +423,10 @@ namespace { int htp_bodycb(http_parser *htp, const char *data, size_t len) { int rv; auto upstream = static_cast(htp->data); + auto handler = upstream->get_client_handler(); + + handler->signal_reset_upstream_conn_rtimer(); + auto downstream = upstream->get_downstream(); rv = downstream->push_upload_data_chunk( reinterpret_cast(data), len); @@ -929,8 +979,7 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) { auto &httpconf = get_config()->http; - if (!get_config()->http2_proxy && !get_config()->client_proxy && - !httpconf.no_location_rewrite) { + if (!get_config()->http2_proxy && !httpconf.no_location_rewrite) { downstream->rewrite_location_response_header( get_client_handler()->get_upstream_scheme()); } @@ -999,7 +1048,7 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) { } } - if (!get_config()->http2_proxy && !get_config()->client_proxy) { + if (!get_config()->http2_proxy) { buf->append("Server: "); buf->append(httpconf.server_name); buf->append("\r\n"); @@ -1025,8 +1074,10 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) { buf->append((*via).value); buf->append(", "); } - buf->append( - http::create_via_header_value(resp.http_major, resp.http_minor)); + std::array viabuf; + auto end = http::create_via_header_value(viabuf.data(), resp.http_major, + resp.http_minor); + buf->append(viabuf.data(), end - std::begin(viabuf)); buf->append("\r\n"); } @@ -1154,8 +1205,7 @@ fail: return 0; } -int HttpsUpstream::initiate_push(Downstream *downstream, const char *uri, - size_t len) { +int HttpsUpstream::initiate_push(Downstream *downstream, const StringRef &uri) { return 0; } diff --git a/src/shrpx_https_upstream.h b/src/shrpx_https_upstream.h index 3ab698bc..0f9b8dd0 100644 --- a/src/shrpx_https_upstream.h +++ b/src/shrpx_https_upstream.h @@ -76,8 +76,7 @@ public: virtual int on_downstream_reset(bool no_retry); virtual int send_reply(Downstream *downstream, const uint8_t *body, size_t bodylen); - virtual int initiate_push(Downstream *downstream, const char *uri, - size_t len); + virtual int initiate_push(Downstream *downstream, const StringRef &uri); virtual int response_riovec(struct iovec *iov, int iovcnt) const; virtual void response_drain(size_t n); virtual bool response_empty() const; diff --git a/src/shrpx_log.cc b/src/shrpx_log.cc index 261b02e5..ae699e8b 100644 --- a/src/shrpx_log.cc +++ b/src/shrpx_log.cc @@ -393,23 +393,23 @@ int reopen_log_files() { auto &accessconf = get_config()->logging.access; auto &errorconf = get_config()->logging.error; - if (!accessconf.syslog && accessconf.file) { - new_accesslog_fd = util::open_log_file(accessconf.file.get()); + if (!accessconf.syslog && !accessconf.file.empty()) { + new_accesslog_fd = util::open_log_file(accessconf.file.c_str()); if (new_accesslog_fd == -1) { - LOG(ERROR) << "Failed to open accesslog file " << accessconf.file.get(); + LOG(ERROR) << "Failed to open accesslog file " << accessconf.file; res = -1; } } - if (!errorconf.syslog && errorconf.file) { - new_errorlog_fd = util::open_log_file(errorconf.file.get()); + if (!errorconf.syslog && !errorconf.file.empty()) { + new_errorlog_fd = util::open_log_file(errorconf.file.c_str()); if (new_errorlog_fd == -1) { if (lgconf->errorlog_fd != -1) { - LOG(ERROR) << "Failed to open errorlog file " << errorconf.file.get(); + LOG(ERROR) << "Failed to open errorlog file " << errorconf.file; } else { - std::cerr << "Failed to open errorlog file " << errorconf.file.get() + std::cerr << "Failed to open errorlog file " << errorconf.file << std::endl; } diff --git a/src/shrpx_memcached_connection.cc b/src/shrpx_memcached_connection.cc index 4df031d8..0b494ba8 100644 --- a/src/shrpx_memcached_connection.cc +++ b/src/shrpx_memcached_connection.cc @@ -96,7 +96,7 @@ MemcachedConnection::MemcachedConnection(const Address *addr, const StringRef &sni_name, MemchunkPool *mcpool) : conn_(loop, -1, nullptr, mcpool, write_timeout, read_timeout, {}, {}, - connectcb, readcb, timeoutcb, this, 0, 0.), + connectcb, readcb, timeoutcb, this, 0, 0., PROTO_MEMCACHED), do_read_(&MemcachedConnection::noop), do_write_(&MemcachedConnection::noop), sni_name_(sni_name.str()), diff --git a/src/shrpx_memcached_connection.h b/src/shrpx_memcached_connection.h index c9552198..a093589a 100644 --- a/src/shrpx_memcached_connection.h +++ b/src/shrpx_memcached_connection.h @@ -35,12 +35,13 @@ #include "shrpx_connection.h" #include "buffer.h" +#include "network.h" + using namespace nghttp2; namespace shrpx { struct MemcachedRequest; -struct Address; enum { MEMCACHED_PARSE_HEADER24, diff --git a/src/shrpx_memcached_dispatcher.h b/src/shrpx_memcached_dispatcher.h index 64021c68..7cca3bcb 100644 --- a/src/shrpx_memcached_dispatcher.h +++ b/src/shrpx_memcached_dispatcher.h @@ -34,12 +34,12 @@ #include #include "memchunk.h" +#include "network.h" namespace shrpx { struct MemcachedRequest; class MemcachedConnection; -struct Address; class MemcachedDispatcher { public: diff --git a/src/shrpx_mruby.cc b/src/shrpx_mruby.cc index 5b1b3967..507d13b9 100644 --- a/src/shrpx_mruby.cc +++ b/src/shrpx_mruby.cc @@ -31,7 +31,6 @@ #include "shrpx_config.h" #include "shrpx_mruby_module.h" #include "shrpx_downstream_connection.h" -#include "template.h" namespace shrpx { @@ -95,14 +94,6 @@ int MRubyContext::run_app(Downstream *downstream, int phase) { mrb_->ud = nullptr; - if (data.request_headers_dirty) { - downstream->request().fs.index_headers(); - } - - if (data.response_headers_dirty) { - downstream->response().fs.index_headers(); - } - return rv; } @@ -146,12 +137,12 @@ mrb_value instantiate_app(mrb_state *mrb, RProc *proc) { // very hard to write these kind of code because mruby has almost no // documentation aobut compiling or generating code, at least at the // time of this writing. -RProc *compile(mrb_state *mrb, const char *filename) { - if (filename == nullptr) { +RProc *compile(mrb_state *mrb, const StringRef &filename) { + if (filename.empty()) { return nullptr; } - auto infile = fopen(filename, "rb"); + auto infile = fopen(filename.c_str(), "rb"); if (infile == nullptr) { return nullptr; } @@ -185,8 +176,8 @@ RProc *compile(mrb_state *mrb, const char *filename) { return proc; } -std::unique_ptr create_mruby_context(const char *filename) { - if (!filename) { +std::unique_ptr create_mruby_context(const StringRef &filename) { + if (filename.empty()) { return make_unique(nullptr, mrb_nil_value(), mrb_nil_value()); } diff --git a/src/shrpx_mruby.h b/src/shrpx_mruby.h index 95f0430c..4466cf6f 100644 --- a/src/shrpx_mruby.h +++ b/src/shrpx_mruby.h @@ -32,6 +32,8 @@ #include #include +#include "template.h" + using namespace nghttp2; namespace shrpx { @@ -65,13 +67,11 @@ enum { struct MRubyAssocData { Downstream *downstream; int phase; - bool request_headers_dirty; - bool response_headers_dirty; }; -RProc *compile(mrb_state *mrb, const char *filename); +RProc *compile(mrb_state *mrb, const StringRef &filename); -std::unique_ptr create_mruby_context(const char *filename); +std::unique_ptr create_mruby_context(const StringRef &filename); // Return interned |ptr|. mrb_sym intern_ptr(mrb_state *mrb, void *ptr); diff --git a/src/shrpx_mruby_module.cc b/src/shrpx_mruby_module.cc index e2f17fb1..27b7769c 100644 --- a/src/shrpx_mruby_module.cc +++ b/src/shrpx_mruby_module.cc @@ -85,7 +85,7 @@ mrb_value init_module(mrb_state *mrb) { return create_env(mrb); } -mrb_value create_headers_hash(mrb_state *mrb, const Headers &headers) { +mrb_value create_headers_hash(mrb_state *mrb, const HeaderRefs &headers) { auto hash = mrb_hash_new(mrb); for (auto &hd : headers) { diff --git a/src/shrpx_mruby_module.h b/src/shrpx_mruby_module.h index 3e417f88..a426bead 100644 --- a/src/shrpx_mruby_module.h +++ b/src/shrpx_mruby_module.h @@ -43,7 +43,7 @@ mrb_value init_module(mrb_state *mrb); void delete_downstream_from_module(mrb_state *mrb, Downstream *downstream); -mrb_value create_headers_hash(mrb_state *mrb, const Headers &headers); +mrb_value create_headers_hash(mrb_state *mrb, const HeaderRefs &headers); } // namespace mruby diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index 0b1ddb15..f98442a4 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -70,7 +70,7 @@ mrb_value request_get_method(mrb_state *mrb, mrb_value self) { const auto &req = downstream->request(); auto method = http2::to_method_string(req.method); - return mrb_str_new_cstr(mrb, method); + return mrb_str_new(mrb, method.c_str(), method.size()); } } // namespace @@ -116,6 +116,8 @@ mrb_value request_set_authority(mrb_state *mrb, mrb_value self) { auto downstream = data->downstream; auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + check_phase(mrb, data->phase, PHASE_REQUEST); const char *authority; @@ -125,7 +127,8 @@ mrb_value request_set_authority(mrb_state *mrb, mrb_value self) { mrb_raise(mrb, E_RUNTIME_ERROR, "authority must not be empty string"); } - req.authority.assign(authority, n); + req.authority = + make_string_ref(balloc, StringRef{authority, static_cast(n)}); return self; } @@ -147,6 +150,8 @@ mrb_value request_set_scheme(mrb_state *mrb, mrb_value self) { auto downstream = data->downstream; auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + check_phase(mrb, data->phase, PHASE_REQUEST); const char *scheme; @@ -156,7 +161,8 @@ mrb_value request_set_scheme(mrb_state *mrb, mrb_value self) { mrb_raise(mrb, E_RUNTIME_ERROR, "scheme must not be empty string"); } - req.scheme.assign(scheme, n); + req.scheme = + make_string_ref(balloc, StringRef{scheme, static_cast(n)}); return self; } @@ -178,13 +184,16 @@ mrb_value request_set_path(mrb_state *mrb, mrb_value self) { auto downstream = data->downstream; auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + check_phase(mrb, data->phase, PHASE_REQUEST); const char *path; mrb_int pathlen; mrb_get_args(mrb, "s", &path, &pathlen); - req.path.assign(path, pathlen); + req.path = + make_string_ref(balloc, StringRef{path, static_cast(pathlen)}); return self; } @@ -204,6 +213,7 @@ mrb_value request_mod_header(mrb_state *mrb, mrb_value self, bool repl) { auto data = static_cast(mrb->ud); auto downstream = data->downstream; auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); check_phase(mrb, data->phase, PHASE_REQUEST); @@ -216,17 +226,21 @@ mrb_value request_mod_header(mrb_state *mrb, mrb_value self, bool repl) { key = mrb_funcall(mrb, key, "downcase", 0); + auto keyref = + make_string_ref(balloc, StringRef{RSTRING_PTR(key), + static_cast(RSTRING_LEN(key))}); + auto token = http2::lookup_token(keyref.byte(), keyref.size()); + if (repl) { size_t p = 0; auto &headers = req.fs.headers(); for (size_t i = 0; i < headers.size(); ++i) { - auto &hd = headers[i]; - if (util::streq(std::begin(hd.name), hd.name.size(), RSTRING_PTR(key), - RSTRING_LEN(key))) { + auto &kv = headers[i]; + if (kv.name == keyref) { continue; } if (i != p) { - headers[p++] = std::move(hd); + headers[p++] = std::move(kv); } } headers.resize(p); @@ -236,16 +250,22 @@ mrb_value request_mod_header(mrb_state *mrb, mrb_value self, bool repl) { auto n = mrb_ary_len(mrb, values); for (int i = 0; i < n; ++i) { auto value = mrb_ary_entry(values, i); - req.fs.add_header(std::string(RSTRING_PTR(key), RSTRING_LEN(key)), - std::string(RSTRING_PTR(value), RSTRING_LEN(value))); + req.fs.add_header_token( + keyref, + make_string_ref(balloc, + StringRef{RSTRING_PTR(value), + static_cast(RSTRING_LEN(value))}), + false, token); } } else if (!mrb_nil_p(values)) { - req.fs.add_header(std::string(RSTRING_PTR(key), RSTRING_LEN(key)), - std::string(RSTRING_PTR(values), RSTRING_LEN(values))); + req.fs.add_header_token( + keyref, + make_string_ref(balloc, + StringRef{RSTRING_PTR(values), + static_cast(RSTRING_LEN(values))}), + false, token); } - data->request_headers_dirty = true; - return mrb_nil_value(); } } // namespace @@ -286,7 +306,7 @@ mrb_value request_push(mrb_state *mrb, mrb_value self) { mrb_int len; mrb_get_args(mrb, "s", &uri, &len); - upstream->initiate_push(downstream, uri, len); + upstream->initiate_push(downstream, StringRef{uri, static_cast(len)}); return mrb_nil_value(); } diff --git a/src/shrpx_mruby_module_response.cc b/src/shrpx_mruby_module_response.cc index 5b88f879..8a89a058 100644 --- a/src/shrpx_mruby_module_response.cc +++ b/src/shrpx_mruby_module_response.cc @@ -107,6 +107,7 @@ mrb_value response_mod_header(mrb_state *mrb, mrb_value self, bool repl) { auto data = static_cast(mrb->ud); auto downstream = data->downstream; auto &resp = downstream->response(); + auto &balloc = downstream->get_block_allocator(); mrb_value key, values; mrb_get_args(mrb, "oo", &key, &values); @@ -117,17 +118,21 @@ mrb_value response_mod_header(mrb_state *mrb, mrb_value self, bool repl) { key = mrb_funcall(mrb, key, "downcase", 0); + auto keyref = + make_string_ref(balloc, StringRef{RSTRING_PTR(key), + static_cast(RSTRING_LEN(key))}); + auto token = http2::lookup_token(keyref.byte(), keyref.size()); + if (repl) { size_t p = 0; auto &headers = resp.fs.headers(); for (size_t i = 0; i < headers.size(); ++i) { - auto &hd = headers[i]; - if (util::streq(std::begin(hd.name), hd.name.size(), RSTRING_PTR(key), - RSTRING_LEN(key))) { + auto &kv = headers[i]; + if (kv.name == keyref) { continue; } if (i != p) { - headers[p++] = std::move(hd); + headers[p++] = std::move(kv); } } headers.resize(p); @@ -137,16 +142,22 @@ mrb_value response_mod_header(mrb_state *mrb, mrb_value self, bool repl) { auto n = mrb_ary_len(mrb, values); for (int i = 0; i < n; ++i) { auto value = mrb_ary_entry(values, i); - resp.fs.add_header(std::string(RSTRING_PTR(key), RSTRING_LEN(key)), - std::string(RSTRING_PTR(value), RSTRING_LEN(value))); + resp.fs.add_header_token( + keyref, + make_string_ref(balloc, + StringRef{RSTRING_PTR(value), + static_cast(RSTRING_LEN(value))}), + false, token); } } else if (!mrb_nil_p(values)) { - resp.fs.add_header(std::string(RSTRING_PTR(key), RSTRING_LEN(key)), - std::string(RSTRING_PTR(values), RSTRING_LEN(values))); + resp.fs.add_header_token( + keyref, + make_string_ref(balloc, + StringRef{RSTRING_PTR(values), + static_cast(RSTRING_LEN(values))}), + false, token); } - data->response_headers_dirty = true; - return mrb_nil_value(); } } // namespace @@ -182,6 +193,8 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { auto &resp = downstream->response(); int rv; + auto &balloc = downstream->get_block_allocator(); + if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { mrb_raise(mrb, E_RUNTIME_ERROR, "response has already been committed"); } @@ -197,22 +210,19 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { resp.http_status = 200; } - if (data->response_headers_dirty) { - resp.fs.index_headers(); - data->response_headers_dirty = false; - } - if (downstream->expect_response_body() && vallen > 0) { body = reinterpret_cast(val); bodylen = vallen; } + auto content_length = util::make_string_ref_uint(balloc, bodylen); + auto cl = resp.fs.header(http2::HD_CONTENT_LENGTH); if (cl) { - cl->value = util::utos(bodylen); + cl->value = content_length; } else { - resp.fs.add_header("content-length", util::utos(bodylen), - http2::HD_CONTENT_LENGTH); + resp.fs.add_header_token(StringRef::from_lit("content-length"), + content_length, false, http2::HD_CONTENT_LENGTH); } resp.fs.content_length = bodylen; @@ -220,7 +230,10 @@ mrb_value response_return(mrb_state *mrb, mrb_value self) { if (!date) { auto lgconf = log_config(); lgconf->update_tstamp(std::chrono::system_clock::now()); - resp.fs.add_header("date", lgconf->time_http_str, http2::HD_DATE); + resp.fs.add_header_token( + StringRef::from_lit("date"), + make_string_ref(balloc, StringRef{lgconf->time_http_str}), false, + http2::HD_DATE); } auto upstream = downstream->get_upstream(); diff --git a/src/shrpx_router.cc b/src/shrpx_router.cc index e0404252..4c88283f 100644 --- a/src/shrpx_router.cc +++ b/src/shrpx_router.cc @@ -66,21 +66,21 @@ void Router::add_node(RNode *node, const char *pattern, size_t patlen, add_next_node(node, std::move(new_node)); } -bool Router::add_route(const char *pattern, size_t patlen, size_t index) { +bool Router::add_route(const StringRef &pattern, size_t index) { auto node = &root_; size_t i = 0; for (;;) { auto next_node = find_next_node(node, pattern[i]); if (next_node == nullptr) { - add_node(node, pattern + i, patlen - i, index); + add_node(node, pattern.c_str() + i, pattern.size() - i, index); return true; } node = next_node; - auto slen = patlen - i; - auto s = pattern + i; + auto slen = pattern.size() - i; + auto s = pattern.c_str() + i; auto n = std::min(node->len, slen); size_t j; for (j = 0; j < n && node->s[j] == s[j]; ++j) @@ -125,8 +125,8 @@ bool Router::add_route(const char *pattern, size_t patlen, size_t index) { i += j; - assert(patlen > i); - add_node(node, pattern + i, patlen - i, index); + assert(pattern.size() > i); + add_node(node, pattern.c_str() + i, pattern.size() - i, index); return true; } @@ -259,18 +259,16 @@ const RNode *match_partial(const RNode *node, size_t offset, const char *first, } } // namespace -ssize_t Router::match(const std::string &host, const char *path, - size_t pathlen) const { +ssize_t Router::match(const StringRef &host, const StringRef &path) const { const RNode *node; size_t offset; - node = - match_complete(&offset, &root_, host.c_str(), host.c_str() + host.size()); + node = match_complete(&offset, &root_, std::begin(host), std::end(host)); if (node == nullptr) { return -1; } - node = match_partial(node, offset, path, path + pathlen); + node = match_partial(node, offset, std::begin(path), std::end(path)); if (node == nullptr || node == &root_) { return -1; } diff --git a/src/shrpx_router.h b/src/shrpx_router.h index 07aadf69..ece0e276 100644 --- a/src/shrpx_router.h +++ b/src/shrpx_router.h @@ -55,11 +55,10 @@ struct RNode { class Router { public: Router(); - // Adds route |pattern| of size |patlen| with its |index|. - bool add_route(const char *pattern, size_t patlen, size_t index); + // Adds route |pattern| with its |index|. + bool add_route(const StringRef &pattern, size_t index); // Returns the matched index of pattern. -1 if there is no match. - ssize_t match(const std::string &host, const char *path, - size_t pathlen) const; + ssize_t match(const StringRef &host, const StringRef &path) const; void add_node(RNode *node, const char *pattern, size_t patlen, size_t index); diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index ee80a144..32cb7b27 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -143,6 +143,10 @@ namespace { void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, spdylay_frame *frame, void *user_data) { auto upstream = static_cast(user_data); + auto handler = upstream->get_client_handler(); + + handler->signal_reset_upstream_conn_rtimer(); + switch (type) { case SPDYLAY_SYN_STREAM: { if (LOG_ENABLED(INFO)) { @@ -155,6 +159,8 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, auto &req = downstream->request(); + auto &balloc = downstream->get_block_allocator(); + downstream->reset_upstream_rtimer(); auto nv = frame->syn_stream.nv; @@ -188,10 +194,15 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, } for (size_t i = 0; nv[i]; i += 2) { - req.fs.add_header(nv[i], nv[i + 1]); + auto name = StringRef{nv[i]}; + auto value = StringRef{nv[i + 1]}; + auto token = http2::lookup_token(name.byte(), name.size()); + req.fs.add_header_token(make_string_ref(balloc, StringRef{name}), + make_string_ref(balloc, StringRef{value}), false, + token); } - if (req.fs.index_headers() != 0) { + if (req.fs.parse_content_length() != 0) { if (upstream->error_reply(downstream, 400) != 0) { ULOG(FATAL, upstream) << "error_reply failed"; } @@ -259,13 +270,12 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, } else { req.scheme = scheme->value; req.authority = host->value; - if (get_config()->http2_proxy || get_config()->client_proxy) { + if (get_config()->http2_proxy) { req.path = path->value; } else if (method_token == HTTP_OPTIONS && path->value == "*") { // Server-wide OPTIONS request. Path is empty. } else { - req.path = http2::rewrite_clean_path(std::begin(path->value), - std::end(path->value)); + req.path = http2::rewrite_clean_path(balloc, path->value); } } @@ -278,7 +288,6 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, downstream->set_request_state(Downstream::HEADER_COMPLETE); #ifdef HAVE_MRUBY - auto handler = upstream->get_client_handler(); auto worker = handler->get_worker(); auto mruby_ctx = worker->get_mruby_context(); @@ -416,6 +425,10 @@ void on_data_recv_callback(spdylay_session *session, uint8_t flags, auto upstream = static_cast(user_data); auto downstream = static_cast( spdylay_session_get_stream_user_data(session, stream_id)); + auto handler = upstream->get_client_handler(); + + handler->signal_reset_upstream_conn_rtimer(); + if (downstream && (flags & SPDYLAY_DATA_FLAG_FIN)) { if (!downstream->validate_request_recv_body_length()) { upstream->rst_stream(downstream, SPDYLAY_PROTOCOL_ERROR); @@ -500,9 +513,7 @@ SpdyUpstream::SpdyUpstream(uint16_t version, ClientHandler *handler) downstream_queue_( get_config()->http2_proxy ? get_config()->conn.downstream.connections_per_host - : get_config()->conn.downstream.proto == PROTO_HTTP - ? get_config()->conn.downstream.connections_per_frontend - : 0, + : get_config()->conn.downstream.connections_per_frontend, !get_config()->http2_proxy), handler_(handler), session_(nullptr) { @@ -544,7 +555,7 @@ SpdyUpstream::SpdyUpstream(uint16_t version, ClientHandler *handler) // TODO Maybe call from outside? std::array entry; entry[0].settings_id = SPDYLAY_SETTINGS_MAX_CONCURRENT_STREAMS; - entry[0].value = http2conf.max_concurrent_streams; + entry[0].value = http2conf.upstream.max_concurrent_streams; entry[0].flags = SPDYLAY_ID_FLAG_SETTINGS_NONE; entry[1].settings_id = SPDYLAY_SETTINGS_INITIAL_WINDOW_SIZE; @@ -1008,8 +1019,7 @@ int SpdyUpstream::on_downstream_header_complete(Downstream *downstream) { auto &httpconf = get_config()->http; - if (!get_config()->http2_proxy && !get_config()->client_proxy && - !httpconf.no_location_rewrite) { + if (!get_config()->http2_proxy && !httpconf.no_location_rewrite) { downstream->rewrite_location_response_header(req.scheme); } @@ -1043,7 +1053,7 @@ int SpdyUpstream::on_downstream_header_complete(Downstream *downstream) { nv[hdidx++] = hd.value.c_str(); } - if (!get_config()->http2_proxy && !get_config()->client_proxy) { + if (!get_config()->http2_proxy) { nv[hdidx++] = "server"; nv[hdidx++] = httpconf.server_name.c_str(); } else { @@ -1062,11 +1072,13 @@ int SpdyUpstream::on_downstream_header_complete(Downstream *downstream) { } } else { if (via) { - via_value = via->value; + via_value = via->value.str(); via_value += ", "; } - via_value += - http::create_via_header_value(resp.http_major, resp.http_minor); + std::array viabuf; + auto end = http::create_via_header_value(std::begin(viabuf), + resp.http_major, resp.http_minor); + via_value.append(std::begin(viabuf), end); nv[hdidx++] = "via"; nv[hdidx++] = via_value.c_str(); } @@ -1212,6 +1224,11 @@ int SpdyUpstream::on_downstream_reset(bool no_retry) { for (auto downstream = downstream_queue_.get_downstreams(); downstream; downstream = downstream->dlnext) { if (downstream->get_dispatch_state() != Downstream::DISPATCH_ACTIVE) { + // This is error condition when we failed push_request_headers() + // in initiate_downstream(). Otherwise, we have + // Downstream::DISPATCH_ACTIVE state, or we did not set + // DownstreamConnection. + downstream->pop_downstream_connection(); continue; } @@ -1252,8 +1269,7 @@ int SpdyUpstream::on_downstream_reset(bool no_retry) { return 0; } -int SpdyUpstream::initiate_push(Downstream *downstream, const char *uri, - size_t len) { +int SpdyUpstream::initiate_push(Downstream *downstream, const StringRef &uri) { return 0; } diff --git a/src/shrpx_spdy_upstream.h b/src/shrpx_spdy_upstream.h index 403c3e5a..c84d3f7e 100644 --- a/src/shrpx_spdy_upstream.h +++ b/src/shrpx_spdy_upstream.h @@ -76,8 +76,7 @@ public: virtual int send_reply(Downstream *downstream, const uint8_t *body, size_t bodylen); - virtual int initiate_push(Downstream *downstream, const char *uri, - size_t len); + virtual int initiate_push(Downstream *downstream, const StringRef &uri); virtual int response_riovec(struct iovec *iov, int iovcnt) const; virtual void response_drain(size_t n); virtual bool response_empty() const; diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc index 5d8a2360..a36b20be 100644 --- a/src/shrpx_ssl.cc +++ b/src/shrpx_ssl.cc @@ -36,6 +36,7 @@ #include #include +#include #include #include @@ -124,13 +125,13 @@ set_alpn_prefs(const std::vector &protos) { namespace { int ssl_pem_passwd_cb(char *buf, int size, int rwflag, void *user_data) { auto config = static_cast(user_data); - int len = (int)strlen(config->tls.private_key_passwd.get()); + auto len = static_cast(config->tls.private_key_passwd.size()); if (size < len + 1) { LOG(ERROR) << "ssl_pem_passwd_cb: buf is too small " << size; return 0; } // Copy string including last '\0'. - memcpy(buf, config->tls.private_key_passwd.get(), len + 1); + memcpy(buf, config->tls.private_key_passwd.c_str(), len + 1); return len; } } // namespace @@ -485,7 +486,7 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file SSL_CTX_set_session_id_context(ssl_ctx, sid_ctx, sizeof(sid_ctx) - 1); SSL_CTX_set_session_cache_mode(ssl_ctx, SSL_SESS_CACHE_SERVER); - if (tlsconf.session_cache.memcached.host) { + if (!tlsconf.session_cache.memcached.host.empty()) { SSL_CTX_sess_set_new_cb(ssl_ctx, tls_session_new_cb); SSL_CTX_sess_set_get_cb(ssl_ctx, tls_session_get_cb); } @@ -493,8 +494,8 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file SSL_CTX_set_timeout(ssl_ctx, tlsconf.session_timeout.count()); const char *ciphers; - if (tlsconf.ciphers) { - ciphers = tlsconf.ciphers.get(); + if (!tlsconf.ciphers.empty()) { + ciphers = tlsconf.ciphers.c_str(); } else { ciphers = nghttp2::ssl::DEFAULT_CIPHER_LIST; } @@ -527,9 +528,9 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file #endif // OPENSSL_NO_EC - if (tlsconf.dh_param_file) { + if (!tlsconf.dh_param_file.empty()) { // Read DH parameters from file - auto bio = BIO_new_file(tlsconf.dh_param_file.get(), "r"); + auto bio = BIO_new_file(tlsconf.dh_param_file.c_str(), "r"); if (bio == nullptr) { LOG(FATAL) << "BIO_new_file() failed: " << ERR_error_string(ERR_get_error(), nullptr); @@ -548,7 +549,7 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file SSL_CTX_set_mode(ssl_ctx, SSL_MODE_AUTO_RETRY); SSL_CTX_set_mode(ssl_ctx, SSL_MODE_RELEASE_BUFFERS); - if (tlsconf.private_key_passwd) { + if (!tlsconf.private_key_passwd.empty()) { SSL_CTX_set_default_passwd_cb(ssl_ctx, ssl_pem_passwd_cb); SSL_CTX_set_default_passwd_cb_userdata(ssl_ctx, (void *)get_config()); } @@ -579,12 +580,12 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file DIE(); } if (tlsconf.client_verify.enabled) { - if (tlsconf.client_verify.cacert) { + if (!tlsconf.client_verify.cacert.empty()) { if (SSL_CTX_load_verify_locations( - ssl_ctx, tlsconf.client_verify.cacert.get(), nullptr) != 1) { + ssl_ctx, tlsconf.client_verify.cacert.c_str(), nullptr) != 1) { LOG(FATAL) << "Could not load trusted ca certificates from " - << tlsconf.client_verify.cacert.get() << ": " + << tlsconf.client_verify.cacert << ": " << ERR_error_string(ERR_get_error(), nullptr); DIE(); } @@ -592,10 +593,10 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file // error even though it returns success. See // http://forum.nginx.org/read.php?29,242540 ERR_clear_error(); - auto list = SSL_load_client_CA_file(tlsconf.client_verify.cacert.get()); + auto list = SSL_load_client_CA_file(tlsconf.client_verify.cacert.c_str()); if (!list) { LOG(FATAL) << "Could not load ca certificates from " - << tlsconf.client_verify.cacert.get() << ": " + << tlsconf.client_verify.cacert << ": " << ERR_error_string(ERR_get_error(), nullptr); DIE(); } @@ -658,12 +659,28 @@ int select_h1_next_proto_cb(SSL *ssl, unsigned char **out, } } // namespace +namespace { +int select_next_proto_cb(SSL *ssl, unsigned char **out, unsigned char *outlen, + const unsigned char *in, unsigned int inlen, + void *arg) { + auto conn = static_cast(SSL_get_app_data(ssl)); + switch (conn->proto) { + case PROTO_HTTP1: + return select_h1_next_proto_cb(ssl, out, outlen, in, inlen, arg); + case PROTO_HTTP2: + return select_h2_next_proto_cb(ssl, out, outlen, in, inlen, arg); + default: + return SSL_TLSEXT_ERR_NOACK; + } +} +} // namespace + SSL_CTX *create_ssl_client_context( #ifdef HAVE_NEVERBLEED neverbleed_t *nb, #endif // HAVE_NEVERBLEED const StringRef &cacert, const StringRef &cert_file, - const StringRef &private_key_file, const StringRef &alpn, + const StringRef &private_key_file, int (*next_proto_select_cb)(SSL *s, unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, void *arg)) { @@ -683,8 +700,8 @@ SSL_CTX *create_ssl_client_context( SSL_CTX_set_options(ssl_ctx, ssl_opts | tlsconf.tls_proto_mask); const char *ciphers; - if (tlsconf.ciphers) { - ciphers = tlsconf.ciphers.get(); + if (!tlsconf.ciphers.empty()) { + ciphers = tlsconf.ciphers.c_str(); } else { ciphers = nghttp2::ssl::DEFAULT_CIPHER_LIST; } @@ -741,14 +758,10 @@ SSL_CTX *create_ssl_client_context( #endif // HAVE_NEVERBLEED } - // NPN selection callback + // NPN selection callback. This is required to set SSL_CTX because + // OpenSSL does not offer SSL_set_next_proto_select_cb. SSL_CTX_set_next_proto_select_cb(ssl_ctx, next_proto_select_cb, nullptr); -#if OPENSSL_VERSION_NUMBER >= 0x10002000L - // ALPN advertisement - SSL_CTX_set_alpn_protos(ssl_ctx, alpn.byte(), alpn.size()); -#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L - return ssl_ctx; } @@ -1245,8 +1258,8 @@ SSL_CTX *setup_server_ssl_context(std::vector &all_ssl_ctx, auto &tlsconf = get_config()->tls; - auto ssl_ctx = ssl::create_ssl_context(tlsconf.private_key_file.get(), - tlsconf.cert_file.get() + auto ssl_ctx = ssl::create_ssl_context(tlsconf.private_key_file.c_str(), + tlsconf.cert_file.c_str() #ifdef HAVE_NEVERBLEED , nb @@ -1281,8 +1294,8 @@ SSL_CTX *setup_server_ssl_context(std::vector &all_ssl_ctx, } } - if (ssl::cert_lookup_tree_add_cert_from_file(cert_tree, ssl_ctx, - tlsconf.cert_file.get()) == -1) { + if (ssl::cert_lookup_tree_add_cert_from_file( + cert_tree, ssl_ctx, tlsconf.cert_file.c_str()) == -1) { LOG(FATAL) << "Failed to add default certificate."; DIE(); } @@ -1302,31 +1315,29 @@ SSL_CTX *setup_downstream_client_ssl_context( } auto &tlsconf = get_config()->tls; - auto &downstreamconf = get_config()->conn.downstream; - - std::vector h2alpn; - StringRef alpn; - int (*next_proto_select_cb)(SSL *s, unsigned char **out, - unsigned char *outlen, const unsigned char *in, - unsigned int inlen, void *arg); - - if (downstreamconf.proto == PROTO_HTTP2) { - h2alpn = util::get_default_alpn(); - alpn = StringRef(h2alpn.data(), h2alpn.size()); - next_proto_select_cb = select_h2_next_proto_cb; - } else { - alpn = StringRef::from_lit(NGHTTP2_H1_1_ALPN); - next_proto_select_cb = select_h1_next_proto_cb; - } return ssl::create_ssl_client_context( #ifdef HAVE_NEVERBLEED nb, #endif // HAVE_NEVERBLEED - StringRef::from_maybe_nullptr(tlsconf.cacert.get()), - StringRef::from_maybe_nullptr(tlsconf.client.cert_file.get()), - StringRef::from_maybe_nullptr(tlsconf.client.private_key_file.get()), - alpn, next_proto_select_cb); + StringRef{tlsconf.cacert}, StringRef{tlsconf.client.cert_file}, + StringRef{tlsconf.client.private_key_file}, select_next_proto_cb); +} + +void setup_downstream_http2_alpn(SSL *ssl) { +#if OPENSSL_VERSION_NUMBER >= 0x10002000L + // ALPN advertisement + auto alpn = util::get_default_alpn(); + SSL_set_alpn_protos(ssl, alpn.data(), alpn.size()); +#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L +} + +void setup_downstream_http1_alpn(SSL *ssl) { +#if OPENSSL_VERSION_NUMBER >= 0x10002000L + // ALPN advertisement + auto alpn = StringRef::from_lit(NGHTTP2_H1_1_ALPN); + SSL_set_alpn_protos(ssl, alpn.byte(), alpn.size()); +#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L } CertLookupTree *create_cert_lookup_tree() { @@ -1337,6 +1348,50 @@ CertLookupTree *create_cert_lookup_tree() { return new ssl::CertLookupTree(); } +namespace { +std::vector serialize_ssl_session(SSL_SESSION *session) { + auto len = i2d_SSL_SESSION(session, nullptr); + auto buf = std::vector(len); + auto p = buf.data(); + i2d_SSL_SESSION(session, &p); + + return buf; +} +} // namespace + +void try_cache_tls_session(DownstreamAddr *addr, SSL_SESSION *session, + ev_tstamp t) { + auto &cache = addr->tls_session_cache; + + if (cache.last_updated + 1_min > t) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Cache for addr=" << util::to_numeric_addr(&addr->addr) + << " is still host. Not updating."; + } + return; + } + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Update cache entry for SSL_SESSION=" << session + << ", addr=" << util::to_numeric_addr(&addr->addr) + << ", timestamp=" << std::fixed << std::setprecision(6) << t; + } + + cache.session_data = serialize_ssl_session(session); + cache.last_updated = t; +} + +SSL_SESSION *reuse_tls_session(const DownstreamAddr *addr) { + auto &cache = addr->tls_session_cache; + + if (cache.session_data.empty()) { + return nullptr; + } + + auto p = cache.session_data.data(); + return d2i_SSL_SESSION(nullptr, &p, cache.session_data.size()); +} + } // namespace ssl } // namespace shrpx diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h index ca93da14..900d9516 100644 --- a/src/shrpx_ssl.h +++ b/src/shrpx_ssl.h @@ -39,6 +39,8 @@ #include #endif // HAVE_NEVERBLEED +#include "network.h" + namespace shrpx { class ClientHandler; @@ -46,7 +48,6 @@ class Worker; class DownstreamConnectionPool; struct DownstreamAddr; struct UpstreamAddr; -struct Address; namespace ssl { @@ -70,13 +71,14 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file #endif // HAVE_NEVERBLEED ); -// Create client side SSL_CTX +// Create client side SSL_CTX. This does not configure ALPN settings. +// |next_proto_select_cb| is for NPN. SSL_CTX *create_ssl_client_context( #ifdef HAVE_NEVERBLEED neverbleed_t *nb, #endif // HAVE_NEVERBLEED const StringRef &cacert, const StringRef &cert_file, - const StringRef &private_key_file, const StringRef &alpn, + const StringRef &private_key_file, int (*next_proto_select_cb)(SSL *s, unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, void *arg)); @@ -200,6 +202,11 @@ SSL_CTX *setup_downstream_client_ssl_context( #endif // HAVE_NEVERBLEED ); +// Sets ALPN settings in |SSL| suitable for HTTP/2 use. +void setup_downstream_http2_alpn(SSL *ssl); +// Sets ALPN settings in |SSL| suitable for HTTP/1.1 use. +void setup_downstream_http1_alpn(SSL *ssl); + // Creates CertLookupTree. If frontend is configured not to use TLS, // this function returns nullptr. CertLookupTree *create_cert_lookup_tree(); @@ -216,6 +223,17 @@ bool downstream_tls_enabled(); bool tls_hostname_match(const char *pattern, size_t plen, const char *hostname, size_t hlen); +// Caches |session| which is associated to remote address |addr|. +// |session| is serialized into ASN1 representation, and stored. |t| +// is used as a time stamp. Depending on the existing cache's time +// stamp, |session| might not be cached. +void try_cache_tls_session(DownstreamAddr *addr, SSL_SESSION *session, + ev_tstamp t); + +// Returns cached session associated |addr|. If no cache entry is +// found associated to |addr|, nullptr will be returned. +SSL_SESSION *reuse_tls_session(const DownstreamAddr *addr); + } // namespace ssl } // namespace shrpx diff --git a/src/shrpx_upstream.h b/src/shrpx_upstream.h index b9639e72..df96830e 100644 --- a/src/shrpx_upstream.h +++ b/src/shrpx_upstream.h @@ -68,8 +68,7 @@ public: virtual int send_reply(Downstream *downstream, const uint8_t *body, size_t bodylen) = 0; - virtual int initiate_push(Downstream *downstream, const char *uri, - size_t len) = 0; + virtual int initiate_push(Downstream *downstream, const StringRef &uri) = 0; // Fills response data in |iov| whose capacity is |iovcnt|. Returns // the number of iovs filled. diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index e663eace..601c8143 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -29,7 +29,6 @@ #endif // HAVE_UNISTD_H #include -#include #include "shrpx_ssl.h" #include "shrpx_log.h" @@ -72,15 +71,14 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, ssl::CertLookupTree *cert_tree, const std::shared_ptr &ticket_keys) : randgen_(rd()), - dconn_pool_(get_config()->conn.downstream.addr_groups.size()), - worker_stat_(get_config()->conn.downstream.addr_groups.size()), - dgrps_(get_config()->conn.downstream.addr_groups.size()), + worker_stat_{}, loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), cert_tree_(cert_tree), ticket_keys_(ticket_keys), - connect_blocker_(make_unique(loop_)), + downstream_addr_groups_(get_config()->conn.downstream.addr_groups.size()), + connect_blocker_(make_unique(randgen_, loop_)), graceful_shutdown_(false) { ev_async_init(&w_, eventcb); w_.data = this; @@ -91,28 +89,34 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, auto &session_cacheconf = get_config()->tls.session_cache; - if (session_cacheconf.memcached.host) { + if (!session_cacheconf.memcached.host.empty()) { session_cache_memcached_dispatcher_ = make_unique( &session_cacheconf.memcached.addr, loop, tls_session_cache_memcached_ssl_ctx, - session_cacheconf.memcached.host.get(), &mcpool_); + StringRef{session_cacheconf.memcached.host}, &mcpool_); } auto &downstreamconf = get_config()->conn.downstream; - if (downstreamconf.proto == PROTO_HTTP2) { - auto n = get_config()->http2.downstream.connections_per_worker; - size_t group = 0; - for (auto &dgrp : dgrps_) { - auto m = n; - if (m == 0) { - m = downstreamconf.addr_groups[group].addrs.size(); - } - for (size_t idx = 0; idx < m; ++idx) { - dgrp.http2sessions.push_back(make_unique( - loop_, cl_ssl_ctx, connect_blocker_.get(), this, group, idx)); - } - ++group; + for (size_t i = 0; i < downstreamconf.addr_groups.size(); ++i) { + auto &src = downstreamconf.addr_groups[i]; + auto &dst = downstream_addr_groups_[i]; + + dst.pattern = src.pattern; + dst.addrs.resize(src.addrs.size()); + dst.proto = src.proto; + + for (size_t j = 0; j < src.addrs.size(); ++j) { + auto &src_addr = src.addrs[j]; + auto &dst_addr = dst.addrs[j]; + + dst_addr.addr = src_addr.addr; + dst_addr.host = src_addr.host; + dst_addr.hostport = src_addr.hostport; + dst_addr.port = src_addr.port; + dst_addr.host_unix = src_addr.host_unix; + + dst_addr.connect_blocker = make_unique(randgen_, loop_); } } } @@ -241,28 +245,6 @@ void Worker::set_ticket_keys(std::shared_ptr ticket_keys) { WorkerStat *Worker::get_worker_stat() { return &worker_stat_; } -DownstreamConnectionPool *Worker::get_dconn_pool() { return &dconn_pool_; } - -Http2Session *Worker::next_http2_session(size_t group) { - auto &dgrp = dgrps_[group]; - auto &http2sessions = dgrp.http2sessions; - if (http2sessions.empty()) { - return nullptr; - } - - auto res = http2sessions[dgrp.next_http2session].get(); - ++dgrp.next_http2session; - if (dgrp.next_http2session >= http2sessions.size()) { - dgrp.next_http2session = 0; - } - - return res; -} - -ConnectBlocker *Worker::get_connect_blocker() const { - return connect_blocker_.get(); -} - struct ev_loop *Worker::get_loop() const { return loop_; } @@ -277,11 +259,6 @@ bool Worker::get_graceful_shutdown() const { return graceful_shutdown_; } MemchunkPool *Worker::get_mcpool() { return &mcpool_; } -DownstreamGroup *Worker::get_dgrp(size_t group) { - assert(group < dgrps_.size()); - return &dgrps_[group]; -} - MemcachedDispatcher *Worker::get_session_cache_memcached_dispatcher() { return session_cache_memcached_dispatcher_.get(); } @@ -290,8 +267,7 @@ std::mt19937 &Worker::get_randgen() { return randgen_; } #ifdef HAVE_MRUBY int Worker::create_mruby_context() { - auto mruby_file = get_config()->mruby_file.get(); - mruby_ctx_ = mruby::create_mruby_context(mruby_file); + mruby_ctx_ = mruby::create_mruby_context(StringRef{get_config()->mruby_file}); if (!mruby_ctx_) { return -1; } @@ -304,62 +280,128 @@ mruby::MRubyContext *Worker::get_mruby_context() const { } #endif // HAVE_MRUBY -namespace { -std::vector serialize_ssl_session(SSL_SESSION *session) { - auto len = i2d_SSL_SESSION(session, nullptr); - auto buf = std::vector(len); - auto p = buf.data(); - i2d_SSL_SESSION(session, &p); - - return buf; +std::vector &Worker::get_downstream_addr_groups() { + return downstream_addr_groups_; } -} // namespace -void Worker::cache_client_tls_session(const Address *addr, SSL_SESSION *session, - ev_tstamp t) { - auto it = client_tls_session_cache_.find(addr); - if (it == std::end(client_tls_session_cache_)) { - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Create cache entry for SSL_SESSION=" << session - << ", addr=" << util::numeric_hostport(&addr->su.sa, addr->len) - << "(" << addr << "), timestamp=" << std::fixed - << std::setprecision(6) << t; - } - client_tls_session_cache_.emplace( - addr, SessionCacheEntry{serialize_ssl_session(session), t}); - return; - } +ConnectBlocker *Worker::get_connect_blocker() const { + return connect_blocker_.get(); +} - auto &ent = (*it).second; - if (ent.last_updated + 1_min > t) { - if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Cache for addr=" - << util::numeric_hostport(&addr->su.sa, addr->len) << "(" - << addr << ") is still host. Not updating."; +namespace { +size_t match_downstream_addr_group_host( + const Router &router, const std::vector &wildcard_patterns, + const StringRef &host, const StringRef &path, + const std::vector &groups, size_t catch_all) { + if (path.empty() || path[0] != '/') { + auto group = router.match(host, StringRef::from_lit("/")); + if (group != -1) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Found pattern with query " << host + << ", matched pattern=" << groups[group].pattern; + } + return group; } - return; + return catch_all; } if (LOG_ENABLED(INFO)) { - LOG(INFO) << "Update cache entry for SSL_SESSION=" << session - << ", addr=" << util::numeric_hostport(&addr->su.sa, addr->len) - << "(" << addr << "), timestamp=" << std::fixed - << std::setprecision(6) << t; + LOG(INFO) << "Perform mapping selection, using host=" << host + << ", path=" << path; } - ent.session_data = serialize_ssl_session(session); - ent.last_updated = t; + auto group = router.match(host, path); + if (group != -1) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Found pattern with query " << host << path + << ", matched pattern=" << groups[group].pattern; + } + return group; + } + + for (auto it = std::begin(wildcard_patterns); + it != std::end(wildcard_patterns); ++it) { + if (!util::ends_with(std::begin(host), std::end(host), + std::begin((*it).host), std::end((*it).host))) { + continue; + } + auto group = (*it).router.match(StringRef{}, path); + if (group != -1) { + // We sorted wildcard_patterns in a way that first match is the + // longest host pattern. + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Found wildcard pattern with query " << host << path + << ", matched pattern=" << groups[group].pattern; + } + return group; + } + } + + group = router.match(StringRef::from_lit(""), path); + if (group != -1) { + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "Found pattern with query " << path + << ", matched pattern=" << groups[group].pattern; + } + return group; + } + + if (LOG_ENABLED(INFO)) { + LOG(INFO) << "None match. Use catch-all pattern"; + } + return catch_all; } +} // namespace -SSL_SESSION *Worker::reuse_client_tls_session(const Address *addr) { - auto it = client_tls_session_cache_.find(addr); - if (it == std::end(client_tls_session_cache_)) { - return nullptr; +size_t match_downstream_addr_group( + const Router &router, const std::vector &wildcard_patterns, + const StringRef &hostport, const StringRef &raw_path, + const std::vector &groups, size_t catch_all) { + if (std::find(std::begin(hostport), std::end(hostport), '/') != + std::end(hostport)) { + // We use '/' specially, and if '/' is included in host, it breaks + // our code. Select catch-all case. + return catch_all; } - const auto &ent = (*it).second; - auto p = ent.session_data.data(); - return d2i_SSL_SESSION(nullptr, &p, ent.session_data.size()); + auto fragment = std::find(std::begin(raw_path), std::end(raw_path), '#'); + auto query = std::find(std::begin(raw_path), fragment, '?'); + auto path = StringRef{std::begin(raw_path), query}; + + if (hostport.empty()) { + return match_downstream_addr_group_host(router, wildcard_patterns, hostport, + path, groups, catch_all); + } + + StringRef host; + if (hostport[0] == '[') { + // assume this is IPv6 numeric address + auto p = std::find(std::begin(hostport), std::end(hostport), ']'); + if (p == std::end(hostport)) { + return catch_all; + } + if (p + 1 < std::end(hostport) && *(p + 1) != ':') { + return catch_all; + } + host = StringRef{std::begin(hostport), p + 1}; + } else { + auto p = std::find(std::begin(hostport), std::end(hostport), ':'); + if (p == std::begin(hostport)) { + return catch_all; + } + host = StringRef{std::begin(hostport), p}; + } + + std::string low_host; + if (std::find_if(std::begin(host), std::end(host), [](char c) { + return 'A' <= c || c <= 'Z'; + }) != std::end(host)) { + low_host = host.str(); + util::inp_strlower(low_host); + host = StringRef{low_host}; + } + return match_downstream_addr_group_host(router, wildcard_patterns, host, path, + groups, catch_all); } } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 8d352429..17a9fee9 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -67,20 +67,41 @@ namespace ssl { class CertLookupTree; } // namespace ssl -struct DownstreamGroup { - DownstreamGroup() : next_http2session(0), next(0) {} +struct DownstreamAddr { + Address addr; + // backend address. If |host_unix| is true, this is UNIX domain + // socket path. + ImmutableString host; + ImmutableString hostport; + // backend port. 0 if |host_unix| is true. + uint16_t port; + // true if |host| contains UNIX domain socket path. + bool host_unix; - std::vector> http2sessions; - // Next index in http2sessions. - size_t next_http2session; - // Next downstream address index corresponding to - // Config::downstream_addr_groups[]. + std::unique_ptr connect_blocker; + // Client side TLS session cache + TLSSessionCache tls_session_cache; +}; + +struct DownstreamAddrGroup { + ImmutableString pattern; + std::vector addrs; + // Application protocol used in this group + shrpx_proto proto; + // List of Http2Session which is not fully utilized (i.e., the + // server advertized maximum concurrency is not reached). We will + // coalesce as much stream as possible in one Http2Session to fully + // utilize TCP connection. + // + // TODO Verify that this approach performs better in performance + // wise. + DList http2_freelist; + DownstreamConnectionPool dconn_pool; + // Next downstream address index in addrs. size_t next; }; struct WorkerStat { - WorkerStat(size_t num_groups) : num_connections(0) {} - size_t num_connections; }; @@ -101,14 +122,6 @@ struct WorkerEvent { std::shared_ptr ticket_keys; }; -struct SessionCacheEntry { - // ASN1 representation of SSL_SESSION object. See - // i2d_SSL_SESSION(3SSL). - std::vector session_data; - // The last time stamp when this cache entry is created or updated. - ev_tstamp last_updated; -}; - class Worker { public: Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, @@ -129,9 +142,6 @@ public: void set_ticket_keys(std::shared_ptr ticket_keys); WorkerStat *get_worker_stat(); - DownstreamConnectionPool *get_dconn_pool(); - Http2Session *next_http2_session(size_t group); - ConnectBlocker *get_connect_blocker() const; struct ev_loop *get_loop() const; SSL_CTX *get_sv_ssl_ctx() const; SSL_CTX *get_cl_ssl_ctx() const; @@ -142,8 +152,6 @@ public: MemchunkPool *get_mcpool(); void schedule_clear_mcpool(); - DownstreamGroup *get_dgrp(size_t group); - MemcachedDispatcher *get_session_cache_memcached_dispatcher(); std::mt19937 &get_randgen(); @@ -154,15 +162,9 @@ public: mruby::MRubyContext *get_mruby_context() const; #endif // HAVE_MRUBY - // Caches |session| which is associated to remote address |addr|. - // |session| is serialized into ASN1 representation, and stored. - // |t| is used as a time stamp. Depending on the existing cache's - // time stamp, |session| might not be cached. - void cache_client_tls_session(const Address *addr, SSL_SESSION *session, - ev_tstamp t); - // Returns cached session associated |addr|. If no cache entry is - // found associated to |addr|, nullptr will be returned. - SSL_SESSION *reuse_client_tls_session(const Address *addr); + std::vector &get_downstream_addr_groups(); + + ConnectBlocker *get_connect_blocker() const; private: #ifndef NOTHREADS @@ -174,14 +176,7 @@ private: ev_async w_; ev_timer mcpool_clear_timer_; MemchunkPool mcpool_; - DownstreamConnectionPool dconn_pool_; WorkerStat worker_stat_; - std::vector dgrps_; - - // Client side SSL_SESSION cache. SSL_SESSION is associated to - // remote address. - std::unordered_map - client_tls_session_cache_; std::unique_ptr session_cache_memcached_dispatcher_; #ifdef HAVE_MRUBY @@ -196,11 +191,25 @@ private: ssl::CertLookupTree *cert_tree_; std::shared_ptr ticket_keys_; + std::vector downstream_addr_groups_; + // Worker level blocker for downstream connection. For example, + // this is used when file decriptor is exhausted. std::unique_ptr connect_blocker_; bool graceful_shutdown_; }; +// Selects group based on request's |hostport| and |path|. |hostport| +// is the value taken from :authority or host header field, and may +// contain port. The |path| may contain query part. We require the +// catch-all pattern in place, so this function always selects one +// group. The catch-all group index is given in |catch_all|. All +// patterns are given in |groups|. +size_t match_downstream_addr_group( + const Router &router, const std::vector &wildcard_patterns, + const StringRef &hostport, const StringRef &path, + const std::vector &groups, size_t catch_all); + } // namespace shrpx #endif // SHRPX_WORKER_H diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc index 26863d82..8d09b5d2 100644 --- a/src/shrpx_worker_process.cc +++ b/src/shrpx_worker_process.cc @@ -64,7 +64,7 @@ void drop_privileges( #endif // HAVE_NEVERBLEED ) { if (getuid() == 0 && get_config()->uid != 0) { - if (initgroups(get_config()->user.get(), get_config()->gid) != 0) { + if (initgroups(get_config()->user.c_str(), get_config()->gid) != 0) { auto error = errno; LOG(FATAL) << "Could not change supplementary groups: " << strerror(error); @@ -86,7 +86,7 @@ void drop_privileges( } #ifdef HAVE_NEVERBLEED if (nb) { - neverbleed_setuidgid(nb, get_config()->user.get(), 1); + neverbleed_setuidgid(nb, get_config()->user.c_str(), 1); } #endif // HAVE_NEVERBLEED } @@ -427,7 +427,7 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { auto &ticketconf = get_config()->tls.ticket; auto &memcachedconf = ticketconf.memcached; - if (ticketconf.memcached.host) { + if (!memcachedconf.host.empty()) { SSL_CTX *ssl_ctx = nullptr; if (memcachedconf.tls) { @@ -437,7 +437,7 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) { conn_handler.set_tls_ticket_key_memcached_dispatcher( make_unique( &ticketconf.memcached.addr, loop, ssl_ctx, - StringRef(memcachedconf.host.get()), &mcpool)); + StringRef{memcachedconf.host}, &mcpool)); ev_timer_init(&renew_ticket_key_timer, memcached_get_ticket_key_cb, 0., 0.); diff --git a/src/shrpx_worker_test.cc b/src/shrpx_worker_test.cc new file mode 100644 index 00000000..ae2f0e80 --- /dev/null +++ b/src/shrpx_worker_test.cc @@ -0,0 +1,216 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "shrpx_worker_test.h" + +#ifdef HAVE_UNISTD_H +#include +#endif // HAVE_UNISTD_H + +#include + +#include + +#include "shrpx_worker.h" +#include "shrpx_connect_blocker.h" + +namespace shrpx { + +void test_shrpx_worker_match_downstream_addr_group(void) { + auto groups = std::vector(); + for (auto &s : {"nghttp2.org/", "nghttp2.org/alpha/bravo/", + "nghttp2.org/alpha/charlie", "nghttp2.org/delta%3A", + "www.nghttp2.org/", "[::1]/", "nghttp2.org/alpha/bravo/delta", + // Check that match is done in the single node + "example.com/alpha/bravo", "192.168.0.1/alpha/", "/golf/"}) { + groups.push_back(DownstreamAddrGroup{ImmutableString(s)}); + } + + Router router; + + for (size_t i = 0; i < groups.size(); ++i) { + auto &g = groups[i]; + router.add_route(StringRef{g.pattern}, i); + } + + std::vector wp; + + CU_ASSERT(0 == match_downstream_addr_group( + router, wp, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/"), groups, 255)); + + // port is removed + CU_ASSERT(0 == match_downstream_addr_group( + router, wp, StringRef::from_lit("nghttp2.org:8080"), + StringRef::from_lit("/"), groups, 255)); + + // host is case-insensitive + CU_ASSERT(4 == match_downstream_addr_group( + router, wp, StringRef::from_lit("WWW.nghttp2.org"), + StringRef::from_lit("/alpha"), groups, 255)); + + CU_ASSERT(1 == match_downstream_addr_group( + router, wp, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/alpha/bravo/"), groups, 255)); + + // /alpha/bravo also matches /alpha/bravo/ + CU_ASSERT(1 == match_downstream_addr_group( + router, wp, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/alpha/bravo"), groups, 255)); + + // path part is case-sensitive + CU_ASSERT(0 == match_downstream_addr_group( + router, wp, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/Alpha/bravo"), groups, 255)); + + CU_ASSERT(1 == match_downstream_addr_group( + router, wp, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/alpha/bravo/charlie"), groups, 255)); + + CU_ASSERT(2 == match_downstream_addr_group( + router, wp, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/alpha/charlie"), groups, 255)); + + // pattern which does not end with '/' must match its entirely. So + // this matches to group 0, not group 2. + CU_ASSERT(0 == match_downstream_addr_group( + router, wp, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/alpha/charlie/"), groups, 255)); + + CU_ASSERT(255 == match_downstream_addr_group( + router, wp, StringRef::from_lit("example.org"), + StringRef::from_lit("/"), groups, 255)); + + CU_ASSERT(255 == + match_downstream_addr_group(router, wp, StringRef::from_lit(""), + StringRef::from_lit("/"), groups, 255)); + + CU_ASSERT(255 == match_downstream_addr_group( + router, wp, StringRef::from_lit(""), + StringRef::from_lit("alpha"), groups, 255)); + + CU_ASSERT(255 == match_downstream_addr_group( + router, wp, StringRef::from_lit("foo/bar"), + StringRef::from_lit("/"), groups, 255)); + + // If path is StringRef::from_lit("*", only match with host + "/"). + CU_ASSERT(0 == match_downstream_addr_group( + router, wp, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("*"), groups, 255)); + + CU_ASSERT( + 5 == match_downstream_addr_group(router, wp, StringRef::from_lit("[::1]"), + StringRef::from_lit("/"), groups, 255)); + CU_ASSERT(5 == match_downstream_addr_group( + router, wp, StringRef::from_lit("[::1]:8080"), + StringRef::from_lit("/"), groups, 255)); + CU_ASSERT(255 == + match_downstream_addr_group(router, wp, StringRef::from_lit("[::1"), + StringRef::from_lit("/"), groups, 255)); + CU_ASSERT(255 == match_downstream_addr_group( + router, wp, StringRef::from_lit("[::1]8000"), + StringRef::from_lit("/"), groups, 255)); + + // Check the case where adding route extends tree + CU_ASSERT(6 == match_downstream_addr_group( + router, wp, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/alpha/bravo/delta"), groups, 255)); + + CU_ASSERT(1 == match_downstream_addr_group( + router, wp, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/alpha/bravo/delta/"), groups, 255)); + + // Check the case where query is done in a single node + CU_ASSERT(7 == match_downstream_addr_group( + router, wp, StringRef::from_lit("example.com"), + StringRef::from_lit("/alpha/bravo"), groups, 255)); + + CU_ASSERT(255 == match_downstream_addr_group( + router, wp, StringRef::from_lit("example.com"), + StringRef::from_lit("/alpha/bravo/"), groups, 255)); + + CU_ASSERT(255 == match_downstream_addr_group( + router, wp, StringRef::from_lit("example.com"), + StringRef::from_lit("/alpha"), groups, 255)); + + // Check the case where quey is done in a single node + CU_ASSERT(8 == match_downstream_addr_group( + router, wp, StringRef::from_lit("192.168.0.1"), + StringRef::from_lit("/alpha"), groups, 255)); + + CU_ASSERT(8 == match_downstream_addr_group( + router, wp, StringRef::from_lit("192.168.0.1"), + StringRef::from_lit("/alpha/"), groups, 255)); + + CU_ASSERT(8 == match_downstream_addr_group( + router, wp, StringRef::from_lit("192.168.0.1"), + StringRef::from_lit("/alpha/bravo"), groups, 255)); + + CU_ASSERT(255 == match_downstream_addr_group( + router, wp, StringRef::from_lit("192.168.0.1"), + StringRef::from_lit("/alph"), groups, 255)); + + CU_ASSERT(255 == match_downstream_addr_group( + router, wp, StringRef::from_lit("192.168.0.1"), + StringRef::from_lit("/"), groups, 255)); + + // Test for wildcard hosts + groups.push_back( + DownstreamAddrGroup{ImmutableString::from_lit("git.nghttp2.org")}); + groups.push_back( + DownstreamAddrGroup{ImmutableString::from_lit(".nghttp2.org")}); + + wp.push_back({ImmutableString("git.nghttp2.org")}); + wp.back().router.add_route(StringRef::from_lit("/echo/"), 10); + + wp.push_back({ImmutableString(".nghttp2.org")}); + wp.back().router.add_route(StringRef::from_lit("/echo/"), 11); + wp.back().router.add_route(StringRef::from_lit("/echo/foxtrot"), 12); + + CU_ASSERT(10 == match_downstream_addr_group( + router, wp, StringRef::from_lit("git.nghttp2.org"), + StringRef::from_lit("/echo"), groups, 255)); + + CU_ASSERT(10 == match_downstream_addr_group( + router, wp, StringRef::from_lit("0git.nghttp2.org"), + StringRef::from_lit("/echo"), groups, 255)); + + CU_ASSERT(11 == match_downstream_addr_group( + router, wp, StringRef::from_lit("it.nghttp2.org"), + StringRef::from_lit("/echo"), groups, 255)); + + CU_ASSERT(12 == match_downstream_addr_group( + router, wp, StringRef::from_lit(".nghttp2.org"), + StringRef::from_lit("/echo/foxtrot"), groups, 255)); + + CU_ASSERT(9 == match_downstream_addr_group( + router, wp, StringRef::from_lit("alpha.nghttp2.org"), + StringRef::from_lit("/golf"), groups, 255)); + + CU_ASSERT(0 == match_downstream_addr_group( + router, wp, StringRef::from_lit("nghttp2.org"), + StringRef::from_lit("/echo"), groups, 255)); +} + +} // namespace shrpx diff --git a/src/shrpx_worker_test.h b/src/shrpx_worker_test.h new file mode 100644 index 00000000..c9bdfe0d --- /dev/null +++ b/src/shrpx_worker_test.h @@ -0,0 +1,38 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2016 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef SHRPX_WORKER_TEST_H +#define SHRPX_WORKER_TEST_H + +#ifdef HAVE_CONFIG_H +#include +#endif // HAVE_CONFIG_H + +namespace shrpx { + +void test_shrpx_worker_match_downstream_addr_group(void); + +} // namespace shrpx + +#endif // SHRPX_WORKER_TEST_H diff --git a/src/template.h b/src/template.h index 3061ad52..fdbc2aac 100644 --- a/src/template.h +++ b/src/template.h @@ -99,14 +99,14 @@ template bool test_flags(T t, F flags) { // T *dlnext, which point to previous element and next element in the // list respectively. template struct DList { - DList() : head(nullptr), tail(nullptr) {} + DList() : head(nullptr), tail(nullptr), n(0) {} DList(const DList &) = delete; - DList &operator=(const DList &) = delete; - DList(DList &&other) : head(other.head), tail(other.tail) { + DList(DList &&other) : head(other.head), tail(other.tail), n(other.n) { other.head = other.tail = nullptr; + other.n = 0; } DList &operator=(DList &&other) { @@ -115,11 +115,16 @@ template struct DList { } head = other.head; tail = other.tail; + n = other.n; + other.head = other.tail = nullptr; + other.n = 0; + return *this; } void append(T *t) { + ++n; if (tail) { tail->dlnext = t; t->dlprev = tail; @@ -130,6 +135,7 @@ template struct DList { } void remove(T *t) { + --n; auto p = t->dlprev; auto n = t->dlnext; if (p) { @@ -149,7 +155,10 @@ template struct DList { bool empty() const { return head == nullptr; } + size_t size() const { return n; } + T *head, *tail; + size_t n; }; template void dlist_delete_all(DList &dl) { @@ -241,6 +250,7 @@ public: using const_reference = const value_type &; using const_pointer = const value_type *; using const_iterator = const_pointer; + using const_reverse_iterator = std::reverse_iterator; ImmutableString() : len(0), base("") {} ImmutableString(const char *s, size_t slen) @@ -299,6 +309,16 @@ public: const_iterator end() const { return base + len; }; const_iterator cend() const { return base + len; }; + const_reverse_iterator rbegin() const { + return const_reverse_iterator{base + len}; + } + const_reverse_iterator crbegin() const { + return const_reverse_iterator{base + len}; + } + + const_reverse_iterator rend() const { return const_reverse_iterator{base}; } + const_reverse_iterator crend() const { return const_reverse_iterator{base}; } + const char *c_str() const { return base; } size_type size() const { return len; } bool empty() const { return len == 0; } @@ -386,21 +406,27 @@ public: using const_reference = const value_type &; using const_pointer = const value_type *; using const_iterator = const_pointer; + using const_reverse_iterator = std::reverse_iterator; constexpr StringRef() : base(""), len(0) {} explicit StringRef(const std::string &s) : base(s.c_str()), len(s.size()) {} explicit StringRef(const ImmutableString &s) : base(s.c_str()), len(s.size()) {} - StringRef(const char *s) : base(s), len(strlen(s)) {} + explicit StringRef(const char *s) : base(s), len(strlen(s)) {} + constexpr StringRef(const char *s, size_t n) : base(s), len(n) {} template - constexpr StringRef(const CharT *s, size_t n) + StringRef(const CharT *s, size_t n) : base(reinterpret_cast(s)), len(n) {} template StringRef(InputIt first, InputIt last) - : base(first), len(std::distance(first, last)) {} + : base(&*first), len(std::distance(first, last)) {} + template + StringRef(InputIt *first, InputIt *last) + : base(reinterpret_cast(first)), + len(std::distance(first, last)) {} template constexpr static StringRef from_lit(const CharT(&s)[N]) { - return StringRef(s, N - 1); + return StringRef{s, N - 1}; } static StringRef from_maybe_nullptr(const char *s) { if (s == nullptr) { @@ -416,6 +442,16 @@ public: const_iterator end() const { return base + len; }; const_iterator cend() const { return base + len; }; + const_reverse_iterator rbegin() const { + return const_reverse_iterator{base + len}; + } + const_reverse_iterator crbegin() const { + return const_reverse_iterator{base + len}; + } + + const_reverse_iterator rend() const { return const_reverse_iterator{base}; } + const_reverse_iterator crend() const { return const_reverse_iterator{base}; } + const char *c_str() const { return base; } size_type size() const { return len; } bool empty() const { return len == 0; } @@ -431,6 +467,11 @@ private: size_type len; }; +inline bool operator==(const StringRef &lhs, const StringRef &rhs) { + return lhs.size() == rhs.size() && + std::equal(std::begin(lhs), std::end(lhs), std::begin(rhs)); +} + inline bool operator==(const StringRef &lhs, const std::string &rhs) { return lhs.size() == rhs.size() && std::equal(std::begin(lhs), std::end(lhs), std::begin(rhs)); @@ -445,6 +486,15 @@ inline bool operator==(const StringRef &lhs, const char *rhs) { std::equal(std::begin(lhs), std::end(lhs), rhs); } +inline bool operator==(const StringRef &lhs, const ImmutableString &rhs) { + return lhs.size() == rhs.size() && + std::equal(std::begin(lhs), std::end(lhs), std::begin(rhs)); +} + +inline bool operator==(const ImmutableString &lhs, const StringRef &rhs) { + return rhs == lhs; +} + inline bool operator==(const char *lhs, const StringRef &rhs) { return rhs == lhs; } @@ -465,6 +515,11 @@ inline bool operator!=(const char *lhs, const StringRef &rhs) { return !(rhs == lhs); } +inline bool operator<(const StringRef &lhs, const StringRef &rhs) { + return std::lexicographical_compare(std::begin(lhs), std::end(lhs), + std::begin(rhs), std::end(rhs)); +} + inline std::ostream &operator<<(std::ostream &o, const StringRef &s) { return o.write(s.c_str(), s.size()); } diff --git a/src/util.cc b/src/util.cc index 0897a0f7..35fc8275 100644 --- a/src/util.cc +++ b/src/util.cc @@ -339,7 +339,7 @@ std::string iso8601_date(int64_t ms) { return res; } -time_t parse_http_date(const std::string &s) { +time_t parse_http_date(const StringRef &s) { tm tm{}; char *r = strptime(s.c_str(), "%a, %d %b %Y %H:%M:%S GMT", &tm); if (r == 0) { @@ -591,13 +591,13 @@ bool fieldeq(const char *uri, const http_parser_url &u, return i == len && !t[i]; } -std::string get_uri_field(const char *uri, const http_parser_url &u, - http_parser_url_fields field) { - if (util::has_uri_field(u, field)) { - return std::string(uri + u.field_data[field].off, u.field_data[field].len); - } else { - return ""; +StringRef get_uri_field(const char *uri, const http_parser_url &u, + http_parser_url_fields field) { + if (!util::has_uri_field(u, field)) { + return StringRef{}; } + + return StringRef{uri + u.field_data[field].off, u.field_data[field].len}; } uint16_t get_default_port(const char *uri, const http_parser_url &u) { @@ -650,15 +650,17 @@ std::string numeric_name(const struct sockaddr *sa, socklen_t salen) { return host.data(); } -std::string numeric_hostport(const struct sockaddr *sa, socklen_t salen) { - if (sa->sa_family == AF_UNIX) { - return "localhost"; +std::string to_numeric_addr(const Address *addr) { + auto family = addr->su.storage.ss_family; + if (family == AF_UNIX) { + return addr->su.un.sun_path; } std::array host; std::array serv; - auto rv = getnameinfo(sa, salen, host.data(), host.size(), serv.data(), - serv.size(), NI_NUMERICHOST | NI_NUMERICSERV); + auto rv = + getnameinfo(&addr->su.sa, addr->len, host.data(), host.size(), + serv.data(), serv.size(), NI_NUMERICHOST | NI_NUMERICSERV); if (rv != 0) { return "unknown"; } @@ -668,7 +670,7 @@ std::string numeric_hostport(const struct sockaddr *sa, socklen_t salen) { std::string s; char *p; - if (sa->sa_family == AF_INET6) { + if (family == AF_INET6) { s.resize(hostlen + servlen + 2 + 1); p = &s[0]; *p++ = '['; @@ -855,6 +857,27 @@ std::vector get_default_alpn() { return res; } +std::vector split_str(const StringRef &s, char delim) { + size_t len = 1; + auto last = std::end(s); + for (auto first = std::begin(s), d = first; + (d = std::find(first, last, delim)) != last; ++len, first = d + 1) + ; + + auto list = std::vector(len); + + len = 0; + for (auto first = std::begin(s);; ++len) { + auto stop = std::find(first, last, delim); + list[len] = StringRef{first, stop}; + if (stop == last) { + break; + } + first = stop + 1; + } + return list; +} + std::vector> split_config_str_list(const char *s, char delim) { size_t len = 1; @@ -1033,6 +1056,10 @@ int64_t parse_uint(const std::string &s) { return parse_uint(reinterpret_cast(s.c_str()), s.size()); } +int64_t parse_uint(const StringRef &s) { + return parse_uint(s.byte(), s.size()); +} + int64_t parse_uint(const uint8_t *s, size_t len) { int64_t n; size_t i; diff --git a/src/util.h b/src/util.h index b1f64d77..2d3208fc 100644 --- a/src/util.h +++ b/src/util.h @@ -50,6 +50,8 @@ #include "http-parser/http_parser.h" #include "template.h" +#include "network.h" +#include "allocator.h" namespace nghttp2 { @@ -148,7 +150,7 @@ std::string common_log_date(time_t t); // 2014-11-15T12:58:24.741Z) std::string iso8601_date(int64_t ms); -time_t parse_http_date(const std::string &s); +time_t parse_http_date(const StringRef &s); char upcase(char c); @@ -224,6 +226,11 @@ bool istarts_with_l(const std::string &a, const CharT(&b)[N]) { return istarts_with(std::begin(a), std::end(a), b, b + N - 1); } +template +bool istarts_with_l(const StringRef &a, const CharT(&b)[N]) { + return istarts_with(std::begin(a), std::end(a), b, b + N - 1); +} + template bool ends_with(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2) { @@ -255,6 +262,11 @@ bool iends_with_l(const std::string &a, const CharT(&b)[N]) { return iends_with(std::begin(a), std::end(a), b, b + N - 1); } +template +bool iends_with_l(const StringRef &a, const CharT(&b)[N]) { + return iends_with(std::begin(a), std::end(a), b, b + N - 1); +} + int strcompare(const char *a, const uint8_t *b, size_t n); template bool strieq(const char *a, InputIt b, size_t bn) { @@ -304,6 +316,11 @@ bool strieq_l(const CharT(&a)[N], const std::string &b) { return strieq(a, N - 1, std::begin(b), b.size()); } +template +bool strieq_l(const CharT(&a)[N], const StringRef &b) { + return strieq(a, N - 1, std::begin(b), b.size()); +} + template bool streq(const char *a, InputIt b, size_t bn) { if (!a) { return false; @@ -371,6 +388,33 @@ template std::string utos(T n) { return res; } +template OutputIt utos(OutputIt dst, T n) { + if (n == 0) { + *dst++ = '0'; + return dst; + } + int i = 0; + T t = n; + for (; t; t /= 10, ++i) + ; + --i; + auto p = dst + i; + auto res = p + 1; + for (; n; --i, n /= 10) { + *p-- = (n % 10) + '0'; + } + return res; +} + +template +StringRef make_string_ref_uint(BlockAllocator &balloc, T n) { + auto iov = make_byte_ref(balloc, str_size("18446744073709551615") + 1); + auto p = iov.base; + p = util::utos(p, n); + *p = '\0'; + return StringRef{iov.base, p}; +} + template std::string utos_unit(T n) { char u = 0; if (n >= (1 << 30)) { @@ -442,8 +486,8 @@ bool fieldeq(const char *uri1, const http_parser_url &u1, const char *uri2, bool fieldeq(const char *uri, const http_parser_url &u, http_parser_url_fields field, const char *t); -std::string get_uri_field(const char *uri, const http_parser_url &u, - http_parser_url_fields field); +StringRef get_uri_field(const char *uri, const http_parser_url &u, + http_parser_url_fields field); uint16_t get_default_port(const char *uri, const http_parser_url &u); @@ -461,10 +505,11 @@ bool numeric_host(const char *hostname, int family); // failed, "unknown" is returned. std::string numeric_name(const struct sockaddr *sa, socklen_t salen); -// Returns string representation of numeric address and port of |addr| -// of length |salen|. The format is like :. For IPv6 -// address, address is enclosed by square brackets ([]). -std::string numeric_hostport(const struct sockaddr *sa, socklen_t salen); +// Returns string representation of numeric address and port of +// |addr|. If address family is AF_UNIX, this return path to UNIX +// domain socket. Otherwise, the format is like :. For +// IPv6 address, address is enclosed by square brackets ([]). +std::string to_numeric_addr(const Address *addr); // Makes internal copy of stderr (and possibly stdout in the future), // which is then used as pointer to /dev/stderr or /proc/self/fd/2 @@ -541,6 +586,11 @@ std::vector parse_config_str_list(const char *s, char delim = ','); std::vector> split_config_str_list(const char *s, char delim); +// Parses delimited strings in |s| and returns Substrings in |s| +// delimited by |delim|. The any white spaces around substring are +// treated as a part of substring. +std::vector split_str(const StringRef &s, char delim); + // Returns given time |tp| in Common Log format (e.g., // 03/Jul/2014:00:19:38 +0900) // Expected type of |tp| is std::chrono::timepoint @@ -597,6 +647,7 @@ int64_t parse_uint_with_unit(const char *s); int64_t parse_uint(const char *s); int64_t parse_uint(const uint8_t *s, size_t len); int64_t parse_uint(const std::string &s); +int64_t parse_uint(const StringRef &s); // Parses NULL terminated string |s| as unsigned integer and returns // the parsed integer casted to double. If |s| ends with "s", the @@ -672,6 +723,11 @@ std::string random_alpha_digit(Generator &gen, size_t len) { return res; } +template +OutputIterator copy_lit(OutputIterator it, CharT(&s)[N]) { + return std::copy_n(s, N - 1, it); +} + } // namespace util } // namespace nghttp2 diff --git a/src/util_test.cc b/src/util_test.cc index 6f605c5b..8a8ee181 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -234,6 +234,24 @@ void test_util_ipv6_numeric_addr(void) { CU_ASSERT(!util::ipv6_numeric_addr("localhost")); } +void test_util_utos(void) { + uint8_t buf[32]; + + CU_ASSERT(("0" == StringRef{buf, util::utos(buf, 0)})); + CU_ASSERT(("123" == StringRef{buf, util::utos(buf, 123)})); + CU_ASSERT(("18446744073709551615" == + StringRef{buf, util::utos(buf, 18446744073709551615ULL)})); +} + +void test_util_make_string_ref_uint(void) { + BlockAllocator balloc(1024, 1024); + + CU_ASSERT("0" == util::make_string_ref_uint(balloc, 0)); + CU_ASSERT("123" == util::make_string_ref_uint(balloc, 123)); + CU_ASSERT("18446744073709551615" == + util::make_string_ref_uint(balloc, 18446744073709551615ULL)); +} + void test_util_utos_unit(void) { CU_ASSERT("0" == util::utos_unit(0)); CU_ASSERT("1023" == util::utos_unit(1023)); @@ -383,8 +401,8 @@ void test_util_ends_with(void) { } void test_util_parse_http_date(void) { - CU_ASSERT(1001939696 == - util::parse_http_date("Mon, 1 Oct 2001 12:34:56 GMT")); + CU_ASSERT(1001939696 == util::parse_http_date(StringRef::from_lit( + "Mon, 1 Oct 2001 12:34:56 GMT"))); } void test_util_localtime_date(void) { @@ -456,14 +474,19 @@ void test_util_parse_config_str_list(void) { } void test_util_make_http_hostport(void) { - CU_ASSERT("localhost" == util::make_http_hostport("localhost", 80)); - CU_ASSERT("[::1]" == util::make_http_hostport("::1", 443)); - CU_ASSERT("localhost:3000" == util::make_http_hostport("localhost", 3000)); + CU_ASSERT("localhost" == + util::make_http_hostport(StringRef::from_lit("localhost"), 80)); + CU_ASSERT("[::1]" == + util::make_http_hostport(StringRef::from_lit("::1"), 443)); + CU_ASSERT("localhost:3000" == + util::make_http_hostport(StringRef::from_lit("localhost"), 3000)); } void test_util_make_hostport(void) { - CU_ASSERT("localhost:80" == util::make_hostport("localhost", 80)); - CU_ASSERT("[::1]:443" == util::make_hostport("::1", 443)); + CU_ASSERT("localhost:80" == + util::make_hostport(StringRef::from_lit("localhost"), 80)); + CU_ASSERT("[::1]:443" == + util::make_hostport(StringRef::from_lit("::1"), 443)); } } // namespace shrpx diff --git a/src/util_test.h b/src/util_test.h index 1b79bbae..b44d5b17 100644 --- a/src/util_test.h +++ b/src/util_test.h @@ -44,6 +44,8 @@ void test_util_utox(void); void test_util_http_date(void); void test_util_select_h2(void); void test_util_ipv6_numeric_addr(void); +void test_util_utos(void); +void test_util_make_string_ref_uint(void); void test_util_utos_unit(void); void test_util_utos_funit(void); void test_util_parse_uint_with_unit(void); diff --git a/tests/main.c b/tests/main.c index 5d1c2533..6a985317 100644 --- a/tests/main.c +++ b/tests/main.c @@ -101,6 +101,8 @@ int main(int argc _U_, char *argv[] _U_) { test_nghttp2_session_recv_settings_header_table_size) || !CU_add_test(pSuite, "session_recv_too_large_frame_length", test_nghttp2_session_recv_too_large_frame_length) || + !CU_add_test(pSuite, "session_recv_extension", + test_nghttp2_session_recv_extension) || !CU_add_test(pSuite, "session_continue", test_nghttp2_session_continue) || !CU_add_test(pSuite, "session_add_frame", test_nghttp2_session_add_frame) || @@ -194,6 +196,7 @@ int main(int argc _U_, char *argv[] _U_) { test_nghttp2_submit_shutdown_notice) || !CU_add_test(pSuite, "submit_invalid_nv", test_nghttp2_submit_invalid_nv) || + !CU_add_test(pSuite, "submit_extension", test_nghttp2_submit_extension) || !CU_add_test(pSuite, "session_open_stream", test_nghttp2_session_open_stream) || !CU_add_test(pSuite, "session_open_stream_with_idle_stream_dep", diff --git a/tests/nghttp2_hd_test.c b/tests/nghttp2_hd_test.c index 5a4d7c5f..3d8ab83b 100644 --- a/tests/nghttp2_hd_test.c +++ b/tests/nghttp2_hd_test.c @@ -295,11 +295,6 @@ void test_nghttp2_hd_inflate_indname_inc(void) { assert_nv_equal(&nv, out.nva, 1, mem); CU_ASSERT(1 == inflater.ctx.hd_table.len); CU_ASSERT(62 == nghttp2_hd_inflate_get_num_table_entries(&inflater)); - assert_nv_equal(&nv, - &nghttp2_hd_table_get(&inflater.ctx, - NGHTTP2_STATIC_TABLE_LENGTH + - inflater.ctx.hd_table.len - 1)->nv, - 1, mem); assert_nv_equal(&nv, nghttp2_hd_inflate_get_table_entry( &inflater, NGHTTP2_STATIC_TABLE_LENGTH + inflater.ctx.hd_table.len), @@ -429,10 +424,9 @@ void test_nghttp2_hd_inflate_newname_inc(void) { CU_ASSERT(1 == out.nvlen); assert_nv_equal(&nv, out.nva, 1, mem); CU_ASSERT(1 == inflater.ctx.hd_table.len); - assert_nv_equal(&nv, - &nghttp2_hd_table_get(&inflater.ctx, - NGHTTP2_STATIC_TABLE_LENGTH + - inflater.ctx.hd_table.len - 1)->nv, + assert_nv_equal(&nv, nghttp2_hd_inflate_get_table_entry( + &inflater, NGHTTP2_STATIC_TABLE_LENGTH + + inflater.ctx.hd_table.len), 1, mem); nva_out_reset(&out, mem); @@ -1352,13 +1346,15 @@ void test_nghttp2_hd_decode_length(void) { void test_nghttp2_hd_huff_encode(void) { int rv; ssize_t len; - nghttp2_bufs bufs, outbufs; + nghttp2_buf outbuf; + nghttp2_bufs bufs; nghttp2_hd_huff_decode_context ctx; const uint8_t t1[] = {22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; + uint8_t b[256]; + nghttp2_buf_wrap_init(&outbuf, b, sizeof(b)); frame_pack_bufs_init(&bufs); - frame_pack_bufs_init(&outbufs); rv = nghttp2_hd_huff_encode(&bufs, t1, sizeof(t1)); @@ -1366,14 +1362,13 @@ void test_nghttp2_hd_huff_encode(void) { nghttp2_hd_huff_decode_context_init(&ctx); - len = nghttp2_hd_huff_decode(&ctx, &outbufs, bufs.cur->buf.pos, + len = nghttp2_hd_huff_decode(&ctx, &outbuf, bufs.cur->buf.pos, nghttp2_bufs_len(&bufs), 1); CU_ASSERT((ssize_t)nghttp2_bufs_len(&bufs) == len); - CU_ASSERT((ssize_t)sizeof(t1) == nghttp2_bufs_len(&outbufs)); + CU_ASSERT((ssize_t)sizeof(t1) == nghttp2_buf_len(&outbuf)); - CU_ASSERT(0 == memcmp(t1, outbufs.cur->buf.pos, sizeof(t1))); + CU_ASSERT(0 == memcmp(t1, outbuf.pos, sizeof(t1))); nghttp2_bufs_free(&bufs); - nghttp2_bufs_free(&outbufs); } diff --git a/tests/nghttp2_session_test.c b/tests/nghttp2_session_test.c index bd07b822..2e7dafee 100644 --- a/tests/nghttp2_session_test.c +++ b/tests/nghttp2_session_test.c @@ -36,10 +36,6 @@ #include "nghttp2_test_helper.h" #include "nghttp2_priority_spec.h" -#define OB_CTRL(ITEM) nghttp2_outbound_item_get_ctrl_frame(ITEM) -#define OB_CTRL_TYPE(ITEM) nghttp2_outbound_item_get_ctrl_frame_type(ITEM) -#define OB_DATA(ITEM) nghttp2_outbound_item_get_data_frame(ITEM) - typedef struct { uint8_t buf[65535]; size_t length; @@ -58,6 +54,7 @@ typedef struct { scripted_data_feed *df; int frame_recv_cb_called, invalid_frame_recv_cb_called; uint8_t recv_frame_type; + nghttp2_frame_hd recv_frame_hd; int frame_send_cb_called; uint8_t sent_frame_type; int frame_not_send_cb_called; @@ -77,6 +74,7 @@ typedef struct { size_t data_chunk_len; size_t padlen; int begin_frame_cb_called; + nghttp2_buf scratchbuf; } my_user_data; static const nghttp2_nv reqnv[] = { @@ -179,6 +177,8 @@ static int on_frame_recv_callback(nghttp2_session *session _U_, my_user_data *ud = (my_user_data *)user_data; ++ud->frame_recv_cb_called; ud->recv_frame_type = frame->hd.type; + ud->recv_frame_hd = frame->hd; + return 0; } @@ -420,6 +420,54 @@ static int on_stream_close_callback(nghttp2_session *session _U_, return 0; } +static ssize_t pack_extension_callback(nghttp2_session *session _U_, + uint8_t *buf, size_t len _U_, + const nghttp2_frame *frame, + void *user_data _U_) { + nghttp2_buf *p = frame->ext.payload; + + memcpy(buf, p->pos, nghttp2_buf_len(p)); + + return (ssize_t)nghttp2_buf_len(p); +} + +static int on_extension_chunk_recv_callback(nghttp2_session *session _U_, + const nghttp2_frame_hd *hd _U_, + const uint8_t *data, size_t len, + void *user_data) { + my_user_data *my_data = (my_user_data *)user_data; + nghttp2_buf *buf = &my_data->scratchbuf; + + buf->last = nghttp2_cpymem(buf->last, data, len); + + return 0; +} + +static int cancel_on_extension_chunk_recv_callback( + nghttp2_session *session _U_, const nghttp2_frame_hd *hd _U_, + const uint8_t *data _U_, size_t len _U_, void *user_data _U_) { + return NGHTTP2_ERR_CANCEL; +} + +static int unpack_extension_callback(nghttp2_session *session _U_, + void **payload, + const nghttp2_frame_hd *hd _U_, + void *user_data) { + my_user_data *my_data = (my_user_data *)user_data; + nghttp2_buf *buf = &my_data->scratchbuf; + + *payload = buf; + + return 0; +} + +static int cancel_unpack_extension_callback(nghttp2_session *session _U_, + void **payload _U_, + const nghttp2_frame_hd *hd _U_, + void *user_data _U_) { + return NGHTTP2_ERR_CANCEL; +} + static nghttp2_settings_entry *dup_iv(const nghttp2_settings_entry *iv, size_t niv) { return nghttp2_frame_iv_copy(iv, niv, nghttp2_mem_default()); @@ -1826,6 +1874,87 @@ void test_nghttp2_session_recv_too_large_frame_length(void) { nghttp2_session_del(session); } +void test_nghttp2_session_recv_extension(void) { + nghttp2_session *session; + nghttp2_session_callbacks callbacks; + my_user_data ud; + nghttp2_buf buf; + nghttp2_frame_hd hd; + nghttp2_mem *mem; + const char data[] = "Hello World!"; + ssize_t rv; + nghttp2_option *option; + + mem = nghttp2_mem_default(); + + memset(&callbacks, 0, sizeof(nghttp2_session_callbacks)); + + callbacks.on_extension_chunk_recv_callback = on_extension_chunk_recv_callback; + callbacks.unpack_extension_callback = unpack_extension_callback; + callbacks.on_frame_recv_callback = on_frame_recv_callback; + + nghttp2_option_new(&option); + nghttp2_option_set_user_recv_extension_type(option, 111); + + nghttp2_buf_init2(&ud.scratchbuf, 4096, mem); + nghttp2_buf_init2(&buf, 4096, mem); + + nghttp2_frame_hd_init(&hd, sizeof(data), 111, 0xab, 1000000007); + nghttp2_frame_pack_frame_hd(buf.last, &hd); + buf.last += NGHTTP2_FRAME_HDLEN; + buf.last = nghttp2_cpymem(buf.last, data, sizeof(data)); + + nghttp2_session_client_new2(&session, &callbacks, &ud, option); + + nghttp2_frame_hd_init(&ud.recv_frame_hd, 0, 0, 0, 0); + rv = nghttp2_session_mem_recv(session, buf.pos, nghttp2_buf_len(&buf)); + + CU_ASSERT(NGHTTP2_FRAME_HDLEN + hd.length == (size_t)rv); + CU_ASSERT(111 == ud.recv_frame_hd.type); + CU_ASSERT(0xab == ud.recv_frame_hd.flags); + CU_ASSERT(1000000007 == ud.recv_frame_hd.stream_id); + CU_ASSERT(0 == memcmp(data, ud.scratchbuf.pos, sizeof(data))); + + nghttp2_session_del(session); + + /* cancel in on_extension_chunk_recv_callback */ + nghttp2_buf_reset(&ud.scratchbuf); + + callbacks.on_extension_chunk_recv_callback = + cancel_on_extension_chunk_recv_callback; + + nghttp2_session_server_new2(&session, &callbacks, &ud, option); + + ud.frame_recv_cb_called = 0; + rv = nghttp2_session_mem_recv(session, buf.pos, nghttp2_buf_len(&buf)); + + CU_ASSERT(NGHTTP2_FRAME_HDLEN + hd.length == (size_t)rv); + CU_ASSERT(0 == ud.frame_recv_cb_called); + + nghttp2_session_del(session); + + /* cancel in unpack_extension_callback */ + nghttp2_buf_reset(&ud.scratchbuf); + + callbacks.on_extension_chunk_recv_callback = on_extension_chunk_recv_callback; + callbacks.unpack_extension_callback = cancel_unpack_extension_callback; + + nghttp2_session_server_new2(&session, &callbacks, &ud, option); + + ud.frame_recv_cb_called = 0; + rv = nghttp2_session_mem_recv(session, buf.pos, nghttp2_buf_len(&buf)); + + CU_ASSERT(NGHTTP2_FRAME_HDLEN + hd.length == (size_t)rv); + CU_ASSERT(0 == ud.frame_recv_cb_called); + + nghttp2_session_del(session); + + nghttp2_buf_free(&buf, mem); + nghttp2_buf_free(&ud.scratchbuf, mem); + + nghttp2_option_del(option); +} + void test_nghttp2_session_continue(void) { nghttp2_session *session; nghttp2_session_callbacks callbacks; @@ -2966,6 +3095,7 @@ void test_nghttp2_session_on_ping_received(void) { nghttp2_frame frame; nghttp2_outbound_item *top; const uint8_t opaque_data[] = "01234567"; + nghttp2_option *option; user_data.frame_recv_cb_called = 0; user_data.invalid_frame_recv_cb_called = 0; @@ -2996,6 +3126,23 @@ void test_nghttp2_session_on_ping_received(void) { nghttp2_frame_ping_free(&frame.ping); nghttp2_session_del(session); + + /* Use nghttp2_option_set_no_auto_ping_ack() */ + nghttp2_option_new(&option); + nghttp2_option_set_no_auto_ping_ack(option, 1); + + nghttp2_session_server_new2(&session, &callbacks, &user_data, option); + nghttp2_frame_ping_init(&frame.ping, NGHTTP2_FLAG_NONE, NULL); + + user_data.frame_recv_cb_called = 0; + + CU_ASSERT(0 == nghttp2_session_on_ping_received(session, &frame)); + CU_ASSERT(1 == user_data.frame_recv_cb_called); + CU_ASSERT(NULL == nghttp2_outbound_queue_top(&session->ob_urgent)); + + nghttp2_frame_ping_free(&frame.ping); + nghttp2_session_del(session); + nghttp2_option_del(option); } void test_nghttp2_session_on_goaway_received(void) { @@ -5009,6 +5156,67 @@ void test_nghttp2_submit_invalid_nv(void) { nghttp2_session_del(session); } +void test_nghttp2_submit_extension(void) { + nghttp2_session *session; + nghttp2_session_callbacks callbacks; + my_user_data ud; + accumulator acc; + nghttp2_mem *mem; + const char data[] = "Hello World!"; + size_t len; + int32_t stream_id; + int rv; + + mem = nghttp2_mem_default(); + + memset(&callbacks, 0, sizeof(nghttp2_session_callbacks)); + + callbacks.pack_extension_callback = pack_extension_callback; + callbacks.send_callback = accumulator_send_callback; + + nghttp2_buf_init2(&ud.scratchbuf, 4096, mem); + + nghttp2_session_client_new(&session, &callbacks, &ud); + + ud.scratchbuf.last = nghttp2_cpymem(ud.scratchbuf.last, data, sizeof(data)); + ud.acc = &acc; + + rv = nghttp2_submit_extension(session, 211, 0x01, 3, &ud.scratchbuf); + + CU_ASSERT(0 == rv); + + acc.length = 0; + + rv = nghttp2_session_send(session); + + CU_ASSERT(0 == rv); + CU_ASSERT(NGHTTP2_FRAME_HDLEN + sizeof(data) == acc.length); + + len = nghttp2_get_uint32(acc.buf) >> 8; + + CU_ASSERT(sizeof(data) == len); + CU_ASSERT(211 == acc.buf[3]); + CU_ASSERT(0x01 == acc.buf[4]); + + stream_id = (int32_t)nghttp2_get_uint32(acc.buf + 5); + + CU_ASSERT(3 == stream_id); + CU_ASSERT(0 == memcmp(data, &acc.buf[NGHTTP2_FRAME_HDLEN], sizeof(data))); + + nghttp2_session_del(session); + + /* submitting standard HTTP/2 frame is error */ + nghttp2_session_server_new(&session, &callbacks, &ud); + + rv = nghttp2_submit_extension(session, NGHTTP2_GOAWAY, NGHTTP2_FLAG_NONE, 0, + NULL); + + CU_ASSERT(NGHTTP2_ERR_INVALID_ARGUMENT == rv); + + nghttp2_session_del(session); + nghttp2_buf_free(&ud.scratchbuf, mem); +} + void test_nghttp2_session_open_stream(void) { nghttp2_session *session; nghttp2_session_callbacks callbacks; @@ -5963,6 +6171,15 @@ void test_nghttp2_session_set_option(void) { CU_ASSERT(99 == session->max_incoming_reserved_streams); nghttp2_session_del(session); + /* Test for nghttp2_option_set_no_auto_ping_ack */ + nghttp2_option_set_no_auto_ping_ack(option, 1); + + nghttp2_session_client_new2(&session, &callbacks, NULL, option); + + CU_ASSERT(session->opt_flags & NGHTTP2_OPTMASK_NO_AUTO_PING_ACK); + + nghttp2_session_del(session); + nghttp2_option_del(option); } diff --git a/tests/nghttp2_session_test.h b/tests/nghttp2_session_test.h index 3a54ba00..3d1d8f28 100644 --- a/tests/nghttp2_session_test.h +++ b/tests/nghttp2_session_test.h @@ -40,6 +40,7 @@ void test_nghttp2_session_recv_unknown_frame(void); void test_nghttp2_session_recv_unexpected_continuation(void); void test_nghttp2_session_recv_settings_header_table_size(void); void test_nghttp2_session_recv_too_large_frame_length(void); +void test_nghttp2_session_recv_extension(void); void test_nghttp2_session_continue(void); void test_nghttp2_session_add_frame(void); void test_nghttp2_session_on_request_headers_received(void); @@ -89,6 +90,7 @@ void test_nghttp2_submit_window_update(void); void test_nghttp2_submit_window_update_local_window_size(void); void test_nghttp2_submit_shutdown_notice(void); void test_nghttp2_submit_invalid_nv(void); +void test_nghttp2_submit_extension(void); void test_nghttp2_session_open_stream(void); void test_nghttp2_session_open_stream_with_idle_stream_dep(void); void test_nghttp2_session_get_next_ob_item(void); diff --git a/third-party/Makefile.am b/third-party/Makefile.am index 3b951c8d..ebc49458 100644 --- a/third-party/Makefile.am +++ b/third-party/Makefile.am @@ -57,7 +57,9 @@ mruby: all-local: mruby clean-local: - -rm -rf "${abs_builddir}/mruby/build" + MRUBY_CONFIG="${srcdir}/build_config.rb" \ + BUILD_DIR="${abs_builddir}/mruby/build" \ + "${srcdir}/mruby/minirake" -f "${srcdir}/mruby/Rakefile" clean endif # HAVE_MRUBY