From 1508c50a45b5321c7ba902c18369be8376ae3308 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 2 Sep 2015 00:19:32 +0900 Subject: [PATCH 01/55] nghttpx: Add basic infrastructure for mruby support --- gennghttpxfun.py | 1 + src/Makefile.am | 6 +- src/shrpx.cc | 17 ++- src/shrpx_config.cc | 10 ++ src/shrpx_config.h | 3 + src/shrpx_connection_handler.cc | 20 +++- src/shrpx_connection_handler.h | 4 +- src/shrpx_downstream.cc | 15 ++- src/shrpx_downstream.h | 5 + src/shrpx_http2_upstream.cc | 8 ++ src/shrpx_https_upstream.cc | 7 ++ src/shrpx_mruby.cc | 163 +++++++++++++++++++++++++++ src/shrpx_mruby.h | 63 +++++++++++ src/shrpx_mruby_module.cc | 193 ++++++++++++++++++++++++++++++++ src/shrpx_mruby_module.h | 44 ++++++++ src/shrpx_worker.cc | 14 +++ src/shrpx_worker.h | 11 ++ 17 files changed, 573 insertions(+), 11 deletions(-) create mode 100644 src/shrpx_mruby.cc create mode 100644 src/shrpx_mruby.h create mode 100644 src/shrpx_mruby_module.cc create mode 100644 src/shrpx_mruby_module.h diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 4e99155c..3b1a3ae0 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -98,6 +98,7 @@ OPTIONS = [ "tls-ticket-key-memcached-interval", "tls-ticket-key-memcached-max-retry", "tls-ticket-key-memcached-max-fail", + "on-request-mruby-file", "conf", ] diff --git a/src/Makefile.am b/src/Makefile.am index 304b6c09..bc3a6ce7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -124,6 +124,8 @@ NGHTTPX_SRCS = \ shrpx_memcached_connection.cc shrpx_memcached_connection.h \ shrpx_memcached_request.h \ shrpx_memcached_result.h \ + shrpx_mruby.cc shrpx_mruby.h \ + shrpx_mruby_module.cc shrpx_mruby_module.h \ buffer.h memchunk.h template.h if HAVE_SPDYLAY @@ -134,7 +136,7 @@ noinst_LIBRARIES = libnghttpx.a libnghttpx_a_SOURCES = ${NGHTTPX_SRCS} nghttpx_SOURCES = shrpx.cc shrpx.h -nghttpx_LDADD = libnghttpx.a ${LDADD} +nghttpx_LDADD = libnghttpx.a ${LDADD} -lmruby if HAVE_CUNIT check_PROGRAMS += nghttpx-unittest @@ -150,7 +152,7 @@ nghttpx_unittest_SOURCES = shrpx-unittest.cc \ memchunk_test.cc memchunk_test.h nghttpx_unittest_CPPFLAGS = ${AM_CPPFLAGS}\ -DNGHTTP2_TESTS_DIR=\"$(top_srcdir)/tests\" -nghttpx_unittest_LDADD = libnghttpx.a ${LDADD} @CUNIT_LIBS@ @TESTLDADD@ +nghttpx_unittest_LDADD = libnghttpx.a ${LDADD} -lmruby @CUNIT_LIBS@ @TESTLDADD@ TESTS += nghttpx-unittest endif # HAVE_CUNIT diff --git a/src/shrpx.cc b/src/shrpx.cc index 17c01f80..51c2cf9d 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -926,9 +926,13 @@ int event_loop() { #endif // !NOTHREADS if (get_config()->num_worker == 1) { - conn_handler->create_single_worker(); + rv = conn_handler->create_single_worker(); } else { - conn_handler->create_worker_thread(get_config()->num_worker); + rv = conn_handler->create_worker_thread(get_config()->num_worker); + } + + if (rv != 0) { + return -1; } #ifndef NOTHREADS @@ -1899,6 +1903,7 @@ int main(int argc, char **argv) { 89}, {SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_MAX_FAIL, required_argument, &flag, 90}, + {SHRPX_OPT_ON_REQUEST_MRUBY_FILE, required_argument, &flag, 91}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2296,6 +2301,10 @@ int main(int argc, char **argv) { cmdcfgs.emplace_back(SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_MAX_FAIL, optarg); break; + case 91: + // --on-request-mruby-file + cmdcfgs.emplace_back(SHRPX_OPT_ON_REQUEST_MRUBY_FILE, optarg); + break; default: break; } @@ -2613,7 +2622,9 @@ int main(int argc, char **argv) { act.sa_handler = SIG_IGN; sigaction(SIGPIPE, &act, nullptr); - event_loop(); + if (event_loop() != 0) { + return -1; + } LOG(NOTICE) << "Shutdown momentarily"; diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 45db8daf..1d4b480c 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -690,6 +690,7 @@ enum { SHRPX_OPTID_NO_VIA, SHRPX_OPTID_NPN_LIST, SHRPX_OPTID_OCSP_UPDATE_INTERVAL, + SHRPX_OPTID_ON_REQUEST_MRUBY_FILE, SHRPX_OPTID_PADDING, SHRPX_OPTID_PID_FILE, SHRPX_OPTID_PRIVATE_KEY_FILE, @@ -1089,6 +1090,11 @@ int option_lookup_token(const char *name, size_t namelen) { return SHRPX_OPTID_BACKEND_TLS_SNI_FIELD; } break; + case 'e': + if (util::strieq_l("on-request-mruby-fil", name, 20)) { + return SHRPX_OPTID_ON_REQUEST_MRUBY_FILE; + } + break; case 'r': if (util::strieq_l("tls-ticket-key-ciphe", name, 20)) { return SHRPX_OPTID_TLS_TICKET_KEY_CIPHER; @@ -1938,6 +1944,10 @@ int parse_config(const char *opt, const char *optarg, case SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_MAX_FAIL: return parse_uint(&mod_config()->tls_ticket_key_memcached_max_fail, opt, optarg); + case SHRPX_OPTID_ON_REQUEST_MRUBY_FILE: + mod_config()->on_request_mruby_file = strcopy(optarg); + + return 0; case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index e75a859c..d7b59c84 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -183,6 +183,7 @@ constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_MAX_RETRY[] = "tls-ticket-key-memcached-max-retry"; constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_MAX_FAIL[] = "tls-ticket-key-memcached-max-fail"; +constexpr char SHRPX_OPT_ON_REQUEST_MRUBY_FILE[] = "on-request-mruby-file"; union sockaddr_union { sockaddr_storage storage; @@ -314,6 +315,8 @@ struct Config { std::unique_ptr user; std::unique_ptr session_cache_memcached_host; std::unique_ptr tls_ticket_key_memcached_host; + std::unique_ptr on_request_mruby_file; + std::unique_ptr on_response_mruby_file; FILE *http2_upstream_dump_request_header; FILE *http2_upstream_dump_response_header; nghttp2_session_callbacks *http2_upstream_callbacks; diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 8b493a55..0dc6dcb3 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -149,7 +149,7 @@ void ConnectionHandler::worker_reopen_log_files() { } } -void ConnectionHandler::create_single_worker() { +int ConnectionHandler::create_single_worker() { auto cert_tree = ssl::create_cert_lookup_tree(); auto sv_ssl_ctx = ssl::setup_server_ssl_context(all_ssl_ctx_, cert_tree); auto cl_ssl_ctx = ssl::setup_client_ssl_context(); @@ -160,9 +160,14 @@ void ConnectionHandler::create_single_worker() { single_worker_ = make_unique(loop_, sv_ssl_ctx, cl_ssl_ctx, cert_tree, ticket_keys_); + if (single_worker_->create_mruby_context() != 0) { + return -1; + } + + return 0; } -void ConnectionHandler::create_worker_thread(size_t num) { +int ConnectionHandler::create_worker_thread(size_t num) { #ifndef NOTHREADS assert(workers_.size() == 0); @@ -179,14 +184,23 @@ void ConnectionHandler::create_worker_thread(size_t num) { auto worker = make_unique(loop, sv_ssl_ctx, cl_ssl_ctx, cert_tree, ticket_keys_); - worker->run_async(); + if (worker->create_mruby_context() != 0) { + return -1; + } + workers_.push_back(std::move(worker)); if (LOG_ENABLED(INFO)) { LLOG(INFO, this) << "Created thread #" << workers_.size() - 1; } } + + for (auto &worker : workers_) { + worker->run_async(); + } #endif // NOTHREADS + + return 0; } void ConnectionHandler::join_worker() { diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h index d46f059c..dc33eb61 100644 --- a/src/shrpx_connection_handler.h +++ b/src/shrpx_connection_handler.h @@ -73,10 +73,10 @@ public: ~ConnectionHandler(); int handle_connection(int fd, sockaddr *addr, int addrlen); // Creates Worker object for single threaded configuration. - void create_single_worker(); + int create_single_worker(); // Creates |num| Worker objects for multi threaded configuration. // The |num| must be strictly more than 1. - void create_worker_thread(size_t num); + int create_worker_thread(size_t num); void set_ticket_keys_to_worker(const std::shared_ptr &ticket_keys); void worker_reopen_log_files(); diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index a37c7f2f..2f55e5d6 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -127,7 +127,7 @@ Downstream::Downstream(Upstream *upstream, MemchunkPool *mcpool, request_http2_expect_body_(false), chunked_response_(false), response_connection_close_(false), response_header_key_prev_(false), response_trailer_key_prev_(false), expect_final_response_(false), - request_pending_(false) { + request_pending_(false), request_headers_dirty_(false) { ev_timer_init(&upstream_rtimer_, &upstream_rtimeoutcb, 0., get_config()->stream_read_timeout); @@ -240,6 +240,8 @@ const Headers &Downstream::get_request_headers() const { return request_headers_; } +Headers &Downstream::get_request_headers() { return request_headers_; } + void Downstream::assemble_request_cookie() { std::string &cookie = assembled_request_cookie_; cookie = ""; @@ -336,6 +338,9 @@ void set_last_header_value(bool &key_prev, size_t &sum, Headers &headers, namespace { int index_headers(http2::HeaderIndex &hdidx, Headers &headers, int64_t &content_length) { + http2::init_hdidx(hdidx); + content_length = -1; + for (size_t i = 0; i < headers.size(); ++i) { auto &kv = headers[i]; util::inp_strlower(kv.name); @@ -1213,4 +1218,12 @@ bool Downstream::can_detach_downstream_connection() const { !response_connection_close_; } +void Downstream::set_request_headers_dirty(bool f) { + request_headers_dirty_ = f; +} + +bool Downstream::get_request_headers_dirty() const { + return request_headers_dirty_; +} + } // namespace shrpx diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index c64aba4f..a12359fa 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -96,6 +96,7 @@ public: const std::string &get_http2_settings() const; // downstream request API const Headers &get_request_headers() const; + Headers &get_request_headers(); // Crumbles (split cookie by ";") in request_headers_ and returns // them. Headers::no_index is inherited. Headers crumble_request_cookie(); @@ -126,6 +127,8 @@ public: void append_last_request_header_value(const char *data, size_t len); // Empties request headers. void clear_request_headers(); + void set_request_headers_dirty(bool f); + bool get_request_headers_dirty() const; size_t get_request_headers_sum() const; @@ -454,6 +457,8 @@ private: // has not been established or should be checked before use; // currently used only with HTTP/2 connection. bool request_pending_; + // true if we need to execute index_request_headers() + bool request_headers_dirty_; }; } // namespace shrpx diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 9659596c..59dc5299 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -37,6 +37,7 @@ #include "shrpx_http.h" #include "shrpx_worker.h" #include "shrpx_http2_session.h" +#include "shrpx_mruby.h" #include "http2.h" #include "util.h" #include "base64.h" @@ -306,6 +307,13 @@ int Http2Upstream::on_request_headers(Downstream *downstream, downstream->set_request_http2_expect_body(true); } + auto upstream = downstream->get_upstream(); + auto handler = upstream->get_client_handler(); + auto worker = handler->get_worker(); + auto mruby_ctx = worker->get_mruby_context(); + + mruby_ctx->run_on_request_proc(downstream); + downstream->inspect_http2_request(); downstream->set_request_state(Downstream::HEADER_COMPLETE); diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 40a29593..4da4fed0 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -37,6 +37,7 @@ #include "shrpx_log_config.h" #include "shrpx_worker.h" #include "shrpx_http2_session.h" +#include "shrpx_mruby.h" #include "http2.h" #include "util.h" #include "template.h" @@ -272,6 +273,12 @@ int htp_hdrs_completecb(http_parser *htp) { return -1; } + auto handler = upstream->get_client_handler(); + auto worker = handler->get_worker(); + auto mruby_ctx = worker->get_mruby_context(); + + mruby_ctx->run_on_request_proc(downstream); + downstream->inspect_http1_request(); if (downstream->get_request_method() != HTTP_CONNECT) { diff --git a/src/shrpx_mruby.cc b/src/shrpx_mruby.cc new file mode 100644 index 00000000..aa3135dd --- /dev/null +++ b/src/shrpx_mruby.cc @@ -0,0 +1,163 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2015 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "shrpx_mruby.h" + +#include +#include + +#include "shrpx_downstream.h" +#include "shrpx_config.h" +#include "shrpx_mruby_module.h" +#include "template.h" + +namespace shrpx { + +namespace mruby { + +MRubyContext::MRubyContext(mrb_state *mrb, RProc *on_request_proc, + RProc *on_response_proc) + : mrb_(mrb), on_request_proc_(on_request_proc), + on_response_proc_(on_response_proc) {} + +MRubyContext::~MRubyContext() { mrb_close(mrb_); } + +int MRubyContext::run_on_request_proc(Downstream *downstream) { + if (!on_request_proc_) { + return 0; + } + + mrb_->ud = downstream; + + int rv = 0; + auto ai = mrb_gc_arena_save(mrb_); + + auto res = mrb_run(mrb_, on_request_proc_, mrb_top_self(mrb_)); + (void)res; + + if (mrb_->exc) { + rv = -1; + auto error = + mrb_str_ptr(mrb_funcall(mrb_, mrb_obj_value(mrb_->exc), "inspect", 0)); + + LOG(ERROR) << "Exception caught while executing mruby code: " + << error->as.heap.ptr; + mrb_->exc = 0; + } + + mrb_->ud = nullptr; + + mrb_gc_arena_restore(mrb_, ai); + + if (downstream->get_request_headers_dirty()) { + downstream->set_request_headers_dirty(false); + downstream->index_request_headers(); + } + + return rv; +} + +int run_on_response_proc(Downstream *downstream) { + // TODO not implemented yet + return 0; +} + +// Based on +// https://github.com/h2o/h2o/blob/master/lib/handler/mruby.c. It is +// very hard to write these kind of code because mruby has almost no +// documentation aobut compiling or generating code, at least at the +// time of this writing. +RProc *compile(mrb_state *mrb, const char *filename) { + if (filename == nullptr) { + return nullptr; + } + + auto infile = fopen(filename, "rb"); + if (infile == nullptr) { + return nullptr; + } + auto infile_d = defer(fclose, infile); + + auto mrbc = mrbc_context_new(mrb); + if (mrbc == nullptr) { + LOG(ERROR) << "mrb_context_new failed"; + return nullptr; + } + auto mrbc_d = defer(mrbc_context_free, mrb, mrbc); + + auto parser = mrb_parse_file(mrb, infile, nullptr); + if (parser == nullptr) { + LOG(ERROR) << "mrb_parse_nstring failed"; + return nullptr; + } + auto parser_d = defer(mrb_parser_free, parser); + + if (parser->nerr != 0) { + LOG(ERROR) << "mruby parser detected parse error"; + return nullptr; + } + + auto proc = mrb_generate_code(mrb, parser); + if (proc == nullptr) { + LOG(ERROR) << "mrb_generate_code failed"; + return nullptr; + } + + return proc; +} + +std::unique_ptr create_mruby_context() { + auto mrb = mrb_open(); + if (mrb == nullptr) { + LOG(ERROR) << "mrb_open failed"; + return nullptr; + } + + init_module(mrb); + + auto req_file = get_config()->on_request_mruby_file.get(); + auto res_file = get_config()->on_response_mruby_file.get(); + + auto req_proc = compile(mrb, req_file); + + if (req_file && !req_proc) { + LOG(ERROR) << "Could not compile mruby code " << req_file; + mrb_close(mrb); + return nullptr; + } + + auto res_proc = compile(mrb, res_file); + + if (res_file && !res_proc) { + LOG(ERROR) << "Could not compile mruby code " << res_file; + mrb_close(mrb); + return nullptr; + } + + return make_unique(mrb, req_proc, res_proc); +} + +} // namespace mruby + +} // namespace shrpx diff --git a/src/shrpx_mruby.h b/src/shrpx_mruby.h new file mode 100644 index 00000000..6e0bc57e --- /dev/null +++ b/src/shrpx_mruby.h @@ -0,0 +1,63 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2015 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef SHRPX_MRUBY_H +#define SHRPX_MRUBY_H + +#include "shrpx.h" + +#include + +#include +#include + +using namespace nghttp2; + +namespace shrpx { + +namespace mruby { + +class MRubyContext { +public: + MRubyContext(mrb_state *mrb, RProc *on_request_proc, RProc *on_response_proc); + ~MRubyContext(); + + int run_on_request_proc(Downstream *downstream); + int run_on_response_proc(Downstream *downstream); + +private: + mrb_state *mrb_; + RProc *on_request_proc_; + RProc *on_response_proc_; +}; + +RProc *compile(mrb_state *mrb, const char *filename); + +std::unique_ptr create_mruby_context(); + +} // namespace mruby + +} // namespace shrpx + +#endif // SHRPX_MRUBY_H diff --git a/src/shrpx_mruby_module.cc b/src/shrpx_mruby_module.cc new file mode 100644 index 00000000..5b45a9d3 --- /dev/null +++ b/src/shrpx_mruby_module.cc @@ -0,0 +1,193 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2015 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "shrpx_mruby_module.h" + +#include +#include + +#include "shrpx_downstream.h" +#include "util.h" + +namespace shrpx { + +namespace mruby { + +namespace { +mrb_value request_init(mrb_state *mrb, mrb_value self) { return self; } +} // namespace + +namespace { +mrb_value request_get_path(mrb_state *mrb, mrb_value self) { + auto downstream = static_cast(mrb->ud); + auto &path = downstream->get_request_path(); + + return mrb_str_new_static(mrb, path.c_str(), path.size()); +} +} // namespace + +namespace { +mrb_value request_set_path(mrb_state *mrb, mrb_value self) { + auto downstream = static_cast(mrb->ud); + + const char *path; + mrb_int pathlen; + mrb_get_args(mrb, "s", &path, &pathlen); + if (pathlen == 0) { + mrb_raise(mrb, E_RUNTIME_ERROR, "path must not be empty string"); + } + + downstream->set_request_path(std::string(path, pathlen)); + + return self; +} +} // namespace + +namespace { +mrb_value request_get_headers(mrb_state *mrb, mrb_value self) { + auto headers = mrb_iv_get(mrb, self, mrb_intern_lit(mrb, "RequestHeaders")); + if (mrb_nil_p(headers)) { + auto module = mrb_module_get(mrb, "Nghttpx"); + auto headers_class = mrb_class_get_under(mrb, module, "RequestHeaders"); + headers = mrb_obj_new(mrb, headers_class, 0, nullptr); + mrb_iv_set(mrb, self, mrb_intern_lit(mrb, "RequestHeaders"), headers); + } + return headers; +} +} // namespace + +namespace { +mrb_value headers_init(mrb_state *mrb, mrb_value self) { return self; } +} // namespace + +namespace { +mrb_value request_headers_get(mrb_state *mrb, mrb_value self) { + auto downstream = static_cast(mrb->ud); + + mrb_value key; + mrb_get_args(mrb, "o", &key); + + key = mrb_funcall(mrb, key, "downcase", 0); + + if (RSTRING_LEN(key) == 0) { + return key; + } + + auto hd = downstream->get_request_header( + std::string(RSTRING_PTR(key), RSTRING_LEN(key))); + + if (hd == nullptr) { + return mrb_nil_value(); + } + + return mrb_str_new_static(mrb, hd->value.c_str(), hd->value.size()); +} +} // namespace + +namespace { +mrb_value request_headers_set(mrb_state *mrb, mrb_value self, bool repl) { + auto downstream = static_cast(mrb->ud); + + mrb_value key, value; + mrb_get_args(mrb, "oo", &key, &value); + + key = mrb_funcall(mrb, key, "downcase", 0); + + if (RSTRING_LEN(key) == 0) { + mrb_raise(mrb, E_RUNTIME_ERROR, "empty key is not allowed"); + } + + if (repl) { + for (auto &hd : downstream->get_request_headers()) { + if (util::streq(std::begin(hd.name), hd.name.size(), RSTRING_PTR(key), + RSTRING_LEN(key))) { + hd.name = ""; + } + } + } + + downstream->add_request_header( + std::string(RSTRING_PTR(key), RSTRING_LEN(key)), + std::string(RSTRING_PTR(value), RSTRING_LEN(value))); + + downstream->set_request_headers_dirty(true); + + return key; +} +} // namespace + +namespace { +mrb_value request_headers_set(mrb_state *mrb, mrb_value self) { + return request_headers_set(mrb, self, true); +} +} // namespace + +namespace { +mrb_value request_headers_add(mrb_state *mrb, mrb_value self) { + return request_headers_set(mrb, self, false); +} +} // namespace + +namespace { +void init_headers_class(mrb_state *mrb, RClass *module, const char *name, + mrb_func_t get, mrb_func_t set, mrb_func_t add) { + auto headers_class = + mrb_define_class_under(mrb, module, name, mrb->object_class); + + mrb_define_method(mrb, headers_class, "initialize", headers_init, + MRB_ARGS_NONE()); + mrb_define_method(mrb, headers_class, "get", get, MRB_ARGS_REQ(1)); + mrb_define_method(mrb, headers_class, "set", set, MRB_ARGS_REQ(2)); + mrb_define_method(mrb, headers_class, "add", set, MRB_ARGS_REQ(2)); +} +} // namespace + +namespace { +void init_request_class(mrb_state *mrb, RClass *module) { + auto request_class = + mrb_define_class_under(mrb, module, "Request", mrb->object_class); + + mrb_define_method(mrb, request_class, "initialize", request_init, + MRB_ARGS_NONE()); + mrb_define_method(mrb, request_class, "path", request_get_path, + MRB_ARGS_NONE()); + mrb_define_method(mrb, request_class, "path=", request_set_path, + MRB_ARGS_REQ(1)); + mrb_define_method(mrb, request_class, "headers", request_get_headers, + MRB_ARGS_NONE()); + + init_headers_class(mrb, module, "RequestHeaders", request_headers_get, + request_headers_set, request_headers_add); +} +} // namespace + +void init_module(mrb_state *mrb) { + auto module = mrb_define_module(mrb, "Nghttpx"); + + init_request_class(mrb, module); +} + +} // namespace mruby + +} // namespace shrpx diff --git a/src/shrpx_mruby_module.h b/src/shrpx_mruby_module.h new file mode 100644 index 00000000..bec51934 --- /dev/null +++ b/src/shrpx_mruby_module.h @@ -0,0 +1,44 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2015 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef SHRPX_MRUBY_MODULE_H +#define SHRPX_MRUBY_MODULE_H + +#include "shrpx.h" + +#include + +using namespace nghttp2; + +namespace shrpx { + +namespace mruby { + +void init_module(mrb_state *mrb); + +} // namespace mruby + +} // namespace shrpx + +#endif // SHRPX_MRUBY_MODULE_H diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 2241f979..006e7e3a 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -37,6 +37,7 @@ #include "shrpx_log_config.h" #include "shrpx_connect_blocker.h" #include "shrpx_memcached_dispatcher.h" +#include "shrpx_mruby.h" #include "util.h" #include "template.h" @@ -265,4 +266,17 @@ MemcachedDispatcher *Worker::get_session_cache_memcached_dispatcher() { return session_cache_memcached_dispatcher_.get(); } +int Worker::create_mruby_context() { + mruby_ctx_ = mruby::create_mruby_context(); + if (!mruby_ctx_) { + return -1; + } + + return 0; +} + +mruby::MRubyContext *Worker::get_mruby_context() const { + return mruby_ctx_.get(); +} + } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 3613b0d5..38868aab 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -51,6 +51,12 @@ class Http2Session; class ConnectBlocker; class MemcachedDispatcher; +namespace mruby { + +class MRubyContext; + +} // namespace mruby + namespace ssl { class CertLookupTree; } // namespace ssl @@ -124,6 +130,10 @@ public: MemcachedDispatcher *get_session_cache_memcached_dispatcher(); + int create_mruby_context(); + + mruby::MRubyContext *get_mruby_context() const; + private: #ifndef NOTHREADS std::future fut_; @@ -137,6 +147,7 @@ private: WorkerStat worker_stat_; std::vector dgrps_; std::unique_ptr session_cache_memcached_dispatcher_; + std::unique_ptr mruby_ctx_; struct ev_loop *loop_; // Following fields are shared across threads if From 9d786919361f9d134217061512332a9db6262b8a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 2 Sep 2015 21:14:20 +0900 Subject: [PATCH 02/55] nghttpx: Fix copy-and-paste bug --- src/shrpx_mruby_module.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shrpx_mruby_module.cc b/src/shrpx_mruby_module.cc index 5b45a9d3..9abf5504 100644 --- a/src/shrpx_mruby_module.cc +++ b/src/shrpx_mruby_module.cc @@ -159,7 +159,7 @@ void init_headers_class(mrb_state *mrb, RClass *module, const char *name, MRB_ARGS_NONE()); mrb_define_method(mrb, headers_class, "get", get, MRB_ARGS_REQ(1)); mrb_define_method(mrb, headers_class, "set", set, MRB_ARGS_REQ(2)); - mrb_define_method(mrb, headers_class, "add", set, MRB_ARGS_REQ(2)); + mrb_define_method(mrb, headers_class, "add", add, MRB_ARGS_REQ(2)); } } // namespace From 2170d958d518fc8c714b838700cb334faedf0972 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 2 Sep 2015 21:14:49 +0900 Subject: [PATCH 03/55] nghttpx: Return copied string for safety --- src/shrpx_mruby_module.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shrpx_mruby_module.cc b/src/shrpx_mruby_module.cc index 9abf5504..fabb1c5d 100644 --- a/src/shrpx_mruby_module.cc +++ b/src/shrpx_mruby_module.cc @@ -43,7 +43,7 @@ mrb_value request_get_path(mrb_state *mrb, mrb_value self) { auto downstream = static_cast(mrb->ud); auto &path = downstream->get_request_path(); - return mrb_str_new_static(mrb, path.c_str(), path.size()); + return mrb_str_new(mrb, path.c_str(), path.size()); } } // namespace @@ -101,7 +101,7 @@ mrb_value request_headers_get(mrb_state *mrb, mrb_value self) { return mrb_nil_value(); } - return mrb_str_new_static(mrb, hd->value.c_str(), hd->value.size()); + return mrb_str_new(mrb, hd->value.c_str(), hd->value.size()); } } // namespace From 45cdb10c4640ec3d88fe96e0e5a8195b041f4694 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 2 Sep 2015 22:25:04 +0900 Subject: [PATCH 04/55] nghttpx: Use ruby block to run request hook --- src/shrpx_downstream.cc | 10 +-- src/shrpx_downstream.h | 4 - src/shrpx_mruby.cc | 33 ++++---- src/shrpx_mruby.h | 5 ++ src/shrpx_mruby_module.cc | 158 ++++++++++++++++++-------------------- 5 files changed, 101 insertions(+), 109 deletions(-) diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 2f55e5d6..d9325699 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -127,7 +127,7 @@ Downstream::Downstream(Upstream *upstream, MemchunkPool *mcpool, request_http2_expect_body_(false), chunked_response_(false), response_connection_close_(false), response_header_key_prev_(false), response_trailer_key_prev_(false), expect_final_response_(false), - request_pending_(false), request_headers_dirty_(false) { + request_pending_(false) { ev_timer_init(&upstream_rtimer_, &upstream_rtimeoutcb, 0., get_config()->stream_read_timeout); @@ -1218,12 +1218,4 @@ bool Downstream::can_detach_downstream_connection() const { !response_connection_close_; } -void Downstream::set_request_headers_dirty(bool f) { - request_headers_dirty_ = f; -} - -bool Downstream::get_request_headers_dirty() const { - return request_headers_dirty_; -} - } // namespace shrpx diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index a12359fa..9139eb5d 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -127,8 +127,6 @@ public: void append_last_request_header_value(const char *data, size_t len); // Empties request headers. void clear_request_headers(); - void set_request_headers_dirty(bool f); - bool get_request_headers_dirty() const; size_t get_request_headers_sum() const; @@ -457,8 +455,6 @@ private: // has not been established or should be checked before use; // currently used only with HTTP/2 connection. bool request_pending_; - // true if we need to execute index_request_headers() - bool request_headers_dirty_; }; } // namespace shrpx diff --git a/src/shrpx_mruby.cc b/src/shrpx_mruby.cc index aa3135dd..8e32387d 100644 --- a/src/shrpx_mruby.cc +++ b/src/shrpx_mruby.cc @@ -43,42 +43,49 @@ MRubyContext::MRubyContext(mrb_state *mrb, RProc *on_request_proc, MRubyContext::~MRubyContext() { mrb_close(mrb_); } -int MRubyContext::run_on_request_proc(Downstream *downstream) { - if (!on_request_proc_) { +namespace { +int run_request_proc(mrb_state *mrb, Downstream *downstream, RProc *proc) { + if (!proc) { return 0; } - mrb_->ud = downstream; + MRubyAssocData data{downstream}; + + mrb->ud = &data; int rv = 0; - auto ai = mrb_gc_arena_save(mrb_); + auto ai = mrb_gc_arena_save(mrb); - auto res = mrb_run(mrb_, on_request_proc_, mrb_top_self(mrb_)); + auto res = mrb_run(mrb, proc, mrb_top_self(mrb)); (void)res; - if (mrb_->exc) { + if (mrb->exc) { rv = -1; auto error = - mrb_str_ptr(mrb_funcall(mrb_, mrb_obj_value(mrb_->exc), "inspect", 0)); + mrb_str_ptr(mrb_funcall(mrb, mrb_obj_value(mrb->exc), "inspect", 0)); LOG(ERROR) << "Exception caught while executing mruby code: " << error->as.heap.ptr; - mrb_->exc = 0; + mrb->exc = 0; } - mrb_->ud = nullptr; + mrb->ud = nullptr; - mrb_gc_arena_restore(mrb_, ai); + mrb_gc_arena_restore(mrb, ai); - if (downstream->get_request_headers_dirty()) { - downstream->set_request_headers_dirty(false); + if (data.request_headers_dirty) { downstream->index_request_headers(); } return rv; } +} // namespace -int run_on_response_proc(Downstream *downstream) { +int MRubyContext::run_on_request_proc(Downstream *downstream) { + return run_request_proc(mrb_, downstream, on_request_proc_); +} + +int MRubyContext::run_on_response_proc(Downstream *downstream) { // TODO not implemented yet return 0; } diff --git a/src/shrpx_mruby.h b/src/shrpx_mruby.h index 6e0bc57e..6494d8f0 100644 --- a/src/shrpx_mruby.h +++ b/src/shrpx_mruby.h @@ -52,6 +52,11 @@ private: RProc *on_response_proc_; }; +struct MRubyAssocData { + Downstream *downstream; + bool request_headers_dirty; +}; + RProc *compile(mrb_state *mrb, const char *filename); std::unique_ptr create_mruby_context(); diff --git a/src/shrpx_mruby_module.cc b/src/shrpx_mruby_module.cc index fabb1c5d..9e0be077 100644 --- a/src/shrpx_mruby_module.cc +++ b/src/shrpx_mruby_module.cc @@ -26,21 +26,46 @@ #include #include +#include +#include #include "shrpx_downstream.h" +#include "shrpx_mruby.h" #include "util.h" namespace shrpx { namespace mruby { +namespace { +mrb_value create_headers_hash(mrb_state *mrb, const Headers &headers) { + auto hash = mrb_hash_new(mrb); + + for (auto &hd : headers) { + if (hd.name.empty() || hd.name[0] == ':') { + continue; + } + auto key = mrb_str_new(mrb, hd.name.c_str(), hd.name.size()); + auto ary = mrb_hash_get(mrb, hash, key); + if (mrb_nil_p(ary)) { + ary = mrb_ary_new(mrb); + mrb_hash_set(mrb, hash, key, ary); + } + mrb_ary_push(mrb, ary, mrb_str_new(mrb, hd.value.c_str(), hd.value.size())); + } + + return hash; +} +} // namespace + namespace { mrb_value request_init(mrb_state *mrb, mrb_value self) { return self; } } // namespace namespace { mrb_value request_get_path(mrb_state *mrb, mrb_value self) { - auto downstream = static_cast(mrb->ud); + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; auto &path = downstream->get_request_path(); return mrb_str_new(mrb, path.c_str(), path.size()); @@ -49,7 +74,8 @@ mrb_value request_get_path(mrb_state *mrb, mrb_value self) { namespace { mrb_value request_set_path(mrb_state *mrb, mrb_value self) { - auto downstream = static_cast(mrb->ud); + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; const char *path; mrb_int pathlen; @@ -66,100 +92,51 @@ mrb_value request_set_path(mrb_state *mrb, mrb_value self) { namespace { mrb_value request_get_headers(mrb_state *mrb, mrb_value self) { - auto headers = mrb_iv_get(mrb, self, mrb_intern_lit(mrb, "RequestHeaders")); - if (mrb_nil_p(headers)) { - auto module = mrb_module_get(mrb, "Nghttpx"); - auto headers_class = mrb_class_get_under(mrb, module, "RequestHeaders"); - headers = mrb_obj_new(mrb, headers_class, 0, nullptr); - mrb_iv_set(mrb, self, mrb_intern_lit(mrb, "RequestHeaders"), headers); - } - return headers; + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + return create_headers_hash(mrb, downstream->get_request_headers()); } } // namespace namespace { -mrb_value headers_init(mrb_state *mrb, mrb_value self) { return self; } -} // namespace +mrb_value request_set_header(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; -namespace { -mrb_value request_headers_get(mrb_state *mrb, mrb_value self) { - auto downstream = static_cast(mrb->ud); - - mrb_value key; - mrb_get_args(mrb, "o", &key); - - key = mrb_funcall(mrb, key, "downcase", 0); - - if (RSTRING_LEN(key) == 0) { - return key; - } - - auto hd = downstream->get_request_header( - std::string(RSTRING_PTR(key), RSTRING_LEN(key))); - - if (hd == nullptr) { - return mrb_nil_value(); - } - - return mrb_str_new(mrb, hd->value.c_str(), hd->value.size()); -} -} // namespace - -namespace { -mrb_value request_headers_set(mrb_state *mrb, mrb_value self, bool repl) { - auto downstream = static_cast(mrb->ud); - - mrb_value key, value; - mrb_get_args(mrb, "oo", &key, &value); - - key = mrb_funcall(mrb, key, "downcase", 0); + mrb_value key, values; + mrb_get_args(mrb, "oo", &key, &values); if (RSTRING_LEN(key) == 0) { mrb_raise(mrb, E_RUNTIME_ERROR, "empty key is not allowed"); } - if (repl) { - for (auto &hd : downstream->get_request_headers()) { - if (util::streq(std::begin(hd.name), hd.name.size(), RSTRING_PTR(key), - RSTRING_LEN(key))) { - hd.name = ""; - } + key = mrb_funcall(mrb, key, "downcase", 0); + + // making name empty will effectively delete header fields + for (auto &hd : downstream->get_request_headers()) { + if (util::streq(std::begin(hd.name), hd.name.size(), RSTRING_PTR(key), + RSTRING_LEN(key))) { + hd.name = ""; } } - downstream->add_request_header( - std::string(RSTRING_PTR(key), RSTRING_LEN(key)), - std::string(RSTRING_PTR(value), RSTRING_LEN(value))); + if (mrb_obj_is_instance_of(mrb, values, mrb->array_class)) { + auto n = mrb_ary_len(mrb, values); + for (int i = 0; i < n; ++i) { + auto value = mrb_ary_entry(values, i); + downstream->add_request_header( + std::string(RSTRING_PTR(key), RSTRING_LEN(key)), + std::string(RSTRING_PTR(value), RSTRING_LEN(value))); + } + } else { + downstream->add_request_header( + std::string(RSTRING_PTR(key), RSTRING_LEN(key)), + std::string(RSTRING_PTR(values), RSTRING_LEN(values))); + } - downstream->set_request_headers_dirty(true); + data->request_headers_dirty = true; - return key; -} -} // namespace - -namespace { -mrb_value request_headers_set(mrb_state *mrb, mrb_value self) { - return request_headers_set(mrb, self, true); -} -} // namespace - -namespace { -mrb_value request_headers_add(mrb_state *mrb, mrb_value self) { - return request_headers_set(mrb, self, false); -} -} // namespace - -namespace { -void init_headers_class(mrb_state *mrb, RClass *module, const char *name, - mrb_func_t get, mrb_func_t set, mrb_func_t add) { - auto headers_class = - mrb_define_class_under(mrb, module, name, mrb->object_class); - - mrb_define_method(mrb, headers_class, "initialize", headers_init, - MRB_ARGS_NONE()); - mrb_define_method(mrb, headers_class, "get", get, MRB_ARGS_REQ(1)); - mrb_define_method(mrb, headers_class, "set", set, MRB_ARGS_REQ(2)); - mrb_define_method(mrb, headers_class, "add", add, MRB_ARGS_REQ(2)); + return mrb_nil_value(); } } // namespace @@ -176,15 +153,30 @@ void init_request_class(mrb_state *mrb, RClass *module) { MRB_ARGS_REQ(1)); mrb_define_method(mrb, request_class, "headers", request_get_headers, MRB_ARGS_NONE()); + mrb_define_method(mrb, request_class, "set_header", request_set_header, + MRB_ARGS_REQ(2)); +} +} // namespace - init_headers_class(mrb, module, "RequestHeaders", request_headers_get, - request_headers_set, request_headers_add); +namespace { +mrb_value run(mrb_state *mrb, mrb_value self) { + mrb_value b; + mrb_get_args(mrb, "&", &b); + + auto module = mrb_module_get(mrb, "Nghttpx"); + auto request_class = mrb_class_get_under(mrb, module, "Request"); + auto request = mrb_obj_new(mrb, request_class, 0, nullptr); + + std::array args{{request}}; + return mrb_yield_argv(mrb, b, args.size(), args.data()); } } // namespace void init_module(mrb_state *mrb) { auto module = mrb_define_module(mrb, "Nghttpx"); + mrb_define_class_method(mrb, module, "run", run, MRB_ARGS_BLOCK()); + init_request_class(mrb, module); } From 068529586d10a939fdef5e767aae322a53005c3a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 2 Sep 2015 22:38:57 +0900 Subject: [PATCH 05/55] nghttpx: Split up request class definition to dedicated files --- src/Makefile.am | 1 + src/shrpx_mruby_module.cc | 145 +++++------------------------- src/shrpx_mruby_module.h | 4 + src/shrpx_mruby_module_request.cc | 141 +++++++++++++++++++++++++++++ src/shrpx_mruby_module_request.h | 44 +++++++++ 5 files changed, 212 insertions(+), 123 deletions(-) create mode 100644 src/shrpx_mruby_module_request.cc create mode 100644 src/shrpx_mruby_module_request.h diff --git a/src/Makefile.am b/src/Makefile.am index bc3a6ce7..bcae9aaa 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -126,6 +126,7 @@ NGHTTPX_SRCS = \ shrpx_memcached_result.h \ shrpx_mruby.cc shrpx_mruby.h \ shrpx_mruby_module.cc shrpx_mruby_module.h \ + shrpx_mruby_module_request.cc shrpx_mruby_module_request.h \ buffer.h memchunk.h template.h if HAVE_SPDYLAY diff --git a/src/shrpx_mruby_module.cc b/src/shrpx_mruby_module.cc index 9e0be077..0b58643e 100644 --- a/src/shrpx_mruby_module.cc +++ b/src/shrpx_mruby_module.cc @@ -24,140 +24,20 @@ */ #include "shrpx_mruby_module.h" +#include + #include #include #include #include -#include "shrpx_downstream.h" #include "shrpx_mruby.h" -#include "util.h" +#include "shrpx_mruby_module_request.h" namespace shrpx { namespace mruby { -namespace { -mrb_value create_headers_hash(mrb_state *mrb, const Headers &headers) { - auto hash = mrb_hash_new(mrb); - - for (auto &hd : headers) { - if (hd.name.empty() || hd.name[0] == ':') { - continue; - } - auto key = mrb_str_new(mrb, hd.name.c_str(), hd.name.size()); - auto ary = mrb_hash_get(mrb, hash, key); - if (mrb_nil_p(ary)) { - ary = mrb_ary_new(mrb); - mrb_hash_set(mrb, hash, key, ary); - } - mrb_ary_push(mrb, ary, mrb_str_new(mrb, hd.value.c_str(), hd.value.size())); - } - - return hash; -} -} // namespace - -namespace { -mrb_value request_init(mrb_state *mrb, mrb_value self) { return self; } -} // namespace - -namespace { -mrb_value request_get_path(mrb_state *mrb, mrb_value self) { - auto data = static_cast(mrb->ud); - auto downstream = data->downstream; - auto &path = downstream->get_request_path(); - - return mrb_str_new(mrb, path.c_str(), path.size()); -} -} // namespace - -namespace { -mrb_value request_set_path(mrb_state *mrb, mrb_value self) { - auto data = static_cast(mrb->ud); - auto downstream = data->downstream; - - const char *path; - mrb_int pathlen; - mrb_get_args(mrb, "s", &path, &pathlen); - if (pathlen == 0) { - mrb_raise(mrb, E_RUNTIME_ERROR, "path must not be empty string"); - } - - downstream->set_request_path(std::string(path, pathlen)); - - return self; -} -} // namespace - -namespace { -mrb_value request_get_headers(mrb_state *mrb, mrb_value self) { - auto data = static_cast(mrb->ud); - auto downstream = data->downstream; - return create_headers_hash(mrb, downstream->get_request_headers()); -} -} // namespace - -namespace { -mrb_value request_set_header(mrb_state *mrb, mrb_value self) { - auto data = static_cast(mrb->ud); - auto downstream = data->downstream; - - mrb_value key, values; - mrb_get_args(mrb, "oo", &key, &values); - - if (RSTRING_LEN(key) == 0) { - mrb_raise(mrb, E_RUNTIME_ERROR, "empty key is not allowed"); - } - - key = mrb_funcall(mrb, key, "downcase", 0); - - // making name empty will effectively delete header fields - for (auto &hd : downstream->get_request_headers()) { - if (util::streq(std::begin(hd.name), hd.name.size(), RSTRING_PTR(key), - RSTRING_LEN(key))) { - hd.name = ""; - } - } - - if (mrb_obj_is_instance_of(mrb, values, mrb->array_class)) { - auto n = mrb_ary_len(mrb, values); - for (int i = 0; i < n; ++i) { - auto value = mrb_ary_entry(values, i); - downstream->add_request_header( - std::string(RSTRING_PTR(key), RSTRING_LEN(key)), - std::string(RSTRING_PTR(value), RSTRING_LEN(value))); - } - } else { - downstream->add_request_header( - std::string(RSTRING_PTR(key), RSTRING_LEN(key)), - std::string(RSTRING_PTR(values), RSTRING_LEN(values))); - } - - data->request_headers_dirty = true; - - return mrb_nil_value(); -} -} // namespace - -namespace { -void init_request_class(mrb_state *mrb, RClass *module) { - auto request_class = - mrb_define_class_under(mrb, module, "Request", mrb->object_class); - - mrb_define_method(mrb, request_class, "initialize", request_init, - MRB_ARGS_NONE()); - mrb_define_method(mrb, request_class, "path", request_get_path, - MRB_ARGS_NONE()); - mrb_define_method(mrb, request_class, "path=", request_set_path, - MRB_ARGS_REQ(1)); - mrb_define_method(mrb, request_class, "headers", request_get_headers, - MRB_ARGS_NONE()); - mrb_define_method(mrb, request_class, "set_header", request_set_header, - MRB_ARGS_REQ(2)); -} -} // namespace - namespace { mrb_value run(mrb_state *mrb, mrb_value self) { mrb_value b; @@ -180,6 +60,25 @@ void init_module(mrb_state *mrb) { init_request_class(mrb, module); } +mrb_value create_headers_hash(mrb_state *mrb, const Headers &headers) { + auto hash = mrb_hash_new(mrb); + + for (auto &hd : headers) { + if (hd.name.empty() || hd.name[0] == ':') { + continue; + } + auto key = mrb_str_new(mrb, hd.name.c_str(), hd.name.size()); + auto ary = mrb_hash_get(mrb, hash, key); + if (mrb_nil_p(ary)) { + ary = mrb_ary_new(mrb); + mrb_hash_set(mrb, hash, key, ary); + } + mrb_ary_push(mrb, ary, mrb_str_new(mrb, hd.value.c_str(), hd.value.size())); + } + + return hash; +} + } // namespace mruby } // namespace shrpx diff --git a/src/shrpx_mruby_module.h b/src/shrpx_mruby_module.h index bec51934..f4e300a4 100644 --- a/src/shrpx_mruby_module.h +++ b/src/shrpx_mruby_module.h @@ -29,6 +29,8 @@ #include +#include "http2.h" + using namespace nghttp2; namespace shrpx { @@ -37,6 +39,8 @@ namespace mruby { void init_module(mrb_state *mrb); +mrb_value create_headers_hash(mrb_state *mrb, const Headers &headers); + } // namespace mruby } // namespace shrpx diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc new file mode 100644 index 00000000..48d8167a --- /dev/null +++ b/src/shrpx_mruby_module_request.cc @@ -0,0 +1,141 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2015 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "shrpx_mruby_module_request.h" + +#include +#include +#include +#include + +#include "shrpx_downstream.h" +#include "shrpx_mruby.h" +#include "shrpx_mruby_module.h" +#include "util.h" + +namespace shrpx { + +namespace mruby { + +namespace { +mrb_value request_init(mrb_state *mrb, mrb_value self) { return self; } +} // namespace + +namespace { +mrb_value request_get_path(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + auto &path = downstream->get_request_path(); + + return mrb_str_new(mrb, path.c_str(), path.size()); +} +} // namespace + +namespace { +mrb_value request_set_path(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + + const char *path; + mrb_int pathlen; + mrb_get_args(mrb, "s", &path, &pathlen); + if (pathlen == 0) { + mrb_raise(mrb, E_RUNTIME_ERROR, "path must not be empty string"); + } + + downstream->set_request_path(std::string(path, pathlen)); + + return self; +} +} // namespace + +namespace { +mrb_value request_get_headers(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + return create_headers_hash(mrb, downstream->get_request_headers()); +} +} // namespace + +namespace { +mrb_value request_set_header(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + + mrb_value key, values; + mrb_get_args(mrb, "oo", &key, &values); + + if (RSTRING_LEN(key) == 0) { + mrb_raise(mrb, E_RUNTIME_ERROR, "empty key is not allowed"); + } + + key = mrb_funcall(mrb, key, "downcase", 0); + + // making name empty will effectively delete header fields + for (auto &hd : downstream->get_request_headers()) { + if (util::streq(std::begin(hd.name), hd.name.size(), RSTRING_PTR(key), + RSTRING_LEN(key))) { + hd.name = ""; + } + } + + if (mrb_obj_is_instance_of(mrb, values, mrb->array_class)) { + auto n = mrb_ary_len(mrb, values); + for (int i = 0; i < n; ++i) { + auto value = mrb_ary_entry(values, i); + downstream->add_request_header( + std::string(RSTRING_PTR(key), RSTRING_LEN(key)), + std::string(RSTRING_PTR(value), RSTRING_LEN(value))); + } + } else { + downstream->add_request_header( + std::string(RSTRING_PTR(key), RSTRING_LEN(key)), + std::string(RSTRING_PTR(values), RSTRING_LEN(values))); + } + + data->request_headers_dirty = true; + + return mrb_nil_value(); +} +} // namespace + +void init_request_class(mrb_state *mrb, RClass *module) { + auto request_class = + mrb_define_class_under(mrb, module, "Request", mrb->object_class); + + mrb_define_method(mrb, request_class, "initialize", request_init, + MRB_ARGS_NONE()); + mrb_define_method(mrb, request_class, "path", request_get_path, + MRB_ARGS_NONE()); + mrb_define_method(mrb, request_class, "path=", request_set_path, + MRB_ARGS_REQ(1)); + mrb_define_method(mrb, request_class, "headers", request_get_headers, + MRB_ARGS_NONE()); + mrb_define_method(mrb, request_class, "set_header", request_set_header, + MRB_ARGS_REQ(2)); +} + +} // namespace mruby + +} // namespace shrpx diff --git a/src/shrpx_mruby_module_request.h b/src/shrpx_mruby_module_request.h new file mode 100644 index 00000000..b27c569b --- /dev/null +++ b/src/shrpx_mruby_module_request.h @@ -0,0 +1,44 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2015 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef SHRPX_MRUBY_MODULE_REQUEST_H +#define SHRPX_MRUBY_MODULE_REQUEST_H + +#include "shrpx.h" + +#include + +using namespace nghttp2; + +namespace shrpx { + +namespace mruby { + +void init_request_class(mrb_state *mrb, RClass *module); + +} // namespace mruby + +} // namespace shrpx + +#endif // SHRPX_MRUBY_MODULE_REQUEST_H From f0d41a5ac2d3ccbc35b1818be1c860c63325f0d2 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 2 Sep 2015 22:41:48 +0900 Subject: [PATCH 06/55] nghttpx: Handle nil given to request.set_header as value --- src/shrpx_mruby_module_request.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index 48d8167a..07257394 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -108,7 +108,7 @@ mrb_value request_set_header(mrb_state *mrb, mrb_value self) { std::string(RSTRING_PTR(key), RSTRING_LEN(key)), std::string(RSTRING_PTR(value), RSTRING_LEN(value))); } - } else { + } else if (!mrb_nil_p(values)) { downstream->add_request_header( std::string(RSTRING_PTR(key), RSTRING_LEN(key)), std::string(RSTRING_PTR(values), RSTRING_LEN(values))); From e436caf0bb81d0c7af44010c7e37f2badae34723 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 2 Sep 2015 23:21:45 +0900 Subject: [PATCH 07/55] nghttpx: Don't forward request if response was commited in mruby hook --- src/shrpx_http2_upstream.cc | 8 ++++++-- src/shrpx_https_upstream.cc | 12 ++++++++---- src/shrpx_mruby.cc | 5 +++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 59dc5299..ecf10b0e 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -307,6 +307,8 @@ int Http2Upstream::on_request_headers(Downstream *downstream, downstream->set_request_http2_expect_body(true); } + downstream->inspect_http2_request(); + auto upstream = downstream->get_upstream(); auto handler = upstream->get_client_handler(); auto worker = handler->get_worker(); @@ -314,8 +316,6 @@ int Http2Upstream::on_request_headers(Downstream *downstream, mruby_ctx->run_on_request_proc(downstream); - downstream->inspect_http2_request(); - downstream->set_request_state(Downstream::HEADER_COMPLETE); if (frame->hd.flags & NGHTTP2_FLAG_END_STREAM) { downstream->disable_upstream_rtimer(); @@ -323,6 +323,10 @@ int Http2Upstream::on_request_headers(Downstream *downstream, downstream->set_request_state(Downstream::MSG_COMPLETE); } + if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { + return 0; + } + start_downstream(downstream); return 0; diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 4da4fed0..1e609037 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -277,8 +277,6 @@ int htp_hdrs_completecb(http_parser *htp) { auto worker = handler->get_worker(); auto mruby_ctx = worker->get_mruby_context(); - mruby_ctx->run_on_request_proc(downstream); - downstream->inspect_http1_request(); if (downstream->get_request_method() != HTTP_CONNECT) { @@ -312,6 +310,14 @@ int htp_hdrs_completecb(http_parser *htp) { } } + mruby_ctx->run_on_request_proc(downstream); + + downstream->set_request_state(Downstream::HEADER_COMPLETE); + + if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { + return 0; + } + rv = downstream->attach_downstream_connection( upstream->get_client_handler()->get_downstream_connection(downstream)); @@ -327,8 +333,6 @@ int htp_hdrs_completecb(http_parser *htp) { return -1; } - downstream->set_request_state(Downstream::HEADER_COMPLETE); - return 0; } } // namespace diff --git a/src/shrpx_mruby.cc b/src/shrpx_mruby.cc index 8e32387d..39940dd1 100644 --- a/src/shrpx_mruby.cc +++ b/src/shrpx_mruby.cc @@ -30,6 +30,7 @@ #include "shrpx_downstream.h" #include "shrpx_config.h" #include "shrpx_mruby_module.h" +#include "shrpx_downstream_connection.h" #include "template.h" namespace shrpx { @@ -77,6 +78,10 @@ int run_request_proc(mrb_state *mrb, Downstream *downstream, RProc *proc) { downstream->index_request_headers(); } + if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { + downstream->pop_downstream_connection(); + } + return rv; } } // namespace From 5cd34920b74ac1979be21fc6c861e97c1abdaabb Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 2 Sep 2015 23:22:33 +0900 Subject: [PATCH 08/55] nghttpx: Add Request class accessors --- src/shrpx_mruby_module_request.cc | 122 ++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index 07257394..98ede3bd 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -33,6 +33,7 @@ #include "shrpx_mruby.h" #include "shrpx_mruby_module.h" #include "util.h" +#include "http2.h" namespace shrpx { @@ -42,6 +43,111 @@ namespace { mrb_value request_init(mrb_state *mrb, mrb_value self) { return self; } } // namespace +namespace { +mrb_value request_get_http_version_major(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + return mrb_fixnum_value(downstream->get_request_major()); +} +} // namespace + +namespace { +mrb_value request_get_http_version_minor(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + return mrb_fixnum_value(downstream->get_request_minor()); +} +} // namespace + +namespace { +mrb_value request_get_method(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + auto method = http2::to_method_string(downstream->get_request_method()); + + return mrb_str_new_cstr(mrb, method); +} +} // namespace + +namespace { +mrb_value request_set_method(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + + const char *method; + mrb_int n; + mrb_get_args(mrb, "s", &method, &n); + if (n == 0) { + mrb_raise(mrb, E_RUNTIME_ERROR, "method must not be empty string"); + } + auto token = + http2::lookup_method_token(reinterpret_cast(method), n); + if (token == -1) { + mrb_raise(mrb, E_RUNTIME_ERROR, "method not supported"); + } + + downstream->set_request_method(token); + + return self; +} +} // namespace + +namespace { +mrb_value request_get_authority(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + auto &authority = downstream->get_request_http2_authority(); + + return mrb_str_new(mrb, authority.c_str(), authority.size()); +} +} // namespace + +namespace { +mrb_value request_set_authority(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + + const char *authority; + mrb_int n; + mrb_get_args(mrb, "s", &authority, &n); + if (n == 0) { + mrb_raise(mrb, E_RUNTIME_ERROR, "authority must not be empty string"); + } + + downstream->set_request_http2_authority(std::string(authority, n)); + + return self; +} +} // namespace + +namespace { +mrb_value request_get_scheme(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + auto &scheme = downstream->get_request_http2_scheme(); + + return mrb_str_new(mrb, scheme.c_str(), scheme.size()); +} +} // namespace + +namespace { +mrb_value request_set_scheme(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + + const char *scheme; + mrb_int n; + mrb_get_args(mrb, "s", &scheme, &n); + if (n == 0) { + mrb_raise(mrb, E_RUNTIME_ERROR, "scheme must not be empty string"); + } + + downstream->set_request_http2_scheme(std::string(scheme, n)); + + return self; +} +} // namespace + namespace { mrb_value request_get_path(mrb_state *mrb, mrb_value self) { auto data = static_cast(mrb->ud); @@ -126,6 +232,22 @@ void init_request_class(mrb_state *mrb, RClass *module) { mrb_define_method(mrb, request_class, "initialize", request_init, MRB_ARGS_NONE()); + mrb_define_method(mrb, request_class, "http_version_major", + request_get_http_version_major, MRB_ARGS_NONE()); + mrb_define_method(mrb, request_class, "http_version_minor", + request_get_http_version_minor, MRB_ARGS_NONE()); + mrb_define_method(mrb, request_class, "method", request_get_method, + MRB_ARGS_NONE()); + mrb_define_method(mrb, request_class, "method=", request_set_method, + MRB_ARGS_REQ(1)); + mrb_define_method(mrb, request_class, "authority", request_get_authority, + MRB_ARGS_NONE()); + mrb_define_method(mrb, request_class, "authority=", request_set_authority, + MRB_ARGS_REQ(1)); + mrb_define_method(mrb, request_class, "scheme", request_get_scheme, + MRB_ARGS_NONE()); + mrb_define_method(mrb, request_class, "scheme=", request_set_scheme, + MRB_ARGS_REQ(1)); mrb_define_method(mrb, request_class, "path", request_get_path, MRB_ARGS_NONE()); mrb_define_method(mrb, request_class, "path=", request_set_path, From e06af02573f4183111726225cc01ed1b7adfc42d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 3 Sep 2015 00:40:14 +0900 Subject: [PATCH 09/55] nghttpx: Add Response mruby object --- src/Makefile.am | 1 + src/http2.cc | 9 ++ src/http2.h | 3 + src/shrpx_downstream.cc | 6 + src/shrpx_downstream.h | 2 + src/shrpx_http2_upstream.cc | 10 +- src/shrpx_https_upstream.cc | 32 +++-- src/shrpx_mruby.cc | 4 + src/shrpx_mruby.h | 1 + src/shrpx_mruby_module.cc | 7 +- src/shrpx_mruby_module_response.cc | 221 +++++++++++++++++++++++++++++ src/shrpx_mruby_module_response.h | 44 ++++++ src/shrpx_spdy_upstream.cc | 20 +++ 13 files changed, 343 insertions(+), 17 deletions(-) create mode 100644 src/shrpx_mruby_module_response.cc create mode 100644 src/shrpx_mruby_module_response.h diff --git a/src/Makefile.am b/src/Makefile.am index bcae9aaa..0384a997 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -127,6 +127,7 @@ NGHTTPX_SRCS = \ shrpx_mruby.cc shrpx_mruby.h \ shrpx_mruby_module.cc shrpx_mruby_module.h \ shrpx_mruby_module_request.cc shrpx_mruby_module_request.h \ + shrpx_mruby_module_response.cc shrpx_mruby_module_response.h \ buffer.h memchunk.h template.h if HAVE_SPDYLAY diff --git a/src/http2.cc b/src/http2.cc index bba0464a..f00aeb0a 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -664,6 +664,15 @@ const Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, return &nva[i]; } +Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, + Headers &nva) { + auto i = hdidx[token]; + if (i == -1) { + return nullptr; + } + return &nva[i]; +} + namespace { template InputIt skip_lws(InputIt first, InputIt last) { for (; first != last; ++first) { diff --git a/src/http2.h b/src/http2.h index 182c45d9..09f04cd2 100644 --- a/src/http2.h +++ b/src/http2.h @@ -262,6 +262,9 @@ bool http2_mandatory_request_headers_presence(const HeaderIndex &hdidx); const Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, const Headers &nva); +Headers::value_type *get_header(const HeaderIndex &hdidx, int16_t token, + Headers &nva); + struct LinkHeader { // The region of URI is [uri.first, uri.second). std::pair uri; diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index d9325699..9c94a2de 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -609,6 +609,8 @@ const Headers &Downstream::get_response_headers() const { return response_headers_; } +Headers &Downstream::get_response_headers() { return response_headers_; } + int Downstream::index_response_headers() { return index_headers(response_hdidx_, response_headers_, response_content_length_); @@ -619,6 +621,10 @@ Downstream::get_response_header(int16_t token) const { return http2::get_header(response_hdidx_, token, response_headers_); } +Headers::value_type *Downstream::get_response_header(int16_t token) { + return http2::get_header(response_hdidx_, token, response_headers_); +} + void Downstream::rewrite_location_response_header( const std::string &upstream_scheme) { auto hd = diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 9139eb5d..860934c5 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -208,6 +208,7 @@ public: bool request_submission_ready() const; // downstream response API const Headers &get_response_headers() const; + Headers &get_response_headers(); // Lower the response header field names and indexes response // headers. If there are invalid headers (e.g., multiple // Content-Length with different values), returns -1. @@ -217,6 +218,7 @@ public: // the beginning. If no such header is found, returns nullptr. // This function must be called after response headers are indexed. const Headers::value_type *get_response_header(int16_t token) const; + Headers::value_type *get_response_header(int16_t token); // Rewrites the location response header field. void rewrite_location_response_header(const std::string &upstream_scheme); void add_response_header(std::string name, std::string value); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index ecf10b0e..01f6d380 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -309,14 +309,20 @@ int Http2Upstream::on_request_headers(Downstream *downstream, downstream->inspect_http2_request(); + downstream->set_request_state(Downstream::HEADER_COMPLETE); + auto upstream = downstream->get_upstream(); auto handler = upstream->get_client_handler(); auto worker = handler->get_worker(); auto mruby_ctx = worker->get_mruby_context(); - mruby_ctx->run_on_request_proc(downstream); + if (mruby_ctx->run_on_request_proc(downstream) != 0) { + if (error_reply(downstream, 500) != 0) { + return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + } + return 0; + } - downstream->set_request_state(Downstream::HEADER_COMPLETE); if (frame->hd.flags & NGHTTP2_FLAG_END_STREAM) { downstream->disable_upstream_rtimer(); diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 1e609037..98ceb3e8 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -273,10 +273,6 @@ int htp_hdrs_completecb(http_parser *htp) { return -1; } - auto handler = upstream->get_client_handler(); - auto worker = handler->get_worker(); - auto mruby_ctx = worker->get_mruby_context(); - downstream->inspect_http1_request(); if (downstream->get_request_method() != HTTP_CONNECT) { @@ -310,10 +306,17 @@ int htp_hdrs_completecb(http_parser *htp) { } } - mruby_ctx->run_on_request_proc(downstream); - downstream->set_request_state(Downstream::HEADER_COMPLETE); + auto handler = upstream->get_client_handler(); + auto worker = handler->get_worker(); + auto mruby_ctx = worker->get_mruby_context(); + + if (mruby_ctx->run_on_request_proc(downstream) != 0) { + downstream->set_response_http_status(500); + return -1; + } + if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { return 0; } @@ -483,13 +486,16 @@ int HttpsUpstream::on_read() { if (htperr == HPE_INVALID_METHOD) { status_code = 501; } else if (downstream) { - if (downstream->get_request_state() == Downstream::CONNECT_FAIL) { - status_code = 503; - } else if (downstream->get_request_state() == - Downstream::HTTP1_REQUEST_HEADER_TOO_LARGE) { - status_code = 431; - } else { - status_code = 400; + status_code = downstream->get_response_http_status(); + if (status_code == 0) { + if (downstream->get_request_state() == Downstream::CONNECT_FAIL) { + status_code = 503; + } else if (downstream->get_request_state() == + Downstream::HTTP1_REQUEST_HEADER_TOO_LARGE) { + status_code = 431; + } else { + status_code = 400; + } } } else { status_code = 400; diff --git a/src/shrpx_mruby.cc b/src/shrpx_mruby.cc index 39940dd1..52ba2aeb 100644 --- a/src/shrpx_mruby.cc +++ b/src/shrpx_mruby.cc @@ -78,6 +78,10 @@ int run_request_proc(mrb_state *mrb, Downstream *downstream, RProc *proc) { downstream->index_request_headers(); } + if (data.response_headers_dirty) { + downstream->index_response_headers(); + } + if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { downstream->pop_downstream_connection(); } diff --git a/src/shrpx_mruby.h b/src/shrpx_mruby.h index 6494d8f0..6f75db3b 100644 --- a/src/shrpx_mruby.h +++ b/src/shrpx_mruby.h @@ -55,6 +55,7 @@ private: struct MRubyAssocData { Downstream *downstream; bool request_headers_dirty; + bool response_headers_dirty; }; RProc *compile(mrb_state *mrb, const char *filename); diff --git a/src/shrpx_mruby_module.cc b/src/shrpx_mruby_module.cc index 0b58643e..c5e66056 100644 --- a/src/shrpx_mruby_module.cc +++ b/src/shrpx_mruby_module.cc @@ -33,6 +33,7 @@ #include "shrpx_mruby.h" #include "shrpx_mruby_module_request.h" +#include "shrpx_mruby_module_response.h" namespace shrpx { @@ -45,9 +46,10 @@ mrb_value run(mrb_state *mrb, mrb_value self) { auto module = mrb_module_get(mrb, "Nghttpx"); auto request_class = mrb_class_get_under(mrb, module, "Request"); - auto request = mrb_obj_new(mrb, request_class, 0, nullptr); + auto response_class = mrb_class_get_under(mrb, module, "Response"); - std::array args{{request}}; + std::array args{{mrb_obj_new(mrb, response_class, 0, nullptr), + mrb_obj_new(mrb, request_class, 0, nullptr)}}; return mrb_yield_argv(mrb, b, args.size(), args.data()); } } // namespace @@ -58,6 +60,7 @@ void init_module(mrb_state *mrb) { mrb_define_class_method(mrb, module, "run", run, MRB_ARGS_BLOCK()); init_request_class(mrb, module); + init_response_class(mrb, module); } mrb_value create_headers_hash(mrb_state *mrb, const Headers &headers) { diff --git a/src/shrpx_mruby_module_response.cc b/src/shrpx_mruby_module_response.cc new file mode 100644 index 00000000..5e0a2d80 --- /dev/null +++ b/src/shrpx_mruby_module_response.cc @@ -0,0 +1,221 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2015 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "shrpx_mruby_module_response.h" + +#include +#include +#include +#include + +#include "shrpx_downstream.h" +#include "shrpx_upstream.h" +#include "shrpx_mruby.h" +#include "shrpx_mruby_module.h" +#include "util.h" +#include "http2.h" + +namespace shrpx { + +namespace mruby { + +namespace { +mrb_value response_init(mrb_state *mrb, mrb_value self) { return self; } +} // namespace + +namespace { +mrb_value response_get_http_version_major(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + return mrb_fixnum_value(downstream->get_response_major()); +} +} // namespace + +namespace { +mrb_value response_get_http_version_minor(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + return mrb_fixnum_value(downstream->get_response_minor()); +} +} // namespace + +namespace { +mrb_value response_get_status(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + + return mrb_fixnum_value(downstream->get_response_http_status()); +} +} // namespace + +namespace { +mrb_value response_set_status(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + + mrb_int status; + mrb_get_args(mrb, "i", &status); + // We don't support 1xx status code for mruby scripting yet. + if (status < 200 || status > 999) { + mrb_raise(mrb, E_RUNTIME_ERROR, + "invalid status; it should be [200, 999], inclusive"); + } + + downstream->set_response_http_status(status); + + return self; +} +} // namespace + +namespace { +mrb_value response_get_headers(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + return create_headers_hash(mrb, downstream->get_response_headers()); +} +} // namespace + +namespace { +mrb_value response_set_header(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + + mrb_value key, values; + mrb_get_args(mrb, "oo", &key, &values); + + if (RSTRING_LEN(key) == 0) { + mrb_raise(mrb, E_RUNTIME_ERROR, "empty key is not allowed"); + } + + key = mrb_funcall(mrb, key, "downcase", 0); + + // making name empty will effectively delete header fields + for (auto &hd : downstream->get_response_headers()) { + if (util::streq(std::begin(hd.name), hd.name.size(), RSTRING_PTR(key), + RSTRING_LEN(key))) { + hd.name = ""; + } + } + + if (mrb_obj_is_instance_of(mrb, values, mrb->array_class)) { + auto n = mrb_ary_len(mrb, values); + for (int i = 0; i < n; ++i) { + auto value = mrb_ary_entry(values, i); + downstream->add_response_header( + std::string(RSTRING_PTR(key), RSTRING_LEN(key)), + std::string(RSTRING_PTR(value), RSTRING_LEN(value))); + } + } else if (!mrb_nil_p(values)) { + downstream->add_response_header( + std::string(RSTRING_PTR(key), RSTRING_LEN(key)), + std::string(RSTRING_PTR(values), RSTRING_LEN(values))); + } + + data->response_headers_dirty = true; + + return mrb_nil_value(); +} +} // namespace + +namespace { +mrb_value response_end(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + int rv; + + if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { + mrb_raise(mrb, E_RUNTIME_ERROR, "response has already been committed"); + } + + mrb_value val; + mrb_get_args(mrb, "|o", &val); + + const char *body = nullptr; + size_t bodylen = 0; + + if (!mrb_nil_p(val)) { + body = RSTRING_PTR(val); + bodylen = RSTRING_LEN(val); + } + + if (downstream->get_response_http_status() == 0) { + downstream->set_response_http_status(200); + } + + if (data->response_headers_dirty) { + downstream->index_response_headers(); + data->response_headers_dirty = false; + } + + auto cl = downstream->get_response_header(http2::HD_CONTENT_LENGTH); + if (cl) { + cl->value = util::utos(bodylen); + } else { + downstream->add_response_header("content-length", util::utos(bodylen), + http2::HD_CONTENT_LENGTH); + } + downstream->set_response_content_length(bodylen); + + auto upstream = downstream->get_upstream(); + + rv = upstream->on_downstream_header_complete(downstream); + if (rv != 0) { + mrb_raise(mrb, E_RUNTIME_ERROR, "could not send response"); + } + + if (downstream->expect_response_body()) { + auto output = downstream->get_response_buf(); + output->append(body, bodylen); + } + + downstream->set_response_state(Downstream::MSG_COMPLETE); + + return self; +} +} // namespace + +void init_response_class(mrb_state *mrb, RClass *module) { + auto response_class = + mrb_define_class_under(mrb, module, "Response", mrb->object_class); + + mrb_define_method(mrb, response_class, "initialize", response_init, + MRB_ARGS_NONE()); + mrb_define_method(mrb, response_class, "http_version_major", + response_get_http_version_major, MRB_ARGS_NONE()); + mrb_define_method(mrb, response_class, "http_version_minor", + response_get_http_version_minor, MRB_ARGS_NONE()); + mrb_define_method(mrb, response_class, "status", response_get_status, + MRB_ARGS_NONE()); + mrb_define_method(mrb, response_class, "status=", response_set_status, + MRB_ARGS_REQ(1)); + mrb_define_method(mrb, response_class, "headers", response_get_headers, + MRB_ARGS_NONE()); + mrb_define_method(mrb, response_class, "set_header", response_set_header, + MRB_ARGS_REQ(2)); + mrb_define_method(mrb, response_class, "end", response_end, MRB_ARGS_OPT(1)); +} + +} // namespace mruby + +} // namespace shrpx diff --git a/src/shrpx_mruby_module_response.h b/src/shrpx_mruby_module_response.h new file mode 100644 index 00000000..32ed0d41 --- /dev/null +++ b/src/shrpx_mruby_module_response.h @@ -0,0 +1,44 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2015 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef SHRPX_MRUBY_MODULE_RESPONSE_H +#define SHRPX_MRUBY_MODULE_RESPONSE_H + +#include "shrpx.h" + +#include + +using namespace nghttp2; + +namespace shrpx { + +namespace mruby { + +void init_response_class(mrb_state *mrb, RClass *module); + +} // namespace mruby + +} // namespace shrpx + +#endif // SHRPX_MRUBY_MODULE_RESPONSE_H diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 89fcb678..5056153e 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -36,6 +36,9 @@ #include "shrpx_downstream_connection.h" #include "shrpx_config.h" #include "shrpx_http.h" +#include "shrpx_mruby.h" +#include "shrpx_worker.h" +#include "shrpx_http2_session.h" #include "http2.h" #include "util.h" #include "template.h" @@ -237,6 +240,19 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, downstream->inspect_http2_request(); downstream->set_request_state(Downstream::HEADER_COMPLETE); + + auto handler = upstream->get_client_handler(); + auto worker = handler->get_worker(); + auto mruby_ctx = worker->get_mruby_context(); + + if (mruby_ctx->run_on_request_proc(downstream) != 0) { + if (upstream->error_reply(downstream, 500) != 0) { + ULOG(FATAL, upstream) << "error_reply failed"; + return; + } + return; + } + if (frame->syn_stream.hd.flags & SPDYLAY_CTRL_FLAG_FIN) { if (!downstream->validate_request_bodylen()) { upstream->rst_stream(downstream, SPDYLAY_PROTOCOL_ERROR); @@ -247,6 +263,10 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, downstream->set_request_state(Downstream::MSG_COMPLETE); } + if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { + return; + } + upstream->start_downstream(downstream); break; From d20229d9b9b307d85247a9fc0c46e320b8e4f831 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 3 Sep 2015 00:43:42 +0900 Subject: [PATCH 10/55] nghttpx: Handle block is not passed --- src/shrpx_mruby_module.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/shrpx_mruby_module.cc b/src/shrpx_mruby_module.cc index c5e66056..2895061d 100644 --- a/src/shrpx_mruby_module.cc +++ b/src/shrpx_mruby_module.cc @@ -44,6 +44,10 @@ mrb_value run(mrb_state *mrb, mrb_value self) { mrb_value b; mrb_get_args(mrb, "&", &b); + if (mrb_nil_p(b)) { + return mrb_nil_value(); + } + auto module = mrb_module_get(mrb, "Nghttpx"); auto request_class = mrb_class_get_under(mrb, module, "Request"); auto response_class = mrb_class_get_under(mrb, module, "Response"); @@ -57,7 +61,7 @@ mrb_value run(mrb_state *mrb, mrb_value self) { void init_module(mrb_state *mrb) { auto module = mrb_define_module(mrb, "Nghttpx"); - mrb_define_class_method(mrb, module, "run", run, MRB_ARGS_BLOCK()); + mrb_define_class_method(mrb, module, "run", run, MRB_ARGS_NONE()); init_request_class(mrb, module); init_response_class(mrb, module); From baadec5ef4eafb30dd7f42239bb096abc2dc12b5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 3 Sep 2015 01:32:15 +0900 Subject: [PATCH 11/55] nghttpx: Add response mruby hook --- gennghttpxfun.py | 1 + src/shrpx.cc | 4 +++ src/shrpx_config.cc | 10 +++++++ src/shrpx_config.h | 1 + src/shrpx_error.h | 1 + src/shrpx_http2_session.cc | 12 ++++++--- src/shrpx_http2_upstream.cc | 16 +++++++++++ src/shrpx_http_downstream_connection.cc | 6 +++++ src/shrpx_https_upstream.cc | 19 ++++++++++++++ src/shrpx_mruby.cc | 35 +++++++++++-------------- src/shrpx_mruby.h | 3 +++ src/shrpx_spdy_upstream.cc | 14 ++++++++++ 12 files changed, 100 insertions(+), 22 deletions(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 3b1a3ae0..74732b98 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -99,6 +99,7 @@ OPTIONS = [ "tls-ticket-key-memcached-max-retry", "tls-ticket-key-memcached-max-fail", "on-request-mruby-file", + "on-response-mruby-file", "conf", ] diff --git a/src/shrpx.cc b/src/shrpx.cc index 51c2cf9d..54132072 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1904,6 +1904,7 @@ int main(int argc, char **argv) { {SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_MAX_FAIL, required_argument, &flag, 90}, {SHRPX_OPT_ON_REQUEST_MRUBY_FILE, required_argument, &flag, 91}, + {SHRPX_OPT_ON_RESPONSE_MRUBY_FILE, required_argument, &flag, 92}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2305,6 +2306,9 @@ int main(int argc, char **argv) { // --on-request-mruby-file cmdcfgs.emplace_back(SHRPX_OPT_ON_REQUEST_MRUBY_FILE, optarg); break; + case 92: + // --on-response-mruby-file + cmdcfgs.emplace_back(SHRPX_OPT_ON_RESPONSE_MRUBY_FILE, optarg); default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 1d4b480c..a923575a 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -691,6 +691,7 @@ enum { SHRPX_OPTID_NPN_LIST, SHRPX_OPTID_OCSP_UPDATE_INTERVAL, SHRPX_OPTID_ON_REQUEST_MRUBY_FILE, + SHRPX_OPTID_ON_RESPONSE_MRUBY_FILE, SHRPX_OPTID_PADDING, SHRPX_OPTID_PID_FILE, SHRPX_OPTID_PRIVATE_KEY_FILE, @@ -1112,6 +1113,11 @@ int option_lookup_token(const char *name, size_t namelen) { break; case 22: switch (name[21]) { + case 'e': + if (util::strieq_l("on-response-mruby-fil", name, 21)) { + return SHRPX_OPTID_ON_RESPONSE_MRUBY_FILE; + } + break; case 'i': if (util::strieq_l("backend-http-proxy-ur", name, 21)) { return SHRPX_OPTID_BACKEND_HTTP_PROXY_URI; @@ -1947,6 +1953,10 @@ int parse_config(const char *opt, const char *optarg, case SHRPX_OPTID_ON_REQUEST_MRUBY_FILE: mod_config()->on_request_mruby_file = strcopy(optarg); + return 0; + case SHRPX_OPTID_ON_RESPONSE_MRUBY_FILE: + mod_config()->on_response_mruby_file = strcopy(optarg); + return 0; case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index d7b59c84..427062b5 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -184,6 +184,7 @@ constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_MAX_RETRY[] = constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_MAX_FAIL[] = "tls-ticket-key-memcached-max-fail"; constexpr char SHRPX_OPT_ON_REQUEST_MRUBY_FILE[] = "on-request-mruby-file"; +constexpr char SHRPX_OPT_ON_RESPONSE_MRUBY_FILE[] = "on-response-mruby-file"; union sockaddr_union { sockaddr_storage storage; diff --git a/src/shrpx_error.h b/src/shrpx_error.h index f1d1bffa..5cb8490c 100644 --- a/src/shrpx_error.h +++ b/src/shrpx_error.h @@ -36,6 +36,7 @@ enum ErrorCode { SHRPX_ERR_NETWORK = -100, SHRPX_ERR_EOF = -101, SHRPX_ERR_INPROGRESS = -102, + SHRPX_ERR_DCONN_CANCELED = -103, }; } // namespace shrpx diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc index ad13531b..6c8bb8b5 100644 --- a/src/shrpx_http2_session.cc +++ b/src/shrpx_http2_session.cc @@ -903,9 +903,15 @@ int on_response_headers(Http2Session *http2session, Downstream *downstream, rv = upstream->on_downstream_header_complete(downstream); if (rv != 0) { - http2session->submit_rst_stream(frame->hd.stream_id, - NGHTTP2_PROTOCOL_ERROR); - downstream->set_response_state(Downstream::MSG_RESET); + // Handling early return (in other words, response was hijacked by + // mruby scripting). + if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { + http2session->submit_rst_stream(frame->hd.stream_id, NGHTTP2_CANCEL); + } else { + http2session->submit_rst_stream(frame->hd.stream_id, + NGHTTP2_INTERNAL_ERROR); + downstream->set_response_state(Downstream::MSG_RESET); + } } return 0; diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 01f6d380..1bebfc08 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1209,6 +1209,22 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { downstream->get_request_http2_scheme()); } + if (!downstream->get_non_final_response()) { + auto worker = handler_->get_worker(); + auto mruby_ctx = worker->get_mruby_context(); + + if (mruby_ctx->run_on_response_proc(downstream) != 0) { + if (error_reply(downstream, 500) != 0) { + return -1; + } + return -1; + } + + if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { + return -1; + } + } + size_t nheader = downstream->get_response_headers().size(); auto nva = std::vector(); // 3 means :status and possible server and via header field. diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 43d1a300..d27b6cd0 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -774,6 +774,12 @@ int HttpDownstreamConnection::on_read() { auto htperr = HTTP_PARSER_ERRNO(&response_htp_); if (htperr != HPE_OK) { + // Handling early return (in other words, response was hijacked + // by mruby scripting). + if (downstream_->get_response_state() == Downstream::MSG_COMPLETE) { + return SHRPX_ERR_DCONN_CANCELED; + } + if (LOG_ENABLED(INFO)) { DCLOG(INFO, this) << "HTTP parser failure: " << "(" << http_errno_name(htperr) << ") " diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 98ceb3e8..4faab808 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -611,6 +611,11 @@ int HttpsUpstream::downstream_read(DownstreamConnection *dconn) { return downstream_eof(dconn); } + if (rv == SHRPX_ERR_DCONN_CANCELED) { + downstream->pop_downstream_connection(); + goto end; + } + if (rv < 0) { return downstream_error(dconn, Downstream::EVENT_ERROR); } @@ -782,6 +787,20 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) { } } + if (!downstream->get_non_final_response()) { + auto worker = handler_->get_worker(); + auto mruby_ctx = worker->get_mruby_context(); + + if (mruby_ctx->run_on_response_proc(downstream) != 0) { + error_reply(500); + return -1; + } + + if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { + return -1; + } + } + auto connect_method = downstream->get_request_method() == HTTP_CONNECT; std::string hdrs = "HTTP/"; diff --git a/src/shrpx_mruby.cc b/src/shrpx_mruby.cc index 52ba2aeb..08d3bdf4 100644 --- a/src/shrpx_mruby.cc +++ b/src/shrpx_mruby.cc @@ -40,39 +40,40 @@ namespace mruby { MRubyContext::MRubyContext(mrb_state *mrb, RProc *on_request_proc, RProc *on_response_proc) : mrb_(mrb), on_request_proc_(on_request_proc), - on_response_proc_(on_response_proc) {} + on_response_proc_(on_response_proc), running_(false) {} MRubyContext::~MRubyContext() { mrb_close(mrb_); } -namespace { -int run_request_proc(mrb_state *mrb, Downstream *downstream, RProc *proc) { - if (!proc) { +int MRubyContext::run_request_proc(Downstream *downstream, RProc *proc) { + if (!proc || running_) { return 0; } + running_ = true; + MRubyAssocData data{downstream}; - mrb->ud = &data; + mrb_->ud = &data; int rv = 0; - auto ai = mrb_gc_arena_save(mrb); + auto ai = mrb_gc_arena_save(mrb_); - auto res = mrb_run(mrb, proc, mrb_top_self(mrb)); + auto res = mrb_run(mrb_, proc, mrb_top_self(mrb_)); (void)res; - if (mrb->exc) { + if (mrb_->exc) { rv = -1; auto error = - mrb_str_ptr(mrb_funcall(mrb, mrb_obj_value(mrb->exc), "inspect", 0)); + mrb_str_ptr(mrb_funcall(mrb_, mrb_obj_value(mrb_->exc), "inspect", 0)); LOG(ERROR) << "Exception caught while executing mruby code: " << error->as.heap.ptr; - mrb->exc = 0; + mrb_->exc = 0; } - mrb->ud = nullptr; + mrb_->ud = nullptr; - mrb_gc_arena_restore(mrb, ai); + mrb_gc_arena_restore(mrb_, ai); if (data.request_headers_dirty) { downstream->index_request_headers(); @@ -82,21 +83,17 @@ int run_request_proc(mrb_state *mrb, Downstream *downstream, RProc *proc) { downstream->index_response_headers(); } - if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { - downstream->pop_downstream_connection(); - } + running_ = false; return rv; } -} // namespace int MRubyContext::run_on_request_proc(Downstream *downstream) { - return run_request_proc(mrb_, downstream, on_request_proc_); + return run_request_proc(downstream, on_request_proc_); } int MRubyContext::run_on_response_proc(Downstream *downstream) { - // TODO not implemented yet - return 0; + return run_request_proc(downstream, on_response_proc_); } // Based on diff --git a/src/shrpx_mruby.h b/src/shrpx_mruby.h index 6f75db3b..2588cfb9 100644 --- a/src/shrpx_mruby.h +++ b/src/shrpx_mruby.h @@ -46,10 +46,13 @@ public: int run_on_request_proc(Downstream *downstream); int run_on_response_proc(Downstream *downstream); + int run_request_proc(Downstream *downstream, RProc *proc); + private: mrb_state *mrb_; RProc *on_request_proc_; RProc *on_response_proc_; + bool running_; }; struct MRubyAssocData { diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 5056153e..bc61b92a 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -865,6 +865,20 @@ int SpdyUpstream::on_downstream_header_complete(Downstream *downstream) { return 0; } + auto worker = handler_->get_worker(); + auto mruby_ctx = worker->get_mruby_context(); + + if (mruby_ctx->run_on_response_proc(downstream) != 0) { + if (error_reply(downstream, 500) != 0) { + return -1; + } + return -1; + } + + if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { + return -1; + } + if (LOG_ENABLED(INFO)) { DLOG(INFO, downstream) << "HTTP response header completed"; } From 14f5240d8f9ff124199bb29319428ede6482b4c9 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 3 Sep 2015 01:37:47 +0900 Subject: [PATCH 12/55] nghttpx: Update doc --- src/shrpx.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/shrpx.cc b/src/shrpx.cc index 54132072..4b5b2dc8 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1730,6 +1730,16 @@ Process: Run this program as . This option is intended to be used to drop root privileges. +Scripting: + --on-request-mruby-file= + Set mruby script file which will be executed when + request header fields are completely received from + frontend. + --on-response-mruby-file= + Set mruby script file which will be executed when + response header fields are completely received from + backend. + Misc: --conf= Load configuration from . From 05a761b628011a1afad32b701983c25c1897c584 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 3 Sep 2015 02:02:28 +0900 Subject: [PATCH 13/55] nghttpx: Add Request.remote_addr which returns client remote address --- src/shrpx_mruby_module_request.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index 98ede3bd..8d7442b1 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -30,6 +30,8 @@ #include #include "shrpx_downstream.h" +#include "shrpx_upstream.h" +#include "shrpx_client_handler.h" #include "shrpx_mruby.h" #include "shrpx_mruby_module.h" #include "util.h" @@ -226,6 +228,19 @@ mrb_value request_set_header(mrb_state *mrb, mrb_value self) { } } // namespace +namespace { +mrb_value request_get_remote_addr(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + auto upstream = downstream->get_upstream(); + auto handler = upstream->get_client_handler(); + + auto &ipaddr = handler->get_ipaddr(); + + return mrb_str_new(mrb, ipaddr.c_str(), ipaddr.size()); +} +} // namespace + void init_request_class(mrb_state *mrb, RClass *module) { auto request_class = mrb_define_class_under(mrb, module, "Request", mrb->object_class); @@ -256,6 +271,8 @@ void init_request_class(mrb_state *mrb, RClass *module) { MRB_ARGS_NONE()); mrb_define_method(mrb, request_class, "set_header", request_set_header, MRB_ARGS_REQ(2)); + mrb_define_method(mrb, request_class, "remote_addr", request_get_remote_addr, + MRB_ARGS_NONE()); } } // namespace mruby From 65f2b16132c446f6d47ef513c0b299398c85aed8 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 3 Sep 2015 22:29:16 +0900 Subject: [PATCH 14/55] nghttpx: More freedom for custom response headers --- src/shrpx_http2_upstream.cc | 55 ++++++++++++++++++++++++++ src/shrpx_http2_upstream.h | 2 + src/shrpx_https_upstream.cc | 56 ++++++++++++++++++++++++++ src/shrpx_https_upstream.h | 2 + src/shrpx_mruby.cc | 5 ++- src/shrpx_mruby_module_response.cc | 27 ++++++------- src/shrpx_spdy_upstream.cc | 63 ++++++++++++++++++++++++++++++ src/shrpx_spdy_upstream.h | 3 ++ src/shrpx_upstream.h | 2 + 9 files changed, 200 insertions(+), 15 deletions(-) diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 1bebfc08..ee603e13 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1138,6 +1138,61 @@ ssize_t downstream_data_read_callback(nghttp2_session *session, } } // namespace +int Http2Upstream::send_reply(Downstream *downstream, const uint8_t *body, + size_t bodylen) { + int rv; + + nghttp2_data_provider data_prd, *data_prd_ptr = nullptr; + + if (bodylen) { + data_prd.source.ptr = downstream; + data_prd.read_callback = downstream_data_read_callback; + data_prd_ptr = &data_prd; + } + + auto status_code_str = util::utos(downstream->get_response_http_status()); + auto &headers = downstream->get_response_headers(); + // 2 for :status and server + auto nva = std::vector(2 + headers.size()); + nva.push_back(http2::make_nv_ls(":status", status_code_str)); + + for (auto &kv : headers) { + if (kv.name.empty() || kv.name[0] == ':') { + continue; + } + switch (kv.token) { + case http2::HD_CONNECTION: + case http2::HD_KEEP_ALIVE: + case http2::HD_PROXY_CONNECTION: + case http2::HD_TE: + case http2::HD_TRANSFER_ENCODING: + case http2::HD_UPGRADE: + continue; + } + nva.push_back(http2::make_nv(kv.name, kv.value, kv.no_index)); + } + + if (!downstream->get_response_header(http2::HD_SERVER)) { + nva.push_back(http2::make_nv_lc("server", get_config()->server_name)); + } + + rv = nghttp2_submit_response(session_, downstream->get_stream_id(), + nva.data(), nva.size(), data_prd_ptr); + if (nghttp2_is_fatal(rv)) { + ULOG(FATAL, this) << "nghttp2_submit_response() failed: " + << nghttp2_strerror(rv); + return -1; + } + + auto buf = downstream->get_response_buf(); + + buf->append(body, bodylen); + + downstream->set_response_state(Downstream::MSG_COMPLETE); + + return 0; +} + int Http2Upstream::error_reply(Downstream *downstream, unsigned int status_code) { int rv; diff --git a/src/shrpx_http2_upstream.h b/src/shrpx_http2_upstream.h index ef8c0bee..944539a0 100644 --- a/src/shrpx_http2_upstream.h +++ b/src/shrpx_http2_upstream.h @@ -78,6 +78,8 @@ public: virtual void on_handler_delete(); virtual int on_downstream_reset(bool no_retry); + virtual int send_reply(Downstream *downstream, const uint8_t *body, + size_t bodylen); bool get_flow_control() const; // Perform HTTP/2 upgrade from |upstream|. On success, this object diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 4faab808..346dda48 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -725,6 +725,62 @@ int HttpsUpstream::downstream_error(DownstreamConnection *dconn, int events) { return 0; } +int HttpsUpstream::send_reply(Downstream *downstream, const uint8_t *body, + size_t bodylen) { + auto major = downstream->get_request_major(); + auto minor = downstream->get_request_minor(); + + auto connection_close = false; + if (major <= 0 || (major == 1 && minor == 0)) { + connection_close = true; + } else { + auto c = downstream->get_response_header(http2::HD_CONNECTION); + if (c && util::strieq_l("close", c->value)) { + connection_close = true; + } + } + + if (connection_close) { + downstream->set_response_connection_close(true); + handler_->set_should_close_after_write(true); + } + + auto output = downstream->get_response_buf(); + + output->append("HTTP/1.1 "); + auto status_str = + http2::get_status_string(downstream->get_response_http_status()); + output->append(status_str.c_str(), status_str.size()); + + for (auto &kv : downstream->get_response_headers()) { + if (kv.name.empty() || kv.name[0] == ':') { + continue; + } + auto name = kv.name; + http2::capitalize(name, 0); + output->append(name.c_str(), name.size()); + output->append(": "); + output->append(kv.value.c_str(), kv.value.size()); + output->append("\r\n"); + } + + if (!downstream->get_response_header(http2::HD_SERVER)) { + output->append("Server: "); + output->append(get_config()->server_name, + strlen(get_config()->server_name)); + output->append("\r\n"); + } + + output->append("\r\n"); + + output->append(body, bodylen); + + downstream->add_response_sent_bodylen(bodylen); + downstream->set_response_state(Downstream::MSG_COMPLETE); + + return 0; +} + void HttpsUpstream::error_reply(unsigned int status_code) { auto html = http::create_error_html(status_code); auto downstream = get_downstream(); diff --git a/src/shrpx_https_upstream.h b/src/shrpx_https_upstream.h index 3b9d5320..f588d4bc 100644 --- a/src/shrpx_https_upstream.h +++ b/src/shrpx_https_upstream.h @@ -74,6 +74,8 @@ public: virtual void on_handler_delete(); virtual int on_downstream_reset(bool no_retry); + virtual int send_reply(Downstream *downstream, const uint8_t *body, + size_t bodylen); void reset_current_header_length(); void log_response_headers(const std::string &hdrs) const; diff --git a/src/shrpx_mruby.cc b/src/shrpx_mruby.cc index 08d3bdf4..50a5ebb0 100644 --- a/src/shrpx_mruby.cc +++ b/src/shrpx_mruby.cc @@ -62,7 +62,10 @@ int MRubyContext::run_request_proc(Downstream *downstream, RProc *proc) { (void)res; if (mrb_->exc) { - rv = -1; + // If response has been committed, ignore error + if (downstream->get_response_state() != Downstream::MSG_COMPLETE) { + rv = -1; + } auto error = mrb_str_ptr(mrb_funcall(mrb_, mrb_obj_value(mrb_->exc), "inspect", 0)); diff --git a/src/shrpx_mruby_module_response.cc b/src/shrpx_mruby_module_response.cc index 5e0a2d80..5398ccfb 100644 --- a/src/shrpx_mruby_module_response.cc +++ b/src/shrpx_mruby_module_response.cc @@ -31,6 +31,7 @@ #include "shrpx_downstream.h" #include "shrpx_upstream.h" +#include "shrpx_client_handler.h" #include "shrpx_mruby.h" #include "shrpx_mruby_module.h" #include "util.h" @@ -139,7 +140,7 @@ mrb_value response_set_header(mrb_state *mrb, mrb_value self) { } // namespace namespace { -mrb_value response_end(mrb_state *mrb, mrb_value self) { +mrb_value response_return(mrb_state *mrb, mrb_value self) { auto data = static_cast(mrb->ud); auto downstream = data->downstream; int rv; @@ -151,14 +152,9 @@ mrb_value response_end(mrb_state *mrb, mrb_value self) { mrb_value val; mrb_get_args(mrb, "|o", &val); - const char *body = nullptr; + const uint8_t *body = nullptr; size_t bodylen = 0; - if (!mrb_nil_p(val)) { - body = RSTRING_PTR(val); - bodylen = RSTRING_LEN(val); - } - if (downstream->get_response_http_status() == 0) { downstream->set_response_http_status(200); } @@ -168,6 +164,11 @@ mrb_value response_end(mrb_state *mrb, mrb_value self) { data->response_headers_dirty = false; } + if (downstream->expect_response_body() && !mrb_nil_p(val)) { + body = reinterpret_cast(RSTRING_PTR(val)); + bodylen = RSTRING_LEN(val); + } + auto cl = downstream->get_response_header(http2::HD_CONTENT_LENGTH); if (cl) { cl->value = util::utos(bodylen); @@ -179,17 +180,14 @@ mrb_value response_end(mrb_state *mrb, mrb_value self) { auto upstream = downstream->get_upstream(); - rv = upstream->on_downstream_header_complete(downstream); + rv = upstream->send_reply(downstream, body, bodylen); if (rv != 0) { mrb_raise(mrb, E_RUNTIME_ERROR, "could not send response"); } - if (downstream->expect_response_body()) { - auto output = downstream->get_response_buf(); - output->append(body, bodylen); - } + auto handler = upstream->get_client_handler(); - downstream->set_response_state(Downstream::MSG_COMPLETE); + handler->signal_write(); return self; } @@ -213,7 +211,8 @@ void init_response_class(mrb_state *mrb, RClass *module) { MRB_ARGS_NONE()); mrb_define_method(mrb, response_class, "set_header", response_set_header, MRB_ARGS_REQ(2)); - mrb_define_method(mrb, response_class, "end", response_end, MRB_ARGS_OPT(1)); + mrb_define_method(mrb, response_class, "return", response_return, + MRB_ARGS_OPT(1)); } } // namespace mruby diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index bc61b92a..9a31e033 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -793,6 +793,69 @@ ssize_t spdy_data_read_callback(spdylay_session *session, int32_t stream_id, } } // namespace +int SpdyUpstream::send_reply(Downstream *downstream, const uint8_t *body, + size_t bodylen) { + int rv; + + spdylay_data_provider data_prd, *data_prd_ptr = nullptr; + if (bodylen) { + data_prd.source.ptr = downstream; + data_prd.read_callback = spdy_data_read_callback; + data_prd_ptr = &data_prd; + } + + auto status_string = + http2::get_status_string(downstream->get_response_http_status()); + + auto &headers = downstream->get_response_headers(); + + // 3 for :status, :version and server + auto nva = std::vector(3 + headers.size()); + + nva.push_back(":status"); + nva.push_back(status_string.c_str()); + nva.push_back(":version"); + nva.push_back("HTTP/1.1"); + + for (auto &kv : headers) { + if (kv.name.empty() || kv.name[0] == ':') { + continue; + } + switch (kv.token) { + case http2::HD_CONNECTION: + case http2::HD_KEEP_ALIVE: + case http2::HD_PROXY_CONNECTION: + case http2::HD_TRANSFER_ENCODING: + continue; + } + nva.push_back(kv.name.c_str()); + nva.push_back(kv.value.c_str()); + } + + if (!downstream->get_response_header(http2::HD_SERVER)) { + nva.push_back("server"); + nva.push_back(get_config()->server_name); + } + + nva.push_back(nullptr); + + rv = spdylay_submit_response(session_, downstream->get_stream_id(), + nva.data(), data_prd_ptr); + if (rv < SPDYLAY_ERR_FATAL) { + ULOG(FATAL, this) << "spdylay_submit_response() failed: " + << spdylay_strerror(rv); + return -1; + } + + auto buf = downstream->get_response_buf(); + + buf->append(body, bodylen); + + downstream->set_response_state(Downstream::MSG_COMPLETE); + + return 0; +} + int SpdyUpstream::error_reply(Downstream *downstream, unsigned int status_code) { int rv; diff --git a/src/shrpx_spdy_upstream.h b/src/shrpx_spdy_upstream.h index 2e4aec47..c3ff1766 100644 --- a/src/shrpx_spdy_upstream.h +++ b/src/shrpx_spdy_upstream.h @@ -74,6 +74,9 @@ public: virtual void on_handler_delete(); virtual int on_downstream_reset(bool no_retry); + virtual int send_reply(Downstream *downstream, const uint8_t *body, + size_t bodylen); + bool get_flow_control() const; int consume(int32_t stream_id, size_t len); diff --git a/src/shrpx_upstream.h b/src/shrpx_upstream.h index fe9a673d..d2ec80a7 100644 --- a/src/shrpx_upstream.h +++ b/src/shrpx_upstream.h @@ -62,6 +62,8 @@ public: virtual void pause_read(IOCtrlReason reason) = 0; virtual int resume_read(IOCtrlReason reason, Downstream *downstream, size_t consumed) = 0; + virtual int send_reply(Downstream *downstream, const uint8_t *body, + size_t bodylen) = 0; }; } // namespace shrpx From 6f9c9f85188714f70f8a9daefe1c4dd2c8fcbb57 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 3 Sep 2015 22:41:51 +0900 Subject: [PATCH 15/55] nghttpx: Fix assertion failure when response was commited early --- src/shrpx_https_upstream.cc | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 346dda48..e5325c07 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -366,6 +366,17 @@ int htp_msg_completecb(http_parser *htp) { downstream->set_request_state(Downstream::MSG_COMPLETE); rv = downstream->end_upload_data(); if (rv != 0) { + if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { + // Here both response and request were completed. One of the + // reason why end_upload_data() failed is when we sent response + // in request phase hook. We only delete and proceed to the + // next request handling (if we don't close the connection). We + // first pause parser here jsut as we normally do, and call + // signal_write() to run on_write(). + http_parser_pause(htp, 1); + + return 0; + } return -1; } @@ -465,6 +476,13 @@ int HttpsUpstream::on_read() { auto htperr = HTTP_PARSER_ERRNO(&htp_); if (htperr == HPE_PAUSED) { + // We may pause parser in htp_msg_completecb when both side are + // completed. Signal write, so that we can run on_write(). + if (downstream && + downstream->get_request_state() == Downstream::MSG_COMPLETE && + downstream->get_response_state() == Downstream::MSG_COMPLETE) { + handler_->signal_write(); + } return 0; } @@ -569,6 +587,11 @@ int HttpsUpstream::on_write() { // We need this if response ends before request. if (downstream->get_request_state() == Downstream::MSG_COMPLETE) { delete_downstream(); + + if (handler_->get_should_close_after_write()) { + return 0; + } + return resume_read(SHRPX_NO_BUFFER, nullptr, 0); } } From 5e3ab6da5a00157f3fc0089113f67c4df925a4b9 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 3 Sep 2015 22:44:41 +0900 Subject: [PATCH 16/55] nghttpx: Perhaps this looks right, but not sure --- src/shrpx_mruby_module.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/shrpx_mruby_module.cc b/src/shrpx_mruby_module.cc index 2895061d..a122ace5 100644 --- a/src/shrpx_mruby_module.cc +++ b/src/shrpx_mruby_module.cc @@ -61,7 +61,8 @@ mrb_value run(mrb_state *mrb, mrb_value self) { void init_module(mrb_state *mrb) { auto module = mrb_define_module(mrb, "Nghttpx"); - mrb_define_class_method(mrb, module, "run", run, MRB_ARGS_NONE()); + mrb_define_class_method(mrb, module, "run", run, + MRB_ARGS_REQ(1) | MRB_ARGS_BLOCK()); init_request_class(mrb, module); init_response_class(mrb, module); From 3235b88253d552c77f0f1badc2f5b3024feeef60 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 3 Sep 2015 22:51:24 +0900 Subject: [PATCH 17/55] nghttpx: Fix assertion error because of empty scheme --- src/shrpx_https_upstream.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index e5325c07..a7a45ed2 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -199,6 +199,10 @@ void rewrite_request_host_path_from_uri(Downstream *downstream, const char *uri, } downstream->set_request_http2_authority(authority); + std::string scheme; + http2::copy_url_component(scheme, &u, UF_SCHEMA, uri); + downstream->set_request_http2_scheme(std::move(scheme)); + std::string path; if (u.field_set & (1 << UF_PATH)) { http2::copy_url_component(path, &u, UF_PATH, uri); @@ -226,10 +230,6 @@ void rewrite_request_host_path_from_uri(Downstream *downstream, const char *uri, downstream->set_request_path( http2::rewrite_clean_path(std::begin(path), std::end(path))); } - - std::string scheme; - http2::copy_url_component(scheme, &u, UF_SCHEMA, uri); - downstream->set_request_http2_scheme(std::move(scheme)); } } // namespace From 02bb2c3e835880253078ed5a2014408bdd7e9993 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Thu, 3 Sep 2015 23:36:49 +0900 Subject: [PATCH 18/55] nghttpx: Create authority from host or authority-form for CONNECT request --- src/shrpx_downstream.cc | 4 ++ src/shrpx_downstream.h | 10 ++-- src/shrpx_http2_downstream_connection.cc | 60 ++++++------------------ src/shrpx_http2_upstream.cc | 5 ++ src/shrpx_http_downstream_connection.cc | 58 ++++++----------------- src/shrpx_https_upstream.cc | 24 ++++++++-- 6 files changed, 63 insertions(+), 98 deletions(-) diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 9c94a2de..5948b1db 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -515,6 +515,10 @@ void Downstream::set_request_http2_authority(std::string authority) { request_http2_authority_ = std::move(authority); } +void Downstream::append_request_http2_authority(const char *data, size_t len) { + request_http2_authority_.append(data, len); +} + void Downstream::set_request_major(int major) { request_major_ = major; } void Downstream::set_request_minor(int minor) { request_minor_ = minor; } diff --git a/src/shrpx_downstream.h b/src/shrpx_downstream.h index 860934c5..f90e80ce 100644 --- a/src/shrpx_downstream.h +++ b/src/shrpx_downstream.h @@ -150,15 +150,19 @@ public: get_request_start_time() const; void append_request_path(const char *data, size_t len); // Returns request path. For HTTP/1.1, this is request-target. For - // HTTP/2, this is :path header field value. + // HTTP/2, this is :path header field value. For CONNECT request, + // this is empty. const std::string &get_request_path() const; // Returns HTTP/2 :scheme header field value. const std::string &get_request_http2_scheme() const; void set_request_http2_scheme(std::string scheme); - // Returns HTTP/2 :authority header field value. We also set the - // value retrieved from absolute-form HTTP/1 request. + // Returns :authority or host header field value. We may deduce it + // from absolute-form HTTP/1 request. We also store authority-form + // HTTP/1 request. This could be empty if request comes from + // HTTP/1.0 without Host header field and origin-form. const std::string &get_request_http2_authority() const; void set_request_http2_authority(std::string authority); + void append_request_http2_authority(const char *data, size_t len); void set_request_major(int major); void set_request_minor(int minor); int get_request_major() const; diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index f4377b44..7e7d9f20 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -265,37 +265,21 @@ int Http2DownstreamConnection::push_request_headers() { .addrs[addr_idx] .hostport.get(); - const char *authority = nullptr, *host = nullptr; - if (!no_host_rewrite) { - if (!downstream_->get_request_http2_authority().empty()) { - authority = downstream_hostport; - } - if (downstream_->get_request_header(http2::HD_HOST)) { - host = downstream_hostport; - } - } else { - if (!downstream_->get_request_http2_authority().empty()) { - authority = downstream_->get_request_http2_authority().c_str(); - } - auto h = downstream_->get_request_header(http2::HD_HOST); - if (h) { - host = h->value.c_str(); - } + // For HTTP/1.0 request, there is no authority in request. In that + // case, we use backend server's host nonetheless. + const char *authority = downstream_hostport; + auto &req_authority = downstream_->get_request_http2_authority(); + if (no_host_rewrite && !req_authority.empty()) { + authority = req_authority.c_str(); } - if (!authority && !host) { - // upstream is HTTP/1.0. We use backend server's host - // nonetheless. - host = downstream_hostport; + if (!authority) { + authority = downstream_hostport; } - if (authority) { - downstream_->set_request_downstream_host(authority); - } else { - downstream_->set_request_downstream_host(host); - } + downstream_->set_request_downstream_host(authority); - size_t nheader = downstream_->get_request_headers().size(); + auto nheader = downstream_->get_request_headers().size(); Headers cookies; if (!get_config()->http2_no_cookie_crumbling) { @@ -306,7 +290,7 @@ int Http2DownstreamConnection::push_request_headers() { // 1. :method // 2. :scheme // 3. :path - // 4. :authority or host (at least either of them exists) + // 4. :authority // 5. via (optional) // 6. x-forwarded-for (optional) // 7. x-forwarded-proto (optional) @@ -320,30 +304,14 @@ int Http2DownstreamConnection::push_request_headers() { auto &scheme = downstream_->get_request_http2_scheme(); - if (downstream_->get_request_method() == HTTP_CONNECT) { - if (authority) { - nva.push_back(http2::make_nv_lc(":authority", authority)); - } else { - nva.push_back( - http2::make_nv_ls(":authority", downstream_->get_request_path())); - } - } else { + nva.push_back(http2::make_nv_lc(":authority", authority)); + + if (downstream_->get_request_method() != HTTP_CONNECT) { assert(!scheme.empty()); nva.push_back(http2::make_nv_ls(":scheme", scheme)); - - if (authority) { - nva.push_back(http2::make_nv_lc(":authority", authority)); - } - nva.push_back(http2::make_nv_ls(":path", downstream_->get_request_path())); } - // only emit host header field if :authority is not emitted. They - // both must be the same value. - if (!authority && host) { - nva.push_back(http2::make_nv_lc("host", host)); - } - http2::copy_headers_to_nva(nva, downstream_->get_request_headers()); bool chunked_encoding = false; diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index ee603e13..245208e0 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -292,7 +292,12 @@ int Http2Upstream::on_request_headers(Downstream *downstream, downstream->set_request_method(method_token); downstream->set_request_http2_scheme(http2::value_to_str(scheme)); + // nghttp2 library guarantees either :authority or host exist + if (!authority) { + authority = downstream->get_request_header(http2::HD_HOST); + } downstream->set_request_http2_authority(http2::value_to_str(authority)); + if (path) { if (get_config()->http2_proxy || get_config()->client_proxy) { downstream->set_request_path(http2::value_to_str(path)); diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index d27b6cd0..b039d4fa 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -209,42 +209,25 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) { } int HttpDownstreamConnection::push_request_headers() { - const char *authority = nullptr, *host = nullptr; auto downstream_hostport = get_config() ->downstream_addr_groups[group_] .addrs[addr_idx_] .hostport.get(); auto connect_method = downstream_->get_request_method() == HTTP_CONNECT; - if (!get_config()->no_host_rewrite && !get_config()->http2_proxy && - !get_config()->client_proxy && !connect_method) { - if (!downstream_->get_request_http2_authority().empty()) { - authority = downstream_hostport; - } - if (downstream_->get_request_header(http2::HD_HOST)) { - host = downstream_hostport; - } - } else { - if (!downstream_->get_request_http2_authority().empty()) { - authority = downstream_->get_request_http2_authority().c_str(); - } - auto h = downstream_->get_request_header(http2::HD_HOST); - if (h) { - host = h->value.c_str(); - } + // For HTTP/1.0 request, there is no authority in request. In that + // case, we use backend server's host nonetheless. + const char *authority = downstream_hostport; + auto &req_authority = downstream_->get_request_http2_authority(); + auto no_host_rewrite = get_config()->no_host_rewrite || + get_config()->http2_proxy || + get_config()->client_proxy || connect_method; + + if (no_host_rewrite && !req_authority.empty()) { + authority = req_authority.c_str(); } - if (!authority && !host) { - // upstream is HTTP/1.0. We use backend server's host - // nonetheless. - host = downstream_hostport; - } - - if (authority) { - downstream_->set_request_downstream_host(authority); - } else { - downstream_->set_request_downstream_host(host); - } + downstream_->set_request_downstream_host(authority); downstream_->assemble_request_cookie(); @@ -255,23 +238,14 @@ int HttpDownstreamConnection::push_request_headers() { auto &scheme = downstream_->get_request_http2_scheme(); if (connect_method) { - if (authority) { - hdrs += authority; - } else { - hdrs += downstream_->get_request_path(); - } + hdrs += authority; } else if (get_config()->http2_proxy || get_config()->client_proxy) { // Construct absolute-form request target because we are going to // send a request to a HTTP/1 proxy. assert(!scheme.empty()); hdrs += scheme; hdrs += "://"; - - if (authority) { - hdrs += authority; - } else { - hdrs += host; - } + hdrs += authority; // Server-wide OPTIONS takes following form in proxy request: // @@ -287,11 +261,7 @@ int HttpDownstreamConnection::push_request_headers() { hdrs += downstream_->get_request_path(); } hdrs += " HTTP/1.1\r\nHost: "; - if (authority) { - hdrs += authority; - } else { - hdrs += host; - } + hdrs += authority; hdrs += "\r\n"; http2::build_http1_headers_from_headers(hdrs, diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index a7a45ed2..d6ab64e1 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -70,8 +70,14 @@ int htp_msg_begin(http_parser *htp) { auto handler = upstream->get_client_handler(); // TODO specify 0 as priority for now - upstream->attach_downstream( - make_unique(upstream, handler->get_mcpool(), 0, 0)); + auto downstream = + make_unique(upstream, handler->get_mcpool(), 0, 0); + + // We happen to have the same value for method token. + downstream->set_request_method(htp->method); + + upstream->attach_downstream(std::move(downstream)); + return 0; } } // namespace @@ -91,7 +97,12 @@ int htp_uricb(http_parser *htp, const char *data, size_t len) { return -1; } downstream->add_request_headers_sum(len); - downstream->append_request_path(data, len); + if (downstream->get_request_method() == HTTP_CONNECT) { + downstream->append_request_http2_authority(data, len); + } else { + downstream->append_request_path(data, len); + } + return 0; } } // namespace @@ -242,8 +253,6 @@ int htp_hdrs_completecb(http_parser *htp) { } auto downstream = upstream->get_downstream(); - // We happen to have the same value for method token. - downstream->set_request_method(htp->method); downstream->set_request_major(htp->http_major); downstream->set_request_minor(htp->http_minor); @@ -296,6 +305,11 @@ int htp_hdrs_completecb(http_parser *htp) { downstream->set_request_path( http2::rewrite_clean_path(std::begin(uri), std::end(uri))); + auto host = downstream->get_request_header(http2::HD_HOST); + if (host) { + downstream->set_request_http2_authority(host->value); + } + if (upstream->get_client_handler()->get_ssl()) { downstream->set_request_http2_scheme("https"); } else { From 200217d8eaec1292880b92389b808f48f7511abd Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 4 Sep 2015 00:14:09 +0900 Subject: [PATCH 19/55] nghttpx: Store empty string to path for server-wide OPTIONS request This change is required to show path attribute to mruby script. It is desirable to construct URI from parts. Just checking method and path is "*" is awkward. --- src/shrpx_client_handler.cc | 47 ++++++++---------------- src/shrpx_http2_downstream_connection.cc | 18 ++++++--- src/shrpx_http2_upstream.cc | 4 +- src/shrpx_http_downstream_connection.cc | 23 +++++------- src/shrpx_https_upstream.cc | 28 +++++++++----- src/shrpx_spdy_upstream.cc | 2 + 6 files changed, 60 insertions(+), 62 deletions(-) diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index 37293a5e..87c1659e 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -739,41 +739,23 @@ namespace { // HttpDownstreamConnection::push_request_headers(), but vastly // simplified since we only care about absolute URI. std::string construct_absolute_request_uri(Downstream *downstream) { - const char *authority = nullptr, *host = nullptr; - if (!downstream->get_request_http2_authority().empty()) { - authority = downstream->get_request_http2_authority().c_str(); - } - auto h = downstream->get_request_header(http2::HD_HOST); - if (h) { - host = h->value.c_str(); - } - if (!authority && !host) { + auto &authority = downstream->get_request_http2_authority(); + if (authority.empty()) { return downstream->get_request_path(); } std::string uri; - if (downstream->get_request_http2_scheme().empty()) { + auto &scheme = downstream->get_request_http2_scheme(); + if (scheme.empty()) { // We may have to log the request which lacks scheme (e.g., // http/1.1 with origin form). uri += "http://"; } else { - uri += downstream->get_request_http2_scheme(); + uri += scheme; uri += "://"; } - if (authority) { - uri += authority; - } else { - uri += host; - } + uri += authority; + uri += downstream->get_request_path(); - // Server-wide OPTIONS takes following form in proxy request: - // - // OPTIONS http://example.org HTTP/1.1 - // - // Notice that no slash after authority. See - // http://tools.ietf.org/html/rfc7230#section-5.3.4 - if (downstream->get_request_path() != "*") { - uri += downstream->get_request_path(); - } return uri; } } // namespace @@ -787,12 +769,15 @@ void ClientHandler::write_accesslog(Downstream *downstream) { downstream, ipaddr_.c_str(), http2::to_method_string(downstream->get_request_method()), - (downstream->get_request_method() != HTTP_CONNECT && - (get_config()->http2_proxy || get_config()->client_proxy)) - ? construct_absolute_request_uri(downstream).c_str() - : downstream->get_request_path().empty() - ? downstream->get_request_http2_authority().c_str() - : downstream->get_request_path().c_str(), + downstream->get_request_method() == HTTP_CONNECT + ? downstream->get_request_http2_authority().c_str() + : (get_config()->http2_proxy || get_config()->client_proxy) + ? construct_absolute_request_uri(downstream).c_str() + : downstream->get_request_path().empty() + ? downstream->get_request_method() == HTTP_OPTIONS + ? "*" + : "-" + : downstream->get_request_path().c_str(), alpn_.c_str(), nghttp2::ssl::get_tls_session_info(&tls_info, conn_.tls.ssl), diff --git a/src/shrpx_http2_downstream_connection.cc b/src/shrpx_http2_downstream_connection.cc index 7e7d9f20..86e200f6 100644 --- a/src/shrpx_http2_downstream_connection.cc +++ b/src/shrpx_http2_downstream_connection.cc @@ -251,10 +251,10 @@ int Http2DownstreamConnection::push_request_headers() { downstream_->set_request_pending(false); + auto method = downstream_->get_request_method(); auto no_host_rewrite = get_config()->no_host_rewrite || get_config()->http2_proxy || - get_config()->client_proxy || - downstream_->get_request_method() == HTTP_CONNECT; + get_config()->client_proxy || method == HTTP_CONNECT; // http2session_ has already in CONNECTED state, so we can get // addr_idx here. @@ -299,17 +299,23 @@ int Http2DownstreamConnection::push_request_headers() { nva.reserve(nheader + 8 + cookies.size() + get_config()->add_request_headers.size()); - nva.push_back(http2::make_nv_lc( - ":method", http2::to_method_string(downstream_->get_request_method()))); + nva.push_back(http2::make_nv_lc(":method", http2::to_method_string(method))); auto &scheme = downstream_->get_request_http2_scheme(); nva.push_back(http2::make_nv_lc(":authority", authority)); - if (downstream_->get_request_method() != HTTP_CONNECT) { + if (method != HTTP_CONNECT) { assert(!scheme.empty()); + nva.push_back(http2::make_nv_ls(":scheme", scheme)); - nva.push_back(http2::make_nv_ls(":path", downstream_->get_request_path())); + + auto &path = downstream_->get_request_path(); + if (method == HTTP_OPTIONS && path.empty()) { + nva.push_back(http2::make_nv_ll(":path", "*")); + } else { + nva.push_back(http2::make_nv_ls(":path", path)); + } } http2::copy_headers_to_nva(nva, downstream_->get_request_headers()); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 245208e0..5a0e075e 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -299,7 +299,9 @@ int Http2Upstream::on_request_headers(Downstream *downstream, downstream->set_request_http2_authority(http2::value_to_str(authority)); if (path) { - if (get_config()->http2_proxy || get_config()->client_proxy) { + if (method_token == HTTP_OPTIONS && path->value == "*") { + // Server-wide OPTIONS request. Path is empty. + } else if (get_config()->http2_proxy || get_config()->client_proxy) { downstream->set_request_path(http2::value_to_str(path)); } else { auto &value = path->value; diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index b039d4fa..8ce8904f 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -213,7 +213,8 @@ int HttpDownstreamConnection::push_request_headers() { ->downstream_addr_groups[group_] .addrs[addr_idx_] .hostport.get(); - auto connect_method = downstream_->get_request_method() == HTTP_CONNECT; + auto method = downstream_->get_request_method(); + auto connect_method = method == HTTP_CONNECT; // For HTTP/1.0 request, there is no authority in request. In that // case, we use backend server's host nonetheless. @@ -232,10 +233,11 @@ int HttpDownstreamConnection::push_request_headers() { downstream_->assemble_request_cookie(); // Assume that method and request path do not contain \r\n. - std::string hdrs = http2::to_method_string(downstream_->get_request_method()); + std::string hdrs = http2::to_method_string(method); hdrs += ' '; auto &scheme = downstream_->get_request_http2_scheme(); + auto &path = downstream_->get_request_path(); if (connect_method) { hdrs += authority; @@ -246,19 +248,12 @@ int HttpDownstreamConnection::push_request_headers() { hdrs += scheme; hdrs += "://"; hdrs += authority; - - // Server-wide OPTIONS takes following form in proxy request: - // - // OPTIONS http://example.org HTTP/1.1 - // - // Notice that no slash after authority. See - // http://tools.ietf.org/html/rfc7230#section-5.3.4 - if (downstream_->get_request_path() != "*") { - hdrs += downstream_->get_request_path(); - } + hdrs += path; + } else if (method == HTTP_OPTIONS && path.empty()) { + // Server-wide OPTIONS + hdrs += "*"; } else { - // No proxy case. - hdrs += downstream_->get_request_path(); + hdrs += path; } hdrs += " HTTP/1.1\r\nHost: "; hdrs += authority; diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index d6ab64e1..60d5e28f 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -224,7 +224,7 @@ void rewrite_request_host_path_from_uri(Downstream *downstream, const char *uri, // // Notice that no slash after authority. See // http://tools.ietf.org/html/rfc7230#section-5.3.4 - downstream->set_request_path("*"); + downstream->set_request_path(""); // we ignore query component here return; } else { @@ -258,10 +258,13 @@ int htp_hdrs_completecb(http_parser *htp) { downstream->set_request_connection_close(!http_should_keep_alive(htp)); + auto method = downstream->get_request_method(); + if (LOG_ENABLED(INFO)) { std::stringstream ss; - ss << http2::to_method_string(downstream->get_request_method()) << " " - << downstream->get_request_path() << " " + ss << http2::to_method_string(method) << " " + << (method == HTTP_CONNECT ? downstream->get_request_http2_authority() + : downstream->get_request_path()) << " " << "HTTP/" << downstream->get_request_major() << "." << downstream->get_request_minor() << "\n"; const auto &headers = downstream->get_request_headers(); @@ -284,13 +287,12 @@ int htp_hdrs_completecb(http_parser *htp) { downstream->inspect_http1_request(); - if (downstream->get_request_method() != HTTP_CONNECT) { + if (method != HTTP_CONNECT) { http_parser_url u{}; // make a copy of request path, since we may set request path // while we are refering to original request path. - auto uri = downstream->get_request_path(); - rv = http_parser_parse_url(uri.c_str(), - downstream->get_request_path().size(), 0, &u); + auto path = downstream->get_request_path(); + rv = http_parser_parse_url(path.c_str(), path.size(), 0, &u); if (rv != 0) { // Expect to respond with 400 bad request return -1; @@ -302,8 +304,12 @@ int htp_hdrs_completecb(http_parser *htp) { return -1; } - downstream->set_request_path( - http2::rewrite_clean_path(std::begin(uri), std::end(uri))); + if (method == HTTP_OPTIONS && path == "*") { + downstream->set_request_path(""); + } else { + downstream->set_request_path( + http2::rewrite_clean_path(std::begin(path), std::end(path))); + } auto host = downstream->get_request_header(http2::HD_HOST); if (host) { @@ -316,7 +322,7 @@ int htp_hdrs_completecb(http_parser *htp) { downstream->set_request_http2_scheme("http"); } } else { - rewrite_request_host_path_from_uri(downstream, uri.c_str(), u); + rewrite_request_host_path_from_uri(downstream, path.c_str(), u); } } @@ -331,6 +337,8 @@ int htp_hdrs_completecb(http_parser *htp) { return -1; } + // mruby hook may change method value + if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { return 0; } diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 9a31e033..4f932431 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -227,6 +227,8 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, downstream->set_request_http2_authority(host->value); if (get_config()->http2_proxy || get_config()->client_proxy) { downstream->set_request_path(path->value); + } else if (method_token == HTTP_OPTIONS && path->value == "*") { + // Server-wide OPTIONS request. Path is empty. } else { downstream->set_request_path(http2::rewrite_clean_path( std::begin(path->value), std::end(path->value))); From d044c58558bbcdc6cc1add493fbedf71630a9108 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 4 Sep 2015 00:54:41 +0900 Subject: [PATCH 20/55] nghttpx: Detect mruby presence and guard mruby related code with ifdef --- configure.ac | 30 ++++++++++++++++++++++++++++++ src/Makefile.am | 20 +++++++++++++------- src/shrpx_connection_handler.cc | 4 ++++ src/shrpx_http2_upstream.cc | 6 ++++++ src/shrpx_https_upstream.cc | 6 ++++++ src/shrpx_spdy_upstream.cc | 6 ++++++ src/shrpx_worker.cc | 4 ++++ src/shrpx_worker.h | 6 ++++++ 8 files changed, 75 insertions(+), 7 deletions(-) diff --git a/configure.ac b/configure.ac index 31a38818..c5a351b9 100644 --- a/configure.ac +++ b/configure.ac @@ -119,6 +119,11 @@ AC_ARG_WITH([spdylay], [Use spdylay [default=check]])], [request_spdylay=$withval], [request_spdylay=check]) +AC_ARG_WITH([mruby], + [AS_HELP_STRING([--with-mruby], + [Use mruby [default=check]])], + [request_mruby=$withval], [request_mruby=check]) + AC_ARG_WITH([cython], [AS_HELP_STRING([--with-cython=PATH], [Use cython in given PATH])], @@ -370,6 +375,30 @@ fi AM_CONDITIONAL([HAVE_SPDYLAY], [ test "x${have_spdylay}" = "xyes" ]) +# mruby (for src/nghttpx) +LIBS_OLD=$LIBS +have_mruby=no +if test "x${request_mruby}" != "xno"; then + AC_CHECK_LIB([mruby], [mrb_open], [have_mruby=yes], [have_mruby=no]) + if test "x${have_mruby}" = "xyes"; then + AC_CHECK_HEADER([mruby.h], [have_mruby=yes], [have_mruby=no]) + if test "x${have_mruby}" = "xyes"; then + LIBMRUBY_LIBS=-lmruby + LIBMRUBY_CFLAGS= + AC_SUBST([LIBMRUBY_LIBS]) + AC_SUBST([LIBMRUBY_CFLAGS]) + fi + fi +fi +LIBS=$LIBS_OLD + +if test "x${request_mruby}" = "xyes" && + test "x${have_mruby}" != "xyes"; then + AC_MSG_ERROR([mruby was requested (--with-mruby) but not found]) +fi + +AM_CONDITIONAL([HAVE_MRUBY], [test "x${have_mruby}" = "xyes"]) + # Check Boost Asio library have_asio_lib=no @@ -717,6 +746,7 @@ AC_MSG_NOTICE([summary of build options: Libev: ${have_libev} Libevent(SSL): ${have_libevent_openssl} Spdylay: ${have_spdylay} + MRuby: ${have_mruby} Jansson: ${have_jansson} Jemalloc: ${have_jemalloc} Zlib: ${have_zlib} diff --git a/src/Makefile.am b/src/Makefile.am index 0384a997..52f50491 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -124,21 +124,26 @@ NGHTTPX_SRCS = \ shrpx_memcached_connection.cc shrpx_memcached_connection.h \ shrpx_memcached_request.h \ shrpx_memcached_result.h \ - shrpx_mruby.cc shrpx_mruby.h \ - shrpx_mruby_module.cc shrpx_mruby_module.h \ - shrpx_mruby_module_request.cc shrpx_mruby_module_request.h \ - shrpx_mruby_module_response.cc shrpx_mruby_module_response.h \ buffer.h memchunk.h template.h if HAVE_SPDYLAY NGHTTPX_SRCS += shrpx_spdy_upstream.cc shrpx_spdy_upstream.h endif # HAVE_SPDYLAY +if HAVE_MRUBY +NGHTTPX_SRCS += \ + shrpx_mruby.cc shrpx_mruby.h \ + shrpx_mruby_module.cc shrpx_mruby_module.h \ + shrpx_mruby_module_request.cc shrpx_mruby_module_request.h \ + shrpx_mruby_module_response.cc shrpx_mruby_module_response.h +endif # HAVE_MRUBY + noinst_LIBRARIES = libnghttpx.a libnghttpx_a_SOURCES = ${NGHTTPX_SRCS} nghttpx_SOURCES = shrpx.cc shrpx.h -nghttpx_LDADD = libnghttpx.a ${LDADD} -lmruby +nghttpx_CPPFLAGS = ${AM_CPPFLAGS} @LIBMRUBY_CFLAGS@ +nghttpx_LDADD = libnghttpx.a ${LDADD} @LIBMRUBY_LIBS@ if HAVE_CUNIT check_PROGRAMS += nghttpx-unittest @@ -152,9 +157,10 @@ nghttpx_unittest_SOURCES = shrpx-unittest.cc \ nghttp2_gzip.c nghttp2_gzip.h \ buffer_test.cc buffer_test.h \ memchunk_test.cc memchunk_test.h -nghttpx_unittest_CPPFLAGS = ${AM_CPPFLAGS}\ +nghttpx_unittest_CPPFLAGS = ${AM_CPPFLAGS} @LIBMRUBY_CFLAGS@ \ -DNGHTTP2_TESTS_DIR=\"$(top_srcdir)/tests\" -nghttpx_unittest_LDADD = libnghttpx.a ${LDADD} -lmruby @CUNIT_LIBS@ @TESTLDADD@ +nghttpx_unittest_LDADD = libnghttpx.a ${LDADD} \ + @LIBMRUBY_LIBS@ @CUNIT_LIBS@ @TESTLDADD@ TESTS += nghttpx-unittest endif # HAVE_CUNIT diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc index 0dc6dcb3..8b82db5f 100644 --- a/src/shrpx_connection_handler.cc +++ b/src/shrpx_connection_handler.cc @@ -160,9 +160,11 @@ int ConnectionHandler::create_single_worker() { single_worker_ = make_unique(loop_, sv_ssl_ctx, cl_ssl_ctx, cert_tree, ticket_keys_); +#ifdef HAVE_MRUBY if (single_worker_->create_mruby_context() != 0) { return -1; } +#endif // HAVE_MRUBY return 0; } @@ -184,9 +186,11 @@ int ConnectionHandler::create_worker_thread(size_t num) { auto worker = make_unique(loop, sv_ssl_ctx, cl_ssl_ctx, cert_tree, ticket_keys_); +#ifdef HAVE_MRUBY if (worker->create_mruby_context() != 0) { return -1; } +#endif // HAVE_MRUBY workers_.push_back(std::move(worker)); diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 5a0e075e..ed26e207 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -37,7 +37,9 @@ #include "shrpx_http.h" #include "shrpx_worker.h" #include "shrpx_http2_session.h" +#ifdef HAVE_MRUBY #include "shrpx_mruby.h" +#endif // HAVE_MRUBY #include "http2.h" #include "util.h" #include "base64.h" @@ -318,6 +320,7 @@ int Http2Upstream::on_request_headers(Downstream *downstream, downstream->set_request_state(Downstream::HEADER_COMPLETE); +#ifdef HAVE_MRUBY auto upstream = downstream->get_upstream(); auto handler = upstream->get_client_handler(); auto worker = handler->get_worker(); @@ -329,6 +332,7 @@ int Http2Upstream::on_request_headers(Downstream *downstream, } return 0; } +#endif // HAVE_MRUBY if (frame->hd.flags & NGHTTP2_FLAG_END_STREAM) { downstream->disable_upstream_rtimer(); @@ -1271,6 +1275,7 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { downstream->get_request_http2_scheme()); } +#ifdef HAVE_MRUBY if (!downstream->get_non_final_response()) { auto worker = handler_->get_worker(); auto mruby_ctx = worker->get_mruby_context(); @@ -1286,6 +1291,7 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { return -1; } } +#endif // HAVE_MRUBY size_t nheader = downstream->get_response_headers().size(); auto nva = std::vector(); diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 60d5e28f..8857c670 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -37,7 +37,9 @@ #include "shrpx_log_config.h" #include "shrpx_worker.h" #include "shrpx_http2_session.h" +#ifdef HAVE_MRUBY #include "shrpx_mruby.h" +#endif // HAVE_MRUBY #include "http2.h" #include "util.h" #include "template.h" @@ -328,6 +330,7 @@ int htp_hdrs_completecb(http_parser *htp) { downstream->set_request_state(Downstream::HEADER_COMPLETE); +#ifdef HAVE_MRUBY auto handler = upstream->get_client_handler(); auto worker = handler->get_worker(); auto mruby_ctx = worker->get_mruby_context(); @@ -336,6 +339,7 @@ int htp_hdrs_completecb(http_parser *htp) { downstream->set_response_http_status(500); return -1; } +#endif // HAVE_MRUBY // mruby hook may change method value @@ -888,6 +892,7 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) { } } +#ifdef HAVE_MRUBY if (!downstream->get_non_final_response()) { auto worker = handler_->get_worker(); auto mruby_ctx = worker->get_mruby_context(); @@ -901,6 +906,7 @@ int HttpsUpstream::on_downstream_header_complete(Downstream *downstream) { return -1; } } +#endif // HAVE_MRUBY auto connect_method = downstream->get_request_method() == HTTP_CONNECT; diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index 4f932431..ee66c228 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -36,7 +36,9 @@ #include "shrpx_downstream_connection.h" #include "shrpx_config.h" #include "shrpx_http.h" +#ifdef HAVE_MRUBY #include "shrpx_mruby.h" +#endif // HAVE_MRUBY #include "shrpx_worker.h" #include "shrpx_http2_session.h" #include "http2.h" @@ -243,6 +245,7 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, downstream->set_request_state(Downstream::HEADER_COMPLETE); +#ifdef HAVE_MRUBY auto handler = upstream->get_client_handler(); auto worker = handler->get_worker(); auto mruby_ctx = worker->get_mruby_context(); @@ -254,6 +257,7 @@ void on_ctrl_recv_callback(spdylay_session *session, spdylay_frame_type type, } return; } +#endif // HAVE_MRUBY if (frame->syn_stream.hd.flags & SPDYLAY_CTRL_FLAG_FIN) { if (!downstream->validate_request_bodylen()) { @@ -930,6 +934,7 @@ int SpdyUpstream::on_downstream_header_complete(Downstream *downstream) { return 0; } +#ifdef HAVE_MRUBY auto worker = handler_->get_worker(); auto mruby_ctx = worker->get_mruby_context(); @@ -943,6 +948,7 @@ int SpdyUpstream::on_downstream_header_complete(Downstream *downstream) { if (downstream->get_response_state() == Downstream::MSG_COMPLETE) { return -1; } +#endif // HAVE_MRUBY if (LOG_ENABLED(INFO)) { DLOG(INFO, downstream) << "HTTP response header completed"; diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 006e7e3a..fadd6494 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -37,7 +37,9 @@ #include "shrpx_log_config.h" #include "shrpx_connect_blocker.h" #include "shrpx_memcached_dispatcher.h" +#ifdef HAVE_MRUBY #include "shrpx_mruby.h" +#endif // HAVE_MRUBY #include "util.h" #include "template.h" @@ -266,6 +268,7 @@ MemcachedDispatcher *Worker::get_session_cache_memcached_dispatcher() { return session_cache_memcached_dispatcher_.get(); } +#ifdef HAVE_MRUBY int Worker::create_mruby_context() { mruby_ctx_ = mruby::create_mruby_context(); if (!mruby_ctx_) { @@ -278,5 +281,6 @@ int Worker::create_mruby_context() { mruby::MRubyContext *Worker::get_mruby_context() const { return mruby_ctx_.get(); } +#endif // HAVE_MRUBY } // namespace shrpx diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 38868aab..6037683d 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -51,11 +51,13 @@ class Http2Session; class ConnectBlocker; class MemcachedDispatcher; +#ifdef HAVE_MRUBY namespace mruby { class MRubyContext; } // namespace mruby +#endif // HAVE_MRUBY namespace ssl { class CertLookupTree; @@ -130,9 +132,11 @@ public: MemcachedDispatcher *get_session_cache_memcached_dispatcher(); +#ifdef HAVE_MRUBY int create_mruby_context(); mruby::MRubyContext *get_mruby_context() const; +#endif // HAVE_MRUBY private: #ifndef NOTHREADS @@ -147,7 +151,9 @@ private: WorkerStat worker_stat_; std::vector dgrps_; std::unique_ptr session_cache_memcached_dispatcher_; +#ifdef HAVE_MRUBY std::unique_ptr mruby_ctx_; +#endif // HAVE_MRUBY struct ev_loop *loop_; // Following fields are shared across threads if From 5967667e9e190cd3dbcd172f6e4023a76b4d6e80 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 4 Sep 2015 01:00:27 +0900 Subject: [PATCH 21/55] nghttpx: Allow empty path assignment in mruby code --- src/shrpx_mruby_module_request.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index 8d7442b1..e1fb1edb 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -168,9 +168,6 @@ mrb_value request_set_path(mrb_state *mrb, mrb_value self) { const char *path; mrb_int pathlen; mrb_get_args(mrb, "s", &path, &pathlen); - if (pathlen == 0) { - mrb_raise(mrb, E_RUNTIME_ERROR, "path must not be empty string"); - } downstream->set_request_path(std::string(path, pathlen)); From d722a095813a1f9957e62ab821b0e59401d20598 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 4 Sep 2015 01:19:57 +0900 Subject: [PATCH 22/55] nghttpx: Rename mruby script options --- gennghttpxfun.py | 4 ++-- src/shrpx.cc | 20 ++++++++++---------- src/shrpx_config.cc | 30 ++++++++++++++---------------- src/shrpx_config.h | 8 ++++---- src/shrpx_mruby.cc | 4 ++-- 5 files changed, 32 insertions(+), 34 deletions(-) diff --git a/gennghttpxfun.py b/gennghttpxfun.py index 74732b98..3114137d 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -98,8 +98,8 @@ OPTIONS = [ "tls-ticket-key-memcached-interval", "tls-ticket-key-memcached-max-retry", "tls-ticket-key-memcached-max-fail", - "on-request-mruby-file", - "on-response-mruby-file", + "request-phase-file", + "response-phase-file", "conf", ] diff --git a/src/shrpx.cc b/src/shrpx.cc index 4b5b2dc8..7ef8e1ed 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1731,14 +1731,14 @@ Process: be used to drop root privileges. Scripting: - --on-request-mruby-file= + --request-phase-file= Set mruby script file which will be executed when request header fields are completely received from - frontend. - --on-response-mruby-file= + frontend. This hook is called request phase hook. + --response-phase-file= Set mruby script file which will be executed when response header fields are completely received from - backend. + backend. This hook is called response phase hook. Misc: --conf= @@ -1913,8 +1913,8 @@ int main(int argc, char **argv) { 89}, {SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_MAX_FAIL, required_argument, &flag, 90}, - {SHRPX_OPT_ON_REQUEST_MRUBY_FILE, required_argument, &flag, 91}, - {SHRPX_OPT_ON_RESPONSE_MRUBY_FILE, required_argument, &flag, 92}, + {SHRPX_OPT_REQUEST_PHASE_FILE, required_argument, &flag, 91}, + {SHRPX_OPT_RESPONSE_PHASE_FILE, required_argument, &flag, 92}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2313,12 +2313,12 @@ int main(int argc, char **argv) { optarg); break; case 91: - // --on-request-mruby-file - cmdcfgs.emplace_back(SHRPX_OPT_ON_REQUEST_MRUBY_FILE, optarg); + // --request-phase-file + cmdcfgs.emplace_back(SHRPX_OPT_REQUEST_PHASE_FILE, optarg); break; case 92: - // --on-response-mruby-file - cmdcfgs.emplace_back(SHRPX_OPT_ON_RESPONSE_MRUBY_FILE, optarg); + // --response-phase-file + cmdcfgs.emplace_back(SHRPX_OPT_RESPONSE_PHASE_FILE, optarg); default: break; } diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index a923575a..09ee2e36 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -690,14 +690,14 @@ enum { SHRPX_OPTID_NO_VIA, SHRPX_OPTID_NPN_LIST, SHRPX_OPTID_OCSP_UPDATE_INTERVAL, - SHRPX_OPTID_ON_REQUEST_MRUBY_FILE, - SHRPX_OPTID_ON_RESPONSE_MRUBY_FILE, SHRPX_OPTID_PADDING, SHRPX_OPTID_PID_FILE, SHRPX_OPTID_PRIVATE_KEY_FILE, SHRPX_OPTID_PRIVATE_KEY_PASSWD_FILE, SHRPX_OPTID_READ_BURST, SHRPX_OPTID_READ_RATE, + SHRPX_OPTID_REQUEST_PHASE_FILE, + SHRPX_OPTID_RESPONSE_PHASE_FILE, SHRPX_OPTID_RLIMIT_NOFILE, SHRPX_OPTID_STREAM_READ_TIMEOUT, SHRPX_OPTID_STREAM_WRITE_TIMEOUT, @@ -1019,6 +1019,11 @@ int option_lookup_token(const char *name, size_t namelen) { break; case 18: switch (name[17]) { + case 'e': + if (util::strieq_l("request-phase-fil", name, 17)) { + return SHRPX_OPTID_REQUEST_PHASE_FILE; + } + break; case 'r': if (util::strieq_l("add-request-heade", name, 17)) { return SHRPX_OPTID_ADD_REQUEST_HEADER; @@ -1037,6 +1042,9 @@ int option_lookup_token(const char *name, size_t namelen) { if (util::strieq_l("no-location-rewrit", name, 18)) { return SHRPX_OPTID_NO_LOCATION_REWRITE; } + if (util::strieq_l("response-phase-fil", name, 18)) { + return SHRPX_OPTID_RESPONSE_PHASE_FILE; + } if (util::strieq_l("tls-ticket-key-fil", name, 18)) { return SHRPX_OPTID_TLS_TICKET_KEY_FILE; } @@ -1091,11 +1099,6 @@ int option_lookup_token(const char *name, size_t namelen) { return SHRPX_OPTID_BACKEND_TLS_SNI_FIELD; } break; - case 'e': - if (util::strieq_l("on-request-mruby-fil", name, 20)) { - return SHRPX_OPTID_ON_REQUEST_MRUBY_FILE; - } - break; case 'r': if (util::strieq_l("tls-ticket-key-ciphe", name, 20)) { return SHRPX_OPTID_TLS_TICKET_KEY_CIPHER; @@ -1113,11 +1116,6 @@ int option_lookup_token(const char *name, size_t namelen) { break; case 22: switch (name[21]) { - case 'e': - if (util::strieq_l("on-response-mruby-fil", name, 21)) { - return SHRPX_OPTID_ON_RESPONSE_MRUBY_FILE; - } - break; case 'i': if (util::strieq_l("backend-http-proxy-ur", name, 21)) { return SHRPX_OPTID_BACKEND_HTTP_PROXY_URI; @@ -1950,12 +1948,12 @@ int parse_config(const char *opt, const char *optarg, case SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED_MAX_FAIL: return parse_uint(&mod_config()->tls_ticket_key_memcached_max_fail, opt, optarg); - case SHRPX_OPTID_ON_REQUEST_MRUBY_FILE: - mod_config()->on_request_mruby_file = strcopy(optarg); + case SHRPX_OPTID_REQUEST_PHASE_FILE: + mod_config()->request_phase_file = strcopy(optarg); return 0; - case SHRPX_OPTID_ON_RESPONSE_MRUBY_FILE: - mod_config()->on_response_mruby_file = strcopy(optarg); + case SHRPX_OPTID_RESPONSE_PHASE_FILE: + mod_config()->response_phase_file = strcopy(optarg); return 0; case SHRPX_OPTID_CONF: diff --git a/src/shrpx_config.h b/src/shrpx_config.h index 427062b5..2bbb9647 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -183,8 +183,8 @@ constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_MAX_RETRY[] = "tls-ticket-key-memcached-max-retry"; constexpr char SHRPX_OPT_TLS_TICKET_KEY_MEMCACHED_MAX_FAIL[] = "tls-ticket-key-memcached-max-fail"; -constexpr char SHRPX_OPT_ON_REQUEST_MRUBY_FILE[] = "on-request-mruby-file"; -constexpr char SHRPX_OPT_ON_RESPONSE_MRUBY_FILE[] = "on-response-mruby-file"; +constexpr char SHRPX_OPT_REQUEST_PHASE_FILE[] = "request-phase-file"; +constexpr char SHRPX_OPT_RESPONSE_PHASE_FILE[] = "response-phase-file"; union sockaddr_union { sockaddr_storage storage; @@ -316,8 +316,8 @@ struct Config { std::unique_ptr user; std::unique_ptr session_cache_memcached_host; std::unique_ptr tls_ticket_key_memcached_host; - std::unique_ptr on_request_mruby_file; - std::unique_ptr on_response_mruby_file; + std::unique_ptr request_phase_file; + std::unique_ptr response_phase_file; FILE *http2_upstream_dump_request_header; FILE *http2_upstream_dump_response_header; nghttp2_session_callbacks *http2_upstream_callbacks; diff --git a/src/shrpx_mruby.cc b/src/shrpx_mruby.cc index 50a5ebb0..f2f23512 100644 --- a/src/shrpx_mruby.cc +++ b/src/shrpx_mruby.cc @@ -152,8 +152,8 @@ std::unique_ptr create_mruby_context() { init_module(mrb); - auto req_file = get_config()->on_request_mruby_file.get(); - auto res_file = get_config()->on_response_mruby_file.get(); + auto req_file = get_config()->request_phase_file.get(); + auto res_file = get_config()->response_phase_file.get(); auto req_proc = compile(mrb, req_file); From c86e8390912800dc88d41b2078d27e3fd03f490e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 4 Sep 2015 01:28:01 +0900 Subject: [PATCH 23/55] nghttpx: Fix mruby detection --- configure.ac | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index c5a351b9..18838c53 100644 --- a/configure.ac +++ b/configure.ac @@ -379,11 +379,12 @@ AM_CONDITIONAL([HAVE_SPDYLAY], [ test "x${have_spdylay}" = "xyes" ]) LIBS_OLD=$LIBS have_mruby=no if test "x${request_mruby}" != "xno"; then - AC_CHECK_LIB([mruby], [mrb_open], [have_mruby=yes], [have_mruby=no]) + AC_CHECK_LIB([mruby], [mrb_open], [have_mruby=yes], [have_mruby=no], [-lm]) if test "x${have_mruby}" = "xyes"; then AC_CHECK_HEADER([mruby.h], [have_mruby=yes], [have_mruby=no]) if test "x${have_mruby}" = "xyes"; then - LIBMRUBY_LIBS=-lmruby + AC_DEFINE([HAVE_MRUBY], [1], [Define to 1 if you have `mruby` library.]) + LIBMRUBY_LIBS="-lmruby -lm" LIBMRUBY_CFLAGS= AC_SUBST([LIBMRUBY_LIBS]) AC_SUBST([LIBMRUBY_CFLAGS]) From 9e685a2734be556244cfca3ebab7d4660936a935 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 4 Sep 2015 02:46:35 +0900 Subject: [PATCH 24/55] nghttpx: Various test failures --- src/shrpx_http2_upstream.cc | 8 +++++++- src/shrpx_https_upstream.cc | 1 + src/shrpx_spdy_upstream.cc | 7 ++++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index ed26e207..3c0c8d28 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -927,6 +927,10 @@ int Http2Upstream::downstream_read(DownstreamConnection *dconn) { if (rv == SHRPX_ERR_EOF) { return downstream_eof(dconn); } + if (rv == SHRPX_ERR_DCONN_CANCELED) { + downstream->pop_downstream_connection(); + return 0; + } if (rv != 0) { if (rv != SHRPX_ERR_NETWORK) { if (LOG_ENABLED(INFO)) { @@ -1163,8 +1167,10 @@ int Http2Upstream::send_reply(Downstream *downstream, const uint8_t *body, auto status_code_str = util::utos(downstream->get_response_http_status()); auto &headers = downstream->get_response_headers(); + auto nva = std::vector(); // 2 for :status and server - auto nva = std::vector(2 + headers.size()); + nva.reserve(2 + headers.size()); + nva.push_back(http2::make_nv_ls(":status", status_code_str)); for (auto &kv : headers) { diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index 8857c670..f1c0b31a 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -800,6 +800,7 @@ int HttpsUpstream::send_reply(Downstream *downstream, const uint8_t *body, auto status_str = http2::get_status_string(downstream->get_response_http_status()); output->append(status_str.c_str(), status_str.size()); + output->append("\r\n"); for (auto &kv : downstream->get_response_headers()) { if (kv.name.empty() || kv.name[0] == ':') { diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index ee66c228..f8d59ea1 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -600,6 +600,10 @@ int SpdyUpstream::downstream_read(DownstreamConnection *dconn) { if (rv == SHRPX_ERR_EOF) { return downstream_eof(dconn); } + if (rv == SHRPX_ERR_DCONN_CANCELED) { + downstream->pop_downstream_connection(); + return 0; + } if (rv != 0) { if (rv != SHRPX_ERR_NETWORK) { if (LOG_ENABLED(INFO)) { @@ -815,8 +819,9 @@ int SpdyUpstream::send_reply(Downstream *downstream, const uint8_t *body, auto &headers = downstream->get_response_headers(); + auto nva = std::vector(); // 3 for :status, :version and server - auto nva = std::vector(3 + headers.size()); + nva.reserve(3 + headers.size()); nva.push_back(":status"); nva.push_back(status_string.c_str()); From 2222a898a19744a3a54326d492a6b33c97d91807 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 4 Sep 2015 02:52:19 +0900 Subject: [PATCH 25/55] integration: Add mruby tests --- integration-tests/nghttpx_http1_test.go | 184 ++++++++++++++++++++++++ integration-tests/nghttpx_http2_test.go | 184 ++++++++++++++++++++++++ integration-tests/nghttpx_spdy_test.go | 184 ++++++++++++++++++++++++ integration-tests/req-set-header.rb | 3 + integration-tests/resp-set-header.rb | 3 + integration-tests/return.rb | 6 + 6 files changed, 564 insertions(+) create mode 100644 integration-tests/req-set-header.rb create mode 100644 integration-tests/resp-set-header.rb create mode 100644 integration-tests/return.rb diff --git a/integration-tests/nghttpx_http1_test.go b/integration-tests/nghttpx_http1_test.go index 71aeeb85..c7658eb5 100644 --- a/integration-tests/nghttpx_http1_test.go +++ b/integration-tests/nghttpx_http1_test.go @@ -355,6 +355,120 @@ func TestH1H1Websocket(t *testing.T) { } } +// TestH1H1ReqPhaseSetHeader tests mruby request phase hook +// modifies request header fields. +func TestH1H1ReqPhaseSetHeader(t *testing.T) { + st := newServerTester([]string{"--request-phase-file=" + testDir + "/req-set-header.rb"}, t, func(w http.ResponseWriter, r *http.Request) { + if got, want := r.Header.Get("User-Agent"), "mruby"; got != want { + t.Errorf("User-Agent = %v; want %v", got, want) + } + }) + defer st.Close() + + res, err := st.http1(requestParam{ + name: "TestH1H1ReqPhaseSetHeader", + }) + if err != nil { + t.Fatalf("Error st.http1() = %v", err) + } + + if got, want := res.status, 200; got != want { + t.Errorf("status = %v; want %v", got, want) + } +} + +// TestH1H1ReqPhaseReturn tests mruby request phase hook returns +// custom response. +func TestH1H1ReqPhaseReturn(t *testing.T) { + st := newServerTester([]string{"--request-phase-file=" + testDir + "/return.rb"}, t, func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("request should not be forwarded") + }) + defer st.Close() + + res, err := st.http1(requestParam{ + name: "TestH1H1ReqPhaseReturn", + }) + if err != nil { + t.Fatalf("Error st.http1() = %v", err) + } + + if got, want := res.status, 404; got != want { + t.Errorf("status = %v; want %v", got, want) + } + + hdtests := []struct { + k, v string + }{ + {"content-length", "11"}, + {"from", "mruby"}, + } + for _, tt := range hdtests { + if got, want := res.header.Get(tt.k), tt.v; got != want { + t.Errorf("%v = %v; want %v", tt.k, got, want) + } + } + + if got, want := string(res.body), "Hello World"; got != want { + t.Errorf("body = %v; want %v", got, want) + } +} + +// TestH1H1RespPhaseSetHeader tests mruby response phase hook modifies +// response header fields. +func TestH1H1RespPhaseSetHeader(t *testing.T) { + st := newServerTester([]string{"--response-phase-file=" + testDir + "/resp-set-header.rb"}, t, noopHandler) + defer st.Close() + + res, err := st.http1(requestParam{ + name: "TestH1H1RespPhaseSetHeader", + }) + if err != nil { + t.Fatalf("Error st.http1() = %v", err) + } + + if got, want := res.status, 200; got != want { + t.Errorf("status = %v; want %v", got, want) + } + + if got, want := res.header.Get("alpha"), "bravo"; got != want { + t.Errorf("alpha = %v; want %v", got, want) + } +} + +// TestH1H1RespPhaseReturn tests mruby response phase hook returns +// custom response. +func TestH1H1RespPhaseReturn(t *testing.T) { + st := newServerTester([]string{"--response-phase-file=" + testDir + "/return.rb"}, t, noopHandler) + defer st.Close() + + res, err := st.http1(requestParam{ + name: "TestH1H1RespPhaseReturn", + }) + if err != nil { + t.Fatalf("Error st.http1() = %v", err) + } + + if got, want := res.status, 404; got != want { + t.Errorf("status = %v; want %v", got, want) + } + + hdtests := []struct { + k, v string + }{ + {"content-length", "11"}, + {"from", "mruby"}, + } + for _, tt := range hdtests { + if got, want := res.header.Get(tt.k), tt.v; got != want { + t.Errorf("%v = %v; want %v", tt.k, got, want) + } + } + + if got, want := string(res.body), "Hello World"; got != want { + t.Errorf("body = %v; want %v", got, want) + } +} + // TestH1H2ConnectFailure tests that server handles the situation that // connection attempt to HTTP/2 backend failed. func TestH1H2ConnectFailure(t *testing.T) { @@ -547,3 +661,73 @@ func TestH1H2NoVia(t *testing.T) { t.Errorf("Via: %v; want %v", got, want) } } + +// TestH1H2ReqPhaseReturn tests mruby request phase hook returns +// custom response. +func TestH1H2ReqPhaseReturn(t *testing.T) { + st := newServerTester([]string{"--http2-bridge", "--request-phase-file=" + testDir + "/return.rb"}, t, func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("request should not be forwarded") + }) + defer st.Close() + + res, err := st.http1(requestParam{ + name: "TestH1H2ReqPhaseReturn", + }) + if err != nil { + t.Fatalf("Error st.http1() = %v", err) + } + + if got, want := res.status, 404; got != want { + t.Errorf("status = %v; want %v", got, want) + } + + hdtests := []struct { + k, v string + }{ + {"content-length", "11"}, + {"from", "mruby"}, + } + for _, tt := range hdtests { + if got, want := res.header.Get(tt.k), tt.v; got != want { + t.Errorf("%v = %v; want %v", tt.k, got, want) + } + } + + if got, want := string(res.body), "Hello World"; got != want { + t.Errorf("body = %v; want %v", got, want) + } +} + +// TestH1H2RespPhaseReturn tests mruby response phase hook returns +// custom response. +func TestH1H2RespPhaseReturn(t *testing.T) { + st := newServerTester([]string{"--http2-bridge", "--response-phase-file=" + testDir + "/return.rb"}, t, noopHandler) + defer st.Close() + + res, err := st.http1(requestParam{ + name: "TestH1H2RespPhaseReturn", + }) + if err != nil { + t.Fatalf("Error st.http1() = %v", err) + } + + if got, want := res.status, 404; got != want { + t.Errorf("status = %v; want %v", got, want) + } + + hdtests := []struct { + k, v string + }{ + {"content-length", "11"}, + {"from", "mruby"}, + } + for _, tt := range hdtests { + if got, want := res.header.Get(tt.k), tt.v; got != want { + t.Errorf("%v = %v; want %v", tt.k, got, want) + } + } + + if got, want := string(res.body), "Hello World"; got != want { + t.Errorf("body = %v; want %v", got, want) + } +} diff --git a/integration-tests/nghttpx_http2_test.go b/integration-tests/nghttpx_http2_test.go index 5fe4c3db..41c527f7 100644 --- a/integration-tests/nghttpx_http2_test.go +++ b/integration-tests/nghttpx_http2_test.go @@ -640,6 +640,120 @@ func TestH2H1HeaderFields(t *testing.T) { } } +// TestH2H1ReqPhaseSetHeader tests mruby request phase hook +// modifies request header fields. +func TestH2H1ReqPhaseSetHeader(t *testing.T) { + st := newServerTester([]string{"--request-phase-file=" + testDir + "/req-set-header.rb"}, t, func(w http.ResponseWriter, r *http.Request) { + if got, want := r.Header.Get("User-Agent"), "mruby"; got != want { + t.Errorf("User-Agent = %v; want %v", got, want) + } + }) + defer st.Close() + + res, err := st.http2(requestParam{ + name: "TestH2H1ReqPhaseSetHeader", + }) + if err != nil { + t.Fatalf("Error st.http2() = %v", err) + } + + if got, want := res.status, 200; got != want { + t.Errorf("status = %v; want %v", got, want) + } +} + +// TestH2H1ReqPhaseReturn tests mruby request phase hook returns +// custom response. +func TestH2H1ReqPhaseReturn(t *testing.T) { + st := newServerTester([]string{"--request-phase-file=" + testDir + "/return.rb"}, t, func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("request should not be forwarded") + }) + defer st.Close() + + res, err := st.http2(requestParam{ + name: "TestH2H1ReqPhaseReturn", + }) + if err != nil { + t.Fatalf("Error st.http2() = %v", err) + } + + if got, want := res.status, 404; got != want { + t.Errorf("status = %v; want %v", got, want) + } + + hdtests := []struct { + k, v string + }{ + {"content-length", "11"}, + {"from", "mruby"}, + } + for _, tt := range hdtests { + if got, want := res.header.Get(tt.k), tt.v; got != want { + t.Errorf("%v = %v; want %v", tt.k, got, want) + } + } + + if got, want := string(res.body), "Hello World"; got != want { + t.Errorf("body = %v; want %v", got, want) + } +} + +// TestH2H1RespPhaseSetHeader tests mruby response phase hook modifies +// response header fields. +func TestH2H1RespPhaseSetHeader(t *testing.T) { + st := newServerTester([]string{"--response-phase-file=" + testDir + "/resp-set-header.rb"}, t, noopHandler) + defer st.Close() + + res, err := st.http2(requestParam{ + name: "TestH2H1RespPhaseSetHeader", + }) + if err != nil { + t.Fatalf("Error st.http2() = %v", err) + } + + if got, want := res.status, 200; got != want { + t.Errorf("status = %v; want %v", got, want) + } + + if got, want := res.header.Get("alpha"), "bravo"; got != want { + t.Errorf("alpha = %v; want %v", got, want) + } +} + +// TestH2H1RespPhaseReturn tests mruby response phase hook returns +// custom response. +func TestH2H1RespPhaseReturn(t *testing.T) { + st := newServerTester([]string{"--response-phase-file=" + testDir + "/return.rb"}, t, noopHandler) + defer st.Close() + + res, err := st.http2(requestParam{ + name: "TestH2H1RespPhaseReturn", + }) + if err != nil { + t.Fatalf("Error st.http2() = %v", err) + } + + if got, want := res.status, 404; got != want { + t.Errorf("status = %v; want %v", got, want) + } + + hdtests := []struct { + k, v string + }{ + {"content-length", "11"}, + {"from", "mruby"}, + } + for _, tt := range hdtests { + if got, want := res.header.Get(tt.k), tt.v; got != want { + t.Errorf("%v = %v; want %v", tt.k, got, want) + } + } + + if got, want := string(res.body), "Hello World"; got != want { + t.Errorf("body = %v; want %v", got, want) + } +} + // TestH2H1Upgrade tests HTTP Upgrade to HTTP/2 func TestH2H1Upgrade(t *testing.T) { st := newServerTester(nil, t, func(w http.ResponseWriter, r *http.Request) {}) @@ -875,3 +989,73 @@ func TestH2H2TLSXfp(t *testing.T) { t.Errorf("res.status: %v; want %v", got, want) } } + +// TestH2H2ReqPhaseReturn tests mruby request phase hook returns +// custom response. +func TestH2H2ReqPhaseReturn(t *testing.T) { + st := newServerTester([]string{"--http2-bridge", "--request-phase-file=" + testDir + "/return.rb"}, t, func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("request should not be forwarded") + }) + defer st.Close() + + res, err := st.http2(requestParam{ + name: "TestH2H2ReqPhaseReturn", + }) + if err != nil { + t.Fatalf("Error st.http2() = %v", err) + } + + if got, want := res.status, 404; got != want { + t.Errorf("status = %v; want %v", got, want) + } + + hdtests := []struct { + k, v string + }{ + {"content-length", "11"}, + {"from", "mruby"}, + } + for _, tt := range hdtests { + if got, want := res.header.Get(tt.k), tt.v; got != want { + t.Errorf("%v = %v; want %v", tt.k, got, want) + } + } + + if got, want := string(res.body), "Hello World"; got != want { + t.Errorf("body = %v; want %v", got, want) + } +} + +// TestH2H2RespPhaseReturn tests mruby response phase hook returns +// custom response. +func TestH2H2RespPhaseReturn(t *testing.T) { + st := newServerTester([]string{"--http2-bridge", "--response-phase-file=" + testDir + "/return.rb"}, t, noopHandler) + defer st.Close() + + res, err := st.http2(requestParam{ + name: "TestH2H2RespPhaseReturn", + }) + if err != nil { + t.Fatalf("Error st.http2() = %v", err) + } + + if got, want := res.status, 404; got != want { + t.Errorf("status = %v; want %v", got, want) + } + + hdtests := []struct { + k, v string + }{ + {"content-length", "11"}, + {"from", "mruby"}, + } + for _, tt := range hdtests { + if got, want := res.header.Get(tt.k), tt.v; got != want { + t.Errorf("%v = %v; want %v", tt.k, got, want) + } + } + + if got, want := string(res.body), "Hello World"; got != want { + t.Errorf("body = %v; want %v", got, want) + } +} diff --git a/integration-tests/nghttpx_spdy_test.go b/integration-tests/nghttpx_spdy_test.go index 76d349e8..e0447fad 100644 --- a/integration-tests/nghttpx_spdy_test.go +++ b/integration-tests/nghttpx_spdy_test.go @@ -230,6 +230,120 @@ func TestS3H1InvalidMethod(t *testing.T) { } } +// TestS3H1ReqPhaseSetHeader tests mruby request phase hook +// modifies request header fields. +func TestS3H1ReqPhaseSetHeader(t *testing.T) { + st := newServerTesterTLS([]string{"--npn-list=spdy/3.1", "--request-phase-file=" + testDir + "/req-set-header.rb"}, t, func(w http.ResponseWriter, r *http.Request) { + if got, want := r.Header.Get("User-Agent"), "mruby"; got != want { + t.Errorf("User-Agent = %v; want %v", got, want) + } + }) + defer st.Close() + + res, err := st.spdy(requestParam{ + name: "TestS3H1ReqPhaseSetHeader", + }) + if err != nil { + t.Fatalf("Error st.spdy() = %v", err) + } + + if got, want := res.status, 200; got != want { + t.Errorf("status = %v; want %v", got, want) + } +} + +// TestS3H1ReqPhaseReturn tests mruby request phase hook returns +// custom response. +func TestS3H1ReqPhaseReturn(t *testing.T) { + st := newServerTesterTLS([]string{"--npn-list=spdy/3.1", "--request-phase-file=" + testDir + "/return.rb"}, t, func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("request should not be forwarded") + }) + defer st.Close() + + res, err := st.spdy(requestParam{ + name: "TestS3H1ReqPhaseReturn", + }) + if err != nil { + t.Fatalf("Error st.spdy() = %v", err) + } + + if got, want := res.status, 404; got != want { + t.Errorf("status = %v; want %v", got, want) + } + + hdtests := []struct { + k, v string + }{ + {"content-length", "11"}, + {"from", "mruby"}, + } + for _, tt := range hdtests { + if got, want := res.header.Get(tt.k), tt.v; got != want { + t.Errorf("%v = %v; want %v", tt.k, got, want) + } + } + + if got, want := string(res.body), "Hello World"; got != want { + t.Errorf("body = %v; want %v", got, want) + } +} + +// TestS3H1RespPhaseSetHeader tests mruby response phase hook modifies +// response header fields. +func TestS3H1RespPhaseSetHeader(t *testing.T) { + st := newServerTesterTLS([]string{"--npn-list=spdy/3.1", "--response-phase-file=" + testDir + "/resp-set-header.rb"}, t, noopHandler) + defer st.Close() + + res, err := st.spdy(requestParam{ + name: "TestS3H1RespPhaseSetHeader", + }) + if err != nil { + t.Fatalf("Error st.spdy() = %v", err) + } + + if got, want := res.status, 200; got != want { + t.Errorf("status = %v; want %v", got, want) + } + + if got, want := res.header.Get("alpha"), "bravo"; got != want { + t.Errorf("alpha = %v; want %v", got, want) + } +} + +// TestS3H1RespPhaseReturn tests mruby response phase hook returns +// custom response. +func TestS3H1RespPhaseReturn(t *testing.T) { + st := newServerTesterTLS([]string{"--npn-list=spdy/3.1", "--response-phase-file=" + testDir + "/return.rb"}, t, noopHandler) + defer st.Close() + + res, err := st.spdy(requestParam{ + name: "TestS3H1RespPhaseReturn", + }) + if err != nil { + t.Fatalf("Error st.spdy() = %v", err) + } + + if got, want := res.status, 404; got != want { + t.Errorf("status = %v; want %v", got, want) + } + + hdtests := []struct { + k, v string + }{ + {"content-length", "11"}, + {"from", "mruby"}, + } + for _, tt := range hdtests { + if got, want := res.header.Get(tt.k), tt.v; got != want { + t.Errorf("%v = %v; want %v", tt.k, got, want) + } + } + + if got, want := string(res.body), "Hello World"; got != want { + t.Errorf("body = %v; want %v", got, want) + } +} + // TestS3H2ConnectFailure tests that server handles the situation that // connection attempt to HTTP/2 backend failed. func TestS3H2ConnectFailure(t *testing.T) { @@ -250,3 +364,73 @@ func TestS3H2ConnectFailure(t *testing.T) { t.Errorf("status: %v; want %v", got, want) } } + +// TestS3H2ReqPhaseReturn tests mruby request phase hook returns +// custom response. +func TestS3H2ReqPhaseReturn(t *testing.T) { + st := newServerTesterTLS([]string{"--npn-list=spdy/3.1", "--http2-bridge", "--request-phase-file=" + testDir + "/return.rb"}, t, func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("request should not be forwarded") + }) + defer st.Close() + + res, err := st.spdy(requestParam{ + name: "TestS3H2ReqPhaseReturn", + }) + if err != nil { + t.Fatalf("Error st.spdy() = %v", err) + } + + if got, want := res.status, 404; got != want { + t.Errorf("status = %v; want %v", got, want) + } + + hdtests := []struct { + k, v string + }{ + {"content-length", "11"}, + {"from", "mruby"}, + } + for _, tt := range hdtests { + if got, want := res.header.Get(tt.k), tt.v; got != want { + t.Errorf("%v = %v; want %v", tt.k, got, want) + } + } + + if got, want := string(res.body), "Hello World"; got != want { + t.Errorf("body = %v; want %v", got, want) + } +} + +// TestS3H2RespPhaseReturn tests mruby response phase hook returns +// custom response. +func TestS3H2RespPhaseReturn(t *testing.T) { + st := newServerTesterTLS([]string{"--npn-list=spdy/3.1", "--http2-bridge", "--response-phase-file=" + testDir + "/return.rb"}, t, noopHandler) + defer st.Close() + + res, err := st.spdy(requestParam{ + name: "TestS3H2RespPhaseReturn", + }) + if err != nil { + t.Fatalf("Error st.spdy() = %v", err) + } + + if got, want := res.status, 404; got != want { + t.Errorf("status = %v; want %v", got, want) + } + + hdtests := []struct { + k, v string + }{ + {"content-length", "11"}, + {"from", "mruby"}, + } + for _, tt := range hdtests { + if got, want := res.header.Get(tt.k), tt.v; got != want { + t.Errorf("%v = %v; want %v", tt.k, got, want) + } + } + + if got, want := string(res.body), "Hello World"; got != want { + t.Errorf("body = %v; want %v", got, want) + } +} diff --git a/integration-tests/req-set-header.rb b/integration-tests/req-set-header.rb new file mode 100644 index 00000000..6f63e357 --- /dev/null +++ b/integration-tests/req-set-header.rb @@ -0,0 +1,3 @@ +Nghttpx.run do |resp, req| + req.set_header "User-Agent", "mruby" +end diff --git a/integration-tests/resp-set-header.rb b/integration-tests/resp-set-header.rb new file mode 100644 index 00000000..a1f2d806 --- /dev/null +++ b/integration-tests/resp-set-header.rb @@ -0,0 +1,3 @@ +Nghttpx.run do |resp, req| + resp.set_header "Alpha", "bravo" +end diff --git a/integration-tests/return.rb b/integration-tests/return.rb new file mode 100644 index 00000000..42b1e345 --- /dev/null +++ b/integration-tests/return.rb @@ -0,0 +1,6 @@ +Nghttpx.run do |resp, req| + resp.clear_headers + resp.status = 404 + resp.add_header "from", "mruby" + resp.return "Hello World" +end From d8ca0a8d4712b1e2f0df7a03fe45c115f73e8655 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 4 Sep 2015 02:53:02 +0900 Subject: [PATCH 26/55] nghttpx: Add clear_headers and add_header methods to Request and Response --- src/shrpx_mruby_module_request.cc | 41 +++++++++++++++++++++++++----- src/shrpx_mruby_module_response.cc | 41 +++++++++++++++++++++++++----- 2 files changed, 70 insertions(+), 12 deletions(-) diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index e1fb1edb..8f883b60 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -184,7 +184,7 @@ mrb_value request_get_headers(mrb_state *mrb, mrb_value self) { } // namespace namespace { -mrb_value request_set_header(mrb_state *mrb, mrb_value self) { +mrb_value request_mod_header(mrb_state *mrb, mrb_value self, bool repl) { auto data = static_cast(mrb->ud); auto downstream = data->downstream; @@ -197,11 +197,13 @@ mrb_value request_set_header(mrb_state *mrb, mrb_value self) { key = mrb_funcall(mrb, key, "downcase", 0); - // making name empty will effectively delete header fields - for (auto &hd : downstream->get_request_headers()) { - if (util::streq(std::begin(hd.name), hd.name.size(), RSTRING_PTR(key), - RSTRING_LEN(key))) { - hd.name = ""; + if (repl) { + // making name empty will effectively delete header fields + for (auto &hd : downstream->get_request_headers()) { + if (util::streq(std::begin(hd.name), hd.name.size(), RSTRING_PTR(key), + RSTRING_LEN(key))) { + hd.name = ""; + } } } @@ -225,6 +227,29 @@ mrb_value request_set_header(mrb_state *mrb, mrb_value self) { } } // namespace +namespace { +mrb_value request_set_header(mrb_state *mrb, mrb_value self) { + return request_mod_header(mrb, self, true); +} +} // namespace + +namespace { +mrb_value request_add_header(mrb_state *mrb, mrb_value self) { + return request_mod_header(mrb, self, false); +} +} // namespace + +namespace { +mrb_value request_clear_headers(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + + downstream->clear_request_headers(); + + return mrb_nil_value(); +} +} // namespace + namespace { mrb_value request_get_remote_addr(mrb_state *mrb, mrb_value self) { auto data = static_cast(mrb->ud); @@ -266,8 +291,12 @@ void init_request_class(mrb_state *mrb, RClass *module) { MRB_ARGS_REQ(1)); mrb_define_method(mrb, request_class, "headers", request_get_headers, MRB_ARGS_NONE()); + mrb_define_method(mrb, request_class, "add_header", request_add_header, + MRB_ARGS_REQ(2)); mrb_define_method(mrb, request_class, "set_header", request_set_header, MRB_ARGS_REQ(2)); + mrb_define_method(mrb, request_class, "clear_headers", request_clear_headers, + MRB_ARGS_NONE()); mrb_define_method(mrb, request_class, "remote_addr", request_get_remote_addr, MRB_ARGS_NONE()); } diff --git a/src/shrpx_mruby_module_response.cc b/src/shrpx_mruby_module_response.cc index 5398ccfb..55a8d702 100644 --- a/src/shrpx_mruby_module_response.cc +++ b/src/shrpx_mruby_module_response.cc @@ -98,7 +98,7 @@ mrb_value response_get_headers(mrb_state *mrb, mrb_value self) { } // namespace namespace { -mrb_value response_set_header(mrb_state *mrb, mrb_value self) { +mrb_value response_mod_header(mrb_state *mrb, mrb_value self, bool repl) { auto data = static_cast(mrb->ud); auto downstream = data->downstream; @@ -111,11 +111,13 @@ mrb_value response_set_header(mrb_state *mrb, mrb_value self) { key = mrb_funcall(mrb, key, "downcase", 0); - // making name empty will effectively delete header fields - for (auto &hd : downstream->get_response_headers()) { - if (util::streq(std::begin(hd.name), hd.name.size(), RSTRING_PTR(key), - RSTRING_LEN(key))) { - hd.name = ""; + if (repl) { + // making name empty will effectively delete header fields + for (auto &hd : downstream->get_response_headers()) { + if (util::streq(std::begin(hd.name), hd.name.size(), RSTRING_PTR(key), + RSTRING_LEN(key))) { + hd.name = ""; + } } } @@ -139,6 +141,29 @@ mrb_value response_set_header(mrb_state *mrb, mrb_value self) { } } // namespace +namespace { +mrb_value response_set_header(mrb_state *mrb, mrb_value self) { + return response_mod_header(mrb, self, true); +} +} // namespace + +namespace { +mrb_value response_add_header(mrb_state *mrb, mrb_value self) { + return response_mod_header(mrb, self, false); +} +} // namespace + +namespace { +mrb_value response_clear_headers(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + + downstream->clear_response_headers(); + + return mrb_nil_value(); +} +} // namespace + namespace { mrb_value response_return(mrb_state *mrb, mrb_value self) { auto data = static_cast(mrb->ud); @@ -209,8 +234,12 @@ void init_response_class(mrb_state *mrb, RClass *module) { MRB_ARGS_REQ(1)); mrb_define_method(mrb, response_class, "headers", response_get_headers, MRB_ARGS_NONE()); + mrb_define_method(mrb, response_class, "add_header", response_add_header, + MRB_ARGS_REQ(2)); mrb_define_method(mrb, response_class, "set_header", response_set_header, MRB_ARGS_REQ(2)); + mrb_define_method(mrb, response_class, "clear_headers", + response_clear_headers, MRB_ARGS_NONE()); mrb_define_method(mrb, response_class, "return", response_return, MRB_ARGS_OPT(1)); } From 4872c9b4c9e6e960628083769e8b43500cf646d5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 4 Sep 2015 22:10:49 +0900 Subject: [PATCH 27/55] Use C++ compiler to detect mruby since we have to enable C++ ABI mode Since default Debian/Ubuntu mruby package does not enable C++ ABI mode, it cannot be used with our C++ code, since mruby exception does not properly cleanup C++ object allocated on stack. We have to build mruby with conf.enable_cxx_abi explicitly. It is safe for us to build our own mruby. --- configure.ac | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configure.ac b/configure.ac index 18838c53..bfdbc554 100644 --- a/configure.ac +++ b/configure.ac @@ -376,6 +376,7 @@ fi AM_CONDITIONAL([HAVE_SPDYLAY], [ test "x${have_spdylay}" = "xyes" ]) # mruby (for src/nghttpx) +AC_LANG_PUSH(C++) LIBS_OLD=$LIBS have_mruby=no if test "x${request_mruby}" != "xno"; then @@ -392,6 +393,7 @@ if test "x${request_mruby}" != "xno"; then fi fi LIBS=$LIBS_OLD +AC_LANG_POP() if test "x${request_mruby}" = "xyes" && test "x${have_mruby}" != "xyes"; then From a9338f1c0efa37d65a04fc17280b9fa0aa161d21 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 4 Sep 2015 22:34:40 +0900 Subject: [PATCH 28/55] nghttpx: Fix hang when error page is sent on response phase hook error --- src/shrpx_http2_upstream.cc | 1 + src/shrpx_spdy_upstream.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 3c0c8d28..4c4ef0cc 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -929,6 +929,7 @@ int Http2Upstream::downstream_read(DownstreamConnection *dconn) { } if (rv == SHRPX_ERR_DCONN_CANCELED) { downstream->pop_downstream_connection(); + handler_->signal_write(); return 0; } if (rv != 0) { diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index f8d59ea1..b3ef81be 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -602,6 +602,7 @@ int SpdyUpstream::downstream_read(DownstreamConnection *dconn) { } if (rv == SHRPX_ERR_DCONN_CANCELED) { downstream->pop_downstream_connection(); + handler_->signal_write(); return 0; } if (rv != 0) { From bddc4a0a04e2296a8ae9a2e5d2069f8f24a85869 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Fri, 4 Sep 2015 23:56:07 +0900 Subject: [PATCH 29/55] nghttpx: Rewrite API; introduce Env object --- src/Makefile.am | 1 + src/shrpx_downstream.cc | 11 ++++ src/shrpx_mruby.cc | 20 +++++- src/shrpx_mruby.h | 5 ++ src/shrpx_mruby_module.cc | 36 +++++++++-- src/shrpx_mruby_module.h | 4 ++ src/shrpx_mruby_module_env.cc | 104 ++++++++++++++++++++++++++++++ src/shrpx_mruby_module_env.h | 44 +++++++++++++ src/shrpx_mruby_module_request.cc | 15 ----- 9 files changed, 218 insertions(+), 22 deletions(-) create mode 100644 src/shrpx_mruby_module_env.cc create mode 100644 src/shrpx_mruby_module_env.h diff --git a/src/Makefile.am b/src/Makefile.am index 52f50491..5751490b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -134,6 +134,7 @@ if HAVE_MRUBY NGHTTPX_SRCS += \ shrpx_mruby.cc shrpx_mruby.h \ shrpx_mruby_module.cc shrpx_mruby_module.h \ + shrpx_mruby_module_env.cc shrpx_mruby_module_env.h \ shrpx_mruby_module_request.cc shrpx_mruby_module_request.h \ shrpx_mruby_module_response.cc shrpx_mruby_module_response.h endif # HAVE_MRUBY diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 5948b1db..29eb5b3b 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -34,6 +34,11 @@ #include "shrpx_error.h" #include "shrpx_downstream_connection.h" #include "shrpx_downstream_queue.h" +#include "shrpx_worker.h" +#include "shrpx_http2_session.h" +#ifdef HAVE_MRUBY +#include "shrpx_mruby.h" +#endif // HAVE_MRUBY #include "util.h" #include "http2.h" @@ -160,6 +165,12 @@ Downstream::~Downstream() { ev_timer_stop(loop, &upstream_wtimer_); ev_timer_stop(loop, &downstream_rtimer_); ev_timer_stop(loop, &downstream_wtimer_); + + auto handler = upstream_->get_client_handler(); + auto worker = handler->get_worker(); + auto mruby_ctx = worker->get_mruby_context(); + + mruby_ctx->delete_downstream(this); } // DownstreamConnection may refer to this object. Delete it now diff --git a/src/shrpx_mruby.cc b/src/shrpx_mruby.cc index f2f23512..1ca5c0f8 100644 --- a/src/shrpx_mruby.cc +++ b/src/shrpx_mruby.cc @@ -99,6 +99,10 @@ int MRubyContext::run_on_response_proc(Downstream *downstream) { return run_request_proc(downstream, on_response_proc_); } +void MRubyContext::delete_downstream(Downstream *downstream) { + delete_downstream_from_module(mrb_, downstream); +} + // Based on // https://github.com/h2o/h2o/blob/master/lib/handler/mruby.c. It is // very hard to write these kind of code because mruby has almost no @@ -144,6 +148,13 @@ RProc *compile(mrb_state *mrb, const char *filename) { } std::unique_ptr create_mruby_context() { + auto req_file = get_config()->request_phase_file.get(); + auto res_file = get_config()->response_phase_file.get(); + + if (!req_file && !res_file) { + return make_unique(nullptr, nullptr, nullptr); + } + auto mrb = mrb_open(); if (mrb == nullptr) { LOG(ERROR) << "mrb_open failed"; @@ -152,9 +163,6 @@ std::unique_ptr create_mruby_context() { init_module(mrb); - auto req_file = get_config()->request_phase_file.get(); - auto res_file = get_config()->response_phase_file.get(); - auto req_proc = compile(mrb, req_file); if (req_file && !req_proc) { @@ -174,6 +182,12 @@ std::unique_ptr create_mruby_context() { return make_unique(mrb, req_proc, res_proc); } +mrb_sym intern_ptr(mrb_state *mrb, void *ptr) { + auto p = reinterpret_cast(ptr); + + return mrb_intern(mrb, reinterpret_cast(&p), sizeof(p)); +} + } // namespace mruby } // namespace shrpx diff --git a/src/shrpx_mruby.h b/src/shrpx_mruby.h index 2588cfb9..24a9e6cd 100644 --- a/src/shrpx_mruby.h +++ b/src/shrpx_mruby.h @@ -48,6 +48,8 @@ public: int run_request_proc(Downstream *downstream, RProc *proc); + void delete_downstream(Downstream *downstream); + private: mrb_state *mrb_; RProc *on_request_proc_; @@ -65,6 +67,9 @@ RProc *compile(mrb_state *mrb, const char *filename); std::unique_ptr create_mruby_context(); +// Return interned |ptr|. +mrb_sym intern_ptr(mrb_state *mrb, void *ptr); + } // namespace mruby } // namespace shrpx diff --git a/src/shrpx_mruby_module.cc b/src/shrpx_mruby_module.cc index a122ace5..16fba07b 100644 --- a/src/shrpx_mruby_module.cc +++ b/src/shrpx_mruby_module.cc @@ -32,6 +32,7 @@ #include #include "shrpx_mruby.h" +#include "shrpx_mruby_module_env.h" #include "shrpx_mruby_module_request.h" #include "shrpx_mruby_module_response.h" @@ -49,21 +50,48 @@ mrb_value run(mrb_state *mrb, mrb_value self) { } auto module = mrb_module_get(mrb, "Nghttpx"); - auto request_class = mrb_class_get_under(mrb, module, "Request"); - auto response_class = mrb_class_get_under(mrb, module, "Response"); - std::array args{{mrb_obj_new(mrb, response_class, 0, nullptr), - mrb_obj_new(mrb, request_class, 0, nullptr)}}; + auto env_sym = mrb_intern_lit(mrb, "env"); + auto env = mrb_obj_iv_get(mrb, reinterpret_cast(module), env_sym); + + if (mrb_nil_p(env)) { + auto env_class = mrb_class_get_under(mrb, module, "Env"); + auto request_class = mrb_class_get_under(mrb, module, "Request"); + auto response_class = mrb_class_get_under(mrb, module, "Response"); + + env = mrb_obj_new(mrb, env_class, 0, nullptr); + auto req = mrb_obj_new(mrb, request_class, 0, nullptr); + auto resp = mrb_obj_new(mrb, response_class, 0, nullptr); + + mrb_iv_set(mrb, env, mrb_intern_lit(mrb, "req"), req); + mrb_iv_set(mrb, env, mrb_intern_lit(mrb, "resp"), resp); + + mrb_obj_iv_set(mrb, reinterpret_cast(module), env_sym, env); + } + + std::array args{{env}}; return mrb_yield_argv(mrb, b, args.size(), args.data()); } } // namespace +void delete_downstream_from_module(mrb_state *mrb, Downstream *downstream) { + auto module = mrb_module_get(mrb, "Nghttpx"); + auto env = mrb_obj_iv_get(mrb, reinterpret_cast(module), + mrb_intern_lit(mrb, "env")); + if (mrb_nil_p(env)) { + return; + } + + mrb_iv_remove(mrb, env, intern_ptr(mrb, downstream)); +} + void init_module(mrb_state *mrb) { auto module = mrb_define_module(mrb, "Nghttpx"); mrb_define_class_method(mrb, module, "run", run, MRB_ARGS_REQ(1) | MRB_ARGS_BLOCK()); + init_env_class(mrb, module); init_request_class(mrb, module); init_response_class(mrb, module); } diff --git a/src/shrpx_mruby_module.h b/src/shrpx_mruby_module.h index f4e300a4..8d5274e2 100644 --- a/src/shrpx_mruby_module.h +++ b/src/shrpx_mruby_module.h @@ -35,10 +35,14 @@ using namespace nghttp2; namespace shrpx { +class Downstream; + namespace mruby { void init_module(mrb_state *mrb); +void delete_downstream_from_module(mrb_state *mrb, Downstream *downstream); + mrb_value create_headers_hash(mrb_state *mrb, const Headers &headers); } // namespace mruby diff --git a/src/shrpx_mruby_module_env.cc b/src/shrpx_mruby_module_env.cc new file mode 100644 index 00000000..a5237b4b --- /dev/null +++ b/src/shrpx_mruby_module_env.cc @@ -0,0 +1,104 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2015 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#include "shrpx_mruby_module_env.h" + +#include +#include +#include +#include + +#include "shrpx_downstream.h" +#include "shrpx_upstream.h" +#include "shrpx_client_handler.h" +#include "shrpx_mruby.h" +#include "shrpx_mruby_module.h" +#include "util.h" +#include "http2.h" + +namespace shrpx { + +namespace mruby { + +namespace { +mrb_value env_init(mrb_state *mrb, mrb_value self) { return self; } +} // namespace + +namespace { +mrb_value env_get_req(mrb_state *mrb, mrb_value self) { + return mrb_iv_get(mrb, self, mrb_intern_lit(mrb, "req")); +} +} // namespace + +namespace { +mrb_value env_get_resp(mrb_state *mrb, mrb_value self) { + return mrb_iv_get(mrb, self, mrb_intern_lit(mrb, "resp")); +} +} // namespace + +namespace { +mrb_value env_get_ctx(mrb_state *mrb, mrb_value self) { + auto data = reinterpret_cast(mrb->ud); + auto downstream = data->downstream; + + auto dsym = intern_ptr(mrb, downstream); + + auto ctx = mrb_iv_get(mrb, self, dsym); + if (mrb_nil_p(ctx)) { + ctx = mrb_hash_new(mrb); + mrb_iv_set(mrb, self, dsym, ctx); + } + + return ctx; +} +} // namespace + +namespace { +mrb_value env_get_remote_addr(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + auto upstream = downstream->get_upstream(); + auto handler = upstream->get_client_handler(); + + auto &ipaddr = handler->get_ipaddr(); + + return mrb_str_new(mrb, ipaddr.c_str(), ipaddr.size()); +} +} // namespace + +void init_env_class(mrb_state *mrb, RClass *module) { + auto env_class = + mrb_define_class_under(mrb, module, "Env", mrb->object_class); + + mrb_define_method(mrb, env_class, "initialize", env_init, MRB_ARGS_NONE()); + mrb_define_method(mrb, env_class, "req", env_get_req, MRB_ARGS_NONE()); + mrb_define_method(mrb, env_class, "resp", env_get_resp, MRB_ARGS_NONE()); + mrb_define_method(mrb, env_class, "ctx", env_get_ctx, MRB_ARGS_NONE()); + mrb_define_method(mrb, env_class, "remote_addr", env_get_remote_addr, + MRB_ARGS_NONE()); +} + +} // namespace mruby + +} // namespace shrpx diff --git a/src/shrpx_mruby_module_env.h b/src/shrpx_mruby_module_env.h new file mode 100644 index 00000000..7515e16e --- /dev/null +++ b/src/shrpx_mruby_module_env.h @@ -0,0 +1,44 @@ +/* + * nghttp2 - HTTP/2 C Library + * + * Copyright (c) 2015 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef SHRPX_MRUBY_MODULE_ENV_H +#define SHRPX_MRUBY_MODULE_ENV_H + +#include "shrpx.h" + +#include + +using namespace nghttp2; + +namespace shrpx { + +namespace mruby { + +void init_env_class(mrb_state *mrb, RClass *module); + +} // namespace mruby + +} // namespace shrpx + +#endif // SHRPX_MRUBY_MODULE_ENV_H diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index 8f883b60..eea4f0ee 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -250,19 +250,6 @@ mrb_value request_clear_headers(mrb_state *mrb, mrb_value self) { } } // namespace -namespace { -mrb_value request_get_remote_addr(mrb_state *mrb, mrb_value self) { - auto data = static_cast(mrb->ud); - auto downstream = data->downstream; - auto upstream = downstream->get_upstream(); - auto handler = upstream->get_client_handler(); - - auto &ipaddr = handler->get_ipaddr(); - - return mrb_str_new(mrb, ipaddr.c_str(), ipaddr.size()); -} -} // namespace - void init_request_class(mrb_state *mrb, RClass *module) { auto request_class = mrb_define_class_under(mrb, module, "Request", mrb->object_class); @@ -297,8 +284,6 @@ void init_request_class(mrb_state *mrb, RClass *module) { MRB_ARGS_REQ(2)); mrb_define_method(mrb, request_class, "clear_headers", request_clear_headers, MRB_ARGS_NONE()); - mrb_define_method(mrb, request_class, "remote_addr", request_get_remote_addr, - MRB_ARGS_NONE()); } } // namespace mruby From 45e0d42c51bf847be676357d4a817847b285c639 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 01:07:29 +0900 Subject: [PATCH 30/55] nghttpx: Check allowed operations per phase --- src/shrpx_mruby.cc | 15 +++++++++++---- src/shrpx_mruby.h | 13 ++++++++++++- src/shrpx_mruby_module.cc | 4 ++++ src/shrpx_mruby_module_request.cc | 12 ++++++++++++ 4 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/shrpx_mruby.cc b/src/shrpx_mruby.cc index 1ca5c0f8..79ba148a 100644 --- a/src/shrpx_mruby.cc +++ b/src/shrpx_mruby.cc @@ -44,14 +44,15 @@ MRubyContext::MRubyContext(mrb_state *mrb, RProc *on_request_proc, MRubyContext::~MRubyContext() { mrb_close(mrb_); } -int MRubyContext::run_request_proc(Downstream *downstream, RProc *proc) { +int MRubyContext::run_request_proc(Downstream *downstream, RProc *proc, + int phase) { if (!proc || running_) { return 0; } running_ = true; - MRubyAssocData data{downstream}; + MRubyAssocData data{downstream, phase}; mrb_->ud = &data; @@ -92,11 +93,11 @@ int MRubyContext::run_request_proc(Downstream *downstream, RProc *proc) { } int MRubyContext::run_on_request_proc(Downstream *downstream) { - return run_request_proc(downstream, on_request_proc_); + return run_request_proc(downstream, on_request_proc_, PHASE_REQUEST); } int MRubyContext::run_on_response_proc(Downstream *downstream) { - return run_request_proc(downstream, on_response_proc_); + return run_request_proc(downstream, on_response_proc_, PHASE_RESPONSE); } void MRubyContext::delete_downstream(Downstream *downstream) { @@ -188,6 +189,12 @@ mrb_sym intern_ptr(mrb_state *mrb, void *ptr) { return mrb_intern(mrb, reinterpret_cast(&p), sizeof(p)); } +void check_phase(mrb_state *mrb, int phase, int phase_mask) { + if ((phase & phase_mask) == 0) { + mrb_raise(mrb, E_RUNTIME_ERROR, "operation was not allowed in this phase"); + } +} + } // namespace mruby } // namespace shrpx diff --git a/src/shrpx_mruby.h b/src/shrpx_mruby.h index 24a9e6cd..a75e4975 100644 --- a/src/shrpx_mruby.h +++ b/src/shrpx_mruby.h @@ -46,7 +46,7 @@ public: int run_on_request_proc(Downstream *downstream); int run_on_response_proc(Downstream *downstream); - int run_request_proc(Downstream *downstream, RProc *proc); + int run_request_proc(Downstream *downstream, RProc *proc, int phase); void delete_downstream(Downstream *downstream); @@ -57,8 +57,15 @@ private: bool running_; }; +enum { + PHASE_NONE = 0, + PHASE_REQUEST = 1, + PHASE_RESPONSE = 1 << 1, +}; + struct MRubyAssocData { Downstream *downstream; + int phase; bool request_headers_dirty; bool response_headers_dirty; }; @@ -70,6 +77,10 @@ std::unique_ptr create_mruby_context(); // Return interned |ptr|. mrb_sym intern_ptr(mrb_state *mrb, void *ptr); +// Checks that |phase| is set in |phase_mask|. If not set, raise +// exception. +void check_phase(mrb_state *mrb, int phase, int phase_mask); + } // namespace mruby } // namespace shrpx diff --git a/src/shrpx_mruby_module.cc b/src/shrpx_mruby_module.cc index 16fba07b..3beb2ca6 100644 --- a/src/shrpx_mruby_module.cc +++ b/src/shrpx_mruby_module.cc @@ -90,6 +90,10 @@ void init_module(mrb_state *mrb) { mrb_define_class_method(mrb, module, "run", run, MRB_ARGS_REQ(1) | MRB_ARGS_BLOCK()); + mrb_define_const(mrb, module, "REQUEST_PHASE", + mrb_fixnum_value(PHASE_REQUEST)); + mrb_define_const(mrb, module, "RESPONSE_PHASE", + mrb_fixnum_value(PHASE_RESPONSE)); init_env_class(mrb, module); init_request_class(mrb, module); diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index eea4f0ee..0b8ead0a 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -76,6 +76,8 @@ mrb_value request_set_method(mrb_state *mrb, mrb_value self) { auto data = static_cast(mrb->ud); auto downstream = data->downstream; + check_phase(mrb, data->phase, PHASE_REQUEST); + const char *method; mrb_int n; mrb_get_args(mrb, "s", &method, &n); @@ -109,6 +111,8 @@ mrb_value request_set_authority(mrb_state *mrb, mrb_value self) { auto data = static_cast(mrb->ud); auto downstream = data->downstream; + check_phase(mrb, data->phase, PHASE_REQUEST); + const char *authority; mrb_int n; mrb_get_args(mrb, "s", &authority, &n); @@ -137,6 +141,8 @@ mrb_value request_set_scheme(mrb_state *mrb, mrb_value self) { auto data = static_cast(mrb->ud); auto downstream = data->downstream; + check_phase(mrb, data->phase, PHASE_REQUEST); + const char *scheme; mrb_int n; mrb_get_args(mrb, "s", &scheme, &n); @@ -165,6 +171,8 @@ mrb_value request_set_path(mrb_state *mrb, mrb_value self) { auto data = static_cast(mrb->ud); auto downstream = data->downstream; + check_phase(mrb, data->phase, PHASE_REQUEST); + const char *path; mrb_int pathlen; mrb_get_args(mrb, "s", &path, &pathlen); @@ -188,6 +196,8 @@ mrb_value request_mod_header(mrb_state *mrb, mrb_value self, bool repl) { auto data = static_cast(mrb->ud); auto downstream = data->downstream; + check_phase(mrb, data->phase, PHASE_REQUEST); + mrb_value key, values; mrb_get_args(mrb, "oo", &key, &values); @@ -244,6 +254,8 @@ mrb_value request_clear_headers(mrb_state *mrb, mrb_value self) { auto data = static_cast(mrb->ud); auto downstream = data->downstream; + check_phase(mrb, data->phase, PHASE_REQUEST); + downstream->clear_request_headers(); return mrb_nil_value(); From 97f488a5c7265ec7ecdea095d098fdac60e8b518 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 01:11:20 +0900 Subject: [PATCH 31/55] nghttpx: Add Env.phase method --- src/shrpx_mruby_module_env.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/shrpx_mruby_module_env.cc b/src/shrpx_mruby_module_env.cc index a5237b4b..1ac7a243 100644 --- a/src/shrpx_mruby_module_env.cc +++ b/src/shrpx_mruby_module_env.cc @@ -74,6 +74,14 @@ mrb_value env_get_ctx(mrb_state *mrb, mrb_value self) { } } // namespace +namespace { +mrb_value env_get_phase(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + + return mrb_fixnum_value(data->phase); +} +} // namespace + namespace { mrb_value env_get_remote_addr(mrb_state *mrb, mrb_value self) { auto data = static_cast(mrb->ud); @@ -95,6 +103,7 @@ void init_env_class(mrb_state *mrb, RClass *module) { mrb_define_method(mrb, env_class, "req", env_get_req, MRB_ARGS_NONE()); mrb_define_method(mrb, env_class, "resp", env_get_resp, MRB_ARGS_NONE()); mrb_define_method(mrb, env_class, "ctx", env_get_ctx, MRB_ARGS_NONE()); + mrb_define_method(mrb, env_class, "phase", env_get_phase, MRB_ARGS_NONE()); mrb_define_method(mrb, env_class, "remote_addr", env_get_remote_addr, MRB_ARGS_NONE()); } From 587f37a59749b0370bce11fbd06b0096910cbfa7 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 01:12:51 +0900 Subject: [PATCH 32/55] nghttpx: Remove unused includes --- src/shrpx_mruby_module_env.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/shrpx_mruby_module_env.cc b/src/shrpx_mruby_module_env.cc index 1ac7a243..1fb75781 100644 --- a/src/shrpx_mruby_module_env.cc +++ b/src/shrpx_mruby_module_env.cc @@ -27,15 +27,12 @@ #include #include #include -#include #include "shrpx_downstream.h" #include "shrpx_upstream.h" #include "shrpx_client_handler.h" #include "shrpx_mruby.h" #include "shrpx_mruby_module.h" -#include "util.h" -#include "http2.h" namespace shrpx { From 226a09b04a0e007041165351eb2dc8a5a662a90e Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 01:14:17 +0900 Subject: [PATCH 33/55] nghttpx: Use gc arena save/restore when creating headers hash --- src/shrpx_mruby_module.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/shrpx_mruby_module.cc b/src/shrpx_mruby_module.cc index 3beb2ca6..e5e397ee 100644 --- a/src/shrpx_mruby_module.cc +++ b/src/shrpx_mruby_module.cc @@ -107,6 +107,8 @@ mrb_value create_headers_hash(mrb_state *mrb, const Headers &headers) { if (hd.name.empty() || hd.name[0] == ':') { continue; } + auto ai = mrb_gc_arena_save(mrb); + auto key = mrb_str_new(mrb, hd.name.c_str(), hd.name.size()); auto ary = mrb_hash_get(mrb, hash, key); if (mrb_nil_p(ary)) { @@ -114,6 +116,8 @@ mrb_value create_headers_hash(mrb_state *mrb, const Headers &headers) { mrb_hash_set(mrb, hash, key, ary); } mrb_ary_push(mrb, ary, mrb_str_new(mrb, hd.value.c_str(), hd.value.size())); + + mrb_gc_arena_restore(mrb, ai); } return hash; From 21e1af2ae7616e4f63928431bb4b28e29ae8cc79 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 01:18:18 +0900 Subject: [PATCH 34/55] nghttpx: Delete replaced header fields --- src/shrpx_mruby_module_request.cc | 12 +++++++++--- src/shrpx_mruby_module_response.cc | 12 +++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index 0b8ead0a..74dad27a 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -208,13 +208,19 @@ mrb_value request_mod_header(mrb_state *mrb, mrb_value self, bool repl) { key = mrb_funcall(mrb, key, "downcase", 0); if (repl) { - // making name empty will effectively delete header fields - for (auto &hd : downstream->get_request_headers()) { + size_t p = 0; + auto &headers = downstream->get_request_headers(); + for (size_t i = 0; i < headers.size(); ++i) { + auto &hd = headers[i]; if (util::streq(std::begin(hd.name), hd.name.size(), RSTRING_PTR(key), RSTRING_LEN(key))) { - hd.name = ""; + continue; + } + if (i != p) { + headers[p++] = std::move(hd); } } + headers.resize(p); } if (mrb_obj_is_instance_of(mrb, values, mrb->array_class)) { diff --git a/src/shrpx_mruby_module_response.cc b/src/shrpx_mruby_module_response.cc index 55a8d702..0b49e925 100644 --- a/src/shrpx_mruby_module_response.cc +++ b/src/shrpx_mruby_module_response.cc @@ -112,13 +112,19 @@ mrb_value response_mod_header(mrb_state *mrb, mrb_value self, bool repl) { key = mrb_funcall(mrb, key, "downcase", 0); if (repl) { - // making name empty will effectively delete header fields - for (auto &hd : downstream->get_response_headers()) { + size_t p = 0; + auto &headers = downstream->get_response_headers(); + for (size_t i = 0; i < headers.size(); ++i) { + auto &hd = headers[i]; if (util::streq(std::begin(hd.name), hd.name.size(), RSTRING_PTR(key), RSTRING_LEN(key))) { - hd.name = ""; + continue; + } + if (i != p) { + headers[p++] = std::move(hd); } } + headers.resize(p); } if (mrb_obj_is_instance_of(mrb, values, mrb->array_class)) { From b4709b9e8c5cbd5b98271613158cd9965bdf45e6 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 01:20:32 +0900 Subject: [PATCH 35/55] nghttpx: Fix crash without mruby script --- src/shrpx_mruby.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/shrpx_mruby.cc b/src/shrpx_mruby.cc index 79ba148a..aadca674 100644 --- a/src/shrpx_mruby.cc +++ b/src/shrpx_mruby.cc @@ -101,6 +101,9 @@ int MRubyContext::run_on_response_proc(Downstream *downstream) { } void MRubyContext::delete_downstream(Downstream *downstream) { + if (!mrb_) { + return; + } delete_downstream_from_module(mrb_, downstream); } From d348ea338468e58b1ada0f20af5a76c69a33ae86 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 01:24:43 +0900 Subject: [PATCH 36/55] integration: Fix test failures --- integration-tests/req-set-header.rb | 4 ++-- integration-tests/resp-set-header.rb | 4 ++-- integration-tests/return.rb | 4 +++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/integration-tests/req-set-header.rb b/integration-tests/req-set-header.rb index 6f63e357..27e9dc34 100644 --- a/integration-tests/req-set-header.rb +++ b/integration-tests/req-set-header.rb @@ -1,3 +1,3 @@ -Nghttpx.run do |resp, req| - req.set_header "User-Agent", "mruby" +Nghttpx.run do |env| + env.req.set_header "User-Agent", "mruby" end diff --git a/integration-tests/resp-set-header.rb b/integration-tests/resp-set-header.rb index a1f2d806..b947ce3b 100644 --- a/integration-tests/resp-set-header.rb +++ b/integration-tests/resp-set-header.rb @@ -1,3 +1,3 @@ -Nghttpx.run do |resp, req| - resp.set_header "Alpha", "bravo" +Nghttpx.run do |env| + env.resp.set_header "Alpha", "bravo" end diff --git a/integration-tests/return.rb b/integration-tests/return.rb index 42b1e345..907d837f 100644 --- a/integration-tests/return.rb +++ b/integration-tests/return.rb @@ -1,4 +1,6 @@ -Nghttpx.run do |resp, req| +Nghttpx.run do |env| + resp = env.resp + resp.clear_headers resp.status = 404 resp.add_header "from", "mruby" From 4ac7152f94c5735dcb4dd39726b904b09c445e5a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 01:32:26 +0900 Subject: [PATCH 37/55] nghttpx: Fix build failure without mruby --- src/shrpx_downstream.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/shrpx_downstream.cc b/src/shrpx_downstream.cc index 29eb5b3b..eefa99f2 100644 --- a/src/shrpx_downstream.cc +++ b/src/shrpx_downstream.cc @@ -166,11 +166,13 @@ Downstream::~Downstream() { ev_timer_stop(loop, &downstream_rtimer_); ev_timer_stop(loop, &downstream_wtimer_); +#ifdef HAVE_MRUBY auto handler = upstream_->get_client_handler(); auto worker = handler->get_worker(); auto mruby_ctx = worker->get_mruby_context(); mruby_ctx->delete_downstream(this); +#endif // HAVE_MRUBY } // DownstreamConnection may refer to this object. Delete it now From aba7e9e7f9efc0133dcce543cc668eb0c6929abd Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 17:49:10 +0900 Subject: [PATCH 38/55] Add mruby as submodule; disable mruby by default for now --- .gitmodules | 3 +++ configure.ac | 34 ++++++++++------------------------ src/Makefile.am | 13 +++++++++---- third-party/Makefile.am | 16 ++++++++++++++++ third-party/mruby | 1 + 5 files changed, 39 insertions(+), 28 deletions(-) create mode 100644 .gitmodules create mode 160000 third-party/mruby diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..301b5b3c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "third-party/mruby"] + path = third-party/mruby + url = https://github.com/mruby/mruby diff --git a/configure.ac b/configure.ac index bfdbc554..ce4a575f 100644 --- a/configure.ac +++ b/configure.ac @@ -121,8 +121,8 @@ AC_ARG_WITH([spdylay], AC_ARG_WITH([mruby], [AS_HELP_STRING([--with-mruby], - [Use mruby [default=check]])], - [request_mruby=$withval], [request_mruby=check]) + [Use mruby [default=no]])], + [request_mruby=$withval], [request_mruby=no]) AC_ARG_WITH([cython], [AS_HELP_STRING([--with-cython=PATH], @@ -376,28 +376,14 @@ fi AM_CONDITIONAL([HAVE_SPDYLAY], [ test "x${have_spdylay}" = "xyes" ]) # mruby (for src/nghttpx) -AC_LANG_PUSH(C++) -LIBS_OLD=$LIBS -have_mruby=no -if test "x${request_mruby}" != "xno"; then - AC_CHECK_LIB([mruby], [mrb_open], [have_mruby=yes], [have_mruby=no], [-lm]) - if test "x${have_mruby}" = "xyes"; then - AC_CHECK_HEADER([mruby.h], [have_mruby=yes], [have_mruby=no]) - if test "x${have_mruby}" = "xyes"; then - AC_DEFINE([HAVE_MRUBY], [1], [Define to 1 if you have `mruby` library.]) - LIBMRUBY_LIBS="-lmruby -lm" - LIBMRUBY_CFLAGS= - AC_SUBST([LIBMRUBY_LIBS]) - AC_SUBST([LIBMRUBY_CFLAGS]) - fi - fi -fi -LIBS=$LIBS_OLD -AC_LANG_POP() - -if test "x${request_mruby}" = "xyes" && - test "x${have_mruby}" != "xyes"; then - AC_MSG_ERROR([mruby was requested (--with-mruby) but not found]) +if test "x${request_mruby}" = "xyes"; then + # We are going to build mruby + have_mruby=yes + AC_DEFINE([HAVE_MRUBY], [1], [Define to 1 if you have `mruby` library.]) + LIBMRUBY_LIBS="-lmruby -lm" + LIBMRUBY_CFLAGS= + AC_SUBST([LIBMRUBY_LIBS]) + AC_SUBST([LIBMRUBY_CFLAGS]) fi AM_CONDITIONAL([HAVE_MRUBY], [test "x${have_mruby}" = "xyes"]) diff --git a/src/Makefile.am b/src/Makefile.am index 5751490b..1c3d8b67 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -141,10 +141,13 @@ endif # HAVE_MRUBY noinst_LIBRARIES = libnghttpx.a libnghttpx_a_SOURCES = ${NGHTTPX_SRCS} +libnghttpx_a_CPPFLAGS = ${AM_CPPFLAGS} \ + -I${top_srcdir}/third-party/mruby/include @LIBMRUBY_CFLAGS@ nghttpx_SOURCES = shrpx.cc shrpx.h -nghttpx_CPPFLAGS = ${AM_CPPFLAGS} @LIBMRUBY_CFLAGS@ -nghttpx_LDADD = libnghttpx.a ${LDADD} @LIBMRUBY_LIBS@ +nghttpx_CPPFLAGS = ${libnghttpx_a_CPPFLAGS} +nghttpx_LDADD = libnghttpx.a ${LDADD} \ + -L${top_builddir}/third-party/mruby/build/lib @LIBMRUBY_LIBS@ if HAVE_CUNIT check_PROGRAMS += nghttpx-unittest @@ -158,10 +161,12 @@ nghttpx_unittest_SOURCES = shrpx-unittest.cc \ nghttp2_gzip.c nghttp2_gzip.h \ buffer_test.cc buffer_test.h \ memchunk_test.cc memchunk_test.h -nghttpx_unittest_CPPFLAGS = ${AM_CPPFLAGS} @LIBMRUBY_CFLAGS@ \ +nghttpx_unittest_CPPFLAGS = ${AM_CPPFLAGS} \ + -I${top_srcdir}/third-party/mruby/include @LIBMRUBY_CFLAGS@ \ -DNGHTTP2_TESTS_DIR=\"$(top_srcdir)/tests\" nghttpx_unittest_LDADD = libnghttpx.a ${LDADD} \ - @LIBMRUBY_LIBS@ @CUNIT_LIBS@ @TESTLDADD@ + -L${top_builddir}/third-party/mruby/build/lib @LIBMRUBY_LIBS@ \ + @CUNIT_LIBS@ @TESTLDADD@ TESTS += nghttpx-unittest endif # HAVE_CUNIT diff --git a/third-party/Makefile.am b/third-party/Makefile.am index 8098e32e..92d6fc4b 100644 --- a/third-party/Makefile.am +++ b/third-party/Makefile.am @@ -30,5 +30,21 @@ libhttp_parser_la_SOURCES = \ http-parser/http_parser.c \ http-parser/http_parser.h +.PHONY: all-local clean mruby + +if HAVE_MRUBY + +mruby: + git submodule update --init + MRUBY_CONFIG=${srcdir}/build_config.rb \ + BUILD_DIR=${abs_builddir}/mruby/build \ + ${srcdir}/mruby/minirake -f ${srcdir}/mruby/Rakefile + +all-local: mruby + +clean-local: + -rm -rf ${abs_builddir}/mruby/build + +endif # HAVE_MRUBY endif # ENABLE_THIRD_PARTY diff --git a/third-party/mruby b/third-party/mruby new file mode 160000 index 00000000..1cbbb7e1 --- /dev/null +++ b/third-party/mruby @@ -0,0 +1 @@ +Subproject commit 1cbbb7e11c02d381a6b76aeebae8db0f54ae9baf From d076d54f67c3014dbf1121c589deb7bfcae850f9 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 18:03:05 +0900 Subject: [PATCH 39/55] nghttpx: Don't add mruby related CPPFLAGS and LDFLAGS if it is disabled Setting unused -L really makes libtool get angry --- .travis.yml | 2 +- src/Makefile.am | 23 +++++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 78e69d34..8d974c15 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ before_script: - autoreconf -i - automake - autoconf - - ./configure --enable-werror + - ./configure --enable-werror --with-mruby script: - make - make check diff --git a/src/Makefile.am b/src/Makefile.am index 1c3d8b67..73b0c05d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -141,13 +141,16 @@ endif # HAVE_MRUBY noinst_LIBRARIES = libnghttpx.a libnghttpx_a_SOURCES = ${NGHTTPX_SRCS} -libnghttpx_a_CPPFLAGS = ${AM_CPPFLAGS} \ - -I${top_srcdir}/third-party/mruby/include @LIBMRUBY_CFLAGS@ +libnghttpx_a_CPPFLAGS = ${AM_CPPFLAGS} nghttpx_SOURCES = shrpx.cc shrpx.h nghttpx_CPPFLAGS = ${libnghttpx_a_CPPFLAGS} -nghttpx_LDADD = libnghttpx.a ${LDADD} \ - -L${top_builddir}/third-party/mruby/build/lib @LIBMRUBY_LIBS@ +nghttpx_LDADD = libnghttpx.a ${LDADD} + +if HAVE_MRUBY +nghttpx_CPPFLAGS += -I${top_srcdir}/third-party/mruby/include @LIBMRUBY_CFLAGS@ +nghttpx_LDADD += -L${top_builddir}/third-party/mruby/build/lib @LIBMRUBY_LIBS@ +endif # HAVE_MRUBY if HAVE_CUNIT check_PROGRAMS += nghttpx-unittest @@ -162,11 +165,15 @@ nghttpx_unittest_SOURCES = shrpx-unittest.cc \ buffer_test.cc buffer_test.h \ memchunk_test.cc memchunk_test.h nghttpx_unittest_CPPFLAGS = ${AM_CPPFLAGS} \ - -I${top_srcdir}/third-party/mruby/include @LIBMRUBY_CFLAGS@ \ -DNGHTTP2_TESTS_DIR=\"$(top_srcdir)/tests\" -nghttpx_unittest_LDADD = libnghttpx.a ${LDADD} \ - -L${top_builddir}/third-party/mruby/build/lib @LIBMRUBY_LIBS@ \ - @CUNIT_LIBS@ @TESTLDADD@ +nghttpx_unittest_LDADD = libnghttpx.a ${LDADD} @CUNIT_LIBS@ @TESTLDADD@ + +if HAVE_MRUBY +nghttpx_unittest_CPPFLAGS += \ + -I${top_srcdir}/third-party/mruby/include @LIBMRUBY_CFLAGS@ +nghttpx_unittest_LDADD += \ + -L${top_builddir}/third-party/mruby/build/lib @LIBMRUBY_LIBS@ +endif # HAVE_MRUBY TESTS += nghttpx-unittest endif # HAVE_CUNIT From 6dcfe1c3f84480392391bd7a892142ed044d645f Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 18:10:10 +0900 Subject: [PATCH 40/55] Add missing build_config.rb --- third-party/build_config.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 third-party/build_config.rb diff --git a/third-party/build_config.rb b/third-party/build_config.rb new file mode 100644 index 00000000..7ddd2e9b --- /dev/null +++ b/third-party/build_config.rb @@ -0,0 +1,14 @@ +MRuby::Build.new do |conf| + # TODO use same compilers configured in configure script + toolchain :clang + + # C++ project needs this. Without this, mruby exception does not + # properly destory C++ object allocated on stack. + conf.enable_cxx_abi + + conf.build_dir = ENV['BUILD_DIR'] + + # include the default GEMs + conf.gembox 'default' + conf.gem :core => 'mruby-eval' +end From 69c3e2114a4d583f92219fa865dc2e7f9d12e3c6 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 18:22:46 +0900 Subject: [PATCH 41/55] Again fix mruby build properly; pass CC, CXX and LD to build_config.rb --- src/Makefile.am | 3 ++- third-party/Makefile.am | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index 73b0c05d..e3575669 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -148,7 +148,8 @@ nghttpx_CPPFLAGS = ${libnghttpx_a_CPPFLAGS} nghttpx_LDADD = libnghttpx.a ${LDADD} if HAVE_MRUBY -nghttpx_CPPFLAGS += -I${top_srcdir}/third-party/mruby/include @LIBMRUBY_CFLAGS@ +libnghttpx_a_CPPFLAGS += \ + -I${top_srcdir}/third-party/mruby/include @LIBMRUBY_CFLAGS@ nghttpx_LDADD += -L${top_builddir}/third-party/mruby/build/lib @LIBMRUBY_LIBS@ endif # HAVE_MRUBY diff --git a/third-party/Makefile.am b/third-party/Makefile.am index 92d6fc4b..e90b297a 100644 --- a/third-party/Makefile.am +++ b/third-party/Makefile.am @@ -36,14 +36,15 @@ if HAVE_MRUBY mruby: git submodule update --init - MRUBY_CONFIG=${srcdir}/build_config.rb \ - BUILD_DIR=${abs_builddir}/mruby/build \ - ${srcdir}/mruby/minirake -f ${srcdir}/mruby/Rakefile + MRUBY_CONFIG="${srcdir}/build_config.rb" \ + BUILD_DIR="${abs_builddir}/mruby/build" \ + CC="${CC}" CXX="${CXX}" LD="${LD}" \ + "${srcdir}/mruby/minirake" -f "${srcdir}/mruby/Rakefile" all-local: mruby clean-local: - -rm -rf ${abs_builddir}/mruby/build + -rm -rf "${abs_builddir}/mruby/build" endif # HAVE_MRUBY From be77b47ab5efababe1c2008471e99ecfd1894b84 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 19:10:55 +0900 Subject: [PATCH 42/55] integration: Add ruby scripts to EXTRA_DIST --- integration-tests/Makefile.am | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integration-tests/Makefile.am b/integration-tests/Makefile.am index 3ad95ece..5159e038 100644 --- a/integration-tests/Makefile.am +++ b/integration-tests/Makefile.am @@ -30,7 +30,10 @@ EXTRA_DIST = \ server.crt \ alt-server.key \ alt-server.crt \ - setenv + setenv \ + req-set-header.rb \ + resp-set-header.rb \ + return.rb itprep-local: go get -d -v github.com/bradfitz/http2 From 3029e5c53019042dc54a71e33623df69ea4475bc Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 21:20:03 +0900 Subject: [PATCH 43/55] Bundle sphinxcontrib.rubydomain Source files were downloaded from https://bitbucket.org/birkenfeld/sphinx-contrib/src/default/rubydomain/ --- doc/_exts/sphinxcontrib/LICENSE.rubydomain | 28 + doc/_exts/sphinxcontrib/__init__.py | 14 + doc/_exts/sphinxcontrib/rubydomain.py | 695 +++++++++++++++++++++ 3 files changed, 737 insertions(+) create mode 100644 doc/_exts/sphinxcontrib/LICENSE.rubydomain create mode 100644 doc/_exts/sphinxcontrib/__init__.py create mode 100644 doc/_exts/sphinxcontrib/rubydomain.py diff --git a/doc/_exts/sphinxcontrib/LICENSE.rubydomain b/doc/_exts/sphinxcontrib/LICENSE.rubydomain new file mode 100644 index 00000000..a560d759 --- /dev/null +++ b/doc/_exts/sphinxcontrib/LICENSE.rubydomain @@ -0,0 +1,28 @@ +If not otherwise noted, the extensions in this package are licensed +under the following license. + +Copyright (c) 2010 by the contributors (see AUTHORS file). +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/doc/_exts/sphinxcontrib/__init__.py b/doc/_exts/sphinxcontrib/__init__.py new file mode 100644 index 00000000..b5a7dc29 --- /dev/null +++ b/doc/_exts/sphinxcontrib/__init__.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +""" + sphinxcontrib + ~~~~~~~~~~~~~ + + This package is a namespace package that contains all extensions + distributed in the ``sphinx-contrib`` distribution. + + :copyright: Copyright 2007-2009 by the Sphinx team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +__import__('pkg_resources').declare_namespace(__name__) + diff --git a/doc/_exts/sphinxcontrib/rubydomain.py b/doc/_exts/sphinxcontrib/rubydomain.py new file mode 100644 index 00000000..bba02f5d --- /dev/null +++ b/doc/_exts/sphinxcontrib/rubydomain.py @@ -0,0 +1,695 @@ +# -*- coding: utf-8 -*- +""" + sphinx.domains.ruby + ~~~~~~~~~~~~~~~~~~~ + + The Ruby domain. + + :copyright: Copyright 2010 by SHIBUKAWA Yoshiki + :license: BSD, see LICENSE for details. +""" + +import re + +from docutils import nodes +from docutils.parsers.rst import directives + +from sphinx import addnodes +from sphinx.roles import XRefRole +from sphinx.locale import l_, _ +from sphinx.domains import Domain, ObjType, Index +from sphinx.directives import ObjectDescription +from sphinx.util.nodes import make_refnode +from sphinx.util.compat import Directive +from sphinx.util.docfields import Field, GroupedField, TypedField + + +# REs for Ruby signatures +rb_sig_re = re.compile( + r'''^ ([\w.]*\.)? # class name(s) + (\$?\w+\??!?) \s* # thing name + (?: \((.*)\) # optional: arguments + (?:\s* -> \s* (.*))? # return annotation + )? $ # and nothing more + ''', re.VERBOSE) + +rb_paramlist_re = re.compile(r'([\[\],])') # split at '[', ']' and ',' + +separators = { + 'method':'#', 'attr_reader':'#', 'attr_writer':'#', 'attr_accessor':'#', + 'function':'.', 'classmethod':'.', 'class':'::', 'module':'::', + 'global':'', 'const':'::'} + +rb_separator = re.compile(r"(?:\w+)?(?:::)?(?:\.)?(?:#)?") + + +def _iteritems(d): + + for k in d: + yield k, d[k] + + +def ruby_rsplit(fullname): + items = [item for item in rb_separator.findall(fullname)] + return ''.join(items[:-2]), items[-1] + + +class RubyObject(ObjectDescription): + """ + Description of a general Ruby object. + """ + option_spec = { + 'noindex': directives.flag, + 'module': directives.unchanged, + } + + doc_field_types = [ + TypedField('parameter', label=l_('Parameters'), + names=('param', 'parameter', 'arg', 'argument'), + typerolename='obj', typenames=('paramtype', 'type')), + TypedField('variable', label=l_('Variables'), rolename='obj', + names=('var', 'ivar', 'cvar'), + typerolename='obj', typenames=('vartype',)), + GroupedField('exceptions', label=l_('Raises'), rolename='exc', + names=('raises', 'raise', 'exception', 'except'), + can_collapse=True), + Field('returnvalue', label=l_('Returns'), has_arg=False, + names=('returns', 'return')), + Field('returntype', label=l_('Return type'), has_arg=False, + names=('rtype',)), + ] + + def get_signature_prefix(self, sig): + """ + May return a prefix to put before the object name in the signature. + """ + return '' + + def needs_arglist(self): + """ + May return true if an empty argument list is to be generated even if + the document contains none. + """ + return False + + def handle_signature(self, sig, signode): + """ + Transform a Ruby signature into RST nodes. + Returns (fully qualified name of the thing, classname if any). + + If inside a class, the current class name is handled intelligently: + * it is stripped from the displayed name if present + * it is added to the full name (return value) if not present + """ + m = rb_sig_re.match(sig) + if m is None: + raise ValueError + name_prefix, name, arglist, retann = m.groups() + if not name_prefix: + name_prefix = "" + # determine module and class name (if applicable), as well as full name + modname = self.options.get( + 'module', self.env.temp_data.get('rb:module')) + classname = self.env.temp_data.get('rb:class') + if self.objtype == 'global': + add_module = False + modname = None + classname = None + fullname = name + elif classname: + add_module = False + if name_prefix and name_prefix.startswith(classname): + fullname = name_prefix + name + # class name is given again in the signature + name_prefix = name_prefix[len(classname):].lstrip('.') + else: + separator = separators[self.objtype] + fullname = classname + separator + name_prefix + name + else: + add_module = True + if name_prefix: + classname = name_prefix.rstrip('.') + fullname = name_prefix + name + else: + classname = '' + fullname = name + + signode['module'] = modname + signode['class'] = self.class_name = classname + signode['fullname'] = fullname + + sig_prefix = self.get_signature_prefix(sig) + if sig_prefix: + signode += addnodes.desc_annotation(sig_prefix, sig_prefix) + + if name_prefix: + signode += addnodes.desc_addname(name_prefix, name_prefix) + # exceptions are a special case, since they are documented in the + # 'exceptions' module. + elif add_module and self.env.config.add_module_names: + if self.objtype == 'global': + nodetext = '' + signode += addnodes.desc_addname(nodetext, nodetext) + else: + modname = self.options.get( + 'module', self.env.temp_data.get('rb:module')) + if modname and modname != 'exceptions': + nodetext = modname + separators[self.objtype] + signode += addnodes.desc_addname(nodetext, nodetext) + + signode += addnodes.desc_name(name, name) + if not arglist: + if self.needs_arglist(): + # for callables, add an empty parameter list + signode += addnodes.desc_parameterlist() + if retann: + signode += addnodes.desc_returns(retann, retann) + return fullname, name_prefix + signode += addnodes.desc_parameterlist() + + stack = [signode[-1]] + for token in rb_paramlist_re.split(arglist): + if token == '[': + opt = addnodes.desc_optional() + stack[-1] += opt + stack.append(opt) + elif token == ']': + try: + stack.pop() + except IndexError: + raise ValueError + elif not token or token == ',' or token.isspace(): + pass + else: + token = token.strip() + stack[-1] += addnodes.desc_parameter(token, token) + if len(stack) != 1: + raise ValueError + if retann: + signode += addnodes.desc_returns(retann, retann) + return fullname, name_prefix + + def get_index_text(self, modname, name): + """ + Return the text for the index entry of the object. + """ + raise NotImplementedError('must be implemented in subclasses') + + def _is_class_member(self): + return self.objtype.endswith('method') or self.objtype.startswith('attr') + + def add_target_and_index(self, name_cls, sig, signode): + if self.objtype == 'global': + modname = '' + else: + modname = self.options.get( + 'module', self.env.temp_data.get('rb:module')) + separator = separators[self.objtype] + if self._is_class_member(): + if signode['class']: + prefix = modname and modname + '::' or '' + else: + prefix = modname and modname + separator or '' + else: + prefix = modname and modname + separator or '' + fullname = prefix + name_cls[0] + # note target + if fullname not in self.state.document.ids: + signode['names'].append(fullname) + signode['ids'].append(fullname) + signode['first'] = (not self.names) + self.state.document.note_explicit_target(signode) + objects = self.env.domaindata['rb']['objects'] + if fullname in objects: + self.env.warn( + self.env.docname, + 'duplicate object description of %s, ' % fullname + + 'other instance in ' + + self.env.doc2path(objects[fullname][0]), + self.lineno) + objects[fullname] = (self.env.docname, self.objtype) + + indextext = self.get_index_text(modname, name_cls) + if indextext: + self.indexnode['entries'].append(('single', indextext, + fullname, fullname)) + + def before_content(self): + # needed for automatic qualification of members (reset in subclasses) + self.clsname_set = False + + def after_content(self): + if self.clsname_set: + self.env.temp_data['rb:class'] = None + + +class RubyModulelevel(RubyObject): + """ + Description of an object on module level (functions, data). + """ + + def needs_arglist(self): + return self.objtype == 'function' + + def get_index_text(self, modname, name_cls): + if self.objtype == 'function': + if not modname: + return _('%s() (global function)') % name_cls[0] + return _('%s() (module function in %s)') % (name_cls[0], modname) + else: + return '' + + +class RubyGloballevel(RubyObject): + """ + Description of an object on module level (functions, data). + """ + + def get_index_text(self, modname, name_cls): + if self.objtype == 'global': + return _('%s (global variable)') % name_cls[0] + else: + return '' + + +class RubyEverywhere(RubyObject): + """ + Description of a class member (methods, attributes). + """ + + def needs_arglist(self): + return self.objtype == 'method' + + def get_index_text(self, modname, name_cls): + name, cls = name_cls + add_modules = self.env.config.add_module_names + if self.objtype == 'method': + try: + clsname, methname = ruby_rsplit(name) + except ValueError: + if modname: + return _('%s() (in module %s)') % (name, modname) + else: + return '%s()' % name + if modname and add_modules: + return _('%s() (%s::%s method)') % (methname, modname, + clsname) + else: + return _('%s() (%s method)') % (methname, clsname) + else: + return '' + + +class RubyClasslike(RubyObject): + """ + Description of a class-like object (classes, exceptions). + """ + + def get_signature_prefix(self, sig): + return self.objtype + ' ' + + def get_index_text(self, modname, name_cls): + if self.objtype == 'class': + if not modname: + return _('%s (class)') % name_cls[0] + return _('%s (class in %s)') % (name_cls[0], modname) + elif self.objtype == 'exception': + return name_cls[0] + else: + return '' + + def before_content(self): + RubyObject.before_content(self) + if self.names: + self.env.temp_data['rb:class'] = self.names[0][0] + self.clsname_set = True + + +class RubyClassmember(RubyObject): + """ + Description of a class member (methods, attributes). + """ + + def needs_arglist(self): + return self.objtype.endswith('method') + + def get_signature_prefix(self, sig): + if self.objtype == 'classmethod': + return "classmethod %s." % self.class_name + elif self.objtype == 'attr_reader': + return "attribute [R] " + elif self.objtype == 'attr_writer': + return "attribute [W] " + elif self.objtype == 'attr_accessor': + return "attribute [R/W] " + return '' + + def get_index_text(self, modname, name_cls): + name, cls = name_cls + add_modules = self.env.config.add_module_names + if self.objtype == 'classmethod': + try: + clsname, methname = ruby_rsplit(name) + except ValueError: + return '%s()' % name + if modname: + return _('%s() (%s.%s class method)') % (methname, modname, + clsname) + else: + return _('%s() (%s class method)') % (methname, clsname) + elif self.objtype.startswith('attr'): + try: + clsname, attrname = ruby_rsplit(name) + except ValueError: + return name + if modname and add_modules: + return _('%s (%s.%s attribute)') % (attrname, modname, clsname) + else: + return _('%s (%s attribute)') % (attrname, clsname) + else: + return '' + + def before_content(self): + RubyObject.before_content(self) + lastname = self.names and self.names[-1][1] + if lastname and not self.env.temp_data.get('rb:class'): + self.env.temp_data['rb:class'] = lastname.strip('.') + self.clsname_set = True + + +class RubyModule(Directive): + """ + Directive to mark description of a new module. + """ + + has_content = False + required_arguments = 1 + optional_arguments = 0 + final_argument_whitespace = False + option_spec = { + 'platform': lambda x: x, + 'synopsis': lambda x: x, + 'noindex': directives.flag, + 'deprecated': directives.flag, + } + + def run(self): + env = self.state.document.settings.env + modname = self.arguments[0].strip() + noindex = 'noindex' in self.options + env.temp_data['rb:module'] = modname + env.domaindata['rb']['modules'][modname] = \ + (env.docname, self.options.get('synopsis', ''), + self.options.get('platform', ''), 'deprecated' in self.options) + targetnode = nodes.target('', '', ids=['module-' + modname], ismod=True) + self.state.document.note_explicit_target(targetnode) + ret = [targetnode] + # XXX this behavior of the module directive is a mess... + if 'platform' in self.options: + platform = self.options['platform'] + node = nodes.paragraph() + node += nodes.emphasis('', _('Platforms: ')) + node += nodes.Text(platform, platform) + ret.append(node) + # the synopsis isn't printed; in fact, it is only used in the + # modindex currently + if not noindex: + indextext = _('%s (module)') % modname + inode = addnodes.index(entries=[('single', indextext, + 'module-' + modname, modname)]) + ret.append(inode) + return ret + + +class RubyCurrentModule(Directive): + """ + This directive is just to tell Sphinx that we're documenting + stuff in module foo, but links to module foo won't lead here. + """ + + has_content = False + required_arguments = 1 + optional_arguments = 0 + final_argument_whitespace = False + option_spec = {} + + def run(self): + env = self.state.document.settings.env + modname = self.arguments[0].strip() + if modname == 'None': + env.temp_data['rb:module'] = None + else: + env.temp_data['rb:module'] = modname + return [] + + +class RubyXRefRole(XRefRole): + def process_link(self, env, refnode, has_explicit_title, title, target): + if not has_explicit_title: + title = title.lstrip('.') # only has a meaning for the target + title = title.lstrip('#') + if title.startswith("::"): + title = title[2:] + target = target.lstrip('~') # only has a meaning for the title + # if the first character is a tilde, don't display the module/class + # parts of the contents + if title[0:1] == '~': + m = re.search(r"(?:\.)?(?:#)?(?:::)?(.*)\Z", title) + if m: + title = m.group(1) + if not title.startswith("$"): + refnode['rb:module'] = env.temp_data.get('rb:module') + refnode['rb:class'] = env.temp_data.get('rb:class') + # if the first character is a dot, search more specific namespaces first + # else search builtins first + if target[0:1] == '.': + target = target[1:] + refnode['refspecific'] = True + return title, target + + +class RubyModuleIndex(Index): + """ + Index subclass to provide the Ruby module index. + """ + + name = 'modindex' + localname = l_('Ruby Module Index') + shortname = l_('modules') + + def generate(self, docnames=None): + content = {} + # list of prefixes to ignore + ignores = self.domain.env.config['modindex_common_prefix'] + ignores = sorted(ignores, key=len, reverse=True) + # list of all modules, sorted by module name + modules = sorted(_iteritems(self.domain.data['modules']), + key=lambda x: x[0].lower()) + # sort out collapsable modules + prev_modname = '' + num_toplevels = 0 + for modname, (docname, synopsis, platforms, deprecated) in modules: + if docnames and docname not in docnames: + continue + + for ignore in ignores: + if modname.startswith(ignore): + modname = modname[len(ignore):] + stripped = ignore + break + else: + stripped = '' + + # we stripped the whole module name? + if not modname: + modname, stripped = stripped, '' + + entries = content.setdefault(modname[0].lower(), []) + + package = modname.split('::')[0] + if package != modname: + # it's a submodule + if prev_modname == package: + # first submodule - make parent a group head + entries[-1][1] = 1 + elif not prev_modname.startswith(package): + # submodule without parent in list, add dummy entry + entries.append([stripped + package, 1, '', '', '', '', '']) + subtype = 2 + else: + num_toplevels += 1 + subtype = 0 + + qualifier = deprecated and _('Deprecated') or '' + entries.append([stripped + modname, subtype, docname, + 'module-' + stripped + modname, platforms, + qualifier, synopsis]) + prev_modname = modname + + # apply heuristics when to collapse modindex at page load: + # only collapse if number of toplevel modules is larger than + # number of submodules + collapse = len(modules) - num_toplevels < num_toplevels + + # sort by first letter + content = sorted(_iteritems(content)) + + return content, collapse + + +class RubyDomain(Domain): + """Ruby language domain.""" + name = 'rb' + label = 'Ruby' + object_types = { + 'function': ObjType(l_('function'), 'func', 'obj'), + 'global': ObjType(l_('global variable'), 'global', 'obj'), + 'method': ObjType(l_('method'), 'meth', 'obj'), + 'class': ObjType(l_('class'), 'class', 'obj'), + 'exception': ObjType(l_('exception'), 'exc', 'obj'), + 'classmethod': ObjType(l_('class method'), 'meth', 'obj'), + 'attr_reader': ObjType(l_('attribute'), 'attr', 'obj'), + 'attr_writer': ObjType(l_('attribute'), 'attr', 'obj'), + 'attr_accessor': ObjType(l_('attribute'), 'attr', 'obj'), + 'const': ObjType(l_('const'), 'const', 'obj'), + 'module': ObjType(l_('module'), 'mod', 'obj'), + } + + directives = { + 'function': RubyModulelevel, + 'global': RubyGloballevel, + 'method': RubyEverywhere, + 'const': RubyEverywhere, + 'class': RubyClasslike, + 'exception': RubyClasslike, + 'classmethod': RubyClassmember, + 'attr_reader': RubyClassmember, + 'attr_writer': RubyClassmember, + 'attr_accessor': RubyClassmember, + 'module': RubyModule, + 'currentmodule': RubyCurrentModule, + } + + roles = { + 'func': RubyXRefRole(fix_parens=False), + 'global':RubyXRefRole(), + 'class': RubyXRefRole(), + 'exc': RubyXRefRole(), + 'meth': RubyXRefRole(fix_parens=False), + 'attr': RubyXRefRole(), + 'const': RubyXRefRole(), + 'mod': RubyXRefRole(), + 'obj': RubyXRefRole(), + } + initial_data = { + 'objects': {}, # fullname -> docname, objtype + 'modules': {}, # modname -> docname, synopsis, platform, deprecated + } + indices = [ + RubyModuleIndex, + ] + + def clear_doc(self, docname): + for fullname, (fn, _) in list(self.data['objects'].items()): + if fn == docname: + del self.data['objects'][fullname] + for modname, (fn, _, _, _) in list(self.data['modules'].items()): + if fn == docname: + del self.data['modules'][modname] + + def find_obj(self, env, modname, classname, name, type, searchorder=0): + """ + Find a Ruby object for "name", perhaps using the given module and/or + classname. + """ + # skip parens + if name[-2:] == '()': + name = name[:-2] + + if not name: + return None, None + + objects = self.data['objects'] + + newname = None + if searchorder == 1: + if modname and classname and \ + modname + '::' + classname + '#' + name in objects: + newname = modname + '::' + classname + '#' + name + elif modname and classname and \ + modname + '::' + classname + '.' + name in objects: + newname = modname + '::' + classname + '.' + name + elif modname and modname + '::' + name in objects: + newname = modname + '::' + name + elif modname and modname + '#' + name in objects: + newname = modname + '#' + name + elif modname and modname + '.' + name in objects: + newname = modname + '.' + name + elif classname and classname + '.' + name in objects: + newname = classname + '.' + name + elif classname and classname + '#' + name in objects: + newname = classname + '#' + name + elif name in objects: + newname = name + else: + if name in objects: + newname = name + elif classname and classname + '.' + name in objects: + newname = classname + '.' + name + elif classname and classname + '#' + name in objects: + newname = classname + '#' + name + elif modname and modname + '::' + name in objects: + newname = modname + '::' + name + elif modname and modname + '#' + name in objects: + newname = modname + '#' + name + elif modname and modname + '.' + name in objects: + newname = modname + '.' + name + elif modname and classname and \ + modname + '::' + classname + '#' + name in objects: + newname = modname + '::' + classname + '#' + name + elif modname and classname and \ + modname + '::' + classname + '.' + name in objects: + newname = modname + '::' + classname + '.' + name + # special case: object methods + elif type in ('func', 'meth') and '.' not in name and \ + 'object.' + name in objects: + newname = 'object.' + name + if newname is None: + return None, None + return newname, objects[newname] + + def resolve_xref(self, env, fromdocname, builder, + typ, target, node, contnode): + if (typ == 'mod' or + typ == 'obj' and target in self.data['modules']): + docname, synopsis, platform, deprecated = \ + self.data['modules'].get(target, ('','','', '')) + if not docname: + return None + else: + title = '%s%s%s' % ((platform and '(%s) ' % platform), + synopsis, + (deprecated and ' (deprecated)' or '')) + return make_refnode(builder, fromdocname, docname, + 'module-' + target, contnode, title) + else: + modname = node.get('rb:module') + clsname = node.get('rb:class') + searchorder = node.hasattr('refspecific') and 1 or 0 + name, obj = self.find_obj(env, modname, clsname, + target, typ, searchorder) + if not obj: + return None + else: + return make_refnode(builder, fromdocname, obj[0], name, + contnode, name) + + def get_objects(self): + for modname, info in _iteritems(self.data['modules']): + yield (modname, modname, 'module', info[0], 'module-' + modname, 0) + for refname, (docname, type) in _iteritems(self.data['objects']): + yield (refname, refname, type, docname, refname, 1) + + +def setup(app): + app.add_domain(RubyDomain) From 18064d1626bef70a606e125d55e4b60032bf77a5 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 21:23:27 +0900 Subject: [PATCH 44/55] Document nghttpx mruby extension --- doc/conf.py.in | 4 +- doc/nghttpx.h2r | 216 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 219 insertions(+), 1 deletion(-) diff --git a/doc/conf.py.in b/doc/conf.py.in index 0e572cf3..e6bd3203 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -41,6 +41,8 @@ import sys, os # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) +sys.path.append(os.path.abspath('_exts')) + # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. @@ -48,7 +50,7 @@ import sys, os # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [] +extensions = ['sphinxcontrib.rubydomain'] # Add any paths that contain templates here, relative to this directory. templates_path = ['@top_srcdir@/_templates'] diff --git a/doc/nghttpx.h2r b/doc/nghttpx.h2r index ac089c37..a590beab 100644 --- a/doc/nghttpx.h2r +++ b/doc/nghttpx.h2r @@ -153,6 +153,222 @@ from the given file. In this case, nghttpx does not rotate key automatically. To rotate key, one has to restart nghttpx (see SIGNALS). +MRUBY SCRIPTING +--------------- + +nghttpx allows users to extend its capability using mruby scripts. +nghttpx has 2 hook points to execute mruby script: request phase and +response phase. The request phase hook is invoked after all request +header fields are received from client. The response phase hook is +invoked after all response header fields are received from backend +server. These hooks allows users to modify header fields, or common +HTTP variables, like authority or request path, and even return custom +response without forwarding request to backend servers. + +To set request phase hook, use :option:`--request-phase-file` option. +To set response phase hook, use :option:`--response-phase-file` +option. + +For request and response phase hook, user calls :rb:meth:`Nghttpx.run` +with block. The :rb:class:`Nghttpx::Env` is passed to the block. +User can can access :rb:class:`Nghttpx::Request` and +:rb:class:`Nghttpx::Response` objects via :rb:attr:`Nghttpx::Env#req` +and :rb:attr:`Nghttpx::Env#resp` respectively. + +.. rb:module:: Nghttpx + +.. rb:classmethod:: run(&block) + + Run request or response phase hook with given *block*. + :rb:class:`Nghttpx::Env` object is passed to the given block. + +.. rb:const:: REQUEST_PHASE + + Constant to represent request phase. + +.. rb:const:: RESPONSE_PHASE + + Constant to represent response phase. + +.. rb:class:: Env + + Object to represent current request specific context. + + .. rb:attr_reader:: req + + Return :rb:class:`Request` object. + + .. rb:attr_reader:: resp + + Return :rb:class:`Response` object. + + .. rb:attr_reader:: ctx + + Return Ruby hash object. It persists until request finishes. + So values set in request phase hoo can be retrieved in + response phase hook. + + .. rb:attr_reader:: phase + + Return the current phase. + + .. rb:attr_reader:: remote_addr + + Return IP address of a remote client. + +.. rb:class:: Request + + Object to represent request from client. The modification to + Request object is allowed only in request phase hook. + + .. rb:attr_reader:: http_version_major + + Return HTTP major version. + + .. rb:attr_reader:: http_version_minor + + Return HTTP minor version. + + .. rb:attr_accessor:: method + + HTTP method. On assignment, copy of given value is assigned. + We don't accept arbitrary method name. We will document them + later, but well known methods, like GET, PUT and POST, are all + supported. + + .. rb:attr_accessor:: authority + + Authority (i.e., example.org), including optional port + component . On assignment, copy of given value is assigned. + + .. rb:attr_accessor:: scheme + + Scheme (i.e., http, https). On assignment, copy of given + value is assigned. + + .. rb:attr_accessor:: path + + Request path, including query component (i.e., /index.html). + On assignment, copy of given value is assigned. The path does + not include authority component of URI. + + .. rb:attr_reader:: headers + + Return Ruby hash containing copy of request header fields. + Changing values in returned hash does not change request + header fields actually used in request processing. Use + :rb:meth:`Nghttpx::Request#add_header` or + :rb:meth:`Nghttpx::Request#set_header` to change request + header fields. + + .. rb:method:: add_header(key, value) + + Add header entry associated with key. The value can be single + string or array of string. It does not replace any existing + values associated with key. + + .. rb:method:: set_header(key, value) + + Set header entry associated with key. The value can be single + string or array of string. It replaces any existing values + associated with key. + + .. rb:method:: clear_headers + + Clear all existing request header fields. + +.. rb:class:: Response + + Object to represent response from backend server. + + .. rb:attr_reader:: http_version_major + + Return HTTP major version. + + .. rb:attr_reader:: http_version_minor + + Return HTTP minor version. + + .. rb:attr_accessor:: status + + HTTP status code. It must be in the range [200, 999], + inclusive. The non-final status code is not supported in + mruby scripting at the moment. + + .. rb:attr_reader:: headers + + Return Ruby hash containing copy of response header fields. + Changing values in returned hash does not change response + header fields actually used in response processing. Use + :rb:meth:`Nghttpx::Response#add_header` or + :rb:meth:`Nghttpx::Response#set_header` to change response + header fields. + + .. rb:method:: add_header(key, value) + + Add header entry associated with key. The value can be single + string or array of string. It does not replace any existing + values associated with key. + + .. rb:method:: set_header(key, value) + + Set header entry associated with key. The value can be single + string or array of string. It replaces any existing values + associated with key. + + .. rb:method:: clear_headers + + Clear all existing response header fields. + + .. rb:method:: return(body) + + Return custom response *body* to a client. When this method + is called in request phase hook, the request is not forwarded + to the backend, and response phase hook for this request will + not be invoked. When this method is called in resonse phase + hook, response from backend server is canceled and discarded. + The status code and response header fields should be set + before using this method. To set status code, use :rb:meth To + set response header fields, use + :rb:attr:`Nghttpx::Response#status`. If status code is not + set, 200 is used. :rb:meth:`Nghttpx::Response#add_header` and + :rb:meth:`Nghttpx::Response#set_header`. When this method is + invoked in response phase hook, the response headers are + filled with the ones received from backend server. To send + completely custom header fields, first call + :rb:meth:`Nghttpx::Response#clear_headers` to erase all + existing header fields, and then add required header fields. + It is an error to call this method twice for a given request. + +MRUBY EXAMPLES +~~~~~~~~~~~~~~ + +Modify requet path: + +.. code-block:: ruby + + Nghttpx.run do |env| + env.req.path = "/apps#{env.req.path}" + end + +Note that the file containing the above script must be set with +:option:`--request-phase-file` option since we modify request path. + +Restrict permission of viewing a content to a specific client +addresses: + +.. code-block:: ruby + + Nghttpx.run do |env| + allowed_clients = ["127.0.0.1", "::1"] + + if env.req.path.start_with?("/log/") && + !allowed_clients.include?(env.remote_addr) then + env.resp.status = 404 + env.resp.return "permission denied" + end + end + SEE ALSO -------- From 6f0c88580f6c685d8b77c29a35d2bd5002713837 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 21:24:54 +0900 Subject: [PATCH 45/55] Add *.pyc to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7adeac3d..582277aa 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ *.lo *.m4 *.o +*.pyc .deps/ .libs/ INSTALL From 7b0113ac7766fa730accc8f05b0e1b56afb7ce3c Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 21:26:58 +0900 Subject: [PATCH 46/55] Remove \r --- .gitignore | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 582277aa..7ed31ec3 100644 --- a/.gitignore +++ b/.gitignore @@ -32,10 +32,10 @@ test-driver # test logs generated by `make check` *.log *.trs - -lib/MSVC_obj/ -_VC_ROOT/ -.depend.MSVC -*.pyd -*.egg-info/ -python/nghttp2.c + +lib/MSVC_obj/ +_VC_ROOT/ +.depend.MSVC +*.pyd +*.egg-info/ +python/nghttp2.c From e1106e0f9130d2ca8478a68b35855c17698e155c Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 21:28:43 +0900 Subject: [PATCH 47/55] Update doc --- doc/nghttpx.h2r | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/nghttpx.h2r b/doc/nghttpx.h2r index a590beab..5362cb9e 100644 --- a/doc/nghttpx.h2r +++ b/doc/nghttpx.h2r @@ -156,6 +156,11 @@ SIGNALS). MRUBY SCRIPTING --------------- +.. warning:: + + The current mruby extension API is experimental and not frozen. The + API is subject to change in the future release. + nghttpx allows users to extend its capability using mruby scripts. nghttpx has 2 hook points to execute mruby script: request phase and response phase. The request phase hook is invoked after all request From 2135c87f944f97fddaba4037777fe8ce67cea7af Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 21:39:26 +0900 Subject: [PATCH 48/55] Update README.rst --- README.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.rst b/README.rst index bffaed04..35fc7259 100644 --- a/README.rst +++ b/README.rst @@ -113,6 +113,14 @@ If you are using Ubuntu 14.04 LTS (trusty), run the following to install the nee spdylay is not packaged in Ubuntu, so you need to build it yourself: http://tatsuhiro-t.github.io/spdylay/ +To enable mruby support for nghttpx, `mruby +`_ is required. We need to build +mruby with C++ ABI explicitly turned on, and probably need other +mrgems, mruby is manged by git submodule under third-party/mruby +directory. Currently, mruby support for nghttpx is disabled by +default. To enable mruby support, use ``--with-mruby`` configure +option. + Building from git ----------------- From 98c959291f7e16752a8907d6c68915b7c769f015 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 22:46:55 +0900 Subject: [PATCH 49/55] Update README.rst --- README.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 35fc7259..91a0ed8d 100644 --- a/README.rst +++ b/README.rst @@ -119,7 +119,9 @@ mruby with C++ ABI explicitly turned on, and probably need other mrgems, mruby is manged by git submodule under third-party/mruby directory. Currently, mruby support for nghttpx is disabled by default. To enable mruby support, use ``--with-mruby`` configure -option. +option. Note that at the time of this writing, libmruby-dev and mruby +packages in Debian/Ubuntu are not usable for nghttp2, since they do +not enable C++ ABI. Building from git ----------------- @@ -135,6 +137,11 @@ used:: To compile the source code, gcc >= 4.8.3 or clang >= 3.4 is required. +.. note:: + + To enable mruby support in nghttpx, use ``--with-mruby`` configure + option. + .. note:: Mac OS X users may need the ``--disable-threads`` configure option to From ef1595672c7136f57a4f45d879474e559f1a1d6d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 22:47:07 +0900 Subject: [PATCH 50/55] nghttpx: Add Request#push in mruby scripting Refactor Http2Upstream so that we can share code between link header field based push and mruby push. --- src/http2.cc | 85 +++++++++++++++++++++ src/http2.h | 15 ++++ src/http2_test.cc | 100 ++++++++++++++++++++++++ src/http2_test.h | 2 + src/shrpx-unittest.cc | 4 + src/shrpx_http2_upstream.cc | 123 ++++++++++++++---------------- src/shrpx_http2_upstream.h | 2 + src/shrpx_https_upstream.cc | 5 ++ src/shrpx_https_upstream.h | 2 + src/shrpx_mruby_module_request.cc | 17 +++++ src/shrpx_spdy_upstream.cc | 5 ++ src/shrpx_spdy_upstream.h | 2 + src/shrpx_upstream.h | 3 + 13 files changed, 301 insertions(+), 64 deletions(-) diff --git a/src/http2.cc b/src/http2.cc index f00aeb0a..8059ac04 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -1308,6 +1308,91 @@ const char *to_method_string(int method_token) { return http_method_str(static_cast(method_token)); } +int get_pure_path_component(const char **base, size_t *baselen, + const std::string &uri) { + int rv; + + http_parser_url u{}; + rv = http_parser_parse_url(uri.c_str(), uri.size(), 0, &u); + if (rv != 0) { + return -1; + } + + if (u.field_set & (1 << UF_PATH)) { + auto &f = u.field_data[UF_PATH]; + *base = uri.c_str() + f.off; + *baselen = f.len; + + return 0; + } + + *base = "/"; + *baselen = 1; + + return 0; +} + +int construct_push_component(std::string &scheme, std::string &authority, + std::string &path, const char *base, + size_t baselen, const char *uri, size_t len) { + int rv; + const char *rel, *relq = nullptr; + size_t rellen, relqlen = 0; + + http_parser_url u{}; + + rv = http_parser_parse_url(uri, len, 0, &u); + + if (rv != 0) { + if (uri[0] == '/') { + return -1; + } + + // treat link_url as relative URI. + auto end = std::find(uri, uri + len, '#'); + auto q = std::find(uri, end, '?'); + + rel = uri; + rellen = q - uri; + if (q != end) { + relq = q + 1; + relqlen = end - relq; + } + } else { + if (u.field_set & (1 << UF_SCHEMA)) { + http2::copy_url_component(scheme, &u, UF_SCHEMA, uri); + } + + if (u.field_set & (1 << UF_HOST)) { + http2::copy_url_component(authority, &u, UF_HOST, uri); + if (u.field_set & (1 << UF_PORT)) { + authority += ":"; + authority += util::utos(u.port); + } + } + + if (u.field_set & (1 << UF_PATH)) { + auto &f = u.field_data[UF_PATH]; + rel = uri + f.off; + rellen = f.len; + } else { + rel = "/"; + rellen = 1; + } + + if (u.field_set & (1 << UF_QUERY)) { + auto &f = u.field_data[UF_QUERY]; + relq = uri + f.off; + relqlen = f.len; + } + } + + path = + http2::path_join(base, baselen, nullptr, 0, rel, rellen, relq, relqlen); + + return 0; +} + } // namespace http2 } // namespace nghttp2 diff --git a/src/http2.h b/src/http2.h index 09f04cd2..8fd32f85 100644 --- a/src/http2.h +++ b/src/http2.h @@ -352,6 +352,21 @@ std::string rewrite_clean_path(InputIt first, InputIt last) { return path; } +// Stores path component of |uri| in *base. Its extracted length is +// stored in *baselen. The extracted path does not include query +// component. This function returns 0 if it succeeds, or -1. +int get_pure_path_component(const char **base, size_t *baselen, + const std::string &uri); + +// Deduces scheme, authority and path from given |uri| of length +// |len|, and stores them in |scheme|, |authority|, and |path| +// respectively. If |uri| is relative path, path resolution is taken +// palce using path given in |base| of length |baselen|. This +// function returns 0 if it succeeds, or -1. +int construct_push_component(std::string &scheme, std::string &authority, + std::string &path, const char *base, + size_t baselen, const char *uri, size_t len); + } // namespace http2 } // namespace nghttp2 diff --git a/src/http2_test.cc b/src/http2_test.cc index 9a99a3a9..cda2a04e 100644 --- a/src/http2_test.cc +++ b/src/http2_test.cc @@ -880,4 +880,104 @@ void test_http2_rewrite_clean_path(void) { CU_ASSERT(src == http2::rewrite_clean_path(std::begin(src), std::end(src))); } +void test_http2_get_pure_path_component(void) { + const char *base; + size_t len; + std::string path; + + path = "/"; + CU_ASSERT(0 == http2::get_pure_path_component(&base, &len, path)); + CU_ASSERT(util::streq_l("/", base, len)); + + path = "/foo"; + CU_ASSERT(0 == http2::get_pure_path_component(&base, &len, path)); + CU_ASSERT(util::streq_l("/foo", base, len)); + + path = "https://example.org/bar"; + CU_ASSERT(0 == http2::get_pure_path_component(&base, &len, path)); + CU_ASSERT(util::streq_l("/bar", base, len)); + + path = "https://example.org/alpha?q=a"; + CU_ASSERT(0 == http2::get_pure_path_component(&base, &len, path)); + CU_ASSERT(util::streq_l("/alpha", base, len)); + + path = "https://example.org/bravo?q=a#fragment"; + CU_ASSERT(0 == http2::get_pure_path_component(&base, &len, path)); + CU_ASSERT(util::streq_l("/bravo", base, len)); + + path = "\x01\x02"; + CU_ASSERT(-1 == http2::get_pure_path_component(&base, &len, path)); +} + +void test_http2_construct_push_component(void) { + const char *base; + size_t baselen; + std::string uri; + std::string scheme, authority, path; + + base = "/b/"; + baselen = 3; + + uri = "https://example.org/foo"; + + CU_ASSERT(0 == http2::construct_push_component(scheme, authority, path, base, + baselen, uri.c_str(), + uri.size())); + CU_ASSERT("https" == scheme); + CU_ASSERT("example.org" == authority); + CU_ASSERT("/foo" == path); + + scheme.clear(); + authority.clear(); + path.clear(); + + uri = "/foo/bar?q=a"; + + CU_ASSERT(0 == http2::construct_push_component(scheme, authority, path, base, + baselen, uri.c_str(), + uri.size())); + CU_ASSERT("" == scheme); + CU_ASSERT("" == authority); + CU_ASSERT("/foo/bar?q=a" == path); + + scheme.clear(); + authority.clear(); + path.clear(); + + uri = "foo/../bar?q=a"; + + CU_ASSERT(0 == http2::construct_push_component(scheme, authority, path, base, + baselen, uri.c_str(), + uri.size())); + CU_ASSERT("" == scheme); + CU_ASSERT("" == authority); + CU_ASSERT("/b/bar?q=a" == path); + + scheme.clear(); + authority.clear(); + path.clear(); + + uri = ""; + + CU_ASSERT(0 == http2::construct_push_component(scheme, authority, path, base, + baselen, uri.c_str(), + uri.size())); + CU_ASSERT("" == scheme); + CU_ASSERT("" == authority); + CU_ASSERT("/" == path); + + scheme.clear(); + authority.clear(); + path.clear(); + + uri = "?q=a"; + + CU_ASSERT(0 == http2::construct_push_component(scheme, authority, path, base, + baselen, uri.c_str(), + uri.size())); + CU_ASSERT("" == scheme); + CU_ASSERT("" == authority); + CU_ASSERT("/b/?q=a" == path); +} + } // namespace shrpx diff --git a/src/http2_test.h b/src/http2_test.h index bc65d453..80f14cd8 100644 --- a/src/http2_test.h +++ b/src/http2_test.h @@ -47,6 +47,8 @@ void test_http2_parse_link_header(void); void test_http2_path_join(void); void test_http2_normalize_path(void); void test_http2_rewrite_clean_path(void); +void test_http2_get_pure_path_component(void); +void test_http2_construct_push_component(void); } // namespace shrpx diff --git a/src/shrpx-unittest.cc b/src/shrpx-unittest.cc index 10eefb8b..03219a65 100644 --- a/src/shrpx-unittest.cc +++ b/src/shrpx-unittest.cc @@ -100,6 +100,10 @@ int main(int argc, char *argv[]) { shrpx::test_http2_normalize_path) || !CU_add_test(pSuite, "http2_rewrite_clean_path", shrpx::test_http2_rewrite_clean_path) || + !CU_add_test(pSuite, "http2_get_pure_path_component", + shrpx::test_http2_get_pure_path_component) || + !CU_add_test(pSuite, "http2_construct_push_component", + shrpx::test_http2_construct_push_component) || !CU_add_test(pSuite, "downstream_index_request_headers", shrpx::test_downstream_index_request_headers) || !CU_add_test(pSuite, "downstream_index_response_headers", diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index ba9c2178..8541f923 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1582,80 +1582,31 @@ int Http2Upstream::on_downstream_reset(bool no_retry) { int Http2Upstream::prepare_push_promise(Downstream *downstream) { int rv; - http_parser_url u{}; - rv = http_parser_parse_url(downstream->get_request_path().c_str(), - downstream->get_request_path().size(), 0, &u); + const char *base; + size_t baselen; + + rv = http2::get_pure_path_component(&base, &baselen, + downstream->get_request_path()); if (rv != 0) { return 0; } - const char *base; - size_t baselen; - if (u.field_set & (1 << UF_PATH)) { - auto &f = u.field_data[UF_PATH]; - base = downstream->get_request_path().c_str() + f.off; - baselen = f.len; - } else { - base = "/"; - baselen = 1; - } + for (auto &kv : downstream->get_response_headers()) { if (kv.token != http2::HD_LINK) { continue; } for (auto &link : http2::parse_link_header(kv.value.c_str(), kv.value.size())) { - auto link_url = link.uri.first; - auto link_urllen = link.uri.second - link.uri.first; - const char *rel; - size_t rellen; - const char *relq = nullptr; - size_t relqlen = 0; + auto uri = link.uri.first; + auto len = link.uri.second - link.uri.first; - std::string authority, scheme; - http_parser_url v{}; - rv = http_parser_parse_url(link_url, link_urllen, 0, &v); + std::string scheme, authority, path; + + rv = http2::construct_push_component(scheme, authority, path, base, + baselen, uri, len); if (rv != 0) { - assert(link_urllen); - if (link_url[0] == '/') { - continue; - } - // treat link_url as relative URI. - auto end = std::find(link_url, link_url + link_urllen, '#'); - auto q = std::find(link_url, end, '?'); - rel = link_url; - rellen = q - link_url; - if (q != end) { - relq = q + 1; - relqlen = end - relq; - } - } else { - if (v.field_set & (1 << UF_SCHEMA)) { - http2::copy_url_component(scheme, &v, UF_SCHEMA, link_url); - } - - if (v.field_set & (1 << UF_HOST)) { - http2::copy_url_component(authority, &v, UF_HOST, link_url); - if (v.field_set & (1 << UF_PORT)) { - authority += ":"; - authority += util::utos(v.port); - } - } - - if (v.field_set & (1 << UF_PATH)) { - auto &f = v.field_data[UF_PATH]; - rel = link_url + f.off; - rellen = f.len; - } else { - rel = "/"; - rellen = 1; - } - - if (v.field_set & (1 << UF_QUERY)) { - auto &f = v.field_data[UF_QUERY]; - relq = link_url + f.off; - relqlen = f.len; - } + continue; } if (scheme.empty()) { @@ -1666,8 +1617,6 @@ int Http2Upstream::prepare_push_promise(Downstream *downstream) { authority = downstream->get_request_http2_authority(); } - auto path = http2::path_join(base, baselen, nullptr, 0, rel, rellen, relq, - relqlen); rv = submit_push_promise(scheme, authority, path, downstream); if (rv != 0) { return -1; @@ -1736,4 +1685,50 @@ int Http2Upstream::submit_push_promise(const std::string &scheme, return 0; } +int Http2Upstream::initiate_push(Downstream *downstream, const char *uri, + size_t len) { + int rv; + + if (len == 0 || + nghttp2_session_get_remote_settings(session_, + NGHTTP2_SETTINGS_ENABLE_PUSH) == 0 || + get_config()->http2_proxy || get_config()->client_proxy || + (downstream->get_stream_id() % 2) == 0) { + return 0; + } + + const char *base; + size_t baselen; + + rv = http2::get_pure_path_component(&base, &baselen, + downstream->get_request_path()); + if (rv != 0) { + return -1; + } + + std::string scheme, authority, path; + + rv = http2::construct_push_component(scheme, authority, path, base, baselen, + uri, len); + if (rv != 0) { + return -1; + } + + if (scheme.empty()) { + scheme = downstream->get_request_http2_scheme(); + } + + if (authority.empty()) { + authority = downstream->get_request_http2_authority(); + } + + rv = submit_push_promise(scheme, authority, path, downstream); + + if (rv != 0) { + return -1; + } + + return 0; +} + } // namespace shrpx diff --git a/src/shrpx_http2_upstream.h b/src/shrpx_http2_upstream.h index c199a9cb..cc877f5c 100644 --- a/src/shrpx_http2_upstream.h +++ b/src/shrpx_http2_upstream.h @@ -80,6 +80,8 @@ public: virtual int on_downstream_reset(bool no_retry); virtual int send_reply(Downstream *downstream, const uint8_t *body, size_t bodylen); + virtual int initiate_push(Downstream *downstream, const char *uri, + size_t len); bool get_flow_control() const; // Perform HTTP/2 upgrade from |upstream|. On success, this object diff --git a/src/shrpx_https_upstream.cc b/src/shrpx_https_upstream.cc index f1c0b31a..ca8a5778 100644 --- a/src/shrpx_https_upstream.cc +++ b/src/shrpx_https_upstream.cc @@ -1160,4 +1160,9 @@ fail: return 0; } +int HttpsUpstream::initiate_push(Downstream *downstream, const char *uri, + size_t len) { + return 0; +} + } // namespace shrpx diff --git a/src/shrpx_https_upstream.h b/src/shrpx_https_upstream.h index f588d4bc..318200ed 100644 --- a/src/shrpx_https_upstream.h +++ b/src/shrpx_https_upstream.h @@ -76,6 +76,8 @@ public: virtual int on_downstream_reset(bool no_retry); virtual int send_reply(Downstream *downstream, const uint8_t *body, size_t bodylen); + virtual int initiate_push(Downstream *downstream, const char *uri, + size_t len); void reset_current_header_length(); void log_response_headers(const std::string &hdrs) const; diff --git a/src/shrpx_mruby_module_request.cc b/src/shrpx_mruby_module_request.cc index 74dad27a..10ef99ec 100644 --- a/src/shrpx_mruby_module_request.cc +++ b/src/shrpx_mruby_module_request.cc @@ -268,6 +268,22 @@ mrb_value request_clear_headers(mrb_state *mrb, mrb_value self) { } } // namespace +namespace { +mrb_value request_push(mrb_state *mrb, mrb_value self) { + auto data = static_cast(mrb->ud); + auto downstream = data->downstream; + auto upstream = downstream->get_upstream(); + + const char *uri; + mrb_int len; + mrb_get_args(mrb, "s", &uri, &len); + + upstream->initiate_push(downstream, uri, len); + + return mrb_nil_value(); +} +} // namespace + void init_request_class(mrb_state *mrb, RClass *module) { auto request_class = mrb_define_class_under(mrb, module, "Request", mrb->object_class); @@ -302,6 +318,7 @@ void init_request_class(mrb_state *mrb, RClass *module) { MRB_ARGS_REQ(2)); mrb_define_method(mrb, request_class, "clear_headers", request_clear_headers, MRB_ARGS_NONE()); + mrb_define_method(mrb, request_class, "push", request_push, MRB_ARGS_REQ(1)); } } // namespace mruby diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc index b3ef81be..d3dea3f9 100644 --- a/src/shrpx_spdy_upstream.cc +++ b/src/shrpx_spdy_upstream.cc @@ -1203,4 +1203,9 @@ int SpdyUpstream::on_downstream_reset(bool no_retry) { return 0; } +int SpdyUpstream::initiate_push(Downstream *downstream, const char *uri, + size_t len) { + return 0; +} + } // namespace shrpx diff --git a/src/shrpx_spdy_upstream.h b/src/shrpx_spdy_upstream.h index c3ff1766..b39f885f 100644 --- a/src/shrpx_spdy_upstream.h +++ b/src/shrpx_spdy_upstream.h @@ -76,6 +76,8 @@ public: virtual int send_reply(Downstream *downstream, const uint8_t *body, size_t bodylen); + virtual int initiate_push(Downstream *downstream, const char *uri, + size_t len); bool get_flow_control() const; diff --git a/src/shrpx_upstream.h b/src/shrpx_upstream.h index d2ec80a7..e367d351 100644 --- a/src/shrpx_upstream.h +++ b/src/shrpx_upstream.h @@ -64,6 +64,9 @@ public: size_t consumed) = 0; virtual int send_reply(Downstream *downstream, const uint8_t *body, size_t bodylen) = 0; + + virtual int initiate_push(Downstream *downstream, const char *uri, + size_t len) = 0; }; } // namespace shrpx From 7273c7d688cb3af6855a10829845405dce62c541 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 5 Sep 2015 23:37:32 +0900 Subject: [PATCH 51/55] Document mruby push --- doc/nghttpx.h2r | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/nghttpx.h2r b/doc/nghttpx.h2r index 5362cb9e..66d8dfde 100644 --- a/doc/nghttpx.h2r +++ b/doc/nghttpx.h2r @@ -282,6 +282,15 @@ and :rb:attr:`Nghttpx::Env#resp` respectively. Clear all existing request header fields. + .. rb:method:: push uri + + Initiate to push resource identified by *uri*. Only HTTP/2 + protocol supports this feature. For the other protocols, this + method is noop. *uri* can be absolute URI, absolute path or + relative path to the current request. For absolute or + relative path, scheme and authority are inherited from the + current request. Currently, method is always GET. + .. rb:class:: Response Object to represent response from backend server. From 1df682140c0c03f93e7d3b378dce84a284aa8493 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 6 Sep 2015 00:06:48 +0900 Subject: [PATCH 52/55] nghttpx: Don't do mruby push with --no-server-push option --- src/shrpx_http2_upstream.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index 8541f923..dd198b9d 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1689,7 +1689,7 @@ int Http2Upstream::initiate_push(Downstream *downstream, const char *uri, size_t len) { int rv; - if (len == 0 || + if (len == 0 || get_config()->no_server_push || nghttp2_session_get_remote_settings(session_, NGHTTP2_SETTINGS_ENABLE_PUSH) == 0 || get_config()->http2_proxy || get_config()->client_proxy || From 75b98662befbd4b66e7713e262b46e285ff3d4b3 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 6 Sep 2015 01:23:11 +0900 Subject: [PATCH 53/55] Don't use `git submodule update --init` in Makefile This is because when we make tar ball, we are no longer under git control. git submodule does not work there. We instead bundle mruby source tree in archive. git submodule command is only required when building from git repository. --- .travis.yml | 1 + README.rst | 5 +++-- makerelease.sh | 3 ++- third-party/Makefile.am | 7 ++++--- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8d974c15..1dee67b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,7 @@ before_script: - autoreconf -i - automake - autoconf + - git submodule update --init - ./configure --enable-werror --with-mruby script: - make diff --git a/README.rst b/README.rst index 91a0ed8d..d69918ff 100644 --- a/README.rst +++ b/README.rst @@ -139,8 +139,9 @@ To compile the source code, gcc >= 4.8.3 or clang >= 3.4 is required. .. note:: - To enable mruby support in nghttpx, use ``--with-mruby`` configure - option. + To enable mruby support in nghttpx, run ``git submodule update + --init`` before running configure script, and use ``--with-mruby`` + configure option. .. note:: diff --git a/makerelease.sh b/makerelease.sh index 054f6a65..60d5ab64 100755 --- a/makerelease.sh +++ b/makerelease.sh @@ -6,6 +6,7 @@ PREV_TAG=$2 git checkout refs/tags/$TAG git log --pretty=fuller --date=short refs/tags/$PREV_TAG..HEAD > ChangeLog -./configure && \ +git submodule update --init +./configure --with-mruby && \ make dist-bzip2 && make dist-gzip && make dist-xz || echo "error" make distclean diff --git a/third-party/Makefile.am b/third-party/Makefile.am index e90b297a..cdabf30c 100644 --- a/third-party/Makefile.am +++ b/third-party/Makefile.am @@ -30,12 +30,13 @@ libhttp_parser_la_SOURCES = \ http-parser/http_parser.c \ http-parser/http_parser.h -.PHONY: all-local clean mruby - if HAVE_MRUBY +EXTRA_DIST = build_config.rb mruby/* + +.PHONY: all-local clean mruby + mruby: - git submodule update --init MRUBY_CONFIG="${srcdir}/build_config.rb" \ BUILD_DIR="${abs_builddir}/mruby/build" \ CC="${CC}" CXX="${CXX}" LD="${LD}" \ From 28defbfb4a82f153b5e3d076bbd82d048214f343 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 6 Sep 2015 15:12:20 +0900 Subject: [PATCH 54/55] nghttpx: Allow link header server push for HTTP/2 backend as well --- src/shrpx_http2_upstream.cc | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index dd198b9d..b4acb275 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -1381,12 +1381,9 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { // We need some conditions that must be fulfilled to initiate server // push. // - // * Server push is disabled for http2 proxy, since incoming headers - // are mixed origins. We don't know how to reliably determine the - // authority yet. - // - // * If downstream is http/2, it is likely that PUSH_PROMISE is - // coming from there, so we don't initiate PUSH_RPOMISE here. + // * Server push is disabled for http2 proxy or client proxy, since + // incoming headers are mixed origins. We don't know how to + // reliably determine the authority yet. // // * We need 200 response code for associated resource. This is too // restrictive, we will review this later. @@ -1397,8 +1394,8 @@ int Http2Upstream::on_downstream_header_complete(Downstream *downstream) { if (!get_config()->no_server_push && nghttp2_session_get_remote_settings(session_, NGHTTP2_SETTINGS_ENABLE_PUSH) == 1 && - get_config()->downstream_proto == PROTO_HTTP && - !get_config()->http2_proxy && (downstream->get_stream_id() % 2) && + !get_config()->http2_proxy && !get_config()->client_proxy && + (downstream->get_stream_id() % 2) && downstream->get_response_header(http2::HD_LINK) && downstream->get_response_http_status() == 200 && (downstream->get_request_method() == HTTP_GET || From 1b63e6d478b75736cbcd75e3db98d3a86205ddbe Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 6 Sep 2015 15:21:36 +0900 Subject: [PATCH 55/55] nghttpx: Call request phase hook for pushed resource as well --- doc/nghttpx.h2r | 5 ++++- src/shrpx_http2_upstream.cc | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/doc/nghttpx.h2r b/doc/nghttpx.h2r index 66d8dfde..c3e6dec5 100644 --- a/doc/nghttpx.h2r +++ b/doc/nghttpx.h2r @@ -289,7 +289,10 @@ and :rb:attr:`Nghttpx::Env#resp` respectively. method is noop. *uri* can be absolute URI, absolute path or relative path to the current request. For absolute or relative path, scheme and authority are inherited from the - current request. Currently, method is always GET. + current request. Currently, method is always GET. nghttpx + will issue request to backend servers to fulfill this request. + The request and response phase hooks will be called for pushed + resource as well. .. rb:class:: Response diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc index b4acb275..8a3160f5 100644 --- a/src/shrpx_http2_upstream.cc +++ b/src/shrpx_http2_upstream.cc @@ -587,6 +587,20 @@ int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame, // downstream is in pending queue. auto ptr = downstream.get(); upstream->add_pending_downstream(std::move(downstream)); + +#ifdef HAVE_MRUBY + auto worker = handler->get_worker(); + auto mruby_ctx = worker->get_mruby_context(); + + if (mruby_ctx->run_on_request_proc(ptr) != 0) { + if (upstream->error_reply(ptr, 500) != 0) { + upstream->rst_stream(ptr, NGHTTP2_INTERNAL_ERROR); + return 0; + } + return 0; + } +#endif // HAVE_MRUBY + upstream->start_downstream(ptr); return 0;