diff --git a/genheaderfunc.py b/genheaderfunc.py index a277fb4f..543ced50 100755 --- a/genheaderfunc.py +++ b/genheaderfunc.py @@ -17,6 +17,7 @@ HEADERS = [ "http2-settings", "server", "via", + "forwarded", "x-forwarded-for", "x-forwarded-proto", "alt-svc", diff --git a/gennghttpxfun.py b/gennghttpxfun.py index e229d78a..496f1875 100755 --- a/gennghttpxfun.py +++ b/gennghttpxfun.py @@ -103,7 +103,11 @@ OPTIONS = [ "conf", "fastopen", "tls-dyn-rec-warmup-threshold", - "tls-dyn-rec-idle-timeout" + "tls-dyn-rec-idle-timeout", + "add-forwarded", + "strip-incoming-forwarded", + "forwarded-by", + "forwarded-for" ] LOGVARS = [ diff --git a/src/http2.cc b/src/http2.cc index 6f9128e1..1b2fa347 100644 --- a/src/http2.cc +++ b/src/http2.cc @@ -348,6 +348,7 @@ void copy_headers_to_nva_internal(std::vector &nva, switch (kv.token) { case HD_COOKIE: case HD_CONNECTION: + case HD_FORWARDED: case HD_HOST: case HD_HTTP2_SETTINGS: case HD_KEEP_ALIVE: @@ -385,6 +386,7 @@ void build_http1_headers_from_headers(DefaultMemchunks *buf, switch (kv.token) { case HD_CONNECTION: case HD_COOKIE: + case HD_FORWARDED: case HD_HOST: case HD_HTTP2_SETTINGS: case HD_KEEP_ALIVE: @@ -638,6 +640,15 @@ int lookup_token(const uint8_t *name, size_t namelen) { break; } break; + case 9: + switch (name[8]) { + case 'd': + if (util::streq_l("forwarde", name, 8)) { + return HD_FORWARDED; + } + break; + } + break; case 10: switch (name[9]) { case 'e': diff --git a/src/http2.h b/src/http2.h index aaf076d6..8f9628c3 100644 --- a/src/http2.h +++ b/src/http2.h @@ -218,6 +218,8 @@ int parse_http_status_code(const std::string &src); // Header fields to be indexed, except HD_MAXIDX which is convenient // member to get maximum value. +// +// generated by genheaderfunc.py enum { HD__AUTHORITY, HD__HOST, @@ -234,6 +236,7 @@ enum { HD_COOKIE, HD_DATE, HD_EXPECT, + HD_FORWARDED, HD_HOST, HD_HTTP2_SETTINGS, HD_IF_MODIFIED_SINCE, diff --git a/src/shrpx.cc b/src/shrpx.cc index 6d603a9c..ddfde86e 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -64,6 +64,7 @@ #include #include #include +#include #include #include @@ -1832,6 +1833,10 @@ int main(int argc, char **argv) { {SHRPX_OPT_FASTOPEN, required_argument, &flag, 94}, {SHRPX_OPT_TLS_DYN_REC_WARMUP_THRESHOLD, required_argument, &flag, 95}, {SHRPX_OPT_TLS_DYN_REC_IDLE_TIMEOUT, required_argument, &flag, 96}, + {SHRPX_OPT_ADD_FORWARDED, required_argument, &flag, 97}, + {SHRPX_OPT_STRIP_INCOMING_FORWARDED, no_argument, &flag, 98}, + {SHRPX_OPT_FORWARDED_BY, required_argument, &flag, 99}, + {SHRPX_OPT_FORWARDED_FOR, required_argument, &flag, 100}, {nullptr, 0, nullptr, 0}}; int option_index = 0; @@ -2245,6 +2250,22 @@ int main(int argc, char **argv) { // --tls-dyn-rec-idle-timeout cmdcfgs.emplace_back(SHRPX_OPT_TLS_DYN_REC_IDLE_TIMEOUT, optarg); break; + case 97: + // --add-forwarded + cmdcfgs.emplace_back(SHRPX_OPT_ADD_FORWARDED, optarg); + break; + case 98: + // --strip-incoming-forwarded + cmdcfgs.emplace_back(SHRPX_OPT_STRIP_INCOMING_FORWARDED, "yes"); + break; + case 99: + // --forwarded-by + cmdcfgs.emplace_back(SHRPX_OPT_FORWARDED_BY, optarg); + break; + case 100: + // --forwarded-for + cmdcfgs.emplace_back(SHRPX_OPT_FORWARDED_FOR, optarg); + break; default: break; } @@ -2555,6 +2576,14 @@ int main(int argc, char **argv) { } } + if (get_config()->forwarded_by_node_type == FORWARDED_NODE_OBFUSCATED) { + std::random_device rd; + std::mt19937 gen(rd()); + auto &dst = mod_config()->forwarded_by_obfuscated; + dst = "_"; + dst += util::random_alpha_digit(gen, SHRPX_OBFUSCATED_NODE_LENGTH); + } + if (get_config()->upstream_frame_debug) { // To make it sync to logging set_output(stderr); diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index c4290e2b..2bb31241 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -27,6 +27,13 @@ #ifdef HAVE_UNISTD_H #include #endif // HAVE_UNISTD_H +#ifdef HAVE_SYS_SOCKET_H +#include +#endif // HAVE_SYS_SOCKET_H +#ifdef HAVE_NETDB_H +#include +#endif // HAVE_NETDB_H + #include #include "shrpx_upstream.h" @@ -396,6 +403,13 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl, } else { setup_upstream_io_callback(); } + + if ((get_config()->forwarded_params & FORWARDED_FOR) && + get_config()->forwarded_for_node_type == FORWARDED_NODE_OBFUSCATED) { + forwarded_for_obfuscated_ = "_"; + forwarded_for_obfuscated_ += util::random_alpha_digit( + worker_->get_randgen(), SHRPX_OBFUSCATED_NODE_LENGTH); + } } void ClientHandler::setup_upstream_io_callback() { @@ -1099,4 +1113,50 @@ int ClientHandler::proxy_protocol_read() { return on_proxy_protocol_finish(); } +const std::string &ClientHandler::get_forwarded_by() { + if (get_config()->forwarded_by_node_type == FORWARDED_NODE_OBFUSCATED) { + return get_config()->forwarded_by_obfuscated; + } + if (!local_hostport_.empty()) { + return local_hostport_; + } + + int rv; + sockaddr_union su; + socklen_t addrlen = sizeof(su); + + rv = getsockname(conn_.fd, &su.sa, &addrlen); + if (rv != 0) { + return local_hostport_; + } + + char host[NI_MAXHOST]; + rv = getnameinfo(&su.sa, addrlen, host, sizeof(host), nullptr, 0, + NI_NUMERICHOST); + if (rv != 0) { + return local_hostport_; + } + + if (su.storage.ss_family == AF_INET6) { + local_hostport_ = "["; + local_hostport_ += host; + local_hostport_ += "]:"; + } else { + local_hostport_ = host; + local_hostport_ += ':'; + } + + local_hostport_ += util::utos(get_config()->port); + + return local_hostport_; +} + +const std::string &ClientHandler::get_forwarded_for() const { + if (get_config()->forwarded_for_node_type == FORWARDED_NODE_OBFUSCATED) { + return forwarded_for_obfuscated_; + } + + return ipaddr_; +} + } // namespace shrpx diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index b05013c8..95c902a5 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -134,6 +134,13 @@ public: void setup_upstream_io_callback(); + // Returns string suitable for use in "by" parameter of Forwarded + // header field. + const std::string &get_forwarded_by(); + // Returns string suitable for use in "for" parameter of Forwarded + // header field. + const std::string &get_forwarded_for() const; + private: Connection conn_; ev_timer reneg_shutdown_timer_; @@ -143,6 +150,11 @@ private: std::string port_; // The ALPN identifier negotiated for this connection. std::string alpn_; + // Host and port of this socket (e.g., "[::1]:8443") + std::string local_hostport_; + // The obfuscated version of client address used in "for" parameter + // of Forwarded header field. + std::string forwarded_for_obfuscated_; std::function read_, write_; std::function on_read_, on_write_; Worker *worker_; diff --git a/src/shrpx_config.cc b/src/shrpx_config.cc index 2281d4ab..a6d2fd19 100644 --- a/src/shrpx_config.cc +++ b/src/shrpx_config.cc @@ -628,12 +628,27 @@ void parse_mapping(const DownstreamAddr &addr, const char *src) { } } // namespace +namespace { +int parse_forwarded_node_type(const std::string &optarg) { + if (util::strieq(optarg, "obfuscated")) { + return FORWARDED_NODE_OBFUSCATED; + } + + if (util::strieq(optarg, "ip")) { + return FORWARDED_NODE_IP; + } + + return -1; +} +} // namespace + // generated by gennghttpxfun.py enum { SHRPX_OPTID_ACCEPT_PROXY_PROTOCOL, SHRPX_OPTID_ACCESSLOG_FILE, SHRPX_OPTID_ACCESSLOG_FORMAT, SHRPX_OPTID_ACCESSLOG_SYSLOG, + SHRPX_OPTID_ADD_FORWARDED, SHRPX_OPTID_ADD_REQUEST_HEADER, SHRPX_OPTID_ADD_RESPONSE_HEADER, SHRPX_OPTID_ADD_X_FORWARDED_FOR, @@ -669,6 +684,8 @@ enum { SHRPX_OPTID_ERRORLOG_SYSLOG, SHRPX_OPTID_FASTOPEN, SHRPX_OPTID_FETCH_OCSP_RESPONSE_FILE, + SHRPX_OPTID_FORWARDED_BY, + SHRPX_OPTID_FORWARDED_FOR, SHRPX_OPTID_FRONTEND, SHRPX_OPTID_FRONTEND_FRAME_DEBUG, SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS, @@ -707,6 +724,7 @@ enum { SHRPX_OPTID_RLIMIT_NOFILE, SHRPX_OPTID_STREAM_READ_TIMEOUT, SHRPX_OPTID_STREAM_WRITE_TIMEOUT, + SHRPX_OPTID_STRIP_INCOMING_FORWARDED, SHRPX_OPTID_STRIP_INCOMING_X_FORWARDED_FOR, SHRPX_OPTID_SUBCERT, SHRPX_OPTID_SYSLOG_FACILITY, @@ -915,11 +933,19 @@ int option_lookup_token(const char *name, size_t namelen) { if (util::strieq_l("client-prox", name, 11)) { return SHRPX_OPTID_CLIENT_PROXY; } + if (util::strieq_l("forwarded-b", name, 11)) { + return SHRPX_OPTID_FORWARDED_BY; + } break; } break; case 13: switch (name[12]) { + case 'd': + if (util::strieq_l("add-forwarde", name, 12)) { + return SHRPX_OPTID_ADD_FORWARDED; + } + break; case 'e': if (util::strieq_l("dh-param-fil", name, 12)) { return SHRPX_OPTID_DH_PARAM_FILE; @@ -931,6 +957,11 @@ int option_lookup_token(const char *name, size_t namelen) { return SHRPX_OPTID_RLIMIT_NOFILE; } break; + case 'r': + if (util::strieq_l("forwarded-fo", name, 12)) { + return SHRPX_OPTID_FORWARDED_FOR; + } + break; case 't': if (util::strieq_l("verify-clien", name, 12)) { return SHRPX_OPTID_VERIFY_CLIENT; @@ -1166,6 +1197,9 @@ int option_lookup_token(const char *name, size_t namelen) { case 24: switch (name[23]) { case 'd': + if (util::strieq_l("strip-incoming-forwarde", name, 23)) { + return SHRPX_OPTID_STRIP_INCOMING_FORWARDED; + } if (util::strieq_l("tls-ticket-key-memcache", name, 23)) { return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED; } @@ -2015,6 +2049,57 @@ int parse_config(const char *opt, const char *optarg, mod_config()->accept_proxy_protocol = util::strieq(optarg, "yes"); return 0; + case SHRPX_OPTID_ADD_FORWARDED: + for (const auto ¶m : util::parse_config_str_list(optarg)) { + if (util::strieq(param, "by")) { + mod_config()->forwarded_params |= FORWARDED_BY; + continue; + } + if (util::strieq(param, "for")) { + mod_config()->forwarded_params |= FORWARDED_FOR; + continue; + } + if (util::strieq(param, "host")) { + mod_config()->forwarded_params |= FORWARDED_HOST; + continue; + } + if (util::strieq(param, "proto")) { + mod_config()->forwarded_params |= FORWARDED_PROTO; + continue; + } + + LOG(ERROR) << opt << ": unknown parameter " << optarg; + + return -1; + } + + return 0; + case SHRPX_OPTID_STRIP_INCOMING_FORWARDED: + mod_config()->strip_incoming_forwarded = util::strieq(optarg, "yes"); + + return 0; + case SHRPX_OPTID_FORWARDED_BY: + case SHRPX_OPTID_FORWARDED_FOR: { + auto type = parse_forwarded_node_type(optarg); + + if (type == -1) { + LOG(ERROR) << opt << ": unknown node type " << optarg; + return -1; + } + + switch (optid) { + case SHRPX_OPTID_FORWARDED_BY: + mod_config()->forwarded_by_node_type = + static_cast(type); + break; + case SHRPX_OPTID_FORWARDED_FOR: + mod_config()->forwarded_for_node_type = + static_cast(type); + break; + } + + return 0; + } case SHRPX_OPTID_CONF: LOG(WARN) << "conf: ignored"; diff --git a/src/shrpx_config.h b/src/shrpx_config.h index f53ebd36..b360cc06 100644 --- a/src/shrpx_config.h +++ b/src/shrpx_config.h @@ -191,6 +191,13 @@ constexpr char SHRPX_OPT_TLS_DYN_REC_WARMUP_THRESHOLD[] = "tls-dyn-rec-warmup-threshold"; constexpr char SHRPX_OPT_TLS_DYN_REC_IDLE_TIMEOUT[] = "tls-dyn-rec-idle-timeout"; +constexpr char SHRPX_OPT_ADD_FORWARDED[] = "add-forwarded"; +constexpr char SHRPX_OPT_STRIP_INCOMING_FORWARDED[] = + "strip-incoming-forwarded"; +constexpr static char SHRPX_OPT_FORWARDED_BY[] = "forwarded-by"; +constexpr char SHRPX_OPT_FORWARDED_FOR[] = "forwarded-for"; + +constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8; union sockaddr_union { sockaddr_storage storage; @@ -207,6 +214,19 @@ struct Address { enum shrpx_proto { PROTO_HTTP2, PROTO_HTTP }; +enum shrpx_forwarded_param { + FORWARDED_NONE = 0, + FORWARDED_BY = 0x1, + FORWARDED_FOR = 0x2, + FORWARDED_HOST = 0x4, + FORWARDED_PROTO = 0x8, +}; + +enum shrpx_forwarded_node_type { + FORWARDED_NODE_OBFUSCATED, + FORWARDED_NODE_IP, +}; + struct AltSvc { AltSvc() : port(0) {} @@ -283,6 +303,9 @@ struct Config { Address session_cache_memcached_addr; Address tls_ticket_key_memcached_addr; Router router; + // obfuscated value used in "by" parameter of Forwarded header + // field. + std::string forwarded_by_obfuscated; std::chrono::seconds tls_session_timeout; ev_tstamp http2_upstream_read_timeout; ev_tstamp upstream_read_timeout; @@ -377,6 +400,14 @@ struct Config { long int tls_proto_mask; // downstream protocol; this will be determined by given options. shrpx_proto downstream_proto; + // bitwise-OR of one or more of shrpx_forwarded_param values. + uint32_t forwarded_params; + // type of value recorded in "by" parameter of Forwarded header + // field. + shrpx_forwarded_node_type forwarded_by_node_type; + // type of value recorded in "for" parameter of Forwarded header + // field. + shrpx_forwarded_node_type forwarded_for_node_type; int syslog_facility; int backlog; int argc; @@ -399,6 +430,7 @@ struct Config { bool client_proxy; bool add_x_forwarded_for; bool strip_incoming_x_forwarded_for; + bool strip_incoming_forwarded; bool no_via; bool upstream_no_tls; bool downstream_no_tls; diff --git a/src/shrpx_http.cc b/src/shrpx_http.cc index 5f2f4b9e..8407f10d 100644 --- a/src/shrpx_http.cc +++ b/src/shrpx_http.cc @@ -62,6 +62,41 @@ std::string create_via_header_value(int major, int minor) { return hdrs; } +std::string create_forwarded(int params, const std::string &node_by, + const std::string &node_for, + const std::string &host, + const std::string &proto) { + std::string res; + if ((params & FORWARDED_BY) && !node_by.empty()) { + res += "by=\""; + res += node_by; + res += "\";"; + } + if ((params & FORWARDED_FOR) && !node_for.empty()) { + res += "for=\""; + res += node_for; + res += "\";"; + } + if ((params & FORWARDED_HOST) && !host.empty()) { + res += "host=\""; + res += host; + res += "\";"; + } + if ((params & FORWARDED_PROTO) && !proto.empty()) { + res += "proto=\""; + res += proto; + res += "\";"; + } + + if (res.empty()) { + return res; + } + + res.erase(res.size() - 1); + + return res; +} + std::string colorizeHeaders(const char *hdrs) { std::string nhdrs; const char *p = strchr(hdrs, '\n'); diff --git a/src/shrpx_http.h b/src/shrpx_http.h index 65dbe0ec..e4c20958 100644 --- a/src/shrpx_http.h +++ b/src/shrpx_http.h @@ -39,6 +39,10 @@ std::string create_error_html(unsigned int status_code); std::string create_via_header_value(int major, int minor); +std::string create_forwarded(int params, const std::string &node_by, + const std::string &node_for, + const std::string &host, const std::string &proto); + // Adds ANSI color codes to HTTP headers |hdrs|. std::string colorizeHeaders(const char *hdrs); diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc index 4f39ab05..170cd4d8 100644 --- a/src/shrpx_http_downstream_connection.cc +++ b/src/shrpx_http_downstream_connection.cc @@ -296,6 +296,41 @@ int HttpDownstreamConnection::push_request_headers() { } } + auto upstream = downstream_->get_upstream(); + auto handler = upstream->get_client_handler(); + + auto fwd = get_config()->strip_incoming_forwarded + ? nullptr + : req.fs.header(http2::HD_FORWARDED); + if (get_config()->forwarded_params) { + auto params = get_config()->forwarded_params; + + if (get_config()->http2_proxy || get_config()->client_proxy || + connect_method) { + params &= ~FORWARDED_PROTO; + } + + auto value = http::create_forwarded(params, handler->get_forwarded_by(), + handler->get_forwarded_for(), + req.authority, req.scheme); + if (fwd || !value.empty()) { + buf->append("Forwarded: "); + if (fwd) { + buf->append(fwd->value); + + if (!value.empty()) { + buf->append(", "); + } + } + buf->append(value); + buf->append("\r\n"); + } + } else if (fwd) { + buf->append("Forwarded: "); + buf->append(fwd->value); + buf->append("\r\n"); + } + auto xff = req.fs.header(http2::HD_X_FORWARDED_FOR); if (get_config()->add_x_forwarded_for) { buf->append("X-Forwarded-For: "); diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc index 665a3d45..3d329900 100644 --- a/src/shrpx_worker.cc +++ b/src/shrpx_worker.cc @@ -62,10 +62,14 @@ void mcpool_clear_cb(struct ev_loop *loop, ev_timer *w, int revents) { } } // namespace +namespace { +std::random_device rd; +} // namespace + Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx, ssl::CertLookupTree *cert_tree, const std::shared_ptr &ticket_keys) - : dconn_pool_(get_config()->downstream_addr_groups.size()), + : randgen_(rd()), dconn_pool_(get_config()->downstream_addr_groups.size()), worker_stat_(get_config()->downstream_addr_groups.size()), dgrps_(get_config()->downstream_addr_groups.size()), loop_(loop), sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), cert_tree_(cert_tree), @@ -268,6 +272,8 @@ MemcachedDispatcher *Worker::get_session_cache_memcached_dispatcher() { return session_cache_memcached_dispatcher_.get(); } +std::mt19937 &Worker::get_randgen() { return randgen_; } + #ifdef HAVE_MRUBY int Worker::create_mruby_context() { auto mruby_file = get_config()->mruby_file.get(); diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h index 6037683d..ce00e46e 100644 --- a/src/shrpx_worker.h +++ b/src/shrpx_worker.h @@ -29,6 +29,7 @@ #include #include +#include #include #ifndef NOTHREADS #include @@ -132,6 +133,8 @@ public: MemcachedDispatcher *get_session_cache_memcached_dispatcher(); + std::mt19937 &get_randgen(); + #ifdef HAVE_MRUBY int create_mruby_context(); @@ -144,6 +147,7 @@ private: #endif // NOTHREADS std::mutex m_; std::vector q_; + std::mt19937 randgen_; ev_async w_; ev_timer mcpool_clear_timer_; MemchunkPool mcpool_; diff --git a/src/util.h b/src/util.h index b09bf550..eb879c15 100644 --- a/src/util.h +++ b/src/util.h @@ -45,6 +45,7 @@ #include #include #include +#include #include "http-parser/http_parser.h" @@ -622,6 +623,18 @@ uint64_t get_uint64(const uint8_t *data); int read_mime_types(std::map &res, const char *filename); +template +std::string random_alpha_digit(Generator &gen, size_t len) { + std::string res; + res.reserve(len); + std::uniform_int_distribution<> dis(0, 26 * 2 + 10 - 1); + for (; len > 0; --len) { + res += "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"[dis( + gen)]; + } + return res; +} + } // namespace util } // namespace nghttp2