nghttpx: Add RFC 7239 Forwarded header field support

genheaderfunc.py

@@ -17,6 +17,7 @@ HEADERS = [
     "http2-settings",
     "server",
     "via",
+    "forwarded",
     "x-forwarded-for",
     "x-forwarded-proto",
     "alt-svc",

gennghttpxfun.py

@@ -103,7 +103,11 @@ OPTIONS = [
     "conf",
     "fastopen",
     "tls-dyn-rec-warmup-threshold",
-    "tls-dyn-rec-idle-timeout"
+    "tls-dyn-rec-idle-timeout",
+    "add-forwarded",
+    "strip-incoming-forwarded",
+    "forwarded-by",
+    "forwarded-for"
 ]
 
 LOGVARS = [

src/http2.cc

@@ -348,6 +348,7 @@ void copy_headers_to_nva_internal(std::vector<nghttp2_nv> &nva,
     switch (kv.token) {
     case HD_COOKIE:
     case HD_CONNECTION:
+    case HD_FORWARDED:
     case HD_HOST:
     case HD_HTTP2_SETTINGS:
     case HD_KEEP_ALIVE:
@@ -385,6 +386,7 @@ void build_http1_headers_from_headers(DefaultMemchunks *buf,
     switch (kv.token) {
     case HD_CONNECTION:
     case HD_COOKIE:
+    case HD_FORWARDED:
     case HD_HOST:
     case HD_HTTP2_SETTINGS:
     case HD_KEEP_ALIVE:
@@ -638,6 +640,15 @@ int lookup_token(const uint8_t *name, size_t namelen) {
       break;
     }
     break;
+  case 9:
+    switch (name[8]) {
+    case 'd':
+      if (util::streq_l("forwarde", name, 8)) {
+        return HD_FORWARDED;
+      }
+      break;
+    }
+    break;
   case 10:
     switch (name[9]) {
     case 'e':

src/http2.h

@@ -218,6 +218,8 @@ int parse_http_status_code(const std::string &src);
 
 // Header fields to be indexed, except HD_MAXIDX which is convenient
 // member to get maximum value.
+//
+// generated by genheaderfunc.py
 enum {
   HD__AUTHORITY,
   HD__HOST,
@@ -234,6 +236,7 @@ enum {
   HD_COOKIE,
   HD_DATE,
   HD_EXPECT,
+  HD_FORWARDED,
   HD_HOST,
   HD_HTTP2_SETTINGS,
   HD_IF_MODIFIED_SINCE,

src/shrpx.cc

@@ -64,6 +64,7 @@
 #include <fstream>
 #include <vector>
 #include <initializer_list>
+#include <random>
 
 #include <openssl/ssl.h>
 #include <openssl/err.h>
@@ -1832,6 +1833,10 @@ int main(int argc, char **argv) {
         {SHRPX_OPT_FASTOPEN, required_argument, &flag, 94},
         {SHRPX_OPT_TLS_DYN_REC_WARMUP_THRESHOLD, required_argument, &flag, 95},
         {SHRPX_OPT_TLS_DYN_REC_IDLE_TIMEOUT, required_argument, &flag, 96},
+        {SHRPX_OPT_ADD_FORWARDED, required_argument, &flag, 97},
+        {SHRPX_OPT_STRIP_INCOMING_FORWARDED, no_argument, &flag, 98},
+        {SHRPX_OPT_FORWARDED_BY, required_argument, &flag, 99},
+        {SHRPX_OPT_FORWARDED_FOR, required_argument, &flag, 100},
         {nullptr, 0, nullptr, 0}};
 
     int option_index = 0;
@@ -2245,6 +2250,22 @@ int main(int argc, char **argv) {
         // --tls-dyn-rec-idle-timeout
         cmdcfgs.emplace_back(SHRPX_OPT_TLS_DYN_REC_IDLE_TIMEOUT, optarg);
         break;
+      case 97:
+        // --add-forwarded
+        cmdcfgs.emplace_back(SHRPX_OPT_ADD_FORWARDED, optarg);
+        break;
+      case 98:
+        // --strip-incoming-forwarded
+        cmdcfgs.emplace_back(SHRPX_OPT_STRIP_INCOMING_FORWARDED, "yes");
+        break;
+      case 99:
+        // --forwarded-by
+        cmdcfgs.emplace_back(SHRPX_OPT_FORWARDED_BY, optarg);
+        break;
+      case 100:
+        // --forwarded-for
+        cmdcfgs.emplace_back(SHRPX_OPT_FORWARDED_FOR, optarg);
+        break;
       default:
         break;
       }
@@ -2555,6 +2576,14 @@ int main(int argc, char **argv) {
     }
   }
 
+  if (get_config()->forwarded_by_node_type == FORWARDED_NODE_OBFUSCATED) {
+    std::random_device rd;
+    std::mt19937 gen(rd());
+    auto &dst = mod_config()->forwarded_by_obfuscated;
+    dst = "_";
+    dst += util::random_alpha_digit(gen, SHRPX_OBFUSCATED_NODE_LENGTH);
+  }
+
   if (get_config()->upstream_frame_debug) {
     // To make it sync to logging
     set_output(stderr);

src/shrpx_client_handler.cc

@@ -27,6 +27,13 @@
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif // HAVE_UNISTD_H
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif // HAVE_SYS_SOCKET_H
+#ifdef HAVE_NETDB_H
+#include <netdb.h>
+#endif // HAVE_NETDB_H
+
 #include <cerrno>
 
 #include "shrpx_upstream.h"
@@ -396,6 +403,13 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl,
   } else {
     setup_upstream_io_callback();
   }
+
+  if ((get_config()->forwarded_params & FORWARDED_FOR) &&
+      get_config()->forwarded_for_node_type == FORWARDED_NODE_OBFUSCATED) {
+    forwarded_for_obfuscated_ = "_";
+    forwarded_for_obfuscated_ += util::random_alpha_digit(
+        worker_->get_randgen(), SHRPX_OBFUSCATED_NODE_LENGTH);
+  }
 }
 
 void ClientHandler::setup_upstream_io_callback() {
@@ -1099,4 +1113,50 @@ int ClientHandler::proxy_protocol_read() {
   return on_proxy_protocol_finish();
 }
 
+const std::string &ClientHandler::get_forwarded_by() {
+  if (get_config()->forwarded_by_node_type == FORWARDED_NODE_OBFUSCATED) {
+    return get_config()->forwarded_by_obfuscated;
+  }
+  if (!local_hostport_.empty()) {
+    return local_hostport_;
+  }
+
+  int rv;
+  sockaddr_union su;
+  socklen_t addrlen = sizeof(su);
+
+  rv = getsockname(conn_.fd, &su.sa, &addrlen);
+  if (rv != 0) {
+    return local_hostport_;
+  }
+
+  char host[NI_MAXHOST];
+  rv = getnameinfo(&su.sa, addrlen, host, sizeof(host), nullptr, 0,
+                   NI_NUMERICHOST);
+  if (rv != 0) {
+    return local_hostport_;
+  }
+
+  if (su.storage.ss_family == AF_INET6) {
+    local_hostport_ = "[";
+    local_hostport_ += host;
+    local_hostport_ += "]:";
+  } else {
+    local_hostport_ = host;
+    local_hostport_ += ':';
+  }
+
+  local_hostport_ += util::utos(get_config()->port);
+
+  return local_hostport_;
+}
+
+const std::string &ClientHandler::get_forwarded_for() const {
+  if (get_config()->forwarded_for_node_type == FORWARDED_NODE_OBFUSCATED) {
+    return forwarded_for_obfuscated_;
+  }
+
+  return ipaddr_;
+}
+
 } // namespace shrpx

src/shrpx_client_handler.h

@@ -134,6 +134,13 @@ public:
 
   void setup_upstream_io_callback();
 
+  // Returns string suitable for use in "by" parameter of Forwarded
+  // header field.
+  const std::string &get_forwarded_by();
+  // Returns string suitable for use in "for" parameter of Forwarded
+  // header field.
+  const std::string &get_forwarded_for() const;
+
 private:
   Connection conn_;
   ev_timer reneg_shutdown_timer_;
@@ -143,6 +150,11 @@ private:
   std::string port_;
   // The ALPN identifier negotiated for this connection.
   std::string alpn_;
+  // Host and port of this socket (e.g., "[::1]:8443")
+  std::string local_hostport_;
+  // The obfuscated version of client address used in "for" parameter
+  // of Forwarded header field.
+  std::string forwarded_for_obfuscated_;
   std::function<int(ClientHandler &)> read_, write_;
   std::function<int(ClientHandler &)> on_read_, on_write_;
   Worker *worker_;

src/shrpx_config.cc

@@ -628,12 +628,27 @@ void parse_mapping(const DownstreamAddr &addr, const char *src) {
 }
 } // namespace
 
+namespace {
+int parse_forwarded_node_type(const std::string &optarg) {
+  if (util::strieq(optarg, "obfuscated")) {
+    return FORWARDED_NODE_OBFUSCATED;
+  }
+
+  if (util::strieq(optarg, "ip")) {
+    return FORWARDED_NODE_IP;
+  }
+
+  return -1;
+}
+} // namespace
+
 // generated by gennghttpxfun.py
 enum {
   SHRPX_OPTID_ACCEPT_PROXY_PROTOCOL,
   SHRPX_OPTID_ACCESSLOG_FILE,
   SHRPX_OPTID_ACCESSLOG_FORMAT,
   SHRPX_OPTID_ACCESSLOG_SYSLOG,
+  SHRPX_OPTID_ADD_FORWARDED,
   SHRPX_OPTID_ADD_REQUEST_HEADER,
   SHRPX_OPTID_ADD_RESPONSE_HEADER,
   SHRPX_OPTID_ADD_X_FORWARDED_FOR,
@@ -669,6 +684,8 @@ enum {
   SHRPX_OPTID_ERRORLOG_SYSLOG,
   SHRPX_OPTID_FASTOPEN,
   SHRPX_OPTID_FETCH_OCSP_RESPONSE_FILE,
+  SHRPX_OPTID_FORWARDED_BY,
+  SHRPX_OPTID_FORWARDED_FOR,
   SHRPX_OPTID_FRONTEND,
   SHRPX_OPTID_FRONTEND_FRAME_DEBUG,
   SHRPX_OPTID_FRONTEND_HTTP2_CONNECTION_WINDOW_BITS,
@@ -707,6 +724,7 @@ enum {
   SHRPX_OPTID_RLIMIT_NOFILE,
   SHRPX_OPTID_STREAM_READ_TIMEOUT,
   SHRPX_OPTID_STREAM_WRITE_TIMEOUT,
+  SHRPX_OPTID_STRIP_INCOMING_FORWARDED,
   SHRPX_OPTID_STRIP_INCOMING_X_FORWARDED_FOR,
   SHRPX_OPTID_SUBCERT,
   SHRPX_OPTID_SYSLOG_FACILITY,
@@ -915,11 +933,19 @@ int option_lookup_token(const char *name, size_t namelen) {
       if (util::strieq_l("client-prox", name, 11)) {
         return SHRPX_OPTID_CLIENT_PROXY;
       }
+      if (util::strieq_l("forwarded-b", name, 11)) {
+        return SHRPX_OPTID_FORWARDED_BY;
+      }
       break;
     }
     break;
   case 13:
     switch (name[12]) {
+    case 'd':
+      if (util::strieq_l("add-forwarde", name, 12)) {
+        return SHRPX_OPTID_ADD_FORWARDED;
+      }
+      break;
     case 'e':
       if (util::strieq_l("dh-param-fil", name, 12)) {
         return SHRPX_OPTID_DH_PARAM_FILE;
@@ -931,6 +957,11 @@ int option_lookup_token(const char *name, size_t namelen) {
         return SHRPX_OPTID_RLIMIT_NOFILE;
       }
       break;
+    case 'r':
+      if (util::strieq_l("forwarded-fo", name, 12)) {
+        return SHRPX_OPTID_FORWARDED_FOR;
+      }
+      break;
     case 't':
       if (util::strieq_l("verify-clien", name, 12)) {
         return SHRPX_OPTID_VERIFY_CLIENT;
@@ -1166,6 +1197,9 @@ int option_lookup_token(const char *name, size_t namelen) {
   case 24:
     switch (name[23]) {
     case 'd':
+      if (util::strieq_l("strip-incoming-forwarde", name, 23)) {
+        return SHRPX_OPTID_STRIP_INCOMING_FORWARDED;
+      }
       if (util::strieq_l("tls-ticket-key-memcache", name, 23)) {
         return SHRPX_OPTID_TLS_TICKET_KEY_MEMCACHED;
       }
@@ -2015,6 +2049,57 @@ int parse_config(const char *opt, const char *optarg,
     mod_config()->accept_proxy_protocol = util::strieq(optarg, "yes");
 
     return 0;
+  case SHRPX_OPTID_ADD_FORWARDED:
+    for (const auto &param : util::parse_config_str_list(optarg)) {
+      if (util::strieq(param, "by")) {
+        mod_config()->forwarded_params |= FORWARDED_BY;
+        continue;
+      }
+      if (util::strieq(param, "for")) {
+        mod_config()->forwarded_params |= FORWARDED_FOR;
+        continue;
+      }
+      if (util::strieq(param, "host")) {
+        mod_config()->forwarded_params |= FORWARDED_HOST;
+        continue;
+      }
+      if (util::strieq(param, "proto")) {
+        mod_config()->forwarded_params |= FORWARDED_PROTO;
+        continue;
+      }
+
+      LOG(ERROR) << opt << ": unknown parameter " << optarg;
+
+      return -1;
+    }
+
+    return 0;
+  case SHRPX_OPTID_STRIP_INCOMING_FORWARDED:
+    mod_config()->strip_incoming_forwarded = util::strieq(optarg, "yes");
+
+    return 0;
+  case SHRPX_OPTID_FORWARDED_BY:
+  case SHRPX_OPTID_FORWARDED_FOR: {
+    auto type = parse_forwarded_node_type(optarg);
+
+    if (type == -1) {
+      LOG(ERROR) << opt << ": unknown node type " << optarg;
+      return -1;
+    }
+
+    switch (optid) {
+    case SHRPX_OPTID_FORWARDED_BY:
+      mod_config()->forwarded_by_node_type =
+          static_cast<shrpx_forwarded_node_type>(type);
+      break;
+    case SHRPX_OPTID_FORWARDED_FOR:
+      mod_config()->forwarded_for_node_type =
+          static_cast<shrpx_forwarded_node_type>(type);
+      break;
+    }
+
+    return 0;
+  }
   case SHRPX_OPTID_CONF:
     LOG(WARN) << "conf: ignored";
 

src/shrpx_config.h

@@ -191,6 +191,13 @@ constexpr char SHRPX_OPT_TLS_DYN_REC_WARMUP_THRESHOLD[] =
     "tls-dyn-rec-warmup-threshold";
 constexpr char SHRPX_OPT_TLS_DYN_REC_IDLE_TIMEOUT[] =
     "tls-dyn-rec-idle-timeout";
+constexpr char SHRPX_OPT_ADD_FORWARDED[] = "add-forwarded";
+constexpr char SHRPX_OPT_STRIP_INCOMING_FORWARDED[] =
+    "strip-incoming-forwarded";
+constexpr static char SHRPX_OPT_FORWARDED_BY[] = "forwarded-by";
+constexpr char SHRPX_OPT_FORWARDED_FOR[] = "forwarded-for";
+
+constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8;
 
 union sockaddr_union {
   sockaddr_storage storage;
@@ -207,6 +214,19 @@ struct Address {
 
 enum shrpx_proto { PROTO_HTTP2, PROTO_HTTP };
 
+enum shrpx_forwarded_param {
+  FORWARDED_NONE = 0,
+  FORWARDED_BY = 0x1,
+  FORWARDED_FOR = 0x2,
+  FORWARDED_HOST = 0x4,
+  FORWARDED_PROTO = 0x8,
+};
+
+enum shrpx_forwarded_node_type {
+  FORWARDED_NODE_OBFUSCATED,
+  FORWARDED_NODE_IP,
+};
+
 struct AltSvc {
   AltSvc() : port(0) {}
 
@@ -283,6 +303,9 @@ struct Config {
   Address session_cache_memcached_addr;
   Address tls_ticket_key_memcached_addr;
   Router router;
+  // obfuscated value used in "by" parameter of Forwarded header
+  // field.
+  std::string forwarded_by_obfuscated;
   std::chrono::seconds tls_session_timeout;
   ev_tstamp http2_upstream_read_timeout;
   ev_tstamp upstream_read_timeout;
@@ -377,6 +400,14 @@ struct Config {
   long int tls_proto_mask;
   // downstream protocol; this will be determined by given options.
   shrpx_proto downstream_proto;
+  // bitwise-OR of one or more of shrpx_forwarded_param values.
+  uint32_t forwarded_params;
+  // type of value recorded in "by" parameter of Forwarded header
+  // field.
+  shrpx_forwarded_node_type forwarded_by_node_type;
+  // type of value recorded in "for" parameter of Forwarded header
+  // field.
+  shrpx_forwarded_node_type forwarded_for_node_type;
   int syslog_facility;
   int backlog;
   int argc;
@@ -399,6 +430,7 @@ struct Config {
   bool client_proxy;
   bool add_x_forwarded_for;
   bool strip_incoming_x_forwarded_for;
+  bool strip_incoming_forwarded;
   bool no_via;
   bool upstream_no_tls;
   bool downstream_no_tls;

src/shrpx_http.cc

@@ -62,6 +62,41 @@ std::string create_via_header_value(int major, int minor) {
   return hdrs;
 }
 
+std::string create_forwarded(int params, const std::string &node_by,
+                             const std::string &node_for,
+                             const std::string &host,
+                             const std::string &proto) {
+  std::string res;
+  if ((params & FORWARDED_BY) && !node_by.empty()) {
+    res += "by=\"";
+    res += node_by;
+    res += "\";";
+  }
+  if ((params & FORWARDED_FOR) && !node_for.empty()) {
+    res += "for=\"";
+    res += node_for;
+    res += "\";";
+  }
+  if ((params & FORWARDED_HOST) && !host.empty()) {
+    res += "host=\"";
+    res += host;
+    res += "\";";
+  }
+  if ((params & FORWARDED_PROTO) && !proto.empty()) {
+    res += "proto=\"";
+    res += proto;
+    res += "\";";
+  }
+
+  if (res.empty()) {
+    return res;
+  }
+
+  res.erase(res.size() - 1);
+
+  return res;
+}
+
 std::string colorizeHeaders(const char *hdrs) {
   std::string nhdrs;
   const char *p = strchr(hdrs, '\n');

src/shrpx_http.h

@@ -39,6 +39,10 @@ std::string create_error_html(unsigned int status_code);
 
 std::string create_via_header_value(int major, int minor);
 
+std::string create_forwarded(int params, const std::string &node_by,
+                             const std::string &node_for,
+                             const std::string &host, const std::string &proto);
+
 // Adds ANSI color codes to HTTP headers |hdrs|.
 std::string colorizeHeaders(const char *hdrs);
 

src/shrpx_http_downstream_connection.cc

@@ -296,6 +296,41 @@ int HttpDownstreamConnection::push_request_headers() {
     }
   }
 
+  auto upstream = downstream_->get_upstream();
+  auto handler = upstream->get_client_handler();
+
+  auto fwd = get_config()->strip_incoming_forwarded
+                 ? nullptr
+                 : req.fs.header(http2::HD_FORWARDED);
+  if (get_config()->forwarded_params) {
+    auto params = get_config()->forwarded_params;
+
+    if (get_config()->http2_proxy || get_config()->client_proxy ||
+        connect_method) {
+      params &= ~FORWARDED_PROTO;
+    }
+
+    auto value = http::create_forwarded(params, handler->get_forwarded_by(),
+                                        handler->get_forwarded_for(),
+                                        req.authority, req.scheme);
+    if (fwd || !value.empty()) {
+      buf->append("Forwarded: ");
+      if (fwd) {
+        buf->append(fwd->value);
+
+        if (!value.empty()) {
+          buf->append(", ");
+        }
+      }
+      buf->append(value);
+      buf->append("\r\n");
+    }
+  } else if (fwd) {
+    buf->append("Forwarded: ");
+    buf->append(fwd->value);
+    buf->append("\r\n");
+  }
+
   auto xff = req.fs.header(http2::HD_X_FORWARDED_FOR);
   if (get_config()->add_x_forwarded_for) {
     buf->append("X-Forwarded-For: ");

src/shrpx_worker.cc

@@ -62,10 +62,14 @@ void mcpool_clear_cb(struct ev_loop *loop, ev_timer *w, int revents) {
 }
 } // namespace
 
+namespace {
+std::random_device rd;
+} // namespace
+
 Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
                ssl::CertLookupTree *cert_tree,
                const std::shared_ptr<TicketKeys> &ticket_keys)
-    : dconn_pool_(get_config()->downstream_addr_groups.size()),
+    : randgen_(rd()), dconn_pool_(get_config()->downstream_addr_groups.size()),
       worker_stat_(get_config()->downstream_addr_groups.size()),
       dgrps_(get_config()->downstream_addr_groups.size()), loop_(loop),
       sv_ssl_ctx_(sv_ssl_ctx), cl_ssl_ctx_(cl_ssl_ctx), cert_tree_(cert_tree),
@@ -268,6 +272,8 @@ MemcachedDispatcher *Worker::get_session_cache_memcached_dispatcher() {
   return session_cache_memcached_dispatcher_.get();
 }
 
+std::mt19937 &Worker::get_randgen() { return randgen_; }
+
 #ifdef HAVE_MRUBY
 int Worker::create_mruby_context() {
   auto mruby_file = get_config()->mruby_file.get();

src/shrpx_worker.h

@@ -29,6 +29,7 @@
 
 #include <mutex>
 #include <vector>
+#include <random>
 #include <thread>
 #ifndef NOTHREADS
 #include <future>
@@ -132,6 +133,8 @@ public:
 
   MemcachedDispatcher *get_session_cache_memcached_dispatcher();
 
+  std::mt19937 &get_randgen();
+
 #ifdef HAVE_MRUBY
   int create_mruby_context();
 
@@ -144,6 +147,7 @@ private:
 #endif // NOTHREADS
   std::mutex m_;
   std::vector<WorkerEvent> q_;
+  std::mt19937 randgen_;
   ev_async w_;
   ev_timer mcpool_clear_timer_;
   MemchunkPool mcpool_;

src/util.h

@@ -45,6 +45,7 @@
 #include <memory>
 #include <chrono>
 #include <map>
+#include <random>
 
 #include "http-parser/http_parser.h"
 
@@ -622,6 +623,18 @@ uint64_t get_uint64(const uint8_t *data);
 int read_mime_types(std::map<std::string, std::string> &res,
                     const char *filename);
 
+template <typename Generator>
+std::string random_alpha_digit(Generator &gen, size_t len) {
+  std::string res;
+  res.reserve(len);
+  std::uniform_int_distribution<> dis(0, 26 * 2 + 10 - 1);
+  for (; len > 0; --len) {
+    res += "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"[dis(
+        gen)];
+  }
+  return res;
+}
+
 } // namespace util
 
 } // namespace nghttp2