Merge pull request #1459 from nghttp2/proxyprotov2

nghttpx: Add PROXY protocol version 2

doc/sources/nghttpx-howto.rst

@@ -401,6 +401,9 @@ like so:
 
    frontend=*,443;proxyproto
 
+nghttpx supports both PROXY protocol v1 and v2.  AF_UNIX in PROXY
+protocol version 2 is ignored.
+
 Session affinity
 ----------------
 

integration-tests/nghttpx_http2_test.go

@@ -9,6 +9,7 @@ import (
 	"golang.org/x/net/http2/hpack"
 	"io"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"regexp"
 	"strings"
@@ -1506,6 +1507,235 @@ func TestH2H1ProxyProtocolV1InvalidID(t *testing.T) {
 	}
 }
 
+// TestH2H1ProxyProtocolV2TCP4 tests PROXY protocol version 2
+// containing AF_INET family is accepted and X-Forwarded-For contains
+// advertised src address.
+func TestH2H1ProxyProtocolV2TCP4(t *testing.T) {
+	st := newServerTester([]string{"--accept-proxy-protocol", "--add-x-forwarded-for", "--add-forwarded=for", "--forwarded-for=ip"}, t, func(w http.ResponseWriter, r *http.Request) {
+		if got, want := r.Header.Get("X-Forwarded-For"), "192.168.0.2"; got != want {
+			t.Errorf("X-Forwarded-For: %v; want %v", got, want)
+		}
+		if got, want := r.Header.Get("Forwarded"), "for=192.168.0.2"; got != want {
+			t.Errorf("Forwarded: %v; want %v", got, want)
+		}
+	})
+	defer st.Close()
+
+	var b bytes.Buffer
+	writeProxyProtocolV2(&b, proxyProtocolV2{
+		command: proxyProtocolV2CommandProxy,
+		sourceAddress: &net.TCPAddr{
+			IP:   net.ParseIP("192.168.0.2").To4(),
+			Port: 12345,
+		},
+		destinationAddress: &net.TCPAddr{
+			IP:   net.ParseIP("192.168.0.100").To4(),
+			Port: 8080,
+		},
+		additionalData: []byte("foobar"),
+	})
+	st.conn.Write(b.Bytes())
+
+	res, err := st.http2(requestParam{
+		name: "TestH2H1ProxyProtocolV2TCP4",
+	})
+
+	if err != nil {
+		t.Fatalf("Error st.http2() = %v", err)
+	}
+
+	if got, want := res.status, 200; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+}
+
+// TestH2H1ProxyProtocolV2TCP6 tests PROXY protocol version 2
+// containing AF_INET6 family is accepted and X-Forwarded-For contains
+// advertised src address.
+func TestH2H1ProxyProtocolV2TCP6(t *testing.T) {
+	st := newServerTester([]string{"--accept-proxy-protocol", "--add-x-forwarded-for", "--add-forwarded=for", "--forwarded-for=ip"}, t, func(w http.ResponseWriter, r *http.Request) {
+		if got, want := r.Header.Get("X-Forwarded-For"), "2001:db8:85a3::8a2e:370:7334"; got != want {
+			t.Errorf("X-Forwarded-For: %v; want %v", got, want)
+		}
+		if got, want := r.Header.Get("Forwarded"), `for="[2001:db8:85a3::8a2e:370:7334]"`; got != want {
+			t.Errorf("Forwarded: %v; want %v", got, want)
+		}
+	})
+	defer st.Close()
+
+	var b bytes.Buffer
+	writeProxyProtocolV2(&b, proxyProtocolV2{
+		command: proxyProtocolV2CommandProxy,
+		sourceAddress: &net.TCPAddr{
+			IP:   net.ParseIP("2001:0db8:85a3:0000:0000:8a2e:0370:7334"),
+			Port: 12345,
+		},
+		destinationAddress: &net.TCPAddr{
+			IP:   net.ParseIP("::1"),
+			Port: 8080,
+		},
+		additionalData: []byte("foobar"),
+	})
+	st.conn.Write(b.Bytes())
+
+	res, err := st.http2(requestParam{
+		name: "TestH2H1ProxyProtocolV2TCP6",
+	})
+
+	if err != nil {
+		t.Fatalf("Error st.http2() = %v", err)
+	}
+
+	if got, want := res.status, 200; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+}
+
+// TestH2H1ProxyProtocolV2Local tests PROXY protocol version 2
+// containing cmd == Local is ignored.
+func TestH2H1ProxyProtocolV2Local(t *testing.T) {
+	st := newServerTester([]string{"--accept-proxy-protocol", "--add-x-forwarded-for", "--add-forwarded=for", "--forwarded-for=ip"}, t, func(w http.ResponseWriter, r *http.Request) {
+		if got, want := r.Header.Get("X-Forwarded-For"), "127.0.0.1"; got != want {
+			t.Errorf("X-Forwarded-For: %v; want %v", got, want)
+		}
+		if got, want := r.Header.Get("Forwarded"), "for=127.0.0.1"; got != want {
+			t.Errorf("Forwarded: %v; want %v", got, want)
+		}
+	})
+	defer st.Close()
+
+	var b bytes.Buffer
+	writeProxyProtocolV2(&b, proxyProtocolV2{
+		command: proxyProtocolV2CommandLocal,
+		sourceAddress: &net.TCPAddr{
+			IP:   net.ParseIP("192.168.0.2").To4(),
+			Port: 12345,
+		},
+		destinationAddress: &net.TCPAddr{
+			IP:   net.ParseIP("192.168.0.100").To4(),
+			Port: 8080,
+		},
+		additionalData: []byte("foobar"),
+	})
+	st.conn.Write(b.Bytes())
+
+	res, err := st.http2(requestParam{
+		name: "TestH2H1ProxyProtocolV2Local",
+	})
+
+	if err != nil {
+		t.Fatalf("Error st.http2() = %v", err)
+	}
+
+	if got, want := res.status, 200; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+}
+
+// TestH2H1ProxyProtocolV2UnknownCmd tests PROXY protocol version 2
+// containing unknown cmd should be rejected.
+func TestH2H1ProxyProtocolV2UnknownCmd(t *testing.T) {
+	st := newServerTester([]string{"--accept-proxy-protocol"}, t, noopHandler)
+	defer st.Close()
+
+	var b bytes.Buffer
+	writeProxyProtocolV2(&b, proxyProtocolV2{
+		command: 0xf,
+		sourceAddress: &net.TCPAddr{
+			IP:   net.ParseIP("192.168.0.2").To4(),
+			Port: 12345,
+		},
+		destinationAddress: &net.TCPAddr{
+			IP:   net.ParseIP("192.168.0.100").To4(),
+			Port: 8080,
+		},
+		additionalData: []byte("foobar"),
+	})
+	st.conn.Write(b.Bytes())
+
+	_, err := st.http2(requestParam{
+		name: "TestH2H1ProxyProtocolV2UnknownCmd",
+	})
+
+	if err == nil {
+		t.Fatalf("connection was not terminated")
+	}
+}
+
+// TestH2H1ProxyProtocolV2Unix tests PROXY protocol version 2
+// containing AF_UNIX family is ignored.
+func TestH2H1ProxyProtocolV2Unix(t *testing.T) {
+	st := newServerTester([]string{"--accept-proxy-protocol", "--add-x-forwarded-for", "--add-forwarded=for", "--forwarded-for=ip"}, t, func(w http.ResponseWriter, r *http.Request) {
+		if got, want := r.Header.Get("X-Forwarded-For"), "127.0.0.1"; got != want {
+			t.Errorf("X-Forwarded-For: %v; want %v", got, want)
+		}
+		if got, want := r.Header.Get("Forwarded"), "for=127.0.0.1"; got != want {
+			t.Errorf("Forwarded: %v; want %v", got, want)
+		}
+	})
+	defer st.Close()
+
+	var b bytes.Buffer
+	writeProxyProtocolV2(&b, proxyProtocolV2{
+		command: proxyProtocolV2CommandProxy,
+		sourceAddress: &net.UnixAddr{
+			Name: "/foo",
+			Net:  "unix",
+		},
+		destinationAddress: &net.UnixAddr{
+			Name: "/bar",
+			Net:  "unix",
+		},
+		additionalData: []byte("foobar"),
+	})
+	st.conn.Write(b.Bytes())
+
+	res, err := st.http2(requestParam{
+		name: "TestH2H1ProxyProtocolV2Unix",
+	})
+
+	if err != nil {
+		t.Fatalf("Error st.http2() = %v", err)
+	}
+
+	if got, want := res.status, 200; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+}
+
+// TestH2H1ProxyProtocolV2Unspec tests PROXY protocol version 2
+// containing AF_UNSPEC family is ignored.
+func TestH2H1ProxyProtocolV2Unspec(t *testing.T) {
+	st := newServerTester([]string{"--accept-proxy-protocol", "--add-x-forwarded-for", "--add-forwarded=for", "--forwarded-for=ip"}, t, func(w http.ResponseWriter, r *http.Request) {
+		if got, want := r.Header.Get("X-Forwarded-For"), "127.0.0.1"; got != want {
+			t.Errorf("X-Forwarded-For: %v; want %v", got, want)
+		}
+		if got, want := r.Header.Get("Forwarded"), "for=127.0.0.1"; got != want {
+			t.Errorf("Forwarded: %v; want %v", got, want)
+		}
+	})
+	defer st.Close()
+
+	var b bytes.Buffer
+	writeProxyProtocolV2(&b, proxyProtocolV2{
+		command:        proxyProtocolV2CommandProxy,
+		additionalData: []byte("foobar"),
+	})
+	st.conn.Write(b.Bytes())
+
+	res, err := st.http2(requestParam{
+		name: "TestH2H1ProxyProtocolV2Unspec",
+	})
+
+	if err != nil {
+		t.Fatalf("Error st.http2() = %v", err)
+	}
+
+	if got, want := res.status, 200; got != want {
+		t.Errorf("res.status: %v; want %v", got, want)
+	}
+}
+
 // TestH2H1ExternalDNS tests that DNS resolution using external DNS
 // with HTTP/1 backend works.
 func TestH2H1ExternalDNS(t *testing.T) {

integration-tests/server_tester.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"crypto/tls"
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"github.com/tatsuhiro-t/go-nghttp2"
@@ -671,3 +672,93 @@ type APIResponse struct {
 	Code   int                    `json:"code,omitempty"`
 	Data   map[string]interface{} `json:"data,omitempty"`
 }
+
+type proxyProtocolV2 struct {
+	command            proxyProtocolV2Command
+	sourceAddress      net.Addr
+	destinationAddress net.Addr
+	additionalData     []byte
+}
+
+type proxyProtocolV2Command int
+
+const (
+	proxyProtocolV2CommandLocal proxyProtocolV2Command = 0x0
+	proxyProtocolV2CommandProxy proxyProtocolV2Command = 0x1
+)
+
+type proxyProtocolV2Family int
+
+const (
+	proxyProtocolV2FamilyUnspec proxyProtocolV2Family = 0x0
+	proxyProtocolV2FamilyInet   proxyProtocolV2Family = 0x1
+	proxyProtocolV2FamilyInet6  proxyProtocolV2Family = 0x2
+	proxyProtocolV2FamilyUnix   proxyProtocolV2Family = 0x3
+)
+
+type proxyProtocolV2Protocol int
+
+const (
+	proxyProtocolV2ProtocolUnspec proxyProtocolV2Protocol = 0x0
+	proxyProtocolV2ProtocolStream proxyProtocolV2Protocol = 0x1
+	proxyProtocolV2ProtocolDgram  proxyProtocolV2Protocol = 0x2
+)
+
+func writeProxyProtocolV2(w io.Writer, hdr proxyProtocolV2) {
+	w.Write([]byte{0x0D, 0x0A, 0x0D, 0x0A, 0x00, 0x0D, 0x0A, 0x51, 0x55, 0x49, 0x54, 0x0A})
+	w.Write([]byte{byte(0x20 | hdr.command)})
+
+	switch srcAddr := hdr.sourceAddress.(type) {
+	case *net.TCPAddr:
+		dstAddr := hdr.destinationAddress.(*net.TCPAddr)
+		if len(srcAddr.IP) != len(dstAddr.IP) {
+			panic("len(srcAddr.IP) != len(dstAddr.IP)")
+		}
+		var fam byte
+		if len(srcAddr.IP) == 4 {
+			fam = byte(proxyProtocolV2FamilyInet << 4)
+		} else {
+			fam = byte(proxyProtocolV2FamilyInet6 << 4)
+		}
+		fam |= byte(proxyProtocolV2ProtocolStream)
+		w.Write([]byte{fam})
+		length := uint16(len(srcAddr.IP)*2 + 4 + len(hdr.additionalData))
+		binary.Write(w, binary.BigEndian, length)
+		w.Write(srcAddr.IP)
+		w.Write(dstAddr.IP)
+		binary.Write(w, binary.BigEndian, uint16(srcAddr.Port))
+		binary.Write(w, binary.BigEndian, uint16(dstAddr.Port))
+	case *net.UnixAddr:
+		dstAddr := hdr.destinationAddress.(*net.UnixAddr)
+		if len(srcAddr.Name) > 108 {
+			panic("too long Unix source address")
+		}
+		if len(dstAddr.Name) > 108 {
+			panic("too long Unix destination address")
+		}
+		fam := byte(proxyProtocolV2FamilyUnix << 4)
+		switch srcAddr.Net {
+		case "unix":
+			fam |= byte(proxyProtocolV2ProtocolStream)
+		case "unixdgram":
+			fam |= byte(proxyProtocolV2ProtocolDgram)
+		default:
+			fam |= byte(proxyProtocolV2ProtocolUnspec)
+		}
+		w.Write([]byte{fam})
+		length := uint16(216 + len(hdr.additionalData))
+		binary.Write(w, binary.BigEndian, length)
+		zeros := make([]byte, 108)
+		w.Write([]byte(srcAddr.Name))
+		w.Write(zeros[:108-len(srcAddr.Name)])
+		w.Write([]byte(dstAddr.Name))
+		w.Write(zeros[:108-len(dstAddr.Name)])
+	default:
+		fam := byte(proxyProtocolV2FamilyUnspec<<4) | byte(proxyProtocolV2ProtocolUnspec)
+		w.Write([]byte{fam})
+		length := uint16(len(hdr.additionalData))
+		binary.Write(w, binary.BigEndian, length)
+	}
+
+	w.Write(hdr.additionalData)
+}

src/shrpx.cc

@@ -1914,7 +1914,7 @@ Connections:
               default.  Any  requests which come through  this address
               are replied with 200 HTTP status, without no body.
 
-              To  accept   PROXY  protocol   version  1   on  frontend
+              To accept  PROXY protocol  version 1  and 2  on frontend
               connection,  specify  "proxyproto" parameter.   This  is
               disabled by default.
 

src/shrpx_client_handler.cc

@@ -447,8 +447,7 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl,
       *p = '\0';
 
       forwarded_for_ = StringRef{buf.base, p};
-    } else if (!faddr_->accept_proxy_protocol &&
+    } else {
-               !config->conn.upstream.accept_proxy_protocol) {
       init_forwarded_for(family, ipaddr_);
     }
   }
@@ -1149,6 +1148,16 @@ int ClientHandler::on_proxy_protocol_finish() {
   return 0;
 }
 
+namespace {
+// PROXY-protocol v2 header signature
+constexpr uint8_t PROXY_PROTO_V2_SIG[] =
+    "\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A";
+
+// PROXY-protocol v2 header length
+constexpr size_t PROXY_PROTO_V2_HDLEN =
+    str_size(PROXY_PROTO_V2_SIG) + /* ver_cmd(1) + fam(1) + len(2) = */ 4;
+} // namespace
+
 // http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt
 int ClientHandler::proxy_protocol_read() {
   if (LOG_ENABLED(INFO)) {
@@ -1157,6 +1166,14 @@ int ClientHandler::proxy_protocol_read() {
 
   auto first = rb_.pos();
 
+  if (rb_.rleft() >= PROXY_PROTO_V2_HDLEN &&
+      (*(first + str_size(PROXY_PROTO_V2_SIG)) & 0xf0) == 0x20) {
+    if (LOG_ENABLED(INFO)) {
+      CLOG(INFO, this) << "PROXY-protocol: Detected v2 header signature";
+    }
+    return proxy_protocol_v2_read();
+  }
+
   // NULL character really destroys functions which expects NULL
   // terminated string.  We won't expect it in PROXY protocol line, so
   // find it here.
@@ -1338,6 +1355,167 @@ int ClientHandler::proxy_protocol_read() {
   return on_proxy_protocol_finish();
 }
 
+int ClientHandler::proxy_protocol_v2_read() {
+  // Assume that first str_size(PROXY_PROTO_V2_SIG) octets match v2
+  // protocol signature and followed by the bytes which indicates v2.
+  assert(rb_.rleft() >= PROXY_PROTO_V2_HDLEN);
+
+  auto p = rb_.pos() + str_size(PROXY_PROTO_V2_SIG);
+
+  assert(((*p) & 0xf0) == 0x20);
+
+  enum { LOCAL, PROXY } cmd;
+
+  auto cmd_bits = (*p++) & 0xf;
+  switch (cmd_bits) {
+  case 0x0:
+    cmd = LOCAL;
+    break;
+  case 0x01:
+    cmd = PROXY;
+    break;
+  default:
+    if (LOG_ENABLED(INFO)) {
+      CLOG(INFO, this) << "PROXY-protocol-v2: Unknown command " << log::hex
+                       << cmd_bits;
+    }
+    return -1;
+  }
+
+  auto fam = *p++;
+  uint16_t len;
+  memcpy(&len, p, sizeof(len));
+  len = ntohs(len);
+
+  p += sizeof(len);
+
+  if (LOG_ENABLED(INFO)) {
+    CLOG(INFO, this) << "PROXY-protocol-v2: Detected family=" << log::hex << fam
+                     << ", len=" << log::dec << len;
+  }
+
+  if (rb_.last() - p < len) {
+    if (LOG_ENABLED(INFO)) {
+      CLOG(INFO, this)
+          << "PROXY-protocol-v2: Prematurely truncated header block; require "
+          << len << " bytes, " << rb_.last() - p << " bytes left";
+    }
+    return -1;
+  }
+
+  int family;
+  std::array<char, std::max(INET_ADDRSTRLEN, INET6_ADDRSTRLEN)> src_addr,
+      dst_addr;
+  size_t addrlen;
+
+  switch (fam) {
+  case 0x11:
+  case 0x12:
+    if (len < 12) {
+      if (LOG_ENABLED(INFO)) {
+        CLOG(INFO, this) << "PROXY-protocol-v2: Too short AF_INET addresses";
+      }
+      return -1;
+    }
+    family = AF_INET;
+    addrlen = 4;
+    break;
+  case 0x21:
+  case 0x22:
+    if (len < 36) {
+      if (LOG_ENABLED(INFO)) {
+        CLOG(INFO, this) << "PROXY-protocol-v2: Too short AF_INET6 addresses";
+      }
+      return -1;
+    }
+    family = AF_INET6;
+    addrlen = 16;
+    break;
+  case 0x31:
+  case 0x32:
+    if (len < 216) {
+      if (LOG_ENABLED(INFO)) {
+        CLOG(INFO, this) << "PROXY-protocol-v2: Too short AF_UNIX addresses";
+      }
+      return -1;
+    }
+    // fall through
+  case 0x00: {
+    // UNSPEC and UNIX are just ignored.
+    if (LOG_ENABLED(INFO)) {
+      CLOG(INFO, this) << "PROXY-protocol-v2: Ignore combination of address "
+                          "family and protocol "
+                       << log::hex << fam;
+    }
+    rb_.drain(PROXY_PROTO_V2_HDLEN + len);
+    return on_proxy_protocol_finish();
+  }
+  default:
+    if (LOG_ENABLED(INFO)) {
+      CLOG(INFO, this) << "PROXY-protocol-v2: Unknown combination of address "
+                          "family and protocol "
+                       << log::hex << fam;
+    }
+    return -1;
+  }
+
+  if (cmd != PROXY) {
+    if (LOG_ENABLED(INFO)) {
+      CLOG(INFO, this) << "PROXY-protocol-v2: Ignore non-PROXY command";
+    }
+    rb_.drain(PROXY_PROTO_V2_HDLEN + len);
+    return on_proxy_protocol_finish();
+  }
+
+  if (inet_ntop(family, p, src_addr.data(), src_addr.size()) == nullptr) {
+    if (LOG_ENABLED(INFO)) {
+      CLOG(INFO, this) << "PROXY-protocol-v2: Unable to parse source address";
+    }
+    return -1;
+  }
+
+  p += addrlen;
+
+  if (inet_ntop(family, p, dst_addr.data(), dst_addr.size()) == nullptr) {
+    if (LOG_ENABLED(INFO)) {
+      CLOG(INFO, this)
+          << "PROXY-protocol-v2: Unable to parse destination address";
+    }
+    return -1;
+  }
+
+  p += addrlen;
+
+  uint16_t src_port;
+
+  memcpy(&src_port, p, sizeof(src_port));
+  src_port = ntohs(src_port);
+
+  // We don't use destination port.
+  p += 4;
+
+  ipaddr_ = make_string_ref(balloc_, StringRef{src_addr.data()});
+  port_ = util::make_string_ref_uint(balloc_, src_port);
+
+  if (LOG_ENABLED(INFO)) {
+    CLOG(INFO, this) << "PROXY-protocol-v2: Finished reading proxy addresses, "
+                     << p - rb_.pos() << " bytes read, "
+                     << PROXY_PROTO_V2_HDLEN + len - (p - rb_.pos())
+                     << " bytes left";
+  }
+
+  auto config = get_config();
+  auto &fwdconf = config->http.forwarded;
+
+  if ((fwdconf.params & FORWARDED_FOR) &&
+      fwdconf.for_node_type == ForwardedNode::IP) {
+    init_forwarded_for(family, ipaddr_);
+  }
+
+  rb_.drain(PROXY_PROTO_V2_HDLEN + len);
+  return on_proxy_protocol_finish();
+}
+
 StringRef ClientHandler::get_forwarded_by() const {
   auto &fwdconf = get_config()->http.forwarded;
 

src/shrpx_client_handler.h

@@ -77,6 +77,7 @@ public:
   int upstream_write();
 
   int proxy_protocol_read();
+  int proxy_protocol_v2_read();
   int on_proxy_protocol_finish();
 
   // Performs I/O operation.  Internally calls on_read()/on_write().