diff --git a/doc/sources/nghttpx-howto.rst b/doc/sources/nghttpx-howto.rst index 11d3eb0b..bbb7c20d 100644 --- a/doc/sources/nghttpx-howto.rst +++ b/doc/sources/nghttpx-howto.rst @@ -401,6 +401,9 @@ like so: frontend=*,443;proxyproto +nghttpx supports both PROXY protocol v1 and v2. AF_UNIX in PROXY +protocol version 2 is ignored. + Session affinity ---------------- diff --git a/integration-tests/nghttpx_http2_test.go b/integration-tests/nghttpx_http2_test.go index 0698dd24..84646fa0 100644 --- a/integration-tests/nghttpx_http2_test.go +++ b/integration-tests/nghttpx_http2_test.go @@ -9,6 +9,7 @@ import ( "golang.org/x/net/http2/hpack" "io" "io/ioutil" + "net" "net/http" "regexp" "strings" @@ -1506,6 +1507,235 @@ func TestH2H1ProxyProtocolV1InvalidID(t *testing.T) { } } +// TestH2H1ProxyProtocolV2TCP4 tests PROXY protocol version 2 +// containing AF_INET family is accepted and X-Forwarded-For contains +// advertised src address. +func TestH2H1ProxyProtocolV2TCP4(t *testing.T) { + st := newServerTester([]string{"--accept-proxy-protocol", "--add-x-forwarded-for", "--add-forwarded=for", "--forwarded-for=ip"}, t, func(w http.ResponseWriter, r *http.Request) { + if got, want := r.Header.Get("X-Forwarded-For"), "192.168.0.2"; got != want { + t.Errorf("X-Forwarded-For: %v; want %v", got, want) + } + if got, want := r.Header.Get("Forwarded"), "for=192.168.0.2"; got != want { + t.Errorf("Forwarded: %v; want %v", got, want) + } + }) + defer st.Close() + + var b bytes.Buffer + writeProxyProtocolV2(&b, proxyProtocolV2{ + command: proxyProtocolV2CommandProxy, + sourceAddress: &net.TCPAddr{ + IP: net.ParseIP("192.168.0.2").To4(), + Port: 12345, + }, + destinationAddress: &net.TCPAddr{ + IP: net.ParseIP("192.168.0.100").To4(), + Port: 8080, + }, + additionalData: []byte("foobar"), + }) + st.conn.Write(b.Bytes()) + + res, err := st.http2(requestParam{ + name: "TestH2H1ProxyProtocolV2TCP4", + }) + + if err != nil { + t.Fatalf("Error st.http2() = %v", err) + } + + if got, want := res.status, 200; got != want { + t.Errorf("res.status: %v; want %v", got, want) + } +} + +// TestH2H1ProxyProtocolV2TCP6 tests PROXY protocol version 2 +// containing AF_INET6 family is accepted and X-Forwarded-For contains +// advertised src address. +func TestH2H1ProxyProtocolV2TCP6(t *testing.T) { + st := newServerTester([]string{"--accept-proxy-protocol", "--add-x-forwarded-for", "--add-forwarded=for", "--forwarded-for=ip"}, t, func(w http.ResponseWriter, r *http.Request) { + if got, want := r.Header.Get("X-Forwarded-For"), "2001:db8:85a3::8a2e:370:7334"; got != want { + t.Errorf("X-Forwarded-For: %v; want %v", got, want) + } + if got, want := r.Header.Get("Forwarded"), `for="[2001:db8:85a3::8a2e:370:7334]"`; got != want { + t.Errorf("Forwarded: %v; want %v", got, want) + } + }) + defer st.Close() + + var b bytes.Buffer + writeProxyProtocolV2(&b, proxyProtocolV2{ + command: proxyProtocolV2CommandProxy, + sourceAddress: &net.TCPAddr{ + IP: net.ParseIP("2001:0db8:85a3:0000:0000:8a2e:0370:7334"), + Port: 12345, + }, + destinationAddress: &net.TCPAddr{ + IP: net.ParseIP("::1"), + Port: 8080, + }, + additionalData: []byte("foobar"), + }) + st.conn.Write(b.Bytes()) + + res, err := st.http2(requestParam{ + name: "TestH2H1ProxyProtocolV2TCP6", + }) + + if err != nil { + t.Fatalf("Error st.http2() = %v", err) + } + + if got, want := res.status, 200; got != want { + t.Errorf("res.status: %v; want %v", got, want) + } +} + +// TestH2H1ProxyProtocolV2Local tests PROXY protocol version 2 +// containing cmd == Local is ignored. +func TestH2H1ProxyProtocolV2Local(t *testing.T) { + st := newServerTester([]string{"--accept-proxy-protocol", "--add-x-forwarded-for", "--add-forwarded=for", "--forwarded-for=ip"}, t, func(w http.ResponseWriter, r *http.Request) { + if got, want := r.Header.Get("X-Forwarded-For"), "127.0.0.1"; got != want { + t.Errorf("X-Forwarded-For: %v; want %v", got, want) + } + if got, want := r.Header.Get("Forwarded"), "for=127.0.0.1"; got != want { + t.Errorf("Forwarded: %v; want %v", got, want) + } + }) + defer st.Close() + + var b bytes.Buffer + writeProxyProtocolV2(&b, proxyProtocolV2{ + command: proxyProtocolV2CommandLocal, + sourceAddress: &net.TCPAddr{ + IP: net.ParseIP("192.168.0.2").To4(), + Port: 12345, + }, + destinationAddress: &net.TCPAddr{ + IP: net.ParseIP("192.168.0.100").To4(), + Port: 8080, + }, + additionalData: []byte("foobar"), + }) + st.conn.Write(b.Bytes()) + + res, err := st.http2(requestParam{ + name: "TestH2H1ProxyProtocolV2Local", + }) + + if err != nil { + t.Fatalf("Error st.http2() = %v", err) + } + + if got, want := res.status, 200; got != want { + t.Errorf("res.status: %v; want %v", got, want) + } +} + +// TestH2H1ProxyProtocolV2UnknownCmd tests PROXY protocol version 2 +// containing unknown cmd should be rejected. +func TestH2H1ProxyProtocolV2UnknownCmd(t *testing.T) { + st := newServerTester([]string{"--accept-proxy-protocol"}, t, noopHandler) + defer st.Close() + + var b bytes.Buffer + writeProxyProtocolV2(&b, proxyProtocolV2{ + command: 0xf, + sourceAddress: &net.TCPAddr{ + IP: net.ParseIP("192.168.0.2").To4(), + Port: 12345, + }, + destinationAddress: &net.TCPAddr{ + IP: net.ParseIP("192.168.0.100").To4(), + Port: 8080, + }, + additionalData: []byte("foobar"), + }) + st.conn.Write(b.Bytes()) + + _, err := st.http2(requestParam{ + name: "TestH2H1ProxyProtocolV2UnknownCmd", + }) + + if err == nil { + t.Fatalf("connection was not terminated") + } +} + +// TestH2H1ProxyProtocolV2Unix tests PROXY protocol version 2 +// containing AF_UNIX family is ignored. +func TestH2H1ProxyProtocolV2Unix(t *testing.T) { + st := newServerTester([]string{"--accept-proxy-protocol", "--add-x-forwarded-for", "--add-forwarded=for", "--forwarded-for=ip"}, t, func(w http.ResponseWriter, r *http.Request) { + if got, want := r.Header.Get("X-Forwarded-For"), "127.0.0.1"; got != want { + t.Errorf("X-Forwarded-For: %v; want %v", got, want) + } + if got, want := r.Header.Get("Forwarded"), "for=127.0.0.1"; got != want { + t.Errorf("Forwarded: %v; want %v", got, want) + } + }) + defer st.Close() + + var b bytes.Buffer + writeProxyProtocolV2(&b, proxyProtocolV2{ + command: proxyProtocolV2CommandProxy, + sourceAddress: &net.UnixAddr{ + Name: "/foo", + Net: "unix", + }, + destinationAddress: &net.UnixAddr{ + Name: "/bar", + Net: "unix", + }, + additionalData: []byte("foobar"), + }) + st.conn.Write(b.Bytes()) + + res, err := st.http2(requestParam{ + name: "TestH2H1ProxyProtocolV2Unix", + }) + + if err != nil { + t.Fatalf("Error st.http2() = %v", err) + } + + if got, want := res.status, 200; got != want { + t.Errorf("res.status: %v; want %v", got, want) + } +} + +// TestH2H1ProxyProtocolV2Unspec tests PROXY protocol version 2 +// containing AF_UNSPEC family is ignored. +func TestH2H1ProxyProtocolV2Unspec(t *testing.T) { + st := newServerTester([]string{"--accept-proxy-protocol", "--add-x-forwarded-for", "--add-forwarded=for", "--forwarded-for=ip"}, t, func(w http.ResponseWriter, r *http.Request) { + if got, want := r.Header.Get("X-Forwarded-For"), "127.0.0.1"; got != want { + t.Errorf("X-Forwarded-For: %v; want %v", got, want) + } + if got, want := r.Header.Get("Forwarded"), "for=127.0.0.1"; got != want { + t.Errorf("Forwarded: %v; want %v", got, want) + } + }) + defer st.Close() + + var b bytes.Buffer + writeProxyProtocolV2(&b, proxyProtocolV2{ + command: proxyProtocolV2CommandProxy, + additionalData: []byte("foobar"), + }) + st.conn.Write(b.Bytes()) + + res, err := st.http2(requestParam{ + name: "TestH2H1ProxyProtocolV2Unspec", + }) + + if err != nil { + t.Fatalf("Error st.http2() = %v", err) + } + + if got, want := res.status, 200; got != want { + t.Errorf("res.status: %v; want %v", got, want) + } +} + // TestH2H1ExternalDNS tests that DNS resolution using external DNS // with HTTP/1 backend works. func TestH2H1ExternalDNS(t *testing.T) { diff --git a/integration-tests/server_tester.go b/integration-tests/server_tester.go index 89e7f29d..113e27d1 100644 --- a/integration-tests/server_tester.go +++ b/integration-tests/server_tester.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "crypto/tls" + "encoding/binary" "errors" "fmt" "github.com/tatsuhiro-t/go-nghttp2" @@ -671,3 +672,93 @@ type APIResponse struct { Code int `json:"code,omitempty"` Data map[string]interface{} `json:"data,omitempty"` } + +type proxyProtocolV2 struct { + command proxyProtocolV2Command + sourceAddress net.Addr + destinationAddress net.Addr + additionalData []byte +} + +type proxyProtocolV2Command int + +const ( + proxyProtocolV2CommandLocal proxyProtocolV2Command = 0x0 + proxyProtocolV2CommandProxy proxyProtocolV2Command = 0x1 +) + +type proxyProtocolV2Family int + +const ( + proxyProtocolV2FamilyUnspec proxyProtocolV2Family = 0x0 + proxyProtocolV2FamilyInet proxyProtocolV2Family = 0x1 + proxyProtocolV2FamilyInet6 proxyProtocolV2Family = 0x2 + proxyProtocolV2FamilyUnix proxyProtocolV2Family = 0x3 +) + +type proxyProtocolV2Protocol int + +const ( + proxyProtocolV2ProtocolUnspec proxyProtocolV2Protocol = 0x0 + proxyProtocolV2ProtocolStream proxyProtocolV2Protocol = 0x1 + proxyProtocolV2ProtocolDgram proxyProtocolV2Protocol = 0x2 +) + +func writeProxyProtocolV2(w io.Writer, hdr proxyProtocolV2) { + w.Write([]byte{0x0D, 0x0A, 0x0D, 0x0A, 0x00, 0x0D, 0x0A, 0x51, 0x55, 0x49, 0x54, 0x0A}) + w.Write([]byte{byte(0x20 | hdr.command)}) + + switch srcAddr := hdr.sourceAddress.(type) { + case *net.TCPAddr: + dstAddr := hdr.destinationAddress.(*net.TCPAddr) + if len(srcAddr.IP) != len(dstAddr.IP) { + panic("len(srcAddr.IP) != len(dstAddr.IP)") + } + var fam byte + if len(srcAddr.IP) == 4 { + fam = byte(proxyProtocolV2FamilyInet << 4) + } else { + fam = byte(proxyProtocolV2FamilyInet6 << 4) + } + fam |= byte(proxyProtocolV2ProtocolStream) + w.Write([]byte{fam}) + length := uint16(len(srcAddr.IP)*2 + 4 + len(hdr.additionalData)) + binary.Write(w, binary.BigEndian, length) + w.Write(srcAddr.IP) + w.Write(dstAddr.IP) + binary.Write(w, binary.BigEndian, uint16(srcAddr.Port)) + binary.Write(w, binary.BigEndian, uint16(dstAddr.Port)) + case *net.UnixAddr: + dstAddr := hdr.destinationAddress.(*net.UnixAddr) + if len(srcAddr.Name) > 108 { + panic("too long Unix source address") + } + if len(dstAddr.Name) > 108 { + panic("too long Unix destination address") + } + fam := byte(proxyProtocolV2FamilyUnix << 4) + switch srcAddr.Net { + case "unix": + fam |= byte(proxyProtocolV2ProtocolStream) + case "unixdgram": + fam |= byte(proxyProtocolV2ProtocolDgram) + default: + fam |= byte(proxyProtocolV2ProtocolUnspec) + } + w.Write([]byte{fam}) + length := uint16(216 + len(hdr.additionalData)) + binary.Write(w, binary.BigEndian, length) + zeros := make([]byte, 108) + w.Write([]byte(srcAddr.Name)) + w.Write(zeros[:108-len(srcAddr.Name)]) + w.Write([]byte(dstAddr.Name)) + w.Write(zeros[:108-len(dstAddr.Name)]) + default: + fam := byte(proxyProtocolV2FamilyUnspec<<4) | byte(proxyProtocolV2ProtocolUnspec) + w.Write([]byte{fam}) + length := uint16(len(hdr.additionalData)) + binary.Write(w, binary.BigEndian, length) + } + + w.Write(hdr.additionalData) +} diff --git a/src/shrpx.cc b/src/shrpx.cc index 2dd0493f..4ff5e478 100644 --- a/src/shrpx.cc +++ b/src/shrpx.cc @@ -1914,7 +1914,7 @@ Connections: default. Any requests which come through this address are replied with 200 HTTP status, without no body. - To accept PROXY protocol version 1 on frontend + To accept PROXY protocol version 1 and 2 on frontend connection, specify "proxyproto" parameter. This is disabled by default. diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc index e1180085..2b6638a6 100644 --- a/src/shrpx_client_handler.cc +++ b/src/shrpx_client_handler.cc @@ -447,8 +447,7 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl, *p = '\0'; forwarded_for_ = StringRef{buf.base, p}; - } else if (!faddr_->accept_proxy_protocol && - !config->conn.upstream.accept_proxy_protocol) { + } else { init_forwarded_for(family, ipaddr_); } } @@ -1149,6 +1148,16 @@ int ClientHandler::on_proxy_protocol_finish() { return 0; } +namespace { +// PROXY-protocol v2 header signature +constexpr uint8_t PROXY_PROTO_V2_SIG[] = + "\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A"; + +// PROXY-protocol v2 header length +constexpr size_t PROXY_PROTO_V2_HDLEN = + str_size(PROXY_PROTO_V2_SIG) + /* ver_cmd(1) + fam(1) + len(2) = */ 4; +} // namespace + // http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt int ClientHandler::proxy_protocol_read() { if (LOG_ENABLED(INFO)) { @@ -1157,6 +1166,14 @@ int ClientHandler::proxy_protocol_read() { auto first = rb_.pos(); + if (rb_.rleft() >= PROXY_PROTO_V2_HDLEN && + (*(first + str_size(PROXY_PROTO_V2_SIG)) & 0xf0) == 0x20) { + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) << "PROXY-protocol: Detected v2 header signature"; + } + return proxy_protocol_v2_read(); + } + // NULL character really destroys functions which expects NULL // terminated string. We won't expect it in PROXY protocol line, so // find it here. @@ -1338,6 +1355,167 @@ int ClientHandler::proxy_protocol_read() { return on_proxy_protocol_finish(); } +int ClientHandler::proxy_protocol_v2_read() { + // Assume that first str_size(PROXY_PROTO_V2_SIG) octets match v2 + // protocol signature and followed by the bytes which indicates v2. + assert(rb_.rleft() >= PROXY_PROTO_V2_HDLEN); + + auto p = rb_.pos() + str_size(PROXY_PROTO_V2_SIG); + + assert(((*p) & 0xf0) == 0x20); + + enum { LOCAL, PROXY } cmd; + + auto cmd_bits = (*p++) & 0xf; + switch (cmd_bits) { + case 0x0: + cmd = LOCAL; + break; + case 0x01: + cmd = PROXY; + break; + default: + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) << "PROXY-protocol-v2: Unknown command " << log::hex + << cmd_bits; + } + return -1; + } + + auto fam = *p++; + uint16_t len; + memcpy(&len, p, sizeof(len)); + len = ntohs(len); + + p += sizeof(len); + + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) << "PROXY-protocol-v2: Detected family=" << log::hex << fam + << ", len=" << log::dec << len; + } + + if (rb_.last() - p < len) { + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) + << "PROXY-protocol-v2: Prematurely truncated header block; require " + << len << " bytes, " << rb_.last() - p << " bytes left"; + } + return -1; + } + + int family; + std::array src_addr, + dst_addr; + size_t addrlen; + + switch (fam) { + case 0x11: + case 0x12: + if (len < 12) { + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) << "PROXY-protocol-v2: Too short AF_INET addresses"; + } + return -1; + } + family = AF_INET; + addrlen = 4; + break; + case 0x21: + case 0x22: + if (len < 36) { + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) << "PROXY-protocol-v2: Too short AF_INET6 addresses"; + } + return -1; + } + family = AF_INET6; + addrlen = 16; + break; + case 0x31: + case 0x32: + if (len < 216) { + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) << "PROXY-protocol-v2: Too short AF_UNIX addresses"; + } + return -1; + } + // fall through + case 0x00: { + // UNSPEC and UNIX are just ignored. + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) << "PROXY-protocol-v2: Ignore combination of address " + "family and protocol " + << log::hex << fam; + } + rb_.drain(PROXY_PROTO_V2_HDLEN + len); + return on_proxy_protocol_finish(); + } + default: + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) << "PROXY-protocol-v2: Unknown combination of address " + "family and protocol " + << log::hex << fam; + } + return -1; + } + + if (cmd != PROXY) { + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) << "PROXY-protocol-v2: Ignore non-PROXY command"; + } + rb_.drain(PROXY_PROTO_V2_HDLEN + len); + return on_proxy_protocol_finish(); + } + + if (inet_ntop(family, p, src_addr.data(), src_addr.size()) == nullptr) { + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) << "PROXY-protocol-v2: Unable to parse source address"; + } + return -1; + } + + p += addrlen; + + if (inet_ntop(family, p, dst_addr.data(), dst_addr.size()) == nullptr) { + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) + << "PROXY-protocol-v2: Unable to parse destination address"; + } + return -1; + } + + p += addrlen; + + uint16_t src_port; + + memcpy(&src_port, p, sizeof(src_port)); + src_port = ntohs(src_port); + + // We don't use destination port. + p += 4; + + ipaddr_ = make_string_ref(balloc_, StringRef{src_addr.data()}); + port_ = util::make_string_ref_uint(balloc_, src_port); + + if (LOG_ENABLED(INFO)) { + CLOG(INFO, this) << "PROXY-protocol-v2: Finished reading proxy addresses, " + << p - rb_.pos() << " bytes read, " + << PROXY_PROTO_V2_HDLEN + len - (p - rb_.pos()) + << " bytes left"; + } + + auto config = get_config(); + auto &fwdconf = config->http.forwarded; + + if ((fwdconf.params & FORWARDED_FOR) && + fwdconf.for_node_type == ForwardedNode::IP) { + init_forwarded_for(family, ipaddr_); + } + + rb_.drain(PROXY_PROTO_V2_HDLEN + len); + return on_proxy_protocol_finish(); +} + StringRef ClientHandler::get_forwarded_by() const { auto &fwdconf = get_config()->http.forwarded; diff --git a/src/shrpx_client_handler.h b/src/shrpx_client_handler.h index c31b1ee4..bc56d482 100644 --- a/src/shrpx_client_handler.h +++ b/src/shrpx_client_handler.h @@ -77,6 +77,7 @@ public: int upstream_write(); int proxy_protocol_read(); + int proxy_protocol_v2_read(); int on_proxy_protocol_finish(); // Performs I/O operation. Internally calls on_read()/on_write().