Merge branch 'rewrite-host'

This commit is contained in:
Tatsuhiro Tsujikawa 2015-02-04 01:44:40 +09:00
commit 83200f3080
13 changed files with 344 additions and 48 deletions

View File

@ -198,8 +198,8 @@ backend connection, use ``--backend-no-tls`` option.
The backend server is supporsed to be a HTTP/2 web server or HTTP/2
proxy. If backend server is HTTP/2 proxy, use
``--no-location-rewrite`` option to disable rewriting location header
field.
``--no-location-rewrite`` and ``--no-host-rewrite`` options to disable
rewriting location, host and :authority header field.
The use-case of this mode is aggregate the incoming connections to one
HTTP/2 connection. One backend HTTP/2 connection is created per

View File

@ -139,6 +139,78 @@ func TestH1H1GracefulShutdown(t *testing.T) {
}
}
// TestH1H1HostRewrite tests that server rewrites Host header field
func TestH1H1HostRewrite(t *testing.T) {
st := newServerTester(nil, t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("request-host", r.Host)
})
defer st.Close()
res, err := st.http1(requestParam{
name: "TestH1H1HostRewrite",
})
if err != nil {
t.Fatalf("Error st.http1() = %v", err)
}
if got, want := res.status, 200; got != want {
t.Errorf("status: %v; want %v", got, want)
}
if got, want := res.header.Get("request-host"), st.backendHost; got != want {
t.Errorf("request-host: %v; want %v", got, want)
}
}
// TestH1H1HTTP10 tests that server can accept HTTP/1.0 request
// without Host header field
func TestH1H1HTTP10(t *testing.T) {
st := newServerTester(nil, t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("request-host", r.Host)
})
defer st.Close()
if _, err := io.WriteString(st.conn, "GET / HTTP/1.0\r\nTest-Case: TestH1H1HTTP10\r\n\r\n"); err != nil {
t.Fatalf("Error io.WriteString() = %v", err)
}
resp, err := http.ReadResponse(bufio.NewReader(st.conn), nil)
if err != nil {
t.Fatalf("Error http.ReadResponse() = %v", err)
}
if got, want := resp.StatusCode, 200; got != want {
t.Errorf("status: %v; want %v", got, want)
}
if got, want := resp.Header.Get("request-host"), st.backendHost; got != want {
t.Errorf("request-host: %v; want %v", got, want)
}
}
// TestH1H1HTTP10NoHostRewrite tests that server generates host header
// field using actual backend server even if --no-http-rewrite is
// used.
func TestH1H1HTTP10NoHostRewrite(t *testing.T) {
st := newServerTester([]string{"--no-host-rewrite"}, t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("request-host", r.Host)
})
defer st.Close()
if _, err := io.WriteString(st.conn, "GET / HTTP/1.0\r\nTest-Case: TestH1H1HTTP10NoHostRewrite\r\n\r\n"); err != nil {
t.Fatalf("Error io.WriteString() = %v", err)
}
resp, err := http.ReadResponse(bufio.NewReader(st.conn), nil)
if err != nil {
t.Fatalf("Error http.ReadResponse() = %v", err)
}
if got, want := resp.StatusCode, 200; got != want {
t.Errorf("status: %v; want %v", got, want)
}
if got, want := resp.Header.Get("request-host"), st.backendHost; got != want {
t.Errorf("request-host: %v; want %v", got, want)
}
}
// TestH1H2ConnectFailure tests that server handles the situation that
// connection attempt to HTTP/2 backend failed.
func TestH1H2ConnectFailure(t *testing.T) {
@ -184,6 +256,57 @@ func TestH1H2NoHost(t *testing.T) {
}
}
// TestH1H2HTTP10 tests that server can accept HTTP/1.0 request
// without Host header field
func TestH1H2HTTP10(t *testing.T) {
st := newServerTester([]string{"--http2-bridge"}, t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("request-host", r.Host)
})
defer st.Close()
if _, err := io.WriteString(st.conn, "GET / HTTP/1.0\r\nTest-Case: TestH1H2HTTP10\r\n\r\n"); err != nil {
t.Fatalf("Error io.WriteString() = %v", err)
}
resp, err := http.ReadResponse(bufio.NewReader(st.conn), nil)
if err != nil {
t.Fatalf("Error http.ReadResponse() = %v", err)
}
if got, want := resp.StatusCode, 200; got != want {
t.Errorf("status: %v; want %v", got, want)
}
if got, want := resp.Header.Get("request-host"), st.backendHost; got != want {
t.Errorf("request-host: %v; want %v", got, want)
}
}
// TestH1H2HTTP10NoHostRewrite tests that server generates host header
// field using actual backend server even if --no-http-rewrite is
// used.
func TestH1H2HTTP10NoHostRewrite(t *testing.T) {
st := newServerTester([]string{"--http2-bridge", "--no-host-rewrite"}, t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("request-host", r.Host)
})
defer st.Close()
if _, err := io.WriteString(st.conn, "GET / HTTP/1.0\r\nTest-Case: TestH1H2HTTP10NoHostRewrite\r\n\r\n"); err != nil {
t.Fatalf("Error io.WriteString() = %v", err)
}
resp, err := http.ReadResponse(bufio.NewReader(st.conn), nil)
if err != nil {
t.Fatalf("Error http.ReadResponse() = %v", err)
}
if got, want := resp.StatusCode, 200; got != want {
t.Errorf("status: %v; want %v", got, want)
}
if got, want := resp.Header.Get("request-host"), st.backendHost; got != want {
t.Errorf("request-host: %v; want %v", got, want)
}
}
// TestH1H2CrumbleCookie tests that Cookies are crumbled and assembled
// when forwarding to HTTP/2 backend link. go-nghttp2 server
// concatenates crumbled Cookies automatically, so this test is not

View File

@ -192,6 +192,49 @@ func TestH2H1NoVia(t *testing.T) {
}
}
// TestH2H1HostRewrite tests that server rewrites host header field
func TestH2H1HostRewrite(t *testing.T) {
st := newServerTester(nil, t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("request-host", r.Host)
})
defer st.Close()
res, err := st.http2(requestParam{
name: "TestH2H1HostRewrite",
})
if err != nil {
t.Fatalf("Error st.http2() = %v", err)
}
if got, want := res.status, 200; got != want {
t.Errorf("status: %v; want %v", got, want)
}
if got, want := res.header.Get("request-host"), st.backendHost; got != want {
t.Errorf("request-host: %v; want %v", got, want)
}
}
// TestH2H1NoHostRewrite tests that server does not rewrite host
// header field
func TestH2H1NoHostRewrite(t *testing.T) {
st := newServerTester([]string{"--no-host-rewrite"}, t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("request-host", r.Host)
})
defer st.Close()
res, err := st.http2(requestParam{
name: "TestH2H1NoHostRewrite",
})
if err != nil {
t.Fatalf("Error st.http2() = %v", err)
}
if got, want := res.status, 200; got != want {
t.Errorf("status: %v; want %v", got, want)
}
if got, want := res.header.Get("request-host"), st.frontendHost; got != want {
t.Errorf("request-host: %v; want %v", got, want)
}
}
// TestH2H1BadRequestCL tests that server rejects request whose
// content-length header field value does not match its request body
// size.
@ -584,3 +627,46 @@ func TestH2H2ConnectFailure(t *testing.T) {
t.Errorf("status: %v; want %v", got, want)
}
}
// TestH2H2HostRewrite tests that server rewrites host header field
func TestH2H2HostRewrite(t *testing.T) {
st := newServerTester([]string{"--http2-bridge"}, t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("request-host", r.Host)
})
defer st.Close()
res, err := st.http2(requestParam{
name: "TestH2H2HostRewrite",
})
if err != nil {
t.Fatalf("Error st.http2() = %v", err)
}
if got, want := res.status, 200; got != want {
t.Errorf("status: %v; want %v", got, want)
}
if got, want := res.header.Get("request-host"), st.backendHost; got != want {
t.Errorf("request-host: %v; want %v", got, want)
}
}
// TestH2H2NoHostRewrite tests that server does not rewrite host
// header field
func TestH2H2NoHostRewrite(t *testing.T) {
st := newServerTester([]string{"--http2-bridge", "--no-host-rewrite"}, t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("request-host", r.Host)
})
defer st.Close()
res, err := st.http2(requestParam{
name: "TestH2H2NoHostRewrite",
})
if err != nil {
t.Fatalf("Error st.http2() = %v", err)
}
if got, want := res.status, 200; got != want {
t.Errorf("status: %v; want %v", got, want)
}
if got, want := res.header.Get("request-host"), st.frontendHost; got != want {
t.Errorf("request-host: %v; want %v", got, want)
}
}

View File

@ -42,6 +42,8 @@ type serverTester struct {
url string // test frontend server URL
t *testing.T
ts *httptest.Server // backend server
frontendHost string // frontend server host
backendHost string // backend server host
conn net.Conn // connection to frontend server
h2PrefaceSent bool // HTTP/2 preface was sent in conn
nextStreamID uint32 // next stream ID
@ -124,6 +126,8 @@ func newServerTesterInternal(args []string, t *testing.T, handler http.HandlerFu
t: t,
ts: ts,
url: fmt.Sprintf("%v://%v", scheme, authority),
frontendHost: fmt.Sprintf("127.0.0.1:%v", serverPort),
backendHost: backendURL.Host,
nextStreamID: 1,
authority: authority,
frCh: make(chan http2.Frame),

View File

@ -224,6 +224,7 @@ void copy_headers_to_nva(std::vector<nghttp2_nv> &nva, const Headers &headers) {
switch (lookup_token(kv.name)) {
case HD_COOKIE:
case HD_CONNECTION:
case HD_HOST:
case HD_HTTP2_SETTINGS:
case HD_KEEP_ALIVE:
case HD_PROXY_CONNECTION:
@ -249,6 +250,7 @@ void build_http1_headers_from_headers(std::string &hdrs,
switch (lookup_token(kv.name)) {
case HD_CONNECTION:
case HD_COOKIE:
case HD_HOST:
case HD_HTTP2_SETTINGS:
case HD_KEEP_ALIVE:
case HD_PROXY_CONNECTION:

View File

@ -766,6 +766,7 @@ void fill_default_config() {
mod_config()->tls_proto_mask = 0;
mod_config()->no_location_rewrite = false;
mod_config()->no_host_rewrite = false;
mod_config()->argc = 0;
mod_config()->argv = nullptr;
mod_config()->downstream_connections_per_host = 8;
@ -1168,6 +1169,11 @@ HTTP:
--client and default mode. For --http2-proxy and
--client-proxy mode, location header field will not be
altered regardless of this option.
--no-host-rewrite
Don't rewrite host and :authority header fields on
--http2-bridge, --client and default mode. For
--http2-proxy and --client-proxy mode, these headers
will not be altered regardless of this option.
--altsvc=<PROTOID,PORT[,HOST,[ORIGIN]]>
Specify protocol ID, port, host and origin of
alternative service. <HOST> and <ORIGIN> are optional.
@ -1332,6 +1338,7 @@ int main(int argc, char **argv) {
{"tls-ctx-per-worker", no_argument, &flag, 70},
{"backend-response-buffer", required_argument, &flag, 71},
{"backend-request-buffer", required_argument, &flag, 72},
{"no-host-rewrite", no_argument, &flag, 73},
{nullptr, 0, nullptr, 0}};
int option_index = 0;
@ -1661,6 +1668,10 @@ int main(int argc, char **argv) {
// --backend-request-buffer
cmdcfgs.emplace_back(SHRPX_OPT_BACKEND_REQUEST_BUFFER, optarg);
break;
case 73:
// --no-host-rewrite
cmdcfgs.emplace_back(SHRPX_OPT_NO_HOST_REWRITE, "yes");
break;
default:
break;
}

View File

@ -133,6 +133,7 @@ const char SHRPX_OPT_ADD_RESPONSE_HEADER[] = "add-response-header";
const char SHRPX_OPT_WORKER_FRONTEND_CONNECTIONS[] =
"worker-frontend-connections";
const char SHRPX_OPT_NO_LOCATION_REWRITE[] = "no-location-rewrite";
const char SHRPX_OPT_NO_HOST_REWRITE[] = "no-host-rewrite";
const char SHRPX_OPT_BACKEND_HTTP1_CONNECTIONS_PER_HOST[] =
"backend-http1-connections-per-host";
const char SHRPX_OPT_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND[] =
@ -1079,6 +1080,12 @@ int parse_config(const char *opt, const char *optarg) {
return 0;
}
if (util::strieq(opt, SHRPX_OPT_NO_HOST_REWRITE)) {
mod_config()->no_host_rewrite = util::strieq(optarg, "yes");
return 0;
}
if (util::strieq(opt, SHRPX_OPT_BACKEND_HTTP1_CONNECTIONS_PER_HOST)) {
int n;

View File

@ -123,6 +123,7 @@ extern const char SHRPX_OPT_ALTSVC[];
extern const char SHRPX_OPT_ADD_RESPONSE_HEADER[];
extern const char SHRPX_OPT_WORKER_FRONTEND_CONNECTIONS[];
extern const char SHRPX_OPT_NO_LOCATION_REWRITE[];
extern const char SHRPX_OPT_NO_HOST_REWRITE[];
extern const char SHRPX_OPT_BACKEND_HTTP1_CONNECTIONS_PER_HOST[];
extern const char SHRPX_OPT_BACKEND_HTTP1_CONNECTIONS_PER_FRONTEND[];
extern const char SHRPX_OPT_LISTENER_DISABLE_TIMEOUT[];
@ -299,6 +300,7 @@ struct Config {
bool http2_no_cookie_crumbling;
bool upstream_frame_debug;
bool no_location_rewrite;
bool no_host_rewrite;
bool auto_tls_ticket_key;
bool tls_ctx_per_worker;
};

View File

@ -232,6 +232,34 @@ int Http2DownstreamConnection::push_request_headers() {
if (!downstream_) {
return 0;
}
const char *authority = nullptr, *host = nullptr;
if (!get_config()->no_host_rewrite && !get_config()->http2_proxy &&
!get_config()->client_proxy) {
// HTTP/2 backend does not support multiple address, so we always
// use index = 0.
if (!downstream_->get_request_http2_authority().empty()) {
authority = get_config()->downstream_addrs[0].hostport.get();
}
if (downstream_->get_request_header(http2::HD_HOST)) {
host = get_config()->downstream_addrs[0].hostport.get();
}
} else {
if (!downstream_->get_request_http2_authority().empty()) {
authority = downstream_->get_request_http2_authority().c_str();
}
auto h = downstream_->get_request_header(http2::HD_HOST);
if (h) {
host = h->value.c_str();
}
}
if (!authority && !host) {
// upstream is HTTP/1.0. We use backend server's host
// nonetheless.
host = get_config()->downstream_addrs[0].hostport.get();
}
size_t nheader = downstream_->get_request_headers().size();
Headers cookies;
@ -239,28 +267,28 @@ int Http2DownstreamConnection::push_request_headers() {
cookies = downstream_->crumble_request_cookie();
}
// 7 means:
// 8 means:
// 1. :method
// 2. :scheme
// 3. :path
// 4. :authority (optional)
// 5. via (optional)
// 6. x-forwarded-for (optional)
// 7. x-forwarded-proto (optional)
// 4. :authority (at least either :authority or host exists)
// 5. host
// 6. via (optional)
// 7. x-forwarded-for (optional)
// 8. x-forwarded-proto (optional)
auto nva = std::vector<nghttp2_nv>();
nva.reserve(nheader + 7 + cookies.size());
nva.reserve(nheader + 8 + cookies.size());
std::string via_value;
std::string xff_value;
std::string scheme, authority, path, query;
std::string scheme, uri_authority, path, query;
// To reconstruct HTTP/1 status line and headers, proxy should
// preserve host header field. See draft-09 section 8.1.3.1.
if (downstream_->get_request_method() == "CONNECT") {
// The upstream may be HTTP/2 or HTTP/1
if (!downstream_->get_request_http2_authority().empty()) {
nva.push_back(http2::make_nv_ls(
":authority", downstream_->get_request_http2_authority()));
if (authority) {
nva.push_back(http2::make_nv_lc(":authority", authority));
} else {
nva.push_back(
http2::make_nv_ls(":authority", downstream_->get_request_path()));
@ -270,14 +298,8 @@ int Http2DownstreamConnection::push_request_headers() {
nva.push_back(
http2::make_nv_ls(":scheme", downstream_->get_request_http2_scheme()));
nva.push_back(http2::make_nv_ls(":path", downstream_->get_request_path()));
if (!downstream_->get_request_http2_authority().empty()) {
nva.push_back(http2::make_nv_ls(
":authority", downstream_->get_request_http2_authority()));
} else if (!downstream_->get_request_header(http2::HD_HOST)) {
if (LOG_ENABLED(INFO)) {
DCLOG(INFO, this) << "host header field missing";
}
return -1;
if (authority) {
nva.push_back(http2::make_nv_lc(":authority", authority));
}
} else {
// The upstream is HTTP/1
@ -288,11 +310,11 @@ int Http2DownstreamConnection::push_request_headers() {
&u);
if (rv == 0) {
http2::copy_url_component(scheme, &u, UF_SCHEMA, url);
http2::copy_url_component(authority, &u, UF_HOST, url);
http2::copy_url_component(uri_authority, &u, UF_HOST, url);
http2::copy_url_component(path, &u, UF_PATH, url);
http2::copy_url_component(query, &u, UF_QUERY, url);
if (path.empty()) {
if (!authority.empty() &&
if (!uri_authority.empty() &&
downstream_->get_request_method() == "OPTIONS") {
path = "*";
} else {
@ -319,28 +341,33 @@ int Http2DownstreamConnection::push_request_headers() {
} else {
nva.push_back(http2::make_nv_ls(":path", path));
}
if (!authority.empty()) {
if (!get_config()->no_host_rewrite && !get_config()->http2_proxy &&
!get_config()->client_proxy) {
if (authority) {
nva.push_back(http2::make_nv_lc(":authority", authority));
}
} else if (!uri_authority.empty()) {
// TODO properly check IPv6 numeric address
if (authority.find(":") != std::string::npos) {
authority = "[" + authority;
authority += "]";
if (uri_authority.find(":") != std::string::npos) {
uri_authority = "[" + uri_authority;
uri_authority += "]";
}
if (u.field_set & (1 << UF_PORT)) {
authority += ":";
authority += util::utos(u.port);
uri_authority += ":";
uri_authority += util::utos(u.port);
}
nva.push_back(http2::make_nv_ls(":authority", authority));
} else if (!downstream_->get_request_header(http2::HD_HOST)) {
if (LOG_ENABLED(INFO)) {
DCLOG(INFO, this) << "host header field missing";
}
return -1;
nva.push_back(http2::make_nv_ls(":authority", uri_authority));
}
}
nva.push_back(
http2::make_nv_ls(":method", downstream_->get_request_method()));
if (host) {
nva.push_back(http2::make_nv_lc("host", host));
}
http2::copy_headers_to_nva(nva, downstream_->get_request_headers());
bool chunked_encoding = false;

View File

@ -634,7 +634,8 @@ int on_stream_close_callback(nghttp2_session *session, int32_t stream_id,
auto http2session = static_cast<Http2Session *>(user_data);
if (LOG_ENABLED(INFO)) {
SSLOG(INFO, http2session) << "Stream stream_id=" << stream_id
<< " is being closed";
<< " is being closed with error code "
<< error_code;
}
auto sd = static_cast<StreamData *>(
nghttp2_session_get_stream_user_data(session, stream_id));

View File

@ -107,7 +107,7 @@ void connectcb(struct ev_loop *loop, ev_io *w, int revents) {
HttpDownstreamConnection::HttpDownstreamConnection(
DownstreamConnectionPool *dconn_pool, struct ev_loop *loop)
: DownstreamConnection(dconn_pool), rlimit_(loop, &rev_, 0, 0),
ioctrl_(&rlimit_), response_htp_{0}, loop_(loop), fd_(-1) {
ioctrl_(&rlimit_), response_htp_{0}, loop_(loop), addr_idx_(0), fd_(-1) {
// We do not know fd yet, so just set dummy fd 0
ev_io_init(&wev_, connectcb, 0, EV_WRITE);
ev_io_init(&rev_, readcb, 0, EV_READ);
@ -198,6 +198,8 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
DCLOG(INFO, this) << "Connecting to downstream server";
}
addr_idx_ = i;
ev_io_set(&wev_, fd_, EV_WRITE);
ev_io_set(&rev_, fd_, EV_READ);
@ -223,27 +225,51 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
}
int HttpDownstreamConnection::push_request_headers() {
const char *authority = nullptr, *host = nullptr;
if (!get_config()->no_host_rewrite && !get_config()->http2_proxy &&
!get_config()->client_proxy) {
if (!downstream_->get_request_http2_authority().empty()) {
authority = get_config()->downstream_addrs[addr_idx_].hostport.get();
}
if (downstream_->get_request_header(http2::HD_HOST)) {
host = get_config()->downstream_addrs[addr_idx_].hostport.get();
}
} else {
if (!downstream_->get_request_http2_authority().empty()) {
authority = downstream_->get_request_http2_authority().c_str();
}
auto h = downstream_->get_request_header(http2::HD_HOST);
if (h) {
host = h->value.c_str();
}
}
if (!authority && !host) {
// upstream is HTTP/1.0. We use backend server's host
// nonetheless.
host = get_config()->downstream_addrs[addr_idx_].hostport.get();
}
downstream_->assemble_request_cookie();
// Assume that method and request path do not contain \r\n.
std::string hdrs = downstream_->get_request_method();
hdrs += " ";
if (downstream_->get_request_method() == "CONNECT") {
if (!downstream_->get_request_http2_authority().empty()) {
hdrs += downstream_->get_request_http2_authority();
if (authority) {
hdrs += authority;
} else {
hdrs += downstream_->get_request_path();
}
} else if (get_config()->http2_proxy &&
!downstream_->get_request_http2_scheme().empty() &&
!downstream_->get_request_http2_authority().empty() &&
!downstream_->get_request_http2_scheme().empty() && authority &&
(downstream_->get_request_path().c_str()[0] == '/' ||
downstream_->get_request_path() == "*")) {
// Construct absolute-form request target because we are going to
// send a request to a HTTP/1 proxy.
hdrs += downstream_->get_request_http2_scheme();
hdrs += "://";
hdrs += downstream_->get_request_http2_authority();
hdrs += authority;
// Server-wide OPTIONS takes following form in proxy request:
//
@ -259,13 +285,14 @@ int HttpDownstreamConnection::push_request_headers() {
// don't care.
hdrs += downstream_->get_request_path();
}
hdrs += " HTTP/1.1\r\n";
if (!downstream_->get_request_header(http2::HD_HOST) &&
!downstream_->get_request_http2_authority().empty()) {
hdrs += "Host: ";
hdrs += downstream_->get_request_http2_authority();
hdrs += "\r\n";
hdrs += " HTTP/1.1\r\nHost: ";
if (authority) {
hdrs += authority;
} else {
hdrs += host;
}
hdrs += "\r\n";
http2::build_http1_headers_from_headers(hdrs,
downstream_->get_request_headers());

View File

@ -70,6 +70,8 @@ private:
IOControl ioctrl_;
http_parser response_htp_;
struct ev_loop *loop_;
// index of get_config()->downstream_addrs this object is using
size_t addr_idx_;
int fd_;
};

View File

@ -160,7 +160,11 @@ int htp_hdrs_completecb(http_parser *htp) {
return -1;
}
downstream->inspect_http1_request();
if (downstream->get_request_major() == 1 &&
downstream->get_request_minor() == 1 &&
!downstream->get_request_header(http2::HD_HOST)) {
return -1;
}
if (get_config()->client_proxy &&
downstream->get_request_method() != "CONNECT") {