diff --git a/src/shrpx.cc b/src/shrpx.cc
index db01ba57..c52a3c99 100644
--- a/src/shrpx.cc
+++ b/src/shrpx.cc
@@ -2067,13 +2067,15 @@ void process_options(
     upstreamconf.no_tls = true;
   }
 
+  shrpx_proto default_proto;
   if (get_config()->client_mode || get_config()->http2_bridge) {
-    downstreamconf.proto = PROTO_HTTP2;
+    default_proto = PROTO_HTTP2;
   } else {
-    downstreamconf.proto = PROTO_HTTP;
+    default_proto = PROTO_HTTP1;
   }
 
-  if (downstreamconf.proto == PROTO_HTTP && !downstreamconf.http1_tls) {
+  if (!get_config()->client_mode && !get_config()->http2_bridge &&
+      !downstreamconf.http1_tls) {
     downstreamconf.no_tls = true;
   }
 
@@ -2102,6 +2104,7 @@ void process_options(
     addr.port = DEFAULT_DOWNSTREAM_PORT;
 
     DownstreamAddrGroupConfig g(StringRef::from_lit("/"));
+    g.proto = default_proto;
     g.addrs.push_back(std::move(addr));
     mod_config()->router.add_route(StringRef{g.pattern}, addr_groups.size());
     addr_groups.push_back(std::move(g));
@@ -2113,12 +2116,19 @@ void process_options(
       std::move(std::begin(g.addrs), std::end(g.addrs),
                 std::back_inserter(catch_all.addrs));
     }
+    catch_all.proto = default_proto;
     std::vector<DownstreamAddrGroupConfig>().swap(addr_groups);
     // maybe not necessary?
     mod_config()->router = Router();
     mod_config()->router.add_route(StringRef{catch_all.pattern},
                                    addr_groups.size());
     addr_groups.push_back(std::move(catch_all));
+  } else {
+    for (auto &g : addr_groups) {
+      if (g.proto == PROTO_NONE) {
+        g.proto = default_proto;
+      }
+    }
   }
 
   if (LOG_ENABLED(INFO)) {
diff --git a/src/shrpx_client_handler.cc b/src/shrpx_client_handler.cc
index 7032471d..9dbd6453 100644
--- a/src/shrpx_client_handler.cc
+++ b/src/shrpx_client_handler.cc
@@ -385,7 +385,7 @@ ClientHandler::ClientHandler(Worker *worker, int fd, SSL *ssl,
             get_config()->conn.upstream.ratelimit.write,
             get_config()->conn.upstream.ratelimit.read, writecb, readcb,
             timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold,
-            get_config()->tls.dyn_rec.idle_timeout),
+            get_config()->tls.dyn_rec.idle_timeout, PROTO_NONE),
       ipaddr_(ipaddr),
       port_(port),
       faddr_(faddr),
@@ -712,7 +712,7 @@ ClientHandler::get_downstream_connection(Downstream *downstream) {
                        << " Create new one";
     }
 
-    if (downstreamconf.proto == PROTO_HTTP2) {
+    if (group.proto == PROTO_HTTP2) {
       if (group.http2_freelist.empty()) {
         if (LOG_ENABLED(INFO)) {
           CLOG(INFO, this)
diff --git a/src/shrpx_config.h b/src/shrpx_config.h
index 039d2a9d..b54951dd 100644
--- a/src/shrpx_config.h
+++ b/src/shrpx_config.h
@@ -235,7 +235,7 @@ constexpr char SHRPX_OPT_BACKEND_HTTP2_MAX_CONCURRENT_STREAMS[] =
 
 constexpr size_t SHRPX_OBFUSCATED_NODE_LENGTH = 8;
 
-enum shrpx_proto { PROTO_HTTP2, PROTO_HTTP };
+enum shrpx_proto { PROTO_NONE, PROTO_HTTP1, PROTO_HTTP2, PROTO_MEMCACHED };
 
 enum shrpx_forwarded_param {
   FORWARDED_NONE = 0,
@@ -306,6 +306,8 @@ struct DownstreamAddrGroupConfig {
 
   ImmutableString pattern;
   std::vector<DownstreamAddrConfig> addrs;
+  // Application protocol used in this group
+  shrpx_proto proto;
 };
 
 struct TicketKey {
@@ -561,8 +563,6 @@ struct ConnectionConfig {
     size_t connections_per_frontend;
     size_t request_buffer_size;
     size_t response_buffer_size;
-    // downstream protocol; this will be determined by given options.
-    shrpx_proto proto;
     // Address family of backend connection.  One of either AF_INET,
     // AF_INET6 or AF_UNSPEC.  This is ignored if backend connection
     // is made via Unix domain socket.
diff --git a/src/shrpx_connection.cc b/src/shrpx_connection.cc
index 6bbaab3b..418afb12 100644
--- a/src/shrpx_connection.cc
+++ b/src/shrpx_connection.cc
@@ -47,7 +47,7 @@ Connection::Connection(struct ev_loop *loop, int fd, SSL *ssl,
                        const RateLimitConfig &read_limit, IOCb writecb,
                        IOCb readcb, TimerCb timeoutcb, void *data,
                        size_t tls_dyn_rec_warmup_threshold,
-                       ev_tstamp tls_dyn_rec_idle_timeout)
+                       ev_tstamp tls_dyn_rec_idle_timeout, shrpx_proto proto)
     : tls{DefaultMemchunks(mcpool), DefaultPeekMemchunks(mcpool)},
       wlimit(loop, &wev, write_limit.rate, write_limit.burst),
       rlimit(loop, &rev, read_limit.rate, read_limit.burst, this),
@@ -58,7 +58,8 @@ Connection::Connection(struct ev_loop *loop, int fd, SSL *ssl,
       data(data),
       fd(fd),
       tls_dyn_rec_warmup_threshold(tls_dyn_rec_warmup_threshold),
-      tls_dyn_rec_idle_timeout(tls_dyn_rec_idle_timeout) {
+      tls_dyn_rec_idle_timeout(tls_dyn_rec_idle_timeout),
+      proto(proto) {
 
   ev_io_init(&wev, writecb, fd, EV_WRITE);
   ev_io_init(&rev, readcb, fd, EV_READ);
diff --git a/src/shrpx_connection.h b/src/shrpx_connection.h
index 30ff448d..71ba33f4 100644
--- a/src/shrpx_connection.h
+++ b/src/shrpx_connection.h
@@ -77,7 +77,7 @@ struct Connection {
              const RateLimitConfig &write_limit,
              const RateLimitConfig &read_limit, IOCb writecb, IOCb readcb,
              TimerCb timeoutcb, void *data, size_t tls_dyn_rec_warmup_threshold,
-             ev_tstamp tls_dyn_rec_idle_timeout);
+             ev_tstamp tls_dyn_rec_idle_timeout, shrpx_proto proto);
   ~Connection();
 
   void disconnect();
@@ -133,6 +133,10 @@ struct Connection {
   int fd;
   size_t tls_dyn_rec_warmup_threshold;
   ev_tstamp tls_dyn_rec_idle_timeout;
+  // Application protocol used over the connection.  This field is not
+  // used in this object at the moment.  The rest of the program may
+  // use this value when it is useful.
+  shrpx_proto proto;
 };
 
 } // namespace shrpx
diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc
index 008f1289..4da62790 100644
--- a/src/shrpx_connection_handler.cc
+++ b/src/shrpx_connection_handler.cc
@@ -203,7 +203,7 @@ int ConnectionHandler::create_single_worker() {
         nb_.get(),
 #endif // HAVE_NEVERBLEED
         StringRef{tlsconf.cacert}, StringRef{memcachedconf.cert_file},
-        StringRef{memcachedconf.private_key_file}, StringRef(), nullptr);
+        StringRef{memcachedconf.private_key_file}, nullptr);
     all_ssl_ctx_.push_back(session_cache_ssl_ctx);
   }
 
@@ -253,7 +253,7 @@ int ConnectionHandler::create_worker_thread(size_t num) {
           nb_.get(),
 #endif // HAVE_NEVERBLEED
           StringRef{tlsconf.cacert}, StringRef{memcachedconf.cert_file},
-          StringRef{memcachedconf.private_key_file}, StringRef{}, nullptr);
+          StringRef{memcachedconf.private_key_file}, nullptr);
       all_ssl_ctx_.push_back(session_cache_ssl_ctx);
     }
     auto worker =
@@ -767,7 +767,7 @@ SSL_CTX *ConnectionHandler::create_tls_ticket_key_memcached_ssl_ctx() {
       nb_.get(),
 #endif // HAVE_NEVERBLEED
       StringRef{tlsconf.cacert}, StringRef{memcachedconf.cert_file},
-      StringRef{memcachedconf.private_key_file}, StringRef{}, nullptr);
+      StringRef{memcachedconf.private_key_file}, nullptr);
 
   all_ssl_ctx_.push_back(ssl_ctx);
 
diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc
index 8bb6ab22..dd8cd5e4 100644
--- a/src/shrpx_http2_session.cc
+++ b/src/shrpx_http2_session.cc
@@ -173,7 +173,7 @@ Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx,
             get_config()->conn.downstream.timeout.write,
             get_config()->conn.downstream.timeout.read, {}, {}, writecb, readcb,
             timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold,
-            get_config()->tls.dyn_rec.idle_timeout),
+            get_config()->tls.dyn_rec.idle_timeout, PROTO_HTTP2),
       wb_(worker->get_mcpool()),
       worker_(worker),
       ssl_ctx_(ssl_ctx),
@@ -398,6 +398,8 @@ int Http2Session::initiate_connection() {
           return -1;
         }
 
+        ssl::setup_downstream_http2_alpn(ssl);
+
         conn_.set_ssl(ssl);
       }
 
diff --git a/src/shrpx_http2_upstream.cc b/src/shrpx_http2_upstream.cc
index 0803433c..aecff732 100644
--- a/src/shrpx_http2_upstream.cc
+++ b/src/shrpx_http2_upstream.cc
@@ -824,9 +824,7 @@ Http2Upstream::Http2Upstream(ClientHandler *handler)
       downstream_queue_(
           get_config()->http2_proxy
               ? get_config()->conn.downstream.connections_per_host
-              : get_config()->conn.downstream.proto == PROTO_HTTP
-                    ? get_config()->conn.downstream.connections_per_frontend
-                    : 0,
+              : get_config()->conn.downstream.connections_per_frontend,
           !get_config()->http2_proxy),
       handler_(handler),
       session_(nullptr),
diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc
index cecc9961..b68bb27e 100644
--- a/src/shrpx_http_downstream_connection.cc
+++ b/src/shrpx_http_downstream_connection.cc
@@ -118,7 +118,7 @@ HttpDownstreamConnection::HttpDownstreamConnection(DownstreamAddrGroup *group,
             get_config()->conn.downstream.timeout.write,
             get_config()->conn.downstream.timeout.read, {}, {}, connectcb,
             readcb, timeoutcb, this, get_config()->tls.dyn_rec.warmup_threshold,
-            get_config()->tls.dyn_rec.idle_timeout),
+            get_config()->tls.dyn_rec.idle_timeout, PROTO_HTTP1),
       do_read_(&HttpDownstreamConnection::noop),
       do_write_(&HttpDownstreamConnection::noop),
       worker_(worker),
@@ -153,6 +153,8 @@ int HttpDownstreamConnection::attach_downstream(Downstream *downstream) {
         return -1;
       }
 
+      ssl::setup_downstream_http1_alpn(ssl);
+
       conn_.set_ssl(ssl);
     }
 
diff --git a/src/shrpx_memcached_connection.cc b/src/shrpx_memcached_connection.cc
index 4df031d8..0b494ba8 100644
--- a/src/shrpx_memcached_connection.cc
+++ b/src/shrpx_memcached_connection.cc
@@ -96,7 +96,7 @@ MemcachedConnection::MemcachedConnection(const Address *addr,
                                          const StringRef &sni_name,
                                          MemchunkPool *mcpool)
     : conn_(loop, -1, nullptr, mcpool, write_timeout, read_timeout, {}, {},
-            connectcb, readcb, timeoutcb, this, 0, 0.),
+            connectcb, readcb, timeoutcb, this, 0, 0., PROTO_MEMCACHED),
       do_read_(&MemcachedConnection::noop),
       do_write_(&MemcachedConnection::noop),
       sni_name_(sni_name.str()),
diff --git a/src/shrpx_spdy_upstream.cc b/src/shrpx_spdy_upstream.cc
index 70d4096c..97e56f10 100644
--- a/src/shrpx_spdy_upstream.cc
+++ b/src/shrpx_spdy_upstream.cc
@@ -503,9 +503,7 @@ SpdyUpstream::SpdyUpstream(uint16_t version, ClientHandler *handler)
       downstream_queue_(
           get_config()->http2_proxy
               ? get_config()->conn.downstream.connections_per_host
-              : get_config()->conn.downstream.proto == PROTO_HTTP
-                    ? get_config()->conn.downstream.connections_per_frontend
-                    : 0,
+              : get_config()->conn.downstream.connections_per_frontend,
           !get_config()->http2_proxy),
       handler_(handler),
       session_(nullptr) {
diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc
index 2f9d200a..208a7342 100644
--- a/src/shrpx_ssl.cc
+++ b/src/shrpx_ssl.cc
@@ -659,12 +659,28 @@ int select_h1_next_proto_cb(SSL *ssl, unsigned char **out,
 }
 } // namespace
 
+namespace {
+int select_next_proto_cb(SSL *ssl, unsigned char **out, unsigned char *outlen,
+                         const unsigned char *in, unsigned int inlen,
+                         void *arg) {
+  auto conn = static_cast<Connection *>(SSL_get_app_data(ssl));
+  switch (conn->proto) {
+  case PROTO_HTTP1:
+    return select_h1_next_proto_cb(ssl, out, outlen, in, inlen, arg);
+  case PROTO_HTTP2:
+    return select_h2_next_proto_cb(ssl, out, outlen, in, inlen, arg);
+  default:
+    return SSL_TLSEXT_ERR_NOACK;
+  }
+}
+} // namespace
+
 SSL_CTX *create_ssl_client_context(
 #ifdef HAVE_NEVERBLEED
     neverbleed_t *nb,
 #endif // HAVE_NEVERBLEED
     const StringRef &cacert, const StringRef &cert_file,
-    const StringRef &private_key_file, const StringRef &alpn,
+    const StringRef &private_key_file,
     int (*next_proto_select_cb)(SSL *s, unsigned char **out,
                                 unsigned char *outlen, const unsigned char *in,
                                 unsigned int inlen, void *arg)) {
@@ -742,14 +758,10 @@ SSL_CTX *create_ssl_client_context(
 #endif // HAVE_NEVERBLEED
   }
 
-  // NPN selection callback
+  // NPN selection callback.  This is required to set SSL_CTX because
+  // OpenSSL does not offer SSL_set_next_proto_select_cb.
   SSL_CTX_set_next_proto_select_cb(ssl_ctx, next_proto_select_cb, nullptr);
 
-#if OPENSSL_VERSION_NUMBER >= 0x10002000L
-  // ALPN advertisement
-  SSL_CTX_set_alpn_protos(ssl_ctx, alpn.byte(), alpn.size());
-#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
-
   return ssl_ctx;
 }
 
@@ -1303,29 +1315,29 @@ SSL_CTX *setup_downstream_client_ssl_context(
   }
 
   auto &tlsconf = get_config()->tls;
-  auto &downstreamconf = get_config()->conn.downstream;
-
-  std::vector<unsigned char> h2alpn;
-  StringRef alpn;
-  int (*next_proto_select_cb)(SSL *s, unsigned char **out,
-                              unsigned char *outlen, const unsigned char *in,
-                              unsigned int inlen, void *arg);
-
-  if (downstreamconf.proto == PROTO_HTTP2) {
-    h2alpn = util::get_default_alpn();
-    alpn = StringRef(h2alpn.data(), h2alpn.size());
-    next_proto_select_cb = select_h2_next_proto_cb;
-  } else {
-    alpn = StringRef::from_lit(NGHTTP2_H1_1_ALPN);
-    next_proto_select_cb = select_h1_next_proto_cb;
-  }
 
   return ssl::create_ssl_client_context(
 #ifdef HAVE_NEVERBLEED
       nb,
 #endif // HAVE_NEVERBLEED
       StringRef{tlsconf.cacert}, StringRef{tlsconf.client.cert_file},
-      StringRef{tlsconf.client.private_key_file}, alpn, next_proto_select_cb);
+      StringRef{tlsconf.client.private_key_file}, select_next_proto_cb);
+}
+
+void setup_downstream_http2_alpn(SSL *ssl) {
+  auto alpn = util::get_default_alpn();
+#if OPENSSL_VERSION_NUMBER >= 0x10002000L
+  // ALPN advertisement
+  SSL_set_alpn_protos(ssl, alpn.data(), alpn.size());
+#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
+}
+
+void setup_downstream_http1_alpn(SSL *ssl) {
+  auto alpn = StringRef::from_lit(NGHTTP2_H1_1_ALPN);
+#if OPENSSL_VERSION_NUMBER >= 0x10002000L
+  // ALPN advertisement
+  SSL_set_alpn_protos(ssl, alpn.byte(), alpn.size());
+#endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
 }
 
 CertLookupTree *create_cert_lookup_tree() {
diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h
index c913e5b9..900d9516 100644
--- a/src/shrpx_ssl.h
+++ b/src/shrpx_ssl.h
@@ -71,13 +71,14 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file
 #endif // HAVE_NEVERBLEED
                             );
 
-// Create client side SSL_CTX
+// Create client side SSL_CTX.  This does not configure ALPN settings.
+// |next_proto_select_cb| is for NPN.
 SSL_CTX *create_ssl_client_context(
 #ifdef HAVE_NEVERBLEED
     neverbleed_t *nb,
 #endif // HAVE_NEVERBLEED
     const StringRef &cacert, const StringRef &cert_file,
-    const StringRef &private_key_file, const StringRef &alpn,
+    const StringRef &private_key_file,
     int (*next_proto_select_cb)(SSL *s, unsigned char **out,
                                 unsigned char *outlen, const unsigned char *in,
                                 unsigned int inlen, void *arg));
@@ -201,6 +202,11 @@ SSL_CTX *setup_downstream_client_ssl_context(
 #endif // HAVE_NEVERBLEED
     );
 
+// Sets ALPN settings in |SSL| suitable for HTTP/2 use.
+void setup_downstream_http2_alpn(SSL *ssl);
+// Sets ALPN settings in |SSL| suitable for HTTP/1.1 use.
+void setup_downstream_http1_alpn(SSL *ssl);
+
 // Creates CertLookupTree.  If frontend is configured not to use TLS,
 // this function returns nullptr.
 CertLookupTree *create_cert_lookup_tree();
diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc
index d2115d25..65d547b6 100644
--- a/src/shrpx_worker.cc
+++ b/src/shrpx_worker.cc
@@ -104,6 +104,7 @@ Worker::Worker(struct ev_loop *loop, SSL_CTX *sv_ssl_ctx, SSL_CTX *cl_ssl_ctx,
 
     dst.pattern = src.pattern;
     dst.addrs.resize(src.addrs.size());
+    dst.proto = src.proto;
 
     for (size_t j = 0; j < src.addrs.size(); ++j) {
       auto &src_addr = src.addrs[j];
diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h
index 1f0c7d78..0602e988 100644
--- a/src/shrpx_worker.h
+++ b/src/shrpx_worker.h
@@ -86,6 +86,8 @@ struct DownstreamAddr {
 struct DownstreamAddrGroup {
   ImmutableString pattern;
   std::vector<DownstreamAddr> addrs;
+  // Application protocol used in this group
+  shrpx_proto proto;
   // List of Http2Session which is not fully utilized (i.e., the
   // server advertized maximum concurrency is not reached).  We will
   // coalesce as much stream as possible in one Http2Session to fully