diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc
index 1ad416fb..12d63087 100644
--- a/src/shrpx_http2_session.cc
+++ b/src/shrpx_http2_session.cc
@@ -1501,6 +1501,7 @@ int Http2Session::connection_made() {
 #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
 
     if (!next_proto) {
+      downstream_failure(addr_);
       return -1;
     }
 
@@ -1509,6 +1510,7 @@ int Http2Session::connection_made() {
       SSLOG(INFO, this) << "Negotiated next protocol: " << proto;
     }
     if (!util::check_h2_is_selected(proto)) {
+      downstream_failure(addr_);
       return -1;
     }
   }
@@ -1897,6 +1899,8 @@ int Http2Session::tls_handshake() {
   }
 
   if (rv < 0) {
+    downstream_failure(addr_);
+
     return rv;
   }
 
@@ -1906,6 +1910,8 @@ int Http2Session::tls_handshake() {
 
   if (!get_config()->tls.insecure &&
       ssl::check_cert(conn_.tls.ssl, addr_) != 0) {
+    downstream_failure(addr_);
+
     return -1;
   }
 
diff --git a/src/shrpx_http_downstream_connection.cc b/src/shrpx_http_downstream_connection.cc
index 495147d3..1014c633 100644
--- a/src/shrpx_http_downstream_connection.cc
+++ b/src/shrpx_http_downstream_connection.cc
@@ -965,6 +965,8 @@ int HttpDownstreamConnection::tls_handshake() {
   }
 
   if (rv < 0) {
+    downstream_failure(addr_);
+
     return rv;
   }
 
@@ -974,6 +976,8 @@ int HttpDownstreamConnection::tls_handshake() {
 
   if (!get_config()->tls.insecure &&
       ssl::check_cert(conn_.tls.ssl, addr_) != 0) {
+    downstream_failure(addr_);
+
     return -1;
   }