nghttpx: Disable TLS renegotiation properly

4ed4efc does not disable TLS renegotiation at all, if client keeps
rengotiations without sending application data. In this change,
we intercept the raw incoming data from the client and if it is a
renegotiation, drop the connection immediately.

src/shrpx_client_handler.cc

@@ -48,10 +48,6 @@ namespace {
 void upstream_readcb(bufferevent *bev, void *arg)
 {
   auto handler = reinterpret_cast<ClientHandler*>(arg);
-  if(handler->get_tls_renegotiation()) {
-    delete handler;
-    return;
-  }
   int rv = handler->on_read();
   if(rv != 0) {
     delete handler;
@@ -64,7 +60,7 @@ void upstream_writecb(bufferevent *bev, void *arg)
 {
   auto handler = reinterpret_cast<ClientHandler*>(arg);
   // We actually depend on write low-warter mark == 0.
-  if(evbuffer_get_length(bufferevent_get_output(bev)) > 0) {
+  if(handler->get_pending_write_length() > 0) {
     // Possibly because of deferred callback, we may get this callback
     // when the output buffer is not empty.
     return;
@@ -223,6 +219,19 @@ void upstream_http1_connhd_readcb(bufferevent *bev, void *arg)
 }
 } // namespace
 
+namespace {
+void tls_raw_readcb(evbuffer *buffer, const evbuffer_cb_info *info, void *arg)
+{
+  auto handler = static_cast<ClientHandler*>(arg);
+  if(handler->get_tls_renegotiation()) {
+    if(LOG_ENABLED(INFO)) {
+      CLOG(INFO, handler) << "Close connection due to TLS renegotiation";
+    }
+    delete handler;
+  }
+}
+} // namespace
+
 ClientHandler::ClientHandler(bufferevent *bev, int fd, SSL *ssl,
                              const char *ipaddr)
   : ipaddr_(ipaddr),
@@ -247,6 +256,8 @@ ClientHandler::ClientHandler(bufferevent *bev, int fd, SSL *ssl,
   if(ssl_) {
     SSL_set_app_data(ssl_, reinterpret_cast<char*>(this));
     set_bev_cb(nullptr, upstream_writecb, upstream_eventcb);
+    auto input = bufferevent_get_input(bufferevent_get_underlying(bev_));
+    evbuffer_add_cb(input, tls_raw_readcb, this);
   } else {
     // For non-TLS version, first create HttpsUpstream. It may be
     // upgraded to HTTP/2.0 through HTTP Upgrade or direct HTTP/2.0
@@ -266,11 +277,16 @@ ClientHandler::~ClientHandler()
     SSL_set_shutdown(ssl_, SSL_RECEIVED_SHUTDOWN);
     SSL_shutdown(ssl_);
   }
+  auto underlying = bufferevent_get_underlying(bev_);
   bufferevent_disable(bev_, EV_READ | EV_WRITE);
   bufferevent_free(bev_);
   if(ssl_) {
     SSL_free(ssl_);
   }
+  if(underlying) {
+    bufferevent_disable(underlying, EV_READ | EV_WRITE);
+    bufferevent_free(underlying);
+  }
   shutdown(fd_, SHUT_WR);
   close(fd_);
   for(auto dconn : dconn_pool_) {
@@ -435,8 +451,12 @@ DownstreamConnection* ClientHandler::get_downstream_connection()
 
 size_t ClientHandler::get_pending_write_length()
 {
-  auto output = bufferevent_get_output(bev_);
-  return evbuffer_get_length(output);
+  auto underlying = bufferevent_get_underlying(bev_);
+  auto len = evbuffer_get_length(bufferevent_get_output(bev_));
+  if(underlying) {
+    len += evbuffer_get_length(bufferevent_get_output(underlying));
+  }
+  return len;
 }
 
 SSL* ClientHandler::get_ssl() const

src/shrpx_http2_upstream.cc

@@ -59,7 +59,8 @@ ssize_t send_callback(nghttp2_session *session,
   auto bev = handler->get_bev();
   auto output = bufferevent_get_output(bev);
   // Check buffer length and return WOULDBLOCK if it is large enough.
-  if(evbuffer_get_length(output) > SHRPX_HTTP2_UPSTREAM_OUTPUT_UPPER_THRES) {
+  if(handler->get_pending_write_length() >
+     SHRPX_HTTP2_UPSTREAM_OUTPUT_UPPER_THRES) {
     return NGHTTP2_ERR_WOULDBLOCK;
   }
 
@@ -594,7 +595,7 @@ int Http2Upstream::on_read()
   }
   if(nghttp2_session_want_read(session_) == 0 &&
      nghttp2_session_want_write(session_) == 0 &&
-     evbuffer_get_length(bufferevent_get_output(bev)) == 0) {
+     handler_->get_pending_write_length() == 0) {
     if(LOG_ENABLED(INFO)) {
       ULOG(INFO, this) << "No more read/write for this HTTP2 session";
     }
@@ -619,7 +620,7 @@ int Http2Upstream::send()
   if(rv == 0) {
     if(nghttp2_session_want_read(session_) == 0 &&
        nghttp2_session_want_write(session_) == 0 &&
-       evbuffer_get_length(bufferevent_get_output(handler_->get_bev())) == 0) {
+       handler_->get_pending_write_length() == 0) {
       if(LOG_ENABLED(INFO)) {
         ULOG(INFO, this) << "No more read/write for this HTTP2 session";
       }

src/shrpx_https_upstream.cc

@@ -446,10 +446,8 @@ void https_downstream_readcb(bufferevent *bev, void *ptr)
         }
       }
     } else {
-      auto handler = upstream->get_client_handler();
-      auto bev = handler->get_bev();
-      size_t outputlen = evbuffer_get_length(bufferevent_get_output(bev));
-      if(outputlen > SHRPX_HTTPS_UPSTREAM_OUTPUT_UPPER_THRES) {
+      if(upstream->get_client_handler()->get_pending_write_length() >
+         SHRPX_HTTPS_UPSTREAM_OUTPUT_UPPER_THRES) {
         downstream->pause_read(SHRPX_NO_BUFFER);
       }
     }

src/shrpx_spdy_upstream.cc

@@ -59,7 +59,8 @@ ssize_t send_callback(spdylay_session *session,
   auto bev = handler->get_bev();
   auto output = bufferevent_get_output(bev);
   // Check buffer length and return WOULDBLOCK if it is large enough.
-  if(evbuffer_get_length(output) > SHRPX_SPDY_UPSTREAM_OUTPUT_UPPER_THRES) {
+  if(handler->get_pending_write_length() >
+     SHRPX_SPDY_UPSTREAM_OUTPUT_UPPER_THRES) {
     return SPDYLAY_ERR_WOULDBLOCK;
   }
 
@@ -451,7 +452,7 @@ int SpdyUpstream::on_read()
   if(rv == 0) {
     if(spdylay_session_want_read(session_) == 0 &&
        spdylay_session_want_write(session_) == 0 &&
-       evbuffer_get_length(bufferevent_get_output(handler_->get_bev())) == 0) {
+       handler_->get_pending_write_length() == 0) {
       if(LOG_ENABLED(INFO)) {
         ULOG(INFO, this) << "No more read/write for this SPDY session";
       }
@@ -477,7 +478,7 @@ int SpdyUpstream::send()
   if(rv == 0) {
     if(spdylay_session_want_read(session_) == 0 &&
        spdylay_session_want_write(session_) == 0 &&
-       evbuffer_get_length(bufferevent_get_output(handler_->get_bev())) == 0) {
+       handler_->get_pending_write_length() == 0) {
       if(LOG_ENABLED(INFO)) {
         ULOG(INFO, this) << "No more read/write for this SPDY session";
       }

src/shrpx_ssl.cc

@@ -458,9 +458,14 @@ ClientHandler* accept_connection(event_base *evbase, SSL_CTX *ssl_ctx,
                    << ERR_error_string(ERR_get_error(), nullptr);
         return nullptr;
       }
-      bev = bufferevent_openssl_socket_new
-        (evbase, fd, ssl,
-         BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_DEFER_CALLBACKS);
+      SSL_set_fd(ssl, fd);
+      // To detect TLS renegotiation and deal with it, we have to use
+      // filter-based OpenSSL bufferevent and set evbuffer callback by
+      // our own.
+      auto underlying_bev = bufferevent_socket_new(evbase, fd, 0);
+      bev = bufferevent_openssl_filter_new(evbase, underlying_bev, ssl,
+                                           BUFFEREVENT_SSL_ACCEPTING,
+                                           BEV_OPT_DEFER_CALLBACKS);
     } else {
       bev = bufferevent_socket_new(evbase, fd, BEV_OPT_DEFER_CALLBACKS);
     }