nghttpd: Avoid to use bufferevent for connection

src/HttpServer.cc

@@ -42,8 +42,18 @@
 #include <zlib.h>
 
 #include <event.h>
-#include <event2/bufferevent_ssl.h>
 #include <event2/listener.h>
+#include <event2/bufferevent.h>
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+#include "nghttp2_helper.h"
+
+#ifdef __cplusplus
+}
+#endif
 
 #include "app_helper.h"
 #include "http2.h"
@@ -66,7 +76,6 @@ const std::string NGHTTPD_SERVER = "nghttpd nghttp2/" NGHTTP2_VERSION;
 
 Config::Config()
   : data_ptr(nullptr),
-    output_upper_thres(1024*1024),
     padding(0),
     num_worker(1),
     header_table_size(-1),
@@ -212,16 +221,27 @@ Http2Handler::Http2Handler(Sessions *sessions,
   : session_id_(session_id),
     session_(nullptr),
     sessions_(sessions),
-    bev_(nullptr),
     ssl_(ssl),
+    rev_(nullptr),
+    wev_(nullptr),
     settings_timerev_(nullptr),
+    pending_data_(nullptr),
+    pending_datalen_(0),
     left_connhd_len_(NGHTTP2_CLIENT_CONNECTION_HEADER_LEN),
     fd_(fd)
-{}
+{
+  nghttp2_buf_wrap_init(&sendbuf_, sendbufarray_, sizeof(sendbufarray_));
+}
 
 Http2Handler::~Http2Handler()
 {
   on_session_closed(this, session_id_);
+  if(rev_) {
+    event_free(rev_);
+  }
+  if(wev_) {
+    event_free(wev_);
+  }
   if(settings_timerev_) {
     event_free(settings_timerev_);
   }
@@ -230,10 +250,6 @@ Http2Handler::~Http2Handler()
     SSL_set_shutdown(ssl_, SSL_RECEIVED_SHUTDOWN);
     SSL_shutdown(ssl_);
   }
-  if(bev_) {
-    bufferevent_disable(bev_, EV_READ | EV_WRITE);
-    bufferevent_free(bev_);
-  }
   if(ssl_) {
     SSL_free(ssl_);
   }
@@ -247,137 +263,325 @@ void Http2Handler::remove_self()
 }
 
 namespace {
-void readcb(bufferevent *bev, void *ptr)
+void rev_cb(evutil_socket_t fd, short what, void *arg)
 {
   int rv;
-  auto handler = static_cast<Http2Handler*>(ptr);
+  auto handler = static_cast<Http2Handler*>(arg);
+
+  if(what & EV_READ) {
     rv = handler->on_read();
-  if(rv != 0) {
+    if(rv == -1) {
       delete_handler(handler);
     }
   }
+}
 } // namespace
 
 namespace {
-void writecb(bufferevent *bev, void *ptr)
+void wev_cb(evutil_socket_t fd, short what, void *arg)
 {
-  if(evbuffer_get_length(bufferevent_get_output(bev)) > 0) {
-    return;
-  }
   int rv;
-  auto handler = static_cast<Http2Handler*>(ptr);
+  auto handler = static_cast<Http2Handler*>(arg);
+
+  if(what & EV_WRITE) {
     rv = handler->on_write();
+    if(rv == -1) {
+      delete_handler(handler);
+    }
+  }
+}
+} // namespace
+
+int Http2Handler::handle_ssl_temporal_error(int err)
+{
+  auto sslerr = SSL_get_error(ssl_, err);
+
+  switch(sslerr) {
+  case SSL_ERROR_WANT_READ:
+    event_add(rev_, nullptr);
+    return 1;
+  case SSL_ERROR_WANT_WRITE:
+    event_add(wev_, nullptr);
+    return 1;
+  }
+
+  return -1;
+}
+
+int Http2Handler::tls_write(const uint8_t *data, size_t datalen)
+{
+  int rv;
+  size_t max_avail;
+
+  // OpenSSL sends at most 16K bytes
+  max_avail = ssl_ ?
+    std::min((ssize_t)16384, nghttp2_buf_avail(&sendbuf_)) :
+    nghttp2_buf_avail(&sendbuf_);
+
+  if(max_avail < datalen) {
+    if(nghttp2_buf_len(&sendbuf_) > 0) {
+      rv = tls_write_pending();
+
+      if(rv == -1) {
+        return -1;
+      }
+
+      if(rv == 1) {
+        pending_data_ = data;
+        pending_datalen_ = datalen;
+
+        return 1;
+      }
+    }
+
+    assert(nghttp2_buf_avail(&sendbuf_) >= (ssize_t)datalen);
+  }
+  //std::cerr << "DBG: copy " << datalen << " bytes" << std::endl;
+  sendbuf_.last = nghttp2_cpymem(sendbuf_.last, data, datalen);
+
+  return 0;
+}
+
+int Http2Handler::tls_write_pending()
+{
+  int rv;
+
+  if(nghttp2_buf_len(&sendbuf_) == 0) {
+    return 0;
+  }
+
+  for(;;) {
+    if(ssl_) {
+      ERR_clear_error();
+
+      rv = SSL_write(ssl_, sendbuf_.pos, nghttp2_buf_len(&sendbuf_));
+
+      if(rv == 0) {
+        return -1;
+      }
+      if(rv < 0) {
+        return handle_ssl_temporal_error(rv);
+      }
+    } else {
+      while((rv = write(fd_, sendbuf_.pos, nghttp2_buf_len(&sendbuf_))) &&
+            rv == -1 && errno == EINTR);
+
+      if(rv == 0) {
+        continue;
+      }
+      if(rv < 0) {
+        if(errno == EAGAIN || errno == EWOULDBLOCK) {
+          event_add(wev_, nullptr);
+          return 1;
+        }
+        return -1;
+      }
+    }
+
+    sendbuf_.pos += rv;
+
+    if(nghttp2_buf_len(&sendbuf_) == 0) {
+      nghttp2_buf_reset(&sendbuf_);
+
+      if(pending_data_) {
+        assert(nghttp2_buf_avail(&sendbuf_) >= (ssize_t)pending_datalen_);
+        sendbuf_.last = nghttp2_cpymem(sendbuf_.last,
+                                      pending_data_, pending_datalen_);
+        pending_data_ = nullptr;
+        pending_datalen_ = 0;
+
+        continue;
+      }
+
+      return 0;
+    }
+  }
+}
+
+namespace {
+void tls_handshake_cb(evutil_socket_t fd, short what, void *arg)
+{
+  int rv;
+  auto handler = static_cast<Http2Handler*>(arg);
+
+  if(what & (EV_READ | EV_WRITE)) {
+    rv = handler->tls_handshake();
+    if(rv == -1) {
+      delete_handler(handler);
+      return;
+    }
+    if(rv == 1) {
+      return;
+    }
+
+    rv = handler->on_connect();
     if(rv != 0) {
       delete_handler(handler);
+      return;
+    }
   }
 }
 } // namespace
 
-namespace {
+int Http2Handler::tls_handshake()
-void eventcb(bufferevent *bev, short events, void *ptr)
 {
-  auto handler = static_cast<Http2Handler*>(ptr);
+  int rv;
-  if(events & BEV_EVENT_CONNECTED) {
-    // SSL/TLS handshake completed
-    if(handler->verify_npn_result() != 0) {
-      delete_handler(handler);
-      return;
-    }
-    if(handler->on_connect() != 0) {
-      delete_handler(handler);
-      return;
-    }
-  } else if(events & BEV_EVENT_EOF) {
-    delete_handler(handler);
-    return;
-  } else if(events & (BEV_EVENT_ERROR | BEV_EVENT_TIMEOUT)) {
-    delete_handler(handler);
-    return;
-  }
-}
-} // namespace
 
-namespace {
+  ERR_clear_error();
-void connhd_readcb(bufferevent *bev, void *ptr)
+
-{
+  rv = SSL_accept(ssl_);
-  uint8_t data[24];
+  if(rv == 0) {
-  auto handler = static_cast<Http2Handler*>(ptr);
+    return -1;
-  size_t leftlen = handler->get_left_connhd_len();
-  auto input = bufferevent_get_input(bev);
-  int readlen = evbuffer_remove(input, data, leftlen);
-  if(readlen == -1) {
-    delete_handler(handler);
-    return;
   }
-  const char *conhead = NGHTTP2_CLIENT_CONNECTION_HEADER;
+  if(rv < 0) {
-  if(memcmp(conhead + NGHTTP2_CLIENT_CONNECTION_HEADER_LEN - leftlen,
+    auto sslerr = SSL_get_error(ssl_, rv);
-            data, readlen) != 0) {
+
-    delete_handler(handler);
+    switch(sslerr) {
-    return;
+    case SSL_ERROR_NONE:
-  }
+    case SSL_ERROR_WANT_X509_LOOKUP:
-  leftlen -= readlen;
+    case SSL_ERROR_ZERO_RETURN:
-  handler->set_left_connhd_len(leftlen);
+      break;
-  if(leftlen == 0) {
+    case SSL_ERROR_WANT_READ:
-    bufferevent_setcb(bev, readcb, writecb, eventcb, ptr);
+      event_add(rev_, nullptr);
-    // Run on_read to process data left in buffer since they are not
+      return 1;
-    // notified further
+    case SSL_ERROR_WANT_WRITE:
-    if(handler->on_read() != 0) {
+      event_add(wev_, nullptr);
-      delete_handler(handler);
+      return 1;
-      return;
     }
   }
+
+  if(sessions_->get_config()->verbose) {
+    std::cerr << "SSL/TLS handshake completed" << std::endl;
+  }
+
+  if(verify_npn_result() != 0) {
+    return -1;
+  }
+
+  event_del(rev_);
+  event_del(wev_);
+
+  event_assign(rev_, sessions_->get_evbase(), fd_, EV_READ, rev_cb, this);
+  event_assign(wev_, sessions_->get_evbase(), fd_, EV_WRITE, wev_cb, this);
+
+  return 0;
 }
-} // namespace
 
 int Http2Handler::setup_bev()
 {
   if(ssl_) {
-    bev_ = bufferevent_openssl_socket_new
+    rev_ = event_new(sessions_->get_evbase(), fd_, EV_READ, tls_handshake_cb,
-      (sessions_->get_evbase(), fd_, ssl_,
+                     this);
-       BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_DEFER_CALLBACKS);
+    wev_ = event_new(sessions_->get_evbase(), fd_, EV_WRITE, tls_handshake_cb,
+                     this);
   } else {
-    bev_ = bufferevent_socket_new(sessions_->get_evbase(), fd_,
+    rev_ = event_new(sessions_->get_evbase(), fd_, EV_READ, rev_cb, this);
-                                  BEV_OPT_DEFER_CALLBACKS);
+    wev_ = event_new(sessions_->get_evbase(), fd_, EV_WRITE, wev_cb, this);
   }
-  bufferevent_enable(bev_, EV_READ);
+
-  bufferevent_setcb(bev_, connhd_readcb, writecb, eventcb, this);
+  event_add(rev_, nullptr);
   // TODO set up timeout here
+
   return 0;
 }
 
+int Http2Handler::wait_events()
+{
+  int active = 0;
+
+  if(nghttp2_session_want_read(session_)) {
+    event_add(rev_, nullptr);
+    active = 1;
+  }
+
+  if(nghttp2_session_want_write(session_)) {
+    event_add(wev_, nullptr);
+    active = 1;
+  }
+
+  return active ? 0 : -1;
+}
+
 int Http2Handler::on_read()
 {
   int rv;
-  auto input = bufferevent_get_input(bev_);
+  uint8_t buf[16384];
-  auto inputlen = evbuffer_get_length(input);
+  uint8_t *bufp;
-  auto mem = evbuffer_pullup(input, -1);
+  size_t nread;
 
-  rv = nghttp2_session_mem_recv(session_, mem, inputlen);
+  if(ssl_) {
+    ERR_clear_error();
+    rv = SSL_read(ssl_, buf, sizeof(buf));
+
+    if(rv == 0) {
+      return -1;
+    }
+    if(rv < 0) {
+      return handle_ssl_temporal_error(rv);
+    }
+  } else {
+    while((rv = read(fd_, buf, sizeof(buf))) && rv == -1 && errno == EINTR);
+
+    if(rv == 0) {
+      return -1;
+    }
+    if(rv < 0) {
+      if(errno == EAGAIN || errno == EWOULDBLOCK) {
+        event_add(rev_, nullptr);
+        return 1;
+      }
+      return -1;
+    }
+  }
+
+  nread = rv;
+  bufp = buf;
+
+  if(left_connhd_len_ > 0) {
+    auto len = std::min(left_connhd_len_, nread);
+    const char *conhead = NGHTTP2_CLIENT_CONNECTION_HEADER;
+
+    if(memcmp(conhead + NGHTTP2_CLIENT_CONNECTION_HEADER_LEN -
+              left_connhd_len_, bufp, len) != 0) {
+      return -1;
+    }
+
+    left_connhd_len_ -= len;
+    nread -= len;
+
+    if(nread == 0) {
+      wait_events();
+
+      return 0;
+    }
+
+    bufp += len;
+  }
+
+  rv = nghttp2_session_mem_recv(session_, bufp, nread);
   if(rv < 0) {
     std::cerr << "nghttp2_session_mem_recv() returned error: "
               << nghttp2_strerror(rv) << std::endl;
     return -1;
   }
 
-  evbuffer_drain(input, rv);
+  return wait_events();
-
-  return on_write();
 }
 
 int Http2Handler::on_write()
 {
   int rv;
-  uint8_t buf[16384];
-  auto output = bufferevent_get_output(bev_);
-  util::EvbufferBuffer evbbuf(output, buf, sizeof(buf));
 
-  for(;;) {
+  //std::cerr << "DBG: on_write" << std::endl;
-    if(evbuffer_get_length(output) + evbbuf.get_buflen() >
+
-       sessions_->get_config()->output_upper_thres) {
+  rv = tls_write_pending();
-      break;
+  if(rv != 0) {
+    return rv;
   }
 
+  for(;;) {
     const uint8_t *data;
+
     auto datalen = nghttp2_session_mem_send(session_, &data);
 
     if(datalen < 0) {
@@ -385,26 +589,24 @@ int Http2Handler::on_write()
                 << nghttp2_strerror(datalen) << std::endl;
       return -1;
     }
+
     if(datalen == 0) {
       break;
     }
-    rv = evbbuf.add(data, datalen);
+
+    rv = tls_write(data, datalen);
+
     if(rv != 0) {
-      std::cerr << "evbuffer_add() failed" << std::endl;
+      return rv;
-      return -1;
     }
   }
-  rv = evbbuf.flush();
+
+  rv = tls_write_pending();
   if(rv != 0) {
-    std::cerr << "evbuffer_add() failed" << std::endl;
+    return rv;
-    return -1;
   }
-  if(nghttp2_session_want_read(session_) == 0 &&
+
-     nghttp2_session_want_write(session_) == 0 &&
+  return wait_events();
-     evbuffer_get_length(output) == 0) {
-    return -1;
-  }
-  return 0;
 }
 
 namespace {
@@ -1252,6 +1454,9 @@ int HttpServer::run()
     SSL_CTX_set_mode(ssl_ctx, SSL_MODE_RELEASE_BUFFERS);
     SSL_CTX_set_mode(ssl_ctx, SSL_MODE_ENABLE_PARTIAL_WRITE);
 
+    SSL_CTX_set_cipher_list(ssl_ctx, "HIGH:!aNULL:!MD5");
+
+
     const unsigned char sid_ctx[] = "nghttpd";
     SSL_CTX_set_session_id_context(ssl_ctx, sid_ctx, sizeof(sid_ctx)-1);
     SSL_CTX_set_session_cache_mode(ssl_ctx, SSL_SESS_CACHE_SERVER);
@@ -1310,7 +1515,11 @@ int HttpServer::run()
 #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
   }
 
-  auto evbase = event_base_new();
+  auto evcfg = event_config_new();
+  event_config_set_flag(evcfg, EVENT_BASE_FLAG_NOLOCK);
+
+  auto evbase = event_base_new_with_config(evcfg);
+
   Sessions sessions(evbase, config_, ssl_ctx);
   if(start_listen(evbase, &sessions, config_) != 0) {
     std::cerr << "Could not listen" << std::endl;

src/HttpServer.h

@@ -40,10 +40,19 @@
 #include <openssl/ssl.h>
 
 #include <event2/event.h>
-#include <event2/bufferevent.h>
 
 #include <nghttp2/nghttp2.h>
 
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+#include "nghttp2_buf.h"
+
+#ifdef __cplusplus
+}
+#endif
+
 #include "http2.h"
 
 namespace nghttp2 {
@@ -55,7 +64,6 @@ struct Config {
   std::string private_key_file;
   std::string cert_file;
   void *data_ptr;
-  size_t output_upper_thres;
   size_t padding;
   size_t num_worker;
   ssize_t header_table_size;
@@ -121,16 +129,26 @@ public:
   void set_left_connhd_len(size_t left);
   void remove_settings_timer();
   void terminate_session(nghttp2_error_code error_code);
+  int tls_handshake();
 private:
+  int handle_ssl_temporal_error(int err);
+  int tls_write(const uint8_t *data, size_t datalen);
+  int tls_write_pending();
+  int wait_events();
+
   std::map<int32_t, std::unique_ptr<Request>> id2req_;
+  nghttp2_buf sendbuf_;
   int64_t session_id_;
   nghttp2_session *session_;
   Sessions *sessions_;
-  bufferevent *bev_;
   SSL* ssl_;
+  event *rev_, *wev_;
   event *settings_timerev_;
+  const uint8_t *pending_data_;
+  size_t pending_datalen_;
   size_t left_connhd_len_;
   int fd_;
+  uint8_t sendbufarray_[65536];
 };
 
 class HttpServer {