Revert "nghttpx: Remove last write/read fields for TLS"

This reverts commit 585af93828.

src/shrpx_connection.cc

@@ -153,14 +153,22 @@ void Connection::update_tls_warmup_writelen(size_t n) {
 }
 
 ssize_t Connection::write_tls(const void *data, size_t len) {
-  // We set SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER, so we don't have to
+  // SSL_write requires the same arguments (buf pointer and its
-  // care about parameters after SSL_ERROR_WANT_READ or
+  // length) on SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE.
-  // SSL_ERROR_WANT_WRITE.
+  // get_write_limit() may return smaller length than previously
+  // passed to SSL_write, which violates OpenSSL assumption.  To avoid
+  // this, we keep last legnth passed to SSL_write to
+  // tls.last_writelen if SSL_write indicated I/O blocking.
+  if (tls.last_writelen == 0) {
     len = std::min(len, wlimit.avail());
     len = std::min(len, get_tls_write_limit());
     if (len == 0) {
       return 0;
     }
+  } else {
+    len = tls.last_writelen;
+    tls.last_writelen = 0;
+  }
 
   auto rv = SSL_write(tls.ssl, data, len);
 
@@ -179,6 +187,7 @@ ssize_t Connection::write_tls(const void *data, size_t len) {
       }
       return SHRPX_ERR_NETWORK;
     case SSL_ERROR_WANT_WRITE:
+      tls.last_writelen = len;
       wlimit.startw();
       ev_timer_again(loop, &wt);
       return 0;
@@ -198,14 +207,22 @@ ssize_t Connection::write_tls(const void *data, size_t len) {
 }
 
 ssize_t Connection::read_tls(void *data, size_t len) {
-  // Although SSL_read manual says it requires the same arguments (buf
+  // SSL_read requires the same arguments (buf pointer and its
-  // pointer and its length) on SSL_ERROR_WANT_READ or
+  // length) on SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE.
-  // SSL_ERROR_WANT_WRITE.  But after reading OpenSSL source code,
+  // rlimit_.avail() or rlimit_.avail() may return different length
-  // there is no such requirement.
+  // than the length previously passed to SSL_read, which violates
+  // OpenSSL assumption.  To avoid this, we keep last legnth passed
+  // to SSL_read to tls_last_readlen_ if SSL_read indicated I/O
+  // blocking.
+  if (tls.last_readlen == 0) {
     len = std::min(len, rlimit.avail());
     if (len == 0) {
       return 0;
     }
+  } else {
+    len = tls.last_readlen;
+    tls.last_readlen = 0;
+  }
 
   auto rv = SSL_read(tls.ssl, data, len);
 
@@ -213,6 +230,7 @@ ssize_t Connection::read_tls(void *data, size_t len) {
     auto err = SSL_get_error(tls.ssl, rv);
     switch (err) {
     case SSL_ERROR_WANT_READ:
+      tls.last_readlen = len;
       return 0;
     case SSL_ERROR_WANT_WRITE:
       if (LOG_ENABLED(INFO)) {

src/shrpx_connection.h

@@ -42,6 +42,10 @@ struct TLSConnection {
   SSL *ssl;
   ev_tstamp last_write_time;
   size_t warmup_writelen;
+  // length passed to SSL_write and SSL_read last time.  This is
+  // required since these functions require the exact same parameters
+  // on non-blocking I/O.
+  size_t last_writelen, last_readlen;
   bool initial_handshake_done;
   bool reneg_started;
 };

src/shrpx_ssl.cc

@@ -396,7 +396,6 @@ SSL_CTX *create_ssl_context(const char *private_key_file,
 
   SSL_CTX_set_mode(ssl_ctx, SSL_MODE_AUTO_RETRY);
   SSL_CTX_set_mode(ssl_ctx, SSL_MODE_RELEASE_BUFFERS);
-  SSL_CTX_set_mode(ssl_ctx, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
   if (get_config()->private_key_passwd) {
     SSL_CTX_set_default_passwd_cb(ssl_ctx, ssl_pem_passwd_cb);
     SSL_CTX_set_default_passwd_cb_userdata(ssl_ctx, (void *)get_config());
@@ -505,7 +504,6 @@ SSL_CTX *create_ssl_client_context() {
 
   SSL_CTX_set_mode(ssl_ctx, SSL_MODE_AUTO_RETRY);
   SSL_CTX_set_mode(ssl_ctx, SSL_MODE_RELEASE_BUFFERS);
-  SSL_CTX_set_mode(ssl_ctx, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
 
   if (SSL_CTX_set_default_verify_paths(ssl_ctx) != 1) {
     LOG(WARN) << "Could not load system trusted ca certificates: "