diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc
index 52c5c94d..51232b1f 100644
--- a/src/shrpx_connection_handler.cc
+++ b/src/shrpx_connection_handler.cc
@@ -165,7 +165,11 @@ int ConnectionHandler::create_single_worker() {
                                                   nb_.get()
 #endif // HAVE_NEVERBLEED
                                                   );
-  auto cl_ssl_ctx = ssl::setup_client_ssl_context();
+  auto cl_ssl_ctx = ssl::setup_client_ssl_context(
+#ifdef HAVE_NEVERBLEED
+      nb_.get()
+#endif // HAVE_NEVERBLEED
+      );
 
   if (cl_ssl_ctx) {
     all_ssl_ctx_.push_back(cl_ssl_ctx);
@@ -193,7 +197,11 @@ int ConnectionHandler::create_worker_thread(size_t num) {
                                                   nb_.get()
 #endif // HAVE_NEVERBLEED
                                                   );
-  auto cl_ssl_ctx = ssl::setup_client_ssl_context();
+  auto cl_ssl_ctx = ssl::setup_client_ssl_context(
+#ifdef HAVE_NEVERBLEED
+      nb_.get()
+#endif // HAVE_NEVERBLEED
+      );
 
   if (cl_ssl_ctx) {
     all_ssl_ctx_.push_back(cl_ssl_ctx);
diff --git a/src/shrpx_ssl.cc b/src/shrpx_ssl.cc
index 6b517b84..a37b5199 100644
--- a/src/shrpx_ssl.cc
+++ b/src/shrpx_ssl.cc
@@ -635,7 +635,11 @@ int select_next_proto_cb(SSL *ssl, unsigned char **out, unsigned char *outlen,
 }
 } // namespace
 
-SSL_CTX *create_ssl_client_context() {
+SSL_CTX *create_ssl_client_context(
+#ifdef HAVE_NEVERBLEED
+    neverbleed_t *nb
+#endif // HAVE_NEVERBLEED
+    ) {
   auto ssl_ctx = SSL_CTX_new(SSLv23_client_method());
   if (!ssl_ctx) {
     LOG(FATAL) << ERR_error_string(ERR_get_error(), nullptr);
@@ -681,6 +685,7 @@ SSL_CTX *create_ssl_client_context() {
   }
 
   if (get_config()->client_private_key_file) {
+#ifndef HAVE_NEVERBLEED
     if (SSL_CTX_use_PrivateKey_file(ssl_ctx,
                                     get_config()->client_private_key_file.get(),
                                     SSL_FILETYPE_PEM) != 1) {
@@ -689,6 +694,16 @@ SSL_CTX *create_ssl_client_context() {
                  << ERR_error_string(ERR_get_error(), nullptr);
       DIE();
     }
+#else  // HAVE_NEVERBLEED
+    std::array<char, NEVERBLEED_ERRBUF_SIZE> errbuf;
+    if (neverbleed_load_private_key_file(
+            nb, ssl_ctx, get_config()->client_private_key_file.get(),
+            errbuf.data()) != 1) {
+      LOG(FATAL) << "neverbleed_load_private_key_file failed: "
+                 << errbuf.data();
+      DIE();
+    }
+#endif // HAVE_NEVERBLEED
   }
   if (get_config()->client_cert_file) {
     if (SSL_CTX_use_certificate_chain_file(
@@ -1165,15 +1180,28 @@ SSL_CTX *setup_server_ssl_context(std::vector<SSL_CTX *> &all_ssl_ctx,
   return ssl_ctx;
 }
 
-SSL_CTX *setup_client_ssl_context() {
+bool downstream_tls_enabled() {
   if (get_config()->client_mode) {
-    return get_config()->downstream_no_tls ? nullptr
-                                           : ssl::create_ssl_client_context();
+    return !get_config()->downstream_no_tls;
   }
 
-  return get_config()->http2_bridge && !get_config()->downstream_no_tls
-             ? ssl::create_ssl_client_context()
-             : nullptr;
+  return get_config()->http2_bridge && !get_config()->downstream_no_tls;
+}
+
+SSL_CTX *setup_client_ssl_context(
+#ifdef HAVE_NEVERBLEED
+    neverbleed_t *nb
+#endif // HAVE_NEVERBLEED
+    ) {
+  if (!downstream_tls_enabled()) {
+    return nullptr;
+  }
+
+  return ssl::create_ssl_client_context(
+#ifdef HAVE_NEVERBLEED
+      nb
+#endif // HAVE_NEVERBLEED
+      );
 }
 
 CertLookupTree *create_cert_lookup_tree() {
diff --git a/src/shrpx_ssl.h b/src/shrpx_ssl.h
index 2a722c07..ffc7bab6 100644
--- a/src/shrpx_ssl.h
+++ b/src/shrpx_ssl.h
@@ -69,7 +69,11 @@ SSL_CTX *create_ssl_context(const char *private_key_file, const char *cert_file
                             );
 
 // Create client side SSL_CTX
-SSL_CTX *create_ssl_client_context();
+SSL_CTX *create_ssl_client_context(
+#ifdef HAVE_NEVERBLEED
+    neverbleed_t *nb
+#endif // HAVE_NEVERBLEED
+    );
 
 ClientHandler *accept_connection(Worker *worker, int fd, sockaddr *addr,
                                  int addrlen);
@@ -179,7 +183,11 @@ SSL_CTX *setup_server_ssl_context(std::vector<SSL_CTX *> &all_ssl_ctx,
 // Setups client side SSL_CTX.  This function inspects get_config()
 // and if downstream_no_tls is true, returns nullptr.  Otherwise, only
 // construct SSL_CTX if either client_mode or http2_bridge is true.
-SSL_CTX *setup_client_ssl_context();
+SSL_CTX *setup_client_ssl_context(
+#ifdef HAVE_NEVERBLEED
+    neverbleed_t *nb
+#endif // HAVE_NEVERBLEED
+    );
 
 // Creates CertLookupTree.  If frontend is configured not to use TLS,
 // this function returns nullptr.
@@ -187,6 +195,9 @@ CertLookupTree *create_cert_lookup_tree();
 
 SSL *create_ssl(SSL_CTX *ssl_ctx);
 
+// Returns true if SSL/TLS is enabled on downstream
+bool downstream_tls_enabled();
+
 } // namespace ssl
 
 } // namespace shrpx
diff --git a/src/shrpx_worker_process.cc b/src/shrpx_worker_process.cc
index f37ced59..9887b63d 100644
--- a/src/shrpx_worker_process.cc
+++ b/src/shrpx_worker_process.cc
@@ -48,6 +48,7 @@
 #include "shrpx_memcached_dispatcher.h"
 #include "shrpx_memcached_request.h"
 #include "shrpx_process.h"
+#include "shrpx_ssl.h"
 #include "util.h"
 #include "app_helper.h"
 #include "template.h"
@@ -84,7 +85,9 @@ void drop_privileges(
       exit(EXIT_FAILURE);
     }
 #ifdef HAVE_NEVERBLEED
-    neverbleed_setuidgid(nb, get_config()->user.get(), 1);
+    if (nb) {
+      neverbleed_setuidgid(nb, get_config()->user.get(), 1);
+    }
 #endif // HAVE_NEVERBLEED
   }
 }
@@ -400,8 +403,8 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) {
   }
 
 #ifdef HAVE_NEVERBLEED
-  std::array<char, NEVERBLEED_ERRBUF_SIZE> errbuf;
-  {
+  if (!get_config()->upstream_no_tls || ssl::downstream_tls_enabled()) {
+    std::array<char, NEVERBLEED_ERRBUF_SIZE> errbuf;
     auto nb = make_unique<neverbleed_t>();
     if (neverbleed_init(nb.get(), errbuf.data()) != 0) {
       LOG(FATAL) << "neverbleed_init failed: " << errbuf.data();
@@ -416,9 +419,11 @@ int worker_process_event_loop(WorkerProcessConfig *wpconf) {
   auto nb = conn_handler.get_neverbleed();
 
   ev_child nb_childev;
-  ev_child_init(&nb_childev, nb_child_cb, nb->daemon_pid, 0);
-  nb_childev.data = nullptr;
-  ev_child_start(loop, &nb_childev);
+  if (nb) {
+    ev_child_init(&nb_childev, nb_child_cb, nb->daemon_pid, 0);
+    nb_childev.data = nullptr;
+    ev_child_start(loop, &nb_childev);
+  }
 
 #endif // HAVE_NEVERBLEED