diff --git a/src/shrpx.cc b/src/shrpx.cc
index e94c9390..4df44a5f 100644
--- a/src/shrpx.cc
+++ b/src/shrpx.cc
@@ -2921,7 +2921,8 @@ SSL/TLS:
               accepts.
               Default: )"
       << util::utos_unit(config->tls.max_early_data) << R"(
-  --tls-ktls  Enable ktls.
+  --tls-ktls  Enable   ktls.    For   server,  ktls   is   enable   if
+              --tls-session-cache-memcached is not configured.
 
 HTTP/2:
   -c, --frontend-http2-max-concurrent-streams=<N>
diff --git a/src/shrpx_connection.cc b/src/shrpx_connection.cc
index f0669638..e8f5192e 100644
--- a/src/shrpx_connection.cc
+++ b/src/shrpx_connection.cc
@@ -150,6 +150,13 @@ void Connection::prepare_client_handshake() {
 }
 
 void Connection::prepare_server_handshake() {
+  auto &tlsconf = get_config()->tls;
+  if (proto != Proto::HTTP3 && !tlsconf.session_cache.memcached.host.empty()) {
+    auto bio = BIO_new(tlsconf.bio_method);
+    BIO_set_data(bio, this);
+    SSL_set_bio(tls.ssl, bio, bio);
+  }
+
   SSL_set_accept_state(tls.ssl);
   tls.server_handshake = true;
 }
@@ -312,13 +319,6 @@ BIO_METHOD *create_bio_method() {
 void Connection::set_ssl(SSL *ssl) {
   tls.ssl = ssl;
 
-  auto &tlsconf = get_config()->tls;
-  if (proto != Proto::HTTP3 && !tlsconf.session_cache.memcached.host.empty()) {
-    auto bio = BIO_new(tlsconf.bio_method);
-    BIO_set_data(bio, this);
-    SSL_set_bio(tls.ssl, bio, bio);
-  }
-
   SSL_set_app_data(tls.ssl, this);
 }
 
@@ -338,7 +338,7 @@ int Connection::tls_handshake() {
 
   auto &tlsconf = get_config()->tls;
 
-  if (tlsconf.session_cache.memcached.host.empty()) {
+  if (!tls.server_handshake || tlsconf.session_cache.memcached.host.empty()) {
     return tls_handshake_simple();
   }
 
@@ -387,7 +387,7 @@ int Connection::tls_handshake() {
 
     set_ssl(ssl);
 
-    SSL_set_accept_state(tls.ssl);
+    prepare_server_handshake();
 
     tls.handshake_state = TLSHandshakeState::NORMAL;
     break;