diff --git a/src/shrpx.cc b/src/shrpx.cc
index 78ef768e..cf913390 100644
--- a/src/shrpx.cc
+++ b/src/shrpx.cc
@@ -1554,14 +1554,6 @@ void fill_default_config(Config *config) {
   {
     quicconf.timeout.idle = 30_s;
 
-    auto &stateless_resetconf = quicconf.stateless_reset;
-    // TODO Find better place to do this and error handling.
-    if (RAND_bytes(stateless_resetconf.secret.data(),
-                   stateless_resetconf.secret.size()) != 1) {
-      LOG(FATAL) << "Unable to generate stateless reset secret";
-      exit(EXIT_FAILURE);
-    }
-
     auto &bpfconf = quicconf.bpf;
     bpfconf.prog_file = StringRef::from_lit(PKGLIBDIR "/reuseport_kern.o");
   }
diff --git a/src/shrpx_config.h b/src/shrpx_config.h
index 789b9582..d6466fcc 100644
--- a/src/shrpx_config.h
+++ b/src/shrpx_config.h
@@ -51,6 +51,9 @@
 #include <nghttp2/nghttp2.h>
 
 #include "shrpx_router.h"
+#if ENABLE_HTTP3
+#  include "shrpx_quic.h"
+#endif // ENABLE_HTTP3
 #include "template.h"
 #include "http2.h"
 #include "network.h"
@@ -572,6 +575,14 @@ struct TLSCertificate {
   std::vector<uint8_t> sct_data;
 };
 
+#ifdef ENABLE_HTTP3
+struct QUICSecret {
+  std::array<uint8_t, SHRPX_QUIC_STATELESS_RESET_SECRETLEN>
+      stateless_reset_secret;
+  std::array<uint8_t, SHRPX_QUIC_TOKEN_SECRETLEN> token_secret;
+};
+#endif // ENABLE_HTTP3
+
 struct HttpProxy {
   Address addr;
   // host in http proxy URI
@@ -711,9 +722,6 @@ struct TLSConfig {
 
 #ifdef ENABLE_HTTP3
 struct QUICConfig {
-  struct {
-    std::array<uint8_t, 32> secret;
-  } stateless_reset;
   struct {
     ev_tstamp idle;
   } timeout;
diff --git a/src/shrpx_connection_handler.cc b/src/shrpx_connection_handler.cc
index a1f2166f..14e20253 100644
--- a/src/shrpx_connection_handler.cc
+++ b/src/shrpx_connection_handler.cc
@@ -291,6 +291,8 @@ int ConnectionHandler::create_single_worker() {
 #endif // HAVE_MRUBY
 
 #ifdef ENABLE_HTTP3
+  single_worker_->set_quic_secret(quic_secret_);
+
   if (single_worker_->setup_quic_server_socket() != 0) {
     return -1;
   }
@@ -394,6 +396,8 @@ int ConnectionHandler::create_worker_thread(size_t num) {
 #  endif // HAVE_MRUBY
 
 #  ifdef ENABLE_HTTP3
+    worker->set_quic_secret(quic_secret_);
+
     if ((!apiconf.enabled || i != 0) &&
         worker->setup_quic_server_socket() != 0) {
       return -1;
@@ -1035,6 +1039,27 @@ int ConnectionHandler::forward_quic_packet(const UpstreamAddr *faddr,
   return -1;
 }
 
+int ConnectionHandler::create_quic_secret() {
+  auto quic_secret = std::make_shared<QUICSecret>();
+
+  if (generate_quic_stateless_reset_secret(
+          quic_secret->stateless_reset_secret.data()) != 0) {
+    LOG(ERROR) << "Failed to generate QUIC Stateless Reset secret";
+
+    return -1;
+  }
+
+  if (generate_quic_token_secret(quic_secret->token_secret.data()) != 0) {
+    LOG(ERROR) << "Failed to generate QUIC token secret";
+
+    return -1;
+  }
+
+  quic_secret_ = std::move(quic_secret);
+
+  return 0;
+}
+
 #  ifdef HAVE_LIBBPF
 std::vector<BPFRef> &ConnectionHandler::get_quic_bpf_refs() {
   return quic_bpf_refs_;
diff --git a/src/shrpx_connection_handler.h b/src/shrpx_connection_handler.h
index 7816c162..7ffc2112 100644
--- a/src/shrpx_connection_handler.h
+++ b/src/shrpx_connection_handler.h
@@ -177,6 +177,8 @@ public:
                           const Address &local_addr, const uint8_t *cid_prefix,
                           const uint8_t *data, size_t datalen);
 
+  int create_quic_secret();
+
 #  ifdef HAVE_LIBBPF
   std::vector<BPFRef> &get_quic_bpf_refs();
 #  endif // HAVE_LIBBPF
@@ -213,6 +215,7 @@ private:
 #  ifdef HAVE_LIBBPF
   std::vector<BPFRef> quic_bpf_refs_;
 #  endif // HAVE_LIBBPF
+  std::shared_ptr<QUICSecret> quic_secret_;
   std::vector<SSL_CTX *> quic_all_ssl_ctx_;
   std::vector<std::vector<SSL_CTX *>> quic_indexed_ssl_ctx_;
 #endif // ENABLE_HTTP3
diff --git a/src/shrpx_http3_upstream.cc b/src/shrpx_http3_upstream.cc
index dab44029..ca2f2930 100644
--- a/src/shrpx_http3_upstream.cc
+++ b/src/shrpx_http3_upstream.cc
@@ -168,9 +168,8 @@ int get_new_connection_id(ngtcp2_conn *conn, ngtcp2_cid *cid, uint8_t *token,
     return NGTCP2_ERR_CALLBACK_FAILURE;
   }
 
-  auto config = get_config();
-  auto &quicconf = config->quic;
-  auto &secret = quicconf.stateless_reset.secret;
+  auto &quic_secret = worker->get_quic_secret();
+  auto &secret = quic_secret->stateless_reset_secret;
 
   if (generate_quic_stateless_reset_token(token, cid, secret.data(),
                                           secret.size()) != 0) {
@@ -483,11 +482,12 @@ int Http3Upstream::init(const UpstreamAddr *faddr, const Address &remote_addr,
       static_cast<ngtcp2_tstamp>(quicconf.timeout.idle * NGTCP2_SECONDS);
   params.original_dcid = initial_hd.dcid;
 
-  auto &stateless_resetconf = quicconf.stateless_reset;
+  auto &quic_secret = worker->get_quic_secret();
+  auto &stateless_reset_secret = quic_secret->stateless_reset_secret;
 
   rv = generate_quic_stateless_reset_token(params.stateless_reset_token, &scid,
-                                           stateless_resetconf.secret.data(),
-                                           stateless_resetconf.secret.size());
+                                           stateless_reset_secret.data(),
+                                           stateless_reset_secret.size());
   if (rv != 0) {
     LOG(ERROR) << "generate_quic_stateless_reset_token failed";
     return -1;
diff --git a/src/shrpx_quic.cc b/src/shrpx_quic.cc
index b99e2dc7..670d20ff 100644
--- a/src/shrpx_quic.cc
+++ b/src/shrpx_quic.cc
@@ -180,4 +180,12 @@ int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid,
   return 0;
 }
 
+int generate_quic_stateless_reset_secret(uint8_t *secret) {
+  return RAND_bytes(secret, SHRPX_QUIC_STATELESS_RESET_SECRETLEN) == 1;
+}
+
+int generate_quic_token_secret(uint8_t *secret) {
+  return RAND_bytes(secret, SHRPX_QUIC_TOKEN_SECRETLEN) == 1;
+}
+
 } // namespace shrpx
diff --git a/src/shrpx_quic.h b/src/shrpx_quic.h
index da57720d..39aecfc5 100644
--- a/src/shrpx_quic.h
+++ b/src/shrpx_quic.h
@@ -38,6 +38,8 @@ struct UpstreamAddr;
 constexpr size_t SHRPX_QUIC_SCIDLEN = 20;
 constexpr size_t SHRPX_QUIC_CID_PREFIXLEN = 8;
 constexpr size_t SHRPX_MAX_UDP_PAYLOAD_SIZE = 1280;
+constexpr size_t SHRPX_QUIC_STATELESS_RESET_SECRETLEN = 32;
+constexpr size_t SHRPX_QUIC_TOKEN_SECRETLEN = 32;
 
 ngtcp2_tstamp quic_timestamp();
 
@@ -55,6 +57,10 @@ int generate_quic_stateless_reset_token(uint8_t *token, const ngtcp2_cid *cid,
                                         const uint8_t *secret,
                                         size_t secretlen);
 
+int generate_quic_stateless_reset_secret(uint8_t *secret);
+
+int generate_quic_token_secret(uint8_t *secret);
+
 } // namespace shrpx
 
 #endif // SHRPX_QUIC_H
diff --git a/src/shrpx_quic_connection_handler.cc b/src/shrpx_quic_connection_handler.cc
index 76681b10..e8f84cc4 100644
--- a/src/shrpx_quic_connection_handler.cc
+++ b/src/shrpx_quic_connection_handler.cc
@@ -254,13 +254,11 @@ int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr,
 
   ngtcp2_cid_init(&cid, dcid, dcidlen);
 
-  auto config = get_config();
-  auto &quicconf = config->quic;
-  auto &stateless_resetconf = quicconf.stateless_reset;
+  auto &quic_secret = worker_->get_quic_secret();
+  auto &secret = quic_secret->stateless_reset_secret;
 
-  rv = generate_quic_stateless_reset_token(token.data(), &cid,
-                                           stateless_resetconf.secret.data(),
-                                           stateless_resetconf.secret.size());
+  rv = generate_quic_stateless_reset_token(token.data(), &cid, secret.data(),
+                                           secret.size());
   if (rv != 0) {
     return -1;
   }
diff --git a/src/shrpx_worker.cc b/src/shrpx_worker.cc
index 02989ea7..0a97674b 100644
--- a/src/shrpx_worker.cc
+++ b/src/shrpx_worker.cc
@@ -954,6 +954,14 @@ int Worker::create_quic_server_socket(UpstreamAddr &faddr) {
 }
 
 const uint8_t *Worker::get_cid_prefix() const { return cid_prefix_.data(); }
+
+void Worker::set_quic_secret(const std::shared_ptr<QUICSecret> &secret) {
+  quic_secret_ = secret;
+}
+
+const std::shared_ptr<QUICSecret> &Worker::get_quic_secret() const {
+  return quic_secret_;
+}
 #endif // ENABLE_HTTP3
 
 namespace {
diff --git a/src/shrpx_worker.h b/src/shrpx_worker.h
index 6af05446..52a75c94 100644
--- a/src/shrpx_worker.h
+++ b/src/shrpx_worker.h
@@ -367,6 +367,10 @@ public:
 
   const uint8_t *get_cid_prefix() const;
 
+  void set_quic_secret(const std::shared_ptr<QUICSecret> &secret);
+
+  const std::shared_ptr<QUICSecret> &get_quic_secret() const;
+
 #  ifdef HAVE_LIBBPF
   bool should_attach_bpf() const;
 
@@ -402,6 +406,7 @@ private:
   std::array<uint8_t, SHRPX_QUIC_CID_PREFIXLEN> cid_prefix_;
   std::vector<UpstreamAddr> quic_upstream_addrs_;
   std::vector<std::unique_ptr<QUICListener>> quic_listeners_;
+  std::shared_ptr<QUICSecret> quic_secret_;
 #endif // ENABLE_HTTP3
 
   std::shared_ptr<DownstreamConfig> downstreamconf_;