diff --git a/src/shrpx_http2_session.cc b/src/shrpx_http2_session.cc
index 66b9595b..05bf62f9 100644
--- a/src/shrpx_http2_session.cc
+++ b/src/shrpx_http2_session.cc
@@ -208,9 +208,9 @@ int Http2Session::disconnect(bool hard) {
   // this object.  In order to achieve this, we first swap dconns_ and
   // streams_.  Upstream::on_downstream_reset() may add
   // Http2DownstreamConnection.
-  std::set<Http2DownstreamConnection *> dconns;
+  std::unordered_set<Http2DownstreamConnection *> dconns;
   dconns.swap(dconns_);
-  std::set<StreamData *> streams;
+  std::unordered_set<StreamData *> streams;
   streams.swap(streams_);
 
   std::set<ClientHandler *> handlers;
diff --git a/src/shrpx_http2_session.h b/src/shrpx_http2_session.h
index 10d056a6..e9ac37d2 100644
--- a/src/shrpx_http2_session.h
+++ b/src/shrpx_http2_session.h
@@ -27,7 +27,7 @@
 
 #include "shrpx.h"
 
-#include <set>
+#include <unordered_set>
 #include <memory>
 
 #include <openssl/ssl.h>
@@ -188,8 +188,8 @@ private:
   // connection check has started, this timer is started again and
   // traps PING ACK timeout.
   ev_timer connchk_timer_;
-  std::set<Http2DownstreamConnection *> dconns_;
-  std::set<StreamData *> streams_;
+  std::unordered_set<Http2DownstreamConnection *> dconns_;
+  std::unordered_set<StreamData *> streams_;
   std::function<int(Http2Session &)> read_, write_;
   std::function<int(Http2Session &)> on_read_, on_write_;
   // Used to parse the response from HTTP proxy