diff --git a/src/shrpx_downstream_queue.cc b/src/shrpx_downstream_queue.cc
index 91e88cff..f7f66b7d 100644
--- a/src/shrpx_downstream_queue.cc
+++ b/src/shrpx_downstream_queue.cc
@@ -31,7 +31,8 @@
 
 namespace shrpx {
 
-DownstreamQueue::HostEntry::HostEntry() : num_active(0) {}
+DownstreamQueue::HostEntry::HostEntry(ImmutableString &&key)
+    : key(std::move(key)), num_active(0) {}
 
 DownstreamQueue::DownstreamQueue(size_t conn_max_per_host, bool unified_host)
     : conn_max_per_host_(conn_max_per_host == 0
@@ -57,25 +58,28 @@ void DownstreamQueue::mark_failure(Downstream *downstream) {
 }
 
 DownstreamQueue::HostEntry &
-DownstreamQueue::find_host_entry(const std::string &host) {
+DownstreamQueue::find_host_entry(const StringRef &host) {
   auto itr = host_entries_.find(host);
   if (itr == std::end(host_entries_)) {
+    auto key = ImmutableString{std::begin(host), std::end(host)};
+    auto key_ref = StringRef{key};
 #ifdef HAVE_STD_MAP_EMPLACE
-    std::tie(itr, std::ignore) = host_entries_.emplace(host, HostEntry());
+    std::tie(itr, std::ignore) =
+        host_entries_.emplace(key_ref, HostEntry(std::move(key)));
 #else  // !HAVE_STD_MAP_EMPLACE
     // for g++-4.7
-    std::tie(itr, std::ignore) =
-        host_entries_.insert(std::make_pair(host, HostEntry()));
+    std::tie(itr, std::ignore) = host_entries_.insert(
+        std::make_pair(key_ref, HostEntry(std::move(key))));
 #endif // !HAVE_STD_MAP_EMPLACE
   }
   return (*itr).second;
 }
 
-std::string DownstreamQueue::make_host_key(const StringRef &host) const {
-  return unified_host_ ? "" : host.str();
+StringRef DownstreamQueue::make_host_key(const StringRef &host) const {
+  return unified_host_ ? StringRef{} : host;
 }
 
-std::string DownstreamQueue::make_host_key(Downstream *downstream) const {
+StringRef DownstreamQueue::make_host_key(Downstream *downstream) const {
   return make_host_key(downstream->request().authority);
 }
 
@@ -108,7 +112,7 @@ bool DownstreamQueue::can_activate(const StringRef &host) const {
 namespace {
 bool remove_host_entry_if_empty(const DownstreamQueue::HostEntry &ent,
                                 DownstreamQueue::HostEntryMap &host_entries,
-                                const std::string &host) {
+                                const StringRef &host) {
   if (ent.blocked.empty() && ent.num_active == 0) {
     host_entries.erase(host);
     return true;
diff --git a/src/shrpx_downstream_queue.h b/src/shrpx_downstream_queue.h
index 47e8555a..1a750343 100644
--- a/src/shrpx_downstream_queue.h
+++ b/src/shrpx_downstream_queue.h
@@ -51,14 +51,23 @@ struct BlockedLink {
 class DownstreamQueue {
 public:
   struct HostEntry {
+    HostEntry(ImmutableString &&key);
+
+    HostEntry(HostEntry &&) = default;
+    HostEntry &operator=(HostEntry &&) = default;
+
+    HostEntry(const HostEntry &) = delete;
+    HostEntry &operator=(const HostEntry &) = delete;
+
+    // Key that associates this object
+    ImmutableString key;
     // Set of stream ID that blocked by conn_max_per_host_.
     DList<BlockedLink> blocked;
     // The number of connections currently made to this host.
     size_t num_active;
-    HostEntry();
   };
 
-  using HostEntryMap = std::map<std::string, HostEntry>;
+  using HostEntryMap = std::map<StringRef, HostEntry, std::less<StringRef>>;
 
   // conn_max_per_host == 0 means no limit for downstream connection.
   DownstreamQueue(size_t conn_max_per_host = 0, bool unified_host = true);
@@ -86,14 +95,14 @@ public:
   Downstream *remove_and_get_blocked(Downstream *downstream,
                                      bool next_blocked = true);
   Downstream *get_downstreams() const;
-  HostEntry &find_host_entry(const std::string &host);
-  std::string make_host_key(const StringRef &host) const;
-  std::string make_host_key(Downstream *downstream) const;
+  HostEntry &find_host_entry(const StringRef &host);
+  StringRef make_host_key(const StringRef &host) const;
+  StringRef make_host_key(Downstream *downstream) const;
 
 private:
   // Per target host structure to keep track of the number of
   // connections to the same host.
-  std::map<std::string, HostEntry> host_entries_;
+  HostEntryMap host_entries_;
   DList<Downstream> downstreams_;
   // Maximum number of concurrent connections to the same host.
   size_t conn_max_per_host_;