diff --git a/src/hb-subset-plan.cc b/src/hb-subset-plan.cc
index bde77d557..eb88657e1 100644
--- a/src/hb-subset-plan.cc
+++ b/src/hb-subset-plan.cc
@@ -524,14 +524,39 @@ _populate_unicodes_to_retain (const hb_set_t *unicodes,
       cmap_unicodes = &plan->accelerator->unicodes;
     }
 
-    for (hb_codepoint_t cp : *cmap_unicodes)
+    if (plan->accelerator &&
+	unicodes->get_population () < cmap_unicodes->get_population () &&
+	glyphs->get_population () < cmap_unicodes->get_population ())
     {
-      hb_codepoint_t gid = (*unicode_glyphid_map)[cp];
-      if (!unicodes->has (cp) && !glyphs->has (gid))
-        continue;
+      auto &gid_to_unicodes = plan->accelerator->gid_to_unicodes;
+      for (hb_codepoint_t gid : *glyphs)
+      {
+        auto unicodes = gid_to_unicodes.get (gid);
 
-      plan->codepoint_to_glyph->set (cp, gid);
-      plan->unicode_to_new_gid_list.push (hb_pair (cp, gid));
+	for (hb_codepoint_t cp : unicodes)
+	{
+	  plan->codepoint_to_glyph->set (cp, gid);
+	  plan->unicode_to_new_gid_list.push (hb_pair (cp, gid));
+	}
+      }
+      for (hb_codepoint_t cp : *unicodes)
+      {
+	hb_codepoint_t gid = (*unicode_glyphid_map)[cp];
+	plan->codepoint_to_glyph->set (cp, gid);
+	plan->unicode_to_new_gid_list.push (hb_pair (cp, gid));
+      }
+    }
+    else
+    {
+      for (hb_codepoint_t cp : *cmap_unicodes)
+      {
+	hb_codepoint_t gid = (*unicode_glyphid_map)[cp];
+	if (!unicodes->has (cp) && !glyphs->has (gid))
+	  continue;
+
+	plan->codepoint_to_glyph->set (cp, gid);
+	plan->unicode_to_new_gid_list.push (hb_pair (cp, gid));
+      }
     }
 
     /* Add gids which where requested, but not mapped in cmap */