diff --git a/src/hb-ot-color-cpal-table.hh b/src/hb-ot-color-cpal-table.hh
index a9deeba9a..24476eda1 100644
--- a/src/hb-ot-color-cpal-table.hh
+++ b/src/hb-ot-color-cpal-table.hh
@@ -197,30 +197,38 @@ struct CPAL
 
   public:
   bool serialize (hb_serialize_context_t *c,
-                  const hb_array_t<const BGRAColor> &color_records,
                   const hb_array_t<const HBUINT16> &color_record_indices,
-                  const hb_map_t &color_record_index_map,
-                  const hb_set_t &retained_color_record_indices) const
+                  const hb_array_t<const BGRAColor> &color_records,
+                  const hb_vector_t<unsigned>& first_color_index_for_layer,
+                  const hb_map_t& first_color_to_layer_index,
+                  const hb_set_t &retained_color_indices) const
   {
     TRACE_SERIALIZE (this);
 
+    // TODO(grieger): limit total final size.
+
     for (const auto idx : color_record_indices)
     {
+      hb_codepoint_t layer_index = first_color_to_layer_index[idx];
+
       HBUINT16 new_idx;
-      if (idx == 0) new_idx = 0;
-      else new_idx = color_record_index_map.get (idx);
+      new_idx = layer_index * retained_color_indices.get_population ();
       if (!c->copy<HBUINT16> (new_idx)) return_trace (false);
     }
 
     c->push ();
-    for (const auto _ : retained_color_record_indices.iter ())
+    for (unsigned first_color_index : first_color_index_for_layer)
     {
-      if (!c->copy<BGRAColor> (color_records[_]))
+      for (hb_codepoint_t color_index : retained_color_indices)
       {
-        c->pop_discard ();
-        return_trace (false);
+        if (!c->copy<BGRAColor> (color_records[first_color_index + color_index]))
+        {
+          c->pop_discard ();
+          return_trace (false);
+        }
       }
     }
+
     c->add_link (colorRecordsZ, c->pop_pack ());
     return_trace (true);
   }
@@ -228,6 +236,8 @@ struct CPAL
   bool subset (hb_subset_context_t *c) const
   {
     TRACE_SUBSET (this);
+    if (!numPalettes) return_trace (false);
+
     const hb_map_t *color_index_map = c->plan->colr_palettes;
     if (color_index_map->is_empty ()) return_trace (false);
 
@@ -242,30 +252,34 @@ struct CPAL
     auto *out = c->serializer->start_embed (*this);
     if (unlikely (!c->serializer->extend_min (out))) return_trace (false);
 
+
     out->version = version;
     out->numColors = retained_color_indices.get_population ();
     out->numPalettes = numPalettes;
 
-    const hb_array_t<const HBUINT16> colorRecordIndices = colorRecordIndicesZ.as_array (numPalettes);
-    hb_map_t color_record_index_map;
-    hb_set_t retained_color_record_indices;
+    hb_vector_t<unsigned> first_color_index_for_layer;
+    hb_map_t first_color_to_layer_index;
 
-    unsigned record_count = 0;
+    const hb_array_t<const HBUINT16> colorRecordIndices = colorRecordIndicesZ.as_array (numPalettes);
     for (const auto first_color_record_idx : colorRecordIndices)
     {
-      for (unsigned retained_color_idx : retained_color_indices.iter ())
-      {
-        unsigned color_record_idx = first_color_record_idx + retained_color_idx;
-        if (color_record_index_map.has (color_record_idx)) continue;
-        color_record_index_map.set (color_record_idx, record_count);
-        retained_color_record_indices.add (color_record_idx);
-        record_count++;
-      }
+      if (first_color_to_layer_index.has (first_color_record_idx)) continue;
+
+      first_color_index_for_layer.push (first_color_record_idx);
+      first_color_to_layer_index.set (first_color_record_idx,
+                                      first_color_index_for_layer.length - 1);
     }
 
-    out->numColorRecords = record_count;
+    out->numColorRecords = first_color_index_for_layer.length
+                           * retained_color_indices.get_population ();
+
     const hb_array_t<const BGRAColor> color_records = (this+colorRecordsZ).as_array (numColorRecords);
-    if (!out->serialize (c->serializer, color_records, colorRecordIndices, color_record_index_map, retained_color_record_indices))
+    if (!out->serialize (c->serializer,
+                         colorRecordIndices,
+                         color_records,
+                         first_color_index_for_layer,
+                         first_color_to_layer_index,
+                         retained_color_indices))
       return_trace (false);
 
     if (version == 1)
diff --git a/test/fuzzing/fonts/clusterfuzz-testcase-minimized-hb-subset-fuzzer-5568200165687296 b/test/fuzzing/fonts/clusterfuzz-testcase-minimized-hb-subset-fuzzer-5568200165687296
new file mode 100644
index 000000000..b2038a601
Binary files /dev/null and b/test/fuzzing/fonts/clusterfuzz-testcase-minimized-hb-subset-fuzzer-5568200165687296 differ
diff --git a/test/subset/data/expected/colr_with_components/colr-table.default.6B.ttf b/test/subset/data/expected/colr_with_components/colr-table.default.6B.ttf
index 67020e24d..8c93da9ff 100644
Binary files a/test/subset/data/expected/colr_with_components/colr-table.default.6B.ttf and b/test/subset/data/expected/colr_with_components/colr-table.default.6B.ttf differ
diff --git a/test/subset/data/expected/colr_with_components/colr-table.drop-hints-retain-gids.6B.ttf b/test/subset/data/expected/colr_with_components/colr-table.drop-hints-retain-gids.6B.ttf
index a2be48415..6857f7be7 100644
Binary files a/test/subset/data/expected/colr_with_components/colr-table.drop-hints-retain-gids.6B.ttf and b/test/subset/data/expected/colr_with_components/colr-table.drop-hints-retain-gids.6B.ttf differ
diff --git a/test/subset/data/expected/colr_with_components/colr-table.drop-hints.6B.ttf b/test/subset/data/expected/colr_with_components/colr-table.drop-hints.6B.ttf
index 67020e24d..8c93da9ff 100644
Binary files a/test/subset/data/expected/colr_with_components/colr-table.drop-hints.6B.ttf and b/test/subset/data/expected/colr_with_components/colr-table.drop-hints.6B.ttf differ
diff --git a/test/subset/data/expected/colr_with_components/colr-table.retain-gids.6B.ttf b/test/subset/data/expected/colr_with_components/colr-table.retain-gids.6B.ttf
index a2be48415..6857f7be7 100644
Binary files a/test/subset/data/expected/colr_with_components/colr-table.retain-gids.6B.ttf and b/test/subset/data/expected/colr_with_components/colr-table.retain-gids.6B.ttf differ
diff --git a/test/subset/data/fonts/colr-table.ttf b/test/subset/data/fonts/colr-table.ttf
index c4b2a55a5..d930faae9 100644
Binary files a/test/subset/data/fonts/colr-table.ttf and b/test/subset/data/fonts/colr-table.ttf differ