[instancer] support COLRv1 full instancing

src/OT/Color/COLR/COLR.hh

@@ -40,7 +40,6 @@
  */
 #define HB_OT_TAG_COLR HB_TAG('C','O','L','R')
 
-
 namespace OT {
 struct hb_paint_context_t;
 }
@@ -242,10 +241,15 @@ struct Variable
   void closurev1 (hb_colrv1_closure_context_t* c) const
   { value.closurev1 (c); }
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer) const
   {
     TRACE_SUBSET (this);
-    if (!value.subset (c)) return_trace (false);
+    if (!value.subset (c, instancer, varIdxBase)) return_trace (false);
+    if (c->plan->all_axes_pinned)
+      return_trace (true);
+
+    //TODO: update varIdxBase for partial-instancing
     return_trace (c->serializer->embed (varIdxBase));
   }
 
@@ -296,10 +300,11 @@ struct NoVariable
   void closurev1 (hb_colrv1_closure_context_t* c) const
   { value.closurev1 (c); }
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer) const
   {
     TRACE_SUBSET (this);
-    return_trace (value.subset (c));
+    return_trace (value.subset (c, instancer, varIdxBase));
   }
 
   bool sanitize (hb_sanitize_context_t *c) const
@@ -337,11 +342,20 @@ struct ColorStop
   void closurev1 (hb_colrv1_closure_context_t* c) const
   { c->add_palette_index (paletteIndex); }
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer,
+               uint32_t varIdxBase) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (*this);
     if (unlikely (!out)) return_trace (false);
+
+    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
+    {
+      out->stopOffset.set_float (stopOffset.to_float(instancer (varIdxBase, 0)));
+      out->alpha.set_float (alpha.to_float (instancer (varIdxBase, 1)));
+    }
+
     return_trace (c->serializer->check_assign (out->paletteIndex, c->plan->colr_palettes.get (paletteIndex),
                                                HB_SERIALIZE_ERROR_INT_OVERFLOW));
   }
@@ -390,7 +404,8 @@ struct ColorLine
       stop.closurev1 (c);
   }
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->start_embed (this);
@@ -402,7 +417,7 @@ struct ColorLine
 
     for (const auto& stop : stops.iter ())
     {
-      if (!stop.subset (c)) return_trace (false);
+      if (!stop.subset (c, instancer)) return_trace (false);
     }
     return_trace (true);
   }
@@ -523,6 +538,25 @@ struct Affine2x3
     return_trace (c->check_struct (this));
   }
 
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer,
+               uint32_t varIdxBase) const
+  {
+    TRACE_SUBSET (this);
+    auto *out = c->serializer->embed (*this);
+    if (unlikely (!out)) return_trace (false);
+    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
+    {
+      out->xx.set_float (xx.to_float(instancer (varIdxBase, 0)));
+      out->yx.set_float (yx.to_float(instancer (varIdxBase, 1)));
+      out->xy.set_float (xy.to_float(instancer (varIdxBase, 2)));
+      out->yy.set_float (yy.to_float(instancer (varIdxBase, 3)));
+      out->dx.set_float (dx.to_float(instancer (varIdxBase, 4)));
+      out->dy.set_float (dy.to_float(instancer (varIdxBase, 5)));
+    }
+    return_trace (true);
+  }
+
   void paint_glyph (hb_paint_context_t *c, uint32_t varIdxBase) const
   {
     c->funcs->push_transform (c->data,
@@ -548,7 +582,8 @@ struct PaintColrLayers
 {
   void closurev1 (hb_colrv1_closure_context_t* c) const;
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer HB_UNUSED) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (this);
@@ -579,11 +614,20 @@ struct PaintSolid
   void closurev1 (hb_colrv1_closure_context_t* c) const
   { c->add_palette_index (paletteIndex); }
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer,
+               uint32_t varIdxBase) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (*this);
     if (unlikely (!out)) return_trace (false);
+
+    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
+      out->alpha.set_float (alpha.to_float (instancer (varIdxBase, 0)));
+
+    if (format == 3 && c->plan->all_axes_pinned)
+        out->format = 2;
+
     return_trace (c->serializer->check_assign (out->paletteIndex, c->plan->colr_palettes.get (paletteIndex),
                                                HB_SERIALIZE_ERROR_INT_OVERFLOW));
   }
@@ -618,13 +662,28 @@ struct PaintLinearGradient
   void closurev1 (hb_colrv1_closure_context_t* c) const
   { (this+colorLine).closurev1 (c); }
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer,
+               uint32_t varIdxBase) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (this);
     if (unlikely (!out)) return_trace (false);
 
-    return_trace (out->colorLine.serialize_subset (c, colorLine, this));
+    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
+    {
+      out->x0 = x0 + (int) roundf (instancer (varIdxBase, 0));
+      out->y0 = y0 + (int) roundf (instancer (varIdxBase, 1));
+      out->x1 = x1 + (int) roundf (instancer (varIdxBase, 2));
+      out->y1 = y1 + (int) roundf (instancer (varIdxBase, 3));
+      out->x2 = x2 + (int) roundf (instancer (varIdxBase, 4));
+      out->y2 = y2 + (int) roundf (instancer (varIdxBase, 5));
+    }
+
+    if (format == 5 && c->plan->all_axes_pinned)
+        out->format = 4;
+
+    return_trace (out->colorLine.serialize_subset (c, colorLine, this, instancer));
   }
 
   bool sanitize (hb_sanitize_context_t *c) const
@@ -669,13 +728,28 @@ struct PaintRadialGradient
   void closurev1 (hb_colrv1_closure_context_t* c) const
   { (this+colorLine).closurev1 (c); }
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer,
+               uint32_t varIdxBase) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (this);
     if (unlikely (!out)) return_trace (false);
 
-    return_trace (out->colorLine.serialize_subset (c, colorLine, this));
+    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
+    {
+      out->x0 = x0 + (int) roundf (instancer (varIdxBase, 0));
+      out->y0 = y0 + (int) roundf (instancer (varIdxBase, 1));
+      out->radius0 = radius0 + (unsigned) roundf (instancer (varIdxBase, 2));
+      out->x1 = x1 + (int) roundf (instancer (varIdxBase, 3));
+      out->y1 = y1 + (int) roundf (instancer (varIdxBase, 4));
+      out->radius1 = radius1 + (unsigned) roundf (instancer (varIdxBase, 5));
+    }
+
+    if (format == 7 && c->plan->all_axes_pinned)
+        out->format = 6;
+
+    return_trace (out->colorLine.serialize_subset (c, colorLine, this, instancer));
   }
 
   bool sanitize (hb_sanitize_context_t *c) const
@@ -720,13 +794,26 @@ struct PaintSweepGradient
   void closurev1 (hb_colrv1_closure_context_t* c) const
   { (this+colorLine).closurev1 (c); }
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer,
+               uint32_t varIdxBase) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (this);
     if (unlikely (!out)) return_trace (false);
 
-    return_trace (out->colorLine.serialize_subset (c, colorLine, this));
+    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
+    {
+      out->centerX = centerX + (int) roundf (instancer (varIdxBase, 0));
+      out->centerY = centerY + (int) roundf (instancer (varIdxBase, 1));
+      out->startAngle.set_float (startAngle.to_float (instancer (varIdxBase, 2)));
+      out->endAngle.set_float (endAngle.to_float (instancer (varIdxBase, 3)));
+    }
+
+    if (format == 9 && c->plan->all_axes_pinned)
+        out->format = 8;
+
+    return_trace (out->colorLine.serialize_subset (c, colorLine, this, instancer));
   }
 
   bool sanitize (hb_sanitize_context_t *c) const
@@ -766,7 +853,8 @@ struct PaintGlyph
 {
   void closurev1 (hb_colrv1_closure_context_t* c) const;
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (this);
@@ -776,7 +864,7 @@ struct PaintGlyph
                                        HB_SERIALIZE_ERROR_INT_OVERFLOW))
       return_trace (false);
 
-    return_trace (out->paint.serialize_subset (c, paint, this));
+    return_trace (out->paint.serialize_subset (c, paint, this, instancer));
   }
 
   bool sanitize (hb_sanitize_context_t *c) const
@@ -807,7 +895,8 @@ struct PaintColrGlyph
 {
   void closurev1 (hb_colrv1_closure_context_t* c) const;
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer HB_UNUSED) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (this);
@@ -836,13 +925,16 @@ struct PaintTransform
 {
   HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (this);
     if (unlikely (!out)) return_trace (false);
-    if (!out->transform.serialize_copy (c->serializer, transform, this)) return_trace (false);
+    if (!out->transform.serialize_subset (c, transform, this, instancer)) return_trace (false);
-    return_trace (out->src.serialize_subset (c, src, this));
+    if (format == 13 && c->plan->all_axes_pinned)
+      out->format = 12;
+    return_trace (out->src.serialize_subset (c, src, this, instancer));
   }
 
   bool sanitize (hb_sanitize_context_t *c) const
@@ -871,13 +963,24 @@ struct PaintTranslate
 {
   HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer,
+               uint32_t varIdxBase) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (this);
     if (unlikely (!out)) return_trace (false);
 
-    return_trace (out->src.serialize_subset (c, src, this));
+    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
+    {
+      out->dx = dx + (int) roundf (instancer (varIdxBase, 0));
+      out->dy = dy + (int) roundf (instancer (varIdxBase, 1));
+    }
+
+    if (format == 15 && c->plan->all_axes_pinned)
+        out->format = 14;
+
+    return_trace (out->src.serialize_subset (c, src, this, instancer));
   }
 
   bool sanitize (hb_sanitize_context_t *c) const
@@ -908,13 +1011,24 @@ struct PaintScale
 {
   HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer,
+               uint32_t varIdxBase) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (this);
     if (unlikely (!out)) return_trace (false);
 
-    return_trace (out->src.serialize_subset (c, src, this));
+    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
+    {
+      out->scaleX.set_float (scaleX.to_float (instancer (varIdxBase, 0)));
+      out->scaleY.set_float (scaleY.to_float (instancer (varIdxBase, 1)));
+    }
+
+    if (format == 17 && c->plan->all_axes_pinned)
+        out->format = 16;
+
+    return_trace (out->src.serialize_subset (c, src, this, instancer));
   }
 
   bool sanitize (hb_sanitize_context_t *c) const
@@ -945,13 +1059,26 @@ struct PaintScaleAroundCenter
 {
   HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer,
+               uint32_t varIdxBase) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (this);
     if (unlikely (!out)) return_trace (false);
 
-    return_trace (out->src.serialize_subset (c, src, this));
+    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
+    {
+      out->scaleX.set_float (scaleX.to_float (instancer (varIdxBase, 0)));
+      out->scaleY.set_float (scaleY.to_float (instancer (varIdxBase, 1)));
+      out->centerX = centerX + (int) roundf (instancer (varIdxBase, 2));
+      out->centerY = centerY + (int) roundf (instancer (varIdxBase, 3));
+    }
+
+    if (format == 19 && c->plan->all_axes_pinned)
+        out->format = 18;
+
+    return_trace (out->src.serialize_subset (c, src, this, instancer));
   }
 
   bool sanitize (hb_sanitize_context_t *c) const
@@ -990,13 +1117,21 @@ struct PaintScaleUniform
 {
   HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer,
+               uint32_t varIdxBase) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (this);
     if (unlikely (!out)) return_trace (false);
 
-    return_trace (out->src.serialize_subset (c, src, this));
+    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
+      out->scale.set_float (scale.to_float (instancer (varIdxBase, 0)));
+
+    if (format == 21 && c->plan->all_axes_pinned)
+        out->format = 20;
+
+    return_trace (out->src.serialize_subset (c, src, this, instancer));
   }
 
   bool sanitize (hb_sanitize_context_t *c) const
@@ -1025,13 +1160,25 @@ struct PaintScaleUniformAroundCenter
 {
   HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer,
+               uint32_t varIdxBase) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (this);
     if (unlikely (!out)) return_trace (false);
 
-    return_trace (out->src.serialize_subset (c, src, this));
+    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
+    {
+      out->scale.set_float (scale.to_float (instancer (varIdxBase, 0)));
+      out->centerX = centerX + (int) roundf (instancer (varIdxBase, 1));
+      out->centerY = centerY + (int) roundf (instancer (varIdxBase, 2));
+    }
+
+    if (format == 23 && c->plan->all_axes_pinned)
+        out->format = 22;
+
+    return_trace (out->src.serialize_subset (c, src, this, instancer));
   }
 
   bool sanitize (hb_sanitize_context_t *c) const
@@ -1068,13 +1215,21 @@ struct PaintRotate
 {
   HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer,
+               uint32_t varIdxBase) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (this);
     if (unlikely (!out)) return_trace (false);
 
-    return_trace (out->src.serialize_subset (c, src, this));
+    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
+      out->angle.set_float (angle.to_float (instancer (varIdxBase, 0)));
+
+    if (format == 25 && c->plan->all_axes_pinned)
+      out->format = 24;
+
+    return_trace (out->src.serialize_subset (c, src, this, instancer));
   }
 
   bool sanitize (hb_sanitize_context_t *c) const
@@ -1103,13 +1258,25 @@ struct PaintRotateAroundCenter
 {
   HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer,
+               uint32_t varIdxBase) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (this);
     if (unlikely (!out)) return_trace (false);
 
-    return_trace (out->src.serialize_subset (c, src, this));
+    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
+    {
+      out->angle.set_float (angle.to_float (instancer (varIdxBase, 0)));
+      out->centerX = centerX + (int) roundf (instancer (varIdxBase, 1));
+      out->centerY = centerY + (int) roundf (instancer (varIdxBase, 2));
+    }
+
+    if (format ==27 && c->plan->all_axes_pinned)
+        out->format = 26;
+
+    return_trace (out->src.serialize_subset (c, src, this, instancer));
   }
 
   bool sanitize (hb_sanitize_context_t *c) const
@@ -1146,13 +1313,24 @@ struct PaintSkew
 {
   HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer,
+               uint32_t varIdxBase) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (this);
     if (unlikely (!out)) return_trace (false);
 
-    return_trace (out->src.serialize_subset (c, src, this));
+    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
+    {
+      out->xSkewAngle.set_float (xSkewAngle.to_float (instancer (varIdxBase, 0)));
+      out->ySkewAngle.set_float (ySkewAngle.to_float (instancer (varIdxBase, 1)));
+    }
+
+    if (format == 29 && c->plan->all_axes_pinned)
+        out->format = 28;
+
+    return_trace (out->src.serialize_subset (c, src, this, instancer));
   }
 
   bool sanitize (hb_sanitize_context_t *c) const
@@ -1183,13 +1361,26 @@ struct PaintSkewAroundCenter
 {
   HB_INTERNAL void closurev1 (hb_colrv1_closure_context_t* c) const;
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer,
+               uint32_t varIdxBase) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (this);
     if (unlikely (!out)) return_trace (false);
 
-    return_trace (out->src.serialize_subset (c, src, this));
+    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
+    {
+      out->xSkewAngle.set_float (xSkewAngle.to_float (instancer (varIdxBase, 0)));
+      out->ySkewAngle.set_float (ySkewAngle.to_float (instancer (varIdxBase, 1)));
+      out->centerX = centerX + (int) roundf (instancer (varIdxBase, 2));
+      out->centerY = centerY + (int) roundf (instancer (varIdxBase, 3));
+    }
+
+    if (format == 31 && c->plan->all_axes_pinned)
+        out->format = 30;
+
+    return_trace (out->src.serialize_subset (c, src, this, instancer));
   }
 
   bool sanitize (hb_sanitize_context_t *c) const
@@ -1228,14 +1419,15 @@ struct PaintComposite
 {
   void closurev1 (hb_colrv1_closure_context_t* c) const;
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->embed (this);
     if (unlikely (!out)) return_trace (false);
 
-    if (!out->src.serialize_subset (c, src, this)) return_trace (false);
+    if (!out->src.serialize_subset (c, src, this, instancer)) return_trace (false);
-    return_trace (out->backdrop.serialize_subset (c, backdrop, this));
+    return_trace (out->backdrop.serialize_subset (c, backdrop, this, instancer));
   }
 
   bool sanitize (hb_sanitize_context_t *c) const
@@ -1283,6 +1475,28 @@ struct ClipBoxFormat1
     clip_box.yMax = yMax;
   }
 
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer,
+               uint32_t varIdxBase) const
+  {
+    TRACE_SUBSET (this);
+    auto *out = c->serializer->embed (*this);
+    if (unlikely (!out)) return_trace (false);
+
+    if (instancer && !c->plan->pinned_at_default && varIdxBase != VarIdx::NO_VARIATION)
+    {
+      out->xMin = xMin + (int) roundf (instancer (varIdxBase, 0));
+      out->yMin = yMin + (int) roundf (instancer (varIdxBase, 1));
+      out->xMax = xMax + (int) roundf (instancer (varIdxBase, 2));
+      out->yMax = yMax + (int) roundf (instancer (varIdxBase, 3));
+    }
+
+    if (format == 2 && c->plan->all_axes_pinned)
+        out->format = 1;
+
+    return_trace (true);
+  }
+
   public:
   HBUINT8	format; /* format = 1(noVar) or 2(Var)*/
   FWORD		xMin;
@@ -1310,13 +1524,14 @@ struct ClipBoxFormat2 : Variable<ClipBoxFormat1>
 
 struct ClipBox
 {
-  ClipBox* copy (hb_serialize_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer) const
   {
-    TRACE_SERIALIZE (this);
+    TRACE_SUBSET (this);
     switch (u.format) {
-    case 1: return_trace (reinterpret_cast<ClipBox *> (c->embed (u.format1)));
+    case 1: return_trace (u.format1.subset (c, instancer, VarIdx::NO_VARIATION));
-    case 2: return_trace (reinterpret_cast<ClipBox *> (c->embed (u.format2)));
+    case 2: return_trace (u.format2.subset (c, instancer));
-    default:return_trace (nullptr);
+    default:return_trace (c->default_return_value ());
     }
   }
 
@@ -1367,13 +1582,15 @@ struct ClipRecord
   int cmp (hb_codepoint_t g) const
   { return g < startGlyphID ? -1 : g <= endGlyphID ? 0 : +1; }
 
-  ClipRecord* copy (hb_serialize_context_t *c, const void *base) const
+  bool subset (hb_subset_context_t *c,
+               const void *base,
+               const VarStoreInstancer &instancer) const
   {
-    TRACE_SERIALIZE (this);
+    TRACE_SUBSET (this);
-    auto *out = c->embed (this);
+    auto *out = c->serializer->embed (*this);
-    if (unlikely (!out)) return_trace (nullptr);
+    if (unlikely (!out)) return_trace (false);
-    if (!out->clipBox.serialize_copy (c, clipBox, base)) return_trace (nullptr);
+
-    return_trace (out);
+    return_trace (out->clipBox.serialize_subset (c, clipBox, base, instancer));
   }
 
   bool sanitize (hb_sanitize_context_t *c, const void *base) const
@@ -1400,7 +1617,8 @@ DECLARE_NULL_NAMESPACE_BYTES (OT, ClipRecord);
 
 struct ClipList
 {
-  unsigned serialize_clip_records (hb_serialize_context_t *c,
+  unsigned serialize_clip_records (hb_subset_context_t *c,
+                                   const VarStoreInstancer &instancer,
                                    const hb_set_t& gids,
                                    const hb_map_t& gid_offset_map) const
   {
@@ -1432,7 +1650,7 @@ struct ClipList
       record.endGlyphID = prev_gid;
       record.clipBox = prev_offset;
 
-      if (!c->copy (record, this)) return_trace (0);
+      if (!record.subset (c, this, instancer)) return_trace (0);
       count++;
 
       start_gid = _;
@@ -1446,13 +1664,14 @@ struct ClipList
       record.startGlyphID = start_gid;
       record.endGlyphID = prev_gid;
       record.clipBox = prev_offset;
-      if (!c->copy (record, this)) return_trace (0);
+      if (!record.subset (c, this, instancer)) return_trace (0);
       count++;
     }
     return_trace (count);
   }
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->start_embed (*this);
@@ -1477,7 +1696,7 @@ struct ClipList
       }
     }
 
-    unsigned count = serialize_clip_records (c->serializer, new_gids, new_gid_offset_map);
+    unsigned count = serialize_clip_records (c, instancer, new_gids, new_gid_offset_map);
     if (!count) return_trace (false);
     return_trace (c->serializer->check_assign (out->clips.len, count, HB_SERIALIZE_ERROR_INT_OVERFLOW));
   }
@@ -1611,7 +1830,8 @@ struct BaseGlyphPaintRecord
   { return g < glyphId ? -1 : g > glyphId ? 1 : 0; }
 
   bool serialize (hb_serialize_context_t *s, const hb_map_t* glyph_map,
-                  const void* src_base, hb_subset_context_t *c) const
+                  const void* src_base, hb_subset_context_t *c,
+                  const VarStoreInstancer &instancer) const
   {
     TRACE_SERIALIZE (this);
     auto *out = s->embed (this);
@@ -1620,7 +1840,7 @@ struct BaseGlyphPaintRecord
                           HB_SERIALIZE_ERROR_INT_OVERFLOW))
       return_trace (false);
 
-    return_trace (out->paint.serialize_subset (c, paint, src_base));
+    return_trace (out->paint.serialize_subset (c, paint, src_base, instancer));
   }
 
   bool sanitize (hb_sanitize_context_t *c, const void *base) const
@@ -1639,7 +1859,8 @@ struct BaseGlyphPaintRecord
 
 struct BaseGlyphList : SortedArray32Of<BaseGlyphPaintRecord>
 {
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->start_embed (this);
@@ -1651,7 +1872,7 @@ struct BaseGlyphList : SortedArray32Of<BaseGlyphPaintRecord>
       unsigned gid = _.glyphId;
       if (!glyphset->has (gid)) continue;
 
-      if (_.serialize (c->serializer, c->plan->glyph_map, this, c)) out->len++;
+      if (_.serialize (c->serializer, c->plan->glyph_map, this, c, instancer)) out->len++;
       else return_trace (false);
     }
 
@@ -1670,7 +1891,8 @@ struct LayerList : Array32OfOffset32To<Paint>
   const Paint& get_paint (unsigned i) const
   { return this+(*this)[i]; }
 
-  bool subset (hb_subset_context_t *c) const
+  bool subset (hb_subset_context_t *c,
+               const VarStoreInstancer &instancer) const
   {
     TRACE_SUBSET (this);
     auto *out = c->serializer->start_embed (this);
@@ -1681,7 +1903,7 @@ struct LayerList : Array32OfOffset32To<Paint>
 
     {
       auto *o = out->serialize_append (c->serializer);
-      if (unlikely (!o) || !o->serialize_subset (c, _.second, this))
+      if (unlikely (!o) || !o->serialize_subset (c, _.second, this, instancer))
         return_trace (false);
     }
     return_trace (true);
@@ -1883,7 +2105,6 @@ struct COLR
   bool subset (hb_subset_context_t *c) const
   {
     TRACE_SUBSET (this);
-
     const hb_map_t &reverse_glyph_map = *c->plan->reverse_glyph_map;
     const hb_set_t& glyphset = c->plan->_glyphset_colred;
 
@@ -1954,7 +2175,12 @@ struct COLR
 
     auto snap = c->serializer->snapshot ();
     if (!c->serializer->allocate_size<void> (5 * HBUINT32::static_size)) return_trace (false);
-    if (!colr_prime->baseGlyphList.serialize_subset (c, baseGlyphList, this))
+
+    VarStoreInstancer instancer (varStore ? &(this+varStore) : nullptr,
+	                         varIdxMap ? &(this+varIdxMap) : nullptr,
+	                         c->plan->normalized_coords.as_array ());
+
+    if (!colr_prime->baseGlyphList.serialize_subset (c, baseGlyphList, this, instancer))
     {
       if (c->serializer->in_error ()) return_trace (false);
       //no more COLRv1 glyphs: downgrade to version 0
@@ -1964,8 +2190,11 @@ struct COLR
 
     if (!colr_prime->serialize_V0 (c->serializer, version, base_it, layer_it)) return_trace (false);
 
-    colr_prime->layerList.serialize_subset (c, layerList, this);
+    colr_prime->layerList.serialize_subset (c, layerList, this, instancer);
-    colr_prime->clipList.serialize_subset (c, clipList, this);
+    colr_prime->clipList.serialize_subset (c, clipList, this, instancer);
+    if (!varStore || c->plan->all_axes_pinned)
+      return_trace (true);
+
     colr_prime->varIdxMap.serialize_copy (c->serializer, varIdxMap, this);
     colr_prime->varStore.serialize_copy (c->serializer, varStore, this);
     return_trace (true);
@@ -1990,8 +2219,8 @@ struct COLR
     if (version != 1)
       return false;
 
-    VarStoreInstancer instancer (this+varStore,
+    VarStoreInstancer instancer (&(this+varStore),
-				 this+varIdxMap,
+				 &(this+varIdxMap),
 				 hb_array (font->coords, font->num_coords));
 
     if (get_clip (glyph, extents, instancer))
@@ -2048,8 +2277,8 @@ struct COLR
   bool
   paint_glyph (hb_font_t *font, hb_codepoint_t glyph, hb_paint_funcs_t *funcs, void *data, unsigned int palette_index, hb_color_t foreground, bool clip = true) const
   {
-    VarStoreInstancer instancer (this+varStore,
+    VarStoreInstancer instancer (&(this+varStore),
-	                         this+varIdxMap,
+	                         &(this+varIdxMap),
 	                         hb_array (font->coords, font->num_coords));
     hb_paint_context_t c (this, funcs, data, font, palette_index, foreground, instancer);
 
@@ -2060,8 +2289,8 @@ struct COLR
       {
         // COLRv1 glyph
 
-	VarStoreInstancer instancer (this+varStore,
+	VarStoreInstancer instancer (&(this+varStore),
-				     this+varIdxMap,
+				     &(this+varIdxMap),
 				     hb_array (font->coords, font->num_coords));
 
 	bool is_bounded = true;

src/hb-ot-var-common.hh

@@ -222,18 +222,20 @@ struct DeltaSetIndexMap
 
 struct VarStoreInstancer
 {
-  VarStoreInstancer (const VariationStore &varStore,
+  VarStoreInstancer (const VariationStore *varStore,
-		     const DeltaSetIndexMap &varIdxMap,
+		     const DeltaSetIndexMap *varIdxMap,
 		     hb_array_t<int> coords) :
     varStore (varStore), varIdxMap (varIdxMap), coords (coords) {}
 
-  operator bool () const { return bool (coords); }
+  operator bool () const { return varStore && bool (coords); }
 
+  /* according to the spec, if colr table has varStore but does not have
+   * varIdxMap, then an implicit identity mapping is used */
   float operator() (uint32_t varIdx, unsigned short offset = 0) const
-  { return varStore.get_delta (varIdxMap.map (VarIdx::add (varIdx, offset)), coords); }
+  { return varStore->get_delta (varIdxMap ? varIdxMap->map (VarIdx::add (varIdx, offset)) : varIdx + offset, coords); }
 
-  const VariationStore &varStore;
+  const VariationStore *varStore;
-  const DeltaSetIndexMap &varIdxMap;
+  const DeltaSetIndexMap *varIdxMap;
   hb_array_t<int> coords;
 };