[instancer] avoid double free for compiled glyph bytes

also increase the HB_GLYF_MAX_POINTS limit to 20000 cause the test file has a
.notdef glyph which is a composite glyph and has 10176 points after
get_points() call

src/OT/glyf/glyf.hh

@@ -98,7 +98,9 @@ struct glyf
       use_short_loca = max_offset < 0x1FFFF;
     }
 
-    glyf_prime->serialize (c->serializer, glyphs.writer (), use_short_loca, c->plan, font);
+    if (!glyf_prime->serialize (c->serializer, glyphs.writer (), use_short_loca, c->plan, font))
+      return false;
+
     if (!use_short_loca) {
       padded_offsets =
           + hb_iter (glyphs)

src/hb-limits.hh

@@ -90,7 +90,7 @@
 
 
 #ifndef HB_GLYF_MAX_POINTS
-#define HB_GLYF_MAX_POINTS 10000
+#define HB_GLYF_MAX_POINTS 20000
 #endif
 
 #ifndef HB_GLYF_MAX_EDGE_COUNT

test/subset/data/Makefile.am

@@ -60,6 +60,7 @@ EXTRA_DIST += \
 	expected/instance_feature_variations \
 	expected/instantiate_glyf \
 	expected/pin_all_at_default \
+	expected/instance_no_double_free \
 	fonts \
 	profiles \
 	$(NULL)

test/subset/data/Makefile.sources

@@ -51,6 +51,7 @@ TESTS = \
 	tests/instance_feature_variations.tests \
 	tests/instantiate_glyf.tests \
 	tests/pin_all_at_default.tests \
+	tests/instance_no_double_free.tests \
 	$(NULL)
 
 # TODO: re-enable once colrv1 subsetting is stabilized.

test/subset/data/tests/instance_no_double_free.tests

@@ -0,0 +1,11 @@
+FONTS:
+Handjet.ttf
+
+PROFILES:
+notdef-outline.txt
+
+SUBSETS:
+*
+
+INSTANCES:
+wght=100,ELGR=1,ELSH=2

test/subset/meson.build

@@ -53,6 +53,7 @@ tests = [
   'instantiate_glyf',
   'full_instance',
   'instance_feature_variations',
+  'instance_no_double_free',
 ]
 
 repack_tests = [