fix for cmap4 and OS_2 subsetting: maximum character code allowed is 0xFFFF

src/hb-ot-cmap-table.hh

@@ -242,8 +242,14 @@ struct CmapSubtableFormat4
     if (unlikely (!c->extend_min (*this))) return;
     this->format = 4;
 
+    auto format4_iter =
+    + it
+    | hb_filter ([&] (const hb_pair_t<hb_codepoint_t, hb_codepoint_t> _)
+		 { return _.first <= 0xFFFF; })
+    ;
+
     //serialize endCode[]
-    HBUINT16 *endCode = serialize_endcode_array (c, it);
+    HBUINT16 *endCode = serialize_endcode_array (c, format4_iter);
     if (unlikely (!endCode)) return;
 
     unsigned segcount = (c->length () - min_size) / HBUINT16::static_size;
@@ -252,14 +258,14 @@ struct CmapSubtableFormat4
     if (unlikely (!c->allocate_size<HBUINT16> (HBUINT16::static_size))) return; // 2 bytes of padding.
 
    // serialize startCode[]
-    HBUINT16 *startCode = serialize_startcode_array (c, it);
+    HBUINT16 *startCode = serialize_startcode_array (c, format4_iter);
     if (unlikely (!startCode)) return;
 
     //serialize idDelta[]
-    HBINT16 *idDelta = serialize_idDelta_array (c, it, endCode, startCode, segcount);
+    HBINT16 *idDelta = serialize_idDelta_array (c, format4_iter, endCode, startCode, segcount);
     if (unlikely (!idDelta)) return;
 
-    HBUINT16 *idRangeOffset = serialize_rangeoffset_glyid (c, it, endCode, startCode, idDelta, segcount);
+    HBUINT16 *idRangeOffset = serialize_rangeoffset_glyid (c, format4_iter, endCode, startCode, idDelta, segcount);
     if (unlikely (!c->check_success (idRangeOffset))) return;
 
     if (unlikely (!c->check_assign(this->length, c->length () - table_initpos))) return;

src/hb-ot-os2-table.hh

@@ -216,8 +216,8 @@ struct OS2
 					  uint16_t *min_cp, /* OUT */
 					  uint16_t *max_cp  /* OUT */)
   {
-    *min_cp = codepoints->get_min ();
+    *min_cp = hb_min (0xFFFFu, codepoints->get_min ());
-    *max_cp = codepoints->get_max ();
+    *max_cp = hb_min (0xFFFFu, codepoints->get_max ());
   }
 
   /* https://github.com/Microsoft/Font-Validator/blob/520aaae/OTFontFileVal/val_OS2.cs#L644-L681 */

test/api/test-subset-cmap.c

@@ -90,6 +90,29 @@ test_subset_cmap_noop (void)
   hb_face_destroy (face_abc);
 }
 
+static void
+test_subset_cmap4_no_exceeding_maximum_codepoint (void)
+{
+  hb_face_t *face_origin = hb_test_open_font_file ("fonts/Mplus1p-Regular.ttf");
+  hb_face_t *face_expected = hb_test_open_font_file ("fonts/Mplus1p-Regular-cmap4-testing.ttf");
+
+  hb_set_t *codepoints = hb_set_create ();
+  hb_face_t *face_subset;
+  hb_set_add (codepoints, 0x20);
+  hb_set_add (codepoints, 0x21);
+  hb_set_add (codepoints, 0x1d542);
+  hb_set_add (codepoints, 0x201a2);
+
+  face_subset = hb_subset_test_create_subset (face_origin, hb_subset_test_create_input (codepoints));
+  hb_set_destroy (codepoints);
+
+  hb_subset_test_check (face_expected, face_subset, HB_TAG ('c','m','a','p'));
+
+  hb_face_destroy (face_subset);
+  hb_face_destroy (face_expected);
+  hb_face_destroy (face_origin);
+}
+
 // TODO(rsheeter) test cmap to no codepoints
 
 int
@@ -100,6 +123,7 @@ main (int argc, char **argv)
   hb_test_add (test_subset_cmap);
   hb_test_add (test_subset_cmap_noop);
   hb_test_add (test_subset_cmap_non_consecutive_glyphs);
+  hb_test_add (test_subset_cmap4_no_exceeding_maximum_codepoint);
 
   return hb_test_run();
 }

test/subset/data/tests/japanese.tests

@@ -11,6 +11,6 @@ acek
 明極珠度輸清
 あいうえおか
 あいう珠度輸
-
+𝜕𝟘AB