Merge branch 'master' into vc++-fixes

NEWS

@@ -1,3 +1,39 @@
+Overview of changes leading to 1.0.1
+Monday, July 27, 2015
+====================================
+
+- Fix out-of-bounds access in USE shaper.
+
+
+Overview of changes leading to 1.0.0
+Sunday, July 26, 2015
+====================================
+
+- Implement Universal Shaping Engine:
+  https://www.microsoft.com/typography/OpenTypeDev/USE/intro.htm
+  http://blogs.windows.com/bloggingwindows/2015/02/23/windows-shapes-the-worlds-languages/
+- Bump version to 1.0.0.  The soname was NOT bumped.
+
+
+Overview of changes leading to 0.9.42
+Thursday, July 26, 2015
+=====================================
+
+- New API to allow for retrieving finer-grained cluster
+  mappings if the client desires to handle them.  Default
+  behavior is unchanged.
+- Fix cluster merging when removing default-ignorables.
+- Update to Unicode 8.0
+- hb-graphite2 fixes.
+- Misc fixes.
+- Removed HB_NO_MERGE_CLUSTERS hack.
+- New API:
+  hb_buffer_cluster_level_t enum
+  hb_buffer_get_cluster_level()
+  hb_buffer_set_cluster_level()
+  hb-shape / hb-view --cluster-level
+
+
 Overview of changes leading to 0.9.41
 Thursday, June 18, 2015
 =====================================
@@ -8,6 +44,7 @@ Thursday, June 18, 2015
 - Fix hb_language_t in language bindings.
 - Misc fixes.
 
+
 Overview of changes leading to 0.9.40
 Friday, March 20, 2015
 =====================================

configure.ac

@@ -1,6 +1,6 @@
 AC_PREREQ([2.64])
 AC_INIT([HarfBuzz],
-        [0.9.41],
+        [1.0.1],
         [http://bugs.freedesktop.org/enter_bug.cgi?product=harfbuzz],
         [harfbuzz],
         [http://harfbuzz.org/])
@@ -51,7 +51,7 @@ m4_if(m4_eval(hb_version_minor % 2), [1],
 m4_define([hb_libtool_age],
 	  m4_eval(hb_version_int - hb_libtool_revision))
 m4_define([hb_libtool_current],
-	  m4_eval(hb_version_major + hb_libtool_age))
+	  m4_eval(hb_libtool_age))
 HB_LIBTOOL_VERSION_INFO=hb_libtool_current:hb_libtool_revision:hb_libtool_age
 AC_SUBST(HB_LIBTOOL_VERSION_INFO)
 
@@ -145,8 +145,10 @@ AC_ARG_WITH(glib,
 			[Use glib @<:@default=auto@:>@])],,
 	[with_glib=auto])
 have_glib=false
+GLIB_DEPS="glib-2.0 >= 2.16"
+AC_SUBST(GLIB_DEPS)
 if test "x$with_glib" = "xyes" -o "x$with_glib" = "xauto"; then
-	PKG_CHECK_MODULES(GLIB, glib-2.0 >= 2.16, have_glib=true, :)
+	PKG_CHECK_MODULES(GLIB, $GLIB_DEPS, have_glib=true, :)
 fi
 if test "x$with_glib" = "xyes" -a "x$have_glib" != "xtrue"; then
 	AC_MSG_ERROR([glib support requested but glib-2.0 not found])
@@ -296,8 +298,10 @@ AC_ARG_WITH(graphite2,
 			[Use the graphite2 library @<:@default=no@:>@])],,
 	[with_graphite2=no])
 have_graphite2=false
+GRAPHITE2_DEPS="graphite2"
+AC_SUBST(GRAPHITE2_DEPS)
 if test "x$with_graphite2" = "xyes" -o "x$with_graphite2" = "xauto"; then
-	PKG_CHECK_MODULES(GRAPHITE2, graphite2, have_graphite2=true, :)
+	PKG_CHECK_MODULES(GRAPHITE2, $GRAPHITE2_DEPS, have_graphite2=true, :)
 fi
 if test "x$with_graphite2" = "xyes" -a "x$have_graphite2" != "xtrue"; then
 	AC_MSG_ERROR([graphite2 support requested but libgraphite2 not found])
@@ -314,9 +318,11 @@ AC_ARG_WITH(freetype,
 			[Use the FreeType library @<:@default=auto@:>@])],,
 	[with_freetype=auto])
 have_freetype=false
+FREETYPE_DEPS="freetype2 >= 12.0.6"
+AC_SUBST(FREETYPE_DEPS)
 if test "x$with_freetype" = "xyes" -o "x$with_freetype" = "xauto"; then
 	# See freetype/docs/VERSION.DLL; 12.0.6 means freetype-2.4.2
-	PKG_CHECK_MODULES(FREETYPE, freetype2 >= 12.0.6, have_freetype=true, :)
+	PKG_CHECK_MODULES(FREETYPE, $FREETYPE_DEPS, have_freetype=true, :)
 fi
 if test "x$with_freetype" = "xyes" -a "x$have_freetype" != "xtrue"; then
 	AC_MSG_ERROR([FreeType support requested but libfreetype2 not found])

src/Makefile.am

@@ -14,12 +14,14 @@ DISTCHECK_CONFIGURE_FLAGS = --enable-introspection
 #AM_CXXFLAGS =
 
 # Convenience targets:
-lib: libharfbuzz.la
+lib: $(BUILT_SOURCES) libharfbuzz.la
 
 lib_LTLIBRARIES = libharfbuzz.la
 
 HBCFLAGS =
 HBLIBS =
+HBNONPCLIBS =
+HBDEPS =
 HBSOURCES =  \
 	hb-atomic-private.hh \
 	hb-blob.cc \
@@ -93,6 +95,7 @@ HBSOURCES += \
 	hb-ot-shape.cc \
 	hb-ot-shape-complex-arabic.cc \
 	hb-ot-shape-complex-arabic-fallback.hh \
+	hb-ot-shape-complex-arabic-private.hh \
 	hb-ot-shape-complex-arabic-table.hh \
 	hb-ot-shape-complex-arabic-win1256.hh \
 	hb-ot-shape-complex-default.cc \
@@ -104,10 +107,12 @@ HBSOURCES += \
 	hb-ot-shape-complex-indic-table.cc \
 	hb-ot-shape-complex-myanmar.cc \
 	hb-ot-shape-complex-myanmar-machine.hh \
-	hb-ot-shape-complex-sea.cc \
-	hb-ot-shape-complex-sea-machine.hh \
 	hb-ot-shape-complex-thai.cc \
 	hb-ot-shape-complex-tibetan.cc \
+	hb-ot-shape-complex-use.cc \
+	hb-ot-shape-complex-use-machine.hh \
+	hb-ot-shape-complex-use-private.hh \
+	hb-ot-shape-complex-use-table.cc \
 	hb-ot-shape-complex-private.hh \
 	hb-ot-shape-normalize-private.hh \
 	hb-ot-shape-normalize.cc \
@@ -130,12 +135,13 @@ endif
 
 if HAVE_PTHREAD
 HBCFLAGS += $(PTHREAD_CFLAGS)
-HBLIBS   += $(PTHREAD_LIBS)
+HBNONPCLIBS += $(PTHREAD_LIBS)
 endif
 
 if HAVE_GLIB
 HBCFLAGS += $(GLIB_CFLAGS)
 HBLIBS   += $(GLIB_LIBS)
+HBDEPS   += $(GLIB_DEPS)
 HBSOURCES += hb-glib.cc
 HBHEADERS += hb-glib.h
 endif
@@ -143,6 +149,7 @@ endif
 if HAVE_FREETYPE
 HBCFLAGS += $(FREETYPE_CFLAGS)
 HBLIBS   += $(FREETYPE_LIBS)
+HBDEPS   += $(FREETYPE_DEPS)
 HBSOURCES += hb-ft.cc
 HBHEADERS += hb-ft.h
 endif
@@ -150,20 +157,21 @@ endif
 if HAVE_GRAPHITE2
 HBCFLAGS += $(GRAPHITE2_CFLAGS)
 HBLIBS   += $(GRAPHITE2_LIBS)
+HBDEPS   += $(GRAPHITE2_DEPS)
 HBSOURCES += hb-graphite2.cc
 HBHEADERS += hb-graphite2.h
 endif
 
 if HAVE_UNISCRIBE
 HBCFLAGS += $(UNISCRIBE_CFLAGS)
-HBLIBS   += $(UNISCRIBE_LIBS)
+HBNONPCLIBS += $(UNISCRIBE_LIBS)
 HBSOURCES += hb-uniscribe.cc
 HBHEADERS += hb-uniscribe.h
 endif
 
 if HAVE_CORETEXT
 HBCFLAGS += $(CORETEXT_CFLAGS)
-HBLIBS   += $(CORETEXT_LIBS)
+HBNONPCLIBS += $(CORETEXT_LIBS)
 HBSOURCES += hb-coretext.cc
 HBHEADERS += hb-coretext.h
 endif
@@ -179,6 +187,8 @@ DIST_SUBDIRS += hb-ucdn
 
 # Put the library together
 
+HBLIBS += $(HBNONPCLIBS)
+
 if OS_WIN32
 export_symbols = -export-symbols harfbuzz.def
 harfbuzz_def_dependency = harfbuzz.def
@@ -253,6 +263,8 @@ EXTRA_DIST += \
 		-e 's@%exec_prefix%@$(exec_prefix)@g' \
 		-e 's@%libdir%@$(libdir)@g' \
 		-e 's@%includedir%@$(includedir)@g' \
+		-e 's@%libs_private%@$(HBNONPCLIBS)@g' \
+		-e 's@%requires_private%@$(HBDEPS)@g' \
 		-e 's@%VERSION%@$(VERSION)@g' \
 	"$<" > "$@" \
 	|| ($(RM) "$@"; false)
@@ -267,7 +279,7 @@ harfbuzz.def: $(HBHEADERS) $(HBNODISTHEADERS)
 	$(EGREP) '^hb_.* \(' | \
 	sed -e 's/ (.*//' | \
 	LANG=C sort; \
-	echo LIBRARY libharfbuzz-$(HB_VERSION_MAJOR).dll; \
+	echo LIBRARY libharfbuzz-0.dll; \
 	) >"$@"
 	@ ! grep -q hb_ERROR "$@" \
 	|| ($(RM) "$@"; false)
@@ -276,29 +288,34 @@ harfbuzz.def: $(HBHEADERS) $(HBNODISTHEADERS)
 GENERATORS = \
 	gen-arabic-table.py \
 	gen-indic-table.py \
+	gen-use-table.py \
 	$(NULL)
 EXTRA_DIST += $(GENERATORS)
 
-unicode-tables: arabic-table indic-table
+unicode-tables: arabic-table indic-table use-table
-
-indic-table: gen-indic-table.py IndicSyllabicCategory.txt IndicMatraCategory.txt Blocks.txt
-	$(AM_V_GEN) $(builddir)/$^ > hb-ot-shape-complex-indic-table.cc \
-	|| ($(RM) hb-ot-shape-complex-indic-table.cc; false)
 
 arabic-table: gen-arabic-table.py ArabicShaping.txt UnicodeData.txt Blocks.txt
 	$(AM_V_GEN) $(builddir)/$^ > hb-ot-shape-complex-arabic-table.hh \
 	|| ($(RM) hb-ot-shape-complex-arabic-table.hh; false)
 
+indic-table: gen-indic-table.py IndicSyllabicCategory-7.0.0.txt IndicMatraCategory-7.0.0.txt Blocks.txt
+	$(AM_V_GEN) $(builddir)/$^ > hb-ot-shape-complex-indic-table.cc \
+	|| ($(RM) hb-ot-shape-complex-indic-table.cc; false)
+
+use-table: gen-use-table.py IndicSyllabicCategory.txt IndicPositionalCategory.txt UnicodeData.txt Blocks.txt
+	$(AM_V_GEN) $(builddir)/$^ > hb-ot-shape-complex-use-table.cc \
+	|| ($(RM) hb-ot-shape-complex-use-table.cc; false)
+
 built-sources: $(BUILT_SOURCES)
 
-.PHONY: unicode-tables arabic-table indic-table built-sources
+.PHONY: unicode-tables arabic-table indic-table use-table built-sources
 
 RAGEL_GENERATED = \
 	$(srcdir)/hb-buffer-deserialize-json.hh \
 	$(srcdir)/hb-buffer-deserialize-text.hh \
 	$(srcdir)/hb-ot-shape-complex-indic-machine.hh \
 	$(srcdir)/hb-ot-shape-complex-myanmar-machine.hh \
-	$(srcdir)/hb-ot-shape-complex-sea-machine.hh \
+	$(srcdir)/hb-ot-shape-complex-use-machine.hh \
 	$(NULL)
 BUILT_SOURCES += $(RAGEL_GENERATED)
 EXTRA_DIST += \
@@ -306,7 +323,7 @@ EXTRA_DIST += \
 	hb-buffer-deserialize-text.rl \
 	hb-ot-shape-complex-indic-machine.rl \
 	hb-ot-shape-complex-myanmar-machine.rl \
-	hb-ot-shape-complex-sea-machine.rl \
+	hb-ot-shape-complex-use-machine.rl \
 	$(NULL)
 MAINTAINERCLEANFILES += $(RAGEL_GENERATED)
 $(srcdir)/%.hh: $(srcdir)/%.rl
@@ -363,7 +380,7 @@ TESTS_ENVIRONMENT = \
 if HAVE_INTROSPECTION
 
 -include $(INTROSPECTION_MAKEFILE)
-INTROSPECTION_GIRS = HarfBuzz-$(HB_VERSION_MAJOR).0.gir # What does the 0 mean anyway?!
+INTROSPECTION_GIRS = HarfBuzz-0.0.gir # What does the 0 mean anyway?!
 INTROSPECTION_SCANNER_ARGS = -I$(srcdir) -n hb --identifier-prefix=hb_ --warn-all
 INTROSPECTION_COMPILER_ARGS = --includedir=$(srcdir)
 INTROSPECTION_SCANNER_ENV = CC="$(CC)"

src/gen-use-table.py

@@ -0,0 +1,476 @@
+#!/usr/bin/python
+
+import sys
+
+if len (sys.argv) != 5:
+	print >>sys.stderr, "usage: ./gen-use-table.py IndicSyllabicCategory.txt IndicPositionalCategory.txt UnicodeData.txt Blocks.txt"
+	sys.exit (1)
+
+BLACKLISTED_BLOCKS = ["Thai", "Lao", "Tibetan"]
+
+files = [file (x) for x in sys.argv[1:]]
+
+headers = [[f.readline () for i in range (2)] for j,f in enumerate(files) if j != 2]
+headers.append (["UnicodeData.txt does not have a header."])
+
+data = [{} for f in files]
+values = [{} for f in files]
+for i, f in enumerate (files):
+	for line in f:
+
+		j = line.find ('#')
+		if j >= 0:
+			line = line[:j]
+
+		fields = [x.strip () for x in line.split (';')]
+		if len (fields) == 1:
+			continue
+
+		uu = fields[0].split ('..')
+		start = int (uu[0], 16)
+		if len (uu) == 1:
+			end = start
+		else:
+			end = int (uu[1], 16)
+
+		t = fields[1 if i != 2 else 2]
+
+		for u in range (start, end + 1):
+			data[i][u] = t
+		values[i][t] = values[i].get (t, 0) + end - start + 1
+
+defaults = ('Other', 'Not_Applicable', 'Cn', 'No_Block')
+
+# TODO Characters that are not in Unicode Indic files, but used in USE
+data[0][0x034F] = defaults[0]
+data[0][0x2060] = defaults[0]
+for u in range (0xFE00, 0xFE0F + 1):
+	data[0][u] = defaults[0]
+
+# Merge data into one dict:
+for i,v in enumerate (defaults):
+	values[i][v] = values[i].get (v, 0) + 1
+combined = {}
+for i,d in enumerate (data):
+	for u,v in d.items ():
+		if i >= 2 and not u in combined:
+			continue
+		if not u in combined:
+			combined[u] = list (defaults)
+		combined[u][i] = v
+combined = {k:v for k,v in combined.items() if v[3] not in BLACKLISTED_BLOCKS}
+data = combined
+del combined
+num = len (data)
+
+
+property_names = [
+	# General_Category
+	'Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Ll', 'Lm', 'Lo', 'Lt', 'Lu', 'Mc',
+	'Me', 'Mn', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Pe', 'Pf', 'Pi', 'Po',
+	'Ps', 'Sc', 'Sk', 'Sm', 'So', 'Zl', 'Zp', 'Zs',
+	# Indic_Syllabic_Category
+	'Other',
+	'Bindu',
+	'Visarga',
+	'Avagraha',
+	'Nukta',
+	'Virama',
+	'Pure_Killer',
+	'Invisible_Stacker',
+	'Vowel_Independent',
+	'Vowel_Dependent',
+	'Vowel',
+	'Consonant_Placeholder',
+	'Consonant',
+	'Consonant_Dead',
+	'Consonant_With_Stacker',
+	'Consonant_Prefixed',
+	'Consonant_Preceding_Repha',
+	'Consonant_Succeeding_Repha',
+	'Consonant_Subjoined',
+	'Consonant_Medial',
+	'Consonant_Final',
+	'Consonant_Head_Letter',
+	'Modifying_Letter',
+	'Tone_Letter',
+	'Tone_Mark',
+	'Gemination_Mark',
+	'Cantillation_Mark',
+	'Register_Shifter',
+	'Syllable_Modifier',
+	'Consonant_Killer',
+	'Non_Joiner',
+	'Joiner',
+	'Number_Joiner',
+	'Number',
+	'Brahmi_Joining_Number',
+	# Indic_Positional_Category
+	'Not_Applicable',
+	'Right',
+	'Left',
+	'Visual_Order_Left',
+	'Left_And_Right',
+	'Top',
+	'Bottom',
+	'Top_And_Bottom',
+	'Top_And_Right',
+	'Top_And_Left',
+	'Top_And_Left_And_Right',
+	'Bottom_And_Right',
+	'Top_And_Bottom_And_Right',
+	'Overstruck',
+]
+
+class PropertyValue(object):
+	def __init__(self, name_):
+		self.name = name_
+	def __str__(self):
+		return self.name
+	def __eq__(self, other):
+		return self.name == (other if isinstance(other, basestring) else other.name)
+	def __ne__(self, other):
+		return not (self == other)
+
+property_values = {}
+
+for name in property_names:
+	value = PropertyValue(name)
+	assert value not in property_values
+	assert value not in globals()
+	property_values[name] = value
+globals().update(property_values)
+
+
+def is_BASE(U, UISC, UGC):
+	return (UISC in [Number, Consonant, Consonant_Head_Letter,
+			#SPEC-OUTDATED Consonant_Placeholder,
+			Tone_Letter] or
+		(UGC == Lo and UISC in [Avagraha, Bindu, Consonant_Final, Consonant_Medial,
+					Consonant_Subjoined, Vowel, Vowel_Dependent]))
+def is_BASE_VOWEL(U, UISC, UGC):
+	return UISC == Vowel_Independent
+def is_BASE_IND(U, UISC, UGC):
+	#SPEC-BROKEN return (UISC in [Consonant_Dead, Modifying_Letter] or UGC == Po)
+	return (UISC in [Consonant_Dead, Modifying_Letter] or
+		(UGC == Po and not is_BASE_OTHER(U, UISC, UGC))) # for 104E
+def is_BASE_NUM(U, UISC, UGC):
+	return UISC == Brahmi_Joining_Number
+def is_BASE_OTHER(U, UISC, UGC):
+	if UISC == Consonant_Placeholder: return True #SPEC-OUTDATED
+	return U in [0x00A0, 0x00D7, 0x2015, 0x2022, 0x25CC,
+		     0x25FB, 0x25FC, 0x25FD, 0x25FE]
+def is_CGJ(U, UISC, UGC):
+	return U == 0x034F
+def is_CONS_FINAL(U, UISC, UGC):
+	return ((UISC == Consonant_Final and UGC != Lo) or
+		UISC == Consonant_Succeeding_Repha)
+def is_CONS_FINAL_MOD(U, UISC, UGC):
+	#SPEC-OUTDATED return  UISC in [Consonant_Final_Modifier, Syllable_Modifier]
+	return  UISC == Syllable_Modifier
+def is_CONS_MED(U, UISC, UGC):
+	return UISC == Consonant_Medial and UGC != Lo
+def is_CONS_MOD(U, UISC, UGC):
+	return UISC in [Nukta, Gemination_Mark, Consonant_Killer]
+def is_CONS_SUB(U, UISC, UGC):
+	#SPEC-OUTDATED return UISC == Consonant_Subjoined
+	return UISC == Consonant_Subjoined and UGC != Lo
+def is_HALANT(U, UISC, UGC):
+	return UISC in [Virama, Invisible_Stacker]
+def is_HALANT_NUM(U, UISC, UGC):
+	return UISC == Number_Joiner
+def is_ZWNJ(U, UISC, UGC):
+	return UISC == Non_Joiner
+def is_ZWJ(U, UISC, UGC):
+	return UISC == Joiner
+def is_Word_Joiner(U, UISC, UGC):
+	return U == 0x2060
+def is_OTHER(U, UISC, UGC):
+	#SPEC-OUTDATED return UGC == Zs # or any other SCRIPT_COMMON characters
+	return (UISC == Other
+		and not is_SYM_MOD(U, UISC, UGC)
+		and not is_CGJ(U, UISC, UGC)
+		and not is_Word_Joiner(U, UISC, UGC)
+		and not is_VARIATION_SELECTOR(U, UISC, UGC)
+	)
+def is_Reserved(U, UISC, UGC):
+	return UGC == 'Cn'
+def is_REPHA(U, UISC, UGC):
+	#return UISC == Consonant_Preceding_Repha
+	#SPEC-OUTDATED hack to categorize Consonant_With_Stacker and Consonant_Prefixed
+	return UISC in [Consonant_Preceding_Repha, Consonant_With_Stacker, Consonant_Prefixed]
+def is_SYM(U, UISC, UGC):
+	if U == 0x25CC: return False #SPEC-OUTDATED
+	#SPEC-OUTDATED return UGC in [So, Sc] or UISC == Symbol_Letter
+	return UGC in [So, Sc]
+def is_SYM_MOD(U, UISC, UGC):
+	return U in [0x1B6B, 0x1B6C, 0x1B6D, 0x1B6E, 0x1B6F, 0x1B70, 0x1B71, 0x1B72, 0x1B73]
+def is_VARIATION_SELECTOR(U, UISC, UGC):
+	return 0xFE00 <= U <= 0xFE0F
+def is_VOWEL(U, UISC, UGC):
+	return (UISC == Pure_Killer or
+		(UGC != Lo and UISC in [Vowel, Vowel_Dependent]))
+def is_VOWEL_MOD(U, UISC, UGC):
+	return (UISC in [Tone_Mark, Cantillation_Mark, Register_Shifter, Visarga] or
+		(UGC != Lo and UISC == Bindu))
+
+use_mapping = {
+	'B':	is_BASE,
+	'IV':	is_BASE_VOWEL,
+	'IND':	is_BASE_IND,
+	'N':	is_BASE_NUM,
+	'GB':	is_BASE_OTHER,
+	'CGJ':	is_CGJ,
+	'F':	is_CONS_FINAL,
+	'FM':	is_CONS_FINAL_MOD,
+	'M':	is_CONS_MED,
+	'CM':	is_CONS_MOD,
+	'SUB':	is_CONS_SUB,
+	'H':	is_HALANT,
+	'HN':	is_HALANT_NUM,
+	'ZWNJ':	is_ZWNJ,
+	'ZWJ':	is_ZWJ,
+	'WJ':	is_Word_Joiner,
+	'O':	is_OTHER,
+	'Rsv':	is_Reserved,
+	'R':	is_REPHA,
+	'S':	is_SYM,
+	'SM':	is_SYM_MOD,
+	'VS':	is_VARIATION_SELECTOR,
+	'V':	is_VOWEL,
+	'VM':	is_VOWEL_MOD,
+}
+
+use_positions = {
+	'F': {
+		'Abv': [Top],
+		'Blw': [Bottom],
+		'Pst': [Right],
+	},
+	'M': {
+		'Abv': [Top],
+		'Blw': [Bottom],
+		'Pst': [Right],
+		'Pre': [Left],
+	},
+	'CM': {
+		'Abv': [Top],
+		'Blw': [Bottom],
+	},
+	'V': {
+		'Abv': [Top, Top_And_Bottom, Top_And_Bottom_And_Right, Top_And_Right],
+		'Blw': [Bottom, Overstruck, Bottom_And_Right],
+		'Pst': [Right],
+		'Pre': [Left, Top_And_Left, Top_And_Left_And_Right, Left_And_Right],
+	},
+	'VM': {
+		'Abv': [Top],
+		'Blw': [Bottom, Overstruck],
+		'Pst': [Right],
+		'Pre': [Left],
+	},
+	'SM': {
+		'Abv': [Top],
+		'Blw': [Bottom],
+	},
+	'H': None,
+	'B': None,
+	'FM': None,
+	'SUB': None,
+}
+
+def map_to_use(data):
+	out = {}
+	items = use_mapping.items()
+	for U,(UISC,UIPC,UGC,UBlock) in data.items():
+
+		# Resolve Indic_Syllabic_Category
+
+		# TODO: These don't have UISC assigned in Unicode 8.0, but
+		# have UIPC
+		if U == 0x17DD: UISC = Vowel_Dependent
+		if 0x1CE2 <= U <= 0x1CE8: UISC = Cantillation_Mark
+
+		# TODO: U+1CED should only be allowed after some of
+		# the nasalization marks, maybe only for U+1CE9..U+1CF1.
+		if U == 0x1CED: UISC = Tone_Mark
+
+		evals = [(k, v(U,UISC,UGC)) for k,v in items]
+		values = [k for k,v in evals if v]
+		assert len(values) == 1, "%s %s %s %s" % (hex(U), UISC, UGC, values)
+		USE = values[0]
+
+		# Resolve Indic_Positional_Category
+
+		# TODO: Not in Unicode 8.0 yet, but in spec.
+		if U == 0x1B6C: UIPC = Bottom
+
+		# TODO: These should die, but have UIPC in Unicode 8.0
+		if U in [0x953, 0x954]: UIPC = Not_Applicable
+
+		# TODO: In USE's override list but not in Unicode 8.0
+		if U == 0x103C: UIPC = Left
+
+		# TODO: These are not in USE's override list that we have, nor are they in Unicode 8.0
+		if 0xA926 <= U <= 0xA92A: UIPC = Top
+		if U == 0x111CA: UIPC = Bottom
+		if U == 0x11300: UIPC = Top
+		if U == 0x1133C: UIPC = Bottom
+		if U == 0x1171E: UIPC = Left # Correct?!
+		if 0x1CF2 <= U <= 0x1CF3: UIPC = Right
+		if 0x1CF8 <= U <= 0x1CF9: UIPC = Top
+
+		assert (UIPC in [Not_Applicable, Visual_Order_Left] or
+			USE in use_positions), "%s %s %s %s %s" % (hex(U), UIPC, USE, UISC, UGC)
+
+		pos_mapping = use_positions.get(USE, None)
+		if pos_mapping:
+			values = [k for k,v in pos_mapping.items() if v and UIPC in v]
+			assert len(values) == 1, "%s %s %s %s %s %s" % (hex(U), UIPC, USE, UISC, UGC, values)
+			USE = USE + values[0]
+
+		out[U] = (USE, UBlock)
+	return out
+
+defaults = ('O', 'No_Block')
+data = map_to_use(data)
+
+# Remove the outliers
+singles = {}
+for u in [0x034F, 0x25CC, 0x1107F]:
+	singles[u] = data[u]
+	del data[u]
+
+print "/* == Start of generated table == */"
+print "/*"
+print " * The following table is generated by running:"
+print " *"
+print " *   ./gen-use-table.py IndicSyllabicCategory.txt IndicPositionalCategory.txt UnicodeData.txt Blocks.txt"
+print " *"
+print " * on files with these headers:"
+print " *"
+for h in headers:
+	for l in h:
+		print " * %s" % (l.strip())
+print " */"
+print
+print '#include "hb-ot-shape-complex-use-private.hh"'
+print
+
+total = 0
+used = 0
+last_block = None
+def print_block (block, start, end, data):
+	global total, used, last_block
+	if block and block != last_block:
+		print
+		print
+		print "  /* %s */" % block
+		if start % 16:
+			print ' ' * (20 + (start % 16 * 6)),
+	num = 0
+	assert start % 8 == 0
+	assert (end+1) % 8 == 0
+	for u in range (start, end+1):
+		if u % 16 == 0:
+			print
+			print "  /* %04X */" % u,
+		if u in data:
+			num += 1
+		d = data.get (u, defaults)
+		sys.stdout.write ("%6s," % d[0])
+
+	total += end - start + 1
+	used += num
+	if block:
+		last_block = block
+
+uu = data.keys ()
+uu.sort ()
+
+last = -100000
+num = 0
+offset = 0
+starts = []
+ends = []
+for k,v in sorted(use_mapping.items()):
+	if k in use_positions and use_positions[k]: continue
+	print "#define %s	USE_%s	/* %s */" % (k, k, v.__name__[3:])
+for k,v in sorted(use_positions.items()):
+	if not v: continue
+	for suf in v.keys():
+		tag = k + suf
+		print "#define %s	USE_%s" % (tag, tag)
+print ""
+print "static const USE_TABLE_ELEMENT_TYPE use_table[] = {"
+for u in uu:
+	if u <= last:
+		continue
+	block = data[u][1]
+
+	start = u//8*8
+	end = start+1
+	while end in uu and block == data[end][1]:
+		end += 1
+	end = (end-1)//8*8 + 7
+
+	if start != last + 1:
+		if start - last <= 1+16*3:
+			print_block (None, last+1, start-1, data)
+			last = start-1
+		else:
+			if last >= 0:
+				ends.append (last + 1)
+				offset += ends[-1] - starts[-1]
+			print
+			print
+			print "#define use_offset_0x%04xu %d" % (start, offset)
+			starts.append (start)
+
+	print_block (block, start, end, data)
+	last = end
+ends.append (last + 1)
+offset += ends[-1] - starts[-1]
+print
+print
+occupancy = used * 100. / total
+page_bits = 12
+print "}; /* Table items: %d; occupancy: %d%% */" % (offset, occupancy)
+print
+print "USE_TABLE_ELEMENT_TYPE"
+print "hb_use_get_categories (hb_codepoint_t u)"
+print "{"
+print "  switch (u >> %d)" % page_bits
+print "  {"
+pages = set([u>>page_bits for u in starts+ends+singles.keys()])
+for p in sorted(pages):
+	print "    case 0x%0Xu:" % p
+	for (start,end) in zip (starts, ends):
+		if p not in [start>>page_bits, end>>page_bits]: continue
+		offset = "use_offset_0x%04xu" % start
+		print "      if (hb_in_range (u, 0x%04Xu, 0x%04Xu)) return use_table[u - 0x%04Xu + %s];" % (start, end-1, start, offset)
+	for u,d in singles.items ():
+		if p != u>>page_bits: continue
+		print "      if (unlikely (u == 0x%04Xu)) return %s;" % (u, d[0])
+	print "      break;"
+	print ""
+print "    default:"
+print "      break;"
+print "  }"
+print "  return USE_O;"
+print "}"
+print
+for k in sorted(use_mapping.keys()):
+	if k in use_positions and use_positions[k]: continue
+	print "#undef %s" % k
+for k,v in sorted(use_positions.items()):
+	if not v: continue
+	for suf in v.keys():
+		tag = k + suf
+		print "#undef %s" % tag
+print
+print "/* == End of generated table == */"
+
+# Maintain at least 50% occupancy in the table */
+if occupancy < 50:
+	raise Exception ("Table too sparse, please investigate: ", occupancy)

src/harfbuzz.pc.in

@@ -8,4 +8,6 @@ Description: HarfBuzz text shaping library
 Version: %VERSION%
 
 Libs: -L${libdir} -lharfbuzz
+Libs.private: %libs_private%
+Requires.private: %requires_private%
 Cflags: -I${includedir}/harfbuzz

src/hb-buffer-private.hh

@@ -50,6 +50,7 @@ struct hb_buffer_t {
   /* Information about how the text in the buffer should be treated */
   hb_unicode_funcs_t *unicode; /* Unicode functions */
   hb_buffer_flags_t flags; /* BOT / EOT / etc. */
+  hb_buffer_cluster_level_t cluster_level;
   hb_codepoint_t replacement; /* U+FFFD or something else. */
 
   /* Buffer contents */
@@ -151,24 +152,6 @@ struct hb_buffer_t {
 
     idx++;
   }
-  inline void
-  next_glyphs (unsigned int count)
-  {
-    if (have_output)
-    {
-      if (unlikely (out_info != info || out_len != idx)) {
-	if (unlikely (!make_room_for (count, count))) return;
-	{
-	  while (count--)
-	    out_info[out_len++] = info[idx++];
-	  return;
-	}
-      }
-      out_len += count;
-    }
-
-    idx += count;
-  }
 
   /* Advance idx without copying to output. */
   inline void skip_glyph (void) { idx++; }
@@ -189,7 +172,14 @@ struct hb_buffer_t {
 			      unsigned int cluster_end);
 
   HB_INTERNAL void merge_clusters (unsigned int start,
-				   unsigned int end);
+				   unsigned int end)
+  {
+    if (end - start < 2)
+      return;
+    merge_clusters_impl (start, end);
+  }
+  HB_INTERNAL void merge_clusters_impl (unsigned int start,
+					unsigned int end);
   HB_INTERNAL void merge_out_clusters (unsigned int start,
 				       unsigned int end);
   /* Merge clusters for deleting current glyph, and skip it. */

src/hb-buffer.cc

@@ -504,14 +504,10 @@ hb_buffer_t::reverse_clusters (void)
 }
 
 void
-hb_buffer_t::merge_clusters (unsigned int start,
+hb_buffer_t::merge_clusters_impl (unsigned int start,
-			     unsigned int end)
+				  unsigned int end)
 {
-#ifdef HB_NO_MERGE_CLUSTERS
+  if (cluster_level == HB_BUFFER_CLUSTER_LEVEL_CHARACTERS)
-  return;
-#endif
-
-  if (unlikely (end - start < 2))
     return;
 
   unsigned int cluster = info[start].cluster;
@@ -539,9 +535,8 @@ void
 hb_buffer_t::merge_out_clusters (unsigned int start,
 				 unsigned int end)
 {
-#ifdef HB_NO_MERGE_CLUSTERS
+  if (cluster_level == HB_BUFFER_CLUSTER_LEVEL_CHARACTERS)
-  return;
+    return;
-#endif
 
   if (unlikely (end - start < 2))
     return;
@@ -741,6 +736,7 @@ hb_buffer_get_empty (void)
 
     const_cast<hb_unicode_funcs_t *> (&_hb_unicode_funcs_nil),
     HB_BUFFER_FLAG_DEFAULT,
+    HB_BUFFER_CLUSTER_LEVEL_DEFAULT,
     HB_BUFFER_REPLACEMENT_CODEPOINT_DEFAULT,
 
     HB_BUFFER_CONTENT_TYPE_INVALID,
@@ -1086,6 +1082,41 @@ hb_buffer_get_flags (hb_buffer_t *buffer)
   return buffer->flags;
 }
 
+/**
+ * hb_buffer_set_cluster_level:
+ * @buffer: a buffer.
+ * @cluster_level: 
+ *
+ * 
+ *
+ * Since: 0.9.42
+ **/
+void
+hb_buffer_set_cluster_level (hb_buffer_t       *buffer,
+		     hb_buffer_cluster_level_t  cluster_level)
+{
+  if (unlikely (hb_object_is_inert (buffer)))
+    return;
+
+  buffer->cluster_level = cluster_level;
+}
+
+/**
+ * hb_buffer_get_cluster_level:
+ * @buffer: a buffer.
+ *
+ * 
+ *
+ * Return value: 
+ *
+ * Since: 0.9.42
+ **/
+hb_buffer_cluster_level_t
+hb_buffer_get_cluster_level (hb_buffer_t *buffer)
+{
+  return buffer->cluster_level;
+}
+
 
 /**
  * hb_buffer_set_replacement_codepoint:

src/hb-buffer.h

@@ -185,7 +185,19 @@ hb_buffer_set_flags (hb_buffer_t       *buffer,
 hb_buffer_flags_t
 hb_buffer_get_flags (hb_buffer_t *buffer);
 
+typedef enum {
+  HB_BUFFER_CLUSTER_LEVEL_MONOTONE_GRAPHEMES	= 0,
+  HB_BUFFER_CLUSTER_LEVEL_MONOTONE_CHARACTERS	= 1,
+  HB_BUFFER_CLUSTER_LEVEL_CHARACTERS		= 2,
+  HB_BUFFER_CLUSTER_LEVEL_DEFAULT = HB_BUFFER_CLUSTER_LEVEL_MONOTONE_GRAPHEMES
+} hb_buffer_cluster_level_t;
 
+void
+hb_buffer_set_cluster_level (hb_buffer_t               *buffer,
+			     hb_buffer_cluster_level_t  cluster_level);
+
+hb_buffer_cluster_level_t
+hb_buffer_get_cluster_level (hb_buffer_t *buffer);
 
 #define HB_BUFFER_REPLACEMENT_CODEPOINT_DEFAULT 0xFFFDu
 

src/hb-common.cc

@@ -493,6 +493,9 @@ hb_script_get_horizontal_direction (hb_script_t script)
     case HB_SCRIPT_PALMYRENE:
     case HB_SCRIPT_PSALTER_PAHLAVI:
 
+    /* Unicode-8.0 additions */
+    case HB_SCRIPT_OLD_HUNGARIAN:
+
       return HB_DIRECTION_RTL;
   }
 

src/hb-common.h

@@ -296,6 +296,13 @@ typedef enum
   /*7.0*/ HB_SCRIPT_TIRHUTA			= HB_TAG ('T','i','r','h'),
   /*7.0*/ HB_SCRIPT_WARANG_CITI			= HB_TAG ('W','a','r','a'),
 
+  /*8.0*/ HB_SCRIPT_AHOM                        = HB_TAG ('A','h','o','m'),
+  /*8.0*/ HB_SCRIPT_ANATOLIAN_HIEROGLYPHS       = HB_TAG ('H','l','u','w'),
+  /*8.0*/ HB_SCRIPT_HATRAN                      = HB_TAG ('H','a','t','r'),
+  /*8.0*/ HB_SCRIPT_MULTANI                     = HB_TAG ('M','u','l','t'),
+  /*8.0*/ HB_SCRIPT_OLD_HUNGARIAN               = HB_TAG ('H','u','n','g'),
+  /*8.0*/ HB_SCRIPT_SIGNWRITING                 = HB_TAG ('S','g','n','w'),
+
   /* No script set. */
   HB_SCRIPT_INVALID				= HB_TAG_NONE,
 

src/hb-graphite2.cc

@@ -228,12 +228,11 @@ _hb_graphite2_shape (hb_shape_plan_t    *shape_plan,
   int lang_len = lang_end ? lang_end - lang : -1;
   gr_feature_val *feats = gr_face_featureval_for_lang (grface, lang ? hb_tag_from_string (lang, lang_len) : 0);
 
-  while (num_features--)
+  for (unsigned int i = 0; i < num_features; i++)
   {
-    const gr_feature_ref *fref = gr_face_find_fref (grface, features->tag);
+    const gr_feature_ref *fref = gr_face_find_fref (grface, features[i].tag);
     if (fref)
-      gr_fref_set_feature_value (fref, features->value, feats);
+      gr_fref_set_feature_value (fref, features[i].value, feats);
-    features++;
   }
 
   gr_segment *seg = NULL;
@@ -249,6 +248,8 @@ _hb_graphite2_shape (hb_shape_plan_t    *shape_plan,
   for (unsigned int i = 0; i < buffer->len; ++i)
     chars[i] = buffer->info[i].codepoint;
 
+  /* TODO ensure_native_direction. */
+
   hb_tag_t script_tag[2];
   hb_ot_tags_from_script (hb_buffer_get_script (buffer), &script_tag[0], &script_tag[1]);
 
@@ -267,9 +268,11 @@ _hb_graphite2_shape (hb_shape_plan_t    *shape_plan,
   if (unlikely (!glyph_count)) {
     if (feats) gr_featureval_destroy (feats);
     gr_seg_destroy (seg);
-    return false;
+    buffer->len = 0;
+    return true;
   }
 
+  buffer->ensure (glyph_count);
   scratch = buffer->get_scratch_buffer (&scratch_size);
   while ((DIV_CEIL (sizeof (hb_graphite2_cluster_t) * buffer->len, sizeof (*scratch)) +
 	  DIV_CEIL (sizeof (hb_codepoint_t) * glyph_count, sizeof (*scratch))) > scratch_size)
@@ -331,7 +334,6 @@ _hb_graphite2_shape (hb_shape_plan_t    *shape_plan,
   }
   ci++;
 
-  //buffer->clear_output ();
   for (unsigned int i = 0; i < ci; ++i)
   {
     for (unsigned int j = 0; j < clusters[i].num_glyphs; ++j)
@@ -342,32 +344,57 @@ _hb_graphite2_shape (hb_shape_plan_t    *shape_plan,
     }
   }
   buffer->len = glyph_count;
-  //buffer->swap_buffers ();
 
-  if (HB_DIRECTION_IS_BACKWARD(buffer->props.direction))
+  /* Positioning. */
-    curradvx = gr_seg_advance_X(seg);
+  if (!HB_DIRECTION_IS_BACKWARD(buffer->props.direction))
-
-  hb_glyph_position_t *pPos;
-  for (pPos = hb_buffer_get_glyph_positions (buffer, NULL), is = gr_seg_first_slot (seg);
-       is; pPos++, is = gr_slot_next_in_segment (is))
   {
-    pPos->x_offset = gr_slot_origin_X (is) - curradvx;
+    hb_glyph_position_t *pPos;
-    pPos->y_offset = gr_slot_origin_Y (is) - curradvy;
+    for (pPos = hb_buffer_get_glyph_positions (buffer, NULL), is = gr_seg_first_slot (seg);
-    pPos->x_advance = gr_slot_advance_X (is, grface, grfont);
+         is; pPos++, is = gr_slot_next_in_segment (is))
-    pPos->y_advance = gr_slot_advance_Y (is, grface, grfont);
+    {
-    if (HB_DIRECTION_IS_BACKWARD (buffer->props.direction))
+      pPos->x_offset = gr_slot_origin_X (is) - curradvx;
-      curradvx -= pPos->x_advance;
+      pPos->y_offset = gr_slot_origin_Y (is) - curradvy;
-    pPos->x_offset = gr_slot_origin_X (is) - curradvx;
+      pPos->x_advance = gr_slot_advance_X (is, grface, grfont);
-    if (!HB_DIRECTION_IS_BACKWARD (buffer->props.direction))
+      pPos->y_advance = gr_slot_advance_Y (is, grface, grfont);
       curradvx += pPos->x_advance;
-    pPos->y_offset = gr_slot_origin_Y (is) - curradvy;
+      curradvy += pPos->y_advance;
-    curradvy += pPos->y_advance;
+    }
-  }
-  if (!HB_DIRECTION_IS_BACKWARD (buffer->props.direction))
     pPos[-1].x_advance += gr_seg_advance_X(seg) - curradvx;
-
+  }
-  if (HB_DIRECTION_IS_BACKWARD (buffer->props.direction))
+  else
+  {
+    hb_glyph_position_t *pPos = hb_buffer_get_glyph_positions (buffer, NULL) + buffer->len - 1;
+    const hb_glyph_info_t *info = buffer->info + buffer->len - 1;
+    const hb_glyph_info_t *tinfo;
+    const gr_slot *tis;
+    int currclus = -1;
+    float clusx = 0., clusy = 0.;
+    for (is = gr_seg_last_slot (seg); is; pPos--, info--, is = gr_slot_prev_in_segment (is))
+    {
+      if (info->cluster != currclus)
+      {
+        curradvx += clusx;
+        curradvy += clusy;
+        currclus = info->cluster;
+        clusx = 0.;
+        clusy = 0.;
+        for (tis = is, tinfo = info; tis && tinfo->cluster == currclus; tis = gr_slot_prev_in_segment (tis), tinfo--)
+        {
+          clusx += gr_slot_advance_X (tis, grface, grfont);
+          clusy += gr_slot_advance_Y (tis, grface, grfont);
+        }
+        curradvx += clusx;
+        curradvy += clusy;
+      }
+      pPos->x_advance = gr_slot_advance_X (is, grface, grfont);
+      pPos->y_advance = gr_slot_advance_Y (is, grface, grfont);
+      curradvx -= pPos->x_advance;
+      curradvy -= pPos->y_advance;
+      pPos->x_offset = gr_slot_origin_X (is) - curradvx;
+      pPos->y_offset = gr_slot_origin_Y (is) - curradvy;
+    }
     hb_buffer_reverse_clusters (buffer);
+  }
 
   if (feats) gr_featureval_destroy (feats);
   gr_seg_destroy (seg);

src/hb-ot-layout-gsubgpos-private.hh

@@ -328,8 +328,7 @@ struct hb_apply_context_t
 
       if (unlikely (_hb_glyph_info_is_default_ignorable (&info) &&
 		    (ignore_zwnj || !_hb_glyph_info_is_zwnj (&info)) &&
-		    (ignore_zwj || !_hb_glyph_info_is_zwj (&info)) &&
+		    (ignore_zwj || !_hb_glyph_info_is_zwj (&info))))
-		    !_hb_glyph_info_ligated (&info)))
 	return SKIP_MAYBE;
 
       return SKIP_NO;
@@ -720,7 +719,7 @@ static inline bool match_input (hb_apply_context_t *c,
 {
   TRACE_APPLY (NULL);
 
-  if (unlikely (count > MAX_CONTEXT_LENGTH)) TRACE_RETURN (false);
+  if (unlikely (count > MAX_CONTEXT_LENGTH)) return TRACE_RETURN (false);
 
   hb_buffer_t *buffer = c->buffer;
 
@@ -2165,7 +2164,7 @@ struct ExtensionFormat1
   {
     TRACE_DISPATCH (this, format);
     if (unlikely (!c->may_dispatch (this, this))) TRACE_RETURN (c->default_return_value ());
-    return get_subtable<typename T::LookupSubTable> ().dispatch (c, get_type ());
+    return TRACE_RETURN (get_subtable<typename T::LookupSubTable> ().dispatch (c, get_type ()));
   }
 
   /* This is called from may_dispatch() above with hb_sanitize_context_t. */

src/hb-ot-layout-private.hh

@@ -36,6 +36,15 @@
 #include "hb-set-private.hh"
 
 
+/* Private API corresponding to hb-ot-layout.h: */
+
+HB_INTERNAL hb_bool_t
+hb_ot_layout_table_find_feature (hb_face_t    *face,
+				 hb_tag_t      table_tag,
+				 hb_tag_t      feature_tag,
+				 unsigned int *feature_index);
+
+
 /*
  * GDEF
  */
@@ -179,6 +188,30 @@ _hb_ot_layout_destroy (hb_ot_layout_t *layout);
 #define lig_props()		var1.u8[2] /* GSUB/GPOS ligature tracking */
 #define syllable()		var1.u8[3] /* GSUB/GPOS shaping boundaries */
 
+
+/* loop over syllables */
+
+#define foreach_syllable(buffer, start, end) \
+  for (unsigned int \
+       _count = buffer->len, \
+       start = 0, end = _count ? _next_syllable (buffer, 0) : 0; \
+       start < _count; \
+       start = end, end = _next_syllable (buffer, start))
+
+static inline unsigned int
+_next_syllable (hb_buffer_t *buffer, unsigned int start)
+{
+  hb_glyph_info_t *info = buffer->info;
+  unsigned int count = buffer->len;
+
+  unsigned int syllable = info[start].syllable();
+  while (++start < count && syllable == info[start].syllable())
+    ;
+
+  return start;
+}
+
+
 /* unicode_props */
 
 enum {
@@ -225,10 +258,12 @@ _hb_glyph_info_get_modified_combining_class (const hb_glyph_info_t *info)
   return info->unicode_props1();
 }
 
+static inline bool _hb_glyph_info_ligated (const hb_glyph_info_t *info);
+
 static inline hb_bool_t
 _hb_glyph_info_is_default_ignorable (const hb_glyph_info_t *info)
 {
-  return !!(info->unicode_props0() & MASK0_IGNORABLE);
+  return (info->unicode_props0() & MASK0_IGNORABLE) && !_hb_glyph_info_ligated (info);
 }
 
 static inline hb_bool_t
@@ -406,6 +441,14 @@ _hb_glyph_info_clear_ligated_and_multiplied (hb_glyph_info_t *info)
 			   HB_OT_LAYOUT_GLYPH_PROPS_MULTIPLIED);
 }
 
+static inline void
+_hb_glyph_info_clear_substituted_and_ligated_and_multiplied (hb_glyph_info_t *info)
+{
+  info->glyph_props() &= ~(HB_OT_LAYOUT_GLYPH_PROPS_SUBSTITUTED |
+			   HB_OT_LAYOUT_GLYPH_PROPS_LIGATED |
+			   HB_OT_LAYOUT_GLYPH_PROPS_MULTIPLIED);
+}
+
 
 /* Allocation / deallocation. */
 

src/hb-ot-layout.cc

@@ -291,6 +291,28 @@ hb_ot_layout_table_get_feature_tags (hb_face_t    *face,
   return g.get_feature_tags (start_offset, feature_count, feature_tags);
 }
 
+hb_bool_t
+hb_ot_layout_table_find_feature (hb_face_t    *face,
+				 hb_tag_t      table_tag,
+				 hb_tag_t      feature_tag,
+				 unsigned int *feature_index)
+{
+  ASSERT_STATIC (OT::Index::NOT_FOUND_INDEX == HB_OT_LAYOUT_NO_FEATURE_INDEX);
+  const OT::GSUBGPOS &g = get_gsubgpos_table (face, table_tag);
+
+  unsigned int num_features = g.get_feature_count ();
+  for (unsigned int i = 0; i < num_features; i++)
+  {
+    if (feature_tag == g.get_feature_tag (i)) {
+      if (feature_index) *feature_index = i;
+      return true;
+    }
+  }
+
+  if (feature_index) *feature_index = HB_OT_LAYOUT_NO_FEATURE_INDEX;
+  return false;
+}
+
 
 unsigned int
 hb_ot_layout_script_get_language_tags (hb_face_t    *face,
@@ -905,7 +927,7 @@ apply_backward (OT::hb_apply_context_t *c,
 
 struct hb_apply_forward_context_t
 {
-  inline const char *get_name (void) { return "APPLY_FORWARD"; }
+  inline const char *get_name (void) { return "APPLY_FWD"; }
   static const unsigned int max_debug_depth = HB_DEBUG_APPLY;
   typedef bool return_t;
   template <typename T, typename F>

src/hb-ot-map-private.hh

@@ -154,9 +154,10 @@ struct hb_ot_map_t
 
 enum hb_ot_map_feature_flags_t {
   F_NONE		= 0x0000u,
-  F_GLOBAL		= 0x0001u,
+  F_GLOBAL		= 0x0001u, /* Feature applies to all characters; results in no mask allocated for it. */
-  F_HAS_FALLBACK	= 0x0002u,
+  F_HAS_FALLBACK	= 0x0002u, /* Has fallback implementation, so include mask bit even if feature not found. */
-  F_MANUAL_ZWJ		= 0x0004u
+  F_MANUAL_ZWJ		= 0x0004u, /* Don't skip over ZWJ when matching. */
+  F_GLOBAL_SEARCH	= 0x0008u  /* If feature not found in LangSys, look for it in global feature list and pick one. */
 };
 /* Macro version for where const is desired. */
 #define F_COMBINE(l,r) (hb_ot_map_feature_flags_t ((unsigned int) (l) | (unsigned int) (r)))

src/hb-ot-map.cc

@@ -216,6 +216,16 @@ hb_ot_map_builder_t::compile (hb_ot_map_t &m)
 						   info->tag,
 						   &feature_index[table_index]);
     }
+    if (!found && (info->flags & F_GLOBAL_SEARCH))
+    {
+      for (unsigned int table_index = 0; table_index < 2; table_index++)
+      {
+	found |= hb_ot_layout_table_find_feature (face,
+						  table_tags[table_index],
+						  info->tag,
+						  &feature_index[table_index]);
+      }
+    }
     if (!found && !(info->flags & F_HAS_FALLBACK))
       continue;
 

src/hb-ot-shape-complex-arabic-private.hh

@@ -0,0 +1,50 @@
+/*
+ * Copyright © 2015  Mozilla Foundation.
+ * Copyright © 2015  Google, Inc.
+ *
+ *  This is part of HarfBuzz, a text shaping library.
+ *
+ * Permission is hereby granted, without written agreement and without
+ * license or royalty fees, to use, copy, modify, and distribute this
+ * software and its documentation for any purpose, provided that the
+ * above copyright notice and the following two paragraphs appear in
+ * all copies of this software.
+ *
+ * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
+ * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
+ * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ *
+ * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
+ * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+ *
+ * Mozilla Author(s): Jonathan Kew
+ * Google Author(s): Behdad Esfahbod
+ */
+
+#ifndef HB_OT_SHAPE_COMPLEX_ARABIC_PRIVATE_HH
+#define HB_OT_SHAPE_COMPLEX_ARABIC_PRIVATE_HH
+
+#include "hb-private.hh"
+
+#include "hb-ot-shape-complex-private.hh"
+
+
+struct arabic_shape_plan_t;
+
+HB_INTERNAL void *
+data_create_arabic (const hb_ot_shape_plan_t *plan);
+
+HB_INTERNAL void
+data_destroy_arabic (void *data);
+
+HB_INTERNAL void
+setup_masks_arabic_plan (const arabic_shape_plan_t *arabic_plan,
+			 hb_buffer_t               *buffer,
+			 hb_script_t                script);
+
+#endif /* HB_OT_SHAPE_COMPLEX_ARABIC_PRIVATE_HH */

src/hb-ot-shape-complex-arabic-table.hh

@@ -6,10 +6,10 @@
  *
  * on files with these headers:
  *
- * # ArabicShaping-7.0.0.txt
+ * # ArabicShaping-8.0.0.txt
- * # Date: 2014-02-14, 21:00:00 GMT [RP, KW, LI]
+ * # Date: 2015-02-17, 23:33:00 GMT [RP]
- * # Blocks-7.0.0.txt
+ * # Blocks-8.0.0.txt
- * # Date: 2014-04-03, 23:23:00 GMT [RP, KW]
+ * # Date: 2014-11-10, 23:04:00 GMT [KW]
  * UnicodeData.txt does not have a header.
  */
 
@@ -76,9 +76,9 @@ static const uint8_t joining_table[] =
 
   /* Arabic Extended-A */
 
-  /* 08A0 */ D,D,D,D,D,D,D,D,D,D,R,R,R,U,R,D,D,R,R,
+  /* 08A0 */ D,D,D,D,D,D,D,D,D,D,R,R,R,U,R,D,D,R,R,D,D,
 
-#define joining_offset_0x1806u 691
+#define joining_offset_0x1806u 693
 
   /* Mongolian */
 
@@ -89,40 +89,40 @@ static const uint8_t joining_table[] =
   /* 1880 */ U,U,U,U,U,U,U,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,
   /* 18A0 */ D,D,D,D,D,D,D,D,D,X,D,
 
-#define joining_offset_0x200cu 856
+#define joining_offset_0x200cu 858
 
   /* General Punctuation */
 
   /* 2000 */                         U,C,
 
-#define joining_offset_0x2066u 858
+#define joining_offset_0x2066u 860
 
   /* General Punctuation */
 
   /* 2060 */             U,U,U,U,
 
-#define joining_offset_0xa840u 862
+#define joining_offset_0xa840u 864
 
   /* Phags-pa */
 
   /* A840 */ D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,
   /* A860 */ D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,D,L,U,
 
-#define joining_offset_0x10ac0u 914
+#define joining_offset_0x10ac0u 916
 
   /* Manichaean */
 
   /* 10AC0 */ D,D,D,D,D,R,U,R,U,R,R,U,U,L,R,R,R,R,R,D,D,D,D,L,D,D,D,D,D,R,D,D,
   /* 10AE0 */ D,R,U,U,R,X,X,X,X,X,X,D,D,D,D,R,
 
-#define joining_offset_0x10b80u 962
+#define joining_offset_0x10b80u 964
 
   /* Psalter Pahlavi */
 
   /* 10B80 */ D,R,D,R,R,R,D,D,D,R,D,D,R,D,R,R,D,R,X,X,X,X,X,X,X,X,X,X,X,X,X,X,
   /* 10BA0 */ X,X,X,X,X,X,X,X,X,R,R,R,R,D,D,U,
 
-}; /* Table items: 1010; occupancy: 57% */
+}; /* Table items: 1012; occupancy: 57% */
 
 
 static unsigned int
@@ -131,7 +131,7 @@ joining_type (hb_codepoint_t u)
   switch (u >> 12)
   {
     case 0x0u:
-      if (hb_in_range (u, 0x0600u, 0x08B2u)) return joining_table[u - 0x0600u + joining_offset_0x0600u];
+      if (hb_in_range (u, 0x0600u, 0x08B4u)) return joining_table[u - 0x0600u + joining_offset_0x0600u];
       break;
 
     case 0x1u:

src/hb-ot-shape-complex-arabic-win1256.hh

@@ -142,7 +142,7 @@
 		OT_UARRAY(Name##Substitute, OT_LIST(ToGlyphs)) \
 	) \
 	OT_COVERAGE1(Name##Coverage, OT_LIST(FromGlyphs)) \
-	/* ASSERT_STATIC_EXPR len(FromGlyphs) == len(ToGlyphs) */
+	/* ASSERT_STATIC_EXPR_ZERO (len(FromGlyphs) == len(ToGlyphs)) */
 
 #define OT_SUBLOOKUP_LIGATURE_SUBST_FORMAT1(Name, FirstGlyphs, LigatureSetOffsets) \
 	OT_SUBLOOKUP(Name, 1, \
@@ -151,7 +151,7 @@
 		OT_UARRAY(Name##LigatureSetOffsetsArray, OT_LIST(LigatureSetOffsets)) \
 	) \
 	OT_COVERAGE1(Name##Coverage, OT_LIST(FirstGlyphs)) \
-	/* ASSERT_STATIC_EXPR len(FirstGlyphs) == len(LigatureSetOffsets) */
+	/* ASSERT_STATIC_EXPR_ZERO (len(FirstGlyphs) == len(LigatureSetOffsets)) */
 
 #define OT_LIGATURE_SET(Name, LigatureSetOffsets) \
 	OT_UARRAY(Name, OT_LIST(LigatureSetOffsets))

src/hb-ot-shape-complex-arabic.cc

@@ -24,7 +24,7 @@
  * Google Author(s): Behdad Esfahbod
  */
 
-#include "hb-ot-shape-complex-private.hh"
+#include "hb-ot-shape-complex-arabic-private.hh"
 #include "hb-ot-shape-private.hh"
 
 
@@ -32,10 +32,14 @@
 #define arabic_shaping_action() complex_var_u8_0() /* arabic shaping action */
 
 
+/*
+ * Joining types:
+ */
+
 /*
  * Bits used in the joining tables
  */
-enum {
+enum hb_arabic_joining_type_t {
   JOINING_TYPE_U		= 0,
   JOINING_TYPE_L		= 1,
   JOINING_TYPE_R		= 2,
@@ -49,10 +53,6 @@ enum {
   JOINING_TYPE_X = 8  /* means: use general-category to choose between U or T. */
 };
 
-/*
- * Joining types:
- */
-
 #include "hb-ot-shape-complex-arabic-table.hh"
 
 static unsigned int get_joining_type (hb_codepoint_t u, hb_unicode_general_category_t gen_cat)
@@ -61,7 +61,7 @@ static unsigned int get_joining_type (hb_codepoint_t u, hb_unicode_general_categ
   if (likely (j_type != JOINING_TYPE_X))
     return j_type;
 
-  return (FLAG(gen_cat) &
+  return (FLAG_SAFE(gen_cat) &
 	  (FLAG(HB_UNICODE_GENERAL_CATEGORY_NON_SPACING_MARK) |
 	   FLAG(HB_UNICODE_GENERAL_CATEGORY_ENCLOSING_MARK) |
 	   FLAG(HB_UNICODE_GENERAL_CATEGORY_FORMAT))
@@ -212,7 +212,7 @@ struct arabic_shape_plan_t
   arabic_fallback_plan_t *fallback_plan;
 };
 
-static void *
+void *
 data_create_arabic (const hb_ot_shape_plan_t *plan)
 {
   arabic_shape_plan_t *arabic_plan = (arabic_shape_plan_t *) calloc (1, sizeof (arabic_shape_plan_t));
@@ -230,7 +230,7 @@ data_create_arabic (const hb_ot_shape_plan_t *plan)
   return arabic_plan;
 }
 
-static void
+void
 data_destroy_arabic (void *data)
 {
   arabic_shape_plan_t *arabic_plan = (arabic_shape_plan_t *) data;
@@ -305,17 +305,15 @@ mongolian_variation_selectors (hb_buffer_t *buffer)
       info[i].arabic_shaping_action() = info[i - 1].arabic_shaping_action();
 }
 
-static void
+void
-setup_masks_arabic (const hb_ot_shape_plan_t *plan,
+setup_masks_arabic_plan (const arabic_shape_plan_t *arabic_plan,
-		    hb_buffer_t              *buffer,
+			 hb_buffer_t               *buffer,
-		    hb_font_t                *font HB_UNUSED)
+			 hb_script_t                script)
 {
   HB_BUFFER_ALLOCATE_VAR (buffer, arabic_shaping_action);
 
-  const arabic_shape_plan_t *arabic_plan = (const arabic_shape_plan_t *) plan->data;
-
   arabic_joining (buffer);
-  if (plan->props.script == HB_SCRIPT_MONGOLIAN)
+  if (script == HB_SCRIPT_MONGOLIAN)
     mongolian_variation_selectors (buffer);
 
   unsigned int count = buffer->len;
@@ -326,6 +324,15 @@ setup_masks_arabic (const hb_ot_shape_plan_t *plan,
   HB_BUFFER_DEALLOCATE_VAR (buffer, arabic_shaping_action);
 }
 
+static void
+setup_masks_arabic (const hb_ot_shape_plan_t *plan,
+		    hb_buffer_t              *buffer,
+		    hb_font_t                *font HB_UNUSED)
+{
+  const arabic_shape_plan_t *arabic_plan = (const arabic_shape_plan_t *) plan->data;
+  setup_masks_arabic_plan (arabic_plan, buffer, plan->props.script);
+}
+
 
 static void
 nuke_joiners (const hb_ot_shape_plan_t *plan HB_UNUSED,

src/hb-ot-shape-complex-hangul.cc

@@ -209,13 +209,8 @@ preprocess_text_hangul (const hb_ot_shape_plan_t *plan,
 	  hb_glyph_info_t tone = info[end];
 	  memmove (&info[start + 1], &info[start], (end - start) * sizeof (hb_glyph_info_t));
 	  info[start] = tone;
+	  buffer->merge_out_clusters (start, end + 1);
 	}
-	/* Merge clusters across the (possibly reordered) syllable+tone.
-	 * We want to merge even in the zero-width tone mark case here,
-	 * so that clustering behavior isn't dependent on how the tone mark
-	 * is handled by the font.
-	 */
-	buffer->merge_out_clusters (start, end + 1);
       }
       else
       {
@@ -296,7 +291,8 @@ preprocess_text_hangul (const hb_ot_shape_plan_t *plan,
 	}
 	else
 	  end = start + 2;
-	buffer->merge_out_clusters (start, end);
+	if (buffer->cluster_level == HB_BUFFER_CLUSTER_LEVEL_MONOTONE_GRAPHEMES)
+	  buffer->merge_out_clusters (start, end);
 	continue;
       }
     }
@@ -368,7 +364,8 @@ preprocess_text_hangul (const hb_ot_shape_plan_t *plan,
 	  info[i++].hangul_shaping_feature() = VJMO;
 	  if (i < end)
 	    info[i++].hangul_shaping_feature() = TJMO;
-	  buffer->merge_out_clusters (start, end);
+	  if (buffer->cluster_level == HB_BUFFER_CLUSTER_LEVEL_MONOTONE_GRAPHEMES)
+	    buffer->merge_out_clusters (start, end);
 	  continue;
 	}
       }

src/hb-ot-shape-complex-indic-private.hh

@@ -161,8 +161,6 @@ enum indic_matra_category_t {
   INDIC_MATRA_CATEGORY_VISUAL_ORDER_LEFT		= POS_PRE_M
 };
 
-/* Note: We use ASSERT_STATIC_EXPR_ZERO() instead of ASSERT_STATIC_EXPR() and the comma operation
- * because gcc fails to optimize the latter and fills the table in at runtime. */
 #define INDIC_COMBINE_CATEGORIES(S,M) \
   (ASSERT_STATIC_EXPR_ZERO (M == INDIC_MATRA_CATEGORY_NOT_APPLICABLE || \
 			    ( \

src/hb-ot-shape-complex-indic.cc

@@ -142,7 +142,7 @@ is_one_of (const hb_glyph_info_t &info, unsigned int flags)
 {
   /* If it ligated, all bets are off. */
   if (_hb_glyph_info_ligated (&info)) return false;
-  return !!(FLAG (info.indic_category()) & flags);
+  return !!(FLAG_SAFE (info.indic_category()) & flags);
 }
 
 static inline bool
@@ -237,7 +237,7 @@ set_indic_properties (hb_glyph_info_t &info)
    * Re-assign position.
    */
 
-  if ((FLAG (cat) & CONSONANT_FLAGS))
+  if ((FLAG_SAFE (cat) & CONSONANT_FLAGS))
   {
     pos = POS_BASE_C;
     if (is_ra (u))
@@ -247,7 +247,7 @@ set_indic_properties (hb_glyph_info_t &info)
   {
     pos = matra_position (u, pos);
   }
-  else if ((FLAG (cat) & (FLAG (OT_SM) | FLAG (OT_VD) | FLAG (OT_A) | FLAG (OT_Symbol))))
+  else if ((FLAG_SAFE (cat) & (FLAG (OT_SM) | FLAG (OT_VD) | FLAG (OT_A) | FLAG (OT_Symbol))))
   {
     pos = POS_SMVD;
   }
@@ -963,7 +963,7 @@ initial_reordering_consonant_syllable (const hb_ot_shape_plan_t *plan,
     indic_position_t last_pos = POS_START;
     for (unsigned int i = start; i < end; i++)
     {
-      if ((FLAG (info[i].indic_category()) & (JOINER_FLAGS | FLAG (OT_N) | FLAG (OT_RS) | MEDIAL_FLAGS | HALANT_OR_COENG_FLAGS)))
+      if ((FLAG_SAFE (info[i].indic_category()) & (JOINER_FLAGS | FLAG (OT_N) | FLAG (OT_RS) | MEDIAL_FLAGS | HALANT_OR_COENG_FLAGS)))
       {
 	info[i].indic_position() = last_pos;
 	if (unlikely (info[i].indic_category() == OT_H &&
@@ -1161,17 +1161,6 @@ initial_reordering_consonant_syllable (const hb_ot_shape_plan_t *plan,
     }
 }
 
-
-static void
-initial_reordering_vowel_syllable (const hb_ot_shape_plan_t *plan,
-				   hb_face_t *face,
-				   hb_buffer_t *buffer,
-				   unsigned int start, unsigned int end)
-{
-  /* We made the vowels look like consonants.  So let's call the consonant logic! */
-  initial_reordering_consonant_syllable (plan, face, buffer, start, end);
-}
-
 static void
 initial_reordering_standalone_cluster (const hb_ot_shape_plan_t *plan,
 				       hb_face_t *face,
@@ -1193,37 +1182,6 @@ initial_reordering_standalone_cluster (const hb_ot_shape_plan_t *plan,
   initial_reordering_consonant_syllable (plan, face, buffer, start, end);
 }
 
-static void
-initial_reordering_broken_cluster (const hb_ot_shape_plan_t *plan,
-				   hb_face_t *face,
-				   hb_buffer_t *buffer,
-				   unsigned int start, unsigned int end)
-{
-  /* We already inserted dotted-circles, so just call the standalone_cluster. */
-  initial_reordering_standalone_cluster (plan, face, buffer, start, end);
-}
-
-static void
-initial_reordering_symbol_cluster (const hb_ot_shape_plan_t *plan HB_UNUSED,
-				   hb_face_t *face HB_UNUSED,
-				   hb_buffer_t *buffer HB_UNUSED,
-				   unsigned int start HB_UNUSED, unsigned int end HB_UNUSED)
-{
-  /* Nothing to do right now.  If we ever switch to using the output
-   * buffer in the reordering process, we'd need to next_glyph() here. */
-}
-
-static void
-initial_reordering_non_indic_cluster (const hb_ot_shape_plan_t *plan HB_UNUSED,
-				      hb_face_t *face HB_UNUSED,
-				      hb_buffer_t *buffer HB_UNUSED,
-				      unsigned int start HB_UNUSED, unsigned int end HB_UNUSED)
-{
-  /* Nothing to do right now.  If we ever switch to using the output
-   * buffer in the reordering process, we'd need to next_glyph() here. */
-}
-
-
 static void
 initial_reordering_syllable (const hb_ot_shape_plan_t *plan,
 			     hb_face_t *face,
@@ -1231,13 +1189,21 @@ initial_reordering_syllable (const hb_ot_shape_plan_t *plan,
 			     unsigned int start, unsigned int end)
 {
   syllable_type_t syllable_type = (syllable_type_t) (buffer->info[start].syllable() & 0x0F);
-  switch (syllable_type) {
+  switch (syllable_type)
-  case consonant_syllable:	initial_reordering_consonant_syllable (plan, face, buffer, start, end); return;
+  {
-  case vowel_syllable:		initial_reordering_vowel_syllable     (plan, face, buffer, start, end); return;
+    case vowel_syllable: /* We made the vowels look like consonants.  So let's call the consonant logic! */
-  case standalone_cluster:	initial_reordering_standalone_cluster (plan, face, buffer, start, end); return;
+    case consonant_syllable:
-  case symbol_cluster:		initial_reordering_symbol_cluster     (plan, face, buffer, start, end); return;
+     initial_reordering_consonant_syllable (plan, face, buffer, start, end);
-  case broken_cluster:		initial_reordering_broken_cluster     (plan, face, buffer, start, end); return;
+     break;
-  case non_indic_cluster:	initial_reordering_non_indic_cluster  (plan, face, buffer, start, end); return;
+
+    case broken_cluster: /* We already inserted dotted-circles, so just call the standalone_cluster. */
+    case standalone_cluster:
+     initial_reordering_standalone_cluster (plan, face, buffer, start, end);
+     break;
+
+    case symbol_cluster:
+    case non_indic_cluster:
+      break;
   }
 }
 
@@ -1310,18 +1276,8 @@ initial_reordering (const hb_ot_shape_plan_t *plan,
   update_consonant_positions (plan, font, buffer);
   insert_dotted_circles (plan, font, buffer);
 
-  hb_glyph_info_t *info = buffer->info;
+  foreach_syllable (buffer, start, end)
-  unsigned int count = buffer->len;
+    initial_reordering_syllable (plan, font->face, buffer, start, end);
-  if (unlikely (!count)) return;
-  unsigned int last = 0;
-  unsigned int last_syllable = info[0].syllable();
-  for (unsigned int i = 1; i < count; i++)
-    if (last_syllable != info[i].syllable()) {
-      initial_reordering_syllable (plan, font->face, buffer, last, i);
-      last = i;
-      last_syllable = info[last].syllable();
-    }
-  initial_reordering_syllable (plan, font->face, buffer, last, count);
 }
 
 static void
@@ -1550,7 +1506,7 @@ final_reordering_syllable (const hb_ot_shape_plan_t *plan,
     {
       new_reph_pos = base;
       while (new_reph_pos < end &&
-	     !( FLAG (info[new_reph_pos + 1].indic_position()) & (FLAG (POS_POST_C) | FLAG (POS_AFTER_POST) | FLAG (POS_SMVD))))
+	     !( FLAG_SAFE (info[new_reph_pos + 1].indic_position()) & (FLAG (POS_POST_C) | FLAG (POS_AFTER_POST) | FLAG (POS_SMVD))))
 	new_reph_pos++;
       if (new_reph_pos < end)
         goto reph_move;
@@ -1701,7 +1657,7 @@ final_reordering_syllable (const hb_ot_shape_plan_t *plan,
   /* Apply 'init' to the Left Matra if it's a word start. */
   if (info[start].indic_position () == POS_PRE_M &&
       (!start ||
-       !(FLAG (_hb_glyph_info_get_general_category (&info[start - 1])) &
+       !(FLAG_SAFE (_hb_glyph_info_get_general_category (&info[start - 1])) &
 	 FLAG_RANGE (HB_UNICODE_GENERAL_CATEGORY_FORMAT, HB_UNICODE_GENERAL_CATEGORY_NON_SPACING_MARK))))
     info[start].mask |= indic_plan->mask_array[INIT];
 
@@ -1737,16 +1693,8 @@ final_reordering (const hb_ot_shape_plan_t *plan,
   unsigned int count = buffer->len;
   if (unlikely (!count)) return;
 
-  hb_glyph_info_t *info = buffer->info;
+  foreach_syllable (buffer, start, end)
-  unsigned int last = 0;
+    final_reordering_syllable (plan, buffer, start, end);
-  unsigned int last_syllable = info[0].syllable();
-  for (unsigned int i = 1; i < count; i++)
-    if (last_syllable != info[i].syllable()) {
-      final_reordering_syllable (plan, buffer, last, i);
-      last = i;
-      last_syllable = info[last].syllable();
-    }
-  final_reordering_syllable (plan, buffer, last, count);
 
   HB_BUFFER_DEALLOCATE_VAR (buffer, indic_category);
   HB_BUFFER_DEALLOCATE_VAR (buffer, indic_position);

src/hb-ot-shape-complex-myanmar.cc

@@ -154,7 +154,7 @@ is_one_of (const hb_glyph_info_t &info, unsigned int flags)
 {
   /* If it ligated, all bets are off. */
   if (_hb_glyph_info_ligated (&info)) return false;
-  return !!(FLAG (info.myanmar_category()) & flags);
+  return !!(FLAG_SAFE (info.myanmar_category()) & flags);
 }
 
 static inline bool
@@ -304,9 +304,7 @@ compare_myanmar_order (const hb_glyph_info_t *pa, const hb_glyph_info_t *pb)
  * http://www.microsoft.com/typography/OpenTypeDev/myanmar/intro.htm */
 
 static void
-initial_reordering_consonant_syllable (const hb_ot_shape_plan_t *plan,
+initial_reordering_consonant_syllable (hb_buffer_t *buffer,
-				       hb_face_t *face,
-				       hb_buffer_t *buffer,
 				       unsigned int start, unsigned int end)
 {
   hb_glyph_info_t *info = buffer->info;
@@ -398,37 +396,6 @@ initial_reordering_consonant_syllable (const hb_ot_shape_plan_t *plan,
   hb_bubble_sort (info + start, end - start, compare_myanmar_order);
 }
 
-static void
-initial_reordering_broken_cluster (const hb_ot_shape_plan_t *plan,
-				   hb_face_t *face,
-				   hb_buffer_t *buffer,
-				   unsigned int start, unsigned int end)
-{
-  /* We already inserted dotted-circles, so just call the consonant_syllable. */
-  initial_reordering_consonant_syllable (plan, face, buffer, start, end);
-}
-
-static void
-initial_reordering_punctuation_cluster (const hb_ot_shape_plan_t *plan HB_UNUSED,
-					hb_face_t *face HB_UNUSED,
-					hb_buffer_t *buffer HB_UNUSED,
-					unsigned int start HB_UNUSED, unsigned int end HB_UNUSED)
-{
-  /* Nothing to do right now.  If we ever switch to using the output
-   * buffer in the reordering process, we'd need to next_glyph() here. */
-}
-
-static void
-initial_reordering_non_myanmar_cluster (const hb_ot_shape_plan_t *plan HB_UNUSED,
-					hb_face_t *face HB_UNUSED,
-					hb_buffer_t *buffer HB_UNUSED,
-					unsigned int start HB_UNUSED, unsigned int end HB_UNUSED)
-{
-  /* Nothing to do right now.  If we ever switch to using the output
-   * buffer in the reordering process, we'd need to next_glyph() here. */
-}
-
-
 static void
 initial_reordering_syllable (const hb_ot_shape_plan_t *plan,
 			     hb_face_t *face,
@@ -437,10 +404,15 @@ initial_reordering_syllable (const hb_ot_shape_plan_t *plan,
 {
   syllable_type_t syllable_type = (syllable_type_t) (buffer->info[start].syllable() & 0x0F);
   switch (syllable_type) {
-  case consonant_syllable:	initial_reordering_consonant_syllable  (plan, face, buffer, start, end); return;
+
-  case punctuation_cluster:	initial_reordering_punctuation_cluster (plan, face, buffer, start, end); return;
+    case broken_cluster: /* We already inserted dotted-circles, so just call the consonant_syllable. */
-  case broken_cluster:		initial_reordering_broken_cluster      (plan, face, buffer, start, end); return;
+    case consonant_syllable:
-  case non_myanmar_cluster:	initial_reordering_non_myanmar_cluster (plan, face, buffer, start, end); return;
+      initial_reordering_consonant_syllable  (buffer, start, end);
+      break;
+
+    case punctuation_cluster:
+    case non_myanmar_cluster:
+      break;
   }
 }
 
@@ -505,18 +477,8 @@ initial_reordering (const hb_ot_shape_plan_t *plan,
 {
   insert_dotted_circles (plan, font, buffer);
 
-  hb_glyph_info_t *info = buffer->info;
+  foreach_syllable (buffer, start, end)
-  unsigned int count = buffer->len;
+    initial_reordering_syllable (plan, font->face, buffer, start, end);
-  if (unlikely (!count)) return;
-  unsigned int last = 0;
-  unsigned int last_syllable = info[0].syllable();
-  for (unsigned int i = 1; i < count; i++)
-    if (last_syllable != info[i].syllable()) {
-      initial_reordering_syllable (plan, font->face, buffer, last, i);
-      last = i;
-      last_syllable = info[last].syllable();
-    }
-  initial_reordering_syllable (plan, font->face, buffer, last, count);
 }
 
 static void

src/hb-ot-shape-complex-private.hh

@@ -59,9 +59,9 @@ enum hb_ot_shape_zero_width_marks_type_t {
   HB_COMPLEX_SHAPER_IMPLEMENT (myanmar_old) \
   HB_COMPLEX_SHAPER_IMPLEMENT (indic) \
   HB_COMPLEX_SHAPER_IMPLEMENT (myanmar) \
-  HB_COMPLEX_SHAPER_IMPLEMENT (sea) \
   HB_COMPLEX_SHAPER_IMPLEMENT (thai) \
   HB_COMPLEX_SHAPER_IMPLEMENT (tibetan) \
+  HB_COMPLEX_SHAPER_IMPLEMENT (use) \
   /* ^--- Add new shapers here */
 
 
@@ -217,61 +217,9 @@ hb_ot_shape_complex_categorize (const hb_ot_shape_planner_t *planner)
 
     /* ^--- Add new shapers here */
 
-
 #if 0
-    /* Note:
-     *
-     * These disabled scripts are listed in ucd/IndicSyllabicCategory.txt, but according
-     * to Martin Hosken and Jonathan Kew do not require complex shaping.
-     *
-     * TODO We should automate figuring out which scripts do not need complex shaping
-     *
-     * TODO We currently keep data for these scripts in our indic table.  Need to fix the
-     * generator to not do that.
-     */
-
-
-    /* Simple? */
-
-    /* Unicode-3.2 additions */
-    case HB_SCRIPT_BUHID:
-    case HB_SCRIPT_HANUNOO:
-
-    /* Unicode-5.1 additions */
-    case HB_SCRIPT_SAURASHTRA:
-
-    /* Unicode-6.0 additions */
-    case HB_SCRIPT_BATAK:
-    case HB_SCRIPT_BRAHMI:
-
-
-    /* Simple */
-
-    /* Unicode-1.1 additions */
-    /* These have their own shaper now. */
-    case HB_SCRIPT_LAO:
-    case HB_SCRIPT_THAI:
-
-    /* Unicode-3.2 additions */
-    case HB_SCRIPT_TAGALOG:
-    case HB_SCRIPT_TAGBANWA:
-
-    /* Unicode-4.0 additions */
-    case HB_SCRIPT_LIMBU:
-    case HB_SCRIPT_TAI_LE:
-
     /* Unicode-4.1 additions */
-    case HB_SCRIPT_KHAROSHTHI:
     case HB_SCRIPT_NEW_TAI_LUE:
-    case HB_SCRIPT_SYLOTI_NAGRI:
-
-    /* Unicode-5.1 additions */
-    case HB_SCRIPT_KAYAH_LI:
-
-    /* Unicode-5.2 additions */
-    case HB_SCRIPT_TAI_VIET:
-
-
 #endif
 
     /* Unicode-1.1 additions */
@@ -288,28 +236,11 @@ hb_ot_shape_complex_categorize (const hb_ot_shape_planner_t *planner)
     /* Unicode-3.0 additions */
     case HB_SCRIPT_SINHALA:
 
-    /* Unicode-5.0 additions */
-    case HB_SCRIPT_BALINESE:
-
-    /* Unicode-5.1 additions */
-    case HB_SCRIPT_LEPCHA:
-    case HB_SCRIPT_REJANG:
-    case HB_SCRIPT_SUNDANESE:
-
     /* Unicode-5.2 additions */
     case HB_SCRIPT_JAVANESE:
-    case HB_SCRIPT_KAITHI:
-    case HB_SCRIPT_MEETEI_MAYEK:
-
-    /* Unicode-6.0 additions */
-
-    /* Unicode-6.1 additions */
-    case HB_SCRIPT_CHAKMA:
-    case HB_SCRIPT_SHARADA:
-    case HB_SCRIPT_TAKRI:
 
       /* If the designer designed the font for the 'DFLT' script,
-       * use the default shaper.  Otherwise, use the Indic shaper.
+       * use the default shaper.  Otherwise, use the specific shaper.
        * Note that for some simple scripts, there may not be *any*
        * GSUB/GPOS needed, so there may be no scripts found! */
       if (planner->map.chosen_script[0] == HB_TAG ('D','F','L','T'))
@@ -341,23 +272,82 @@ hb_ot_shape_complex_categorize (const hb_ot_shape_planner_t *planner)
       else
 	return &_hb_ot_complex_shaper_default;
 
+
+    /* Unicode-2.0 additions */
+    //case HB_SCRIPT_TIBETAN:
+
+    /* Unicode-3.0 additions */
+    //case HB_SCRIPT_MONGOLIAN:
+    //case HB_SCRIPT_SINHALA:
+
+    /* Unicode-3.2 additions */
+    case HB_SCRIPT_BUHID:
+    case HB_SCRIPT_HANUNOO:
+    case HB_SCRIPT_TAGALOG:
+    case HB_SCRIPT_TAGBANWA:
+
+    /* Unicode-4.0 additions */
+    case HB_SCRIPT_LIMBU:
+    case HB_SCRIPT_TAI_LE:
+
     /* Unicode-4.1 additions */
     case HB_SCRIPT_BUGINESE:
+    case HB_SCRIPT_KHAROSHTHI:
+    case HB_SCRIPT_SYLOTI_NAGRI:
+    case HB_SCRIPT_TIFINAGH:
+
+    /* Unicode-5.0 additions */
+    case HB_SCRIPT_BALINESE:
+    //case HB_SCRIPT_NKO:
+    //case HB_SCRIPT_PHAGS_PA:
 
     /* Unicode-5.1 additions */
     case HB_SCRIPT_CHAM:
+    case HB_SCRIPT_KAYAH_LI:
+    case HB_SCRIPT_LEPCHA:
+    case HB_SCRIPT_REJANG:
+    case HB_SCRIPT_SAURASHTRA:
+    case HB_SCRIPT_SUNDANESE:
 
     /* Unicode-5.2 additions */
+    case HB_SCRIPT_EGYPTIAN_HIEROGLYPHS:
+    //case HB_SCRIPT_JAVANESE:
+    case HB_SCRIPT_KAITHI:
+    case HB_SCRIPT_MEETEI_MAYEK:
     case HB_SCRIPT_TAI_THAM:
+    case HB_SCRIPT_TAI_VIET:
+
+    /* Unicode-6.0 additions */
+    case HB_SCRIPT_BATAK:
+    case HB_SCRIPT_BRAHMI:
+    //case HB_SCRIPT_MANDAIC:
+
+    /* Unicode-6.1 additions */
+    case HB_SCRIPT_CHAKMA:
+    case HB_SCRIPT_SHARADA:
+    case HB_SCRIPT_TAKRI:
+
+    /* Unicode-7.0 additions */
+    case HB_SCRIPT_DUPLOYAN:
+    case HB_SCRIPT_GRANTHA:
+    case HB_SCRIPT_KHOJKI:
+    case HB_SCRIPT_KHUDAWADI:
+    case HB_SCRIPT_MAHAJANI:
+    //case HB_SCRIPT_MANICHAEAN:
+    case HB_SCRIPT_MODI:
+    case HB_SCRIPT_PAHAWH_HMONG:
+    //case HB_SCRIPT_PSALTER_PAHLAVI:
+    case HB_SCRIPT_SIDDHAM:
+    case HB_SCRIPT_TIRHUTA:
 
       /* If the designer designed the font for the 'DFLT' script,
-       * use the default shaper.  Otherwise, use the Indic shaper.
+       * use the default shaper.  Otherwise, use the specific shaper.
        * Note that for some simple scripts, there may not be *any*
        * GSUB/GPOS needed, so there may be no scripts found! */
       if (planner->map.chosen_script[0] == HB_TAG ('D','F','L','T'))
 	return &_hb_ot_complex_shaper_default;
       else
-	return &_hb_ot_complex_shaper_sea;
+	return &_hb_ot_complex_shaper_use;
   }
 }
 

src/hb-ot-shape-complex-sea-machine.rl

@@ -1,102 +0,0 @@
-/*
- * Copyright © 2011,2012,2013  Google, Inc.
- *
- *  This is part of HarfBuzz, a text shaping library.
- *
- * Permission is hereby granted, without written agreement and without
- * license or royalty fees, to use, copy, modify, and distribute this
- * software and its documentation for any purpose, provided that the
- * above copyright notice and the following two paragraphs appear in
- * all copies of this software.
- *
- * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
- * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
- * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
- * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- *
- * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
- * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
- * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
- * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
- * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
- *
- * Google Author(s): Behdad Esfahbod
- */
-
-#ifndef HB_OT_SHAPE_COMPLEX_SEA_MACHINE_HH
-#define HB_OT_SHAPE_COMPLEX_SEA_MACHINE_HH
-
-#include "hb-private.hh"
-
-%%{
-  machine sea_syllable_machine;
-  alphtype unsigned char;
-  write data;
-}%%
-
-%%{
-
-# Same order as enum sea_category_t.  Not sure how to avoid duplication.
-C    = 1;
-GB   = 12; # Generic Base
-H    = 4;  # Halant
-IV   = 2;  # Independent Vowel
-MR   = 22; # Medial Ra
-CM   = 17; # Consonant Medial
-VAbv = 26;
-VBlw = 27;
-VPre = 28;
-VPst = 29;
-T    = 3;  # Tone Marks
-A    = 10; # Anusvara
-
-syllable_tail = (VPre|VAbv|VBlw|VPst|H.C|CM|MR|T|A)*;
-
-consonant_syllable =	(C|IV|GB) syllable_tail;
-broken_cluster =	syllable_tail;
-other =			any;
-
-main := |*
-	consonant_syllable	=> { found_syllable (consonant_syllable); };
-	broken_cluster		=> { found_syllable (broken_cluster); };
-	other			=> { found_syllable (non_sea_cluster); };
-*|;
-
-
-}%%
-
-#define found_syllable(syllable_type) \
-  HB_STMT_START { \
-    if (0) fprintf (stderr, "syllable %d..%d %s\n", last, p+1, #syllable_type); \
-    for (unsigned int i = last; i < p+1; i++) \
-      info[i].syllable() = (syllable_serial << 4) | syllable_type; \
-    last = p+1; \
-    syllable_serial++; \
-    if (unlikely (syllable_serial == 16)) syllable_serial = 1; \
-  } HB_STMT_END
-
-static void
-find_syllables (hb_buffer_t *buffer)
-{
-  unsigned int p, pe, eof, ts HB_UNUSED, te HB_UNUSED, act HB_UNUSED;
-  int cs;
-  hb_glyph_info_t *info = buffer->info;
-  %%{
-    write init;
-    getkey info[p].sea_category();
-  }%%
-
-  p = 0;
-  pe = eof = buffer->len;
-
-  unsigned int last = 0;
-  unsigned int syllable_serial = 1;
-  %%{
-    write exec;
-  }%%
-}
-
-#undef found_syllable
-
-#endif /* HB_OT_SHAPE_COMPLEX_SEA_MACHINE_HH */

src/hb-ot-shape-complex-sea.cc

@@ -1,380 +0,0 @@
-/*
- * Copyright © 2011,2012,2013  Google, Inc.
- *
- *  This is part of HarfBuzz, a text shaping library.
- *
- * Permission is hereby granted, without written agreement and without
- * license or royalty fees, to use, copy, modify, and distribute this
- * software and its documentation for any purpose, provided that the
- * above copyright notice and the following two paragraphs appear in
- * all copies of this software.
- *
- * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
- * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
- * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
- * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- *
- * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
- * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
- * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
- * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
- * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
- *
- * Google Author(s): Behdad Esfahbod
- */
-
-#include "hb-ot-shape-complex-indic-private.hh"
-
-/* buffer var allocations */
-#define sea_category() complex_var_u8_0() /* indic_category_t */
-#define sea_position() complex_var_u8_1() /* indic_position_t */
-
-
-/*
- * South-East Asian shaper.
- * Loosely based on the Myanmar spec / shaper.
- * There is no OpenType spec for this.
- */
-
-static const hb_tag_t
-basic_features[] =
-{
-  /*
-   * Basic features.
-   * These features are applied in order, one at a time, after initial_reordering.
-   */
-  HB_TAG('p','r','e','f'),
-  HB_TAG('a','b','v','f'),
-  HB_TAG('b','l','w','f'),
-  HB_TAG('p','s','t','f'),
-};
-static const hb_tag_t
-other_features[] =
-{
-  /*
-   * Other features.
-   * These features are applied all at once, after final_reordering.
-   */
-  HB_TAG('p','r','e','s'),
-  HB_TAG('a','b','v','s'),
-  HB_TAG('b','l','w','s'),
-  HB_TAG('p','s','t','s'),
-  /* Positioning features, though we don't care about the types. */
-  HB_TAG('d','i','s','t'),
-};
-
-static void
-setup_syllables (const hb_ot_shape_plan_t *plan,
-		 hb_font_t *font,
-		 hb_buffer_t *buffer);
-static void
-initial_reordering (const hb_ot_shape_plan_t *plan,
-		    hb_font_t *font,
-		    hb_buffer_t *buffer);
-static void
-final_reordering (const hb_ot_shape_plan_t *plan,
-		  hb_font_t *font,
-		  hb_buffer_t *buffer);
-
-static void
-collect_features_sea (hb_ot_shape_planner_t *plan)
-{
-  hb_ot_map_builder_t *map = &plan->map;
-
-  /* Do this before any lookups have been applied. */
-  map->add_gsub_pause (setup_syllables);
-
-  map->add_global_bool_feature (HB_TAG('l','o','c','l'));
-  /* The Indic specs do not require ccmp, but we apply it here since if
-   * there is a use of it, it's typically at the beginning. */
-  map->add_global_bool_feature (HB_TAG('c','c','m','p'));
-
-  map->add_gsub_pause (initial_reordering);
-  for (unsigned int i = 0; i < ARRAY_LENGTH (basic_features); i++)
-  {
-    map->add_feature (basic_features[i], 1, F_GLOBAL | F_MANUAL_ZWJ);
-    map->add_gsub_pause (NULL);
-  }
-  map->add_gsub_pause (final_reordering);
-  for (unsigned int i = 0; i < ARRAY_LENGTH (other_features); i++)
-    map->add_feature (other_features[i], 1, F_GLOBAL | F_MANUAL_ZWJ);
-}
-
-static void
-override_features_sea (hb_ot_shape_planner_t *plan)
-{
-  plan->map.add_feature (HB_TAG('l','i','g','a'), 0, F_GLOBAL);
-}
-
-
-enum syllable_type_t {
-  consonant_syllable,
-  broken_cluster,
-  non_sea_cluster,
-};
-
-#include "hb-ot-shape-complex-sea-machine.hh"
-
-
-/* Note: This enum is duplicated in the -machine.rl source file.
- * Not sure how to avoid duplication. */
-enum sea_category_t {
-//  OT_C    = 1,
-  OT_GB   = 12, /* Generic Base XXX DOTTED CIRCLE only for now */
-//  OT_H    = 4,  /* Halant */
-  OT_IV   = 2,  /* Independent Vowel */
-  OT_MR   = 22, /* Medial Ra */
-//  OT_CM   = 17, /* Consonant Medial */
-  OT_VAbv = 26,
-  OT_VBlw = 27,
-  OT_VPre = 28,
-  OT_VPst = 29,
-  OT_T    = 3,  /* Tone Marks */
-//  OT_A    = 10, /* Anusvara */
-};
-
-static inline void
-set_sea_properties (hb_glyph_info_t &info)
-{
-  hb_codepoint_t u = info.codepoint;
-  unsigned int type = hb_indic_get_categories (u);
-  indic_category_t cat = (indic_category_t) (type & 0x7Fu);
-  indic_position_t pos = (indic_position_t) (type >> 8);
-
-  /* Medial Ra */
-  if (u == 0x1A55u || u == 0xAA34u)
-    cat = (indic_category_t) OT_MR;
-
-  if (cat == OT_M)
-  {
-    switch ((int) pos)
-    {
-      case POS_PRE_C:	cat = (indic_category_t) OT_VPre; break;
-      case POS_ABOVE_C:	cat = (indic_category_t) OT_VAbv; break;
-      case POS_BELOW_C:	cat = (indic_category_t) OT_VBlw; break;
-      case POS_POST_C:	cat = (indic_category_t) OT_VPst; break;
-    }
-  }
-
-  info.sea_category() = (sea_category_t) cat;
-  info.sea_position() = pos;
-}
-
-
-static void
-setup_masks_sea (const hb_ot_shape_plan_t *plan HB_UNUSED,
-		 hb_buffer_t              *buffer,
-		 hb_font_t                *font HB_UNUSED)
-{
-  HB_BUFFER_ALLOCATE_VAR (buffer, sea_category);
-  HB_BUFFER_ALLOCATE_VAR (buffer, sea_position);
-
-  /* We cannot setup masks here.  We save information about characters
-   * and setup masks later on in a pause-callback. */
-
-  unsigned int count = buffer->len;
-  hb_glyph_info_t *info = buffer->info;
-  for (unsigned int i = 0; i < count; i++)
-    set_sea_properties (info[i]);
-}
-
-static void
-setup_syllables (const hb_ot_shape_plan_t *plan HB_UNUSED,
-		 hb_font_t *font HB_UNUSED,
-		 hb_buffer_t *buffer)
-{
-  find_syllables (buffer);
-}
-
-static int
-compare_sea_order (const hb_glyph_info_t *pa, const hb_glyph_info_t *pb)
-{
-  int a = pa->sea_position();
-  int b = pb->sea_position();
-
-  return a < b ? -1 : a == b ? 0 : +1;
-}
-
-
-static void
-initial_reordering_consonant_syllable (const hb_ot_shape_plan_t *plan,
-				       hb_face_t *face,
-				       hb_buffer_t *buffer,
-				       unsigned int start, unsigned int end)
-{
-  hb_glyph_info_t *info = buffer->info;
-  unsigned int base = start;
-
-  /* Reorder! */
-  unsigned int i = start;
-  for (; i < base; i++)
-    info[i].sea_position() = POS_PRE_C;
-  if (i < end)
-  {
-    info[i].sea_position() = POS_BASE_C;
-    i++;
-  }
-  for (; i < end; i++)
-  {
-    if (info[i].sea_category() == OT_MR) /* Pre-base reordering */
-    {
-      info[i].sea_position() = POS_PRE_C;
-      continue;
-    }
-    if (info[i].sea_category() == OT_VPre) /* Left matra */
-    {
-      info[i].sea_position() = POS_PRE_M;
-      continue;
-    }
-
-    info[i].sea_position() = POS_AFTER_MAIN;
-  }
-
-  buffer->merge_clusters (start, end);
-  /* Sit tight, rock 'n roll! */
-  hb_bubble_sort (info + start, end - start, compare_sea_order);
-}
-
-static void
-initial_reordering_broken_cluster (const hb_ot_shape_plan_t *plan,
-				   hb_face_t *face,
-				   hb_buffer_t *buffer,
-				   unsigned int start, unsigned int end)
-{
-  /* We already inserted dotted-circles, so just call the consonant_syllable. */
-  initial_reordering_consonant_syllable (plan, face, buffer, start, end);
-}
-
-static void
-initial_reordering_non_sea_cluster (const hb_ot_shape_plan_t *plan HB_UNUSED,
-				    hb_face_t *face HB_UNUSED,
-				    hb_buffer_t *buffer HB_UNUSED,
-				    unsigned int start HB_UNUSED, unsigned int end HB_UNUSED)
-{
-  /* Nothing to do right now.  If we ever switch to using the output
-   * buffer in the reordering process, we'd need to next_glyph() here. */
-}
-
-
-static void
-initial_reordering_syllable (const hb_ot_shape_plan_t *plan,
-			     hb_face_t *face,
-			     hb_buffer_t *buffer,
-			     unsigned int start, unsigned int end)
-{
-  syllable_type_t syllable_type = (syllable_type_t) (buffer->info[start].syllable() & 0x0F);
-  switch (syllable_type) {
-  case consonant_syllable:	initial_reordering_consonant_syllable  (plan, face, buffer, start, end); return;
-  case broken_cluster:		initial_reordering_broken_cluster      (plan, face, buffer, start, end); return;
-  case non_sea_cluster:		initial_reordering_non_sea_cluster     (plan, face, buffer, start, end); return;
-  }
-}
-
-static inline void
-insert_dotted_circles (const hb_ot_shape_plan_t *plan HB_UNUSED,
-		       hb_font_t *font,
-		       hb_buffer_t *buffer)
-{
-  /* Note: This loop is extra overhead, but should not be measurable. */
-  bool has_broken_syllables = false;
-  unsigned int count = buffer->len;
-  hb_glyph_info_t *info = buffer->info;
-  for (unsigned int i = 0; i < count; i++)
-    if ((info[i].syllable() & 0x0F) == broken_cluster)
-    {
-      has_broken_syllables = true;
-      break;
-    }
-  if (likely (!has_broken_syllables))
-    return;
-
-
-  hb_codepoint_t dottedcircle_glyph;
-  if (!font->get_glyph (0x25CCu, 0, &dottedcircle_glyph))
-    return;
-
-  hb_glyph_info_t dottedcircle = {0};
-  dottedcircle.codepoint = 0x25CCu;
-  set_sea_properties (dottedcircle);
-  dottedcircle.codepoint = dottedcircle_glyph;
-
-  buffer->clear_output ();
-
-  buffer->idx = 0;
-  unsigned int last_syllable = 0;
-  while (buffer->idx < buffer->len)
-  {
-    unsigned int syllable = buffer->cur().syllable();
-    syllable_type_t syllable_type = (syllable_type_t) (syllable & 0x0F);
-    if (unlikely (last_syllable != syllable && syllable_type == broken_cluster))
-    {
-      last_syllable = syllable;
-
-      hb_glyph_info_t info = dottedcircle;
-      info.cluster = buffer->cur().cluster;
-      info.mask = buffer->cur().mask;
-      info.syllable() = buffer->cur().syllable();
-
-      buffer->output_info (info);
-    }
-    else
-      buffer->next_glyph ();
-  }
-
-  buffer->swap_buffers ();
-}
-
-static void
-initial_reordering (const hb_ot_shape_plan_t *plan,
-		    hb_font_t *font,
-		    hb_buffer_t *buffer)
-{
-  insert_dotted_circles (plan, font, buffer);
-
-  hb_glyph_info_t *info = buffer->info;
-  unsigned int count = buffer->len;
-  if (unlikely (!count)) return;
-  unsigned int last = 0;
-  unsigned int last_syllable = info[0].syllable();
-  for (unsigned int i = 1; i < count; i++)
-    if (last_syllable != info[i].syllable()) {
-      initial_reordering_syllable (plan, font->face, buffer, last, i);
-      last = i;
-      last_syllable = info[last].syllable();
-    }
-  initial_reordering_syllable (plan, font->face, buffer, last, count);
-}
-
-static void
-final_reordering (const hb_ot_shape_plan_t *plan,
-		  hb_font_t *font HB_UNUSED,
-		  hb_buffer_t *buffer)
-{
-  hb_glyph_info_t *info = buffer->info;
-  unsigned int count = buffer->len;
-
-  /* Zero syllables now... */
-  for (unsigned int i = 0; i < count; i++)
-    info[i].syllable() = 0;
-
-  HB_BUFFER_DEALLOCATE_VAR (buffer, sea_category);
-  HB_BUFFER_DEALLOCATE_VAR (buffer, sea_position);
-}
-
-
-const hb_ot_complex_shaper_t _hb_ot_complex_shaper_sea =
-{
-  "sea",
-  collect_features_sea,
-  override_features_sea,
-  NULL, /* data_create */
-  NULL, /* data_destroy */
-  NULL, /* preprocess_text */
-  HB_OT_SHAPE_NORMALIZATION_MODE_COMPOSED_DIACRITICS_NO_SHORT_CIRCUIT,
-  NULL, /* decompose */
-  NULL, /* compose */
-  setup_masks_sea,
-  HB_OT_SHAPE_ZERO_WIDTH_MARKS_NONE,
-  false, /* fallback_position */
-};

src/hb-ot-shape-complex-use-machine.rl

@@ -0,0 +1,180 @@
+/*
+ * Copyright © 2015  Mozilla Foundation.
+ * Copyright © 2015  Google, Inc.
+ *
+ *  This is part of HarfBuzz, a text shaping library.
+ *
+ * Permission is hereby granted, without written agreement and without
+ * license or royalty fees, to use, copy, modify, and distribute this
+ * software and its documentation for any purpose, provided that the
+ * above copyright notice and the following two paragraphs appear in
+ * all copies of this software.
+ *
+ * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
+ * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
+ * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ *
+ * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
+ * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+ *
+ * Mozilla Author(s): Jonathan Kew
+ * Google Author(s): Behdad Esfahbod
+ */
+
+#ifndef HB_OT_SHAPE_COMPLEX_USE_MACHINE_HH
+#define HB_OT_SHAPE_COMPLEX_USE_MACHINE_HH
+
+#include "hb-private.hh"
+
+%%{
+  machine use_syllable_machine;
+  alphtype unsigned char;
+  write data;
+}%%
+
+%%{
+
+# Same order as enum use_category_t.  Not sure how to avoid duplication.
+
+O	= 0; # OTHER
+
+B	= 1; # BASE
+IV	= 2; # BASE_VOWEL
+IND	= 3; # BASE_IND
+N	= 4; # BASE_NUM
+GB	= 5; # BASE_OTHER
+CGJ	= 6; # CGJ
+#F	= 7; # CONS_FINAL
+FM	= 8; # CONS_FINAL_MOD
+#M	= 9; # CONS_MED
+#CM	= 10; # CONS_MOD
+SUB	= 11; # CONS_SUB
+H	= 12; # HALANT
+
+HN	= 13; # HALANT_NUM
+ZWNJ	= 14; # Zero width non-joiner
+ZWJ	= 15; # Zero width joiner
+WJ	= 16; # Word joiner
+Rsv	= 17; # Reserved characters
+R	= 18; # REPHA
+S	= 19; # SYM
+#SM	= 20; # SYM_MOD
+VS	= 21; # VARIATION_SELECTOR
+#V	= 36; # VOWEL
+#VM	= 40; # VOWEL_MOD
+
+FAbv	= 24; # CONS_FINAL_ABOVE
+FBlw	= 25; # CONS_FINAL_BELOW
+FPst	= 26; # CONS_FINAL_POST
+MAbv	= 27; # CONS_MED_ABOVE
+MBlw	= 28; # CONS_MED_BELOW
+MPst	= 29; # CONS_MED_POST
+MPre	= 30; # CONS_MED_PRE
+CMAbv	= 31; # CONS_MOD_ABOVE
+CMBlw	= 32; # CONS_MOD_BELOW
+VAbv	= 33; # VOWEL_ABOVE / VOWEL_ABOVE_BELOW / VOWEL_ABOVE_BELOW_POST / VOWEL_ABOVE_POST
+VBlw	= 34; # VOWEL_BELOW / VOWEL_BELOW_POST
+VPst	= 35; # VOWEL_POST	UIPC = Right
+VPre	= 22; # VOWEL_PRE / VOWEL_PRE_ABOVE / VOWEL_PRE_ABOVE_POST / VOWEL_PRE_POST
+VMAbv	= 37; # VOWEL_MOD_ABOVE
+VMBlw	= 38; # VOWEL_MOD_BELOW
+VMPst	= 39; # VOWEL_MOD_POST
+VMPre	= 23; # VOWEL_MOD_PRE
+SMAbv	= 41; # SYM_MOD_ABOVE
+SMBlw	= 42; # SYM_MOD_BELOW
+
+
+consonant_modifiers = CMAbv* CMBlw* ((H B | SUB) VS? CMAbv? CMBlw*)*;
+medial_consonants = MPre? MAbv? MBlw? MPst?;
+dependent_vowels = VPre* VAbv* VBlw* VPst*;
+vowel_modifiers = VMPre* VMAbv* VMBlw* VMPst*;
+final_consonants = FAbv* FBlw* FPst* FM?;
+
+virama_terminated_cluster =
+	R? (B | GB | IV) VS?
+	consonant_modifiers
+	H
+;
+consonant_cluster =
+	R? (B | GB) VS?
+	consonant_modifiers
+	medial_consonants
+	dependent_vowels
+	vowel_modifiers
+	final_consonants
+;
+vowel_cluster =
+	R? (IV) VS?
+	consonant_modifiers
+	medial_consonants
+	vowel_modifiers
+	final_consonants
+;
+
+broken_cluster =
+	R?
+	consonant_modifiers
+	medial_consonants
+	dependent_vowels
+	vowel_modifiers
+	final_consonants
+;
+
+number_joiner_terminated_cluster = N VS? (HN N VS?)* H;
+numeral_cluster = N VS? (HN N VS?)*;
+symbol_cluster = S VS? SMAbv* SMBlw*;
+independent_cluster = (IND | O | Rsv | WJ) VS?;
+
+main := |*
+	independent_cluster			=> { found_syllable (independent_cluster); };
+	virama_terminated_cluster		=> { found_syllable (virama_terminated_cluster); };
+	consonant_cluster			=> { found_syllable (consonant_cluster); };
+	vowel_cluster				=> { found_syllable (vowel_cluster); };
+	number_joiner_terminated_cluster	=> { found_syllable (number_joiner_terminated_cluster); };
+	numeral_cluster				=> { found_syllable (numeral_cluster); };
+	symbol_cluster				=> { found_syllable (symbol_cluster); };
+	broken_cluster				=> { found_syllable (broken_cluster); };
+*|;
+
+
+}%%
+
+#define found_syllable(syllable_type) \
+  HB_STMT_START { \
+    if (0) fprintf (stderr, "syllable %d..%d %s\n", last, p+1, #syllable_type); \
+    for (unsigned int i = last; i < p+1; i++) \
+      info[i].syllable() = (syllable_serial << 4) | syllable_type; \
+    last = p+1; \
+    syllable_serial++; \
+    if (unlikely (syllable_serial == 16)) syllable_serial = 1; \
+  } HB_STMT_END
+
+static void
+find_syllables (hb_buffer_t *buffer)
+{
+  unsigned int p, pe, eof, ts HB_UNUSED, te HB_UNUSED, act HB_UNUSED;
+  int cs;
+  hb_glyph_info_t *info = buffer->info;
+  %%{
+    write init;
+    getkey info[p].use_category();
+  }%%
+
+  p = 0;
+  pe = eof = buffer->len;
+
+  unsigned int last = 0;
+  unsigned int syllable_serial = 1;
+  %%{
+    write exec;
+  }%%
+}
+
+#undef found_syllable
+
+#endif /* HB_OT_SHAPE_COMPLEX_USE_MACHINE_HH */

src/hb-ot-shape-complex-use-private.hh

@@ -0,0 +1,97 @@
+/*
+ * Copyright © 2015  Mozilla Foundation.
+ * Copyright © 2015  Google, Inc.
+ *
+ *  This is part of HarfBuzz, a text shaping library.
+ *
+ * Permission is hereby granted, without written agreement and without
+ * license or royalty fees, to use, copy, modify, and distribute this
+ * software and its documentation for any purpose, provided that the
+ * above copyright notice and the following two paragraphs appear in
+ * all copies of this software.
+ *
+ * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
+ * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
+ * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ *
+ * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
+ * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+ *
+ * Mozilla Author(s): Jonathan Kew
+ * Google Author(s): Behdad Esfahbod
+ */
+
+#ifndef HB_OT_SHAPE_COMPLEX_USE_PRIVATE_HH
+#define HB_OT_SHAPE_COMPLEX_USE_PRIVATE_HH
+
+#include "hb-private.hh"
+
+
+#include "hb-ot-shape-complex-private.hh"
+
+
+#define USE_TABLE_ELEMENT_TYPE uint8_t
+
+/* Cateories used in the Universal Shaping Engine spec:
+ * https://www.microsoft.com/typography/OpenTypeDev/USE/intro.htm
+ */
+/* Note: This enum is duplicated in the -machine.rl source file.
+ * Not sure how to avoid duplication. */
+enum use_category_t {
+  USE_O		= 0,	/* OTHER */
+
+  USE_B		= 1,	/* BASE */
+  USE_IV	= 2,	/* BASE_VOWEL */
+  USE_IND	= 3,	/* BASE_IND */
+  USE_N		= 4,	/* BASE_NUM */
+  USE_GB	= 5,	/* BASE_OTHER */
+  USE_CGJ	= 6,	/* CGJ */
+//  USE_F		= 7,	/* CONS_FINAL */
+  USE_FM	= 8,	/* CONS_FINAL_MOD */
+//  USE_M		= 9,	/* CONS_MED */
+//  USE_CM	= 10,	/* CONS_MOD */
+  USE_SUB	= 11,	/* CONS_SUB */
+  USE_H		= 12,	/* HALANT */
+
+  USE_HN	= 13,	/* HALANT_NUM */
+  USE_ZWNJ	= 14,	/* Zero width non-joiner */
+  USE_ZWJ	= 15,	/* Zero width joiner */
+  USE_WJ	= 16,	/* Word joiner */
+  USE_Rsv	= 17,	/* Reserved characters */
+  USE_R		= 18,	/* REPHA */
+  USE_S		= 19,	/* SYM */
+//  USE_SM	= 20,	/* SYM_MOD */
+  USE_VS	= 21,	/* VARIATION_SELECTOR */
+//  USE_V	= 36,	/* VOWEL */
+//  USE_VM	= 40,	/* VOWEL_MOD */
+
+  USE_FAbv	= 24,	/* CONS_FINAL_ABOVE */
+  USE_FBlw	= 25,	/* CONS_FINAL_BELOW */
+  USE_FPst	= 26,	/* CONS_FINAL_POST */
+  USE_MAbv	= 27,	/* CONS_MED_ABOVE */
+  USE_MBlw	= 28,	/* CONS_MED_BELOW */
+  USE_MPst	= 29,	/* CONS_MED_POST */
+  USE_MPre	= 30,	/* CONS_MED_PRE */
+  USE_CMAbv	= 31,	/* CONS_MOD_ABOVE */
+  USE_CMBlw	= 32,	/* CONS_MOD_BELOW */
+  USE_VAbv	= 33,	/* VOWEL_ABOVE / VOWEL_ABOVE_BELOW / VOWEL_ABOVE_BELOW_POST / VOWEL_ABOVE_POST */
+  USE_VBlw	= 34,	/* VOWEL_BELOW / VOWEL_BELOW_POST */
+  USE_VPst	= 35,	/* VOWEL_POST	UIPC = Right */
+  USE_VPre	= 22,	/* VOWEL_PRE / VOWEL_PRE_ABOVE / VOWEL_PRE_ABOVE_POST / VOWEL_PRE_POST */
+  USE_VMAbv	= 37,	/* VOWEL_MOD_ABOVE */
+  USE_VMBlw	= 38,	/* VOWEL_MOD_BELOW */
+  USE_VMPst	= 39,	/* VOWEL_MOD_POST */
+  USE_VMPre	= 23,	/* VOWEL_MOD_PRE */
+  USE_SMAbv	= 41,	/* SYM_MOD_ABOVE */
+  USE_SMBlw	= 42	/* SYM_MOD_BELOW */
+};
+
+HB_INTERNAL USE_TABLE_ELEMENT_TYPE
+hb_use_get_categories (hb_codepoint_t u);
+
+#endif /* HB_OT_SHAPE_COMPLEX_USE_PRIVATE_HH */

src/hb-ot-shape-complex-use-table.cc

@@ -0,0 +1,696 @@
+/* == Start of generated table == */
+/*
+ * The following table is generated by running:
+ *
+ *   ./gen-use-table.py IndicSyllabicCategory.txt IndicPositionalCategory.txt UnicodeData.txt Blocks.txt
+ *
+ * on files with these headers:
+ *
+ * # IndicSyllabicCategory-8.0.0.txt
+ * # Date: 2015-05-12, 10:00:00 GMT [RP, KW, LI]
+ * # IndicPositionalCategory-8.0.0.txt
+ * # Date: 2015-05-12, 10:00:00 GMT [RP, KW, LI]
+ * # Blocks-8.0.0.txt
+ * # Date: 2014-11-10, 23:04:00 GMT [KW]
+ * UnicodeData.txt does not have a header.
+ */
+
+#include "hb-ot-shape-complex-use-private.hh"
+
+#define B	USE_B	/* BASE */
+#define CGJ	USE_CGJ	/* CGJ */
+#define FM	USE_FM	/* CONS_FINAL_MOD */
+#define GB	USE_GB	/* BASE_OTHER */
+#define H	USE_H	/* HALANT */
+#define HN	USE_HN	/* HALANT_NUM */
+#define IND	USE_IND	/* BASE_IND */
+#define IV	USE_IV	/* BASE_VOWEL */
+#define N	USE_N	/* BASE_NUM */
+#define O	USE_O	/* OTHER */
+#define R	USE_R	/* REPHA */
+#define Rsv	USE_Rsv	/* Reserved */
+#define S	USE_S	/* SYM */
+#define SUB	USE_SUB	/* CONS_SUB */
+#define VS	USE_VS	/* VARIATION_SELECTOR */
+#define WJ	USE_WJ	/* Word_Joiner */
+#define ZWJ	USE_ZWJ	/* ZWJ */
+#define ZWNJ	USE_ZWNJ	/* ZWNJ */
+#define CMBlw	USE_CMBlw
+#define CMAbv	USE_CMAbv
+#define FBlw	USE_FBlw
+#define FPst	USE_FPst
+#define FAbv	USE_FAbv
+#define MPre	USE_MPre
+#define MBlw	USE_MBlw
+#define MPst	USE_MPst
+#define MAbv	USE_MAbv
+#define SMBlw	USE_SMBlw
+#define SMAbv	USE_SMAbv
+#define VPre	USE_VPre
+#define VBlw	USE_VBlw
+#define VPst	USE_VPst
+#define VAbv	USE_VAbv
+#define VMPre	USE_VMPre
+#define VMBlw	USE_VMBlw
+#define VMPst	USE_VMPst
+#define VMAbv	USE_VMAbv
+
+static const USE_TABLE_ELEMENT_TYPE use_table[] = {
+
+
+#define use_offset_0x0028u 0
+
+
+  /* Basic Latin */
+                                                                         O,     O,     O,     O,     O,    GB,     O,     O,
+  /* 0030 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
+
+#define use_offset_0x00a0u 24
+
+
+  /* Latin-1 Supplement */
+
+  /* 00A0 */    GB,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* 00B0 */     O,     O,    FM,    FM,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* 00C0 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* 00D0 */     O,     O,     O,     O,     O,     O,     O,    GB,
+
+#define use_offset_0x0900u 80
+
+
+  /* Devanagari */
+
+  /* 0900 */ VMAbv, VMAbv, VMAbv, VMPst,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,
+  /* 0910 */    IV,    IV,    IV,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0920 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0930 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,  VAbv,  VPst, CMBlw,     B,  VPst,  VPre,
+  /* 0940 */  VPst,  VBlw,  VBlw,  VBlw,  VBlw,  VAbv,  VAbv,  VAbv,  VAbv,  VPst,  VPst,  VPst,  VPst,     H,  VPre,  VPst,
+  /* 0950 */     O, VMAbv, VMBlw,     O,     O,  VAbv,  VBlw,  VBlw,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0960 */    IV,    IV,  VBlw,  VBlw,     O,     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0970 */     O,     O,    IV,    IV,    IV,    IV,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,
+
+  /* Bengali */
+
+  /* 0980 */     O, VMAbv, VMPst, VMPst,     O,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,     O,     O,    IV,
+  /* 0990 */    IV,     O,     O,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 09A0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     B,     B,     B,     B,     B,     B,
+  /* 09B0 */     B,     O,     B,     O,     O,     O,     B,     B,     B,     B,     O,     O, CMBlw,     B,  VPst,  VPre,
+  /* 09C0 */  VPst,  VBlw,  VBlw,  VBlw,  VBlw,     O,     O,  VPre,  VPre,     O,     O,  VPre,  VPre,     H,   IND,     O,
+  /* 09D0 */     O,     O,     O,     O,     O,     O,     O,  VPst,     O,     O,     O,     O,     B,     B,     O,     B,
+  /* 09E0 */    IV,    IV,  VBlw,  VBlw,     O,     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 09F0 */     B,     B,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Gurmukhi */
+
+  /* 0A00 */     O, VMAbv, VMAbv, VMPst,     O,    IV,    IV,    IV,    IV,    IV,    IV,     O,     O,     O,     O,    IV,
+  /* 0A10 */    IV,     O,     O,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0A20 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     B,     B,     B,     B,     B,     B,
+  /* 0A30 */     B,     O,     B,     B,     O,     B,     B,     O,     B,     B,     O,     O, CMBlw,     O,  VPst,  VPre,
+  /* 0A40 */  VPst,  VBlw,  VBlw,     O,     O,     O,     O,  VAbv,  VAbv,     O,     O,  VAbv,  VAbv,     H,     O,     O,
+  /* 0A50 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     B,     B,     B,     B,     O,     B,     O,
+  /* 0A60 */     O,     O,     O,     O,     O,     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0A70 */ VMAbv, CMAbv,    GB,    GB,     O,  MBlw,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Gujarati */
+
+  /* 0A80 */     O, VMAbv, VMAbv, VMPst,     O,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,     O,    IV,
+  /* 0A90 */    IV,    IV,     O,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0AA0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     B,     B,     B,     B,     B,     B,
+  /* 0AB0 */     B,     O,     B,     B,     O,     B,     B,     B,     B,     B,     O,     O, CMBlw,     B,  VPst,  VPre,
+  /* 0AC0 */  VPst,  VBlw,  VBlw,  VBlw,  VBlw,  VAbv,     O,  VAbv,  VAbv,  VAbv,     O,  VPst,  VPst,     H,     O,     O,
+  /* 0AD0 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* 0AE0 */    IV,    IV,  VBlw,  VBlw,     O,     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0AF0 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     B,     O,     O,     O,     O,     O,     O,
+
+  /* Oriya */
+
+  /* 0B00 */     O, VMAbv, VMPst, VMPst,     O,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,     O,     O,    IV,
+  /* 0B10 */    IV,     O,     O,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0B20 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     B,     B,     B,     B,     B,     B,
+  /* 0B30 */     B,     O,     B,     B,     O,     B,     B,     B,     B,     B,     O,     O, CMBlw,     B,  VPst,  VAbv,
+  /* 0B40 */  VPst,  VBlw,  VBlw,  VBlw,  VBlw,     O,     O,  VPre,  VPre,     O,     O,  VPre,  VPre,     H,     O,     O,
+  /* 0B50 */     O,     O,     O,     O,     O,     O,  VAbv,  VAbv,     O,     O,     O,     O,     B,     B,     O,     B,
+  /* 0B60 */    IV,    IV,  VBlw,  VBlw,     O,     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0B70 */     O,     B,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Tamil */
+
+  /* 0B80 */     O,     O, VMAbv,   IND,     O,    IV,    IV,    IV,    IV,    IV,    IV,     O,     O,     O,    IV,    IV,
+  /* 0B90 */    IV,     O,    IV,    IV,    IV,     B,     O,     O,     O,     B,     B,     O,     B,     O,     B,     B,
+  /* 0BA0 */     O,     O,     O,     B,     B,     O,     O,     O,     B,     B,     B,     O,     O,     O,     B,     B,
+  /* 0BB0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,  VPst,  VPst,
+  /* 0BC0 */  VAbv,  VPst,  VPst,     O,     O,     O,  VPre,  VPre,  VPre,     O,  VPre,  VPre,  VPre,     H,     O,     O,
+  /* 0BD0 */     O,     O,     O,     O,     O,     O,     O,  VPst,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* 0BE0 */     O,     O,     O,     O,     O,     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0BF0 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Telugu */
+
+  /* 0C00 */ VMAbv, VMPst, VMPst, VMPst,     O,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,     O,    IV,    IV,
+  /* 0C10 */    IV,     O,    IV,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0C20 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     B,     B,     B,     B,     B,     B,
+  /* 0C30 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     B,  VAbv,  VAbv,
+  /* 0C40 */  VAbv,  VPst,  VPst,  VPst,  VPst,     O,  VAbv,  VAbv,  VAbv,     O,  VAbv,  VAbv,  VAbv,     H,     O,     O,
+  /* 0C50 */     O,     O,     O,     O,     O,  VAbv,  VBlw,     O,     B,     B,     B,     O,     O,     O,     O,     O,
+  /* 0C60 */    IV,    IV,  VBlw,  VBlw,     O,     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0C70 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Kannada */
+
+  /* 0C80 */     O, VMAbv, VMPst, VMPst,     O,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,     O,    IV,    IV,
+  /* 0C90 */    IV,     O,    IV,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0CA0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     B,     B,     B,     B,     B,     B,
+  /* 0CB0 */     B,     B,     B,     B,     O,     B,     B,     B,     B,     B,     O,     O, CMBlw,     B,  VPst,  VAbv,
+  /* 0CC0 */  VAbv,  VPst,  VPst,  VPst,  VPst,     O,  VAbv,  VAbv,  VAbv,     O,  VAbv,  VAbv,  VAbv,     H,     O,     O,
+  /* 0CD0 */     O,     O,     O,     O,     O,  VPst,  VPst,     O,     O,     O,     O,     O,     O,     O,     B,     O,
+  /* 0CE0 */    IV,    IV,  VBlw,  VBlw,     O,     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0CF0 */     O,     R,     R,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Malayalam */
+
+  /* 0D00 */     O, VMAbv, VMPst, VMPst,     O,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,     O,    IV,    IV,
+  /* 0D10 */    IV,     O,    IV,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0D20 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0D30 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     B,  VPst,  VPst,
+  /* 0D40 */  VPst,  VPst,  VPst,  VBlw,  VBlw,     O,  VPre,  VPre,  VPre,     O,  VPre,  VPre,  VPre,     H,     R,     O,
+  /* 0D50 */     O,     O,     O,     O,     O,     O,     O,  VPst,     O,     O,     O,     O,     O,     O,     O,    IV,
+  /* 0D60 */    IV,    IV,  VBlw,  VBlw,     O,     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0D70 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,   IND,   IND,   IND,   IND,   IND,   IND,
+
+  /* Sinhala */
+
+  /* 0D80 */     O,     O, VMPst, VMPst,     O,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,
+  /* 0D90 */    IV,    IV,    IV,    IV,    IV,    IV,    IV,     O,     O,     O,     B,     B,     B,     B,     B,     B,
+  /* 0DA0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0DB0 */     B,     B,     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     B,     O,     O,
+  /* 0DC0 */     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     H,     O,     O,     O,     O,  VPst,
+  /* 0DD0 */  VPst,  VPst,  VAbv,  VAbv,  VBlw,     O,  VBlw,     O,  VPst,  VPre,  VPre,  VPre,  VPre,  VPre,  VPre,  VPst,
+  /* 0DE0 */     O,     O,     O,     O,     O,     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0DF0 */     O,     O,  VPst,  VPst,     O,     O,     O,     O,
+
+#define use_offset_0x1000u 1352
+
+
+  /* Myanmar */
+
+  /* 1000 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1010 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1020 */     B,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,  VPst,  VPst,  VAbv,  VAbv,  VBlw,
+  /* 1030 */  VBlw,  VPre,  VAbv,  VAbv,  VAbv,  VAbv, VMAbv, VMBlw, VMPst,     H,  VAbv,  MPst,  MPre,  MBlw,  MBlw,     B,
+  /* 1040 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,    GB,     O,
+  /* 1050 */     B,     B,    IV,    IV,    IV,    IV,  VPst,  VPst,  VBlw,  VBlw,     B,     B,     B,     B,  MBlw,  MBlw,
+  /* 1060 */  MBlw,     B,  VPst, VMPst, VMPst,     B,     B,  VPst,  VPst, VMPst, VMPst, VMPst, VMPst, VMPst,     B,     B,
+  /* 1070 */     B,  VAbv,  VAbv,  VAbv,  VAbv,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1080 */     B,     B,  MBlw,  VPst,  VPre,  VAbv,  VAbv, VMPst, VMPst, VMPst, VMPst, VMPst, VMPst, VMBlw,     B, VMPst,
+  /* 1090 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B, VMPst, VMPst,  VPst,  VAbv,     O,     O,
+
+#define use_offset_0x1700u 1512
+
+
+  /* Tagalog */
+
+  /* 1700 */    IV,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     B,     B,
+  /* 1710 */     B,     B,  VAbv,  VBlw,  VBlw,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Hanunoo */
+
+  /* 1720 */    IV,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1730 */     B,     B,  VAbv,  VBlw,  VBlw,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Buhid */
+
+  /* 1740 */    IV,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1750 */     B,     B,  VAbv,  VBlw,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Tagbanwa */
+
+  /* 1760 */    IV,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     B,     B,
+  /* 1770 */     B,     O,  VAbv,  VBlw,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Khmer */
+
+  /* 1780 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1790 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 17A0 */     B,     B,     B,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,
+  /* 17B0 */    IV,    IV,    IV,    IV,     O,     O,  VPst,  VAbv,  VAbv,  VAbv,  VAbv,  VBlw,  VBlw,  VBlw,  VPre,  VPre,
+  /* 17C0 */  VPre,  VPre,  VPre,  VPre,  VPre,  VPre, VMAbv, VMPst,  VPst, VMAbv, VMAbv,    FM,  FAbv, CMAbv,    FM,    FM,
+  /* 17D0 */    FM,  VAbv,     H,    FM,     O,     O,     O,     O,     O,     O,     O,     O,     B,  VAbv,     O,     O,
+  /* 17E0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
+
+#define use_offset_0x1900u 1752
+
+
+  /* Limbu */
+
+  /* 1900 */    GB,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1910 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,
+  /* 1920 */  VAbv,  VAbv,  VBlw,  VPst,  VPst,  VAbv,  VAbv,  VAbv,  VAbv,   SUB,   SUB,   SUB,     O,     O,     O,     O,
+  /* 1930 */  FPst,  FPst, VMBlw,  FPst,  FPst,  FPst,  FPst,  FPst,  FPst,  FBlw,  VAbv,    FM,     O,     O,     O,     O,
+  /* 1940 */     O,     O,     O,     O,     O,     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+
+  /* Tai Le */
+
+  /* 1950 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1960 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,
+  /* 1970 */     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* New Tai Lue */
+
+  /* 1980 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1990 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 19A0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,
+  /* 19B0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 19C0 */     B,     B,     B,     B,     B,     B,     B,     B, VMPst, VMPst,     O,     O,     O,     O,     O,     O,
+  /* 19D0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
+  /* 19E0 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* 19F0 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Buginese */
+
+  /* 1A00 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1A10 */     B,     B,     B,     B,     B,     B,     B,  VAbv,  VBlw,  VPre,  VPst,  VAbv,     O,     O,     O,     O,
+
+  /* Tai Tham */
+
+  /* 1A20 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1A30 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1A40 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,    IV,    IV,    IV,
+  /* 1A50 */    IV,    IV,    IV,     B,     B,  MPre,  MBlw,  FPst,  FAbv,  FAbv,  FAbv,  FBlw,  FBlw,  FBlw,  FBlw,     O,
+  /* 1A60 */     H,  VPst,  VAbv,  VPst,  VPst,  VAbv,  VAbv,  VAbv,  VAbv,  VBlw,  VBlw,  VAbv,  VBlw,  VPst,  VPre,  VPre,
+  /* 1A70 */  VPre,  VPre,  VPre,  VAbv,  VAbv, VMAbv, VMAbv, VMAbv, VMAbv, VMAbv,    FM,    FM,    FM,     O,     O,    FM,
+  /* 1A80 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
+  /* 1A90 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
+
+#define use_offset_0x1b00u 2168
+
+
+  /* Balinese */
+
+  /* 1B00 */ VMAbv, VMAbv, VMAbv,  FAbv, VMPst,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,
+  /* 1B10 */    IV,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1B20 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1B30 */     B,     B,     B,     B, CMAbv,  VPst,  VAbv,  VAbv,  VBlw,  VBlw,  VBlw,  VBlw,  VAbv,  VAbv,  VPre,  VPre,
+  /* 1B40 */  VPre,  VPre,  VAbv,  VAbv,     H,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,
+  /* 1B50 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
+  /* 1B60 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O, SMAbv, SMBlw, SMAbv, SMAbv, SMAbv,
+  /* 1B70 */ SMAbv, SMAbv, SMAbv, SMAbv,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Sundanese */
+
+  /* 1B80 */ VMAbv,  FAbv, VMPst,    IV,    IV,    IV,    IV,    IV,    IV,    IV,     B,     B,     B,     B,     B,     B,
+  /* 1B90 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1BA0 */     B,   SUB,   SUB,   SUB,  VAbv,  VBlw,  VPre,  VPst,  VAbv,  VAbv,  VPst,     H,   SUB,   SUB,     B,     B,
+  /* 1BB0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+
+  /* Batak */
+
+  /* 1BC0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1BD0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1BE0 */     B,     B,     B,     B,    IV,    IV, CMAbv,  VPst,  VAbv,  VAbv,  VPst,  VPst,  VPst,  VAbv,  VPst,  VAbv,
+  /* 1BF0 */  FAbv,  FAbv,  VPst,  VPst,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Lepcha */
+
+  /* 1C00 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1C10 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 1C20 */     B,     B,     B,     B,   SUB,   SUB,  VPst,  VPre,  VPre,  VPre,  VPst,  VPst,  VBlw,  FAbv,  FAbv,  FAbv,
+  /* 1C30 */  FAbv,  FAbv,  FAbv,  FAbv, VMPre, VMPre,    FM, CMBlw,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* 1C40 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     B,     B,     B,
+
+#define use_offset_0x1cd0u 2504
+
+
+  /* Vedic Extensions */
+
+  /* 1CD0 */ VMAbv, VMAbv, VMAbv,     O, VMBlw, VMBlw, VMBlw, VMBlw, VMBlw, VMBlw, VMAbv, VMAbv, VMBlw, VMBlw, VMBlw, VMBlw,
+  /* 1CE0 */ VMAbv, VMPst, VMBlw, VMBlw, VMBlw, VMBlw, VMBlw, VMBlw, VMBlw,     O,     O,     O,     O, VMBlw,     O,     O,
+  /* 1CF0 */     O,     O, VMPst, VMPst, VMAbv,     O,     O,     O, VMAbv, VMAbv,     O,     O,     O,     O,     O,     O,
+
+#define use_offset_0x2008u 2552
+
+
+  /* General Punctuation */
+                                                                         O,     O,     O,     O,  ZWNJ,   ZWJ,     O,     O,
+  /* 2010 */    GB,    GB,    GB,    GB,    GB,     O,     O,     O,
+
+#define use_offset_0x2060u 2568
+
+  /* 2060 */    WJ,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Superscripts and Subscripts */
+
+  /* 2070 */     O,     O,     O,     O,    FM,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* 2080 */     O,     O,    FM,    FM,    FM,     O,     O,     O,
+
+#define use_offset_0xa800u 2608
+
+
+  /* Syloti Nagri */
+
+  /* A800 */    IV,    IV,     O,    IV,    IV,    IV,  VAbv,     B,     B,     B,     B, VMAbv,     B,     B,     B,     B,
+  /* A810 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* A820 */     B,     B,     B,  VPst,  VPst,  VBlw,  VAbv,  VPst,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* A830 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Phags-pa */
+
+  /* A840 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* A850 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* A860 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* A870 */     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Saurashtra */
+
+  /* A880 */ VMPst, VMPst,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,
+  /* A890 */    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* A8A0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* A8B0 */     B,     B,     B,     B,  FPst,  VPst,  VPst,  VPst,  VPst,  VPst,  VPst,  VPst,  VPst,  VPst,  VPst,  VPst,
+  /* A8C0 */  VPst,  VPst,  VPst,  VPst,     H,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* A8D0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
+
+  /* Devanagari Extended */
+
+  /* A8E0 */ VMAbv, VMAbv, VMAbv, VMAbv, VMAbv, VMAbv, VMAbv, VMAbv, VMAbv, VMAbv, VMAbv, VMAbv, VMAbv, VMAbv, VMAbv, VMAbv,
+  /* A8F0 */ VMAbv, VMAbv,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Kayah Li */
+
+  /* A900 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* A910 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* A920 */     B,     B,     B,     B,     B,     B,  VAbv,  VAbv,  VAbv,  VAbv,  VAbv, VMBlw, VMBlw, VMBlw,     O,     O,
+
+  /* Rejang */
+
+  /* A930 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* A940 */     B,     B,     B,     B,     B,     B,     B,  VBlw,  VBlw,  VBlw,  VAbv,  VBlw,  VBlw,  VBlw,  VBlw,  FAbv,
+  /* A950 */  FAbv,  FAbv,  FPst,  VPst,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* A960 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* A970 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Javanese */
+
+  /* A980 */ VMAbv, VMAbv,  FAbv, VMPst,    IV,    IV,    IV,    IV,    IV,     B,     B,     B,    IV,    IV,    IV,     B,
+  /* A990 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* A9A0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* A9B0 */     B,     B,     B, CMAbv,  VPst,  VPst,  VAbv,  VAbv,  VBlw,  VBlw,  VPre,  VPre,  VAbv,   SUB,  MPst,  MPst,
+  /* A9C0 */     H,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* A9D0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
+
+  /* Myanmar Extended-B */
+
+  /* A9E0 */     B,     B,     B,     B,     B,  VAbv,     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* A9F0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,
+
+  /* Cham */
+
+  /* AA00 */    IV,    IV,    IV,    IV,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* AA10 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* AA20 */     B,     B,     B,     B,     B,     B,     B,     B,     B,  VAbv,  VAbv,  VAbv,  VAbv,  VBlw,  VAbv,  VPre,
+  /* AA30 */  VPre,  VAbv,  VBlw,  MPst,  MPre,  MBlw,  MBlw,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* AA40 */     B,     B,     B,  FAbv,     B,     B,     B,     B,     B,     B,     B,     B,  FAbv,  FPst,     O,     O,
+  /* AA50 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
+
+  /* Myanmar Extended-A */
+
+  /* AA60 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* AA70 */     O,     B,     B,     B,     O,     O,     O,     O,     O,     O,     B, VMPst, VMAbv, VMPst,     B,     B,
+
+  /* Tai Viet */
+
+  /* AA80 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* AA90 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* AAA0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* AAB0 */  VAbv,     B,  VAbv,  VAbv,  VBlw,     B,     B,  VAbv,  VAbv,     B,     B,     B,     B,     B,  VAbv, VMAbv,
+  /* AAC0 */     B, VMAbv,     B,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* AAD0 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Meetei Mayek Extensions */
+
+  /* AAE0 */    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,  VPre,  VBlw,  VAbv,  VPre,  VPst,
+  /* AAF0 */     O,     O,     O,     O,     O, VMPst,     H,     O,
+
+#define use_offset_0xabc0u 3368
+
+
+  /* Meetei Mayek */
+
+  /* ABC0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,    IV,    IV,
+  /* ABD0 */     B,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* ABE0 */     B,     B,     B,  VPst,  VPst,  VAbv,  VPst,  VPst,  VBlw,  VPst,  VPst,     O, VMPst,  VBlw,     O,     O,
+  /* ABF0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
+
+#define use_offset_0xfe00u 3432
+
+
+  /* Variation Selectors */
+
+  /* FE00 */    VS,    VS,    VS,    VS,    VS,    VS,    VS,    VS,    VS,    VS,    VS,    VS,    VS,    VS,    VS,    VS,
+
+#define use_offset_0x10a00u 3448
+
+
+  /* Kharoshthi */
+
+  /* 10A00 */     B,  VBlw,  VBlw,  VBlw,     O,  VAbv,  VBlw,     O,     O,     O,     O,     O,  VBlw,  VBlw, VMBlw, VMAbv,
+  /* 10A10 */     B,     B,     B,     B,     O,     B,     B,     B,     O,     B,     B,     B,     B,     B,     B,     B,
+  /* 10A20 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 10A30 */     B,     B,     B,     B,     O,     O,     O,     O, CMAbv, CMBlw, CMBlw,     O,     O,     O,     O,     H,
+  /* 10A40 */     B,     B,     B,     B,     B,     B,     B,     B,
+
+#define use_offset_0x11000u 3520
+
+
+  /* Brahmi */
+
+  /* 11000 */ VMPst, VMAbv, VMPst,     R,     R,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,
+  /* 11010 */    IV,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 11020 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 11030 */     B,     B,     B,     B,     B,     B,     B,     B,  VAbv,  VAbv,  VAbv,  VAbv,  VBlw,  VBlw,  VBlw,  VBlw,
+  /* 11040 */  VBlw,  VBlw,  VAbv,  VAbv,  VAbv,  VAbv,     H,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* 11050 */     O,     O,     N,     N,     N,     N,     N,     N,     N,     N,     N,     N,     N,     N,     N,     N,
+  /* 11060 */     N,     N,     N,     N,     N,     N,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 11070 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Kaithi */
+
+  /* 11080 */ VMAbv, VMAbv, VMPst,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,     B,     B,     B,
+  /* 11090 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 110A0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 110B0 */  VPst,  VPre,  VPst,  VBlw,  VBlw,  VAbv,  VAbv,  VPst,  VPst,     H, CMBlw,     O,     O,     O,     O,     O,
+
+#define use_offset_0x11100u 3712
+
+
+  /* Chakma */
+
+  /* 11100 */ VMAbv, VMAbv, VMAbv,    IV,    IV,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 11110 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 11120 */     B,     B,     B,     B,     B,     B,     B,  VAbv,  VAbv,  VAbv,  VBlw,  VBlw,  VPre,  VAbv,  VAbv,  VAbv,
+  /* 11130 */  VAbv,  VBlw,  VBlw,     H,  VAbv,     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 11140 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Mahajani */
+
+  /* 11150 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 11160 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 11170 */     B,     B,     B, CMBlw,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Sharada */
+
+  /* 11180 */ VMAbv, VMAbv, VMPst,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,
+  /* 11190 */    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 111A0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 111B0 */     B,     B,     B,  VPst,  VPre,  VPst,  VBlw,  VBlw,  VBlw,  VBlw,  VBlw,  VBlw,  VAbv,  VAbv,  VAbv,  VAbv,
+  /* 111C0 */     H,     B,     R,     R,     O,     O,     O,     O,     O,     O, CMBlw,  VAbv,  VBlw,     O,     O,     O,
+  /* 111D0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
+
+  /* Sinhala Archaic Numbers */
+
+  /* 111E0 */     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 111F0 */     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Khojki */
+
+  /* 11200 */    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 11210 */     B,     B,     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 11220 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,  VPst,  VPst,  VPst,  VBlw,
+  /* 11230 */  VAbv,  VAbv,  VAbv,  VAbv, VMAbv,     H, CMAbv, CMAbv,
+
+#define use_offset_0x11280u 4024
+
+
+  /* Multani */
+
+  /* 11280 */    IV,    IV,    IV,    IV,     B,     B,     B,     O,     B,     O,     B,     B,     B,     B,     O,     B,
+  /* 11290 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     B,
+  /* 112A0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Khudawadi */
+
+  /* 112B0 */    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,     B,     B,     B,     B,     B,     B,
+  /* 112C0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 112D0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B, VMAbv,
+  /* 112E0 */  VPst,  VPre,  VPst,  VBlw,  VBlw,  VAbv,  VAbv,  VAbv,  VAbv, CMBlw,  VBlw,     O,     O,     O,     O,     O,
+  /* 112F0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
+
+  /* Grantha */
+
+  /* 11300 */ VMAbv, VMAbv, VMPst, VMPst,     O,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,     O,     O,    IV,
+  /* 11310 */    IV,     O,     O,    IV,    IV,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 11320 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     B,     B,     B,     B,     B,     B,
+  /* 11330 */     B,     O,     B,     B,     O,     B,     B,     B,     B,     B,     O,     O, CMBlw,     B,  VPst,  VPst,
+  /* 11340 */  VAbv,  VPst,  VPst,  VPst,  VPst,     O,     O,  VPre,  VPre,     O,     O,  VPre,  VPre,     H,     O,     O,
+  /* 11350 */     O,     O,     O,     O,     O,     O,     O,  VPst,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* 11360 */    IV,    IV,  VPst,  VPst,     O,     O, VMAbv, VMAbv, VMAbv, VMAbv, VMAbv, VMAbv, VMAbv,     O,     O,     O,
+  /* 11370 */ VMAbv, VMAbv, VMAbv, VMAbv, VMAbv,     O,     O,     O,
+
+#define use_offset_0x11480u 4272
+
+
+  /* Tirhuta */
+
+  /* 11480 */     O,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,     B,
+  /* 11490 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 114A0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 114B0 */  VPst,  VPre,  VPst,  VBlw,  VBlw,  VBlw,  VBlw,  VBlw,  VBlw,  VPre,  VAbv,  VPre,  VPre,  VPst,  VPre, VMAbv,
+  /* 114C0 */ VMAbv, VMPst,     H, CMBlw,     B,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* 114D0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
+
+#define use_offset_0x11580u 4368
+
+
+  /* Siddham */
+
+  /* 11580 */    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,     B,     B,
+  /* 11590 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 115A0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,  VPst,
+  /* 115B0 */  VPre,  VPst,  VBlw,  VBlw,  VBlw,  VBlw,     O,     O,  VPre,  VPre,  VPre,  VPre, VMAbv, VMAbv, VMPst,     H,
+  /* 115C0 */ CMBlw,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* 115D0 */     O,     O,     O,     O,     O,     O,     O,     O,    IV,    IV,    IV,    IV,  VBlw,  VBlw,     O,     O,
+  /* 115E0 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* 115F0 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Modi */
+
+  /* 11600 */    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,     B,     B,
+  /* 11610 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 11620 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 11630 */  VPst,  VPst,  VPst,  VBlw,  VBlw,  VBlw,  VBlw,  VBlw,  VBlw,  VAbv,  VAbv,  VPst,  VPst, VMAbv, VMPst,     H,
+  /* 11640 */  VAbv,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* 11650 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
+  /* 11660 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* 11670 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Takri */
+
+  /* 11680 */    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,    IV,     B,     B,     B,     B,     B,     B,
+  /* 11690 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 116A0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B, VMAbv, VMPst,  VAbv,  VPre,  VPst,
+  /* 116B0 */  VBlw,  VBlw,  VAbv,  VAbv,  VAbv,  VAbv,     H, CMBlw,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* 116C0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
+  /* 116D0 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* 116E0 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+  /* 116F0 */     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
+
+  /* Ahom */
+
+  /* 11700 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 11710 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,  MBlw,  MPre,  MAbv,
+  /* 11720 */  VPst,  VPst,  VAbv,  VAbv,  VBlw,  VBlw,  VPre,  VAbv,  VBlw,  VAbv,  VAbv,  VAbv,     O,     O,     O,     O,
+  /* 11730 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,
+
+}; /* Table items: 4816; occupancy: 72% */
+
+USE_TABLE_ELEMENT_TYPE
+hb_use_get_categories (hb_codepoint_t u)
+{
+  switch (u >> 12)
+  {
+    case 0x0u:
+      if (hb_in_range (u, 0x0028u, 0x003Fu)) return use_table[u - 0x0028u + use_offset_0x0028u];
+      if (hb_in_range (u, 0x00A0u, 0x00D7u)) return use_table[u - 0x00A0u + use_offset_0x00a0u];
+      if (hb_in_range (u, 0x0900u, 0x0DF7u)) return use_table[u - 0x0900u + use_offset_0x0900u];
+      if (unlikely (u == 0x034Fu)) return CGJ;
+      break;
+
+    case 0x1u:
+      if (hb_in_range (u, 0x1000u, 0x109Fu)) return use_table[u - 0x1000u + use_offset_0x1000u];
+      if (hb_in_range (u, 0x1700u, 0x17EFu)) return use_table[u - 0x1700u + use_offset_0x1700u];
+      if (hb_in_range (u, 0x1900u, 0x1A9Fu)) return use_table[u - 0x1900u + use_offset_0x1900u];
+      if (hb_in_range (u, 0x1B00u, 0x1C4Fu)) return use_table[u - 0x1B00u + use_offset_0x1b00u];
+      if (hb_in_range (u, 0x1CD0u, 0x1CFFu)) return use_table[u - 0x1CD0u + use_offset_0x1cd0u];
+      break;
+
+    case 0x2u:
+      if (hb_in_range (u, 0x2008u, 0x2017u)) return use_table[u - 0x2008u + use_offset_0x2008u];
+      if (hb_in_range (u, 0x2060u, 0x2087u)) return use_table[u - 0x2060u + use_offset_0x2060u];
+      if (unlikely (u == 0x25CCu)) return GB;
+      break;
+
+    case 0xAu:
+      if (hb_in_range (u, 0xA800u, 0xAAF7u)) return use_table[u - 0xA800u + use_offset_0xa800u];
+      if (hb_in_range (u, 0xABC0u, 0xABFFu)) return use_table[u - 0xABC0u + use_offset_0xabc0u];
+      break;
+
+    case 0xFu:
+      if (hb_in_range (u, 0xFE00u, 0xFE0Fu)) return use_table[u - 0xFE00u + use_offset_0xfe00u];
+      break;
+
+    case 0x10u:
+      if (hb_in_range (u, 0x10A00u, 0x10A47u)) return use_table[u - 0x10A00u + use_offset_0x10a00u];
+      break;
+
+    case 0x11u:
+      if (hb_in_range (u, 0x11000u, 0x110BFu)) return use_table[u - 0x11000u + use_offset_0x11000u];
+      if (hb_in_range (u, 0x11100u, 0x11237u)) return use_table[u - 0x11100u + use_offset_0x11100u];
+      if (hb_in_range (u, 0x11280u, 0x11377u)) return use_table[u - 0x11280u + use_offset_0x11280u];
+      if (hb_in_range (u, 0x11480u, 0x114DFu)) return use_table[u - 0x11480u + use_offset_0x11480u];
+      if (hb_in_range (u, 0x11580u, 0x1173Fu)) return use_table[u - 0x11580u + use_offset_0x11580u];
+      if (unlikely (u == 0x1107Fu)) return HN;
+      break;
+
+    default:
+      break;
+  }
+  return USE_O;
+}
+
+#undef B
+#undef CGJ
+#undef FM
+#undef GB
+#undef H
+#undef HN
+#undef IND
+#undef IV
+#undef N
+#undef O
+#undef R
+#undef Rsv
+#undef S
+#undef SUB
+#undef VS
+#undef WJ
+#undef ZWJ
+#undef ZWNJ
+#undef CMBlw
+#undef CMAbv
+#undef FBlw
+#undef FPst
+#undef FAbv
+#undef MPre
+#undef MBlw
+#undef MPst
+#undef MAbv
+#undef SMBlw
+#undef SMAbv
+#undef VPre
+#undef VBlw
+#undef VPst
+#undef VAbv
+#undef VMPre
+#undef VMBlw
+#undef VMPst
+#undef VMAbv
+
+/* == End of generated table == */

src/hb-ot-shape-complex-use.cc

@@ -0,0 +1,593 @@
+/*
+ * Copyright © 2015  Mozilla Foundation.
+ * Copyright © 2015  Google, Inc.
+ *
+ *  This is part of HarfBuzz, a text shaping library.
+ *
+ * Permission is hereby granted, without written agreement and without
+ * license or royalty fees, to use, copy, modify, and distribute this
+ * software and its documentation for any purpose, provided that the
+ * above copyright notice and the following two paragraphs appear in
+ * all copies of this software.
+ *
+ * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
+ * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
+ * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ *
+ * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
+ * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+ *
+ * Mozilla Author(s): Jonathan Kew
+ * Google Author(s): Behdad Esfahbod
+ */
+
+#include "hb-ot-shape-complex-use-private.hh"
+#include "hb-ot-shape-complex-arabic-private.hh"
+
+/* buffer var allocations */
+#define use_category() complex_var_u8_0()
+
+
+/*
+ * Universal Shaping Engine.
+ * https://www.microsoft.com/typography/OpenTypeDev/USE/intro.htm
+ */
+
+static const hb_tag_t
+basic_features[] =
+{
+  /*
+   * Basic features.
+   * These features are applied all at once, before reordering.
+   */
+  HB_TAG('r','k','r','f'),
+  HB_TAG('a','b','v','f'),
+  HB_TAG('b','l','w','f'),
+  HB_TAG('h','a','l','f'),
+  HB_TAG('p','s','t','f'),
+  HB_TAG('v','a','t','u'),
+  HB_TAG('c','j','c','t'),
+};
+static const hb_tag_t
+arabic_features[] =
+{
+  HB_TAG('i','s','o','l'),
+  HB_TAG('i','n','i','t'),
+  HB_TAG('m','e','d','i'),
+  HB_TAG('f','i','n','a'),
+  /* The spec doesn't specify these but we apply anyway, since our Arabic shaper
+   * does.  These are only used in Syriac spec. */
+  HB_TAG('m','e','d','2'),
+  HB_TAG('f','i','n','2'),
+  HB_TAG('f','i','n','3'),
+};
+/* Same order as arabic_features.  Don't need Syriac stuff.*/
+enum joining_form_t {
+  ISOL,
+  INIT,
+  MEDI,
+  FINA,
+  _NONE
+};
+static const hb_tag_t
+other_features[] =
+{
+  /*
+   * Other features.
+   * These features are applied all at once, after reordering.
+   */
+  HB_TAG('a','b','v','s'),
+  HB_TAG('b','l','w','s'),
+  HB_TAG('h','a','l','n'),
+  HB_TAG('p','r','e','s'),
+  HB_TAG('p','s','t','s'),
+  /* Positioning features, though we don't care about the types. */
+  HB_TAG('d','i','s','t'),
+  HB_TAG('a','b','v','m'),
+  HB_TAG('b','l','w','m'),
+};
+
+static void
+setup_syllables (const hb_ot_shape_plan_t *plan,
+		 hb_font_t *font,
+		 hb_buffer_t *buffer);
+static void
+clear_substitution_flags (const hb_ot_shape_plan_t *plan,
+			  hb_font_t *font,
+			  hb_buffer_t *buffer);
+static void
+record_rphf (const hb_ot_shape_plan_t *plan,
+	     hb_font_t *font,
+	     hb_buffer_t *buffer);
+static void
+record_pref (const hb_ot_shape_plan_t *plan,
+	     hb_font_t *font,
+	     hb_buffer_t *buffer);
+static void
+reorder (const hb_ot_shape_plan_t *plan,
+	 hb_font_t *font,
+	 hb_buffer_t *buffer);
+
+static void
+collect_features_use (hb_ot_shape_planner_t *plan)
+{
+  hb_ot_map_builder_t *map = &plan->map;
+
+  /* Do this before any lookups have been applied. */
+  map->add_gsub_pause (setup_syllables);
+
+  /* "Default glyph pre-processing group" */
+  map->add_global_bool_feature (HB_TAG('l','o','c','l'));
+  map->add_global_bool_feature (HB_TAG('c','c','m','p'));
+  map->add_global_bool_feature (HB_TAG('n','u','k','t'));
+  map->add_global_bool_feature (HB_TAG('a','k','h','n'));
+
+  /* "Reordering group" */
+  map->add_gsub_pause (clear_substitution_flags);
+  map->add_feature (HB_TAG('r','p','h','f'), 1, F_MANUAL_ZWJ);
+  map->add_gsub_pause (record_rphf);
+  map->add_gsub_pause (clear_substitution_flags);
+  map->add_feature (HB_TAG('p','r','e','f'), 1, F_GLOBAL | F_MANUAL_ZWJ);
+  map->add_gsub_pause (record_pref);
+
+  /* "Orthographic unit shaping group" */
+  for (unsigned int i = 0; i < ARRAY_LENGTH (basic_features); i++)
+    map->add_feature (basic_features[i], 1, F_GLOBAL | F_MANUAL_ZWJ);
+
+  map->add_gsub_pause (reorder);
+
+  /* "Topographical features" */
+  for (unsigned int i = 0; i < ARRAY_LENGTH (arabic_features); i++)
+    map->add_feature (arabic_features[i], 1, F_NONE);
+  map->add_gsub_pause (NULL);
+
+  /* "Standard typographic presentation" and "Positional feature application" */
+  for (unsigned int i = 0; i < ARRAY_LENGTH (other_features); i++)
+    map->add_feature (other_features[i], 1, F_GLOBAL | F_MANUAL_ZWJ);
+}
+
+struct use_shape_plan_t
+{
+  ASSERT_POD ();
+
+  hb_mask_t rphf_mask;
+
+  arabic_shape_plan_t *arabic_plan;
+};
+
+static bool
+has_arabic_joining (hb_script_t script)
+{
+  /* List of scripts that have data in arabic-table. */
+  switch ((int) script)
+  {
+    /* Unicode-1.1 additions */
+    case HB_SCRIPT_ARABIC:
+
+    /* Unicode-3.0 additions */
+    case HB_SCRIPT_MONGOLIAN:
+    case HB_SCRIPT_SYRIAC:
+
+    /* Unicode-5.0 additions */
+    case HB_SCRIPT_NKO:
+    case HB_SCRIPT_PHAGS_PA:
+
+    /* Unicode-6.0 additions */
+    case HB_SCRIPT_MANDAIC:
+
+    /* Unicode-7.0 additions */
+    case HB_SCRIPT_MANICHAEAN:
+    case HB_SCRIPT_PSALTER_PAHLAVI:
+
+      return true;
+
+    default:
+      return false;
+  }
+}
+
+static void *
+data_create_use (const hb_ot_shape_plan_t *plan)
+{
+  use_shape_plan_t *use_plan = (use_shape_plan_t *) calloc (1, sizeof (use_shape_plan_t));
+  if (unlikely (!use_plan))
+    return NULL;
+
+  use_plan->rphf_mask = plan->map.get_1_mask (HB_TAG('r','p','h','f'));
+
+  if (has_arabic_joining (plan->props.script))
+  {
+    use_plan->arabic_plan = (arabic_shape_plan_t *) data_create_arabic (plan);
+    if (unlikely (!use_plan->arabic_plan))
+    {
+      free (use_plan);
+      return NULL;
+    }
+  }
+
+  return use_plan;
+}
+
+static void
+data_destroy_use (void *data)
+{
+  use_shape_plan_t *use_plan = (use_shape_plan_t *) data;
+
+  if (use_plan->arabic_plan)
+    data_destroy_arabic (use_plan->arabic_plan);
+
+  free (data);
+}
+
+enum syllable_type_t {
+  independent_cluster,
+  virama_terminated_cluster,
+  consonant_cluster,
+  vowel_cluster,
+  number_joiner_terminated_cluster,
+  numeral_cluster,
+  symbol_cluster,
+  broken_cluster,
+};
+
+#include "hb-ot-shape-complex-use-machine.hh"
+
+
+static inline void
+set_use_properties (hb_glyph_info_t &info)
+{
+  hb_codepoint_t u = info.codepoint;
+  info.use_category() = hb_use_get_categories (u);
+}
+
+
+static void
+setup_masks_use (const hb_ot_shape_plan_t *plan,
+		 hb_buffer_t              *buffer,
+		 hb_font_t                *font HB_UNUSED)
+{
+  const use_shape_plan_t *use_plan = (const use_shape_plan_t *) plan->data;
+
+  /* Do this before allocating use_category(). */
+  if (use_plan->arabic_plan)
+  {
+    setup_masks_arabic_plan (use_plan->arabic_plan, buffer, plan->props.script);
+  }
+
+  HB_BUFFER_ALLOCATE_VAR (buffer, use_category);
+
+  /* We cannot setup masks here.  We save information about characters
+   * and setup masks later on in a pause-callback. */
+
+  unsigned int count = buffer->len;
+  hb_glyph_info_t *info = buffer->info;
+  for (unsigned int i = 0; i < count; i++)
+    info[i].use_category() = hb_use_get_categories (info[i].codepoint);
+}
+
+static void
+setup_rphf_mask (const hb_ot_shape_plan_t *plan,
+		 hb_buffer_t *buffer)
+{
+  const use_shape_plan_t *use_plan = (const use_shape_plan_t *) plan->data;
+
+  hb_mask_t mask = use_plan->rphf_mask;
+  if (!mask) return;
+
+  hb_glyph_info_t *info = buffer->info;
+
+  foreach_syllable (buffer, start, end)
+  {
+    unsigned int limit = info[start].use_category() == USE_R ? 1 : MIN (3u, end - start);
+    for (unsigned int i = start; i < start + limit; i++)
+      info[i].mask |= mask;
+  }
+}
+
+static void
+setup_topographical_masks (const hb_ot_shape_plan_t *plan,
+			   hb_buffer_t *buffer)
+{
+
+  ASSERT_STATIC (INIT < 4 && ISOL < 4 && MEDI < 4 && FINA < 4);
+  hb_mask_t masks[4], all_masks = 0;
+  for (unsigned int i = 0; i < 4; i++)
+  {
+    masks[i] = plan->map.get_1_mask (arabic_features[i]);
+    if (masks[i] == plan->map.get_global_mask ())
+      masks[i] = 0;
+    all_masks |= masks[i];
+  }
+  if (!all_masks)
+    return;
+  hb_mask_t other_masks = ~all_masks;
+
+  unsigned int last_start = 0;
+  joining_form_t last_form = _NONE;
+  hb_glyph_info_t *info = buffer->info;
+  foreach_syllable (buffer, start, end)
+  {
+    syllable_type_t syllable_type = (syllable_type_t) (info[start].syllable() & 0x0F);
+    switch (syllable_type)
+    {
+      case independent_cluster:
+      case symbol_cluster:
+	/* These don't join.  Nothing to do. */
+	last_form = _NONE;
+	break;
+
+      case virama_terminated_cluster:
+      case consonant_cluster:
+      case vowel_cluster:
+      case number_joiner_terminated_cluster:
+      case numeral_cluster:
+      case broken_cluster:
+
+	bool join = last_form == FINA || last_form == ISOL;
+
+	if (join)
+	{
+	  /* Fixup previous syllable's form. */
+	  last_form = last_form == FINA ? MEDI : INIT;
+	  for (unsigned int i = last_start; i < start; i++)
+	    info[i].mask = (info[i].mask & other_masks) | masks[last_form];
+	}
+
+	/* Form for this syllable. */
+	last_form = join ? FINA : ISOL;
+	for (unsigned int i = start; i < end; i++)
+	  info[i].mask = (info[i].mask & other_masks) | masks[last_form];
+
+	break;
+    }
+
+    last_start = start;
+  }
+}
+
+static void
+setup_syllables (const hb_ot_shape_plan_t *plan,
+		 hb_font_t *font HB_UNUSED,
+		 hb_buffer_t *buffer)
+{
+  find_syllables (buffer);
+  setup_rphf_mask (plan, buffer);
+  setup_topographical_masks (plan, buffer);
+}
+
+static void
+clear_substitution_flags (const hb_ot_shape_plan_t *plan,
+			  hb_font_t *font HB_UNUSED,
+			  hb_buffer_t *buffer)
+{
+  hb_glyph_info_t *info = buffer->info;
+  unsigned int count = buffer->len;
+  for (unsigned int i = 0; i < count; i++)
+    _hb_glyph_info_clear_substituted_and_ligated_and_multiplied (&info[i]);
+}
+
+static void
+record_rphf (const hb_ot_shape_plan_t *plan,
+	     hb_font_t *font,
+	     hb_buffer_t *buffer)
+{
+  const use_shape_plan_t *use_plan = (const use_shape_plan_t *) plan->data;
+
+  hb_mask_t mask = use_plan->rphf_mask;
+  if (!mask) return;
+  hb_glyph_info_t *info = buffer->info;
+
+  foreach_syllable (buffer, start, end)
+  {
+    /* Mark a substituted repha as USE_R. */
+    for (unsigned int i = start; i < end && (info[i].mask & mask); i++)
+      if (_hb_glyph_info_substituted (&info[i]))
+      {
+	info[i].use_category() = USE_R;
+	break;
+      }
+  }
+}
+
+static void
+record_pref (const hb_ot_shape_plan_t *plan,
+	     hb_font_t *font,
+	     hb_buffer_t *buffer)
+{
+  hb_glyph_info_t *info = buffer->info;
+
+  foreach_syllable (buffer, start, end)
+  {
+    /* Mark a substituted pref as VPre, as they behave the same way. */
+    for (unsigned int i = start; i < end; i++)
+      if (_hb_glyph_info_substituted (&info[i]))
+      {
+	info[i].use_category() = USE_VPre;
+	break;
+      }
+  }
+}
+
+static void
+reorder_syllable (hb_buffer_t *buffer, unsigned int start, unsigned int end)
+{
+  syllable_type_t syllable_type = (syllable_type_t) (buffer->info[start].syllable() & 0x0F);
+  /* Only a few syllable types need reordering. */
+  if (unlikely (!(FLAG_SAFE (syllable_type) &
+		  (FLAG (virama_terminated_cluster) |
+		   FLAG (consonant_cluster) |
+		   FLAG (vowel_cluster) |
+		   FLAG (broken_cluster) |
+		   0))))
+    return;
+
+  hb_glyph_info_t *info = buffer->info;
+
+#define HALANT_FLAGS FLAG(USE_H)
+#define BASE_FLAGS (FLAG (USE_B) | FLAG (USE_GB) | FLAG (USE_IV))
+
+  /* Move things forward. */
+  if (info[start].use_category() == USE_R && end - start > 1)
+  {
+    /* Got a repha.  Reorder it to after first base, before first halant. */
+    for (unsigned int i = start + 1; i < end; i++)
+      if (FLAG_UNSAFE (info[i].use_category()) & (HALANT_FLAGS | BASE_FLAGS))
+      {
+	/* If we hit a halant, move before it; otherwise it's a base: move to it's
+	 * place, and shift things in between backward. */
+
+	if (info[i].use_category() == USE_H)
+	  i--;
+
+	hb_glyph_info_t t = info[start];
+	memmove (&info[start], &info[start + 1], (i - start) * sizeof (info[0]));
+	info[i] = t;
+	buffer->merge_clusters (start, i + 1);
+
+	break;
+      }
+  }
+
+  /* Move things back. */
+  unsigned int j = end;
+  for (unsigned int i = start; i < end; i++)
+  {
+    uint32_t flag = FLAG_UNSAFE (info[i].use_category());
+    if (flag & (HALANT_FLAGS | BASE_FLAGS))
+    {
+      /* If we hit a halant, move before it; otherwise it's a base: move to it's
+       * place, and shift things in between backward. */
+      if (info[i].use_category() == USE_H)
+	j = i + 1;
+      else
+	j = i;
+    }
+    else if (((flag) & (FLAG (USE_VPre) | FLAG (USE_VMPre))) &&
+	     /* Only move the first component of a MultipleSubst. */
+	     0 == _hb_glyph_info_get_lig_comp (&info[i]) &&
+	     j < i)
+    {
+      hb_glyph_info_t t = info[i];
+      memmove (&info[j + 1], &info[j], (i - j) * sizeof (info[0]));
+      info[j] = t;
+      buffer->merge_clusters (j, i + 1);
+    }
+  }
+}
+
+static inline void
+insert_dotted_circles (const hb_ot_shape_plan_t *plan HB_UNUSED,
+		       hb_font_t *font,
+		       hb_buffer_t *buffer)
+{
+  /* Note: This loop is extra overhead, but should not be measurable. */
+  bool has_broken_syllables = false;
+  unsigned int count = buffer->len;
+  hb_glyph_info_t *info = buffer->info;
+  for (unsigned int i = 0; i < count; i++)
+    if ((info[i].syllable() & 0x0F) == broken_cluster)
+    {
+      has_broken_syllables = true;
+      break;
+    }
+  if (likely (!has_broken_syllables))
+    return;
+
+
+  hb_codepoint_t dottedcircle_glyph;
+  if (!font->get_glyph (0x25CCu, 0, &dottedcircle_glyph))
+    return;
+
+  hb_glyph_info_t dottedcircle = {0};
+  if (!font->get_glyph (0x25CCu, 0, &dottedcircle.codepoint))
+    return;
+  dottedcircle.use_category() = hb_use_get_categories (0x25CC);
+
+  buffer->clear_output ();
+
+  buffer->idx = 0;
+
+  unsigned int last_syllable = 0;
+  while (buffer->idx < buffer->len)
+  {
+    unsigned int syllable = buffer->cur().syllable();
+    syllable_type_t syllable_type = (syllable_type_t) (syllable & 0x0F);
+    if (unlikely (last_syllable != syllable && syllable_type == broken_cluster))
+    {
+      last_syllable = syllable;
+
+      hb_glyph_info_t info = dottedcircle;
+      info.cluster = buffer->cur().cluster;
+      info.mask = buffer->cur().mask;
+      info.syllable() = buffer->cur().syllable();
+      /* TODO Set glyph_props? */
+
+      /* Insert dottedcircle after possible Repha. */
+      while (buffer->idx < buffer->len &&
+	     last_syllable == buffer->cur().syllable() &&
+	     buffer->cur().use_category() == USE_R)
+        buffer->next_glyph ();
+
+      buffer->output_info (info);
+    }
+    else
+      buffer->next_glyph ();
+  }
+
+  buffer->swap_buffers ();
+}
+
+static void
+reorder (const hb_ot_shape_plan_t *plan,
+	 hb_font_t *font,
+	 hb_buffer_t *buffer)
+{
+  insert_dotted_circles (plan, font, buffer);
+
+  hb_glyph_info_t *info = buffer->info;
+
+  foreach_syllable (buffer, start, end)
+    reorder_syllable (buffer, start, end);
+
+  /* Zero syllables now... */
+  unsigned int count = buffer->len;
+  for (unsigned int i = 0; i < count; i++)
+    info[i].syllable() = 0;
+
+  HB_BUFFER_DEALLOCATE_VAR (buffer, use_category);
+}
+
+static bool
+compose_use (const hb_ot_shape_normalize_context_t *c,
+	     hb_codepoint_t  a,
+	     hb_codepoint_t  b,
+	     hb_codepoint_t *ab)
+{
+  /* Avoid recomposing split matras. */
+  if (HB_UNICODE_GENERAL_CATEGORY_IS_MARK (c->unicode->general_category (a)))
+    return false;
+
+  return c->unicode->compose (a, b, ab);
+}
+
+
+const hb_ot_complex_shaper_t _hb_ot_complex_shaper_use =
+{
+  "use",
+  collect_features_use,
+  NULL, /* override_features */
+  data_create_use,
+  data_destroy_use,
+  NULL, /* preprocess_text */
+  HB_OT_SHAPE_NORMALIZATION_MODE_COMPOSED_DIACRITICS_NO_SHORT_CIRCUIT,
+  NULL, /* decompose */
+  compose_use,
+  setup_masks_use,
+  HB_OT_SHAPE_ZERO_WIDTH_MARKS_NONE,
+  false, /* fallback_position */
+};

src/hb-ot-shape-fallback.cc

@@ -418,13 +418,12 @@ _hb_ot_shape_fallback_position (const hb_ot_shape_plan_t *plan,
   _hb_buffer_assert_gsubgpos_vars (buffer);
 
   unsigned int start = 0;
-  unsigned int last_cluster = buffer->info[0].cluster;
   unsigned int count = buffer->len;
+  hb_glyph_info_t *info = buffer->info;
   for (unsigned int i = 1; i < count; i++)
-    if (buffer->info[i].cluster != last_cluster) {
+    if (likely (!HB_UNICODE_GENERAL_CATEGORY_IS_MARK (_hb_glyph_info_get_general_category (&info[i])))) {
       position_cluster (plan, font, buffer, start, i);
       start = i;
-      last_cluster = buffer->info[i].cluster;
     }
   position_cluster (plan, font, buffer, start, count);
 }

src/hb-ot-shape-normalize.cc

@@ -323,7 +323,7 @@ _hb_ot_shape_normalize (const hb_ot_shape_plan_t *plan,
   {
     unsigned int end;
     for (end = buffer->idx + 1; end < count; end++)
-      if (buffer->cur().cluster != buffer->info[end].cluster)
+      if (likely (!HB_UNICODE_GENERAL_CATEGORY_IS_MARK (_hb_glyph_info_get_general_category (&buffer->info[end]))))
         break;
 
     decompose_cluster (&c, end, might_short_circuit, always_short_circuit);

src/hb-ot-shape.cc

@@ -59,10 +59,6 @@ static hb_tag_t horizontal_features[] = {
   HB_TAG('r','c','l','t'),
 };
 
-static hb_tag_t vertical_features[] = {
-  HB_TAG('v','e','r','t'),
-};
-
 
 
 static void
@@ -105,10 +101,13 @@ hb_ot_shape_collect_features (hb_ot_shape_planner_t          *planner,
 			(horizontal_features[i] == HB_TAG('k','e','r','n') ?
 			 F_HAS_FALLBACK : F_NONE));
   else
-    for (unsigned int i = 0; i < ARRAY_LENGTH (vertical_features); i++)
+  {
-      map->add_feature (vertical_features[i], 1, F_GLOBAL |
+    /* We really want to find a 'vert' feature if there's any in the font, no
-			(vertical_features[i] == HB_TAG('v','k','r','n') ?
+     * matter which script/langsys it is listed (or not) under.
-			 F_HAS_FALLBACK : F_NONE));
+     * See various bugs referenced from:
+     * https://github.com/behdad/harfbuzz/issues/63 */
+    map->add_feature (HB_TAG ('v','e','r','t'), 1, F_GLOBAL | F_GLOBAL_SEARCH);
+  }
 
   if (planner->shaper->override_features)
     planner->shaper->override_features (planner);
@@ -264,11 +263,22 @@ hb_insert_dotted_circle (hb_buffer_t *buffer, hb_font_t *font)
 static void
 hb_form_clusters (hb_buffer_t *buffer)
 {
+  if (buffer->cluster_level != HB_BUFFER_CLUSTER_LEVEL_MONOTONE_GRAPHEMES)
+    return;
+
+  /* Loop duplicated in hb_ensure_native_direction(). */
+  unsigned int base = 0;
   unsigned int count = buffer->len;
   hb_glyph_info_t *info = buffer->info;
   for (unsigned int i = 1; i < count; i++)
-    if (HB_UNICODE_GENERAL_CATEGORY_IS_MARK (_hb_glyph_info_get_general_category (&info[i])))
+  {
-      buffer->merge_clusters (i - 1, i + 1);
+    if (likely (!HB_UNICODE_GENERAL_CATEGORY_IS_MARK (_hb_glyph_info_get_general_category (&info[i]))))
+    {
+      buffer->merge_clusters (base, i);
+      base = i;
+    }
+  }
+  buffer->merge_clusters (base, count);
 }
 
 static void
@@ -283,7 +293,27 @@ hb_ensure_native_direction (hb_buffer_t *buffer)
   if ((HB_DIRECTION_IS_HORIZONTAL (direction) && direction != hb_script_get_horizontal_direction (buffer->props.script)) ||
       (HB_DIRECTION_IS_VERTICAL   (direction) && direction != HB_DIRECTION_TTB))
   {
-    hb_buffer_reverse_clusters (buffer);
+    /* Same loop as hb_form_clusters().
+     * Since form_clusters() merged clusters already, we don't merge. */
+    unsigned int base = 0;
+    unsigned int count = buffer->len;
+    hb_glyph_info_t *info = buffer->info;
+    for (unsigned int i = 1; i < count; i++)
+    {
+      if (likely (!HB_UNICODE_GENERAL_CATEGORY_IS_MARK (_hb_glyph_info_get_general_category (&info[i]))))
+      {
+	buffer->reverse_range (base, i);
+	if (buffer->cluster_level == HB_BUFFER_CLUSTER_LEVEL_MONOTONE_CHARACTERS)
+	  buffer->merge_clusters (base, i);
+	base = i;
+      }
+    }
+    buffer->reverse_range (base, count);
+    if (buffer->cluster_level == HB_BUFFER_CLUSTER_LEVEL_MONOTONE_CHARACTERS)
+      buffer->merge_clusters (base, count);
+
+    buffer->reverse ();
+
     buffer->props.direction = HB_DIRECTION_REVERSE (buffer->props.direction);
   }
 }
@@ -305,7 +335,7 @@ hb_ot_mirror_chars (hb_ot_shape_context_t *c)
   hb_glyph_info_t *info = buffer->info;
   for (unsigned int i = 0; i < count; i++) {
     hb_codepoint_t codepoint = unicode->mirroring (info[i].codepoint);
-    if (likely (codepoint == info[i].codepoint))
+    if (likely (codepoint == info[i].codepoint || !c->font->has_glyph (codepoint)))
       info[i].mask |= rtlm_mask;
     else
       info[i].codepoint = codepoint;
@@ -380,6 +410,101 @@ hb_ot_shape_setup_masks (hb_ot_shape_context_t *c)
   }
 }
 
+static void
+hb_ot_zero_width_default_ignorables (hb_ot_shape_context_t *c)
+{
+  hb_buffer_t *buffer = c->buffer;
+
+  if (buffer->flags & HB_BUFFER_FLAG_PRESERVE_DEFAULT_IGNORABLES)
+    return;
+
+  unsigned int count = buffer->len;
+  hb_glyph_info_t *info = buffer->info;
+  hb_glyph_position_t *pos = buffer->pos;
+  unsigned int i = 0;
+  for (i = 0; i < count; i++)
+    if (unlikely (_hb_glyph_info_is_default_ignorable (&info[i])))
+      pos[i].x_advance = pos[i].y_advance = pos[i].x_offset = pos[i].y_offset = 0;
+}
+
+static void
+hb_ot_hide_default_ignorables (hb_ot_shape_context_t *c)
+{
+  hb_buffer_t *buffer = c->buffer;
+
+  if (buffer->flags & HB_BUFFER_FLAG_PRESERVE_DEFAULT_IGNORABLES)
+    return;
+
+  unsigned int count = buffer->len;
+  hb_glyph_info_t *info = buffer->info;
+  hb_glyph_position_t *pos = buffer->pos;
+  unsigned int i = 0;
+  for (i = 0; i < count; i++)
+  {
+    if (unlikely (_hb_glyph_info_is_default_ignorable (&info[i])))
+      break;
+  }
+
+  /* No default-ignorables found; return. */
+  if (i == count)
+    return;
+
+  hb_codepoint_t space;
+  if (c->font->get_glyph (' ', 0, &space))
+  {
+    /* Replace default-ignorables with a zero-advance space glyph. */
+    for (/*continue*/; i < count; i++)
+    {
+      if (_hb_glyph_info_is_default_ignorable (&info[i]))
+	info[i].codepoint = space;
+    }
+  }
+  else
+  {
+    /* Merge clusters and delete default-ignorables.
+     * NOTE! We can't use out-buffer as we have positioning data. */
+    unsigned int j = i;
+    for (; i < count; i++)
+    {
+      if (_hb_glyph_info_is_default_ignorable (&info[i]))
+      {
+	/* Merge clusters.
+	 * Same logic as buffer->delete_glyph(), but for in-place removal. */
+
+	unsigned int cluster = info[i].cluster;
+	if (i + 1 < count && cluster == info[i + 1].cluster)
+	  continue; /* Cluster survives; do nothing. */
+
+	if (j)
+	{
+	  /* Merge cluster backward. */
+	  if (cluster < info[j - 1].cluster)
+	  {
+	    unsigned int old_cluster = info[j - 1].cluster;
+	    for (unsigned k = j; k && info[k - 1].cluster == old_cluster; k--)
+	      info[k - 1].cluster = cluster;
+	  }
+	  continue;
+	}
+
+	if (i + 1 < count)
+	  buffer->merge_clusters (i, i + 2); /* Merge cluster forward. */
+
+	continue;
+      }
+
+      if (j != i)
+      {
+	info[j] = info[i];
+	pos[j] = pos[i];
+      }
+      j++;
+    }
+    buffer->len = j;
+  }
+}
+
+
 static inline void
 hb_ot_map_glyphs_fast (hb_buffer_t  *buffer)
 {
@@ -625,6 +750,8 @@ hb_ot_position (hb_ot_shape_context_t *c)
 
   hb_bool_t fallback = !hb_ot_position_complex (c);
 
+  hb_ot_zero_width_default_ignorables (c);
+
   hb_ot_layout_position_finish (c->font, c->buffer);
 
   if (fallback && c->plan->shaper->fallback_position)
@@ -642,66 +769,6 @@ hb_ot_position (hb_ot_shape_context_t *c)
 }
 
 
-/* Post-process */
-
-static void
-hb_ot_hide_default_ignorables (hb_ot_shape_context_t *c)
-{
-  hb_buffer_t *buffer = c->buffer;
-
-  if (buffer->flags & HB_BUFFER_FLAG_PRESERVE_DEFAULT_IGNORABLES)
-    return;
-
-  unsigned int count = buffer->len;
-  hb_glyph_info_t *info = buffer->info;
-  hb_glyph_position_t *pos = buffer->pos;
-  unsigned int i = 0;
-  for (i = 0; i < count; i++)
-  {
-    if (unlikely (!_hb_glyph_info_ligated (&info[i]) &&
-		  _hb_glyph_info_is_default_ignorable (&info[i])))
-      break;
-  }
-
-  /* No default-ignorables found; return. */
-  if (i == count)
-    return;
-
-  hb_codepoint_t space;
-  if (c->font->get_glyph (' ', 0, &space))
-  {
-    /* Replace default-ignorables with a zero-advance space glyph. */
-    for (/*continue*/; i < count; i++)
-    {
-      if (!_hb_glyph_info_ligated (&info[i]) &&
-	   _hb_glyph_info_is_default_ignorable (&info[i]))
-      {
-	info[i].codepoint = space;
-	pos[i].x_advance = pos[i].y_advance = pos[i].x_offset = pos[i].y_offset = 0;
-      }
-    }
-  }
-  else
-  {
-    /* Merge clusters and delete default-ignorables. */
-    buffer->clear_output ();
-    buffer->idx = 0;
-    buffer->next_glyphs (i);
-    while (buffer->idx < buffer->len)
-    {
-      if (!_hb_glyph_info_ligated (&info[buffer->idx]) &&
-	   _hb_glyph_info_is_default_ignorable (&info[buffer->idx]))
-      {
-	buffer->delete_glyph ();
-	continue;
-      }
-      buffer->next_glyph ();
-    }
-    buffer->swap_buffers ();
-  }
-}
-
-
 /* Pull it all together! */
 
 static void

src/hb-private.hh

@@ -196,7 +196,8 @@ static inline unsigned int ARRAY_LENGTH (const Type (&)[n]) { return n; }
 #define _ASSERT_STATIC0(_line, _cond)	_ASSERT_STATIC1 (_line, (_cond))
 #define ASSERT_STATIC(_cond)		_ASSERT_STATIC0 (__LINE__, (_cond))
 
-#define ASSERT_STATIC_EXPR(_cond)((void) sizeof (char[(_cond) ? 1 : -1]))
+/* Note: C++ allows sizeof() of variable-lengh arrays.  So, if _cond is not
+ * constant, it still compiles (ouch!), but at least we'll get a -Wvla warning. */
 #define ASSERT_STATIC_EXPR_ZERO(_cond) (0 * sizeof (char[(_cond) ? 1 : -1]))
 
 #define _PASTE1(a,b) a##b
@@ -848,9 +849,11 @@ hb_in_ranges (T u, T lo1, T hi1, T lo2, T hi2, T lo3, T hi3)
 
 /* Useful for set-operations on small enums.
  * For example, for testing "x ∈ {x1, x2, x3}" use:
- * (FLAG(x) & (FLAG(x1) | FLAG(x2) | FLAG(x3)))
+ * (FLAG_SAFE(x) & (FLAG(x1) | FLAG(x2) | FLAG(x3)))
  */
-#define FLAG(x) (1<<(x))
+#define FLAG(x) (ASSERT_STATIC_EXPR_ZERO ((x) < 32) + (1U << (x)))
+#define FLAG_SAFE(x) (1U << (x))
+#define FLAG_UNSAFE(x) ((x) < 32 ? FLAG_SAFE(x) : 0)
 #define FLAG_RANGE(x,y) (ASSERT_STATIC_EXPR_ZERO ((x) < (y)) + FLAG(y+1) - FLAG(x))
 
 

src/hb-set-private.hh

@@ -36,7 +36,15 @@
  * "approximate member query".  Conceptually these are like Bloom
  * Filter and Quotient Filter, however, much smaller, faster, and
  * designed to fit the requirements of our uses for glyph coverage
- * queries.  As a result, our filters have much higher.
+ * queries.
+ *
+ * Our filters are highly accurate if the lookup covers fairly local
+ * set of glyphs, but fully flooded and ineffective if coverage is
+ * all over the place.
+ *
+ * The frozen-set can be used instead of a digest, to trade more
+ * memory for 100% accuracy, but in practice, that doesn't look like
+ * an attractive trade-off.
  */
 
 template <typename mask_t, unsigned int shift>

src/hb-ucdn.cc

@@ -148,6 +148,12 @@ static const hb_script_t ucdn_script_translate[] =
     HB_SCRIPT_SIDDHAM,
     HB_SCRIPT_TIRHUTA,
     HB_SCRIPT_WARANG_CITI,
+    HB_SCRIPT_AHOM,
+    HB_SCRIPT_ANATOLIAN_HIEROGLYPHS,
+    HB_SCRIPT_HATRAN,
+    HB_SCRIPT_MULTANI,
+    HB_SCRIPT_OLD_HUNGARIAN,
+    HB_SCRIPT_SIGNWRITING,
 };
 
 static hb_unicode_combining_class_t

src/hb-ucdn/ucdn.h

@@ -191,6 +191,12 @@ typedef unsigned __int64 uint64_t;
 #define UCDN_SCRIPT_SIDDHAM 123
 #define UCDN_SCRIPT_TIRHUTA 124
 #define UCDN_SCRIPT_WARANG_CITI 125
+#define UCDN_SCRIPT_AHOM 126
+#define UCDN_SCRIPT_ANATOLIAN_HIEROGLYPHS 127
+#define UCDN_SCRIPT_HATRAN 128
+#define UCDN_SCRIPT_MULTANI 129
+#define UCDN_SCRIPT_OLD_HUNGARIAN 130
+#define UCDN_SCRIPT_SIGNWRITING 131
 
 #define UCDN_GENERAL_CATEGORY_CC 0
 #define UCDN_GENERAL_CATEGORY_CF 1

src/hb-unicode-private.hh

@@ -308,7 +308,7 @@ extern HB_INTERNAL const hb_unicode_funcs_t _hb_unicode_funcs_nil;
 /* Misc */
 
 #define HB_UNICODE_GENERAL_CATEGORY_IS_MARK(gen_cat) \
-	(FLAG (gen_cat) & \
+	(FLAG_SAFE (gen_cat) & \
 	 (FLAG (HB_UNICODE_GENERAL_CATEGORY_SPACING_MARK) | \
 	  FLAG (HB_UNICODE_GENERAL_CATEGORY_ENCLOSING_MARK) | \
 	  FLAG (HB_UNICODE_GENERAL_CATEGORY_NON_SPACING_MARK)))

src/hb-unicode.cc

@@ -146,14 +146,9 @@ hb_unicode_funcs_get_default (void)
 }
 
 #if !defined(HB_NO_UNICODE_FUNCS) && defined(HB_UNICODE_FUNCS_NIL)
-#ifdef _MSC_VER
-#pragma error("Could not find any Unicode functions implementation, you have to provide your own.")
-#pragma error("Consider building hb-ucdn.c.  If you absolutely want to build without any, check the code.")
-#else
 #error "Could not find any Unicode functions implementation, you have to provide your own"
 #error "Consider building hb-ucdn.c.  If you absolutely want to build without any, check the code."
 #endif
-#endif
 
 /**
  * hb_unicode_funcs_create: (Xconstructor)

src/hb-warning.cc

@@ -29,21 +29,11 @@
 
 
 #if defined(HB_ATOMIC_INT_NIL)
-#ifdef _MSC_VER
-#pragma error("Could not find any system to define atomic_int macros, library WILL NOT be thread-safe")
-#pragma error("Check hb-atomic-private.hh for possible resolutions.")
-#else
 #error "Could not find any system to define atomic_int macros, library WILL NOT be thread-safe"
 #error "Check hb-atomic-private.hh for possible resolutions."
 #endif
-#endif
 
 #if defined(HB_MUTEX_IMPL_NIL)
-#ifdef _MSC_VER
-#pragma error("Could not find any system to define mutex macros, library WILL NOT be thread-safe")
-#pragma error("Check hb-mutex-private.hh for possible resolutions.")
-#else
 #error "Could not find any system to define mutex macros, library WILL NOT be thread-safe"
 #error "Check hb-mutex-private.hh for possible resolutions."
 #endif
-#endif

test/api/Makefile.am

@@ -34,7 +34,7 @@ test_unicode_CPPFLAGS += $(GLIB_CFLAGS)
 endif
 if HAVE_ICU
 test_unicode_CPPFLAGS += $(ICU_CFLAGS)
-test_unicode_LDADD += $(top_builddir)/src/libharfbuzz-icu.la
+test_unicode_LDADD += $(top_builddir)/src/libharfbuzz-icu.la $(ICU_LIBS)
 endif
 
 

test/shaping/Makefile.am

@@ -38,12 +38,15 @@ CLEANFILES += \
 TESTS = \
 	tests/arabic-fallback-shaping.tests \
 	tests/arabic-feature-order.tests \
+	tests/cluster.tests \
 	tests/context-matching.tests \
+	tests/default-ignorables.tests \
 	tests/hangul-jamo.tests \
 	tests/indic-joiner-candrabindu.tests \
 	tests/indic-old-spec.tests \
 	tests/indic-pref-blocking.tests \
 	tests/mongolian-variation-selector.tests \
+	tests/vertical.tests \
 	tests/zero-width-marks.tests \
 	$(NULL)
 

test/shaping/fonts/sha1sum/MANIFEST

@@ -1,9 +1,12 @@
+051d92f8bc6ff724511b296c27623f824de256e9.ttf
+191826b9643e3f124d865d617ae609db6a2ce203.ttf
 226bc2deab3846f1a682085f70c67d0421014144.ttf
 270b89df543a7e48e206a2d830c0e10e5265c630.ttf
 37033cc5cf37bb223d7355153016b6ccece93b28.ttf
 4cce528e99f600ed9c25a2b69e32eb94a03b4ae8.ttf
 5028afb650b1bb718ed2131e872fbcce57828fff.ttf
 57a9d9f83020155cbb1d2be1f43d82388cbecc88.ttf
+6466d38c62e73a39202435a4f73bf5d6acbb73c0.ttf
 757ebd573617a24aa9dfbf0b885c54875c6fe06b.ttf
 7e14e7883ed152baa158b80e207b66114c823a8b.ttf
 813c2f8e5512187fd982417a7fb4286728e6f4a8.ttf

test/shaping/record-test.sh

@@ -5,11 +5,42 @@ dir=`mktemp --directory`
 hb_shape=$1
 shift
 fontfile=$1
+if test "x${fontfile:0:1}" == 'x-'; then
+	echo "Specify font file before other options." >&2
+	exit 1
+fi
 shift
-hb_shape="$hb_shape $@"
+if ! echo "$hb_shape" | grep -q 'hb-shape'; then
-unicodes=`./hb-unicode-decode`
+	echo "Specify hb-shape (not hb-view, etc)." >&2
-text=`./hb-unicode-encode "$unicodes"`
+	exit 1
-glyphs=`echo "$text" | $hb_shape "$fontfile"`
+fi
+options=
+have_text=false
+for arg in "$@"; do
+	if test "x${arg:0:1}" == 'x-'; then
+		if echo "$arg" | grep -q ' '; then
+			echo "Space in argument is not supported: '$arg'." >&2
+			exit 1
+		fi
+		options="$options${options:+ }$arg"
+		continue
+	fi
+	if $have_text; then
+		echo "Too many arguments found...  Use '=' notation for options: '$arg'" >&2
+		exit 1;
+	fi
+	text="$arg"
+	have_text=true
+done
+if ! $have_text; then
+	text=`cat`
+fi
+unicodes=`./hb-unicode-decode "$text"`
+glyphs=`echo "$text" | $hb_shape $options "$fontfile"`
+if test $? != 0; then
+	echo "hb-shape failed." >&2
+	exit 2
+fi
 
 cp "$fontfile" "$dir/font.ttf"
 pyftsubset \
@@ -22,14 +53,14 @@ if ! test -s "$dir/font.ttf.subset"; then
 fi
 
 # Verify that subset font produces same glyphs!
-glyphs_subset=`echo "$text" | $hb_shape "$dir/font.ttf.subset"`
+glyphs_subset=`echo "$text" | $hb_shape $options "$dir/font.ttf.subset"`
 
 if ! test "x$glyphs" = "x$glyphs_subset"; then
 	echo "Subset font produced different glyphs!" >&2
 	echo "Perhaps font doesn't have glyph names; checking visually..." >&2
 	hb_view=${hb_shape/shape/view}
-	echo "$text" | $hb_view "$dir/font.ttf" --output-format=png --output-file="$dir/orig.png"
+	echo "$text" | $hb_view $options "$dir/font.ttf" --output-format=png --output-file="$dir/orig.png"
-	echo "$text" | $hb_view "$dir/font.ttf.subset" --output-format=png --output-file="$dir/subset.png"
+	echo "$text" | $hb_view $options "$dir/font.ttf.subset" --output-format=png --output-file="$dir/subset.png"
 	if ! cmp "$dir/orig.png" "$dir/subset.png"; then
 		echo "Images differ.  Please inspect $dir/*.png." >&2
 		echo "$glyphs"
@@ -46,7 +77,7 @@ sha1sum=`sha1sum "$dir/font.ttf.subset" | cut -d' ' -f1`
 subset="fonts/sha1sum/$sha1sum.ttf"
 mv "$dir/font.ttf.subset" "$subset"
 
-echo "$subset:$unicodes:$glyphs"
+echo "$subset:$options:$unicodes:$glyphs"
 
 rm -f "$dir/font.ttf"
 rmdir "$dir"

test/shaping/run-tests.sh

@@ -15,9 +15,14 @@ fi
 IFS=:
 for f in "$@"; do
 	echo "Running tests in $f"
-	while read fontfile unicodes glyphs_expected; do
+	while read fontfile options unicodes glyphs_expected; do
 		echo "Testing $fontfile:$unicodes"
-		glyphs=`$srcdir/hb-unicode-encode "$unicodes" | $hb_shape "$srcdir/$fontfile"`
+		glyphs=`$srcdir/hb-unicode-encode "$unicodes" | $hb_shape $options "$srcdir/$fontfile"`
+		if test $? != 0; then
+			echo "hb-shape failed." >&2
+			fails=$((fails+1))
+			continue
+		fi
 		if ! test "x$glyphs" = "x$glyphs_expected"; then
 			echo "Actual:   $glyphs" >&2
 			echo "Expected: $glyphs_expected" >&2

test/shaping/tests/MANIFEST

@@ -1,9 +1,12 @@
 arabic-fallback-shaping.tests
 arabic-feature-order.tests
+cluster.tests
 context-matching.tests
+default-ignorables.tests
 hangul-jamo.tests
 indic-joiner-candrabindu.tests
 indic-old-spec.tests
 indic-pref-blocking.tests
 mongolian-variation-selector.tests
+vertical.tests
 zero-width-marks.tests

test/shaping/tests/arabic-fallback-shaping.tests

@@ -1 +1 @@
-fonts/sha1sum/df768b9c257e0c9c35786c47cae15c46571d56be.ttf:U+0633,U+064F,U+0644,U+064E,U+0651,U+0627,U+0651,U+0650,U+0645,U+062A,U+06CC:[uni06CC.fina=10+1655|uni062A.medi=9+868|uni0645.init=8+1098|uni0650=2@221,0+0|uni0651=2@260,736+0|uni064E=2@935,1259+0|uni0651=2@974,736+0|uni06440627.fina=2+1470|uni064F=0@558,-10+0|uni0633.init=0+1585]
+fonts/sha1sum/df768b9c257e0c9c35786c47cae15c46571d56be.ttf::U+0633,U+064F,U+0644,U+064E,U+0651,U+0627,U+0651,U+0650,U+0645,U+062A,U+06CC:[uni06CC.fina=10+1655|uni062A.medi=9+868|uni0645.init=8+1098|uni0650=2@221,0+0|uni0651=2@260,736+0|uni064E=2@935,1259+0|uni0651=2@974,736+0|uni06440627.fina=2+1470|uni064F=0@558,-10+0|uni0633.init=0+1585]

test/shaping/tests/arabic-feature-order.tests

@@ -1,3 +1,3 @@
-fonts/sha1sum/813c2f8e5512187fd982417a7fb4286728e6f4a8.ttf:U+1820,U+180B:[uni2048.E81A=0+1550]
+fonts/sha1sum/813c2f8e5512187fd982417a7fb4286728e6f4a8.ttf::U+1820,U+180B:[uni2048.E81A=0+1550]
-fonts/sha1sum/8a9fea2a7384f2116e5b84a9b31f83be7850ce21.ttf:U+1820,U+180B:[uni2048.E81A=0+1550]
+fonts/sha1sum/8a9fea2a7384f2116e5b84a9b31f83be7850ce21.ttf::U+1820,U+180B:[uni2048.E81A=0+1550]
-fonts/sha1sum/a919b33197965846f21074b24e30250d67277bce.ttf:U+0644,U+0644,U+0647:[Lellah=0+1503]
+fonts/sha1sum/a919b33197965846f21074b24e30250d67277bce.ttf::U+0644,U+0644,U+0647:[Lellah=0+1503]

test/shaping/tests/cluster.tests

@@ -0,0 +1 @@
+fonts/sha1sum/6466d38c62e73a39202435a4f73bf5d6acbb73c0.ttf:--cluster-level=2:U+0078,U+030A,U+0058,U+030A:[gid2=0+1083|gid4=1@-555,-8+0|gid1=2+1200|gid4=3@-614,349+0]

test/shaping/tests/context-matching.tests

@@ -1,3 +1,3 @@
-fonts/sha1sum/4cce528e99f600ed9c25a2b69e32eb94a03b4ae8.ttf:U+1A48,U+1A58,U+1A25,U+1A48,U+1A58,U+1A25,U+1A6E,U+1A63:[uni1A48=0+1212|uni1A25=0+1912|uni1A58=0+0|uni1A48=3+1212|uni1A6E=3+1212|uni1A25=3+1912|uni1A58=3+0|uni1A63=3+1212]
+fonts/sha1sum/4cce528e99f600ed9c25a2b69e32eb94a03b4ae8.ttf::U+1A48,U+1A58,U+1A25,U+1A48,U+1A58,U+1A25,U+1A6E,U+1A63:[uni1A48=0+1212|uni1A25=0+1912|uni1A58=0+0|uni1A48=3+1212|uni1A6E=3+1212|uni1A25=3+1912|uni1A58=3+0|uni1A63=3+1212]
-fonts/sha1sum/d629e7fedc0b350222d7987345fe61613fa3929a.ttf:U+0915,U+093F,U+0915,U+093F:[ivowelsign03deva=0+530|kadeva=0+1561|ivowelsign03deva=2+530|kadeva=2+1561]
+fonts/sha1sum/d629e7fedc0b350222d7987345fe61613fa3929a.ttf::U+0915,U+093F,U+0915,U+093F:[ivowelsign03deva=0+530|kadeva=0+1561|ivowelsign03deva=2+530|kadeva=2+1561]
-fonts/sha1sum/f499fbc23865022234775c43503bba2e63978fe1.ttf:U+09B0,U+09CD,U+09A5,U+09CD,U+09AF,U+09C0:[gid1=0+1320|gid13=0+523|gid18=0+545]
+fonts/sha1sum/f499fbc23865022234775c43503bba2e63978fe1.ttf::U+09B0,U+09CD,U+09A5,U+09CD,U+09AF,U+09C0:[gid1=0+1320|gid13=0+523|gid18=0+545]

test/shaping/tests/default-ignorables.tests

@@ -0,0 +1 @@
+fonts/sha1sum/051d92f8bc6ff724511b296c27623f824de256e9.ttf::U+0075,U+0361,U+034F,U+0301,U+0069:[gid2=0+1266|gid7=0@-617,442+0|gid5=0@-7,0+0|gid1=4+528]

test/shaping/tests/hangul-jamo.tests

@@ -1,2 +1,2 @@
-fonts/sha1sum/757ebd573617a24aa9dfbf0b885c54875c6fe06b.ttf:U+115F,U+11A2:[gid3=0+920|gid4=0+0]
+fonts/sha1sum/757ebd573617a24aa9dfbf0b885c54875c6fe06b.ttf::U+115F,U+11A2:[gid3=0+920|gid4=0+0]
-fonts/sha1sum/7e14e7883ed152baa158b80e207b66114c823a8b.ttf:U+11A2:[gid1=0+920]
+fonts/sha1sum/7e14e7883ed152baa158b80e207b66114c823a8b.ttf::U+11A2:[gid1=0+920]

test/shaping/tests/indic-joiner-candrabindu.tests

@@ -1,2 +1,2 @@
-fonts/sha1sum/5028afb650b1bb718ed2131e872fbcce57828fff.ttf:U+0B13,U+200D,U+0B01:[omorya=0+1450]
+fonts/sha1sum/5028afb650b1bb718ed2131e872fbcce57828fff.ttf::U+0B13,U+200D,U+0B01:[omorya=0+1450]
-fonts/sha1sum/5028afb650b1bb718ed2131e872fbcce57828fff.ttf:U+0B13,U+200C,U+0B01:[oorya=0+1309|space=1+0|candrabinduorya=1+0]
+fonts/sha1sum/5028afb650b1bb718ed2131e872fbcce57828fff.ttf::U+0B13,U+200C,U+0B01:[oorya=0+1309|space=1+0|candrabinduorya=1+0]

test/shaping/tests/indic-old-spec.tests

@@ -1,2 +1,2 @@
-fonts/sha1sum/57a9d9f83020155cbb1d2be1f43d82388cbecc88.ttf:U+0C9A,U+0CCD,U+0C9A,U+0CCD:[U0C9A_U0CCD.haln=0+1066|U0C9A_0CCD.blwf=0+0]
+fonts/sha1sum/57a9d9f83020155cbb1d2be1f43d82388cbecc88.ttf::U+0C9A,U+0CCD,U+0C9A,U+0CCD:[U0C9A_U0CCD.haln=0+1066|U0C9A_0CCD.blwf=0+0]
-fonts/sha1sum/270b89df543a7e48e206a2d830c0e10e5265c630.ttf:U+0D38,U+0D4D,U+0D31,U+0D4D,U+0D31,U+0D4D:[glyph201=0+1183|U0D4D=0+0]
+fonts/sha1sum/270b89df543a7e48e206a2d830c0e10e5265c630.ttf::U+0D38,U+0D4D,U+0D31,U+0D4D,U+0D31,U+0D4D:[glyph201=0+1183|U0D4D=0+0]

test/shaping/tests/indic-pref-blocking.tests

@@ -1,2 +1,2 @@
-fonts/sha1sum/226bc2deab3846f1a682085f70c67d0421014144.ttf:U+0D2F,U+0D4D,U+0D30,U+0D46:[evowelsignmlym=0+1465|rapostmlym=0+499|yamlym=0+2120]
+fonts/sha1sum/226bc2deab3846f1a682085f70c67d0421014144.ttf::U+0D2F,U+0D4D,U+0D30,U+0D46:[evowelsignmlym=0+1465|rapostmlym=0+499|yamlym=0+2120]
-fonts/sha1sum/e207635780b42f898d58654b65098763e340f5c7.ttf:U+0D2F,U+0D4D,U+0D30,U+0D46:[yamlym=0+2120|viramamlym=0+0|evowelsignmlym=0+1465|ramlym=0+1507]
+fonts/sha1sum/e207635780b42f898d58654b65098763e340f5c7.ttf::U+0D2F,U+0D4D,U+0D30,U+0D46:[yamlym=0+2120|viramamlym=0+0|evowelsignmlym=0+1465|ramlym=0+1507]

test/shaping/tests/mongolian-variation-selector.tests

@@ -1,3 +1,3 @@
-fonts/sha1sum/37033cc5cf37bb223d7355153016b6ccece93b28.ttf:U+1826,U+180B,U+1826:[uni1826.E85E_ue.init1=0+599|uni1826.E856_ue.fina=2+750]
+fonts/sha1sum/37033cc5cf37bb223d7355153016b6ccece93b28.ttf::U+1826,U+180B,U+1826:[uni1826.E85E_ue.init1=0+599|uni1826.E856_ue.fina=2+750]
-fonts/sha1sum/ef86fe710cfea877bbe0dbb6946a1f88d0661031.ttf:U+1820,U+180B:[uni1820.E821_a.isol1=0+1199]
+fonts/sha1sum/ef86fe710cfea877bbe0dbb6946a1f88d0661031.ttf::U+1820,U+180B:[uni1820.E821_a.isol1=0+1199]
-fonts/sha1sum/bb29ce50df2bdba2d10726427c6b7609bf460e04.ttf:U+183A,U+1823,U+182E,U+182B,U+1822,U+1826,U+180B,U+1832,U+180B,U+1827,U+1837,U+0020,U+182D,U+182D,U+180B,U+0020,U+182D,U+180C,U+0020,U+182D,U+180D,U+200D,U+0020,U+182D,U+200D,U+182D,U+180B,U+200D,U+0020,U+182D,U+180C,U+200D,U+0020,U+182D,U+180D,U+200D,U+0020,U+200D,U+182D,U+200D,U+200D,U+182D,U+180B,U+200D,U+0020,U+200D,U+182D,U+180C,U+200D,U+0020,U+200D,U+182D,U+180D,U+200D,U+0020,U+200D,U+182D,U+200D,U+182D,U+180B,U+0020,U+200D,U+182D,U+180C,U+0020,U+1820,U+200C,U+182D,U+1820,U+1837,U+0020,U+1830,U+1824,U+1837,U+200D,U+200D,U+182D,U+1820,U+200D,U+0020,U+200D,U+182D,U+1824,U+182F,U+1822,U+0020,U+182A,U+1820,U+1822,U+182D,U+180E,U+1820,U+202F,U+1836,U+1822,U+1828:[uni183A1823.E971_ko.init=0+950|uni182E.E904_m.medi=2+400|uni182B1822.E8A6_pi.medi=3+1150|uni1826.E854_ue.medi1=5+1100|uni1832.E916_t.medi1=7+1000|uni1827.E85C_ee.medi=9+750|uni1837.E931_r.fina=10+750|space=11+500|uni182D.E8E2_g.init=12+1000|uni182D.E8E8_g.fina1=13+1250|space=15+500|uni182D.EA1B_g.isol2=16+1000|space=18+500|uni182D.EA1E_g.init3=19+650|space=21+0|space=22+500|uni182D.E8E2_g.init=23+1000|space=24+0|uni182D.E8E5_g.medi1=25+800|space=27+0|space=28+500|uni182D.EA1D_g.init2=29+950|space=31+0|space=32+500|uni182D.EA1E_g.init3=33+650|space=35+0|space=36+500|space=37+0|uni182D.E8E4_g.medi=38+800|space=39+0|space=40+0|uni182D.E8E5_g.medi1=41+800|space=43+0|space=44+500|space=45+0|uni182D.E8E6_g.medi2=46+650|space=48+0|space=49+500|space=50+0|uni182D.E8E6_g.medi2=51+650|space=53+0|space=54+500|space=55+0|uni182D.E8E4_g.medi=56+800|space=57+0|uni182D.E8E8_g.fina1=58+1250|space=60+500|space=61+0|uni182D.E8E9_g.fina2=62+1050|space=64+500|uni1820.E820_a.isol=65+1550|space=66+0|uni182D.E8E2_g.init=67+1000|uni1820.E823_a.medi=68+400|uni1837.E931_r.fina=69+750|space=70+500|uni1830.E90B_s.init=71+850|uni1824.E844_u.medi=72+600|uni1837.E930_r.medi=73+600|space=74+0|space=75+0|uni182D.E8E5_g.medi1=76+800|uni1820.E823_a.medi=77+400|space=78+0|space=79+500|space=80+0|uni182D.E8E5_g.medi1=81+800|uni1824.E844_u.medi=82+600|uni182F.E908_l.medi=83+400|uni1822.E837_i.fina=84+600|space=85+500|uni182A1820.E875_ba.init=86+1000|uni1822.E836_i.medi2=88+1000|uni182D.E8E8_g.fina1=89+1250|space=90+0|uni1820.E827_a.fina2=91+600|uni202F.nobreak=92+500|uni1836.E92B_y.init1=93+500|uni1822.E834_i.medi=94+500|uni1828.E866_n.fina=95+850]
+fonts/sha1sum/bb29ce50df2bdba2d10726427c6b7609bf460e04.ttf::U+183A,U+1823,U+182E,U+182B,U+1822,U+1826,U+180B,U+1832,U+180B,U+1827,U+1837,U+0020,U+182D,U+182D,U+180B,U+0020,U+182D,U+180C,U+0020,U+182D,U+180D,U+200D,U+0020,U+182D,U+200D,U+182D,U+180B,U+200D,U+0020,U+182D,U+180C,U+200D,U+0020,U+182D,U+180D,U+200D,U+0020,U+200D,U+182D,U+200D,U+200D,U+182D,U+180B,U+200D,U+0020,U+200D,U+182D,U+180C,U+200D,U+0020,U+200D,U+182D,U+180D,U+200D,U+0020,U+200D,U+182D,U+200D,U+182D,U+180B,U+0020,U+200D,U+182D,U+180C,U+0020,U+1820,U+200C,U+182D,U+1820,U+1837,U+0020,U+1830,U+1824,U+1837,U+200D,U+200D,U+182D,U+1820,U+200D,U+0020,U+200D,U+182D,U+1824,U+182F,U+1822,U+0020,U+182A,U+1820,U+1822,U+182D,U+180E,U+1820,U+202F,U+1836,U+1822,U+1828:[uni183A1823.E971_ko.init=0+950|uni182E.E904_m.medi=2+400|uni182B1822.E8A6_pi.medi=3+1150|uni1826.E854_ue.medi1=5+1100|uni1832.E916_t.medi1=7+1000|uni1827.E85C_ee.medi=9+750|uni1837.E931_r.fina=10+750|space=11+500|uni182D.E8E2_g.init=12+1000|uni182D.E8E8_g.fina1=13+1250|space=15+500|uni182D.EA1B_g.isol2=16+1000|space=18+500|uni182D.EA1E_g.init3=19+650|space=21+0|space=22+500|uni182D.E8E2_g.init=23+1000|space=24+0|uni182D.E8E5_g.medi1=25+800|space=27+0|space=28+500|uni182D.EA1D_g.init2=29+950|space=31+0|space=32+500|uni182D.EA1E_g.init3=33+650|space=35+0|space=36+500|space=37+0|uni182D.E8E4_g.medi=38+800|space=39+0|space=40+0|uni182D.E8E5_g.medi1=41+800|space=43+0|space=44+500|space=45+0|uni182D.E8E6_g.medi2=46+650|space=48+0|space=49+500|space=50+0|uni182D.E8E6_g.medi2=51+650|space=53+0|space=54+500|space=55+0|uni182D.E8E4_g.medi=56+800|space=57+0|uni182D.E8E8_g.fina1=58+1250|space=60+500|space=61+0|uni182D.E8E9_g.fina2=62+1050|space=64+500|uni1820.E820_a.isol=65+1550|space=66+0|uni182D.E8E2_g.init=67+1000|uni1820.E823_a.medi=68+400|uni1837.E931_r.fina=69+750|space=70+500|uni1830.E90B_s.init=71+850|uni1824.E844_u.medi=72+600|uni1837.E930_r.medi=73+600|space=74+0|space=75+0|uni182D.E8E5_g.medi1=76+800|uni1820.E823_a.medi=77+400|space=78+0|space=79+500|space=80+0|uni182D.E8E5_g.medi1=81+800|uni1824.E844_u.medi=82+600|uni182F.E908_l.medi=83+400|uni1822.E837_i.fina=84+600|space=85+500|uni182A1820.E875_ba.init=86+1000|uni1822.E836_i.medi2=88+1000|uni182D.E8E8_g.fina1=89+1250|space=90+0|uni1820.E827_a.fina2=91+600|uni202F.nobreak=92+500|uni1836.E92B_y.init1=93+500|uni1822.E834_i.medi=94+500|uni1828.E866_n.fina=95+850]

test/shaping/tests/vertical.tests

@@ -0,0 +1 @@
+fonts/sha1sum/191826b9643e3f124d865d617ae609db6a2ce203.ttf:--direction=t:U+300C:[uni300C.vert=0@-512,-578+0,-1024]

test/shaping/tests/zero-width-marks.tests

@@ -1,2 +1,2 @@
-fonts/sha1sum/bb9473d2403488714043bcfb946c9f78b86ad627.ttf:U+1030:[circledash=0+636|u1030.med=0@-162,0+0]
+fonts/sha1sum/bb9473d2403488714043bcfb946c9f78b86ad627.ttf::U+1030:[circledash=0+636|u1030.med=0@-162,0+0]
-fonts/sha1sum/8454d22037f892e76614e1645d066689a0200e61.ttf:U+05E0,U+05B8,U+0591,U+05DA,U+05B0:[uni05DA05B0=3+991|uni2009=0+200|uni0591=0@75,0+0|uni05B8=0@495,0+0|uni05E0=0+683]
+fonts/sha1sum/8454d22037f892e76614e1645d066689a0200e61.ttf::U+05E0,U+05B8,U+0591,U+05DA,U+05B0:[uni05DA05B0=3+991|uni2009=0+200|uni0591=0@75,0+0|uni05B8=0@495,0+0|uni05E0=0+683]

test/shaping/texts/in-tree/MANIFEST

@@ -4,6 +4,6 @@ shaper-hangul
 shaper-hebrew
 shaper-indic
 shaper-myanmar
-shaper-sea
 shaper-thai
 shaper-tibetan
+shaper-use

test/shaping/texts/in-tree/shaper-sea/MANIFEST

@@ -1,3 +0,0 @@
-script-cham
-script-new-tai-lue
-script-tai-tham

test/shaping/texts/in-tree/shaper-sea/script-cham/MANIFEST

@@ -1 +0,0 @@
-misc

test/shaping/texts/in-tree/shaper-sea/script-new-tai-lue/MANIFEST

@@ -1 +0,0 @@
-misc

test/shaping/texts/in-tree/shaper-sea/script-new-tai-lue/misc/misc.txt

@@ -1 +0,0 @@
-ᦀᦷᧃᧈ

test/shaping/texts/in-tree/shaper-sea/script-tai-tham/MANIFEST

@@ -1 +0,0 @@
-misc

test/shaping/texts/in-tree/shaper-use/MANIFEST

@@ -0,0 +1,5 @@
+script-batak
+script-buginese
+script-cham
+script-kharoshti
+script-tai-tham

test/shaping/texts/in-tree/shaper-use/script-batak/misc.txt

@@ -0,0 +1,9 @@
+ᯂᯩ
+ᯄ᯦ᯩ
+ᯇᯪᯰ
+ᯓᯩᯰ
+ᯄᯮ
+ᯃᯮ
+ᯎᯮ
+ᯞᯮ
+ᯖᯪᯇ᯲

test/shaping/texts/in-tree/shaper-use/script-buginese/misc.txt

@@ -0,0 +1,70 @@
+ᨒᨚᨈᨑ
+ᨔᨑ
+ᨅᨔ ᨈᨚ ᨅᨙᨀ
+ᨕᨒᨚ ᨆᨒᨗᨕᨘ ᨅᨛᨈᨘᨕᨊ
+ᨕᨗᨉᨚ ᨔᨘᨑᨛ
+ᨕᨗᨊ ᨔᨘᨑᨛ
+ᨕᨊ ᨔᨘᨑᨛ
+
+ᨊᨀᨚ	ᨕᨛᨃ	ᨈᨕᨘᨄᨔᨒ᨞	ᨕᨍ	ᨆᨘᨄᨈᨒᨒᨚᨓᨗ	ᨄᨌᨒᨆᨘ	ᨑᨗᨈᨚᨄᨔᨒᨕᨙ᨞
+ᨄᨔᨗᨈᨘᨍᨘᨓᨗᨆᨘᨈᨚᨓᨗᨔ	ᨕᨔᨒᨊ	ᨄᨌᨒᨆᨘ᨞	ᨕᨄ	ᨕᨗᨀᨚᨊᨈᨘ	ᨊᨁᨗᨒᨗ	ᨉᨙᨓᨈᨕᨙ᨞
+ᨊᨀᨚ	ᨅᨕᨗᨌᨘᨆᨘᨄᨗ	ᨕᨔᨒᨊ	ᨈᨕᨘᨓᨙ᨞	ᨆᨘᨄᨙᨑᨍᨕᨗᨔ	ᨄᨉᨈᨚᨓᨗ᨞
+ᨊᨀᨚ	ᨄᨔᨒᨕᨗ	ᨈᨕᨘᨓᨙ᨞	ᨕᨍ	ᨈᨗᨆᨘᨌᨒᨕᨗ	ᨑᨗᨔᨗᨈᨗᨊᨍᨊᨕᨙᨈᨚᨔ	ᨕᨔᨒᨊ᨞
+
+ᨕᨛᨛᨃ	ᨕᨛᨃ	ᨄ ᨙᨑ᨞	ᨕᨛᨃ	 ᨙᨔᨕᨘᨓ	ᨓᨛᨈᨘ᨞
+ᨕᨛᨃ	 ᨙᨔᨕᨘᨓ	ᨕᨑᨘ	ᨆᨀᨘᨋᨕᨗ	ᨑᨗ	ᨒᨘᨓᨘ᨞	ᨆᨔᨒ	ᨕᨘᨒᨗ᨞
+
+ᨄᨘᨑᨊᨗᨀᨚ	ᨆᨙᨋ?
+ᨉᨙᨄ
+
+ᨆᨙᨒᨚ ᨀ ᨌᨛᨙᨆ
+ᨔᨙᨉᨗ	
+ᨉᨘᨓ	
+ᨈᨛᨒᨘ	
+ᨕᨛᨄ	
+ᨒᨗᨆ	
+ᨕᨛᨊᨛ	
+ᨄᨗᨈᨘ	
+ᨕᨑᨘᨓ	
+ᨕᨙᨔᨑ	
+ᨔᨄᨘᨒᨚ	
+ᨉᨘᨓᨄᨘᨒᨚ	
+ᨈᨛᨒᨘᨄᨘᨒᨚ	
+ᨄᨈᨄᨘᨒᨚ	
+ᨒᨗᨆᨄᨘᨒᨚ	
+ᨕᨛᨊᨛᨄᨘᨒᨚᨊ	
+ᨄᨗᨈᨘᨄᨘᨒᨚ	
+ᨕᨑᨘᨓᨄᨘᨒᨚᨊ	
+ᨕᨙᨔᨑᨄᨘᨒᨚᨊ	
+ᨔᨗᨑᨈᨘ	
+ᨔᨗᨔᨛᨅᨘ	
+ᨔᨗᨒᨔ	
+ᨔᨗᨀᨚᨈᨗ	
+
+ᨅᨔ ᨕᨘᨁᨗ
+
+ᨅᨔ ᨆᨀᨔᨑ
+ᨅᨒ	
+ᨅᨚᨒᨚ	
+ᨅᨅ	
+ᨌᨗᨄᨘᨑᨘ	
+ᨉᨚᨕᨙ	
+ᨕᨗᨐᨚ	
+ᨒᨚᨄᨚ	
+ᨔᨒᨚ	
+ᨈ ᨅᨙᨙ	
+ᨈᨙᨊ	
+ᨀᨑᨕᨙ	
+ᨕᨄ ᨀᨑᨙᨅ?	
+ᨒᨀᨙᨀᨚ ᨆᨕᨙ?	
+ᨅᨒ	
+ᨅᨚᨈᨚ	
+ᨑᨈᨔ	
+ᨅᨈᨒ	
+ᨅᨗᨒ	
+ᨁᨙᨒᨙ ᨁᨙᨒᨙ	
+ᨀᨚᨀᨚ	
+ᨍᨑ	
+ᨅᨙᨅᨙ	
+ᨆᨚᨈᨙᨑᨙ	
+ᨂᨑᨙ	

test/shaping/texts/in-tree/shaper-use/script-cham/MANIFEST

@@ -0,0 +1 @@
+misc.txt

test/shaping/texts/in-tree/shaper-use/script-kharoshti/MANIFEST

@@ -0,0 +1 @@
+misc.txt

test/shaping/texts/in-tree/shaper-use/script-kharoshti/misc.txt

@@ -0,0 +1,36 @@
+𐨤𐨪𐨌𐨪𐨿𐨗𐨸𐨅𐨌𐨏
+𐨀𐨁
+𐨐𐨁
+𐨠𐨁
+𐨀𐨂
+𐨱𐨂
+𐨨𐨂
+𐨀𐨃
+𐨨𐨃
+𐨀𐨅
+𐨐𐨅
+𐨠𐨅
+𐨡𐨅
+𐨀𐨆
+𐨤𐨆
+𐨨𐨌
+𐨯𐨍
+𐨀𐨎
+𐨐𐨏
+𐨗𐨸
+𐨒𐨹
+𐨨𐨺
+𐨢𐨁𐨐𐨿
+𐨐𐨿𐨮
+𐨨𐨿𐨪
+𐨬𐨿𐨱
+𐨯𐨿𐨟
+𐨯𐨿𐨩
+𐨪𐨿𐨟
+𐨟𐨿𐨪
+𐨫𐨿𐨤
+𐨤𐨿𐨫
+𐨐𐨿𐨫
+𐨟𐨿𐨬
+𐨐𐨿𐨟
+𐨑𐨿𐨐𐨿𐨮

util/options.cc

@@ -291,6 +291,7 @@ shape_options_t::add_options (option_parser_t *parser)
     {"eot",		0, 0, G_OPTION_ARG_NONE,	&this->eot,			"Treat text as end-of-paragraph",	NULL},
     {"preserve-default-ignorables",0, 0, G_OPTION_ARG_NONE,	&this->preserve_default_ignorables,	"Preserve Default-Ignorable characters",	NULL},
     {"utf8-clusters",	0, 0, G_OPTION_ARG_NONE,	&this->utf8_clusters,		"Use UTF8 byte indices, not char indices",	NULL},
+    {"cluster-level",	0, 0, G_OPTION_ARG_INT,		&this->cluster_level,		"Cluster merging level (default: 0)",	"0/1/2"},
     {"normalize-glyphs",0, 0, G_OPTION_ARG_NONE,	&this->normalize_glyphs,	"Rearrange glyph clusters in nominal order",	NULL},
     {"num-iterations",	0, 0, G_OPTION_ARG_INT,		&this->num_iterations,		"Run shaper N times (default: 1)",	"N"},
     {NULL}

util/options.hh

@@ -180,6 +180,7 @@ struct shape_options_t : option_group_t
     num_features = 0;
     shapers = NULL;
     utf8_clusters = false;
+    cluster_level = HB_BUFFER_CLUSTER_LEVEL_DEFAULT;
     normalize_glyphs = false;
     num_iterations = 1;
 
@@ -202,6 +203,7 @@ struct shape_options_t : option_group_t
 			 (bot ? HB_BUFFER_FLAG_BOT : 0) |
 			 (eot ? HB_BUFFER_FLAG_EOT : 0) |
 			 (preserve_default_ignorables ? HB_BUFFER_FLAG_PRESERVE_DEFAULT_IGNORABLES : 0)));
+    hb_buffer_set_cluster_level (buffer, cluster_level);
     hb_buffer_guess_segment_properties (buffer);
   }
 
@@ -265,6 +267,7 @@ struct shape_options_t : option_group_t
   unsigned int num_features;
   char **shapers;
   hb_bool_t utf8_clusters;
+  hb_buffer_cluster_level_t cluster_level;
   hb_bool_t normalize_glyphs;
   unsigned int num_iterations;
 };