Merge branch 'master' into cff-subset

.circleci/config.yml

@@ -89,11 +89,11 @@ jobs:
       - run: echo "deb http://apt.llvm.org/cosmic/ llvm-toolchain-cosmic main" > /etc/apt/sources.list.d/llvmdev.list
       - run: echo "deb-src http://apt.llvm.org/cosmic/ llvm-toolchain-cosmic main" > /etc/apt/sources.list.d/llvmdevsrc.list
       - run: apt update || true
-      - run: apt install -y clang lld binutils libtool autoconf automake make pkg-config gtk-doc-tools ragel libfreetype6-dev libglib2.0-dev libcairo2-dev libicu-dev libgraphite2-dev python python-pip
+      - run: apt install -y clang lld binutils libtool autoconf automake make pkg-config gtk-doc-tools ragel libfreetype6-dev libfontconfig1-dev libglib2.0-dev libcairo2-dev libicu-dev libgraphite2-dev python python-pip
       - run: pip install fonttools
-      - run: CFLAGS="-Weverything -Werror -fPIC -Wno-unused-parameter -Wno-missing-variable-declarations -Wno-padded -Wno-cast-qual -Wno-sign-conversion -Wno-conversion -Wno-documentation -Wno-documentation-unknown-command -Wno-reserved-id-macro -Wno-shadow -Wno-reserved-id-macro -Wno-disabled-macro-expansion -Wno-missing-variable-declarations -Wno-unused-macros -Wno-unreachable-code-return" CXXFLAGS="-Weverything -Werror -fPIC -Wno-undef -Wno-deprecated-declarations -Wno-weak-vtables -Wno-old-style-cast -Wno-documentation -Wno-documentation-unknown-command -Wno-conversion -Wno-sign-conversion -Wno-c++98-compat -Wno-extra-semi -Wno-c++98-compat-pedantic -Wno-padded -Wno-shift-sign-overflow -Wno-missing-field-initializers -Wno-double-promotion -Wno-reserved-id-macro -Wno-cast-qual -Wno-unused-parameter -Wno-comma -Wno-shadow -Wno-used-but-marked-unused -Wno-format-pedantic -Wno-zero-as-null-pointer-constant -Wno-disabled-macro-expansion -Wno-covered-switch-default -Wno-conditional-uninitialized -Wno-unreachable-code -Wno-unused-macros -Wno-float-equal -Wno-missing-prototypes" CC=clang CXX=clang++ ./autogen.sh --with-freetype --with-glib --with-cairo --with-icu --with-graphite2
+      - run: CFLAGS="-Weverything -Wno-reserved-id-macro -Wno-conversion -Wno-padded -Wno-sign-conversion -Wno-cast-qual -Wno-documentation -Wno-documentation-unknown-command" CXXFLAGS="-Weverything -Wno-old-style-cast -Wno-documentation -Wno-documentation-unknown-command -Wno-c++98-compat -Wno-cast-qual -Wno-c++98-compat-pedantic -Wno-sign-conversion -Wno-padded -Wno-shorten-64-to-32 -Wno-extra-semi -Wno-reserved-id-macro -Wno-float-conversion -Wno-format-pedantic -Wno-shadow -Wno-conversion -Wno-zero-as-null-pointer-constant -Wno-missing-field-initializers -Wno-double-promotion -Wno-used-but-marked-unused -Wno-unused-macros -Wno-comma -Wno-float-equal -Wno-disabled-macro-expansion -Wno-weak-vtables -Wno-unused-parameter -Wno-covered-switch-default -Wno-unreachable-code -Wno-deprecated-declarations" CC=clang CXX=clang++ ./autogen.sh --with-freetype --with-glib --with-cairo --with-icu --with-graphite2 --with-fontconfig
-      - run: make -j32
+      - run: make -j32 CPPFLAGS="-Werror"
-      - run: make check || .ci/fail.sh
+      - run: make check CPPFLAGS="-Werror" || .ci/fail.sh
 
   clang-asan:
     docker:
@@ -121,14 +121,13 @@ jobs:
       - run: echo "deb http://apt.llvm.org/cosmic/ llvm-toolchain-cosmic main" > /etc/apt/sources.list.d/llvmdev.list
       - run: echo "deb-src http://apt.llvm.org/cosmic/ llvm-toolchain-cosmic main" > /etc/apt/sources.list.d/llvmdevsrc.list
       - run: apt update || true
-      - run: apt install -y clang lld binutils libtool autoconf automake automake1.11 gtk-doc-tools gettext make pkg-config ragel libfreetype6-dev libglib2.0-dev libcairo2-dev libicu-dev libgraphite2-dev python python-pip
+      - run: apt install -y clang lld binutils libtool autoconf automake gtk-doc-tools gettext make pkg-config ragel libcairo2-dev libicu-dev libmount-dev libgraphite2-dev python python-pip
       - run: pip install fonttools
       - run: update-alternatives --install "/usr/bin/ld" "ld" "/usr/bin/ld.lld" 10
-      - run: wget https://ftp.gnome.org/pub/gnome/sources/glib/2.28/glib-2.28.0.tar.bz2 && tar xf glib-2.28.0.tar.bz2 && cd glib-2.28.0 && ./autogen.sh || true && ./configure CPPFLAGS="-fsanitize=memory" LDFLAGS="-fsanitize=memory" CFLAGS="-fsanitize=memory" CXXFLAGS="-fsanitize=memory" LD=ld.lld CC=clang CXX=clang++ && make -j32 && make install && cd ..
+      - run: wget https://ftp.gnome.org/pub/gnome/sources/glib/2.58/glib-2.58.1.tar.xz && tar xf glib-2.58.1.tar.xz && cd glib-2.58.1 && ./autogen.sh --with-pcre CPPFLAGS="-fsanitize=memory" LDFLAGS="-fsanitize=memory" CFLAGS="-fsanitize=memory" CXXFLAGS="-fsanitize=memory" LD=ld.lld CC=clang CXX=clang++ && make -j32 && make install && cd ..
       - run: wget http://download.savannah.gnu.org/releases/freetype/freetype-2.9.tar.bz2 && tar xf freetype-2.9.tar.bz2 && cd freetype-2.9 && ./autogen.sh && ./configure CPPFLAGS="-fsanitize=memory" LDFLAGS="-fsanitize=memory -O1 -g -fno-omit-frame-pointer" CFLAGS="-fsanitize=memory -O1 -g -fno-omit-frame-pointer" CXXFLAGS="-fsanitize=memory -O1 -g -fno-omit-frame-pointer" LD=ld.lld CC=clang CXX=clang++ && make -j32 && make install && cd ..
       - run: CPPFLAGS="-fsanitize=memory" LDFLAGS="-fsanitize=memory -O1 -g -fno-omit-frame-pointer" CFLAGS="-fsanitize=memory -O1 -g -fno-omit-frame-pointer" CXXFLAGS="-fsanitize=memory -O1 -g -fno-omit-frame-pointer" LD=ld.lld CC=clang CXX=clang++ ./autogen.sh --with-freetype --with-glib --without-icu
-      - run: make -j32
+      - run: make -j32 && MSAN_OPTIONS=exitcode=42 make check || .ci/fail.sh | asan_symbolize | c++filt
-      - run: MSAN_OPTIONS=exitcode=42 SKIPCHECKSYMBOLS=1 SKIPFUZZERTESTS=1 make check || .ci/fail.sh | asan_symbolize | c++filt
 
   clang-tsan:
     docker:

build/test/api/test-collect-unicodes

@@ -0,0 +1,210 @@
+#! /bin/sh
+
+# test-collect-unicodes - temporary wrapper script for .libs/test-collect-unicodes
+# Generated by libtool (GNU libtool) 2.4.6
+#
+# The test-collect-unicodes program cannot be directly executed until all the libtool
+# libraries that it depends on are installed.
+#
+# This wrapper script should never be moved out of the build directory.
+# If it is, it will not operate correctly.
+
+# Sed substitution that helps us do robust quoting.  It backslashifies
+# metacharacters that are still active within double-quoted strings.
+sed_quote_subst='s|\([`"$\\]\)|\\\1|g'
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+  setopt NO_GLOB_SUBST
+else
+  case `(set -o) 2>/dev/null` in *posix*) set -o posix;; esac
+fi
+BIN_SH=xpg4; export BIN_SH # for Tru64
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# The HP-UX ksh and POSIX shell print the target directory to stdout
+# if CDPATH is set.
+(unset CDPATH) >/dev/null 2>&1 && unset CDPATH
+
+relink_command=""
+
+# This environment variable determines our operation mode.
+if test "$libtool_install_magic" = "%%%MAGIC variable%%%"; then
+  # install mode needs the following variables:
+  generated_by_libtool_version='2.4.6'
+  notinst_deplibs=' ../../src/libharfbuzz.la'
+else
+  # When we are sourced in execute mode, $file and $ECHO are already set.
+  if test "$libtool_execute_magic" != "%%%MAGIC variable%%%"; then
+    file="$0"
+
+# A function that is used when there is no print builtin or printf.
+func_fallback_echo ()
+{
+  eval 'cat <<_LTECHO_EOF
+$1
+_LTECHO_EOF'
+}
+    ECHO="printf %s\\n"
+  fi
+
+# Very basic option parsing. These options are (a) specific to
+# the libtool wrapper, (b) are identical between the wrapper
+# /script/ and the wrapper /executable/ that is used only on
+# windows platforms, and (c) all begin with the string --lt-
+# (application programs are unlikely to have options that match
+# this pattern).
+#
+# There are only two supported options: --lt-debug and
+# --lt-dump-script. There is, deliberately, no --lt-help.
+#
+# The first argument to this parsing function should be the
+# script's ../../libtool value, followed by no.
+lt_option_debug=
+func_parse_lt_options ()
+{
+  lt_script_arg0=$0
+  shift
+  for lt_opt
+  do
+    case "$lt_opt" in
+    --lt-debug) lt_option_debug=1 ;;
+    --lt-dump-script)
+        lt_dump_D=`$ECHO "X$lt_script_arg0" | /usr/bin/sed -e 's/^X//' -e 's%/[^/]*$%%'`
+        test "X$lt_dump_D" = "X$lt_script_arg0" && lt_dump_D=.
+        lt_dump_F=`$ECHO "X$lt_script_arg0" | /usr/bin/sed -e 's/^X//' -e 's%^.*/%%'`
+        cat "$lt_dump_D/$lt_dump_F"
+        exit 0
+      ;;
+    --lt-*)
+        $ECHO "Unrecognized --lt- option: '$lt_opt'" 1>&2
+        exit 1
+      ;;
+    esac
+  done
+
+  # Print the debug banner immediately:
+  if test -n "$lt_option_debug"; then
+    echo "test-collect-unicodes:test-collect-unicodes:$LINENO: libtool wrapper (GNU libtool) 2.4.6" 1>&2
+  fi
+}
+
+# Used when --lt-debug. Prints its arguments to stdout
+# (redirection is the responsibility of the caller)
+func_lt_dump_args ()
+{
+  lt_dump_args_N=1;
+  for lt_arg
+  do
+    $ECHO "test-collect-unicodes:test-collect-unicodes:$LINENO: newargv[$lt_dump_args_N]: $lt_arg"
+    lt_dump_args_N=`expr $lt_dump_args_N + 1`
+  done
+}
+
+# Core function for launching the target application
+func_exec_program_core ()
+{
+
+      if test -n "$lt_option_debug"; then
+        $ECHO "test-collect-unicodes:test-collect-unicodes:$LINENO: newargv[0]: $progdir/$program" 1>&2
+        func_lt_dump_args ${1+"$@"} 1>&2
+      fi
+      exec "$progdir/$program" ${1+"$@"}
+
+      $ECHO "$0: cannot exec $program $*" 1>&2
+      exit 1
+}
+
+# A function to encapsulate launching the target application
+# Strips options in the --lt-* namespace from $@ and
+# launches target application with the remaining arguments.
+func_exec_program ()
+{
+  case " $* " in
+  *\ --lt-*)
+    for lt_wr_arg
+    do
+      case $lt_wr_arg in
+      --lt-*) ;;
+      *) set x "$@" "$lt_wr_arg"; shift;;
+      esac
+      shift
+    done ;;
+  esac
+  func_exec_program_core ${1+"$@"}
+}
+
+  # Parse options
+  func_parse_lt_options "$0" ${1+"$@"}
+
+  # Find the directory that this script lives in.
+  thisdir=`$ECHO "$file" | /usr/bin/sed 's%/[^/]*$%%'`
+  test "x$thisdir" = "x$file" && thisdir=.
+
+  # Follow symbolic links until we get to the real thisdir.
+  file=`ls -ld "$file" | /usr/bin/sed -n 's/.*-> //p'`
+  while test -n "$file"; do
+    destdir=`$ECHO "$file" | /usr/bin/sed 's%/[^/]*$%%'`
+
+    # If there was a directory component, then change thisdir.
+    if test "x$destdir" != "x$file"; then
+      case "$destdir" in
+      [\\/]* | [A-Za-z]:[\\/]*) thisdir="$destdir" ;;
+      *) thisdir="$thisdir/$destdir" ;;
+      esac
+    fi
+
+    file=`$ECHO "$file" | /usr/bin/sed 's%^.*/%%'`
+    file=`ls -ld "$thisdir/$file" | /usr/bin/sed -n 's/.*-> //p'`
+  done
+
+  # Usually 'no', except on cygwin/mingw when embedded into
+  # the cwrapper.
+  WRAPPER_SCRIPT_BELONGS_IN_OBJDIR=no
+  if test "$WRAPPER_SCRIPT_BELONGS_IN_OBJDIR" = "yes"; then
+    # special case for '.'
+    if test "$thisdir" = "."; then
+      thisdir=`pwd`
+    fi
+    # remove .libs from thisdir
+    case "$thisdir" in
+    *[\\/].libs ) thisdir=`$ECHO "$thisdir" | /usr/bin/sed 's%[\\/][^\\/]*$%%'` ;;
+    .libs )   thisdir=. ;;
+    esac
+  fi
+
+  # Try to get the absolute directory name.
+  absdir=`cd "$thisdir" && pwd`
+  test -n "$absdir" && thisdir="$absdir"
+
+  program='test-collect-unicodes'
+  progdir="$thisdir/.libs"
+
+
+  if test -f "$progdir/$program"; then
+    # Add our own library path to DYLD_LIBRARY_PATH
+    DYLD_LIBRARY_PATH="/Users/ariza/git/harfbuzz/build/src/.libs:$DYLD_LIBRARY_PATH"
+
+    # Some systems cannot cope with colon-terminated DYLD_LIBRARY_PATH
+    # The second colon is a workaround for a bug in BeOS R4 sed
+    DYLD_LIBRARY_PATH=`$ECHO "$DYLD_LIBRARY_PATH" | /usr/bin/sed 's/::*$//'`
+
+    export DYLD_LIBRARY_PATH
+
+    if test "$libtool_execute_magic" != "%%%MAGIC variable%%%"; then
+      # Run the actual program with our arguments.
+      func_exec_program ${1+"$@"}
+    fi
+  else
+    # The program doesn't exist.
+    $ECHO "$0: error: '$progdir/$program' does not exist" 1>&2
+    $ECHO "This script is just a wrapper for $program." 1>&2
+    $ECHO "See the libtool documentation for more information." 1>&2
+    exit 1
+  fi
+fi

build/test/api/test-multithread

@@ -0,0 +1,210 @@
+#! /bin/sh
+
+# test-multithread - temporary wrapper script for .libs/test-multithread
+# Generated by libtool (GNU libtool) 2.4.6
+#
+# The test-multithread program cannot be directly executed until all the libtool
+# libraries that it depends on are installed.
+#
+# This wrapper script should never be moved out of the build directory.
+# If it is, it will not operate correctly.
+
+# Sed substitution that helps us do robust quoting.  It backslashifies
+# metacharacters that are still active within double-quoted strings.
+sed_quote_subst='s|\([`"$\\]\)|\\\1|g'
+
+# Be Bourne compatible
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  # Zsh 3.x and 4.x performs word splitting on ${1+"$@"}, which
+  # is contrary to our usage.  Disable this feature.
+  alias -g '${1+"$@"}'='"$@"'
+  setopt NO_GLOB_SUBST
+else
+  case `(set -o) 2>/dev/null` in *posix*) set -o posix;; esac
+fi
+BIN_SH=xpg4; export BIN_SH # for Tru64
+DUALCASE=1; export DUALCASE # for MKS sh
+
+# The HP-UX ksh and POSIX shell print the target directory to stdout
+# if CDPATH is set.
+(unset CDPATH) >/dev/null 2>&1 && unset CDPATH
+
+relink_command=""
+
+# This environment variable determines our operation mode.
+if test "$libtool_install_magic" = "%%%MAGIC variable%%%"; then
+  # install mode needs the following variables:
+  generated_by_libtool_version='2.4.6'
+  notinst_deplibs=' ../../src/libharfbuzz.la'
+else
+  # When we are sourced in execute mode, $file and $ECHO are already set.
+  if test "$libtool_execute_magic" != "%%%MAGIC variable%%%"; then
+    file="$0"
+
+# A function that is used when there is no print builtin or printf.
+func_fallback_echo ()
+{
+  eval 'cat <<_LTECHO_EOF
+$1
+_LTECHO_EOF'
+}
+    ECHO="printf %s\\n"
+  fi
+
+# Very basic option parsing. These options are (a) specific to
+# the libtool wrapper, (b) are identical between the wrapper
+# /script/ and the wrapper /executable/ that is used only on
+# windows platforms, and (c) all begin with the string --lt-
+# (application programs are unlikely to have options that match
+# this pattern).
+#
+# There are only two supported options: --lt-debug and
+# --lt-dump-script. There is, deliberately, no --lt-help.
+#
+# The first argument to this parsing function should be the
+# script's ../../libtool value, followed by no.
+lt_option_debug=
+func_parse_lt_options ()
+{
+  lt_script_arg0=$0
+  shift
+  for lt_opt
+  do
+    case "$lt_opt" in
+    --lt-debug) lt_option_debug=1 ;;
+    --lt-dump-script)
+        lt_dump_D=`$ECHO "X$lt_script_arg0" | /usr/bin/sed -e 's/^X//' -e 's%/[^/]*$%%'`
+        test "X$lt_dump_D" = "X$lt_script_arg0" && lt_dump_D=.
+        lt_dump_F=`$ECHO "X$lt_script_arg0" | /usr/bin/sed -e 's/^X//' -e 's%^.*/%%'`
+        cat "$lt_dump_D/$lt_dump_F"
+        exit 0
+      ;;
+    --lt-*)
+        $ECHO "Unrecognized --lt- option: '$lt_opt'" 1>&2
+        exit 1
+      ;;
+    esac
+  done
+
+  # Print the debug banner immediately:
+  if test -n "$lt_option_debug"; then
+    echo "test-multithread:test-multithread:$LINENO: libtool wrapper (GNU libtool) 2.4.6" 1>&2
+  fi
+}
+
+# Used when --lt-debug. Prints its arguments to stdout
+# (redirection is the responsibility of the caller)
+func_lt_dump_args ()
+{
+  lt_dump_args_N=1;
+  for lt_arg
+  do
+    $ECHO "test-multithread:test-multithread:$LINENO: newargv[$lt_dump_args_N]: $lt_arg"
+    lt_dump_args_N=`expr $lt_dump_args_N + 1`
+  done
+}
+
+# Core function for launching the target application
+func_exec_program_core ()
+{
+
+      if test -n "$lt_option_debug"; then
+        $ECHO "test-multithread:test-multithread:$LINENO: newargv[0]: $progdir/$program" 1>&2
+        func_lt_dump_args ${1+"$@"} 1>&2
+      fi
+      exec "$progdir/$program" ${1+"$@"}
+
+      $ECHO "$0: cannot exec $program $*" 1>&2
+      exit 1
+}
+
+# A function to encapsulate launching the target application
+# Strips options in the --lt-* namespace from $@ and
+# launches target application with the remaining arguments.
+func_exec_program ()
+{
+  case " $* " in
+  *\ --lt-*)
+    for lt_wr_arg
+    do
+      case $lt_wr_arg in
+      --lt-*) ;;
+      *) set x "$@" "$lt_wr_arg"; shift;;
+      esac
+      shift
+    done ;;
+  esac
+  func_exec_program_core ${1+"$@"}
+}
+
+  # Parse options
+  func_parse_lt_options "$0" ${1+"$@"}
+
+  # Find the directory that this script lives in.
+  thisdir=`$ECHO "$file" | /usr/bin/sed 's%/[^/]*$%%'`
+  test "x$thisdir" = "x$file" && thisdir=.
+
+  # Follow symbolic links until we get to the real thisdir.
+  file=`ls -ld "$file" | /usr/bin/sed -n 's/.*-> //p'`
+  while test -n "$file"; do
+    destdir=`$ECHO "$file" | /usr/bin/sed 's%/[^/]*$%%'`
+
+    # If there was a directory component, then change thisdir.
+    if test "x$destdir" != "x$file"; then
+      case "$destdir" in
+      [\\/]* | [A-Za-z]:[\\/]*) thisdir="$destdir" ;;
+      *) thisdir="$thisdir/$destdir" ;;
+      esac
+    fi
+
+    file=`$ECHO "$file" | /usr/bin/sed 's%^.*/%%'`
+    file=`ls -ld "$thisdir/$file" | /usr/bin/sed -n 's/.*-> //p'`
+  done
+
+  # Usually 'no', except on cygwin/mingw when embedded into
+  # the cwrapper.
+  WRAPPER_SCRIPT_BELONGS_IN_OBJDIR=no
+  if test "$WRAPPER_SCRIPT_BELONGS_IN_OBJDIR" = "yes"; then
+    # special case for '.'
+    if test "$thisdir" = "."; then
+      thisdir=`pwd`
+    fi
+    # remove .libs from thisdir
+    case "$thisdir" in
+    *[\\/].libs ) thisdir=`$ECHO "$thisdir" | /usr/bin/sed 's%[\\/][^\\/]*$%%'` ;;
+    .libs )   thisdir=. ;;
+    esac
+  fi
+
+  # Try to get the absolute directory name.
+  absdir=`cd "$thisdir" && pwd`
+  test -n "$absdir" && thisdir="$absdir"
+
+  program='test-multithread'
+  progdir="$thisdir/.libs"
+
+
+  if test -f "$progdir/$program"; then
+    # Add our own library path to DYLD_LIBRARY_PATH
+    DYLD_LIBRARY_PATH="/Users/ariza/git/harfbuzz/build/src/.libs:$DYLD_LIBRARY_PATH"
+
+    # Some systems cannot cope with colon-terminated DYLD_LIBRARY_PATH
+    # The second colon is a workaround for a bug in BeOS R4 sed
+    DYLD_LIBRARY_PATH=`$ECHO "$DYLD_LIBRARY_PATH" | /usr/bin/sed 's/::*$//'`
+
+    export DYLD_LIBRARY_PATH
+
+    if test "$libtool_execute_magic" != "%%%MAGIC variable%%%"; then
+      # Run the actual program with our arguments.
+      func_exec_program ${1+"$@"}
+    fi
+  else
+    # The program doesn't exist.
+    $ECHO "$0: error: '$progdir/$program' does not exist" 1>&2
+    $ECHO "This script is just a wrapper for $program." 1>&2
+    $ECHO "See the libtool documentation for more information." 1>&2
+    exit 1
+  fi
+fi

src/Makefile.sources

@@ -141,7 +141,6 @@ HB_OT_sources = \
 	hb-ot-shape-complex-myanmar.hh \
 	hb-ot-shape-complex-myanmar.cc \
 	hb-ot-shape-complex-thai.cc \
-	hb-ot-shape-complex-tibetan.cc \
 	hb-ot-shape-complex-use.cc \
 	hb-ot-shape-complex-use.hh \
 	hb-ot-shape-complex-use-table.cc \

src/check-symbols.sh

@@ -3,8 +3,6 @@
 LC_ALL=C
 export LC_ALL
 
-test -z "$SKIPCHECKSYMBOLS" || exit 77
-
 test -z "$srcdir" && srcdir=.
 test -z "$libs" && libs=.libs
 stat=0

src/dump-emoji.cc

@@ -45,8 +45,8 @@
 #include <stdlib.h>
 #include <stdio.h>
 
-void cbdt_callback (const uint8_t* data, unsigned int length,
+static void cbdt_callback (const uint8_t* data, unsigned int length,
-                    unsigned int group, unsigned int gid)
+			   unsigned int group, unsigned int gid)
 {
   char output_path[255];
   sprintf (output_path, "out/cbdt-%d-%d.png", group, gid);
@@ -55,8 +55,8 @@ void cbdt_callback (const uint8_t* data, unsigned int length,
   fclose (f);
 }
 
-void sbix_callback (const uint8_t* data, unsigned int length,
+static void sbix_callback (const uint8_t* data, unsigned int length,
-                    unsigned int group, unsigned int gid)
+			   unsigned int group, unsigned int gid)
 {
   char output_path[255];
   sprintf (output_path, "out/sbix-%d-%d.png", group, gid);
@@ -65,8 +65,8 @@ void sbix_callback (const uint8_t* data, unsigned int length,
   fclose (f);
 }
 
-void svg_callback (const uint8_t* data, unsigned int length,
+static void svg_callback (const uint8_t* data, unsigned int length,
-                   unsigned int start_glyph, unsigned int end_glyph)
+			  unsigned int start_glyph, unsigned int end_glyph)
 {
   char output_path[255];
   if (start_glyph == end_glyph)
@@ -83,8 +83,8 @@ void svg_callback (const uint8_t* data, unsigned int length,
   fclose (f);
 }
 
-void colr_cpal_rendering (cairo_font_face_t *cairo_face, unsigned int upem, unsigned int num_glyphs,
+static void colr_cpal_rendering (cairo_font_face_t *cairo_face, unsigned int upem, unsigned int num_glyphs,
-			  const OT::COLR *colr, const OT::CPAL *cpal)
+				 const OT::COLR *colr, const OT::CPAL *cpal)
 {
   for (unsigned int i = 0; i < num_glyphs; ++i)
   {
@@ -162,7 +162,8 @@ void colr_cpal_rendering (cairo_font_face_t *cairo_face, unsigned int upem, unsi
   }
 }
 
-void dump_glyphs (cairo_font_face_t *cairo_face, unsigned int upem, unsigned int num_glyphs)
+static void dump_glyphs (cairo_font_face_t *cairo_face, unsigned int upem,
+			 unsigned int num_glyphs)
 {
   // Dump every glyph available on the font
   return; // disabled for now

src/dump-khmer-data.cc

@@ -34,10 +34,8 @@ main (void)
     hb_glyph_info_t info;
     info.codepoint = u;
     set_khmer_properties (info);
-    if (info.khmer_category() != INDIC_SYLLABIC_CATEGORY_OTHER ||
+    if (info.khmer_category() != INDIC_SYLLABIC_CATEGORY_OTHER)
-	info.khmer_position() != INDIC_MATRA_CATEGORY_NOT_APPLICABLE)
+      printf("U+%04X	%u\n", u,
-      printf("U+%04X	%u	%u\n", u,
+	     info.khmer_category());
-	     info.khmer_category(),
-	     info.khmer_position());
   }
 }

src/gen-use-table.py

@@ -8,7 +8,7 @@ if len (sys.argv) != 5:
 	print ("usage: ./gen-use-table.py IndicSyllabicCategory.txt IndicPositionalCategory.txt UnicodeData.txt Blocks.txt", file=sys.stderr)
 	sys.exit (1)
 
-BLACKLISTED_BLOCKS = ["Thai", "Lao", "Tibetan"]
+BLACKLISTED_BLOCKS = ["Thai", "Lao"]
 
 files = [io.open (x, encoding='utf-8') for x in sys.argv[1:]]
 
@@ -307,11 +307,28 @@ def map_to_use(data):
 
 		# Resolve Indic_Syllabic_Category
 
-		# TODO: These don't have UISC assigned in Unicode 8.0, but
+		# TODO: These don't have UISC assigned in Unicode 8.0, but have UIPC
-		# have UIPC
 		if U == 0x17DD: UISC = Vowel_Dependent
 		if 0x1CE2 <= U <= 0x1CE8: UISC = Cantillation_Mark
 
+		# Tibetan:
+		# TODO: These don't have UISC assigned in Unicode 11.0, but have UIPC
+		if 0x0F18 <= U <= 0x0F19 or 0x0F3E <= U <= 0x0F3F: UISC = Vowel_Dependent
+		if 0x0F86 <= U <= 0x0F87: UISC = Tone_Mark
+		# Overrides to allow NFC order matching syllable
+		# https://github.com/harfbuzz/harfbuzz/issues/1012
+		if UBlock == 'Tibetan' and is_VOWEL (U, UISC, UGC):
+			if UIPC == Top:
+				UIPC = Bottom
+
+		# TODO: https://github.com/harfbuzz/harfbuzz/pull/982
+		# also  https://github.com/harfbuzz/harfbuzz/issues/1012
+		if UBlock == 'Chakma' and is_VOWEL (U, UISC, UGC):
+			if UIPC == Top:
+				UIPC = Bottom
+			elif UIPC == Bottom:
+				UIPC = Top
+
 		# TODO: https://github.com/harfbuzz/harfbuzz/pull/627
 		if 0x1BF2 <= U <= 0x1BF3: UISC = Nukta; UIPC = Bottom
 
@@ -359,13 +376,6 @@ def map_to_use(data):
 		# https://github.com/roozbehp/unicode-data/issues/8
 		if U == 0x0A51: UIPC = Bottom
 
-		# TODO: https://github.com/harfbuzz/harfbuzz/pull/982
-		if UBlock == 'Chakma' and is_VOWEL (U, UISC, UGC):
-			if UIPC == Top:
-				UIPC = Bottom
-			elif UIPC == Bottom:
-				UIPC = Top
-
 		assert (UIPC in [Not_Applicable, Visual_Order_Left] or
 			USE in use_positions), "%s %s %s %s %s" % (hex(U), UIPC, USE, UISC, UGC)
 

src/hb-buffer.hh

@@ -68,8 +68,7 @@ enum hb_buffer_scratch_flags_t {
   HB_BUFFER_SCRATCH_FLAG_HAS_SPACE_FALLBACK		= 0x00000004u,
   HB_BUFFER_SCRATCH_FLAG_HAS_GPOS_ATTACHMENT		= 0x00000008u,
   HB_BUFFER_SCRATCH_FLAG_HAS_UNSAFE_TO_BREAK		= 0x00000010u,
-  HB_BUFFER_SCRATCH_FLAG_HAS_JOINERS			= 0x00000020u,
+  HB_BUFFER_SCRATCH_FLAG_HAS_CGJ			= 0x00000020u,
-  HB_BUFFER_SCRATCH_FLAG_HAS_CGJ			= 0x00000040u,
 
   /* Reserved for complex shapers' internal use. */
   HB_BUFFER_SCRATCH_FLAG_COMPLEX0			= 0x01000000u,

src/hb-font.cc

@@ -1005,7 +1005,7 @@ hb_font_get_glyph_from_name (hb_font_t *font,
  * hb_font_get_extents_for_direction:
  * @font: a font.
  * @direction:
- * @extents:
+ * @extents: (out):
  *
  *
  *

src/hb-ot-face.cc

@@ -41,19 +41,19 @@
 void hb_ot_face_data_t::init0 (hb_face_t *face)
 {
   this->face = face;
-#define HB_OT_LAYOUT_TABLE(Namespace, Type) Type.init0 ();
+#define HB_OT_TABLE(Namespace, Type) Type.init0 ();
-#define HB_OT_LAYOUT_ACCELERATOR(Namespace, Type) HB_OT_LAYOUT_TABLE (Namespace, Type)
+#define HB_OT_ACCELERATOR(Namespace, Type) HB_OT_TABLE (Namespace, Type)
-  HB_OT_LAYOUT_TABLES
+  HB_OT_TABLES
-#undef HB_OT_LAYOUT_ACCELERATOR
+#undef HB_OT_ACCELERATOR
-#undef HB_OT_LAYOUT_TABLE
+#undef HB_OT_TABLE
 }
 void hb_ot_face_data_t::fini (void)
 {
-#define HB_OT_LAYOUT_TABLE(Namespace, Type) Type.fini ();
+#define HB_OT_TABLE(Namespace, Type) Type.fini ();
-#define HB_OT_LAYOUT_ACCELERATOR(Namespace, Type) HB_OT_LAYOUT_TABLE (Namespace, Type)
+#define HB_OT_ACCELERATOR(Namespace, Type) HB_OT_TABLE (Namespace, Type)
-  HB_OT_LAYOUT_TABLES
+  HB_OT_TABLES
-#undef HB_OT_LAYOUT_ACCELERATOR
+#undef HB_OT_ACCELERATOR
-#undef HB_OT_LAYOUT_TABLE
+#undef HB_OT_TABLE
 }
 
 hb_ot_face_data_t *

src/hb-ot-face.hh

@@ -43,67 +43,67 @@
 
 /* Most of these tables are NOT needed for shaping.  But we need to hook them *somewhere*.
  * This is as good as any place. */
-#define HB_OT_LAYOUT_TABLES \
+#define HB_OT_TABLES \
     /* OpenType shaping. */ \
-    HB_OT_LAYOUT_TABLE(OT, JSTF) \
+    HB_OT_TABLE(OT, JSTF) \
-    HB_OT_LAYOUT_TABLE(OT, BASE) \
+    HB_OT_TABLE(OT, BASE) \
     /* AAT shaping. */ \
-    HB_OT_LAYOUT_TABLE(AAT, morx) \
+    HB_OT_TABLE(AAT, morx) \
-    HB_OT_LAYOUT_TABLE(AAT, kerx) \
+    HB_OT_TABLE(AAT, kerx) \
-    HB_OT_LAYOUT_TABLE(AAT, ankr) \
+    HB_OT_TABLE(AAT, ankr) \
-    HB_OT_LAYOUT_TABLE(AAT, trak) \
+    HB_OT_TABLE(AAT, trak) \
     /* OpenType variations. */ \
-    HB_OT_LAYOUT_TABLE(OT, fvar) \
+    HB_OT_TABLE(OT, fvar) \
-    HB_OT_LAYOUT_TABLE(OT, avar) \
+    HB_OT_TABLE(OT, avar) \
-    HB_OT_LAYOUT_TABLE(OT, MVAR) \
+    HB_OT_TABLE(OT, MVAR) \
     /* OpenType math. */ \
-    HB_OT_LAYOUT_TABLE(OT, MATH) \
+    HB_OT_TABLE(OT, MATH) \
     /* OpenType fundamentals. */ \
-    HB_OT_LAYOUT_ACCELERATOR(OT, GDEF) \
+    HB_OT_ACCELERATOR(OT, GDEF) \
-    HB_OT_LAYOUT_ACCELERATOR(OT, GSUB) \
+    HB_OT_ACCELERATOR(OT, GSUB) \
-    HB_OT_LAYOUT_ACCELERATOR(OT, GPOS) \
+    HB_OT_ACCELERATOR(OT, GPOS) \
-    HB_OT_LAYOUT_ACCELERATOR(OT, cmap) \
+    HB_OT_ACCELERATOR(OT, cmap) \
-    HB_OT_LAYOUT_ACCELERATOR(OT, hmtx) \
+    HB_OT_ACCELERATOR(OT, hmtx) \
-    HB_OT_LAYOUT_ACCELERATOR(OT, vmtx) \
+    HB_OT_ACCELERATOR(OT, vmtx) \
-    HB_OT_LAYOUT_ACCELERATOR(OT, post) \
+    HB_OT_ACCELERATOR(OT, post) \
-    HB_OT_LAYOUT_ACCELERATOR(OT, kern) \
+    HB_OT_ACCELERATOR(OT, kern) \
-    HB_OT_LAYOUT_ACCELERATOR(OT, glyf) \
+    HB_OT_ACCELERATOR(OT, glyf) \
-    HB_OT_LAYOUT_ACCELERATOR(OT, cff1) \
+    HB_OT_ACCELERATOR(OT, cff1) \
-    HB_OT_LAYOUT_ACCELERATOR(OT, CBDT) \
+    HB_OT_ACCELERATOR(OT, CBDT) \
     /* */
 
 /* Declare tables. */
-#define HB_OT_LAYOUT_TABLE(Namespace, Type) namespace Namespace { struct Type; }
+#define HB_OT_TABLE(Namespace, Type) namespace Namespace { struct Type; }
-#define HB_OT_LAYOUT_ACCELERATOR(Namespace, Type) HB_OT_LAYOUT_TABLE (Namespace, Type##_accelerator_t)
+#define HB_OT_ACCELERATOR(Namespace, Type) HB_OT_TABLE (Namespace, Type##_accelerator_t)
-HB_OT_LAYOUT_TABLES
+HB_OT_TABLES
-#undef HB_OT_LAYOUT_ACCELERATOR
+#undef HB_OT_ACCELERATOR
-#undef HB_OT_LAYOUT_TABLE
+#undef HB_OT_TABLE
 
 struct hb_ot_face_data_t
 {
   HB_INTERNAL void init0 (hb_face_t *face);
   HB_INTERNAL void fini (void);
 
-#define HB_OT_LAYOUT_TABLE_ORDER(Namespace, Type) \
+#define HB_OT_TABLE_ORDER(Namespace, Type) \
     HB_PASTE (ORDER_, HB_PASTE (Namespace, HB_PASTE (_, Type)))
   enum order_t
   {
     ORDER_ZERO,
-#define HB_OT_LAYOUT_TABLE(Namespace, Type) HB_OT_LAYOUT_TABLE_ORDER (Namespace, Type),
+#define HB_OT_TABLE(Namespace, Type) HB_OT_TABLE_ORDER (Namespace, Type),
-#define HB_OT_LAYOUT_ACCELERATOR(Namespace, Type) HB_OT_LAYOUT_TABLE (Namespace, Type)
+#define HB_OT_ACCELERATOR(Namespace, Type) HB_OT_TABLE (Namespace, Type)
-    HB_OT_LAYOUT_TABLES
+    HB_OT_TABLES
-#undef HB_OT_LAYOUT_ACCELERATOR
+#undef HB_OT_ACCELERATOR
-#undef HB_OT_LAYOUT_TABLE
+#undef HB_OT_TABLE
   };
 
   hb_face_t *face; /* MUST be JUST before the lazy loaders. */
-#define HB_OT_LAYOUT_TABLE(Namespace, Type) \
+#define HB_OT_TABLE(Namespace, Type) \
-  hb_table_lazy_loader_t<Namespace::Type, HB_OT_LAYOUT_TABLE_ORDER (Namespace, Type)> Type;
+  hb_table_lazy_loader_t<Namespace::Type, HB_OT_TABLE_ORDER (Namespace, Type)> Type;
-#define HB_OT_LAYOUT_ACCELERATOR(Namespace, Type) \
+#define HB_OT_ACCELERATOR(Namespace, Type) \
-  hb_face_lazy_loader_t<Namespace::Type##_accelerator_t, HB_OT_LAYOUT_TABLE_ORDER (Namespace, Type)> Type;
+  hb_face_lazy_loader_t<Namespace::Type##_accelerator_t, HB_OT_TABLE_ORDER (Namespace, Type)> Type;
-  HB_OT_LAYOUT_TABLES
+  HB_OT_TABLES
-#undef HB_OT_LAYOUT_ACCELERATOR
+#undef HB_OT_ACCELERATOR
-#undef HB_OT_LAYOUT_TABLE
+#undef HB_OT_TABLE
 };
 
 

src/hb-ot-layout-gsub-table.hh

@@ -762,7 +762,6 @@ struct Ligature
       return_trace (true);
     }
 
-    bool is_mark_ligature = false;
     unsigned int total_component_count = 0;
 
     unsigned int match_length = 0;
@@ -774,7 +773,6 @@ struct Ligature
 			      nullptr,
 			      &match_length,
 			      match_positions,
-			      &is_mark_ligature,
 			      &total_component_count)))
       return_trace (false);
 
@@ -783,7 +781,6 @@ struct Ligature
 		  match_positions,
 		  match_length,
 		  ligGlyph,
-		  is_mark_ligature,
 		  total_component_count);
 
     return_trace (true);

src/hb-ot-layout-gsubgpos.hh

@@ -731,7 +731,6 @@ static inline bool match_input (hb_ot_apply_context_t *c,
 				const void *match_data,
 				unsigned int *end_offset,
 				unsigned int match_positions[HB_MAX_CONTEXT_LENGTH],
-				bool *p_is_mark_ligature = nullptr,
 				unsigned int *p_total_component_count = nullptr)
 {
   TRACE_APPLY (nullptr);
@@ -768,8 +767,6 @@ static inline bool match_input (hb_ot_apply_context_t *c,
    *     https://github.com/harfbuzz/harfbuzz/issues/545
    */
 
-  bool is_mark_ligature = _hb_glyph_info_is_mark (&buffer->cur());
-
   unsigned int total_component_count = 0;
   total_component_count += _hb_glyph_info_get_lig_num_comps (&buffer->cur());
 
@@ -836,15 +833,11 @@ static inline bool match_input (hb_ot_apply_context_t *c,
 	return_trace (false);
     }
 
-    is_mark_ligature = is_mark_ligature && _hb_glyph_info_is_mark (&buffer->info[skippy_iter.idx]);
     total_component_count += _hb_glyph_info_get_lig_num_comps (&buffer->info[skippy_iter.idx]);
   }
 
   *end_offset = skippy_iter.idx - buffer->idx + 1;
 
-  if (p_is_mark_ligature)
-    *p_is_mark_ligature = is_mark_ligature;
-
   if (p_total_component_count)
     *p_total_component_count = total_component_count;
 
@@ -855,7 +848,6 @@ static inline bool ligate_input (hb_ot_apply_context_t *c,
 				 unsigned int match_positions[HB_MAX_CONTEXT_LENGTH], /* Including the first glyph */
 				 unsigned int match_length,
 				 hb_codepoint_t lig_glyph,
-				 bool is_mark_ligature,
 				 unsigned int total_component_count)
 {
   TRACE_APPLY (nullptr);
@@ -864,11 +856,15 @@ static inline bool ligate_input (hb_ot_apply_context_t *c,
 
   buffer->merge_clusters (buffer->idx, buffer->idx + match_length);
 
-  /*
+  /* - If a base and one or more marks ligate, consider that as a base, NOT
-   * - If it *is* a mark ligature, we don't allocate a new ligature id, and leave
+   *   ligature, such that all following marks can still attach to it.
+   *   https://github.com/harfbuzz/harfbuzz/issues/1109
+   *
+   * - If all components of the ligature were marks, we call this a mark ligature.
+   *   If it *is* a mark ligature, we don't allocate a new ligature id, and leave
    *   the ligature to keep its old ligature id.  This will allow it to attach to
    *   a base ligature in GPOS.  Eg. if the sequence is: LAM,LAM,SHADDA,FATHA,HEH,
-   *   and LAM,LAM,HEH for a ligature, they will leave SHADDA and FATHA wit a
+   *   and LAM,LAM,HEH for a ligature, they will leave SHADDA and FATHA with a
    *   ligature id and component value of 2.  Then if SHADDA,FATHA form a ligature
    *   later, we don't want them to lose their ligature id/component, otherwise
    *   GPOS will fail to correctly position the mark ligature on top of the
@@ -892,13 +888,24 @@ static inline bool ligate_input (hb_ot_apply_context_t *c,
    *   https://bugzilla.gnome.org/show_bug.cgi?id=437633
    */
 
-  unsigned int klass = is_mark_ligature ? 0 : HB_OT_LAYOUT_GLYPH_PROPS_LIGATURE;
+  bool is_base_ligature = _hb_glyph_info_is_base_glyph (&buffer->info[match_positions[0]]);
-  unsigned int lig_id = is_mark_ligature ? 0 : _hb_allocate_lig_id (buffer);
+  bool is_mark_ligature = _hb_glyph_info_is_mark (&buffer->info[match_positions[0]]);
+  for (unsigned int i = 1; i < count; i++)
+    if (!_hb_glyph_info_is_mark (&buffer->info[match_positions[i]]))
+    {
+      is_base_ligature = false;
+      is_mark_ligature = false;
+      break;
+    }
+  bool is_ligature = !is_base_ligature && !is_mark_ligature;
+
+  unsigned int klass = is_ligature ? HB_OT_LAYOUT_GLYPH_PROPS_LIGATURE : 0;
+  unsigned int lig_id = is_ligature ? _hb_allocate_lig_id (buffer) : 0;
   unsigned int last_lig_id = _hb_glyph_info_get_lig_id (&buffer->cur());
   unsigned int last_num_components = _hb_glyph_info_get_lig_num_comps (&buffer->cur());
   unsigned int components_so_far = last_num_components;
 
-  if (!is_mark_ligature)
+  if (is_ligature)
   {
     _hb_glyph_info_set_lig_props_for_ligature (&buffer->cur(), lig_id, total_component_count);
     if (_hb_glyph_info_get_general_category (&buffer->cur()) == HB_UNICODE_GENERAL_CATEGORY_NON_SPACING_MARK)
@@ -912,7 +919,8 @@ static inline bool ligate_input (hb_ot_apply_context_t *c,
   {
     while (buffer->idx < match_positions[i] && buffer->successful)
     {
-      if (!is_mark_ligature) {
+      if (is_ligature)
+      {
         unsigned int this_comp = _hb_glyph_info_get_lig_comp (&buffer->cur());
 	if (this_comp == 0)
 	  this_comp = last_num_components;

src/hb-ot-layout.hh

@@ -224,16 +224,8 @@ _hb_glyph_info_set_unicode_props (hb_glyph_info_t *info, hb_buffer_t *buffer)
     {
       buffer->scratch_flags |= HB_BUFFER_SCRATCH_FLAG_HAS_DEFAULT_IGNORABLES;
       props |=  UPROPS_MASK_IGNORABLE;
-      if (u == 0x200Cu)
+      if (u == 0x200Cu) props |= UPROPS_MASK_Cf_ZWNJ;
-      {
+      else if (u == 0x200Du) props |= UPROPS_MASK_Cf_ZWJ;
-	props |= UPROPS_MASK_Cf_ZWNJ;
-	buffer->scratch_flags |= HB_BUFFER_SCRATCH_FLAG_HAS_JOINERS;
-      }
-      else if (u == 0x200Du)
-      {
-	props |= UPROPS_MASK_Cf_ZWJ;
-	buffer->scratch_flags |= HB_BUFFER_SCRATCH_FLAG_HAS_JOINERS;
-      }
       /* Mongolian Free Variation Selectors need to be remembered
        * because although we need to hide them like default-ignorables,
        * they need to non-ignorable during shaping.  This is similar to

src/hb-ot-map.hh

@@ -175,6 +175,7 @@ enum hb_ot_map_feature_flags_t
   F_MANUAL_ZWJ		= 0x0008u, /* Don't skip over ZWJ when matching **input**. */
   F_MANUAL_JOINERS	= F_MANUAL_ZWNJ | F_MANUAL_ZWJ,
   F_GLOBAL_MANUAL_JOINERS= F_GLOBAL | F_MANUAL_JOINERS,
+  F_GLOBAL_HAS_FALLBACK = F_GLOBAL | F_HAS_FALLBACK,
   F_GLOBAL_SEARCH	= 0x0010u, /* If feature not found in LangSys, look for it in global feature list and pick one. */
   F_RANDOM		= 0x0020u  /* Randomly select a glyph from an AlternateSubstFormat1 subtable. */
 };
@@ -204,8 +205,10 @@ struct hb_ot_map_builder_t
   inline void add_feature (const hb_ot_map_feature_t &feat)
   { add_feature (feat.tag, feat.flags); }
 
-  inline void enable_feature (hb_tag_t tag)
+  inline void enable_feature (hb_tag_t tag,
-  { add_feature (tag, F_GLOBAL); }
+			      hb_ot_map_feature_flags_t flags=F_NONE,
+			      unsigned int value=1)
+  { add_feature (tag, F_GLOBAL | flags, value); }
 
   inline void disable_feature (hb_tag_t tag)
   { add_feature (tag, F_GLOBAL, 0); }

src/hb-ot-shape-complex-arabic.cc

@@ -158,11 +158,6 @@ static const struct arabic_state_table_entry {
 };
 
 
-static void
-flip_joiners (const hb_ot_shape_plan_t *plan,
-	      hb_font_t *font,
-	      hb_buffer_t *buffer);
-
 static void
 arabic_fallback_shape (const hb_ot_shape_plan_t *plan,
 		       hb_font_t *font,
@@ -217,28 +212,20 @@ collect_features_arabic (hb_ot_shape_planner_t *plan)
   }
 
   /* Normally, Unicode says a ZWNJ means "don't ligate".  In Arabic script
-   * however, it says a ZWJ should also mean "don't ligate".  So we convert
+   * however, it says a ZWJ should also mean "don't ligate".  So we run
-   * a ZWJ to a ZWNJ for GSUB.  We want to revert it back to ZWJ before
+   * the main ligating features as MANUAL_ZWJ. */
-   * GPOS processing though.  So we just flip their roles, and flip back
-   * later.  Note that this makes a ZWNJ into ZWJ for GSUB stage, which
-   * means it would *not* break ligatures.  But since ligatures around
-   * ZWNJ are rare, we don't care.
-   *
-   * Since we don't currently have a way to apply a pause before GPOS
-   * starts, let's just do this dance around a few required GUSB features. */
-  map->add_gsub_pause (flip_joiners);
 
-  map->add_feature (HB_TAG('r','l','i','g'), F_GLOBAL | F_HAS_FALLBACK);
+  map->enable_feature (HB_TAG('r','l','i','g'), F_MANUAL_ZWJ | F_HAS_FALLBACK);
 
   if (plan->props.script == HB_SCRIPT_ARABIC)
     map->add_gsub_pause (arabic_fallback_shape);
 
   /* No pause after rclt.  See 98460779bae19e4d64d29461ff154b3527bf8420. */
-  map->enable_feature (HB_TAG('r','c','l','t'));
+  map->enable_feature (HB_TAG('r','c','l','t'), F_MANUAL_ZWJ);
-  map->enable_feature (HB_TAG('c','a','l','t'));
+  map->enable_feature (HB_TAG('c','a','l','t'), F_MANUAL_ZWJ);
+  map->add_gsub_pause (nullptr);
 
   /* And undo here. */
-  map->add_gsub_pause (flip_joiners);
 
   /* The spec includes 'cswh'.  Earlier versions of Windows
    * used to enable this by default, but testing suggests
@@ -393,22 +380,6 @@ setup_masks_arabic (const hb_ot_shape_plan_t *plan,
   setup_masks_arabic_plan (arabic_plan, buffer, plan->props.script);
 }
 
-
-static void
-flip_joiners (const hb_ot_shape_plan_t *plan HB_UNUSED,
-	      hb_font_t *font HB_UNUSED,
-	      hb_buffer_t *buffer)
-{
-  if (!(buffer->scratch_flags & HB_BUFFER_SCRATCH_FLAG_HAS_JOINERS))
-    return;
-
-  unsigned int count = buffer->len;
-  hb_glyph_info_t *info = buffer->info;
-  for (unsigned int i = 0; i < count; i++)
-    if (_hb_glyph_info_is_joiner (&info[i]))
-      _hb_glyph_info_flip_joiners (&info[i]);
-}
-
 static void
 arabic_fallback_shape (const hb_ot_shape_plan_t *plan,
 		       hb_font_t *font,
@@ -731,7 +702,7 @@ const hb_ot_complex_shaper_t _hb_ot_complex_shaper_arabic =
   nullptr, /* decompose */
   nullptr, /* compose */
   setup_masks_arabic,
-  nullptr, /* disable_otl */
+  HB_TAG_NONE, /* gpos_tag */
   reorder_marks_arabic,
   HB_OT_SHAPE_ZERO_WIDTH_MARKS_BY_GDEF_LATE,
   true, /* fallback_position */

src/hb-ot-shape-complex-default.cc

@@ -39,7 +39,7 @@ const hb_ot_complex_shaper_t _hb_ot_complex_shaper_default =
   nullptr, /* decompose */
   nullptr, /* compose */
   nullptr, /* setup_masks */
-  nullptr, /* disable_otl */
+  HB_TAG_NONE, /* gpos_tag */
   nullptr, /* reorder_marks */
   HB_OT_SHAPE_ZERO_WIDTH_MARKS_BY_GDEF_LATE,
   true, /* fallback_position */

src/hb-ot-shape-complex-hangul.cc

@@ -424,7 +424,7 @@ const hb_ot_complex_shaper_t _hb_ot_complex_shaper_hangul =
   nullptr, /* decompose */
   nullptr, /* compose */
   setup_masks_hangul,
-  nullptr, /* disable_otl */
+  HB_TAG_NONE, /* gpos_tag */
   nullptr, /* reorder_marks */
   HB_OT_SHAPE_ZERO_WIDTH_MARKS_NONE,
   false, /* fallback_position */

src/hb-ot-shape-complex-hebrew.cc

@@ -70,7 +70,7 @@ compose_hebrew (const hb_ot_shape_normalize_context_t *c,
 
   bool found = (bool) c->unicode->compose (a, b, ab);
 
-  if (!found && !c->plan->has_mark)
+  if (!found && !c->plan->has_gpos_mark)
   {
       /* Special-case Hebrew presentation forms that are excluded from
        * standard normalization, but wanted for old fonts. */
@@ -154,18 +154,6 @@ compose_hebrew (const hb_ot_shape_normalize_context_t *c,
   return found;
 }
 
-static bool
-disable_otl_hebrew (const hb_ot_shape_plan_t *plan)
-{
-  /* For Hebrew shaper, use fallback if GPOS does not have 'hebr'
-   * script.  This matches Uniscribe better, and makes fonts like
-   * Arial that have GSUB/GPOS/GDEF but no data for Hebrew work.
-   * See:
-   * https://github.com/harfbuzz/harfbuzz/issues/347#issuecomment-267838368
-   */
-  return plan->map.chosen_script[1] != HB_TAG ('h','e','b','r');
-}
-
 
 const hb_ot_complex_shaper_t _hb_ot_complex_shaper_hebrew =
 {
@@ -179,7 +167,7 @@ const hb_ot_complex_shaper_t _hb_ot_complex_shaper_hebrew =
   nullptr, /* decompose */
   compose_hebrew,
   nullptr, /* setup_masks */
-  disable_otl_hebrew,
+  HB_TAG ('h','e','b','r'), /* gpos_tag. https://github.com/harfbuzz/harfbuzz/issues/347#issuecomment-267838368 */
   nullptr, /* reorder_marks */
   HB_OT_SHAPE_ZERO_WIDTH_MARKS_BY_GDEF_LATE,
   true, /* fallback_position */

src/hb-ot-shape-complex-indic-machine.rl

@@ -52,7 +52,6 @@ DOTTEDCIRCLE = 12;
 RS    = 13;
 Repha = 15;
 Ra    = 16;
-CM    = 17;
 Symbol= 18;
 CS    = 19;
 
@@ -68,15 +67,16 @@ matra_group = z{0,3}.M.N?.(H | forced_rakar)?;
 syllable_tail = (z?.SM.SM?.ZWNJ?)? A{0,3}?;
 halant_group = (z?.H.(ZWJ.N?)?);
 final_halant_group = halant_group | H.ZWNJ;
-medial_group = CM?;
 halant_or_matra_group = (final_halant_group | matra_group{0,4});
 
+complex_syllable_tail = (halant_group.cn){0,4} halant_or_matra_group syllable_tail;
 
-consonant_syllable =	(Repha|CS)? (cn.halant_group){0,4} cn medial_group halant_or_matra_group syllable_tail;
+
-vowel_syllable =	reph? V.n? (ZWJ | (halant_group.cn){0,4} medial_group halant_or_matra_group syllable_tail);
+consonant_syllable =	(Repha|CS)? cn complex_syllable_tail;
-standalone_cluster =	((Repha|CS)? PLACEHOLDER | reph? DOTTEDCIRCLE).n? (halant_group.cn){0,4} medial_group halant_or_matra_group syllable_tail;
+vowel_syllable =	reph? V.n? (ZWJ | complex_syllable_tail);
+standalone_cluster =	((Repha|CS)? PLACEHOLDER | reph? DOTTEDCIRCLE).n? complex_syllable_tail;
 symbol_cluster = 	symbol syllable_tail;
-broken_cluster =	reph? n? (halant_group.cn){0,4} medial_group halant_or_matra_group syllable_tail;
+broken_cluster =	reph? n? complex_syllable_tail;
 other =			any;
 
 main := |*
@@ -93,10 +93,9 @@ main := |*
 
 #define found_syllable(syllable_type) \
   HB_STMT_START { \
-    if (0) fprintf (stderr, "syllable %d..%d %s\n", last, p+1, #syllable_type); \
+    if (0) fprintf (stderr, "syllable %d..%d %s\n", ts, te, #syllable_type); \
-    for (unsigned int i = last; i < p+1; i++) \
+    for (unsigned int i = ts; i < te; i++) \
       info[i].syllable() = (syllable_serial << 4) | syllable_type; \
-    last = p+1; \
     syllable_serial++; \
     if (unlikely (syllable_serial == 16)) syllable_serial = 1; \
   } HB_STMT_END
@@ -104,7 +103,7 @@ main := |*
 static void
 find_syllables (hb_buffer_t *buffer)
 {
-  unsigned int p, pe, eof, ts HB_UNUSED, te, act;
+  unsigned int p, pe, eof, ts, te, act;
   int cs;
   hb_glyph_info_t *info = buffer->info;
   %%{
@@ -115,7 +114,6 @@ find_syllables (hb_buffer_t *buffer)
   p = 0;
   pe = eof = buffer->len;
 
-  unsigned int last = 0;
   unsigned int syllable_serial = 1;
   %%{
     write exec;

src/hb-ot-shape-complex-indic.cc

@@ -720,7 +720,7 @@ initial_reordering_consonant_syllable (const hb_ot_shape_plan_t *plan,
     indic_position_t last_pos = POS_START;
     for (unsigned int i = start; i < end; i++)
     {
-      if ((FLAG_UNSAFE (info[i].indic_category()) & (JOINER_FLAGS | FLAG (OT_N) | FLAG (OT_RS) | MEDIAL_FLAGS | FLAG (OT_H))))
+      if ((FLAG_UNSAFE (info[i].indic_category()) & (JOINER_FLAGS | FLAG (OT_N) | FLAG (OT_RS) | FLAG (OT_H))))
       {
 	info[i].indic_position() = last_pos;
 	if (unlikely (info[i].indic_category() == OT_H &&
@@ -1620,7 +1620,7 @@ const hb_ot_complex_shaper_t _hb_ot_complex_shaper_indic =
   decompose_indic,
   compose_indic,
   setup_masks_indic,
-  nullptr, /* disable_otl */
+  HB_TAG_NONE, /* gpos_tag */
   nullptr, /* reorder_marks */
   HB_OT_SHAPE_ZERO_WIDTH_MARKS_NONE,
   false, /* fallback_position */

src/hb-ot-shape-complex-indic.hh

@@ -29,9 +29,7 @@
 
 #include "hb.hh"
 
-
 #include "hb-ot-shape-complex.hh"
-#include "hb-ot-shape.hh" /* XXX Remove */
 
 
 /* buffer var allocations */
@@ -64,19 +62,17 @@ enum indic_category_t {
   OT_Coeng = 14, /* Khmer-style Virama. */
   OT_Repha = 15, /* Atomically-encoded logical or visual repha. */
   OT_Ra = 16,
-  OT_CM = 17,  /* Consonant-Medial. */
+  OT_CM = 17,  /* Consonant-Medial; Unused by Indic shaper. */
   OT_Symbol = 18, /* Avagraha, etc that take marks (SM,A,VD). */
   OT_CS = 19
 };
 
-#define MEDIAL_FLAGS (FLAG (OT_CM))
-
 /* Note:
  *
  * We treat Vowels and placeholders as if they were consonants.  This is safe because Vowels
  * cannot happen in a consonant syllable.  The plus side however is, we can call the
  * consonant syllable logic from the vowel syllable function and get it all right! */
-#define CONSONANT_FLAGS (FLAG (OT_C) | FLAG (OT_CS) | FLAG (OT_Ra) | MEDIAL_FLAGS | FLAG (OT_V) | FLAG (OT_PLACEHOLDER) | FLAG (OT_DOTTEDCIRCLE))
+#define CONSONANT_FLAGS (FLAG (OT_C) | FLAG (OT_CS) | FLAG (OT_Ra) | FLAG (OT_V) | FLAG (OT_PLACEHOLDER) | FLAG (OT_DOTTEDCIRCLE))
 #define JOINER_FLAGS (FLAG (OT_ZWJ) | FLAG (OT_ZWNJ))
 
 
@@ -125,7 +121,7 @@ enum indic_syllabic_category_t {
   INDIC_SYLLABIC_CATEGORY_CONSONANT_PRECEDING_REPHA	= OT_Repha,
   INDIC_SYLLABIC_CATEGORY_CONSONANT_PREFIXED		= OT_X, /* Don't care. */
   INDIC_SYLLABIC_CATEGORY_CONSONANT_SUBJOINED		= OT_CM,
-  INDIC_SYLLABIC_CATEGORY_CONSONANT_SUCCEEDING_REPHA	= OT_N,
+  INDIC_SYLLABIC_CATEGORY_CONSONANT_SUCCEEDING_REPHA	= OT_CM,
   INDIC_SYLLABIC_CATEGORY_CONSONANT_WITH_STACKER	= OT_CS,
   INDIC_SYLLABIC_CATEGORY_GEMINATION_MARK		= OT_SM, /* https://github.com/harfbuzz/harfbuzz/issues/552 */
   INDIC_SYLLABIC_CATEGORY_INVISIBLE_STACKER		= OT_Coeng,

src/hb-ot-shape-complex-khmer-machine.hh

@@ -34,138 +34,207 @@
 
 #line 36 "hb-ot-shape-complex-khmer-machine.hh"
 static const unsigned char _khmer_syllable_machine_trans_keys[] = {
-	7u, 7u, 1u, 16u, 13u, 13u, 1u, 16u, 7u, 13u, 7u, 7u, 1u, 16u, 13u, 13u, 
+	5u, 26u, 5u, 21u, 5u, 26u, 5u, 21u, 1u, 16u, 5u, 21u, 5u, 26u, 5u, 21u, 
-	1u, 16u, 7u, 13u, 1u, 16u, 3u, 14u, 3u, 14u, 5u, 14u, 3u, 14u, 5u, 14u, 
+	5u, 26u, 5u, 21u, 1u, 16u, 5u, 21u, 5u, 26u, 5u, 21u, 1u, 16u, 5u, 21u, 
-	8u, 8u, 3u, 13u, 3u, 8u, 8u, 8u, 3u, 8u, 3u, 14u, 3u, 14u, 5u, 14u, 
+	5u, 26u, 5u, 21u, 5u, 26u, 5u, 21u, 5u, 26u, 1u, 16u, 1u, 29u, 5u, 29u, 
-	3u, 14u, 5u, 14u, 8u, 8u, 3u, 13u, 3u, 8u, 8u, 8u, 3u, 8u, 3u, 14u, 
+	5u, 29u, 5u, 29u, 22u, 22u, 5u, 22u, 5u, 29u, 5u, 29u, 5u, 29u, 5u, 26u, 
-	3u, 14u, 7u, 13u, 7u, 7u, 1u, 16u, 0
+	5u, 29u, 5u, 29u, 22u, 22u, 5u, 22u, 5u, 29u, 5u, 29u, 1u, 16u, 5u, 29u, 
+	5u, 29u, 0
 };
 
 static const char _khmer_syllable_machine_key_spans[] = {
-	1, 16, 1, 16, 7, 1, 16, 1, 
+	22, 17, 22, 17, 16, 17, 22, 17, 
-	16, 7, 16, 12, 12, 10, 12, 10, 
+	22, 17, 16, 17, 22, 17, 16, 17, 
-	1, 11, 6, 1, 6, 12, 12, 10, 
+	22, 17, 22, 17, 22, 16, 29, 25, 
-	12, 10, 1, 11, 6, 1, 6, 12, 
+	25, 25, 1, 18, 25, 25, 25, 22, 
-	12, 7, 1, 16
+	25, 25, 1, 18, 25, 25, 16, 25, 
+	25
 };
 
 static const short _khmer_syllable_machine_index_offsets[] = {
-	0, 2, 19, 21, 38, 46, 48, 65, 
+	0, 23, 41, 64, 82, 99, 117, 140, 
-	67, 84, 92, 109, 122, 135, 146, 159, 
+	158, 181, 199, 216, 234, 257, 275, 292, 
-	170, 172, 184, 191, 193, 200, 213, 226, 
+	310, 333, 351, 374, 392, 415, 432, 462, 
-	237, 250, 261, 263, 275, 282, 284, 291, 
+	488, 514, 540, 542, 561, 587, 613, 639, 
-	304, 317, 325, 327
+	662, 688, 714, 716, 735, 761, 787, 804, 
+	830
 };
 
 static const char _khmer_syllable_machine_indicies[] = {
-	1, 0, 2, 2, 0, 0, 0, 0, 
+	1, 1, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 2, 
+	3, 0, 0, 0, 0, 4, 0, 1, 
+	1, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 3, 
+	0, 1, 1, 0, 0, 0, 0, 0, 
 	0, 0, 0, 0, 0, 0, 0, 0, 
-	0, 2, 0, 3, 0, 4, 4, 0, 
+	0, 3, 0, 0, 0, 0, 4, 0, 
+	5, 5, 0, 0, 0, 0, 0, 0, 
 	0, 0, 0, 0, 0, 0, 0, 0, 
-	0, 0, 0, 0, 4, 0, 1, 0, 
+	4, 0, 6, 6, 0, 0, 0, 0, 
-	0, 0, 0, 0, 5, 0, 7, 6, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
-	8, 8, 6, 6, 6, 6, 6, 6, 
+	0, 6, 0, 7, 7, 0, 0, 0, 
-	6, 6, 6, 6, 6, 6, 6, 8, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
-	6, 9, 6, 10, 10, 6, 6, 6, 
+	0, 0, 0, 8, 0, 9, 9, 0, 
-	6, 6, 6, 6, 6, 6, 6, 6, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
-	6, 6, 10, 6, 7, 6, 6, 6, 
+	0, 0, 0, 0, 0, 10, 0, 0, 
-	6, 6, 11, 6, 4, 4, 13, 12, 
+	0, 0, 4, 0, 9, 9, 0, 0, 
-	14, 15, 7, 16, 12, 12, 4, 4, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
-	11, 17, 12, 4, 12, 19, 18, 20, 
+	0, 0, 0, 0, 10, 0, 11, 11, 
-	21, 1, 22, 18, 18, 18, 18, 5, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
-	23, 18, 24, 18, 21, 21, 1, 22, 
+	0, 0, 0, 0, 0, 0, 12, 0, 
-	18, 18, 18, 18, 18, 23, 18, 21, 
+	0, 0, 0, 4, 0, 11, 11, 0, 
-	21, 1, 22, 18, 18, 18, 18, 18, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
-	23, 18, 25, 18, 21, 21, 1, 22, 
+	0, 0, 0, 0, 0, 12, 0, 13, 
-	18, 18, 18, 18, 18, 26, 18, 21, 
+	13, 0, 0, 0, 0, 0, 0, 0, 
-	21, 1, 22, 18, 18, 18, 18, 18, 
+	0, 0, 0, 0, 0, 0, 13, 0, 
-	26, 18, 27, 18, 28, 18, 29, 18, 
+	15, 15, 14, 14, 14, 14, 14, 14, 
-	18, 22, 18, 18, 18, 18, 3, 18, 
+	14, 14, 14, 14, 14, 14, 14, 14, 
-	30, 18, 18, 18, 18, 22, 18, 22, 
+	16, 14, 15, 15, 17, 17, 17, 17, 
-	18, 28, 18, 18, 18, 18, 22, 18, 
+	17, 17, 17, 17, 17, 17, 17, 17, 
-	19, 18, 21, 21, 1, 22, 18, 18, 
+	17, 17, 16, 17, 17, 17, 17, 18, 
-	18, 18, 18, 23, 18, 32, 31, 33, 
+	17, 19, 19, 17, 17, 17, 17, 17, 
-	33, 7, 16, 31, 31, 31, 31, 31, 
+	17, 17, 17, 17, 17, 17, 17, 17, 
-	34, 31, 33, 33, 7, 16, 31, 31, 
+	17, 18, 17, 20, 20, 17, 17, 17, 
-	31, 31, 31, 34, 31, 35, 31, 33, 
+	17, 17, 17, 17, 17, 17, 17, 17, 
-	33, 7, 16, 31, 31, 31, 31, 31, 
+	17, 17, 20, 17, 21, 21, 17, 17, 
-	36, 31, 33, 33, 7, 16, 31, 31, 
+	17, 17, 17, 17, 17, 17, 17, 17, 
-	31, 31, 31, 36, 31, 37, 31, 38, 
+	17, 17, 17, 17, 22, 17, 23, 23, 
-	31, 39, 31, 31, 16, 31, 31, 31, 
+	17, 17, 17, 17, 17, 17, 17, 17, 
-	31, 9, 31, 40, 31, 31, 31, 31, 
+	17, 17, 17, 17, 17, 17, 24, 17, 
-	16, 31, 16, 31, 38, 31, 31, 31, 
+	17, 17, 17, 18, 17, 23, 23, 17, 
-	31, 16, 31, 13, 31, 41, 33, 7, 
+	17, 17, 17, 17, 17, 17, 17, 17, 
-	16, 31, 31, 31, 31, 11, 34, 31, 
+	17, 17, 17, 17, 17, 24, 17, 25, 
-	13, 31, 33, 33, 7, 16, 31, 31, 
+	25, 17, 17, 17, 17, 17, 17, 17, 
-	31, 31, 31, 34, 31, 7, 42, 42, 
+	17, 17, 17, 17, 17, 17, 17, 26, 
-	42, 42, 42, 11, 42, 7, 42, 10, 
+	17, 17, 17, 17, 18, 17, 25, 25, 
-	10, 42, 42, 42, 42, 42, 42, 42, 
+	17, 17, 17, 17, 17, 17, 17, 17, 
-	42, 42, 42, 42, 42, 42, 10, 42, 
+	17, 17, 17, 17, 17, 17, 26, 17, 
+	15, 15, 17, 17, 17, 17, 17, 17, 
+	17, 17, 17, 17, 17, 17, 17, 27, 
+	16, 17, 17, 17, 17, 18, 17, 28, 
+	28, 17, 17, 17, 17, 17, 17, 17, 
+	17, 17, 17, 17, 17, 17, 28, 17, 
+	13, 13, 29, 29, 30, 30, 29, 29, 
+	29, 29, 2, 2, 29, 31, 29, 13, 
+	29, 29, 29, 29, 16, 20, 29, 29, 
+	29, 18, 24, 26, 22, 29, 33, 33, 
+	32, 32, 32, 32, 32, 32, 32, 34, 
+	32, 32, 32, 32, 32, 2, 3, 6, 
+	32, 32, 32, 4, 10, 12, 8, 32, 
+	35, 35, 32, 32, 32, 32, 32, 32, 
+	32, 36, 32, 32, 32, 32, 32, 32, 
+	3, 6, 32, 32, 32, 4, 10, 12, 
+	8, 32, 5, 5, 32, 32, 32, 32, 
+	32, 32, 32, 36, 32, 32, 32, 32, 
+	32, 32, 4, 6, 32, 32, 32, 32, 
+	32, 32, 8, 32, 6, 32, 7, 7, 
+	32, 32, 32, 32, 32, 32, 32, 36, 
+	32, 32, 32, 32, 32, 32, 8, 6, 
+	32, 37, 37, 32, 32, 32, 32, 32, 
+	32, 32, 36, 32, 32, 32, 32, 32, 
+	32, 10, 6, 32, 32, 32, 4, 32, 
+	32, 8, 32, 38, 38, 32, 32, 32, 
+	32, 32, 32, 32, 36, 32, 32, 32, 
+	32, 32, 32, 12, 6, 32, 32, 32, 
+	4, 10, 32, 8, 32, 35, 35, 32, 
+	32, 32, 32, 32, 32, 32, 34, 32, 
+	32, 32, 32, 32, 32, 3, 6, 32, 
+	32, 32, 4, 10, 12, 8, 32, 15, 
+	15, 39, 39, 39, 39, 39, 39, 39, 
+	39, 39, 39, 39, 39, 39, 39, 16, 
+	39, 39, 39, 39, 18, 39, 41, 41, 
+	40, 40, 40, 40, 40, 40, 40, 42, 
+	40, 40, 40, 40, 40, 40, 16, 20, 
+	40, 40, 40, 18, 24, 26, 22, 40, 
+	19, 19, 40, 40, 40, 40, 40, 40, 
+	40, 42, 40, 40, 40, 40, 40, 40, 
+	18, 20, 40, 40, 40, 40, 40, 40, 
+	22, 40, 20, 40, 21, 21, 40, 40, 
+	40, 40, 40, 40, 40, 42, 40, 40, 
+	40, 40, 40, 40, 22, 20, 40, 43, 
+	43, 40, 40, 40, 40, 40, 40, 40, 
+	42, 40, 40, 40, 40, 40, 40, 24, 
+	20, 40, 40, 40, 18, 40, 40, 22, 
+	40, 44, 44, 40, 40, 40, 40, 40, 
+	40, 40, 42, 40, 40, 40, 40, 40, 
+	40, 26, 20, 40, 40, 40, 18, 24, 
+	40, 22, 40, 28, 28, 39, 39, 39, 
+	39, 39, 39, 39, 39, 39, 39, 39, 
+	39, 39, 28, 39, 45, 45, 40, 40, 
+	40, 40, 40, 40, 40, 46, 40, 40, 
+	40, 40, 40, 27, 16, 20, 40, 40, 
+	40, 18, 24, 26, 22, 40, 41, 41, 
+	40, 40, 40, 40, 40, 40, 40, 46, 
+	40, 40, 40, 40, 40, 40, 16, 20, 
+	40, 40, 40, 18, 24, 26, 22, 40, 
 	0
 };
 
 static const char _khmer_syllable_machine_trans_targs[] = {
-	10, 14, 17, 20, 11, 21, 10, 24, 
+	22, 1, 30, 24, 25, 3, 26, 5, 
-	27, 30, 31, 32, 10, 22, 33, 34, 
+	27, 7, 28, 9, 29, 23, 22, 11, 
-	26, 35, 10, 12, 4, 0, 16, 3, 
+	32, 22, 33, 13, 34, 15, 35, 17, 
-	13, 15, 1, 10, 18, 2, 19, 10, 
+	36, 19, 37, 40, 39, 22, 31, 38, 
-	23, 5, 8, 25, 6, 10, 28, 7, 
+	22, 0, 10, 2, 4, 6, 8, 22, 
-	29, 9, 10
+	22, 12, 14, 16, 18, 20, 21
 };
 
 static const char _khmer_syllable_machine_trans_actions[] = {
-	1, 2, 2, 0, 2, 2, 3, 2, 
+	1, 0, 2, 2, 2, 0, 0, 0, 
-	2, 0, 2, 2, 6, 2, 0, 0, 
+	2, 0, 2, 0, 2, 2, 3, 0, 
-	0, 0, 7, 2, 0, 0, 0, 0, 
+	4, 5, 2, 0, 0, 0, 2, 0, 
-	2, 2, 0, 8, 0, 0, 0, 9, 
+	2, 0, 2, 4, 4, 8, 9, 0, 
-	2, 0, 0, 2, 0, 10, 0, 0, 
+	10, 0, 0, 0, 0, 0, 0, 11, 
-	0, 0, 11
+	12, 0, 0, 0, 0, 0, 0
 };
 
 static const char _khmer_syllable_machine_to_state_actions[] = {
 	0, 0, 0, 0, 0, 0, 0, 0, 
-	0, 0, 4, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 6, 0, 
 	0, 0, 0, 0, 0, 0, 0, 0, 
 	0, 0, 0, 0, 0, 0, 0, 0, 
-	0, 0, 0, 0
+	0
 };
 
 static const char _khmer_syllable_machine_from_state_actions[] = {
 	0, 0, 0, 0, 0, 0, 0, 0, 
-	0, 0, 5, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 0, 0, 
+	0, 0, 0, 0, 0, 0, 7, 0, 
 	0, 0, 0, 0, 0, 0, 0, 0, 
 	0, 0, 0, 0, 0, 0, 0, 0, 
-	0, 0, 0, 0
+	0
 };
 
 static const unsigned char _khmer_syllable_machine_eof_trans[] = {
-	1, 1, 1, 1, 1, 7, 7, 7, 
+	1, 1, 1, 1, 1, 1, 1, 1, 
-	7, 7, 0, 19, 19, 19, 19, 19, 
+	1, 1, 1, 15, 18, 18, 18, 18, 
-	19, 19, 19, 19, 19, 19, 32, 32, 
+	18, 18, 18, 18, 18, 18, 0, 33, 
-	32, 32, 32, 32, 32, 32, 32, 32, 
+	33, 33, 33, 33, 33, 33, 33, 40, 
-	32, 43, 43, 43
+	41, 41, 41, 41, 41, 41, 40, 41, 
+	41
 };
 
-static const int khmer_syllable_machine_start = 10;
+static const int khmer_syllable_machine_start = 22;
-static const int khmer_syllable_machine_first_final = 10;
+static const int khmer_syllable_machine_first_final = 22;
 static const int khmer_syllable_machine_error = -1;
 
-static const int khmer_syllable_machine_en_main = 10;
+static const int khmer_syllable_machine_en_main = 22;
 
 
 #line 36 "hb-ot-shape-complex-khmer-machine.rl"
 
 
 
-#line 74 "hb-ot-shape-complex-khmer-machine.rl"
+#line 80 "hb-ot-shape-complex-khmer-machine.rl"
 
 
 #define found_syllable(syllable_type) \
   HB_STMT_START { \
-    if (0) fprintf (stderr, "syllable %d..%d %s\n", last, p+1, #syllable_type); \
+    if (0) fprintf (stderr, "syllable %d..%d %s\n", ts, te, #syllable_type); \
-    for (unsigned int i = last; i < p+1; i++) \
+    for (unsigned int i = ts; i < te; i++) \
       info[i].syllable() = (syllable_serial << 4) | syllable_type; \
-    last = p+1; \
     syllable_serial++; \
     if (unlikely (syllable_serial == 16)) syllable_serial = 1; \
   } HB_STMT_END
@@ -173,11 +242,11 @@ static const int khmer_syllable_machine_en_main = 10;
 static void
 find_syllables (hb_buffer_t *buffer)
 {
-  unsigned int p, pe, eof, ts HB_UNUSED, te, act HB_UNUSED;
+  unsigned int p, pe, eof, ts, te, act HB_UNUSED;
   int cs;
   hb_glyph_info_t *info = buffer->info;
   
-#line 181 "hb-ot-shape-complex-khmer-machine.hh"
+#line 250 "hb-ot-shape-complex-khmer-machine.hh"
 	{
 	cs = khmer_syllable_machine_start;
 	ts = 0;
@@ -185,16 +254,15 @@ find_syllables (hb_buffer_t *buffer)
 	act = 0;
 	}
 
-#line 95 "hb-ot-shape-complex-khmer-machine.rl"
+#line 100 "hb-ot-shape-complex-khmer-machine.rl"
 
 
   p = 0;
   pe = eof = buffer->len;
 
-  unsigned int last = 0;
   unsigned int syllable_serial = 1;
   
-#line 198 "hb-ot-shape-complex-khmer-machine.hh"
+#line 266 "hb-ot-shape-complex-khmer-machine.hh"
 	{
 	int _slen;
 	int _trans;
@@ -204,11 +272,11 @@ find_syllables (hb_buffer_t *buffer)
 		goto _test_eof;
 _resume:
 	switch ( _khmer_syllable_machine_from_state_actions[cs] ) {
-	case 5:
+	case 7:
 #line 1 "NONE"
 	{ts = p;}
 	break;
-#line 212 "hb-ot-shape-complex-khmer-machine.hh"
+#line 280 "hb-ot-shape-complex-khmer-machine.hh"
 	}
 
 	_keys = _khmer_syllable_machine_trans_keys + (cs<<1);
@@ -231,47 +299,63 @@ _eof_trans:
 	{te = p+1;}
 	break;
 	case 8:
-#line 68 "hb-ot-shape-complex-khmer-machine.rl"
+#line 76 "hb-ot-shape-complex-khmer-machine.rl"
-	{te = p+1;{ found_syllable (consonant_syllable); }}
-	break;
-	case 10:
-#line 69 "hb-ot-shape-complex-khmer-machine.rl"
-	{te = p+1;{ found_syllable (broken_cluster); }}
-	break;
-	case 6:
-#line 70 "hb-ot-shape-complex-khmer-machine.rl"
 	{te = p+1;{ found_syllable (non_khmer_cluster); }}
 	break;
-	case 7:
+	case 10:
-#line 68 "hb-ot-shape-complex-khmer-machine.rl"
+#line 74 "hb-ot-shape-complex-khmer-machine.rl"
 	{te = p;p--;{ found_syllable (consonant_syllable); }}
 	break;
-	case 9:
+	case 12:
-#line 69 "hb-ot-shape-complex-khmer-machine.rl"
+#line 75 "hb-ot-shape-complex-khmer-machine.rl"
 	{te = p;p--;{ found_syllable (broken_cluster); }}
 	break;
 	case 11:
-#line 70 "hb-ot-shape-complex-khmer-machine.rl"
+#line 76 "hb-ot-shape-complex-khmer-machine.rl"
 	{te = p;p--;{ found_syllable (non_khmer_cluster); }}
 	break;
 	case 1:
-#line 68 "hb-ot-shape-complex-khmer-machine.rl"
+#line 74 "hb-ot-shape-complex-khmer-machine.rl"
 	{{p = ((te))-1;}{ found_syllable (consonant_syllable); }}
 	break;
-	case 3:
+	case 5:
-#line 69 "hb-ot-shape-complex-khmer-machine.rl"
+#line 75 "hb-ot-shape-complex-khmer-machine.rl"
 	{{p = ((te))-1;}{ found_syllable (broken_cluster); }}
 	break;
-#line 266 "hb-ot-shape-complex-khmer-machine.hh"
+	case 3:
+#line 1 "NONE"
+	{	switch( act ) {
+	case 2:
+	{{p = ((te))-1;} found_syllable (broken_cluster); }
+	break;
+	case 3:
+	{{p = ((te))-1;} found_syllable (non_khmer_cluster); }
+	break;
+	}
+	}
+	break;
+	case 4:
+#line 1 "NONE"
+	{te = p+1;}
+#line 75 "hb-ot-shape-complex-khmer-machine.rl"
+	{act = 2;}
+	break;
+	case 9:
+#line 1 "NONE"
+	{te = p+1;}
+#line 76 "hb-ot-shape-complex-khmer-machine.rl"
+	{act = 3;}
+	break;
+#line 350 "hb-ot-shape-complex-khmer-machine.hh"
 	}
 
 _again:
 	switch ( _khmer_syllable_machine_to_state_actions[cs] ) {
-	case 4:
+	case 6:
 #line 1 "NONE"
 	{ts = 0;}
 	break;
-#line 275 "hb-ot-shape-complex-khmer-machine.hh"
+#line 359 "hb-ot-shape-complex-khmer-machine.hh"
 	}
 
 	if ( ++p != pe )
@@ -287,7 +371,7 @@ _again:
 
 	}
 
-#line 104 "hb-ot-shape-complex-khmer-machine.rl"
+#line 108 "hb-ot-shape-complex-khmer-machine.rl"
 
 }
 

src/hb-ot-shape-complex-khmer-machine.rl

@@ -40,28 +40,34 @@
 # Same order as enum khmer_category_t.  Not sure how to avoid duplication.
 C    = 1;
 V    = 2;
-N    = 3;
 ZWNJ = 5;
 ZWJ  = 6;
-M    = 7;
-SM   = 8;
 PLACEHOLDER = 11;
 DOTTEDCIRCLE = 12;
-RS    = 13;
+Coeng= 14;
-Coeng = 14;
+Ra   = 16;
-Ra    = 16;
+Robatic = 20;
+Xgroup  = 21;
+Ygroup  = 22;
+VAbv = 26;
+VBlw = 27;
+VPre = 28;
+VPst = 29;
 
-c = (C | Ra | V);		# is_consonant
+c = (C | Ra | V);
-n = ((ZWNJ?.RS)? (N.N?)?);	# is_consonant_modifier
+cn = c.((ZWJ|ZWNJ)?.Robatic)?;
-z = ZWJ|ZWNJ;			# is_joiner
+joiner = (ZWJ | ZWNJ);
+xgroup = (joiner*.Xgroup)*;
+ygroup = Ygroup*;
 
-cn = c.n?;
+# This grammar was experimentally extracted from what Uniscribe allows.
-matra_group = z?.M.N?;
+
-syllable_tail = (SM.SM?)?;
+matra_group = VPre? xgroup VBlw? xgroup (joiner?.VAbv)? xgroup VPst?;
+syllable_tail = xgroup matra_group xgroup (Coeng.c)? ygroup;
 
 
-broken_cluster =	n? (Coeng.cn)* matra_group* (Coeng.cn)? syllable_tail;
+broken_cluster =	(Coeng.cn)* syllable_tail;
-consonant_syllable =	(c|PLACEHOLDER|DOTTEDCIRCLE) broken_cluster;
+consonant_syllable =	(cn|PLACEHOLDER|DOTTEDCIRCLE) broken_cluster;
 other =			any;
 
 main := |*
@@ -75,10 +81,9 @@ main := |*
 
 #define found_syllable(syllable_type) \
   HB_STMT_START { \
-    if (0) fprintf (stderr, "syllable %d..%d %s\n", last, p+1, #syllable_type); \
+    if (0) fprintf (stderr, "syllable %d..%d %s\n", ts, te, #syllable_type); \
-    for (unsigned int i = last; i < p+1; i++) \
+    for (unsigned int i = ts; i < te; i++) \
       info[i].syllable() = (syllable_serial << 4) | syllable_type; \
-    last = p+1; \
     syllable_serial++; \
     if (unlikely (syllable_serial == 16)) syllable_serial = 1; \
   } HB_STMT_END
@@ -86,7 +91,7 @@ main := |*
 static void
 find_syllables (hb_buffer_t *buffer)
 {
-  unsigned int p, pe, eof, ts HB_UNUSED, te, act HB_UNUSED;
+  unsigned int p, pe, eof, ts, te, act HB_UNUSED;
   int cs;
   hb_glyph_info_t *info = buffer->info;
   %%{
@@ -97,7 +102,6 @@ find_syllables (hb_buffer_t *buffer)
   p = 0;
   pe = eof = buffer->len;
 
-  unsigned int last = 0;
   unsigned int syllable_serial = 1;
   %%{
     write exec;

src/hb-ot-shape-complex-khmer.cc

@@ -241,7 +241,6 @@ setup_masks_khmer (const hb_ot_shape_plan_t *plan HB_UNUSED,
 		   hb_font_t                *font HB_UNUSED)
 {
   HB_BUFFER_ALLOCATE_VAR (buffer, khmer_category);
-  HB_BUFFER_ALLOCATE_VAR (buffer, khmer_position);
 
   /* We cannot setup masks here.  We save information about characters
    * and setup masks later on in a pause-callback. */
@@ -330,7 +329,7 @@ reorder_consonant_syllable (const hb_ot_shape_plan_t *plan,
     }
 
     /* Reorder left matra piece. */
-    else if (info[i].khmer_position() == POS_PRE_M)
+    else if (info[i].khmer_category() == OT_VPre)
     {
       /* Move to the start. */
       buffer->merge_clusters (start, i + 1);
@@ -432,7 +431,6 @@ reorder (const hb_ot_shape_plan_t *plan,
     initial_reordering_syllable (plan, font->face, buffer, start, end);
 
   HB_BUFFER_DEALLOCATE_VAR (buffer, khmer_category);
-  HB_BUFFER_DEALLOCATE_VAR (buffer, khmer_position);
 }
 
 static void
@@ -498,7 +496,7 @@ const hb_ot_complex_shaper_t _hb_ot_complex_shaper_khmer =
   decompose_khmer,
   compose_khmer,
   setup_masks_khmer,
-  nullptr, /* disable_otl */
+  HB_TAG_NONE, /* gpos_tag */
   nullptr, /* reorder_marks */
   HB_OT_SHAPE_ZERO_WIDTH_MARKS_NONE,
   false, /* fallback_position */

src/hb-ot-shape-complex-khmer.hh

@@ -34,42 +34,21 @@
 
 /* buffer var allocations */
 #define khmer_category() indic_category() /* khmer_category_t */
-#define khmer_position() indic_position() /* khmer_position_t */
 
 
-typedef indic_category_t khmer_category_t;
+/* Note: This enum is duplicated in the -machine.rl source file.
-typedef indic_position_t khmer_position_t;
+ * Not sure how to avoid duplication. */
-
+enum khmer_category_t
-
-static inline khmer_position_t
-matra_position_khmer (khmer_position_t side)
 {
-  switch ((int) side)
+  OT_Robatic = 20,
-  {
+  OT_Xgroup  = 21,
-    case POS_PRE_C:
+  OT_Ygroup  = 22,
-      return POS_PRE_M;
 
-    case POS_POST_C:
+  OT_VAbv    = 26,
-    case POS_ABOVE_C:
+  OT_VBlw    = 27,
-    case POS_BELOW_C:
+  OT_VPre    = 28,
-      return POS_AFTER_POST;
+  OT_VPst    = 29,
-
+};
-    default:
-      return side;
-  };
-}
-
-static inline bool
-is_consonant_or_vowel (const hb_glyph_info_t &info)
-{
-  return is_one_of (info, CONSONANT_FLAGS | FLAG (OT_V));
-}
-
-static inline bool
-is_coeng (const hb_glyph_info_t &info)
-{
-  return is_one_of (info, FLAG (OT_Coeng));
-}
 
 static inline void
 set_khmer_properties (hb_glyph_info_t &info)
@@ -77,47 +56,58 @@ set_khmer_properties (hb_glyph_info_t &info)
   hb_codepoint_t u = info.codepoint;
   unsigned int type = hb_indic_get_categories (u);
   khmer_category_t cat = (khmer_category_t) (type & 0x7Fu);
-  khmer_position_t pos = (khmer_position_t) (type >> 8);
+  indic_position_t pos = (indic_position_t) (type >> 8);
 
 
   /*
    * Re-assign category
+   *
+   * These categories are experimentally extracted from what Uniscribe allows.
    */
-
+  switch (u)
-  if (unlikely (u == 0x17C6u)) cat = OT_N; /* Khmer Bindu doesn't like to be repositioned. */
-  else if (unlikely (hb_in_range<hb_codepoint_t> (u, 0x17CDu, 0x17D1u) ||
-		     u == 0x17CBu || u == 0x17D3u || u == 0x17DDu)) /* Khmer Various signs */
   {
-    /* These can occur mid-syllable (eg. before matras), even though Unicode marks them as Syllable_Modifier.
+    case 0x179Au:
-     * https://github.com/roozbehp/unicode-data/issues/5 */
+      cat = (khmer_category_t) OT_Ra;
-    cat = OT_M;
+      break;
-    pos = POS_ABOVE_C;
-  }
-  else if (unlikely (hb_in_range<hb_codepoint_t> (u, 0x2010u, 0x2011u))) cat = OT_PLACEHOLDER;
-  else if (unlikely (u == 0x25CCu)) cat = OT_DOTTEDCIRCLE;
 
+    case 0x17CCu:
+    case 0x17C9u:
+    case 0x17CAu:
+      cat = OT_Robatic;
+      break;
+
+    case 0x17C6u:
+    case 0x17CBu:
+    case 0x17CDu:
+    case 0x17CEu:
+    case 0x17CFu:
+    case 0x17D0u:
+    case 0x17D1u:
+      cat = OT_Xgroup;
+      break;
+
+    case 0x17C7u:
+    case 0x17C8u:
+    case 0x17DDu:
+    case 0x17D3u: /* Just guessing. Uniscribe doesn't categorize it. */
+      cat = OT_Ygroup;
+      break;
+  }
 
   /*
    * Re-assign position.
    */
-
+  if (cat == (khmer_category_t) OT_M)
-  if ((FLAG_UNSAFE (cat) & CONSONANT_FLAGS))
+    switch ((int) pos)
-  {
+    {
-    pos = POS_BASE_C;
+      case POS_PRE_C:	cat = OT_VPre; break;
-    if (u == 0x179Au)
+      case POS_BELOW_C:	cat = OT_VBlw; break;
-      cat = OT_Ra;
+      case POS_ABOVE_C:	cat = OT_VAbv; break;
-  }
+      case POS_POST_C:	cat = OT_VPst; break;
-  else if (cat == OT_M)
+      default: assert (0);
-  {
+    };
-    pos = matra_position_khmer (pos);
-  }
-  else if ((FLAG_UNSAFE (cat) & (FLAG (OT_SM) | FLAG (OT_A) | FLAG (OT_Symbol))))
-  {
-    pos = POS_SMVD;
-  }
 
   info.khmer_category() = cat;
-  info.khmer_position() = pos;
 }
 
 

src/hb-ot-shape-complex-myanmar-machine.hh

@@ -283,10 +283,9 @@ static const int myanmar_syllable_machine_en_main = 0;
 
 #define found_syllable(syllable_type) \
   HB_STMT_START { \
-    if (0) fprintf (stderr, "syllable %d..%d %s\n", last, p+1, #syllable_type); \
+    if (0) fprintf (stderr, "syllable %d..%d %s\n", ts, te, #syllable_type); \
-    for (unsigned int i = last; i < p+1; i++) \
+    for (unsigned int i = ts; i < te; i++) \
       info[i].syllable() = (syllable_serial << 4) | syllable_type; \
-    last = p+1; \
     syllable_serial++; \
     if (unlikely (syllable_serial == 16)) syllable_serial = 1; \
   } HB_STMT_END
@@ -294,11 +293,11 @@ static const int myanmar_syllable_machine_en_main = 0;
 static void
 find_syllables (hb_buffer_t *buffer)
 {
-  unsigned int p, pe, eof, ts HB_UNUSED, te HB_UNUSED, act HB_UNUSED;
+  unsigned int p, pe, eof, ts, te, act HB_UNUSED;
   int cs;
   hb_glyph_info_t *info = buffer->info;
   
-#line 302 "hb-ot-shape-complex-myanmar-machine.hh"
+#line 301 "hb-ot-shape-complex-myanmar-machine.hh"
 	{
 	cs = myanmar_syllable_machine_start;
 	ts = 0;
@@ -306,16 +305,15 @@ find_syllables (hb_buffer_t *buffer)
 	act = 0;
 	}
 
-#line 115 "hb-ot-shape-complex-myanmar-machine.rl"
+#line 114 "hb-ot-shape-complex-myanmar-machine.rl"
 
 
   p = 0;
   pe = eof = buffer->len;
 
-  unsigned int last = 0;
   unsigned int syllable_serial = 1;
   
-#line 319 "hb-ot-shape-complex-myanmar-machine.hh"
+#line 317 "hb-ot-shape-complex-myanmar-machine.hh"
 	{
 	int _slen;
 	int _trans;
@@ -329,7 +327,7 @@ _resume:
 #line 1 "NONE"
 	{ts = p;}
 	break;
-#line 333 "hb-ot-shape-complex-myanmar-machine.hh"
+#line 331 "hb-ot-shape-complex-myanmar-machine.hh"
 	}
 
 	_keys = _myanmar_syllable_machine_trans_keys + (cs<<1);
@@ -379,7 +377,7 @@ _eof_trans:
 #line 90 "hb-ot-shape-complex-myanmar-machine.rl"
 	{te = p;p--;{ found_syllable (non_myanmar_cluster); }}
 	break;
-#line 383 "hb-ot-shape-complex-myanmar-machine.hh"
+#line 381 "hb-ot-shape-complex-myanmar-machine.hh"
 	}
 
 _again:
@@ -388,7 +386,7 @@ _again:
 #line 1 "NONE"
 	{ts = 0;}
 	break;
-#line 392 "hb-ot-shape-complex-myanmar-machine.hh"
+#line 390 "hb-ot-shape-complex-myanmar-machine.hh"
 	}
 
 	if ( ++p != pe )
@@ -404,7 +402,7 @@ _again:
 
 	}
 
-#line 124 "hb-ot-shape-complex-myanmar-machine.rl"
+#line 122 "hb-ot-shape-complex-myanmar-machine.rl"
 
 }
 

src/hb-ot-shape-complex-myanmar-machine.rl

@@ -95,10 +95,9 @@ main := |*
 
 #define found_syllable(syllable_type) \
   HB_STMT_START { \
-    if (0) fprintf (stderr, "syllable %d..%d %s\n", last, p+1, #syllable_type); \
+    if (0) fprintf (stderr, "syllable %d..%d %s\n", ts, te, #syllable_type); \
-    for (unsigned int i = last; i < p+1; i++) \
+    for (unsigned int i = ts; i < te; i++) \
       info[i].syllable() = (syllable_serial << 4) | syllable_type; \
-    last = p+1; \
     syllable_serial++; \
     if (unlikely (syllable_serial == 16)) syllable_serial = 1; \
   } HB_STMT_END
@@ -106,7 +105,7 @@ main := |*
 static void
 find_syllables (hb_buffer_t *buffer)
 {
-  unsigned int p, pe, eof, ts HB_UNUSED, te HB_UNUSED, act HB_UNUSED;
+  unsigned int p, pe, eof, ts, te, act HB_UNUSED;
   int cs;
   hb_glyph_info_t *info = buffer->info;
   %%{
@@ -117,7 +116,6 @@ find_syllables (hb_buffer_t *buffer)
   p = 0;
   pe = eof = buffer->len;
 
-  unsigned int last = 0;
   unsigned int syllable_serial = 1;
   %%{
     write exec;

src/hb-ot-shape-complex-myanmar.cc

@@ -106,14 +106,14 @@ collect_features_myanmar (hb_ot_shape_planner_t *plan)
 
   for (unsigned int i = 0; i < ARRAY_LENGTH (basic_features); i++)
   {
-    map->add_feature (basic_features[i], F_GLOBAL | F_MANUAL_ZWJ);
+    map->enable_feature (basic_features[i], F_MANUAL_ZWJ);
     map->add_gsub_pause (nullptr);
   }
 
   map->add_gsub_pause (final_reordering);
 
   for (unsigned int i = 0; i < ARRAY_LENGTH (other_features); i++)
-    map->add_feature (other_features[i], F_GLOBAL | F_MANUAL_ZWJ);
+    map->enable_feature (other_features[i], F_MANUAL_ZWJ);
 
   for (unsigned int i = 0; i < ARRAY_LENGTH (positioning_features); i++)
     map->enable_feature (positioning_features[i]);
@@ -389,7 +389,7 @@ const hb_ot_complex_shaper_t _hb_ot_complex_shaper_myanmar_old =
   nullptr, /* decompose */
   nullptr, /* compose */
   nullptr, /* setup_masks */
-  nullptr, /* disable_otl */
+  HB_TAG_NONE, /* gpos_tag */
   nullptr, /* reorder_marks */
   HB_OT_SHAPE_ZERO_WIDTH_MARKS_BY_GDEF_LATE,
   true, /* fallback_position */
@@ -407,7 +407,7 @@ const hb_ot_complex_shaper_t _hb_ot_complex_shaper_myanmar =
   nullptr, /* decompose */
   nullptr, /* compose */
   setup_masks_myanmar,
-  nullptr, /* disable_otl */
+  HB_TAG_NONE, /* gpos_tag */
   nullptr, /* reorder_marks */
   HB_OT_SHAPE_ZERO_WIDTH_MARKS_BY_GDEF_EARLY,
   false, /* fallback_position */

src/hb-ot-shape-complex-thai.cc

@@ -376,7 +376,7 @@ const hb_ot_complex_shaper_t _hb_ot_complex_shaper_thai =
   nullptr, /* decompose */
   nullptr, /* compose */
   nullptr, /* setup_masks */
-  nullptr, /* disable_otl */
+  HB_TAG_NONE, /* gpos_tag */
   nullptr, /* reorder_marks */
   HB_OT_SHAPE_ZERO_WIDTH_MARKS_BY_GDEF_LATE,
   false,/* fallback_position */

src/hb-ot-shape-complex-tibetan.cc

@@ -1,63 +0,0 @@
-/*
- * Copyright © 2010,2012  Google, Inc.
- *
- *  This is part of HarfBuzz, a text shaping library.
- *
- * Permission is hereby granted, without written agreement and without
- * license or royalty fees, to use, copy, modify, and distribute this
- * software and its documentation for any purpose, provided that the
- * above copyright notice and the following two paragraphs appear in
- * all copies of this software.
- *
- * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
- * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
- * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
- * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- *
- * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
- * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
- * FITNESS FOR A PARTICULAR PURPOSE.  THE SOFTWARE PROVIDED HEREUNDER IS
- * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
- * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
- *
- * Google Author(s): Behdad Esfahbod
- */
-
-#include "hb-ot-shape-complex.hh"
-
-
-static const hb_tag_t tibetan_features[] =
-{
-  HB_TAG('a','b','v','s'),
-  HB_TAG('b','l','w','s'),
-  HB_TAG('a','b','v','m'),
-  HB_TAG('b','l','w','m'),
-  HB_TAG_NONE
-};
-
-static void
-collect_features_tibetan (hb_ot_shape_planner_t *plan)
-{
-  for (const hb_tag_t *script_features = tibetan_features; script_features && *script_features; script_features++)
-    plan->map.enable_feature (*script_features);
-}
-
-
-const hb_ot_complex_shaper_t _hb_ot_complex_shaper_tibetan =
-{
-  collect_features_tibetan,
-  nullptr, /* override_features */
-  nullptr, /* data_create */
-  nullptr, /* data_destroy */
-  nullptr, /* preprocess_text */
-  nullptr, /* postprocess_glyphs */
-  HB_OT_SHAPE_NORMALIZATION_MODE_DEFAULT,
-  nullptr, /* decompose */
-  nullptr, /* compose */
-  nullptr, /* setup_masks */
-  nullptr, /* disable_otl */
-  nullptr, /* reorder_marks */
-  HB_OT_SHAPE_ZERO_WIDTH_MARKS_BY_GDEF_LATE,
-  true, /* fallback_position */
-};

src/hb-ot-shape-complex-use-machine.hh

@@ -315,15 +315,14 @@ static const int use_syllable_machine_en_main = 4;
 
 
 
-#line 141 "hb-ot-shape-complex-use-machine.rl"
+#line 140 "hb-ot-shape-complex-use-machine.rl"
 
 
 #define found_syllable(syllable_type) \
   HB_STMT_START { \
-    if (0) fprintf (stderr, "syllable %d..%d %s\n", last, p+1, #syllable_type); \
+    if (0) fprintf (stderr, "syllable %d..%d %s\n", ts, te, #syllable_type); \
-    for (unsigned int i = last; i < p+1; i++) \
+    for (unsigned int i = ts; i < te; i++) \
       info[i].syllable() = (syllable_serial << 4) | syllable_type; \
-    last = p+1; \
     syllable_serial++; \
     if (unlikely (syllable_serial == 16)) syllable_serial = 1; \
   } HB_STMT_END
@@ -331,11 +330,11 @@ static const int use_syllable_machine_en_main = 4;
 static void
 find_syllables (hb_buffer_t *buffer)
 {
-  unsigned int p, pe, eof, ts HB_UNUSED, te, act;
+  unsigned int p, pe, eof, ts, te, act;
   int cs;
   hb_glyph_info_t *info = buffer->info;
   
-#line 339 "hb-ot-shape-complex-use-machine.hh"
+#line 338 "hb-ot-shape-complex-use-machine.hh"
 	{
 	cs = use_syllable_machine_start;
 	ts = 0;
@@ -343,16 +342,15 @@ find_syllables (hb_buffer_t *buffer)
 	act = 0;
 	}
 
-#line 162 "hb-ot-shape-complex-use-machine.rl"
+#line 160 "hb-ot-shape-complex-use-machine.rl"
 
 
   p = 0;
   pe = eof = buffer->len;
 
-  unsigned int last = 0;
   unsigned int syllable_serial = 1;
   
-#line 356 "hb-ot-shape-complex-use-machine.hh"
+#line 354 "hb-ot-shape-complex-use-machine.hh"
 	{
 	int _slen;
 	int _trans;
@@ -366,7 +364,7 @@ _resume:
 #line 1 "NONE"
 	{ts = p;}
 	break;
-#line 370 "hb-ot-shape-complex-use-machine.hh"
+#line 368 "hb-ot-shape-complex-use-machine.hh"
 	}
 
 	_keys = _use_syllable_machine_trans_keys + (cs<<1);
@@ -389,59 +387,59 @@ _eof_trans:
 	{te = p+1;}
 	break;
 	case 12:
-#line 130 "hb-ot-shape-complex-use-machine.rl"
+#line 129 "hb-ot-shape-complex-use-machine.rl"
 	{te = p+1;{ found_syllable (independent_cluster); }}
 	break;
 	case 14:
-#line 132 "hb-ot-shape-complex-use-machine.rl"
+#line 131 "hb-ot-shape-complex-use-machine.rl"
 	{te = p+1;{ found_syllable (standard_cluster); }}
 	break;
 	case 9:
-#line 136 "hb-ot-shape-complex-use-machine.rl"
+#line 135 "hb-ot-shape-complex-use-machine.rl"
 	{te = p+1;{ found_syllable (broken_cluster); }}
 	break;
 	case 8:
-#line 137 "hb-ot-shape-complex-use-machine.rl"
+#line 136 "hb-ot-shape-complex-use-machine.rl"
 	{te = p+1;{ found_syllable (non_cluster); }}
 	break;
 	case 11:
-#line 130 "hb-ot-shape-complex-use-machine.rl"
+#line 129 "hb-ot-shape-complex-use-machine.rl"
 	{te = p;p--;{ found_syllable (independent_cluster); }}
 	break;
 	case 15:
-#line 131 "hb-ot-shape-complex-use-machine.rl"
+#line 130 "hb-ot-shape-complex-use-machine.rl"
 	{te = p;p--;{ found_syllable (virama_terminated_cluster); }}
 	break;
 	case 13:
-#line 132 "hb-ot-shape-complex-use-machine.rl"
+#line 131 "hb-ot-shape-complex-use-machine.rl"
 	{te = p;p--;{ found_syllable (standard_cluster); }}
 	break;
 	case 17:
-#line 133 "hb-ot-shape-complex-use-machine.rl"
+#line 132 "hb-ot-shape-complex-use-machine.rl"
 	{te = p;p--;{ found_syllable (number_joiner_terminated_cluster); }}
 	break;
 	case 16:
-#line 134 "hb-ot-shape-complex-use-machine.rl"
+#line 133 "hb-ot-shape-complex-use-machine.rl"
 	{te = p;p--;{ found_syllable (numeral_cluster); }}
 	break;
 	case 20:
-#line 135 "hb-ot-shape-complex-use-machine.rl"
+#line 134 "hb-ot-shape-complex-use-machine.rl"
 	{te = p;p--;{ found_syllable (symbol_cluster); }}
 	break;
 	case 18:
-#line 136 "hb-ot-shape-complex-use-machine.rl"
+#line 135 "hb-ot-shape-complex-use-machine.rl"
 	{te = p;p--;{ found_syllable (broken_cluster); }}
 	break;
 	case 19:
-#line 137 "hb-ot-shape-complex-use-machine.rl"
+#line 136 "hb-ot-shape-complex-use-machine.rl"
 	{te = p;p--;{ found_syllable (non_cluster); }}
 	break;
 	case 1:
-#line 132 "hb-ot-shape-complex-use-machine.rl"
+#line 131 "hb-ot-shape-complex-use-machine.rl"
 	{{p = ((te))-1;}{ found_syllable (standard_cluster); }}
 	break;
 	case 4:
-#line 136 "hb-ot-shape-complex-use-machine.rl"
+#line 135 "hb-ot-shape-complex-use-machine.rl"
 	{{p = ((te))-1;}{ found_syllable (broken_cluster); }}
 	break;
 	case 2:
@@ -459,16 +457,16 @@ _eof_trans:
 	case 3:
 #line 1 "NONE"
 	{te = p+1;}
-#line 136 "hb-ot-shape-complex-use-machine.rl"
+#line 135 "hb-ot-shape-complex-use-machine.rl"
 	{act = 7;}
 	break;
 	case 10:
 #line 1 "NONE"
 	{te = p+1;}
-#line 137 "hb-ot-shape-complex-use-machine.rl"
+#line 136 "hb-ot-shape-complex-use-machine.rl"
 	{act = 8;}
 	break;
-#line 472 "hb-ot-shape-complex-use-machine.hh"
+#line 470 "hb-ot-shape-complex-use-machine.hh"
 	}
 
 _again:
@@ -477,7 +475,7 @@ _again:
 #line 1 "NONE"
 	{ts = 0;}
 	break;
-#line 481 "hb-ot-shape-complex-use-machine.hh"
+#line 479 "hb-ot-shape-complex-use-machine.hh"
 	}
 
 	if ( ++p != pe )
@@ -493,7 +491,7 @@ _again:
 
 	}
 
-#line 171 "hb-ot-shape-complex-use-machine.rl"
+#line 168 "hb-ot-shape-complex-use-machine.rl"
 
 }
 

src/hb-ot-shape-complex-use-machine.rl

@@ -97,13 +97,7 @@ dependent_vowels = VPre* VAbv* VBlw* VPst*;
 vowel_modifiers = VMPre* VMAbv* VMBlw* VMPst*;
 final_consonants = FAbv* FBlw* FPst* FM?;
 
-virama_terminated_cluster =
+complex_syllable_tail =
-	(R|CS)? (B | GB) VS?
-	consonant_modifiers
-	ZWJ?.H.ZWJ?
-;
-standard_cluster =
-	(R|CS)? (B | GB) VS?
 	consonant_modifiers
 	medial_consonants
 	dependent_vowels
@@ -111,13 +105,18 @@ standard_cluster =
 	final_consonants
 ;
 
+virama_terminated_cluster =
+	(R|CS)? (B | GB) VS?
+	consonant_modifiers
+	ZWJ?.H.ZWJ?
+;
+standard_cluster =
+	(R|CS)? (B | GB) VS?
+	complex_syllable_tail
+;
 broken_cluster =
 	R?
-	consonant_modifiers
+	complex_syllable_tail
-	medial_consonants
-	dependent_vowels
-	vowel_modifiers
-	final_consonants
 ;
 
 number_joiner_terminated_cluster = N VS? (HN N VS?)* HN;
@@ -142,10 +141,9 @@ main := |*
 
 #define found_syllable(syllable_type) \
   HB_STMT_START { \
-    if (0) fprintf (stderr, "syllable %d..%d %s\n", last, p+1, #syllable_type); \
+    if (0) fprintf (stderr, "syllable %d..%d %s\n", ts, te, #syllable_type); \
-    for (unsigned int i = last; i < p+1; i++) \
+    for (unsigned int i = ts; i < te; i++) \
       info[i].syllable() = (syllable_serial << 4) | syllable_type; \
-    last = p+1; \
     syllable_serial++; \
     if (unlikely (syllable_serial == 16)) syllable_serial = 1; \
   } HB_STMT_END
@@ -153,7 +151,7 @@ main := |*
 static void
 find_syllables (hb_buffer_t *buffer)
 {
-  unsigned int p, pe, eof, ts HB_UNUSED, te, act;
+  unsigned int p, pe, eof, ts, te, act;
   int cs;
   hb_glyph_info_t *info = buffer->info;
   %%{
@@ -164,7 +162,6 @@ find_syllables (hb_buffer_t *buffer)
   p = 0;
   pe = eof = buffer->len;
 
-  unsigned int last = 0;
   unsigned int syllable_serial = 1;
   %%{
     write exec;

src/hb-ot-shape-complex-use-table.cc

@@ -194,7 +194,24 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 0DE0 */     O,     O,     O,     O,     O,     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
   /* 0DF0 */     O,     O,  VPst,  VPst,     O,     O,     O,     O,
 
-#define use_offset_0x1000u 1360
+#define use_offset_0x0f18u 1360
+
+
+  /* Tibetan */
+                                                                      VBlw,  VBlw,     O,     O,     O,     O,     O,     O,
+  /* 0F20 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0F30 */     B,     B,     B,     B,     O,    FM,     O,    FM,     O, CMAbv,     O,     O,     O,     O,  VPst,  VPre,
+  /* 0F40 */     B,     B,     B,     B,     B,     B,     B,     B,     O,     B,     B,     B,     B,     B,     B,     B,
+  /* 0F50 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 0F60 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,
+  /* 0F70 */     O,  VBlw,  VBlw,  VAbv,  VBlw,  VBlw,  VAbv,  VAbv,  VAbv,  VAbv,  VBlw,  VBlw,  VBlw,  VBlw, VMAbv, VMPst,
+  /* 0F80 */  VBlw,  VAbv, VMAbv, VMAbv,  VBlw,   IND, VMAbv, VMAbv,     B,     B,     B,     B,     B,   SUB,   SUB,   SUB,
+  /* 0F90 */   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,     O,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,
+  /* 0FA0 */   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,
+  /* 0FB0 */   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,     O,     O,     O,
+  /* 0FC0 */     O,     O,     O,     O,     O,     O,    FM,     O,
+
+#define use_offset_0x1000u 1536
 
 
   /* Myanmar */
@@ -210,7 +227,7 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 1080 */     B,     B,  MBlw,  VPst,  VPre,  VAbv,  VAbv, VMPst, VMPst, VMPst, VMPst, VMPst, VMPst, VMBlw,     B, VMPst,
   /* 1090 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B, VMPst, VMPst,  VPst,  VAbv,     O,     O,
 
-#define use_offset_0x1700u 1520
+#define use_offset_0x1700u 1696
 
 
   /* Tagalog */
@@ -243,7 +260,7 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 17D0 */    FM,  VAbv,     H,    FM,     O,     O,     O,     O,     O,     O,     O,     O,     B,  VAbv,     O,     O,
   /* 17E0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
 
-#define use_offset_0x1900u 1760
+#define use_offset_0x1900u 1936
 
 
   /* Limbu */
@@ -287,7 +304,7 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 1A80 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
   /* 1A90 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
 
-#define use_offset_0x1b00u 2176
+#define use_offset_0x1b00u 2352
 
 
   /* Balinese */
@@ -323,7 +340,7 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 1C30 */  FAbv,  FAbv,  FAbv,  FAbv, VMPre, VMPre,    FM, CMBlw,     O,     O,     O,     O,     O,     O,     O,     O,
   /* 1C40 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     B,     B,     B,
 
-#define use_offset_0x1cd0u 2512
+#define use_offset_0x1cd0u 2688
 
 
   /* Vedic Extensions */
@@ -332,20 +349,20 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 1CE0 */ VMAbv, VMPst, VMBlw, VMBlw, VMBlw, VMBlw, VMBlw, VMBlw, VMBlw,     O,     O,     O,     O, VMBlw,     O,     O,
   /* 1CF0 */     O,     O, VMPst, VMPst, VMAbv,    CS,    CS, VMPst, VMAbv, VMAbv,     O,     O,     O,     O,     O,     O,
 
-#define use_offset_0x1df8u 2560
+#define use_offset_0x1df8u 2736
 
 
   /* Combining Diacritical Marks Supplement */
                                                                          O,     O,     O,    FM,     O,     O,     O,     O,
 
-#define use_offset_0x2008u 2568
+#define use_offset_0x2008u 2744
 
 
   /* General Punctuation */
                                                                          O,     O,     O,     O,  ZWNJ,   ZWJ,     O,     O,
   /* 2010 */    GB,    GB,    GB,    GB,    GB,     O,     O,     O,
 
-#define use_offset_0x2060u 2584
+#define use_offset_0x2060u 2760
 
   /* 2060 */    WJ,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
 
@@ -354,20 +371,20 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 2070 */     O,     O,     O,     O,    FM,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
   /* 2080 */     O,     O,    FM,    FM,    FM,     O,     O,     O,
 
-#define use_offset_0x20f0u 2624
+#define use_offset_0x20f0u 2800
 
 
   /* Combining Diacritical Marks for Symbols */
 
   /* 20F0 */ VMAbv,     O,     O,     O,     O,     O,     O,     O,
 
-#define use_offset_0x25c8u 2632
+#define use_offset_0x25c8u 2808
 
 
   /* Geometric Shapes */
                                                                          O,     O,     O,     O,    GB,     O,     O,     O,
 
-#define use_offset_0xa800u 2640
+#define use_offset_0xa800u 2816
 
 
   /* Syloti Nagri */
@@ -454,7 +471,7 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* AAE0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,  VPre,  VBlw,  VAbv,  VPre,  VPst,
   /* AAF0 */     O,     O,     O,     O,     O, VMPst,     H,     O,
 
-#define use_offset_0xabc0u 3400
+#define use_offset_0xabc0u 3576
 
 
   /* Meetei Mayek */
@@ -464,14 +481,14 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* ABE0 */     B,     B,     B,  VPst,  VPst,  VAbv,  VPst,  VPst,  VBlw,  VPst,  VPst,     O, VMPst,  VBlw,     O,     O,
   /* ABF0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
 
-#define use_offset_0xfe00u 3464
+#define use_offset_0xfe00u 3640
 
 
   /* Variation Selectors */
 
   /* FE00 */    VS,    VS,    VS,    VS,    VS,    VS,    VS,    VS,    VS,    VS,    VS,    VS,    VS,    VS,    VS,    VS,
 
-#define use_offset_0x10a00u 3480
+#define use_offset_0x10a00u 3656
 
 
   /* Kharoshthi */
@@ -482,7 +499,7 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 10A30 */     B,     B,     B,     B,     B,     B,     O,     O, CMAbv, CMBlw, CMBlw,     O,     O,     O,     O,     H,
   /* 10A40 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,     O,
 
-#define use_offset_0x11000u 3560
+#define use_offset_0x11000u 3736
 
 
   /* Brahmi */
@@ -503,7 +520,7 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 110A0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
   /* 110B0 */  VPst,  VPre,  VPst,  VBlw,  VBlw,  VAbv,  VAbv,  VPst,  VPst,     H, CMBlw,     O,     O,     O,     O,     O,
 
-#define use_offset_0x11100u 3752
+#define use_offset_0x11100u 3928
 
 
   /* Chakma */
@@ -511,7 +528,7 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 11100 */ VMAbv, VMAbv, VMAbv,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
   /* 11110 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
   /* 11120 */     B,     B,     B,     B,     B,     B,     B,  VBlw,  VBlw,  VBlw,  VAbv,  VAbv,  VPre,  VBlw,  VAbv,  VAbv,
-  /* 11130 */  VBlw,  VAbv,  VAbv,     H, CMAbv,     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
+  /* 11130 */  VBlw,  VAbv,  VAbv,     H, CMBlw,     O,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
   /* 11140 */     O,     O,     O,     O,     B,  VPst,  VPst,     O,     O,     O,     O,     O,     O,     O,     O,     O,
 
   /* Mahajani */
@@ -541,7 +558,7 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 11220 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,  VPst,  VPst,  VPst,  VBlw,
   /* 11230 */  VAbv,  VAbv,  VAbv,  VAbv, VMAbv,     H, CMAbv, CMAbv,     O,     O,     O,     O,     O,     O, VMAbv,     O,
 
-#define use_offset_0x11280u 4072
+#define use_offset_0x11280u 4248
 
 
   /* Multani */
@@ -569,7 +586,7 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 11360 */     B,     B,  VPst,  VPst,     O,     O, VMAbv, VMAbv, VMAbv, VMAbv, VMAbv, VMAbv, VMAbv,     O,     O,     O,
   /* 11370 */ VMAbv, VMAbv, VMAbv, VMAbv, VMAbv,     O,     O,     O,
 
-#define use_offset_0x11400u 4320
+#define use_offset_0x11400u 4496
 
 
   /* Newa */
@@ -592,7 +609,7 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 114C0 */ VMAbv, VMPst,     H, CMBlw,     B,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,     O,
   /* 114D0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
 
-#define use_offset_0x11580u 4544
+#define use_offset_0x11580u 4720
 
 
   /* Siddham */
@@ -635,7 +652,7 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 11720 */  VPst,  VPst,  VAbv,  VAbv,  VBlw,  VBlw,  VPre,  VAbv,  VBlw,  VAbv,  VAbv,  VAbv,     O,     O,     O,     O,
   /* 11730 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,
 
-#define use_offset_0x11800u 4992
+#define use_offset_0x11800u 5168
 
 
   /* Dogra */
@@ -645,7 +662,7 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 11820 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,  VPst,  VPre,  VPst,  VBlw,
   /* 11830 */  VBlw,  VBlw,  VBlw,  VAbv,  VAbv,  VAbv,  VAbv, VMAbv, VMPst,     H, CMBlw,     O,     O,     O,     O,     O,
 
-#define use_offset_0x11a00u 5056
+#define use_offset_0x11a00u 5232
 
 
   /* Zanabazar Square */
@@ -664,7 +681,7 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 11A80 */     B,     B,     B,     B,     O,     O,     R,     R,     R,     R,  FBlw,  FBlw,  FBlw,  FBlw,  FBlw,  FBlw,
   /* 11A90 */  FBlw,  FBlw,  FBlw,  FBlw,  FBlw,  FBlw, VMAbv, VMPst, CMAbv,     H,     O,     O,     O,     B,     O,     O,
 
-#define use_offset_0x11c00u 5216
+#define use_offset_0x11c00u 5392
 
 
   /* Bhaiksuki */
@@ -685,7 +702,7 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 11CA0 */   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,     O,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,   SUB,
   /* 11CB0 */  VBlw,  VPre,  VBlw,  VAbv,  VPst, VMAbv, VMAbv,     O,
 
-#define use_offset_0x11d00u 5400
+#define use_offset_0x11d00u 5576
 
 
   /* Masaram Gondi */
@@ -705,7 +722,7 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 11D90 */  VAbv,  VAbv,     O,  VPst,  VPst, VMAbv, VMPst,     H,     O,     O,     O,     O,     O,     O,     O,     O,
   /* 11DA0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     O,     O,     O,     O,     O,     O,
 
-#define use_offset_0x11ee0u 5576
+#define use_offset_0x11ee0u 5752
 
 
   /* Makasar */
@@ -713,7 +730,7 @@ static const USE_TABLE_ELEMENT_TYPE use_table[] = {
   /* 11EE0 */     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,     B,
   /* 11EF0 */     B,     B,    GB,  VAbv,  VBlw,  VPre,  VPst,     O,
 
-}; /* Table items: 5600; occupancy: 73% */
+}; /* Table items: 5776; occupancy: 74% */
 
 USE_TABLE_ELEMENT_TYPE
 hb_use_get_category (hb_codepoint_t u)
@@ -725,6 +742,7 @@ hb_use_get_category (hb_codepoint_t u)
       if (hb_in_range<hb_codepoint_t> (u, 0x00A0u, 0x00D7u)) return use_table[u - 0x00A0u + use_offset_0x00a0u];
       if (hb_in_range<hb_codepoint_t> (u, 0x0348u, 0x034Fu)) return use_table[u - 0x0348u + use_offset_0x0348u];
       if (hb_in_range<hb_codepoint_t> (u, 0x0900u, 0x0DF7u)) return use_table[u - 0x0900u + use_offset_0x0900u];
+      if (hb_in_range<hb_codepoint_t> (u, 0x0F18u, 0x0FC7u)) return use_table[u - 0x0F18u + use_offset_0x0f18u];
       break;
 
     case 0x1u:

src/hb-ot-shape-complex-use.cc

@@ -132,19 +132,19 @@ collect_features_use (hb_ot_shape_planner_t *plan)
   map->enable_feature (HB_TAG('l','o','c','l'));
   map->enable_feature (HB_TAG('c','c','m','p'));
   map->enable_feature (HB_TAG('n','u','k','t'));
-  map->add_feature (HB_TAG('a','k','h','n'), F_GLOBAL | F_MANUAL_ZWJ);
+  map->enable_feature (HB_TAG('a','k','h','n'), F_MANUAL_ZWJ);
 
   /* "Reordering group" */
   map->add_gsub_pause (clear_substitution_flags);
   map->add_feature (HB_TAG('r','p','h','f'), F_MANUAL_ZWJ);
   map->add_gsub_pause (record_rphf);
   map->add_gsub_pause (clear_substitution_flags);
-  map->add_feature (HB_TAG('p','r','e','f'), F_GLOBAL | F_MANUAL_ZWJ);
+  map->enable_feature (HB_TAG('p','r','e','f'), F_MANUAL_ZWJ);
   map->add_gsub_pause (record_pref);
 
   /* "Orthographic unit shaping group" */
   for (unsigned int i = 0; i < ARRAY_LENGTH (basic_features); i++)
-    map->add_feature (basic_features[i], F_GLOBAL | F_MANUAL_ZWJ);
+    map->enable_feature (basic_features[i], F_MANUAL_ZWJ);
 
   map->add_gsub_pause (reorder);
 
@@ -155,7 +155,7 @@ collect_features_use (hb_ot_shape_planner_t *plan)
 
   /* "Standard typographic presentation" */
   for (unsigned int i = 0; i < ARRAY_LENGTH (other_features); i++)
-    map->add_feature (other_features[i], F_GLOBAL | F_MANUAL_ZWJ);
+    map->enable_feature (other_features[i], F_MANUAL_ZWJ);
 
   /* "Positional feature application" */
   for (unsigned int i = 0; i < ARRAY_LENGTH (positioning_features); i++)
@@ -597,7 +597,7 @@ const hb_ot_complex_shaper_t _hb_ot_complex_shaper_use =
   nullptr, /* decompose */
   compose_use,
   setup_masks_use,
-  nullptr, /* disable_otl */
+  HB_TAG_NONE, /* gpos_tag */
   nullptr, /* reorder_marks */
   HB_OT_SHAPE_ZERO_WIDTH_MARKS_BY_GDEF_EARLY,
   false, /* fallback_position */

src/hb-ot-shape-complex.hh

@@ -29,11 +29,11 @@
 
 #include "hb.hh"
 
+#include "hb-ot-layout.hh"
 #include "hb-ot-shape.hh"
 #include "hb-ot-shape-normalize.hh"
 
 
-
 /* buffer var allocations, used by complex shapers */
 #define complex_var_u8_0()	var2.u8[2]
 #define complex_var_u8_1()	var2.u8[3]
@@ -59,7 +59,6 @@ enum hb_ot_shape_zero_width_marks_type_t {
   HB_COMPLEX_SHAPER_IMPLEMENT (myanmar) \
   HB_COMPLEX_SHAPER_IMPLEMENT (myanmar_old) \
   HB_COMPLEX_SHAPER_IMPLEMENT (thai) \
-  HB_COMPLEX_SHAPER_IMPLEMENT (tibetan) \
   HB_COMPLEX_SHAPER_IMPLEMENT (use) \
   /* ^--- Add new shapers here */
 
@@ -147,13 +146,11 @@ struct hb_ot_complex_shaper_t
 		       hb_buffer_t              *buffer,
 		       hb_font_t                *font);
 
-  /* disable_otl()
+  /* gpos_tag()
-   * Called during shape().
+   * If not HB_TAG_NONE, then must match found GPOS script tag for
-   * If set and returns true, GDEF/GSUB/GPOS of the font are ignored
+   * GPOS to be applied.  Otherwise, fallback positioning will be used.
-   * and fallback operations used.
-   * May be nullptr.
    */
-  bool (*disable_otl) (const hb_ot_shape_plan_t *plan);
+  hb_tag_t gpos_tag;
 
   /* reorder_marks()
    * Called during shape().
@@ -234,12 +231,6 @@ hb_ot_shape_complex_categorize (const hb_ot_shape_planner_t *planner)
       return &_hb_ot_complex_shaper_hangul;
 
 
-    /* Unicode-2.0 additions */
-    case HB_SCRIPT_TIBETAN:
-
-      return &_hb_ot_complex_shaper_tibetan;
-
-
     /* Unicode-1.1 additions */
     case HB_SCRIPT_HEBREW:
 
@@ -291,7 +282,7 @@ hb_ot_shape_complex_categorize (const hb_ot_shape_planner_t *planner)
 
 
     /* Unicode-2.0 additions */
-    //case HB_SCRIPT_TIBETAN:
+    case HB_SCRIPT_TIBETAN:
 
     /* Unicode-3.0 additions */
     //case HB_SCRIPT_MONGOLIAN:

src/hb-ot-shape-fallback.cc

@@ -441,7 +441,7 @@ _hb_ot_shape_fallback_kern (const hb_ot_shape_plan_t *plan,
 			    hb_font_t *font,
 			    hb_buffer_t  *buffer)
 {
-  if (!plan->has_kern) return;
+  if (!plan->kerning_requested) return;
 
   OT::hb_ot_apply_context_t c (1, font, buffer);
   hb_mask_t kern_mask = plan->kern_mask;

src/hb-ot-shape-normalize.cc

@@ -296,7 +296,7 @@ _hb_ot_shape_normalize (const hb_ot_shape_plan_t *plan,
   hb_ot_shape_normalization_mode_t mode = plan->shaper->normalization_preference;
   if (mode == HB_OT_SHAPE_NORMALIZATION_MODE_AUTO)
   {
-    if (plan->has_mark)
+    if (plan->has_gpos_mark)
       // https://github.com/harfbuzz/harfbuzz/issues/653#issuecomment-423905920
       //mode = HB_OT_SHAPE_NORMALIZATION_MODE_DECOMPOSED;
       mode = HB_OT_SHAPE_NORMALIZATION_MODE_COMPOSED_DIACRITICS;

src/hb-ot-shape.cc

@@ -36,33 +36,62 @@
 #include "hb-ot-shape-normalize.hh"
 
 #include "hb-ot-face.hh"
-#include "hb-ot-layout.hh"
+
-#include "hb-unicode.hh"
 #include "hb-set.hh"
 
-#include "hb-ot-layout-gsubgpos.hh"
 #include "hb-aat-layout.hh"
 
-static hb_tag_t common_features[] = {
+
-  HB_TAG('c','c','m','p'),
+void
-  HB_TAG('l','o','c','l'),
+hb_ot_shape_planner_t::compile (hb_ot_shape_plan_t &plan,
-  HB_TAG('m','a','r','k'),
+				const int          *coords,
-  HB_TAG('m','k','m','k'),
+				unsigned int        num_coords)
-  HB_TAG('r','l','i','g'),
+{
+  plan.props = props;
+  plan.shaper = shaper;
+  map.compile (plan.map, coords, num_coords);
+
+  plan.rtlm_mask = plan.map.get_1_mask (HB_TAG ('r','t','l','m'));
+  plan.frac_mask = plan.map.get_1_mask (HB_TAG ('f','r','a','c'));
+  plan.numr_mask = plan.map.get_1_mask (HB_TAG ('n','u','m','r'));
+  plan.dnom_mask = plan.map.get_1_mask (HB_TAG ('d','n','o','m'));
+
+  plan.kern_mask = plan.map.get_mask (HB_DIRECTION_IS_HORIZONTAL (plan.props.direction) ?
+				      HB_TAG ('k','e','r','n') : HB_TAG ('v','k','r','n'));
+
+  plan.has_frac = plan.frac_mask || (plan.numr_mask && plan.dnom_mask);
+  plan.kerning_requested = !!plan.kern_mask;
+  plan.has_gpos_mark = !!plan.map.get_1_mask (HB_TAG ('m','a','r','k'));
+
+  bool disable_gpos = plan.shaper->gpos_tag &&
+		      plan.shaper->gpos_tag != plan.map.chosen_script[1];
+  plan.fallback_positioning = disable_gpos || !hb_ot_layout_has_positioning (face);
+  plan.fallback_glyph_classes = !hb_ot_layout_has_glyph_classes (face);
+}
+
+
+static const hb_ot_map_feature_t
+common_features[] =
+{
+  {HB_TAG('c','c','m','p'), F_GLOBAL},
+  {HB_TAG('l','o','c','l'), F_GLOBAL},
+  {HB_TAG('m','a','r','k'), F_GLOBAL_MANUAL_JOINERS},
+  {HB_TAG('m','k','m','k'), F_GLOBAL_MANUAL_JOINERS},
+  {HB_TAG('r','l','i','g'), F_GLOBAL},
 };
 
 
-static hb_tag_t horizontal_features[] = {
+static const hb_ot_map_feature_t
-  HB_TAG('c','a','l','t'),
+horizontal_features[] =
-  HB_TAG('c','l','i','g'),
+{
-  HB_TAG('c','u','r','s'),
+  {HB_TAG('c','a','l','t'), F_GLOBAL},
-  HB_TAG('k','e','r','n'),
+  {HB_TAG('c','l','i','g'), F_GLOBAL},
-  HB_TAG('l','i','g','a'),
+  {HB_TAG('c','u','r','s'), F_GLOBAL},
-  HB_TAG('r','c','l','t'),
+  {HB_TAG('k','e','r','n'), F_GLOBAL_HAS_FALLBACK},
+  {HB_TAG('l','i','g','a'), F_GLOBAL},
+  {HB_TAG('r','c','l','t'), F_GLOBAL},
 };
 
-
-
 static void
 hb_ot_shape_collect_features (hb_ot_shape_planner_t          *planner,
 			      const hb_segment_properties_t  *props,
@@ -96,26 +125,24 @@ hb_ot_shape_collect_features (hb_ot_shape_planner_t          *planner,
   map->add_feature (HB_TAG ('d','n','o','m'));
 
   /* Random! */
-  map->add_feature (HB_TAG ('r','a','n','d'), F_GLOBAL | F_RANDOM, HB_OT_MAP_MAX_VALUE);
+  map->enable_feature (HB_TAG ('r','a','n','d'), F_RANDOM, HB_OT_MAP_MAX_VALUE);
 
   if (planner->shaper->collect_features)
     planner->shaper->collect_features (planner);
 
   for (unsigned int i = 0; i < ARRAY_LENGTH (common_features); i++)
-    map->enable_feature (common_features[i]);
+    map->add_feature (common_features[i]);
 
   if (HB_DIRECTION_IS_HORIZONTAL (props->direction))
     for (unsigned int i = 0; i < ARRAY_LENGTH (horizontal_features); i++)
-      map->add_feature (horizontal_features[i], F_GLOBAL |
+      map->add_feature (horizontal_features[i]);
-			(horizontal_features[i] == HB_TAG('k','e','r','n') ?
-			 F_HAS_FALLBACK : F_NONE));
   else
   {
     /* We really want to find a 'vert' feature if there's any in the font, no
      * matter which script/langsys it is listed (or not) under.
      * See various bugs referenced from:
      * https://github.com/harfbuzz/harfbuzz/issues/63 */
-    map->add_feature (HB_TAG ('v','e','r','t'), F_GLOBAL | F_GLOBAL_SEARCH);
+    map->enable_feature (HB_TAG ('v','e','r','t'), F_GLOBAL_SEARCH);
   }
 
   if (planner->shaper->override_features)
@@ -235,8 +262,6 @@ struct hb_ot_shape_context_t
   unsigned int        num_user_features;
 
   /* Transient stuff */
-  bool fallback_positioning;
-  bool fallback_glyph_classes;
   hb_direction_t target_direction;
 };
 
@@ -612,7 +637,7 @@ hb_ot_substitute_default (hb_ot_shape_context_t *c)
   hb_ot_shape_setup_masks (c);
 
   /* This is unfortunate to go here, but necessary... */
-  if (c->fallback_positioning)
+  if (c->plan->fallback_positioning)
     _hb_ot_shape_fallback_position_recategorize_marks (c->plan, c->font, buffer);
 
   hb_ot_map_glyphs_fast (buffer);
@@ -627,7 +652,7 @@ hb_ot_substitute_complex (hb_ot_shape_context_t *c)
 
   hb_ot_layout_substitute_start (c->font, buffer);
 
-  if (!hb_ot_layout_has_glyph_classes (c->face))
+  if (c->plan->fallback_glyph_classes)
     hb_synthesize_glyph_classes (c);
 
   c->plan->substitute (c->font, buffer);
@@ -726,7 +751,7 @@ hb_ot_position_complex (hb_ot_shape_context_t *c)
    * If fallback positinoing happens or GPOS is present, we don't
    * care.
    */
-  bool adjust_offsets_when_zeroing = c->fallback_positioning &&
+  bool adjust_offsets_when_zeroing = c->plan->fallback_positioning &&
 				     !c->plan->shaper->fallback_position &&
 				     HB_DIRECTION_IS_FORWARD (c->buffer->props.direction);
 
@@ -753,7 +778,7 @@ hb_ot_position_complex (hb_ot_shape_context_t *c)
       break;
   }
 
-  if (likely (!c->fallback_positioning))
+  if (likely (!c->plan->fallback_positioning))
     c->plan->position (c->font, c->buffer);
 
   switch (c->plan->shaper->zero_width_marks)
@@ -790,7 +815,7 @@ hb_ot_position (hb_ot_shape_context_t *c)
 
   hb_ot_position_complex (c);
 
-  if (c->fallback_positioning && c->plan->shaper->fallback_position)
+  if (c->plan->fallback_positioning && c->plan->shaper->fallback_position)
     _hb_ot_shape_fallback_position (c->plan, c->font, c->buffer);
 
   if (HB_DIRECTION_IS_BACKWARD (c->buffer->props.direction))
@@ -798,7 +823,7 @@ hb_ot_position (hb_ot_shape_context_t *c)
 
   /* Visual fallback goes here. */
 
-  if (c->fallback_positioning)
+  if (c->plan->fallback_positioning)
     _hb_ot_shape_fallback_kern (c->plan, c->font, c->buffer);
 
   _hb_buffer_deallocate_gsubgpos_vars (c->buffer);
@@ -850,11 +875,6 @@ hb_ot_shape_internal (hb_ot_shape_context_t *c)
 			      (unsigned) HB_BUFFER_MAX_OPS_MIN);
   }
 
-  bool disable_otl = c->plan->shaper->disable_otl && c->plan->shaper->disable_otl (c->plan);
-  //c->fallback_substitute     = disable_otl || !hb_ot_layout_has_substitution (c->face);
-  c->fallback_positioning    = disable_otl || !hb_ot_layout_has_positioning (c->face);
-  c->fallback_glyph_classes  = disable_otl || !hb_ot_layout_has_glyph_classes (c->face);
-
   /* Save the original direction, we use it later. */
   c->target_direction = c->buffer->props.direction;
 

src/hb-ot-shape.hh

@@ -30,7 +30,7 @@
 #include "hb.hh"
 
 #include "hb-ot-map.hh"
-#include "hb-ot-layout.hh"
+#include "hb-shape-plan.hh"
 
 
 
@@ -42,9 +42,12 @@ struct hb_ot_shape_plan_t
   const void *data;
   hb_mask_t rtlm_mask, frac_mask, numr_mask, dnom_mask;
   hb_mask_t kern_mask;
-  unsigned int has_frac : 1;
+
-  unsigned int has_kern : 1;
+  bool has_frac : 1;
-  unsigned int has_mark : 1;
+  bool kerning_requested : 1;
+  bool has_gpos_mark : 1;
+  bool fallback_positioning : 1;
+  bool fallback_glyph_classes : 1;
 
   inline void collect_lookups (hb_tag_t table_tag, hb_set_t *lookups) const
   {
@@ -83,26 +86,9 @@ struct hb_ot_shape_planner_t
 			 shaper (nullptr),
 			 map (face, &props) {}
 
-  inline void compile (hb_ot_shape_plan_t &plan,
+  HB_INTERNAL void compile (hb_ot_shape_plan_t &plan,
-		       const int          *coords,
+			    const int          *coords,
-		       unsigned int        num_coords)
+			    unsigned int        num_coords);
-  {
-    plan.props = props;
-    plan.shaper = shaper;
-    map.compile (plan.map, coords, num_coords);
-
-    plan.rtlm_mask = plan.map.get_1_mask (HB_TAG ('r','t','l','m'));
-    plan.frac_mask = plan.map.get_1_mask (HB_TAG ('f','r','a','c'));
-    plan.numr_mask = plan.map.get_1_mask (HB_TAG ('n','u','m','r'));
-    plan.dnom_mask = plan.map.get_1_mask (HB_TAG ('d','n','o','m'));
-
-    plan.kern_mask = plan.map.get_mask (HB_DIRECTION_IS_HORIZONTAL (plan.props.direction) ?
-					HB_TAG ('k','e','r','n') : HB_TAG ('v','k','r','n'));
-
-    plan.has_frac = plan.frac_mask || (plan.numr_mask && plan.dnom_mask);
-    plan.has_kern = !!plan.kern_mask;
-    plan.has_mark = !!plan.map.get_1_mask (HB_TAG ('m','a','r','k'));
-  }
 
   private:
   HB_DISALLOW_COPY_AND_ASSIGN (hb_ot_shape_planner_t);

src/hb-ucdn.cc

@@ -274,6 +274,9 @@ void free_static_ucdn_funcs (void)
 #endif
 
 extern "C" HB_INTERNAL
+hb_unicode_funcs_t *
+hb_ucdn_get_unicode_funcs (void);
+
 hb_unicode_funcs_t *
 hb_ucdn_get_unicode_funcs (void)
 {

src/sample.py

@@ -54,11 +54,11 @@ if False:
 	# buffer:
 	hb.buffer_add_utf8 (buf, text.encode('utf-8'), 0, -1)
 	# Otherwise, then following handles both narrow and wide
-	# Python builds:
+	# Python builds (the first item in the array is BOM, so we skip it):
 elif sys.maxunicode == 0x10FFFF:
-	hb.buffer_add_utf32 (buf, array.array('I', text.encode('utf-32')), 0, -1)
+	hb.buffer_add_utf32 (buf, array.array('I', text.encode('utf-32'))[1:], 0, -1)
 else:
-	hb.buffer_add_utf16 (buf, array.array('H', text.encode('utf-16')), 0, -1)
+	hb.buffer_add_utf16 (buf, array.array('H', text.encode('utf-16'))[1:], 0, -1)
 
 
 hb.buffer_guess_segment_properties (buf)

src/test-unicode-ranges.cc

@@ -28,7 +28,7 @@
 
 #include "hb-ot-os2-unicode-ranges.hh"
 
-void
+static void
 test (hb_codepoint_t cp, unsigned int bit)
 {
   if (OT::hb_get_unicode_range_bit (cp) != bit)
@@ -41,7 +41,7 @@ test (hb_codepoint_t cp, unsigned int bit)
   }
 }
 
-void
+static void
 test_get_unicode_range_bit (void)
 {
   test (0x0000, 0);

test/api/test-buffer.c

@@ -92,14 +92,14 @@ fixture_init (fixture_t *fixture, gconstpointer user_data)
 }
 
 static void
-fixture_finish (fixture_t *fixture, gconstpointer user_data)
+fixture_finish (fixture_t *fixture, gconstpointer user_data HB_UNUSED)
 {
   hb_buffer_destroy (fixture->buffer);
 }
 
 
 static void
-test_buffer_properties (fixture_t *fixture, gconstpointer user_data)
+test_buffer_properties (fixture_t *fixture, gconstpointer user_data HB_UNUSED)
 {
   hb_buffer_t *b = fixture->buffer;
   hb_unicode_funcs_t *ufuncs;
@@ -294,7 +294,7 @@ test_buffer_contents (fixture_t *fixture, gconstpointer user_data)
 }
 
 static void
-test_buffer_positions (fixture_t *fixture, gconstpointer user_data)
+test_buffer_positions (fixture_t *fixture, gconstpointer user_data HB_UNUSED)
 {
   hb_buffer_t *b = fixture->buffer;
   unsigned int i, len, len2;
@@ -319,7 +319,7 @@ test_buffer_positions (fixture_t *fixture, gconstpointer user_data)
 }
 
 static void
-test_buffer_allocation (fixture_t *fixture, gconstpointer user_data)
+test_buffer_allocation (fixture_t *fixture, gconstpointer user_data HB_UNUSED)
 {
   hb_buffer_t *b = fixture->buffer;
 

test/api/test-multithread.c

@@ -49,7 +49,7 @@ static int num_iters = 200;
 static hb_font_t *font;
 static hb_buffer_t *ref_buffer;
 
-pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
 
 static void
 fill_the_buffer (hb_buffer_t *buffer)
@@ -105,7 +105,6 @@ static void
 test_body (void)
 {
   int i;
-  int num_threads = 30;
   pthread_t *threads = calloc (num_threads, sizeof (pthread_t));
   hb_buffer_t **buffers = calloc (num_threads, sizeof (hb_buffer_t *));
 

test/fuzzing/hb-subset-fuzzer.cc

@@ -6,11 +6,12 @@
 
 #include "hb-subset.h"
 
-void trySubset (hb_face_t *face,
+static void
-                const hb_codepoint_t text[],
+trySubset (hb_face_t *face,
-                int text_length,
+	   const hb_codepoint_t text[],
-                bool drop_hints,
+	   int text_length,
-                bool drop_layout)
+	   bool drop_hints,
+	   bool drop_layout)
 {
   hb_subset_input_t *input = hb_subset_input_create_or_fail ();
   hb_subset_input_set_drop_hints (input, drop_hints);
@@ -28,16 +29,17 @@ void trySubset (hb_face_t *face,
   hb_subset_input_destroy (input);
 }
 
-void trySubset (hb_face_t *face,
+static void
-                const hb_codepoint_t text[],
+trySubset (hb_face_t *face,
-                int text_length)
+	   const hb_codepoint_t text[],
+	   int text_length)
 {
   for (unsigned int drop_hints = 0; drop_hints < 2; drop_hints++)
   {
     for (unsigned int drop_layout = 0; drop_layout < 2; drop_layout++)
     {
       trySubset (face, text, text_length,
-                 (bool) drop_hints, (bool) drop_layout);
+		 (bool) drop_hints, (bool) drop_layout);
     }
   }
 }
@@ -45,22 +47,22 @@ void trySubset (hb_face_t *face,
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
 {
   hb_blob_t *blob = hb_blob_create ((const char *)data, size,
-                                    HB_MEMORY_MODE_READONLY, NULL, NULL);
+				    HB_MEMORY_MODE_READONLY, NULL, NULL);
   hb_face_t *face = hb_face_create (blob, 0);
 
   const hb_codepoint_t text[] =
       {
-        'A', 'B', 'C', 'D', 'E', 'X', 'Y', 'Z', '1', '2',
+	'A', 'B', 'C', 'D', 'E', 'X', 'Y', 'Z', '1', '2',
-        '3', '@', '_', '%', '&', ')', '*', '$', '!'
+	'3', '@', '_', '%', '&', ')', '*', '$', '!'
       };
 
   trySubset (face, text, sizeof (text) / sizeof (hb_codepoint_t));
 
   hb_codepoint_t text_from_data[16];
   if (size > sizeof(text_from_data)) {
-    memcpy(text_from_data,
+    memcpy (text_from_data,
-           data + size - sizeof(text_from_data),
+	    data + size - sizeof(text_from_data),
-           sizeof(text_from_data));
+	    sizeof(text_from_data));
     unsigned int text_size = sizeof (text_from_data) / sizeof (hb_codepoint_t);
     trySubset (face, text_from_data, text_size);
   }

test/fuzzing/main.cc

@@ -1,21 +1,17 @@
 #include "hb-fuzzer.hh"
 
-#include <iostream>
+#include <stdio.h>
-#include <iterator>
-#include <fstream>
 #include <assert.h>
 
-std::string FileToString(const std::string &Path) {
-  /* TODO This silently passes if file does not exist.  Fix it! */
-  std::ifstream T(Path.c_str());
-  return std::string((std::istreambuf_iterator<char>(T)),
-                     std::istreambuf_iterator<char>());
-}
-
 int main(int argc, char **argv) {
+  hb_blob_t *blob = hb_blob_create_from_file (argv[1]);
+  unsigned int len;
+  const char *font_data = hb_blob_get_data (blob, &len);
+
   for (int i = 1; i < argc; i++) {
-    std::string s = FileToString(argv[i]);
+    printf ("%s\n", argv[i]);
-    std::cout << argv[i] << std::endl;
+    LLVMFuzzerTestOneInput((const uint8_t *) font_data, len);
-    LLVMFuzzerTestOneInput((const unsigned char*)s.data(), s.size());
   }
+
+  hb_blob_destroy (blob);
 }

test/fuzzing/run-shape-fuzzer-tests.py

@@ -4,8 +4,6 @@ from __future__ import print_function, division, absolute_import
 
 import sys, os, subprocess
 
-if os.environ.get ("SKIPFUZZERTESTS", "") != "": sys.exit (0)
-
 srcdir = os.environ.get ("srcdir", ".")
 EXEEXT = os.environ.get ("EXEEXT", "")
 top_builddir = os.environ.get ("top_builddir", ".")

test/fuzzing/run-subset-fuzzer-tests.py

@@ -4,8 +4,6 @@ from __future__ import print_function, division, absolute_import
 
 import sys, os, subprocess
 
-if os.environ.get ("SKIPFUZZERTESTS", "") != "": sys.exit (0)
-
 srcdir = os.environ.get ("srcdir", ".")
 EXEEXT = os.environ.get ("EXEEXT", "")
 top_builddir = os.environ.get ("top_builddir", ".")

test/shaping/data/in-house/Makefile.sources

@@ -2,6 +2,7 @@ TESTS = \
 	tests/arabic-fallback-shaping.tests \
 	tests/arabic-feature-order.tests \
 	tests/arabic-like-joining.tests \
+	tests/arabic-mark-attach.tests \
 	tests/arabic-mark-order.tests \
 	tests/arabic-stch.tests \
 	tests/automatic-fractions.tests \
@@ -26,6 +27,8 @@ TESTS = \
 	tests/indic-script-extensions.tests \
 	tests/indic-special-cases.tests \
 	tests/indic-syllable.tests \
+	tests/khmer-mark-order.tests \
+	tests/khmer-misc.tests \
 	tests/language-tags.tests \
 	tests/ligature-id.tests \
 	tests/mark-attachment.tests \

test/shaping/data/in-house/tests/arabic-mark-attach.tests

@@ -0,0 +1 @@
+../fonts/641ca9d7808b01cafa9a666c13811c9b56eb9c52.ttf::U+064A,U+0633,U+06E1,U+200D,U+0654,U+064E,U+0644:[afii57444.zz04=6+1091|afii57454=1@75,925+0|uni0654=1+0|space=1+0|uni06E1=1@950,1115+0|afii57427.zz03_calt=1+1847|afii57450.zz21=0+345]

test/shaping/data/in-house/tests/fuzzed.tests

@@ -1,23 +1,23 @@
-../fonts/1a6f1687b7a221f9f2c834b0b360d3c8463b6daf.ttf:--font-funcs=ot:U+0041:[gid0=0+1000]
+../fonts/1a6f1687b7a221f9f2c834b0b360d3c8463b6daf.ttf:--font-funcs=ot:U+0041:*
-../fonts/5a5daf5eb5a4db77a2baa3ad9c7a6ed6e0655fa8.ttf:--font-funcs=ot:U+0041:[gid0=0+1000]
+../fonts/5a5daf5eb5a4db77a2baa3ad9c7a6ed6e0655fa8.ttf:--font-funcs=ot:U+0041:*
-../fonts/0509e80afb379d16560e9e47bdd7d888bebdebc6.ttf:--font-funcs=ot:U+0041:[gid0=0+1000]
+../fonts/0509e80afb379d16560e9e47bdd7d888bebdebc6.ttf:--font-funcs=ot:U+0041:*
-../fonts/641bd9db850193064d17575053ae2bf8ec149ddc.ttf:--font-funcs=ot:U+0041:[gid0=0+1000]
+../fonts/641bd9db850193064d17575053ae2bf8ec149ddc.ttf:--font-funcs=ot:U+0041:*
-../fonts/375d6ae32a3cbe52fbf81a4e5777e3377675d5a3.ttf:--font-funcs=ot:U+0041:[gid0=0+4352]
+../fonts/375d6ae32a3cbe52fbf81a4e5777e3377675d5a3.ttf:--font-funcs=ot:U+0041:*
-../fonts/8240789f6d12d4cfc4b5e8e6f246c3701bcf861f.ttf:--font-funcs=ot:U+0041:[gid0=0+1024]
+../fonts/8240789f6d12d4cfc4b5e8e6f246c3701bcf861f.ttf:--font-funcs=ot:U+0041:*
-../fonts/b9e2aaa0d75fcef6971ec3a96d806ba4a6b31fe2.ttf:--font-funcs=ot:U+0041:[gid0=0+1000|gid1=0+1000|gid8=0+1000|gid3=0+1000|gid0=0+1000|gid1=0+1000|gid1=0+1000|gid8=0+1000|gid3=0+1000|gid0=0+1000|gid1=0+1000|gid8=0+1000|gid3=0+1000|gid0=0+1000|gid1=0+1000|gid1=0+1000]
+../fonts/b9e2aaa0d75fcef6971ec3a96d806ba4a6b31fe2.ttf:--font-funcs=ot:U+0041:*
-../fonts/43979b90b2dd929723cf4fe1715990bcb9c9a56b.ttf:--font-funcs=ot:U+0041:[gid0=0+1000]
+../fonts/43979b90b2dd929723cf4fe1715990bcb9c9a56b.ttf:--font-funcs=ot:U+0041:*
-../fonts/3511ff5c1647150595846ac414c595cccac34f18.ttf:--font-funcs=ot --no-positions --no-clusters --no-glyph-names:U+0041:[0|512|15104|11004|3408|18244|17872|17961|0|992|15616|0|14151|20559|20992|5440|256|0|10|8960|256|1024|1490|0|768|4096|256|2216|0|256|256|0|768|10752|11004|3408|18244|17734|53248|256|0|512|14848|10793|57344|768|18227|20285|20480|0|256|0|810|0|11004|3408|18244|17734|53289|57344|768|15667|71|0|20559|21248|256|0|2816|2776|0|51516|0|32|26209|28005|65249|29690|0|51548|0|2454|28783|29556|1291|3458|80|0|2804|210|28786|25968|45763|50546|0|59136|0|38144|256|0|2560|30208|52224|580|17996|21504|6734|108|116|24846|1024|0|255|65280|256|0|8704|1345|23109|8192|10823|21076|8192|12877|20300|8192|6738|20301|8192|16980|21067|8251|18944|255|65280|15360|256|255|65280|256|768|255|65280|256|768|255|65280|256|1024|12|65280|256|1280|255|65280|256|1536|1899|25970|110|11264|27502|29285|12907|25974|28160|14443|25970|28288|3|118|18259|21826|45716|46369|0|0|1|16|17|256|4|16|18244|17734|28|12|0|284|0|28|18256|20307|45114|47616|226|10296|0|57927|1|0|0|21248|5440|256|0|10|768|256|1024|512|0|297|16|24833|28774|10794|2304|29|32|42|64515|42|42|64525|20551|17477|18128|10720|3|61|3408|18244|17734|53289|57344|768|15616|512|55|10576|20307|0|255|56063|53504|42|42|64525|12288|18176|80|20307|1|0|62]
+../fonts/3511ff5c1647150595846ac414c595cccac34f18.ttf:--font-funcs=ot --no-positions --no-clusters --no-glyph-names:U+0041:*
-../fonts/fab39d60d758cb586db5a504f218442cd1395725.ttf:--font-funcs=ot:U+0041,U+0041:[gid0=0+1000|gid0=1+1000]
+../fonts/fab39d60d758cb586db5a504f218442cd1395725.ttf:--font-funcs=ot:U+0041,U+0041:*
-../fonts/205edd09bd3d141cc9580f650109556cc28b22cb.ttf:--font-funcs=ot:U+0041:[gid0=0+1000]
+../fonts/205edd09bd3d141cc9580f650109556cc28b22cb.ttf:--font-funcs=ot:U+0041:*
-../fonts/217a934cfe15c548b572c203dceb2befdf026462.ttf:--font-funcs=ot:U+0061,U+0061,U+0061:[]
+../fonts/217a934cfe15c548b572c203dceb2befdf026462.ttf:--font-funcs=ot:U+0061,U+0061,U+0061:*
-../fonts/558661aa659912f4d30ecd27bd09835171a8e2b0.ttf:--font-funcs=ot:U+FFFD,U+E0100,U+FFFD,U+E0010:[gid3584=0+1000|gid1024=0+1000|gid1=0+1000|gid8=0+1000|gid3=0+1000|gid0=0+1000|gid1=0+1000|gid3584=0+1000|gid3584=2+1000|gid1024=2+1000|gid1=2+1000|gid8=2+1000|gid3=2+1000|gid0=2+1000|gid1=2+1000|gid3584=2+1000]
+../fonts/558661aa659912f4d30ecd27bd09835171a8e2b0.ttf:--font-funcs=ot:U+FFFD,U+E0100,U+FFFD,U+E0010:*
-../fonts/a34a9191d9376bda419836effeef7e75c1386016.ttf:--font-funcs=ot:U+0041:[]
+../fonts/a34a9191d9376bda419836effeef7e75c1386016.ttf:--font-funcs=ot:U+0041:*
-../fonts/a69118c2c2ada48ff803d9149daa54c9ebdae30e.ttf:--font-funcs=ot:U+0041:[gid0=0+1229]
+../fonts/a69118c2c2ada48ff803d9149daa54c9ebdae30e.ttf:--font-funcs=ot:U+0041:*
-../fonts/b6acef662e0beb8d5fcf5b61c6b0ca69537b7402.ttf:--font-funcs=ot:U+0041:[gid0=0+1000]
+../fonts/b6acef662e0beb8d5fcf5b61c6b0ca69537b7402.ttf:--font-funcs=ot:U+0041:*
-../fonts/e88c339237f52d21e01c55f01b9c1b4cc14a0467.ttf:--font-funcs=ot:U+0041:[gid0=0+1000]
+../fonts/e88c339237f52d21e01c55f01b9c1b4cc14a0467.ttf:--font-funcs=ot:U+0041:*
-../fonts/243798dd281c1c77c065958e1ff467420faa9bde.ttf:--font-funcs=ot:U+0041:[gid0=0+1000]
+../fonts/243798dd281c1c77c065958e1ff467420faa9bde.ttf:--font-funcs=ot:U+0041:*
-../fonts/dd9f0c7c7c36f75a18be0cab1cddf8f3ab0f366b.ttf:--font-funcs=ot --no-positions --no-clusters --no-glyph-names:U+0041:[0|0|2|0|0|2|0|0|2|0|0|2|0|0|2|0|0|2|0|0|0|2|0|0|0|2|0|0|2|0|0|2|0|0|2|0|0|2|0|0|0|2|0|0|2|0|0|2|0|0|2|0]
+../fonts/dd9f0c7c7c36f75a18be0cab1cddf8f3ab0f366b.ttf:--font-funcs=ot --no-positions --no-clusters --no-glyph-names:U+0041:*
-../fonts/ef2511f215aa3ca847cbfffbf861793b42170875.ttf:--font-funcs=ot:U+0041:[gid0=0+1000]
+../fonts/ef2511f215aa3ca847cbfffbf861793b42170875.ttf:--font-funcs=ot:U+0041:*
-../fonts/9d8a94a67932a3ab75a596fc8b5c6d0392ca9e49.ttf:--font-funcs=ot:U+0041:[gid0=0+1000]
+../fonts/9d8a94a67932a3ab75a596fc8b5c6d0392ca9e49.ttf:--font-funcs=ot:U+0041:*
-../fonts/bbf4a308c402f0678c3e82844892a4da2ebe598f.ttf:--font-funcs=ot:U+0041:[gid0=0+1000]
+../fonts/bbf4a308c402f0678c3e82844892a4da2ebe598f.ttf:--font-funcs=ot:U+0041:*
-../fonts/233c1e252e737ca79e03a9fd56b71aaa4a230f2b.ttf:--font-funcs=ot:U+0041:[gid0=0+1000]
+../fonts/233c1e252e737ca79e03a9fd56b71aaa4a230f2b.ttf:--font-funcs=ot:U+0041:*

test/shaping/data/in-house/tests/khmer-mark-order.tests

@@ -0,0 +1,25 @@
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17CA,U+17BE,U+1794:[uni17C1=0+288|uni179F=0+928|uni17BB=0@-6,-26+0|uni17B8=0@-32,-29+0|uni1794=3+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17C9,U+17BE,U+17BB,U+1794:[uni17C1=0+288|uni179F=0+928|uni17C9=0@-32,-29+0|uni17B8=0@-32,237+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni1794=4+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17C9,U+17C1,U+17B8,U+17BB,U+1794:[uni17C1=0+288|uni179F=0+928|uni17C9=0@-32,-29+0|uni17B8=0@-32,237+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni1794=5+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17C9,U+17B8,U+17C1,U+17BB,U+1794:[uni179F=0+928|uni17C9=0@-32,-29+0|uni17B8=0@-32,237+0|uni17C1=0+288|uni25CC=0+635|uni17BB=0@-20,-26+0|uni1794=5+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17C9,U+17BE,U+17BB,U+17BB,U+1794:[uni17C1=0+288|uni179F=0+928|uni17C9=0@-32,-29+0|uni17B8=0@-32,237+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni1794=5+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17C9,U+17C1,U+17B8,U+17BB,U+17BB,U+1794:[uni17C1=0+288|uni179F=0+928|uni17C9=0@-32,-29+0|uni17B8=0@-32,237+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni1794=6+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17C9,U+17B8,U+17C1,U+17BB,U+17BB,U+1794:[uni179F=0+928|uni17C9=0@-32,-29+0|uni17B8=0@-32,237+0|uni17C1=0+288|uni25CC=0+635|uni17BB=0@-20,-26+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni1794=6+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17CA,U+17BE,U+17BB,U+1794:[uni17C1=0+288|uni179F=0+928|uni17BB=0@-6,-26+0|uni17B8=0@-32,-29+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni1794=4+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17CA,U+17C1,U+17B8,U+17BB,U+1794:[uni17C1=0+288|uni179F=0+928|uni17BB=0@-6,-26+0|uni17B8=0@-32,-29+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni1794=5+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17CA,U+17B8,U+17C1,U+17BB,U+1794:[uni179F=0+928|uni17BB=0@-6,-26+0|uni17B8=0@-32,-29+0|uni17C1=0+288|uni25CC=0+635|uni17BB=0@-20,-26+0|uni1794=5+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17CA,U+17BE,U+17BB,U+17BB,U+1794:[uni17C1=0+288|uni179F=0+928|uni17BB=0@-6,-26+0|uni17B8=0@-32,-29+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni1794=5+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17CA,U+17C1,U+17B8,U+17BB,U+17BB,U+1794:[uni17C1=0+288|uni179F=0+928|uni17BB=0@-6,-26+0|uni17B8=0@-32,-29+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni1794=6+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17CA,U+17B8,U+17C1,U+17BB,U+17BB,U+1794:[uni179F=0+928|uni17BB=0@-6,-26+0|uni17B8=0@-32,-29+0|uni17C1=0+288|uni25CC=0+635|uni17BB=0@-20,-26+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni1794=6+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17C9,U+17BE,U+17B8,U+1794:[uni17C1=0+288|uni179F=0+928|uni17C9=0@-32,-29+0|uni17B8=0@-32,237+0|uni25CC=0+635|uni17B8=0@-20,-84+0|uni1794=4+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17C9,U+17B8,U+17BE,U+1794:[uni179F=0+928|uni17C9=0@-32,-29+0|uni17B8=0@-32,237+0|uni17C1=0+288|uni25CC=0+635|uni17B8=0@-20,-84+0|uni1794=4+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17C9,U+17BE,U+17B8,U+17BB,U+1794:[uni17C1=0+288|uni179F=0+928|uni17C9=0@-32,-29+0|uni17B8=0@-32,237+0|uni25CC=0+635|uni17B8=0@-20,-84+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni1794=5+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17C9,U+17B8,U+17BE,U+17BB,U+1794:[uni179F=0+928|uni17C9=0@-32,-29+0|uni17B8=0@-32,237+0|uni17C1=0+288|uni25CC=0+635|uni17B8=0@-20,-84+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni1794=5+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17CA,U+17BE,U+17B8,U+1794:[uni17C1=0+288|uni179F=0+928|uni17BB=0@-6,-26+0|uni17B8=0@-32,-29+0|uni25CC=0+635|uni17B8=0@-20,-84+0|uni1794=4+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17CA,U+17B8,U+17BE,U+1794:[uni179F=0+928|uni17BB=0@-6,-26+0|uni17B8=0@-32,-29+0|uni17C1=0+288|uni25CC=0+635|uni17B8=0@-20,-84+0|uni1794=4+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17CA,U+17BE,U+17B8,U+17BB,U+1794:[uni17C1=0+288|uni179F=0+928|uni17BB=0@-6,-26+0|uni17B8=0@-32,-29+0|uni25CC=0+635|uni17B8=0@-20,-84+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni1794=5+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17CA,U+17B8,U+17BE,U+17BB,U+1794:[uni179F=0+928|uni17BB=0@-6,-26+0|uni17B8=0@-32,-29+0|uni17C1=0+288|uni25CC=0+635|uni17B8=0@-20,-84+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni1794=5+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17BE,U+17B8,U+17BB,U+1794:[uni17C1=0+288|uni179F=0+928|uni17B8=0@-32,-29+0|uni25CC=0+635|uni17B8=0@-20,-84+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni1794=4+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17BE,U+17BB,U+17B8,U+1794:[uni17C1=0+288|uni179F=0+928|uni17B8=0@-32,-29+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni17B8=0@-20,-84+0|uni1794=4+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17B8,U+17BE,U+17BB,U+1794:[uni179F=0+928|uni17B8=0@-32,-29+0|uni17C1=0+288|uni25CC=0+635|uni17B8=0@-20,-84+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni1794=4+635]
+../fonts/b6031119874ae9ff1dd65383a335e361c0962220.ttf::U+179F,U+17B8,U+17BB,U+17BE,U+1794:[uni179F=0+928|uni17B8=0@-32,-29+0|uni25CC=0+635|uni17BB=0@-20,-26+0|uni17C1=0+288|uni25CC=0+635|uni17B8=0@-20,-84+0|uni1794=4+635]

test/shaping/run-tests.py

@@ -84,7 +84,7 @@ for filename in args:
 			print (":".join ([fontfile, options, unicodes, glyphs1]))
 			continue
 
-		if glyphs1.strip() != glyphs_expected.strip():
+		if glyphs1.strip() != glyphs_expected.strip() and glyphs_expected.strip() != '*':
 			print ("Actual:   " + glyphs1) # file=sys.stderr
 			print ("Expected: " + glyphs_expected) # file=sys.stderr
 			fails = fails + 1

test/shaping/texts/in-house/shaper-indic/script-bengali/bengali-vowel-leters.txt

@@ -0,0 +1,3 @@
+আ অা
+ৠ ঋৃ
+ৡ ঌৢ

test/shaping/texts/in-house/shaper-indic/script-devanagari/devanagari-atomic-consonants.txt

@@ -0,0 +1,33 @@
+ख ख्ा ख्‍ा
+ग ग्ा ग्‍ा
+घ घ्ा घ्‍ा
+च च्ा च्‍ा
+ज ज्ा ज्‍ा
+झ झ्ा झ्‍ा
+ञ ञ्ा ञ्‍ा
+ण ण्ा ण्‍ा
+त त्ा त्‍ा
+थ थ्ा थ्‍ा
+ध ध्ा ध्‍ा
+न न्ा न्‍ा
+ऩ ऩ्ा ऩ्‍ा ऩ्ा ऩ्‍ा
+प प्ा प्‍ा
+ब ब्ा ब्‍ा
+भ भ्ा भ्‍ा
+म म्ा म्‍ा
+य य्ा य्‍ा
+ल ल्ा ल्‍ा
+व व्ा व्‍ा
+श श्ा श्‍ा
+ष ष्ा ष्‍ा
+स स्ा स्‍ा
+ख़ ख़्ा ख़्‍ा ख़्ा ख़्‍ा
+ग़ ग़्ा ग़्‍ा ग़्ा ग़्‍ा
+ज़ ज़्ा ज़्‍ा ज़्ा ज़्‍ा
+य़ य़्ा य़्‍ा य़्ा य़्‍ा
+ॹ ॹ्ा ॹ्‍ा
+ॺ ॺ्ा ॺ्‍ा
+ज़ ॻ्ा ॻ्‍ा
+ॼ ॼ्ा ॼ्‍ा
+ॾ ॾ्ा ॾ्‍ा
+ॿ ॿ्ा ॿ्‍ा

test/shaping/texts/in-house/shaper-indic/script-devanagari/devanagari-vowel-letters.txt

@@ -0,0 +1,17 @@
+ऄ अॆ
+आ अा
+ई र्इ
+ऊ उु
+ऍ एॅ
+ऎ एॆ
+ऐ एे
+ऑ अॉ आॅ
+ऒ अॊ आॆ
+ओ अो आे
+औ अौ आै
+ॲ अॅ
+ॳ अऺ
+ॴ अऻ आऺ
+ॵ अॏ
+ॶ अॖ
+ॷ अॗ

test/shaping/texts/in-house/shaper-indic/script-gujarati/gujarati-vowel-letter.txt

@@ -0,0 +1,8 @@
+આ અા
+ઍ અૅ
+એ અે
+ઐ અૈ
+ઑ અૉ
+ઓ અો અાૅ
+ઔ અૌ અાૈ
+ૉ ૅા

test/shaping/texts/in-house/shaper-indic/script-gurmukhi/gurmukhi-vowel-letters.txt

@@ -0,0 +1,9 @@
+ਆ ਅਾ
+ਇ ੲਿ
+ਈ ੲੀ
+ਉ ੳੁ
+ਊ ੳੂ
+ਏ ੲੇ
+ਐ ਅੈ
+ਓ ੳੋ
+ਔ ਅੌ

test/shaping/texts/in-house/shaper-indic/script-kannada/kannada-vowel-letters.txt

@@ -0,0 +1,3 @@
+ಊ ಉಾ
+ಔ ಒೌ
+ೠ ಋಾ

test/shaping/texts/in-house/shaper-indic/script-malayalam/malayalam-vowel-letters.txt

@@ -0,0 +1,5 @@
+ഈ ഇൗ
+ഊ ഉൗ
+ഐ എെ
+ഓ ഒാ
+ഔ ഒൗ

test/shaping/texts/in-house/shaper-indic/script-oriya/oriya-vowel-letters.txt

@@ -0,0 +1,3 @@
+ଆ ଅା
+ଐ ଏୗ
+ଔ ଓୗ

test/shaping/texts/in-house/shaper-indic/script-telugu/telugu-vowel-letters.txt

@@ -0,0 +1,5 @@
+ఓ ఒౕ
+ఔ ఒౌ
+ీ ిౕ
+ే ెౕ
+ో ొౕ

util/ansi-print.cc

@@ -71,7 +71,7 @@ struct color_t
 {
   static color_t from_ansi (unsigned int x)
   {
-    color_t c = {(0xFF<<24) | ((0xFF*(x&1))<<16) | ((0xFF*((x >> 1)&1))<<8) | (0xFF*((x >> 2)&1))};
+    color_t c = {(0xFFu<<24) | ((0xFFu*(x&1))<<16) | ((0xFFu*((x >> 1)&1))<<8) | (0xFFu*((x >> 2)&1))};
     return c;
   }
   unsigned int to_ansi (void)
@@ -223,7 +223,7 @@ struct biimage_t
   uint8_t * const data;
 };
 
-const char *
+static const char *
 block_best (const biimage_t &bi, bool *inverse)
 {
   assert (bi.width  <= CELL_W);