harfbuzz/test/shaping/hb_test_tools.py

573 lines
14 KiB
Python
Raw Normal View History

#!/usr/bin/env python
from __future__ import print_function
import sys, os, re, difflib, unicodedata, errno, cgi
2012-05-08 19:38:49 +02:00
from itertools import *
diff_symbols = "-+=*&^%$#@!~/"
diff_colors = ['red', 'green', 'blue']
try:
unichr = unichr
if sys.maxunicode < 0x10FFFF:
# workarounds for Python 2 "narrow" builds with UCS2-only support.
_narrow_unichr = unichr
def unichr(i):
"""
Return the unicode character whose Unicode code is the integer 'i'.
The valid range is 0 to 0x10FFFF inclusive.
>>> _narrow_unichr(0xFFFF + 1)
Traceback (most recent call last):
File "<stdin>", line 1, in ?
ValueError: unichr() arg not in range(0x10000) (narrow Python build)
>>> unichr(0xFFFF + 1) == u'\U00010000'
True
>>> unichr(1114111) == u'\U0010FFFF'
True
>>> unichr(0x10FFFF + 1)
Traceback (most recent call last):
File "<stdin>", line 1, in ?
ValueError: unichr() arg not in range(0x110000)
"""
try:
return _narrow_unichr(i)
except ValueError:
try:
padded_hex_str = hex(i)[2:].zfill(8)
escape_str = "\\U" + padded_hex_str
return escape_str.decode("unicode-escape")
except UnicodeDecodeError:
raise ValueError('unichr() arg not in range(0x110000)')
except NameError:
unichr = chr
try:
unicode = unicode
except NameError:
unicode = str
def tounicode(s, encoding='ascii', errors='strict'):
if not isinstance(s, unicode):
return s.decode(encoding, errors)
else:
return s
class ColorFormatter:
class Null:
@staticmethod
def start_color (c): return ''
@staticmethod
def end_color (): return ''
@staticmethod
def escape (s): return s
@staticmethod
def newline (): return '\n'
class ANSI:
@staticmethod
def start_color (c):
return {
'red': '\033[41;37;1m',
'green': '\033[42;37;1m',
'blue': '\033[44;37;1m',
}[c]
@staticmethod
def end_color ():
return '\033[m'
@staticmethod
def escape (s): return s
@staticmethod
def newline (): return '\n'
class HTML:
@staticmethod
def start_color (c):
return '<span style="background:%s">' % c
@staticmethod
def end_color ():
return '</span>'
@staticmethod
def escape (s): return cgi.escape (s)
@staticmethod
def newline (): return '<br/>\n'
@staticmethod
def Auto (argv = [], out = sys.stdout):
format = ColorFormatter.ANSI
if "--format" in argv:
argv.remove ("--format")
format = ColorFormatter.ANSI
if "--format=ansi" in argv:
argv.remove ("--format=ansi")
format = ColorFormatter.ANSI
if "--format=html" in argv:
argv.remove ("--format=html")
format = ColorFormatter.HTML
if "--no-format" in argv:
argv.remove ("--no-format")
format = ColorFormatter.Null
return format
class DiffColorizer:
diff_regex = re.compile ('([a-za-z0-9_]*)([^a-za-z0-9_]?)')
def __init__ (self, formatter, colors=diff_colors, symbols=diff_symbols):
self.formatter = formatter
self.colors = colors
self.symbols = symbols
2012-01-22 01:40:30 +01:00
def colorize_lines (self, lines):
lines = (l if l else '' for l in lines)
ss = [self.diff_regex.sub (r'\1\n\2\n', l).splitlines (True) for l in lines]
oo = ["",""]
st = [False, False]
for l in difflib.Differ().compare (*ss):
if l[0] == '?':
continue
if l[0] == ' ':
for i in range(2):
if st[i]:
oo[i] += self.formatter.end_color ()
st[i] = False
oo = [o + self.formatter.escape (l[2:]) for o in oo]
continue
if l[0] in self.symbols:
i = self.symbols.index (l[0])
if not st[i]:
oo[i] += self.formatter.start_color (self.colors[i])
st[i] = True
oo[i] += self.formatter.escape (l[2:])
continue
for i in range(2):
if st[i]:
oo[i] += self.formatter.end_color ()
st[i] = False
oo = [o.replace ('\n', '') for o in oo]
return [s1+s2+self.formatter.newline () for (s1,s2) in zip (self.symbols, oo) if s2]
def colorize_diff (self, f):
lines = [None, None]
for l in f:
if l[0] not in self.symbols:
yield self.formatter.escape (l).replace ('\n', self.formatter.newline ())
continue
i = self.symbols.index (l[0])
if lines[i]:
# Flush
for line in self.colorize_lines (lines):
yield line
lines = [None, None]
lines[i] = l[1:]
if (all (lines)):
# Flush
for line in self.colorize_lines (lines):
yield line
lines = [None, None]
if (any (lines)):
# Flush
for line in self.colorize_lines (lines):
yield line
class ZipDiffer:
@staticmethod
def diff_files (files, symbols=diff_symbols):
files = tuple (files) # in case it's a generator, copy it
2012-01-22 01:15:41 +01:00
try:
for lines in izip_longest (*files):
if all (lines[0] == line for line in lines[1:]):
sys.stdout.writelines ([" ", lines[0]])
2012-01-22 01:15:41 +01:00
continue
for i, l in enumerate (lines):
if l:
sys.stdout.writelines ([symbols[i], l])
2012-01-22 01:15:41 +01:00
except IOError as e:
if e.errno != errno.EPIPE:
print ("%s: %s: %s" % (sys.argv[0], e.filename, e.strerror), file=sys.stderr)
2012-01-22 01:15:41 +01:00
sys.exit (1)
class DiffFilters:
@staticmethod
def filter_failures (f):
2012-05-09 08:16:15 +02:00
for key, lines in DiffHelpers.separate_test_cases (f):
lines = list (lines)
2012-05-09 07:45:17 +02:00
if not DiffHelpers.test_passed (lines):
for l in lines: yield l
2012-05-09 09:54:54 +02:00
class Stat:
def __init__ (self):
self.count = 0
self.freq = 0
def add (self, test):
self.count += 1
self.freq += test.freq
class Stats:
def __init__ (self):
self.passed = Stat ()
self.failed = Stat ()
self.total = Stat ()
def add (self, test):
self.total.add (test)
if test.passed:
self.passed.add (test)
else:
self.failed.add (test)
def mean (self):
return float (self.passed.count) / self.total.count
def variance (self):
return (float (self.passed.count) / self.total.count) * \
(float (self.failed.count) / self.total.count)
def stddev (self):
return self.variance () ** .5
def zscore (self, population):
"""Calculate the standard score.
Population is the Stats for population.
Self is Stats for sample.
Returns larger absolute value if sample is highly unlikely to be random.
Anything outside of -3..+3 is very unlikely to be random.
See: http://en.wikipedia.org/wiki/Standard_score"""
return (self.mean () - population.mean ()) / population.stddev ()
2012-05-09 07:45:17 +02:00
class DiffSinks:
@staticmethod
def print_stat (f):
passed = 0
failed = 0
2012-05-09 09:54:54 +02:00
# XXX port to Stats, but that would really slow us down here
2012-05-09 08:16:15 +02:00
for key, lines in DiffHelpers.separate_test_cases (f):
2012-05-09 07:45:17 +02:00
if DiffHelpers.test_passed (lines):
passed += 1
else:
failed += 1
total = passed + failed
print ("%d out of %d tests passed. %d failed (%g%%)" % (passed, total, failed, 100. * failed / total))
2012-05-09 07:45:17 +02:00
2012-05-09 08:57:29 +02:00
@staticmethod
def print_ngrams (f, ns=(1,2,3)):
gens = tuple (Ngram.generator (n) for n in ns)
2012-05-09 09:54:54 +02:00
allstats = Stats ()
allgrams = {}
2012-05-09 08:57:29 +02:00
for key, lines in DiffHelpers.separate_test_cases (f):
test = Test (lines)
2012-05-09 09:54:54 +02:00
allstats.add (test)
2012-05-09 08:57:29 +02:00
for gen in gens:
2012-05-09 09:54:54 +02:00
for ngram in gen (test.unicodes):
if ngram not in allgrams:
allgrams[ngram] = Stats ()
allgrams[ngram].add (test)
importantgrams = {}
for ngram, stats in allgrams.iteritems ():
if stats.failed.count >= 30: # for statistical reasons
importantgrams[ngram] = stats
allgrams = importantgrams
del importantgrams
for ngram, stats in allgrams.iteritems ():
print ("zscore: %9f failed: %6d passed: %6d ngram: <%s>" % (stats.zscore (allstats), stats.failed.count, stats.passed.count, ','.join ("U+%04X" % u for u in ngram)))
2012-05-09 08:57:29 +02:00
class Test:
def __init__ (self, lines):
2012-05-09 09:54:54 +02:00
self.freq = 1
2012-05-09 08:57:29 +02:00
self.passed = True
self.identifier = None
self.text = None
self.unicodes = None
self.glyphs = None
for l in lines:
symbol = l[0]
if symbol != ' ':
self.passed = False
i = 1
if ':' in l:
i = l.index (':')
if not self.identifier:
self.identifier = l[1:i]
i = i + 2 # Skip colon and space
j = -1
if l[j] == '\n':
j -= 1
brackets = l[i] + l[j]
l = l[i+1:-2]
if brackets == '()':
self.text = l
elif brackets == '<>':
self.unicodes = Unicode.parse (l)
elif brackets == '[]':
# XXX we don't handle failed tests here
self.glyphs = l
class DiffHelpers:
@staticmethod
def separate_test_cases (f):
'''Reads lines from f, and if the lines have identifiers, ie.
have a colon character, groups them by identifier,
yielding lists of all lines with the same identifier.'''
2012-05-09 08:16:15 +02:00
def identifier (l):
if ':' in l[1:]:
return l[1:l.index (':')]
return l
return groupby (f, key=identifier)
2012-05-09 07:45:17 +02:00
@staticmethod
def test_passed (lines):
lines = list (lines)
# XXX This is a hack, but does the job for now.
if any (l.find("space+0|space+0") >= 0 for l in lines if l[0] == '+'): return True
Fixup test failure reporting After we implemented dotted-circle, we were still ignoring any tests that had dottedcircle in it for any of the shapers. That meant that if we wrongly outputted dottedcircle, the test was being ignored. Ouch! Fixing that shows regressions across the board. Most are Uniscribe bugs: NOT inserting dotted-circle when it should. Some are arou machine bugs. This is in fact a nice way to catch Indic-machine deficiencies and when I fix the regressions, our clusters should be much closer to Uniscribe. For now, we regressed from: BENGALI: 353997 out of 354285 tests passed. 288 failed (0.0812905%) DEVANAGARI: 707339 out of 707394 tests passed. 55 failed (0.00777502%) GUJARATI: 366489 out of 366506 tests passed. 17 failed (0.0046384%) GURMUKHI: 60769 out of 60809 tests passed. 40 failed (0.0657797%) KANNADA: 951086 out of 951913 tests passed. 827 failed (0.0868777%) KHMER: 299106 out of 299124 tests passed. 18 failed (0.00601757%) LAO: 53611 out of 53644 tests passed. 33 failed (0.0615167%) MALAYALAM: 1048104 out of 1048416 tests passed. 312 failed (0.0297592%) ORIYA: 42320 out of 42329 tests passed. 9 failed (0.021262%) SINHALA: 271747 out of 271847 tests passed. 100 failed (0.0367854%) TAMIL: 1091837 out of 1091837 tests passed. 0 failed (0%) TELUGU: 970558 out of 970573 tests passed. 15 failed (0.00154548%) TIBETAN: 208469 out of 208469 tests passed. 0 failed (0%) To: BENGALI: 353990 out of 354285 tests passed. 295 failed (0.0832663%) DEVANAGARI: 707315 out of 707394 tests passed. 79 failed (0.0111678%) GUJARATI: 366447 out of 366506 tests passed. 59 failed (0.016098%) GURMUKHI: 60707 out of 60809 tests passed. 102 failed (0.167738%) KANNADA: 951042 out of 951913 tests passed. 871 failed (0.0915%) KHMER: 298962 out of 299124 tests passed. 162 failed (0.0541581%) LAO: 53611 out of 53644 tests passed. 33 failed (0.0615167%) MALAYALAM: 1048074 out of 1048416 tests passed. 342 failed (0.0326206%) ORIYA: 42320 out of 42329 tests passed. 9 failed (0.021262%) SINHALA: 271666 out of 271847 tests passed. 181 failed (0.0665816%) TAMIL: 1091835 out of 1091837 tests passed. 2 failed (0.000183178%) TELUGU: 970553 out of 970573 tests passed. 20 failed (0.00206064%) TIBETAN: 208469 out of 208469 tests passed. 0 failed (0%) Investigating.
2012-09-05 21:50:47 +02:00
if any (l.find("uni25CC") >= 0 for l in lines if l[0] == '+'): return True
if any (l.find("dottedcircle") >= 0 for l in lines if l[0] == '+'): return True
if any (l.find("glyph0") >= 0 for l in lines if l[0] == '+'): return True
2012-10-30 03:42:19 +01:00
if any (l.find("gid0") >= 0 for l in lines if l[0] == '+'): return True
Fixup test failure reporting After we implemented dotted-circle, we were still ignoring any tests that had dottedcircle in it for any of the shapers. That meant that if we wrongly outputted dottedcircle, the test was being ignored. Ouch! Fixing that shows regressions across the board. Most are Uniscribe bugs: NOT inserting dotted-circle when it should. Some are arou machine bugs. This is in fact a nice way to catch Indic-machine deficiencies and when I fix the regressions, our clusters should be much closer to Uniscribe. For now, we regressed from: BENGALI: 353997 out of 354285 tests passed. 288 failed (0.0812905%) DEVANAGARI: 707339 out of 707394 tests passed. 55 failed (0.00777502%) GUJARATI: 366489 out of 366506 tests passed. 17 failed (0.0046384%) GURMUKHI: 60769 out of 60809 tests passed. 40 failed (0.0657797%) KANNADA: 951086 out of 951913 tests passed. 827 failed (0.0868777%) KHMER: 299106 out of 299124 tests passed. 18 failed (0.00601757%) LAO: 53611 out of 53644 tests passed. 33 failed (0.0615167%) MALAYALAM: 1048104 out of 1048416 tests passed. 312 failed (0.0297592%) ORIYA: 42320 out of 42329 tests passed. 9 failed (0.021262%) SINHALA: 271747 out of 271847 tests passed. 100 failed (0.0367854%) TAMIL: 1091837 out of 1091837 tests passed. 0 failed (0%) TELUGU: 970558 out of 970573 tests passed. 15 failed (0.00154548%) TIBETAN: 208469 out of 208469 tests passed. 0 failed (0%) To: BENGALI: 353990 out of 354285 tests passed. 295 failed (0.0832663%) DEVANAGARI: 707315 out of 707394 tests passed. 79 failed (0.0111678%) GUJARATI: 366447 out of 366506 tests passed. 59 failed (0.016098%) GURMUKHI: 60707 out of 60809 tests passed. 102 failed (0.167738%) KANNADA: 951042 out of 951913 tests passed. 871 failed (0.0915%) KHMER: 298962 out of 299124 tests passed. 162 failed (0.0541581%) LAO: 53611 out of 53644 tests passed. 33 failed (0.0615167%) MALAYALAM: 1048074 out of 1048416 tests passed. 342 failed (0.0326206%) ORIYA: 42320 out of 42329 tests passed. 9 failed (0.021262%) SINHALA: 271666 out of 271847 tests passed. 181 failed (0.0665816%) TAMIL: 1091835 out of 1091837 tests passed. 2 failed (0.000183178%) TELUGU: 970553 out of 970573 tests passed. 20 failed (0.00206064%) TIBETAN: 208469 out of 208469 tests passed. 0 failed (0%) Investigating.
2012-09-05 21:50:47 +02:00
if any (l.find("notdef") >= 0 for l in lines if l[0] == '+'): return True
2012-05-09 07:45:17 +02:00
return all (l[0] == ' ' for l in lines)
2012-01-22 02:03:25 +01:00
class FilterHelpers:
2012-01-22 01:55:16 +01:00
@staticmethod
2012-01-22 02:03:25 +01:00
def filter_printer_function (filter_callback):
2012-01-22 01:55:16 +01:00
def printer (f):
2012-01-22 02:03:25 +01:00
for line in filter_callback (f):
print (line)
2012-01-22 01:55:16 +01:00
return printer
2012-01-22 02:03:25 +01:00
@staticmethod
def filter_printer_function_no_newline (filter_callback):
def printer (f):
for line in filter_callback (f):
sys.stdout.writelines ([line])
return printer
2012-05-09 08:57:29 +02:00
class Ngram:
@staticmethod
def generator (n):
def gen (f):
l = []
for x in f:
l.append (x)
if len (l) == n:
yield tuple (l)
l[:1] = []
gen.n = n
return gen
2012-01-22 02:03:25 +01:00
class UtilMains:
2012-01-22 01:55:16 +01:00
@staticmethod
2012-01-21 03:16:34 +01:00
def process_multiple_files (callback, mnemonic = "FILE"):
if "--help" in sys.argv:
print ("Usage: %s %s..." % (sys.argv[0], mnemonic))
sys.exit (1)
2012-01-22 01:15:41 +01:00
try:
files = sys.argv[1:] if len (sys.argv) > 1 else ['-']
for s in files:
2012-01-22 01:15:41 +01:00
callback (FileHelpers.open_file_or_stdin (s))
except IOError as e:
if e.errno != errno.EPIPE:
print ("%s: %s: %s" % (sys.argv[0], e.filename, e.strerror), file=sys.stderr)
2012-01-22 01:15:41 +01:00
sys.exit (1)
@staticmethod
2012-01-21 03:16:34 +01:00
def process_multiple_args (callback, mnemonic):
if len (sys.argv) == 1 or "--help" in sys.argv:
print ("Usage: %s %s..." % (sys.argv[0], mnemonic))
2012-01-21 03:16:34 +01:00
sys.exit (1)
2012-01-22 01:15:41 +01:00
try:
for s in sys.argv[1:]:
callback (s)
except IOError as e:
if e.errno != errno.EPIPE:
print ("%s: %s: %s" % (sys.argv[0], e.filename, e.strerror), file=sys.stderr)
2012-01-22 01:15:41 +01:00
sys.exit (1)
2012-01-21 03:16:34 +01:00
@staticmethod
def filter_multiple_strings_or_stdin (callback, mnemonic, \
separator = " ", \
concat_separator = False):
if "--help" in sys.argv:
print ("Usage:\n %s %s...\nor:\n %s\n\nWhen called with no arguments, input is read from standard input." \
% (sys.argv[0], mnemonic, sys.argv[0]))
sys.exit (1)
2012-01-22 01:15:41 +01:00
try:
if len (sys.argv) == 1:
2012-01-22 01:15:41 +01:00
while (1):
line = sys.stdin.readline ()
if not len (line):
break
2012-01-22 22:21:19 +01:00
if line[-1] == '\n':
line = line[:-1]
print (callback (line))
2012-01-22 01:15:41 +01:00
else:
args = sys.argv[1:]
if concat_separator != False:
args = [concat_separator.join (args)]
print (separator.join (callback (x) for x in (args)))
2012-01-22 01:15:41 +01:00
except IOError as e:
if e.errno != errno.EPIPE:
print ("%s: %s: %s" % (sys.argv[0], e.filename, e.strerror), file=sys.stderr)
2012-01-22 01:15:41 +01:00
sys.exit (1)
class Unicode:
@staticmethod
def decode (s):
return u','.join ("U+%04X" % ord (u) for u in tounicode (s, 'utf-8'))
@staticmethod
2012-05-09 08:57:29 +02:00
def parse (s):
s = re.sub (r"0[xX]", " ", s)
s = re.sub (r"[<+>{},;&#\\xXuUnNiI\n\t]", " ", s)
2014-08-14 01:42:01 +02:00
return [int (x, 16) for x in s.split ()]
2012-05-09 08:57:29 +02:00
@staticmethod
def encode (s):
s = u''.join (unichr (x) for x in Unicode.parse (s))
if sys.version_info[0] == 2: s = s.encode ('utf-8')
return s
shorthands = {
"ZERO WIDTH NON-JOINER": "ZWNJ",
"ZERO WIDTH JOINER": "ZWJ",
"NARROW NO-BREAK SPACE": "NNBSP",
"COMBINING GRAPHEME JOINER": "CGJ",
"LEFT-TO-RIGHT MARK": "LRM",
"RIGHT-TO-LEFT MARK": "RLM",
"LEFT-TO-RIGHT EMBEDDING": "LRE",
"RIGHT-TO-LEFT EMBEDDING": "RLE",
"POP DIRECTIONAL FORMATTING": "PDF",
"LEFT-TO-RIGHT OVERRIDE": "LRO",
"RIGHT-TO-LEFT OVERRIDE": "RLO",
}
@staticmethod
def pretty_name (u):
try:
s = unicodedata.name (u)
except ValueError:
return "XXX"
s = re.sub (".* LETTER ", "", s)
s = re.sub (".* VOWEL SIGN (.*)", r"\1-MATRA", s)
s = re.sub (".* SIGN ", "", s)
s = re.sub (".* COMBINING ", "", s)
if re.match (".* VIRAMA", s):
s = "HALANT"
if s in Unicode.shorthands:
s = Unicode.shorthands[s]
return s
@staticmethod
def pretty_names (s):
s = re.sub (r"[<+>\\uU]", " ", s)
s = re.sub (r"0[xX]", " ", s)
s = [unichr (int (x, 16)) for x in re.split ('[, \n]', s) if len (x)]
2012-01-21 01:32:17 +01:00
return u' + '.join (Unicode.pretty_name (x) for x in s).encode ('utf-8')
2012-01-21 03:16:34 +01:00
class FileHelpers:
@staticmethod
def open_file_or_stdin (f):
if f == '-':
return sys.stdin
return file (f)
2012-01-21 03:16:34 +01:00
class Manifest:
@staticmethod
2012-01-22 01:37:31 +01:00
def read (s, strict = True):
2012-01-21 03:16:34 +01:00
if not os.path.exists (s):
if strict:
print ("%s: %s does not exist" % (sys.argv[0], s), file=sys.stderr)
2012-01-21 03:16:34 +01:00
sys.exit (1)
return
2012-01-22 01:37:31 +01:00
s = os.path.normpath (s)
2012-01-21 03:16:34 +01:00
2012-01-22 01:37:31 +01:00
if os.path.isdir (s):
2012-01-21 03:16:34 +01:00
try:
2012-01-22 01:31:51 +01:00
m = file (os.path.join (s, "MANIFEST"))
2012-01-21 03:16:34 +01:00
items = [x.strip () for x in m.readlines ()]
for f in items:
2012-01-22 01:37:31 +01:00
for p in Manifest.read (os.path.join (s, f)):
yield p
2012-01-21 03:16:34 +01:00
except IOError:
if strict:
print ("%s: %s does not exist" % (sys.argv[0], os.path.join (s, "MANIFEST")), file=sys.stderr)
2012-01-21 03:16:34 +01:00
sys.exit (1)
return
else:
2012-01-22 01:37:31 +01:00
yield s
2012-01-22 01:31:51 +01:00
@staticmethod
def update_recursive (s):
for dirpath, dirnames, filenames in os.walk (s, followlinks=True):
2012-01-22 22:26:49 +01:00
for f in ["MANIFEST", "README", "LICENSE", "COPYING", "AUTHORS", "SOURCES", "ChangeLog"]:
2012-01-22 01:31:51 +01:00
if f in dirnames:
dirnames.remove (f)
if f in filenames:
filenames.remove (f)
dirnames.sort ()
filenames.sort ()
ms = os.path.join (dirpath, "MANIFEST")
print (" GEN %s" % ms)
2012-01-22 01:31:51 +01:00
m = open (ms, "w")
for f in filenames:
print (f, file=m)
2012-01-22 01:31:51 +01:00
for f in dirnames:
print (f, file=m)
2012-01-22 01:31:51 +01:00
for f in dirnames:
Manifest.update_recursive (os.path.join (dirpath, f))
if __name__ == '__main__':
pass