[valgrind] Use libtool and support run-subset-fuzzer-tests (#1668)

test/fuzzing/Makefile.am

@@ -55,8 +55,8 @@ hb_subset_fuzzer_CPPFLAGS = $(AM_CPPFLAGS)
 hb_subset_fuzzer_DEPENDENCIES = $(top_builddir)/src/libharfbuzz-subset.la
 
 check:
-	EXEEXT="$(EXEEXT)" srcdir="$(srcdir)" builddir="$(builddir)" $(srcdir)/run-shape-fuzzer-tests.py
+	EXEEXT="$(EXEEXT)" srcdir="$(srcdir)" builddir="$(builddir)" LIBTOOL="$(LIBTOOL)" $(srcdir)/run-shape-fuzzer-tests.py
-	EXEEXT="$(EXEEXT)" srcdir="$(srcdir)" builddir="$(builddir)" $(srcdir)/run-subset-fuzzer-tests.py
+	EXEEXT="$(EXEEXT)" srcdir="$(srcdir)" builddir="$(builddir)" LIBTOOL="$(LIBTOOL)" $(srcdir)/run-subset-fuzzer-tests.py
 check-valgrind:
 	$(AM_V_at)RUN_VALGRIND=1 $(MAKE) $(AM_MAKEFLGS) check
 

test/fuzzing/run-shape-fuzzer-tests.py

@@ -67,36 +67,36 @@ please provide it as the first argument to the tool""")
 print ('hb_shape_fuzzer:', hb_shape_fuzzer)
 fails = 0
 
+libtool = os.environ.get('LIBTOOL')
 valgrind = None
 if os.environ.get('RUN_VALGRIND', ''):
 	valgrind = which ('valgrind')
 	if valgrind is None:
 		print ("""Valgrind requested but not found.""")
 		sys.exit (1)
+	if libtool is None:
+		print ("""Valgrind support is currently autotools only and needs libtool but not found.""")
+
 
 parent_path = os.path.join (srcdir, "fonts")
 for file in os.listdir (parent_path):
 	path = os.path.join(parent_path, file)
 
-	text, returncode = cmd ([hb_shape_fuzzer, path])
+	if valgrind:
-	if text.strip ():
+		text, returncode = cmd (libtool.split(' ') + ['--mode=execute', valgrind + ' --leak-check=full --error-exitcode=1', '--', hb_shape_fuzzer, path])
+	else:
+		text, returncode = cmd ([hb_shape_fuzzer, path])
+		if 'error' in text:
+			returncode = 1
+
+	if not valgrind and text.strip ():
 		print (text)
 
-	failed = False
+	if returncode != 0:
-	if returncode != 0 or 'error' in text:
 		print ('failure on %s' % file)
-		failed = True
-
-	if valgrind:
-		text, returncode = cmd ([valgrind, '--error-exitcode=1', '--leak-check=full', hb_shape_fuzzer, path])
-		if returncode:
-			print (text)
-			print ('failure on %s' % file)
-			failed = True
-
-	if failed:
 		fails = fails + 1
 
+
 if fails:
 	print ("%i shape fuzzer related tests failed." % fails)
 	sys.exit (1)

test/fuzzing/run-subset-fuzzer-tests.py

@@ -2,7 +2,54 @@
 
 from __future__ import print_function, division, absolute_import
 
-import sys, os, subprocess
+import sys, os, subprocess, tempfile, threading
+
+
+def which(program):
+	# https://stackoverflow.com/a/377028
+	def is_exe(fpath):
+		return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+	fpath, _ = os.path.split(program)
+	if fpath:
+		if is_exe(program):
+			return program
+	else:
+		for path in os.environ["PATH"].split(os.pathsep):
+			exe_file = os.path.join(path, program)
+			if is_exe(exe_file):
+				return exe_file
+
+	return None
+
+
+def cmd(command):
+	# https://stackoverflow.com/a/4408409
+	# https://stackoverflow.com/a/10012262
+	with tempfile.TemporaryFile() as tempf:
+		p = subprocess.Popen (command, stderr=tempf)
+		is_killed = {'value': False}
+
+		def timeout(p, is_killed):
+			is_killed['value'] = True
+			p.kill()
+		timer = threading.Timer (2, timeout, [p, is_killed])
+
+		try:
+			timer.start()
+			p.wait ()
+			tempf.seek (0)
+			text = tempf.read().decode ("utf-8").strip ()
+			returncode = p.returncode
+		finally:
+			timer.cancel()
+
+		if is_killed['value']:
+			text = 'error: timeout, ' + text
+			returncode = 1
+
+		return text, returncode
+
 
 srcdir = os.environ.get ("srcdir", ".")
 EXEEXT = os.environ.get ("EXEEXT", "")
@@ -20,21 +67,37 @@ please provide it as the first argument to the tool""")
 print ('hb_subset_fuzzer:', hb_subset_fuzzer)
 fails = 0
 
+libtool = os.environ.get('LIBTOOL')
+valgrind = None
+if os.environ.get('RUN_VALGRIND', ''):
+	valgrind = which ('valgrind')
+	if valgrind is None:
+		print ("""Valgrind requested but not found.""")
+		sys.exit (1)
+	if libtool is None:
+		print ("""Valgrind support is currently autotools only and needs libtool but not found.""")
+
+
 def run_dir (parent_path):
 	global fails
 	for file in os.listdir (parent_path):
 		path = os.path.join(parent_path, file)
 
 		print ("running subset fuzzer against %s" % path)
-		p = subprocess.Popen ([hb_subset_fuzzer, path])
+		if valgrind:
+			text, returncode = cmd (libtool.split(' ') + ['--mode=execute', valgrind + ' --leak-check=full --show-leak-kinds=all --error-exitcode=1', '--', hb_subset_fuzzer, path])
+		else:
+			text, returncode = cmd ([hb_subset_fuzzer, path])
+			if 'error' in text:
+				returncode = 1
 
-		if p.wait () != 0:
+		if not valgrind and text.strip ():
+			print (text)
+
+		if returncode != 0:
 			print ("failed for %s" % path)
 			fails = fails + 1
 
-		if p.wait () != 0:
-			print ("failed for %s" % path)
-			fails = fails + 1
 
 run_dir (os.path.join (srcdir, "..", "subset", "data", "fonts"))
 # TODO running these tests very slow tests.  Fix and re-enable