diff --git a/.circleci/config.yml b/.circleci/config.yml
index 8f164384f..3ddc693bb 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -14,14 +14,14 @@ jobs:
       - run: make -Cbuild test
       - run: make -Cbuild install
 
-  fedora:
+  fedora-out-of-tree:
     docker:
       - image: fedora
     steps:
       - checkout
       - run: dnf install -y pkg-config ragel gcc gcc-c++ automake autoconf libtool make which glib2-devel freetype-devel cairo-devel libicu-devel gobject-introspection-devel graphite2-devel redhat-rpm-config || true
-      - run: ./autogen.sh --with-freetype --with-glib --with-gobject --with-cairo --with-icu --with-graphite2
-      - run: make && make check
+      - run: NOCONFIGURE=1 ./autogen.sh --with-freetype --with-glib --with-gobject --with-cairo --with-icu --with-graphite2
+      - run: mkdir build && cd build && ../configure && make && make check
 
   archlinux:
     docker:
@@ -140,7 +140,7 @@ workflows:
   build:
     jobs:
       - oracledeveloperstudio
-      - fedora
+      - fedora-out-of-tree
       - archlinux
       - freebsd9
       - base
diff --git a/test/fuzzing/Makefile.am b/test/fuzzing/Makefile.am
index 6120afce7..a7f736239 100644
--- a/test/fuzzing/Makefile.am
+++ b/test/fuzzing/Makefile.am
@@ -46,23 +46,7 @@ hb_fuzzer_DEPENDENCIES = \
 	lib \
 	$(NULL)
 
-AM_TESTS_ENVIRONMENT = \
-	EXEEXT="$(EXEEXT)"; \
-	export EXEEXT; \
-	srcdir="$(srcdir)"; \
-	export srcdir; \
-	builddir="$(builddir)"; \
-	export builddir; \
-	$(NULL)
-
-if AUTOMAKE_OLDER_THAN_1_13
-TESTS_ENVIRONMENT = \
-	$(AM_TESTS_ENVIRONMENT) \
-	$(TESTS_LOG_COMPILER) \
-	$(NULL)
-endif
-
 check:
-	$(srcdir)/run-fuzzer-tests.py
+	EXEEXT="$(EXEEXT)" srcdir="$(srcdir)" builddir="$(builddir)" $(srcdir)/run-fuzzer-tests.py
 
 -include $(top_srcdir)/git.mk
diff --git a/test/fuzzing/run-fuzzer-tests.py b/test/fuzzing/run-fuzzer-tests.py
index aa615af70..673ee9a96 100755
--- a/test/fuzzing/run-fuzzer-tests.py
+++ b/test/fuzzing/run-fuzzer-tests.py
@@ -8,10 +8,12 @@ EXEEXT = os.environ.get ("EXEEXT", "")
 top_builddir = os.environ.get ("top_builddir", ".")
 hb_fuzzer = os.path.join (top_builddir, "hb-fuzzer" + EXEEXT)
 
-if hb_fuzzer == ".\\hb-fuzzer":
-	hb_fuzzer = "./hb-fuzzer.exe"
-
 if not os.path.exists (hb_fuzzer):
+	if len (sys.argv) == 1 or not os.path.exists (sys.argv[1]):
+		print ("""Failed to find hb-fuzzer binary automatically,
+please provide it as the first argument to the tool""")
+		sys.exit (1)
+
 	hb_fuzzer = sys.argv[1]
 
 print ('hb_fuzzer:', hb_fuzzer)
diff --git a/test/shaping/run-tests.py b/test/shaping/run-tests.py
index 342c006ef..306fcc3e0 100755
--- a/test/shaping/run-tests.py
+++ b/test/shaping/run-tests.py
@@ -31,6 +31,11 @@ hb_shape = os.path.join (top_builddir, utildir, "hb-shape" + EXEEXT)
 args = sys.argv[1:]
 
 if not os.path.exists (hb_shape):
+	if len (sys.argv) == 1 or not os.path.exists (sys.argv[1]):
+		print ("""Failed to find hb-shape binary automatically,
+please provide it as the first argument to the tool""")
+		sys.exit (1)
+
 	hb_shape = args[0]
 	args = args[1:]