Update fuzzing tools and build scripts
This commit is contained in:
parent
21d2d51911
commit
97f8ae52c1
11
configure.ac
11
configure.ac
|
@ -73,6 +73,14 @@ AM_PATH_PYTHON([2.7])
|
|||
|
||||
PKG_PROG_PKG_CONFIG
|
||||
|
||||
AC_ARG_ENABLE([fuzzing],
|
||||
[AS_HELP_STRING([--enable-fuzzing], [Turn on fuzzing build (for developers)])],
|
||||
[enable_fuzzing=yes;
|
||||
AC_SUBST([LIB_FUZZING_ENGINE])
|
||||
AC_DEFINE([FUZZING], 1, [Define to 1 if this is a fuzzing build])
|
||||
], [enable_fuzzing=no; LIB_FUZZING_ENGINE=""])
|
||||
AM_CONDITIONAL([FUZZING], [test "$enable_fuzzing" = "yes"])
|
||||
|
||||
AC_ARG_ENABLE([cfi],
|
||||
[AS_HELP_STRING([--enable-cfi], [Turn on clang's Control Flow Integrity (CFI)])],
|
||||
[
|
||||
|
@ -90,6 +98,7 @@ AC_ARG_ENABLE([ubsan],
|
|||
[AS_HELP_STRING([--enable-ubsan], [Turn on Undefined Behavior Sanitizer (UBSan)])],
|
||||
[
|
||||
if test "$enable_ubsan" = yes; then
|
||||
# Set basic UBSAN compiler flags. Add your own to CFLAGS.
|
||||
CFLAGS=$CFLAGS" -fsanitize=undefined -fno-sanitize-recover=undefined"
|
||||
fi
|
||||
], [ enable_ubsan=no ])
|
||||
|
@ -98,6 +107,7 @@ AC_ARG_ENABLE([asan],
|
|||
[AS_HELP_STRING([--enable-asan], [Turn on Address Sanitizer (ASan)])],
|
||||
[
|
||||
if test "$enable_asan" = yes; then
|
||||
# Set basic ASAN compiler flags. Add your own to CFLAGS.
|
||||
CFLAGS=$CFLAGS" -fsanitize=address -fno-omit-frame-pointer"
|
||||
AX_CHECK_COMPILE_FLAG([-fsanitize-address-use-after-scope], [CFLAGS="$CFLAGS -fsanitize-address-use-after-scope"])
|
||||
fi
|
||||
|
@ -409,4 +419,5 @@ AC_MSG_NOTICE([Summary of build options:
|
|||
Docs: $enable_gtk_doc
|
||||
Man pages: $enable_man
|
||||
Tests: ${TESTS_INFO}
|
||||
Fuzzing build: $enable_fuzzing, $LIB_FUZZING_ENGINE
|
||||
])
|
||||
|
|
|
@ -1,15 +1,18 @@
|
|||
AM_CFLAGS = $(WERROR_CFLAGS) $(WARN_CFLAGS)
|
||||
AM_CPPFLAGS = -I$(top_srcdir)/include -I$(srcdir) -DSRCDIR=\"$(abs_srcdir)\" -DTEST_RUN
|
||||
AM_CPPFLAGS = -I$(top_srcdir)/include -I$(srcdir) -DSRCDIR=\"$(abs_srcdir)\"
|
||||
#AM_LDFLAGS = -static
|
||||
AM_LDFLAGS = -no-install
|
||||
LDADD = ../src/libpsl.la $(LIBICU_LIBS) $(LIBIDN_LIBS) $(LIBIDN2_LIBS)
|
||||
|
||||
if !FUZZING
|
||||
MAIN = main.c fuzzer.h
|
||||
endif
|
||||
|
||||
if WITH_LIBICU
|
||||
XLIBS = $(LIBICU_LIBS)
|
||||
XTYPE = _icu
|
||||
libpsl_icu_fuzzer_SOURCES = libpsl_fuzzer.c main.c fuzzer.h
|
||||
libpsl_icu_load_fuzzer_SOURCES = libpsl_load_fuzzer.c main.c fuzzer.h
|
||||
libpsl_icu_load_dafsa_fuzzer_SOURCES = libpsl_load_dafsa_fuzzer.c main.c fuzzer.h
|
||||
libpsl_icu_fuzzer_SOURCES = libpsl_fuzzer.c $(MAIN)
|
||||
libpsl_icu_load_fuzzer_SOURCES = libpsl_load_fuzzer.c $(MAIN)
|
||||
libpsl_icu_load_dafsa_fuzzer_SOURCES = libpsl_load_dafsa_fuzzer.c $(MAIN)
|
||||
PSL_TESTS = \
|
||||
libpsl_icu_fuzzer$(EXEEXT) \
|
||||
libpsl_icu_load_fuzzer$(EXEEXT) \
|
||||
|
@ -18,9 +21,9 @@ else
|
|||
if WITH_LIBIDN2
|
||||
XLIBS = -lidn2 -lunistring
|
||||
XTYPE = _idn2
|
||||
libpsl_idn2_fuzzer_SOURCES = libpsl_fuzzer.c main.c fuzzer.h
|
||||
libpsl_idn2_load_fuzzer_SOURCES = libpsl_load_fuzzer.c main.c fuzzer.h
|
||||
libpsl_idn2_load_dafsa_fuzzer_SOURCES = libpsl_load_dafsa_fuzzer.c main.c fuzzer.h
|
||||
libpsl_idn2_fuzzer_SOURCES = libpsl_fuzzer.c $(MAIN)
|
||||
libpsl_idn2_load_fuzzer_SOURCES = libpsl_load_fuzzer.c $(MAIN)
|
||||
libpsl_idn2_load_dafsa_fuzzer_SOURCES = libpsl_load_dafsa_fuzzer.c $(MAIN)
|
||||
PSL_TESTS = \
|
||||
libpsl_idn2_fuzzer$(EXEEXT) \
|
||||
libpsl_idn2_load_fuzzer$(EXEEXT) \
|
||||
|
@ -29,9 +32,9 @@ else
|
|||
if WITH_LIBIDN
|
||||
XLIBS = -lidn -lunistring
|
||||
XTYPE = _idn
|
||||
libpsl_idn_fuzzer_SOURCES = libpsl_fuzzer.c main.c fuzzer.h
|
||||
libpsl_idn_load_fuzzer_SOURCES = libpsl_load_fuzzer.c main.c fuzzer.h
|
||||
libpsl_idn_load_dafsa_fuzzer_SOURCES = libpsl_load_dafsa_fuzzer.c main.c fuzzer.h
|
||||
libpsl_idn_fuzzer_SOURCES = libpsl_fuzzer.c $(MAIN)
|
||||
libpsl_idn_load_fuzzer_SOURCES = libpsl_load_fuzzer.c $(MAIN)
|
||||
libpsl_idn_load_dafsa_fuzzer_SOURCES = libpsl_load_dafsa_fuzzer.c $(MAIN)
|
||||
PSL_TESTS = \
|
||||
libpsl_idn_fuzzer$(EXEEXT) \
|
||||
libpsl_idn_load_fuzzer$(EXEEXT) \
|
||||
|
@ -39,9 +42,9 @@ if WITH_LIBIDN
|
|||
else
|
||||
XLIBS =
|
||||
XTYPE =
|
||||
libpsl_fuzzer_SOURCES = libpsl_fuzzer.c main.c fuzzer.h
|
||||
libpsl_load_fuzzer_SOURCES = libpsl_load_fuzzer.c main.c fuzzer.h
|
||||
libpsl_load_dafsa_fuzzer_SOURCES = libpsl_load_dafsa_fuzzer.c main.c fuzzer.h
|
||||
libpsl_fuzzer_SOURCES = libpsl_fuzzer.c $(MAIN)
|
||||
libpsl_load_fuzzer_SOURCES = libpsl_load_fuzzer.c $(MAIN)
|
||||
libpsl_load_dafsa_fuzzer_SOURCES = libpsl_load_dafsa_fuzzer.c $(MAIN)
|
||||
PSL_TESTS = \
|
||||
libpsl_fuzzer$(EXEEXT) \
|
||||
libpsl_load_fuzzer$(EXEEXT) \
|
||||
|
@ -50,7 +53,16 @@ endif
|
|||
endif
|
||||
endif
|
||||
|
||||
check_PROGRAMS = $(PSL_TESTS)
|
||||
if FUZZING
|
||||
bin_PROGRAMS = $(PSL_TESTS)
|
||||
LDADD += $(LIB_FUZZING_ENGINE)
|
||||
AM_LDFLAGS = -no-install
|
||||
else
|
||||
AM_CPPFLAGS += -DTEST_RUN
|
||||
AM_TESTS_ENVIRONMENT = export VALGRIND_TESTS"=@VALGRIND_TESTS@";
|
||||
TESTS = $(PSL_TESTS)
|
||||
check_PROGRAMS = $(PSL_TESTS)
|
||||
endif
|
||||
|
||||
EXTRA_DIST = meson.build
|
||||
|
||||
|
@ -61,7 +73,6 @@ dist-hook:
|
|||
find . -name '*.repro' -exec cp -vr '{}' $(distdir) ';'
|
||||
|
||||
TESTS_ENVIRONMENT = TESTS_VALGRIND="@VALGRIND_ENVIRONMENT@"
|
||||
TESTS = $(PSL_TESTS)
|
||||
|
||||
clean-local:
|
||||
rm -rf *.gc?? *.log lcov coverage.info *_fuzzer *.o
|
||||
|
@ -69,7 +80,7 @@ clean-local:
|
|||
fuzz-coverage: $(PSL_TESTS)
|
||||
find . -name '*_fuzzer' -exec ./coverage.sh '{}' ';'
|
||||
|
||||
CXX ?= clang-5.0
|
||||
CXX ?= clang
|
||||
CXXFLAGS ?= $(CFLAGS)
|
||||
|
||||
oss-fuzz:
|
||||
|
|
|
@ -12,24 +12,58 @@ regression testing with top dir 'make check' or 'make check-valgrind'.
|
|||
The ./configure runs below are for libidn2.
|
||||
To test libicu replace 'libidn2' with 'libicu', to test with
|
||||
libidn replace 'libidn2' by 'libidn'.
|
||||
To test without IDNA libraries replace `--enable-runtime=...` with `--disable-runtime`
|
||||
and replace `--enable-builtin=...` with `--disable-builtin`.
|
||||
|
||||
|
||||
# Running a fuzzer using clang
|
||||
|
||||
Use the following commands on top dir:
|
||||
```
|
||||
export CC=clang-5.0
|
||||
export CFLAGS="-O1 -fno-omit-frame-pointer -gline-tables-only -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize-address-use-after-scope -fsanitize-coverage=trace-pc-guard,trace-cmp"
|
||||
./configure --enable-static --disable-gtk-doc --enable-runtime=libidn2 --enable-builtin=libidn2
|
||||
make clean
|
||||
export CC=clang
|
||||
export LIB_FUZZING_ENGINE="-lFuzzer -lstdc++"
|
||||
export UBSAN_OPTIONS=print_stacktrace=1
|
||||
export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer
|
||||
|
||||
# set flags for clang asan
|
||||
CFLAGS="-O1 -fno-omit-frame-pointer -gline-tables-only -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address,leak,nonnull-attribute -fsanitize-address-use-after-scope -fsanitize-coverage=trace-pc-guard,trace-cmp"
|
||||
|
||||
# set flags for clang ubsan
|
||||
CFLAGS="$CFLAGS -fsanitize=bool,array-bounds,float-divide-by-zero,function,integer-divide-by-zero,return,shift,signed-integer-overflow,vla-bound,vptr,undefined,alignment,null,enum,integer,builtin,float-divide-by-zero,function,object-size,returns-nonnull-attribute,unsigned-integer-overflow,unreachable -fsanitize=fuzzer-no-link"
|
||||
|
||||
# unsigned-integer-overflow is not UB, so recover when we see it.
|
||||
CFLAGS="$CFLAGS -fno-sanitize-recover=all -fsanitize-recover=unsigned-integer-overflow"
|
||||
|
||||
export CFLAGS
|
||||
|
||||
./configure --enable-static --disable-gtk-doc --enable-fuzzing --enable-runtime=libidn2 --enable-builtin=libidn2
|
||||
make -j$(nproc)
|
||||
cd fuzz
|
||||
|
||||
# build and run libpsl_fuzzer
|
||||
# run libpsl_fuzzer
|
||||
./run-clang.sh libpsl_fuzzer
|
||||
```
|
||||
|
||||
|
||||
If you see a crash, then a crash corpora is written that can be used for further
|
||||
investigation. E.g.
|
||||
```
|
||||
==2410==ERROR: AddressSanitizer: heap-use-after-free on address 0x602000004e90 at pc 0x00000049cf9c bp 0x7fffb5543f70 sp 0x7ff
|
||||
fb5543720
|
||||
...
|
||||
Test unit written to ./crash-adc83b19e793491b1c6ea0fd8b46cd9f32e592fc
|
||||
```
|
||||
|
||||
To reproduce the crash:
|
||||
```
|
||||
./libpsl_fuzzer < ./crash-adc83b19e793491b1c6ea0fd8b46cd9f32e592fc
|
||||
```
|
||||
|
||||
You can also copy/move that file into libpsl_fuzzer.repro/
|
||||
and re-build the project without fuzzing for a valgrind run, if you like that better.
|
||||
Just a `./configure` and a `make check-valgrind` should reproduce it.
|
||||
|
||||
|
||||
# Running a fuzzer using AFL
|
||||
|
||||
Use the following commands on top dir:
|
||||
|
@ -67,7 +101,7 @@ directory.
|
|||
|
||||
# Clang CFI instrumentation
|
||||
```
|
||||
CC=clang-5.0 CFLAGS="-B/usr/bin/gold -O0 -fsanitize=cfi -flto -fvisibility=default -fno-sanitize-trap=all" ./configure
|
||||
CC=clang CFLAGS="-B/usr/bin/gold -O0 -fsanitize=cfi -flto -fvisibility=default -fno-sanitize-trap=all" ./configure
|
||||
make clean
|
||||
make
|
||||
make check
|
||||
|
|
|
@ -1,15 +1,8 @@
|
|||
#!/bin/sh -eu
|
||||
|
||||
# First step: In the top directory execute
|
||||
# export CC=clang-5.0
|
||||
# export CFLAGS="-O1 -fno-omit-frame-pointer -gline-tables-only -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address -fsanitize-address-use-after-scope -fsanitize-coverage=trace-pc-guard,trace-cmp"
|
||||
# ./configure --enable-static --disable-gtk-doc --enable-runtime=libidn2 --enable-builtin=libidn2
|
||||
# make clean
|
||||
# make -j$(nproc) LIB_FUZZING_ENGINE="-lFuzzer"
|
||||
# cd fuzz
|
||||
# make -j$(nproc) check
|
||||
# As a first step see README.md and follow the steps under "Running a fuzzer using clang".
|
||||
|
||||
#
|
||||
# You might need 'gsutil' to download new corpora from the Google cloud:
|
||||
# Read the docs at https://github.com/google/oss-fuzz/blob/master/docs/corpora.md
|
||||
# then install 'google-cloud-sdk' and execute 'gcloud init'.
|
||||
# Now 'gsutil' should be ready to use.
|
||||
|
@ -20,13 +13,20 @@ if test -z "$1"; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if ! grep -q FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION Makefile; then
|
||||
echo "The fuzzers haven't been built for fuzzing (maybe for regression testing !?)"
|
||||
echo "Please built regarding README.md and try again."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
fuzzer=$1
|
||||
project=$(echo $1 | cut -d'_' -f1)
|
||||
|
||||
# sync/copy the OSS-Fuzz corpora into the .new directory
|
||||
mkdir -p ${fuzzer}.new
|
||||
cp -p ${fuzzer}.in/* ${fuzzer}.new
|
||||
gsutil -m rsync gs://${project}-corpus.clusterfuzz-external.appspot.com/libFuzzer/${fuzzer} ${fuzzer}.new
|
||||
mkdir -p ${fuzzer}.in ${fuzzer}.new
|
||||
cp -fp ${fuzzer}.in/* ${fuzzer}.new 2>/dev/null || true
|
||||
gsutil cp $(gsutil ls gs://${project}-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/${fuzzer}|tail -n 1) ${fuzzer}.new/
|
||||
(cd ${fuzzer}.new && unzip -o *.zip && mv *.zip ..)
|
||||
|
||||
# create fuzzer target
|
||||
BUILD_ONLY=1 ./run-clang.sh ${fuzzer}
|
||||
|
@ -35,8 +35,9 @@ BUILD_ONLY=1 ./run-clang.sh ${fuzzer}
|
|||
./${fuzzer} -merge=1 ${fuzzer}.in ${fuzzer}.new
|
||||
|
||||
# now clear .new dir and put all corpora there
|
||||
rm -f ${fuzzer}.new/*
|
||||
mv ${fuzzer}.in/* ${fuzzer}.new
|
||||
rm -rf ${fuzzer}.new
|
||||
mv ${fuzzer}.in ${fuzzer}.new
|
||||
mkdir ${fuzzer}.in
|
||||
|
||||
# now merge again (optimizes number of corpora)
|
||||
./${fuzzer} -merge=1 ${fuzzer}.in ${fuzzer}.new
|
||||
|
|
|
@ -35,10 +35,21 @@ if test -z "$1"; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if ! grep -q FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION Makefile; then
|
||||
echo "The fuzzers haven't been built for fuzzing (maybe for regression testing !?)"
|
||||
echo "Please built regarding README.md and try again."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# you'll need ~2GB free memory per worker !
|
||||
fuzzer=$1
|
||||
workers=$(($(nproc) - 1))
|
||||
workers=$(($(nproc) - 0))
|
||||
jobs=$workers
|
||||
|
||||
if test -n "$BUILD_ONLY"; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
case $fuzzer in
|
||||
libpsl_idn2_*)
|
||||
cfile="libpsl_"$(echo $fuzzer|cut -d'_' -f3-)".c"
|
||||
|
@ -54,11 +65,11 @@ case $fuzzer in
|
|||
XLIBS=
|
||||
esac
|
||||
|
||||
clang-5.0 \
|
||||
clang \
|
||||
$CFLAGS -Og -g -I../include -I.. \
|
||||
${cfile} -o ${fuzzer} \
|
||||
-Wl,-Bstatic ../src/.libs/libpsl.a -lFuzzer \
|
||||
-Wl,-Bdynamic $XLIBS -lclang-5.0 -lpthread -lm -lstdc++
|
||||
-Wl,-Bdynamic $XLIBS -lpthread -lm -lstdc++
|
||||
|
||||
if test -n "$BUILD_ONLY"; then
|
||||
exit 0
|
||||
|
|
Loading…
Reference in New Issue