pcre2/maint/ManyConfigTests

#! /bin/sh

# This is a script for the use of PCRE2 maintainers. It configures and rebuilds
# PCRE2 with a variety of configuration options, and in each case runs the
# tests to ensure that all goes well. Every possible combination would take far
# too long, so we use a representative sample. This script should be run in the
# PCRE2 source directory.

# While debugging, it is sometimes useful to be able to cut out some of the
# tests, in order to run those that are giving errors. The following options
# do this:
#
# -noasan      skip the test that uses -fsanitize=address
# -nousan      skip the test that uses -fsanitize=undefined
# -nodebug     skip the test that uses --enable-debug
# -nojit       skip JIT tests
# -nomain      skip the main set of tests
# -notmp       skip the test in a temporary directory
# -novalgrind  skip the valgrind tests

# The -v option causes a call to 'pcre2test -C' to happen for each
# configuration.

# Some of the tests are automatically skipped when PCRE2 is built with non-Unix
# newline recognition because they don't work. I am hoping to reduce this as
# much as possible in due course.

useasan=1
useusan=1
# Temporarily disabled.
usedebug=1
usejit=1
usemain=1
usetmp=1
usevalgrind=1
verbose=0

while [ $# -gt 0 ] ; do
  case $1 in
    -noasan)       useasan=0;;
    -nousan)       useusan=0;; 
    -nodebug)      usedebug=0;; 
    -nojit)        usejit=0;;
    -nomain)       usemain=0;;
    -notmp)        usetmp=0;;
    -novalgrind)   usevalgrind=0;;
    -v)            verbose=1;;
    *)             echo "Unknown option '$1'"; exit 1;;
  esac
  shift
done

# This is in case the caller has set aliases (as I do - PH)

unset cp ls mv rm

# This is a temporary directory for testing out-of-line builds

tmp=/tmp/pcre2testing

# Don't bother with compiler optimization for most tests; it just slows down
# compilation a lot (and running the tests themselves is quick). However, one
# special test turns optimization on, because it can provoke some compiler
# warnings.

CFLAGS="-g"
OFLAGS="-O0"
ISGCC=0

# If the compiler is gcc, add a lot of warning switches.

cc --version >/tmp/pcre2ccversion 2>/dev/null
if [ $? -eq 0 ] && grep GCC /tmp/pcre2ccversion >/dev/null; then
  ISGCC=1
  CFLAGS="$CFLAGS -Wall"
  CFLAGS="$CFLAGS -Wno-overlength-strings"
  CFLAGS="$CFLAGS -Wpointer-arith"
  CFLAGS="$CFLAGS -Wwrite-strings"
  CFLAGS="$CFLAGS -Wundef -Wshadow"
  CFLAGS="$CFLAGS -Wmissing-field-initializers"
  CFLAGS="$CFLAGS -Wunused-parameter"
  CFLAGS="$CFLAGS -Wextra -Wformat"
  CFLAGS="$CFLAGS -Wbad-function-cast"
  CFLAGS="$CFLAGS -Wmissing-declarations"
  CFLAGS="$CFLAGS -Wnested-externs"
  CFLAGS="$CFLAGS -pedantic"
  CFLAGS="$CFLAGS -Wuninitialized"
  CFLAGS="$CFLAGS -Wmissing-prototypes"
  CFLAGS="$CFLAGS -Wstrict-prototypes"
fi
rm -f /tmp/pcre2ccversion

# This function runs a single test with the set of configuration options that
# are in $opts. The source directory must be set in srcdir. The function must
# be defined as "runtest()" not "function runtest()" in order to run on
# Solaris.

runtest()
  {
  rm -f $srcdir/pcre2test $srcdir/pcre2grep $srcdir/pcre2_jit_test
  testcount=`expr $testcount + 1`

  if [ "$opts" = "" ] ; then
    echo "[$testcount/$testtotal] Configuring with: default settings"
  else
    echo "[$testcount/$testtotal] Configuring with:"
    echo "  $opts"
  fi

  CFLAGS="$CFLAGS" \
    $srcdir/configure $opts >/dev/null 2>teststderr

  if [ $? -ne 0 ]; then
    echo " "
    echo "******** Error while configuring ********"
    cat teststderr
    exit 1
  fi

  echo "Making"
  make -j >/dev/null 2>teststderr
  if [ $? -ne 0 -o -s teststderr ]; then
    echo " "
    echo "******** Errors or warnings while making ********"
    echo " "
    cat teststderr
    exit 1
  fi

  if [ $verbose -eq 1 ]; then
    ./pcre2test -C
  fi

  nl=`./pcre2test -C newline`
  if [ "$nl" = "LF" -o "$nl" = "ANY" -o "$nl" = "ANYCRLF" ]; then
    nlok=1
  else
    nlok=0
  fi

  ./pcre2test -C jit >/dev/null
  jit=$?
  ./pcre2test -C pcre2-8 >/dev/null
  pcre2_8=$?

  if [ $nlok -gt 0 ]; then
    echo "Running PCRE2 library tests $withvalgrind"
    $srcdir/RunTest $valgrind >teststdout 2>teststderr
    if [ $? -ne 0 -o -s teststderr ]; then
      echo " "
      echo "**** Test failed ****"
      cat teststderr
      if [ -s teststdout ] ; then cat teststdout; fi
      exit 1
    fi
  else
    echo "Skipping PCRE2 library tests: newline is $nl"
  fi

  if [ $nlok -gt 0 -a $pcre2_8 -gt 0 ]; then
    echo "Running pcre2grep tests $withvalgrind"
    $srcdir/RunGrepTest $valgrind >teststdout 2>teststderr
    if [ $? -ne 0 -o -s teststderr ]; then
      echo " "
      echo "**** Test failed ****"
      cat teststderr
      cat teststdout
      exit 1
    fi
  elif [ $nlok -gt 0 ]; then
    echo "Skipping pcre2grep tests: 8-bit library not compiled"
  else
    echo "Skipping pcre2grep tests: newline is $nl"
  fi

  if [ "$jit" -gt 0 ]; then
    echo "Running JIT regression tests $withvalgrind"
    $cvalgrind $srcdir/pcre2_jit_test >teststdout 2>teststderr
    if [ $? -ne 0 -o -s teststderr ]; then
      echo " "
      echo "**** Test failed ****"
      cat teststderr
      cat teststdout
      exit 1
    fi
  else
    echo "Skipping JIT regression tests: JIT is not enabled"
  fi
  }

# Update the total count whenever a new test is added; it is used to show
# progess as each test is run.

testtotal=`expr 20 \* $usemain + \
  1 \* $usemain \* $usedebug + \
  1 \* $usetmp + \
  1 \* $ISGCC \* $usemain + \
  1 \* $ISGCC \* $usemain \* $useasan + \
  1 \* $ISGCC \* $usemain \* $useusan + \
  13 \* $usejit + \
  \( 3 + 2 \* $usejit \) \* $usevalgrind`
testcount=0

valgrind=
cvalgrind=
withvalgrind=
srcdir=.
export srcdir

if [ $usejit -ne 0 ]; then
  enable_jit=--enable-jit
else
  enable_jit=
fi

# If gcc is in use, run a maximally configured test with -O2, because that can
# throw up warnings that are not detected with -O0. Then run a second test with
# -fsanitize=address, which also may throw up new warnings as well as checking
# things at runtime. Using -fsanitize=address increases the size of stack
# frames by a lot, so run this test with --disable-stack-for-recursion, as
# otherwise RunTest may fail on test 2. Finally, run another test using
# -fsanitize=undefined -std-gnu99 to check for runtime actions that are not
# well defined. However, we also use -fno-sanitize=shift to avoid warnings for
# shifts of negative numbers, which occur in src/pcre2_jit_compile.c.

if [ $ISGCC -ne 0 -a $usemain -ne 0 ]; then
  echo "---------- Maximally configured test with -O2 ----------"
  SAVECFLAGS="$CFLAGS"
  CFLAGS="-O2 $CFLAGS"
  echo "CFLAGS=$CFLAGS"
  opts="--disable-shared $enable_jit --enable-pcre2-16 --enable-pcre2-32"
  runtest
  if [ $useasan -ne 0 ]; then
    echo "---------- Maximally configured test with -fsanitize=address ----------"
    CFLAGS="$OFLAGS $SAVECFLAGS -fsanitize=address"
    echo "CFLAGS=$CFLAGS"
    opts="--disable-shared $enable_jit --disable-stack-for-recursion --enable-pcre2-16 --enable-pcre2-32"
    runtest
  fi
  if [ $useusan -ne 0 ]; then
    echo "------- Maximally configured test with -fsanitize=undefined -fno-sanitize=shift -fno-sanitize=alignment -std=gnu99 -------"
    CFLAGS="$OFLAGS $SAVECFLAGS -fsanitize=undefined -fno-sanitize=shift -fno-sanitize=alignment -std=gnu99"
    echo "CFLAGS=$CFLAGS"
    opts="--disable-shared $enable_jit --disable-stack-for-recursion --enable-pcre2-16 --enable-pcre2-32"
    runtest
  fi
  CFLAGS="$OFLAGS $SAVECFLAGS"
fi

# This set of tests builds PCRE2 and runs the tests with a variety of configure
# options, in the current (source) directory. The empty configuration builds
# with all the default settings. As well as testing that these options work, we
# use --disable-shared or --disable-static except for the default test (which
# builds both) to save a bit of time by building only one version of the
# library for the subsequent tests.

echo "---------- CFLAGS for the remaining tests ----------"
echo "CFLAGS=$CFLAGS"

if [ $usemain -ne 0 ]; then
  if [ $usedebug -ne 0 ]; then
    echo "---------- Maximally configured test with --enable-debug ----------"
    opts="--disable-shared $enable_jit --enable-pcre2-16 --enable-pcre2-32 --enable-debug"
    runtest
  fi   

  echo "---------- Non-JIT tests in the current directory ----------"
  for opts in \
    "" \
    "--disable-static" \
    "--disable-shared" \
    "--disable-unicode --disable-stack-for-recursion --disable-shared" \
    "--disable-stack-for-recursion --disable-shared" \
    "--with-link-size=3 --disable-shared" \
    "--disable-unicode --enable-rebuild-chartables --disable-shared" \
    "--disable-unicode --enable-newline-is-any --disable-shared" \
    "--disable-unicode --enable-newline-is-cr --disable-shared" \
    "--disable-unicode --enable-newline-is-crlf --disable-shared" \
    "--disable-unicode --enable-newline-is-anycrlf --enable-bsr-anycrlf --disable-shared" \
    "--enable-newline-is-any --disable-stack-for-recursion --disable-static" \
    "--disable-unicode --enable-pcre2-16" \
    "--disable-unicode --enable-pcre2-16 --disable-stack-for-recursion --disable-shared" \
    "--enable-pcre2-16 --disable-stack-for-recursion --disable-shared" \
    "--disable-unicode --enable-pcre2-32" \
    "--disable-unicode --enable-pcre2-32 --disable-stack-for-recursion --disable-shared" \
    "--enable-pcre2-32 --disable-stack-for-recursion --disable-shared" \
    "--disable-unicode --enable-pcre2-32 --enable-pcre2-16 --disable-shared" \
    "--disable-unicode --enable-pcre2-32 --enable-pcre2-16 --disable-pcre2-8 --disable-shared"
  do
    runtest
  done
fi

# Now run the JIT tests unless disabled

if [ $usejit -ne 0 ]; then
  echo "---------- JIT tests in the current directory ----------"
  for opts in \
    "--disable-unicode --enable-jit --disable-shared" \
    "--enable-jit --disable-shared" \
    "--enable-jit --with-link-size=3 --disable-shared" \
    "--enable-jit --enable-pcre2-16 --disable-shared" \
    "--disable-unicode --enable-jit --enable-pcre2-16 --disable-pcre2-8 --disable-shared" \
    "--enable-jit --enable-pcre2-16 --disable-pcre2-8 --disable-shared" \
    "--enable-jit --enable-pcre2-16 --with-link-size=3 --disable-shared" \
    "--enable-jit --enable-pcre2-16 --with-link-size=4 --disable-shared" \
    "--enable-jit --enable-pcre2-32 --disable-shared" \
    "--disable-unicode --enable-jit --enable-pcre2-32 --disable-pcre2-8 --disable-shared" \
    "--enable-jit --enable-pcre2-32 --disable-pcre2-8 --disable-shared" \
    "--enable-jit --enable-pcre2-32 --with-link-size=4 --disable-shared" \
    "--enable-jit --enable-pcre2-32 --enable-pcre2-16 --disable-pcre2-8 --enable-newline-is-anycrlf --enable-bsr-anycrlf --disable-shared"
  do
    runtest
  done
fi

# Now re-run some of the tests under valgrind.

if [ $usevalgrind -ne 0 ]; then
  echo "---------- Tests in the current directory using valgrind ----------"
  valgrind=valgrind
  cvalgrind="valgrind -q --smc-check=all"
  withvalgrind="with valgrind"

  for opts in \
    "--disable-stack-for-recursion --disable-shared" \
    "--with-link-size=3 --enable-pcre2-16 --enable-pcre2-32 --disable-shared" \
    "--disable-unicode --disable-shared"
  do
    opts="--enable-valgrind $opts"
    runtest
  done

  if [ $usejit -ne 0 ]; then
    for opts in \
      "--enable-jit --disable-shared" \
      "--enable-jit --enable-pcre2-16 --enable-pcre2-32"
    do
      opts="--enable-valgrind $opts"
      runtest
    done
  fi
fi

valgrind=
cvalgrind=
withvalgrind=

# Clean up the distribution and then do at least one build and test in a
# directory other than the source directory. It doesn't work unless the
# source directory is cleaned up first.

if [ -f Makefile ]; then
  echo "Running 'make distclean'"
  make distclean >/dev/null 2>&1
  if [ $? -ne 0 ]; then
    echo "** 'make distclean' failed"
    exit 1
  fi
fi

echo "---------- Tests in the $tmp directory ----------"
srcdir=`pwd`
export srcdir

if [ ! -e $tmp ]; then
  mkdir $tmp
fi

if [ ! -d $tmp ]; then
  echo "** Failed to create $tmp or it is not a directory"
  exit 1
fi

cd $tmp
if [ $? -ne 0 ]; then
  echo "** Failed to cd to $tmp"
  exit 1
fi

if [ $usetmp -ne 0 ]; then
  for opts in \
    "--disable-shared"
  do
    runtest
  done
fi

echo "Removing $tmp"

rm -rf $tmp

echo "All done"

# End