diff --git a/tools/bisect/README.md b/tools/bisect/README.md new file mode 100644 index 000000000..2c820e2c5 --- /dev/null +++ b/tools/bisect/README.md @@ -0,0 +1,91 @@ +# Bisecting + +NOTE: THIS IS WORK IN PROGRESS + +`bisect.sh` is a script to bisect issues. + +## Command + +``` +./bisect.sh "" +``` + +`hash-good` - the last known good commit hash - in case of daca it is the last tagged minor release (not patch release - i.e. 2.x) +`hash-bad` - the known bad commit hash - in case of daca the one from the `head-info:` line +`cppcheck-options` - the options for the Cppcheck invokation - in case of daca the ones from the `cppcheck-options:` line and the path to the folder/file to scan + +If possible use `main` as the function to test stuff with since it won't emit an `unusedFunction` warning. + +## Bisecting scan time regressions + +We use daca to track differences in scan time. An overview of regressions in scan time can be found at http://cppcheck1.osuosl.org:8000/time_gt.html. + +You need to download the archive as specified by the second line in the output and extract it. + +If the overall scan time regressed you need to specify the whole folder. + +If a timeout (potential hang) was introduced you can simply specify the file from `error: Internal error: Child process crashed with signal 15 [cppcheckError]`. + + +## Bisecting result regressions + +Results regressions are being bisected based on the `--error-exitcode=` result. +If nothing is found the result will be `0` and it is treated as a _good_ commit. +If a finding occurs the result will be `1` which is treated as a _bad_ commit. + +### False positive + +Provide a code sample which will trigger the false postive. + +```cpp +// cppcheck-suppress unusedFunction +static void f() +{ + +} +``` + +### False negative + +Provide a code sample which will trigger a `unmatchedSuppression`. + +```cpp +// cppcheck-suppress unusedFunction +static void f() +{ + // cppcheck-suppress unreadVariable + int i; +} +``` + +## Notes + +### Compilation issues: + +- 2.5 and before can only be built with GCC<=10 because of missing includes caused by cleanups within the standard headers. You need to specify `CXX=g++-10`. +- 1.88 and 1.89 cannot be compiled: +``` +make: python: No such file or directory +``` +- 1.39 to 1.49 (possibly more versions - 1.54 and up work) cannot be compiled: +``` +lib/mathlib.cpp:70:42: error: invalid conversion from ‘char’ to ‘char**’ [-fpermissive] + 70 | return std::strtoul(str.c_str(), '\0', 16); + | ^~~~ + | | + | char +``` +- some commits between 2.0 and 2.2 cannot be compiled: +``` +cli/cppcheckexecutor.cpp:333:22: error: size of array ‘mytstack’ is not an integral constant-expression + 333 | static char mytstack[MYSTACKSIZE]= {0}; // alternative stack for signal handler + | ^~~~~~~~~~~ +``` +RESOLVED: a hot-patch is applied before compilation. +- some commits between 1.54 and 1.55 cannot be compiled: +``` +lib/preprocessor.cpp:2103:5: error: ‘errorLogger’ was not declared in this scope; did you mean ‘_errorLogger’? +2103 | errorLogger->reportInfo(errmsg); +| ^~~~~~~~~~~ +| _errorLogger +``` \ No newline at end of file diff --git a/tools/bisect/bisect.sh b/tools/bisect/bisect.sh new file mode 100644 index 000000000..034410076 --- /dev/null +++ b/tools/bisect/bisect.sh @@ -0,0 +1,96 @@ +#!/bin/sh + +# TODO: set -e +set -x + +# TODO: check parameters +hash_good=$1 +hash_bad=$2 +options=$3 + +# TODO: verify "good" commit happened before "bad" commit + +hang=0 +expected="" + +script_dir="$(dirname "$(realpath "$0")")" + +# TODO: make configurable +bisect_dir=~/.bisect + +mkdir -p "$bisect_dir" || exit 1 + +cd "$bisect_dir" || exit 1 + +if [ ! -d 'cppcheck' ]; then + git clone https://github.com/danmar/cppcheck.git || exit 1 +fi + +bisect_repo_dir="$bisect_dir/cppcheck" + +cd $bisect_repo_dir || exit 1 + +git fetch --all --tags || exit 1 + +# clean up in case we previously exited prematurely +git restore . || exit 1 +git clean -df || exit 1 + +# reset potentially unfinished bisect - also reverts to 'main' branch +git bisect reset || exit 1 + +# update `main` branch +git pull || exit 1 + +# TODO: filter addons, cfg and platforms based on the options +# limit to paths which actually affect the analysis +git bisect start -- Makefile 'addons/*.py' 'cfg/*.cfg' 'cli/*.cpp' 'cli/*.h' 'externals/**/*.cpp' 'externals/**/*.h' 'lib/*.cpp' 'lib/*.h' platforms tools/matchcompiler.py || exit 1 + +git checkout "$hash_good" || exit 1 + +if [ $hang -eq 1 ]; then + # TODO: exitcode overflow on 255 + # get expected time from good commit + python3 "$script_dir/bisect_hang.py" "$bisect_dir" "$options" + elapsed_time=$? +else + # verify the given commit is actually "good" + python3 "$script_dir/bisect_res.py" "$bisect_dir" "$options" "$expected" + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + echo "given good commit is not actually good" + exit 1 + fi +fi + +# mark commit as "good" +git bisect good || exit 1 + +git checkout "$hash_bad" || exit 1 + +# verify the given commit is actually "bad" +if [ $hang -eq 1 ]; then + python3 "$script_dir/bisect_hang.py" "$bisect_dir" "$options" $elapsed_time +else + python3 "$script_dir/bisect_res.py" "$bisect_dir" "$options" "$expected" +fi + +if [ $? -ne 1 ]; then + echo "given bad commit is not actually bad" + exit 1 +fi + +# mark commit as "bad" +git bisect bad || exit 1 + +# perform the actual bisect +if [ $hang -eq 1 ]; then + git bisect run python3 "$script_dir/bisect_hang.py" "$bisect_dir" "$options" $elapsed_time || exit 1 +else + git bisect run python3 "$script_dir/bisect_res.py" "$bisect_dir" "$options" "$expected" || exit 1 +fi + +# show the bisect log +git bisect log || exit 1 + +git bisect reset || exit 1 \ No newline at end of file diff --git a/tools/bisect/bisect_common.py b/tools/bisect/bisect_common.py new file mode 100644 index 000000000..424a52a7a --- /dev/null +++ b/tools/bisect/bisect_common.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +import subprocess +import os +import shutil + +EC_GOOD = 0 # tells bisect that the commit is "good" +EC_BAD = 1 # tells bisect that the commit is "bad" +EC_SKIP = 125 # tells bisect to skip this commit since it cannot be tested + +def build_cppcheck(bisect_path): + commit_hash = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).decode('ascii').strip() + install_path = os.path.join(bisect_path, commit_hash) + cppcheck_path = os.path.join(install_path, 'cppcheck') + if os.path.exists(install_path): + print('binary for {} already exists'.format(commit_hash)) + return cppcheck_path + + bisect_repo_dir = os.path.join(bisect_path, 'cppcheck') + + if os.path.exists(os.path.join(bisect_repo_dir, 'cppcheck')): + os.remove(os.path.join(bisect_repo_dir, 'cppcheck')) + + print('patching cli/cppcheckexecutor.cpp') + subprocess.check_call(['sed', '-i', 's/SIGSTKSZ/32768/g', os.path.join(bisect_repo_dir, 'cli', 'cppcheckexecutor.cpp')]) + + # TODO: older versions do not build because of include changes in libstdc++ - check compiler version and try to use an earlier one + # TODO: make jobs configurable + # TODO: use "make install"? + # TODO: allow CXXFLAGS overrides to workaround compiling issues in older versions + print('building {}'.format(commit_hash)) + subprocess.check_call(['make', '-C', bisect_repo_dir, '-j6', 'MATCHCOMPILER=yes', 'CXXFLAGS=-O2 -w -pipe', '-s']) + + # TODO: remove folder if installation failed + print('installing {}'.format(commit_hash)) + os.mkdir(install_path) + if os.path.exists(os.path.join(bisect_repo_dir, 'cfg')): + shutil.copytree(os.path.join(bisect_repo_dir, 'cfg'), os.path.join(install_path, 'cfg')) + if os.path.exists(os.path.join(bisect_repo_dir, 'platforms')): + shutil.copytree(os.path.join(bisect_repo_dir, 'platforms'), os.path.join(install_path, 'platforms')) + shutil.copy(os.path.join(bisect_repo_dir, 'cppcheck'), cppcheck_path) + + # reset the patches so the subsequent checkout works + print('resetting repo') + subprocess.check_call(['git', 'reset', '--hard']) + + return cppcheck_path \ No newline at end of file diff --git a/tools/bisect/bisect_hang.py b/tools/bisect/bisect_hang.py new file mode 100644 index 000000000..fef81ee63 --- /dev/null +++ b/tools/bisect/bisect_hang.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +import time +import sys + +from bisect_common import * + +# TODO: detect missing file +def run(cppcheck_path, options, elapsed_time=None): + timeout = None + if elapsed_time: + timeout = elapsed_time * 2 + cmd = options.split() + cmd.insert(0, cppcheck_path) + print('running {}'.format(cppcheck_path)) + p = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + p.communicate(timeout=timeout) + if p.returncode != 0: + print('error') + return None + print('done') + except subprocess.TimeoutExpired: + print('timeout') + p.kill() + p.communicate() + return False + + return True + + +# TODO: check arguments +bisect_path = sys.argv[1] +options = sys.argv[2] +if '--error-exitcode=0' not in options: + options += ' --error-exitcode=0' +if len(sys.argv) == 4: + elapsed_time = float(sys.argv[3]) +else: + elapsed_time = None + +try: + cppcheck_path = build_cppcheck(bisect_path) +except Exception as e: + # TODO: how to persist this so we don't keep compiling these + print(e) + sys.exit(EC_SKIP) + +if not elapsed_time: + t = time.perf_counter() + # TODO: handle error result + run(cppcheck_path, options) + elapsed_time = time.perf_counter() - t + print('elapsed_time: {}'.format(elapsed_time)) + # TODO: write to stdout and redirect all all printing to stderr + sys.exit(round(elapsed_time + .5)) # return the time + +t = time.perf_counter() +run_res = run(cppcheck_path, options, elapsed_time) +run_time = time.perf_counter() - t + +if not elapsed_time: + # TODO: handle error result + print('elapsed_time: {}'.format(run_time)) + # TODO: write to stdout and redirect all printing to stderr + sys.exit(round(run_time + .5)) # return the time + +if run_res is None: + sys.exit(EC_SKIP) # error occured + +if not run_res: + sys.exit(EC_BAD) # timeout occured + +print('run_time: {}'.format(run_time)) + +sys.exit(EC_GOOD) # no timeout diff --git a/tools/bisect/bisect_res.py b/tools/bisect/bisect_res.py new file mode 100644 index 000000000..3c6f98d94 --- /dev/null +++ b/tools/bisect/bisect_res.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +import sys + +from bisect_common import * + +# TODO: detect missing file +def run(cppcheck_path, options): + cmd = options.split() + cmd.insert(0, cppcheck_path) + print('running {}'.format(cppcheck_path)) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + stdout, stderr = p.communicate() + # only 0 and 1 are well-defined in this case + if p.returncode > 1: + print('error') + return None, None, None + # signals are report as negative exitcode (e.g. SIGSEGV -> -11) + if p.returncode < 0: + print('crash') + return None, None, None + print('done') + return p.returncode, stderr, stdout + + +# TODO: check arguments +bisect_path = sys.argv[1] +options = sys.argv[2] +expected = None +if len(sys.argv) == 4: + expected = sys.argv[3] +if len(expected) == 0: + expected = None +if expected is None: + if '--error-exitcode=1' not in options: + options += ' --error-exitcode=1' +else: + if '--error-exitcode=0' not in options: + options += ' --error-exitcode=0' + +try: + cppcheck_path = build_cppcheck(bisect_path) +except Exception as e: + # TODO: how to persist this so we don't keep compiling these + print(e) + sys.exit(EC_SKIP) + +run_ec, run_stderr, run_stdout = run(cppcheck_path, options) + +if expected is None: + print(run_ec) +print(run_stdout) +print(run_stderr) + +# if no ec is set we encountered an unexpected error +if run_ec is None: + sys.exit(EC_SKIP) # error occured + +# check output for expected string +if expected is not None: + if (expected not in run_stderr) and (expected not in run_stdout): + sys.exit(EC_BAD) # output not found occured + + sys.exit(EC_GOOD) # output found + +sys.exit(run_ec) # return the elapsed time - not a result for bisect