added a script to bisect regressions (#4286)
This commit is contained in:
Oliver Stöneberg
GitHub
parent
132ae52428
commit
41d2d1b0b5
|
|
@ -0,0 +1,91 @@
|
|||
# Bisecting
|
||||
|
||||
NOTE: THIS IS WORK IN PROGRESS
|
||||
|
||||
`bisect.sh` is a script to bisect issues.
|
||||
|
||||
## Command
|
||||
|
||||
```
|
||||
./bisect.sh <hash-good> <hash-bad> "<cppcheck-options>"
|
||||
```
|
||||
|
||||
`hash-good` - the last known good commit hash - in case of daca it is the last tagged minor release (not patch release - i.e. 2.x)
|
||||
`hash-bad` - the known bad commit hash - in case of daca the one from the `head-info:` line
|
||||
`cppcheck-options` - the options for the Cppcheck invokation - in case of daca the ones from the `cppcheck-options:` line and the path to the folder/file to scan
|
||||
|
||||
If possible use `main` as the function to test stuff with since it won't emit an `unusedFunction` warning.
|
||||
|
||||
## Bisecting scan time regressions
|
||||
|
||||
We use daca to track differences in scan time. An overview of regressions in scan time can be found at http://cppcheck1.osuosl.org:8000/time_gt.html.
|
||||
|
||||
You need to download the archive as specified by the second line in the output and extract it.
|
||||
|
||||
If the overall scan time regressed you need to specify the whole folder.
|
||||
|
||||
If a timeout (potential hang) was introduced you can simply specify the file from `error: Internal error: Child process crashed with signal 15 [cppcheckError]`.
|
||||
|
||||
|
||||
## Bisecting result regressions
|
||||
|
||||
Results regressions are being bisected based on the `--error-exitcode=` result.
|
||||
If nothing is found the result will be `0` and it is treated as a _good_ commit.
|
||||
If a finding occurs the result will be `1` which is treated as a _bad_ commit.
|
||||
|
||||
### False positive
|
||||
|
||||
Provide a code sample which will trigger the false postive.
|
||||
|
||||
```cpp
|
||||
// cppcheck-suppress unusedFunction
|
||||
static void f()
|
||||
{
|
||||
<code triggering FP>
|
||||
}
|
||||
```
|
||||
|
||||
### False negative
|
||||
|
||||
Provide a code sample which will trigger a `unmatchedSuppression`.
|
||||
|
||||
```cpp
|
||||
// cppcheck-suppress unusedFunction
|
||||
static void f()
|
||||
{
|
||||
// cppcheck-suppress unreadVariable
|
||||
int i;
|
||||
}
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
### Compilation issues:
|
||||
|
||||
- 2.5 and before can only be built with GCC<=10 because of missing includes caused by cleanups within the standard headers. You need to specify `CXX=g++-10`.
|
||||
- 1.88 and 1.89 cannot be compiled:
|
||||
```
|
||||
make: python: No such file or directory
|
||||
```
|
||||
- 1.39 to 1.49 (possibly more versions - 1.54 and up work) cannot be compiled:
|
||||
```
|
||||
lib/mathlib.cpp:70:42: error: invalid conversion from ‘char’ to ‘char**’ [-fpermissive]
|
||||
70 | return std::strtoul(str.c_str(), '\0', 16);
|
||||
| ^~~~
|
||||
| |
|
||||
| char
|
||||
```
|
||||
- some commits between 2.0 and 2.2 cannot be compiled:
|
||||
```
|
||||
cli/cppcheckexecutor.cpp:333:22: error: size of array ‘mytstack’ is not an integral constant-expression
|
||||
333 | static char mytstack[MYSTACKSIZE]= {0}; // alternative stack for signal handler
|
||||
| ^~~~~~~~~~~
|
||||
```
|
||||
RESOLVED: a hot-patch is applied before compilation.
|
||||
- some commits between 1.54 and 1.55 cannot be compiled:
|
||||
```
|
||||
lib/preprocessor.cpp:2103:5: error: ‘errorLogger’ was not declared in this scope; did you mean ‘_errorLogger’?
|
||||
2103 | errorLogger->reportInfo(errmsg);
|
||||
| ^~~~~~~~~~~
|
||||
| _errorLogger
|
||||
```
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
#!/bin/sh
|
||||
|
||||
# TODO: set -e
|
||||
set -x
|
||||
|
||||
# TODO: check parameters
|
||||
hash_good=$1
|
||||
hash_bad=$2
|
||||
options=$3
|
||||
|
||||
# TODO: verify "good" commit happened before "bad" commit
|
||||
|
||||
hang=0
|
||||
expected=""
|
||||
|
||||
script_dir="$(dirname "$(realpath "$0")")"
|
||||
|
||||
# TODO: make configurable
|
||||
bisect_dir=~/.bisect
|
||||
|
||||
mkdir -p "$bisect_dir" || exit 1
|
||||
|
||||
cd "$bisect_dir" || exit 1
|
||||
|
||||
if [ ! -d 'cppcheck' ]; then
|
||||
git clone https://github.com/danmar/cppcheck.git || exit 1
|
||||
fi
|
||||
|
||||
bisect_repo_dir="$bisect_dir/cppcheck"
|
||||
|
||||
cd $bisect_repo_dir || exit 1
|
||||
|
||||
git fetch --all --tags || exit 1
|
||||
|
||||
# clean up in case we previously exited prematurely
|
||||
git restore . || exit 1
|
||||
git clean -df || exit 1
|
||||
|
||||
# reset potentially unfinished bisect - also reverts to 'main' branch
|
||||
git bisect reset || exit 1
|
||||
|
||||
# update `main` branch
|
||||
git pull || exit 1
|
||||
|
||||
# TODO: filter addons, cfg and platforms based on the options
|
||||
# limit to paths which actually affect the analysis
|
||||
git bisect start -- Makefile 'addons/*.py' 'cfg/*.cfg' 'cli/*.cpp' 'cli/*.h' 'externals/**/*.cpp' 'externals/**/*.h' 'lib/*.cpp' 'lib/*.h' platforms tools/matchcompiler.py || exit 1
|
||||
|
||||
git checkout "$hash_good" || exit 1
|
||||
|
||||
if [ $hang -eq 1 ]; then
|
||||
# TODO: exitcode overflow on 255
|
||||
# get expected time from good commit
|
||||
python3 "$script_dir/bisect_hang.py" "$bisect_dir" "$options"
|
||||
elapsed_time=$?
|
||||
else
|
||||
# verify the given commit is actually "good"
|
||||
python3 "$script_dir/bisect_res.py" "$bisect_dir" "$options" "$expected"
|
||||
# shellcheck disable=SC2181
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "given good commit is not actually good"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# mark commit as "good"
|
||||
git bisect good || exit 1
|
||||
|
||||
git checkout "$hash_bad" || exit 1
|
||||
|
||||
# verify the given commit is actually "bad"
|
||||
if [ $hang -eq 1 ]; then
|
||||
python3 "$script_dir/bisect_hang.py" "$bisect_dir" "$options" $elapsed_time
|
||||
else
|
||||
python3 "$script_dir/bisect_res.py" "$bisect_dir" "$options" "$expected"
|
||||
fi
|
||||
|
||||
if [ $? -ne 1 ]; then
|
||||
echo "given bad commit is not actually bad"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# mark commit as "bad"
|
||||
git bisect bad || exit 1
|
||||
|
||||
# perform the actual bisect
|
||||
if [ $hang -eq 1 ]; then
|
||||
git bisect run python3 "$script_dir/bisect_hang.py" "$bisect_dir" "$options" $elapsed_time || exit 1
|
||||
else
|
||||
git bisect run python3 "$script_dir/bisect_res.py" "$bisect_dir" "$options" "$expected" || exit 1
|
||||
fi
|
||||
|
||||
# show the bisect log
|
||||
git bisect log || exit 1
|
||||
|
||||
git bisect reset || exit 1
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
#!/usr/bin/env python
|
||||
import subprocess
|
||||
import os
|
||||
import shutil
|
||||
|
||||
EC_GOOD = 0 # tells bisect that the commit is "good"
|
||||
EC_BAD = 1 # tells bisect that the commit is "bad"
|
||||
EC_SKIP = 125 # tells bisect to skip this commit since it cannot be tested
|
||||
|
||||
def build_cppcheck(bisect_path):
|
||||
commit_hash = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).decode('ascii').strip()
|
||||
install_path = os.path.join(bisect_path, commit_hash)
|
||||
cppcheck_path = os.path.join(install_path, 'cppcheck')
|
||||
if os.path.exists(install_path):
|
||||
print('binary for {} already exists'.format(commit_hash))
|
||||
return cppcheck_path
|
||||
|
||||
bisect_repo_dir = os.path.join(bisect_path, 'cppcheck')
|
||||
|
||||
if os.path.exists(os.path.join(bisect_repo_dir, 'cppcheck')):
|
||||
os.remove(os.path.join(bisect_repo_dir, 'cppcheck'))
|
||||
|
||||
print('patching cli/cppcheckexecutor.cpp')
|
||||
subprocess.check_call(['sed', '-i', 's/SIGSTKSZ/32768/g', os.path.join(bisect_repo_dir, 'cli', 'cppcheckexecutor.cpp')])
|
||||
|
||||
# TODO: older versions do not build because of include changes in libstdc++ - check compiler version and try to use an earlier one
|
||||
# TODO: make jobs configurable
|
||||
# TODO: use "make install"?
|
||||
# TODO: allow CXXFLAGS overrides to workaround compiling issues in older versions
|
||||
print('building {}'.format(commit_hash))
|
||||
subprocess.check_call(['make', '-C', bisect_repo_dir, '-j6', 'MATCHCOMPILER=yes', 'CXXFLAGS=-O2 -w -pipe', '-s'])
|
||||
|
||||
# TODO: remove folder if installation failed
|
||||
print('installing {}'.format(commit_hash))
|
||||
os.mkdir(install_path)
|
||||
if os.path.exists(os.path.join(bisect_repo_dir, 'cfg')):
|
||||
shutil.copytree(os.path.join(bisect_repo_dir, 'cfg'), os.path.join(install_path, 'cfg'))
|
||||
if os.path.exists(os.path.join(bisect_repo_dir, 'platforms')):
|
||||
shutil.copytree(os.path.join(bisect_repo_dir, 'platforms'), os.path.join(install_path, 'platforms'))
|
||||
shutil.copy(os.path.join(bisect_repo_dir, 'cppcheck'), cppcheck_path)
|
||||
|
||||
# reset the patches so the subsequent checkout works
|
||||
print('resetting repo')
|
||||
subprocess.check_call(['git', 'reset', '--hard'])
|
||||
|
||||
return cppcheck_path
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
#!/usr/bin/env python
|
||||
import time
|
||||
import sys
|
||||
|
||||
from bisect_common import *
|
||||
|
||||
# TODO: detect missing file
|
||||
def run(cppcheck_path, options, elapsed_time=None):
|
||||
timeout = None
|
||||
if elapsed_time:
|
||||
timeout = elapsed_time * 2
|
||||
cmd = options.split()
|
||||
cmd.insert(0, cppcheck_path)
|
||||
print('running {}'.format(cppcheck_path))
|
||||
p = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
try:
|
||||
p.communicate(timeout=timeout)
|
||||
if p.returncode != 0:
|
||||
print('error')
|
||||
return None
|
||||
print('done')
|
||||
except subprocess.TimeoutExpired:
|
||||
print('timeout')
|
||||
p.kill()
|
||||
p.communicate()
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
# TODO: check arguments
|
||||
bisect_path = sys.argv[1]
|
||||
options = sys.argv[2]
|
||||
if '--error-exitcode=0' not in options:
|
||||
options += ' --error-exitcode=0'
|
||||
if len(sys.argv) == 4:
|
||||
elapsed_time = float(sys.argv[3])
|
||||
else:
|
||||
elapsed_time = None
|
||||
|
||||
try:
|
||||
cppcheck_path = build_cppcheck(bisect_path)
|
||||
except Exception as e:
|
||||
# TODO: how to persist this so we don't keep compiling these
|
||||
print(e)
|
||||
sys.exit(EC_SKIP)
|
||||
|
||||
if not elapsed_time:
|
||||
t = time.perf_counter()
|
||||
# TODO: handle error result
|
||||
run(cppcheck_path, options)
|
||||
elapsed_time = time.perf_counter() - t
|
||||
print('elapsed_time: {}'.format(elapsed_time))
|
||||
# TODO: write to stdout and redirect all all printing to stderr
|
||||
sys.exit(round(elapsed_time + .5)) # return the time
|
||||
|
||||
t = time.perf_counter()
|
||||
run_res = run(cppcheck_path, options, elapsed_time)
|
||||
run_time = time.perf_counter() - t
|
||||
|
||||
if not elapsed_time:
|
||||
# TODO: handle error result
|
||||
print('elapsed_time: {}'.format(run_time))
|
||||
# TODO: write to stdout and redirect all printing to stderr
|
||||
sys.exit(round(run_time + .5)) # return the time
|
||||
|
||||
if run_res is None:
|
||||
sys.exit(EC_SKIP) # error occured
|
||||
|
||||
if not run_res:
|
||||
sys.exit(EC_BAD) # timeout occured
|
||||
|
||||
print('run_time: {}'.format(run_time))
|
||||
|
||||
sys.exit(EC_GOOD) # no timeout
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
#!/usr/bin/env python
|
||||
import sys
|
||||
|
||||
from bisect_common import *
|
||||
|
||||
# TODO: detect missing file
|
||||
def run(cppcheck_path, options):
|
||||
cmd = options.split()
|
||||
cmd.insert(0, cppcheck_path)
|
||||
print('running {}'.format(cppcheck_path))
|
||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
|
||||
stdout, stderr = p.communicate()
|
||||
# only 0 and 1 are well-defined in this case
|
||||
if p.returncode > 1:
|
||||
print('error')
|
||||
return None, None, None
|
||||
# signals are report as negative exitcode (e.g. SIGSEGV -> -11)
|
||||
if p.returncode < 0:
|
||||
print('crash')
|
||||
return None, None, None
|
||||
print('done')
|
||||
return p.returncode, stderr, stdout
|
||||
|
||||
|
||||
# TODO: check arguments
|
||||
bisect_path = sys.argv[1]
|
||||
options = sys.argv[2]
|
||||
expected = None
|
||||
if len(sys.argv) == 4:
|
||||
expected = sys.argv[3]
|
||||
if len(expected) == 0:
|
||||
expected = None
|
||||
if expected is None:
|
||||
if '--error-exitcode=1' not in options:
|
||||
options += ' --error-exitcode=1'
|
||||
else:
|
||||
if '--error-exitcode=0' not in options:
|
||||
options += ' --error-exitcode=0'
|
||||
|
||||
try:
|
||||
cppcheck_path = build_cppcheck(bisect_path)
|
||||
except Exception as e:
|
||||
# TODO: how to persist this so we don't keep compiling these
|
||||
print(e)
|
||||
sys.exit(EC_SKIP)
|
||||
|
||||
run_ec, run_stderr, run_stdout = run(cppcheck_path, options)
|
||||
|
||||
if expected is None:
|
||||
print(run_ec)
|
||||
print(run_stdout)
|
||||
print(run_stderr)
|
||||
|
||||
# if no ec is set we encountered an unexpected error
|
||||
if run_ec is None:
|
||||
sys.exit(EC_SKIP) # error occured
|
||||
|
||||
# check output for expected string
|
||||
if expected is not None:
|
||||
if (expected not in run_stderr) and (expected not in run_stdout):
|
||||
sys.exit(EC_BAD) # output not found occured
|
||||
|
||||
sys.exit(EC_GOOD) # output found
|
||||
|
||||
sys.exit(run_ec) # return the elapsed time - not a result for bisect
|
||||
Loading…
Reference in New Issue