diff --git a/tools/bisect/README.md b/tools/bisect/README.md index 053c278c2..e4e92a146 100644 --- a/tools/bisect/README.md +++ b/tools/bisect/README.md @@ -85,9 +85,26 @@ Provide the expected error ID (`unreadVariable`) as the `expected` parameter. ## Bisecting scan time regressions -We use daca@home to track differences in scan time. An overview of regressions in scan time can be found at http://cppcheck1.osuosl.org:8000/time_gt.html. +It is also possible to bisect for a regression in scan time. -You need to download the archive as specified by the second line in the output and extract it. +This is done by determinaing the time it took for the "good" commit to finish and setting a timeout twice that size for the calls to determine the "bad" commit. + +To bisect these kinds of regressions you currently need to adjust the `bisect.sh` script and set the `hang` variable to appropriate value:
+`1` - find the commit which started the hang
+`2` - find the commit which resolved the hang
+ +### General notes + +As we are currently using the process exitcode to pass the elapsed time to the script it will not work properly with vey long runtime (>= 255 seconds) as it will overflow. + +In case the run-time before the regression was very short (<= 1 second) you might need to adjust the `elapsed_time` variable in `bisect.sh` to a higher value to avoid potential false positives. +This might also be necessary to determine one of multiple regressions in the commit range. + +After the bisect finished you should take a look at the output and make sure the elpased time of the repective commit looks as expected. + +### daca@home notes + +We use daca@home to track differences in scan time. An overview of regressions in scan time can be found at http://cppcheck1.osuosl.org:8000/time_gt.html. If the overall scan time regressed you need to specify the whole folder. @@ -97,6 +114,8 @@ If a timeout (potential hang) was introduced you can simply specify the file fro ### Bisecting daca@home issues +You need to download the archive as specified by the second line in the output and extract it. + Use the following data as respective parameters: `hash-good` the latest tagged release - the second value from the `cppcheck:` line
diff --git a/tools/bisect/bisect.sh b/tools/bisect/bisect.sh index edb040979..cf0d4cd5b 100644 --- a/tools/bisect/bisect.sh +++ b/tools/bisect/bisect.sh @@ -11,6 +11,9 @@ expected=$4 # TODO: verify "good" commit happened before "bad" commit +# 0 - regular result based bisect +# 1 - find commit which started hang +# 2 - find commit which resolved hang hang=0 script_dir="$(dirname "$(realpath "$0")")" @@ -48,7 +51,7 @@ git bisect start -- Makefile 'addons/*.py' 'cfg/*.cfg' 'cli/*.cpp' 'cli/*.h' 'ex git checkout "$hash_good" || exit 1 -if [ $hang -eq 1 ]; then +if [ $hang -ne 0 ]; then # TODO: exitcode overflow on 255 # get expected time from good commit python3 "$script_dir/bisect_hang.py" "$bisect_dir" "$options" @@ -69,8 +72,8 @@ git bisect good || exit 1 git checkout "$hash_bad" || exit 1 # verify the given commit is actually "bad" -if [ $hang -eq 1 ]; then - python3 "$script_dir/bisect_hang.py" "$bisect_dir" "$options" $elapsed_time +if [ $hang -ne 0 ]; then + python3 "$script_dir/bisect_hang.py" "$bisect_dir" "$options" $elapsed_time $hang else python3 "$script_dir/bisect_res.py" "$bisect_dir" "$options" "$expected" fi @@ -84,8 +87,8 @@ fi git bisect bad || exit 1 # perform the actual bisect -if [ $hang -eq 1 ]; then - git bisect run python3 "$script_dir/bisect_hang.py" "$bisect_dir" "$options" $elapsed_time || exit 1 +if [ $hang -ne 0 ]; then + git bisect run python3 "$script_dir/bisect_hang.py" "$bisect_dir" "$options" $elapsed_time $hang || exit 1 else git bisect run python3 "$script_dir/bisect_res.py" "$bisect_dir" "$options" "$expected" || exit 1 fi diff --git a/tools/bisect/bisect_hang.py b/tools/bisect/bisect_hang.py index fef81ee63..a7283d2ef 100644 --- a/tools/bisect/bisect_hang.py +++ b/tools/bisect/bisect_hang.py @@ -33,10 +33,14 @@ bisect_path = sys.argv[1] options = sys.argv[2] if '--error-exitcode=0' not in options: options += ' --error-exitcode=0' -if len(sys.argv) == 4: +if len(sys.argv) >= 4: elapsed_time = float(sys.argv[3]) else: elapsed_time = None +if len(sys.argv) == 5: + invert = sys.argv[4] == "2" +else: + invert = False try: cppcheck_path = build_cppcheck(bisect_path) @@ -68,8 +72,8 @@ if run_res is None: sys.exit(EC_SKIP) # error occured if not run_res: - sys.exit(EC_BAD) # timeout occured + sys.exit(EC_BAD if not invert else EC_GOOD) # timeout occured print('run_time: {}'.format(run_time)) -sys.exit(EC_GOOD) # no timeout +sys.exit(EC_GOOD if not invert else EC_BAD) # no timeout