triage_version.py: added `--perf` to collect performance data in CSV format (#4833)

* removed old timing tracking code

* tools/triage_py/README.md: updated

* triage_version.py: fixed output when when stderr and stdout are not empty

* triage_version.py: added `--perf` to collect performance data in CSV format

* triage_version.py: added TODO about providing additional options

* triage_version.py: added `--start` to specify the first tag/commit to execute
This commit is contained in:
Oliver Stöneberg 2023-04-30 22:19:04 +02:00 committed by GitHub
parent 5a2c31a41c
commit 4807bffdbf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 49 additions and 165 deletions

View File

@ -1,23 +0,0 @@
#!/bin/bash
#
# Simple script to generate times-tags.log that contains timing information for a range of revisions
# Typically these commands shall be used to get times.txt:
# mkdir src
# cp lib/* src/ <- fill src/ with some source code
# tools/times-tags.sh
# gcc -o tools/times-tags tools/times-tags.c
# tools/times-tags
rm times-tags.txt
for i in $(seq $1 $2);
do
echo "1.$i"
echo "1.$i" >> times-tags.txt
git checkout "1.$i" -b "$i"
make clean
make -j4 > /dev/null
/usr/bin/time -a -o times-tags.txt ./cppcheck sources -q 2> /dev/null
git checkout main
git branch -D "$i"
done

View File

@ -1,58 +0,0 @@
#!/usr/bin/env python3
# Times script using Visual Studio compiler in Windows
#
# This script assumes that you have:
# Python 3
# Visual Studio (script assumes VS2013, manipulate the sed command otherwise)
# Cygwin64 for the sed command
# Command line svn. TortoiseSVN with that feature selected works.
#
# Usage:
# Open VS command prompt.
# cd c:\users\...
# svn checkout https://github.com/danmar/cppcheck/trunk cppcheck-svn
# cd cppcheck-svn
# c:\python34\python.exe times-vs.py rev1:rev2
import subprocess
import glob
import re
import sys
if len(sys.argv) != 2:
print('revisions not specified')
sys.exit(1)
res = re.match(r'([0-9]+):([0-9]+)', sys.argv[1])
if res is None:
print('invalid format, 11111:22222')
sys.exit(1)
rev1 = int(res.group(1))
rev2 = int(res.group(2))
if rev1 > rev2 or rev1 < 10000 or rev2 > 20000 or rev2 - rev1 > 500:
print('range, aborting')
sys.exit(1)
print('Revisions: ' + str(rev1) + ':' + str(rev2))
f = open('results.txt', 'wt')
f.write('\n')
f.close()
for rev in range(rev1, rev2):
subprocess.call(['svn', 'revert', '-R', '.'])
subprocess.call(['svn', 'up', '-r' + str(rev)])
for vcxproj in glob.glob('*/*.vcxproj'):
subprocess.call([r'c:\cygwin64\bin\sed.exe', '-i', 's/140/120/', vcxproj])
subprocess.call('msbuild cppcheck.sln /t:build /p:configuration=Release,platform=x64'.split())
print('Revision:' + str(rev))
p = subprocess.Popen(r'bin\cppcheck.exe src -q --showtime=summary'.split(),
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
comm = p.communicate()
f = open('results.txt', 'at')
f.write('\nREV ' + str(rev) + '\n')
f.write(comm[0].decode('utf-8'))
f.close()

View File

@ -1,47 +0,0 @@
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
static void revncpy(char *dst, const char *src, size_t len)
{
int n = 0;
while (n++<len && *src && *src!=' ' && *src!='\r' && *src!='\n')
*dst++ = *src++;
}
int main()
{
FILE *f = fopen("times.log", "rt");
if (!f)
return 1;
char lines[0xffff][64] = {0};
int n = 0;
float mintime=0.0f, maxtime=0.0f;
char rev[10] = {0};
char line[128] = {0};
while (fgets(line,sizeof(line),f) && n < (sizeof(lines)/sizeof(*lines))) {
if (strncmp(line,"HEAD is now at ", 15) == 0) {
if (rev[0])
sprintf(lines[n++],"%s\t%.1f\t%.1f", rev, mintime, maxtime);
revncpy(rev, line+15, sizeof(rev)-1);
mintime = 0.0f;
maxtime = 0.0f;
}
if (strncmp(line,"Overall time:",13)==0) {
float time = atof(line+14);
if (mintime < 0.1f || time < mintime)
mintime = time;
if (time > maxtime)
maxtime = time;
}
}
while (n > 0)
printf("%s\n", lines[--n]);
fclose(f);
return 0;
}

View File

@ -1,31 +0,0 @@
#!/bin/bash
#
# Simple script to generate times.log that contains timing information for the last <n> revisions
# Typically these commands shall be used to get times.txt:
mkdir -p src || exit 1
cp lib/* src/ || exit 2 # fill src/ with some source code
#NOTE: also try with some files other then from cppcheck!
iterations=4
# if "old" already exists for some reason, do NOT work on current branch but bail out
git checkout -b old || exit
make clean
git reset --hard HEAD > times.log
for i in $(seq 1 50); do
git_head=$(git log -1 --format=%h)
# if build fails, make clean and try again
make SRCDIR=build CXXFLAGS=-O2 -j4 || make clean ; make MATCHCOMPILER=yes CXXFLAGS=-O2 -j4
echo "Run number $i"
for j in $(seq 1 ${iterations}); do
./cppcheck --quiet --showtime=summary --enable=all --inconclusive src 2> /dev/null | tee -a times.log
done
grep "Overall" times.log | tail -${iterations} | sed s/s// | awk -v "i=$i" -v "iterations=$iterations" -v "git_head=$git_head" '{ sum+=$3} END {print "Run " i", "git_head " Average: " sum/iterations}' | tee -a times.log
git reset --hard HEAD^1 | tee -a times.log
done
gcc -o tools/times tools/times.c
tools/times

View File

@ -5,9 +5,10 @@ A script to run a code sample against a given set of Cppcheck versions.
## Usage
```
usage: triage_version.py [-h] [--compare] [--verbose] [--debug] [--debug-warnings] [--check-library] [--timeout TIMEOUT]
[--compact]
dir infile [repo]
usage: triage_version.py [-h] [--compare] [--verbose] [--debug] [--debug-warnings] [--check-library]
[--timeout TIMEOUT] [--compact] [--no-quiet] [--perf] [--start START]
[--no-stderr | --no-stdout]
dir infile [repo]
positional arguments:
dir directory with versioned folders
@ -23,6 +24,11 @@ options:
--check-library passed through to binary if supported
--timeout TIMEOUT the amount of seconds to wait for the analysis to finish
--compact only print versions with changes with --compare
--no-quiet do not specify -q
--perf output duration of execution in seconds (CSV format)
--start START specify the start version/commit
--no-stderr do not display stdout
--no-stdout do not display stderr
```
### Structure of `dir`

View File

@ -3,6 +3,7 @@ import os.path
import subprocess
import sys
import argparse
import time
from packaging.version import Version
@ -18,6 +19,8 @@ parser.add_argument('--check-library', action='store_true', help='passed through
parser.add_argument('--timeout', type=int, default=2, help='the amount of seconds to wait for the analysis to finish')
parser.add_argument('--compact', action='store_true', help='only print versions with changes with --compare')
parser.add_argument('--no-quiet', action='store_true', default=False, help='do not specify -q')
parser.add_argument('--perf', action='store_true', default=False, help='output duration of execution in seconds (CSV format)')
parser.add_argument('--start', default=None, help='specify the start version/commit')
package_group = parser.add_mutually_exclusive_group()
package_group.add_argument('--no-stderr', action='store_true', default=False, help='do not display stdout')
package_group.add_argument('--no-stdout', action='store_true', default=False, help='do not display stderr')
@ -39,6 +42,13 @@ if args.compact:
if not do_compare:
print('error: --compact requires --compare')
sys.exit(1)
if args.perf:
if args.compact:
print('error: --compact has no effect with --perf')
if args.no_stdout:
print('error: --no-stdout has no effect with --perf')
if args.no_stderr:
print('error: --no-stderr has no effect with --perf')
directory = args.dir
input_file = args.infile
@ -91,7 +101,17 @@ if verbose:
last_ec = None
last_out = None
if args.perf:
print('version,time')
start_entry = args.start
for entry in versions:
if start_entry:
if start_entry != entry:
continue
start_entry = None
exe_path = os.path.join(directory, entry)
exe = os.path.join(exe_path, 'cppcheck')
@ -134,16 +154,23 @@ for entry in versions:
else:
# TODO: re-add inconclusive: {callstack}: ({severity}{inconclusive:, inconclusive}) {message
cmd.append('--template={callstack}: ({severity}) {message} [{id}]')
# TODO: how to pass addtional options?
if args.perf:
cmd.append('--error-exitcode=0')
cmd.append(input_file)
if verbose:
print("running '{}'". format(' '.join(cmd)))
if args.perf:
start = time.time_ns()
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=exe_path, universal_newlines=True)
try:
comm = p.communicate(timeout=args.timeout)
if args.perf:
end = time.time_ns()
out = ''
if not args.no_stdout:
out += comm[0]
if not args.no_stderr and not args.no_stderr:
if not args.no_stdout and not args.no_stderr:
out += '\n'
if not args.no_stderr:
out += comm[1]
@ -156,9 +183,19 @@ for entry in versions:
if not do_compare:
if not use_hashes:
print(version)
ver_str = version
else:
print('{} ({})'.format(entry, version))
ver_str = '{} ({})'.format(entry, version)
if args.perf:
if out == "timeout":
data_str = "0.0" # TODO: how to handle these properly?
elif not ec == 0:
continue # skip errors
else:
data_str = '{}'.format((end - start) / 1000.0 / 1000.0 / 1000.0)
print('"{}",{}'.format(ver_str, data_str))
continue
print(ver_str)
print(ec)
print(out)
continue