triage_version.py: added `--perf` to collect performance data in CSV format (#4833)
* removed old timing tracking code * tools/triage_py/README.md: updated * triage_version.py: fixed output when when stderr and stdout are not empty * triage_version.py: added `--perf` to collect performance data in CSV format * triage_version.py: added TODO about providing additional options * triage_version.py: added `--start` to specify the first tag/commit to execute
This commit is contained in:
Oliver Stöneberg
GitHub
parent
5a2c31a41c
commit
4807bffdbf
|
|
@ -1,23 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Simple script to generate times-tags.log that contains timing information for a range of revisions
|
||||
# Typically these commands shall be used to get times.txt:
|
||||
# mkdir src
|
||||
# cp lib/* src/ <- fill src/ with some source code
|
||||
# tools/times-tags.sh
|
||||
# gcc -o tools/times-tags tools/times-tags.c
|
||||
# tools/times-tags
|
||||
|
||||
rm times-tags.txt
|
||||
|
||||
for i in $(seq $1 $2);
|
||||
do
|
||||
echo "1.$i"
|
||||
echo "1.$i" >> times-tags.txt
|
||||
git checkout "1.$i" -b "$i"
|
||||
make clean
|
||||
make -j4 > /dev/null
|
||||
/usr/bin/time -a -o times-tags.txt ./cppcheck sources -q 2> /dev/null
|
||||
git checkout main
|
||||
git branch -D "$i"
|
||||
done
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# Times script using Visual Studio compiler in Windows
|
||||
#
|
||||
# This script assumes that you have:
|
||||
# Python 3
|
||||
# Visual Studio (script assumes VS2013, manipulate the sed command otherwise)
|
||||
# Cygwin64 for the sed command
|
||||
# Command line svn. TortoiseSVN with that feature selected works.
|
||||
#
|
||||
# Usage:
|
||||
# Open VS command prompt.
|
||||
# cd c:\users\...
|
||||
# svn checkout https://github.com/danmar/cppcheck/trunk cppcheck-svn
|
||||
# cd cppcheck-svn
|
||||
# c:\python34\python.exe times-vs.py rev1:rev2
|
||||
|
||||
|
||||
import subprocess
|
||||
import glob
|
||||
import re
|
||||
import sys
|
||||
|
||||
if len(sys.argv) != 2:
|
||||
print('revisions not specified')
|
||||
sys.exit(1)
|
||||
|
||||
res = re.match(r'([0-9]+):([0-9]+)', sys.argv[1])
|
||||
if res is None:
|
||||
print('invalid format, 11111:22222')
|
||||
sys.exit(1)
|
||||
|
||||
rev1 = int(res.group(1))
|
||||
rev2 = int(res.group(2))
|
||||
|
||||
if rev1 > rev2 or rev1 < 10000 or rev2 > 20000 or rev2 - rev1 > 500:
|
||||
print('range, aborting')
|
||||
sys.exit(1)
|
||||
|
||||
print('Revisions: ' + str(rev1) + ':' + str(rev2))
|
||||
|
||||
f = open('results.txt', 'wt')
|
||||
f.write('\n')
|
||||
f.close()
|
||||
|
||||
for rev in range(rev1, rev2):
|
||||
subprocess.call(['svn', 'revert', '-R', '.'])
|
||||
subprocess.call(['svn', 'up', '-r' + str(rev)])
|
||||
for vcxproj in glob.glob('*/*.vcxproj'):
|
||||
subprocess.call([r'c:\cygwin64\bin\sed.exe', '-i', 's/140/120/', vcxproj])
|
||||
subprocess.call('msbuild cppcheck.sln /t:build /p:configuration=Release,platform=x64'.split())
|
||||
print('Revision:' + str(rev))
|
||||
p = subprocess.Popen(r'bin\cppcheck.exe src -q --showtime=summary'.split(),
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
comm = p.communicate()
|
||||
f = open('results.txt', 'at')
|
||||
f.write('\nREV ' + str(rev) + '\n')
|
||||
f.write(comm[0].decode('utf-8'))
|
||||
f.close()
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
static void revncpy(char *dst, const char *src, size_t len)
|
||||
{
|
||||
int n = 0;
|
||||
while (n++<len && *src && *src!=' ' && *src!='\r' && *src!='\n')
|
||||
*dst++ = *src++;
|
||||
}
|
||||
|
||||
int main()
|
||||
{
|
||||
FILE *f = fopen("times.log", "rt");
|
||||
if (!f)
|
||||
return 1;
|
||||
|
||||
char lines[0xffff][64] = {0};
|
||||
|
||||
int n = 0;
|
||||
float mintime=0.0f, maxtime=0.0f;
|
||||
char rev[10] = {0};
|
||||
char line[128] = {0};
|
||||
|
||||
while (fgets(line,sizeof(line),f) && n < (sizeof(lines)/sizeof(*lines))) {
|
||||
if (strncmp(line,"HEAD is now at ", 15) == 0) {
|
||||
if (rev[0])
|
||||
sprintf(lines[n++],"%s\t%.1f\t%.1f", rev, mintime, maxtime);
|
||||
revncpy(rev, line+15, sizeof(rev)-1);
|
||||
mintime = 0.0f;
|
||||
maxtime = 0.0f;
|
||||
}
|
||||
if (strncmp(line,"Overall time:",13)==0) {
|
||||
float time = atof(line+14);
|
||||
if (mintime < 0.1f || time < mintime)
|
||||
mintime = time;
|
||||
if (time > maxtime)
|
||||
maxtime = time;
|
||||
}
|
||||
}
|
||||
|
||||
while (n > 0)
|
||||
printf("%s\n", lines[--n]);
|
||||
|
||||
fclose(f);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Simple script to generate times.log that contains timing information for the last <n> revisions
|
||||
# Typically these commands shall be used to get times.txt:
|
||||
mkdir -p src || exit 1
|
||||
cp lib/* src/ || exit 2 # fill src/ with some source code
|
||||
#NOTE: also try with some files other then from cppcheck!
|
||||
|
||||
iterations=4
|
||||
|
||||
# if "old" already exists for some reason, do NOT work on current branch but bail out
|
||||
git checkout -b old || exit
|
||||
|
||||
make clean
|
||||
|
||||
git reset --hard HEAD > times.log
|
||||
|
||||
for i in $(seq 1 50); do
|
||||
git_head=$(git log -1 --format=%h)
|
||||
# if build fails, make clean and try again
|
||||
make SRCDIR=build CXXFLAGS=-O2 -j4 || make clean ; make MATCHCOMPILER=yes CXXFLAGS=-O2 -j4
|
||||
echo "Run number $i"
|
||||
for j in $(seq 1 ${iterations}); do
|
||||
./cppcheck --quiet --showtime=summary --enable=all --inconclusive src 2> /dev/null | tee -a times.log
|
||||
done
|
||||
grep "Overall" times.log | tail -${iterations} | sed s/s// | awk -v "i=$i" -v "iterations=$iterations" -v "git_head=$git_head" '{ sum+=$3} END {print "Run " i", "git_head " Average: " sum/iterations}' | tee -a times.log
|
||||
git reset --hard HEAD^1 | tee -a times.log
|
||||
done
|
||||
|
||||
gcc -o tools/times tools/times.c
|
||||
tools/times
|
||||
|
|
@ -5,9 +5,10 @@ A script to run a code sample against a given set of Cppcheck versions.
|
|||
## Usage
|
||||
|
||||
```
|
||||
usage: triage_version.py [-h] [--compare] [--verbose] [--debug] [--debug-warnings] [--check-library] [--timeout TIMEOUT]
|
||||
[--compact]
|
||||
dir infile [repo]
|
||||
usage: triage_version.py [-h] [--compare] [--verbose] [--debug] [--debug-warnings] [--check-library]
|
||||
[--timeout TIMEOUT] [--compact] [--no-quiet] [--perf] [--start START]
|
||||
[--no-stderr | --no-stdout]
|
||||
dir infile [repo]
|
||||
|
||||
positional arguments:
|
||||
dir directory with versioned folders
|
||||
|
|
@ -23,6 +24,11 @@ options:
|
|||
--check-library passed through to binary if supported
|
||||
--timeout TIMEOUT the amount of seconds to wait for the analysis to finish
|
||||
--compact only print versions with changes with --compare
|
||||
--no-quiet do not specify -q
|
||||
--perf output duration of execution in seconds (CSV format)
|
||||
--start START specify the start version/commit
|
||||
--no-stderr do not display stdout
|
||||
--no-stdout do not display stderr
|
||||
```
|
||||
|
||||
### Structure of `dir`
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import os.path
|
|||
import subprocess
|
||||
import sys
|
||||
import argparse
|
||||
import time
|
||||
|
||||
from packaging.version import Version
|
||||
|
||||
|
|
@ -18,6 +19,8 @@ parser.add_argument('--check-library', action='store_true', help='passed through
|
|||
parser.add_argument('--timeout', type=int, default=2, help='the amount of seconds to wait for the analysis to finish')
|
||||
parser.add_argument('--compact', action='store_true', help='only print versions with changes with --compare')
|
||||
parser.add_argument('--no-quiet', action='store_true', default=False, help='do not specify -q')
|
||||
parser.add_argument('--perf', action='store_true', default=False, help='output duration of execution in seconds (CSV format)')
|
||||
parser.add_argument('--start', default=None, help='specify the start version/commit')
|
||||
package_group = parser.add_mutually_exclusive_group()
|
||||
package_group.add_argument('--no-stderr', action='store_true', default=False, help='do not display stdout')
|
||||
package_group.add_argument('--no-stdout', action='store_true', default=False, help='do not display stderr')
|
||||
|
|
@ -39,6 +42,13 @@ if args.compact:
|
|||
if not do_compare:
|
||||
print('error: --compact requires --compare')
|
||||
sys.exit(1)
|
||||
if args.perf:
|
||||
if args.compact:
|
||||
print('error: --compact has no effect with --perf')
|
||||
if args.no_stdout:
|
||||
print('error: --no-stdout has no effect with --perf')
|
||||
if args.no_stderr:
|
||||
print('error: --no-stderr has no effect with --perf')
|
||||
|
||||
directory = args.dir
|
||||
input_file = args.infile
|
||||
|
|
@ -91,7 +101,17 @@ if verbose:
|
|||
last_ec = None
|
||||
last_out = None
|
||||
|
||||
if args.perf:
|
||||
print('version,time')
|
||||
|
||||
start_entry = args.start
|
||||
|
||||
for entry in versions:
|
||||
if start_entry:
|
||||
if start_entry != entry:
|
||||
continue
|
||||
start_entry = None
|
||||
|
||||
exe_path = os.path.join(directory, entry)
|
||||
exe = os.path.join(exe_path, 'cppcheck')
|
||||
|
||||
|
|
@ -134,16 +154,23 @@ for entry in versions:
|
|||
else:
|
||||
# TODO: re-add inconclusive: {callstack}: ({severity}{inconclusive:, inconclusive}) {message
|
||||
cmd.append('--template={callstack}: ({severity}) {message} [{id}]')
|
||||
# TODO: how to pass addtional options?
|
||||
if args.perf:
|
||||
cmd.append('--error-exitcode=0')
|
||||
cmd.append(input_file)
|
||||
if verbose:
|
||||
print("running '{}'". format(' '.join(cmd)))
|
||||
if args.perf:
|
||||
start = time.time_ns()
|
||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=exe_path, universal_newlines=True)
|
||||
try:
|
||||
comm = p.communicate(timeout=args.timeout)
|
||||
if args.perf:
|
||||
end = time.time_ns()
|
||||
out = ''
|
||||
if not args.no_stdout:
|
||||
out += comm[0]
|
||||
if not args.no_stderr and not args.no_stderr:
|
||||
if not args.no_stdout and not args.no_stderr:
|
||||
out += '\n'
|
||||
if not args.no_stderr:
|
||||
out += comm[1]
|
||||
|
|
@ -156,9 +183,19 @@ for entry in versions:
|
|||
|
||||
if not do_compare:
|
||||
if not use_hashes:
|
||||
print(version)
|
||||
ver_str = version
|
||||
else:
|
||||
print('{} ({})'.format(entry, version))
|
||||
ver_str = '{} ({})'.format(entry, version)
|
||||
if args.perf:
|
||||
if out == "timeout":
|
||||
data_str = "0.0" # TODO: how to handle these properly?
|
||||
elif not ec == 0:
|
||||
continue # skip errors
|
||||
else:
|
||||
data_str = '{}'.format((end - start) / 1000.0 / 1000.0 / 1000.0)
|
||||
print('"{}",{}'.format(ver_str, data_str))
|
||||
continue
|
||||
print(ver_str)
|
||||
print(ec)
|
||||
print(out)
|
||||
continue
|
||||
|
|
|
|||
Loading…
Reference in New Issue