From 4807bffdbffe8f7b8cb33478cf2c7e558aa76932 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Sun, 30 Apr 2023 22:19:04 +0200 Subject: [PATCH] triage_version.py: added `--perf` to collect performance data in CSV format (#4833) * removed old timing tracking code * tools/triage_py/README.md: updated * triage_version.py: fixed output when when stderr and stdout are not empty * triage_version.py: added `--perf` to collect performance data in CSV format * triage_version.py: added TODO about providing additional options * triage_version.py: added `--start` to specify the first tag/commit to execute --- tools/times-tags.sh | 23 ------------ tools/times-vs.py | 58 ------------------------------- tools/times.c | 47 ------------------------- tools/times.sh | 31 ----------------- tools/triage_py/README.md | 12 +++++-- tools/triage_py/triage_version.py | 43 +++++++++++++++++++++-- 6 files changed, 49 insertions(+), 165 deletions(-) delete mode 100755 tools/times-tags.sh delete mode 100644 tools/times-vs.py delete mode 100644 tools/times.c delete mode 100755 tools/times.sh diff --git a/tools/times-tags.sh b/tools/times-tags.sh deleted file mode 100755 index 524efbd2b..000000000 --- a/tools/times-tags.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# -# Simple script to generate times-tags.log that contains timing information for a range of revisions -# Typically these commands shall be used to get times.txt: -# mkdir src -# cp lib/* src/ <- fill src/ with some source code -# tools/times-tags.sh -# gcc -o tools/times-tags tools/times-tags.c -# tools/times-tags - -rm times-tags.txt - -for i in $(seq $1 $2); -do - echo "1.$i" - echo "1.$i" >> times-tags.txt - git checkout "1.$i" -b "$i" - make clean - make -j4 > /dev/null - /usr/bin/time -a -o times-tags.txt ./cppcheck sources -q 2> /dev/null - git checkout main - git branch -D "$i" -done diff --git a/tools/times-vs.py b/tools/times-vs.py deleted file mode 100644 index cd072fbc7..000000000 --- a/tools/times-vs.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 -# Times script using Visual Studio compiler in Windows -# -# This script assumes that you have: -# Python 3 -# Visual Studio (script assumes VS2013, manipulate the sed command otherwise) -# Cygwin64 for the sed command -# Command line svn. TortoiseSVN with that feature selected works. -# -# Usage: -# Open VS command prompt. -# cd c:\users\... -# svn checkout https://github.com/danmar/cppcheck/trunk cppcheck-svn -# cd cppcheck-svn -# c:\python34\python.exe times-vs.py rev1:rev2 - - -import subprocess -import glob -import re -import sys - -if len(sys.argv) != 2: - print('revisions not specified') - sys.exit(1) - -res = re.match(r'([0-9]+):([0-9]+)', sys.argv[1]) -if res is None: - print('invalid format, 11111:22222') - sys.exit(1) - -rev1 = int(res.group(1)) -rev2 = int(res.group(2)) - -if rev1 > rev2 or rev1 < 10000 or rev2 > 20000 or rev2 - rev1 > 500: - print('range, aborting') - sys.exit(1) - -print('Revisions: ' + str(rev1) + ':' + str(rev2)) - -f = open('results.txt', 'wt') -f.write('\n') -f.close() - -for rev in range(rev1, rev2): - subprocess.call(['svn', 'revert', '-R', '.']) - subprocess.call(['svn', 'up', '-r' + str(rev)]) - for vcxproj in glob.glob('*/*.vcxproj'): - subprocess.call([r'c:\cygwin64\bin\sed.exe', '-i', 's/140/120/', vcxproj]) - subprocess.call('msbuild cppcheck.sln /t:build /p:configuration=Release,platform=x64'.split()) - print('Revision:' + str(rev)) - p = subprocess.Popen(r'bin\cppcheck.exe src -q --showtime=summary'.split(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - comm = p.communicate() - f = open('results.txt', 'at') - f.write('\nREV ' + str(rev) + '\n') - f.write(comm[0].decode('utf-8')) - f.close() diff --git a/tools/times.c b/tools/times.c deleted file mode 100644 index 446227dce..000000000 --- a/tools/times.c +++ /dev/null @@ -1,47 +0,0 @@ -#include -#include -#include - -static void revncpy(char *dst, const char *src, size_t len) -{ - int n = 0; - while (n++ maxtime) - maxtime = time; - } - } - - while (n > 0) - printf("%s\n", lines[--n]); - - fclose(f); - return 0; -} diff --git a/tools/times.sh b/tools/times.sh deleted file mode 100755 index e5e98da63..000000000 --- a/tools/times.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# -# Simple script to generate times.log that contains timing information for the last revisions -# Typically these commands shall be used to get times.txt: -mkdir -p src || exit 1 -cp lib/* src/ || exit 2 # fill src/ with some source code -#NOTE: also try with some files other then from cppcheck! - -iterations=4 - -# if "old" already exists for some reason, do NOT work on current branch but bail out -git checkout -b old || exit - -make clean - -git reset --hard HEAD > times.log - -for i in $(seq 1 50); do - git_head=$(git log -1 --format=%h) - # if build fails, make clean and try again - make SRCDIR=build CXXFLAGS=-O2 -j4 || make clean ; make MATCHCOMPILER=yes CXXFLAGS=-O2 -j4 - echo "Run number $i" - for j in $(seq 1 ${iterations}); do - ./cppcheck --quiet --showtime=summary --enable=all --inconclusive src 2> /dev/null | tee -a times.log - done - grep "Overall" times.log | tail -${iterations} | sed s/s// | awk -v "i=$i" -v "iterations=$iterations" -v "git_head=$git_head" '{ sum+=$3} END {print "Run " i", "git_head " Average: " sum/iterations}' | tee -a times.log - git reset --hard HEAD^1 | tee -a times.log -done - -gcc -o tools/times tools/times.c -tools/times diff --git a/tools/triage_py/README.md b/tools/triage_py/README.md index b162cc92e..2d7e0705b 100644 --- a/tools/triage_py/README.md +++ b/tools/triage_py/README.md @@ -5,9 +5,10 @@ A script to run a code sample against a given set of Cppcheck versions. ## Usage ``` -usage: triage_version.py [-h] [--compare] [--verbose] [--debug] [--debug-warnings] [--check-library] [--timeout TIMEOUT] - [--compact] - dir infile [repo] +usage: triage_version.py [-h] [--compare] [--verbose] [--debug] [--debug-warnings] [--check-library] + [--timeout TIMEOUT] [--compact] [--no-quiet] [--perf] [--start START] + [--no-stderr | --no-stdout] + dir infile [repo] positional arguments: dir directory with versioned folders @@ -23,6 +24,11 @@ options: --check-library passed through to binary if supported --timeout TIMEOUT the amount of seconds to wait for the analysis to finish --compact only print versions with changes with --compare + --no-quiet do not specify -q + --perf output duration of execution in seconds (CSV format) + --start START specify the start version/commit + --no-stderr do not display stdout + --no-stdout do not display stderr ``` ### Structure of `dir` diff --git a/tools/triage_py/triage_version.py b/tools/triage_py/triage_version.py index c72b97beb..42be79ba1 100644 --- a/tools/triage_py/triage_version.py +++ b/tools/triage_py/triage_version.py @@ -3,6 +3,7 @@ import os.path import subprocess import sys import argparse +import time from packaging.version import Version @@ -18,6 +19,8 @@ parser.add_argument('--check-library', action='store_true', help='passed through parser.add_argument('--timeout', type=int, default=2, help='the amount of seconds to wait for the analysis to finish') parser.add_argument('--compact', action='store_true', help='only print versions with changes with --compare') parser.add_argument('--no-quiet', action='store_true', default=False, help='do not specify -q') +parser.add_argument('--perf', action='store_true', default=False, help='output duration of execution in seconds (CSV format)') +parser.add_argument('--start', default=None, help='specify the start version/commit') package_group = parser.add_mutually_exclusive_group() package_group.add_argument('--no-stderr', action='store_true', default=False, help='do not display stdout') package_group.add_argument('--no-stdout', action='store_true', default=False, help='do not display stderr') @@ -39,6 +42,13 @@ if args.compact: if not do_compare: print('error: --compact requires --compare') sys.exit(1) +if args.perf: + if args.compact: + print('error: --compact has no effect with --perf') + if args.no_stdout: + print('error: --no-stdout has no effect with --perf') + if args.no_stderr: + print('error: --no-stderr has no effect with --perf') directory = args.dir input_file = args.infile @@ -91,7 +101,17 @@ if verbose: last_ec = None last_out = None +if args.perf: + print('version,time') + +start_entry = args.start + for entry in versions: + if start_entry: + if start_entry != entry: + continue + start_entry = None + exe_path = os.path.join(directory, entry) exe = os.path.join(exe_path, 'cppcheck') @@ -134,16 +154,23 @@ for entry in versions: else: # TODO: re-add inconclusive: {callstack}: ({severity}{inconclusive:, inconclusive}) {message cmd.append('--template={callstack}: ({severity}) {message} [{id}]') + # TODO: how to pass addtional options? + if args.perf: + cmd.append('--error-exitcode=0') cmd.append(input_file) if verbose: print("running '{}'". format(' '.join(cmd))) + if args.perf: + start = time.time_ns() p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=exe_path, universal_newlines=True) try: comm = p.communicate(timeout=args.timeout) + if args.perf: + end = time.time_ns() out = '' if not args.no_stdout: out += comm[0] - if not args.no_stderr and not args.no_stderr: + if not args.no_stdout and not args.no_stderr: out += '\n' if not args.no_stderr: out += comm[1] @@ -156,9 +183,19 @@ for entry in versions: if not do_compare: if not use_hashes: - print(version) + ver_str = version else: - print('{} ({})'.format(entry, version)) + ver_str = '{} ({})'.format(entry, version) + if args.perf: + if out == "timeout": + data_str = "0.0" # TODO: how to handle these properly? + elif not ec == 0: + continue # skip errors + else: + data_str = '{}'.format((end - start) / 1000.0 / 1000.0 / 1000.0) + print('"{}",{}'.format(ver_str, data_str)) + continue + print(ver_str) print(ec) print(out) continue