triage_version.py: added `--perf` to collect performance data in CSV format (#4833)

* removed old timing tracking code

* tools/triage_py/README.md: updated

* triage_version.py: fixed output when when stderr and stdout are not empty

* triage_version.py: added `--perf` to collect performance data in CSV format

* triage_version.py: added TODO about providing additional options

* triage_version.py: added `--start` to specify the first tag/commit to execute

tools/times-tags.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-#
-# Simple script to generate times-tags.log that contains timing information for a range of revisions
-# Typically these commands shall be used to get times.txt:
-#   mkdir src
-#   cp lib/* src/   <- fill src/ with some source code
-#   tools/times-tags.sh
-#   gcc -o tools/times-tags tools/times-tags.c
-#   tools/times-tags
-
-rm times-tags.txt
-
-for i in $(seq $1 $2);
-do
-    echo "1.$i"
-    echo "1.$i" >> times-tags.txt
-    git checkout "1.$i" -b "$i"
-    make clean
-    make -j4 > /dev/null
-    /usr/bin/time -a -o times-tags.txt ./cppcheck sources -q 2> /dev/null
-    git checkout main
-    git branch -D "$i"
-done

tools/times-vs.py

@@ -1,58 +0,0 @@
-#!/usr/bin/env python3
-# Times script using Visual Studio compiler in Windows
-#
-# This script assumes that you have:
-# Python 3
-# Visual Studio (script assumes VS2013, manipulate the sed command otherwise)
-# Cygwin64 for the sed command
-# Command line svn. TortoiseSVN with that feature selected works.
-#
-# Usage:
-# Open VS command prompt.
-# cd c:\users\...
-# svn checkout https://github.com/danmar/cppcheck/trunk cppcheck-svn
-# cd cppcheck-svn
-# c:\python34\python.exe times-vs.py rev1:rev2
-
-
-import subprocess
-import glob
-import re
-import sys
-
-if len(sys.argv) != 2:
-    print('revisions not specified')
-    sys.exit(1)
-
-res = re.match(r'([0-9]+):([0-9]+)', sys.argv[1])
-if res is None:
-    print('invalid format, 11111:22222')
-    sys.exit(1)
-
-rev1 = int(res.group(1))
-rev2 = int(res.group(2))
-
-if rev1 > rev2 or rev1 < 10000 or rev2 > 20000 or rev2 - rev1 > 500:
-    print('range, aborting')
-    sys.exit(1)
-
-print('Revisions: ' + str(rev1) + ':' + str(rev2))
-
-f = open('results.txt', 'wt')
-f.write('\n')
-f.close()
-
-for rev in range(rev1, rev2):
-    subprocess.call(['svn', 'revert', '-R',  '.'])
-    subprocess.call(['svn', 'up', '-r' + str(rev)])
-    for vcxproj in glob.glob('*/*.vcxproj'):
-        subprocess.call([r'c:\cygwin64\bin\sed.exe', '-i', 's/140/120/', vcxproj])
-    subprocess.call('msbuild cppcheck.sln /t:build /p:configuration=Release,platform=x64'.split())
-    print('Revision:' + str(rev))
-    p = subprocess.Popen(r'bin\cppcheck.exe src -q --showtime=summary'.split(),
-                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    comm = p.communicate()
-    f = open('results.txt', 'at')
-    f.write('\nREV ' + str(rev) + '\n')
-    f.write(comm[0].decode('utf-8'))
-    f.close()

tools/times.c

@@ -1,47 +0,0 @@
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-static void revncpy(char *dst, const char *src, size_t len)
-{
-    int n = 0;
-    while (n++<len && *src && *src!=' ' && *src!='\r' && *src!='\n')
-        *dst++ = *src++;
-}
-
-int main()
-{
-    FILE *f = fopen("times.log", "rt");
-    if (!f)
-        return 1;
-
-    char lines[0xffff][64] = {0};
-
-    int n = 0;
-    float mintime=0.0f, maxtime=0.0f;
-    char rev[10] = {0};
-    char line[128] = {0};
-
-    while (fgets(line,sizeof(line),f) && n < (sizeof(lines)/sizeof(*lines))) {
-        if (strncmp(line,"HEAD is now at ", 15) == 0) {
-            if (rev[0])
-                sprintf(lines[n++],"%s\t%.1f\t%.1f", rev, mintime, maxtime);
-            revncpy(rev, line+15, sizeof(rev)-1);
-            mintime = 0.0f;
-            maxtime = 0.0f;
-        }
-        if (strncmp(line,"Overall time:",13)==0) {
-            float time = atof(line+14);
-            if (mintime < 0.1f || time < mintime)
-                mintime = time;
-            if (time > maxtime)
-                maxtime = time;
-        }
-    }
-
-    while (n > 0)
-        printf("%s\n", lines[--n]);
-
-    fclose(f);
-    return 0;
-}

tools/times.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-#
-# Simple script to generate times.log that contains timing information for the last <n> revisions
-# Typically these commands shall be used to get times.txt:
-mkdir -p src  || exit 1
-cp lib/* src/ || exit 2  # fill src/ with some source code
-#NOTE: also try with some files other then from cppcheck!
-
-iterations=4
-
-# if "old" already exists for some reason, do NOT work on current branch but bail out
-git checkout -b old || exit
-
-make clean
-
-git reset --hard HEAD > times.log
-
-for i in $(seq 1 50); do
-    git_head=$(git log -1 --format=%h)
-    # if build fails, make clean and try again
-    make SRCDIR=build CXXFLAGS=-O2 -j4 || make clean ; make MATCHCOMPILER=yes CXXFLAGS=-O2 -j4
-    echo "Run number $i"
-    for j in $(seq 1 ${iterations}); do
-        ./cppcheck --quiet --showtime=summary --enable=all --inconclusive src 2> /dev/null | tee -a times.log
-    done
-    grep "Overall" times.log | tail -${iterations} | sed s/s// | awk -v "i=$i" -v "iterations=$iterations" -v "git_head=$git_head"  '{ sum+=$3} END {print "Run " i",  "git_head  "  Average: " sum/iterations}' | tee -a times.log
-    git reset --hard HEAD^1 | tee -a times.log
-done
-
-gcc -o tools/times tools/times.c
-tools/times

tools/triage_py/README.md

@@ -5,8 +5,9 @@ A script to run a code sample against a given set of Cppcheck versions.
 ## Usage
 
 ```
-usage: triage_version.py [-h] [--compare] [--verbose] [--debug] [--debug-warnings] [--check-library] [--timeout TIMEOUT]
+usage: triage_version.py [-h] [--compare] [--verbose] [--debug] [--debug-warnings] [--check-library]
-                 [--compact]
+                         [--timeout TIMEOUT] [--compact] [--no-quiet] [--perf] [--start START]
+                         [--no-stderr | --no-stdout]
                          dir infile [repo]
 
 positional arguments:
@@ -23,6 +24,11 @@ options:
   --check-library    passed through to binary if supported
   --timeout TIMEOUT  the amount of seconds to wait for the analysis to finish
   --compact          only print versions with changes with --compare
+  --no-quiet         do not specify -q
+  --perf             output duration of execution in seconds (CSV format)
+  --start START      specify the start version/commit
+  --no-stderr        do not display stdout
+  --no-stdout        do not display stderr
 ```
 
 ### Structure of `dir`

tools/triage_py/triage_version.py

@@ -3,6 +3,7 @@ import os.path
 import subprocess
 import sys
 import argparse
+import time
 
 from packaging.version import Version
 
@@ -18,6 +19,8 @@ parser.add_argument('--check-library', action='store_true', help='passed through
 parser.add_argument('--timeout', type=int, default=2, help='the amount of seconds to wait for the analysis to finish')
 parser.add_argument('--compact', action='store_true', help='only print versions with changes with --compare')
 parser.add_argument('--no-quiet', action='store_true', default=False, help='do not specify -q')
+parser.add_argument('--perf', action='store_true', default=False, help='output duration of execution in seconds (CSV format)')
+parser.add_argument('--start', default=None, help='specify the start version/commit')
 package_group = parser.add_mutually_exclusive_group()
 package_group.add_argument('--no-stderr', action='store_true', default=False, help='do not display stdout')
 package_group.add_argument('--no-stdout', action='store_true', default=False, help='do not display stderr')
@@ -39,6 +42,13 @@ if args.compact:
     if not do_compare:
         print('error: --compact requires --compare')
         sys.exit(1)
+if args.perf:
+    if args.compact:
+        print('error: --compact has no effect with --perf')
+    if args.no_stdout:
+        print('error: --no-stdout has no effect with --perf')
+    if args.no_stderr:
+        print('error: --no-stderr has no effect with --perf')
 
 directory = args.dir
 input_file = args.infile
@@ -91,7 +101,17 @@ if verbose:
 last_ec = None
 last_out = None
 
+if args.perf:
+    print('version,time')
+
+start_entry = args.start
+
 for entry in versions:
+    if start_entry:
+        if start_entry != entry:
+            continue
+        start_entry = None
+
     exe_path = os.path.join(directory, entry)
     exe = os.path.join(exe_path, 'cppcheck')
 
@@ -134,16 +154,23 @@ for entry in versions:
         else:
             # TODO: re-add inconclusive: {callstack}: ({severity}{inconclusive:, inconclusive}) {message
             cmd.append('--template={callstack}: ({severity}) {message} [{id}]')
+    # TODO: how to pass addtional options?
+    if args.perf:
+        cmd.append('--error-exitcode=0')
     cmd.append(input_file)
     if verbose:
         print("running '{}'". format(' '.join(cmd)))
+    if args.perf:
+        start = time.time_ns()
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=exe_path, universal_newlines=True)
     try:
         comm = p.communicate(timeout=args.timeout)
+        if args.perf:
+            end = time.time_ns()
         out = ''
         if not args.no_stdout:
             out += comm[0]
-        if not args.no_stderr and not args.no_stderr:
+        if not args.no_stdout and not args.no_stderr:
             out += '\n'
         if not args.no_stderr:
             out += comm[1]
@@ -156,9 +183,19 @@ for entry in versions:
 
     if not do_compare:
         if not use_hashes:
-            print(version)
+            ver_str = version
         else:
-            print('{} ({})'.format(entry, version))
+            ver_str = '{} ({})'.format(entry, version)
+        if args.perf:
+            if out == "timeout":
+                data_str = "0.0" # TODO: how to handle these properly?
+            elif not ec == 0:
+                continue # skip errors
+            else:
+                data_str = '{}'.format((end - start) / 1000.0 / 1000.0 / 1000.0)
+            print('"{}",{}'.format(ver_str, data_str))
+            continue
+        print(ver_str)
         print(ec)
         print(out)
         continue