Fixed #8643 (daca2: comparison of versions)

tools/daca2-diff.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+import os
+import sys
+import re
+
+def parseResults(filename):
+    ftp = ''
+    warnings = []
+    pattern = re.compile(r'.*: (error|warning|style|performance|portability):.* \[([a-zA-Z0-9_\\-]+)\]')
+    for line in open(filename, 'rt'):
+        line = line.strip('\r\n')
+        if line.startswith('ftp://'):
+            ftp = line
+            continue
+        if pattern.match(line):
+            warnings.append(ftp + '\n' + line)
+    return warnings
+
+def getUnique(warnings1, warnings2):
+    ret = ''
+    for w in warnings1:
+        if w not in warnings2:
+            ret = ret + w + '\n'
+    return ret
+
+daca2folder = os.path.expanduser('~/daca2/')
+reportpath = ''
+for arg in sys.argv[1:]:
+    if arg.startswith('--daca2='):
+        daca2folder = arg[8:]
+        if daca2folder[-1] != '/':
+            daca2folder += '/'
+    else:
+        reportpath = arg
+        if reportpath[-1] != '/':
+            reportpath += '/'
+
+warnings_base = []
+warnings_head = []
+
+for lib in ['', 'lib']:
+    for a in "0123456789abcdefghijklmnopqrstuvwxyz":
+        if not os.path.isfile(daca2folder + lib + a + '/results-1.84.txt'):
+            continue
+        if not os.path.isfile(daca2folder + lib + a + '/results-head.txt'):
+            continue
+
+        warnings_base.extend(parseResults(daca2folder + lib + a + '/results-1.84.txt'))
+        warnings_head.extend(parseResults(daca2folder + lib + a + '/results-head.txt'))
+
+f = open(reportpath + 'diff-1.84.txt', 'wt')
+f.write(getUnique(warnings_base, warnings_head))
+f.close()
+
+f = open(reportpath + 'diff-head.txt', 'wt')
+f.write(getUnique(warnings_head, warnings_base))
+f.close()

tools/daca2-report.py

@@ -95,14 +95,14 @@ for severity in ['error', 'warning', 'style', 'portability', 'performance']:
 
 daca2 = daca2folder
 pattern = re.compile(r'.*: (error|warning|style|performance|portability):.* \[([a-zA-Z0-9_\\-]+)\]')
-for lib in (False, True):
+for lib in [False, True]:
     for a in "0123456789abcdefghijklmnopqrstuvwxyz":
         if lib:
             a = "lib" + a
-        if not os.path.isfile(daca2 + a + '/results.txt'):
+        if not os.path.isfile(daca2 + a + '/results-head.txt'):
             continue
 
-        f = open(daca2 + a + '/results.txt', 'rt')
+        f = open(daca2 + a + '/results-head.txt', 'rt')
         data = f.read()
         f.close()
 

tools/daca2.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 #
 # 1. Create a folder daca2 in your HOME folder
-# 2. Put cppcheck-O2 in daca2. It should be built with all optimisations.
+# 2. Put cppcheck-head in daca2. It should be built with all optimisations.
 # 3. Optional: Put a file called "suppressions.txt" in the daca2 folder.
 # 4. Optional: tweak FTPSERVER and FTPPATH in this script below.
 # 5. Run the daca2 script:  python daca2.py FOLDER
@@ -14,11 +14,11 @@ import glob
 import os
 import datetime
 import time
-import logging
 
 DEBIAN = ('ftp://ftp.se.debian.org/debian/',
           'ftp://ftp.debian.org/debian/')
 
+RESULTS_FILES = ['results.txt']
 
 def wget(filepath):
     filename = filepath
@@ -85,7 +85,7 @@ def removeAllExceptResults():
             try:
                 if os.path.isdir(filename):
                     shutil.rmtree(filename, onerror=handleRemoveReadonly)
-                elif filename != RESULTS_FILENAME:
+                elif filename not in RESULTS_FILES:
                     os.remove(filename)
                 break
             except WindowsError as err:
@@ -113,7 +113,7 @@ def removeLargeFiles(path):
                 shutil.rmtree(g, onerror=handleRemoveReadonly)
             else:
                 removeLargeFiles(g + '/')
-        elif os.path.isfile(g) and g[-4:] != '.txt':
+        elif os.path.isfile(g) and not g.endswith('.txt'):
             statinfo = os.stat(g)
             if statinfo.st_size > 1000000:
                 try:
@@ -126,15 +126,15 @@ def strfCurrTime(fmt):
     return datetime.time.strftime(datetime.datetime.now().time(), fmt)
 
 
-def scanarchive(filepath, jobs, cpulimit):
+def scanarchive(filepath, args, run, resultsFile):
     # remove all files/folders except RESULTS_FILENAME
     removeAllExceptResults()
 
-    logging.info(DEBIAN[0] + filepath)
+    resultsFile.write(DEBIAN[0] + filepath + '\n')
 
     if not wget(filepath):
         if not wget(filepath):
-            logging.error('wget failed at {}'.format(filepath))
+            resultsFile.write('wget failed at {}'.format(filepath))
             return
 
     filename = filepath[filepath.rfind('/') + 1:]
@@ -149,27 +149,29 @@ def scanarchive(filepath, jobs, cpulimit):
 
     print(strfCurrTime('[%H:%M] cppcheck ') + filename)
 
-    if cpulimit:
+    if args.cpulimit:
-        cmd = 'cpulimit --limit=' + cpulimit
+        cmd = 'cpulimit --limit=' + args.cpulimit
     else:
         cmd = 'nice --adjustment=1000'
     # TODO: The --exception-handling=stderr is skipped right now because it hangs (#8589)
-    cmd = cmd + ' ../cppcheck-O2 -D__GCC__ --enable=style --inconclusive --error-exitcode=0 ' +\
+    cppcheck = '../cppcheck-' + run
-        jobs + ' --template=daca2 .'
+    cmd = cmd + ' ' + cppcheck + ' -D__GCC__ --enable=style --inconclusive --error-exitcode=0 ' +\
+        args.jobs + ' --template=daca2 .'
     cmds = cmd.split()
 
     p = subprocess.Popen(cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     comm = p.communicate()
 
     if p.returncode == 0:
-        logging.info(comm[1] + strfCurrTime('[%H:%M]'))
+        resultsFile.write(comm[1] + strfCurrTime('[%H:%M]') + '\n')
     elif 'cppcheck: error: could not find or open any of the paths given.' not in comm[0]:
         stdout = comm[0]
         pos1 = stdout.rfind('Checking ')
         if pos1 > 0:
-            logging.error(stdout[pos1:])
+            resultsFile.write(stdout[pos1:]+'\n')
-        logging.error(comm[1] + strfCurrTime('[%H:%M]'))
+        resultsFile.write(comm[1] + strfCurrTime('[%H:%M]')+'\n')
-        logging.error('Exit code is not zero! Crash?\n')
+        resultsFile.write('Exit code is not zero! Crash?\n')
+    resultsFile.write('\n')
 
 
 parser = argparse.ArgumentParser(description='Checks debian source code')
@@ -179,57 +181,58 @@ parser.add_argument('--workdir', default='~/daca2')
 parser.add_argument('-j', '--jobs', default='-j1')
 parser.add_argument('--skip', default=[], action='append')
 parser.add_argument('--cpulimit')
+parser.add_argument('--baseversion')
 
 args = parser.parse_args()
 
 workdir = os.path.expanduser(args.workdir)
 if not os.path.isdir(workdir):
-    logging.critical('workdir \'' + workdir + '\' is not a folder')
+    print('workdir \'' + workdir + '\' is not a folder')
+    sys.exit(1)
+os.chdir(workdir)
+
+archives = getpackages(args.folder)
+if len(archives) == 0:
+    print('failed to load packages')
     sys.exit(1)
 
 workdir = os.path.join(workdir, args.folder)
-if not os.path.isdir(workdir):
-    os.makedirs(workdir)
-
-RESULTS_FILENAME = 'results.txt'
-RESULTS_FILE = os.path.join(workdir, RESULTS_FILENAME)
-
-logging.basicConfig(
-    filename=RESULTS_FILE,
-    level=logging.INFO,
-    format='%(message)s')
-
-print(workdir)
-
-archives = getpackages(args.folder)
-if len(archives) == 0:
-    logging.critical('failed to load packages')
-    sys.exit(1)
-
 if not os.path.isdir(workdir):
     os.makedirs(workdir)
 os.chdir(workdir)
 
-try:
+versions = ['head']
-    logging.info('STARTDATE ' + str(datetime.date.today()))
+if args.baseversion:
-    logging.info('STARTTIME ' + strfCurrTime('%H:%M:%S'))
+    versions.append(args.baseversion)
-    if args.rev:
+    RESULTS_FILES = ['results-head.txt', 'results-' + args.baseversion + '.txt']
-        logging.info('GIT-REVISION ' + args.rev + '\n')
-    logging.info('')
 
-    for archive in archives:
+for run in versions:
-        if len(args.skip) > 0:
+    try:
-            a = archive[:archive.rfind('/')]
+        f = None
-            a = a[a.rfind('/') + 1:]
+        if args.baseversion:
-            if a in args.skip:
+            f = open('results-' + run + '.txt', 'wt')
-                continue
+        else:
-        scanarchive(archive, args.jobs, args.cpulimit)
+            f = open('results.txt', 'wt')
+        f.write('STARTDATE ' + str(datetime.date.today()) + '\n')
+        f.write('STARTTIME ' + strfCurrTime('%H:%M:%S') + '\n')
+        if args.rev:
+            f.write('GIT-REVISION ' + args.rev + '\n')
+        f.write('\n')
 
-    logging.info('DATE {}'.format(datetime.date.today()))
+        for archive in archives:
-    logging.info('TIME {}'.format(strfCurrTime('%H:%M:%S')))
+            if len(args.skip) > 0:
+                a = archive[:archive.rfind('/')]
+                a = a[a.rfind('/') + 1:]
+                if a in args.skip:
+                    continue
+            scanarchive(archive, args, run, f)
 
-except EOFError:
+        f.write('DATE {}'.format(datetime.date.today()) + '\n')
-    pass
+        f.write('TIME {}'.format(strfCurrTime('%H:%M:%S')) + '\n')
+        f.close()
+
+    except EOFError:
+        pass
 
 # remove all files/folders except RESULTS_FILENAME
 removeAllExceptResults()

tools/rundaca2.py

@@ -21,20 +21,21 @@ def compilecppcheck(CPPFLAGS):
     subprocess.call(['nice', 'make', 'clean'])
     subprocess.call(['nice', 'make', 'SRCDIR=build', 'CFGDIR=' +
                     os.path.expanduser('~/cppcheck/cfg'), 'CXXFLAGS=-g -O2', 'CPPFLAGS=' + CPPFLAGS])
-    subprocess.call(['cp', 'cppcheck', os.path.expanduser('~/daca2/cppcheck-O2')])
+    subprocess.call(['cp', 'cppcheck', os.path.expanduser('~/daca2/cppcheck-head')])
 
 
 def runcppcheck(rev, folder):
     subprocess.call(['rm', '-rf', os.path.expanduser('~/daca2/' + folder)])
     subprocess.call(['nice', '--adjustment=19', 'python',
                     os.path.expanduser('~/cppcheck/tools/daca2.py'), folder, '--rev=' + rev,
-                    '--skip=hashdeep', '--skip=lice'])
+                    '--baseversion=1.84', '--skip=hashdeep', '--skip=lice'])
 
 
 def daca2report(reportfolder):
     subprocess.call(['rm', '-rf', reportfolder])
     subprocess.call(['mkdir', reportfolder])
     subprocess.call(['python', os.path.expanduser('~/cppcheck/tools/daca2-report.py'), reportfolder])
+    subprocess.call(['python', os.path.expanduser('~/cppcheck/tools/daca2-diff.py'), reportfolder])
 
 
 # Upload file to sourceforge server using scp
@@ -71,10 +72,17 @@ def daca2(foldernum):
     rev = comm[0]
     rev = rev[:rev.find('\n')]
 
+    print('rundaca2.py: compile cppcheck')
     compilecppcheck('-DMAXTIME=600 -DDACA2')
+
+    print('rundaca2.py: runcppcheck')
     runcppcheck(rev, folder)
     runcppcheck(rev, 'lib' + folder)
+
+    print('rundaca2.py: daca2 report')
     daca2report(os.path.expanduser('~/daca2-report'))
+
+    print('rundaca2.py: upload')
     upload(os.path.expanduser('~/daca2-report'), 'devinfo/')
 
 foldernum = START