donate-cpu.py: add stack traces for daca@home crashes / bugfixes (#1764)

* Get stack traces for daca@home crashes

If a command in daca@home crashes, execute it again within gdb to get a stack trace.

* donate-cpu.py: added "gdb" to checkRequirements()

* donate-cpu.py: handle wget failures

* donate-cpu.py: added --no-upload option to disable all uploads

* donate-cpu.py: set max_packages to 1 if --package is provided to avoid endless processing of the same package

* donate-cpu.py: no longer treat missing sources as a crash

* donate-cpu.py: fixed wget "http://: Invalid host name." error caused by empty argument in subprocess.call()

* donate-cpu.py: added --no-upload to --help

* donate-cpu.py: detect crashes when using -j1

* donate-cpu.py: added -g to compiler flags

* donate-cpu.py: fixed gdb call and stacktrace printing / always pass "-j1" to gdb call so the exception will actually occur in the application

* donate-cpu.py: removed left-over --verbose from wget call

* donate-cpu.py: removed unnecessary break

* donate-cpu.py: only use gdb for crash in head run / actually provide the stack trace for the output

* donate-cpu.py: include the last checked file with the stack trace

* donate-cpu.py: removed unnecessary wget() call and a sleep in it / also inverted some logic

* donate-cpu.py: small hasInclude() optimization

* donate-cpu.py: bumped version number

* donate-cpu.py: detect start of gdb output when Cygwin is used

The Cygwin output looks like this:

Thread 1 "cppcheck" received signal SIGSEGV, Segmentation fault.

Co-Authored-By: firewave <firewave@users.noreply.github.com>

tools/donate-cpu.py

@@ -12,6 +12,7 @@
 #                       Examples: --bandwidth-limit=250k => max. 250 kilobytes per second
 #                                 --bandwidth-limit=2m => max. 2 megabytes per second
 #  --max-packages=N     Process N packages and then exit. A value of 0 means infinitely.
+#  --no-upload          Do not upload anything. Defaults to False.
 #
 # What this script does:
 # 1. Check requirements
@@ -38,12 +39,12 @@ import platform
 # Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/
 # Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic
 # changes)
-CLIENT_VERSION = "1.1.16"
+CLIENT_VERSION = "1.1.17"
 
 
 def checkRequirements():
     result = True
-    for app in ['g++', 'git', 'make', 'wget']:
+    for app in ['g++', 'git', 'make', 'wget', 'gdb']:
         try:
             subprocess.call([app, '--version'])
         except OSError:
@@ -76,7 +77,7 @@ def compile_version(workPath, jobs, version):
     os.chdir(workPath + '/cppcheck')
     subprocess.call(['git', 'checkout', version])
     subprocess.call(['make', 'clean'])
-    subprocess.call(['make', jobs, 'SRCDIR=build', 'CXXFLAGS=-O2'])
+    subprocess.call(['make', jobs, 'SRCDIR=build', 'CXXFLAGS=-O2 -g'])
     if os.path.isfile(workPath + '/cppcheck/cppcheck'):
         os.mkdir(workpath + '/' + version)
         destPath = workpath + '/' + version + '/'
@@ -94,7 +95,7 @@ def compile(cppcheckPath, jobs):
     print('Compiling Cppcheck..')
     try:
         os.chdir(cppcheckPath)
-        subprocess.call(['make', jobs, 'SRCDIR=build', 'CXXFLAGS=-O2'])
+        subprocess.call(['make', jobs, 'SRCDIR=build', 'CXXFLAGS=-O2 -g'])
         subprocess.call([cppcheckPath + '/cppcheck', '--version'])
     except OSError:
         return False
@@ -159,24 +160,24 @@ def wget(url, destfile, bandwidth_limit):
         else:
             print('Error: ' + destfile + ' exists but it is not a file! Please check the path and delete it manually.')
             sys.exit(1)
-    limit_rate_option = ''
+    wget_call = ['wget', '--tries=10', '--timeout=300', '-O', destfile, url]
     if bandwidth_limit and isinstance(bandwidth_limit, str):
-        limit_rate_option = '--limit-rate=' + bandwidth_limit
+        wget_call.append('--limit-rate=' + bandwidth_limit)
-    subprocess.call(
+    exitcode = subprocess.call(wget_call)
-            ['wget', '--tries=10', '--timeout=300', limit_rate_option, '-O', destfile, url])
+    if exitcode != 0:
-    if os.path.isfile(destfile):
+        print('wget failed with ' + str(exitcode))
-        return True
+        os.remove(destfile)
-    print('Sleep for 10 seconds..')
+        return False
-    time.sleep(10)
+    if not os.path.isfile(destfile):
-    return False
+        return False
+    return True
 
 
 def downloadPackage(workPath, package, bandwidth_limit):
     print('Download package ' + package)
     destfile = workPath + '/temp.tgz'
     if not wget(package, destfile, bandwidth_limit):
-        if not wget(package, destfile, bandwidth_limit):
+        return None
-            return None
     return destfile
 
 
@@ -205,6 +206,8 @@ def unpackPackage(workPath, tgz):
 
 
 def hasInclude(path, includes):
+    re_includes = [re.escape(inc) for inc in includes]
+    re_expr = '^[ \t]*#[ \t]*include[ \t]*(' + '|'.join(re_includes) + ')'
     for root, _, files in os.walk(path):
         for name in files:
             filename = os.path.join(root, name)
@@ -221,13 +224,24 @@ def hasInclude(path, includes):
                     # Python3 directly reads the data into a string object that has no decode()
                     pass
                 f.close()
-                re_includes = [re.escape(inc) for inc in includes]
+                if re.search(re_expr, filedata, re.MULTILINE):
-                if re.search('^[ \t]*#[ \t]*include[ \t]*(' + '|'.join(re_includes) + ')', filedata, re.MULTILINE):
                     return True
             except IOError:
                 pass
     return False
 
+	
+def runCommand(cmd):
+    print(cmd)
+    startTime = time.time()
+    p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    comm = p.communicate()
+    stopTime = time.time()
+    stdout = comm[0].decode(encoding='utf-8', errors='ignore')
+    stderr = comm[1].decode(encoding='utf-8', errors='ignore')
+    elapsedTime = stopTime - startTime
+    return p.returncode, stdout, stderr, elapsedTime
+
 
 def scanPackage(workPath, cppcheckPath, jobs):
     print('Analyze..')
@@ -255,25 +269,27 @@ def scanPackage(workPath, cppcheckPath, jobs):
         if os.path.exists(os.path.join(cppcheckPath, 'cfg', library + '.cfg')) and hasInclude('temp', includes):
             libraries += ' --library=' + library
 
-# Reference for GNU C: https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html
+    # Reference for GNU C: https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html
     options = jobs + libraries + ' -D__GNUC__ --check-library --inconclusive --enable=style,information --platform=unix64 --template=daca2 -rp=temp temp'
-    cmd = 'nice ' + cppcheckPath + '/cppcheck' + ' ' + options
+    cppcheck_cmd = cppcheckPath + '/cppcheck' + ' ' + options
-    print(cmd)
+    cmd = 'nice ' + cppcheck_cmd
-    startTime = time.time()
+    returncode, stdout, stderr, elapsedTime = runCommand(cmd)
-    p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    if returncode == -11 or stderr.find('Internal error: Child process crashed with signal 11 [cppcheckError]') > 0:
-    comm = p.communicate()
-    stopTime = time.time()
-    stdout = comm[0].decode(encoding='utf-8', errors='ignore')
-    stderr = comm[1].decode(encoding='utf-8', errors='ignore')
-    if p.returncode != 0 and 'cppcheck: error: could not find or open any of the paths given.' not in stdout:
         # Crash!
         print('Crash!')
-        return -1, '', '', -1, options
+        stacktrace = ''
-    if stderr.find('Internal error: Child process crashed with signal 11 [cppcheckError]') > 0:
+        if cppcheckPath == 'cppcheck':
-        # Crash!
+            # re-run within gdb to get a stacktrace
-        print('Crash!')
+            cmd = 'gdb --batch --eval-command=run --eval-command=bt --return-child-result --args ' + cppcheck_cmd + " -j1"
-        return -1, '', '', -1, options
+            returncode, stdout, stderr, elapsedTime = runCommand(cmd)
-    elapsedTime = stopTime - startTime
+            gdb_pos = stdout.find(" received signal")
+            if not gdb_pos == -1:
+                last_check_pos = stdout.rfind('Checking ', 0, gdb_pos)
+                if last_check_pos == -1:
+                    stacktrace = stdout[gdb_pos:]
+                else:
+                    stacktrace = stdout[last_check_pos:]
+        return -1, stacktrace, '', -1, options
     information_messages_list = []
     issue_messages_list = []
     count = 0
@@ -385,6 +401,7 @@ packageUrl = None
 server_address = ('cppcheck.osuosl.org', 8000)
 bandwidth_limit = None
 max_packages = None
+do_upload = True
 for arg in sys.argv[1:]:
     # --stop-time=12:00 => run until ~12:00 and then stop
     if arg.startswith('--stop-time='):
@@ -420,6 +437,8 @@ for arg in sys.argv[1:]:
         # 0 means infinitely, no counting needed.
         if max_packages == 0:
             max_packages = None
+    elif arg.startswith('--no-upload'):
+        do_upload = False
     elif arg == '--help':
         print('Donate CPU to Cppcheck project')
         print('')
@@ -433,6 +452,7 @@ for arg in sys.argv[1:]:
         print('                       Examples: --bandwidth-limit=250k => max. 250 kilobytes per second')
         print('                                 --bandwidth-limit=2m => max. 2 megabytes per second')
         print('  --max-packages=N     Process N packages and then exit. A value of 0 means infinitely.')
+        print('  --no-upload          Do not upload anything. Defaults to False.')
         print('')
         print('Quick start: just run this script without any arguments')
         sys.exit(0)
@@ -449,6 +469,8 @@ if bandwidth_limit and isinstance(bandwidth_limit, str):
         sys.exit(1)
     else:
         print('Bandwidth-limit: ' + bandwidth_limit)
+if packageUrl:
+    max_packages = 1
 if max_packages:
     print('Maximum number of packages to download and analyze: {}'.format(max_packages))
 if not os.path.exists(workpath):
@@ -492,6 +514,9 @@ while True:
         time.sleep(30)
         package = getPackage(server_address)
     tgz = downloadPackage(workpath, package, bandwidth_limit)
+    if tgz is None:
+        print("No package downloaded")
+        continue
     unpackPackage(workpath, tgz)
     crash = False
     count = ''
@@ -541,11 +566,11 @@ while True:
         print('=========================================================')
         print(output)
         print('=========================================================')
-        break
+    if do_upload:
-    if crash or results_exist:
+        if crash or results_exist:
-        uploadResults(package, output, server_address)
+            uploadResults(package, output, server_address)
-    if info_exists:
+        if info_exists:
-        uploadInfo(package, info_output, server_address)
+            uploadInfo(package, info_output, server_address)
     if not max_packages or packages_processed < max_packages:
         print('Sleep 5 seconds..')
         time.sleep(5)