walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

donate-cpu.py: add stack traces for daca@home crashes / bugfixes (#1764) 

* Get stack traces for daca@home crashes

If a command in daca@home crashes, execute it again within gdb to get a stack trace.

* donate-cpu.py: added "gdb" to checkRequirements()

* donate-cpu.py: handle wget failures

* donate-cpu.py: added --no-upload option to disable all uploads

* donate-cpu.py: set max_packages to 1 if --package is provided to avoid endless processing of the same package

* donate-cpu.py: no longer treat missing sources as a crash

* donate-cpu.py: fixed wget "http://: Invalid host name." error caused by empty argument in subprocess.call()

* donate-cpu.py: added --no-upload to --help

* donate-cpu.py: detect crashes when using -j1

* donate-cpu.py: added -g to compiler flags

* donate-cpu.py: fixed gdb call and stacktrace printing / always pass "-j1" to gdb call so the exception will actually occur in the application

* donate-cpu.py: removed left-over --verbose from wget call

* donate-cpu.py: removed unnecessary break

* donate-cpu.py: only use gdb for crash in head run / actually provide the stack trace for the output

* donate-cpu.py: include the last checked file with the stack trace

* donate-cpu.py: removed unnecessary wget() call and a sleep in it / also inverted some logic

* donate-cpu.py: small hasInclude() optimization

* donate-cpu.py: bumped version number

* donate-cpu.py: detect start of gdb output when Cygwin is used

The Cygwin output looks like this:

Thread 1 "cppcheck" received signal SIGSEGV, Segmentation fault.

Co-Authored-By: firewave <firewave@users.noreply.github.com>
This commit is contained in:
Oliver Stöneberg 2019-03-28 15:49:20 +01:00 committed by Sebastian
parent e88a0c00c1
commit 7d383d1684
1 changed files with 63 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
tools/donate-cpu.py
View File

@ -12,6 +12,7 @@
# Examples: --bandwidth-limit=250k => max. 250 kilobytes per second
# --bandwidth-limit=2m => max. 2 megabytes per second
# --max-packages=N Process N packages and then exit. A value of 0 means infinitely.
# --no-upload Do not upload anything. Defaults to False.
#
# What this script does:
# 1. Check requirements
@ -38,12 +39,12 @@ import platform
# Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/
# Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic
# changes)
CLIENT_VERSION = "1.1.16"
CLIENT_VERSION = "1.1.17"
def checkRequirements():
result = True
for app in ['g++', 'git', 'make', 'wget']:
for app in ['g++', 'git', 'make', 'wget', 'gdb']:
try:
subprocess.call([app, '--version'])
except OSError:
@ -76,7 +77,7 @@ def compile_version(workPath, jobs, version):
os.chdir(workPath + '/cppcheck')
subprocess.call(['git', 'checkout', version])
subprocess.call(['make', 'clean'])
subprocess.call(['make', jobs, 'SRCDIR=build', 'CXXFLAGS=-O2'])
subprocess.call(['make', jobs, 'SRCDIR=build', 'CXXFLAGS=-O2 -g'])
if os.path.isfile(workPath + '/cppcheck/cppcheck'):
os.mkdir(workpath + '/' + version)
destPath = workpath + '/' + version + '/'
@ -94,7 +95,7 @@ def compile(cppcheckPath, jobs):
print('Compiling Cppcheck..')
try:
os.chdir(cppcheckPath)
subprocess.call(['make', jobs, 'SRCDIR=build', 'CXXFLAGS=-O2'])
subprocess.call(['make', jobs, 'SRCDIR=build', 'CXXFLAGS=-O2 -g'])
subprocess.call([cppcheckPath + '/cppcheck', '--version'])
except OSError:
return False
@ -159,24 +160,24 @@ def wget(url, destfile, bandwidth_limit):
else:
print('Error: ' + destfile + ' exists but it is not a file! Please check the path and delete it manually.')
sys.exit(1)
limit_rate_option = ''
wget_call = ['wget', '--tries=10', '--timeout=300', '-O', destfile, url]
if bandwidth_limit and isinstance(bandwidth_limit, str):
limit_rate_option = '--limit-rate=' + bandwidth_limit
subprocess.call(
['wget', '--tries=10', '--timeout=300', limit_rate_option, '-O', destfile, url])
if os.path.isfile(destfile):
return True
print('Sleep for 10 seconds..')
time.sleep(10)
return False
wget_call.append('--limit-rate=' + bandwidth_limit)
exitcode = subprocess.call(wget_call)
if exitcode != 0:
print('wget failed with ' + str(exitcode))
os.remove(destfile)
return False
if not os.path.isfile(destfile):
return False
return True
def downloadPackage(workPath, package, bandwidth_limit):
print('Download package ' + package)
destfile = workPath + '/temp.tgz'
if not wget(package, destfile, bandwidth_limit):
if not wget(package, destfile, bandwidth_limit):
return None
return None
return destfile
@ -205,6 +206,8 @@ def unpackPackage(workPath, tgz):
def hasInclude(path, includes):
re_includes = [re.escape(inc) for inc in includes]
re_expr = '^[ \t]*#[ \t]*include[ \t]*(' + '|'.join(re_includes) + ')'
for root, _, files in os.walk(path):
for name in files:
filename = os.path.join(root, name)
@ -221,13 +224,24 @@ def hasInclude(path, includes):
# Python3 directly reads the data into a string object that has no decode()
pass
f.close()
re_includes = [re.escape(inc) for inc in includes]
if re.search('^[ \t]*#[ \t]*include[ \t]*(' + '|'.join(re_includes) + ')', filedata, re.MULTILINE):
if re.search(re_expr, filedata, re.MULTILINE):
return True
except IOError:
pass
return False
def runCommand(cmd):
print(cmd)
startTime = time.time()
p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
comm = p.communicate()
stopTime = time.time()
stdout = comm[0].decode(encoding='utf-8', errors='ignore')
stderr = comm[1].decode(encoding='utf-8', errors='ignore')
elapsedTime = stopTime - startTime
return p.returncode, stdout, stderr, elapsedTime
def scanPackage(workPath, cppcheckPath, jobs):
print('Analyze..')
@ -255,25 +269,27 @@ def scanPackage(workPath, cppcheckPath, jobs):
if os.path.exists(os.path.join(cppcheckPath, 'cfg', library + '.cfg')) and hasInclude('temp', includes):
libraries += ' --library=' + library
# Reference for GNU C: https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html
# Reference for GNU C: https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html
options = jobs + libraries + ' -D__GNUC__ --check-library --inconclusive --enable=style,information --platform=unix64 --template=daca2 -rp=temp temp'
cmd = 'nice ' + cppcheckPath + '/cppcheck' + ' ' + options
print(cmd)
startTime = time.time()
p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
comm = p.communicate()
stopTime = time.time()
stdout = comm[0].decode(encoding='utf-8', errors='ignore')
stderr = comm[1].decode(encoding='utf-8', errors='ignore')
if p.returncode != 0 and 'cppcheck: error: could not find or open any of the paths given.' not in stdout:
cppcheck_cmd = cppcheckPath + '/cppcheck' + ' ' + options
cmd = 'nice ' + cppcheck_cmd
returncode, stdout, stderr, elapsedTime = runCommand(cmd)
if returncode == -11 or stderr.find('Internal error: Child process crashed with signal 11 [cppcheckError]') > 0:
# Crash!
print('Crash!')
return -1, '', '', -1, options
if stderr.find('Internal error: Child process crashed with signal 11 [cppcheckError]') > 0:
# Crash!
print('Crash!')
return -1, '', '', -1, options
elapsedTime = stopTime - startTime
stacktrace = ''
if cppcheckPath == 'cppcheck':
# re-run within gdb to get a stacktrace
cmd = 'gdb --batch --eval-command=run --eval-command=bt --return-child-result --args ' + cppcheck_cmd + " -j1"
returncode, stdout, stderr, elapsedTime = runCommand(cmd)
gdb_pos = stdout.find(" received signal")
if not gdb_pos == -1:
last_check_pos = stdout.rfind('Checking ', 0, gdb_pos)
if last_check_pos == -1:
stacktrace = stdout[gdb_pos:]
else:
stacktrace = stdout[last_check_pos:]
return -1, stacktrace, '', -1, options
information_messages_list = []
issue_messages_list = []
count = 0
@ -385,6 +401,7 @@ packageUrl = None
server_address = ('cppcheck.osuosl.org', 8000)
bandwidth_limit = None
max_packages = None
do_upload = True
for arg in sys.argv[1:]:
# --stop-time=12:00 => run until ~12:00 and then stop
if arg.startswith('--stop-time='):
@ -420,6 +437,8 @@ for arg in sys.argv[1:]:
# 0 means infinitely, no counting needed.
if max_packages == 0:
max_packages = None
elif arg.startswith('--no-upload'):
do_upload = False
elif arg == '--help':
print('Donate CPU to Cppcheck project')
print('')
@ -433,6 +452,7 @@ for arg in sys.argv[1:]:
print(' Examples: --bandwidth-limit=250k => max. 250 kilobytes per second')
print(' --bandwidth-limit=2m => max. 2 megabytes per second')
print(' --max-packages=N Process N packages and then exit. A value of 0 means infinitely.')
print(' --no-upload Do not upload anything. Defaults to False.')
print('')
print('Quick start: just run this script without any arguments')
sys.exit(0)
@ -449,6 +469,8 @@ if bandwidth_limit and isinstance(bandwidth_limit, str):
sys.exit(1)
else:
print('Bandwidth-limit: ' + bandwidth_limit)
if packageUrl:
max_packages = 1
if max_packages:
print('Maximum number of packages to download and analyze: {}'.format(max_packages))
if not os.path.exists(workpath):
@ -492,6 +514,9 @@ while True:
time.sleep(30)
package = getPackage(server_address)
tgz = downloadPackage(workpath, package, bandwidth_limit)
if tgz is None:
print("No package downloaded")
continue
unpackPackage(workpath, tgz)
crash = False
count = ''
@ -541,11 +566,11 @@ while True:
print('=========================================================')
print(output)
print('=========================================================')
break
if crash or results_exist:
uploadResults(package, output, server_address)
if info_exists:
uploadInfo(package, info_output, server_address)
if do_upload:
if crash or results_exist:
uploadResults(package, output, server_address)
if info_exists:
uploadInfo(package, info_output, server_address)
if not max_packages or packages_processed < max_packages:
print('Sleep 5 seconds..')
time.sleep(5)