1b28fb2d1a
Yesterday, I observed that some client with a wrong jobs setting (only "-j") requested one package after another and always uploaded results where it only said that the argument "-j" is invalid for Cppcheck. This check should avoid such cases where results are overwritten with useless data and the server is kept busy for nothing.
244 lines
10 KiB
Python
Executable File
244 lines
10 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# Donate CPU
|
|
#
|
|
# A script a user can run to donate CPU to cppcheck project
|
|
#
|
|
# Syntax: donate-cpu.py [-jN] [--package=url] [--stop-time=HH:MM] [--work-path=path] [--test] [--bandwidth-limit=limit]
|
|
# -jN Use N threads in compilation/analysis. Default is 1.
|
|
# --package=url Check a specific package and then stop. Can be useful if you want to reproduce
|
|
# some warning/crash/exception/etc..
|
|
# --stop-time=HH:MM Stop analysis when time has passed. Default is that you must terminate the script.
|
|
# --work-path=path Work folder path. Default path is cppcheck-donate-cpu-workfolder in your home folder.
|
|
# --test Connect to a donate-cpu-server that is running locally on port 8001 for testing.
|
|
# --bandwidth-limit=limit Limit download rate for packages. Format for limit is the same that wget uses.
|
|
# Examples: --bandwidth-limit=250k => max. 250 kilobytes per second
|
|
# --bandwidth-limit=2m => max. 2 megabytes per second
|
|
# --max-packages=N Process N packages and then exit. A value of 0 means infinitely.
|
|
# --no-upload Do not upload anything. Defaults to False.
|
|
#
|
|
# What this script does:
|
|
# 1. Check requirements
|
|
# 2. Pull & compile Cppcheck
|
|
# 3. Select a package
|
|
# 4. Download package
|
|
# 5. Analyze source code
|
|
# 6. Upload results
|
|
# 7. Repeat from step 2
|
|
#
|
|
# Quick start: just run this script without any arguments
|
|
|
|
import shutil
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import socket
|
|
import time
|
|
import re
|
|
import tarfile
|
|
import platform
|
|
from donate_cpu_lib import *
|
|
|
|
for arg in sys.argv[1:]:
|
|
# --stop-time=12:00 => run until ~12:00 and then stop
|
|
if arg.startswith('--stop-time='):
|
|
stop_time = arg[-5:]
|
|
print('Stop time:' + stop_time)
|
|
elif arg.startswith('-j'):
|
|
if not re.match(r'-j\d+', arg):
|
|
print('Argument "{}" is invalid.'.format(arg))
|
|
print('"-j" must be followed by a positive number.')
|
|
sys.exit(1)
|
|
jobs = arg
|
|
print('Jobs:' + jobs[2:])
|
|
elif arg.startswith('--package='):
|
|
package_url = arg[arg.find('=')+1:]
|
|
print('Package:' + package_url)
|
|
elif arg.startswith('--work-path='):
|
|
work_path = arg[arg.find('=')+1:]
|
|
print('work_path:' + work_path)
|
|
if not os.path.exists(work_path):
|
|
print('work path does not exist!')
|
|
sys.exit(1)
|
|
elif arg == '--test':
|
|
server_address = ('localhost', 8001)
|
|
elif arg.startswith('--bandwidth-limit='):
|
|
bandwidth_limit = arg[arg.find('=')+1:]
|
|
elif arg.startswith('--max-packages='):
|
|
arg_value = arg[arg.find('=')+1:]
|
|
try:
|
|
max_packages = int(arg_value)
|
|
except ValueError:
|
|
max_packages = None
|
|
if max_packages < 0:
|
|
max_packages = None
|
|
if max_packages is None:
|
|
print('Error: Max. packages value "{}" is invalid. Must be a positive number or 0.'.format(arg_value))
|
|
sys.exit(1)
|
|
# 0 means infinitely, no counting needed.
|
|
if max_packages == 0:
|
|
max_packages = None
|
|
elif arg.startswith('--no-upload'):
|
|
do_upload = False
|
|
elif arg == '--help':
|
|
print('Donate CPU to Cppcheck project')
|
|
print('')
|
|
print('Syntax: donate-cpu.py [-jN] [--stop-time=HH:MM] [--work-path=path]')
|
|
print(' -jN Use N threads in compilation/analysis. Default is 1.')
|
|
print(' --package=url Check a specific package and then stop. Can be useful if you want to reproduce')
|
|
print(' some warning/crash/exception/etc..')
|
|
print(' --stop-time=HH:MM Stop analysis when time has passed. Default is that you must terminate the script.')
|
|
print(' --work-path=path Work folder path. Default path is ' + work_path)
|
|
print(' --bandwidth-limit=limit Limit download rate for packages. Format for limit is the same that wget uses.')
|
|
print(' Examples: --bandwidth-limit=250k => max. 250 kilobytes per second')
|
|
print(' --bandwidth-limit=2m => max. 2 megabytes per second')
|
|
print(' --max-packages=N Process N packages and then exit. A value of 0 means infinitely.')
|
|
print(' --no-upload Do not upload anything. Defaults to False.')
|
|
print('')
|
|
print('Quick start: just run this script without any arguments')
|
|
sys.exit(0)
|
|
else:
|
|
print('Unhandled argument: ' + arg)
|
|
sys.exit(1)
|
|
|
|
if sys.version_info.major < 3:
|
|
print("#" * 80)
|
|
print("#" * 80)
|
|
print("IMPORTANT")
|
|
print("Please consider switching to Python 3.")
|
|
print("")
|
|
print("We plan to completely drop Python 2 support")
|
|
print("for the Donate CPU client in the near future.")
|
|
print("")
|
|
print("For further information and reporting complaints, ideas, ... see:")
|
|
print("https://sourceforge.net/p/cppcheck/discussion/development/thread/86813a8a53/")
|
|
print("#" * 80)
|
|
print("#" * 80)
|
|
time.sleep(5)
|
|
|
|
print('Thank you!')
|
|
if not check_requirements():
|
|
sys.exit(1)
|
|
if bandwidth_limit and isinstance(bandwidth_limit, str):
|
|
if subprocess.call(['wget', '--limit-rate=' + bandwidth_limit, '-q', '--spider', 'cppcheck1.osuosl.org']) == 2:
|
|
print('Error: Bandwidth limit value "' + bandwidth_limit + '" is invalid.')
|
|
sys.exit(1)
|
|
else:
|
|
print('Bandwidth-limit: ' + bandwidth_limit)
|
|
if package_url:
|
|
max_packages = 1
|
|
if max_packages:
|
|
print('Maximum number of packages to download and analyze: {}'.format(max_packages))
|
|
if not os.path.exists(work_path):
|
|
os.mkdir(work_path)
|
|
cppcheck_path = os.path.join(work_path, 'cppcheck')
|
|
packages_processed = 0
|
|
while True:
|
|
if max_packages:
|
|
if packages_processed >= max_packages:
|
|
print('Processed the specified number of {} package(s). Exiting now.'.format(max_packages))
|
|
break
|
|
print('Processing package {} of the specified {} package(s).'.format(packages_processed + 1, max_packages))
|
|
packages_processed += 1
|
|
if stop_time:
|
|
print('stop_time:' + stop_time + '. Time:' + time.strftime('%H:%M') + '.')
|
|
if stop_time < time.strftime('%H:%M'):
|
|
print('Stopping. Thank you!')
|
|
sys.exit(0)
|
|
if not get_cppcheck(cppcheck_path, work_path):
|
|
print('Failed to clone Cppcheck, retry later')
|
|
sys.exit(1)
|
|
cppcheck_versions = get_cppcheck_versions(server_address)
|
|
if cppcheck_versions is None:
|
|
print('Failed to communicate with server, retry later')
|
|
sys.exit(1)
|
|
if len(cppcheck_versions) == 0:
|
|
print('Did not get any cppcheck versions from server, retry later')
|
|
sys.exit(1)
|
|
for ver in cppcheck_versions:
|
|
if ver == 'head':
|
|
if not compile(cppcheck_path, jobs):
|
|
print('Failed to compile Cppcheck, retry later')
|
|
sys.exit(1)
|
|
elif not compile_version(work_path, jobs, ver):
|
|
print('Failed to compile Cppcheck-{}, retry later'.format(ver))
|
|
sys.exit(1)
|
|
if package_url:
|
|
package = package_url
|
|
else:
|
|
package = get_package(server_address)
|
|
while len(package) == 0:
|
|
print("network or server might be temporarily down.. will try again in 30 seconds..")
|
|
time.sleep(30)
|
|
package = get_package(server_address)
|
|
tgz = download_package(work_path, package, bandwidth_limit)
|
|
if tgz is None:
|
|
print("No package downloaded")
|
|
continue
|
|
if not unpack_package(work_path, tgz):
|
|
print("No files to process")
|
|
continue
|
|
crash = False
|
|
count = ''
|
|
elapsed_time = ''
|
|
results_to_diff = []
|
|
cppcheck_options = ''
|
|
head_info_msg = ''
|
|
head_timing_info = ''
|
|
old_timing_info = ''
|
|
cppcheck_head_info = ''
|
|
libraries = get_libraries()
|
|
|
|
for ver in cppcheck_versions:
|
|
if ver == 'head':
|
|
current_cppcheck_dir = 'cppcheck'
|
|
cppcheck_head_info = get_cppcheck_info(work_path + '/cppcheck')
|
|
else:
|
|
current_cppcheck_dir = ver
|
|
c, errout, info, t, cppcheck_options, timing_info = scan_package(work_path, current_cppcheck_dir, jobs, libraries)
|
|
if c < 0:
|
|
if c == -101 and 'error: could not find or open any of the paths given.' in errout:
|
|
# No sourcefile found (for example only headers present)
|
|
count += ' 0'
|
|
else:
|
|
crash = True
|
|
count += ' Crash!'
|
|
else:
|
|
count += ' ' + str(c)
|
|
elapsed_time += " {:.1f}".format(t)
|
|
results_to_diff.append(errout)
|
|
if ver == 'head':
|
|
head_info_msg = info
|
|
head_timing_info = timing_info
|
|
else:
|
|
old_timing_info = timing_info
|
|
|
|
output = 'cppcheck-options: ' + cppcheck_options + '\n'
|
|
output += 'platform: ' + platform.platform() + '\n'
|
|
output += 'python: ' + platform.python_version() + '\n'
|
|
output += 'client-version: ' + CLIENT_VERSION + '\n'
|
|
output += 'cppcheck: ' + ' '.join(cppcheck_versions) + '\n'
|
|
output += 'head-info: ' + cppcheck_head_info + '\n'
|
|
output += 'count:' + count + '\n'
|
|
output += 'elapsed-time:' + elapsed_time + '\n'
|
|
output += 'head-timing-info:\n' + head_timing_info + '\n'
|
|
output += 'old-timing-info:\n' + old_timing_info + '\n'
|
|
info_output = output
|
|
info_output += 'info messages:\n' + head_info_msg
|
|
if 'head' in cppcheck_versions:
|
|
output += 'head results:\n' + results_to_diff[cppcheck_versions.index('head')]
|
|
if not crash:
|
|
output += 'diff:\n' + diff_results(work_path, cppcheck_versions[0], results_to_diff[0], cppcheck_versions[1], results_to_diff[1]) + '\n'
|
|
if package_url:
|
|
print('=========================================================')
|
|
print(output)
|
|
print('=========================================================')
|
|
print(info_output)
|
|
print('=========================================================')
|
|
if do_upload:
|
|
upload_results(package, output, server_address)
|
|
upload_info(package, info_output, server_address)
|
|
if not max_packages or packages_processed < max_packages:
|
|
print('Sleep 5 seconds..')
|
|
time.sleep(5)
|