diff --git a/tools/donate-cpu.py b/tools/donate-cpu.py index 40ba87b3e..360dbc973 100644 --- a/tools/donate-cpu.py +++ b/tools/donate-cpu.py @@ -43,7 +43,7 @@ import platform CLIENT_VERSION = "1.1.22" -def checkRequirements(): +def check_requirements(): result = True for app in ['g++', 'git', 'make', 'wget', 'gdb']: try: @@ -54,16 +54,16 @@ def checkRequirements(): return result -def getCppcheck(cppcheckPath): +def get_cppcheck(cppcheck_path): print('Get Cppcheck..') for i in range(5): - if os.path.exists(cppcheckPath): - os.chdir(cppcheckPath) + if os.path.exists(cppcheck_path): + os.chdir(cppcheck_path) subprocess.call(['git', 'checkout', '-f']) subprocess.call(['git', 'pull']) else: - subprocess.call(['git', 'clone', 'https://github.com/danmar/cppcheck.git', cppcheckPath]) - if not os.path.exists(cppcheckPath): + subprocess.call(['git', 'clone', 'https://github.com/danmar/cppcheck.git', cppcheck_path]) + if not os.path.exists(cppcheck_path): print('Failed to clone, will try again in 10 minutes..') time.sleep(600) continue @@ -72,38 +72,38 @@ def getCppcheck(cppcheckPath): return False -def compile_version(workPath, jobs, version): - if os.path.isfile(workPath + '/' + version + '/cppcheck'): +def compile_version(work_path, jobs, version): + if os.path.isfile(work_path + '/' + version + '/cppcheck'): return True - os.chdir(workPath + '/cppcheck') + os.chdir(work_path + '/cppcheck') subprocess.call(['git', 'checkout', version]) subprocess.call(['make', 'clean']) subprocess.call(['make', jobs, 'MATCHCOMPILER=yes', 'CXXFLAGS=-O2 -g']) - if os.path.isfile(workPath + '/cppcheck/cppcheck'): - os.mkdir(workpath + '/' + version) - destPath = workpath + '/' + version + '/' - subprocess.call(['cp', '-R', workPath + '/cppcheck/cfg', destPath]) + if os.path.isfile(work_path + '/cppcheck/cppcheck'): + os.mkdir(work_path + '/' + version) + destPath = work_path + '/' + version + '/' + subprocess.call(['cp', '-R', work_path + '/cppcheck/cfg', destPath]) subprocess.call(['cp', 'cppcheck', destPath]) subprocess.call(['git', 'checkout', 'master']) try: - subprocess.call([workPath + '/' + version + '/cppcheck', '--version']) + subprocess.call([work_path + '/' + version + '/cppcheck', '--version']) except OSError: return False return True -def compile(cppcheckPath, jobs): +def compile(cppcheck_path, jobs): print('Compiling Cppcheck..') try: - os.chdir(cppcheckPath) + os.chdir(cppcheck_path) subprocess.call(['make', jobs, 'MATCHCOMPILER=yes', 'CXXFLAGS=-O2 -g']) - subprocess.call([cppcheckPath + '/cppcheck', '--version']) + subprocess.call([cppcheck_path + '/cppcheck', '--version']) except OSError: return False return True -def getCppcheckVersions(server_address): +def get_cppcheck_versions(server_address): print('Connecting to server to get Cppcheck versions..') sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: @@ -117,7 +117,7 @@ def getCppcheckVersions(server_address): return versions.decode('utf-8').split() -def getPackage(server_address): +def get_package(server_address): print('Connecting to server to get assigned work..') package = None sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -131,7 +131,7 @@ def getPackage(server_address): return package.decode('utf-8') -def handleRemoveReadonly(func, path, exc): +def handle_remove_readonly(func, path, exc): import stat if not os.access(path, os.W_OK): # Is the error an access error ? @@ -139,14 +139,14 @@ def handleRemoveReadonly(func, path, exc): func(path) -def removeTree(folderName): +def remove_tree(folderName): if not os.path.exists(folderName): return count = 5 while count > 0: count -= 1 try: - shutil.rmtree(folderName, onerror=handleRemoveReadonly) + shutil.rmtree(folderName, onerror=handle_remove_readonly) break except OSError as err: time.sleep(30) @@ -175,20 +175,20 @@ def wget(url, destfile, bandwidth_limit): return True -def downloadPackage(workPath, package, bandwidth_limit): +def download_package(work_path, package, bandwidth_limit): print('Download package ' + package) - destfile = workPath + '/temp.tgz' + destfile = work_path + '/temp.tgz' if not wget(package, destfile, bandwidth_limit): return None return destfile -def unpackPackage(workPath, tgz): +def unpack_package(work_path, tgz): print('Unpacking..') - tempPath = workPath + '/temp' - removeTree(tempPath) - os.mkdir(tempPath) - os.chdir(tempPath) + temp_path = work_path + '/temp' + remove_tree(temp_path) + os.mkdir(temp_path) + os.chdir(temp_path) found = False if tarfile.is_tarfile(tgz): tf = tarfile.open(tgz) @@ -206,11 +206,11 @@ def unpackPackage(workPath, tgz): except AttributeError: pass tf.close() - os.chdir(workPath) + os.chdir(work_path) return found -def hasInclude(path, includes): +def has_include(path, includes): re_includes = [re.escape(inc) for inc in includes] re_expr = '^[ \t]*#[ \t]*include[ \t]*(' + '|'.join(re_includes) + ')' for root, _, files in os.walk(path): @@ -236,24 +236,24 @@ def hasInclude(path, includes): return False -def runCommand(cmd): +def run_command(cmd): print(cmd) startTime = time.time() p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) comm = p.communicate() - stopTime = time.time() + stop_time = time.time() stdout = comm[0].decode(encoding='utf-8', errors='ignore') stderr = comm[1].decode(encoding='utf-8', errors='ignore') - elapsedTime = stopTime - startTime - return p.returncode, stdout, stderr, elapsedTime + elapsed_time = stop_time - startTime + return p.returncode, stdout, stderr, elapsed_time -def scanPackage(workPath, cppcheckPath, jobs): +def scan_package(work_path, cppcheck_path, jobs): print('Analyze..') - os.chdir(workPath) + os.chdir(work_path) libraries = ' --library=posix --library=gnu' - libraryIncludes = {'boost': [''], 'gtk': ['', '', ''], } - for library, includes in libraryIncludes.items(): - if os.path.exists(os.path.join(cppcheckPath, 'cfg', library + '.cfg')) and hasInclude('temp', includes): + for library, includes in library_includes.items(): + if os.path.exists(os.path.join(cppcheck_path, 'cfg', library + '.cfg')) and has_include('temp', includes): libraries += ' --library=' + library # Reference for GNU C: https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html options = jobs + libraries + ' -D__GNUC__ --check-library --inconclusive --enable=style,information --platform=unix64 --template=daca2 -rp=temp temp' - cppcheck_cmd = cppcheckPath + '/cppcheck' + ' ' + options + cppcheck_cmd = cppcheck_path + '/cppcheck' + ' ' + options cmd = 'nice ' + cppcheck_cmd - returncode, stdout, stderr, elapsedTime = runCommand(cmd) + returncode, stdout, stderr, elapsed_time = run_command(cmd) print('cppcheck finished with ' + str(returncode)) if returncode == -11 or stderr.find('Internal error: Child process crashed with signal 11 [cppcheckError]') > 0: print('Crash!') stacktrace = '' - if cppcheckPath == 'cppcheck': + if cppcheck_path == 'cppcheck': # re-run within gdb to get a stacktrace cmd = 'gdb --batch --eval-command=run --eval-command=bt --return-child-result --args ' + cppcheck_cmd + " -j1" - returncode, stdout, stderr, elapsedTime = runCommand(cmd) + returncode, stdout, stderr, elapsed_time = run_command(cmd) gdb_pos = stdout.find(" received signal") if not gdb_pos == -1: last_check_pos = stdout.rfind('Checking ', 0, gdb_pos) @@ -322,10 +322,10 @@ def scanPackage(workPath, cppcheckPath, jobs): if re.match(r'.*:[0-9]+:.*\]$', line): count += 1 print('Number of issues: ' + str(count)) - return count, ''.join(issue_messages_list), ''.join(information_messages_list), elapsedTime, options + return count, ''.join(issue_messages_list), ''.join(information_messages_list), elapsed_time, options -def splitResults(results): +def split_results(results): ret = [] w = None for line in results.split('\n'): @@ -340,11 +340,11 @@ def splitResults(results): return ret -def diffResults(workPath, ver1, results1, ver2, results2): +def diff_results(work_path, ver1, results1, ver2, results2): print('Diff results..') ret = '' - r1 = sorted(splitResults(results1)) - r2 = sorted(splitResults(results2)) + r1 = sorted(split_results(results1)) + r2 = sorted(split_results(results2)) i1 = 0 i2 = 0 while i1 < len(r1) and i2 < len(r2): @@ -367,7 +367,7 @@ def diffResults(workPath, ver1, results1, ver2, results2): return ret -def sendAll(connection, data): +def send_all(connection, data): bytes = data.encode('ascii', 'ignore') while bytes: num = connection.send(bytes) @@ -377,7 +377,7 @@ def sendAll(connection, data): bytes = None -def uploadResults(package, results, server_address): +def upload_results(package, results, server_address): print('Uploading results.. ' + str(len(results)) + ' bytes') max_retries = 4 for retry in range(max_retries): @@ -385,7 +385,7 @@ def uploadResults(package, results, server_address): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(server_address) cmd = 'write\n' - sendAll(sock, cmd + package + '\n' + results + '\nDONE') + send_all(sock, cmd + package + '\n' + results + '\nDONE') sock.close() print('Results have been successfully uploaded.') return True @@ -398,14 +398,14 @@ def uploadResults(package, results, server_address): return False -def uploadInfo(package, info_output, server_address): +def upload_info(package, info_output, server_address): print('Uploading information output.. ' + str(len(info_output)) + ' bytes') max_retries = 3 for retry in range(max_retries): try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(server_address) - sendAll(sock, 'write_info\n' + package + '\n' + info_output + '\nDONE') + send_all(sock, 'write_info\n' + package + '\n' + info_output + '\nDONE') sock.close() print('Information output has been successfully uploaded.') return True @@ -419,9 +419,9 @@ def uploadInfo(package, info_output, server_address): jobs = '-j1' -stopTime = None -workpath = os.path.expanduser('~/cppcheck-donate-cpu-workfolder') -packageUrl = None +stop_time = None +work_path = os.path.expanduser('~/cppcheck-donate-cpu-workfolder') +package_url = None server_address = ('cppcheck.osuosl.org', 8000) bandwidth_limit = None max_packages = None @@ -429,18 +429,18 @@ do_upload = True for arg in sys.argv[1:]: # --stop-time=12:00 => run until ~12:00 and then stop if arg.startswith('--stop-time='): - stopTime = arg[-5:] - print('Stop time:' + stopTime) + stop_time = arg[-5:] + print('Stop time:' + stop_time) elif arg.startswith('-j'): jobs = arg print('Jobs:' + jobs[2:]) elif arg.startswith('--package='): - packageUrl = arg[arg.find('=')+1:] - print('Package:' + packageUrl) + package_url = arg[arg.find('=')+1:] + print('Package:' + package_url) elif arg.startswith('--work-path='): - workpath = arg[arg.find('=')+1:] - print('workpath:' + workpath) - if not os.path.exists(workpath): + work_path = arg[arg.find('=')+1:] + print('work_path:' + work_path) + if not os.path.exists(work_path): print('work path does not exist!') sys.exit(1) elif arg == '--test': @@ -471,7 +471,7 @@ for arg in sys.argv[1:]: print(' --package=url Check a specific package and then stop. Can be useful if you want to reproduce') print(' some warning/crash/exception/etc..') print(' --stop-time=HH:MM Stop analysis when time has passed. Default is that you must terminate the script.') - print(' --work-path=path Work folder path. Default path is ' + workpath) + print(' --work-path=path Work folder path. Default path is ' + work_path) print(' --bandwidth-limit=limit Limit download rate for packages. Format for limit is the same that wget uses.') print(' Examples: --bandwidth-limit=250k => max. 250 kilobytes per second') print(' --bandwidth-limit=2m => max. 2 megabytes per second') @@ -485,7 +485,7 @@ for arg in sys.argv[1:]: sys.exit(1) print('Thank you!') -if not checkRequirements(): +if not check_requirements(): sys.exit(1) if bandwidth_limit and isinstance(bandwidth_limit, str): if subprocess.call(['wget', '--limit-rate=' + bandwidth_limit, '-q', '--spider', 'cppcheck.osuosl.org']) is 2: @@ -493,13 +493,13 @@ if bandwidth_limit and isinstance(bandwidth_limit, str): sys.exit(1) else: print('Bandwidth-limit: ' + bandwidth_limit) -if packageUrl: +if package_url: max_packages = 1 if max_packages: print('Maximum number of packages to download and analyze: {}'.format(max_packages)) -if not os.path.exists(workpath): - os.mkdir(workpath) -cppcheckPath = workpath + '/cppcheck' +if not os.path.exists(work_path): + os.mkdir(work_path) +cppcheck_path = os.path.join(work_path, 'cppcheck') packages_processed = 0 while True: if max_packages: @@ -509,63 +509,63 @@ while True: else: print('Processing package {} of the specified {} package(s).'.format(packages_processed + 1, max_packages)) packages_processed += 1 - if stopTime: - print('stopTime:' + stopTime + '. Time:' + time.strftime('%H:%M') + '.') - if stopTime < time.strftime('%H:%M'): + if stop_time: + print('stop_time:' + stop_time + '. Time:' + time.strftime('%H:%M') + '.') + if stop_time < time.strftime('%H:%M'): print('Stopping. Thank you!') sys.exit(0) - if not getCppcheck(cppcheckPath): + if not get_cppcheck(cppcheck_path): print('Failed to clone Cppcheck, retry later') sys.exit(1) - cppcheckVersions = getCppcheckVersions(server_address) - if cppcheckVersions is None: + cppcheck_versions = get_cppcheck_versions(server_address) + if cppcheck_versions is None: print('Failed to communicate with server, retry later') sys.exit(1) - if len(cppcheckVersions) == 0: + if len(cppcheck_versions) == 0: print('Did not get any cppcheck versions from server, retry later') sys.exit(1) - for ver in cppcheckVersions: + for ver in cppcheck_versions: if ver == 'head': - if not compile(cppcheckPath, jobs): + if not compile(cppcheck_path, jobs): print('Failed to compile Cppcheck, retry later') sys.exit(1) - elif not compile_version(workpath, jobs, ver): + elif not compile_version(work_path, jobs, ver): print('Failed to compile Cppcheck-{}, retry later'.format(ver)) sys.exit(1) - if packageUrl: - package = packageUrl + if package_url: + package = package_url else: - package = getPackage(server_address) + package = get_package(server_address) while len(package) == 0: print("network or server might be temporarily down.. will try again in 30 seconds..") time.sleep(30) - package = getPackage(server_address) - tgz = downloadPackage(workpath, package, bandwidth_limit) + package = get_package(server_address) + tgz = download_package(work_path, package, bandwidth_limit) if tgz is None: print("No package downloaded") continue - if not unpackPackage(workpath, tgz): + if not unpack_package(work_path, tgz): print("No files to process") continue crash = False count = '' - elapsedTime = '' - resultsToDiff = [] + elapsed_time = '' + results_to_diff = [] cppcheck_options = '' head_info_msg = '' - for ver in cppcheckVersions: + for ver in cppcheck_versions: if ver == 'head': current_cppcheck_dir = 'cppcheck' else: current_cppcheck_dir = ver - c, errout, info, t, cppcheck_options = scanPackage(workpath, current_cppcheck_dir, jobs) + c, errout, info, t, cppcheck_options = scan_package(work_path, current_cppcheck_dir, jobs) if c < 0: crash = True count += ' Crash!' else: count += ' ' + str(c) - elapsedTime += " {:.1f}".format(t) - resultsToDiff.append(errout) + elapsed_time += " {:.1f}".format(t) + results_to_diff.append(errout) if ver == 'head': head_info_msg = info @@ -573,24 +573,24 @@ while True: output += 'platform: ' + platform.platform() + '\n' output += 'python: ' + platform.python_version() + '\n' output += 'client-version: ' + CLIENT_VERSION + '\n' - output += 'cppcheck: ' + ' '.join(cppcheckVersions) + '\n' + output += 'cppcheck: ' + ' '.join(cppcheck_versions) + '\n' output += 'count:' + count + '\n' - output += 'elapsed-time:' + elapsedTime + '\n' + output += 'elapsed-time:' + elapsed_time + '\n' info_output = output info_output += 'info messages:\n' + head_info_msg - if 'head' in cppcheckVersions: - output += 'head results:\n' + resultsToDiff[cppcheckVersions.index('head')] + if 'head' in cppcheck_versions: + output += 'head results:\n' + results_to_diff[cppcheck_versions.index('head')] if not crash: - output += 'diff:\n' + diffResults(workpath, cppcheckVersions[0], resultsToDiff[0], cppcheckVersions[1], resultsToDiff[1]) + '\n' - if packageUrl: + output += 'diff:\n' + diff_results(work_path, cppcheck_versions[0], results_to_diff[0], cppcheck_versions[1], results_to_diff[1]) + '\n' + if package_url: print('=========================================================') print(output) print('=========================================================') print(info_output) print('=========================================================') if do_upload: - uploadResults(package, output, server_address) - uploadInfo(package, info_output, server_address) + upload_results(package, output, server_address) + upload_info(package, info_output, server_address) if not max_packages or packages_processed < max_packages: print('Sleep 5 seconds..') time.sleep(5)