diff --git a/tools/donate-cpu.py b/tools/donate-cpu.py index 03d6eb724..951a5418c 100755 --- a/tools/donate-cpu.py +++ b/tools/donate-cpu.py @@ -47,7 +47,7 @@ for arg in sys.argv[1:]: package_url = arg[arg.find('=')+1:] print('Package:' + package_url) elif arg.startswith('--work-path='): - work_path = arg[arg.find('=')+1:] + work_path = os.path.abspath(arg[arg.find('=')+1:]) print('work_path:' + work_path) if not os.path.exists(work_path): print('work path does not exist!') @@ -116,8 +116,19 @@ if max_packages: print('Maximum number of packages to download and analyze: {}'.format(max_packages)) if not os.path.exists(work_path): os.mkdir(work_path) -cppcheck_path = os.path.join(work_path, 'cppcheck') +repo_path = os.path.join(work_path, 'repo') +# This is a temporary migration step which should be removed in the future +migrate_repo_path = os.path.join(work_path, 'cppcheck') + packages_processed = 0 + +print('Get Cppcheck..') +try: + try_retry(clone_cppcheck, fargs=(repo_path, migrate_repo_path)) +except: + print('Error: Failed to clone Cppcheck, retry later') + sys.exit(1) + while True: if max_packages: if packages_processed >= max_packages: @@ -130,9 +141,6 @@ while True: if stop_time < time.strftime('%H:%M'): print('Stopping. Thank you!') sys.exit(0) - if not get_cppcheck(cppcheck_path, work_path): - print('Failed to clone Cppcheck, retry later') - sys.exit(1) cppcheck_versions = get_cppcheck_versions(server_address) if cppcheck_versions is None: print('Failed to communicate with server, retry later') @@ -142,12 +150,25 @@ while True: sys.exit(1) for ver in cppcheck_versions: if ver == 'head': - if not compile_cppcheck(cppcheck_path, jobs): - print('Failed to compile Cppcheck, retry later') - sys.exit(1) - elif not compile_version(work_path, jobs, ver): - print('Failed to compile Cppcheck-{}, retry later'.format(ver)) + ver = 'main' + current_cppcheck_dir = os.path.join(work_path, 'tree-'+ver) + try: + print('Fetching Cppcheck-{}..'.format(ver)) + try_retry(checkout_cppcheck_version, fargs=(repo_path, ver, current_cppcheck_dir)) + except KeyboardInterrupt as e: + # Passthrough for user abort + raise e + except: + print('Failed to update Cppcheck, retry later') sys.exit(1) + if ver == 'main': + if not compile_cppcheck(current_cppcheck_dir, jobs): + print('Failed to compile Cppcheck-{}, retry later'.format(ver)) + sys.exit(1) + else: + if not compile_version(current_cppcheck_dir, jobs): + print('Failed to compile Cppcheck-{}, retry later'.format(ver)) + sys.exit(1) if package_url: package = package_url else: @@ -172,12 +193,11 @@ while True: libraries = get_libraries() for ver in cppcheck_versions: + tree_path = os.path.join(work_path, 'tree-'+ver) if ver == 'head': - current_cppcheck_dir = 'cppcheck' - cppcheck_head_info = get_cppcheck_info(work_path + '/cppcheck') - else: - current_cppcheck_dir = ver - c, errout, info, t, cppcheck_options, timing_info = scan_package(work_path, current_cppcheck_dir, jobs, libraries) + tree_path = os.path.join(work_path, 'tree-main') + cppcheck_head_info = get_cppcheck_info(tree_path) + c, errout, info, t, cppcheck_options, timing_info = scan_package(work_path, tree_path, jobs, libraries) if c < 0: if c == -101 and 'error: could not find or open any of the paths given.' in errout: # No sourcefile found (for example only headers present) diff --git a/tools/donate_cpu_lib.py b/tools/donate_cpu_lib.py index 3fbe29235..492700eab 100644 --- a/tools/donate_cpu_lib.py +++ b/tools/donate_cpu_lib.py @@ -15,11 +15,13 @@ import shlex # Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/ # Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic # changes) -CLIENT_VERSION = "1.3.16" +CLIENT_VERSION = "1.3.17" # Timeout for analysis with Cppcheck in seconds CPPCHECK_TIMEOUT = 30 * 60 +CPPCHECK_REPO_URL = "https://github.com/danmar/cppcheck.git" + # Return code that is used to mark a timed out analysis RETURN_CODE_TIMEOUT = -999 @@ -35,42 +37,59 @@ def check_requirements(): return result -def get_cppcheck(cppcheck_path, work_path): - print('Get Cppcheck..') - for i in range(5): - if os.path.exists(cppcheck_path): - try: - os.chdir(cppcheck_path) - try: - subprocess.check_call(['git', 'checkout', '-f', 'main']) - except subprocess.CalledProcessError: - subprocess.check_call(['git', 'checkout', '-f', 'master']) - subprocess.check_call(['git', 'pull']) - subprocess.check_call(['git', 'checkout', 'origin/main', '-b', 'main']) - subprocess.check_call(['git', 'pull']) - except: - print('Failed to update Cppcheck sources! Retrying..') - time.sleep(10) - continue - else: - try: - subprocess.check_call(['git', 'clone', 'https://github.com/danmar/cppcheck.git', cppcheck_path]) - except: - print('Failed to clone, will try again in 10 minutes..') - time.sleep(600) - continue - time.sleep(2) - return True - if os.path.exists(cppcheck_path): - print('Failed to update Cppcheck sources, trying a fresh clone..') +# Try and retry with exponential backoff if an exception is raised +def try_retry(fun, fargs=(), max_tries=5): + sleep_duration = 5.0 + for i in range(max_tries): try: - os.chdir(work_path) - shutil.rmtree(cppcheck_path) - get_cppcheck(cppcheck_path, work_path) - except: - print('Failed to remove Cppcheck folder, please manually remove ' + work_path) - return False - return False + return fun(*fargs) + except KeyboardInterrupt as e: + # Do not retry in case of user abort + raise e + except BaseException as e: + if i < max_tries - 1: + print("{} in {}: {}".format(type(e).__name__, fun.__name__, str(e))) + print("Trying {} again in {} seconds".format(fun.__name__, sleep_duration)) + time.sleep(sleep_duration) + sleep_duration *= 2.0 + else: + print("Maximum number of tries reached for {}".format(fun.__name__)) + raise e + + +def clone_cppcheck(repo_path, migrate_from_path): + repo_git_dir = os.path.join(repo_path, '.git') + if os.path.exists(repo_git_dir): + return + # Attempt to migrate clone directory used prior to 1.3.17 + if os.path.exists(migrate_from_path): + os.rename(migrate_from_path, repo_path) + else: + # A shallow git clone (depth = 1) is enough for building and scanning. + # Do not checkout until fetch_cppcheck_version. + subprocess.check_call(['git', 'clone', '--depth=1', '--no-checkout', CPPCHECK_REPO_URL, repo_path]) + # Checkout an empty branch to allow "git worktree add" for main later on + try: + # git >= 2.27 + subprocess.check_call(['git', 'switch', '--orphan', 'empty'], cwd=repo_path) + except subprocess.CalledProcessError: + subprocess.check_call(['git', 'checkout','--orphan', 'empty'], cwd=repo_path) + + +def checkout_cppcheck_version(repo_path, version, cppcheck_path): + if not os.path.isabs(cppcheck_path): + raise ValueError("cppcheck_path is not an absolute path") + if os.path.exists(cppcheck_path): + subprocess.check_call(['git', 'checkout' , '-f', version], cwd=cppcheck_path) + # It is possible to pull branches, not tags + if version == 'main': + subprocess.check_call(['git', 'pull'], cwd=cppcheck_path) + else: + if version != 'main': + # Since this is a shallow clone, explicitly fetch the remote version tag + refspec = 'refs/tags/' + version + ':ref/tags/' + version + subprocess.check_call(['git', 'fetch', '--depth=1', 'origin', refspec], cwd=repo_path) + subprocess.check_call(['git', 'worktree', 'add', cppcheck_path, version], cwd=repo_path) def get_cppcheck_info(cppcheck_path): @@ -81,28 +100,18 @@ def get_cppcheck_info(cppcheck_path): return '' -def compile_version(work_path, jobs, version): - if os.path.isfile(work_path + '/' + version + '/cppcheck'): +def compile_version(cppcheck_path, jobs): + if os.path.isfile(os.path.join(cppcheck_path, 'cppcheck')): return True - os.chdir(work_path + '/cppcheck') - subprocess.call(['git', 'checkout', version]) - subprocess.call(['make', 'clean']) - subprocess.call(['make', jobs, 'MATCHCOMPILER=yes', 'CXXFLAGS=-O2 -g']) - if os.path.isfile(work_path + '/cppcheck/cppcheck'): - os.mkdir(work_path + '/' + version) - dest_path = work_path + '/' + version + '/' - subprocess.call(['cp', '-R', work_path + '/cppcheck/cfg', dest_path]) - subprocess.call(['cp', 'cppcheck', dest_path]) - subprocess.call(['git', 'checkout', 'main']) - try: - subprocess.call([work_path + '/' + version + '/cppcheck', '--version']) - except OSError: - return False - return True + # Build + ret = compile_cppcheck(cppcheck_path, jobs) + # Clean intermediate build files + subprocess.call(['git', 'clean', '-f', '-d', '-x', '--exclude', 'cppcheck'], cwd=cppcheck_path) + return ret def compile_cppcheck(cppcheck_path, jobs): - print('Compiling Cppcheck..') + print('Compiling {}'.format(os.path.basename(cppcheck_path))) try: os.chdir(cppcheck_path) subprocess.call(['make', jobs, 'MATCHCOMPILER=yes', 'CXXFLAGS=-O2 -g']) diff --git a/tools/test-my-pr.py b/tools/test-my-pr.py index 34daaffe4..c646abe5a 100755 --- a/tools/test-my-pr.py +++ b/tools/test-my-pr.py @@ -33,7 +33,9 @@ if __name__ == "__main__": work_path = os.path.abspath(args.work_path) if not os.path.exists(work_path): os.makedirs(work_path) - main_dir = os.path.join(work_path, 'cppcheck') + repo_dir = os.path.join(work_path, 'repo') + old_repo_dir = os.path.join(work_path, 'cppcheck') + main_dir = os.path.join(work_path, 'tree-main') jobs = '-j' + str(args.j) result_file = os.path.join(work_path, args.o) @@ -46,8 +48,18 @@ if __name__ == "__main__": if os.path.exists(timing_file): os.remove(timing_file) - if not lib.get_cppcheck(main_dir, work_path): - print('Failed to clone main of Cppcheck, retry later') + try: + lib.clone_cppcheck(repo_dir, old_repo_dir) + pass + except: + print('Failed to clone Cppcheck repository, retry later') + sys.exit(1) + + try: + lib.checkout_cppcheck_version(repo_dir, 'main', main_dir) + pass + except: + print('Failed to checkout main, retry later') sys.exit(1) try: @@ -62,8 +74,10 @@ if __name__ == "__main__": 'Package', 'main', 'your', 'Factor', package_width=package_width, timing_width=timing_width)) os.chdir(main_dir) + subprocess.check_call(['git', 'fetch', '--depth=1', 'origin', commit_id]) subprocess.check_call(['git', 'checkout', '-f', commit_id]) - except: + except BaseException as e: + print('Error: {}'.format(e)) print('Failed to switch to common ancestor of your branch and main') sys.exit(1)