walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

donate-cpu: improved client stability by consistently re-trying remote calls / cleanups (#4545)

This commit is contained in:
Oliver Stöneberg 2022-11-20 14:47:19 +01:00 committed by GitHub
parent e235297a14
commit a7dde5d470
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 64 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
tools/donate-cpu.py
View File

@ -173,20 +173,18 @@ while True:
if stop_time < time.strftime('%H:%M'): if stop_time < time.strftime('%H:%M'):
print('Stopping. Thank you!') print('Stopping. Thank you!')
sys.exit(0) sys.exit(0)
cppcheck_versions = lib.get_cppcheck_versions() try:
if cppcheck_versions is None: cppcheck_versions = lib.try_retry(lib.get_cppcheck_versions, max_tries=3, sleep_duration=30.0, sleep_factor=1.0)
print('Failed to communicate with server, retry later') except Exception as e:
sys.exit(1) print('Failed to get cppcheck versions from server ({}), retry later'.format(e))
if len(cppcheck_versions) == 0:
print('Did not get any cppcheck versions from server, retry later')
sys.exit(1) sys.exit(1)
for ver in cppcheck_versions: for ver in cppcheck_versions:
if ver == 'head': if ver == 'head':
ver = 'main' ver = 'main'
current_cppcheck_dir = os.path.join(work_path, 'tree-'+ver) current_cppcheck_dir = os.path.join(work_path, 'tree-'+ver)
try:
print('Fetching Cppcheck-{}..'.format(ver)) print('Fetching Cppcheck-{}..'.format(ver))
lib.try_retry(lib.checkout_cppcheck_version, fargs=(repo_path, ver, current_cppcheck_dir)) try:
lib.try_retry(lib.checkout_cppcheck_version, fargs=(repo_path, ver, current_cppcheck_dir), max_tries=3, sleep_duration=30.0, sleep_factor=1.0)
except KeyboardInterrupt as e: except KeyboardInterrupt as e:
# Passthrough for user abort # Passthrough for user abort
raise e raise e
@ -204,7 +202,11 @@ while True:
if package_urls: if package_urls:
package = package_urls[packages_processed-1] package = package_urls[packages_processed-1]
else: else:
package = lib.get_package() try:
package = lib.try_retry(lib.get_package, max_tries=3, sleep_duration=30.0, sleep_factor=1.0)
except Exception as e:
print('Error: Failed to get package ({}), retry later'.format(e))
sys.exit(1)
tgz = lib.download_package(work_path, package, bandwidth_limit) tgz = lib.download_package(work_path, package, bandwidth_limit)
if tgz is None: if tgz is None:
print("No package downloaded") print("No package downloaded")
@ -243,7 +245,7 @@ while True:
p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, universal_newlines=True) p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, universal_newlines=True)
try: try:
comm = p.communicate() comm = p.communicate()
return comm[0] return comm[0].strip()
except: except:
return None return None

79
tools/donate_cpu_lib.py
View File

@ -15,7 +15,7 @@ import shlex
# Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/ # Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/
# Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic # Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic
# changes) # changes)
CLIENT_VERSION = "1.3.37" CLIENT_VERSION = "1.3.38"
# Timeout for analysis with Cppcheck in seconds # Timeout for analysis with Cppcheck in seconds
CPPCHECK_TIMEOUT = 30 * 60 CPPCHECK_TIMEOUT = 30 * 60
@ -80,8 +80,7 @@ def check_requirements():
# Try and retry with exponential backoff if an exception is raised # Try and retry with exponential backoff if an exception is raised
def try_retry(fun, fargs=(), max_tries=5): def try_retry(fun, fargs=(), max_tries=5, sleep_duration=5.0, sleep_factor=2.0):
sleep_duration = 5.0
for i in range(max_tries): for i in range(max_tries):
try: try:
return fun(*fargs) return fun(*fargs)
@ -93,7 +92,7 @@ def try_retry(fun, fargs=(), max_tries=5):
print("{} in {}: {}".format(type(e).__name__, fun.__name__, str(e))) print("{} in {}: {}".format(type(e).__name__, fun.__name__, str(e)))
print("Trying {} again in {} seconds".format(fun.__name__, sleep_duration)) print("Trying {} again in {} seconds".format(fun.__name__, sleep_duration))
time.sleep(sleep_duration) time.sleep(sleep_duration)
sleep_duration *= 2.0 sleep_duration *= sleep_factor
else: else:
print("Maximum number of tries reached for {}".format(fun.__name__)) print("Maximum number of tries reached for {}".format(fun.__name__))
raise e raise e
@ -210,35 +209,30 @@ def compile_cppcheck(cppcheck_path):
def get_cppcheck_versions(): def get_cppcheck_versions():
print('Connecting to server to get Cppcheck versions..') print('Connecting to server to get Cppcheck versions..')
try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.connect(__server_address) sock.connect(__server_address)
sock.send(b'GetCppcheckVersions\n') sock.send(b'GetCppcheckVersions\n')
versions = sock.recv(256) versions = sock.recv(256)
except socket.error as err: # TODO: sock.recv() sometimes hangs and returns b'' afterwards
print('Failed to get cppcheck versions: ' + str(err)) if not versions:
return None raise Exception('received empty response')
return versions.decode('utf-8').split() return versions.decode('utf-8').split()
def get_packages_count(): def get_packages_count():
print('Connecting to server to get count of packages..') print('Connecting to server to get count of packages..')
try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.connect(__server_address) sock.connect(__server_address)
sock.send(b'getPackagesCount\n') sock.send(b'getPackagesCount\n')
packages = int(sock.recv(64)) packages = sock.recv(64)
except socket.error as err: # TODO: sock.recv() sometimes hangs and returns b'' afterwards
print('Failed to get count of packages: ' + str(err)) if not packages:
return None raise Exception('received empty response')
return packages return int(packages)
def get_package(package_index=None): def get_package(package_index=None):
package = b''
while not package:
print('Connecting to server to get assigned work..') print('Connecting to server to get assigned work..')
try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.connect(__server_address) sock.connect(__server_address)
if package_index is None: if package_index is None:
@ -247,9 +241,9 @@ def get_package(package_index=None):
request = 'getPackageIdx:' + str(package_index) + '\n' request = 'getPackageIdx:' + str(package_index) + '\n'
sock.send(request.encode()) sock.send(request.encode())
package = sock.recv(256) package = sock.recv(256)
except socket.error: # TODO: sock.recv() sometimes hangs and returns b'' afterwards
print("network or server might be temporarily down.. will try again in 30 seconds..") if not package:
time.sleep(30) raise Exception('received empty response')
return package.decode('utf-8') return package.decode('utf-8')
@ -586,29 +580,28 @@ def __send_all(connection, data):
bytes_ = None bytes_ = None
def __upload(cmd, data, cmd_info):
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.connect(__server_address)
__send_all(sock, '{}\n{}'.format(cmd, data))
print('{} has been successfully uploaded.'.format(cmd_info))
return True
def upload_results(package, results): def upload_results(package, results):
if not __make_cmd == 'make': if not __make_cmd == 'make':
print('Error: Result upload not performed - only make build binaries are currently fully supported') print('Error: Result upload not performed - only make build binaries are currently fully supported')
return False return False
print('Uploading results.. ' + str(len(results)) + ' bytes') print('Uploading results.. ' + str(len(results)) + ' bytes')
max_retries = 4
for retry in range(max_retries):
try: try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: try_retry(__upload, fargs=('write\n' + package, results + '\nDONE', 'Result'), max_tries=4, sleep_duration=30, sleep_factor=1)
sock.connect(__server_address) except Exception as e:
cmd = 'write\n' print('Result upload permanently failed ({})!'.format(e))
__send_all(sock, cmd + package + '\n' + results + '\nDONE')
print('Results have been successfully uploaded.')
return True
except socket.error as err:
print('Upload error: ' + str(err))
if retry < (max_retries - 1):
print('Retrying upload in 30 seconds')
time.sleep(30)
print('Upload permanently failed!')
return False return False
return True
def upload_info(package, info_output): def upload_info(package, info_output):
if not __make_cmd == 'make': if not __make_cmd == 'make':
@ -616,22 +609,14 @@ def upload_info(package, info_output):
return False return False
print('Uploading information output.. ' + str(len(info_output)) + ' bytes') print('Uploading information output.. ' + str(len(info_output)) + ' bytes')
max_retries = 3
for retry in range(max_retries):
try: try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: try_retry(__upload, fargs=('write_info\n' + package, info_output + '\nDONE', 'Information'), max_tries=3, sleep_duration=30, sleep_factor=1)
sock.connect(__server_address) except Exception as e:
__send_all(sock, 'write_info\n' + package + '\n' + info_output + '\nDONE') print('Information upload permanently failed ({})!'.format(e))
print('Information output has been successfully uploaded.')
return True
except socket.error as err:
print('Upload error: ' + str(err))
if retry < (max_retries - 1):
print('Retrying upload in 30 seconds')
time.sleep(30)
print('Upload permanently failed!')
return False return False
return True
class LibraryIncludes: class LibraryIncludes:
def __init__(self): def __init__(self):