donate-cpu: improved client stability by consistently re-trying remote calls / cleanups (#4545)
This commit is contained in:
parent
e235297a14
commit
a7dde5d470
|
@ -173,20 +173,18 @@ while True:
|
||||||
if stop_time < time.strftime('%H:%M'):
|
if stop_time < time.strftime('%H:%M'):
|
||||||
print('Stopping. Thank you!')
|
print('Stopping. Thank you!')
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
cppcheck_versions = lib.get_cppcheck_versions()
|
try:
|
||||||
if cppcheck_versions is None:
|
cppcheck_versions = lib.try_retry(lib.get_cppcheck_versions, max_tries=3, sleep_duration=30.0, sleep_factor=1.0)
|
||||||
print('Failed to communicate with server, retry later')
|
except Exception as e:
|
||||||
sys.exit(1)
|
print('Failed to get cppcheck versions from server ({}), retry later'.format(e))
|
||||||
if len(cppcheck_versions) == 0:
|
|
||||||
print('Did not get any cppcheck versions from server, retry later')
|
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
for ver in cppcheck_versions:
|
for ver in cppcheck_versions:
|
||||||
if ver == 'head':
|
if ver == 'head':
|
||||||
ver = 'main'
|
ver = 'main'
|
||||||
current_cppcheck_dir = os.path.join(work_path, 'tree-'+ver)
|
current_cppcheck_dir = os.path.join(work_path, 'tree-'+ver)
|
||||||
|
print('Fetching Cppcheck-{}..'.format(ver))
|
||||||
try:
|
try:
|
||||||
print('Fetching Cppcheck-{}..'.format(ver))
|
lib.try_retry(lib.checkout_cppcheck_version, fargs=(repo_path, ver, current_cppcheck_dir), max_tries=3, sleep_duration=30.0, sleep_factor=1.0)
|
||||||
lib.try_retry(lib.checkout_cppcheck_version, fargs=(repo_path, ver, current_cppcheck_dir))
|
|
||||||
except KeyboardInterrupt as e:
|
except KeyboardInterrupt as e:
|
||||||
# Passthrough for user abort
|
# Passthrough for user abort
|
||||||
raise e
|
raise e
|
||||||
|
@ -204,7 +202,11 @@ while True:
|
||||||
if package_urls:
|
if package_urls:
|
||||||
package = package_urls[packages_processed-1]
|
package = package_urls[packages_processed-1]
|
||||||
else:
|
else:
|
||||||
package = lib.get_package()
|
try:
|
||||||
|
package = lib.try_retry(lib.get_package, max_tries=3, sleep_duration=30.0, sleep_factor=1.0)
|
||||||
|
except Exception as e:
|
||||||
|
print('Error: Failed to get package ({}), retry later'.format(e))
|
||||||
|
sys.exit(1)
|
||||||
tgz = lib.download_package(work_path, package, bandwidth_limit)
|
tgz = lib.download_package(work_path, package, bandwidth_limit)
|
||||||
if tgz is None:
|
if tgz is None:
|
||||||
print("No package downloaded")
|
print("No package downloaded")
|
||||||
|
@ -243,7 +245,7 @@ while True:
|
||||||
p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, universal_newlines=True)
|
p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, universal_newlines=True)
|
||||||
try:
|
try:
|
||||||
comm = p.communicate()
|
comm = p.communicate()
|
||||||
return comm[0]
|
return comm[0].strip()
|
||||||
except:
|
except:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
|
@ -15,7 +15,7 @@ import shlex
|
||||||
# Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/
|
# Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/
|
||||||
# Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic
|
# Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic
|
||||||
# changes)
|
# changes)
|
||||||
CLIENT_VERSION = "1.3.37"
|
CLIENT_VERSION = "1.3.38"
|
||||||
|
|
||||||
# Timeout for analysis with Cppcheck in seconds
|
# Timeout for analysis with Cppcheck in seconds
|
||||||
CPPCHECK_TIMEOUT = 30 * 60
|
CPPCHECK_TIMEOUT = 30 * 60
|
||||||
|
@ -80,8 +80,7 @@ def check_requirements():
|
||||||
|
|
||||||
|
|
||||||
# Try and retry with exponential backoff if an exception is raised
|
# Try and retry with exponential backoff if an exception is raised
|
||||||
def try_retry(fun, fargs=(), max_tries=5):
|
def try_retry(fun, fargs=(), max_tries=5, sleep_duration=5.0, sleep_factor=2.0):
|
||||||
sleep_duration = 5.0
|
|
||||||
for i in range(max_tries):
|
for i in range(max_tries):
|
||||||
try:
|
try:
|
||||||
return fun(*fargs)
|
return fun(*fargs)
|
||||||
|
@ -93,7 +92,7 @@ def try_retry(fun, fargs=(), max_tries=5):
|
||||||
print("{} in {}: {}".format(type(e).__name__, fun.__name__, str(e)))
|
print("{} in {}: {}".format(type(e).__name__, fun.__name__, str(e)))
|
||||||
print("Trying {} again in {} seconds".format(fun.__name__, sleep_duration))
|
print("Trying {} again in {} seconds".format(fun.__name__, sleep_duration))
|
||||||
time.sleep(sleep_duration)
|
time.sleep(sleep_duration)
|
||||||
sleep_duration *= 2.0
|
sleep_duration *= sleep_factor
|
||||||
else:
|
else:
|
||||||
print("Maximum number of tries reached for {}".format(fun.__name__))
|
print("Maximum number of tries reached for {}".format(fun.__name__))
|
||||||
raise e
|
raise e
|
||||||
|
@ -210,46 +209,41 @@ def compile_cppcheck(cppcheck_path):
|
||||||
|
|
||||||
def get_cppcheck_versions():
|
def get_cppcheck_versions():
|
||||||
print('Connecting to server to get Cppcheck versions..')
|
print('Connecting to server to get Cppcheck versions..')
|
||||||
try:
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
sock.connect(__server_address)
|
||||||
sock.connect(__server_address)
|
sock.send(b'GetCppcheckVersions\n')
|
||||||
sock.send(b'GetCppcheckVersions\n')
|
versions = sock.recv(256)
|
||||||
versions = sock.recv(256)
|
# TODO: sock.recv() sometimes hangs and returns b'' afterwards
|
||||||
except socket.error as err:
|
if not versions:
|
||||||
print('Failed to get cppcheck versions: ' + str(err))
|
raise Exception('received empty response')
|
||||||
return None
|
|
||||||
return versions.decode('utf-8').split()
|
return versions.decode('utf-8').split()
|
||||||
|
|
||||||
|
|
||||||
def get_packages_count():
|
def get_packages_count():
|
||||||
print('Connecting to server to get count of packages..')
|
print('Connecting to server to get count of packages..')
|
||||||
try:
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
sock.connect(__server_address)
|
||||||
sock.connect(__server_address)
|
sock.send(b'getPackagesCount\n')
|
||||||
sock.send(b'getPackagesCount\n')
|
packages = sock.recv(64)
|
||||||
packages = int(sock.recv(64))
|
# TODO: sock.recv() sometimes hangs and returns b'' afterwards
|
||||||
except socket.error as err:
|
if not packages:
|
||||||
print('Failed to get count of packages: ' + str(err))
|
raise Exception('received empty response')
|
||||||
return None
|
return int(packages)
|
||||||
return packages
|
|
||||||
|
|
||||||
|
|
||||||
def get_package(package_index=None):
|
def get_package(package_index=None):
|
||||||
package = b''
|
print('Connecting to server to get assigned work..')
|
||||||
while not package:
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||||
print('Connecting to server to get assigned work..')
|
sock.connect(__server_address)
|
||||||
try:
|
if package_index is None:
|
||||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
sock.send(b'get\n')
|
||||||
sock.connect(__server_address)
|
else:
|
||||||
if package_index is None:
|
request = 'getPackageIdx:' + str(package_index) + '\n'
|
||||||
sock.send(b'get\n')
|
sock.send(request.encode())
|
||||||
else:
|
package = sock.recv(256)
|
||||||
request = 'getPackageIdx:' + str(package_index) + '\n'
|
# TODO: sock.recv() sometimes hangs and returns b'' afterwards
|
||||||
sock.send(request.encode())
|
if not package:
|
||||||
package = sock.recv(256)
|
raise Exception('received empty response')
|
||||||
except socket.error:
|
|
||||||
print("network or server might be temporarily down.. will try again in 30 seconds..")
|
|
||||||
time.sleep(30)
|
|
||||||
return package.decode('utf-8')
|
return package.decode('utf-8')
|
||||||
|
|
||||||
|
|
||||||
|
@ -586,28 +580,27 @@ def __send_all(connection, data):
|
||||||
bytes_ = None
|
bytes_ = None
|
||||||
|
|
||||||
|
|
||||||
|
def __upload(cmd, data, cmd_info):
|
||||||
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||||
|
sock.connect(__server_address)
|
||||||
|
__send_all(sock, '{}\n{}'.format(cmd, data))
|
||||||
|
print('{} has been successfully uploaded.'.format(cmd_info))
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def upload_results(package, results):
|
def upload_results(package, results):
|
||||||
if not __make_cmd == 'make':
|
if not __make_cmd == 'make':
|
||||||
print('Error: Result upload not performed - only make build binaries are currently fully supported')
|
print('Error: Result upload not performed - only make build binaries are currently fully supported')
|
||||||
return False
|
return False
|
||||||
|
|
||||||
print('Uploading results.. ' + str(len(results)) + ' bytes')
|
print('Uploading results.. ' + str(len(results)) + ' bytes')
|
||||||
max_retries = 4
|
try:
|
||||||
for retry in range(max_retries):
|
try_retry(__upload, fargs=('write\n' + package, results + '\nDONE', 'Result'), max_tries=4, sleep_duration=30, sleep_factor=1)
|
||||||
try:
|
except Exception as e:
|
||||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
print('Result upload permanently failed ({})!'.format(e))
|
||||||
sock.connect(__server_address)
|
return False
|
||||||
cmd = 'write\n'
|
|
||||||
__send_all(sock, cmd + package + '\n' + results + '\nDONE')
|
return True
|
||||||
print('Results have been successfully uploaded.')
|
|
||||||
return True
|
|
||||||
except socket.error as err:
|
|
||||||
print('Upload error: ' + str(err))
|
|
||||||
if retry < (max_retries - 1):
|
|
||||||
print('Retrying upload in 30 seconds')
|
|
||||||
time.sleep(30)
|
|
||||||
print('Upload permanently failed!')
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def upload_info(package, info_output):
|
def upload_info(package, info_output):
|
||||||
|
@ -616,21 +609,13 @@ def upload_info(package, info_output):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
print('Uploading information output.. ' + str(len(info_output)) + ' bytes')
|
print('Uploading information output.. ' + str(len(info_output)) + ' bytes')
|
||||||
max_retries = 3
|
try:
|
||||||
for retry in range(max_retries):
|
try_retry(__upload, fargs=('write_info\n' + package, info_output + '\nDONE', 'Information'), max_tries=3, sleep_duration=30, sleep_factor=1)
|
||||||
try:
|
except Exception as e:
|
||||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
print('Information upload permanently failed ({})!'.format(e))
|
||||||
sock.connect(__server_address)
|
return False
|
||||||
__send_all(sock, 'write_info\n' + package + '\n' + info_output + '\nDONE')
|
|
||||||
print('Information output has been successfully uploaded.')
|
return True
|
||||||
return True
|
|
||||||
except socket.error as err:
|
|
||||||
print('Upload error: ' + str(err))
|
|
||||||
if retry < (max_retries - 1):
|
|
||||||
print('Retrying upload in 30 seconds')
|
|
||||||
time.sleep(30)
|
|
||||||
print('Upload permanently failed!')
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
class LibraryIncludes:
|
class LibraryIncludes:
|
||||||
|
|
Loading…
Reference in New Issue