donate-cpu-server.py: some minor improvements and cleanups (#4492)
* donate-cpu-server.py: bumped version * donate-cpu-server.py: adjusted some variables names * donate-cpu-server.py: small date string parsing improvement * donate-cpu-server.py: added timestamp to all messages * donate-cpu-server.py: bail out if `donated-results` folder is missing * donate-cpu-server.py: use more detailed timestamp in logging * donate-cpu-server.py: use `with` scopes
This commit is contained in:
Oliver Stöneberg
GitHub
parent
bb2ba53bf5
commit
1a464ec5ab
|
|
@ -25,7 +25,7 @@ import html as html_lib
|
|||
# Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/
|
||||
# Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic
|
||||
# changes)
|
||||
SERVER_VERSION = "1.3.28"
|
||||
SERVER_VERSION = "1.3.29"
|
||||
|
||||
OLD_VERSION = '2.9'
|
||||
|
||||
|
|
@ -46,6 +46,10 @@ handler_file.setLevel(logging.ERROR)
|
|||
logger.addHandler(handler_file)
|
||||
|
||||
|
||||
def print_ts(msg):
|
||||
print('[' + strDateTime() + '] ' + msg)
|
||||
|
||||
|
||||
# Set up an exception hook for all uncaught exceptions so they can be logged
|
||||
def handle_uncaught_exception(exc_type, exc_value, exc_traceback):
|
||||
if issubclass(exc_type, KeyboardInterrupt):
|
||||
|
|
@ -59,7 +63,7 @@ sys.excepthook = handle_uncaught_exception
|
|||
|
||||
|
||||
def strDateTime() -> str:
|
||||
return datetime.datetime.now().strftime('%Y-%m-%d %H:%M')
|
||||
return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
|
||||
|
||||
|
||||
def dateTimeFromStr(datestr: str) -> datetime.datetime:
|
||||
|
|
@ -148,13 +152,13 @@ def latestReport(latestResults: list) -> str:
|
|||
package = filename[filename.rfind('/')+1:]
|
||||
current_year = datetime.date.today().year
|
||||
|
||||
datestr = ''
|
||||
datestr = None
|
||||
count = ['0', '0']
|
||||
lost = 0
|
||||
added = 0
|
||||
for line in open(filename, 'rt'):
|
||||
line = line.strip()
|
||||
if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
|
||||
if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
|
||||
datestr = line
|
||||
#elif line.startswith('cppcheck:'):
|
||||
# cppcheck = line[9:]
|
||||
|
|
@ -185,8 +189,8 @@ def crashReport(results_path: str) -> str:
|
|||
for filename in sorted(glob.glob(os.path.expanduser(results_path + '/*'))):
|
||||
if not os.path.isfile(filename) or filename.endswith('.diff'):
|
||||
continue
|
||||
datestr = ''
|
||||
with open(filename, 'rt') as file_:
|
||||
datestr = None
|
||||
for line in file_:
|
||||
line = line.strip()
|
||||
if line.startswith('cppcheck: '):
|
||||
|
|
@ -196,21 +200,21 @@ def crashReport(results_path: str) -> str:
|
|||
else:
|
||||
# Current package, parse on
|
||||
continue
|
||||
if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
|
||||
if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
|
||||
datestr = line
|
||||
elif line.startswith('count:'):
|
||||
if line.find('Crash') < 0:
|
||||
break
|
||||
package = filename[filename.rfind('/')+1:]
|
||||
counts = line.split(' ')
|
||||
c2 = ''
|
||||
c_version = ''
|
||||
if counts[2] == 'Crash!':
|
||||
c2 = 'Crash'
|
||||
c1 = ''
|
||||
c_version = 'Crash'
|
||||
c_head = ''
|
||||
if counts[1] == 'Crash!':
|
||||
c1 = 'Crash'
|
||||
html += fmt(package, datestr, c2, c1) + '\n'
|
||||
if c1 != 'Crash':
|
||||
c_head = 'Crash'
|
||||
html += fmt(package, datestr, c_version, c_head) + '\n'
|
||||
if c_head != 'Crash':
|
||||
break
|
||||
elif line.find(' received signal ') != -1:
|
||||
crash_line = next(file_, '').strip()
|
||||
|
|
@ -243,7 +247,7 @@ def crashReport(results_path: str) -> str:
|
|||
stack_trace.append(m.group('number') + ' ' + m.group('function') + '(...) at ' + m.group('location'))
|
||||
continue
|
||||
|
||||
print('{} - unmatched stack frame - {}'.format(package, l))
|
||||
print_ts('{} - unmatched stack frame - {}'.format(package, l))
|
||||
break
|
||||
key = hash(' '.join(stack_trace))
|
||||
|
||||
|
|
@ -279,8 +283,8 @@ def timeoutReport(results_path: str) -> str:
|
|||
for filename in sorted(glob.glob(os.path.expanduser(results_path + '/*'))):
|
||||
if not os.path.isfile(filename) or filename.endswith('.diff'):
|
||||
continue
|
||||
datestr = ''
|
||||
with open(filename, 'rt') as file_:
|
||||
datestr = None
|
||||
for line in file_:
|
||||
line = line.strip()
|
||||
if line.startswith('cppcheck: '):
|
||||
|
|
@ -290,7 +294,7 @@ def timeoutReport(results_path: str) -> str:
|
|||
else:
|
||||
# Current package, parse on
|
||||
continue
|
||||
if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
|
||||
if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
|
||||
datestr = line
|
||||
elif line.startswith('count:'):
|
||||
if line.find('TO!') < 0:
|
||||
|
|
@ -680,7 +684,7 @@ def timeReport(resultPath: str, show_gt: bool) -> str:
|
|||
for filename in glob.glob(resultPath + '/*'):
|
||||
if not os.path.isfile(filename) or filename.endswith('.diff'):
|
||||
continue
|
||||
datestr = ''
|
||||
datestr = None
|
||||
for line in open(filename, 'rt'):
|
||||
line = line.strip()
|
||||
if line.startswith('cppcheck: '):
|
||||
|
|
@ -690,7 +694,7 @@ def timeReport(resultPath: str, show_gt: bool) -> str:
|
|||
else:
|
||||
# Current package, parse on
|
||||
continue
|
||||
if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
|
||||
if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
|
||||
datestr = line
|
||||
continue
|
||||
if not line.startswith('elapsed-time:'):
|
||||
|
|
@ -765,7 +769,7 @@ def timeReportSlow(resultPath: str) -> str:
|
|||
for filename in glob.glob(resultPath + '/*'):
|
||||
if not os.path.isfile(filename) or filename.endswith('.diff'):
|
||||
continue
|
||||
datestr = ''
|
||||
datestr = None
|
||||
for line in open(filename, 'rt'):
|
||||
line = line.strip()
|
||||
if line.startswith('cppcheck: '):
|
||||
|
|
@ -775,7 +779,7 @@ def timeReportSlow(resultPath: str) -> str:
|
|||
else:
|
||||
# Current package, parse on
|
||||
continue
|
||||
if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
|
||||
if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
|
||||
datestr = line
|
||||
continue
|
||||
elif line.startswith('count:'):
|
||||
|
|
@ -815,7 +819,7 @@ def timeReportSlow(resultPath: str) -> str:
|
|||
def check_library_report(result_path: str, message_id: str) -> str:
|
||||
if message_id not in ('checkLibraryNoReturn', 'checkLibraryFunction', 'checkLibraryUseIgnore', 'checkLibraryCheckType'):
|
||||
error_message = 'Invalid value ' + message_id + ' for message_id parameter.'
|
||||
print(error_message)
|
||||
print_ts(error_message)
|
||||
return error_message
|
||||
|
||||
if message_id == 'checkLibraryCheckType':
|
||||
|
|
@ -941,7 +945,7 @@ class HttpClientThread(Thread):
|
|||
def run(self):
|
||||
try:
|
||||
cmd = self.cmd
|
||||
print('[' + strDateTime() + '] ' + cmd)
|
||||
print_ts(cmd)
|
||||
res = re.match(r'GET /([a-zA-Z0-9_\-\.\+%]*) HTTP', cmd)
|
||||
if res is None:
|
||||
self.connection.close()
|
||||
|
|
@ -1012,16 +1016,15 @@ class HttpClientThread(Thread):
|
|||
else:
|
||||
filename = resultPath + '/' + url
|
||||
if not os.path.isfile(filename):
|
||||
print('HTTP/1.1 404 Not Found')
|
||||
print_ts('HTTP/1.1 404 Not Found')
|
||||
self.connection.send(b'HTTP/1.1 404 Not Found\r\n\r\n')
|
||||
else:
|
||||
f = open(filename, 'rt')
|
||||
data = f.read()
|
||||
f.close()
|
||||
with open(filename, 'rt') as f:
|
||||
data = f.read()
|
||||
httpGetResponse(self.connection, data, 'text/plain')
|
||||
except:
|
||||
tb = "".join(traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))
|
||||
print(tb)
|
||||
print_ts(tb)
|
||||
httpGetResponse(self.connection, tb, 'text/plain')
|
||||
finally:
|
||||
time.sleep(1)
|
||||
|
|
@ -1042,12 +1045,12 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
|
|||
with open('latest.txt', 'rt') as f:
|
||||
latestResults = f.read().strip().split(' ')
|
||||
|
||||
print('[' + strDateTime() + '] version ' + SERVER_VERSION)
|
||||
print('[' + strDateTime() + '] listening on port ' + str(server_address_port))
|
||||
print_ts('version ' + SERVER_VERSION)
|
||||
print_ts('listening on port ' + str(server_address_port))
|
||||
|
||||
while True:
|
||||
# wait for a connection
|
||||
print('[' + strDateTime() + '] waiting for a connection')
|
||||
print_ts('waiting for a connection')
|
||||
connection, client_address = sock.accept()
|
||||
try:
|
||||
bytes_received = connection.recv(128)
|
||||
|
|
@ -1057,7 +1060,7 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
|
|||
continue
|
||||
except UnicodeDecodeError as e:
|
||||
connection.close()
|
||||
print('Error: Decoding failed: ' + str(e))
|
||||
print_ts('Error: Decoding failed: ' + str(e))
|
||||
continue
|
||||
if cmd.find('\n') < 1:
|
||||
continue
|
||||
|
|
@ -1070,7 +1073,7 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
|
|||
newThread.start()
|
||||
elif cmd == 'GetCppcheckVersions\n':
|
||||
reply = 'head ' + OLD_VERSION
|
||||
print('[' + strDateTime() + '] GetCppcheckVersions: ' + reply)
|
||||
print_ts('GetCppcheckVersions: ' + reply)
|
||||
connection.send(reply.encode('utf-8', 'ignore'))
|
||||
connection.close()
|
||||
elif cmd == 'get\n':
|
||||
|
|
@ -1079,11 +1082,10 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
|
|||
if packageIndex >= len(packages):
|
||||
packageIndex = 0
|
||||
|
||||
f = open('package-index.txt', 'wt')
|
||||
f.write(str(packageIndex) + '\n')
|
||||
f.close()
|
||||
with open('package-index.txt', 'wt') as f:
|
||||
f.write(str(packageIndex) + '\n')
|
||||
|
||||
print('[' + strDateTime() + '] get:' + pkg)
|
||||
print_ts('get:' + pkg)
|
||||
connection.send(pkg.encode('utf-8', 'ignore'))
|
||||
connection.close()
|
||||
elif cmd.startswith('write\nftp://') or cmd.startswith('write\nhttp://'):
|
||||
|
|
@ -1098,7 +1100,7 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
|
|||
try:
|
||||
text_received = bytes_received.decode('utf-8', 'ignore')
|
||||
except UnicodeDecodeError as e:
|
||||
print('Error: Decoding failed (write): ' + str(e))
|
||||
print_ts('Error: Decoding failed (write): ' + str(e))
|
||||
data = ''
|
||||
break
|
||||
t = 0.0
|
||||
|
|
@ -1114,17 +1116,17 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
|
|||
if pos < 10:
|
||||
continue
|
||||
url = data[:pos]
|
||||
print('[' + strDateTime() + '] write:' + url)
|
||||
print_ts('write:' + url)
|
||||
|
||||
# save data
|
||||
res = re.match(r'ftp://.*pool/main/[^/]+/([^/]+)/[^/]*tar.(gz|bz2|xz)', url)
|
||||
if res is None:
|
||||
res = re.match(r'https?://cppcheck\.sf\.net/([a-z]+).tgz', url)
|
||||
if res is None:
|
||||
print('results not written. res is None.')
|
||||
print_ts('results not written. res is None.')
|
||||
continue
|
||||
if url not in packages:
|
||||
print('results not written. url is not in packages.')
|
||||
print_ts('results not written. url is not in packages.')
|
||||
continue
|
||||
# Verify that head was compared to correct OLD_VERSION
|
||||
versions_found = False
|
||||
|
|
@ -1133,17 +1135,17 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
|
|||
if line.startswith('cppcheck: '):
|
||||
versions_found = True
|
||||
if OLD_VERSION not in line.split():
|
||||
print('Compared to wrong old version. Should be ' + OLD_VERSION + '. Versions compared: ' +
|
||||
print_ts('Compared to wrong old version. Should be ' + OLD_VERSION + '. Versions compared: ' +
|
||||
line)
|
||||
print('Ignoring data.')
|
||||
print_ts('Ignoring data.')
|
||||
old_version_wrong = True
|
||||
break
|
||||
if not versions_found:
|
||||
print('Cppcheck versions missing in result data. Ignoring data.')
|
||||
print_ts('Cppcheck versions missing in result data. Ignoring data.')
|
||||
continue
|
||||
if old_version_wrong:
|
||||
continue
|
||||
print('results added for package ' + res.group(1))
|
||||
print_ts('results added for package ' + res.group(1))
|
||||
filename = os.path.join(resultPath, res.group(1))
|
||||
with open(filename, 'wt') as f:
|
||||
f.write(strDateTime() + '\n' + data)
|
||||
|
|
@ -1167,7 +1169,7 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
|
|||
try:
|
||||
text_received = bytes_received.decode('utf-8', 'ignore')
|
||||
except UnicodeDecodeError as e:
|
||||
print('Error: Decoding failed (write_info): ' + str(e))
|
||||
print_ts('Error: Decoding failed (write_info): ' + str(e))
|
||||
data = ''
|
||||
break
|
||||
t = 0.0
|
||||
|
|
@ -1183,19 +1185,19 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
|
|||
if pos < 10:
|
||||
continue
|
||||
url = data[:pos]
|
||||
print('[' + strDateTime() + '] write_info:' + url)
|
||||
print_ts('write_info:' + url)
|
||||
|
||||
# save data
|
||||
res = re.match(r'ftp://.*pool/main/[^/]+/([^/]+)/[^/]*tar.(gz|bz2|xz)', url)
|
||||
if res is None:
|
||||
res = re.match(r'https://cppcheck\.sf\.net/([a-z]+).tgz', url)
|
||||
if res is None:
|
||||
print('info output not written. res is None.')
|
||||
print_ts('info output not written. res is None.')
|
||||
continue
|
||||
if url not in packages:
|
||||
print('info output not written. url is not in packages.')
|
||||
print_ts('info output not written. url is not in packages.')
|
||||
continue
|
||||
print('adding info output for package ' + res.group(1))
|
||||
print_ts('adding info output for package ' + res.group(1))
|
||||
info_path = resultPath + '/' + 'info_output'
|
||||
if not os.path.exists(info_path):
|
||||
os.mkdir(info_path)
|
||||
|
|
@ -1206,7 +1208,7 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
|
|||
packages_count = str(len(packages))
|
||||
connection.send(packages_count.encode('utf-8', 'ignore'))
|
||||
connection.close()
|
||||
print('[' + strDateTime() + '] getPackagesCount: ' + packages_count)
|
||||
print_ts('getPackagesCount: ' + packages_count)
|
||||
continue
|
||||
elif cmd.startswith('getPackageIdx'):
|
||||
request_idx = abs(int(cmd[len('getPackageIdx:'):]))
|
||||
|
|
@ -1214,20 +1216,20 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa
|
|||
pkg = packages[request_idx]
|
||||
connection.send(pkg.encode('utf-8', 'ignore'))
|
||||
connection.close()
|
||||
print('[' + strDateTime() + '] getPackageIdx: ' + pkg)
|
||||
print_ts('getPackageIdx: ' + pkg)
|
||||
else:
|
||||
connection.close()
|
||||
print('[' + strDateTime() + '] getPackageIdx: index is out of range')
|
||||
print_ts('getPackageIdx: index is out of range')
|
||||
continue
|
||||
else:
|
||||
if cmd.find('\n') < 0:
|
||||
print('[' + strDateTime() + '] invalid command: "' + firstLine + '"')
|
||||
print_ts('invalid command: "' + firstLine + '"')
|
||||
else:
|
||||
lines = cmd.split('\n')
|
||||
s = '\\n'.join(lines[:2])
|
||||
if len(lines) > 2:
|
||||
s += '...'
|
||||
print('[' + strDateTime() + '] invalid command: "' + s + '"')
|
||||
print_ts('invalid command: "' + s + '"')
|
||||
connection.close()
|
||||
|
||||
|
||||
|
|
@ -1236,26 +1238,27 @@ if __name__ == "__main__":
|
|||
if not os.path.isdir(workPath):
|
||||
workPath = os.path.expanduser('~/daca@home')
|
||||
os.chdir(workPath)
|
||||
print('work path: ' + workPath)
|
||||
print_ts('work path: ' + workPath)
|
||||
resultPath = workPath + '/donated-results'
|
||||
if not os.path.isdir(resultPath):
|
||||
print_ts("fatal: result path '{}' is missing".format(resultPath))
|
||||
sys.exit(1)
|
||||
|
||||
f = open('packages.txt', 'rt')
|
||||
packages = [val.strip() for val in f.readlines()]
|
||||
f.close()
|
||||
with open('packages.txt', 'rt') as f:
|
||||
packages = [val.strip() for val in f.readlines()]
|
||||
|
||||
print('packages: ' + str(len(packages)))
|
||||
print_ts('packages: ' + str(len(packages)))
|
||||
|
||||
if len(packages) == 0:
|
||||
print('fatal: there are no packages')
|
||||
print_ts('fatal: there are no packages')
|
||||
sys.exit(1)
|
||||
|
||||
packageIndex = 0
|
||||
if os.path.isfile('package-index.txt'):
|
||||
f = open('package-index.txt', 'rt')
|
||||
packageIndex = int(f.read())
|
||||
with open('package-index.txt', 'rt') as f:
|
||||
packageIndex = int(f.read())
|
||||
if packageIndex < 0 or packageIndex >= len(packages):
|
||||
packageIndex = 0
|
||||
f.close()
|
||||
|
||||
server_address_port = 8000
|
||||
if '--test' in sys.argv[1:]:
|
||||
|
|
@ -1264,4 +1267,4 @@ if __name__ == "__main__":
|
|||
try:
|
||||
server(server_address_port, packages, packageIndex, resultPath)
|
||||
except socket.timeout:
|
||||
print('Timeout!')
|
||||
print_ts('Timeout!')
|
||||
|
|
|
|||
Loading…
Reference in New Issue