diff --git a/tools/donate-cpu-server.py b/tools/donate-cpu-server.py index 600f0eef9..47e38e335 100755 --- a/tools/donate-cpu-server.py +++ b/tools/donate-cpu-server.py @@ -25,7 +25,7 @@ import html as html_lib # Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/ # Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic # changes) -SERVER_VERSION = "1.3.28" +SERVER_VERSION = "1.3.29" OLD_VERSION = '2.9' @@ -46,6 +46,10 @@ handler_file.setLevel(logging.ERROR) logger.addHandler(handler_file) +def print_ts(msg): + print('[' + strDateTime() + '] ' + msg) + + # Set up an exception hook for all uncaught exceptions so they can be logged def handle_uncaught_exception(exc_type, exc_value, exc_traceback): if issubclass(exc_type, KeyboardInterrupt): @@ -59,7 +63,7 @@ sys.excepthook = handle_uncaught_exception def strDateTime() -> str: - return datetime.datetime.now().strftime('%Y-%m-%d %H:%M') + return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') def dateTimeFromStr(datestr: str) -> datetime.datetime: @@ -148,13 +152,13 @@ def latestReport(latestResults: list) -> str: package = filename[filename.rfind('/')+1:] current_year = datetime.date.today().year - datestr = '' + datestr = None count = ['0', '0'] lost = 0 added = 0 for line in open(filename, 'rt'): line = line.strip() - if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'): + if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'): datestr = line #elif line.startswith('cppcheck:'): # cppcheck = line[9:] @@ -185,8 +189,8 @@ def crashReport(results_path: str) -> str: for filename in sorted(glob.glob(os.path.expanduser(results_path + '/*'))): if not os.path.isfile(filename) or filename.endswith('.diff'): continue - datestr = '' with open(filename, 'rt') as file_: + datestr = None for line in file_: line = line.strip() if line.startswith('cppcheck: '): @@ -196,21 +200,21 @@ def crashReport(results_path: str) -> str: else: # Current package, parse on continue - if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'): + if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'): datestr = line elif line.startswith('count:'): if line.find('Crash') < 0: break package = filename[filename.rfind('/')+1:] counts = line.split(' ') - c2 = '' + c_version = '' if counts[2] == 'Crash!': - c2 = 'Crash' - c1 = '' + c_version = 'Crash' + c_head = '' if counts[1] == 'Crash!': - c1 = 'Crash' - html += fmt(package, datestr, c2, c1) + '\n' - if c1 != 'Crash': + c_head = 'Crash' + html += fmt(package, datestr, c_version, c_head) + '\n' + if c_head != 'Crash': break elif line.find(' received signal ') != -1: crash_line = next(file_, '').strip() @@ -243,7 +247,7 @@ def crashReport(results_path: str) -> str: stack_trace.append(m.group('number') + ' ' + m.group('function') + '(...) at ' + m.group('location')) continue - print('{} - unmatched stack frame - {}'.format(package, l)) + print_ts('{} - unmatched stack frame - {}'.format(package, l)) break key = hash(' '.join(stack_trace)) @@ -279,8 +283,8 @@ def timeoutReport(results_path: str) -> str: for filename in sorted(glob.glob(os.path.expanduser(results_path + '/*'))): if not os.path.isfile(filename) or filename.endswith('.diff'): continue - datestr = '' with open(filename, 'rt') as file_: + datestr = None for line in file_: line = line.strip() if line.startswith('cppcheck: '): @@ -290,7 +294,7 @@ def timeoutReport(results_path: str) -> str: else: # Current package, parse on continue - if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'): + if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'): datestr = line elif line.startswith('count:'): if line.find('TO!') < 0: @@ -680,7 +684,7 @@ def timeReport(resultPath: str, show_gt: bool) -> str: for filename in glob.glob(resultPath + '/*'): if not os.path.isfile(filename) or filename.endswith('.diff'): continue - datestr = '' + datestr = None for line in open(filename, 'rt'): line = line.strip() if line.startswith('cppcheck: '): @@ -690,7 +694,7 @@ def timeReport(resultPath: str, show_gt: bool) -> str: else: # Current package, parse on continue - if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'): + if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'): datestr = line continue if not line.startswith('elapsed-time:'): @@ -765,7 +769,7 @@ def timeReportSlow(resultPath: str) -> str: for filename in glob.glob(resultPath + '/*'): if not os.path.isfile(filename) or filename.endswith('.diff'): continue - datestr = '' + datestr = None for line in open(filename, 'rt'): line = line.strip() if line.startswith('cppcheck: '): @@ -775,7 +779,7 @@ def timeReportSlow(resultPath: str) -> str: else: # Current package, parse on continue - if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'): + if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'): datestr = line continue elif line.startswith('count:'): @@ -815,7 +819,7 @@ def timeReportSlow(resultPath: str) -> str: def check_library_report(result_path: str, message_id: str) -> str: if message_id not in ('checkLibraryNoReturn', 'checkLibraryFunction', 'checkLibraryUseIgnore', 'checkLibraryCheckType'): error_message = 'Invalid value ' + message_id + ' for message_id parameter.' - print(error_message) + print_ts(error_message) return error_message if message_id == 'checkLibraryCheckType': @@ -941,7 +945,7 @@ class HttpClientThread(Thread): def run(self): try: cmd = self.cmd - print('[' + strDateTime() + '] ' + cmd) + print_ts(cmd) res = re.match(r'GET /([a-zA-Z0-9_\-\.\+%]*) HTTP', cmd) if res is None: self.connection.close() @@ -1012,16 +1016,15 @@ class HttpClientThread(Thread): else: filename = resultPath + '/' + url if not os.path.isfile(filename): - print('HTTP/1.1 404 Not Found') + print_ts('HTTP/1.1 404 Not Found') self.connection.send(b'HTTP/1.1 404 Not Found\r\n\r\n') else: - f = open(filename, 'rt') - data = f.read() - f.close() + with open(filename, 'rt') as f: + data = f.read() httpGetResponse(self.connection, data, 'text/plain') except: tb = "".join(traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) - print(tb) + print_ts(tb) httpGetResponse(self.connection, tb, 'text/plain') finally: time.sleep(1) @@ -1042,12 +1045,12 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa with open('latest.txt', 'rt') as f: latestResults = f.read().strip().split(' ') - print('[' + strDateTime() + '] version ' + SERVER_VERSION) - print('[' + strDateTime() + '] listening on port ' + str(server_address_port)) + print_ts('version ' + SERVER_VERSION) + print_ts('listening on port ' + str(server_address_port)) while True: # wait for a connection - print('[' + strDateTime() + '] waiting for a connection') + print_ts('waiting for a connection') connection, client_address = sock.accept() try: bytes_received = connection.recv(128) @@ -1057,7 +1060,7 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa continue except UnicodeDecodeError as e: connection.close() - print('Error: Decoding failed: ' + str(e)) + print_ts('Error: Decoding failed: ' + str(e)) continue if cmd.find('\n') < 1: continue @@ -1070,7 +1073,7 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa newThread.start() elif cmd == 'GetCppcheckVersions\n': reply = 'head ' + OLD_VERSION - print('[' + strDateTime() + '] GetCppcheckVersions: ' + reply) + print_ts('GetCppcheckVersions: ' + reply) connection.send(reply.encode('utf-8', 'ignore')) connection.close() elif cmd == 'get\n': @@ -1079,11 +1082,10 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa if packageIndex >= len(packages): packageIndex = 0 - f = open('package-index.txt', 'wt') - f.write(str(packageIndex) + '\n') - f.close() + with open('package-index.txt', 'wt') as f: + f.write(str(packageIndex) + '\n') - print('[' + strDateTime() + '] get:' + pkg) + print_ts('get:' + pkg) connection.send(pkg.encode('utf-8', 'ignore')) connection.close() elif cmd.startswith('write\nftp://') or cmd.startswith('write\nhttp://'): @@ -1098,7 +1100,7 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa try: text_received = bytes_received.decode('utf-8', 'ignore') except UnicodeDecodeError as e: - print('Error: Decoding failed (write): ' + str(e)) + print_ts('Error: Decoding failed (write): ' + str(e)) data = '' break t = 0.0 @@ -1114,17 +1116,17 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa if pos < 10: continue url = data[:pos] - print('[' + strDateTime() + '] write:' + url) + print_ts('write:' + url) # save data res = re.match(r'ftp://.*pool/main/[^/]+/([^/]+)/[^/]*tar.(gz|bz2|xz)', url) if res is None: res = re.match(r'https?://cppcheck\.sf\.net/([a-z]+).tgz', url) if res is None: - print('results not written. res is None.') + print_ts('results not written. res is None.') continue if url not in packages: - print('results not written. url is not in packages.') + print_ts('results not written. url is not in packages.') continue # Verify that head was compared to correct OLD_VERSION versions_found = False @@ -1133,17 +1135,17 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa if line.startswith('cppcheck: '): versions_found = True if OLD_VERSION not in line.split(): - print('Compared to wrong old version. Should be ' + OLD_VERSION + '. Versions compared: ' + + print_ts('Compared to wrong old version. Should be ' + OLD_VERSION + '. Versions compared: ' + line) - print('Ignoring data.') + print_ts('Ignoring data.') old_version_wrong = True break if not versions_found: - print('Cppcheck versions missing in result data. Ignoring data.') + print_ts('Cppcheck versions missing in result data. Ignoring data.') continue if old_version_wrong: continue - print('results added for package ' + res.group(1)) + print_ts('results added for package ' + res.group(1)) filename = os.path.join(resultPath, res.group(1)) with open(filename, 'wt') as f: f.write(strDateTime() + '\n' + data) @@ -1167,7 +1169,7 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa try: text_received = bytes_received.decode('utf-8', 'ignore') except UnicodeDecodeError as e: - print('Error: Decoding failed (write_info): ' + str(e)) + print_ts('Error: Decoding failed (write_info): ' + str(e)) data = '' break t = 0.0 @@ -1183,19 +1185,19 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa if pos < 10: continue url = data[:pos] - print('[' + strDateTime() + '] write_info:' + url) + print_ts('write_info:' + url) # save data res = re.match(r'ftp://.*pool/main/[^/]+/([^/]+)/[^/]*tar.(gz|bz2|xz)', url) if res is None: res = re.match(r'https://cppcheck\.sf\.net/([a-z]+).tgz', url) if res is None: - print('info output not written. res is None.') + print_ts('info output not written. res is None.') continue if url not in packages: - print('info output not written. url is not in packages.') + print_ts('info output not written. url is not in packages.') continue - print('adding info output for package ' + res.group(1)) + print_ts('adding info output for package ' + res.group(1)) info_path = resultPath + '/' + 'info_output' if not os.path.exists(info_path): os.mkdir(info_path) @@ -1206,7 +1208,7 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa packages_count = str(len(packages)) connection.send(packages_count.encode('utf-8', 'ignore')) connection.close() - print('[' + strDateTime() + '] getPackagesCount: ' + packages_count) + print_ts('getPackagesCount: ' + packages_count) continue elif cmd.startswith('getPackageIdx'): request_idx = abs(int(cmd[len('getPackageIdx:'):])) @@ -1214,20 +1216,20 @@ def server(server_address_port: int, packages: list, packageIndex: int, resultPa pkg = packages[request_idx] connection.send(pkg.encode('utf-8', 'ignore')) connection.close() - print('[' + strDateTime() + '] getPackageIdx: ' + pkg) + print_ts('getPackageIdx: ' + pkg) else: connection.close() - print('[' + strDateTime() + '] getPackageIdx: index is out of range') + print_ts('getPackageIdx: index is out of range') continue else: if cmd.find('\n') < 0: - print('[' + strDateTime() + '] invalid command: "' + firstLine + '"') + print_ts('invalid command: "' + firstLine + '"') else: lines = cmd.split('\n') s = '\\n'.join(lines[:2]) if len(lines) > 2: s += '...' - print('[' + strDateTime() + '] invalid command: "' + s + '"') + print_ts('invalid command: "' + s + '"') connection.close() @@ -1236,26 +1238,27 @@ if __name__ == "__main__": if not os.path.isdir(workPath): workPath = os.path.expanduser('~/daca@home') os.chdir(workPath) - print('work path: ' + workPath) + print_ts('work path: ' + workPath) resultPath = workPath + '/donated-results' + if not os.path.isdir(resultPath): + print_ts("fatal: result path '{}' is missing".format(resultPath)) + sys.exit(1) - f = open('packages.txt', 'rt') - packages = [val.strip() for val in f.readlines()] - f.close() + with open('packages.txt', 'rt') as f: + packages = [val.strip() for val in f.readlines()] - print('packages: ' + str(len(packages))) + print_ts('packages: ' + str(len(packages))) if len(packages) == 0: - print('fatal: there are no packages') + print_ts('fatal: there are no packages') sys.exit(1) packageIndex = 0 if os.path.isfile('package-index.txt'): - f = open('package-index.txt', 'rt') - packageIndex = int(f.read()) + with open('package-index.txt', 'rt') as f: + packageIndex = int(f.read()) if packageIndex < 0 or packageIndex >= len(packages): packageIndex = 0 - f.close() server_address_port = 8000 if '--test' in sys.argv[1:]: @@ -1264,4 +1267,4 @@ if __name__ == "__main__": try: server(server_address_port, packages, packageIndex, resultPath) except socket.timeout: - print('Timeout!') + print_ts('Timeout!')