walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

donate-cpu-server.py: added support for basic `information` reports / some cleanups (#5014) 

* donate-cpu-server.py: bumped version

* donate-cpu-server.py: fixed some PyCharm inspection warnings

* donate-cpu-server.py: use `os.path.join()`

* donate-cpu-server.py: added support for basic `information` reports
This commit is contained in:
Oliver Stöneberg 2023-04-28 15:33:25 +02:00 committed by GitHub
parent 1cd1cbabe9
commit 9239549598
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 74 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
tools/donate-cpu-server.py
View File

@ -26,10 +26,13 @@ from urllib.parse import urlparse
# Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/ # Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/
# Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic # Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic
# changes) # changes)
SERVER_VERSION = "1.3.39" SERVER_VERSION = "1.3.40"
OLD_VERSION = '2.10' OLD_VERSION = '2.10'
HEAD_MARKER = 'head results:'
INFO_MARKER = 'info messages:'
# Set up logging # Set up logging
logger = logging.getLogger() logger = logging.getLogger()
@ -82,6 +85,7 @@ def overviewReport() -> str:
html += '<a href="stale.html">Stale report</a><br>\n' html += '<a href="stale.html">Stale report</a><br>\n'
html += '<a href="diff.html">Diff report</a><br>\n' html += '<a href="diff.html">Diff report</a><br>\n'
html += '<a href="head.html">HEAD report</a><br>\n' html += '<a href="head.html">HEAD report</a><br>\n'
html += '<a href="headinfo.html">HEAD (info) report</a><br>\n'
html += '<a href="latest.html">Latest results</a><br>\n' html += '<a href="latest.html">Latest results</a><br>\n'
html += '<a href="time_lt.html">Time report (improved)</a><br>\n' html += '<a href="time_lt.html">Time report (improved)</a><br>\n'
html += '<a href="time_gt.html">Time report (regressed)</a> - <a href="time_gt.html?pkgs=1">packages.txt</a><br>\n' html += '<a href="time_gt.html">Time report (regressed)</a> - <a href="time_gt.html?pkgs=1">packages.txt</a><br>\n'
@ -537,7 +541,7 @@ def diffMessageIdTodayReport(resultPath: str, messageId: str) -> str:
return text return text
def headReportFromDict(out: dict, today: str) -> str: def summaryReportFromDict(out: dict, prefix: str, today: str) -> str:
html = '<pre>\n' html = '<pre>\n'
html += '<b>MessageID Count</b>\n' html += '<b>MessageID Count</b>\n'
sumTotal = 0 sumTotal = 0
@ -550,7 +554,7 @@ def headReportFromDict(out: dict, today: str) -> str:
while len(line) < 48 - len(c): while len(line) < 48 - len(c):
line += ' ' line += ' '
line += c + ' ' line += c + ' '
line = '<a href="head' + today + '-' + messageId + '">' + messageId + '</a>' + line[line.find(' '):] line = '<a href="' + prefix + today + '-' + messageId + '">' + messageId + '</a>' + line[line.find(' '):]
html += line + '\n' html += line + '\n'
# Sum # Sum
@ -565,7 +569,7 @@ def headReportFromDict(out: dict, today: str) -> str:
return html return html
def headReport(resultsPath: str) -> str: def summaryReport(resultsPath: str, name: str, prefix: str, marker: str) -> str:
out = {} out = {}
outToday = {} outToday = {}
today = strDateTime()[:10] today = strDateTime()[:10]
@ -575,7 +579,7 @@ def headReport(resultsPath: str) -> str:
continue continue
uploadedToday = False uploadedToday = False
firstLine = True firstLine = True
headResults = False inResults = False
for line in open(filename, 'rt'): for line in open(filename, 'rt'):
if firstLine: if firstLine:
if line.startswith(today): if line.startswith(today):
@ -590,13 +594,13 @@ def headReport(resultsPath: str) -> str:
else: else:
# Current package, parse on # Current package, parse on
continue continue
if line.startswith('head results:'): if line.startswith(marker):
headResults = True inResults = True
continue continue
if line.startswith('diff:'): if line.startswith('diff:'):
if headResults: if inResults:
break break
if not headResults: if not inResults:
continue continue
if not line.endswith(']'): if not line.endswith(']'):
continue continue
@ -623,14 +627,22 @@ def headReport(resultsPath: str) -> str:
html += '<html><head><title>HEAD report</title></head><body>\n' html += '<html><head><title>HEAD report</title></head><body>\n'
html += '<h1>HEAD report</h1>\n' html += '<h1>HEAD report</h1>\n'
html += '<h2>Uploaded today</h2>' html += '<h2>Uploaded today</h2>'
html += headReportFromDict(outToday, 'today') html += summaryReportFromDict(outToday, prefix, 'today')
html += '<h2>All</h2>' html += '<h2>All</h2>'
html += headReportFromDict(out, '') html += summaryReportFromDict(out, prefix, '')
return html return html
def headMessageIdReport(resultPath: str, messageId: str, query_params: dict) -> str: def headReport(resultsPath: str) -> str:
return summaryReport(resultsPath, 'HEAD', 'head', HEAD_MARKER)
def infoReport(resultsPath: str) -> str:
return summaryReport(resultsPath, 'HEAD (info)', 'headinfo', INFO_MARKER)
def messageIdReport(resultPath: str, marker: str, messageId: str, query_params: dict) -> str:
pkgs = '' if query_params.get('pkgs') == '1' else None pkgs = '' if query_params.get('pkgs') == '1' else None
text = messageId + '\n' text = messageId + '\n'
e = '[' + messageId + ']\n' e = '[' + messageId + ']\n'
@ -638,15 +650,15 @@ def headMessageIdReport(resultPath: str, messageId: str, query_params: dict) ->
if not os.path.isfile(filename) or filename.endswith('.diff'): if not os.path.isfile(filename) or filename.endswith('.diff'):
continue continue
url = None url = None
headResults = False inResults = False
for line in open(filename, 'rt'): for line in open(filename, 'rt'):
if line.startswith('ftp://'): if line.startswith('ftp://'):
url = line url = line
elif line.startswith('head results:'): elif line.startswith(marker):
headResults = True inResults = True
elif not headResults: elif not inResults:
continue continue
elif headResults and line.startswith('diff:'): elif inResults and line.startswith('diff:'):
break break
elif line.endswith(e): elif line.endswith(e):
if url: if url:
@ -660,7 +672,15 @@ def headMessageIdReport(resultPath: str, messageId: str, query_params: dict) ->
return text return text
def headMessageIdTodayReport(resultPath: str, messageId: str) -> str: def headMessageIdReport(resultPath: str, messageId: str, query_params: dict) -> str:
return messageIdReport(resultPath, HEAD_MARKER, messageId, query_params)
def infoMessageIdReport(resultPath: str, messageId: str, query_params: dict) -> str:
return messageIdReport(resultPath, INFO_MARKER, messageId, query_params)
def messageIdTodayReport(resultPath: str, messageId: str, marker: str) -> str:
text = messageId + '\n' text = messageId + '\n'
e = '[' + messageId + ']\n' e = '[' + messageId + ']\n'
today = strDateTime()[:10] today = strDateTime()[:10]
@ -668,7 +688,7 @@ def headMessageIdTodayReport(resultPath: str, messageId: str) -> str:
if not os.path.isfile(filename) or filename.endswith('.diff'): if not os.path.isfile(filename) or filename.endswith('.diff'):
continue continue
url = None url = None
headResults = False inResults = False
firstLine = True firstLine = True
for line in open(filename, 'rt'): for line in open(filename, 'rt'):
if firstLine: if firstLine:
@ -677,11 +697,11 @@ def headMessageIdTodayReport(resultPath: str, messageId: str) -> str:
break break
if line.startswith('ftp://'): if line.startswith('ftp://'):
url = line url = line
elif line.startswith('head results:'): elif line.startswith(marker):
headResults = True inResults = True
elif not headResults: elif not inResults:
continue continue
elif headResults and line.startswith('diff:'): elif inResults and line.startswith('diff:'):
break break
elif line.endswith(e): elif line.endswith(e):
if url: if url:
@ -691,7 +711,16 @@ def headMessageIdTodayReport(resultPath: str, messageId: str) -> str:
return text return text
def timeReport(resultPath: str, show_gt: bool, query_params: dict) -> str: def headMessageIdTodayReport(resultPath: str, messageId: str) -> str:
return messageIdTodayReport(resultPath, messageId, HEAD_MARKER)
def infoMessageIdTodayReport(resultPath: str, messageId: str) -> str:
return messageIdTodayReport(resultPath, messageId, INFO_MARKER)
# TODO: needs to dinicate that it returns 'tuple[str, str]' but that isn't supported until Python 3.9
def timeReport(resultPath: str, show_gt: bool, query_params: dict):
# no need for package report support in "improved" report # no need for package report support in "improved" report
pkgs = '' if show_gt and query_params and query_params.get('pkgs') == '1' else None pkgs = '' if show_gt and query_params and query_params.get('pkgs') == '1' else None
factor = float(query_params.get('factor')) if query_params and 'factor' in query_params else 2.0 factor = float(query_params.get('factor')) if query_params and 'factor' in query_params else 2.0
@ -991,9 +1020,11 @@ class HttpClientThread(Thread):
self.connection = connection self.connection = connection
self.cmd = cmd[:cmd.find('\r\n')] self.cmd = cmd[:cmd.find('\r\n')]
self.resultPath = resultPath self.resultPath = resultPath
self.infoPath = os.path.join(self.resultPath, 'info_output')
self.latestResults = latestResults self.latestResults = latestResults
# TODO: use a proper parser # TODO: use a proper parser
@staticmethod
def parse_req(cmd): def parse_req(cmd):
req_parts = cmd.split(' ') req_parts = cmd.split(' ')
if len(req_parts) != 3 or req_parts[0] != 'GET' or not req_parts[2].startswith('HTTP'): if len(req_parts) != 3 or req_parts[0] != 'GET' or not req_parts[2].startswith('HTTP'):
@ -1005,7 +1036,7 @@ class HttpClientThread(Thread):
try: try:
cmd = self.cmd cmd = self.cmd
print_ts(cmd) print_ts(cmd)
url, queryParams = HttpClientThread.parse_req(cmd) url, queryParams = self.parse_req(cmd)
if url is None: if url is None:
print_ts('invalid request: {}'.format(cmd)) print_ts('invalid request: {}'.format(cmd))
self.connection.close() self.connection.close()
@ -1039,14 +1070,25 @@ class HttpClientThread(Thread):
elif url == '/head.html': elif url == '/head.html':
html = headReport(self.resultPath) html = headReport(self.resultPath)
httpGetResponse(self.connection, html, 'text/html') httpGetResponse(self.connection, html, 'text/html')
elif url == '/headinfo.html':
html = infoReport(self.infoPath)
httpGetResponse(self.connection, html, 'text/html')
elif url.startswith('/headtoday-'): elif url.startswith('/headtoday-'):
messageId = url[len('/headtoday-'):] messageId = url[len('/headtoday-'):]
text = headMessageIdTodayReport(self.resultPath, messageId) text = headMessageIdTodayReport(self.resultPath, messageId)
httpGetResponse(self.connection, text, 'text/plain') httpGetResponse(self.connection, text, 'text/plain')
elif url.startswith('/headinfotoday-'):
messageId = url[len('/headinfotoday-'):]
text = infoMessageIdTodayReport(self.infoPath, messageId)
httpGetResponse(self.connection, text, 'text/plain')
elif url.startswith('/head-'): elif url.startswith('/head-'):
messageId = url[len('/head-'):] messageId = url[len('/head-'):]
text = headMessageIdReport(self.resultPath, messageId, queryParams) text = headMessageIdReport(self.resultPath, messageId, queryParams)
httpGetResponse(self.connection, text, 'text/plain') httpGetResponse(self.connection, text, 'text/plain')
elif url.startswith('/headinfo-'):
messageId = url[len('/headinfo-'):]
text = infoMessageIdReport(self.infoPath, messageId, queryParams)
httpGetResponse(self.connection, text, 'text/plain')
elif url == '/time_lt.html': elif url == '/time_lt.html':
text, mime = timeReport(self.resultPath, False, queryParams) text, mime = timeReport(self.resultPath, False, queryParams)
httpGetResponse(self.connection, text, mime) httpGetResponse(self.connection, text, mime)
@ -1057,20 +1099,20 @@ class HttpClientThread(Thread):
text = timeReportSlow(self.resultPath) text = timeReportSlow(self.resultPath)
httpGetResponse(self.connection, text, 'text/html') httpGetResponse(self.connection, text, 'text/html')
elif url == '/check_library_function_report.html': elif url == '/check_library_function_report.html':
text = check_library_report(self.resultPath + '/' + 'info_output', message_id='checkLibraryFunction') text = check_library_report(self.infoPath, message_id='checkLibraryFunction')
httpGetResponse(self.connection, text, 'text/html') httpGetResponse(self.connection, text, 'text/html')
elif url == '/check_library_noreturn_report.html': elif url == '/check_library_noreturn_report.html':
text = check_library_report(self.resultPath + '/' + 'info_output', message_id='checkLibraryNoReturn') text = check_library_report(self.infoPath, message_id='checkLibraryNoReturn')
httpGetResponse(self.connection, text, 'text/html') httpGetResponse(self.connection, text, 'text/html')
elif url == '/check_library_use_ignore_report.html': elif url == '/check_library_use_ignore_report.html':
text = check_library_report(self.resultPath + '/' + 'info_output', message_id='checkLibraryUseIgnore') text = check_library_report(self.infoPath, message_id='checkLibraryUseIgnore')
httpGetResponse(self.connection, text, 'text/html') httpGetResponse(self.connection, text, 'text/html')
elif url == '/check_library_check_type_report.html': elif url == '/check_library_check_type_report.html':
text = check_library_report(self.resultPath + '/' + 'info_output', message_id='checkLibraryCheckType') text = check_library_report(self.infoPath, message_id='checkLibraryCheckType')
httpGetResponse(self.connection, text, 'text/html') httpGetResponse(self.connection, text, 'text/html')
elif url.startswith('/check_library-'): elif url.startswith('/check_library-'):
function_name = url[len('/check_library-'):] function_name = url[len('/check_library-'):]
text = check_library_function_name(self.resultPath + '/' + 'info_output', function_name) text = check_library_function_name(self.infoPath, function_name)
httpGetResponse(self.connection, text, 'text/plain') httpGetResponse(self.connection, text, 'text/plain')
else: else:
filename = resultPath + url filename = resultPath + url
@ -1092,8 +1134,8 @@ class HttpClientThread(Thread):
def read_data(connection, cmd, pos_nl, max_data_size, check_done, cmd_name, timeout=10): def read_data(connection, cmd, pos_nl, max_data_size, check_done, cmd_name, timeout=10):
data = cmd[pos_nl+1:] data = cmd[pos_nl+1:]
try:
t = 0.0 t = 0.0
try:
while (len(data) < max_data_size) and (not check_done or not data.endswith('\nDONE')) and (timeout > 0 and t < timeout): while (len(data) < max_data_size) and (not check_done or not data.endswith('\nDONE')) and (timeout > 0 and t < timeout):
bytes_received = connection.recv(1024) bytes_received = connection.recv(1024)
if bytes_received: if bytes_received:
@ -1115,7 +1157,7 @@ def read_data(connection, cmd, pos_nl, max_data_size, check_done, cmd_name, time
print_ts('Socket error occurred ({}): {}'.format(cmd_name, e)) print_ts('Socket error occurred ({}): {}'.format(cmd_name, e))
data = None data = None
if (timeout > 0 and t >= timeout): if timeout > 0 and t >= timeout:
print_ts('Timeout occurred ({}).'.format(cmd_name)) print_ts('Timeout occurred ({}).'.format(cmd_name))
data = None data = None