From 9239549598fea77c0b4183ddd553e7c3ed80d70b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20St=C3=B6neberg?= Date: Fri, 28 Apr 2023 15:33:25 +0200 Subject: [PATCH] donate-cpu-server.py: added support for basic `information` reports / some cleanups (#5014) * donate-cpu-server.py: bumped version * donate-cpu-server.py: fixed some PyCharm inspection warnings * donate-cpu-server.py: use `os.path.join()` * donate-cpu-server.py: added support for basic `information` reports --- tools/donate-cpu-server.py | 106 ++++++++++++++++++++++++++----------- 1 file changed, 74 insertions(+), 32 deletions(-) diff --git a/tools/donate-cpu-server.py b/tools/donate-cpu-server.py index d525575c7..a51845549 100755 --- a/tools/donate-cpu-server.py +++ b/tools/donate-cpu-server.py @@ -26,10 +26,13 @@ from urllib.parse import urlparse # Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/ # Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic # changes) -SERVER_VERSION = "1.3.39" +SERVER_VERSION = "1.3.40" OLD_VERSION = '2.10' +HEAD_MARKER = 'head results:' +INFO_MARKER = 'info messages:' + # Set up logging logger = logging.getLogger() @@ -82,6 +85,7 @@ def overviewReport() -> str: html += 'Stale report
\n' html += 'Diff report
\n' html += 'HEAD report
\n' + html += 'HEAD (info) report
\n' html += 'Latest results
\n' html += 'Time report (improved)
\n' html += 'Time report (regressed) - packages.txt
\n' @@ -537,7 +541,7 @@ def diffMessageIdTodayReport(resultPath: str, messageId: str) -> str: return text -def headReportFromDict(out: dict, today: str) -> str: +def summaryReportFromDict(out: dict, prefix: str, today: str) -> str: html = '
\n'
     html += 'MessageID                                  Count\n'
     sumTotal = 0
@@ -550,7 +554,7 @@ def headReportFromDict(out: dict, today: str) -> str:
             while len(line) < 48 - len(c):
                 line += ' '
             line += c + ' '
-        line = '' + messageId + '' + line[line.find(' '):]
+        line = '' + messageId + '' + line[line.find(' '):]
         html += line + '\n'
 
     # Sum
@@ -565,7 +569,7 @@ def headReportFromDict(out: dict, today: str) -> str:
     return html
 
 
-def headReport(resultsPath: str) -> str:
+def summaryReport(resultsPath: str, name: str, prefix: str, marker: str) -> str:
     out = {}
     outToday = {}
     today = strDateTime()[:10]
@@ -575,7 +579,7 @@ def headReport(resultsPath: str) -> str:
             continue
         uploadedToday = False
         firstLine = True
-        headResults = False
+        inResults = False
         for line in open(filename, 'rt'):
             if firstLine:
                 if line.startswith(today):
@@ -590,13 +594,13 @@ def headReport(resultsPath: str) -> str:
                 else:
                     # Current package, parse on
                     continue
-            if line.startswith('head results:'):
-                headResults = True
+            if line.startswith(marker):
+                inResults = True
                 continue
             if line.startswith('diff:'):
-                if headResults:
+                if inResults:
                     break
-            if not headResults:
+            if not inResults:
                 continue
             if not line.endswith(']'):
                 continue
@@ -623,14 +627,22 @@ def headReport(resultsPath: str) -> str:
     html += 'HEAD report\n'
     html += '
HEAD report
\n'
     html += '
Uploaded today
'
-    html += headReportFromDict(outToday, 'today')
+    html += summaryReportFromDict(outToday, prefix, 'today')
     html += '
All
'
-    html += headReportFromDict(out, '')
+    html += summaryReportFromDict(out, prefix, '')
 
     return html
 
 
-def headMessageIdReport(resultPath: str, messageId: str, query_params: dict) -> str:
+def headReport(resultsPath: str) -> str:
+    return summaryReport(resultsPath, 'HEAD', 'head', HEAD_MARKER)
+
+
+def infoReport(resultsPath: str) -> str:
+    return summaryReport(resultsPath, 'HEAD (info)', 'headinfo', INFO_MARKER)
+
+
+def messageIdReport(resultPath: str, marker: str, messageId: str, query_params: dict) -> str:
     pkgs = '' if query_params.get('pkgs') == '1' else None
     text = messageId + '\n'
     e = '[' + messageId + ']\n'
@@ -638,15 +650,15 @@ def headMessageIdReport(resultPath: str, messageId: str, query_params: dict) ->
         if not os.path.isfile(filename) or filename.endswith('.diff'):
             continue
         url = None
-        headResults = False
+        inResults = False
         for line in open(filename, 'rt'):
             if line.startswith('ftp://'):
                 url = line
-            elif line.startswith('head results:'):
-                headResults = True
-            elif not headResults:
+            elif line.startswith(marker):
+                inResults = True
+            elif not inResults:
                 continue
-            elif headResults and line.startswith('diff:'):
+            elif inResults and line.startswith('diff:'):
                 break
             elif line.endswith(e):
                 if url:
@@ -660,7 +672,15 @@ def headMessageIdReport(resultPath: str, messageId: str, query_params: dict) ->
     return text
 
 
-def headMessageIdTodayReport(resultPath: str, messageId: str) -> str:
+def headMessageIdReport(resultPath: str, messageId: str, query_params: dict) -> str:
+    return messageIdReport(resultPath, HEAD_MARKER, messageId, query_params)
+
+
+def infoMessageIdReport(resultPath: str, messageId: str, query_params: dict) -> str:
+    return messageIdReport(resultPath, INFO_MARKER, messageId, query_params)
+
+
+def messageIdTodayReport(resultPath: str, messageId: str, marker: str) -> str:
     text = messageId + '\n'
     e = '[' + messageId + ']\n'
     today = strDateTime()[:10]
@@ -668,7 +688,7 @@ def headMessageIdTodayReport(resultPath: str, messageId: str) -> str:
         if not os.path.isfile(filename) or filename.endswith('.diff'):
             continue
         url = None
-        headResults = False
+        inResults = False
         firstLine = True
         for line in open(filename, 'rt'):
             if firstLine:
@@ -677,11 +697,11 @@ def headMessageIdTodayReport(resultPath: str, messageId: str) -> str:
                     break
             if line.startswith('ftp://'):
                 url = line
-            elif line.startswith('head results:'):
-                headResults = True
-            elif not headResults:
+            elif line.startswith(marker):
+                inResults = True
+            elif not inResults:
                 continue
-            elif headResults and line.startswith('diff:'):
+            elif inResults and line.startswith('diff:'):
                 break
             elif line.endswith(e):
                 if url:
@@ -691,7 +711,16 @@ def headMessageIdTodayReport(resultPath: str, messageId: str) -> str:
     return text
 
 
-def timeReport(resultPath: str, show_gt: bool, query_params: dict) -> str:
+def headMessageIdTodayReport(resultPath: str, messageId: str) -> str:
+    return messageIdTodayReport(resultPath, messageId, HEAD_MARKER)
+
+
+def infoMessageIdTodayReport(resultPath: str, messageId: str) -> str:
+    return messageIdTodayReport(resultPath, messageId, INFO_MARKER)
+
+
+# TODO: needs to dinicate that it returns 'tuple[str, str]' but that isn't supported until Python 3.9
+def timeReport(resultPath: str, show_gt: bool, query_params: dict):
     # no need for package report support in "improved" report
     pkgs = '' if show_gt and query_params and query_params.get('pkgs') == '1' else None
     factor = float(query_params.get('factor')) if query_params and 'factor' in query_params else 2.0
@@ -991,9 +1020,11 @@ class HttpClientThread(Thread):
         self.connection = connection
         self.cmd = cmd[:cmd.find('\r\n')]
         self.resultPath = resultPath
+        self.infoPath = os.path.join(self.resultPath, 'info_output')
         self.latestResults = latestResults
 
     # TODO: use a proper parser
+    @staticmethod
     def parse_req(cmd):
         req_parts = cmd.split(' ')
         if len(req_parts) != 3 or req_parts[0] != 'GET' or not req_parts[2].startswith('HTTP'):
@@ -1005,7 +1036,7 @@ class HttpClientThread(Thread):
         try:
             cmd = self.cmd
             print_ts(cmd)
-            url, queryParams = HttpClientThread.parse_req(cmd)
+            url, queryParams = self.parse_req(cmd)
             if url is None:
                 print_ts('invalid request: {}'.format(cmd))
                 self.connection.close()
@@ -1039,14 +1070,25 @@ class HttpClientThread(Thread):
             elif url == '/head.html':
                 html = headReport(self.resultPath)
                 httpGetResponse(self.connection, html, 'text/html')
+            elif url == '/headinfo.html':
+                html = infoReport(self.infoPath)
+                httpGetResponse(self.connection, html, 'text/html')
             elif url.startswith('/headtoday-'):
                 messageId = url[len('/headtoday-'):]
                 text = headMessageIdTodayReport(self.resultPath, messageId)
                 httpGetResponse(self.connection, text, 'text/plain')
+            elif url.startswith('/headinfotoday-'):
+                messageId = url[len('/headinfotoday-'):]
+                text = infoMessageIdTodayReport(self.infoPath, messageId)
+                httpGetResponse(self.connection, text, 'text/plain')
             elif url.startswith('/head-'):
                 messageId = url[len('/head-'):]
                 text = headMessageIdReport(self.resultPath, messageId, queryParams)
                 httpGetResponse(self.connection, text, 'text/plain')
+            elif url.startswith('/headinfo-'):
+                messageId = url[len('/headinfo-'):]
+                text = infoMessageIdReport(self.infoPath, messageId, queryParams)
+                httpGetResponse(self.connection, text, 'text/plain')
             elif url == '/time_lt.html':
                 text, mime = timeReport(self.resultPath, False, queryParams)
                 httpGetResponse(self.connection, text, mime)
@@ -1057,20 +1099,20 @@ class HttpClientThread(Thread):
                 text = timeReportSlow(self.resultPath)
                 httpGetResponse(self.connection, text, 'text/html')
             elif url == '/check_library_function_report.html':
-                text = check_library_report(self.resultPath + '/' + 'info_output', message_id='checkLibraryFunction')
+                text = check_library_report(self.infoPath, message_id='checkLibraryFunction')
                 httpGetResponse(self.connection, text, 'text/html')
             elif url == '/check_library_noreturn_report.html':
-                text = check_library_report(self.resultPath + '/' + 'info_output', message_id='checkLibraryNoReturn')
+                text = check_library_report(self.infoPath, message_id='checkLibraryNoReturn')
                 httpGetResponse(self.connection, text, 'text/html')
             elif url == '/check_library_use_ignore_report.html':
-                text = check_library_report(self.resultPath + '/' + 'info_output', message_id='checkLibraryUseIgnore')
+                text = check_library_report(self.infoPath, message_id='checkLibraryUseIgnore')
                 httpGetResponse(self.connection, text, 'text/html')
             elif url == '/check_library_check_type_report.html':
-                text = check_library_report(self.resultPath + '/' + 'info_output', message_id='checkLibraryCheckType')
+                text = check_library_report(self.infoPath, message_id='checkLibraryCheckType')
                 httpGetResponse(self.connection, text, 'text/html')
             elif url.startswith('/check_library-'):
                 function_name = url[len('/check_library-'):]
-                text = check_library_function_name(self.resultPath + '/' + 'info_output', function_name)
+                text = check_library_function_name(self.infoPath, function_name)
                 httpGetResponse(self.connection, text, 'text/plain')
             else:
                 filename = resultPath + url
@@ -1092,8 +1134,8 @@ class HttpClientThread(Thread):
 
 def read_data(connection, cmd, pos_nl, max_data_size, check_done, cmd_name, timeout=10):
     data = cmd[pos_nl+1:]
+    t = 0.0
     try:
-        t = 0.0
         while (len(data) < max_data_size) and (not check_done or not data.endswith('\nDONE')) and (timeout > 0 and t < timeout):
             bytes_received = connection.recv(1024)
             if bytes_received:
@@ -1115,7 +1157,7 @@ def read_data(connection, cmd, pos_nl, max_data_size, check_done, cmd_name, time
         print_ts('Socket error occurred ({}): {}'.format(cmd_name, e))
         data = None
 
-    if (timeout > 0 and t >= timeout):
+    if timeout > 0 and t >= timeout:
         print_ts('Timeout occurred ({}).'.format(cmd_name))
         data = None