various daca@home improvements (#3139)

tools/donate-cpu-server.py

@@ -25,7 +25,7 @@ import html as html_lib
 # Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/
 # Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic
 # changes)
-SERVER_VERSION = "1.3.15"
+SERVER_VERSION = "1.3.16"
 
 OLD_VERSION = '2.3'
 
@@ -77,6 +77,7 @@ def overviewReport() -> str:
     html += '<a href="latest.html">Latest results</a><br>\n'
     html += '<a href="time_lt.html">Time report (improved)</a><br>\n'
     html += '<a href="time_gt.html">Time report (regressed)</a><br>\n'
+    html += '<a href="time_slow.html">Time report (slowest)</a><br>\n'
     html += '<a href="check_library_function_report.html">checkLibraryFunction report</a><br>\n'
     html += '<a href="check_library_noreturn_report.html">checkLibraryNoReturn report</a><br>\n'
     html += '<a href="check_library_use_ignore_report.html">checkLibraryUseIgnore report</a><br>\n'
@@ -158,7 +159,7 @@ def crashReport(results_path: str) -> str:
     current_year = datetime.date.today().year
     stack_traces = {}
     for filename in sorted(glob.glob(os.path.expanduser(results_path + '/*'))):
-        if not os.path.isfile(filename):
+        if not os.path.isfile(filename) or filename.endswith('.diff'):
             continue
         datestr = ''
         with open(filename, 'rt') as file_:
@@ -177,7 +178,7 @@ def crashReport(results_path: str) -> str:
                     if line.find('Crash') < 0:
                         break
                     package = filename[filename.rfind('/')+1:]
-                    counts = line.strip().split(' ')
+                    counts = line.split(' ')
                     c2 = ''
                     if counts[2] == 'Crash!':
                         c2 = 'Crash'
@@ -249,7 +250,7 @@ def timeoutReport(results_path: str) -> str:
     html += '<b>' + fmt('Package', 'Date       Time', OLD_VERSION, 'Head', link=False) + '</b>\n'
     current_year = datetime.date.today().year
     for filename in sorted(glob.glob(os.path.expanduser(results_path + '/*'))):
-        if not os.path.isfile(filename):
+        if not os.path.isfile(filename) or filename.endswith('.diff'):
             continue
         datestr = ''
         with open(filename, 'rt') as file_:
@@ -268,7 +269,7 @@ def timeoutReport(results_path: str) -> str:
                     if line.find('TO!') < 0:
                         break
                     package = filename[filename.rfind('/')+1:]
-                    counts = line.strip().split(' ')
+                    counts = line.split(' ')
                     c2 = ''
                     if counts[2] == 'TO!':
                         c2 = 'Timeout'
@@ -290,7 +291,7 @@ def staleReport(results_path: str) -> str:
     html += '<b>' + fmt('Package', 'Date       Time', link=False) + '</b>\n'
     current_year = datetime.date.today().year
     for filename in sorted(glob.glob(os.path.expanduser(results_path + '/*'))):
-        if not os.path.isfile(filename):
+        if not os.path.isfile(filename) or filename.endswith('.diff'):
             continue
         for line in open(filename, 'rt'):
             line = line.strip()
@@ -519,7 +520,7 @@ def headReport(resultsPath: str) -> str:
     today = strDateTime()[:10]
 
     for filename in sorted(glob.glob(resultsPath + '/*')):
-        if not os.path.isfile(filename):
+        if not os.path.isfile(filename) or filename.endswith('.diff'):
             continue
         uploadedToday = False
         firstLine = True
@@ -581,7 +582,7 @@ def headMessageIdReport(resultPath: str, messageId: str) -> str:
     text = messageId + '\n'
     e = '[' + messageId + ']\n'
     for filename in sorted(glob.glob(resultPath + '/*')):
-        if not os.path.isfile(filename):
+        if not os.path.isfile(filename) or filename.endswith('.diff'):
             continue
         url = None
         headResults = False
@@ -607,7 +608,7 @@ def headMessageIdTodayReport(resultPath: str, messageId: str) -> str:
     e = '[' + messageId + ']\n'
     today = strDateTime()[:10]
     for filename in sorted(glob.glob(resultPath + '/*')):
-        if not os.path.isfile(filename):
+        if not os.path.isfile(filename) or filename.endswith('.diff'):
             continue
         url = None
         headResults = False
@@ -650,10 +651,11 @@ def timeReport(resultPath: str, show_gt: bool) -> str:
     total_time_base = 0.0
     total_time_head = 0.0
     for filename in glob.glob(resultPath + '/*'):
-        if not os.path.isfile(filename):
+        if not os.path.isfile(filename) or filename.endswith('.diff'):
             continue
         datestr = ''
         for line in open(filename, 'rt'):
+            line = line.strip()
             if line.startswith('cppcheck: '):
                 if OLD_VERSION not in line:
                     # Package results seem to be too old, skip
@@ -666,7 +668,7 @@ def timeReport(resultPath: str, show_gt: bool) -> str:
                 continue
             if not line.startswith('elapsed-time:'):
                 continue
-            split_line = line.strip().split()
+            split_line = line.split()
             time_base = float(split_line[2])
             time_head = float(split_line[1])
             if time_base < 0.0 or time_head < 0.0:
@@ -688,10 +690,7 @@ def timeReport(resultPath: str, show_gt: bool) -> str:
                 data[pkg_name] = (datestr, split_line[2], split_line[1], time_factor)
             break
 
-    sorted_data = sorted(data.items(), key=lambda kv: kv[1][3])
+    sorted_data = sorted(data.items(), key=lambda kv: kv[1][3], reverse=show_gt)
-    if show_gt:
-        sorted_data.reverse()
-
     sorted_dict = collections.OrderedDict(sorted_data)
     for key in sorted_dict:
         html += fmt(key, sorted_dict[key][0], sorted_dict[key][1], sorted_dict[key][2], '{:.2f}'.format(sorted_dict[key][3]),
@@ -702,7 +701,7 @@ def timeReport(resultPath: str, show_gt: bool) -> str:
     if show_gt:
         html += '&gt;2.00'
     else:
-        html += '&lt;0.50'
+        html += '&lt;=0.50'
     html += ')\n'
     html += '\n'
     if total_time_base > 0.0:
@@ -723,6 +722,69 @@ def timeReport(resultPath: str, show_gt: bool) -> str:
     return html
 
 
+def timeReportSlow(resultPath: str) -> str:
+    title = 'Time report (slowest)'
+    html = '<html><head><title>{}</title></head><body>\n'.format(title)
+    html += '<h1>{}</h1>\n'.format(title)
+    html += '<pre>\n'
+    html += '<b>'
+    html += fmt('Package', 'Date       Time', OLD_VERSION, 'Head', link=False)
+    html += '</b>\n'
+
+    current_year = datetime.date.today().year
+
+    data = {}
+
+    for filename in glob.glob(resultPath + '/*'):
+        if not os.path.isfile(filename) or filename.endswith('.diff'):
+            continue
+        datestr = ''
+        for line in open(filename, 'rt'):
+            line = line.strip()
+            if line.startswith('cppcheck: '):
+                if OLD_VERSION not in line:
+                    # Package results seem to be too old, skip
+                    break
+                else:
+                    # Current package, parse on
+                    continue
+            if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
+                datestr = line
+                continue
+            elif line.startswith('count:'):
+                count_head = line.split()[1]
+                if count_head == 'TO!':
+                    # ignore results with timeouts
+                    break
+                continue
+            if not line.startswith('elapsed-time:'):
+                continue
+            split_line = line.split()
+            time_base = float(split_line[2])
+            time_head = float(split_line[1])
+            if time_base < 0.0 or time_head < 0.0:
+                # ignore results with crashes / errors
+                break
+            pkg_name = filename[len(resultPath)+1:]
+            data[pkg_name] = (datestr, split_line[2], split_line[1], time_head)
+            break
+
+        sorted_data = sorted(data.items(), key=lambda kv: kv[1][3])
+        if len(data) > 100:
+            first_key, _ = sorted_data[0]
+            # remove the entry with the lowest run-time
+            del data[first_key]
+
+    sorted_data = sorted(data.items(), key=lambda kv: kv[1][3], reverse=True)
+    sorted_dict = collections.OrderedDict(sorted_data)
+    for key in sorted_dict:
+        html += fmt(key, sorted_dict[key][0], sorted_dict[key][1], sorted_dict[key][2]) + '\n'
+    html += '</pre>\n'
+    html += '</body></html>\n'
+
+    return html
+
+
 def check_library_report(result_path: str, message_id: str) -> str:
     if message_id not in ('checkLibraryNoReturn', 'checkLibraryFunction', 'checkLibraryUseIgnore'):
         error_message = 'Invalid value ' + message_id + ' for message_id parameter.'
@@ -742,7 +804,7 @@ def check_library_report(result_path: str, message_id: str) -> str:
 
     function_counts = {}
     for filename in glob.glob(result_path + '/*'):
-        if not os.path.isfile(filename):
+        if not os.path.isfile(filename) or filename.endswith('.diff'):
             continue
         info_messages = False
         for line in open(filename, 'rt'):
@@ -784,7 +846,7 @@ def check_library_function_name(result_path: str, function_name: str) -> str:
     function_name = urllib.parse.unquote_plus(function_name)
     output_lines_list = []
     for filename in glob.glob(result_path + '/*'):
-        if not os.path.isfile(filename):
+        if not os.path.isfile(filename) or filename.endswith('.diff'):
             continue
         info_messages = False
         url = None
@@ -890,6 +952,9 @@ class HttpClientThread(Thread):
             elif url == 'time_gt.html':
                 text = timeReport(self.resultPath, True)
                 httpGetResponse(self.connection, text, 'text/html')
+            elif url == 'time_slow.html':
+                text = timeReportSlow(self.resultPath)
+                httpGetResponse(self.connection, text, 'text/html')
             elif url == 'check_library_function_report.html':
                 text = check_library_report(self.resultPath + '/' + 'info_output', message_id='checkLibraryFunction')
                 httpGetResponse(self.connection, text, 'text/html')