#!/usr/bin/env python3 # Server for 'donate-cpu.py' # Runs only under Python 3. import collections import glob import json import os import socket import re import datetime import time import traceback from threading import Thread import sys import urllib.request import urllib.parse import urllib.error import logging import logging.handlers import operator import html as html_lib from urllib.parse import urlparse # Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/ # Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic # changes) SERVER_VERSION = "1.3.47" OLD_VERSION = '2.13.0' HEAD_MARKER = 'head results:' INFO_MARKER = 'info messages:' # Set up logging logger = logging.getLogger() logger.setLevel(logging.INFO) # Logging to console handler_stream = logging.StreamHandler() logger.addHandler(handler_stream) # Log errors to a rotating file logfile = sys.path[0] if logfile: logfile += '/' logfile += 'donate-cpu-server.log' handler_file = logging.handlers.RotatingFileHandler(filename=logfile, maxBytes=100*1024, backupCount=1) handler_file.setFormatter(logging.Formatter('%(asctime)s %(message)s')) handler_file.setLevel(logging.ERROR) logger.addHandler(handler_file) def print_ts(msg) -> None: dt = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') print('[{}] {}'.format(dt, msg)) # Set up an exception hook for all uncaught exceptions so they can be logged def handle_uncaught_exception(exc_type, exc_value, exc_traceback): if issubclass(exc_type, KeyboardInterrupt): sys.__excepthook__(exc_type, exc_value, exc_traceback) return logging.error("Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback)) sys.excepthook = handle_uncaught_exception def strDateTime() -> str: return datetime.datetime.now().strftime('%Y-%m-%d %H:%M') def dateTimeFromStr(datestr: str) -> datetime.datetime: return datetime.datetime.strptime(datestr, '%Y-%m-%d %H:%M') def overviewReport() -> str: html = '\n' html += 'daca@home\n' html += '

daca@home

\n' html += 'Crash report - packages.txt
\n' html += 'Timeout report
\n' html += 'Stale report
\n' html += 'Diff report
\n' html += 'HEAD report
\n' html += 'HEAD (information) report
\n' html += 'Latest results
\n' html += 'Time report (improved)
\n' html += 'Time report (regressed) - packages.txt
\n' html += 'Time report (slowest)
\n' html += '
\n' html += '--check-library:
\n' html += 'checkLibraryFunction report
\n' html += 'checkLibraryNoReturn report
\n' html += 'checkLibraryUseIgnore report
\n' html += 'checkLibraryCheckType report
\n' html += '
\n' html += 'Debug warnings:
\n' html += 'debug
\n' html += 'varid0
\n' html += 'valueType
\n' html += 'noparamend
\n' html += 'simplifyTypedef
\n' html += 'simplifyUsingUnmatchedBodyEnd
\n' html += 'simplifyUsing
\n' html += 'valueFlowMaxIterations
\n' html += 'templateInstantiation
\n' #html += 'autoNoType
\n' #html += 'valueFlowBailout
\n' #html += 'bailoutUninitVar
\n' #html += 'symbolDatabaseWarning
\n' html += '
\n' html += 'Custom reports:
\n' html += 'valueFlowBailoutIncompleteVar report
\n' html += 'unknownMacro report
\n' html += '
\n' html += 'Important errors:
\n' html += 'cppcheckError
\n' html += 'internalError
\n' html += 'internalAstError
\n' html += 'syntaxError
\n' html += 'DacaWrongData
\n' html += 'dacaWrongSplitTemplateRightAngleBrackets
\n' html += '
\n' html += 'version ' + SERVER_VERSION + '\n' html += '' return html def fmt(a: str, b: str, c: str = None, d: str = None, e: str = None, link: bool = True, column_width=None) -> str: if column_width is None: column_width = [40, 10, 5, 7, 7, 8] ret = a while len(ret) < column_width[0]: ret += ' ' if len(ret) == column_width[0]: ret += ' ' + b[:10] while len(ret) < (column_width[0] + 1 + column_width[1]): ret += ' ' ret += ' ' if len(b) > 10: ret += b[-5:].rjust(column_width[2]) + ' ' if c is not None: ret += c.rjust(column_width[3]) + ' ' if d is not None: ret += d.rjust(column_width[4]) + ' ' if e is not None: ret += e.rjust(column_width[5]) if link: pos = ret.find(' ') ret = '' + a + '' + ret[pos:] return ret def latestReport(latestResults: list) -> str: html = '\n' html += 'Latest daca@home results\n' html += '

Latest daca@home results

\n' html += '
\n' + fmt('Package', 'Date       Time', OLD_VERSION, 'Head', 'Diff', link=False) + '\n'

    # Write report for latest results
    for filename in latestResults:
        if not os.path.isfile(filename):
            continue
        package = filename[filename.rfind('/')+1:]
        current_year = datetime.date.today().year

        datestr = None
        count = ['0', '0']
        lost = 0
        added = 0
        for line in open(filename, 'rt'):
            line = line.strip()
            if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
                datestr = line
            #elif line.startswith('cppcheck:'):
            #    cppcheck = line[9:]
            elif line.startswith('count: '):
                count = line.split(' ')[1:]
            elif line.startswith('head ') and not line.startswith('head results:'):
                added += 1
            elif line.startswith(OLD_VERSION + ' '):
                lost += 1
        diff = ''
        if lost > 0:
            diff += '-' + str(lost)
        if added > 0:
            diff += '+' + str(added)
        html += fmt(package, datestr, count[1], count[0], diff) + '\n'

    html += '
\n' return html def crashReport(results_path: str, query_params: dict): pkgs = '' if query_params.get('pkgs') == '1' else None html = '\n' html += 'Crash report\n' html += '

Crash report

\n' html += '
\n'
    html += '' + fmt('Package', 'Date       Time', OLD_VERSION, 'Head', link=False) + '\n'
    current_year = datetime.date.today().year
    stack_traces = {}
    for filename in sorted(glob.glob(os.path.expanduser(results_path + '/*'))):
        if not os.path.isfile(filename) or filename.endswith('.diff'):
            continue
        with open(filename, 'rt') as file_:
            datestr = None
            package_url = None
            for line in file_:
                line = line.strip()
                if line.startswith('cppcheck: '):
                    if OLD_VERSION not in line:
                        # Package results seem to be too old, skip
                        break
                    else:
                        # Current package, parse on
                        continue
                if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
                    datestr = line
                elif pkgs is not None and package_url is None and line.startswith('ftp://'):
                    package_url = line
                elif line.startswith('count:'):
                    if line.find('Crash') < 0:
                        break
                    package = filename[filename.rfind('/')+1:]
                    counts = line.split(' ')
                    c_version = ''
                    if counts[2] == 'Crash!':
                        c_version = 'Crash'
                    c_head = ''
                    if counts[1] == 'Crash!':
                        c_head = 'Crash'
                    html += fmt(package, datestr, c_version, c_head) + '\n'
                    if c_head != 'Crash':
                        break
                    if package_url is not None:
                        pkgs += '{}\n'.format(package_url)
                elif line.find(' received signal ') != -1:
                    crash_line = next(file_, '').strip()
                    location_index = crash_line.rfind(' at ')
                    if location_index > 0:
                        code_line = next(file_, '').strip()
                    else:
                        code_line = ''
                    stack_trace = []
                    while True:
                        l = next(file_, '')
                        if not l.strip():
                            break
                        # #0  0x00007ffff71cbf67 in raise () from /lib64/libc.so.6
                        m = re.search(r'(?P#\d+) .* in (?P.+)\(.*\) from (?P.*)$', l)
                        if m:
                            #print('0 - {} - {} - {}'.format(m.group('number'), m.group('function'), m.group('binary')))
                            stack_trace.append(m.group('number') + ' ' + m.group('function') + '(...) from ' + m.group('binary'))
                            continue
                        # #11 0x00000000006f2414 in valueFlowNumber (tokenlist=tokenlist@entry=0x7fffffffc610) at build/valueflow.cpp:2503
                        m = re.search(r'(?P#\d+) .* in (?P.+?) \(.*\) at (?P.*)$', l)
                        if m:
                            #print('1 - {} - {} - {}'.format(m.group('number'), m.group('function'), m.group('location')))
                            stack_trace.append(m.group('number') + ' ' + m.group('function') + '(...) at ' + m.group('location'))
                            continue
                        # #18 ForwardTraversal::updateRecursive (this=0x7fffffffb3c0, tok=0x14668a0) at build/forwardanalyzer.cpp:415
                        m = re.search(r'(?P#\d+) (?P.+)\(.*\) at (?P.*)$', l)
                        if m:
                            #print('2 - {} - {} - {}'.format(m.group('number'), m.group('function'), m.group('location')))
                            stack_trace.append(m.group('number') + ' ' + m.group('function') + '(...) at ' + m.group('location'))
                            continue

                        print_ts('{} - unmatched stack frame - {}'.format(package, l))
                        break
                    key = hash(' '.join(stack_trace))

                    if key in stack_traces:
                        stack_traces[key]['code_line'] = code_line
                        stack_traces[key]['stack_trace'] = stack_trace
                        stack_traces[key]['n'] += 1
                        stack_traces[key]['packages'].append(package)
                    else:
                        stack_traces[key] = {'stack_trace': stack_trace, 'n': 1, 'code_line': code_line, 'packages': [package], 'crash_line': crash_line}
                    break

    html += '
\n' html += '
\n'
    html += 'Stack traces\n'
    for stack_trace in sorted(list(stack_traces.values()), key=lambda x: x['n'], reverse=True):
        html += 'Packages: ' + ' '.join(['' + p + '' for p in stack_trace['packages']]) + '\n'
        html += html_lib.escape(stack_trace['crash_line']) + '\n'
        html += html_lib.escape(stack_trace['code_line']) + '\n'
        html += html_lib.escape('\n'.join(stack_trace['stack_trace'])) + '\n\n'
    html += '
\n' html += '\n' if pkgs is not None: return pkgs, 'text/plain' return html, 'text/html' def timeoutReport(results_path: str) -> str: html = '\n' html += 'Timeout report\n' html += '

Timeout report

\n' html += '
\n'
    html += '' + fmt('Package', 'Date       Time', OLD_VERSION, 'Head', link=False) + '\n'
    current_year = datetime.date.today().year
    for filename in sorted(glob.glob(os.path.expanduser(results_path + '/*'))):
        if not os.path.isfile(filename) or filename.endswith('.diff'):
            continue
        with open(filename, 'rt') as file_:
            datestr = None
            for line in file_:
                line = line.strip()
                if line.startswith('cppcheck: '):
                    if OLD_VERSION not in line:
                        # Package results seem to be too old, skip
                        break
                    else:
                        # Current package, parse on
                        continue
                if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
                    datestr = line
                elif line.startswith('count:'):
                    if line.find('TO!') < 0:
                        break
                    package = filename[filename.rfind('/')+1:]
                    counts = line.split(' ')
                    c2 = ''
                    if counts[2] == 'TO!':
                        c2 = 'Timeout'
                    c1 = ''
                    if counts[1] == 'TO!':
                        c1 = 'Timeout'
                    html += fmt(package, datestr, c2, c1) + '\n'
                    break

    html += '
\n' html += '\n' return html def staleReport(results_path: str) -> str: html = '\n' html += 'Stale report\n' html += '

Stale report

\n' html += '
\n'
    html += '' + fmt('Package', 'Date       Time', link=False) + '\n'
    current_year = datetime.date.today().year
    for filename in sorted(glob.glob(os.path.expanduser(results_path + '/*'))):
        if not os.path.isfile(filename) or filename.endswith('.diff'):
            continue
        for line in open(filename, 'rt'):
            line = line.strip()
            if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
                datestr = line
            else:
                continue
            dt = dateTimeFromStr(datestr)
            diff = datetime.datetime.now() - dt
            if diff.days < 30:
                continue
            package = filename[filename.rfind('/')+1:]
            html += fmt(package, datestr) + '\n'
            break
    html += '
\n' html += '\n' return html def diffReportFromDict(out: dict, today: str) -> str: html = '
\n'
    html += 'MessageID                           ' + OLD_VERSION + '    Head\n'
    sum0 = 0
    sum1 = 0
    for messageId in sorted(out.keys()):
        line = messageId + ' '
        counts = out[messageId]
        sum0 += counts[0]
        sum1 += counts[1]
        if counts[0] > 0:
            c = str(counts[0])
            while len(line) < 40 - len(c):
                line += ' '
            line += c + ' '
        if counts[1] > 0:
            c = str(counts[1])
            while len(line) < 48 - len(c):
                line += ' '
            line += c
        line = '' + messageId + '' + line[line.find(' '):]
        html += line + '\n'

    # Sum
    html += '================================================\n'
    line = ''
    while len(line) < 40 - len(str(sum0)):
        line += ' '
    line += str(sum0) + ' '
    while len(line) < 48 - len(str(sum1)):
        line += ' '
    line += str(sum1)
    html += line + '\n'
    html += '
\n' return html def diffReport(resultsPath: str) -> str: out = {} outToday = {} today = strDateTime()[:10] for filename in sorted(glob.glob(resultsPath + '/*.diff')): if not os.path.isfile(filename): continue with open(filename, 'rt') as f: data = json.loads(f.read()) uploadedToday = data['date'] == today for messageId in data['sums']: sums = data['sums'][messageId] if OLD_VERSION not in sums: continue if messageId not in out: out[messageId] = [0, 0] out[messageId][0] += sums[OLD_VERSION] out[messageId][1] += sums['head'] if uploadedToday: if messageId not in outToday: outToday[messageId] = [0, 0] outToday[messageId][0] += sums[OLD_VERSION] outToday[messageId][1] += sums['head'] html = '\n' html += 'Diff report\n' html += '

Diff report

\n' html += '

Uploaded today

' html += diffReportFromDict(outToday, 'today') html += '

All

' html += diffReportFromDict(out, '') return html def generate_package_diff_statistics(filename: str) -> None: is_diff = False sums = {} for line in open(filename, 'rt'): line = line.strip() if line == 'diff:': is_diff = True continue elif not is_diff: continue if not line.endswith(']'): continue if line.startswith(OLD_VERSION + ' '): version = OLD_VERSION elif line.startswith('head '): version = 'head' else: continue messageId = line[line.rfind('[')+1:len(line)-1] if messageId not in sums: sums[messageId] = {OLD_VERSION: 0, 'head': 0} sums[messageId][version] += 1 output = {'date': strDateTime()[:10], 'sums': sums} filename_diff = filename + '.diff' if sums: with open(filename_diff, 'wt') as f: f.write(json.dumps(output)) elif os.path.isfile(filename_diff): os.remove(filename_diff) def diffMessageIdReport(resultPath: str, messageId: str) -> str: text = messageId + '\n' e = '[' + messageId + ']\n' for filename in sorted(glob.glob(resultPath + '/*.diff')): if not os.path.isfile(filename): continue with open(filename, 'rt') as f: diff_stats = f.read() if messageId not in diff_stats: continue url = None diff = False for line in open(filename[:-5], 'rt'): if line.startswith('ftp://'): url = line elif line == 'diff:\n': diff = True elif not diff: continue elif line.endswith(e): if url: text += url url = None text += line return text def diffMessageIdTodayReport(resultPath: str, messageId: str) -> str: text = messageId + '\n' e = '[' + messageId + ']\n' today = strDateTime()[:10] for filename in sorted(glob.glob(resultPath + '/*.diff')): if not os.path.isfile(filename): continue with open(filename, 'rt') as f: diff_stats = f.read() if messageId not in diff_stats: continue if today not in diff_stats: continue url = None diff = False firstLine = True for line in open(filename[:-5], 'rt'): if firstLine: firstLine = False if not line.startswith(today): break if line.startswith('ftp://'): url = line elif line == 'diff:\n': diff = True elif not diff: continue elif line.endswith(e): if url: text += url url = None text += line return text def summaryReportFromDict(out: dict, prefix: str, today: str) -> str: html = '
\n'
    html += 'MessageID                                  Count\n'
    sumTotal = 0
    for messageId in sorted(out.keys()):
        line = messageId + ' '
        counts = out[messageId]
        sumTotal += counts
        if counts > 0:
            c = str(counts)
            while len(line) < 48 - len(c):
                line += ' '
            line += c + ' '
        line = '' + messageId + '' + line[line.find(' '):]
        html += line + '\n'

    # Sum
    html += '================================================\n'
    line = ''
    while len(line) < 48 - len(str(sumTotal)):
        line += ' '
    line += str(sumTotal) + ' '
    html += line + '\n'
    html += '
\n' return html def summaryReport(resultsPath: str, name: str, prefix: str, marker: str) -> str: out = {} outToday = {} today = strDateTime()[:10] for filename in sorted(glob.glob(resultsPath + '/*')): if not os.path.isfile(filename) or filename.endswith('.diff'): continue uploadedToday = False firstLine = True inResults = False for line in open(filename, 'rt'): if firstLine: if line.startswith(today): uploadedToday = True firstLine = False continue line = line.strip() if line.startswith('cppcheck: '): if OLD_VERSION not in line: # Package results seem to be too old, skip break else: # Current package, parse on continue if line.startswith(marker): inResults = True continue if not inResults: continue if line.startswith('diff:'): break if not line.endswith(']'): continue if ': note: ' in line: # notes normally do not contain message ids but can end with ']' continue message_id_start_pos = line.rfind('[') if message_id_start_pos <= 0: continue messageId = line[message_id_start_pos+1:len(line)-1] if ' ' in messageId: # skip invalid messageIds continue if messageId not in out: out[messageId] = 0 out[messageId] += 1 if uploadedToday: if messageId not in outToday: outToday[messageId] = 0 outToday[messageId] += 1 html = '\n' html += '{} report\n'.format(name) html += '

HEAD report

\n' html += '

Uploaded today

' html += summaryReportFromDict(outToday, prefix, 'today') html += '

All

' html += summaryReportFromDict(out, prefix, '') return html def headReport(resultsPath: str) -> str: return summaryReport(resultsPath, 'HEAD', 'head', HEAD_MARKER) def infoReport(resultsPath: str) -> str: return summaryReport(resultsPath, 'HEAD (information)', 'headinfo', INFO_MARKER) def messageIdReport(resultPath: str, marker: str, messageId: str, query_params: dict) -> str: pkgs = '' if query_params.get('pkgs') == '1' else None text = messageId + '\n' e = '[' + messageId + ']\n' for filename in sorted(glob.glob(resultPath + '/*')): if not os.path.isfile(filename) or filename.endswith('.diff'): continue url = None inResults = False for line in open(filename, 'rt'): if line.startswith('cppcheck: '): if OLD_VERSION not in line: # Package results seem to be too old, skip break else: # Current package, parse on continue if line.startswith('ftp://'): url = line continue if not inResults: if line.startswith(marker): inResults = True continue if line.startswith('diff:'): break if line.endswith(e): if url: text += url if pkgs is not None: pkgs += url url = None text += line if pkgs is not None: return pkgs return text def headMessageIdReport(resultPath: str, messageId: str, query_params: dict) -> str: return messageIdReport(resultPath, HEAD_MARKER, messageId, query_params) def infoMessageIdReport(resultPath: str, messageId: str, query_params: dict) -> str: return messageIdReport(resultPath, INFO_MARKER, messageId, query_params) def messageIdTodayReport(resultPath: str, messageId: str, marker: str) -> str: text = messageId + '\n' e = '[' + messageId + ']\n' today = strDateTime()[:10] for filename in sorted(glob.glob(resultPath + '/*')): if not os.path.isfile(filename) or filename.endswith('.diff'): continue url = None inResults = False firstLine = True for line in open(filename, 'rt'): if firstLine: firstLine = False if not line.startswith(today): break if line.startswith('ftp://'): url = line continue if not inResults: if line.startswith(marker): inResults = True continue if line.startswith('diff:'): break if line.endswith(e): if url: text += url url = None text += line return text def headMessageIdTodayReport(resultPath: str, messageId: str) -> str: return messageIdTodayReport(resultPath, messageId, HEAD_MARKER) def infoMessageIdTodayReport(resultPath: str, messageId: str) -> str: return messageIdTodayReport(resultPath, messageId, INFO_MARKER) # TODO: needs to dinicate that it returns 'tuple[str, str]' but that isn't supported until Python 3.9 def timeReport(resultPath: str, show_gt: bool, query_params: dict): # no need for package report support in "improved" report pkgs = '' if show_gt and query_params and query_params.get('pkgs') == '1' else None factor = float(query_params.get('factor')) if query_params and 'factor' in query_params else 2.0 if not show_gt: factor = 1.0 / factor title = 'Time report ({})'.format('regressed' if show_gt else 'improved') html = '\n' html += '{}\n'.format(title) html += '

{}

\n'.format(title) html += '
\n'
    column_width = [40, 10, 10, 10, 10, 10]
    html += ''
    html += fmt('Package', 'Date       Time', OLD_VERSION, 'Head', 'Factor', link=False, column_width=column_width)
    html += '\n'

    current_year = datetime.date.today().year

    data = {}

    total_time_base = 0.0
    total_time_head = 0.0
    for filename in glob.glob(resultPath + '/*'):
        if not os.path.isfile(filename) or filename.endswith('.diff'):
            continue
        datestr = None
        package_url = None
        for line in open(filename, 'rt'):
            line = line.strip()
            if line.startswith('cppcheck: '):
                if OLD_VERSION not in line:
                    # Package results seem to be too old, skip
                    break
                else:
                    # Current package, parse on
                    continue
            if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
                datestr = line
                continue
            elif pkgs is not None and package_url is None and line.startswith('ftp://'):
                package_url = line
            if not line.startswith('elapsed-time:'):
                continue
            split_line = line.split()
            time_base = float(split_line[2])
            time_head = float(split_line[1])
            if time_base < 0.0 or time_head < 0.0:
                # ignore results with crashes / errors for the time report
                break
            if time_base == 0.0 and time_head == 0.0:
                # no difference possible
                break
            total_time_base += time_base
            total_time_head += time_head
            if time_base == time_head:
                # no difference
                break
            if time_base > 0.0 and time_head > 0.0:
                time_factor = time_head / time_base
            elif time_base == 0.0:
                # the smallest possible value is 0.1 so treat that as an increase of 100%
                # on top of the existing 100% (treating the base 0.0 as such).
                time_factor = 1.0 + (time_head * 10)
            else:
                time_factor = 0.0
            suspicious_time_difference = False
            if show_gt and time_factor > factor:
                suspicious_time_difference = True
            elif not show_gt and time_factor < factor:
                suspicious_time_difference = True
            if suspicious_time_difference:
                pkg_name = filename[len(resultPath)+1:]
                data[pkg_name] = (datestr, split_line[2], split_line[1], time_factor)

                if package_url is not None:
                    pkgs += '{}\n'.format(package_url)
            break

    sorted_data = sorted(data.items(), key=lambda kv: kv[1][3], reverse=show_gt)
    sorted_dict = collections.OrderedDict(sorted_data)
    for key in sorted_dict:
        html += fmt(key, sorted_dict[key][0], sorted_dict[key][1], sorted_dict[key][2], '{:.2f}'.format(sorted_dict[key][3]),
                    column_width=column_width) + '\n'

    html += '\n'
    html += '(listed above are all suspicious timings with a factor '
    html += '>' if show_gt else '<'
    html += ' {}'.format(format(factor, '.2f'))
    html += ')\n'
    html += '\n'
    if total_time_base > 0.0:
        total_time_factor = total_time_head / total_time_base
    else:
        total_time_factor = 0.0
    html += 'Time for all packages (not just the ones listed above):\n'
    html += fmt('Total time:',
            '',
            '{:.1f}'.format(total_time_base),
            '{:.1f}'.format(total_time_head),
            '{:.2f}'.format(total_time_factor), link=False, column_width=column_width)

    html += '\n'
    html += '
\n' html += '\n' if pkgs is not None: return pkgs, 'text/plain' return html, 'text/html' def timeReportSlow(resultPath: str) -> str: title = 'Time report (slowest)' html = '\n' html += '{}\n'.format(title) html += '

{}

\n'.format(title) html += '
\n'
    html += ''
    html += fmt('Package', 'Date       Time', OLD_VERSION, 'Head', link=False)
    html += '\n'

    current_year = datetime.date.today().year

    data = {}

    for filename in glob.glob(resultPath + '/*'):
        if not os.path.isfile(filename) or filename.endswith('.diff'):
            continue
        datestr = None
        for line in open(filename, 'rt'):
            line = line.strip()
            if line.startswith('cppcheck: '):
                if OLD_VERSION not in line:
                    # Package results seem to be too old, skip
                    break
                else:
                    # Current package, parse on
                    continue
            if datestr is None and line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
                datestr = line
                continue
            elif line.startswith('count:'):
                count_head = line.split()[1]
                if count_head == 'TO!':
                    # ignore results with timeouts
                    break
                continue
            if not line.startswith('elapsed-time:'):
                continue
            split_line = line.split()
            time_base = float(split_line[2])
            time_head = float(split_line[1])
            if time_base < 0.0 or time_head < 0.0:
                # ignore results with crashes / errors
                break
            pkg_name = filename[len(resultPath)+1:]
            data[pkg_name] = (datestr, split_line[2], split_line[1], time_head)
            break

        sorted_data = sorted(data.items(), key=lambda kv: kv[1][3])
        if len(data) > 100:
            first_key, _ = sorted_data[0]
            # remove the entry with the lowest run-time
            del data[first_key]

    sorted_data = sorted(data.items(), key=lambda kv: kv[1][3], reverse=True)
    sorted_dict = collections.OrderedDict(sorted_data)
    for key in sorted_dict:
        html += fmt(key, sorted_dict[key][0], sorted_dict[key][1], sorted_dict[key][2]) + '\n'
    html += '
\n' html += '\n' return html def check_library_report(result_path: str, message_id: str) -> str: if message_id not in ('checkLibraryNoReturn', 'checkLibraryFunction', 'checkLibraryUseIgnore', 'checkLibraryCheckType', 'valueFlowBailoutIncompleteVar', 'unknownMacro'): error_message = 'Invalid value ' + message_id + ' for message_id parameter.' print_ts(error_message) return error_message if message_id == 'unknownMacro': metric = 'macros' m_column = 'macro' metric_link = 'unknown_macro' start_marker = HEAD_MARKER elif message_id == 'valueFlowBailoutIncompleteVar': metric = 'variables' m_column = 'Variable' metric_link = 'incomplete_var' start_marker = HEAD_MARKER elif message_id == 'checkLibraryCheckType': metric = 'types' m_column = 'Type' metric_link = 'check_library' start_marker = INFO_MARKER else: metric = 'functions' m_column = 'Function' metric_link = 'check_library' start_marker = INFO_MARKER functions_shown_max = 5000 html = '\n' html += '' + message_id + ' report\n' html += '

' + message_id + ' report

\n' html += 'Top ' + str(functions_shown_max) + ' ' + metric + ' are shown.' html += '
\n'
    column_widths = [10, 100]
    html += ''
    html += 'Count'.rjust(column_widths[0]) + ' ' + m_column
    html += '\n'

    function_counts = {}
    for filename in glob.glob(result_path + '/*'):
        if not os.path.isfile(filename) or filename.endswith('.diff'):
            continue
        in_results = False
        for line in open(filename, 'rt'):
            if line.startswith('cppcheck: '):
                if OLD_VERSION not in line:
                    # Package results seem to be too old, skip
                    break
                else:
                    # Current package, parse on
                    continue
            if not in_results:
                if line.startswith(start_marker):
                    in_results = True
                continue
            if line.startswith('diff:'):
                break
            if line.endswith('[' + message_id + ']\n'):
                if message_id == 'unknownMacro':
                    marker = 'required. If '
                    function_name = line[(line.find(marker) + len(marker)):line.rfind('is a macro') - 1]
                elif message_id == 'valueFlowBailoutIncompleteVar':
                    marker = 'incomplete variable '
                    function_name = line[(line.find(marker) + len(marker)):line.rfind('[') - 1]
                elif message_id == 'checkLibraryFunction':
                    marker = 'for function '
                    function_name = line[(line.find(marker) + len(marker)):line.rfind('[') - 1]
                elif message_id == 'checkLibraryCheckType':
                    marker = 'configuration for '
                    function_name = line[(line.find(marker) + len(marker)):line.rfind('[') - 1]
                else:
                    marker = ': Function '
                    function_name = line[(line.find(marker) + len(marker)):line.rfind('should have') - 1]
                function_counts[function_name] = function_counts.setdefault(function_name, 0) + 1

    function_details_list = []
    for function_name, count in sorted(list(function_counts.items()), key=operator.itemgetter(1), reverse=True):
        if len(function_details_list) >= functions_shown_max:
            break
        function_details_list.append(str(count).rjust(column_widths[0]) + ' ' +
                '' + function_name + '\n')

    html += ''.join(function_details_list)
    html += '
\n' html += '\n' return html # Lists all checkLibrary* messages regarding the given function name def check_library_function_name(result_path: str, function_name: str, query_params: dict, nonfunc_id: str='') -> str: pkgs = '' if query_params.get('pkgs') == '1' else None function_name = urllib.parse.unquote_plus(function_name) if nonfunc_id: id = '[' + nonfunc_id marker = HEAD_MARKER else: if function_name.endswith('()'): id = '[checkLibrary' else: id = '[checkLibraryCheckType]' marker = INFO_MARKER output_lines_list = [] for filename in glob.glob(result_path + '/*'): if not os.path.isfile(filename) or filename.endswith('.diff'): continue in_results = False package_url = None cppcheck_options = None for line in open(filename, 'rt'): if line.startswith('cppcheck: '): if OLD_VERSION not in line: # Package results seem to be too old, skip break else: # Current package, parse on continue if line.startswith('ftp://'): package_url = line continue if line.startswith('cppcheck-options:'): cppcheck_options = line continue if not in_results: if line.startswith(marker): in_results = True continue if line.startswith('diff:'): break if id not in line: continue if not (' ' + function_name + ' ') in line: continue if pkgs is not None and package_url is not None: pkgs += '{}\n'.format(package_url.strip()) break if package_url: output_lines_list.append(package_url) package_url = None if cppcheck_options: output_lines_list.append(cppcheck_options) cppcheck_options = None output_lines_list.append(line) if pkgs is not None: return pkgs return ''.join(output_lines_list) def sendAll(connection: socket.socket, text: str) -> None: data = text.encode('utf-8', 'ignore') while data: num = connection.send(data) if num < len(data): data = data[num:] else: data = None def httpGetResponse(connection: socket.socket, data: str, contentType: str) -> None: resp = 'HTTP/1.1 200 OK\r\n' resp += 'Connection: close\r\n' resp += 'Content-length: ' + str(len(data)) + '\r\n' resp += 'Content-type: ' + contentType + '\r\n\r\n' resp += data sendAll(connection, resp) class HttpClientThread(Thread): def __init__(self, connection: socket.socket, cmd: str, resultPath: str, latestResults: list) -> None: Thread.__init__(self) self.connection = connection self.cmd = cmd[:cmd.find('\r\n')] self.resultPath = resultPath self.infoPath = os.path.join(self.resultPath, 'info_output') self.latestResults = latestResults # TODO: use a proper parser @staticmethod def parse_req(cmd): req_parts = cmd.split(' ') if len(req_parts) != 3 or req_parts[0] != 'GET' or not req_parts[2].startswith('HTTP'): return None, None url_obj = urlparse(req_parts[1]) return url_obj.path, dict(urllib.parse.parse_qsl(url_obj.query)) def run(self): try: cmd = self.cmd print_ts(cmd) url, queryParams = self.parse_req(cmd) if url is None: print_ts('invalid request: {}'.format(cmd)) self.connection.close() return if url == '/': html = overviewReport() httpGetResponse(self.connection, html, 'text/html') elif url == '/latest.html': html = latestReport(self.latestResults) httpGetResponse(self.connection, html, 'text/html') elif url == '/crash.html': text, mime = crashReport(self.resultPath, queryParams) httpGetResponse(self.connection, text, mime) elif url == '/timeout.html': html = timeoutReport(self.resultPath) httpGetResponse(self.connection, html, 'text/html') elif url == '/stale.html': html = staleReport(self.resultPath) httpGetResponse(self.connection, html, 'text/html') elif url == '/diff.html': html = diffReport(self.resultPath) httpGetResponse(self.connection, html, 'text/html') elif url.startswith('/difftoday-'): messageId = url[len('/difftoday-'):] text = diffMessageIdTodayReport(self.resultPath, messageId) httpGetResponse(self.connection, text, 'text/plain') elif url.startswith('/diff-'): messageId = url[len('/diff-'):] text = diffMessageIdReport(self.resultPath, messageId) httpGetResponse(self.connection, text, 'text/plain') elif url == '/head.html': html = headReport(self.resultPath) httpGetResponse(self.connection, html, 'text/html') elif url == '/headinfo.html': html = infoReport(self.infoPath) httpGetResponse(self.connection, html, 'text/html') elif url.startswith('/headtoday-'): messageId = url[len('/headtoday-'):] text = headMessageIdTodayReport(self.resultPath, messageId) httpGetResponse(self.connection, text, 'text/plain') elif url.startswith('/headinfotoday-'): messageId = url[len('/headinfotoday-'):] text = infoMessageIdTodayReport(self.infoPath, messageId) httpGetResponse(self.connection, text, 'text/plain') elif url.startswith('/head-'): messageId = url[len('/head-'):] text = headMessageIdReport(self.resultPath, messageId, queryParams) httpGetResponse(self.connection, text, 'text/plain') elif url.startswith('/headinfo-'): messageId = url[len('/headinfo-'):] text = infoMessageIdReport(self.infoPath, messageId, queryParams) httpGetResponse(self.connection, text, 'text/plain') elif url == '/time_lt.html': text, mime = timeReport(self.resultPath, False, queryParams) httpGetResponse(self.connection, text, mime) elif url == '/time_gt.html': text, mime = timeReport(self.resultPath, True, queryParams) httpGetResponse(self.connection, text, mime) elif url == '/time_slow.html': text = timeReportSlow(self.resultPath) httpGetResponse(self.connection, text, 'text/html') elif url == '/check_library_function_report.html': text = check_library_report(self.infoPath, message_id='checkLibraryFunction') httpGetResponse(self.connection, text, 'text/html') elif url == '/check_library_noreturn_report.html': text = check_library_report(self.infoPath, message_id='checkLibraryNoReturn') httpGetResponse(self.connection, text, 'text/html') elif url == '/check_library_use_ignore_report.html': text = check_library_report(self.infoPath, message_id='checkLibraryUseIgnore') httpGetResponse(self.connection, text, 'text/html') elif url == '/check_library_check_type_report.html': text = check_library_report(self.infoPath, message_id='checkLibraryCheckType') httpGetResponse(self.connection, text, 'text/html') elif url.startswith('/check_library-'): function_name = url[len('/check_library-'):] text = check_library_function_name(self.infoPath, function_name, queryParams) httpGetResponse(self.connection, text, 'text/plain') elif url == '/value_flow_bailout_incomplete_var.html': text = check_library_report(self.resultPath, message_id='valueFlowBailoutIncompleteVar') httpGetResponse(self.connection, text, 'text/html') elif url == '/unknown_macro.html': text = check_library_report(self.resultPath, message_id='unknownMacro') httpGetResponse(self.connection, text, 'text/html') elif url.startswith('/incomplete_var-'): var_name = url[len('/incomplete_var-'):] text = check_library_function_name(self.resultPath, var_name, queryParams, nonfunc_id='valueFlowBailoutIncompleteVar') httpGetResponse(self.connection, text, 'text/plain') elif url.startswith('/unknown_macro-'): var_name = url[len('/unknown_macro-'):] text = check_library_function_name(self.resultPath, var_name, queryParams, nonfunc_id='unknownMacro') httpGetResponse(self.connection, text, 'text/plain') else: filename = resultPath + url if not os.path.isfile(filename): print_ts('HTTP/1.1 404 Not Found') self.connection.send(b'HTTP/1.1 404 Not Found\r\n\r\n') else: with open(filename, 'rt') as f: data = f.read() httpGetResponse(self.connection, data, 'text/plain') except: tb = "".join(traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])) print_ts(tb) httpGetResponse(self.connection, tb, 'text/plain') finally: time.sleep(1) self.connection.close() def read_data(connection, cmd, pos_nl, max_data_size, check_done, cmd_name, timeout=10): data = cmd[pos_nl+1:] t = 0.0 try: while (len(data) < max_data_size) and (not check_done or not data.endswith('\nDONE')) and (timeout > 0 and t < timeout): bytes_received = connection.recv(1024) if bytes_received: try: text_received = bytes_received.decode('utf-8', 'ignore') except UnicodeDecodeError as e: print_ts('Error: Decoding failed ({}): {}'.format(cmd_name, e)) data = None break t = 0.0 data += text_received elif not check_done: break else: time.sleep(0.2) t += 0.2 connection.close() except socket.error as e: print_ts('Socket error occurred ({}): {}'.format(cmd_name, e)) data = None if timeout > 0 and t >= timeout: print_ts('Timeout occurred ({}).'.format(cmd_name)) data = None return data def server(server_address_port: int, packages: list, packageIndex: int, resultPath: str) -> None: socket.setdefaulttimeout(30) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server_address = ('', server_address_port) sock.bind(server_address) sock.listen(1) latestResults = [] if os.path.isfile('latest.txt'): with open('latest.txt', 'rt') as f: latestResults = f.read().strip().split(' ') print_ts('version ' + SERVER_VERSION) print_ts('listening on port ' + str(server_address_port)) while True: # wait for a connection print_ts('waiting for a connection') connection, client_address = sock.accept() try: bytes_received = connection.recv(128) cmd = bytes_received.decode('utf-8', 'ignore') except socket.error: connection.close() continue except UnicodeDecodeError as e: connection.close() print_ts('Error: Decoding failed: ' + str(e)) continue pos_nl = cmd.find('\n') if pos_nl < 1: print_ts('No newline found in data.') continue firstLine = cmd[:pos_nl] if re.match('[a-zA-Z0-9./ ]+', firstLine) is None: print_ts('Unsupported characters found in command: {}'.format(firstLine)) connection.close() continue if cmd.startswith('GET /'): newThread = HttpClientThread(connection, cmd, resultPath, latestResults) newThread.start() elif cmd == 'GetCppcheckVersions\n': reply = 'head ' + OLD_VERSION print_ts('GetCppcheckVersions: ' + reply) connection.send(reply.encode('utf-8', 'ignore')) connection.close() elif cmd == 'get\n': while True: pkg = packages[packageIndex] packageIndex += 1 if packageIndex >= len(packages): packageIndex = 0 if pkg is not None: break with open('package-index.txt', 'wt') as f: f.write(str(packageIndex) + '\n') print_ts('get:' + pkg) connection.send(pkg.encode('utf-8', 'ignore')) connection.close() elif cmd.startswith('write\nftp://') or cmd.startswith('write\nhttp://'): data = read_data(connection, cmd, pos_nl, max_data_size=2 * 1024 * 1024, check_done=True, cmd_name='write') if data is None: continue pos = data.find('\n') if pos == -1: print_ts('No newline found in data. Ignoring result data.') continue if pos < 10: print_ts('Data is less than 10 characters. Ignoring result data.') continue url = data[:pos] print_ts('write:' + url) # save data res = re.match(r'ftp://.*pool/main/[^/]+/([^/]+)/[^/]*tar.(gz|bz2|xz)', url) if res is None: res = re.match(r'https?://cppcheck\.sf\.net/([a-z]+).tgz', url) if res is None: print_ts('res is None. Ignoring result data.') continue if url not in packages: print_ts('Url is not in packages. Ignoring result data.') continue # Verify that head was compared to correct OLD_VERSION versions_found = False old_version_wrong = False for line in data.split('\n', 20): if line.startswith('cppcheck: '): versions_found = True if OLD_VERSION not in line.split(): print_ts('Compared to wrong old version. Should be ' + OLD_VERSION + '. Versions compared: ' + line + '. Ignoring result data.') old_version_wrong = True break if not versions_found: print_ts('Cppcheck versions missing in result data. Ignoring result data.') continue if old_version_wrong: print_ts('Unexpected old version. Ignoring result data.') continue print_ts('results added for package ' + res.group(1)) filename = os.path.join(resultPath, res.group(1)) with open(filename, 'wt') as f: f.write(strDateTime() + '\n' + data) # track latest added results.. if len(latestResults) >= 20: latestResults = latestResults[1:] latestResults.append(filename) with open('latest.txt', 'wt') as f: f.write(' '.join(latestResults)) # generate package.diff.. generate_package_diff_statistics(filename) elif cmd.startswith('write_info\nftp://') or cmd.startswith('write_info\nhttp://'): data = read_data(connection, cmd, pos_nl, max_data_size=1024 * 1024, check_done=True, cmd_name='write_info') if data is None: continue pos = data.find('\n') if pos == -1: print_ts('No newline found in data. Ignoring information data.') continue if pos < 10: print_ts('Data is less than 10 characters. Ignoring information data.') continue url = data[:pos] print_ts('write_info:' + url) # save data res = re.match(r'ftp://.*pool/main/[^/]+/([^/]+)/[^/]*tar.(gz|bz2|xz)', url) if res is None: res = re.match(r'https://cppcheck\.sf\.net/([a-z]+).tgz', url) if res is None: print_ts('res is None. Ignoring information data.') continue if url not in packages: print_ts('Url is not in packages. Ignoring information data.') continue print_ts('adding info output for package ' + res.group(1)) info_path = resultPath + '/' + 'info_output' if not os.path.exists(info_path): os.mkdir(info_path) filename = info_path + '/' + res.group(1) with open(filename, 'wt') as f: f.write(strDateTime() + '\n' + data) elif cmd == 'getPackagesCount\n': packages_count = str(len(packages)) connection.send(packages_count.encode('utf-8', 'ignore')) connection.close() print_ts('getPackagesCount: ' + packages_count) continue elif cmd.startswith('getPackageIdx'): request_idx = abs(int(cmd[len('getPackageIdx:'):])) if request_idx < len(packages): pkg = packages[request_idx] connection.send(pkg.encode('utf-8', 'ignore')) connection.close() print_ts('getPackageIdx: ' + pkg) else: connection.close() print_ts('getPackageIdx: index is out of range') continue elif cmd.startswith('write_nodata\nftp://'): data = read_data(connection, cmd, pos_nl, max_data_size=8 * 1024, check_done=False, cmd_name='write_nodata') if data is None: continue pos = data.find('\n') if pos == -1: print_ts('No newline found in data. Ignoring no-data data.') continue if pos < 10: print_ts('Data is less than 10 characters ({}). Ignoring no-data data.'.format(pos)) continue url = data[:pos] startIdx = packageIndex currentIdx = packageIndex while True: if packages[currentIdx] == url: packages[currentIdx] = None print_ts('write_nodata:' + url) with open('packages_nodata.txt', 'at') as f: f.write(url + '\n') break if currentIdx == 0: currentIdx = len(packages) - 1 else: currentIdx -= 1 if currentIdx == startIdx: print_ts('write_nodata:' + url + ' - package not found') break connection.close() else: if pos_nl < 0: print_ts('invalid command: "' + firstLine + '"') else: lines = cmd.split('\n') s = '\\n'.join(lines[:2]) if len(lines) > 2: s += '...' print_ts('invalid command: "' + s + '"') connection.close() if __name__ == "__main__": workPath = '/var/daca@home' if not os.path.isdir(workPath): workPath = os.path.expanduser('~/daca@home') os.chdir(workPath) print_ts('work path: ' + workPath) resultPath = workPath + '/donated-results' if not os.path.isdir(resultPath): print_ts("fatal: result path '{}' is missing".format(resultPath)) sys.exit(1) with open('packages.txt', 'rt') as f: packages = [val.strip() for val in f.readlines()] print_ts('packages: {}'.format(len(packages))) if os.path.isfile('packages_nodata.txt'): with open('packages_nodata.txt', 'rt') as f: packages_nodata = [val.strip() for val in f.readlines()] packages_nodata.sort() print_ts('packages_nodata: {}'.format(len(packages_nodata))) print_ts('removing packages with no files to process'.format(len(packages_nodata))) packages_nodata_clean = [] for pkg_n in packages_nodata: if pkg_n in packages: packages.remove(pkg_n) packages_nodata_clean.append(pkg_n) packages_nodata_diff = len(packages_nodata) - len(packages_nodata_clean) if packages_nodata_diff: with open('packages_nodata.txt', 'wt') as f: for pkg in packages_nodata_clean: f.write(pkg + '\n') print_ts('removed {} packages from packages_nodata.txt'.format(packages_nodata_diff)) print_ts('packages: {}'.format(len(packages))) if len(packages) == 0: print_ts('fatal: there are no packages') sys.exit(1) packageIndex = 0 if os.path.isfile('package-index.txt'): with open('package-index.txt', 'rt') as f: packageIndex = int(f.read()) if packageIndex < 0 or packageIndex >= len(packages): packageIndex = 0 server_address_port = 8000 if '--test' in sys.argv[1:]: server_address_port = 8001 try: server(server_address_port, packages, packageIndex, resultPath) except socket.timeout: print_ts('Timeout!')