cppcheck/tools/donate-cpu-server.py

1228 lines
46 KiB
Python
Raw Normal View History

#!/usr/bin/env python3
# Server for 'donate-cpu.py'
# Runs only under Python 3.
import collections
import glob
import json
import os
import socket
import re
2018-08-24 13:04:25 +02:00
import datetime
import time
import traceback
from threading import Thread
import sys
import urllib.request
import urllib.parse
import urllib.error
import logging
import logging.handlers
import operator
import html as html_lib
2018-08-24 13:04:25 +02:00
# Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/
# Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic
# changes)
2021-02-21 08:47:58 +01:00
SERVER_VERSION = "1.3.16"
OLD_VERSION = '2.4'
2018-12-08 11:39:44 +01:00
# Set up logging
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# Logging to console
handler_stream = logging.StreamHandler()
logger.addHandler(handler_stream)
# Log errors to a rotating file
logfile = sys.path[0]
if logfile:
logfile += '/'
logfile += 'donate-cpu-server.log'
handler_file = logging.handlers.RotatingFileHandler(filename=logfile, maxBytes=100*1024, backupCount=1)
handler_file.setLevel(logging.ERROR)
logger.addHandler(handler_file)
# Set up an exception hook for all uncaught exceptions so they can be logged
def handle_uncaught_exception(exc_type, exc_value, exc_traceback):
if issubclass(exc_type, KeyboardInterrupt):
sys.__excepthook__(exc_type, exc_value, exc_traceback)
return
logging.error("Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback))
sys.excepthook = handle_uncaught_exception
def strDateTime() -> str:
return datetime.datetime.now().strftime('%Y-%m-%d %H:%M')
2018-08-24 13:04:25 +02:00
def dateTimeFromStr(datestr: str) -> datetime.datetime:
return datetime.datetime.strptime(datestr, '%Y-%m-%d %H:%M')
def overviewReport() -> str:
html = '<html><head><title>daca@home</title></head><body>\n'
html += '<h1>daca@home</h1>\n'
html += '<a href="crash.html">Crash report</a><br>\n'
html += '<a href="timeout.html">Timeout report</a><br>\n'
html += '<a href="stale.html">Stale report</a><br>\n'
html += '<a href="diff.html">Diff report</a><br>\n'
html += '<a href="head.html">HEAD report</a><br>\n'
2018-09-02 09:35:38 +02:00
html += '<a href="latest.html">Latest results</a><br>\n'
html += '<a href="time_lt.html">Time report (improved)</a><br>\n'
html += '<a href="time_gt.html">Time report (regressed)</a><br>\n'
2021-02-21 08:47:58 +01:00
html += '<a href="time_slow.html">Time report (slowest)</a><br>\n'
html += '<a href="check_library_function_report.html">checkLibraryFunction report</a><br>\n'
html += '<a href="check_library_noreturn_report.html">checkLibraryNoReturn report</a><br>\n'
html += '<a href="check_library_use_ignore_report.html">checkLibraryUseIgnore report</a><br>\n'
html += '<br>\n'
html += 'version ' + SERVER_VERSION + '\n'
html += '</body></html>'
return html
def fmt(a: str, b: str, c: str = None, d: str = None, e: str = None, link: bool = True, column_width=None) -> str:
if column_width is None:
column_width = [40, 10, 5, 7, 7, 8]
ret = a
while len(ret) < column_width[0]:
ret += ' '
if len(ret) == column_width[0]:
ret += ' ' + b[:10]
while len(ret) < (column_width[0] + 1 + column_width[1]):
ret += ' '
ret += ' '
Donate cpu server crash locations (#2276) donate-cpu-server: Print grouped stack traces in crash report Example output: Packages: psi xenomai Token::hasKnownValue (this=0x0) at lib/token.h:988 988 return mImpl->mValues && std::any_of(mImpl->mValues->begin(), mImpl->mValues->end(), std::mem_fn(&ValueFlow::Value::isKnown)); #0 Token::hasKnownValue (...) at lib/token.h:988 #1 valueFlowReverse (...) at build/valueflow.cpp:3775 #2 valueFlowBeforeCondition (...) at /usr/include/c++/8/bits/stl_list.h:301 #3 ValueFlow::setValues (...) at build/valueflow.cpp:8403 #4 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #5 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #6 CppCheck::check (...) at /usr/include/c++/8/bits/basic_string.h:936 #7 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #8 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #9 main (...) at cli/main.cpp:95 Packages: broker valueFlowReverse (tokenlist=tokenlist@entry=0x7fffffffbce0, tok=tok@entry=0x555555cc9930, varToken=varToken@entry=0x555555cc9b70, val=..., val2=..., errorLogger=errorLogger@entry=0x7fffffffcb20, settings=0x7fffffffcd40) at build/valueflow.cpp:3775 3775 if (!assignTok->hasKnownValue()) { #0 valueFlowReverse (...) at build/valueflow.cpp:3775 #1 valueFlowBeforeCondition (...) at build/valueflow.cpp:4092 #2 ValueFlow::setValues (...) at build/valueflow.cpp:8406 #3 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #4 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #5 CppCheck::check (...) at build/cppcheck.cpp:197 #6 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #7 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #8 main (...) at cli/main.cpp:95
2019-10-18 09:56:15 +02:00
if len(b) > 10:
ret += b[-5:].rjust(column_width[2]) + ' '
if c is not None:
ret += c.rjust(column_width[3]) + ' '
if d is not None:
ret += d.rjust(column_width[4]) + ' '
if e is not None:
ret += e.rjust(column_width[5])
Donate cpu server crash locations (#2276) donate-cpu-server: Print grouped stack traces in crash report Example output: Packages: psi xenomai Token::hasKnownValue (this=0x0) at lib/token.h:988 988 return mImpl->mValues && std::any_of(mImpl->mValues->begin(), mImpl->mValues->end(), std::mem_fn(&ValueFlow::Value::isKnown)); #0 Token::hasKnownValue (...) at lib/token.h:988 #1 valueFlowReverse (...) at build/valueflow.cpp:3775 #2 valueFlowBeforeCondition (...) at /usr/include/c++/8/bits/stl_list.h:301 #3 ValueFlow::setValues (...) at build/valueflow.cpp:8403 #4 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #5 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #6 CppCheck::check (...) at /usr/include/c++/8/bits/basic_string.h:936 #7 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #8 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #9 main (...) at cli/main.cpp:95 Packages: broker valueFlowReverse (tokenlist=tokenlist@entry=0x7fffffffbce0, tok=tok@entry=0x555555cc9930, varToken=varToken@entry=0x555555cc9b70, val=..., val2=..., errorLogger=errorLogger@entry=0x7fffffffcb20, settings=0x7fffffffcd40) at build/valueflow.cpp:3775 3775 if (!assignTok->hasKnownValue()) { #0 valueFlowReverse (...) at build/valueflow.cpp:3775 #1 valueFlowBeforeCondition (...) at build/valueflow.cpp:4092 #2 ValueFlow::setValues (...) at build/valueflow.cpp:8406 #3 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #4 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #5 CppCheck::check (...) at build/cppcheck.cpp:197 #6 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #7 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #8 main (...) at cli/main.cpp:95
2019-10-18 09:56:15 +02:00
if link:
2018-08-25 10:59:49 +02:00
pos = ret.find(' ')
ret = '<a href="' + a + '">' + a + '</a>' + ret[pos:]
return ret
def latestReport(latestResults: list) -> str:
2018-08-26 16:47:20 +02:00
html = '<html><head><title>Latest daca@home results</title></head><body>\n'
html += '<h1>Latest daca@home results</h1>\n'
Donate cpu server crash locations (#2276) donate-cpu-server: Print grouped stack traces in crash report Example output: Packages: psi xenomai Token::hasKnownValue (this=0x0) at lib/token.h:988 988 return mImpl->mValues && std::any_of(mImpl->mValues->begin(), mImpl->mValues->end(), std::mem_fn(&ValueFlow::Value::isKnown)); #0 Token::hasKnownValue (...) at lib/token.h:988 #1 valueFlowReverse (...) at build/valueflow.cpp:3775 #2 valueFlowBeforeCondition (...) at /usr/include/c++/8/bits/stl_list.h:301 #3 ValueFlow::setValues (...) at build/valueflow.cpp:8403 #4 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #5 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #6 CppCheck::check (...) at /usr/include/c++/8/bits/basic_string.h:936 #7 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #8 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #9 main (...) at cli/main.cpp:95 Packages: broker valueFlowReverse (tokenlist=tokenlist@entry=0x7fffffffbce0, tok=tok@entry=0x555555cc9930, varToken=varToken@entry=0x555555cc9b70, val=..., val2=..., errorLogger=errorLogger@entry=0x7fffffffcb20, settings=0x7fffffffcd40) at build/valueflow.cpp:3775 3775 if (!assignTok->hasKnownValue()) { #0 valueFlowReverse (...) at build/valueflow.cpp:3775 #1 valueFlowBeforeCondition (...) at build/valueflow.cpp:4092 #2 ValueFlow::setValues (...) at build/valueflow.cpp:8406 #3 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #4 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #5 CppCheck::check (...) at build/cppcheck.cpp:197 #6 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #7 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #8 main (...) at cli/main.cpp:95
2019-10-18 09:56:15 +02:00
html += '<pre>\n<b>' + fmt('Package', 'Date Time', OLD_VERSION, 'Head', 'Diff', link=False) + '</b>\n'
# Write report for latest results
2018-08-25 09:06:15 +02:00
for filename in latestResults:
if not os.path.isfile(filename):
continue
package = filename[filename.rfind('/')+1:]
current_year = datetime.date.today().year
datestr = ''
count = ['0', '0']
2018-08-26 16:47:20 +02:00
lost = 0
added = 0
for line in open(filename, 'rt'):
line = line.strip()
if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
datestr = line
2018-08-26 16:47:20 +02:00
#elif line.startswith('cppcheck:'):
# cppcheck = line[9:]
elif line.startswith('count: '):
count = line.split(' ')[1:]
elif line.startswith('head ') and not line.startswith('head results:'):
2018-08-26 16:47:20 +02:00
added += 1
2018-12-08 11:39:44 +01:00
elif line.startswith(OLD_VERSION + ' '):
lost += 1
2018-08-26 16:47:20 +02:00
diff = ''
if lost > 0:
diff += '-' + str(lost)
if added > 0:
diff += '+' + str(added)
html += fmt(package, datestr, count[1], count[0], diff) + '\n'
html += '</pre></body></html>\n'
return html
def crashReport(results_path: str) -> str:
2018-08-31 14:28:01 +02:00
html = '<html><head><title>Crash report</title></head><body>\n'
html += '<h1>Crash report</h1>\n'
html += '<pre>\n'
Donate cpu server crash locations (#2276) donate-cpu-server: Print grouped stack traces in crash report Example output: Packages: psi xenomai Token::hasKnownValue (this=0x0) at lib/token.h:988 988 return mImpl->mValues && std::any_of(mImpl->mValues->begin(), mImpl->mValues->end(), std::mem_fn(&ValueFlow::Value::isKnown)); #0 Token::hasKnownValue (...) at lib/token.h:988 #1 valueFlowReverse (...) at build/valueflow.cpp:3775 #2 valueFlowBeforeCondition (...) at /usr/include/c++/8/bits/stl_list.h:301 #3 ValueFlow::setValues (...) at build/valueflow.cpp:8403 #4 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #5 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #6 CppCheck::check (...) at /usr/include/c++/8/bits/basic_string.h:936 #7 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #8 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #9 main (...) at cli/main.cpp:95 Packages: broker valueFlowReverse (tokenlist=tokenlist@entry=0x7fffffffbce0, tok=tok@entry=0x555555cc9930, varToken=varToken@entry=0x555555cc9b70, val=..., val2=..., errorLogger=errorLogger@entry=0x7fffffffcb20, settings=0x7fffffffcd40) at build/valueflow.cpp:3775 3775 if (!assignTok->hasKnownValue()) { #0 valueFlowReverse (...) at build/valueflow.cpp:3775 #1 valueFlowBeforeCondition (...) at build/valueflow.cpp:4092 #2 ValueFlow::setValues (...) at build/valueflow.cpp:8406 #3 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #4 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #5 CppCheck::check (...) at build/cppcheck.cpp:197 #6 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #7 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #8 main (...) at cli/main.cpp:95
2019-10-18 09:56:15 +02:00
html += '<b>' + fmt('Package', 'Date Time', OLD_VERSION, 'Head', link=False) + '</b>\n'
current_year = datetime.date.today().year
Donate cpu server crash locations (#2276) donate-cpu-server: Print grouped stack traces in crash report Example output: Packages: psi xenomai Token::hasKnownValue (this=0x0) at lib/token.h:988 988 return mImpl->mValues && std::any_of(mImpl->mValues->begin(), mImpl->mValues->end(), std::mem_fn(&ValueFlow::Value::isKnown)); #0 Token::hasKnownValue (...) at lib/token.h:988 #1 valueFlowReverse (...) at build/valueflow.cpp:3775 #2 valueFlowBeforeCondition (...) at /usr/include/c++/8/bits/stl_list.h:301 #3 ValueFlow::setValues (...) at build/valueflow.cpp:8403 #4 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #5 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #6 CppCheck::check (...) at /usr/include/c++/8/bits/basic_string.h:936 #7 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #8 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #9 main (...) at cli/main.cpp:95 Packages: broker valueFlowReverse (tokenlist=tokenlist@entry=0x7fffffffbce0, tok=tok@entry=0x555555cc9930, varToken=varToken@entry=0x555555cc9b70, val=..., val2=..., errorLogger=errorLogger@entry=0x7fffffffcb20, settings=0x7fffffffcd40) at build/valueflow.cpp:3775 3775 if (!assignTok->hasKnownValue()) { #0 valueFlowReverse (...) at build/valueflow.cpp:3775 #1 valueFlowBeforeCondition (...) at build/valueflow.cpp:4092 #2 ValueFlow::setValues (...) at build/valueflow.cpp:8406 #3 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #4 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #5 CppCheck::check (...) at build/cppcheck.cpp:197 #6 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #7 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #8 main (...) at cli/main.cpp:95
2019-10-18 09:56:15 +02:00
stack_traces = {}
for filename in sorted(glob.glob(os.path.expanduser(results_path + '/*'))):
2021-02-21 08:47:58 +01:00
if not os.path.isfile(filename) or filename.endswith('.diff'):
2018-08-31 14:28:01 +02:00
continue
datestr = ''
with open(filename, 'rt') as file_:
for line in file_:
Donate cpu server crash locations (#2276) donate-cpu-server: Print grouped stack traces in crash report Example output: Packages: psi xenomai Token::hasKnownValue (this=0x0) at lib/token.h:988 988 return mImpl->mValues && std::any_of(mImpl->mValues->begin(), mImpl->mValues->end(), std::mem_fn(&ValueFlow::Value::isKnown)); #0 Token::hasKnownValue (...) at lib/token.h:988 #1 valueFlowReverse (...) at build/valueflow.cpp:3775 #2 valueFlowBeforeCondition (...) at /usr/include/c++/8/bits/stl_list.h:301 #3 ValueFlow::setValues (...) at build/valueflow.cpp:8403 #4 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #5 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #6 CppCheck::check (...) at /usr/include/c++/8/bits/basic_string.h:936 #7 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #8 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #9 main (...) at cli/main.cpp:95 Packages: broker valueFlowReverse (tokenlist=tokenlist@entry=0x7fffffffbce0, tok=tok@entry=0x555555cc9930, varToken=varToken@entry=0x555555cc9b70, val=..., val2=..., errorLogger=errorLogger@entry=0x7fffffffcb20, settings=0x7fffffffcd40) at build/valueflow.cpp:3775 3775 if (!assignTok->hasKnownValue()) { #0 valueFlowReverse (...) at build/valueflow.cpp:3775 #1 valueFlowBeforeCondition (...) at build/valueflow.cpp:4092 #2 ValueFlow::setValues (...) at build/valueflow.cpp:8406 #3 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #4 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #5 CppCheck::check (...) at build/cppcheck.cpp:197 #6 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #7 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #8 main (...) at cli/main.cpp:95
2019-10-18 09:56:15 +02:00
line = line.strip()
if line.startswith('cppcheck: '):
if OLD_VERSION not in line:
# Package results seem to be too old, skip
break
else:
# Current package, parse on
continue
if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
datestr = line
elif line.startswith('count:'):
Donate cpu server crash locations (#2276) donate-cpu-server: Print grouped stack traces in crash report Example output: Packages: psi xenomai Token::hasKnownValue (this=0x0) at lib/token.h:988 988 return mImpl->mValues && std::any_of(mImpl->mValues->begin(), mImpl->mValues->end(), std::mem_fn(&ValueFlow::Value::isKnown)); #0 Token::hasKnownValue (...) at lib/token.h:988 #1 valueFlowReverse (...) at build/valueflow.cpp:3775 #2 valueFlowBeforeCondition (...) at /usr/include/c++/8/bits/stl_list.h:301 #3 ValueFlow::setValues (...) at build/valueflow.cpp:8403 #4 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #5 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #6 CppCheck::check (...) at /usr/include/c++/8/bits/basic_string.h:936 #7 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #8 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #9 main (...) at cli/main.cpp:95 Packages: broker valueFlowReverse (tokenlist=tokenlist@entry=0x7fffffffbce0, tok=tok@entry=0x555555cc9930, varToken=varToken@entry=0x555555cc9b70, val=..., val2=..., errorLogger=errorLogger@entry=0x7fffffffcb20, settings=0x7fffffffcd40) at build/valueflow.cpp:3775 3775 if (!assignTok->hasKnownValue()) { #0 valueFlowReverse (...) at build/valueflow.cpp:3775 #1 valueFlowBeforeCondition (...) at build/valueflow.cpp:4092 #2 ValueFlow::setValues (...) at build/valueflow.cpp:8406 #3 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #4 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #5 CppCheck::check (...) at build/cppcheck.cpp:197 #6 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #7 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #8 main (...) at cli/main.cpp:95
2019-10-18 09:56:15 +02:00
if line.find('Crash') < 0:
break
package = filename[filename.rfind('/')+1:]
2021-02-21 08:47:58 +01:00
counts = line.split(' ')
Donate cpu server crash locations (#2276) donate-cpu-server: Print grouped stack traces in crash report Example output: Packages: psi xenomai Token::hasKnownValue (this=0x0) at lib/token.h:988 988 return mImpl->mValues && std::any_of(mImpl->mValues->begin(), mImpl->mValues->end(), std::mem_fn(&ValueFlow::Value::isKnown)); #0 Token::hasKnownValue (...) at lib/token.h:988 #1 valueFlowReverse (...) at build/valueflow.cpp:3775 #2 valueFlowBeforeCondition (...) at /usr/include/c++/8/bits/stl_list.h:301 #3 ValueFlow::setValues (...) at build/valueflow.cpp:8403 #4 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #5 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #6 CppCheck::check (...) at /usr/include/c++/8/bits/basic_string.h:936 #7 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #8 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #9 main (...) at cli/main.cpp:95 Packages: broker valueFlowReverse (tokenlist=tokenlist@entry=0x7fffffffbce0, tok=tok@entry=0x555555cc9930, varToken=varToken@entry=0x555555cc9b70, val=..., val2=..., errorLogger=errorLogger@entry=0x7fffffffcb20, settings=0x7fffffffcd40) at build/valueflow.cpp:3775 3775 if (!assignTok->hasKnownValue()) { #0 valueFlowReverse (...) at build/valueflow.cpp:3775 #1 valueFlowBeforeCondition (...) at build/valueflow.cpp:4092 #2 ValueFlow::setValues (...) at build/valueflow.cpp:8406 #3 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #4 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #5 CppCheck::check (...) at build/cppcheck.cpp:197 #6 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #7 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #8 main (...) at cli/main.cpp:95
2019-10-18 09:56:15 +02:00
c2 = ''
if counts[2] == 'Crash!':
c2 = 'Crash'
c1 = ''
if counts[1] == 'Crash!':
c1 = 'Crash'
html += fmt(package, datestr, c2, c1) + '\n'
if c1 != 'Crash':
break
elif line.find(' received signal ') != -1:
crash_line = next(file_, '').strip()
location_index = crash_line.rfind(' at ')
Donate cpu server crash locations (#2276) donate-cpu-server: Print grouped stack traces in crash report Example output: Packages: psi xenomai Token::hasKnownValue (this=0x0) at lib/token.h:988 988 return mImpl->mValues && std::any_of(mImpl->mValues->begin(), mImpl->mValues->end(), std::mem_fn(&ValueFlow::Value::isKnown)); #0 Token::hasKnownValue (...) at lib/token.h:988 #1 valueFlowReverse (...) at build/valueflow.cpp:3775 #2 valueFlowBeforeCondition (...) at /usr/include/c++/8/bits/stl_list.h:301 #3 ValueFlow::setValues (...) at build/valueflow.cpp:8403 #4 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #5 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #6 CppCheck::check (...) at /usr/include/c++/8/bits/basic_string.h:936 #7 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #8 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #9 main (...) at cli/main.cpp:95 Packages: broker valueFlowReverse (tokenlist=tokenlist@entry=0x7fffffffbce0, tok=tok@entry=0x555555cc9930, varToken=varToken@entry=0x555555cc9b70, val=..., val2=..., errorLogger=errorLogger@entry=0x7fffffffcb20, settings=0x7fffffffcd40) at build/valueflow.cpp:3775 3775 if (!assignTok->hasKnownValue()) { #0 valueFlowReverse (...) at build/valueflow.cpp:3775 #1 valueFlowBeforeCondition (...) at build/valueflow.cpp:4092 #2 ValueFlow::setValues (...) at build/valueflow.cpp:8406 #3 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #4 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #5 CppCheck::check (...) at build/cppcheck.cpp:197 #6 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #7 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #8 main (...) at cli/main.cpp:95
2019-10-18 09:56:15 +02:00
if location_index > 0:
code_line = next(file_, '').strip()
else:
code_line = ''
stack_trace = []
while True:
l = next(file_, '')
if not l.strip():
break
# #0 0x00007ffff71cbf67 in raise () from /lib64/libc.so.6
m = re.search(r'(?P<number>#\d+) .* in (?P<function>.+)\(.*\) from (?P<binary>.*)$', l)
if m:
stack_trace.append(m.group('number') + ' ' + m.group('function') + '(...) from ' + m.group('binary'))
continue
# #11 0x00000000006f2414 in valueFlowNumber (tokenlist=tokenlist@entry=0x7fffffffc610) at build/valueflow.cpp:2503
m = re.search(r'(?P<number>#\d+) .* in (?P<function>.+)\(.*\) at (?P<location>.*)$', l)
if m:
stack_trace.append(m.group('number') + ' ' + m.group('function') + '(...) at ' + m.group('location'))
continue
# #18 ForwardTraversal::updateRecursive (this=0x7fffffffb3c0, tok=0x14668a0) at build/forwardanalyzer.cpp:415
m = re.search(r'(?P<number>#\d+) (?P<function>.+)\(.*\) at (?P<location>.*)$', l)
if m:
Donate cpu server crash locations (#2276) donate-cpu-server: Print grouped stack traces in crash report Example output: Packages: psi xenomai Token::hasKnownValue (this=0x0) at lib/token.h:988 988 return mImpl->mValues && std::any_of(mImpl->mValues->begin(), mImpl->mValues->end(), std::mem_fn(&ValueFlow::Value::isKnown)); #0 Token::hasKnownValue (...) at lib/token.h:988 #1 valueFlowReverse (...) at build/valueflow.cpp:3775 #2 valueFlowBeforeCondition (...) at /usr/include/c++/8/bits/stl_list.h:301 #3 ValueFlow::setValues (...) at build/valueflow.cpp:8403 #4 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #5 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #6 CppCheck::check (...) at /usr/include/c++/8/bits/basic_string.h:936 #7 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #8 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #9 main (...) at cli/main.cpp:95 Packages: broker valueFlowReverse (tokenlist=tokenlist@entry=0x7fffffffbce0, tok=tok@entry=0x555555cc9930, varToken=varToken@entry=0x555555cc9b70, val=..., val2=..., errorLogger=errorLogger@entry=0x7fffffffcb20, settings=0x7fffffffcd40) at build/valueflow.cpp:3775 3775 if (!assignTok->hasKnownValue()) { #0 valueFlowReverse (...) at build/valueflow.cpp:3775 #1 valueFlowBeforeCondition (...) at build/valueflow.cpp:4092 #2 ValueFlow::setValues (...) at build/valueflow.cpp:8406 #3 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #4 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #5 CppCheck::check (...) at build/cppcheck.cpp:197 #6 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #7 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #8 main (...) at cli/main.cpp:95
2019-10-18 09:56:15 +02:00
stack_trace.append(m.group('number') + ' ' + m.group('function') + '(...) at ' + m.group('location'))
continue
print('{} - unmatched stack frame - {}'.format(package, l))
break
key = hash(' '.join(stack_trace))
if key in stack_traces:
stack_traces[key]['code_line'] = code_line
stack_traces[key]['stack_trace'] = stack_trace
stack_traces[key]['n'] += 1
stack_traces[key]['packages'].append(package)
else:
stack_traces[key] = {'stack_trace': stack_trace, 'n': 1, 'code_line': code_line, 'packages': [package], 'crash_line': crash_line}
break
Donate cpu server crash locations (#2276) donate-cpu-server: Print grouped stack traces in crash report Example output: Packages: psi xenomai Token::hasKnownValue (this=0x0) at lib/token.h:988 988 return mImpl->mValues && std::any_of(mImpl->mValues->begin(), mImpl->mValues->end(), std::mem_fn(&ValueFlow::Value::isKnown)); #0 Token::hasKnownValue (...) at lib/token.h:988 #1 valueFlowReverse (...) at build/valueflow.cpp:3775 #2 valueFlowBeforeCondition (...) at /usr/include/c++/8/bits/stl_list.h:301 #3 ValueFlow::setValues (...) at build/valueflow.cpp:8403 #4 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #5 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #6 CppCheck::check (...) at /usr/include/c++/8/bits/basic_string.h:936 #7 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #8 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #9 main (...) at cli/main.cpp:95 Packages: broker valueFlowReverse (tokenlist=tokenlist@entry=0x7fffffffbce0, tok=tok@entry=0x555555cc9930, varToken=varToken@entry=0x555555cc9b70, val=..., val2=..., errorLogger=errorLogger@entry=0x7fffffffcb20, settings=0x7fffffffcd40) at build/valueflow.cpp:3775 3775 if (!assignTok->hasKnownValue()) { #0 valueFlowReverse (...) at build/valueflow.cpp:3775 #1 valueFlowBeforeCondition (...) at build/valueflow.cpp:4092 #2 ValueFlow::setValues (...) at build/valueflow.cpp:8406 #3 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #4 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #5 CppCheck::check (...) at build/cppcheck.cpp:197 #6 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #7 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #8 main (...) at cli/main.cpp:95
2019-10-18 09:56:15 +02:00
html += '</pre>\n'
html += '<pre>\n'
html += '<b>Stack traces</b>\n'
for stack_trace in sorted(list(stack_traces.values()), key=lambda x: x['n'], reverse=True):
Donate cpu server crash locations (#2276) donate-cpu-server: Print grouped stack traces in crash report Example output: Packages: psi xenomai Token::hasKnownValue (this=0x0) at lib/token.h:988 988 return mImpl->mValues && std::any_of(mImpl->mValues->begin(), mImpl->mValues->end(), std::mem_fn(&ValueFlow::Value::isKnown)); #0 Token::hasKnownValue (...) at lib/token.h:988 #1 valueFlowReverse (...) at build/valueflow.cpp:3775 #2 valueFlowBeforeCondition (...) at /usr/include/c++/8/bits/stl_list.h:301 #3 ValueFlow::setValues (...) at build/valueflow.cpp:8403 #4 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #5 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #6 CppCheck::check (...) at /usr/include/c++/8/bits/basic_string.h:936 #7 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #8 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #9 main (...) at cli/main.cpp:95 Packages: broker valueFlowReverse (tokenlist=tokenlist@entry=0x7fffffffbce0, tok=tok@entry=0x555555cc9930, varToken=varToken@entry=0x555555cc9b70, val=..., val2=..., errorLogger=errorLogger@entry=0x7fffffffcb20, settings=0x7fffffffcd40) at build/valueflow.cpp:3775 3775 if (!assignTok->hasKnownValue()) { #0 valueFlowReverse (...) at build/valueflow.cpp:3775 #1 valueFlowBeforeCondition (...) at build/valueflow.cpp:4092 #2 ValueFlow::setValues (...) at build/valueflow.cpp:8406 #3 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #4 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #5 CppCheck::check (...) at build/cppcheck.cpp:197 #6 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #7 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #8 main (...) at cli/main.cpp:95
2019-10-18 09:56:15 +02:00
html += 'Packages: ' + ' '.join(['<a href="' + p + '">' + p + '</a>' for p in stack_trace['packages']]) + '\n'
html += html_lib.escape(stack_trace['crash_line']) + '\n'
html += html_lib.escape(stack_trace['code_line']) + '\n'
html += html_lib.escape('\n'.join(stack_trace['stack_trace'])) + '\n\n'
html += '</pre>\n'
html += '</body></html>\n'
return html
def timeoutReport(results_path: str) -> str:
html = '<html><head><title>Timeout report</title></head><body>\n'
html += '<h1>Timeout report</h1>\n'
html += '<pre>\n'
html += '<b>' + fmt('Package', 'Date Time', OLD_VERSION, 'Head', link=False) + '</b>\n'
current_year = datetime.date.today().year
for filename in sorted(glob.glob(os.path.expanduser(results_path + '/*'))):
2021-02-21 08:47:58 +01:00
if not os.path.isfile(filename) or filename.endswith('.diff'):
continue
datestr = ''
with open(filename, 'rt') as file_:
for line in file_:
line = line.strip()
if line.startswith('cppcheck: '):
if OLD_VERSION not in line:
# Package results seem to be too old, skip
break
else:
# Current package, parse on
continue
if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
datestr = line
elif line.startswith('count:'):
if line.find('TO!') < 0:
break
package = filename[filename.rfind('/')+1:]
2021-02-21 08:47:58 +01:00
counts = line.split(' ')
c2 = ''
if counts[2] == 'TO!':
c2 = 'Timeout'
c1 = ''
if counts[1] == 'TO!':
c1 = 'Timeout'
html += fmt(package, datestr, c2, c1) + '\n'
break
html += '</pre>\n'
html += '</body></html>\n'
return html
def staleReport(results_path: str) -> str:
html = '<html><head><title>Stale report</title></head><body>\n'
html += '<h1>Stale report</h1>\n'
html += '<pre>\n'
Donate cpu server crash locations (#2276) donate-cpu-server: Print grouped stack traces in crash report Example output: Packages: psi xenomai Token::hasKnownValue (this=0x0) at lib/token.h:988 988 return mImpl->mValues && std::any_of(mImpl->mValues->begin(), mImpl->mValues->end(), std::mem_fn(&ValueFlow::Value::isKnown)); #0 Token::hasKnownValue (...) at lib/token.h:988 #1 valueFlowReverse (...) at build/valueflow.cpp:3775 #2 valueFlowBeforeCondition (...) at /usr/include/c++/8/bits/stl_list.h:301 #3 ValueFlow::setValues (...) at build/valueflow.cpp:8403 #4 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #5 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #6 CppCheck::check (...) at /usr/include/c++/8/bits/basic_string.h:936 #7 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #8 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #9 main (...) at cli/main.cpp:95 Packages: broker valueFlowReverse (tokenlist=tokenlist@entry=0x7fffffffbce0, tok=tok@entry=0x555555cc9930, varToken=varToken@entry=0x555555cc9b70, val=..., val2=..., errorLogger=errorLogger@entry=0x7fffffffcb20, settings=0x7fffffffcd40) at build/valueflow.cpp:3775 3775 if (!assignTok->hasKnownValue()) { #0 valueFlowReverse (...) at build/valueflow.cpp:3775 #1 valueFlowBeforeCondition (...) at build/valueflow.cpp:4092 #2 ValueFlow::setValues (...) at build/valueflow.cpp:8406 #3 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #4 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #5 CppCheck::check (...) at build/cppcheck.cpp:197 #6 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #7 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #8 main (...) at cli/main.cpp:95
2019-10-18 09:56:15 +02:00
html += '<b>' + fmt('Package', 'Date Time', link=False) + '</b>\n'
current_year = datetime.date.today().year
for filename in sorted(glob.glob(os.path.expanduser(results_path + '/*'))):
2021-02-21 08:47:58 +01:00
if not os.path.isfile(filename) or filename.endswith('.diff'):
continue
for line in open(filename, 'rt'):
line = line.strip()
if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
datestr = line
else:
continue
dt = dateTimeFromStr(datestr)
diff = datetime.datetime.now() - dt
if diff.days < 30:
continue
package = filename[filename.rfind('/')+1:]
Donate cpu server crash locations (#2276) donate-cpu-server: Print grouped stack traces in crash report Example output: Packages: psi xenomai Token::hasKnownValue (this=0x0) at lib/token.h:988 988 return mImpl->mValues && std::any_of(mImpl->mValues->begin(), mImpl->mValues->end(), std::mem_fn(&ValueFlow::Value::isKnown)); #0 Token::hasKnownValue (...) at lib/token.h:988 #1 valueFlowReverse (...) at build/valueflow.cpp:3775 #2 valueFlowBeforeCondition (...) at /usr/include/c++/8/bits/stl_list.h:301 #3 ValueFlow::setValues (...) at build/valueflow.cpp:8403 #4 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #5 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #6 CppCheck::check (...) at /usr/include/c++/8/bits/basic_string.h:936 #7 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #8 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #9 main (...) at cli/main.cpp:95 Packages: broker valueFlowReverse (tokenlist=tokenlist@entry=0x7fffffffbce0, tok=tok@entry=0x555555cc9930, varToken=varToken@entry=0x555555cc9b70, val=..., val2=..., errorLogger=errorLogger@entry=0x7fffffffcb20, settings=0x7fffffffcd40) at build/valueflow.cpp:3775 3775 if (!assignTok->hasKnownValue()) { #0 valueFlowReverse (...) at build/valueflow.cpp:3775 #1 valueFlowBeforeCondition (...) at build/valueflow.cpp:4092 #2 ValueFlow::setValues (...) at build/valueflow.cpp:8406 #3 Tokenizer::simplifyTokens1 (...) at build/tokenize.cpp:11095 #4 CppCheck::checkFile (...) at build/cppcheck.cpp:513 #5 CppCheck::check (...) at build/cppcheck.cpp:197 #6 CppCheckExecutor::check_internal (...) at cli/cppcheckexecutor.cpp:884 #7 CppCheckExecutor::check (...) at cli/cppcheckexecutor.cpp:198 #8 main (...) at cli/main.cpp:95
2019-10-18 09:56:15 +02:00
html += fmt(package, datestr) + '\n'
2018-08-31 14:28:01 +02:00
break
html += '</pre>\n'
2018-11-28 20:36:19 +01:00
2018-08-31 14:28:01 +02:00
html += '</body></html>\n'
return html
def diffReportFromDict(out: dict, today: str) -> str:
html = '<pre>\n'
2018-12-08 11:39:44 +01:00
html += '<b>MessageID ' + OLD_VERSION + ' Head</b>\n'
sum0 = 0
sum1 = 0
for messageId in sorted(out.keys()):
line = messageId + ' '
counts = out[messageId]
sum0 += counts[0]
sum1 += counts[1]
if counts[0] > 0:
c = str(counts[0])
while len(line) < 40 - len(c):
line += ' '
line += c + ' '
if counts[1] > 0:
c = str(counts[1])
while len(line) < 48 - len(c):
line += ' '
line += c
line = '<a href="diff' + today + '-' + messageId + '">' + messageId + '</a>' + line[line.find(' '):]
html += line + '\n'
# Sum
html += '================================================\n'
line = ''
while len(line) < 40 - len(str(sum0)):
line += ' '
line += str(sum0) + ' '
while len(line) < 48 - len(str(sum1)):
line += ' '
line += str(sum1)
html += line + '\n'
html += '</pre>\n'
return html
def diffReport(resultsPath: str) -> str:
out = {}
outToday = {}
today = strDateTime()[:10]
for filename in sorted(glob.glob(resultsPath + '/*.diff')):
if not os.path.isfile(filename):
continue
with open(filename, 'rt') as f:
data = json.loads(f.read())
uploadedToday = data['date'] == today
2019-06-30 14:14:02 +02:00
for messageId in data['sums']:
sums = data['sums'][messageId]
if OLD_VERSION not in sums:
continue
if messageId not in out:
out[messageId] = [0, 0]
out[messageId][0] += sums[OLD_VERSION]
out[messageId][1] += sums['head']
if uploadedToday:
if messageId not in outToday:
outToday[messageId] = [0, 0]
outToday[messageId][0] += sums[OLD_VERSION]
outToday[messageId][1] += sums['head']
html = '<html><head><title>Diff report</title></head><body>\n'
html += '<h1>Diff report</h1>\n'
html += '<h2>Uploaded today</h2>'
html += diffReportFromDict(outToday, 'today')
html += '<h2>All</h2>'
html += diffReportFromDict(out, '')
return html
def generate_package_diff_statistics(filename: str) -> None:
is_diff = False
sums = {}
for line in open(filename, 'rt'):
line = line.strip()
if line == 'diff:':
is_diff = True
continue
elif not is_diff:
continue
if not line.endswith(']'):
continue
if line.startswith(OLD_VERSION + ' '):
version = OLD_VERSION
elif line.startswith('head '):
version = 'head'
else:
continue
messageId = line[line.rfind('[')+1:len(line)-1]
if messageId not in sums:
sums[messageId] = {OLD_VERSION: 0, 'head': 0}
sums[messageId][version] += 1
output = {'date': strDateTime()[:10], 'sums': sums}
filename_diff = filename + '.diff'
if sums:
with open(filename_diff, 'wt') as f:
f.write(json.dumps(output))
elif os.path.isfile(filename_diff):
os.remove(filename_diff)
def diffMessageIdReport(resultPath: str, messageId: str) -> str:
text = messageId + '\n'
e = '[' + messageId + ']\n'
for filename in sorted(glob.glob(resultPath + '/*.diff')):
if not os.path.isfile(filename):
continue
with open(filename, 'rt') as f:
diff_stats = f.read()
if messageId not in diff_stats:
continue
url = None
diff = False
for line in open(filename[:-5], 'rt'):
if line.startswith('ftp://'):
url = line
elif line == 'diff:\n':
diff = True
elif not diff:
continue
elif line.endswith(e):
if url:
text += url
url = None
text += line
return text
def diffMessageIdTodayReport(resultPath: str, messageId: str) -> str:
text = messageId + '\n'
e = '[' + messageId + ']\n'
today = strDateTime()[:10]
for filename in sorted(glob.glob(resultPath + '/*.diff')):
if not os.path.isfile(filename):
continue
with open(filename, 'rt') as f:
diff_stats = f.read()
if messageId not in diff_stats:
continue
if today not in diff_stats:
continue
url = None
diff = False
firstLine = True
for line in open(filename[:-5], 'rt'):
if firstLine:
firstLine = False
if not line.startswith(today):
break
if line.startswith('ftp://'):
url = line
elif line == 'diff:\n':
diff = True
elif not diff:
continue
elif line.endswith(e):
if url:
text += url
url = None
text += line
return text
def headReportFromDict(out: dict, today: str) -> str:
html = '<pre>\n'
html += '<b>MessageID Count</b>\n'
sumTotal = 0
for messageId in sorted(out.keys()):
line = messageId + ' '
counts = out[messageId]
sumTotal += counts
if counts > 0:
c = str(counts)
while len(line) < 48 - len(c):
line += ' '
line += c + ' '
line = '<a href="head' + today + '-' + messageId + '">' + messageId + '</a>' + line[line.find(' '):]
html += line + '\n'
# Sum
html += '================================================\n'
line = ''
while len(line) < 48 - len(str(sumTotal)):
line += ' '
line += str(sumTotal) + ' '
html += line + '\n'
html += '</pre>\n'
return html
def headReport(resultsPath: str) -> str:
out = {}
outToday = {}
today = strDateTime()[:10]
for filename in sorted(glob.glob(resultsPath + '/*')):
2021-02-21 08:47:58 +01:00
if not os.path.isfile(filename) or filename.endswith('.diff'):
continue
uploadedToday = False
firstLine = True
headResults = False
for line in open(filename, 'rt'):
if firstLine:
if line.startswith(today):
uploadedToday = True
firstLine = False
continue
line = line.strip()
if line.startswith('cppcheck: '):
if OLD_VERSION not in line:
# Package results seem to be too old, skip
break
else:
# Current package, parse on
continue
if line.startswith('head results:'):
headResults = True
continue
if line.startswith('diff:'):
if headResults:
break
if not headResults:
continue
if not line.endswith(']'):
continue
if ': note: ' in line:
# notes normally do not contain message ids but can end with ']'
continue
message_id_start_pos = line.rfind('[')
if message_id_start_pos <= 0:
continue
messageId = line[message_id_start_pos+1:len(line)-1]
if ' ' in messageId:
# skip invalid messageIds
continue
if messageId not in out:
out[messageId] = 0
out[messageId] += 1
if uploadedToday:
if messageId not in outToday:
outToday[messageId] = 0
outToday[messageId] += 1
html = '<html><head><title>HEAD report</title></head><body>\n'
html += '<h1>HEAD report</h1>\n'
html += '<h2>Uploaded today</h2>'
html += headReportFromDict(outToday, 'today')
html += '<h2>All</h2>'
html += headReportFromDict(out, '')
return html
def headMessageIdReport(resultPath: str, messageId: str) -> str:
text = messageId + '\n'
e = '[' + messageId + ']\n'
for filename in sorted(glob.glob(resultPath + '/*')):
2021-02-21 08:47:58 +01:00
if not os.path.isfile(filename) or filename.endswith('.diff'):
continue
url = None
headResults = False
for line in open(filename, 'rt'):
if line.startswith('ftp://'):
url = line
elif line.startswith('head results:'):
headResults = True
elif not headResults:
continue
elif headResults and line.startswith('diff:'):
break
elif line.endswith(e):
if url:
text += url
url = None
text += line
return text
def headMessageIdTodayReport(resultPath: str, messageId: str) -> str:
text = messageId + '\n'
e = '[' + messageId + ']\n'
today = strDateTime()[:10]
for filename in sorted(glob.glob(resultPath + '/*')):
2021-02-21 08:47:58 +01:00
if not os.path.isfile(filename) or filename.endswith('.diff'):
continue
url = None
headResults = False
firstLine = True
for line in open(filename, 'rt'):
if firstLine:
firstLine = False
if not line.startswith(today):
break
if line.startswith('ftp://'):
url = line
elif line.startswith('head results:'):
headResults = True
elif not headResults:
continue
elif headResults and line.startswith('diff:'):
break
elif line.endswith(e):
if url:
text += url
url = None
text += line
return text
def timeReport(resultPath: str, show_gt: bool) -> str:
title = 'Time report ({})'.format('regressed' if show_gt else 'improved')
html = '<html><head><title>{}</title></head><body>\n'.format(title)
html += '<h1>{}</h1>\n'.format(title)
html += '<pre>\n'
column_width = [40, 10, 10, 10, 10, 10]
html += '<b>'
html += fmt('Package', 'Date Time', OLD_VERSION, 'Head', 'Factor', link=False, column_width=column_width)
html += '</b>\n'
current_year = datetime.date.today().year
data = {}
total_time_base = 0.0
total_time_head = 0.0
2018-09-02 09:35:38 +02:00
for filename in glob.glob(resultPath + '/*'):
2021-02-21 08:47:58 +01:00
if not os.path.isfile(filename) or filename.endswith('.diff'):
continue
datestr = ''
for line in open(filename, 'rt'):
2021-02-21 08:47:58 +01:00
line = line.strip()
if line.startswith('cppcheck: '):
if OLD_VERSION not in line:
# Package results seem to be too old, skip
break
else:
# Current package, parse on
continue
if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
datestr = line
continue
2018-09-02 09:35:38 +02:00
if not line.startswith('elapsed-time:'):
continue
2021-02-21 08:47:58 +01:00
split_line = line.split()
time_base = float(split_line[2])
time_head = float(split_line[1])
if time_base < 0.0 or time_head < 0.0:
# ignore results with crashes / errors for the time report
break
total_time_base += time_base
total_time_head += time_head
suspicious_time_difference = False
if show_gt and time_base > 1 and time_base*2 < time_head:
suspicious_time_difference = True
elif not show_gt and time_head > 1 and time_head*2 < time_base:
suspicious_time_difference = True
if suspicious_time_difference:
if time_base > 0.0:
time_factor = time_head / time_base
else:
time_factor = 0.0
pkg_name = filename[len(resultPath)+1:]
data[pkg_name] = (datestr, split_line[2], split_line[1], time_factor)
2018-09-02 09:35:38 +02:00
break
2021-02-21 08:47:58 +01:00
sorted_data = sorted(data.items(), key=lambda kv: kv[1][3], reverse=show_gt)
sorted_dict = collections.OrderedDict(sorted_data)
for key in sorted_dict:
html += fmt(key, sorted_dict[key][0], sorted_dict[key][1], sorted_dict[key][2], '{:.2f}'.format(sorted_dict[key][3]),
column_width=column_width) + '\n'
html += '\n'
html += '(listed above are all suspicious timings with a factor '
if show_gt:
html += '&gt;2.00'
else:
2021-02-21 08:47:58 +01:00
html += '&lt;=0.50'
html += ')\n'
html += '\n'
if total_time_base > 0.0:
total_time_factor = total_time_head / total_time_base
else:
total_time_factor = 0.0
html += 'Time for all packages (not just the ones listed above):\n'
html += fmt('Total time:',
'',
'{:.1f}'.format(total_time_base),
'{:.1f}'.format(total_time_head),
'{:.2f}'.format(total_time_factor), link=False, column_width=column_width)
html += '\n'
html += '</pre>\n'
html += '</body></html>\n'
return html
2018-09-02 09:35:38 +02:00
2021-02-21 08:47:58 +01:00
def timeReportSlow(resultPath: str) -> str:
title = 'Time report (slowest)'
html = '<html><head><title>{}</title></head><body>\n'.format(title)
html += '<h1>{}</h1>\n'.format(title)
html += '<pre>\n'
html += '<b>'
html += fmt('Package', 'Date Time', OLD_VERSION, 'Head', link=False)
html += '</b>\n'
current_year = datetime.date.today().year
data = {}
for filename in glob.glob(resultPath + '/*'):
if not os.path.isfile(filename) or filename.endswith('.diff'):
continue
datestr = ''
for line in open(filename, 'rt'):
line = line.strip()
if line.startswith('cppcheck: '):
if OLD_VERSION not in line:
# Package results seem to be too old, skip
break
else:
# Current package, parse on
continue
if line.startswith(str(current_year) + '-') or line.startswith(str(current_year - 1) + '-'):
datestr = line
continue
elif line.startswith('count:'):
count_head = line.split()[1]
if count_head == 'TO!':
# ignore results with timeouts
break
continue
if not line.startswith('elapsed-time:'):
continue
split_line = line.split()
time_base = float(split_line[2])
time_head = float(split_line[1])
if time_base < 0.0 or time_head < 0.0:
# ignore results with crashes / errors
break
pkg_name = filename[len(resultPath)+1:]
data[pkg_name] = (datestr, split_line[2], split_line[1], time_head)
break
sorted_data = sorted(data.items(), key=lambda kv: kv[1][3])
if len(data) > 100:
first_key, _ = sorted_data[0]
# remove the entry with the lowest run-time
del data[first_key]
sorted_data = sorted(data.items(), key=lambda kv: kv[1][3], reverse=True)
sorted_dict = collections.OrderedDict(sorted_data)
for key in sorted_dict:
html += fmt(key, sorted_dict[key][0], sorted_dict[key][1], sorted_dict[key][2]) + '\n'
html += '</pre>\n'
html += '</body></html>\n'
return html
def check_library_report(result_path: str, message_id: str) -> str:
if message_id not in ('checkLibraryNoReturn', 'checkLibraryFunction', 'checkLibraryUseIgnore'):
error_message = 'Invalid value ' + message_id + ' for message_id parameter.'
print(error_message)
return error_message
functions_shown_max = 50000
html = '<html><head><title>' + message_id + ' report</title></head><body>\n'
html += '<h1>' + message_id + ' report</h1>\n'
html += 'Top ' + str(functions_shown_max) + ' functions are shown.'
html += '<pre>\n'
column_widths = [10, 100]
html += '<b>'
html += 'Count'.rjust(column_widths[0]) + ' ' + \
'Function'
html += '</b>\n'
function_counts = {}
for filename in glob.glob(result_path + '/*'):
2021-02-21 08:47:58 +01:00
if not os.path.isfile(filename) or filename.endswith('.diff'):
continue
info_messages = False
for line in open(filename, 'rt'):
if line.startswith('cppcheck: '):
if OLD_VERSION not in line:
# Package results seem to be too old, skip
break
else:
# Current package, parse on
continue
if line == 'info messages:\n':
info_messages = True
if not info_messages:
continue
if line.endswith('[' + message_id + ']\n'):
if message_id == 'checkLibraryFunction':
function_name = line[(line.find('for function ') + len('for function ')):line.rfind('[') - 1]
else:
function_name = line[(line.find(': Function ') + len(': Function ')):line.rfind('should have') - 1]
function_counts[function_name] = function_counts.setdefault(function_name, 0) + 1
function_details_list = []
for function_name, count in sorted(list(function_counts.items()), key=operator.itemgetter(1), reverse=True):
if len(function_details_list) >= functions_shown_max:
break
function_details_list.append(str(count).rjust(column_widths[0]) + ' ' +
'<a href="check_library-' + urllib.parse.quote_plus(function_name) + '">' + function_name + '</a>\n')
html += ''.join(function_details_list)
html += '</pre>\n'
html += '</body></html>\n'
return html
# Lists all checkLibrary* messages regarding the given function name
def check_library_function_name(result_path: str, function_name: str) -> str:
print('check_library_function_name')
function_name = urllib.parse.unquote_plus(function_name)
output_lines_list = []
for filename in glob.glob(result_path + '/*'):
2021-02-21 08:47:58 +01:00
if not os.path.isfile(filename) or filename.endswith('.diff'):
continue
info_messages = False
url = None
cppcheck_options = None
for line in open(filename, 'rt'):
if line.startswith('ftp://'):
url = line
elif line.startswith('cppcheck-options:'):
cppcheck_options = line
elif line == 'info messages:\n':
info_messages = True
if not info_messages:
continue
if '[checkLibrary' in line:
if (' ' + function_name) in line:
if url:
output_lines_list.append(url)
url = None
if cppcheck_options:
output_lines_list.append(cppcheck_options)
cppcheck_options = None
output_lines_list.append(line)
return ''.join(output_lines_list)
def sendAll(connection: socket.socket, text: str) -> None:
data = text.encode('utf-8', 'ignore')
2018-08-25 10:25:05 +02:00
while data:
num = connection.send(data)
if num < len(data):
data = data[num:]
2018-08-25 10:25:05 +02:00
else:
data = None
def httpGetResponse(connection: socket.socket, data: str, contentType: str) -> None:
2018-08-26 11:17:18 +02:00
resp = 'HTTP/1.1 200 OK\r\n'
resp += 'Connection: close\r\n'
resp += 'Content-length: ' + str(len(data)) + '\r\n'
resp += 'Content-type: ' + contentType + '\r\n\r\n'
2018-08-25 10:59:49 +02:00
resp += data
sendAll(connection, resp)
class HttpClientThread(Thread):
def __init__(self, connection: socket.socket, cmd: str, resultPath: str, latestResults: list) -> None:
Thread.__init__(self)
self.connection = connection
self.cmd = cmd[:cmd.find('\n')]
self.resultPath = resultPath
self.latestResults = latestResults
def run(self):
try:
cmd = self.cmd
print('[' + strDateTime() + '] ' + cmd)
res = re.match(r'GET /([a-zA-Z0-9_\-\.\+%]*) HTTP', cmd)
if res is None:
self.connection.close()
return
url = res.group(1)
if url == '':
html = overviewReport()
httpGetResponse(self.connection, html, 'text/html')
elif url == 'latest.html':
html = latestReport(self.latestResults)
httpGetResponse(self.connection, html, 'text/html')
elif url == 'crash.html':
html = crashReport(self.resultPath)
2018-08-31 14:28:01 +02:00
httpGetResponse(self.connection, html, 'text/html')
elif url == 'timeout.html':
html = timeoutReport(self.resultPath)
httpGetResponse(self.connection, html, 'text/html')
elif url == 'stale.html':
html = staleReport(self.resultPath)
httpGetResponse(self.connection, html, 'text/html')
elif url == 'diff.html':
html = diffReport(self.resultPath)
httpGetResponse(self.connection, html, 'text/html')
elif url.startswith('difftoday-'):
messageId = url[10:]
text = diffMessageIdTodayReport(self.resultPath, messageId)
httpGetResponse(self.connection, text, 'text/plain')
elif url.startswith('diff-'):
messageId = url[5:]
2018-08-27 18:44:17 +02:00
text = diffMessageIdReport(self.resultPath, messageId)
httpGetResponse(self.connection, text, 'text/plain')
elif url == 'head.html':
html = headReport(self.resultPath)
httpGetResponse(self.connection, html, 'text/html')
elif url.startswith('headtoday-'):
messageId = url[10:]
text = headMessageIdTodayReport(self.resultPath, messageId)
httpGetResponse(self.connection, text, 'text/plain')
elif url.startswith('head-'):
messageId = url[5:]
text = headMessageIdReport(self.resultPath, messageId)
httpGetResponse(self.connection, text, 'text/plain')
elif url == 'time_lt.html':
text = timeReport(self.resultPath, False)
httpGetResponse(self.connection, text, 'text/html')
elif url == 'time_gt.html':
text = timeReport(self.resultPath, True)
httpGetResponse(self.connection, text, 'text/html')
2021-02-21 08:47:58 +01:00
elif url == 'time_slow.html':
text = timeReportSlow(self.resultPath)
httpGetResponse(self.connection, text, 'text/html')
elif url == 'check_library_function_report.html':
text = check_library_report(self.resultPath + '/' + 'info_output', message_id='checkLibraryFunction')
httpGetResponse(self.connection, text, 'text/html')
elif url == 'check_library_noreturn_report.html':
text = check_library_report(self.resultPath + '/' + 'info_output', message_id='checkLibraryNoReturn')
httpGetResponse(self.connection, text, 'text/html')
elif url == 'check_library_use_ignore_report.html':
text = check_library_report(self.resultPath + '/' + 'info_output', message_id='checkLibraryUseIgnore')
httpGetResponse(self.connection, text, 'text/html')
elif url.startswith('check_library-'):
print('check library function !')
function_name = url[len('check_library-'):]
text = check_library_function_name(self.resultPath + '/' + 'info_output', function_name)
httpGetResponse(self.connection, text, 'text/plain')
else:
filename = resultPath + '/' + url
if not os.path.isfile(filename):
print('HTTP/1.1 404 Not Found')
self.connection.send(b'HTTP/1.1 404 Not Found\r\n\r\n')
else:
f = open(filename, 'rt')
data = f.read()
f.close()
httpGetResponse(self.connection, data, 'text/plain')
except:
tb = "".join(traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]))
print(tb)
httpGetResponse(self.connection, tb, 'text/plain')
2018-08-26 11:17:18 +02:00
finally:
time.sleep(1)
self.connection.close()
def server(server_address_port: int, packages: list, packageIndex: int, resultPath: str) -> None:
socket.setdefaulttimeout(30)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_address = ('', server_address_port)
sock.bind(server_address)
sock.listen(1)
latestResults = []
if os.path.isfile('latest.txt'):
with open('latest.txt', 'rt') as f:
latestResults = f.read().strip().split(' ')
print('[' + strDateTime() + '] version ' + SERVER_VERSION)
print('[' + strDateTime() + '] listening on port ' + str(server_address_port))
while True:
# wait for a connection
print('[' + strDateTime() + '] waiting for a connection')
connection, client_address = sock.accept()
2018-08-25 20:00:04 +02:00
try:
bytes_received = connection.recv(128)
cmd = bytes_received.decode('utf-8', 'ignore')
2018-08-25 20:43:20 +02:00
except socket.error:
connection.close()
2018-08-25 20:00:04 +02:00
continue
except UnicodeDecodeError as e:
connection.close()
print('Error: Decoding failed: ' + str(e))
continue
if cmd.find('\n') < 1:
continue
firstLine = cmd[:cmd.find('\n')]
if re.match('[a-zA-Z0-9./ ]+', firstLine) is None:
connection.close()
continue
if cmd.startswith('GET /'):
newThread = HttpClientThread(connection, cmd, resultPath, latestResults)
newThread.start()
elif cmd == 'GetCppcheckVersions\n':
2018-12-08 11:39:44 +01:00
reply = 'head ' + OLD_VERSION
print('[' + strDateTime() + '] GetCppcheckVersions: ' + reply)
connection.send(reply.encode('utf-8', 'ignore'))
connection.close()
elif cmd == 'get\n':
pkg = packages[packageIndex]
packageIndex += 1
if packageIndex >= len(packages):
packageIndex = 0
f = open('package-index.txt', 'wt')
f.write(str(packageIndex) + '\n')
f.close()
print('[' + strDateTime() + '] get:' + pkg)
connection.send(pkg.encode('utf-8', 'ignore'))
connection.close()
elif cmd.startswith('write\nftp://') or cmd.startswith('write\nhttp://'):
# read data
data = cmd[cmd.find('\n')+1:]
try:
t = 0.0
max_data_size = 2 * 1024 * 1024
while (len(data) < max_data_size) and (not data.endswith('\nDONE')) and (t < 10):
bytes_received = connection.recv(1024)
if bytes_received:
try:
text_received = bytes_received.decode('utf-8', 'ignore')
except UnicodeDecodeError as e:
print('Error: Decoding failed: ' + str(e))
data = ''
break
t = 0.0
data += text_received
else:
time.sleep(0.2)
t += 0.2
connection.close()
except socket.error:
pass
pos = data.find('\n')
if pos < 10:
continue
url = data[:pos]
2018-08-26 16:47:20 +02:00
print('[' + strDateTime() + '] write:' + url)
# save data
res = re.match(r'ftp://.*pool/main/[^/]+/([^/]+)/[^/]*tar.(gz|bz2|xz)', url)
if res is None:
res = re.match(r'http://cppcheck.sf.net/([a-z]+).tgz', url)
if res is None:
print('results not written. res is None.')
continue
if url not in packages:
print('results not written. url is not in packages.')
continue
# Verify that head was compared to correct OLD_VERSION
versions_found = False
old_version_wrong = False
for line in data.split('\n', 20):
if line.startswith('cppcheck: '):
versions_found = True
if OLD_VERSION not in line.split():
print('Compared to wrong old version. Should be ' + OLD_VERSION + '. Versions compared: ' +
line)
print('Ignoring data.')
old_version_wrong = True
break
if not versions_found:
print('Cppcheck versions missing in result data. Ignoring data.')
continue
if old_version_wrong:
continue
print('results added for package ' + res.group(1))
filename = os.path.join(resultPath, res.group(1))
with open(filename, 'wt') as f:
f.write(strDateTime() + '\n' + data)
# track latest added results..
if len(latestResults) >= 20:
latestResults = latestResults[1:]
latestResults.append(filename)
with open('latest.txt', 'wt') as f:
f.write(' '.join(latestResults))
# generate package.diff..
generate_package_diff_statistics(filename)
elif cmd.startswith('write_info\nftp://') or cmd.startswith('write_info\nhttp://'):
# read data
data = cmd[cmd.find('\n') + 1:]
try:
t = 0.0
max_data_size = 1024 * 1024
while (len(data) < max_data_size) and (not data.endswith('\nDONE')) and (t < 10):
bytes_received = connection.recv(1024)
if bytes_received:
try:
text_received = bytes_received.decode('utf-8', 'ignore')
except UnicodeDecodeError as e:
print('Error: Decoding failed: ' + str(e))
data = ''
break
t = 0.0
data += text_received
else:
time.sleep(0.2)
t += 0.2
connection.close()
except socket.error:
pass
pos = data.find('\n')
if pos < 10:
continue
url = data[:pos]
print('[' + strDateTime() + '] write_info:' + url)
# save data
res = re.match(r'ftp://.*pool/main/[^/]+/([^/]+)/[^/]*tar.(gz|bz2|xz)', url)
if res is None:
res = re.match(r'http://cppcheck.sf.net/([a-z]+).tgz', url)
if res is None:
print('info output not written. res is None.')
continue
if url not in packages:
print('info output not written. url is not in packages.')
continue
print('adding info output for package ' + res.group(1))
info_path = resultPath + '/' + 'info_output'
if not os.path.exists(info_path):
os.mkdir(info_path)
filename = info_path + '/' + res.group(1)
with open(filename, 'wt') as f:
f.write(strDateTime() + '\n' + data)
elif cmd == 'getPackagesCount\n':
packages_count = str(len(packages))
connection.send(packages_count.encode('utf-8', 'ignore'))
connection.close()
print('[' + strDateTime() + '] getPackagesCount: ' + packages_count)
continue
elif cmd.startswith('getPackageIdx'):
request_idx = abs(int(cmd[len('getPackageIdx:'):]))
if request_idx < len(packages):
pkg = packages[request_idx]
connection.send(pkg.encode('utf-8', 'ignore'))
connection.close()
print('[' + strDateTime() + '] getPackageIdx: ' + pkg)
else:
connection.close()
print('[' + strDateTime() + '] getPackageIdx: index is out of range')
continue
2018-08-23 22:13:53 +02:00
else:
if cmd.find('\n') < 0:
print('[' + strDateTime() + '] invalid command: "' + firstLine + '"')
else:
lines = cmd.split('\n')
s = '\\n'.join(lines[:2])
if len(lines) > 2:
s += '...'
print('[' + strDateTime() + '] invalid command: "' + s + '"')
connection.close()
if __name__ == "__main__":
workPath = '/var/daca@home'
if not os.path.isdir(workPath):
workPath = os.path.expanduser('~/daca@home')
os.chdir(workPath)
print('work path: ' + workPath)
resultPath = workPath + '/donated-results'
f = open('packages.txt', 'rt')
packages = [val.strip() for val in f.readlines()]
f.close()
print('packages: ' + str(len(packages)))
if len(packages) == 0:
print('fatal: there are no packages')
sys.exit(1)
packageIndex = 0
if os.path.isfile('package-index.txt'):
f = open('package-index.txt', 'rt')
packageIndex = int(f.read())
if packageIndex < 0 or packageIndex >= len(packages):
packageIndex = 0
f.close()
server_address_port = 8000
if '--test' in sys.argv[1:]:
server_address_port = 8001
try:
server(server_address_port, packages, packageIndex, resultPath)
except socket.timeout:
print('Timeout!')