walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Donate CPU: Collect information messages and provide checkLibrary reports (#1610) 

Related trac ticket: https://trac.cppcheck.net/ticket/8947
Enable information messages and "--check-library" in the Cppcheck
parameters.
Store the information messages and the rest of the messages in different
variables and upload them separately.
The server stores the information messages in a sub-directory similarly
to the normal issue messages in one file per package.
Reports for "checkLibraryFunction" and "checkLibraryNoReturn" message ids
are generated by the server now.
This commit is contained in:
Sebastian 2019-01-22 15:27:13 +01:00 committed by GitHub
parent 6b61866d59
commit a410cea59a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 193 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
tools/donate-cpu-server.py
View File

@ -9,6 +9,7 @@ import datetime
import time import time
from threading import Thread from threading import Thread
import sys import sys
import urllib
OLD_VERSION = '1.86' OLD_VERSION = '1.86'
@ -27,6 +28,8 @@ def overviewReport():
html += '<a href="head">HEAD report</a><br>\n' html += '<a href="head">HEAD report</a><br>\n'
html += '<a href="latest.html">Latest results</a><br>\n' html += '<a href="latest.html">Latest results</a><br>\n'
html += '<a href="time.html">Time report</a><br>\n' html += '<a href="time.html">Time report</a><br>\n'
html += '<a href="check_library_function_report.html">checkLibraryFunction report</a><br>\n'
html += '<a href="check_library_noreturn_report.html">checkLibraryNoReturn report</a><br>\n'
html += '</body></html>' html += '</body></html>'
return html return html
@ -211,6 +214,8 @@ def diffMessageIdReport(resultPath, messageId):
text = messageId + '\n' text = messageId + '\n'
e = '[' + messageId + ']\n' e = '[' + messageId + ']\n'
for filename in sorted(glob.glob(resultPath + '/*')): for filename in sorted(glob.glob(resultPath + '/*')):
if not os.path.isfile(filename):
continue
url = None url = None
diff = False diff = False
for line in open(filename, 'rt'): for line in open(filename, 'rt'):
@ -233,6 +238,8 @@ def diffMessageIdTodayReport(resultPath, messageId):
e = '[' + messageId + ']\n' e = '[' + messageId + ']\n'
today = strDateTime()[:10] today = strDateTime()[:10]
for filename in sorted(glob.glob(resultPath + '/*')): for filename in sorted(glob.glob(resultPath + '/*')):
if not os.path.isfile(filename):
continue
url = None url = None
diff = False diff = False
firstLine = True firstLine = True
@ -338,6 +345,8 @@ def headMessageIdReport(resultPath, messageId):
text = messageId + '\n' text = messageId + '\n'
e = '[' + messageId + ']\n' e = '[' + messageId + ']\n'
for filename in sorted(glob.glob(resultPath + '/*')): for filename in sorted(glob.glob(resultPath + '/*')):
if not os.path.isfile(filename):
continue
url = None url = None
headResults = False headResults = False
for line in open(filename, 'rt'): for line in open(filename, 'rt'):
@ -362,6 +371,8 @@ def headMessageIdTodayReport(resultPath, messageId):
e = '[' + messageId + ']\n' e = '[' + messageId + ']\n'
today = strDateTime()[:10] today = strDateTime()[:10]
for filename in sorted(glob.glob(resultPath + '/*')): for filename in sorted(glob.glob(resultPath + '/*')):
if not os.path.isfile(filename):
continue
url = None url = None
headResults = False headResults = False
firstLine = True firstLine = True
@ -401,6 +412,8 @@ def timeReport(resultPath):
total_time_base = 0.0 total_time_base = 0.0
total_time_head = 0.0 total_time_head = 0.0
for filename in glob.glob(resultPath + '/*'): for filename in glob.glob(resultPath + '/*'):
if not os.path.isfile(filename):
continue
for line in open(filename, 'rt'): for line in open(filename, 'rt'):
if not line.startswith('elapsed-time:'): if not line.startswith('elapsed-time:'):
continue continue
@ -442,6 +455,83 @@ def timeReport(resultPath):
return html return html
def check_library_report(result_path, message_id):
if message_id not in ('checkLibraryNoReturn', 'checkLibraryFunction'):
error_message = 'Invalid value ' + message_id + ' for message_id parameter.'
print(error_message)
return error_message
html = '<html><head><title>' + message_id + ' report</title></head><body>\n'
html += '<h1>' + message_id + ' report</h1>\n'
html += '<pre>\n'
column_widths = [10, 100]
html += '<b>'
html += 'Count'.rjust(column_widths[0]) + ' ' + \
'Function'
html += '</b>\n'
function_counts = dict()
for filename in glob.glob(result_path + '/*'):
if not os.path.isfile(filename):
continue
info_messages = False
for line in open(filename, 'rt'):
if line == 'info messages:\n':
info_messages = True
if not info_messages:
continue
if line.endswith('[' + message_id + ']\n'):
if message_id is 'checkLibraryNoReturn':
function_name = line[(line.find(': Function ') + len(': Function ')):line.rfind('should have') - 1]
else:
function_name = line[(line.find('for function ') + len('for function ')):line.rfind('[') - 1]
function_counts[function_name] = function_counts.setdefault(function_name, 0) + 1
for function_name, count in sorted(function_counts.iteritems(), key=lambda (k, v): (v, k), reverse=True):
if count < 10:
break
html += str(count).rjust(column_widths[0]) + ' ' + \
'<a href="check_library-' + urllib.quote_plus(function_name) + '">' + function_name + '</a>\n'
html += '\n'
html += '</pre>\n'
html += '</body></html>\n'
return html
# Lists all checkLibrary* messages regarding the given function name
def check_library_function_name(result_path, function_name):
print('check_library_function_name')
text = ''
function_name = urllib.unquote_plus(function_name)
for filename in glob.glob(result_path + '/*'):
if not os.path.isfile(filename):
continue
info_messages = False
url = None
cppcheck_options = None
for line in open(filename, 'rt'):
if line.startswith('ftp://'):
url = line
elif line.startswith('cppcheck-options:'):
cppcheck_options = line
elif line == 'info messages:\n':
info_messages = True
if not info_messages:
continue
if '[checkLibrary' in line:
if (' ' + function_name) in line:
if url:
text += url
url = None
if cppcheck_options:
text += cppcheck_options
cppcheck_options = None
text += line
return text
def sendAll(connection, data): def sendAll(connection, data):
while data: while data:
num = connection.send(data) num = connection.send(data)
@ -472,7 +562,7 @@ class HttpClientThread(Thread):
try: try:
cmd = self.cmd cmd = self.cmd
print('[' + strDateTime() + '] ' + cmd) print('[' + strDateTime() + '] ' + cmd)
res = re.match(r'GET /([a-zA-Z0-9_\-\.\+]*) HTTP', cmd) res = re.match(r'GET /([a-zA-Z0-9_\-\.\+%]*) HTTP', cmd)
if res is None: if res is None:
self.connection.close() self.connection.close()
return return
@ -511,6 +601,17 @@ class HttpClientThread(Thread):
elif url == 'time.html': elif url == 'time.html':
text = timeReport(self.resultPath) text = timeReport(self.resultPath)
httpGetResponse(self.connection, text, 'text/html') httpGetResponse(self.connection, text, 'text/html')
elif url == 'check_library_function_report.html':
text = check_library_report(self.resultPath + '/' + 'info_output', message_id='checkLibraryFunction')
httpGetResponse(self.connection, text, 'text/html')
elif url == 'check_library_noreturn_report.html':
text = check_library_report(self.resultPath + '/' + 'info_output', message_id='checkLibraryNoReturn')
httpGetResponse(self.connection, text, 'text/html')
elif url.startswith('check_library-'):
print('check library function !')
function_name = url[len('check_library-'):]
text = check_library_function_name(self.resultPath + '/' + 'info_output', function_name)
httpGetResponse(self.connection, text, 'text/plain')
else: else:
filename = resultPath + '/' + url filename = resultPath + '/' + url
if not os.path.isfile(filename): if not os.path.isfile(filename):
@ -621,6 +722,47 @@ def server(server_address_port, packages, packageIndex, resultPath):
latestResults.append(filename) latestResults.append(filename)
with open('latest.txt', 'wt') as f: with open('latest.txt', 'wt') as f:
f.write(' '.join(latestResults)) f.write(' '.join(latestResults))
elif cmd.startswith('write_info\nftp://'):
# read data
data = cmd[11:]
try:
t = 0
max_data_size = 1024 * 1024
while (len(data) < max_data_size) and (not data.endswith('\nDONE')) and (t < 10):
d = connection.recv(1024)
if d:
t = 0
data += d
else:
time.sleep(0.2)
t += 0.2
connection.close()
except socket.error as e:
pass
pos = data.find('\n')
if pos < 10:
continue
url = data[:pos]
print('[' + strDateTime() + '] write_info:' + url)
# save data
res = re.match(r'ftp://.*pool/main/[^/]+/([^/]+)/[^/]*tar.gz', url)
if res is None:
print('info output not written. res is None.')
continue
if url not in packages:
url2 = url + '\n'
if url2 not in packages:
print('info output not written. url is not in packages.')
continue
print('adding info output for package ' + res.group(1))
info_path = resultPath + '/' + 'info_output'
if not os.path.exists(info_path):
os.mkdir(info_path)
filename = info_path + '/' + res.group(1)
with open(filename, 'wt') as f:
f.write(strDateTime() + '\n' + data)
else: else:
print('[' + strDateTime() + '] invalid command: ' + firstLine) print('[' + strDateTime() + '] invalid command: ' + firstLine)
connection.close() connection.close()

62
tools/donate-cpu.py
View File

@ -231,7 +231,7 @@ def scanPackage(workPath, cppcheck, jobs):
libraries += ' --library=gtk' libraries += ' --library=gtk'
# if hasInclude('temp', '<boost/'): # if hasInclude('temp', '<boost/'):
# libraries += ' --library=boost' # libraries += ' --library=boost'
options = jobs + libraries + ' -D__GCC__ --inconclusive --enable=style --platform=unix64 --template=daca2 -rp=temp temp' options = jobs + libraries + ' -D__GCC__ --check-library --inconclusive --enable=style,information --platform=unix64 --template=daca2 -rp=temp temp'
cmd = 'nice ' + cppcheck + ' ' + options cmd = 'nice ' + cppcheck + ' ' + options
print(cmd) print(cmd)
startTime = time.time() startTime = time.time()
@ -243,18 +243,25 @@ def scanPackage(workPath, cppcheck, jobs):
if p.returncode != 0 and 'cppcheck: error: could not find or open any of the paths given.' not in stdout: if p.returncode != 0 and 'cppcheck: error: could not find or open any of the paths given.' not in stdout:
# Crash! # Crash!
print('Crash!') print('Crash!')
return -1, '', -1, options return -1, '', '', -1, options
if stderr.find('Internal error: Child process crashed with signal 11 [cppcheckError]') > 0: if stderr.find('Internal error: Child process crashed with signal 11 [cppcheckError]') > 0:
# Crash! # Crash!
print('Crash!') print('Crash!')
return -1, '', -1, options return -1, '', '', -1, options
elapsedTime = stopTime - startTime elapsedTime = stopTime - startTime
information_messages = ''
issue_messages = ''
count = 0 count = 0
for line in stderr.split('\n'): for line in stderr.split('\n'):
if re.match(r'.*:[0-9]+:.*\]$', line): if ': information: ' in line:
count += 1 information_messages += line + '\n'
else:
if len(line) > 0:
issue_messages += line + '\n'
if re.match(r'.*:[0-9]+:.*\]$', line):
count += 1
print('Number of issues: ' + str(count)) print('Number of issues: ' + str(count))
return count, stderr, elapsedTime, options return count, issue_messages, information_messages, elapsedTime, options
def splitResults(results): def splitResults(results):
@ -317,11 +324,29 @@ def uploadResults(package, results, server_address):
sock.connect(server_address) sock.connect(server_address)
sendAll(sock, 'write\n' + package + '\n' + results + '\nDONE') sendAll(sock, 'write\n' + package + '\n' + results + '\nDONE')
sock.close() sock.close()
print('Results have been successfully uploaded.')
return True return True
except socket.error: except socket.error:
print('Upload failed, retry in 60 seconds') print('Upload failed, retry in 30 seconds')
time.sleep(30) time.sleep(30)
pass print('Upload permanently failed!')
return False
def uploadInfo(package, info_output, server_address):
print('Uploading information output..')
for retry in range(3):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(server_address)
sendAll(sock, 'write_info\n' + package + '\n' + info_output + '\nDONE')
sock.close()
print('Information output has been successfully uploaded.')
return True
except socket.error:
print('Upload failed, retry in 30 seconds')
time.sleep(30)
print('Upload permanently failed!')
return False return False
@ -419,12 +444,13 @@ while True:
elapsedTime = '' elapsedTime = ''
resultsToDiff = [] resultsToDiff = []
cppcheck_options = '' cppcheck_options = ''
head_info_msg = ''
for ver in cppcheckVersions: for ver in cppcheckVersions:
if ver == 'head': if ver == 'head':
cppcheck = 'cppcheck/cppcheck' cppcheck = 'cppcheck/cppcheck'
else: else:
cppcheck = ver + '/cppcheck' cppcheck = ver + '/cppcheck'
c, errout, t, cppcheck_options = scanPackage(workpath, cppcheck, jobs) c, errout, info, t, cppcheck_options = scanPackage(workpath, cppcheck, jobs)
if c < 0: if c < 0:
crash = True crash = True
count += ' Crash!' count += ' Crash!'
@ -432,7 +458,15 @@ while True:
count += ' ' + str(c) count += ' ' + str(c)
elapsedTime += " {:.1f}".format(t) elapsedTime += " {:.1f}".format(t)
resultsToDiff.append(errout) resultsToDiff.append(errout)
if not crash and len(resultsToDiff[0]) + len(resultsToDiff[1]) == 0: if ver == 'head':
head_info_msg = info
results_exist = True
if len(resultsToDiff[0]) + len(resultsToDiff[1]) == 0:
results_exist = False
info_exists = True
if len(head_info_msg) == 0:
info_exists = False
if not crash and not results_exist and not info_exists:
print('No results') print('No results')
continue continue
output = 'cppcheck-options: ' + cppcheck_options + '\n' output = 'cppcheck-options: ' + cppcheck_options + '\n'
@ -441,6 +475,8 @@ while True:
output += 'cppcheck: ' + ' '.join(cppcheckVersions) + '\n' output += 'cppcheck: ' + ' '.join(cppcheckVersions) + '\n'
output += 'count:' + count + '\n' output += 'count:' + count + '\n'
output += 'elapsed-time:' + elapsedTime + '\n' output += 'elapsed-time:' + elapsedTime + '\n'
info_output = output
info_output += 'info messages:\n' + head_info_msg
if 'head' in cppcheckVersions: if 'head' in cppcheckVersions:
output += 'head results:\n' + resultsToDiff[cppcheckVersions.index('head')] output += 'head results:\n' + resultsToDiff[cppcheckVersions.index('head')]
if not crash: if not crash:
@ -450,7 +486,9 @@ while True:
print(output) print(output)
print('=========================================================') print('=========================================================')
break break
uploadResults(package, output, server_address) if results_exist:
print('Results have been uploaded') uploadResults(package, output, server_address)
if info_exists:
uploadInfo(package, info_output, server_address)
print('Sleep 5 seconds..') print('Sleep 5 seconds..')
time.sleep(5) time.sleep(5)