Donate CPU: Collect information messages and provide checkLibrary reports (#1610)
Related trac ticket: https://trac.cppcheck.net/ticket/8947 Enable information messages and "--check-library" in the Cppcheck parameters. Store the information messages and the rest of the messages in different variables and upload them separately. The server stores the information messages in a sub-directory similarly to the normal issue messages in one file per package. Reports for "checkLibraryFunction" and "checkLibraryNoReturn" message ids are generated by the server now.
This commit is contained in:
Sebastian
GitHub
parent
6b61866d59
commit
a410cea59a
|
|
@ -9,6 +9,7 @@ import datetime
|
|||
import time
|
||||
from threading import Thread
|
||||
import sys
|
||||
import urllib
|
||||
|
||||
OLD_VERSION = '1.86'
|
||||
|
||||
|
|
@ -27,6 +28,8 @@ def overviewReport():
|
|||
html += '<a href="head">HEAD report</a><br>\n'
|
||||
html += '<a href="latest.html">Latest results</a><br>\n'
|
||||
html += '<a href="time.html">Time report</a><br>\n'
|
||||
html += '<a href="check_library_function_report.html">checkLibraryFunction report</a><br>\n'
|
||||
html += '<a href="check_library_noreturn_report.html">checkLibraryNoReturn report</a><br>\n'
|
||||
html += '</body></html>'
|
||||
return html
|
||||
|
||||
|
|
@ -211,6 +214,8 @@ def diffMessageIdReport(resultPath, messageId):
|
|||
text = messageId + '\n'
|
||||
e = '[' + messageId + ']\n'
|
||||
for filename in sorted(glob.glob(resultPath + '/*')):
|
||||
if not os.path.isfile(filename):
|
||||
continue
|
||||
url = None
|
||||
diff = False
|
||||
for line in open(filename, 'rt'):
|
||||
|
|
@ -233,6 +238,8 @@ def diffMessageIdTodayReport(resultPath, messageId):
|
|||
e = '[' + messageId + ']\n'
|
||||
today = strDateTime()[:10]
|
||||
for filename in sorted(glob.glob(resultPath + '/*')):
|
||||
if not os.path.isfile(filename):
|
||||
continue
|
||||
url = None
|
||||
diff = False
|
||||
firstLine = True
|
||||
|
|
@ -338,6 +345,8 @@ def headMessageIdReport(resultPath, messageId):
|
|||
text = messageId + '\n'
|
||||
e = '[' + messageId + ']\n'
|
||||
for filename in sorted(glob.glob(resultPath + '/*')):
|
||||
if not os.path.isfile(filename):
|
||||
continue
|
||||
url = None
|
||||
headResults = False
|
||||
for line in open(filename, 'rt'):
|
||||
|
|
@ -362,6 +371,8 @@ def headMessageIdTodayReport(resultPath, messageId):
|
|||
e = '[' + messageId + ']\n'
|
||||
today = strDateTime()[:10]
|
||||
for filename in sorted(glob.glob(resultPath + '/*')):
|
||||
if not os.path.isfile(filename):
|
||||
continue
|
||||
url = None
|
||||
headResults = False
|
||||
firstLine = True
|
||||
|
|
@ -401,6 +412,8 @@ def timeReport(resultPath):
|
|||
total_time_base = 0.0
|
||||
total_time_head = 0.0
|
||||
for filename in glob.glob(resultPath + '/*'):
|
||||
if not os.path.isfile(filename):
|
||||
continue
|
||||
for line in open(filename, 'rt'):
|
||||
if not line.startswith('elapsed-time:'):
|
||||
continue
|
||||
|
|
@ -442,6 +455,83 @@ def timeReport(resultPath):
|
|||
return html
|
||||
|
||||
|
||||
def check_library_report(result_path, message_id):
|
||||
if message_id not in ('checkLibraryNoReturn', 'checkLibraryFunction'):
|
||||
error_message = 'Invalid value ' + message_id + ' for message_id parameter.'
|
||||
print(error_message)
|
||||
return error_message
|
||||
html = '<html><head><title>' + message_id + ' report</title></head><body>\n'
|
||||
html += '<h1>' + message_id + ' report</h1>\n'
|
||||
html += '<pre>\n'
|
||||
column_widths = [10, 100]
|
||||
html += '<b>'
|
||||
html += 'Count'.rjust(column_widths[0]) + ' ' + \
|
||||
'Function'
|
||||
html += '</b>\n'
|
||||
|
||||
function_counts = dict()
|
||||
for filename in glob.glob(result_path + '/*'):
|
||||
if not os.path.isfile(filename):
|
||||
continue
|
||||
info_messages = False
|
||||
for line in open(filename, 'rt'):
|
||||
if line == 'info messages:\n':
|
||||
info_messages = True
|
||||
if not info_messages:
|
||||
continue
|
||||
if line.endswith('[' + message_id + ']\n'):
|
||||
if message_id is 'checkLibraryNoReturn':
|
||||
function_name = line[(line.find(': Function ') + len(': Function ')):line.rfind('should have') - 1]
|
||||
else:
|
||||
function_name = line[(line.find('for function ') + len('for function ')):line.rfind('[') - 1]
|
||||
function_counts[function_name] = function_counts.setdefault(function_name, 0) + 1
|
||||
|
||||
for function_name, count in sorted(function_counts.iteritems(), key=lambda (k, v): (v, k), reverse=True):
|
||||
if count < 10:
|
||||
break
|
||||
html += str(count).rjust(column_widths[0]) + ' ' + \
|
||||
'<a href="check_library-' + urllib.quote_plus(function_name) + '">' + function_name + '</a>\n'
|
||||
|
||||
html += '\n'
|
||||
html += '</pre>\n'
|
||||
html += '</body></html>\n'
|
||||
|
||||
return html
|
||||
|
||||
|
||||
# Lists all checkLibrary* messages regarding the given function name
|
||||
def check_library_function_name(result_path, function_name):
|
||||
print('check_library_function_name')
|
||||
text = ''
|
||||
function_name = urllib.unquote_plus(function_name)
|
||||
for filename in glob.glob(result_path + '/*'):
|
||||
if not os.path.isfile(filename):
|
||||
continue
|
||||
info_messages = False
|
||||
url = None
|
||||
cppcheck_options = None
|
||||
for line in open(filename, 'rt'):
|
||||
if line.startswith('ftp://'):
|
||||
url = line
|
||||
elif line.startswith('cppcheck-options:'):
|
||||
cppcheck_options = line
|
||||
elif line == 'info messages:\n':
|
||||
info_messages = True
|
||||
if not info_messages:
|
||||
continue
|
||||
if '[checkLibrary' in line:
|
||||
if (' ' + function_name) in line:
|
||||
if url:
|
||||
text += url
|
||||
url = None
|
||||
if cppcheck_options:
|
||||
text += cppcheck_options
|
||||
cppcheck_options = None
|
||||
text += line
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def sendAll(connection, data):
|
||||
while data:
|
||||
num = connection.send(data)
|
||||
|
|
@ -472,7 +562,7 @@ class HttpClientThread(Thread):
|
|||
try:
|
||||
cmd = self.cmd
|
||||
print('[' + strDateTime() + '] ' + cmd)
|
||||
res = re.match(r'GET /([a-zA-Z0-9_\-\.\+]*) HTTP', cmd)
|
||||
res = re.match(r'GET /([a-zA-Z0-9_\-\.\+%]*) HTTP', cmd)
|
||||
if res is None:
|
||||
self.connection.close()
|
||||
return
|
||||
|
|
@ -511,6 +601,17 @@ class HttpClientThread(Thread):
|
|||
elif url == 'time.html':
|
||||
text = timeReport(self.resultPath)
|
||||
httpGetResponse(self.connection, text, 'text/html')
|
||||
elif url == 'check_library_function_report.html':
|
||||
text = check_library_report(self.resultPath + '/' + 'info_output', message_id='checkLibraryFunction')
|
||||
httpGetResponse(self.connection, text, 'text/html')
|
||||
elif url == 'check_library_noreturn_report.html':
|
||||
text = check_library_report(self.resultPath + '/' + 'info_output', message_id='checkLibraryNoReturn')
|
||||
httpGetResponse(self.connection, text, 'text/html')
|
||||
elif url.startswith('check_library-'):
|
||||
print('check library function !')
|
||||
function_name = url[len('check_library-'):]
|
||||
text = check_library_function_name(self.resultPath + '/' + 'info_output', function_name)
|
||||
httpGetResponse(self.connection, text, 'text/plain')
|
||||
else:
|
||||
filename = resultPath + '/' + url
|
||||
if not os.path.isfile(filename):
|
||||
|
|
@ -621,6 +722,47 @@ def server(server_address_port, packages, packageIndex, resultPath):
|
|||
latestResults.append(filename)
|
||||
with open('latest.txt', 'wt') as f:
|
||||
f.write(' '.join(latestResults))
|
||||
elif cmd.startswith('write_info\nftp://'):
|
||||
# read data
|
||||
data = cmd[11:]
|
||||
try:
|
||||
t = 0
|
||||
max_data_size = 1024 * 1024
|
||||
while (len(data) < max_data_size) and (not data.endswith('\nDONE')) and (t < 10):
|
||||
d = connection.recv(1024)
|
||||
if d:
|
||||
t = 0
|
||||
data += d
|
||||
else:
|
||||
time.sleep(0.2)
|
||||
t += 0.2
|
||||
connection.close()
|
||||
except socket.error as e:
|
||||
pass
|
||||
|
||||
pos = data.find('\n')
|
||||
if pos < 10:
|
||||
continue
|
||||
url = data[:pos]
|
||||
print('[' + strDateTime() + '] write_info:' + url)
|
||||
|
||||
# save data
|
||||
res = re.match(r'ftp://.*pool/main/[^/]+/([^/]+)/[^/]*tar.gz', url)
|
||||
if res is None:
|
||||
print('info output not written. res is None.')
|
||||
continue
|
||||
if url not in packages:
|
||||
url2 = url + '\n'
|
||||
if url2 not in packages:
|
||||
print('info output not written. url is not in packages.')
|
||||
continue
|
||||
print('adding info output for package ' + res.group(1))
|
||||
info_path = resultPath + '/' + 'info_output'
|
||||
if not os.path.exists(info_path):
|
||||
os.mkdir(info_path)
|
||||
filename = info_path + '/' + res.group(1)
|
||||
with open(filename, 'wt') as f:
|
||||
f.write(strDateTime() + '\n' + data)
|
||||
else:
|
||||
print('[' + strDateTime() + '] invalid command: ' + firstLine)
|
||||
connection.close()
|
||||
|
|
|
|||
|
|
@ -231,7 +231,7 @@ def scanPackage(workPath, cppcheck, jobs):
|
|||
libraries += ' --library=gtk'
|
||||
# if hasInclude('temp', '<boost/'):
|
||||
# libraries += ' --library=boost'
|
||||
options = jobs + libraries + ' -D__GCC__ --inconclusive --enable=style --platform=unix64 --template=daca2 -rp=temp temp'
|
||||
options = jobs + libraries + ' -D__GCC__ --check-library --inconclusive --enable=style,information --platform=unix64 --template=daca2 -rp=temp temp'
|
||||
cmd = 'nice ' + cppcheck + ' ' + options
|
||||
print(cmd)
|
||||
startTime = time.time()
|
||||
|
|
@ -243,18 +243,25 @@ def scanPackage(workPath, cppcheck, jobs):
|
|||
if p.returncode != 0 and 'cppcheck: error: could not find or open any of the paths given.' not in stdout:
|
||||
# Crash!
|
||||
print('Crash!')
|
||||
return -1, '', -1, options
|
||||
return -1, '', '', -1, options
|
||||
if stderr.find('Internal error: Child process crashed with signal 11 [cppcheckError]') > 0:
|
||||
# Crash!
|
||||
print('Crash!')
|
||||
return -1, '', -1, options
|
||||
return -1, '', '', -1, options
|
||||
elapsedTime = stopTime - startTime
|
||||
information_messages = ''
|
||||
issue_messages = ''
|
||||
count = 0
|
||||
for line in stderr.split('\n'):
|
||||
if re.match(r'.*:[0-9]+:.*\]$', line):
|
||||
count += 1
|
||||
if ': information: ' in line:
|
||||
information_messages += line + '\n'
|
||||
else:
|
||||
if len(line) > 0:
|
||||
issue_messages += line + '\n'
|
||||
if re.match(r'.*:[0-9]+:.*\]$', line):
|
||||
count += 1
|
||||
print('Number of issues: ' + str(count))
|
||||
return count, stderr, elapsedTime, options
|
||||
return count, issue_messages, information_messages, elapsedTime, options
|
||||
|
||||
|
||||
def splitResults(results):
|
||||
|
|
@ -317,11 +324,29 @@ def uploadResults(package, results, server_address):
|
|||
sock.connect(server_address)
|
||||
sendAll(sock, 'write\n' + package + '\n' + results + '\nDONE')
|
||||
sock.close()
|
||||
print('Results have been successfully uploaded.')
|
||||
return True
|
||||
except socket.error:
|
||||
print('Upload failed, retry in 60 seconds')
|
||||
print('Upload failed, retry in 30 seconds')
|
||||
time.sleep(30)
|
||||
pass
|
||||
print('Upload permanently failed!')
|
||||
return False
|
||||
|
||||
|
||||
def uploadInfo(package, info_output, server_address):
|
||||
print('Uploading information output..')
|
||||
for retry in range(3):
|
||||
try:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.connect(server_address)
|
||||
sendAll(sock, 'write_info\n' + package + '\n' + info_output + '\nDONE')
|
||||
sock.close()
|
||||
print('Information output has been successfully uploaded.')
|
||||
return True
|
||||
except socket.error:
|
||||
print('Upload failed, retry in 30 seconds')
|
||||
time.sleep(30)
|
||||
print('Upload permanently failed!')
|
||||
return False
|
||||
|
||||
|
||||
|
|
@ -419,12 +444,13 @@ while True:
|
|||
elapsedTime = ''
|
||||
resultsToDiff = []
|
||||
cppcheck_options = ''
|
||||
head_info_msg = ''
|
||||
for ver in cppcheckVersions:
|
||||
if ver == 'head':
|
||||
cppcheck = 'cppcheck/cppcheck'
|
||||
else:
|
||||
cppcheck = ver + '/cppcheck'
|
||||
c, errout, t, cppcheck_options = scanPackage(workpath, cppcheck, jobs)
|
||||
c, errout, info, t, cppcheck_options = scanPackage(workpath, cppcheck, jobs)
|
||||
if c < 0:
|
||||
crash = True
|
||||
count += ' Crash!'
|
||||
|
|
@ -432,7 +458,15 @@ while True:
|
|||
count += ' ' + str(c)
|
||||
elapsedTime += " {:.1f}".format(t)
|
||||
resultsToDiff.append(errout)
|
||||
if not crash and len(resultsToDiff[0]) + len(resultsToDiff[1]) == 0:
|
||||
if ver == 'head':
|
||||
head_info_msg = info
|
||||
results_exist = True
|
||||
if len(resultsToDiff[0]) + len(resultsToDiff[1]) == 0:
|
||||
results_exist = False
|
||||
info_exists = True
|
||||
if len(head_info_msg) == 0:
|
||||
info_exists = False
|
||||
if not crash and not results_exist and not info_exists:
|
||||
print('No results')
|
||||
continue
|
||||
output = 'cppcheck-options: ' + cppcheck_options + '\n'
|
||||
|
|
@ -441,6 +475,8 @@ while True:
|
|||
output += 'cppcheck: ' + ' '.join(cppcheckVersions) + '\n'
|
||||
output += 'count:' + count + '\n'
|
||||
output += 'elapsed-time:' + elapsedTime + '\n'
|
||||
info_output = output
|
||||
info_output += 'info messages:\n' + head_info_msg
|
||||
if 'head' in cppcheckVersions:
|
||||
output += 'head results:\n' + resultsToDiff[cppcheckVersions.index('head')]
|
||||
if not crash:
|
||||
|
|
@ -450,7 +486,9 @@ while True:
|
|||
print(output)
|
||||
print('=========================================================')
|
||||
break
|
||||
uploadResults(package, output, server_address)
|
||||
print('Results have been uploaded')
|
||||
if results_exist:
|
||||
uploadResults(package, output, server_address)
|
||||
if info_exists:
|
||||
uploadInfo(package, info_output, server_address)
|
||||
print('Sleep 5 seconds..')
|
||||
time.sleep(5)
|
||||
|
|
|
|||
Loading…
Reference in New Issue