Donate CPU: Collect information messages and provide checkLibrary reports (#1610)
Related trac ticket: https://trac.cppcheck.net/ticket/8947 Enable information messages and "--check-library" in the Cppcheck parameters. Store the information messages and the rest of the messages in different variables and upload them separately. The server stores the information messages in a sub-directory similarly to the normal issue messages in one file per package. Reports for "checkLibraryFunction" and "checkLibraryNoReturn" message ids are generated by the server now.
This commit is contained in:
parent
6b61866d59
commit
a410cea59a
|
@ -9,6 +9,7 @@ import datetime
|
||||||
import time
|
import time
|
||||||
from threading import Thread
|
from threading import Thread
|
||||||
import sys
|
import sys
|
||||||
|
import urllib
|
||||||
|
|
||||||
OLD_VERSION = '1.86'
|
OLD_VERSION = '1.86'
|
||||||
|
|
||||||
|
@ -27,6 +28,8 @@ def overviewReport():
|
||||||
html += '<a href="head">HEAD report</a><br>\n'
|
html += '<a href="head">HEAD report</a><br>\n'
|
||||||
html += '<a href="latest.html">Latest results</a><br>\n'
|
html += '<a href="latest.html">Latest results</a><br>\n'
|
||||||
html += '<a href="time.html">Time report</a><br>\n'
|
html += '<a href="time.html">Time report</a><br>\n'
|
||||||
|
html += '<a href="check_library_function_report.html">checkLibraryFunction report</a><br>\n'
|
||||||
|
html += '<a href="check_library_noreturn_report.html">checkLibraryNoReturn report</a><br>\n'
|
||||||
html += '</body></html>'
|
html += '</body></html>'
|
||||||
return html
|
return html
|
||||||
|
|
||||||
|
@ -211,6 +214,8 @@ def diffMessageIdReport(resultPath, messageId):
|
||||||
text = messageId + '\n'
|
text = messageId + '\n'
|
||||||
e = '[' + messageId + ']\n'
|
e = '[' + messageId + ']\n'
|
||||||
for filename in sorted(glob.glob(resultPath + '/*')):
|
for filename in sorted(glob.glob(resultPath + '/*')):
|
||||||
|
if not os.path.isfile(filename):
|
||||||
|
continue
|
||||||
url = None
|
url = None
|
||||||
diff = False
|
diff = False
|
||||||
for line in open(filename, 'rt'):
|
for line in open(filename, 'rt'):
|
||||||
|
@ -233,6 +238,8 @@ def diffMessageIdTodayReport(resultPath, messageId):
|
||||||
e = '[' + messageId + ']\n'
|
e = '[' + messageId + ']\n'
|
||||||
today = strDateTime()[:10]
|
today = strDateTime()[:10]
|
||||||
for filename in sorted(glob.glob(resultPath + '/*')):
|
for filename in sorted(glob.glob(resultPath + '/*')):
|
||||||
|
if not os.path.isfile(filename):
|
||||||
|
continue
|
||||||
url = None
|
url = None
|
||||||
diff = False
|
diff = False
|
||||||
firstLine = True
|
firstLine = True
|
||||||
|
@ -338,6 +345,8 @@ def headMessageIdReport(resultPath, messageId):
|
||||||
text = messageId + '\n'
|
text = messageId + '\n'
|
||||||
e = '[' + messageId + ']\n'
|
e = '[' + messageId + ']\n'
|
||||||
for filename in sorted(glob.glob(resultPath + '/*')):
|
for filename in sorted(glob.glob(resultPath + '/*')):
|
||||||
|
if not os.path.isfile(filename):
|
||||||
|
continue
|
||||||
url = None
|
url = None
|
||||||
headResults = False
|
headResults = False
|
||||||
for line in open(filename, 'rt'):
|
for line in open(filename, 'rt'):
|
||||||
|
@ -362,6 +371,8 @@ def headMessageIdTodayReport(resultPath, messageId):
|
||||||
e = '[' + messageId + ']\n'
|
e = '[' + messageId + ']\n'
|
||||||
today = strDateTime()[:10]
|
today = strDateTime()[:10]
|
||||||
for filename in sorted(glob.glob(resultPath + '/*')):
|
for filename in sorted(glob.glob(resultPath + '/*')):
|
||||||
|
if not os.path.isfile(filename):
|
||||||
|
continue
|
||||||
url = None
|
url = None
|
||||||
headResults = False
|
headResults = False
|
||||||
firstLine = True
|
firstLine = True
|
||||||
|
@ -401,6 +412,8 @@ def timeReport(resultPath):
|
||||||
total_time_base = 0.0
|
total_time_base = 0.0
|
||||||
total_time_head = 0.0
|
total_time_head = 0.0
|
||||||
for filename in glob.glob(resultPath + '/*'):
|
for filename in glob.glob(resultPath + '/*'):
|
||||||
|
if not os.path.isfile(filename):
|
||||||
|
continue
|
||||||
for line in open(filename, 'rt'):
|
for line in open(filename, 'rt'):
|
||||||
if not line.startswith('elapsed-time:'):
|
if not line.startswith('elapsed-time:'):
|
||||||
continue
|
continue
|
||||||
|
@ -442,6 +455,83 @@ def timeReport(resultPath):
|
||||||
return html
|
return html
|
||||||
|
|
||||||
|
|
||||||
|
def check_library_report(result_path, message_id):
|
||||||
|
if message_id not in ('checkLibraryNoReturn', 'checkLibraryFunction'):
|
||||||
|
error_message = 'Invalid value ' + message_id + ' for message_id parameter.'
|
||||||
|
print(error_message)
|
||||||
|
return error_message
|
||||||
|
html = '<html><head><title>' + message_id + ' report</title></head><body>\n'
|
||||||
|
html += '<h1>' + message_id + ' report</h1>\n'
|
||||||
|
html += '<pre>\n'
|
||||||
|
column_widths = [10, 100]
|
||||||
|
html += '<b>'
|
||||||
|
html += 'Count'.rjust(column_widths[0]) + ' ' + \
|
||||||
|
'Function'
|
||||||
|
html += '</b>\n'
|
||||||
|
|
||||||
|
function_counts = dict()
|
||||||
|
for filename in glob.glob(result_path + '/*'):
|
||||||
|
if not os.path.isfile(filename):
|
||||||
|
continue
|
||||||
|
info_messages = False
|
||||||
|
for line in open(filename, 'rt'):
|
||||||
|
if line == 'info messages:\n':
|
||||||
|
info_messages = True
|
||||||
|
if not info_messages:
|
||||||
|
continue
|
||||||
|
if line.endswith('[' + message_id + ']\n'):
|
||||||
|
if message_id is 'checkLibraryNoReturn':
|
||||||
|
function_name = line[(line.find(': Function ') + len(': Function ')):line.rfind('should have') - 1]
|
||||||
|
else:
|
||||||
|
function_name = line[(line.find('for function ') + len('for function ')):line.rfind('[') - 1]
|
||||||
|
function_counts[function_name] = function_counts.setdefault(function_name, 0) + 1
|
||||||
|
|
||||||
|
for function_name, count in sorted(function_counts.iteritems(), key=lambda (k, v): (v, k), reverse=True):
|
||||||
|
if count < 10:
|
||||||
|
break
|
||||||
|
html += str(count).rjust(column_widths[0]) + ' ' + \
|
||||||
|
'<a href="check_library-' + urllib.quote_plus(function_name) + '">' + function_name + '</a>\n'
|
||||||
|
|
||||||
|
html += '\n'
|
||||||
|
html += '</pre>\n'
|
||||||
|
html += '</body></html>\n'
|
||||||
|
|
||||||
|
return html
|
||||||
|
|
||||||
|
|
||||||
|
# Lists all checkLibrary* messages regarding the given function name
|
||||||
|
def check_library_function_name(result_path, function_name):
|
||||||
|
print('check_library_function_name')
|
||||||
|
text = ''
|
||||||
|
function_name = urllib.unquote_plus(function_name)
|
||||||
|
for filename in glob.glob(result_path + '/*'):
|
||||||
|
if not os.path.isfile(filename):
|
||||||
|
continue
|
||||||
|
info_messages = False
|
||||||
|
url = None
|
||||||
|
cppcheck_options = None
|
||||||
|
for line in open(filename, 'rt'):
|
||||||
|
if line.startswith('ftp://'):
|
||||||
|
url = line
|
||||||
|
elif line.startswith('cppcheck-options:'):
|
||||||
|
cppcheck_options = line
|
||||||
|
elif line == 'info messages:\n':
|
||||||
|
info_messages = True
|
||||||
|
if not info_messages:
|
||||||
|
continue
|
||||||
|
if '[checkLibrary' in line:
|
||||||
|
if (' ' + function_name) in line:
|
||||||
|
if url:
|
||||||
|
text += url
|
||||||
|
url = None
|
||||||
|
if cppcheck_options:
|
||||||
|
text += cppcheck_options
|
||||||
|
cppcheck_options = None
|
||||||
|
text += line
|
||||||
|
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
def sendAll(connection, data):
|
def sendAll(connection, data):
|
||||||
while data:
|
while data:
|
||||||
num = connection.send(data)
|
num = connection.send(data)
|
||||||
|
@ -472,7 +562,7 @@ class HttpClientThread(Thread):
|
||||||
try:
|
try:
|
||||||
cmd = self.cmd
|
cmd = self.cmd
|
||||||
print('[' + strDateTime() + '] ' + cmd)
|
print('[' + strDateTime() + '] ' + cmd)
|
||||||
res = re.match(r'GET /([a-zA-Z0-9_\-\.\+]*) HTTP', cmd)
|
res = re.match(r'GET /([a-zA-Z0-9_\-\.\+%]*) HTTP', cmd)
|
||||||
if res is None:
|
if res is None:
|
||||||
self.connection.close()
|
self.connection.close()
|
||||||
return
|
return
|
||||||
|
@ -511,6 +601,17 @@ class HttpClientThread(Thread):
|
||||||
elif url == 'time.html':
|
elif url == 'time.html':
|
||||||
text = timeReport(self.resultPath)
|
text = timeReport(self.resultPath)
|
||||||
httpGetResponse(self.connection, text, 'text/html')
|
httpGetResponse(self.connection, text, 'text/html')
|
||||||
|
elif url == 'check_library_function_report.html':
|
||||||
|
text = check_library_report(self.resultPath + '/' + 'info_output', message_id='checkLibraryFunction')
|
||||||
|
httpGetResponse(self.connection, text, 'text/html')
|
||||||
|
elif url == 'check_library_noreturn_report.html':
|
||||||
|
text = check_library_report(self.resultPath + '/' + 'info_output', message_id='checkLibraryNoReturn')
|
||||||
|
httpGetResponse(self.connection, text, 'text/html')
|
||||||
|
elif url.startswith('check_library-'):
|
||||||
|
print('check library function !')
|
||||||
|
function_name = url[len('check_library-'):]
|
||||||
|
text = check_library_function_name(self.resultPath + '/' + 'info_output', function_name)
|
||||||
|
httpGetResponse(self.connection, text, 'text/plain')
|
||||||
else:
|
else:
|
||||||
filename = resultPath + '/' + url
|
filename = resultPath + '/' + url
|
||||||
if not os.path.isfile(filename):
|
if not os.path.isfile(filename):
|
||||||
|
@ -621,6 +722,47 @@ def server(server_address_port, packages, packageIndex, resultPath):
|
||||||
latestResults.append(filename)
|
latestResults.append(filename)
|
||||||
with open('latest.txt', 'wt') as f:
|
with open('latest.txt', 'wt') as f:
|
||||||
f.write(' '.join(latestResults))
|
f.write(' '.join(latestResults))
|
||||||
|
elif cmd.startswith('write_info\nftp://'):
|
||||||
|
# read data
|
||||||
|
data = cmd[11:]
|
||||||
|
try:
|
||||||
|
t = 0
|
||||||
|
max_data_size = 1024 * 1024
|
||||||
|
while (len(data) < max_data_size) and (not data.endswith('\nDONE')) and (t < 10):
|
||||||
|
d = connection.recv(1024)
|
||||||
|
if d:
|
||||||
|
t = 0
|
||||||
|
data += d
|
||||||
|
else:
|
||||||
|
time.sleep(0.2)
|
||||||
|
t += 0.2
|
||||||
|
connection.close()
|
||||||
|
except socket.error as e:
|
||||||
|
pass
|
||||||
|
|
||||||
|
pos = data.find('\n')
|
||||||
|
if pos < 10:
|
||||||
|
continue
|
||||||
|
url = data[:pos]
|
||||||
|
print('[' + strDateTime() + '] write_info:' + url)
|
||||||
|
|
||||||
|
# save data
|
||||||
|
res = re.match(r'ftp://.*pool/main/[^/]+/([^/]+)/[^/]*tar.gz', url)
|
||||||
|
if res is None:
|
||||||
|
print('info output not written. res is None.')
|
||||||
|
continue
|
||||||
|
if url not in packages:
|
||||||
|
url2 = url + '\n'
|
||||||
|
if url2 not in packages:
|
||||||
|
print('info output not written. url is not in packages.')
|
||||||
|
continue
|
||||||
|
print('adding info output for package ' + res.group(1))
|
||||||
|
info_path = resultPath + '/' + 'info_output'
|
||||||
|
if not os.path.exists(info_path):
|
||||||
|
os.mkdir(info_path)
|
||||||
|
filename = info_path + '/' + res.group(1)
|
||||||
|
with open(filename, 'wt') as f:
|
||||||
|
f.write(strDateTime() + '\n' + data)
|
||||||
else:
|
else:
|
||||||
print('[' + strDateTime() + '] invalid command: ' + firstLine)
|
print('[' + strDateTime() + '] invalid command: ' + firstLine)
|
||||||
connection.close()
|
connection.close()
|
||||||
|
|
|
@ -231,7 +231,7 @@ def scanPackage(workPath, cppcheck, jobs):
|
||||||
libraries += ' --library=gtk'
|
libraries += ' --library=gtk'
|
||||||
# if hasInclude('temp', '<boost/'):
|
# if hasInclude('temp', '<boost/'):
|
||||||
# libraries += ' --library=boost'
|
# libraries += ' --library=boost'
|
||||||
options = jobs + libraries + ' -D__GCC__ --inconclusive --enable=style --platform=unix64 --template=daca2 -rp=temp temp'
|
options = jobs + libraries + ' -D__GCC__ --check-library --inconclusive --enable=style,information --platform=unix64 --template=daca2 -rp=temp temp'
|
||||||
cmd = 'nice ' + cppcheck + ' ' + options
|
cmd = 'nice ' + cppcheck + ' ' + options
|
||||||
print(cmd)
|
print(cmd)
|
||||||
startTime = time.time()
|
startTime = time.time()
|
||||||
|
@ -243,18 +243,25 @@ def scanPackage(workPath, cppcheck, jobs):
|
||||||
if p.returncode != 0 and 'cppcheck: error: could not find or open any of the paths given.' not in stdout:
|
if p.returncode != 0 and 'cppcheck: error: could not find or open any of the paths given.' not in stdout:
|
||||||
# Crash!
|
# Crash!
|
||||||
print('Crash!')
|
print('Crash!')
|
||||||
return -1, '', -1, options
|
return -1, '', '', -1, options
|
||||||
if stderr.find('Internal error: Child process crashed with signal 11 [cppcheckError]') > 0:
|
if stderr.find('Internal error: Child process crashed with signal 11 [cppcheckError]') > 0:
|
||||||
# Crash!
|
# Crash!
|
||||||
print('Crash!')
|
print('Crash!')
|
||||||
return -1, '', -1, options
|
return -1, '', '', -1, options
|
||||||
elapsedTime = stopTime - startTime
|
elapsedTime = stopTime - startTime
|
||||||
|
information_messages = ''
|
||||||
|
issue_messages = ''
|
||||||
count = 0
|
count = 0
|
||||||
for line in stderr.split('\n'):
|
for line in stderr.split('\n'):
|
||||||
|
if ': information: ' in line:
|
||||||
|
information_messages += line + '\n'
|
||||||
|
else:
|
||||||
|
if len(line) > 0:
|
||||||
|
issue_messages += line + '\n'
|
||||||
if re.match(r'.*:[0-9]+:.*\]$', line):
|
if re.match(r'.*:[0-9]+:.*\]$', line):
|
||||||
count += 1
|
count += 1
|
||||||
print('Number of issues: ' + str(count))
|
print('Number of issues: ' + str(count))
|
||||||
return count, stderr, elapsedTime, options
|
return count, issue_messages, information_messages, elapsedTime, options
|
||||||
|
|
||||||
|
|
||||||
def splitResults(results):
|
def splitResults(results):
|
||||||
|
@ -317,11 +324,29 @@ def uploadResults(package, results, server_address):
|
||||||
sock.connect(server_address)
|
sock.connect(server_address)
|
||||||
sendAll(sock, 'write\n' + package + '\n' + results + '\nDONE')
|
sendAll(sock, 'write\n' + package + '\n' + results + '\nDONE')
|
||||||
sock.close()
|
sock.close()
|
||||||
|
print('Results have been successfully uploaded.')
|
||||||
return True
|
return True
|
||||||
except socket.error:
|
except socket.error:
|
||||||
print('Upload failed, retry in 60 seconds')
|
print('Upload failed, retry in 30 seconds')
|
||||||
time.sleep(30)
|
time.sleep(30)
|
||||||
pass
|
print('Upload permanently failed!')
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def uploadInfo(package, info_output, server_address):
|
||||||
|
print('Uploading information output..')
|
||||||
|
for retry in range(3):
|
||||||
|
try:
|
||||||
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
|
sock.connect(server_address)
|
||||||
|
sendAll(sock, 'write_info\n' + package + '\n' + info_output + '\nDONE')
|
||||||
|
sock.close()
|
||||||
|
print('Information output has been successfully uploaded.')
|
||||||
|
return True
|
||||||
|
except socket.error:
|
||||||
|
print('Upload failed, retry in 30 seconds')
|
||||||
|
time.sleep(30)
|
||||||
|
print('Upload permanently failed!')
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
@ -419,12 +444,13 @@ while True:
|
||||||
elapsedTime = ''
|
elapsedTime = ''
|
||||||
resultsToDiff = []
|
resultsToDiff = []
|
||||||
cppcheck_options = ''
|
cppcheck_options = ''
|
||||||
|
head_info_msg = ''
|
||||||
for ver in cppcheckVersions:
|
for ver in cppcheckVersions:
|
||||||
if ver == 'head':
|
if ver == 'head':
|
||||||
cppcheck = 'cppcheck/cppcheck'
|
cppcheck = 'cppcheck/cppcheck'
|
||||||
else:
|
else:
|
||||||
cppcheck = ver + '/cppcheck'
|
cppcheck = ver + '/cppcheck'
|
||||||
c, errout, t, cppcheck_options = scanPackage(workpath, cppcheck, jobs)
|
c, errout, info, t, cppcheck_options = scanPackage(workpath, cppcheck, jobs)
|
||||||
if c < 0:
|
if c < 0:
|
||||||
crash = True
|
crash = True
|
||||||
count += ' Crash!'
|
count += ' Crash!'
|
||||||
|
@ -432,7 +458,15 @@ while True:
|
||||||
count += ' ' + str(c)
|
count += ' ' + str(c)
|
||||||
elapsedTime += " {:.1f}".format(t)
|
elapsedTime += " {:.1f}".format(t)
|
||||||
resultsToDiff.append(errout)
|
resultsToDiff.append(errout)
|
||||||
if not crash and len(resultsToDiff[0]) + len(resultsToDiff[1]) == 0:
|
if ver == 'head':
|
||||||
|
head_info_msg = info
|
||||||
|
results_exist = True
|
||||||
|
if len(resultsToDiff[0]) + len(resultsToDiff[1]) == 0:
|
||||||
|
results_exist = False
|
||||||
|
info_exists = True
|
||||||
|
if len(head_info_msg) == 0:
|
||||||
|
info_exists = False
|
||||||
|
if not crash and not results_exist and not info_exists:
|
||||||
print('No results')
|
print('No results')
|
||||||
continue
|
continue
|
||||||
output = 'cppcheck-options: ' + cppcheck_options + '\n'
|
output = 'cppcheck-options: ' + cppcheck_options + '\n'
|
||||||
|
@ -441,6 +475,8 @@ while True:
|
||||||
output += 'cppcheck: ' + ' '.join(cppcheckVersions) + '\n'
|
output += 'cppcheck: ' + ' '.join(cppcheckVersions) + '\n'
|
||||||
output += 'count:' + count + '\n'
|
output += 'count:' + count + '\n'
|
||||||
output += 'elapsed-time:' + elapsedTime + '\n'
|
output += 'elapsed-time:' + elapsedTime + '\n'
|
||||||
|
info_output = output
|
||||||
|
info_output += 'info messages:\n' + head_info_msg
|
||||||
if 'head' in cppcheckVersions:
|
if 'head' in cppcheckVersions:
|
||||||
output += 'head results:\n' + resultsToDiff[cppcheckVersions.index('head')]
|
output += 'head results:\n' + resultsToDiff[cppcheckVersions.index('head')]
|
||||||
if not crash:
|
if not crash:
|
||||||
|
@ -450,7 +486,9 @@ while True:
|
||||||
print(output)
|
print(output)
|
||||||
print('=========================================================')
|
print('=========================================================')
|
||||||
break
|
break
|
||||||
|
if results_exist:
|
||||||
uploadResults(package, output, server_address)
|
uploadResults(package, output, server_address)
|
||||||
print('Results have been uploaded')
|
if info_exists:
|
||||||
|
uploadInfo(package, info_output, server_address)
|
||||||
print('Sleep 5 seconds..')
|
print('Sleep 5 seconds..')
|
||||||
time.sleep(5)
|
time.sleep(5)
|
||||||
|
|
Loading…
Reference in New Issue