walkero
/
cppcheck
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

donate-cpu-server.py: Upgrade to work with Python 3 (drop Python 2 support) (#2292) 

* donate-cpu-server.py: Use tools to prepare code to work with Python 3

The following commands were used for these changes:
futurize -1 -w donate-cpu-server.py
2to3 -w donate-cpu-server.py

* Make the server work under Python 3

Manually fixed the Unicode issues. Received data is decoded, sent data
is encoded.

* Add backward compatible type hints (in comments)

This enables better static analysis and suggestions in an IDE.

* Fix Pylint warning "Comparison to literal"

* .travis.yml: Fix/enhance pylint verification and Python compilation

donate-cpu-server.py is only Python 3 compatible, so it must be ignored
for pylint verification under Python 2.
All Python scripts that were verified with pylint under Python 2 are
now also verified with pylint under Python 3.

* donate-cpu-server.py: Add shebang and mark script as executable

* start_donate_cpu_server_test_local.sh: Directly execute server

Since the server script is executable now and has a shebang it can
be directly executed.

* Use Python 3.0 function annotations instead of comment type hints

Reference: https://www.python.org/dev/peps/pep-3107/
This commit is contained in:
Sebastian 2019-10-26 21:10:21 +02:00 committed by GitHub
parent f03945a9e2
commit ec521fba36
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 79 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.travis.yml
View File

@ -25,9 +25,14 @@ before_install:
- travis_retry python2 -m pip install --user pytest==4.6.4 - travis_retry python2 -m pip install --user pytest==4.6.4
- travis_retry python2 -m pip install --user pylint - travis_retry python2 -m pip install --user pylint
- travis_retry python2 -m pip install --user unittest2 - travis_retry python2 -m pip install --user unittest2
- travis_retry python2 -m pip install --user pexpect - travis_retry python2 -m pip install --user pexpect # imported by tools/ci.py
# Python 3 modules # Python 3 modules
- travis_retry python3 -m pip install --user setuptools --upgrade
- travis_retry python3 -m pip install --user pytest - travis_retry python3 -m pip install --user pytest
- travis_retry python3 -m pip install --user pylint
- travis_retry python3 -m pip install --user unittest2
- travis_retry python3 -m pip install --user pexpect # imported by tools/ci.py
- travis_retry python3 -m pip install --user requests # imported by tools/pr.py
matrix: matrix:
# do notify immediately about it when a job of a build fails. # do notify immediately about it when a job of a build fails.
@ -95,11 +100,13 @@ matrix:
# run pylint # run pylint
- pylint --rcfile=pylintrc_travis addons/*.py - pylint --rcfile=pylintrc_travis addons/*.py
- pylint --rcfile=pylintrc_travis htmlreport/*.py - pylint --rcfile=pylintrc_travis htmlreport/*.py
- pylint --rcfile=pylintrc_travis tools/*.py - pylint --rcfile=pylintrc_travis --ignore=donate-cpu-server.py tools/*.py
- python3 -m pylint --rcfile=pylintrc_travis addons/*.py
- python3 -m pylint --rcfile=pylintrc_travis htmlreport/*.py
- python3 -m pylint --rcfile=pylintrc_travis tools/*.py
# check python syntax by compiling some selected scripts # check python syntax by compiling some selected scripts
- python -m py_compile ./tools/donate-cpu.py - python -m py_compile ./tools/donate-cpu.py
- python3 -m py_compile ./tools/donate-cpu.py - python3 -m py_compile ./tools/donate-cpu.py
- python -m py_compile ./tools/donate-cpu-server.py
- python3 -m py_compile ./tools/donate-cpu-server.py - python3 -m py_compile ./tools/donate-cpu-server.py
# check addons/misc.py # check addons/misc.py
- cd addons/test - cd addons/test

115
tools/donate-cpu-server.py Normal file → Executable file
View File

@ -1,5 +1,7 @@
#!/usr/bin/env python3
# Server for 'donate-cpu.py' # Server for 'donate-cpu.py'
# Runs only under Python 3.
import glob import glob
import json import json
@ -10,7 +12,9 @@ import datetime
import time import time
from threading import Thread from threading import Thread
import sys import sys
import urllib import urllib.request
import urllib.parse
import urllib.error
import logging import logging
import logging.handlers import logging.handlers
import operator import operator
@ -18,7 +22,7 @@ import operator
# Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/ # Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/
# Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic # Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic
# changes) # changes)
SERVER_VERSION = "1.2.0" SERVER_VERSION = "1.3.0"
OLD_VERSION = '1.89' OLD_VERSION = '1.89'
@ -51,15 +55,15 @@ def handle_uncaught_exception(exc_type, exc_value, exc_traceback):
sys.excepthook = handle_uncaught_exception sys.excepthook = handle_uncaught_exception
def strDateTime(): def strDateTime() -> str:
return datetime.datetime.now().strftime('%Y-%m-%d %H:%M') return datetime.datetime.now().strftime('%Y-%m-%d %H:%M')
def dateTimeFromStr(datestr): def dateTimeFromStr(datestr: str) -> datetime.datetime:
return datetime.datetime.strptime(datestr, '%Y-%m-%d %H:%M') return datetime.datetime.strptime(datestr, '%Y-%m-%d %H:%M')
def overviewReport(): def overviewReport() -> str:
html = '<html><head><title>daca@home</title></head><body>\n' html = '<html><head><title>daca@home</title></head><body>\n'
html += '<h1>daca@home</h1>\n' html += '<h1>daca@home</h1>\n'
html += '<a href="crash.html">Crash report</a><br>\n' html += '<a href="crash.html">Crash report</a><br>\n'
@ -77,7 +81,7 @@ def overviewReport():
return html return html
def fmt(a, b, c=None, d=None, e=None, link=True): def fmt(a: str, b: str, c: str = None, d: str = None, e: str = None, link: bool = True) -> str:
column_width = [40, 10, 5, 6, 6, 8] column_width = [40, 10, 5, 6, 6, 8]
ret = a ret = a
while len(ret) < column_width[0]: while len(ret) < column_width[0]:
@ -101,7 +105,7 @@ def fmt(a, b, c=None, d=None, e=None, link=True):
return ret return ret
def latestReport(latestResults): def latestReport(latestResults: list) -> str:
html = '<html><head><title>Latest daca@home results</title></head><body>\n' html = '<html><head><title>Latest daca@home results</title></head><body>\n'
html += '<h1>Latest daca@home results</h1>\n' html += '<h1>Latest daca@home results</h1>\n'
html += '<pre>\n<b>' + fmt('Package', 'Date Time', OLD_VERSION, 'Head', 'Diff', link=False) + '</b>\n' html += '<pre>\n<b>' + fmt('Package', 'Date Time', OLD_VERSION, 'Head', 'Diff', link=False) + '</b>\n'
@ -140,7 +144,7 @@ def latestReport(latestResults):
return html return html
def crashReport(results_path): def crashReport(results_path: str) -> str:
html = '<html><head><title>Crash report</title></head><body>\n' html = '<html><head><title>Crash report</title></head><body>\n'
html += '<h1>Crash report</h1>\n' html += '<h1>Crash report</h1>\n'
html += '<pre>\n' html += '<pre>\n'
@ -203,7 +207,7 @@ def crashReport(results_path):
html += '</pre>\n' html += '</pre>\n'
html += '<pre>\n' html += '<pre>\n'
html += '<b>Stack traces</b>\n' html += '<b>Stack traces</b>\n'
for stack_trace in sorted(stack_traces.values(), key=lambda x: x['n'], reverse=True): for stack_trace in sorted(list(stack_traces.values()), key=lambda x: x['n'], reverse=True):
html += 'Packages: ' + ' '.join(['<a href="' + p + '">' + p + '</a>' for p in stack_trace['packages']]) + '\n' html += 'Packages: ' + ' '.join(['<a href="' + p + '">' + p + '</a>' for p in stack_trace['packages']]) + '\n'
html += stack_trace['crash_line'] + '\n' html += stack_trace['crash_line'] + '\n'
html += stack_trace['code_line'] + '\n' html += stack_trace['code_line'] + '\n'
@ -214,7 +218,7 @@ def crashReport(results_path):
return html return html
def staleReport(results_path): def staleReport(results_path: str) -> str:
html = '<html><head><title>Stale report</title></head><body>\n' html = '<html><head><title>Stale report</title></head><body>\n'
html += '<h1>Stale report</h1>\n' html += '<h1>Stale report</h1>\n'
html += '<pre>\n' html += '<pre>\n'
@ -242,7 +246,7 @@ def staleReport(results_path):
return html return html
def diffReportFromDict(out, today): def diffReportFromDict(out: dict, today: str) -> str:
html = '<pre>\n' html = '<pre>\n'
html += '<b>MessageID ' + OLD_VERSION + ' Head</b>\n' html += '<b>MessageID ' + OLD_VERSION + ' Head</b>\n'
sum0 = 0 sum0 = 0
@ -280,7 +284,7 @@ def diffReportFromDict(out, today):
return html return html
def diffReport(resultsPath): def diffReport(resultsPath: str) -> str:
out = {} out = {}
outToday = {} outToday = {}
today = strDateTime()[:10] today = strDateTime()[:10]
@ -315,7 +319,7 @@ def diffReport(resultsPath):
return html return html
def generate_package_diff_statistics(filename): def generate_package_diff_statistics(filename: str) -> None:
is_diff = False is_diff = False
sums = {} sums = {}
@ -355,7 +359,7 @@ def generate_package_diff_statistics(filename):
os.remove(filename_diff) os.remove(filename_diff)
def diffMessageIdReport(resultPath, messageId): def diffMessageIdReport(resultPath: str, messageId: str) -> str:
text = messageId + '\n' text = messageId + '\n'
e = '[' + messageId + ']\n' e = '[' + messageId + ']\n'
for filename in sorted(glob.glob(resultPath + '/*.diff')): for filename in sorted(glob.glob(resultPath + '/*.diff')):
@ -382,7 +386,7 @@ def diffMessageIdReport(resultPath, messageId):
return text return text
def diffMessageIdTodayReport(resultPath, messageId): def diffMessageIdTodayReport(resultPath: str, messageId: str) -> str:
text = messageId + '\n' text = messageId + '\n'
e = '[' + messageId + ']\n' e = '[' + messageId + ']\n'
today = strDateTime()[:10] today = strDateTime()[:10]
@ -417,7 +421,7 @@ def diffMessageIdTodayReport(resultPath, messageId):
return text return text
def headReportFromDict(out, today): def headReportFromDict(out: dict, today: str) -> str:
html = '<pre>\n' html = '<pre>\n'
html += '<b>MessageID Count</b>\n' html += '<b>MessageID Count</b>\n'
sumTotal = 0 sumTotal = 0
@ -445,7 +449,7 @@ def headReportFromDict(out, today):
return html return html
def headReport(resultsPath): def headReport(resultsPath: str) -> str:
out = {} out = {}
outToday = {} outToday = {}
today = strDateTime()[:10] today = strDateTime()[:10]
@ -509,7 +513,7 @@ def headReport(resultsPath):
return html return html
def headMessageIdReport(resultPath, messageId): def headMessageIdReport(resultPath: str, messageId: str) -> str:
text = messageId + '\n' text = messageId + '\n'
e = '[' + messageId + ']\n' e = '[' + messageId + ']\n'
for filename in sorted(glob.glob(resultPath + '/*')): for filename in sorted(glob.glob(resultPath + '/*')):
@ -534,7 +538,7 @@ def headMessageIdReport(resultPath, messageId):
return text return text
def headMessageIdTodayReport(resultPath, messageId): def headMessageIdTodayReport(resultPath: str, messageId: str) -> str:
text = messageId + '\n' text = messageId + '\n'
e = '[' + messageId + ']\n' e = '[' + messageId + ']\n'
today = strDateTime()[:10] today = strDateTime()[:10]
@ -565,7 +569,7 @@ def headMessageIdTodayReport(resultPath, messageId):
return text return text
def timeReport(resultPath): def timeReport(resultPath: str) -> str:
html = '<html><head><title>Time report</title></head><body>\n' html = '<html><head><title>Time report</title></head><body>\n'
html += '<h1>Time report</h1>\n' html += '<h1>Time report</h1>\n'
html += '<pre>\n' html += '<pre>\n'
@ -634,7 +638,7 @@ def timeReport(resultPath):
return html return html
def check_library_report(result_path, message_id): def check_library_report(result_path: str, message_id: str) -> str:
if message_id not in ('checkLibraryNoReturn', 'checkLibraryFunction', 'checkLibraryUseIgnore'): if message_id not in ('checkLibraryNoReturn', 'checkLibraryFunction', 'checkLibraryUseIgnore'):
error_message = 'Invalid value ' + message_id + ' for message_id parameter.' error_message = 'Invalid value ' + message_id + ' for message_id parameter.'
print(error_message) print(error_message)
@ -669,18 +673,18 @@ def check_library_report(result_path, message_id):
if not info_messages: if not info_messages:
continue continue
if line.endswith('[' + message_id + ']\n'): if line.endswith('[' + message_id + ']\n'):
if message_id is 'checkLibraryFunction': if message_id == 'checkLibraryFunction':
function_name = line[(line.find('for function ') + len('for function ')):line.rfind('[') - 1] function_name = line[(line.find('for function ') + len('for function ')):line.rfind('[') - 1]
else: else:
function_name = line[(line.find(': Function ') + len(': Function ')):line.rfind('should have') - 1] function_name = line[(line.find(': Function ') + len(': Function ')):line.rfind('should have') - 1]
function_counts[function_name] = function_counts.setdefault(function_name, 0) + 1 function_counts[function_name] = function_counts.setdefault(function_name, 0) + 1
function_details_list = [] function_details_list = []
for function_name, count in sorted(function_counts.items(), key=operator.itemgetter(1), reverse=True): for function_name, count in sorted(list(function_counts.items()), key=operator.itemgetter(1), reverse=True):
if len(function_details_list) >= functions_shown_max: if len(function_details_list) >= functions_shown_max:
break break
function_details_list.append(str(count).rjust(column_widths[0]) + ' ' + function_details_list.append(str(count).rjust(column_widths[0]) + ' ' +
'<a href="check_library-' + urllib.quote_plus(function_name) + '">' + function_name + '</a>\n') '<a href="check_library-' + urllib.parse.quote_plus(function_name) + '">' + function_name + '</a>\n')
html += ''.join(function_details_list) html += ''.join(function_details_list)
html += '</pre>\n' html += '</pre>\n'
@ -690,9 +694,9 @@ def check_library_report(result_path, message_id):
# Lists all checkLibrary* messages regarding the given function name # Lists all checkLibrary* messages regarding the given function name
def check_library_function_name(result_path, function_name): def check_library_function_name(result_path: str, function_name: str) -> str:
print('check_library_function_name') print('check_library_function_name')
function_name = urllib.unquote_plus(function_name) function_name = urllib.parse.unquote_plus(function_name)
output_lines_list = [] output_lines_list = []
for filename in glob.glob(result_path + '/*'): for filename in glob.glob(result_path + '/*'):
if not os.path.isfile(filename): if not os.path.isfile(filename):
@ -722,7 +726,8 @@ def check_library_function_name(result_path, function_name):
return ''.join(output_lines_list) return ''.join(output_lines_list)
def sendAll(connection, data): def sendAll(connection: socket.socket, text: str) -> None:
data = text.encode('utf-8', 'ignore')
while data: while data:
num = connection.send(data) num = connection.send(data)
if num < len(data): if num < len(data):
@ -731,7 +736,7 @@ def sendAll(connection, data):
data = None data = None
def httpGetResponse(connection, data, contentType): def httpGetResponse(connection: socket.socket, data: str, contentType: str) -> None:
resp = 'HTTP/1.1 200 OK\r\n' resp = 'HTTP/1.1 200 OK\r\n'
resp += 'Connection: close\r\n' resp += 'Connection: close\r\n'
resp += 'Content-length: ' + str(len(data)) + '\r\n' resp += 'Content-length: ' + str(len(data)) + '\r\n'
@ -741,7 +746,7 @@ def httpGetResponse(connection, data, contentType):
class HttpClientThread(Thread): class HttpClientThread(Thread):
def __init__(self, connection, cmd, resultPath, latestResults): def __init__(self, connection: socket.socket, cmd: str, resultPath: str, latestResults: list) -> None:
Thread.__init__(self) Thread.__init__(self)
self.connection = connection self.connection = connection
self.cmd = cmd[:cmd.find('\n')] self.cmd = cmd[:cmd.find('\n')]
@ -812,7 +817,7 @@ class HttpClientThread(Thread):
filename = resultPath + '/' + url filename = resultPath + '/' + url
if not os.path.isfile(filename): if not os.path.isfile(filename):
print('HTTP/1.1 404 Not Found') print('HTTP/1.1 404 Not Found')
self.connection.send('HTTP/1.1 404 Not Found\r\n\r\n') self.connection.send(b'HTTP/1.1 404 Not Found\r\n\r\n')
else: else:
f = open(filename, 'rt') f = open(filename, 'rt')
data = f.read() data = f.read()
@ -823,7 +828,7 @@ class HttpClientThread(Thread):
self.connection.close() self.connection.close()
def server(server_address_port, packages, packageIndex, resultPath): def server(server_address_port: int, packages: list, packageIndex: int, resultPath: str) -> None:
socket.setdefaulttimeout(30) socket.setdefaulttimeout(30)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
@ -845,10 +850,15 @@ def server(server_address_port, packages, packageIndex, resultPath):
print('[' + strDateTime() + '] waiting for a connection') print('[' + strDateTime() + '] waiting for a connection')
connection, client_address = sock.accept() connection, client_address = sock.accept()
try: try:
cmd = connection.recv(128) bytes_received = connection.recv(128)
cmd = bytes_received.decode('utf-8', 'ignore')
except socket.error: except socket.error:
connection.close() connection.close()
continue continue
except UnicodeDecodeError as e:
connection.close()
print('Error: Decoding failed: ' + str(e))
continue
if cmd.find('\n') < 1: if cmd.find('\n') < 1:
continue continue
firstLine = cmd[:cmd.find('\n')] firstLine = cmd[:cmd.find('\n')]
@ -861,7 +871,7 @@ def server(server_address_port, packages, packageIndex, resultPath):
elif cmd == 'GetCppcheckVersions\n': elif cmd == 'GetCppcheckVersions\n':
reply = 'head ' + OLD_VERSION reply = 'head ' + OLD_VERSION
print('[' + strDateTime() + '] GetCppcheckVersions: ' + reply) print('[' + strDateTime() + '] GetCppcheckVersions: ' + reply)
connection.send(reply) connection.send(reply.encode('utf-8', 'ignore'))
connection.close() connection.close()
elif cmd == 'get\n': elif cmd == 'get\n':
pkg = packages[packageIndex] pkg = packages[packageIndex]
@ -874,19 +884,25 @@ def server(server_address_port, packages, packageIndex, resultPath):
f.close() f.close()
print('[' + strDateTime() + '] get:' + pkg) print('[' + strDateTime() + '] get:' + pkg)
connection.send(pkg) connection.send(pkg.encode('utf-8', 'ignore'))
connection.close() connection.close()
elif cmd.startswith('write\nftp://'): elif cmd.startswith('write\nftp://'):
# read data # read data
data = cmd[cmd.find('ftp'):] data = cmd[cmd.find('ftp'):]
try: try:
t = 0 t = 0.0
max_data_size = 2 * 1024 * 1024 max_data_size = 2 * 1024 * 1024
while (len(data) < max_data_size) and (not data.endswith('\nDONE')) and (t < 10): while (len(data) < max_data_size) and (not data.endswith('\nDONE')) and (t < 10):
d = connection.recv(1024) bytes_received = connection.recv(1024)
if d: if bytes_received:
t = 0 try:
data += d text_received = bytes_received.decode('utf-8', 'ignore')
except UnicodeDecodeError as e:
print('Error: Decoding failed: ' + str(e))
data = ''
break
t = 0.0
data += text_received
else: else:
time.sleep(0.2) time.sleep(0.2)
t += 0.2 t += 0.2
@ -941,13 +957,19 @@ def server(server_address_port, packages, packageIndex, resultPath):
# read data # read data
data = cmd[11:] data = cmd[11:]
try: try:
t = 0 t = 0.0
max_data_size = 1024 * 1024 max_data_size = 1024 * 1024
while (len(data) < max_data_size) and (not data.endswith('\nDONE')) and (t < 10): while (len(data) < max_data_size) and (not data.endswith('\nDONE')) and (t < 10):
d = connection.recv(1024) bytes_received = connection.recv(1024)
if d: if bytes_received:
t = 0 try:
data += d text_received = bytes_received.decode('utf-8', 'ignore')
except UnicodeDecodeError as e:
print('Error: Decoding failed: ' + str(e))
data = ''
break
t = 0.0
data += text_received
else: else:
time.sleep(0.2) time.sleep(0.2)
t += 0.2 t += 0.2
@ -978,7 +1000,7 @@ def server(server_address_port, packages, packageIndex, resultPath):
f.write(strDateTime() + '\n' + data) f.write(strDateTime() + '\n' + data)
elif cmd == 'getPackagesCount\n': elif cmd == 'getPackagesCount\n':
packages_count = str(len(packages)) packages_count = str(len(packages))
connection.send(packages_count) connection.send(packages_count.encode('utf-8', 'ignore'))
connection.close() connection.close()
print('[' + strDateTime() + '] getPackagesCount: ' + packages_count) print('[' + strDateTime() + '] getPackagesCount: ' + packages_count)
continue continue
@ -986,7 +1008,7 @@ def server(server_address_port, packages, packageIndex, resultPath):
request_idx = abs(int(cmd[len('getPackageIdx:'):])) request_idx = abs(int(cmd[len('getPackageIdx:'):]))
if request_idx < len(packages): if request_idx < len(packages):
pkg = packages[request_idx] pkg = packages[request_idx]
connection.send(pkg) connection.send(pkg.encode('utf-8', 'ignore'))
connection.close() connection.close()
print('[' + strDateTime() + '] getPackageIdx: ' + pkg) print('[' + strDateTime() + '] getPackageIdx: ' + pkg)
else: else:
@ -1032,4 +1054,3 @@ if __name__ == "__main__":
server(server_address_port, packages, packageIndex, resultPath) server(server_address_port, packages, packageIndex, resultPath)
except socket.timeout: except socket.timeout:
print('Timeout!') print('Timeout!')

2
tools/test/start_donate_cpu_server_test_local.sh
View File

@ -27,5 +27,5 @@ fi
while : while :
do do
python "${cppcheck_tools_path}/donate-cpu-server.py" --test "${cppcheck_tools_path}/donate-cpu-server.py" --test
done done