From ec521fba36ce83a707f0f43f8aea3de903cab986 Mon Sep 17 00:00:00 2001
From: Sebastian <versat@users.noreply.github.com>
Date: Sat, 26 Oct 2019 21:10:21 +0200
Subject: [PATCH] donate-cpu-server.py: Upgrade to work with Python 3 (drop
 Python 2 support) (#2292)

* donate-cpu-server.py: Use tools to prepare code to work with Python 3

The following commands were used for these changes:
futurize -1 -w donate-cpu-server.py
2to3 -w donate-cpu-server.py

* Make the server work under Python 3

Manually fixed the Unicode issues. Received data is decoded, sent data
is encoded.

* Add backward compatible type hints (in comments)

This enables better static analysis and suggestions in an IDE.

* Fix Pylint warning "Comparison to literal"

* .travis.yml: Fix/enhance pylint verification and Python compilation

donate-cpu-server.py is only Python 3 compatible, so it must be ignored
for pylint verification under Python 2.
All Python scripts that were verified with pylint under Python 2 are
now also verified with pylint under Python 3.

* donate-cpu-server.py: Add shebang and mark script as executable

* start_donate_cpu_server_test_local.sh: Directly execute server

Since the server script is executable now and has a shebang it can
be directly executed.

* Use Python 3.0 function annotations instead of comment type hints

Reference: https://www.python.org/dev/peps/pep-3107/
---
 .travis.yml                                   |  13 +-
 tools/donate-cpu-server.py                    | 115 +++++++++++-------
 .../start_donate_cpu_server_test_local.sh     |   2 +-
 3 files changed, 79 insertions(+), 51 deletions(-)
 mode change 100644 => 100755 tools/donate-cpu-server.py

diff --git a/.travis.yml b/.travis.yml
index 4125525da..9aaa5a9d6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -25,9 +25,14 @@ before_install:
  - travis_retry python2 -m pip install --user pytest==4.6.4
  - travis_retry python2 -m pip install --user pylint
  - travis_retry python2 -m pip install --user unittest2
- - travis_retry python2 -m pip install --user pexpect
+ - travis_retry python2 -m pip install --user pexpect # imported by tools/ci.py
 # Python 3 modules
+ - travis_retry python3 -m pip install --user setuptools --upgrade
  - travis_retry python3 -m pip install --user pytest
+ - travis_retry python3 -m pip install --user pylint
+ - travis_retry python3 -m pip install --user unittest2
+ - travis_retry python3 -m pip install --user pexpect # imported by tools/ci.py
+ - travis_retry python3 -m pip install --user requests # imported by tools/pr.py
 
 matrix:
 # do notify immediately about it when a job of a build fails.
@@ -95,11 +100,13 @@ matrix:
 # run pylint
         - pylint --rcfile=pylintrc_travis addons/*.py
         - pylint --rcfile=pylintrc_travis htmlreport/*.py
-        - pylint --rcfile=pylintrc_travis tools/*.py
+        - pylint --rcfile=pylintrc_travis --ignore=donate-cpu-server.py tools/*.py
+        - python3 -m pylint --rcfile=pylintrc_travis addons/*.py
+        - python3 -m pylint --rcfile=pylintrc_travis htmlreport/*.py
+        - python3 -m pylint --rcfile=pylintrc_travis tools/*.py
 # check python syntax by compiling some selected scripts
         - python -m py_compile ./tools/donate-cpu.py
         - python3 -m py_compile ./tools/donate-cpu.py
-        - python -m py_compile ./tools/donate-cpu-server.py
         - python3 -m py_compile ./tools/donate-cpu-server.py
 # check addons/misc.py
         - cd addons/test
diff --git a/tools/donate-cpu-server.py b/tools/donate-cpu-server.py
old mode 100644
new mode 100755
index cf4ad5aa4..6172b728c
--- a/tools/donate-cpu-server.py
+++ b/tools/donate-cpu-server.py
@@ -1,5 +1,7 @@
+#!/usr/bin/env python3
 
 # Server for 'donate-cpu.py'
+# Runs only under Python 3.
 
 import glob
 import json
@@ -10,7 +12,9 @@ import datetime
 import time
 from threading import Thread
 import sys
-import urllib
+import urllib.request
+import urllib.parse
+import urllib.error
 import logging
 import logging.handlers
 import operator
@@ -18,7 +22,7 @@ import operator
 # Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/
 # Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic
 # changes)
-SERVER_VERSION = "1.2.0"
+SERVER_VERSION = "1.3.0"
 
 OLD_VERSION = '1.89'
 
@@ -51,15 +55,15 @@ def handle_uncaught_exception(exc_type, exc_value, exc_traceback):
 sys.excepthook = handle_uncaught_exception
 
 
-def strDateTime():
+def strDateTime() -> str:
     return datetime.datetime.now().strftime('%Y-%m-%d %H:%M')
 
 
-def dateTimeFromStr(datestr):
+def dateTimeFromStr(datestr: str) -> datetime.datetime:
     return datetime.datetime.strptime(datestr, '%Y-%m-%d %H:%M')
 
 
-def overviewReport():
+def overviewReport() -> str:
     html = '<html><head><title>daca@home</title></head><body>\n'
     html += '<h1>daca@home</h1>\n'
     html += '<a href="crash.html">Crash report</a><br>\n'
@@ -77,7 +81,7 @@ def overviewReport():
     return html
 
 
-def fmt(a, b, c=None, d=None, e=None, link=True):
+def fmt(a: str, b: str, c: str = None, d: str = None, e: str = None, link: bool = True) -> str:
     column_width = [40, 10, 5, 6, 6, 8]
     ret = a
     while len(ret) < column_width[0]:
@@ -101,7 +105,7 @@ def fmt(a, b, c=None, d=None, e=None, link=True):
     return ret
 
 
-def latestReport(latestResults):
+def latestReport(latestResults: list) -> str:
     html = '<html><head><title>Latest daca@home results</title></head><body>\n'
     html += '<h1>Latest daca@home results</h1>\n'
     html += '<pre>\n<b>' + fmt('Package', 'Date       Time', OLD_VERSION, 'Head', 'Diff', link=False) + '</b>\n'
@@ -140,7 +144,7 @@ def latestReport(latestResults):
     return html
 
 
-def crashReport(results_path):
+def crashReport(results_path: str) -> str:
     html = '<html><head><title>Crash report</title></head><body>\n'
     html += '<h1>Crash report</h1>\n'
     html += '<pre>\n'
@@ -203,7 +207,7 @@ def crashReport(results_path):
     html += '</pre>\n'
     html += '<pre>\n'
     html += '<b>Stack traces</b>\n'
-    for stack_trace in sorted(stack_traces.values(), key=lambda x: x['n'], reverse=True):
+    for stack_trace in sorted(list(stack_traces.values()), key=lambda x: x['n'], reverse=True):
         html += 'Packages: ' + ' '.join(['<a href="' + p + '">' + p + '</a>' for p in stack_trace['packages']]) + '\n'
         html += stack_trace['crash_line'] + '\n'
         html += stack_trace['code_line'] + '\n'
@@ -214,7 +218,7 @@ def crashReport(results_path):
     return html
 
 
-def staleReport(results_path):
+def staleReport(results_path: str) -> str:
     html = '<html><head><title>Stale report</title></head><body>\n'
     html += '<h1>Stale report</h1>\n'
     html += '<pre>\n'
@@ -242,7 +246,7 @@ def staleReport(results_path):
     return html
 
 
-def diffReportFromDict(out, today):
+def diffReportFromDict(out: dict, today: str) -> str:
     html = '<pre>\n'
     html += '<b>MessageID                           ' + OLD_VERSION + '    Head</b>\n'
     sum0 = 0
@@ -280,7 +284,7 @@ def diffReportFromDict(out, today):
     return html
 
 
-def diffReport(resultsPath):
+def diffReport(resultsPath: str) -> str:
     out = {}
     outToday = {}
     today = strDateTime()[:10]
@@ -315,7 +319,7 @@ def diffReport(resultsPath):
     return html
 
 
-def generate_package_diff_statistics(filename):
+def generate_package_diff_statistics(filename: str) -> None:
     is_diff = False
 
     sums = {}
@@ -355,7 +359,7 @@ def generate_package_diff_statistics(filename):
         os.remove(filename_diff)
 
 
-def diffMessageIdReport(resultPath, messageId):
+def diffMessageIdReport(resultPath: str, messageId: str) -> str:
     text = messageId + '\n'
     e = '[' + messageId + ']\n'
     for filename in sorted(glob.glob(resultPath + '/*.diff')):
@@ -382,7 +386,7 @@ def diffMessageIdReport(resultPath, messageId):
     return text
 
 
-def diffMessageIdTodayReport(resultPath, messageId):
+def diffMessageIdTodayReport(resultPath: str, messageId: str) -> str:
     text = messageId + '\n'
     e = '[' + messageId + ']\n'
     today = strDateTime()[:10]
@@ -417,7 +421,7 @@ def diffMessageIdTodayReport(resultPath, messageId):
     return text
 
 
-def headReportFromDict(out, today):
+def headReportFromDict(out: dict, today: str) -> str:
     html = '<pre>\n'
     html += '<b>MessageID                                  Count</b>\n'
     sumTotal = 0
@@ -445,7 +449,7 @@ def headReportFromDict(out, today):
     return html
 
 
-def headReport(resultsPath):
+def headReport(resultsPath: str) -> str:
     out = {}
     outToday = {}
     today = strDateTime()[:10]
@@ -509,7 +513,7 @@ def headReport(resultsPath):
     return html
 
 
-def headMessageIdReport(resultPath, messageId):
+def headMessageIdReport(resultPath: str, messageId: str) -> str:
     text = messageId + '\n'
     e = '[' + messageId + ']\n'
     for filename in sorted(glob.glob(resultPath + '/*')):
@@ -534,7 +538,7 @@ def headMessageIdReport(resultPath, messageId):
     return text
 
 
-def headMessageIdTodayReport(resultPath, messageId):
+def headMessageIdTodayReport(resultPath: str, messageId: str) -> str:
     text = messageId + '\n'
     e = '[' + messageId + ']\n'
     today = strDateTime()[:10]
@@ -565,7 +569,7 @@ def headMessageIdTodayReport(resultPath, messageId):
     return text
 
 
-def timeReport(resultPath):
+def timeReport(resultPath: str) -> str:
     html = '<html><head><title>Time report</title></head><body>\n'
     html += '<h1>Time report</h1>\n'
     html += '<pre>\n'
@@ -634,7 +638,7 @@ def timeReport(resultPath):
     return html
 
 
-def check_library_report(result_path, message_id):
+def check_library_report(result_path: str, message_id: str) -> str:
     if message_id not in ('checkLibraryNoReturn', 'checkLibraryFunction', 'checkLibraryUseIgnore'):
         error_message = 'Invalid value ' + message_id + ' for message_id parameter.'
         print(error_message)
@@ -669,18 +673,18 @@ def check_library_report(result_path, message_id):
             if not info_messages:
                 continue
             if line.endswith('[' + message_id + ']\n'):
-                if message_id is 'checkLibraryFunction':
+                if message_id == 'checkLibraryFunction':
                     function_name = line[(line.find('for function ') + len('for function ')):line.rfind('[') - 1]
                 else:
                     function_name = line[(line.find(': Function ') + len(': Function ')):line.rfind('should have') - 1]
                 function_counts[function_name] = function_counts.setdefault(function_name, 0) + 1
 
     function_details_list = []
-    for function_name, count in sorted(function_counts.items(), key=operator.itemgetter(1), reverse=True):
+    for function_name, count in sorted(list(function_counts.items()), key=operator.itemgetter(1), reverse=True):
         if len(function_details_list) >= functions_shown_max:
             break
         function_details_list.append(str(count).rjust(column_widths[0]) + ' ' +
-                '<a href="check_library-' + urllib.quote_plus(function_name) + '">' + function_name + '</a>\n')
+                '<a href="check_library-' + urllib.parse.quote_plus(function_name) + '">' + function_name + '</a>\n')
 
     html += ''.join(function_details_list)
     html += '</pre>\n'
@@ -690,9 +694,9 @@ def check_library_report(result_path, message_id):
 
 
 # Lists all checkLibrary* messages regarding the given function name
-def check_library_function_name(result_path, function_name):
+def check_library_function_name(result_path: str, function_name: str) -> str:
     print('check_library_function_name')
-    function_name = urllib.unquote_plus(function_name)
+    function_name = urllib.parse.unquote_plus(function_name)
     output_lines_list = []
     for filename in glob.glob(result_path + '/*'):
         if not os.path.isfile(filename):
@@ -722,7 +726,8 @@ def check_library_function_name(result_path, function_name):
     return ''.join(output_lines_list)
 
 
-def sendAll(connection, data):
+def sendAll(connection: socket.socket, text: str) -> None:
+    data = text.encode('utf-8', 'ignore')
     while data:
         num = connection.send(data)
         if num < len(data):
@@ -731,7 +736,7 @@ def sendAll(connection, data):
             data = None
 
 
-def httpGetResponse(connection, data, contentType):
+def httpGetResponse(connection: socket.socket, data: str, contentType: str) -> None:
     resp = 'HTTP/1.1 200 OK\r\n'
     resp += 'Connection: close\r\n'
     resp += 'Content-length: ' + str(len(data)) + '\r\n'
@@ -741,7 +746,7 @@ def httpGetResponse(connection, data, contentType):
 
 
 class HttpClientThread(Thread):
-    def __init__(self, connection, cmd, resultPath, latestResults):
+    def __init__(self, connection: socket.socket, cmd: str, resultPath: str, latestResults: list) -> None:
         Thread.__init__(self)
         self.connection = connection
         self.cmd = cmd[:cmd.find('\n')]
@@ -812,7 +817,7 @@ class HttpClientThread(Thread):
                 filename = resultPath + '/' + url
                 if not os.path.isfile(filename):
                     print('HTTP/1.1 404 Not Found')
-                    self.connection.send('HTTP/1.1 404 Not Found\r\n\r\n')
+                    self.connection.send(b'HTTP/1.1 404 Not Found\r\n\r\n')
                 else:
                     f = open(filename, 'rt')
                     data = f.read()
@@ -823,7 +828,7 @@ class HttpClientThread(Thread):
             self.connection.close()
 
 
-def server(server_address_port, packages, packageIndex, resultPath):
+def server(server_address_port: int, packages: list, packageIndex: int, resultPath: str) -> None:
     socket.setdefaulttimeout(30)
     sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
@@ -845,10 +850,15 @@ def server(server_address_port, packages, packageIndex, resultPath):
         print('[' + strDateTime() + '] waiting for a connection')
         connection, client_address = sock.accept()
         try:
-            cmd = connection.recv(128)
+            bytes_received = connection.recv(128)
+            cmd = bytes_received.decode('utf-8', 'ignore')
         except socket.error:
             connection.close()
             continue
+        except UnicodeDecodeError as e:
+            connection.close()
+            print('Error: Decoding failed: ' + str(e))
+            continue
         if cmd.find('\n') < 1:
             continue
         firstLine = cmd[:cmd.find('\n')]
@@ -861,7 +871,7 @@ def server(server_address_port, packages, packageIndex, resultPath):
         elif cmd == 'GetCppcheckVersions\n':
             reply = 'head ' + OLD_VERSION
             print('[' + strDateTime() + '] GetCppcheckVersions: ' + reply)
-            connection.send(reply)
+            connection.send(reply.encode('utf-8', 'ignore'))
             connection.close()
         elif cmd == 'get\n':
             pkg = packages[packageIndex]
@@ -874,19 +884,25 @@ def server(server_address_port, packages, packageIndex, resultPath):
             f.close()
 
             print('[' + strDateTime() + '] get:' + pkg)
-            connection.send(pkg)
+            connection.send(pkg.encode('utf-8', 'ignore'))
             connection.close()
         elif cmd.startswith('write\nftp://'):
             # read data
             data = cmd[cmd.find('ftp'):]
             try:
-                t = 0
+                t = 0.0
                 max_data_size = 2 * 1024 * 1024
                 while (len(data) < max_data_size) and (not data.endswith('\nDONE')) and (t < 10):
-                    d = connection.recv(1024)
-                    if d:
-                        t = 0
-                        data += d
+                    bytes_received = connection.recv(1024)
+                    if bytes_received:
+                        try:
+                            text_received = bytes_received.decode('utf-8', 'ignore')
+                        except UnicodeDecodeError as e:
+                            print('Error: Decoding failed: ' + str(e))
+                            data = ''
+                            break
+                        t = 0.0
+                        data += text_received
                     else:
                         time.sleep(0.2)
                         t += 0.2
@@ -941,13 +957,19 @@ def server(server_address_port, packages, packageIndex, resultPath):
             # read data
             data = cmd[11:]
             try:
-                t = 0
+                t = 0.0
                 max_data_size = 1024 * 1024
                 while (len(data) < max_data_size) and (not data.endswith('\nDONE')) and (t < 10):
-                    d = connection.recv(1024)
-                    if d:
-                        t = 0
-                        data += d
+                    bytes_received = connection.recv(1024)
+                    if bytes_received:
+                        try:
+                            text_received = bytes_received.decode('utf-8', 'ignore')
+                        except UnicodeDecodeError as e:
+                            print('Error: Decoding failed: ' + str(e))
+                            data = ''
+                            break
+                        t = 0.0
+                        data += text_received
                     else:
                         time.sleep(0.2)
                         t += 0.2
@@ -978,7 +1000,7 @@ def server(server_address_port, packages, packageIndex, resultPath):
                 f.write(strDateTime() + '\n' + data)
         elif cmd == 'getPackagesCount\n':
             packages_count = str(len(packages))
-            connection.send(packages_count)
+            connection.send(packages_count.encode('utf-8', 'ignore'))
             connection.close()
             print('[' + strDateTime() + '] getPackagesCount: ' + packages_count)
             continue
@@ -986,7 +1008,7 @@ def server(server_address_port, packages, packageIndex, resultPath):
             request_idx = abs(int(cmd[len('getPackageIdx:'):]))
             if request_idx < len(packages):
                 pkg = packages[request_idx]
-                connection.send(pkg)
+                connection.send(pkg.encode('utf-8', 'ignore'))
                 connection.close()
                 print('[' + strDateTime() + '] getPackageIdx: ' + pkg)
             else:
@@ -1032,4 +1054,3 @@ if __name__ == "__main__":
         server(server_address_port, packages, packageIndex, resultPath)
     except socket.timeout:
         print('Timeout!')
-
diff --git a/tools/test/start_donate_cpu_server_test_local.sh b/tools/test/start_donate_cpu_server_test_local.sh
index 796c08745..b53124707 100644
--- a/tools/test/start_donate_cpu_server_test_local.sh
+++ b/tools/test/start_donate_cpu_server_test_local.sh
@@ -27,5 +27,5 @@ fi
 
 while :
 do
-    python "${cppcheck_tools_path}/donate-cpu-server.py" --test
+    "${cppcheck_tools_path}/donate-cpu-server.py" --test
 done