Donate CPU: Tried to make scripts more robust

tools/donate-cpu-server.py

@@ -7,6 +7,7 @@ import socket
 import re
 import datetime
 import time
+from threading import Thread
 
 def strDateTime():
     d = datetime.date.strftime(datetime.datetime.now().date(), '%Y-%m-%d')
@@ -60,16 +61,17 @@ def latestReport(latestResults):
     html += '</pre></body></html>\n'
     return html
 
+
 def sendAll(connection, data):
     while data:
-        bytes = connection.send(data)
+        num = connection.send(data)
-        if bytes < len(data):
+        if num < len(data):
-            data = data[bytes:]
+            data = data[num:]
         else:
             data = None
-    time.sleep(0.5)
 
-def httpGetResponse(data, contentType):
+
+def httpGetResponse(connection, data, contentType):
     resp = 'HTTP/1.1 200 OK\n'
     resp += 'Connection: close\n'
     resp += 'Content-length: ' + str(len(data)) + '\n'
@@ -77,54 +79,111 @@ def httpGetResponse(data, contentType):
     resp += data
     sendAll(connection, resp)
 
-resultPath = os.path.expanduser('~/donated-results')
 
-f = open('packages.txt', 'rt')
+class HttpClientThread(Thread):
-packages = f.readlines()
+    def __init__(self, connection, cmd, latestResults):
-f.close()
+        Thread.__init__(self)
+        self.connection = connection
+        self.cmd = cmd[:cmd.find('\n')]
+        self.latestResults = latestResults
 
-print('packages:'+str(len(packages)))
+    def run(self):
+        try:
+            cmd = self.cmd
+            print('[' + strDateTime() + '] ' + cmd)
+            if cmd.startswith('GET /latest.html '):
+                html = latestReport(self.latestResults)
+                httpGetResponse(self.connection, html, 'text/html')
+            else:
+                package = cmd[5:]
+                if package.find(' ') > 0:
+                    package = package[:package.find(' ')]
+                filename = os.path.expanduser('~/donated-results/') + package
+                if not os.path.isfile(filename):
+                    print('HTTP/1.1 404 Not Found\n\n')
+                    connection.send('HTTP/1.1 404 Not Found\n\n')
+                else:
+                    f = open(filename,'rt')
+                    data = f.read()
+                    f.close()
+                    httpGetResponse(self.connection, data, 'text/plain')
+            print('HttpClientThread: sleep 30 seconds..')
+            time.sleep(30)
+            print('HttpClientThread: stopping')
+            self.connection.close()
+        except:
+            return
 
-if len(packages) == 0:
+if __name__ == "__main__":
+    resultPath = os.path.expanduser('~/donated-results')
+
+    f = open('packages.txt', 'rt')
+    packages = f.readlines()
+    f.close()
+
+    print('packages: ' + str(len(packages)))
+
+    if len(packages) == 0:
         print('fatal: there are no packages')
         sys.exit(1)
 
-packageIndex = 0
+    packageIndex = int(time.time()) % len(packages)
 
-sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-server_address = ('', 8000)
+    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-sock.bind(server_address)
+    server_address = ('', 8000)
+    sock.bind(server_address)
 
-sock.listen(1)
+    sock.listen(1)
 
-latestResults = []
+    latestResults = []
 
-while True:
+    while True:
         # wait for a connection
         print('[' + strDateTime() + '] waiting for a connection')
         connection, client_address = sock.accept()
-
+        cmd = connection.recv(128)
-    try:
+        if cmd.find('\n') < 1:
-        cmd = connection.recv(16)
+            continue
-        if cmd=='get\n':
+        firstLine = cmd[:cmd.find('\n')]
+        if re.match('[a-zA-Z0-9./ ]+',firstLine) is None:
+            connection.close()
+            continue;
+        if cmd.startswith('GET /'):
+            newThread = HttpClientThread(connection, cmd, latestResults)
+            newThread.start()
+        elif cmd=='get\n':
             packages[packageIndex] = packages[packageIndex].strip()
             print('[' + strDateTime() + '] get:' + packages[packageIndex])
-            connection.sendall(packages[packageIndex])
+            connection.send(packages[packageIndex])
             packageIndex += 1
             if packageIndex >= len(packages):
                 packageIndex = 0
-        elif cmd.startswith('write\n'):
+            connection.close()
+        elif cmd.startswith('write\nftp://'):
+            # read data
             data = cmd[6:]
-            while len(data) < 1024 * 1024:
+            try:
+                t = 0
+                while (len(data) < 1024 * 1024) and (not data.endswith('\nDONE')) and (t < 10):
                     d = connection.recv(1024)
                     if d:
+                        t = 0
                         data += d
                     else:
-                    break
+                        time.sleep(0.2)
+                        t += 0.2
+                connection.close()
+            except socket.error as e:
+                pass
+
             pos = data.find('\n')
-            if data.startswith('ftp://') and pos > 10:
+            if pos < 10:
+                continue
             url = data[:pos]
             print('[' + strDateTime() + '] write:'+url)
+
+            # save data
             res = re.match(r'ftp://.*pool/main/[^/]+/([^/]+)/[^/]*tar.gz',url)
             if res and url in packages:
                 print('results added for package ' + res.group(1))
@@ -132,30 +191,10 @@ while True:
                 f = open(filename, 'wt')
                 f.write(strDateTime() + '\n' + data[pos+1:])
                 f.close()
+                # track latest added results..
                 if len(latestResults) >= 20:
                     latestResults = latestResults[1:]
                 latestResults.append(filename)
-        elif cmd=='GET /latest.html':
-            print('[' + strDateTime() + '] ' + cmd)
-            html = latestReport(latestResults)
-            httpGetResponse(html, 'text/html')
-        elif cmd.startswith('GET /'):
-            print('[' + strDateTime() + '] ' + cmd)
-            package = cmd[5:]
-            if package.find(' ') > 0:
-                package = package[:package.find(' ')]
-            filename = resultPath + '/' + package
-            print('filename:"' + filename + '"')
-            if not os.path.isfile(filename):
-                connection.send('HTTP/1.1 404 Not Found\n\n')
-                print(404)
         else:
-                f = open(filename,'rt')
+            print('[' + strDateTime() + '] invalid command: ' + firstLine)
-                data = f.read()
-                f.close()
-                httpGetResponse(data, 'text/plain')
-        else:
-            print('[' + strDateTime() + '] invalid command: ' + cmd)
-    finally:
             connection.close()
-

tools/donate-cpu.py

@@ -139,7 +139,6 @@ def sendAll(connection, data):
             data = data[bytes:]
         else:
             data = None
-    time.sleep(0.5)
 
 
 def uploadResults(package, results):
@@ -148,9 +147,10 @@ def uploadResults(package, results):
     server_address = ('cppcheck.osuosl.org', 8000)
     sock.connect(server_address)
     try:
-        sendAll(sock, 'write\n' + package + '\n' + results)
+        sendAll(sock, 'write\n' + package + '\n' + results + '\nDONE')
-    finally:
         sock.close()
+    except socket.error as e:
+        pass
     return package
 
 
@@ -188,5 +188,5 @@ while True:
         print('No results to upload')
     else:
         uploadResults(package, results)
-        print('Results are uploaded')
+        print('Results have been uploaded')
-        time.sleep(2)
+        time.sleep(5)