From 4002fcd3e3475505fa61507809d82e9792571d99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Marjam=C3=A4ki?= Date: Mon, 27 Aug 2018 18:21:16 +0200 Subject: [PATCH] Donate CPU: Improved the server, now you can see diff --- tools/donate-cpu-server.py | 171 ++++++++++++++++++++++++++++++------- 1 file changed, 141 insertions(+), 30 deletions(-) diff --git a/tools/donate-cpu-server.py b/tools/donate-cpu-server.py index 15111aebd..002e38c1a 100644 --- a/tools/donate-cpu-server.py +++ b/tools/donate-cpu-server.py @@ -8,6 +8,8 @@ import re import datetime import time from threading import Thread +import subprocess +import sys def strDateTime(): d = datetime.date.strftime(datetime.datetime.now().date(), '%Y-%m-%d') @@ -72,6 +74,92 @@ def latestReport(latestResults): return html +def diffReport(): + html = 'Diff report\n' + html += '

Diff report

\n' + html += '
\n'
+
+    out = {}
+
+    # grep '^1.84 .*\]$' donated-results/* | sed 's/.*\[\(.*\)\]/\1/' | sort | uniq -c
+    p = subprocess.Popen(['./getdiff.sh', '1.84'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    comm = p.communicate()
+    stdout = comm[0]
+    for line in stdout.split('\n'):
+        a = line.strip().split()
+        if len(a) == 2:
+            count     = a[0]
+            messageId = a[1]
+            out[messageId] = [count, '0']
+
+    # grep '^head .*\]$' donated-results/* | sed 's/.*\[\(.*\)\]/\1/' | sort | uniq -c
+    p = subprocess.Popen(['./getdiff.sh', 'head'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    comm = p.communicate()
+    stdout = comm[0]
+    for line in stdout.split('\n'):
+        a = line.strip().split()
+        if len(a) == 2:
+            count     = a[0]
+            messageId = a[1]
+            if messageId in out:
+                out[messageId][1] = count
+            else:
+                out[messageId] = ['0',count]
+
+    html += 'MessageID                           1.84    Head\n'
+    sum0 = 0
+    sum1 = 0
+    for messageId in sorted(out.keys()):
+        line = messageId + ' '
+        counts = out[messageId]
+        sum0 += int(counts[0])
+        sum1 += int(counts[1])
+        if counts[0] > 0:
+            c = counts[0]
+            while len(line) < 40 - len(c):
+                line += ' '
+            line += c + ' '
+        if counts[1] > 0:
+            c = counts[1]
+            while len(line) < 48 - len(c):
+                line += ' '
+            line += c
+        line = '' + messageId + '' + line[line.find(' '):]
+        html += line + '\n'
+
+    # Sum
+    html += '================================================\n'
+    line = ''
+    while len(line) < 40 - len(str(sum0)):
+        line += ' '
+    line += str(sum0) + ' '
+    while len(line) < 48 - len(str(sum1)):
+        line += ' '
+    line += str(sum1)
+    html += line + '\n'
+
+    return html
+
+def diffMessageIdReport(messageId):
+    text = messageId + '\n'
+    e = '[' + messageId + ']\n'
+    for filename in sorted(glob.glob(os.path.expanduser('~/donated-results/*'))):
+        url = None
+        diff = False
+        for line in open(filename,'rt'):
+            if line.startswith('ftp://'):
+                url = line
+            elif line == 'diff:\n':
+                diff = True
+            elif not diff:
+                continue
+            elif line.endswith(e):
+                if url:
+                    text += url
+                    url = None
+                text += line
+    return text
+
 def sendAll(connection, data):
     while data:
         num = connection.send(data)
@@ -91,27 +179,38 @@ def httpGetResponse(connection, data, contentType):
 
 
 class HttpClientThread(Thread):
-    def __init__(self, connection, cmd, latestResults):
+    def __init__(self, connection, cmd, resultPath, latestResults):
         Thread.__init__(self)
         self.connection = connection
         self.cmd = cmd[:cmd.find('\n')]
+        self.resultPath = resultPath
         self.latestResults = latestResults
 
     def run(self):
         try:
             cmd = self.cmd
             print('[' + strDateTime() + '] ' + cmd)
-            if cmd.startswith('GET /latest.html '):
+            res = re.match(r'GET /([a-zA-Z0-9_\-\.]+) HTTP', cmd)
+            if res is None:
+                self.connection.close()
+                return
+            url = res.group(1)
+            if url == 'latest.html':
                 html = latestReport(self.latestResults)
                 httpGetResponse(self.connection, html, 'text/html')
+            elif url == 'diff':
+                html = diffReport()
+                httpGetResponse(self.connection, html, 'text/html')
+            elif url.startswith('diff-'):
+                messageId = url[5:]
+                text = diffMessageIdReport(messageId)
+                print(text)
+                httpGetResponse(self.connection, text, 'text/plain')
             else:
-                package = cmd[5:]
-                if package.find(' ') > 0:
-                    package = package[:package.find(' ')]
-                filename = os.path.expanduser('~/donated-results/') + package
+                filename = resultPath + '/' + url
                 if not os.path.isfile(filename):
                     print('HTTP/1.1 404 Not Found')
-                    connection.send('HTTP/1.1 404 Not Found\r\n\r\n')
+                    self.connection.send('HTTP/1.1 404 Not Found\r\n\r\n')
                 else:
                     f = open(filename,'rt')
                     data = f.read()
@@ -121,30 +220,10 @@ class HttpClientThread(Thread):
             time.sleep(1)
             self.connection.close()
 
-if __name__ == "__main__":
-    resultPath = os.path.expanduser('~/donated-results')
-
-    f = open('packages.txt', 'rt')
-    packages = f.readlines()
-    f.close()
-
-    print('packages: ' + str(len(packages)))
-
-    if len(packages) == 0:
-        print('fatal: there are no packages')
-        sys.exit(1)
-
-    packageIndex = 0
-    if os.path.isfile('package-index.txt'):
-        f = open('package-index.txt', 'rt')
-        packageIndex = int(f.read())
-        if packageIndex < 0 or packageIndex >= len(packages):
-            packageIndex = 0
-        f.close()
-
+def server(server_address_port, packages, packageIndex, resultPath):
     sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-    server_address = ('', 8000)
+    server_address = ('', server_address_port)
     sock.bind(server_address)
 
     sock.listen(1)
@@ -158,6 +237,7 @@ if __name__ == "__main__":
         try:
             cmd = connection.recv(128)
         except socket.error:
+            connection.close()
             continue
         if cmd.find('\n') < 1:
             continue
@@ -166,7 +246,7 @@ if __name__ == "__main__":
             connection.close()
             continue;
         if cmd.startswith('GET /'):
-            newThread = HttpClientThread(connection, cmd, latestResults)
+            newThread = HttpClientThread(connection, cmd, resultPath, latestResults)
             newThread.start()
         elif cmd=='get\n':
             packages[packageIndex] = packages[packageIndex].strip()
@@ -217,3 +297,34 @@ if __name__ == "__main__":
         else:
             print('[' + strDateTime() + '] invalid command: ' + firstLine)
             connection.close()
+
+if __name__ == "__main__":
+    workPath = os.path.expanduser('~/daca@home')
+    os.chdir(workPath)
+    resultPath = 'donated-results'
+
+    f = open('packages.txt', 'rt')
+    packages = f.readlines()
+    f.close()
+
+    print('packages: ' + str(len(packages)))
+
+    if len(packages) == 0:
+        print('fatal: there are no packages')
+        sys.exit(1)
+
+    packageIndex = 0
+    if os.path.isfile('package-index.txt'):
+        f = open('package-index.txt', 'rt')
+        packageIndex = int(f.read())
+        if packageIndex < 0 or packageIndex >= len(packages):
+            packageIndex = 0
+        f.close()
+
+    server_address_port = 8000
+    if '--test' in sys.argv[1:]:
+        server_address_port = 8001
+
+    server(server_address_port, packages, packageIndex, resultPath)
+
+