Donate CPU: Improved the server, now you can see diff
This commit is contained in:
Daniel Marjamäki
parent
6223204a06
commit
4002fcd3e3
|
|
@ -8,6 +8,8 @@ import re
|
||||||
import datetime
|
import datetime
|
||||||
import time
|
import time
|
||||||
from threading import Thread
|
from threading import Thread
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
def strDateTime():
|
def strDateTime():
|
||||||
d = datetime.date.strftime(datetime.datetime.now().date(), '%Y-%m-%d')
|
d = datetime.date.strftime(datetime.datetime.now().date(), '%Y-%m-%d')
|
||||||
|
|
@ -72,6 +74,92 @@ def latestReport(latestResults):
|
||||||
return html
|
return html
|
||||||
|
|
||||||
|
|
||||||
|
def diffReport():
|
||||||
|
html = '<html><head><title>Diff report</title></head><body>\n'
|
||||||
|
html += '<h1>Diff report</h1>\n'
|
||||||
|
html += '<pre>\n'
|
||||||
|
|
||||||
|
out = {}
|
||||||
|
|
||||||
|
# grep '^1.84 .*\]$' donated-results/* | sed 's/.*\[\(.*\)\]/\1/' | sort | uniq -c
|
||||||
|
p = subprocess.Popen(['./getdiff.sh', '1.84'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
|
comm = p.communicate()
|
||||||
|
stdout = comm[0]
|
||||||
|
for line in stdout.split('\n'):
|
||||||
|
a = line.strip().split()
|
||||||
|
if len(a) == 2:
|
||||||
|
count = a[0]
|
||||||
|
messageId = a[1]
|
||||||
|
out[messageId] = [count, '0']
|
||||||
|
|
||||||
|
# grep '^head .*\]$' donated-results/* | sed 's/.*\[\(.*\)\]/\1/' | sort | uniq -c
|
||||||
|
p = subprocess.Popen(['./getdiff.sh', 'head'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
|
comm = p.communicate()
|
||||||
|
stdout = comm[0]
|
||||||
|
for line in stdout.split('\n'):
|
||||||
|
a = line.strip().split()
|
||||||
|
if len(a) == 2:
|
||||||
|
count = a[0]
|
||||||
|
messageId = a[1]
|
||||||
|
if messageId in out:
|
||||||
|
out[messageId][1] = count
|
||||||
|
else:
|
||||||
|
out[messageId] = ['0',count]
|
||||||
|
|
||||||
|
html += '<b>MessageID 1.84 Head</b>\n'
|
||||||
|
sum0 = 0
|
||||||
|
sum1 = 0
|
||||||
|
for messageId in sorted(out.keys()):
|
||||||
|
line = messageId + ' '
|
||||||
|
counts = out[messageId]
|
||||||
|
sum0 += int(counts[0])
|
||||||
|
sum1 += int(counts[1])
|
||||||
|
if counts[0] > 0:
|
||||||
|
c = counts[0]
|
||||||
|
while len(line) < 40 - len(c):
|
||||||
|
line += ' '
|
||||||
|
line += c + ' '
|
||||||
|
if counts[1] > 0:
|
||||||
|
c = counts[1]
|
||||||
|
while len(line) < 48 - len(c):
|
||||||
|
line += ' '
|
||||||
|
line += c
|
||||||
|
line = '<a href="diff-' + messageId + '">' + messageId + '</a>' + line[line.find(' '):]
|
||||||
|
html += line + '\n'
|
||||||
|
|
||||||
|
# Sum
|
||||||
|
html += '================================================\n'
|
||||||
|
line = ''
|
||||||
|
while len(line) < 40 - len(str(sum0)):
|
||||||
|
line += ' '
|
||||||
|
line += str(sum0) + ' '
|
||||||
|
while len(line) < 48 - len(str(sum1)):
|
||||||
|
line += ' '
|
||||||
|
line += str(sum1)
|
||||||
|
html += line + '\n'
|
||||||
|
|
||||||
|
return html
|
||||||
|
|
||||||
|
def diffMessageIdReport(messageId):
|
||||||
|
text = messageId + '\n'
|
||||||
|
e = '[' + messageId + ']\n'
|
||||||
|
for filename in sorted(glob.glob(os.path.expanduser('~/donated-results/*'))):
|
||||||
|
url = None
|
||||||
|
diff = False
|
||||||
|
for line in open(filename,'rt'):
|
||||||
|
if line.startswith('ftp://'):
|
||||||
|
url = line
|
||||||
|
elif line == 'diff:\n':
|
||||||
|
diff = True
|
||||||
|
elif not diff:
|
||||||
|
continue
|
||||||
|
elif line.endswith(e):
|
||||||
|
if url:
|
||||||
|
text += url
|
||||||
|
url = None
|
||||||
|
text += line
|
||||||
|
return text
|
||||||
|
|
||||||
def sendAll(connection, data):
|
def sendAll(connection, data):
|
||||||
while data:
|
while data:
|
||||||
num = connection.send(data)
|
num = connection.send(data)
|
||||||
|
|
@ -91,27 +179,38 @@ def httpGetResponse(connection, data, contentType):
|
||||||
|
|
||||||
|
|
||||||
class HttpClientThread(Thread):
|
class HttpClientThread(Thread):
|
||||||
def __init__(self, connection, cmd, latestResults):
|
def __init__(self, connection, cmd, resultPath, latestResults):
|
||||||
Thread.__init__(self)
|
Thread.__init__(self)
|
||||||
self.connection = connection
|
self.connection = connection
|
||||||
self.cmd = cmd[:cmd.find('\n')]
|
self.cmd = cmd[:cmd.find('\n')]
|
||||||
|
self.resultPath = resultPath
|
||||||
self.latestResults = latestResults
|
self.latestResults = latestResults
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
try:
|
try:
|
||||||
cmd = self.cmd
|
cmd = self.cmd
|
||||||
print('[' + strDateTime() + '] ' + cmd)
|
print('[' + strDateTime() + '] ' + cmd)
|
||||||
if cmd.startswith('GET /latest.html '):
|
res = re.match(r'GET /([a-zA-Z0-9_\-\.]+) HTTP', cmd)
|
||||||
|
if res is None:
|
||||||
|
self.connection.close()
|
||||||
|
return
|
||||||
|
url = res.group(1)
|
||||||
|
if url == 'latest.html':
|
||||||
html = latestReport(self.latestResults)
|
html = latestReport(self.latestResults)
|
||||||
httpGetResponse(self.connection, html, 'text/html')
|
httpGetResponse(self.connection, html, 'text/html')
|
||||||
|
elif url == 'diff':
|
||||||
|
html = diffReport()
|
||||||
|
httpGetResponse(self.connection, html, 'text/html')
|
||||||
|
elif url.startswith('diff-'):
|
||||||
|
messageId = url[5:]
|
||||||
|
text = diffMessageIdReport(messageId)
|
||||||
|
print(text)
|
||||||
|
httpGetResponse(self.connection, text, 'text/plain')
|
||||||
else:
|
else:
|
||||||
package = cmd[5:]
|
filename = resultPath + '/' + url
|
||||||
if package.find(' ') > 0:
|
|
||||||
package = package[:package.find(' ')]
|
|
||||||
filename = os.path.expanduser('~/donated-results/') + package
|
|
||||||
if not os.path.isfile(filename):
|
if not os.path.isfile(filename):
|
||||||
print('HTTP/1.1 404 Not Found')
|
print('HTTP/1.1 404 Not Found')
|
||||||
connection.send('HTTP/1.1 404 Not Found\r\n\r\n')
|
self.connection.send('HTTP/1.1 404 Not Found\r\n\r\n')
|
||||||
else:
|
else:
|
||||||
f = open(filename,'rt')
|
f = open(filename,'rt')
|
||||||
data = f.read()
|
data = f.read()
|
||||||
|
|
@ -121,30 +220,10 @@ class HttpClientThread(Thread):
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
self.connection.close()
|
self.connection.close()
|
||||||
|
|
||||||
if __name__ == "__main__":
|
def server(server_address_port, packages, packageIndex, resultPath):
|
||||||
resultPath = os.path.expanduser('~/donated-results')
|
|
||||||
|
|
||||||
f = open('packages.txt', 'rt')
|
|
||||||
packages = f.readlines()
|
|
||||||
f.close()
|
|
||||||
|
|
||||||
print('packages: ' + str(len(packages)))
|
|
||||||
|
|
||||||
if len(packages) == 0:
|
|
||||||
print('fatal: there are no packages')
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
packageIndex = 0
|
|
||||||
if os.path.isfile('package-index.txt'):
|
|
||||||
f = open('package-index.txt', 'rt')
|
|
||||||
packageIndex = int(f.read())
|
|
||||||
if packageIndex < 0 or packageIndex >= len(packages):
|
|
||||||
packageIndex = 0
|
|
||||||
f.close()
|
|
||||||
|
|
||||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||||
server_address = ('', 8000)
|
server_address = ('', server_address_port)
|
||||||
sock.bind(server_address)
|
sock.bind(server_address)
|
||||||
|
|
||||||
sock.listen(1)
|
sock.listen(1)
|
||||||
|
|
@ -158,6 +237,7 @@ if __name__ == "__main__":
|
||||||
try:
|
try:
|
||||||
cmd = connection.recv(128)
|
cmd = connection.recv(128)
|
||||||
except socket.error:
|
except socket.error:
|
||||||
|
connection.close()
|
||||||
continue
|
continue
|
||||||
if cmd.find('\n') < 1:
|
if cmd.find('\n') < 1:
|
||||||
continue
|
continue
|
||||||
|
|
@ -166,7 +246,7 @@ if __name__ == "__main__":
|
||||||
connection.close()
|
connection.close()
|
||||||
continue;
|
continue;
|
||||||
if cmd.startswith('GET /'):
|
if cmd.startswith('GET /'):
|
||||||
newThread = HttpClientThread(connection, cmd, latestResults)
|
newThread = HttpClientThread(connection, cmd, resultPath, latestResults)
|
||||||
newThread.start()
|
newThread.start()
|
||||||
elif cmd=='get\n':
|
elif cmd=='get\n':
|
||||||
packages[packageIndex] = packages[packageIndex].strip()
|
packages[packageIndex] = packages[packageIndex].strip()
|
||||||
|
|
@ -217,3 +297,34 @@ if __name__ == "__main__":
|
||||||
else:
|
else:
|
||||||
print('[' + strDateTime() + '] invalid command: ' + firstLine)
|
print('[' + strDateTime() + '] invalid command: ' + firstLine)
|
||||||
connection.close()
|
connection.close()
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
workPath = os.path.expanduser('~/daca@home')
|
||||||
|
os.chdir(workPath)
|
||||||
|
resultPath = 'donated-results'
|
||||||
|
|
||||||
|
f = open('packages.txt', 'rt')
|
||||||
|
packages = f.readlines()
|
||||||
|
f.close()
|
||||||
|
|
||||||
|
print('packages: ' + str(len(packages)))
|
||||||
|
|
||||||
|
if len(packages) == 0:
|
||||||
|
print('fatal: there are no packages')
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
packageIndex = 0
|
||||||
|
if os.path.isfile('package-index.txt'):
|
||||||
|
f = open('package-index.txt', 'rt')
|
||||||
|
packageIndex = int(f.read())
|
||||||
|
if packageIndex < 0 or packageIndex >= len(packages):
|
||||||
|
packageIndex = 0
|
||||||
|
f.close()
|
||||||
|
|
||||||
|
server_address_port = 8000
|
||||||
|
if '--test' in sys.argv[1:]:
|
||||||
|
server_address_port = 8001
|
||||||
|
|
||||||
|
server(server_address_port, packages, packageIndex, resultPath)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue