Donate CPU: Tried to make scripts more robust
This commit is contained in:
parent
a5f4c5d0eb
commit
7049249460
|
@ -7,6 +7,7 @@ import socket
|
||||||
import re
|
import re
|
||||||
import datetime
|
import datetime
|
||||||
import time
|
import time
|
||||||
|
from threading import Thread
|
||||||
|
|
||||||
def strDateTime():
|
def strDateTime():
|
||||||
d = datetime.date.strftime(datetime.datetime.now().date(), '%Y-%m-%d')
|
d = datetime.date.strftime(datetime.datetime.now().date(), '%Y-%m-%d')
|
||||||
|
@ -60,16 +61,17 @@ def latestReport(latestResults):
|
||||||
html += '</pre></body></html>\n'
|
html += '</pre></body></html>\n'
|
||||||
return html
|
return html
|
||||||
|
|
||||||
|
|
||||||
def sendAll(connection, data):
|
def sendAll(connection, data):
|
||||||
while data:
|
while data:
|
||||||
bytes = connection.send(data)
|
num = connection.send(data)
|
||||||
if bytes < len(data):
|
if num < len(data):
|
||||||
data = data[bytes:]
|
data = data[num:]
|
||||||
else:
|
else:
|
||||||
data = None
|
data = None
|
||||||
time.sleep(0.5)
|
|
||||||
|
|
||||||
def httpGetResponse(data, contentType):
|
|
||||||
|
def httpGetResponse(connection, data, contentType):
|
||||||
resp = 'HTTP/1.1 200 OK\n'
|
resp = 'HTTP/1.1 200 OK\n'
|
||||||
resp += 'Connection: close\n'
|
resp += 'Connection: close\n'
|
||||||
resp += 'Content-length: ' + str(len(data)) + '\n'
|
resp += 'Content-length: ' + str(len(data)) + '\n'
|
||||||
|
@ -77,6 +79,42 @@ def httpGetResponse(data, contentType):
|
||||||
resp += data
|
resp += data
|
||||||
sendAll(connection, resp)
|
sendAll(connection, resp)
|
||||||
|
|
||||||
|
|
||||||
|
class HttpClientThread(Thread):
|
||||||
|
def __init__(self, connection, cmd, latestResults):
|
||||||
|
Thread.__init__(self)
|
||||||
|
self.connection = connection
|
||||||
|
self.cmd = cmd[:cmd.find('\n')]
|
||||||
|
self.latestResults = latestResults
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
try:
|
||||||
|
cmd = self.cmd
|
||||||
|
print('[' + strDateTime() + '] ' + cmd)
|
||||||
|
if cmd.startswith('GET /latest.html '):
|
||||||
|
html = latestReport(self.latestResults)
|
||||||
|
httpGetResponse(self.connection, html, 'text/html')
|
||||||
|
else:
|
||||||
|
package = cmd[5:]
|
||||||
|
if package.find(' ') > 0:
|
||||||
|
package = package[:package.find(' ')]
|
||||||
|
filename = os.path.expanduser('~/donated-results/') + package
|
||||||
|
if not os.path.isfile(filename):
|
||||||
|
print('HTTP/1.1 404 Not Found\n\n')
|
||||||
|
connection.send('HTTP/1.1 404 Not Found\n\n')
|
||||||
|
else:
|
||||||
|
f = open(filename,'rt')
|
||||||
|
data = f.read()
|
||||||
|
f.close()
|
||||||
|
httpGetResponse(self.connection, data, 'text/plain')
|
||||||
|
print('HttpClientThread: sleep 30 seconds..')
|
||||||
|
time.sleep(30)
|
||||||
|
print('HttpClientThread: stopping')
|
||||||
|
self.connection.close()
|
||||||
|
except:
|
||||||
|
return
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
resultPath = os.path.expanduser('~/donated-results')
|
resultPath = os.path.expanduser('~/donated-results')
|
||||||
|
|
||||||
f = open('packages.txt', 'rt')
|
f = open('packages.txt', 'rt')
|
||||||
|
@ -89,9 +127,10 @@ if len(packages) == 0:
|
||||||
print('fatal: there are no packages')
|
print('fatal: there are no packages')
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
packageIndex = 0
|
packageIndex = int(time.time()) % len(packages)
|
||||||
|
|
||||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||||
server_address = ('', 8000)
|
server_address = ('', 8000)
|
||||||
sock.bind(server_address)
|
sock.bind(server_address)
|
||||||
|
|
||||||
|
@ -103,28 +142,48 @@ while True:
|
||||||
# wait for a connection
|
# wait for a connection
|
||||||
print('[' + strDateTime() + '] waiting for a connection')
|
print('[' + strDateTime() + '] waiting for a connection')
|
||||||
connection, client_address = sock.accept()
|
connection, client_address = sock.accept()
|
||||||
|
cmd = connection.recv(128)
|
||||||
try:
|
if cmd.find('\n') < 1:
|
||||||
cmd = connection.recv(16)
|
continue
|
||||||
if cmd=='get\n':
|
firstLine = cmd[:cmd.find('\n')]
|
||||||
|
if re.match('[a-zA-Z0-9./ ]+',firstLine) is None:
|
||||||
|
connection.close()
|
||||||
|
continue;
|
||||||
|
if cmd.startswith('GET /'):
|
||||||
|
newThread = HttpClientThread(connection, cmd, latestResults)
|
||||||
|
newThread.start()
|
||||||
|
elif cmd=='get\n':
|
||||||
packages[packageIndex] = packages[packageIndex].strip()
|
packages[packageIndex] = packages[packageIndex].strip()
|
||||||
print('[' + strDateTime() + '] get:' + packages[packageIndex])
|
print('[' + strDateTime() + '] get:' + packages[packageIndex])
|
||||||
connection.sendall(packages[packageIndex])
|
connection.send(packages[packageIndex])
|
||||||
packageIndex += 1
|
packageIndex += 1
|
||||||
if packageIndex >= len(packages):
|
if packageIndex >= len(packages):
|
||||||
packageIndex = 0
|
packageIndex = 0
|
||||||
elif cmd.startswith('write\n'):
|
connection.close()
|
||||||
|
elif cmd.startswith('write\nftp://'):
|
||||||
|
# read data
|
||||||
data = cmd[6:]
|
data = cmd[6:]
|
||||||
while len(data) < 1024 * 1024:
|
try:
|
||||||
|
t = 0
|
||||||
|
while (len(data) < 1024 * 1024) and (not data.endswith('\nDONE')) and (t < 10):
|
||||||
d = connection.recv(1024)
|
d = connection.recv(1024)
|
||||||
if d:
|
if d:
|
||||||
|
t = 0
|
||||||
data += d
|
data += d
|
||||||
else:
|
else:
|
||||||
break
|
time.sleep(0.2)
|
||||||
|
t += 0.2
|
||||||
|
connection.close()
|
||||||
|
except socket.error as e:
|
||||||
|
pass
|
||||||
|
|
||||||
pos = data.find('\n')
|
pos = data.find('\n')
|
||||||
if data.startswith('ftp://') and pos > 10:
|
if pos < 10:
|
||||||
|
continue
|
||||||
url = data[:pos]
|
url = data[:pos]
|
||||||
print('[' + strDateTime() + '] write:'+url)
|
print('[' + strDateTime() + '] write:'+url)
|
||||||
|
|
||||||
|
# save data
|
||||||
res = re.match(r'ftp://.*pool/main/[^/]+/([^/]+)/[^/]*tar.gz',url)
|
res = re.match(r'ftp://.*pool/main/[^/]+/([^/]+)/[^/]*tar.gz',url)
|
||||||
if res and url in packages:
|
if res and url in packages:
|
||||||
print('results added for package ' + res.group(1))
|
print('results added for package ' + res.group(1))
|
||||||
|
@ -132,30 +191,10 @@ while True:
|
||||||
f = open(filename, 'wt')
|
f = open(filename, 'wt')
|
||||||
f.write(strDateTime() + '\n' + data[pos+1:])
|
f.write(strDateTime() + '\n' + data[pos+1:])
|
||||||
f.close()
|
f.close()
|
||||||
|
# track latest added results..
|
||||||
if len(latestResults) >= 20:
|
if len(latestResults) >= 20:
|
||||||
latestResults = latestResults[1:]
|
latestResults = latestResults[1:]
|
||||||
latestResults.append(filename)
|
latestResults.append(filename)
|
||||||
elif cmd=='GET /latest.html':
|
|
||||||
print('[' + strDateTime() + '] ' + cmd)
|
|
||||||
html = latestReport(latestResults)
|
|
||||||
httpGetResponse(html, 'text/html')
|
|
||||||
elif cmd.startswith('GET /'):
|
|
||||||
print('[' + strDateTime() + '] ' + cmd)
|
|
||||||
package = cmd[5:]
|
|
||||||
if package.find(' ') > 0:
|
|
||||||
package = package[:package.find(' ')]
|
|
||||||
filename = resultPath + '/' + package
|
|
||||||
print('filename:"' + filename + '"')
|
|
||||||
if not os.path.isfile(filename):
|
|
||||||
connection.send('HTTP/1.1 404 Not Found\n\n')
|
|
||||||
print(404)
|
|
||||||
else:
|
else:
|
||||||
f = open(filename,'rt')
|
print('[' + strDateTime() + '] invalid command: ' + firstLine)
|
||||||
data = f.read()
|
|
||||||
f.close()
|
|
||||||
httpGetResponse(data, 'text/plain')
|
|
||||||
else:
|
|
||||||
print('[' + strDateTime() + '] invalid command: ' + cmd)
|
|
||||||
finally:
|
|
||||||
connection.close()
|
connection.close()
|
||||||
|
|
||||||
|
|
|
@ -139,7 +139,6 @@ def sendAll(connection, data):
|
||||||
data = data[bytes:]
|
data = data[bytes:]
|
||||||
else:
|
else:
|
||||||
data = None
|
data = None
|
||||||
time.sleep(0.5)
|
|
||||||
|
|
||||||
|
|
||||||
def uploadResults(package, results):
|
def uploadResults(package, results):
|
||||||
|
@ -148,9 +147,10 @@ def uploadResults(package, results):
|
||||||
server_address = ('cppcheck.osuosl.org', 8000)
|
server_address = ('cppcheck.osuosl.org', 8000)
|
||||||
sock.connect(server_address)
|
sock.connect(server_address)
|
||||||
try:
|
try:
|
||||||
sendAll(sock, 'write\n' + package + '\n' + results)
|
sendAll(sock, 'write\n' + package + '\n' + results + '\nDONE')
|
||||||
finally:
|
|
||||||
sock.close()
|
sock.close()
|
||||||
|
except socket.error as e:
|
||||||
|
pass
|
||||||
return package
|
return package
|
||||||
|
|
||||||
|
|
||||||
|
@ -188,5 +188,5 @@ while True:
|
||||||
print('No results to upload')
|
print('No results to upload')
|
||||||
else:
|
else:
|
||||||
uploadResults(package, results)
|
uploadResults(package, results)
|
||||||
print('Results are uploaded')
|
print('Results have been uploaded')
|
||||||
time.sleep(2)
|
time.sleep(5)
|
||||||
|
|
Loading…
Reference in New Issue