Reduce memory footprint
This commit is contained in:
parent
4223b7e822
commit
0deff10e66
|
@ -96,9 +96,10 @@ def downloadSpeed(files, quiet=False):
|
||||||
while len(finished) < total_files:
|
while len(finished) < total_files:
|
||||||
thread = q.get(True)
|
thread = q.get(True)
|
||||||
thread.join()
|
thread.join()
|
||||||
finished.append(thread.get_result())
|
finished.append(len(thread.result))
|
||||||
|
thread.result = ''
|
||||||
|
|
||||||
q = Queue(3)
|
q = Queue(6)
|
||||||
start = time.time()
|
start = time.time()
|
||||||
prod_thread = threading.Thread(target=producer, args=(q, files))
|
prod_thread = threading.Thread(target=producer, args=(q, files))
|
||||||
cons_thread = threading.Thread(target=consumer, args=(q, len(files)))
|
cons_thread = threading.Thread(target=consumer, args=(q, len(files)))
|
||||||
|
@ -106,7 +107,7 @@ def downloadSpeed(files, quiet=False):
|
||||||
cons_thread.start()
|
cons_thread.start()
|
||||||
prod_thread.join()
|
prod_thread.join()
|
||||||
cons_thread.join()
|
cons_thread.join()
|
||||||
return (len(''.join(finished))/(time.time()-start))
|
return (sum(finished)/(time.time()-start))
|
||||||
|
|
||||||
|
|
||||||
class FilePutter(threading.Thread):
|
class FilePutter(threading.Thread):
|
||||||
|
@ -153,9 +154,10 @@ def uploadSpeed(url, sizes, quiet=False):
|
||||||
while len(finished) < total_sizes:
|
while len(finished) < total_sizes:
|
||||||
thread = q.get(True)
|
thread = q.get(True)
|
||||||
thread.join()
|
thread.join()
|
||||||
finished.append(thread.get_result())
|
finished.append(len(thread.result))
|
||||||
|
thread.result = ''
|
||||||
|
|
||||||
q = Queue(9)
|
q = Queue(6)
|
||||||
start = time.time()
|
start = time.time()
|
||||||
prod_thread = threading.Thread(target=producer, args=(q, sizes))
|
prod_thread = threading.Thread(target=producer, args=(q, sizes))
|
||||||
cons_thread = threading.Thread(target=consumer, args=(q, len(sizes)))
|
cons_thread = threading.Thread(target=consumer, args=(q, len(sizes)))
|
||||||
|
@ -163,7 +165,7 @@ def uploadSpeed(url, sizes, quiet=False):
|
||||||
cons_thread.start()
|
cons_thread.start()
|
||||||
prod_thread.join()
|
prod_thread.join()
|
||||||
cons_thread.join()
|
cons_thread.join()
|
||||||
return (len(''.join(finished))/(time.time()-start))
|
return (sum(finished)/(time.time()-start))
|
||||||
|
|
||||||
|
|
||||||
def getAttributesByTagName(dom, tagName):
|
def getAttributesByTagName(dom, tagName):
|
||||||
|
|
|
@ -89,9 +89,10 @@ def downloadSpeed(files, quiet=False):
|
||||||
while len(finished) < total_files:
|
while len(finished) < total_files:
|
||||||
thread = q.get(True)
|
thread = q.get(True)
|
||||||
thread.join()
|
thread.join()
|
||||||
finished.append(thread.get_result())
|
finished.append(len(thread.result))
|
||||||
|
thread.result = ''
|
||||||
|
|
||||||
q = Queue(3)
|
q = Queue(6)
|
||||||
start = time.time()
|
start = time.time()
|
||||||
prod_thread = threading.Thread(target=producer, args=(q, files))
|
prod_thread = threading.Thread(target=producer, args=(q, files))
|
||||||
cons_thread = threading.Thread(target=consumer, args=(q, len(files)))
|
cons_thread = threading.Thread(target=consumer, args=(q, len(files)))
|
||||||
|
@ -99,8 +100,7 @@ def downloadSpeed(files, quiet=False):
|
||||||
cons_thread.start()
|
cons_thread.start()
|
||||||
prod_thread.join()
|
prod_thread.join()
|
||||||
cons_thread.join()
|
cons_thread.join()
|
||||||
return (len(b''.join([chunk if isinstance(chunk, bytes) else chunk.encode()
|
return (sum(finished)/(time.time()-start))
|
||||||
for chunk in finished]))/(time.time()-start))
|
|
||||||
|
|
||||||
|
|
||||||
class FilePutter(threading.Thread):
|
class FilePutter(threading.Thread):
|
||||||
|
@ -147,9 +147,10 @@ def uploadSpeed(url, sizes, quiet=False):
|
||||||
while len(finished) < total_sizes:
|
while len(finished) < total_sizes:
|
||||||
thread = q.get(True)
|
thread = q.get(True)
|
||||||
thread.join()
|
thread.join()
|
||||||
finished.append(thread.get_result())
|
finished.append(len(thread.result))
|
||||||
|
thread.result = ''
|
||||||
|
|
||||||
q = Queue(9)
|
q = Queue(6)
|
||||||
start = time.time()
|
start = time.time()
|
||||||
prod_thread = threading.Thread(target=producer, args=(q, sizes))
|
prod_thread = threading.Thread(target=producer, args=(q, sizes))
|
||||||
cons_thread = threading.Thread(target=consumer, args=(q, len(sizes)))
|
cons_thread = threading.Thread(target=consumer, args=(q, len(sizes)))
|
||||||
|
@ -157,8 +158,7 @@ def uploadSpeed(url, sizes, quiet=False):
|
||||||
cons_thread.start()
|
cons_thread.start()
|
||||||
prod_thread.join()
|
prod_thread.join()
|
||||||
cons_thread.join()
|
cons_thread.join()
|
||||||
return (len(b''.join([chunk if isinstance(chunk, bytes) else chunk.encode()
|
return (sum(finished)/(time.time()-start))
|
||||||
for chunk in finished]))/(time.time()-start))
|
|
||||||
|
|
||||||
|
|
||||||
def getConfig():
|
def getConfig():
|
||||||
|
|
Loading…
Reference in New Issue