daca2: refactoring. gather list of archives from ftp server then disconnect.
This commit is contained in:
Daniel Marjamäki
parent
a305698442
commit
554790afcf
126
tools/daca2.py
126
tools/daca2.py
|
|
@ -15,6 +15,7 @@ import glob
|
||||||
import os
|
import os
|
||||||
import socket
|
import socket
|
||||||
import datetime
|
import datetime
|
||||||
|
import time
|
||||||
|
|
||||||
FTPSERVER = 'ftp.sunet.se'
|
FTPSERVER = 'ftp.sunet.se'
|
||||||
FTPPATH = '/pub/Linux/distributions/Debian/debian/pool/main/'
|
FTPPATH = '/pub/Linux/distributions/Debian/debian/pool/main/'
|
||||||
|
|
@ -30,50 +31,17 @@ def removeAllExceptResults():
|
||||||
os.remove(filename)
|
os.remove(filename)
|
||||||
|
|
||||||
|
|
||||||
def generateDaca2Report(allfolders):
|
def scanarchive(fullpath):
|
||||||
filename = os.path.expanduser('~/daca2/daca2.html')
|
results = open('results.txt', 'at')
|
||||||
f = open(filename, 'wt')
|
results.write(fullpath + '\n')
|
||||||
f.write('<html>\n')
|
results.close()
|
||||||
f.write('<body>\n')
|
|
||||||
f.write('<h1>DACA2</h1>\n')
|
|
||||||
|
|
||||||
f.write('<h2>All folders</h2>\n')
|
filename = fullpath[fullpath.rfind('/') + 1:]
|
||||||
for folder in allfolders:
|
|
||||||
results = os.path.expanduser('~/daca2/' + folder + '/results.txt')
|
|
||||||
if os.path.isfile(results):
|
|
||||||
f.write(
|
|
||||||
'<a href="' +
|
|
||||||
folder +
|
|
||||||
'/results.txt">' +
|
|
||||||
folder +
|
|
||||||
'</a><br>\n')
|
|
||||||
else:
|
|
||||||
f.write(folder + '<br>\n')
|
|
||||||
f.write('</body>\n')
|
|
||||||
f.write('</html>\n')
|
|
||||||
f.close()
|
|
||||||
|
|
||||||
|
|
||||||
def scanpackage(package, f):
|
|
||||||
print('package:' + package)
|
|
||||||
filename = None
|
|
||||||
path = FTPPATH + FOLDER + '/' + package
|
|
||||||
try:
|
|
||||||
for s in f.nlst(path):
|
|
||||||
if s[-12:] == '.orig.tar.gz':
|
|
||||||
filename = s
|
|
||||||
except socket.error:
|
|
||||||
pass
|
|
||||||
except ftplib.error_temp:
|
|
||||||
pass
|
|
||||||
|
|
||||||
if not filename:
|
|
||||||
return
|
|
||||||
|
|
||||||
fullpath = 'ftp://' + FTPSERVER + path + '/' + filename
|
|
||||||
subprocess.call(['wget', fullpath])
|
subprocess.call(['wget', fullpath])
|
||||||
subprocess.call(['tar', 'xzvf', filename])
|
if filename[-3:] == '.gz':
|
||||||
subprocess.call(['rm', filename])
|
subprocess.call(['tar', 'xzvf', filename])
|
||||||
|
elif filename[-4:] == '.bz2':
|
||||||
|
subprocess.call(['tar', 'xjvf', filename])
|
||||||
|
|
||||||
dirname = None
|
dirname = None
|
||||||
for s in glob.glob(filename[:2] + '*'):
|
for s in glob.glob(filename[:2] + '*'):
|
||||||
|
|
@ -95,13 +63,53 @@ def scanpackage(package, f):
|
||||||
comm = p.communicate()
|
comm = p.communicate()
|
||||||
|
|
||||||
results = open('results.txt', 'at')
|
results = open('results.txt', 'at')
|
||||||
results.write(fullpath + '\n')
|
|
||||||
results.write(comm[1] + '\n')
|
results.write(comm[1] + '\n')
|
||||||
results.close()
|
results.close()
|
||||||
|
|
||||||
# remove all files/folders except results.txt
|
if len(sys.argv) == 2:
|
||||||
removeAllExceptResults()
|
FOLDER = sys.argv[1]
|
||||||
|
|
||||||
|
print('Connect to ' + FTPSERVER)
|
||||||
|
f = ftplib.FTP(FTPSERVER)
|
||||||
|
f.login()
|
||||||
|
|
||||||
|
print('Get package list in folder ' + FOLDER)
|
||||||
|
packages = f.nlst(FTPPATH + FOLDER)
|
||||||
|
|
||||||
|
archives = []
|
||||||
|
for package in packages:
|
||||||
|
filename = None
|
||||||
|
path = FTPPATH + FOLDER + '/' + package
|
||||||
|
try:
|
||||||
|
files = f.nlst(path)
|
||||||
|
|
||||||
|
for s in files:
|
||||||
|
if s.find('.orig.tar.') > 0:
|
||||||
|
filename = s
|
||||||
|
|
||||||
|
if not filename:
|
||||||
|
for s in files:
|
||||||
|
if s.find('.tar.') > 0:
|
||||||
|
filename = s
|
||||||
|
except socket.error:
|
||||||
|
print('socket.error')
|
||||||
|
pass
|
||||||
|
except ftplib.error_temp:
|
||||||
|
print('ftplib.error_temp')
|
||||||
|
pass
|
||||||
|
except EOFError:
|
||||||
|
print('EOFError')
|
||||||
|
pass
|
||||||
|
|
||||||
|
if not filename:
|
||||||
|
print('archive not found for ' + package)
|
||||||
|
else:
|
||||||
|
archives.append(package + '/' + filename)
|
||||||
|
|
||||||
|
print('Disconnect')
|
||||||
|
f.quit()
|
||||||
|
|
||||||
|
time.sleep(30)
|
||||||
|
|
||||||
workdir = os.path.expanduser('~/daca2/')
|
workdir = os.path.expanduser('~/daca2/')
|
||||||
|
|
||||||
|
|
@ -111,40 +119,22 @@ if not os.path.isfile(workdir + 'suppressions.txt'):
|
||||||
suppressions.write('\n')
|
suppressions.write('\n')
|
||||||
suppressions.close()
|
suppressions.close()
|
||||||
|
|
||||||
if len(sys.argv) == 2:
|
|
||||||
FOLDER = sys.argv[1]
|
|
||||||
|
|
||||||
print('~/daca2/' + FOLDER)
|
print('~/daca2/' + FOLDER)
|
||||||
if not os.path.isdir(workdir + FOLDER):
|
if not os.path.isdir(workdir + FOLDER):
|
||||||
os.makedirs(workdir + FOLDER)
|
os.makedirs(workdir + FOLDER)
|
||||||
os.chdir(workdir + FOLDER)
|
os.chdir(workdir + FOLDER)
|
||||||
if os.path.isfile('results.txt'):
|
|
||||||
os.remove('results.txt')
|
|
||||||
|
|
||||||
print('Connect to ' + FTPSERVER)
|
|
||||||
f = ftplib.FTP(FTPSERVER)
|
|
||||||
f.login()
|
|
||||||
|
|
||||||
print('Get package list in folder ' + FOLDER)
|
|
||||||
packages = f.nlst(FTPPATH + FOLDER)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# remove all files/folders except results.txt
|
|
||||||
removeAllExceptResults()
|
|
||||||
|
|
||||||
results = open('results.txt', 'wt')
|
results = open('results.txt', 'wt')
|
||||||
results.write('DATE ' + str(datetime.date.today()) + '\n\n')
|
results.write('DATE ' + str(datetime.date.today()) + '\n\n')
|
||||||
results.close()
|
results.close()
|
||||||
|
|
||||||
for package in packages:
|
for archive in archives:
|
||||||
scanpackage(package, f)
|
# remove all files/folders except results.txt
|
||||||
except EOFError:
|
removeAllExceptResults()
|
||||||
pass
|
|
||||||
|
scanarchive('ftp://' + FTPSERVER + FTPPATH + FOLDER + '/' + archive)
|
||||||
|
|
||||||
try:
|
|
||||||
generateDaca2Report(f.nlst(FTPPATH))
|
|
||||||
except socket.error:
|
|
||||||
pass
|
|
||||||
except EOFError:
|
except EOFError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue