From c250e385d3fb9674c890266210ef6d8a11e5ac2d Mon Sep 17 00:00:00 2001 From: Boris Egorov Date: Mon, 28 Nov 2016 10:22:30 +0700 Subject: [PATCH 1/2] daca2: Use argparse instead of manual argument parsing --- tools/daca2.py | 64 +++++++++++++++++++++----------------------------- 1 file changed, 27 insertions(+), 37 deletions(-) diff --git a/tools/daca2.py b/tools/daca2.py index e1319646b..56b971e58 100644 --- a/tools/daca2.py +++ b/tools/daca2.py @@ -6,6 +6,7 @@ # 4. Optional: tweak FTPSERVER and FTPPATH in this script below. # 5. Run the daca2 script: python daca2.py FOLDER +import argparse import subprocess import sys import shutil @@ -176,63 +177,52 @@ def scanarchive(filepath, jobs, cpulimit): results.write('\n') results.close() -FOLDER = None -JOBS = '-j1' -REV = None -SKIP = [] -WORKDIR = os.path.expanduser('~/daca2') -CPULIMIT = None -for arg in sys.argv[1:]: - if arg[:6] == '--rev=': - REV = arg[6:] - elif arg[:2] == '-j': - JOBS = arg - elif arg.startswith('--skip='): - SKIP.append(arg[7:]) - elif arg.startswith('--workdir='): - WORKDIR = arg[10:] - elif arg.startswith('--cpulimit='): - CPULIMIT = arg[11:] - else: - FOLDER = arg +parser = argparse.ArgumentParser(description='Checks debian source code') +parser.add_argument('folder', metavar='FOLDER') +parser.add_argument('--rev') +parser.add_argument('--workdir', default='~/daca2') +parser.add_argument('-j', '--jobs', default='-j1') +parser.add_argument('--skip', default=[], action='append') +parser.add_argument('--cpulimit') -if not FOLDER: - print('no folder given') +args = parser.parse_args() + +workdir = os.path.expanduser(args.workdir) +if not os.path.isdir(workdir): + print('workdir \'' + workdir + '\' is not a folder') sys.exit(1) -if not os.path.isdir(WORKDIR): - print('workdir \'' + WORKDIR + '\' is not a folder') - sys.exit(1) +workdir = os.path.join(workdir, args.folder) +if not os.path.isdir(workdir): + os.makedirs(workdir) -archives = getpackages(FOLDER) +print(workdir) + +archives = getpackages(args.folder) if len(archives) == 0: print('failed to load packages') sys.exit(1) -if not WORKDIR.endswith('/'): - WORKDIR = WORKDIR + '/' - -print('~/daca2/' + FOLDER) -if not os.path.isdir(WORKDIR + FOLDER): - os.makedirs(WORKDIR + FOLDER) -os.chdir(WORKDIR + FOLDER) +if not os.path.isdir(workdir): + os.makedirs(workdir) +os.chdir(workdir) try: results = open('results.txt', 'wt') results.write('STARTDATE ' + str(datetime.date.today()) + '\n') results.write('STARTTIME ' + strfCurrTime('%H:%M:%S') + '\n') - if REV: - results.write('GIT-REVISION ' + REV + '\n') + if args.rev: + results.write('GIT-REVISION ' + args.rev + '\n') results.write('\n') results.close() for archive in archives: - if len(SKIP) > 0: + if len(args.skip) > 0: a = archive[:archive.rfind('/')] a = a[a.rfind('/')+1:] - if a in SKIP: + if a in args.skip: continue - scanarchive(archive, JOBS, CPULIMIT) + scanarchive(archive, args.jobs, args.cpulimit) results = open('results.txt', 'at') results.write('DATE ' + str(datetime.date.today()) + '\n') From 743dcdd669540fa9f77bb09516cc58efe6cf5829 Mon Sep 17 00:00:00 2001 From: Boris Egorov Date: Mon, 28 Nov 2016 11:13:36 +0700 Subject: [PATCH 2/2] daca2: Use logging module instead of opening/closing results file --- tools/daca2.py | 63 ++++++++++++++++++++++---------------------------- 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/tools/daca2.py b/tools/daca2.py index 56b971e58..d14169a52 100644 --- a/tools/daca2.py +++ b/tools/daca2.py @@ -14,6 +14,7 @@ import glob import os import datetime import time +import logging DEBIAN = ['ftp://ftp.se.debian.org/debian/', 'ftp://ftp.debian.org/debian/'] @@ -88,21 +89,17 @@ def removeAllExceptResults(): try: if os.path.isdir(filename): shutil.rmtree(filename, onerror=handleRemoveReadonly) - elif filename != 'results.txt': + elif filename != RESULTS_FILENAME: os.remove(filename) break except WindowsError as err: time.sleep(30) if count == 0: - f = open('results.txt','at') - f.write('Failed to cleanup ' + filename + ': ' + str(err)) - f.close() + logging.error('Failed to cleanup {}: {}'.format(filename, err)) except OSError as err: time.sleep(30) if count == 0: - f = open('results.txt','at') - f.write('Failed to cleanup ' + filename + ': ' + str(err)) - f.close() + logging.error('Failed to cleanup {}: {}'.format(filename, err)) def removeLargeFiles(path): @@ -123,26 +120,20 @@ def removeLargeFiles(path): try: os.remove(g) except OSError as err: - f = open('results.txt','at') - f.write('Failed to remove ' + g + ': ' + str(err)) - f.close() + logging.error('Failed to remove {}: {}'.format(g, err)) def strfCurrTime(fmt): return datetime.time.strftime(datetime.datetime.now().time(), fmt) def scanarchive(filepath, jobs, cpulimit): - # remove all files/folders except results.txt + # remove all files/folders except RESULTS_FILENAME removeAllExceptResults() - results = open('results.txt', 'at') - results.write(DEBIAN[0] + filepath + '\n') - results.close() + logging.info(DEBIAN[0] + filepath) if not wget(filepath): if not wget(filepath): - results = open('results.txt', 'at') - results.write('wget failed\n') - results.close() + logging.error('wget failed at {}', filepath) return filename = filepath[filepath.rfind('/') + 1:] @@ -168,14 +159,12 @@ def scanarchive(filepath, jobs, cpulimit): p = subprocess.Popen(cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE) comm = p.communicate() - results = open('results.txt', 'at') if p.returncode == 0: - results.write(comm[1] + strfCurrTime('[%H:%M]') + '\n') + logging.info(comm[1] + strfCurrTime('[%H:%M]')) elif comm[0].find('cppcheck: error: could not find or open any of the paths given.') < 0: - results.write(comm[1] + strfCurrTime('[%H:%M]') + '\n') - results.write('Exit code is not zero! Crash?\n') - results.write('\n') - results.close() + logging.error(comm[1] + strfCurrTime('[%H:%M]')) + logging.error('Exit code is not zero! Crash?\n') + parser = argparse.ArgumentParser(description='Checks debian source code') parser.add_argument('folder', metavar='FOLDER') @@ -196,11 +185,19 @@ workdir = os.path.join(workdir, args.folder) if not os.path.isdir(workdir): os.makedirs(workdir) +RESULTS_FILENAME = 'results.txt' +RESULTS_FILE = os.path.join(workdir, RESULTS_FILENAME) + +logging.basicConfig( + filename=RESULTS_FILE, + level=logging.INFO, + format='%(message)s') + print(workdir) archives = getpackages(args.folder) if len(archives) == 0: - print('failed to load packages') + logging.critical('failed to load packages') sys.exit(1) if not os.path.isdir(workdir): @@ -208,13 +205,11 @@ if not os.path.isdir(workdir): os.chdir(workdir) try: - results = open('results.txt', 'wt') - results.write('STARTDATE ' + str(datetime.date.today()) + '\n') - results.write('STARTTIME ' + strfCurrTime('%H:%M:%S') + '\n') + logging.info('STARTDATE ' + str(datetime.date.today())) + logging.info('STARTTIME ' + strfCurrTime('%H:%M:%S')) if args.rev: - results.write('GIT-REVISION ' + args.rev + '\n') - results.write('\n') - results.close() + logging.info('GIT-REVISION ' + args.rev + '\n') + logging.info('') for archive in archives: if len(args.skip) > 0: @@ -224,13 +219,11 @@ try: continue scanarchive(archive, args.jobs, args.cpulimit) - results = open('results.txt', 'at') - results.write('DATE ' + str(datetime.date.today()) + '\n') - results.write('TIME ' + strfCurrTime('%H:%M:%S') + '\n') - results.close() + logging.info('DATE {}'.format(datetime.date.today())) + logging.info('TIME {}'.format(strfCurrTime('%H:%M:%S'))) except EOFError: pass -# remove all files/folders except results.txt +# remove all files/folders except RESULTS_FILENAME removeAllExceptResults()