From eed71e755fca18ebc271b3a3b72e70c0822f55d2 Mon Sep 17 00:00:00 2001 From: Brian Dolbec Date: Mon, 2 Jul 2018 14:57:23 -0700 Subject: gkeys: Implement use of the new Fetch class Signed-off-by: Brian Dolbec Signed-off-by: Brian Dolbec --- gkeys/gkeys/actions.py | 51 +++++------------------------------- gkeys/gkeys/base.py | 6 +++++ gkeys/gkeys/seedhandler.py | 64 ++++++++++++++++++++++++++++++++++------------ 3 files changed, 61 insertions(+), 60 deletions(-) diff --git a/gkeys/gkeys/actions.py b/gkeys/gkeys/actions.py index 64d6123..0f03880 100644 --- a/gkeys/gkeys/actions.py +++ b/gkeys/gkeys/actions.py @@ -29,6 +29,7 @@ from snakeoil.demandload import demandload demandload( "gkeys.base:Args", + "gkeys.fetch:Fetch", "json:load", ) @@ -718,9 +719,7 @@ class Actions(ActionBase): def installed(self, args): '''Lists the installed key directories''' - if args.category: - keyring = self.config.get_key('keyring') - else: + if not args.category: return (False, ["Please specify a category."]) catdir = self._set_category(args.category) self.logger.debug("ACTIONS: installed; catdir = %s" % catdir) @@ -818,45 +817,10 @@ class Actions(ActionBase): climit = 0 sig_path = None if isurl: - from sslfetch.connections import Connector - connector_output = { - 'info': self.logger.info, - 'debug': self.logger.debug, - 'error': self.logger.error, - 'exception': self.logger.exception, - # we want any warnings to be printed to the terminal - # so assign it to logging.error - 'warning': self.logger.error, - 'kwargs-info': {}, - 'kwargs-debug': {}, - 'kwargs-error': {}, - 'kwargs-exception': {}, - 'kwargs-warning': {}, - } - fetcher = Connector(connector_output, None, "Gentoo Keys") - self.logger.debug( - _unicode("ACTIONS: verify; fetching %s signed file ") % filepath) - self.logger.debug( - _unicode("ACTIONS: verify; timestamp path: %s") % timestamp_path) - success, signedfile, timestamp = fetcher.fetch_file( - url, filepath, timestamp_path, climit=climit) - if not success: - messages.append(_unicode("File %s cannot be retrieved.") % filepath) - elif '.' + url.rsplit('.', 1)[1] not in EXTENSIONS: - if not signature: - success_fetch = False - for ext in EXTENSIONS: - sig_path = filepath + ext - if isurl: - signature = url + ext - self.logger.debug( - _unicode("ACTIONS: verify; fetching %s signature ") - % signature) - success_fetch, sig, timestamp = fetcher.fetch_file(signature, sig_path) - if success_fetch: - break - else: - signature = None + fetcher = Fetch(self.logger) + success, msgs = fetcher.fetch_url(url, filepath, signature, timestamp_path=timestamp_path, + climit=climit) + messages.extend(msgs) elif signature is not None and os.path.exists(signature): sig_path = signature else: @@ -864,9 +828,8 @@ class Actions(ActionBase): self.logger.debug( _unicode("ACTIONS: verify; local file %s") % filepath) success = os.path.isfile(filepath) - if (not signature + if (success and not signature and '.' + filepath.rsplit('.', 1)[-1] not in EXTENSIONS): - success_fetch = False for ext in EXTENSIONS: sig_path = filepath + ext sig_path = os.path.abspath(sig_path) diff --git a/gkeys/gkeys/base.py b/gkeys/gkeys/base.py index a67b330..2de1493 100644 --- a/gkeys/gkeys/base.py +++ b/gkeys/gkeys/base.py @@ -130,6 +130,12 @@ class CliBase(object): action='store_true', default=False, help='Use CASE matching in searches') + @staticmethod + def _option_fetcthonly(parser=None): + parser.add_argument('--fetchonly', + dest='fetchonly', default=False, + help="Only fetch the seed file if there is an update or doesn't exist locally") + @staticmethod def _option_file(parser=None): parser.add_argument('-F', '--file', dest='filename', default=None, diff --git a/gkeys/gkeys/seedhandler.py b/gkeys/gkeys/seedhandler.py index afe5971..cb082ef 100644 --- a/gkeys/gkeys/seedhandler.py +++ b/gkeys/gkeys/seedhandler.py @@ -17,12 +17,15 @@ import re from snakeoil.demandload import demandload from gkeys.gkey import GKEY +from gkeys.lock import LockDir from gkeys.seed import Seeds, decoder demandload( "json:load", "gkeys.exception:UpdateDbError", "gkeys.fileops:ensure_dirs", + "gkeys.fetch:Fetch", + "sslfetch.connections:get_timestamp", ) @@ -34,6 +37,8 @@ class SeedHandler(object): self.fingerprint_re = re.compile('[0-9A-Fa-f]{40}') self.finerprint_re2 = re.compile('[0-9A-Fa-f]{4}( [0-9A-Fa-f]{4}){9}') self.seeds = None + self.seedsdir_lock = None + self.update_lock = None def new(self, args, checkgkey=False): @@ -70,10 +75,10 @@ class SeedHandler(object): if attr in GKEY._fields: keyinfo[attr] = None return keyinfo - + def compare_seeds(self, seeds1, seeds2) : '''Compares two seed lists and returns the differences - + @param seeds1: set of seeds to be compared @param seeds2: set of seeds to be compared @return added_gkeys: list of keys that are included in seed2 but not seed1 @@ -98,7 +103,7 @@ class SeedHandler(object): if old_gkey not in new_gkeys and old_gkey not in old_changed_gkeys: removed_gkeys.append(old_gkey) else: - added_gkeys = new_gkeys + added_gkeys = new_gkeys return(added_gkeys, changed_gkeys, removed_gkeys) def compare_seeds(self, seeds1, seeds2) : @@ -181,7 +186,8 @@ class SeedHandler(object): with open(gkey_path, 'r') as fileseed: seed = load(fileseed) except IOError as error: - self.logger.debug("SeedHandler: load_category; IOError loading seed file %s." % gkey_path) + self.logger.debug("SeedHandler: load_category; IOError loading seed file %s." + % gkey_path) self.logger.debug("Error was: %s" % str(error)) if seed: for nick in sorted(seed): @@ -206,6 +212,7 @@ class SeedHandler(object): '''Fetch new seed files @param seeds: list of seed nicks to download + @param args: argparse namespace instance @param verified_dl: Function pointer to the Actions.verify() instance needed to do the download and verification ''' @@ -224,20 +231,45 @@ class SeedHandler(object): except KeyError: pass succeeded = [] - seedsdir = self.config.get_key('seedsdir') + seedsdir = os.path.join(self.config.get_key('seedsdir')) + updatedir = os.path.join(seedsdir, "__updates__") mode = int(self.config.get_key('permissions', 'directories'),0) - ensure_dirs(seedsdir, mode=mode) + ensure_dirs(updatedir, mode=mode) + self.update_lock = LockDir(updatedir) + self.update_lock.write_lock() + fetcher = Fetch(self.logger) for (seed, url, filepath) in urls: - verify_info = self.config.get_key('verify-seeds', seed).split() - args.category = verify_info[0] - args.nick = verify_info[1] - args.filename = url - args.signature = None - args.timestamp = True - args.destination = filepath - verified, messages_ = verified_dl(args) - succeeded.append(verified) - messages.append(messages_) + tmppath = os.path.join(updatedir, os.path.split(filepath)[-1]) + # use the real timestamp file for the dl timestamp + tpath = filepath + ".timestamp" + # verify the re-fetch cycle timer + if fetcher.verify_cycle(tpath, climit=60): + timestamp = get_timestamp(filepath + ".timestamp") + success, msgs = fetcher.fetch_url(url, tmppath, timestamp=timestamp) + messages.extend(msgs) + if success: + verify_info = self.config.get_key('verify-seeds', seed).split() + args.category = verify_info[0] + args.nick = verify_info[1] + args.filename = url + args.signature = tmppath + ".timestamp" + #args.timestamp = True + args.destination = tmppath + verified, messages_ = verified_dl(args) + messages.append(messages_) + if verified and not args.fetchonly: + self.seedsdir_lock = LockDir(seedsdir) + if updateseeds(tmppath, filepath) and updateseeds(args.signature, tpath): + self.logger.info("Updated seed file...: %s ... OK" % (filepath)) + succeeded.append(verified) + else: + self.logger.info("Updating seed file...: %s ... Failed" % (filepath)) + succeeded.append(False) + self.seedsdir_lock.unlock() + else: + # sha512sum the 2 files + pass + self.update_lock.unlock() return (succeeded, messages) def check_gkey(self, args): -- cgit v1.2.3-65-gdbad