diff options
| -rw-r--r-- | TODO | 1 | ||||
| -rwxr-xr-x | bin/euscan | 1 | ||||
| -rw-r--r-- | euscanwww/djeuscan/processing/misc/update_portage_trees.py | 11 | ||||
| -rw-r--r-- | euscanwww/djeuscan/processing/scan/scan_metadata.py | 3 | ||||
| -rw-r--r-- | euscanwww/djeuscan/tasks.py | 4 | ||||
| -rw-r--r-- | euscanwww/euscanwww/settings.py | 1 | ||||
| -rw-r--r-- | pym/euscan/handlers/berlios.py | 2 | ||||
| -rw-r--r-- | pym/euscan/handlers/google_code.py | 4 | ||||
| -rw-r--r-- | pym/euscan/handlers/sourceforge.py | 8 | ||||
| -rw-r--r-- | pym/euscan/scan.py | 12 |
10 files changed, 30 insertions, 17 deletions
@@ -22,6 +22,7 @@ euscan Site Handlers ------------- +- store un-expanded SRC_URI and use it in handlers - berlios - sourceforge: http://sourceforge.net/api/file/index/project-name/vboxgtk/mtime/desc/limit/20/rss http://sourceforge.net/api/release/index/project-id/264534/rss - MySQL: should use http://downloads.mysql.com/archives/ @@ -38,7 +38,6 @@ from euscan.out import progress_bar isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty() isatty_stderr = os.environ.get('TERM') != 'dumb' and sys.stderr.isatty() - def exit_helper(status): if CONFIG["format"]: print(output.get_formatted_output()) diff --git a/euscanwww/djeuscan/processing/misc/update_portage_trees.py b/euscanwww/djeuscan/processing/misc/update_portage_trees.py index 46eb9c6..e91a18c 100644 --- a/euscanwww/djeuscan/processing/misc/update_portage_trees.py +++ b/euscanwww/djeuscan/processing/misc/update_portage_trees.py @@ -53,6 +53,14 @@ def emerge_sync(logger): "--config-root", settings.PORTAGE_CONFIGROOT] return _launch_command(cmd, logger) +def emerge_metadata(logger): + """ + Launches an emerge --metadata + """ + cmd = ["emerge", "--metadata", "--root", settings.PORTAGE_ROOT, + "--config-root", settings.PORTAGE_CONFIGROOT] + return _launch_command(cmd, logger) + def layman_sync(logger, cache=True): """ @@ -106,7 +114,8 @@ def update_portage_trees(logger=None): logger.info("Running layman --sync") layman_sync(logger, cache=True) #logger.info("Running emerge --regen") - #emerge_regen() + #emerge_regen(logger) + emerge_metadata(logger) logger.info("Running eix-update") eix_update(logger) logger.info("Done!") diff --git a/euscanwww/djeuscan/processing/scan/scan_metadata.py b/euscanwww/djeuscan/processing/scan/scan_metadata.py index 097229f..97a2b70 100644 --- a/euscanwww/djeuscan/processing/scan/scan_metadata.py +++ b/euscanwww/djeuscan/processing/scan/scan_metadata.py @@ -91,8 +91,10 @@ class ScanMetadata(object): try: metadata = None pkg, metadata = self.metadata_from_db(query, pkg) + if not metadata: pkg, metadata = self.metadata_from_portage(query, pkg) + if not metadata: return except Exception as e: @@ -148,6 +150,7 @@ class ScanMetadata(object): self.style.ERROR("Bad maintainer: '%s' '%s'" % \ (maintainer.name, maintainer.email)) ) + pkg.save() def store_herd(self, name, email): diff --git a/euscanwww/djeuscan/tasks.py b/euscanwww/djeuscan/tasks.py index 056f0ad..acfb3d7 100644 --- a/euscanwww/djeuscan/tasks.py +++ b/euscanwww/djeuscan/tasks.py @@ -223,7 +223,7 @@ def update_upstream(): def scan_package(package): scan_portage([package], purge_packages=True, purge_versions=True) scan_metadata([package]) - scan_upstream([package]) + scan_upstream([package], purge_versions=True) return True @@ -254,7 +254,7 @@ def consume_refresh_queue(locked=False): pkg = query.package query.delete() scan_package_user.delay(pkg) - logger.info('Done (%s)' % pkg) + logger.info('Selected: %s' % pkg) except IndexError: pass finally: diff --git a/euscanwww/euscanwww/settings.py b/euscanwww/euscanwww/settings.py index 6ae512d..d6b8616 100644 --- a/euscanwww/euscanwww/settings.py +++ b/euscanwww/euscanwww/settings.py @@ -222,7 +222,6 @@ ACCOUNT_ACTIVATION_DAYS = 7 RECAPTCHA_PUBLIC_KEY = "" RECAPTCHA_PRIVATE_KEY = "" EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' -EMAIL_FROM = 'noreply@iksaif.net' # djeuscan tasks PORTAGE_ROOT = "/" diff --git a/pym/euscan/handlers/berlios.py b/pym/euscan/handlers/berlios.py index 9f716de..bbfe6e1 100644 --- a/pym/euscan/handlers/berlios.py +++ b/pym/euscan/handlers/berlios.py @@ -12,7 +12,7 @@ CONFIDENCE = 90 PRIORITY = 90 -berlios_regex = r"berlios.de/([^/]+)/([^/]+)" +berlios_regex = r"mirror://berlios/([^/]+)/([^/]+)" def can_handle(pkg, url=None): diff --git a/pym/euscan/handlers/google_code.py b/pym/euscan/handlers/google_code.py index a5cb037..892064f 100644 --- a/pym/euscan/handlers/google_code.py +++ b/pym/euscan/handlers/google_code.py @@ -32,6 +32,6 @@ def scan_url(pkg, url, options): result = url_scan(pkg, base_url, file_pattern) ret = [] - for found_url, pv, _, _ in result: - ret.append((found_url, pv, HANDLER_NAME, CONFIDENCE)) + for url, pv, _, _ in result: + ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) return ret diff --git a/pym/euscan/handlers/sourceforge.py b/pym/euscan/handlers/sourceforge.py index df80863..752dba9 100644 --- a/pym/euscan/handlers/sourceforge.py +++ b/pym/euscan/handlers/sourceforge.py @@ -12,7 +12,7 @@ PRIORITY = 90 def can_handle(pkg, url=None): - return url and "sourceforge.net/" in url + return url and "mirror://sourceforge/" in url def scan_url(pkg, url, options): @@ -20,7 +20,7 @@ def scan_url(pkg, url, options): cp, ver, rev = portage.pkgsplit(pkg.cpv) - project, filename = re.search("sourceforge.net/([^/]+)/(?:.*/)?([^/]+)", + project, filename = re.search("mirror://sourceforge/([^/]+)/(?:.*/)?([^/]+)", url).groups() base_url = "http://qa.debian.org/watch/sf.php/%s" % project file_pattern = regex_from_template( @@ -30,6 +30,6 @@ def scan_url(pkg, url, options): result = url_scan(pkg, base_url, file_pattern) ret = [] - for found_url, pv, _, _ in result: - ret.append((found_url, pv, HANDLER_NAME, CONFIDENCE)) + for url, pv, _, _ in result: + ret.append((url, pv, HANDLER_NAME, CONFIDENCE)) return ret diff --git a/pym/euscan/scan.py b/pym/euscan/scan.py index dc7df75..1fc13bb 100644 --- a/pym/euscan/scan.py +++ b/pym/euscan/scan.py @@ -54,8 +54,6 @@ def parse_src_uri(uris): if '://' not in uri: continue - if 'mirror://' in uri: - uri = from_mirror(uri) if uris and uris[-1] == "->": operator = uris.pop() @@ -102,7 +100,7 @@ def scan_upstream(query, on_progress=None): else: matches = Query(query).find( include_masked=True, - in_installed=False + in_installed=False, ) if not matches: @@ -165,9 +163,13 @@ def scan_upstream(query, on_progress=None): uris = pkg.environment('SRC_URI') cpv = pkg.cpv - urls = parse_src_uri(uris) - versions = handlers.scan(pkg, urls, on_progress) + uris = parse_src_uri(uris) + uris_expanded = [ from_mirror(uri) if 'mirror://' in uri else uri for uri in uris ] + pkg._uris = uris + pkg._uris_expanded = uris_expanded + + versions = handlers.scan(pkg, uris, on_progress) cp, ver, rev = portage.pkgsplit(pkg.cpv) |