w

Author: nadove-ucsc

scripts/mirror.py

@@ -5,9 +5,6 @@
 import argparse
 import logging
 import sys
-from typing import (
-    Iterable,
-)
 
 from azul import (
     CatalogName,
@@ -20,9 +17,6 @@
 from azul.azulclient import (
     AzulClient,
 )
-from azul.indexer import (
-    SourceRef,
-)
 from azul.logging import (
     configure_script_logging,
 )
@@ -44,28 +38,31 @@ def mirror_catalog(azul: AzulClient,
         source.spec: source
         for source in plugin.list_sources(authentication=None)
     }
-    source_refs: Iterable[SourceRef]
-    # When the user doesn't specify a source or provides "*" as a source glob,
-    # we implicitly filter out managed-access sources. This lets us assert that
-    # all sources matching the provided globs are public, without forcing the
-    # user to manually specify every public source.
+    source_specs = azul.matching_sources([catalog], source_globs)[catalog]
     if '*' in source_globs:
-        source_refs = public_sources_by_spec.values()
+        # When the user doesn't specify a source or provides "*" as a source glob,
+        # we implicitly filter out managed-access sources. This lets us assert that
+        # all sources matching the provided globs are public, without forcing the
+        # user to manually specify every public source.
+        source_specs = {
+            spec: cfg
+            for spec, cfg in source_specs.items()
+            if spec in public_sources_by_spec
+        }
+    try:
+        source_refs = {
+            public_sources_by_spec[spec]: cfg
+            for spec, cfg in source_specs.items()
+        }
+    except KeyError as e:
+        assert False, R(
+            'Cannot mirror managed-access source', e.args[0])
     else:
-        source_specs = azul.matching_sources([catalog], source_globs)[catalog]
-        try:
-            source_refs = {
-                public_sources_by_spec[spec]
-                for spec in source_specs
-            }
-        except KeyError as e:
-            assert False, R(
-                'Cannot mirror managed-access source', e.args[0])
-    azul.remote_mirror(catalog, source_refs)
-    if wait:
-        azul.wait_for_mirroring()
-        assert azul.is_queue_empty(fail_queue), R(
-            'There are messages in the fail queue', fail_queue)
+        azul.remote_mirror(catalog, source_refs.items())
+        if wait:
+            azul.wait_for_mirroring()
+            assert azul.is_queue_empty(fail_queue), R(
+                'There are messages in the fail queue', fail_queue)
 
 
 def main(args):

src/azul/azulclient.py

@@ -1,6 +1,3 @@
-import fnmatch
-import logging
-import uuid
 from collections import (
     defaultdict,
 )
@@ -14,15 +11,18 @@
 from enum import (
     auto,
 )
+import fnmatch
 from functools import (
     partial,
 )
-from itertools import starmap
+import logging
 from pprint import (
     PrettyPrinter,
 )
 from typing import (
-    AbstractSet, )
+    AbstractSet,
+)
+import uuid
 
 import attrs
 import requests
@@ -248,7 +248,7 @@ def matching_sources(self,
                         matched_globs.add(glob)
                         log.debug('Source glob %r matched sources %r in catalog %r',
                                   glob, _matching_raw_specs, catalog)
-                specs = {spec: cfg for spec, cfg in specs if str(spec) in matching_raw_specs}
+                specs = {spec: cfg for spec, cfg in specs.items() if str(spec) in matching_raw_specs}
             result[catalog] = specs
         unmatched_globs = globs - matched_globs
         if unmatched_globs:

test/azul_test_case.py

@@ -456,7 +456,7 @@ def catalog_config(cls) -> dict[CatalogName, Config.Catalog]:
                                         mirror_max_file_size=None,
                                         plugins=dict(metadata=config.Catalog.Plugin(name='hca'),
                                                      repository=config.Catalog.Plugin(name='dss')),
-                                        sources={str(cls.source.spec): {}})
+                                        sources={str(cls.source.spec): {'mirror': True}})
         }
 
 
@@ -480,7 +480,7 @@ def _patch_tdr_service_url(cls):
 
     @classmethod
     def _sources(cls):
-        return {str(cls.source.spec): {}}
+        return {str(cls.source.spec): {'mirror': True}}
 
     @classmethod
     def _patch_source_cache(cls):
@@ -522,5 +522,5 @@ def catalog_config(cls) -> dict[CatalogName, Config.Catalog]:
                                         mirror_max_file_size=None,
                                         plugins=dict(metadata=config.Catalog.Plugin(name='anvil'),
                                                      repository=config.Catalog.Plugin(name='tdr_anvil')),
-                                        sources={str(cls.source.spec): {}})
+                                        sources={str(cls.source.spec): {'mirror': True}})
         }