[u r] Add source spec flags

Author: nadove-ucsc

UPGRADING.rst

@@ -19,6 +19,12 @@ branch that does not have the listed changes, the steps would need to be
 reverted. This is all fairly informal and loosely defined. Hopefully we won't
 have too many entries in this file.
 
+#7066 Configure mirroring per catalog and source
+================================================
+
+Update the environment.py files for your personal deployments, using the changes
+to the sandbox/anvilbox deployment as a reference.
+
 
 #7076 Populate schema property in info objects
 ==============================================

deployments/anvilbox/environment.py

@@ -10,6 +10,7 @@
 is_sandbox = True
 
 pop = 1  # remove snapshot
+no_mirror = 2  # do not mirror files from snapshot. redundant for MA snapshots.
 
 
 def bqsrc(google_project: str,
@@ -33,14 +34,15 @@ def mksrc(source_type: Literal['bigquery', 'parquet'],
           prefix: str = ''
           ) -> tuple[str, str | None]:
     project = '_'.join(snapshot.split('_')[1:-3])
-    assert flags <= pop
+    assert flags <= pop | no_mirror
     source = None if flags & pop else ':'.join([
         'tdr',
         source_type,
         'gcp',
         google_project,
         snapshot,
-        prefix
+        prefix,
+        'no_mirror' if flags & no_mirror else ''
     ])
     return project, source
 

deployments/anvildev/environment.py

@@ -8,6 +8,7 @@
 )
 
 pop = 1  # remove snapshot
+no_mirror = 2  # do not mirror files from snapshot. redundant for MA snapshots.
 
 
 def bqsrc(google_project: str,
@@ -31,14 +32,15 @@ def mksrc(source_type: Literal['bigquery', 'parquet'],
           prefix: str = ''
           ) -> tuple[str, str | None]:
     project = '_'.join(snapshot.split('_')[1:-3])
-    assert flags <= pop
+    assert flags <= pop | no_mirror
     source = None if flags & pop else ':'.join([
         'tdr',
         source_type,
         'gcp',
         google_project,
         snapshot,
-        prefix
+        prefix,
+        'no_mirror' if flags & no_mirror else ''
     ])
     return project, source
 

deployments/anvilprod/environment.py

@@ -8,6 +8,7 @@
 )
 
 pop = 1  # remove snapshot
+no_mirror = 2  # do not mirror files from snapshot. redundant for MA snapshots.
 
 
 def bqsrc(google_project: str,
@@ -32,14 +33,15 @@ def mksrc(source_type: Literal['bigquery', 'parquet'],
           prefix: str = ''
           ) -> tuple[str, str | None]:
     project = '_'.join(snapshot.split('_')[1:-3])
-    assert flags <= pop
+    assert flags <= pop | no_mirror
     source = None if flags & pop else ':'.join([
         'tdr',
         source_type,
         'gcp',
         google_project,
         snapshot,
-        prefix
+        prefix,
+        'no_mirror' if flags & no_mirror else ''
     ])
     return project, source
 

deployments/dev/environment.py

@@ -8,6 +8,7 @@
 )
 
 pop = 1  # remove snapshot
+no_mirror = 2  # do not mirror files from snapshot. redundant for MA snapshots.
 
 
 def mksrc(source_type: Literal['bigquery', 'parquet'],
@@ -18,14 +19,15 @@ def mksrc(source_type: Literal['bigquery', 'parquet'],
           prefix: str = ''
           ) -> tuple[str, str | None]:
     _, env, project, _ = snapshot.split('_', 3)
-    assert flags <= pop
+    assert flags <= pop | no_mirror
     source = None if flags & pop else ':'.join([
         'tdr',
         source_type,
         'gcp',
         google_project,
         snapshot,
-        prefix
+        prefix,
+        'no_mirror' if flags & no_mirror else ''
     ])
     return project, source
 

deployments/hammerbox/environment.py

@@ -10,6 +10,7 @@
 is_sandbox = True
 
 pop = 1  # remove snapshot
+no_mirror = 2  # do not mirror files from snapshot. redundant for MA snapshots.
 
 
 def bqsrc(google_project: str,
@@ -34,14 +35,15 @@ def mksrc(source_type: Literal['bigquery', 'parquet'],
           prefix: str = ''
           ) -> tuple[str, str | None]:
     project = '_'.join(snapshot.split('_')[1:-3])
-    assert flags <= pop
+    assert flags <= pop | no_mirror
     source = None if flags & pop else ':'.join([
         'tdr',
         source_type,
         'gcp',
         google_project,
         snapshot,
-        prefix
+        prefix,
+        'no_mirror' if flags & no_mirror else ''
     ])
     return project, source
 

deployments/prod/environment.py

@@ -10,6 +10,7 @@
 )
 
 pop = 1  # remove snapshot
+no_mirror = 2  # do not mirror files from snapshot. redundant for MA snapshots.
 
 
 def mksrc(source_type: Literal['bigquery', 'parquet'],
@@ -20,14 +21,15 @@ def mksrc(source_type: Literal['bigquery', 'parquet'],
           prefix: str = ''
           ) -> tuple[str, str | None]:
     _, env, project, _ = snapshot.split('_', 3)
-    assert flags <= pop
+    assert flags <= pop | no_mirror
     source = None if flags & pop else ':'.join([
         'tdr',
         source_type,
         'gcp',
         google_project,
         snapshot,
-        prefix
+        prefix,
+        'no_mirror' if flags & no_mirror else ''
     ])
     return project, source
 

deployments/sandbox/environment.py

@@ -10,6 +10,7 @@
 is_sandbox = True
 
 pop = 1  # remove snapshot
+no_mirror = 2  # do not mirror files from snapshot. redundant for MA snapshots.
 
 
 def mksrc(source_type: Literal['bigquery', 'parquet'],
@@ -20,14 +21,15 @@ def mksrc(source_type: Literal['bigquery', 'parquet'],
           prefix: str = ''
           ) -> tuple[str, str | None]:
     _, env, project, _ = snapshot.split('_', 3)
-    assert flags <= pop
+    assert flags <= pop | no_mirror
     source = None if flags & pop else ':'.join([
         'tdr',
         source_type,
         'gcp',
         google_project,
         snapshot,
-        prefix
+        prefix,
+        'no_mirror' if flags & no_mirror else ''
     ])
     return project, source
 

deployments/tempdev/environment.py

@@ -8,6 +8,7 @@
 )
 
 pop = 1  # remove snapshot
+no_mirror = 2  # do not mirror files from snapshot. redundant for MA snapshots.
 
 
 def mksrc(source_type: Literal['bigquery', 'parquet'],
@@ -18,14 +19,15 @@ def mksrc(source_type: Literal['bigquery', 'parquet'],
           prefix: str = ''
           ) -> tuple[str, str | None]:
     project = '_'.join(snapshot.split('_')[1:-3])
-    assert flags <= pop
+    assert flags <= pop | no_mirror
     source = None if flags & pop else ':'.join([
         'tdr',
         source_type,
         'gcp',
         google_project,
         snapshot,
-        prefix
+        prefix,
+        'no_mirror' if flags & no_mirror else ''
     ])
     return project, source
 

environment.py

@@ -639,7 +639,7 @@ def env() -> Mapping[str, Optional[str]]:
         #
         # Examples:
         #
-        # https://dss.data.humancellatlas.org/v1:/1
+        # https://dss.data.humancellatlas.org/v1:/1:
         # https://dss.data.humancellatlas.org/v1:aa/1
         #
         'AZUL_DSS_SOURCE': None,