Skip to content

Commit dec55f8

Browse files
nadove-ucscnadove-ucsc
committed
[u 5/6] Convert source config to dictionary (#7066)
1 parent 9aa0203 commit dec55f8

File tree

17 files changed

+166
-109
lines changed

17 files changed

+166
-109
lines changed

‎UPGRADING.rst‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@ In your personal deployments' ``environment.py`` files:
3030

3131
2. Remove the ``prefix`` parameter and its uses from ``bqsrc`` and ``mksrc``.
3232

33+
2. Update the function body of ``mksrc``.
34+
35+
3. Update the assignment of the ``sources`` parameter in the ``AZUL_CATALOGS``
36+
variable.
37+
3338
As always, use the sandbox deployment's ``environment.py`` as a model when
3439
upgrading personal deployments.
3540

‎deployments/anvilbox/environment.py‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,52 +12,57 @@
1212

1313
type ProjectName = str
1414
type SourceSpec = str
15+
type SourceConfig = dict[str, str | int | float | bool | None]
16+
type SourceItem = tuple[SourceSpec, SourceConfig]
1517

1618

1719
def bqsrc(google_project: str,
1820
snapshot: str,
1921
flags: int = 0,
20-
) -> tuple[ProjectName, SourceSpec | None]:
22+
**config: str | int | float | bool | None
23+
) -> tuple[ProjectName, SourceItem | None]:
2124
assert len(google_project) == 8, google_project
2225
project = 'datarepo-dev-' + google_project
2326
assert not snapshot.startswith('ANVIL_'), snapshot
2427
snapshot = 'ANVIL_' + snapshot
25-
return mksrc('bigquery', project, snapshot, flags)
28+
return mksrc('bigquery', project, snapshot, flags, **config)
2629

2730

2831
def mksrc(source_type: Literal['bigquery', 'parquet'],
2932
google_project,
3033
snapshot,
3134
flags: int = 0,
32-
) -> tuple[ProjectName, SourceSpec | None]:
35+
**config: str | int | float | bool | None
36+
) -> tuple[ProjectName, SourceItem | None]:
3337
project = '_'.join(snapshot.split('_')[1:-3])
3438
assert flags <= pop
35-
source = None if flags & pop else ':'.join([
39+
source = None if flags & pop else (':'.join([
3640
'tdr',
3741
source_type,
3842
'gcp',
3943
google_project,
4044
snapshot,
41-
])
45+
]), config)
4246
return project, source
4347

4448

45-
def mkdelta(items: list[tuple[ProjectName, SourceSpec | None]]
46-
) -> dict[ProjectName, SourceSpec | None]:
49+
def mkdelta(items: list[tuple[ProjectName, SourceItem | None]]
50+
) -> dict[ProjectName, SourceItem | None]:
4751
result = dict(items)
4852
assert len(items) == len(result), 'collisions detected'
4953
assert list(result.keys()) == sorted(result.keys()), 'input not sorted'
5054
return result
5155

5256

53-
def mksrcs(catalog: dict[ProjectName, SourceSpec | None]) -> list[SourceSpec]:
54-
return list(filter(None, catalog.values()))
57+
def mksrcs(catalog: dict[ProjectName, SourceItem | None]
58+
) -> dict[SourceSpec, SourceConfig]:
59+
return dict(filter(None, catalog.values()))
5560

5661

57-
def mkdict(previous_catalog: dict[ProjectName, SourceSpec | None],
62+
def mkdict(previous_catalog: dict[ProjectName, SourceItem | None],
5863
num_expected: int,
59-
delta: dict[ProjectName, SourceSpec | None],
60-
) -> dict[ProjectName, SourceSpec | None]:
64+
delta: dict[ProjectName, SourceItem | None],
65+
) -> dict[ProjectName, SourceItem | None]:
6166
catalog = previous_catalog | delta
6267
num_actual = len(mksrcs(catalog))
6368
assert num_expected == num_actual, (num_expected, num_actual)

‎deployments/anvildev/environment.py‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,52 +10,57 @@
1010

1111
type ProjectName = str
1212
type SourceSpec = str
13+
type SourceConfig = dict[str, str | int | float | bool | None]
14+
type SourceItem = tuple[SourceSpec, SourceConfig]
1315

1416

1517
def bqsrc(google_project: str,
1618
snapshot: str,
1719
flags: int = 0,
18-
) -> tuple[ProjectName, SourceSpec | None]:
20+
**config: str | int | float | bool | None
21+
) -> tuple[ProjectName, SourceItem | None]:
1922
assert len(google_project) == 8, google_project
2023
project = 'datarepo-dev-' + google_project
2124
assert not snapshot.startswith('ANVIL_'), snapshot
2225
snapshot = 'ANVIL_' + snapshot
23-
return mksrc('bigquery', project, snapshot, flags)
26+
return mksrc('bigquery', project, snapshot, flags, **config)
2427

2528

2629
def mksrc(source_type: Literal['bigquery', 'parquet'],
2730
google_project,
2831
snapshot,
2932
flags: int = 0,
30-
) -> tuple[ProjectName, SourceSpec | None]:
33+
**config: str | int | float | bool | None
34+
) -> tuple[ProjectName, SourceItem | None]:
3135
project = '_'.join(snapshot.split('_')[1:-3])
3236
assert flags <= pop
33-
source = None if flags & pop else ':'.join([
37+
source = None if flags & pop else (':'.join([
3438
'tdr',
3539
source_type,
3640
'gcp',
3741
google_project,
3842
snapshot,
39-
])
43+
]), config)
4044
return project, source
4145

4246

43-
def mkdelta(items: list[tuple[ProjectName, SourceSpec | None]]
44-
) -> dict[ProjectName, SourceSpec | None]:
47+
def mkdelta(items: list[tuple[ProjectName, SourceItem | None]]
48+
) -> dict[ProjectName, SourceItem | None]:
4549
result = dict(items)
4650
assert len(items) == len(result), 'collisions detected'
4751
assert list(result.keys()) == sorted(result.keys()), 'input not sorted'
4852
return result
4953

5054

51-
def mksrcs(catalog: dict[ProjectName, SourceSpec | None]) -> list[SourceSpec]:
52-
return list(filter(None, catalog.values()))
55+
def mksrcs(catalog: dict[ProjectName, SourceItem | None]
56+
) -> dict[SourceSpec, SourceConfig]:
57+
return dict(filter(None, catalog.values()))
5358

5459

55-
def mkdict(previous_catalog: dict[ProjectName, SourceSpec | None],
60+
def mkdict(previous_catalog: dict[ProjectName, SourceItem | None],
5661
num_expected: int,
57-
delta: dict[ProjectName, SourceSpec | None],
58-
) -> dict[ProjectName, SourceSpec | None]:
62+
delta: dict[ProjectName, SourceItem | None],
63+
) -> dict[ProjectName, SourceItem | None]:
5964
catalog = previous_catalog | delta
6065
num_actual = len(mksrcs(catalog))
6166
assert num_expected == num_actual, (num_expected, num_actual)

‎deployments/anvilprod/environment.py‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,53 +12,58 @@
1212

1313
type ProjectName = str
1414
type SourceSpec = str
15+
type SourceConfig = dict[str, str | int | float | bool | None]
16+
type SourceItem = tuple[SourceSpec, SourceConfig]
1517

1618

1719
def bqsrc(google_project: str,
1820
snapshot: str,
1921
flags: int = 0,
20-
) -> tuple[ProjectName, SourceSpec | None]:
22+
**config: str | int | float | bool | None
23+
) -> tuple[ProjectName, SourceItem | None]:
2124
assert len(google_project) == 8, google_project
2225
project = 'datarepo-' + google_project
2326
# Some snapshots start with AnVIL instead of ANVIL
2427
if not snapshot.upper().startswith('ANVIL_'):
2528
snapshot = 'ANVIL_' + snapshot
26-
return mksrc('bigquery', project, snapshot, flags)
29+
return mksrc('bigquery', project, snapshot, flags, **config)
2730

2831

2932
def mksrc(source_type: Literal['bigquery', 'parquet'],
3033
google_project,
3134
snapshot,
3235
flags: int = 0,
33-
) -> tuple[ProjectName, SourceSpec | None]:
36+
**config: str | int | float | bool | None
37+
) -> tuple[ProjectName, SourceItem | None]:
3438
project = '_'.join(snapshot.split('_')[1:-3])
3539
assert flags <= pop
36-
source = None if flags & pop else ':'.join([
40+
source = None if flags & pop else (':'.join([
3741
'tdr',
3842
source_type,
3943
'gcp',
4044
google_project,
4145
snapshot,
42-
])
46+
]), config)
4347
return project, source
4448

4549

46-
def mkdelta(items: list[tuple[ProjectName, SourceSpec | None]]
47-
) -> dict[ProjectName, SourceSpec | None]:
50+
def mkdelta(items: list[tuple[ProjectName, SourceItem | None]]
51+
) -> dict[ProjectName, SourceItem | None]:
4852
result = dict(items)
4953
assert len(items) == len(result), 'collisions detected'
5054
assert list(result.keys()) == sorted(result.keys()), 'input not sorted'
5155
return result
5256

5357

54-
def mksrcs(catalog: dict[ProjectName, SourceSpec | None]) -> list[SourceSpec]:
55-
return list(filter(None, catalog.values()))
58+
def mksrcs(catalog: dict[ProjectName, SourceItem | None]
59+
) -> dict[SourceSpec, SourceConfig]:
60+
return dict(filter(None, catalog.values()))
5661

5762

58-
def mkdict(previous_catalog: dict[ProjectName, SourceSpec | None],
63+
def mkdict(previous_catalog: dict[ProjectName, SourceItem | None],
5964
num_expected: int,
60-
delta: dict[ProjectName, SourceSpec | None],
61-
) -> dict[ProjectName, SourceSpec | None]:
65+
delta: dict[ProjectName, SourceItem | None],
66+
) -> dict[ProjectName, SourceItem | None]:
6267
catalog = previous_catalog | delta
6368
num_actual = len(mksrcs(catalog))
6469
assert num_expected == num_actual, (num_expected, num_actual)

‎deployments/dev/environment.py‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,41 +10,45 @@
1010

1111
type ProjectName = str
1212
type SourceSpec = str
13+
type SourceConfig = dict[str, str | int | float | bool | None]
14+
type SourceItem = tuple[SourceSpec, SourceConfig]
1315

1416

1517
def mksrc(source_type: Literal['bigquery', 'parquet'],
1618
google_project,
1719
snapshot,
1820
flags: int = 0,
19-
) -> tuple[ProjectName, SourceSpec | None]:
21+
**config: str | int | float | bool | None
22+
) -> tuple[ProjectName, SourceItem | None]:
2023
_, env, project, _ = snapshot.split('_', 3)
2124
assert flags <= pop
22-
source = None if flags & pop else ':'.join([
25+
source = None if flags & pop else (':'.join([
2326
'tdr',
2427
source_type,
2528
'gcp',
2629
google_project,
2730
snapshot,
28-
])
31+
]), config)
2932
return project, source
3033

3134

32-
def mkdelta(items: list[tuple[ProjectName, SourceSpec | None]]
33-
) -> dict[ProjectName, SourceSpec | None]:
35+
def mkdelta(items: list[tuple[ProjectName, SourceItem | None]]
36+
) -> dict[ProjectName, SourceItem | None]:
3437
result = dict(items)
3538
assert len(items) == len(result), 'collisions detected'
3639
assert list(result.keys()) == sorted(result.keys()), 'input not sorted'
3740
return result
3841

3942

40-
def mksrcs(catalog: dict[ProjectName, SourceSpec | None]) -> list[SourceSpec]:
41-
return list(filter(None, catalog.values()))
43+
def mksrcs(catalog: dict[ProjectName, SourceItem | None]
44+
) -> dict[SourceSpec, SourceConfig]:
45+
return dict(filter(None, catalog.values()))
4246

4347

44-
def mkdict(previous_catalog: dict[ProjectName, SourceSpec | None],
48+
def mkdict(previous_catalog: dict[ProjectName, SourceItem | None],
4549
num_expected: int,
46-
delta: dict[ProjectName, SourceSpec | None],
47-
) -> dict[ProjectName, SourceSpec | None]:
50+
delta: dict[ProjectName, SourceItem | None],
51+
) -> dict[ProjectName, SourceItem | None]:
4852
catalog = previous_catalog | delta
4953
num_actual = len(mksrcs(catalog))
5054
assert num_expected == num_actual, (num_expected, num_actual)

‎deployments/hammerbox/environment.py‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,53 +14,58 @@
1414

1515
type ProjectName = str
1616
type SourceSpec = str
17+
type SourceConfig = dict[str, str | int | float | bool | None]
18+
type SourceItem = tuple[SourceSpec, SourceConfig]
1719

1820

1921
def bqsrc(google_project: str,
2022
snapshot: str,
2123
flags: int = 0,
22-
) -> tuple[ProjectName, SourceSpec | None]:
24+
**config: str | int | float | bool | None
25+
) -> tuple[ProjectName, SourceItem | None]:
2326
assert len(google_project) == 8, google_project
2427
project = 'datarepo-' + google_project
2528
# Some snapshots start with AnVIL instead of ANVIL
2629
if not snapshot.upper().startswith('ANVIL_'):
2730
snapshot = 'ANVIL_' + snapshot
28-
return mksrc('bigquery', project, snapshot, flags)
31+
return mksrc('bigquery', project, snapshot, flags, **config)
2932

3033

3134
def mksrc(source_type: Literal['bigquery', 'parquet'],
3235
google_project,
3336
snapshot,
3437
flags: int = 0,
35-
) -> tuple[ProjectName, SourceSpec | None]:
38+
**config: str | int | float | bool | None
39+
) -> tuple[ProjectName, SourceItem | None]:
3640
project = '_'.join(snapshot.split('_')[1:-3])
3741
assert flags <= pop
38-
source = None if flags & pop else ':'.join([
42+
source = None if flags & pop else (':'.join([
3943
'tdr',
4044
source_type,
4145
'gcp',
4246
google_project,
4347
snapshot,
44-
])
48+
]), config)
4549
return project, source
4650

4751

48-
def mkdelta(items: list[tuple[ProjectName, SourceSpec | None]]
49-
) -> dict[ProjectName, SourceSpec | None]:
52+
def mkdelta(items: list[tuple[ProjectName, SourceItem | None]]
53+
) -> dict[ProjectName, SourceItem | None]:
5054
result = dict(items)
5155
assert len(items) == len(result), 'collisions detected'
5256
assert list(result.keys()) == sorted(result.keys()), 'input not sorted'
5357
return result
5458

5559

56-
def mksrcs(catalog: dict[ProjectName, SourceSpec | None]) -> list[SourceSpec]:
57-
return list(filter(None, catalog.values()))
60+
def mksrcs(catalog: dict[ProjectName, SourceItem | None]
61+
) -> dict[SourceSpec, SourceConfig]:
62+
return dict(filter(None, catalog.values()))
5863

5964

60-
def mkdict(previous_catalog: dict[ProjectName, SourceSpec | None],
65+
def mkdict(previous_catalog: dict[ProjectName, SourceItem | None],
6166
num_expected: int,
62-
delta: dict[ProjectName, SourceSpec | None],
63-
) -> dict[ProjectName, SourceSpec | None]:
67+
delta: dict[ProjectName, SourceItem | None],
68+
) -> dict[ProjectName, SourceItem | None]:
6469
catalog = previous_catalog | delta
6570
num_actual = len(mksrcs(catalog))
6671
assert num_expected == num_actual, (num_expected, num_actual)

0 commit comments

Comments
 (0)