10
10
import json
11
11
import os
12
12
import subprocess
13
- import sys
14
13
import tempfile
15
14
import traceback
16
15
from collections import defaultdict
19
18
from functools import lru_cache
20
19
from itertools import islice
21
20
from pathlib import Path
22
- from typing import Any , ClassVar , Dict , List , Optional , Sequence , Tuple , Union
21
+ from typing import Any , ClassVar , Dict , List , Optional , Tuple
23
22
24
- import backoff # type: ignore
25
23
import gcsfs # type: ignore
26
24
import pendulum
27
25
import sentry_sdk
28
26
import typer
29
- from aiohttp .client_exceptions import (
30
- ClientOSError ,
31
- ClientResponseError ,
32
- ServerDisconnectedError ,
33
- )
34
27
from calitp_data_infra .storage import ( # type: ignore
35
28
JSONL_GZIP_EXTENSION ,
36
29
GTFSDownloadConfig ,
37
- GTFSFeedExtract ,
38
30
GTFSFeedType ,
39
31
GTFSRTFeedExtract ,
40
32
GTFSScheduleFeedExtract ,
@@ -565,6 +557,7 @@ def process(
565
557
exception = e ,
566
558
process_stderr = stderr ,
567
559
)
560
+ for extract in self .aggregation .extracts
568
561
]
569
562
570
563
if not outcomes :
@@ -581,13 +574,13 @@ def process(
581
574
fg = typer .colors .YELLOW ,
582
575
)
583
576
584
- for e in extracts :
577
+ for extract in extracts :
585
578
outcomes .append (
586
579
RTFileProcessingOutcome (
587
580
step = self .aggregation .step ,
588
581
success = False ,
589
- extract = e ,
590
- aggregation = aggregation ,
582
+ exception = e ,
583
+ extract = extract ,
591
584
)
592
585
)
593
586
continue
@@ -682,7 +675,7 @@ def process(
682
675
feed .ParseFromString (f .read ())
683
676
parsed = json_format .MessageToDict (feed )
684
677
except DecodeError as e :
685
- if verbose :
678
+ if self . verbose :
686
679
typer .secho (
687
680
f"WARNING: DecodeError for { str (extract .path )} " ,
688
681
fg = typer .colors .YELLOW ,
@@ -854,10 +847,10 @@ def main(
854
847
):
855
848
hourly_feed_files = FeedStorage (feed_type ).get_hour (hour )
856
849
if not hourly_feed_files .valid ():
857
- typer .secho (f"missing: { files_missing_metadata } " )
858
- typer .secho (f"invalid: { files_invalid_metadata } " )
850
+ typer .secho (f"missing: { hourly_feed_files . files_missing_metadata } " )
851
+ typer .secho (f"invalid: { hourly_feed_files . files_invalid_metadata } " )
859
852
raise RuntimeError (
860
- f"too many files have missing/invalid metadata; { total - len (files )} of { total } " # noqa: E702
853
+ f"too many files have missing/invalid metadata; { hourly_feed_files . total - len (hourly_feed_files . files )} of { hourly_feed_files . total } "
861
854
)
862
855
aggregated_feed = hourly_feed_files .get_query (step , feed_type )
863
856
aggregations_to_process = (
0 commit comments