Added data issues repository path as an explicit parameter to AWS

Author: ladsmund

src/pypromice/process/L1toL2.py

@@ -3,6 +3,7 @@
 AWS Level 1 (L1) to Level 2 (L2) data processing
 """
 import logging
+from pathlib import Path
 
 import numpy as np
 import pandas as pd
@@ -23,6 +24,8 @@
 def toL2(
     L1: xr.Dataset,
     vars_df: pd.DataFrame,
+    data_flags_dir: Path,
+    data_adjustments_dir: Path,
     T_0=273.15,
     ews=1013.246,
     ei0=6.1071,
@@ -72,9 +75,9 @@ def toL2(
     ds = L1.copy(deep=True)                                                    # Reassign dataset
     ds.attrs['level'] = 'L2'
     try:
-        ds = adjustTime(ds)                                                    # Adjust time after a user-defined csv files
-        ds = flagNAN(ds)                                                       # Flag NaNs after a user-defined csv files
-        ds = adjustData(ds)                                                    # Adjust data after a user-defined csv files
+        ds = adjustTime(ds, adj_dir=data_adjustments_dir.as_posix())       # Adjust time after a user-defined csv files
+        ds = flagNAN(ds, flag_dir=data_flags_dir.as_posix())             # Flag NaNs after a user-defined csv files
+        ds = adjustData(ds, adj_dir=data_adjustments_dir.as_posix())       # Adjust data after a user-defined csv files
     except Exception:
         logger.exception('Flagging and fixing failed:')
 

src/pypromice/process/aws.py

@@ -4,7 +4,8 @@
 """
 import json
 import warnings
-warnings.simplefilter(action='ignore', category=FutureWarning)
+
+warnings.simplefilter(action="ignore", category=FutureWarning)
 
 import logging, os
 from pathlib import Path
@@ -33,6 +34,7 @@ def __init__(
         self,
         config_file,
         inpath,
+        data_issues_repository: Path | str,
         var_file=None,
         meta_file=None,
     ):
@@ -59,23 +61,23 @@ def __init__(
         self.config = self.loadConfig(config_file, inpath)
         self.vars = pypromice.resources.load_variables(var_file)
         self.meta = pypromice.resources.load_metadata(meta_file)
+        self.data_issues_repository = Path(data_issues_repository)
 
         config_hash = get_commit_hash_and_check_dirty(Path(config_file))
         config_source_string = f"{Path(config_file).name}:{config_hash}"
         inpath_hash = get_commit_hash_and_check_dirty(Path(inpath))
-        inpath_source_string = f"{Path(inpath).name}:{inpath_hash}"
-
+        data_issues_hash = get_commit_hash_and_check_dirty(self.data_issues_repository)
         source_dict = dict(
-            pypromice = metadata.version("pypromice"),
-            l0_config_file = config_source_string,
-            l0_data_root = inpath_source_string,
+            pypromice=metadata.version("pypromice"),
+            l0_config_file=config_source_string,
+            l0_data_root=inpath_hash,
+            data_issues=data_issues_hash,
         )
         self.meta["source"] = json.dumps(source_dict)
 
-
         # Load config file
         L0 = self.loadL0()
-        self.L0=[]
+        self.L0 = []
         for l in L0:
             n = write.getColNames(self.vars, l)
             self.L0.append(utilities.popCols(l, n))
@@ -98,7 +100,9 @@ def __init__(
     def process(self):
         """Perform L0 to L3 data processing"""
         try:
-            logger.info(f'Commencing {self.L0.attrs["number_of_booms"]}-boom processing...')
+            logger.info(
+                f'Commencing {self.L0.attrs["number_of_booms"]}-boom processing...'
+            )
             logger.info(
                 f'Commencing {self.L0.attrs["number_of_booms"]}-boom processing...'
             )
@@ -137,7 +141,13 @@ def getL1(self):
     def getL2(self):
         """Perform L1 to L2 data processing"""
         logger.info("Level 2 processing...")
-        self.L2 = toL2(self.L1A, vars_df=self.vars)
+
+        self.L2 = toL2(
+            self.L1A,
+            vars_df=self.vars,
+            data_flags_dir=self.data_issues_repository / "flags",
+            data_adjustments_dir=self.data_issues_repository / "adjustments",
+        )
 
     def getL3(self):
         """Perform L2 to L3 data processing, including resampling and metadata
@@ -186,7 +196,7 @@ def loadConfig(self, config_file, inpath):
         return conf
 
     def loadL0(self):
-        '''Load level 0 (L0) data from associated TOML-formatted
+        """Load level 0 (L0) data from associated TOML-formatted
         config file and L0 data file
 
         Try readL0file() using the config with msg_lat & msg_lon appended. The
@@ -201,7 +211,7 @@ def loadL0(self):
         -------
         ds_list : list
             List of L0 xr.Dataset objects
-        '''
+        """
         ds_list = []
         for k in self.config.keys():
             target = self.config[k]
@@ -211,14 +221,14 @@ def loadL0(self):
             except pd.errors.ParserError as e:
                 # ParserError: Too many columns specified: expected 40 and found 38
                 # logger.info(f'-----> No msg_lat or msg_lon for {k}')
-                for item in ['msg_lat', 'msg_lon']:
-                    target['columns'].remove(item)                           # Also removes from self.config
+                for item in ["msg_lat", "msg_lon"]:
+                    target["columns"].remove(item)  # Also removes from self.config
                 ds_list.append(self.readL0file(target))
-            logger.info(f'L0 data successfully loaded from {k}')
+            logger.info(f"L0 data successfully loaded from {k}")
         return ds_list
 
     def readL0file(self, conf):
-        '''Read L0 .txt file to Dataset object using config dictionary and
+        """Read L0 .txt file to Dataset object using config dictionary and
         populate with initial metadata
 
         Parameters
@@ -230,9 +240,15 @@ def readL0file(self, conf):
         -------
         ds : xr.Dataset
             L0 data
-        '''
-        file_version = conf.get('file_version', -1)  
-        ds = load.getL0(conf['file'], conf['nodata'], conf['columns'], 
-                   conf["skiprows"], file_version, time_offset=conf.get('time_offset'))
+        """
+        file_version = conf.get("file_version", -1)
+        ds = load.getL0(
+            conf["file"],
+            conf["nodata"],
+            conf["columns"],
+            conf["skiprows"],
+            file_version,
+            time_offset=conf.get("time_offset"),
+        )
         ds = utilities.populateMeta(ds, conf, ["columns", "skiprows", "modem"])
         return ds

src/pypromice/process/get_l2.py

@@ -1,9 +1,14 @@
 #!/usr/bin/env python
-import logging, os, sys, unittest
+import logging
+import os
+import sys
 from argparse import ArgumentParser
+from pathlib import Path
+
 from pypromice.process.aws import AWS
 from pypromice.process.write import prepare_and_write
 
+
 def parse_arguments_l2():
     parser = ArgumentParser(description="AWS L2 processor")
 
@@ -17,24 +22,19 @@ def parse_arguments_l2():
                         required=False, help='File path to variables look-up table')
     parser.add_argument('-m', '--metadata', default=None, type=str, 
                         required=False, help='File path to metadata')
+    parser.add_argument('--data_issues_path', default=None, help="Path to data issues repository")
     args = parser.parse_args()
     return args
 
 
-def get_l2(config_file, inpath, outpath, variables, metadata) -> AWS:
-    logging.basicConfig(
-        format="%(asctime)s; %(levelname)s; %(name)s; %(message)s",
-        level=logging.INFO,
-        stream=sys.stdout,
-    )
-    
+def get_l2(config_file, inpath, outpath, variables, metadata, data_issues_path: Path) -> AWS:
     # Define input path
     station_name = config_file.split('/')[-1].split('.')[0] 
     station_path = os.path.join(inpath, station_name)
     if os.path.exists(station_path):
-        aws = AWS(config_file, station_path, variables, metadata)
+        aws = AWS(config_file, station_path, data_issues_repository=data_issues_path, var_file=variables, meta_file=metadata)
     else:
-        aws = AWS(config_file, inpath, variables, metadata)
+        aws = AWS(config_file, inpath, data_issues_repository=data_issues_path, var_file=variables, meta_file=metadata)
 
     # Perform level 1 and 2 processing
     aws.getL1()
@@ -51,7 +51,29 @@ def get_l2(config_file, inpath, outpath, variables, metadata) -> AWS:
 
 def main():
     args = parse_arguments_l2()
-    _ = get_l2(args.config_file, args.inpath, args.outpath, args.variables, args.metadata)
+
+    logging.basicConfig(
+        format="%(asctime)s; %(levelname)s; %(name)s; %(message)s",
+        level=logging.INFO,
+        stream=sys.stdout,
+    )
+
+    data_issues_path = args.data_issues_path
+    if data_issues_path is None:
+        data_issues_path = Path("../PROMICE-AWS-data-issues")
+        if data_issues_path.exists():
+            logging.warning(f"data_issues_path is missing. Using default data issues path: {data_issues_path}")
+        else:
+            raise ValueError(f"data_issues_path is missing. Please provide a valid path to the data issues repository")
+
+    _ = get_l2(
+        args.config_file,
+        args.inpath,
+        args.outpath,
+        args.variables,
+        args.metadata,
+        data_issues_path=data_issues_path,
+    )
 
 
 if __name__ == "__main__":  

src/pypromice/qc/github_data_issues.py

@@ -1,7 +1,5 @@
 import logging
 import os
-import urllib.request
-from urllib.error import HTTPError, URLError
 
 import numpy as np
 import pandas as pd
@@ -16,8 +14,7 @@
 logger = logging.getLogger(__name__)
 
 
-def flagNAN(ds_in,
-            flag_dir='../PROMICE-AWS-data-issues/flags'):
+def flagNAN(ds_in, flag_dir):
     '''Read flagged data from .csv file. For each variable, and downstream
     dependents, flag as invalid (or other) if set in the flag .csv
 
@@ -73,9 +70,7 @@ def flagNAN(ds_in,
     return ds
 
 
-def adjustTime(ds,
-               adj_dir='../PROMICE-AWS-data-issues/adjustments/',
-               var_list=[], skip_var=[]):
+def adjustTime(ds, adj_dir, var_list=[], skip_var=[]):
     '''Read adjustment data from .csv file. Only applies the "time_shift" adjustment
 
     Parameters
@@ -134,9 +129,7 @@ def adjustTime(ds,
     return ds_out
 
 
-def adjustData(ds,
-               adj_dir='../PROMICE-AWS-data-issues/adjustments/',
-               var_list=[], skip_var=[]):
+def adjustData(ds, adj_dir, var_list=[], skip_var=[]):
     '''Read adjustment data from .csv file. For each variable, and downstream
     dependents, adjust data accordingly if set in the adjustment .csv
 

tests/e2e/test_process.py

@@ -62,7 +62,13 @@ def test_add_all(self):
 
     def test_l0_to_l3(self):
         '''Test L0 to L3 processing'''
-        pAWS = AWS(TEST_CONFIG_PATH.as_posix(), TEST_DATA_ROOT_PATH.as_posix())
+        pAWS = AWS(
+            TEST_CONFIG_PATH.as_posix(),
+            TEST_DATA_ROOT_PATH.as_posix(),
+            data_issues_repository=TEST_DATA_ROOT_PATH / 'data_issues',
+            var_file=None,
+            meta_file=None
+        )
         pAWS.process()
         self.assertIsInstance(pAWS.L2, xr.Dataset)
         self.assertTrue(pAWS.L2.attrs['station_id']=='TEST1')
@@ -71,17 +77,17 @@ def get_l2_cli(self):
         '''Test get_l2 CLI'''
         exit_status = os.system('get_l2 -h')
         self.assertEqual(exit_status, 0)
-        
+
     def test_join_l2_cli(self):
         '''Test join_l2 CLI'''
         exit_status = os.system('join_l2 -h')
         self.assertEqual(exit_status, 0)
-        
+
     def test_l2_to_l3_cli(self):
         """Test get_l2tol3 CLI"""
         exit_status = os.system('get_l2tol3 -h')
         self.assertEqual(exit_status, 0)
-        
+
     def test_join_l3_cli(self):
         """Test join_l3 CLI"""
         exit_status = os.system('join_l3 -h')
@@ -100,18 +106,21 @@ def test_full_e2e(self):
             output_path_raw = root / "station_l2_raw"
             config_file_tx = TEST_DATA_ROOT_PATH / "test_config1_tx.toml"
             config_file_raw = TEST_DATA_ROOT_PATH / "test_config1_raw.toml"
+            data_issues_path = TEST_DATA_ROOT_PATH / "data_issues"
             station_id = "TEST1"
             aws_tx_l2 = get_l2(
                 config_file=config_file_tx.as_posix(),
                 inpath=TEST_DATA_ROOT_PATH.as_posix(),
                 outpath=output_path_tx,
+                data_issues_path=data_issues_path,
                 variables=None,
                 metadata=None,
             )
             aws_raw_l2 = get_l2(
                 config_file=config_file_raw.as_posix(),
                 inpath=TEST_DATA_ROOT_PATH.as_posix(),
                 outpath=output_path_raw,
+                data_issues_path=data_issues_path,
                 variables=None,
                 metadata=None,
             )