Add preliminary pipeline class structure and move functions to proces…

…sing.

Not everything works, this is mostly a progress save

climateset/__init__.py

@@ -6,6 +6,7 @@
 DATA_DIR = PROJECT_ROOT / "data"
 RAW_DATA = DATA_DIR / "raw"
 PROCESSED_DATA = DATA_DIR / "processed"
+FINAL_DATA = DATA_DIR / "final"
 LOAD_DATA = DATA_DIR / "load"
 META_DATA = DATA_DIR / "meta"
 SCRIPT_DIR = PROJECT_ROOT / "scripts"

climateset/processing/abstract_processor_step.py

@@ -0,0 +1,13 @@
+from abc import ABC, abstractmethod
+
+
+class AbstractProcessorStep(ABC):
+    def __init__(self):
+        self.results_directory = None
+
+    @abstractmethod
+    def execute(self, input_directory):
+        pass
+
+    def get_results_directory(self):
+        return self.results_directory

climateset/processing/raw/abstract_raw_processing.py → climateset/processing/raw/abstract_raw_processor.py

@@ -12,7 +12,7 @@
 LOGGER = create_logger(__name__)
 
 
-class AbstractRawProcesser(ABC):
+class AbstractRawProcessor(ABC):
     """Abstract class for raw processing."""
 
     def __init__(
@@ -53,7 +53,7 @@ def type_class_meta(self) -> str:
         """Returns the name tag of the subclass."""
 
     @abstractmethod
-    def preprocess_subdir(
+    def process_directory(
         self,
         input_dir: Path,
         cleaned_dir: Path,
@@ -99,6 +99,6 @@ def add_processing_step(self, steps: Union[str, list]):
                 self.processing_steps.append(step)
 
     def list_available_steps(self):
-        step_list = [step for step in self.available_steps.keys() if not step.startswith("_")]
+        step_list = [step for step in self.available_steps if not step.startswith("_")]
         LOGGER.info(f"Available steps: {step_list}")
         return step_list

climateset/processing/raw/checker.py

@@ -178,18 +178,11 @@ def add_lonlat_check(self):
     def add_variables_check(self):
         self.checker_steps.append(VERIFY_VARIABLES)
 
-    def check_directory(self, log_file: Path = None, model: str = "") -> bool:
+    def check_directory(self) -> bool:
         """
         Checking all files in a sub dir.
 
         TAKES TIME.
-        Args:
-            sub_dir (Path): The dir that should be checked
-            log_file (Path): Where problematic files should be logged
-            mode (str): 'w' writes the logs into the file, 'a' appends it to
-                an existing file.
-            model (str): Default is "". Set this, if you want that only
-                the data of this model is checked.
         Returns:
             bool: True if all checks were successful, False if not
         """
@@ -374,7 +367,8 @@ def check_units_ok(self):
             if found_unit != expected_unit:
                 self.results["units_ok"] = False
                 LOGGER.warning(
-                    f"The following file has the unit {found_unit}, but the unit {expected_unit} was expected: \n{self.input_file}"
+                    f"The following file has the unit {found_unit}, but the unit {expected_unit} "
+                    f"was expected: \n{self.input_file}"
                 )
             else:
                 self.results["units_ok"] = True

climateset/processing/raw/cmip6_processing.py → climateset/processing/raw/cmip6_processor.py

@@ -9,13 +9,13 @@
 import xmip.preprocessing as xmip_preprocessing
 from tqdm import tqdm
 
-from climateset.processing.raw.abstract_raw_processing import AbstractRawProcesser
+from climateset.processing.raw.abstract_raw_processor import AbstractRawProcessor
 from climateset.processing.raw.utils import create_generic_output_path
 
 
 # Attention: we need to write _desired_units for our all our data
 # Attention: Before you apply, make sure this is only applied to CMIP6 data!
-class ClimateModelProcesser(AbstractRawProcesser):
+class ClimateModelProcesser(AbstractRawProcessor):
     """
     Can be called to apply xmip preprocessing.
 
@@ -36,7 +36,6 @@ def __init__(
         correct_time_axis: bool = True,
         correct_calendar: bool = True,
         sum_levels: bool = True,
-        **kwargs,
     ):
         """Init function for xmip processer
         Args:
@@ -111,7 +110,7 @@ def update_desired_units_dict(self, var: str, unit: str):
         xmip.preprocessing._desired_units[var] = unit
 
     # share this with input4mips?
-    def preprocess_subdir(
+    def process_directory(
         self,
         sub_dir: Path,
         output_dir: Path,