Merge pull request #87 from ioannis-vm/2024-12_use_submodule_files_v2

Use submodule files.

.github/workflows/build_docs.yaml

@@ -12,8 +12,9 @@ jobs:
         os: [ubuntu-latest]
         python-version: ['3.10']
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
         uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}

.github/workflows/deploy_to_pypi.yaml

@@ -1,30 +1,26 @@
 name: Deploy to PyPI
-
 on:
   release:
     types: [created]
-
 jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
-
+        uses: actions/checkout@v4
+        with:
+          submodules: true
       - name: Set up Python
         uses: actions/setup-python@v3
         with:
           python-version: '3.10'
-
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           python -m pip install setuptools wheel twine
-
       - name: Build package
         run: |
           python setup.py sdist bdist_wheel
-
       - name: Publish package to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:

.github/workflows/docs_check.yaml

@@ -1,32 +1,28 @@
 name: Check for Sphinx Warnings
-
 on:
   pull_request:
     paths:
       - "doc/**"
       - "**/*.rst"
       - ".github/workflows/docs_check.yaml"
       - "setup.py"
-
 jobs:
   build:
     runs-on: ubuntu-latest
-
     steps:
     - name: Check out the repository
-      uses: actions/checkout@v3
-
+      uses: actions/checkout@v4
+        with:
+          submodules: true
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
         python-version: '3.x'
-
     - name: Install dependencies
       run: |
           sudo apt-get update
           sudo apt-get install -y pandoc
           python -m pip install -e .[development]
-
     - name: Check for Sphinx warnings
       run: |
         sphinx-build -M html ./doc/source ./doc/_build --fail-on-warning

.github/workflows/format_check.yml

@@ -5,6 +5,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
       - uses: chartboost/ruff-action@v1
         with:
           args: 'format --check'

.github/workflows/lint.yml

@@ -5,6 +5,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
       - uses: chartboost/ruff-action@v1
         with:
           version: 0.7.0

.github/workflows/spell_check.yml

@@ -1,15 +1,12 @@
 name: Spell Check
-
 on: [push, pull_request]
-
 jobs:
   spell-check:
     runs-on: ubuntu-latest
-
     steps:
     - name: Checkout code
-      uses: actions/checkout@v2
-
+      uses: actions/checkout@v4
+        with:
+          submodules: true
     - name: Run codespell
       uses: codespell-project/actions-codespell@v2
-

.github/workflows/tests.yml

@@ -1,9 +1,7 @@
 name: Tests
-
 on:
   - push
   - pull_request
-
 jobs:
   test:
     runs-on: ${{ matrix.os }}
@@ -12,7 +10,9 @@ jobs:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: ['3.9', '3.10', '3.11','3.12']
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:

pelicun/assessment.py

@@ -218,7 +218,10 @@ def get_default_data(self, data_name: str) -> pd.DataFrame:
                 'Please use `loss_repair_DB` instead.'
             )
 
-        data_path = f'{base.pelicun_path}/resources/SimCenterDBDL/{data_name}.csv'
+        data_path = file_io.substitute_default_path(
+            [f'PelicunDefault/{data_name}.csv']
+        )[0]
+        assert isinstance(data_path, str)
 
         data = file_io.load_data(
             data_path, None, orientation=1, reindex=False, log=self.log
@@ -249,7 +252,10 @@ def get_default_metadata(self, data_name: str) -> dict:
                 '`fragility_DB` is deprecated and will be dropped in '
                 'future versions of pelicun. Please use `damage_DB` instead.'
             )
-        data_path = f'{base.pelicun_path}/resources/SimCenterDBDL/{data_name}.json'
+        data_path = file_io.substitute_default_path(
+            [f'PelicunDefault/{data_name}.json']
+        )[0]
+        assert isinstance(data_path, str)
 
         with Path(data_path).open(encoding='utf-8') as f:
             data = json.load(f)

pelicun/file_io.py

@@ -43,6 +43,7 @@
 
 from __future__ import annotations
 
+import json
 from pathlib import Path
 
 import numpy as np
@@ -250,30 +251,33 @@ def substitute_default_path(
     data_paths: list[str | pd.DataFrame],
 ) -> list[str | pd.DataFrame]:
     """
-    Substitute the default directory path with a specified path.
+    Substitute the default directory path.
 
     This function iterates over a list of data paths and replaces
-    occurrences of the 'PelicunDefault/' substring with the path
-    specified by `base.pelicun_path` concatenated with
-    '/resources/SimCenterDBDL/'. This operation is performed to update
-    paths that are using a default location to a user-defined location
-    within the pelicun framework. The updated list of paths is then
-    returned.
+    occurrences of the 'PelicunDefault/' substring by looking up the
+    filename in a resource mapping file located at
+    `{base.pelicun_path}/resources/dlml_resource_paths.json`. If a
+    match is found, the file path is replaced with the value found in
+    the `resource_paths` dictionary.  The updated list of paths is
+    then returned.
 
     Parameters
     ----------
-    data_paths: list of str
+    data_paths: list of str or pd.DataFrame
         A list containing the paths to data files. These paths may
         include a placeholder directory 'PelicunDefault/' that needs
-        to be substituted with the actual path specified in
-        `base.pelicun_path`.
+        to be substituted with the actual path specified in the
+        resource mapping.
 
     Returns
     -------
-    list of str
-        The list with updated paths where 'PelicunDefault/' has been
-        replaced with the specified path in `base.pelicun_path`
-        concatenated with '/resources/SimCenterDBDL/'.
+    list of str or pd.DataFrame
+
+    Raises
+    ------
+    KeyError
+      If the file after 'PelicunDefault/' does not exist in the
+      `resource_paths` keys.
 
     Notes
     -----
@@ -282,26 +286,50 @@ def substitute_default_path(
       are located.
     - If a path in the input list does not contain 'PelicunDefault/',
       it is added to the output list unchanged.
+    - If the file after 'PelicunDefault/' does not exist in the
+      `resource_paths` keys, a `KeyError` is raised.
 
     Examples
     --------
-    >>> data_paths = ['PelicunDefault/data/file1.txt',
-        'data/file2.txt']
+    >>> data_paths = ['PelicunDefault/damage_DB_SimCenter_HU.csv', 'data/file2.txt']
     >>> substitute_default_path(data_paths)
-    ['{base.pelicun_path}/resources/SimCenterDBDL/data/file1.txt',
-    'data/file2.txt']
+    ['{base.pelicun_path}/resources/DamageAndLossModelLibrary/'
+      'hurricane/building/component/fragility.csv',
+      'data/file2.txt']
 
     """
+    # Load the resource paths from the JSON file
+    resource_file_path = (
+        Path(base.pelicun_path) / 'resources' / 'dlml_resource_paths.json'
+    )
+    with resource_file_path.open('r') as file:
+        resource_paths = json.load(file)
+
     updated_paths: list[str | pd.DataFrame] = []
-    for data_path in data_paths:
-        if isinstance(data_path, str) and 'PelicunDefault/' in data_path:
-            path = data_path.replace(
-                'PelicunDefault/',
-                f'{base.pelicun_path}/resources/SimCenterDBDL/',
+    for data_path_str in data_paths:
+        if isinstance(data_path_str, str) and 'PelicunDefault/' in data_path_str:
+            data_path = Path(data_path_str)
+            # Extract the filename after 'PelicunDefault/'
+            file_name = data_path.parts[-1]
+
+            # Check if the filename exists in the resource paths
+            # dictionary
+            if file_name not in resource_paths:
+                msg = f'File `{file_name}` not found in resource paths.'
+                raise KeyError(msg)
+
+            # Substitute the path with the corresponding value from
+            # the dictionary
+            updated_path = str(
+                Path(base.pelicun_path)
+                / 'resources'
+                / 'DamageAndLossModelLibrary'
+                / resource_paths[file_name]
             )
-            updated_paths.append(path)
+            updated_paths.append(updated_path)
         else:
-            updated_paths.append(data_path)
+            updated_paths.append(data_path_str)
+
     return updated_paths
 
 

pelicun/model/damage_model.py

@@ -1313,7 +1313,7 @@ def map_ds(values: np.ndarray, offset: int) -> np.ndarray:
         else:
             # if there are contents, ensure they are valid.
             # See docstring for an example of what is expected.
-            parsed_scaling_specification = {}
+            parsed_scaling_specification: dict = {}
             # validate contents
             for key, value in scaling_specification.items():
                 css = 'capacity adjustment specification'