Merge branch 'develop' into renaming_3to4_name_identifier

Author: dee-mew

docs/source/cli.rst

@@ -121,8 +121,8 @@ process these files with ``imas process-db-analysis``. This will:
     variable).
 2.  These results are summarized in a table, showing per IDS:
 
-    - The number of data fields [#data_fields]_ that were filled in *any* of the
-      analyzed data entries.
+    - The number of data fields [#data_fields]_ that were filled in *any* occurrence of
+      the IDS in *any* of the analyzed data entries.
     - The total number of data fields [#data_fields]_ that the Data Dictionary
       defines for this IDS.
     - The percentage of fields filled.

imas/db_entry.py

@@ -2,9 +2,12 @@
 # You should have received the IMAS-Python LICENSE file with this project.
 """Logic for interacting with IMAS Data Entries."""
 
+from __future__ import annotations
+
 import logging
 import os
-from typing import Any, List, Optional, Tuple, Type, Union, overload
+import pathlib
+from typing import Any, Type, overload
 
 import numpy
 
@@ -33,14 +36,14 @@
 logger = logging.getLogger(__name__)
 
 
-def _get_uri_mode(uri, mode) -> Tuple[str, str]:
+def _get_uri_mode(uri, mode) -> tuple[str, str]:
     """Helper method to parse arguments of DBEntry.__init__."""
     return uri, mode
 
 
 def _get_legacy_params(
     backend_id, db_name, pulse, run, user_name=None, data_version=None
-) -> Tuple[int, str, int, int, Optional[str], Optional[str]]:
+) -> tuple[int, str, int, int, str | None, str | None]:
     """Helper method to parse arguments of DBEntry.__init__."""
     return backend_id, db_name, pulse, run, user_name, data_version
 
@@ -74,8 +77,8 @@ def __init__(
         uri: str,
         mode: str,
         *,
-        dd_version: Optional[str] = None,
-        xml_path: Optional[str] = None,
+        dd_version: str | None = None,
+        xml_path: str | pathlib.Path | None = None,
     ) -> None: ...
 
     @overload
@@ -85,19 +88,19 @@ def __init__(
         db_name: str,
         pulse: int,
         run: int,
-        user_name: Optional[str] = None,
-        data_version: Optional[str] = None,
+        user_name: str | None = None,
+        data_version: str | None = None,
         *,
-        shot: Optional[int] = None,
-        dd_version: Optional[str] = None,
-        xml_path: Optional[str] = None,
+        shot: int | None = None,
+        dd_version: str | None = None,
+        xml_path: str | pathlib.Path | None = None,
     ) -> None: ...
 
     def __init__(
         self,
         *args,
-        dd_version: Optional[str] = None,
-        xml_path: Optional[str] = None,
+        dd_version: str | None = None,
+        xml_path: str | pathlib.Path | None = None,
         **kwargs,
     ):
         """Open or create a Data Entry based on the provided URI and mode, or prepare a
@@ -162,7 +165,7 @@ def __init__(
                 ) from None
 
         # Actual intializiation
-        self._dbe_impl: Optional[DBEntryImpl] = None
+        self._dbe_impl: DBEntryImpl | None = None
         self._dd_version = dd_version
         self._xml_path = xml_path
         self._ids_factory = IDSFactory(dd_version, xml_path)
@@ -186,7 +189,7 @@ def __init__(
             self._dbe_impl = cls.from_uri(self.uri, mode, self._ids_factory)
 
     @staticmethod
-    def _select_implementation(uri: Optional[str]) -> Type[DBEntryImpl]:
+    def _select_implementation(uri: str | None) -> Type[DBEntryImpl]:
         """Select which DBEntry implementation to use based on the URI."""
         if uri and uri.endswith(".nc") and not uri.startswith("imas:"):
             from imas.backends.netcdf.db_entry_nc import NCDBEntryImpl as impl
@@ -307,7 +310,7 @@ def get(
         lazy: bool = False,
         autoconvert: bool = True,
         ignore_unknown_dd_version: bool = False,
-        destination: Optional[IDSToplevel] = None,
+        destination: IDSToplevel | None = None,
     ) -> IDSToplevel:
         """Read the contents of an IDS into memory.
 
@@ -370,7 +373,7 @@ def get_slice(
         lazy: bool = False,
         autoconvert: bool = True,
         ignore_unknown_dd_version: bool = False,
-        destination: Optional[IDSToplevel] = None,
+        destination: IDSToplevel | None = None,
     ) -> IDSToplevel:
         """Read a single time slice from an IDS in this Database Entry.
 
@@ -434,14 +437,14 @@ def get_sample(
         ids_name: str,
         tmin: float,
         tmax: float,
-        dtime: Optional[Union[float, numpy.ndarray]] = None,
-        interpolation_method: Optional[int] = None,
+        dtime: float | numpy.ndarray | None = None,
+        interpolation_method: int | None = None,
         occurrence: int = 0,
         *,
         lazy: bool = False,
         autoconvert: bool = True,
         ignore_unknown_dd_version: bool = False,
-        destination: Optional[IDSToplevel] = None,
+        destination: IDSToplevel | None = None,
     ) -> IDSToplevel:
         """Read a range of time slices from an IDS in this Database Entry.
 
@@ -547,8 +550,8 @@ def _get(
         self,
         ids_name: str,
         occurrence: int,
-        parameters: Union[None, GetSliceParameters, GetSampleParameters],
-        destination: Optional[IDSToplevel],
+        parameters: None | GetSliceParameters | GetSampleParameters,
+        destination: IDSToplevel | None,
         lazy: bool,
         autoconvert: bool,
         ignore_unknown_dd_version: bool,
@@ -751,12 +754,12 @@ def delete_data(self, ids_name: str, occurrence: int = 0) -> None:
     @overload
     def list_all_occurrences(
         self, ids_name: str, node_path: None = None
-    ) -> List[int]: ...
+    ) -> list[int]: ...
 
     @overload
     def list_all_occurrences(
         self, ids_name: str, node_path: str
-    ) -> Tuple[List[int], List[IDSBase]]: ...
+    ) -> tuple[list[int], list[IDSBase]]: ...
 
     def list_all_occurrences(self, ids_name, node_path=None):
         """List all non-empty occurrences of an IDS

imas/ids_convert.py

@@ -349,13 +349,18 @@ def _add_rename(self, old_path: str, new_path: str) -> None:
 
     def _apply_3to4_conversion(self, old: Element, new: Element) -> None:
         # Postprocessing for COCOS definition change:
+        cocos_paths = []
         for psi_like in ["psi_like", "dodpsi_like"]:
             xpath_query = f".//field[@cocos_label_transformation='{psi_like}']"
             for old_item in old.iterfind(xpath_query):
-                old_path = old_item.get("path")
-                new_path = self.old_to_new.path.get(old_path, old_path)
-                self.new_to_old.post_process[new_path] = _cocos_change
-                self.old_to_new.post_process[old_path] = _cocos_change
+                cocos_paths.append(old_item.get("path"))
+        # Sign flips not covered by the generic rule:
+        cocos_paths.extend(_3to4_sign_flip_paths.get(self.ids_name, []))
+        for old_path in cocos_paths:
+            new_path = self.old_to_new.path.get(old_path, old_path)
+            self.new_to_old.post_process[new_path] = _cocos_change
+            self.old_to_new.post_process[old_path] = _cocos_change
+
         # Definition change for pf_active circuit/connections
         if self.ids_name == "pf_active":
             path = "circuit/connections"
@@ -729,6 +734,113 @@ def _copy_structure(
             callback(item, target_item)
 
 
+_3to4_sign_flip_paths = {
+    "core_instant_changes": [
+        "change/profiles_1d/grid/psi_magnetic_axis",
+        "change/profiles_1d/grid/psi_boundary",
+    ],
+    "core_profiles": [
+        "profiles_1d/grid/psi_magnetic_axis",
+        "profiles_1d/grid/psi_boundary",
+    ],
+    "core_sources": [
+        "source/profiles_1d/grid/psi_magnetic_axis",
+        "source/profiles_1d/grid/psi_boundary",
+    ],
+    "core_transport": [
+        "model/profiles_1d/grid_d/psi_magnetic_axis",
+        "model/profiles_1d/grid_d/psi_boundary",
+        "model/profiles_1d/grid_v/psi_magnetic_axis",
+        "model/profiles_1d/grid_v/psi_boundary",
+        "model/profiles_1d/grid_flux/psi_magnetic_axis",
+        "model/profiles_1d/grid_flux/psi_boundary",
+    ],
+    "disruption": [
+        "global_quantities/psi_halo_boundary",
+        "profiles_1d/grid/psi_magnetic_axis",
+        "profiles_1d/grid/psi_boundary",
+    ],
+    "ece": [
+        "channel/beam_tracing/beam/position/psi",
+        "psi_normalization/psi_magnetic_axis",
+        "psi_normalization/psi_boundary",
+    ],
+    "edge_profiles": [
+        "profiles_1d/grid/psi",
+        "profiles_1d/grid/psi_magnetic_axis",
+        "profiles_1d/grid/psi_boundary",
+    ],
+    "equilibrium": [
+        "time_slice/boundary/psi",
+        "time_slice/global_quantities/q_min/psi",
+        "time_slice/ggd/psi/values",
+    ],
+    "mhd": ["ggd/psi/values"],
+    "pellets": ["time_slice/pellet/path_profiles/psi"],
+    "plasma_profiles": [
+        "profiles_1d/grid/psi",
+        "profiles_1d/grid/psi_magnetic_axis",
+        "profiles_1d/grid/psi_boundary",
+        "ggd/psi/values",
+    ],
+    "plasma_sources": [
+        "source/profiles_1d/grid/psi",
+        "source/profiles_1d/grid/psi_magnetic_axis",
+        "source/profiles_1d/grid/psi_boundary",
+    ],
+    "plasma_transport": [
+        "model/profiles_1d/grid_d/psi",
+        "model/profiles_1d/grid_d/psi_magnetic_axis",
+        "model/profiles_1d/grid_d/psi_boundary",
+        "model/profiles_1d/grid_v/psi",
+        "model/profiles_1d/grid_v/psi_magnetic_axis",
+        "model/profiles_1d/grid_v/psi_boundary",
+        "model/profiles_1d/grid_flux/psi",
+        "model/profiles_1d/grid_flux/psi_magnetic_axis",
+        "model/profiles_1d/grid_flux/psi_boundary",
+    ],
+    "radiation": [
+        "process/profiles_1d/grid/psi_magnetic_axis",
+        "process/profiles_1d/grid/psi_boundary",
+    ],
+    "reflectometer_profile": [
+        "psi_normalization/psi_magnetic_axis",
+        "psi_normalization/psi_boundary",
+    ],
+    "reflectometer_fluctuation": [
+        "psi_normalization/psi_magnetic_axis",
+        "psi_normalization/psi_boundary",
+    ],
+    "runaway_electrons": [
+        "profiles_1d/grid/psi_magnetic_axis",
+        "profiles_1d/grid/psi_boundary",
+    ],
+    "sawteeth": [
+        "profiles_1d/grid/psi_magnetic_axis",
+        "profiles_1d/grid/psi_boundary",
+    ],
+    "summary": [
+        "global_quantities/psi_external_average/value",
+        "local/magnetic_axis/position/psi",
+    ],
+    "transport_solver_numerics": [
+        "solver_1d/grid/psi_magnetic_axis",
+        "solver_1d/grid/psi_boundary",
+        "derivatives_1d/grid/psi_magnetic_axis",
+        "derivatives_1d/grid/psi_boundary",
+    ],
+    "wall": ["description_ggd/ggd/psi/values"],
+    "waves": [
+        "coherent_wave/profiles_1d/grid/psi_magnetic_axis",
+        "coherent_wave/profiles_1d/grid/psi_boundary",
+        "coherent_wave/profiles_2d/grid/psi",
+        "coherent_wave/beam_tracing/beam/position/psi",
+    ],
+}
+"""List of paths per IDS that require a COCOS sign change, but aren't covered by the
+generic rule."""
+
+
 ########################################################################################
 # Type changed handlers and post-processing functions                                  #
 ########################################################################################

imas/ids_factory.py

@@ -1,11 +1,14 @@
 # This file is part of IMAS-Python.
 # You should have received the IMAS-Python LICENSE file with this project.
-"""Tools for generating IDSs from a Data Dictionary version.
-"""
+"""Tools for generating IDSs from a Data Dictionary version."""
+
+from __future__ import annotations
 
 import logging
+import pathlib
+from collections.abc import Iterable, Iterator
 from functools import partial
-from typing import Any, Iterable, Iterator, List, Optional
+from typing import Any
 
 from imas import dd_zip
 from imas.exception import IDSNameError
@@ -27,7 +30,7 @@ class IDSFactory:
     """
 
     def __init__(
-        self, version: Optional[str] = None, xml_path: Optional[str] = None
+        self, version: str | None = None, xml_path: str | pathlib.Path | None = None
     ) -> None:
         """Create a new IDS Factory
 
@@ -77,7 +80,7 @@ def __iter__(self) -> Iterator[str]:
         """Iterate over the IDS names defined by the loaded Data Dictionary"""
         return iter(self._ids_elements)
 
-    def ids_names(self) -> List[str]:
+    def ids_names(self) -> list[str]:
         """Get a list of all known IDS names in the loaded Data Dictionary"""
         return list(self._ids_elements)
 

imas/ids_structure.py

@@ -1,7 +1,6 @@
 # This file is part of IMAS-Python.
 # You should have received the IMAS-Python LICENSE file with this project.
-"""A structure in an IDS
-"""
+"""A structure in an IDS"""
 
 import logging
 from copy import deepcopy
@@ -151,7 +150,9 @@ def __deepcopy__(self, memo):
         for child in self._children:
             if child in self.__dict__:
                 child_copy = deepcopy(getattr(self, child), memo)
-                setattr(copy, child, child_copy)
+                # bypass __setattr__:
+                copy.__dict__[child] = child_copy
+                child_copy._parent = copy
         return copy
 
     def __dir__(self) -> List[str]:

imas/test/test_ids_convert.py

@@ -12,12 +12,14 @@
 
 from imas import identifiers
 from imas.ids_convert import (
+    _3to4_sign_flip_paths,
     _get_ctxpath,
     _get_tbp,
     convert_ids,
     dd_version_map_from_factories,
     iter_parents,
 )
+from imas.ids_data_type import IDSDataType
 from imas.ids_defs import (
     ASCII_BACKEND,
     IDS_TIME_MODE_HETEROGENEOUS,
@@ -571,3 +573,22 @@ def test_4to3_name_identifier_mapping_magnetics():
 
     # DD4 name -> DD3 identifier
     assert dst.b_field_pol_probe[0].identifier == "TEST_NAME"
+
+
+def test_3to4_cocos_hardcoded_paths():
+    # Check for existence in 3.42.0
+    factory = IDSFactory("3.42.0")
+    for ids_name, paths in _3to4_sign_flip_paths.items():
+        ids = factory.new(ids_name)
+        for path in paths:
+            # Check path exists and is not a FLT
+            metadata = ids.metadata[path]
+            assert metadata.data_type is IDSDataType.FLT
+
+    # Test a conversion
+    eq = factory.equilibrium()
+    eq.time_slice.resize(1)
+    eq.time_slice[0].boundary.psi = 3.141
+
+    eq4 = convert_ids(eq, "4.0.0")
+    assert eq4.time_slice[0].boundary.psi == -3.141