Merge branch 'refs/heads/dev' into ms/#827-use-java-streams-on-sql-fe…

…tsches-effectively # Conflicts: # CHANGELOG.md
ie3-institute · Jun 13, 2024 · 76b1c29 · 76b1c29
2 parents abad5ed + e392ecc
commit 76b1c29
Show file tree

Hide file tree

Showing 26 changed files with 382 additions and 505 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -16,13 +16,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 - Improvements to the search for corner points in `IdCoordinateSource` [#1016](https://github.com/ie3-institute/PowerSystemDataModel/issues/1016)
+- Refactor `CsvFileConnector` and `CsvDataSource` [#1007](https://github.com/ie3-institute/PowerSystemDataModel/issues/1007)
 - Improving usage of streams on sql fetches [#827](https://github.com/ie3-institute/PowerSystemDataModel/issues/827)
 
-
 ## [5.0.1] - 2024-03-07
 
 ### Fixed
 - Fixed `equals` of `ResultEntity` and `TimeSeriesEntry` [#1037](https://github.com/ie3-institute/PowerSystemDataModel/issues/1037)
+- Fixed "depth of discharge" in documentation [#872](https://github.com/ie3-institute/PowerSystemDataModel/issues/872)
 
 ## [5.0.0] - 2024-03-06
 

diff --git a/build.gradle b/build.gradle
@@ -5,7 +5,7 @@ plugins {
   id 'signing'
   id 'pmd' // code check, working on source code
   id 'com.diffplug.spotless' version '6.25.0' //code format
-  id 'com.github.spotbugs' version '6.0.15' // code check, working on byte code
+  id 'com.github.spotbugs' version '6.0.16' // code check, working on byte code
   id 'de.undercouch.download' version '5.6.0'
   id 'kr.motd.sphinx' version '2.10.1' // documentation generation
   id 'jacoco' // java code coverage plugin
@@ -73,7 +73,7 @@ dependencies {
   testImplementation 'org.junit.jupiter:junit-jupiter:5.10.2'
   testImplementation "org.spockframework:spock-core:2.3-groovy-$groovyVersion"
   testImplementation 'org.objenesis:objenesis:3.4' // Mock creation with constructor parameters
-  testImplementation 'net.bytebuddy:byte-buddy:1.14.16' // Mocks of classes
+  testImplementation 'net.bytebuddy:byte-buddy:1.14.17' // Mocks of classes
 
   // testcontainers (docker framework for testing)
   testImplementation "org.testcontainers:testcontainers:$testcontainersVersion"

diff --git a/docs/readthedocs/models/input/participant/storage.md b/docs/readthedocs/models/input/participant/storage.md
@@ -60,25 +60,6 @@ Model of an ideal electrical battery energy storage.
      - %
      - Efficiency of the electrical inverter
 
-   * - dod
-     - %
-     - | Maximum permissible depth of discharge. 80 % dod
-       | is equivalent to a state of charge of 20 %.
-
-   * - lifeTime
-     - h
-     - Permissible hours of full use
-
-   * - lifeCycle
-     - --
-     - Permissible amount of full cycles
-
-   * - em
-     - --
-     - | UUID reference to an :ref:`Energy Management Unit<em_model>` that is controlling
-       | this system participant. Field can be empty or missing, if this participant
-       | is not controlled.
-
 ```
 
 ### Entity Model
@@ -125,7 +106,12 @@ Model of an ideal electrical battery energy storage.
      - --
      - | Foreseen operation strategy of the storage.
        | Eligible input: *"market"*, *"grid"*, *"self"*
-
+       
+   * - em
+     - --
+     - | UUID reference to an :ref:`Energy Management Unit<em_model>` that is controlling
+       | this system participant. Field can be empty or missing, if this participant
+       | is not controlled.
 
 ```
 

diff --git a/docs/uml/main/input/SystemDatamodelConcept.puml b/docs/uml/main/input/SystemDatamodelConcept.puml
@@ -153,9 +153,6 @@ package models {
                     - pMax: ComparableQuantity<Power> [kW]
                     - activePowerGradient: ComparableQuantity<DimensionlessRate> [%/h]
                     - eta: ComparableQuantity<Dimensionless> [%]
-                    - dod: ComparableQuantity<Dimensionless> [%]
-                    - lifeTime: ComparableQuantity<Time> [h]
-                    - lifeCycle: int
                 }
                 StorageTypeInput --|> SystemParticipantTypeInput
 

diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java
@@ -7,29 +7,23 @@
 
 import edu.ie3.datamodel.exceptions.ConnectorException;
 import edu.ie3.datamodel.io.IoUtil;
-import edu.ie3.datamodel.io.csv.*;
-import edu.ie3.datamodel.io.naming.FileNamingStrategy;
-import edu.ie3.datamodel.io.naming.TimeSeriesMetaInformation;
-import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme;
-import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation;
+import edu.ie3.datamodel.io.csv.BufferedCsvWriter;
+import edu.ie3.datamodel.io.csv.CsvFileDefinition;
 import edu.ie3.datamodel.models.Entity;
 import edu.ie3.datamodel.models.timeseries.TimeSeries;
 import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry;
 import edu.ie3.datamodel.models.value.Value;
 import java.io.*;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.*;
-import java.util.function.Function;
-import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * Provides the connector (here: buffered writer) for specific files to be used by a {@link
- * edu.ie3.datamodel.io.sink.CsvFileSink}
+ * edu.ie3.datamodel.io.sink.CsvFileSink} or {@link edu.ie3.datamodel.io.source.csv.CsvDataSource}
  *
  * @version 0.1
  * @since 19.03.20
@@ -39,27 +33,26 @@ public class CsvFileConnector implements DataConnector {
 
   private final Map<Class<? extends Entity>, BufferedCsvWriter> entityWriters = new HashMap<>();
   private final Map<UUID, BufferedCsvWriter> timeSeriesWriters = new HashMap<>();
-
-  private final FileNamingStrategy fileNamingStrategy;
   private final Path baseDirectory;
-
   private static final String FILE_ENDING = ".csv";
 
-  public CsvFileConnector(Path baseDirectory, FileNamingStrategy fileNamingStrategy) {
+  public CsvFileConnector(Path baseDirectory) {
     this.baseDirectory = baseDirectory;
-    this.fileNamingStrategy = fileNamingStrategy;
+  }
+
+  /** Returns the base directory of this connector. */
+  public Path getBaseDirectory() {
+    return baseDirectory;
   }
 
   public synchronized BufferedCsvWriter getOrInitWriter(
-      Class<? extends Entity> clz, String[] headerElements, String csvSep)
-      throws ConnectorException {
-    /* Try to the the right writer */
+      Class<? extends Entity> clz, CsvFileDefinition fileDefinition) throws ConnectorException {
+    /* Try to the right writer */
     BufferedCsvWriter predefinedWriter = entityWriters.get(clz);
     if (predefinedWriter != null) return predefinedWriter;
 
     /* If it is not available, build and register one */
     try {
-      CsvFileDefinition fileDefinition = buildFileDefinition(clz, headerElements, csvSep);
       BufferedCsvWriter newWriter = initWriter(baseDirectory, fileDefinition);
 
       entityWriters.put(clz, newWriter);
@@ -71,15 +64,14 @@ public synchronized BufferedCsvWriter getOrInitWriter(
   }
 
   public synchronized <T extends TimeSeries<E, V>, E extends TimeSeriesEntry<V>, V extends Value>
-      BufferedCsvWriter getOrInitWriter(T timeSeries, String[] headerElements, String csvSep)
+      BufferedCsvWriter getOrInitWriter(T timeSeries, CsvFileDefinition fileDefinition)
           throws ConnectorException {
-    /* Try to the the right writer */
+    /* Try to the right writer */
     BufferedCsvWriter predefinedWriter = timeSeriesWriters.get(timeSeries.getUuid());
     if (predefinedWriter != null) return predefinedWriter;
 
     /* If it is not available, build and register one */
     try {
-      CsvFileDefinition fileDefinition = buildFileDefinition(timeSeries, headerElements, csvSep);
       BufferedCsvWriter newWriter = initWriter(baseDirectory, fileDefinition);
 
       timeSeriesWriters.put(timeSeries.getUuid(), newWriter);
@@ -131,8 +123,7 @@ public synchronized void closeTimeSeriesWriter(UUID uuid) throws IOException {
     Optional<BufferedCsvWriter> maybeWriter = Optional.ofNullable(timeSeriesWriters.get(uuid));
     if (maybeWriter.isPresent()) {
       log.debug("Remove reference to time series writer for UUID '{}'.", uuid);
-      timeSeriesWriters.remove(uuid);
-      maybeWriter.get().close();
+      timeSeriesWriters.remove(uuid).close();
     } else {
       log.warn("No writer found for time series '{}'.", uuid);
     }
@@ -149,8 +140,7 @@ public synchronized <C extends Entity> void closeEntityWriter(Class<C> clz) thro
     Optional<BufferedCsvWriter> maybeWriter = Optional.ofNullable(entityWriters.get(clz));
     if (maybeWriter.isPresent()) {
       log.debug("Remove reference to entity writer for class '{}'.", clz);
-      entityWriters.remove(clz);
-      maybeWriter.get().close();
+      entityWriters.remove(clz).close();
     } else {
       log.warn("No writer found for class '{}'.", clz);
     }
@@ -170,106 +160,6 @@ public BufferedReader initReader(Path filePath) throws FileNotFoundException {
         new InputStreamReader(new FileInputStream(fullPath), StandardCharsets.UTF_8), 16384);
   }
 
-  /**
-   * Receive the information for specific time series. They are given back filtered by the column
-   * scheme in order to allow for accounting the different content types.
-   *
-   * @param columnSchemes the column schemes to initialize readers for. If no scheme is given, all
-   *     possible readers will be initialized.
-   * @return A mapping from column scheme to the individual time series meta information
-   */
-  public Map<UUID, CsvIndividualTimeSeriesMetaInformation>
-      getCsvIndividualTimeSeriesMetaInformation(final ColumnScheme... columnSchemes) {
-    return getIndividualTimeSeriesFilePaths().parallelStream()
-        .map(
-            filePath -> {
-              /* Extract meta information from file path and enhance it with the file path itself */
-              IndividualTimeSeriesMetaInformation metaInformation =
-                  fileNamingStrategy.individualTimeSeriesMetaInformation(filePath.toString());
-              return new CsvIndividualTimeSeriesMetaInformation(
-                  metaInformation, FileNamingStrategy.removeFileNameEnding(filePath.getFileName()));
-            })
-        .filter(
-            metaInformation ->
-                columnSchemes == null
-                    || columnSchemes.length == 0
-                    || Stream.of(columnSchemes)
-                        .anyMatch(scheme -> scheme.equals(metaInformation.getColumnScheme())))
-        .collect(Collectors.toMap(TimeSeriesMetaInformation::getUuid, Function.identity()));
-  }
-
-  /**
-   * Returns a set of relative paths strings to time series files, with respect to the base folder
-   * path
-   *
-   * @return A set of relative paths to time series files, with respect to the base folder path
-   */
-  private Set<Path> getIndividualTimeSeriesFilePaths() {
-    try (Stream<Path> pathStream = Files.walk(baseDirectory)) {
-      return pathStream
-          .map(baseDirectory::relativize)
-          .filter(
-              path -> {
-                Path withoutEnding =
-                    Path.of(FileNamingStrategy.removeFileNameEnding(path.toString()));
-                return fileNamingStrategy
-                    .getIndividualTimeSeriesPattern()
-                    .matcher(withoutEnding.toString())
-                    .matches();
-              })
-          .collect(Collectors.toSet());
-    } catch (IOException e) {
-      log.error("Unable to determine time series files readers for time series.", e);
-      return Collections.emptySet();
-    }
-  }
-
-  /**
-   * Builds a new file definition consisting of file name and head line elements
-   *
-   * @param timeSeries Time series to derive naming information from
-   * @param headLineElements Array of head line elements
-   * @param csvSep Separator for csv columns
-   * @return A suitable file definition
-   * @throws ConnectorException If the definition cannot be determined
-   */
-  private <T extends TimeSeries<E, V>, E extends TimeSeriesEntry<V>, V extends Value>
-      CsvFileDefinition buildFileDefinition(T timeSeries, String[] headLineElements, String csvSep)
-          throws ConnectorException {
-    Path directoryPath = fileNamingStrategy.getDirectoryPath(timeSeries).orElse(Path.of(""));
-    String fileName =
-        fileNamingStrategy
-            .getEntityName(timeSeries)
-            .orElseThrow(
-                () ->
-                    new ConnectorException(
-                        "Cannot determine the file name for time series '" + timeSeries + "'."));
-    return new CsvFileDefinition(fileName, directoryPath, headLineElements, csvSep);
-  }
-
-  /**
-   * Builds a new file definition consisting of file name and head line elements
-   *
-   * @param clz Class that is meant to be serialized into this file
-   * @param headLineElements Array of head line elements
-   * @param csvSep Separator for csv columns
-   * @return A suitable file definition
-   * @throws ConnectorException If the definition cannot be determined
-   */
-  private CsvFileDefinition buildFileDefinition(
-      Class<? extends Entity> clz, String[] headLineElements, String csvSep)
-      throws ConnectorException {
-    Path directoryPath = fileNamingStrategy.getDirectoryPath(clz).orElse(Path.of(""));
-    String fileName =
-        fileNamingStrategy
-            .getEntityName(clz)
-            .orElseThrow(
-                () ->
-                    new ConnectorException(
-                        "Cannot determine the file name for class '" + clz.getSimpleName() + "'."));
-    return new CsvFileDefinition(fileName, directoryPath, headLineElements, csvSep);
-  }
-
   @Override
   public void shutdown() {
     Stream.of(entityWriters.values(), timeSeriesWriters.values())

diff --git a/src/main/java/edu/ie3/datamodel/io/csv/CsvFileDefinition.java b/src/main/java/edu/ie3/datamodel/io/csv/CsvFileDefinition.java
@@ -5,6 +5,12 @@
 */
 package edu.ie3.datamodel.io.csv;
 
+import edu.ie3.datamodel.exceptions.FileException;
+import edu.ie3.datamodel.io.naming.FileNamingStrategy;
+import edu.ie3.datamodel.models.Entity;
+import edu.ie3.datamodel.models.timeseries.TimeSeries;
+import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry;
+import edu.ie3.datamodel.models.value.Value;
 import edu.ie3.datamodel.utils.FileUtils;
 import java.nio.file.Path;
 import java.util.Arrays;
@@ -23,6 +29,67 @@ public CsvFileDefinition(
     this(FileUtils.ofCsv(fileName, directoryPath), headLineElements, csvSep);
   }
 
+  /**
+   * Builds a new file definition consisting of file name and headline elements
+   *
+   * @param clz Class that is meant to be serialized into this file
+   * @param headLineElements Array of headline elements
+   * @param csvSep Separator for csv columns
+   * @param fileNamingStrategy that should be used
+   * @throws FileException If the definition cannot be determined
+   */
+  public CsvFileDefinition(
+      Class<? extends Entity> clz,
+      String[] headLineElements,
+      String csvSep,
+      FileNamingStrategy fileNamingStrategy)
+      throws FileException {
+    this(
+        FileUtils.ofCsv(
+            fileNamingStrategy
+                .getEntityName(clz)
+                .orElseThrow(
+                    () ->
+                        new FileException(
+                            "Cannot determine the file name for class '"
+                                + clz.getSimpleName()
+                                + "'.")),
+            fileNamingStrategy.getDirectoryPath(clz).orElse(Path.of(""))),
+        headLineElements,
+        csvSep);
+  }
+
+  /**
+   * Builds a new file definition consisting of file name and headline elements
+   *
+   * @param timeSeries Time series to derive naming information from
+   * @param headLineElements Array of headline elements
+   * @param csvSep Separator for csv columns
+   * @param fileNamingStrategy that should be used
+   * @throws FileException If the definition cannot be determined
+   */
+  public <T extends TimeSeries<E, V>, E extends TimeSeriesEntry<V>, V extends Value>
+      CsvFileDefinition(
+          T timeSeries,
+          String[] headLineElements,
+          String csvSep,
+          FileNamingStrategy fileNamingStrategy)
+          throws FileException {
+    this(
+        FileUtils.ofCsv(
+            fileNamingStrategy
+                .getEntityName(timeSeries)
+                .orElseThrow(
+                    () ->
+                        new FileException(
+                            "Cannot determine the file name for time series '"
+                                + timeSeries
+                                + "'.")),
+            fileNamingStrategy.getDirectoryPath(timeSeries).orElse(Path.of(""))),
+        headLineElements,
+        csvSep);
+  }
+
   /**
    * @return The path to the file relative to a not explicitly defined base directory, including the
    *     file extension