From 72801af024da8298a9bceaa27a2cacfb3c8b326e Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Mon, 4 Mar 2024 15:11:17 +0100 Subject: [PATCH 1/3] Refactor `CsvFileConnector` and `CsvDataSource`. --- CHANGELOG.md | 1 + .../io/connectors/CsvFileConnector.java | 142 ++----------- .../datamodel/io/csv/CsvFileDefinition.java | 67 ++++++ .../ie3/datamodel/io/sink/CsvFileSink.java | 17 +- .../io/source/csv/CsvDataSource.java | 67 +++++- .../CsvTimeSeriesMetaInformationSource.java | 2 +- .../io/source/csv/CsvWeatherSource.java | 5 +- .../io/connectors/CsvFileConnectorTest.groovy | 197 +----------------- .../io/csv/CsvFileDefinitionTest.groovy | 110 ++++++++++ .../io/source/csv/CsvDataSourceTest.groovy | 78 ++++++- 10 files changed, 356 insertions(+), 330 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9788bc342..a15a87625 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Removing `uuid` as required column from input and result time series [#826](https://github.com/ie3-institute/PowerSystemDataModel/issues/826) - Removing the support for the old csv format that was marked `deprecated` back in version `1.1.0` [#795](https://github.com/ie3-institute/PowerSystemDataModel/issues/795) - BREAKING: Updating PowerSystemUtils dependency to 2.2 [#1006](https://github.com/ie3-institute/PowerSystemDataModel/issues/1006) +- Refactor `CsvFileConnector` and `CsvDataSource` [#1007](https://github.com/ie3-institute/PowerSystemDataModel/issues/1007) ## [4.1.0] - 2023-11-02 diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java index 8bf12c7ce..aaa61974d 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java @@ -6,23 +6,19 @@ package edu.ie3.datamodel.io.connectors; import edu.ie3.datamodel.exceptions.ConnectorException; +import edu.ie3.datamodel.exceptions.FileException; import edu.ie3.datamodel.io.IoUtil; -import edu.ie3.datamodel.io.csv.*; -import edu.ie3.datamodel.io.naming.FileNamingStrategy; -import edu.ie3.datamodel.io.naming.TimeSeriesMetaInformation; -import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; -import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation; +import edu.ie3.datamodel.io.csv.BufferedCsvWriter; +import edu.ie3.datamodel.io.csv.CsvFileDefinition; import edu.ie3.datamodel.models.Entity; import edu.ie3.datamodel.models.timeseries.TimeSeries; import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry; import edu.ie3.datamodel.models.value.Value; +import edu.ie3.datamodel.utils.Try.TrySupplier; import java.io.*; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; import java.util.*; -import java.util.function.Function; -import java.util.stream.Collectors; import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,52 +35,52 @@ public class CsvFileConnector implements DataConnector { private final Map, BufferedCsvWriter> entityWriters = new HashMap<>(); private final Map timeSeriesWriters = new HashMap<>(); - - private final FileNamingStrategy fileNamingStrategy; private final Path baseDirectory; - private static final String FILE_ENDING = ".csv"; - public CsvFileConnector(Path baseDirectory, FileNamingStrategy fileNamingStrategy) { + public CsvFileConnector(Path baseDirectory) { this.baseDirectory = baseDirectory; - this.fileNamingStrategy = fileNamingStrategy; + } + + /** Returns the base directory of this connector. */ + public Path getBaseDirectory() { + return baseDirectory; } public synchronized BufferedCsvWriter getOrInitWriter( - Class clz, String[] headerElements, String csvSep) + Class clz, TrySupplier supplier) throws ConnectorException { - /* Try to the the right writer */ + /* Try to the right writer */ BufferedCsvWriter predefinedWriter = entityWriters.get(clz); if (predefinedWriter != null) return predefinedWriter; /* If it is not available, build and register one */ try { - CsvFileDefinition fileDefinition = buildFileDefinition(clz, headerElements, csvSep); - BufferedCsvWriter newWriter = initWriter(baseDirectory, fileDefinition); + BufferedCsvWriter newWriter = initWriter(baseDirectory, supplier.get()); entityWriters.put(clz, newWriter); return newWriter; - } catch (ConnectorException | IOException e) { + } catch (ConnectorException | FileException | IOException e) { throw new ConnectorException( "Can neither find suitable writer nor build the correct one in CsvFileConnector.", e); } } public synchronized , E extends TimeSeriesEntry, V extends Value> - BufferedCsvWriter getOrInitWriter(T timeSeries, String[] headerElements, String csvSep) + BufferedCsvWriter getOrInitWriter( + T timeSeries, TrySupplier supplier) throws ConnectorException { - /* Try to the the right writer */ + /* Try to the right writer */ BufferedCsvWriter predefinedWriter = timeSeriesWriters.get(timeSeries.getUuid()); if (predefinedWriter != null) return predefinedWriter; /* If it is not available, build and register one */ try { - CsvFileDefinition fileDefinition = buildFileDefinition(timeSeries, headerElements, csvSep); - BufferedCsvWriter newWriter = initWriter(baseDirectory, fileDefinition); + BufferedCsvWriter newWriter = initWriter(baseDirectory, supplier.get()); timeSeriesWriters.put(timeSeries.getUuid(), newWriter); return newWriter; - } catch (ConnectorException | IOException e) { + } catch (ConnectorException | FileException | IOException e) { throw new ConnectorException( "Can neither find suitable writer nor build the correct one in CsvFileConnector.", e); } @@ -170,106 +166,6 @@ public BufferedReader initReader(Path filePath) throws FileNotFoundException { new InputStreamReader(new FileInputStream(fullPath), StandardCharsets.UTF_8), 16384); } - /** - * Receive the information for specific time series. They are given back filtered by the column - * scheme in order to allow for accounting the different content types. - * - * @param columnSchemes the column schemes to initialize readers for. If no scheme is given, all - * possible readers will be initialized. - * @return A mapping from column scheme to the individual time series meta information - */ - public Map - getCsvIndividualTimeSeriesMetaInformation(final ColumnScheme... columnSchemes) { - return getIndividualTimeSeriesFilePaths().parallelStream() - .map( - filePath -> { - /* Extract meta information from file path and enhance it with the file path itself */ - IndividualTimeSeriesMetaInformation metaInformation = - fileNamingStrategy.individualTimeSeriesMetaInformation(filePath.toString()); - return new CsvIndividualTimeSeriesMetaInformation( - metaInformation, FileNamingStrategy.removeFileNameEnding(filePath.getFileName())); - }) - .filter( - metaInformation -> - columnSchemes == null - || columnSchemes.length == 0 - || Stream.of(columnSchemes) - .anyMatch(scheme -> scheme.equals(metaInformation.getColumnScheme()))) - .collect(Collectors.toMap(TimeSeriesMetaInformation::getUuid, Function.identity())); - } - - /** - * Returns a set of relative paths strings to time series files, with respect to the base folder - * path - * - * @return A set of relative paths to time series files, with respect to the base folder path - */ - private Set getIndividualTimeSeriesFilePaths() { - try (Stream pathStream = Files.walk(baseDirectory)) { - return pathStream - .map(baseDirectory::relativize) - .filter( - path -> { - Path withoutEnding = - Path.of(FileNamingStrategy.removeFileNameEnding(path.toString())); - return fileNamingStrategy - .getIndividualTimeSeriesPattern() - .matcher(withoutEnding.toString()) - .matches(); - }) - .collect(Collectors.toSet()); - } catch (IOException e) { - log.error("Unable to determine time series files readers for time series.", e); - return Collections.emptySet(); - } - } - - /** - * Builds a new file definition consisting of file name and head line elements - * - * @param timeSeries Time series to derive naming information from - * @param headLineElements Array of head line elements - * @param csvSep Separator for csv columns - * @return A suitable file definition - * @throws ConnectorException If the definition cannot be determined - */ - private , E extends TimeSeriesEntry, V extends Value> - CsvFileDefinition buildFileDefinition(T timeSeries, String[] headLineElements, String csvSep) - throws ConnectorException { - Path directoryPath = fileNamingStrategy.getDirectoryPath(timeSeries).orElse(Path.of("")); - String fileName = - fileNamingStrategy - .getEntityName(timeSeries) - .orElseThrow( - () -> - new ConnectorException( - "Cannot determine the file name for time series '" + timeSeries + "'.")); - return new CsvFileDefinition(fileName, directoryPath, headLineElements, csvSep); - } - - /** - * Builds a new file definition consisting of file name and head line elements - * - * @param clz Class that is meant to be serialized into this file - * @param headLineElements Array of head line elements - * @param csvSep Separator for csv columns - * @return A suitable file definition - * @throws ConnectorException If the definition cannot be determined - */ - private CsvFileDefinition buildFileDefinition( - Class clz, String[] headLineElements, String csvSep) - throws ConnectorException { - Path directoryPath = fileNamingStrategy.getDirectoryPath(clz).orElse(Path.of("")); - String fileName = - fileNamingStrategy - .getEntityName(clz) - .orElseThrow( - () -> - new ConnectorException( - "Cannot determine the file name for class '" + clz.getSimpleName() + "'.")); - return new CsvFileDefinition(fileName, directoryPath, headLineElements, csvSep); - } - @Override public void shutdown() { Stream.of(entityWriters.values(), timeSeriesWriters.values()) diff --git a/src/main/java/edu/ie3/datamodel/io/csv/CsvFileDefinition.java b/src/main/java/edu/ie3/datamodel/io/csv/CsvFileDefinition.java index 6e2429a45..6afe95a39 100644 --- a/src/main/java/edu/ie3/datamodel/io/csv/CsvFileDefinition.java +++ b/src/main/java/edu/ie3/datamodel/io/csv/CsvFileDefinition.java @@ -5,6 +5,12 @@ */ package edu.ie3.datamodel.io.csv; +import edu.ie3.datamodel.exceptions.FileException; +import edu.ie3.datamodel.io.naming.FileNamingStrategy; +import edu.ie3.datamodel.models.Entity; +import edu.ie3.datamodel.models.timeseries.TimeSeries; +import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry; +import edu.ie3.datamodel.models.value.Value; import edu.ie3.datamodel.utils.FileUtils; import java.nio.file.Path; import java.util.Arrays; @@ -23,6 +29,67 @@ public CsvFileDefinition( this(FileUtils.ofCsv(fileName, directoryPath), headLineElements, csvSep); } + /** + * Builds a new file definition consisting of file name and headline elements + * + * @param clz Class that is meant to be serialized into this file + * @param headLineElements Array of headline elements + * @param csvSep Separator for csv columns + * @param fileNamingStrategy that should be used + * @throws FileException If the definition cannot be determined + */ + public CsvFileDefinition( + Class clz, + String[] headLineElements, + String csvSep, + FileNamingStrategy fileNamingStrategy) + throws FileException { + this( + FileUtils.ofCsv( + fileNamingStrategy + .getEntityName(clz) + .orElseThrow( + () -> + new FileException( + "Cannot determine the file name for class '" + + clz.getSimpleName() + + "'.")), + fileNamingStrategy.getDirectoryPath(clz).orElse(Path.of(""))), + headLineElements, + csvSep); + } + + /** + * Builds a new file definition consisting of file name and headline elements + * + * @param timeSeries Time series to derive naming information from + * @param headLineElements Array of headline elements + * @param csvSep Separator for csv columns + * @param fileNamingStrategy that should be used + * @throws FileException If the definition cannot be determined + */ + public , E extends TimeSeriesEntry, V extends Value> + CsvFileDefinition( + T timeSeries, + String[] headLineElements, + String csvSep, + FileNamingStrategy fileNamingStrategy) + throws FileException { + this( + FileUtils.ofCsv( + fileNamingStrategy + .getEntityName(timeSeries) + .orElseThrow( + () -> + new FileException( + "Cannot determine the file name for time series '" + + timeSeries + + "'.")), + fileNamingStrategy.getDirectoryPath(timeSeries).orElse(Path.of(""))), + headLineElements, + csvSep); + } + /** * @return The path to the file relative to a not explicitly defined base directory, including the * file extension diff --git a/src/main/java/edu/ie3/datamodel/io/sink/CsvFileSink.java b/src/main/java/edu/ie3/datamodel/io/sink/CsvFileSink.java index a7d34a5b7..6cd224bfe 100644 --- a/src/main/java/edu/ie3/datamodel/io/sink/CsvFileSink.java +++ b/src/main/java/edu/ie3/datamodel/io/sink/CsvFileSink.java @@ -8,6 +8,7 @@ import edu.ie3.datamodel.exceptions.*; import edu.ie3.datamodel.io.connectors.CsvFileConnector; import edu.ie3.datamodel.io.csv.BufferedCsvWriter; +import edu.ie3.datamodel.io.csv.CsvFileDefinition; import edu.ie3.datamodel.io.extractor.Extractor; import edu.ie3.datamodel.io.extractor.NestedEntity; import edu.ie3.datamodel.io.naming.FileNamingStrategy; @@ -53,7 +54,7 @@ public class CsvFileSink implements InputDataSink, OutputDataSink { private final CsvFileConnector connector; private final ProcessorProvider processorProvider; - + private final FileNamingStrategy fileNamingStrategy; private final String csvSep; public CsvFileSink(Path baseFolderPath) throws EntityProcessorException { @@ -95,7 +96,8 @@ public CsvFileSink( String csvSep) { this.csvSep = csvSep; this.processorProvider = processorProvider; - this.connector = new CsvFileConnector(baseFolderPath, fileNamingStrategy); + this.connector = new CsvFileConnector(baseFolderPath); + this.fileNamingStrategy = fileNamingStrategy; } @Override @@ -246,7 +248,10 @@ public , V extends Value> void persistTimeSeries( try { TimeSeriesProcessorKey key = new TimeSeriesProcessorKey(timeSeries); String[] headerElements = csvHeaderElements(processorProvider.getHeaderElements(key)); - BufferedCsvWriter writer = connector.getOrInitWriter(timeSeries, headerElements, csvSep); + BufferedCsvWriter writer = + connector.getOrInitWriter( + timeSeries, + () -> new CsvFileDefinition(timeSeries, headerElements, csvSep, fileNamingStrategy)); persistTimeSeries(timeSeries, writer); connector.closeTimeSeriesWriter(timeSeries.getUuid()); } catch (ProcessorProviderException e) { @@ -292,7 +297,11 @@ private void write(C entity) { processorProvider.handleEntity(entity).map(this::csvEntityFieldData).getOrThrow(); String[] headerElements = processorProvider.getHeaderElements(entity.getClass()); BufferedCsvWriter writer = - connector.getOrInitWriter(entity.getClass(), headerElements, csvSep); + connector.getOrInitWriter( + entity.getClass(), + () -> + new CsvFileDefinition( + entity.getClass(), headerElements, csvSep, fileNamingStrategy)); writer.write(entityFieldData); } catch (ProcessorProviderException e) { log.error( diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java index e234f597e..d8a341303 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java @@ -7,7 +7,11 @@ import edu.ie3.datamodel.exceptions.SourceException; import edu.ie3.datamodel.io.connectors.CsvFileConnector; +import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.naming.FileNamingStrategy; +import edu.ie3.datamodel.io.naming.TimeSeriesMetaInformation; +import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme; +import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation; import edu.ie3.datamodel.io.source.DataSource; import edu.ie3.datamodel.models.Entity; import edu.ie3.datamodel.utils.Try; @@ -17,8 +21,10 @@ import java.io.BufferedReader; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.*; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -26,7 +32,7 @@ import org.slf4j.LoggerFactory; /** - * Parent class of all .csv file related sources containing methods and fields consumed by allmost + * Parent class of all .csv file related sources containing methods and fields consumed by almost * all implementations of .csv file related sources. * * @version 0.1 @@ -44,7 +50,7 @@ public class CsvDataSource implements DataSource { public CsvDataSource(String csvSep, Path folderPath, FileNamingStrategy fileNamingStrategy) { this.csvSep = csvSep; - this.connector = new CsvFileConnector(folderPath, fileNamingStrategy); + this.connector = new CsvFileConnector(folderPath); this.fileNamingStrategy = fileNamingStrategy; } @@ -86,6 +92,63 @@ public FileNamingStrategy getNamingStrategy() { return fileNamingStrategy; } + /** + * Receive the information for specific time series. They are given back filtered by the column + * scheme in order to allow for accounting the different content types. + * + * @param columnSchemes the column schemes to initialize readers for. If no scheme is given, all + * possible readers will be initialized. + * @return A mapping from column scheme to the individual time series meta information + */ + public Map + getCsvIndividualTimeSeriesMetaInformation(final ColumnScheme... columnSchemes) { + return getIndividualTimeSeriesFilePaths().parallelStream() + .map( + filePath -> { + /* Extract meta information from file path and enhance it with the file path itself */ + IndividualTimeSeriesMetaInformation metaInformation = + fileNamingStrategy.individualTimeSeriesMetaInformation(filePath.toString()); + return new CsvIndividualTimeSeriesMetaInformation( + metaInformation, FileNamingStrategy.removeFileNameEnding(filePath.getFileName())); + }) + .filter( + metaInformation -> + columnSchemes == null + || columnSchemes.length == 0 + || Stream.of(columnSchemes) + .anyMatch(scheme -> scheme.equals(metaInformation.getColumnScheme()))) + .collect(Collectors.toMap(TimeSeriesMetaInformation::getUuid, Function.identity())); + } + + /** + * Returns a set of relative paths strings to time series files, with respect to the base folder + * path + * + * @return A set of relative paths to time series files, with respect to the base folder path + */ + protected Set getIndividualTimeSeriesFilePaths() { + Path baseDirectory = connector.getBaseDirectory(); + try (Stream pathStream = Files.walk(baseDirectory)) { + return pathStream + .map(baseDirectory::relativize) + .filter( + path -> { + Path withoutEnding = + Path.of(FileNamingStrategy.removeFileNameEnding(path.toString())); + return fileNamingStrategy + .getIndividualTimeSeriesPattern() + .matcher(withoutEnding.toString()) + .matches(); + }) + .collect(Collectors.toSet()); + } catch (IOException e) { + log.error("Unable to determine time series files readers for time series.", e); + return Collections.emptySet(); + } + } + + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + /** * Takes a row string of a .csv file and a string array of the csv file headline, tries to split * the csv row string based and zip it together with the headline. This method does not contain diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMetaInformationSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMetaInformationSource.java index d657d30cb..dbf5114cf 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMetaInformationSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesMetaInformationSource.java @@ -39,7 +39,7 @@ public CsvTimeSeriesMetaInformationSource( this.dataSource = new CsvDataSource(csvSep, folderPath, fileNamingStrategy); // retrieve only the desired time series this.timeSeriesMetaInformation = - dataSource.connector.getCsvIndividualTimeSeriesMetaInformation( + dataSource.getCsvIndividualTimeSeriesMetaInformation( TimeSeriesUtils.getAcceptedColumnSchemes().toArray(new ColumnScheme[0])); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java index fbf78b4f9..9a2eb69aa 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java @@ -139,10 +139,7 @@ private Map> getWeatherTimeSeries() throws SourceException { /* Get only weather time series meta information */ Collection weatherCsvMetaInformation = - dataSource - .connector - .getCsvIndividualTimeSeriesMetaInformation(ColumnScheme.WEATHER) - .values(); + dataSource.getCsvIndividualTimeSeriesMetaInformation(ColumnScheme.WEATHER).values(); return readWeatherTimeSeries(Set.copyOf(weatherCsvMetaInformation), dataSource.connector); } diff --git a/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy index 25210bd3c..aafba5e51 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy @@ -7,26 +7,16 @@ package edu.ie3.datamodel.io.connectors import edu.ie3.datamodel.exceptions.ConnectorException import edu.ie3.datamodel.io.csv.CsvFileDefinition -import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation import edu.ie3.datamodel.io.naming.DefaultDirectoryHierarchy import edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy import edu.ie3.datamodel.io.naming.FileNamingStrategy -import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme -import edu.ie3.datamodel.models.StandardUnits import edu.ie3.datamodel.models.input.NodeInput -import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries -import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue -import edu.ie3.datamodel.models.timeseries.repetitive.RepetitiveTimeSeries -import edu.ie3.datamodel.models.value.EnergyPriceValue import edu.ie3.util.io.FileIOUtils import spock.lang.Shared import spock.lang.Specification -import tech.units.indriya.quantity.Quantities import java.nio.file.Files import java.nio.file.Path -import java.time.ZonedDateTime -import java.util.stream.Collectors class CsvFileConnectorTest extends Specification { @Shared @@ -38,28 +28,12 @@ class CsvFileConnectorTest extends Specification { @Shared CsvFileConnector cfc - @Shared - Set timeSeriesPaths - - @Shared - Set pathsToIgnore - def setupSpec() { tmpDirectory = Files.createTempDirectory("psdm_csv_file_connector_") fileNamingStrategy = new FileNamingStrategy() - cfc = new CsvFileConnector(tmpDirectory, fileNamingStrategy) + cfc = new CsvFileConnector(tmpDirectory) def gridPaths = [Path.of("node_input.csv")] - timeSeriesPaths = [ - "its_pq_53990eea-1b5d-47e8-9134-6d8de36604bf.csv", - "its_p_fcf0b851-a836-4bde-8090-f44c382ed226.csv", - "its_pqh_5022a70e-a58f-4bac-b8ec-1c62376c216b.csv", - "its_c_b88dee50-5484-4136-901d-050d8c1c97d1.csv", - "its_c_c7b0d9d6-5044-4f51-80b4-f221d8b1f14b.csv" - ].stream().map { file -> Path.of(file) }.collect(Collectors.toSet()) - pathsToIgnore = [ - Path.of("file_to_be_ignored.txt") - ] - (gridPaths + pathsToIgnore + timeSeriesPaths).forEach { path -> Files.createFile(tmpDirectory.resolve(path)) } + gridPaths.forEach { path -> Files.createFile(tmpDirectory.resolve(path)) } } def cleanupSpec() { @@ -67,52 +41,6 @@ class CsvFileConnectorTest extends Specification { FileIOUtils.deleteRecursively(tmpDirectory) } - def "The csv file connector is able to provide correct paths to time series files"() { - when: - def actual = cfc.individualTimeSeriesFilePaths - - then: - noExceptionThrown() - - actual.size() == timeSeriesPaths.size() - actual.containsAll(timeSeriesPaths) - } - - def "The csv file connector is able to build correct uuid to meta information mapping"() { - given: - def expected = [ - (UUID.fromString("53990eea-1b5d-47e8-9134-6d8de36604bf")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("53990eea-1b5d-47e8-9134-6d8de36604bf"), ColumnScheme.APPARENT_POWER, Path.of("its_pq_53990eea-1b5d-47e8-9134-6d8de36604bf")), - (UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226"), ColumnScheme.ACTIVE_POWER, Path.of("its_p_fcf0b851-a836-4bde-8090-f44c382ed226")), - (UUID.fromString("5022a70e-a58f-4bac-b8ec-1c62376c216b")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("5022a70e-a58f-4bac-b8ec-1c62376c216b"), ColumnScheme.APPARENT_POWER_AND_HEAT_DEMAND, Path.of("its_pqh_5022a70e-a58f-4bac-b8ec-1c62376c216b")), - (UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_b88dee50-5484-4136-901d-050d8c1c97d1")), - (UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")) - ] - - when: - def actual = cfc.getCsvIndividualTimeSeriesMetaInformation() - - then: - actual == expected - } - - def "The csv file connector is able to build correct uuid to meta information mapping when restricting column schemes"() { - given: - def expected = [ - (UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_b88dee50-5484-4136-901d-050d8c1c97d1")), - (UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")), - (UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226"), ColumnScheme.ACTIVE_POWER, Path.of("its_p_fcf0b851-a836-4bde-8090-f44c382ed226")) - ] - - when: - def actual = cfc.getCsvIndividualTimeSeriesMetaInformation( - ColumnScheme.ENERGY_PRICE, - ColumnScheme.ACTIVE_POWER - ) - - then: - actual == expected - } - def "The csv file connector throws an Exception, if the foreseen file cannot be found"() { when: cfc.initReader(tmpDirectory.resolve("path-does-not-exist")) @@ -135,14 +63,14 @@ class CsvFileConnectorTest extends Specification { def baseDirectory = tmpDirectory.resolve("directoryHierarchy") def directoryHierarchy = new DefaultDirectoryHierarchy(baseDirectory, "test") def fileNamingStrategy = new FileNamingStrategy(new EntityPersistenceNamingStrategy(), directoryHierarchy) - def connector = new CsvFileConnector(baseDirectory, fileNamingStrategy) + def connector = new CsvFileConnector(baseDirectory) and: "expected results" def nodeFile = baseDirectory.resolve(Path.of("test", "input", "grid", "node_input.csv")).toFile() when: /* The head line is of no interest here */ - connector.getOrInitWriter(NodeInput, [] as String[], ",") + connector.getOrInitWriter(NodeInput, () -> new CsvFileDefinition(NodeInput, [] as String[], ",", fileNamingStrategy)) then: noExceptionThrown() @@ -154,14 +82,14 @@ class CsvFileConnectorTest extends Specification { given: "a suitable connector" def baseDirectory = tmpDirectory.resolve("directoryHierarchy") def fileNamingStrategy = new FileNamingStrategy() - def connector = new CsvFileConnector(baseDirectory, fileNamingStrategy) + def connector = new CsvFileConnector(baseDirectory) and: "expected results" def nodeFile = baseDirectory.resolve("node_input.csv").toFile() when: /* The head line is of no interest here */ - connector.getOrInitWriter(NodeInput, [] as String[], ",") + connector.getOrInitWriter(NodeInput, () -> new CsvFileDefinition(NodeInput, [] as String[], ",", fileNamingStrategy)) then: noExceptionThrown() @@ -169,119 +97,6 @@ class CsvFileConnectorTest extends Specification { nodeFile.file // is it a file? } - def "The csv file connector throws ConnectorException if no csv file definition can be built from class information"() { - given: - def fileNamingStrategy = new FileNamingStrategy() - def connector = new CsvFileConnector(tmpDirectory, fileNamingStrategy) - - when: - connector.buildFileDefinition(String, ["a", "b", "c"] as String[], ",") - - then: - def ex = thrown(ConnectorException) - ex.message == "Cannot determine the file name for class 'String'." - } - - def "The csv file connector is able to build correct csv file definition from class upon request"() { - given: - def fileNamingStrategy = new FileNamingStrategy() - def connector = new CsvFileConnector(tmpDirectory, fileNamingStrategy) - def expected = new CsvFileDefinition("node_input.csv", Path.of(""), ["a", "b", "c"] as String[], ",") - - when: - def actual = connector.buildFileDefinition(NodeInput, ["a", "b", "c"] as String[], ",") - - then: - actual.with { - assert it.filePath == expected.filePath - assert it.headLineElements() == expected.headLineElements() - assert it.csvSep() == expected.csvSep() - } - } - - def "The csv file connector is able to build correct csv file definition from class upon request, utilizing directory hierarchy"() { - given: - def fileNamingStrategy = new FileNamingStrategy(new EntityPersistenceNamingStrategy(), new DefaultDirectoryHierarchy(tmpDirectory, "test")) - def connector = new CsvFileConnector(tmpDirectory, fileNamingStrategy) - def expected = new CsvFileDefinition("node_input.csv", Path.of("test", "input", "grid"), ["a", "b", "c"] as String[], ",") - - when: - def actual = connector.buildFileDefinition(NodeInput, ["a", "b", "c"] as String[], ",") - - then: - actual.with { - assert it.filePath == expected.filePath - assert it.headLineElements() == expected.headLineElements() - assert it.csvSep() == expected.csvSep() - } - } - - def "The csv file connector throws ConnectorException if no csv file definition can be built from time series"() { - given: "a suitable connector" - def fileNamingStrategy = new FileNamingStrategy() - def connector = new CsvFileConnector(tmpDirectory, fileNamingStrategy) - - and: "credible input" - def timeSeries = Mock(RepetitiveTimeSeries) - - when: - connector.buildFileDefinition(timeSeries, ["a", "b", "c"] as String[], ",") - - then: - def ex = thrown(ConnectorException) - ex.message == "Cannot determine the file name for time series 'Mock for type 'RepetitiveTimeSeries' named 'timeSeries''." - } - - def "The csv file connector is able to build correct csv file definition from time series upon request"() { - given: "a suitable connector" - def fileNamingStrategy = new FileNamingStrategy() - def connector = new CsvFileConnector(tmpDirectory, fileNamingStrategy) - def expected = new CsvFileDefinition("its_c_0c03ce9f-ab0e-4715-bc13-f9d903f26dbf.csv", Path.of(""), ["a", "b", "c"] as String[], ",") - - and: "credible input" - def entries = [ - new TimeBasedValue(ZonedDateTime.now(), new EnergyPriceValue(Quantities.getQuantity(50d, StandardUnits.ENERGY_PRICE))) - ] as SortedSet - def timeSeries = Mock(IndividualTimeSeries) - timeSeries.uuid >> UUID.fromString("0c03ce9f-ab0e-4715-bc13-f9d903f26dbf") - timeSeries.entries >> entries - - when: - def actual = connector.buildFileDefinition(timeSeries, ["a", "b", "c"] as String[], ",") - - then: - actual.with { - assert it.filePath == expected.filePath - assert it.headLineElements() == expected.headLineElements() - assert it.csvSep() == expected.csvSep() - } - } - - def "The csv file connector is able to build correct csv file definition from time series upon request, utilizing directory hierarchy"() { - given: "a suitable connector" - def fileNamingStrategy = new FileNamingStrategy(new EntityPersistenceNamingStrategy(), new DefaultDirectoryHierarchy(tmpDirectory, "test")) - def connector = new CsvFileConnector(tmpDirectory, fileNamingStrategy) - def expected = new CsvFileDefinition("its_c_0c03ce9f-ab0e-4715-bc13-f9d903f26dbf.csv", Path.of("test", "input", "participants", "time_series"), ["a", "b", "c"] as String[], ",") - - and: "credible input" - def entries = [ - new TimeBasedValue(ZonedDateTime.now(), new EnergyPriceValue(Quantities.getQuantity(50d, StandardUnits.ENERGY_PRICE))) - ] as SortedSet - def timeSeries = Mock(IndividualTimeSeries) - timeSeries.uuid >> UUID.fromString("0c03ce9f-ab0e-4715-bc13-f9d903f26dbf") - timeSeries.entries >> entries - - when: - def actual = connector.buildFileDefinition(timeSeries, ["a", "b", "c"] as String[], ",") - - then: - actual.with { - assert it.filePath == expected.filePath - assert it.headLineElements() == expected.headLineElements() - assert it.csvSep() == expected.csvSep() - } - } - def "Initialising a writer with incorrect base directory leads to ConnectorException"() { given: def baseFolder = tmpDirectory.resolve("helloWorld.txt") diff --git a/src/test/groovy/edu/ie3/datamodel/io/csv/CsvFileDefinitionTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/csv/CsvFileDefinitionTest.groovy index 747d64120..682c7975e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/csv/CsvFileDefinitionTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/csv/CsvFileDefinitionTest.groovy @@ -5,11 +5,24 @@ */ package edu.ie3.datamodel.io.csv +import edu.ie3.datamodel.exceptions.FileException +import edu.ie3.datamodel.io.naming.DefaultDirectoryHierarchy +import edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy +import edu.ie3.datamodel.io.naming.FileNamingStrategy +import edu.ie3.datamodel.models.StandardUnits +import edu.ie3.datamodel.models.input.NodeInput +import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries +import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue +import edu.ie3.datamodel.models.timeseries.repetitive.RepetitiveTimeSeries +import edu.ie3.datamodel.models.value.EnergyPriceValue import org.apache.commons.io.FilenameUtils import spock.lang.Shared import spock.lang.Specification +import tech.units.indriya.quantity.Quantities +import java.nio.file.Files import java.nio.file.Path +import java.time.ZonedDateTime class CsvFileDefinitionTest extends Specification { @Shared @@ -74,4 +87,101 @@ class CsvFileDefinitionTest extends Specification { assert it.csvSep() == this.csvSep } } + + def "The csv file definition can be build correctly from class upon request"() { + given: + def fileNamingStrategy = new FileNamingStrategy() + def expected = new CsvFileDefinition("node_input.csv", Path.of(""), ["a", "b", "c"] as String[], ",") + + when: + def actual = new CsvFileDefinition(NodeInput, ["a", "b", "c"] as String[], ",", fileNamingStrategy) + + then: + actual.with { + assert it.filePath == expected.filePath + assert it.headLineElements() == expected.headLineElements() + assert it.csvSep() == expected.csvSep() + } + } + + def "The csv file connector can be build correctly from class upon request, utilizing directory hierarchy"() { + given: + def tmpDirectory = Files.createTempDirectory("psdm_csv_file_") + def fileNamingStrategy = new FileNamingStrategy(new EntityPersistenceNamingStrategy(), new DefaultDirectoryHierarchy(tmpDirectory, "test")) + def expected = new CsvFileDefinition("node_input.csv", Path.of("test", "input", "grid"), ["a", "b", "c"] as String[], ",") + + when: + def actual = new CsvFileDefinition(NodeInput, ["a", "b", "c"] as String[], ",", fileNamingStrategy) + + then: + actual.with { + assert it.filePath == expected.filePath + assert it.headLineElements() == expected.headLineElements() + assert it.csvSep() == expected.csvSep() + } + } + + def "The csv file definition throws FileException if it cannot be build from time series"() { + given: + def fileNamingStrategy = new FileNamingStrategy() + + and: "credible input" + def timeSeries = Mock(RepetitiveTimeSeries) + + when: + new CsvFileDefinition(timeSeries, ["a", "b", "c"] as String[], ",", fileNamingStrategy) + + then: + def ex = thrown(FileException) + ex.message == "Cannot determine the file name for time series 'Mock for type 'RepetitiveTimeSeries' named 'timeSeries''." + } + + def "The csv file definition can be build correctly from time series upon request"() { + given: + def fileNamingStrategy = new FileNamingStrategy() + def expected = new CsvFileDefinition("its_c_0c03ce9f-ab0e-4715-bc13-f9d903f26dbf.csv", Path.of(""), ["a", "b", "c"] as String[], ",") + + and: "credible input" + def entries = [ + new TimeBasedValue(ZonedDateTime.now(), new EnergyPriceValue(Quantities.getQuantity(50d, StandardUnits.ENERGY_PRICE))) + ] as SortedSet + def timeSeries = Mock(IndividualTimeSeries) + timeSeries.uuid >> UUID.fromString("0c03ce9f-ab0e-4715-bc13-f9d903f26dbf") + timeSeries.entries >> entries + + when: + def actual = new CsvFileDefinition(timeSeries, ["a", "b", "c"] as String[], ",", fileNamingStrategy) + + then: + actual.with { + assert it.filePath == expected.filePath + assert it.headLineElements() == expected.headLineElements() + assert it.csvSep() == expected.csvSep() + } + } + + def "The csv file definition can be build correctly from time series upon request, utilizing directory hierarchy"() { + given: + def tmpDirectory = Files.createTempDirectory("psdm_csv_file_") + def fileNamingStrategy = new FileNamingStrategy(new EntityPersistenceNamingStrategy(), new DefaultDirectoryHierarchy(tmpDirectory, "test")) + def expected = new CsvFileDefinition("its_c_0c03ce9f-ab0e-4715-bc13-f9d903f26dbf.csv", Path.of("test", "input", "participants", "time_series"), ["a", "b", "c"] as String[], ",") + + and: "credible input" + def entries = [ + new TimeBasedValue(ZonedDateTime.now(), new EnergyPriceValue(Quantities.getQuantity(50d, StandardUnits.ENERGY_PRICE))) + ] as SortedSet + def timeSeries = Mock(IndividualTimeSeries) + timeSeries.uuid >> UUID.fromString("0c03ce9f-ab0e-4715-bc13-f9d903f26dbf") + timeSeries.entries >> entries + + when: + def actual = new CsvFileDefinition(timeSeries, ["a", "b", "c"] as String[], ",", fileNamingStrategy) + + then: + actual.with { + assert it.filePath == expected.filePath + assert it.headLineElements() == expected.headLineElements() + assert it.csvSep() == expected.csvSep() + } + } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy index c1dbe8bf2..4979f8b77 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvDataSourceTest.groovy @@ -5,12 +5,16 @@ */ package edu.ie3.datamodel.io.source.csv +import edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation import edu.ie3.datamodel.io.naming.FileNamingStrategy +import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme import edu.ie3.datamodel.models.input.system.LoadInput import spock.lang.Shared import spock.lang.Specification +import java.nio.file.Files import java.nio.file.Path +import java.util.stream.Collectors class CsvDataSourceTest extends Specification implements CsvTestDataMeta { @@ -36,20 +40,38 @@ class CsvDataSourceTest extends Specification implements CsvTestDataMeta { } @Shared - String csvSep = "," + String csvSep @Shared - Path testBaseFolderPath = Path.of("testBaseFolderPath") // does not have to exist for this test + Path testBaseFolderPath @Shared - FileNamingStrategy fileNamingStrategy = new FileNamingStrategy() + FileNamingStrategy fileNamingStrategy + @Shared + Set timeSeriesPaths @Shared - DummyCsvSource dummyCsvSource = new DummyCsvSource(csvSep, testBaseFolderPath, fileNamingStrategy) + DummyCsvSource dummyCsvSource + + def setupSpec() { + csvSep = "," + testBaseFolderPath = Files.createTempDirectory("testBaseFolderPath") + fileNamingStrategy = new FileNamingStrategy() + + dummyCsvSource = new DummyCsvSource(csvSep, testBaseFolderPath, fileNamingStrategy) + + timeSeriesPaths = [ + "its_pq_53990eea-1b5d-47e8-9134-6d8de36604bf.csv", + "its_p_fcf0b851-a836-4bde-8090-f44c382ed226.csv", + "its_pqh_5022a70e-a58f-4bac-b8ec-1c62376c216b.csv", + "its_c_b88dee50-5484-4136-901d-050d8c1c97d1.csv", + "its_c_c7b0d9d6-5044-4f51-80b4-f221d8b1f14b.csv" + ].stream().map { file -> Path.of(file) }.collect(Collectors.toSet()) + timeSeriesPaths.forEach { path -> Files.createFile(testBaseFolderPath.resolve(path)) } + } def "A DataSource should contain a valid connector after initialization"() { expect: dummyCsvSource.connector != null dummyCsvSource.connector.baseDirectory == testBaseFolderPath - dummyCsvSource.connector.fileNamingStrategy == fileNamingStrategy dummyCsvSource.connector.entityWriters.isEmpty() } @@ -275,4 +297,50 @@ class CsvDataSourceTest extends Specification implements CsvTestDataMeta { "5ebd8f7e-dedb-4017-bb86-6373c4b68eb8,25.0,100.0,0.95,98.0,test_bmTypeInput" || "too less columns" "5ebd8f7e-dedb-4017-bb86-6373c4b68eb8,25.0,100.0,0.95,98.0,test_bmTypeInput,,,," || "too much columns" } + + def "The CsvDataSource is able to provide correct paths to time series files"() { + when: + def actual = dummyCsvSource.getIndividualTimeSeriesFilePaths() + + then: + noExceptionThrown() + + actual.size() == timeSeriesPaths.size() + actual.containsAll(timeSeriesPaths) + } + + def "The CsvDataSource is able to build correct uuid to meta information mapping"() { + given: + def expected = [ + (UUID.fromString("53990eea-1b5d-47e8-9134-6d8de36604bf")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("53990eea-1b5d-47e8-9134-6d8de36604bf"), ColumnScheme.APPARENT_POWER, Path.of("its_pq_53990eea-1b5d-47e8-9134-6d8de36604bf")), + (UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226"), ColumnScheme.ACTIVE_POWER, Path.of("its_p_fcf0b851-a836-4bde-8090-f44c382ed226")), + (UUID.fromString("5022a70e-a58f-4bac-b8ec-1c62376c216b")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("5022a70e-a58f-4bac-b8ec-1c62376c216b"), ColumnScheme.APPARENT_POWER_AND_HEAT_DEMAND, Path.of("its_pqh_5022a70e-a58f-4bac-b8ec-1c62376c216b")), + (UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_b88dee50-5484-4136-901d-050d8c1c97d1")), + (UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")) + ] + + when: + def actual = dummyCsvSource.getCsvIndividualTimeSeriesMetaInformation() + + then: + actual == expected + } + + def "The CsvDataSource is able to build correct uuid to meta information mapping when restricting column schemes"() { + given: + def expected = [ + (UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("b88dee50-5484-4136-901d-050d8c1c97d1"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_b88dee50-5484-4136-901d-050d8c1c97d1")), + (UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("c7b0d9d6-5044-4f51-80b4-f221d8b1f14b"), ColumnScheme.ENERGY_PRICE, Path.of("its_c_c7b0d9d6-5044-4f51-80b4-f221d8b1f14b")), + (UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226")): new CsvIndividualTimeSeriesMetaInformation(UUID.fromString("fcf0b851-a836-4bde-8090-f44c382ed226"), ColumnScheme.ACTIVE_POWER, Path.of("its_p_fcf0b851-a836-4bde-8090-f44c382ed226")) + ] + + when: + def actual = dummyCsvSource.getCsvIndividualTimeSeriesMetaInformation( + ColumnScheme.ENERGY_PRICE, + ColumnScheme.ACTIVE_POWER + ) + + then: + actual == expected + } } From fd7629a78ca3cd6b8dedc26aee06929a47b46bc6 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Mon, 15 Apr 2024 11:34:16 +0200 Subject: [PATCH 2/3] Small changes to how `CsvFileDefinition` is provided as an argument. --- CHANGELOG.md | 12 ++++++++-- .../io/connectors/CsvFileConnector.java | 22 +++++++------------ .../ie3/datamodel/io/sink/CsvFileSink.java | 10 ++++----- .../io/source/csv/CsvDataSource.java | 4 ++-- .../io/connectors/CsvFileConnectorTest.groovy | 4 ++-- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bada53b23..b9f9a95ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased/Snapshot] +### Added +- Enhancing `VoltageLevel` with `equals` method [#1063](https://github.com/ie3-institute/PowerSystemDataModel/issues/1063) + +### Fixed + + +### Changed +- Refactor `CsvFileConnector` and `CsvDataSource` [#1007](https://github.com/ie3-institute/PowerSystemDataModel/issues/1007) + + ## [5.0.1] - 2024-03-07 ### Fixed @@ -23,7 +33,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added test for invalid input data in `CsvRawGridSource` [#1021](https://github.com/ie3-institute/PowerSystemDataModel/issues/1021) - Added `CsvThermalGridSource` [#1009](https://github.com/ie3-institute/PowerSystemDataModel/issues/1009) - Enhance documentation for CSV timeseries [#825](https://github.com/ie3-institute/PowerSystemDataModel/issues/825) -- Enhancing `VoltageLevel` with `equals` method [#1063](https://github.com/ie3-institute/PowerSystemDataModel/issues/1063) ### Fixed - Fixed Couchbase integration tests that randomly failed [#755](https://github.com/ie3-institute/PowerSystemDataModel/issues/755) @@ -51,7 +60,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Removing `uuid` as required column from input and result time series [#826](https://github.com/ie3-institute/PowerSystemDataModel/issues/826) - Removing the support for the old csv format that was marked `deprecated` back in version `1.1.0` [#795](https://github.com/ie3-institute/PowerSystemDataModel/issues/795) - BREAKING: Updating PowerSystemUtils dependency to 2.2 [#1006](https://github.com/ie3-institute/PowerSystemDataModel/issues/1006) -- Refactor `CsvFileConnector` and `CsvDataSource` [#1007](https://github.com/ie3-institute/PowerSystemDataModel/issues/1007) ## [4.1.0] - 2023-11-02 diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java index aaa61974d..8c1997abf 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java @@ -6,7 +6,6 @@ package edu.ie3.datamodel.io.connectors; import edu.ie3.datamodel.exceptions.ConnectorException; -import edu.ie3.datamodel.exceptions.FileException; import edu.ie3.datamodel.io.IoUtil; import edu.ie3.datamodel.io.csv.BufferedCsvWriter; import edu.ie3.datamodel.io.csv.CsvFileDefinition; @@ -14,7 +13,6 @@ import edu.ie3.datamodel.models.timeseries.TimeSeries; import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry; import edu.ie3.datamodel.models.value.Value; -import edu.ie3.datamodel.utils.Try.TrySupplier; import java.io.*; import java.nio.charset.StandardCharsets; import java.nio.file.Path; @@ -48,27 +46,25 @@ public Path getBaseDirectory() { } public synchronized BufferedCsvWriter getOrInitWriter( - Class clz, TrySupplier supplier) - throws ConnectorException { + Class clz, CsvFileDefinition fileDefinition) throws ConnectorException { /* Try to the right writer */ BufferedCsvWriter predefinedWriter = entityWriters.get(clz); if (predefinedWriter != null) return predefinedWriter; /* If it is not available, build and register one */ try { - BufferedCsvWriter newWriter = initWriter(baseDirectory, supplier.get()); + BufferedCsvWriter newWriter = initWriter(baseDirectory, fileDefinition); entityWriters.put(clz, newWriter); return newWriter; - } catch (ConnectorException | FileException | IOException e) { + } catch (ConnectorException | IOException e) { throw new ConnectorException( "Can neither find suitable writer nor build the correct one in CsvFileConnector.", e); } } public synchronized , E extends TimeSeriesEntry, V extends Value> - BufferedCsvWriter getOrInitWriter( - T timeSeries, TrySupplier supplier) + BufferedCsvWriter getOrInitWriter(T timeSeries, CsvFileDefinition fileDefinition) throws ConnectorException { /* Try to the right writer */ BufferedCsvWriter predefinedWriter = timeSeriesWriters.get(timeSeries.getUuid()); @@ -76,11 +72,11 @@ BufferedCsvWriter getOrInitWriter( /* If it is not available, build and register one */ try { - BufferedCsvWriter newWriter = initWriter(baseDirectory, supplier.get()); + BufferedCsvWriter newWriter = initWriter(baseDirectory, fileDefinition); timeSeriesWriters.put(timeSeries.getUuid(), newWriter); return newWriter; - } catch (ConnectorException | FileException | IOException e) { + } catch (ConnectorException | IOException e) { throw new ConnectorException( "Can neither find suitable writer nor build the correct one in CsvFileConnector.", e); } @@ -127,8 +123,7 @@ public synchronized void closeTimeSeriesWriter(UUID uuid) throws IOException { Optional maybeWriter = Optional.ofNullable(timeSeriesWriters.get(uuid)); if (maybeWriter.isPresent()) { log.debug("Remove reference to time series writer for UUID '{}'.", uuid); - timeSeriesWriters.remove(uuid); - maybeWriter.get().close(); + timeSeriesWriters.remove(uuid).close(); } else { log.warn("No writer found for time series '{}'.", uuid); } @@ -145,8 +140,7 @@ public synchronized void closeEntityWriter(Class clz) thro Optional maybeWriter = Optional.ofNullable(entityWriters.get(clz)); if (maybeWriter.isPresent()) { log.debug("Remove reference to entity writer for class '{}'.", clz); - entityWriters.remove(clz); - maybeWriter.get().close(); + entityWriters.remove(clz).close(); } else { log.warn("No writer found for class '{}'.", clz); } diff --git a/src/main/java/edu/ie3/datamodel/io/sink/CsvFileSink.java b/src/main/java/edu/ie3/datamodel/io/sink/CsvFileSink.java index 6cd224bfe..8635534a7 100644 --- a/src/main/java/edu/ie3/datamodel/io/sink/CsvFileSink.java +++ b/src/main/java/edu/ie3/datamodel/io/sink/CsvFileSink.java @@ -251,13 +251,13 @@ public , V extends Value> void persistTimeSeries( BufferedCsvWriter writer = connector.getOrInitWriter( timeSeries, - () -> new CsvFileDefinition(timeSeries, headerElements, csvSep, fileNamingStrategy)); + new CsvFileDefinition(timeSeries, headerElements, csvSep, fileNamingStrategy)); persistTimeSeries(timeSeries, writer); connector.closeTimeSeriesWriter(timeSeries.getUuid()); } catch (ProcessorProviderException e) { log.error( "Exception occurred during receiving of header elements. Cannot write this element.", e); - } catch (ConnectorException e) { + } catch (ConnectorException | FileException e) { log.error("Exception occurred during acquisition of writer.", e); } catch (IOException e) { log.error("Exception occurred during closing of writer.", e); @@ -299,14 +299,12 @@ private void write(C entity) { BufferedCsvWriter writer = connector.getOrInitWriter( entity.getClass(), - () -> - new CsvFileDefinition( - entity.getClass(), headerElements, csvSep, fileNamingStrategy)); + new CsvFileDefinition(entity.getClass(), headerElements, csvSep, fileNamingStrategy)); writer.write(entityFieldData); } catch (ProcessorProviderException e) { log.error( "Exception occurred during receiving of header elements. Cannot write this element.", e); - } catch (ConnectorException e) { + } catch (ConnectorException | FileException e) { log.error("Exception occurred during retrieval of writer. Cannot write this element.", e); } catch (IOException e) { log.error("Exception occurred during writing of this element. Cannot write this element.", e); diff --git a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java index d8a341303..3232f2a24 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java @@ -48,9 +48,9 @@ public class CsvDataSource implements DataSource { private final FileNamingStrategy fileNamingStrategy; - public CsvDataSource(String csvSep, Path folderPath, FileNamingStrategy fileNamingStrategy) { + public CsvDataSource(String csvSep, Path directoryPath, FileNamingStrategy fileNamingStrategy) { this.csvSep = csvSep; - this.connector = new CsvFileConnector(folderPath); + this.connector = new CsvFileConnector(directoryPath); this.fileNamingStrategy = fileNamingStrategy; } diff --git a/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy index aafba5e51..273993e10 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/connectors/CsvFileConnectorTest.groovy @@ -70,7 +70,7 @@ class CsvFileConnectorTest extends Specification { when: /* The head line is of no interest here */ - connector.getOrInitWriter(NodeInput, () -> new CsvFileDefinition(NodeInput, [] as String[], ",", fileNamingStrategy)) + connector.getOrInitWriter(NodeInput, new CsvFileDefinition(NodeInput, [] as String[], ",", fileNamingStrategy)) then: noExceptionThrown() @@ -89,7 +89,7 @@ class CsvFileConnectorTest extends Specification { when: /* The head line is of no interest here */ - connector.getOrInitWriter(NodeInput, () -> new CsvFileDefinition(NodeInput, [] as String[], ",", fileNamingStrategy)) + connector.getOrInitWriter(NodeInput, new CsvFileDefinition(NodeInput, [] as String[], ",", fileNamingStrategy)) then: noExceptionThrown() From 3eff07dce27855fedf6bf2491692075005596843 Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Thu, 13 Jun 2024 10:51:39 +0200 Subject: [PATCH 3/3] Updating JavaDoc --- .../java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java index 8c1997abf..7628bf3b2 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java @@ -23,7 +23,7 @@ /** * Provides the connector (here: buffered writer) for specific files to be used by a {@link - * edu.ie3.datamodel.io.sink.CsvFileSink} + * edu.ie3.datamodel.io.sink.CsvFileSink} or {@link edu.ie3.datamodel.io.source.csv.CsvDataSource} * * @version 0.1 * @since 19.03.20