Merge branch 'dev' into jb/#762-sql-grid-datasources

CHANGELOG.md

@@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased/Snapshot]
 
+### Added
+
+### Fixed
+
+### Changed
+
+## [5.1.0] - 2024-06-24
+
 ### Added
 - Enhancing `VoltageLevel` with `equals` method [#1063](https://github.com/ie3-institute/PowerSystemDataModel/issues/1063)
 - `ConnectorValidationUtils` checks if parallel devices is > 0 [#1077](https://github.com/ie3-institute/PowerSystemDataModel/issues/1077)
@@ -14,17 +22,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 - Fixed `MappingEntryies` not getting processed by adding `Getter` methods for record fields [#1084](https://github.com/ie3-institute/PowerSystemDataModel/issues/1084) 
+- Fixed "depth of discharge" in documentation [#872](https://github.com/ie3-institute/PowerSystemDataModel/issues/872)
+- Fixed project being build twice in CI [#994](https://github.com/ie3-institute/PowerSystemDataModel/issues/994)
 
 ### Changed
 - Improvements to the search for corner points in `IdCoordinateSource` [#1016](https://github.com/ie3-institute/PowerSystemDataModel/issues/1016)
 - Refactor `CsvFileConnector` and `CsvDataSource` [#1007](https://github.com/ie3-institute/PowerSystemDataModel/issues/1007)
-
+- Make `EntitySource` completely static [#975](https://github.com/ie3-institute/PowerSystemDataModel/issues/975)
+- Abstract commonly used functionality from `EntitySource` [#981](https://github.com/ie3-institute/PowerSystemDataModel/issues/981)
 
 ## [5.0.1] - 2024-03-07
 
 ### Fixed
 - Fixed `equals` of `ResultEntity` and `TimeSeriesEntry` [#1037](https://github.com/ie3-institute/PowerSystemDataModel/issues/1037)
-- Fixed "depth of discharge" in documentation [#872](https://github.com/ie3-institute/PowerSystemDataModel/issues/872)
 
 ## [5.0.0] - 2024-03-06
 
@@ -292,9 +302,10 @@ coordinates or multiple exactly equal coordinates possible
 -   CsvDataSource now stops trying to get an operator for empty operator uuid field in entities
 -   CsvDataSource now parsing multiple geoJson strings correctly
 
-[Unreleased/Snapshot]: https://github.com/ie3-institute/powersystemdatamodel/compare/5.0.1...HEAD
-[5.0.0]: https://github.com/ie3-institute/powersystemdatamodel/compare/5.0.0...5.0.1
-[4.1.0]: https://github.com/ie3-institute/powersystemdatamodel/compare/4.1.0...5.0.0
+[Unreleased/Snapshot]: https://github.com/ie3-institute/powersystemdatamodel/compare/5.1.0...HEAD
+[5.1.0]: https://github.com/ie3-institute/powersystemdatamodel/compare/5.0.1...5.1.0
+[5.0.1]: https://github.com/ie3-institute/powersystemdatamodel/compare/5.0.0...5.0.1
+[5.0.0]: https://github.com/ie3-institute/powersystemdatamodel/compare/4.1.0...5.0.0
 [4.1.0]: https://github.com/ie3-institute/powersystemdatamodel/compare/4.0.0...4.1.0
 [4.0.0]: https://github.com/ie3-institute/powersystemdatamodel/compare/3.0.0...4.0.0
 [3.0.0]: https://github.com/ie3-institute/powersystemdatamodel/compare/2.1.0...3.0.0

Jenkinsfile

@@ -113,7 +113,7 @@ node {
         gradle('--refresh-dependencies clean spotlessCheck pmdMain pmdTest spotbugsMain ' +
             'spotbugsTest test jacocoTestReport jacocoTestCoverageVerification', projectName)
 
-        sh(script: """set +x && cd $projectName""" + ''' set +x; ./gradlew clean javadoc''', returnStdout: true)
+        sh(script: """set +x && cd $projectName""" + ''' set +x; ./gradlew javadoc''', returnStdout: true)
       }
 
       // sonarqube analysis
@@ -156,7 +156,7 @@ node {
              */
             sh(
                 script: """set +x && cd $projectName""" +
-                ''' set +x; ./gradlew clean javadoc''',
+                ''' set +x; ./gradlew javadoc''',
                 returnStdout: true
                 )
 

build.gradle

@@ -5,7 +5,7 @@ plugins {
   id 'signing'
   id 'pmd' // code check, working on source code
   id 'com.diffplug.spotless' version '6.25.0' //code format
-  id 'com.github.spotbugs' version '6.0.16' // code check, working on byte code
+  id 'com.github.spotbugs' version '6.0.18' // code check, working on byte code
   id 'de.undercouch.download' version '5.6.0'
   id 'kr.motd.sphinx' version '2.10.1' // documentation generation
   id 'jacoco' // java code coverage plugin
@@ -17,7 +17,7 @@ ext {
   //version (changing these should be considered thoroughly!)
   javaVersion = JavaVersion.VERSION_17
   groovyVersion = "4.0"
-  groovyBinaryVersion = "4.0.21"
+  groovyBinaryVersion = "4.0.22"
   testcontainersVersion = '1.19.8'
 
   scriptsLocation = 'gradle' + File.separator + 'scripts' + File.separator //location of script plugins
@@ -70,7 +70,7 @@ dependencies {
   // testing
   testImplementation "org.apache.groovy:groovy:$groovyBinaryVersion"
 
-  testImplementation 'org.junit.jupiter:junit-jupiter:5.10.2'
+  testImplementation 'org.junit.jupiter:junit-jupiter:5.10.3'
   testImplementation "org.spockframework:spock-core:2.3-groovy-$groovyVersion"
   testImplementation 'org.objenesis:objenesis:3.4' // Mock creation with constructor parameters
   testImplementation 'net.bytebuddy:byte-buddy:1.14.17' // Mocks of classes
@@ -90,7 +90,7 @@ dependencies {
 
   // Databases
   implementation 'org.influxdb:influxdb-java:2.24'
-  implementation 'com.couchbase.client:java-client:3.6.2'
+  implementation 'com.couchbase.client:java-client:3.7.0'
   runtimeOnly 'org.postgresql:postgresql:42.7.3' // postgresql jdbc driver required during runtime
 
   implementation 'commons-io:commons-io:2.16.1' // I/O functionalities

docs/uml/main/EntitySourceClassDiagram.puml

@@ -3,27 +3,38 @@
 note "Assuming all classes to implement \nthe abstract methods of their interfaces\n\n" as generalNotes
 
 abstract class EntitySource {
-    # DataSource dataSource
-    # {static} <E extends EntityData, T extends UniqueEntity, R extends E> Try<R, SourceException> enrichEntityData(E, String, Map<UUID, T>, BiFunction<E, T, R>)
-    # {static} <E extends EntityData, T1 extends UniqueEntity, T2 extends UniqueEntity, R extends E> Try<R, SourceException> enrichEntityData(E, String, Map<UUID, T1>, String, Map<UUID, T2>, TriFunction<E, T1, T2, R>)
-    # {static} <T extends UniqueEntity> Try<T, SourceException> getLinkedEntity(EntityData, String, Map<UUID, T>)
-    # {static} <E extends EntityData, T extends UniqueEntity, R extends E> Try<R, SourceException> optionallyEnrichEntityData(E, String, Map<UUID, T>, T, BiFunction<E, T, R>)
-    # Stream<Try<NodeAssetInputEntityData, SourceException>> buildNodeAssetEntities(Class<T>, Map<UUID, OperatorInput>, Map<UUID, NodeInput>)
-    # {static} Stream<Try<NodeAssetInputEntityData, SourceException>> nodeAssetInputEntityDataStream(Stream<Try<AssetInputEntityData, SourceException>>, Map<UUID, NodeInput>)
-    # Stream<Try<AssetInputEntityData, SourceException>> buildAssetInputEntities(Class<? extends AssetInput>, Map<UUID, OperatorInput>)
-    # {static} Stream<Try<AssetInputEntityData, SourceException>> assetInputEntityDataStream(Stream<Try<EntityData, SourceException>>, Map<UUID, OperatorInput>)
-    # Stream<Try<EntityData, SourceException>> buildEntityData(Class<? extends UniqueEntity>)
-    # {static} <S extends UniqueEntity> Map<UUID, S> unpackMap(Stream<Try<S, FactoryException>>, Class<S>) throws SourceException
-    # {static} <S extends UniqueEntity> Set<S> unpackSet(Stream<Try<S, FactoryException>>, Class<S>) throws SourceException
-    # {static} <S, E extends Exception> Stream<S> unpackSet(Stream<Try<S, E>>, Class<S>) throws SourceException
+    + void validate() throws ValidationException
 }
 
+EntitySource <|-- EnergyManagementSource
 EntitySource <|-- GraphicSource
+EntitySource <|-- IdCoordinateSource
+IdCoordinateSource <|-- CsvIdCoordinateSource
+IdCoordinateSource <|-- SqlIdCoordinateSource
 EntitySource <|-- RawGridSource
 EntitySource <|-- ResultEntitySource
 EntitySource <|-- SystemParticipantSource
 EntitySource <|-- ThermalSource
+EntitySource <|-- TimeSeriesMappingSource
+TimeSeriesMappingSource <|-- CsvTimeSeriesMappingSource
+TimeSeriesMappingSource <|-- SqlTimeSeriesMappingSource
+EntitySource <|-- TimeSeriesSource
+TimeSeriesSource <|-- CsvTimeSeriesSource
+TimeSeriesSource <|-- SqlTimeSeriesSource
 EntitySource <|-- TypeSource
+EntitySource <|-- WeatherSource
+WeatherSource <|-- CouchbaseWeatherSource
+WeatherSource <|-- CsvWeatherSource
+WeatherSource <|-- InfluxDbWeatherSource
+WeatherSource <|-- SqlWeatherSource
+
+class EnergyManagementSource {
+    - TypeSource typeSource
+    - EmInputFactory emInputFactory
+    + EnergyManagementSource(TypeSource, DataSource)
+    + Map<UUID, EmInput> getEmUnits() throws SourceException
+    + Map<UUID, EmInput> getEmUnits(Map<UUID, OperatorInput>) throws SourceException
+}
 
 class GraphicSource {
     - TypeSource typeSource
@@ -39,6 +50,34 @@ class GraphicSource {
     + Set<LineGraphicInput> getLineGraphicInput(Map<UUID, LineInput>) throws SourceException
 }
 
+abstract class IdCoordinateSource {
+    + Optional<Set<String>> getSourceFields() throws SourceException
+    + Optional<Point> getCoordinate(int)
+    + Collection<Point> getCoordinates(int[])
+    + Optional<Integer> getId(Point)
+    + Collection<Point> getAllCoordinates()
+    + List<CoordinateDistance> getNearestCoordinates(Point, int)
+    + List<CoordinateDistance> getClosestCoordinates(Point, int, ComparableQuantity<Length>)
+    + List<CoordinateDistance> calculateCoordinateDistances(Point, int, Collection<Point>)
+    + List<CoordinateDistance> findCornerPoints(Point, ComparableQuantity<Length>)
+    + List<CoordinateDistance> findCornerPoints(Point, Collection<CoordinateDistance>)
+}
+
+class CsvIdCoordinateSource {
+      - Map<Integer, Point> idToCoordinate;
+      - Map<Point, Integer> coordinateToId;
+      - CsvDataSource dataSource;
+      - IdCoordinateFactory factory;
+      + CsvIdCoordinateSource(IdCoordinateFactory, CsvDataSource) throws SourceException
+      + int getCoordinateCount()
+}
+
+class SqlIdCoordinateSource {
+    - SqlDataSource dataSource
+    - SqlIdCoordinateFactory factory
+    + SqlIdCoordinateSource(SqlIdCoordinateFactory, String, SqlDataSource)
+}
+
 class RawGridSource {
     - TypeSource typeSource
     - NodeInputFactory nodeInputFactory
@@ -70,9 +109,10 @@ class ResultEntitySource {
     - SwitchResultFactory switchResultFactory
     - NodeResultFactory nodeResultFactory
     - ConnectorResultFactory connectorResultFactory
+    - CongestionResultFactory congestionResultFactory
     - FlexOptionsResultFactory flexOptionsResultFactory
     + ResultEntitySource(DataSource)
-    + ResultEntitySource(DataSource, String)
+    + ResultEntitySource(DataSource, DateTimeFormatter)
     + Set<NodeResult> getNodeResults() throws SourceException
     + Set<SwitchResult> getSwitchResults() throws SourceException
     + Set<LineResult> getLineResults() throws SourceException
@@ -92,12 +132,14 @@ class ResultEntitySource {
     + Set<CylindricalStorageResult> getCylindricalStorageResult() throws SourceException
     + Set<ThermalHouseResult> getThermalHouseResults() throws SourceException
     + Set<EmResult> getEmResults() throws SourceException
+    + Set<CongestionResult> getCongestionResults() throws SourceException
 }
 
 class SystemParticipantSource{
     - TypeSource typeSource
     - RawGridSource rawGridSource
     - ThermalSource thermalSource
+    - EnergyManagementSource energyManagementSource
     - BmInputFactory bmInputFactory
     - ChpInputFactory chpInputFactory
     - EvInputFactory evInputFactory
@@ -108,7 +150,7 @@ class SystemParticipantSource{
     - StorageInputFactory storageInputFactory
     - WecInputFactory wecInputFactory
     - EvcsInputFactory evcsInputFactory
-    + SystemParticipantSource(TypeSource, ThermalSource, RawGridSource, DataSource)
+    + SystemParticipantSource(TypeSource, ThermalSource, RawGridSource, EnergyManagementSource, DataSource)
     + SystemParticipants getSystemParticipants() throws SourceException
     + SystemParticipants getSystemParticipants(Map<UUID, OperatorInput>, Map<UUID, NodeInput>) throws SourceException
     + Set<BmInput> getBmPlants() throws SourceException
@@ -141,20 +183,65 @@ class ThermalSource {
     + ThermalSource(TypeSource, DataSource)
     + Map<UUID, ThermalBusInput> getThermalBuses() throws SourceException
     + Map<UUID, ThermalBusInput> getThermalBuses(Map<UUID, OperatorInput>) throws SourceException
-    + Set<ThermalStorageInput> getThermalStorages() throws SourceException
-    + Set<ThermalStorageInput> getThermalStorages(Map<UUID, OperatorInput>, Map<UUID, ThermalBusInput>) throws SourceException
-    + Set<ThermalHouseInput> getThermalHouses() throws SourceException
-    + Set<ThermalHouseInput> getThermalHouses(Map<UUID, OperatorInput>, Map<UUID, ThermalBusInput>) throws SourceException
+    + Map<UUID, ThermalStorageInput> getThermalStorages() throws SourceException
+    + Map<UUID, ThermalStorageInput> getThermalStorages(Map<UUID, OperatorInput>, Map<UUID, ThermalBusInput>) throws SourceException
+    + Map<UUID, ThermalHouseInput> getThermalHouses() throws SourceException
+    + Map<UUID, ThermalHouseInput> getThermalHouses(Map<UUID, OperatorInput>, Map<UUID, ThermalBusInput>) throws SourceException
     + Set<CylindricalStorageInput> getCylindricStorages() throws SourceException
     + Set<CylindricalStorageInput> getCylindricStorages(Map<UUID, OperatorInput>, Map<UUID, ThermalBusInput>) throws SourceException
 }
 
+abstract class TimeSeriesMappingSource {
+    - TimeSeriesMappingFactory mappingFactory
+    + Map<UUID, UUID> getMapping() throws SourceException
+    + Optional<UUID> getTimeSeriesUuid(UUID) throws SourceException
+    + Stream<Map<String, String>> getMappingSourceData() throws SourceException
+    + Optional<Set<String>> getSourceFields() throws SourceException
+}
+
+class CsvTimeSeriesMappingSource {
+    - CsvDataSource dataSource
+    + CsvTimeSeriesMappingSource(String, Path, FileNamingStrategy)
+}
+
+class SqlTimeSeriesMappingSource {
+    - EntityPersistenceNamingStrategy entityPersistenceNamingStrategy
+    - SqlDataSource dataSource
+    + SqlTimeSeriesMappingSource(SqlConnector, String, EntityPersistenceNamingStrategy)
+}
+
+abstract class TimeSeriesSource {
+    - Class<V> valueClass
+    - TimeBasedSimpleValueFactory<V> valueFactory
+    + TimeSeriesSource(Class<V>, TimeBasedSimpleValueFactory<V>)
+    + IndividualTimeSeries<V> getTimeSeries()
+    + IndividualTimeSeries<V> getTimeSeries(ClosedInterval<ZonedDateTime>) throws SourceException
+    + Optional<V> getValue(ZonedDateTime) throws SourceException
+}
+
+class CsvTimeSeriesSource {
+    - IndividualTimeSeries<V> timeSeries
+    - CsvDataSource dataSource
+    - Path filePath
+    + {static} CsvTimeSeriesSource<? extends Value> getSource(String, Path, FileNamingStrategy, CsvIndividualTimeSeriesMetaInformation)
+    + CsvTimeSeriesSource(String, Path, FileNamingStrategy, UUID, Path, Class<V>, TimeBasedSimpleValueFactory<V>)
+}
+
+class SqlTimeSeriesSource {
+    - SqlDataSource dataSource
+    - UUID timeSeriesUuid
+    + SqlTimeSeriesSource(SqlDataSource, UUID, Class<V>, TimeBasedSimpleValueFactory<V>)
+    + SqlTimeSeriesSource(SqlConnector, String, DatabaseNamingStrategy, UUID, Class<V>, TimeBasedSimpleValueFactory<V>)
+    + SqlTimeSeriesSource<? extends Value> createSource(SqlConnector, String, DatabaseNamingStrategy, IndividualTimeSeriesMetaInformation, DateTimeFormatter) throws SourceException
+}
+
 class TypeSource {
     - OperatorInputFactory operatorInputFactory
     - Transformer2WTypeInputFactory transformer2WTypeInputFactory
     - LineTypeInputFactory lineTypeInputFactory
     - Transformer3WTypeInputFactory transformer3WTypeInputFactory
     - SystemParticipantTypeInputFactory systemParticipantTypeInputFactory
+    - DataSource dataSource
     + TypeSource(DataSource)
     + Map<UUID, Transformer2WTypeInput> getTransformer2WTypes() throws SourceException
     + Map<UUID, Transformer3WTypeInput> getTransformer3WTypes() throws SourceException
@@ -167,4 +254,38 @@ class TypeSource {
     + Map<UUID, WecTypeInput> getWecTypes() throws SourceException
     + Map<UUID, EvTypeInput> getEvTypes() throws SourceException
 }
+
+abstract class WeatherSource {
+    - TimeBasedWeatherValueFactory weatherFactory
+    - IdCoordinateSource idCoordinateSource
+    + WeatherSource(IdCoordinateSource, TimeBasedWeatherValueFactory)
+    + Optional<Set<String>> getSourceFields() throws SourceException
+    + Map<Point, IndividualTimeSeries<WeatherValue>> getWeather(ClosedInterval<ZonedDateTime>) throws SourceException
+    + Map<Point, IndividualTimeSeries<WeatherValue>> getWeather(ClosedInterval<ZonedDateTime>, Collection<Point>) throws SourceException
+    + Optional<TimeBasedValue<WeatherValue>> getWeather(ZonedDateTime, Point) throws SourceException
+}
+
+class CouchbaseWeatherSource {
+    - CouchbaseConnector connector
+    + CouchbaseWeatherSource(CouchbaseConnector, IdCoordinateSource, String, TimeBasedWeatherValueFactory, String)
+    + CouchbaseWeatherSource(CouchbaseConnector, IdCoordinateSource, String, String, TimeBasedWeatherValueFactory, String)
+}
+
+class CsvWeatherSource {
+    - CsvDataSource dataSource
+    - Map<Point, IndividualTimeSeries<WeatherValue>> coordinateToTimeSeries
+    + CsvWeatherSource(String, Path, FileNamingStrategy, IdCoordinateSource, TimeBasedWeatherValueFactory) throws SourceException
+}
+
+class InfluxDbWeatherSource {
+    - InfluxDbConnector connector
+    + InfluxDbWeatherSource(InfluxDbConnector, IdCoordinateSource, TimeBasedWeatherValueFactory)
+    + IndividualTimeSeries<WeatherValue> getWeather(ClosedInterval<ZonedDateTime>, Point)
+}
+
+class SqlWeatherSource {
+    - SqlDataSource dataSource
+    + SqlWeatherSource(SqlConnector, IdCoordinateSource, String, String, TimeBasedWeatherValueFactory)
+}
+
 @enduml

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

gradlew.bat

@@ -43,11 +43,11 @@ set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
 if %ERRORLEVEL% equ 0 goto execute
 
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
+echo. 1>&2
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
+echo. 1>&2
+echo Please set the JAVA_HOME variable in your environment to match the 1>&2
+echo location of your Java installation. 1>&2
 
 goto fail
 
@@ -57,11 +57,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe
 
 if exist "%JAVA_EXE%" goto execute
 
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
+echo. 1>&2
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
+echo. 1>&2
+echo Please set the JAVA_HOME variable in your environment to match the 1>&2
+echo location of your Java installation. 1>&2
 
 goto fail
 

src/main/java/edu/ie3/datamodel/io/factory/input/AssetInputEntityFactory.java

@@ -27,6 +27,7 @@ public abstract class AssetInputEntityFactory<T extends AssetInput, D extends As
   private static final String OPERATES_FROM = "operatesFrom";
   private static final String OPERATES_UNTIL = "operatesUntil";
 
+  @SafeVarargs
   protected AssetInputEntityFactory(Class<? extends T>... allowedClasses) {
     super(allowedClasses);
   }