diff --git a/.gitattributes b/.gitattributes index c0b77356d..6ef183f09 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,3 @@ # The following file will be copied to a unix Docker image and imported to InfluxDB data base. Therefore, the line # ending plays a crucial role. This prevents the endings from being adjusted with 'core.autocrlf=true' -src/test/resources/testContainerFiles/influxDb/weather.txt eol=lf +src/test/resources/testContainerFiles/influxDb/cosmo/weather.txt eol=lf diff --git a/CHANGELOG.md b/CHANGELOG.md index 13dad1f23..d107a0b77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `edu.ie3.datamodel.io.csv.timeseries.LoadProfileTimeSeriesMetaInformation` - `edu.ie3.datamodel.io.connectors.CsvFileConnector.CsvIndividualTimeSeriesMetaInformation` - and related methods +- BREAKING: Comprehensive harmonization around weather sources [#267](https://github.com/ie3-institute/PowerSystemDataModel/issues/267) + - Adapted the expected column scheme + - General weather model + - `coordinate` to `coordinateid` + - DWD COSMO model + - `diffuseirradiation` to `diffuseirradiance` + - `directirradiation` to `directirradiance` ## [2.1.0] - 2022-01-05 diff --git a/build.gradle b/build.gradle index fc3e30d23..ab808d600 100644 --- a/build.gradle +++ b/build.gradle @@ -40,7 +40,7 @@ apply from: scriptsLocation + 'vcs.gradle' apply from: scriptsLocation + 'semVer.gradle' repositories { - mavenCentral() //searches in bintray's repository 'jCenter', which contains Maven Central + mavenCentral() // searches in Sonatype's repository 'Maven Central' maven { url 'https://www.jitpack.io' } // allows github repos as dependencies // sonatype snapshot repo diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java similarity index 87% rename from src/main/java/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactory.java rename to src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java index f9ddd6af1..a95affade 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java @@ -25,31 +25,25 @@ * Factory implementation of {@link TimeBasedWeatherValueFactory}, that is able to handle field to * value mapping in the typical PowerSystemDataModel (PSDM) column scheme */ -public class PsdmTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory { - private static final String COORDINATE = "coordinate"; - private static final String DIFFUSE_IRRADIANCE = "diffuseirradiation"; - private static final String DIRECT_IRRADIANCE = "directirradiation"; +public class CosmoTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory { + private static final String DIFFUSE_IRRADIANCE = "diffuseirradiance"; + private static final String DIRECT_IRRADIANCE = "directirradiance"; private static final String TEMPERATURE = "temperature"; private static final String WIND_DIRECTION = "winddirection"; private static final String WIND_VELOCITY = "windvelocity"; - public PsdmTimeBasedWeatherValueFactory(TimeUtil timeUtil) { + public CosmoTimeBasedWeatherValueFactory(TimeUtil timeUtil) { super(timeUtil); } - public PsdmTimeBasedWeatherValueFactory(String timePattern) { + public CosmoTimeBasedWeatherValueFactory(String timePattern) { super(timePattern); } - public PsdmTimeBasedWeatherValueFactory() { + public CosmoTimeBasedWeatherValueFactory() { super(); } - @Override - public String getCoordinateIdFieldString() { - return COORDINATE; - } - @Override public String getTimeFieldString() { return TIME; diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java index b26c9910d..a2e23b1f2 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java @@ -29,7 +29,6 @@ */ public class IconTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory { /* Redefine the column names to meet the icon specifications */ - private static final String COORDINATE = "coordinateid"; private static final String TIME = "datum"; private static final String DIFFUSE_IRRADIANCE = "aswdifdS"; private static final String DIRECT_IRRADIANCE = "aswdirS"; @@ -49,11 +48,6 @@ public IconTimeBasedWeatherValueFactory() { super(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd HH:mm:ss")); } - @Override - public String getCoordinateIdFieldString() { - return COORDINATE; - } - @Override public String getTimeFieldString() { return TIME; diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java index 526850763..4ad3b3348 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java @@ -18,6 +18,7 @@ public abstract class TimeBasedWeatherValueFactory extends TimeBasedValueFactory { protected static final String UUID = "uuid"; protected static final String TIME = "time"; + protected static final String COORDINATE_ID = "coordinateid"; protected final TimeUtil timeUtil; @@ -39,7 +40,9 @@ protected TimeBasedWeatherValueFactory(TimeUtil timeUtil) { * * @return the field name for the coordinate id */ - public abstract String getCoordinateIdFieldString(); + public String getCoordinateIdFieldString() { + return COORDINATE_ID; + } /** * Return the field name for the date time diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index a424cf894..a183a3dbc 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -29,8 +29,9 @@ public class InfluxDbWeatherSource implements WeatherSource { private static final String BASIC_QUERY_STRING = "Select * from weather"; private static final String WHERE = " where "; private static final String MEASUREMENT_NAME_WEATHER = "weather"; + private static final String COORDINATE_ID_COLUMN_NAME = "coordinate_id"; private static final int MILLI_TO_NANO_FACTOR = 1000000; - private final String coordinateIdColumnName; + private final InfluxDbConnector connector; private final IdCoordinateSource coordinateSource; private final TimeBasedWeatherValueFactory weatherValueFactory; @@ -50,7 +51,6 @@ public InfluxDbWeatherSource( this.connector = connector; this.coordinateSource = coordinateSource; this.weatherValueFactory = weatherValueFactory; - this.coordinateIdColumnName = weatherValueFactory.getCoordinateIdFieldString(); } @Override @@ -113,7 +113,7 @@ public Map> getWeather( public IndividualTimeSeries getWeather( ClosedInterval timeInterval, Point coordinate) { Optional coordinateId = coordinateSource.getId(coordinate); - if (!coordinateId.isPresent()) { + if (coordinateId.isEmpty()) { return new IndividualTimeSeries<>(UUID.randomUUID(), Collections.emptySet()); } try (InfluxDB session = connector.getSession()) { @@ -130,7 +130,7 @@ public IndividualTimeSeries getWeather( @Override public Optional> getWeather(ZonedDateTime date, Point coordinate) { Optional coordinateId = coordinateSource.getId(coordinate); - if (!coordinateId.isPresent()) { + if (coordinateId.isEmpty()) { return Optional.empty(); } try (InfluxDB session = connector.getSession()) { @@ -141,35 +141,37 @@ public Optional> getWeather(ZonedDateTime date, Poi } /** - * Parses an influxQL QueryResult and then transforms them into a Stream of optional - * TimeBasedValue<WeatherValue>, with a present Optional value, if the transformation was - * successful and an empty optional otherwise. + * Parses an influxQL QueryResult and then transforms it into a Stream of optional + * TimeBasedValue<WeatherValue>, with a present Optional value if the transformation was + * successful and an empty Optional otherwise. */ private Stream>> optTimeBasedValueStream( QueryResult queryResult) { Map>> measurementsMap = InfluxDbConnector.parseQueryResult(queryResult, MEASUREMENT_NAME_WEATHER); + final String coordinateIdFieldName = weatherValueFactory.getCoordinateIdFieldString(); return measurementsMap.get(MEASUREMENT_NAME_WEATHER).stream() .map( fieldToValue -> { - Optional coordinate = - coordinateSource.getCoordinate( - Integer.parseInt(fieldToValue.remove(coordinateIdColumnName))); - if (!coordinate.isPresent()) return null; - fieldToValue.putIfAbsent("uuid", UUID.randomUUID().toString()); - - /* The factory expects camel case id's for fields -> Convert the keys */ - Map camelCaseFields = + /* The factory expects flat case id's for fields -> Convert the keys */ + Map flatCaseFields = fieldToValue.entrySet().stream() .collect( Collectors.toMap( - entry -> StringUtils.snakeCaseToCamelCase(entry.getKey()), + entry -> + StringUtils.snakeCaseToCamelCase(entry.getKey()).toLowerCase(), Map.Entry::getValue)); - return new TimeBasedWeatherValueData(camelCaseFields, coordinate.get()); - }) - .filter(Objects::nonNull) - .map(weatherValueFactory::get); + /* Add a random UUID if necessary */ + flatCaseFields.putIfAbsent("uuid", UUID.randomUUID().toString()); + + /* Get the corresponding coordinate id from map AND REMOVE THE ENTRY !!! */ + int coordinateId = Integer.parseInt(flatCaseFields.remove(coordinateIdFieldName)); + return coordinateSource + .getCoordinate(coordinateId) + .map(point -> new TimeBasedWeatherValueData(flatCaseFields, point)) + .flatMap(weatherValueFactory::get); + }); } private String createQueryStringForCoordinateAndTimeInterval( @@ -205,7 +207,7 @@ private String createTimeConstraint(ZonedDateTime date) { } private String createCoordinateConstraintString(int coordinateId) { - return "coordinate='" + coordinateId + "'"; + return COORDINATE_ID_COLUMN_NAME + "='" + coordinateId + "'"; } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 60610d77d..ab38ac5f0 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -60,17 +60,17 @@ public SqlWeatherSource( String dbTimeColumnName = getDbColumnName(weatherFactory.getTimeFieldString(), weatherTableName); - String dbCoordColumnName = getDbColumnName(factoryCoordinateFieldName, weatherTableName); + String dbCoordinateIdColumnName = getDbColumnName(factoryCoordinateFieldName, weatherTableName); // setup queries this.queryTimeInterval = createQueryStringForTimeInterval(schemaName, weatherTableName, dbTimeColumnName); this.queryTimeAndCoordinate = createQueryStringForTimeAndCoordinate( - schemaName, weatherTableName, dbTimeColumnName, dbCoordColumnName); + schemaName, weatherTableName, dbTimeColumnName, dbCoordinateIdColumnName); this.queryTimeIntervalAndCoordinates = createQueryStringForTimeIntervalAndCoordinates( - schemaName, weatherTableName, dbTimeColumnName, dbCoordColumnName); + schemaName, weatherTableName, dbTimeColumnName, dbCoordinateIdColumnName); } @Override diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactoryTest.groovy similarity index 67% rename from src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy rename to src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactoryTest.groovy index b28d89f33..df296966b 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactoryTest.groovy @@ -8,27 +8,27 @@ package edu.ie3.datamodel.io.factory.timeseries import edu.ie3.datamodel.models.StandardUnits import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.util.TimeUtil import spock.lang.Specification import tech.units.indriya.quantity.Quantities -class PsdmTimeBasedWeatherValueFactoryTest extends Specification { +class CosmoTimeBasedWeatherValueFactoryTest extends Specification { def "A PsdmTimeBasedWeatherValueFactory should be able to create time series with missing values"() { given: - def factory = new PsdmTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss") - def coordinate = PsdmWeatherTestData.COORDINATE_193186 + def factory = new CosmoTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss") + def coordinate = CosmoWeatherTestData.COORDINATE_193186 def time = TimeUtil.withDefaults.toZonedDateTime("2019-01-01 00:00:00") Map parameter = [ - "uuid" : "980f7714-8def-479f-baae-4deed6c8d6d1", - "time" : TimeUtil.withDefaults.toString(time), - "diffuseirradiation": "282.671997070312", - "directirradiation" : "286.872985839844", - "temperature" : "", - "winddirection" : "0", - "windvelocity" : "1.66103506088257" + "uuid" : "980f7714-8def-479f-baae-4deed6c8d6d1", + "time" : TimeUtil.withDefaults.toString(time), + "diffuseirradiance": "282.671997070312", + "directirradiance" : "286.872985839844", + "temperature" : "", + "winddirection" : "0", + "windvelocity" : "1.66103506088257" ] def data = new TimeBasedWeatherValueData(parameter, coordinate) @@ -50,18 +50,18 @@ class PsdmTimeBasedWeatherValueFactoryTest extends Specification { def "A PsdmTimeBasedWeatherValueFactory should be able to create time series values"() { given: - def factory = new PsdmTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss") - def coordinate = PsdmWeatherTestData.COORDINATE_193186 + def factory = new CosmoTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss") + def coordinate = CosmoWeatherTestData.COORDINATE_193186 def time = TimeUtil.withDefaults.toZonedDateTime("2019-01-01 00:00:00") Map parameter = [ - "time" : TimeUtil.withDefaults.toString(time), - "uuid" : "980f7714-8def-479f-baae-4deed6c8d6d1", - "diffuseirradiation": "282.671997070312", - "directirradiation" : "286.872985839844", - "temperature" : "278.019012451172", - "winddirection" : "0", - "windvelocity" : "1.66103506088257" + "time" : TimeUtil.withDefaults.toString(time), + "uuid" : "980f7714-8def-479f-baae-4deed6c8d6d1", + "diffuseirradiance": "282.671997070312", + "directirradiance" : "286.872985839844", + "temperature" : "278.019012451172", + "winddirection" : "0", + "windvelocity" : "1.66103506088257" ] def data = new TimeBasedWeatherValueData(parameter, coordinate) diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy index 93bc7b37d..fb37a49bd 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy @@ -6,7 +6,7 @@ package edu.ie3.datamodel.io.factory.timeseries import edu.ie3.datamodel.models.StandardUnits -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.util.TimeUtil import edu.ie3.util.quantities.PowerSystemUnits import edu.ie3.util.quantities.QuantityUtil @@ -73,7 +73,7 @@ class IconTimeBasedWeatherValueFactoryTest extends Specification { def "A time based weather value factory for ICON column scheme builds a single time based value correctly"() { given: def factory = new IconTimeBasedWeatherValueFactory() - def coordinate = PsdmWeatherTestData.COORDINATE_67775 + def coordinate = CosmoWeatherTestData.COORDINATE_67775 def parameter = [ "datum" : "2019-08-01 01:00:00", diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy similarity index 59% rename from src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy rename to src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index 643c28b0c..117f48081 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -6,11 +6,11 @@ package edu.ie3.datamodel.io.source.couchbase import edu.ie3.datamodel.io.connectors.CouchbaseConnector -import edu.ie3.datamodel.io.factory.timeseries.PsdmTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.TimeUtil import edu.ie3.util.interval.ClosedInterval @@ -25,7 +25,7 @@ import spock.lang.Specification import java.time.ZoneId @Testcontainers -class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourceTestHelper { +class CouchbaseWeatherSourceCosmoIT extends Specification implements WeatherSourceTestHelper { @Shared BucketDefinition bucketDefinition = new BucketDefinition("ie3_in") @@ -37,11 +37,11 @@ class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourc @Shared CouchbaseWeatherSource source - static String coordinateIdColumnName = "coordinate" + static String coordinateIdColumnName = "coordinateid" def setupSpec() { // Copy import file with json array of documents into docker - MountableFile couchbaseWeatherJsonsFile = MountableFile.forClasspathResource("/testcontainersFiles/couchbase/weather.json") + MountableFile couchbaseWeatherJsonsFile = MountableFile.forClasspathResource("/testcontainersFiles/couchbase/cosmo/weather.json") couchbaseContainer.copyFileToContainer(couchbaseWeatherJsonsFile, "/home/weather.json") // create an index for the document keys @@ -62,8 +62,8 @@ class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourc "--dataset", "file:///home/weather.json") def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) - def weatherFactory = new PsdmTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd'T'HH:mm:ssxxx")) - source = new CouchbaseWeatherSource(connector, PsdmWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory) + def weatherFactory = new CosmoTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd'T'HH:mm:ssxxx")) + source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory) } def "The test container can establish a valid connection"() { @@ -75,10 +75,10 @@ class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourc def "A CouchbaseWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { given: - def expectedTimeBasedValue = new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H) + def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) when: - def optTimeBasedValue = source.getWeather(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.COORDINATE_193186) + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: optTimeBasedValue.present @@ -88,54 +88,54 @@ class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourc def "A CouchbaseWeatherSource can read multiple time series values for multiple coordinates"() { given: def coordinates = [ - PsdmWeatherTestData.COORDINATE_193186, - PsdmWeatherTestData.COORDINATE_193187 + CosmoWeatherTestData.COORDINATE_193186, + CosmoWeatherTestData.COORDINATE_193187 ] - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.TIME_17H) def timeSeries193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H) + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H) ] as Set) def timeSeries193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H) + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H) ] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) then: coordinateToTimeSeries.keySet().size() == 2 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186), timeSeries193186) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187), timeSeries193187) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186), timeSeries193186) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeSeries193187) } def "A CouchbaseWeatherSource can read all weather data in a given time interval"() { given: - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) def timeSeries193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H) ] as Set) def timeSeries193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193187_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193187_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H) ] as Set) def timeSeries193188 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193188_15H) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193188_15H) ] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval) then: coordinateToTimeSeries.keySet().size() == 3 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy index b51356c0a..f3c5459e3 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy @@ -20,6 +20,9 @@ trait CsvTestDataMeta { static String participantsFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("participants") static String resultEntitiesFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("results") static String timeSeriesFolderPath = testTimeSeriesBaseFolderPath + static String weatherFolderPath = new File(CsvTestDataMeta.getResource('weather').toURI()).absolutePath + static String cosmoWeatherFolderPath = weatherFolderPath.concat(File.separator).concat("cosmo") + static String iconWeatherFolderPath = weatherFolderPath.concat(File.separator).concat("icon") static String thermalFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("thermal") static String coordinatesFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("coordinates") diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy new file mode 100644 index 000000000..b93cdbf11 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -0,0 +1,218 @@ +/* + * © 2021. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.csv + +import static edu.ie3.datamodel.models.StandardUnits.* + +import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.naming.FileNamingStrategy +import edu.ie3.datamodel.io.source.IdCoordinateSource +import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries +import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue +import edu.ie3.datamodel.models.value.SolarIrradianceValue +import edu.ie3.datamodel.models.value.TemperatureValue +import edu.ie3.datamodel.models.value.WeatherValue +import edu.ie3.datamodel.models.value.WindValue +import edu.ie3.test.common.CosmoWeatherTestData +import edu.ie3.test.helper.WeatherSourceTestHelper +import edu.ie3.util.TimeUtil +import edu.ie3.util.geo.GeoUtils +import edu.ie3.util.interval.ClosedInterval +import org.locationtech.jts.geom.Coordinate +import org.locationtech.jts.geom.Point +import spock.lang.Shared +import spock.lang.Specification +import tech.units.indriya.quantity.Quantities + +class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta, WeatherSourceTestHelper { + + @Shared + CsvWeatherSource source + + @Shared + IdCoordinateSource coordinateSource + + def setupSpec() { + coordinateSource = CosmoWeatherTestData.coordinateSource + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() + source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) + } + + def "A CsvWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { + given: + def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) + when: + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) + then: + optTimeBasedValue.present + equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) + } + + def "A CsvWeatherSource can read multiple time series values for multiple coordinates"() { + given: + def coordinates = [ + CosmoWeatherTestData.COORDINATE_193186, + CosmoWeatherTestData.COORDINATE_193187 + ] + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.TIME_17H) + def timeSeries193186 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] + as Set) + def timeSeries193187 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + when: + Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) + then: + coordinateToTimeSeries.keySet().size() == 2 + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186), timeSeries193186) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeSeries193187) + } + + + def "A CsvWeatherSource can read all weather data in a given time interval"() { + given: + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) + def timeSeries193186 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + def timeSeries193187 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193187_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + def timeSeries193188 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) + when: + Map> coordinateToTimeSeries = source.getWeather(timeInterval) + then: + coordinateToTimeSeries.keySet().size() == 3 + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) + } + + def "The CsvWeatherSource is able to build a single WeatherValue from field to value mapping"() { + given: + def defaultCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(7.4116482, 51.4843281)) + def coordinateSource = Mock(IdCoordinateSource) + coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) + def fieldToValues = [ + "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", + "time" : "2020-10-16T12:40:42Z", + "coordinateid" : "5", + "directirradiance" : "1.234", + "diffuseirradiance": "5.678", + "temperature" : "9.1011", + "windvelocity" : "12.1314", + "winddirection" : "15.1617" + ] + def expectedValue = new TimeBasedValue( + UUID.fromString("71a79f59-eebf-40c1-8358-ba7414077d57"), + TimeUtil.withDefaults.toZonedDateTime("2020-10-16 12:40:42"), + new WeatherValue( + defaultCoordinate, + new SolarIrradianceValue( + Quantities.getQuantity(1.234, SOLAR_IRRADIANCE), + Quantities.getQuantity(5.678, SOLAR_IRRADIANCE) + ), + new TemperatureValue( + Quantities.getQuantity(9.1011, TEMPERATURE) + ), + new WindValue( + Quantities.getQuantity(12.1314, WIND_DIRECTION), + Quantities.getQuantity(15.1617, WIND_VELOCITY) + ) + ) + ) + + when: + def actual = source.buildWeatherValue(fieldToValues) + + then: + actual.present + actual.get() == expectedValue + } + + def "The CsvWeatherSource returns no WeatherValue, if the coordinate field is empty"() { + given: + def defaultCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(7.4116482, 51.4843281)) + def coordinateSource = Mock(IdCoordinateSource) + coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) + def fieldToValues = [ + "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", + "time" : "2020-10-16T12:40:42Z", + "coordinateid" : "", + "directirradiance" : "1.234", + "diffuseirradiance": "5.678", + "temperature" : "9.1011", + "windvelocity" : "12.1314", + "winddirection" : "15.1617" + ] + + when: + def actual = source.buildWeatherValue(fieldToValues) + + then: + !actual.present + } + + def "The CsvWeatherSource returns no WeatherValue, if the coordinate field is missing"() { + given: + def defaultCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(7.4116482, 51.4843281)) + def coordinateSource = Mock(IdCoordinateSource) + coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) + def fieldToValues = [ + "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", + "time" : "2020-10-16T12:40:42Z", + "directirradiance" : "1.234", + "diffuseirradiance": "5.678", + "temperature" : "9.1011", + "windvelocity" : "12.1314", + "winddirection" : "15.1617" + ] + + when: + def actual = source.buildWeatherValue(fieldToValues) + + then: + !actual.present + } + + def "The CsvWeatherSource returns no WeatherValue, if the coordinate cannot be obtained"() { + given: + def coordinateSource = Mock(IdCoordinateSource) + coordinateSource.getCoordinate(_) >> Optional.empty() + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) + def fieldToValues = [ + "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", + "time" : "2020-10-16T12:40:42Z", + "coordinateid" : "6", + "directirradiance" : "1.234", + "diffuseirradiance": "5.678", + "temperature" : "9.1011", + "windvelocity" : "12.1314", + "winddirection" : "15.1617" + ] + + when: + def actual = source.buildWeatherValue(fieldToValues) + + then: + !actual.present + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy index fe7b041d3..b4869c6ad 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy @@ -29,14 +29,10 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, @Shared IdCoordinateSource coordinateSource - @Shared - String folderPath - def setupSpec() { coordinateSource = WeatherTestData.coordinateSource def weatherFactory = new IconTimeBasedWeatherValueFactory() - folderPath = new File(getClass().getResource('/weather/icon').toURI()).absolutePath - source = new CsvWeatherSource(",", folderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) + source = new CsvWeatherSource(",", iconWeatherFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) } def "A CsvWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { @@ -109,7 +105,7 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 67775 ? Optional.of(expectedCoordinate) : Optional.empty() } def weatherFactory = new IconTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(",", folderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(",", iconWeatherFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = new TreeMap<>(String.CASE_INSENSITIVE_ORDER) fieldToValues.putAll( [ @@ -156,7 +152,7 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, given: def coordinateSource = new WeatherTestData.DummyIdCoordinateSource() def weatherFactory = new IconTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(",", folderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(",", iconWeatherFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "datum" : "2019-08-01 01:00:00", "albRad" : "13.015240669", @@ -200,7 +196,7 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, given: def coordinateSource = new WeatherTestData.DummyIdCoordinateSource() def weatherFactory = new IconTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(",", folderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(",", iconWeatherFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "datum" : "2019-08-01 01:00:00", "albRad" : "13.015240669", @@ -243,7 +239,7 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, given: def coordinateSource = new WeatherTestData.DummyIdCoordinateSource() def weatherFactory = new IconTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(",", folderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(",", iconWeatherFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "datum" : "2019-08-01 01:00:00", "albrad" : "13.015240669", diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourcePsdmTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourcePsdmTest.groovy deleted file mode 100644 index 845f44338..000000000 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourcePsdmTest.groovy +++ /dev/null @@ -1,227 +0,0 @@ -/* - * © 2021. TU Dortmund University, - * Institute of Energy Systems, Energy Efficiency and Energy Economics, - * Research group Distribution grid planning and operation - */ -package edu.ie3.datamodel.io.source.csv - -import edu.ie3.datamodel.io.naming.FileNamingStrategy - -import static edu.ie3.datamodel.models.StandardUnits.SOLAR_IRRADIANCE -import static edu.ie3.datamodel.models.StandardUnits.TEMPERATURE -import static edu.ie3.datamodel.models.StandardUnits.WIND_DIRECTION -import static edu.ie3.datamodel.models.StandardUnits.WIND_VELOCITY - -import edu.ie3.datamodel.io.factory.timeseries.PsdmTimeBasedWeatherValueFactory -import edu.ie3.datamodel.models.value.SolarIrradianceValue -import edu.ie3.datamodel.io.source.IdCoordinateSource -import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries -import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue -import edu.ie3.datamodel.models.value.TemperatureValue -import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.datamodel.models.value.WindValue -import edu.ie3.test.common.PsdmWeatherTestData -import edu.ie3.test.helper.WeatherSourceTestHelper -import edu.ie3.util.TimeUtil -import edu.ie3.util.geo.GeoUtils -import edu.ie3.util.interval.ClosedInterval -import org.locationtech.jts.geom.Coordinate -import org.locationtech.jts.geom.Point -import spock.lang.Shared -import spock.lang.Specification -import tech.units.indriya.quantity.Quantities - -class CsvWeatherSourcePsdmTest extends Specification implements CsvTestDataMeta, WeatherSourceTestHelper { - - @Shared - CsvWeatherSource source - - @Shared - IdCoordinateSource coordinateSource - - def setupSpec() { - coordinateSource = PsdmWeatherTestData.coordinateSource - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() - source = new CsvWeatherSource(";", timeSeriesFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) - } - - def "A CsvWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { - given: - def expectedTimeBasedValue = new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H) - when: - def optTimeBasedValue = source.getWeather(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.COORDINATE_193186) - then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) - } - - def "A CsvWeatherSource can read multiple time series values for multiple coordinates"() { - given: - def coordinates = [ - PsdmWeatherTestData.COORDINATE_193186, - PsdmWeatherTestData.COORDINATE_193187 - ] - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.TIME_17H) - def timeSeries193186 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H) - ] - as Set) - def timeSeries193187 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H) - ] as Set) - when: - Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) - then: - coordinateToTimeSeries.keySet().size() == 2 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186), timeSeries193186) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187), timeSeries193187) - } - - - def "A CsvWeatherSource can read all weather data in a given time interval"() { - given: - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.TIME_17H) - def timeSeries193186 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H) - ] as Set) - def timeSeries193187 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193187_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H) - ] as Set) - def timeSeries193188 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193188_15H) - ] as Set) - when: - Map> coordinateToTimeSeries = source.getWeather(timeInterval) - then: - coordinateToTimeSeries.keySet().size() == 3 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) - } - - def "The CsvWeatherSource is able to build a single WeatherValue from field to value mapping"() { - given: - def defaultCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(7.4116482, 51.4843281)) - def coordinateSource = Mock(IdCoordinateSource) - coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", timeSeriesFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) - def fieldToValues = [ - "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", - "time" : "2020-10-16T12:40:42Z", - "coordinate" : "5", - "directirradiation" : "1.234", - "diffuseirradiation": "5.678", - "temperature" : "9.1011", - "windvelocity" : "12.1314", - "winddirection" : "15.1617" - ] - def expectedValue = new TimeBasedValue( - UUID.fromString("71a79f59-eebf-40c1-8358-ba7414077d57"), - TimeUtil.withDefaults.toZonedDateTime("2020-10-16 12:40:42"), - new WeatherValue( - defaultCoordinate, - new SolarIrradianceValue( - Quantities.getQuantity(1.234, SOLAR_IRRADIANCE), - Quantities.getQuantity(5.678, SOLAR_IRRADIANCE) - ), - new TemperatureValue( - Quantities.getQuantity(9.1011, TEMPERATURE) - ), - new WindValue( - Quantities.getQuantity(12.1314, WIND_DIRECTION), - Quantities.getQuantity(15.1617, WIND_VELOCITY) - ) - ) - ) - - when: - def actual = source.buildWeatherValue(fieldToValues) - - then: - actual.present - actual.get() == expectedValue - } - - def "The CsvWeatherSource returns no WeatherValue, if the coordinate field is empty"() { - given: - def defaultCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(7.4116482, 51.4843281)) - def coordinateSource = Mock(IdCoordinateSource) - coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", timeSeriesFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) - def fieldToValues = [ - "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", - "time" : "2020-10-16T12:40:42Z", - "coordinate" : "", - "directirradiation" : "1.234", - "diffuseirradiation": "5.678", - "temperature" : "9.1011", - "windvelocity" : "12.1314", - "winddirection" : "15.1617" - ] - - when: - def actual = source.buildWeatherValue(fieldToValues) - - then: - !actual.present - } - - def "The CsvWeatherSource returns no WeatherValue, if the coordinate field is missing"() { - given: - def defaultCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(7.4116482, 51.4843281)) - def coordinateSource = Mock(IdCoordinateSource) - coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", timeSeriesFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) - def fieldToValues = [ - "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", - "time" : "2020-10-16T12:40:42Z", - "directirradiation" : "1.234", - "diffuseirradiation": "5.678", - "temperature" : "9.1011", - "windvelocity" : "12.1314", - "winddirection" : "15.1617" - ] - - when: - def actual = source.buildWeatherValue(fieldToValues) - - then: - !actual.present - } - - def "The CsvWeatherSource returns no WeatherValue, if the coordinate cannot be obtained"() { - given: - def coordinateSource = Mock(IdCoordinateSource) - coordinateSource.getCoordinate(_) >> Optional.empty() - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", timeSeriesFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) - def fieldToValues = [ - "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", - "time" : "2020-10-16T12:40:42Z", - "coordinate" : "6", - "directirradiation" : "1.234", - "diffuseirradiation": "5.678", - "temperature" : "9.1011", - "windvelocity" : "12.1314", - "winddirection" : "15.1617" - ] - - when: - def actual = source.buildWeatherValue(fieldToValues) - - then: - !actual.present - } -} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy similarity index 56% rename from src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy rename to src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy index ddf83fee4..2c5a2b5b4 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy @@ -6,26 +6,27 @@ package edu.ie3.datamodel.io.source.influxdb import edu.ie3.datamodel.io.connectors.InfluxDbConnector -import edu.ie3.datamodel.io.factory.timeseries.PsdmTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.geo.GeoUtils import edu.ie3.util.interval.ClosedInterval import org.locationtech.jts.geom.Point import org.testcontainers.containers.InfluxDBContainer import org.testcontainers.spock.Testcontainers +import org.testcontainers.utility.DockerImageName import org.testcontainers.utility.MountableFile import spock.lang.Shared import spock.lang.Specification @Testcontainers -class InfluxDbWeatherSourcePsdmIT extends Specification implements WeatherSourceTestHelper { +class InfluxDbWeatherSourceCosmoIT extends Specification implements WeatherSourceTestHelper { @Shared - InfluxDBContainer influxDbContainer = new InfluxDBContainer("1.8.4") + InfluxDBContainer influxDbContainer = new InfluxDBContainer(DockerImageName.parse("influxdb:1.8.10")) .withAuthEnabled(false) .withDatabase("test_weather") @@ -35,7 +36,7 @@ class InfluxDbWeatherSourcePsdmIT extends Specification implements WeatherSource def setupSpec() { // Copy import file into docker and then import it via influx CLI // more information on file format and usage here: https://docs.influxdata.com/influxdb/v1.7/tools/shell/#import-data-from-a-file-with-import - MountableFile influxWeatherImportFile = MountableFile.forClasspathResource("/testcontainersFiles/influxDb/weather.txt") + MountableFile influxWeatherImportFile = MountableFile.forClasspathResource("/testcontainersFiles/influxDb/cosmo/weather.txt") influxDbContainer.copyFileToContainer(influxWeatherImportFile, "/home/weather.txt") def execResult = influxDbContainer.execInContainer("influx", "-import", "-path=/home/weather.txt", "-precision=ms") println "Command \"influx -import -path=/home/weather.txt -precision=ms\" returned:" @@ -43,8 +44,8 @@ class InfluxDbWeatherSourcePsdmIT extends Specification implements WeatherSource if(!execResult.stdout.isEmpty()) println execResult.getStdout() def connector = new InfluxDbConnector(influxDbContainer.url,"test_weather", "test_scenario") - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() - source = new InfluxDbWeatherSource(connector, PsdmWeatherTestData.coordinateSource, weatherFactory) + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() + source = new InfluxDbWeatherSource(connector, CosmoWeatherTestData.coordinateSource, weatherFactory) } @@ -57,9 +58,9 @@ class InfluxDbWeatherSourcePsdmIT extends Specification implements WeatherSource def "An InfluxDbWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { given: - def expectedTimeBasedValue = new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H) + def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) when: - def optTimeBasedValue = source.getWeather(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.COORDINATE_193186) + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: optTimeBasedValue.present equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) @@ -68,66 +69,66 @@ class InfluxDbWeatherSourcePsdmIT extends Specification implements WeatherSource def "An InfluxDbWeatherSource can read multiple time series values for multiple coordinates"() { given: def coordinates = [ - PsdmWeatherTestData.COORDINATE_193186, - PsdmWeatherTestData.COORDINATE_193187 + CosmoWeatherTestData.COORDINATE_193186, + CosmoWeatherTestData.COORDINATE_193187 ] - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.TIME_17H) def timeseries_193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H) + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H) ] as Set) def timeseries_193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H) + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H) ] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) then: coordinateToTimeSeries.keySet().size() == 2 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186), timeseries_193186) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187), timeseries_193187) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186), timeseries_193186) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeseries_193187) } def "An InfluxDbWeatherSource can read all weather data in a given time interval"() { given: - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) def timeseries_193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H) ] as Set) def timeseries_193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193187_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193187_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H) ] as Set) def timeseries_193188 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193188_15H) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193188_15H) ] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval) then: coordinateToTimeSeries.keySet().size() == 3 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186).getEntries(), timeseries_193186.getEntries()) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187).getEntries(), timeseries_193187.getEntries()) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193188).getEntries(), timeseries_193188.getEntries()) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186).entries, timeseries_193186.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187).entries, timeseries_193187.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).entries, timeseries_193188.entries) } def "An InfluxDbWeatherSource will return an equivalent to 'empty' when being unable to map a coordinate to it's ID"() { - def validCoordinate = PsdmWeatherTestData.COORDINATE_193186 + def validCoordinate = CosmoWeatherTestData.COORDINATE_193186 def invalidCoordinate = GeoUtils.xyToPoint(48d, 7d) - def time = PsdmWeatherTestData.TIME_15H - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.TIME_17H) + def time = CosmoWeatherTestData.TIME_15H + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) def emptyTimeSeries = new IndividualTimeSeries(UUID.randomUUID(), Collections.emptySet()) def timeseries_193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H) ] as Set) when: def coordinateAtDate = source.getWeather(time, invalidCoordinate) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy new file mode 100644 index 000000000..21da4b6f5 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -0,0 +1,106 @@ +/* + * © 2021. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.sql + +import edu.ie3.datamodel.io.connectors.SqlConnector +import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory +import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries +import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue +import edu.ie3.datamodel.models.value.WeatherValue +import edu.ie3.test.common.CosmoWeatherTestData +import edu.ie3.test.helper.WeatherSourceTestHelper +import edu.ie3.util.TimeUtil +import edu.ie3.util.interval.ClosedInterval +import org.locationtech.jts.geom.Point +import org.testcontainers.containers.PostgreSQLContainer +import org.testcontainers.spock.Testcontainers +import org.testcontainers.utility.MountableFile +import spock.lang.Shared +import spock.lang.Specification + +@Testcontainers +class SqlWeatherSourceCosmoIT extends Specification implements WeatherSourceTestHelper { + + @Shared + PostgreSQLContainer postgreSQLContainer = new PostgreSQLContainer("postgres:11.9") + + @Shared + SqlWeatherSource source + + static String schemaName = "public" + static String weatherTableName = "weather" + + def setupSpec() { + // Copy sql import script into docker + MountableFile sqlImportFile = MountableFile.forClasspathResource("/testcontainersFiles/sql/cosmo/weather.sql") + postgreSQLContainer.copyFileToContainer(sqlImportFile, "/home/weather.sql") + // Execute import script + postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/weather.sql") + + def connector = new SqlConnector(postgreSQLContainer.jdbcUrl, postgreSQLContainer.username, postgreSQLContainer.password) + def weatherFactory = new CosmoTimeBasedWeatherValueFactory(TimeUtil.withDefaults) + source = new SqlWeatherSource(connector, CosmoWeatherTestData.coordinateSource, schemaName, weatherTableName, weatherFactory) + } + + def "A NativeSqlWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { + given: + def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) + when: + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) + then: + optTimeBasedValue.present + equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue ) + } + + def "A NativeSqlWeatherSource can read multiple timeseries values for multiple coordinates"() { + given: + def coordinates = [ + CosmoWeatherTestData.COORDINATE_193186, + CosmoWeatherTestData.COORDINATE_193187 + ] + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.TIME_17H) + def timeSeries193186 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] + as Set) + def timeSeries193187 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + when: + Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) + then: + coordinateToTimeSeries.keySet().size() == 2 + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186), timeSeries193186) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeSeries193187) + } + + + + def "A NativeSqlWeatherSource can read all weather data in a given time interval"() { + given: + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) + def timeSeries193186 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + def timeSeries193187 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193187_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + def timeSeries193188 = new IndividualTimeSeries(null, + [ + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) + when: + Map> coordinateToTimeSeries = source.getWeather(timeInterval) + then: + coordinateToTimeSeries.keySet().size() == 3 + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourcePsdmIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourcePsdmIT.groovy deleted file mode 100644 index 99123ff34..000000000 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourcePsdmIT.groovy +++ /dev/null @@ -1,141 +0,0 @@ -/* - * © 2021. TU Dortmund University, - * Institute of Energy Systems, Energy Efficiency and Energy Economics, - * Research group Distribution grid planning and operation - */ -package edu.ie3.datamodel.io.source.sql - -import edu.ie3.datamodel.io.connectors.SqlConnector -import edu.ie3.datamodel.io.factory.timeseries.PsdmTimeBasedWeatherValueFactory -import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries -import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue -import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.test.common.PsdmWeatherTestData -import edu.ie3.test.helper.WeatherSourceTestHelper -import edu.ie3.util.TimeUtil -import edu.ie3.util.geo.GeoUtils -import edu.ie3.util.interval.ClosedInterval -import org.locationtech.jts.geom.Point -import org.testcontainers.containers.Container -import org.testcontainers.containers.PostgreSQLContainer -import org.testcontainers.spock.Testcontainers -import org.testcontainers.utility.MountableFile -import spock.lang.Shared -import spock.lang.Specification - -@Testcontainers -class SqlWeatherSourcePsdmIT extends Specification implements WeatherSourceTestHelper { - - @Shared - PostgreSQLContainer postgreSQLContainer = new PostgreSQLContainer("postgres:11.14") - - @Shared - SqlWeatherSource source - - static String schemaName = "public" - static String weatherTableName = "weather" - - def setupSpec() { - // Copy sql import script into docker - MountableFile sqlImportFile = MountableFile.forClasspathResource("/testcontainersFiles/sql/psdm/weather.sql") - postgreSQLContainer.copyFileToContainer(sqlImportFile, "/home/weather.sql") - // Execute import script - Container.ExecResult res = postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/weather.sql") - assert res.stderr.empty - - def connector = new SqlConnector(postgreSQLContainer.jdbcUrl, postgreSQLContainer.username, postgreSQLContainer.password) - def weatherFactory = new PsdmTimeBasedWeatherValueFactory(TimeUtil.withDefaults) - source = new SqlWeatherSource(connector, PsdmWeatherTestData.coordinateSource, schemaName, weatherTableName, weatherFactory) - } - - def "A SqlWeatherSourcePsdm can read and correctly parse a single value for a specific date and coordinate"() { - given: - def expectedTimeBasedValue = new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H) - - when: - def optTimeBasedValue = source.getWeather(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.COORDINATE_193186) - - then: - optTimeBasedValue.present - equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue ) - } - - def "A SqlWeatherSourcePsdm returns nothing for an invalid coordinate"() { - when: - def optTimeBasedValue = source.getWeather(PsdmWeatherTestData.TIME_15H, GeoUtils.xyToPoint(88d, 89d)) - - then: - !optTimeBasedValue.present - } - - def "A SqlWeatherSourcePsdm can read multiple timeseries values for multiple coordinates"() { - given: - def coordinates = [ - PsdmWeatherTestData.COORDINATE_193186, - PsdmWeatherTestData.COORDINATE_193187 - ] - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.TIME_17H) - def timeSeries193186 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H) - ] - as Set) - def timeSeries193187 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H) - ] as Set) - - when: - Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) - - then: - coordinateToTimeSeries.keySet().size() == 2 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186), timeSeries193186) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187), timeSeries193187) - } - - def "A SqlWeatherSourcePsdm returns nothing for invalid coordinates"() { - given: - def coordinates = [ - GeoUtils.xyToPoint(88d, 89d), - GeoUtils.xyToPoint(89d, 89d) - ] - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.TIME_17H) - - when: - Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) - - then: - coordinateToTimeSeries.keySet().empty - } - - def "A SqlWeatherSourcePsdm can read all weather data in a given time interval"() { - given: - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.TIME_17H) - def timeSeries193186 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H) - ] as Set) - def timeSeries193187 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193187_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H) - ] as Set) - def timeSeries193188 = new IndividualTimeSeries(null, - [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193188_15H) - ] as Set) - - when: - Map> coordinateToTimeSeries = source.getWeather(timeInterval) - - then: - coordinateToTimeSeries.keySet().size() == 3 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) - } -} diff --git a/src/test/groovy/edu/ie3/test/common/PsdmWeatherTestData.groovy b/src/test/groovy/edu/ie3/test/common/CosmoWeatherTestData.groovy similarity index 98% rename from src/test/groovy/edu/ie3/test/common/PsdmWeatherTestData.groovy rename to src/test/groovy/edu/ie3/test/common/CosmoWeatherTestData.groovy index 4a6c95a8e..33612e176 100644 --- a/src/test/groovy/edu/ie3/test/common/PsdmWeatherTestData.groovy +++ b/src/test/groovy/edu/ie3/test/common/CosmoWeatherTestData.groovy @@ -12,7 +12,7 @@ import tech.units.indriya.quantity.Quantities import java.time.ZoneId import java.time.ZonedDateTime -class PsdmWeatherTestData extends WeatherTestData { +class CosmoWeatherTestData extends WeatherTestData { public static final ZonedDateTime TIME_15H = ZonedDateTime.of(2020, 04, 28, 15, 0, 0, 0, ZoneId.of("UTC")) public static final ZonedDateTime TIME_16H = ZonedDateTime.of(2020, 04, 28, 16, 0, 0, 0, ZoneId.of("UTC")) diff --git a/src/test/resources/testTimeSeriesFiles/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv b/src/test/resources/edu/ie3/datamodel/io/source/csv/weather/cosmo/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv similarity index 85% rename from src/test/resources/testTimeSeriesFiles/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv rename to src/test/resources/edu/ie3/datamodel/io/source/csv/weather/cosmo/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv index f15c6237a..d73cac992 100644 --- a/src/test/resources/testTimeSeriesFiles/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv +++ b/src/test/resources/edu/ie3/datamodel/io/source/csv/weather/cosmo/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv @@ -1,4 +1,4 @@ -"uuid";"coordinate";"time";"diffuse_irradiation";"direct_irradiation";"temperature";"wind_direction";"wind_velocity" +"uuid";"coordinate_id";"time";"diffuse_irradiance";"direct_irradiance";"temperature";"wind_direction";"wind_velocity" "3cee46d5-1fe7-419e-a652-32f9be6703be";193186;2020-04-28T15:00:00Z;286.872985839844;282.671997070312;278.019012451172;0;1.66103506088257 "1fbc6f7e-52fe-4668-93c8-33454d7feb6c";193187;2020-04-28T15:00:00Z;287.872985839844;283.671997070312;279.019012451172;0;1.76103506088257 "78c7a057-ad6c-4ea1-bb74-7e28fbb08957";193188;2020-04-28T15:00:00Z;288.872985839844;284.671997070312;280.019012451172;0;1.86103506088257 diff --git a/src/test/resources/weather/icon/its_weather_513606bc-539e-445b-9675-2f98be3d9231.csv b/src/test/resources/edu/ie3/datamodel/io/source/csv/weather/icon/its_weather_513606bc-539e-445b-9675-2f98be3d9231.csv similarity index 100% rename from src/test/resources/weather/icon/its_weather_513606bc-539e-445b-9675-2f98be3d9231.csv rename to src/test/resources/edu/ie3/datamodel/io/source/csv/weather/icon/its_weather_513606bc-539e-445b-9675-2f98be3d9231.csv diff --git a/src/test/resources/testcontainersFiles/couchbase/weather.json b/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json similarity index 56% rename from src/test/resources/testcontainersFiles/couchbase/weather.json rename to src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json index e075cecab..163837bf5 100644 --- a/src/test/resources/testcontainersFiles/couchbase/weather.json +++ b/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json @@ -1,47 +1,47 @@ -[{"coordinate":193186, +[{"coordinateid":193186, "time":"2020-04-28T15:00:00+00:00", - "diffuseirradiation":286.872985839844, - "directirradiation":282.671997070312, + "diffuseirradiance":286.872985839844, + "directirradiance":282.671997070312, "temperature":278.019012451172, "winddirection":0, "windvelocity":1.66103506088257}, - {"coordinate":193187, + {"coordinateid":193187, "time":"2020-04-28T15:00:00+00:00", - "diffuseirradiation":287.872985839844, - "directirradiation":283.671997070312, + "diffuseirradiance":287.872985839844, + "directirradiance":283.671997070312, "temperature":279.019012451172, "winddirection":0, "windvelocity":1.76103506088257}, - {"coordinate":193188, + {"coordinateid":193188, "time":"2020-04-28T15:00:00+00:00", - "diffuseirradiation":288.872985839844, - "directirradiation":284.671997070312, + "diffuseirradiance":288.872985839844, + "directirradiance":284.671997070312, "temperature":280.019012451172, "winddirection":0, "windvelocity":1.86103506088257}, - {"coordinate":193186, + {"coordinateid":193186, "time":"2020-04-28T16:00:00+00:00", - "diffuseirradiation":286.872, - "directirradiation":282.672, + "diffuseirradiance":286.872, + "directirradiance":282.672, "temperature":278.012, "winddirection":0, "windvelocity":1.662}, - {"coordinate":193187, + {"coordinateid":193187, "time":"2020-04-28T16:00:00+00:00", - "diffuseirradiation":287.872, - "directirradiation":283.672, + "diffuseirradiance":287.872, + "directirradiance":283.672, "temperature":279.012, "winddirection":0, "windvelocity":1.762}, - {"coordinate":193186, + {"coordinateid":193186, "time":"2020-04-28T17:00:00+00:00", - "diffuseirradiation":286.873, - "directirradiation":282.673, + "diffuseirradiance":286.873, + "directirradiance":282.673, "temperature":278.013, "winddirection":0, "windvelocity":1.663}] diff --git a/src/test/resources/testcontainersFiles/influxDb/cosmo/weather.txt b/src/test/resources/testcontainersFiles/influxDb/cosmo/weather.txt new file mode 100644 index 000000000..982e7ea5b --- /dev/null +++ b/src/test/resources/testcontainersFiles/influxDb/cosmo/weather.txt @@ -0,0 +1,13 @@ +# DDL +CREATE DATABASE test_weather + +# DML +# CONTEXT-DATABASE: test_weather + + +weather,coordinate_id=193186 diffuse_irradiance=286.872985839844,direct_irradiance=282.671997070312,temperature=278.019012451172,wind_direction=0,wind_velocity=1.66103506088257 1588086000000 +weather,coordinate_id=193187 diffuse_irradiance=287.872985839844,direct_irradiance=283.671997070312,temperature=279.019012451172,wind_direction=0,wind_velocity=1.76103506088257 1588086000000 +weather,coordinate_id=193188 diffuse_irradiance=288.872985839844,direct_irradiance=284.671997070312,temperature=280.019012451172,wind_direction=0,wind_velocity=1.86103506088257 1588086000000 +weather,coordinate_id=193186 diffuse_irradiance=286.872,direct_irradiance=282.672,temperature=278.012,wind_direction=0,wind_velocity=1.662 1588089600000 +weather,coordinate_id=193187 diffuse_irradiance=287.872,direct_irradiance=283.672,temperature=279.012,wind_direction=0,wind_velocity=1.762 1588089600000 +weather,coordinate_id=193186 diffuse_irradiance=286.873,direct_irradiance=282.673,temperature=278.013,wind_direction=0,wind_velocity=1.663 1588093200000 diff --git a/src/test/resources/testcontainersFiles/influxDb/weather.txt b/src/test/resources/testcontainersFiles/influxDb/weather.txt deleted file mode 100644 index 8ee925e99..000000000 --- a/src/test/resources/testcontainersFiles/influxDb/weather.txt +++ /dev/null @@ -1,13 +0,0 @@ -# DDL -CREATE DATABASE test_weather - -# DML -# CONTEXT-DATABASE: test_weather - - -weather,coordinate=193186 diffuse_irradiation=286.872985839844,direct_irradiation=282.671997070312,temperature=278.019012451172,wind_direction=0,wind_velocity=1.66103506088257 1588086000000 -weather,coordinate=193187 diffuse_irradiation=287.872985839844,direct_irradiation=283.671997070312,temperature=279.019012451172,wind_direction=0,wind_velocity=1.76103506088257 1588086000000 -weather,coordinate=193188 diffuse_irradiation=288.872985839844,direct_irradiation=284.671997070312,temperature=280.019012451172,wind_direction=0,wind_velocity=1.86103506088257 1588086000000 -weather,coordinate=193186 diffuse_irradiation=286.872,direct_irradiation=282.672,temperature=278.012,wind_direction=0,wind_velocity=1.662 1588089600000 -weather,coordinate=193187 diffuse_irradiation=287.872,direct_irradiation=283.672,temperature=279.012,wind_direction=0,wind_velocity=1.762 1588089600000 -weather,coordinate=193186 diffuse_irradiation=286.873,direct_irradiation=282.673,temperature=278.013,wind_direction=0,wind_velocity=1.663 1588093200000 diff --git a/src/test/resources/testcontainersFiles/sql/psdm/weather.sql b/src/test/resources/testcontainersFiles/sql/cosmo/weather.sql similarity index 66% rename from src/test/resources/testcontainersFiles/sql/psdm/weather.sql rename to src/test/resources/testcontainersFiles/sql/cosmo/weather.sql index c5361bbf7..055698ba1 100644 --- a/src/test/resources/testcontainersFiles/sql/psdm/weather.sql +++ b/src/test/resources/testcontainersFiles/sql/cosmo/weather.sql @@ -1,33 +1,33 @@ CREATE TABLE public.weather ( time timestamp with time zone, - coordinate integer, - diffuse_irradiation double precision, - direct_irradiation double precision, + coordinate_id integer, + diffuse_irradiance double precision, + direct_irradiance double precision, wind_direction double precision, wind_velocity double precision, temperature double precision, tid serial, CONSTRAINT weather_pkey PRIMARY KEY (tid), - CONSTRAINT "weather_datum_coordinate_unique" UNIQUE (time, coordinate) + CONSTRAINT "weather_datum_coordinate_id_unique" UNIQUE (time, coordinate_id) ) WITH ( OIDS = FALSE ) TABLESPACE pg_default; -CREATE INDEX weather_coordinate_idx +CREATE INDEX weather_coordinate_id_idx ON public.weather USING btree - (coordinate ASC NULLS LAST) + (coordinate_id ASC NULLS LAST) TABLESPACE pg_default; -CREATE INDEX weather_coordinate_time_idx +CREATE INDEX weather_coordinate_id_time_idx ON public.weather USING btree - (coordinate ASC NULLS LAST, time ASC NULLS LAST) + (coordinate_id ASC NULLS LAST, time ASC NULLS LAST) TABLESPACE pg_default; INSERT INTO - public.weather (time, coordinate, diffuse_irradiation, direct_irradiation, wind_direction, wind_velocity, temperature) + public.weather (time, coordinate_id, diffuse_irradiance, direct_irradiance, wind_direction, wind_velocity, temperature) VALUES ('2020-04-28 15:00:00+0', 193186, 286.872985839844, 282.671997070312, 0, 1.66103506088257, 278.019012451172), ('2020-04-28 15:00:00+0', 193187, 287.872985839844, 283.671997070312, 0, 1.76103506088257, 279.019012451172),