Merge pull request #558 from ie3-institute/sp/#267-cosmo-weather-sche…

…me-adaptions

Adapt to changed scheme of COSMO weather data

.gitattributes

@@ -1,3 +1,3 @@
 # The following file will be copied to a unix Docker image and imported to InfluxDB data base. Therefore, the line
 # ending plays a crucial role. This prevents the endings from being adjusted with 'core.autocrlf=true'
-src/test/resources/testContainerFiles/influxDb/weather.txt eol=lf
+src/test/resources/testContainerFiles/influxDb/cosmo/weather.txt eol=lf

CHANGELOG.md

@@ -35,6 +35,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - `edu.ie3.datamodel.io.csv.timeseries.LoadProfileTimeSeriesMetaInformation`
   - `edu.ie3.datamodel.io.connectors.CsvFileConnector.CsvIndividualTimeSeriesMetaInformation`
   - and related methods
+- BREAKING: Comprehensive harmonization around weather sources [#267](https://github.com/ie3-institute/PowerSystemDataModel/issues/267)
+    - Adapted the expected column scheme
+      - General weather model
+        - `coordinate` to `coordinateid`
+      - DWD COSMO model
+        - `diffuseirradiation` to `diffuseirradiance`
+        - `directirradiation` to `directirradiance`
 
 ## [2.1.0] - 2022-01-05
 

build.gradle

@@ -40,7 +40,7 @@ apply from: scriptsLocation + 'vcs.gradle'
 apply from: scriptsLocation + 'semVer.gradle'
 
 repositories {
-	mavenCentral() //searches in bintray's repository 'jCenter', which contains Maven Central
+	mavenCentral() // searches in Sonatype's repository 'Maven Central'
 	maven { url 'https://www.jitpack.io' } // allows github repos as dependencies
 
 	// sonatype snapshot repo

src/main/java/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactory.java → src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java

@@ -25,31 +25,25 @@
  * Factory implementation of {@link TimeBasedWeatherValueFactory}, that is able to handle field to
  * value mapping in the typical PowerSystemDataModel (PSDM) column scheme
  */
-public class PsdmTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory {
-  private static final String COORDINATE = "coordinate";
-  private static final String DIFFUSE_IRRADIANCE = "diffuseirradiation";
-  private static final String DIRECT_IRRADIANCE = "directirradiation";
+public class CosmoTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory {
+  private static final String DIFFUSE_IRRADIANCE = "diffuseirradiance";
+  private static final String DIRECT_IRRADIANCE = "directirradiance";
   private static final String TEMPERATURE = "temperature";
   private static final String WIND_DIRECTION = "winddirection";
   private static final String WIND_VELOCITY = "windvelocity";
 
-  public PsdmTimeBasedWeatherValueFactory(TimeUtil timeUtil) {
+  public CosmoTimeBasedWeatherValueFactory(TimeUtil timeUtil) {
     super(timeUtil);
   }
 
-  public PsdmTimeBasedWeatherValueFactory(String timePattern) {
+  public CosmoTimeBasedWeatherValueFactory(String timePattern) {
     super(timePattern);
   }
 
-  public PsdmTimeBasedWeatherValueFactory() {
+  public CosmoTimeBasedWeatherValueFactory() {
     super();
   }
 
-  @Override
-  public String getCoordinateIdFieldString() {
-    return COORDINATE;
-  }
-
   @Override
   public String getTimeFieldString() {
     return TIME;

src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java

@@ -29,7 +29,6 @@
  */
 public class IconTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory {
   /* Redefine the column names to meet the icon specifications */
-  private static final String COORDINATE = "coordinateid";
   private static final String TIME = "datum";
   private static final String DIFFUSE_IRRADIANCE = "aswdifdS";
   private static final String DIRECT_IRRADIANCE = "aswdirS";
@@ -49,11 +48,6 @@ public IconTimeBasedWeatherValueFactory() {
     super(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd HH:mm:ss"));
   }
 
-  @Override
-  public String getCoordinateIdFieldString() {
-    return COORDINATE;
-  }
-
   @Override
   public String getTimeFieldString() {
     return TIME;

src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java

@@ -18,6 +18,7 @@ public abstract class TimeBasedWeatherValueFactory
     extends TimeBasedValueFactory<TimeBasedWeatherValueData, WeatherValue> {
   protected static final String UUID = "uuid";
   protected static final String TIME = "time";
+  protected static final String COORDINATE_ID = "coordinateid";
 
   protected final TimeUtil timeUtil;
 
@@ -39,7 +40,9 @@ protected TimeBasedWeatherValueFactory(TimeUtil timeUtil) {
    *
    * @return the field name for the coordinate id
    */
-  public abstract String getCoordinateIdFieldString();
+  public String getCoordinateIdFieldString() {
+    return COORDINATE_ID;
+  }
 
   /**
    * Return the field name for the date time

src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java

@@ -29,8 +29,9 @@ public class InfluxDbWeatherSource implements WeatherSource {
   private static final String BASIC_QUERY_STRING = "Select * from weather";
   private static final String WHERE = " where ";
   private static final String MEASUREMENT_NAME_WEATHER = "weather";
+  private static final String COORDINATE_ID_COLUMN_NAME = "coordinate_id";
   private static final int MILLI_TO_NANO_FACTOR = 1000000;
-  private final String coordinateIdColumnName;
+
   private final InfluxDbConnector connector;
   private final IdCoordinateSource coordinateSource;
   private final TimeBasedWeatherValueFactory weatherValueFactory;
@@ -50,7 +51,6 @@ public InfluxDbWeatherSource(
     this.connector = connector;
     this.coordinateSource = coordinateSource;
     this.weatherValueFactory = weatherValueFactory;
-    this.coordinateIdColumnName = weatherValueFactory.getCoordinateIdFieldString();
   }
 
   @Override
@@ -113,7 +113,7 @@ public Map<Point, IndividualTimeSeries<WeatherValue>> getWeather(
   public IndividualTimeSeries<WeatherValue> getWeather(
       ClosedInterval<ZonedDateTime> timeInterval, Point coordinate) {
     Optional<Integer> coordinateId = coordinateSource.getId(coordinate);
-    if (!coordinateId.isPresent()) {
+    if (coordinateId.isEmpty()) {
       return new IndividualTimeSeries<>(UUID.randomUUID(), Collections.emptySet());
     }
     try (InfluxDB session = connector.getSession()) {
@@ -130,7 +130,7 @@ public IndividualTimeSeries<WeatherValue> getWeather(
   @Override
   public Optional<TimeBasedValue<WeatherValue>> getWeather(ZonedDateTime date, Point coordinate) {
     Optional<Integer> coordinateId = coordinateSource.getId(coordinate);
-    if (!coordinateId.isPresent()) {
+    if (coordinateId.isEmpty()) {
       return Optional.empty();
     }
     try (InfluxDB session = connector.getSession()) {
@@ -141,35 +141,37 @@ public Optional<TimeBasedValue<WeatherValue>> getWeather(ZonedDateTime date, Poi
   }
 
   /**
-   * Parses an influxQL QueryResult and then transforms them into a Stream of optional
-   * TimeBasedValue&lt;WeatherValue&gt;, with a present Optional value, if the transformation was
-   * successful and an empty optional otherwise.
+   * Parses an influxQL QueryResult and then transforms it into a Stream of optional
+   * TimeBasedValue&lt;WeatherValue&gt;, with a present Optional value if the transformation was
+   * successful and an empty Optional otherwise.
    */
   private Stream<Optional<TimeBasedValue<WeatherValue>>> optTimeBasedValueStream(
       QueryResult queryResult) {
     Map<String, Set<Map<String, String>>> measurementsMap =
         InfluxDbConnector.parseQueryResult(queryResult, MEASUREMENT_NAME_WEATHER);
+    final String coordinateIdFieldName = weatherValueFactory.getCoordinateIdFieldString();
     return measurementsMap.get(MEASUREMENT_NAME_WEATHER).stream()
         .map(
             fieldToValue -> {
-              Optional<Point> coordinate =
-                  coordinateSource.getCoordinate(
-                      Integer.parseInt(fieldToValue.remove(coordinateIdColumnName)));
-              if (!coordinate.isPresent()) return null;
-              fieldToValue.putIfAbsent("uuid", UUID.randomUUID().toString());
-
-              /* The factory expects camel case id's for fields -> Convert the keys */
-              Map<String, String> camelCaseFields =
+              /* The factory expects flat case id's for fields -> Convert the keys */
+              Map<String, String> flatCaseFields =
                   fieldToValue.entrySet().stream()
                       .collect(
                           Collectors.toMap(
-                              entry -> StringUtils.snakeCaseToCamelCase(entry.getKey()),
+                              entry ->
+                                  StringUtils.snakeCaseToCamelCase(entry.getKey()).toLowerCase(),
                               Map.Entry::getValue));
 
-              return new TimeBasedWeatherValueData(camelCaseFields, coordinate.get());
-            })
-        .filter(Objects::nonNull)
-        .map(weatherValueFactory::get);
+              /* Add a random UUID if necessary */
+              flatCaseFields.putIfAbsent("uuid", UUID.randomUUID().toString());
+
+              /* Get the corresponding coordinate id from map AND REMOVE THE ENTRY !!! */
+              int coordinateId = Integer.parseInt(flatCaseFields.remove(coordinateIdFieldName));
+              return coordinateSource
+                  .getCoordinate(coordinateId)
+                  .map(point -> new TimeBasedWeatherValueData(flatCaseFields, point))
+                  .flatMap(weatherValueFactory::get);
+            });
   }
 
   private String createQueryStringForCoordinateAndTimeInterval(
@@ -205,7 +207,7 @@ private String createTimeConstraint(ZonedDateTime date) {
   }
 
   private String createCoordinateConstraintString(int coordinateId) {
-    return "coordinate='" + coordinateId + "'";
+    return COORDINATE_ID_COLUMN_NAME + "='" + coordinateId + "'";
   }
 
   /**

src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java

@@ -60,17 +60,17 @@ public SqlWeatherSource(
 
     String dbTimeColumnName =
         getDbColumnName(weatherFactory.getTimeFieldString(), weatherTableName);
-    String dbCoordColumnName = getDbColumnName(factoryCoordinateFieldName, weatherTableName);
+    String dbCoordinateIdColumnName = getDbColumnName(factoryCoordinateFieldName, weatherTableName);
 
     // setup queries
     this.queryTimeInterval =
         createQueryStringForTimeInterval(schemaName, weatherTableName, dbTimeColumnName);
     this.queryTimeAndCoordinate =
         createQueryStringForTimeAndCoordinate(
-            schemaName, weatherTableName, dbTimeColumnName, dbCoordColumnName);
+            schemaName, weatherTableName, dbTimeColumnName, dbCoordinateIdColumnName);
     this.queryTimeIntervalAndCoordinates =
         createQueryStringForTimeIntervalAndCoordinates(
-            schemaName, weatherTableName, dbTimeColumnName, dbCoordColumnName);
+            schemaName, weatherTableName, dbTimeColumnName, dbCoordinateIdColumnName);
   }
 
   @Override

src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy → src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactoryTest.groovy

@@ -8,27 +8,27 @@ package edu.ie3.datamodel.io.factory.timeseries
 import edu.ie3.datamodel.models.StandardUnits
 import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue
 import edu.ie3.datamodel.models.value.WeatherValue
-import edu.ie3.test.common.PsdmWeatherTestData
+import edu.ie3.test.common.CosmoWeatherTestData
 import edu.ie3.util.TimeUtil
 import spock.lang.Specification
 import tech.units.indriya.quantity.Quantities
 
-class PsdmTimeBasedWeatherValueFactoryTest extends Specification {
+class CosmoTimeBasedWeatherValueFactoryTest extends Specification {
 
 	def "A PsdmTimeBasedWeatherValueFactory should be able to create time series with missing values"() {
 		given:
-		def factory = new PsdmTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss")
-		def coordinate = PsdmWeatherTestData.COORDINATE_193186
+		def factory = new CosmoTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss")
+		def coordinate = CosmoWeatherTestData.COORDINATE_193186
 		def time = TimeUtil.withDefaults.toZonedDateTime("2019-01-01 00:00:00")
 
 		Map<String, String> parameter = [
-			"uuid"               : "980f7714-8def-479f-baae-4deed6c8d6d1",
-			"time"               : TimeUtil.withDefaults.toString(time),
-			"diffuseirradiation": "282.671997070312",
-			"directirradiation" : "286.872985839844",
-			"temperature"        : "",
-			"winddirection"     : "0",
-			"windvelocity"      : "1.66103506088257"
+			"uuid"             : "980f7714-8def-479f-baae-4deed6c8d6d1",
+			"time"             : TimeUtil.withDefaults.toString(time),
+			"diffuseirradiance": "282.671997070312",
+			"directirradiance" : "286.872985839844",
+			"temperature"      : "",
+			"winddirection"    : "0",
+			"windvelocity"     : "1.66103506088257"
 		]
 
 		def data = new TimeBasedWeatherValueData(parameter, coordinate)
@@ -50,18 +50,18 @@ class PsdmTimeBasedWeatherValueFactoryTest extends Specification {
 
 	def "A PsdmTimeBasedWeatherValueFactory should be able to create time series values"() {
 		given:
-		def factory = new PsdmTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss")
-		def coordinate = PsdmWeatherTestData.COORDINATE_193186
+		def factory = new CosmoTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss")
+		def coordinate = CosmoWeatherTestData.COORDINATE_193186
 		def time = TimeUtil.withDefaults.toZonedDateTime("2019-01-01 00:00:00")
 
 		Map<String, String> parameter = [
-			"time"               : TimeUtil.withDefaults.toString(time),
-			"uuid"               : "980f7714-8def-479f-baae-4deed6c8d6d1",
-			"diffuseirradiation": "282.671997070312",
-			"directirradiation" : "286.872985839844",
-			"temperature"        : "278.019012451172",
-			"winddirection"     : "0",
-			"windvelocity"      : "1.66103506088257"
+			"time"             : TimeUtil.withDefaults.toString(time),
+			"uuid"             : "980f7714-8def-479f-baae-4deed6c8d6d1",
+			"diffuseirradiance": "282.671997070312",
+			"directirradiance" : "286.872985839844",
+			"temperature"      : "278.019012451172",
+			"winddirection"    : "0",
+			"windvelocity"     : "1.66103506088257"
 		]
 
 		def data = new TimeBasedWeatherValueData(parameter, coordinate)

src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy

@@ -6,7 +6,7 @@
 package edu.ie3.datamodel.io.factory.timeseries
 
 import edu.ie3.datamodel.models.StandardUnits
-import edu.ie3.test.common.PsdmWeatherTestData
+import edu.ie3.test.common.CosmoWeatherTestData
 import edu.ie3.util.TimeUtil
 import edu.ie3.util.quantities.PowerSystemUnits
 import edu.ie3.util.quantities.QuantityUtil
@@ -73,7 +73,7 @@ class IconTimeBasedWeatherValueFactoryTest extends Specification {
 	def "A time based weather value factory for ICON column scheme builds a single time based value correctly"() {
 		given:
 		def factory = new IconTimeBasedWeatherValueFactory()
-		def coordinate = PsdmWeatherTestData.COORDINATE_67775
+		def coordinate = CosmoWeatherTestData.COORDINATE_67775
 
 		def parameter = [
 			"datum"        : "2019-08-01 01:00:00",