Change scheme of COSMO weather data

.gitattributes

@@ -1,3 +1,3 @@
 # The following file will be copied to a unix Docker image and imported to InfluxDB data base. Therefore, the line
 # ending plays a crucial role. This prevents the endings from being adjusted with 'core.autocrlf=true'
-src/test/resources/testContainerFiles/influxDb/weather.txt eol=lf
+src/test/resources/testContainerFiles/influxDb/cosmo/weather.txt eol=lf

CHANGELOG.md

@@ -35,6 +35,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - `edu.ie3.datamodel.io.csv.timeseries.LoadProfileTimeSeriesMetaInformation`
   - `edu.ie3.datamodel.io.connectors.CsvFileConnector.CsvIndividualTimeSeriesMetaInformation`
   - and related methods
+- BREAKING: Comprehensive harmonization around weather sources [#267](https://github.com/ie3-institute/PowerSystemDataModel/issues/267)
+    - Adapted the expected column scheme
+      - DWD COSMO model
+        - `"coordinate"` to `"coordinate id"` (as composite word)
+        - `"diffuseirradiation"` to `"diffuse irradiance"` (as composite word)
+    - Harmonized the column name definition for coordinate id across implementations of `WeatherSource`
+      - Allows for different case conventions based on the used data source technology
 
 ## [2.1.0] - 2022-01-05
 

build.gradle

@@ -40,7 +40,7 @@ apply from: scriptsLocation + 'vcs.gradle'
 apply from: scriptsLocation + 'semVer.gradle'
 
 repositories {
-	mavenCentral() //searches in bintray's repository 'jCenter', which contains Maven Central
+	mavenCentral() // searches in Sonatype's repository 'Maven Central'
 	maven { url 'https://www.jitpack.io' } // allows github repos as dependencies
 
 	// sonatype snapshot repo

src/main/java/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactory.java → src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java

@@ -25,31 +25,25 @@
  * Factory implementation of {@link TimeBasedWeatherValueFactory}, that is able to handle field to
  * value mapping in the typical PowerSystemDataModel (PSDM) column scheme
  */
-public class PsdmTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory {
-  private static final String COORDINATE = "coordinate";
-  private static final String DIFFUSE_IRRADIANCE = "diffuseirradiation";
-  private static final String DIRECT_IRRADIANCE = "directirradiation";
+public class CosmoTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory {
+  private static final String DIFFUSE_IRRADIANCE = "diffuseirradiance";
+  private static final String DIRECT_IRRADIANCE = "directirradiance";
   private static final String TEMPERATURE = "temperature";
   private static final String WIND_DIRECTION = "winddirection";
   private static final String WIND_VELOCITY = "windvelocity";
 
-  public PsdmTimeBasedWeatherValueFactory(TimeUtil timeUtil) {
+  public CosmoTimeBasedWeatherValueFactory(TimeUtil timeUtil) {
     super(timeUtil);
   }
 
-  public PsdmTimeBasedWeatherValueFactory(String timePattern) {
+  public CosmoTimeBasedWeatherValueFactory(String timePattern) {
     super(timePattern);
   }
 
-  public PsdmTimeBasedWeatherValueFactory() {
+  public CosmoTimeBasedWeatherValueFactory() {
     super();
   }
 
-  @Override
-  public String getCoordinateIdFieldString() {
-    return COORDINATE;
-  }
-
   @Override
   public String getTimeFieldString() {
     return TIME;

src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java

@@ -29,7 +29,6 @@
  */
 public class IconTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory {
   /* Redefine the column names to meet the icon specifications */
-  private static final String COORDINATE = "coordinateid";
   private static final String TIME = "datum";
   private static final String DIFFUSE_IRRADIANCE = "aswdifdS";
   private static final String DIRECT_IRRADIANCE = "aswdirS";
@@ -49,11 +48,6 @@ public IconTimeBasedWeatherValueFactory() {
     super(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd HH:mm:ss"));
   }
 
-  @Override
-  public String getCoordinateIdFieldString() {
-    return COORDINATE;
-  }
-
   @Override
   public String getTimeFieldString() {
     return TIME;

src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java

@@ -7,6 +7,8 @@
 
 import edu.ie3.datamodel.models.value.WeatherValue;
 import edu.ie3.util.TimeUtil;
+import edu.ie3.util.naming.Naming;
+import edu.ie3.util.naming.NamingConvention;
 import java.time.ZoneId;
 import java.util.*;
 
@@ -18,6 +20,8 @@ public abstract class TimeBasedWeatherValueFactory
     extends TimeBasedValueFactory<TimeBasedWeatherValueData, WeatherValue> {
   protected static final String UUID = "uuid";
   protected static final String TIME = "time";
+  /* Hold a case agnostic representation of the composite word "coordinate id", that allows for later case conversion */
+  protected static final Naming COORDINATE_ID_NAMING = Naming.from("coordinate", "id");
 
   protected final TimeUtil timeUtil;
 
@@ -35,11 +39,23 @@ protected TimeBasedWeatherValueFactory(TimeUtil timeUtil) {
   }
 
   /**
-   * Return the field name for the coordinate id
+   * Return the field name for the coordinate id in flat case.
    *
    * @return the field name for the coordinate id
    */
-  public abstract String getCoordinateIdFieldString();
+  public String getCoordinateIdFieldString() {
+    return COORDINATE_ID_NAMING.flatCase();
+  }
+
+  /**
+   * Return the field name for the coordinate id in desired case.
+   *
+   * @param convention The desired naming convention / casing
+   * @return the field name for the coordinate id in appropriate case
+   */
+  public String getCoordinateIdFieldString(NamingConvention convention) {
+    return COORDINATE_ID_NAMING.as(convention);
+  }
 
   /**
    * Return the field name for the date time

src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java

@@ -19,6 +19,7 @@
 import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue;
 import edu.ie3.datamodel.models.value.WeatherValue;
 import edu.ie3.util.interval.ClosedInterval;
+import edu.ie3.util.naming.NamingConvention;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
@@ -33,6 +34,9 @@
 /** Couchbase Source for weather data */
 public class CouchbaseWeatherSource implements WeatherSource {
   private static final Logger logger = LoggerFactory.getLogger(CouchbaseWeatherSource.class);
+
+  /* Column names in couchbase come in "flat" case by default */
+  private static final NamingConvention DEFAULT_NAMING_CONVENTION = NamingConvention.FLAT;
   private static final String DEFAULT_TIMESTAMP_PATTERN = "yyyy-MM-dd'T'HH:mm:ssxxx";
   /** The start of the document key, comparable to a table name in relational databases */
   private static final String DEFAULT_KEY_PREFIX = "weather";
@@ -42,26 +46,47 @@ public class CouchbaseWeatherSource implements WeatherSource {
   private final String keyPrefix;
   private final CouchbaseConnector connector;
   private final IdCoordinateSource coordinateSource;
+  /* Final name of the column within the database */
   private final String coordinateIdColumnName;
 
   /**
    * Instantiate a weather source utilising a connection to a couchbase instance obtained via the
    * connector. This convenient constructor uses the {@link
-   * CouchbaseWeatherSource#DEFAULT_KEY_PREFIX} as key prefix.
+   * CouchbaseWeatherSource#DEFAULT_KEY_PREFIX} as key prefix and {@link
+   * CouchbaseWeatherSource#DEFAULT_NAMING_CONVENTION} as naming convention.
    *
    * @param connector Connector, that establishes the connection to the couchbase instance
    * @param coordinateSource Source to obtain actual coordinates from
-   * @param coordinateIdColumnName Name of the column containing the information about the
-   *     coordinate identifier
    * @param weatherFactory Factory to transfer field to value mapping into actual java object
    *     instances
    */
   public CouchbaseWeatherSource(
       CouchbaseConnector connector,
       IdCoordinateSource coordinateSource,
-      String coordinateIdColumnName,
       TimeBasedWeatherValueFactory weatherFactory) {
-    this(connector, coordinateSource, coordinateIdColumnName, DEFAULT_KEY_PREFIX, weatherFactory);
+    this(
+        connector, coordinateSource, DEFAULT_KEY_PREFIX, DEFAULT_NAMING_CONVENTION, weatherFactory);
+  }
+
+  /**
+   * Instantiate a weather source utilising a connection to a couchbase instance obtained via the
+   * connector. Uses {@link CouchbaseWeatherSource#DEFAULT_NAMING_CONVENTION} as naming convention.
+   *
+   * @param connector Connector, that establishes the connection to the couchbase instance
+   * @param coordinateSource Source to obtain actual coordinates from
+   * @param keyPrefix Prefix of entries, that belong to weather
+   * @param weatherFactory Factory to transfer field to value mapping into actual java object
+   *     instances
+   * @deprecated Use {@link CouchbaseWeatherSource#CouchbaseWeatherSource(CouchbaseConnector,
+   *     IdCoordinateSource, String, NamingConvention, TimeBasedWeatherValueFactory)} instead
+   */
+  @Deprecated(since = "3.0", forRemoval = true)
+  public CouchbaseWeatherSource(
+      CouchbaseConnector connector,
+      IdCoordinateSource coordinateSource,
+      String keyPrefix,
+      TimeBasedWeatherValueFactory weatherFactory) {
+    this(connector, coordinateSource, keyPrefix, DEFAULT_NAMING_CONVENTION, weatherFactory);
   }
 
   /**
@@ -70,23 +95,22 @@ public CouchbaseWeatherSource(
    *
    * @param connector Connector, that establishes the connection to the couchbase instance
    * @param coordinateSource Source to obtain actual coordinates from
-   * @param coordinateIdColumnName Name of the column containing the information about the
-   *     coordinate identifier
    * @param keyPrefix Prefix of entries, that belong to weather
+   * @param namingConvention the (case) convention, how columns are named
    * @param weatherFactory Factory to transfer field to value mapping into actual java object
    *     instances
    */
   public CouchbaseWeatherSource(
       CouchbaseConnector connector,
       IdCoordinateSource coordinateSource,
-      String coordinateIdColumnName,
       String keyPrefix,
+      NamingConvention namingConvention,
       TimeBasedWeatherValueFactory weatherFactory) {
     this.connector = connector;
     this.coordinateSource = coordinateSource;
-    this.coordinateIdColumnName = coordinateIdColumnName;
     this.keyPrefix = keyPrefix;
     this.weatherFactory = weatherFactory;
+    this.coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention);
   }
 
   @Override

src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java

@@ -15,6 +15,7 @@
 import edu.ie3.datamodel.models.value.WeatherValue;
 import edu.ie3.util.StringUtils;
 import edu.ie3.util.interval.ClosedInterval;
+import edu.ie3.util.naming.Naming;
 import java.time.ZonedDateTime;
 import java.util.*;
 import java.util.stream.Collectors;
@@ -30,7 +31,11 @@ public class InfluxDbWeatherSource implements WeatherSource {
   private static final String WHERE = " where ";
   private static final String MEASUREMENT_NAME_WEATHER = "weather";
   private static final int MILLI_TO_NANO_FACTOR = 1000000;
-  private final String coordinateIdColumnName;
+
+  /* Final name of the coordinate id field for use in factories */
+  private final String coordinateIdFieldName;
+  /* Final name of the column within the database */
+  private final String coordinateIdColumnName = Naming.from("coordinate", "id").snakeCase();
   private final InfluxDbConnector connector;
   private final IdCoordinateSource coordinateSource;
   private final TimeBasedWeatherValueFactory weatherValueFactory;
@@ -50,7 +55,7 @@ public InfluxDbWeatherSource(
     this.connector = connector;
     this.coordinateSource = coordinateSource;
     this.weatherValueFactory = weatherValueFactory;
-    this.coordinateIdColumnName = weatherValueFactory.getCoordinateIdFieldString();
+    this.coordinateIdFieldName = weatherValueFactory.getCoordinateIdFieldString();
   }
 
   @Override
@@ -152,24 +157,25 @@ private Stream<Optional<TimeBasedValue<WeatherValue>>> optTimeBasedValueStream(
     return measurementsMap.get(MEASUREMENT_NAME_WEATHER).stream()
         .map(
             fieldToValue -> {
-              Optional<Point> coordinate =
-                  coordinateSource.getCoordinate(
-                      Integer.parseInt(fieldToValue.remove(coordinateIdColumnName)));
-              if (!coordinate.isPresent()) return null;
-              fieldToValue.putIfAbsent("uuid", UUID.randomUUID().toString());
-
-              /* The factory expects camel case id's for fields -> Convert the keys */
-              Map<String, String> camelCaseFields =
+              /* The factory expects flat case id's for fields -> Convert the keys */
+              Map<String, String> flatCaseFields =
                   fieldToValue.entrySet().stream()
                       .collect(
                           Collectors.toMap(
-                              entry -> StringUtils.snakeCaseToCamelCase(entry.getKey()),
+                              entry ->
+                                  StringUtils.snakeCaseToCamelCase(entry.getKey()).toLowerCase(),
                               Map.Entry::getValue));
 
-              return new TimeBasedWeatherValueData(camelCaseFields, coordinate.get());
-            })
-        .filter(Objects::nonNull)
-        .map(weatherValueFactory::get);
+              /* Add a random UUID if necessary */
+              flatCaseFields.putIfAbsent("uuid", UUID.randomUUID().toString());
+
+              /* Get the corresponding coordinate id from map AND REMOVE THE ENTRY !!! */
+              int coordinateId = Integer.parseInt(flatCaseFields.remove(coordinateIdFieldName));
+              return coordinateSource
+                  .getCoordinate(coordinateId)
+                  .map(point -> new TimeBasedWeatherValueData(flatCaseFields, point))
+                  .flatMap(weatherValueFactory::get);
+            });
   }
 
   private String createQueryStringForCoordinateAndTimeInterval(
@@ -205,7 +211,7 @@ private String createTimeConstraint(ZonedDateTime date) {
   }
 
   private String createCoordinateConstraintString(int coordinateId) {
-    return "coordinate='" + coordinateId + "'";
+    return coordinateIdColumnName + "='" + coordinateId + "'";
   }
 
   /**

src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java

@@ -14,6 +14,7 @@
 import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue;
 import edu.ie3.datamodel.models.value.WeatherValue;
 import edu.ie3.util.interval.ClosedInterval;
+import edu.ie3.util.naming.NamingConvention;
 import java.sql.*;
 import java.time.ZonedDateTime;
 import java.util.*;
@@ -24,6 +25,8 @@
 public class SqlWeatherSource extends SqlDataSource<TimeBasedValue<WeatherValue>>
     implements WeatherSource {
   private static final String WHERE = " WHERE ";
+  /* Column names in sql come in "snake" case by default */
+  private static final NamingConvention DEFAULT_NAMING_CONVENTION = NamingConvention.SNAKE;
 
   private final IdCoordinateSource idCoordinateSource;
   private final String factoryCoordinateFieldName;
@@ -38,39 +41,69 @@ public class SqlWeatherSource extends SqlDataSource<TimeBasedValue<WeatherValue>
   private final String queryTimeAndCoordinate;
   private final String queryTimeIntervalAndCoordinates;
 
+  /**
+   * Initializes a new SqlWeatherSource. Uses {@link SqlWeatherSource#DEFAULT_NAMING_CONVENTION} as
+   * naming convention.
+   *
+   * @param connector the connector needed for database connection
+   * @param idCoordinateSource a coordinate source to map ids to points
+   * @param schemaName the database schema to use
+   * @param weatherTableName the name of the table containing weather data
+   * @param weatherFactory instance of a time based weather value factory
+   * @deprecated Use {@link SqlWeatherSource#SqlWeatherSource(SqlConnector, IdCoordinateSource,
+   *     String, String, NamingConvention, TimeBasedWeatherValueFactory)} instead
+   */
+  @Deprecated(since = "3.0", forRemoval = true)
+  public SqlWeatherSource(
+      SqlConnector connector,
+      IdCoordinateSource idCoordinateSource,
+      String schemaName,
+      String weatherTableName,
+      TimeBasedWeatherValueFactory weatherFactory) {
+    this(
+        connector,
+        idCoordinateSource,
+        schemaName,
+        weatherTableName,
+        DEFAULT_NAMING_CONVENTION,
+        weatherFactory);
+  }
   /**
    * Initializes a new SqlWeatherSource
    *
    * @param connector the connector needed for database connection
    * @param idCoordinateSource a coordinate source to map ids to points
    * @param schemaName the database schema to use
    * @param weatherTableName the name of the table containing weather data
+   * @param namingConvention the (case) convention, how columns are named
    * @param weatherFactory instance of a time based weather value factory
    */
   public SqlWeatherSource(
       SqlConnector connector,
       IdCoordinateSource idCoordinateSource,
       String schemaName,
       String weatherTableName,
+      NamingConvention namingConvention,
       TimeBasedWeatherValueFactory weatherFactory) {
     super(connector);
     this.idCoordinateSource = idCoordinateSource;
     this.weatherFactory = weatherFactory;
     this.factoryCoordinateFieldName = weatherFactory.getCoordinateIdFieldString();
 
+    /* Determine the correct column names in database */
+    String coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention);
     String dbTimeColumnName =
         getDbColumnName(weatherFactory.getTimeFieldString(), weatherTableName);
-    String dbCoordColumnName = getDbColumnName(factoryCoordinateFieldName, weatherTableName);
 
     // setup queries
     this.queryTimeInterval =
         createQueryStringForTimeInterval(schemaName, weatherTableName, dbTimeColumnName);
     this.queryTimeAndCoordinate =
         createQueryStringForTimeAndCoordinate(
-            schemaName, weatherTableName, dbTimeColumnName, dbCoordColumnName);
+            schemaName, weatherTableName, dbTimeColumnName, coordinateIdColumnName);
     this.queryTimeIntervalAndCoordinates =
         createQueryStringForTimeIntervalAndCoordinates(
-            schemaName, weatherTableName, dbTimeColumnName, dbCoordColumnName);
+            schemaName, weatherTableName, dbTimeColumnName, coordinateIdColumnName);
   }
 
   @Override