From 80460e71d3a927c18767d9e927474b99c1791234 Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 10 Nov 2025 02:40:50 +0100 Subject: [PATCH 1/8] Add JSON source infrastructure and Markov factory --- build.gradle | 2 + .../io/connectors/JsonFileConnector.java | 53 +++ .../markov/MarkovLoadModelFactory.java | 317 ++++++++++++++++++ .../io/factory/markov/MarkovModelData.java | 26 ++ .../edu/ie3/datamodel/io/file/FileType.java | 3 +- .../io/source/json/JsonDataSource.java | 142 ++++++++ .../source/json/JsonMarkovProfileSource.java | 117 +++++++ .../profile/markov/MarkovLoadModel.java | 63 ++++ 8 files changed, 722 insertions(+), 1 deletion(-) create mode 100644 src/main/java/edu/ie3/datamodel/io/connectors/JsonFileConnector.java create mode 100644 src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java create mode 100644 src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java create mode 100644 src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java create mode 100644 src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java create mode 100644 src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java diff --git a/build.gradle b/build.gradle index bf0966b5a..e420517cd 100644 --- a/build.gradle +++ b/build.gradle @@ -105,6 +105,8 @@ dependencies { implementation 'commons-io:commons-io:2.20.0' // I/O functionalities implementation 'commons-codec:commons-codec:1.20.0' // needed by commons-compress implementation 'org.apache.commons:commons-compress:1.28.0' // I/O functionalities + + implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.2' } tasks.withType(JavaCompile) { diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/JsonFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/JsonFileConnector.java new file mode 100644 index 000000000..40764d551 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/connectors/JsonFileConnector.java @@ -0,0 +1,53 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.connectors; + +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.function.Function; + +/** Connector for JSON-based sources and sinks. */ +public class JsonFileConnector extends FileConnector { + private static final String FILE_ENDING = ".json"; + + public JsonFileConnector(Path baseDirectory) { + super(baseDirectory); + } + + public JsonFileConnector(Path baseDirectory, Function customInputStream) { + super(baseDirectory, customInputStream); + } + + /** + * Opens a buffered reader for the given JSON file, using UTF-8 decoding. + * + * @param filePath relative path without ending + * @return buffered reader referencing the JSON file + */ + public BufferedReader initReader(Path filePath) throws FileNotFoundException { + InputStream inputStream = openInputStream(filePath); + return new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8), 16384); + } + + /** + * Opens an input stream for the given JSON file. + * + * @param filePath relative path without ending + * @return input stream for the file + */ + public InputStream initInputStream(Path filePath) throws FileNotFoundException { + return openInputStream(filePath); + } + + @Override + protected String getFileEnding() { + return FILE_ENDING; + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java new file mode 100644 index 000000000..aeabdffe2 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -0,0 +1,317 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.factory.markov; + +import com.fasterxml.jackson.databind.JsonNode; +import edu.ie3.datamodel.exceptions.FactoryException; +import edu.ie3.datamodel.io.factory.Factory; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.*; +import java.time.ZonedDateTime; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; + +/** Factory turning Markov JSON data into {@link MarkovLoadModel}s. */ +public class MarkovLoadModelFactory + extends Factory { + + public MarkovLoadModelFactory() { + super(MarkovLoadModel.class); + } + + @Override + protected MarkovLoadModel buildModel(MarkovModelData data) { + JsonNode root = data.getRoot(); + String schema = requireText(root, "schema"); + ZonedDateTime generatedAt = parseTimestamp(requireText(root, "generated_at")); + Generator generator = parseGenerator(requireNode(root, "generator")); + TimeModel timeModel = parseTimeModel(requireNode(root, "time_model")); + ValueModel valueModel = parseValueModel(requireNode(root, "value_model")); + Parameters parameters = parseParameters(root.path("parameters")); + + JsonNode dataNode = requireNode(root, "data"); + TransitionData transitionData = + parseTransitions(dataNode, timeModel.bucketCount(), valueModel.discretization().states()); + Optional gmmBuckets = parseGmmBuckets(dataNode.path("gmms")); + + return new MarkovLoadModel( + schema, + generatedAt, + generator, + timeModel, + valueModel, + parameters, + transitionData, + gmmBuckets); + } + + @Override + protected List> getFields(Class entityClass) { + Set requiredFields = + newSet( + "schema", + "generated_at", + "generator.name", + "generator.version", + "time_model.bucket_count", + "time_model.bucket_encoding.formula", + "time_model.sampling_interval_minutes", + "time_model.timezone", + "value_model.value_unit", + "value_model.normalization.method", + "value_model.discretization.states", + "value_model.discretization.thresholds_right", + "data.transitions.shape", + "data.transitions.values", + "data.gmms.buckets"); + return List.of(requiredFields); + } + + private static Generator parseGenerator(JsonNode generatorNode) { + String name = requireText(generatorNode, "name"); + String version = requireText(generatorNode, "version"); + Map config = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + JsonNode configNode = generatorNode.path("config"); + if (configNode.isObject()) { + Iterator> fields = configNode.fields(); + while (fields.hasNext()) { + Map.Entry entry = fields.next(); + config.put(entry.getKey(), entry.getValue().asText()); + } + } + return new Generator(name, version, config); + } + + private static TimeModel parseTimeModel(JsonNode timeNode) { + int bucketCount = requireInt(timeNode, "bucket_count"); + String formula = requireNode(timeNode, "bucket_encoding").path("formula").asText(""); + if (formula.isEmpty()) { + throw new FactoryException("Missing bucket encoding formula"); + } + int samplingInterval = requireInt(timeNode, "sampling_interval_minutes"); + String timezone = requireText(timeNode, "timezone"); + return new TimeModel(bucketCount, formula, samplingInterval, timezone); + } + + private static ValueModel parseValueModel(JsonNode valueNode) { + String valueUnit = requireText(valueNode, "value_unit"); + JsonNode normalizationNode = requireNode(valueNode, "normalization"); + String normalizationMethod = requireText(normalizationNode, "method"); + ValueModel.Normalization normalization = new ValueModel.Normalization(normalizationMethod); + + JsonNode discretizationNode = requireNode(valueNode, "discretization"); + int states = requireInt(discretizationNode, "states"); + List thresholds = new ArrayList<>(); + JsonNode thresholdsNode = requireNode(discretizationNode, "thresholds_right"); + if (!thresholdsNode.isArray()) { + throw new FactoryException("thresholds_right must be an array"); + } + thresholdsNode.forEach(element -> thresholds.add(element.asDouble())); + ValueModel.Discretization discretization = + new ValueModel.Discretization(states, List.copyOf(thresholds)); + + return new ValueModel(valueUnit, normalization, discretization); + } + + private static Parameters parseParameters(JsonNode parametersNode) { + Parameters.TransitionParameters transitions = + new Parameters.TransitionParameters( + parametersNode.path("transitions").path("empty_row_strategy").asText("")); + if (transitions.emptyRowStrategy().isEmpty()) { + transitions = null; + } + + JsonNode gmmNode = parametersNode.path("gmm"); + Parameters.GmmParameters gmm = + gmmNode.isMissingNode() || gmmNode.isNull() || gmmNode.size() == 0 + ? null + : new Parameters.GmmParameters( + gmmNode.path("value_col").asText(""), + optionalInt(gmmNode, "verbose"), + optionalInt(gmmNode, "heartbeat_seconds")); + + return new Parameters(transitions, gmm); + } + + private static Optional optionalInt(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isNull()) return Optional.empty(); + return Optional.of(value.asInt()); + } + + private static TransitionData parseTransitions( + JsonNode dataNode, int expectedBucketCount, int stateCount) { + JsonNode transitionsNode = requireNode(dataNode, "transitions"); + String dtype = requireText(transitionsNode, "dtype"); + String encoding = requireText(transitionsNode, "encoding"); + + JsonNode shapeNode = requireNode(transitionsNode, "shape"); + if (!shapeNode.isArray() || shapeNode.size() != 3) { + throw new FactoryException("Transition shape must contain three dimensions"); + } + int buckets = shapeNode.get(0).asInt(); + int rows = shapeNode.get(1).asInt(); + int columns = shapeNode.get(2).asInt(); + if (buckets != expectedBucketCount) { + throw new FactoryException( + "Transition bucket count mismatch. Expected " + + expectedBucketCount + + " but was " + + buckets); + } + if (rows != stateCount || columns != stateCount) { + throw new FactoryException( + "Transition state dimension mismatch. Expected " + + stateCount + + " but was rows=" + + rows + + ", columns=" + + columns); + } + + JsonNode valuesNode = requireNode(transitionsNode, "values"); + if (!valuesNode.isArray()) { + throw new FactoryException("Transition values must be a three dimensional array"); + } + + double[][][] values = new double[buckets][stateCount][stateCount]; + int bucketIndex = 0; + for (JsonNode bucketNode : valuesNode) { + if (bucketIndex >= buckets) { + throw new FactoryException("More transition buckets present than specified in shape"); + } + int rowIndex = 0; + for (JsonNode rowNode : bucketNode) { + if (rowIndex >= stateCount) { + throw new FactoryException( + "Too many rows in transition matrix for bucket " + bucketIndex); + } + int columnIndex = 0; + for (JsonNode probNode : rowNode) { + if (columnIndex >= stateCount) { + throw new FactoryException( + "Too many columns in transition matrix for bucket " + + bucketIndex + + ", row " + + rowIndex); + } + values[bucketIndex][rowIndex][columnIndex] = probNode.asDouble(); + columnIndex++; + } + if (columnIndex != stateCount) { + throw new FactoryException( + "Row " + + rowIndex + + " in bucket " + + bucketIndex + + " had " + + columnIndex + + " columns. Expected " + + stateCount); + } + rowIndex++; + } + if (rowIndex != stateCount) { + throw new FactoryException( + "Bucket " + bucketIndex + " contained " + rowIndex + " rows. Expected " + stateCount); + } + bucketIndex++; + } + if (bucketIndex != buckets) { + throw new FactoryException( + "Transition values provided only " + bucketIndex + " buckets. Expected " + buckets); + } + + return new TransitionData(dtype, encoding, values); + } + + private static Optional parseGmmBuckets(JsonNode gmmsNode) { + if (gmmsNode == null + || gmmsNode.isMissingNode() + || gmmsNode.isNull() + || !gmmsNode.has("buckets")) { + return Optional.empty(); + } + JsonNode bucketsNode = gmmsNode.get("buckets"); + if (!bucketsNode.isArray()) { + throw new FactoryException("data.gmms.buckets must be an array"); + } + List buckets = new ArrayList<>(); + for (JsonNode bucketNode : bucketsNode) { + JsonNode statesNode = bucketNode.get("states"); + if (statesNode == null || !statesNode.isArray()) { + throw new FactoryException("Each GMM bucket must contain an array 'states'"); + } + List> states = new ArrayList<>(); + for (JsonNode stateNode : statesNode) { + if (stateNode == null || stateNode.isNull()) { + states.add(Optional.empty()); + continue; + } + List weights = readDoubleArray(stateNode, "weights"); + List means = readDoubleArray(stateNode, "means"); + List variances = readDoubleArray(stateNode, "variances"); + states.add(Optional.of(new GmmBuckets.GmmState(weights, means, variances))); + } + buckets.add(new GmmBuckets.GmmBucket(List.copyOf(states))); + } + return Optional.of(new GmmBuckets(List.copyOf(buckets))); + } + + private static List readDoubleArray(JsonNode node, String field) { + JsonNode arrayNode = node.get(field); + if (arrayNode == null || !arrayNode.isArray()) { + throw new FactoryException("Field '" + field + "' must be an array"); + } + List values = new ArrayList<>(); + arrayNode.forEach(element -> values.add(element.asDouble())); + return List.copyOf(values); + } + + private static JsonNode requireNode(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode()) { + throw new FactoryException("Missing field '" + field + "'"); + } + return value; + } + + private static String requireText(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode() || value.isNull()) { + throw new FactoryException("Missing field '" + field + "'"); + } + if (!value.isTextual()) { + throw new FactoryException("Field '" + field + "' must be textual"); + } + return value.asText(); + } + + private static int requireInt(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode() || value.isNull()) { + throw new FactoryException("Missing field '" + field + "'"); + } + if (!value.canConvertToInt()) { + throw new FactoryException("Field '" + field + "' must be an integer"); + } + return value.asInt(); + } + + private static ZonedDateTime parseTimestamp(String timestamp) { + try { + return ZonedDateTime.parse(timestamp); + } catch (DateTimeParseException e) { + throw new FactoryException("Unable to parse generated_at timestamp '" + timestamp + "'", e); + } + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java new file mode 100644 index 000000000..e1cbdf3b0 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java @@ -0,0 +1,26 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.factory.markov; + +import com.fasterxml.jackson.databind.JsonNode; +import edu.ie3.datamodel.io.factory.FactoryData; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; +import java.util.Collections; +import java.util.Objects; + +/** Factory data wrapper around a parsed Markov-load JSON tree. */ +public class MarkovModelData extends FactoryData { + private final JsonNode root; + + public MarkovModelData(JsonNode root) { + super(Collections.emptyMap(), MarkovLoadModel.class); + this.root = Objects.requireNonNull(root, "root"); + } + + public JsonNode getRoot() { + return root; + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/file/FileType.java b/src/main/java/edu/ie3/datamodel/io/file/FileType.java index b9ac1f3f7..08886b255 100644 --- a/src/main/java/edu/ie3/datamodel/io/file/FileType.java +++ b/src/main/java/edu/ie3/datamodel/io/file/FileType.java @@ -10,7 +10,8 @@ import java.util.stream.Collectors; public enum FileType { - CSV(".csv"); + CSV(".csv"), + JSON(".json"); public final String fileEnding; diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java new file mode 100644 index 000000000..945a3b11a --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java @@ -0,0 +1,142 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.source.json; + +import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.io.connectors.JsonFileConnector; +import edu.ie3.datamodel.io.naming.FileNamingStrategy; +import edu.ie3.datamodel.io.source.file.FileDataSource; +import edu.ie3.datamodel.models.Entity; +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.nio.file.Path; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Stream; + +/** Data source abstraction for JSON files. */ +public class JsonDataSource extends FileDataSource { + + private final JsonFileConnector connector; + + public JsonDataSource(Path directoryPath, FileNamingStrategy fileNamingStrategy) { + this(new JsonFileConnector(directoryPath), fileNamingStrategy); + } + + public JsonDataSource(JsonFileConnector connector, FileNamingStrategy fileNamingStrategy) { + super(connector.getBaseDirectory(), fileNamingStrategy); + this.connector = connector; + } + + /** + * Opens a buffered reader for the provided file path. + * + * @param filePath relative path without ending + * @return buffered reader + * @throws SourceException if the file cannot be opened + */ + public BufferedReader initReader(Path filePath) throws SourceException { + try { + return connector.initReader(filePath); + } catch (FileNotFoundException e) { + throw new SourceException("Unable to open JSON file '" + filePath + "'.", e); + } + } + + /** + * Opens an input stream for the provided file path. + * + * @param filePath relative path without ending + * @return input stream + * @throws SourceException if the file cannot be opened + */ + public InputStream initInputStream(Path filePath) throws SourceException { + try { + return connector.initInputStream(filePath); + } catch (FileNotFoundException e) { + throw new SourceException("Unable to open JSON file '" + filePath + "'.", e); + } + } + + /** + * Utility method that reads the entire JSON file into memory. + * + * @param filePath relative path without ending + * @return optional JSON string (empty if the file does not exist) + * @throws SourceException if reading fails + */ + public Optional readRaw(Path filePath) throws SourceException { + try (Reader reader = connector.initReader(filePath)) { + return Optional.of(readAll(reader)); + } catch (FileNotFoundException e) { + return Optional.empty(); + } catch (IOException e) { + throw new SourceException("Unable to read JSON file '" + filePath + "'.", e); + } + } + + /** + * Reads the JSON file using the provided consumer function. + * + * @param filePath relative path without ending + * @param readerFunction function that consumes the reader + * @param result type + * @return optional result (empty if file not found) + * @throws SourceException if reading fails + */ + public Optional readWith(Path filePath, Function readerFunction) + throws SourceException { + try (BufferedReader reader = connector.initReader(filePath)) { + return Optional.ofNullable(readerFunction.apply(reader)); + } catch (FileNotFoundException e) { + return Optional.empty(); + } catch (IOException e) { + throw new SourceException("Unable to read JSON file '" + filePath + "'.", e); + } + } + + @Override + public Optional> getSourceFields(Class entityClass) + throws SourceException { + throw unsupportedTabularAccess("getSourceFields(Class)"); + } + + @Override + public Stream> getSourceData(Class entityClass) + throws SourceException { + throw unsupportedTabularAccess("getSourceData(Class)"); + } + + @Override + public Optional> getSourceFields(Path filePath) throws SourceException { + throw unsupportedTabularAccess("getSourceFields(Path)"); + } + + @Override + public Stream> getSourceData(Path filePath) throws SourceException { + throw unsupportedTabularAccess("getSourceData(Path)"); + } + + private UnsupportedOperationException unsupportedTabularAccess(String method) { + return new UnsupportedOperationException( + "JsonDataSource does not support '" + method + "', as JSON sources are not tabular."); + } + + private String readAll(Reader reader) throws IOException { + StringBuilder builder = new StringBuilder(); + char[] buffer = new char[4096]; + int read; + while ((read = reader.read(buffer)) != -1) { + builder.append(buffer, 0, read); + } + return builder.toString(); + } +} diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java new file mode 100644 index 000000000..d3240a896 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSource.java @@ -0,0 +1,117 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.io.source.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.ie3.datamodel.exceptions.FactoryException; +import edu.ie3.datamodel.exceptions.FailedValidationException; +import edu.ie3.datamodel.exceptions.SourceException; +import edu.ie3.datamodel.exceptions.ValidationException; +import edu.ie3.datamodel.io.factory.markov.MarkovLoadModelFactory; +import edu.ie3.datamodel.io.factory.markov.MarkovModelData; +import edu.ie3.datamodel.io.file.FileType; +import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation; +import edu.ie3.datamodel.io.source.EntitySource; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; + +/** Source that reads Markov-based load models from JSON files. */ +public class JsonMarkovProfileSource extends EntitySource { + + private final JsonDataSource dataSource; + private final FileLoadProfileMetaInformation metaInformation; + private final MarkovLoadModelFactory factory; + private final ObjectMapper objectMapper = new ObjectMapper(); + private MarkovLoadModel cachedModel; + + public JsonMarkovProfileSource( + JsonDataSource dataSource, FileLoadProfileMetaInformation metaInformation) { + this(dataSource, metaInformation, new MarkovLoadModelFactory()); + } + + public JsonMarkovProfileSource( + JsonDataSource dataSource, + FileLoadProfileMetaInformation metaInformation, + MarkovLoadModelFactory factory) { + this.dataSource = Objects.requireNonNull(dataSource, "dataSource"); + this.metaInformation = Objects.requireNonNull(metaInformation, "metaInformation"); + this.factory = Objects.requireNonNull(factory, "factory"); + if (metaInformation.getFileType() != FileType.JSON) { + throw new IllegalArgumentException("Markov profile source requires JSON meta information."); + } + } + + /** + * Returns the parsed Markov model, parsing the underlying file if needed. + * + * @throws SourceException if reading or parsing fails + */ + public synchronized MarkovLoadModel getModel() throws SourceException { + if (cachedModel == null) { + JsonNode root = readModelTree(); + try { + cachedModel = factory.get(new MarkovModelData(root)).getOrThrow(); + } catch (FactoryException e) { + throw new SourceException( + "Unable to build Markov load model from '" + metaInformation.getProfile() + "'.", e); + } + } + return cachedModel; + } + + @Override + public void validate() throws ValidationException { + JsonNode root; + try { + root = readModelTree(); + } catch (SourceException e) { + throw new FailedValidationException( + "Unable to read Markov model '" + metaInformation.getProfile() + "' for validation.", e); + } + Set fields = collectFieldNames(root); + factory.validate(fields, MarkovLoadModel.class).getOrThrow(); + } + + private JsonNode readModelTree() throws SourceException { + Path filePath = metaInformation.getFullFilePath(); + try (InputStream inputStream = dataSource.initInputStream(filePath)) { + return objectMapper.readTree(inputStream); + } catch (IOException e) { + throw new SourceException("Unable to read Markov model JSON from '" + filePath + "'.", e); + } + } + + private static Set collectFieldNames(JsonNode node) { + Set fields = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + collectFields("", node, fields); + return fields; + } + + private static void collectFields(String prefix, JsonNode node, Set collector) { + if (node.isArray()) { + if (!prefix.isEmpty()) { + collector.add(prefix); + } + return; + } + if (node.isObject()) { + node.fieldNames() + .forEachRemaining(name -> collectFields(join(prefix, name), node.get(name), collector)); + } else if (!prefix.isEmpty()) { + collector.add(prefix); + } + } + + private static String join(String prefix, String name) { + return prefix.isEmpty() ? name : prefix + "." + name; + } +} diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java new file mode 100644 index 000000000..3ba8163b6 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -0,0 +1,63 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation +*/ +package edu.ie3.datamodel.models.profile.markov; + +import java.time.ZonedDateTime; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** Container for Markov-chain-based load models produced by simonaMarkovLoad. */ +public record MarkovLoadModel( + String schema, + ZonedDateTime generatedAt, + Generator generator, + TimeModel timeModel, + ValueModel valueModel, + Parameters parameters, + TransitionData transitionData, + Optional gmmBuckets) { + + public record Generator(String name, String version, Map config) {} + + public record TimeModel( + int bucketCount, + String bucketEncodingFormula, + int samplingIntervalMinutes, + String timezone) {} + + public record ValueModel( + String valueUnit, Normalization normalization, Discretization discretization) { + + public record Normalization(String method) {} + + public record Discretization(int states, List thresholdsRight) {} + } + + public record Parameters(TransitionParameters transitions, GmmParameters gmm) { + + public record TransitionParameters(String emptyRowStrategy) {} + + public record GmmParameters( + String valueColumn, Optional verbose, Optional heartbeatSeconds) {} + } + + public record TransitionData(String dtype, String encoding, double[][][] values) { + public int bucketCount() { + return values.length; + } + + public int stateCount() { + return values.length == 0 ? 0 : values[0].length; + } + } + + public record GmmBuckets(List buckets) { + public record GmmBucket(List> states) {} + + public record GmmState(List weights, List means, List variances) {} + } +} From 80e932dcaa134da5033bc53e0c37ce523d1f8f69 Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 10 Nov 2025 03:10:06 +0100 Subject: [PATCH 2/8] bug Fix --- .../profile/markov/MarkovLoadModel.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java index 3ba8163b6..2b525a966 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -6,8 +6,10 @@ package edu.ie3.datamodel.models.profile.markov; import java.time.ZonedDateTime; +import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; /** Container for Markov-chain-based load models produced by simonaMarkovLoad. */ @@ -53,6 +55,35 @@ public int bucketCount() { public int stateCount() { return values.length == 0 ? 0 : values[0].length; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof TransitionData(String dtype1, String encoding1, double[][][] values1))) + return false; + return Objects.equals(dtype, dtype1) + && Objects.equals(encoding, encoding1) + && Arrays.deepEquals(values, values1); + } + + @Override + public int hashCode() { + return Objects.hash(dtype, encoding, Arrays.deepHashCode(values)); + } + + @Override + public String toString() { + return "TransitionData{" + + "dtype='" + + dtype + + '\'' + + ", encoding='" + + encoding + + '\'' + + ", values=" + + Arrays.deepToString(values) + + '}'; + } } public record GmmBuckets(List buckets) { From a590311d63c7064a636aa488719e69e1208d072d Mon Sep 17 00:00:00 2001 From: Philipp Date: Mon, 17 Nov 2025 02:40:57 +0100 Subject: [PATCH 3/8] little fix --- .../markov/MarkovLoadModelFactory.java | 15 ++--- .../io/source/json/JsonDataSource.java | 66 ------------------- 2 files changed, 6 insertions(+), 75 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java index aeabdffe2..6ebe5e178 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -41,7 +41,7 @@ protected MarkovLoadModel buildModel(MarkovModelData data) { JsonNode dataNode = requireNode(root, "data"); TransitionData transitionData = parseTransitions(dataNode, timeModel.bucketCount(), valueModel.discretization().states()); - Optional gmmBuckets = parseGmmBuckets(dataNode.path("gmms")); + GmmBuckets gmmBuckets = parseGmmBuckets(requireNode(dataNode, "gmms")); return new MarkovLoadModel( schema, @@ -51,7 +51,7 @@ protected MarkovLoadModel buildModel(MarkovModelData data) { valueModel, parameters, transitionData, - gmmBuckets); + Optional.of(gmmBuckets)); } @Override @@ -234,12 +234,9 @@ private static TransitionData parseTransitions( return new TransitionData(dtype, encoding, values); } - private static Optional parseGmmBuckets(JsonNode gmmsNode) { - if (gmmsNode == null - || gmmsNode.isMissingNode() - || gmmsNode.isNull() - || !gmmsNode.has("buckets")) { - return Optional.empty(); + private static GmmBuckets parseGmmBuckets(JsonNode gmmsNode) { + if (gmmsNode == null || gmmsNode.isMissingNode() || gmmsNode.isNull()) { + throw new FactoryException("Missing field 'gmms'"); } JsonNode bucketsNode = gmmsNode.get("buckets"); if (!bucketsNode.isArray()) { @@ -264,7 +261,7 @@ private static Optional parseGmmBuckets(JsonNode gmmsNode) { } buckets.add(new GmmBuckets.GmmBucket(List.copyOf(states))); } - return Optional.of(new GmmBuckets(List.copyOf(buckets))); + return new GmmBuckets(List.copyOf(buckets)); } private static List readDoubleArray(JsonNode node, String field) { diff --git a/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java b/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java index 945a3b11a..eb542bb4a 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/json/JsonDataSource.java @@ -10,16 +10,12 @@ import edu.ie3.datamodel.io.naming.FileNamingStrategy; import edu.ie3.datamodel.io.source.file.FileDataSource; import edu.ie3.datamodel.models.Entity; -import java.io.BufferedReader; import java.io.FileNotFoundException; -import java.io.IOException; import java.io.InputStream; -import java.io.Reader; import java.nio.file.Path; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.function.Function; import java.util.stream.Stream; /** Data source abstraction for JSON files. */ @@ -36,21 +32,6 @@ public JsonDataSource(JsonFileConnector connector, FileNamingStrategy fileNaming this.connector = connector; } - /** - * Opens a buffered reader for the provided file path. - * - * @param filePath relative path without ending - * @return buffered reader - * @throws SourceException if the file cannot be opened - */ - public BufferedReader initReader(Path filePath) throws SourceException { - try { - return connector.initReader(filePath); - } catch (FileNotFoundException e) { - throw new SourceException("Unable to open JSON file '" + filePath + "'.", e); - } - } - /** * Opens an input stream for the provided file path. * @@ -66,43 +47,6 @@ public InputStream initInputStream(Path filePath) throws SourceException { } } - /** - * Utility method that reads the entire JSON file into memory. - * - * @param filePath relative path without ending - * @return optional JSON string (empty if the file does not exist) - * @throws SourceException if reading fails - */ - public Optional readRaw(Path filePath) throws SourceException { - try (Reader reader = connector.initReader(filePath)) { - return Optional.of(readAll(reader)); - } catch (FileNotFoundException e) { - return Optional.empty(); - } catch (IOException e) { - throw new SourceException("Unable to read JSON file '" + filePath + "'.", e); - } - } - - /** - * Reads the JSON file using the provided consumer function. - * - * @param filePath relative path without ending - * @param readerFunction function that consumes the reader - * @param result type - * @return optional result (empty if file not found) - * @throws SourceException if reading fails - */ - public Optional readWith(Path filePath, Function readerFunction) - throws SourceException { - try (BufferedReader reader = connector.initReader(filePath)) { - return Optional.ofNullable(readerFunction.apply(reader)); - } catch (FileNotFoundException e) { - return Optional.empty(); - } catch (IOException e) { - throw new SourceException("Unable to read JSON file '" + filePath + "'.", e); - } - } - @Override public Optional> getSourceFields(Class entityClass) throws SourceException { @@ -129,14 +73,4 @@ private UnsupportedOperationException unsupportedTabularAccess(String method) { return new UnsupportedOperationException( "JsonDataSource does not support '" + method + "', as JSON sources are not tabular."); } - - private String readAll(Reader reader) throws IOException { - StringBuilder builder = new StringBuilder(); - char[] buffer = new char[4096]; - int read; - while ((read = reader.read(buffer)) != -1) { - builder.append(buffer, 0, read); - } - return builder.toString(); - } } From 26a9a5489886d0a56b032145655b581dce1dbcdc Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 26 Nov 2025 09:08:13 +0300 Subject: [PATCH 4/8] tests and regex --- .../markov/MarkovLoadModelFactory.java | 18 +-- .../EntityPersistenceNamingStrategy.java | 2 +- .../profile/markov/MarkovLoadModel.java | 9 +- .../connectors/JsonFileConnectorTest.groovy | 55 +++++++ .../markov/MarkovLoadModelFactoryTest.groovy | 111 ++++++++++++++ .../ie3/datamodel/io/file/FileTypeTest.groovy | 30 ++++ ...EntityPersistenceNamingStrategyTest.groovy | 25 ++++ .../io/naming/FileNamingStrategyTest.groovy | 4 +- .../io/source/json/JsonDataSourceTest.groovy | 60 ++++++++ .../json/JsonMarkovProfileSourceTest.groovy | 137 ++++++++++++++++++ 10 files changed, 434 insertions(+), 17 deletions(-) create mode 100644 src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy create mode 100644 src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy create mode 100644 src/test/groovy/edu/ie3/datamodel/io/file/FileTypeTest.groovy create mode 100644 src/test/groovy/edu/ie3/datamodel/io/source/json/JsonDataSourceTest.groovy create mode 100644 src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java index 6ebe5e178..55e906de9 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -59,17 +59,17 @@ protected List> getFields(Class entityClass) { Set requiredFields = newSet( "schema", - "generated_at", + "generatedAt", "generator.name", "generator.version", - "time_model.bucket_count", - "time_model.bucket_encoding.formula", - "time_model.sampling_interval_minutes", - "time_model.timezone", - "value_model.value_unit", - "value_model.normalization.method", - "value_model.discretization.states", - "value_model.discretization.thresholds_right", + "timeModel.bucketCount", + "timeModel.bucketEncoding.formula", + "timeModel.samplingIntervalMinutes", + "timeModel.timezone", + "valueModel.valueUnit", + "valueModel.normalization.method", + "valueModel.discretization.states", + "valueModel.discretization.thresholdsRight", "data.transitions.shape", "data.transitions.values", "data.gmms.buckets"); diff --git a/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java b/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java index 12a42e922..76818163d 100644 --- a/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java +++ b/src/main/java/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategy.java @@ -61,7 +61,7 @@ public class EntityPersistenceNamingStrategy { * profile is accessible via the named capturing group "profile", the uuid by the group "uuid" */ private static final String LOAD_PROFILE_TIME_SERIES = - "lpts_(?[a-zA-Z]{1,11}[0-9]{0,3})"; + "(?:lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})"; /** * Pattern to identify load profile time series in this instance of the naming strategy (takes diff --git a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java index 2b525a966..c791491b1 100644 --- a/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java +++ b/src/main/java/edu/ie3/datamodel/models/profile/markov/MarkovLoadModel.java @@ -59,11 +59,10 @@ public int stateCount() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof TransitionData(String dtype1, String encoding1, double[][][] values1))) - return false; - return Objects.equals(dtype, dtype1) - && Objects.equals(encoding, encoding1) - && Arrays.deepEquals(values, values1); + if (!(o instanceof TransitionData other)) return false; + return Objects.equals(dtype, other.dtype) + && Objects.equals(encoding, other.encoding) + && Arrays.deepEquals(values, other.values); } @Override diff --git a/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy new file mode 100644 index 000000000..49a3877d3 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/connectors/JsonFileConnectorTest.groovy @@ -0,0 +1,55 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.connectors + +import spock.lang.Specification + +import java.nio.file.Files +import java.nio.file.Path + +class JsonFileConnectorTest extends Specification { + + Path tempDir + + def setup() { + tempDir = Files.createTempDirectory("jsonFileConnector") + } + + def cleanup() { + if (tempDir != null) { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .forEach { Files.deleteIfExists(it) } + } + } + + def "initInputStream resolves .json ending and reads content"() { + given: + def file = tempDir.resolve("model.json") + Files.writeString(file, """{"foo":"bar"}""") + def connector = new JsonFileConnector(tempDir) + + when: + def content = connector.initInputStream(Path.of("model")).text + + then: + content == """{"foo":"bar"}""" + } + + def "initReader returns buffered reader with UTF-8 decoding"() { + given: + def file = tempDir.resolve("data.json") + Files.writeString(file, "[1,2,3]") + def connector = new JsonFileConnector(tempDir) + + when: + def reader = connector.initReader(Path.of("data")) + def line = reader.readLine() + + then: + line == "[1,2,3]" + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy new file mode 100644 index 000000000..b60d2864a --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactoryTest.groovy @@ -0,0 +1,111 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.factory.markov + +import com.fasterxml.jackson.databind.ObjectMapper +import edu.ie3.datamodel.exceptions.FactoryException +import spock.lang.Specification + +class MarkovLoadModelFactoryTest extends Specification { + private final ObjectMapper objectMapper = new ObjectMapper() + private final MarkovLoadModelFactory factory = new MarkovLoadModelFactory() + + def "buildModel returns parsed Markov load model from valid JSON"() { + given: + def root = objectMapper.readTree(validModelJson()) + + when: + def model = factory.get(new MarkovModelData(root)).getOrThrow() + + then: + model.schema() == "markov.load.v1" + model.generator().name() == "simonaMarkovLoad" + model.timeModel().bucketCount() == 1 + model.valueModel().discretization().states() == 2 + model.transitionData().bucketCount() == 1 + model.transitionData().stateCount() == 2 + model.transitionData().values()[0][0][1] == 0.9d + model.gmmBuckets().isPresent() + def gmmState = model.gmmBuckets().get().buckets().first().states().first().get() + gmmState.weights() == [0.6d] + gmmState.means() == [1.0d] + gmmState.variances() == [0.2d] + } + + def "buildModel throws FactoryException on transition dimension mismatch"() { + given: + def invalidJson = objectMapper.readTree(validModelJson().replace("\"shape\": [1,2,2]", "\"shape\": [2,2,2]")) + + when: + factory.get(new MarkovModelData(invalidJson)).getOrThrow() + + then: + thrown(FactoryException) + } + + private static String validModelJson() { + return """ + { + "schema": "markov.load.v1", + "generated_at": "2025-01-01T00:00:00Z", + "generator": { + "name": "simonaMarkovLoad", + "version": "1.0.0", + "config": { "foo": "bar" } + }, + "time_model": { + "bucket_count": 1, + "bucket_encoding": { "formula": "hour_of_day" }, + "sampling_interval_minutes": 60, + "timezone": "UTC" + }, + "value_model": { + "value_unit": "W", + "normalization": { "method": "none" }, + "discretization": { + "states": 2, + "thresholds_right": [0.5] + } + }, + "parameters": { + "transitions": { "empty_row_strategy": "fill" }, + "gmm": { + "value_col": "p", + "verbose": 1, + "heartbeat_seconds": 5 + } + }, + "data": { + "transitions": { + "dtype": "float64", + "encoding": "dense", + "shape": [1,2,2], + "values": [ + [ + [0.1, 0.9], + [0.3, 0.7] + ] + ] + }, + "gmms": { + "buckets": [ + { + "states": [ + { + "weights": [0.6], + "means": [1.0], + "variances": [0.2] + }, + null + ] + } + ] + } + } + } + """.stripIndent() + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/file/FileTypeTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/file/FileTypeTest.groovy new file mode 100644 index 000000000..ecfb744d2 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/file/FileTypeTest.groovy @@ -0,0 +1,30 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.file + +import edu.ie3.datamodel.exceptions.ParsingException +import spock.lang.Specification + +class FileTypeTest extends Specification { + + def "getFileType resolves CSV and JSON endings"() { + expect: + FileType.getFileType(fileName) == expected + + where: + fileName || expected + "data.csv" || FileType.CSV + "model.json" || FileType.JSON + } + + def "getFileType throws ParsingException on unknown ending"() { + when: + FileType.getFileType("unknown.txt") + + then: + thrown(ParsingException) + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy index 8688aced0..8011e4cb7 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/naming/EntityPersistenceNamingStrategyTest.groovy @@ -94,6 +94,19 @@ class EntityPersistenceNamingStrategyTest extends Specification { matcher.group("profile") == "g3" } + def "The pattern for a Markov load profile time series file name matches and extracts the correct profile"() { + given: + def ens = new EntityPersistenceNamingStrategy() + def validFileName = "markov_demo1" + + when: + def matcher = ens.loadProfileTimeSeriesPattern.matcher(validFileName) + + then: + matcher.matches() + matcher.group("profile") == "demo1" + } + def "Trying to extract individual time series meta information throws an Exception, if it is provided a malformed string"() { given: def ens = new EntityPersistenceNamingStrategy() @@ -120,6 +133,18 @@ class EntityPersistenceNamingStrategyTest extends Specification { ex.message == "Cannot extract meta information on load profile time series from 'foo'." } + def "loadProfileTimesSeriesMetaInformation extracts profile from Markov load profile file name"() { + given: + def ens = new EntityPersistenceNamingStrategy() + def fileName = "markov_demo2" + + when: + def meta = ens.loadProfileTimesSeriesMetaInformation(fileName) + + then: + meta.profile == "demo2" + } + def "The EntityPersistenceNamingStrategy is able to prepare the prefix properly"() { when: String actual = EntityPersistenceNamingStrategy.preparePrefix(prefix) diff --git a/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy index 3b12faef4..b68a5f166 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/naming/FileNamingStrategyTest.groovy @@ -767,7 +767,7 @@ class FileNamingStrategyTest extends Specification { def actual = strategy.loadProfileTimeSeriesPattern.pattern() then: - actual == "test_grid" + escapedFileSeparator + "input" + escapedFileSeparator + "participants" + escapedFileSeparator + "time_series" + escapedFileSeparator + "lpts_(?[a-zA-Z]{1,11}[0-9]{0,3})" + actual == "test_grid" + escapedFileSeparator + "input" + escapedFileSeparator + "participants" + escapedFileSeparator + "time_series" + escapedFileSeparator + "(?:lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})" } def "A FileNamingStrategy with FlatHierarchy returns correct individual time series file name pattern"() { @@ -789,7 +789,7 @@ class FileNamingStrategyTest extends Specification { def actual = strategy.loadProfileTimeSeriesPattern.pattern() then: - actual == "lpts_(?[a-zA-Z]{1,11}[0-9]{0,3})" + actual == "(?:lpts|markov)_(?[a-zA-Z]{1,11}[0-9]{0,3})" } def "Trying to extract time series meta information throws an Exception, if it is provided a malformed string"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonDataSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonDataSourceTest.groovy new file mode 100644 index 000000000..423c1af23 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonDataSourceTest.groovy @@ -0,0 +1,60 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.json + +import edu.ie3.datamodel.exceptions.SourceException +import edu.ie3.datamodel.io.connectors.JsonFileConnector +import edu.ie3.datamodel.io.naming.FileNamingStrategy +import edu.ie3.datamodel.models.Entity +import spock.lang.Specification + +import java.nio.file.Files +import java.nio.file.Path + +class JsonDataSourceTest extends Specification { + + Path tempDir + JsonDataSource dataSource + + def setup() { + tempDir = Files.createTempDirectory("jsonDataSource") + dataSource = new JsonDataSource(tempDir, new FileNamingStrategy()) + } + + def cleanup() { + if (tempDir != null) { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .forEach { Files.deleteIfExists(it) } + } + } + + def "initInputStream opens JSON file via connector"() { + given: + def file = tempDir.resolve("sample.json") + Files.writeString(file, """{"key":42}""") + + when: + def content = dataSource.initInputStream(Path.of("sample")).text + + then: + content == """{"key":42}""" + } + + def "tabular access methods are unsupported"() { + when: + dataSource.getSourceFields(Entity) + + then: + thrown(UnsupportedOperationException) + + when: + dataSource.getSourceData(Entity) + + then: + thrown(UnsupportedOperationException) + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy new file mode 100644 index 000000000..1b20cd92f --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/json/JsonMarkovProfileSourceTest.groovy @@ -0,0 +1,137 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.json + +import edu.ie3.datamodel.exceptions.SourceException +import edu.ie3.datamodel.io.file.FileType +import edu.ie3.datamodel.io.naming.FileNamingStrategy +import edu.ie3.datamodel.io.naming.timeseries.FileLoadProfileMetaInformation +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel +import spock.lang.Specification + +import java.nio.file.Files +import java.nio.file.Path + +class JsonMarkovProfileSourceTest extends Specification { + + Path tempDir + Path jsonFile + + def setup() { + tempDir = Files.createTempDirectory("markovProfileSource") + jsonFile = tempDir.resolve("model.json") + } + + def cleanup() { + if (tempDir != null) { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .forEach { Files.deleteIfExists(it) } + } + } + + def "getModel reads and caches Markov model from JSON file"() { + given: + Files.writeString(jsonFile, validModelJson()) + def source = new JsonMarkovProfileSource( + new JsonDataSource(tempDir, new FileNamingStrategy()), + new FileLoadProfileMetaInformation("profile1", jsonFile, FileType.JSON) + ) + + when: + MarkovLoadModel modelFirst = source.getModel() + MarkovLoadModel modelSecond = source.getModel() + + then: + modelFirst.is(modelSecond) // cached instance reused + modelFirst.schema() == "markov.load.v1" + noExceptionThrown() + + when: "validation is executed on the same file" + source.validate() + + then: + noExceptionThrown() + } + + def "getModel throws SourceException on invalid JSON file"() { + given: + Files.writeString(jsonFile, "{}") + def source = new JsonMarkovProfileSource( + new JsonDataSource(tempDir, new FileNamingStrategy()), + new FileLoadProfileMetaInformation("brokenProfile", jsonFile, FileType.JSON) + ) + + when: + source.getModel() + + then: + thrown(SourceException) + } + + private static String validModelJson() { + return """ + { + "schema": "markov.load.v1", + "generated_at": "2025-01-01T00:00:00Z", + "generator": { + "name": "simonaMarkovLoad", + "version": "1.0.0", + "config": { "foo": "bar" } + }, + "time_model": { + "bucket_count": 1, + "bucket_encoding": { "formula": "hour_of_day" }, + "sampling_interval_minutes": 60, + "timezone": "UTC" + }, + "value_model": { + "value_unit": "W", + "normalization": { "method": "none" }, + "discretization": { + "states": 2, + "thresholds_right": [0.5] + } + }, + "parameters": { + "transitions": { "empty_row_strategy": "fill" }, + "gmm": { + "value_col": "p", + "verbose": 1, + "heartbeat_seconds": 5 + } + }, + "data": { + "transitions": { + "dtype": "float64", + "encoding": "dense", + "shape": [1,2,2], + "values": [ + [ + [0.1, 0.9], + [0.3, 0.7] + ] + ] + }, + "gmms": { + "buckets": [ + { + "states": [ + { + "weights": [0.6], + "means": [1.0], + "variances": [0.2] + }, + null + ] + } + ] + } + } + } + """.stripIndent() + } +} From b84ecb3b89bc5ed3ebfe09531e1079ce32f0a4e8 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 26 Nov 2025 09:32:41 +0300 Subject: [PATCH 5/8] refactor --- .../markov/MarkovLoadModelFactory.java | 112 +++++++++++------- 1 file changed, 68 insertions(+), 44 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java index 55e906de9..9461c2ffe 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -154,13 +154,28 @@ private static TransitionData parseTransitions( String dtype = requireText(transitionsNode, "dtype"); String encoding = requireText(transitionsNode, "encoding"); + int[] shape = parseTransitionShape(transitionsNode); + int buckets = shape[0]; + int rows = shape[1]; + int columns = shape[2]; + validateTransitionShape(expectedBucketCount, stateCount, buckets, rows, columns); + + JsonNode valuesNode = requireNode(transitionsNode, "values"); + double[][][] values = parseTransitionValues(valuesNode, buckets, stateCount); + + return new TransitionData(dtype, encoding, values); + } + + private static int[] parseTransitionShape(JsonNode transitionsNode) { JsonNode shapeNode = requireNode(transitionsNode, "shape"); if (!shapeNode.isArray() || shapeNode.size() != 3) { throw new FactoryException("Transition shape must contain three dimensions"); } - int buckets = shapeNode.get(0).asInt(); - int rows = shapeNode.get(1).asInt(); - int columns = shapeNode.get(2).asInt(); + return new int[] {shapeNode.get(0).asInt(), shapeNode.get(1).asInt(), shapeNode.get(2).asInt()}; + } + + private static void validateTransitionShape( + int expectedBucketCount, int stateCount, int buckets, int rows, int columns) { if (buckets != expectedBucketCount) { throw new FactoryException( "Transition bucket count mismatch. Expected " @@ -177,61 +192,70 @@ private static TransitionData parseTransitions( + ", columns=" + columns); } + } - JsonNode valuesNode = requireNode(transitionsNode, "values"); + private static double[][][] parseTransitionValues( + JsonNode valuesNode, int buckets, int stateCount) { if (!valuesNode.isArray()) { throw new FactoryException("Transition values must be a three dimensional array"); } - double[][][] values = new double[buckets][stateCount][stateCount]; int bucketIndex = 0; for (JsonNode bucketNode : valuesNode) { - if (bucketIndex >= buckets) { - throw new FactoryException("More transition buckets present than specified in shape"); - } - int rowIndex = 0; - for (JsonNode rowNode : bucketNode) { - if (rowIndex >= stateCount) { - throw new FactoryException( - "Too many rows in transition matrix for bucket " + bucketIndex); - } - int columnIndex = 0; - for (JsonNode probNode : rowNode) { - if (columnIndex >= stateCount) { - throw new FactoryException( - "Too many columns in transition matrix for bucket " - + bucketIndex - + ", row " - + rowIndex); - } - values[bucketIndex][rowIndex][columnIndex] = probNode.asDouble(); - columnIndex++; - } - if (columnIndex != stateCount) { - throw new FactoryException( - "Row " - + rowIndex - + " in bucket " - + bucketIndex - + " had " - + columnIndex - + " columns. Expected " - + stateCount); - } - rowIndex++; - } - if (rowIndex != stateCount) { - throw new FactoryException( - "Bucket " + bucketIndex + " contained " + rowIndex + " rows. Expected " + stateCount); - } + fillBucket(values, bucketNode, bucketIndex, stateCount); bucketIndex++; } if (bucketIndex != buckets) { throw new FactoryException( "Transition values provided only " + bucketIndex + " buckets. Expected " + buckets); } + return values; + } - return new TransitionData(dtype, encoding, values); + private static void fillBucket( + double[][][] values, JsonNode bucketNode, int bucketIndex, int stateCount) { + if (bucketIndex >= values.length) { + throw new FactoryException("More transition buckets present than specified in shape"); + } + int rowIndex = 0; + for (JsonNode rowNode : bucketNode) { + fillRow(values, rowNode, bucketIndex, rowIndex, stateCount); + rowIndex++; + } + if (rowIndex != stateCount) { + throw new FactoryException( + "Bucket " + bucketIndex + " contained " + rowIndex + " rows. Expected " + stateCount); + } + } + + private static void fillRow( + double[][][] values, JsonNode rowNode, int bucketIndex, int rowIndex, int stateCount) { + if (rowIndex >= stateCount) { + throw new FactoryException("Too many rows in transition matrix for bucket " + bucketIndex); + } + int columnIndex = 0; + for (JsonNode probNode : rowNode) { + if (columnIndex >= stateCount) { + throw new FactoryException( + "Too many columns in transition matrix for bucket " + + bucketIndex + + ", row " + + rowIndex); + } + values[bucketIndex][rowIndex][columnIndex] = probNode.asDouble(); + columnIndex++; + } + if (columnIndex != stateCount) { + throw new FactoryException( + "Row " + + rowIndex + + " in bucket " + + bucketIndex + + " had " + + columnIndex + + " columns. Expected " + + stateCount); + } } private static GmmBuckets parseGmmBuckets(JsonNode gmmsNode) { From dd38436fb7e0760b50303d269d0833c8706791f0 Mon Sep 17 00:00:00 2001 From: Philipp Schmelter Date: Wed, 26 Nov 2025 09:40:58 +0300 Subject: [PATCH 6/8] refactor for sonar --- .../io/factory/markov/MarkovModelData.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java index e1cbdf3b0..ca735ac2a 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelData.java @@ -23,4 +23,18 @@ public MarkovModelData(JsonNode root) { public JsonNode getRoot() { return root; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof MarkovModelData that)) return false; + return Objects.equals(getTargetClass(), that.getTargetClass()) + && Objects.equals(getFieldsToValues(), that.getFieldsToValues()) + && Objects.equals(root, that.root); + } + + @Override + public int hashCode() { + return Objects.hash(getTargetClass(), getFieldsToValues(), root); + } } From b5a3ccd1b9443e17841f30dd5d28d20fd590cbb4 Mon Sep 17 00:00:00 2001 From: Philipp Date: Thu, 11 Dec 2025 01:53:06 +0100 Subject: [PATCH 7/8] abstraction --- .../markov/MarkovLoadModelFactory.java | 269 +--------------- .../markov/MarkovModelParsingSupport.java | 290 ++++++++++++++++++ 2 files changed, 292 insertions(+), 267 deletions(-) create mode 100644 src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java index 9461c2ffe..7249fd3e2 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovLoadModelFactory.java @@ -6,23 +6,18 @@ package edu.ie3.datamodel.io.factory.markov; import com.fasterxml.jackson.databind.JsonNode; -import edu.ie3.datamodel.exceptions.FactoryException; import edu.ie3.datamodel.io.factory.Factory; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel; import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.*; import java.time.ZonedDateTime; -import java.time.format.DateTimeParseException; -import java.util.ArrayList; -import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.TreeMap; /** Factory turning Markov JSON data into {@link MarkovLoadModel}s. */ public class MarkovLoadModelFactory - extends Factory { + extends Factory + implements MarkovModelParsingSupport { public MarkovLoadModelFactory() { super(MarkovLoadModel.class); @@ -75,264 +70,4 @@ protected List> getFields(Class entityClass) { "data.gmms.buckets"); return List.of(requiredFields); } - - private static Generator parseGenerator(JsonNode generatorNode) { - String name = requireText(generatorNode, "name"); - String version = requireText(generatorNode, "version"); - Map config = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - JsonNode configNode = generatorNode.path("config"); - if (configNode.isObject()) { - Iterator> fields = configNode.fields(); - while (fields.hasNext()) { - Map.Entry entry = fields.next(); - config.put(entry.getKey(), entry.getValue().asText()); - } - } - return new Generator(name, version, config); - } - - private static TimeModel parseTimeModel(JsonNode timeNode) { - int bucketCount = requireInt(timeNode, "bucket_count"); - String formula = requireNode(timeNode, "bucket_encoding").path("formula").asText(""); - if (formula.isEmpty()) { - throw new FactoryException("Missing bucket encoding formula"); - } - int samplingInterval = requireInt(timeNode, "sampling_interval_minutes"); - String timezone = requireText(timeNode, "timezone"); - return new TimeModel(bucketCount, formula, samplingInterval, timezone); - } - - private static ValueModel parseValueModel(JsonNode valueNode) { - String valueUnit = requireText(valueNode, "value_unit"); - JsonNode normalizationNode = requireNode(valueNode, "normalization"); - String normalizationMethod = requireText(normalizationNode, "method"); - ValueModel.Normalization normalization = new ValueModel.Normalization(normalizationMethod); - - JsonNode discretizationNode = requireNode(valueNode, "discretization"); - int states = requireInt(discretizationNode, "states"); - List thresholds = new ArrayList<>(); - JsonNode thresholdsNode = requireNode(discretizationNode, "thresholds_right"); - if (!thresholdsNode.isArray()) { - throw new FactoryException("thresholds_right must be an array"); - } - thresholdsNode.forEach(element -> thresholds.add(element.asDouble())); - ValueModel.Discretization discretization = - new ValueModel.Discretization(states, List.copyOf(thresholds)); - - return new ValueModel(valueUnit, normalization, discretization); - } - - private static Parameters parseParameters(JsonNode parametersNode) { - Parameters.TransitionParameters transitions = - new Parameters.TransitionParameters( - parametersNode.path("transitions").path("empty_row_strategy").asText("")); - if (transitions.emptyRowStrategy().isEmpty()) { - transitions = null; - } - - JsonNode gmmNode = parametersNode.path("gmm"); - Parameters.GmmParameters gmm = - gmmNode.isMissingNode() || gmmNode.isNull() || gmmNode.size() == 0 - ? null - : new Parameters.GmmParameters( - gmmNode.path("value_col").asText(""), - optionalInt(gmmNode, "verbose"), - optionalInt(gmmNode, "heartbeat_seconds")); - - return new Parameters(transitions, gmm); - } - - private static Optional optionalInt(JsonNode node, String field) { - JsonNode value = node.get(field); - if (value == null || value.isNull()) return Optional.empty(); - return Optional.of(value.asInt()); - } - - private static TransitionData parseTransitions( - JsonNode dataNode, int expectedBucketCount, int stateCount) { - JsonNode transitionsNode = requireNode(dataNode, "transitions"); - String dtype = requireText(transitionsNode, "dtype"); - String encoding = requireText(transitionsNode, "encoding"); - - int[] shape = parseTransitionShape(transitionsNode); - int buckets = shape[0]; - int rows = shape[1]; - int columns = shape[2]; - validateTransitionShape(expectedBucketCount, stateCount, buckets, rows, columns); - - JsonNode valuesNode = requireNode(transitionsNode, "values"); - double[][][] values = parseTransitionValues(valuesNode, buckets, stateCount); - - return new TransitionData(dtype, encoding, values); - } - - private static int[] parseTransitionShape(JsonNode transitionsNode) { - JsonNode shapeNode = requireNode(transitionsNode, "shape"); - if (!shapeNode.isArray() || shapeNode.size() != 3) { - throw new FactoryException("Transition shape must contain three dimensions"); - } - return new int[] {shapeNode.get(0).asInt(), shapeNode.get(1).asInt(), shapeNode.get(2).asInt()}; - } - - private static void validateTransitionShape( - int expectedBucketCount, int stateCount, int buckets, int rows, int columns) { - if (buckets != expectedBucketCount) { - throw new FactoryException( - "Transition bucket count mismatch. Expected " - + expectedBucketCount - + " but was " - + buckets); - } - if (rows != stateCount || columns != stateCount) { - throw new FactoryException( - "Transition state dimension mismatch. Expected " - + stateCount - + " but was rows=" - + rows - + ", columns=" - + columns); - } - } - - private static double[][][] parseTransitionValues( - JsonNode valuesNode, int buckets, int stateCount) { - if (!valuesNode.isArray()) { - throw new FactoryException("Transition values must be a three dimensional array"); - } - double[][][] values = new double[buckets][stateCount][stateCount]; - int bucketIndex = 0; - for (JsonNode bucketNode : valuesNode) { - fillBucket(values, bucketNode, bucketIndex, stateCount); - bucketIndex++; - } - if (bucketIndex != buckets) { - throw new FactoryException( - "Transition values provided only " + bucketIndex + " buckets. Expected " + buckets); - } - return values; - } - - private static void fillBucket( - double[][][] values, JsonNode bucketNode, int bucketIndex, int stateCount) { - if (bucketIndex >= values.length) { - throw new FactoryException("More transition buckets present than specified in shape"); - } - int rowIndex = 0; - for (JsonNode rowNode : bucketNode) { - fillRow(values, rowNode, bucketIndex, rowIndex, stateCount); - rowIndex++; - } - if (rowIndex != stateCount) { - throw new FactoryException( - "Bucket " + bucketIndex + " contained " + rowIndex + " rows. Expected " + stateCount); - } - } - - private static void fillRow( - double[][][] values, JsonNode rowNode, int bucketIndex, int rowIndex, int stateCount) { - if (rowIndex >= stateCount) { - throw new FactoryException("Too many rows in transition matrix for bucket " + bucketIndex); - } - int columnIndex = 0; - for (JsonNode probNode : rowNode) { - if (columnIndex >= stateCount) { - throw new FactoryException( - "Too many columns in transition matrix for bucket " - + bucketIndex - + ", row " - + rowIndex); - } - values[bucketIndex][rowIndex][columnIndex] = probNode.asDouble(); - columnIndex++; - } - if (columnIndex != stateCount) { - throw new FactoryException( - "Row " - + rowIndex - + " in bucket " - + bucketIndex - + " had " - + columnIndex - + " columns. Expected " - + stateCount); - } - } - - private static GmmBuckets parseGmmBuckets(JsonNode gmmsNode) { - if (gmmsNode == null || gmmsNode.isMissingNode() || gmmsNode.isNull()) { - throw new FactoryException("Missing field 'gmms'"); - } - JsonNode bucketsNode = gmmsNode.get("buckets"); - if (!bucketsNode.isArray()) { - throw new FactoryException("data.gmms.buckets must be an array"); - } - List buckets = new ArrayList<>(); - for (JsonNode bucketNode : bucketsNode) { - JsonNode statesNode = bucketNode.get("states"); - if (statesNode == null || !statesNode.isArray()) { - throw new FactoryException("Each GMM bucket must contain an array 'states'"); - } - List> states = new ArrayList<>(); - for (JsonNode stateNode : statesNode) { - if (stateNode == null || stateNode.isNull()) { - states.add(Optional.empty()); - continue; - } - List weights = readDoubleArray(stateNode, "weights"); - List means = readDoubleArray(stateNode, "means"); - List variances = readDoubleArray(stateNode, "variances"); - states.add(Optional.of(new GmmBuckets.GmmState(weights, means, variances))); - } - buckets.add(new GmmBuckets.GmmBucket(List.copyOf(states))); - } - return new GmmBuckets(List.copyOf(buckets)); - } - - private static List readDoubleArray(JsonNode node, String field) { - JsonNode arrayNode = node.get(field); - if (arrayNode == null || !arrayNode.isArray()) { - throw new FactoryException("Field '" + field + "' must be an array"); - } - List values = new ArrayList<>(); - arrayNode.forEach(element -> values.add(element.asDouble())); - return List.copyOf(values); - } - - private static JsonNode requireNode(JsonNode node, String field) { - JsonNode value = node.get(field); - if (value == null || value.isMissingNode()) { - throw new FactoryException("Missing field '" + field + "'"); - } - return value; - } - - private static String requireText(JsonNode node, String field) { - JsonNode value = node.get(field); - if (value == null || value.isMissingNode() || value.isNull()) { - throw new FactoryException("Missing field '" + field + "'"); - } - if (!value.isTextual()) { - throw new FactoryException("Field '" + field + "' must be textual"); - } - return value.asText(); - } - - private static int requireInt(JsonNode node, String field) { - JsonNode value = node.get(field); - if (value == null || value.isMissingNode() || value.isNull()) { - throw new FactoryException("Missing field '" + field + "'"); - } - if (!value.canConvertToInt()) { - throw new FactoryException("Field '" + field + "' must be an integer"); - } - return value.asInt(); - } - - private static ZonedDateTime parseTimestamp(String timestamp) { - try { - return ZonedDateTime.parse(timestamp); - } catch (DateTimeParseException e) { - throw new FactoryException("Unable to parse generated_at timestamp '" + timestamp + "'", e); - } - } } diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java new file mode 100644 index 000000000..db593f0f3 --- /dev/null +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -0,0 +1,290 @@ +/* + * ЖИ 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.factory.markov; + +import com.fasterxml.jackson.databind.JsonNode; +import edu.ie3.datamodel.exceptions.FactoryException; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.Generator; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.GmmBuckets; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.Parameters; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.TimeModel; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.TransitionData; +import edu.ie3.datamodel.models.profile.markov.MarkovLoadModel.ValueModel; +import java.time.ZonedDateTime; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; + +/** + * Shared parsing helpers for Markov model JSON documents. This is intentionally package-private as + * it is only meant to be reused across factory implementations in this package. + */ +interface MarkovModelParsingSupport { + + default Generator parseGenerator(JsonNode generatorNode) { + String name = requireText(generatorNode, "name"); + String version = requireText(generatorNode, "version"); + Map config = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + JsonNode configNode = generatorNode.path("config"); + if (configNode.isObject()) { + Iterator> fields = configNode.fields(); + while (fields.hasNext()) { + Map.Entry entry = fields.next(); + config.put(entry.getKey(), entry.getValue().asText()); + } + } + return new Generator(name, version, config); + } + + default TimeModel parseTimeModel(JsonNode timeNode) { + int bucketCount = requireInt(timeNode, "bucket_count"); + String formula = requireNode(timeNode, "bucket_encoding").path("formula").asText(""); + if (formula.isEmpty()) { + throw new FactoryException("Missing bucket encoding formula"); + } + int samplingInterval = requireInt(timeNode, "sampling_interval_minutes"); + String timezone = requireText(timeNode, "timezone"); + return new TimeModel(bucketCount, formula, samplingInterval, timezone); + } + + default ValueModel parseValueModel(JsonNode valueNode) { + String valueUnit = requireText(valueNode, "value_unit"); + JsonNode normalizationNode = requireNode(valueNode, "normalization"); + String normalizationMethod = requireText(normalizationNode, "method"); + ValueModel.Normalization normalization = new ValueModel.Normalization(normalizationMethod); + + JsonNode discretizationNode = requireNode(valueNode, "discretization"); + int states = requireInt(discretizationNode, "states"); + List thresholds = new ArrayList<>(); + JsonNode thresholdsNode = requireNode(discretizationNode, "thresholds_right"); + if (!thresholdsNode.isArray()) { + throw new FactoryException("thresholds_right must be an array"); + } + thresholdsNode.forEach(element -> thresholds.add(element.asDouble())); + ValueModel.Discretization discretization = + new ValueModel.Discretization(states, List.copyOf(thresholds)); + + return new ValueModel(valueUnit, normalization, discretization); + } + + default Parameters parseParameters(JsonNode parametersNode) { + Parameters.TransitionParameters transitions = + new Parameters.TransitionParameters( + parametersNode.path("transitions").path("empty_row_strategy").asText("")); + if (transitions.emptyRowStrategy().isEmpty()) { + transitions = null; + } + + JsonNode gmmNode = parametersNode.path("gmm"); + Parameters.GmmParameters gmm = + gmmNode.isMissingNode() || gmmNode.isNull() || gmmNode.size() == 0 + ? null + : new Parameters.GmmParameters( + gmmNode.path("value_col").asText(""), + optionalInt(gmmNode, "verbose"), + optionalInt(gmmNode, "heartbeat_seconds")); + + return new Parameters(transitions, gmm); + } + + default TransitionData parseTransitions( + JsonNode dataNode, int expectedBucketCount, int stateCount) { + JsonNode transitionsNode = requireNode(dataNode, "transitions"); + String dtype = requireText(transitionsNode, "dtype"); + String encoding = requireText(transitionsNode, "encoding"); + + int[] shape = parseTransitionShape(transitionsNode); + int buckets = shape[0]; + int rows = shape[1]; + int columns = shape[2]; + validateTransitionShape(expectedBucketCount, stateCount, buckets, rows, columns); + + JsonNode valuesNode = requireNode(transitionsNode, "values"); + double[][][] values = parseTransitionValues(valuesNode, buckets, stateCount); + + return new TransitionData(dtype, encoding, values); + } + + default GmmBuckets parseGmmBuckets(JsonNode gmmsNode) { + if (gmmsNode == null || gmmsNode.isMissingNode() || gmmsNode.isNull()) { + throw new FactoryException("Missing field 'gmms'"); + } + JsonNode bucketsNode = gmmsNode.get("buckets"); + if (!bucketsNode.isArray()) { + throw new FactoryException("data.gmms.buckets must be an array"); + } + List buckets = new ArrayList<>(); + for (JsonNode bucketNode : bucketsNode) { + JsonNode statesNode = bucketNode.get("states"); + if (statesNode == null || !statesNode.isArray()) { + throw new FactoryException("Each GMM bucket must contain an array 'states'"); + } + List> states = new ArrayList<>(); + for (JsonNode stateNode : statesNode) { + if (stateNode == null || stateNode.isNull()) { + states.add(Optional.empty()); + continue; + } + List weights = readDoubleArray(stateNode, "weights"); + List means = readDoubleArray(stateNode, "means"); + List variances = readDoubleArray(stateNode, "variances"); + states.add(Optional.of(new GmmBuckets.GmmState(weights, means, variances))); + } + buckets.add(new GmmBuckets.GmmBucket(List.copyOf(states))); + } + return new GmmBuckets(List.copyOf(buckets)); + } + + default JsonNode requireNode(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode()) { + throw new FactoryException("Missing field '" + field + "'"); + } + return value; + } + + default String requireText(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode() || value.isNull()) { + throw new FactoryException("Missing field '" + field + "'"); + } + if (!value.isTextual()) { + throw new FactoryException("Field '" + field + "' must be textual"); + } + return value.asText(); + } + + default int requireInt(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isMissingNode() || value.isNull()) { + throw new FactoryException("Missing field '" + field + "'"); + } + if (!value.canConvertToInt()) { + throw new FactoryException("Field '" + field + "' must be an integer"); + } + return value.asInt(); + } + + default ZonedDateTime parseTimestamp(String timestamp) { + try { + return ZonedDateTime.parse(timestamp); + } catch (DateTimeParseException e) { + throw new FactoryException("Unable to parse generated_at timestamp '" + timestamp + "'", e); + } + } + + default Optional optionalInt(JsonNode node, String field) { + JsonNode value = node.get(field); + if (value == null || value.isNull()) return Optional.empty(); + return Optional.of(value.asInt()); + } + + default int[] parseTransitionShape(JsonNode transitionsNode) { + JsonNode shapeNode = requireNode(transitionsNode, "shape"); + if (!shapeNode.isArray() || shapeNode.size() != 3) { + throw new FactoryException("Transition shape must contain three dimensions"); + } + return new int[] {shapeNode.get(0).asInt(), shapeNode.get(1).asInt(), shapeNode.get(2).asInt()}; + } + + default void validateTransitionShape( + int expectedBucketCount, int stateCount, int buckets, int rows, int columns) { + if (buckets != expectedBucketCount) { + throw new FactoryException( + "Transition bucket count mismatch. Expected " + + expectedBucketCount + + " but was " + + buckets); + } + if (rows != stateCount || columns != stateCount) { + throw new FactoryException( + "Transition state dimension mismatch. Expected " + + stateCount + + " but was rows=" + + rows + + ", columns=" + + columns); + } + } + + default double[][][] parseTransitionValues( + JsonNode valuesNode, int buckets, int stateCount) { + if (!valuesNode.isArray()) { + throw new FactoryException("Transition values must be a three dimensional array"); + } + double[][][] values = new double[buckets][stateCount][stateCount]; + int bucketIndex = 0; + for (JsonNode bucketNode : valuesNode) { + fillBucket(values, bucketNode, bucketIndex, stateCount); + bucketIndex++; + } + if (bucketIndex != buckets) { + throw new FactoryException( + "Transition values provided only " + bucketIndex + " buckets. Expected " + buckets); + } + return values; + } + + default void fillBucket( + double[][][] values, JsonNode bucketNode, int bucketIndex, int stateCount) { + if (bucketIndex >= values.length) { + throw new FactoryException("More transition buckets present than specified in shape"); + } + int rowIndex = 0; + for (JsonNode rowNode : bucketNode) { + fillRow(values, rowNode, bucketIndex, rowIndex, stateCount); + rowIndex++; + } + if (rowIndex != stateCount) { + throw new FactoryException( + "Bucket " + bucketIndex + " contained " + rowIndex + " rows. Expected " + stateCount); + } + } + + default void fillRow( + double[][][] values, JsonNode rowNode, int bucketIndex, int rowIndex, int stateCount) { + if (rowIndex >= stateCount) { + throw new FactoryException("Too many rows in transition matrix for bucket " + bucketIndex); + } + int columnIndex = 0; + for (JsonNode probNode : rowNode) { + if (columnIndex >= stateCount) { + throw new FactoryException( + "Too many columns in transition matrix for bucket " + + bucketIndex + + ", row " + + rowIndex); + } + values[bucketIndex][rowIndex][columnIndex] = probNode.asDouble(); + columnIndex++; + } + if (columnIndex != stateCount) { + throw new FactoryException( + "Row " + + rowIndex + + " in bucket " + + bucketIndex + + " had " + + columnIndex + + " columns. Expected " + + stateCount); + } + } + + default List readDoubleArray(JsonNode node, String field) { + JsonNode arrayNode = node.get(field); + if (arrayNode == null || !arrayNode.isArray()) { + throw new FactoryException("Field '" + field + "' must be an array"); + } + List values = new ArrayList<>(); + arrayNode.forEach(element -> values.add(element.asDouble())); + return List.copyOf(values); + } +} From fbf01ee3b07e7deda2fb1dd08c74bd3102d240e2 Mon Sep 17 00:00:00 2001 From: Philipp Date: Thu, 11 Dec 2025 02:28:31 +0100 Subject: [PATCH 8/8] spotless --- .../io/factory/markov/MarkovModelParsingSupport.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java index db593f0f3..fa360374e 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/markov/MarkovModelParsingSupport.java @@ -1,8 +1,8 @@ /* - * ЖИ 2025. TU Dortmund University, + * © 2025. TU Dortmund University, * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation - */ +*/ package edu.ie3.datamodel.io.factory.markov; import com.fasterxml.jackson.databind.JsonNode; @@ -214,8 +214,7 @@ default void validateTransitionShape( } } - default double[][][] parseTransitionValues( - JsonNode valuesNode, int buckets, int stateCount) { + default double[][][] parseTransitionValues(JsonNode valuesNode, int buckets, int stateCount) { if (!valuesNode.isArray()) { throw new FactoryException("Transition values must be a three dimensional array"); }