From 0fe5b0a7b6e24df8a82802332f86ff7328bab901 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Wed, 8 May 2024 17:51:43 +0200 Subject: [PATCH 1/6] Preventing unconnected nodes or subgrids. --- .../edu/ie3/osmogrid/graph/OsmGraph.scala | 17 ++ .../osmogrid/lv/LvGridGeneratorSupport.scala | 232 ++++-------------- 2 files changed, 70 insertions(+), 179 deletions(-) diff --git a/src/main/scala/edu/ie3/osmogrid/graph/OsmGraph.scala b/src/main/scala/edu/ie3/osmogrid/graph/OsmGraph.scala index 718cbcd9..18c11eb9 100644 --- a/src/main/scala/edu/ie3/osmogrid/graph/OsmGraph.scala +++ b/src/main/scala/edu/ie3/osmogrid/graph/OsmGraph.scala @@ -119,6 +119,23 @@ class OsmGraph( } } + /** Returns the other node of an edge. + * + * @param source + * the source node + * @param edge + * the considered edge + * @return + * the node on the other side of the edge + */ + def getOtherEdgeNode( + source: Node, + edge: DistanceWeightedEdge + ): Node = { + if (getEdgeSource(edge) == source) getEdgeTarget(edge) + else getEdgeSource(edge) + } + /** Returns true if at least two edges of this graph intersects each other. */ def containsEdgeIntersection(): Boolean = { diff --git a/src/main/scala/edu/ie3/osmogrid/lv/LvGridGeneratorSupport.scala b/src/main/scala/edu/ie3/osmogrid/lv/LvGridGeneratorSupport.scala index 6255f9aa..18cad520 100644 --- a/src/main/scala/edu/ie3/osmogrid/lv/LvGridGeneratorSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/lv/LvGridGeneratorSupport.scala @@ -7,7 +7,6 @@ package edu.ie3.osmogrid.lv import com.typesafe.scalalogging.LazyLogging -import edu.ie3.datamodel.graph.DistanceWeightedEdge import edu.ie3.datamodel.models.input.NodeInput import edu.ie3.datamodel.models.input.connector.LineInput import edu.ie3.datamodel.models.input.connector.`type`.{ @@ -16,21 +15,16 @@ import edu.ie3.datamodel.models.input.connector.`type`.{ } import edu.ie3.datamodel.models.input.container.SubGridContainer import edu.ie3.datamodel.models.input.system.LoadInput -import edu.ie3.datamodel.models.voltagelevels.{ - GermanVoltageLevelUtils, - VoltageLevel -} +import edu.ie3.datamodel.models.voltagelevels.VoltageLevel import edu.ie3.osmogrid.exception.IllegalStateException import edu.ie3.osmogrid.graph.OsmGraph import edu.ie3.osmogrid.lv.LvGraphGeneratorSupport.BuildingGraphConnection import edu.ie3.util.osm.model.OsmEntity.Node -import tech.units.indriya.ComparableQuantity +import edu.ie3.util.quantities.QuantityUtils._ import utils.Clustering import utils.Clustering.Cluster import utils.GridConversion._ -import javax.measure.quantity.ElectricPotential -import scala.annotation.tailrec import scala.collection.Set import scala.collection.parallel.{ParMap, ParSeq} import scala.jdk.CollectionConverters._ @@ -165,27 +159,23 @@ object LvGridGeneratorSupport extends LazyLogging { if (gridElements.loads.isEmpty) { return List.empty } - val (startNode, startNodeInput) = gridElements.nodes.headOption.getOrElse( - throw new IllegalArgumentException( - "We have no electrical nodes to convert." - ) - ) - val (visitedNodes, lineInputs) = traverseGraph( - startNode, - startNodeInput, - osmGraph, - Set.empty, - Set.empty, - gridElements.nodes ++ gridElements.substations, - lineType - ) - val unvisitedNodes = osmGraph.vertexSet().asScala.diff(visitedNodes) + val nodeToNodeInput = gridElements.nodes ++ gridElements.substations + val reducedGraph = reduceGraph(osmGraph, nodeToNodeInput.keySet) - // todo: happens with connected graphs with size of 1 - if (unvisitedNodes.nonEmpty) { - logger.error( - "We did not visit all nodes while traversing the graph. Unvisited Nodes: " + unvisitedNodes + val lineInputs = reducedGraph.edgeSet().asScala.map { edge => + val source = reducedGraph.getEdgeSource(edge) + val target = reducedGraph.getEdgeTarget(edge) + val nodeA = nodeToNodeInput(source) + val nodeB = nodeToNodeInput(target) + + buildLine( + s"Line between: ${nodeA.getId}-${nodeB.getId}", + nodeA, + nodeB, + 1, + lineType, + edge.getDistance ) } @@ -266,166 +256,50 @@ object LvGridGeneratorSupport extends LazyLogging { } } - /** Recursively traverses the graph by starting at a given node and follows - * all connected edges sequentially, building all lines between nodes in the - * process. + /** This method will reduce the graph by removing some vertices. A vertex is + * removed if its degree is < 2 and it is not defined to be kept. * - * @param currentNode - * the osm node at which we start - * @param currentNodeInput - * the node input associated with the osm node at which we start * @param osmGraph - * the osm graph wich we traverse - * @param alreadyVisited - * nodes we have already visited - * @param lines - * lines we have already built - * @param nodeToNodeInput - * a mapping from osm node to node input - * @param lineTypeInput - * the line type we use for building lines + * graph to reduce + * @param keep + * all [[Node]]s that should be kept * @return - * a tuple of the set of already visited nodes and lines we have built */ - private def traverseGraph( - currentNode: Node, - currentNodeInput: NodeInput, + private def reduceGraph( osmGraph: OsmGraph, - alreadyVisited: Set[Node], - lines: Set[LineInput], - nodeToNodeInput: Map[Node, NodeInput], - lineTypeInput: LineTypeInput - ): (Set[Node], Set[LineInput]) = { - if (alreadyVisited.contains(currentNode)) return (alreadyVisited, lines) - val connectedEdges = osmGraph.edgesOf(currentNode).asScala - // traverse through every edge of the current node to build lines - connectedEdges.foldLeft((alreadyVisited ++ Seq(currentNode), lines)) { - case ((updatedAlreadyVisited, updatedLines), edge) => - val nextNode = getOtherEdgeNode(osmGraph, currentNode, edge) - if (!alreadyVisited.contains(nextNode)) { - // follow the edge along until the next node input is found if there is any - val (maybeNextNodeInput, maybeNextNode, passedStreetNodes) = - findNextNodeInput( - osmGraph, - nextNode, - edge, - updatedAlreadyVisited ++ Seq(currentNode), - nodeToNodeInput - ) - maybeNextNodeInput.zip(maybeNextNode) match { - // if a node input is found along the edge we build a line - case Some((nextNodeInput, nextNode)) => - val newLine = - buildLine( - currentNodeInput, - nextNodeInput, - // for building the line we want to consider the whole street section we went along - currentNode +: passedStreetNodes :+ nextNode, - lineTypeInput - ) - val (visitedNodes, builtLines) = traverseGraph( - nextNode, - nextNodeInput, - osmGraph, - alreadyVisited ++ passedStreetNodes ++ Seq(currentNode), - lines ++ Seq(newLine), - nodeToNodeInput, - lineTypeInput - ) - ( - updatedAlreadyVisited ++ visitedNodes, - updatedLines ++ builtLines - ) - // if there is no more node input along the edge we are done with this branch of the graph - case None => - ( - updatedAlreadyVisited ++ passedStreetNodes ++ Seq(currentNode), - updatedLines - ) + keep: Set[Node] + ): OsmGraph = { + osmGraph + .vertexSet() + .asScala + .diff(keep) + .foldLeft(osmGraph) { case (graph, currentNode) => + if (graph.degreeOf(currentNode) <= 2) { + // the current node can be removed + val edges = graph.edgesOf(currentNode).asScala + + if (edges.size != 1) { + edges.headOption.zip(edges.lastOption).foreach { + case (edgeA, edgeB) => + val source = graph.getOtherEdgeNode(currentNode, edgeA) + val target = graph.getOtherEdgeNode(currentNode, edgeB) + + if (source != target) { + val distance = edgeA.getDistance + .add(edgeB.getDistance) + .getValue + .doubleValue() + .asMetre + + graph.addWeightedEdge(source, target, distance) + } + } } - } else { - // we've already been at this node before so we are done on this branch of the graph - (updatedAlreadyVisited, updatedLines) - } - } - } - /** Looks for the next [[Node]] that has an associated [[NodeInput]]. Returns - * all passed [[Node]]s we passed during the search. Includes the node at - * which we found a [[NodeInput]]. - * - * @param graph - * graph we traverse - * @param currentNode - * node at which we start - * @param lastEdge - * edge from where we came - * @param alreadyVisited - * nodes we already visited - * @param nodeToNodeInput - * map of all created [[NodeInput]]s - * @param passedNodes - * nodes we passed while traversing - * @return - * An optional of the found [[NodeInput]], an optional of the associated - * [[Node]] and all [[Node]]s we looked at while traversin - */ - @tailrec - private def findNextNodeInput( - graph: OsmGraph, - currentNode: Node, - lastEdge: DistanceWeightedEdge, - alreadyVisited: Set[Node], - nodeToNodeInput: Map[Node, NodeInput], - passedNodes: Seq[Node] = Seq.empty - ): (Option[NodeInput], Option[Node], Seq[Node]) = { - if (alreadyVisited.contains(currentNode)) - return (None, None, passedNodes) - nodeToNodeInput.get(currentNode) match { - case Some(nodeInput) => - (Some(nodeInput), Some(currentNode), passedNodes) - case None => - graph.edgesOf(currentNode).asScala.filter(_ != lastEdge).toSeq match { - case Seq(nextEdge) => - val nextNode = getOtherEdgeNode(graph, currentNode, nextEdge) - findNextNodeInput( - graph, - nextNode, - nextEdge, - alreadyVisited ++ Seq(currentNode), - nodeToNodeInput, - passedNodes :+ currentNode - ) - case Nil => - // this means we arrived at a dead end at which no node is connected - (None, None, passedNodes :+ currentNode) - case seq => - throw IllegalStateException( - s"Node: $currentNode has no associated node input but more than two associated edges. Edges: $seq" - ) + graph.removeVertex(currentNode) } - } - } - - /** Returns the other node of an edge. - * - * @param graph - * the graph - * @param source - * the source node - * @param edge - * the considered edge - * @return - * the node on the other side of the edge - */ - private def getOtherEdgeNode( - graph: OsmGraph, - source: Node, - edge: DistanceWeightedEdge - ): Node = { - if (graph.getEdgeSource(edge) == source) graph.getEdgeTarget(edge) - else graph.getEdgeSource(edge) + graph + } } - } From 6056dff9d97fd4f9a070f9837b84050af40b7ce5 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Wed, 8 May 2024 17:54:22 +0200 Subject: [PATCH 2/6] Updating `CHANGELOG`. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e72b85b2..97ac7e74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - LV Coordinator dies unexpectedly [#361](https://github.com/ie3-institute/OSMoGrid/issues/361) - Some bugs fixed [#405](https://github.com/ie3-institute/OSMoGrid/issues/405) - Fixed number of parallel lines from zero to one [#419](https://github.com/ie3-institute/OSMoGrid/issues/419) +- Preventing unconnected nodes or subgrids [#415](https://github.com/ie3-institute/OSMoGrid/issues/415) ### Removed - Legacy Java code From 28fc06548ef833349549324f2d5e4deeee3e5103 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Fri, 10 May 2024 13:26:18 +0200 Subject: [PATCH 3/6] Fixing some issues in `Clustering`. --- .../osmogrid/exception/ClusterException.scala | 12 ++ .../osmogrid/lv/LvGridGeneratorSupport.scala | 4 +- src/main/scala/utils/Clustering.scala | 118 ++++++++++-------- src/main/scala/utils/Connections.scala | 6 +- .../ie3/osmogrid/utils/ClusteringSpec.scala | 27 ++-- 5 files changed, 99 insertions(+), 68 deletions(-) create mode 100644 src/main/scala/edu/ie3/osmogrid/exception/ClusterException.scala diff --git a/src/main/scala/edu/ie3/osmogrid/exception/ClusterException.scala b/src/main/scala/edu/ie3/osmogrid/exception/ClusterException.scala new file mode 100644 index 00000000..fe76d1d0 --- /dev/null +++ b/src/main/scala/edu/ie3/osmogrid/exception/ClusterException.scala @@ -0,0 +1,12 @@ +/* + * © 2024. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ + +package edu.ie3.osmogrid.exception + +final case class ClusterException( + msg: String = "Error during clustering of lv grids.", + cause: Throwable = None.orNull +) extends Exception(msg, cause) diff --git a/src/main/scala/edu/ie3/osmogrid/lv/LvGridGeneratorSupport.scala b/src/main/scala/edu/ie3/osmogrid/lv/LvGridGeneratorSupport.scala index 18cad520..38ffa6dd 100644 --- a/src/main/scala/edu/ie3/osmogrid/lv/LvGridGeneratorSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/lv/LvGridGeneratorSupport.scala @@ -226,7 +226,7 @@ object LvGridGeneratorSupport extends LazyLogging { // converting the cluster into an actual psdm subgrid cluster.map { c => val substation = c.substation - val nodes = c.nodes.toSet + substation + val nodes = c.nodes ++ Set(substation) val lines: Map[(NodeInput, NodeInput), LineInput] = lineMap.filter { case ((nodeA, nodeB), _) => nodes.contains(nodeA) && nodes.contains(nodeB) @@ -245,7 +245,7 @@ object LvGridGeneratorSupport extends LazyLogging { val transformer2W = buildTransformer2W(mvNode, substation, 1, transformer2WTypeInput) - val allNodes: Set[NodeInput] = nodes + mvNode + val allNodes = nodes ++ Set(mvNode) buildGridContainer( gridNameBase, diff --git a/src/main/scala/utils/Clustering.scala b/src/main/scala/utils/Clustering.scala index 3be4bd03..f190e42f 100644 --- a/src/main/scala/utils/Clustering.scala +++ b/src/main/scala/utils/Clustering.scala @@ -11,6 +11,7 @@ import edu.ie3.datamodel.models.input.NodeInput import edu.ie3.datamodel.models.input.connector.LineInput import edu.ie3.datamodel.models.input.connector.`type`.Transformer2WTypeInput import edu.ie3.datamodel.models.input.system.LoadInput +import edu.ie3.osmogrid.exception.ClusterException import edu.ie3.osmogrid.lv.LvGridGeneratorSupport.GridElements import tech.units.indriya.ComparableQuantity import tech.units.indriya.quantity.Quantities @@ -18,12 +19,14 @@ import utils.Clustering.{Cluster, isImprovement, totalDistance} import java.util.concurrent.ThreadLocalRandom import javax.measure.quantity.Power +import scala.collection.parallel.CollectionConverters.ImmutableIterableIsParallelizable final case class Clustering( connections: Connections[NodeInput], - osmSubstations: List[NodeInput], - additionalSubstations: List[NodeInput], - nodes: List[NodeInput] + osmSubstations: Set[NodeInput], + additionalSubstations: Set[NodeInput], + nodes: Set[NodeInput], + nodeSize: Int ) { /** Method to run the algorithm. This algorithm is based on PAM (Partitioning @@ -61,31 +64,31 @@ final case class Clustering( } /** Calculates the best next step. - * @param list + * @param clusters * last step * @return * an option */ - private def calculateStep(list: List[Cluster]): Option[List[Cluster]] = { - // list of changeable substations - val changeable: List[NodeInput] = - list.filter { l => !osmSubstations.contains(l.substation) }.map { c => - c.substation - } - + private def calculateStep(clusters: List[Cluster]): Option[List[Cluster]] = { // calculates all swaps and returns the best as an option - changeable - .flatMap { s => - nodes.map { n => + additionalSubstations.par + .flatMap { substation => + nodes.map { node => + val currentSubstation = Set(substation) + val currentNode = Set(node) + // swaps two nodes - val substations = changeable.filter { c => c != s } :+ n - val other = nodes.filter { c => c != n } :+ s + val updatedSubstations = + additionalSubstations -- currentSubstation ++ currentNode + val updatedNodes = nodes -- currentNode ++ currentSubstation - val clusters = createClusters(substations :++ osmSubstations, other) + val clusters = + createClusters(updatedSubstations ++ osmSubstations, updatedNodes) (clusters, totalDistance(clusters)) } } + .seq .minByOption(_._2) .map(_._1) } @@ -99,31 +102,41 @@ final case class Clustering( * a list of [[Cluster]] */ private def createClusters( - substations: List[NodeInput], - nodes: List[NodeInput] + substations: Set[NodeInput], + nodes: Set[NodeInput] ): List[Cluster] = { - nodes - .map { n => - val closest = substations - .map { s => s -> connections.getDistance(n, s) } - .sortBy(_._2) - .toList(0) - ._1 - - (n, closest) - } - .groupMap(_._2)(_._1) - .map { case (s, list) => - list - .flatMap { n => - connections.getDistance(s, n).map(_.getValue.doubleValue()) + if (substations.size + nodes.size != nodeSize) { + List.empty + } else { + nodes + .map { n => + val closest = substations + .map { s => s -> connections.getDistance(n, s) } + .minByOption(_._2) + .map(_._1) + .getOrElse( + throw ClusterException(s"No substation found for node: $n") + ) + + (n, closest) + } + .groupMap(_._2)(_._1) + .map { case (substation, connectedNodes) => + // calculate all distances + val distances = connectedNodes.map(node => + connections + .getDistance(substation, node) + .map(_.getValue.doubleValue()) + .getOrElse(Double.MaxValue) + ) + + distances.reduceOption { (a, b) => a + b } match { + case Some(distance) => Cluster(substation, connectedNodes, distance) + case None => Cluster(substation, connectedNodes, Double.MaxValue) } - .reduceOption { (a, b) => a + b } match { - case Some(value) => Cluster(s, list, value) - case None => Cluster(s, list, Double.MaxValue) } - } - .toList + .toList + } } } @@ -142,11 +155,11 @@ object Clustering { val connections: Connections[NodeInput] = Connections(gridElements, lines.toSeq) - val osmSubstations = gridElements.substations.values.toList + val osmSubstations = gridElements.substations.values.toSet val additionalSubstationCount = substationCount - osmSubstations.size - val additionalSubstations: List[NodeInput] = + val additionalSubstations: Set[NodeInput] = if (additionalSubstationCount > 0) { val maxNr = gridElements.nodes.size val nodes = gridElements.nodes.values.toList @@ -155,16 +168,17 @@ object Clustering { Range .Int(0, additionalSubstationCount, 1) .map { _ => nodes(ThreadLocalRandom.current().nextInt(0, maxNr)) } - .toList + .toSet } else { - List.empty[NodeInput] + Set.empty[NodeInput] } Clustering( connections, osmSubstations, additionalSubstations, - gridElements.nodes.values.toList.diff(additionalSubstations) + gridElements.nodes.values.toSet.diff(additionalSubstations), + gridElements.nodes.size + gridElements.substations.size ) } @@ -208,7 +222,7 @@ object Clustering { } } - /** Checks if there are still improvements + /** Checks if there are still improvements greater than 1 %. * * @param old * clusters @@ -220,7 +234,7 @@ object Clustering { def isImprovement( old: List[Cluster], current: List[Cluster] - ): Boolean = totalDistance(current) <= totalDistance(old) * 0.95 + ): Boolean = totalDistance(current) <= totalDistance(old) * 0.99 /** Calculates the total connection distance of a list of [[Cluster]]s. * @@ -230,15 +244,19 @@ object Clustering { * either the total distance or [[Double.MaxValue]] */ private def totalDistance(list: List[Cluster]): Double = { - list.map { l => l.distances }.reduceOption { (a, b) => a + b } match { - case Some(value) => value - case None => Double.MaxValue + if (list.isEmpty) { + Double.MaxValue + } else { + list.map { l => l.distances }.reduceOption { (a, b) => a + b } match { + case Some(value) => value + case None => Double.MaxValue + } } } final case class Cluster( substation: NodeInput, - nodes: List[NodeInput], + nodes: Set[NodeInput], distances: Double ) } diff --git a/src/main/scala/utils/Connections.scala b/src/main/scala/utils/Connections.scala index 1ca85b63..915657c8 100644 --- a/src/main/scala/utils/Connections.scala +++ b/src/main/scala/utils/Connections.scala @@ -129,8 +129,8 @@ object Connections { val graph: DistanceWeightedGraph = new DistanceWeightedGraph() // adding all nodes to the graph - val nodes: List[NodeInput] = - (elements.nodes.values ++ elements.substations.values).toList + val nodes: Set[NodeInput] = + (elements.nodes.values ++ elements.substations.values).toSet nodes.foreach { n => graph.addVertex(n) } lines.foreach { line => @@ -143,7 +143,7 @@ object Connections { val connectionList: List[Connection[NodeInput]] = buildUndirectedShortestPathConnections(graph, shortestPath) - Connections(nodes, connectionList) + Connections(nodes.toList, connectionList) } def apply[T]( diff --git a/src/test/scala/edu/ie3/osmogrid/utils/ClusteringSpec.scala b/src/test/scala/edu/ie3/osmogrid/utils/ClusteringSpec.scala index 2fb632a3..20b24b91 100644 --- a/src/test/scala/edu/ie3/osmogrid/utils/ClusteringSpec.scala +++ b/src/test/scala/edu/ie3/osmogrid/utils/ClusteringSpec.scala @@ -49,7 +49,7 @@ class ClusteringSpec extends UnitSpec with ClusterTestData { ) clustering.osmSubstations should contain allOf (p2_1, p1_1) - clustering.additionalSubstations shouldBe List.empty + clustering.additionalSubstations shouldBe Set.empty clustering.nodes should contain only (p2_4, p2_2, p1_3, p1_2, p2_3, p1_4) } @@ -58,8 +58,8 @@ class ClusteringSpec extends UnitSpec with ClusterTestData { val clustering = Clustering.setup(elements, lines, trafo_10kV_to_lv, 0.5) val clusters = clustering invokePrivate createClusters( - elements.substations.values.toList, - elements.nodes.values.toList + elements.substations.values.toSet, + elements.nodes.values.toSet ) val map = clusters.map { c => c.substation -> c }.toMap @@ -69,15 +69,15 @@ class ClusteringSpec extends UnitSpec with ClusterTestData { } "check for improvements correctly" in { - val old: List[Cluster] = List(Cluster(p1_1, List(p1_2), 500)) + val old: List[Cluster] = List(Cluster(p1_1, Set(p1_2), 500)) Clustering.isImprovement( old, - List(Cluster(p1_1, List(p1_2), 450)) + List(Cluster(p1_1, Set(p1_2), 450)) ) shouldBe true Clustering.isImprovement( old, - List(Cluster(p1_1, List(p1_2), 490)) + List(Cluster(p1_1, Set(p1_2), 496)) // improvement is less than 1 % ) shouldBe false } @@ -85,9 +85,9 @@ class ClusteringSpec extends UnitSpec with ClusterTestData { val totalDistance = PrivateMethod[Double](Symbol("totalDistance")) val clusters = List( - Cluster(p1_1, List(p1_2), 500), - Cluster(p2_1, List(p2_2), 200), - Cluster(p1_3, List(p1_4), 300) + Cluster(p1_1, Set(p1_2), 500), + Cluster(p2_1, Set(p2_2), 200), + Cluster(p1_3, Set(p1_4), 300) ) Clustering invokePrivate totalDistance(clusters) shouldBe 1000 @@ -99,17 +99,18 @@ class ClusteringSpec extends UnitSpec with ClusterTestData { val connections = Connections(elements, lines.toSeq) // suboptimal additional substation found - val additionalSubstation = List(p1_2) + val additionalSubstation = Set(p1_2) val clustering = Clustering( connections, - List(p1_1), + Set(p1_1), additionalSubstation, - elements.nodes.values.toList + elements.nodes.values.toSet, + 8 ) val initialClusters = clustering invokePrivate createClusters( - List(p1_1, p1_2), + Set(p1_1, p1_2), clustering.nodes ) val firstStep = clustering invokePrivate calculateStep(initialClusters) From 2f43187247353ec2d36784d21e8422e744f8a45e Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Thu, 16 May 2024 12:57:54 +0200 Subject: [PATCH 4/6] Fixing some issues in `Clustering`. --- src/main/scala/utils/Clustering.scala | 194 +++++++++++------- .../ie3/osmogrid/utils/ClusteringSpec.scala | 69 ++++++- 2 files changed, 183 insertions(+), 80 deletions(-) diff --git a/src/main/scala/utils/Clustering.scala b/src/main/scala/utils/Clustering.scala index f190e42f..9abec7ea 100644 --- a/src/main/scala/utils/Clustering.scala +++ b/src/main/scala/utils/Clustering.scala @@ -6,6 +6,7 @@ package utils +import com.typesafe.scalalogging.LazyLogging import edu.ie3.datamodel.models.StandardUnits import edu.ie3.datamodel.models.input.NodeInput import edu.ie3.datamodel.models.input.connector.LineInput @@ -15,7 +16,7 @@ import edu.ie3.osmogrid.exception.ClusterException import edu.ie3.osmogrid.lv.LvGridGeneratorSupport.GridElements import tech.units.indriya.ComparableQuantity import tech.units.indriya.quantity.Quantities -import utils.Clustering.{Cluster, isImprovement, totalDistance} +import utils.Clustering.{Cluster, isImprovement} import java.util.concurrent.ThreadLocalRandom import javax.measure.quantity.Power @@ -26,101 +27,129 @@ final case class Clustering( osmSubstations: Set[NodeInput], additionalSubstations: Set[NodeInput], nodes: Set[NodeInput], - nodeSize: Int -) { + nodeCount: Int, + substationCount: Int +) extends LazyLogging { /** Method to run the algorithm. This algorithm is based on PAM (Partitioning * Around Medoids). + * * @param maxIteration * maximal number of iterations * @return * a list of [[Cluster]]s */ - def run(implicit maxIteration: Int = 10): List[Cluster] = { + def run(implicit maxIteration: Int = 50): List[Cluster] = { // creates the initial clusters - var clusters: List[Cluster] = createClusters( + val initialClusters: Set[Cluster] = createClusters( osmSubstations ++ additionalSubstations, nodes ) - var i = 0 - var finished = false - - // tries to improve the cluster - while (!finished && i < maxIteration) { - calculateStep(clusters) match { - case Some(current) => - if (isImprovement(clusters, current)) { - clusters = current - i += 1 - } else { - finished = true + val initialSubstations = Set(initialClusters.map(_.substation)) + + val (clusters, _) = Range.Int + .apply(0, maxIteration, 1) + .foldLeft(initialClusters, initialSubstations) { + case ((currentClusters, substationCombinations), _) => + // calculate a swap for all swappable substations + val swaps = currentClusters + .filter(cluster => !osmSubstations.contains(cluster.substation)) + .map { cluster => + val node = + cluster.nodes.find(_ => true).getOrElse(cluster.substation) + (cluster.substation, node) + } + + calculateStep(swaps, substationCombinations) match { + case Some(maybeUpdate) => + if (isImprovement(currentClusters, maybeUpdate)) { + val substations = Set(maybeUpdate.map(_.substation)) + + val updatedFoundCombinations: Set[Set[NodeInput]] = + substationCombinations ++ substations + + (maybeUpdate, updatedFoundCombinations) + } else { + (currentClusters, substationCombinations) + } + case None => + (currentClusters, substationCombinations) } - case None => - finished = true } - } - clusters + clusters.toList } /** Calculates the best next step. - * @param clusters - * last step + * + * @param swaps + * a set of substation and node swaps * @return * an option */ - private def calculateStep(clusters: List[Cluster]): Option[List[Cluster]] = { - // calculates all swaps and returns the best as an option - additionalSubstations.par - .flatMap { substation => - nodes.map { node => - val currentSubstation = Set(substation) - val currentNode = Set(node) - - // swaps two nodes - val updatedSubstations = - additionalSubstations -- currentSubstation ++ currentNode - val updatedNodes = nodes -- currentNode ++ currentSubstation - - val clusters = - createClusters(updatedSubstations ++ osmSubstations, updatedNodes) - - (clusters, totalDistance(clusters)) - } + private def calculateStep( + swaps: Set[(NodeInput, NodeInput)], + substationCombinations: Set[Set[NodeInput]] + ): Option[Set[Cluster]] = { + val (updatedSubstations, updatedNodes) = + swaps.foldLeft((osmSubstations ++ additionalSubstations, nodes)) { + case ((substations, allNodes), swap) => + val newSubstations = Set(swap._2) + val newNode = Set(swap._1) + + val currentSubstations = substations -- newNode ++ newSubstations + val currentNodes = allNodes -- newSubstations ++ newNode + + (currentSubstations, currentNodes) } - .seq - .minByOption(_._2) - .map(_._1) + + if (substationCombinations.contains(updatedSubstations)) { + None + } else { + val clusters = createClusters(updatedSubstations, updatedNodes) + + Option.when(clusters.nonEmpty)(clusters) + } } /** Creates [[Cluster]] based on the given data. + * * @param substations * substations - * @param nodes + * @param others * all other nodes * @return - * a list of [[Cluster]] + * a set of [[Cluster]] */ private def createClusters( substations: Set[NodeInput], - nodes: Set[NodeInput] - ): List[Cluster] = { - if (substations.size + nodes.size != nodeSize) { - List.empty + others: Set[NodeInput] + ): Set[Cluster] = { + if (substations.size + nodes.size != nodeCount) { + logger.debug( + s"The number of found nodes ${substations.size + nodes.size} does not equal the expected number $nodeCount! Discarding the found option." + ) + Set.empty } else { - nodes - .map { n => - val closest = substations - .map { s => s -> connections.getDistance(n, s) } - .minByOption(_._2) - .map(_._1) - .getOrElse( - throw ClusterException(s"No substation found for node: $n") - ) - - (n, closest) - } - .groupMap(_._2)(_._1) + val nodeToSubstation = findClosestSubstation(substations, others) + val updatedSubstations = nodeToSubstation.map(_._2) + + val updatedNodeToSubstation = + if (updatedSubstations.size != substationCount) { + val newNodes = substations.diff(updatedSubstations) + + val unusedIds = newNodes.map(_.getId) + logger.info( + s"More substations than uses. Convert unused substations to normal nodes. Unused substations: $unusedIds" + ) + + findClosestSubstation(updatedSubstations, others ++ newNodes) + } else nodeToSubstation + + val groupedNodes: Map[NodeInput, Set[NodeInput]] = + updatedNodeToSubstation.groupMap(_._2)(_._1) + + groupedNodes.par .map { case (substation, connectedNodes) => // calculate all distances val distances = connectedNodes.map(node => @@ -131,13 +160,39 @@ final case class Clustering( ) distances.reduceOption { (a, b) => a + b } match { - case Some(distance) => Cluster(substation, connectedNodes, distance) + case Some(distance) => + Cluster(substation, connectedNodes, distance) case None => Cluster(substation, connectedNodes, Double.MaxValue) } } - .toList + .seq + .toSet } } + + /** Method for finding the closest substation for the given nodes. + * @param substations + * a set of all known substations + * @param others + * all other nodes + * @return + * a set of nodes to closest substation + */ + private def findClosestSubstation( + substations: Set[NodeInput], + others: Set[NodeInput] + ): Set[(NodeInput, NodeInput)] = + others.map { n => + val closest = substations + .map { s => s -> connections.getDistance(n, s) } + .minByOption(_._2) + .map(_._1) + .getOrElse( + throw ClusterException(s"No substation found for node: $n") + ) + + (n, closest) + } } object Clustering { @@ -178,7 +233,8 @@ object Clustering { osmSubstations, additionalSubstations, gridElements.nodes.values.toSet.diff(additionalSubstations), - gridElements.nodes.size + gridElements.substations.size + gridElements.nodes.size + gridElements.substations.size, + osmSubstations.size + additionalSubstations.size ) } @@ -232,8 +288,8 @@ object Clustering { * true if the are still improvements */ def isImprovement( - old: List[Cluster], - current: List[Cluster] + old: Set[Cluster], + current: Set[Cluster] ): Boolean = totalDistance(current) <= totalDistance(old) * 0.99 /** Calculates the total connection distance of a list of [[Cluster]]s. @@ -243,7 +299,7 @@ object Clustering { * @return * either the total distance or [[Double.MaxValue]] */ - private def totalDistance(list: List[Cluster]): Double = { + private def totalDistance(list: Set[Cluster]): Double = { if (list.isEmpty) { Double.MaxValue } else { diff --git a/src/test/scala/edu/ie3/osmogrid/utils/ClusteringSpec.scala b/src/test/scala/edu/ie3/osmogrid/utils/ClusteringSpec.scala index 20b24b91..aa1e3d1d 100644 --- a/src/test/scala/edu/ie3/osmogrid/utils/ClusteringSpec.scala +++ b/src/test/scala/edu/ie3/osmogrid/utils/ClusteringSpec.scala @@ -6,6 +6,7 @@ package edu.ie3.osmogrid.utils +import edu.ie3.datamodel.models.input.NodeInput import edu.ie3.test.common.{ClusterTestData, UnitSpec} import utils.{Clustering, Connections} import utils.Clustering.Cluster @@ -13,10 +14,10 @@ import utils.Clustering.Cluster class ClusteringSpec extends UnitSpec with ClusterTestData { "A Clustering" should { val createClusters = - PrivateMethod[List[Cluster]](Symbol("createClusters")) + PrivateMethod[Set[Cluster]](Symbol("createClusters")) val calculateStep = - PrivateMethod[Option[List[Cluster]]](Symbol("calculateStep")) + PrivateMethod[Option[Set[Cluster]]](Symbol("calculateStep")) "calculate the number of substations correctly" in { val getSubstationCount = PrivateMethod[Int](Symbol("getSubstationCount")) @@ -51,6 +52,8 @@ class ClusteringSpec extends UnitSpec with ClusterTestData { clustering.osmSubstations should contain allOf (p2_1, p1_1) clustering.additionalSubstations shouldBe Set.empty clustering.nodes should contain only (p2_4, p2_2, p1_3, p1_2, p2_3, p1_4) + clustering.nodeCount shouldBe 8 + clustering.substationCount shouldBe 2 } "create clusters correctly" in { @@ -69,22 +72,22 @@ class ClusteringSpec extends UnitSpec with ClusterTestData { } "check for improvements correctly" in { - val old: List[Cluster] = List(Cluster(p1_1, Set(p1_2), 500)) + val old = Set(Cluster(p1_1, Set(p1_2), 500)) Clustering.isImprovement( old, - List(Cluster(p1_1, Set(p1_2), 450)) + Set(Cluster(p1_1, Set(p1_2), 450)) ) shouldBe true Clustering.isImprovement( old, - List(Cluster(p1_1, Set(p1_2), 496)) // improvement is less than 1 % + Set(Cluster(p1_1, Set(p1_2), 496)) // improvement is less than 1 % ) shouldBe false } "calculate the total distance of a list of clusters correctly" in { val totalDistance = PrivateMethod[Double](Symbol("totalDistance")) - val clusters = List( + val clusters = Set( Cluster(p1_1, Set(p1_2), 500), Cluster(p2_1, Set(p2_2), 200), Cluster(p1_3, Set(p1_4), 300) @@ -106,19 +109,25 @@ class ClusteringSpec extends UnitSpec with ClusterTestData { Set(p1_1), additionalSubstation, elements.nodes.values.toSet, - 8 + 9, + 2 ) - val initialClusters = clustering invokePrivate createClusters( - Set(p1_1, p1_2), - clustering.nodes + val initialSubstations = Set(p1_1, p1_2) + + val swaps = Set( + (p1_1, p1_1), + (p1_2, p2_3) ) - val firstStep = clustering invokePrivate calculateStep(initialClusters) + + val firstStep = + clustering invokePrivate calculateStep(swaps, initialSubstations) firstStep match { case Some(value) => val map = value.map { c => c.substation -> c }.toMap map.keySet should contain allOf (p1_1, p2_3) + map(p1_1).nodes should contain allOf (p1_2, p1_3, p1_4) map(p2_3).nodes should contain allOf (p2_1, p2_2, p2_4) case None => @@ -126,6 +135,44 @@ class ClusteringSpec extends UnitSpec with ClusterTestData { } } + "find the closest substation correctly" in { + // with one osm substation + val elements = gridElements(List(p1_1)) + val connections = Connections(elements, lines.toSeq) + + val clustering = Clustering( + connections, + Set(p1_1), + Set(p2_1), + elements.nodes.values.toSet, + 9, + 2 + ) + + val findClosestSubstation = + PrivateMethod[Set[(NodeInput, NodeInput)]]( + Symbol("findClosestSubstation") + ) + + val cases = Table( + ("substations", "node", "expectedSubstation"), + (Set(p1_1, p2_1), p1_2, Set((p1_2, p1_1))), + (Set(p1_1, p2_1), p1_3, Set((p1_3, p1_1))), + (Set(p1_1, p2_1), p2_4, Set((p2_4, p2_1))), + (Set(p1_1, p2_3), p2_4, Set((p2_4, p2_3))) + ) + + forAll(cases) { (substations, node, expectedSubstation) => + val actual = clustering invokePrivate findClosestSubstation( + substations, + Set(node) + ) + + actual shouldBe expectedSubstation + } + + } + "run the calculation correctly" in { val clustering = Clustering.setup( gridElements(List(p1_1, p2_1)), From 0221c37518eb65e82ae4b185d7f18ea4b2877bdf Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Tue, 21 May 2024 12:31:39 +0200 Subject: [PATCH 5/6] Possible `Sonarqube` fix. Adding grid summary. --- input/fuerweiler/fuerweiler.conf | 2 +- .../osmogrid/main/RunOsmoGridStandalone.scala | 62 ++++++++++++++++++- src/main/scala/utils/Clustering.scala | 5 +- .../ie3/osmogrid/utils/ClusteringSpec.scala | 8 +-- 4 files changed, 67 insertions(+), 10 deletions(-) diff --git a/input/fuerweiler/fuerweiler.conf b/input/fuerweiler/fuerweiler.conf index da188d96..e6b43c7b 100644 --- a/input/fuerweiler/fuerweiler.conf +++ b/input/fuerweiler/fuerweiler.conf @@ -28,7 +28,7 @@ voltage.hv.default = 110.0 ################################################################## # Generation parameters ################################################################## -generation.lv.averagePowerDensity = 2000 +generation.lv.averagePowerDensity = 500 generation.lv.considerHouseConnectionPoints = false generation.lv.minDistance = 10 diff --git a/src/main/scala/edu/ie3/osmogrid/main/RunOsmoGridStandalone.scala b/src/main/scala/edu/ie3/osmogrid/main/RunOsmoGridStandalone.scala index ed0f2efb..769291f6 100644 --- a/src/main/scala/edu/ie3/osmogrid/main/RunOsmoGridStandalone.scala +++ b/src/main/scala/edu/ie3/osmogrid/main/RunOsmoGridStandalone.scala @@ -6,9 +6,17 @@ package edu.ie3.osmogrid.main -import org.apache.pekko.actor.typed.ActorSystem +import edu.ie3.datamodel.io.source.csv.CsvJointGridContainerSource +import edu.ie3.osmogrid.cfg.OsmoGridConfig.Output import edu.ie3.osmogrid.cfg.{ArgsParser, OsmoGridConfig} -import edu.ie3.osmogrid.guardian.{GuardianRequest, OsmoGridGuardian, Run} +import edu.ie3.osmogrid.exception.{GridException, IllegalConfigException} +import edu.ie3.osmogrid.guardian.{OsmoGridGuardian, Run} +import org.apache.pekko.actor.typed.ActorSystem + +import java.nio.file.Path +import scala.concurrent.Await +import scala.concurrent.duration.Duration +import scala.jdk.CollectionConverters.CollectionHasAsScala object RunOsmoGridStandalone { @@ -17,5 +25,55 @@ object RunOsmoGridStandalone { val actorSystem = ActorSystem(OsmoGridGuardian(), "OSMoGridGuardian") actorSystem ! Run(cfg) + + Await.result(actorSystem.whenTerminated, Duration.Inf) + summarizeGrid(cfg) + } + + private def summarizeGrid(cfg: OsmoGridConfig): Unit = { + println(" ") + + val grid = cfg.output match { + case Output(Some(csv), gridName) => + CsvJointGridContainerSource.read( + gridName, + csv.separator, + Path.of(csv.directory), + csv.hierarchic + ) + case Output(None, _) => + throw IllegalConfigException("No output given.") + } + + println(" ") + + // get nodes + val nodes = grid.getRawGrid.getNodes.asScala + + // check sub grids + val subnets = nodes.groupBy(_.getSubnet) + subnets + .map { case (i, subgridNodes) => + val voltLvl = subgridNodes.map(_.getVoltLvl.getNominalVoltage).toSet + + if (voltLvl.size != 1) { + throw GridException(s"In subgrid $i: $voltLvl") + } + + i -> s"In subgrid $i: ${voltLvl.toSeq(0)} with ${subgridNodes.size} node(s)" + } + .toList + .sortBy(_._1) + .map(_._2) + .foreach(println) + + println(s"Number of slack nodes: ${nodes.toSeq.count(_.isSlack)}") + println(s"Number of lv nodes: ${nodes.toSeq + .count(_.getVoltLvl.getNominalVoltage.getValue.doubleValue() < 10)}") + println(s"Number of mv nodes: ${nodes.toSeq + .count(_.getVoltLvl.getNominalVoltage.getValue.doubleValue() == 10)}") + println(s"Number of hv nodes: ${nodes.toSeq + .count(_.getVoltLvl.getNominalVoltage.getValue.doubleValue() == 110)}") + } } diff --git a/src/main/scala/utils/Clustering.scala b/src/main/scala/utils/Clustering.scala index 9abec7ea..21704231 100644 --- a/src/main/scala/utils/Clustering.scala +++ b/src/main/scala/utils/Clustering.scala @@ -290,7 +290,8 @@ object Clustering { def isImprovement( old: Set[Cluster], current: Set[Cluster] - ): Boolean = totalDistance(current) <= totalDistance(old) * 0.99 + ): Boolean = + calculateTotalLineLength(current) <= calculateTotalLineLength(old) * 0.99 /** Calculates the total connection distance of a list of [[Cluster]]s. * @@ -299,7 +300,7 @@ object Clustering { * @return * either the total distance or [[Double.MaxValue]] */ - private def totalDistance(list: Set[Cluster]): Double = { + def calculateTotalLineLength(list: Set[Cluster]): Double = { if (list.isEmpty) { Double.MaxValue } else { diff --git a/src/test/scala/edu/ie3/osmogrid/utils/ClusteringSpec.scala b/src/test/scala/edu/ie3/osmogrid/utils/ClusteringSpec.scala index aa1e3d1d..2aa3c77c 100644 --- a/src/test/scala/edu/ie3/osmogrid/utils/ClusteringSpec.scala +++ b/src/test/scala/edu/ie3/osmogrid/utils/ClusteringSpec.scala @@ -8,8 +8,8 @@ package edu.ie3.osmogrid.utils import edu.ie3.datamodel.models.input.NodeInput import edu.ie3.test.common.{ClusterTestData, UnitSpec} -import utils.{Clustering, Connections} import utils.Clustering.Cluster +import utils.{Clustering, Connections} class ClusteringSpec extends UnitSpec with ClusterTestData { "A Clustering" should { @@ -84,16 +84,14 @@ class ClusteringSpec extends UnitSpec with ClusterTestData { ) shouldBe false } - "calculate the total distance of a list of clusters correctly" in { - val totalDistance = PrivateMethod[Double](Symbol("totalDistance")) - + "calculate the total line length of a list of clusters correctly" in { val clusters = Set( Cluster(p1_1, Set(p1_2), 500), Cluster(p2_1, Set(p2_2), 200), Cluster(p1_3, Set(p1_4), 300) ) - Clustering invokePrivate totalDistance(clusters) shouldBe 1000 + Clustering.calculateTotalLineLength(clusters) shouldBe 1000 } "calculate the next step correctly" in { From 1dbee0cae6385298edbc180ccb1ea0408eb4c021 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Tue, 21 May 2024 12:33:55 +0200 Subject: [PATCH 6/6] Improvements to `scalaDoc`. --- src/main/scala/edu/ie3/osmogrid/lv/LvGridGeneratorSupport.scala | 2 +- src/main/scala/utils/Clustering.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/edu/ie3/osmogrid/lv/LvGridGeneratorSupport.scala b/src/main/scala/edu/ie3/osmogrid/lv/LvGridGeneratorSupport.scala index 38ffa6dd..1881616d 100644 --- a/src/main/scala/edu/ie3/osmogrid/lv/LvGridGeneratorSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/lv/LvGridGeneratorSupport.scala @@ -257,7 +257,7 @@ object LvGridGeneratorSupport extends LazyLogging { } /** This method will reduce the graph by removing some vertices. A vertex is - * removed if its degree is < 2 and it is not defined to be kept. + * removed if its degree is <= 2 and it is not defined to be kept. * * @param osmGraph * graph to reduce diff --git a/src/main/scala/utils/Clustering.scala b/src/main/scala/utils/Clustering.scala index 21704231..cd1dccbd 100644 --- a/src/main/scala/utils/Clustering.scala +++ b/src/main/scala/utils/Clustering.scala @@ -140,7 +140,7 @@ final case class Clustering( val unusedIds = newNodes.map(_.getId) logger.info( - s"More substations than uses. Convert unused substations to normal nodes. Unused substations: $unusedIds" + s"Unused substations exists. Converted these substations to normal nodes. Unused substations: $unusedIds" ) findClosestSubstation(updatedSubstations, others ++ newNodes)