fixes #3: Partitioning bug (#4)

conker84 · web-flow · commit 4f867703f518 · 2022-09-27T14:25:50.000+02:00
diff --git a/src/main/scala/org/neo4j/dwh/connector/Neo4jDWHConnector.scala b/src/main/scala/org/neo4j/dwh/connector/Neo4jDWHConnector.scala
@@ -86,7 +86,7 @@ class Neo4jDWHConnector(session: SparkSession, job: JobConfig) {
     }
 
     if (source.partition.number > 0) {
-      dataFrame = if (StringUtils.isNotBlank(source.partition.by)) {
+      dataFrame = if (StringUtils.isBlank(source.partition.by)) {
         dataFrame.repartition(source.partition.number)
       } else {
         dataFrame.repartition(source.partition.number, new sql.Column(source.partition.by))

Original file line number	Diff line number	Diff line change
`@@ -86,7 +86,7 @@ class Neo4jDWHConnector(session: SparkSession, job: JobConfig) {`
`86`	`86`	`}`
`87`	`87`
`88`	`88`	`if (source.partition.number > 0) {`
`89`		`- dataFrame = if (StringUtils.isNotBlank(source.partition.by)) {`
	`89`	`+ dataFrame = if (StringUtils.isBlank(source.partition.by)) {`
`90`	`90`	`dataFrame.repartition(source.partition.number)`
`91`	`91`	`} else {`
`92`	`92`	`dataFrame.repartition(source.partition.number, new sql.Column(source.partition.by))`