Skip to content

Commit

Permalink
Add Spark 3.5 support (#514)
Browse files Browse the repository at this point in the history
* Add Spark 3.5 support

* Replace with DataTypeUtils.fromAttributes

* Remove unintended new line
  • Loading branch information
jhchee authored Feb 8, 2024
1 parent f3222c1 commit f6fb888
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 3 deletions.
4 changes: 2 additions & 2 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

<groupId>com.amazon.deequ</groupId>
<artifactId>deequ</artifactId>
<version>2.0.6-spark-3.4</version>
<version>2.0.6-spark-3.5</version>

<properties>
<maven.compiler.source>1.8</maven.compiler.source>
Expand All @@ -18,7 +18,7 @@
<artifact.scala.version>${scala.major.version}</artifact.scala.version>
<scala-maven-plugin.version>4.8.1</scala-maven-plugin.version>

<spark.version>3.4.1</spark.version>
<spark.version>3.5.0</spark.version>
</properties>

<name>deequ</name>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.types._
import org.apache.spark.sql.catalyst.expressions.aggregate.HLLConstants._
import org.apache.spark.sql.catalyst.trees.UnaryLike
import org.apache.spark.sql.catalyst.types.DataTypeUtils

/** Adjusted version of org.apache.spark.sql.catalyst.expressions.aggregate.HyperloglogPlus */
private[sql] case class StatefulHyperloglogPlus(
Expand Down Expand Up @@ -59,7 +60,7 @@ private[sql] case class StatefulHyperloglogPlus(

override def dataType: DataType = BinaryType

override def aggBufferSchema: StructType = StructType.fromAttributes(aggBufferAttributes)
override def aggBufferSchema: StructType = DataTypeUtils.fromAttributes(aggBufferAttributes)

/** Allocate enough words to store all registers. */
override val aggBufferAttributes: Seq[AttributeReference] = Seq.tabulate(NUM_WORDS) { i =>
Expand Down

0 comments on commit f6fb888

Please sign in to comment.