From ff372947384a4daacfdee8342d6eb9672ae30418 Mon Sep 17 00:00:00 2001 From: Tom Wollnik Date: Wed, 28 Apr 2021 14:19:51 +0200 Subject: [PATCH 1/3] simplify pom.xml and Makefile --- Makefile | 17 +----------- pom.xml | 84 ++++++++++++++------------------------------------------ 2 files changed, 21 insertions(+), 80 deletions(-) diff --git a/Makefile b/Makefile index fdedf1ca9..bbba98aab 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,2 @@ -# mvn profiles for the different supported -# Spark and Scala versions. Uncomment -# the one that you want to use. You can also -# override the profile on the command line: -# `make MVN_PROFILE=spark-2.4-scala-2.11 build` -MVN_PROFILE := spark-3.0-scala-2.12 -# MVN_PROFILE := spark-2.4-scala-2.11 -# MVN_PROFILE := spark-2.3-scala-2.11 -# MVN_PROFILE := spark-2.2-scala-2.11 - -# Build the project for specific Spark and -# Scala versions. You can change the profile -# variable to use a different Scala or Spark -# version (see list above). -# If you need more log ouput remove the -q flag. build: - mvn clean install -q -P $(MVN_PROFILE) + mvn clean install diff --git a/pom.xml b/pom.xml index 8ad0f782b..831b114d3 100644 --- a/pom.xml +++ b/pom.xml @@ -5,8 +5,23 @@ 4.0.0 com.amazon.deequ - deequ${artifact.scala.version}${artifact.spark.version} - 1.1.0-SNAPSHOT + deequ + 1.2.2-spark-2.2 + + + 1.8 + 1.8 + UTF-8 + + + 2.11 + ${scala.major.version}.10 + ${scala.major.version} + 4.4.0 + + + 2.2.3 + deequ Deequ is a library built on top of Apache Spark for defining "unit tests for data", @@ -54,28 +69,6 @@ https://github.com/awslabs/deequ - - 1.8 - 1.8 - UTF-8 - - - ${scala-211.major.version} - ${scala.major.version}.10 - 2.11 - 2.12 - - 4.4.0 - - - ${spark-24.version} - 2.2.2 - 2.3.2 - 2.4.2 - 3.0.0 - - - org.scala-lang @@ -244,6 +237,9 @@ org.apache.maven.plugins maven-jar-plugin 3.2.0 + + ${project.artifactId}_${scala.major.version}-${project.version} + @@ -403,46 +399,6 @@ - - spark-2.2-scala-2.11 - - ${spark-22.version} - ${scala-211.major.version} - ${scala.major.version}.10 - _scala-${scala.major.version} - _spark-${spark.version} - - - - spark-2.3-scala-2.11 - - ${spark-23.version} - ${scala-211.major.version} - ${scala.major.version}.10 - _scala-${scala.major.version} - _spark-${spark.version} - - - - spark-2.4-scala-2.11 - - ${spark-24.version} - ${scala-211.major.version} - ${scala.major.version}.10 - _scala-${scala.major.version} - _spark-${spark.version} - - - - spark-3.0-scala-2.12 - - ${spark-30.version} - ${scala-212.major.version} - ${scala.major.version}.10 - _scala-${scala.major.version} - _spark-${spark.version} - - From 0c3d78d428881346f4b0c206c42c9b0d69ac163a Mon Sep 17 00:00:00 2001 From: Tom Wollnik Date: Wed, 28 Apr 2021 14:24:51 +0200 Subject: [PATCH 2/3] set version to ..-SNAPSHOT --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 831b114d3..cc2e50b88 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.amazon.deequ deequ - 1.2.2-spark-2.2 + 1.2.2-SNAPSHOT 1.8 @@ -14,13 +14,13 @@ UTF-8 - 2.11 + 2.12 ${scala.major.version}.10 ${scala.major.version} 4.4.0 - 2.2.3 + 2.4.7 deequ From 2065c5a18bd600e6ad3d73c58c986ebf5bda6139 Mon Sep 17 00:00:00 2001 From: Tom Wollnik Date: Wed, 28 Apr 2021 15:10:58 +0200 Subject: [PATCH 3/3] update README --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c853fc887..499007a21 100644 --- a/README.md +++ b/README.md @@ -10,22 +10,22 @@ Python users may also be interested in PyDeequ, a Python interface for Deequ. Yo ## Requirements and Installation -__Deequ__ depends on Java 8. We provide releases compatible with Apache Spark versions 2.2.x to 2.4.x and 3.0.x. +__Deequ__ depends on Java 8. We provide releases compatible with Apache Spark versions 2.2.x to 3.0.x. The Spark 2.2.x and 2.3.x releases depend on Scala 2.11 and the Spark 2.4.x and 3.0.x releases depend on Scala 2.12. Available via [maven central](http://mvnrepository.com/artifact/com.amazon.deequ/deequ). -Choose the latest release that matches your Scala and Spark versions from the [available versions](https://repo1.maven.org/maven2/com/amazon/deequ/deequ/). Add the release as a dependency to your project. For example: +Choose the latest release that matches your Spark version from the [available versions](https://repo1.maven.org/maven2/com/amazon/deequ/deequ/). Add the release as a dependency to your project. For example for spark 3.0.x: __Maven__ ``` com.amazon.deequ deequ - 1.1.0_spark-3.0-scala-2.12 + 1.2.2-spark-3.0 ``` __sbt__ ``` -libraryDependencies += "com.amazon.deequ" % "deequ" % "1.1.0_spark-3.0-scala-2.12" +libraryDependencies += "com.amazon.deequ" % "deequ" % "1.2.2-spark-3.0" ``` ## Example