Skip to content

Commit 75375e3

Browse files
colindeancolindean
committed
Implements basic crossbuild for 2.{11,12,13}
Done: * Compile and tests for 2.11 * Assembly for 2.11 TODO: * Compile and tests for 2.12 and 2.13 * Assembly for 2.12 and 2.13 Work toward #84
1 parent 379163e commit 75375e3

File tree

4 files changed

+86
-60
lines changed

4 files changed

+86
-60
lines changed

‎build.sbt

Lines changed: 80 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,21 @@
1-
name := "data-validator"
2-
organization := "com.target"
3-
4-
scalaVersion := "2.11.12"
5-
6-
val sparkVersion = "2.3.1"
7-
81
val circeVersion = "0.10.0"
92

10-
//addDependencyTreePlugin
3+
val scala211 = "2.11.12"
4+
val scala212 = "2.12.15"
5+
val scala213 = "2.13.8"
6+
7+
ThisBuild / organization := "com.target"
118
enablePlugins(GitVersioning)
12-
git.useGitDescribe := true
9+
ThisBuild / git.useGitDescribe := true
1310
ThisBuild / versionScheme := Some("early-semver")
1411

12+
// sbt auto-reload on changes
13+
Global / onChangedBuildSource := ReloadOnSourceChanges
14+
15+
// Enforces scalastyle checks
16+
val compileScalastyle = TaskKey[Unit]("compileScalastyle")
17+
val generateTestData = TaskKey[Unit]("generateTestData")
18+
1519
/////////////
1620
// Publishing
1721
/////////////
@@ -23,59 +27,77 @@ githubRepository := "data-validator"
2327
githubTokenSource := (TokenSource.Environment("GITHUB_TOKEN") ||
2428
TokenSource.GitConfig("github.token") ||
2529
TokenSource.Environment("SHELL")) // it's safe to assume this exists and is not unique
26-
2730
publishTo := githubPublishTo.value
2831

29-
enablePlugins(BuildInfoPlugin)
30-
buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion)
31-
buildInfoPackage := "com.target.data_validator"
32+
lazy val commonSettings: SettingsDefinition = Def.settings(
33+
name := "data-validator",
34+
buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion),
35+
buildInfoPackage := "com.target.data_validator",
36+
libraryDependencies ++= Seq(
37+
"com.typesafe.scala-logging" %% "scala-logging" % "3.8.0",
38+
"com.github.scopt" %% "scopt" % "3.7.0",
39+
"com.sun.mail" % "javax.mail" % "1.6.2",
40+
"com.lihaoyi" %% "scalatags" % "0.6.7",
41+
"io.circe" %% "circe-yaml" % "0.9.0",
42+
"io.circe" %% "circe-core" % circeVersion,
43+
"io.circe" %% "circe-generic" % circeVersion,
44+
"io.circe" %% "circe-parser" % circeVersion,
45+
// "org.apache.spark" %% "spark-sql" % sparkVersion % Provided,
46+
"org.scalatest" %% "scalatest" % "3.0.5" % Test,
47+
"junit" % "junit" % "4.12" % Test,
48+
"com.novocode" % "junit-interface" % "0.11" % Test exclude ("junit", "junit-dep")
49+
),
50+
(Test / fork) := true,
51+
javaOptions ++= Seq("-Xms512M", "-Xmx2048M", "-XX:+CMSClassUnloadingEnabled"),
52+
(Test / parallelExecution) := false,
3253

33-
libraryDependencies ++= Seq(
34-
"com.typesafe.scala-logging" %% "scala-logging" % "3.8.0",
35-
"com.github.scopt" %% "scopt" % "3.7.0",
36-
"com.sun.mail" % "javax.mail" % "1.6.2",
37-
"com.lihaoyi" %% "scalatags" % "0.6.7",
38-
"io.circe" %% "circe-yaml" % "0.9.0",
39-
"io.circe" %% "circe-core" % circeVersion,
40-
"io.circe" %% "circe-generic" % circeVersion,
41-
"io.circe" %% "circe-parser" % circeVersion,
42-
"org.apache.spark" %% "spark-sql" % sparkVersion % Provided,
43-
44-
"org.scalatest" %% "scalatest" % "3.0.5" % Test,
45-
"junit" % "junit" % "4.12" % Test,
46-
"com.novocode" % "junit-interface" % "0.11" % Test exclude ("junit", "junit-dep")
47-
)
54+
// required for unit tests, but not set in some environments
55+
(Test / envVars) ++= Map(
56+
"JAVA_HOME" ->
57+
Option(System.getenv("JAVA_HOME"))
58+
.getOrElse(System.getProperty("java.home"))
59+
),
60+
(assembly / mainClass) := Some("com.target.data_validator.Main"),
61+
scalastyleFailOnWarning := true,
62+
scalastyleFailOnError := true,
63+
compileScalastyle := (Compile / scalastyle).toTask("").value,
64+
(Compile / compile) := ((Compile / compile) dependsOn compileScalastyle).value,
65+
(Compile / run) := Defaults
66+
.runTask(Compile / fullClasspath, Compile / run / mainClass, Compile / run / runner)
67+
.evaluated,
4868

49-
Test / fork := true
50-
javaOptions ++= Seq("-Xms512M", "-Xmx2048M", "-XX:+CMSClassUnloadingEnabled")
51-
Test / parallelExecution := false
52-
// required for unit tests, but not set in some environments
53-
Test / envVars ++= Map(
54-
"JAVA_HOME" ->
55-
Option(System.getenv("JAVA_HOME"))
56-
.getOrElse(System.getProperty("java.home"))
69+
/////////////
70+
// Publishing
71+
/////////////
72+
githubOwner := "target",
73+
githubRepository := "data-validator",
74+
// this unfortunately must be set strangely because GitHub requires a token for pulling packages
75+
// and sbt-github-packages does not allow the user to configure the resolver not to be used.
76+
// https://github.com/djspiewak/sbt-github-packages/issues/28
77+
githubTokenSource := (TokenSource.Environment("GITHUB_TOKEN") ||
78+
TokenSource.GitConfig("github.token") ||
79+
TokenSource.Environment("SHELL")), // it's safe to assume this exists and is not unique
80+
publishTo := githubPublishTo.value
5781
)
5882

59-
assembly / mainClass := Some("com.target.data_validator.Main")
60-
61-
// Enforces scalastyle checks
62-
val compileScalastyle = TaskKey[Unit]("compileScalastyle")
63-
scalastyleFailOnWarning := true
64-
scalastyleFailOnError := true
65-
66-
compileScalastyle := (Compile / scalastyle).toTask("").value
67-
(Compile / compile) := ((Compile / compile) dependsOn compileScalastyle).value
68-
69-
(Compile / run) := Defaults
70-
.runTask(
71-
Compile / fullClasspath,
72-
Compile / run / mainClass,
73-
Compile / run / runner
83+
lazy val root = (projectMatrix in file("."))
84+
.enablePlugins(BuildInfoPlugin)
85+
.settings(commonSettings)
86+
.jvmPlatform(
87+
scalaVersions = Seq(scala211),
88+
settings = Seq(
89+
libraryDependencies += "org.apache.spark" %% "spark-sql" % "2.3.4" % Provided,
90+
(Compile / runMain) := Defaults.runMainTask(Compile / fullClasspath, Compile / run / runner).evaluated,
91+
generateTestData := {
92+
(Compile / runMain).toTask(" com.target.data_validator.GenTestData").value
93+
}
94+
)
95+
)
96+
.jvmPlatform(
97+
scalaVersions = Seq(scala212),
98+
settings = Seq(libraryDependencies += "org.apache.spark" %% "spark-sql" % "2.4.8" % Provided)
99+
)
100+
.jvmPlatform(
101+
scalaVersions = Seq(scala213),
102+
settings = Seq(libraryDependencies += "org.apache.spark" %% "spark-sql" % "3.2.1" % Provided)
74103
)
75-
.evaluated
76-
77-
(Compile / runMain) := Defaults.runMainTask(Compile / fullClasspath, Compile / run / runner).evaluated
78-
TaskKey[Unit]("generateTestData") := {
79-
libraryDependencies += "org.apache.spark" %% "spark-sql" % sparkVersion
80-
(Compile / runMain).toTask(" com.target.data_validator.GenTestData").value
81-
}

‎project/plugins.sbt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0")
33
addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.9.0")
44
addSbtPlugin("com.codecommit" % "sbt-github-packages" % "0.5.3")
55
addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6")
6+
addSbtPlugin("com.eed3si9n" % "sbt-projectmatrix" % "0.9.0")

‎src/test/scala/com/target/data_validator/ConfigParserSpec.scala

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import com.target.data_validator.validator.{MinNumRows, NullCheck}
55
import io.circe.Json
66
import org.scalatest.{BeforeAndAfterAll, FunSpec}
77

8+
import scala.io.Source
9+
810
class ConfigParserSpec extends FunSpec with BeforeAndAfterAll {
911

1012
// Silence is golden!
@@ -106,7 +108,8 @@ class ConfigParserSpec extends FunSpec with BeforeAndAfterAll {
106108
describe("parseFile") {
107109

108110
it("should support loading config files by path") {
109-
val output = ConfigParser.parseFile("src/test/resources/test_config.yaml", Map.empty)
111+
val path = getClass.getResource("/test_config.yaml").getPath
112+
val output = ConfigParser.parseFile(path, Map.empty)
110113
assert(output == Right(expectedConfiguration))
111114
}
112115

‎src/test/scala/com/target/data_validator/ValidatorSpecifiedFormatLoaderSpec.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ class ValidatorSpecifiedFormatLoaderSpec extends WordSpec with Matchers with Tes
1919
MinNumRows(JsonUtils.string2Json("9"))
2020
),
2121
options = None,
22-
loadData = Some(List("src/test/resources/format_test.jsonl"))
22+
loadData = Some(List(getClass.getResource("/format_test.jsonl").getPath))
2323
)
2424

2525
val didFail = loader.quickChecks(spark, mkDict())(mkConfig(List(loader)))

0 commit comments

Comments
 (0)