diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c4a65c..203dc5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,8 +6,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 - - uses: olafurpg/setup-scala@v11 + - uses: actions/setup-java@v4 with: - java-version: adopt@1.11 + distribution: temurin + java-version: 17 + cache: sbt + - uses: sbt/setup-sbt@v1 - name: Test - run: sbt test \ No newline at end of file + run: sbt "++test" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..1bbe495 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,39 @@ +name: Publish +on: + workflow_dispatch: +jobs: + build: + runs-on: ubuntu-latest + env: + SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} + SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 11 + cache: sbt + - uses: sbt/setup-sbt@v1 + - name: Import GPG key + id: import_gpg + uses: crazy-max/ghaction-import-gpg@v5 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSWORD }} + - name: Build and publish maven artifacts + run: sbt "+clean; +publishSigned" + - name: Setup buildx builder + uses: docker/setup-buildx-action@v3 + with: + platforms: linux/amd64,linux/arm64 + config-inline: | + [worker.oci] + max-parallelism = 1 + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push images + run: ./build-images.sh --push diff --git a/.sbtopts b/.sbtopts new file mode 100644 index 0000000..837257c --- /dev/null +++ b/.sbtopts @@ -0,0 +1 @@ +-J--add-opens=java.base/java.util=ALL-UNNAMED diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c282f2e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +ARG FLINK_VERSION="invalid" + +FROM flink:${FLINK_VERSION}-scala_2.12-java17 + +RUN rm $FLINK_HOME/lib/flink-scala*.jar + +ARG FLINK_SCALA_VERSION="invalid" +COPY --chown=flink:flink flink-scala-assembly-${FLINK_SCALA_VERSION}.jar $FLINK_HOME/lib/ diff --git a/README.md b/README.md index fbd4ebc..002b2bc 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,25 @@ -# flink-scala-2.13 +# flink-scala -For now Flink does not support Scala 2.13. For more refer to . +This module is a replacement for the `org.apache.flink:flink-scala` lib shipped with flink distribution, +and allows using flink with higher scala versions than 2.12.8. -Our solution to deploy Scala 2.13 code to Flink, until it's officially supported (or Flink becomes really scala-free): +For more refer to . +## Replacing flink-scala in flink distribution ```bash rm $FLINK_HOME/lib/flink-scala*.jar -wget https://repo1.maven.org/maven2/pl/touk/flink-scala-2-13_2.13/1.1.2/flink-scala-2-13_2.13-1.1.2-assembly.jar -O $FLINK_HOME/lib/flink-scala-2-13_2.13-1.1.1-assembly.jar + +wget https://central.sonatype.com/repository/maven-snapshots/pl/touk/flink-scala_2.13/1.1.3-SNAPSHOT/flink-scala_2.13-1.1.3-SNAPSHOT-assembly.jar -O $FLINK_HOME/lib/flink-scala_2.13-1.1.3-SNAPSHOT-assembly.jar ``` +## Using as a lib (probably only sufficient when child-first classloading is enabled on flink) ```scala -libraryDependencies += "pl.touk" %% "flink-scala-2-13" % "1.1.2" +libraryDependencies += "pl.touk" %% "flink-scala" % "1.1.3-SNAPSHOT" ``` +## Prebuild flink images +* we provide prebuild flink docker images for scala 2.12 and 2.13 on [Docker Hub](https://hub.docker.com/r/touk/flink) + ## Publishing ``` sbt publishSigned sonatypeBundleRelease diff --git a/build-images.sh b/build-images.sh new file mode 100755 index 0000000..7836c18 --- /dev/null +++ b/build-images.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -e + +VERSION=$(sbt --no-colors -Dsbt.supershell=false "show version" | grep "info" | tail -1 | awk '{print $2}') +FLINK_VERSION=$(sbt --no-colors -Dsbt.supershell=false "show flinkV" | grep "info" | tail -1 | awk '{print $2}') +echo "FLINK_SCALA_VERSION: ${VERSION}" +echo "FLINK_VERSION: ${FLINK_VERSION}" + +if [[ "$1" == "--push" ]]; then + OUTPUT_TYPE="registry" +else + OUTPUT_TYPE="docker" +fi + +sbt "+clean; +assembly" + +IMAGE_TAG="${VERSION}-flink${FLINK_VERSION}-scala_2.12" +echo "Building Docker image with version: $IMAGE_TAG" + +cp target/scala-2.12/flink-scala-assembly-${VERSION}.jar . + +docker buildx build \ + --build-arg FLINK_VERSION=$FLINK_VERSION \ + --build-arg FLINK_SCALA_VERSION=$VERSION \ + --platform linux/amd64,linux/arm64 \ + --tag touk/flink:$IMAGE_TAG \ + --output=type=$OUTPUT_TYPE . + +IMAGE_TAG="${VERSION}-flink${FLINK_VERSION}-scala_2.13" +echo "Building Docker image with version: $IMAGE_TAG" + +cp target/scala-2.13/flink-scala-assembly-${VERSION}.jar . + +docker buildx build \ + --build-arg FLINK_VERSION=$FLINK_VERSION \ + --build-arg FLINK_SCALA_VERSION=$VERSION \ + --platform linux/amd64,linux/arm64 \ + --tag touk/flink:$IMAGE_TAG \ + --output=type=$OUTPUT_TYPE . diff --git a/build.sbt b/build.sbt index c1532d5..2b352bb 100644 --- a/build.sbt +++ b/build.sbt @@ -1,67 +1,103 @@ -version := "1.1.2" +import sbtassembly.MergeStrategy -scalaVersion := "2.13.15" +name := "flink-scala" +version := "1.1.3-SNAPSHOT" -name := "flink-scala-2.13" +val scala212 = "2.12.20" +val scala213 = "2.13.16" -lazy val flinkV = "1.16.2" -lazy val scalaTestV = "3.2.17" +scalaVersion := scala212 +crossScalaVersions := List(scala212, scala213) +val flinkV = settingKey[String]("Flink version") // to extract using `show flinkV` +flinkV := "1.20.1" -assembly / artifact := { - val art = (assembly / artifact).value - art.withClassifier(Some("assembly")) -} +lazy val scalaTestV = "3.2.19" -addArtifact(assembly / artifact, assembly) +lazy val assemblySettings = Seq( + assembly / artifact := { + val art = (assembly / artifact).value + art.withClassifier(Some("assembly")) + }, + assembly / assemblyMergeStrategy := { + case PathList(ps@_*) if ps.last == "module-info.class" => MergeStrategy.discard + case x => MergeStrategy.defaultMergeStrategy(x) + }, + addArtifact(assembly / artifact, assembly) +) lazy val publishSettings = Seq( publishMavenStyle := true, + sonatypeCredentialHost := "central.sonatype.com", publishTo := { - val defaultNexusUrl = "https://oss.sonatype.org/" - if (isSnapshot.value) - Some("snapshots" at defaultNexusUrl + "content/repositories/snapshots") - else { - sonatypePublishToBundle.value - } + if (isSnapshot.value) + Some("snapshots" at "https://central.sonatype.com/repository/maven-snapshots/") + else { + sonatypePublishToBundle.value //todo: full release not tested yet + } }, Test / publishArtifact := false, //We don't put scm information here, it will be added by release plugin and if scm provided here is different than the one from scm //we'll end up with two scm sections and invalid pom... pomExtra in Global := { - scm:git:github.com/TouK/flink-scala-2.13.git - scm:git:git@github.com:TouK/flink-scala-2.13.git - github.com/TouK/flink-scala-2.13 + scm:git:github.com/TouK/flink-scala.git + scm:git:git@github.com:TouK/flink-scala.git + github.com/TouK/flink-scala - - - TouK - TouK - https://touk.pl - - + + + TouK + TouK + https://touk.pl + + }, organization := "pl.touk", ) lazy val root = (project in file(".")) .settings( - name := "flink-scala-2.13", + name := "flink-scala", organization := "pl.touk", licenses := Seq("Apache 2" -> url("http://www.apache.org/licenses/LICENSE-2.0.txt")), - homepage := Some(url("https://github.com/TouK/flink-scala-2.13")), - libraryDependencies ++= { - Seq( - "org.scala-lang" % "scala-compiler" % scalaVersion.value, - "org.scala-lang" % "scala-reflect" % scalaVersion.value, - - "org.apache.flink" % "flink-streaming-java" % flinkV % "provided", - "com.twitter" %% "chill" % "0.9.5" exclude("com.esotericsoftware", "kryo-shaded"), - "com.esotericsoftware.kryo" % "kryo" % "2.24.0" % "provided", - - "org.scalatest" %% "scalatest" % scalaTestV % "test", - ) - } + homepage := Some(url("https://github.com/TouK/flink-scala")), + libraryDependencies ++= (forScalaVersion(scalaVersion.value) { + case (2, 12) => + Seq( + "org.apache.flink" %% "flink-scala" % flinkV.value excludeAll( + ExclusionRule(organization = "org.apache.flink", name = "flink-core"), + ExclusionRule(organization = "org.apache.flink", name = "flink-java"), + ExclusionRule(organization = "org.apache.flink", name = "flink-shaded-asm-9"), + ExclusionRule(organization = "org.slf4j", name = "slf4j-api"), + ExclusionRule(organization = "com.google.code.findbugs", name = "jsr305"), + ), + "com.esotericsoftware.kryo" % "kryo" % "2.24.0" % Test, + "org.apache.flink" % "flink-java" % flinkV.value % Test, + ) + case (2, 13) => + Seq( + "org.apache.flink" % "flink-streaming-java" % flinkV.value % "provided", + "com.twitter" %% "chill" % "0.9.5" exclude("com.esotericsoftware", "kryo-shaded"), + "com.esotericsoftware.kryo" % "kryo" % "2.24.0" % "provided", + ) + } ++ Seq( + "org.scala-lang" % "scala-library" % scalaVersion.value, + "org.scala-lang" % "scala-compiler" % scalaVersion.value, + "org.scala-lang" % "scala-reflect" % scalaVersion.value, + "org.scalatest" %% "scalatest" % scalaTestV % Test, + )) ) + .settings(assemblySettings *) .settings(publishSettings) + +def forScalaVersion[T](version: String)(provide: PartialFunction[(Int, Int), T]): T = { + CrossVersion.partialVersion(version) match { + case Some((major, minor)) if provide.isDefinedAt((major.toInt, minor.toInt)) => + provide((major.toInt, minor.toInt)) + case Some(_) => + throw new IllegalArgumentException(s"Scala version $version is not handled") + case None => + throw new IllegalArgumentException(s"Invalid Scala version $version") + } +} diff --git a/project/plugins.sbt b/project/plugins.sbt index 3a6596d..dc39144 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1,3 @@ -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.4") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.3.1") addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.10.0") addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.2.1") diff --git a/src/main/scala/org/apache/flink/runtime/types/FlinkScalaKryoInstantiator.scala b/src/main/scala-2.13/org/apache/flink/runtime/types/FlinkScalaKryoInstantiator.scala similarity index 99% rename from src/main/scala/org/apache/flink/runtime/types/FlinkScalaKryoInstantiator.scala rename to src/main/scala-2.13/org/apache/flink/runtime/types/FlinkScalaKryoInstantiator.scala index d26b895..e21c6b6 100644 --- a/src/main/scala/org/apache/flink/runtime/types/FlinkScalaKryoInstantiator.scala +++ b/src/main/scala-2.13/org/apache/flink/runtime/types/FlinkScalaKryoInstantiator.scala @@ -233,4 +233,4 @@ class ScalaCollectionsRegistrarCompat extends IKryoRegistrar { ) ) } -} \ No newline at end of file +} diff --git a/src/main/scala/org/apache/flink/runtime/types/JavaWrapperScala2_13Serializer.scala b/src/main/scala-2.13/org/apache/flink/runtime/types/JavaWrapperScala2_13Serializer.scala similarity index 100% rename from src/main/scala/org/apache/flink/runtime/types/JavaWrapperScala2_13Serializer.scala rename to src/main/scala-2.13/org/apache/flink/runtime/types/JavaWrapperScala2_13Serializer.scala diff --git a/src/test/scala/org/apache/flink/runtime/types/FlinkScalaKryoInstantiatorSpec.scala b/src/test/scala-2.13/org/apache/flink/runtime/types/FlinkScalaKryoInstantiatorSpec.scala similarity index 100% rename from src/test/scala/org/apache/flink/runtime/types/FlinkScalaKryoInstantiatorSpec.scala rename to src/test/scala-2.13/org/apache/flink/runtime/types/FlinkScalaKryoInstantiatorSpec.scala