Jdk17 scala 2.12.20 and 2.13.16

.github/workflows/ci.yml

@@ -6,8 +6,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v1
-      - uses: olafurpg/setup-scala@v11
+      - uses: actions/setup-java@v4
         with:
-          java-version: [email protected]
+          distribution: temurin
+          java-version: 17
+          cache: sbt
+      - uses: sbt/setup-sbt@v1
       - name: Test
-        run: sbt test
+        run: sbt "++test"

.github/workflows/publish.yml

@@ -0,0 +1,39 @@
+name: Publish
+on:
+  workflow_dispatch:
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
+      SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 11
+          cache: sbt
+      - uses: sbt/setup-sbt@v1
+      - name: Import GPG key
+        id: import_gpg
+        uses: crazy-max/ghaction-import-gpg@v5
+        with:
+          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSWORD }}
+      - name: Build and publish maven artifacts
+        run: sbt "+clean; +publishSigned"
+      - name: Setup buildx builder
+        uses: docker/setup-buildx-action@v3
+        with:
+          platforms: linux/amd64,linux/arm64
+          config-inline: |
+            [worker.oci]
+              max-parallelism = 1
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push images
+        run: ./build-images.sh --push

.sbtopts

@@ -0,0 +1 @@
+-J--add-opens=java.base/java.util=ALL-UNNAMED

Dockerfile

@@ -0,0 +1,8 @@
+ARG FLINK_VERSION="invalid"
+
+FROM flink:${FLINK_VERSION}-scala_2.12-java17
+
+RUN rm $FLINK_HOME/lib/flink-scala*.jar
+
+ARG FLINK_SCALA_VERSION="invalid"
+COPY --chown=flink:flink flink-scala-assembly-${FLINK_SCALA_VERSION}.jar $FLINK_HOME/lib/

README.md

@@ -1,18 +1,25 @@
-# flink-scala-2.13
+# flink-scala
 
-For now Flink does not support Scala 2.13. For more refer to <https://issues.apache.org/jira/browse/FLINK-13414>.
+This module is a replacement for the `org.apache.flink:flink-scala` lib shipped with flink distribution,
+and allows using flink with higher scala versions than 2.12.8.
 
-Our solution to deploy Scala 2.13 code to Flink, until it's officially supported (or Flink becomes really scala-free):
+For more refer to <https://issues.apache.org/jira/browse/FLINK-13414>.
 
+## Replacing flink-scala in flink distribution
 ```bash
 rm $FLINK_HOME/lib/flink-scala*.jar
-wget https://repo1.maven.org/maven2/pl/touk/flink-scala-2-13_2.13/1.1.2/flink-scala-2-13_2.13-1.1.2-assembly.jar -O $FLINK_HOME/lib/flink-scala-2-13_2.13-1.1.1-assembly.jar
+
+wget https://central.sonatype.com/repository/maven-snapshots/pl/touk/flink-scala_2.13/1.1.3-SNAPSHOT/flink-scala_2.13-1.1.3-SNAPSHOT-assembly.jar -O $FLINK_HOME/lib/flink-scala_2.13-1.1.3-SNAPSHOT-assembly.jar
 ```
 
+## Using as a lib (probably only sufficient when child-first classloading is enabled on flink)
 ```scala
-libraryDependencies += "pl.touk" %% "flink-scala-2-13" % "1.1.2"
+libraryDependencies += "pl.touk" %% "flink-scala" % "1.1.3-SNAPSHOT"
 ```
 
+## Prebuild flink images
+* we provide prebuild flink docker images for scala 2.12 and 2.13 on [Docker Hub](https://hub.docker.com/r/touk/flink)
+
 ## Publishing
 ```
 sbt publishSigned sonatypeBundleRelease

build-images.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -e
+
+VERSION=$(sbt --no-colors -Dsbt.supershell=false "show version" | grep "info" | tail -1 | awk '{print $2}')
+FLINK_VERSION=$(sbt --no-colors -Dsbt.supershell=false "show flinkV" | grep "info" | tail -1 | awk '{print $2}')
+echo "FLINK_SCALA_VERSION: ${VERSION}"
+echo "FLINK_VERSION: ${FLINK_VERSION}"
+
+if [[ "$1" == "--push" ]]; then
+  OUTPUT_TYPE="registry"
+else
+  OUTPUT_TYPE="docker"
+fi
+
+sbt "+clean; +assembly"
+
+IMAGE_TAG="${VERSION}-flink${FLINK_VERSION}-scala_2.12"
+echo "Building Docker image with version: $IMAGE_TAG"
+
+cp target/scala-2.12/flink-scala-assembly-${VERSION}.jar .
+
+docker buildx build \
+  --build-arg FLINK_VERSION=$FLINK_VERSION \
+  --build-arg FLINK_SCALA_VERSION=$VERSION \
+  --platform linux/amd64,linux/arm64 \
+  --tag touk/flink:$IMAGE_TAG \
+  --output=type=$OUTPUT_TYPE .
+
+IMAGE_TAG="${VERSION}-flink${FLINK_VERSION}-scala_2.13"
+echo "Building Docker image with version: $IMAGE_TAG"
+
+cp target/scala-2.13/flink-scala-assembly-${VERSION}.jar .
+
+docker buildx build \
+  --build-arg FLINK_VERSION=$FLINK_VERSION \
+  --build-arg FLINK_SCALA_VERSION=$VERSION \
+  --platform linux/amd64,linux/arm64 \
+  --tag touk/flink:$IMAGE_TAG \
+  --output=type=$OUTPUT_TYPE .

build.sbt

@@ -1,67 +1,103 @@
-version := "1.1.2"
+import sbtassembly.MergeStrategy
 
-scalaVersion := "2.13.15"
+name := "flink-scala"
+version := "1.1.3-SNAPSHOT"
 
-name := "flink-scala-2.13"
+val scala212 = "2.12.20"
+val scala213 = "2.13.16"
 
-lazy val flinkV = "1.16.2"
-lazy val scalaTestV = "3.2.17"
+scalaVersion := scala212
+crossScalaVersions := List(scala212, scala213)
 
+val flinkV = settingKey[String]("Flink version") // to extract using `show flinkV`
+flinkV := "1.20.1"
 
-assembly / artifact := {
-  val art = (assembly / artifact).value
-  art.withClassifier(Some("assembly"))
-}
+lazy val scalaTestV = "3.2.19"
 
-addArtifact(assembly / artifact, assembly)
+lazy val assemblySettings = Seq(
+  assembly / artifact := {
+    val art = (assembly / artifact).value
+    art.withClassifier(Some("assembly"))
+  },
+  assembly / assemblyMergeStrategy := {
+    case PathList(ps@_*) if ps.last == "module-info.class" => MergeStrategy.discard
+    case x => MergeStrategy.defaultMergeStrategy(x)
+  },
+  addArtifact(assembly / artifact, assembly)
+)
 
 lazy val publishSettings = Seq(
   publishMavenStyle := true,
+  sonatypeCredentialHost := "central.sonatype.com",
   publishTo := {
-      val defaultNexusUrl = "https://oss.sonatype.org/"
-      if (isSnapshot.value)
-        Some("snapshots" at defaultNexusUrl + "content/repositories/snapshots")
-      else {
-        sonatypePublishToBundle.value
-      }
+    if (isSnapshot.value)
+      Some("snapshots" at "https://central.sonatype.com/repository/maven-snapshots/")
+    else {
+      sonatypePublishToBundle.value //todo: full release not tested yet
+    }
   },
   Test / publishArtifact := false,
   //We don't put scm information here, it will be added by release plugin and if scm provided here is different than the one from scm
   //we'll end up with two scm sections and invalid pom...
   pomExtra in Global := {
     <scm>
-      <connection>scm:git:github.com/TouK/flink-scala-2.13.git</connection>
-      <developerConnection>scm:git:[email protected]:TouK/flink-scala-2.13.git</developerConnection>
-      <url>github.com/TouK/flink-scala-2.13</url>
+      <connection>scm:git:github.com/TouK/flink-scala.git</connection>
+      <developerConnection>scm:git:[email protected]:TouK/flink-scala.git</developerConnection>
+      <url>github.com/TouK/flink-scala</url>
     </scm>
-    <developers>
-      <developer>
-        <id>TouK</id>
-        <name>TouK</name>
-        <url>https://touk.pl</url>
-      </developer>
-    </developers>
+      <developers>
+        <developer>
+          <id>TouK</id>
+          <name>TouK</name>
+          <url>https://touk.pl</url>
+        </developer>
+      </developers>
   },
   organization := "pl.touk",
 )
 
 lazy val root = (project in file("."))
   .settings(
-    name := "flink-scala-2.13",
+    name := "flink-scala",
     organization := "pl.touk",
     licenses := Seq("Apache 2" -> url("http://www.apache.org/licenses/LICENSE-2.0.txt")),
-    homepage := Some(url("https://github.com/TouK/flink-scala-2.13")),
-    libraryDependencies ++= {
-      Seq(
-        "org.scala-lang" % "scala-compiler" % scalaVersion.value,
-        "org.scala-lang" % "scala-reflect" % scalaVersion.value,
-
-        "org.apache.flink" % "flink-streaming-java" % flinkV % "provided",
-        "com.twitter" %% "chill" % "0.9.5" exclude("com.esotericsoftware", "kryo-shaded"),
-        "com.esotericsoftware.kryo" % "kryo" % "2.24.0" % "provided",
-
-        "org.scalatest" %% "scalatest" % scalaTestV % "test",
-      )
-    }
+    homepage := Some(url("https://github.com/TouK/flink-scala")),
+    libraryDependencies ++= (forScalaVersion(scalaVersion.value) {
+      case (2, 12) =>
+        Seq(
+          "org.apache.flink" %% "flink-scala" % flinkV.value excludeAll(
+            ExclusionRule(organization = "org.apache.flink", name = "flink-core"),
+            ExclusionRule(organization = "org.apache.flink", name = "flink-java"),
+            ExclusionRule(organization = "org.apache.flink", name = "flink-shaded-asm-9"),
+            ExclusionRule(organization = "org.slf4j", name = "slf4j-api"),
+            ExclusionRule(organization = "com.google.code.findbugs", name = "jsr305"),
+          ),
+          "com.esotericsoftware.kryo" % "kryo" % "2.24.0" % Test,
+          "org.apache.flink" % "flink-java" % flinkV.value % Test,
+        )
+      case (2, 13) =>
+        Seq(
+          "org.apache.flink" % "flink-streaming-java" % flinkV.value % "provided",
+          "com.twitter" %% "chill" % "0.9.5" exclude("com.esotericsoftware", "kryo-shaded"),
+          "com.esotericsoftware.kryo" % "kryo" % "2.24.0" % "provided",
+        )
+    } ++ Seq(
+      "org.scala-lang" % "scala-library" % scalaVersion.value,
+      "org.scala-lang" % "scala-compiler" % scalaVersion.value,
+      "org.scala-lang" % "scala-reflect" % scalaVersion.value,
+      "org.scalatest" %% "scalatest" % scalaTestV % Test,
+    ))
   )
+  .settings(assemblySettings *)
   .settings(publishSettings)
+
+def forScalaVersion[T](version: String)(provide: PartialFunction[(Int, Int), T]): T = {
+  CrossVersion.partialVersion(version) match {
+    case Some((major, minor)) if provide.isDefinedAt((major.toInt, minor.toInt)) =>
+      provide((major.toInt, minor.toInt))
+    case Some(_) =>
+      throw new IllegalArgumentException(s"Scala version $version is not handled")
+    case None =>
+      throw new IllegalArgumentException(s"Invalid Scala version $version")
+  }
+}

project/plugins.sbt

@@ -1,3 +1,3 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.4")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.3.1")
 addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.10.0")
 addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.2.1")

src/main/scala/org/apache/flink/runtime/types/FlinkScalaKryoInstantiator.scala → src/main/scala-2.13/org/apache/flink/runtime/types/FlinkScalaKryoInstantiator.scala

@@ -233,4 +233,4 @@ class ScalaCollectionsRegistrarCompat extends IKryoRegistrar {
       )
     )
   }
-}
+}