From 9b9e7405f61545c9a330fa4740e416950508c1a2 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Wed, 7 Feb 2024 14:46:21 +0100 Subject: [PATCH 01/40] dockerfile --- build.sbt | 19 ++- project/Dependencies.scala | 5 +- project/plugins.sbt | 3 +- server/Dockerfile | 32 ++--- server/Dockerfile_old | 109 ++++++++++++++++++ .../scala/za/co/absa/atum/server/Main.scala | 7 +- .../api/database/TestTransactorProvider.scala | 56 +++++++++ 7 files changed, 201 insertions(+), 30 deletions(-) create mode 100644 server/Dockerfile_old create mode 100644 server/src/main/scala/za/co/absa/atum/server/api/database/TestTransactorProvider.scala diff --git a/build.sbt b/build.sbt index dce47ea74..8ddd2b0be 100644 --- a/build.sbt +++ b/build.sbt @@ -22,7 +22,7 @@ import sbt.Keys.name ThisBuild / organization := "za.co.absa" -ThisBuild / scalaVersion := Versions.scala212 // default version +ThisBuild / scalaVersion := Versions.scala213 // default version ThisBuild / versionScheme := Some("early-semver") @@ -44,7 +44,7 @@ lazy val commonSettings = Seq( ) val mergeStrategy: Def.SettingsDefinition = assembly / assemblyMergeStrategy := { - case PathList("META-INF", _) => MergeStrategy.discard + // case PathList("META-INF", _) => MergeStrategy.discard case "application.conf" => MergeStrategy.concat case "reference.conf" => MergeStrategy.concat case _ => MergeStrategy.first @@ -70,13 +70,24 @@ lazy val root = (projectMatrix in file(".")) lazy val server = (projectMatrix in file("server")) .settings( commonSettings ++ Seq( + assemblyMergeStrategy in assembly := { + case PathList("META-INF", "maven", "org.webjars", "swagger-ui", "pom.properties") => MergeStrategy.singleOrError + case PathList("META-INF", "resources", "webjars", "swagger-ui", _*) => MergeStrategy.singleOrError + case PathList("META-INF", _*) => MergeStrategy.discard + case PathList("META-INF", "versions", "9", xs @ _*) => MergeStrategy.discard + case PathList("module-info.class") => MergeStrategy.discard + case x => + val oldStrategy = (assemblyMergeStrategy in assembly).value + oldStrategy(x) + }, name := "atum-server", + javacOptions ++= Seq("-source", "11", "-target", "11"), libraryDependencies ++= Dependencies.serverDependencies, scalacOptions ++= Seq("-Ymacro-annotations"), Compile / packageBin / publishArtifact := false, (Compile / compile) := ((Compile / compile) dependsOn printSparkScalaVersion).value, packageBin := (Compile / assembly).value, - artifactPath / (Compile / packageBin) := baseDirectory.value / s"target/${name.value}-${version.value}.war", + artifactPath / (Compile / packageBin) := baseDirectory.value / s"target/${name.value}-${version.value}.jar", webappWebInfClasses := true, inheritJarManifest := true, testFrameworks += new TestFramework("zio.test.sbt.ZTestFramework") @@ -87,7 +98,7 @@ lazy val server = (projectMatrix in file("server")) jacocoExcludes := jacocoProjectExcludes() ) .enablePlugins(AssemblyPlugin) - .enablePlugins(TomcatPlugin) + // .enablePlugins(TomcatPlugin) .enablePlugins(AutomateHeaderPlugin) .jvmPlatform(scalaVersions = Seq(Versions.serviceScalaVersion)) .dependsOn(model) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index cf6de2eb0..05460e01a 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -61,7 +61,7 @@ object Dependencies { val sbtJunitInterface = "0.13.3" val tapir = "1.9.6" val http4sBlazeBackend = "0.23.15" - val playJson = "2.9.4" + val playJson = "3.0.1"//"2.9.4" val postgresql = "42.5.4" } @@ -128,7 +128,8 @@ object Dependencies { val tapirOrg = "com.softwaremill.sttp.tapir" val http4sOrg = "org.http4s" val faDbOrg = "za.co.absa.fa-db" - val playOrg = "com.typesafe.play" +// val playOrg = "com.typesafe.play" + val playOrg = "org.playframework" val sbtOrg = "com.github.sbt" // zio diff --git a/project/plugins.sbt b/project/plugins.sbt index 9004f7d68..7b4472c43 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -23,7 +23,8 @@ addSbtPlugin("com.eed3si9n" % "sbt-projectmatrix" % "0.9.1") addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.12") // Plugins to build the server module as a war file -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.10") +//addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.10") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.5") // sbt-jacoco dependency downloading lazy val ow2Version = "9.5" diff --git a/server/Dockerfile b/server/Dockerfile index a5bb8f1ac..041e9f0fa 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -32,8 +32,8 @@ # Conditional Docker image creation idea: https://stackoverflow.com/a/60820156/1773349 # change to OFF to disable SSL ARG SSL=true -# Tomcat OS base image, feel free to use another (e.g. your hardened one) -ARG BASE_IMAGE=tomcat:9-jdk8-corretto +# OpenJDK base image +ARG BASE_IMAGE=openjdk:11-jdk-slim # --- Base image part (common for SSL true|false) --- FROM $BASE_IMAGE as base @@ -41,7 +41,7 @@ FROM $BASE_IMAGE as base # Provide your proxy if needed, e.g. http://my.proxy.examle.com:3128 ARG BUILD_PROXY # Override of the example application config is possible -ARG CONFIG=./src/main/resources/application.properties +ARG CONFIG=./src/main/resources/resource.conf # Provide path to the directory with LDAP certs in PEM format ARG LDAP_SSL_CERTS_PATH # ARG SSL_DNAME is defined below in the SSL-enabled image @@ -52,13 +52,11 @@ ARG SCALA_VERSION=2.13 LABEL org.opencontainers.image.authors="ABSA" -# Copy Spring application properties -COPY $CONFIG /opt/application.properties +# The application's jar file +ARG JAR_FILE=${AS_PREFIX}/target/jvm-${SCALA_VERSION}/*.jar -# deploy as root application in tomcat -COPY ${AS_PREFIX}/target/jvm-${SCALA_VERSION}/*.war /usr/local/tomcat/webapps/ROOT.war - -ENV SPRING_CONFIG_LOCATION=/opt/application.properties +# Add the application's jar to the container +ADD ${JAR_FILE} app.jar ENV http_proxy=$BUILD_PROXY ENV https_proxy=$BUILD_PROXY @@ -74,27 +72,21 @@ do \ echo yes | keytool -import -file $file -alias ldaps$RANDOM -keystore $JAVA_HOME/jre/lib/security/cacerts -storepass changeit; \ done -# uncomment and add packages you would need to install via yum if any -#RUN yum -y update -#RUN yum -y install htop procps - # --- SSL=true image specifics --- FROM base AS base-ssl-true ENV SSL_ENABLED=true RUN echo "This stage sets SSL=$SSL_ENABLED" -# Enable SSL: Add our SSL connector after is found in server.xml; .bak = backup -RUN sed -i.bak '//a ' /usr/local/tomcat/conf/server.xml # DNAME for self-signed cert, only applied for SSL=true ARG SSL_DNAME="CN=*.my.example.com, OU=project1, O=yourcompany, L=Johannesburg, ST=Gauteng, C=za" # A self-seigned certificate for HTTPS -RUN keytool -genkeypair -keyalg RSA -alias tomcat -keysize 2048 \ +RUN keytool -genkeypair -keyalg RSA -alias selfsigned -keysize 2048 \ -dname "$SSL_DNAME" \ - -validity 365 -storepass changeit -keystore /usr/local/tomcat/conf/selfsigned.p12 -storetype PKCS12 + -validity 365 -storepass changeit -keystore /usr/local/openjdk-11/conf/selfsigned.p12 -storetype PKCS12 EXPOSE 8080 8443 -CMD ["catalina.sh", "run"] +ENTRYPOINT ["java","-jar","/app.jar"] # --- SSL=false image specifics --- FROM base AS base-ssl-false @@ -102,8 +94,8 @@ ENV SSL_ENABLED=false RUN echo "This stage sets SSL=$SSL_ENABLED" EXPOSE 8080 -CMD ["catalina.sh", "run"] +ENTRYPOINT ["java","-jar","/app.jar"] # --- Final image assembly --- FROM base-ssl-${SSL} AS final -RUN echo "Running final stage with SSL_ENABLED=$SSL_ENABLED." +RUN echo "Running final stage with SSL_ENABLED=$SSL_ENABLED." \ No newline at end of file diff --git a/server/Dockerfile_old b/server/Dockerfile_old new file mode 100644 index 000000000..a5bb8f1ac --- /dev/null +++ b/server/Dockerfile_old @@ -0,0 +1,109 @@ +# +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +# Atum Service docker file + +# build via (docker root = project root): +# docker build -t absaoss/atum-service:latest \ +# --build-arg BUILD_PROXY=http://my.cool.proxy.here:3128 \ +# --build-arg CONFIG=./path/to/my.awesome.local.application.properties \ +# --build-arg SSL=true \ +# --build-arg SSL_DNAME="CN=*.my.domain.com, OU=project1, O=mycorp, L=Johannesburg, ST=Gauteng, C=za" . +# run via +# docker run -p 8080:8080 -p 8443:8443 absaoss/atum-service:latest + +# test via: +# http://localhost:8080/token/public-key +# https://localhost:8443/token/public-key + +# Conditional Docker image creation idea: https://stackoverflow.com/a/60820156/1773349 +# change to OFF to disable SSL +ARG SSL=true +# Tomcat OS base image, feel free to use another (e.g. your hardened one) +ARG BASE_IMAGE=tomcat:9-jdk8-corretto + +# --- Base image part (common for SSL true|false) --- +FROM $BASE_IMAGE as base + +# Provide your proxy if needed, e.g. http://my.proxy.examle.com:3128 +ARG BUILD_PROXY +# Override of the example application config is possible +ARG CONFIG=./src/main/resources/application.properties +# Provide path to the directory with LDAP certs in PEM format +ARG LDAP_SSL_CERTS_PATH +# ARG SSL_DNAME is defined below in the SSL-enabled image +# In case you build the Dockerfile from another location than the default 'service' dir, provide a prefix to reach it +ARG AS_PREFIX=. +# Version of scala that server was built against +ARG SCALA_VERSION=2.13 + +LABEL org.opencontainers.image.authors="ABSA" + +# Copy Spring application properties +COPY $CONFIG /opt/application.properties + +# deploy as root application in tomcat +COPY ${AS_PREFIX}/target/jvm-${SCALA_VERSION}/*.war /usr/local/tomcat/webapps/ROOT.war + +ENV SPRING_CONFIG_LOCATION=/opt/application.properties + +ENV http_proxy=$BUILD_PROXY +ENV https_proxy=$BUILD_PROXY +ENV HTTP_PROXY=$BUILD_PROXY +ENV HTTPS_PROXY=$BUILD_PROXY + +RUN mkdir -p /opt/certs + +COPY $LDAP_SSL_CERTS_PATH /opt/certs/ + +RUN for file in `ls /opt/certs/*.pem`; \ +do \ + echo yes | keytool -import -file $file -alias ldaps$RANDOM -keystore $JAVA_HOME/jre/lib/security/cacerts -storepass changeit; \ +done + +# uncomment and add packages you would need to install via yum if any +#RUN yum -y update +#RUN yum -y install htop procps + +# --- SSL=true image specifics --- +FROM base AS base-ssl-true +ENV SSL_ENABLED=true +RUN echo "This stage sets SSL=$SSL_ENABLED" +# Enable SSL: Add our SSL connector after is found in server.xml; .bak = backup +RUN sed -i.bak '//a ' /usr/local/tomcat/conf/server.xml + +# DNAME for self-signed cert, only applied for SSL=true +ARG SSL_DNAME="CN=*.my.example.com, OU=project1, O=yourcompany, L=Johannesburg, ST=Gauteng, C=za" + +# A self-seigned certificate for HTTPS +RUN keytool -genkeypair -keyalg RSA -alias tomcat -keysize 2048 \ + -dname "$SSL_DNAME" \ + -validity 365 -storepass changeit -keystore /usr/local/tomcat/conf/selfsigned.p12 -storetype PKCS12 + +EXPOSE 8080 8443 +CMD ["catalina.sh", "run"] + +# --- SSL=false image specifics --- +FROM base AS base-ssl-false +ENV SSL_ENABLED=false +RUN echo "This stage sets SSL=$SSL_ENABLED" + +EXPOSE 8080 +CMD ["catalina.sh", "run"] + +# --- Final image assembly --- +FROM base-ssl-${SSL} AS final +RUN echo "Running final stage with SSL_ENABLED=$SSL_ENABLED." diff --git a/server/src/main/scala/za/co/absa/atum/server/Main.scala b/server/src/main/scala/za/co/absa/atum/server/Main.scala index 9c0d21d0d..2d4f59eb9 100644 --- a/server/src/main/scala/za/co/absa/atum/server/Main.scala +++ b/server/src/main/scala/za/co/absa/atum/server/Main.scala @@ -17,7 +17,7 @@ package za.co.absa.atum.server import za.co.absa.atum.server.api.controller._ -import za.co.absa.atum.server.api.database.{PostgresDatabaseProvider, TransactorProvider} +import za.co.absa.atum.server.api.database.{PostgresDatabaseProvider, TestTransactorProvider, TransactorProvider} import za.co.absa.atum.server.api.database.runs.functions.{CreatePartitioningIfNotExists, WriteCheckpoint} import za.co.absa.atum.server.api.http.Server import za.co.absa.atum.server.api.repository.{CheckpointRepositoryImpl, PartitioningRepositoryImpl} @@ -42,8 +42,9 @@ object Main extends ZIOAppDefault with Server { CreatePartitioningIfNotExists.layer, WriteCheckpoint.layer, PostgresDatabaseProvider.layer, - TransactorProvider.layer, - zio.Scope.default +// TransactorProvider.layer, + TestTransactorProvider.layerWithoutRollback, +// zio.Scope.default ) override val bootstrap: ZLayer[Any, Config.Error, Unit] = diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/TestTransactorProvider.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/TestTransactorProvider.scala new file mode 100644 index 000000000..dcee99726 --- /dev/null +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/TestTransactorProvider.scala @@ -0,0 +1,56 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.server.api.database + +import doobie.HC +import doobie.util.transactor.{Strategy, Transactor} +import za.co.absa.atum.server.config.PostgresConfig +import zio._ +import zio.interop.catz._ + +object TestTransactorProvider { + + val layerWithoutRollback: ZLayer[Any, Config.Error, Transactor[Task]] = ZLayer { + for { + postgresConfig <- ZIO.config[PostgresConfig](PostgresConfig.config) + transactor <- ZIO.succeed( + Transactor.fromDriverManager[Task]( + postgresConfig.dataSourceClass, + s"jdbc:postgresql://${postgresConfig.serverName}:${postgresConfig.portNumber}/${postgresConfig.databaseName}", + postgresConfig.user, + postgresConfig.password + ) + ) + } yield transactor + } + + val layerWithRollback: ZLayer[Any, Config.Error, Transactor[Task]] = ZLayer { + for { + postgresConfig <- ZIO.config[PostgresConfig](PostgresConfig.config) + transactor <- ZIO.succeed( + Transactor.fromDriverManager[Task]( + postgresConfig.dataSourceClass, + s"jdbc:postgresql://${postgresConfig.serverName}:${postgresConfig.portNumber}/${postgresConfig.databaseName}", + postgresConfig.user, + postgresConfig.password + ) + ) + transactorWithRollback = Transactor.strategy.set(transactor, Strategy.default.copy(after = HC.rollback)) + } yield transactorWithRollback + } + +} From a9190517dc8e7604679c9e8b7985f04936417e3a Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Wed, 7 Feb 2024 15:21:07 +0100 Subject: [PATCH 02/40] dockerfile --- server/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/Dockerfile b/server/Dockerfile index 041e9f0fa..799e9ab81 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -19,7 +19,7 @@ # build via (docker root = project root): # docker build -t absaoss/atum-service:latest \ # --build-arg BUILD_PROXY=http://my.cool.proxy.here:3128 \ -# --build-arg CONFIG=./path/to/my.awesome.local.application.properties \ +# --build-arg CONFIG=./path/to/my.awesome.local.reference.conf \ # --build-arg SSL=true \ # --build-arg SSL_DNAME="CN=*.my.domain.com, OU=project1, O=mycorp, L=Johannesburg, ST=Gauteng, C=za" . # run via From 139529d4af8bda246745d8962b67e070a3966962 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Thu, 8 Feb 2024 09:11:54 +0100 Subject: [PATCH 03/40] dockerfile --- server/Dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/server/Dockerfile b/server/Dockerfile index 799e9ab81..1b16b66fa 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -32,8 +32,8 @@ # Conditional Docker image creation idea: https://stackoverflow.com/a/60820156/1773349 # change to OFF to disable SSL ARG SSL=true -# OpenJDK base image -ARG BASE_IMAGE=openjdk:11-jdk-slim +# Amazon correto base image +ARG BASE_IMAGE=amazoncorretto:11.0.22 # --- Base image part (common for SSL true|false) --- FROM $BASE_IMAGE as base @@ -69,7 +69,7 @@ COPY $LDAP_SSL_CERTS_PATH /opt/certs/ RUN for file in `ls /opt/certs/*.pem`; \ do \ - echo yes | keytool -import -file $file -alias ldaps$RANDOM -keystore $JAVA_HOME/jre/lib/security/cacerts -storepass changeit; \ + echo yes | keytool -import -file $file -alias ldaps$RANDOM -keystore /usr/lib/jvm/java-11-amazon-corretto/lib/security/cacerts -storepass changeit; \ done # --- SSL=true image specifics --- @@ -83,7 +83,9 @@ ARG SSL_DNAME="CN=*.my.example.com, OU=project1, O=yourcompany, L=Johannesburg, # A self-seigned certificate for HTTPS RUN keytool -genkeypair -keyalg RSA -alias selfsigned -keysize 2048 \ -dname "$SSL_DNAME" \ - -validity 365 -storepass changeit -keystore /usr/local/openjdk-11/conf/selfsigned.p12 -storetype PKCS12 +# /usr/local/openjdk-11/conf/selfsigned.p12 +# -validity 365 -storepass changeit -keystore /usr/local/java-11-amazon-corretto/conf/selfsigned.p12 -storetype PKCS12 + -validity 365 -storepass changeit -keystore /etc/ssl/certs/selfsigned.p12 -storetype PKCS12 EXPOSE 8080 8443 ENTRYPOINT ["java","-jar","/app.jar"] From 64c9d0bf44bddb82517b46c2a34e482a08fda7ef Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Thu, 8 Feb 2024 09:13:32 +0100 Subject: [PATCH 04/40] dockerfile --- server/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/Dockerfile b/server/Dockerfile index 1b16b66fa..c23683b4e 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -83,8 +83,6 @@ ARG SSL_DNAME="CN=*.my.example.com, OU=project1, O=yourcompany, L=Johannesburg, # A self-seigned certificate for HTTPS RUN keytool -genkeypair -keyalg RSA -alias selfsigned -keysize 2048 \ -dname "$SSL_DNAME" \ -# /usr/local/openjdk-11/conf/selfsigned.p12 -# -validity 365 -storepass changeit -keystore /usr/local/java-11-amazon-corretto/conf/selfsigned.p12 -storetype PKCS12 -validity 365 -storepass changeit -keystore /etc/ssl/certs/selfsigned.p12 -storetype PKCS12 EXPOSE 8080 8443 From 850408020a2f7b242235acc95696f1672a804a87 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Thu, 8 Feb 2024 21:15:41 +0100 Subject: [PATCH 05/40] ssl stuff --- .gitignore | 3 +++ server/Dockerfile | 2 +- server/src/main/resources/reference.conf | 2 +- .../scala/za/co/absa/atum/server/Main.scala | 2 +- .../co/absa/atum/server/api/http/Server.scala | 24 ++++++++++++++++++- 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index e946ec93d..10ec50bc8 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,6 @@ _site utils/resources/*.conf .bsp +/server/certs/ +/server/selfsigned.crt +/server/selfsigned.p12 diff --git a/server/Dockerfile b/server/Dockerfile index c23683b4e..bbf05b6c4 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -31,7 +31,7 @@ # Conditional Docker image creation idea: https://stackoverflow.com/a/60820156/1773349 # change to OFF to disable SSL -ARG SSL=true +ARG SSL # Amazon correto base image ARG BASE_IMAGE=amazoncorretto:11.0.22 diff --git a/server/src/main/resources/reference.conf b/server/src/main/resources/reference.conf index 7dabe41e0..251c29391 100644 --- a/server/src/main/resources/reference.conf +++ b/server/src/main/resources/reference.conf @@ -5,7 +5,7 @@ # log filter filter { # rootLevel sets the minimum level of log messages that will be displayed - rootLevel = INFO + rootLevel = ERROR } } postgres { diff --git a/server/src/main/scala/za/co/absa/atum/server/Main.scala b/server/src/main/scala/za/co/absa/atum/server/Main.scala index 2d4f59eb9..716e42363 100644 --- a/server/src/main/scala/za/co/absa/atum/server/Main.scala +++ b/server/src/main/scala/za/co/absa/atum/server/Main.scala @@ -23,7 +23,7 @@ import za.co.absa.atum.server.api.http.Server import za.co.absa.atum.server.api.repository.{CheckpointRepositoryImpl, PartitioningRepositoryImpl} import za.co.absa.atum.server.api.service.{CheckpointServiceImpl, PartitioningServiceImpl} import zio.config.typesafe.TypesafeConfigProvider -import zio.logging.consoleLogger +import zio.logging.{consoleLogger, loggerConfigPath} import zio._ object Main extends ZIOAppDefault with Server { diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Server.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Server.scala index f38fd5d63..8d8cc88ef 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Server.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Server.scala @@ -31,13 +31,17 @@ import sttp.tapir.server.interceptor.decodefailure.DefaultDecodeFailureHandler.r import sttp.tapir.server.model.ValuedEndpointOutput import sttp.tapir.swagger.bundle.SwaggerInterpreter import sttp.tapir.ztapir._ -import sttp.tapir.{DecodeResult, Endpoint, PublicEndpoint, headers, statusCode} +import sttp.tapir.{DecodeResult, PublicEndpoint, headers, statusCode} import za.co.absa.atum.server.Constants.{SwaggerApiName, SwaggerApiVersion} import za.co.absa.atum.server.api.controller._ import za.co.absa.atum.server.model.BadRequestResponse import zio.interop.catz._ import zio.{RIO, ZIO} +import java.io.FileInputStream +import java.security.{KeyStore, SecureRandom} +import javax.net.ssl.{KeyManagerFactory, SSLContext, TrustManagerFactory} + trait Server extends Endpoints { type Env = PartitioningController with CheckpointController @@ -88,9 +92,27 @@ trait Server extends Endpoints { .toRoutes } + val password: Array[Char] = "changeit".toCharArray // replace with your keystore password + + val ks: KeyStore = KeyStore.getInstance("PKCS12") + val fis = new FileInputStream("/etc/ssl/certs/selfsigned.p12") // replace with your keystore path + ks.load(fis, password) + + val kmf: KeyManagerFactory = KeyManagerFactory.getInstance("SunX509") + kmf.init(ks, password) + + val tmf: TrustManagerFactory = TrustManagerFactory.getInstance("SunX509") + tmf.init(ks) + + val sslContext: SSLContext = SSLContext.getInstance("SSL") +// val sslContext: SSLContext = SSLContext.getInstance("TLS") +// sslContext.init(kmf.getKeyManagers, tmf.getTrustManagers, new SecureRandom) + sslContext.init(kmf.getKeyManagers, null, null) + protected val server: ZIO[Env, Throwable, Unit] = ZIO.executor.flatMap { executor => BlazeServerBuilder[F] + .withSslContext(sslContext) .withExecutionContext(executor.asExecutionContext) .withHttpApp(Router("/" -> (createAllServerRoutes <+> createSwaggerRoutes)).orNotFound) .serve From 43096ba9808802a6edeac807b15232beed7cdc11 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Fri, 9 Feb 2024 22:02:32 +0100 Subject: [PATCH 06/40] ssl context, http and https servers, logging backend, configuration, dockerfile --- project/Dependencies.scala | 7 +- server/Dockerfile | 7 +- server/Dockerfile_old | 109 ------------------ server/src/main/resources/logback.xml | 13 +++ server/src/main/resources/reference.conf | 16 +-- .../scala/za/co/absa/atum/server/Main.scala | 13 +-- .../api/database/TestTransactorProvider.scala | 56 --------- .../za/co/absa/atum/server/api/http/SSL.scala | 39 +++++++ .../co/absa/atum/server/api/http/Server.scala | 46 +++----- .../absa/atum/server/config/SslConfig.scala | 14 +++ server/src/test/resources/logback-test.xml | 12 ++ server/src/test/resources/reference.conf | 9 -- 12 files changed, 118 insertions(+), 223 deletions(-) delete mode 100644 server/Dockerfile_old create mode 100644 server/src/main/resources/logback.xml delete mode 100644 server/src/main/scala/za/co/absa/atum/server/api/database/TestTransactorProvider.scala create mode 100644 server/src/main/scala/za/co/absa/atum/server/api/http/SSL.scala create mode 100644 server/src/main/scala/za/co/absa/atum/server/config/SslConfig.scala create mode 100644 server/src/test/resources/logback-test.xml diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 05460e01a..d9dd021ee 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -57,6 +57,7 @@ object Dependencies { val zio = "2.0.19" val zioLogging = "2.2.0" + val logbackZio = "1.4.7" val zioConfig = "4.0.1" val sbtJunitInterface = "0.13.3" val tapir = "1.9.6" @@ -128,14 +129,16 @@ object Dependencies { val tapirOrg = "com.softwaremill.sttp.tapir" val http4sOrg = "org.http4s" val faDbOrg = "za.co.absa.fa-db" -// val playOrg = "com.typesafe.play" val playOrg = "org.playframework" val sbtOrg = "com.github.sbt" + val logbackOrg = "ch.qos.logback" // zio lazy val zioCore = zioOrg %% "zio" % Versions.zio lazy val zioMacros = zioOrg %% "zio-macros" % Versions.zio lazy val zioLogging = zioOrg %% "zio-logging" % Versions.zioLogging + lazy val slf4jLogging = zioOrg %% "zio-logging-slf4j2" % Versions.zioLogging + lazy val logback = logbackOrg % "logback-classic" % Versions.logbackZio lazy val zioConfig = zioOrg %% "zio-config" % Versions.zioConfig lazy val zioConfigMagnolia = zioOrg %% "zio-config-magnolia" % Versions.zioConfig lazy val zioConfigTypesafe = zioOrg %% "zio-config-typesafe" % Versions.zioConfig @@ -165,6 +168,8 @@ object Dependencies { zioCore, zioMacros, zioLogging, + slf4jLogging, + logback, zioConfig, zioConfigMagnolia, zioConfigTypesafe, diff --git a/server/Dockerfile b/server/Dockerfile index bbf05b6c4..7a8a54afa 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -13,7 +13,6 @@ # limitations under the License. # - # Atum Service docker file # build via (docker root = project root): @@ -30,7 +29,7 @@ # https://localhost:8443/token/public-key # Conditional Docker image creation idea: https://stackoverflow.com/a/60820156/1773349 -# change to OFF to disable SSL +# change to false to disable SSL ARG SSL # Amazon correto base image ARG BASE_IMAGE=amazoncorretto:11.0.22 @@ -80,10 +79,10 @@ RUN echo "This stage sets SSL=$SSL_ENABLED" # DNAME for self-signed cert, only applied for SSL=true ARG SSL_DNAME="CN=*.my.example.com, OU=project1, O=yourcompany, L=Johannesburg, ST=Gauteng, C=za" -# A self-seigned certificate for HTTPS +# A self-signed certificate for HTTPS RUN keytool -genkeypair -keyalg RSA -alias selfsigned -keysize 2048 \ -dname "$SSL_DNAME" \ - -validity 365 -storepass changeit -keystore /etc/ssl/certs/selfsigned.p12 -storetype PKCS12 + -validity 365 -storepass changeit -keystore /etc/ssl/certs/selfsigned.jks -storetype JKS EXPOSE 8080 8443 ENTRYPOINT ["java","-jar","/app.jar"] diff --git a/server/Dockerfile_old b/server/Dockerfile_old deleted file mode 100644 index a5bb8f1ac..000000000 --- a/server/Dockerfile_old +++ /dev/null @@ -1,109 +0,0 @@ -# -# Copyright 2021 ABSA Group Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -# Atum Service docker file - -# build via (docker root = project root): -# docker build -t absaoss/atum-service:latest \ -# --build-arg BUILD_PROXY=http://my.cool.proxy.here:3128 \ -# --build-arg CONFIG=./path/to/my.awesome.local.application.properties \ -# --build-arg SSL=true \ -# --build-arg SSL_DNAME="CN=*.my.domain.com, OU=project1, O=mycorp, L=Johannesburg, ST=Gauteng, C=za" . -# run via -# docker run -p 8080:8080 -p 8443:8443 absaoss/atum-service:latest - -# test via: -# http://localhost:8080/token/public-key -# https://localhost:8443/token/public-key - -# Conditional Docker image creation idea: https://stackoverflow.com/a/60820156/1773349 -# change to OFF to disable SSL -ARG SSL=true -# Tomcat OS base image, feel free to use another (e.g. your hardened one) -ARG BASE_IMAGE=tomcat:9-jdk8-corretto - -# --- Base image part (common for SSL true|false) --- -FROM $BASE_IMAGE as base - -# Provide your proxy if needed, e.g. http://my.proxy.examle.com:3128 -ARG BUILD_PROXY -# Override of the example application config is possible -ARG CONFIG=./src/main/resources/application.properties -# Provide path to the directory with LDAP certs in PEM format -ARG LDAP_SSL_CERTS_PATH -# ARG SSL_DNAME is defined below in the SSL-enabled image -# In case you build the Dockerfile from another location than the default 'service' dir, provide a prefix to reach it -ARG AS_PREFIX=. -# Version of scala that server was built against -ARG SCALA_VERSION=2.13 - -LABEL org.opencontainers.image.authors="ABSA" - -# Copy Spring application properties -COPY $CONFIG /opt/application.properties - -# deploy as root application in tomcat -COPY ${AS_PREFIX}/target/jvm-${SCALA_VERSION}/*.war /usr/local/tomcat/webapps/ROOT.war - -ENV SPRING_CONFIG_LOCATION=/opt/application.properties - -ENV http_proxy=$BUILD_PROXY -ENV https_proxy=$BUILD_PROXY -ENV HTTP_PROXY=$BUILD_PROXY -ENV HTTPS_PROXY=$BUILD_PROXY - -RUN mkdir -p /opt/certs - -COPY $LDAP_SSL_CERTS_PATH /opt/certs/ - -RUN for file in `ls /opt/certs/*.pem`; \ -do \ - echo yes | keytool -import -file $file -alias ldaps$RANDOM -keystore $JAVA_HOME/jre/lib/security/cacerts -storepass changeit; \ -done - -# uncomment and add packages you would need to install via yum if any -#RUN yum -y update -#RUN yum -y install htop procps - -# --- SSL=true image specifics --- -FROM base AS base-ssl-true -ENV SSL_ENABLED=true -RUN echo "This stage sets SSL=$SSL_ENABLED" -# Enable SSL: Add our SSL connector after is found in server.xml; .bak = backup -RUN sed -i.bak '//a ' /usr/local/tomcat/conf/server.xml - -# DNAME for self-signed cert, only applied for SSL=true -ARG SSL_DNAME="CN=*.my.example.com, OU=project1, O=yourcompany, L=Johannesburg, ST=Gauteng, C=za" - -# A self-seigned certificate for HTTPS -RUN keytool -genkeypair -keyalg RSA -alias tomcat -keysize 2048 \ - -dname "$SSL_DNAME" \ - -validity 365 -storepass changeit -keystore /usr/local/tomcat/conf/selfsigned.p12 -storetype PKCS12 - -EXPOSE 8080 8443 -CMD ["catalina.sh", "run"] - -# --- SSL=false image specifics --- -FROM base AS base-ssl-false -ENV SSL_ENABLED=false -RUN echo "This stage sets SSL=$SSL_ENABLED" - -EXPOSE 8080 -CMD ["catalina.sh", "run"] - -# --- Final image assembly --- -FROM base-ssl-${SSL} AS final -RUN echo "Running final stage with SSL_ENABLED=$SSL_ENABLED." diff --git a/server/src/main/resources/logback.xml b/server/src/main/resources/logback.xml new file mode 100644 index 000000000..6a008063f --- /dev/null +++ b/server/src/main/resources/logback.xml @@ -0,0 +1,13 @@ + + + + + %d{HH:mm:ss.SSS} %-5level [%thread] %logger{36}:%L - %msg %ex{short}%n + + + + + + + + diff --git a/server/src/main/resources/reference.conf b/server/src/main/resources/reference.conf index 251c29391..4029484fb 100644 --- a/server/src/main/resources/reference.conf +++ b/server/src/main/resources/reference.conf @@ -1,17 +1,8 @@ { - logger { - # This format includes timestamp, level, thread (fiberId), message, and cause - format = "%label{timestamp}{%fixed{32}{%timestamp}} %label{level}{%level} %label{thread}{%fiberId} %label{class}{%name:%line} %label{message}{%message} %label{cause}{%cause}" - # log filter - filter { - # rootLevel sets the minimum level of log messages that will be displayed - rootLevel = ERROR - } - } postgres { # The JDBC driver class dataSourceClass=org.postgresql.Driver - serverName=localhost + serverName=127.0.0.1 # set to non-default port, to avoid local collision portNumber=5432 databaseName=atum_db @@ -20,4 +11,9 @@ # maximum number of connections that HikariCP will keep in the pool, including both idle and in-use connections maxPoolSize=10 } + ssl { + enabled=false + keyStorePassword=password + keyStorePath="/your/path/to/selfsigned.jks" + } } diff --git a/server/src/main/scala/za/co/absa/atum/server/Main.scala b/server/src/main/scala/za/co/absa/atum/server/Main.scala index 716e42363..6afb1f2ed 100644 --- a/server/src/main/scala/za/co/absa/atum/server/Main.scala +++ b/server/src/main/scala/za/co/absa/atum/server/Main.scala @@ -17,14 +17,14 @@ package za.co.absa.atum.server import za.co.absa.atum.server.api.controller._ -import za.co.absa.atum.server.api.database.{PostgresDatabaseProvider, TestTransactorProvider, TransactorProvider} import za.co.absa.atum.server.api.database.runs.functions.{CreatePartitioningIfNotExists, WriteCheckpoint} +import za.co.absa.atum.server.api.database.{PostgresDatabaseProvider, TransactorProvider} import za.co.absa.atum.server.api.http.Server import za.co.absa.atum.server.api.repository.{CheckpointRepositoryImpl, PartitioningRepositoryImpl} import za.co.absa.atum.server.api.service.{CheckpointServiceImpl, PartitioningServiceImpl} -import zio.config.typesafe.TypesafeConfigProvider -import zio.logging.{consoleLogger, loggerConfigPath} import zio._ +import zio.config.typesafe.TypesafeConfigProvider +import zio.logging.backend.SLF4J object Main extends ZIOAppDefault with Server { @@ -42,12 +42,11 @@ object Main extends ZIOAppDefault with Server { CreatePartitioningIfNotExists.layer, WriteCheckpoint.layer, PostgresDatabaseProvider.layer, -// TransactorProvider.layer, - TestTransactorProvider.layerWithoutRollback, -// zio.Scope.default + TransactorProvider.layer, + zio.Scope.default ) override val bootstrap: ZLayer[Any, Config.Error, Unit] = - Runtime.removeDefaultLoggers >>> Runtime.setConfigProvider(configProvider) >>> consoleLogger() + Runtime.removeDefaultLoggers >>> SLF4J.slf4j >>> Runtime.setConfigProvider(configProvider) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/TestTransactorProvider.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/TestTransactorProvider.scala deleted file mode 100644 index dcee99726..000000000 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/TestTransactorProvider.scala +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package za.co.absa.atum.server.api.database - -import doobie.HC -import doobie.util.transactor.{Strategy, Transactor} -import za.co.absa.atum.server.config.PostgresConfig -import zio._ -import zio.interop.catz._ - -object TestTransactorProvider { - - val layerWithoutRollback: ZLayer[Any, Config.Error, Transactor[Task]] = ZLayer { - for { - postgresConfig <- ZIO.config[PostgresConfig](PostgresConfig.config) - transactor <- ZIO.succeed( - Transactor.fromDriverManager[Task]( - postgresConfig.dataSourceClass, - s"jdbc:postgresql://${postgresConfig.serverName}:${postgresConfig.portNumber}/${postgresConfig.databaseName}", - postgresConfig.user, - postgresConfig.password - ) - ) - } yield transactor - } - - val layerWithRollback: ZLayer[Any, Config.Error, Transactor[Task]] = ZLayer { - for { - postgresConfig <- ZIO.config[PostgresConfig](PostgresConfig.config) - transactor <- ZIO.succeed( - Transactor.fromDriverManager[Task]( - postgresConfig.dataSourceClass, - s"jdbc:postgresql://${postgresConfig.serverName}:${postgresConfig.portNumber}/${postgresConfig.databaseName}", - postgresConfig.user, - postgresConfig.password - ) - ) - transactorWithRollback = Transactor.strategy.set(transactor, Strategy.default.copy(after = HC.rollback)) - } yield transactorWithRollback - } - -} diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/SSL.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/SSL.scala new file mode 100644 index 000000000..fafc72120 --- /dev/null +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/SSL.scala @@ -0,0 +1,39 @@ +package za.co.absa.atum.server.api.http + +import za.co.absa.atum.server.config.SslConfig +import zio.ZIO + +import java.io.FileInputStream +import java.security.{KeyStore, SecureRandom} +import javax.net.ssl.{KeyManagerFactory, SSLContext, TrustManagerFactory} + +object SSL { + + def context: ZIO[Any, Throwable, SSLContext] = { + for { + sslConfig <- ZIO.config[SslConfig](SslConfig.config) + keyStore <- ZIO.attempt { + val keyStore = KeyStore.getInstance(KeyStore.getDefaultType) + val in = new FileInputStream(sslConfig.keyStorePath) + keyStore.load(in, sslConfig.keyStorePassword.toCharArray) + keyStore + } + keyManagerFactory <- ZIO.attempt { + val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm) + keyManagerFactory.init(keyStore, sslConfig.keyStorePassword.toCharArray) + keyManagerFactory + } + trustManagerFactory <- ZIO.attempt { + val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm) + trustManagerFactory.init(keyStore) + trustManagerFactory + } + sslContext <- ZIO.attempt { + val sslContext = SSLContext.getInstance("TLS") + sslContext.init(keyManagerFactory.getKeyManagers, trustManagerFactory.getTrustManagers, new SecureRandom()) + sslContext + } + } yield sslContext + } + +} diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Server.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Server.scala index 8d8cc88ef..9acde0a67 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Server.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Server.scala @@ -34,13 +34,12 @@ import sttp.tapir.ztapir._ import sttp.tapir.{DecodeResult, PublicEndpoint, headers, statusCode} import za.co.absa.atum.server.Constants.{SwaggerApiName, SwaggerApiVersion} import za.co.absa.atum.server.api.controller._ +import za.co.absa.atum.server.config.SslConfig import za.co.absa.atum.server.model.BadRequestResponse import zio.interop.catz._ -import zio.{RIO, ZIO} +import zio._ -import java.io.FileInputStream -import java.security.{KeyStore, SecureRandom} -import javax.net.ssl.{KeyManagerFactory, SSLContext, TrustManagerFactory} +import javax.net.ssl.SSLContext trait Server extends Endpoints { @@ -92,32 +91,25 @@ trait Server extends Endpoints { .toRoutes } - val password: Array[Char] = "changeit".toCharArray // replace with your keystore password - - val ks: KeyStore = KeyStore.getInstance("PKCS12") - val fis = new FileInputStream("/etc/ssl/certs/selfsigned.p12") // replace with your keystore path - ks.load(fis, password) - - val kmf: KeyManagerFactory = KeyManagerFactory.getInstance("SunX509") - kmf.init(ks, password) - - val tmf: TrustManagerFactory = TrustManagerFactory.getInstance("SunX509") - tmf.init(ks) - - val sslContext: SSLContext = SSLContext.getInstance("SSL") -// val sslContext: SSLContext = SSLContext.getInstance("TLS") -// sslContext.init(kmf.getKeyManagers, tmf.getTrustManagers, new SecureRandom) - sslContext.init(kmf.getKeyManagers, null, null) - - protected val server: ZIO[Env, Throwable, Unit] = + private def createServer(port: Int, sslContext: Option[SSLContext] = None): ZIO[Env, Throwable, Unit] = ZIO.executor.flatMap { executor => - BlazeServerBuilder[F] - .withSslContext(sslContext) + val builder = BlazeServerBuilder[F] + .bindHttp(port, "0.0.0.0") .withExecutionContext(executor.asExecutionContext) .withHttpApp(Router("/" -> (createAllServerRoutes <+> createSwaggerRoutes)).orNotFound) - .serve - .compile - .drain + + val builderWithSsl = sslContext.fold(builder)(ctx => builder.withSslContext(ctx)) + builderWithSsl.serve.compile.drain } + private def httpServer: ZIO[Env, Throwable, Unit] = createServer(8080) + private def httpsServer: ZIO[Env, Throwable, Unit] = SSL.context.flatMap { context => + createServer(8443, Some(context)) + } + + protected val server: ZIO[Env, Throwable, Unit] = for { + sslConfig <- ZIO.config[SslConfig](SslConfig.config) + server <- if (sslConfig.enabled) httpsServer else httpServer + } yield server + } diff --git a/server/src/main/scala/za/co/absa/atum/server/config/SslConfig.scala b/server/src/main/scala/za/co/absa/atum/server/config/SslConfig.scala new file mode 100644 index 000000000..47bf90526 --- /dev/null +++ b/server/src/main/scala/za/co/absa/atum/server/config/SslConfig.scala @@ -0,0 +1,14 @@ +package za.co.absa.atum.server.config + +import zio.Config +import zio.config.magnolia.deriveConfig + +case class SslConfig( + enabled: Boolean, + keyStorePassword: String, + keyStorePath: String +) + +object SslConfig { + val config: Config[SslConfig] = deriveConfig[SslConfig].nested("ssl") +} diff --git a/server/src/test/resources/logback-test.xml b/server/src/test/resources/logback-test.xml new file mode 100644 index 000000000..6e2afba1d --- /dev/null +++ b/server/src/test/resources/logback-test.xml @@ -0,0 +1,12 @@ + + + + + %d{HH:mm:ss.SSS} [%thread] [%kvp] %-5level %logger{36} %msg%n + + + + + + + diff --git a/server/src/test/resources/reference.conf b/server/src/test/resources/reference.conf index 7b8fc5b4f..9279d9667 100644 --- a/server/src/test/resources/reference.conf +++ b/server/src/test/resources/reference.conf @@ -1,13 +1,4 @@ { - logger { - # This format includes timestamp, level, thread (fiberId), message, and cause - format = "%label{timestamp}{%fixed{32}{%timestamp}} %label{level}{%level} %label{thread}{%fiberId} %label{class}{%name:%line} %label{message}{%message} %label{cause}{%cause}" - # log filter - filter { - # rootLevel sets the minimum level of log messages that will be displayed - rootLevel = INFO - } - } postgres { # The JDBC driver class dataSourceClass=org.postgresql.Driver From 7112c69cbfbb84f0fad7426d4447072670babf11 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Sat, 10 Feb 2024 20:04:16 +0100 Subject: [PATCH 07/40] sbt setup --- .github/workflows/build.yml | 3 ++- .github/workflows/jacoco_check.yml | 2 +- .github/workflows/jacoco_check_server.yml | 2 +- build.sbt | 11 +++++----- .../za/co/absa/atum/server/api/http/SSL.scala | 21 +++++++++++++++++++ .../absa/atum/server/config/SslConfig.scala | 16 ++++++++++++++ 6 files changed, 46 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 75f68a8c7..d235890d5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,7 +55,8 @@ jobs: - name: Setup Scala uses: olafurpg/setup-scala@v10 with: - java-version: "adopt@1.8" + java-version: "adopt@11" +# java-version: "adopt@1.8" - name: Prepare testing database run: sbt flywayMigrate diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index c3e2671e3..9035c2077 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -43,7 +43,7 @@ jobs: - name: Setup Scala uses: olafurpg/setup-scala@v10 with: - java-version: "adopt@1.8" + java-version: "adopt@11" - name: Build and run tests run: sbt "++${{matrix.scala}}; project agent; jacoco" # Agent module code coverage diff --git a/.github/workflows/jacoco_check_server.yml b/.github/workflows/jacoco_check_server.yml index 6c4c6ee99..e46e9da4f 100644 --- a/.github/workflows/jacoco_check_server.yml +++ b/.github/workflows/jacoco_check_server.yml @@ -53,7 +53,7 @@ jobs: - name: Setup Scala uses: olafurpg/setup-scala@v10 with: - java-version: "adopt@1.8" + java-version: "adopt@11" - name: Prepare testing database run: sbt flywayMigrate - name: Build and run tests diff --git a/build.sbt b/build.sbt index 8ddd2b0be..07b37b364 100644 --- a/build.sbt +++ b/build.sbt @@ -44,7 +44,7 @@ lazy val commonSettings = Seq( ) val mergeStrategy: Def.SettingsDefinition = assembly / assemblyMergeStrategy := { - // case PathList("META-INF", _) => MergeStrategy.discard + case PathList("META-INF", _) => MergeStrategy.discard case "application.conf" => MergeStrategy.concat case "reference.conf" => MergeStrategy.concat case _ => MergeStrategy.first @@ -70,15 +70,15 @@ lazy val root = (projectMatrix in file(".")) lazy val server = (projectMatrix in file("server")) .settings( commonSettings ++ Seq( - assemblyMergeStrategy in assembly := { + assembly / assemblyMergeStrategy := { case PathList("META-INF", "maven", "org.webjars", "swagger-ui", "pom.properties") => MergeStrategy.singleOrError case PathList("META-INF", "resources", "webjars", "swagger-ui", _*) => MergeStrategy.singleOrError case PathList("META-INF", _*) => MergeStrategy.discard case PathList("META-INF", "versions", "9", xs @ _*) => MergeStrategy.discard case PathList("module-info.class") => MergeStrategy.discard - case x => - val oldStrategy = (assemblyMergeStrategy in assembly).value - oldStrategy(x) + case "application.conf" => MergeStrategy.concat + case "reference.conf" => MergeStrategy.concat + case _ => MergeStrategy.first }, name := "atum-server", javacOptions ++= Seq("-source", "11", "-target", "11"), @@ -98,7 +98,6 @@ lazy val server = (projectMatrix in file("server")) jacocoExcludes := jacocoProjectExcludes() ) .enablePlugins(AssemblyPlugin) - // .enablePlugins(TomcatPlugin) .enablePlugins(AutomateHeaderPlugin) .jvmPlatform(scalaVersions = Seq(Versions.serviceScalaVersion)) .dependsOn(model) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/SSL.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/SSL.scala index fafc72120..e03906d7a 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/SSL.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/SSL.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.server.api.http import za.co.absa.atum.server.config.SslConfig @@ -12,27 +28,32 @@ object SSL { def context: ZIO[Any, Throwable, SSLContext] = { for { sslConfig <- ZIO.config[SslConfig](SslConfig.config) + _ <- ZIO.logDebug("Attempting to initialize SSLContext") keyStore <- ZIO.attempt { val keyStore = KeyStore.getInstance(KeyStore.getDefaultType) val in = new FileInputStream(sslConfig.keyStorePath) keyStore.load(in, sslConfig.keyStorePassword.toCharArray) keyStore } + _ <- ZIO.logDebug("KeyStore instantiated") keyManagerFactory <- ZIO.attempt { val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm) keyManagerFactory.init(keyStore, sslConfig.keyStorePassword.toCharArray) keyManagerFactory } + _ <- ZIO.logDebug("KeyManagerFactory initialized") trustManagerFactory <- ZIO.attempt { val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm) trustManagerFactory.init(keyStore) trustManagerFactory } + _ <- ZIO.logDebug("TrustManagerFactory initialized") sslContext <- ZIO.attempt { val sslContext = SSLContext.getInstance("TLS") sslContext.init(keyManagerFactory.getKeyManagers, trustManagerFactory.getTrustManagers, new SecureRandom()) sslContext } + _ <- ZIO.logDebug("SSLContext successfully initialized") } yield sslContext } diff --git a/server/src/main/scala/za/co/absa/atum/server/config/SslConfig.scala b/server/src/main/scala/za/co/absa/atum/server/config/SslConfig.scala index 47bf90526..77b032548 100644 --- a/server/src/main/scala/za/co/absa/atum/server/config/SslConfig.scala +++ b/server/src/main/scala/za/co/absa/atum/server/config/SslConfig.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.server.config import zio.Config From 591b1b2032826505f2d071d156515fdde726e62f Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Sat, 10 Feb 2024 20:08:43 +0100 Subject: [PATCH 08/40] adopt@1.11.0-11 --- .github/workflows/build.yml | 2 +- .github/workflows/jacoco_check.yml | 2 +- .github/workflows/jacoco_check_server.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d235890d5..b8b658c6a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,7 +55,7 @@ jobs: - name: Setup Scala uses: olafurpg/setup-scala@v10 with: - java-version: "adopt@11" + java-version: "adopt@1.11.0-11" # java-version: "adopt@1.8" - name: Prepare testing database diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 9035c2077..79d4ca482 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -43,7 +43,7 @@ jobs: - name: Setup Scala uses: olafurpg/setup-scala@v10 with: - java-version: "adopt@11" + java-version: "adopt@1.11.0-11" - name: Build and run tests run: sbt "++${{matrix.scala}}; project agent; jacoco" # Agent module code coverage diff --git a/.github/workflows/jacoco_check_server.yml b/.github/workflows/jacoco_check_server.yml index e46e9da4f..63ddc033f 100644 --- a/.github/workflows/jacoco_check_server.yml +++ b/.github/workflows/jacoco_check_server.yml @@ -53,7 +53,7 @@ jobs: - name: Setup Scala uses: olafurpg/setup-scala@v10 with: - java-version: "adopt@11" + java-version: "adopt@1.11.0-11" - name: Prepare testing database run: sbt flywayMigrate - name: Build and run tests From 809548608b88e1a35ec4a91f4a1108572ee99348 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Sat, 10 Feb 2024 20:32:28 +0100 Subject: [PATCH 09/40] build.sbt --- build.sbt | 14 +++++++++----- project/plugins.sbt | 1 - 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/build.sbt b/build.sbt index 07b37b364..d033c36c9 100644 --- a/build.sbt +++ b/build.sbt @@ -40,6 +40,7 @@ lazy val commonSettings = Seq( libraryDependencies ++= commonDependencies, scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings"), javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), + scalacOptions ++= Seq("-target:jvm-1.8"), Test / parallelExecution := false ) @@ -62,15 +63,19 @@ lazy val root = (projectMatrix in file(".")) .aggregate(model, server, agent) .settings( name := "atum-service-root", - javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), publish / skip := true, mergeStrategy ) lazy val server = (projectMatrix in file("server")) .settings( - commonSettings ++ Seq( - assembly / assemblyMergeStrategy := { + Seq( + libraryDependencies ++= commonDependencies, + scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings"), + javacOptions ++= Seq("-source", "11", "-target", "11"), + scalacOptions ++= Seq("-target:jvm-11"), + Test / parallelExecution := false, + assembly / assemblyMergeStrategy := { case PathList("META-INF", "maven", "org.webjars", "swagger-ui", "pom.properties") => MergeStrategy.singleOrError case PathList("META-INF", "resources", "webjars", "swagger-ui", _*) => MergeStrategy.singleOrError case PathList("META-INF", _*) => MergeStrategy.discard @@ -79,9 +84,8 @@ lazy val server = (projectMatrix in file("server")) case "application.conf" => MergeStrategy.concat case "reference.conf" => MergeStrategy.concat case _ => MergeStrategy.first - }, + }, name := "atum-server", - javacOptions ++= Seq("-source", "11", "-target", "11"), libraryDependencies ++= Dependencies.serverDependencies, scalacOptions ++= Seq("-Ymacro-annotations"), Compile / packageBin / publishArtifact := false, diff --git a/project/plugins.sbt b/project/plugins.sbt index 7b4472c43..cdd55d27f 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -23,7 +23,6 @@ addSbtPlugin("com.eed3si9n" % "sbt-projectmatrix" % "0.9.1") addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.12") // Plugins to build the server module as a war file -//addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.10") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.5") // sbt-jacoco dependency downloading From 8b5a6d0edf25fbb23277cdd43d75bb35c92cd8d0 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Sat, 10 Feb 2024 20:37:14 +0100 Subject: [PATCH 10/40] build.sbt --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index d033c36c9..9dc40d9b2 100644 --- a/build.sbt +++ b/build.sbt @@ -40,7 +40,7 @@ lazy val commonSettings = Seq( libraryDependencies ++= commonDependencies, scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings"), javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), - scalacOptions ++= Seq("-target:jvm-1.8"), + scalacOptions ++= Seq("-release", "8"), Test / parallelExecution := false ) @@ -73,7 +73,7 @@ lazy val server = (projectMatrix in file("server")) libraryDependencies ++= commonDependencies, scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings"), javacOptions ++= Seq("-source", "11", "-target", "11"), - scalacOptions ++= Seq("-target:jvm-11"), + scalacOptions ++= Seq("-release", "11"), Test / parallelExecution := false, assembly / assemblyMergeStrategy := { case PathList("META-INF", "maven", "org.webjars", "swagger-ui", "pom.properties") => MergeStrategy.singleOrError From 28432b76500b283b63c7a3f10229bbc5d12b9f09 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Sat, 10 Feb 2024 21:53:06 +0100 Subject: [PATCH 11/40] PartitioningForDB serialized with Play --- build.sbt | 8 +++++--- .../functions/CreatePartitioningIfNotExists.scala | 12 ++++++------ .../database/runs/functions/WriteCheckpoint.scala | 2 -- .../absa/atum/server/model/PartitioningForDB.scala | 4 ++++ .../absa/atum/server/model/PlayJsonImplicits.scala | 2 ++ 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/build.sbt b/build.sbt index 9dc40d9b2..316e4e317 100644 --- a/build.sbt +++ b/build.sbt @@ -40,7 +40,10 @@ lazy val commonSettings = Seq( libraryDependencies ++= commonDependencies, scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings"), javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), - scalacOptions ++= Seq("-release", "8"), + scalacOptions ++= (CrossVersion.partialVersion(scalaVersion.value) match { + case Some((2, major)) if major >= 12 => Seq("-release", "8") + case _ => Seq("-target:jvm-1.8") + }), Test / parallelExecution := false ) @@ -70,7 +73,6 @@ lazy val root = (projectMatrix in file(".")) lazy val server = (projectMatrix in file("server")) .settings( Seq( - libraryDependencies ++= commonDependencies, scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings"), javacOptions ++= Seq("-source", "11", "-target", "11"), scalacOptions ++= Seq("-release", "11"), @@ -86,7 +88,7 @@ lazy val server = (projectMatrix in file("server")) case _ => MergeStrategy.first }, name := "atum-server", - libraryDependencies ++= Dependencies.serverDependencies, + libraryDependencies ++= Dependencies.serverDependencies ++ commonDependencies, scalacOptions ++= Seq("-Ymacro-annotations"), Compile / packageBin / publishArtifact := false, (Compile / compile) := ((Compile / compile) dependsOn printSparkScalaVersion).value, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index 9ac29565f..b73815c0f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -19,6 +19,7 @@ package za.co.absa.atum.server.api.database.runs.functions import doobie.Fragment import doobie.implicits.toSqlInterpolator import doobie.util.Read +import play.api.libs.json.Json import za.co.absa.atum.model.dto.PartitioningSubmitDTO import za.co.absa.atum.model.utils.SerializationUtils import za.co.absa.atum.server.model.PartitioningForDB @@ -28,7 +29,6 @@ import za.co.absa.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatu import za.co.absa.fadb.status.handling.implementations.StandardStatusHandling import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs - import zio._ import zio.interop.catz._ @@ -38,22 +38,22 @@ class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieE override def sql(values: PartitioningSubmitDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningNormalized = SerializationUtils.asJson(partitioning) + val partitioningJsonString = Json.toJson(partitioning).toString() - val parentPartitioningNormalized = values.parentPartitioning.map { parentPartitioning => + val parentPartitioningJsonString = values.parentPartitioning.map { parentPartitioning => val parentPartitioningForDB = PartitioningForDB.fromSeqPartitionDTO(parentPartitioning) - SerializationUtils.asJson(parentPartitioningForDB) + Json.toJson(parentPartitioningForDB).toString() } sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( ${ import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString - partitioningNormalized + partitioningJsonString }, ${values.authorIfNew}, ${ import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString - parentPartitioningNormalized + parentPartitioningJsonString } ) ${Fragment.const(alias)};""" } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index bcd9882a1..a192f0644 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -28,10 +28,8 @@ import za.co.absa.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatu import za.co.absa.fadb.status.handling.implementations.StandardStatusHandling import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs - import zio._ import zio.interop.catz._ - import doobie.postgres.implicits._ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala index df2ba109e..94c558012 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala @@ -16,6 +16,7 @@ package za.co.absa.atum.server.model +import play.api.libs.json.{Json, Reads, Writes} import za.co.absa.atum.model.dto.PartitioningDTO private[server] case class PartitioningForDB private ( @@ -32,4 +33,7 @@ object PartitioningForDB { PartitioningForDB(keys = allKeys, keysToValuesMap = mapOfKeysAndValues) } + + implicit val writes: Writes[PartitioningForDB] = Json.writes + } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala index 1f3aae830..c4e492836 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala @@ -77,4 +77,6 @@ object PlayJsonImplicits { implicit val readsAtumContextDTO: Reads[AtumContextDTO] = Json.reads[AtumContextDTO] implicit val writesAtumContextDTO: Writes[AtumContextDTO] = Json.writes[AtumContextDTO] + + } From 6b91971b9889ce5de5a5545c6eab046f1eaa8163 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Sun, 11 Feb 2024 10:29:16 +0100 Subject: [PATCH 12/40] build.sbt --- build.sbt | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/build.sbt b/build.sbt index 316e4e317..da76e187d 100644 --- a/build.sbt +++ b/build.sbt @@ -39,11 +39,6 @@ ThisBuild / printSparkScalaVersion := { lazy val commonSettings = Seq( libraryDependencies ++= commonDependencies, scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings"), - javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), - scalacOptions ++= (CrossVersion.partialVersion(scalaVersion.value) match { - case Some((2, major)) if major >= 12 => Seq("-release", "8") - case _ => Seq("-target:jvm-1.8") - }), Test / parallelExecution := false ) @@ -72,10 +67,11 @@ lazy val root = (projectMatrix in file(".")) lazy val server = (projectMatrix in file("server")) .settings( + commonSettings, Seq( scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings"), - javacOptions ++= Seq("-source", "11", "-target", "11"), scalacOptions ++= Seq("-release", "11"), + javacOptions ++= Seq("-source", "11", "-target", "11", "-Xlint"), Test / parallelExecution := false, assembly / assemblyMergeStrategy := { case PathList("META-INF", "maven", "org.webjars", "swagger-ui", "pom.properties") => MergeStrategy.singleOrError @@ -110,7 +106,13 @@ lazy val server = (projectMatrix in file("server")) lazy val agent = (projectMatrix in file("agent")) .settings( - commonSettings ++ Seq( + commonSettings, + scalacOptions ++= (CrossVersion.partialVersion(scalaVersion.value) match { + case Some((2, major)) if major >= 12 => Seq("-release", "8") + case _ => Seq("-target:jvm-1.8") + }), + javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), + Seq( name := "atum-agent", libraryDependencies ++= Dependencies.agentDependencies( if (scalaVersion.value == Versions.scala211) Versions.spark2 else Versions.spark3, @@ -131,7 +133,13 @@ lazy val agent = (projectMatrix in file("agent")) lazy val model = (projectMatrix in file("model")) .settings( - commonSettings ++ Seq( + commonSettings, + scalacOptions ++= (CrossVersion.partialVersion(scalaVersion.value) match { + case Some((2, major)) if major >= 12 => Seq("-release", "8") + case _ => Seq("-target:jvm-1.8") + }), + javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), + Seq( name := "atum-model", libraryDependencies ++= Dependencies.modelDependencies(scalaVersion.value), (Compile / compile) := ((Compile / compile) dependsOn printSparkScalaVersion).value, From 6f90c77d1720853cc6d86386892331a30279217e Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Sun, 11 Feb 2024 10:32:37 +0100 Subject: [PATCH 13/40] build.sbt --- build.sbt | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/build.sbt b/build.sbt index da76e187d..43eb4d36b 100644 --- a/build.sbt +++ b/build.sbt @@ -67,7 +67,7 @@ lazy val root = (projectMatrix in file(".")) lazy val server = (projectMatrix in file("server")) .settings( - commonSettings, + commonSettings ++ Seq( scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings"), scalacOptions ++= Seq("-release", "11"), @@ -106,14 +106,13 @@ lazy val server = (projectMatrix in file("server")) lazy val agent = (projectMatrix in file("agent")) .settings( - commonSettings, - scalacOptions ++= (CrossVersion.partialVersion(scalaVersion.value) match { - case Some((2, major)) if major >= 12 => Seq("-release", "8") - case _ => Seq("-target:jvm-1.8") - }), - javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), - Seq( + commonSettings ++ Seq( name := "atum-agent", + scalacOptions ++= (CrossVersion.partialVersion(scalaVersion.value) match { + case Some((2, major)) if major >= 12 => Seq("-release", "8") + case _ => Seq("-target:jvm-1.8") + }), + javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), libraryDependencies ++= Dependencies.agentDependencies( if (scalaVersion.value == Versions.scala211) Versions.spark2 else Versions.spark3, scalaVersion.value @@ -133,14 +132,13 @@ lazy val agent = (projectMatrix in file("agent")) lazy val model = (projectMatrix in file("model")) .settings( - commonSettings, - scalacOptions ++= (CrossVersion.partialVersion(scalaVersion.value) match { - case Some((2, major)) if major >= 12 => Seq("-release", "8") - case _ => Seq("-target:jvm-1.8") - }), - javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), - Seq( + commonSettings ++ Seq( name := "atum-model", + scalacOptions ++= (CrossVersion.partialVersion(scalaVersion.value) match { + case Some((2, major)) if major >= 12 => Seq("-release", "8") + case _ => Seq("-target:jvm-1.8") + }), + javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), libraryDependencies ++= Dependencies.modelDependencies(scalaVersion.value), (Compile / compile) := ((Compile / compile) dependsOn printSparkScalaVersion).value, ): _* From 1bd421d7d6ffc0f99656c76342cbc3ed0be5aa59 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Sun, 11 Feb 2024 11:13:30 +0100 Subject: [PATCH 14/40] build.sbt --- .github/workflows/build.yml | 70 ++++++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b8b658c6a..0268e8712 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,9 +23,67 @@ on: types: [ opened, synchronize, reopened ] jobs: - test: +# test: +# runs-on: ubuntu-latest +# +# services: +# postgres: +# image: postgres:15 +# env: +# POSTGRES_PASSWORD: postgres +# POSTGRES_DB: atum_db +# options: >- +# --health-cmd pg_isready +# --health-interval 10s +# --health-timeout 5s +# --health-retries 5 +# ports: +# - 5432:5432 + +# strategy: +# fail-fast: false +# matrix: +# scala: [2.11.12, 2.12.18, 2.13.11] +# +# name: Scala ${{matrix.scala}} +# +# steps: +# - name: Checkout code +# uses: actions/checkout@v2 +# - uses: coursier/cache-action@v5 +# +# - name: Setup Scala +# uses: olafurpg/setup-scala@v10 +# with: +# java-version: "adopt@1.11.0-11" +## java-version: "adopt@1.8" +# +# - name: Prepare testing database +# run: sbt flywayMigrate +# +# - name: Build and run tests +# run: sbt ++${{matrix.scala}} test doc + + build-agent-and-model: runs-on: ubuntu-latest + strategy: + matrix: + scala: [2.11.12, 2.12.18, 2.13.11] + steps: + - name: Checkout code + uses: actions/checkout@v2 + - uses: coursier/cache-action@v5 + - name: Setup Scala + uses: olafurpg/setup-scala@v10 + with: + java-version: "adopt@1.8" + + - name: Build and run tests + run: sbt "project agent" ++${{matrix.scala}} test "project model" ++${{matrix.scala}} test + + build-server: + runs-on: ubuntu-latest services: postgres: image: postgres:15 @@ -39,14 +97,9 @@ jobs: --health-retries 5 ports: - 5432:5432 - strategy: - fail-fast: false matrix: - scala: [2.11.12, 2.12.18, 2.13.11] - - name: Scala ${{matrix.scala}} - + scala: [2.13.11] steps: - name: Checkout code uses: actions/checkout@v2 @@ -56,10 +109,9 @@ jobs: uses: olafurpg/setup-scala@v10 with: java-version: "adopt@1.11.0-11" -# java-version: "adopt@1.8" - name: Prepare testing database run: sbt flywayMigrate - name: Build and run tests - run: sbt ++${{matrix.scala}} test doc + run: sbt "project server" ++${{matrix.scala}} test From 84b713d6b4295cfd33d4c2a1c58b0e90b73d2d60 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Sun, 11 Feb 2024 12:28:53 +0100 Subject: [PATCH 15/40] SerializationUtils --- .../runs/functions/CreatePartitioningIfNotExists.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index b73815c0f..c07142b6e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -38,11 +38,13 @@ class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieE override def sql(values: PartitioningSubmitDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningJsonString = Json.toJson(partitioning).toString() +// val partitioningJsonString = Json.toJson(partitioning).toString() + val partitioningJsonString = SerializationUtils.asJson(partitioning) val parentPartitioningJsonString = values.parentPartitioning.map { parentPartitioning => val parentPartitioningForDB = PartitioningForDB.fromSeqPartitionDTO(parentPartitioning) - Json.toJson(parentPartitioningForDB).toString() +// Json.toJson(parentPartitioningForDB).toString() + SerializationUtils.asJson(parentPartitioningForDB) } sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( From 7e1549d47c54017eeae6db10009ddbd39be1bd43 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Sun, 11 Feb 2024 12:46:00 +0100 Subject: [PATCH 16/40] SerializationUtils --- build.sbt | 46 +++++++++---------- .../CreatePartitioningIfNotExists.scala | 6 +-- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/build.sbt b/build.sbt index 43eb4d36b..4306f2cbc 100644 --- a/build.sbt +++ b/build.sbt @@ -42,13 +42,24 @@ lazy val commonSettings = Seq( Test / parallelExecution := false ) -val mergeStrategy: Def.SettingsDefinition = assembly / assemblyMergeStrategy := { +val mergeStrategy = assembly / assemblyMergeStrategy := { case PathList("META-INF", _) => MergeStrategy.discard case "application.conf" => MergeStrategy.concat case "reference.conf" => MergeStrategy.concat case _ => MergeStrategy.first } +val serverMergeStrategy = assembly / assemblyMergeStrategy := { + case PathList("META-INF", "maven", "org.webjars", "swagger-ui", "pom.properties") => MergeStrategy.singleOrError + case PathList("META-INF", "resources", "webjars", "swagger-ui", _*) => MergeStrategy.singleOrError + case PathList("META-INF", _*) => MergeStrategy.discard + case PathList("META-INF", "versions", "9", xs@_*) => MergeStrategy.discard + case PathList("module-info.class") => MergeStrategy.discard + case "application.conf" => MergeStrategy.concat + case "reference.conf" => MergeStrategy.concat + case _ => MergeStrategy.first +} + enablePlugins(FlywayPlugin) flywayUrl := FlywayConfiguration.flywayUrl flywayUser := FlywayConfiguration.flywayUser @@ -69,30 +80,19 @@ lazy val server = (projectMatrix in file("server")) .settings( commonSettings ++ Seq( - scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings"), scalacOptions ++= Seq("-release", "11"), javacOptions ++= Seq("-source", "11", "-target", "11", "-Xlint"), - Test / parallelExecution := false, - assembly / assemblyMergeStrategy := { - case PathList("META-INF", "maven", "org.webjars", "swagger-ui", "pom.properties") => MergeStrategy.singleOrError - case PathList("META-INF", "resources", "webjars", "swagger-ui", _*) => MergeStrategy.singleOrError - case PathList("META-INF", _*) => MergeStrategy.discard - case PathList("META-INF", "versions", "9", xs @ _*) => MergeStrategy.discard - case PathList("module-info.class") => MergeStrategy.discard - case "application.conf" => MergeStrategy.concat - case "reference.conf" => MergeStrategy.concat - case _ => MergeStrategy.first - }, - name := "atum-server", - libraryDependencies ++= Dependencies.serverDependencies ++ commonDependencies, - scalacOptions ++= Seq("-Ymacro-annotations"), - Compile / packageBin / publishArtifact := false, - (Compile / compile) := ((Compile / compile) dependsOn printSparkScalaVersion).value, - packageBin := (Compile / assembly).value, - artifactPath / (Compile / packageBin) := baseDirectory.value / s"target/${name.value}-${version.value}.jar", - webappWebInfClasses := true, - inheritJarManifest := true, - testFrameworks += new TestFramework("zio.test.sbt.ZTestFramework") + serverMergeStrategy, + name := "atum-server", + libraryDependencies ++= Dependencies.serverDependencies ++ commonDependencies, + scalacOptions ++= Seq("-Ymacro-annotations"), + Compile / packageBin / publishArtifact := false, + (Compile / compile) := ((Compile / compile) dependsOn printSparkScalaVersion).value, + packageBin := (Compile / assembly).value, + artifactPath / (Compile / packageBin) := baseDirectory.value / s"target/${name.value}-${version.value}.jar", + webappWebInfClasses := true, + inheritJarManifest := true, + testFrameworks += new TestFramework("zio.test.sbt.ZTestFramework") ): _* ) .settings( diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index c07142b6e..b73815c0f 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -38,13 +38,11 @@ class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieE override def sql(values: PartitioningSubmitDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) -// val partitioningJsonString = Json.toJson(partitioning).toString() - val partitioningJsonString = SerializationUtils.asJson(partitioning) + val partitioningJsonString = Json.toJson(partitioning).toString() val parentPartitioningJsonString = values.parentPartitioning.map { parentPartitioning => val parentPartitioningForDB = PartitioningForDB.fromSeqPartitionDTO(parentPartitioning) -// Json.toJson(parentPartitioningForDB).toString() - SerializationUtils.asJson(parentPartitioningForDB) + Json.toJson(parentPartitioningForDB).toString() } sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( From 9df1bd6ad269da3672632feceb8781a1ebfeeb83 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Sun, 11 Feb 2024 12:50:26 +0100 Subject: [PATCH 17/40] build.yml --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0268e8712..50a2ae73b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,7 +14,7 @@ # limitations under the License. # -name: Build +name: Build all modules on: pull_request: @@ -80,7 +80,7 @@ jobs: java-version: "adopt@1.8" - name: Build and run tests - run: sbt "project agent" ++${{matrix.scala}} test "project model" ++${{matrix.scala}} test + run: sbt "project agent" ++${{matrix.scala}} test doc "project model" ++${{matrix.scala}} test doc build-server: runs-on: ubuntu-latest @@ -114,4 +114,4 @@ jobs: run: sbt flywayMigrate - name: Build and run tests - run: sbt "project server" ++${{matrix.scala}} test + run: sbt "project server" ++${{matrix.scala}} test doc From 9b78d68def9987b55d2d98fce8749b5cb86dd84f Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Mon, 12 Feb 2024 07:59:05 +0100 Subject: [PATCH 18/40] without scalac options --- build.sbt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/build.sbt b/build.sbt index 4306f2cbc..2b78888fa 100644 --- a/build.sbt +++ b/build.sbt @@ -80,7 +80,7 @@ lazy val server = (projectMatrix in file("server")) .settings( commonSettings ++ Seq( - scalacOptions ++= Seq("-release", "11"), +// scalacOptions ++= Seq("-release", "11"), javacOptions ++= Seq("-source", "11", "-target", "11", "-Xlint"), serverMergeStrategy, name := "atum-server", @@ -108,10 +108,10 @@ lazy val agent = (projectMatrix in file("agent")) .settings( commonSettings ++ Seq( name := "atum-agent", - scalacOptions ++= (CrossVersion.partialVersion(scalaVersion.value) match { - case Some((2, major)) if major >= 12 => Seq("-release", "8") - case _ => Seq("-target:jvm-1.8") - }), +// scalacOptions ++= (CrossVersion.partialVersion(scalaVersion.value) match { +// case Some((2, major)) if major >= 12 => Seq("-release", "8") +// case _ => Seq("-target:jvm-1.8") +// }), javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), libraryDependencies ++= Dependencies.agentDependencies( if (scalaVersion.value == Versions.scala211) Versions.spark2 else Versions.spark3, @@ -134,10 +134,10 @@ lazy val model = (projectMatrix in file("model")) .settings( commonSettings ++ Seq( name := "atum-model", - scalacOptions ++= (CrossVersion.partialVersion(scalaVersion.value) match { - case Some((2, major)) if major >= 12 => Seq("-release", "8") - case _ => Seq("-target:jvm-1.8") - }), +// scalacOptions ++= (CrossVersion.partialVersion(scalaVersion.value) match { +// case Some((2, major)) if major >= 12 => Seq("-release", "8") +// case _ => Seq("-target:jvm-1.8") +// }), javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), libraryDependencies ++= Dependencies.modelDependencies(scalaVersion.value), (Compile / compile) := ((Compile / compile) dependsOn printSparkScalaVersion).value, From 3b95300b13478f04e61d1db3a9a8db4707d97545 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Mon, 12 Feb 2024 08:22:33 +0100 Subject: [PATCH 19/40] clean up --- .github/workflows/build.yml | 41 ------------------- .github/workflows/jacoco_check.yml | 2 +- build.sbt | 15 ++----- .../CreatePartitioningIfNotExists.scala | 4 +- .../atum/server/model/PartitioningForDB.scala | 2 - 5 files changed, 6 insertions(+), 58 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 50a2ae73b..303c39179 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,47 +23,6 @@ on: types: [ opened, synchronize, reopened ] jobs: -# test: -# runs-on: ubuntu-latest -# -# services: -# postgres: -# image: postgres:15 -# env: -# POSTGRES_PASSWORD: postgres -# POSTGRES_DB: atum_db -# options: >- -# --health-cmd pg_isready -# --health-interval 10s -# --health-timeout 5s -# --health-retries 5 -# ports: -# - 5432:5432 - -# strategy: -# fail-fast: false -# matrix: -# scala: [2.11.12, 2.12.18, 2.13.11] -# -# name: Scala ${{matrix.scala}} -# -# steps: -# - name: Checkout code -# uses: actions/checkout@v2 -# - uses: coursier/cache-action@v5 -# -# - name: Setup Scala -# uses: olafurpg/setup-scala@v10 -# with: -# java-version: "adopt@1.11.0-11" -## java-version: "adopt@1.8" -# -# - name: Prepare testing database -# run: sbt flywayMigrate -# -# - name: Build and run tests -# run: sbt ++${{matrix.scala}} test doc - build-agent-and-model: runs-on: ubuntu-latest strategy: diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 79d4ca482..c3e2671e3 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -43,7 +43,7 @@ jobs: - name: Setup Scala uses: olafurpg/setup-scala@v10 with: - java-version: "adopt@1.11.0-11" + java-version: "adopt@1.8" - name: Build and run tests run: sbt "++${{matrix.scala}}; project agent; jacoco" # Agent module code coverage diff --git a/build.sbt b/build.sbt index 2b78888fa..66ecdc3dc 100644 --- a/build.sbt +++ b/build.sbt @@ -80,10 +80,8 @@ lazy val server = (projectMatrix in file("server")) .settings( commonSettings ++ Seq( -// scalacOptions ++= Seq("-release", "11"), - javacOptions ++= Seq("-source", "11", "-target", "11", "-Xlint"), - serverMergeStrategy, name := "atum-server", + javacOptions ++= Seq("-source", "11", "-target", "11", "-Xlint"), libraryDependencies ++= Dependencies.serverDependencies ++ commonDependencies, scalacOptions ++= Seq("-Ymacro-annotations"), Compile / packageBin / publishArtifact := false, @@ -92,7 +90,8 @@ lazy val server = (projectMatrix in file("server")) artifactPath / (Compile / packageBin) := baseDirectory.value / s"target/${name.value}-${version.value}.jar", webappWebInfClasses := true, inheritJarManifest := true, - testFrameworks += new TestFramework("zio.test.sbt.ZTestFramework") + testFrameworks += new TestFramework("zio.test.sbt.ZTestFramework"), + serverMergeStrategy, ): _* ) .settings( @@ -108,10 +107,6 @@ lazy val agent = (projectMatrix in file("agent")) .settings( commonSettings ++ Seq( name := "atum-agent", -// scalacOptions ++= (CrossVersion.partialVersion(scalaVersion.value) match { -// case Some((2, major)) if major >= 12 => Seq("-release", "8") -// case _ => Seq("-target:jvm-1.8") -// }), javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), libraryDependencies ++= Dependencies.agentDependencies( if (scalaVersion.value == Versions.scala211) Versions.spark2 else Versions.spark3, @@ -134,10 +129,6 @@ lazy val model = (projectMatrix in file("model")) .settings( commonSettings ++ Seq( name := "atum-model", -// scalacOptions ++= (CrossVersion.partialVersion(scalaVersion.value) match { -// case Some((2, major)) if major >= 12 => Seq("-release", "8") -// case _ => Seq("-target:jvm-1.8") -// }), javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), libraryDependencies ++= Dependencies.modelDependencies(scalaVersion.value), (Compile / compile) := ((Compile / compile) dependsOn printSparkScalaVersion).value, diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index b73815c0f..8c23de021 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -38,11 +38,11 @@ class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieE override def sql(values: PartitioningSubmitDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningJsonString = Json.toJson(partitioning).toString() + val partitioningJsonString = SerializationUtils.asJson(partitioning) val parentPartitioningJsonString = values.parentPartitioning.map { parentPartitioning => val parentPartitioningForDB = PartitioningForDB.fromSeqPartitionDTO(parentPartitioning) - Json.toJson(parentPartitioningForDB).toString() + SerializationUtils.asJson(parentPartitioningForDB) } sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala index 94c558012..1963e2a7e 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala @@ -34,6 +34,4 @@ object PartitioningForDB { PartitioningForDB(keys = allKeys, keysToValuesMap = mapOfKeysAndValues) } - implicit val writes: Writes[PartitioningForDB] = Json.writes - } From e64cc23dbc9e3ab8cfb440f665a32873223881e5 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Mon, 12 Feb 2024 08:36:16 +0100 Subject: [PATCH 20/40] clean up --- .../src/main/scala/za/co/absa/atum/server/api/http/SSL.scala | 2 +- .../main/scala/za/co/absa/atum/server/api/http/Server.scala | 4 ++-- .../za/co/absa/atum/server/model/PartitioningForDB.scala | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/SSL.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/SSL.scala index e03906d7a..d31072301 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/SSL.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/SSL.scala @@ -25,7 +25,7 @@ import javax.net.ssl.{KeyManagerFactory, SSLContext, TrustManagerFactory} object SSL { - def context: ZIO[Any, Throwable, SSLContext] = { + val context: ZIO[Any, Throwable, SSLContext] = { for { sslConfig <- ZIO.config[SslConfig](SslConfig.config) _ <- ZIO.logDebug("Attempting to initialize SSLContext") diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Server.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Server.scala index 9acde0a67..b38a67626 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Server.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Server.scala @@ -102,8 +102,8 @@ trait Server extends Endpoints { builderWithSsl.serve.compile.drain } - private def httpServer: ZIO[Env, Throwable, Unit] = createServer(8080) - private def httpsServer: ZIO[Env, Throwable, Unit] = SSL.context.flatMap { context => + private val httpServer: ZIO[Env, Throwable, Unit] = createServer(8080) + private val httpsServer: ZIO[Env, Throwable, Unit] = SSL.context.flatMap { context => createServer(8443, Some(context)) } diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala index 1963e2a7e..da5fc2579 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala @@ -16,7 +16,6 @@ package za.co.absa.atum.server.model -import play.api.libs.json.{Json, Reads, Writes} import za.co.absa.atum.model.dto.PartitioningDTO private[server] case class PartitioningForDB private ( From d8ba3251776eac0800899d885de3d372224d7c61 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Mon, 12 Feb 2024 12:44:03 +0100 Subject: [PATCH 21/40] tmp --- build.sbt | 24 ++++++++++------- project/Dependencies.scala | 27 ++++++++++++++----- project/plugins.sbt | 3 ++- server/src/main/resources/logback.xml | 2 +- server/src/main/resources/reference.conf | 6 ++--- .../CreatePartitioningIfNotExists.scala | 6 +++-- .../atum/server/model/PartitioningForDB.scala | 3 +++ 7 files changed, 48 insertions(+), 23 deletions(-) diff --git a/build.sbt b/build.sbt index 66ecdc3dc..442109c40 100644 --- a/build.sbt +++ b/build.sbt @@ -14,10 +14,11 @@ * limitations under the License. */ -import Dependencies._ -import SparkVersionAxis._ -import JacocoSetup._ +import Dependencies.* +import SparkVersionAxis.* +import JacocoSetup.* import sbt.Keys.name +import sbtassembly.AssemblyPlugin.autoImport.assemblyJarName ThisBuild / organization := "za.co.absa" @@ -37,7 +38,7 @@ ThisBuild / printSparkScalaVersion := { } lazy val commonSettings = Seq( - libraryDependencies ++= commonDependencies, +// libraryDependencies ++= commonDependencies, scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings"), Test / parallelExecution := false ) @@ -82,14 +83,17 @@ lazy val server = (projectMatrix in file("server")) Seq( name := "atum-server", javacOptions ++= Seq("-source", "11", "-target", "11", "-Xlint"), - libraryDependencies ++= Dependencies.serverDependencies ++ commonDependencies, + scalacOptions ++= Seq("-release", "11"), +// libraryDependencies ++= Dependencies.serverDependencies ++ commonDependencies, + libraryDependencies ++= Dependencies.serverDependencies ++ testDependencies, +// unmanagedJars in Compile += file("model/target/jvm-2.13/model.jar"), scalacOptions ++= Seq("-Ymacro-annotations"), Compile / packageBin / publishArtifact := false, (Compile / compile) := ((Compile / compile) dependsOn printSparkScalaVersion).value, packageBin := (Compile / assembly).value, artifactPath / (Compile / packageBin) := baseDirectory.value / s"target/${name.value}-${version.value}.jar", - webappWebInfClasses := true, - inheritJarManifest := true, +// webappWebInfClasses := true, +// inheritJarManifest := true, testFrameworks += new TestFramework("zio.test.sbt.ZTestFramework"), serverMergeStrategy, ): _* @@ -108,7 +112,7 @@ lazy val agent = (projectMatrix in file("agent")) commonSettings ++ Seq( name := "atum-agent", javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), - libraryDependencies ++= Dependencies.agentDependencies( + libraryDependencies ++= commonDependencies ++ testDependencies ++ Dependencies.agentDependencies( if (scalaVersion.value == Versions.scala211) Versions.spark2 else Versions.spark3, scalaVersion.value ), @@ -130,8 +134,10 @@ lazy val model = (projectMatrix in file("model")) commonSettings ++ Seq( name := "atum-model", javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), - libraryDependencies ++= Dependencies.modelDependencies(scalaVersion.value), + assembly / assemblyJarName := "model.jar", + libraryDependencies ++= commonDependencies ++ testDependencies ++ Dependencies.modelDependencies(scalaVersion.value), (Compile / compile) := ((Compile / compile) dependsOn printSparkScalaVersion).value, + mergeStrategy ): _* ) .settings( diff --git a/project/Dependencies.scala b/project/Dependencies.scala index d9dd021ee..3f12520d6 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -98,6 +98,16 @@ object Dependencies { } } + def testDependencies: Seq[ModuleID] = { + lazy val scalatest = "org.scalatest" %% "scalatest" % Versions.scalatest % Test + lazy val mockito = "org.mockito" %% "mockito-scala" % Versions.scalaMockito % Test + + Seq( + scalatest, + mockito, + ) + } + def commonDependencies: Seq[ModuleID] = { val json4sVersion = json4sVersionForScala(Versions.scala212) @@ -108,9 +118,9 @@ object Dependencies { lazy val json4sJackson = "org.json4s" %% "json4s-jackson" % json4sVersion lazy val json4sNative = "org.json4s" %% "json4s-native" % json4sVersion % Provided - lazy val logback = "ch.qos.logback" % "logback-classic" % Versions.logback - lazy val scalatest = "org.scalatest" %% "scalatest" % Versions.scalatest % Test - lazy val mockito = "org.mockito" %% "mockito-scala" % Versions.scalaMockito % Test +// lazy val logback = "ch.qos.logback" % "logback-classic" % Versions.logback +// lazy val scalatest = "org.scalatest" %% "scalatest" % Versions.scalatest % Test +// lazy val mockito = "org.mockito" %% "mockito-scala" % Versions.scalaMockito % Test Seq( jacksonModuleScala, @@ -118,9 +128,9 @@ object Dependencies { json4sCore, json4sJackson, json4sNative, - logback, - scalatest, - mockito, +// logback, +// scalatest, +// mockito, ) } @@ -198,13 +208,16 @@ object Dependencies { lazy val sttp = "com.softwaremill.sttp.client3" %% "core" % Versions.sttp + lazy val logback = "ch.qos.logback" % "logback-classic" % Versions.logback + Seq( sparkCore, sparkSql, typeSafeConfig, sparkCommons, sparkCommonsTest, - sttp + sttp, + logback ) } diff --git a/project/plugins.sbt b/project/plugins.sbt index cdd55d27f..f41be589b 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -23,7 +23,8 @@ addSbtPlugin("com.eed3si9n" % "sbt-projectmatrix" % "0.9.1") addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.12") // Plugins to build the server module as a war file -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.5") +//addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.5") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.10") // sbt-jacoco dependency downloading lazy val ow2Version = "9.5" diff --git a/server/src/main/resources/logback.xml b/server/src/main/resources/logback.xml index 6a008063f..acd40c3d8 100644 --- a/server/src/main/resources/logback.xml +++ b/server/src/main/resources/logback.xml @@ -5,7 +5,7 @@ %d{HH:mm:ss.SSS} %-5level [%thread] %logger{36}:%L - %msg %ex{short}%n - + diff --git a/server/src/main/resources/reference.conf b/server/src/main/resources/reference.conf index 4029484fb..bdeafe461 100644 --- a/server/src/main/resources/reference.conf +++ b/server/src/main/resources/reference.conf @@ -12,8 +12,8 @@ maxPoolSize=10 } ssl { - enabled=false - keyStorePassword=password - keyStorePath="/your/path/to/selfsigned.jks" + enabled=true + keyStorePassword=changeit + keyStorePath="/etc/ssl/certs/selfsigned.jks" } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index 8c23de021..606c63f94 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -38,11 +38,13 @@ class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieE override def sql(values: PartitioningSubmitDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningJsonString = SerializationUtils.asJson(partitioning) +// val partitioningJsonString = SerializationUtils.asJson(partitioning) + val partitioningJsonString = Json.toJson(partitioning).toString val parentPartitioningJsonString = values.parentPartitioning.map { parentPartitioning => val parentPartitioningForDB = PartitioningForDB.fromSeqPartitionDTO(parentPartitioning) - SerializationUtils.asJson(parentPartitioningForDB) +// SerializationUtils.asJson(parentPartitioningForDB) + Json.toJson(parentPartitioningForDB).toString } sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala index da5fc2579..b03c89c3c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala @@ -16,6 +16,7 @@ package za.co.absa.atum.server.model +import play.api.libs.json.{Json, Writes} import za.co.absa.atum.model.dto.PartitioningDTO private[server] case class PartitioningForDB private ( @@ -33,4 +34,6 @@ object PartitioningForDB { PartitioningForDB(keys = allKeys, keysToValuesMap = mapOfKeysAndValues) } + implicit val writes: Writes[PartitioningForDB] = Json.writes + } From cdd84861e7af2edce3376ecaeb1c944c9a03d96c Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Mon, 12 Feb 2024 16:41:27 +0100 Subject: [PATCH 22/40] fix stuff, added docker ci test --- .github/workflows/docker_image.yml | 67 +++++++++++++++++++ build.sbt | 11 +-- server/certs/.gitkeep | 0 server/src/main/resources/reference.conf | 30 ++++----- .../CreatePartitioningIfNotExists.scala | 3 - .../runs/functions/WriteCheckpoint.scala | 11 +-- 6 files changed, 90 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/docker_image.yml create mode 100644 server/certs/.gitkeep diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml new file mode 100644 index 000000000..3a06dc224 --- /dev/null +++ b/.github/workflows/docker_image.yml @@ -0,0 +1,67 @@ +# +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Docker Image CI + +on: + pull_request: + branches: [ master ] + types: [ opened, edited, synchronize, reopened ] + +jobs: + build: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:15 + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: atum_db + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - uses: actions/checkout@v2 + + - name: Setup Scala environment + uses: olafurpg/setup-scala@v10 + with: + java-version: adopt@1.11 + + - name: Run sbt assembly + run: sbt server/assembly + + - name: Prepare testing database + run: sbt flywayMigrate + + - name: Build the Docker image + run: docker build -t absaoss/atum-service:latest \ + --build-arg CONFIG=./src/main/resources/reference.conf \ + --build-arg SSL=false ./ + + - name: Run the Docker image + run: docker run -d -p 8080:8080 absaoss/atum-service:latest + + - name: Execute HTTP call + run: | + sleep 10 # Wait for the service to start + curl -Lf http://localhost:8080/docs \ No newline at end of file diff --git a/build.sbt b/build.sbt index 442109c40..28a407ae2 100644 --- a/build.sbt +++ b/build.sbt @@ -18,8 +18,6 @@ import Dependencies.* import SparkVersionAxis.* import JacocoSetup.* import sbt.Keys.name -import sbtassembly.AssemblyPlugin.autoImport.assemblyJarName - ThisBuild / organization := "za.co.absa" @@ -38,7 +36,6 @@ ThisBuild / printSparkScalaVersion := { } lazy val commonSettings = Seq( -// libraryDependencies ++= commonDependencies, scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings"), Test / parallelExecution := false ) @@ -51,6 +48,7 @@ val mergeStrategy = assembly / assemblyMergeStrategy := { } val serverMergeStrategy = assembly / assemblyMergeStrategy := { + case PathList("META-INF", "services", xs @ _*) => MergeStrategy.filterDistinctLines case PathList("META-INF", "maven", "org.webjars", "swagger-ui", "pom.properties") => MergeStrategy.singleOrError case PathList("META-INF", "resources", "webjars", "swagger-ui", _*) => MergeStrategy.singleOrError case PathList("META-INF", _*) => MergeStrategy.discard @@ -74,7 +72,6 @@ lazy val root = (projectMatrix in file(".")) .settings( name := "atum-service-root", publish / skip := true, - mergeStrategy ) lazy val server = (projectMatrix in file("server")) @@ -84,16 +81,12 @@ lazy val server = (projectMatrix in file("server")) name := "atum-server", javacOptions ++= Seq("-source", "11", "-target", "11", "-Xlint"), scalacOptions ++= Seq("-release", "11"), -// libraryDependencies ++= Dependencies.serverDependencies ++ commonDependencies, libraryDependencies ++= Dependencies.serverDependencies ++ testDependencies, -// unmanagedJars in Compile += file("model/target/jvm-2.13/model.jar"), scalacOptions ++= Seq("-Ymacro-annotations"), Compile / packageBin / publishArtifact := false, (Compile / compile) := ((Compile / compile) dependsOn printSparkScalaVersion).value, packageBin := (Compile / assembly).value, artifactPath / (Compile / packageBin) := baseDirectory.value / s"target/${name.value}-${version.value}.jar", -// webappWebInfClasses := true, -// inheritJarManifest := true, testFrameworks += new TestFramework("zio.test.sbt.ZTestFramework"), serverMergeStrategy, ): _* @@ -134,10 +127,8 @@ lazy val model = (projectMatrix in file("model")) commonSettings ++ Seq( name := "atum-model", javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), - assembly / assemblyJarName := "model.jar", libraryDependencies ++= commonDependencies ++ testDependencies ++ Dependencies.modelDependencies(scalaVersion.value), (Compile / compile) := ((Compile / compile) dependsOn printSparkScalaVersion).value, - mergeStrategy ): _* ) .settings( diff --git a/server/certs/.gitkeep b/server/certs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/server/src/main/resources/reference.conf b/server/src/main/resources/reference.conf index bdeafe461..857353de1 100644 --- a/server/src/main/resources/reference.conf +++ b/server/src/main/resources/reference.conf @@ -1,19 +1,19 @@ { - postgres { - # The JDBC driver class - dataSourceClass=org.postgresql.Driver - serverName=127.0.0.1 - # set to non-default port, to avoid local collision - portNumber=5432 - databaseName=atum_db - user=atum_user - password=changeme - # maximum number of connections that HikariCP will keep in the pool, including both idle and in-use connections - maxPoolSize=10 - } + postgres { + # The JDBC driver class + dataSourceClass=org.postgresql.Driver + serverName=localhost // host.docker.internal for local run in docker against db on its host machine + # set to non-default port, to avoid local collision + portNumber=5432 + databaseName=atum_db + user=atum_user + password=changeme + # maximum number of connections that HikariCP will keep in the pool, including both idle and in-use connections + maxPoolSize=10 + } ssl { - enabled=true - keyStorePassword=changeit - keyStorePath="/etc/ssl/certs/selfsigned.jks" + enabled=false + keyStorePassword=password + keyStorePath="/path/to/your/cert" } } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index 606c63f94..8c6135476 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -21,7 +21,6 @@ import doobie.implicits.toSqlInterpolator import doobie.util.Read import play.api.libs.json.Json import za.co.absa.atum.model.dto.PartitioningSubmitDTO -import za.co.absa.atum.model.utils.SerializationUtils import za.co.absa.atum.server.model.PartitioningForDB import za.co.absa.fadb.DBSchema import za.co.absa.fadb.doobie.{DoobieEngine, StatusWithData} @@ -38,12 +37,10 @@ class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieE override def sql(values: PartitioningSubmitDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) -// val partitioningJsonString = SerializationUtils.asJson(partitioning) val partitioningJsonString = Json.toJson(partitioning).toString val parentPartitioningJsonString = values.parentPartitioning.map { parentPartitioning => val parentPartitioningForDB = PartitioningForDB.fromSeqPartitionDTO(parentPartitioning) -// SerializationUtils.asJson(parentPartitioningForDB) Json.toJson(parentPartitioningForDB).toString } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index a192f0644..35b615dad 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -20,7 +20,6 @@ import doobie.Fragment import doobie.implicits._ import doobie.util.Read import za.co.absa.atum.model.dto.CheckpointDTO -import za.co.absa.atum.model.utils.SerializationUtils import za.co.absa.atum.server.model.PartitioningForDB import za.co.absa.fadb.DBSchema import za.co.absa.fadb.doobie.{DoobieEngine, StatusWithData} @@ -30,6 +29,9 @@ import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs import zio._ import zio.interop.catz._ +import play.api.libs.json.Json +import za.co.absa.atum.server.model.PlayJsonImplicits.writesMeasurementDTO + import doobie.postgres.implicits._ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) @@ -38,11 +40,12 @@ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) override def sql(values: CheckpointDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningNormalized = SerializationUtils.asJson(partitioning) + val partitioningNormalized = Json.toJson(partitioning).toString // List[String] containing json data has to be properly escaped // It would be safer to use Json data type and derive Put instance - val measurementsNormalized = - values.measurements.map(x => s"\"${SerializationUtils.asJson(x).replaceAll("\"", "\\\\\"")}\"") + val measurementsNormalized = { + values.measurements.map(x => s"\"${Json.toJson(x).toString.replaceAll("\"", "\\\\\"")}\"") + } sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( ${ From e0b4506abe601e3f03c9c968d597bbb308ed0797 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Feb 2024 12:22:40 +0100 Subject: [PATCH 23/40] minors in pipeline --- .github/workflows/docker_image.yml | 4 +- .github/workflows/jacoco_check.yml | 33 +---------- .github/workflows/jacoco_check_server.yml | 69 +++++++++++++++++------ .github/workflows/release_publish.yml | 9 --- build.sbt | 12 ++-- server/src/main/resources/reference.conf | 2 +- 6 files changed, 61 insertions(+), 68 deletions(-) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index 3a06dc224..15a35edf1 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -43,9 +43,9 @@ jobs: - uses: actions/checkout@v2 - name: Setup Scala environment - uses: olafurpg/setup-scala@v10 + uses: olafurpg/setup-scala@v14 with: - java-version: adopt@1.11 + java-version: "adopt@1.11.0-11" - name: Run sbt assembly run: sbt server/assembly diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index e9757b0e3..6134b812e 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -14,7 +14,7 @@ # limitations under the License. # -name: JaCoCo report +name: JaCoCo report agent and module on: pull_request: @@ -24,8 +24,6 @@ on: env: scalaLong12: 2.12.18 scalaShort12: "2.12" - scalaLong13: 2.13.11 - scalaShort13: "2.13" overall: 80.0 changed: 80.0 @@ -34,20 +32,6 @@ jobs: name: Build and test runs-on: ubuntu-latest - services: - postgres: - image: postgres:15 - env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: atum_db - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - steps: - name: Checkout code uses: actions/checkout@v4 @@ -61,18 +45,6 @@ jobs: continue-on-error: true id: jacocorun run: sbt jacoco - # server module code coverage - - name: Add coverage to PR - if: steps.jacocorun.outcome == 'success' - id: jacoco-server - uses: madrapps/jacoco-report@v1.6.1 - with: - paths: ${{ github.workspace }}/server/target/jvm-${{ env.scalaShort13 }}/jacoco/report/jacoco.xml - token: ${{ secrets.GITHUB_TOKEN }} - min-coverage-overall: ${{env.overall }} - min-coverage-changed-files: ${{ env.changed }} - title: JaCoCo server module code coverage report - scala ${{ env.scalaLong13 }} - update-comment: true # agent module code coverage - name: Add coverage to PR if: steps.jacocorun.outcome == 'success' @@ -102,8 +74,6 @@ jobs: - name: Get the Coverage info if: steps.jacocorun.outcome == 'success' run: | - echo "Total sever module coverage ${{ steps.jacoco-server.outputs.coverage-overall }}" - echo "Changed Files coverage ${{ steps.jacoco-server.outputs.coverage-changed-files }}" echo "Total agent module coverage ${{ steps.jacoco-agent.outputs.coverage-overall }}" echo "Changed Files coverage ${{ steps.jacoco-agent.outputs.coverage-changed-files }}" echo "Total model module coverage ${{ steps.jacoco-model.outputs.coverage-overall }}" @@ -114,7 +84,6 @@ jobs: with: script: | const coverageCheckFailed = - Number('${{ steps.jacoco-server.outputs.coverage-changed-files }}') < Number('${{ env.changed }}') || Number('${{ steps.jacoco-agent.outputs.coverage-changed-files }}') < Number('${{ env.changed }}') || Number('${{ steps.jacoco-model.outputs.coverage-changed-files }}') < Number('${{ env.changed }}'); if (coverageCheckFailed) { diff --git a/.github/workflows/jacoco_check_server.yml b/.github/workflows/jacoco_check_server.yml index 63ddc033f..48e41b463 100644 --- a/.github/workflows/jacoco_check_server.yml +++ b/.github/workflows/jacoco_check_server.yml @@ -21,6 +21,12 @@ on: branches: [ master ] types: [ opened, edited, synchronize, reopened ] +env: + scalaLong13: 2.13.11 + scalaShort13: "2.13" + overall: 80.0 + changed: 80.0 + jobs: test: name: Build and test @@ -40,45 +46,72 @@ jobs: ports: - 5432:5432 - strategy: - matrix: - include: - - scala: 2.13.11 - scalaShort: "2.13" - overall: 80.0 - changed: 80.0 steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup Scala - uses: olafurpg/setup-scala@v10 + uses: olafurpg/setup-scala@v14 with: java-version: "adopt@1.11.0-11" - name: Prepare testing database run: sbt flywayMigrate - name: Build and run tests - run: sbt "++${{matrix.scala}}; project server; jacoco" + continue-on-error: true + id: jacocorun + run: sbt "project server; jacoco" # server module code coverage - name: Add coverage to PR + if: steps.jacocorun.outcome == 'success' id: jacoco-server uses: madrapps/jacoco-report@v1.6.1 with: - paths: ${{ github.workspace }}/server/target/jvm-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml + paths: ${{ github.workspace }}/server/target/jvm-${{ env.scalaShort13 }}/jacoco/report/jacoco.xml token: ${{ secrets.GITHUB_TOKEN }} - min-coverage-overall: ${{matrix.overall }} - min-coverage-changed-files: ${{ matrix.changed }} - title: JaCoCo server module code coverage report - scala ${{ matrix.scala }} + min-coverage-overall: ${{env.overall }} + min-coverage-changed-files: ${{ env.changed }} + title: JaCoCo server module code coverage report - scala ${{ env.scalaLong13 }} update-comment: true - name: Get the Coverage info + if: steps.jacocorun.outcome == 'success' run: | echo "Total sever module coverage ${{ steps.jacoco-server.outputs.coverage-overall }}" echo "Changed Files coverage ${{ steps.jacoco-server.outputs.coverage-changed-files }}" - - name: Fail PR if changed files coverage is less than ${{ matrix.changed }}% - if: | - ${{ steps.jacoco-server.outputs.coverage-changed-files < 80.0 }} + - name: Fail PR if changed files coverage is less than ${{ env.changed }}% + if: steps.jacocorun.outcome == 'success' uses: actions/github-script@v6 with: script: | - core.setFailed('Changed files coverage is less than ${{ matrix.changed }}%!') + const coverageCheckFailed = + Number('${{ steps.jacoco-server.outputs.coverage-changed-files }}') < Number('${{ env.changed }}'); + if (coverageCheckFailed) { + core.setFailed('Changed files coverage is less than ${{ env.changed }}%!'); + } + - name: Edit JaCoCo comments on build failure + if: steps.jacocorun.outcome != 'success' + uses: actions/github-script@v6 + with: + script: | + const issue_number = context.issue.number; + const owner = context.repo.owner; + const repo = context.repo.repo; + const jacocoReportRegExp = /^### JaCoCo .* code coverage report .*/; + + const comments = await github.rest.issues.listComments({ + owner, + repo, + issue_number, + }); + for (const comment of comments.data) { + const lines = comment.body.split('\n'); + if (lines.length > 0 && jacocoReportRegExp.test(lines[0])) { + await github.rest.issues.updateComment({ + owner, + repo, + comment_id: comment.id, + body: lines[0] + "\n\n### Build Failed", + }); + } + } + core.setFailed('JaCoCo test coverage report generation failed, and related PR comments were updated.'); diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 4d52f137f..e6ac47685 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -34,12 +34,3 @@ jobs: PGP_SECRET: ${{ secrets.PGP_SECRET }} SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} - - - name: Find WAR file - id: find_war - run: echo "WAR_PATH=$(find . -name '*.war' | head -n 1)" >> $GITHUB_ENV - - - name: Upload WAR file to GitHub Release - run: gh release upload ${{ github.event.release.tag_name }} ${{ env.WAR_PATH }} --repo ${{ github.repository }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/build.sbt b/build.sbt index 4573fc0b3..26758e47e 100644 --- a/build.sbt +++ b/build.sbt @@ -40,12 +40,12 @@ lazy val commonSettings = Seq( jacocoExcludes := jacocoProjectExcludes() ) -val mergeStrategy = assembly / assemblyMergeStrategy := { - case PathList("META-INF", _) => MergeStrategy.discard - case "application.conf" => MergeStrategy.concat - case "reference.conf" => MergeStrategy.concat - case _ => MergeStrategy.first -} +//val mergeStrategy = assembly / assemblyMergeStrategy := { +// case PathList("META-INF", _) => MergeStrategy.discard +// case "application.conf" => MergeStrategy.concat +// case "reference.conf" => MergeStrategy.concat +// case _ => MergeStrategy.first +//} val serverMergeStrategy = assembly / assemblyMergeStrategy := { case PathList("META-INF", "services", xs @ _*) => MergeStrategy.filterDistinctLines diff --git a/server/src/main/resources/reference.conf b/server/src/main/resources/reference.conf index 857353de1..6b93fdc5f 100644 --- a/server/src/main/resources/reference.conf +++ b/server/src/main/resources/reference.conf @@ -2,7 +2,7 @@ postgres { # The JDBC driver class dataSourceClass=org.postgresql.Driver - serverName=localhost // host.docker.internal for local run in docker against db on its host machine + serverName=localhost // host.docker.internal for local run in docker against db on its host machine; localhost otherwise for testing and for the gh pipeline # set to non-default port, to avoid local collision portNumber=5432 databaseName=atum_db From 66d0aa8a862c0a8f9dde63feab630d7ef55ef4a5 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Feb 2024 12:29:47 +0100 Subject: [PATCH 24/40] jacoco --- .github/workflows/jacoco_check.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 6134b812e..09d19c805 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -39,12 +39,12 @@ jobs: uses: olafurpg/setup-scala@v14 with: java-version: "adopt@1.8" - - name: Prepare testing database - run: sbt flywayMigrate - name: Build and run tests continue-on-error: true id: jacocorun - run: sbt jacoco + run: | + sbt "project agent; jacoco" + sbt "project model; jacoco" # agent module code coverage - name: Add coverage to PR if: steps.jacocorun.outcome == 'success' From 84e21373a5f3aadccf46a7cb48109635b7656530 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Feb 2024 12:33:32 +0100 Subject: [PATCH 25/40] docker ci --- .github/workflows/docker_image.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index 15a35edf1..ffdd558f2 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -47,12 +47,12 @@ jobs: with: java-version: "adopt@1.11.0-11" - - name: Run sbt assembly - run: sbt server/assembly - - name: Prepare testing database run: sbt flywayMigrate + - name: Run sbt assembly + run: sbt server/assembly + - name: Build the Docker image run: docker build -t absaoss/atum-service:latest \ --build-arg CONFIG=./src/main/resources/reference.conf \ @@ -64,4 +64,4 @@ jobs: - name: Execute HTTP call run: | sleep 10 # Wait for the service to start - curl -Lf http://localhost:8080/docs \ No newline at end of file + curl -Lf http://localhost:8080/docs From 85c84b20daff0a57fc0956c08360e8696c316bad Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Feb 2024 12:46:14 +0100 Subject: [PATCH 26/40] docker ci --- .github/workflows/docker_image.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index ffdd558f2..efed1754b 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -54,9 +54,7 @@ jobs: run: sbt server/assembly - name: Build the Docker image - run: docker build -t absaoss/atum-service:latest \ - --build-arg CONFIG=./src/main/resources/reference.conf \ - --build-arg SSL=false ./ + run: docker buildx build --platform linux/amd64 -t absaoss/atum-service:latest --build-arg CONFIG=./src/main/resources/reference.conf --build-arg SSL=false ./ - name: Run the Docker image run: docker run -d -p 8080:8080 absaoss/atum-service:latest From 60c1e67a584242b38d656a8c9d079883a047f51c Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Feb 2024 12:49:24 +0100 Subject: [PATCH 27/40] docker ci --- .github/workflows/docker_image.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index efed1754b..4acbe0467 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -54,7 +54,9 @@ jobs: run: sbt server/assembly - name: Build the Docker image - run: docker buildx build --platform linux/amd64 -t absaoss/atum-service:latest --build-arg CONFIG=./src/main/resources/reference.conf --build-arg SSL=false ./ + run: | + cd server + docker buildx build --platform linux/amd64 -t absaoss/atum-service:latest --build-arg CONFIG=./src/main/resources/reference.conf --build-arg SSL=false ./ - name: Run the Docker image run: docker run -d -p 8080:8080 absaoss/atum-service:latest From d49ad5c2c818d47446097ce65594c1cc59cf75e3 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Feb 2024 14:04:49 +0100 Subject: [PATCH 28/40] docker ci --- .github/workflows/docker_image.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index 4acbe0467..d5d40f641 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -58,10 +58,8 @@ jobs: cd server docker buildx build --platform linux/amd64 -t absaoss/atum-service:latest --build-arg CONFIG=./src/main/resources/reference.conf --build-arg SSL=false ./ - - name: Run the Docker image - run: docker run -d -p 8080:8080 absaoss/atum-service:latest - - - name: Execute HTTP call + - name: Run the Docker image and execute HTTP call run: | + docker run -d -p 8080:8080 absaoss/atum-service:latest sleep 10 # Wait for the service to start curl -Lf http://localhost:8080/docs From a9fd875a84dec4fc69f40523d04e7f99c4a814e0 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Feb 2024 14:09:07 +0100 Subject: [PATCH 29/40] docker ci --- .github/workflows/docker_image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index d5d40f641..ab383ec4f 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -61,5 +61,6 @@ jobs: - name: Run the Docker image and execute HTTP call run: | docker run -d -p 8080:8080 absaoss/atum-service:latest + docker container ls sleep 10 # Wait for the service to start curl -Lf http://localhost:8080/docs From 31e7c4aaf7ac0de81cc4ca23d74e5b8e8df31915 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Feb 2024 14:09:19 +0100 Subject: [PATCH 30/40] docker ci --- .github/workflows/docker_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index ab383ec4f..57a663841 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -62,5 +62,5 @@ jobs: run: | docker run -d -p 8080:8080 absaoss/atum-service:latest docker container ls - sleep 10 # Wait for the service to start + sleep 15 # Wait for the service to start curl -Lf http://localhost:8080/docs From 8d4a071a9dd1c3c07ecefd7c7b0255982ba42f70 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Feb 2024 14:14:08 +0100 Subject: [PATCH 31/40] docker ci --- .github/workflows/docker_image.yml | 2 +- build.sbt | 7 ------- project/Dependencies.scala | 15 ++++----------- project/plugins.sbt | 2 -- 4 files changed, 5 insertions(+), 21 deletions(-) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index 57a663841..f4fb58943 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -63,4 +63,4 @@ jobs: docker run -d -p 8080:8080 absaoss/atum-service:latest docker container ls sleep 15 # Wait for the service to start - curl -Lf http://localhost:8080/docs + curl -Lf http://127.0.0.1:8080/docs diff --git a/build.sbt b/build.sbt index 26758e47e..bc57efb14 100644 --- a/build.sbt +++ b/build.sbt @@ -40,13 +40,6 @@ lazy val commonSettings = Seq( jacocoExcludes := jacocoProjectExcludes() ) -//val mergeStrategy = assembly / assemblyMergeStrategy := { -// case PathList("META-INF", _) => MergeStrategy.discard -// case "application.conf" => MergeStrategy.concat -// case "reference.conf" => MergeStrategy.concat -// case _ => MergeStrategy.first -//} - val serverMergeStrategy = assembly / assemblyMergeStrategy := { case PathList("META-INF", "services", xs @ _*) => MergeStrategy.filterDistinctLines case PathList("META-INF", "maven", "org.webjars", "swagger-ui", "pom.properties") => MergeStrategy.singleOrError diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 68101fee4..38f860ae9 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -49,7 +49,7 @@ object Dependencies { val sttp = "3.5.2" val postgresql = "42.6.0" - + val fadb = "0.3.0" val json4s_spark2 = "3.5.3" @@ -64,7 +64,7 @@ object Dependencies { val sbtJunitInterface = "0.13.3" val tapir = "1.9.6" val http4sBlazeBackend = "0.23.15" - val playJson = "3.0.1"//"2.9.4" + val playJson = "3.0.1" } @@ -120,19 +120,12 @@ object Dependencies { lazy val json4sJackson = "org.json4s" %% "json4s-jackson" % json4sVersion lazy val json4sNative = "org.json4s" %% "json4s-native" % json4sVersion % Provided -// lazy val logback = "ch.qos.logback" % "logback-classic" % Versions.logback -// lazy val scalatest = "org.scalatest" %% "scalatest" % Versions.scalatest % Test -// lazy val mockito = "org.mockito" %% "mockito-scala" % Versions.scalaMockito % Test - Seq( jacksonModuleScala, json4sExt, json4sCore, json4sJackson, - json4sNative, -// logback, -// scalatest, -// mockito, + json4sNative ) } @@ -239,7 +232,7 @@ object Dependencies { balta, ) } - + def flywayDependencies: Seq[ModuleID] = { val postgresql = "org.postgresql" % "postgresql" % Versions.postgresql diff --git a/project/plugins.sbt b/project/plugins.sbt index e2b10d39f..3a360d5fd 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -27,7 +27,6 @@ addSbtPlugin("io.github.davidmweber" % "flyway-sbt" % "7.4.0") addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.12") // Plugins to build the server module as a war file -//addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.5") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.10") // sbt-jacoco dependency downloading @@ -48,4 +47,3 @@ addSbtPlugin("org.ow2.asm" % "asm-commons" % ow2Version from ow2Url("asm-commons addSbtPlugin("org.ow2.asm" % "asm-tree" % ow2Version from ow2Url("asm-tree")) addSbtPlugin("za.co.absa.sbt" % "sbt-jacoco" % "3.4.1-absa.4" from "https://github.com/AbsaOSS/sbt-jacoco/releases/download/3.4.1-absa.4/sbt-jacoco-3.4.1-absa.4.jar") - From 5a9ecaa662d6daa9c4c09d54bd73bc4ee4f979b7 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Feb 2024 14:19:12 +0100 Subject: [PATCH 32/40] docker ci --- .github/workflows/docker_image.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index f4fb58943..0c4397498 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -62,5 +62,6 @@ jobs: run: | docker run -d -p 8080:8080 absaoss/atum-service:latest docker container ls - sleep 15 # Wait for the service to start + sleep 10 + docker logs $(docker ps -lq) curl -Lf http://127.0.0.1:8080/docs From 8452f00652886dc43b219e07dee0a49297a2df17 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Feb 2024 14:23:07 +0100 Subject: [PATCH 33/40] docker ci --- server/src/main/resources/reference.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/resources/reference.conf b/server/src/main/resources/reference.conf index 6b93fdc5f..be3a3a78f 100644 --- a/server/src/main/resources/reference.conf +++ b/server/src/main/resources/reference.conf @@ -2,7 +2,7 @@ postgres { # The JDBC driver class dataSourceClass=org.postgresql.Driver - serverName=localhost // host.docker.internal for local run in docker against db on its host machine; localhost otherwise for testing and for the gh pipeline + serverName=host.docker.internal // host.docker.internal for local run in docker against db on its host machine; localhost otherwise for testing and for the gh pipeline # set to non-default port, to avoid local collision portNumber=5432 databaseName=atum_db From de4d204a561d56863f7d6119a0e54b4168b29243 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Feb 2024 14:29:26 +0100 Subject: [PATCH 34/40] docker ci --- server/src/main/resources/reference.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/resources/reference.conf b/server/src/main/resources/reference.conf index be3a3a78f..6b93fdc5f 100644 --- a/server/src/main/resources/reference.conf +++ b/server/src/main/resources/reference.conf @@ -2,7 +2,7 @@ postgres { # The JDBC driver class dataSourceClass=org.postgresql.Driver - serverName=host.docker.internal // host.docker.internal for local run in docker against db on its host machine; localhost otherwise for testing and for the gh pipeline + serverName=localhost // host.docker.internal for local run in docker against db on its host machine; localhost otherwise for testing and for the gh pipeline # set to non-default port, to avoid local collision portNumber=5432 databaseName=atum_db From 63181546abb5681d0cd864892bdb5918c5219954 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Feb 2024 14:32:46 +0100 Subject: [PATCH 35/40] docker ci --- server/src/main/resources/reference.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/resources/reference.conf b/server/src/main/resources/reference.conf index 6b93fdc5f..6ee05e598 100644 --- a/server/src/main/resources/reference.conf +++ b/server/src/main/resources/reference.conf @@ -2,7 +2,7 @@ postgres { # The JDBC driver class dataSourceClass=org.postgresql.Driver - serverName=localhost // host.docker.internal for local run in docker against db on its host machine; localhost otherwise for testing and for the gh pipeline + serverName=127.0.0.1 // host.docker.internal for local run in docker against db on its host machine; localhost otherwise for testing and for the gh pipeline # set to non-default port, to avoid local collision portNumber=5432 databaseName=atum_db From be484123e1cba5ba1542405096fb84819e964418 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Feb 2024 14:44:36 +0100 Subject: [PATCH 36/40] docker ci --- .github/workflows/docker_image.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml index 0c4397498..c07c1f781 100644 --- a/.github/workflows/docker_image.yml +++ b/.github/workflows/docker_image.yml @@ -38,6 +38,7 @@ jobs: --health-retries 5 ports: - 5432:5432 + network: network_name steps: - uses: actions/checkout@v2 @@ -60,8 +61,8 @@ jobs: - name: Run the Docker image and execute HTTP call run: | - docker run -d -p 8080:8080 absaoss/atum-service:latest + docker run -d -p 8080:8080 --network network_name absaoss/atum-service:latest docker container ls - sleep 10 + sleep 10 docker logs $(docker ps -lq) curl -Lf http://127.0.0.1:8080/docs From 0f98bbfebcea9814911373a9e767d82a1fccc867 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Feb 2024 14:52:16 +0100 Subject: [PATCH 37/40] docker ci --- .github/workflows/docker_image.yml | 68 ------------------------ server/src/main/resources/reference.conf | 2 +- 2 files changed, 1 insertion(+), 69 deletions(-) delete mode 100644 .github/workflows/docker_image.yml diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml deleted file mode 100644 index c07c1f781..000000000 --- a/.github/workflows/docker_image.yml +++ /dev/null @@ -1,68 +0,0 @@ -# -# Copyright 2021 ABSA Group Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: Docker Image CI - -on: - pull_request: - branches: [ master ] - types: [ opened, edited, synchronize, reopened ] - -jobs: - build: - runs-on: ubuntu-latest - - services: - postgres: - image: postgres:15 - env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: atum_db - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - network: network_name - - steps: - - uses: actions/checkout@v2 - - - name: Setup Scala environment - uses: olafurpg/setup-scala@v14 - with: - java-version: "adopt@1.11.0-11" - - - name: Prepare testing database - run: sbt flywayMigrate - - - name: Run sbt assembly - run: sbt server/assembly - - - name: Build the Docker image - run: | - cd server - docker buildx build --platform linux/amd64 -t absaoss/atum-service:latest --build-arg CONFIG=./src/main/resources/reference.conf --build-arg SSL=false ./ - - - name: Run the Docker image and execute HTTP call - run: | - docker run -d -p 8080:8080 --network network_name absaoss/atum-service:latest - docker container ls - sleep 10 - docker logs $(docker ps -lq) - curl -Lf http://127.0.0.1:8080/docs diff --git a/server/src/main/resources/reference.conf b/server/src/main/resources/reference.conf index 6ee05e598..6b93fdc5f 100644 --- a/server/src/main/resources/reference.conf +++ b/server/src/main/resources/reference.conf @@ -2,7 +2,7 @@ postgres { # The JDBC driver class dataSourceClass=org.postgresql.Driver - serverName=127.0.0.1 // host.docker.internal for local run in docker against db on its host machine; localhost otherwise for testing and for the gh pipeline + serverName=localhost // host.docker.internal for local run in docker against db on its host machine; localhost otherwise for testing and for the gh pipeline # set to non-default port, to avoid local collision portNumber=5432 databaseName=atum_db From d1a4ac9ce0b483b5b80b6aae255332c9fa0ca612 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Feb 2024 15:11:33 +0100 Subject: [PATCH 38/40] minors --- server/Dockerfile | 2 +- server/src/main/resources/reference.conf | 1 - .../scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/server/Dockerfile b/server/Dockerfile index 7a8a54afa..68b8ee4b7 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -97,4 +97,4 @@ ENTRYPOINT ["java","-jar","/app.jar"] # --- Final image assembly --- FROM base-ssl-${SSL} AS final -RUN echo "Running final stage with SSL_ENABLED=$SSL_ENABLED." \ No newline at end of file +RUN echo "Running final stage with SSL_ENABLED=$SSL_ENABLED." diff --git a/server/src/main/resources/reference.conf b/server/src/main/resources/reference.conf index 6b93fdc5f..817c72bbc 100644 --- a/server/src/main/resources/reference.conf +++ b/server/src/main/resources/reference.conf @@ -3,7 +3,6 @@ # The JDBC driver class dataSourceClass=org.postgresql.Driver serverName=localhost // host.docker.internal for local run in docker against db on its host machine; localhost otherwise for testing and for the gh pipeline - # set to non-default port, to avoid local collision portNumber=5432 databaseName=atum_db user=atum_user diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala b/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala index c4e492836..1f3aae830 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/PlayJsonImplicits.scala @@ -77,6 +77,4 @@ object PlayJsonImplicits { implicit val readsAtumContextDTO: Reads[AtumContextDTO] = Json.reads[AtumContextDTO] implicit val writesAtumContextDTO: Writes[AtumContextDTO] = Json.writes[AtumContextDTO] - - } From 9dc5d1b9f51be8bd84484bd1eaa7e18d309c79b4 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Feb 2024 15:59:18 +0100 Subject: [PATCH 39/40] minors --- .github/workflows/build.yml | 4 ++-- .github/workflows/release_publish.yml | 11 ++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 303c39179..e7c9bac0b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,7 +34,7 @@ jobs: - uses: coursier/cache-action@v5 - name: Setup Scala - uses: olafurpg/setup-scala@v10 + uses: olafurpg/setup-scala@v14 with: java-version: "adopt@1.8" @@ -65,7 +65,7 @@ jobs: - uses: coursier/cache-action@v5 - name: Setup Scala - uses: olafurpg/setup-scala@v10 + uses: olafurpg/setup-scala@v14 with: java-version: "adopt@1.11.0-11" diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index e6ac47685..a75203377 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -27,10 +27,19 @@ jobs: with: fetch-depth: 0 - uses: olafurpg/setup-scala@v14 - - name: Run sbt ci-release (produces war as well) + - name: Run sbt ci-release (produces jar as well) run: sbt ci-release env: PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} PGP_SECRET: ${{ secrets.PGP_SECRET }} SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} + + - name: Find JAR file + id: find_jar + run: echo "JAR_PATH=$(find . -name 'server/target/jvm-2.13/*.jar' | head -n 1)" >> $GITHUB_ENV + + - name: Upload JAR file to GitHub Release + run: gh release upload ${{ github.event.release.tag_name }} ${{ env.JAR_PATH }} --repo ${{ github.repository }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 8afce36ae25821c64038660275e4061af0a040e2 Mon Sep 17 00:00:00 2001 From: Pavel Salamon Date: Tue, 13 Feb 2024 16:12:26 +0100 Subject: [PATCH 40/40] json serde etc --- build.sbt | 6 ++---- project/Dependencies.scala | 2 +- project/plugins.sbt | 2 +- server/README.md | 5 +++-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/build.sbt b/build.sbt index bc57efb14..b3edec56c 100644 --- a/build.sbt +++ b/build.sbt @@ -18,8 +18,6 @@ import Dependencies._ import JacocoSetup._ import sbt.Keys.name -import scala.collection.immutable.Seq - ThisBuild / organization := "za.co.absa.atum-service" sonatypeProfileName := "za.co.absa" @@ -90,7 +88,7 @@ lazy val agent = (projectMatrix in file("agent")) commonSettings ++ Seq( name := "atum-agent", javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), - libraryDependencies ++= commonDependencies ++ testDependencies ++ Dependencies.agentDependencies( + libraryDependencies ++= jsonSerdeDependencies ++ testDependencies ++ Dependencies.agentDependencies( if (scalaVersion.value == Versions.scala211) Versions.spark2 else Versions.spark3, scalaVersion.value ), @@ -111,7 +109,7 @@ lazy val model = (projectMatrix in file("model")) commonSettings ++ Seq( name := "atum-model", javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), - libraryDependencies ++= commonDependencies ++ testDependencies ++ Dependencies.modelDependencies(scalaVersion.value), + libraryDependencies ++= jsonSerdeDependencies ++ testDependencies ++ Dependencies.modelDependencies(scalaVersion.value), printScalaVersion := { val log = streams.value.log log.info(s"Building ${name.value} with Scala ${scalaVersion.value}") diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 38f860ae9..f8b8541e9 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -110,7 +110,7 @@ object Dependencies { ) } - def commonDependencies: Seq[ModuleID] = { + def jsonSerdeDependencies: Seq[ModuleID] = { val json4sVersion = json4sVersionForScala(Versions.scala212) lazy val jacksonModuleScala = "com.fasterxml.jackson.module" %% "jackson-module-scala" % Versions.jacksonModuleScala diff --git a/project/plugins.sbt b/project/plugins.sbt index 3a360d5fd..321d068ee 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -26,7 +26,7 @@ addSbtPlugin("io.github.davidmweber" % "flyway-sbt" % "7.4.0") // To add release plugin addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.12") -// Plugins to build the server module as a war file +// Plugins to build the server module as a jar file addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.10") // sbt-jacoco dependency downloading diff --git a/server/README.md b/server/README.md index 7ee745bd9..551854cd8 100644 --- a/server/README.md +++ b/server/README.md @@ -2,10 +2,11 @@ ## How to build and run (will be updated soon) -To create a war file that can be deployed to tomcat just run: +To create a jar file that can be executed: ```shell -> sbt package +> sbt clean "project server" assembly +> java -jar server/target/jvm-2.13/*.jar ``` If you want to quickly build and run from sbt you can run using the command below (alternatively you can execute za.co.absa.atum.server.Main within your IDE). This deploys it to `localhost:8080`.