diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 75f68a8c7..e7c9bac0b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,7 +14,7 @@ # limitations under the License. # -name: Build +name: Build all modules on: pull_request: @@ -23,9 +23,26 @@ on: types: [ opened, synchronize, reopened ] jobs: - test: + build-agent-and-model: runs-on: ubuntu-latest + strategy: + matrix: + scala: [2.11.12, 2.12.18, 2.13.11] + steps: + - name: Checkout code + uses: actions/checkout@v2 + - uses: coursier/cache-action@v5 + - name: Setup Scala + uses: olafurpg/setup-scala@v14 + with: + java-version: "adopt@1.8" + + - name: Build and run tests + run: sbt "project agent" ++${{matrix.scala}} test doc "project model" ++${{matrix.scala}} test doc + + build-server: + runs-on: ubuntu-latest services: postgres: image: postgres:15 @@ -39,26 +56,21 @@ jobs: --health-retries 5 ports: - 5432:5432 - strategy: - fail-fast: false matrix: - scala: [2.11.12, 2.12.18, 2.13.11] - - name: Scala ${{matrix.scala}} - + scala: [2.13.11] steps: - name: Checkout code uses: actions/checkout@v2 - uses: coursier/cache-action@v5 - name: Setup Scala - uses: olafurpg/setup-scala@v10 + uses: olafurpg/setup-scala@v14 with: - java-version: "adopt@1.8" + java-version: "adopt@1.11.0-11" - name: Prepare testing database run: sbt flywayMigrate - name: Build and run tests - run: sbt ++${{matrix.scala}} test doc + run: sbt "project server" ++${{matrix.scala}} test doc diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index e9757b0e3..09d19c805 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -14,7 +14,7 @@ # limitations under the License. # -name: JaCoCo report +name: JaCoCo report agent and module on: pull_request: @@ -24,8 +24,6 @@ on: env: scalaLong12: 2.12.18 scalaShort12: "2.12" - scalaLong13: 2.13.11 - scalaShort13: "2.13" overall: 80.0 changed: 80.0 @@ -34,20 +32,6 @@ jobs: name: Build and test runs-on: ubuntu-latest - services: - postgres: - image: postgres:15 - env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: atum_db - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - steps: - name: Checkout code uses: actions/checkout@v4 @@ -55,24 +39,12 @@ jobs: uses: olafurpg/setup-scala@v14 with: java-version: "adopt@1.8" - - name: Prepare testing database - run: sbt flywayMigrate - name: Build and run tests continue-on-error: true id: jacocorun - run: sbt jacoco - # server module code coverage - - name: Add coverage to PR - if: steps.jacocorun.outcome == 'success' - id: jacoco-server - uses: madrapps/jacoco-report@v1.6.1 - with: - paths: ${{ github.workspace }}/server/target/jvm-${{ env.scalaShort13 }}/jacoco/report/jacoco.xml - token: ${{ secrets.GITHUB_TOKEN }} - min-coverage-overall: ${{env.overall }} - min-coverage-changed-files: ${{ env.changed }} - title: JaCoCo server module code coverage report - scala ${{ env.scalaLong13 }} - update-comment: true + run: | + sbt "project agent; jacoco" + sbt "project model; jacoco" # agent module code coverage - name: Add coverage to PR if: steps.jacocorun.outcome == 'success' @@ -102,8 +74,6 @@ jobs: - name: Get the Coverage info if: steps.jacocorun.outcome == 'success' run: | - echo "Total sever module coverage ${{ steps.jacoco-server.outputs.coverage-overall }}" - echo "Changed Files coverage ${{ steps.jacoco-server.outputs.coverage-changed-files }}" echo "Total agent module coverage ${{ steps.jacoco-agent.outputs.coverage-overall }}" echo "Changed Files coverage ${{ steps.jacoco-agent.outputs.coverage-changed-files }}" echo "Total model module coverage ${{ steps.jacoco-model.outputs.coverage-overall }}" @@ -114,7 +84,6 @@ jobs: with: script: | const coverageCheckFailed = - Number('${{ steps.jacoco-server.outputs.coverage-changed-files }}') < Number('${{ env.changed }}') || Number('${{ steps.jacoco-agent.outputs.coverage-changed-files }}') < Number('${{ env.changed }}') || Number('${{ steps.jacoco-model.outputs.coverage-changed-files }}') < Number('${{ env.changed }}'); if (coverageCheckFailed) { diff --git a/.github/workflows/jacoco_check_server.yml b/.github/workflows/jacoco_check_server.yml new file mode 100644 index 000000000..48e41b463 --- /dev/null +++ b/.github/workflows/jacoco_check_server.yml @@ -0,0 +1,117 @@ +# +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: JaCoCo report server + +on: + pull_request: + branches: [ master ] + types: [ opened, edited, synchronize, reopened ] + +env: + scalaLong13: 2.13.11 + scalaShort13: "2.13" + overall: 80.0 + changed: 80.0 + +jobs: + test: + name: Build and test + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:15 + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: atum_db + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Setup Scala + uses: olafurpg/setup-scala@v14 + with: + java-version: "adopt@1.11.0-11" + - name: Prepare testing database + run: sbt flywayMigrate + - name: Build and run tests + continue-on-error: true + id: jacocorun + run: sbt "project server; jacoco" + # server module code coverage + - name: Add coverage to PR + if: steps.jacocorun.outcome == 'success' + id: jacoco-server + uses: madrapps/jacoco-report@v1.6.1 + with: + paths: ${{ github.workspace }}/server/target/jvm-${{ env.scalaShort13 }}/jacoco/report/jacoco.xml + token: ${{ secrets.GITHUB_TOKEN }} + min-coverage-overall: ${{env.overall }} + min-coverage-changed-files: ${{ env.changed }} + title: JaCoCo server module code coverage report - scala ${{ env.scalaLong13 }} + update-comment: true + - name: Get the Coverage info + if: steps.jacocorun.outcome == 'success' + run: | + echo "Total sever module coverage ${{ steps.jacoco-server.outputs.coverage-overall }}" + echo "Changed Files coverage ${{ steps.jacoco-server.outputs.coverage-changed-files }}" + - name: Fail PR if changed files coverage is less than ${{ env.changed }}% + if: steps.jacocorun.outcome == 'success' + uses: actions/github-script@v6 + with: + script: | + const coverageCheckFailed = + Number('${{ steps.jacoco-server.outputs.coverage-changed-files }}') < Number('${{ env.changed }}'); + if (coverageCheckFailed) { + core.setFailed('Changed files coverage is less than ${{ env.changed }}%!'); + } + - name: Edit JaCoCo comments on build failure + if: steps.jacocorun.outcome != 'success' + uses: actions/github-script@v6 + with: + script: | + const issue_number = context.issue.number; + const owner = context.repo.owner; + const repo = context.repo.repo; + const jacocoReportRegExp = /^### JaCoCo .* code coverage report .*/; + + const comments = await github.rest.issues.listComments({ + owner, + repo, + issue_number, + }); + + for (const comment of comments.data) { + const lines = comment.body.split('\n'); + if (lines.length > 0 && jacocoReportRegExp.test(lines[0])) { + await github.rest.issues.updateComment({ + owner, + repo, + comment_id: comment.id, + body: lines[0] + "\n\n### Build Failed", + }); + } + } + + core.setFailed('JaCoCo test coverage report generation failed, and related PR comments were updated.'); diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 4d52f137f..a75203377 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -27,7 +27,7 @@ jobs: with: fetch-depth: 0 - uses: olafurpg/setup-scala@v14 - - name: Run sbt ci-release (produces war as well) + - name: Run sbt ci-release (produces jar as well) run: sbt ci-release env: PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} @@ -35,11 +35,11 @@ jobs: SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} - - name: Find WAR file - id: find_war - run: echo "WAR_PATH=$(find . -name '*.war' | head -n 1)" >> $GITHUB_ENV + - name: Find JAR file + id: find_jar + run: echo "JAR_PATH=$(find . -name 'server/target/jvm-2.13/*.jar' | head -n 1)" >> $GITHUB_ENV - - name: Upload WAR file to GitHub Release - run: gh release upload ${{ github.event.release.tag_name }} ${{ env.WAR_PATH }} --repo ${{ github.repository }} + - name: Upload JAR file to GitHub Release + run: gh release upload ${{ github.event.release.tag_name }} ${{ env.JAR_PATH }} --repo ${{ github.repository }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index e1726edd4..68eb03cb2 100644 --- a/.gitignore +++ b/.gitignore @@ -88,3 +88,6 @@ _site utils/resources/*.conf .bsp +/server/certs/ +/server/selfsigned.crt +/server/selfsigned.p12 diff --git a/build.sbt b/build.sbt index 46cc208d9..b3edec56c 100644 --- a/build.sbt +++ b/build.sbt @@ -21,7 +21,7 @@ import sbt.Keys.name ThisBuild / organization := "za.co.absa.atum-service" sonatypeProfileName := "za.co.absa" -ThisBuild / scalaVersion := Versions.scala212 // default version +ThisBuild / scalaVersion := Versions.scala213 // default version ThisBuild / versionScheme := Some("early-semver") @@ -33,13 +33,23 @@ lazy val printSparkScalaVersion = taskKey[Unit]("Print Spark and Scala versions lazy val printScalaVersion = taskKey[Unit]("Print Scala versions for atum-service is being built for.") lazy val commonSettings = Seq( - libraryDependencies ++= commonDependencies, scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings"), - javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), Test / parallelExecution := false, jacocoExcludes := jacocoProjectExcludes() ) +val serverMergeStrategy = assembly / assemblyMergeStrategy := { + case PathList("META-INF", "services", xs @ _*) => MergeStrategy.filterDistinctLines + case PathList("META-INF", "maven", "org.webjars", "swagger-ui", "pom.properties") => MergeStrategy.singleOrError + case PathList("META-INF", "resources", "webjars", "swagger-ui", _*) => MergeStrategy.singleOrError + case PathList("META-INF", _*) => MergeStrategy.discard + case PathList("META-INF", "versions", "9", xs@_*) => MergeStrategy.discard + case PathList("module-info.class") => MergeStrategy.discard + case "application.conf" => MergeStrategy.concat + case "reference.conf" => MergeStrategy.concat + case _ => MergeStrategy.first +} + enablePlugins(FlywayPlugin) flywayUrl := FlywayConfiguration.flywayUrl flywayUser := FlywayConfiguration.flywayUser @@ -52,8 +62,9 @@ lazy val server = (projectMatrix in file("server")) .settings( commonSettings ++ Seq( name := "atum-server", - libraryDependencies ++= Dependencies.serverDependencies, - scalacOptions ++= Seq("-Ymacro-annotations"), + libraryDependencies ++= Dependencies.serverDependencies ++ testDependencies, + javacOptions ++= Seq("-source", "11", "-target", "11", "-Xlint"), + scalacOptions ++= Seq("-release", "11", "-Ymacro-annotations"), Compile / packageBin / publishArtifact := false, printScalaVersion := { val log = streams.value.log @@ -61,15 +72,13 @@ lazy val server = (projectMatrix in file("server")) }, (Compile / compile) := ((Compile / compile) dependsOn printScalaVersion).value, packageBin := (Compile / assembly).value, - artifactPath / (Compile / packageBin) := baseDirectory.value / s"target/${name.value}-${version.value}.war", - webappWebInfClasses := true, - inheritJarManifest := true, - publish / skip := true, - jacocoReportSettings := jacocoSettings(scalaVersion.value, "atum-server") + artifactPath / (Compile / packageBin) := baseDirectory.value / s"target/${name.value}-${version.value}.jar", + testFrameworks += new TestFramework("zio.test.sbt.ZTestFramework"), + jacocoReportSettings := jacocoSettings(scalaVersion.value, "atum-server"), + serverMergeStrategy ): _* ) .enablePlugins(AssemblyPlugin) - .enablePlugins(TomcatPlugin) .enablePlugins(AutomateHeaderPlugin) .jvmPlatform(scalaVersions = Seq(Versions.serviceScalaVersion)) .dependsOn(model) @@ -77,8 +86,9 @@ lazy val server = (projectMatrix in file("server")) lazy val agent = (projectMatrix in file("agent")) .settings( commonSettings ++ Seq( - name := "agent", - libraryDependencies ++= Dependencies.agentDependencies( + name := "atum-agent", + javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), + libraryDependencies ++= jsonSerdeDependencies ++ testDependencies ++ Dependencies.agentDependencies( if (scalaVersion.value == Versions.scala211) Versions.spark2 else Versions.spark3, scalaVersion.value ), @@ -97,8 +107,9 @@ lazy val agent = (projectMatrix in file("agent")) lazy val model = (projectMatrix in file("model")) .settings( commonSettings ++ Seq( - name := "model", - libraryDependencies ++= Dependencies.modelDependencies(scalaVersion.value), + name := "atum-model", + javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), + libraryDependencies ++= jsonSerdeDependencies ++ testDependencies ++ Dependencies.modelDependencies(scalaVersion.value), printScalaVersion := { val log = streams.value.log log.info(s"Building ${name.value} with Scala ${scalaVersion.value}") diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 385e6e1eb..f8b8541e9 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -49,7 +49,7 @@ object Dependencies { val sttp = "3.5.2" val postgresql = "42.6.0" - + val fadb = "0.3.0" val json4s_spark2 = "3.5.3" @@ -59,11 +59,12 @@ object Dependencies { val zio = "2.0.19" val zioLogging = "2.2.0" + val logbackZio = "1.4.7" val zioConfig = "4.0.1" val sbtJunitInterface = "0.13.3" val tapir = "1.9.6" val http4sBlazeBackend = "0.23.15" - val playJson = "2.9.4" + val playJson = "3.0.1" } @@ -99,7 +100,17 @@ object Dependencies { } } - def commonDependencies: Seq[ModuleID] = { + def testDependencies: Seq[ModuleID] = { + lazy val scalatest = "org.scalatest" %% "scalatest" % Versions.scalatest % Test + lazy val mockito = "org.mockito" %% "mockito-scala" % Versions.scalaMockito % Test + + Seq( + scalatest, + mockito, + ) + } + + def jsonSerdeDependencies: Seq[ModuleID] = { val json4sVersion = json4sVersionForScala(Versions.scala212) lazy val jacksonModuleScala = "com.fasterxml.jackson.module" %% "jackson-module-scala" % Versions.jacksonModuleScala @@ -109,19 +120,12 @@ object Dependencies { lazy val json4sJackson = "org.json4s" %% "json4s-jackson" % json4sVersion lazy val json4sNative = "org.json4s" %% "json4s-native" % json4sVersion % Provided - lazy val logback = "ch.qos.logback" % "logback-classic" % Versions.logback - lazy val scalatest = "org.scalatest" %% "scalatest" % Versions.scalatest % Test - lazy val mockito = "org.mockito" %% "mockito-scala" % Versions.scalaMockito % Test - Seq( jacksonModuleScala, json4sExt, json4sCore, json4sJackson, - json4sNative, - logback, - scalatest, - mockito, + json4sNative ) } @@ -130,13 +134,16 @@ object Dependencies { val tapirOrg = "com.softwaremill.sttp.tapir" val http4sOrg = "org.http4s" val faDbOrg = "za.co.absa.fa-db" - val playOrg = "com.typesafe.play" + val playOrg = "org.playframework" val sbtOrg = "com.github.sbt" + val logbackOrg = "ch.qos.logback" // zio lazy val zioCore = zioOrg %% "zio" % Versions.zio lazy val zioMacros = zioOrg %% "zio-macros" % Versions.zio lazy val zioLogging = zioOrg %% "zio-logging" % Versions.zioLogging + lazy val slf4jLogging = zioOrg %% "zio-logging-slf4j2" % Versions.zioLogging + lazy val logback = logbackOrg % "logback-classic" % Versions.logbackZio lazy val zioConfig = zioOrg %% "zio-config" % Versions.zioConfig lazy val zioConfigMagnolia = zioOrg %% "zio-config-magnolia" % Versions.zioConfig lazy val zioConfigTypesafe = zioOrg %% "zio-config-typesafe" % Versions.zioConfig @@ -166,6 +173,8 @@ object Dependencies { zioCore, zioMacros, zioLogging, + slf4jLogging, + logback, zioConfig, zioConfigMagnolia, zioConfigTypesafe, @@ -194,13 +203,16 @@ object Dependencies { lazy val sttp = "com.softwaremill.sttp.client3" %% "core" % Versions.sttp + lazy val logback = "ch.qos.logback" % "logback-classic" % Versions.logback + Seq( sparkCore, sparkSql, typeSafeConfig, sparkCommons, sparkCommonsTest, - sttp + sttp, + logback ) } @@ -220,7 +232,7 @@ object Dependencies { balta, ) } - + def flywayDependencies: Seq[ModuleID] = { val postgresql = "org.postgresql" % "postgresql" % Versions.postgresql diff --git a/project/plugins.sbt b/project/plugins.sbt index f5ca6af36..321d068ee 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -26,7 +26,7 @@ addSbtPlugin("io.github.davidmweber" % "flyway-sbt" % "7.4.0") // To add release plugin addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.12") -// Plugins to build the server module as a war file +// Plugins to build the server module as a jar file addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.10") // sbt-jacoco dependency downloading @@ -47,4 +47,3 @@ addSbtPlugin("org.ow2.asm" % "asm-commons" % ow2Version from ow2Url("asm-commons addSbtPlugin("org.ow2.asm" % "asm-tree" % ow2Version from ow2Url("asm-tree")) addSbtPlugin("za.co.absa.sbt" % "sbt-jacoco" % "3.4.1-absa.4" from "https://github.com/AbsaOSS/sbt-jacoco/releases/download/3.4.1-absa.4/sbt-jacoco-3.4.1-absa.4.jar") - diff --git a/server/Dockerfile b/server/Dockerfile index a5bb8f1ac..68b8ee4b7 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -13,13 +13,12 @@ # limitations under the License. # - # Atum Service docker file # build via (docker root = project root): # docker build -t absaoss/atum-service:latest \ # --build-arg BUILD_PROXY=http://my.cool.proxy.here:3128 \ -# --build-arg CONFIG=./path/to/my.awesome.local.application.properties \ +# --build-arg CONFIG=./path/to/my.awesome.local.reference.conf \ # --build-arg SSL=true \ # --build-arg SSL_DNAME="CN=*.my.domain.com, OU=project1, O=mycorp, L=Johannesburg, ST=Gauteng, C=za" . # run via @@ -30,10 +29,10 @@ # https://localhost:8443/token/public-key # Conditional Docker image creation idea: https://stackoverflow.com/a/60820156/1773349 -# change to OFF to disable SSL -ARG SSL=true -# Tomcat OS base image, feel free to use another (e.g. your hardened one) -ARG BASE_IMAGE=tomcat:9-jdk8-corretto +# change to false to disable SSL +ARG SSL +# Amazon correto base image +ARG BASE_IMAGE=amazoncorretto:11.0.22 # --- Base image part (common for SSL true|false) --- FROM $BASE_IMAGE as base @@ -41,7 +40,7 @@ FROM $BASE_IMAGE as base # Provide your proxy if needed, e.g. http://my.proxy.examle.com:3128 ARG BUILD_PROXY # Override of the example application config is possible -ARG CONFIG=./src/main/resources/application.properties +ARG CONFIG=./src/main/resources/resource.conf # Provide path to the directory with LDAP certs in PEM format ARG LDAP_SSL_CERTS_PATH # ARG SSL_DNAME is defined below in the SSL-enabled image @@ -52,13 +51,11 @@ ARG SCALA_VERSION=2.13 LABEL org.opencontainers.image.authors="ABSA" -# Copy Spring application properties -COPY $CONFIG /opt/application.properties - -# deploy as root application in tomcat -COPY ${AS_PREFIX}/target/jvm-${SCALA_VERSION}/*.war /usr/local/tomcat/webapps/ROOT.war +# The application's jar file +ARG JAR_FILE=${AS_PREFIX}/target/jvm-${SCALA_VERSION}/*.jar -ENV SPRING_CONFIG_LOCATION=/opt/application.properties +# Add the application's jar to the container +ADD ${JAR_FILE} app.jar ENV http_proxy=$BUILD_PROXY ENV https_proxy=$BUILD_PROXY @@ -71,30 +68,24 @@ COPY $LDAP_SSL_CERTS_PATH /opt/certs/ RUN for file in `ls /opt/certs/*.pem`; \ do \ - echo yes | keytool -import -file $file -alias ldaps$RANDOM -keystore $JAVA_HOME/jre/lib/security/cacerts -storepass changeit; \ + echo yes | keytool -import -file $file -alias ldaps$RANDOM -keystore /usr/lib/jvm/java-11-amazon-corretto/lib/security/cacerts -storepass changeit; \ done -# uncomment and add packages you would need to install via yum if any -#RUN yum -y update -#RUN yum -y install htop procps - # --- SSL=true image specifics --- FROM base AS base-ssl-true ENV SSL_ENABLED=true RUN echo "This stage sets SSL=$SSL_ENABLED" -# Enable SSL: Add our SSL connector after is found in server.xml; .bak = backup -RUN sed -i.bak '//a ' /usr/local/tomcat/conf/server.xml # DNAME for self-signed cert, only applied for SSL=true ARG SSL_DNAME="CN=*.my.example.com, OU=project1, O=yourcompany, L=Johannesburg, ST=Gauteng, C=za" -# A self-seigned certificate for HTTPS -RUN keytool -genkeypair -keyalg RSA -alias tomcat -keysize 2048 \ +# A self-signed certificate for HTTPS +RUN keytool -genkeypair -keyalg RSA -alias selfsigned -keysize 2048 \ -dname "$SSL_DNAME" \ - -validity 365 -storepass changeit -keystore /usr/local/tomcat/conf/selfsigned.p12 -storetype PKCS12 + -validity 365 -storepass changeit -keystore /etc/ssl/certs/selfsigned.jks -storetype JKS EXPOSE 8080 8443 -CMD ["catalina.sh", "run"] +ENTRYPOINT ["java","-jar","/app.jar"] # --- SSL=false image specifics --- FROM base AS base-ssl-false @@ -102,7 +93,7 @@ ENV SSL_ENABLED=false RUN echo "This stage sets SSL=$SSL_ENABLED" EXPOSE 8080 -CMD ["catalina.sh", "run"] +ENTRYPOINT ["java","-jar","/app.jar"] # --- Final image assembly --- FROM base-ssl-${SSL} AS final diff --git a/server/README.md b/server/README.md index 7ee745bd9..551854cd8 100644 --- a/server/README.md +++ b/server/README.md @@ -2,10 +2,11 @@ ## How to build and run (will be updated soon) -To create a war file that can be deployed to tomcat just run: +To create a jar file that can be executed: ```shell -> sbt package +> sbt clean "project server" assembly +> java -jar server/target/jvm-2.13/*.jar ``` If you want to quickly build and run from sbt you can run using the command below (alternatively you can execute za.co.absa.atum.server.Main within your IDE). This deploys it to `localhost:8080`. diff --git a/server/certs/.gitkeep b/server/certs/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/server/src/main/resources/logback.xml b/server/src/main/resources/logback.xml new file mode 100644 index 000000000..acd40c3d8 --- /dev/null +++ b/server/src/main/resources/logback.xml @@ -0,0 +1,13 @@ + + + + + %d{HH:mm:ss.SSS} %-5level [%thread] %logger{36}:%L - %msg %ex{short}%n + + + + + + + + diff --git a/server/src/main/resources/reference.conf b/server/src/main/resources/reference.conf index 7dabe41e0..817c72bbc 100644 --- a/server/src/main/resources/reference.conf +++ b/server/src/main/resources/reference.conf @@ -1,23 +1,18 @@ { - logger { - # This format includes timestamp, level, thread (fiberId), message, and cause - format = "%label{timestamp}{%fixed{32}{%timestamp}} %label{level}{%level} %label{thread}{%fiberId} %label{class}{%name:%line} %label{message}{%message} %label{cause}{%cause}" - # log filter - filter { - # rootLevel sets the minimum level of log messages that will be displayed - rootLevel = INFO - } - } - postgres { - # The JDBC driver class - dataSourceClass=org.postgresql.Driver - serverName=localhost - # set to non-default port, to avoid local collision - portNumber=5432 - databaseName=atum_db - user=atum_user - password=changeme - # maximum number of connections that HikariCP will keep in the pool, including both idle and in-use connections - maxPoolSize=10 - } + postgres { + # The JDBC driver class + dataSourceClass=org.postgresql.Driver + serverName=localhost // host.docker.internal for local run in docker against db on its host machine; localhost otherwise for testing and for the gh pipeline + portNumber=5432 + databaseName=atum_db + user=atum_user + password=changeme + # maximum number of connections that HikariCP will keep in the pool, including both idle and in-use connections + maxPoolSize=10 + } + ssl { + enabled=false + keyStorePassword=password + keyStorePath="/path/to/your/cert" + } } diff --git a/server/src/main/scala/za/co/absa/atum/server/Main.scala b/server/src/main/scala/za/co/absa/atum/server/Main.scala index 9c0d21d0d..6afb1f2ed 100644 --- a/server/src/main/scala/za/co/absa/atum/server/Main.scala +++ b/server/src/main/scala/za/co/absa/atum/server/Main.scala @@ -17,14 +17,14 @@ package za.co.absa.atum.server import za.co.absa.atum.server.api.controller._ -import za.co.absa.atum.server.api.database.{PostgresDatabaseProvider, TransactorProvider} import za.co.absa.atum.server.api.database.runs.functions.{CreatePartitioningIfNotExists, WriteCheckpoint} +import za.co.absa.atum.server.api.database.{PostgresDatabaseProvider, TransactorProvider} import za.co.absa.atum.server.api.http.Server import za.co.absa.atum.server.api.repository.{CheckpointRepositoryImpl, PartitioningRepositoryImpl} import za.co.absa.atum.server.api.service.{CheckpointServiceImpl, PartitioningServiceImpl} -import zio.config.typesafe.TypesafeConfigProvider -import zio.logging.consoleLogger import zio._ +import zio.config.typesafe.TypesafeConfigProvider +import zio.logging.backend.SLF4J object Main extends ZIOAppDefault with Server { @@ -47,6 +47,6 @@ object Main extends ZIOAppDefault with Server { ) override val bootstrap: ZLayer[Any, Config.Error, Unit] = - Runtime.removeDefaultLoggers >>> Runtime.setConfigProvider(configProvider) >>> consoleLogger() + Runtime.removeDefaultLoggers >>> SLF4J.slf4j >>> Runtime.setConfigProvider(configProvider) } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala index 9ac29565f..8c6135476 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/CreatePartitioningIfNotExists.scala @@ -19,8 +19,8 @@ package za.co.absa.atum.server.api.database.runs.functions import doobie.Fragment import doobie.implicits.toSqlInterpolator import doobie.util.Read +import play.api.libs.json.Json import za.co.absa.atum.model.dto.PartitioningSubmitDTO -import za.co.absa.atum.model.utils.SerializationUtils import za.co.absa.atum.server.model.PartitioningForDB import za.co.absa.fadb.DBSchema import za.co.absa.fadb.doobie.{DoobieEngine, StatusWithData} @@ -28,7 +28,6 @@ import za.co.absa.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatu import za.co.absa.fadb.status.handling.implementations.StandardStatusHandling import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs - import zio._ import zio.interop.catz._ @@ -38,22 +37,22 @@ class CreatePartitioningIfNotExists(implicit schema: DBSchema, dbEngine: DoobieE override def sql(values: PartitioningSubmitDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningNormalized = SerializationUtils.asJson(partitioning) + val partitioningJsonString = Json.toJson(partitioning).toString - val parentPartitioningNormalized = values.parentPartitioning.map { parentPartitioning => + val parentPartitioningJsonString = values.parentPartitioning.map { parentPartitioning => val parentPartitioningForDB = PartitioningForDB.fromSeqPartitionDTO(parentPartitioning) - SerializationUtils.asJson(parentPartitioningForDB) + Json.toJson(parentPartitioningForDB).toString } sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( ${ import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString - partitioningNormalized + partitioningJsonString }, ${values.authorIfNew}, ${ import za.co.absa.atum.server.api.database.DoobieImplicits.Jsonb.jsonbPutUsingString - parentPartitioningNormalized + parentPartitioningJsonString } ) ${Fragment.const(alias)};""" } diff --git a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala index bcd9882a1..35b615dad 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/database/runs/functions/WriteCheckpoint.scala @@ -20,7 +20,6 @@ import doobie.Fragment import doobie.implicits._ import doobie.util.Read import za.co.absa.atum.model.dto.CheckpointDTO -import za.co.absa.atum.model.utils.SerializationUtils import za.co.absa.atum.server.model.PartitioningForDB import za.co.absa.fadb.DBSchema import za.co.absa.fadb.doobie.{DoobieEngine, StatusWithData} @@ -28,9 +27,10 @@ import za.co.absa.fadb.doobie.DoobieFunction.DoobieSingleResultFunctionWithStatu import za.co.absa.fadb.status.handling.implementations.StandardStatusHandling import za.co.absa.atum.server.api.database.PostgresDatabaseProvider import za.co.absa.atum.server.api.database.runs.Runs - import zio._ import zio.interop.catz._ +import play.api.libs.json.Json +import za.co.absa.atum.server.model.PlayJsonImplicits.writesMeasurementDTO import doobie.postgres.implicits._ @@ -40,11 +40,12 @@ class WriteCheckpoint(implicit schema: DBSchema, dbEngine: DoobieEngine[Task]) override def sql(values: CheckpointDTO)(implicit read: Read[StatusWithData[Unit]]): Fragment = { val partitioning = PartitioningForDB.fromSeqPartitionDTO(values.partitioning) - val partitioningNormalized = SerializationUtils.asJson(partitioning) + val partitioningNormalized = Json.toJson(partitioning).toString // List[String] containing json data has to be properly escaped // It would be safer to use Json data type and derive Put instance - val measurementsNormalized = - values.measurements.map(x => s"\"${SerializationUtils.asJson(x).replaceAll("\"", "\\\\\"")}\"") + val measurementsNormalized = { + values.measurements.map(x => s"\"${Json.toJson(x).toString.replaceAll("\"", "\\\\\"")}\"") + } sql"""SELECT ${Fragment.const(selectEntry)} FROM ${Fragment.const(functionName)}( ${ diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/SSL.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/SSL.scala new file mode 100644 index 000000000..d31072301 --- /dev/null +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/SSL.scala @@ -0,0 +1,60 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.server.api.http + +import za.co.absa.atum.server.config.SslConfig +import zio.ZIO + +import java.io.FileInputStream +import java.security.{KeyStore, SecureRandom} +import javax.net.ssl.{KeyManagerFactory, SSLContext, TrustManagerFactory} + +object SSL { + + val context: ZIO[Any, Throwable, SSLContext] = { + for { + sslConfig <- ZIO.config[SslConfig](SslConfig.config) + _ <- ZIO.logDebug("Attempting to initialize SSLContext") + keyStore <- ZIO.attempt { + val keyStore = KeyStore.getInstance(KeyStore.getDefaultType) + val in = new FileInputStream(sslConfig.keyStorePath) + keyStore.load(in, sslConfig.keyStorePassword.toCharArray) + keyStore + } + _ <- ZIO.logDebug("KeyStore instantiated") + keyManagerFactory <- ZIO.attempt { + val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm) + keyManagerFactory.init(keyStore, sslConfig.keyStorePassword.toCharArray) + keyManagerFactory + } + _ <- ZIO.logDebug("KeyManagerFactory initialized") + trustManagerFactory <- ZIO.attempt { + val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm) + trustManagerFactory.init(keyStore) + trustManagerFactory + } + _ <- ZIO.logDebug("TrustManagerFactory initialized") + sslContext <- ZIO.attempt { + val sslContext = SSLContext.getInstance("TLS") + sslContext.init(keyManagerFactory.getKeyManagers, trustManagerFactory.getTrustManagers, new SecureRandom()) + sslContext + } + _ <- ZIO.logDebug("SSLContext successfully initialized") + } yield sslContext + } + +} diff --git a/server/src/main/scala/za/co/absa/atum/server/api/http/Server.scala b/server/src/main/scala/za/co/absa/atum/server/api/http/Server.scala index f38fd5d63..b38a67626 100644 --- a/server/src/main/scala/za/co/absa/atum/server/api/http/Server.scala +++ b/server/src/main/scala/za/co/absa/atum/server/api/http/Server.scala @@ -31,12 +31,15 @@ import sttp.tapir.server.interceptor.decodefailure.DefaultDecodeFailureHandler.r import sttp.tapir.server.model.ValuedEndpointOutput import sttp.tapir.swagger.bundle.SwaggerInterpreter import sttp.tapir.ztapir._ -import sttp.tapir.{DecodeResult, Endpoint, PublicEndpoint, headers, statusCode} +import sttp.tapir.{DecodeResult, PublicEndpoint, headers, statusCode} import za.co.absa.atum.server.Constants.{SwaggerApiName, SwaggerApiVersion} import za.co.absa.atum.server.api.controller._ +import za.co.absa.atum.server.config.SslConfig import za.co.absa.atum.server.model.BadRequestResponse import zio.interop.catz._ -import zio.{RIO, ZIO} +import zio._ + +import javax.net.ssl.SSLContext trait Server extends Endpoints { @@ -88,14 +91,25 @@ trait Server extends Endpoints { .toRoutes } - protected val server: ZIO[Env, Throwable, Unit] = + private def createServer(port: Int, sslContext: Option[SSLContext] = None): ZIO[Env, Throwable, Unit] = ZIO.executor.flatMap { executor => - BlazeServerBuilder[F] + val builder = BlazeServerBuilder[F] + .bindHttp(port, "0.0.0.0") .withExecutionContext(executor.asExecutionContext) .withHttpApp(Router("/" -> (createAllServerRoutes <+> createSwaggerRoutes)).orNotFound) - .serve - .compile - .drain + + val builderWithSsl = sslContext.fold(builder)(ctx => builder.withSslContext(ctx)) + builderWithSsl.serve.compile.drain } + private val httpServer: ZIO[Env, Throwable, Unit] = createServer(8080) + private val httpsServer: ZIO[Env, Throwable, Unit] = SSL.context.flatMap { context => + createServer(8443, Some(context)) + } + + protected val server: ZIO[Env, Throwable, Unit] = for { + sslConfig <- ZIO.config[SslConfig](SslConfig.config) + server <- if (sslConfig.enabled) httpsServer else httpServer + } yield server + } diff --git a/server/src/main/scala/za/co/absa/atum/server/config/SslConfig.scala b/server/src/main/scala/za/co/absa/atum/server/config/SslConfig.scala new file mode 100644 index 000000000..77b032548 --- /dev/null +++ b/server/src/main/scala/za/co/absa/atum/server/config/SslConfig.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package za.co.absa.atum.server.config + +import zio.Config +import zio.config.magnolia.deriveConfig + +case class SslConfig( + enabled: Boolean, + keyStorePassword: String, + keyStorePath: String +) + +object SslConfig { + val config: Config[SslConfig] = deriveConfig[SslConfig].nested("ssl") +} diff --git a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala index df2ba109e..b03c89c3c 100644 --- a/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala +++ b/server/src/main/scala/za/co/absa/atum/server/model/PartitioningForDB.scala @@ -16,6 +16,7 @@ package za.co.absa.atum.server.model +import play.api.libs.json.{Json, Writes} import za.co.absa.atum.model.dto.PartitioningDTO private[server] case class PartitioningForDB private ( @@ -32,4 +33,7 @@ object PartitioningForDB { PartitioningForDB(keys = allKeys, keysToValuesMap = mapOfKeysAndValues) } + + implicit val writes: Writes[PartitioningForDB] = Json.writes + } diff --git a/server/src/test/resources/logback-test.xml b/server/src/test/resources/logback-test.xml new file mode 100644 index 000000000..6e2afba1d --- /dev/null +++ b/server/src/test/resources/logback-test.xml @@ -0,0 +1,12 @@ + + + + + %d{HH:mm:ss.SSS} [%thread] [%kvp] %-5level %logger{36} %msg%n + + + + + + + diff --git a/server/src/test/resources/reference.conf b/server/src/test/resources/reference.conf index 7b8fc5b4f..9279d9667 100644 --- a/server/src/test/resources/reference.conf +++ b/server/src/test/resources/reference.conf @@ -1,13 +1,4 @@ { - logger { - # This format includes timestamp, level, thread (fiberId), message, and cause - format = "%label{timestamp}{%fixed{32}{%timestamp}} %label{level}{%level} %label{thread}{%fiberId} %label{class}{%name:%line} %label{message}{%message} %label{cause}{%cause}" - # log filter - filter { - # rootLevel sets the minimum level of log messages that will be displayed - rootLevel = INFO - } - } postgres { # The JDBC driver class dataSourceClass=org.postgresql.Driver