Move Data Science Pipelines Dockerfiles to a different path

opendatahub-io · Nov 21, 2024 · 1806d20 · 1806d20
1 parent 7c3b020
commit 1806d20
Show file tree

Hide file tree

Showing 24 changed files with 652 additions and 147 deletions.
diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml
@@ -74,17 +74,17 @@ jobs:
       matrix:
         include:
           - image: ds-pipelines-api-server
-            dockerfile: backend/Dockerfile
+            dockerfile: odh_images/Dockerfile
           - image: ds-pipelines-frontend
             dockerfile: frontend/Dockerfile
           - image: ds-pipelines-persistenceagent
-            dockerfile: backend/Dockerfile.persistenceagent
+            dockerfile: odh_images/Dockerfile.persistenceagent
           - image: ds-pipelines-scheduledworkflow
-            dockerfile: backend/Dockerfile.scheduledworkflow
+            dockerfile: odh_images/Dockerfile.scheduledworkflow
           - image: ds-pipelines-driver
-            dockerfile: backend/Dockerfile.driver
+            dockerfile: odh_images/Dockerfile.driver
           - image: ds-pipelines-launcher
-            dockerfile: backend/Dockerfile.launcher
+            dockerfile: odh_images/Dockerfile.launcher
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/build

diff --git a/.github/workflows/build-master.yml b/.github/workflows/build-master.yml
@@ -20,17 +20,17 @@ jobs:
       matrix:
         include:
           - image: ds-pipelines-api-server
-            dockerfile: backend/Dockerfile
+            dockerfile: odh_images/Dockerfile
           - image: ds-pipelines-frontend
             dockerfile: frontend/Dockerfile
           - image: ds-pipelines-persistenceagent
-            dockerfile: backend/Dockerfile.persistenceagent
+            dockerfile: odh_images/Dockerfile.persistenceagent
           - image: ds-pipelines-scheduledworkflow
-            dockerfile: backend/Dockerfile.scheduledworkflow
+            dockerfile: odh_images/Dockerfile.scheduledworkflow
           - image: ds-pipelines-driver
-            dockerfile: backend/Dockerfile.driver
+            dockerfile: odh_images/Dockerfile.driver
           - image: ds-pipelines-launcher
-            dockerfile: backend/Dockerfile.launcher
+            dockerfile: odh_images/Dockerfile.launcher
     steps:
       - uses: actions/checkout@v3
       - name: Generate Tag

diff --git a/.github/workflows/build-prs.yml b/.github/workflows/build-prs.yml
@@ -69,17 +69,17 @@ jobs:
       matrix:
         include:
           - image: ds-pipelines-api-server
-            dockerfile: backend/Dockerfile
+            dockerfile: odh_images/Dockerfile
           - image: ds-pipelines-frontend
             dockerfile: frontend/Dockerfile
           - image: ds-pipelines-persistenceagent
-            dockerfile: backend/Dockerfile.persistenceagent
+            dockerfile: odh_images/Dockerfile.persistenceagent
           - image: ds-pipelines-scheduledworkflow
-            dockerfile: backend/Dockerfile.scheduledworkflow
+            dockerfile: odh_images/Dockerfile.scheduledworkflow
           - image: ds-pipelines-driver
-            dockerfile: backend/Dockerfile.driver
+            dockerfile: odh_images/Dockerfile.driver
           - image: ds-pipelines-launcher
-            dockerfile: backend/Dockerfile.launcher
+            dockerfile: odh_images/Dockerfile.launcher
     steps:
       - uses: actions/checkout@v3
       - name: Build Image

diff --git a/backend/Dockerfile b/backend/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2021-2024 The Kubeflow Authors
+# Copyright 2021-2022 The Kubeflow Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,47 +12,79 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Build arguments
-ARG SOURCE_CODE=.
+# 1. Build api server application
+FROM golang:1.21.7-bookworm as builder
+RUN apt-get update && apt-get install -y cmake clang musl-dev openssl
+WORKDIR /go/src/github.com/kubeflow/pipelines
 
-FROM registry.access.redhat.com/ubi8/go-toolset:1.21 as builder
+COPY ./go.mod ./
+COPY ./go.sum ./
+COPY ./hack/install-go-licenses.sh ./hack/
+
+RUN GO111MODULE=on go mod download
+RUN ./hack/install-go-licenses.sh
+
+COPY . .
+RUN GO111MODULE=on go build -o /bin/apiserver backend/src/apiserver/*.go
+# Check licenses and comply with license terms.
+# First, make sure there's no forbidden license.
+RUN go-licenses check ./backend/src/apiserver
+RUN go-licenses csv ./backend/src/apiserver > /tmp/licenses.csv && \
+  diff /tmp/licenses.csv backend/third_party_licenses/apiserver.csv && \
+  go-licenses save ./backend/src/apiserver --save_path /tmp/NOTICES
+
+# 2. Compile preloaded pipeline samples
+FROM python:3.9 as compiler
+RUN apt-get update -y && apt-get install --no-install-recommends -y -q default-jdk python3-setuptools python3-dev jq
+RUN wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py
+COPY backend/requirements.txt .
+RUN python3 -m pip install -r requirements.txt --no-cache-dir
 
-USER root
 # Downloading Argo CLI so that the samples are validated
 ENV ARGO_VERSION v3.4.17
 RUN curl -sLO https://github.com/argoproj/argo-workflows/releases/download/${ARGO_VERSION}/argo-linux-amd64.gz && \
     gunzip argo-linux-amd64.gz && \
     chmod +x argo-linux-amd64 && \
     mv ./argo-linux-amd64 /usr/local/bin/argo
 
-RUN dnf install -y cmake clang openssl
-
-COPY ${SOURCE_CODE}/go.mod ./
-COPY ${SOURCE_CODE}/go.sum ./
+WORKDIR /
+COPY ./samples /samples
+COPY backend/src/apiserver/config/sample_config.json /samples/
 
-RUN GO111MODULE=on go mod download
-
-# Copy the source
-COPY ${SOURCE_CODE}/ ./
+# Compiling the preloaded samples.
+# The default image is replaced with the GCR-hosted python image.
+RUN set -e; \
+    < /samples/sample_config.json jq .[].file --raw-output | while read pipeline_yaml; do \
+    pipeline_py="${pipeline_yaml%.yaml}"; \
+    python3 "$pipeline_py"; \
+    done
 
-RUN GO111MODULE=on go build -o /bin/apiserver ./backend/src/apiserver/ && \
-    dnf clean all
+# 3. Start api web server
+FROM debian:stable
 
-FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
+ARG COMMIT_SHA=unknown
+ENV COMMIT_SHA=${COMMIT_SHA}
+ARG TAG_NAME=unknown
+ENV TAG_NAME=${TAG_NAME}
+ENV LOG_LEVEL info 
 
 WORKDIR /bin
 
-COPY --from=builder /opt/app-root/src/backend/src/apiserver/config/ /config
+COPY backend/src/apiserver/config/ /config
 COPY --from=builder /bin/apiserver /bin/apiserver
 
+# Copy licenses and notices.
+COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv
+COPY --from=builder /tmp/NOTICES /third_party/NOTICES
+COPY --from=compiler /samples/ /samples/
 RUN chmod +x /bin/apiserver
 
-USER root
-
 # Adding CA certificate so API server can download pipeline through URL and wget is used for liveness/readiness probe command
-RUN microdnf install -y ca-certificates wget
+RUN apt-get update && apt-get install -y ca-certificates wget
 
-USER 1001
+# Pin sample doc links to the commit that built the backend image
+RUN sed -E "s#/(blob|tree)/master/#/\1/${COMMIT_SHA}/#g" -i /config/sample_config.json && \
+    sed -E "s/%252Fmaster/%252F${COMMIT_SHA}/#g" -i /config/sample_config.json
 
 # Expose apiserver port
 EXPOSE 8888

diff --git a/backend/Dockerfile.conformance b/backend/Dockerfile.conformance
@@ -19,6 +19,12 @@ RUN apk update && apk upgrade && \
     apk add --no-cache bash git openssh gcc musl-dev
 
 WORKDIR /go/src/github.com/kubeflow/pipelines
+
+COPY ./go.mod ./
+COPY ./go.sum ./
+
+RUN GO111MODULE=on go mod download
+
 COPY . .
 
 # Compile the test
@@ -34,7 +40,7 @@ RUN chmod +x /test/integration/run.sh
 RUN tar -czvf /test.tar.gz /test
 
 
-FROM alpine:3.8
+FROM alpine:3.9
 
 COPY --from=builder /test.tar.gz /
 RUN tar -xzvf /test.tar.gz

diff --git a/backend/Dockerfile.driver b/backend/Dockerfile.driver
@@ -1,4 +1,4 @@
-# Copyright 2021-2024 The Kubeflow Authors
+# Copyright 2023 The Kubeflow Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,37 +12,38 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Build arguments
-ARG SOURCE_CODE=.
+FROM golang:1.21.7-alpine3.19 as builder
 
-# Use ubi8/nodejs-14 as base image
-FROM registry.access.redhat.com/ubi8/go-toolset:1.21 as builder
+WORKDIR /go/src/github.com/kubeflow/pipelines
 
-
-## Build args to be used at this step
-ARG SOURCE_CODE
-
-## Switch to root as required for some operations
-USER root
-
-COPY ${SOURCE_CODE}/go.mod ./
-COPY ${SOURCE_CODE}/go.sum ./
+COPY ./go.mod ./
+COPY ./go.sum ./
+COPY ./hack/install-go-licenses.sh ./hack/
 
 RUN GO111MODULE=on go mod download
+RUN ./hack/install-go-licenses.sh
 
-# Copy the source
-COPY ${SOURCE_CODE}/ ./
+COPY . .
 
 RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags netgo -ldflags '-extldflags "-static"' -o /bin/driver ./backend/src/v2/cmd/driver/*.go
 
-FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
+# Check licenses and comply with license terms.
+# First, make sure there's no forbidden license.
+RUN go-licenses check ./backend/src/v2/cmd/driver
+RUN go-licenses csv ./backend/src/v2/cmd/driver > /tmp/licenses.csv && \
+    diff /tmp/licenses.csv backend/third_party_licenses/driver.csv && \
+    go-licenses save ./backend/src/v2/cmd/driver --save_path /tmp/NOTICES
+
+FROM alpine:3.19
+
+RUN adduser -S appuser
+USER appuser
 
 WORKDIR /bin
 
 COPY --from=builder /bin/driver /bin/driver
-RUN chmod +x /bin/driver
-
-ENTRYPOINT ["/bin/driver"]
+# Copy licenses and notices.
+COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv
+COPY --from=builder /tmp/NOTICES /third_party/NOTICES
 
-LABEL name="ds-pipelines-driver" \
-      summary="DSP Driver"
+ENTRYPOINT [ "/bin/driver" ]
diff --git a/backend/Dockerfile.launcher b/backend/Dockerfile.launcher
@@ -1,4 +1,4 @@
-# Copyright 2021-2024 The Kubeflow Authors
+# Copyright 2023 The Kubeflow Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,39 +12,38 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Build arguments
-ARG SOURCE_CODE=.
-ARG CI_CONTAINER_VERSION="unknown"
+FROM golang:1.21.7-alpine3.19 as builder
 
+WORKDIR /go/src/github.com/kubeflow/pipelines
 
-# Use ubi8/nodejs-14 as base image
-FROM registry.access.redhat.com/ubi8/go-toolset:1.21 as builder
-
-
-## Build args to be used at this step
-ARG SOURCE_CODE
-
-## Switch to root as required for some operations
-USER root
-
-COPY ${SOURCE_CODE}/go.mod ./
-COPY ${SOURCE_CODE}/go.sum ./
+COPY ./go.mod ./
+COPY ./go.sum ./
+COPY ./hack/install-go-licenses.sh ./hack/
 
 RUN GO111MODULE=on go mod download
+RUN ./hack/install-go-licenses.sh
 
-# Copy the source
-COPY ${SOURCE_CODE}/ ./
+COPY . .
 
 RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags netgo -ldflags '-extldflags "-static"' -o /bin/launcher-v2 ./backend/src/v2/cmd/launcher-v2/*.go
 
-FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
+# Check licenses and comply with license terms.
+# First, make sure there's no forbidden license.
+RUN go-licenses check ./backend/src/v2/cmd/launcher-v2
+RUN go-licenses csv ./backend/src/v2/cmd/launcher-v2 > /tmp/licenses.csv && \
+    diff /tmp/licenses.csv backend/third_party_licenses/launcher.csv && \
+    go-licenses save ./backend/src/v2/cmd/launcher-v2 --save_path /tmp/NOTICES
+
+FROM alpine:3.19
+
+RUN adduser -S appuser
+USER appuser
 
 WORKDIR /bin
 
 COPY --from=builder /bin/launcher-v2 /bin/launcher-v2
-RUN chmod +x /bin/launcher-v2
-
-ENTRYPOINT ["/bin/launcher-v2"]
+# Copy licenses and notices.
+COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv
+COPY --from=builder /tmp/NOTICES /third_party/NOTICES
 
-LABEL name="ds-pipelines-launcher" \
-      summary="DSP launcher"
+ENTRYPOINT [ "/bin/launcher-v2" ]
diff --git a/backend/Dockerfile.persistenceagent b/backend/Dockerfile.persistenceagent
@@ -12,35 +12,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Build arguments
-ARG SOURCE_CODE=.
-ARG CI_CONTAINER_VERSION="unknown"
+FROM golang:1.21.7-alpine3.19 as builder
 
+WORKDIR /go/src/github.com/kubeflow/pipelines
 
-# Use ubi8/go-toolset as base image
-FROM registry.access.redhat.com/ubi8/go-toolset:1.21 as builder
+COPY ./go.mod ./
+COPY ./go.sum ./
+COPY ./hack/install-go-licenses.sh ./hack/
 
-## Build args to be used at this step
-ARG SOURCE_CODE
-
-USER root
+RUN GO111MODULE=on go mod download
+RUN ./hack/install-go-licenses.sh
 
-RUN dnf install -y bash git openssh gcc
+COPY . .
 
-COPY ${SOURCE_CODE}/go.mod ./
-COPY ${SOURCE_CODE}/go.sum ./
+# Needed musl-dev for github.com/mattn/go-sqlite3
+RUN apk update && apk upgrade && \
+    apk add --no-cache bash git openssh gcc musl-dev
 
-RUN GO111MODULE=on go mod download
+RUN GO111MODULE=on go build -o /bin/persistence_agent backend/src/agent/persistence/*.go
+# Check licenses and comply with license terms.
+# First, make sure there's no forbidden license.
+RUN go-licenses check ./backend/src/agent/persistence
+RUN go-licenses csv ./backend/src/agent/persistence > /tmp/licenses.csv && \
+    diff /tmp/licenses.csv backend/third_party_licenses/persistence_agent.csv && \
+    go-licenses save ./backend/src/agent/persistence --save_path /tmp/NOTICES
 
-# Copy the source
-COPY ${SOURCE_CODE}/ ./
+FROM alpine:3.19
 
-RUN GO111MODULE=on go build -o /bin/persistence_agent backend/src/agent/persistence/*.go
+RUN adduser -S appuser
+USER appuser
 
-FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
 WORKDIR /bin
 
 COPY --from=builder /bin/persistence_agent /bin/persistence_agent
+# Copy licenses and notices.
+COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv
+COPY --from=builder /tmp/NOTICES /third_party/NOTICES
 
 ENV NAMESPACE ""
 
@@ -53,4 +60,4 @@ ENV LOG_LEVEL info
 
 ENV EXECUTIONTYPE Workflow
 
-CMD persistence_agent --logtostderr=true --namespace=${NAMESPACE} --ttlSecondsAfterWorkflowFinish=${TTL_SECONDS_AFTER_WORKFLOW_FINISH} --numWorker ${NUM_WORKERS} --executionType ${EXECUTIONTYPE} --logLevel=${LOG_LEVEL}
+CMD persistence_agent --logtostderr=true --namespace=${NAMESPACE} --ttlSecondsAfterWorkflowFinish=${TTL_SECONDS_AFTER_WORKFLOW_FINISH} --numWorker ${NUM_WORKERS} --executionType ${EXECUTIONTYPE} --logLevel=${LOG_LEVEL}