Skip to content

Commit 3066dd2

Browse files
VaniHaripriyaVaniHaripriya
committed
chore: Move Data Science Pipelines Dockerfiles to a different path
Signed-off-by: VaniHaripriya <[email protected]>
1 parent 7c3b020 commit 3066dd2

16 files changed

+418
-156
lines changed

‎.github/workflows/build-images.yaml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -74,17 +74,17 @@ jobs:
7474
matrix:
7575
include:
7676
- image: ds-pipelines-api-server
77-
dockerfile: backend/Dockerfile
77+
dockerfile: odh_images/Dockerfile
7878
- image: ds-pipelines-frontend
7979
dockerfile: frontend/Dockerfile
8080
- image: ds-pipelines-persistenceagent
81-
dockerfile: backend/Dockerfile.persistenceagent
81+
dockerfile: odh_images/Dockerfile.persistenceagent
8282
- image: ds-pipelines-scheduledworkflow
83-
dockerfile: backend/Dockerfile.scheduledworkflow
83+
dockerfile: odh_images/Dockerfile.scheduledworkflow
8484
- image: ds-pipelines-driver
85-
dockerfile: backend/Dockerfile.driver
85+
dockerfile: odh_images/Dockerfile.driver
8686
- image: ds-pipelines-launcher
87-
dockerfile: backend/Dockerfile.launcher
87+
dockerfile: odh_images/Dockerfile.launcher
8888
steps:
8989
- uses: actions/checkout@v3
9090
- uses: ./.github/actions/build

‎.github/workflows/build-master.yml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,17 +20,17 @@ jobs:
2020
matrix:
2121
include:
2222
- image: ds-pipelines-api-server
23-
dockerfile: backend/Dockerfile
23+
dockerfile: odh_images/Dockerfile
2424
- image: ds-pipelines-frontend
2525
dockerfile: frontend/Dockerfile
2626
- image: ds-pipelines-persistenceagent
27-
dockerfile: backend/Dockerfile.persistenceagent
27+
dockerfile: odh_images/Dockerfile.persistenceagent
2828
- image: ds-pipelines-scheduledworkflow
29-
dockerfile: backend/Dockerfile.scheduledworkflow
29+
dockerfile: odh_images/Dockerfile.scheduledworkflow
3030
- image: ds-pipelines-driver
31-
dockerfile: backend/Dockerfile.driver
31+
dockerfile: odh_images/Dockerfile.driver
3232
- image: ds-pipelines-launcher
33-
dockerfile: backend/Dockerfile.launcher
33+
dockerfile: odh_images/Dockerfile.launcher
3434
steps:
3535
- uses: actions/checkout@v3
3636
- name: Generate Tag

‎.github/workflows/build-prs.yml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,17 +69,17 @@ jobs:
6969
matrix:
7070
include:
7171
- image: ds-pipelines-api-server
72-
dockerfile: backend/Dockerfile
72+
dockerfile: odh_images/Dockerfile
7373
- image: ds-pipelines-frontend
7474
dockerfile: frontend/Dockerfile
7575
- image: ds-pipelines-persistenceagent
76-
dockerfile: backend/Dockerfile.persistenceagent
76+
dockerfile: odh_images/Dockerfile.persistenceagent
7777
- image: ds-pipelines-scheduledworkflow
78-
dockerfile: backend/Dockerfile.scheduledworkflow
78+
dockerfile: odh_images/Dockerfile.scheduledworkflow
7979
- image: ds-pipelines-driver
80-
dockerfile: backend/Dockerfile.driver
80+
dockerfile: odh_images/Dockerfile.driver
8181
- image: ds-pipelines-launcher
82-
dockerfile: backend/Dockerfile.launcher
82+
dockerfile: odh_images/Dockerfile.launcher
8383
steps:
8484
- uses: actions/checkout@v3
8585
- name: Build Image

‎backend/Dockerfile

Lines changed: 47 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2021-2024 The Kubeflow Authors
1+
# Copyright 2021-2022 The Kubeflow Authors
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -12,47 +12,72 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
# Build arguments
16-
ARG SOURCE_CODE=.
15+
# 1. Build api server application
16+
FROM golang:1.21.7-bookworm as builder
17+
RUN apt-get update && apt-get install -y cmake clang musl-dev openssl
18+
WORKDIR /go/src/github.com/kubeflow/pipelines
19+
COPY . .
20+
RUN GO111MODULE=on go build -o /bin/apiserver backend/src/apiserver/*.go
21+
# Check licenses and comply with license terms.
22+
RUN ./hack/install-go-licenses.sh
23+
# First, make sure there's no forbidden license.
24+
RUN go-licenses check ./backend/src/apiserver
25+
RUN go-licenses csv ./backend/src/apiserver > /tmp/licenses.csv && \
26+
diff /tmp/licenses.csv backend/third_party_licenses/apiserver.csv && \
27+
go-licenses save ./backend/src/apiserver --save_path /tmp/NOTICES
28+
29+
# 2. Compile preloaded pipeline samples
30+
FROM python:3.8 as compiler
31+
RUN apt-get update -y && apt-get install --no-install-recommends -y -q default-jdk python3-setuptools python3-dev jq
32+
RUN wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py
33+
COPY backend/requirements.txt .
34+
RUN python3 -m pip install -r requirements.txt --no-cache-dir
1735

18-
FROM registry.access.redhat.com/ubi8/go-toolset:1.21 as builder
19-
20-
USER root
2136
# Downloading Argo CLI so that the samples are validated
2237
ENV ARGO_VERSION v3.4.17
2338
RUN curl -sLO https://github.com/argoproj/argo-workflows/releases/download/${ARGO_VERSION}/argo-linux-amd64.gz && \
2439
gunzip argo-linux-amd64.gz && \
2540
chmod +x argo-linux-amd64 && \
2641
mv ./argo-linux-amd64 /usr/local/bin/argo
2742

28-
RUN dnf install -y cmake clang openssl
29-
30-
COPY ${SOURCE_CODE}/go.mod ./
31-
COPY ${SOURCE_CODE}/go.sum ./
43+
WORKDIR /
44+
COPY ./samples /samples
45+
COPY backend/src/apiserver/config/sample_config.json /samples/
3246

33-
RUN GO111MODULE=on go mod download
47+
# Compiling the preloaded samples.
48+
# The default image is replaced with the GCR-hosted python image.
49+
RUN set -e; \
50+
< /samples/sample_config.json jq .[].file --raw-output | while read pipeline_yaml; do \
51+
pipeline_py="${pipeline_yaml%.yaml}"; \
52+
python3 "$pipeline_py"; \
53+
done
3454

35-
# Copy the source
36-
COPY ${SOURCE_CODE}/ ./
55+
# 3. Start api web server
56+
FROM debian:stable
3757

38-
RUN GO111MODULE=on go build -o /bin/apiserver ./backend/src/apiserver/ && \
39-
dnf clean all
40-
41-
FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
58+
ARG COMMIT_SHA=unknown
59+
ENV COMMIT_SHA=${COMMIT_SHA}
60+
ARG TAG_NAME=unknown
61+
ENV TAG_NAME=${TAG_NAME}
62+
ENV LOG_LEVEL info
4263

4364
WORKDIR /bin
4465

45-
COPY --from=builder /opt/app-root/src/backend/src/apiserver/config/ /config
66+
COPY backend/src/apiserver/config/ /config
4667
COPY --from=builder /bin/apiserver /bin/apiserver
4768

69+
# Copy licenses and notices.
70+
COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv
71+
COPY --from=builder /tmp/NOTICES /third_party/NOTICES
72+
COPY --from=compiler /samples/ /samples/
4873
RUN chmod +x /bin/apiserver
4974

50-
USER root
51-
5275
# Adding CA certificate so API server can download pipeline through URL and wget is used for liveness/readiness probe command
53-
RUN microdnf install -y ca-certificates wget
76+
RUN apt-get update && apt-get install -y ca-certificates wget
5477

55-
USER 1001
78+
# Pin sample doc links to the commit that built the backend image
79+
RUN sed -E "s#/(blob|tree)/master/#/\1/${COMMIT_SHA}/#g" -i /config/sample_config.json && \
80+
sed -E "s/%252Fmaster/%252F${COMMIT_SHA}/#g" -i /config/sample_config.json
5681

5782
# Expose apiserver port
5883
EXPOSE 8888

‎backend/Dockerfile.cacheserver

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,12 @@ RUN apk update && apk upgrade && \
1919
apk add --no-cache bash git openssh gcc musl-dev
2020

2121
WORKDIR /go/src/github.com/kubeflow/pipelines
22-
23-
COPY ./go.mod ./
24-
COPY ./go.sum ./
25-
COPY ./hack/install-go-licenses.sh ./hack/
26-
27-
RUN GO111MODULE=on go mod download
28-
RUN ./hack/install-go-licenses.sh
29-
3022
COPY . .
3123

3224
RUN GO111MODULE=on go build -o /bin/cache_server backend/src/cache/*.go
3325

3426
# Check licenses and comply with license terms.
27+
RUN ./hack/install-go-licenses.sh
3528
# First, make sure there's no forbidden license.
3629
RUN go-licenses check ./backend/src/cache
3730
RUN go-licenses csv ./backend/src/cache > /tmp/licenses.csv && \

‎backend/Dockerfile.driver

Lines changed: 19 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2021-2024 The Kubeflow Authors
1+
# Copyright 2023 The Kubeflow Authors
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -12,37 +12,31 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
# Build arguments
16-
ARG SOURCE_CODE=.
15+
FROM golang:1.21.7-alpine3.19 as builder
1716

18-
# Use ubi8/nodejs-14 as base image
19-
FROM registry.access.redhat.com/ubi8/go-toolset:1.21 as builder
17+
WORKDIR /go/src/github.com/kubeflow/pipelines
18+
COPY . .
2019

20+
RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags netgo -ldflags '-extldflags "-static"' -o /bin/driver ./backend/src/v2/cmd/driver/*.go
2121

22-
## Build args to be used at this step
23-
ARG SOURCE_CODE
24-
25-
## Switch to root as required for some operations
26-
USER root
27-
28-
COPY ${SOURCE_CODE}/go.mod ./
29-
COPY ${SOURCE_CODE}/go.sum ./
30-
31-
RUN GO111MODULE=on go mod download
32-
33-
# Copy the source
34-
COPY ${SOURCE_CODE}/ ./
22+
# Check licenses and comply with license terms.
23+
RUN ./hack/install-go-licenses.sh
24+
# First, make sure there's no forbidden license.
25+
RUN go-licenses check ./backend/src/v2/cmd/driver
26+
RUN go-licenses csv ./backend/src/v2/cmd/driver > /tmp/licenses.csv && \
27+
diff /tmp/licenses.csv backend/third_party_licenses/driver.csv && \
28+
go-licenses save ./backend/src/v2/cmd/driver --save_path /tmp/NOTICES
3529

36-
RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags netgo -ldflags '-extldflags "-static"' -o /bin/driver ./backend/src/v2/cmd/driver/*.go
30+
FROM alpine:3.19
3731

38-
FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
32+
RUN adduser -S appuser
33+
USER appuser
3934

4035
WORKDIR /bin
4136

4237
COPY --from=builder /bin/driver /bin/driver
43-
RUN chmod +x /bin/driver
44-
45-
ENTRYPOINT ["/bin/driver"]
38+
# Copy licenses and notices.
39+
COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv
40+
COPY --from=builder /tmp/NOTICES /third_party/NOTICES
4641

47-
LABEL name="ds-pipelines-driver" \
48-
summary="DSP Driver"
42+
ENTRYPOINT [ "/bin/driver" ]

‎backend/Dockerfile.launcher

Lines changed: 19 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2021-2024 The Kubeflow Authors
1+
# Copyright 2023 The Kubeflow Authors
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -12,39 +12,31 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
# Build arguments
16-
ARG SOURCE_CODE=.
17-
ARG CI_CONTAINER_VERSION="unknown"
15+
FROM golang:1.21.7-alpine3.19 as builder
1816

17+
WORKDIR /go/src/github.com/kubeflow/pipelines
18+
COPY . .
1919

20-
# Use ubi8/nodejs-14 as base image
21-
FROM registry.access.redhat.com/ubi8/go-toolset:1.21 as builder
22-
23-
24-
## Build args to be used at this step
25-
ARG SOURCE_CODE
26-
27-
## Switch to root as required for some operations
28-
USER root
29-
30-
COPY ${SOURCE_CODE}/go.mod ./
31-
COPY ${SOURCE_CODE}/go.sum ./
32-
33-
RUN GO111MODULE=on go mod download
20+
RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags netgo -ldflags '-extldflags "-static"' -o /bin/launcher-v2 ./backend/src/v2/cmd/launcher-v2/*.go
3421

35-
# Copy the source
36-
COPY ${SOURCE_CODE}/ ./
22+
# Check licenses and comply with license terms.
23+
RUN ./hack/install-go-licenses.sh
24+
# First, make sure there's no forbidden license.
25+
RUN go-licenses check ./backend/src/v2/cmd/launcher-v2
26+
RUN go-licenses csv ./backend/src/v2/cmd/launcher-v2 > /tmp/licenses.csv && \
27+
diff /tmp/licenses.csv backend/third_party_licenses/launcher.csv && \
28+
go-licenses save ./backend/src/v2/cmd/launcher-v2 --save_path /tmp/NOTICES
3729

38-
RUN GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags netgo -ldflags '-extldflags "-static"' -o /bin/launcher-v2 ./backend/src/v2/cmd/launcher-v2/*.go
30+
FROM alpine:3.19
3931

40-
FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
32+
RUN adduser -S appuser
33+
USER appuser
4134

4235
WORKDIR /bin
4336

4437
COPY --from=builder /bin/launcher-v2 /bin/launcher-v2
45-
RUN chmod +x /bin/launcher-v2
46-
47-
ENTRYPOINT ["/bin/launcher-v2"]
38+
# Copy licenses and notices.
39+
COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv
40+
COPY --from=builder /tmp/NOTICES /third_party/NOTICES
4841

49-
LABEL name="ds-pipelines-launcher" \
50-
summary="DSP launcher"
42+
ENTRYPOINT [ "/bin/launcher-v2" ]

‎backend/Dockerfile.persistenceagent

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -12,35 +12,35 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
# Build arguments
16-
ARG SOURCE_CODE=.
17-
ARG CI_CONTAINER_VERSION="unknown"
15+
FROM golang:1.21.7-alpine3.19 as builder
1816

17+
WORKDIR /go/src/github.com/kubeflow/pipelines
18+
COPY . .
1919

20-
# Use ubi8/go-toolset as base image
21-
FROM registry.access.redhat.com/ubi8/go-toolset:1.21 as builder
20+
# Needed musl-dev for github.com/mattn/go-sqlite3
21+
RUN apk update && apk upgrade && \
22+
apk add --no-cache bash git openssh gcc musl-dev
2223

23-
## Build args to be used at this step
24-
ARG SOURCE_CODE
25-
26-
USER root
27-
28-
RUN dnf install -y bash git openssh gcc
29-
30-
COPY ${SOURCE_CODE}/go.mod ./
31-
COPY ${SOURCE_CODE}/go.sum ./
32-
33-
RUN GO111MODULE=on go mod download
24+
RUN GO111MODULE=on go build -o /bin/persistence_agent backend/src/agent/persistence/*.go
25+
# Check licenses and comply with license terms.
26+
RUN ./hack/install-go-licenses.sh
27+
# First, make sure there's no forbidden license.
28+
RUN go-licenses check ./backend/src/agent/persistence
29+
RUN go-licenses csv ./backend/src/agent/persistence > /tmp/licenses.csv && \
30+
diff /tmp/licenses.csv backend/third_party_licenses/persistence_agent.csv && \
31+
go-licenses save ./backend/src/agent/persistence --save_path /tmp/NOTICES
3432

35-
# Copy the source
36-
COPY ${SOURCE_CODE}/ ./
33+
FROM alpine:3.19
3734

38-
RUN GO111MODULE=on go build -o /bin/persistence_agent backend/src/agent/persistence/*.go
35+
RUN adduser -S appuser
36+
USER appuser
3937

40-
FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
4138
WORKDIR /bin
4239

4340
COPY --from=builder /bin/persistence_agent /bin/persistence_agent
41+
# Copy licenses and notices.
42+
COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv
43+
COPY --from=builder /tmp/NOTICES /third_party/NOTICES
4444

4545
ENV NAMESPACE ""
4646

0 commit comments

Comments
 (0)