From d51ed4ae1a778735c0253dcbfb5794a7df2c624b Mon Sep 17 00:00:00 2001 From: Oliver Koenig Date: Fri, 22 Nov 2024 09:58:47 +0100 Subject: [PATCH 1/2] ci: Add `build-test-publish-wheel` workflow Signed-off-by: Oliver Koenig --- .../workflows/build-test-publish-wheel.yml | 49 +++++++++++++++++++ Dockerfile | 31 ++++++++++++ src/nemo_run/__init__.py | 23 ++++++--- src/nemo_run/package_info.py | 37 ++++++++++++++ 4 files changed, 134 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/build-test-publish-wheel.yml create mode 100644 Dockerfile create mode 100644 src/nemo_run/package_info.py diff --git a/.github/workflows/build-test-publish-wheel.yml b/.github/workflows/build-test-publish-wheel.yml new file mode 100644 index 0000000..d109b74 --- /dev/null +++ b/.github/workflows/build-test-publish-wheel.yml @@ -0,0 +1,49 @@ +# Copyright (c) 2020-2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Build, test, and publish a PyPi wheel (to testpypi) + +on: + push: + branches: + - main + - 'r**' + +defaults: + run: + shell: bash -x -e -u -o pipefail {0} + +jobs: + build-test-publish-wheel: + uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/_build_test_publish_wheel.yml@v0.12.2 + with: + image-name: nemo_container + dockerfile: Dockerfile + image-label: nemo-aligner + build-args: | + IMAGE_LABEL=nemo-aligner + MAX_JOBS=32 + ALIGNER_COMMIT=${{ github.event.pull_request.head.sha || github.sha }} + prune-filter-timerange: 24h + dry-run: true + python-package: nemo_aligner + container-workdir: /opt/NeMo-Aligner + environment: public + secrets: + TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} + SLACK_WEBHOOK_ADMIN: ${{ secrets.SLACK_WEBHOOK_ADMIN }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..486b45d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.12 as nemo-run-update +WORKDIR /opt +ARG NEMO_RUN_COMMIT +RUN <<"EOF" bash -exu +if [[ ! -d NeMo-Run ]]; then + git clone https://github.com/NVIDIA/NeMo-Run.git +fi +cd NeMo-Run +git init +git remote add origin https://github.com/NVIDIA/NeMo-Aligner.git +git fetch --all +git fetch origin '+refs/pull/*/merge:refs/remotes/pull/*/merge' +git checkout $CURATOR_COMMIT +EOF + +FROM python:3.12 + +RUN \ + --mount=type=bind,source=/opt/NeMo-Run/src/nemo_run/__init__.py,target=/opt/NeMo-Run/src/nemo_run/__init__.py,from=nemo-run-update \ + --mount=type=bind,source=/opt/NeMo-Run/src/nemo_run/package_info.py,target=/opt/NeMo-Run/src/nemo_run/package_info.py,from=nemo-run-update \ + --mount=type=bind,source=/opt/NeMo-Run/pyproject.toml,target=/opt/NeMo-Curator/pyproject.toml,from=nemo-run-update \ + cd /opt/NeMo-Run && \ + pip install . + +COPY --from=nemo-run-update /opt/NeMo-Run/ /opt/NeMo-Run/ + +# Clone the user's repository, find the relevant commit, and install everything we need +RUN bash -exu < Date: Fri, 22 Nov 2024 10:00:16 +0100 Subject: [PATCH 2/2] fixes Signed-off-by: Oliver Koenig --- .github/workflows/build-test-publish-wheel.yml | 17 +++++++---------- Dockerfile | 2 +- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-test-publish-wheel.yml b/.github/workflows/build-test-publish-wheel.yml index d109b74..cc70317 100644 --- a/.github/workflows/build-test-publish-wheel.yml +++ b/.github/workflows/build-test-publish-wheel.yml @@ -14,7 +14,7 @@ name: Build, test, and publish a PyPi wheel (to testpypi) -on: +on: push: branches: - main @@ -28,22 +28,19 @@ jobs: build-test-publish-wheel: uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/_build_test_publish_wheel.yml@v0.12.2 with: - image-name: nemo_container + image-name: nemo_run_container dockerfile: Dockerfile - image-label: nemo-aligner + image-label: nemo-run build-args: | - IMAGE_LABEL=nemo-aligner - MAX_JOBS=32 - ALIGNER_COMMIT=${{ github.event.pull_request.head.sha || github.sha }} + IMAGE_LABEL=nemo-run + NEMO_RUN_COMMIT=${{ github.event.pull_request.head.sha || github.sha }} prune-filter-timerange: 24h dry-run: true - python-package: nemo_aligner - container-workdir: /opt/NeMo-Aligner + python-package: nemo_run/src + container-workdir: /opt/NeMo-Run environment: public secrets: TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} SLACK_WEBHOOK_ADMIN: ${{ secrets.SLACK_WEBHOOK_ADMIN }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - - \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 486b45d..f2ffd35 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ git init git remote add origin https://github.com/NVIDIA/NeMo-Aligner.git git fetch --all git fetch origin '+refs/pull/*/merge:refs/remotes/pull/*/merge' -git checkout $CURATOR_COMMIT +git checkout $NEMO_RUN_COMMIT EOF FROM python:3.12