ci: Add build-test-publish-wheel workflow

.github/workflows/build-test-publish-wheel.yml

@@ -0,0 +1,46 @@
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Build, test, and publish a PyPi wheel (to testpypi)
+
+on:
+  push:
+    branches:
+      - main
+      - 'r**'
+
+defaults:
+  run:
+    shell: bash -x -e -u -o pipefail {0}
+
+jobs:
+  build-test-publish-wheel:
+    uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/[email protected]
+    with:
+      image-name: nemo_run_container
+      dockerfile: Dockerfile
+      image-label: nemo-run
+      build-args: |
+        IMAGE_LABEL=nemo-run
+        NEMO_RUN_COMMIT=${{ github.event.pull_request.head.sha || github.sha }}
+      prune-filter-timerange: 24h
+      dry-run: true
+      python-package: nemo_run/src
+      container-workdir: /opt/NeMo-Run
+      environment: public
+    secrets:
+      TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+      TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+      SLACK_WEBHOOK_ADMIN: ${{ secrets.SLACK_WEBHOOK_ADMIN }}
+      SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}

Dockerfile

@@ -0,0 +1,31 @@
+FROM python:3.12 as nemo-run-update
+WORKDIR /opt
+ARG NEMO_RUN_COMMIT
+RUN <<"EOF" bash -exu
+if [[ ! -d NeMo-Run ]]; then
+    git clone https://github.com/NVIDIA/NeMo-Run.git
+fi
+cd NeMo-Run
+git init
+git remote add origin https://github.com/NVIDIA/NeMo-Aligner.git
+git fetch --all
+git fetch origin '+refs/pull/*/merge:refs/remotes/pull/*/merge'
+git checkout $NEMO_RUN_COMMIT
+EOF
+
+FROM python:3.12
+
+RUN \
+    --mount=type=bind,source=/opt/NeMo-Run/src/nemo_run/__init__.py,target=/opt/NeMo-Run/src/nemo_run/__init__.py,from=nemo-run-update \
+    --mount=type=bind,source=/opt/NeMo-Run/src/nemo_run/package_info.py,target=/opt/NeMo-Run/src/nemo_run/package_info.py,from=nemo-run-update \
+    --mount=type=bind,source=/opt/NeMo-Run/pyproject.toml,target=/opt/NeMo-Curator/pyproject.toml,from=nemo-run-update \
+    cd /opt/NeMo-Run && \
+    pip install .
+
+COPY --from=nemo-run-update /opt/NeMo-Run/ /opt/NeMo-Run/
+
+# Clone the user's repository, find the relevant commit, and install everything we need
+RUN bash -exu <<EOF
+  cd /opt/NeMo-Run/
+  pip install .
+EOF

src/nemo_run/__init__.py

@@ -16,7 +16,8 @@
 from nemo_run import cli
 from nemo_run.api import autoconvert, dryrun_fn
 from nemo_run.config import Config, ConfigurableMixin, Partial, Script
-from nemo_run.core.execution.base import Executor, ExecutorMacros, FaultTolerance, Torchrun
+from nemo_run.core.execution.base import (Executor, ExecutorMacros,
+                                          FaultTolerance, Torchrun)
 from nemo_run.core.execution.docker import DockerExecutor
 from nemo_run.core.execution.local import LocalExecutor
 from nemo_run.core.execution.skypilot import SkypilotExecutor
@@ -32,6 +33,11 @@
 from nemo_run.run.experiment import Experiment
 from nemo_run.run.plugin import ExperimentPlugin as Plugin
 
+from .package_info import (__contact_emails__, __contact_names__,
+                           __description__, __download_url__, __keywords__,
+                           __license__, __package_name__, __repository_url__,
+                           __shortversion__, __version__)
+
 __all__ = [
     "autoconvert",
     "cli",
@@ -61,9 +67,14 @@
     "SlurmExecutor",
     "SSHTunnel",
     "Torchrun",
+    "__contact_emails__",
+    "__contact_names__",
+    "__description__",
+    "__download_url__",
+    "__keywords__",
+    "__license__",
+    "__package_name__",
+    "__repository_url__",
+    "__shortversion__",
+    "__version__"
 ]
-
-try:
-    from nemo_run._version import __version__
-except Exception:
-    __version__ = "0.0.1"

src/nemo_run/package_info.py

@@ -0,0 +1,37 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+MAJOR = 0
+MINOR = 0
+PATCH = 1
+PRE_RELEASE = "rc0"
+DEV = "dev1"
+
+# Use the following formatting: (major, minor, patch, pre-release)
+VERSION = (MAJOR, MINOR, PATCH, PRE_RELEASE, DEV)
+
+__shortversion__ = ".".join(map(str, VERSION[:3]))
+__version__ = __shortversion__ + VERSION[3] + "." + ".".join(VERSION[4:])
+
+__package_name__ = "nemo_run"
+__contact_names__ = "NVIDIA"
+__contact_emails__ = "[email protected]"
+__repository_url__ = "https://github.com/NVIDIA/NeMo-Run"
+__download_url__ = "https://github.com/NVIDIA/NeMo-Run/releases"
+__description__ = (
+    "NeMo-Run - A powerful tool designed to streamline the configuration, execution and management of Machine Learning experiments across various computing environments."
+)
+__license__ = "Apache2"
+__keywords__ = "deep learning, machine learning, gpu, NLP, NeMo, nvidia, pytorch, torch, language, preprocessing, LLM, large language model"