Merge pull request #1876 from swahtz/feature/fvdb

fVDB code format updates

fvdb/Dockerfile

@@ -1,52 +1,17 @@
-ARG CUDA_VERSION=12.1.1
-ARG CUDNN_VERSION=8
+FROM nvcr.io/nvidia/pytorch:24.04-py3
 
-FROM nvidia/cuda:${CUDA_VERSION}-cudnn${CUDNN_VERSION}-devel-ubuntu20.04
-
-ENV PATH /usr/local/cuda/bin:$PATH
-ENV LD_LIBRARY_PATH /usr/lib64:/usr/local/cuda/lib64:/usr/local/cuda/lib:${LD_LIBRARY_PATH}
-
-# # nvidia-container-runtime
-ENV NVIDIA_VISIBLE_DEVICES all
-ENV NVIDIA_DRIVER_CAPABILITIES compute,utility,graphics
-
-RUN echo "Acquire { https::Verify-Peer false }" > /etc/apt/apt.conf.d/99verify-peer.conf \
-    && if [ -f /etc/apt/sources.list.d/cuda.list ]; then \
-        rm /etc/apt/sources.list.d/cuda.list; \
-    fi \
-    && if [ -f /etc/apt/sources.list.d/nvidia-ml.list ]; then \
-        rm /etc/apt/sources.list.d/nvidia-ml.list; \
-    fi \
-    && apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated ca-certificates \
-    && rm /etc/apt/apt.conf.d/99verify-peer.conf \
-    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-		wget \
-		rsync \
-		vim \
-		git \
-    	curl \
-		ninja-build \
-		cmake \
-		build-essential \
-		xauth \
-		openssh-server \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN curl -o ~/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
-    bash ~/miniconda.sh -b -p /opt/conda && \
-    rm ~/miniconda.sh
-
-ENV PATH /opt/conda/bin:$PATH
-ENV TORCH_CUDA_ARCH_LIST "6.1;7.0;7.5;8.0;8.6+PTX"
+ARG MODE=production
+RUN echo "Building fVDB container in $MODE mode"
 
 # used for cross-compilation in docker build
 ENV FORCE_CUDA=1
 
 WORKDIR /fvdb
-COPY env/test_environment.yml .
-
-RUN /opt/conda/bin/conda env create -f test_environment.yml \
-    && /opt/conda/bin/conda clean -ya \
-    && /opt/conda/bin/conda init bash
+COPY . .
+RUN  pip install --no-cache-dir -r env/build_requirements.txt
+
+RUN if [ "$MODE" = "production" ]; then \
+     MAX_JOBS=$(free -g | awk '/^Mem:/{jobs=int($4/2.5); if(jobs<1) jobs=1; print jobs}')  \
+     TORCH_CUDA_ARCH_LIST="6.1;7.0;7.5;8.0;8.6+PTX" \
+     python setup.py install; \
+    fi

fvdb/README.md

@@ -46,7 +46,62 @@ conda activate fvdb_learn
 
 
 ## Building *f*VDB from Source
-*f*VDB is a Python library implemented as a C++ Pytorch extension.
+
+### Environment Management
+ƒVDB is a Python library implemented as a C++ Pytorch extension.  Of course you can build ƒVDB in whatever environment suits you, but we provide two paths to constructing reliable environments for building and running ƒVDB:  using [docker](#setting-up-a-docker-container) and using [conda](#setting-up-a-conda-environment).
+
+`conda` tends to be more flexible since reconfiguring toolchains and modules to suit your larger project can be dynamic, but at the same time this can be a more brittle experience compared to using a virtualized `docker` container.  Using `conda` is generally recommended for development and testing, while using `docker` is recommended for CI/CD and deployment.
+
+#### Setting up a Docker Container
+
+Running a docker container is a great way to ensure that you have a consistent environment for building and running ƒVDB.
+
+Our provided [`Dockerfile`](Dockerfile) has two modes for building the image: `dev` and `production`.  `production` constructs an image capable of building ƒVDB, builds and installs the ƒVDB libraries and is read for you to start running python code that uses the `fvdb` module.  `dev` mode constructs an image which is ready to build ƒVDB but does not build the ƒVDB libraries.
+
+Building the docker image in `production` mode is the default and is as simple as running the following command from the root of this repository:
+```shell
+# Build the docker image in production mode
+docker build -t fvdb/prod .
+```
+
+Building the docker mage in `dev` mode is done by setting the `BUILD_MODE` argument to `dev`:
+```shell
+# Build the docker image in dev mode
+docker build --build-arg  MODE=dev -t fvdb/dev .
+```
+
+Running the docker container is done with the following command:
+```shell
+# Run an interactive bash shell (or replace with your command)
+docker run -it --gpus all --rm \
+  fvdb/dev:latest \
+  /bin/bash
+```
+
+
+#### Setting up a Conda Environment
+
+In order to get resolved package versions in your conda environment consistent with our testing, it is necessary to configure your `.condarc` since not all package resolving behaviour can be controlled with an `environment.yml` file.  We recommend using `strict` channel priority in your conda configuration.  This can be done by running the following command:
+
+```shell
+conda config --set channel_priority strict
+```
+
+Further, it is recommend to not mix the `defaults` and `conda-forge` package channels when resolving environments.  We have generally used `conda-forge` as the primary channel for our dependencies.  You can remove the `defaults` channel and add `conda-forge` with the following command:
+
+```shell
+conda config --remove channels defaults
+conda config --add channels conda-forge
+```
+
+With these changes, it is recommended that your `.condarc` file looks like the following:
+
+```yaml
+channel_priority: strict
+channels:
+  - conda-forge
+```
+
 
 **(Optional) Install libMamba for a huge quality of life improvement when using Conda**
 ```
@@ -55,7 +110,6 @@ conda install -n base conda-libmamba-solver
 conda config --set solver libmamba
 ```
 
-### Conda Environment
 
 Next, create the `fvdb` conda environment by running the following command from the root of this repository, and then grabbing a ☕:
 ```shell
@@ -106,22 +160,6 @@ sphinx-build -E -a docs/ build/sphinx
 open build/sphinx/index.html
 ```
 
-### Docker Image
-
-To build and test *f*VDB, we have the dockerfile available:
-```shell
-# Build fvdb
-docker build . -t fvdb-dev
-# Run fvdb (or replace with your command)
-docker run -it --gpus all --rm \
-  --user $(id -u):$(id -g) \
-  --mount type=bind,source="$HOME/.ssh",target=/root/.ssh \
-  --mount type=bind,source="$(pwd)",target=/fvdb \
-  fvdb-dev:latest \
-  conda run -n fvdb_test --no-capture-output python setup.py develop
-```
-
-
 
 
 ## Usage Examples

fvdb/docs/conf.py

@@ -9,27 +9,23 @@
 
 import os
 import sys
-sys.path.insert(0, os.path.abspath('..'))
+
+sys.path.insert(0, os.path.abspath(".."))
 
 
 # -- Project information -----------------------------------------------------
 
-project = 'fVDB'
-copyright = '2023, NVIDIA Corporation'
-author = 'NVIDIA Corporation'
+project = "fVDB"
+copyright = "2023, NVIDIA Corporation"
+author = "NVIDIA Corporation"
 
 
 # -- General configuration ---------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.viewcode',
-    'sphinx.ext.napoleon',
-    'myst_parser'
-]
+extensions = ["sphinx.ext.autodoc", "sphinx.ext.viewcode", "sphinx.ext.napoleon", "myst_parser"]
 
 myst_enable_extensions = [
     "amsmath",
@@ -49,28 +45,26 @@
 ]
 
 # Fix return-type in google-style docstrings
-napoleon_custom_sections = [('Returns', 'params_style')]
+napoleon_custom_sections = [("Returns", "params_style")]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
-source_suffix = ['.rst', '.md']
+source_suffix = [".rst", ".md"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
-autodoc_default_options = {
-    'undoc-members': 'forward, extra_repr'
-}
+autodoc_default_options = {"undoc-members": "forward, extra_repr"}
 
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
@@ -80,6 +74,7 @@
 
 # -- Custom hooks ------------------------------------------------------------
 
+
 def process_signature(app, what, name, obj, options, signature, return_annotation):
     if signature is not None:
         signature = signature.replace("._Cpp", "")
@@ -91,5 +86,6 @@ def process_signature(app, what, name, obj, options, signature, return_annotatio
 
     return signature, return_annotation
 
+
 def setup(app):
     app.connect("autodoc-process-signature", process_signature)

fvdb/env/build_environment.yml

@@ -1,7 +1,8 @@
 name: fvdb_build
 channels:
-  - nvidia/label/cuda-12.1.0
   - pytorch
+  - nvidia
+  - conda-forge
 dependencies:
   - python=3.10
   - pytorch::pytorch=2.2
@@ -11,14 +12,14 @@ dependencies:
   - ca-certificates
   - certifi
   - openssl
-  - nvidia/label/cuda-12.1.0::cuda
-  - nvidia/label/cuda-12.1.0::cuda-tools
-  - nvidia/label/cuda-12.1.0::cuda-nvcc
-  - nvidia/label/cuda-12.1.0::cuda-cccl
-  - nvidia/label/cuda-12.1.0::cuda-libraries-static
+  - cuda-toolkit=12.1
+  - cuda-compiler=12.1
+  - cuda-nvcc=12.1
+  - cuda-cccl=12.1
+  - cuda-libraries-static=12.1
   - gcc_linux-64=11
   - gxx_linux-64=11
-  - setuptools
+  - setuptools>=68.2.2
   - cmake
   - make
   - ninja

fvdb/env/learn_environment.yml

@@ -28,6 +28,7 @@ dependencies:
   - pytest-benchmark
   - polyscope
   - numpy<2
+  - pyrender
   - pip:
     - point-cloud-utils
     - linkify-it-py

fvdb/env/test_environment.yml

@@ -1,28 +1,29 @@
 name: fvdb_test
 channels:
   - pyg
-  - nvidia/label/cuda-12.1.0
   - pytorch
+  - nvidia
+  - conda-forge
 dependencies:
   - python=3.10
   - pytorch::pytorch=2.2
   - pytorch::pytorch-cuda=12.1
   - tensorboard
-  - pip
+  - pip>=23.3.1
   - git
   - gitpython
   - ca-certificates
   - certifi
   - openssl
-  - nvidia/label/cuda-12.1.0::cuda
-  - nvidia/label/cuda-12.1.0::cuda-tools
-  - nvidia/label/cuda-12.1.0::cuda-nvcc
-  - nvidia/label/cuda-12.1.0::cuda-cccl
-  - nvidia/label/cuda-12.1.0::cuda-libraries-static
+  - cuda-toolkit=12.1
+  - cuda-compiler=12.1
+  - cuda-nvcc=12.1
+  - cuda-cccl=12.1
+  - cuda-libraries-static=12.1
   - parameterized
   - gcc_linux-64=11
   - gxx_linux-64=11
-  - setuptools
+  - setuptools>=68.2.2
   - cmake
   - make
   - ninja