Merge fVDB into OpenVDB

doc/nanovdb/SourceTree.md

@@ -5,12 +5,12 @@
 * [NanoVDB.h](https://github.com/AcademySoftwareFoundation/openvdb/blob/master/nanovdb/nanovdb/NanoVDB.h) C++11 implementation of the core data structure and its access methods.
 * [CNanoVDB.h](https://github.com/AcademySoftwareFoundation/openvdb/blob/master/nanovdb/nanovdb/CNanoVDB.h) Incomplete but useable C99 implementation of the core data structure and its access methods.  Designed in particular for use in OpenCL kernels.  Note that this relies on zero-sized arrays for the _reserved padding, so will not work on all compilers (with MSVC being a particular example)
 * [PNanoVDB.h](https://github.com/AcademySoftwareFoundation/openvdb/blob/master/nanovdb/nanovdb/PNanoVDB.h) C99 implementation of the core data structure and its access methods. More complete coverage than CNanoVDB.  This version is pointer-less and supports virtually all graphics APIs.
-* [util/GridHandle.h](https://github.com/AcademySoftwareFoundation/openvdb/blob/master/nanovdb/nanovdb/util/GridHandle.h) defines a handler for the memory allocated to a NanoVDB grid.
-* [util/IO.h](https://github.com/AcademySoftwareFoundation/openvdb/blob/master/nanovdb/nanovdb/util/IO.h) implements I/O support.
-* [util/OpenToNanoVDB.h](https://github.com/AcademySoftwareFoundation/openvdb/blob/master/nanovdb/nanovdb/util/OpenToNanoVDB.h) defines the converter from OpenVDB to NanoVDB and obviously depends on the OpenVDB library (as the only header file).
-* [Ray.h](https://github.com/AcademySoftwareFoundation/openvdb/blob/master/nanovdb/nanovdb/util/Ray.h) Ray class.
-* [HDDA.h](https://github.com/AcademySoftwareFoundation/openvdb/blob/master/nanovdb/nanovdb/util/HDDA.h) HDDA related.
-* [SampleFromVoxels.h](https://github.com/AcademySoftwareFoundation/openvdb/blob/master/nanovdb/nanovdb/util/SampleFromVoxels.h) interpolation.
+* [GridHandle.h](https://github.com/AcademySoftwareFoundation/openvdb/blob/master/nanovdb/nanovdb/GridHandle.h) defines a handler for the memory allocated to a NanoVDB grid.
+* [io/IO.h](https://github.com/AcademySoftwareFoundation/openvdb/blob/master/nanovdb/nanovdb/io/IO.h) implements I/O support.
+* [tools/CreateNanoGrid.h](https://github.com/AcademySoftwareFoundation/openvdb/blob/master/nanovdb/nanovdb/tools/CreateNanoGrid.h) defines the converter from OpenVDB to NanoVDB and obviously depends on the OpenVDB library (as the only header file).
+* [util/Ray.h](https://github.com/AcademySoftwareFoundation/openvdb/blob/master/nanovdb/nanovdb/util/Ray.h) Ray class.
+* [util/HDDA.h](https://github.com/AcademySoftwareFoundation/openvdb/blob/master/nanovdb/nanovdb/util/HDDA.h) HDDA related.
+* [util/SampleFromVoxels.h](https://github.com/AcademySoftwareFoundation/openvdb/blob/master/nanovdb/nanovdb/util/SampleFromVoxels.h) interpolation.
 
 ```bash
 foo@bar:~$ tree
@@ -22,25 +22,19 @@ foo@bar:~$ tree
 │   │   └── nanovdb_convert.cc
 │   ├── print
 │   │   └── nanovdb_print.cc
+│   ├── updateFiles.sh
 │   └── validate
 │       └── nanovdb_validate.cc
 ├── CNanoVDB.h
+├── cuda
+│   ├── DeviceBuffer.h
+│   ├── GridHandle.cuh
+│   └── NodeManager.cuh
 ├── docs
 │   ├── CMakeLists.txt
 │   ├── codingstyle.txt
 │   └── doxygen-config
 ├── examples
-│   ├── benchmark
-│   │   ├── BenchKernels_dense.cu
-│   │   ├── BenchKernels_nano.cu
-│   │   ├── Benchmark_dense.cu
-│   │   ├── Benchmark_nano.cu
-│   │   ├── Camera.h
-│   │   ├── CMakeLists.txt
-│   │   ├── DenseGrid.h
-│   │   ├── Image.h
-│   │   ├── TestBenchmark.cc
-│   │   └── TestBenchmark.cu
 │   ├── CMakeLists.txt
 │   ├── ex_bump_pool_buffer
 │   │   └── bump_pool_buffer.cc
@@ -50,7 +44,7 @@ foo@bar:~$ tree
 │   │   ├── nanovdb.cu
 │   │   └── openvdb.cc
 │   ├── ex_index_grid_cuda
-│   │   ├── index_grid_cuda.cu
+│   │   ├── index_grid_cuda.cc
 │   │   └── index_grid_cuda_kernel.cu
 │   ├── ex_make_custom_nanovdb
 │   │   └── make_custom_nanovdb.cc
@@ -66,6 +60,7 @@ foo@bar:~$ tree
 │   ├── ex_map_pool_buffer
 │   │   └── map_pool_buffer.cc
 │   ├── ex_modify_nanovdb_thrust
+│   │   ├── modify_nanovdb_thrust.cc
 │   │   └── modify_nanovdb_thrust.cu
 │   ├── ex_nodemanager_cuda
 │   │   ├── nodemanager_cuda.cc
@@ -103,9 +98,38 @@ foo@bar:~$ tree
 │   │   └── VoxToNanoVDB.h
 │   └── ex_write_nanovdb_grids
 │       └── write_nanovdb_grids.cc
+├── GridHandle.h
+├── HostBuffer.h
+├── io
+│   └── IO.h
+├── math
+│   ├── CSampleFromVoxels.h
+│   ├── DitherLUT.h
+│   ├── HDDA.h
+│   ├── Math.h
+│   ├── Ray.h
+│   ├── SampleFromVoxels.h
+│   └── Stencils.h
 ├── NanoVDB.h
+├── NodeManager.h
 ├── PNanoVDB.h
 ├── Readme.md
+├── tools
+│   ├── CreateNanoGrid.h
+│   ├── CreatePrimitives.h
+│   ├── cuda
+│   │   ├── AddBlindData.cuh
+│   │   ├── GridChecksum.cuh
+│   │   ├── GridStats.cuh
+│   │   ├── GridValidator.cuh
+│   │   ├── IndexToGrid.cuh
+│   │   ├── PointsToGrid.cuh
+│   │   └── SignedFloodFill.cuh
+│   ├── GridBuilder.h
+│   ├── GridChecksum.h
+│   ├── GridStats.h
+│   ├── GridValidator.h
+│   └── NanoToOpenVDB.h
 ├── unittest
 │   ├── CMakeLists.txt
 │   ├── pnanovdb_validate_strides.h
@@ -115,21 +139,25 @@ foo@bar:~$ tree
 └── util
     ├── CpuTimer.h
     ├── CreateNanoGrid.h
-    ├── CSampleFromVoxels.h
     ├── cuda
     │   ├── CudaAddBlindData.cuh
     │   ├── CudaDeviceBuffer.h
+    │   ├── CudaGridChecksum.cuh
     │   ├── CudaGridHandle.cuh
+    │   ├── CudaGridStats.cuh
+    │   ├── CudaGridValidator.cuh
     │   ├── CudaIndexToGrid.cuh
+    │   ├── CudaNodeManager.cuh
     │   ├── CudaPointsToGrid.cuh
     │   ├── CudaSignedFloodFill.cuh
     │   ├── CudaUtils.h
-    │   └── GpuTimer.cuh
+    │   ├── GpuTimer.h
+    │   ├── Timer.h
+    │   └── Util.h
     ├── DitherLUT.h
     ├── ForEach.h
     ├── GridBuilder.h
     ├── GridChecksum.h
-    ├── GridHandle.h
     ├── GridStats.h
     ├── GridValidator.h
     ├── HDDA.h
@@ -145,5 +173,6 @@ foo@bar:~$ tree
     ├── Ray.h
     ├── Reduce.h
     ├── SampleFromVoxels.h
-    └── Stencils.h
-```
+    ├── Stencils.h
+    ├── Timer.h
+    └── Util.h

fvdb/.github/workflows/benchmarks.yml

@@ -0,0 +1,57 @@
+name: Running Benchmarks
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+    contents: write
+    deployments: write
+
+jobs:
+  benchmarks:
+    name: Run fVDB continuous benchmarking
+    runs-on:
+      - self-hosted
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up fvdb_test Conda env
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniconda-version: "latest"
+          mamba-version: "*"
+          channels: conda-forge,defaults
+          channel-priority: true
+          activate-environment: fvdb_test
+          environment-file: env/test_environment.yml
+
+
+      - name: Run benchmarks
+        run: |
+          conda activate fvdb_test
+          MAX_JOBS=$(($(nproc) < $(free -g | awk '/^Mem:/{jobs=int($4/2.5); if(jobs<1) jobs=1; print jobs}') ? $(nproc) : $(free -g | awk '/^Mem:/{jobs=int($4/2.5); if(jobs<1) jobs=1; print jobs}'))) python setup.py develop
+          pytest tests/benchmark --benchmark-json tests/benchmark/output.json
+        shell:
+          bash -el {0}
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+            name: Python Benchmark with pytest-benchmark
+            tool: 'pytest'
+            output-file-path: tests/benchmark/output.json
+            # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096
+            github-token: ${{ secrets.GITHUB_TOKEN }}
+            auto-push: true
+            # Show alert with commit comment on detecting possible performance regression
+            alert-threshold: '200%'
+            comment-on-alert: true
+            fail-on-alert: true
+            alert-comment-cc-users: '@swahtz'
+
+      - name: Clean Conda
+        run: |
+          conda clean -pty
+        shell:
+          bash -el {0}

fvdb/.github/workflows/building.yml

@@ -0,0 +1,125 @@
+name: Building Release Wheels
+
+on:
+  release:
+    types: [published, edited]
+  workflow_dispatch:
+
+jobs:
+
+  wheel:
+    runs-on:
+      - self-hosted
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # support version based on: https://download.pytorch.org/whl/torch/
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        torch-version: [2.1.0, 2.1.1, 2.1.2, 2.2.0, 2.2.1, 2.2.2, 2.3.0]
+        cuda-version: ['12.1.1']
+        exclude:
+          - torch-version: 2.1.2
+            python-version: '3.12'
+          - torch-version: 2.1.1
+            python-version: '3.12'
+          - torch-version: 2.1.0
+            python-version: '3.12'
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Generate release_environment.yml
+        run: |
+          echo "cp env/release_base_environment.yml release_environment.yml"
+          cp env/release_base_environment.yml release_environment.yml
+          echo "  - python=${{ matrix.python-version }}" >> release_environment.yml
+          echo "  - pytorch=${{ matrix.torch-version }}" >> release_environment.yml
+          perl -E '"${{ matrix.cuda-version }}" =~ /^(\d+\.\d+)/; say "  - pytorch-cuda=$1"' >> release_environment.yml
+          perl -E '"${{ matrix.cuda-version }}" =~ /^(\d+)\.(\d+)/; say "  - cuda>=$1.$2.0, <$1." . ($2+1) . ".0"' >> release_environment.yml
+          perl -E '"${{ matrix.cuda-version }}" =~ /^(\d+)\.(\d+)/; say "  - cuda-nvcc>=$1.$2.0, <$1." . ($2+1) . ".0"' >> release_environment.yml
+
+      - name: Set up fvdb_release Conda env
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniconda-version: "latest"
+          mamba-version: "*"
+          channels: nvidia,pytorch,conda-forge,defaults
+          channel-priority: true
+          activate-environment: fvdb_release
+          environment-file: release_environment.yml
+
+      - name: Set version
+        run: |
+          VERSION=`sed -n "s/^__version__ = '\(.*\)'/\1/p" fvdb/__init__.py`
+          TORCH_VERSION=`echo "pt${{ matrix.torch-version }}" | sed "s/..$//" | sed "s/\.//g"`
+          CUDA_VERSION=`perl -E '"${{ matrix.cuda-version }}" =~ /^(\d+)\.(\d+)/; say "cu$1$2"'`
+          echo "New version name: $VERSION+$TORCH_VERSION$CUDA_VERSION"
+          sed -i "s/$VERSION/$VERSION+$TORCH_VERSION$CUDA_VERSION/" fvdb/__init__.py
+        shell:
+          bash -el {0}
+
+      - name: Build wheel
+        run: |
+          conda activate fvdb_release
+          mkdir dist
+          TORCH_CUDA_ARCH_LIST="6.1;7.0;7.5;8.0;8.6+PTX" python setup.py bdist_wheel --dist-dir=dist
+        shell:
+          bash -el {0}
+
+      - name: Rename wheel
+        run: |
+          python scripts/rename_wheels.py ${{ matrix.torch-version }} `perl -E '"${{ matrix.cuda-version }}" =~ /^(\d+)\.(\d+)/; say "cu$1$2"'`
+        shell:
+          bash -el {0}
+
+      # - name: Push wheel to fvdb-wheels
+      #   id: push_directory
+      #   uses: swahtz/github-action-push-to-another-repository@main
+      #   env:
+      #     SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_WHL_KEY }}
+      #   with:
+      #     source-directory: 'dist'
+      #     destination-github-username: 'voxel-foundation'
+      #     destination-repository-name: 'fvdb-wheels'
+      #     user-email: [email protected]
+      #     target-branch: main
+      #     target-directory: 'main'
+
+      # - name: Upload wheels to URM
+      #   run: |
+      #     pip install twine
+      #     twine upload -u jswartz -p ${{ secrets.URM_TOKEN }} --repository-url https://urm.nvidia.com/artifactory/api/pypi/nv-shared-pypi-local dist/*whl  --verbose
+      #   shell:
+      #     bash
+
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          token: ${{ secrets.RELEASE_ACTION_TOKEN }}
+          files: |
+            dist/*.whl
+
+      # - name: Upload to artifacts
+      #   uses: actions/upload-artifact@v3
+      #   with:
+      #     name: wheels
+      #     path: |
+      #       dist/*.whl
+
+      # - name: Configure AWS
+      #   uses: aws-actions/configure-aws-credentials@v1
+      #   with:
+      #     aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      #     aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      #     aws-region: ap-northeast-1
+
+      # - name: Upload wheel
+      #   run: |
+      #     aws s3 sync dist s3://nksr/whl/torch-${{ matrix.torch-version }}+${{ matrix.cuda-version }} --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
+
+      - name: Clean Conda
+        run: |
+          conda clean -pty
+        shell:
+          bash -el {0}

fvdb/.github/workflows/tests.yml

@@ -0,0 +1,39 @@
+name: Running Tests
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+
+  tests:
+    name: Unit Tests
+    runs-on:
+      - self-hosted
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up fvdb_test Conda env
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniconda-version: "latest"
+          mamba-version: "*"
+          channels: conda-forge,defaults
+          channel-priority: true
+          activate-environment: fvdb_test
+          environment-file: env/test_environment.yml
+
+
+      - name: Run tests
+        run: |
+          conda activate fvdb_test
+          MAX_JOBS=$(($(nproc) < $(free -g | awk '/^Mem:/{jobs=int($4/2.5); if(jobs<1) jobs=1; print jobs}') ? $(nproc) : $(free -g | awk '/^Mem:/{jobs=int($4/2.5); if(jobs<1) jobs=1; print jobs}'))) python setup.py test
+        shell:
+          bash -el {0}
+
+      - name: Clean Conda
+        run: |
+          conda clean -pty
+        shell:
+          bash -el {0}