CTS - CI Integration

.github/workflows/anari_sdk_ci.yml

@@ -6,16 +6,16 @@ on:
   pull_request:
     branches: [ main, next_release ]
 
-env:
-  ANARI_LIBRARY: helide
-
 jobs:
   build:
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         config: [Release, Debug]
+    env:
+      ANARI_LIBRARY: helide
 
     steps:
     - uses: actions/checkout@v3
@@ -36,6 +36,7 @@ jobs:
         -DBUILD_REMOTE_DEVICE=${{ matrix.os == 'ubuntu-latest' }}
         -DBUILD_TESTING=ON
         -DBUILD_VIEWER=OFF
+        -DCTS_ENABLE_GLTF=OFF
 
     - name: Build
       run: cmake --build ${{github.workspace}}/build --config ${{ matrix.config }} --target install
@@ -51,3 +52,73 @@ jobs:
     - name: Tutorial Tests
       working-directory: ${{github.workspace}}/build
       run: ctest -R anariTutorial -C ${{ matrix.config }}
+
+    - name: Build CTS
+      run: cmake --build ${{github.workspace}}/build --config ${{ matrix.config }} --target anariCTSBackend
+
+    - name: Debug Output Windows
+      working-directory: ${{github.workspace}}/build/install/
+      if: ${{ matrix.os == 'windows-latest' }}
+      run: |
+        echo "Directories in build/install/:"
+        ls
+
+    - name: CTS test case creation Linux
+      working-directory: ${{github.workspace}}/build/install/share/anari/cts
+      env: 
+        PYTHONPATH: ${{github.workspace}}/build/install/lib
+      if: ${{ matrix.os == 'ubuntu-latest' }}
+      run: |
+        sudo apt install pipx
+        pipx ensurepath
+        pipx install poetry
+        poetry install
+        poetry run python3 ./createTestCasesFromGltf.py
+
+    - name: CTS test case creation Windows
+      working-directory: ${{github.workspace}}/build/install/share/anari/cts
+      env: 
+        PYTHONPATH: ${{github.workspace}}/build/install/bin
+      if: ${{ matrix.os == 'windows-latest' }}
+      run: |
+        python -m pip install --user pipx
+        python -m pipx ensurepath
+        pipx install poetry
+        poetry install
+        poetry run python ./createTestCasesFromGltf.py
+
+    - name: CTS test case creation MacOS
+      working-directory: ${{github.workspace}}/build/install/share/anari/cts
+      env: 
+        PYTHONPATH: ${{github.workspace}}/build/install/lib
+      if: ${{ matrix.os == 'macos-latest' }}
+      run: |
+        brew install pipx
+        pipx ensurepath
+        pipx install poetry
+        poetry install
+        poetry run python3 ./createTestCasesFromGltf.py
+
+    - name: CTS create reference data Linux
+      working-directory: ${{github.workspace}}/build/install/share/anari/cts
+      env: 
+        PYTHONPATH: ${{github.workspace}}/build/install/lib
+      if: ${{ matrix.os == 'ubuntu-latest' }}
+      run: |
+        poetry run python3 ./createReferenceData.py
+
+    - name: CTS create reference data Windows
+      working-directory: ${{github.workspace}}/build/install/share/anari/cts
+      env: 
+        PYTHONPATH: ${{github.workspace}}/build/install/bin
+      if: ${{ matrix.os == 'windows-latest' }}
+      run: |
+        poetry run python ./createReferenceData.py
+
+    - name: CTS create reference data MacOS
+      working-directory: ${{github.workspace}}/build/install/share/anari/cts
+      env: 
+        PYTHONPATH: ${{github.workspace}}/build/install/lib
+      if: ${{ matrix.os == 'macos-latest' }}
+      run: |
+        poetry run python3 ./createReferenceData.py

.gitignore

@@ -22,3 +22,4 @@ cts/*.exp
 cts/*.lib
 cts/*.ilk
 cts/*.pdb
+cts/glTF-Sample-Assets

CMakeLists.txt

@@ -45,7 +45,19 @@ set(CMAKE_INSTALL_RPATH "$ORIGIN:$ORIGIN/../${CMAKE_INSTALL_LIBDIR}")
 option(BUILD_SHARED_LIBS "Build shared libraries instead of static" ON)
 option(BUILD_TESTING "Build tests for CTest" ON)
 # ANARI specific options
+
 option(BUILD_CTS "Build cts toolkit" ON)
+cmake_dependent_option(CTS_ENABLE_GLTF "Enable glTF support for cts" ON "BUILD_CTS" ON)
+option(USE_DRACO "Use draco when loading glTF files" OFF)
+option(USE_WEBP "Use webp when loading glTF files" OFF)
+option(USE_KTX "Use ktx when loading glTF files" OFF)
+
+if (CTS_ENABLE_GLTF)
+    set(USE_KTX ON)
+    set(USE_WEBP ON)
+    set(USE_DRACO ON)
+endif()
+
 option(BUILD_EXAMPLES "Build example applications and example device" ON)
 cmake_dependent_option(BUILD_VIEWER
   "Build interactive viewer app (requires GLFW)"
@@ -68,12 +80,19 @@ cmake_dependent_option(VIEWER_ENABLE_GLTF
   "BUILD_VIEWER"
   OFF
 )
+if (VIEWER_ENABLE_GLTF)
+    set(USE_WEBP ON)
+    set(USE_DRACO ON)
+endif()
 cmake_dependent_option(VIEWER_ENABLE_KTX
   "Enable KTX support in viewer"
   OFF
   "BUILD_VIEWER"
   OFF
 )
+if (VIEWER_ENABLE_KTX)
+    set(USE_KTX ON)
+endif()
 
 ## The generate_all targets collects all offline code generation targets
 

cts/CMakeLists.txt

@@ -3,6 +3,13 @@
 
 cmake_minimum_required(VERSION 3.1)
 
+# Fix "DOWNLOAD_EXTRACT_TIMESTAMP" warning
+if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0")
+  cmake_policy(SET CMP0135 NEW)
+endif()
+
+include(FetchContent)
+
 if (TARGET anari)
   list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../cmake)
 endif()
@@ -69,6 +76,17 @@ if (CTS_DEV AND NOT BUILD_SHARED_LIBS)
   )
 endif()
 
+message("Fetching glTF Sample Models")
+FetchContent_Declare(glTFSampleAssets
+  GIT_REPOSITORY https://github.com/KhronosGroup/glTF-Sample-Assets.git
+  GIT_TAG 754a32a333b27698b6c36d8bec8ff7ed5f729538)
+FetchContent_MakeAvailable(glTFSampleAssets)
+
+message("Copying glTF Sample Models")
+file(GLOB GLTF_SAMPLE_ASSETS_FILES ${gltfsampleassets_SOURCE_DIR}/Models/*)
+list(FILTER GLTF_SAMPLE_ASSETS_FILES EXCLUDE REGEX ".*Unicode❤♻Test")
+file(COPY ${GLTF_SAMPLE_ASSETS_FILES} DESTINATION ${CMAKE_CURRENT_LIST_DIR}/gltf-Sample-Assets)
+
 ## Installation ##
 
 # .pyd files are considered library files
@@ -95,3 +113,7 @@ install(FILES ${python_files} ${CMAKE_CURRENT_LIST_DIR}/default_test_scene.json
 install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/test_scenes
   DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/anari/cts
 )
+
+install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/gltf-Sample-Assets
+  DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/anari/cts
+)

cts/README.md

@@ -8,11 +8,23 @@ It contains the following features:
 - Verification of object/parameter info metadata
 - Verification of known object properties
 - List core extensions implemented by a device
-- Create pdf report
+- Create a pdf report
 
 ## Requirements
 
-The project uses poetry to manage package versions. If using poetry is not wanted, one can manually install the dependencies specified in [pyproject.toml](pyproject.toml).
+### ANARI Feature Support
+
+Any libraries/devices being tested need to at least support the following features:
+
+- ANARI_KHR_MATERIAL_MATTE
+- ANARI_KHR_GEOMETRY_TRIANGLE
+- ANARI_KHR_CAMERA_PERSPECTIVE
+- ANARI_KHR_RENDERER_BACKGROUND_COLOR
+- ANARI_KHR_RENDERER_AMBIENT_LIGHT
+
+### Software Dependencies
+
+This project uses poetry to manage package versions. To bypass using poetry the packages listed in [pyproject.toml](pyproject.toml) can be installed manually.
 
 - Your ANARI library files copied into this folder
 - [Python 3.9+](https://www.python.org/downloads/)
@@ -154,7 +166,6 @@ Here is the [sphere test file](test_scenes/primitives/sphere/sphere.json) as an
   - `spatial_field_dimensions`: A 3-integer array, describing the dimensions of all spatial fields in this test. If no spatial field is defined in `anariObjects`, a default one will be used.
   - `frameCompletionCallback`: When set to `true`, checks if `ANARI_KHR_FRAME_COMPLETION_CALLBACK` works correctly. Renders a green image on success and a red image otherwise.
   - `progressiveRendering`: When set to `true`, checks if `ANARI_KHR_PROGRESSIVE_RENDERING` works correctly. Renders a green image if more then 10 pixels differ from the previous rendering and a red image otherwise.
-  - `camera_generate_transform`: Enables automatic adaptation of the camera position to the scene.
   - `anari_objects`: This JSON object can be used to set generic parameters for all ANARI objects except World. It holds arrays that each have their type name as their key. Each array should only have one element. This array contains objects with parameters for each ANARI object. The parameter `subtype` is mandatory for all object types that define it. `null` can be used to reset/use the default of a parameter (useful for permutations). To set a parameter to a reference to another object, use the `ref_` prefix, then add the type and index, e.g. `ref_sampler_0`. All other `sceneParameters` take precedence over the ones set using the generic `anari_objects`. This also implies that for geometries, subtypes can not be mixed and `subtype` in `anariObjects` need to match `geometrySubtype`. Additionally, if `instances` are used no objects are added to the world despite the instances. There is no need to create e.g. a default `surface` to display materials or geometry. If an object is missing, a default object is used instead. To set `Array1D` or `Array2D` parameters, the value needs to be a JSON object containing the type as key together with its value, e.g. `{"Array1D": [0, 1, 2, 3]}`.
 -  `simplified_permutations`: When set to `true`, only the first value of each permutation list is used with permutations of other parameters, therefore reducing the amount of tests to N_0 + N_1 + ... instead of a cartesian product
 - `permutations` and `variants`: These specify scene parameters where different values should be tested. Therefore, these JSON objects contain the name of a scene parameter and a list of values for each of them. The permutations specify changes which result in a different outcome (e.g. 1 or 16 primitives). The variants should not change the outcome, therefore only one reference rendering is needed for these (e.g. the soup and indexed variants should look the same). The cartesian product is performed for all parameters. In the current example this results in four test images: 1 primitive + soup, 1 primitive + indexed, 16 primitives + soup, 16 primitives + indexed; As well as two reference images: 1 primitive (soup or indexed), 16 primitives (soup or indexed). To permutate a parameter in an ANARI object, the object needs to be defined in `anariObjects` without the parameter. In the permutation or variant object a JSON pointer e.g. `"/anari_objects/material/0/color"` should be used as key as seen in the example.

cts/copy_post_build.cmake

@@ -1,6 +1,8 @@
+message("Copying files to ${CMAKE_CURRENT_LIST_DIR}")
+
 file (GLOB pyd_files "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${BUILD_CONFIG_DIR}/*.pyd")
 file (GLOB example_lib "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${BUILD_CONFIG_DIR}/anari_library_${REFERENCE_LIB}*")
 file (GLOB debug_lib "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${BUILD_CONFIG_DIR}/anari_library_debug*")
 
-
+message("files: ${pyd_files}")
 file(COPY ${pyd_files} ${example_lib} ${debug_lib} DESTINATION "${CMAKE_CURRENT_LIST_DIR}")

cts/createReferenceData.py

@@ -4,9 +4,10 @@
 import argparse
 import math
 
+
 def writeMetaData(sceneGenerator, scene_location, permutationString):
     original_json = {}
-    with open(scene_location, "r") as scene_file:
+    with open(scene_location, "r", encoding="utf-8") as scene_file:
         original_json = json.load(scene_file)
     metaData = {"bounds": {}}
     bounds = sceneGenerator.getBounds()
@@ -23,19 +24,19 @@ def writeMetaData(sceneGenerator, scene_location, permutationString):
         if "metaData" not in original_json:
             original_json["metaData"] = {}
         original_json["metaData"][permutationString] = metaData
-    with open(scene_location, "w") as scene_file:
+    with open(scene_location, "w", encoding="utf-8") as scene_file:
         json.dump(original_json, scene_file, indent=2)
 
 
 def cleanMetaData(
     _lib, _dev, _ren, scene_location, _test_name, _permutationString, _variantString
 ):
     original_json = {}
-    with open(scene_location, "r") as scene_file:
+    with open(scene_location, "r", encoding="utf-8") as scene_file:
         original_json = json.load(scene_file)
     if "metaData" in original_json:
         del original_json["metaData"]
-        with open(scene_location, "w") as scene_file:
+        with open(scene_location, "w", encoding="utf-8") as scene_file:
             json.dump(original_json, scene_file, indent=2)
 
 
@@ -68,10 +69,17 @@ def createReferenceData(
     parser.add_argument("-r", "--renderer", default="default")
     parser.add_argument("-t", "--test_scenes", default="test_scenes")
     parser.add_argument("--ignore_features", action="store_true")
+    parser.add_argument("--no_tangent_generation", action="store_true")
 
     args = parser.parse_args()
     cts.apply_to_scenes(
-        cleanMetaData, args.library, args.device, args.renderer, args.test_scenes, True, not args.ignore_features
+        cleanMetaData,
+        args.library,
+        args.device,
+        args.renderer,
+        args.test_scenes,
+        True,
+        not args.ignore_features,
     )
     cts.apply_to_scenes(
         createReferenceData,
@@ -80,5 +88,7 @@ def createReferenceData(
         args.renderer,
         args.test_scenes,
         True,
-        not args.ignore_features
+        not args.ignore_features,
+        True,
+        not args.no_tangent_generation,
     )