Install headers & morpheus-config.cmake (nv-morpheus#1448)

* Allows users to create C++ stages/messages without a source code checkout of morpheus. * Explicitly install cpython extensions as a part of the python install. * Remove usage of `YAPF_EXCLUDE_FLAGS` work-around for an old yapf bug which has been fixed in our current version. * Update C++ developer guide examples to optionally build independently of morpheus. * Install versioneer as a conda package, remove in-repo copy of versioneer.py Closes nv-morpheus#331 ## By Submitting this PR I confirm: - I am familiar with the [Contributing Guidelines](https://github.com/nv-morpheus/Morpheus/blob/main/docs/source/developer_guide/contributing.md). - When the PR is ready for review, new or existing tests cover these changes. - When the PR is ready for review, the documentation is up to date with these changes. Authors: - David Gardner (https://github.com/dagardner-nv) - Michael Demoret (https://github.com/mdemoret-nv) Approvers: - Michael Demoret (https://github.com/mdemoret-nv) - Christopher Harris (https://github.com/cwharris) URL: nv-morpheus#1448
pdmack · Jan 22, 2024 · c0237fe · c0237fe
1 parent 1b52ab8
commit c0237fe
Show file tree

Hide file tree

Showing 67 changed files with 2,185 additions and 2,479 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -99,7 +99,10 @@
 
     "features": {
         "ghcr.io/devcontainers/features/docker-from-docker": {},
-        "ghcr.io/devcontainers/features/dotnet:1": {}
+        "ghcr.io/devcontainers/features/dotnet:1": {
+            "version": "6.0",
+            "installUsingApt": false
+        }
     },
 
     "customizations": {
@@ -116,7 +119,6 @@
 			],
             "settings": {
                 "cmake.cmakePath": "/tmp/.current-conda-env/bin/cmake",
-                "cmake.languageSupport.dotnetPath": "/usr/bin/dotnet",
                 "C_Cpp.intelliSenseEngine": "disabled",
                 "python.terminal.activateEnvironment": false
             }

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,10 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 cmake_minimum_required(VERSION 3.25 FATAL_ERROR)
 
-
 list(APPEND CMAKE_MESSAGE_CONTEXT "morpheus")
 
 # Global options (Keep sorted!)
@@ -42,19 +40,9 @@ set(MORPHEUS_RAPIDS_VERSION "23.06" CACHE STRING "Sets default versions for RAPI
 set(MORPHEUS_CACHE_DIR "${CMAKE_SOURCE_DIR}/.cache" CACHE PATH "Directory to contain all CPM and CCache data")
 mark_as_advanced(MORPHEUS_CACHE_DIR)
 
-set(CMAKE_CXX_STANDARD 20)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(CMAKE_CXX_EXTENSIONS ON)
-set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)
-set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
-set(CMAKE_INSTALL_RPATH "$ORIGIN")
-
-# Disable compile commands until after dependencies
-set(CMAKE_EXPORT_COMPILE_COMMANDS OFF)
-
 enable_testing()
 
-if (MORPHEUS_USE_IWYU AND MORPHEUS_USE_CCACHE)
+if(MORPHEUS_USE_IWYU AND MORPHEUS_USE_CCACHE)
   message(FATAL_ERROR "MORPHEUS_USE_IWYU and MORPHEUS_USE_CCACHE cannot be set simultaneously")
 endif()
 
@@ -80,66 +68,88 @@ set(MORPHEUS_UTILS_RAPIDS_VERSION ${MORPHEUS_RAPIDS_VERSION} CACHE STRING "" FOR
 # Load morpheus utils and update CMake paths
 include(morpheus_utils/load)
 
-# Configure project package manager
 morpheus_utils_initialize_package_manager(
-    MORPHEUS_USE_CONDA
-    BUILD_SHARED_LIBS
+  MORPHEUS_USE_CONDA
+  BUILD_SHARED_LIBS
 )
 
-# Configure CUDA architecture
-# NOTE: This MUST occur before any 'project' calls because of rapids_cmake requirements.
-if (DEFINED MORPHEUS_CUDA_ARCHITECTURES)
+# Initialize CUDA
+# This is a two-step process. We need to call morpheus_utils_initialize_cuda_arch which in turn calls
+# rapids_cuda_init_architectures prior to calling project(). This is because rapids_cuda_init_architectures defines a
+# `CMAKE_PROJECT_<PROJECT-NAME>_INCLUDE` hook which is invoked by the project() call. This hook is what allows us to
+# set `CMAKE_CUDA_ARCHITECTURES=rapids` when performing a release build which will be expanded to the current list of
+# supported architectures by our version of rapids.
+#
+# After the call to project() we can then call morpheus_utils_enable_cuda() which will set some CUDA+clang settings
+# which can only be performed after calling project(), but which must be set prior to calling enable_language(CUDA)
+if(DEFINED MORPHEUS_CUDA_ARCHITECTURES)
   set(CMAKE_CUDA_ARCHITECTURES "${MORPHEUS_CUDA_ARCHITECTURES}")
 endif()
 
 morpheus_utils_initialize_cuda_arch(morpheus)
 
-# Set a default build type if none was specified
-rapids_cmake_build_type(Release)
 # Project definition
+# Note intentionally excluding CUDA from the LANGUAGES list allowing us to set some clang specific settings later when
+# we call morpheus_utils_enable_cuda()
 project(morpheus
-    VERSION 24.03.00
-    LANGUAGES C CXX CUDA)
+  VERSION 24.03.00
+  LANGUAGES C CXX
+)
+
+# This sets some clang specific settings for CUDA prior to calling enable_language(CUDA)
+morpheus_utils_enable_cuda()
 
 rapids_cmake_write_version_file(${CMAKE_BINARY_DIR}/autogenerated/include/morpheus/version.hpp)
 
-# Ccache configuration
+# Set a default build type if none was specified
+rapids_cmake_build_type(Release)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS ON)
+set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)
+set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+set(CMAKE_INSTALL_RPATH "$ORIGIN")
+
+# Setup cache before dependencies
+# Configure CCache if requested
 include(environment/init_ccache)
 
+# Disable exporting compile commands for dependencies
+set(CMAKE_EXPORT_COMPILE_COMMANDS OFF)
+
+# Create a custom target to allow preparing for style checks
+add_custom_target(${PROJECT_NAME}_style_checks
+  COMMENT "Building dependencies for style checks"
+)
+
 # Configure all dependencies
 include(dependencies)
 
-####################################
-# - Post dependencies setup --------
-morpheus_utils_compiler_set_defaults(MORPHEUS_USE_CLANG_TIDY)
-
-# Setup IWYU if enabled
-include(environment/init_iwyu)
+# Enable for all first party code
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 # To make it easier for CI to find output files, set the default executable suffix to .x if not set
 if("${CMAKE_EXECUTABLE_SUFFIX}" STREQUAL "")
   set(CMAKE_EXECUTABLE_SUFFIX ".x")
 endif()
 
-# Create a custom target to allow preparing for style checks
-add_custom_target(${PROJECT_NAME}_style_checks
-  COMMENT "Building dependencies for style checks"
-)
-
-
-##################################
-##### Morpheus Python Setup ######
-##################################
+# ###################################
+# - Post dependencies setup --------
+morpheus_utils_compiler_set_defaults(MORPHEUS_USE_CLANG_TIDY)
 
-# Re-enable compile commands
-set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+# Setup IWYU if enabled
+include(environment/init_iwyu)
 
+# #################################
+# #### Morpheus Python Setup ######
+# #################################
 morpheus_utils_python_configure()
 
 # Include the main morpheus code
 morpheus_utils_create_python_package(morpheus
-    PROJECT_DIRECTORY "${CMAKE_SOURCE_DIR}"
-    SOURCE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/morpheus"
+  PROJECT_DIRECTORY "${CMAKE_SOURCE_DIR}"
+  SOURCE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/morpheus"
 )
 
 add_subdirectory(morpheus)
@@ -174,9 +184,9 @@ if(MORPHEUS_ENABLE_DEBUG_INFO)
   morpheus_utils_print_all_targets()
 
   morpheus_utils_print_target_properties(
-      TARGETS
-        morpheus morpheus._lib.llm
-      WRITE_TO_FILE
+    TARGETS
+      morpheus
+    WRITE_TO_FILE
   )
 
   morpheus_utils_print_global_properties(

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,4 +1,3 @@
-include versioneer.py
 include morpheus/_version.py
 recursive-include morpheus/data *
-recursive-include morpheus *.so py.typed *.pyi
+recursive-include morpheus *.cpython*.so py.typed *.pyi
diff --git a/ci/conda/recipes/morpheus/morpheus_build.sh b/ci/conda/recipes/morpheus/morpheus_build.sh
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -90,7 +90,7 @@ cmake -B ${BUILD_DIR} \
    .
 
 # Build the components
-cmake --build ${BUILD_DIR} -j${PARALLEL_LEVEL:-$(nproc)}
+cmake --build ${BUILD_DIR} -j${PARALLEL_LEVEL:-$(nproc)} --target install
 
 # Install just the python wheel components
 ${PYTHON} -m pip install -vv ${BUILD_DIR}/dist/*.whl
diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,14 +22,11 @@ export PY_ROOT="${MORPHEUS_ROOT}"
 export PY_CFG="${PY_ROOT}/setup.cfg"
 export PY_DIRS="${PY_ROOT} ci/scripts"
 
-# work-around for known yapf issue https://github.com/google/yapf/issues/984
-export YAPF_EXCLUDE_FLAGS="-e versioneer.py -e morpheus/_version.py"
-
 # Determine the commits to compare against. If running in CI, these will be set. Otherwise, diff with main
 export BASE_SHA=${CHANGE_TARGET:-${BASE_SHA:-$(${SCRIPT_DIR}/gitutils.py get_merge_target)}}
 export COMMIT_SHA=${GIT_COMMIT:-${COMMIT_SHA:-HEAD}}
 
-export CPP_FILE_REGEX='^(\.\/)?(morpheus|tests)\/.*\.(cc|cpp|h|hpp)$'
+export CPP_FILE_REGEX='^(\.\/)?(examples|morpheus|tests)\/.*\.(cc|cpp|h|hpp)$'
 export PYTHON_FILE_REGEX='^(\.\/)?(?!\.|build|external).*\.(py|pyx|pxd)$'
 
 # Use these options to skip any of the checks

diff --git a/ci/scripts/fix_all.sh b/ci/scripts/fix_all.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -105,5 +105,5 @@ fi
 # Run yapf
 if [[ "${SKIP_YAPF}" == "" ]]; then
    echo "Running yapf..."
-   python3 -m yapf -i --style ${PY_CFG} ${YAPF_EXCLUDE_FLAGS} -r ${PY_MODIFIED_FILES[@]}
+   python3 -m yapf -i --style ${PY_CFG} -r ${PY_MODIFIED_FILES[@]}
 fi
diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -71,10 +71,14 @@ function update_conda_env() {
     # Deactivate the environment first before updating
     conda deactivate
 
-    rapids-logger "Checking for updates to conda env"
 
-    # Update the packages
-    rapids-mamba-retry env update -n morpheus --prune -q --file "$1"
+    if [[ "${SKIP_CONDA_ENV_UPDATE}" == "" ]]; then
+        rapids-logger "Checking for updates to conda env"
+
+
+        # Update the packages
+        rapids-mamba-retry env update -n morpheus --prune -q --file "$1"
+    fi
 
     # Finally, reactivate
     conda activate morpheus
@@ -107,7 +111,6 @@ function fetch_base_branch_gh_api() {
 
 function fetch_base_branch_local() {
     rapids-logger "Retrieving base branch from git"
-    git remote remove upstream
     git remote add upstream ${GIT_UPSTREAM_URL}
     git fetch upstream --tags
     source ${MORPHEUS_ROOT}/ci/scripts/common.sh

diff --git a/ci/scripts/github/docs.sh b/ci/scripts/github/docs.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -40,10 +40,11 @@ ${MORPHEUS_ROOT}/scripts/fetch_data.py fetch docs examples
 git submodule update --init --recursive
 
 rapids-logger "Configuring for docs"
-cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} -DMORPHEUS_PYTHON_BUILD_STUBS=OFF -DMORPHEUS_BUILD_DOCS=ON .
+cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} -DCMAKE_INSTALL_PREFIX=${CONDA_PREFIX} -DMORPHEUS_PYTHON_BUILD_STUBS=OFF -DMORPHEUS_BUILD_DOCS=ON .
 
 rapids-logger "Building docs"
-cmake --build build --target morpheus_docs
+cmake --build build --parallel ${PARALLEL_LEVEL} --target install
+cmake --build build --parallel ${PARALLEL_LEVEL} --target morpheus_docs
 
 rapids-logger "Archiving the docs"
 tar cfj "${WORKSPACE_TMP}/docs.tar.bz" build/docs/html

diff --git a/ci/scripts/github/test.sh b/ci/scripts/github/test.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -26,13 +26,7 @@ rapids-dependency-file-generator \
 
 update_conda_env env.yaml
 
-rapids-logger "Check versions"
-python3 --version
-x86_64-conda-linux-gnu-cc --version
-x86_64-conda-linux-gnu-c++ --version
-cmake --version
-ninja --version
-sccache --version
+log_toolchain
 
 git submodule update --init --recursive
 
@@ -41,6 +35,7 @@ CMAKE_FLAGS="${CMAKE_FLAGS} -DCMAKE_BUILD_RPATH_USE_ORIGIN=ON"
 CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_STUBS=ON"
 CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_WHEEL=OFF"
 CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_PERFORM_INSTALL=ON"
+CMAKE_FLAGS="${CMAKE_FLAGS} -DCMAKE_INSTALL_PREFIX=${CONDA_PREFIX}"
 if [[ "${LOCAL_CI}" == "" ]]; then
     CMAKE_FLAGS="${CMAKE_FLAGS} -DCCACHE_PROGRAM_PATH=$(which sccache)"
 fi
@@ -49,7 +44,7 @@ rapids-logger "Configuring cmake for Morpheus with ${CMAKE_FLAGS}"
 cmake -B build -G Ninja ${CMAKE_FLAGS} .
 
 rapids-logger "Building Morpheus"
-cmake --build build --parallel ${PARALLEL_LEVEL}
+cmake --build build --parallel ${PARALLEL_LEVEL} --target install
 
 if [[ "${LOCAL_CI}" == "" ]]; then
     rapids-logger "sccache usage for morpheus build:"

diff --git a/ci/scripts/python_checks.sh b/ci/scripts/python_checks.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# SPDX-FileCopyrightText: Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -59,7 +59,7 @@ if [[ -n "${MORPHEUS_MODIFIED_FILES}" ]]; then
 
    if [[ "${SKIP_YAPF}" == "" ]]; then
       # Run yapf. Will return 1 if there are any diffs
-      YAPF_OUTPUT=`python3 -m yapf --style ${PY_CFG} ${YAPF_EXCLUDE_FLAGS} --diff ${MORPHEUS_MODIFIED_FILES[@]} 2>&1`
+      YAPF_OUTPUT=`python3 -m yapf --style ${PY_CFG} --diff ${MORPHEUS_MODIFIED_FILES[@]} 2>&1`
       YAPF_RETVAL=$?
    fi