diff --git a/.github/actions/setup_python/action.yml b/.github/actions/setup_python/action.yml index d067842135cd9d..c076c5156da039 100644 --- a/.github/actions/setup_python/action.yml +++ b/.github/actions/setup_python/action.yml @@ -29,9 +29,9 @@ runs: run: apt-get update && apt-get install -y ca-certificates software-properties-common - if: ${{ runner.os == 'Linux' && runner.arch == 'ARM64' }} - name: Setup sudo + name: Setup sudo and python3 shell: bash - run: apt-get update && apt-get install -y sudo # Needed for the deadsnakes action + run: apt-get update && apt-get install -y sudo python3 # Needed for the deadsnakes action - if: ${{ runner.os == 'Linux' && runner.arch == 'ARM64' }} name: Setup Python ${{ inputs.version }} diff --git a/.github/workflows/job_cpu_functional_tests.yml b/.github/workflows/job_cpu_functional_tests.yml new file mode 100644 index 00000000000000..b1f2e6bbf08b59 --- /dev/null +++ b/.github/workflows/job_cpu_functional_tests.yml @@ -0,0 +1,124 @@ +name: CPU functional tests + +on: + workflow_call: + inputs: + runner: + description: 'Machine on which the tests would run' + type: string + required: true + image: + description: 'Docker image in which the tests would run' + type: string + required: false + default: null + +jobs: + CPU_Functional_Tests: + name: CPU functional tests + timeout-minutes: 25 + runs-on: ${{ inputs.runner }} + container: + image: ${{ inputs.image }} + defaults: + run: + shell: bash + env: + DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input + INSTALL_DIR: ${{ github.workspace }}/install + INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests + PARALLEL_TEST_SCRIPT: ${{ github.workspace }}/install/tests/functional_test_utils/layer_tests_summary/run_parallel.py + PARALLEL_TEST_CACHE: ${{ github.workspace }}/install/tests/test_cache.lst + steps: + - name: Download OpenVINO package + uses: actions/download-artifact@v3 + with: + name: openvino_package + path: ${{ env.INSTALL_DIR }} + + - name: Download OpenVINO tests package + uses: actions/download-artifact@v3 + with: + name: openvino_tests + path: ${{ env.INSTALL_TEST_DIR }} + + # Needed as ${{ github.workspace }} is not working correctly when using Docker + - name: Setup Variables + run: | + echo "INSTALL_DIR=$GITHUB_WORKSPACE/install" >> "$GITHUB_ENV" + echo "INSTALL_TEST_DIR=$GITHUB_WORKSPACE/install/tests" >> "$GITHUB_ENV" + echo "PARALLEL_TEST_SCRIPT=$GITHUB_WORKSPACE/install/tests/functional_test_utils/layer_tests_summary/run_parallel.py" >> "$GITHUB_ENV" + echo "PARALLEL_TEST_CACHE=$GITHUB_WORKSPACE/install/tests/test_cache.lst" >> "$GITHUB_ENV" + + - name: Extract OpenVINO packages + run: | + pushd $INSTALL_DIR + tar -xzf openvino_package.tar.gz -C $INSTALL_DIR + popd + + pushd $INSTALL_TEST_DIR + tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR + popd + + - name: Install OpenVINO dependencies (Linux) + if: runner.os == 'Linux' + run: $INSTALL_DIR/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -c=gpu -y + + - name: Fetch setup_python action + uses: actions/checkout@v4 + with: + sparse-checkout: | + .github/actions/setup_python/action.yml + sparse-checkout-cone-mode: false + path: 'openvino' + + - name: Setup Python 3.11 + uses: ./openvino/.github/actions/setup_python + with: + version: '3.11' + should-setup-pip-paths: 'false' + self-hosted-runner: ${{ runner.os == 'Linux' }} + + - name: Install python dependencies for run_parallel.py + run: python3 -m pip install -r ${INSTALL_TEST_DIR}/functional_test_utils/layer_tests_summary/requirements.txt + + - name: Restore tests execution time + uses: actions/cache/restore@v3 + with: + path: ${{ env.PARALLEL_TEST_CACHE }} + key: ${{ runner.os }}-${{ runner.arch }}-tests-functional-cpu-stamp-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-tests-functional-cpu-stamp + + - name: Intel CPU plugin func tests (parallel) + run: | + # Needed as the Linux CC does not require setupvars to work + if [[ -f "${INSTALL_DIR}/setupvars.sh" ]]; then + source ${INSTALL_DIR}/setupvars.sh + fi + + python3 ${PARALLEL_TEST_SCRIPT} -e ${INSTALL_TEST_DIR}/ov_cpu_func_tests -c ${PARALLEL_TEST_CACHE} -w ${INSTALL_TEST_DIR} -s suite -rf 0 -- --gtest_print_time=1 --gtest_filter=*smoke* + timeout-minutes: 20 + + - name: Save tests execution time + uses: actions/cache/save@v3 + if: github.ref_name == 'master' + with: + path: ${{ env.PARALLEL_TEST_CACHE }} + key: ${{ runner.os }}-${{ runner.arch }}-tests-functional-cpu-stamp-${{ github.sha }} + + - name: Upload Test Results + uses: actions/upload-artifact@v3 + if: ${{ !cancelled() }} + with: + name: test-results-functional-cpu + path: | + ${{ env.INSTALL_TEST_DIR }}/temp/*.log + ${{ env.INSTALL_TEST_DIR }}/logs/*.log + ${{ env.INSTALL_TEST_DIR }}/logs/failed/*.log + ${{ env.INSTALL_TEST_DIR }}/logs/crashed/*.log + ${{ env.INSTALL_TEST_DIR }}/logs/hanged/*.log + ${{ env.INSTALL_TEST_DIR }}/logs/interapted/*.log + ${{ env.INSTALL_TEST_DIR }}/logs/hash_table.csv + ${{ env.PARALLEL_TEST_CACHE }} + if-no-files-found: 'error' diff --git a/.github/workflows/job_cxx_unit_tests.yml b/.github/workflows/job_cxx_unit_tests.yml new file mode 100644 index 00000000000000..bc6157d3478a8c --- /dev/null +++ b/.github/workflows/job_cxx_unit_tests.yml @@ -0,0 +1,279 @@ +name: Samples + +on: + workflow_call: + inputs: + runner: + description: 'Machine on which the tests would run' + type: string + required: true + image: + description: 'Docker image in which the tests would run' + type: string + required: false + default: null + affected-components: + description: 'Components that are affected by changes in the commit defined by the Smart CI Action' + type: string + required: true + +jobs: + CXX_Unit_Tests: + name: C++ unit tests + timeout-minutes: 30 + runs-on: ${{ inputs.runner }} + container: + image: ${{ inputs.image }} + defaults: + run: + shell: bash + env: + DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input + INSTALL_DIR: ${{ github.workspace }}/install + INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests + steps: + - name: Download OpenVINO package + uses: actions/download-artifact@v3 + with: + name: openvino_package + path: ${{ env.INSTALL_DIR }} + + - name: Download OpenVINO tests package + uses: actions/download-artifact@v3 + with: + name: openvino_tests + path: ${{ env.INSTALL_TEST_DIR }} + + # Needed as ${{ github.workspace }} is not working correctly when using Docker + - name: Setup Variables + run: | + echo "INSTALL_DIR=$GITHUB_WORKSPACE/install" >> "$GITHUB_ENV" + echo "INSTALL_TEST_DIR=$GITHUB_WORKSPACE/install/tests" >> "$GITHUB_ENV" + + - name: Extract OpenVINO packages + run: | + pushd $INSTALL_DIR + tar -xzf openvino_package.tar.gz -C $INSTALL_DIR + popd + pushd $INSTALL_TEST_DIR + tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR + popd + + - name: Install OpenVINO dependencies (Linux) + if: runner.os == 'Linux' + run: $INSTALL_DIR/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -c=gpu -y + + # + # Tests + # + + - name: OpenVINO Core Unit Tests + if: fromJSON(inputs.affected-components).Core.test + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_core_unit_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OVCoreUT.xml + + - name: OpenVINO Inference Functional Tests + if: fromJSON(inputs.affected-components).inference.test + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_inference_functional_tests --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-InferenceFunc.xml + + - name: OpenVINO Inference Unit Tests + if: fromJSON(inputs.affected-components).inference.test + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_inference_unit_tests --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-InferenceUnit.xml + + - name: Low Precision Transformations Tests + if: fromJSON(inputs.affected-components).LP_transformations.test + run: | + source ${INSTALL_DIR}/setupvars.sh + + ${INSTALL_TEST_DIR}/ov_lp_transformations_tests --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-LpTransformations.xml + + - name: OpenVINO Conditional compilation tests + if: fromJSON(inputs.affected-components).Core.test + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_conditional_compilation_tests --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ConditionalCompilation.xml + + - name: IR frontend tests + if: fromJSON(inputs.affected-components).IR_FE.test + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_ir_frontend_tests --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-IRFrontend.xml + + - name: PaddlePaddle frontend tests + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/paddle_tests --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-PaddleTests.xml + + - name: ONNX frontend tests + if: ${{ fromJSON(inputs.affected-components).ONNX_FE.test && runner.arch != 'ARM64' }} # Ticket for macOS ARM64: 122663, for Linux ARM64: 126280 + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_onnx_frontend_tests --gtest_print_time=1 \ + --gtest_filter=-*IE_GPU* \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ONNXFrontend.xml + + - name: TensorFlow Common frontend tests + if: fromJSON(inputs.affected-components).TF_FE.test || + fromJSON(inputs.affected-components).TFL_FE.test && + (runner.os != 'macOS' && runner.arch != 'ARM64') + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_tensorflow_common_tests --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TensorFlowCommonFrontend.xml + + - name: TensorFlow frontend tests + if: fromJSON(inputs.affected-components).TF_FE.test + run: | + source ${INSTALL_DIR}/setupvars.sh + + ${INSTALL_TEST_DIR}/ov_tensorflow_frontend_tests --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TensorFlowFrontend.xml + + - name: TensorFlow Lite frontend tests + if: fromJSON(inputs.affected-components).TFL_FE.test + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_tensorflow_lite_frontend_tests --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TensorFlowLiteFrontend.xml + + - name: Transformations func tests + if: ${{ fromJSON(inputs.affected-components).transformations.test && runner.arch != 'ARM64' }} # Ticket: 126281 + run: | + source ${INSTALL_DIR}/setupvars.sh + + ${INSTALL_TEST_DIR}/ov_transformations_tests --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-Transformations.xml + + - name: Legacy Transformations func tests + if: fromJSON(inputs.affected-components).GNA.test && + (runner.os != 'macOS' && runner.arch != 'ARM64') + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_legacy_transformations_tests --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-LegacyTransformations.xml + + - name: Inference Engine 1.0 unit tests + if: fromJSON(inputs.affected-components).GNA.test && + (runner.os != 'macOS' && runner.arch != 'ARM64') + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/InferenceEngineUnitTests --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-InferenceEngineUnitTests.xml + + - name: Common test utils tests + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_util_tests --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-CommonUtilTests.xml + + - name: Snippets func tests + if: fromJSON(inputs.affected-components).CPU.test + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_snippets_func_tests --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-SnippetsFuncTests.xml + + - name: CPU plugin unit tests + if: fromJSON(inputs.affected-components).CPU.test + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_cpu_unit_tests --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-CPUUnitTests.xml + + - name: ov_subgraphs_dumper_tests tests + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_subgraphs_dumper_tests --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_subgraphs_dumper_tests.xml + + - name: Template OpImpl tests + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_op_conformance_tests --gtest_print_time=1 --device=TEMPLATE --gtest_filter=*OpImpl*\ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OpImplTests.xml + + - name: AUTO unit tests + if: fromJSON(inputs.affected-components).AUTO.test + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_auto_unit_tests --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_unit_tests.xml + + - name: AUTO func Tests + if: fromJSON(inputs.affected-components).AUTO.test + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_auto_func_tests --gtest_print_time=1 \ + --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_func_tests.xml + + - name: Template plugin func tests + if: fromJSON(inputs.affected-components).TEMPLATE.test + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_template_func_tests --gtest_print_time=1 \ + --gtest_filter=*smoke* \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TemplateFuncTests.xml + + - name: Inference Engine C API tests + if: fromJSON(inputs.affected-components).C_API.test + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/InferenceEngineCAPITests --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-InferenceEngineCAPITests.xml + + - name: OpenVINO C API tests + if: fromJSON(inputs.affected-components).C_API.test + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_capi_test --gtest_print_time=1 \ + --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OpenVINOCAPITests.xml + + - name: AutoBatch unit tests + if: fromJSON(inputs.affected-components).AUTO_BATCH.test + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_auto_batch_unit_tests --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_batch_unit_tests.xml + + - name: AutoBatch func tests + if: fromJSON(inputs.affected-components).AUTO_BATCH.test + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_auto_batch_func_tests --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_batch_func_tests.xml + + - name: Proxy Plugin func tests + if: fromJSON(inputs.affected-components).PROXY.test + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OVProxyTests.xml + + - name: Hetero unit tests + if: fromJSON(inputs.affected-components).HETERO.test + run: | + source ${{ env.INSTALL_DIR }}/setupvars.sh + ${{ env.INSTALL_TEST_DIR }}/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVHeteroUnitTests.xml + + - name: Hetero func tests + if: fromJSON(inputs.affected-components).HETERO.test + run: | + source ${INSTALL_DIR}/setupvars.sh + ${INSTALL_TEST_DIR}/ov_hetero_func_tests --gtest_print_time=1 --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OVHeteroFuncTests.xml + + - name: Upload Test Results + uses: actions/upload-artifact@v3 + if: ${{ !cancelled() }} + with: + name: test-results-cpp + path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml + if-no-files-found: 'warn' diff --git a/.github/workflows/job_debian_packages.yml b/.github/workflows/job_debian_packages.yml new file mode 100644 index 00000000000000..f063a7734b8aec --- /dev/null +++ b/.github/workflows/job_debian_packages.yml @@ -0,0 +1,83 @@ +name: Debian Packages + +on: + workflow_call: + inputs: + runner: + description: 'Machine on which the tests would run' + type: string + required: true + image: + description: 'Docker image in which the tests would run' + type: string + required: false + default: null + +jobs: + Debian_Packages: + name: Debian Packages + runs-on: ${{ inputs.runner }} + container: + image: ${{ inputs.image }} + defaults: + run: + shell: bash + env: + DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input + DEBIAN_PACKAGES_DIR: ${{ github.workspace }}/packages + steps: + + - name: Download OpenVINO debian packages + uses: actions/download-artifact@v3 + with: + name: openvino_debian_packages + path: ${{ env.DEBIAN_PACKAGES_DIR }} + + # Needed as ${{ github.workspace }} is not working correctly when using Docker + - name: Setup Variables + run: echo "DEBIAN_PACKAGES_DIR=$GITHUB_WORKSPACE/packages" >> "$GITHUB_ENV" + + - name: Install debian packages & check conflicts + run: | + apt-get update -y + + if [[ "${{ runner.arch }}" == "X64" ]]; then + # Install debian packages from previous release + apt-get install --no-install-recommends -y gnupg wget ca-certificates + wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB + apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB + echo "deb https://apt.repos.intel.com/openvino/2023 ubuntu20 main" | tee /etc/apt/sources.list.d/intel-openvino-2023.list + apt-get update -y + apt-get install -y openvino + fi + + # install our local one and make sure the conflicts are resolved + apt-get install --no-install-recommends -y dpkg-dev + dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz + echo "deb [trusted=yes] file:${DEBIAN_PACKAGES_DIR} ./" | tee /etc/apt/sources.list.d/openvino-local.list + apt-get update -y + apt-get install openvino -y + working-directory: ${{ env.DEBIAN_PACKAGES_DIR }} + + - name: Test debian packages + run: | + /usr/share/openvino/samples/cpp/build_samples.sh + /usr/share/openvino/samples/c/build_samples.sh + + [[ "${{ runner.arch }}" == "X64" ]] && path_by_arch="intel64" || path_by_arch="aarch64" + ~/openvino_cpp_samples_build/$path_by_arch/Release/hello_query_device + + python3 /usr/share/openvino/samples/python/hello_query_device/hello_query_device.py + python3 -c 'from openvino import Core; Core().get_property("CPU", "AVAILABLE_DEVICES")' + + if [[ "${{ runner.arch }}" == "X64" ]]; then + python3 -c 'from openvino import Core; Core().get_property("GPU", "AVAILABLE_DEVICES")' + fi + + python3 -c 'from openvino import Core; Core().get_property("AUTO", "SUPPORTED_METRICS")' + python3 -c 'from openvino import Core; Core().get_property("MULTI", "SUPPORTED_METRICS")' + python3 -c 'from openvino import Core; Core().get_property("HETERO", "SUPPORTED_METRICS")' + python3 -c 'from openvino import Core; Core().get_property("BATCH", "SUPPORTED_METRICS")' + python3 -c 'from openvino.frontend import FrontEndManager; assert len(FrontEndManager().get_available_front_ends()) == 6' + benchmark_app --help + ovc --help diff --git a/.github/workflows/job_onnx_runtime.yml b/.github/workflows/job_onnx_runtime.yml new file mode 100644 index 00000000000000..5a6f5cb27eceae --- /dev/null +++ b/.github/workflows/job_onnx_runtime.yml @@ -0,0 +1,157 @@ +name: ONNX Runtime Integration + +on: + workflow_call: + inputs: + runner: + description: 'Machine on which the tests would run' + type: string + required: true + container: + description: 'JSON to be converted to the value of the "container" configuration for the job' + type: string + required: false + default: '{"image": null}' + sccache-azure-key-prefix: + description: 'Key prefix for the cache folder on the Azure' + type: string + required: true + +jobs: + ONNX_Runtime: + name: ONNX Runtime Integration + timeout-minutes: 60 + runs-on: ${{ inputs.runner }} + container: ${{ fromJSON(inputs.container) }} + defaults: + run: + shell: bash + env: + DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input + OPENVINO_REPO: ${{ github.workspace }}/openvino + INSTALL_DIR: ${{ github.workspace }}/install + CMAKE_GENERATOR: 'Ninja Multi-Config' + CMAKE_CXX_COMPILER_LAUNCHER: sccache + CMAKE_C_COMPILER_LAUNCHER: sccache + SCCACHE_AZURE_KEY_PREFIX: ${{ inputs.sccache-azure-key-prefix }} + ONNX_RUNTIME_REPO: ${{ github.workspace }}/onnxruntime + ONNX_RUNTIME_UTILS: ${{ github.workspace }}/install/onnxruntime + ONNX_RUNTIME_BUILD_DIR: ${{ github.workspace }}/onnxruntime/build + steps: + - name: Download OpenVINO package + uses: actions/download-artifact@v3 + with: + name: openvino_package + path: ${{ env.INSTALL_DIR }} + + # Needed as ${{ github.workspace }} is not working correctly when using Docker + - name: Setup Variables + run: | + echo "OPENVINO_REPO=$GITHUB_WORKSPACE/openvino" >> "$GITHUB_ENV" + echo "INSTALL_DIR=$GITHUB_WORKSPACE/install" >> "$GITHUB_ENV" + echo "ONNX_RUNTIME_REPO=$GITHUB_WORKSPACE/onnxruntime" >> "$GITHUB_ENV" + echo "ONNX_RUNTIME_UTILS=$GITHUB_WORKSPACE/install/onnxruntime" >> "$GITHUB_ENV" + echo "ONNX_RUNTIME_BUILD_DIR=$GITHUB_WORKSPACE/onnxruntime/build" >> "$GITHUB_ENV" + + - name: Fetch install_build_dependencies.sh and setup_python action + uses: actions/checkout@v4 + with: + sparse-checkout: | + install_build_dependencies.sh + .github/actions/setup_python/action.yml + sparse-checkout-cone-mode: false + path: 'openvino' + + - name: Install git + run: | + apt-get update + apt-get install --assume-yes --no-install-recommends git ca-certificates + + - name: Setup Python ${{ env.PYTHON_VERSION }} + uses: ./openvino/.github/actions/setup_python + with: + version: '3.11' + should-setup-pip-paths: 'false' + + - name: Extract OpenVINO package + run: | + pushd ${INSTALL_DIR} + tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} + popd + + - name: Install OpenVINO dependencies + run: ${INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -y + + - name: Clone ONNX Runtime + run: | + branch=`tr -s '\n ' < ${ONNX_RUNTIME_UTILS}/version` + git clone --branch $branch --single-branch --recursive https://github.com/microsoft/onnxruntime.git ${ONNX_RUNTIME_REPO} + + # + # Tests + # + + - name: Install Build Dependencies + run: bash ${OPENVINO_REPO}/install_build_dependencies.sh + + - name: Install sccache + uses: mozilla-actions/sccache-action@v0.0.3 + with: + version: "v0.5.4" + + - name: Build Lin ONNX Runtime + run: | + source ${INSTALL_DIR}/setupvars.sh + + ${ONNX_RUNTIME_REPO}/build.sh \ + --config RelWithDebInfo \ + --use_openvino CPU_FP32 \ + --build_shared_lib \ + --parallel \ + --skip_tests \ + --compile_no_warning_as_error \ + --build_dir ${ONNX_RUNTIME_BUILD_DIR} + env: + CXXFLAGS: "-Wno-error=deprecated-declarations" + + - name: Show sccache stats + run: ${SCCACHE_PATH} --show-stats + + - name: Run onnxruntime_test_all + if: ${{ runner.arch != 'ARM64' }} # Ticket: 126277 + run: | + source ${INSTALL_DIR}/setupvars.sh + skip_tests=$(tr -s '\n ' ':' < ${ONNX_RUNTIME_UTILS}/skip_tests) + + ./onnxruntime_test_all --gtest_filter=-$skip_tests + working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo + + - name: Run onnxruntime_shared_lib_test + run: | + source ${INSTALL_DIR}/setupvars.sh + ./onnxruntime_shared_lib_test --gtest_filter=-CApiTest.test_custom_op_openvino_wrapper_library + working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo + + - name: Run onnxruntime_global_thread_pools_test + run: | + source ${INSTALL_DIR}/setupvars.sh + ./onnxruntime_global_thread_pools_test + working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo + + - name: Run onnxruntime_api_tests_without_env + run: | + source ${INSTALL_DIR}/setupvars.sh + ./onnxruntime_api_tests_without_env + working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo + + - name: Run pytorch-converted tests + run: | + source ${INSTALL_DIR}/setupvars.sh + ./onnx_test_runner "${ONNX_RUNTIME_REPO}/cmake/external/onnx/onnx/backend/test/data/pytorch-converted" + working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo + + - name: Run pytorch-operator tests + run: | + source ${INSTALL_DIR}/setupvars.sh + ./onnx_test_runner "${ONNX_RUNTIME_REPO}/cmake/external/onnx/onnx/backend/test/data/pytorch-operator" + working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo diff --git a/.github/workflows/job_python_unit_tests.yml b/.github/workflows/job_python_unit_tests.yml new file mode 100644 index 00000000000000..d81c50c0a69539 --- /dev/null +++ b/.github/workflows/job_python_unit_tests.yml @@ -0,0 +1,323 @@ +name: Python unit tests + +on: + workflow_call: + inputs: + runner: + description: 'Machine on which the tests would run' + type: string + required: true + container: + description: 'JSON to be converted to the value of the "container" configuration for the job' + type: string + required: false + default: '{"image": null}' + affected-components: + description: 'Components that are affected by changes in the commit defined by the Smart CI Action' + type: string + required: true + +env: + PIP_CACHE_PATH: /mount/caches/pip/linux + PYTHON_VERSION: '3.11' + +jobs: + Python_Unit_Tests: + name: Python unit tests + timeout-minutes: 60 + runs-on: ${{ inputs.runner }} + container: ${{ fromJSON(inputs.container) }} + defaults: + run: + shell: bash + env: + DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input + OPENVINO_REPO: ${{ github.workspace }}/openvino + INSTALL_DIR: ${{ github.workspace }}/install + INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests + LAYER_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/layer_tests + steps: + - name: Download OpenVINO package + uses: actions/download-artifact@v3 + with: + name: openvino_package + path: ${{ env.INSTALL_DIR }} + + - name: Download OpenVINO tests package + uses: actions/download-artifact@v3 + with: + name: openvino_tests + path: ${{ env.INSTALL_TEST_DIR }} + + # Needed as ${{ github.workspace }} is not working correctly when using Docker + - name: Setup Variables + run: | + echo "OPENVINO_REPO=$GITHUB_WORKSPACE/openvino" >> "$GITHUB_ENV" + echo "INSTALL_DIR=$GITHUB_WORKSPACE/install" >> "$GITHUB_ENV" + echo "INSTALL_TEST_DIR=$GITHUB_WORKSPACE/install/tests" >> "$GITHUB_ENV" + echo "LAYER_TESTS_INSTALL_DIR=$GITHUB_WORKSPACE/install/tests/layer_tests" >> "$GITHUB_ENV" + + - name: Extract OpenVINO packages + run: | + pushd $INSTALL_DIR + tar -xzf openvino_package.tar.gz -C $INSTALL_DIR + popd + pushd $INSTALL_TEST_DIR + tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR + popd + + - name: Install OpenVINO dependencies (Linux) + if: runner.os == 'Linux' + run: $INSTALL_DIR/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -y + + - name: Fetch setup_python action + uses: actions/checkout@v4 + with: + sparse-checkout: | + .github/actions/setup_python/action.yml + sparse-checkout-cone-mode: false + path: 'openvino' + + - name: Setup Python ${{ env.PYTHON_VERSION }} + uses: ./openvino/.github/actions/setup_python + with: + version: ${{ env.PYTHON_VERSION }} + pip-cache-path: ${{ runner.os == 'Linux' && env.PIP_CACHE_PATH || '' }} + should-setup-pip-paths: ${{ runner.os == 'Linux' }} + self-hosted-runner: ${{ runner.os == 'Linux' }} + + # + # Tests + # + + - name: Install OpenVINO Python wheels + run: | + # Install the core OV wheel + python3 -m pip install ${INSTALL_DIR}/tools/openvino-*.whl + + extras_to_install="caffe,kaldi,onnx,tensorflow2,pytorch" + + if [[ "${{ runner.arch }}" != "ARM64" ]]; then + extras_to_install="mxnet,$extras_to_install" + fi + + # Find and install OV dev wheel + pushd ${INSTALL_DIR}/tools + ov_dev_wheel_name=$(find . -name 'openvino_dev*.whl') + python3 -m pip install $ov_dev_wheel_name[$extras_to_install] + popd + + - name: Install Python API tests dependencies + run: | + # To enable pytest parallel features + python3 -m pip install pytest-xdist[psutil] + # For torchvision to OpenVINO preprocessing converter + python3 -m pip install -r ${INSTALL_TEST_DIR}/python/preprocess/torchvision/requirements.txt + + # TODO: replace with Python API tests requirements + python3 -m pip install -r ${INSTALL_TEST_DIR}/mo/requirements_dev.txt + + # + # Tests + # + + - name: Python API 1.0 Tests + if: fromJSON(inputs.affected-components).Python_API.test + run: | + python3 -m pytest -s ${INSTALL_TEST_DIR}/pyngraph \ + --junitxml=${INSTALL_TEST_DIR}/TEST-Pyngraph.xml \ + --ignore=${INSTALL_TEST_DIR}/pyngraph/tests_compatibility/test_onnx/test_zoo_models.py \ + --ignore=${INSTALL_TEST_DIR}/pyngraph/tests_compatibility/test_onnx/test_backend.py + + - name: Python API 2.0 Tests + if: ${{ fromJSON(inputs.affected-components).Python_API.test && runner.arch != 'ARM64' }} # Ticket: 126380 + run: | + # for 'template' extension + export LD_LIBRARY_PATH=${INSTALL_TEST_DIR}:$LD_LIBRARY_PATH + python3 -m pytest -sv ${INSTALL_TEST_DIR}/pyopenvino \ + --junitxml=${INSTALL_TEST_DIR}/TEST-Pyngraph.xml \ + --ignore=${INSTALL_TEST_DIR}/pyopenvino/tests/test_utils/test_utils.py + + - name: Model Optimizer unit tests + if: fromJSON(inputs.affected-components).MO.test + run: | + skip_filter='' + if [[ "${{ runner.os }}" != "Linux" ]] && [[ "${{ runner.arch }} != "ARM64" ]] || [[ "${{ runner.os }} != "macOS" ]]; then + # required for MxNet + apt-get install -y libgomp1 libquadmath0 + else + # Skips under Ticket: 122666 + skip_filter='--ignore-glob=**/mo/unit_tests/mo/front/mxnet/**' + fi + + python3 -m pytest -s ${INSTALL_TEST_DIR}/mo/unit_tests \ + --junitxml=${INSTALL_TEST_DIR}/TEST-ModelOptimizer.xml \ + "$skip_filter" + + - name: Python ONNX operators tests + if: fromJSON(inputs.affected-components).Python_API.test || + fromJSON(inputs.affected-components).ONNX_FE.test && runner.os != 'macOS' # Ticket: 123325 + run: | + # Skip test_onnx/test_zoo_models and test_onnx/test_backend due to long execution time - ONNX Model Zoo tests are run separately + python3 -m pytest -sv ${INSTALL_TEST_DIR}/onnx -k 'not cuda' \ + --junitxml=${INSTALL_TEST_DIR}/TEST-onnx_frontend.xml \ + --ignore=${INSTALL_TEST_DIR}/onnx/test_python/test_zoo_models.py + + - name: OVC unit tests + if: fromJSON(inputs.affected-components).MO.test + run: python3 -m pytest -s ${INSTALL_TEST_DIR}/ovc/unit_tests --junitxml=${INSTALL_TEST_DIR}/TEST-OpenVinoConversion.xml + + - name: Install Python Layer tests dependencies + run: | + # layer test requirements + python3 -m pip install -r ${LAYER_TESTS_INSTALL_DIR}/requirements.txt + + - name: MO Python API Tests + if: fromJSON(inputs.affected-components).MO.test + run: | + # Import 'test_utils' installed in '/tests/python/openvino' + export LD_LIBRARY_PATH=${PIP_INSTALL_PATH}/openvino/libs:$LD_LIBRARY_PATH + export PYTHONPATH=${INSTALL_TEST_DIR}/python + + if [[ "${{ runner.os }}" == "Linux" ]] && [[ "${{ runner.arch }}" == "ARM64" ]]; then + # Find gomp lib + GOMP_LIB=$(find "${PIP_INSTALL_PATH}/torch/lib/../../torch.libs/" -name '*libgomp-*so*') + export LD_PRELOAD=${GOMP_LIB} + fi + + python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/mo_python_api_tests --junitxml=${INSTALL_TEST_DIR}/TEST-test_mo_convert.xml + env: + TEST_DEVICE: CPU + TEST_PRECISION: FP16 + + - name: OVC Python API Tests + if: fromJSON(inputs.affected-components).MO.test + run: | + # Import 'test_utils' installed in '/tests/python/openvino' + export PYTHONPATH=${INSTALL_TEST_DIR}/python + export LD_LIBRARY_PATH=${PIP_INSTALL_PATH}/openvino/libs:$LD_LIBRARY_PATH + + if [[ "${{ runner.os }}" == "Linux" ]] && [[ "${{ runner.arch }}" == "ARM64" ]]; then + # Find gomp lib + GOMP_LIB=$(find "${PIP_INSTALL_PATH}/torch/lib/../../torch.libs/" -name '*libgomp-*so*') + export LD_PRELOAD=${GOMP_LIB} + fi + + python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/ovc_python_api_tests --junitxml=${INSTALL_TEST_DIR}/TEST-test_ovc_convert.xml + env: + TEST_DEVICE: CPU + TEST_PRECISION: FP16 + + - name: Python Frontend tests + if: fromJSON(inputs.affected-components).PyTorch_FE.test || + fromJSON(inputs.affected-components).PDPD_FE.test + run: | + # to allow 'libtest_builtin_extensions.so' to find 'libopenvino_onnx_frontend.so' + export LD_LIBRARY_PATH=${PIP_INSTALL_PATH}/openvino/libs:$LD_LIBRARY_PATH + python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/py_frontend_tests --junitxml=${INSTALL_TEST_DIR}/TEST-test_py_fontend.xml + + - name: PyTorch Layer Tests + if: ${{ fromJSON(inputs.affected-components).PyTorch_FE.test && runner.arch != 'ARM64' }} # Ticket: 126287 + run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/pytorch_tests -n logical -m precommit --junitxml=${INSTALL_TEST_DIR}/TEST-pytorch.xml + env: + TEST_DEVICE: CPU + TEST_PRECISION: FP32 + + - name: PyTorch torch.compile TORCHFX Layer Tests + if: ${{ fromJSON(inputs.affected-components).PyTorch_FE.test && runner.os != 'macOS' }} + run: | + python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/pytorch_tests -m precommit_fx_backend --junitxml=${INSTALL_TEST_DIR}/TEST-pytorch.xml + env: + TEST_DEVICE: CPU + TEST_PRECISION: FP32 + PYTORCH_TRACING_MODE: TORCHFX + + - name: PyTorch torch.compile TORCHSCRIPT Layer Tests + if: ${{ fromJSON(inputs.affected-components).PyTorch_FE.test && runner.os != 'macOS' }} + run: | + python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/pytorch_tests -m precommit_ts_backend --junitxml=${INSTALL_TEST_DIR}/TEST-pytorch.xml + env: + TEST_DEVICE: CPU + TEST_PRECISION: FP32 + PYTORCH_TRACING_MODE: TORCHSCRIPT + + - name: ONNX Layer Tests + if: fromJSON(inputs.affected-components).ONNX_FE.test + run: | + # requires 'unit_tests' from 'tools/mo' + export PYTHONPATH=${INSTALL_TEST_DIR}/mo:$PYTHONPATH + python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/onnx_tests -m "not launch_only_if_manually_specified and precommit" --junitxml=${INSTALL_TEST_DIR}/TEST-onnx.xml + env: + TEST_DEVICE: CPU + TEST_PRECISION: FP16 + + - name: TensorFlow 1 Layer Tests - TF FE + if: fromJSON(inputs.affected-components).TF_FE.test + run: | + # requires 'unit_tests' from 'mo' + export PYTHONPATH=${INSTALL_TEST_DIR}/mo + python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow_tests/ --use_new_frontend -m precommit_tf_fe --junitxml=${INSTALL_TEST_DIR}/TEST-tf_fe.xml + env: + TEST_DEVICE: CPU + TEST_PRECISION: FP16 + + - name: TensorFlow 2 Layer Tests - TF FE + if: fromJSON(inputs.affected-components).TF_FE.test && runner.os != 'macOS' # Ticket: 123322 + run: | + # requires 'unit_tests' from 'mo' + export PYTHONPATH=${INSTALL_TEST_DIR}/mo + python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow2_keras_tests/ --use_new_frontend -m precommit_tf_fe --junitxml=${INSTALL_TEST_DIR}/TEST-tf2_fe.xml + env: + TEST_DEVICE: CPU + TEST_PRECISION: FP16 + + - name: JAX Layer Tests - TF FE + if: ${{ fromJSON(inputs.affected-components).TF_FE.test && runner.arch != 'ARM64' }} + run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/jax_tests/ -m precommit --junitxml=${INSTALL_TEST_DIR}/TEST-jax.xml + env: + TEST_DEVICE: CPU + + - name: TensorFlow 1 Layer Tests - Legacy FE + if: fromJSON(inputs.affected-components).TF_FE.test + run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow_tests/test_tf_Roll.py --ir_version=10 --junitxml=${INSTALL_TEST_DIR}/TEST-tf_Roll.xml + + - name: TensorFlow 2 Layer Tests - Legacy FE + if: fromJSON(inputs.affected-components).TF_FE.test + run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow2_keras_tests/test_tf2_keras_activation.py --ir_version=11 -k "sigmoid" --junitxml=${INSTALL_TEST_DIR}/TEST-tf2_Activation.xml + env: + TEST_DEVICE: CPU + TEST_PRECISION: FP16 + + - name: TensorFlow Lite Layer Tests - TFL FE + if: fromJSON(inputs.affected-components).TFL_FE.test + run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow_lite_tests/ --junitxml=${INSTALL_TEST_DIR}/TEST-tfl_fe.xml + env: + TEST_DEVICE: CPU + TEST_PRECISION: FP16 + + - name: Clone API snippets + if: runner.os != 'macOS' + uses: actions/checkout@v4 + with: + sparse-checkout: openvino/docs/snippets + path: ${{ env.OPENVINO_REPO }} + submodules: 'false' + + - name: Docs Python snippets + if: runner.os != 'macOS' + run: | + # to find 'snippets' module in docs + export PYTHONPATH=${OPENVINO_REPO}/docs + # for 'template' extension + export LD_LIBRARY_PATH=${INSTALL_TEST_DIR}:$LD_LIBRARY_PATH + python3 ${OPENVINO_REPO}/docs/snippets/main.py + + - name: Upload Test Results + uses: actions/upload-artifact@v3 + if: ${{ !cancelled() }} + with: + name: test-results-python + path: | + ${{ env.INSTALL_TEST_DIR }}/TEST*.html + ${{ env.INSTALL_TEST_DIR }}/TEST*.xml + if-no-files-found: 'warn' diff --git a/.github/workflows/job_pytorch_models_tests.yml b/.github/workflows/job_pytorch_models_tests.yml new file mode 100644 index 00000000000000..1640bea02c413b --- /dev/null +++ b/.github/workflows/job_pytorch_models_tests.yml @@ -0,0 +1,126 @@ +name: PyTorch Models tests + +on: + workflow_call: + inputs: + runner: + description: 'Machine on which the tests would run' + type: string + required: true + container: + description: 'JSON to be converted to the value of the "container" configuration for the job' + type: string + required: false + default: '{"image": null}' + event: + description: 'Event that triggered the workflow. E.g., "schedule" for nightly runs' + type: string + required: true + +jobs: + PyTorch_Models_Tests: + name: PyTorch Models tests + timeout-minutes: ${{ inputs.event == 'schedule' && 400 || 30 }} + runs-on: ${{ inputs.runner }} + container: ${{ fromJSON(inputs.container) }} + defaults: + run: + shell: bash + env: + DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input + OPENVINO_REPO: ${{ github.workspace }}/openvino + INSTALL_DIR: ${{ github.workspace }}/install + INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests + MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests + steps: + + - name: Check sudo + if: ${{ runner.os == 'Linux' }} + run: if [ "$(id -u)" -eq 0 ]; then apt update && apt --assume-yes install sudo; fi + + - name: Download OpenVINO package + uses: actions/download-artifact@v3 + with: + name: openvino_package + path: ${{ env.INSTALL_DIR }} + + - name: Download OpenVINO tests package + uses: actions/download-artifact@v3 + with: + name: openvino_tests + path: ${{ env.INSTALL_TEST_DIR }} + + # Needed as ${{ github.workspace }} is not working correctly when using Docker + - name: Setup Variables + run: | + echo "OPENVINO_REPO=$GITHUB_WORKSPACE/openvino" >> "$GITHUB_ENV" + echo "INSTALL_DIR=$GITHUB_WORKSPACE/install" >> "$GITHUB_ENV" + echo "INSTALL_TEST_DIR=$GITHUB_WORKSPACE/install/tests" >> "$GITHUB_ENV" + echo "MODEL_HUB_TESTS_INSTALL_DIR=$GITHUB_WORKSPACE/install/tests/model_hub_tests" >> "$GITHUB_ENV" + + - name: Extract OpenVINO packages + run: | + pushd ${INSTALL_DIR} + tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} + popd + + pushd ${INSTALL_TEST_DIR} + tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} + popd + + - name: Fetch setup_python action + uses: actions/checkout@v4 + with: + sparse-checkout: | + .github/actions/setup_python/action.yml + sparse-checkout-cone-mode: false + path: 'openvino' + + - name: Install dependencies + if: ${{ runner.os == 'Linux' }} + run: | + # install git (required to build pip deps from the sources) + # install 'g++' to build 'detectron2' and 'natten' wheels + sudo apt-get install --assume-yes --no-install-recommends g++ git ca-certificates + + - name: Setup Python 3.11 + uses: ./openvino/.github/actions/setup_python + with: + version: '3.11' + should-setup-pip-paths: 'false' + self-hosted-runner: ${{ contains(inputs.runner, 'aks') }} + + - name: Install OpenVINO Python wheels + run: python3 -m pip install ${INSTALL_DIR}/tools/openvino-* + + - name: Install PyTorch tests requirements + run: | + python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/requirements.txt + python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/requirements_secondary.txt + echo "Available storage:" + df -h + env: + CPLUS_INCLUDE_PATH: ${{ env.Python_ROOT_DIR }}/include/python${{ env.PYTHON_VERSION }} + + - name: PyTorch Models Tests + run: | + export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH + python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_model_tests.html --self-contained-html -v + env: + TYPE: ${{ inputs.event == 'schedule' && 'nightly' || 'precommit'}} + TEST_DEVICE: CPU + USE_SYSTEM_CACHE: False + + - name: Available storage after tests + run: | + echo "Available storage:" + df -h + + - name: Upload Test Results + uses: actions/upload-artifact@v3 + if: ${{ !cancelled() }} + with: + name: test-results-torch-models + path: | + ${{ env.INSTALL_TEST_DIR }}/TEST*.html + if-no-files-found: 'error' diff --git a/.github/workflows/job_samples_tests.yml b/.github/workflows/job_samples_tests.yml new file mode 100644 index 00000000000000..8c1401a819d845 --- /dev/null +++ b/.github/workflows/job_samples_tests.yml @@ -0,0 +1,132 @@ +name: Samples + +on: + workflow_call: + inputs: + runner: + description: 'Machine on which the tests would run' + type: string + required: true + image: + description: 'Docker image in which the tests would run' + type: string + required: false + default: null + affected-components: + description: 'Components that are affected by changes in the commit defined by the Smart CI Action' + type: string + required: true + +jobs: + Samples: + runs-on: ${{ inputs.runner }} + container: + image: ${{ inputs.image }} + defaults: + run: + shell: bash + env: + DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input + INSTALL_DIR: ${{ github.workspace }}/install + INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests + BUILD_DIR: ${{ github.workspace }}/build + steps: + - name: Download OpenVINO package + uses: actions/download-artifact@v3 + with: + name: openvino_package + path: ${{ env.INSTALL_DIR }} + + - name: Download OpenVINO tests package + uses: actions/download-artifact@v3 + with: + name: openvino_tests + path: ${{ env.INSTALL_TEST_DIR }} + + # Needed as ${{ github.workspace }} is not working correctly when using Docker + - name: Setup Variables + run: | + echo "INSTALL_DIR=$GITHUB_WORKSPACE/install" >> "$GITHUB_ENV" + echo "INSTALL_TEST_DIR=$GITHUB_WORKSPACE/install/tests" >> "$GITHUB_ENV" + echo "BUILD_DIR=$GITHUB_WORKSPACE/build" >> "$GITHUB_ENV" + + - name: Extract OpenVINO packages + run: | + pushd $INSTALL_DIR + tar -xzf openvino_package.tar.gz -C $INSTALL_DIR + popd + pushd $INSTALL_TEST_DIR + tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR + popd + + - name: Install OpenVINO dependencies (Linux) + if: runner.os == 'Linux' + run: $INSTALL_DIR/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -y + + - name: Install OpenVINO dependencies (mac) + if: runner.os == 'macOS' + run: brew install coreutils + + - name: Fetch setup_python action + uses: actions/checkout@v4 + with: + sparse-checkout: | + .github/actions/setup_python/action.yml + sparse-checkout-cone-mode: false + path: 'openvino' + + - name: Setup Python 3.11 + uses: ./openvino/.github/actions/setup_python + with: + version: '3.11' + should-setup-pip-paths: 'false' + self-hosted-runner: ${{ runner.os == 'Linux' }} + + - name: Build cpp samples - GCC + run: $INSTALL_DIR/samples/cpp/build_samples.sh -i $INSTALL_DIR -b $BUILD_DIR/cpp_samples + env: + CMAKE_COMPILE_WARNING_AS_ERROR: 'ON' + + - name: Build cpp samples - Clang + if: runner.os == 'Linux' + run: | + apt-get install -y clang + $INSTALL_DIR/samples/cpp/build_samples.sh -i $INSTALL_DIR -b $BUILD_DIR/cpp_samples_clang + env: + CMAKE_COMPILE_WARNING_AS_ERROR: 'ON' + CC: clang + CXX: clang++ + + - name: Build c samples + run: $INSTALL_DIR/samples/c/build_samples.sh -i $INSTALL_DIR -b $BUILD_DIR/c_samples + env: + CMAKE_COMPILE_WARNING_AS_ERROR: 'ON' + + # + # Tests + # + + - name: Samples tests + if: fromJSON(inputs.affected-components).samples.test + run: | + export WORKSPACE=$INSTALL_DIR + export IE_APP_PATH=$INSTALL_DIR/samples_bin + export IE_APP_PYTHON_PATH=$INSTALL_DIR/samples/python + export SHARE=$INSTALL_TEST_DIR/smoke_tests/samples_smoke_tests_data + + python3 -m pip install --ignore-installed PyYAML -r $INSTALL_TEST_DIR/smoke_tests/requirements.txt + export LD_LIBRARY_PATH=${IE_APP_PATH}:$LD_LIBRARY_PATH + + source ${INSTALL_DIR}/setupvars.sh + + python3 -m pytest -sv $INSTALL_TEST_DIR/smoke_tests \ + --env_conf $INSTALL_TEST_DIR/smoke_tests/env_config.yml \ + --junitxml=$INSTALL_TEST_DIR/TEST-SamplesSmokeTests.xml + + - name: Upload Test Results + uses: actions/upload-artifact@v3 + if: ${{ !cancelled() }} + with: + name: test-results-samples + path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml + if-no-files-found: 'warn' diff --git a/.github/workflows/job_tensorflow_hub_models_tests.yml b/.github/workflows/job_tensorflow_hub_models_tests.yml new file mode 100644 index 00000000000000..6dcecaa267e8c4 --- /dev/null +++ b/.github/workflows/job_tensorflow_hub_models_tests.yml @@ -0,0 +1,113 @@ +name: TensorFlow Hub Models tests + +on: + workflow_call: + inputs: + runner: + description: 'Machine on which the tests would run' + type: string + required: true + container: + description: 'JSON to be converted to the value of the "container" configuration for the job' + type: string + required: false + default: '{"image": null}' + event: + description: 'Event that triggered the workflow. E.g., "schedule" for nightly runs' + type: string + required: true + +jobs: + TensorFlow_Hub_Models_Tests: + name: TensorFlow Hub Models tests + timeout-minutes: ${{ inputs.event == 'schedule' && 400 || 25 }} + runs-on: ${{ inputs.runner }} + container: ${{ fromJSON(inputs.container) }} + defaults: + run: + shell: bash + env: + DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input + OPENVINO_REPO: ${{ github.workspace }}/openvino + INSTALL_DIR: ${{ github.workspace }}/install + INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests + MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests + steps: + + - name: Check sudo + if: ${{ runner.os == 'Linux' }} + run: if [ "$(id -u)" -eq 0 ]; then apt update && apt --assume-yes install sudo; fi + + - name: Download OpenVINO package + uses: actions/download-artifact@v3 + with: + name: openvino_package + path: ${{ env.INSTALL_DIR }} + + - name: Download OpenVINO tests package + uses: actions/download-artifact@v3 + with: + name: openvino_tests + path: ${{ env.INSTALL_TEST_DIR }} + + # Needed as ${{ github.workspace }} is not working correctly when using Docker + - name: Setup Variables + run: | + echo "OPENVINO_REPO=$GITHUB_WORKSPACE/openvino" >> "$GITHUB_ENV" + echo "INSTALL_DIR=$GITHUB_WORKSPACE/install" >> "$GITHUB_ENV" + echo "INSTALL_TEST_DIR=$GITHUB_WORKSPACE/install/tests" >> "$GITHUB_ENV" + echo "MODEL_HUB_TESTS_INSTALL_DIR=$GITHUB_WORKSPACE/install/tests/model_hub_tests" >> "$GITHUB_ENV" + + - name: Extract OpenVINO packages + run: | + pushd ${INSTALL_DIR} + tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} + popd + + pushd ${INSTALL_TEST_DIR} + tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} + popd + + - name: Fetch setup_python action + uses: actions/checkout@v4 + with: + sparse-checkout: | + .github/actions/setup_python/action.yml + sparse-checkout-cone-mode: false + path: 'openvino' + + - name: Install dependencies + if: ${{ runner.os == 'Linux' }} + run: | + # install git (required to build pip deps from the sources) + sudo apt-get install --assume-yes --no-install-recommends g++ git ca-certificates + + - name: Setup Python 3.11 + uses: ./openvino/.github/actions/setup_python + with: + version: '3.11' + should-setup-pip-paths: 'false' + self-hosted-runner: ${{ contains(inputs.runner, 'aks') }} + + - name: Install OpenVINO Python wheels + run: python3 -m pip install ${INSTALL_DIR}/tools/openvino-* + + - name: Install TF Hub tests requirements + run: python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/tf_hub_tests/requirements.txt + + - name: TensorFlow Hub Tests - TF FE + run: | + export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH + python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/tf_hub_tests/ -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-tf_hub_tf_fe.html --self-contained-html -v + env: + TYPE: ${{ inputs.event == 'schedule' && 'nightly' || 'precommit'}} + TEST_DEVICE: CPU + + - name: Upload Test Results + uses: actions/upload-artifact@v3 + if: ${{ !cancelled() }} + with: + name: test-results-tensorflow-hub-models + path: | + ${{ env.INSTALL_TEST_DIR }}/TEST*.html + if-no-files-found: 'error' diff --git a/.github/workflows/job_tensorflow_hub_performance_models_tests.yml b/.github/workflows/job_tensorflow_hub_performance_models_tests.yml new file mode 100644 index 00000000000000..6e00f25e0c0ebb --- /dev/null +++ b/.github/workflows/job_tensorflow_hub_performance_models_tests.yml @@ -0,0 +1,116 @@ +name: TensorFlow Hub Performance Models tests + +on: + workflow_call: + inputs: + runner: + description: 'Machine on which the tests would run' + type: string + required: true + container: + description: 'JSON to be converted to the value of the "container" configuration for the job' + type: string + required: false + default: '{"image": null}' + event: + description: 'Event that triggered the workflow. E.g., "schedule" for nightly runs' + type: string + required: true + +jobs: + TensorFlow_Hub_Performance_Models_Tests: + name: TensorFlow Hub Performance Models tests + timeout-minutes: ${{ inputs.event == 'schedule' && 400 || 25 }} + runs-on: ${{ inputs.runner }} + container: ${{ fromJSON(inputs.container) }} + defaults: + run: + shell: bash + env: + DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input + OPENVINO_REPO: ${{ github.workspace }}/openvino + INSTALL_DIR: ${{ github.workspace }}/install + INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests + MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests + steps: + + - name: Check sudo + if: ${{ runner.os == 'Linux' }} + run: if [ "$(id -u)" -eq 0 ]; then apt update && apt --assume-yes install sudo; fi + + - name: Download OpenVINO package + uses: actions/download-artifact@v3 + with: + name: openvino_package + path: ${{ env.INSTALL_DIR }} + + - name: Download OpenVINO tests package + uses: actions/download-artifact@v3 + with: + name: openvino_tests + path: ${{ env.INSTALL_TEST_DIR }} + + # Needed as ${{ github.workspace }} is not working correctly when using Docker + - name: Setup Variables + run: | + echo "OPENVINO_REPO=$GITHUB_WORKSPACE/openvino" >> "$GITHUB_ENV" + echo "INSTALL_DIR=$GITHUB_WORKSPACE/install" >> "$GITHUB_ENV" + echo "INSTALL_TEST_DIR=$GITHUB_WORKSPACE/install/tests" >> "$GITHUB_ENV" + echo "MODEL_HUB_TESTS_INSTALL_DIR=$GITHUB_WORKSPACE/install/tests/model_hub_tests" >> "$GITHUB_ENV" + + - name: Extract OpenVINO packages + run: | + pushd ${INSTALL_DIR} + tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} + popd + + pushd ${INSTALL_TEST_DIR} + tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} + popd + + - name: Fetch setup_python action + uses: actions/checkout@v4 + with: + sparse-checkout: | + .github/actions/setup_python/action.yml + sparse-checkout-cone-mode: false + path: 'openvino' + + - name: Install dependencies + if: ${{ runner.os == 'Linux' }} + run: | + # install git (required to build pip deps from the sources) + sudo apt-get install --assume-yes --no-install-recommends g++ git ca-certificates + + - name: Setup Python 3.11 + uses: ./openvino/.github/actions/setup_python + with: + version: '3.11' + should-setup-pip-paths: 'false' + self-hosted-runner: ${{ contains(inputs.runner, 'aks') }} + + - name: Install OpenVINO Python wheels + run: python3 -m pip install ${INSTALL_DIR}/tools/openvino-* + + - name: Install TF Hub tests requirements + run: python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/tf_hub_tests/requirements.txt + + - name: Install Hub Performance tests requirements + run: python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/performance_tests/requirements.txt + + - name: Performance Hub Tests + run: | + export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH + python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/performance_tests/ -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-tf_hub_performance.html --self-contained-html -v + env: + TYPE: ${{ inputs.event == 'schedule' && 'nightly' || 'precommit'}} + TEST_DEVICE: CPU + + - name: Upload Test Results + uses: actions/upload-artifact@v3 + if: ${{ !cancelled() }} + with: + name: test-results-tensorflow-hub-performance-models + path: | + ${{ env.INSTALL_TEST_DIR }}/TEST*.html + if-no-files-found: 'error' diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 900d29a34d7fd7..5cdc7804c7b06e 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -261,164 +261,22 @@ jobs: Debian_Packages: name: Debian Packages needs: Build - timeout-minutes: 5 - defaults: - run: - shell: bash - runs-on: ubuntu-20.04 - container: - image: ubuntu:20.04 - env: - DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input - DEBIAN_PACKAGES_DIR: /__w/openvino/packages/ - - steps: - - name: Download OpenVINO debian packages - uses: actions/download-artifact@v3 - with: - name: openvino_debian_packages - path: ${{ env.DEBIAN_PACKAGES_DIR }} - - - name: Install debian packages & check conflicts - run: | - apt-get update -y - # Install debian packages from previous release - apt-get install --no-install-recommends -y gnupg wget ca-certificates - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - echo "deb https://apt.repos.intel.com/openvino/2023 ubuntu20 main" | tee /etc/apt/sources.list.d/intel-openvino-2023.list - apt-get update -y - apt-get install -y openvino - # install our local one and make sure the conflicts are resolved - apt-get install --no-install-recommends -y dpkg-dev - dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz - echo "deb [trusted=yes] file:${DEBIAN_PACKAGES_DIR} ./" | tee /etc/apt/sources.list.d/openvino-local.list - apt-get update -y - apt-get install openvino -y - working-directory: ${{ env.DEBIAN_PACKAGES_DIR }} - - - name: Test debian packages - run: | - /usr/share/openvino/samples/cpp/build_samples.sh - /usr/share/openvino/samples/c/build_samples.sh - ~/openvino_cpp_samples_build/intel64/Release/hello_query_device - python3 /usr/share/openvino/samples/python/hello_query_device/hello_query_device.py - python3 -c 'from openvino import Core; Core().get_property("CPU", "AVAILABLE_DEVICES")' - python3 -c 'from openvino import Core; Core().get_property("GPU", "AVAILABLE_DEVICES")' - python3 -c 'from openvino import Core; Core().get_property("AUTO", "SUPPORTED_METRICS")' - python3 -c 'from openvino import Core; Core().get_property("MULTI", "SUPPORTED_METRICS")' - python3 -c 'from openvino import Core; Core().get_property("HETERO", "SUPPORTED_METRICS")' - python3 -c 'from openvino import Core; Core().get_property("BATCH", "SUPPORTED_METRICS")' - python3 -c 'from openvino.frontend import FrontEndManager; assert len(FrontEndManager().get_available_front_ends()) == 6' - benchmark_app --help - ovc --help + uses: ./.github/workflows/job_debian_packages.yml + with: + runner: 'aks-linux-4-cores-16gb' + image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' Samples: - needs: [Build, Smart_CI] - timeout-minutes: 20 - defaults: - run: - shell: bash - runs-on: ubuntu-20.04 - container: - image: ubuntu:20.04 - env: - DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input - INSTALL_DIR: /__w/openvino/openvino/install - INSTALL_TEST_DIR: /__w/openvino/openvino/install/tests - BUILD_DIR: /__w/openvino/openvino/build + needs: [ Build, Smart_CI ] if: fromJSON(needs.smart_ci.outputs.affected_components).samples - - steps: - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package - path: ${{ env.INSTALL_DIR }} - - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} - popd - pushd ${INSTALL_TEST_DIR} - tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} - popd - - - name: Install OpenVINO dependencies - run: ${INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -y - - - name: Fetch setup_python action - uses: actions/checkout@v4 - with: - sparse-checkout: | - .github/actions/setup_python/action.yml - sparse-checkout-cone-mode: false - path: 'openvino' - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: ./openvino/.github/actions/setup_python - with: - version: ${{ env.PYTHON_VERSION }} - should-setup-pip-paths: 'false' - self-hosted-runner: 'false' - - - name: Build cpp samples - GCC - run: ${INSTALL_DIR}/samples/cpp/build_samples.sh -i ${INSTALL_DIR} -b ${BUILD_DIR}/cpp_samples - env: - CMAKE_COMPILE_WARNING_AS_ERROR: 'ON' - - - name: Build cpp samples - Clang - run: | - apt-get install -y clang - ${INSTALL_DIR}/samples/cpp/build_samples.sh -i ${INSTALL_DIR} -b ${BUILD_DIR}/cpp_samples_clang - env: - CMAKE_COMPILE_WARNING_AS_ERROR: 'ON' - CC: clang - CXX: clang++ - - - name: Build c samples - run: ${INSTALL_DIR}/samples/c/build_samples.sh -i ${INSTALL_DIR} -b ${BUILD_DIR}/c_samples - env: - CMAKE_COMPILE_WARNING_AS_ERROR: 'ON' - - # - # Tests - # - - - name: Samples tests - if: fromJSON(needs.smart_ci.outputs.affected_components).samples.test - run: | - export WORKSPACE=${INSTALL_DIR} - export IE_APP_PATH=${INSTALL_DIR}/samples_bin - export IE_APP_PYTHON_PATH=${INSTALL_DIR}/samples/python - export SHARE=${INSTALL_TEST_DIR}/smoke_tests/samples_smoke_tests_data - - python3 -m pip install --ignore-installed PyYAML -r ${INSTALL_TEST_DIR}/smoke_tests/requirements.txt - export LD_LIBRARY_PATH=${IE_APP_PATH}:$LD_LIBRARY_PATH - - source ${INSTALL_DIR}/setupvars.sh - - python3 -m pytest -sv ${INSTALL_TEST_DIR}/smoke_tests \ - --env_conf ${INSTALL_TEST_DIR}/smoke_tests/env_config.yml \ - --junitxml=${INSTALL_TEST_DIR}/TEST-SamplesSmokeTests.xml - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ !cancelled() }} - with: - name: test-results-samples - path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml - if-no-files-found: 'warn' + uses: ./.github/workflows/job_samples_tests.yml + with: + runner: 'aks-linux-4-cores-16gb' + image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} Conformance: - needs: [Build, Smart_CI] + needs: [ Build, Smart_CI ] timeout-minutes: ${{ matrix.TEST_TYPE == 'API' && 5 || 15 }} defaults: run: @@ -522,1027 +380,88 @@ jobs: ONNX_Runtime: name: ONNX Runtime Integration - needs: [Build, Smart_CI] - timeout-minutes: 20 - defaults: - run: - shell: bash - runs-on: aks-linux-16-cores-32gb - container: - image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 - volumes: - - /mount/caches:/mount/caches - options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING - env: - DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input - CMAKE_GENERATOR: 'Ninja Multi-Config' - CMAKE_CXX_COMPILER_LAUNCHER: sccache - CMAKE_C_COMPILER_LAUNCHER: sccache - OPENVINO_REPO: /__w/openvino/openvino/openvino - INSTALL_DIR: /__w/openvino/openvino/install - SCCACHE_AZURE_KEY_PREFIX: ubuntu20_x86_64_onnxruntime - ONNX_RUNTIME_REPO: /__w/openvino/openvino/onnxruntime - ONNX_RUNTIME_UTILS: /__w/openvino/openvino/install/onnxruntime - ONNX_RUNTIME_BUILD_DIR: /__w/openvino/openvino/onnxruntime/build if: fromJSON(needs.smart_ci.outputs.affected_components).ONNX_RT - - steps: - - name: Fetch install_build_dependencies.sh and setup_python action - uses: actions/checkout@v4 - with: - sparse-checkout: | - install_build_dependencies.sh - .github/actions/setup_python/action.yml - sparse-checkout-cone-mode: false - path: ${{ env.OPENVINO_REPO }} - - - name: Install git - run: | - apt-get update - apt-get install --assume-yes --no-install-recommends git ca-certificates - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: ./openvino/.github/actions/setup_python - with: - version: ${{ env.PYTHON_VERSION }} - should-setup-pip-paths: 'false' - - # - # Initialize OpenVINO - # - - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package - path: ${{ env.INSTALL_DIR }} - - - name: Extract OpenVINO package - run: | - pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} - popd - - - name: Install OpenVINO dependencies - run: ${INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -y - - - name: Clone ONNX Runtime - run: | - branch=`tr -s '\n ' < ${ONNX_RUNTIME_UTILS}/version` - git clone --branch $branch --single-branch --recursive https://github.com/microsoft/onnxruntime.git ${ONNX_RUNTIME_REPO} - - # - # Tests - # - - - name: Install Build Dependencies - run: bash ${OPENVINO_REPO}/install_build_dependencies.sh - - - name: Install sccache - uses: mozilla-actions/sccache-action@v0.0.3 - with: - version: "v0.5.4" - - - name: Build Lin ONNX Runtime - run: | - source ${INSTALL_DIR}/setupvars.sh - - ${ONNX_RUNTIME_REPO}/build.sh \ - --config RelWithDebInfo \ - --use_openvino CPU_FP32 \ - --build_shared_lib \ - --parallel \ - --skip_tests \ - --compile_no_warning_as_error \ - --build_dir ${ONNX_RUNTIME_BUILD_DIR} - env: - CXXFLAGS: "-Wno-error=deprecated-declarations" - - - name: Show sccache stats - run: ${SCCACHE_PATH} --show-stats - - - name: Run onnxruntime_test_all - run: | - source ${INSTALL_DIR}/setupvars.sh - skip_tests=$(tr -s '\n ' ':' < ${ONNX_RUNTIME_UTILS}/skip_tests) - - ./onnxruntime_test_all --gtest_filter=-$skip_tests - working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo - - - name: Run onnxruntime_shared_lib_test - run: | - source ${INSTALL_DIR}/setupvars.sh - ./onnxruntime_shared_lib_test --gtest_filter=-CApiTest.test_custom_op_openvino_wrapper_library - working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo - - - name: Run onnxruntime_global_thread_pools_test - run: | - source ${INSTALL_DIR}/setupvars.sh - ./onnxruntime_global_thread_pools_test - working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo - - - name: Run onnxruntime_api_tests_without_env - run: | - source ${INSTALL_DIR}/setupvars.sh - ./onnxruntime_api_tests_without_env - working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo - - - name: Run pytorch-converted tests - run: | - source ${INSTALL_DIR}/setupvars.sh - ./onnx_test_runner "${ONNX_RUNTIME_REPO}/cmake/external/onnx/onnx/backend/test/data/pytorch-converted" - working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo - - - name: Run pytorch-operator tests - run: | - source ${INSTALL_DIR}/setupvars.sh - ./onnx_test_runner "${ONNX_RUNTIME_REPO}/cmake/external/onnx/onnx/backend/test/data/pytorch-operator" - working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_onnx_runtime.yml + with: + runner: 'aks-linux-16-cores-32gb' + container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04", "volumes": ["/mount/caches:/mount/caches"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}' + sccache-azure-key-prefix: 'ubuntu20_x86_64_onnxruntime' CXX_Unit_Tests: name: C++ unit tests - needs: [Build, Smart_CI] - timeout-minutes: 20 - defaults: - run: - shell: bash - runs-on: aks-linux-4-cores-16gb - container: - image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 - env: - INSTALL_DIR: /__w/openvino/openvino/install - INSTALL_TEST_DIR: /__w/openvino/openvino/install/tests - - steps: - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package - path: ${{ env.INSTALL_DIR }} - - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} - popd - pushd ${INSTALL_TEST_DIR} - tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} - popd - - - name: Install OpenVINO dependencies - run: ${INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh -c=core -c=gpu -y - - # - # Tests - # - - - name: OpenVINO Core Unit Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).Core.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_core_unit_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OVCoreUT.xml - - - name: OpenVINO Inference Functional Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).inference.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_inference_functional_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-InferenceFunc.xml - - - name: OpenVINO Inference Unit Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).inference.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_inference_unit_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-InferenceUnit.xml - - - name: Low Precision Transformations Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).LP_transformations.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_lp_transformations_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-LpTransformations.xml - - - name: OpenVINO Conditional compilation tests - if: fromJSON(needs.smart_ci.outputs.affected_components).Core.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_conditional_compilation_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ConditionalCompilation.xml - - - name: IR frontend tests - if: fromJSON(needs.smart_ci.outputs.affected_components).IR_FE.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_ir_frontend_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-IRFrontend.xml - - - name: PaddlePaddle frontend tests - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/paddle_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-PaddleTests.xml - - - name: ONNX frontend tests - if: fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_onnx_frontend_tests --gtest_print_time=1 \ - --gtest_filter=-*IE_GPU* \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ONNXFrontend.xml - - - name: TensorFlow Common frontend tests - if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test || - fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_tensorflow_common_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TensorFlowCommonFrontend.xml - - - name: TensorFlow frontend tests - if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_tensorflow_frontend_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TensorFlowFrontend.xml - - - name: TensorFlow Lite frontend tests - if: fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_tensorflow_lite_frontend_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TensorFlowLiteFrontend.xml - - - name: Transformations func tests - if: fromJSON(needs.smart_ci.outputs.affected_components).transformations.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_transformations_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-Transformations.xml - - - name: Legacy Transformations func tests - if: fromJSON(needs.smart_ci.outputs.affected_components).GNA.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_legacy_transformations_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-LegacyTransformations.xml - - - name: Inference Engine 1.0 unit tests - if: fromJSON(needs.smart_ci.outputs.affected_components).GNA.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/InferenceEngineUnitTests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-InferenceEngineUnitTests.xml + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_cxx_unit_tests.yml + with: + runner: 'aks-linux-4-cores-16gb' + image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} - - name: Common test utils tests - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_util_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-CommonUtilTests.xml - - - name: Snippets func tests - if: fromJSON(needs.smart_ci.outputs.affected_components).CPU.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_snippets_func_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-SnippetsFuncTests.xml - - - name: CPU plugin unit tests - if: fromJSON(needs.smart_ci.outputs.affected_components).CPU.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_cpu_unit_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-CPUUnitTests.xml - - - name: ov_subgraphs_dumper_tests tests - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_subgraphs_dumper_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_subgraphs_dumper_tests.xml - - - name: Template OpImpl tests - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_op_conformance_tests --gtest_print_time=1 --device=TEMPLATE --gtest_filter=*OpImpl*\ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OpImplTests.xml - - - name: AUTO unit tests - if: fromJSON(needs.smart_ci.outputs.affected_components).AUTO.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_auto_unit_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_unit_tests.xml - - - name: AUTO func Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).AUTO.test - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_auto_func_tests --gtest_print_time=1 \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_func_tests.xml - - - name: Template plugin func tests - if: fromJSON(needs.smart_ci.outputs.affected_components).TEMPLATE.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_template_func_tests --gtest_print_time=1 \ - --gtest_filter=*smoke* \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TemplateFuncTests.xml - - - name: Inference Engine C API tests - if: fromJSON(needs.smart_ci.outputs.affected_components).C_API.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/InferenceEngineCAPITests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-InferenceEngineCAPITests.xml - - - name: OpenVINO C API tests - if: fromJSON(needs.smart_ci.outputs.affected_components).C_API.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_capi_test --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OpenVINOCAPITests.xml - - - name: AutoBatch unit tests - if: fromJSON(needs.smart_ci.outputs.affected_components).AUTO_BATCH.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_auto_batch_unit_tests --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_batch_unit_tests.xml - - - name: AutoBatch func tests - if: fromJSON(needs.smart_ci.outputs.affected_components).AUTO_BATCH.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_auto_batch_func_tests --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_batch_func_tests.xml - - - name: Proxy Plugin func tests - if: fromJSON(needs.smart_ci.outputs.affected_components).PROXY.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OVProxyTests.xml - - - name: Hetero unit tests - if: fromJSON(needs.smart_ci.outputs.affected_components).HETERO.test - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVHeteroUnitTests.xml - - - name: Hetero func tests - if: fromJSON(needs.smart_ci.outputs.affected_components).HETERO.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_hetero_func_tests --gtest_print_time=1 --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OVHeteroFuncTests.xml - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ !cancelled() }} - with: - name: test-results-cpp - path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml - if-no-files-found: 'warn' - - Python_Unit_Tests: - name: Python unit tests - needs: [Build, Smart_CI] - timeout-minutes: 40 - defaults: - run: - shell: bash - runs-on: aks-linux-4-cores-16gb - container: - image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 - volumes: - - /mount/caches:/mount/caches - env: - OPENVINO_REPO: /__w/openvino/openvino/openvino - INSTALL_DIR: /__w/openvino/openvino/install - INSTALL_TEST_DIR: /__w/openvino/openvino/install/tests - LAYER_TESTS_INSTALL_DIR: /__w/openvino/openvino/install/tests/layer_tests - - steps: - # - # Initialize OpenVINO - # - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package - path: ${{ env.INSTALL_DIR }} - - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} - popd - - pushd ${INSTALL_TEST_DIR} - tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} - popd - - - name: Fetch setup_python action - uses: actions/checkout@v4 - with: - sparse-checkout: | - .github/actions/setup_python/action.yml - sparse-checkout-cone-mode: false - path: ${{ env.OPENVINO_REPO }} - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: ./openvino/.github/actions/setup_python - with: - version: ${{ env.PYTHON_VERSION }} - pip-cache-path: ${{ env.PIP_CACHE_PATH }} - should-setup-pip-paths: 'true' - - - name: Install OpenVINO dependencies - run: ${INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh -c=core -y - - - name: Install OpenVINO Python wheels - run: | - # Install the core OV wheel - python3 -m pip install ${INSTALL_DIR}/tools/openvino-*.whl - - # Find and install OV dev wheel - pushd ${INSTALL_DIR}/tools - ov_dev_wheel_name=$(find . -name 'openvino_dev*.whl') - python3 -m pip install $ov_dev_wheel_name[mxnet,caffe,kaldi,onnx,tensorflow2,pytorch] - popd - - - name: Install Python API tests dependencies - run: | - # To enable pytest parallel features - python3 -m pip install pytest-xdist[psutil] - # For torchvision to OpenVINO preprocessing converter - python3 -m pip install -r ${INSTALL_TEST_DIR}/python/preprocess/torchvision/requirements.txt - - # TODO: replace with Python API tests requirements - python3 -m pip install -r ${INSTALL_TEST_DIR}/mo/requirements_dev.txt - - # - # Tests - # - - - name: Python API 1.0 Tests - #if: fromJSON(needs.smart_ci.outputs.affected_components).Python_API.test - run: | - python3 -m pytest -s ${INSTALL_TEST_DIR}/pyngraph \ - --junitxml=${INSTALL_TEST_DIR}/TEST-Pyngraph.xml \ - --ignore=${INSTALL_TEST_DIR}/pyngraph/tests_compatibility/test_onnx/test_zoo_models.py \ - --ignore=${INSTALL_TEST_DIR}/pyngraph/tests_compatibility/test_onnx/test_backend.py - - - name: Python API 2.0 Tests - #if: fromJSON(needs.smart_ci.outputs.affected_components).Python_API.test - run: | - # for 'template' extension - export LD_LIBRARY_PATH=${INSTALL_TEST_DIR}:$LD_LIBRARY_PATH - python3 -m pytest -sv ${INSTALL_TEST_DIR}/pyopenvino \ - --junitxml=${INSTALL_TEST_DIR}/TEST-Pyngraph.xml \ - --ignore=${INSTALL_TEST_DIR}/pyopenvino/tests/test_utils/test_utils.py - - - name: Model Optimizer unit tests - if: fromJSON(needs.smart_ci.outputs.affected_components).MO.test - run: | - # required for MxNet - apt-get install -y libgomp1 libquadmath0 - - python3 -m pytest -s ${INSTALL_TEST_DIR}/mo/unit_tests \ - --junitxml=${INSTALL_TEST_DIR}/TEST-ModelOptimizer.xml - - - name: Python ONNX operators tests - if: fromJSON(needs.smart_ci.outputs.affected_components).Python_API.test || - fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE.test - run: | - # Skip test_onnx/test_zoo_models and test_onnx/test_backend due to long execution time - ONNX Model Zoo tests are run separately - python3 -m pytest -sv ${INSTALL_TEST_DIR}/onnx -k 'not cuda' \ - --junitxml=${INSTALL_TEST_DIR}/TEST-onnx_frontend.xml \ - --ignore=${INSTALL_TEST_DIR}/onnx/test_python/test_zoo_models.py - - - name: OVC unit tests - if: fromJSON(needs.smart_ci.outputs.affected_components).MO.test - run: python3 -m pytest -s ${INSTALL_TEST_DIR}/ovc/unit_tests --junitxml=${INSTALL_TEST_DIR}/TEST-OpenVinoConversion.xml - - - name: Install Python Layer tests dependencies - run: | - # layer test requirements - python3 -m pip install -r ${LAYER_TESTS_INSTALL_DIR}/requirements.txt - - - name: MO Python API Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).MO.test - run: | - # Import 'test_utils' installed in '/tests/python/openvino' - export LD_LIBRARY_PATH=${PIP_INSTALL_PATH}/openvino/libs:$LD_LIBRARY_PATH - export PYTHONPATH=${INSTALL_TEST_DIR}/python - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/mo_python_api_tests --junitxml=${INSTALL_TEST_DIR}/TEST-test_mo_convert.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: OVC Python API Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).MO.test - run: | - # Import 'test_utils' installed in '/tests/python/openvino' - export PYTHONPATH=${INSTALL_TEST_DIR}/python - export LD_LIBRARY_PATH=${PIP_INSTALL_PATH}/openvino/libs:$LD_LIBRARY_PATH - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/ovc_python_api_tests --junitxml=${INSTALL_TEST_DIR}/TEST-test_ovc_convert.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: Python Frontend tests - if: fromJSON(needs.smart_ci.outputs.affected_components).PyTorch_FE.test || - fromJSON(needs.smart_ci.outputs.affected_components).PDPD_FE.test - run: | - # to allow 'libtest_builtin_extensions.so' to find 'libopenvino_onnx_frontend.so' - export LD_LIBRARY_PATH=${PIP_INSTALL_PATH}/openvino/libs:$LD_LIBRARY_PATH - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/py_frontend_tests --junitxml=${INSTALL_TEST_DIR}/TEST-test_py_fontend.xml - - - name: PyTorch Layer Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).PyTorch_FE.test - run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/pytorch_tests -n logical -m precommit --junitxml=${INSTALL_TEST_DIR}/TEST-pytorch.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP32 - - - name: PyTorch torch.compile TORCHFX Layer Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).PyTorch_FE.test - run: | - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/pytorch_tests -m precommit_fx_backend --junitxml=${INSTALL_TEST_DIR}/TEST-pytorch.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP32 - PYTORCH_TRACING_MODE: TORCHFX - - - name: PyTorch torch.compile TORCHSCRIPT Layer Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).PyTorch_FE.test - run: | - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/pytorch_tests -m precommit_ts_backend --junitxml=${INSTALL_TEST_DIR}/TEST-pytorch.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP32 - PYTORCH_TRACING_MODE: TORCHSCRIPT - - - name: ONNX Layer Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE.test - run: | - # requires 'unit_tests' from 'tools/mo' - export PYTHONPATH=${INSTALL_TEST_DIR}/mo:$PYTHONPATH - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/onnx_tests -m "not launch_only_if_manually_specified and precommit" --junitxml=${INSTALL_TEST_DIR}/TEST-onnx.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: TensorFlow 1 Layer Tests - TF FE - if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test - run: | - # requires 'unit_tests' from 'mo' - export PYTHONPATH=${INSTALL_TEST_DIR}/mo - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow_tests/ --use_new_frontend -m precommit_tf_fe --junitxml=${INSTALL_TEST_DIR}/TEST-tf_fe.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: TensorFlow 2 Layer Tests - TF FE - if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test - run: | - # requires 'unit_tests' from 'mo' - export PYTHONPATH=${INSTALL_TEST_DIR}/mo - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow2_keras_tests/ --use_new_frontend -m precommit_tf_fe --junitxml=${INSTALL_TEST_DIR}/TEST-tf2_fe.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: JAX Layer Tests - TF FE - if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test - run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/jax_tests/ -m precommit --junitxml=${INSTALL_TEST_DIR}/TEST-jax.xml - env: - TEST_DEVICE: CPU - - - name: TensorFlow 1 Layer Tests - Legacy FE - if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test - run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow_tests/test_tf_Roll.py --ir_version=10 --junitxml=${INSTALL_TEST_DIR}/TEST-tf_Roll.xml - - - name: TensorFlow 2 Layer Tests - Legacy FE - if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test - run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow2_keras_tests/test_tf2_keras_activation.py --ir_version=11 -k "sigmoid" --junitxml=${INSTALL_TEST_DIR}/TEST-tf2_Activation.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: TensorFlow Lite Layer Tests - TFL FE - if: fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test - run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow_lite_tests/ --junitxml=${INSTALL_TEST_DIR}/TEST-tfl_fe.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: Clone API snippets - uses: actions/checkout@v4 - with: - sparse-checkout: openvino/docs/snippets - path: ${{ env.OPENVINO_REPO }} - submodules: 'false' - - - name: Docs Python snippets - run: | - # to find 'snippets' module in docs - export PYTHONPATH=${OPENVINO_REPO}/docs - # for 'template' extension - export LD_LIBRARY_PATH=${INSTALL_TEST_DIR}:$LD_LIBRARY_PATH - python3 ${OPENVINO_REPO}/docs/snippets/main.py - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ !cancelled() }} - with: - name: test-results-python - path: | - ${{ env.INSTALL_TEST_DIR }}/TEST*.html - ${{ env.INSTALL_TEST_DIR }}/TEST*.xml - if-no-files-found: 'warn' + Python_Unit_Tests: + name: Python unit tests + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_python_unit_tests.yml + with: + runner: 'aks-linux-4-cores-16gb' + container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04", "volumes": ["/mount/caches:/mount/caches"]}' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} CPU_Functional_Tests: name: CPU functional tests - needs: [Build, Smart_CI] - timeout-minutes: 25 - defaults: - run: - shell: bash - runs-on: aks-linux-8-cores-32gb - container: - image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 - env: - OPENVINO_REPO: /__w/openvino/openvino/openvino - INSTALL_DIR: /__w/openvino/openvino/install - INSTALL_TEST_DIR: /__w/openvino/openvino/install/tests - PARALLEL_TEST_SCRIPT: /__w/openvino/openvino/install/tests/functional_test_utils/layer_tests_summary/run_parallel.py - PARALLEL_TEST_CACHE: /__w/openvino/openvino/install/tests/test_cache.lst if: fromJSON(needs.smart_ci.outputs.affected_components).CPU.test - steps: - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package - path: ${{ env.INSTALL_DIR }} - - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} - popd - pushd ${INSTALL_TEST_DIR} - tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} - popd - - - name: Install OpenVINO dependencies - run: bash ${INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh -c=core -y - - - name: Fetch setup_python action - uses: actions/checkout@v4 - with: - sparse-checkout: | - .github/actions/setup_python/action.yml - sparse-checkout-cone-mode: false - path: ${{ env.OPENVINO_REPO }} - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: ./openvino/.github/actions/setup_python - with: - version: ${{ env.PYTHON_VERSION }} - should-setup-pip-paths: 'false' - - - name: Install python dependencies for run_parallel.py - run: python3 -m pip install -r ${INSTALL_TEST_DIR}/functional_test_utils/layer_tests_summary/requirements.txt - - - name: Restore tests execution time - uses: actions/cache/restore@v3 - with: - path: ${{ env.PARALLEL_TEST_CACHE }} - key: ${{ runner.os }}-tests-functional-cpu-stamp-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-tests-functional-cpu-stamp - - - name: Intel CPU plugin func tests (parallel) - run: | - source ${INSTALL_DIR}/setupvars.sh - python3 ${PARALLEL_TEST_SCRIPT} -e ${INSTALL_TEST_DIR}/ov_cpu_func_tests -c ${PARALLEL_TEST_CACHE} -w ${INSTALL_TEST_DIR} -s suite -rf 0 -- --gtest_print_time=1 --gtest_filter=*smoke* - timeout-minutes: 20 - - - name: Save tests execution time - uses: actions/cache/save@v3 - if: github.ref_name == 'master' - with: - path: ${{ env.PARALLEL_TEST_CACHE }} - key: ${{ runner.os }}-tests-functional-cpu-stamp-${{ github.sha }} - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ !cancelled() }} - with: - name: test-results-functional-cpu - path: | - ${{ env.INSTALL_TEST_DIR }}/temp/*.log - ${{ env.INSTALL_TEST_DIR }}/logs/*.log - ${{ env.INSTALL_TEST_DIR }}/logs/failed/*.log - ${{ env.INSTALL_TEST_DIR }}/logs/crashed/*.log - ${{ env.INSTALL_TEST_DIR }}/logs/hanged/*.log - ${{ env.INSTALL_TEST_DIR }}/logs/interapted/*.log - ${{ env.INSTALL_TEST_DIR }}/logs/hash_table.csv - ${{ env.PARALLEL_TEST_CACHE }} - if-no-files-found: 'error' - + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_cpu_functional_tests.yml + with: + runner: 'aks-linux-8-cores-32gb' + image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' + + # TODO: Switch back to self-hosted runners + # container: + # image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 + # volumes: + # - /mount/caches:/mount/caches TensorFlow_Hub_Models_Tests: name: TensorFlow Hub Models tests - needs: [Build, Smart_CI] - defaults: - run: - shell: bash - runs-on: ${{ github.event_name == 'schedule' && 'ubuntu-20.04-16-cores' || 'ubuntu-20.04-8-cores'}} - timeout-minutes: ${{ github.event_name == 'schedule' && 400 || 25 }} - # TODO: Switch back to self-hosted runners - # container: - # image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 - # volumes: - # - /mount/caches:/mount/caches - env: - OPENVINO_REPO: ${{ github.workspace }}/openvino - INSTALL_DIR: ${{ github.workspace }}/install - INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests - MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test || fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test - - steps: - - name: Check sudo - run: if [ "$(id -u)" -eq 0 ]; then apt update && apt --assume-yes install sudo; fi - - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package - path: ${{ env.INSTALL_DIR }} - - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} - popd - - pushd ${INSTALL_TEST_DIR} - tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} - popd - - - name: Fetch setup_python action - uses: actions/checkout@v4 - with: - sparse-checkout: | - .github/actions/setup_python/action.yml - sparse-checkout-cone-mode: false - path: 'openvino' - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: ./openvino/.github/actions/setup_python - with: - version: ${{ env.PYTHON_VERSION }} - should-setup-pip-paths: 'false' - self-hosted-runner: 'false' - - - name: Install OpenVINO Python wheels - run: python3 -m pip install ${INSTALL_DIR}/tools/openvino-* - - - name: Install TF Hub tests requirements - run: | - python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/tf_hub_tests/requirements.txt - - - name: TensorFlow Hub Tests - TF FE - run: | - export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH - python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/tf_hub_tests/ -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-tf_hub_tf_fe.html --self-contained-html -v - env: - TYPE: ${{ github.event_name == 'schedule' && 'nightly' || 'precommit'}} - TEST_DEVICE: CPU - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ !cancelled() }} - with: - name: test-results-tensorflow-hub-models - path: | - ${{ env.INSTALL_TEST_DIR }}/TEST*.html - if-no-files-found: 'error' - + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_tensorflow_hub_models_tests.yml + with: + runner: ${{ github.event_name == 'schedule' && 'ubuntu-20.04-16-cores' || 'ubuntu-20.04-8-cores' }} + event: ${{ github.event_name }} + + # TODO: Switch back to self-hosted runners + # container: + # image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 + # volumes: + # - /mount/caches:/mount/caches TensorFlow_Hub_Performance_Models_Tests: name: TensorFlow Hub Performance Models tests - needs: [Build, Smart_CI] - defaults: - run: - shell: bash - runs-on: ${{ github.event_name == 'schedule' && 'ubuntu-20.04-16-cores' || 'ubuntu-20.04-8-cores'}} - timeout-minutes: ${{ github.event_name == 'schedule' && 400 || 25 }} - env: - OPENVINO_REPO: ${{ github.workspace }}/openvino - INSTALL_DIR: ${{ github.workspace }}/install - INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests - MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test || fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test - - steps: - - name: Check sudo - run: if [ "$(id -u)" -eq 0 ]; then apt update && apt --assume-yes install sudo; fi - - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package - path: ${{ env.INSTALL_DIR }} - - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} - popd - - pushd ${INSTALL_TEST_DIR} - tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} - popd - - - name: Fetch setup_python action - uses: actions/checkout@v4 - with: - sparse-checkout: | - .github/actions/setup_python/action.yml - sparse-checkout-cone-mode: false - path: 'openvino' - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: ./openvino/.github/actions/setup_python - with: - version: ${{ env.PYTHON_VERSION }} - should-setup-pip-paths: 'false' - self-hosted-runner: 'false' - - - name: Install OpenVINO Python wheels - run: python3 -m pip install ${INSTALL_DIR}/tools/openvino-* - - - name: Install TF Hub tests requirements - run: | - python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/tf_hub_tests/requirements.txt - - - name: Install Hub Performance tests requirements - run: | - python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/performance_tests/requirements.txt - - - name: Performance Hub Tests - run: | - export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH - python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/performance_tests/ -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-tf_hub_performance.html --self-contained-html -v - env: - TYPE: ${{ github.event_name == 'schedule' && 'nightly' || 'precommit'}} - TEST_DEVICE: CPU - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ !cancelled() }} - with: - name: test-results-tensorflow-hub-performance-models - path: | - ${{ env.INSTALL_TEST_DIR }}/TEST*.html - if-no-files-found: 'error' - + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_tensorflow_hub_performance_models_tests.yml + with: + runner: ${{ github.event_name == 'schedule' && 'ubuntu-20.04-16-cores' || 'ubuntu-20.04-8-cores' }} + event: ${{ github.event_name }} + + # TODO: Switch back to self-hosted runners + # container: + # image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 + # volumes: + # - /mount/caches:/mount/caches PyTorch_Models_Tests: name: PyTorch Models tests - needs: [Build, Smart_CI] - timeout-minutes: ${{ github.event_name == 'schedule' && 400 || 30 }} - defaults: - run: - shell: bash - runs-on: ${{ github.event_name == 'schedule' && 'ubuntu-20.04-16-cores' || 'ubuntu-20.04-8-cores'}} - # TODO: Switch back to self-hosted runners - # container: - # image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 - # volumes: - # - /mount/caches:/mount/caches - env: - OPENVINO_REPO: ${{ github.workspace }}/openvino - INSTALL_DIR: ${{ github.workspace }}/install - INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests - MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests if: fromJSON(needs.smart_ci.outputs.affected_components).PyTorch_FE.test - - steps: - - name: Check sudo - run: if [ "$(id -u)" -eq 0 ]; then apt update && apt --assume-yes install sudo; fi - - - name: Install dependencies - run: | - # install git (required to build pip deps from the sources) - # install 'g++' to build 'detectron2' and 'natten' wheels - sudo apt-get install --assume-yes --no-install-recommends g++ git ca-certificates - - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package - path: ${{ env.INSTALL_DIR }} - - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} - popd - pushd ${INSTALL_TEST_DIR} - tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} - popd - - - name: Fetch setup_python action - uses: actions/checkout@v4 - with: - sparse-checkout: | - .github/actions/setup_python/action.yml - sparse-checkout-cone-mode: false - path: 'openvino' - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: ./openvino/.github/actions/setup_python - with: - version: ${{ env.PYTHON_VERSION }} - should-setup-pip-paths: 'false' - self-hosted-runner: 'false' - - - name: Install OpenVINO Python wheels - run: python3 -m pip install ${INSTALL_DIR}/tools/openvino-* - - - name: Install PyTorch tests requirements - run: | - python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/requirements.txt - python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/requirements_secondary.txt - echo "Available storage:" - df -h - env: - CPLUS_INCLUDE_PATH: ${{ env.Python_ROOT_DIR }}/include/python${{ env.PYTHON_VERSION }} - - - name: PyTorch Models Tests - run: | - export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH - python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_model_tests.html --self-contained-html -v - env: - TYPE: ${{ github.event_name == 'schedule' && 'nightly' || 'precommit'}} - TEST_DEVICE: CPU - USE_SYSTEM_CACHE: False - - - name: Available storage after tests - run: | - echo "Available storage:" - df -h - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ !cancelled() }} - with: - name: test-results-torch-models - path: | - ${{ env.INSTALL_TEST_DIR }}/TEST*.html - if-no-files-found: 'error' + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_pytorch_models_tests.yml + with: + runner: ${{ github.event_name == 'schedule' && 'ubuntu-20.04-16-cores' || 'ubuntu-20.04-8-cores' }} + event: ${{ github.event_name }} NVIDIA_Plugin: name: NVIDIA plugin - needs: [Build, Smart_CI] + needs: [ Build, Smart_CI ] timeout-minutes: 15 defaults: run: @@ -1659,7 +578,7 @@ jobs: run: ${SCCACHE_PATH} --show-stats GPU_Stub: - needs: [Build, Smart_CI] + needs: [ Build, Smart_CI ] runs-on: ubuntu-latest if: fromJSON(needs.smart_ci.outputs.affected_components).GPU steps: diff --git a/.github/workflows/linux_arm64.yml b/.github/workflows/linux_arm64.yml index a611f59db5a55c..4c45648d1cdfc6 100644 --- a/.github/workflows/linux_arm64.yml +++ b/.github/workflows/linux_arm64.yml @@ -1,7 +1,7 @@ name: Linux ARM64 (Ubuntu 20.04, Python 3.11) on: workflow_dispatch: - # pull_request: +# pull_request: push: branches: - master @@ -258,1204 +258,92 @@ jobs: Debian_Packages: name: Debian Packages needs: Build - timeout-minutes: 10 - defaults: - run: - shell: bash - runs-on: 'aks-linux-16-cores-arm' - container: + uses: ./.github/workflows/job_debian_packages.yml + with: + runner: 'aks-linux-16-cores-arm' image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' - env: - DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input - DEBIAN_PACKAGES_DIR: /__w/openvino/packages/ - - steps: - - name: Download OpenVINO debian packages - uses: actions/download-artifact@v3 - with: - name: openvino_debian_packages - path: ${{ env.DEBIAN_PACKAGES_DIR }} - - - name: Install debian packages & check conflicts - run: | - apt-get update -y - # install our local one - apt-get install --no-install-recommends -y dpkg-dev - dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz - echo "deb [trusted=yes] file:${DEBIAN_PACKAGES_DIR} ./" | tee /etc/apt/sources.list.d/openvino-local.list - apt-get update -y - apt-get install openvino -y - working-directory: ${{ env.DEBIAN_PACKAGES_DIR }} - - - name: Test debian packages - run: | - /usr/share/openvino/samples/cpp/build_samples.sh - /usr/share/openvino/samples/c/build_samples.sh - - ~/openvino_cpp_samples_build/aarch64/Release/hello_query_device - - python3 /usr/share/openvino/samples/python/hello_query_device/hello_query_device.py - python3 -c 'from openvino import Core; Core().get_property("CPU", "AVAILABLE_DEVICES")' - python3 -c 'from openvino import Core; Core().get_property("AUTO", "SUPPORTED_METRICS")' - python3 -c 'from openvino import Core; Core().get_property("MULTI", "SUPPORTED_METRICS")' - python3 -c 'from openvino import Core; Core().get_property("HETERO", "SUPPORTED_METRICS")' - python3 -c 'from openvino import Core; Core().get_property("BATCH", "SUPPORTED_METRICS")' - python3 -c 'from openvino.frontend import FrontEndManager; assert len(FrontEndManager().get_available_front_ends()) == 6' - benchmark_app --help - ovc --help Samples: - needs: [Build, Smart_CI] - timeout-minutes: 20 - defaults: - run: - shell: bash - runs-on: 'aks-linux-16-cores-arm' - container: - image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' - env: - DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input - INSTALL_DIR: /__w/openvino/openvino/install - INSTALL_TEST_DIR: /__w/openvino/openvino/install/tests - BUILD_DIR: /__w/openvino/openvino/build + needs: [ Build, Smart_CI ] if: fromJSON(needs.smart_ci.outputs.affected_components).samples - - steps: - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package - path: ${{ env.INSTALL_DIR }} - - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} - popd - pushd ${INSTALL_TEST_DIR} - tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} - popd - - - name: Install OpenVINO dependencies - run: ${INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -y - - - name: Fetch setup_python action - uses: actions/checkout@v4 - with: - sparse-checkout: | - .github/actions/setup_python/action.yml - sparse-checkout-cone-mode: false - path: 'openvino' - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: ./openvino/.github/actions/setup_python - with: - version: ${{ env.PYTHON_VERSION }} - should-setup-pip-paths: 'false' - self-hosted-runner: 'true' - - - name: Build cpp samples - GCC - run: ${INSTALL_DIR}/samples/cpp/build_samples.sh -i ${INSTALL_DIR} -b ${BUILD_DIR}/cpp_samples - env: - CMAKE_COMPILE_WARNING_AS_ERROR: 'ON' - - - name: Build cpp samples - Clang - run: | - apt-get install -y clang - ${INSTALL_DIR}/samples/cpp/build_samples.sh -i ${INSTALL_DIR} -b ${BUILD_DIR}/cpp_samples_clang - env: - CMAKE_COMPILE_WARNING_AS_ERROR: 'ON' - CC: clang - CXX: clang++ - - - name: Build c samples - run: ${INSTALL_DIR}/samples/c/build_samples.sh -i ${INSTALL_DIR} -b ${BUILD_DIR}/c_samples - env: - CMAKE_COMPILE_WARNING_AS_ERROR: 'ON' - - # - # Tests - # - - - name: Samples tests - if: fromJSON(needs.smart_ci.outputs.affected_components).samples.test - run: | - export WORKSPACE=${INSTALL_DIR} - export IE_APP_PATH=${INSTALL_DIR}/samples_bin - export IE_APP_PYTHON_PATH=${INSTALL_DIR}/samples/python - export SHARE=${INSTALL_TEST_DIR}/smoke_tests/samples_smoke_tests_data - - python3 -m pip install --ignore-installed PyYAML -r ${INSTALL_TEST_DIR}/smoke_tests/requirements.txt - export LD_LIBRARY_PATH=${IE_APP_PATH}:$LD_LIBRARY_PATH - - source ${INSTALL_DIR}/setupvars.sh - - python3 -m pytest -sv ${INSTALL_TEST_DIR}/smoke_tests \ - --env_conf ${INSTALL_TEST_DIR}/smoke_tests/env_config.yml \ - --junitxml=${INSTALL_TEST_DIR}/TEST-SamplesSmokeTests.xml - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ !cancelled() }} - with: - name: test-results-samples - path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml - if-no-files-found: 'warn' + uses: ./.github/workflows/job_samples_tests.yml + with: + runner: 'aks-linux-16-cores-arm' + image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} ONNX_Runtime: name: ONNX Runtime Integration - needs: [Build, Smart_CI] - timeout-minutes: 30 - defaults: - run: - shell: bash - runs-on: 'aks-linux-16-cores-arm' - container: - image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 - volumes: - - /mount/caches:/mount/caches - options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING - env: - DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input - CMAKE_GENERATOR: 'Ninja Multi-Config' - CMAKE_CXX_COMPILER_LAUNCHER: sccache - CMAKE_C_COMPILER_LAUNCHER: sccache - OPENVINO_REPO: /__w/openvino/openvino/openvino - INSTALL_DIR: /__w/openvino/openvino/install - SCCACHE_AZURE_KEY_PREFIX: 'ubuntu20_aarch64_onnxruntime' - ONNX_RUNTIME_REPO: /__w/openvino/openvino/onnxruntime - ONNX_RUNTIME_UTILS: /__w/openvino/openvino/install/onnxruntime - ONNX_RUNTIME_BUILD_DIR: /__w/openvino/openvino/onnxruntime/build if: fromJSON(needs.smart_ci.outputs.affected_components).ONNX_RT - - steps: - - name: Fetch install_build_dependencies.sh and setup_python action - uses: actions/checkout@v4 - with: - sparse-checkout: | - install_build_dependencies.sh - .github/actions/setup_python/action.yml - sparse-checkout-cone-mode: false - path: ${{ env.OPENVINO_REPO }} - - - name: Install git - run: | - apt-get update - apt-get install --assume-yes --no-install-recommends git ca-certificates - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: ./openvino/.github/actions/setup_python - with: - version: ${{ env.PYTHON_VERSION }} - should-setup-pip-paths: 'false' - - # - # Initialize OpenVINO - # - - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package - path: ${{ env.INSTALL_DIR }} - - - name: Extract OpenVINO package - run: | - pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} - popd - - - name: Install OpenVINO dependencies - run: ${INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -y - - - name: Clone ONNX Runtime - run: | - branch=`tr -s '\n ' < ${ONNX_RUNTIME_UTILS}/version` - git clone --branch $branch --single-branch --recursive https://github.com/microsoft/onnxruntime.git ${ONNX_RUNTIME_REPO} - - # - # Tests - # - - - name: Install Build Dependencies - run: bash ${OPENVINO_REPO}/install_build_dependencies.sh - - - name: Install sccache - uses: mozilla-actions/sccache-action@v0.0.3 - with: - version: "v0.5.4" - - - name: Build Lin ONNX Runtime - run: | - source ${INSTALL_DIR}/setupvars.sh - - ${ONNX_RUNTIME_REPO}/build.sh \ - --config RelWithDebInfo \ - --use_openvino CPU_FP32 \ - --build_shared_lib \ - --parallel \ - --skip_tests \ - --compile_no_warning_as_error \ - --build_dir ${ONNX_RUNTIME_BUILD_DIR} - env: - CXXFLAGS: "-Wno-error=deprecated-declarations" - - - name: Show sccache stats - run: ${SCCACHE_PATH} --show-stats - - - name: Run onnxruntime_test_all - if: ${{ 'false' }} # Ticket: 126277 - run: | - source ${INSTALL_DIR}/setupvars.sh - skip_tests=$(tr -s '\n ' ':' < ${ONNX_RUNTIME_UTILS}/skip_tests) - - ./onnxruntime_test_all --gtest_filter=-$skip_tests - working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo - - - name: Run onnxruntime_shared_lib_test - run: | - source ${INSTALL_DIR}/setupvars.sh - ./onnxruntime_shared_lib_test --gtest_filter=-CApiTest.test_custom_op_openvino_wrapper_library - working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo - - - name: Run onnxruntime_global_thread_pools_test - run: | - source ${INSTALL_DIR}/setupvars.sh - ./onnxruntime_global_thread_pools_test - working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo - - - name: Run onnxruntime_api_tests_without_env - run: | - source ${INSTALL_DIR}/setupvars.sh - ./onnxruntime_api_tests_without_env - working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo - - - name: Run pytorch-converted tests - run: | - source ${INSTALL_DIR}/setupvars.sh - ./onnx_test_runner "${ONNX_RUNTIME_REPO}/cmake/external/onnx/onnx/backend/test/data/pytorch-converted" - working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo - - - name: Run pytorch-operator tests - run: | - source ${INSTALL_DIR}/setupvars.sh - ./onnx_test_runner "${ONNX_RUNTIME_REPO}/cmake/external/onnx/onnx/backend/test/data/pytorch-operator" - working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_onnx_runtime.yml + with: + runner: 'aks-linux-16-cores-arm' + container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04", "volumes": ["/mount/caches:/mount/caches"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}' + sccache-azure-key-prefix: 'ubuntu20_aarch64_onnxruntime' CXX_Unit_Tests: name: C++ unit tests - needs: [Build, Smart_CI] - timeout-minutes: 20 - defaults: - run: - shell: bash - runs-on: 'aks-linux-16-cores-arm' - container: - image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 - env: - INSTALL_DIR: /__w/openvino/openvino/install - INSTALL_TEST_DIR: /__w/openvino/openvino/install/tests - - steps: - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package - path: ${{ env.INSTALL_DIR }} - - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} - popd - pushd ${INSTALL_TEST_DIR} - tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} - popd - - - name: Install OpenVINO dependencies - run: ${INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh -c=core -c=gpu -y - - # - # Tests - # - - - name: OpenVINO Core Unit Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).Core.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_core_unit_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OVCoreUT.xml - - - name: OpenVINO Inference Functional Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).inference.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_inference_functional_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-InferenceFunc.xml - - - name: OpenVINO Inference Unit Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).inference.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_inference_unit_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-InferenceUnit.xml - - - name: Low Precision Transformations Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).LP_transformations.test - run: | - source ${INSTALL_DIR}/setupvars.sh - - # Skip filter ticket: 126279 - ${INSTALL_TEST_DIR}/ov_lp_transformations_tests --gtest_print_time=1 \ - --gtest_filter=-*smoke_LPT/FoldFakeQuantizeInTransformations.CompareFunctions* \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-LpTransformations.xml - - - name: OpenVINO Conditional compilation tests - if: fromJSON(needs.smart_ci.outputs.affected_components).Core.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_conditional_compilation_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ConditionalCompilation.xml - - - name: IR frontend tests - if: fromJSON(needs.smart_ci.outputs.affected_components).IR_FE.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_ir_frontend_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-IRFrontend.xml - - - name: PaddlePaddle frontend tests - if: ${{ 'false' }} - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/paddle_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-PaddleTests.xml - - - name: ONNX frontend tests - if: ${{ 'false' }} # Ticket: 126280 - #if: fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_onnx_frontend_tests --gtest_print_time=1 \ - --gtest_filter=-*IE_GPU* \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ONNXFrontend.xml - - - name: TensorFlow Common frontend tests - if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test || - fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_tensorflow_common_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TensorFlowCommonFrontend.xml - - - name: TensorFlow frontend tests - if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_tensorflow_frontend_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TensorFlowFrontend.xml - - - name: TensorFlow Lite frontend tests - if: fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_tensorflow_lite_frontend_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TensorFlowLiteFrontend.xml - - - name: Transformations func tests - if: ${{ 'false' }} # Ticket: 126281 - #if: fromJSON(needs.smart_ci.outputs.affected_components).transformations.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_transformations_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-Transformations.xml - - - name: Common test utils tests - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_util_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-CommonUtilTests.xml - - - name: Snippets func tests - #if: fromJSON(needs.smart_ci.outputs.affected_components).CPU.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_snippets_func_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-SnippetsFuncTests.xml - - - name: CPU plugin unit tests - if: fromJSON(needs.smart_ci.outputs.affected_components).CPU.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_cpu_unit_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-CPUUnitTests.xml - - - name: ov_subgraphs_dumper_tests tests - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_subgraphs_dumper_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_subgraphs_dumper_tests.xml - - - name: Template OpImpl tests - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_op_conformance_tests --gtest_print_time=1 --device=TEMPLATE --gtest_filter=*OpImpl*\ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OpImplTests.xml - - - name: AUTO unit tests - if: fromJSON(needs.smart_ci.outputs.affected_components).AUTO.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_auto_unit_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_unit_tests.xml - - - name: AUTO func Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).AUTO.test - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_auto_func_tests --gtest_print_time=1 \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_func_tests.xml - - - name: Template plugin func tests - if: fromJSON(needs.smart_ci.outputs.affected_components).TEMPLATE.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_template_func_tests --gtest_print_time=1 \ - --gtest_filter=*smoke* \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TemplateFuncTests.xml - - - name: Inference Engine C API tests - if: fromJSON(needs.smart_ci.outputs.affected_components).C_API.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/InferenceEngineCAPITests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-InferenceEngineCAPITests.xml - - - name: OpenVINO C API tests - if: fromJSON(needs.smart_ci.outputs.affected_components).C_API.test - run: | - source ${INSTALL_DIR}/setupvars.sh - - # Skip filter ticket: 126283 - ${INSTALL_TEST_DIR}/ov_capi_test --gtest_print_time=1 \ - --gtest_filter=-*ov_core/ov_core_test.ov_core_compile_model_with_property* \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OpenVINOCAPITests.xml - - - name: AutoBatch unit tests - if: fromJSON(needs.smart_ci.outputs.affected_components).AUTO_BATCH.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_auto_batch_unit_tests --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_batch_unit_tests.xml - - - name: AutoBatch func tests - if: fromJSON(needs.smart_ci.outputs.affected_components).AUTO_BATCH.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_auto_batch_func_tests --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_auto_batch_func_tests.xml - - - name: Proxy Plugin func tests - if: fromJSON(needs.smart_ci.outputs.affected_components).PROXY.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OVProxyTests.xml - - - name: Hetero unit tests - if: fromJSON(needs.smart_ci.outputs.affected_components).HETERO.test - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVHeteroUnitTests.xml - - - name: Hetero func tests - if: fromJSON(needs.smart_ci.outputs.affected_components).HETERO.test - run: | - source ${INSTALL_DIR}/setupvars.sh - ${INSTALL_TEST_DIR}/ov_hetero_func_tests --gtest_print_time=1 --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-OVHeteroFuncTests.xml - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ !cancelled() }} - with: - name: test-results-cpp - path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml - if-no-files-found: 'warn' + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_cxx_unit_tests.yml + with: + runner: 'aks-linux-16-cores-arm' + image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} Python_Unit_Tests: name: Python unit tests - needs: [Build, Smart_CI] - timeout-minutes: 180 - defaults: - run: - shell: bash - runs-on: 'aks-linux-16-cores-arm' - container: - image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 - volumes: - - /mount/caches:/mount/caches - env: - OPENVINO_REPO: /__w/openvino/openvino/openvino - INSTALL_DIR: /__w/openvino/openvino/install - INSTALL_TEST_DIR: /__w/openvino/openvino/install/tests - LAYER_TESTS_INSTALL_DIR: /__w/openvino/openvino/install/tests/layer_tests - - steps: - # - # Initialize OpenVINO - # - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package - path: ${{ env.INSTALL_DIR }} - - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} - popd - - pushd ${INSTALL_TEST_DIR} - tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} - popd - - - name: Fetch setup_python action - uses: actions/checkout@v4 - with: - sparse-checkout: | - .github/actions/setup_python/action.yml - sparse-checkout-cone-mode: false - path: ${{ env.OPENVINO_REPO }} - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: ./openvino/.github/actions/setup_python - with: - version: ${{ env.PYTHON_VERSION }} - pip-cache-path: ${{ env.PIP_CACHE_PATH }} - should-setup-pip-paths: 'true' - - - name: Install OpenVINO dependencies - run: | - apt-get update && apt-get install -y gcc python3-dev # Needed for building `psutil` - ${INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh -c=core -y - - - name: Install OpenVINO Python wheels - run: | - # Install the core OV wheel - python3 -m pip install ${INSTALL_DIR}/tools/openvino-*.whl - - # Find and install OV dev wheel - pushd ${INSTALL_DIR}/tools - ov_dev_wheel_name=$(find . -name 'openvino_dev*.whl') - python3 -m pip install $ov_dev_wheel_name[mxnet,caffe,kaldi,onnx,tensorflow2,pytorch] - popd - - - name: Install Python API tests dependencies - run: | - # To enable pytest parallel features - python3 -m pip install pytest-xdist[psutil] - # For torchvision to OpenVINO preprocessing converter - python3 -m pip install -r ${INSTALL_TEST_DIR}/python/preprocess/torchvision/requirements.txt - - # TODO: replace with Python API tests requirements - python3 -m pip install -r ${INSTALL_TEST_DIR}/mo/requirements_dev.txt - - # - # Tests - # - - - name: Python API 1.0 Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).Python_API.test - run: | - python3 -m pytest -s ${INSTALL_TEST_DIR}/pyngraph \ - --junitxml=${INSTALL_TEST_DIR}/TEST-Pyngraph.xml \ - --ignore=${INSTALL_TEST_DIR}/pyngraph/tests_compatibility/test_onnx/test_zoo_models.py \ - --ignore=${INSTALL_TEST_DIR}/pyngraph/tests_compatibility/test_onnx/test_backend.py - - - name: Python API 2.0 Tests - timeout-minutes: 30 - if: ${{ 'false' }} # Ticket: 126380 - #if: fromJSON(needs.smart_ci.outputs.affected_components).Python_API.test - run: | - # for 'template' extension - export LD_LIBRARY_PATH=${INSTALL_TEST_DIR}:$LD_LIBRARY_PATH - python3 -m pytest -sv ${INSTALL_TEST_DIR}/pyopenvino \ - --junitxml=${INSTALL_TEST_DIR}/TEST-Pyngraph.xml \ - --ignore=${INSTALL_TEST_DIR}/pyopenvino/tests/test_utils/test_utils.py - - - name: Model Optimizer unit tests - if: fromJSON(needs.smart_ci.outputs.affected_components).MO.test - run: | - python3 -m pytest -s ${INSTALL_TEST_DIR}/mo/unit_tests \ - --junitxml=${INSTALL_TEST_DIR}/TEST-ModelOptimizer.xml \ - --ignore-glob="**/mo/front/mxnet/**" - - - name: Python ONNX operators tests - if: fromJSON(needs.smart_ci.outputs.affected_components).Python_API.test || - fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE.test - run: | - # Skip test_onnx/test_zoo_models and test_onnx/test_backend due to long execution time - ONNX Model Zoo tests are run separately - python3 -m pytest -sv ${INSTALL_TEST_DIR}/onnx -k 'not cuda' \ - --junitxml=${INSTALL_TEST_DIR}/TEST-onnx_frontend.xml \ - --ignore=${INSTALL_TEST_DIR}/onnx/test_python/test_zoo_models.py - - - name: OVC unit tests - if: fromJSON(needs.smart_ci.outputs.affected_components).MO.test - run: python3 -m pytest -s ${INSTALL_TEST_DIR}/ovc/unit_tests --junitxml=${INSTALL_TEST_DIR}/TEST-OpenVinoConversion.xml - - - name: Install Python Layer tests dependencies - if: ${{ always() }} - run: | - # layer test requirements - python3 -m pip install -r ${LAYER_TESTS_INSTALL_DIR}/requirements.txt - - - name: MO Python API Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).MO.test - run: | - # Import 'test_utils' installed in '/tests/python/openvino' - export LD_LIBRARY_PATH=${PIP_INSTALL_PATH}/openvino/libs:$LD_LIBRARY_PATH - export PYTHONPATH=${INSTALL_TEST_DIR}/python - export LD_PRELOAD=${PIP_INSTALL_PATH}/torch/lib/../../torch.libs/libgomp-d22c30c5.so.1.0.0 - - echo ${PIP_INSTALL_PATH} - echo ${PIP_INSTALL_PATH} - echo ${PIP_INSTALL_PATH} - - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/mo_python_api_tests --junitxml=${INSTALL_TEST_DIR}/TEST-test_mo_convert.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: OVC Python API Tests - if: fromJSON(needs.smart_ci.outputs.affected_components).MO.test - run: | - # Import 'test_utils' installed in '/tests/python/openvino' - export PYTHONPATH=${INSTALL_TEST_DIR}/python - export LD_LIBRARY_PATH=${PIP_INSTALL_PATH}/openvino/libs:$LD_LIBRARY_PATH - export LD_PRELOAD=${PIP_INSTALL_PATH}/torch/lib/../../torch.libs/libgomp-d22c30c5.so.1.0.0 - - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/ovc_python_api_tests --junitxml=${INSTALL_TEST_DIR}/TEST-test_ovc_convert.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: Python Frontend tests - if: fromJSON(needs.smart_ci.outputs.affected_components).PyTorch_FE.test || - fromJSON(needs.smart_ci.outputs.affected_components).PDPD_FE.test - run: | - # to allow 'libtest_builtin_extensions.so' to find 'libopenvino_onnx_frontend.so' - export LD_LIBRARY_PATH=${PIP_INSTALL_PATH}/openvino/libs:$LD_LIBRARY_PATH - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/py_frontend_tests --junitxml=${INSTALL_TEST_DIR}/TEST-test_py_fontend.xml - - - name: PyTorch Layer Tests - timeout-minutes: 20 - if: ${{ 'false' }} # Ticket: 126287 - #if: fromJSON(needs.smart_ci.outputs.affected_components).PyTorch_FE.test - run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/pytorch_tests -n logical -m precommit --junitxml=${INSTALL_TEST_DIR}/TEST-pytorch.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP32 - - - name: PyTorch torch.compile TORCHFX Layer Tests - if: ${{ 'false' }} # RuntimeError: Python 3.11+ not yet supported for torch.compile, torch 2.0.1 is installed on Linux ARM64, it works in torch 2.1.1 - timeout-minutes: 20 - #if: fromJSON(needs.smart_ci.outputs.affected_components).PyTorch_FE.test - run: | - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/pytorch_tests -m precommit_fx_backend --junitxml=${INSTALL_TEST_DIR}/TEST-pytorch.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP32 - PYTORCH_TRACING_MODE: TORCHFX - - - name: PyTorch torch.compile TORCHSCRIPT Layer Tests - if: ${{ 'false' }} # RuntimeError: Python 3.11+ not yet supported for torch.compile, torch 2.0.1 is installed on Linux ARM64, it works in torch 2.1.1 - timeout-minutes: 20 - #if: fromJSON(needs.smart_ci.outputs.affected_components).PyTorch_FE.test - run: | - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/pytorch_tests -m precommit_ts_backend --junitxml=${INSTALL_TEST_DIR}/TEST-pytorch.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP32 - PYTORCH_TRACING_MODE: TORCHSCRIPT - - - name: ONNX Layer Tests - timeout-minutes: 30 - if: fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE.test - run: | - # requires 'unit_tests' from 'tools/mo' - export PYTHONPATH=${INSTALL_TEST_DIR}/mo:$PYTHONPATH - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/onnx_tests -m "not launch_only_if_manually_specified and precommit" --junitxml=${INSTALL_TEST_DIR}/TEST-onnx.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: TensorFlow 1 Layer Tests - TF FE - timeout-minutes: 30 - if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test - run: | - # requires 'unit_tests' from 'mo' - export PYTHONPATH=${INSTALL_TEST_DIR}/mo - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow_tests/ --use_new_frontend -m precommit_tf_fe --junitxml=${INSTALL_TEST_DIR}/TEST-tf_fe.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: TensorFlow 2 Layer Tests - TF FE - timeout-minutes: 30 - if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test - run: | - # requires 'unit_tests' from 'mo' - export PYTHONPATH=${INSTALL_TEST_DIR}/mo - python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow2_keras_tests/ --use_new_frontend -m precommit_tf_fe --junitxml=${INSTALL_TEST_DIR}/TEST-tf2_fe.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: JAX Layer Tests - TF FE - timeout-minutes: 30 - if: ${{ 'false' }} - #if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test - run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/jax_tests/ -m precommit --junitxml=${INSTALL_TEST_DIR}/TEST-jax.xml - env: - TEST_DEVICE: CPU - - - name: TensorFlow 1 Layer Tests - Legacy FE - timeout-minutes: 30 - run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow_tests/test_tf_Roll.py --ir_version=10 --junitxml=${INSTALL_TEST_DIR}/TEST-tf_Roll.xml - - - name: TensorFlow 2 Layer Tests - Legacy FE - timeout-minutes: 30 - if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test - run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow_tests/test_tf_Roll.py --ir_version=10 --junitxml=${INSTALL_TEST_DIR}/TEST-tf_Roll.xml - - - name: TensorFlow 2 Layer Tests - Legacy FE - if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test - run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow2_keras_tests/test_tf2_keras_activation.py --ir_version=11 -k "sigmoid" --junitxml=${INSTALL_TEST_DIR}/TEST-tf2_Activation.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: TensorFlow Lite Layer Tests - TFL FE - timeout-minutes: 30 - if: fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test - run: python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow_lite_tests/ --junitxml=${INSTALL_TEST_DIR}/TEST-tfl_fe.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: Clone API snippets - if: ${{ always() }} - uses: actions/checkout@v4 - with: - sparse-checkout: openvino/docs/snippets - path: ${{ env.OPENVINO_REPO }} - submodules: 'false' - - - name: Docs Python snippets - if: ${{ always() }} - run: | - # to find 'snippets' module in docs - export PYTHONPATH=${OPENVINO_REPO}/docs - # for 'template' extension - export LD_LIBRARY_PATH=${INSTALL_TEST_DIR}:$LD_LIBRARY_PATH - python3 ${OPENVINO_REPO}/docs/snippets/main.py - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ !cancelled() }} - with: - name: test-results-python - path: | - ${{ env.INSTALL_TEST_DIR }}/TEST*.html - ${{ env.INSTALL_TEST_DIR }}/TEST*.xml - if-no-files-found: 'warn' + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_python_unit_tests.yml + with: + runner: 'aks-linux-16-cores-arm' + container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04", "volumes": ["/mount/caches:/mount/caches"]}' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} CPU_Functional_Tests: name: CPU functional tests - needs: [Build, Smart_CI] - timeout-minutes: 60 - defaults: - run: - shell: bash - runs-on: 'aks-linux-16-cores-arm' - container: - image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 - env: - OPENVINO_REPO: /__w/openvino/openvino/openvino - INSTALL_DIR: /__w/openvino/openvino/install - INSTALL_TEST_DIR: /__w/openvino/openvino/install/tests - PARALLEL_TEST_SCRIPT: /__w/openvino/openvino/install/tests/functional_test_utils/layer_tests_summary/run_parallel.py - PARALLEL_TEST_CACHE: /__w/openvino/openvino/install/tests/test_cache.lst # if: fromJSON(needs.smart_ci.outputs.affected_components).CPU.test if: ${{ 'false' }} # Ticket: 126379 - steps: - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package - path: ${{ env.INSTALL_DIR }} - - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} - popd - pushd ${INSTALL_TEST_DIR} - tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} - popd - - - name: Install OpenVINO dependencies - run: bash ${INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh -c=core -y - - - name: Fetch setup_python action - uses: actions/checkout@v4 - with: - sparse-checkout: | - .github/actions/setup_python/action.yml - sparse-checkout-cone-mode: false - path: ${{ env.OPENVINO_REPO }} - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: ./openvino/.github/actions/setup_python - with: - version: ${{ env.PYTHON_VERSION }} - should-setup-pip-paths: 'false' - - - name: Install python dependencies for run_parallel.py - run: python3 -m pip install -r ${INSTALL_TEST_DIR}/functional_test_utils/layer_tests_summary/requirements.txt - - - name: Restore tests execution time - uses: actions/cache/restore@v3 - with: - path: ${{ env.PARALLEL_TEST_CACHE }} - key: ${{ runner.os }}-${{ runner.arch }}-tests-functional-cpu-stamp-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-tests-functional-cpu-stamp - - - name: Intel CPU plugin func tests (parallel) - run: | - source ${INSTALL_DIR}/setupvars.sh - python3 ${PARALLEL_TEST_SCRIPT} -e ${INSTALL_TEST_DIR}/ov_cpu_func_tests -c ${PARALLEL_TEST_CACHE} -w ${INSTALL_TEST_DIR} -s suite -rf 0 -- --gtest_print_time=1 --gtest_filter=*smoke* - timeout-minutes: 40 - - - name: Save tests execution time - uses: actions/cache/save@v3 - if: github.ref_name == 'master' - with: - path: ${{ env.PARALLEL_TEST_CACHE }} - key: ${{ runner.os }}-${{ runner.arch }}-tests-functional-cpu-stamp-${{ github.sha }} - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ !cancelled() }} - with: - name: test-results-functional-cpu - path: | - ${{ env.INSTALL_TEST_DIR }}/temp/*.log - ${{ env.INSTALL_TEST_DIR }}/logs/*.log - ${{ env.INSTALL_TEST_DIR }}/logs/failed/*.log - ${{ env.INSTALL_TEST_DIR }}/logs/crashed/*.log - ${{ env.INSTALL_TEST_DIR }}/logs/hanged/*.log - ${{ env.INSTALL_TEST_DIR }}/logs/interapted/*.log - ${{ env.INSTALL_TEST_DIR }}/logs/hash_table.csv - ${{ env.PARALLEL_TEST_CACHE }} - if-no-files-found: 'error' + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_cpu_functional_tests.yml + with: + runner: 'aks-linux-16-cores-arm' + image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' TensorFlow_Hub_Models_Tests: name: TensorFlow Hub Models tests - needs: [Build, Smart_CI] - defaults: - run: - shell: bash - runs-on: 'aks-linux-16-cores-arm' - timeout-minutes: ${{ github.event_name == 'schedule' && 400 || 5 }} - container: - image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 - volumes: - - /mount/caches:/mount/caches - env: - OPENVINO_REPO: ${{ github.workspace }}/openvino - INSTALL_DIR: ${{ github.workspace }}/install - INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests - MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests + if: ${{ 'false' }} # TODO: Enable once the dependencies are ready for arm (no tensorflow-text available for arm from PyPI) # if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test || # fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test - if: ${{ 'false' }} # TODO: Enable once the self-hosted runners are ready for them - - steps: - - name: Check sudo - run: if [ "$(id -u)" -eq 0 ]; then apt update && apt --assume-yes install sudo; fi - - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package_x86_64 - path: ${{ env.INSTALL_DIR }} - - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests_x86_64 - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} - popd - - pushd ${INSTALL_TEST_DIR} - tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} - popd - - - name: Fetch setup_python action - uses: actions/checkout@v4 - with: - sparse-checkout: | - .github/actions/setup_python/action.yml - sparse-checkout-cone-mode: false - path: 'openvino' - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: ./openvino/.github/actions/setup_python - with: - version: ${{ env.PYTHON_VERSION }} - should-setup-pip-paths: 'false' - self-hosted-runner: 'false' - - - name: Install OpenVINO Python wheels - run: python3 -m pip install ${INSTALL_DIR}/tools/openvino-* - - - name: Install TF Hub tests requirements - run: | - python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/tf_hub_tests/requirements.txt - - - name: TensorFlow Hub Tests - TF FE - run: | - export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH - python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/tf_hub_tests/ -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-tf_hub_tf_fe.html --self-contained-html -v - env: - TYPE: ${{ github.event_name == 'schedule' && 'nightly' || 'precommit'}} - TEST_DEVICE: CPU - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ !cancelled() }} - with: - name: test-results-tensorflow-hub-models - path: | - ${{ env.INSTALL_TEST_DIR }}/TEST*.html - if-no-files-found: 'error' + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_tensorflow_hub_models_tests.yml + with: + runner: 'aks-linux-16-cores-arm' + container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04"}' + event: ${{ github.event_name }} TensorFlow_Hub_Performance_Models_Tests: name: TensorFlow Hub Performance Models tests - needs: [Build, Smart_CI] - defaults: - run: - shell: bash - runs-on: 'aks-linux-16-cores-arm' - timeout-minutes: ${{ github.event_name == 'schedule' && 400 || 5 }} - env: - OPENVINO_REPO: ${{ github.workspace }}/openvino - INSTALL_DIR: ${{ github.workspace }}/install - INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests - MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests + if: ${{ 'false' }} # TODO: Enable once the dependencies are ready for arm (no tensorflow-text available for arm from PyPI) # if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test || # fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test - if: ${{ 'false' }} # TODO: Enable once the self-hosted runners are ready for them - - steps: - - name: Check sudo - run: if [ "$(id -u)" -eq 0 ]; then apt update && apt --assume-yes install sudo; fi - - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package_x86_64 - path: ${{ env.INSTALL_DIR }} - - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests_x86_64 - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} - popd - - pushd ${INSTALL_TEST_DIR} - tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} - popd - - - name: Fetch setup_python action - uses: actions/checkout@v4 - with: - sparse-checkout: | - .github/actions/setup_python/action.yml - sparse-checkout-cone-mode: false - path: 'openvino' + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_tensorflow_hub_performance_models_tests.yml + with: + runner: 'aks-linux-16-cores-arm' + container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04"}' + event: ${{ github.event_name }} - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: ./openvino/.github/actions/setup_python - with: - version: ${{ env.PYTHON_VERSION }} - should-setup-pip-paths: 'false' - self-hosted-runner: 'false' - - - name: Install OpenVINO Python wheels - run: python3 -m pip install ${INSTALL_DIR}/tools/openvino-* - - - name: Install TF Hub tests requirements - run: | - python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/tf_hub_tests/requirements.txt - - - name: Install Hub Performance tests requirements - run: | - python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/performance_tests/requirements.txt - - - name: Performance Hub Tests - run: | - export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH - python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/performance_tests/ -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-tf_hub_performance.html --self-contained-html -v - env: - TYPE: ${{ github.event_name == 'schedule' && 'nightly' || 'precommit'}} - TEST_DEVICE: CPU - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ !cancelled() }} - with: - name: test-results-tensorflow-hub-performance-models - path: | - ${{ env.INSTALL_TEST_DIR }}/TEST*.html - if-no-files-found: 'error' - - # TODO: Enable once they are ready for self-hosted runners PyTorch_Models_Tests: name: PyTorch Models tests - needs: [Build, Smart_CI] - timeout-minutes: ${{ github.event_name == 'schedule' && 400 || 30 }} - defaults: - run: - shell: bash - runs-on: 'aks-linux-16-cores-arm' - container: - image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 - volumes: - - /mount/caches:/mount/caches - env: - OPENVINO_REPO: ${{ github.workspace }}/openvino - INSTALL_DIR: ${{ github.workspace }}/install - INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests - MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests + if: ${{ 'false' }} # TODO: Enable once the dependencies are ready for arm (no tensorflow-text available for arm from PyPI) # if: fromJSON(needs.smart_ci.outputs.affected_components).PyTorch_FE.test - if: ${{ 'false' }} # TODO: Enable once the self-hosted runners are ready for them - - steps: - - name: Check sudo - run: if [ "$(id -u)" -eq 0 ]; then apt update && apt --assume-yes install sudo; fi - - - name: Install dependencies - run: | - # install git (required to build pip deps from the sources) - # install 'g++' to build 'detectron2' and 'natten' wheels - sudo apt-get install --assume-yes --no-install-recommends g++ git ca-certificates - - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package_x86_64 - path: ${{ env.INSTALL_DIR }} - - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests_x86_64 - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} - popd - pushd ${INSTALL_TEST_DIR} - tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} - popd - - - name: Fetch setup_python action - uses: actions/checkout@v4 - with: - sparse-checkout: | - .github/actions/setup_python/action.yml - sparse-checkout-cone-mode: false - path: 'openvino' - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: ./openvino/.github/actions/setup_python - with: - version: ${{ env.PYTHON_VERSION }} - should-setup-pip-paths: 'false' - self-hosted-runner: 'false' - - - name: Install OpenVINO Python wheels - run: python3 -m pip install ${INSTALL_DIR}/tools/openvino-* - - - name: Install PyTorch tests requirements - run: | - python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/requirements.txt - python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/requirements_secondary.txt - echo "Available storage:" - df -h - env: - CPLUS_INCLUDE_PATH: ${{ env.Python_ROOT_DIR }}/include/python${{ env.PYTHON_VERSION }} - - - name: PyTorch Models Tests - run: | - export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH - python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_model_tests.html --self-contained-html -v - env: - TYPE: ${{ github.event_name == 'schedule' && 'nightly' || 'precommit'}} - TEST_DEVICE: CPU - USE_SYSTEM_CACHE: False - - - name: Available storage after tests - run: | - echo "Available storage:" - df -h - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ !cancelled() }} - with: - name: test-results-torch-models - path: | - ${{ env.INSTALL_TEST_DIR }}/TEST*.html - if-no-files-found: 'error' + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_pytorch_models_tests.yml + with: + runner: 'aks-linux-16-cores-arm' + container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04"}' + event: ${{ github.event_name }} Overall_Status: name: ci/gha_overall_status diff --git a/.github/workflows/linux_conditional_compilation.yml b/.github/workflows/linux_conditional_compilation.yml index c5d09112d2e7e7..ad9218fd1ef6b2 100644 --- a/.github/workflows/linux_conditional_compilation.yml +++ b/.github/workflows/linux_conditional_compilation.yml @@ -60,6 +60,7 @@ jobs: GITHUB_WORKSPACE: '/__w/openvino/openvino' OPENVINO_REPO: /__w/openvino/openvino/openvino INSTALL_DIR: /__w/openvino/openvino/openvino_install + INSTALL_TEST_DIR: /__w/openvino/openvino/tests_install BUILD_DIR: /__w/openvino/openvino/openvino_build SELECTIVE_BUILD_STAT_DIR: /__w/openvino/openvino/selective_build_stat MODELS_PATH: /__w/openvino/openvino/testdata @@ -163,7 +164,9 @@ jobs: run: ${SCCACHE_PATH} --show-stats - name: Cmake install - OpenVINO - run: cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -P ${BUILD_DIR}/cmake_install.cmake + run: | + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -P ${BUILD_DIR}/cmake_install.cmake + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_TEST_DIR} -DCOMPONENT=tests -P ${BUILD_DIR}/cmake_install.cmake - name: Build C++ samples - OpenVINO build tree run: | @@ -188,14 +191,26 @@ jobs: pushd ${SELECTIVE_BUILD_STAT_DIR} tar -czvf ${BUILD_DIR}/openvino_selective_build_stat.tar.gz * popd + + pushd ${INSTALL_DIR} + tar -czvf ${BUILD_DIR}/openvino_package.tar.gz \ + install_dependencies/install_openvino_dependencies.sh + popd - pushd ${OPENVINO_REPO} + pushd ${INSTALL_TEST_DIR} tar -czvf ${BUILD_DIR}/openvino_tests.tar.gz \ - bin/intel64/Release/ov_cpu_func_tests \ - src/tests/test_utils/functional_test_utils/layer_tests_summary/* \ - scripts/install_dependencies/* + tests/ov_cpu_func_tests \ + tests/functional_test_utils/layer_tests_summary/* popd + - name: Upload openvino package + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: openvino_package + path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz + if-no-files-found: 'error' + - name: Upload selective build statistics package if: ${{ always() }} uses: actions/upload-artifact@v3 @@ -310,74 +325,9 @@ jobs: CPU_Functional_Tests: name: CPU functional tests - needs: [Build, Smart_CI] - timeout-minutes: 25 - defaults: - run: - shell: bash - runs-on: aks-linux-8-cores-32gb - container: - image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04 - env: - OPENVINO_REPO: /__w/openvino/openvino/openvino - DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input - INSTALL_TEST_DIR: /__w/openvino/openvino/install/tests - PARALLEL_TEST_SCRIPT: /__w/openvino/openvino/install/tests/src/tests/test_utils/functional_test_utils/layer_tests_summary/run_parallel.py - PARALLEL_TEST_CACHE: /__w/openvino/openvino/install/tests/test_cache.lst if: fromJSON(needs.smart_ci.outputs.affected_components).CPU.test - - steps: - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO tests package - run: tar -xvzf ${INSTALL_TEST_DIR}/openvino_tests.tar.gz -C ${INSTALL_TEST_DIR} - - - name: Install OpenVINO dependencies - run: bash ${INSTALL_TEST_DIR}/scripts/install_dependencies/install_openvino_dependencies.sh -c=core -c=gpu -y - - - name: Fetch setup_python action - uses: actions/checkout@v4 - with: - sparse-checkout: | - .github/actions/setup_python/action.yml - sparse-checkout-cone-mode: false - path: ${{ env.OPENVINO_REPO }} - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: ./openvino/.github/actions/setup_python - with: - version: ${{ env.PYTHON_VERSION }} - should-setup-pip-paths: 'false' - - - name: Install python dependencies for run_parallel.py - run: python3 -m pip install -r ${INSTALL_TEST_DIR}/src/tests/test_utils/functional_test_utils/layer_tests_summary/requirements.txt - - - name: Restore tests execution time - uses: actions/cache/restore@v3 - with: - path: ${{ env.PARALLEL_TEST_CACHE }} - key: ${{ runner.os }}-tests-functional-cpu-stamp-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-tests-functional-cpu-stamp - - - name: Intel CPU plugin func tests (parallel) - run: python3 ${PARALLEL_TEST_SCRIPT} -e ${INSTALL_TEST_DIR}/bin/intel64/Release/ov_cpu_func_tests -c ${PARALLEL_TEST_CACHE} -w ${INSTALL_TEST_DIR} -s suite -rf 0 -- --gtest_print_time=1 --gtest_filter=*smoke* - timeout-minutes: 20 - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ !cancelled() }} - with: - name: test-results-functional-cpu - path: | - ${{ env.INSTALL_TEST_DIR }}/TEST*.xml - ${{ env.INSTALL_TEST_DIR }}/logs/failed/*.log - ${{ env.INSTALL_TEST_DIR }}/logs/crashed/*.log - ${{ env.INSTALL_TEST_DIR }}/logs/hanged/*.log - ${{ env.INSTALL_TEST_DIR }}/logs/interapted/*.log - ${{ env.INSTALL_TEST_DIR }}/logs/disabled_tests.log - if-no-files-found: 'error' + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_cpu_functional_tests.yml + with: + runner: 'aks-linux-8-cores-32gb' + image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04' diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index d8ac800e18b7d5..2a59d9de4d50b2 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -33,27 +33,41 @@ env: PYTHON_VERSION: '3.11' jobs: + + Smart_CI: + runs-on: ubuntu-latest + outputs: + affected_components: "${{ steps.smart_ci.outputs.affected_components }}" + skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" + steps: + - name: checkout action + uses: actions/checkout@v4 + with: + sparse-checkout: .github/actions/smart-ci + + - name: Get affected components + id: smart_ci + uses: ./.github/actions/smart-ci + with: + repository: ${{ github.repository }} + pr: ${{ github.event.number }} + commit_sha: ${{ github.sha }} + component_pattern: "category: (.*)" + repo_token: ${{ secrets.GITHUB_TOKEN }} + skip_when_only_listed_labels_set: 'docs' + skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg' + Build: + needs: Smart_CI timeout-minutes: 150 defaults: run: shell: bash - strategy: - max-parallel: 2 - fail-fast: false - matrix: - include: - - arhitecture: 'x86_64' - machine: 'macos-13-large' - macos_deployment_target: '10.12' - - arhitecture: 'arm64' - machine: 'macos-13-xlarge' - macos_deployment_target: '11.0' - runs-on: ${{ matrix.machine }} + runs-on: 'macos-13-large' env: CMAKE_BUILD_TYPE: 'Release' CMAKE_GENERATOR: 'Ninja Multi-Config' - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos_deployment_target }} + MACOSX_DEPLOYMENT_TARGET: '10.12' CMAKE_CXX_COMPILER_LAUNCHER: ccache CMAKE_C_COMPILER_LAUNCHER: ccache OPENVINO_REPO: ${{ github.workspace }}/openvino @@ -125,9 +139,9 @@ jobs: # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push save: ${{ github.ref_name == 'master' && 'true' || 'false' }} verbose: 2 - key: ${{ runner.os }}-${{ matrix.arhitecture }}-main + key: ${{ runner.os }}-${{ runner.arch }}-main restore-keys: | - ${{ runner.os }}-${{ matrix.arhitecture }}-main + ${{ runner.os }}-${{ runner.arch }}-main - name: CMake configure run: | @@ -184,7 +198,7 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@v3 with: - name: openvino_package_${{ matrix.arhitecture }} + name: openvino_package path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz if-no-files-found: 'error' @@ -192,650 +206,39 @@ jobs: if: ${{ always() }} uses: actions/upload-artifact@v3 with: - name: openvino_tests_${{ matrix.arhitecture }} + name: openvino_tests path: ${{ env.BUILD_DIR }}/openvino_tests.tar.gz if-no-files-found: 'error' Samples: - needs: Build - timeout-minutes: 5 - defaults: - run: - shell: bash - strategy: - max-parallel: 2 - fail-fast: false - matrix: - include: - - arhitecture: 'x86_64' - machine: 'macos-13' - - arhitecture: 'arm64' - machine: 'macos-13-xlarge' - runs-on: ${{ matrix.machine }} - env: - OPENVINO_REPO: ${{ github.workspace }}/openvino - INSTALL_DIR: ${{ github.workspace }}/install - INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests - BUILD_DIR: ${{ github.workspace }}/build - - steps: - - # - # Initialize OpenVINO - # - - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package_${{ matrix.arhitecture }} - path: ${{ env.INSTALL_DIR }} - - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests_${{ matrix.arhitecture }} - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} - popd - - pushd ${INSTALL_TEST_DIR} - tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} - popd - - - name: Install dependencies - run: brew install coreutils - - - name: Fetch setup_python action - uses: actions/checkout@v4 - with: - sparse-checkout: | - .github/actions/setup_python/action.yml - sparse-checkout-cone-mode: false - path: 'openvino' - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: ./openvino/.github/actions/setup_python - with: - version: ${{ env.PYTHON_VERSION }} - should-setup-pip-paths: 'false' - self-hosted-runner: 'false' - - - name: Build cpp samples - run: ${INSTALL_DIR}/samples/cpp/build_samples.sh -i ${INSTALL_DIR} -b ${BUILD_DIR}/cpp_samples - env: - CMAKE_COMPILE_WARNING_AS_ERROR: 'ON' - - - name: Build c samples - run: ${INSTALL_DIR}/samples/c/build_samples.sh -i ${INSTALL_DIR} -b ${BUILD_DIR}/c_samples - env: - CMAKE_COMPILE_WARNING_AS_ERROR: 'ON' - - # - # Tests - # - - - name: Samples tests - run: | - export WORKSPACE=${INSTALL_DIR} - export IE_APP_PATH=${INSTALL_DIR}/samples_bin - export IE_APP_PYTHON_PATH=${INSTALL_DIR}/samples/python - export SHARE=${INSTALL_TEST_DIR}/smoke_tests/samples_smoke_tests_data - - python3 -m pip install --ignore-installed PyYAML -r ${INSTALL_TEST_DIR}/smoke_tests/requirements.txt - - source ${INSTALL_DIR}/setupvars.sh - - python3 -m pytest -sv ${INSTALL_TEST_DIR}/smoke_tests \ - --env_conf ${INSTALL_TEST_DIR}/smoke_tests/env_config.yml \ - --junitxml=${INSTALL_TEST_DIR}/TEST-SamplesSmokeTests.xml - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ !cancelled() }} - with: - name: test-results-samples-${{ matrix.arhitecture }} - path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml - if-no-files-found: 'error' + needs: [ Build, Smart_CI ] + if: fromJSON(needs.smart_ci.outputs.affected_components).samples + uses: ./.github/workflows/job_samples_tests.yml + with: + runner: 'macos-13' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} CXX_Unit_Tests: - name: C++ Unit tests - needs: Build - timeout-minutes: 20 - defaults: - run: - shell: bash - strategy: - max-parallel: 2 - fail-fast: false - matrix: - include: - - arhitecture: 'x86_64' - machine: 'macos-13' - - arhitecture: 'arm64' - machine: 'macos-13-xlarge' - runs-on: ${{ matrix.machine }} - env: - INSTALL_DIR: ${{ github.workspace }}/install - INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests - - steps: - # - # Dependencies - # - - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package_${{ matrix.arhitecture }} - path: ${{ env.INSTALL_DIR }} - - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests_${{ matrix.arhitecture }} - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${{ env.INSTALL_DIR }} - tar -xzf openvino_package.tar.gz -C ${{ env.INSTALL_DIR }} && rm openvino_package.tar.gz || exit 1 - popd - pushd ${{ env.INSTALL_TEST_DIR }} - tar -xzf openvino_tests.tar.gz -C ${{ env.INSTALL_DIR }} && rm openvino_tests.tar.gz || exit 1 - popd - - # - # Tests - # - - - name: OpenVINO Core Unit Tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_core_unit_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-NGraphUT.xml - - - name: OpenVINO Inference Functional Tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_inference_functional_tests --gtest_print_time=1 \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-InferenceFunc.xml - - - name: OpenVINO Inference Unit Tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_inference_unit_tests --gtest_print_time=1 \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-InferenceUnit.xml - - - name: Low Precision Transformations Tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - - # Skips under Ticket: 122660 - skip_filter=${{ matrix.arhitecture == 'arm64' && '--gtest_filter=-*smoke_LPT/FoldFakeQuantizeInTransformations.CompareFunctions*' || '' }} - - ${{ env.INSTALL_TEST_DIR }}/ov_lp_transformations_tests --gtest_print_time=1 "$skip_filter" \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-LpTransformations.xml - - - name: OpenVINO Conditional compilation tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_conditional_compilation_tests --gtest_print_time=1 \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ConditionalCompilation.xml - - - name: IR frontend tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_ir_frontend_tests --gtest_print_time=1 \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-IRFrontend.xml - - - name: PaddlePaddle frontend tests - if: ${{ 'false' }} - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/paddle_tests --gtest_print_time=1 --gtest_filter=*smoke* \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-PaddleTests.xml - - - name: ONNX frontend tests - if: ${{ matrix.arhitecture == 'x86_64' }} # Ticket for ARM64: 122663 - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - - ${{ env.INSTALL_TEST_DIR }}/ov_onnx_frontend_tests --gtest_print_time=1 --gtest_filter=-*IE_GPU* \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ONNXFrontend.xml - - - name: TensorFlow Common tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_tensorflow_common_tests --gtest_print_time=1 \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-TensorFlowCommonFrontend.xml - - - name: TensorFlow frontend tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - - # Skips under Ticket: 122666 - skip_filter=${{ matrix.arhitecture == 'arm64' && '--gtest_filter=-*CompileModelsTests.ModelWithSplitConvConcat*:*NgramCompilation*' || '' }} - - ${{ env.INSTALL_TEST_DIR }}/ov_tensorflow_frontend_tests --gtest_print_time=1 "$skip_filter" \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-TensorFlowFrontend.xml - - - name: TensorFlow Lite frontend tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_tensorflow_lite_frontend_tests --gtest_print_time=1 \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-TensorFlowLiteFrontend.xml - - - name: Transformations func tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - - # Skips under Ticket: 122668 - skip_filter=${{ matrix.arhitecture == 'arm64' && '--gtest_filter=-*TransformationTestsF.CompressQuantizeWeights*:*TransformationTests/CompressQuantizeWeightsTests.FusionTest*' || '' }} - - ${{ env.INSTALL_TEST_DIR }}/ov_transformations_tests --gtest_print_time=1 "$skip_filter" \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-Transformations.xml - - - name: Common test utils tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_util_tests --gtest_print_time=1 \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-commonUtilsTests.xml - - - name: Snippets func tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_snippets_func_tests --gtest_print_time=1 \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-SnippetsFuncTests.xml - - - name: CPU plugin unit tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_cpu_unit_tests --gtest_print_time=1 \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-CPUUnitTests.xml - - - name: ov_subgraphs_dumper_tests tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_subgraphs_dumper_tests --gtest_print_time=1 \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-ov_subgraphs_dumper_tests.xml - - - name: Template OpImpl tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_op_conformance_tests --gtest_print_time=1 --device=TEMPLATE --gtest_filter="*OpImpl*" \ - --gtest_output=xml:${INSTALL_TEST_DIR}/TEST-TemplateOpImplTests.xml - - - name: AUTO unit tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_auto_unit_tests --gtest_print_time=1 \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_unit_tests.xml - - - name: AUTO func Tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_auto_func_tests --gtest_print_time=1 \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_func_tests.xml - - - name: Template plugin func tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_template_func_tests --gtest_print_time=1 \ - --gtest_filter=*smoke* \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-TemplateFuncTests.xml - - - name: Inference Engine C API tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/InferenceEngineCAPITests --gtest_print_time=1 \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-InferenceEngineCAPITests.xml - - - name: OpenVINO C API tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_capi_test --gtest_print_time=1 \ - --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OpenVINOCAPITests.xml - - - name: AutoBatch unit tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_auto_batch_unit_tests --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_batch_unit_tests.xml - - - name: AutoBatch func tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_auto_batch_func_tests --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-ov_auto_batch_func_tests.xml - - - name: Proxy Plugin func tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_proxy_plugin_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVProxyTests.xml - - - name: Hetero unit tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_hetero_unit_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVHeteroUnitTests.xml - - - name: Hetero func tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - ${{ env.INSTALL_TEST_DIR }}/ov_hetero_func_tests --gtest_print_time=1 --gtest_output=xml:${{ env.INSTALL_TEST_DIR }}/TEST-OVHeteroFuncTests.xml - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ always() }} - with: - name: test-results-cpp-${{ matrix.arhitecture }} - path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml - if-no-files-found: 'error' + name: C++ unit tests + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_cxx_unit_tests.yml + with: + runner: 'macos-13' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} Python_Unit_Tests: name: Python unit tests - needs: Build - timeout-minutes: 55 - defaults: - run: - shell: bash - strategy: - max-parallel: 2 - fail-fast: false - matrix: - include: - - arhitecture: 'x86_64' - machine: 'macos-13' - - arhitecture: 'arm64' - machine: 'macos-13-xlarge' - runs-on: ${{ matrix.machine }} - env: - OPENVINO_REPO: ${{ github.workspace }}/openvino - OPENVINO_CONTRIB_REPO: ${{ github.workspace }}/openvino_contrib - INSTALL_DIR: ${{ github.workspace }}/install - INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests - LAYER_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/layer_tests - steps: - - name: Fetch setup_python action - uses: actions/checkout@v4 - with: - sparse-checkout: | - .github/actions/setup_python/action.yml - sparse-checkout-cone-mode: false - path: 'openvino' - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: ./openvino/.github/actions/setup_python - with: - version: ${{ env.PYTHON_VERSION }} - should-setup-pip-paths: 'false' - self-hosted-runner: 'false' - - # - # Dependencies - # - - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package_${{ matrix.arhitecture }} - path: ${{ env.INSTALL_DIR }} - - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests_${{ matrix.arhitecture }} - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${{ env.INSTALL_DIR }} - tar -xzf openvino_package.tar.gz -C ${{ env.INSTALL_DIR }} - popd - - pushd ${{ env.INSTALL_TEST_DIR }} - tar -xzf openvino_tests.tar.gz -C ${{ env.INSTALL_DIR }} - popd - - - name: Install OpenVINO Python wheels - run: | - # Install the core OV wheel - python3 -m pip install ${{ env.INSTALL_DIR }}/tools/openvino-*.whl - - # mxnet is only available on x86_64 - extras_to_install="caffe,kaldi,onnx,tensorflow2,pytorch" - if [[ "${{ matrix.arhitecture }}" == "x86_64" ]]; then - extras_to_install="mxnet,$extras_to_install" - fi - - # Find and install OV dev wheel - pushd ${{ env.INSTALL_DIR }}/tools - ov_dev_wheel_name=$(find . -name 'openvino_dev*.whl') - python3 -m pip install $ov_dev_wheel_name[$extras_to_install] - popd - - - name: Install Python API tests dependencies - run: | - # For torchvision to OpenVINO preprocessing converter - python3 -m pip install -r ${{ env.INSTALL_TEST_DIR }}/python/preprocess/torchvision/requirements.txt - - # TODO: replace with Python API tests requirements - python3 -m pip install -r ${{ env.INSTALL_TEST_DIR }}/mo/requirements_dev.txt - - - name: Python API 1.0 Tests - run: | - python3 -m pytest -s ${{ env.INSTALL_TEST_DIR }}/pyngraph \ - --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-Pyngraph.xml \ - --ignore=${{ env.INSTALL_TEST_DIR }}/pyngraph/tests_compatibility/test_onnx/test_zoo_models.py \ - --ignore=${{ env.INSTALL_TEST_DIR }}/pyngraph/tests_compatibility/test_onnx/test_backend.py - - - name: Python API 2.0 Tests - run: | - # For python imports to import pybind_mock_frontend - export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}:$PYTHONPATH - # for 'template' extension - export DYLD_LIBRARY_PATH=${{ env.INSTALL_TEST_DIR }}:$DYLD_LIBRARY_PATH - - python3 -m pytest -sv ${{ env.INSTALL_TEST_DIR }}/pyopenvino \ - --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-Pyngraph.xml \ - --ignore=${{ env.INSTALL_TEST_DIR }}/pyopenvino/tests/test_utils/test_utils.py - - - name: MO Python API Tests - run: | - python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt - - # Used for 'test_utils' installed in '/python/openvino/test_utils' - export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}/python/openvino/test_utils:${{ env.INSTALL_TEST_DIR }}/python:$PYTHONPATH - - python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/mo_python_api_tests/ --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-test_mo_convert.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: OVC Python API Tests - run: | - python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt - - # Used for 'test_utils' installed in '/python/openvino/test_utils' - export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}/python/openvino/test_utils:${{ env.INSTALL_TEST_DIR }}/python:$PYTHONPATH - - python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/ovc_python_api_tests --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-test_ovc_convert.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: Model Optimizer unit tests - run: | - export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}:$PYTHONPATH - python3 -m pytest -s ${{ env.INSTALL_TEST_DIR }}/mo/unit_tests \ - --ignore=${{ env.INSTALL_TEST_DIR }}/mo/unit_tests/mo/front/mxnet \ - --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-ModelOptimizer.xml - - - name: PyTorch Layer Tests - run: | - python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt - export PYTHONPATH=${{ env.LAYER_TESTS_INSTALL_DIR }}:$PYTHONPATH - - python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/pytorch_tests -m precommit --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-pytorch.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: ONNX Layer Tests - run: | - python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt - - export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}/mo:$PYTHONPATH - - python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/onnx_tests -m "not launch_only_if_manually_specified and precommit" --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-onnx.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: TensorFlow 1 Layer Tests - TF FE - run: | - python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt - - export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}/mo:$PYTHONPATH - - python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow_tests/ --use_new_frontend -m precommit_tf_fe --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf_fe.xml - env: - TEST_DEVICE: CPU - - - name: TensorFlow 2 Layer Tests - TF FE - if: ${{ 'false' }} # Ticket: 123322 - run: | - python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt - - export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}/mo:$PYTHONPATH - - python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow2_keras_tests/ --use_new_frontend -m precommit_tf_fe --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf2_fe.xml - env: - TEST_DEVICE: CPU - - - name: TensorFlow 1 Layer Tests - Legacy FE - run: | - python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt - - export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}/mo:$PYTHONPATH - - python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow_tests/test_tf_Roll.py --ir_version=10 --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf_Roll.xml - - - name: TensorFlow 2 Layer Tests - Legacy FE - run: | - python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt - - export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}/mo:$PYTHONPATH - - python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow2_keras_tests/test_tf2_keras_activation.py \ - --ir_version=11 --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf2_Activation.xml -k "sigmoid" - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: TensorFlow Lite Layer Tests - TFL FE - run: | - python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt - - export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}/mo:$PYTHONPATH - - python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow_lite_tests/ --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tfl_fe.xml - env: - TEST_DEVICE: CPU - TEST_PRECISION: FP16 - - - name: Python ONNX operators tests - if: ${{ 'false' }} # Ticket: 123325 - run: | - # Skip test_onnx/test_zoo_models and test_onnx/test_backend due to long execution time - ONNX Model Zoo tests are run separately - python3 -m pytest -sv ${{ env.INSTALL_TEST_DIR }}/onnx -k 'not cuda' \ - --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-onnx_frontend.xml \ - --ignore=${{ env.INSTALL_TEST_DIR }}/onnx/test_python/test_zoo_models.py - - - name: Python Frontend tests - run: | - python3 -m pip install -r ${{ env.LAYER_TESTS_INSTALL_DIR }}/requirements.txt - - export PYTHONPATH=${{ env.INSTALL_TEST_DIR }}/mo:$PYTHONPATH - - # to allow 'libtest_builtin_extensions.so' to find 'libopenvino_onnx_frontend.so' - source ${{ env.INSTALL_DIR }}/setupvars.sh - - python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/py_frontend_tests --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-test_py_fontend.xml - - # TODO: install to 'tests' component via cpack - - name: OVC unit tests - run: python3 -m pytest -s ${{ env.INSTALL_TEST_DIR }}/ovc/unit_tests --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-OpenVinoConversion.xml - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ always() }} - with: - name: test-results-python-${{ matrix.arhitecture }} - path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml - if-no-files-found: 'error' + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_python_unit_tests.yml + with: + runner: 'macos-13' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} CPU_Functional_Tests: name: CPU functional tests - needs: Build - timeout-minutes: 25 - defaults: - run: - shell: bash - strategy: - max-parallel: 2 - fail-fast: false - matrix: - include: - # ticket: 122001 - # - arhitecture: 'x86_64' - # machine: 'macos-13' - - arhitecture: 'arm64' - machine: 'macos-13-xlarge' - runs-on: ${{ matrix.machine }} - env: - INSTALL_DIR: ${{ github.workspace }}/install - INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests - - steps: - - name: Create Directories - run: mkdir -p ${{ env.INSTALL_DIR }} ${{ env.INSTALL_TEST_DIR }} - - - name: Download OpenVINO package - uses: actions/download-artifact@v3 - with: - name: openvino_package_${{ matrix.arhitecture }} - path: ${{ env.INSTALL_DIR }} - - - name: Download OpenVINO tests package - uses: actions/download-artifact@v3 - with: - name: openvino_tests_${{ matrix.arhitecture }} - path: ${{ env.INSTALL_TEST_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${{ env.INSTALL_DIR }} - tar -xzf openvino_package.tar.gz -C ${{ env.INSTALL_DIR }} && rm openvino_package.tar.gz - popd - pushd ${{ env.INSTALL_TEST_DIR }} - tar -xzf openvino_tests.tar.gz -C ${{ env.INSTALL_DIR }} && rm openvino_tests.tar.gz - popd - - - name: CPU plugin func tests - run: | - source ${{ env.INSTALL_DIR }}/setupvars.sh - - # Skips under Ticket: 122769 - skip_filter=${{ matrix.arhitecture == 'arm64' && '--gtest_filter=-*smoke_nonzero/NonZeroLayerTest.Inference/IS*:*smoke_NormalizeL2_*:*Extension.XmlModelWithExtensionFromDSO*:*Extension.OnnxModelWithExtensionFromDSO*:*ONNXQuantizedModels/QuantizedModelsTests.MaxPool*:*ONNXQuantizedModels/QuantizedModelsTests.Convolution*:**' || '' }} - - ${{ env.INSTALL_TEST_DIR }}/ov_cpu_func_tests --gtest_print_time=1 --gtest_filter=*smoke* "$skip_filter" --gtest_output=xml:"${{ env.INSTALL_TEST_DIR }}/TEST-CPUFuncTests.xml" - - - name: Upload Test Results - uses: actions/upload-artifact@v3 - if: ${{ always() }} - with: - name: test-results-functional-cpu-${{ matrix.arhitecture }} - path: ${{ env.INSTALL_TEST_DIR }}/TEST*.xml - if-no-files-found: 'error' + # if: fromJSON(needs.smart_ci.outputs.affected_components).CPU.test + if: ${{ 'false' }} # Ticket: 122001 + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_cpu_functional_tests.yml + with: + runner: 'macos-13' diff --git a/.github/workflows/mac_arm64.yml b/.github/workflows/mac_arm64.yml new file mode 100644 index 00000000000000..ea96b26c465a7e --- /dev/null +++ b/.github/workflows/mac_arm64.yml @@ -0,0 +1,241 @@ +name: macOS ARM64 (Python 3.11) +on: + workflow_dispatch: + schedule: + # at 00:00 on workdays + - cron: '0 0 * * 1,2,3,4,5' +# pull_request: +# paths-ignore: +# - '**/docs/**' +# - 'docs/**' +# - '**/**.md' +# - '**.md' +# - '**/layer_tests_summary/**' +# - '**/conformance/**' +# push: +# paths-ignore: +# - '**/docs/**' +# - 'docs/**' +# - '**/**.md' +# - '**.md' +# - '**/layer_tests_summary/**' +# - '**/conformance/**' +# branches: +# - master +# - 'releases/**' + +concurrency: + # github.ref is not unique in post-commit + group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-mac-arm64 + cancel-in-progress: true + +env: + PYTHON_VERSION: '3.11' + +jobs: + Smart_CI: + runs-on: ubuntu-latest + outputs: + affected_components: "${{ steps.smart_ci.outputs.affected_components }}" + skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" + steps: + - name: checkout action + uses: actions/checkout@v4 + with: + sparse-checkout: .github/actions/smart-ci + + - name: Get affected components + id: smart_ci + uses: ./.github/actions/smart-ci + with: + repository: ${{ github.repository }} + pr: ${{ github.event.number }} + commit_sha: ${{ github.sha }} + component_pattern: "category: (.*)" + repo_token: ${{ secrets.GITHUB_TOKEN }} + skip_when_only_listed_labels_set: 'docs' + skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg' + + Build: + needs: Smart_CI + timeout-minutes: 150 + defaults: + run: + shell: bash + runs-on: 'macos-13-xlarge' + env: + CMAKE_BUILD_TYPE: 'Release' + CMAKE_GENERATOR: 'Ninja Multi-Config' + MACOSX_DEPLOYMENT_TARGET: '11.0' + CMAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_C_COMPILER_LAUNCHER: ccache + OPENVINO_REPO: ${{ github.workspace }}/openvino + OPENVINO_CONTRIB_REPO: ${{ github.workspace }}/openvino_contrib + INSTALL_DIR: ${{ github.workspace }}/openvino_install + INSTALL_TEST_DIR: ${{ github.workspace }}/tests_install + BUILD_DIR: ${{ github.workspace }}/build + steps: + - name: Clone OpenVINO + uses: actions/checkout@v4 + with: + path: 'openvino' + submodules: 'true' + + - name: Clone OpenVINO Contrib + uses: actions/checkout@v4 + with: + repository: 'openvinotoolkit/openvino_contrib' + path: 'openvino_contrib' + + # + # Print system info + # + + - name: System info + uses: ./openvino/.github/actions/system_info + + # + # Dependencies + # + + - name: Install build dependencies + run: brew install coreutils ninja scons + + - name: Setup Python ${{ env.PYTHON_VERSION }} + uses: ./openvino/.github/actions/setup_python + with: + version: ${{ env.PYTHON_VERSION }} + should-setup-pip-paths: 'false' + self-hosted-runner: 'false' + + - name: Install python dependencies + run: | + # For Python API + python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/wheel/requirements-dev.txt + python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/requirements.txt + + # For running Python API tests + python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/src/compatibility/openvino/requirements-dev.txt + + # For running ONNX frontend unit tests + python3 -m pip install --force-reinstall -r ${{ env.OPENVINO_REPO }}/src/frontends/onnx/tests/requirements.txt + + # For running TensorFlow frontend unit tests + python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/tensorflow/tests/requirements.txt + + # For running Paddle frontend unit tests + python3 -m pip install -r ${{ env.OPENVINO_REPO }}/src/frontends/paddle/tests/requirements.txt + + # + # Build + # + + - name: Setup ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + max-size: "2000M" + # Should save cache only if run in the master branch of the base repo + # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push + save: ${{ github.ref_name == 'master' && 'true' || 'false' }} + verbose: 2 + key: ${{ runner.os }}-${{ runner.arch }}-main + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-main + + - name: CMake configure + run: | + cmake \ + -G "${{ env.CMAKE_GENERATOR }}" \ + -DENABLE_CPPLINT=OFF \ + -DENABLE_NCC_STYLE=OFF \ + -DENABLE_TESTS=ON \ + -DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \ + -DENABLE_STRICT_DEPENDENCIES=OFF \ + -DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CMAKE_CXX_COMPILER_LAUNCHER }} \ + -DCMAKE_C_COMPILER_LAUNCHER=${{ env.CMAKE_C_COMPILER_LAUNCHER }} \ + -S ${{ env.OPENVINO_REPO }} \ + -B ${{ env.BUILD_DIR }} + + - name: Cmake build - OpenVINO + run: cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + + - name: Show ccache stats + run: ccache --show-stats + + - name: Cmake install - OpenVINO + run: | + cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} -P ${{ env.BUILD_DIR }}/cmake_install.cmake + cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_TEST_DIR }} -DCOMPONENT=tests -P ${{ env.BUILD_DIR }}/cmake_install.cmake + cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} -DCOMPONENT=python_wheels -P ${{ env.BUILD_DIR }}/cmake_install.cmake + + - name: Pack Artifacts + run: | + pushd ${{ env.INSTALL_DIR }} + tar -czvf ${{ env.BUILD_DIR }}/openvino_package.tar.gz * + popd + + pushd ${{ env.INSTALL_TEST_DIR }} + tar -czvf ${{ env.BUILD_DIR }}/openvino_tests.tar.gz * + popd + + - name: Cmake & Build - OpenVINO Contrib + run: | + cmake \ + -DBUILD_nvidia_plugin=OFF \ + -DBUILD_java_api=OFF \ + -DCUSTOM_OPERATIONS="calculate_grid;complex_mul;fft;grid_sample;sparse_conv;sparse_conv_transpose" \ + -DOPENVINO_EXTRA_MODULES=${{ env.OPENVINO_CONTRIB_REPO }}/modules \ + -S ${{ env.OPENVINO_REPO }} \ + -B ${{ env.BUILD_DIR }} + cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + + # + # Upload build artifacts + # + + - name: Upload openvino package + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: openvino_package + path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz + if-no-files-found: 'error' + + - name: Upload openvino tests package + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: openvino_tests + path: ${{ env.BUILD_DIR }}/openvino_tests.tar.gz + if-no-files-found: 'error' + + Samples: + needs: Build + uses: ./.github/workflows/job_samples_tests.yml + with: + runner: 'macos-13-xlarge' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} + + CXX_Unit_Tests: + name: C++ unit tests + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_cxx_unit_tests.yml + with: + runner: 'macos-13-xlarge' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} + + Python_Unit_Tests: + name: Python unit tests + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_python_unit_tests.yml + with: + runner: 'macos-13-xlarge' + affected-components: ${{ needs.smart_ci.outputs.affected_components }} + + CPU_Functional_Tests: + name: CPU functional tests + if: fromJSON(needs.smart_ci.outputs.affected_components).CPU.test + needs: [ Build, Smart_CI ] + uses: ./.github/workflows/job_cpu_functional_tests.yml + with: + runner: 'macos-13-xlarge' diff --git a/src/bindings/c/tests/ov_core_test.cpp b/src/bindings/c/tests/ov_core_test.cpp index 90ed036cec32b4..6579c9268418fe 100644 --- a/src/bindings/c/tests/ov_core_test.cpp +++ b/src/bindings/c/tests/ov_core_test.cpp @@ -154,7 +154,12 @@ TEST_P(ov_core_test, ov_core_compile_model) { ov_core_free(core); } +#ifdef OPENVINO_ARCH_ARM64 +// Ticket: 126283 +TEST_P(ov_core_test, DISABLED_ov_core_compile_model_with_property) { +#else TEST_P(ov_core_test, ov_core_compile_model_with_property) { +#endif auto device_name = GetParam(); ov_core_t* core = nullptr; OV_EXPECT_OK(ov_core_create(&core)); diff --git a/src/common/low_precision_transformations/tests/fold_fake_quantize_in_transformations.cpp b/src/common/low_precision_transformations/tests/fold_fake_quantize_in_transformations.cpp index 91d101559c6b36..566806e7ee7af1 100644 --- a/src/common/low_precision_transformations/tests/fold_fake_quantize_in_transformations.cpp +++ b/src/common/low_precision_transformations/tests/fold_fake_quantize_in_transformations.cpp @@ -115,11 +115,20 @@ class FoldFakeQuantizeInTransformations } }; +#ifdef OPENVINO_ARCH_ARM64 +// Ticket: 122660 +TEST_P(FoldFakeQuantizeInTransformations, DISABLED_CompareFunctions) { + actualFunction->validate_nodes_and_infer_types(); + auto res = compare_functions(actualFunction, referenceFunction, true, false); + ASSERT_TRUE(res.first) << res.second; +} +#else TEST_P(FoldFakeQuantizeInTransformations, CompareFunctions) { actualFunction->validate_nodes_and_infer_types(); auto res = compare_functions(actualFunction, referenceFunction, true, false); ASSERT_TRUE(res.first) << res.second; } +#endif const std::vector testValues = { { diff --git a/src/common/transformations/tests/utils/compress_quantize_weights.cpp b/src/common/transformations/tests/utils/compress_quantize_weights.cpp index 0b730d0b36be5a..f76b4c901321d6 100644 --- a/src/common/transformations/tests/utils/compress_quantize_weights.cpp +++ b/src/common/transformations/tests/utils/compress_quantize_weights.cpp @@ -146,7 +146,12 @@ INSTANTIATE_TEST_SUITE_P(TransformationTests, CompressQuantizeWeightsTests, ::testing::Combine(::testing::ValuesIn(params), ::testing::ValuesIn(data_precisions))); +#ifdef OPENVINO_ARCH_ARM64 +// Ticket: 122666 +TEST_F(TransformationTestsF, DISABLED_CompressQuantizeWeightsWithDequantizationSubgraph) { +#else TEST_F(TransformationTestsF, CompressQuantizeWeightsWithDequantizationSubgraph) { +#endif { auto data = opset8::Constant::create(element::f32, Shape{2, 4, 1, 1}, {-1, 0, 1, 2, 3, 4, 5, 11}); auto input_low = opset8::Constant::create(element::f32, Shape{}, {1}); @@ -178,7 +183,12 @@ TEST_F(TransformationTestsF, CompressQuantizeWeightsWithDequantizationSubgraph) comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } +#ifdef OPENVINO_ARCH_ARM64 +// Ticket: 122666 +TEST_F(TransformationTestsF, DISABLED_CompressQuantizeWeightsWithDequantizationSubgraphFP16) { +#else TEST_F(TransformationTestsF, CompressQuantizeWeightsWithDequantizationSubgraphFP16) { +#endif { auto data = opset8::Constant::create(element::f16, Shape{2, 4, 1, 1}, {-1, 0, 1, 2, 3, 4, 5, 11}); auto convert_to_f32 = std::make_shared(data, element::f32); @@ -213,7 +223,12 @@ TEST_F(TransformationTestsF, CompressQuantizeWeightsWithDequantizationSubgraphFP comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } +#ifdef OPENVINO_ARCH_ARM64 +// Ticket: 122666 +TEST_F(TransformationTestsF, DISABLED_CompressQuantizeWeightsWithZeroPointEliminated) { +#else TEST_F(TransformationTestsF, CompressQuantizeWeightsWithZeroPointEliminated) { +#endif { auto data = opset8::Constant::create(element::f32, Shape{3, 1, 1, 1}, {-0.144816, 0.0858578, 0.110928}); auto input_low = opset8::Constant::create(element::f32, Shape{3, 1, 1, 1}, {-0.402659, -0.383148, -0.34054}); @@ -237,7 +252,12 @@ TEST_F(TransformationTestsF, CompressQuantizeWeightsWithZeroPointEliminated) { comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } +#ifdef OPENVINO_ARCH_ARM64 +// Ticket: 122666 +TEST_F(TransformationTestsF, DISABLED_CompressQuantizeWeightsWithZeroPointEliminatedZeroScale) { +#else TEST_F(TransformationTestsF, CompressQuantizeWeightsWithZeroPointEliminatedZeroScale) { +#endif { auto data = opset8::Constant::create(element::f32, Shape{3, 1, 1, 1}, {-0.144816, 0.0858578, 0.110928}); auto input_low = opset8::Constant::create(element::f32, Shape{3, 1, 1, 1}, {-0.402659, -0.383148, -0.34054}); @@ -261,7 +281,12 @@ TEST_F(TransformationTestsF, CompressQuantizeWeightsWithZeroPointEliminatedZeroS comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } +#ifdef OPENVINO_ARCH_ARM64 +// Ticket: 122666 +TEST_F(TransformationTestsF, DISABLED_CompressQuantizeWeightsWithZeroPointEliminatedFP16) { +#else TEST_F(TransformationTestsF, CompressQuantizeWeightsWithZeroPointEliminatedFP16) { +#endif { auto data = opset8::Constant::create(element::f16, Shape{3, 1, 1, 1}, {0.2, 1.2, 1.2}); auto input_low = @@ -290,7 +315,12 @@ TEST_F(TransformationTestsF, CompressQuantizeWeightsWithZeroPointEliminatedFP16) comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } +#ifdef OPENVINO_ARCH_ARM64 +// Ticket: 122666 +TEST_F(TransformationTestsF, DISABLED_NegativeCompressQuantizeWeights) { +#else TEST_F(TransformationTestsF, NegativeCompressQuantizeWeights) { +#endif { auto data = opset8::Constant::create(element::f32, Shape{2, 4, 1, 1}, {-1, 0, 1, 2, 3, 4, 5, 11}); auto input_low = opset8::Constant::create(element::f32, Shape{}, {1}); @@ -315,7 +345,12 @@ TEST_F(TransformationTestsF, NegativeCompressQuantizeWeights) { comparator.enable(FunctionsComparator::CmpValues::ACCURACY); } +#ifdef OPENVINO_ARCH_ARM64 +// Ticket: 122666 +TEST_F(TransformationTestsF, DISABLED_NegativeCompressQuantizeWeightsNonConstantInput) { +#else TEST_F(TransformationTestsF, NegativeCompressQuantizeWeightsNonConstantInput) { +#endif auto data = std::make_shared(element::f32, Shape{2, 4, 1, 1}); auto input_low = opset8::Constant::create(element::f32, Shape{}, {1}); auto input_high = opset8::Constant::create(element::f32, Shape{}, {9}); diff --git a/src/frontends/tensorflow/tests/compilation.cpp b/src/frontends/tensorflow/tests/compilation.cpp index e34a159a45dd06..dc3ef5187b987e 100644 --- a/src/frontends/tensorflow/tests/compilation.cpp +++ b/src/frontends/tensorflow/tests/compilation.cpp @@ -14,6 +14,20 @@ using namespace ov::frontend::tensorflow::tests; class CompileModelsTests : public ::testing::Test {}; +#ifdef OPENVINO_ARCH_ARM64 +// Ticket: 122666 +TEST_F(CompileModelsTests, DISABLED_NgramCompilation) { + ov::Core core; + auto model = convert_model("model_ngram/model_ngram.pbtxt"); + ov::CompiledModel compiled_model = core.compile_model(model, "CPU"); + const auto runtime_model = compiled_model.get_runtime_model(); + + // A convert node will be inserted for CPU plugin API 2.0 + EXPECT_EQ(runtime_model->get_ordered_ops().size(), 5); + EXPECT_EQ(runtime_model->get_parameters().size(), 2); + EXPECT_EQ(runtime_model->get_results().size(), 1); +} +#else TEST_F(CompileModelsTests, NgramCompilation) { ov::Core core; auto model = convert_model("model_ngram/model_ngram.pbtxt"); @@ -25,6 +39,7 @@ TEST_F(CompileModelsTests, NgramCompilation) { EXPECT_EQ(runtime_model->get_parameters().size(), 2); EXPECT_EQ(runtime_model->get_results().size(), 1); } +#endif #ifdef OPENVINO_ARCH_ARM64 // Ticket: CVS-122396 diff --git a/src/frontends/tests/frontend/shared/src/op_fuzzy.cpp b/src/frontends/tests/frontend/shared/src/op_fuzzy.cpp index 8a911e6544f094..90826a6e164e2b 100644 --- a/src/frontends/tests/frontend/shared/src/op_fuzzy.cpp +++ b/src/frontends/tests/frontend/shared/src/op_fuzzy.cpp @@ -107,7 +107,12 @@ void FrontEndFuzzyOpTest::runConvertedModel(const std::shared_ptr mod } } +#ifdef OPENVINO_ARCH_ARM64 +// Ticket: 126830 +TEST_P(FrontEndFuzzyOpTest, DISABLED_testOpFuzzy) { +#else TEST_P(FrontEndFuzzyOpTest, testOpFuzzy) { +#endif // load ASSERT_NO_THROW(doLoadFromFile()); diff --git a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp index 6348e1afccfd40..92e5777a94755c 100644 --- a/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp +++ b/src/plugins/intel_cpu/tests/functional/shared_tests_instances/skip_tests_config.cpp @@ -260,6 +260,13 @@ std::vector disabledTestPatterns() { // Issue: 123321 retVector.emplace_back( R"(.*smoke_RNNSequenceCommonZeroClip/RNNSequenceTest.Inference.*hidden_size=1.*relu.*direction=reverse.*)"); + // Ticket: 122769 + retVector.emplace_back(R"(.*smoke_nonzero/NonZeroLayerTest.Inference/IS.*)"); + retVector.emplace_back(R"(.*smoke_NormalizeL2_.*)"); + retVector.emplace_back(R"(.*Extension.XmlModelWithExtensionFromDSO.*)"); + retVector.emplace_back(R"(.*Extension.OnnxModelWithExtensionFromDSO.*)"); + retVector.emplace_back(R"(.*ONNXQuantizedModels/QuantizedModelsTests.MaxPool.*)"); + retVector.emplace_back(R"(.*ONNXQuantizedModels/QuantizedModelsTests.Convolution.*)"); } // invalid test: checks u8 precision for runtime graph, while it should be f32 retVector.emplace_back(R"(smoke_NegativeQuantizedMatMulMultiplyFusion.*)"); diff --git a/src/plugins/intel_cpu/tests/unit/graph/scaled_attn.cpp b/src/plugins/intel_cpu/tests/unit/graph/scaled_attn.cpp index 91673df7d05a86..48db2d5fd701eb 100644 --- a/src/plugins/intel_cpu/tests/unit/graph/scaled_attn.cpp +++ b/src/plugins/intel_cpu/tests/unit/graph/scaled_attn.cpp @@ -28,7 +28,12 @@ using namespace ov::intel_cpu; +#ifdef OPENVINO_ARCH_ARM64 +// Ticket: 126975 +TEST(ScaledAttnGraphTest, DISABLED_smoke_Check_Scaled_Concat_Noplace) { +#else TEST(ScaledAttnGraphTest, smoke_Check_Scaled_Concat_Noplace) { +#endif auto build_graph = [](const ov::Shape& shape, float* qkv_val, float* past_kv_val) { auto qkv = ov::op::v0::Constant::create(ov::element::f32, shape, qkv_val); qkv->set_friendly_name("qkv_const"); diff --git a/tests/layer_tests/requirements.txt b/tests/layer_tests/requirements.txt index fd15d8bf32405d..fc55322ccb8b32 100644 --- a/tests/layer_tests/requirements.txt +++ b/tests/layer_tests/requirements.txt @@ -8,5 +8,5 @@ torchvision transformers pytest tensorflow-addons; python_version <= '3.10' -jax; sys_platform == "linux" -jaxlib; sys_platform == "linux" +jax; sys_platform == "linux" and platform_machine == "x86_64" # https://jax.readthedocs.io/en/latest/installation.html#pip-installation-cpu - wheels are for "x86_64" only +jaxlib; sys_platform == "linux" and platform_machine == "x86_64" # https://jax.readthedocs.io/en/latest/installation.html#pip-installation-cpu - wheels are for "x86_64" only diff --git a/tests/layer_tests/tensorflow_tests/test_tf_ArgMinMax.py b/tests/layer_tests/tensorflow_tests/test_tf_ArgMinMax.py index ca4f7f051e1079..da60c1ef21a79e 100644 --- a/tests/layer_tests/tensorflow_tests/test_tf_ArgMinMax.py +++ b/tests/layer_tests/tensorflow_tests/test_tf_ArgMinMax.py @@ -1,6 +1,8 @@ # Copyright (C) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +import platform + import numpy as np import pytest import tensorflow as tf @@ -50,6 +52,10 @@ def create_argmin_max_net(self, input_shape, dimension, input_type, output_type, @pytest.mark.parametrize("op_type", [tf.raw_ops.ArgMax, tf.raw_ops.ArgMin]) @pytest.mark.precommit_tf_fe @pytest.mark.nightly + @pytest.mark.xfail(condition=platform.system() == 'Linux' and platform.machine() in ['arm', 'armv7l', + 'aarch64', + 'arm64', 'ARM64'], + reason='Ticket - 126314') def test_argmin_max_net(self, params, input_type, output_type, op_type, ie_device, precision, ir_version, temp_dir, use_new_frontend, use_old_api): self._test(*self.create_argmin_max_net(**params, input_type=input_type,