diff --git a/app/api/api_v1/endpoints/test_run_executions.py b/app/api/api_v1/endpoints/test_run_executions.py index a67a8e98..4926b68d 100644 --- a/app/api/api_v1/endpoints/test_run_executions.py +++ b/app/api/api_v1/endpoints/test_run_executions.py @@ -161,6 +161,11 @@ def start_test_run_execution( status_code=HTTPStatus.NOT_FOUND, detail="Test Run Execution not found" ) + if len(test_run_execution.project.pics.clusters) == 0: + raise HTTPException( + status_code=HTTPStatus.UNPROCESSABLE_ENTITY, detail="No PICS were informed." + ) + test_runner = TestRunner() try: diff --git a/app/tests/api/api_v1/test_test_run_executions.py b/app/tests/api/api_v1/test_test_run_executions.py index cba0091f..b4d829e2 100644 --- a/app/tests/api/api_v1/test_test_run_executions.py +++ b/app/tests/api/api_v1/test_test_run_executions.py @@ -796,6 +796,25 @@ async def test_test_run_execution_start(async_client: AsyncClient, db: Session) assert content["id"] == test_run_execution.id +@pytest.mark.asyncio +async def test_test_run_execution_start_no_pics( + async_client: AsyncClient, db: Session +) -> None: + test_run_execution = create_test_run_execution_with_some_test_cases(db=db, pics={}) + + # First attempt to start test run + response = await async_client.post( + f"{settings.API_V1_STR}/test_run_executions/{test_run_execution.id}/start", + ) + + # Assert 422 UNPROCESSABLE_ENTITY and a detail error message + assert response.status_code == HTTPStatus.UNPROCESSABLE_ENTITY + content = response.json() + assert isinstance(content, dict) + assert "detail" in content.keys() + assert content["detail"] == "No PICS were informed." + + @pytest.mark.asyncio async def test_test_run_execution_busy(async_client: AsyncClient, db: Session) -> None: test_run_execution = create_test_run_execution_with_some_test_cases(db=db) diff --git a/app/tests/utils/test_run_execution.py b/app/tests/utils/test_run_execution.py index e0e76123..4cac7a9f 100644 --- a/app/tests/utils/test_run_execution.py +++ b/app/tests/utils/test_run_execution.py @@ -23,8 +23,10 @@ from app.models import TestRunExecution from app.models.test_enums import TestStateEnum from app.schemas import TestSelection +from app.schemas.pics import PICS from app.schemas.test_run_execution import TestRunExecutionCreate from app.tests.utils.project import create_random_project +from app.tests.utils.test_pics_data import create_random_pics fake = Faker() @@ -85,12 +87,15 @@ def create_random_test_run_execution_archived( def create_random_test_run_execution( - db: Session, selected_tests: Optional[TestSelection] = {}, **kwargs: Any + db: Session, + selected_tests: Optional[TestSelection] = {}, + pics: Optional[PICS] = PICS(), + **kwargs: Any ) -> models.TestRunExecution: test_run_execution_dict = random_test_run_execution_dict(**kwargs) if test_run_execution_dict.get("project_id") is None: - project = create_random_project(db, config={}) + project = create_random_project(db, config={}, pics=pics) test_run_execution_dict["project_id"] = project.id test_run_execution_in = TestRunExecutionCreate(**test_run_execution_dict) @@ -110,7 +115,7 @@ def create_random_test_run_execution_with_test_case_states( "sample_tests": {"SampleTestSuite1": {"TCSS1001": num_test_cases}} } test_run_execution = create_random_test_run_execution( - db=db, selected_tests=selected_tests + db=db, selected_tests=selected_tests, pics=create_random_pics() ) test_suite_execution = test_run_execution.test_suite_executions[0] @@ -128,7 +133,7 @@ def create_random_test_run_execution_with_test_case_states( def create_test_run_execution_with_some_test_cases( - db: Session, **kwargs: Any + db: Session, pics: Optional[PICS] = create_random_pics(), **kwargs: Any ) -> TestRunExecution: return create_random_test_run_execution( db=db, @@ -137,6 +142,7 @@ def create_test_run_execution_with_some_test_cases( "SampleTestSuite1": {"TCSS1001": 1, "TCSS1002": 2, "TCSS1003": 3} } }, + pics=pics, **kwargs ) diff --git a/test_collections/matter/scripts/OTBR/otbr_start.sh b/test_collections/matter/scripts/OTBR/otbr_start.sh index 6e80e300..11b7f66d 100755 --- a/test_collections/matter/scripts/OTBR/otbr_start.sh +++ b/test_collections/matter/scripts/OTBR/otbr_start.sh @@ -1,28 +1,49 @@ #! /usr/bin/env bash - - # - # Copyright (c) 2023 Project CHIP Authors - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. +# +# Copyright (c) 2024 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. ROOT_DIR=$(realpath $(dirname "$0")/../../../../..) TH_SCRIPTS_DIR="$ROOT_DIR/scripts" -DEFAULT_OTBR_INTERFACE="eth0" -BR_INTERFACE=${1:-$DEFAULT_OTBR_INTERFACE} +BR_INTERFACE="eth0" +BR_VARIANT="35" +BR_CHANNEL=25 # The Thread communication channel used BR_IMAGE_BASE="nrfconnect/otbr" BR_IMAGE_TAG="9185bda" BR_IMAGE=$BR_IMAGE_BASE":"$BR_IMAGE_TAG +while getopts ":i:v:" opt; do + case $opt in + i) + echo "Using BR Interface: $OPTARG" >&2 + BR_INTERFACE=$OPTARG + ;; + v) + echo "Using BR Variant: $OPTARG" >&2 + BR_VARIANT=$OPTARG + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac +done + source "$TH_SCRIPTS_DIR/utils.sh" print_start_of_script @@ -47,15 +68,11 @@ sudo docker run --privileged -d --network host --name otbr-chip -e NAT64=1 -e DN print_script_step "Waiting 10 seconds to give the the docker container enough time to start up..." sleep 10 -# The configuration values used below are matching the TH's default Thread configuration -# Please, change the following attributes to create a custom Thread network as desired -# -BR_CHANNEL=15 # The Thread communication channel used BR_CHANNEL_HEX=$(printf '%02x' $BR_CHANNEL) -BR_PANID="1234" # The 2-byte Personal Area Network ID is a unique Thread identifier -BR_EXTPANID="1111111122222222" # The 8-byte Extended Personal Area Network ID is a unique Thread identifier -BR_NETWORKNAME="DEMO" # The human-readable Network Name is a unique Thread identifier -BR_IPV6PREFIX="fd11:22::/64" # The Mesh-Local prefix used to reach interfaces in the same network +BR_PANID="5b${BR_VARIANT}" # The 2-byte Personal Area Network ID is a unique Thread identifier +BR_EXTPANID="5b${BR_VARIANT}dead5b${BR_VARIANT}beef" # The 8-byte Extended Personal Area Network ID is a unique Thread identifier +BR_NETWORKNAME="5b${BR_VARIANT}" # The human-readable Network Name is a unique Thread identifier +BR_IPV6PREFIX="fd11:${BR_VARIANT}::/64" # The Mesh-Local prefix used to reach interfaces in the same network BR_NETWORKKEY="00112233445566778899aabbccddeeff" # The Thread authentication key value BR_PARAMS=( @@ -89,3 +106,4 @@ print_script_step "Restarting the Raspi avahi to have it in a clean state" sudo service avahi-daemon restart print_end_of_script + diff --git a/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py b/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py index b9f1fbf6..d19561f3 100644 --- a/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py +++ b/test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py @@ -36,6 +36,7 @@ "TC_IDM_10_2", "TC_IDM_10_3", "TC_IDM_10_4", + "TC_IDM_10_5", "TC_IDM_12_1", ] @@ -158,8 +159,8 @@ def __parse_test_case( try: tc_desc = __retrieve_description(desc_method) except Exception as e: - logger.error( - f"Error while parsing description for {tc_name}, Error:{str(e)}" + logger.warning( + f"Failed parsing description method for {tc_name}, Error:{str(e)}" ) # If the python test does not implement the steps template method, @@ -170,14 +171,14 @@ def __parse_test_case( try: tc_steps = __retrieve_steps(steps_method) except Exception as e: - logger.error(f"Error while parsing steps for {tc_name}, Error:{str(e)}") + logger.warning(f"Failed parsing steps method for {tc_name}, Error:{str(e)}") pics_method = __get_method_by_name(pics_method_name, methods) if pics_method: try: tc_pics = __retrieve_pics(pics_method) except Exception as e: - logger.error(f"Error while parsing PICS for {tc_name}, Error:{str(e)}") + logger.warning(f"Failed parsing PICS method for {tc_name}, Error:{str(e)}") # - PythonTestType.COMMISSIONING: test cases that have a commissioning first step # - PythonTestType.NO_COMMISSIONING: test cases that follow the expected template @@ -237,7 +238,9 @@ def __retrieve_steps(method: FunctionDefType) -> List[MatterTestStep]: step_name = step.args[ARG_STEP_DESCRIPTION_INDEX].value parsed_step_name = step_name except Exception as e: - logger.error(f"Error while parsing step from {method.name}, Error:{str(e)}") + logger.warning( + f"Failed parsing step name from {method.name}, Error:{str(e)}" + ) parsed_step_name = "UNABLE TO PARSE TEST STEP NAME" python_steps.append( diff --git a/test_collections/matter/sdk_tests/support/python_testing/models/test_case.py b/test_collections/matter/sdk_tests/support/python_testing/models/test_case.py index 0863c074..98e750e4 100644 --- a/test_collections/matter/sdk_tests/support/python_testing/models/test_case.py +++ b/test_collections/matter/sdk_tests/support/python_testing/models/test_case.py @@ -17,7 +17,6 @@ from asyncio import sleep from enum import IntEnum from inspect import iscoroutinefunction -from multiprocessing.managers import BaseManager from pathlib import Path from socket import SocketIO from typing import Any, Optional, Type, TypeVar @@ -38,10 +37,7 @@ from ...sdk_container import SDKContainer from ...utils import prompt_for_commissioning_mode from .python_test_models import PythonTest, PythonTestType -from .python_testing_hooks_proxy import ( - SDKPythonTestResultBase, - SDKPythonTestRunnerHooks, -) +from .python_testing_hooks_proxy import SDKPythonTestResultBase from .utils import ( EXECUTABLE, RUNNER_CLASS_PATH, @@ -115,7 +111,14 @@ def test_skipped(self, filename: str, name: str) -> None: self.skip_to_last_step() def step_skipped(self, name: str, expression: str) -> None: - self.current_test_step.mark_as_not_applicable("Test step skipped") + # From TH perspective, Legacy test cases shows only 2 steps in UI + # but it may have several in the script file. + # So TH should not skip the step in order to keep the test execution flow + skiped_msg = "Test step skipped" + if self.python_test.python_test_type == PythonTestType.LEGACY: + logger.info(skiped_msg) + else: + self.current_test_step.mark_as_not_applicable(skiped_msg) def step_start(self, name: str) -> None: self.step_over() @@ -254,9 +257,7 @@ async def execute(self) -> None: try: logger.info("Running Python Test: " + self.python_test.name) - BaseManager.register("TestRunnerHooks", SDKPythonTestRunnerHooks) - manager = BaseManager(address=("0.0.0.0", 50000), authkey=b"abc") - manager.start() + manager = self.sdk_container.manager test_runner_hooks = manager.TestRunnerHooks() # type: ignore if not self.python_test.path: diff --git a/test_collections/matter/sdk_tests/support/python_testing/models/test_suite.py b/test_collections/matter/sdk_tests/support/python_testing/models/test_suite.py index 943522b4..ac2618ed 100644 --- a/test_collections/matter/sdk_tests/support/python_testing/models/test_suite.py +++ b/test_collections/matter/sdk_tests/support/python_testing/models/test_suite.py @@ -29,8 +29,8 @@ ) from ...sdk_container import SDKContainer -from ...utils import prompt_for_commissioning_mode -from .utils import commission_device +from ...utils import PromptOption, prompt_for_commissioning_mode +from .utils import DUTCommissioningError, commission_device class SuiteType(Enum): @@ -124,7 +124,14 @@ class CommissioningPythonTestSuite(PythonTestSuite, UserPromptSupport): async def setup(self) -> None: await super().setup() - await prompt_for_commissioning_mode(self, logger, None, self.cancel) + user_response = await prompt_for_commissioning_mode( + self, logger, None, self.cancel + ) + + if user_response == PromptOption.FAIL: + raise DUTCommissioningError( + "User chose prompt option FAILED for DUT is in Commissioning Mode" + ) logger.info("Commission DUT") diff --git a/test_collections/matter/sdk_tests/support/sdk_container.py b/test_collections/matter/sdk_tests/support/sdk_container.py index b0f6c8e2..beb2c964 100644 --- a/test_collections/matter/sdk_tests/support/sdk_container.py +++ b/test_collections/matter/sdk_tests/support/sdk_container.py @@ -15,6 +15,7 @@ # from __future__ import annotations +from multiprocessing.managers import BaseManager from pathlib import Path from typing import Optional, Union @@ -29,6 +30,7 @@ from .exec_run_in_container import ExecResultExtended, exec_run_in_container from .pics import set_pics_command +from .python_testing.models.python_testing_hooks_proxy import SDKPythonTestRunnerHooks # Trace mount LOCAL_LOGS_PATH = Path("/var/tmp") @@ -136,6 +138,7 @@ def __init__( self.__pics_file_created = False self.logger = logger + self.manager: BaseManager | None = None @property def pics_file_created(self) -> bool: @@ -150,6 +153,13 @@ def __destroy_existing_container(self) -> None: ) container_manager.destroy(existing_container) + def __create_manager(self) -> BaseManager: + BaseManager.register("TestRunnerHooks", SDKPythonTestRunnerHooks) + manager = BaseManager(address=("0.0.0.0", 50000), authkey=b"abc") + manager.start() + + return manager + def is_running(self) -> bool: if self.__container is None: return False @@ -176,6 +186,9 @@ async def start(self) -> None: self.image_tag, self.run_parameters ) + # Create the BaseManager for multiprocess data share + self.manager = self.__create_manager() + self.logger.info( f"{self.container_name} container started" f" with configuration: {self.run_parameters}" diff --git a/test_collections/matter/sdk_tests/support/tests/test_sdk_container.py b/test_collections/matter/sdk_tests/support/tests/test_sdk_container.py index abe7a897..e5577d84 100644 --- a/test_collections/matter/sdk_tests/support/tests/test_sdk_container.py +++ b/test_collections/matter/sdk_tests/support/tests/test_sdk_container.py @@ -76,6 +76,10 @@ async def test_destroy_container_running() -> None: with mock.patch.object( target=sdk_container, attribute="is_running", return_value=False + ), mock.patch.object( + target=sdk_container, + attribute="_SDKContainer__create_manager", + return_value=None, ), mock.patch.object( target=container_manager, attribute="get_container", return_value=None ), mock.patch.object( @@ -112,6 +116,10 @@ async def test_destroy_container_once() -> None: with mock.patch.object( target=sdk_container, attribute="is_running", return_value=False + ), mock.patch.object( + target=sdk_container, + attribute="_SDKContainer__create_manager", + return_value=None, ), mock.patch.object( target=container_manager, attribute="get_container", return_value=None ), mock.patch.object( @@ -147,6 +155,10 @@ async def test_send_command_default_prefix() -> None: with mock.patch.object( target=sdk_container, attribute="is_running", return_value=False + ), mock.patch.object( + target=sdk_container, + attribute="_SDKContainer__create_manager", + return_value=None, ), mock.patch.object( target=container_manager, attribute="get_container", return_value=None ), mock.patch.object( @@ -188,6 +200,10 @@ async def test_send_command_custom_prefix() -> None: with mock.patch.object( target=sdk_container, attribute="is_running", return_value=False + ), mock.patch.object( + target=sdk_container, + attribute="_SDKContainer__create_manager", + return_value=None, ), mock.patch.object( target=container_manager, attribute="get_container", return_value=None ), mock.patch.object( diff --git a/test_collections/matter/sdk_tests/support/utils.py b/test_collections/matter/sdk_tests/support/utils.py index 7903099a..d35a0e9f 100644 --- a/test_collections/matter/sdk_tests/support/utils.py +++ b/test_collections/matter/sdk_tests/support/utils.py @@ -35,7 +35,7 @@ async def prompt_for_commissioning_mode( logger: loguru.Logger, on_success: Optional[Callable] = None, on_failure: Optional[Callable] = None, -) -> None: +) -> PromptOption: prompt = "Make sure the DUT is in Commissioning Mode" options = { "DONE": PromptOption.PASS, @@ -62,3 +62,4 @@ async def prompt_for_commissioning_mode( f"Received unknown prompt option for \ commissioning step: {prompt_response.response}" ) + return prompt_response.response