Move SDK test stuff to matter test program folder (project-chip#58)

* Move SDK test stuff to matter program folder

* Search for prestart script only into directories

* Fixed prestart script and some references updated

* update python test commit SHA

* fix flake8 error

* fix flake8 error

* Update test_collections/matter/sdk_tests/scripts/fetch_sdk_tests_and_runner.sh

* Update .gitkeep

* Update .gitkeep

* restore version information file

* Update .version_information

* Update test_collections/matter/sdk_tests/sdk_checkout/python_testing/scripts/custom/.gitkeep

* Update test_collections/matter/sdk_tests/sdk_checkout/python_testing/scripts/custom/.gitkeep

* Fix ROOT_DIR folder

* update python test SHA

* fixed unit tests

* update python tests SHA

* checkout submodules on git action workflows

* checkout submodules on git action workflows

* update python tests SHA

.coveragerc

@@ -1,4 +1,4 @@
 [run]
 omit =
         */app/tests/*
-        */test_collections/sdk_tests/support/tests/*
+        */test_collections/matter/sdk_tests/support/tests/*

.flake8

@@ -1,7 +1,7 @@
 [flake8]
 max-line-length = 88
 extend-ignore = E203, W503
-exclude = .git,__pycache__,__init__.py,.mypy_cache,.pytest_cache,third_party,test_collections/sdk_tests/sdk_checkout
+exclude = .git,__pycache__,__init__.py,.mypy_cache,.pytest_cache,third_party,test_collections/matter/sdk_tests/sdk_checkout
 ; Ignore max-line-length in autogenerated files 
 per-file-ignores =
     test_collections/automated_and_semi_automated/**/*:E501

.github/workflows/python-lint.yml

@@ -17,6 +17,8 @@ jobs:
     steps:
       - name: Check out Git repository
         uses: actions/checkout@v3
+        with:
+          submodules: recursive
 
       - name: Set up Python
         uses: actions/setup-python@v1

.github/workflows/spell-check.yml

@@ -11,6 +11,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
       - uses: streetsidesoftware/cspell-action@v2
         with:
           incremental_files_only: false

.github/workflows/unittests.yml

@@ -35,6 +35,8 @@ jobs:
     steps:
       - name: Check out Git repository
         uses: actions/checkout@v3
+        with:
+          submodules: recursive
 
       - name: Load .env file to job environent
         uses: cardinalby/export-env-action@v2
@@ -63,7 +65,7 @@ jobs:
       - name: Fetch yaml tests and runner
         run: |
           source $(poetry env info --path)/bin/activate
-          ./test_collections/sdk_tests/scripts/fetch_sdk_tests_and_runner.sh
+          ./test_collections/matter/sdk_tests/scripts/fetch_sdk_tests_and_runner.sh
         env:
           SERVER_NAME: localhost
           SERVER_HOST: http://localhost

.gitignore

@@ -12,4 +12,4 @@ test_environment.config
 SerialTests.lock
 test_db_creation.lock
 .sha_information
-test_collections/sdk_tests/sdk_checkout
+test_collections/matter/sdk_tests/sdk_checkout

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "python_tests"]
-	path = test_collections/python_tests
+	path = test_collections/matter/python_tests
 	url = https://github.com/project-chip/matter-test-scripts.git

.isort.cfg

@@ -5,4 +5,4 @@ force_grid_wrap = 0
 use_parentheses = True
 ensure_newline_before_comments = True
 line_length = 88
-skip=test_collections/sdk_tests/sdk_checkout
+skip=test_collections/matter/sdk_tests/sdk_checkout

.vscode/settings.json

@@ -35,10 +35,10 @@
     "python.testing.pytestPath": "/usr/local/bin/pytest",
     "python.testing.pytestArgs": [
         "app/tests",
-        "test_collections/sdk_tests/support/tests",
+        "test_collections/matter/sdk_tests/support/tests",
         "--no-cov"
     ],
     "python.linting.ignorePatterns": [
-        "test_collections/sdk_tests/sdk_checkout/**/*.py"
+        "test_collections/matter/sdk_tests/sdk_checkout/**/*.py"
     ]
 }

cspell.json

@@ -97,15 +97,15 @@
     "allowCompoundWords": true,
     "ignorePaths": [
         "logs",
-        "test_collections/python_tests/onboarding_payload_test_suite/",
+        "test_collections/matter/python_tests/onboarding_payload_test_suite/",
         "app/otbr_manager/avahi/avahi-daemon.conf",
         "*.ini",
         "*.toml",
         "docker-compose*",
         "Dockerfile",
         "*.pyc",
         ".vscode/*",
-        "test_collections/sdk_tests/sdk_checkout",
+        "test_collections/matter/sdk_tests/sdk_checkout",
         "sdk_patch",
         ".devcontainer"
     ],

mypy.ini

@@ -25,10 +25,10 @@ warn_unused_ignores = True
 warn_unreachable = True
 namespace_packages = True
 exclude = (?x)(
-    # ignore python files in test_collections/sdk_tests/sdk_checkout
+    # ignore python files in test_collections/matter/sdk_tests/sdk_checkout
     # - sdk_runner
     # - yaml_tests/yaml/sdk
-    ^test_collections\/sdk_tests\/sdk_checkout\/
+    ^test_collections\/matter\/sdk_tests\/sdk_checkout\/
         (
             sdk_runner |
             yaml_tests\/yaml\/sdk |

prestart.sh

@@ -27,9 +27,15 @@ python ./app/initial_data.py
 # Run Prestart scripts in test collections
 for dir in ./test_collections/*
 do
-    prestart=$dir/prestart.sh
-    # Only run prestart.sh if present/
-    [ -x $prestart ] && $prestart
+    if [ -d $dir ]; then 
+        prestart=$dir/prestart.sh
+
+        # Only run prestart.sh if present and it's executable
+        if [ -x $prestart ]; then 
+            echo "Running prestart script: $prestart"
+            $prestart
+        fi
+    fi
 done
 
 # We echo "complete" to ensure this scripts last command has exit code 0.

scripts/test-local.sh

@@ -33,8 +33,8 @@ done
 
 if [[ $RUN_ALL_TESTS -eq 1 ]]; then
     echo "Running all tests"
-    pytest --cov-config=.coveragerc --cov=app --cov=test_collections --cov-report=term-missing app/tests test_collections/sdk_tests/support/tests "${@}"
+    pytest --cov-config=.coveragerc --cov=app --cov=test_collections --cov-report=term-missing app/tests test_collections/matter/sdk_tests/support/tests "${@}"
 else
     echo "Skipping platform dependant tests"
-    pytest --cov-config=.coveragerc --cov=app --cov=test_collections --cov-report=term-missing --ignore=app/tests/platform_dependent_tests app/tests test_collections/sdk_tests/support/tests "${@}"
+    pytest --cov-config=.coveragerc --cov=app --cov=test_collections --cov-report=term-missing --ignore=app/tests/platform_dependent_tests app/tests test_collections/matter/sdk_tests/support/tests "${@}"
 fi

test_collections/sdk_tests/__init__.py → test_collections/matter/__init__.py

@@ -13,5 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from .support.python_testing import custom_python_collection, sdk_python_collection
-from .support.yaml_tests import custom_collection, sdk_collection
+from .python_tests import onboarding_payload_collection
+from .sdk_tests.support.python_testing import (
+    custom_python_collection,
+    sdk_python_collection,
+)
+from .sdk_tests.support.yaml_tests import custom_collection, sdk_collection

test_collections/sdk_tests/prestart.sh → test_collections/matter/prestart.sh

@@ -16,8 +16,10 @@
  # limitations under the License.
 
  # Paths
-SDK_TESTS_DIR=$(dirname "$0")
+SDK_TESTS_DIR=$(dirname "$0")/sdk_tests
+
 cd $SDK_TESTS_DIR
 
 # Fetch code from SDK
-./scripts/fetch_sdk_tests_and_runner.sh
+./scripts/fetch_sdk_tests_and_runner.sh
+

test_collections/matter/sdk_tests/__init__.py

@@ -0,0 +1 @@
+

test_collections/sdk_tests/scripts/fetch_sdk_tests_and_runner.sh → test_collections/matter/sdk_tests/scripts/fetch_sdk_tests_and_runner.sh

@@ -16,14 +16,14 @@ set -e
  # See the License for the specific language governing permissions and
  # limitations under the License.
 
-# Usage: ./test_collections/sdk_tests/fetch_sdk_tests_and_runner.sh [sdk path]
+# Usage: ./test_collections/matter/sdk_tests/fetch_sdk_tests_and_runner.sh [sdk path]
 # 
 # When an SDK path is supplied, the SDK_SHA from .env is ignored.
 # Otherwise a temporary checkout of matter sdk will be made.
 
 # Paths
-ROOT_DIR=$(realpath $(dirname "$0")/../../..)
-TEST_YAML_LOCK_PATH="$ROOT_DIR/.lock-test-yaml"
+ROOT_DIR=$(realpath $(dirname "$0")/../..)
+
 TMP_SDK_FOLDER="sdk-sparse"
 TMP_SDK_PATH="/tmp/$TMP_SDK_FOLDER"
 
@@ -35,11 +35,7 @@ SDK_EXAMPLE_CHIP_TOOL_PATH="examples/chip-tool"
 SDK_EXAMPLE_PLACEHOLDER_PATH="examples/placeholder"
 SDK_DATA_MODEL_PATH="src/app/zap-templates/zcl/data-model/chip"
 
-ENV_FILE="$ROOT_DIR/.env"
-
-TEST_COLLECTIONS_PATH="$ROOT_DIR/test_collections"
-TEST_COLLECTIONS_SDK_CHECKOUT_PATH="$TEST_COLLECTIONS_PATH/sdk_tests/sdk_checkout"
-
+TEST_COLLECTIONS_SDK_CHECKOUT_PATH="$ROOT_DIR/sdk_tests/sdk_checkout"
 
 # YAML Files
 YAML_TEST_COLLECTION_PATH="$TEST_COLLECTIONS_SDK_CHECKOUT_PATH/yaml_tests"
@@ -85,10 +81,6 @@ else
     then 
         echo "Update is forced."
         SDK_CHECKOUT_VERSION=$SDK_SHA
-    elif [ -f "$TEST_YAML_LOCK_PATH" ]
-    then 
-        echo "Test yaml are locked by '.lock-test-yaml', remove file to update."
-        exit 0
     elif [ ! -f "$CURRENT_SDK_CHECKOUT_VERSION" ] || [[ $(< "$CURRENT_SDK_CHECKOUT_VERSION") != "$SDK_SHA" ]]
     then    echo "Current version of test yaml needs to be updated to SDK: $SDK_SHA"
         SDK_CHECKOUT_VERSION=$SDK_SHA
@@ -143,11 +135,6 @@ cd "$SDK_PATH/$SDK_PYTHON_DATA_MODEL_PATH"
 mkdir -p "$PYTHON_TESTING_TEST_COLLECTION_PATH/data_model"
 cp -R * "$PYTHON_TESTING_TEST_COLLECTION_PATH/data_model"
 
-# Delete deprecated codegenerated python wrappers for yaml 
-rm -Rf "$TEST_COLLECTIONS_PATH/manual_tests"
-rm -Rf "$TEST_COLLECTIONS_PATH/automated_and_semi_automated"
-rm -Rf "$TEST_COLLECTIONS_PATH/app1_tests"
-
 ###
 # Extract sdk runner and dependencies
 ###

test_collections/sdk_tests/sdk_checkout/python_testing/scripts/custom/.gitkeep → test_collections/matter/sdk_tests/sdk_checkout/python_testing/scripts/custom/.gitkeep

@@ -1,3 +1,3 @@
-- This is a placeholder file for the custom Python test folder
-- Use this directory to include all customized Python test files
-- The tests will be presented in a new test collection on the TH UI
+- This is a placeholder file for the custom Python test folder
+- Use this directory to include all customized Python test files
+- The tests will be presented in a new test collection on the TH UI

test_collections/sdk_tests/sdk_checkout/yaml_tests/yaml/custom/.gitkeep → test_collections/matter/sdk_tests/sdk_checkout/yaml_tests/yaml/custom/.gitkeep

@@ -1,3 +1,3 @@
-- This is a placeholder file for the custom YAML folder
-- Use this directory to include all customized YAML files
-- The tests will be presented in a new test collection on the TH UI
+- This is a placeholder file for the custom YAML folder
+- Use this directory to include all customized YAML files
+- The tests will be presented in a new test collection on the TH UI

test_collections/sdk_tests/support/chip/chip_tool.py → test_collections/matter/sdk_tests/support/chip/chip_tool.py

@@ -43,13 +43,10 @@
 from app.singleton import Singleton
 from app.test_engine.logger import CHIP_LOG_FORMAT, CHIPTOOL_LEVEL
 from app.test_engine.logger import test_engine_logger as logger
-from test_collections.sdk_tests.support.paths import SDK_CHECKOUT_PATH
-from test_collections.sdk_tests.support.pics import PICS_FILE_PATH, set_pics_command
-from test_collections.sdk_tests.support.sdk_container import (
-    DOCKER_LOGS_PATH,
-    DOCKER_PAA_CERTS_PATH,
-    SDKContainer,
-)
+
+from ..paths import SDK_CHECKOUT_PATH
+from ..pics import PICS_FILE_PATH, set_pics_command
+from ..sdk_container import DOCKER_LOGS_PATH, DOCKER_PAA_CERTS_PATH, SDKContainer
 
 # Chip Tool Parameters
 CHIP_TOOL_EXE = "./chip-tool"

test_collections/sdk_tests/support/models/sdk_test_folder.py → test_collections/matter/sdk_tests/support/models/sdk_test_folder.py

@@ -15,7 +15,7 @@
 #
 from pathlib import Path
 
-from test_collections.sdk_tests.support.paths import SDK_CHECKOUT_PATH
+from ..paths import SDK_CHECKOUT_PATH
 
 UNKNOWN_version = "Unknown"
 VERSION_FILE_FILENAME = ".version"
@@ -34,7 +34,7 @@ def __init__(self, path: Path, filename_pattern: str = "*") -> None:
 
     def __version(self) -> str:
         """Read version string from .version file in
-        /app/backend/test_collections/sdk_tests/sdk_checkout path."""
+        /app/backend/test_collections/matter/sdk_tests/sdk_checkout path."""
         version_file_path = SDK_CHECKOUT_PATH / VERSION_FILE_FILENAME
 
         if not version_file_path.exists():

test_collections/sdk_tests/support/python_testing/models/python_test_parser.py → test_collections/matter/sdk_tests/support/python_testing/models/python_test_parser.py

@@ -18,11 +18,7 @@
 from pathlib import Path
 from typing import Any, List, Optional, Union
 
-from test_collections.sdk_tests.support.models.matter_test_models import (
-    MatterTestStep,
-    MatterTestType,
-)
-
+from ...models.matter_test_models import MatterTestStep, MatterTestType
 from .python_test_models import PythonTest, PythonTestType
 
 ARG_STEP_DESCRIPTION_INDEX = 1

test_collections/sdk_tests/support/python_testing/models/test_case.py → test_collections/matter/sdk_tests/support/python_testing/models/test_case.py

@@ -25,10 +25,10 @@
 from app.test_engine.models.test_case import CUSTOM_TEST_IDENTIFIER
 from app.user_prompt_support.prompt_request import OptionsSelectPromptRequest
 from app.user_prompt_support.user_prompt_support import UserPromptSupport
-from test_collections.sdk_tests.support.chip.chip_tool import PICS_FILE_PATH
-from test_collections.sdk_tests.support.sdk_container import SDKContainer
-from test_collections.sdk_tests.support.utils import prompt_for_commissioning_mode
 
+from ...chip.chip_tool import PICS_FILE_PATH
+from ...sdk_container import SDKContainer
+from ...utils import prompt_for_commissioning_mode
 from .python_test_models import PythonTest, PythonTestType
 from .python_testing_hooks_proxy import (
     SDKPythonTestResultBase,

test_collections/sdk_tests/support/python_testing/models/test_declarations.py → test_collections/matter/sdk_tests/support/python_testing/models/test_declarations.py

@@ -20,8 +20,8 @@
     TestCollectionDeclaration,
     TestSuiteDeclaration,
 )
-from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder
 
+from ...models.sdk_test_folder import SDKTestFolder
 from .python_test_models import MatterTestType, PythonTest
 from .test_case import PythonTestCase
 from .test_suite import PythonTestSuite, SuiteType

test_collections/sdk_tests/support/python_testing/models/test_suite.py → test_collections/matter/sdk_tests/support/python_testing/models/test_suite.py

@@ -19,11 +19,10 @@
 from app.test_engine.logger import test_engine_logger as logger
 from app.test_engine.models import TestSuite
 from app.user_prompt_support.user_prompt_support import UserPromptSupport
-from test_collections.sdk_tests.support.python_testing.models.utils import (
-    commission_device,
-)
-from test_collections.sdk_tests.support.sdk_container import SDKContainer
-from test_collections.sdk_tests.support.utils import prompt_for_commissioning_mode
+
+from ...sdk_container import SDKContainer
+from ...utils import prompt_for_commissioning_mode
+from .utils import commission_device
 
 
 class SuiteType(Enum):

test_collections/sdk_tests/support/python_testing/models/utils.py → test_collections/matter/sdk_tests/support/python_testing/models/utils.py

@@ -22,7 +22,8 @@
 
 from app.schemas.test_environment_config import TestEnvironmentConfig
 from app.test_engine.logger import PYTHON_TEST_LEVEL
-from test_collections.sdk_tests.support.sdk_container import SDKContainer
+
+from ...sdk_container import SDKContainer
 
 # Command line params
 RUNNER_CLASS_PATH = "/root/python_testing/scripts/sdk/test_harness_client.py"

test_collections/sdk_tests/support/python_testing/sdk_python_tests.py → test_collections/matter/sdk_tests/support/python_testing/sdk_python_tests.py

@@ -16,9 +16,8 @@
 from pathlib import Path
 from typing import Optional
 
-from test_collections.sdk_tests.support.models.sdk_test_folder import SDKTestFolder
-from test_collections.sdk_tests.support.paths import SDK_CHECKOUT_PATH
-
+from ..models.sdk_test_folder import SDKTestFolder
+from ..paths import SDK_CHECKOUT_PATH
 from .models.python_test_models import PythonTestType
 from .models.python_test_parser import parse_python_script
 from .models.test_declarations import (
@@ -30,9 +29,9 @@
 
 ###
 # This file hosts logic to load and parse Python test cases, located in
-# `test_collections/sdk_tests/sdk_checkout/python_testing/scripts/sdk`.
+# `test_collections/matter/sdk_tests/sdk_checkout/python_testing/scripts/sdk`.
 # The `sdk` sub-folder here is automatically maintained using the
-# `test_collections/sdk_tests/fetch_sdk_tests_and_runner.sh` script.
+# `test_collections/matter/sdk_tests/fetch_sdk_tests_and_runner.sh` script.
 #
 # The Python Tests are organized into 1 Test Suite:
 #        - Automated

test_collections/sdk_tests/support/sdk_container.py → test_collections/matter/sdk_tests/support/sdk_container.py

@@ -26,9 +26,9 @@
 from app.schemas.pics import PICS, PICSError
 from app.singleton import Singleton
 from app.test_engine.logger import test_engine_logger as logger
-from test_collections.sdk_tests.support.pics import set_pics_command
 
 from .exec_run_in_container import ExecResultExtended, exec_run_in_container
+from .pics import set_pics_command
 
 # Trace mount
 LOCAL_LOGS_PATH = Path("/var/tmp")
@@ -43,7 +43,10 @@
 DOCKER_CREDENTIALS_DEVELOPMENT_PATH = "/credentials/development"
 
 # Python Testing Folder
-LOCAL_TEST_COLLECTIONS_PATH = "/home/ubuntu/certification-tool/backend/test_collections"
+LOCAL_TEST_COLLECTIONS_PATH = (
+    "/home/ubuntu/certification-tool/backend/test_collections/matter"
+)
+
 LOCAL_PYTHON_TESTING_PATH = Path(
     LOCAL_TEST_COLLECTIONS_PATH + "/sdk_tests/sdk_checkout/python_testing"
 )