🎨E2E: improvements on ClassicTIP test (#5955)

.vscode/extensions.json

@@ -5,12 +5,10 @@
     "DevSoft.svg-viewer-vscode",
     "eamodio.gitlens",
     "exiasr.hadolint",
-    "hediet.vscode-drawio",
     "ms-azuretools.vscode-docker",
     "ms-python.black-formatter",
     "ms-python.pylint",
     "ms-python.python",
-    "ms-vscode.makefile-tools",
     "njpwerner.autodocstring",
     "samuelcolvin.jinjahtml",
     "timonwong.shellcheck",

packages/pytest-simcore/src/pytest_simcore/logging_utils.py

@@ -124,8 +124,7 @@ def log_context(
         error_message = (
             f"{ctx_msg.raised} ({_timedelta_as_minute_second_ms(elapsed_time)})"
         )
-        logger.log(
-            logging.ERROR,
+        logger.exception(
             error_message,
             *args,
             **kwargs,

packages/pytest-simcore/src/pytest_simcore/playwright_utils.py

@@ -1,15 +1,21 @@
+import contextlib
 import json
 import logging
+import re
+from collections import defaultdict
 from contextlib import ExitStack
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import Enum, unique
 from typing import Any, Final
 
-from playwright.sync_api import WebSocket
+from playwright.sync_api import FrameLocator, Page, Request, WebSocket, expect
 from pytest_simcore.logging_utils import log_context
 
 SECOND: Final[int] = 1000
 MINUTE: Final[int] = 60 * SECOND
+NODE_START_REQUEST_PATTERN: Final[re.Pattern[str]] = re.compile(
+    r"/projects/[^/]+/nodes/[^:]+:start"
+)
 
 
 @unique
@@ -42,6 +48,28 @@ def is_running(self) -> bool:
         )
 
 
+@unique
+class NodeProgressType(str, Enum):
+    # NOTE: this is a partial duplicate of models_library/rabbitmq_messages.py
+    # It must remain as such until that module is pydantic V2 compatible
+    CLUSTER_UP_SCALING = "CLUSTER_UP_SCALING"
+    SERVICE_INPUTS_PULLING = "SERVICE_INPUTS_PULLING"
+    SIDECARS_PULLING = "SIDECARS_PULLING"
+    SERVICE_OUTPUTS_PULLING = "SERVICE_OUTPUTS_PULLING"
+    SERVICE_STATE_PULLING = "SERVICE_STATE_PULLING"
+    SERVICE_IMAGES_PULLING = "SERVICE_IMAGES_PULLING"
+
+    @classmethod
+    def required_types_for_started_service(cls) -> set["NodeProgressType"]:
+        return {
+            NodeProgressType.SERVICE_INPUTS_PULLING,
+            NodeProgressType.SIDECARS_PULLING,
+            NodeProgressType.SERVICE_OUTPUTS_PULLING,
+            NodeProgressType.SERVICE_STATE_PULLING,
+            NodeProgressType.SERVICE_IMAGES_PULLING,
+        }
+
+
 class ServiceType(str, Enum):
     DYNAMIC = "DYNAMIC"
     COMPUTATIONAL = "COMPUTATIONAL"
@@ -84,6 +112,28 @@ def retrieve_project_state_from_decoded_message(event: SocketIOEvent) -> Running
     return RunningState(event.obj["data"]["state"]["value"])
 
 
+@dataclass(frozen=True, slots=True, kw_only=True)
+class NodeProgressEvent:
+    node_id: str
+    progress_type: NodeProgressType
+    current_progress: float
+    total_progress: float
+
+
+def retrieve_node_progress_from_decoded_message(
+    event: SocketIOEvent,
+) -> NodeProgressEvent:
+    assert event.name == _OSparcMessages.NODE_PROGRESS.value
+    assert "progress_type" in event.obj
+    assert "progress_report" in event.obj
+    return NodeProgressEvent(
+        node_id=event.obj["node_id"],
+        progress_type=NodeProgressType(event.obj["progress_type"]),
+        current_progress=float(event.obj["progress_report"]["actual_value"]),
+        total_progress=float(event.obj["progress_report"]["total"]),
+    )
+
+
 @dataclass
 class SocketIOProjectClosedWaiter:
     def __call__(self, message: str) -> bool:
@@ -139,6 +189,44 @@ def __call__(self, message: str) -> None:
                 print("WS Message:", decoded_message.name, decoded_message.obj)
 
 
+@dataclass
+class SocketIONodeProgressCompleteWaiter:
+    node_id: str
+    _current_progress: dict[NodeProgressType, float] = field(
+        default_factory=defaultdict
+    )
+
+    def __call__(self, message: str) -> bool:
+        with log_context(logging.DEBUG, msg=f"handling websocket {message=}") as ctx:
+            # socket.io encodes messages like so
+            # https://stackoverflow.com/questions/24564877/what-do-these-numbers-mean-in-socket-io-payload
+            if message.startswith(_SOCKETIO_MESSAGE_PREFIX):
+                decoded_message = decode_socketio_42_message(message)
+                if decoded_message.name == _OSparcMessages.NODE_PROGRESS.value:
+                    node_progress_event = retrieve_node_progress_from_decoded_message(
+                        decoded_message
+                    )
+                    if node_progress_event.node_id == self.node_id:
+                        self._current_progress[node_progress_event.progress_type] = (
+                            node_progress_event.current_progress
+                            / node_progress_event.total_progress
+                        )
+                        ctx.logger.info(
+                            "current startup progress: %s",
+                            f"{json.dumps({k:round(v,1) for k,v in self._current_progress.items()})}",
+                        )
+
+                    return all(
+                        progress_type in self._current_progress
+                        for progress_type in NodeProgressType.required_types_for_started_service()
+                    ) and all(
+                        round(progress, 1) == 1.0
+                        for progress in self._current_progress.values()
+                    )
+
+            return False
+
+
 def wait_for_pipeline_state(
     current_state: RunningState,
     *,
@@ -187,3 +275,52 @@ def on_web_socket_default_handler(ws) -> None:
     ws.on("framesent", lambda payload: ctx.logger.info("⬇️ %s", payload))
     ws.on("framereceived", lambda payload: ctx.logger.info("⬆️ %s", payload))
     ws.on("close", lambda payload: stack.close())  # noqa: ARG005
+
+
+def _node_started_predicate(request: Request) -> bool:
+    return bool(
+        re.search(NODE_START_REQUEST_PATTERN, request.url)
+        and request.method.upper() == "POST"
+    )
+
+
+def _trigger_service_start_if_button_available(page: Page, node_id: str) -> None:
+    # wait for the start button to auto-disappear if it is still around after the timeout, then we click it
+    with log_context(logging.INFO, msg="trigger start button if needed") as ctx:
+        start_button_locator = page.get_by_test_id(f"Start_{node_id}")
+        with contextlib.suppress(AssertionError, TimeoutError):
+            expect(start_button_locator).to_be_visible(timeout=5000)
+            expect(start_button_locator).to_be_enabled(timeout=5000)
+            with page.expect_request(_node_started_predicate):
+                start_button_locator.click()
+                ctx.logger.info("triggered start button")
+
+
+def wait_for_service_running(
+    *,
+    page: Page,
+    node_id: str,
+    websocket: WebSocket,
+    timeout: int,
+) -> FrameLocator:
+    """NOTE: if the service was already started this will not work as some of the required websocket events will not be emitted again
+    In which case this will need further adjutment"""
+
+    waiter = SocketIONodeProgressCompleteWaiter(node_id=node_id)
+    with (
+        log_context(logging.INFO, msg="Waiting for node to run"),
+        websocket.expect_event("framereceived", waiter, timeout=timeout),
+    ):
+        _trigger_service_start_if_button_available(page, node_id)
+    return page.frame_locator(f'[osparc-test-id="iframe_{node_id}"]')
+
+
+def app_mode_trigger_next_app(page: Page) -> None:
+    with (
+        log_context(logging.INFO, msg="triggering next app"),
+        page.expect_request(_node_started_predicate),
+    ):
+        # Move to next step (this auto starts the next service)
+        next_button_locator = page.get_by_test_id("AppMode_NextBtn")
+        if next_button_locator.is_visible() and next_button_locator.is_enabled():
+            page.get_by_test_id("AppMode_NextBtn").click()

scripts/maintenance/computational-clusters/autoscaled_monitor/cli.py

@@ -81,6 +81,7 @@ def main(
         if "license" in file_path.name:
             continue
         # very bad HACK
+        rich.print(f"checking {file_path.name}")
         if (
             any(_ in f"{file_path}" for _ in ("sim4life.io", "osparc-master"))
             and "openssh" not in f"{file_path}"

scripts/maintenance/computational-clusters/autoscaled_monitor/constants.py

@@ -8,7 +8,9 @@
     str
 ] = r"osparc-computational-cluster-{role}-{swarm_stack_name}-user_id:{user_id:d}-wallet_id:{wallet_id:d}"
 DEFAULT_DYNAMIC_EC2_FORMAT: Final[str] = r"osparc-dynamic-autoscaled-worker-{key_name}"
-DEPLOY_SSH_KEY_PARSER: Final[parse.Parser] = parse.compile(r"osparc-{random_name}.pem")
+DEPLOY_SSH_KEY_PARSER: Final[parse.Parser] = parse.compile(
+    r"{base_name}-{random_name}.pem"
+)
 
 MINUTE: Final[int] = 60
 HOUR: Final[int] = 60 * MINUTE

tests/e2e-playwright/.gitignore

@@ -1,6 +1,5 @@
 test-results
 assets
 report.html
-.e2e-playwright-env.txt
-.e2e-playwright-jupyterlab-env.txt
+.e2e-playwright-*.txt
 report.xml

tests/e2e-playwright/Makefile

@@ -92,7 +92,7 @@ test-sleepers: _check_venv_active ## runs sleepers test on local deploy
 		--product-url=http://$(get_my_ip):9081 \
 		--autoregister \
 		--tracing=retain-on-failure \
-		$(CURDIR)/tests/sleepers/sleepers.py
+		$(CURDIR)/tests/sleepers/test_sleepers.py
 
 
 .PHONY: test-sleepers-dev
@@ -104,63 +104,56 @@ test-sleepers-dev: _check_venv_active ## runs sleepers test on local deploy
 		--product-url=http://$(get_my_ip):9081 \
 		--headed \
 		--autoregister \
-		$(CURDIR)/tests/sleepers/sleepers.py
+		$(CURDIR)/tests/sleepers/test_sleepers.py
+
+
+# Define the files where user input will be saved
+SLEEPERS_INPUT_FILE := .e2e-playwright-sleepers-env.txt
+JUPYTER_LAB_INPUT_FILE := .e2e-playwright-jupyterlab-env.txt
+CLASSIC_TIP_INPUT_FILE := .e2e-playwright-classictip-env.txt
+
+# Prompt the user for input and store it into variables
+$(SLEEPERS_INPUT_FILE) $(JUPYTER_LAB_INPUT_FILE) $(CLASSIC_TIP_INPUT_FILE):
+	@read -p "Enter your product URL: " PRODUCT_URL; \
+	read -p "Is the product billable [y/n]: " BILLABLE; \
+	read -p "Is the test running in autoscaled deployment [y/n]: " AUTOSCALED; \
+	read -p "Enter your username: " USER_NAME; \
+	read -s -p "Enter your password: " PASSWORD; echo ""; \
+	echo "--product-url=$$PRODUCT_URL --user-name=$$USER_NAME --password=$$PASSWORD" > $@; \
+	if [ "$$BILLABLE" = "y" ]; then \
+		echo "--product-billable" >> $@; \
+	fi; \
+	if [ "$$AUTOSCALED" = "y" ]; then \
+		echo "--autoscaled" >> $@; \
+	fi; \
+	if [ "$@" = "$(JUPYTER_LAB_INPUT_FILE)" ]; then \
+		read -p "Enter the size of the large file (human readable form e.g. 3Gib): " LARGE_FILE_SIZE; \
+		echo "--service-key=jupyter-math --large-file-size=$$LARGE_FILE_SIZE" >> $@; \
+	elif [ "$@" = "$(SLEEPERS_INPUT_FILE)" ]; then \
+		read -p "Enter the number of sleepers: " NUM_SLEEPERS; \
+		echo "--num-sleepers=$$NUM_SLEEPERS" >> $@; \
+	fi
 
+# Run the tests
+test-sleepers-anywhere: _check_venv_active $(SLEEPERS_INPUT_FILE)
+	@$(call run_test, $(SLEEPERS_INPUT_FILE), tests/sleepers/test_sleepers.py)
 
-# Define the file where user input will be saved
-USER_INPUT_FILE := .e2e-playwright-env.txt
-$(USER_INPUT_FILE):## Prompt the user for input and store it into variables
-		@read -p "Enter your product URL: " PRODUCT_URL; \
-		read -p "Is the product billable [y/n]: " BILLABLE; \
-		read -p "Enter your username: " USER_NAME; \
-		read -s -p "Enter your password: " PASSWORD; echo ""; \
-		read -p "Enter the number of sleepers: " NUM_SLEEPERS; \
-		echo "$$PRODUCT_URL $$USER_NAME $$PASSWORD $$NUM_SLEEPERS $$BILLABLE" > $(USER_INPUT_FILE)
-
-# Read user input from the file and run the test
-test-sleepers-anywhere: _check_venv_active $(USER_INPUT_FILE) ## test sleepers anywhere and keeps a cache as to where
-	@IFS=' ' read -r PRODUCT_URL USER_NAME PASSWORD NUM_SLEEPERS BILLABLE < $(USER_INPUT_FILE); \
-	BILLABLE_FLAG=""; \
-    if [ "$$BILLABLE" = "y" ]; then \
-        BILLABLE_FLAG="--product-billable"; \
-    fi; \
-	pytest -s tests/sleepers/sleepers.py \
-		--color=yes \
-		--product-url=$$PRODUCT_URL \
-		--user-name=$$USER_NAME \
-		--password=$$PASSWORD \
-		--num-sleepers=$$NUM_SLEEPERS \
-		$$BILLABLE_FLAG \
-		--browser chromium \
-		--headed
-
-# Define the file where user input will be saved
-JUPYTER_USER_INPUT_FILE := .e2e-playwright-jupyterlab-env.txt
-$(JUPYTER_USER_INPUT_FILE): ## Prompt the user for input and store it into variables
-		@read -p "Enter your product URL: " PRODUCT_URL; \
-		read -p "Is the product billable [y/n]: " BILLABLE; \
-		read -p "Enter your username: " USER_NAME; \
-		read -s -p "Enter your password: " PASSWORD; echo ""; \
-		read -p "Enter the size of the large file (human readable form e.g. 3Gib): " LARGE_FILE_SIZE; \
-		echo "$$PRODUCT_URL $$USER_NAME $$PASSWORD $$LARGE_FILE_SIZE $$BILLABLE" > $(JUPYTER_USER_INPUT_FILE)
-
-test-jupyterlab-anywhere: _check_venv_active $(JUPYTER_USER_INPUT_FILE) ## test jupyterlabs anywhere and keeps a cache as to where
-	@IFS=' ' read -r PRODUCT_URL USER_NAME PASSWORD LARGE_FILE_SIZE BILLABLE < $(JUPYTER_USER_INPUT_FILE); \
-	BILLABLE_FLAG=""; \
-    if [ "$$BILLABLE" = "y" ]; then \
-        BILLABLE_FLAG="--product-billable"; \
-    fi; \
-	pytest -s tests/jupyterlabs/ \
+test-jupyterlab-anywhere: _check_venv_active $(JUPYTER_LAB_INPUT_FILE)
+	@$(call run_test, $(JUPYTER_LAB_INPUT_FILE), tests/jupyterlabs/test_jupyterlab.py)
+
+test-tip-anywhere: _check_venv_active $(CLASSIC_TIP_INPUT_FILE)
+	$(call run_test, $(CLASSIC_TIP_INPUT_FILE), tests/tip/test_ti_plan.py)
+
+# Define the common test running function
+define run_test
+	TEST_ARGS=$$(cat $1 | xargs); \
+	echo $$TEST_ARGS; \
+	pytest -s $2 \
 		--color=yes \
-		--product-url=$$PRODUCT_URL \
-		--user-name=$$USER_NAME \
-		--password=$$PASSWORD \
-		--large-file-size=$$LARGE_FILE_SIZE \
-		--service-key=jupyter-math \
-		$$BILLABLE_FLAG \
 		--browser chromium \
-		--headed
+		--headed \
+		$$TEST_ARGS
+endef
 
 clean:
-	@rm -rf $(USER_INPUT_FILE)
-	@rm -rf $(JUPYTER_USER_INPUT_FILE)
+	@rm -rf $(SLEEPERS_INPUT_FILE) $(JUPYTER_LAB_INPUT_FILE) $(CLASSIC_TIP_INPUT_FILE)