Fix issue with Workflows blocks for Roboflow models v2 not using base64

inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v1.py

@@ -347,7 +347,7 @@ def _post_process_result(
         predictions: List[dict],
         class_filter: Optional[List[str]],
     ) -> BlockResult:
-        inference_id = predictions[0].get(INFERENCE_ID_KEY, None)
+        inference_ids = [p.get(INFERENCE_ID_KEY, None) for p in predictions]
         predictions = convert_inference_detections_batch_to_sv_detections(predictions)
         predictions = attach_prediction_type_info_to_sv_detections_batch(
             predictions=predictions,
@@ -363,5 +363,5 @@ def _post_process_result(
         )
         return [
             {"inference_id": inference_id, "predictions": prediction}
-            for prediction in predictions
+            for inference_id, prediction in zip(inference_ids, predictions)
         ]

inference/core/workflows/core_steps/models/roboflow/instance_segmentation/v2.py

@@ -325,7 +325,7 @@ def run_remotely(
             source="workflow-execution",
         )
         client.configure(inference_configuration=client_config)
-        inference_images = [i.numpy_image for i in images]
+        inference_images = [i.base64_image for i in images]
         predictions = client.infer(
             inference_input=inference_images,
             model_id=model_id,
@@ -344,7 +344,7 @@ def _post_process_result(
         predictions: List[dict],
         class_filter: Optional[List[str]],
     ) -> BlockResult:
-        inference_id = predictions[0].get(INFERENCE_ID_KEY, None)
+        inference_ids = [p.get(INFERENCE_ID_KEY, None) for p in predictions]
         predictions = convert_inference_detections_batch_to_sv_detections(predictions)
         predictions = attach_prediction_type_info_to_sv_detections_batch(
             predictions=predictions,
@@ -360,5 +360,5 @@ def _post_process_result(
         )
         return [
             {"inference_id": inference_id, "predictions": prediction}
-            for prediction in predictions
+            for inference_id, prediction in zip(inference_ids, predictions)
         ]

inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v1.py

@@ -332,7 +332,7 @@ def _post_process_result(
         predictions: List[dict],
         class_filter: Optional[List[str]],
     ) -> BlockResult:
-        inference_id = predictions[0].get(INFERENCE_ID_KEY, None)
+        inference_ids = [p.get(INFERENCE_ID_KEY, None) for p in predictions]
         detections = convert_inference_detections_batch_to_sv_detections(predictions)
         for prediction, image_detections in zip(predictions, detections):
             add_inference_keypoints_to_sv_detections(
@@ -353,5 +353,5 @@ def _post_process_result(
         )
         return [
             {"inference_id": inference_id, "predictions": image_detections}
-            for image_detections in detections
+            for inference_id, image_detections in zip(inference_ids, detections)
         ]

inference/core/workflows/core_steps/models/roboflow/keypoint_detection/v2.py

@@ -309,7 +309,7 @@ def run_remotely(
             source="workflow-execution",
         )
         client.configure(inference_configuration=client_config)
-        inference_images = [i.numpy_image for i in images]
+        inference_images = [i.base64_image for i in images]
         predictions = client.infer(
             inference_input=inference_images,
             model_id=model_id,
@@ -328,7 +328,7 @@ def _post_process_result(
         predictions: List[dict],
         class_filter: Optional[List[str]],
     ) -> BlockResult:
-        inference_id = predictions[0].get(INFERENCE_ID_KEY, None)
+        inference_ids = [p.get(INFERENCE_ID_KEY, None) for p in predictions]
         detections = convert_inference_detections_batch_to_sv_detections(predictions)
         for prediction, image_detections in zip(predictions, detections):
             add_inference_keypoints_to_sv_detections(
@@ -349,5 +349,5 @@ def _post_process_result(
         )
         return [
             {"inference_id": inference_id, "predictions": image_detections}
-            for image_detections in detections
+            for inference_id, image_detections in zip(inference_ids, detections)
         ]

inference/core/workflows/core_steps/models/roboflow/multi_class_classification/v2.py

@@ -221,7 +221,7 @@ def run_remotely(
             source="workflow-execution",
         )
         client.configure(inference_configuration=client_config)
-        non_empty_inference_images = [i.numpy_image for i in images]
+        non_empty_inference_images = [i.base64_image for i in images]
         predictions = client.infer(
             inference_input=non_empty_inference_images,
             model_id=model_id,

inference/core/workflows/core_steps/models/roboflow/multi_label_classification/v2.py

@@ -221,7 +221,7 @@ def run_remotely(
             source="workflow-execution",
         )
         client.configure(inference_configuration=client_config)
-        non_empty_inference_images = [i.numpy_image for i in images]
+        non_empty_inference_images = [i.base64_image for i in images]
         predictions = client.infer(
             inference_input=non_empty_inference_images,
             model_id=model_id,
@@ -235,7 +235,6 @@ def _post_process_result(
         images: Batch[WorkflowImageData],
         predictions: List[dict],
     ) -> List[dict]:
-        inference_id = predictions[0].get(INFERENCE_ID_KEY, None)
         predictions = attach_prediction_type_info(
             predictions=predictions,
             prediction_type="classification",

inference/core/workflows/core_steps/models/roboflow/object_detection/v1.py

@@ -314,7 +314,7 @@ def _post_process_result(
         predictions: List[dict],
         class_filter: Optional[List[str]],
     ) -> BlockResult:
-        inference_id = predictions[0].get(INFERENCE_ID_KEY, None)
+        inference_ids = [p.get(INFERENCE_ID_KEY, None) for p in predictions]
         predictions = convert_inference_detections_batch_to_sv_detections(predictions)
         predictions = attach_prediction_type_info_to_sv_detections_batch(
             predictions=predictions,
@@ -330,5 +330,5 @@ def _post_process_result(
         )
         return [
             {"inference_id": inference_id, "predictions": prediction}
-            for prediction in predictions
+            for inference_id, prediction in zip(inference_ids, predictions)
         ]

inference/core/workflows/core_steps/models/roboflow/object_detection/v2.py

@@ -291,7 +291,7 @@ def run_remotely(
             source="workflow-execution",
         )
         client.configure(inference_configuration=client_config)
-        non_empty_inference_images = [i.numpy_image for i in images]
+        non_empty_inference_images = [i.base64_image for i in images]
         predictions = client.infer(
             inference_input=non_empty_inference_images,
             model_id=model_id,
@@ -310,7 +310,7 @@ def _post_process_result(
         predictions: List[dict],
         class_filter: Optional[List[str]],
     ) -> BlockResult:
-        inference_id = predictions[0].get(INFERENCE_ID_KEY, None)
+        inference_ids = [p.get(INFERENCE_ID_KEY, None) for p in predictions]
         predictions = convert_inference_detections_batch_to_sv_detections(predictions)
         predictions = attach_prediction_type_info_to_sv_detections_batch(
             predictions=predictions,
@@ -326,5 +326,5 @@ def _post_process_result(
         )
         return [
             {"inference_id": inference_id, "predictions": prediction}
-            for prediction in predictions
+            for inference_id, prediction in zip(inference_ids, predictions)
         ]

tests/inference/hosted_platform_tests/conftest.py

@@ -62,13 +62,17 @@ class PlatformEnvironment(Enum):
         "object-detection": "coin-counting/137",
         "instance-segmentation": "asl-poly-instance-seg/53",
         "classification": "catdog-w9i9e/18",
+        "multi_class_classification": "vehicle-classification-eapcd/2",
         "yolov8n-640": "yolov8n-640",
+        "yolov8n-pose-640": "yolov8n-pose-640",
     },
     PlatformEnvironment.ROBOFLOW_STAGING: {
         "object-detection": "eye-detection/35",
         "instance-segmentation": "asl-instance-seg/116",
         "classification": "catdog/28",
+        "multi_class_classification": "car-classification/23",
         "yolov8n-640": "microsoft-coco-obj-det/8",
+        "yolov8n-pose-640": "microsoft-coco-pose/1",
     },
 }
 
@@ -122,6 +126,13 @@ def classification_model_id(platform_environment: PlatformEnvironment) -> str:
     return MODELS_TO_BE_USED[platform_environment]["classification"]
 
 
+@pytest.fixture(scope="session")
+def multi_class_classification_model_id(
+    platform_environment: PlatformEnvironment,
+) -> str:
+    return MODELS_TO_BE_USED[platform_environment]["multi_class_classification"]
+
+
 @pytest.fixture(scope="session")
 def detection_model_id(platform_environment: PlatformEnvironment) -> str:
     return MODELS_TO_BE_USED[platform_environment]["object-detection"]
@@ -132,6 +143,11 @@ def yolov8n_640_model_id(platform_environment: PlatformEnvironment) -> str:
     return MODELS_TO_BE_USED[platform_environment]["yolov8n-640"]
 
 
+@pytest.fixture(scope="session")
+def yolov8n_pose_640_model_id(platform_environment: PlatformEnvironment) -> str:
+    return MODELS_TO_BE_USED[platform_environment]["yolov8n-pose-640"]
+
+
 @pytest.fixture(scope="session")
 def segmentation_model_id(platform_environment: PlatformEnvironment) -> str:
     return MODELS_TO_BE_USED[platform_environment]["instance-segmentation"]

tests/inference/hosted_platform_tests/workflows_examples/conftest.py

@@ -33,3 +33,8 @@ def license_plate_image() -> np.ndarray:
 @pytest.fixture(scope="function")
 def dogs_image() -> np.ndarray:
     return cv2.imread(os.path.join(ASSETS_DIR, "dogs.jpg"))
+
+
+@pytest.fixture(scope="function")
+def asl_image() -> np.ndarray:
+    return cv2.imread(os.path.join(ASSETS_DIR, "asl_image.jpg"))

tests/inference/hosted_platform_tests/workflows_examples/roboflow_models/v1/test_workflow_for_classification.py

@@ -0,0 +1,174 @@
+import numpy as np
+import pytest
+
+from inference_sdk import InferenceHTTPClient
+from tests.inference.hosted_platform_tests.conftest import (
+    ROBOFLOW_API_KEY,
+    PlatformEnvironment,
+)
+
+MULTI_CLASS_CLASSIFICATION_WORKFLOW = {
+    "version": "1.0",
+    "inputs": [
+        {"type": "WorkflowImage", "name": "image"},
+        {"type": "WorkflowParameter", "name": "model_id"},
+    ],
+    "steps": [
+        {
+            "type": "roboflow_core/roboflow_classification_model@v1",
+            "name": "classifier",
+            "image": "$inputs.image",
+            "model_id": "$inputs.model_id",
+        }
+    ],
+    "outputs": [
+        {
+            "type": "JsonField",
+            "name": "predictions",
+            "selector": "$steps.classifier.predictions",
+        },
+        {
+            "type": "JsonField",
+            "name": "inference_id",
+            "selector": "$steps.classifier.inference_id",
+        },
+    ],
+}
+
+MULTI_CLASS_CLASSIFICATION_RESULTS_FOR_ENVIRONMENT = {
+    PlatformEnvironment.ROBOFLOW_STAGING: [
+        0.3673,
+        0.593,
+    ],
+    PlatformEnvironment.ROBOFLOW_PLATFORM: [0.8252, 0.9962],
+}
+
+
+@pytest.mark.flaky(retries=4, delay=1)
+def test_multi_class_classification_workflow(
+    platform_environment: PlatformEnvironment,
+    classification_service_url: str,
+    multi_class_classification_model_id: str,
+    dogs_image: np.ndarray,
+    license_plate_image: np.ndarray,
+) -> None:
+    # given
+    client = InferenceHTTPClient(
+        api_url=classification_service_url,
+        api_key=ROBOFLOW_API_KEY,
+    )
+
+    # when
+    result = client.run_workflow(
+        specification=MULTI_CLASS_CLASSIFICATION_WORKFLOW,
+        images={
+            "image": [dogs_image, license_plate_image],
+        },
+        parameters={
+            "model_id": multi_class_classification_model_id,
+        },
+    )
+
+    # then
+    assert len(result) == 2, "2 images submitted, expected two outputs"
+    assert set(result[0].keys()) == {
+        "predictions",
+        "inference_id",
+    }, "Expected all outputs to be registered"
+    assert set(result[1].keys()) == {
+        "predictions",
+        "inference_id",
+    }, "Expected all outputs to be registered"
+    unique_inference_ids = {r["inference_id"] for r in result}
+    assert len(unique_inference_ids) == 2, "Expected unique inference ids granted"
+    predicted_confidences = [r["predictions"]["confidence"] for r in result]
+    assert np.allclose(
+        predicted_confidences,
+        MULTI_CLASS_CLASSIFICATION_RESULTS_FOR_ENVIRONMENT[platform_environment],
+        atol=1e-3,
+    ), "Expected classification predictions to match expectations"
+
+
+MULTI_LABEL_CLASSIFICATION_WORKFLOW = {
+    "version": "1.0",
+    "inputs": [
+        {"type": "WorkflowImage", "name": "image"},
+        {"type": "WorkflowParameter", "name": "model_id"},
+    ],
+    "steps": [
+        {
+            "type": "roboflow_core/roboflow_multi_label_classification_model@v1",
+            "name": "classifier",
+            "image": "$inputs.image",
+            "model_id": "$inputs.model_id",
+        }
+    ],
+    "outputs": [
+        {
+            "type": "JsonField",
+            "name": "predictions",
+            "selector": "$steps.classifier.predictions",
+        },
+        {
+            "type": "JsonField",
+            "name": "inference_id",
+            "selector": "$steps.classifier.inference_id",
+        },
+    ],
+}
+
+
+MULTI_LABEL_CLASSIFICATION_RESULTS_FOR_ENVIRONMENT = {
+    PlatformEnvironment.ROBOFLOW_STAGING: [
+        {"dog"},
+        {"cat", "dog"},
+    ],
+    PlatformEnvironment.ROBOFLOW_PLATFORM: [
+        {"dog"},
+        set(),
+    ],
+}
+
+
+@pytest.mark.flaky(retries=4, delay=1)
+def test_multi_label_classification_workflow(
+    platform_environment: PlatformEnvironment,
+    classification_service_url: str,
+    classification_model_id: str,
+    dogs_image: np.ndarray,
+    license_plate_image: np.ndarray,
+) -> None:
+    # given
+    client = InferenceHTTPClient(
+        api_url=classification_service_url,
+        api_key=ROBOFLOW_API_KEY,
+    )
+
+    # when
+    result = client.run_workflow(
+        specification=MULTI_LABEL_CLASSIFICATION_WORKFLOW,
+        images={
+            "image": [dogs_image, license_plate_image],
+        },
+        parameters={
+            "model_id": classification_model_id,
+        },
+    )
+
+    # then
+    assert len(result) == 2, "2 images submitted, expected two outputs"
+    assert set(result[0].keys()) == {
+        "predictions",
+        "inference_id",
+    }, "Expected all outputs to be registered"
+    assert set(result[1].keys()) == {
+        "predictions",
+        "inference_id",
+    }, "Expected all outputs to be registered"
+    unique_inference_ids = {r["inference_id"] for r in result}
+    assert len(unique_inference_ids) == 2, "Expected unique inference ids granted"
+    predicted_classes = [set(r["predictions"]["predicted_classes"]) for r in result]
+    assert (
+        predicted_classes
+        == MULTI_LABEL_CLASSIFICATION_RESULTS_FOR_ENVIRONMENT[platform_environment]
+    )