inference-cli with batch processing for Workflows

.github/workflows/integration_tests_inference_cli_depending_on_inference_x86.yml

@@ -0,0 +1,47 @@
+name: INTEGRATION TESTS - inference CLI + inference CORE
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  call_is_mergeable:
+    uses: ./.github/workflows/is_mergeable.yml
+    secrets: inherit
+  build-dev-test:
+    needs: call_is_mergeable
+    if: ${{ github.event_name != 'pull_request' || needs.call_is_mergeable.outputs.mergeable_state != 'not_clean' }}
+    runs-on:
+      labels: depot-ubuntu-22.04-small
+      group: public-depot
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+    steps:
+      - name: 🛎️ Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+      - name: 🐍 Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          check-latest: true
+      - name: 📦 Cache Python packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('requirements/**') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-${{ matrix.python-version }}-
+      - name: 📦 Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade setuptools
+          pip install --extra-index-url https://download.pytorch.org/whl/cpu -r requirements/_requirements.txt -r requirements/requirements.cpu.txt -r requirements/requirements.sdk.http.txt -r requirements/requirements.test.unit.txt -r requirements/requirements.http.txt -r requirements/requirements.yolo_world.txt -r requirements/requirements.doctr.txt -r requirements/requirements.sam.txt -r requirements/requirements.transformers.txt -r requirements/requirements.cli.txt -r requirements/requirements.sdk.http.txt
+      - name: 🧪 Integration Tests of Inference CLI
+        run: RUN_TESTS_WITH_INFERENCE_PACKAGE=True INFERENCE_CLI_TESTS_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m pytest tests/inference_cli/integration_tests/test_workflows.py

.github/workflows/integration_tests_inference_cli_x86.yml

@@ -44,4 +44,4 @@ jobs:
           pip install --upgrade setuptools
           pip install -r requirements/requirements.cli.txt -r requirements/requirements.sdk.http.txt -r requirements/requirements.test.unit.txt
       - name: 🧪 Integration Tests of Inference CLI
-        run: python -m pytest tests/inference_cli/integration_tests
+        run: RUN_TESTS_EXPECTING_ERROR_WHEN_INFERENCE_NOT_INSTALLED=True INFERENCE_CLI_TESTS_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m pytest tests/inference_cli/integration_tests

.github/workflows/load_test_hosted_inference.yml

@@ -51,35 +51,36 @@ jobs:
       - name: 🏋️‍♂️ Load test 🚨 PRODUCTION 🚨 | object-detection 🔥🔥🔥🔥
         if: ${{ github.event.inputs.environment == 'production' && github.event.inputs.model_type == 'object-detection' }}
         run: |
-          ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m inference_cli.main benchmark api-speed -m coco/16 -d coco -rps 5 -br 500 -h https://detect.roboflow.com --yes --output_location test_results.json
+          ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m inference_cli.main benchmark api-speed -m coco/16 -d coco -rps 5 -br 500 -h https://detect.roboflow.com --yes --output_location test_results.json --max_error_rate 5.0
       - name: 🏋️‍♂️ Load test 🚨 PRODUCTION 🚨 | instance-segmentation 🔥🔥🔥🔥
         if: ${{ github.event.inputs.environment == 'production' && github.event.inputs.model_type == 'instance-segmentation' }}
         run: |
-          ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m inference_cli.main benchmark api-speed -m asl-poly-instance-seg/53 -d coco -rps 5 -br 500 -h https://outline.roboflow.com --yes --output_location test_results.json
+          ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m inference_cli.main benchmark api-speed -m asl-poly-instance-seg/53 -d coco -rps 5 -br 500 -h https://outline.roboflow.com --yes --output_location test_results.json --max_error_rate 5.0
       - name: 🏋️‍♂️ Load test 🚨 PRODUCTION 🚨 | classification 🔥🔥🔥🔥
         if: ${{ github.event.inputs.environment == 'production' && github.event.inputs.model_type == 'classification' }}
         run: |
-          ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m inference_cli.main benchmark api-speed -m vehicle-classification-eapcd/2 -d coco -rps 5 -br 500 -h https://classify.roboflow.com --yes --output_location test_results.json
+          ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m inference_cli.main benchmark api-speed -m vehicle-classification-eapcd/2 -d coco -rps 5 -br 500 -h https://classify.roboflow.com --yes --output_location test_results.json --max_error_rate 5.0
       - name: 🏋️‍♂️ Load test 🚨 PRODUCTION 🚨 | workflows 🔥🔥🔥🔥
         if: ${{ github.event.inputs.environment == 'production' && github.event.inputs.model_type == 'workflows' }}
         run: |
-          ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m inference_cli.main benchmark api-speed -wid workflows-production-test -wn paul-guerrie-tang1 -d coco -rps 5 -br 500 -h https://classify.roboflow.com --yes --output_location test_results.json
+          ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m inference_cli.main benchmark api-speed -wid workflows-production-test -wn paul-guerrie-tang1 -d coco -rps 5 -br 500 -h https://classify.roboflow.com --yes --output_location test_results.json --max_error_rate 5.0
 
       - name: 🏋️‍♂️ Load test 😎 STAGING 😎 | object-detection 🔥🔥🔥🔥
         if: ${{ github.event.inputs.environment == 'staging' && github.event.inputs.model_type == 'object-detection' }}
         run: |
-          ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_STAGING_API_KEY }} python -m inference_cli.main benchmark api-speed -m eye-detection/35  -d coco -rps 5 -br 500 -h https://lambda-object-detection.staging.roboflow.com --legacy-endpoints --yes --output_location test_results.json
+          ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_STAGING_API_KEY }} python -m inference_cli.main benchmark api-speed -m eye-detection/35  -d coco -rps 5 -br 500 -h https://lambda-object-detection.staging.roboflow.com --legacy-endpoints --yes --output_location test_results.json --max_error_rate 5.0
       - name: 🏋️‍♂️ Load test 😎 STAGING 😎 | instance-segmentation 🔥🔥🔥🔥
         if: ${{ github.event.inputs.environment == 'staging' && github.event.inputs.model_type == 'instance-segmentation' }}
         run: |
-          ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_STAGING_API_KEY }} python -m inference_cli.main benchmark api-speed -m asl-instance-seg/116 -d coco -rps 5 -br 500 -h https://lambda-instance-segmentation.staging.roboflow.com --legacy-endpoints --yes --output_location test_results.json
+          ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_STAGING_API_KEY }} python -m inference_cli.main benchmark api-speed -m asl-instance-seg/116 -d coco -rps 5 -br 500 -h https://lambda-instance-segmentation.staging.roboflow.com --legacy-endpoints --yes --output_location test_results.json --max_error_rate 5.0
       - name: 🏋️‍♂️ Load test 😎 STAGING 😎 | classification 🔥🔥🔥🔥
         if: ${{ github.event.inputs.environment == 'staging' && github.event.inputs.model_type == 'classification' }}
         run: |
-          ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_STAGING_API_KEY }} python -m inference_cli.main benchmark api-speed -m catdog/28 -d coco -rps 5 -br 500 -h https://lambda-classification.staging.roboflow.com --legacy-endpoints --yes --output_location test_results.json
+          ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_STAGING_API_KEY }} python -m inference_cli.main benchmark api-speed -m catdog/28 -d coco -rps 5 -br 500 -h https://lambda-classification.staging.roboflow.com --legacy-endpoints --yes --output_location test_results.json --max_error_rate 5.0
       - name: 🏋️‍♂️ Load test 😎 STAGING 😎 | workflows 🔥🔥🔥🔥
         if: ${{ github.event.inputs.environment == 'staging' && github.event.inputs.model_type == 'workflows' }}
         run: |
-          ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_STAGING_API_KEY }} python -m inference_cli.main benchmark api-speed -wid workflows-staging-test -wn paul-guerrie -d coco -rps 5 -br 500 -h https://lambda-classification.staging.roboflow.com --legacy-endpoints --yes --output_location test_results.json
+          ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_STAGING_API_KEY }} python -m inference_cli.main benchmark api-speed -wid workflows-staging-test -wn paul-guerrie -d coco -rps 5 -br 500 -h https://lambda-classification.staging.roboflow.com --legacy-endpoints --yes --output_location test_results.json --max_error_rate 5.0
       - name: 📈 RESULTS
         run: cat test_results.json | jq
+        if: always()

.gitignore

@@ -170,6 +170,6 @@ docs/workflows/gallery/*
 !tests/workflows/integration_tests/execution/assets/rock_paper_scissors/*.jpg
 !tests/workflows/unit_tests/core_steps/models/third_party/assets/*.png
 !tests/workflows/integration_tests/execution/assets/*.png
-
+tests/inference_cli/integration_tests/assets/test_images/
 inference_profiling
 tests/inference_sdk/unit_tests/http/inference_profiling

docs/inference_helpers/inference_cli.md

@@ -360,6 +360,92 @@ option can be used (and `-c` will be ignored). Value provided in `--rps` option
 are to be spawned **each second** without waiting for previous requests to be handled. In I/O intensive benchmark 
 scenarios - we suggest running command from multiple separate processes and possibly multiple hosts.
 
+### inference workflows
+
+In release `0.29.0`, `inference-cli` was extended with command to process data using Workflows. It is possible to 
+process:
+
+* individual images
+
+* directories of images
+
+* video files
+
+#### Processing individual image
+
+Basic usage of the command is illustrated below:
+
+```bash
+inference workflows process-image \
+    --image_path {your-input-image} \
+    --output_dir {your-output-dir} \
+    --workspace_name {your-roboflow-workspace-url} \
+    --workflow_id {your-workflow-id} \
+    --api-key {your_roboflow_api_key}
+```
+
+which would take your input image, run it against your Workflow and save results in output directory. By default, 
+Workflow will be processed using Roboflow Hosted API. You can tweak behaviour of the command:
+
+* if you want to process the image locally, using `inference` Python package - use 
+`--processing_target inference_package` option (*requires `inference` to be installed*)
+
+* to see all options, use `inference workflows process-image --help` command
+
+* any option that starts from `--` which is not enlisted in `--help` command will be treated as input parameter 
+to the workflow execution
+
+
+#### Processing directory of images
+
+Basic usage of the command is illustrated below:
+
+```bash
+inference workflows process-images-directory \
+    -i {your_input_directory} \
+    -o {your_output_directory} \
+    --workspace_name {your-roboflow-workspace-url} \
+    --workflow_id {your-workflow-id} \
+    --api-key {your_roboflow_api_key}
+```
+
+You can tweak behaviour of the command:
+
+* if you want to process the image locally, using `inference` Python package - use 
+`--processing_target inference_package` option (*requires `inference` to be installed*)
+
+* to see all options, use `inference workflows process-image --help` command
+
+* any option that starts from `--` which is not enlisted in `--help` command will be treated as input parameter 
+to the workflow execution
+
+#### Processing video file 
+
+!!! Note "`inference` required"
+
+    This command requires `inference` to be installed.
+
+Basic usage of the command is illustrated below:
+
+```bash
+inference workflows process-video \
+    --video_path {video_to_be_processed} \
+    --output_dir {empty_directory} \
+    --workspace_name {your-roboflow-workspace-url} \
+    --workflow_id {your-workflow-id} \
+    --api-key {your_roboflow_api_key}
+```
+
+You can tweak behaviour of the command:
+
+* `--max_fps` option can be used to subsample video frames while processing
+
+* to see all options, use `inference workflows process-image --help` command
+
+* any option that starts from `--` which is not enlisted in `--help` command will be treated as input parameter 
+to the workflow execution
+
+
 ## Supported Devices
 
 Roboflow Inference CLI currently supports the following device targets:

docs/workflows/kinds.md

@@ -6,7 +6,7 @@ provided at runtime, either from user inputs or from other function outputs.
 
 To manage this, Workflows use *selectors*, which act like references, pointing to data without containing it directly. 
 
-!!! Example *selectors*
+!!! example "selectors"
 
     Selectors might refer to a named input - for example input image - like `$inputs.image` 
     or predictions generated by a previous step - like `$steps.my_model.predictions`

docs/workflows/modes_of_running.md

@@ -221,6 +221,42 @@ Explore the example below to see how to combine `InferencePipeline` with Workflo
 
         Make sure you have `inference` or `inference-gpu` package installed in your Python environment
 
+## Batch processing using `inference-cli`
+
+[`inference-cli`](/inference_helpers/inference_cli/) is command-line wrapper library around `inference`. You can use it
+to process your data using Workflows without writing a single line of code. You simply point the data to be processed,
+select your Workflow and specify where results should be saved. Thanks to `inference-cli` you can process:
+
+* individual images
+
+* directories of images
+
+* video files
+
+!!! example "Processing directory of images"
+
+    You can start the processing using the following command:
+
+    ```bash
+    inference workflows process-images-directory \
+        -i {your_input_directory} \
+        -o {your_output_directory} \[workflows.py](..%2F..%2Finference_cli%2Fworkflows.py)
+        --workspace_name {your-roboflow-workspace-url} \
+        --workflow_id {your-workflow-id} \
+        --api-key {your_roboflow_api_key}
+    ```
+
+    As a result, in the directory specified in `-o` option you should be able to find:
+
+    * sub-directories named after files in your original directory with `results.json` file that contain Worklfow 
+    results and optionally additional `*.jpg` files with images created during Workflow execution
+
+    * `aggregated_results.csv` file that contain concatenated results of Workflow execution for all input image file
+
+    !!! note
+
+        Make sure you have `inference` or `inference-cli` package installed in your Python environment
+
 
 ## Workflows in Python package
 

inference/core/entities/responses/workflows.py

@@ -160,7 +160,9 @@ class ExecutionEngineVersions(BaseModel):
 
 
 class WorkflowsBlocksSchemaDescription(BaseModel):
-    schema: dict = Field(description="Schema for validating block definitions")
+    blocks_schema: dict = Field(
+        description="Schema for validating block definitions", alias="schema"
+    )
 
 
 class DescribeInterfaceResponse(BaseModel):

inference/core/interfaces/stream/inference_pipeline.py

@@ -466,6 +466,7 @@ def init_with_workflow(
         batch_collection_timeout: Optional[float] = None,
         profiling_directory: str = "./inference_profiling",
         use_workflow_definition_cache: bool = True,
+        serialize_results: bool = False,
     ) -> "InferencePipeline":
         """
         This class creates the abstraction for making inferences from given workflow against video stream.
@@ -540,6 +541,8 @@ def init_with_workflow(
             use_workflow_definition_cache (bool): Controls usage of cache for workflow definitions. Set this to False
                 when you frequently modify definition saved in Roboflow app and want to fetch the
                 newest version for the request. Only applies for Workflows definitions saved on Roboflow platform.
+            serialize_results (bool): Boolean flag to decide if ExecutionEngine run should serialize workflow
+                results for each frame. If that is set true, sinks will receive serialized workflow responses.
 
         Other ENV variables involved in low-level configuration:
         * INFERENCE_PIPELINE_PREDICTIONS_QUEUE_SIZE - size of buffer for predictions that are ready for dispatching
@@ -604,8 +607,6 @@ def init_with_workflow(
                 model_manager,
                 max_size=MAX_ACTIVE_MODELS,
             )
-            if api_key is None:
-                api_key = API_KEY
             if workflow_init_parameters is None:
                 workflow_init_parameters = {}
             thread_pool_executor = ThreadPoolExecutor(
@@ -629,6 +630,7 @@ def init_with_workflow(
                 execution_engine=execution_engine,
                 image_input_name=image_input_name,
                 video_metadata_input_name=video_metadata_input_name,
+                serialize_results=serialize_results,
             )
         except ImportError as error:
             raise CannotInitialiseModelError(

inference/core/interfaces/stream/model_handlers/workflows.py

@@ -14,6 +14,7 @@ def run_workflow(
         execution_engine: ExecutionEngine,
         image_input_name: str,
         video_metadata_input_name: str,
+        serialize_results: bool = False,
     ) -> List[dict]:
         if workflows_parameters is None:
             workflows_parameters = {}
@@ -53,4 +54,5 @@ def run_workflow(
         return execution_engine.run(
             runtime_parameters=workflows_parameters,
             fps=fps,
+            serialize_results=serialize_results,
         )

inference/core/workflows/core_steps/sinks/roboflow/model_monitoring_inference_aggregator/v1.py

@@ -148,6 +148,10 @@ def get_execution_engine_compatibility(cls) -> Optional[str]:
 
 
 class ParsedPrediction(BaseModel):
+    model_config = ConfigDict(
+        protected_namespaces=(),
+    )
+
     class_name: str
     confidence: float
     inference_id: str

inference/core/workflows/execution_engine/v1/compiler/syntactic_parser.py

@@ -97,4 +97,4 @@ def get_workflow_schema_description() -> WorkflowsBlocksSchemaDescription:
         available_blocks=available_blocks
     )
     schema = workflow_definition_class.model_json_schema()
-    return WorkflowsBlocksSchemaDescription(schema=schema)
+    return WorkflowsBlocksSchemaDescription(blocks_schema=schema)

inference_cli/benchmark.py

@@ -136,6 +136,14 @@ def api_speed(
             help="Boolean flag to decide on auto `yes` answer given on user input required.",
         ),
     ] = False,
+    max_error_rate: Annotated[
+        Optional[float],
+        typer.Option(
+            "--max_error_rate",
+            help="Max error rate for API speed benchmark - if given and the error rate is higher - command will "
+            "return non-success error code. Expected percentage values in range 0.0-100.0",
+        ),
+    ] = None,
 ):
     if "roboflow.com" in host and not proceed_automatically:
         proceed = input(
@@ -158,6 +166,7 @@ def api_speed(
                 model_configuration=model_configuration,
                 output_location=output_location,
                 enforce_legacy_endpoints=enforce_legacy_endpoints,
+                max_error_rate=max_error_rate,
             )
         else:
             if workflow_specification:
@@ -179,6 +188,7 @@ def api_speed(
                 api_key=api_key,
                 model_configuration=model_configuration,
                 output_location=output_location,
+                max_error_rate=max_error_rate,
             )
     except Exception as error:
         typer.echo(f"Command failed. Cause: {error}")

inference_cli/lib/benchmark/api_speed.py

@@ -25,7 +25,11 @@ def run_api_warm_up(
     for _ in tqdm(
         range(warm_up_requests), desc="Warming up API...", total=warm_up_requests
     ):
-        _ = client.infer(inference_input=image)
+        try:
+            _ = client.infer(inference_input=image)
+        except Exception:
+            # ignoring errors, without slight API instability may terminate benchmark
+            pass
 
 
 def coordinate_infer_api_speed_benchmark(