From dd1fa444e3dbd1f7fca9ab190cd2367759b2808b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Mon, 25 Nov 2024 18:45:49 +0100 Subject: [PATCH 01/16] WIP --- inference_cli/lib/workflows_adapter.py | 158 +++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 inference_cli/lib/workflows_adapter.py diff --git a/inference_cli/lib/workflows_adapter.py b/inference_cli/lib/workflows_adapter.py new file mode 100644 index 000000000..9d51e2fee --- /dev/null +++ b/inference_cli/lib/workflows_adapter.py @@ -0,0 +1,158 @@ +import os.path +from collections import defaultdict +from typing import List, Literal, Union, Optional, Dict + +import cv2 +import numpy as np +from supervision import VideoInfo + +from inference.core.interfaces.camera.entities import VideoFrame +from inference.core.workflows.execution_engine.entities.base import WorkflowImageData + + +def process_video_with_workflow() -> None: + pass + + +class WorkflowsStructuredDataSink: + + @classmethod + def init( + cls, + output_directory: str, + results_log_type: Literal["csv", "jsonl"], + max_entries_in_logs_chunk: int, + ) -> "WorkflowsStructuredDataSink": + return cls( + output_directory=output_directory, + structured_results_buffer=[], + results_log_type=results_log_type, + max_entries_in_logs_chunk=max_entries_in_logs_chunk, + ) + + def __init__( + self, + output_directory: str, + structured_results_buffer: List[dict], + results_log_type: Literal["csv", "jsonl"], + max_entries_in_logs_chunk: int, + ): + self._output_directory = output_directory + self._structured_results_buffer = structured_results_buffer + self._results_log_type = results_log_type + self._max_entries_in_logs_chunk = max_entries_in_logs_chunk + + def on_prediction( + self, + predictions: Union[Optional[dict], List[Optional[dict]]], + video_frames: Union[Optional[VideoFrame], List[Optional[VideoFrame]]], + ) -> None: + if not isinstance(predictions, list): + predictions = [predictions] + for prediction in predictions: + if prediction is None: + continue + + def __del__(self): + pass + + +def dump_content() -> None: + pass + + +class WorkflowsVideoSink: + + @classmethod + def init( + cls, + input_video_path: str, + output_directory: str, + ) -> "WorkflowsVideoSink": + source_video_info = VideoInfo.from_video_path(video_path=input_video_path) + return cls( + source_video_info=source_video_info, + output_directory=output_directory, + ) + + def __init__( + self, + source_video_info: VideoInfo, + output_directory: str + ): + self._video_sinks: Dict[int, Dict[str, VideoSink]] = defaultdict(dict) + self._source_video_info = source_video_info + self._output_directory = output_directory + + def on_prediction( + self, + predictions: Union[Optional[dict], List[Optional[dict]]], + video_frames: Union[Optional[VideoFrame], List[Optional[VideoFrame]]], + ) -> None: + if not isinstance(predictions, list): + predictions = [predictions] + for stream_idx, prediction in enumerate(predictions): + if prediction is None: + continue + stream_sinks = self._video_sinks[stream_idx] + for key, value in prediction.items(): + if not isinstance(value, WorkflowImageData): + continue + if key not in stream_sinks: + video_target_path = _generate_target_path_for_video( + output_directory=self._output_directory, + source_id=stream_idx, + field_name=key, + ) + stream_sinks[key] = VideoSink( + target_path=video_target_path, + video_info=self._source_video_info, + ) + stream_sinks[key].start() + stream_sinks[key].write_frame(frame=value.numpy_image) + + def __del__(self): + for stream_sinks in self._video_sinks.values(): + for sink in stream_sinks.values(): + sink.release() + + + +class VideoSink: + + def __init__(self, target_path: str, video_info: VideoInfo, codec: str = "mp4v"): + self.target_path = target_path + self.video_info = video_info + self.__codec = codec + self.__writer = None + + def start(self) -> None: + try: + self.__fourcc = cv2.VideoWriter_fourcc(*self.__codec) + except TypeError as e: + print(str(e) + ". Defaulting to mp4v...") + self.__fourcc = cv2.VideoWriter_fourcc(*"mp4v") + self.__writer = cv2.VideoWriter( + self.target_path, + self.__fourcc, + self.video_info.fps, + self.video_info.resolution_wh, + ) + + def write_frame(self, frame: np.ndarray): + """ + Writes a single video frame to the target video file. + + Args: + frame (np.ndarray): The video frame to be written to the file. The frame + must be in BGR color format. + """ + self.__writer.write(frame) + + def release(self) -> None: + self.__writer.release() + + +def _generate_target_path_for_video(output_directory: str, source_id: int, field_name: str) -> str: + os.makedirs(os.path.abspath(output_directory), exist_ok=True) + return os.path.join(os.path.abspath(output_directory), f"source_{source_id}_output_{field_name}_preview.mp4") \ No newline at end of file From 86cecaa003bf789ecfc6ccd04b51629ac4ac4bca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Tue, 26 Nov 2024 12:39:04 +0100 Subject: [PATCH 02/16] Add first working version of video processing --- .../core/entities/responses/workflows.py | 4 +- .../interfaces/stream/inference_pipeline.py | 6 +- .../stream/model_handlers/workflows.py | 2 + .../v1.py | 4 + .../v1/compiler/syntactic_parser.py | 2 +- .../lib/benchmark/python_package_speed.py | 26 +- inference_cli/lib/benchmark_adapter.py | 8 +- inference_cli/lib/utils.py | 28 ++ inference_cli/lib/workflows/__init__.py | 0 inference_cli/lib/workflows/core.py | 39 +++ inference_cli/lib/workflows/entities.py | 6 + inference_cli/lib/workflows/video_adapter.py | 307 ++++++++++++++++++ inference_cli/lib/workflows_adapter.py | 158 --------- inference_cli/main.py | 2 + inference_cli/workflows.py | 212 ++++++++++++ 15 files changed, 618 insertions(+), 186 deletions(-) create mode 100644 inference_cli/lib/workflows/__init__.py create mode 100644 inference_cli/lib/workflows/core.py create mode 100644 inference_cli/lib/workflows/entities.py create mode 100644 inference_cli/lib/workflows/video_adapter.py delete mode 100644 inference_cli/lib/workflows_adapter.py create mode 100644 inference_cli/workflows.py diff --git a/inference/core/entities/responses/workflows.py b/inference/core/entities/responses/workflows.py index afc94f93b..c8255f995 100644 --- a/inference/core/entities/responses/workflows.py +++ b/inference/core/entities/responses/workflows.py @@ -160,7 +160,9 @@ class ExecutionEngineVersions(BaseModel): class WorkflowsBlocksSchemaDescription(BaseModel): - schema: dict = Field(description="Schema for validating block definitions") + blocks_schema: dict = Field( + description="Schema for validating block definitions", alias="schema" + ) class DescribeInterfaceResponse(BaseModel): diff --git a/inference/core/interfaces/stream/inference_pipeline.py b/inference/core/interfaces/stream/inference_pipeline.py index 896e06620..01d77ea8a 100644 --- a/inference/core/interfaces/stream/inference_pipeline.py +++ b/inference/core/interfaces/stream/inference_pipeline.py @@ -466,6 +466,7 @@ def init_with_workflow( batch_collection_timeout: Optional[float] = None, profiling_directory: str = "./inference_profiling", use_workflow_definition_cache: bool = True, + serialize_results: bool = False, ) -> "InferencePipeline": """ This class creates the abstraction for making inferences from given workflow against video stream. @@ -540,6 +541,8 @@ def init_with_workflow( use_workflow_definition_cache (bool): Controls usage of cache for workflow definitions. Set this to False when you frequently modify definition saved in Roboflow app and want to fetch the newest version for the request. Only applies for Workflows definitions saved on Roboflow platform. + serialize_results (bool): Boolean flag to decide if ExecutionEngine run should serialize workflow + results for each frame. If that is set true, sinks will receive serialized workflow responses. Other ENV variables involved in low-level configuration: * INFERENCE_PIPELINE_PREDICTIONS_QUEUE_SIZE - size of buffer for predictions that are ready for dispatching @@ -604,8 +607,6 @@ def init_with_workflow( model_manager, max_size=MAX_ACTIVE_MODELS, ) - if api_key is None: - api_key = API_KEY if workflow_init_parameters is None: workflow_init_parameters = {} thread_pool_executor = ThreadPoolExecutor( @@ -629,6 +630,7 @@ def init_with_workflow( execution_engine=execution_engine, image_input_name=image_input_name, video_metadata_input_name=video_metadata_input_name, + serialize_results=serialize_results, ) except ImportError as error: raise CannotInitialiseModelError( diff --git a/inference/core/interfaces/stream/model_handlers/workflows.py b/inference/core/interfaces/stream/model_handlers/workflows.py index c1da0c440..102ef92ca 100644 --- a/inference/core/interfaces/stream/model_handlers/workflows.py +++ b/inference/core/interfaces/stream/model_handlers/workflows.py @@ -14,6 +14,7 @@ def run_workflow( execution_engine: ExecutionEngine, image_input_name: str, video_metadata_input_name: str, + serialize_results: bool = False, ) -> List[dict]: if workflows_parameters is None: workflows_parameters = {} @@ -53,4 +54,5 @@ def run_workflow( return execution_engine.run( runtime_parameters=workflows_parameters, fps=fps, + serialize_results=serialize_results, ) diff --git a/inference/core/workflows/core_steps/sinks/roboflow/model_monitoring_inference_aggregator/v1.py b/inference/core/workflows/core_steps/sinks/roboflow/model_monitoring_inference_aggregator/v1.py index d77680284..d03d4e17f 100644 --- a/inference/core/workflows/core_steps/sinks/roboflow/model_monitoring_inference_aggregator/v1.py +++ b/inference/core/workflows/core_steps/sinks/roboflow/model_monitoring_inference_aggregator/v1.py @@ -148,6 +148,10 @@ def get_execution_engine_compatibility(cls) -> Optional[str]: class ParsedPrediction(BaseModel): + model_config = ConfigDict( + protected_namespaces=(), + ) + class_name: str confidence: float inference_id: str diff --git a/inference/core/workflows/execution_engine/v1/compiler/syntactic_parser.py b/inference/core/workflows/execution_engine/v1/compiler/syntactic_parser.py index 9eeb4f873..e1f9df680 100644 --- a/inference/core/workflows/execution_engine/v1/compiler/syntactic_parser.py +++ b/inference/core/workflows/execution_engine/v1/compiler/syntactic_parser.py @@ -97,4 +97,4 @@ def get_workflow_schema_description() -> WorkflowsBlocksSchemaDescription: available_blocks=available_blocks ) schema = workflow_definition_class.model_json_schema() - return WorkflowsBlocksSchemaDescription(schema=schema) + return WorkflowsBlocksSchemaDescription(blocks_schema=schema) diff --git a/inference_cli/lib/benchmark/python_package_speed.py b/inference_cli/lib/benchmark/python_package_speed.py index dc0c6f0c8..f41dd4397 100644 --- a/inference_cli/lib/benchmark/python_package_speed.py +++ b/inference_cli/lib/benchmark/python_package_speed.py @@ -1,5 +1,4 @@ import random -import subprocess import time from typing import Any, Dict, List, Optional @@ -7,29 +6,10 @@ from supervision.utils.file import read_yaml_file from tqdm import tqdm +from inference import get_model +from inference.core.models.base import Model +from inference.core.registries.roboflow import get_model_type from inference_cli.lib.benchmark.results_gathering import ResultsCollector -from inference_cli.lib.exceptions import InferencePackageMissingError - -try: - from inference import get_model - from inference.core.models.base import Model - from inference.core.registries.roboflow import get_model_type -except Exception as error: - print( - "You need to have `inference` package installed. Do you want the package to be installed? [YES/no]" - ) - user_choice = input() - if user_choice.lower() != "yes": - raise InferencePackageMissingError( - "You need to install `inference` package to use this feature. Run `pip install inference`" - ) from error - try: - subprocess.run("pip install inference".split(), check=True) - import inference - except Exception as inner_error: - raise InferencePackageMissingError( - f"Installation of package failed. Cause: {inner_error}" - ) from inner_error def run_python_package_speed_benchmark( diff --git a/inference_cli/lib/benchmark_adapter.py b/inference_cli/lib/benchmark_adapter.py index b238bd2f7..790b084e4 100644 --- a/inference_cli/lib/benchmark_adapter.py +++ b/inference_cli/lib/benchmark_adapter.py @@ -15,7 +15,11 @@ InferenceStatistics, ResultsCollector, ) -from inference_cli.lib.utils import dump_json, initialise_client +from inference_cli.lib.utils import ( + dump_json, + ensure_inference_is_installed, + initialise_client, +) def run_infer_api_speed_benchmark( @@ -149,6 +153,8 @@ def run_python_package_speed_benchmark( model_configuration: Optional[str] = None, output_location: Optional[str] = None, ) -> None: + ensure_inference_is_installed() + # importing here not to affect other entrypoints by missing `inference` core library from inference_cli.lib.benchmark.python_package_speed import ( run_python_package_speed_benchmark, diff --git a/inference_cli/lib/utils.py b/inference_cli/lib/utils.py index 9dee2f99e..4d3688999 100644 --- a/inference_cli/lib/utils.py +++ b/inference_cli/lib/utils.py @@ -1,14 +1,42 @@ import json import os.path +import subprocess from typing import Dict, List, Optional, Union from supervision.utils.file import read_yaml_file from inference_cli.lib.env import ROBOFLOW_API_KEY +from inference_cli.lib.exceptions import InferencePackageMissingError from inference_cli.lib.logger import CLI_LOGGER from inference_sdk import InferenceConfiguration, InferenceHTTPClient +def ensure_inference_is_installed() -> None: + try: + from inference import get_model + except Exception as error: + print( + "You need to have `inference` package installed. Do you want the package to be installed? [YES/no]" + ) + user_choice = input() + if user_choice.lower() != "yes": + raise InferencePackageMissingError( + "You need to install `inference` package to use this feature. Run `pip install inference`" + ) from error + try: + subprocess.run("pip install inference".split(), check=True) + import inference + except Exception as inner_error: + raise InferencePackageMissingError( + f"Installation of package failed. Cause: {inner_error}" + ) from inner_error + + +def read_json(path: str) -> dict: + with open(path) as f: + return json.load(f) + + def read_env_file(path: str) -> Dict[str, str]: file_lines = read_file_lines(path=path) result = {} diff --git a/inference_cli/lib/workflows/__init__.py b/inference_cli/lib/workflows/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/inference_cli/lib/workflows/core.py b/inference_cli/lib/workflows/core.py new file mode 100644 index 000000000..97cc9b317 --- /dev/null +++ b/inference_cli/lib/workflows/core.py @@ -0,0 +1,39 @@ +import os +from typing import Any, Dict, Literal, Optional, Union + +from inference_cli.lib.utils import ensure_inference_is_installed, read_json +from inference_cli.lib.workflows.entities import OutputFileType + + +def run_video_processing_with_workflows( + input_video_path: str, + output_directory: str, + output_file_type: OutputFileType, + workflow_specification: Optional[dict] = None, + workspace_name: Optional[str] = None, + workflow_id: Optional[str] = None, + workflow_parameters: Optional[Dict[str, Any]] = None, + max_fps: Optional[float] = None, + save_image_outputs_as_video: bool = True, + api_key: Optional[str] = None, +) -> None: + # enabling new behaviour ensuring frame rate will be subsample (needed until + # this becomes default) + os.environ["ENABLE_FRAME_DROP_ON_VIDEO_FILE_RATE_LIMITING"] = "True" + + ensure_inference_is_installed() + + from inference_cli.lib.workflows.video_adapter import process_video_with_workflow + + process_video_with_workflow( + input_video_path=input_video_path, + output_directory=output_directory, + output_file_type=output_file_type, + workflow_specification=workflow_specification, + workspace_name=workspace_name, + workflow_id=workflow_id, + workflow_parameters=workflow_parameters, + max_fps=max_fps, + save_image_outputs_as_video=save_image_outputs_as_video, + api_key=api_key, + ) diff --git a/inference_cli/lib/workflows/entities.py b/inference_cli/lib/workflows/entities.py new file mode 100644 index 000000000..be61abf2a --- /dev/null +++ b/inference_cli/lib/workflows/entities.py @@ -0,0 +1,6 @@ +from enum import Enum + + +class OutputFileType(str, Enum): + JSONL = "jsonl" + CSV = "csv" diff --git a/inference_cli/lib/workflows/video_adapter.py b/inference_cli/lib/workflows/video_adapter.py new file mode 100644 index 000000000..f1e8f8b2c --- /dev/null +++ b/inference_cli/lib/workflows/video_adapter.py @@ -0,0 +1,307 @@ +import json +import os.path +from collections import defaultdict +from functools import partial +from glob import glob +from typing import Any, Dict, List, Optional, Union + +import cv2 +import numpy as np +import pandas as pd +import supervision as sv +from rich.progress import Progress, TaskID + +from inference import InferencePipeline +from inference.core.interfaces.camera.entities import VideoFrame +from inference.core.interfaces.stream.sinks import multi_sink +from inference.core.utils.image_utils import load_image_bgr +from inference_cli.lib.workflows.entities import OutputFileType + + +def process_video_with_workflow( + input_video_path: str, + output_directory: str, + output_file_type: OutputFileType, + workflow_specification: Optional[dict] = None, + workspace_name: Optional[str] = None, + workflow_id: Optional[str] = None, + workflow_parameters: Optional[Dict[str, Any]] = None, + max_fps: Optional[float] = None, + save_image_outputs_as_video: bool = True, + api_key: Optional[str] = None, +) -> None: + structured_sink = WorkflowsStructuredDataSink( + output_directory=output_directory, + output_file_type=output_file_type, + ) + progress_sink = ProgressSink.init(input_video_path=input_video_path) + sinks = [structured_sink.on_prediction, progress_sink.on_prediction] + video_sink: Optional[WorkflowsVideoSink] = None + if save_image_outputs_as_video: + video_sink = WorkflowsVideoSink.init( + input_video_path=input_video_path, + output_directory=output_directory, + ) + sinks.append(video_sink.on_prediction) + pipeline = InferencePipeline.init_with_workflow( + video_reference=[input_video_path], + workflow_specification=workflow_specification, + workspace_name=workspace_name, + workflow_id=workflow_id, + api_key=api_key, + on_prediction=partial(multi_sink, sinks=sinks), + workflows_parameters=workflow_parameters, + serialize_results=True, + max_fps=max_fps, + ) + progress_sink.start() + pipeline.start(use_main_thread=True) + pipeline.join() + progress_sink.stop() + structured_sink.flush() + if video_sink is not None: + video_sink.release() + + +class WorkflowsStructuredDataSink: + + def __init__( + self, + output_directory: str, + output_file_type: OutputFileType, + ): + self._output_directory = output_directory + self._structured_results_buffer = defaultdict(list) + self._output_file_type = output_file_type + + def on_prediction( + self, + predictions: Union[Optional[dict], List[Optional[dict]]], + video_frames: Union[Optional[VideoFrame], List[Optional[VideoFrame]]], + ) -> None: + if not isinstance(predictions, list): + predictions = [predictions] + for stream_idx, prediction in enumerate(predictions): + if prediction is None: + continue + prediction = deduct_images(result=prediction) + if self._output_file_type is OutputFileType.CSV: + prediction = { + k: dump_objects_to_json(value=v) for k, v in prediction.items() + } + self._structured_results_buffer[stream_idx].append(prediction) + + def flush(self) -> None: + for stream_idx, buffer in self._structured_results_buffer.items(): + self._flush_stream_buffer(stream_idx=stream_idx) + + def _flush_stream_buffer(self, stream_idx: int) -> None: + content = self._structured_results_buffer[stream_idx] + if len(content) == 0: + return None + file_path = generate_results_chunk_file_name( + output_directory=self._output_directory, + results_log_type=self._output_file_type, + stream_id=stream_idx, + ) + os.makedirs(os.path.dirname(file_path), exist_ok=True) + if self._output_file_type is OutputFileType.CSV: + data_frame = pd.DataFrame(content) + data_frame.to_csv(file_path, index=False) + else: + dump_to_jsonl(path=file_path, content=content) + self._structured_results_buffer[stream_idx] = [] + + def __del__(self): + self.flush() + + +def deduct_images(result: Any) -> Any: + if isinstance(result, list): + return [deduct_images(result=e) for e in result] + if isinstance(result, set): + return {deduct_images(result=e) for e in result} + if ( + isinstance(result, dict) + and result.get("type") == "base64" + and "value" in result + ): + return "" + if isinstance(result, dict): + return {k: deduct_images(result=v) for k, v in result.items()} + return result + + +def dump_objects_to_json(value: Any) -> Any: + if isinstance(value, list) or isinstance(value, dict) or isinstance(value, set): + return json.dumps(value) + return value + + +def generate_results_chunk_file_name( + output_directory: str, + results_log_type: OutputFileType, + stream_id: int, +) -> str: + output_directory = os.path.abspath(output_directory) + chunks = glob( + os.path.join( + output_directory, + f"workflow_results_source_{stream_id}_part_*.{results_log_type.value}", + ) + ) + chunk_id = len(chunks) + return os.path.join( + output_directory, + f"workflow_results_source_{stream_id}_part_{chunk_id}.{results_log_type.value}", + ) + + +def dump_to_jsonl(path: str, content: list) -> None: + with open(path, "w") as f: + for line in content: + f.write(f"{json.dumps(line)}\n") + + +class WorkflowsVideoSink: + + @classmethod + def init( + cls, + input_video_path: str, + output_directory: str, + ) -> "WorkflowsVideoSink": + source_video_info = sv.VideoInfo.from_video_path(video_path=input_video_path) + return cls( + source_video_info=source_video_info, + output_directory=output_directory, + ) + + def __init__(self, source_video_info: sv.VideoInfo, output_directory: str): + self._video_sinks: Dict[int, Dict[str, VideoSink]] = defaultdict(dict) + self._source_video_info = source_video_info + self._output_directory = output_directory + + def on_prediction( + self, + predictions: Union[Optional[dict], List[Optional[dict]]], + video_frames: Union[Optional[VideoFrame], List[Optional[VideoFrame]]], + ) -> None: + if not isinstance(predictions, list): + predictions = [predictions] + for stream_idx, prediction in enumerate(predictions): + if prediction is None: + continue + stream_sinks = self._video_sinks[stream_idx] + for key, value in prediction.items(): + if ( + not isinstance(value, dict) + or "value" not in value + or value.get("type") != "base64" + ): + continue + if key not in stream_sinks: + video_target_path = _generate_target_path_for_video( + output_directory=self._output_directory, + source_id=stream_idx, + field_name=key, + ) + stream_sinks[key] = VideoSink( + target_path=video_target_path, + video_info=self._source_video_info, + ) + stream_sinks[key].start() + image = load_image_bgr(value) + stream_sinks[key].write_frame(frame=image) + + def release(self) -> None: + for stream_sinks in self._video_sinks.values(): + for sink in stream_sinks.values(): + sink.release() + self._video_sinks = defaultdict(dict) + + def __del__(self): + self.release() + + +class ProgressSink: + + @classmethod + def init( + cls, + input_video_path: str, + ) -> "ProgressSink": + source_video_info = sv.VideoInfo.from_video_path(video_path=input_video_path) + return cls(total_frames=source_video_info.total_frames) + + def __init__(self, total_frames: Optional[int]): + self._total_frames = total_frames + self._progress_bar = Progress() + self._task: Optional[TaskID] = None + + def start(self) -> None: + self._progress_bar.start() + self._task = self._progress_bar.add_task( + "Processing video...", + total=self._total_frames, + ) + + def on_prediction( + self, + predictions: Union[Optional[dict], List[Optional[dict]]], + video_frames: Union[Optional[VideoFrame], List[Optional[VideoFrame]]], + ) -> None: + if video_frames is None: + return None + if isinstance(video_frames, list): + raise NotImplementedError( + "ProgressSink is only to be used against single video file" + ) + self._progress_bar.update( + self._task, + completed=video_frames.frame_id, + ) + + def stop(self) -> None: + self._progress_bar.stop() + + def __del__(self): + self.stop() + + +class VideoSink: + + def __init__(self, target_path: str, video_info: sv.VideoInfo, codec: str = "mp4v"): + self.target_path = target_path + self.video_info = video_info + self.__codec = codec + self.__writer = None + + def start(self) -> None: + try: + self.__fourcc = cv2.VideoWriter_fourcc(*self.__codec) + except TypeError as e: + print(str(e) + ". Defaulting to mp4v...") + self.__fourcc = cv2.VideoWriter_fourcc(*"mp4v") + self.__writer = cv2.VideoWriter( + self.target_path, + self.__fourcc, + self.video_info.fps, + self.video_info.resolution_wh, + ) + + def write_frame(self, frame: np.ndarray): + self.__writer.write(frame) + + def release(self) -> None: + self.__writer.release() + + +def _generate_target_path_for_video( + output_directory: str, source_id: int, field_name: str +) -> str: + os.makedirs(os.path.abspath(output_directory), exist_ok=True) + return os.path.join( + os.path.abspath(output_directory), + f"source_{source_id}_output_{field_name}_preview.mp4", + ) diff --git a/inference_cli/lib/workflows_adapter.py b/inference_cli/lib/workflows_adapter.py deleted file mode 100644 index 9d51e2fee..000000000 --- a/inference_cli/lib/workflows_adapter.py +++ /dev/null @@ -1,158 +0,0 @@ -import os.path -from collections import defaultdict -from typing import List, Literal, Union, Optional, Dict - -import cv2 -import numpy as np -from supervision import VideoInfo - -from inference.core.interfaces.camera.entities import VideoFrame -from inference.core.workflows.execution_engine.entities.base import WorkflowImageData - - -def process_video_with_workflow() -> None: - pass - - -class WorkflowsStructuredDataSink: - - @classmethod - def init( - cls, - output_directory: str, - results_log_type: Literal["csv", "jsonl"], - max_entries_in_logs_chunk: int, - ) -> "WorkflowsStructuredDataSink": - return cls( - output_directory=output_directory, - structured_results_buffer=[], - results_log_type=results_log_type, - max_entries_in_logs_chunk=max_entries_in_logs_chunk, - ) - - def __init__( - self, - output_directory: str, - structured_results_buffer: List[dict], - results_log_type: Literal["csv", "jsonl"], - max_entries_in_logs_chunk: int, - ): - self._output_directory = output_directory - self._structured_results_buffer = structured_results_buffer - self._results_log_type = results_log_type - self._max_entries_in_logs_chunk = max_entries_in_logs_chunk - - def on_prediction( - self, - predictions: Union[Optional[dict], List[Optional[dict]]], - video_frames: Union[Optional[VideoFrame], List[Optional[VideoFrame]]], - ) -> None: - if not isinstance(predictions, list): - predictions = [predictions] - for prediction in predictions: - if prediction is None: - continue - - def __del__(self): - pass - - -def dump_content() -> None: - pass - - -class WorkflowsVideoSink: - - @classmethod - def init( - cls, - input_video_path: str, - output_directory: str, - ) -> "WorkflowsVideoSink": - source_video_info = VideoInfo.from_video_path(video_path=input_video_path) - return cls( - source_video_info=source_video_info, - output_directory=output_directory, - ) - - def __init__( - self, - source_video_info: VideoInfo, - output_directory: str - ): - self._video_sinks: Dict[int, Dict[str, VideoSink]] = defaultdict(dict) - self._source_video_info = source_video_info - self._output_directory = output_directory - - def on_prediction( - self, - predictions: Union[Optional[dict], List[Optional[dict]]], - video_frames: Union[Optional[VideoFrame], List[Optional[VideoFrame]]], - ) -> None: - if not isinstance(predictions, list): - predictions = [predictions] - for stream_idx, prediction in enumerate(predictions): - if prediction is None: - continue - stream_sinks = self._video_sinks[stream_idx] - for key, value in prediction.items(): - if not isinstance(value, WorkflowImageData): - continue - if key not in stream_sinks: - video_target_path = _generate_target_path_for_video( - output_directory=self._output_directory, - source_id=stream_idx, - field_name=key, - ) - stream_sinks[key] = VideoSink( - target_path=video_target_path, - video_info=self._source_video_info, - ) - stream_sinks[key].start() - stream_sinks[key].write_frame(frame=value.numpy_image) - - def __del__(self): - for stream_sinks in self._video_sinks.values(): - for sink in stream_sinks.values(): - sink.release() - - - -class VideoSink: - - def __init__(self, target_path: str, video_info: VideoInfo, codec: str = "mp4v"): - self.target_path = target_path - self.video_info = video_info - self.__codec = codec - self.__writer = None - - def start(self) -> None: - try: - self.__fourcc = cv2.VideoWriter_fourcc(*self.__codec) - except TypeError as e: - print(str(e) + ". Defaulting to mp4v...") - self.__fourcc = cv2.VideoWriter_fourcc(*"mp4v") - self.__writer = cv2.VideoWriter( - self.target_path, - self.__fourcc, - self.video_info.fps, - self.video_info.resolution_wh, - ) - - def write_frame(self, frame: np.ndarray): - """ - Writes a single video frame to the target video file. - - Args: - frame (np.ndarray): The video frame to be written to the file. The frame - must be in BGR color format. - """ - self.__writer.write(frame) - - def release(self) -> None: - self.__writer.release() - - -def _generate_target_path_for_video(output_directory: str, source_id: int, field_name: str) -> str: - os.makedirs(os.path.abspath(output_directory), exist_ok=True) - return os.path.join(os.path.abspath(output_directory), f"source_{source_id}_output_{field_name}_preview.mp4") \ No newline at end of file diff --git a/inference_cli/main.py b/inference_cli/main.py index 1258db3f1..0073efa83 100644 --- a/inference_cli/main.py +++ b/inference_cli/main.py @@ -7,11 +7,13 @@ from inference_cli.benchmark import benchmark_app from inference_cli.cloud import cloud_app from inference_cli.server import server_app +from inference_cli.workflows import workflows_app app = typer.Typer() app.add_typer(server_app, name="server") app.add_typer(cloud_app, name="cloud") app.add_typer(benchmark_app, name="benchmark") +app.add_typer(workflows_app, name="workflows") def version_callback(value: bool): diff --git a/inference_cli/workflows.py b/inference_cli/workflows.py new file mode 100644 index 000000000..40a1afbdb --- /dev/null +++ b/inference_cli/workflows.py @@ -0,0 +1,212 @@ +from typing import Any, Dict, List, Optional + +import typer +from typing_extensions import Annotated + +from inference_cli.lib.utils import read_json +from inference_cli.lib.workflows.core import run_video_processing_with_workflows +from inference_cli.lib.workflows.entities import OutputFileType + +workflows_app = typer.Typer(help="Commands for interacting with Roboflow Workflows") + + +@workflows_app.command( + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, + help="Process video file with your Workflow locally (inference Python package required)", +) +def process_video( + context: typer.Context, + video_path: Annotated[ + str, + typer.Option( + "--video_path", + "-v", + help="Path to video to be processed", + ), + ], + output_directory: Annotated[ + str, + typer.Option( + "--output_dir", + "-o", + help="Path to output directory", + ), + ], + output_file_type: Annotated[ + OutputFileType, + typer.Option( + "--output_file_type", + "-ft", + help="Type of the output file", + case_sensitive=False, + ), + ] = OutputFileType.CSV, + workflow_specification_path: Annotated[ + Optional[str], + typer.Option( + "--workflow_spec", + "-ws", + help="Path to JSON file with Workflow definition " + "(mutually exclusive with `workspace_name` and `workflow_id`)", + ), + ] = None, + workspace_name: Annotated[ + Optional[str], + typer.Option( + "--workspace_name", + "-wn", + help="Name of Roboflow workspace the that Workflow belongs to " + "(mutually exclusive with `workflow_specification_path`)", + ), + ] = None, + workflow_id: Annotated[ + Optional[str], + typer.Option( + "--workflow_id", + "-wid", + help="Identifier of a Workflow on Roboflow platform " + "(mutually exclusive with `workflow_specification_path`)", + ), + ] = None, + workflow_parameters_path: Annotated[ + Optional[str], + typer.Option( + "--workflow_params_file", + help="Path to JSON document with Workflow parameters - helpful when Workflow is parametrized and " + "passing the parameters in CLI is not handy / impossible due to typing conversion issues.", + ), + ] = None, + max_fps: Annotated[ + Optional[float], + typer.Option( + "--max_fps", + help="Use the parameter to limit video FPS (additional frames will be skipped in processing).", + ), + ] = None, + image_outputs_as_video: Annotated[ + bool, + typer.Option( + "--save_out_video/--no_save_out_video", + help="Flag deciding if image outputs of the workflow should be saved as video file", + ), + ] = True, + api_key: Annotated[ + Optional[str], + typer.Option( + "--api-key", + "-a", + help="Roboflow API key for your workspace. If not given - env variable `ROBOFLOW_API_KEY` will be used", + ), + ] = None, + debug_mode: Annotated[ + bool, + typer.Option( + "--debug_mode/--no_debug_mode", + help="Flag enabling errors stack traces to be displayed (helpful for debugging)", + ), + ] = False, +): + try: + workflow_parameters = prepare_workflow_parameters( + context=context, + workflows_parameters_path=workflow_parameters_path, + ) + workflow_specification = None + if workflow_specification_path is not None: + workflow_specification = read_json(path=workflow_specification_path) + run_video_processing_with_workflows( + input_video_path=video_path, + output_directory=output_directory, + output_file_type=output_file_type, + workflow_specification=workflow_specification, + workspace_name=workspace_name, + workflow_id=workflow_id, + workflow_parameters=workflow_parameters, + max_fps=max_fps, + save_image_outputs_as_video=image_outputs_as_video, + api_key=api_key, + ) + except KeyboardInterrupt: + print("Command interrupted - results may not be fully consistent.") + return + except Exception as error: + if debug_mode: + raise error + typer.echo(f"Command failed. Cause: {error}") + raise typer.Exit(code=1) + + +def prepare_workflow_parameters( + context: typer.Context, + workflows_parameters_path: Optional[str], +) -> Optional[dict]: + workflow_parameters = _parse_extra_args(context=context) + if workflows_parameters_path is None: + return workflow_parameters + workflows_parameters_from_file = read_json(path=workflows_parameters_path) + if workflow_parameters is None: + return workflows_parameters_from_file + # explicit params in CLI override file params + workflows_parameters_from_file.update(workflow_parameters) + return workflows_parameters_from_file + + +def _parse_extra_args(context: typer.Context) -> Optional[Dict[str, Any]]: + indices_of_named_parameters = _get_indices_of_named_parameters( + arguments=context.args + ) + if not indices_of_named_parameters: + return None + params_values_spans = _calculate_spans_between_indices( + indices=indices_of_named_parameters, + list_length=len(context.args), + ) + result = {} + for param_index, values_span in zip( + indices_of_named_parameters, params_values_spans + ): + if values_span < 1: + continue + name = context.args[param_index].lstrip("-") + values = [ + _parse_value(value=v) + for v in context.args[param_index + 1 : param_index + values_span + 1] + ] + if len(values) == 1: + values = values[0] + result[name] = values + if not result: + return None + return result + + +def _get_indices_of_named_parameters(arguments: List[str]) -> List[int]: + result = [] + for index, arg in enumerate(arguments): + if arg.startswith("-"): + result.append(index) + return result + + +def _calculate_spans_between_indices(indices: List[int], list_length: int) -> List[int]: + if not indices: + return [] + diffs = [x - y - 1 for x, y in zip(indices[1:], indices)] + diffs.append(list_length - indices[-1] - 1) + return diffs + + +def _parse_value(value: str) -> Any: + try: + return float(value) + except ValueError: + pass + try: + return int(value) + except ValueError: + pass + if value.lower() in {"y", "yes", "true"}: + return True + if value.lower() in {"n", "no", "false"}: + return False + return value From 33160afa49861d21f2c2a405d44bbcbfc624c954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Tue, 26 Nov 2024 16:55:10 +0100 Subject: [PATCH 03/16] Add commands to execute batch processing on directory of images --- inference_cli/lib/utils.py | 10 +- inference_cli/lib/workflows/common.py | 244 +++++++++++ inference_cli/lib/workflows/core.py | 125 +++++- inference_cli/lib/workflows/entities.py | 5 + .../lib/workflows/local_image_adapter.py | 321 +++++++++++++++ .../lib/workflows/remote_image_adapter.py | 289 +++++++++++++ inference_cli/lib/workflows/video_adapter.py | 37 +- inference_cli/workflows.py | 388 +++++++++++++++++- 8 files changed, 1381 insertions(+), 38 deletions(-) create mode 100644 inference_cli/lib/workflows/common.py create mode 100644 inference_cli/lib/workflows/local_image_adapter.py create mode 100644 inference_cli/lib/workflows/remote_image_adapter.py diff --git a/inference_cli/lib/utils.py b/inference_cli/lib/utils.py index 4d3688999..a3da7e133 100644 --- a/inference_cli/lib/utils.py +++ b/inference_cli/lib/utils.py @@ -1,7 +1,7 @@ import json import os.path import subprocess -from typing import Dict, List, Optional, Union +from typing import Dict, Iterable, List, Optional, Union from supervision.utils.file import read_yaml_file @@ -64,6 +64,14 @@ def dump_json(path: str, content: Union[dict, list]) -> None: json.dump(content, f) +def dump_jsonl(path: str, content: Iterable[dict]) -> None: + parent_dir = os.path.dirname(os.path.abspath(path)) + os.makedirs(parent_dir, exist_ok=True) + with open(path, "w") as f: + for line in content: + f.write(f"{json.dumps(line)}\n") + + def initialise_client( host: str, api_key: Optional[str], model_configuration: Optional[str], **kwargs ) -> InferenceHTTPClient: diff --git a/inference_cli/lib/workflows/common.py b/inference_cli/lib/workflows/common.py new file mode 100644 index 000000000..27991cb8d --- /dev/null +++ b/inference_cli/lib/workflows/common.py @@ -0,0 +1,244 @@ +import json +import os.path +import re +from datetime import datetime +from threading import Lock +from typing import Any, Dict, List, Optional, Set, TextIO, Tuple + +import cv2 +import numpy as np +import pandas as pd +import pybase64 +import supervision as sv +from rich.progress import track + +from inference_cli.lib.utils import dump_json, dump_jsonl, read_json +from inference_cli.lib.workflows.entities import OutputFileType + +BASE64_DATA_TYPE_PATTERN = re.compile(r"^data:image\/[a-z]+;base64,") + +IMAGES_EXTENSIONS = [ + "bmp", + "dib", + "jpeg", + "jpg", + "jpe", + "jp2", + "png", + "webp", +] +IMAGES_EXTENSIONS += [e.upper() for e in IMAGES_EXTENSIONS] + + +def open_progress_log(output_directory: str) -> Tuple[TextIO, Set[str]]: + os.makedirs(output_directory, exist_ok=True) + log_path = get_progress_log_path(output_directory=output_directory) + if not os.path.exists(log_path): + file_descriptor = open(log_path, "w+") + else: + file_descriptor = open(log_path, "r+") + all_processed_files = set(line.strip() for line in file_descriptor.readlines()) + return file_descriptor, all_processed_files + + +def denote_image_processed( + log_file: TextIO, image_path: str, lock: Optional[Lock] = None +) -> None: + image_name = os.path.basename(image_path) + if lock is None: + log_file.write(f"{image_name}\n") + log_file.flush() + return None + with lock: + log_file.write(f"{image_name}\n") + log_file.flush() + return None + + +def get_progress_log_path(output_directory: str) -> str: + return os.path.abspath(os.path.join(output_directory, "progress.log")) + + +def dump_image_processing_results( + result: Dict[str, Any], + image_path: str, + output_directory: str, + save_image_outputs: bool, +) -> None: + images_in_result = [] + if save_image_outputs: + images_in_result = extract_images_from_result(result=result) + structured_content = deduct_images(result=result) + image_results_dir = construct_image_output_dir_path( + image_path=image_path, + output_directory=output_directory, + ) + os.makedirs(image_results_dir, exist_ok=True) + structured_results_path = os.path.join(image_results_dir, "results.json") + dump_json( + path=structured_results_path, + content=structured_content, + ) + dump_images_outputs( + image_results_dir=image_results_dir, + images_in_result=images_in_result, + ) + + +def dump_images_outputs( + image_results_dir: str, + images_in_result: List[Tuple[str, np.ndarray]], +) -> None: + for image_key, image in images_in_result: + target_path = os.path.join(image_results_dir, f"{image_key}.jpg") + target_path_dir = os.path.dirname(target_path) + os.makedirs(target_path_dir, exist_ok=True) + cv2.imwrite(target_path, image) + + +def construct_image_output_dir_path(image_path: str, output_directory: str) -> str: + image_file_name = os.path.basename(image_path) + return os.path.abspath(os.path.join(output_directory, image_file_name)) + + +def deduct_images(result: Any) -> Any: + if isinstance(result, list): + return [deduct_images(result=e) for e in result] + if isinstance(result, set): + return {deduct_images(result=e) for e in result} + if ( + isinstance(result, dict) + and result.get("type") == "base64" + and "value" in result + ): + return "" + if isinstance(result, np.ndarray): + return "" + if isinstance(result, dict): + return {k: deduct_images(result=v) for k, v in result.items()} + return result + + +def extract_images_from_result( + result: Any, key_prefix: str = "" +) -> List[Tuple[str, np.ndarray]]: + if ( + isinstance(result, dict) + and result.get("type") == "base64" + and "value" in result + ): + loaded_image = decode_base64_image(result["value"]) + return [(key_prefix, loaded_image)] + if isinstance(result, np.ndarray): + return [(key_prefix, result)] + current_result = [] + if isinstance(result, dict): + for key, value in result.items(): + current_result.extend( + extract_images_from_result( + result=value, key_prefix=f"{key_prefix}/{key}".lstrip("/") + ) + ) + elif isinstance(result, list): + for idx, element in enumerate(result): + current_result.extend( + extract_images_from_result( + result=element, key_prefix=f"{key_prefix}/{idx}".lstrip("/") + ) + ) + return current_result + + +def decode_base64_image(payload: str) -> np.ndarray: + value = BASE64_DATA_TYPE_PATTERN.sub("", payload) + value = pybase64.b64decode(value) + image_np = np.frombuffer(value, np.uint8) + result = cv2.imdecode(image_np, cv2.IMREAD_COLOR) + if result is None: + raise ValueError("Could not decode image") + return result + + +def get_all_images_in_directory(input_directory: str) -> List[str]: + if os.name == "nt": + # Windows paths are case-insensitive, hence deduplication + # is needed, as IMAGES_EXTENSIONS contains extensions + # in lower- and upper- cases version. + return list( + { + path.as_posix().lower() + for path in sv.list_files_with_extensions( + directory=input_directory, + extensions=IMAGES_EXTENSIONS, + ) + } + ) + return [ + path.as_posix() + for path in sv.list_files_with_extensions( + directory=input_directory, + extensions=IMAGES_EXTENSIONS, + ) + ] + + +def report_failed_files( + failed_files: List[Tuple[str, str]], output_directory: str +) -> None: + if not failed_files: + return None + os.makedirs(output_directory, exist_ok=True) + timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f") + failed_files_path = os.path.abspath( + os.path.join(output_directory, f"failed_files_processing_{timestamp}.json") + ) + content = [{"file_path": e[0], "cause": e[1]} for e in failed_files] + dump_json(path=failed_files_path, content=content) + print( + f"Detected {len(failed_files)} processing failures. Details saved under: {failed_files_path}" + ) + + +def aggregate_batch_processing_results( + output_directory: str, + aggregation_format: OutputFileType, +) -> None: + file_descriptor, all_processed_files = open_progress_log( + output_directory=output_directory + ) + file_descriptor.close() + all_results = [ + os.path.join(output_directory, f, "results.json") + for f in all_processed_files + if os.path.exists(os.path.join(output_directory, f, "results.json")) + ] + decoded_content = [] + for result_path in track(all_results, description="Grabbing processing results..."): + decoded_content.append(read_json(path=result_path)) + if aggregation_format is OutputFileType.JSONL: + aggregated_results_path = os.path.join( + output_directory, "aggregated_results.jsonl" + ) + dump_jsonl( + path=aggregated_results_path, + content=track( + decoded_content, description="Dumping aggregated results to JSONL..." + ), + ) + return None + dumped_results = [] + for decoded_result in track( + decoded_content, description="Dumping aggregated results to CSV..." + ): + dumped_results.append( + {k: dump_objects_to_json(value=v) for k, v in decoded_result.items()} + ) + data_frame = pd.DataFrame(dumped_results) + aggregated_results_path = os.path.join(output_directory, "aggregated_results.csv") + data_frame.to_csv(aggregated_results_path, index=False) + + +def dump_objects_to_json(value: Any) -> Any: + if isinstance(value, list) or isinstance(value, dict) or isinstance(value, set): + return json.dumps(value) + return value diff --git a/inference_cli/lib/workflows/core.py b/inference_cli/lib/workflows/core.py index 97cc9b317..beed91f99 100644 --- a/inference_cli/lib/workflows/core.py +++ b/inference_cli/lib/workflows/core.py @@ -1,8 +1,12 @@ import os -from typing import Any, Dict, Literal, Optional, Union +from typing import Any, Dict, Optional -from inference_cli.lib.utils import ensure_inference_is_installed, read_json -from inference_cli.lib.workflows.entities import OutputFileType +from inference_cli.lib.utils import ensure_inference_is_installed +from inference_cli.lib.workflows.entities import OutputFileType, ProcessingTarget +from inference_cli.lib.workflows.remote_image_adapter import ( + process_image_directory_with_workflow_using_api, + process_image_with_workflow_using_api, +) def run_video_processing_with_workflows( @@ -13,6 +17,7 @@ def run_video_processing_with_workflows( workspace_name: Optional[str] = None, workflow_id: Optional[str] = None, workflow_parameters: Optional[Dict[str, Any]] = None, + image_input_name: str = "image", max_fps: Optional[float] = None, save_image_outputs_as_video: bool = True, api_key: Optional[str] = None, @@ -33,7 +38,121 @@ def run_video_processing_with_workflows( workspace_name=workspace_name, workflow_id=workflow_id, workflow_parameters=workflow_parameters, + image_input_name=image_input_name, max_fps=max_fps, save_image_outputs_as_video=save_image_outputs_as_video, api_key=api_key, ) + + +def process_image_with_workflow( + image_path: str, + output_directory: str, + processing_target: ProcessingTarget, + workflow_specification: Optional[dict] = None, + workspace_name: Optional[str] = None, + workflow_id: Optional[str] = None, + workflow_parameters: Optional[Dict[str, Any]] = None, + image_input_name: str = "image", + api_key: Optional[str] = None, + api_url: str = "https://detect.roboflow.com", + save_image_outputs: bool = True, + force_reprocessing: bool = False, +) -> None: + if processing_target is ProcessingTarget.INFERENCE_PACKAGE: + + ensure_inference_is_installed() + + from inference_cli.lib.workflows.local_image_adapter import ( + process_image_with_workflow_using_inference_package, + ) + + process_image_with_workflow_using_inference_package( + image_path=image_path, + output_directory=output_directory, + workflow_specification=workflow_specification, + workspace_name=workspace_name, + workflow_id=workflow_id, + workflow_parameters=workflow_parameters, + image_input_name=image_input_name, + api_key=api_key, + save_image_outputs=save_image_outputs, + force_reprocessing=force_reprocessing, + ) + return None + process_image_with_workflow_using_api( + image_path=image_path, + output_directory=output_directory, + workflow_specification=workflow_specification, + workspace_name=workspace_name, + workflow_id=workflow_id, + workflow_parameters=workflow_parameters, + image_input_name=image_input_name, + api_key=api_key, + api_url=api_url, + save_image_outputs=save_image_outputs, + force_reprocessing=force_reprocessing, + ) + return None + + +def process_images_directory_with_workflow( + input_directory: str, + output_directory: str, + processing_target: ProcessingTarget, + workflow_specification: Optional[dict] = None, + workspace_name: Optional[str] = None, + workflow_id: Optional[str] = None, + workflow_parameters: Optional[Dict[str, Any]] = None, + image_input_name: str = "image", + api_key: Optional[str] = None, + save_image_outputs: bool = True, + force_reprocessing: bool = False, + aggregate_structured_results: bool = True, + aggregation_format: OutputFileType = OutputFileType.JSONL, + debug_mode: bool = False, + api_url: str = "https://detect.roboflow.com", + processing_threads: Optional[int] = None, +) -> None: + if processing_target is ProcessingTarget.INFERENCE_PACKAGE: + + ensure_inference_is_installed() + + from inference_cli.lib.workflows.local_image_adapter import ( + process_image_directory_with_workflow_using_inference_package, + ) + + process_image_directory_with_workflow_using_inference_package( + input_directory=input_directory, + output_directory=output_directory, + workflow_specification=workflow_specification, + workspace_name=workspace_name, + workflow_id=workflow_id, + workflow_parameters=workflow_parameters, + image_input_name=image_input_name, + api_key=api_key, + save_image_outputs=save_image_outputs, + force_reprocessing=force_reprocessing, + aggregate_structured_results=aggregate_structured_results, + aggregation_format=aggregation_format, + debug_mode=debug_mode, + ) + return None + process_image_directory_with_workflow_using_api( + input_directory=input_directory, + output_directory=output_directory, + workflow_specification=workflow_specification, + workspace_name=workspace_name, + workflow_id=workflow_id, + workflow_parameters=workflow_parameters, + image_input_name=image_input_name, + api_key=api_key, + api_url=api_url, + save_image_outputs=save_image_outputs, + force_reprocessing=force_reprocessing, + aggregate_structured_results=aggregate_structured_results, + aggregation_format=aggregation_format, + debug_mode=debug_mode, + processing_threads=processing_threads, + ) + return None diff --git a/inference_cli/lib/workflows/entities.py b/inference_cli/lib/workflows/entities.py index be61abf2a..a3cfe2c85 100644 --- a/inference_cli/lib/workflows/entities.py +++ b/inference_cli/lib/workflows/entities.py @@ -4,3 +4,8 @@ class OutputFileType(str, Enum): JSONL = "jsonl" CSV = "csv" + + +class ProcessingTarget(str, Enum): + API = "api" + INFERENCE_PACKAGE = "inference_package" diff --git a/inference_cli/lib/workflows/local_image_adapter.py b/inference_cli/lib/workflows/local_image_adapter.py new file mode 100644 index 000000000..da6ced4bd --- /dev/null +++ b/inference_cli/lib/workflows/local_image_adapter.py @@ -0,0 +1,321 @@ +import os +from concurrent.futures import ThreadPoolExecutor +from functools import partial +from threading import Lock +from typing import Any, Callable, Dict, List, Optional, TextIO, Tuple + +import cv2 +from rich.progress import Progress, TaskID + +from inference.core.cache import cache +from inference.core.env import API_KEY, MAX_ACTIVE_MODELS +from inference.core.exceptions import MissingApiKeyError +from inference.core.managers.active_learning import BackgroundTaskActiveLearningManager +from inference.core.managers.decorators.base import ModelManagerDecorator +from inference.core.managers.decorators.fixed_size_cache import WithFixedSizeCache +from inference.core.registries.roboflow import RoboflowModelRegistry +from inference.core.roboflow_api import get_workflow_specification +from inference.core.workflows.execution_engine.core import ExecutionEngine +from inference.core.workflows.execution_engine.profiling.core import ( + NullWorkflowsProfiler, +) +from inference.models.utils import ROBOFLOW_MODEL_TYPES +from inference_cli.lib.logger import CLI_LOGGER +from inference_cli.lib.workflows.common import ( + aggregate_batch_processing_results, + denote_image_processed, + dump_image_processing_results, + get_all_images_in_directory, + open_progress_log, + report_failed_files, +) +from inference_cli.lib.workflows.entities import OutputFileType + + +def process_image_with_workflow_using_inference_package( + image_path: str, + output_directory: str, + workflow_specification: Optional[dict] = None, + workspace_name: Optional[str] = None, + workflow_id: Optional[str] = None, + workflow_parameters: Optional[Dict[str, Any]] = None, + image_input_name: str = "image", + api_key: Optional[str] = None, + save_image_outputs: bool = True, + force_reprocessing: bool = False, +) -> None: + if api_key is None: + api_key = API_KEY + log_file, log_content = open_progress_log(output_directory=output_directory) + try: + image_name = os.path.basename(image_path) + if image_name in log_content and not force_reprocessing: + return None + model_manager = _prepare_model_manager() + workflow_specification = _get_workflow_specification( + workflow_specification=workflow_specification, + workspace_name=workspace_name, + workflow_id=workflow_id, + api_key=api_key, + ) + result = _run_workflow_for_single_image_with_inference( + model_manager=model_manager, + image_path=image_path, + workflow_specification=workflow_specification, + workflow_id=workflow_id, + image_input_name=image_input_name, + workflow_parameters=workflow_parameters, + api_key=api_key, + ) + dump_image_processing_results( + result=result, + image_path=image_path, + output_directory=output_directory, + save_image_outputs=save_image_outputs, + ) + denote_image_processed(log_file=log_file, image_path=image_path) + except Exception as e: + raise RuntimeError(f"Could not process image: {image_path}") from e + finally: + log_file.close() + + +def process_image_directory_with_workflow_using_inference_package( + input_directory: str, + output_directory: str, + workflow_specification: Optional[dict] = None, + workspace_name: Optional[str] = None, + workflow_id: Optional[str] = None, + workflow_parameters: Optional[Dict[str, Any]] = None, + image_input_name: str = "image", + api_key: Optional[str] = None, + save_image_outputs: bool = True, + force_reprocessing: bool = False, + aggregate_structured_results: bool = True, + aggregation_format: OutputFileType = OutputFileType.JSONL, + debug_mode: bool = False, +) -> None: + if api_key is None: + api_key = API_KEY + files_to_process = get_all_images_in_directory(input_directory=input_directory) + log_file, log_content = open_progress_log(output_directory=output_directory) + try: + remaining_files = [ + f + for f in files_to_process + if os.path.basename(f) not in log_content or force_reprocessing + ] + print(f"Files to process: {len(remaining_files)}") + failed_files = _process_images_within_directory( + files_to_process=remaining_files, + output_directory=output_directory, + workflow_specification=workflow_specification, + workspace_name=workspace_name, + workflow_id=workflow_id, + workflow_parameters=workflow_parameters, + image_input_name=image_input_name, + api_key=api_key, + save_image_outputs=save_image_outputs, + log_file=log_file, + debug_mode=debug_mode, + ) + finally: + log_file.close() + report_failed_files(failed_files=failed_files, output_directory=output_directory) + if not aggregate_structured_results: + return None + aggregate_batch_processing_results( + output_directory=output_directory, + aggregation_format=aggregation_format, + ) + + +def _process_images_within_directory( + files_to_process: List[str], + output_directory: str, + workflow_specification: Dict[str, Any], + workspace_name: Optional[str], + workflow_id: Optional[str], + workflow_parameters: Optional[Dict[str, Any]], + image_input_name: str, + api_key: Optional[str], + save_image_outputs: bool, + log_file: TextIO, + debug_mode: bool = False, +) -> List[Tuple[str, str]]: + workflow_specification = _get_workflow_specification( + workflow_specification=workflow_specification, + workspace_name=workspace_name, + workflow_id=workflow_id, + api_key=api_key, + ) + model_manager = _prepare_model_manager() + progress_bar = Progress() + processing_task = progress_bar.add_task( + description="Processing images...", + total=len(files_to_process), + ) + failed_files = [] + on_success = partial( + _on_success, progress_bar=progress_bar, task_id=processing_task + ) + on_failure = partial( + _on_failure, + failed_files=failed_files, + progress_bar=progress_bar, + task_id=processing_task, + ) + processing_fun = partial( + _process_single_image_from_directory, + model_manager=model_manager, + workflow_specification=workflow_specification, + workflow_id=workflow_id, + image_input_name=image_input_name, + workflow_parameters=workflow_parameters, + api_key=api_key, + output_directory=output_directory, + save_image_outputs=save_image_outputs, + log_file=log_file, + on_success=on_success, + on_failure=on_failure, + debug_mode=debug_mode, + ) + with progress_bar: + for image_path in files_to_process: + processing_fun(image_path) + return failed_files + + +def _on_success( + path: str, + progress_bar: Progress, + task_id: TaskID, +) -> None: + progress_bar.update(task_id, advance=1) + + +def _on_failure( + path: str, + cause: str, + failed_files: List[Tuple[str, str]], + progress_bar: Progress, + task_id: TaskID, +) -> None: + failed_files.append((path, cause)) + progress_bar.update(task_id, advance=1) + + +def _process_single_image_from_directory( + image_path: str, + model_manager: ModelManagerDecorator, + workflow_specification: Optional[Dict[str, Any]], + workflow_id: Optional[str], + image_input_name: str, + workflow_parameters: Optional[Dict[str, Any]], + api_key: Optional[str], + output_directory: str, + save_image_outputs: bool, + log_file: TextIO, + on_success: Callable[[str], None], + on_failure: Callable[[str, str], None], + log_file_lock: Optional[Lock] = None, + debug_mode: bool = False, +) -> None: + try: + result = _run_workflow_for_single_image_with_inference( + model_manager=model_manager, + image_path=image_path, + workflow_specification=workflow_specification, + workflow_id=workflow_id, + image_input_name=image_input_name, + workflow_parameters=workflow_parameters, + api_key=api_key, + ) + dump_image_processing_results( + result=result, + image_path=image_path, + output_directory=output_directory, + save_image_outputs=save_image_outputs, + ) + denote_image_processed( + log_file=log_file, image_path=image_path, lock=log_file_lock + ) + on_success(image_path) + except Exception as error: + error_summary = f"Error in processing {image_path}. Error type: {error.__class__.__name__} - {error}" + if debug_mode: + CLI_LOGGER.exception(error_summary) + on_failure(image_path, error_summary) + + +def _get_workflow_specification( + workflow_specification: Optional[dict] = None, + workspace_name: Optional[str] = None, + workflow_id: Optional[str] = None, + api_key: Optional[str] = None, +) -> Dict[str, Any]: + if workflow_specification is not None: + return workflow_specification + named_workflow_specified = (workspace_name is not None) and ( + workflow_id is not None + ) + if not (named_workflow_specified != (workflow_specification is not None)): + raise ValueError( + "Parameters (`workspace_name`, `workflow_id`) can be used mutually exclusive with " + "`workflow_specification`, but at least one must be set." + ) + if api_key is None: + raise MissingApiKeyError( + "Roboflow API key needs to be provided either as parameter or via env variable " + "ROBOFLOW_API_KEY. If you do not know how to get API key - visit " + "https://docs.roboflow.com/api-reference/authentication#retrieve-an-api-key to learn how to " + "retrieve one." + ) + return get_workflow_specification( + api_key=api_key, + workspace_id=workspace_name, + workflow_id=workflow_id, + use_cache=False, + ) + + +def _prepare_model_manager() -> ModelManagerDecorator: + model_registry = RoboflowModelRegistry(ROBOFLOW_MODEL_TYPES) + model_manager = BackgroundTaskActiveLearningManager( + model_registry=model_registry, cache=cache + ) + return WithFixedSizeCache( + model_manager, + max_size=MAX_ACTIVE_MODELS, + ) + + +def _run_workflow_for_single_image_with_inference( + model_manager: ModelManagerDecorator, + image_path: str, + workflow_specification: Dict[str, Any], + workflow_id: Optional[str], + image_input_name: str, + workflow_parameters: Optional[Dict[str, Any]], + api_key: Optional[str], +) -> Dict[str, Any]: + profiler = NullWorkflowsProfiler.init() + with ThreadPoolExecutor() as thread_pool_executor: + workflow_init_parameters = { + "workflows_core.model_manager": model_manager, + "workflows_core.api_key": api_key, + "workflows_core.thread_pool_executor": thread_pool_executor, + } + execution_engine = ExecutionEngine.init( + workflow_definition=workflow_specification, + init_parameters=workflow_init_parameters, + workflow_id=workflow_id, + profiler=profiler, + ) + runtime_parameters = workflow_parameters or {} + runtime_parameters[image_input_name] = cv2.imread(image_path) + results = execution_engine.run( + runtime_parameters=runtime_parameters, + serialize_results=True, + ) + return results[0] diff --git a/inference_cli/lib/workflows/remote_image_adapter.py b/inference_cli/lib/workflows/remote_image_adapter.py new file mode 100644 index 000000000..8f8861096 --- /dev/null +++ b/inference_cli/lib/workflows/remote_image_adapter.py @@ -0,0 +1,289 @@ +import logging +import os +from concurrent.futures import ThreadPoolExecutor +from functools import partial +from multiprocessing.pool import ThreadPool +from threading import Lock +from typing import Any, Callable, Dict, List, Optional, TextIO, Tuple + +import backoff +from rich.progress import Progress + +from inference_cli.lib.logger import CLI_LOGGER +from inference_cli.lib.workflows.common import ( + aggregate_batch_processing_results, + denote_image_processed, + dump_image_processing_results, + get_all_images_in_directory, + open_progress_log, + report_failed_files, +) +from inference_cli.lib.workflows.entities import OutputFileType +from inference_cli.lib.workflows.local_image_adapter import _on_failure, _on_success +from inference_sdk import ( + InferenceConfiguration, + InferenceHTTPClient, + VisualisationResponseFormat, +) +from inference_sdk.http.errors import HTTPCallErrorError + +HOSTED_API_URLS = { + "https://detect.roboflow.com", + "https://outline.roboflow.com", + "https://classify.roboflow.com", + "https://lambda-object-detection.staging.roboflow.com", + "https://lambda-instance-segmentation.staging.roboflow.com", + "https://lambda-classification.staging.roboflow.com", +} + + +def process_image_with_workflow_using_api( + image_path: str, + output_directory: str, + workflow_specification: Optional[dict] = None, + workspace_name: Optional[str] = None, + workflow_id: Optional[str] = None, + workflow_parameters: Optional[Dict[str, Any]] = None, + image_input_name: str = "image", + api_key: Optional[str] = None, + api_url: str = "https://detect.roboflow.com", + save_image_outputs: bool = True, + force_reprocessing: bool = False, +) -> None: + if api_key is None: + api_key = _get_api_key_from_env() + log_file, log_content = open_progress_log(output_directory=output_directory) + try: + image_name = os.path.basename(image_path) + if image_name in log_content and not force_reprocessing: + return None + result = _run_workflow_for_single_image_through_api( + image_path=image_path, + workflow_specification=workflow_specification, + workspace_name=workspace_name, + workflow_id=workflow_id, + image_input_name=image_input_name, + workflow_parameters=workflow_parameters, + api_key=api_key, + api_url=api_url, + ) + if result is None: + return None + dump_image_processing_results( + result=result, + image_path=image_path, + output_directory=output_directory, + save_image_outputs=save_image_outputs, + ) + denote_image_processed(log_file=log_file, image_path=image_path) + except Exception as e: + raise RuntimeError(f"Could not process image: {image_path}") from e + finally: + log_file.close() + + +def process_image_directory_with_workflow_using_api( + input_directory: str, + output_directory: str, + workflow_specification: Optional[dict] = None, + workspace_name: Optional[str] = None, + workflow_id: Optional[str] = None, + workflow_parameters: Optional[Dict[str, Any]] = None, + image_input_name: str = "image", + api_key: Optional[str] = None, + api_url: str = "https://detect.roboflow.com", + save_image_outputs: bool = True, + force_reprocessing: bool = False, + aggregate_structured_results: bool = True, + aggregation_format: OutputFileType = OutputFileType.JSONL, + debug_mode: bool = False, + processing_threads: Optional[int] = None, +) -> None: + if api_key is None: + api_key = _get_api_key_from_env() + if processing_threads is None: + if _is_roboflow_hosted_api(api_url=api_url): + processing_threads = 32 + else: + processing_threads = 1 + files_to_process = get_all_images_in_directory(input_directory=input_directory) + log_file, log_content = open_progress_log(output_directory=output_directory) + try: + remaining_files = [ + f + for f in files_to_process + if os.path.basename(f) not in log_content or force_reprocessing + ] + print(f"Files to process: {len(remaining_files)}") + failed_files = _process_images_within_directory_with_api( + files_to_process=remaining_files, + output_directory=output_directory, + workflow_specification=workflow_specification, + workspace_name=workspace_name, + workflow_id=workflow_id, + workflow_parameters=workflow_parameters, + image_input_name=image_input_name, + api_key=api_key, + save_image_outputs=save_image_outputs, + log_file=log_file, + api_url=api_url, + processing_threads=processing_threads, + debug_mode=debug_mode, + ) + finally: + log_file.close() + report_failed_files(failed_files=failed_files, output_directory=output_directory) + if not aggregate_structured_results: + return None + aggregate_batch_processing_results( + output_directory=output_directory, + aggregation_format=aggregation_format, + ) + + +def _process_images_within_directory_with_api( + files_to_process: List[str], + output_directory: str, + workflow_specification: Dict[str, Any], + workspace_name: Optional[str], + workflow_id: Optional[str], + workflow_parameters: Optional[Dict[str, Any]], + image_input_name: str, + api_key: Optional[str], + save_image_outputs: bool, + log_file: TextIO, + api_url: str, + processing_threads: int, + debug_mode: bool = False, +) -> List[Tuple[str, str]]: + progress_bar = Progress() + processing_task = progress_bar.add_task( + description="Processing images...", + total=len(files_to_process), + ) + failed_files = [] + on_success = partial( + _on_success, progress_bar=progress_bar, task_id=processing_task + ) + on_failure = partial( + _on_failure, + failed_files=failed_files, + progress_bar=progress_bar, + task_id=processing_task, + ) + log_file_lock = Lock() + processing_fun = partial( + _process_single_image_from_directory, + workflow_specification=workflow_specification, + workspace_name=workspace_name, + workflow_id=workflow_id, + image_input_name=image_input_name, + workflow_parameters=workflow_parameters, + api_url=api_url, + api_key=api_key, + output_directory=output_directory, + save_image_outputs=save_image_outputs, + log_file=log_file, + on_success=on_success, + on_failure=on_failure, + log_file_lock=log_file_lock, + debug_mode=debug_mode, + ) + with progress_bar: + with ThreadPool(processes=processing_threads) as pool: + _ = pool.map( + processing_fun, + files_to_process, + ) + return failed_files + + +def _process_single_image_from_directory( + image_path: str, + workflow_specification: Dict[str, Any], + workspace_name: Optional[str], + workflow_id: Optional[str], + image_input_name: str, + workflow_parameters: Optional[Dict[str, Any]], + api_url: str, + api_key: Optional[str], + output_directory: str, + save_image_outputs: bool, + log_file: TextIO, + on_success: Callable[[str], None], + on_failure: Callable[[str, str], None], + log_file_lock: Optional[Lock] = None, + debug_mode: bool = False, +) -> None: + try: + result = _run_workflow_for_single_image_through_api( + image_path=image_path, + workflow_specification=workflow_specification, + workspace_name=workspace_name, + workflow_id=workflow_id, + image_input_name=image_input_name, + workflow_parameters=workflow_parameters, + api_key=api_key, + api_url=api_url, + ) + dump_image_processing_results( + result=result, + image_path=image_path, + output_directory=output_directory, + save_image_outputs=save_image_outputs, + ) + denote_image_processed( + log_file=log_file, image_path=image_path, lock=log_file_lock + ) + on_success(image_path) + except Exception as error: + error_summary = f"Error in processing {image_path}. Error type: {error.__class__.__name__} - {error}" + if debug_mode: + CLI_LOGGER.exception(error_summary) + on_failure(image_path, error_summary) + + +@backoff.on_exception( + backoff.constant, + exception=HTTPCallErrorError, + max_tries=3, + interval=1, + backoff_log_level=logging.DEBUG, + giveup_log_level=logging.DEBUG, +) +def _run_workflow_for_single_image_through_api( + image_path: str, + workflow_specification: Optional[dict], + workspace_name: Optional[str], + workflow_id: Optional[str], + image_input_name: str, + workflow_parameters: Optional[Dict[str, Any]], + api_key: Optional[str], + api_url: str, +) -> Dict[str, Any]: + client = InferenceHTTPClient( + api_url=api_url, + api_key=api_key, + ).configure( + InferenceConfiguration( + output_visualisation_format=VisualisationResponseFormat.NUMPY + ) + ) + result = client.run_workflow( + workspace_name=workspace_name, + workflow_id=workflow_id, + specification=workflow_specification, + images={ + image_input_name: [image_path], + }, + parameters=workflow_parameters, + )[0] + return result + + +def _is_roboflow_hosted_api(api_url: str) -> bool: + return api_url in HOSTED_API_URLS + + +def _get_api_key_from_env() -> Optional[str]: + return os.getenv("ROBOFLOW_API_KEY") or os.getenv("API_KEY") diff --git a/inference_cli/lib/workflows/video_adapter.py b/inference_cli/lib/workflows/video_adapter.py index f1e8f8b2c..f4f5f3f4f 100644 --- a/inference_cli/lib/workflows/video_adapter.py +++ b/inference_cli/lib/workflows/video_adapter.py @@ -1,4 +1,3 @@ -import json import os.path from collections import defaultdict from functools import partial @@ -15,6 +14,8 @@ from inference.core.interfaces.camera.entities import VideoFrame from inference.core.interfaces.stream.sinks import multi_sink from inference.core.utils.image_utils import load_image_bgr +from inference_cli.lib.utils import dump_jsonl +from inference_cli.lib.workflows.common import deduct_images, dump_objects_to_json from inference_cli.lib.workflows.entities import OutputFileType @@ -26,6 +27,7 @@ def process_video_with_workflow( workspace_name: Optional[str] = None, workflow_id: Optional[str] = None, workflow_parameters: Optional[Dict[str, Any]] = None, + image_input_name: str = "image", max_fps: Optional[float] = None, save_image_outputs_as_video: bool = True, api_key: Optional[str] = None, @@ -52,6 +54,7 @@ def process_video_with_workflow( on_prediction=partial(multi_sink, sinks=sinks), workflows_parameters=workflow_parameters, serialize_results=True, + image_input_name=image_input_name, max_fps=max_fps, ) progress_sink.start() @@ -109,35 +112,13 @@ def _flush_stream_buffer(self, stream_idx: int) -> None: data_frame = pd.DataFrame(content) data_frame.to_csv(file_path, index=False) else: - dump_to_jsonl(path=file_path, content=content) + dump_jsonl(path=file_path, content=content) self._structured_results_buffer[stream_idx] = [] def __del__(self): self.flush() -def deduct_images(result: Any) -> Any: - if isinstance(result, list): - return [deduct_images(result=e) for e in result] - if isinstance(result, set): - return {deduct_images(result=e) for e in result} - if ( - isinstance(result, dict) - and result.get("type") == "base64" - and "value" in result - ): - return "" - if isinstance(result, dict): - return {k: deduct_images(result=v) for k, v in result.items()} - return result - - -def dump_objects_to_json(value: Any) -> Any: - if isinstance(value, list) or isinstance(value, dict) or isinstance(value, set): - return json.dumps(value) - return value - - def generate_results_chunk_file_name( output_directory: str, results_log_type: OutputFileType, @@ -157,12 +138,6 @@ def generate_results_chunk_file_name( ) -def dump_to_jsonl(path: str, content: list) -> None: - with open(path, "w") as f: - for line in content: - f.write(f"{json.dumps(line)}\n") - - class WorkflowsVideoSink: @classmethod @@ -242,7 +217,7 @@ def __init__(self, total_frames: Optional[int]): def start(self) -> None: self._progress_bar.start() self._task = self._progress_bar.add_task( - "Processing video...", + description="Processing video...", total=self._total_frames, ) diff --git a/inference_cli/workflows.py b/inference_cli/workflows.py index 40a1afbdb..de36d6b32 100644 --- a/inference_cli/workflows.py +++ b/inference_cli/workflows.py @@ -1,11 +1,16 @@ +import os.path from typing import Any, Dict, List, Optional import typer from typing_extensions import Annotated from inference_cli.lib.utils import read_json -from inference_cli.lib.workflows.core import run_video_processing_with_workflows -from inference_cli.lib.workflows.entities import OutputFileType +from inference_cli.lib.workflows.core import ( + process_image_with_workflow, + process_images_directory_with_workflow, + run_video_processing_with_workflows, +) +from inference_cli.lib.workflows.entities import OutputFileType, ProcessingTarget workflows_app = typer.Typer(help="Commands for interacting with Roboflow Workflows") @@ -71,11 +76,18 @@ def process_video( workflow_parameters_path: Annotated[ Optional[str], typer.Option( - "--workflow_params_file", + "--workflow_params", help="Path to JSON document with Workflow parameters - helpful when Workflow is parametrized and " "passing the parameters in CLI is not handy / impossible due to typing conversion issues.", ), ] = None, + image_input_name: Annotated[ + str, + typer.Option( + "--image_input_name", + help="Name of the Workflow input that defines placeholder for image to be processed", + ), + ] = "image", max_fps: Annotated[ Optional[float], typer.Option( @@ -98,6 +110,13 @@ def process_video( help="Roboflow API key for your workspace. If not given - env variable `ROBOFLOW_API_KEY` will be used", ), ] = None, + allow_override: Annotated[ + bool, + typer.Option( + "--allow_override/--no_override", + help="Flag to decide if content of output directory can be overridden.", + ), + ] = False, debug_mode: Annotated[ bool, typer.Option( @@ -107,6 +126,10 @@ def process_video( ] = False, ): try: + ensure_target_directory_is_empty( + output_directory=output_directory, + allow_override=allow_override, + ) workflow_parameters = prepare_workflow_parameters( context=context, workflows_parameters_path=workflow_parameters_path, @@ -122,6 +145,7 @@ def process_video( workspace_name=workspace_name, workflow_id=workflow_id, workflow_parameters=workflow_parameters, + image_input_name=image_input_name, max_fps=max_fps, save_image_outputs_as_video=image_outputs_as_video, api_key=api_key, @@ -136,6 +160,364 @@ def process_video( raise typer.Exit(code=1) +@workflows_app.command( + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, + help="Process single image with Workflows (inference Package may be needed dependent on mode)", +) +def process_image( + context: typer.Context, + image_path: Annotated[ + str, + typer.Option( + "--image_path", + "-i", + help="Path to image to be processed", + ), + ], + output_directory: Annotated[ + str, + typer.Option( + "--output_dir", + "-o", + help="Path to output directory", + ), + ], + processing_target: Annotated[ + ProcessingTarget, + typer.Option( + "--processing_target", + "-pt", + help="Defines where the actual processing will be done, either in inference Python package " + "running locally, or behind the API (which ensures greater throughput).", + case_sensitive=False, + ), + ] = ProcessingTarget.API, + workflow_specification_path: Annotated[ + Optional[str], + typer.Option( + "--workflow_spec", + "-ws", + help="Path to JSON file with Workflow definition " + "(mutually exclusive with `workspace_name` and `workflow_id`)", + ), + ] = None, + workspace_name: Annotated[ + Optional[str], + typer.Option( + "--workspace_name", + "-wn", + help="Name of Roboflow workspace the that Workflow belongs to " + "(mutually exclusive with `workflow_specification_path`)", + ), + ] = None, + workflow_id: Annotated[ + Optional[str], + typer.Option( + "--workflow_id", + "-wid", + help="Identifier of a Workflow on Roboflow platform " + "(mutually exclusive with `workflow_specification_path`)", + ), + ] = None, + workflow_parameters_path: Annotated[ + Optional[str], + typer.Option( + "--workflow_params", + help="Path to JSON document with Workflow parameters - helpful when Workflow is parametrized and " + "passing the parameters in CLI is not handy / impossible due to typing conversion issues.", + ), + ] = None, + image_input_name: Annotated[ + str, + typer.Option( + "--image_input_name", + help="Name of the Workflow input that defines placeholder for image to be processed", + ), + ] = "image", + api_key: Annotated[ + Optional[str], + typer.Option( + "--api-key", + "-a", + help="Roboflow API key for your workspace. If not given - env variable `ROBOFLOW_API_KEY` will be used", + ), + ] = None, + api_url: Annotated[ + str, + typer.Option( + "--api_url", + help="URL of the API that will be used for processing, when API processing target pointed.", + ), + ] = "https://detect.roboflow.com", + allow_override: Annotated[ + bool, + typer.Option( + "--allow_override/--no_override", + help="Flag to decide if content of output directory can be overridden.", + ), + ] = False, + save_image_outputs: Annotated[ + bool, + typer.Option( + "--save_image_outputs/--no_save_image_outputs", + help="Flag controlling persistence of Workflow outputs that are images", + ), + ] = True, + force_reprocessing: Annotated[ + bool, + typer.Option( + "--force_reprocessing/--no_reprocessing", + help="Flag to enforce re-processing of specific images. Images are identified by file name.", + ), + ] = False, + debug_mode: Annotated[ + bool, + typer.Option( + "--debug_mode/--no_debug_mode", + help="Flag enabling errors stack traces to be displayed (helpful for debugging)", + ), + ] = False, +): + try: + ensure_target_directory_is_empty( + output_directory=output_directory, + allow_override=allow_override, + only_files=False, + ) + workflow_parameters = prepare_workflow_parameters( + context=context, + workflows_parameters_path=workflow_parameters_path, + ) + workflow_specification = None + if workflow_specification_path is not None: + workflow_specification = read_json(path=workflow_specification_path) + process_image_with_workflow( + image_path=image_path, + output_directory=output_directory, + processing_target=processing_target, + workflow_specification=workflow_specification, + workspace_name=workspace_name, + workflow_id=workflow_id, + workflow_parameters=workflow_parameters, + image_input_name=image_input_name, + api_key=api_key, + api_url=api_url, + save_image_outputs=save_image_outputs, + force_reprocessing=force_reprocessing, + ) + except KeyboardInterrupt: + print("Command interrupted - results may not be fully consistent.") + return + except Exception as error: + if debug_mode: + raise error + typer.echo(f"Command failed. Cause: {error}") + raise typer.Exit(code=1) + + +@workflows_app.command( + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, + help="Process whole images directory with Workflows (inference Package may be needed dependent on mode)", +) +def process_images_directory( + context: typer.Context, + input_directory: Annotated[ + str, + typer.Option( + "--input_directory", + "-i", + help="Path to directory with images", + ), + ], + output_directory: Annotated[ + str, + typer.Option( + "--output_dir", + "-o", + help="Path to output directory", + ), + ], + processing_target: Annotated[ + ProcessingTarget, + typer.Option( + "--processing_target", + "-pt", + help="Defines where the actual processing will be done, either in inference Python package " + "running locally, or behind the API (which ensures greater throughput).", + case_sensitive=False, + ), + ] = ProcessingTarget.API, + workflow_specification_path: Annotated[ + Optional[str], + typer.Option( + "--workflow_spec", + "-ws", + help="Path to JSON file with Workflow definition " + "(mutually exclusive with `workspace_name` and `workflow_id`)", + ), + ] = None, + workspace_name: Annotated[ + Optional[str], + typer.Option( + "--workspace_name", + "-wn", + help="Name of Roboflow workspace the that Workflow belongs to " + "(mutually exclusive with `workflow_specification_path`)", + ), + ] = None, + workflow_id: Annotated[ + Optional[str], + typer.Option( + "--workflow_id", + "-wid", + help="Identifier of a Workflow on Roboflow platform " + "(mutually exclusive with `workflow_specification_path`)", + ), + ] = None, + workflow_parameters_path: Annotated[ + Optional[str], + typer.Option( + "--workflow_params", + help="Path to JSON document with Workflow parameters - helpful when Workflow is parametrized and " + "passing the parameters in CLI is not handy / impossible due to typing conversion issues.", + ), + ] = None, + image_input_name: Annotated[ + str, + typer.Option( + "--image_input_name", + help="Name of the Workflow input that defines placeholder for image to be processed", + ), + ] = "image", + api_key: Annotated[ + Optional[str], + typer.Option( + "--api-key", + "-a", + help="Roboflow API key for your workspace. If not given - env variable `ROBOFLOW_API_KEY` will be used", + ), + ] = None, + api_url: Annotated[ + str, + typer.Option( + "--api_url", + help="URL of the API that will be used for processing, when API processing target pointed.", + ), + ] = "https://detect.roboflow.com", + allow_override: Annotated[ + bool, + typer.Option( + "--allow_override/--no_override", + help="Flag to decide if content of output directory can be overridden.", + ), + ] = False, + save_image_outputs: Annotated[ + bool, + typer.Option( + "--save_image_outputs/--no_save_image_outputs", + help="Flag controlling persistence of Workflow outputs that are images", + ), + ] = True, + force_reprocessing: Annotated[ + bool, + typer.Option( + "--force_reprocessing/--no_reprocessing", + help="Flag to enforce re-processing of specific images. Images are identified by file name.", + ), + ] = False, + aggregate_structured_results: Annotated[ + bool, + typer.Option( + "--aggregate/--no_aggregate", + help="Flag to decide if processing results for a directory should be aggregated to a single file " + "at the end of processing.", + ), + ] = True, + aggregation_format: Annotated[ + OutputFileType, + typer.Option( + "--aggregation_format", + "-af", + help="Defines the format of aggregated results - either CSV of JSONL", + case_sensitive=False, + ), + ] = OutputFileType.CSV, + processing_threads: Annotated[ + Optional[int], + typer.Option( + "--threads", + help="Defines number of threads that will be used to send requests when processing target is API. " + "Default for Roboflow Hosted API is 32, and for on-prem deployments: 1.", + ), + ] = None, + debug_mode: Annotated[ + bool, + typer.Option( + "--debug_mode/--no_debug_mode", + help="Flag enabling errors stack traces to be displayed (helpful for debugging)", + ), + ] = False, +): + try: + ensure_target_directory_is_empty( + output_directory=output_directory, + allow_override=allow_override, + only_files=False, + ) + workflow_parameters = prepare_workflow_parameters( + context=context, + workflows_parameters_path=workflow_parameters_path, + ) + workflow_specification = None + if workflow_specification_path is not None: + workflow_specification = read_json(path=workflow_specification_path) + process_images_directory_with_workflow( + input_directory=input_directory, + output_directory=output_directory, + processing_target=processing_target, + workflow_specification=workflow_specification, + workspace_name=workspace_name, + workflow_id=workflow_id, + workflow_parameters=workflow_parameters, + image_input_name=image_input_name, + api_key=api_key, + api_url=api_url, + save_image_outputs=save_image_outputs, + force_reprocessing=force_reprocessing, + aggregate_structured_results=aggregate_structured_results, + aggregation_format=aggregation_format, + processing_threads=processing_threads, + debug_mode=debug_mode, + ) + except KeyboardInterrupt: + print("Command interrupted - results may not be fully consistent.") + return + except Exception as error: + if debug_mode: + raise error + typer.echo(f"Command failed. Cause: {error}") + raise typer.Exit(code=1) + + +def ensure_target_directory_is_empty( + output_directory: str, allow_override: bool, only_files: bool = True +) -> None: + if allow_override: + return None + if not os.path.exists(output_directory): + return None + files_in_directory = [ + f + for f in os.listdir(output_directory) + if not only_files or os.path.isfile(os.path.join(output_directory, f)) + ] + if files_in_directory: + raise RuntimeError( + f"Detected content in output directory: {output_directory}. " + f"Command cannot run, as content override is forbidden. Use `--allow_override` to proceed." + ) + + def prepare_workflow_parameters( context: typer.Context, workflows_parameters_path: Optional[str], From 8a3689211711b44821be1db1145acb8ff011ad9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Tue, 26 Nov 2024 18:15:34 +0100 Subject: [PATCH 04/16] Fix import issues --- .../lib/workflows/remote_image_adapter.py | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/inference_cli/lib/workflows/remote_image_adapter.py b/inference_cli/lib/workflows/remote_image_adapter.py index 8f8861096..f9a9fb86c 100644 --- a/inference_cli/lib/workflows/remote_image_adapter.py +++ b/inference_cli/lib/workflows/remote_image_adapter.py @@ -1,13 +1,12 @@ import logging import os -from concurrent.futures import ThreadPoolExecutor from functools import partial from multiprocessing.pool import ThreadPool from threading import Lock from typing import Any, Callable, Dict, List, Optional, TextIO, Tuple import backoff -from rich.progress import Progress +from rich.progress import Progress, TaskID from inference_cli.lib.logger import CLI_LOGGER from inference_cli.lib.workflows.common import ( @@ -19,7 +18,6 @@ report_failed_files, ) from inference_cli.lib.workflows.entities import OutputFileType -from inference_cli.lib.workflows.local_image_adapter import _on_failure, _on_success from inference_sdk import ( InferenceConfiguration, InferenceHTTPClient, @@ -198,6 +196,25 @@ def _process_images_within_directory_with_api( return failed_files +def _on_success( + path: str, + progress_bar: Progress, + task_id: TaskID, +) -> None: + progress_bar.update(task_id, advance=1) + + +def _on_failure( + path: str, + cause: str, + failed_files: List[Tuple[str, str]], + progress_bar: Progress, + task_id: TaskID, +) -> None: + failed_files.append((path, cause)) + progress_bar.update(task_id, advance=1) + + def _process_single_image_from_directory( image_path: str, workflow_specification: Dict[str, Any], From 22001c864319cc75e0f46a5df4689aa6edc2e14f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Wed, 27 Nov 2024 12:11:59 +0100 Subject: [PATCH 05/16] Fix requirements --- requirements/requirements.cli.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements/requirements.cli.txt b/requirements/requirements.cli.txt index 16e5272a7..6f85f1b89 100644 --- a/requirements/requirements.cli.txt +++ b/requirements/requirements.cli.txt @@ -10,3 +10,5 @@ GPUtil~=1.4.0 py-cpuinfo~=9.0.0 aiohttp>=3.9.0,<=3.10.11 backoff~=2.2.0 +pandas>=2.0.0,<2.3.0 +rich~=13.0.0 From 469a4debe8cff25a025efad1a14f9f6d947b43d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Wed, 27 Nov 2024 12:16:14 +0100 Subject: [PATCH 06/16] Fix requirements --- requirements/requirements.cli.txt | 1 + tests/inference_cli/unit_tests/test_workflows.py | 0 2 files changed, 1 insertion(+) create mode 100644 tests/inference_cli/unit_tests/test_workflows.py diff --git a/requirements/requirements.cli.txt b/requirements/requirements.cli.txt index 6f85f1b89..023ec54ae 100644 --- a/requirements/requirements.cli.txt +++ b/requirements/requirements.cli.txt @@ -12,3 +12,4 @@ aiohttp>=3.9.0,<=3.10.11 backoff~=2.2.0 pandas>=2.0.0,<2.3.0 rich~=13.0.0 +pybase64~=1.0.0 diff --git a/tests/inference_cli/unit_tests/test_workflows.py b/tests/inference_cli/unit_tests/test_workflows.py new file mode 100644 index 000000000..e69de29bb From fbaba5a1d8622f7691075c9c110b6b3fb15fdb8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Wed, 27 Nov 2024 12:54:50 +0100 Subject: [PATCH 07/16] Add basic test on parsing workflows parameters --- inference_cli/workflows.py | 4 +- .../unit_tests/test_workflows.py | 315 ++++++++++++++++++ 2 files changed, 317 insertions(+), 2 deletions(-) diff --git a/inference_cli/workflows.py b/inference_cli/workflows.py index de36d6b32..e2cce6ad6 100644 --- a/inference_cli/workflows.py +++ b/inference_cli/workflows.py @@ -580,11 +580,11 @@ def _calculate_spans_between_indices(indices: List[int], list_length: int) -> Li def _parse_value(value: str) -> Any: try: - return float(value) + return int(value) except ValueError: pass try: - return int(value) + return float(value) except ValueError: pass if value.lower() in {"y", "yes", "true"}: diff --git a/tests/inference_cli/unit_tests/test_workflows.py b/tests/inference_cli/unit_tests/test_workflows.py index e69de29bb..1a5979805 100644 --- a/tests/inference_cli/unit_tests/test_workflows.py +++ b/tests/inference_cli/unit_tests/test_workflows.py @@ -0,0 +1,315 @@ +import json +from typing import List, Tuple + +import typer +from typer.testing import CliRunner +from typing_extensions import Annotated + +from inference_cli.workflows import prepare_workflow_parameters + +test_app = typer.Typer(help="This is test app to verify kwargs parsing") + + +@test_app.command( + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, +) +def verify_workflow_parameters_parsing( + context: typer.Context, + string_param: Annotated[ + str, + typer.Option( + "--string_param", + "-sp", + ), + ], + int_param: Annotated[ + int, + typer.Option( + "--int_param", + "-ip", + ), + ], + float_param: Annotated[ + float, + typer.Option( + "--float_param", + "-fp", + ), + ], + tuple_param: Annotated[ + Tuple[int, float, str], + typer.Option( + "--tuple_param", + "-tp", + ), + ], + list_param: Annotated[ + List[int], + typer.Option( + "--list_param", + "-lp", + ), + ], + bool_param: Annotated[ + bool, + typer.Option("--yes/--no"), + ], + default_param: Annotated[ + str, + typer.Option( + "--default_param", + "-dp", + ), + ] = "default", +) -> None: + workflow_parameters = prepare_workflow_parameters( + context=context, + workflows_parameters_path=None, + ) + print( + json.dumps( + { + "string_param": string_param, + "int_param": int_param, + "float_param": float_param, + "tuple_param": tuple_param, + "list_param": list_param, + "bool_param": bool_param, + "default_param": default_param, + "workflow_parameters": workflow_parameters, + } + ) + ) + + +def test_command_parsing_workflows_parameters_when_no_additional_params_passed_and_long_param_names_used() -> ( + None +): + # given + runner = CliRunner() + + # when + result = runner.invoke( + test_app, + "verify_workflow_parameters_parsing " + "--string_param value " + "--int_param 2137 " + "--float_param 21.37 " + "--tuple_param 2 1.0 37 " + "--list_param 2 1 3 7 " + "--yes".split(" "), + ) + + # then + last_output_line = result.stdout.strip().split("\n")[-1] + parsed_output = json.loads(last_output_line) + assert parsed_output == { + "string_param": "value", + "int_param": 2137, + "float_param": 21.37, + "tuple_param": [2, 1.0, "37"], + "list_param": [2], + "bool_param": True, + "default_param": "default", + "workflow_parameters": None, + } + + +def test_command_parsing_workflows_parameters_when_additional_params_passed_and_long_param_names_used() -> ( + None +): + # given + runner = CliRunner() + + # when + result = runner.invoke( + test_app, + "verify_workflow_parameters_parsing " + "--string_param value " + "--int_param 2137 " + "--float_param 21.37 " + "--tuple_param 2 1.0 37 " + "--list_param 2 1 3 7 " + "--yes " + "--additional_bool yes " + "--additional_float 3.2 " + "--additional_int 3 " + "--additional_string custom " + "--additional_list 1 2.0 some".split(" "), + ) + + # then + last_output_line = result.stdout.strip().split("\n")[-1] + parsed_output = json.loads(last_output_line) + assert parsed_output == { + "string_param": "value", + "int_param": 2137, + "float_param": 21.37, + "tuple_param": [2, 1.0, "37"], + "list_param": [2], + "bool_param": True, + "default_param": "default", + "workflow_parameters": { + "additional_bool": True, + "additional_float": 3.2, + "additional_int": 3, + "additional_string": "custom", + "additional_list": [1, 2.0, "some"], + }, + } + + +def test_command_parsing_workflows_parameters_when_no_additional_params_passed_and_short_param_names_used() -> ( + None +): + # given + runner = CliRunner() + + # when + result = runner.invoke( + test_app, + "verify_workflow_parameters_parsing " + "-sp value " + "-ip 2137 " + "-fp 21.37 " + "-tp 2 1.0 37 " + "-lp 2 1 3 7 " + "--yes " + "-dp some".split(" "), + ) + + # then + last_output_line = result.stdout.strip().split("\n")[-1] + parsed_output = json.loads(last_output_line) + assert parsed_output == { + "string_param": "value", + "int_param": 2137, + "float_param": 21.37, + "tuple_param": [2, 1.0, "37"], + "list_param": [2], + "bool_param": True, + "default_param": "some", + "workflow_parameters": None, + } + + +def test_command_parsing_workflows_parameters_when_additional_params_passed_and_short_param_names_used() -> ( + None +): + # given + runner = CliRunner() + + # when + result = runner.invoke( + test_app, + "verify_workflow_parameters_parsing " + "-sp value " + "-ip 2137 " + "-fp 21.37 " + "-tp 2 1.0 37 " + "-lp 2 1 3 7 " + "--yes " + "--additional_bool yes " + "--additional_float 3.2 " + "--additional_int 3 " + "--additional_string custom " + "--additional_list 1 2.0 some".split(" "), + ) + + # then + last_output_line = result.stdout.strip().split("\n")[-1] + parsed_output = json.loads(last_output_line) + assert parsed_output == { + "string_param": "value", + "int_param": 2137, + "float_param": 21.37, + "tuple_param": [2, 1.0, "37"], + "list_param": [2], + "bool_param": True, + "default_param": "default", + "workflow_parameters": { + "additional_bool": True, + "additional_float": 3.2, + "additional_int": 3, + "additional_string": "custom", + "additional_list": [1, 2.0, "some"], + }, + } + + +def test_command_parsing_workflows_parameters_when_no_additional_params_passed_and_short_mixed_names_used() -> ( + None +): + # given + runner = CliRunner() + + # when + result = runner.invoke( + test_app, + "verify_workflow_parameters_parsing " + "--string_param value " + "-ip 2137 " + "-fp 21.37 " + "--tuple_param 2 1.0 37 " + "-lp 2 1 3 7 " + "--no " + "-dp some".split(" "), + ) + + # then + last_output_line = result.stdout.strip().split("\n")[-1] + parsed_output = json.loads(last_output_line) + assert parsed_output == { + "string_param": "value", + "int_param": 2137, + "float_param": 21.37, + "tuple_param": [2, 1.0, "37"], + "list_param": [2], + "bool_param": False, + "default_param": "some", + "workflow_parameters": None, + } + + +def test_command_parsing_workflows_parameters_when_additional_params_passed_and_short_mixed_names_used() -> ( + None +): + # given + runner = CliRunner() + + # when + result = runner.invoke( + test_app, + "verify_workflow_parameters_parsing " + "--string_param value " + "-ip 2137 " + "-fp 21.37 " + "--tuple_param 2 1.0 37 " + "-lp 2 1 3 7 " + "--no " + "--additional_bool yes " + "--additional_float 3.2 " + "--additional_int 3 " + "--additional_string custom " + "--additional_list 1 2.0 some".split(" "), + ) + + # then + last_output_line = result.stdout.strip().split("\n")[-1] + parsed_output = json.loads(last_output_line) + assert parsed_output == { + "string_param": "value", + "int_param": 2137, + "float_param": 21.37, + "tuple_param": [2, 1.0, "37"], + "list_param": [2], + "bool_param": False, + "default_param": "default", + "workflow_parameters": { + "additional_bool": True, + "additional_float": 3.2, + "additional_int": 3, + "additional_string": "custom", + "additional_list": [1, 2.0, "some"], + }, + } From ef72b3af8968eaf69248359d6c7de32f038c71a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Wed, 27 Nov 2024 13:49:54 +0100 Subject: [PATCH 08/16] Add tests to check workflow params parsing --- tests/inference_cli/unit_tests/conftest.py | 10 +++ .../unit_tests/test_workflows.py | 72 +++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 tests/inference_cli/unit_tests/conftest.py diff --git a/tests/inference_cli/unit_tests/conftest.py b/tests/inference_cli/unit_tests/conftest.py new file mode 100644 index 000000000..c05d37a6d --- /dev/null +++ b/tests/inference_cli/unit_tests/conftest.py @@ -0,0 +1,10 @@ +import tempfile +from typing import Generator + +import pytest + + +@pytest.fixture(scope="function") +def empty_directory() -> Generator[str, None, None]: + with tempfile.TemporaryDirectory() as tmp_dir: + yield tmp_dir diff --git a/tests/inference_cli/unit_tests/test_workflows.py b/tests/inference_cli/unit_tests/test_workflows.py index 1a5979805..225d57254 100644 --- a/tests/inference_cli/unit_tests/test_workflows.py +++ b/tests/inference_cli/unit_tests/test_workflows.py @@ -1,5 +1,7 @@ import json +import os.path from typing import List, Tuple +from unittest.mock import MagicMock import typer from typer.testing import CliRunner @@ -313,3 +315,73 @@ def test_command_parsing_workflows_parameters_when_additional_params_passed_and_ "additional_list": [1, 2.0, "some"], }, } + + +def test_prepare_workflow_parameters_when_neither_file_nor_additional_args_are_used() -> None: + # when + result = prepare_workflow_parameters( + context=typer.Context(command=MagicMock()), + workflows_parameters_path=None, + ) + + # then + assert result is None + + +def test_prepare_workflow_parameters_when_only_args_are_provided() -> None: + # given + context = typer.Context(command=MagicMock()) + context.args = ["--some", "value", "--list", "1", "2.9", "other", "--flag", "true"] + + # when + result = prepare_workflow_parameters( + context=context, + workflows_parameters_path=None, + ) + + # then + assert result == { + "some": "value", + "list": [1, 2.9, "other"], + "flag": True, + } + + +def test_prepare_workflow_parameters_when_only_file_provided(empty_directory: str) -> None: + # given + workflows_parameters_path = os.path.join(empty_directory, "config.json") + with open(workflows_parameters_path, "w") as f: + json.dump({"some": "value", "flag": True}, f) + + # when + result = prepare_workflow_parameters( + context=typer.Context(command=MagicMock()), + workflows_parameters_path=workflows_parameters_path, + ) + + # then + assert result == { + "some": "value", + "flag": True, + } + + +def test_prepare_workflow_parameters_when_file_and_args_provided(empty_directory: str) -> None: + # given + workflows_parameters_path = os.path.join(empty_directory, "config.json") + with open(workflows_parameters_path, "w") as f: + json.dump({"some": "value", "flag": True}, f) + context = typer.Context(command=MagicMock()) + context.args = ["--flag", "false"] + + # when + result = prepare_workflow_parameters( + context=context, + workflows_parameters_path=workflows_parameters_path, + ) + + # then + assert result == { + "some": "value", + "flag": False, + }, "Expected explicit arg to override config value and other config values to be remained" From ee5c49482937e0f929f5656294c48682cb3717b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Wed, 27 Nov 2024 14:26:12 +0100 Subject: [PATCH 09/16] Add threshold for erros in API speed benchmark --- .../workflows/load_test_hosted_inference.yml | 17 ++--- inference_cli/benchmark.py | 10 +++ inference_cli/lib/benchmark/api_speed.py | 6 +- inference_cli/lib/benchmark_adapter.py | 26 +++++++ inference_cli/lib/utils.py | 19 +++++ inference_cli/workflows.py | 22 +----- .../unit_tests/lib/test_benchmark_adapter.py | 30 ++++++++ .../unit_tests/lib/test_utils.py | 71 ++++++++++++++++++- .../unit_tests/lib/workflows/__init__.py | 0 .../unit_tests/lib/workflows/test_common.py | 0 10 files changed, 170 insertions(+), 31 deletions(-) create mode 100644 tests/inference_cli/unit_tests/lib/test_benchmark_adapter.py create mode 100644 tests/inference_cli/unit_tests/lib/workflows/__init__.py create mode 100644 tests/inference_cli/unit_tests/lib/workflows/test_common.py diff --git a/.github/workflows/load_test_hosted_inference.yml b/.github/workflows/load_test_hosted_inference.yml index 4aa697501..74d7dc3ed 100644 --- a/.github/workflows/load_test_hosted_inference.yml +++ b/.github/workflows/load_test_hosted_inference.yml @@ -51,35 +51,36 @@ jobs: - name: 🏋️‍♂️ Load test 🚨 PRODUCTION 🚨 | object-detection 🔥🔥🔥🔥 if: ${{ github.event.inputs.environment == 'production' && github.event.inputs.model_type == 'object-detection' }} run: | - ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m inference_cli.main benchmark api-speed -m coco/16 -d coco -rps 5 -br 500 -h https://detect.roboflow.com --yes --output_location test_results.json + ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m inference_cli.main benchmark api-speed -m coco/16 -d coco -rps 5 -br 500 -h https://detect.roboflow.com --yes --output_location test_results.json --max_error_rate 5.0 - name: 🏋️‍♂️ Load test 🚨 PRODUCTION 🚨 | instance-segmentation 🔥🔥🔥🔥 if: ${{ github.event.inputs.environment == 'production' && github.event.inputs.model_type == 'instance-segmentation' }} run: | - ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m inference_cli.main benchmark api-speed -m asl-poly-instance-seg/53 -d coco -rps 5 -br 500 -h https://outline.roboflow.com --yes --output_location test_results.json + ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m inference_cli.main benchmark api-speed -m asl-poly-instance-seg/53 -d coco -rps 5 -br 500 -h https://outline.roboflow.com --yes --output_location test_results.json --max_error_rate 5.0 - name: 🏋️‍♂️ Load test 🚨 PRODUCTION 🚨 | classification 🔥🔥🔥🔥 if: ${{ github.event.inputs.environment == 'production' && github.event.inputs.model_type == 'classification' }} run: | - ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m inference_cli.main benchmark api-speed -m vehicle-classification-eapcd/2 -d coco -rps 5 -br 500 -h https://classify.roboflow.com --yes --output_location test_results.json + ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m inference_cli.main benchmark api-speed -m vehicle-classification-eapcd/2 -d coco -rps 5 -br 500 -h https://classify.roboflow.com --yes --output_location test_results.json --max_error_rate 5.0 - name: 🏋️‍♂️ Load test 🚨 PRODUCTION 🚨 | workflows 🔥🔥🔥🔥 if: ${{ github.event.inputs.environment == 'production' && github.event.inputs.model_type == 'workflows' }} run: | - ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m inference_cli.main benchmark api-speed -wid workflows-production-test -wn paul-guerrie-tang1 -d coco -rps 5 -br 500 -h https://classify.roboflow.com --yes --output_location test_results.json + ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m inference_cli.main benchmark api-speed -wid workflows-production-test -wn paul-guerrie-tang1 -d coco -rps 5 -br 500 -h https://classify.roboflow.com --yes --output_location test_results.json --max_error_rate 5.0 - name: 🏋️‍♂️ Load test 😎 STAGING 😎 | object-detection 🔥🔥🔥🔥 if: ${{ github.event.inputs.environment == 'staging' && github.event.inputs.model_type == 'object-detection' }} run: | - ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_STAGING_API_KEY }} python -m inference_cli.main benchmark api-speed -m eye-detection/35 -d coco -rps 5 -br 500 -h https://lambda-object-detection.staging.roboflow.com --legacy-endpoints --yes --output_location test_results.json + ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_STAGING_API_KEY }} python -m inference_cli.main benchmark api-speed -m eye-detection/35 -d coco -rps 5 -br 500 -h https://lambda-object-detection.staging.roboflow.com --legacy-endpoints --yes --output_location test_results.json --max_error_rate 5.0 - name: 🏋️‍♂️ Load test 😎 STAGING 😎 | instance-segmentation 🔥🔥🔥🔥 if: ${{ github.event.inputs.environment == 'staging' && github.event.inputs.model_type == 'instance-segmentation' }} run: | - ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_STAGING_API_KEY }} python -m inference_cli.main benchmark api-speed -m asl-instance-seg/116 -d coco -rps 5 -br 500 -h https://lambda-instance-segmentation.staging.roboflow.com --legacy-endpoints --yes --output_location test_results.json + ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_STAGING_API_KEY }} python -m inference_cli.main benchmark api-speed -m asl-instance-seg/116 -d coco -rps 5 -br 500 -h https://lambda-instance-segmentation.staging.roboflow.com --legacy-endpoints --yes --output_location test_results.json --max_error_rate 5.0 - name: 🏋️‍♂️ Load test 😎 STAGING 😎 | classification 🔥🔥🔥🔥 if: ${{ github.event.inputs.environment == 'staging' && github.event.inputs.model_type == 'classification' }} run: | - ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_STAGING_API_KEY }} python -m inference_cli.main benchmark api-speed -m catdog/28 -d coco -rps 5 -br 500 -h https://lambda-classification.staging.roboflow.com --legacy-endpoints --yes --output_location test_results.json + ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_STAGING_API_KEY }} python -m inference_cli.main benchmark api-speed -m catdog/28 -d coco -rps 5 -br 500 -h https://lambda-classification.staging.roboflow.com --legacy-endpoints --yes --output_location test_results.json --max_error_rate 5.0 - name: 🏋️‍♂️ Load test 😎 STAGING 😎 | workflows 🔥🔥🔥🔥 if: ${{ github.event.inputs.environment == 'staging' && github.event.inputs.model_type == 'workflows' }} run: | - ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_STAGING_API_KEY }} python -m inference_cli.main benchmark api-speed -wid workflows-staging-test -wn paul-guerrie -d coco -rps 5 -br 500 -h https://lambda-classification.staging.roboflow.com --legacy-endpoints --yes --output_location test_results.json + ROBOFLOW_API_KEY=${{ secrets.LOAD_TEST_STAGING_API_KEY }} python -m inference_cli.main benchmark api-speed -wid workflows-staging-test -wn paul-guerrie -d coco -rps 5 -br 500 -h https://lambda-classification.staging.roboflow.com --legacy-endpoints --yes --output_location test_results.json --max_error_rate 5.0 - name: 📈 RESULTS run: cat test_results.json | jq + if: always() diff --git a/inference_cli/benchmark.py b/inference_cli/benchmark.py index 24a4fbbb3..b43e34eee 100644 --- a/inference_cli/benchmark.py +++ b/inference_cli/benchmark.py @@ -136,6 +136,14 @@ def api_speed( help="Boolean flag to decide on auto `yes` answer given on user input required.", ), ] = False, + max_error_rate: Annotated[ + Optional[float], + typer.Option( + "--max_error_rate", + help="Max error rate for API speed benchmark - if given and the error rate is higher - command will " + "return non-success error code. Expected percentage values in range 0.0-100.0", + ), + ] = None, ): if "roboflow.com" in host and not proceed_automatically: proceed = input( @@ -158,6 +166,7 @@ def api_speed( model_configuration=model_configuration, output_location=output_location, enforce_legacy_endpoints=enforce_legacy_endpoints, + max_error_rate=max_error_rate, ) else: if workflow_specification: @@ -179,6 +188,7 @@ def api_speed( api_key=api_key, model_configuration=model_configuration, output_location=output_location, + max_error_rate=max_error_rate, ) except Exception as error: typer.echo(f"Command failed. Cause: {error}") diff --git a/inference_cli/lib/benchmark/api_speed.py b/inference_cli/lib/benchmark/api_speed.py index df3d5e1f1..ba75cbe54 100644 --- a/inference_cli/lib/benchmark/api_speed.py +++ b/inference_cli/lib/benchmark/api_speed.py @@ -25,7 +25,11 @@ def run_api_warm_up( for _ in tqdm( range(warm_up_requests), desc="Warming up API...", total=warm_up_requests ): - _ = client.infer(inference_input=image) + try: + _ = client.infer(inference_input=image) + except Exception: + # ignoring errors, without slight API instability may terminate benchmark + pass def coordinate_infer_api_speed_benchmark( diff --git a/inference_cli/lib/benchmark_adapter.py b/inference_cli/lib/benchmark_adapter.py index 790b084e4..3bc1d09da 100644 --- a/inference_cli/lib/benchmark_adapter.py +++ b/inference_cli/lib/benchmark_adapter.py @@ -35,6 +35,7 @@ def run_infer_api_speed_benchmark( model_configuration: Optional[str] = None, output_location: Optional[str] = None, enforce_legacy_endpoints: bool = False, + max_error_rate: Optional[float] = None, ) -> None: dataset_images = load_dataset_images( dataset_reference=dataset_reference, @@ -61,6 +62,10 @@ def run_infer_api_speed_benchmark( requests_per_second=requests_per_second, ) if output_location is None: + ensure_error_rate_is_below_threshold( + error_rate=benchmark_results.error_rate, + threshold=max_error_rate, + ) return None benchmark_parameters = { "datetime": datetime.now().isoformat(), @@ -78,6 +83,10 @@ def run_infer_api_speed_benchmark( benchmark_parameters=benchmark_parameters, benchmark_results=benchmark_results, ) + ensure_error_rate_is_below_threshold( + error_rate=benchmark_results.error_rate, + threshold=max_error_rate, + ) def run_workflow_api_speed_benchmark( @@ -95,6 +104,7 @@ def run_workflow_api_speed_benchmark( api_key: Optional[str] = None, model_configuration: Optional[str] = None, output_location: Optional[str] = None, + max_error_rate: Optional[float] = None, ) -> None: dataset_images = load_dataset_images( dataset_reference=dataset_reference, @@ -120,6 +130,10 @@ def run_workflow_api_speed_benchmark( requests_per_second=requests_per_second, ) if output_location is None: + ensure_error_rate_is_below_threshold( + error_rate=benchmark_results.error_rate, + threshold=max_error_rate, + ) return None benchmark_parameters = { "datetime": datetime.now().isoformat(), @@ -141,6 +155,10 @@ def run_workflow_api_speed_benchmark( benchmark_parameters=benchmark_parameters, benchmark_results=benchmark_results, ) + ensure_error_rate_is_below_threshold( + error_rate=benchmark_results.error_rate, + threshold=max_error_rate, + ) def run_python_package_speed_benchmark( @@ -218,3 +236,11 @@ def dump_benchmark_results( "platform": platform_specifics, } dump_json(path=target_path, content=results) + + +def ensure_error_rate_is_below_threshold(error_rate: float, threshold: Optional[float]) -> None: + if threshold is None: + return None + if error_rate <= threshold: + return None + raise RuntimeError(f"Benchmark error rate: {error_rate}% is higher than threshold ({threshold}%)") diff --git a/inference_cli/lib/utils.py b/inference_cli/lib/utils.py index a3da7e133..58aae9ac9 100644 --- a/inference_cli/lib/utils.py +++ b/inference_cli/lib/utils.py @@ -88,3 +88,22 @@ def initialise_client( config = InferenceConfiguration(**raw_configuration) client.configure(inference_configuration=config) return client + + +def ensure_target_directory_is_empty( + output_directory: str, allow_override: bool, only_files: bool = True +) -> None: + if allow_override: + return None + if not os.path.exists(output_directory): + return None + files_in_directory = [ + f + for f in os.listdir(output_directory) + if not only_files or os.path.isfile(os.path.join(output_directory, f)) + ] + if files_in_directory: + raise RuntimeError( + f"Detected content in output directory: {output_directory}. " + f"Command cannot run, as content override is forbidden. Use `--allow_override` to proceed." + ) diff --git a/inference_cli/workflows.py b/inference_cli/workflows.py index e2cce6ad6..3dfe96da2 100644 --- a/inference_cli/workflows.py +++ b/inference_cli/workflows.py @@ -1,10 +1,9 @@ -import os.path from typing import Any, Dict, List, Optional import typer from typing_extensions import Annotated -from inference_cli.lib.utils import read_json +from inference_cli.lib.utils import read_json, ensure_target_directory_is_empty from inference_cli.lib.workflows.core import ( process_image_with_workflow, process_images_directory_with_workflow, @@ -499,25 +498,6 @@ def process_images_directory( raise typer.Exit(code=1) -def ensure_target_directory_is_empty( - output_directory: str, allow_override: bool, only_files: bool = True -) -> None: - if allow_override: - return None - if not os.path.exists(output_directory): - return None - files_in_directory = [ - f - for f in os.listdir(output_directory) - if not only_files or os.path.isfile(os.path.join(output_directory, f)) - ] - if files_in_directory: - raise RuntimeError( - f"Detected content in output directory: {output_directory}. " - f"Command cannot run, as content override is forbidden. Use `--allow_override` to proceed." - ) - - def prepare_workflow_parameters( context: typer.Context, workflows_parameters_path: Optional[str], diff --git a/tests/inference_cli/unit_tests/lib/test_benchmark_adapter.py b/tests/inference_cli/unit_tests/lib/test_benchmark_adapter.py new file mode 100644 index 000000000..58d512e2e --- /dev/null +++ b/tests/inference_cli/unit_tests/lib/test_benchmark_adapter.py @@ -0,0 +1,30 @@ +import pytest + +from inference_cli.lib.benchmark_adapter import ensure_error_rate_is_below_threshold + + +def test_ensure_error_rate_is_below_threshold_when_threshold_not_given() -> None: + # when + ensure_error_rate_is_below_threshold(error_rate=30.0, threshold=None) + + # then - no error + + +def test_ensure_error_rate_is_below_threshold_when_value_below_threshold() -> None: + # when + ensure_error_rate_is_below_threshold(error_rate=30.0, threshold=30.5) + + # then - no error + + +def test_ensure_error_rate_is_below_threshold_when_value_equal_to_threshold() -> None: + # when + ensure_error_rate_is_below_threshold(error_rate=30.5, threshold=30.5) + + # then - no error + + +def test_ensure_error_rate_is_below_threshold_when_value_above_threshold() -> None: + # when + with pytest.raises(RuntimeError): + ensure_error_rate_is_below_threshold(error_rate=30.51, threshold=30.5) diff --git a/tests/inference_cli/unit_tests/lib/test_utils.py b/tests/inference_cli/unit_tests/lib/test_utils.py index 916a0a9c7..9a47bf90a 100644 --- a/tests/inference_cli/unit_tests/lib/test_utils.py +++ b/tests/inference_cli/unit_tests/lib/test_utils.py @@ -1,7 +1,9 @@ import json import os.path -from inference_cli.lib.utils import dump_json, read_env_file, read_file_lines +import pytest + +from inference_cli.lib.utils import dump_json, read_env_file, read_file_lines, ensure_target_directory_is_empty def test_read_file_lines(text_file_path: str) -> None: @@ -39,3 +41,70 @@ def test_dump_json(empty_directory: str) -> None: with open(target_path, "r") as f: result = json.load(f) assert result == {"some": "content"} + + +@pytest.mark.parametrize("allow_override", [True, False]) +@pytest.mark.parametrize("only_files", [True, False]) +def test_ensure_target_directory_is_empty_when_empty_directory_given( + empty_directory: str, + allow_override: bool, + only_files: bool, +) -> None: + # when + ensure_target_directory_is_empty( + output_directory=empty_directory, + allow_override=allow_override, + only_files=only_files, + ) + + # then - no errors + + +def test_ensure_target_directory_is_empty_when_directory_with_sub_dir_provided_but_only_files_matter( + empty_directory: str, +) -> None: + # given + sub_dir_path = os.path.join(empty_directory, "sub_dir") + os.makedirs(sub_dir_path, exist_ok=True) + + # when + ensure_target_directory_is_empty( + output_directory=empty_directory, + allow_override=False, + only_files=True, + ) + + # then - no errors + + +def test_ensure_target_directory_is_empty_when_directory_with_sub_dir_provided_but_not_only_files_matter( + empty_directory: str, +) -> None: + # given + sub_dir_path = os.path.join(empty_directory, "sub_dir") + os.makedirs(sub_dir_path, exist_ok=True) + + # when + with pytest.raises(RuntimeError): + ensure_target_directory_is_empty( + output_directory=empty_directory, + allow_override=False, + only_files=False, + ) + + +def test_ensure_target_directory_is_empty_when_directory_with_sub_dir_provided_but_not_only_files_matter_and_override_allowed( + empty_directory: str, +) -> None: + # given + sub_dir_path = os.path.join(empty_directory, "sub_dir") + os.makedirs(sub_dir_path, exist_ok=True) + + # when + ensure_target_directory_is_empty( + output_directory=empty_directory, + allow_override=True, + only_files=False, + ) + + # then - no errors diff --git a/tests/inference_cli/unit_tests/lib/workflows/__init__.py b/tests/inference_cli/unit_tests/lib/workflows/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/inference_cli/unit_tests/lib/workflows/test_common.py b/tests/inference_cli/unit_tests/lib/workflows/test_common.py new file mode 100644 index 000000000..e69de29bb From b284abb231d35bf2a93c27106cb0d3778ac2273e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Wed, 27 Nov 2024 15:34:30 +0100 Subject: [PATCH 10/16] Add tests for part of common utils for workflows command --- inference_cli/benchmark.py | 2 +- inference_cli/lib/benchmark_adapter.py | 8 +- inference_cli/lib/workflows/common.py | 15 +- inference_cli/workflows.py | 2 +- .../unit_tests/lib/test_utils.py | 7 +- .../unit_tests/lib/workflows/test_common.py | 249 ++++++++++++++++++ 6 files changed, 275 insertions(+), 8 deletions(-) diff --git a/inference_cli/benchmark.py b/inference_cli/benchmark.py index b43e34eee..6ad4ff842 100644 --- a/inference_cli/benchmark.py +++ b/inference_cli/benchmark.py @@ -141,7 +141,7 @@ def api_speed( typer.Option( "--max_error_rate", help="Max error rate for API speed benchmark - if given and the error rate is higher - command will " - "return non-success error code. Expected percentage values in range 0.0-100.0", + "return non-success error code. Expected percentage values in range 0.0-100.0", ), ] = None, ): diff --git a/inference_cli/lib/benchmark_adapter.py b/inference_cli/lib/benchmark_adapter.py index 3bc1d09da..e259946a9 100644 --- a/inference_cli/lib/benchmark_adapter.py +++ b/inference_cli/lib/benchmark_adapter.py @@ -238,9 +238,13 @@ def dump_benchmark_results( dump_json(path=target_path, content=results) -def ensure_error_rate_is_below_threshold(error_rate: float, threshold: Optional[float]) -> None: +def ensure_error_rate_is_below_threshold( + error_rate: float, threshold: Optional[float] +) -> None: if threshold is None: return None if error_rate <= threshold: return None - raise RuntimeError(f"Benchmark error rate: {error_rate}% is higher than threshold ({threshold}%)") + raise RuntimeError( + f"Benchmark error rate: {error_rate}% is higher than threshold ({threshold}%)" + ) diff --git a/inference_cli/lib/workflows/common.py b/inference_cli/lib/workflows/common.py index 27991cb8d..400eb1374 100644 --- a/inference_cli/lib/workflows/common.py +++ b/inference_cli/lib/workflows/common.py @@ -19,15 +19,22 @@ IMAGES_EXTENSIONS = [ "bmp", + "BMP", "dib", + "DIB", "jpeg", + "JPEG", "jpg", + "JPG", "jpe", + "JPE", "jp2", + "JP2", "png", + "PNG", "webp", + "WEBP", ] -IMAGES_EXTENSIONS += [e.upper() for e in IMAGES_EXTENSIONS] def open_progress_log(output_directory: str) -> Tuple[TextIO, Set[str]]: @@ -190,10 +197,10 @@ def report_failed_files( os.makedirs(output_directory, exist_ok=True) timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f") failed_files_path = os.path.abspath( - os.path.join(output_directory, f"failed_files_processing_{timestamp}.json") + os.path.join(output_directory, f"failed_files_processing_{timestamp}.jsonl") ) content = [{"file_path": e[0], "cause": e[1]} for e in failed_files] - dump_json(path=failed_files_path, content=content) + dump_jsonl(path=failed_files_path, content=content) print( f"Detected {len(failed_files)} processing failures. Details saved under: {failed_files_path}" ) @@ -239,6 +246,8 @@ def aggregate_batch_processing_results( def dump_objects_to_json(value: Any) -> Any: + if isinstance(value, set): + value = list(value) if isinstance(value, list) or isinstance(value, dict) or isinstance(value, set): return json.dumps(value) return value diff --git a/inference_cli/workflows.py b/inference_cli/workflows.py index 3dfe96da2..ea88b92ca 100644 --- a/inference_cli/workflows.py +++ b/inference_cli/workflows.py @@ -3,7 +3,7 @@ import typer from typing_extensions import Annotated -from inference_cli.lib.utils import read_json, ensure_target_directory_is_empty +from inference_cli.lib.utils import ensure_target_directory_is_empty, read_json from inference_cli.lib.workflows.core import ( process_image_with_workflow, process_images_directory_with_workflow, diff --git a/tests/inference_cli/unit_tests/lib/test_utils.py b/tests/inference_cli/unit_tests/lib/test_utils.py index 9a47bf90a..078d99715 100644 --- a/tests/inference_cli/unit_tests/lib/test_utils.py +++ b/tests/inference_cli/unit_tests/lib/test_utils.py @@ -3,7 +3,12 @@ import pytest -from inference_cli.lib.utils import dump_json, read_env_file, read_file_lines, ensure_target_directory_is_empty +from inference_cli.lib.utils import ( + dump_json, + ensure_target_directory_is_empty, + read_env_file, + read_file_lines, +) def test_read_file_lines(text_file_path: str) -> None: diff --git a/tests/inference_cli/unit_tests/lib/workflows/test_common.py b/tests/inference_cli/unit_tests/lib/workflows/test_common.py index e69de29bb..f35749587 100644 --- a/tests/inference_cli/unit_tests/lib/workflows/test_common.py +++ b/tests/inference_cli/unit_tests/lib/workflows/test_common.py @@ -0,0 +1,249 @@ +import json +import os.path +from pathlib import Path +from typing import Any, Optional + +import pandas as pd +import pytest +from pandas.errors import EmptyDataError + +from inference_cli.lib.workflows.common import ( + IMAGES_EXTENSIONS, + aggregate_batch_processing_results, + denote_image_processed, + dump_objects_to_json, + get_all_images_in_directory, + open_progress_log, + report_failed_files, +) +from inference_cli.lib.workflows.entities import OutputFileType + + +@pytest.mark.parametrize("value", [3, 3.5, "some", True]) +def test_dump_objects_to_json_when_primitive_type_given(value: Any) -> None: + # when + result = dump_objects_to_json(value=value) + + # then + assert result == value + + +def test_dump_objects_to_json_when_list_given() -> None: + # when + result = dump_objects_to_json(value=[1, 2, 3]) + + # then + assert json.loads(result) == [1, 2, 3] + + +def test_dump_objects_to_json_when_set_given() -> None: + # when + result = dump_objects_to_json(value={1, 2, 3}) + + # then + assert set(json.loads(result)) == {1, 2, 3} + + +def test_dump_objects_to_json_when_dict_given() -> None: + # when + result = dump_objects_to_json(value={"some": "value", "other": [1, 2, 3]}) + + # then + assert json.loads(result) == {"some": "value", "other": [1, 2, 3]} + + +def test_aggregate_batch_processing_results_when_json_output_is_expected_and_results_present( + empty_directory: str, +) -> None: + # given + _prepare_dummy_results(root_dir=empty_directory, sub_dir_name="some.jpg") + _prepare_dummy_results(root_dir=empty_directory, sub_dir_name="other.jpg") + + # when + file_descriptor, _ = open_progress_log(output_directory=empty_directory) + denote_image_processed(log_file=file_descriptor, image_path="/my/path/some.jpg") + denote_image_processed(log_file=file_descriptor, image_path="/my/path/other.jpg") + file_descriptor.close() + aggregate_batch_processing_results( + output_directory=empty_directory, + aggregation_format=OutputFileType.JSONL, + ) + + # then + expected_output_path = os.path.join(empty_directory, "aggregated_results.jsonl") + decoded_results = [] + with open(expected_output_path, "r") as f: + for line in f.readlines(): + if len(line.strip()) == 0: + continue + decoded_results.append(json.loads(line)) + assert decoded_results == [ + {"some": "value", "other": 3.0, "list": [1, 2, 3], "object": {"nested": "value"}} + ] * 2 + + +def test_aggregate_batch_processing_results_when_json_output_is_expected_and_results_not_present( + empty_directory: str, +) -> None: + # when + aggregate_batch_processing_results( + output_directory=empty_directory, + aggregation_format=OutputFileType.JSONL, + ) + + # then + expected_output_path = os.path.join(empty_directory, "aggregated_results.jsonl") + decoded_results = [] + with open(expected_output_path, "r") as f: + for line in f.readlines(): + if len(line.strip()) == 0: + continue + decoded_results.append(json.loads(line)) + assert decoded_results == [] + + +def test_aggregate_batch_processing_results_when_csv_output_is_expected_and_results_present( + empty_directory: str, +) -> None: + # given + _prepare_dummy_results(root_dir=empty_directory, sub_dir_name="some.jpg") + _prepare_dummy_results(root_dir=empty_directory, sub_dir_name="other.jpg") + + # when + file_descriptor, _ = open_progress_log(output_directory=empty_directory) + denote_image_processed(log_file=file_descriptor, image_path="/my/path/some.jpg") + denote_image_processed(log_file=file_descriptor, image_path="/my/path/other.jpg") + file_descriptor.close() + aggregate_batch_processing_results( + output_directory=empty_directory, + aggregation_format=OutputFileType.CSV, + ) + + # then + expected_output_path = os.path.join(empty_directory, "aggregated_results.csv") + df = pd.read_csv(expected_output_path) + assert len(df) == 2, "Expected 2 records" + assert df.iloc[0].some == "value" + assert df.iloc[0].other == 3.0 + assert json.loads(df.iloc[0].list) == [1, 2, 3] + assert json.loads(df.iloc[0].object) == {"nested": "value"} + assert df.iloc[1].some == "value" + assert df.iloc[1].other == 3.0 + assert json.loads(df.iloc[1].list) == [1, 2, 3] + assert json.loads(df.iloc[1].object) == {"nested": "value"} + + +def test_aggregate_batch_processing_results_when_csv_output_is_expected_and_results_present_but_with_inconsistent_schema( + empty_directory: str, +) -> None: + # given + _prepare_dummy_results(root_dir=empty_directory, sub_dir_name="some.jpg") + _prepare_dummy_results(root_dir=empty_directory, sub_dir_name="other.jpg", extra_data={"extra": "column"}) + + # when + file_descriptor, _ = open_progress_log(output_directory=empty_directory) + denote_image_processed(log_file=file_descriptor, image_path="/my/path/some.jpg") + denote_image_processed(log_file=file_descriptor, image_path="/my/path/other.jpg") + file_descriptor.close() + aggregate_batch_processing_results( + output_directory=empty_directory, + aggregation_format=OutputFileType.CSV, + ) + + # then + expected_output_path = os.path.join(empty_directory, "aggregated_results.csv") + df = pd.read_csv(expected_output_path) + assert len(df) == 2, "Expected 2 records" + assert df.iloc[0].some == "value" + assert df.iloc[0].other == 3.0 + assert json.loads(df.iloc[0].list) == [1, 2, 3] + assert json.loads(df.iloc[0].object) == {"nested": "value"} + assert df.iloc[1].some == "value" + assert df.iloc[1].other == 3.0 + assert json.loads(df.iloc[1].list) == [1, 2, 3] + assert json.loads(df.iloc[1].object) == {"nested": "value"} + assert df.iloc[1].extra == "column" or df.iloc[0].extra == "column" and df.iloc[1].extra != df.iloc[0].extra, \ + "Expected one record to have value and other to have none in extra column" + + +def test_aggregate_batch_processing_results_when_csv_output_is_expected_and_results_not_present( + empty_directory: str, +) -> None: + # when + aggregate_batch_processing_results( + output_directory=empty_directory, + aggregation_format=OutputFileType.CSV, + ) + + # then + expected_output_path = os.path.join(empty_directory, "aggregated_results.csv") + with pytest.raises(EmptyDataError): + _ = pd.read_csv(expected_output_path) + + +def _prepare_dummy_results( + root_dir: str, + sub_dir_name: str, + extra_data: Optional[dict] = None, +) -> None: + if extra_data is None: + extra_data = {} + sub_dir_path = os.path.join(root_dir, sub_dir_name) + os.makedirs(sub_dir_path, exist_ok=True) + results = {"some": "value", "other": 3.0, "list": [1, 2, 3], "object": {"nested": "value"}} + results.update(extra_data) + results_path = os.path.join(sub_dir_path, "results.json") + with open(results_path, "w") as f: + json.dump(results, f) + + +def test_report_failed_files_when_no_errors_detected(empty_directory: str) -> None: + # when + report_failed_files(failed_files=[], output_directory=empty_directory) + + # then + assert len(os.listdir(empty_directory)) == 0 + + +def test_report_failed_files_when_errors_detected(empty_directory: str) -> None: + # when + report_failed_files( + failed_files=[ + ("some.jpg", "some"), + ("other.jpg", "other") + ], + output_directory=empty_directory, + ) + + # then + dir_content = os.listdir(empty_directory) + assert len(dir_content) == 1 + file_path = os.path.join(empty_directory, dir_content[0]) + with open(file_path, "r") as f: + decoded_file = [ + json.loads(line) for line in f.readlines() + if len(line.strip()) > 0 + ] + assert decoded_file == [ + {"file_path": "some.jpg", "cause": "some"}, + {"file_path": "other.jpg", "cause": "other"}, + ] + + +def test_get_all_images_in_directory(empty_directory: str) -> None: + # given + for extension in IMAGES_EXTENSIONS: + _create_empty_file(directory=empty_directory, file_name=f"image.{extension}") + expected_files = len(IMAGES_EXTENSIONS) // 2 # dividing by two, as we have each extension lower- and upper- case + + # when + result = get_all_images_in_directory(input_directory=empty_directory) + + # then + assert len(result) == expected_files + + +def _create_empty_file(directory: str, file_name: str) -> None: + file_path = os.path.join(directory, file_name) + path = Path(file_path) + path.touch(exist_ok=True) From e26727a77e9e9ac906ecfd670688d2bdb43f9c28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Wed, 27 Nov 2024 19:18:22 +0100 Subject: [PATCH 11/16] Add tests for common utils --- inference_cli/lib/workflows/common.py | 66 +++-- .../unit_tests/lib/workflows/test_common.py | 264 +++++++++++++++++- .../unit_tests/test_workflows.py | 28 +- 3 files changed, 306 insertions(+), 52 deletions(-) diff --git a/inference_cli/lib/workflows/common.py b/inference_cli/lib/workflows/common.py index 400eb1374..9c52381ca 100644 --- a/inference_cli/lib/workflows/common.py +++ b/inference_cli/lib/workflows/common.py @@ -2,6 +2,7 @@ import os.path import re from datetime import datetime +from functools import lru_cache from threading import Lock from typing import Any, Dict, List, Optional, Set, TextIO, Tuple @@ -17,6 +18,11 @@ BASE64_DATA_TYPE_PATTERN = re.compile(r"^data:image\/[a-z]+;base64,") +TYPE_KEY = "type" +BASE_64_TYPE = "base64" +VALUE_KEY = "value" +DEDUCTED_IMAGE = "" + IMAGES_EXTENSIONS = [ "bmp", "BMP", @@ -115,12 +121,12 @@ def deduct_images(result: Any) -> Any: return {deduct_images(result=e) for e in result} if ( isinstance(result, dict) - and result.get("type") == "base64" - and "value" in result + and result.get(TYPE_KEY) == BASE_64_TYPE + and VALUE_KEY in result ): - return "" + return DEDUCTED_IMAGE if isinstance(result, np.ndarray): - return "" + return DEDUCTED_IMAGE if isinstance(result, dict): return {k: deduct_images(result=v) for k, v in result.items()} return result @@ -131,10 +137,10 @@ def extract_images_from_result( ) -> List[Tuple[str, np.ndarray]]: if ( isinstance(result, dict) - and result.get("type") == "base64" - and "value" in result + and result.get(TYPE_KEY) == BASE_64_TYPE + and VALUE_KEY in result ): - loaded_image = decode_base64_image(result["value"]) + loaded_image = decode_base64_image(result[VALUE_KEY]) return [(key_prefix, loaded_image)] if isinstance(result, np.ndarray): return [(key_prefix, result)] @@ -167,26 +173,32 @@ def decode_base64_image(payload: str) -> np.ndarray: def get_all_images_in_directory(input_directory: str) -> List[str]: - if os.name == "nt": - # Windows paths are case-insensitive, hence deduplication - # is needed, as IMAGES_EXTENSIONS contains extensions - # in lower- and upper- cases version. - return list( - { - path.as_posix().lower() - for path in sv.list_files_with_extensions( - directory=input_directory, - extensions=IMAGES_EXTENSIONS, - ) - } - ) - return [ - path.as_posix() - for path in sv.list_files_with_extensions( - directory=input_directory, - extensions=IMAGES_EXTENSIONS, - ) - ] + file_system_is_case_sensitive = _is_file_system_case_sensitive() + if file_system_is_case_sensitive: + return [ + path.as_posix() + for path in sv.list_files_with_extensions( + directory=input_directory, + extensions=IMAGES_EXTENSIONS, + ) + ] + return list( + { + path.as_posix().lower() + for path in sv.list_files_with_extensions( + directory=input_directory, + extensions=IMAGES_EXTENSIONS, + ) + } + ) + + +@lru_cache() +def _is_file_system_case_sensitive() -> bool: + fs_is_case_insensitive = os.path.exists(__file__.upper()) and os.path.exists( + __file__.lower() + ) + return not fs_is_case_insensitive def report_failed_files( diff --git a/tests/inference_cli/unit_tests/lib/workflows/test_common.py b/tests/inference_cli/unit_tests/lib/workflows/test_common.py index f35749587..0af36d179 100644 --- a/tests/inference_cli/unit_tests/lib/workflows/test_common.py +++ b/tests/inference_cli/unit_tests/lib/workflows/test_common.py @@ -1,8 +1,11 @@ +import base64 import json import os.path from pathlib import Path from typing import Any, Optional +import cv2 +import numpy as np import pandas as pd import pytest from pandas.errors import EmptyDataError @@ -10,8 +13,13 @@ from inference_cli.lib.workflows.common import ( IMAGES_EXTENSIONS, aggregate_batch_processing_results, + decode_base64_image, + deduct_images, denote_image_processed, + dump_image_processing_results, + dump_images_outputs, dump_objects_to_json, + extract_images_from_result, get_all_images_in_directory, open_progress_log, report_failed_files, @@ -77,9 +85,18 @@ def test_aggregate_batch_processing_results_when_json_output_is_expected_and_res if len(line.strip()) == 0: continue decoded_results.append(json.loads(line)) - assert decoded_results == [ - {"some": "value", "other": 3.0, "list": [1, 2, 3], "object": {"nested": "value"}} - ] * 2 + assert ( + decoded_results + == [ + { + "some": "value", + "other": 3.0, + "list": [1, 2, 3], + "object": {"nested": "value"}, + } + ] + * 2 + ) def test_aggregate_batch_processing_results_when_json_output_is_expected_and_results_not_present( @@ -138,7 +155,11 @@ def test_aggregate_batch_processing_results_when_csv_output_is_expected_and_resu ) -> None: # given _prepare_dummy_results(root_dir=empty_directory, sub_dir_name="some.jpg") - _prepare_dummy_results(root_dir=empty_directory, sub_dir_name="other.jpg", extra_data={"extra": "column"}) + _prepare_dummy_results( + root_dir=empty_directory, + sub_dir_name="other.jpg", + extra_data={"extra": "column"}, + ) # when file_descriptor, _ = open_progress_log(output_directory=empty_directory) @@ -162,8 +183,11 @@ def test_aggregate_batch_processing_results_when_csv_output_is_expected_and_resu assert df.iloc[1].other == 3.0 assert json.loads(df.iloc[1].list) == [1, 2, 3] assert json.loads(df.iloc[1].object) == {"nested": "value"} - assert df.iloc[1].extra == "column" or df.iloc[0].extra == "column" and df.iloc[1].extra != df.iloc[0].extra, \ - "Expected one record to have value and other to have none in extra column" + assert ( + df.iloc[1].extra == "column" + or df.iloc[0].extra == "column" + and df.iloc[1].extra != df.iloc[0].extra + ), "Expected one record to have value and other to have none in extra column" def test_aggregate_batch_processing_results_when_csv_output_is_expected_and_results_not_present( @@ -190,7 +214,12 @@ def _prepare_dummy_results( extra_data = {} sub_dir_path = os.path.join(root_dir, sub_dir_name) os.makedirs(sub_dir_path, exist_ok=True) - results = {"some": "value", "other": 3.0, "list": [1, 2, 3], "object": {"nested": "value"}} + results = { + "some": "value", + "other": 3.0, + "list": [1, 2, 3], + "object": {"nested": "value"}, + } results.update(extra_data) results_path = os.path.join(sub_dir_path, "results.json") with open(results_path, "w") as f: @@ -208,10 +237,7 @@ def test_report_failed_files_when_no_errors_detected(empty_directory: str) -> No def test_report_failed_files_when_errors_detected(empty_directory: str) -> None: # when report_failed_files( - failed_files=[ - ("some.jpg", "some"), - ("other.jpg", "other") - ], + failed_files=[("some.jpg", "some"), ("other.jpg", "other")], output_directory=empty_directory, ) @@ -221,8 +247,7 @@ def test_report_failed_files_when_errors_detected(empty_directory: str) -> None: file_path = os.path.join(empty_directory, dir_content[0]) with open(file_path, "r") as f: decoded_file = [ - json.loads(line) for line in f.readlines() - if len(line.strip()) > 0 + json.loads(line) for line in f.readlines() if len(line.strip()) > 0 ] assert decoded_file == [ {"file_path": "some.jpg", "cause": "some"}, @@ -234,7 +259,9 @@ def test_get_all_images_in_directory(empty_directory: str) -> None: # given for extension in IMAGES_EXTENSIONS: _create_empty_file(directory=empty_directory, file_name=f"image.{extension}") - expected_files = len(IMAGES_EXTENSIONS) // 2 # dividing by two, as we have each extension lower- and upper- case + _create_empty_file(directory=empty_directory, file_name=f".tmp") + _create_empty_file(directory=empty_directory, file_name=f".bin") + expected_files = len(os.listdir(empty_directory)) - 2 # when result = get_all_images_in_directory(input_directory=empty_directory) @@ -247,3 +274,212 @@ def _create_empty_file(directory: str, file_name: str) -> None: file_path = os.path.join(directory, file_name) path = Path(file_path) path.touch(exist_ok=True) + + +def test_decode_base64_image_when_base64_header_present() -> None: + # given + image = np.zeros((192, 168, 3), dtype=np.uint8) + encoded = _encode_image_to_base64(image=image) + encoded = f"data:image/jpeg;base64,{encoded}" + + # when + result = decode_base64_image(payload=encoded) + + # then + assert np.allclose(result, image) + + +def test_decode_base64_image_when_base64_header_not_present() -> None: + # given + image = np.zeros((192, 168, 3), dtype=np.uint8) + encoded = _encode_image_to_base64(image=image) + + # when + result = decode_base64_image(payload=encoded) + + # then + assert np.allclose(result, image) + + +def test_extract_images_from_result() -> None: + # given + result = { + "some": "value", + "other": { + "type": "base64", + "value": _encode_image_to_base64(np.zeros((192, 168, 3), dtype=np.uint8)), + }, + "dict": { + "a": 1, + "b": 2, + "c": [np.zeros((192, 192, 3), dtype=np.uint8), 1, "some"], + }, + "list": [["a", "b"], ["c", np.zeros((168, 168, 3), dtype=np.uint8)]], + } + + # when + result = extract_images_from_result(result=result) + + # then + assert len(result) == 3, "Expected three images returned" + key_to_image = {key: image for key, image in result} + assert key_to_image["other"].shape == (192, 168, 3) + assert key_to_image["dict/c/0"].shape == (192, 192, 3) + assert key_to_image["list/1/1"].shape == (168, 168, 3) + + +def test_deduct_images() -> None: + # given + result = { + "some": "value", + "other": { + "type": "base64", + "value": _encode_image_to_base64(np.zeros((192, 168, 3), dtype=np.uint8)), + }, + "dict": { + "a": 1, + "b": 2, + "c": [np.zeros((192, 192, 3), dtype=np.uint8), 1, "some"], + }, + "list": [["a", "b"], ["c", np.zeros((168, 168, 3), dtype=np.uint8)]], + } + + # when + result = deduct_images(result=result) + + # then + assert result == { + "some": "value", + "other": "", + "dict": {"a": 1, "b": 2, "c": ["", 1, "some"]}, + "list": [["a", "b"], ["c", ""]], + } + + +def test_dump_images_outputs(empty_directory: str) -> None: + # given + images_in_result = [ + ("visualization", np.zeros((168, 168, 3), dtype=np.uint8)), + ("some/crops/1", np.zeros((192, 192, 3), dtype=np.uint8)), + ] + + # when + dump_images_outputs( + image_results_dir=empty_directory, + images_in_result=images_in_result, + ) + + # then + visualization_image = cv2.imread(os.path.join(empty_directory, "visualization.jpg")) + assert visualization_image.shape == (168, 168, 3) + crop_image = cv2.imread(os.path.join(empty_directory, "some/crops/1.jpg")) + assert crop_image.shape == (192, 192, 3) + + +def test_dump_image_processing_results_when_images_are_to_be_saved( + empty_directory: str, +) -> None: + # given + result = { + "some": "value", + "other": { + "type": "base64", + "value": _encode_image_to_base64(np.zeros((192, 168, 3), dtype=np.uint8)), + }, + "dict": { + "a": 1, + "b": 2, + "c": [np.zeros((192, 192, 3), dtype=np.uint8), 1, "some"], + }, + "list": [["a", "b"], ["c", np.zeros((168, 168, 3), dtype=np.uint8)]], + } + + # when + dump_image_processing_results( + result=result, + image_path="/some/directory/my_image.jpeg", + output_directory=empty_directory, + save_image_outputs=True, + ) + + # then + assert os.path.isdir(os.path.join(empty_directory, "my_image.jpeg")) + structured_results_path = os.path.join( + empty_directory, "my_image.jpeg", "results.json" + ) + with open(structured_results_path) as f: + structured_results = json.load(f) + assert structured_results == { + "some": "value", + "other": "", + "dict": {"a": 1, "b": 2, "c": ["", 1, "some"]}, + "list": [["a", "b"], ["c", ""]], + } + other_image = cv2.imread( + os.path.join(empty_directory, "my_image.jpeg", "other.jpg") + ) + assert other_image.shape == (192, 168, 3) + dict_nested_image = cv2.imread( + os.path.join(empty_directory, "my_image.jpeg", "dict", "c", "0.jpg") + ) + assert dict_nested_image.shape == (192, 192, 3) + list_nested_image = cv2.imread( + os.path.join(empty_directory, "my_image.jpeg", "list", "1", "1.jpg") + ) + assert list_nested_image.shape == (168, 168, 3) + + +def test_dump_image_processing_results_when_images_not_to_be_saved( + empty_directory: str, +) -> None: + # given + result = { + "some": "value", + "other": { + "type": "base64", + "value": _encode_image_to_base64(np.zeros((192, 168, 3), dtype=np.uint8)), + }, + "dict": { + "a": 1, + "b": 2, + "c": [np.zeros((192, 192, 3), dtype=np.uint8), 1, "some"], + }, + "list": [["a", "b"], ["c", np.zeros((168, 168, 3), dtype=np.uint8)]], + } + + # when + dump_image_processing_results( + result=result, + image_path="/some/directory/my_image.jpeg", + output_directory=empty_directory, + save_image_outputs=False, + ) + + # then + assert os.path.isdir(os.path.join(empty_directory, "my_image.jpeg")) + structured_results_path = os.path.join( + empty_directory, "my_image.jpeg", "results.json" + ) + with open(structured_results_path) as f: + structured_results = json.load(f) + assert structured_results == { + "some": "value", + "other": "", + "dict": {"a": 1, "b": 2, "c": ["", 1, "some"]}, + "list": [["a", "b"], ["c", ""]], + } + assert not os.path.exists( + os.path.join(empty_directory, "my_image.jpeg", "other.jpg") + ) + assert not os.path.exists( + os.path.join(empty_directory, "my_image.jpeg", "dict", "c", "0.jpg") + ) + assert not os.path.exists( + os.path.join(empty_directory, "my_image.jpeg", "list", "1", "1.jpg") + ) + + +def _encode_image_to_base64(image: np.ndarray) -> str: + _, img_encoded = cv2.imencode(".jpg", image) + image_bytes = np.array(img_encoded).tobytes() + return base64.b64encode(image_bytes).decode("utf-8") diff --git a/tests/inference_cli/unit_tests/test_workflows.py b/tests/inference_cli/unit_tests/test_workflows.py index 225d57254..8174de8f3 100644 --- a/tests/inference_cli/unit_tests/test_workflows.py +++ b/tests/inference_cli/unit_tests/test_workflows.py @@ -9,10 +9,10 @@ from inference_cli.workflows import prepare_workflow_parameters -test_app = typer.Typer(help="This is test app to verify kwargs parsing") +dummy_app = typer.Typer(help="This is test app to verify kwargs parsing") -@test_app.command( +@dummy_app.command( context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, ) def verify_workflow_parameters_parsing( @@ -92,7 +92,7 @@ def test_command_parsing_workflows_parameters_when_no_additional_params_passed_a # when result = runner.invoke( - test_app, + dummy_app, "verify_workflow_parameters_parsing " "--string_param value " "--int_param 2137 " @@ -125,7 +125,7 @@ def test_command_parsing_workflows_parameters_when_additional_params_passed_and_ # when result = runner.invoke( - test_app, + dummy_app, "verify_workflow_parameters_parsing " "--string_param value " "--int_param 2137 " @@ -169,7 +169,7 @@ def test_command_parsing_workflows_parameters_when_no_additional_params_passed_a # when result = runner.invoke( - test_app, + dummy_app, "verify_workflow_parameters_parsing " "-sp value " "-ip 2137 " @@ -203,7 +203,7 @@ def test_command_parsing_workflows_parameters_when_additional_params_passed_and_ # when result = runner.invoke( - test_app, + dummy_app, "verify_workflow_parameters_parsing " "-sp value " "-ip 2137 " @@ -247,7 +247,7 @@ def test_command_parsing_workflows_parameters_when_no_additional_params_passed_a # when result = runner.invoke( - test_app, + dummy_app, "verify_workflow_parameters_parsing " "--string_param value " "-ip 2137 " @@ -281,7 +281,7 @@ def test_command_parsing_workflows_parameters_when_additional_params_passed_and_ # when result = runner.invoke( - test_app, + dummy_app, "verify_workflow_parameters_parsing " "--string_param value " "-ip 2137 " @@ -317,7 +317,9 @@ def test_command_parsing_workflows_parameters_when_additional_params_passed_and_ } -def test_prepare_workflow_parameters_when_neither_file_nor_additional_args_are_used() -> None: +def test_prepare_workflow_parameters_when_neither_file_nor_additional_args_are_used() -> ( + None +): # when result = prepare_workflow_parameters( context=typer.Context(command=MagicMock()), @@ -347,7 +349,9 @@ def test_prepare_workflow_parameters_when_only_args_are_provided() -> None: } -def test_prepare_workflow_parameters_when_only_file_provided(empty_directory: str) -> None: +def test_prepare_workflow_parameters_when_only_file_provided( + empty_directory: str, +) -> None: # given workflows_parameters_path = os.path.join(empty_directory, "config.json") with open(workflows_parameters_path, "w") as f: @@ -366,7 +370,9 @@ def test_prepare_workflow_parameters_when_only_file_provided(empty_directory: st } -def test_prepare_workflow_parameters_when_file_and_args_provided(empty_directory: str) -> None: +def test_prepare_workflow_parameters_when_file_and_args_provided( + empty_directory: str, +) -> None: # given workflows_parameters_path = os.path.join(empty_directory, "config.json") with open(workflows_parameters_path, "w") as f: From f669696a031f6b8844208e8e1021311c2f70896b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Wed, 27 Nov 2024 21:11:10 +0100 Subject: [PATCH 12/16] Add new tests suiteS --- ...ference_cli_depending_on_inference_x86.yml | 47 ++ .../integration_tests_inference_cli_x86.yml | 2 +- .gitignore | 2 +- inference_cli/lib/utils.py | 7 + inference_cli/lib/workflows/video_adapter.py | 13 +- .../integration_tests/conftest.py | 66 +++ .../integration_tests/test_workflows.py | 485 ++++++++++++++++++ 7 files changed, 610 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/integration_tests_inference_cli_depending_on_inference_x86.yml create mode 100644 tests/inference_cli/integration_tests/test_workflows.py diff --git a/.github/workflows/integration_tests_inference_cli_depending_on_inference_x86.yml b/.github/workflows/integration_tests_inference_cli_depending_on_inference_x86.yml new file mode 100644 index 000000000..ec0e1381f --- /dev/null +++ b/.github/workflows/integration_tests_inference_cli_depending_on_inference_x86.yml @@ -0,0 +1,47 @@ +name: INTEGRATION TESTS - inference CLI + inference CORE + +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + +jobs: + call_is_mergeable: + uses: ./.github/workflows/is_mergeable.yml + secrets: inherit + build-dev-test: + needs: call_is_mergeable + if: ${{ github.event_name != 'pull_request' || needs.call_is_mergeable.outputs.mergeable_state != 'not_clean' }} + runs-on: + labels: depot-ubuntu-22.04-small + group: public-depot + timeout-minutes: 30 + strategy: + matrix: + python-version: ["3.9", "3.10"] + steps: + - name: 🛎️ Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + - name: 🐍 Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + check-latest: true + - name: 📦 Cache Python packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('requirements/**') }} + restore-keys: | + ${{ runner.os }}-pip-${{ matrix.python-version }}- + - name: 📦 Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade setuptools + pip install --extra-index-url https://download.pytorch.org/whl/cpu -r requirements/_requirements.txt -r requirements/requirements.cpu.txt -r requirements/requirements.sdk.http.txt -r requirements/requirements.test.unit.txt -r requirements/requirements.http.txt -r requirements/requirements.yolo_world.txt -r requirements/requirements.doctr.txt -r requirements/requirements.sam.txt -r requirements/requirements.transformers.txt -r requirements/requirements.cli.txt -r requirements/requirements.sdk.http.txt + - name: 🧪 Integration Tests of Inference CLI + run: RUN_TESTS_WITH_INFERENCE_PACKAGE=True INFERENCE_CLI_TESTS_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m pytest tests/inference_cli/integration_tests/test_workflows.py diff --git a/.github/workflows/integration_tests_inference_cli_x86.yml b/.github/workflows/integration_tests_inference_cli_x86.yml index b2527851c..ff0929406 100644 --- a/.github/workflows/integration_tests_inference_cli_x86.yml +++ b/.github/workflows/integration_tests_inference_cli_x86.yml @@ -44,4 +44,4 @@ jobs: pip install --upgrade setuptools pip install -r requirements/requirements.cli.txt -r requirements/requirements.sdk.http.txt -r requirements/requirements.test.unit.txt - name: 🧪 Integration Tests of Inference CLI - run: python -m pytest tests/inference_cli/integration_tests + run: RUN_TESTS_EXPECTING_ERROR_WHEN_INFERENCE_NOT_INSTALLED=True INFERENCE_CLI_TESTS_API_KEY=${{ secrets.LOAD_TEST_PRODUCTION_API_KEY }} python -m pytest tests/inference_cli/integration_tests diff --git a/.gitignore b/.gitignore index 43524e3c6..36a6fd331 100644 --- a/.gitignore +++ b/.gitignore @@ -170,6 +170,6 @@ docs/workflows/gallery/* !tests/workflows/integration_tests/execution/assets/rock_paper_scissors/*.jpg !tests/workflows/unit_tests/core_steps/models/third_party/assets/*.png !tests/workflows/integration_tests/execution/assets/*.png - +tests/inference_cli/integration_tests/assets/test_images/ inference_profiling tests/inference_sdk/unit_tests/http/inference_profiling diff --git a/inference_cli/lib/utils.py b/inference_cli/lib/utils.py index 58aae9ac9..291cd61c7 100644 --- a/inference_cli/lib/utils.py +++ b/inference_cli/lib/utils.py @@ -15,6 +15,13 @@ def ensure_inference_is_installed() -> None: try: from inference import get_model except Exception as error: + if ( + os.getenv("ALLOW_INTERACTIVE_INFERENCE_INSTALLATION", "True").lower() + == "false" + ): + raise InferencePackageMissingError( + "You need to install `inference` package to use this feature. Run `pip install inference`" + ) from error print( "You need to have `inference` package installed. Do you want the package to be installed? [YES/no]" ) diff --git a/inference_cli/lib/workflows/video_adapter.py b/inference_cli/lib/workflows/video_adapter.py index f4f5f3f4f..1de519542 100644 --- a/inference_cli/lib/workflows/video_adapter.py +++ b/inference_cli/lib/workflows/video_adapter.py @@ -102,7 +102,7 @@ def _flush_stream_buffer(self, stream_idx: int) -> None: content = self._structured_results_buffer[stream_idx] if len(content) == 0: return None - file_path = generate_results_chunk_file_name( + file_path = generate_results_file_name( output_directory=self._output_directory, results_log_type=self._output_file_type, stream_id=stream_idx, @@ -119,22 +119,15 @@ def __del__(self): self.flush() -def generate_results_chunk_file_name( +def generate_results_file_name( output_directory: str, results_log_type: OutputFileType, stream_id: int, ) -> str: output_directory = os.path.abspath(output_directory) - chunks = glob( - os.path.join( - output_directory, - f"workflow_results_source_{stream_id}_part_*.{results_log_type.value}", - ) - ) - chunk_id = len(chunks) return os.path.join( output_directory, - f"workflow_results_source_{stream_id}_part_{chunk_id}.{results_log_type.value}", + f"workflow_results_source_{stream_id}.{results_log_type.value}", ) diff --git a/tests/inference_cli/integration_tests/conftest.py b/tests/inference_cli/integration_tests/conftest.py index 3956b035d..4f7d40275 100644 --- a/tests/inference_cli/integration_tests/conftest.py +++ b/tests/inference_cli/integration_tests/conftest.py @@ -1,6 +1,37 @@ import os.path +import tempfile +from typing import Generator import pytest +import requests + +ASSETS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "assets")) +IMAGES_URLS = [ + "https://source.roboflow.com/BTRTpB7nxxjUchrOQ9vT/aFq7tthQAK6d4pvtupX7/original.jpg", + "https://source.roboflow.com/BTRTpB7nxxjUchrOQ9vT/KmFskd2RQMfcnDNjzeeA/original.jpg", + "https://source.roboflow.com/BTRTpB7nxxjUchrOQ9vT/3FBCYL5SX7VPrg0OVkdN/original.jpg", + "https://source.roboflow.com/BTRTpB7nxxjUchrOQ9vT/K2KrTzjxYu0kJCScGcoH/original.jpg", + "https://source.roboflow.com/BTRTpB7nxxjUchrOQ9vT/XzDB9zVrIxJm17iVKleP/original.jpg", + "https://source.roboflow.com/BTRTpB7nxxjUchrOQ9vT/0fsReHjmHk3hBadXdNk4/original.jpg", + "https://source.roboflow.com/BTRTpB7nxxjUchrOQ9vT/t23lZ0inksJwRRLd3J1b/original.jpg", + "https://source.roboflow.com/BTRTpB7nxxjUchrOQ9vT/3iCH40NuJxcf8l2tXgQn/original.jpg", +] +VIDEO_URL = "https://media.roboflow.com/inference/people-walking.mp4" + +INFERENCE_CLI_TESTS_API_KEY = os.getenv("INFERENCE_CLI_TESTS_API_KEY") +RUN_TESTS_WITH_INFERENCE_PACKAGE = ( + os.getenv("RUN_TESTS_WITH_INFERENCE_PACKAGE", "False").lower() == "true" +) +RUN_TESTS_EXPECTING_ERROR_WHEN_INFERENCE_NOT_INSTALLED = ( + os.getenv("RUN_TESTS_EXPECTING_ERROR_WHEN_INFERENCE_NOT_INSTALLED", "False").lower() + == "true" +) + + +@pytest.fixture(scope="function") +def empty_directory() -> Generator[str, None, None]: + with tempfile.TemporaryDirectory() as tmp_dir: + yield tmp_dir @pytest.fixture(scope="function") @@ -8,3 +39,38 @@ def example_env_file_path() -> str: return os.path.abspath( os.path.join(os.path.dirname(__file__), "assets", "example.env") ) + + +@pytest.fixture +def dataset_directory() -> str: + dataset_directory = os.path.join(ASSETS_DIR, "test_images") + os.makedirs(dataset_directory, exist_ok=True) + expected_video_name = "video.mp4" + current_content = set(os.listdir(dataset_directory)) + all_images_present = all( + f"{i}.jpg" in current_content for i in range(len(IMAGES_URLS)) + ) + if all_images_present and expected_video_name in current_content: + return dataset_directory + for i, image_url in enumerate(IMAGES_URLS): + response = requests.get(image_url) + response.raise_for_status() + image_bytes = response.content + with open(os.path.join(dataset_directory, f"{i}.jpg"), "wb") as f: + f.write(image_bytes) + response = requests.get(VIDEO_URL) + response.raise_for_status() + video_bytes = response.content + with open(os.path.join(dataset_directory, "video.mp4"), "wb") as f: + f.write(video_bytes) + return dataset_directory + + +@pytest.fixture +def video_to_be_processed(dataset_directory: str) -> str: + return os.path.join(dataset_directory, "video.mp4") + + +@pytest.fixture +def image_to_be_processed(dataset_directory: str) -> str: + return os.path.join(dataset_directory, "0.jpg") diff --git a/tests/inference_cli/integration_tests/test_workflows.py b/tests/inference_cli/integration_tests/test_workflows.py new file mode 100644 index 000000000..b0d729d6a --- /dev/null +++ b/tests/inference_cli/integration_tests/test_workflows.py @@ -0,0 +1,485 @@ +import json +import os +import subprocess +from copy import deepcopy + +import cv2 +import pandas as pd +import pytest +import supervision as sv + +from tests.inference_cli.integration_tests.conftest import ( + INFERENCE_CLI_TESTS_API_KEY, + RUN_TESTS_EXPECTING_ERROR_WHEN_INFERENCE_NOT_INSTALLED, + RUN_TESTS_WITH_INFERENCE_PACKAGE, +) + + +@pytest.mark.skipif( + INFERENCE_CLI_TESTS_API_KEY is None, + reason="`INFERENCE_CLI_TESTS_API_KEY` not provided.", +) +@pytest.mark.timeout(120) +def test_processing_image_with_hosted_api( + image_to_be_processed: str, + empty_directory: str, +) -> None: + # given + command = ( + f"python -m inference_cli.main workflows process-image " + f"--image_path {image_to_be_processed} " + f"--output_dir {empty_directory} " + f"--processing_target api " + f"--workspace_name paul-guerrie-tang1 " + f"--workflow_id prod-test-workflow " + f"--api-key {INFERENCE_CLI_TESTS_API_KEY} " + f"--model_id yolov8n-640" + ).split() + new_process_env = deepcopy(os.environ) + new_process_env["ALLOW_INTERACTIVE_INFERENCE_INSTALLATION"] = "False" + + # when + result = subprocess.run(command, env=new_process_env) + + # then + assert result.returncode == 0 + result_dir = os.path.join(empty_directory, os.path.basename(image_to_be_processed)) + results_json = os.path.join(result_dir, "results.json") + result_image = os.path.join(result_dir, "bounding_box_visualization.jpg") + with open(results_json) as f: + decoded_json = json.load(f) + assert set(decoded_json.keys()) == { + "model_predictions", + "bounding_box_visualization", + } + assert decoded_json["bounding_box_visualization"] == "" + assert cv2.imread(result_image) is not None + + +@pytest.mark.skipif( + INFERENCE_CLI_TESTS_API_KEY is None, + reason="`INFERENCE_CLI_TESTS_API_KEY` not provided.", +) +@pytest.mark.timeout(120) +def test_processing_images_directory_with_hosted_api( + dataset_directory: str, + empty_directory: str, +) -> None: + # given + command = ( + f"python -m inference_cli.main workflows process-images-directory " + f"--input_directory {dataset_directory} " + f"--output_dir {empty_directory} " + f"--processing_target api " + f"--workspace_name paul-guerrie-tang1 " + f"--workflow_id prod-test-workflow " + f"--api-key {INFERENCE_CLI_TESTS_API_KEY} " + f"--model_id yolov8n-640 " + ).split() + new_process_env = deepcopy(os.environ) + new_process_env["ALLOW_INTERACTIVE_INFERENCE_INSTALLATION"] = "False" + + # when + result = subprocess.run(command, env=new_process_env) + + # then + assert result.returncode == 0 + assert ( + len(os.listdir(empty_directory)) == 10 + ), "Expected 8 images dirs, log file and aggregated results" + for i in range(8): + image_results_dir = os.path.join(empty_directory, f"{i}.jpg") + image_results_dir_content = set(os.listdir(image_results_dir)) + assert image_results_dir_content == { + "results.json", + "bounding_box_visualization.jpg", + } + result_csv = pd.read_csv(os.path.join(empty_directory, "aggregated_results.csv")) + assert len(result_csv) == 8 + assert ( + len(result_csv.columns) == 2 + ), "Two columns expected - predictions and deducted visualization" + + +@pytest.mark.skipif( + INFERENCE_CLI_TESTS_API_KEY is None, + reason="`INFERENCE_CLI_TESTS_API_KEY` not provided.", +) +@pytest.mark.skipif( + not RUN_TESTS_WITH_INFERENCE_PACKAGE, + reason="`RUN_TESTS_WITH_INFERENCE_PACKAGE` set to False", +) +@pytest.mark.timeout(120) +def test_processing_image_with_inference_package( + image_to_be_processed: str, + empty_directory: str, +) -> None: + # given + command = ( + f"python -m inference_cli.main workflows process-image " + f"--image_path {image_to_be_processed} " + f"--output_dir {empty_directory} " + f"--processing_target inference_package " + f"--workspace_name paul-guerrie-tang1 " + f"--workflow_id prod-test-workflow " + f"--api-key {INFERENCE_CLI_TESTS_API_KEY} " + f"--model_id yolov8n-640" + ).split() + new_process_env = deepcopy(os.environ) + new_process_env["ALLOW_INTERACTIVE_INFERENCE_INSTALLATION"] = "False" + + # when + result = subprocess.run(command, env=new_process_env) + + # then + assert result.returncode == 0 + result_dir = os.path.join(empty_directory, os.path.basename(image_to_be_processed)) + results_json = os.path.join(result_dir, "results.json") + result_image = os.path.join(result_dir, "bounding_box_visualization.jpg") + with open(results_json) as f: + decoded_json = json.load(f) + assert set(decoded_json.keys()) == { + "model_predictions", + "bounding_box_visualization", + } + assert decoded_json["bounding_box_visualization"] == "" + assert cv2.imread(result_image) is not None + + +@pytest.mark.skipif( + INFERENCE_CLI_TESTS_API_KEY is None, + reason="`INFERENCE_CLI_TESTS_API_KEY` not provided.", +) +@pytest.mark.skipif( + not RUN_TESTS_WITH_INFERENCE_PACKAGE, + reason="`RUN_TESTS_WITH_INFERENCE_PACKAGE` set to False", +) +@pytest.mark.timeout(120) +def test_processing_image_with_inference_package_when_output_images_should_not_be_preserved( + image_to_be_processed: str, + empty_directory: str, +) -> None: + # given + command = ( + f"python -m inference_cli.main workflows process-image " + f"--image_path {image_to_be_processed} " + f"--output_dir {empty_directory} " + f"--processing_target inference_package " + f"--workspace_name paul-guerrie-tang1 " + f"--workflow_id prod-test-workflow " + f"--api-key {INFERENCE_CLI_TESTS_API_KEY} " + f"--model_id yolov8n-640 " + f"--no_save_image_outputs" + ).split() + new_process_env = deepcopy(os.environ) + new_process_env["ALLOW_INTERACTIVE_INFERENCE_INSTALLATION"] = "False" + + # when + result = subprocess.run(command, env=new_process_env) + + # then + assert result.returncode == 0 + result_dir = os.path.join(empty_directory, os.path.basename(image_to_be_processed)) + results_json = os.path.join(result_dir, "results.json") + assert not os.path.exists( + os.path.join(result_dir, "bounding_box_visualization.jpg") + ) + with open(results_json) as f: + decoded_json = json.load(f) + assert set(decoded_json.keys()) == { + "model_predictions", + "bounding_box_visualization", + } + assert decoded_json["bounding_box_visualization"] == "" + + +@pytest.mark.skipif( + INFERENCE_CLI_TESTS_API_KEY is None, + reason="`INFERENCE_CLI_TESTS_API_KEY` not provided.", +) +@pytest.mark.skipif( + not RUN_TESTS_WITH_INFERENCE_PACKAGE, + reason="`RUN_TESTS_WITH_INFERENCE_PACKAGE` set to False", +) +@pytest.mark.timeout(120) +def test_processing_images_directory_with_inference_package( + dataset_directory: str, + empty_directory: str, +) -> None: + # given + command = ( + f"python -m inference_cli.main workflows process-images-directory " + f"--input_directory {dataset_directory} " + f"--output_dir {empty_directory} " + f"--processing_target inference_package " + f"--workspace_name paul-guerrie-tang1 " + f"--workflow_id prod-test-workflow " + f"--api-key {INFERENCE_CLI_TESTS_API_KEY} " + f"--model_id yolov8n-640 " + ).split() + new_process_env = deepcopy(os.environ) + new_process_env["ALLOW_INTERACTIVE_INFERENCE_INSTALLATION"] = "False" + + # when + result = subprocess.run(command, env=new_process_env) + + # then + assert result.returncode == 0 + assert ( + len(os.listdir(empty_directory)) == 10 + ), "Expected 8 images dirs, log file and aggregated results" + for i in range(8): + image_results_dir = os.path.join(empty_directory, f"{i}.jpg") + image_results_dir_content = set(os.listdir(image_results_dir)) + assert image_results_dir_content == { + "results.json", + "bounding_box_visualization.jpg", + } + result_csv = pd.read_csv(os.path.join(empty_directory, "aggregated_results.csv")) + assert len(result_csv) == 8 + assert ( + len(result_csv.columns) == 2 + ), "Two columns expected - predictions and deducted visualization" + + +@pytest.mark.skipif( + INFERENCE_CLI_TESTS_API_KEY is None, + reason="`INFERENCE_CLI_TESTS_API_KEY` not provided.", +) +@pytest.mark.skipif( + not RUN_TESTS_WITH_INFERENCE_PACKAGE, + reason="`RUN_TESTS_WITH_INFERENCE_PACKAGE` set to False", +) +@pytest.mark.timeout(120) +def test_processing_video_with_inference_package( + video_to_be_processed: str, + empty_directory: str, +) -> None: + # given + command = ( + f"python -m inference_cli.main workflows process-video " + f"--video_path {video_to_be_processed} " + f"--output_dir {empty_directory} " + f"--workspace_name paul-guerrie-tang1 " + f"--workflow_id prod-test-workflow " + f"--api-key {INFERENCE_CLI_TESTS_API_KEY} " + f"--model_id yolov8n-640" + ).split() + new_process_env = deepcopy(os.environ) + new_process_env["ALLOW_INTERACTIVE_INFERENCE_INSTALLATION"] = "False" + + result = subprocess.run(command, env=new_process_env) + + # then + assert result.returncode == 0 + result_csv = pd.read_csv( + os.path.join(empty_directory, "workflow_results_source_0.csv") + ) + assert len(result_csv) == 341 + assert len(result_csv.columns) == 2 + output_video_info = sv.VideoInfo.from_video_path( + video_path=os.path.join( + empty_directory, "source_0_output_bounding_box_visualization_preview.mp4" + ) + ) + assert output_video_info.total_frames == 341 + + +@pytest.mark.skipif( + INFERENCE_CLI_TESTS_API_KEY is None, + reason="`INFERENCE_CLI_TESTS_API_KEY` not provided.", +) +@pytest.mark.skipif( + not RUN_TESTS_WITH_INFERENCE_PACKAGE, + reason="`RUN_TESTS_WITH_INFERENCE_PACKAGE` set to False", +) +@pytest.mark.timeout(120) +def test_processing_video_with_inference_package_with_modulated_fps( + video_to_be_processed: str, + empty_directory: str, +) -> None: + # given + command = ( + f"python -m inference_cli.main workflows process-video " + f"--video_path {video_to_be_processed} " + f"--output_dir {empty_directory} " + f"--workspace_name paul-guerrie-tang1 " + f"--workflow_id prod-test-workflow " + f"--api-key {INFERENCE_CLI_TESTS_API_KEY} " + f"--model_id yolov8n-640 " + f"--max_fps 1.0" + ).split() + new_process_env = deepcopy(os.environ) + new_process_env["ALLOW_INTERACTIVE_INFERENCE_INSTALLATION"] = "False" + + result = subprocess.run(command, env=new_process_env, stdout=subprocess.PIPE) + + # then + assert result.returncode == 0 + result_csv = pd.read_csv( + os.path.join(empty_directory, "workflow_results_source_0.csv") + ) + assert len(result_csv) == 14 + assert len(result_csv.columns) == 2 + output_video_info = sv.VideoInfo.from_video_path( + video_path=os.path.join( + empty_directory, "source_0_output_bounding_box_visualization_preview.mp4" + ) + ) + assert output_video_info.total_frames == 14 + + +@pytest.mark.skipif( + INFERENCE_CLI_TESTS_API_KEY is None, + reason="`INFERENCE_CLI_TESTS_API_KEY` not provided.", +) +@pytest.mark.skipif( + not RUN_TESTS_WITH_INFERENCE_PACKAGE, + reason="`RUN_TESTS_WITH_INFERENCE_PACKAGE` set to False", +) +@pytest.mark.timeout(120) +def test_processing_video_with_inference_package_with_modulated_fps_when_video_should_not_be_preserved( + video_to_be_processed: str, + empty_directory: str, +) -> None: + # given + command = ( + f"python -m inference_cli.main workflows process-video " + f"--video_path {video_to_be_processed} " + f"--output_dir {empty_directory} " + f"--workspace_name paul-guerrie-tang1 " + f"--workflow_id prod-test-workflow " + f"--api-key {INFERENCE_CLI_TESTS_API_KEY} " + f"--model_id yolov8n-640 " + f"--max_fps 1.0 " + f"--no_save_out_video" + ).split() + new_process_env = deepcopy(os.environ) + new_process_env["ALLOW_INTERACTIVE_INFERENCE_INSTALLATION"] = "False" + + result = subprocess.run(command, env=new_process_env, stdout=subprocess.PIPE) + + # then + assert result.returncode == 0 + result_csv = pd.read_csv( + os.path.join(empty_directory, "workflow_results_source_0.csv") + ) + assert len(result_csv) == 14 + assert len(result_csv.columns) == 2 + assert not os.path.exists( + os.path.join( + empty_directory, "source_0_output_bounding_box_visualization_preview.mp4" + ) + ) + + +@pytest.mark.skipif( + INFERENCE_CLI_TESTS_API_KEY is None, + reason="`INFERENCE_CLI_TESTS_API_KEY` not provided.", +) +@pytest.mark.skipif( + not RUN_TESTS_EXPECTING_ERROR_WHEN_INFERENCE_NOT_INSTALLED, + reason="`RUN_TESTS_EXPECTING_ERROR_WHEN_INFERENCE_NOT_INSTALLED` set to False", +) +@pytest.mark.timeout(120) +def test_processing_image_with_inference_package_when_inference_not_installed( + empty_directory: str, + image_to_be_processed: str, +) -> None: + # given + command = ( + f"python -m inference_cli.main workflows process-image " + f"--image_path {image_to_be_processed} " + f"--output_dir {empty_directory} " + f"--processing_target inference_package " + f"--workspace_name paul-guerrie-tang1 " + f"--workflow_id prod-test-workflow " + f"--api-key {INFERENCE_CLI_TESTS_API_KEY} " + f"--model_id yolov8n-640" + ).split() + new_process_env = deepcopy(os.environ) + new_process_env["ALLOW_INTERACTIVE_INFERENCE_INSTALLATION"] = "False" + + # when + result = subprocess.run(command, env=new_process_env, stdout=subprocess.PIPE) + + # then + assert result.returncode != 0 + assert ( + "You need to install `inference` package to use this feature" + in result.stdout.decode("utf-8") + ) + + +@pytest.mark.skipif( + INFERENCE_CLI_TESTS_API_KEY is None, + reason="`INFERENCE_CLI_TESTS_API_KEY` not provided.", +) +@pytest.mark.skipif( + not RUN_TESTS_EXPECTING_ERROR_WHEN_INFERENCE_NOT_INSTALLED, + reason="`RUN_TESTS_EXPECTING_ERROR_WHEN_INFERENCE_NOT_INSTALLED` set to False", +) +@pytest.mark.timeout(120) +def test_processing_images_directory_with_inference_package_when_inference_not_installed( + empty_directory: str, + dataset_directory: str, +) -> None: + # given + command = ( + f"python -m inference_cli.main workflows process-images-directory " + f"--input_directory {dataset_directory} " + f"--output_dir {empty_directory} " + f"--processing_target inference_package " + f"--workspace_name paul-guerrie-tang1 " + f"--workflow_id prod-test-workflow " + f"--api-key {INFERENCE_CLI_TESTS_API_KEY} " + f"--model_id yolov8n-640 " + ).split() + new_process_env = deepcopy(os.environ) + new_process_env["ALLOW_INTERACTIVE_INFERENCE_INSTALLATION"] = "False" + + # when + result = subprocess.run(command, env=new_process_env, stdout=subprocess.PIPE) + + # then + assert result.returncode != 0 + assert ( + "You need to install `inference` package to use this feature" + in result.stdout.decode("utf-8") + ) + + +@pytest.mark.skipif( + INFERENCE_CLI_TESTS_API_KEY is None, + reason="`INFERENCE_CLI_TESTS_API_KEY` not provided.", +) +@pytest.mark.skipif( + not RUN_TESTS_EXPECTING_ERROR_WHEN_INFERENCE_NOT_INSTALLED, + reason="`RUN_TESTS_EXPECTING_ERROR_WHEN_INFERENCE_NOT_INSTALLED` set to False", +) +@pytest.mark.timeout(120) +def test_processing_video_with_inference_package_when_inference_not_installed( + empty_directory: str, + video_to_be_processed: str, +) -> None: + # given + command = ( + f"python -m inference_cli.main workflows process-video " + f"--video_path {video_to_be_processed} " + f"--output_dir {empty_directory} " + f"--workspace_name paul-guerrie-tang1 " + f"--workflow_id prod-test-workflow " + f"--api-key {INFERENCE_CLI_TESTS_API_KEY} " + f"--model_id yolov8n-640" + ).split() + new_process_env = deepcopy(os.environ) + new_process_env["ALLOW_INTERACTIVE_INFERENCE_INSTALLATION"] = "False" + + result = subprocess.run(command, env=new_process_env, stdout=subprocess.PIPE) + + # then + assert result.returncode != 0 + assert ( + "You need to install `inference` package to use this feature" + in result.stdout.decode("utf-8") + ) From 73e83867ad708033b46bdf975616c41f231c48d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Wed, 27 Nov 2024 21:23:36 +0100 Subject: [PATCH 13/16] Correct tests --- .../integration_tests/test_workflows.py | 43 ------------------- .../unit_tests/lib/workflows/test_common.py | 24 +++++------ 2 files changed, 12 insertions(+), 55 deletions(-) diff --git a/tests/inference_cli/integration_tests/test_workflows.py b/tests/inference_cli/integration_tests/test_workflows.py index b0d729d6a..79b7347ee 100644 --- a/tests/inference_cli/integration_tests/test_workflows.py +++ b/tests/inference_cli/integration_tests/test_workflows.py @@ -242,49 +242,6 @@ def test_processing_images_directory_with_inference_package( ), "Two columns expected - predictions and deducted visualization" -@pytest.mark.skipif( - INFERENCE_CLI_TESTS_API_KEY is None, - reason="`INFERENCE_CLI_TESTS_API_KEY` not provided.", -) -@pytest.mark.skipif( - not RUN_TESTS_WITH_INFERENCE_PACKAGE, - reason="`RUN_TESTS_WITH_INFERENCE_PACKAGE` set to False", -) -@pytest.mark.timeout(120) -def test_processing_video_with_inference_package( - video_to_be_processed: str, - empty_directory: str, -) -> None: - # given - command = ( - f"python -m inference_cli.main workflows process-video " - f"--video_path {video_to_be_processed} " - f"--output_dir {empty_directory} " - f"--workspace_name paul-guerrie-tang1 " - f"--workflow_id prod-test-workflow " - f"--api-key {INFERENCE_CLI_TESTS_API_KEY} " - f"--model_id yolov8n-640" - ).split() - new_process_env = deepcopy(os.environ) - new_process_env["ALLOW_INTERACTIVE_INFERENCE_INSTALLATION"] = "False" - - result = subprocess.run(command, env=new_process_env) - - # then - assert result.returncode == 0 - result_csv = pd.read_csv( - os.path.join(empty_directory, "workflow_results_source_0.csv") - ) - assert len(result_csv) == 341 - assert len(result_csv.columns) == 2 - output_video_info = sv.VideoInfo.from_video_path( - video_path=os.path.join( - empty_directory, "source_0_output_bounding_box_visualization_preview.mp4" - ) - ) - assert output_video_info.total_frames == 341 - - @pytest.mark.skipif( INFERENCE_CLI_TESTS_API_KEY is None, reason="`INFERENCE_CLI_TESTS_API_KEY` not provided.", diff --git a/tests/inference_cli/unit_tests/lib/workflows/test_common.py b/tests/inference_cli/unit_tests/lib/workflows/test_common.py index 0af36d179..753a80547 100644 --- a/tests/inference_cli/unit_tests/lib/workflows/test_common.py +++ b/tests/inference_cli/unit_tests/lib/workflows/test_common.py @@ -91,8 +91,8 @@ def test_aggregate_batch_processing_results_when_json_output_is_expected_and_res { "some": "value", "other": 3.0, - "list": [1, 2, 3], - "object": {"nested": "value"}, + "list_field": [1, 2, 3], + "object_field": {"nested": "value"}, } ] * 2 @@ -142,12 +142,12 @@ def test_aggregate_batch_processing_results_when_csv_output_is_expected_and_resu assert len(df) == 2, "Expected 2 records" assert df.iloc[0].some == "value" assert df.iloc[0].other == 3.0 - assert json.loads(df.iloc[0].list) == [1, 2, 3] - assert json.loads(df.iloc[0].object) == {"nested": "value"} + assert json.loads(df.iloc[0].list_field) == [1, 2, 3] + assert json.loads(df.iloc[0].object_field) == {"nested": "value"} assert df.iloc[1].some == "value" assert df.iloc[1].other == 3.0 - assert json.loads(df.iloc[1].list) == [1, 2, 3] - assert json.loads(df.iloc[1].object) == {"nested": "value"} + assert json.loads(df.iloc[1].list_field) == [1, 2, 3] + assert json.loads(df.iloc[1].object_field) == {"nested": "value"} def test_aggregate_batch_processing_results_when_csv_output_is_expected_and_results_present_but_with_inconsistent_schema( @@ -177,12 +177,12 @@ def test_aggregate_batch_processing_results_when_csv_output_is_expected_and_resu assert len(df) == 2, "Expected 2 records" assert df.iloc[0].some == "value" assert df.iloc[0].other == 3.0 - assert json.loads(df.iloc[0].list) == [1, 2, 3] - assert json.loads(df.iloc[0].object) == {"nested": "value"} + assert json.loads(df.iloc[0].list_field) == [1, 2, 3] + assert json.loads(df.iloc[0].object_field) == {"nested": "value"} assert df.iloc[1].some == "value" assert df.iloc[1].other == 3.0 - assert json.loads(df.iloc[1].list) == [1, 2, 3] - assert json.loads(df.iloc[1].object) == {"nested": "value"} + assert json.loads(df.iloc[1].list_field) == [1, 2, 3] + assert json.loads(df.iloc[1].object_field) == {"nested": "value"} assert ( df.iloc[1].extra == "column" or df.iloc[0].extra == "column" @@ -217,8 +217,8 @@ def _prepare_dummy_results( results = { "some": "value", "other": 3.0, - "list": [1, 2, 3], - "object": {"nested": "value"}, + "list_field": [1, 2, 3], + "object_field": {"nested": "value"}, } results.update(extra_data) results_path = os.path.join(sub_dir_path, "results.json") From 39afe60b55b39d4a297a7a0aac8418cb600f30ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Wed, 27 Nov 2024 21:25:38 +0100 Subject: [PATCH 14/16] Reduce size of assets for tests --- tests/inference_cli/integration_tests/conftest.py | 5 ----- .../integration_tests/test_workflows.py | 12 ++++++------ 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/tests/inference_cli/integration_tests/conftest.py b/tests/inference_cli/integration_tests/conftest.py index 4f7d40275..4e373e0a3 100644 --- a/tests/inference_cli/integration_tests/conftest.py +++ b/tests/inference_cli/integration_tests/conftest.py @@ -10,11 +10,6 @@ "https://source.roboflow.com/BTRTpB7nxxjUchrOQ9vT/aFq7tthQAK6d4pvtupX7/original.jpg", "https://source.roboflow.com/BTRTpB7nxxjUchrOQ9vT/KmFskd2RQMfcnDNjzeeA/original.jpg", "https://source.roboflow.com/BTRTpB7nxxjUchrOQ9vT/3FBCYL5SX7VPrg0OVkdN/original.jpg", - "https://source.roboflow.com/BTRTpB7nxxjUchrOQ9vT/K2KrTzjxYu0kJCScGcoH/original.jpg", - "https://source.roboflow.com/BTRTpB7nxxjUchrOQ9vT/XzDB9zVrIxJm17iVKleP/original.jpg", - "https://source.roboflow.com/BTRTpB7nxxjUchrOQ9vT/0fsReHjmHk3hBadXdNk4/original.jpg", - "https://source.roboflow.com/BTRTpB7nxxjUchrOQ9vT/t23lZ0inksJwRRLd3J1b/original.jpg", - "https://source.roboflow.com/BTRTpB7nxxjUchrOQ9vT/3iCH40NuJxcf8l2tXgQn/original.jpg", ] VIDEO_URL = "https://media.roboflow.com/inference/people-walking.mp4" diff --git a/tests/inference_cli/integration_tests/test_workflows.py b/tests/inference_cli/integration_tests/test_workflows.py index 79b7347ee..e3e6660ca 100644 --- a/tests/inference_cli/integration_tests/test_workflows.py +++ b/tests/inference_cli/integration_tests/test_workflows.py @@ -85,8 +85,8 @@ def test_processing_images_directory_with_hosted_api( # then assert result.returncode == 0 assert ( - len(os.listdir(empty_directory)) == 10 - ), "Expected 8 images dirs, log file and aggregated results" + len(os.listdir(empty_directory)) == 5 + ), "Expected 3 images dirs, log file and aggregated results" for i in range(8): image_results_dir = os.path.join(empty_directory, f"{i}.jpg") image_results_dir_content = set(os.listdir(image_results_dir)) @@ -95,7 +95,7 @@ def test_processing_images_directory_with_hosted_api( "bounding_box_visualization.jpg", } result_csv = pd.read_csv(os.path.join(empty_directory, "aggregated_results.csv")) - assert len(result_csv) == 8 + assert len(result_csv) == 3 assert ( len(result_csv.columns) == 2 ), "Two columns expected - predictions and deducted visualization" @@ -226,8 +226,8 @@ def test_processing_images_directory_with_inference_package( # then assert result.returncode == 0 assert ( - len(os.listdir(empty_directory)) == 10 - ), "Expected 8 images dirs, log file and aggregated results" + len(os.listdir(empty_directory)) == 5 + ), "Expected 3 images dirs, log file and aggregated results" for i in range(8): image_results_dir = os.path.join(empty_directory, f"{i}.jpg") image_results_dir_content = set(os.listdir(image_results_dir)) @@ -236,7 +236,7 @@ def test_processing_images_directory_with_inference_package( "bounding_box_visualization.jpg", } result_csv = pd.read_csv(os.path.join(empty_directory, "aggregated_results.csv")) - assert len(result_csv) == 8 + assert len(result_csv) == 3 assert ( len(result_csv.columns) == 2 ), "Two columns expected - predictions and deducted visualization" From 8c348274d3a5198c4ad35e339c09537a0ca90683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Wed, 27 Nov 2024 22:21:51 +0100 Subject: [PATCH 15/16] Add docs --- docs/inference_helpers/inference_cli.md | 86 +++++++++++++++++++ docs/workflows/kinds.md | 2 +- docs/workflows/modes_of_running.md | 36 ++++++++ inference_cli/workflows.py | 40 ++++++++- .../integration_tests/test_workflows.py | 8 +- 5 files changed, 166 insertions(+), 6 deletions(-) diff --git a/docs/inference_helpers/inference_cli.md b/docs/inference_helpers/inference_cli.md index 8def634c1..809e8b60c 100644 --- a/docs/inference_helpers/inference_cli.md +++ b/docs/inference_helpers/inference_cli.md @@ -360,6 +360,92 @@ option can be used (and `-c` will be ignored). Value provided in `--rps` option are to be spawned **each second** without waiting for previous requests to be handled. In I/O intensive benchmark scenarios - we suggest running command from multiple separate processes and possibly multiple hosts. +### inference workflows + +In release `0.29.0`, `inference-cli` was extended with command to process data using Workflows. It is possible to +process: + +* individual images + +* directories of images + +* video files + +#### Processing individual image + +Basic usage of the command is illustrated below: + +```bash +inference workflows process-image \ + --image_path {your-input-image} \ + --output_dir {your-output-dir} \ + --workspace_name {your-roboflow-workspace-url} \ + --workflow_id {your-workflow-id} \ + --api-key {your_roboflow_api_key} +``` + +which would take your input image, run it against your Workflow and save results in output directory. By default, +Workflow will be processed using Roboflow Hosted API. You can tweak behaviour of the command: + +* if you want to process the image locally, using `inference` Python package - use +`--processing_target inference_package` option (*requires `inference` to be installed*) + +* to see all options, use `inference workflows process-image --help` command + +* any option that starts from `--` which is not enlisted in `--help` command will be treated as input parameter +to the workflow execution + + +#### Processing directory of images + +Basic usage of the command is illustrated below: + +```bash +inference workflows process-images-directory \ + -i {your_input_directory} \ + -o {your_output_directory} \ + --workspace_name {your-roboflow-workspace-url} \ + --workflow_id {your-workflow-id} \ + --api-key {your_roboflow_api_key} +``` + +You can tweak behaviour of the command: + +* if you want to process the image locally, using `inference` Python package - use +`--processing_target inference_package` option (*requires `inference` to be installed*) + +* to see all options, use `inference workflows process-image --help` command + +* any option that starts from `--` which is not enlisted in `--help` command will be treated as input parameter +to the workflow execution + +#### Processing video file + +!!! Note "`inference` required" + + This command requires `inference` to be installed. + +Basic usage of the command is illustrated below: + +```bash +inference workflows process-video \ + --video_path {video_to_be_processed} \ + --output_dir {empty_directory} \ + --workspace_name {your-roboflow-workspace-url} \ + --workflow_id {your-workflow-id} \ + --api-key {your_roboflow_api_key} +``` + +You can tweak behaviour of the command: + +* `--max_fps` option can be used to subsample video frames while processing + +* to see all options, use `inference workflows process-image --help` command + +* any option that starts from `--` which is not enlisted in `--help` command will be treated as input parameter +to the workflow execution + + ## Supported Devices Roboflow Inference CLI currently supports the following device targets: diff --git a/docs/workflows/kinds.md b/docs/workflows/kinds.md index fc3557d77..3c1c22a52 100644 --- a/docs/workflows/kinds.md +++ b/docs/workflows/kinds.md @@ -6,7 +6,7 @@ provided at runtime, either from user inputs or from other function outputs. To manage this, Workflows use *selectors*, which act like references, pointing to data without containing it directly. -!!! Example *selectors* +!!! example "selectors" Selectors might refer to a named input - for example input image - like `$inputs.image` or predictions generated by a previous step - like `$steps.my_model.predictions` diff --git a/docs/workflows/modes_of_running.md b/docs/workflows/modes_of_running.md index 983ce340e..d0fa7c690 100644 --- a/docs/workflows/modes_of_running.md +++ b/docs/workflows/modes_of_running.md @@ -221,6 +221,42 @@ Explore the example below to see how to combine `InferencePipeline` with Workflo Make sure you have `inference` or `inference-gpu` package installed in your Python environment +## Batch processing using `inference-cli` + +[`inference-cli`](/inference_helpers/inference_cli/) is command-line wrapper library around `inference`. You can use it +to process your data using Workflows without writing a single line of code. You simply point the data to be processed, +select your Workflow and specify where results should be saved. Thanks to `inference-cli` you can process: + +* individual images + +* directories of images + +* video files + +!!! example "Processing directory of images" + + You can start the processing using the following command: + + ```bash + inference workflows process-images-directory \ + -i {your_input_directory} \ + -o {your_output_directory} \[workflows.py](..%2F..%2Finference_cli%2Fworkflows.py) + --workspace_name {your-roboflow-workspace-url} \ + --workflow_id {your-workflow-id} \ + --api-key {your_roboflow_api_key} + ``` + + As a result, in the directory specified in `-o` option you should be able to find: + + * sub-directories named after files in your original directory with `results.json` file that contain Worklfow + results and optionally additional `*.jpg` files with images created during Workflow execution + + * `aggregated_results.csv` file that contain concatenated results of Workflow execution for all input image file + + !!! note + + Make sure you have `inference` or `inference-cli` package installed in your Python environment + ## Workflows in Python package diff --git a/inference_cli/workflows.py b/inference_cli/workflows.py index ea88b92ca..38e44780a 100644 --- a/inference_cli/workflows.py +++ b/inference_cli/workflows.py @@ -276,7 +276,25 @@ def process_image( help="Flag enabling errors stack traces to be displayed (helpful for debugging)", ), ] = False, -): + proceed_automatically: Annotated[ + bool, + typer.Option( + "--yes/--no", + "-y/-n", + help="Boolean flag to decide on auto `yes` answer given on user input required.", + ), + ] = False, +) -> None: + if ( + processing_target is ProcessingTarget.API + and "roboflow.com" in api_url + and not proceed_automatically + ): + proceed = input( + "This action may easily exceed your Roboflow inference credits. Are you sure? [y/N] " + ) + if proceed.lower() != "y": + return None try: ensure_target_directory_is_empty( output_directory=output_directory, @@ -456,7 +474,25 @@ def process_images_directory( help="Flag enabling errors stack traces to be displayed (helpful for debugging)", ), ] = False, -): + proceed_automatically: Annotated[ + bool, + typer.Option( + "--yes/--no", + "-y/-n", + help="Boolean flag to decide on auto `yes` answer given on user input required.", + ), + ] = False, +) -> None: + if ( + processing_target is ProcessingTarget.API + and "roboflow.com" in api_url + and not proceed_automatically + ): + proceed = input( + "This action may easily exceed your Roboflow inference credits. Are you sure? [y/N] " + ) + if proceed.lower() != "y": + return None try: ensure_target_directory_is_empty( output_directory=output_directory, diff --git a/tests/inference_cli/integration_tests/test_workflows.py b/tests/inference_cli/integration_tests/test_workflows.py index e3e6660ca..fa2d9a2a6 100644 --- a/tests/inference_cli/integration_tests/test_workflows.py +++ b/tests/inference_cli/integration_tests/test_workflows.py @@ -33,7 +33,8 @@ def test_processing_image_with_hosted_api( f"--workspace_name paul-guerrie-tang1 " f"--workflow_id prod-test-workflow " f"--api-key {INFERENCE_CLI_TESTS_API_KEY} " - f"--model_id yolov8n-640" + f"--model_id yolov8n-640 " + f"--yes" ).split() new_process_env = deepcopy(os.environ) new_process_env["ALLOW_INTERACTIVE_INFERENCE_INSTALLATION"] = "False" @@ -75,6 +76,7 @@ def test_processing_images_directory_with_hosted_api( f"--workflow_id prod-test-workflow " f"--api-key {INFERENCE_CLI_TESTS_API_KEY} " f"--model_id yolov8n-640 " + f"--yes" ).split() new_process_env = deepcopy(os.environ) new_process_env["ALLOW_INTERACTIVE_INFERENCE_INSTALLATION"] = "False" @@ -87,7 +89,7 @@ def test_processing_images_directory_with_hosted_api( assert ( len(os.listdir(empty_directory)) == 5 ), "Expected 3 images dirs, log file and aggregated results" - for i in range(8): + for i in range(3): image_results_dir = os.path.join(empty_directory, f"{i}.jpg") image_results_dir_content = set(os.listdir(image_results_dir)) assert image_results_dir_content == { @@ -228,7 +230,7 @@ def test_processing_images_directory_with_inference_package( assert ( len(os.listdir(empty_directory)) == 5 ), "Expected 3 images dirs, log file and aggregated results" - for i in range(8): + for i in range(3): image_results_dir = os.path.join(empty_directory, f"{i}.jpg") image_results_dir_content = set(os.listdir(image_results_dir)) assert image_results_dir_content == { From b230a02740bf052e09f95214a12d9413a4fdaec9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20P=C4=99czek?= Date: Thu, 28 Nov 2024 10:43:50 +0100 Subject: [PATCH 16/16] Modify docs and remove security issue --- .../cli_commands/benchmark.md | 71 +++ docs/inference_helpers/cli_commands/cloud.md | 123 +++++ docs/inference_helpers/cli_commands/infer.md | 139 ++++++ docs/inference_helpers/cli_commands/server.md | 68 +++ .../cli_commands/workflows.md | 123 +++++ docs/inference_helpers/inference_cli.md | 436 +----------------- inference_cli/workflows.py | 36 -- mkdocs.yml | 8 +- .../integration_tests/test_workflows.py | 2 - 9 files changed, 541 insertions(+), 465 deletions(-) create mode 100644 docs/inference_helpers/cli_commands/benchmark.md create mode 100644 docs/inference_helpers/cli_commands/cloud.md create mode 100644 docs/inference_helpers/cli_commands/infer.md create mode 100644 docs/inference_helpers/cli_commands/server.md create mode 100644 docs/inference_helpers/cli_commands/workflows.md diff --git a/docs/inference_helpers/cli_commands/benchmark.md b/docs/inference_helpers/cli_commands/benchmark.md new file mode 100644 index 000000000..f295e7da8 --- /dev/null +++ b/docs/inference_helpers/cli_commands/benchmark.md @@ -0,0 +1,71 @@ +# Benchmarking `inference` + +`inference benchmark` offers you an easy way to check the performance of `inference` in your setup. The command +is capable of benchmarking both `inference` server and `inference` Python package. + +!!! Tip "Discovering command capabilities" + + To check detail of the command, run: + + ```bash + inference benchmark --help + ``` + + Additionally, help guide is also available for each sub-command: + + ```bash + inference benchmark api-speed --help + ``` + +## Benchmarking `inference` Python package + +!!! Important "`inference` needs to be installed" + + Running this command, make sure `inference` package is installed. + + ```bash + pip install inference + ``` + + +Basic benchmark can be run using the following command: + +```bash +inference benchmark python-package-speed \ + -m {your_model_id} \ + -d {pre-configured dataset name or path to directory with images} \ + -o {output_directory} +``` +Command runs specified number of inferences using pointed model and saves statistics (including benchmark +parameter, throughput, latency, errors and platform details) in pointed directory. + + +## Benchmarking `inference` server + +!!! note + + Before running API benchmark of your local `inference` server - make sure the server is up and running: + + ```bash + inference server start + ``` +Basic benchmark can be run using the following command: + +```bash +inference benchmark api-speed \ + -m {your_model_id} \ + -d {pre-configured dataset name or path to directory with images} \ + -o {output_directory} +``` +Command runs specified number of inferences using pointed model and saves statistics (including benchmark +parameter, throughput, latency, errors and platform details) in pointed directory. + +This benchmark has more configuration options to support different ways HTTP API profiling. In default mode, +single client will be spawned, and it will send one request after another sequentially. This may be suboptimal +in specific cases, so one may specify number of concurrent clients using `-c {number_of_clients}` option. +Each client will send next request once previous is handled. This option will also not cover all scenarios +of tests. For instance one may want to send `x` requests each second (which is closer to the scenario of +production environment where multiple clients are sending requests concurrently). In this scenario, `--rps {value}` +option can be used (and `-c` will be ignored). Value provided in `--rps` option specifies how many requests +are to be spawned **each second** without waiting for previous requests to be handled. In I/O intensive benchmark +scenarios - we suggest running command from multiple separate processes and possibly multiple hosts. diff --git a/docs/inference_helpers/cli_commands/cloud.md b/docs/inference_helpers/cli_commands/cloud.md new file mode 100644 index 000000000..fcd470b1d --- /dev/null +++ b/docs/inference_helpers/cli_commands/cloud.md @@ -0,0 +1,123 @@ +# Deploying `inference` to Cloud + +You can deploy Roboflow Inference containers to virtual machines in the cloud. These VMs are configured to run CPU or +GPU-based Inference servers under the hood, so you don't have to deal with OS/GPU drivers/docker installations, etc! +The Inference cli currently supports deploying the Roboflow Inference container images into a virtual machine running +on Google (GCP) or Amazon cloud (AWS). + +The Roboflow Inference CLI assumes the corresponding cloud CLI is configured for the project you want to deploy the +virtual machine into. Read instructions for setting up [Google/GCP - gcloud cli](https://cloud.google.com/sdk/docs/install) or the [Amazon/AWS aws cli](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html). + +Roboflow Inference cloud deploy is powered by the popular [Skypilot project](https://github.com/skypilot-org/skypilot). + +!!! Important "Make sure `cloud-deploy` extras is installed" + + To run commands presented below, you need to have `cloud-deploy` extras installed: + + ```bash + pip install "inference-cli[cloud-deploy]" + ``` + +!!! Tip "Discovering command capabilities" + + To check detail of the command, run: + + ```bash + inference cloud --help + ``` + + Additionally, help guide is also available for each sub-command: + + ```bash + inference cloud deploy --help + ``` + +## `inference cloud deploy` + +We illustrate Inference cloud deploy with some examples, below. + +*Deploy GPU or CPU inference to AWS or GCP* + +```bash +# Deploy the roboflow Inference GPU container into a GPU-enabled VM in AWS + +inference cloud deploy --provider aws --compute-type gpu +``` + +```bash +# Deploy the roboflow Inference CPU container into a CPU-only VM in GCP + +inference cloud deploy --provider gcp --compute-type cpu +``` + +Note the "cluster name" printed after the deployment completes. This handle is used in many subsequent commands. +The deploy command also prints helpful debug and cost information about your VM. + +Deploying Inference into a cloud VM will also print out an endpoint of the form "http://1.2.3.4:9001"; you can now run inferences against this endpoint. + +Note that the port 9001 is automatically opened - check with your security admin if this is acceptable for your cloud/project. + +## `inference cloud status` + +To check the status of your deployment, run: + +```bash +inference cloud status +``` + +## Stop and start deployments + +You can start and stop your deployment using: + +```bash +inference cloud start +``` + +and + +```bash +# Stop the VM, you only pay for disk storage while the VM is stopped +inference cloud stop + +``` + +## `inference cloud undeploy` + +To delete (undeploy) your deployment, run: + +```bash +inference cloud undeploy +``` + +## SSH into the cloud deployment + +You can SSH into your cloud deployment with the following command: +```bash +ssh +``` + +The required SSH key is automatically added to your `~/.ssh/config`, you don't need to configure this manually. + + +## Cloud Deploy Customization + +Roboflow Inference cloud deploy will create VMs based on internally tested templates. + +For advanced usecases and to customize the template, you can use your [sky yaml](https://skypilot.readthedocs.io/en/latest/reference/yaml-spec.html) template on the command-line, like so: + +```bash +inference cloud deploy --custom /path/to/sky-template.yaml +``` + +If you want you can download the standard template stored in the roboflow cli and the modify it for your needs, this command will do that. + +```bash +# This command will print out the standard gcp/cpu sky template. +inference cloud deploy --dry-run --provider gcp --compute-type cpu +``` + +Then you can deploy a custom template based off your changes. + +As an aside, you can also use the [sky cli](https://skypilot.readthedocs.io/en/latest/reference/cli.html) to control your deployment(s) and access some more advanced functionality. + +Roboflow Inference deploy currently supports AWS and GCP, please open an issue on the [Inference GitHub repository](https://github.com/roboflow/inference/issues) if you would like to see other cloud providers supported. diff --git a/docs/inference_helpers/cli_commands/infer.md b/docs/inference_helpers/cli_commands/infer.md new file mode 100644 index 000000000..91e99974b --- /dev/null +++ b/docs/inference_helpers/cli_commands/infer.md @@ -0,0 +1,139 @@ +# Making predictions from your models + +`inference infer` command offers an easy way to make predictions from your model based on your input images or video +files sending requests to `inference` server, depending on command configuration. + +!!! Tip "Discovering command capabilities" + + To check detail of the command, run: + + ```bash + inference infer --help + ``` + +## Command details + +`inference infer` takes input path / url and model version to produce predictions (and optionally make visualisation +using `supervision`). You can also specify a host to run inference on our hosted inference server. + +!!! note + + If you decided to use hosted inference server - make sure command `inference server start` was used first + +!!! tip + + Roboflow API key can be provided via `ROBOFLOW_API_KEY` environment variable + +## Examples + +Below, you have usage examples illustrated. + +### Predict On Local Image + +This command is going to make a prediction from local image using selected model and print the prediction on +the console. + +```bash +inference infer -i ./image.jpg -m {your_project}/{version} --api-key {YOUR_API_KEY} +``` + +To display visualised prediction use `-D` option. To save prediction and visualisation in a local directory, +use `-o {path_to_your_directory}` option. Those options work also in other modes. + +```bash +inference infer -i ./image.jpg -m {your_project}/{version} --api-key {YOUR_API_KEY} -D -o {path_to_your_output_directory} +``` + +### Predict On Image URL + +```bash +inference infer -i https://[YOUR_HOSTED_IMAGE_URL] -m {your_project}/{version} --api-key {YOUR_API_KEY} +``` + +### Using Hosted API + +```bash +inference infer -i ./image.jpg -m {your_project}/{version} --api-key {YOUR_API_KEY} -h https://detect.roboflow.com +``` + +### Predict From Local Directory + +```bash +inference infer -i {your_directory_with_images} -m {your_project}/{version} -o {path_to_your_output_directory} --api-key {YOUR_API_KEY} +``` + +### Predict On Video File + +```bash +inference infer -i {path_to_your_video_file} -m {your_project}/{version} -o {path_to_your_output_directory} --api-key {YOUR_API_KEY} +``` + +### Configure The Visualization + +Option `-c` can be provided with a path to `*.yml` file configuring `supervision` visualisation. +There are few pre-defined configs: +- `bounding_boxes` - with `BoxAnnotator` and `LabelAnnotator` annotators +- `bounding_boxes_tracing` - with `ByteTracker` and annotators (`BoxAnnotator`, `LabelAnnotator`) +- `masks` - with `MaskAnnotator` and `LabelAnnotator` annotators +- `polygons` - with `PolygonAnnotator` and `LabelAnnotator` annotators + +Custom configuration can be created following the schema: +```yaml +annotators: + - type: "bounding_box" + params: + thickness: 2 + - type: "label" + params: + text_scale: 0.5 + text_thickness: 2 + text_padding: 5 + - type: "trace" + params: + trace_length: 60 + thickness: 2 +tracking: + track_activation_threshold: 0.25 + lost_track_buffer: 30 + minimum_matching_threshold: 0.8 + frame_rate: 30 +``` +`annotators` field is a list of dictionaries with two keys: `type` and `param`. `type` points to +name of annotator class: +```python +from supervision import * +ANNOTATOR_TYPE2CLASS = { + "bounding_box": BoxAnnotator, + "box": BoxAnnotator, + "mask": MaskAnnotator, + "polygon": PolygonAnnotator, + "color": ColorAnnotator, + "halo": HaloAnnotator, + "ellipse": EllipseAnnotator, + "box_corner": BoxCornerAnnotator, + "circle": CircleAnnotator, + "dot": DotAnnotator, + "label": LabelAnnotator, + "blur": BlurAnnotator, + "trace": TraceAnnotator, + "heat_map": HeatMapAnnotator, + "pixelate": PixelateAnnotator, + "triangle": TriangleAnnotator, +} +``` +`param` is a dictionary of annotator constructor parameters (check them in +[`supervision`](https://github.com/roboflow/supervision) docs - you would only be able +to use primitive values, classes and enums that are defined in constructors may not be possible +to resolve from yaml config). + +`tracking` is an optional key that holds a dictionary with constructor parameters for +`ByteTrack`. + +### Provide Inference Hyperparameters + +`-mc` parameter can be provided with path to `*.yml` file that specifies +model configuration (like confidence threshold or IoU threshold). If given, +configuration will be used to initialise `InferenceConfiguration` object +from `inference_sdk` library. See [sdk docs](../inference_sdk.md) to discover +which options can be configured via `*.yml` file - configuration keys must match +with names of fields in `InferenceConfiguration` object. diff --git a/docs/inference_helpers/cli_commands/server.md b/docs/inference_helpers/cli_commands/server.md new file mode 100644 index 000000000..ee83da89c --- /dev/null +++ b/docs/inference_helpers/cli_commands/server.md @@ -0,0 +1,68 @@ +# Controlling `inference` server + +`inference server` command provides a control layer around HTTP server exposing `inference`. + +!!! Tip "Discovering command capabilities" + + To check detail of the command, run: + + ```bash + inference server --help + ``` + + Additionally, help guide is also available for each sub-command: + + ```bash + inference server start --help + ``` + +## `inference server start` + +Starts a local Inference server. It optionally takes a port number (default is 9001) and will only start the docker container if there is not already a container running on that port. + +If you would rather run your server on a virtual machine in Google cloud or Amazon cloud, skip to the section titled "Deploy Inference on Cloud" below. + +Before you begin, ensure that you have Docker installed on your machine. Docker provides a containerized environment, +allowing the Roboflow Inference Server to run in a consistent and isolated manner, regardless of the host system. If +you haven't installed Docker yet, you can get it from Docker's official website. + +The CLI will automatically detect the device you are running on and pull the appropriate Docker image. + +```bash +inference server start --port 9001 [-e {optional_path_to_file_with_env_variables}] +``` + +Parameter `--env-file` (or `-e`) is the optional path for .env file that will be loaded into your Inference server +in case that values of internal parameters needs to be adjusted. Any value passed explicitly as command parameter +is considered as more important and will shadow the value defined in `.env` file under the same target variable name. + + +### Development Mode + +Use the `--dev` flag to start the Inference Server in development mode. Development mode enables the Inference Server's built in notebook environment for easy testing and development. + +### Tunnel + +Use the `--tunnel` flag to start the Inference Server with a tunnel to expose inference to external requests on a TLS-enabled endpoint. + +The random generated address will be on server start output: + +``` +Tunnel to local inference running on https://somethingrandom-ip-192-168-0-1.roboflow.run +``` + +## inference server status + +Checks the status of the local inference server. + +```bash +inference server status +``` + +## inference server stop + +Stops the inference server. + +```bash +inference server stop +``` \ No newline at end of file diff --git a/docs/inference_helpers/cli_commands/workflows.md b/docs/inference_helpers/cli_commands/workflows.md new file mode 100644 index 000000000..3db6f219f --- /dev/null +++ b/docs/inference_helpers/cli_commands/workflows.md @@ -0,0 +1,123 @@ +# Processing data with Workflows + +`inference workflows` command provides a way to process images and videos with Workflow. It is possible to +process: + +* individual images + +* directories of images + +* video files + + +!!! Tip "Discovering command capabilities" + + To check detail of the command, run: + + ```bash + inference workflows --help + ``` + + Additionally, help guide is also available for each sub-command: + + ```bash + inference workflows process-image --help + ``` + + +## Process individual image + +Basic usage of the command is illustrated below: + +```bash +inference workflows process-image \ + --image_path {your-input-image} \ + --output_dir {your-output-dir} \ + --workspace_name {your-roboflow-workspace-url} \ + --workflow_id {your-workflow-id} \ + --api-key {your_roboflow_api_key} +``` + +which would take your input image, run it against your Workflow and save results in output directory. By default, +Workflow will be processed using Roboflow Hosted API. You can tweak behaviour of the command: + +* if you want to process the image locally, using `inference` Python package - use +`--processing_target inference_package` option (*requires `inference` to be installed*) + +* to see all options, use `inference workflows process-image --help` command + +* any option that starts from `--` which is not enlisted in `--help` command will be treated as input parameter +to the workflow execution - with automatic type conversion applied. Additionally `--workflow_params` option may +specify path to `*.json` file providing workflow parameters (explicit parameters will override parameters defined +in file) + +* if your Workflow defines image parameter placeholder under a name different from `image`, you can point the +proper image input by `--image_input_name` + +* `--allow_override` flag must be used if output directory is not empty + + +## Process directory of images + +Basic usage of the command is illustrated below: + +```bash +inference workflows process-images-directory \ + -i {your_input_directory} \ + -o {your_output_directory} \ + --workspace_name {your-roboflow-workspace-url} \ + --workflow_id {your-workflow-id} \ + --api-key {your_roboflow_api_key} +``` + +You can tweak behaviour of the command: + +* if you want to process the image locally, using `inference` Python package - use +`--processing_target inference_package` option (*requires `inference` to be installed*) + +* to see all options, use `inference workflows process-image --help` command + +* any option that starts from `--` which is not enlisted in `--help` command will be treated as input parameter +to the workflow execution - with automatic type conversion applied. Additionally `--workflow_params` option may +specify path to `*.json` file providing workflow parameters (explicit parameters will override parameters defined +in file) + +* if your Workflow defines image parameter placeholder under a name different from `image`, you can point the +proper image input by `--image_input_name` + +* `--allow_override` flag must be used if output directory is not empty + +* `--threads` option can specify number of threads used to run the requests when processing target is API + +## Process video file + +!!! Note "`inference` required" + + This command requires `inference` to be installed. + +Basic usage of the command is illustrated below: + +```bash +inference workflows process-video \ + --video_path {video_to_be_processed} \ + --output_dir {empty_directory} \ + --workspace_name {your-roboflow-workspace-url} \ + --workflow_id {your-workflow-id} \ + --api-key {your_roboflow_api_key} +``` + +You can tweak behaviour of the command: + +* `--max_fps` option can be used to subsample video frames while processing + +* to see all options, use `inference workflows process-image --help` command + +* any option that starts from `--` which is not enlisted in `--help` command will be treated as input parameter +to the workflow execution - with automatic type conversion applied. Additionally `--workflow_params` option may +specify path to `*.json` file providing workflow parameters (explicit parameters will override parameters defined +in file) + +* if your Workflow defines image parameter placeholder under a name different from `image`, you can point the +proper image input by `--image_input_name` + +* `--allow_override` flag must be used if output directory is not empty diff --git a/docs/inference_helpers/inference_cli.md b/docs/inference_helpers/inference_cli.md index 809e8b60c..bd1e5dfac 100644 --- a/docs/inference_helpers/inference_cli.md +++ b/docs/inference_helpers/inference_cli.md @@ -7,443 +7,27 @@ ## Roboflow Inference CLI -Roboflow Inference CLI offers a lightweight interface for running the Roboflow inference server locally or the Roboflow Hosted API. +Roboflow Inference CLI is command-line interface for `inference` ecosystem, providing an easy way to: -To create custom Inference server Docker images, go to the parent package, Roboflow Inference. +* run and manage [`inference` server](./cli_commands/server.md) locally -Roboflow has everything you need to deploy a computer vision model to a range of devices and environments. Inference supports object detection, classification, and instance segmentation models, and running foundation models (CLIP and SAM). +* process data with [Workflows](/workflows/about/) -### Installation - -```bash -pip install roboflow-cli -``` - -## Examples - -### inference server start - -Starts a local Inference server. It optionally takes a port number (default is 9001) and will only start the docker container if there is not already a container running on that port. - -If you would rather run your server on a virtual machine in Google cloud or Amazon cloud, skip to the section titled "Deploy Inference on Cloud" below. - -Before you begin, ensure that you have Docker installed on your machine. Docker provides a containerized environment, -allowing the Roboflow Inference Server to run in a consistent and isolated manner, regardless of the host system. If -you haven't installed Docker yet, you can get it from Docker's official website. - -The CLI will automatically detect the device you are running on and pull the appropriate Docker image. - -```bash -inference server start --port 9001 [-e {optional_path_to_file_with_env_variables}] -``` - -Parameter `--env-file` (or `-e`) is the optional path for .env file that will be loaded into your Inference server -in case that values of internal parameters needs to be adjusted. Any value passed explicitly as command parameter -is considered as more important and will shadow the value defined in `.env` file under the same target variable name. - -#### Development Mode - -Use the `--dev` flag to start the Inference Server in development mode. Development mode enables the Inference Server's built in notebook environment for easy testing and development. - -#### Tunnel - -Use the `--tunnel` flag to start the Inference Server with a tunnel to expose inference to external requests on a TLS-enabled endpoint. - -The random generated address will be on server start output: - -``` -Tunnel to local inference running on https://somethingrandom-ip-192-168-0-1.roboflow.run -``` - -### inference server status - -Checks the status of the local inference server. - -```bash -inference server status -``` - -### inference server stop - -Stops the inference server. - -```bash -inference server stop -``` - -## Deploy Inference on a Cloud VM - -You can deploy Roboflow Inference containers to virtual machines in the cloud. These VMs are configured to run CPU or GPU-based Inference servers under the hood, so you don't have to deal with OS/GPU drivers/docker installations, etc! The Inference cli currently supports deploying the Roboflow Inference container images into a virtual machine running on Google (GCP) or Amazon cloud (AWS). - -The Roboflow Inference CLI assumes the corresponding cloud CLI is configured for the project you want to deploy the virtual machine into. Read instructions for setting up [Google/GCP - gcloud cli](https://cloud.google.com/sdk/docs/install) or the [Amazon/AWS aws cli](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html). - -Roboflow Inference cloud deploy is powered by the popular [Skypilot project](https://github.com/skypilot-org/skypilot). - -### Important: Cloud Deploy Installation - -Before using cloud deploy, optional dependencies must be installed: - -```bash -# Install dependencies required for cloud deploy -pip install inference[cloud-deploy] -``` - - -### Cloud Deploy Examples - -We illustrate Inference cloud deploy with some examples, below. +* [benchmark](./cli_commands/benchmark.md) `inference` performance -*Deploy GPU or CPU inference to AWS or GCP* - -```bash -# Deploy the roboflow Inference GPU container into a GPU-enabled VM in AWS - -inference cloud deploy --provider aws --compute-type gpu -``` - -```bash -# Deploy the roboflow Inference CPU container into a CPU-only VM in GCP - -inference cloud deploy --provider gcp --compute-type cpu - -``` - -Note the "cluster name" printed after the deployment completes. This handle is used in many subsequent commands. -The deploy command also prints helpful debug and cost information about your VM. - -Deploying Inference into a cloud VM will also print out an endpoint of the form "http://1.2.3.4:9001"; you can now run inferences against this endpoint. - -Note that the port 9001 is automatically opened - check with your security admin if this is acceptable for your cloud/project. - -### View status of deployments - -```bash -inference cloud status -``` - -### Stop and start deployments - -```bash -# Stop the VM, you only pay for disk storage while the VM is stopped -inference cloud stop - -``` - -### Restart deployments - -```bash -inference cloud start -``` - -### Undeploy (delete) the cloud deployment - -```bash -inference cloud undeploy -``` - -### SSH into the cloud deployment - -You can SSH into your cloud deployment with the following command: -```bash -ssh -``` - -The required SSH key is automatically added to your .ssh/config, you don't need to configure this manually. - -### Cloud Deploy Customization - -Roboflow Inference cloud deploy will create VMs based on internally tested templates. - -For advanced usecases and to customize the template, you can use your [sky yaml](https://skypilot.readthedocs.io/en/latest/reference/yaml-spec.html) template on the command-line, like so: - -```bash -inference cloud deploy --custom /path/to/sky-template.yaml -``` - -If you want you can download the standard template stored in the roboflow cli and the modify it for your needs, this command will do that. - -```bash -# This command will print out the standard gcp/cpu sky template. -inference cloud deploy --dry-run --provider gcp --compute-type cpu -``` +* make [predictions](./cli_commands/infer.md) from your models -Then you can deploy a custom template based off your changes. +* deploy `inference` server in [cloud](./cli_commands/cloud.md) -As an aside, you can also use the [sky cli](https://skypilot.readthedocs.io/en/latest/reference/cli.html) to control your deployment(s) and access some more advanced functionality. - -Roboflow Inference deploy currently supports AWS and GCP, please open an issue on the [Inference GitHub repository](https://github.com/roboflow/inference/issues) if you would like to see other cloud providers supported. - - -### inference infer - -It takes input path / url and model version to produce predictions (and optionally make visualisation using -`supervision`). You can also specify a host to run inference on our hosted inference server. - -!!! note - - If you decided to use hosted inference server - make sure command `inference server start` was used first - -!!! tip - - Use `inference infer --help` to display description of parameters - -!!! tip - - Roboflow API key can be provided via `ROBOFLOW_API_KEY` environment variable - -#### Local image - -This command is going to make a prediction from local image using selected model and print the prediction on -the console. - -```bash -inference infer -i ./image.jpg -m {your_project}/{version} --api-key {YOUR_API_KEY} -``` - -To display visualised prediction use `-D` option. To save prediction and visualisation in a local directory, -use `-o {path_to_your_directory}` option. Those options work also in other modes. - -```bash -inference infer -i ./image.jpg -m {your_project}/{version} --api-key {YOUR_API_KEY} -D -o {path_to_your_output_directory} -``` - -#### Hosted image - -```bash -inference infer -i https://[YOUR_HOSTED_IMAGE_URL] -m {your_project}/{version} --api-key {YOUR_API_KEY} -``` - -#### Hosted API inference - -```bash -inference infer -i ./image.jpg -m {your_project}/{version} --api-key {YOUR_API_KEY} -h https://detect.roboflow.com -``` - -#### Local directory - -```bash -inference infer -i {your_directory_with_images} -m {your_project}/{version} -o {path_to_your_output_directory} --api-key {YOUR_API_KEY} -``` - -#### Video file - -```bash -inference infer -i {path_to_your_video_file} -m {your_project}/{version} -o {path_to_your_output_directory} --api-key {YOUR_API_KEY} -``` - -#### Configuration of visualisation -Option `-c` can be provided with a path to `*.yml` file configuring `supervision` visualisation. -There are few pre-defined configs: -- `bounding_boxes` - with `BoxAnnotator` and `LabelAnnotator` annotators -- `bounding_boxes_tracing` - with `ByteTracker` and annotators (`BoxAnnotator`, `LabelAnnotator`) -- `masks` - with `MaskAnnotator` and `LabelAnnotator` annotators -- `polygons` - with `PolygonAnnotator` and `LabelAnnotator` annotators - -Custom configuration can be created following the schema: -```yaml -annotators: - - type: "bounding_box" - params: - thickness: 2 - - type: "label" - params: - text_scale: 0.5 - text_thickness: 2 - text_padding: 5 - - type: "trace" - params: - trace_length: 60 - thickness: 2 -tracking: - track_activation_threshold: 0.25 - lost_track_buffer: 30 - minimum_matching_threshold: 0.8 - frame_rate: 30 -``` -`annotators` field is a list of dictionaries with two keys: `type` and `param`. `type` points to -name of annotator class: -```python -from supervision import * -ANNOTATOR_TYPE2CLASS = { - "bounding_box": BoxAnnotator, - "box": BoxAnnotator, - "mask": MaskAnnotator, - "polygon": PolygonAnnotator, - "color": ColorAnnotator, - "halo": HaloAnnotator, - "ellipse": EllipseAnnotator, - "box_corner": BoxCornerAnnotator, - "circle": CircleAnnotator, - "dot": DotAnnotator, - "label": LabelAnnotator, - "blur": BlurAnnotator, - "trace": TraceAnnotator, - "heat_map": HeatMapAnnotator, - "pixelate": PixelateAnnotator, - "triangle": TriangleAnnotator, -} -``` -`param` is a dictionary of annotator constructor parameters (check them in -[`supervision`](https://github.com/roboflow/supervision) docs - you would only be able -to use primitive values, classes and enums that are defined in constructors may not be possible -to resolve from yaml config). - -`tracking` is an optional key that holds a dictionary with constructor parameters for -`ByteTrack`. - -#### Configuration of model -`-mc` parameter can be provided with path to `*.yml` file that specifies -model configuration (like confidence threshold or IoU threshold). If given, -configuration will be used to initialise `InferenceConfiguration` object -from `inference_sdk` library. See [sdk docs](./inference_sdk.md) to discover -which options can be configured via `*.yml` file - configuration keys must match -with names of fields in `InferenceConfiguration` object. - -### inference benchmark - -!!! note - - The command is introduced in `inference_cli>=0.9.10` - -`inference benchmark` is a set of command suited to run benchmarks of `inference`. There are two types of benchmark -available `inference benchmark api-speed` - to test `inference` HTTP server and `inference benchmark python-package-speed` -to verify the performance of `inference` Python package. - -!!! tip - - Use `inference benchmark api-speed --help` / `inference benchmark python-package-speed --help` to - display all options of benchmark commands. - -!!! tip - - Roboflow API key can be provided via `ROBOFLOW_API_KEY` environment variable - -#### Running benchmark of Python package - -Basic benchmark can be run using the following command: - -```bash -inference benchmark python-package-speed \ - -m {your_model_id} \ - -d {pre-configured dataset name or path to directory with images} \ - -o {output_directory} -``` -Command runs specified number of inferences using pointed model and saves statistics (including benchmark -parameter, throughput, latency, errors and platform details) in pointed directory. - -#### Running benchmark of `inference server` - -!!! note - - Before running API benchmark - make sure the server is up and running: - ```bash - inference server start - ``` -Basic benchmark can be run using the following command: - -```bash -inference benchmark api-speed \ - -m {your_model_id} \ - -d {pre-configured dataset name or path to directory with images} \ - -o {output_directory} -``` -Command runs specified number of inferences using pointed model and saves statistics (including benchmark -parameter, throughput, latency, errors and platform details) in pointed directory. - -This benchmark has more configuration options to support different ways HTTP API profiling. In default mode, -single client will be spawned, and it will send one request after another sequentially. This may be suboptimal -in specific cases, so one may specify number of concurrent clients using `-c {number_of_clients}` option. -Each client will send next request once previous is handled. This option will also not cover all scenarios -of tests. For instance one may want to send `x` requests each second (which is closer to the scenario of -production environment where multiple clients are sending requests concurrently). In this scenario, `--rps {value}` -option can be used (and `-c` will be ignored). Value provided in `--rps` option specifies how many requests -are to be spawned **each second** without waiting for previous requests to be handled. In I/O intensive benchmark -scenarios - we suggest running command from multiple separate processes and possibly multiple hosts. - -### inference workflows - -In release `0.29.0`, `inference-cli` was extended with command to process data using Workflows. It is possible to -process: - -* individual images - -* directories of images - -* video files - -#### Processing individual image - -Basic usage of the command is illustrated below: - -```bash -inference workflows process-image \ - --image_path {your-input-image} \ - --output_dir {your-output-dir} \ - --workspace_name {your-roboflow-workspace-url} \ - --workflow_id {your-workflow-id} \ - --api-key {your_roboflow_api_key} -``` - -which would take your input image, run it against your Workflow and save results in output directory. By default, -Workflow will be processed using Roboflow Hosted API. You can tweak behaviour of the command: - -* if you want to process the image locally, using `inference` Python package - use -`--processing_target inference_package` option (*requires `inference` to be installed*) - -* to see all options, use `inference workflows process-image --help` command - -* any option that starts from `--` which is not enlisted in `--help` command will be treated as input parameter -to the workflow execution - - -#### Processing directory of images - -Basic usage of the command is illustrated below: - -```bash -inference workflows process-images-directory \ - -i {your_input_directory} \ - -o {your_output_directory} \ - --workspace_name {your-roboflow-workspace-url} \ - --workflow_id {your-workflow-id} \ - --api-key {your_roboflow_api_key} -``` - -You can tweak behaviour of the command: - -* if you want to process the image locally, using `inference` Python package - use -`--processing_target inference_package` option (*requires `inference` to be installed*) - -* to see all options, use `inference workflows process-image --help` command - -* any option that starts from `--` which is not enlisted in `--help` command will be treated as input parameter -to the workflow execution - -#### Processing video file - -!!! Note "`inference` required" - - This command requires `inference` to be installed. - -Basic usage of the command is illustrated below: +### Installation ```bash -inference workflows process-video \ - --video_path {video_to_be_processed} \ - --output_dir {empty_directory} \ - --workspace_name {your-roboflow-workspace-url} \ - --workflow_id {your-workflow-id} \ - --api-key {your_roboflow_api_key} +pip install roboflow-cli ``` -You can tweak behaviour of the command: - -* `--max_fps` option can be used to subsample video frames while processing - -* to see all options, use `inference workflows process-image --help` command +!!! Tip "`inference-cli` is part of `inference`" -* any option that starts from `--` which is not enlisted in `--help` command will be treated as input parameter -to the workflow execution + If you have installed `inference` Python package, the CLI extensions is already included. ## Supported Devices diff --git a/inference_cli/workflows.py b/inference_cli/workflows.py index 38e44780a..03ddafcd4 100644 --- a/inference_cli/workflows.py +++ b/inference_cli/workflows.py @@ -276,25 +276,7 @@ def process_image( help="Flag enabling errors stack traces to be displayed (helpful for debugging)", ), ] = False, - proceed_automatically: Annotated[ - bool, - typer.Option( - "--yes/--no", - "-y/-n", - help="Boolean flag to decide on auto `yes` answer given on user input required.", - ), - ] = False, ) -> None: - if ( - processing_target is ProcessingTarget.API - and "roboflow.com" in api_url - and not proceed_automatically - ): - proceed = input( - "This action may easily exceed your Roboflow inference credits. Are you sure? [y/N] " - ) - if proceed.lower() != "y": - return None try: ensure_target_directory_is_empty( output_directory=output_directory, @@ -474,25 +456,7 @@ def process_images_directory( help="Flag enabling errors stack traces to be displayed (helpful for debugging)", ), ] = False, - proceed_automatically: Annotated[ - bool, - typer.Option( - "--yes/--no", - "-y/-n", - help="Boolean flag to decide on auto `yes` answer given on user input required.", - ), - ] = False, ) -> None: - if ( - processing_target is ProcessingTarget.API - and "roboflow.com" in api_url - and not proceed_automatically - ): - proceed = input( - "This action may easily exceed your Roboflow inference credits. Are you sure? [y/N] " - ) - if proceed.lower() != "y": - return None try: ensure_target_directory_is_empty( output_directory=output_directory, diff --git a/mkdocs.yml b/mkdocs.yml index 794a647b0..5d8f69cd8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -109,7 +109,13 @@ nav: - Stream Management API: enterprise/stream_management_api.md - Inference Helpers: - Inference Landing Page: inference_helpers/inference_landing_page.md - - Inference CLI: inference_helpers/inference_cli.md + - Inference CLI: + - About CLI: inference_helpers/inference_cli.md + - Control The Server: inference_helpers/cli_commands/server.md + - Run Workflows: inference_helpers/cli_commands/workflows.md + - Benchmark Inference: inference_helpers/cli_commands/benchmark.md + - Make Predictions: inference_helpers/cli_commands/infer.md + - Deploy To Cloud: inference_helpers/cli_commands/cloud.md - Inference SDK: inference_helpers/inference_sdk.md - inference configuration: - Environmental variables: server_configuration/environmental_variables.md diff --git a/tests/inference_cli/integration_tests/test_workflows.py b/tests/inference_cli/integration_tests/test_workflows.py index fa2d9a2a6..440431d56 100644 --- a/tests/inference_cli/integration_tests/test_workflows.py +++ b/tests/inference_cli/integration_tests/test_workflows.py @@ -34,7 +34,6 @@ def test_processing_image_with_hosted_api( f"--workflow_id prod-test-workflow " f"--api-key {INFERENCE_CLI_TESTS_API_KEY} " f"--model_id yolov8n-640 " - f"--yes" ).split() new_process_env = deepcopy(os.environ) new_process_env["ALLOW_INTERACTIVE_INFERENCE_INSTALLATION"] = "False" @@ -76,7 +75,6 @@ def test_processing_images_directory_with_hosted_api( f"--workflow_id prod-test-workflow " f"--api-key {INFERENCE_CLI_TESTS_API_KEY} " f"--model_id yolov8n-640 " - f"--yes" ).split() new_process_env = deepcopy(os.environ) new_process_env["ALLOW_INTERACTIVE_INFERENCE_INSTALLATION"] = "False"