From a1c9ebb75df6e54def52a233f3003050577c3a68 Mon Sep 17 00:00:00 2001 From: SundarRajan28 Date: Thu, 17 Oct 2024 08:17:16 +0000 Subject: [PATCH 1/2] Adding sharding info changes for readers --- rocAL_pybind/amd/rocal/fn.py | 9 +++++++-- rocAL_pybind/amd/rocal/readers.py | 31 ++++++++++++++++++++++--------- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/rocAL_pybind/amd/rocal/fn.py b/rocAL_pybind/amd/rocal/fn.py index 0bbfab79..24f1ef5b 100644 --- a/rocAL_pybind/amd/rocal/fn.py +++ b/rocAL_pybind/amd/rocal/fn.py @@ -1056,7 +1056,7 @@ def box_iou_matcher(*inputs, anchors, high_threshold=0.5, return (box_iou_matcher, []) -def external_source(source, device=None, color_format=types.RGB, random_shuffle=False, mode=types.EXTSOURCE_FNAME, max_width=2000, max_height=2000, last_batch_policy=types.LAST_BATCH_FILL, last_batch_padded=True): +def external_source(source, device=None, color_format=types.RGB, random_shuffle=False, mode=types.EXTSOURCE_FNAME, max_width=2000, max_height=2000, last_batch_policy=types.LAST_BATCH_FILL, pad_last_batch_repeated=False, stick_to_shard=True, shard_size=-1): """ External Source Reader - User can pass a iterator or callable source. @param source (iterator or callable) The source iterator or callable object. @@ -1073,8 +1073,13 @@ def external_source(source, device=None, color_format=types.RGB, random_shuffle= Pipeline._current_pipeline._external_source_mode = mode Pipeline._current_pipeline._external_source_user_given_width = max_width Pipeline._current_pipeline._external_source_user_given_height = max_height + RocalShardingInfo = b.RocalShardingInfo() + RocalShardingInfo.last_batch_policy = last_batch_policy + RocalShardingInfo.pad_last_batch_repeated = pad_last_batch_repeated + RocalShardingInfo.stick_to_shard = stick_to_shard + RocalShardingInfo.shard_size = shard_size kwargs_pybind = {"rocal_color_format": color_format, "is_output": False, "shuffle": random_shuffle, "loop": False, "decode_size_policy": types.USER_GIVEN_SIZE, - "max_width": max_width, "max_height": max_height, "dec_type": types.DECODER_TJPEG, "external_source_mode": mode, "last_batch_info": (last_batch_policy, last_batch_padded)} + "max_width": max_width, "max_height": max_height, "dec_type": types.DECODER_TJPEG, "external_source_mode": mode, "sharding_info": RocalShardingInfo} external_source_operator = b.externalFileSource( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) return (external_source_operator, []) # Labels is Empty diff --git a/rocAL_pybind/amd/rocal/readers.py b/rocAL_pybind/amd/rocal/readers.py index 23dd4606..5fdce737 100644 --- a/rocAL_pybind/amd/rocal/readers.py +++ b/rocAL_pybind/amd/rocal/readers.py @@ -189,7 +189,7 @@ def caffe2(path, bbox=False, stick_to_shard=False, pad_last_batch=False): def video(sequence_length, file_list_frame_num=False, file_root="", image_type=types.RGB, num_shards=1, random_shuffle=False, step=1, stride=1, decoder_mode=types.SOFTWARE_DECODE, enable_frame_num=False, enable_timestamps=False, file_list="", stick_to_shard=False, pad_last_batch=False, - file_list_include_preceding_frame=False, normalized=False, skip_vfr_check=False, last_batch_policy=types.LAST_BATCH_FILL, last_batch_padded=True): + file_list_include_preceding_frame=False, normalized=False, skip_vfr_check=False, last_batch_policy=types.LAST_BATCH_FILL, pad_last_batch_repeated=False, shard_size=-1): """!Creates a VideoDecoder node for loading video sequences. @param sequence_length Number of frames in video sequence. @@ -223,7 +223,11 @@ def video(sequence_length, file_list_frame_num=False, file_root="", image_type=t "file_list_frame_num": file_list_frame_num} # VideoMetaDataReader b.videoMetaDataReader(Pipeline._current_pipeline._handle, *(kwargs_pybind_reader.values())) - + RocalShardingInfo = b.RocalShardingInfo() + RocalShardingInfo.last_batch_policy = last_batch_policy + RocalShardingInfo.pad_last_batch_repeated = pad_last_batch_repeated + RocalShardingInfo.stick_to_shard = stick_to_shard + RocalShardingInfo.shard_size = shard_size kwargs_pybind_decoder = { "source_path": file_root, "color_format": image_type, @@ -236,7 +240,7 @@ def video(sequence_length, file_list_frame_num=False, file_root="", image_type=t "frame_step": step, "frame_stride": stride, "file_list_frame_num": file_list_frame_num, - "last_batch_info": (last_batch_policy, last_batch_padded)} # VideoDecoder + "sharding_info": RocalShardingInfo} # VideoDecoder videos = b.videoDecoder( Pipeline._current_pipeline._handle, *(kwargs_pybind_decoder.values())) return (videos) @@ -248,9 +252,9 @@ def video_resize(sequence_length, resize_width, resize_height, file_list_frame_n stride=3, decoder_mode=types.SOFTWARE_DECODE, scaling_mode=types.SCALING_MODE_DEFAULT, interpolation_type=types.LINEAR_INTERPOLATION, resize_longer=0, resize_shorter=0, max_size=[], enable_frame_num=False, - enable_timestamps=False, file_list="", stick_to_shard=False, pad_last_batch=False, + enable_timestamps=False, file_list="", stick_to_shard=True, pad_last_batch=False, file_list_include_preceding_frame=False, normalized=False, skip_vfr_check=False, - last_batch_policy=types.LAST_BATCH_FILL, last_batch_padded=True): + last_batch_policy=types.LAST_BATCH_FILL, pad_last_batch_repeated=False, shard_size=-1): """!Creates a VideoDecoderResize node in the pipeline for loading and resizing video sequences. @param sequence_length Number of frames in video sequence. @@ -291,18 +295,22 @@ def video_resize(sequence_length, resize_width, resize_height, file_list_frame_n "file_list_frame_num": file_list_frame_num} # VideoMetaDataReader meta_data = b.videoMetaDataReader( Pipeline._current_pipeline._handle, *(kwargs_pybind_reader.values())) - + RocalShardingInfo = b.RocalShardingInfo() + RocalShardingInfo.last_batch_policy = last_batch_policy + RocalShardingInfo.pad_last_batch_repeated = pad_last_batch_repeated + RocalShardingInfo.stick_to_shard = stick_to_shard + RocalShardingInfo.shard_size = shard_size kwargs_pybind_decoder = {"source_path": file_root, "color_format": image_type, "decoder_mode": decoder_mode, "shard_count": num_shards, "sequence_length": sequence_length, "resize_width": resize_width, "resize_height": resize_height, "shuffle": random_shuffle, "is_output": False, "loop": False, "frame_step": step, "frame_stride": stride, "file_list_frame_num": file_list_frame_num, "scaling_mode": scaling_mode, "max_size": max_size, - "resize_shorter": resize_shorter, "resize_longer": resize_longer, "interpolation_type": interpolation_type, "last_batch_info": (last_batch_policy, last_batch_padded)} + "resize_shorter": resize_shorter, "resize_longer": resize_longer, "interpolation_type": interpolation_type, "sharding_info": RocalShardingInfo} videos = b.videoDecoderResize( Pipeline._current_pipeline._handle, *(kwargs_pybind_decoder.values())) return (videos, meta_data) -def sequence_reader(file_root, sequence_length, image_type=types.RGB, num_shards=1, random_shuffle=False, step=3, stride=1, stick_to_shard=False, pad_last_batch=False, last_batch_policy=types.LAST_BATCH_FILL, last_batch_padded=True): +def sequence_reader(file_root, sequence_length, image_type=types.RGB, num_shards=1, random_shuffle=False, step=3, stride=1, stick_to_shard=False, last_batch_policy=types.LAST_BATCH_FILL, pad_last_batch_repeated=False, shard_size=-1): """!Creates a SequenceReader node for loading image sequences. @param file_root Root directory containing image sequences. @@ -318,6 +326,11 @@ def sequence_reader(file_root, sequence_length, image_type=types.RGB, num_shards @return list of loaded image sequences. """ Pipeline._current_pipeline._reader = "SequenceReader" + RocalShardingInfo = b.RocalShardingInfo() + RocalShardingInfo.last_batch_policy = last_batch_policy + RocalShardingInfo.pad_last_batch_repeated = pad_last_batch_repeated + RocalShardingInfo.stick_to_shard = stick_to_shard + RocalShardingInfo.shard_size = shard_size # Output kwargs_pybind = { "source_path": file_root, @@ -329,7 +342,7 @@ def sequence_reader(file_root, sequence_length, image_type=types.RGB, num_shards "loop": False, "frame_step": step, "frame_stride": stride, - "last_batch_info": (last_batch_policy, last_batch_padded)} + "sharding_info": RocalShardingInfo} frames = b.sequenceReader( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) return (frames) From 7f90251822fbe4686021154f6ade1ce27618555a Mon Sep 17 00:00:00 2001 From: SundarRajan28 Date: Thu, 17 Oct 2024 08:22:50 +0000 Subject: [PATCH 2/2] Adding absolute path for images dataset --- tests/python_api/decoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python_api/decoder.py b/tests/python_api/decoder.py index 3e24ceef..31f7e4a1 100644 --- a/tests/python_api/decoder.py +++ b/tests/python_api/decoder.py @@ -8,7 +8,7 @@ import matplotlib.pyplot as plt seed = 1549361629 -image_dir = "../../data/images/AMD-tinyDataSet/" +image_dir = "/opt/rocm/share/rocal/test/data/images/AMD-tinyDataSet" batch_size = 4 gpu_id = 0