Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix errors in python tests #224

Open
wants to merge 2 commits into
base: develop
Choose a base branch
from
+30 −12
Open

Fix errors in python tests #224

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a1c9ebb
Adding sharding info changes for readers
SundarRajan28 Oct 17, 2024
7f90251
Adding absolute path for images dataset
SundarRajan28 Oct 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions rocAL_pybind/amd/rocal/fn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1056,7 +1056,7 @@ def box_iou_matcher(*inputs, anchors, high_threshold=0.5,
return (box_iou_matcher, [])


def external_source(source, device=None, color_format=types.RGB, random_shuffle=False, mode=types.EXTSOURCE_FNAME, max_width=2000, max_height=2000, last_batch_policy=types.LAST_BATCH_FILL, last_batch_padded=True):
def external_source(source, device=None, color_format=types.RGB, random_shuffle=False, mode=types.EXTSOURCE_FNAME, max_width=2000, max_height=2000, last_batch_policy=types.LAST_BATCH_FILL, pad_last_batch_repeated=False, stick_to_shard=True, shard_size=-1):
"""
External Source Reader - User can pass a iterator or callable source.
@param source (iterator or callable) The source iterator or callable object.
Expand All @@ -1073,8 +1073,13 @@ def external_source(source, device=None, color_format=types.RGB, random_shuffle=
Pipeline._current_pipeline._external_source_mode = mode
Pipeline._current_pipeline._external_source_user_given_width = max_width
Pipeline._current_pipeline._external_source_user_given_height = max_height
RocalShardingInfo = b.RocalShardingInfo()
RocalShardingInfo.last_batch_policy = last_batch_policy
RocalShardingInfo.pad_last_batch_repeated = pad_last_batch_repeated
RocalShardingInfo.stick_to_shard = stick_to_shard
RocalShardingInfo.shard_size = shard_size
kwargs_pybind = {"rocal_color_format": color_format, "is_output": False, "shuffle": random_shuffle, "loop": False, "decode_size_policy": types.USER_GIVEN_SIZE,
"max_width": max_width, "max_height": max_height, "dec_type": types.DECODER_TJPEG, "external_source_mode": mode, "last_batch_info": (last_batch_policy, last_batch_padded)}
"max_width": max_width, "max_height": max_height, "dec_type": types.DECODER_TJPEG, "external_source_mode": mode, "sharding_info": RocalShardingInfo}
external_source_operator = b.externalFileSource(
Pipeline._current_pipeline._handle, *(kwargs_pybind.values()))
return (external_source_operator, []) # Labels is Empty
Expand Down
31 changes: 22 additions & 9 deletions rocAL_pybind/amd/rocal/readers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ def caffe2(path, bbox=False, stick_to_shard=False, pad_last_batch=False):
def video(sequence_length, file_list_frame_num=False, file_root="", image_type=types.RGB, num_shards=1,
random_shuffle=False, step=1, stride=1, decoder_mode=types.SOFTWARE_DECODE, enable_frame_num=False,
enable_timestamps=False, file_list="", stick_to_shard=False, pad_last_batch=False,
file_list_include_preceding_frame=False, normalized=False, skip_vfr_check=False, last_batch_policy=types.LAST_BATCH_FILL, last_batch_padded=True):
file_list_include_preceding_frame=False, normalized=False, skip_vfr_check=False, last_batch_policy=types.LAST_BATCH_FILL, pad_last_batch_repeated=False, shard_size=-1):
"""!Creates a VideoDecoder node for loading video sequences.

@param sequence_length Number of frames in video sequence.
Expand Down Expand Up @@ -223,7 +223,11 @@ def video(sequence_length, file_list_frame_num=False, file_root="", image_type=t
"file_list_frame_num": file_list_frame_num} # VideoMetaDataReader
b.videoMetaDataReader(Pipeline._current_pipeline._handle,
*(kwargs_pybind_reader.values()))

RocalShardingInfo = b.RocalShardingInfo()
RocalShardingInfo.last_batch_policy = last_batch_policy
RocalShardingInfo.pad_last_batch_repeated = pad_last_batch_repeated
RocalShardingInfo.stick_to_shard = stick_to_shard
RocalShardingInfo.shard_size = shard_size
kwargs_pybind_decoder = {
"source_path": file_root,
"color_format": image_type,
Expand All @@ -236,7 +240,7 @@ def video(sequence_length, file_list_frame_num=False, file_root="", image_type=t
"frame_step": step,
"frame_stride": stride,
"file_list_frame_num": file_list_frame_num,
"last_batch_info": (last_batch_policy, last_batch_padded)} # VideoDecoder
"sharding_info": RocalShardingInfo} # VideoDecoder
videos = b.videoDecoder(
Pipeline._current_pipeline._handle, *(kwargs_pybind_decoder.values()))
return (videos)
Expand All @@ -248,9 +252,9 @@ def video_resize(sequence_length, resize_width, resize_height, file_list_frame_n
stride=3, decoder_mode=types.SOFTWARE_DECODE,
scaling_mode=types.SCALING_MODE_DEFAULT, interpolation_type=types.LINEAR_INTERPOLATION,
resize_longer=0, resize_shorter=0, max_size=[], enable_frame_num=False,
enable_timestamps=False, file_list="", stick_to_shard=False, pad_last_batch=False,
enable_timestamps=False, file_list="", stick_to_shard=True, pad_last_batch=False,
file_list_include_preceding_frame=False, normalized=False, skip_vfr_check=False,
last_batch_policy=types.LAST_BATCH_FILL, last_batch_padded=True):
last_batch_policy=types.LAST_BATCH_FILL, pad_last_batch_repeated=False, shard_size=-1):
"""!Creates a VideoDecoderResize node in the pipeline for loading and resizing video sequences.

@param sequence_length Number of frames in video sequence.
Expand Down Expand Up @@ -291,18 +295,22 @@ def video_resize(sequence_length, resize_width, resize_height, file_list_frame_n
"file_list_frame_num": file_list_frame_num} # VideoMetaDataReader
meta_data = b.videoMetaDataReader(
Pipeline._current_pipeline._handle, *(kwargs_pybind_reader.values()))

RocalShardingInfo = b.RocalShardingInfo()
RocalShardingInfo.last_batch_policy = last_batch_policy
RocalShardingInfo.pad_last_batch_repeated = pad_last_batch_repeated
RocalShardingInfo.stick_to_shard = stick_to_shard
RocalShardingInfo.shard_size = shard_size
kwargs_pybind_decoder = {"source_path": file_root, "color_format": image_type, "decoder_mode": decoder_mode, "shard_count": num_shards,
"sequence_length": sequence_length, "resize_width": resize_width, "resize_height": resize_height,
"shuffle": random_shuffle, "is_output": False, "loop": False, "frame_step": step, "frame_stride": stride,
"file_list_frame_num": file_list_frame_num, "scaling_mode": scaling_mode, "max_size": max_size,
"resize_shorter": resize_shorter, "resize_longer": resize_longer, "interpolation_type": interpolation_type, "last_batch_info": (last_batch_policy, last_batch_padded)}
"resize_shorter": resize_shorter, "resize_longer": resize_longer, "interpolation_type": interpolation_type, "sharding_info": RocalShardingInfo}
videos = b.videoDecoderResize(
Pipeline._current_pipeline._handle, *(kwargs_pybind_decoder.values()))
return (videos, meta_data)


def sequence_reader(file_root, sequence_length, image_type=types.RGB, num_shards=1, random_shuffle=False, step=3, stride=1, stick_to_shard=False, pad_last_batch=False, last_batch_policy=types.LAST_BATCH_FILL, last_batch_padded=True):
def sequence_reader(file_root, sequence_length, image_type=types.RGB, num_shards=1, random_shuffle=False, step=3, stride=1, stick_to_shard=False, last_batch_policy=types.LAST_BATCH_FILL, pad_last_batch_repeated=False, shard_size=-1):
"""!Creates a SequenceReader node for loading image sequences.

@param file_root Root directory containing image sequences.
Expand All @@ -318,6 +326,11 @@ def sequence_reader(file_root, sequence_length, image_type=types.RGB, num_shards
@return list of loaded image sequences.
"""
Pipeline._current_pipeline._reader = "SequenceReader"
RocalShardingInfo = b.RocalShardingInfo()
RocalShardingInfo.last_batch_policy = last_batch_policy
RocalShardingInfo.pad_last_batch_repeated = pad_last_batch_repeated
RocalShardingInfo.stick_to_shard = stick_to_shard
RocalShardingInfo.shard_size = shard_size
# Output
kwargs_pybind = {
"source_path": file_root,
Expand All @@ -329,7 +342,7 @@ def sequence_reader(file_root, sequence_length, image_type=types.RGB, num_shards
"loop": False,
"frame_step": step,
"frame_stride": stride,
"last_batch_info": (last_batch_policy, last_batch_padded)}
"sharding_info": RocalShardingInfo}
frames = b.sequenceReader(
Pipeline._current_pipeline._handle, *(kwargs_pybind.values()))
return (frames)
Expand Down
2 changes: 1 addition & 1 deletion tests/python_api/decoder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import matplotlib.pyplot as plt

seed = 1549361629
image_dir = "../../data/images/AMD-tinyDataSet/"
image_dir = "/opt/rocm/share/rocal/test/data/images/AMD-tinyDataSet"
batch_size = 4
gpu_id = 0

Expand Down