diff --git a/rocAL/include/api/rocal_api_data_loaders.h b/rocAL/include/api/rocal_api_data_loaders.h index eec0c9a64..ec903eebc 100644 --- a/rocAL/include/api/rocal_api_data_loaders.h +++ b/rocAL/include/api/rocal_api_data_loaders.h @@ -45,7 +45,7 @@ THE SOFTWARE. * \param [in] max_width The maximum width of the decoded images, larger or smaller will be resized to closest * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest * \param [in] rocal_decoder_type Determines the decoder_type, tjpeg or hwdec - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalJpegFileSource(RocalContext context, @@ -56,7 +56,8 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegFileSource(RocalContext context, bool shuffle = false, bool loop = false, RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MOST_FREQUENT_SIZE, - unsigned max_width = 0, unsigned max_height = 0, RocalDecoderType rocal_decoder_type = RocalDecoderType::ROCAL_DECODER_TJPEG, std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + unsigned max_width = 0, unsigned max_height = 0, RocalDecoderType rocal_decoder_type = RocalDecoderType::ROCAL_DECODER_TJPEG, + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates JPEG image reader and decoder. It allocates the resources and objects required to read and decode Jpeg images stored on the file systems. It accepts external sharding information to load a singe shard. only * \ingroup group_rocal_data_loaders @@ -72,7 +73,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegFileSource(RocalContext context, * \param [in] max_width The maximum width of the decoded images, larger or smaller will be resized to closest * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest * \param [in] rocal_decoder_type Determines the decoder_type, tjpeg or hwdec - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ @@ -86,7 +87,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegFileSourceSingleShard(RocalContex bool loop = false, RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MOST_FREQUENT_SIZE, unsigned max_width = 0, unsigned max_height = 0, RocalDecoderType rocal_decoder_type = RocalDecoderType::ROCAL_DECODER_TJPEG, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates JPEG image reader and decoder. Reads [Frames] sequences from a directory representing a collection of streams. * \ingroup group_rocal_data_loaders @@ -100,7 +101,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegFileSourceSingleShard(RocalContex * \param [in] loop Determines if the user wants to indefinitely loops through images or not. * \param [in] step: Frame interval between each sequence. * \param [in] stride: Frame interval between frames in a sequence. - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor. */ extern "C" RocalTensor ROCAL_API_CALL rocalSequenceReader(RocalContext context, @@ -113,7 +114,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalSequenceReader(RocalContext context, bool loop = false, unsigned step = 0, unsigned stride = 0, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates JPEG image reader and decoder. Reads [Frames] sequences from a directory representing a collection of streams. It accepts external sharding information to load a singe shard only. * \ingroup group_rocal_data_loaders @@ -128,7 +129,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalSequenceReader(RocalContext context, * \param [in] loop Determines if the user wants to indefinitely loops through images or not. * \param [in] step: Frame interval between each sequence. * \param [in] stride: Frame interval between frames in a sequence. - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalSequenceReaderSingleShard(RocalContext context, @@ -142,7 +143,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalSequenceReaderSingleShard(RocalContex bool loop = false, unsigned step = 0, unsigned stride = 0, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief JPEG image reader and decoder. It allocates the resources and objects required to read and decode COCO Jpeg images stored on the file systems. It has internal sharding capability to load/decode in parallel is user wants. If images are not Jpeg compressed they will be ignored. * \ingroup group_rocal_data_loaders @@ -156,7 +157,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalSequenceReaderSingleShard(RocalContex * \param [in] max_width The maximum width of the decoded images, larger or smaller will be resized to closest * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest * \param [in] rocal_decoder_type Determines the decoder_type, tjpeg or hwdec - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCOCOFileSource(RocalContext context, @@ -170,7 +171,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCOCOFileSource(RocalContext conte RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MOST_FREQUENT_SIZE, unsigned max_width = 0, unsigned max_height = 0, RocalDecoderType rocal_decoder_type = RocalDecoderType::ROCAL_DECODER_TJPEG, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief JPEG image reader and partial decoder. It allocates the resources and objects required to read and decode COCO Jpeg images stored on the file systems. It has internal sharding capability to load/decode in parallel is user wants. If images are not Jpeg compressed they will be ignored. * \ingroup group_rocal_data_loaders @@ -186,7 +187,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCOCOFileSource(RocalContext conte * \param [in] decode_size_policy * \param [in] max_width The maximum width of the decoded images, larger or smaller will be resized to closest * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCOCOFileSourcePartial(RocalContext p_context, @@ -202,7 +203,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCOCOFileSourcePartial(RocalContex bool loop = false, RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MOST_FREQUENT_SIZE, unsigned max_width = 0, unsigned max_height = 0, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates JPEG image reader and partial decoder. It allocates the resources and objects required to read and decode COCO Jpeg images stored on the file systems. It has internal sharding capability to load/decode in parallel is user wants. If images are not Jpeg compressed they will be ignored. * \ingroup group_rocal_data_loaders @@ -218,7 +219,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCOCOFileSourcePartial(RocalContex * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest * \param [in] area_factor Determines how much area to be cropped. Ranges from from 0.08 - 1. * \param [in] aspect_ratio Determines the aspect ration of crop. Ranges from 0.75 to 1.33. - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCOCOFileSourcePartialSingleShard(RocalContext p_context, @@ -235,7 +236,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCOCOFileSourcePartialSingleShard( bool loop = false, RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MOST_FREQUENT_SIZE, unsigned max_width = 0, unsigned max_height = 0, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates JPEG image reader. It allocates the resources and objects required to read and decode COCO Jpeg images stored on the file systems. It has internal sharding capability to load/decode in parallel is user wants. If images are not Jpeg compressed they will be ignored. * \ingroup group_rocal_data_loaders @@ -250,7 +251,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCOCOFileSourcePartialSingleShard( * \param [in] max_width The maximum width of the decoded images, larger or smaller will be resized to closest * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest * \param [in] rocal_decoder_type Determines the decoder_type, tjpeg or hwdec - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCOCOFileSourceSingleShard(RocalContext context, @@ -265,7 +266,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCOCOFileSourceSingleShard(RocalCo RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MOST_FREQUENT_SIZE, unsigned max_width = 0, unsigned max_height = 0, RocalDecoderType rocal_decoder_type = RocalDecoderType::ROCAL_DECODER_TJPEG, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates JPEG image reader and decoder for Caffe LMDB records. It allocates the resources and objects required to read and decode Jpeg images stored in Caffe LMDB Records. It has internal sharding capability to load/decode in parallel is user wants. If images are not Jpeg compressed they will be ignored. * \ingroup group_rocal_data_loaders @@ -279,7 +280,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCOCOFileSourceSingleShard(RocalCo * \param [in] decode_size_policy * \param [in] max_width The maximum width of the decoded images, larger or smaller will be resized to closest * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCaffeLMDBRecordSource(RocalContext context, @@ -292,7 +293,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCaffeLMDBRecordSource(RocalContex RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MOST_FREQUENT_SIZE, unsigned max_width = 0, unsigned max_height = 0, RocalDecoderType rocal_decoder_type = RocalDecoderType::ROCAL_DECODER_TJPEG, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates JPEG image reader and decoder for Caffe LMDB records. It allocates the resources and objects required to read and decode Jpeg images stored in Caffe2 LMDB Records. It has internal sharding capability to load/decode in parallel is user wants. * \ingroup group_rocal_data_loaders @@ -308,7 +309,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCaffeLMDBRecordSource(RocalContex * \param [in] max_width The maximum width of the decoded images, larger or smaller will be resized to closest * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest * \param [in] rocal_decoder_type Determines the decoder_type, tjpeg or hwdec - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCaffeLMDBRecordSourceSingleShard(RocalContext p_context, @@ -322,7 +323,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCaffeLMDBRecordSourceSingleShard( RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MOST_FREQUENT_SIZE, unsigned max_width = 0, unsigned max_height = 0, RocalDecoderType rocal_decoder_type = RocalDecoderType::ROCAL_DECODER_TJPEG, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates JPEG image reader and decoder for Caffe2 LMDB records. It allocates the resources and objects required to read and decode Jpeg images stored in Caffe2 LMDB Records. It has internal sharding capability to load/decode in parallel is user wants. If images are not Jpeg compressed they will be ignored. * \ingroup group_rocal_data_loaders @@ -337,7 +338,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCaffeLMDBRecordSourceSingleShard( * \param [in] max_width The maximum width of the decoded images, larger or smaller will be resized to closest * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest * \param [in] rocal_decoder_type Determines the decoder_type, tjpeg or hwdec - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCaffe2LMDBRecordSource(RocalContext context, @@ -350,7 +351,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCaffe2LMDBRecordSource(RocalConte RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MOST_FREQUENT_SIZE, unsigned max_width = 0, unsigned max_height = 0, RocalDecoderType rocal_decoder_type = RocalDecoderType::ROCAL_DECODER_TJPEG, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates JPEG image reader and decoder for Caffe2 LMDB records. It allocates the resources and objects required to read and decode Jpeg images stored on the Caffe2 LMDB Records. It accepts external sharding information to load a singe shard. only * \ingroup group_rocal_data_loaders @@ -366,7 +367,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCaffe2LMDBRecordSource(RocalConte * \param [in] max_width The maximum width of the decoded images, larger or smaller will be resized to closest * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest * \param [in] rocal_decoder_type Determines the decoder_type, tjpeg or hwdec - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCaffe2LMDBRecordSourceSingleShard(RocalContext p_context, @@ -380,7 +381,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCaffe2LMDBRecordSourceSingleShard RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MOST_FREQUENT_SIZE, unsigned max_width = 0, unsigned max_height = 0, RocalDecoderType rocal_decoder_type = RocalDecoderType::ROCAL_DECODER_TJPEG, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates JPEG image reader and decoder for MXNet records. It allocates the resources and objects required to read and decode Jpeg images stored in MXNet Records. It has internal sharding capability to load/decode in parallel is user wants. If images are not Jpeg compressed they will be ignored. * \ingroup group_rocal_data_loaders @@ -395,7 +396,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCaffe2LMDBRecordSourceSingleShard * \param [in] max_width The maximum width of the decoded images, larger or smaller will be resized to closest * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest * \param [in] rocal_decoder_type Determines the decoder_type, tjpeg or hwdec - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalMXNetRecordSource(RocalContext context, @@ -408,7 +409,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalMXNetRecordSource(RocalContext contex RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MOST_FREQUENT_SIZE, unsigned max_width = 0, unsigned max_height = 0, RocalDecoderType rocal_decoder_type = RocalDecoderType::ROCAL_DECODER_TJPEG, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates JPEG image reader and decoder for MXNet records. It allocates the resources and objects required to read and decode Jpeg images stored on the MXNet records. It accepts external sharding information to load a singe shard. only * \ingroup group_rocal_data_loaders @@ -424,7 +425,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalMXNetRecordSource(RocalContext contex * \param [in] max_width The maximum width of the decoded images, larger or smaller will be resized to closest * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest * \param [in] rocal_decoder_type Determines the decoder_type, tjpeg or hwdec - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalMXNetRecordSourceSingleShard(RocalContext p_context, @@ -438,7 +439,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalMXNetRecordSourceSingleShard(RocalCon RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MOST_FREQUENT_SIZE, unsigned max_width = 0, unsigned max_height = 0, RocalDecoderType rocal_decoder_type = RocalDecoderType::ROCAL_DECODER_TJPEG, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates JPEG image reader and partial decoder. It allocates the resources and objects required to read and decode Jpeg images stored on the file systems. It has internal sharding capability to load/decode in parallel is user wants. If images are not Jpeg compressed they will be ignored and Crops t * \ingroup group_rocal_data_loaders @@ -455,7 +456,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalMXNetRecordSourceSingleShard(RocalCon * \param [in] decode_size_policy * \param [in] max_width The maximum width of the decoded images, larger or smaller will be resized to closest * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalFusedJpegCrop(RocalContext context, @@ -470,7 +471,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalFusedJpegCrop(RocalContext context, bool loop = false, RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MOST_FREQUENT_SIZE, unsigned max_width = 0, unsigned max_height = 0, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates JPEG image reader and partial decoder. It allocates the resources and objects required to read and decode Jpeg images stored on the file systems. It accepts external sharding information to load a singe shard. only * \ingroup group_rocal_data_loaders @@ -486,7 +487,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalFusedJpegCrop(RocalContext context, * \param [in] decode_size_policy * \param [in] max_width The maximum width of the decoded images, larger or smaller will be resized to closest * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalFusedJpegCropSingleShard(RocalContext context, @@ -502,7 +503,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalFusedJpegCropSingleShard(RocalContext bool loop = false, RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MOST_FREQUENT_SIZE, unsigned max_width = 0, unsigned max_height = 0, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates TensorFlow records JPEG image reader and decoder. It allocates the resources and objects required to read and decode Jpeg images stored on the file systems. It has internal sharding capability to load/decode in parallel is user wants. If images are not Jpeg compressed they will be ignored. * \ingroup group_rocal_data_loaders @@ -517,7 +518,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalFusedJpegCropSingleShard(RocalContext * \param [in] max_width The maximum width of the decoded images, larger or smaller will be resized to closest * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest * \param [in] rocal_decoder_type Determines the decoder_type, tjpeg or hwdec - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output image */ extern "C" RocalTensor ROCAL_API_CALL rocalJpegTFRecordSource(RocalContext context, @@ -532,7 +533,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegTFRecordSource(RocalContext conte RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MOST_FREQUENT_SIZE, unsigned max_width = 0, unsigned max_height = 0, RocalDecoderType rocal_decoder_type = RocalDecoderType::ROCAL_DECODER_TJPEG, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates TensorFlow records JPEG image reader and decoder. It allocates the resources and objects required to read and decode Jpeg images stored on the file systems. It accepts external sharding information to load a singe shard. only * \ingroup group_rocal_data_loaders @@ -548,7 +549,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegTFRecordSource(RocalContext conte * \param [in] max_width The maximum width of the decoded images, larger or smaller will be resized to closest * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest * \param [in] rocal_decoder_type Determines the decoder_type, tjpeg or hwdec - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalJpegTFRecordSourceSingleShard(RocalContext context, @@ -562,7 +563,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegTFRecordSourceSingleShard(RocalCo RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MOST_FREQUENT_SIZE, unsigned max_width = 0, unsigned max_height = 0, RocalDecoderType rocal_decoder_type = RocalDecoderType::ROCAL_DECODER_TJPEG, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates Raw image loader. It allocates the resources and objects required to load images stored on the file systems. * \ingroup group_rocal_data_loaders @@ -574,7 +575,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegTFRecordSourceSingleShard(RocalCo * \param [in] loop: repeat data loading * \param [in] out_width The output_width of raw image * \param [in] out_height The output height of raw image - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalRawTFRecordSource(RocalContext p_context, @@ -587,7 +588,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalRawTFRecordSource(RocalContext p_cont bool loop = false, unsigned out_width = 0, unsigned out_height = 0, const char* record_name_prefix = "", - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates Raw image loader. It allocates the resources and objects required to load images stored on the file systems. * \ingroup group_rocal_data_loaders @@ -601,7 +602,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalRawTFRecordSource(RocalContext p_cont * \param [in] out_width The output_width of raw image * \param [in] out_height The output height of raw image * \param [in] record_name_prefix : if nonempty reader will only read records with certain prefix - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalRawTFRecordSourceSingleShard(RocalContext p_context, @@ -614,7 +615,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalRawTFRecordSourceSingleShard(RocalCon bool loop = false, unsigned out_width = 0, unsigned out_height = 0, const char* record_name_prefix = "", - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! * \brief Creates a video reader and decoder as a source. It allocates the resources and objects required to read and decode mp4 videos stored on the file systems. @@ -631,7 +632,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalRawTFRecordSourceSingleShard(RocalCon * \param [in] step: Frame interval between each sequence. * \param [in] stride: Frame interval between frames in a sequence. * \param [in] file_list_frame_num: Determines if the user wants to read frame number or timestamps if a text file is passed in the source_path. - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalVideoFileSource(RocalContext context, @@ -646,7 +647,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalVideoFileSource(RocalContext context, unsigned step = 0, unsigned stride = 0, bool file_list_frame_num = true, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates a video reader and decoder as a source. It allocates the resources and objects required to read and decode mp4 videos stored on the file systems. It accepts external sharding information to load a singe shard only. * \ingroup group_rocal_data_loaders @@ -663,7 +664,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalVideoFileSource(RocalContext context, * \param [in] step: Frame interval between each sequence. * \param [in] stride: Frame interval between frames in a sequence. * \param [in] file_list_frame_num: Determines if the user wants to read frame number or timestamps if a text file is passed in the source_path. - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalVideoFileSourceSingleShard(RocalContext context, @@ -679,7 +680,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalVideoFileSourceSingleShard(RocalConte unsigned step = 0, unsigned stride = 0, bool file_list_frame_num = true, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates a video reader and decoder as a source. It allocates the resources and objects required to read and decode mp4 videos stored on the file systems. Resizes the decoded frames to the dest width and height. * \ingroup group_rocal_data_loaders @@ -697,7 +698,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalVideoFileSourceSingleShard(RocalConte * \param [in] step: Frame interval between each sequence. * \param [in] stride: Frame interval between frames in a sequence. * \param [in] file_list_frame_num: Determines if the user wants to read frame number or timestamps if a text file is passed in the source_path. - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalVideoFileResize(RocalContext context, @@ -719,7 +720,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalVideoFileResize(RocalContext context, unsigned resize_shorter = 0, unsigned resize_longer = 0, RocalResizeInterpolationType interpolation_type = ROCAL_LINEAR_INTERPOLATION, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates a video reader and decoder as a source. It allocates the resources and objects required to read and decode mp4 videos stored on the file systems. Resizes the decoded frames to the dest width and height. It accepts external sharding information to load a singe shard only. * \ingroup group_rocal_data_loaders @@ -738,7 +739,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalVideoFileResize(RocalContext context, * \param [in] step: Frame interval between each sequence. * \param [in] stride: Frame interval between frames in a sequence. * \param [in] file_list_frame_num: Determines if the user wants to read frame number or timestamps if a text file is passed in the source_path. - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalVideoFileResizeSingleShard(RocalContext context, @@ -761,7 +762,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalVideoFileResizeSingleShard(RocalConte unsigned resize_shorter = 0, unsigned resize_longer = 0, RocalResizeInterpolationType interpolation_type = ROCAL_LINEAR_INTERPOLATION, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates CIFAR10 raw data reader and loader. It allocates the resources and objects required to read raw data stored on the file systems. * \ingroup group_rocal_data_loaders @@ -772,7 +773,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalVideoFileResizeSingleShard(RocalConte * \param [in] out_width output width * \param [in] out_height output_height * \param [in] filename_prefix if set loader will only load files with the given prefix name - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalRawCIFAR10Source(RocalContext context, @@ -805,7 +806,7 @@ extern "C" RocalStatus ROCAL_API_CALL rocalResetLoaders(RocalContext context); * \param [in] decode_size_policy * \param [in] max_width The maximum width of the decoded images, larger or smaller will be resized to closest * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCaffeLMDBRecordSourcePartialSingleShard(RocalContext p_context, @@ -821,7 +822,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCaffeLMDBRecordSourcePartialSingl bool loop = false, RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MOST_FREQUENT_SIZE, unsigned max_width = 0, unsigned max_height = 0, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates JPEG image reader and partial decoder for Caffe2 LMDB records. It allocates the resources and objects required to read and decode Jpeg images stored in Caffe22 LMDB Records. It has internal sharding capability to load/decode in parallel is user wants. * \ingroup group_rocal_data_loaders @@ -836,7 +837,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCaffeLMDBRecordSourcePartialSingl * \param [in] decode_size_policy * \param [in] max_width The maximum width of the decoded images, larger or smaller will be resized to closest * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCaffe2LMDBRecordSourcePartialSingleShard(RocalContext p_context, @@ -852,7 +853,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCaffe2LMDBRecordSourcePartialSing bool loop = false, RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MOST_FREQUENT_SIZE, unsigned max_width = 0, unsigned max_height = 0, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! \brief Creates JPEG external source image reader. * \ingroup group_rocal_data_loaders * \param [in] rocal_context Rocal context @@ -865,7 +866,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegCaffe2LMDBRecordSourcePartialSing * \param [in] max_height The maximum height of the decoded images, larger or smaller will be resized to closest * \param [in] rocal_decoder_type Determines the decoder_type, tjpeg or hwdec * \param [in] external_source_mode Determines the mode of the source passed from the user - file_names / uncompressed data / compressed data - * \param [in] last_batch_info Determines the handling of the last batch when the shard size is not divisible by the batch size. Check RocalLastBatchPolicy() enum for possible values & If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output tensor */ extern "C" RocalTensor ROCAL_API_CALL rocalJpegExternalFileSource(RocalContext p_context, @@ -877,7 +878,7 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegExternalFileSource(RocalContext p unsigned max_width = 0, unsigned max_height = 0, RocalDecoderType rocal_decoder_type = RocalDecoderType::ROCAL_DECODER_TJPEG, RocalExternalSourceMode external_source_mode = RocalExternalSourceMode::ROCAL_EXTSOURCE_FNAME, - std::pair last_batch_info = {RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL, true}); + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! Creates Audio file reader and decoder. It allocates the resources and objects required to read and decode audio files stored on the file systems. It has internal sharding capability to load/decode in parallel if user wants. * If the files are not in standard audio compression formats they will be ignored, Currently wav format is supported @@ -889,6 +890,10 @@ extern "C" RocalTensor ROCAL_API_CALL rocalJpegExternalFileSource(RocalContext p * \param [in] shuffle Boolean variable to shuffle the dataset. * \param [in] loop Boolean variable to indefinitely loop through audio. * \param [in] downmix Boolean variable to downmix all input channels to mono. If downmixing is turned on, the decoder output is 1D. If downmixing is turned off, it produces 2D output with interleaved channels incase of multichannel audio. + * \param [in] decode_size_policy is the RocalImageSizeEvaluationPolicy for decoding. + * \param [in] max_decoded_samples The maximum samples of the decoded audio data. + * \param [in] max_decoded_channels The maximum channels of the decoded audio data. + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output audio */ extern "C" RocalTensor ROCAL_API_CALL rocalAudioFileSource(RocalContext context, @@ -898,7 +903,11 @@ extern "C" RocalTensor ROCAL_API_CALL rocalAudioFileSource(RocalContext context, bool is_output, bool shuffle = false, bool loop = false, - bool downmix = false); + bool downmix = false, + RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MAX_SIZE, + unsigned max_decoded_samples = 0, + unsigned max_decoded_channels = 0, + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); /*! Creates Audio file reader and decoder. It allocates the resources and objects required to read and decode audio files stored on the file systems. It has internal sharding capability to load/decode in parallel is user wants. * If the files are not in standard audio compression formats they will be ignored. @@ -911,6 +920,10 @@ extern "C" RocalTensor ROCAL_API_CALL rocalAudioFileSource(RocalContext context, * \param [in] shuffle Boolean variable to shuffle the dataset. * \param [in] loop Boolean variable to indefinitely loop through audio. * \param [in] downmix Boolean variable to downmix all input channels to mono. If downmixing is turned on, the decoder output is 1D. If downmixing is turned off, it produces 2D output with interleaved channels incase of multichannel audio. + * \param [in] decode_size_policy is the RocalImageSizeEvaluationPolicy for decoding. + * \param [in] max_decoded_samples The maximum samples of the decoded audio data. + * \param [in] max_decoded_channels The maximum channels of the decoded audio data. + * \param [in] rocal_sharding_info The members of RocalShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. * \return Reference to the output audio */ extern "C" RocalTensor ROCAL_API_CALL rocalAudioFileSourceSingleShard(RocalContext p_context, @@ -921,6 +934,10 @@ extern "C" RocalTensor ROCAL_API_CALL rocalAudioFileSourceSingleShard(RocalConte bool is_output, bool shuffle = false, bool loop = false, - bool downmix = false); + bool downmix = false, + RocalImageSizeEvaluationPolicy decode_size_policy = ROCAL_USE_MAX_SIZE, + unsigned max_decoded_samples = 0, + unsigned max_decoded_channels = 0, + RocalShardingInfo rocal_sharding_info = RocalShardingInfo()); #endif // MIVISIONX_ROCAL_API_DATA_LOADERS_H diff --git a/rocAL/include/api/rocal_api_info.h b/rocAL/include/api/rocal_api_info.h index 9ea36fbb8..eab1c77eb 100644 --- a/rocAL/include/api/rocal_api_info.h +++ b/rocAL/include/api/rocal_api_info.h @@ -133,7 +133,7 @@ extern "C" TimingInfo ROCAL_API_CALL rocalGetTimingInfo(RocalContext rocal_conte * \brief Retrieves the information about the size of the last batch. * \ingroup group_rocal_info * \param rocal_context - * \return The number of samples that were padded in the last batch in adherence with last_batch_policy and last_batch_padded + * \return The number of samples that were padded in the last batch in adherence with last_batch_policy and pad_last_batch_repeated. */ extern "C" size_t ROCAL_API_CALL rocalGetLastBatchPaddedSize(RocalContext rocal_context); diff --git a/rocAL/include/api/rocal_api_tensor.h b/rocAL/include/api/rocal_api_tensor.h index 5faf0e0ee..579af260b 100644 --- a/rocAL/include/api/rocal_api_tensor.h +++ b/rocAL/include/api/rocal_api_tensor.h @@ -40,6 +40,7 @@ class rocalTensor { virtual ~rocalTensor() = default; virtual void* buffer() = 0; virtual unsigned copy_data(void* user_buffer, RocalOutputMemType external_mem_type = ROCAL_MEMCPY_HOST) = 0; + virtual unsigned copy_data(void* user_buffer, uint x_offset, uint y_offset, uint max_cols, uint max_rows) = 0; // Copy only the ROI to the user_buffer [The padded region is not copied] virtual unsigned num_of_dims() = 0; virtual unsigned batch_size() = 0; virtual std::vector dims() = 0; diff --git a/rocAL/include/api/rocal_api_types.h b/rocAL/include/api/rocal_api_types.h index e63a5a915..b446008a0 100644 --- a/rocAL/include/api/rocal_api_types.h +++ b/rocAL/include/api/rocal_api_types.h @@ -438,7 +438,7 @@ enum RocalMelScaleFormula { ROCAL_MELSCALE_HTK }; -/*! \brief Tensor Last Batch Policies +/*! \brief Tensor Last Batch Policy Type enum * \ingroup group_rocal_types */ enum RocalLastBatchPolicy { @@ -448,9 +448,27 @@ enum RocalLastBatchPolicy { /*! \brief ROCAL_LAST_BATCH_DROP - The last batch is dropped if there are not enough samples from the current epoch. */ ROCAL_LAST_BATCH_DROP = 1, - /*! \brief ROCAL_LAST_BATCH_PARTIAL - The last batch is partially filled with the remaining data from the current epoch, keeping the rest of the samples empty. (currently this policy works similar to FILL in rocAL, PARTIAL policy needs to be handled from python end) + /*! \brief ROCAL_LAST_BATCH_PARTIAL - The last batch is partially filled with the remaining data from the current epoch, keeping the rest of the samples empty. (currently this policy works similar to FILL in rocAL, PARTIAL policy needs to be handled in the python iterator) */ ROCAL_LAST_BATCH_PARTIAL = 2 }; +/*! \brief rocAL RocalShardingInfo enum + * \ingroup group_rocal_types + */ +struct RocalShardingInfo { + RocalLastBatchPolicy last_batch_policy; + bool pad_last_batch_repeated; + bool stick_to_shard; + int32_t shard_size; + + // Constructor with default values + RocalShardingInfo() + : last_batch_policy(RocalLastBatchPolicy::ROCAL_LAST_BATCH_FILL), + pad_last_batch_repeated(false), + stick_to_shard(true), + shard_size(-1) + {} +}; + #endif // MIVISIONX_ROCAL_API_TYPES_H diff --git a/rocAL/include/loaders/audio/audio_loader.h b/rocAL/include/loaders/audio/audio_loader.h index 8fa87ee48..4ca375881 100644 --- a/rocAL/include/loaders/audio/audio_loader.h +++ b/rocAL/include/loaders/audio/audio_loader.h @@ -56,6 +56,7 @@ class AudioLoader : public LoaderModule { void feed_external_input(const std::vector& input_images_names, const std::vector& input_buffer, const std::vector& roi_xywh, unsigned int max_width, unsigned int max_height, unsigned int channels, ExternalSourceFileMode mode, bool eos) override { THROW("external source feed is not supported in audio loader") } + size_t last_batch_padded_size() override; private: bool is_out_of_data(); diff --git a/rocAL/include/loaders/audio/audio_loader_sharded.h b/rocAL/include/loaders/audio/audio_loader_sharded.h index 76eb63cad..c93ab02aa 100644 --- a/rocAL/include/loaders/audio/audio_loader_sharded.h +++ b/rocAL/include/loaders/audio/audio_loader_sharded.h @@ -45,6 +45,7 @@ class AudioLoaderSharded : public LoaderModule { void feed_external_input(const std::vector& input_images_names, const std::vector& input_buffer, const std::vector& roi_xywh, unsigned int max_width, unsigned int max_height, unsigned int channels, ExternalSourceFileMode mode, bool eos) override { THROW("external source feed is not supported in audio loader") } + size_t last_batch_padded_size() override; private: void increment_loader_idx(); diff --git a/rocAL/include/loaders/audio/audio_read_and_decode.h b/rocAL/include/loaders/audio/audio_read_and_decode.h index d121869e6..f31c4bc81 100644 --- a/rocAL/include/loaders/audio/audio_read_and_decode.h +++ b/rocAL/include/loaders/audio/audio_read_and_decode.h @@ -64,6 +64,7 @@ class AudioReadAndDecode { const size_t max_decoded_channels); // returns timing info or other status information Timing GetTiming(); + size_t last_batch_padded_size(); // The number of padded samples in the last batch private: std::vector> _decoder; diff --git a/rocAL/include/loaders/audio/node_audio_loader.h b/rocAL/include/loaders/audio/node_audio_loader.h index b3d04b68c..8fdf131bf 100644 --- a/rocAL/include/loaders/audio/node_audio_loader.h +++ b/rocAL/include/loaders/audio/node_audio_loader.h @@ -44,11 +44,13 @@ class AudioLoaderNode : public Node { /// \param load_batch_count Defines the quantum count of the Audios to be loaded. It's usually equal to the user's batch size. /// \param mem_type Memory type, host or device /// \param meta_data_reader Determines the meta-data information + /// \param sharding_info The members of ShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. /// The loader will repeat Audios if necessary to be able to have Audios in multiples of the load_batch_count, /// for example if there are 10 Audios in the dataset and load_batch_count is 3, the loader repeats 2 Audios as if there are 12 Audios available. void Init(unsigned internal_shard_count, unsigned cpu_num_threads, const std::string &source_path, const std::string &file_list_path, StorageType storage_type, DecoderType decoder_type, bool shuffle, bool loop, - size_t load_batch_count, RocalMemType mem_type, std::shared_ptr meta_data_reader); + size_t load_batch_count, RocalMemType mem_type, std::shared_ptr meta_data_reader, + const ShardingInfo& sharding_info); std::shared_ptr GetLoaderModule(); protected: diff --git a/rocAL/include/loaders/audio/node_audio_loader_single_shard.h b/rocAL/include/loaders/audio/node_audio_loader_single_shard.h index 851a4eb39..6a1a8082f 100644 --- a/rocAL/include/loaders/audio/node_audio_loader_single_shard.h +++ b/rocAL/include/loaders/audio/node_audio_loader_single_shard.h @@ -42,11 +42,13 @@ class AudioLoaderSingleShardNode : public Node { /// \param load_batch_count Defines the quantum count of the Audios to be loaded. It's usually equal to the user's batch size. /// \param mem_type Memory type, host or device /// \param meta_data_reader Determines the meta-data information + /// \param sharding_info The members of ShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. /// The loader will repeat Audios if necessary to be able to have Audios in multiples of the load_batch_count, /// for example if there are 10 Audios in the dataset and load_batch_count is 3, the loader repeats 2 Audios as if there are 12 Audios available. void Init(unsigned shard_id, unsigned shard_count, unsigned cpu_num_threads, const std::string &source_path, const std::string &file_list_path, StorageType storage_type, DecoderType decoder_type, bool shuffle, - bool loop, size_t load_batch_count, RocalMemType mem_type, std::shared_ptr meta_data_reader); + bool loop, size_t load_batch_count, RocalMemType mem_type, std::shared_ptr meta_data_reader, + const ShardingInfo& sharding_info); std::shared_ptr GetLoaderModule(); protected: diff --git a/rocAL/include/loaders/image/node_fused_jpeg_crop.h b/rocAL/include/loaders/image/node_fused_jpeg_crop.h index 841b7f091..cba00aea1 100644 --- a/rocAL/include/loaders/image/node_fused_jpeg_crop.h +++ b/rocAL/include/loaders/image/node_fused_jpeg_crop.h @@ -42,7 +42,7 @@ class FusedJpegCropNode : public Node { /// for example if there are 10 images in the dataset and load_batch_count is 3, the loader repeats 2 images as if there are 12 images available. void init(unsigned internal_shard_count, unsigned cpu_num_threads, const std::string &source_path, const std::string &json_path, StorageType storage_type, DecoderType decoder_type, bool shuffle, bool loop, size_t load_batch_count, RocalMemType mem_type, std::shared_ptr meta_data_reader, - unsigned num_attempts, std::vector &random_area, std::vector &random_aspect_ratio, std::pair last_batch_info = {RocalBatchPolicy::FILL, true}); + unsigned num_attempts, std::vector &random_area, std::vector &random_aspect_ratio, const ShardingInfo& sharding_info = ShardingInfo()); std::shared_ptr get_loader_module(); diff --git a/rocAL/include/loaders/image/node_fused_jpeg_crop_single_shard.h b/rocAL/include/loaders/image/node_fused_jpeg_crop_single_shard.h index 87e3a1d80..462e04411 100644 --- a/rocAL/include/loaders/image/node_fused_jpeg_crop_single_shard.h +++ b/rocAL/include/loaders/image/node_fused_jpeg_crop_single_shard.h @@ -39,7 +39,7 @@ class FusedJpegCropSingleShardNode : public Node { /// for example if there are 10 images in the dataset and load_batch_count is 3, the loader repeats 2 images as if there are 12 images available. void init(unsigned shard_id, unsigned shard_count, unsigned cpu_num_threads, const std::string &source_path, const std::string &json_path, StorageType storage_type, DecoderType decoder_type, bool shuffle, bool loop, size_t load_batch_count, RocalMemType mem_type, std::shared_ptr meta_data_reader, - unsigned num_attempts, std::vector &random_area, std::vector &random_aspect_ratio, std::pair last_batch_info = {RocalBatchPolicy::FILL, true}); + unsigned num_attempts, std::vector &random_area, std::vector &random_aspect_ratio, const ShardingInfo& sharding_info = ShardingInfo()); std::shared_ptr get_loader_module(); diff --git a/rocAL/include/loaders/image/node_image_loader.h b/rocAL/include/loaders/image/node_image_loader.h index 43b847650..18bcf6d44 100644 --- a/rocAL/include/loaders/image/node_image_loader.h +++ b/rocAL/include/loaders/image/node_image_loader.h @@ -40,7 +40,7 @@ class ImageLoaderNode : public Node { /// The loader will repeat images if necessary to be able to have images in multiples of the load_batch_count, /// for example if there are 10 images in the dataset and load_batch_count is 3, the loader repeats 2 images as if there are 12 images available. void init(unsigned internal_shard_count, unsigned cpu_num_threads, const std::string &source_path, const std::string &json_path, const std::map feature_key_map, StorageType storage_type, DecoderType decoder_type, bool shuffle, bool loop, - size_t load_batch_count, RocalMemType mem_type, std::shared_ptr meta_data_reader, bool decoder_keep_orig = false, std::pair last_batch_info = {RocalBatchPolicy::FILL, true}, const char *prefix = "", unsigned sequence_length = 0, unsigned step = 0, unsigned stride = 0, ExternalSourceFileMode external_file_mode = ExternalSourceFileMode::NONE); + size_t load_batch_count, RocalMemType mem_type, std::shared_ptr meta_data_reader, bool decoder_keep_orig = false, const ShardingInfo& sharding_info = ShardingInfo(), const char *prefix = "", unsigned sequence_length = 0, unsigned step = 0, unsigned stride = 0, ExternalSourceFileMode external_file_mode = ExternalSourceFileMode::NONE); std::shared_ptr get_loader_module(); diff --git a/rocAL/include/loaders/image/node_image_loader_single_shard.h b/rocAL/include/loaders/image/node_image_loader_single_shard.h index 30cffe1ad..0ee91e69f 100644 --- a/rocAL/include/loaders/image/node_image_loader_single_shard.h +++ b/rocAL/include/loaders/image/node_image_loader_single_shard.h @@ -37,7 +37,7 @@ class ImageLoaderSingleShardNode : public Node { /// The loader will repeat images if necessary to be able to have images in multiples of the load_batch_count, /// for example if there are 10 images in the dataset and load_batch_count is 3, the loader repeats 2 images as if there are 12 images available. void init(unsigned shard_id, unsigned shard_count, unsigned cpu_num_threads, const std::string &source_path, const std::string &json_path, StorageType storage_type, DecoderType decoder_type, - bool shuffle, bool loop, size_t load_batch_count, RocalMemType mem_type, std::shared_ptr meta_data_reader, bool decoder_keep_orig = false, std::pair last_batch_info = {RocalBatchPolicy::FILL, true}, + bool shuffle, bool loop, size_t load_batch_count, RocalMemType mem_type, std::shared_ptr meta_data_reader, bool decoder_keep_orig = false, const ShardingInfo& sharding_info = ShardingInfo(), const std::map feature_key_map = std::map(), unsigned sequence_length = 0, unsigned step = 0, unsigned stride = 0, ExternalSourceFileMode external_file_mode = ExternalSourceFileMode::NONE); std::shared_ptr get_loader_module(); diff --git a/rocAL/include/meta_data/meta_data_reader.h b/rocAL/include/meta_data/meta_data_reader.h index 111d6ead5..b9e977c0a 100644 --- a/rocAL/include/meta_data/meta_data_reader.h +++ b/rocAL/include/meta_data/meta_data_reader.h @@ -100,4 +100,5 @@ class MetaDataReader { virtual ImgSize lookup_image_size(const std::string& image_name) { return {}; } virtual void set_aspect_ratio_grouping(bool aspect_ratio_grouping) { return; } virtual bool get_aspect_ratio_grouping() const { return {}; } + virtual std::vector get_relative_file_path() { return {}; } // Returns the relative file_path's of the reader }; diff --git a/rocAL/include/meta_data/text_file_meta_data_reader.h b/rocAL/include/meta_data/text_file_meta_data_reader.h index 502f21e87..db91fbb91 100644 --- a/rocAL/include/meta_data/text_file_meta_data_reader.h +++ b/rocAL/include/meta_data/text_file_meta_data_reader.h @@ -36,6 +36,7 @@ class TextFileMetaDataReader : public MetaDataReader { bool set_timestamp_mode() override { return false; } const std::map>& get_map_content() override { return _map_content; } + std::vector get_relative_file_path() override { return _relative_file_path; } TextFileMetaDataReader(); private: @@ -45,4 +46,5 @@ class TextFileMetaDataReader : public MetaDataReader { void add(std::string image_name, int label); std::map> _map_content; std::string _path; + std::vector _relative_file_path {}; }; diff --git a/rocAL/include/pipeline/commons.h b/rocAL/include/pipeline/commons.h index 6fe70ae9c..1f4716392 100644 --- a/rocAL/include/pipeline/commons.h +++ b/rocAL/include/pipeline/commons.h @@ -159,11 +159,11 @@ struct Timing { long long unsigned video_process_time= 0; }; -/*! \brief Tensor Last Batch Policies +/*! \brief Tensor Last Batch Policy Type enum These policies the last batch policies determine the behavior when there are not enough samples in the epoch to fill the last batch FILL - The last batch is filled by either repeating the last sample or by wrapping up the data set. DROP - The last batch is dropped if it cannot be fully filled with data from the current epoch. - PARTIAL - The last batch is partially filled with the remaining data from the current epoch, and padding the remaining samples with either last image or wrapping up the dataset - the padded images are removed in the python end + PARTIAL - The last batch is partially filled with the remaining data from the current epoch, keeping the rest of the samples empty. (currently this policy works similar to FILL in rocAL, PARTIAL policy needs to be handled in the pytorch iterator) */ enum RocalBatchPolicy { FILL = 0, diff --git a/rocAL/include/pipeline/tensor.h b/rocAL/include/pipeline/tensor.h index 5540d49ee..cebaf8769 100644 --- a/rocAL/include/pipeline/tensor.h +++ b/rocAL/include/pipeline/tensor.h @@ -326,7 +326,7 @@ class Tensor : public rocalTensor { #endif unsigned copy_data(void* user_buffer, RocalOutputMemType external_mem_type) override; //! Copying the output buffer with specified max_cols and max_rows values for the 2D buffer of size batch_size - unsigned copy_data(void* user_buffer, uint max_rows, uint max_cols); + unsigned copy_data(void* user_buffer, uint x_offset, uint y_offset, uint max_rows, uint max_cols); //! Default destructor /*! Releases the OpenVX Tensor object */ ~Tensor(); diff --git a/rocAL/include/readers/file_source_reader.h b/rocAL/include/readers/file_source_reader.h index eeefdefd5..1fd66e973 100644 --- a/rocAL/include/readers/file_source_reader.h +++ b/rocAL/include/readers/file_source_reader.h @@ -28,8 +28,8 @@ THE SOFTWARE. #include #include "pipeline/commons.h" -#include "readers/image/image_reader.h" #include "pipeline/timing_debug.h" +#include "readers/image/image_reader.h" class FileSourceReader : public Reader { public: @@ -67,9 +67,11 @@ class FileSourceReader : public Reader { FileSourceReader(); - //! Returns the number of images in the last batch - size_t last_batch_padded_size() override; + size_t last_batch_padded_size() override; // The size of the number of samples padded in the last batch + + std::string get_root_folder_path() override; // Returns the root folder path + std::vector get_file_paths_from_meta_data_reader() override; // Returns the relative file path from the meta-data reader private: //! opens the folder containnig the images Reader::Status open_folder(); @@ -83,30 +85,37 @@ class FileSourceReader : public Reader { unsigned _curr_file_idx; FILE *_current_fPtr; unsigned _current_file_size; + unsigned _shard_start_idx; + std::vector _shard_start_idx_vector, _shard_end_idx_vector; std::string _last_id; - std::string _last_file_name, _last_file_path; + std::string _last_file_name, _last_file_path, _absolute_file_path; size_t _shard_id = 0; size_t _shard_count = 1; // equivalent of batch size - //!< _batch_count Defines the quantum count of the images to be read. It's usually equal to the user's batch size. - /// The loader will repeat images if necessary to be able to have images available in multiples of the load_batch_count, - /// for instance if there are 10 images in the dataset and _batch_count is 3, the loader repeats 2 images as if there are 12 images available. - size_t _batch_count = 1; - size_t _file_id = 0; - size_t _in_batch_read_count = 0; + int32_t _shard_size = -1; + size_t _batch_size = 1; + size_t _padded_samples = 0; bool _loop; bool _shuffle; int _read_counter = 0; //!< _file_count_all_shards total_number of files in to figure out the max_batch_size (usually needed for distributed training). size_t _file_count_all_shards; void incremenet_read_ptr(); + void increment_curr_file_idx(); int release(); - size_t get_file_shard_id(); - void incremenet_file_id() { _file_id++; } void fill_last_batch(); void replicate_last_batch_to_pad_partial_shard(); std::shared_ptr _meta_data_reader = nullptr; - //! Pair containing the last batch policy and last_batch_padded values for deciding what to do with last batch - std::pair _last_batch_info; - size_t _last_batch_padded_size = 0; - Reader::Status generate_file_names(); + //! Pair containing the last batch policy and pad_last_batch_repeated values for deciding what to do with last batch + ShardingInfo _last_batch_info = ShardingInfo(); // The members of ShardingInfo determines how the data is distributed among the shards and how the last batch is processed by the pipeline. + size_t _last_batch_padded_size = 0; // The size of number of padded samples in the last batch + size_t _num_padded_samples = 0; //! Number of samples that are padded in the last batch which would differ for each shard. + bool _stick_to_shard = false; + bool _pad_last_batch_repeated = false; + Reader::Status generate_file_names(); // Function that would generate _file_names containing all the samples in the dataset + void compute_start_and_end_idx_of_all_shards(); // Start Idx of all the Shards + size_t get_dataset_size(); // DataSet Size + size_t actual_shard_size_without_padding(); // Actual Number of Files present in the shard (without padding) + size_t largest_shard_size_without_padding(); // The size of the shard having largest files (without padding) + //!< Used to advance to the next shard's data to increase the entropy of the data seen by the pipeline> + void increment_shard_id(); }; diff --git a/rocAL/include/readers/image/image_reader.h b/rocAL/include/readers/image/image_reader.h index 7c3cd0eaa..d17904dbb 100644 --- a/rocAL/include/readers/image/image_reader.h +++ b/rocAL/include/readers/image/image_reader.h @@ -57,6 +57,27 @@ enum class ExternalSourceFileMode { NONE = 3, }; +struct ShardingInfo { + RocalBatchPolicy last_batch_policy; + bool pad_last_batch_repeated; + bool stick_to_shard; + int32_t shard_size; + + // Constructor with default values + ShardingInfo() + : last_batch_policy(RocalBatchPolicy::FILL), + pad_last_batch_repeated(false), + stick_to_shard(true), + shard_size(-1) {} + + // Parameterized constructor + ShardingInfo(RocalBatchPolicy policy, bool pad_repeated, bool stick, int32_t size) + : last_batch_policy(policy), + pad_last_batch_repeated(pad_repeated), + stick_to_shard(stick), + shard_size(size) {} +}; + struct ReaderConfig { explicit ReaderConfig(StorageType type, std::string path = "", std::string json_path = "", const std::map feature_key_map = std::map(), @@ -80,8 +101,8 @@ struct ReaderConfig { void set_frame_step(unsigned step) { _sequence_frame_step = step; } void set_frame_stride(unsigned stride) { _sequence_frame_stride = stride; } void set_external_filemode(ExternalSourceFileMode mode) { _file_mode = mode; } - void set_last_batch_policy(std::pair last_batch_info) { - _last_batch_info = last_batch_info; + void set_sharding_info(const ShardingInfo& sharding_info) { + _sharding_info = sharding_info; } size_t get_shard_count() { return _shard_count; } size_t get_shard_id() { return _shard_id; } @@ -103,7 +124,7 @@ struct ReaderConfig { std::string file_list_path() { return _file_list_path; } std::shared_ptr meta_data_reader() { return _meta_data_reader; } ExternalSourceFileMode mode() { return _file_mode; } - std::pair get_last_batch_policy() { return _last_batch_info; } + const ShardingInfo& get_sharding_info() { return _sharding_info; } private: StorageType _type = StorageType::FILE_SYSTEM; @@ -123,7 +144,7 @@ struct ReaderConfig { std::string _file_list_path; //!< to read only files present in the file list std::shared_ptr _meta_data_reader = nullptr; ExternalSourceFileMode _file_mode = ExternalSourceFileMode::NONE; - std::pair _last_batch_info = {RocalBatchPolicy::FILL, true}; + ShardingInfo _sharding_info; #ifdef ROCAL_VIDEO VideoProperties _video_prop; #endif @@ -187,6 +208,10 @@ class Reader { virtual ~Reader() = default; + virtual std::string get_root_folder_path() { return {}; } + + virtual std::vector get_file_paths_from_meta_data_reader() { return {}; } + //! Returns the number of images in the last batch virtual size_t last_batch_padded_size() { return 0; } }; diff --git a/rocAL/source/api/rocal_api_data_loaders.cpp b/rocAL/source/api/rocal_api_data_loaders.cpp index 169ba0224..af78c923e 100644 --- a/rocAL/source/api/rocal_api_data_loaders.cpp +++ b/rocAL/source/api/rocal_api_data_loaders.cpp @@ -46,10 +46,11 @@ THE SOFTWARE. #ifdef ROCAL_AUDIO std::tuple evaluate_audio_data_set(StorageType storage_type, DecoderType decoder_type, - const std::string& source_path, const std::string& file_list_path) { + const std::string& source_path, const std::string& file_list_path, std::shared_ptr meta_data_reader) { AudioSourceEvaluator source_evaluator; auto reader_config = ReaderConfig(storage_type, source_path); reader_config.set_file_list_path(file_list_path); + reader_config.set_meta_data_reader(meta_data_reader); if (source_evaluator.Create(reader_config, DecoderConfig(decoder_type)) != AudioSourceEvaluatorStatus::OK) THROW("Initializing file source input evaluator failed") auto max_samples = source_evaluator.GetMaxSamples(); @@ -157,7 +158,7 @@ auto convert_last_batch_policy = [](RocalLastBatchPolicy last_batch_policy) { case ROCAL_LAST_BATCH_DROP: return RocalBatchPolicy::DROP; default: - THROW("Unsupported decoder mode" + TOSTR(last_batch_policy)) + THROW("Unsupported Last Batch Policy Mode" + TOSTR(last_batch_policy)) } }; @@ -175,7 +176,7 @@ rocalJpegFileSourceSingleShard( unsigned max_width, unsigned max_height, RocalDecoderType dec_type, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -208,9 +209,8 @@ rocalJpegFileSourceSingleShard( color_format); output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(shard_count); - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); - context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::FILE_SYSTEM, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, policy_info); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); + context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::FILE_SYSTEM, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, sharding_info); context->master_graph->set_loop(loop); if (is_output) { @@ -238,7 +238,7 @@ rocalJpegFileSource( unsigned max_width, unsigned max_height, RocalDecoderType dec_type, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -258,8 +258,7 @@ rocalJpegFileSource( } auto [width, height] = use_input_dimension ? std::make_tuple(max_width, max_height) : evaluate_image_data_set(decode_size_policy, StorageType::FILE_SYSTEM, DecoderType::TURBO_JPEG, source_path, ""); - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), height, width); INFO("Internal buffer size width = " + TOSTR(width) + " height = " + TOSTR(height) + " depth = " + TOSTR(num_of_planes)) @@ -271,7 +270,7 @@ rocalJpegFileSource( output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(1); - context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, "", std::map(), StorageType::FILE_SYSTEM, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, policy_info); + context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, "", std::map(), StorageType::FILE_SYSTEM, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, sharding_info); context->master_graph->set_loop(loop); if (is_output) { @@ -298,7 +297,7 @@ rocalSequenceReader( bool loop, unsigned step, unsigned stride, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; if (p_context == nullptr) { ERR("Invalid ROCAL context or invalid input image") @@ -338,10 +337,9 @@ rocalSequenceReader( output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(1); - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); - context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, "", std::map(), StorageType::SEQUENCE_FILE_SYSTEM, DecoderType::TURBO_JPEG, shuffle, loop, context->master_graph->sequence_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, policy_info, "", sequence_length, step, stride); + context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, "", std::map(), StorageType::SEQUENCE_FILE_SYSTEM, DecoderType::TURBO_JPEG, shuffle, loop, context->master_graph->sequence_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, sharding_info, "", sequence_length, step, stride); context->master_graph->set_loop(loop); if (is_output) { @@ -369,8 +367,7 @@ rocalSequenceReaderSingleShard( bool loop, unsigned step, unsigned stride, - std::pair last_batch_info) { - + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; if (p_context == nullptr) { ERR("Invalid ROCAL context or invalid input image") @@ -412,10 +409,8 @@ rocalSequenceReaderSingleShard( info.set_max_shape(); output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(shard_count); - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); - context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::SEQUENCE_FILE_SYSTEM, DecoderType::TURBO_JPEG, shuffle, loop, context->master_graph->sequence_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, policy_info, - std::map(), sequence_length, step, stride); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); + context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::SEQUENCE_FILE_SYSTEM, DecoderType::TURBO_JPEG, shuffle, loop, context->master_graph->sequence_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, sharding_info, std::map(), sequence_length, step, stride); context->master_graph->set_loop(loop); if (is_output) { @@ -443,7 +438,7 @@ rocalJpegCaffe2LMDBRecordSource( unsigned max_width, unsigned max_height, RocalDecoderType dec_type, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -473,9 +468,8 @@ rocalJpegCaffe2LMDBRecordSource( color_format); output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(1); - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); - context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, "", std::map(), StorageType::CAFFE2_LMDB_RECORD, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, policy_info); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); + context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, "", std::map(), StorageType::CAFFE2_LMDB_RECORD, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, sharding_info); context->master_graph->set_loop(loop); if (is_output) { @@ -504,7 +498,7 @@ rocalJpegCaffe2LMDBRecordSourceSingleShard( unsigned max_width, unsigned max_height, RocalDecoderType dec_type, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -529,8 +523,7 @@ rocalJpegCaffe2LMDBRecordSourceSingleShard( auto [width, height] = use_input_dimension ? std::make_tuple(max_width, max_height) : evaluate_image_data_set(decode_size_policy, StorageType::CAFFE2_LMDB_RECORD, DecoderType::TURBO_JPEG, source_path, ""); auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), height, width); INFO("Internal buffer size width = " + TOSTR(width) + " height = " + TOSTR(height) + " depth = " + TOSTR(num_of_planes)) - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), @@ -540,7 +533,7 @@ rocalJpegCaffe2LMDBRecordSourceSingleShard( output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(shard_count); - context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::CAFFE2_LMDB_RECORD, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, policy_info); + context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::CAFFE2_LMDB_RECORD, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, sharding_info); context->master_graph->set_loop(loop); if (is_output) { @@ -568,7 +561,7 @@ rocalJpegCaffeLMDBRecordSource( unsigned max_width, unsigned max_height, RocalDecoderType dec_type, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -590,8 +583,7 @@ rocalJpegCaffeLMDBRecordSource( auto [width, height] = use_input_dimension ? std::make_tuple(max_width, max_height) : evaluate_image_data_set(decode_size_policy, StorageType::CAFFE_LMDB_RECORD, DecoderType::TURBO_JPEG, source_path, ""); auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), height, width); INFO("Internal buffer size width = " + TOSTR(width) + " height = " + TOSTR(height) + " depth = " + TOSTR(num_of_planes)) - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), @@ -601,7 +593,7 @@ rocalJpegCaffeLMDBRecordSource( output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(1); - context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, "", std::map(), StorageType::CAFFE_LMDB_RECORD, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, policy_info); + context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, "", std::map(), StorageType::CAFFE_LMDB_RECORD, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, sharding_info); context->master_graph->set_loop(loop); @@ -631,7 +623,7 @@ rocalJpegCaffeLMDBRecordSourceSingleShard( unsigned max_width, unsigned max_height, RocalDecoderType dec_type, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -656,8 +648,7 @@ rocalJpegCaffeLMDBRecordSourceSingleShard( auto [width, height] = use_input_dimension ? std::make_tuple(max_width, max_height) : evaluate_image_data_set(decode_size_policy, StorageType::CAFFE_LMDB_RECORD, DecoderType::TURBO_JPEG, source_path, ""); auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), height, width); INFO("Internal buffer size width = " + TOSTR(width) + " height = " + TOSTR(height) + " depth = " + TOSTR(num_of_planes)) - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), @@ -667,7 +658,7 @@ rocalJpegCaffeLMDBRecordSourceSingleShard( output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(shard_count); - context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::CAFFE_LMDB_RECORD, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, policy_info); + context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::CAFFE_LMDB_RECORD, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, sharding_info); context->master_graph->set_loop(loop); if (is_output) { @@ -698,7 +689,7 @@ rocalJpegCaffeLMDBRecordSourcePartialSingleShard( RocalImageSizeEvaluationPolicy decode_size_policy, unsigned max_width, unsigned max_height, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -719,8 +710,7 @@ rocalJpegCaffeLMDBRecordSourcePartialSingleShard( auto [width, height] = use_input_dimension ? std::make_tuple(max_width, max_height) : evaluate_image_data_set(decode_size_policy, StorageType::CAFFE_LMDB_RECORD, DecoderType::FUSED_TURBO_JPEG, source_path, ""); auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), height, width); INFO("Internal buffer size width = " + TOSTR(width) + " height = " + TOSTR(height) + " depth = " + TOSTR(num_of_planes)) - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), @@ -730,7 +720,7 @@ rocalJpegCaffeLMDBRecordSourcePartialSingleShard( output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(shard_count); - context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::CAFFE_LMDB_RECORD, DecoderType::FUSED_TURBO_JPEG, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), num_attempts, area_factor, aspect_ratio, policy_info); + context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::CAFFE_LMDB_RECORD, DecoderType::FUSED_TURBO_JPEG, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), num_attempts, area_factor, aspect_ratio, sharding_info); context->master_graph->set_loop(loop); @@ -762,7 +752,7 @@ rocalJpegCaffe2LMDBRecordSourcePartialSingleShard( RocalImageSizeEvaluationPolicy decode_size_policy, unsigned max_width, unsigned max_height, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -783,8 +773,7 @@ rocalJpegCaffe2LMDBRecordSourcePartialSingleShard( auto [width, height] = use_input_dimension ? std::make_tuple(max_width, max_height) : evaluate_image_data_set(decode_size_policy, StorageType::CAFFE2_LMDB_RECORD, DecoderType::FUSED_TURBO_JPEG, source_path, ""); auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), height, width); INFO("Internal buffer size width = " + TOSTR(width) + " height = " + TOSTR(height) + " depth = " + TOSTR(num_of_planes)) - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), @@ -794,7 +783,7 @@ rocalJpegCaffe2LMDBRecordSourcePartialSingleShard( output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(shard_count); - context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::CAFFE2_LMDB_RECORD, DecoderType::FUSED_TURBO_JPEG, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), num_attempts, area_factor, aspect_ratio, policy_info); + context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::CAFFE2_LMDB_RECORD, DecoderType::FUSED_TURBO_JPEG, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), num_attempts, area_factor, aspect_ratio, sharding_info); context->master_graph->set_loop(loop); if (is_output) { @@ -825,7 +814,7 @@ rocalMXNetRecordSource( RocalImageSizeEvaluationPolicy decode_size_policy, unsigned max_width, unsigned max_height, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -846,8 +835,7 @@ rocalMXNetRecordSource( auto [width, height] = use_input_dimension ? std::make_tuple(max_width, max_height) : evaluate_image_data_set(decode_size_policy, StorageType::CAFFE_LMDB_RECORD, DecoderType::FUSED_TURBO_JPEG, source_path, ""); auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), height, width); INFO("Internal buffer size width = " + TOSTR(width) + " height = " + TOSTR(height) + " depth = " + TOSTR(num_of_planes)) - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), @@ -857,7 +845,7 @@ rocalMXNetRecordSource( output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(shard_count); - context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::CAFFE_LMDB_RECORD, DecoderType::FUSED_TURBO_JPEG, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), num_attempts, area_factor, aspect_ratio, policy_info); + context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::CAFFE_LMDB_RECORD, DecoderType::FUSED_TURBO_JPEG, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), num_attempts, area_factor, aspect_ratio, sharding_info); context->master_graph->set_loop(loop); @@ -886,7 +874,7 @@ rocalMXNetRecordSource( unsigned max_width, unsigned max_height, RocalDecoderType dec_type, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; if (p_context == nullptr) { ERR("Invalid ROCAL context or invalid input image") @@ -912,8 +900,7 @@ rocalMXNetRecordSource( auto [width, height] = use_input_dimension ? std::make_tuple(max_width, max_height) : evaluate_image_data_set(decode_size_policy, StorageType::MXNET_RECORDIO, DecoderType::TURBO_JPEG, source_path, ""); auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), height, width); INFO("Internal buffer size width = " + TOSTR(width) + " height = " + TOSTR(height) + " depth = " + TOSTR(num_of_planes)) - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), @@ -923,7 +910,7 @@ rocalMXNetRecordSource( output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(1); - context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, "", std::map(), StorageType::MXNET_RECORDIO, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, policy_info); + context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, "", std::map(), StorageType::MXNET_RECORDIO, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, sharding_info); context->master_graph->set_loop(loop); @@ -953,7 +940,7 @@ rocalMXNetRecordSourceSingleShard( unsigned max_width, unsigned max_height, RocalDecoderType dec_type, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; if (p_context == nullptr) { ERR("Invalid ROCAL context or invalid input image") @@ -982,8 +969,7 @@ rocalMXNetRecordSourceSingleShard( auto [width, height] = use_input_dimension ? std::make_tuple(max_width, max_height) : evaluate_image_data_set(decode_size_policy, StorageType::MXNET_RECORDIO, DecoderType::TURBO_JPEG, source_path, ""); auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), height, width); INFO("Internal buffer size width = " + TOSTR(width) + " height = " + TOSTR(height) + " depth = " + TOSTR(num_of_planes)) - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), @@ -993,7 +979,7 @@ rocalMXNetRecordSourceSingleShard( output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(shard_count); - context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::MXNET_RECORDIO, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, policy_info); + context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::MXNET_RECORDIO, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, sharding_info); context->master_graph->set_loop(loop); if (is_output) { @@ -1022,7 +1008,7 @@ rocalJpegCOCOFileSource( unsigned max_width, unsigned max_height, RocalDecoderType dec_type, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -1045,8 +1031,7 @@ rocalJpegCOCOFileSource( auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), height, width); INFO("Internal buffer size width = " + TOSTR(width) + " height = " + TOSTR(height) + " depth = " + TOSTR(num_of_planes)) - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), @@ -1056,7 +1041,7 @@ rocalJpegCOCOFileSource( output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(1); - context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, json_path, std::map(), StorageType::COCO_FILE_SYSTEM, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, policy_info); + context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, json_path, std::map(), StorageType::COCO_FILE_SYSTEM, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, sharding_info); context->master_graph->set_loop(loop); @@ -1087,7 +1072,7 @@ rocalJpegCOCOFileSourceSingleShard( unsigned max_width, unsigned max_height, RocalDecoderType dec_type, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -1112,8 +1097,7 @@ rocalJpegCOCOFileSourceSingleShard( auto [width, height] = use_input_dimension ? std::make_tuple(max_width, max_height) : evaluate_image_data_set(decode_size_policy, StorageType::COCO_FILE_SYSTEM, DecoderType::TURBO_JPEG, source_path, json_path); auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), height, width); INFO("Internal buffer size width = " + TOSTR(width) + " height = " + TOSTR(height) + " depth = " + TOSTR(num_of_planes)) - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), @@ -1123,7 +1107,7 @@ rocalJpegCOCOFileSourceSingleShard( output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(shard_count); - context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, json_path, StorageType::COCO_FILE_SYSTEM, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, policy_info); + context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, json_path, StorageType::COCO_FILE_SYSTEM, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, sharding_info); context->master_graph->set_loop(loop); if (is_output) { @@ -1153,7 +1137,7 @@ rocalFusedJpegCrop( RocalImageSizeEvaluationPolicy decode_size_policy, unsigned max_width, unsigned max_height, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -1171,8 +1155,7 @@ rocalFusedJpegCrop( auto [width, height] = use_input_dimension ? std::make_tuple(max_width, max_height) : evaluate_image_data_set(decode_size_policy, StorageType::FILE_SYSTEM, DecoderType::FUSED_TURBO_JPEG, source_path, ""); auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), height, width); - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), @@ -1181,7 +1164,7 @@ rocalFusedJpegCrop( color_format); output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(1); - context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, "", StorageType::FILE_SYSTEM, DecoderType::FUSED_TURBO_JPEG, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), num_attempts, area_factor, aspect_ratio, policy_info); + context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, "", StorageType::FILE_SYSTEM, DecoderType::FUSED_TURBO_JPEG, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), num_attempts, area_factor, aspect_ratio, sharding_info); context->master_graph->set_loop(loop); if (is_output) { @@ -1212,7 +1195,7 @@ rocalJpegCOCOFileSourcePartial( RocalImageSizeEvaluationPolicy decode_size_policy, unsigned max_width, unsigned max_height, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -1231,8 +1214,7 @@ rocalJpegCOCOFileSourcePartial( auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), height, width); INFO("Internal buffer size width = " + TOSTR(width) + " height = " + TOSTR(height) + " depth = " + TOSTR(num_of_planes)) - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), @@ -1242,7 +1224,7 @@ rocalJpegCOCOFileSourcePartial( output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(1); - context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, json_path, StorageType::COCO_FILE_SYSTEM, DecoderType::FUSED_TURBO_JPEG, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), num_attempts, area_factor, aspect_ratio, policy_info); + context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, json_path, StorageType::COCO_FILE_SYSTEM, DecoderType::FUSED_TURBO_JPEG, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), num_attempts, area_factor, aspect_ratio, sharding_info); context->master_graph->set_loop(loop); @@ -1275,7 +1257,7 @@ rocalJpegCOCOFileSourcePartialSingleShard( RocalImageSizeEvaluationPolicy decode_size_policy, unsigned max_width, unsigned max_height, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -1296,8 +1278,7 @@ rocalJpegCOCOFileSourcePartialSingleShard( auto [width, height] = use_input_dimension ? std::make_tuple(max_width, max_height) : evaluate_image_data_set(decode_size_policy, StorageType::COCO_FILE_SYSTEM, DecoderType::FUSED_TURBO_JPEG, source_path, json_path); auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), height, width); - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), @@ -1307,7 +1288,7 @@ rocalJpegCOCOFileSourcePartialSingleShard( output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(shard_count); - context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, json_path, StorageType::COCO_FILE_SYSTEM, DecoderType::FUSED_TURBO_JPEG, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), num_attempts, area_factor, aspect_ratio, policy_info); + context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, json_path, StorageType::COCO_FILE_SYSTEM, DecoderType::FUSED_TURBO_JPEG, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), num_attempts, area_factor, aspect_ratio, sharding_info); context->master_graph->set_loop(loop); @@ -1338,7 +1319,7 @@ rocalJpegTFRecordSource( unsigned max_width, unsigned max_height, RocalDecoderType dec_type, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -1366,8 +1347,7 @@ rocalJpegTFRecordSource( auto [width, height] = use_input_dimension ? std::make_tuple(max_width, max_height) : evaluate_image_data_set(decode_size_policy, StorageType::TF_RECORD, DecoderType::TURBO_JPEG, source_path, ""); auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), height, width); INFO("Internal buffer size width = " + TOSTR(width) + " height = " + TOSTR(height) + " depth = " + TOSTR(num_of_planes)) - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), @@ -1377,7 +1357,7 @@ rocalJpegTFRecordSource( output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(1); - context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, "", feature_key_map, StorageType::TF_RECORD, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), false, policy_info); + context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, "", feature_key_map, StorageType::TF_RECORD, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), false, sharding_info); context->master_graph->set_loop(loop); if (is_output) { @@ -1406,7 +1386,7 @@ rocalJpegTFRecordSourceSingleShard( unsigned max_width, unsigned max_height, RocalDecoderType dec_type, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -1431,8 +1411,7 @@ rocalJpegTFRecordSourceSingleShard( auto [width, height] = use_input_dimension ? std::make_tuple(max_width, max_height) : evaluate_image_data_set(decode_size_policy, StorageType::TF_RECORD, DecoderType::TURBO_JPEG, source_path, ""); auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), height, width); INFO("Internal buffer size width = " + TOSTR(width) + " height = " + TOSTR(height) + " depth = " + TOSTR(num_of_planes)) - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), @@ -1442,7 +1421,7 @@ rocalJpegTFRecordSourceSingleShard( output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(shard_count); - context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::TF_RECORD, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, policy_info); + context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::TF_RECORD, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, sharding_info); context->master_graph->set_loop(loop); if (is_output) { @@ -1470,7 +1449,7 @@ rocalRawTFRecordSource( unsigned out_width, unsigned out_height, const char* record_name_prefix, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; if (p_context == nullptr) { ERR("Invalid ROCAL context or invalid input image") @@ -1492,8 +1471,7 @@ rocalRawTFRecordSource( } auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), out_height, out_width); - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), @@ -1503,7 +1481,7 @@ rocalRawTFRecordSource( output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(1); - context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, "", feature_key_map, StorageType::TF_RECORD, DecoderType::SKIP_DECODE, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), false, policy_info, record_name_prefix, 0, 0, 0); + context->master_graph->add_node({}, {output})->init(internal_shard_count, cpu_num_threads, source_path, "", feature_key_map, StorageType::TF_RECORD, DecoderType::SKIP_DECODE, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), false, sharding_info, record_name_prefix, 0, 0, 0); context->master_graph->set_loop(loop); if (is_output) { @@ -1530,7 +1508,7 @@ rocalRawTFRecordSourceSingleShard( bool loop, unsigned out_width, unsigned out_height, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -1548,8 +1526,7 @@ rocalRawTFRecordSourceSingleShard( auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), out_height, out_width); INFO("Internal buffer size width = " + TOSTR(out_width) + " height = " + TOSTR(out_height) + " depth = " + TOSTR(num_of_planes)) - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), RocalTensorDataType::UINT8, @@ -1558,7 +1535,7 @@ rocalRawTFRecordSourceSingleShard( output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(shard_count); - context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::TF_RECORD, DecoderType::SKIP_DECODE, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), false, policy_info); + context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::TF_RECORD, DecoderType::SKIP_DECODE, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), false, sharding_info); context->master_graph->set_loop(loop); if (is_output) { @@ -1589,7 +1566,7 @@ rocalFusedJpegCropSingleShard( RocalImageSizeEvaluationPolicy decode_size_policy, unsigned max_width, unsigned max_height, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -1610,8 +1587,7 @@ rocalFusedJpegCropSingleShard( auto [width, height] = use_input_dimension ? std::make_tuple(max_width, max_height) : evaluate_image_data_set(decode_size_policy, StorageType::FILE_SYSTEM, DecoderType::FUSED_TURBO_JPEG, source_path, ""); auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), height, width); - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), RocalTensorDataType::UINT8, @@ -1619,7 +1595,7 @@ rocalFusedJpegCropSingleShard( color_format); output = context->master_graph->create_loader_output_tensor(info); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(shard_count); - context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::FILE_SYSTEM, DecoderType::FUSED_TURBO_JPEG, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), num_attempts, area_factor, aspect_ratio, policy_info); + context->master_graph->add_node({}, {output})->init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::FILE_SYSTEM, DecoderType::FUSED_TURBO_JPEG, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), num_attempts, area_factor, aspect_ratio, sharding_info); context->master_graph->set_loop(loop); if (is_output) { @@ -1648,7 +1624,7 @@ rocalVideoFileSource( unsigned step, unsigned stride, bool file_list_frame_num, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; if (p_context == nullptr) { ERR("Invalid ROCAL context or invalid input image") @@ -1714,7 +1690,7 @@ rocalVideoFileSourceSingleShard( unsigned step, unsigned stride, bool file_list_frame_num, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; if (p_context == nullptr) { ERR("Invalid ROCAL context") @@ -1792,7 +1768,7 @@ rocalVideoFileResize( unsigned resize_shorter, unsigned resize_longer, RocalResizeInterpolationType interpolation_type, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* resize_output = nullptr; if (p_context == nullptr) { ERR("Invalid ROCAL context or invalid input image") @@ -1945,7 +1921,7 @@ rocalVideoFileResizeSingleShard( unsigned resize_shorter, unsigned resize_longer, RocalResizeInterpolationType interpolation_type, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* resize_output = nullptr; if (p_context == nullptr) { ERR("Invalid ROCAL context or invalid input image") @@ -2138,7 +2114,7 @@ rocalJpegExternalFileSource( unsigned max_height, RocalDecoderType dec_type, RocalExternalSourceMode external_source_mode, - std::pair last_batch_info) { + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -2158,9 +2134,7 @@ rocalJpegExternalFileSource( auto [width, height] = std::make_tuple(max_width, max_height); auto [color_format, tensor_layout, dims, num_of_planes] = convert_color_format(rocal_color_format, context->user_batch_size(), height, width); INFO("Internal buffer size width = " + TOSTR(width) + " height = " + TOSTR(height) + " depth = " + TOSTR(num_of_planes)) - auto last_batch_policy = convert_last_batch_policy(last_batch_info.first); - std::pair policy_info = std::make_pair(last_batch_policy, last_batch_info.second); - + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto info = TensorInfo(std::move(dims), context->master_graph->mem_type(), RocalTensorDataType::UINT8, @@ -2168,12 +2142,10 @@ rocalJpegExternalFileSource( color_format); output = context->master_graph->create_loader_output_tensor(info); context->master_graph->set_external_source_reader_flag(); - + unsigned shard_count = 1; // Hardcoding the shard count to 1 for now. auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(shard_count); - context->master_graph->add_node({}, {output})->init(shard_count, cpu_num_threads, "", "", std::map(), StorageType::EXTERNAL_FILE_SOURCE, - decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), - decoder_keep_original, policy_info, "", 0, 0, 0, ExternalSourceFileMode(external_source_mode)); + context->master_graph->add_node({}, {output})->init(shard_count, cpu_num_threads, "", "", std::map(), StorageType::EXTERNAL_FILE_SOURCE, decType, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), decoder_keep_original, sharding_info, "", 0, 0, 0, ExternalSourceFileMode(external_source_mode)); context->master_graph->set_loop(loop); if (is_output) { @@ -2198,7 +2170,11 @@ rocalAudioFileSourceSingleShard( bool is_output, bool shuffle, bool loop, - bool downmix) { + bool downmix, + RocalImageSizeEvaluationPolicy decode_size_policy, + unsigned max_decoded_samples, + unsigned max_decoded_channels, + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { @@ -2207,7 +2183,14 @@ rocalAudioFileSourceSingleShard( THROW("Shard count should be bigger than 0") if (shard_id >= shard_count) THROW("Shard id should be smaller than shard count") - auto [max_sample_length, max_channels] = evaluate_audio_data_set(StorageType::FILE_SYSTEM, DecoderType::AUDIO_SOFTWARE_DECODE, source_path, source_file_list_path); + bool use_input_dimension = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE) || (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED); + + if (use_input_dimension && (max_decoded_samples == 0 || max_decoded_channels == 0)) { + THROW("Invalid input max width and height"); + } else { + LOG("User input size " + TOSTR(max_decoded_samples) + " x " + TOSTR(max_decoded_channels)) + } + auto [max_sample_length, max_channels] = use_input_dimension ? std::make_tuple(max_decoded_samples, max_decoded_channels) : evaluate_audio_data_set(StorageType::FILE_SYSTEM, DecoderType::AUDIO_SOFTWARE_DECODE, source_path, source_file_list_path, context->master_graph->meta_data_reader()); INFO("Internal buffer size for audio samples = " + TOSTR(max_sample_length) + " and channels = " + TOSTR(max_channels)) RocalTensorDataType tensor_data_type = RocalTensorDataType::FP32; std::vector dims = {context->user_batch_size(), max_sample_length, max_channels}; @@ -2218,7 +2201,8 @@ rocalAudioFileSourceSingleShard( output = context->master_graph->create_loader_output_tensor(info); output->reset_audio_sample_rate(); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(shard_count); - context->master_graph->add_node({}, {output})->Init(shard_id, shard_count, cpu_num_threads, source_path, "", StorageType::FILE_SYSTEM, DecoderType::AUDIO_SOFTWARE_DECODE, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader()); + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); + context->master_graph->add_node({}, {output})->Init(shard_id, shard_count, cpu_num_threads, source_path, source_file_list_path, StorageType::FILE_SYSTEM, DecoderType::AUDIO_SOFTWARE_DECODE, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), sharding_info); context->master_graph->set_loop(loop); if (downmix && (max_channels > 1)) { TensorInfo output_info = info; @@ -2255,12 +2239,23 @@ rocalAudioFileSource( bool is_output, bool shuffle, bool loop, - bool downmix) { + bool downmix, + RocalImageSizeEvaluationPolicy decode_size_policy, + unsigned max_decoded_samples, + unsigned max_decoded_channels, + RocalShardingInfo rocal_sharding_info) { Tensor* output = nullptr; auto context = static_cast(p_context); try { #ifdef ROCAL_AUDIO - auto [max_sample_length, max_channels] = evaluate_audio_data_set(StorageType::FILE_SYSTEM, DecoderType::AUDIO_SOFTWARE_DECODE, source_path, source_file_list_path); + bool use_input_dimension = (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE) || (decode_size_policy == ROCAL_USE_USER_GIVEN_SIZE_RESTRICTED); + + if (use_input_dimension && (max_decoded_samples == 0 || max_decoded_channels == 0)) { + THROW("Invalid input max width and height"); + } else { + LOG("User input size " + TOSTR(max_decoded_samples) + " x " + TOSTR(max_decoded_channels)) + } + auto [max_sample_length, max_channels] = use_input_dimension ? std::make_tuple(max_decoded_samples, max_decoded_channels) : evaluate_audio_data_set(StorageType::FILE_SYSTEM, DecoderType::AUDIO_SOFTWARE_DECODE, source_path, source_file_list_path, context->master_graph->meta_data_reader()); INFO("Internal buffer size for audio samples = " + TOSTR(max_sample_length) + " and channels = " + TOSTR(max_channels)) RocalTensorDataType tensor_data_type = RocalTensorDataType::FP32; std::vector dims = {context->user_batch_size(), max_sample_length, max_channels}; @@ -2273,9 +2268,9 @@ rocalAudioFileSource( if (shard_count < 1) THROW("internal shard count should be bigger than 0") - + ShardingInfo sharding_info(convert_last_batch_policy(rocal_sharding_info.last_batch_policy), rocal_sharding_info.pad_last_batch_repeated, rocal_sharding_info.stick_to_shard, rocal_sharding_info.shard_size); auto cpu_num_threads = context->master_graph->calculate_cpu_num_threads(shard_count); - context->master_graph->add_node({}, {output})->Init(shard_count, cpu_num_threads, source_path, "", StorageType::FILE_SYSTEM, DecoderType::AUDIO_SOFTWARE_DECODE, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader()); + context->master_graph->add_node({}, {output})->Init(shard_count, cpu_num_threads, source_path, source_file_list_path, StorageType::FILE_SYSTEM, DecoderType::AUDIO_SOFTWARE_DECODE, shuffle, loop, context->user_batch_size(), context->master_graph->mem_type(), context->master_graph->meta_data_reader(), sharding_info); context->master_graph->set_loop(loop); if (downmix && (max_channels > 1)) { TensorInfo output_info = info; diff --git a/rocAL/source/loaders/audio/audio_loader.cpp b/rocAL/source/loaders/audio/audio_loader.cpp index 64a01247a..eda29bd55 100644 --- a/rocAL/source/loaders/audio/audio_loader.cpp +++ b/rocAL/source/loaders/audio/audio_loader.cpp @@ -199,8 +199,13 @@ AudioLoader::load_routine() { return LoaderModuleStatus::OK; } +// Uses the same condition in audio_read_and_decode.cpp to check if enough samples are present bool AudioLoader::is_out_of_data() { - return (remaining_count() < 0); + return (remaining_count() < _batch_size); +} + +size_t AudioLoader::last_batch_padded_size() { + return _audio_loader->last_batch_padded_size(); } LoaderModuleStatus diff --git a/rocAL/source/loaders/audio/audio_loader_sharded.cpp b/rocAL/source/loaders/audio/audio_loader_sharded.cpp index e067e1b50..b99116875 100644 --- a/rocAL/source/loaders/audio/audio_loader_sharded.cpp +++ b/rocAL/source/loaders/audio/audio_loader_sharded.cpp @@ -48,6 +48,13 @@ AudioLoaderSharded::~AudioLoaderSharded() { _loaders.clear(); } +size_t AudioLoaderSharded::last_batch_padded_size() { + size_t sum = 0; + for (auto& loader : _loaders) + sum += loader->last_batch_padded_size(); + return sum; +} + void AudioLoaderSharded::fast_forward_through_empty_loaders() { int loaders_count = _loaders.size(); // reject empty loaders and get to a loader that still has audios to play diff --git a/rocAL/source/loaders/audio/audio_read_and_decode.cpp b/rocAL/source/loaders/audio/audio_read_and_decode.cpp index 2fbff4ac6..e5acf52c0 100644 --- a/rocAL/source/loaders/audio/audio_read_and_decode.cpp +++ b/rocAL/source/loaders/audio/audio_read_and_decode.cpp @@ -72,6 +72,10 @@ AudioReadAndDecode::Count() { return _reader->count_items(); } +size_t AudioReadAndDecode::last_batch_padded_size() { + return _reader->last_batch_padded_size(); +} + LoaderModuleStatus AudioReadAndDecode::Load(float *audio_buffer, DecodedDataInfo& audio_info, diff --git a/rocAL/source/loaders/audio/audio_source_evaluator.cpp b/rocAL/source/loaders/audio/audio_source_evaluator.cpp index 7a7e3d409..6fd6f5e80 100644 --- a/rocAL/source/loaders/audio/audio_source_evaluator.cpp +++ b/rocAL/source/loaders/audio/audio_source_evaluator.cpp @@ -47,27 +47,50 @@ AudioSourceEvaluator::Create(ReaderConfig reader_cfg, DecoderConfig decoder_cfg) void AudioSourceEvaluator::FindMaxDimension() { _reader->reset(); - while (_reader->count_items()) { - size_t fsize = _reader->open(); - if (!fsize) continue; - auto file_name = _reader->file_path(); - if (_decoder->Initialize(file_name.c_str()) != AudioDecoder::Status::OK) { - WRN("Could not initialize audio decoder for file : " + _reader->id()) - continue; + auto root_folder_path = _reader->get_root_folder_path(); + auto relative_file_paths = _reader->get_file_paths_from_meta_data_reader(); + if ((relative_file_paths.size() > 0)) { + for (auto rel_file_path : relative_file_paths) { + std::string file_name = root_folder_path + "/" + rel_file_path; + if (_decoder->Initialize(file_name.c_str()) != AudioDecoder::Status::OK) { + WRN("Could not initialize audio decoder for file : " + _reader->id()) + continue; + } + int samples, channels; + float sample_rate; + if (_decoder->DecodeInfo(&samples, &channels, &sample_rate) != AudioDecoder::Status::OK) { + WRN("Could not decode the header of the: " + _reader->id()) + continue; + } + if (samples <= 0 || channels <= 0) + continue; + _samples_max = std::max(samples, _samples_max); + _channels_max = std::max(channels, _channels_max); + _decoder->Release(); } - int samples, channels; - float sample_rate; - if (_decoder->DecodeInfo(&samples, &channels, &sample_rate) != AudioDecoder::Status::OK) { - WRN("Could not decode the header of the: " + _reader->id()) - continue; + } else { + while (_reader->count_items()) { + size_t fsize = _reader->open(); + if (!fsize) continue; + auto file_name = _reader->file_path(); + if (_decoder->Initialize(file_name.c_str()) != AudioDecoder::Status::OK) { + WRN("Could not initialize audio decoder for file : " + _reader->id()) + continue; + } + int samples, channels; + float sample_rate; + if (_decoder->DecodeInfo(&samples, &channels, &sample_rate) != AudioDecoder::Status::OK) { + WRN("Could not decode the header of the: " + _reader->id()) + continue; + } + if (samples <= 0 || channels <= 0) + continue; + _samples_max = std::max(samples, _samples_max); + _channels_max = std::max(channels, _channels_max); + _decoder->Release(); } - if (samples <= 0 || channels <= 0) - continue; - _samples_max = std::max(samples, _samples_max); - _channels_max = std::max(channels, _channels_max); - _decoder->Release(); } - // return the reader read pointer to the begining of the resource + // return the reader read pointer to the beginning of the resource _reader->reset(); } #endif diff --git a/rocAL/source/loaders/audio/node_audio_loader.cpp b/rocAL/source/loaders/audio/node_audio_loader.cpp index c1d3cc511..83f0367f6 100644 --- a/rocAL/source/loaders/audio/node_audio_loader.cpp +++ b/rocAL/source/loaders/audio/node_audio_loader.cpp @@ -29,8 +29,10 @@ AudioLoaderNode::AudioLoaderNode(Tensor *output, void *device_resources) : Node( _loader_module = std::make_shared(device_resources); } -void AudioLoaderNode::Init(unsigned internal_shard_count, unsigned cpu_num_threads, const std::string &source_path, const std::string &file_list_path, StorageType storage_type, - DecoderType decoder_type, bool shuffle, bool loop, size_t load_batch_count, RocalMemType mem_type, std::shared_ptr meta_data_reader) { +void AudioLoaderNode::Init(unsigned internal_shard_count, unsigned cpu_num_threads, const std::string &source_path, const std::string &file_list_path, + StorageType storage_type, DecoderType decoder_type, bool shuffle, bool loop, + size_t load_batch_count, RocalMemType mem_type, std::shared_ptr meta_data_reader, + const ShardingInfo& sharding_info) { if (!_loader_module) THROW("ERROR: loader module is not set for AudioLoaderNode, cannot initialize") if (internal_shard_count < 1) @@ -43,6 +45,7 @@ void AudioLoaderNode::Init(unsigned internal_shard_count, unsigned cpu_num_threa reader_cfg.set_meta_data_reader(meta_data_reader); reader_cfg.set_cpu_num_threads(cpu_num_threads); reader_cfg.set_file_list_path(file_list_path); + reader_cfg.set_sharding_info(sharding_info); _loader_module->initialize(reader_cfg, DecoderConfig(decoder_type), mem_type, _batch_size, false); _loader_module->start_loading(); } diff --git a/rocAL/source/loaders/audio/node_audio_loader_single_shard.cpp b/rocAL/source/loaders/audio/node_audio_loader_single_shard.cpp index c7238fab8..8795570f2 100644 --- a/rocAL/source/loaders/audio/node_audio_loader_single_shard.cpp +++ b/rocAL/source/loaders/audio/node_audio_loader_single_shard.cpp @@ -31,7 +31,7 @@ AudioLoaderSingleShardNode::AudioLoaderSingleShardNode(Tensor *output, void *dev void AudioLoaderSingleShardNode::Init(unsigned shard_id, unsigned shard_count, unsigned cpu_num_threads, const std::string &source_path, const std::string &file_list_path, StorageType storage_type, DecoderType decoder_type, bool shuffle, bool loop, size_t load_batch_count, - RocalMemType mem_type, std::shared_ptr meta_data_reader) { + RocalMemType mem_type, std::shared_ptr meta_data_reader, const ShardingInfo& sharding_info) { if (!_loader_module) THROW("ERROR: loader module is not set for AudioLoaderNode, cannot initialize") if (shard_count < 1) @@ -47,6 +47,7 @@ void AudioLoaderSingleShardNode::Init(unsigned shard_id, unsigned shard_count, u reader_cfg.set_meta_data_reader(meta_data_reader); reader_cfg.set_cpu_num_threads(cpu_num_threads); reader_cfg.set_file_list_path(file_list_path); + reader_cfg.set_sharding_info(sharding_info); _loader_module->initialize(reader_cfg, DecoderConfig(decoder_type), mem_type, _batch_size); _loader_module->start_loading(); } diff --git a/rocAL/source/loaders/image/node_fused_jpeg_crop.cpp b/rocAL/source/loaders/image/node_fused_jpeg_crop.cpp index df0bd59da..ecf87b552 100644 --- a/rocAL/source/loaders/image/node_fused_jpeg_crop.cpp +++ b/rocAL/source/loaders/image/node_fused_jpeg_crop.cpp @@ -30,7 +30,7 @@ FusedJpegCropNode::FusedJpegCropNode(Tensor *output, void *device_resources) : N void FusedJpegCropNode::init(unsigned internal_shard_count, unsigned cpu_num_threads, const std::string &source_path, const std::string &json_path, StorageType storage_type, DecoderType decoder_type, bool shuffle, bool loop, size_t load_batch_count, RocalMemType mem_type, std::shared_ptr meta_data_reader, - unsigned num_attempts, std::vector &random_area, std::vector &random_aspect_ratio, std::pair last_batch_info) { + unsigned num_attempts, std::vector &random_area, std::vector &random_aspect_ratio, const ShardingInfo& sharding_info) { if (!_loader_module) THROW("ERROR: loader module is not set for FusedJpegCropNode, cannot initialize") if (internal_shard_count < 1) @@ -42,7 +42,7 @@ void FusedJpegCropNode::init(unsigned internal_shard_count, unsigned cpu_num_thr reader_cfg.set_cpu_num_threads(cpu_num_threads); reader_cfg.set_batch_count(load_batch_count); reader_cfg.set_meta_data_reader(meta_data_reader); - reader_cfg.set_last_batch_policy(last_batch_info); + reader_cfg.set_sharding_info(sharding_info); auto decoder_cfg = DecoderConfig(decoder_type); decoder_cfg.set_random_area(random_area); diff --git a/rocAL/source/loaders/image/node_fused_jpeg_crop_single_shard.cpp b/rocAL/source/loaders/image/node_fused_jpeg_crop_single_shard.cpp index 89fcb99bf..7cc2a29c6 100644 --- a/rocAL/source/loaders/image/node_fused_jpeg_crop_single_shard.cpp +++ b/rocAL/source/loaders/image/node_fused_jpeg_crop_single_shard.cpp @@ -30,7 +30,7 @@ FusedJpegCropSingleShardNode::FusedJpegCropSingleShardNode(Tensor *output, void void FusedJpegCropSingleShardNode::init(unsigned shard_id, unsigned shard_count, unsigned cpu_num_threads, const std::string &source_path, const std::string &json_path, StorageType storage_type, DecoderType decoder_type, bool shuffle, bool loop, size_t load_batch_count, RocalMemType mem_type, std::shared_ptr meta_data_reader, - unsigned num_attempts, std::vector &area_factor, std::vector &aspect_ratio, std::pair last_batch_info) { + unsigned num_attempts, std::vector &area_factor, std::vector &aspect_ratio, const ShardingInfo& sharding_info) { if (!_loader_module) THROW("ERROR: loader module is not set for FusedJpegCropSingleShardNode, cannot initialize") if (shard_count < 1) @@ -45,8 +45,7 @@ void FusedJpegCropSingleShardNode::init(unsigned shard_id, unsigned shard_count, reader_cfg.set_cpu_num_threads(cpu_num_threads); reader_cfg.set_batch_count(load_batch_count); reader_cfg.set_meta_data_reader(meta_data_reader); - reader_cfg.set_last_batch_policy(last_batch_info); - + reader_cfg.set_sharding_info(sharding_info); auto decoder_cfg = DecoderConfig(decoder_type); decoder_cfg.set_random_area(area_factor); diff --git a/rocAL/source/loaders/image/node_image_loader.cpp b/rocAL/source/loaders/image/node_image_loader.cpp index 9d2676240..96c614067 100644 --- a/rocAL/source/loaders/image/node_image_loader.cpp +++ b/rocAL/source/loaders/image/node_image_loader.cpp @@ -29,7 +29,7 @@ ImageLoaderNode::ImageLoaderNode(Tensor *output, void *device_resources) : Node( } void ImageLoaderNode::init(unsigned internal_shard_count, unsigned cpu_num_threads, const std::string &source_path, const std::string &json_path, const std::map feature_key_map, StorageType storage_type, DecoderType decoder_type, bool shuffle, bool loop, - size_t load_batch_count, RocalMemType mem_type, std::shared_ptr meta_data_reader, bool decoder_keep_orig, std::pair last_batch_info, const char *file_prefix, unsigned sequence_length, unsigned step, unsigned stride, ExternalSourceFileMode external_file_mode) { + size_t load_batch_count, RocalMemType mem_type, std::shared_ptr meta_data_reader, bool decoder_keep_orig, const ShardingInfo& sharding_info, const char *file_prefix, unsigned sequence_length, unsigned step, unsigned stride, ExternalSourceFileMode external_file_mode) { if (!_loader_module) THROW("ERROR: loader module is not set for ImageLoaderNode, cannot initialize") if (internal_shard_count < 1) @@ -47,7 +47,7 @@ void ImageLoaderNode::init(unsigned internal_shard_count, unsigned cpu_num_threa reader_cfg.set_frame_step(step); reader_cfg.set_frame_stride(stride); reader_cfg.set_external_filemode(external_file_mode); - reader_cfg.set_last_batch_policy(last_batch_info); + reader_cfg.set_sharding_info(sharding_info); _loader_module->initialize(reader_cfg, DecoderConfig(decoder_type), mem_type, _batch_size, decoder_keep_orig); diff --git a/rocAL/source/loaders/image/node_image_loader_single_shard.cpp b/rocAL/source/loaders/image/node_image_loader_single_shard.cpp index 9b1be455a..0560805b5 100644 --- a/rocAL/source/loaders/image/node_image_loader_single_shard.cpp +++ b/rocAL/source/loaders/image/node_image_loader_single_shard.cpp @@ -30,7 +30,7 @@ ImageLoaderSingleShardNode::ImageLoaderSingleShardNode(Tensor *output, void *dev void ImageLoaderSingleShardNode::init(unsigned shard_id, unsigned shard_count, unsigned cpu_num_threads, const std::string &source_path, const std::string &json_path, StorageType storage_type, DecoderType decoder_type, bool shuffle, bool loop, size_t load_batch_count, RocalMemType mem_type, std::shared_ptr meta_data_reader, - bool decoder_keep_original, std::pair last_batch_info, const std::map feature_key_map, unsigned sequence_length, unsigned step, unsigned stride, ExternalSourceFileMode external_file_mode) { + bool decoder_keep_original, const ShardingInfo& sharding_info, const std::map feature_key_map, unsigned sequence_length, unsigned step, unsigned stride, ExternalSourceFileMode external_file_mode) { if (!_loader_module) THROW("ERROR: loader module is not set for ImageLoaderNode, cannot initialize") if (shard_count < 1) @@ -50,7 +50,7 @@ void ImageLoaderSingleShardNode::init(unsigned shard_id, unsigned shard_count, u reader_cfg.set_frame_step(step); reader_cfg.set_frame_stride(stride); reader_cfg.set_external_filemode(external_file_mode); - reader_cfg.set_last_batch_policy(last_batch_info); + reader_cfg.set_sharding_info(sharding_info); _loader_module->initialize(reader_cfg, DecoderConfig(decoder_type), mem_type, _batch_size, decoder_keep_original); diff --git a/rocAL/source/meta_data/text_file_meta_data_reader.cpp b/rocAL/source/meta_data/text_file_meta_data_reader.cpp index f326f3d9a..49ceb369a 100644 --- a/rocAL/source/meta_data/text_file_meta_data_reader.cpp +++ b/rocAL/source/meta_data/text_file_meta_data_reader.cpp @@ -76,7 +76,7 @@ void TextFileMetaDataReader::read_all(const std::string &path) { std::string file_name; if (!(line_ss >> file_name >> label)) continue; - // process pair (file_name, label) + _relative_file_path.push_back(file_name); // to be used in file source reader to reduce I/O operations auto last_id = file_name; auto last_slash_idx = last_id.find_last_of("\\/"); if (std::string::npos != last_slash_idx) { diff --git a/rocAL/source/pipeline/tensor.cpp b/rocAL/source/pipeline/tensor.cpp index 619317211..70fbc26bf 100644 --- a/rocAL/source/pipeline/tensor.cpp +++ b/rocAL/source/pipeline/tensor.cpp @@ -449,20 +449,20 @@ unsigned Tensor::copy_data(void *user_buffer, RocalOutputMemType external_mem_ty return 0; } -unsigned Tensor::copy_data(void *user_buffer, uint max_rows, uint max_cols) { +unsigned Tensor::copy_data(void *user_buffer, uint x_offset, uint y_offset, uint roi_width, uint roi_height) { if (_mem_handle == nullptr) return 0; // TODO : Handle this case for HIP buffer auto max_shape_rows = _info.max_shape().at(1); auto dtype_size = _info.data_type_size(); auto num_of_bytes_max_rows = max_shape_rows * dtype_size; auto src_stride = (_info.max_shape().at(0) * num_of_bytes_max_rows); - auto num_of_bytes_rows = max_cols * dtype_size; - auto dst_stride = (max_rows * num_of_bytes_rows); + auto num_of_bytes_rows = roi_width * dtype_size; + auto dst_stride = (roi_height * num_of_bytes_rows); for (uint i = 0; i < _info._batch_size; i++) { - auto temp_src_ptr = static_cast(_mem_handle) + i * src_stride; + auto temp_src_ptr = static_cast(_mem_handle) + i * src_stride + _info.max_shape().at(0) * y_offset + x_offset; auto temp_dst_ptr = static_cast(user_buffer) + i * dst_stride; - for (uint height = 0; height < max_rows; height++) { + for (uint height = 0; height < roi_height; height++) { memcpy(temp_dst_ptr, temp_src_ptr, num_of_bytes_rows); temp_src_ptr += num_of_bytes_max_rows; temp_dst_ptr += num_of_bytes_rows; diff --git a/rocAL/source/readers/file_source_reader.cpp b/rocAL/source/readers/file_source_reader.cpp index 564bef03c..e26b32988 100644 --- a/rocAL/source/readers/file_source_reader.cpp +++ b/rocAL/source/readers/file_source_reader.cpp @@ -23,6 +23,7 @@ THE SOFTWARE. #include #include #include +#include #include "pipeline/commons.h" #include "readers/file_source_reader.h" #include "pipeline/filesystem.h" @@ -35,55 +36,81 @@ FileSourceReader::FileSourceReader() { _current_file_size = 0; _current_fPtr = nullptr; _loop = false; - _file_id = 0; _shuffle = false; _file_count_all_shards = 0; } unsigned FileSourceReader::count_items() { - if (_loop) - return _file_names.size(); - - int ret = ((int)_file_names.size() - _read_counter); + int ret = 0; + if (_shard_size == -1) { // When shard_size is set to -1, The shard_size variable is not used + if (_loop) return largest_shard_size_without_padding(); // Return the size of the largest shard amongst all the shard's size + int size = std::max(largest_shard_size_without_padding(), _batch_size); + ret = (size - _read_counter); + // Formula used to calculate - [_last_batch_padded_size = _batch_size - (_shard_size % _batch_size) ] + // Since "size" doesnt involve padding - we add the count of padded samples to the number of remaining elements + // which equals to the shard size with padding + if (_last_batch_info.last_batch_policy == RocalBatchPolicy::PARTIAL || _last_batch_info.last_batch_policy == RocalBatchPolicy::FILL) { + ret += _last_batch_padded_size; + } else if (_last_batch_info.last_batch_policy == RocalBatchPolicy::DROP && + _last_batch_info.pad_last_batch_repeated == true) { // When pad_last_batch_repeated is False - Enough + // number of samples would not be present in the last batch - hence + // dropped by condition handled in the loader + ret -= _batch_size; + } + } else if (_shard_size > 0) { + auto largest_shard_size_with_padding = + _shard_size + (_batch_size - (_shard_size % _batch_size)); // The shard size used here is padded + if (_loop) + return largest_shard_size_with_padding; + int size = std::max(largest_shard_size_with_padding, _batch_size); + ret = (size - _read_counter); + if (_last_batch_info.last_batch_policy == RocalBatchPolicy::DROP) // The shard size is padded at the beginning of the condition, hence dropping the last batch + ret -= _batch_size; + } return ((ret < 0) ? 0 : ret); } Reader::Status FileSourceReader::initialize(ReaderConfig desc) { auto ret = Reader::Status::OK; - _file_id = 0; _folder_path = desc.path(); _file_list_path = desc.file_list_path(); _shard_id = desc.get_shard_id(); _shard_count = desc.get_shard_count(); - _batch_count = desc.get_batch_size(); + _batch_size = desc.get_batch_size(); _shuffle = desc.shuffle(); _loop = desc.loop(); _meta_data_reader = desc.meta_data_reader(); - _last_batch_info = desc.get_last_batch_policy(); + _last_batch_info = desc.get_sharding_info(); + _pad_last_batch_repeated = _last_batch_info.pad_last_batch_repeated; + _stick_to_shard = _last_batch_info.stick_to_shard; + _shard_size = _last_batch_info.shard_size; ret = subfolder_reading(); // shuffle dataset if set if (ret == Reader::Status::OK && _shuffle) - std::random_shuffle(_file_names.begin(), _file_names.end()); + std::random_shuffle(_file_names.begin() + _shard_start_idx_vector[_shard_id], + _file_names.begin() + _shard_end_idx_vector[_shard_id]); return ret; } +void FileSourceReader::increment_curr_file_idx() { + // The condition satisfies for both pad_last_batch = True (or) False + if (_stick_to_shard == false) { // The elements of each shard rotate in a round-robin fashion once the elements in particular shard is exhausted + _curr_file_idx = (_curr_file_idx + 1) % _file_names.size(); + } else { // Stick to only elements from the current shard + if (_curr_file_idx >= _shard_start_idx_vector[_shard_id] && + _curr_file_idx < _shard_end_idx_vector[_shard_id]) // checking if current-element lies within the shard size [begin_idx, last_idx -1] + _curr_file_idx = (_curr_file_idx + 1); + else + _curr_file_idx = _shard_start_idx_vector[_shard_id]; + } +} + void FileSourceReader::incremenet_read_ptr() { _read_counter++; - _curr_file_idx = (_curr_file_idx + 1) % _file_names.size(); - if (_last_batch_info.first == RocalBatchPolicy::DROP) { - if (_last_batch_info.second == true) { - // Check for the last batch and skip it by incrementing with batch_size - hence dropping the last batch - if ((_file_names.size() / _batch_count) == _curr_file_idx) // To check if it the last batch - { - _curr_file_idx += _batch_count; // This increaments the ptr with batch size - meaning the batch is skipped. - _curr_file_idx = (_curr_file_idx + 1) % _file_names.size(); // When the last_batch_pad is true, next iter should start from beginning. This line ensures, pointer from end is brought back to beginning. - } - } else { - THROW("Not implemented"); - } - } + increment_curr_file_idx(); } + size_t FileSourceReader::open() { auto file_path = _file_names[_curr_file_idx]; // Get next file name incremenet_read_ptr(); @@ -141,9 +168,23 @@ int FileSourceReader::release() { } void FileSourceReader::reset() { - if (_shuffle) std::random_shuffle(_file_names.begin(), _file_names.end()); + if (_shuffle) + std::random_shuffle(_file_names.begin() + _shard_start_idx_vector[_shard_id], + _file_names.begin() + _shard_start_idx_vector[_shard_id] + actual_shard_size_without_padding()); + + if (_stick_to_shard == false) // Pick elements from the next shard - hence increment shard_id + increment_shard_id(); // Should work for both single and multiple shards + _read_counter = 0; - _curr_file_idx = 0; + + if (_last_batch_info.last_batch_policy == RocalBatchPolicy::DROP) { // Skipping the dropped batch in next epoch + for (uint i = 0; i < _batch_size; i++) + increment_curr_file_idx(); + } +} + +void FileSourceReader::increment_shard_id() { + _shard_id = (_shard_id + 1) % _shard_count; } Reader::Status FileSourceReader::generate_file_names() { @@ -163,37 +204,41 @@ Reader::Status FileSourceReader::generate_file_names() { auto ret = Reader::Status::OK; if (!_file_list_path.empty()) { // Reads the file paths from the file list and adds to file_names vector for decoding - std::ifstream fp(_file_list_path); - if (fp.is_open()) { - while (fp) { - std::string file_label_path; - std::getline(fp, file_label_path); - std::istringstream ss(file_label_path); - std::string file_path; - std::getline(ss, file_path, ' '); + if (_meta_data_reader) { + auto vec_rel_file_path = _meta_data_reader->get_relative_file_path(); // Get the relative file path's from meta_data_reader + for (auto file_path : vec_rel_file_path) { if (filesys::path(file_path).is_relative()) { // Only add root path if the file list contains relative file paths if (!filesys::exists(_folder_path)) THROW("File list contains relative paths but root path doesn't exists"); - file_path = _folder_path + "/" + file_path; + _absolute_file_path = _folder_path + "/" + file_path; } - std::string file_name = file_path.substr(file_path.find_last_of("/\\") + 1); + if (filesys::is_regular_file(_absolute_file_path)) { + _file_names.push_back(_absolute_file_path); + _file_count_all_shards++; + } + } + _last_file_name = _absolute_file_path; + } else { + std::ifstream fp(_file_list_path); + if (fp.is_open()) { + while (fp) { + std::string file_label_path; + std::getline(fp, file_label_path); + std::istringstream ss(file_label_path); + std::string file_path; + std::getline(ss, file_path, ' '); + if (filesys::path(file_path).is_relative()) { // Only add root path if the file list contains relative file paths + if (!filesys::exists(_folder_path)) + THROW("File list contains relative paths but root path doesn't exists"); + file_path = _folder_path + "/" + file_path; + } + std::string file_name = file_path.substr(file_path.find_last_of("/\\") + 1); - if (!_meta_data_reader || _meta_data_reader->exists(file_name)) { // Check if the file is present in metadata reader and add to file names list, to avoid issues while lookup if (filesys::is_regular_file(file_path)) { - if (get_file_shard_id() != _shard_id) { - _file_count_all_shards++; - incremenet_file_id(); - continue; - } - _in_batch_read_count++; - _in_batch_read_count = (_in_batch_read_count % _batch_count == 0) ? 0 : _in_batch_read_count; _last_file_name = file_path; _file_names.push_back(file_path); _file_count_all_shards++; - incremenet_file_id(); } - } else { - WRN("Skipping file," + std::string(file_path) + " as it is not present in metadata reader") } } } @@ -224,56 +269,45 @@ Reader::Status FileSourceReader::generate_file_names() { if (_file_names.empty()) ERR("FileReader ShardID [" + TOSTR(_shard_id) + "] Did not load any file from " + _folder_path) - // the following code is required to make every shard the same size - required for the multi-gpu training - uint images_to_pad_shard = (ceil(_file_count_all_shards / _shard_count) * _shard_count) - _file_count_all_shards; - if (!images_to_pad_shard) { - for (uint i = 0; i < images_to_pad_shard; i++) { - if (get_file_shard_id() != _shard_id) { - _file_count_all_shards++; - incremenet_file_id(); - continue; + auto dataset_size = _file_count_all_shards; + // Pad the _file_names with last element of the shard in the vector when _pad_last_batch_repeated is True + _padded_samples = ((_shard_size > 0) ? _shard_size : largest_shard_size_without_padding()) % _batch_size; + _last_batch_padded_size = (_batch_size > 1) ? (_batch_size - _padded_samples) : 0; + + if (_pad_last_batch_repeated == true) { + // pad the last sample when the dataset_size is not divisible by + // the number of shard's (or) when the shard's size is not + // divisible by the batch size making each shard having equal + // number of samples + uint32_t total_padded_samples = 0; // initialize the total_padded_samples to 0 + for (uint32_t shard_id = 0; shard_id < _shard_count; shard_id++) { + uint32_t start_idx = (dataset_size * shard_id) / _shard_count; + uint32_t actual_shard_size_without_padding = std::floor((shard_id + 1) * dataset_size / _shard_count) - std::floor(shard_id * dataset_size / _shard_count); + uint32_t largest_shard_size = std::ceil(dataset_size * 1.0 / _shard_count); + auto start = _file_names.begin() + start_idx + total_padded_samples; + auto end = start + actual_shard_size_without_padding; + if (largest_shard_size % _batch_size) { + _num_padded_samples = (largest_shard_size - actual_shard_size_without_padding) + _batch_size - (largest_shard_size % _batch_size); + _file_count_all_shards += _num_padded_samples; + _file_names.insert(end, _num_padded_samples, _file_names[start_idx + actual_shard_size_without_padding + total_padded_samples - 1]); + total_padded_samples += _num_padded_samples; } - _last_file_name = _file_names.at(i); - _file_names.push_back(_last_file_name); - _file_count_all_shards++; - incremenet_file_id(); } } + _last_file_name = _file_names[_file_names.size() - 1]; + compute_start_and_end_idx_of_all_shards(); + return ret; } Reader::Status FileSourceReader::subfolder_reading() { auto ret = generate_file_names(); - - if (_in_batch_read_count > 0 && _in_batch_read_count < _batch_count) { - // This is to pad within a batch in a shard. Need to change this according to fill / drop or partial. - // Adjust last batch only if the last batch padded is true. - fill_last_batch(); - LOG("FileReader ShardID [" + TOSTR(_shard_id) + "] Replicated " + _folder_path + _last_file_name + " " + TOSTR((_batch_count - _in_batch_read_count)) + " times to fill the last batch") - } if (!_file_names.empty()) - LOG("FileReader ShardID [" + TOSTR(_shard_id) + "] Total of " + TOSTR(_file_names.size()) + " images loaded from " + _full_path) + LOG("FileReader ShardID [" + TOSTR(_shard_id) + "] Total of " + TOSTR(_file_names.size()) + " images loaded from " + STR(_folder_path)) return ret; } -void FileSourceReader::fill_last_batch() { - if (_last_batch_info.first == RocalBatchPolicy::FILL || _last_batch_info.first == RocalBatchPolicy::PARTIAL) { - if (_last_batch_info.second == true) { - for (size_t i = 0; i < (_batch_count - _in_batch_read_count); i++) - _file_names.push_back(_last_file_name); - } else { - THROW("Not implemented"); - } - } else if (_last_batch_info.first == RocalBatchPolicy::DROP) { - for (size_t i = 0; i < _in_batch_read_count; i++) - _file_names.pop_back(); - } - if (_last_batch_info.first == RocalBatchPolicy::PARTIAL) { - _last_batch_padded_size = _batch_count - _in_batch_read_count; - } -} - Reader::Status FileSourceReader::open_folder() { if ((_src_dir = opendir(_folder_path.c_str())) == nullptr) THROW("FileReader ShardID [" + TOSTR(_shard_id) + "] ERROR: Failed opening the directory at " + _folder_path); @@ -296,34 +330,54 @@ Reader::Status FileSourceReader::open_folder() { continue; } if (!_meta_data_reader || _meta_data_reader->exists(filename)) { // Check if the file is present in metadata reader and add to file names list, to avoid issues while lookup - if (get_file_shard_id() != _shard_id) { - _file_count_all_shards++; - incremenet_file_id(); - continue; - } - _in_batch_read_count++; - _in_batch_read_count = (_in_batch_read_count % _batch_count == 0) ? 0 : _in_batch_read_count; _file_names.push_back(file_path); + _last_file_name = file_path; _file_count_all_shards++; - incremenet_file_id(); } else { WRN("Skipping file," + filename + " as it is not present in metadata reader") } } + if (_file_names.empty()) ERR("FileReader ShardID [" + TOSTR(_shard_id) + "] Did not load any file from " + _folder_path) - _last_file_name = _file_names[_file_names.size() - 1]; - closedir(_src_dir); return Reader::Status::OK; } -size_t FileSourceReader::get_file_shard_id() { - if (_batch_count == 0 || _shard_count == 0) - THROW("Shard (Batch) size cannot be set to 0") - return _file_id % _shard_count; -} - size_t FileSourceReader::last_batch_padded_size() { return _last_batch_padded_size; } +std::string FileSourceReader::get_root_folder_path() { + return _folder_path; +} + +std::vector FileSourceReader::get_file_paths_from_meta_data_reader() { + if (_meta_data_reader) { + return _meta_data_reader->get_relative_file_path(); + } else { + std::cout << "\n Meta Data Reader is not initialized!"; + return {}; + } +} + +void FileSourceReader::compute_start_and_end_idx_of_all_shards() { + for (uint shard_id = 0; shard_id < _shard_count; shard_id++) { + auto start_idx_of_shard = (_file_count_all_shards * shard_id) / _shard_count; + auto end_idx_of_shard = start_idx_of_shard + actual_shard_size_without_padding() - 1; + _shard_start_idx_vector.push_back(start_idx_of_shard); + _shard_end_idx_vector.push_back(end_idx_of_shard); + + } +} + +size_t FileSourceReader::get_dataset_size() { + return _file_count_all_shards; +} + +size_t FileSourceReader::actual_shard_size_without_padding() { + return std::floor((_shard_id + 1) * _file_count_all_shards / _shard_count) - std::floor(_shard_id * _file_count_all_shards / _shard_count); +} + +size_t FileSourceReader::largest_shard_size_without_padding() { + return std::ceil(_file_count_all_shards * 1.0 / _shard_count); +} \ No newline at end of file diff --git a/rocAL_pybind/amd/rocal/decoders.py b/rocAL_pybind/amd/rocal/decoders.py index 038034d6a..abaa81cf1 100644 --- a/rocAL_pybind/amd/rocal/decoders.py +++ b/rocAL_pybind/amd/rocal/decoders.py @@ -31,7 +31,7 @@ def image(*inputs, user_feature_key_map=None, path='', file_root='', annotations_file='', shard_id=0, num_shards=1, random_shuffle=False, output_type=types.RGB, decoder_type=types.DECODER_TJPEG, device=None, decode_size_policy=types.USER_GIVEN_SIZE_ORIG, max_decoded_width=1000, max_decoded_height=1000, - last_batch_policy=types.LAST_BATCH_FILL, last_batch_padded=True): + last_batch_policy=types.LAST_BATCH_FILL, pad_last_batch=True, stick_to_shard=True, shard_size=-1): """!Decodes images using different readers and decoders. @param inputs list of input images. @@ -53,9 +53,11 @@ def image(*inputs, user_feature_key_map=None, path='', file_root='', annotations """ reader = Pipeline._current_pipeline._reader Pipeline._current_pipeline._last_batch_policy = last_batch_policy - if last_batch_padded is False: - print("last_batch_padded = False is not implemented in rocAL. Setting last_batch_padded to True") - last_batch_padded = True + RocalShardingInfo = b.RocalShardingInfo() + RocalShardingInfo.last_batch_policy = last_batch_policy + RocalShardingInfo.pad_last_batch_repeated = pad_last_batch + RocalShardingInfo.stick_to_shard = stick_to_shard + RocalShardingInfo.shard_size = shard_size if (device == "gpu"): decoder_type = types.DECODER_HW_JEPG else: @@ -74,7 +76,7 @@ def image(*inputs, user_feature_key_map=None, path='', file_root='', annotations "max_width": max_decoded_width, "max_height": max_decoded_height, "dec_type": decoder_type, - "last_batch_info": (last_batch_policy, last_batch_padded)} + "sharding_info": RocalShardingInfo} decoded_image = b.cocoImageDecoderShard( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) @@ -92,7 +94,7 @@ def image(*inputs, user_feature_key_map=None, path='', file_root='', annotations "max_width": max_decoded_width, "max_height": max_decoded_height, "dec_type": decoder_type, - "last_batch_info": (last_batch_policy, last_batch_padded)} + "sharding_info": RocalShardingInfo} decoded_image = b.tfImageDecoder( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) @@ -109,7 +111,7 @@ def image(*inputs, user_feature_key_map=None, path='', file_root='', annotations "max_width": max_decoded_width, "max_height": max_decoded_height, "dec_type": decoder_type, - "last_batch_info": (last_batch_policy, last_batch_padded)} + "sharding_info": RocalShardingInfo} decoded_image = b.caffe2ImageDecoderShard( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) @@ -126,7 +128,7 @@ def image(*inputs, user_feature_key_map=None, path='', file_root='', annotations "max_width": max_decoded_width, "max_height": max_decoded_height, "dec_type": decoder_type, - "last_batch_info": (last_batch_policy, last_batch_padded)} + "sharding_info": RocalShardingInfo} decoded_image = b.caffeImageDecoderShard( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) @@ -143,7 +145,7 @@ def image(*inputs, user_feature_key_map=None, path='', file_root='', annotations "max_width": max_decoded_width, "max_height": max_decoded_height, "dec_type": decoder_type, - "last_batch_info": (last_batch_policy, last_batch_padded)} + "sharding_info": RocalShardingInfo} decoded_image = b.mxnetDecoder( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) @@ -160,7 +162,7 @@ def image(*inputs, user_feature_key_map=None, path='', file_root='', annotations "max_width": max_decoded_width, "max_height": max_decoded_height, "dec_type": decoder_type, - "last_batch_info": (last_batch_policy, last_batch_padded)} + "sharding_info": RocalShardingInfo} decoded_image = b.imageDecoderShard( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) @@ -168,7 +170,7 @@ def image(*inputs, user_feature_key_map=None, path='', file_root='', annotations def image_raw(*inputs, user_feature_key_map=None, path='', random_shuffle=False, output_type=types.RGB, max_decoded_width=1000, max_decoded_height=1000, - last_batch_policy=types.LAST_BATCH_FILL, last_batch_padded=True): + last_batch_policy=types.LAST_BATCH_FILL, pad_last_batch=True, stick_to_shard=True, shard_size=-1): """!Decodes raw images for TF reader and decoder. @param inputs list of input images. @@ -183,10 +185,11 @@ def image_raw(*inputs, user_feature_key_map=None, path='', random_shuffle=False, """ reader = Pipeline._current_pipeline._reader Pipeline._current_pipeline._last_batch_policy = last_batch_policy - if last_batch_padded is False: - print("last_batch_padded = False is not implemented in rocAL. Setting last_batch_padded to True") - last_batch_padded = True - + RocalShardingInfo = b.RocalShardingInfo() + RocalShardingInfo.last_batch_policy = last_batch_policy + RocalShardingInfo.pad_last_batch_repeated = pad_last_batch + RocalShardingInfo.stick_to_shard = stick_to_shard + RocalShardingInfo.shard_size = shard_size if (reader == "TFRecordReaderClassification" or reader == "TFRecordReaderDetection"): kwargs_pybind = { "source_path": path, @@ -198,7 +201,7 @@ def image_raw(*inputs, user_feature_key_map=None, path='', random_shuffle=False, "loop": False, "max_width": max_decoded_width, "max_height": max_decoded_height, - "last_batch_info": (last_batch_policy, last_batch_padded)} + "sharding_info": RocalShardingInfo} decoded_image = b.tfImageDecoderRaw( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) return (decoded_image) @@ -208,7 +211,7 @@ def image_random_crop(*inputs, user_feature_key_map=None, path='', file_root='', random_shuffle=False, num_attempts=10, output_type=types.RGB, random_area=[0.08, 1.0], random_aspect_ratio=[0.8, 1.25], decode_size_policy=types.USER_GIVEN_SIZE_ORIG, max_decoded_width=1000, max_decoded_height=1000, decoder_type=types.DECODER_TJPEG, - last_batch_policy=types.LAST_BATCH_FILL, last_batch_padded=True): + last_batch_policy=types.LAST_BATCH_FILL, pad_last_batch=True, stick_to_shard=True, shard_size=-1): """!Applies random cropping to images using different readers and decoders. @param inputs list of input images. @@ -232,9 +235,11 @@ def image_random_crop(*inputs, user_feature_key_map=None, path='', file_root='', """ reader = Pipeline._current_pipeline._reader Pipeline._current_pipeline._last_batch_policy = last_batch_policy - if last_batch_padded is False: - print("last_batch_padded = False is not implemented in rocAL. Setting last_batch_padded to True") - last_batch_padded = True + RocalShardingInfo = b.RocalShardingInfo() + RocalShardingInfo.last_batch_policy = last_batch_policy + RocalShardingInfo.pad_last_batch_repeated = pad_last_batch + RocalShardingInfo.stick_to_shard = stick_to_shard + RocalShardingInfo.shard_size = shard_size # Internally calls the C++ Partial decoder's if (reader == 'COCOReader'): kwargs_pybind = { @@ -252,7 +257,7 @@ def image_random_crop(*inputs, user_feature_key_map=None, path='', file_root='', "decode_size_policy": decode_size_policy, "max_width": max_decoded_width, "max_height": max_decoded_height, - "last_batch_info": (last_batch_policy, last_batch_padded)} + "sharding_info": RocalShardingInfo} crop_output_image = b.cocoImageDecoderSliceShard( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) elif (reader == "TFRecordReaderClassification" or reader == "TFRecordReaderDetection"): @@ -269,7 +274,7 @@ def image_random_crop(*inputs, user_feature_key_map=None, path='', file_root='', "max_width": max_decoded_width, "max_height": max_decoded_height, "dec_type": decoder_type, - "last_batch_info": (last_batch_policy, last_batch_padded)} + "sharding_info": RocalShardingInfo} crop_output_image = b.tfImageDecoder( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) elif (reader == "CaffeReader" or reader == "CaffeReaderDetection"): @@ -287,7 +292,7 @@ def image_random_crop(*inputs, user_feature_key_map=None, path='', file_root='', "decode_size_policy": decode_size_policy, "max_width": max_decoded_width, "max_height": max_decoded_height, - "last_batch_info": (last_batch_policy, last_batch_padded)} + "sharding_info": RocalShardingInfo} crop_output_image = b.caffeImageDecoderPartialShard( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) elif (reader == "Caffe2Reader" or reader == "Caffe2ReaderDetection"): @@ -305,7 +310,7 @@ def image_random_crop(*inputs, user_feature_key_map=None, path='', file_root='', "decode_size_policy": decode_size_policy, "max_width": max_decoded_width, "max_height": max_decoded_height, - "last_batch_info": (last_batch_policy, last_batch_padded)} + "sharding_info": RocalShardingInfo} crop_output_image = b.caffe2ImageDecoderPartialShard( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) else: @@ -323,7 +328,7 @@ def image_random_crop(*inputs, user_feature_key_map=None, path='', file_root='', "decode_size_policy": decode_size_policy, "max_width": max_decoded_width, "max_height": max_decoded_height, - "last_batch_info": (last_batch_policy, last_batch_padded)} + "sharding_info": RocalShardingInfo} crop_output_image = b.fusedDecoderCropShard( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) @@ -332,7 +337,7 @@ def image_random_crop(*inputs, user_feature_key_map=None, path='', file_root='', def image_slice(*inputs, file_root='', path='', annotations_file='', shard_id=0, num_shards=1, random_shuffle=False, random_aspect_ratio=[0.75, 1.33333], random_area=[0.08, 1.0], num_attempts=100, output_type=types.RGB, - decode_size_policy=types.USER_GIVEN_SIZE_ORIG, max_decoded_width=1000, max_decoded_height=1000, last_batch_policy=types.LAST_BATCH_FILL, last_batch_padded=True): + decode_size_policy=types.USER_GIVEN_SIZE_ORIG, max_decoded_width=1000, max_decoded_height=1000, last_batch_policy=types.LAST_BATCH_FILL, pad_last_batch=True, stick_to_shard=True, shard_size=-1): """!Slices images randomly using different readers and decoders. @param inputs list of input images. @@ -354,9 +359,11 @@ def image_slice(*inputs, file_root='', path='', annotations_file='', shard_id=0, """ reader = Pipeline._current_pipeline._reader Pipeline._current_pipeline._last_batch_policy = last_batch_policy - if last_batch_padded is False: - print("last_batch_padded = False is not implemented in rocAL. Setting last_batch_padded to True") - last_batch_padded = True + RocalShardingInfo = b.RocalShardingInfo() + RocalShardingInfo.last_batch_policy = last_batch_policy + RocalShardingInfo.pad_last_batch_repeated = pad_last_batch + RocalShardingInfo.stick_to_shard = stick_to_shard + RocalShardingInfo.shard_size = shard_size # Reader -> Randon BBox Crop -> ImageDecoderSlice # Random crop parameters taken from pytorch's RandomResizedCrop default function arguments # TODO:To pass the crop co-ordinates from random_bbox_crop to image_slice @@ -379,7 +386,7 @@ def image_slice(*inputs, file_root='', path='', annotations_file='', shard_id=0, "decode_size_policy": decode_size_policy, "max_width": max_decoded_width, "max_height": max_decoded_height, - "last_batch_info": (last_batch_policy, last_batch_padded)} + "sharding_info": RocalShardingInfo} image_decoder_slice = b.cocoImageDecoderSliceShard( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) elif (reader == "CaffeReader" or reader == "CaffeReaderDetection"): @@ -397,7 +404,7 @@ def image_slice(*inputs, file_root='', path='', annotations_file='', shard_id=0, "decode_size_policy": decode_size_policy, "max_width": max_decoded_width, "max_height": max_decoded_height, - "last_batch_info": (last_batch_policy, last_batch_padded)} + "sharding_info": RocalShardingInfo} image_decoder_slice = b.caffeImageDecoderPartialShard( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) elif (reader == "Caffe2Reader" or reader == "Caffe2ReaderDetection"): @@ -415,7 +422,7 @@ def image_slice(*inputs, file_root='', path='', annotations_file='', shard_id=0, "decode_size_policy": decode_size_policy, "max_width": max_decoded_width, "max_height": max_decoded_height, - "last_batch_info": (last_batch_policy, last_batch_padded)} + "sharding_info": RocalShardingInfo} image_decoder_slice = b.caffe2ImageDecoderPartialShard( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) else: @@ -433,25 +440,40 @@ def image_slice(*inputs, file_root='', path='', annotations_file='', shard_id=0, "decode_size_policy": decode_size_policy, "max_width": max_decoded_width, "max_height": max_decoded_height, - "last_batch_info": (last_batch_policy, last_batch_padded)} + "sharding_info": RocalShardingInfo} image_decoder_slice = b.fusedDecoderCropShard( Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) return (image_decoder_slice) -def audio(*inputs, file_root='', file_list_path='', shard_id=0, num_shards=1, random_shuffle=False, downmix=False, stick_to_shard=False, shard_size=-1): +def audio(*inputs, file_root='', file_list_path='', bytes_per_sample_hint=[0], shard_id=0, num_shards=1, random_shuffle=False, downmix=False, dtype=types.FLOAT, quality=50.0, sample_rate=0.0, seed=1, stick_to_shard=True, shard_size=-1, last_batch_policy=types.LAST_BATCH_FILL, pad_last_batch_repeated=False, + decode_size_policy=types.MAX_SIZE, max_decoded_samples=522320, max_decoded_channels=1): """!Decodes wav audio files. - @param inputs List of input audio. - @param file_root Folder Path to the audio data. - @param file_list_path (for future use) Path to the text file containing list of files and the labels - @param shard_id Shard ID for parallel processing. - @param num_shards Total number of shards for parallel processing. - @param random_shuffle Whether to shuffle audio samples randomly. - @param downmix Converts the audio data to single channel when enabled - @param stick_to_shard The reader sticks to the data for it's corresponding shard when enabled - @param shard_size Provides the number of files in an epoch of a particular shard. - @return Decoded audio. + @param inputs list of input audio. + @param file_root Folder Path to the audio data. + @param file_list_path Text file containing list of files and the labels + @param shard_id Shard ID for parallel processing. + @param num_shards Total number of shards for parallel processing. + @param random_shuffle Whether to shuffle images randomly. + @param downmix Converts the audio data to single channel when enabled + @param dtype Data type of the decoded audio + @param quality Resampling quality + @param sample_rate Sample rate for the decoded audio. + @param seed Random seed. + @param stick_to_shard The reader sticks to the data for it's corresponding shard when enabled + @param shard_size Number of files in an epoch + @param last_batch_policy Determines the handling of the last batch when the shard size is not divisible by the batch size. Check types.py enum for possible values. + @param pad_last_batch_repeated If set to True, pads the shards last batch by repeating the last sample's data (dummy data). + @param decode_size_policy Size policy for decoding images. + @param max_decoded_samples Maximum samples for decoded images. + @param max_decoded_channels Maximum channels for decoded images. + @return Decoded audio. """ + RocalShardingInfo = b.RocalShardingInfo() + RocalShardingInfo.last_batch_policy = last_batch_policy + RocalShardingInfo.pad_last_batch_repeated = pad_last_batch_repeated + RocalShardingInfo.stick_to_shard = stick_to_shard + RocalShardingInfo.shard_size = shard_size kwargs_pybind = { "source_path": file_root, "source_file_list_path": file_list_path, @@ -460,6 +482,11 @@ def audio(*inputs, file_root='', file_list_path='', shard_id=0, num_shards=1, ra "is_output": False, "shuffle": random_shuffle, "loop": False, - "downmix": downmix} + "downmix": downmix, + "decode_size_policy": decode_size_policy, + "max_width": max_decoded_samples, + "max_height": max_decoded_channels, + "sharding_info": RocalShardingInfo} + Pipeline._current_pipeline._last_batch_policy = last_batch_policy decoded_audio = b.audioDecoderSingleShard(Pipeline._current_pipeline._handle, *(kwargs_pybind.values())) return decoded_audio diff --git a/rocAL_pybind/amd/rocal/pipeline.py b/rocAL_pybind/amd/rocal/pipeline.py index 5e9eb5527..0cbb6fc1e 100644 --- a/rocAL_pybind/amd/rocal/pipeline.py +++ b/rocAL_pybind/amd/rocal/pipeline.py @@ -66,7 +66,7 @@ class Pipeline(object): def __init__(self, batch_size=-1, num_threads=0, device_id=0, seed=1, exec_pipelined=True, prefetch_queue_depth=2, exec_async=True, bytes_per_sample=0, - rocal_cpu=False, max_streams=-1, default_cuda_stream_priority=0, tensor_layout=types.NCHW, reverse_channels=False, mean=None, std=None, tensor_dtype=types.FLOAT, output_memory_type=None): + rocal_cpu=False, max_streams=-1, default_cuda_stream_priority=0, tensor_layout=types.NCHW, reverse_channels=False, mean=None, std=None, tensor_dtype=types.FLOAT, output_memory_type=None): if (rocal_cpu): self._handle = b.rocalCreate( batch_size, types.CPU, device_id, num_threads, prefetch_queue_depth, tensor_dtype) diff --git a/rocAL_pybind/amd/rocal/plugin/pytorch.py b/rocAL_pybind/amd/rocal/plugin/pytorch.py index 611777603..00495b418 100644 --- a/rocAL_pybind/amd/rocal/plugin/pytorch.py +++ b/rocAL_pybind/amd/rocal/plugin/pytorch.py @@ -216,9 +216,8 @@ def __next__(self): # Check if last batch policy is partial and only return the valid images in last batch if (self.last_batch_policy is (types.LAST_BATCH_PARTIAL)) and b.getRemainingImages(self.loader._handle) < self.batch_size: - if (self.last_batch_size is None): - self.last_batch_size = self.batch_size - \ - b.getLastBatchPaddedSize(self.loader._handle) + self.last_batch_size = self.batch_size - \ + b.getLastBatchPaddedSize(self.loader._handle) return [inner_list[0:self.last_batch_size, :] for inner_list in self.output_list], self.labels_tensor[0:self.last_batch_size] else: return self.output_list, self.labels_tensor @@ -286,6 +285,7 @@ def __init__(self, super(ROCALClassificationIterator, self).__init__(pipe, tensor_layout=pipe._tensor_layout, tensor_dtype=pipe._tensor_dtype, multiplier=pipe._multiplier, offset=pipe._offset, display=display, device=device, device_id=device_id) + class ROCALAudioIterator(object): """! rocAL iterator for audio tasks for PyTorch The Tensors that are returned by the iterator will be owned by rocAL and would be valid until next iteration. @@ -296,51 +296,92 @@ class ROCALAudioIterator(object): @param device The device to use for processing - CPU / GPU @param device_id The ID of the device to use """ - def __init__(self, pipeline, tensor_dtype = types.FLOAT, size = -1, auto_reset = False, device = "cpu", device_id = 0): + + def __init__(self, pipeline, tensor_dtype=types.FLOAT, size=-1, auto_reset=False, device="cpu", device_id=0): self.loader = pipeline self.device = device self.device_id = device_id self.output = None - self.iterator_length = b.getRemainingImages(self.loader._handle) # To change the name of getRemainingImages to getRemainingSamples in upcoming PRs + # To change the name of getRemainingImages to getRemainingSamples in upcoming PRs + self.iterator_length = b.getRemainingImages(self.loader._handle) self.max_shape = None self.batch_size = self.loader._batch_size self.output_list = None self.labels_size = self.batch_size self.output_memory_type = self.loader._output_memory_type + self.last_batch_padded_size = b.getLastBatchPaddedSize( + self.loader._handle) + self.last_batch_policy = self.loader._last_batch_policy + self.shard_size = size + self.auto_reset = auto_reset + self.batch_count = 0 def next(self): return self.__next__() def __next__(self): - if self.loader.rocal_run() != 0: + rocal_run = self.loader.rocal_run() + if (rocal_run != 0 or self.shard_size > 0 and self.batch_count >= self.shard_size): + if self.auto_reset: + self.reset() raise StopIteration else: self.output_tensor_list = self.loader.get_output_tensors() - # Output list used to store pipeline outputs - can support multiple augmentation outputs + self.batch_count += self.batch_size + self.last_batch_size = self.batch_size - \ + b.getLastBatchPaddedSize(self.loader._handle) self.output_list = [] for i in range(len(self.output_tensor_list)): dimensions = self.output_tensor_list[i].dimensions() self.num_roi_dims = self.output_tensor_list[i].roi_dims_size() self.num_of_rois = self.num_roi_dims * 2 - self.roi_array = np.zeros(self.batch_size * self.num_of_rois, dtype=np.int32) + self.roi_array = np.zeros( + self.batch_size * self.num_of_rois, dtype=np.int32) self.output_tensor_list[i].copy_roi(self.roi_array) - torch_dtype = self.output_tensor_list[i].dtype() + roi = self.roi_array.reshape(self.batch_size, self.num_of_rois) + max_x1 = np.max(roi[..., self.num_of_rois-2:self.num_of_rois-1]) + max_y1 = np.max(roi[..., self.num_of_rois-1:self.num_of_rois]) if self.device == "cpu": - output = torch.empty(dimensions, dtype=getattr(torch, torch_dtype)) - self.labels_tensor = torch.empty(self.labels_size, dtype=getattr(torch, torch_dtype)) + torch_dtype = self.output_tensor_list[i].dtype() + if (max_x1 == 0 or max_y1 == 0): + output = torch.empty( + dimensions, dtype=getattr(torch, torch_dtype)) + else: + output = torch.empty( + dimensions[0], max_x1, max_y1, dtype=getattr(torch, torch_dtype)) + self.labels_tensor = torch.empty( + self.labels_size, dtype=getattr(torch, torch_dtype)) else: torch_gpu_device = torch.device('cuda', self.device_id) - output = torch.empty(dimensions, dtype=getattr(torch, torch_dtype), device=torch_gpu_device) - self.labels_tensor = torch.empty(self.labels_size, dtype=getattr(torch, torch_dtype), device=torch_gpu_device) + torch_dtype = self.output_tensor_list[i].dtype() + if (max_x1 == 0 or max_y1 == 0): + output = torch.empty(dimensions, dtype=getattr( + torch, torch_dtype), device=torch_gpu_device) + else: + output = torch.empty( + dimensions[0], max_x1, max_y1, dtype=getattr(torch, torch_dtype)) + self.labels_tensor = torch.empty(self.labels_size, dtype=getattr( + torch, torch_dtype), device=torch_gpu_device) - self.output_tensor_list[i].copy_data(ctypes.c_void_p(output.data_ptr()), self.output_memory_type) + if (max_x1 == 0 or max_y1 == 0): + self.output_tensor_list[i].copy_data(ctypes.c_void_p( + output.data_ptr()), self.output_memory_type) + else: + self.output_tensor_list[i].copy_data( + ctypes.c_void_p(output.data_ptr()), 0, 0, max_y1, max_x1) self.output_list.append(output) self.labels = self.loader.get_image_labels() - self.labels_tensor = self.labels_tensor.copy_(torch.from_numpy(self.labels)).long() - return self.output_list, self.labels_tensor, torch.tensor(self.roi_array.reshape(self.batch_size, self.num_of_rois)[...,self.num_of_rois-2:self.num_of_rois]) + self.labels_tensor = self.labels_tensor.copy_( + torch.from_numpy(self.labels)).long() + + if (self.last_batch_policy is (types.LAST_BATCH_PARTIAL)) and b.getRemainingImages(self.loader._handle) < self.batch_size: + return [inner_list[0:self.last_batch_size, :] for inner_list in self.output_list], self.labels_tensor[0:self.last_batch_size], torch.from_numpy(self.roi_array.reshape(self.batch_size, self.num_of_rois)[..., self.num_of_rois-2:self.num_of_rois][0:self.last_batch_size, :]) + else: + return self.output_list, self.labels_tensor, torch.from_numpy(self.roi_array.reshape(self.batch_size, self.num_of_rois)[..., self.num_of_rois-2:self.num_of_rois]) def reset(self): + self.batch_count = 0 b.rocalResetLoaders(self.loader._handle) def __iter__(self): diff --git a/rocAL_pybind/rocal_pybind.cpp b/rocAL_pybind/rocal_pybind.cpp index 22f40d24b..69eb92fa4 100644 --- a/rocAL_pybind/rocal_pybind.cpp +++ b/rocAL_pybind/rocal_pybind.cpp @@ -270,6 +270,14 @@ PYBIND11_MODULE(rocal_pybind, m) { R"code( Copies the ring buffer data to cupy arrays. )code") + .def( + "copy_data", [](rocalTensor &output_tensor, py::object p, uint x_offset, uint y_offset, uint roi_width, uint roi_height) { + auto ptr = ctypes_void_ptr(p); + output_tensor.copy_data(static_cast(ptr), x_offset, y_offset, roi_width, roi_height); + }, + R"code( + Copies the ring buffer data to python buffer pointers given a ROI with dimensions in x and y direction. + )code") .def( "at", [](rocalTensor &output_tensor, uint idx) { std::vector stride_per_sample(output_tensor.strides()); @@ -452,6 +460,12 @@ PYBIND11_MODULE(rocal_pybind, m) { .def_readwrite("y", &ROIxywh::y) .def_readwrite("w", &ROIxywh::w) .def_readwrite("h", &ROIxywh::h); + py::class_(m, "RocalShardingInfo") + .def(py::init<>()) + .def_readwrite("last_batch_policy", &RocalShardingInfo::last_batch_policy) + .def_readwrite("pad_last_batch_repeated", &RocalShardingInfo::pad_last_batch_repeated) + .def_readwrite("stick_to_shard", &RocalShardingInfo::stick_to_shard) + .def_readwrite("shard_size", &RocalShardingInfo::shard_size); // rocal_api_info.h m.def("getRemainingImages", &rocalGetRemainingImages); m.def("getImageName", &wrapper_image_name); diff --git a/tests/python_api/audio_unit_test.py b/tests/python_api/audio_unit_test.py index 438bfff4f..b1f752c3a 100644 --- a/tests/python_api/audio_unit_test.py +++ b/tests/python_api/audio_unit_test.py @@ -80,7 +80,7 @@ def verify_output(audio_tensor, rocal_data_path, roi_tensor, test_results, case_ for i in range(roi_data[0]): for j in range(roi_data[1]): ref_val = data_array[i * roi_data[1] + j] - out_val = audio_data[i * dimensions[2] + j] + out_val = audio_data[i * roi_data[1] + j] # Stride upto max_roi # ensuring that out_val is not exactly zero while ref_val is non-zero. invalid_comparison = (out_val == 0.0) and (ref_val != 0.0) #comparing the absolute difference between the output value (out_val) and the reference value (ref_val) with a tolerance threshold of 1e-20. @@ -106,7 +106,8 @@ def audio_decoder_pipeline(path, file_list, downmix=False): downmix=downmix, shard_id=0, num_shards=1, - stick_to_shard=False) + stick_to_shard=False, + last_batch_policy=types.LAST_BATCH_DROP, pad_last_batch_repeated=False) @pipeline_def(seed=seed) def pre_emphasis_filter_pipeline(path, file_list): @@ -118,7 +119,8 @@ def pre_emphasis_filter_pipeline(path, file_list): downmix=False, shard_id=0, num_shards=1, - stick_to_shard=False) + stick_to_shard=False, + last_batch_policy=types.LAST_BATCH_DROP, pad_last_batch_repeated=False) return fn.preemphasis_filter(decoded_audio) @pipeline_def(seed=seed) @@ -131,7 +133,8 @@ def spectrogram_pipeline(path, file_list): downmix=False, shard_id=0, num_shards=1, - stick_to_shard=False) + stick_to_shard=False, + last_batch_policy=types.LAST_BATCH_DROP, pad_last_batch_repeated=False) spec = fn.spectrogram( decoded_audio, nfft=512, @@ -150,7 +153,8 @@ def to_decibels_pipeline(path, file_list): downmix=False, shard_id=0, num_shards=1, - stick_to_shard=False) + stick_to_shard=False, + last_batch_policy=types.LAST_BATCH_DROP, pad_last_batch_repeated=False) return fn.to_decibels( decoded_audio, multiplier=np.log(10), @@ -168,7 +172,8 @@ def resample_pipeline(path, file_list): downmix=True, shard_id=0, num_shards=1, - stick_to_shard=False) + stick_to_shard=False, + last_batch_policy=types.LAST_BATCH_DROP, pad_last_batch_repeated=False) input_sample_rate = 16000.00 uniform_distribution_resample = fn.random.uniform(decoded_audio, range=[1.15, 1.15]) resampled_rate = uniform_distribution_resample * input_sample_rate @@ -188,7 +193,8 @@ def tensor_add_tensor_pipeline(path, file_list): downmix=False, shard_id=0, num_shards=1, - stick_to_shard=False) + stick_to_shard=False, + last_batch_policy=types.LAST_BATCH_DROP, pad_last_batch_repeated=False) uniform_distribution_sample = fn.random.uniform(decoded_audio, range=[1.15, 1.15]) return decoded_audio + uniform_distribution_sample @@ -202,7 +208,8 @@ def tensor_mul_scalar_pipeline(path, file_list): downmix=True, shard_id=0, num_shards=1, - stick_to_shard=False) + stick_to_shard=False, + last_batch_policy=types.LAST_BATCH_DROP, pad_last_batch_repeated=False) return decoded_audio * 1.15 @pipeline_def(seed=seed) @@ -215,7 +222,8 @@ def non_silent_region(path, file_list): downmix=False, shard_id=0, num_shards=1, - stick_to_shard=False) + stick_to_shard=False, + last_batch_policy=types.LAST_BATCH_DROP, pad_last_batch_repeated=False) begin, length = fn.nonsilent_region(decoded_audio, cutoff_db=-60) return begin, length @@ -229,7 +237,8 @@ def non_silent_region_and_slice(path, file_list): downmix=False, shard_id=0, num_shards=1, - stick_to_shard=False) + stick_to_shard=False, + last_batch_policy=types.LAST_BATCH_DROP, pad_last_batch_repeated=False) begin, length = fn.nonsilent_region(decoded_audio, cutoff_db=-60) trim_silence = fn.slice( decoded_audio, @@ -248,7 +257,8 @@ def mel_filter_bank_pipeline(path, file_list): downmix=False, shard_id=0, num_shards=1, - stick_to_shard=False) + stick_to_shard=False, + last_batch_policy=types.LAST_BATCH_DROP, pad_last_batch_repeated=False) spec = fn.spectrogram( decoded_audio, nfft=512, @@ -272,7 +282,8 @@ def normalize_pipeline(path, file_list): downmix=False, shard_id=0, num_shards=1, - stick_to_shard=False) + stick_to_shard=False, + last_batch_policy=types.LAST_BATCH_DROP, pad_last_batch_repeated=False) spec = fn.spectrogram( decoded_audio, nfft=512,