Audio PR - Augmentation support [ Mel-Filter Bank and Normalize ]

CHANGELOG.md

@@ -25,8 +25,10 @@
 * Support for Audio augmentation - Resample
 * Support for TensorTensorAdd and TensorScalarMultiply operations
 * Support for Uniform and Normal distribution nodes
 * Support for Audio augmentation - NonSilentRegionDetection 
 * Support for generic augmentation - Slice 
+* Support for generic augmentation - Normalize
+* Support for Audio augmentation - MelFilterBank
 
 ### Optimizations
 

rocAL/include/api/rocal_api_augmentation.h

@@ -1277,4 +1277,51 @@ extern "C" RocalTensor ROCAL_API_CALL rocalSlice(RocalContext context,
                                                  RocalOutOfBoundsPolicy policy = RocalOutOfBoundsPolicy::ROCAL_ERROR,
                                                  RocalTensorOutputType output_datatype = ROCAL_FP32);
 
+/*! \brief Performs mean-stddev normalization on images.
+ * \ingroup group_rocal_augmentations
+ * \param [in] context Rocal context
+ * \param [in] input Input Rocal tensor
+ * \param [in] axes axes list for tensor normalization
+ * \param [in] mean mean value (specified for each channel) for tensor normalization
+ * \param [in] std_dev standard deviation value (specified for each channel) for tensor normalization
+ * \param [in] is_output is the output tensor part of the graph output
+ * \param [in] scale scale value (specified for each channel) for tensor normalization
+ * \param [in] shift shift value (specified for each channel) for tensor normalization
+ * \param [in] output_datatype the data type of the output tensor
+ * \return RocalTensor
+ */
+extern "C" RocalTensor ROCAL_API_CALL rocalNormalize(RocalContext context, RocalTensor input,
+                                                     std::vector<unsigned> &axes,
+                                                     std::vector<float> &mean,
+                                                     std::vector<float> &std_dev,
+                                                     bool is_output,
+                                                     float scale = 1.0, float shift = 0.0,
+                                                     RocalTensorOutputType output_datatype = ROCAL_FP32);
+
+/*! \brief Applies mel-filter bank augmentation on the given input tensor
+ * \ingroup group_rocal_augmentations
+ * \param [in] p_context Rocal context
+ * \param [in] p_input Input Rocal tensor
+ * \param [in] is_output is the output tensor part of the graph output
+ * \param [in] freq_high maximum frequency
+ * \param [in] freq_low minimum frequency
+ * \param [in] mel_formula formula used to convert frequencies from hertz to mel and from mel to hertz
+ * \param [in] nfilter number of mel filters
+ * \param [in] normalize boolean variable that determine whether to normalize weights / not
+ * \param [in] sample_rate sampling rate of the audio data
+ * \param [in] output_datatype the data type of the output tensor
+ * \return RocalTensor
+ */
+
+extern "C" RocalTensor ROCAL_API_CALL rocalMelFilterBank(RocalContext p_context,
+                                                         RocalTensor p_input,
+                                                         bool is_output,
+                                                         float freq_high,
+                                                         float freq_low,
+                                                         RocalMelScaleFormula mel_formula,
+                                                         int nfilter,
+                                                         bool normalize,
+                                                         float sample_rate,
+                                                         RocalTensorOutputType output_datatype);
+
 #endif  // MIVISIONX_ROCAL_API_AUGMENTATION_H

rocAL/include/api/rocal_api_types.h

@@ -424,6 +424,20 @@ enum RocalOutOfBoundsPolicy {
     ROCAL_ERROR
 };
 
+/*! \brief rocAL MelScale formula enum
+ * \ingroup group_rocal_types
+ */
+enum RocalMelScaleFormula {
+    /*! \brief Slaney
+     * Follows Slaney’s MATLAB Auditory Modelling Work behavior
+     */
+    ROCAL_MELSCALE_SLANEY = 0,
+    /*! \brief HTK
+     * Follows O’Shaughnessy’s book formula, consistent with Hidden Markov Toolkit(HTK), m = 2595 * log10(1 + (f/700))
+     */
+    ROCAL_MELSCALE_HTK
+};
+
 /*! \brief Tensor Last Batch Policies
  *  \ingroup group_rocal_types
  */

rocAL/include/augmentations/audio_augmentations/node_mel_filter_bank.h

@@ -0,0 +1,45 @@
+/*
+Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+#include "pipeline/graph.h"
+#include "pipeline/node.h"
+#include "rocal_api_types.h"
+
+class MelFilterBankNode : public Node {
+   public:
+     MelFilterBankNode(const std::vector<Tensor *> &inputs, const std::vector<Tensor *> &outputs);
+     MelFilterBankNode() = delete;
+     void init(float freq_high, float freq_low, RocalMelScaleFormula mel_formula, int nfilter, bool normalize, float sample_rate);
+
+   protected:
+     void create_node() override;
+     void update_node() override;
+
+   private:
+     float _freq_high = 0;
+     float _freq_low = 0;
+     int _mel_formula = 0;
+     int _nfilter = 128;
+     float _sample_rate = 44100;
+     bool _normalize = true;
+};

rocAL/include/augmentations/augmentations_nodes.h

@@ -65,3 +65,5 @@ THE SOFTWARE.
 #include "augmentations/arithmetic_augmentations/node_tensor_add_tensor.h"
 #include "augmentations/audio_augmentations/node_non_silent_region_detection.h"
 #include "augmentations/geometry_augmentations/node_slice.h"
+#include "augmentations/effects_augmentations/node_normalize.h"
+#include "augmentations/audio_augmentations/node_mel_filter_bank.h"

rocAL/include/augmentations/effects_augmentations/node_normalize.h

@@ -0,0 +1,51 @@
+/*
+Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+#include "pipeline/graph.h"
+#include "pipeline/node.h"
+#include "parameters/parameter_vx.h"
+
+class NormalizeNode : public Node {
+   public:
+     NormalizeNode(const std::vector<Tensor *> &inputs, const std::vector<Tensor *> &outputs);
+     NormalizeNode() = delete;
+     void init(std::vector<unsigned> &axes, std::vector<float> &mean, std::vector<float> &std_dev, float scale, float shift);
+
+   protected:
+     void create_node() override;
+     void update_node() override {};
+
+   private:
+     int _axis_mask = 0;
+     vx_array _mean_vx_array, _stddev_vx_array;
+     std::vector<unsigned> _axes;
+     std::vector<float> _mean, _std_dev;
+     float _scale, _shift;
+     std::vector<std::vector<uint32_t>> _normalize_roi;
+     enum NormalizeModes {
+         DO_NOT_COMPUTE = 0,      // Mean and Stddev values are passed from user
+         COMPUTE_MEAN = 1,        // Compute mean from specified axes of input
+         COMPUTE_STDDEV = 2,      // Compute stddev from specified axes of input
+         COMPUTE_MEAN_STDDEV = 3  // Compute both mean and stddev from specified axes of input
+     };
+};

rocAL/source/api/rocal_api_augmentation.cpp

@@ -2505,3 +2505,70 @@ rocalSlice(
     }
     return output;
 }
+
+RocalTensor ROCAL_API_CALL
+rocalNormalize(RocalContext p_context, RocalTensor p_input, std::vector<unsigned>& axes,
+               std::vector<float>& mean, std::vector<float>& std_dev, bool is_output,
+               float scale, float shift,
+               RocalTensorOutputType output_datatype) {
+    Tensor* output = nullptr;
+    if ((p_context == nullptr) || (p_input == nullptr)) {
+        ERR("Invalid ROCAL context or invalid input tensor")
+        return output;
+    }
+    auto context = static_cast<Context*>(p_context);
+    auto input = static_cast<Tensor*>(p_input);
+    try {
+        RocalTensorDataType op_tensor_datatype = static_cast<RocalTensorDataType>(output_datatype);
+        TensorInfo output_info = input->info();
+        output_info.set_data_type(op_tensor_datatype);
+        output = context->master_graph->create_tensor(output_info, is_output);
+        std::shared_ptr<NormalizeNode> normalize_node = context->master_graph->add_node<NormalizeNode>({input}, {output});
+        normalize_node->init(axes, mean, std_dev, scale, shift);
+    } catch (const std::exception& e) {
+        context->capture_error(e.what());
+        ERR(e.what())
+    }
+    return output;
+}
+
+RocalTensor ROCAL_API_CALL
+rocalMelFilterBank(
+    RocalContext p_context,
+    RocalTensor p_input,
+    bool is_output,
+    float freq_high,
+    float freq_low,
+    RocalMelScaleFormula mel_formula,
+    int nfilter,
+    bool normalize,
+    float sample_rate,
+    RocalTensorOutputType output_datatype) {
+    Tensor* output = nullptr;
+    if ((p_context == nullptr) || (p_input == nullptr)) {
+        ERR("Invalid ROCAL context or invalid input tensor")
+        return output;
+    }
+    auto context = static_cast<Context*>(p_context);
+    auto input = static_cast<Tensor*>(p_input);
+    try {
+        RocalTensorDataType op_tensor_data_type = (RocalTensorDataType)output_datatype;
+        if (op_tensor_data_type != RocalTensorDataType::FP32) {
+            THROW("Only FP32 dtype is supported for MelFilterBank augmentation.")
+        }
+        TensorInfo output_info = input->info();
+        std::vector<size_t> max_dims = output_info.max_shape();
+        int max_frame = std::max(0ul, max_dims[1]);
+        std::vector<size_t> dims = output_info.dims();
+        dims[1] = nfilter;
+        dims[2] = max_frame;
+        output_info.set_dims(dims);
+        output_info.set_data_type(op_tensor_data_type);
+        output = context->master_graph->create_tensor(output_info, is_output);
+        context->master_graph->add_node<MelFilterBankNode>({input}, {output})->init(freq_high, freq_low, mel_formula, nfilter, normalize, sample_rate);
+    } catch (const std::exception& e) {
+        context->capture_error(e.what());
+        ERR(e.what())
+    }
+    return output;
+}

rocAL/source/augmentations/audio_augmentations/node_mel_filter_bank.cpp

@@ -0,0 +1,63 @@
+/*
+Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#include "augmentations/audio_augmentations/node_mel_filter_bank.h"
+
+#include <vx_ext_rpp.h>
+
+#include "pipeline/exception.h"
+
+MelFilterBankNode::MelFilterBankNode(const std::vector<Tensor *> &inputs, const std::vector<Tensor *> &outputs) : Node(inputs, outputs) {}
+
+void MelFilterBankNode::create_node() {
+    if (_node)
+        return;
+
+    vx_scalar freq_high_vx = vxCreateScalar(vxGetContext((vx_reference)_graph->get()), VX_TYPE_FLOAT32, &_freq_high);
+    vx_scalar freq_low_vx = vxCreateScalar(vxGetContext((vx_reference)_graph->get()), VX_TYPE_FLOAT32, &_freq_low);
+    vx_scalar mel_formula_vx = vxCreateScalar(vxGetContext((vx_reference)_graph->get()), VX_TYPE_INT32, &_mel_formula);
+    vx_scalar nfilter_vx = vxCreateScalar(vxGetContext((vx_reference)_graph->get()), VX_TYPE_INT32, &_nfilter);
+    vx_scalar normalize_vx = vxCreateScalar(vxGetContext((vx_reference)_graph->get()), VX_TYPE_BOOL, &_normalize);
+    vx_scalar sample_rate_vx = vxCreateScalar(vxGetContext((vx_reference)_graph->get()), VX_TYPE_FLOAT32, &_sample_rate);
+    int input_layout = static_cast<int>(_inputs[0]->info().layout());
+    int output_layout = static_cast<int>(_outputs[0]->info().layout());
+    vx_scalar input_layout_vx = vxCreateScalar(vxGetContext((vx_reference)_graph->get()), VX_TYPE_INT32, &input_layout);
+    vx_scalar output_layout_vx = vxCreateScalar(vxGetContext((vx_reference)_graph->get()), VX_TYPE_INT32, &output_layout);
+    _node = vxExtRppMelFilterBank(_graph->get(), _inputs[0]->handle(), _inputs[0]->get_roi_tensor(), _outputs[0]->handle(), _outputs[0]->get_roi_tensor(), freq_high_vx,
+                                  freq_low_vx, mel_formula_vx, nfilter_vx, normalize_vx, sample_rate_vx, input_layout_vx, output_layout_vx);
+
+    vx_status status;
+    if ((status = vxGetStatus((vx_reference)_node)) != VX_SUCCESS)
+        THROW("Adding the mel filter bank (vxRppMelFilterBank) node failed: " + TOSTR(status))
+}
+
+void MelFilterBankNode::update_node() {}
+
+void MelFilterBankNode::init(float freq_high, float freq_low, RocalMelScaleFormula mel_formula,
+                             int nfilter, bool normalize, float sample_rate) {
+    _freq_high = freq_high;
+    _freq_low = freq_low;
+    _mel_formula = static_cast<int>(mel_formula);
+    _nfilter = nfilter;
+    _normalize = normalize;
+    _sample_rate = sample_rate;
+}