diff --git a/components/acoustics-porting/CMakeLists.txt b/components/acoustics-porting/CMakeLists.txt index 38f9c6f..e508598 100644 --- a/components/acoustics-porting/CMakeLists.txt +++ b/components/acoustics-porting/CMakeLists.txt @@ -215,6 +215,21 @@ else() endif() endif() +if(NOT DEFINED PORTING_BOARD_MODEL_RESPEAKER_LITE) + set(PORTING_BOARD_MODEL_RESPEAKER_LITE 0) +endif() +if(NOT DEFINED PORTING_BOARD_MODEL_XIAO_S3) + set(PORTING_BOARD_MODEL_XIAO_S3 1) +endif() + +if(PORTING_BOARD_MODEL_RESPEAKER_LITE) + message(STATUS "acoustics-porting: ReSpeaker Lite board support ENABLED.") +elseif(PORTING_BOARD_MODEL_XIAO_S3) + message(STATUS "acoustics-porting: XIAO ESP32S3 board support ENABLED (default).") +else() + message(STATUS "acoustics-porting: No specific board model enabled.") +endif() + idf_component_register( SRCS ${ACOUSTICS_PORTING_SRCS} INCLUDE_DIRS ${ACOUSTICS_PORTING_INCLUDES_DIR} @@ -232,6 +247,8 @@ target_compile_definitions(${COMPONENT_LIB} PUBLIC LIB_OPUS_ENABLE=${LIB_OPUS_ENABLE} PORTING_LIB_TFLM_ENABLE=${PORTING_LIB_TFLM_ENABLE} PORTING_LIB_DL_FFT_ENABLE=${PORTING_LIB_DL_FFT_ENABLE} + PORTING_BOARD_MODEL_RESPEAKER_LITE=${PORTING_BOARD_MODEL_RESPEAKER_LITE} + PORTING_BOARD_MODEL_XIAO_S3=${PORTING_BOARD_MODEL_XIAO_S3} ) if(LIB_OPUS_ENABLE) diff --git a/components/acoustics-porting/porting/device_esp32s3.cpp b/components/acoustics-porting/porting/device_esp32s3.cpp index 83205b3..0df2031 100644 --- a/components/acoustics-porting/porting/device_esp32s3.cpp +++ b/components/acoustics-porting/porting/device_esp32s3.cpp @@ -58,7 +58,13 @@ static size_t getFreeMemorySize() noexcept static constexpr const char DEVICE_MODEL[] = "ESP32-S3"; static constexpr const char DEVICE_VERSION[] = "1.0.0"; +#if defined(PORTING_BOARD_MODEL_RESPEAKER_LITE) && PORTING_BOARD_MODEL_RESPEAKER_LITE +static constexpr const char DEVICE_NAME[] = "ReSpeaker Lite (XIAO ESP32S3)"; +#elif defined(PORTING_BOARD_MODEL_XIAO_S3) && PORTING_BOARD_MODEL_XIAO_S3 static constexpr const char DEVICE_NAME[] = "XIAO ESP32-S3"; +#else +static constexpr const char DEVICE_NAME[] = "XIAO ESP32-S3"; +#endif static constexpr const size_t DEVICE_MEMORY_SIZE = 8 * 1024 * 1024; static constexpr const size_t DEVICE_NAME_LENGTH_MAX = 64; diff --git a/components/acoustics-porting/porting/sensor/i2sxmos.hpp b/components/acoustics-porting/porting/sensor/i2sxmos.hpp new file mode 100644 index 0000000..e93a9c7 --- /dev/null +++ b/components/acoustics-porting/porting/sensor/i2sxmos.hpp @@ -0,0 +1,394 @@ +#pragma once +#ifndef I2SXMOS_HPP +#define I2SXMOS_HPP + +#include "hal/sensor.hpp" + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace porting { + +using namespace hal; + +class SensorI2SXMOS final: public Sensor +{ +public: + static inline core::ConfigObjectMap DEFAULT_CONFIGS() noexcept + { + return { CONFIG_OBJECT_DECL_INTEGER("bclk_pin", "I2S BCLK pin", 8, 0, 48), + CONFIG_OBJECT_DECL_INTEGER("ws_pin", "I2S WS/LRCLK pin", 7, 0, 48), + CONFIG_OBJECT_DECL_INTEGER("dout_pin", "I2S DOUT pin (unused for RX)", 43, 0, 48), + CONFIG_OBJECT_DECL_INTEGER("din_pin", "I2S DIN pin", 44, 0, 48), + CONFIG_OBJECT_DECL_INTEGER("sr", "PCM sample rate in Hz", 16000, 8000, 48000), + CONFIG_OBJECT_DECL_INTEGER("channels", "Number of channels", 1, 1, 1), + CONFIG_OBJECT_DECL_INTEGER("buffered_duration", "Time duration of buffered data for DMA in seconds", 2, 1, + 5) }; + } + + SensorI2SXMOS() noexcept : Sensor(Info(3, "I2S XMOS (ReSpeaker)", Type::Microphone, { DEFAULT_CONFIGS() })) { } + + core::Status init() noexcept override + { + const std::lock_guard lock(_lock); + + if (_info.status != Status::Uninitialized) [[unlikely]] + { + return STATUS(ENXIO, "Sensor is already initialized or in an invalid state"); + } + + const size_t sr = _info.configs["sr"].getValue(); + _channels = _info.configs["channels"].getValue(); + _buffered_duration = _info.configs["buffered_duration"].getValue(); + _data_buffer_capacity_frames = _buffered_duration * sr; + + if (!_data_buffer) [[likely]] + { + _data_buffer_capacity_bytes = _data_buffer_capacity_frames * _channels * sizeof(int16_t); + _data_buffer = std::shared_ptr(new std::byte[_data_buffer_capacity_bytes]); + if (!_data_buffer) [[unlikely]] + { + LOG(ERROR, "Failed to allocate data buffer, size: %zu bytes", _data_buffer_capacity_bytes); + return STATUS(ENOMEM, "Failed to allocate data buffer"); + } + LOG(DEBUG, "Allocated data buffer of size: %zu bytes at %p", _data_buffer_capacity_bytes, + _data_buffer.get()); + } + + if (!_rx_chan) [[likely]] + { + const uint32_t frames_count = 800; + const uint32_t slots_count = _data_buffer_capacity_frames / frames_count; + LOG(DEBUG, + "Creating I2S channel with sample rate %d, buffered frames %d, sync frame count %ld, slots count %ld", + sr, _data_buffer_capacity_frames, frames_count, slots_count); + _rx_chan_cfg.dma_desc_num = slots_count; + _rx_chan_cfg.dma_frame_num = frames_count; + + auto ret = i2s_new_channel(&_rx_chan_cfg, nullptr, &_rx_chan); + if (ret != ESP_OK) [[unlikely]] + { + LOG(ERROR, "Failed to create I2S channel: %s", esp_err_to_name(ret)); + return STATUS(EIO, "Failed to create I2S channel"); + } + + ret = i2s_channel_register_event_callback(_rx_chan, &_event_cbs, this); + if (ret != ESP_OK) [[unlikely]] + { + LOG(ERROR, "Failed to register I2S channel event callback: %s", esp_err_to_name(ret)); + return STATUS(EIO, "Failed to register I2S channel event callback"); + } + } + + { + const gpio_num_t bclk = static_cast(_info.configs["bclk_pin"].getValue()); + const gpio_num_t ws = static_cast(_info.configs["ws_pin"].getValue()); + const gpio_num_t dout = static_cast(_info.configs["dout_pin"].getValue()); + const gpio_num_t din = static_cast(_info.configs["din_pin"].getValue()); + const uint32_t sample_rate = static_cast(_info.configs["sr"].getValue()); + + _std_cfg = { + .clk_cfg = I2S_STD_CLK_DEFAULT_CONFIG(sample_rate), + .slot_cfg = { + .data_bit_width = I2S_DATA_BIT_WIDTH_16BIT, + .slot_bit_width = I2S_SLOT_BIT_WIDTH_32BIT, + .slot_mode = I2S_SLOT_MODE_MONO, + .slot_mask = I2S_STD_SLOT_LEFT, + .ws_width = 32, + .ws_pol = false, + .bit_shift = true, + .left_align = true, + .big_endian = false, + .bit_order_lsb = false, + }, + .gpio_cfg = { + .mclk = I2S_GPIO_UNUSED, + .bclk = bclk, .ws = ws, .dout = dout, .din = din, + .invert_flags = { .mclk_inv = false, .bclk_inv = false, .ws_inv = false }, + }, + }; + + auto ret = i2s_channel_init_std_mode(_rx_chan, &_std_cfg); + if (ret != ESP_OK) [[unlikely]] + { + LOG(ERROR, "Failed to initialize I2S channel in PDM RX mode: %s", esp_err_to_name(ret)); + return STATUS(EIO, "Failed to initialize I2S channel in PDM RX mode"); + } + } + + _frame_index = 0; + _data_bytes_available = 0; + { + auto ret = i2s_channel_enable(_rx_chan); + if (ret != ESP_OK) [[unlikely]] + { + LOG(ERROR, "Failed to enable I2S channel: %s", esp_err_to_name(ret)); + return STATUS(EIO, "Failed to enable I2S channel"); + } + } + + _info.status = Status::Idle; + + return STATUS_OK(); + } + + core::Status deinit() noexcept override + { + const std::lock_guard lock(_lock); + + if (_info.status == Status::Locked) [[unlikely]] + { + return STATUS(EBUSY, "Sensor is locked"); + } + + _data_buffer_capacity_frames = 0; + _data_buffer_capacity_bytes = 0; + + _frame_index = 0; + _data_bytes_available = 0; + + if (_data_buffer) [[likely]] + { + if (_data_buffer.use_count() > 1) [[unlikely]] + { + LOG(ERROR, "Data buffer is already in use by another process"); + return STATUS(EBUSY, "Data buffer is already in use"); + } + + _data_buffer.reset(); + } + + if (_rx_chan) [[likely]] + { + auto ret = i2s_channel_disable(_rx_chan); + if (ret != ESP_OK) [[unlikely]] + { + LOG(ERROR, "Failed to disable I2S channel: %s", esp_err_to_name(ret)); + return STATUS(EIO, "Failed to disable I2S channel"); + } + ret = i2s_del_channel(_rx_chan); + if (ret != ESP_OK) [[unlikely]] + { + LOG(ERROR, "Failed to delete I2S channel: %s", esp_err_to_name(ret)); + return STATUS(EIO, "Failed to delete I2S channel"); + } + _rx_chan = nullptr; + } + + _info.status = Status::Uninitialized; + + return STATUS_OK(); + } + + core::Status updateConfig(const core::ConfigMap &) noexcept override + { + return STATUS(ENOTSUP, "Update config is not supported for SensorI2SXMOS"); + } + + inline size_t dataAvailable() const noexcept override + { + if (!initialized()) [[unlikely]] + { + return 0; + } + + const std::lock_guard lock(_lock); + + return internalDataAvailable(); + } + + inline size_t dataClear() noexcept override + { + if (!initialized()) [[unlikely]] + { + return 0; + } + const std::lock_guard lock(_lock); + + const size_t data_available = internalDataAvailable(); + if (data_available == 0) [[unlikely]] + { + return 0; + } + + return internalDataDiscard(data_available); + } + + inline core::Status readDataFrame(core::DataFrame> &data_frame, + size_t batch_size) noexcept override + { + if (batch_size == 0) [[unlikely]] + { + LOG(ERROR, "Batch size cannot be zero"); + return STATUS(EINVAL, "Batch size cannot be zero"); + } + + if (!initialized()) [[unlikely]] + { + LOG(ERROR, "Sensor is not initialized"); + return STATUS(ENXIO, "Sensor is not initialized"); + } + if (_info.status != Status::Idle) [[unlikely]] + { + LOG(ERROR, "Sensor is not in idle state"); + return STATUS(EINVAL, "Sensor is not in idle state"); + } + if (_data_buffer.use_count() > 1) [[unlikely]] + { + LOG(ERROR, "Data buffer is already in use by another process"); + return STATUS(EBUSY, "Data buffer is already in use"); + } + + if (batch_size > _data_buffer_capacity_frames) [[unlikely]] + { + LOG(WARNING, "Batch size exceeds buffer capacity: %zu > %zu", batch_size, _data_buffer_capacity_frames); + batch_size = _data_buffer_capacity_frames; + } + + const std::lock_guard lock(_lock); + + _info.status = Status::Locked; + + data_frame.timestamp = std::chrono::steady_clock::now(); + + size_t read = 0; + { + size_t size = batch_size * _channels * sizeof(int16_t); + auto ret = i2s_channel_read(_rx_chan, _data_buffer.get(), size, &read, + pdMS_TO_TICKS((_buffered_duration + 1) * 1000)); + if (ret != ESP_OK) [[unlikely]] + { + LOG(ERROR, "Failed to read data from I2S channel: %s", esp_err_to_name(ret)); + _info.status = Status::Idle; + return STATUS(EIO, "Failed to read data from I2S channel"); + } + internalConsumeData(read); + + if (read != size) [[unlikely]] + { + LOG(WARNING, "Read %zu bytes, expected %zu bytes", read, size); + if (!read) [[unlikely]] + { + _info.status = Status::Idle; + return STATUS(EIO, "No data read from I2S channel"); + } + batch_size = read / (_channels * sizeof(int16_t)); + read = batch_size * (_channels * sizeof(int16_t)); + } + } + data_frame.data = core::Tensor::create(core::Tensor::Type::Int16, + core::Tensor::Shape(static_cast(batch_size), static_cast(_channels)), _data_buffer, read); + + data_frame.index = _frame_index; + _frame_index += batch_size; + + _info.status = Status::Idle; + + return data_frame.data ? STATUS_OK() : STATUS(ENOMEM, "Failed to create data frame tensor"); + } + +protected: + static inline bool isrOnReceive(i2s_chan_handle_t, i2s_event_data_t *event, void *user_ctx) + { + if (!event || !user_ctx) [[unlikely]] + { + return false; + } + auto *self = static_cast(user_ctx); + + size_t size = event->size; + size_t data_bytes_available = self->_data_bytes_available.load(std::memory_order_acquire); + while (1) + { + const size_t new_data_bytes_available + = std::min(data_bytes_available + size, self->_data_buffer_capacity_bytes); + if (self->_data_bytes_available.compare_exchange_strong(data_bytes_available, new_data_bytes_available, + std::memory_order_release, std::memory_order_relaxed)) [[likely]] + { + return false; + } + } + + return false; + } + +private: + inline size_t internalDataAvailable() const noexcept + { + return _data_bytes_available.load(std::memory_order_acquire) / (_channels * sizeof(int16_t)); + } + + inline size_t internalDataDiscard(size_t size) noexcept + { + size_t discarded = 0; + while (discarded < size) + { + size_t read = std::min(static_cast((size - discarded) * _channels * sizeof(int16_t)), + _data_buffer_capacity_bytes); + auto ret = i2s_channel_read(_rx_chan, _data_buffer.get(), read, &read, + pdMS_TO_TICKS((_buffered_duration + 1) * 1000)); + if (ret != ESP_OK) [[unlikely]] + { + LOG(ERROR, "Failed to read data from I2S channel: %s", esp_err_to_name(ret)); + } + internalConsumeData(read); + discarded += read; + } + return size; + } + + inline void internalConsumeData(size_t size) noexcept + { + if (!size) [[unlikely]] + { + return; + } + size_t data_bytes_available = _data_bytes_available.load(std::memory_order_acquire); + while (1) + { + size = std::min(size, data_bytes_available); + const size_t new_data_bytes_available = data_bytes_available - size; + if (_data_bytes_available.compare_exchange_strong(data_bytes_available, new_data_bytes_available, + std::memory_order_release, std::memory_order_relaxed)) [[likely]] + { + return; + } + } + } + + mutable std::mutex _lock; + + std::shared_ptr _data_buffer = nullptr; + size_t _data_buffer_capacity_frames = 0; + static inline size_t _data_buffer_capacity_bytes = 0; + + size_t _channels = 1; + size_t _buffered_duration = 0; + + size_t _frame_index = 0; + static inline std::atomic _data_bytes_available = 0; + + i2s_chan_handle_t _rx_chan = nullptr; + i2s_chan_config_t _rx_chan_cfg = I2S_CHANNEL_DEFAULT_CONFIG(I2S_NUM_AUTO, I2S_ROLE_SLAVE); + i2s_std_config_t _std_cfg = {}; + i2s_event_callbacks_t _event_cbs + = { .on_recv = isrOnReceive, .on_recv_q_ovf = nullptr, .on_sent = nullptr, .on_send_q_ovf = nullptr }; +}; + +} // namespace porting + +#endif \ No newline at end of file diff --git a/components/acoustics-porting/porting/sensor_esp32s3.cpp b/components/acoustics-porting/porting/sensor_esp32s3.cpp index de2816d..517fd51 100644 --- a/components/acoustics-porting/porting/sensor_esp32s3.cpp +++ b/components/acoustics-porting/porting/sensor_esp32s3.cpp @@ -2,13 +2,22 @@ #include "sensor/i2smic.hpp" #include "sensor/lis3dhtr.hpp" +#if defined(PORTING_BOARD_MODEL_RESPEAKER_LITE) && PORTING_BOARD_MODEL_RESPEAKER_LITE +#include "sensor/i2sxmos.hpp" +#endif namespace bridge { void __REGISTER_SENSORS__() { [[maybe_unused]] static porting::SensorLIS3DHTR sensor_lis3dhtr; +#if defined(PORTING_BOARD_MODEL_RESPEAKER_LITE) && PORTING_BOARD_MODEL_RESPEAKER_LITE + [[maybe_unused]] static porting::SensorI2SXMOS sensor_i2sxmos; +#elif defined(PORTING_BOARD_MODEL_XIAO_S3) && PORTING_BOARD_MODEL_XIAO_S3 [[maybe_unused]] static porting::SensorI2SMic sensor_i2smic; +#else + [[maybe_unused]] static porting::SensorI2SMic sensor_i2smic; +#endif } } // namespace bridge diff --git a/components/acoustics/api/v1/task_sc.hpp b/components/acoustics/api/v1/task_sc.hpp index 4e399fa..0f2fb42 100644 --- a/components/acoustics/api/v1/task_sc.hpp +++ b/components/acoustics/api/v1/task_sc.hpp @@ -549,4 +549,4 @@ struct TaskSC final } // namespace v1 -#endif +#endif \ No newline at end of file diff --git a/examples/sound_classification/main/CMakeLists.txt b/examples/sound_classification/main/CMakeLists.txt index f36d079..29cdd0e 100644 --- a/examples/sound_classification/main/CMakeLists.txt +++ b/examples/sound_classification/main/CMakeLists.txt @@ -15,6 +15,21 @@ set(ACOUSTICS_LIB_OPUS_ENABLE ON GLOBAL BOOL "Enable OPUS support in acoustics-p set(ACOUSTICS_PORTING_LIB_TFLM_ENABLE ON GLOBAL BOOL "Enable TFLM support in acoustics-porting") set(ACOUSTICS_PORTING_LIB_DL_FFT_ENABLE ON GLOBAL BOOL "Enable DL_FFT support in acoustics-porting") +if(NOT DEFINED PORTING_BOARD_MODEL_RESPEAKER_LITE) + set(PORTING_BOARD_MODEL_RESPEAKER_LITE 0) +endif() +if(NOT DEFINED PORTING_BOARD_MODEL_XIAO_S3) + set(PORTING_BOARD_MODEL_XIAO_S3 1) +endif() + +if(PORTING_BOARD_MODEL_RESPEAKER_LITE) + message(STATUS "Board: ReSpeaker Lite enabled") +elseif(PORTING_BOARD_MODEL_XIAO_S3) + message(STATUS "Board: XIAO ESP32S3 enabled (default)") +else() + message(STATUS "Board: No specific board model enabled") +endif() + idf_component_register( SRCS ${PROJ_SRCS} INCLUDE_DIRS ${PROJ_INCLUDE_DIRS}