diff --git a/.gitignore b/.gitignore index aa6da67..32e6264 100644 --- a/.gitignore +++ b/.gitignore @@ -140,4 +140,5 @@ obj # Node stuff node_modules -~$*.docx \ No newline at end of file +~$*.docx +*.a diff --git a/labs/README.md b/labs/README.md index 7dfa8cf..195485f 100644 --- a/labs/README.md +++ b/labs/README.md @@ -22,6 +22,12 @@ These labs cover running AI workloads either in the cloud from the IoT device, o * [Assembly line QA](./ai-edge/vision/manufacturing-part-check/) - A prototype of an AI image classification based quality assurance tool for manufacturing showing how to detect broken parts on an assembly line using AI, controlled from another device * [Speech](./ai-edge/speech) - Speech to text, text to speech and speech translation using a Raspberry Pi and USB microphone/speaker +## TinyML + +These labs cover training tiny machine learning workloads and running them on embedded hardware such as Adruino microcontrollers. + +* [Audio classifier](./tiny-ml/audio-classifier/) - a TinyML lab to capture audio data, use it to train a model, then classify audio data using that model on an Arduino Nano 33 Sense BLE board. + ## Digital Agriculture These labs cover scenarios around digital agriculture. diff --git a/labs/ai-edge/vision/manufacturing-part-check/code/analytics/esp-eye/src/ImageHandler.cpp b/labs/ai-edge/vision/manufacturing-part-check/code/analytics/esp-eye/src/ImageHandler.cpp index 76c0dad..8ea9a6f 100644 --- a/labs/ai-edge/vision/manufacturing-part-check/code/analytics/esp-eye/src/ImageHandler.cpp +++ b/labs/ai-edge/vision/manufacturing-part-check/code/analytics/esp-eye/src/ImageHandler.cpp @@ -64,7 +64,7 @@ bool ImageHandler::SavePhoto(camera_fb_t *frameBuffer, const char *fileName) // Checks if the photo has been saved by validating the file size bool ImageHandler::CheckPhoto(const char *fileName) { - // Open the file from teh SPIFFS file system + // Open the file from the SPIFFS file system File f_pic = SPIFFS.open(fileName); // Get the file size diff --git a/labs/ai-edge/vision/manufacturing-part-check/code/custom-vision/src/ImageHandler.cpp b/labs/ai-edge/vision/manufacturing-part-check/code/custom-vision/src/ImageHandler.cpp index 76c0dad..8ea9a6f 100644 --- a/labs/ai-edge/vision/manufacturing-part-check/code/custom-vision/src/ImageHandler.cpp +++ b/labs/ai-edge/vision/manufacturing-part-check/code/custom-vision/src/ImageHandler.cpp @@ -64,7 +64,7 @@ bool ImageHandler::SavePhoto(camera_fb_t *frameBuffer, const char *fileName) // Checks if the photo has been saved by validating the file size bool ImageHandler::CheckPhoto(const char *fileName) { - // Open the file from teh SPIFFS file system + // Open the file from the SPIFFS file system File f_pic = SPIFFS.open(fileName); // Get the file size diff --git a/labs/ai-edge/vision/manufacturing-part-check/code/image-capture/src/ImageHandler.cpp b/labs/ai-edge/vision/manufacturing-part-check/code/image-capture/src/ImageHandler.cpp index 76c0dad..8ea9a6f 100644 --- a/labs/ai-edge/vision/manufacturing-part-check/code/image-capture/src/ImageHandler.cpp +++ b/labs/ai-edge/vision/manufacturing-part-check/code/image-capture/src/ImageHandler.cpp @@ -64,7 +64,7 @@ bool ImageHandler::SavePhoto(camera_fb_t *frameBuffer, const char *fileName) // Checks if the photo has been saved by validating the file size bool ImageHandler::CheckPhoto(const char *fileName) { - // Open the file from teh SPIFFS file system + // Open the file from the SPIFFS file system File f_pic = SPIFFS.open(fileName); // Get the file size diff --git a/labs/ai-edge/vision/manufacturing-part-check/code/pi-control/esp-eye/src/ImageHandler.cpp b/labs/ai-edge/vision/manufacturing-part-check/code/pi-control/esp-eye/src/ImageHandler.cpp index 76c0dad..8ea9a6f 100644 --- a/labs/ai-edge/vision/manufacturing-part-check/code/pi-control/esp-eye/src/ImageHandler.cpp +++ b/labs/ai-edge/vision/manufacturing-part-check/code/pi-control/esp-eye/src/ImageHandler.cpp @@ -64,7 +64,7 @@ bool ImageHandler::SavePhoto(camera_fb_t *frameBuffer, const char *fileName) // Checks if the photo has been saved by validating the file size bool ImageHandler::CheckPhoto(const char *fileName) { - // Open the file from teh SPIFFS file system + // Open the file from the SPIFFS file system File f_pic = SPIFFS.open(fileName); // Get the file size diff --git a/labs/ai-edge/vision/manufacturing-part-check/steps/image-capture.md b/labs/ai-edge/vision/manufacturing-part-check/steps/image-capture.md index e94852c..cfa106b 100644 --- a/labs/ai-edge/vision/manufacturing-part-check/steps/image-capture.md +++ b/labs/ai-edge/vision/manufacturing-part-check/steps/image-capture.md @@ -77,7 +77,7 @@ To access the camera and host a web server, you can use some pre-existing Arduin ESP Async WebServer ``` - This will add the [Espressif ESP32 Camera driver](https://github.com/espressif/esp32-camera) and the [ESP Async WebServer](https://github.com/me-no-dev/ESPAsyncWebServer) libraries to the project. The ESP32 camera driver is a library to talk to the camera on the ESP-EYE board, and ht eESP Async WebServer allows you to run a web server on the ESP board. + This will add the [Espressif ESP32 Camera driver](https://github.com/espressif/esp32-camera) and the [ESP Async WebServer](https://github.com/me-no-dev/ESPAsyncWebServer) libraries to the project. The ESP32 camera driver is a library to talk to the camera on the ESP-EYE board, and the ESP Async WebServer allows you to run a web server on the ESP board. Next time the app is compiled, it will bring in these libraries and compile them into the build. @@ -362,7 +362,7 @@ Just like code for interacting with the camera, the code for interacting with th // Checks if the photo has been saved by validating the file size bool ImageHandler::CheckPhoto(const char *fileName) { - // Open the file from teh SPIFFS file system + // Open the file from the SPIFFS file system File f_pic = SPIFFS.open(fileName); // Get the file size diff --git a/labs/ai-edge/vision/manufacturing-part-check/steps/pi-configuration.md b/labs/ai-edge/vision/manufacturing-part-check/steps/pi-configuration.md deleted file mode 100644 index 963a5b6..0000000 --- a/labs/ai-edge/vision/manufacturing-part-check/steps/pi-configuration.md +++ /dev/null @@ -1,39 +0,0 @@ -# Pi Configuration - -```sh -# Update all the things -sudo apt update && sudo apt upgrade -y -sudo reboot - -# Install the Grove stuff -sudo curl -kL dexterindustries.com/update_grovepi | bash -sudo reboot - -# Update the Grove firmware -git clone https://github.com/DexterInd/GrovePi.git -cd GrovePi/Firmware -sudo ./firmware_update.sh - -# Install docker -curl -sSL https://get.docker.com | sh -sudo usermod -aG docker pi -sudo reboot - -# Install IoT Edge -curl https://packages.microsoft.com/config/debian/stretch/multiarch/prod.list > ./microsoft-prod.list -sudo cp ./microsoft-prod.list /etc/apt/sources.list.d/ -curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg -sudo cp ./microsoft.gpg /etc/apt/trusted.gpg.d/ -sudo apt update -sudo apt -y install iotedge - -sudo nano /etc/iotedge/config.yaml -# Set connection string -sudo systemctl restart iotedge - -# Install IoT Hub python packages -sudo apt install cmake libssl-dev -y -pip3 install azure-iot-hub -pip3 install python-dotenv - -``` \ No newline at end of file diff --git a/labs/images/vscode-platformio-menu.png b/labs/images/vscode-platformio-menu.png index b422d34..2b209d9 100644 Binary files a/labs/images/vscode-platformio-menu.png and b/labs/images/vscode-platformio-menu.png differ diff --git a/labs/tiny-ml/audio-classifier/README.md b/labs/tiny-ml/audio-classifier/README.md new file mode 100644 index 0000000..c1b09e6 --- /dev/null +++ b/labs/tiny-ml/audio-classifier/README.md @@ -0,0 +1,48 @@ +# TinyML Audio classifier + +This folder contains a lab with multiple parts working towards an audio classifier running on an Arduino Nano 33 BLE Sense microcontroller, taking advantage of the built-in microphone. + +> This lab was inspired by [Better word classification with Arduino Nano 33 BLE Sense and Machine Learning](https://eloquentarduino.github.io/2020/08/better-word-classification-with-arduino-33-ble-sense-and-machine-learning/) by [Eloquent Arduino](https://eloquentarduino.github.io/about-me/). + +| Author | [Jim Bennett](https://github.com/JimBobBennett) | +|:---|:---| +| Target platform | | +| Hardware required | | +| Software required | | +| Programming Language | | +| Prerequisites | Basic proficiency in using VS Code, C++ and Python.
If you want to learn Python, check out these free resources:
| +| Date | February 2021 | +| Learning Objectives | | +| Time to complete | 1 hours | + +## The lab parts + +This lab has the following parts + +1. Program the Arduino device for audio capture and capture training data +1. Train a ML model using the training data +1. Program the Arduino device to classify audio + +## Audio classification + +Audo classification is the process of classifying a sound based on labelled training data. For example - you could train a model by using multiple recordings of someone saying the word "Yes" labelled as `Yes`, and multiple recordings of someone saying the word "No" labelled as `No`. The model could then take a new sound recording and classify it as either `Yes` or `No`. + +This lab starts by coding the Arduino to record multiple samples that are labelled, then these labelled samples are used to train a model, which is then add to device code that runs on the microcontroller to classify new audio data. + +The classifier you will create here needs at least 2 labels in the model, and will pick the most probable one to classify the audio. + +## TinyML + +TinyML is the coming together of machine learning and embedded systems. It involves training ML models that are tiny - that is substantially smaller than the models historically created, and able to run on microcontrollers with limited memory and power. It was originally defined as ML models that can run using less than 1mW of power, but has become a general term for running ML on microcontrollers. + +Microcontrollers have memory limits usually in the kilobytes, meaning traditional ML models that are many megabytes in size cannot even be installed on the device, let alone run. By using TinyML models you can bring the world of ML to a microcontroller. An audio classifier (a model that can distinguish between multiple sounds) or a wake word model (a model that can detect one specific sound, such as the command to wake a smart device up), for example, can be compressed to less than 20KB using TinyML. + +There is a trade off - with smaller models you lose accuracy, but this is an acceptable tradeoff for the advantages of smaller, low powered models. For example, if you were creating a voice activated smart device you would want it to consume as little power as possible waiting for a wake word, and only then powering up to listen for more instructions and using bigger models, or even sending data to the cloud. If you are using audio to classify animal behavior on a smart collar, you want long battery life and the device to be as small and light-weight as possible to reduce the inconvenience to the animal being tracked. + +In this lab, the device in question is an [Arduino Nano 33 BLE Sense microcontroller](https://store.arduino.cc/usa/nano-33-ble-sense) - a board that has a built in microphone that can be used to detect and classify audio signals. It has 256KB of memory available. + +## Labs + +1. [Create the audio capture tool](./steps/audio-capture.md) +1. [Train the TinyML model](./steps/train-model.md) +1. [Create the audio classifier](./steps/audio-classifier.md) diff --git a/labs/tiny-ml/audio-classifier/code/audio-capture/.gitignore b/labs/tiny-ml/audio-classifier/code/audio-capture/.gitignore new file mode 100644 index 0000000..89cc49c --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/audio-capture/.gitignore @@ -0,0 +1,5 @@ +.pio +.vscode/.browse.c_cpp.db* +.vscode/c_cpp_properties.json +.vscode/launch.json +.vscode/ipch diff --git a/labs/tiny-ml/audio-classifier/code/audio-capture/.vscode/extensions.json b/labs/tiny-ml/audio-classifier/code/audio-capture/.vscode/extensions.json new file mode 100644 index 0000000..0f0d740 --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/audio-capture/.vscode/extensions.json @@ -0,0 +1,7 @@ +{ + // See http://go.microsoft.com/fwlink/?LinkId=827846 + // for the documentation about the extensions.json format + "recommendations": [ + "platformio.platformio-ide" + ] +} diff --git a/labs/tiny-ml/audio-classifier/code/audio-capture/README.md b/labs/tiny-ml/audio-classifier/code/audio-capture/README.md new file mode 100644 index 0000000..1bcde42 --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/audio-capture/README.md @@ -0,0 +1,30 @@ +# Audio Capture + +This folder contains a [PlatformIO](https://platformio.org/platformio-ide) project to build an audio capture app that runs on an [Arduino Nano 33 BLE Sense](https://store.arduino.cc/usa/nano-33-ble-sense) board. + +You can read more about this project in the [top level README.md file](../README.md). + +To build and deploy this project, you will need to open this folder in [Visual Studio Code](https://code.visualstudio.com/?WT.mc_id=academic-7372-jabenn) with the [PlatformIO extension](https://platformio.org/platformio-ide) installed. You will then be able to build the project and upload to a Arduino Nano 33 BLE Sense board. + +## Building this project + +This project has a library dependency on the ARM CMSIS static library. You will need to download this from the ARM GitHub repo. + +* Head to [the CMSIS GCC libs folder in the GitHub repo](https://github.com/ARM-software/CMSIS_5/tree/5.7.0/CMSIS/DSP/Lib/GCC) +* Download `libarm_cortexM4l_math.a` from that repo page +* Add the file to the root folder of this project +* Build and upload as normal + +## Running the project + +Once the project is running on the board, it will listen for audio and output RMS values to the serial port. + +* Connect to the serial monitor to view the audio values +* Make the one of the relevant noises into the microphone of the board. Pause after each noise and you will see a line output to the serial monitor. +* Repeat 15-30 times for that one noise +* Copy the values from the serial monitor into a CSV file named using the name of the label (for eample if you were capturing numbers you would put the data for one into `one.csv`, for two into `two.csv` and so on). +* Clear the serial output and repeat the above steps for all the noises you are interested in + +## Processing the output + +Refer to the [top level README.md file](../README.md) for instructions on how to process and use the output to classify noises. diff --git a/labs/tiny-ml/audio-classifier/code/audio-capture/include/README b/labs/tiny-ml/audio-classifier/code/audio-capture/include/README new file mode 100644 index 0000000..194dcd4 --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/audio-capture/include/README @@ -0,0 +1,39 @@ + +This directory is intended for project header files. + +A header file is a file containing C declarations and macro definitions +to be shared between several project source files. You request the use of a +header file in your project source file (C, C++, etc) located in `src` folder +by including it, with the C preprocessing directive `#include'. + +```src/main.c + +#include "header.h" + +int main (void) +{ + ... +} +``` + +Including a header file produces the same results as copying the header file +into each source file that needs it. Such copying would be time-consuming +and error-prone. With a header file, the related declarations appear +in only one place. If they need to be changed, they can be changed in one +place, and programs that include the header file will automatically use the +new version when next recompiled. The header file eliminates the labor of +finding and changing all the copies as well as the risk that a failure to +find one copy will result in inconsistencies within a program. + +In C, the usual convention is to give header files names that end with `.h'. +It is most portable to use only letters, digits, dashes, and underscores in +header file names, and at most one dot. + +Read more about using header files in official GCC documentation: + +* Include Syntax +* Include Operation +* Once-Only Headers +* Computed Includes + +https://gcc.gnu.org/onlinedocs/cpp/Header-Files.html diff --git a/labs/tiny-ml/audio-classifier/code/audio-capture/platformio.ini b/labs/tiny-ml/audio-classifier/code/audio-capture/platformio.ini new file mode 100644 index 0000000..1b1b1f5 --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/audio-capture/platformio.ini @@ -0,0 +1,18 @@ +; PlatformIO Project Configuration File +; +; Build options: build flags, source filter +; Upload options: custom upload port, speed and extra flags +; Library options: dependencies, extra library storages +; Advanced options: extra scripting +; +; Please visit documentation for the other options and examples +; https://docs.platformio.org/page/projectconf.html + +[env:nano33ble] +platform = nordicnrf52 +board = nano33ble +framework = arduino +build_flags = + -L. + -l arm_cortexM4l_math + -w diff --git a/labs/tiny-ml/audio-classifier/code/audio-capture/src/main.cpp b/labs/tiny-ml/audio-classifier/code/audio-capture/src/main.cpp new file mode 100644 index 0000000..a2a2c88 --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/audio-capture/src/main.cpp @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * Audio Capture + * + * This program listens on the microphone capturing audio data and writing it out as + * CSV file lines to the serial port. + * + * Audio data is captured as a full 512 byte data buffer (1/64 second of audio data). + * Each buffer is then converted into a root mean square value of the buffer to create a smaller + * representation of the audio. + * + * If this root mean square value is over a threshold (i.e. not silence), then the next SAMPLE_SIZE + * buffers are captured and the RMS values calculated. Once a full set of samples is retrieved, a + * callback function is invoked, passing this set of samples. + * + * The default for SAMPLE_SIZE is 128 - 2 seconds of audio data. You can configure this in the + * sample_capture.h header file. + * + * This essentially converts audio data to a smaller representation for use with training TinyML models. + * For example - 2 seconds of audio * becomes 128 4-byte float values (512 bytes) instead of + * 32,768 2-byte integerss (65,536 bytes). + * + * The output sent to the serial port can be saved as a CSV file and used to train the model + * + */ + +#include "sample_capture.h" + +// A helper class that captures audio from the microphone +SampleCapture sampleCapture; + +// A buffer used to store data read from the Sample Capture class +float _samples[SAMPLE_SIZE]; + +// Tracks if we have samples ready to log to the serial port +bool _ready; + +/** + * @brief A callback that is called whenever the sample capture object has a full buffer of audio + * RMS values ready for processing + */ +void onSamples(float *samples) +{ + memcpy(_samples, samples, SAMPLE_SIZE * sizeof(float)); + _ready = true; +} + +/** + * @brief Process the samples, writing them out to the serial port + */ +void procesSamples() +{ + // print the audio data to serial port + for (int i = 0; i < SAMPLE_SIZE; i++) + { + Serial.print(_samples[i], 6); + + // Seperate the audio values with commas, at the last value + // send a newline + Serial.print(i == SAMPLE_SIZE - 1 ? '\n' : ','); + } +} + +/** + * @brief Sets up the serial port and the sample capture object + */ +void setup() +{ + // Start the serial connection so the captured audio data can be output + Serial.begin(115200); + + // Start the sample capture object to listen for audio and callback when + // it has a full set of samples + sampleCapture.init(onSamples); + + // Wait 3 seconds for everything to get started + delay(3000); +} + +/** + * @brief Runs continuously capturing audio data and writing it to + * the serial port + */ +void loop() +{ + // check to see if we have audio data + if (_ready) + { + // If we do, mark it as read ready for the next loop + _ready = false; + + // Process the samples + procesSamples(); + } + + // Sleep to allow background microphone processing + // Each sample is 2 seconds, so sleeping for 1 second is fine + delay(1000); +} diff --git a/labs/tiny-ml/audio-classifier/code/audio-capture/src/sample_capture.h b/labs/tiny-ml/audio-classifier/code/audio-capture/src/sample_capture.h new file mode 100644 index 0000000..423e8db --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/audio-capture/src/sample_capture.h @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +#ifndef __SAMPLE_CAPTURE_H__ +#define __SAMPLE_CAPTURE_H__ + +#include +#include + +// The size of the data coming in from the microphone +#define BUFFER_SIZE 512U + +// 128 samples is enough for 2 seconds of audio - it's captured at 64 samples per second +#define SAMPLE_SIZE 128 +#define GAIN (1.0f / 50) +#define SOUND_THRESHOLD 1000 + +/** + * @brief The function signature for the callback function that is called when we have a full set of samples. + */ +typedef void (*samples_ready_callback)(float*); + +/** + * @brief A helper class for accessing the BLEs microphone. + * + * Audio data is captured as a full 512 byte data buffer (1/64 second of audio data). + * Each buffer is then converted into a root mean square value of the buffer to create a smaller + * representation of the audio. + * + * If this root mean square value is over a threshold (i.e. not silence), then the next SAMPLE_SIZE + * buffers are captured and the RMS values calculated. Once a full set of samples is retrieved, a + * callback function is invoked, passing this set of samples. + * + * This essentially converts audio data to a smaller representation for use with training and inference + * with TinyML models. + * For example - 2 seconds of audio * becomes 128 4-byte float values (512 bytes) instead of + * 32,768 2-byte integerss (65,536 bytes). + */ +class SampleCapture +{ +public: + /** + * @brief Setup the PDM library to access the microphone at a sample rate of 16KHz + * @param callback A callback function to call when there is a full set of samples ready. + * This callback will take the samples as a parameter as a SAMPLE_SIZE array of floats. + * This array is owned by this class, so the callback will need to take a copy. + */ + void init(samples_ready_callback callback) + { + // Set up the audio callback + PDMHelper::setSampleCapture(this); + + // Set up the callback when a set of samples is ready + _callback = callback; + + // Start listening on the microphone at a sample rate of 16KHz + PDM.begin(1, 16000); + PDM.setGain(20); + } + +private: + /** + * @brief A helper wrapper class that can connect a method on the SampleCapture object + * to the PDM callback that expects a static method + */ + class PDMHelper + { + public: + /** + * @brief Sets up the PDM callback to a method on the SampleCapture class + */ + static void setSampleCapture(SampleCapture *sampleCapture) + { + // Store the sample capture + _sampleCapture = sampleCapture; + + // Set up the callback + PDM.onReceive(PDMHelper::onReceive); + } + + private: + /** + * @brief The callback from the PDM class, calls the SampleCapture update method + */ + static void onReceive() + { + _sampleCapture->onReceive(); + } + + inline static SampleCapture *_sampleCapture; + }; + + /** + * @brief Reads the audio data from the PDM buffer and calculates the + * root mean square value, adding it to the samples if needed. + */ + void onReceive() + { + // Check we have a full buffers worth + if (PDM.available() == BUFFER_SIZE) + { + // Read from the buffer + PDM.read(_buffer, BUFFER_SIZE); + + // Calculate the root mean square value of the buffer + int16_t rms; + arm_rms_q15((q15_t *)_buffer, BUFFER_SIZE/sizeof(int16_t), (q15_t *)&rms); + + // If we are not currently collecting samples, check if the RMS value is + // above our threshold - as in we've heard something, not just silence. + // If we hear something, start collecting samples + if (!_started) + { + if (rms > SOUND_THRESHOLD) + { + _started = true; + _position = 0; + } + } + + // If were collecting data, either because we already were, or because we've + // just detected audio that's not slience, add it to the next slot in the samples + // array. + if (_started) + { + // Add the RMS value to the samples array in the next slot, multiplied by a + // gain value to give the signal a boost + _samples[_position] = rms * GAIN; + + // Move to the next slot in the samples + _position++; + + // If we're filled the samples buffer, stop collecting data and call the callback + if (_position >= SAMPLE_SIZE) + { + _started = false; + _position = 0; + + // Pass the samples to the callback + _callback(_samples); + } + } + } + } + + // The buffer to read from the PDM into - use this as a field to reduce overhead + // creating and deleteing it every time the buffer is full + int16_t _buffer[BUFFER_SIZE/sizeof(int16_t)]; + + // The samples buffer- use this as a field to reduce overhead + // creating and deleteing it every time the buffer is full + float _samples[SAMPLE_SIZE]; + + // Are we currently capturing data - we capture from when audio above the SOUND_THRESHOLD + // is detected, for SAMPLE_SIZE samples + bool _started; + + // The current position in the samples array to write the next sample to + int _position; + + // The callback to call when we have a full set of samples + samples_ready_callback _callback; +}; + +#endif __SAMPLE_CAPTURE_H__ \ No newline at end of file diff --git a/labs/tiny-ml/audio-classifier/code/audio-capture/test/README b/labs/tiny-ml/audio-classifier/code/audio-capture/test/README new file mode 100644 index 0000000..b94d089 --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/audio-capture/test/README @@ -0,0 +1,11 @@ + +This directory is intended for PlatformIO Unit Testing and project tests. + +Unit Testing is a software testing method by which individual units of +source code, sets of one or more MCU program modules together with associated +control data, usage procedures, and operating procedures, are tested to +determine whether they are fit for use. Unit testing finds problems early +in the development cycle. + +More information about PlatformIO Unit Testing: +- https://docs.platformio.org/page/plus/unit-testing.html diff --git a/labs/tiny-ml/audio-classifier/code/audio-classifier/.gitignore b/labs/tiny-ml/audio-classifier/code/audio-classifier/.gitignore new file mode 100644 index 0000000..89cc49c --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/audio-classifier/.gitignore @@ -0,0 +1,5 @@ +.pio +.vscode/.browse.c_cpp.db* +.vscode/c_cpp_properties.json +.vscode/launch.json +.vscode/ipch diff --git a/labs/tiny-ml/audio-classifier/code/audio-classifier/.vscode/extensions.json b/labs/tiny-ml/audio-classifier/code/audio-classifier/.vscode/extensions.json new file mode 100644 index 0000000..0f0d740 --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/audio-classifier/.vscode/extensions.json @@ -0,0 +1,7 @@ +{ + // See http://go.microsoft.com/fwlink/?LinkId=827846 + // for the documentation about the extensions.json format + "recommendations": [ + "platformio.platformio-ide" + ] +} diff --git a/labs/tiny-ml/audio-classifier/code/audio-classifier/README.md b/labs/tiny-ml/audio-classifier/code/audio-classifier/README.md new file mode 100644 index 0000000..3a7b19b --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/audio-classifier/README.md @@ -0,0 +1,24 @@ +# Audio Classifier + +This folder contains a [PlatformIO](https://platformio.org/platformio-ide) project to build an audio classifier app that runs on an [Arduino Nano 33 BLE Sense](https://store.arduino.cc/usa/nano-33-ble-sense) board. + +You can read more about this project in the [top level README.md file](../README.md). + +To build and deploy this project, you will need to open this folder in [Visual Studio Code](https://code.visualstudio.com/?WT.mc_id=academic-7372-jabenn) with the [PlatformIO extension](https://platformio.org/platformio-ide) installed. You will then be able to build the project and upload to a Arduino Nano 33 BLE Sense board. + +## Building this project + +This project has a library dependency on the ARM CMSIS static library. You will need to download this from the ARM GitHub repo. + +* Head to [the CMSIS GCC libs folder in the GitHub repo](https://github.com/ARM-software/CMSIS_5/tree/5.7.0/CMSIS/DSP/Lib/GCC) +* Download `libarm_cortexM4l_math.a` from that repo page +* Add the file to the root folder of this project + +You will also need to add code to the [`classifier.h`](./src/classifier.h) file. This code is generated by the model training utility. Refer to the [top level README.md file](../README.md) for more details on creating the contents of this file. + +## Running the project + +Once the project is running on the board, it will listen for audio, classify based on a trained model, then output the label of the predicted value to the serial port. + +* Connect to the serial monitor to view the audio values +* Make the one of the relevant noises into the microphone of the board. Pause after each noise and you will see the predicted label for that sound output to the serial monitor. diff --git a/labs/tiny-ml/audio-classifier/code/audio-classifier/include/README b/labs/tiny-ml/audio-classifier/code/audio-classifier/include/README new file mode 100644 index 0000000..194dcd4 --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/audio-classifier/include/README @@ -0,0 +1,39 @@ + +This directory is intended for project header files. + +A header file is a file containing C declarations and macro definitions +to be shared between several project source files. You request the use of a +header file in your project source file (C, C++, etc) located in `src` folder +by including it, with the C preprocessing directive `#include'. + +```src/main.c + +#include "header.h" + +int main (void) +{ + ... +} +``` + +Including a header file produces the same results as copying the header file +into each source file that needs it. Such copying would be time-consuming +and error-prone. With a header file, the related declarations appear +in only one place. If they need to be changed, they can be changed in one +place, and programs that include the header file will automatically use the +new version when next recompiled. The header file eliminates the labor of +finding and changing all the copies as well as the risk that a failure to +find one copy will result in inconsistencies within a program. + +In C, the usual convention is to give header files names that end with `.h'. +It is most portable to use only letters, digits, dashes, and underscores in +header file names, and at most one dot. + +Read more about using header files in official GCC documentation: + +* Include Syntax +* Include Operation +* Once-Only Headers +* Computed Includes + +https://gcc.gnu.org/onlinedocs/cpp/Header-Files.html diff --git a/labs/tiny-ml/audio-classifier/code/audio-classifier/platformio.ini b/labs/tiny-ml/audio-classifier/code/audio-classifier/platformio.ini new file mode 100644 index 0000000..1b1b1f5 --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/audio-classifier/platformio.ini @@ -0,0 +1,18 @@ +; PlatformIO Project Configuration File +; +; Build options: build flags, source filter +; Upload options: custom upload port, speed and extra flags +; Library options: dependencies, extra library storages +; Advanced options: extra scripting +; +; Please visit documentation for the other options and examples +; https://docs.platformio.org/page/projectconf.html + +[env:nano33ble] +platform = nordicnrf52 +board = nano33ble +framework = arduino +build_flags = + -L. + -l arm_cortexM4l_math + -w diff --git a/labs/tiny-ml/audio-classifier/code/audio-classifier/src/classifier.h b/labs/tiny-ml/audio-classifier/code/audio-classifier/src/classifier.h new file mode 100644 index 0000000..d26f225 --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/audio-classifier/src/classifier.h @@ -0,0 +1,4 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +// Copy the file created by the train_classifier script over this one \ No newline at end of file diff --git a/labs/tiny-ml/audio-classifier/code/audio-classifier/src/main.cpp b/labs/tiny-ml/audio-classifier/code/audio-classifier/src/main.cpp new file mode 100644 index 0000000..33ad893 --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/audio-classifier/src/main.cpp @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/** + * Audio Capture + * + * This program listens on the microphone capturing audio data, classifying it and outputting the + * classification label to the Serial port. + * + * Audio data is captured as a full 512 byte data buffer (1/64 second of audio data). + * Each buffer is then converted into a root mean square value of the buffer to create a smaller + * representation of the audio. + * + * If this root mean square value is over a threshold (i.e. not silence), then the next SAMPLE_SIZE + * buffers are captured and the RMS values calculated. Once a full set of samples is retrieved, a + * callback function is invoked, passing this set of samples. + * + * The default for SAMPLE_SIZE is 128 - 2 seconds of audio data. You can configure this in the + * sample_capture.h header file. + * + * This essentially converts audio data to a smaller representation for use with training TinyML models. + * For example - 2 seconds of audio * becomes 128 4-byte float values (512 bytes) instead of + * 32,768 2-byte integerss (65,536 bytes). + * + * Once a sample is colected, it is classified. + * + */ + +#include "sample_capture.h" +#include "classifier.h" + +// The classifier +Eloquent::ML::Port::SVM clf; + +// A helper class that captures audio from the microphone +SampleCapture sampleCapture; + +// A buffer used to store data read from the Sample Capture class +float _samples[SAMPLE_SIZE]; + +// Tracks if we have samples ready to log to the serial port +bool _ready; + +/** + * @brief A callback that is called whenever the sample capture object has a full buffer of audio + * RMS values ready for processing + */ +void onSamples(float *samples) +{ + memcpy(_samples, samples, SAMPLE_SIZE * sizeof(float)); + _ready = true; +} + +/** + * @brief Classify the samples, writing the label to the serial port + */ +void procesSamples() +{ + // Write out the classification to the serial port + Serial.print("Label: "); + Serial.println(clf.predictLabel(_samples)); +} + +/** + * @brief Sets up the serial port and the sample capture object + */ +void setup() +{ + // Start the serial connection so the captured audio data can be output + Serial.begin(115200); + + // Start the sample capture object to listen for audio and callback when + // it has a full set of samples + sampleCapture.init(onSamples); + + // Wait 3 seconds for everything to get started + delay(3000); +} + +/** + * @brief Runs continuously capturing audio data and writing it to + * the serial port + */ +void loop() +{ + // check to see if we have audio data + if (_ready) + { + // If we do, mark it as read ready for the next loop + _ready = false; + + // Process the samples + procesSamples(); + } + + // Sleep to allow background microphone processing + // Each sample is 2 seconds, so sleeping for 1 second is fine + delay(1000); +} diff --git a/labs/tiny-ml/audio-classifier/code/audio-classifier/src/sample_capture.h b/labs/tiny-ml/audio-classifier/code/audio-classifier/src/sample_capture.h new file mode 100644 index 0000000..423e8db --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/audio-classifier/src/sample_capture.h @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +#ifndef __SAMPLE_CAPTURE_H__ +#define __SAMPLE_CAPTURE_H__ + +#include +#include + +// The size of the data coming in from the microphone +#define BUFFER_SIZE 512U + +// 128 samples is enough for 2 seconds of audio - it's captured at 64 samples per second +#define SAMPLE_SIZE 128 +#define GAIN (1.0f / 50) +#define SOUND_THRESHOLD 1000 + +/** + * @brief The function signature for the callback function that is called when we have a full set of samples. + */ +typedef void (*samples_ready_callback)(float*); + +/** + * @brief A helper class for accessing the BLEs microphone. + * + * Audio data is captured as a full 512 byte data buffer (1/64 second of audio data). + * Each buffer is then converted into a root mean square value of the buffer to create a smaller + * representation of the audio. + * + * If this root mean square value is over a threshold (i.e. not silence), then the next SAMPLE_SIZE + * buffers are captured and the RMS values calculated. Once a full set of samples is retrieved, a + * callback function is invoked, passing this set of samples. + * + * This essentially converts audio data to a smaller representation for use with training and inference + * with TinyML models. + * For example - 2 seconds of audio * becomes 128 4-byte float values (512 bytes) instead of + * 32,768 2-byte integerss (65,536 bytes). + */ +class SampleCapture +{ +public: + /** + * @brief Setup the PDM library to access the microphone at a sample rate of 16KHz + * @param callback A callback function to call when there is a full set of samples ready. + * This callback will take the samples as a parameter as a SAMPLE_SIZE array of floats. + * This array is owned by this class, so the callback will need to take a copy. + */ + void init(samples_ready_callback callback) + { + // Set up the audio callback + PDMHelper::setSampleCapture(this); + + // Set up the callback when a set of samples is ready + _callback = callback; + + // Start listening on the microphone at a sample rate of 16KHz + PDM.begin(1, 16000); + PDM.setGain(20); + } + +private: + /** + * @brief A helper wrapper class that can connect a method on the SampleCapture object + * to the PDM callback that expects a static method + */ + class PDMHelper + { + public: + /** + * @brief Sets up the PDM callback to a method on the SampleCapture class + */ + static void setSampleCapture(SampleCapture *sampleCapture) + { + // Store the sample capture + _sampleCapture = sampleCapture; + + // Set up the callback + PDM.onReceive(PDMHelper::onReceive); + } + + private: + /** + * @brief The callback from the PDM class, calls the SampleCapture update method + */ + static void onReceive() + { + _sampleCapture->onReceive(); + } + + inline static SampleCapture *_sampleCapture; + }; + + /** + * @brief Reads the audio data from the PDM buffer and calculates the + * root mean square value, adding it to the samples if needed. + */ + void onReceive() + { + // Check we have a full buffers worth + if (PDM.available() == BUFFER_SIZE) + { + // Read from the buffer + PDM.read(_buffer, BUFFER_SIZE); + + // Calculate the root mean square value of the buffer + int16_t rms; + arm_rms_q15((q15_t *)_buffer, BUFFER_SIZE/sizeof(int16_t), (q15_t *)&rms); + + // If we are not currently collecting samples, check if the RMS value is + // above our threshold - as in we've heard something, not just silence. + // If we hear something, start collecting samples + if (!_started) + { + if (rms > SOUND_THRESHOLD) + { + _started = true; + _position = 0; + } + } + + // If were collecting data, either because we already were, or because we've + // just detected audio that's not slience, add it to the next slot in the samples + // array. + if (_started) + { + // Add the RMS value to the samples array in the next slot, multiplied by a + // gain value to give the signal a boost + _samples[_position] = rms * GAIN; + + // Move to the next slot in the samples + _position++; + + // If we're filled the samples buffer, stop collecting data and call the callback + if (_position >= SAMPLE_SIZE) + { + _started = false; + _position = 0; + + // Pass the samples to the callback + _callback(_samples); + } + } + } + } + + // The buffer to read from the PDM into - use this as a field to reduce overhead + // creating and deleteing it every time the buffer is full + int16_t _buffer[BUFFER_SIZE/sizeof(int16_t)]; + + // The samples buffer- use this as a field to reduce overhead + // creating and deleteing it every time the buffer is full + float _samples[SAMPLE_SIZE]; + + // Are we currently capturing data - we capture from when audio above the SOUND_THRESHOLD + // is detected, for SAMPLE_SIZE samples + bool _started; + + // The current position in the samples array to write the next sample to + int _position; + + // The callback to call when we have a full set of samples + samples_ready_callback _callback; +}; + +#endif __SAMPLE_CAPTURE_H__ \ No newline at end of file diff --git a/labs/tiny-ml/audio-classifier/code/audio-classifier/test/README b/labs/tiny-ml/audio-classifier/code/audio-classifier/test/README new file mode 100644 index 0000000..b94d089 --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/audio-classifier/test/README @@ -0,0 +1,11 @@ + +This directory is intended for PlatformIO Unit Testing and project tests. + +Unit Testing is a software testing method by which individual units of +source code, sets of one or more MCU program modules together with associated +control data, usage procedures, and operating procedures, are tested to +determine whether they are fit for use. Unit testing finds problems early +in the development cycle. + +More information about PlatformIO Unit Testing: +- https://docs.platformio.org/page/plus/unit-testing.html diff --git a/labs/tiny-ml/audio-classifier/code/model-trainer/.gitignore b/labs/tiny-ml/audio-classifier/code/model-trainer/.gitignore new file mode 100644 index 0000000..ffc839f --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/model-trainer/.gitignore @@ -0,0 +1,121 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# Junit test results (used by VSTS) +junit + +# Coverage results +coverage/ + +# Pycharm +.idea/ + +# vim +*.swp + +# Certificates +*.pem +demoCA/ +*.rnd \ No newline at end of file diff --git a/labs/tiny-ml/audio-classifier/code/model-trainer/.vscode/extensions.json b/labs/tiny-ml/audio-classifier/code/model-trainer/.vscode/extensions.json new file mode 100644 index 0000000..90e4b09 --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/model-trainer/.vscode/extensions.json @@ -0,0 +1,7 @@ +{ + // See http://go.microsoft.com/fwlink/?LinkId=827846 + // for the documentation about the extensions.json format + "recommendations": [ + "ms-python.vscode-pylance" + ] +} diff --git a/labs/tiny-ml/audio-classifier/code/model-trainer/README.md b/labs/tiny-ml/audio-classifier/code/model-trainer/README.md new file mode 100644 index 0000000..1576b7f --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/model-trainer/README.md @@ -0,0 +1,26 @@ +# Audio model trainer + +This Python program takes the output from the [audio capture program](../audio-capture) and uses it to train a TinyML model that can then be used in the the [audio classifier program](../audio-classifier). + +## Setting up the Python environment + +* Create a virtual environment for this folder using Python 3.8 or above +* Install the pip packages in the `requirements.txt` file + + > If you are using a new Apple Silicon based Mac, then the packages may not install as at the time of writing there is no supported Scikit-Learn package for the Mac. Instead you will need to use MiniForge. Refer to [this blog post](https://dev.to/jimbobbennett/installing-scikit-learn-on-an-apple-m1-114d) for instructions. +* Copy the CSV files created from the [audio capture program](../audio-capture) into the [`data`](./data) folder. These files should be named for the relevant label for the audio data. +* Run the `train_classifier.py` file + +The model will be trained, and the output sent to the console. + +```output +(.venv) ➜ model-trainer git:(master) ✗ python train_classifier.py +Accuracy 1.0 +Exported classifier to plain C +#pragma once +#include +namespace Eloquent { +} +``` + +Copy everything from `#pragma once` to the end of the output and paste it into the [`classifier.h`](../audio-classifier/src/classifier.h) header file in the audio classifier project. \ No newline at end of file diff --git a/labs/tiny-ml/audio-classifier/code/model-trainer/requirements.txt b/labs/tiny-ml/audio-classifier/code/model-trainer/requirements.txt new file mode 100644 index 0000000..cfc4179 --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/model-trainer/requirements.txt @@ -0,0 +1,2 @@ +scikit-learn +micromlgen \ No newline at end of file diff --git a/labs/tiny-ml/audio-classifier/code/model-trainer/train_classifier.py b/labs/tiny-ml/audio-classifier/code/model-trainer/train_classifier.py new file mode 100644 index 0000000..14334a1 --- /dev/null +++ b/labs/tiny-ml/audio-classifier/code/model-trainer/train_classifier.py @@ -0,0 +1,60 @@ +import numpy as np +import os +from micromlgen import port +from sklearn.svm import SVC +from sklearn.model_selection import train_test_split + +label_idx = 0 + +# A map of label indexes to labels +label_map = {} + +# Prepare a dataset to hold the CSV data +dataset = None + +# Prepare an array to hold the labels +dataset_labels = None + +# Make the random numbers predictable so the model is the same each time +np.random.seed(0) + +# Iterate through the files in the data folder loading the data +# This assumes the files are all valid csv files and the label +# is the file name without the .csv extensiojn +for file in os.listdir('data'): + # Get the file name to use as the label + file_name, ext = os.path.splitext(file) + label_map[label_idx] = file_name + + # Load the data from the CSV files and build a dataset by concatenating + # all the data into one big array + file_contents = np.loadtxt('data/' + file, delimiter=',') + dataset = file_contents if dataset is None else np.vstack((dataset, file_contents)) + + # Create an array of labels containing one row for each row in the main dataset + # with the value for each row the label index + labels = np.full((len(file_contents), 1), label_idx) + dataset_labels = labels if dataset_labels is None else np.vstack((dataset_labels, labels)) + + # Increment the label index for the next file + label_idx = label_idx + 1 + +# Split the data into a training and testing set to test the accuracy of the model +# If you are happy with the accuracy of the model, you can remove this split +dataset_train, dataset_test, label_train, label_test = train_test_split(dataset, dataset_labels.ravel(), test_size=0.2) + +# Build the support vector classification for our data and train the model +svc = SVC(kernel='poly', degree=2, gamma=0.1, C=100) +svc.fit(dataset_train, label_train) + +# Test the accuracy of the model +print('Accuracy:', svc.score(dataset_test, label_test)) +print() + +# Convert the model to C code and write to the classifier.h file +c_code = port(svc, classmap=label_map) +with open('classifier.h', 'w') as f: + f.write(c_code) + f.close() + +print('Classifier written to classifier.h.') diff --git a/labs/tiny-ml/audio-classifier/images/vscode-platformio-new-capture-project.png b/labs/tiny-ml/audio-classifier/images/vscode-platformio-new-capture-project.png new file mode 100644 index 0000000..7e77107 Binary files /dev/null and b/labs/tiny-ml/audio-classifier/images/vscode-platformio-new-capture-project.png differ diff --git a/labs/tiny-ml/audio-classifier/steps/audio-capture.md b/labs/tiny-ml/audio-classifier/steps/audio-capture.md new file mode 100644 index 0000000..d5813ae --- /dev/null +++ b/labs/tiny-ml/audio-classifier/steps/audio-capture.md @@ -0,0 +1,485 @@ +# Create the audio capture tool + +In this step you will program the Arduino device to capture audio data ready to classify to train the model. + +## Arduino Nano 33 Sense + +The Arduino Nano 33 Sense is an Arduino board with a number of sensors, including a microphone which makes it ideal for audio classification. This board is programmed in C++, using Platform IO. + +## Platform IO + +[Visual Studio Code](http://code.visualstudio.com?WT.mc_id=academic-7372-jabenn) (VS Code) is a free developer text editor that can be expanded with extensions to support multiple features or programming languages, including C++ to program microcontrollers, and Python to train ML models. + +[Platform IO](https://platformio.org) is an open source extension for VS Code that gives you the ability to program embedded devices without any additional dependencies. + +### Install Platform IO + +To install Platform IO: + +* Make sure you have VS Code installed. If not, you can install it from the [Visual Studio Code Downloads page](https://code.visualstudio.com/Download?WT.mc_id=academic-7372-jabenn) +* From VS Code, select the **Extensions** tab from the side menu + + ![The VS Code extensions menu](../../../images/vscode-extensions-menu.png) + +1. Search for `PlatformIO` and select the **PlatformIO IDE**. Select the **Install** button to install this extension. + + ![The install button for the Platform IO extension](../../../images/vscode-extensions-platformio-install-button.png) + +> PlatformIO is a self-contained embedded development environment. If you have the Arduino extension installed, it can clash with PlatformIO, so make sure to uninstall the Arduino extension before using PlatformIO. + +## Create the project + +Once you have PlatformIO installed, it can be used to write the audio capture tool to gather data for classification. + +### Create the PlatformIO project + +1. From VS Code, select the **PlatformIO** tab from the side menu + + ![The VS Code Platform IO menu](../../../images/vscode-platformio-menu.png) + +1. The *PlatformIO: Quick Access* bar will appear. Select *PIO Home->Open* + + ![The platform IO home open option](../../../images/vscode-platformio-home-open.png) + +1. From the home screen, select **+ New Project** from the *Quick Access* section + + ![The new platform button](../../../images/vscode-platformio-new-project-button.png) + +1. Name the project `audio-capture` + +1. For the *Board*, type in `Arduinno Nano 33 BLE` and select the `Arduinno Nano 33 BLE` board + +1. Set the *Framework* to **Arduino** + +1. For the location, either leave the default option checked, or uncheck it and select a location for the project. If you hover over the question mark you can see the default project location. + +1. Select the **Finish** button + + ![The completed project options](../images/vscode-platformio-new-capture-project.png) + +PlatformIO will take some time to download the components it needs to use the board, and create the project. Once done, the project will be opened in the VS Code window. + +### Navigate the project + +The PlatformIO project contains a number of files and folders: + +* `.pio` - this folder contains files used by PlatformIO such as the output of the build and other temporary files. You should never need to read or edit any files in this folder. This folder would not normally be checked in to source code control. +* `.vscode` - this folder contains configuration information used by VS Code and may be specific to your PC or Mac, such as paths to library files. You should never need to read or edit any files in this folder. +* `include` - this folder is for any header files for your project that you would want to expose to anyone using your code. This is normally when building libraries for consumption by other developers, and won't be used when writing code for a board. +* `lib` - this folder is for any additional libraries you want to use in your project. PlatformIO has a library manager than can import third party libraries, but if you want to use an additional libraries you need to add them to this folder. +* `src` - this folder is where the source code for your app lives. It will have a single file in it `main.cpp`. This file contains the code for an empty Arduino project. +* `test` - this folder is where you would put any unit test you write against your code. +* `.gitignore` - this is a [`gitignore`](https://git-scm.com/docs/gitignore) file that defines what will be ignored if you add this project to Git source code control. +* `platform.ini` - this file defines the configuration for the PlatformIO project including the board types (PlatformIO projects can target multiple boards), any additional libraries to use, and any build flags to control the code compilation. + +This project uses the [Arduino framework](https://www.arduino.cc), an open source framework for software and hardware. When writing arduino code, the core flow is around two methods that are defined in your main source code file. In a PlatformIO project, this file is called `main.cpp`. + +* `setup()` - the setup method is called once and only once when the board starts up. This function is used to do one-time setup, such as configuring the microphone or seial port connection. +* `loop()` - the loop method is called after setup, and once the method completes it is called again and again. All the time the board is running, this method will be called as soon as it completes. + +![The flow of an Arduino app](../../../images/arduino-setup-loop-flow.png) + +## Write the code + +The code needs to listen to the microphone and capture data, saving it out in a way that can be used to train a classifier. + +Traditionally, audio data is sampled at a rate of many of thousands of values a second - for example low quality audio is sampled at 16KHz, or 16 thousand values every second. This would give a huge amount of inputs to our model, meaning we'd need a large model - something TinyML is trying to avoid. Instead the audio can be averaged out into smaller samples, only a few a second. This gives a less accurate representation of the audio data, but is good enough to distinguish between different sounds. This is done by calculating the root mean square (RMS) value of the sampled input for a period of time. + +You can find all the code for this step in the [code/audio-capture](../code/audio-capture) folder. + +### Import a library + +The code to calculate the RMS value is at the time of writing not shipped with the code to support this Arduino board in PlatformIO. You can get round this by downloading a static library with the implementation that can be linked into your code. + +1. Head to the [ARM CMSIS GitHub repo compiled static libraries page](https://github.com/ARM-software/CMSIS_5/tree/5.7.0/CMSIS/DSP/Lib/GCC) + +1. Download the `libarm_cortexM4l_math.a` file and add it to the root folder of the project. + > There are a number of libraries with similar names targetting different M chips, so make sure to download the correct one +1. Open the `platformio.ini` file from the root of the project in VS Code + +1. Add the following to the end of the `[env:nano33ble]` section to tell the linker to look for libraries in the root and link this file when building the project: + + ```sh + build_flags = + -L. + -l arm_cortexM4l_math + -w + ``` + + The `-L.` flag tells the linker to look for libraries in the root folder. + + The `-l arm_cortexM4l_math` flag tells the linker to link the `libarm_cortexM4l_math.a` library + + The `-w` flag ignores warnings that come from the Arduino libraries. Although it is generally good practice to find and fix all warnings, in this case the Arduino libraries have warnings that can make it harder to catch errors in your code, so it is easier to ignore them + +1. Save the file + + > VS Code has an auto save option if you don't want to have to keep saving files. Enable this by selecting *File -> Auto save*. + +### Add code to read audio samples from the microphone + +To keep the code cleaner, the code to read from the microphone can be added to a separate file, and this file re-used later when building the audio classifier. + +1. In VS Code, create a new file in the `src` folder called `sample_capture.h`. This file will contain code to read the data from the microphone using a standard Arduino library, and calculate the RMS value of periods of audio. + +1. Add the following code to this file: + + ```cpp + #ifndef __SAMPLE_CAPTURE_H__ + #define __SAMPLE_CAPTURE_H__ + + #include + #include + + // The size of the data coming in from the microphone + #define BUFFER_SIZE 512U + + // 128 samples is enough for 2 seconds of audio - it's captured at 64 samples per second + #define SAMPLE_SIZE 128 + #define GAIN (1.0f / 50) + #define SOUND_THRESHOLD 1000 + + /** + * @brief The function signature for the callback function that is called when we have a full set of samples. + */ + typedef void (*samples_ready_callback)(float*); + + /** + * @brief A helper class for accessing the BLEs microphone. + * + * Audio data is captured as a full 512 byte data buffer (1/64 second of audio data). + * Each buffer is then converted into a root mean square value of the buffer to create a smaller + * representation of the audio. + * + * If this root mean square value is over a threshold (i.e. not silence), then the next SAMPLE_SIZE + * buffers are captured and the RMS values calculated. Once a full set of samples is retrieved, a + * callback function is invoked, passing this set of samples. + * + * This essentially converts audio data to a smaller representation for use with training and inference + * with TinyML models. + * For example - 2 seconds of audio * becomes 128 4-byte float values (512 bytes) instead of + * 32,768 2-byte integerss (65,536 bytes). + */ + class SampleCapture + { + public: + /** + * @brief Setup the PDM library to access the microphone at a sample rate of 16KHz + * @param callback A callback function to call when there is a full set of samples ready. + * This callback will take the samples as a parameter as a SAMPLE_SIZE array of floats. + * This array is owned by this class, so the callback will need to take a copy. + */ + void init(samples_ready_callback callback) + { + // Set up the audio callback + PDMHelper::setSampleCapture(this); + + // Set up the callback when a set of samples is ready + _callback = callback; + + // Start listening on the microphone at a sample rate of 16KHz + PDM.begin(1, 16000); + PDM.setGain(20); + } + + private: + /** + * @brief A helper wrapper class that can connect a method on the SampleCapture object + * to the PDM callback that expects a static method + */ + class PDMHelper + { + public: + /** + * @brief Sets up the PDM callback to a method on the SampleCapture class + */ + static void setSampleCapture(SampleCapture *sampleCapture) + { + // Store the sample capture + _sampleCapture = sampleCapture; + + // Set up the callback + PDM.onReceive(PDMHelper::onReceive); + } + + private: + /** + * @brief The callback from the PDM class, calls the SampleCapture update method + */ + static void onReceive() + { + _sampleCapture->onReceive(); + } + + inline static SampleCapture *_sampleCapture; + }; + + /** + * @brief Reads the audio data from the PDM buffer and calculates the + * root mean square value, adding it to the samples if needed. + */ + void onReceive() + { + // Check we have a full buffers worth + if (PDM.available() == BUFFER_SIZE) + { + // Read from the buffer + PDM.read(_buffer, BUFFER_SIZE); + + // Calculate the root mean square value of the buffer + int16_t rms; + arm_rms_q15((q15_t *)_buffer, BUFFER_SIZE/sizeof(int16_t), (q15_t *)&rms); + + // If we are not currently collecting samples, check if the RMS value is + // above our threshold - as in we've heard something, not just silence. + // If we hear something, start collecting samples + if (!_started) + { + if (rms > SOUND_THRESHOLD) + { + _started = true; + _position = 0; + } + } + + // If were collecting data, either because we already were, or because we've + // just detected audio that's not slience, add it to the next slot in the samples + // array. + if (_started) + { + // Add the RMS value to the samples array in the next slot, multiplied by a + // gain value to give the signal a boost + _samples[_position] = rms * GAIN; + + // Move to the next slot in the samples + _position++; + + // If we're filled the samples buffer, stop collecting data and call the callback + if (_position >= SAMPLE_SIZE) + { + _started = false; + _position = 0; + + // Pass the samples to the callback + _callback(_samples); + } + } + } + } + + // The buffer to read from the PDM into - use this as a field to reduce overhead + // creating and deleteing it every time the buffer is full + int16_t _buffer[BUFFER_SIZE/sizeof(int16_t)]; + + // The samples buffer- use this as a field to reduce overhead + // creating and deleteing it every time the buffer is full + float _samples[SAMPLE_SIZE]; + + // Are we currently capturing data - we capture from when audio above the SOUND_THRESHOLD + // is detected, for SAMPLE_SIZE samples + bool _started; + + // The current position in the samples array to write the next sample to + int _position; + + // The callback to call when we have a full set of samples + samples_ready_callback _callback; + }; + + #endif __SAMPLE_CAPTURE_H__ + ``` + + This code defines a class called `SampleCapture` that uses the [Arduino `PDM` library](https://www.arduino.cc/en/Reference/PDM) to get data from the microphone at a sample rate of 16KHz, then uses the `arm_math` library to calculate the RMS of the incoming data from the microphone buffer. + + The PDM buffer is read in blocks of 256 2-byte values, with each full buffer converted to a single RMS value. If this RMS value is above a threshold - meaning audio has been detected, then recording begins, storing 128 RMS values to represent 2 seconds of data. Once this 2 seconds of data has been captured, a callback is made to pass the data to the rest of the code. This way there's no output diring periods of slience, only when audio is detected. + + You can read the code and the comments to understand how each call works. If you need to change the length of data sampled, change the value of `SAMPLE_SIZE` to the number of seconds needed multiplied by 64. For example, if you only need 1 second of data, set `SAMPLE_SIZE` to be 64. To keep this model small, set this to be as small as possible. + +1. Save the file + +### Add the program code + +The main program now needs to be written to read data from the microphone at regular intervals, then output the 2 seconds of RMS values to the serial port. The data that is captured needs to be stored somewhere so that it can be used to train the model, and the easiest way to do this is to output it from the Arduino device to the serial port, then monitor that serial port using PlatformIO. + +1. Open the `main.cpp` file + +1. Replace the existing code in this file with the following: + + ```cpp + // Copyright (c) Microsoft Corporation. + // Licensed under the MIT license. + + /** + * Audio Capture + * + * This program listens on the microphone capturing audio data and writing it out as + * CSV file lines to the serial port. + * + * Audio data is captured as a full 512 byte data buffer (1/64 second of audio data). + * Each buffer is then converted into a root mean square value of the buffer to create a smaller + * representation of the audio. + * + * If this root mean square value is over a threshold (i.e. not silence), then the next SAMPLE_SIZE + * buffers are captured and the RMS values calculated. Once a full set of samples is retrieved, a + * callback function is invoked, passing this set of samples. + * + * The default for SAMPLE_SIZE is 128 - 2 seconds of audio data. You can configure this in the + * sample_capture.h header file. + * + * This essentially converts audio data to a smaller representation for use with training TinyML models. + * For example - 2 seconds of audio * becomes 128 4-byte float values (512 bytes) instead of + * 32,768 2-byte integerss (65,536 bytes). + * + * The output sent to the serial port can be saved as a CSV file and used to train the model + * + */ + + #include "sample_capture.h" + + // A helper class that captures audio from the microphone + SampleCapture sampleCapture; + + // A buffer used to store data read from the Sample Capture class + float _samples[SAMPLE_SIZE]; + + // Tracks if we have samples ready to log to the serial port + bool _ready; + + /** + * @brief A callback that is called whenever the sample capture object has a full buffer of audio + * RMS values ready for processing + */ + void onSamples(float *samples) + { + memcpy(_samples, samples, SAMPLE_SIZE * sizeof(float)); + _ready = true; + } + + /** + * @brief Process the samples, writing them out to the serial port + */ + void procesSamples() + { + // print the audio data to serial port + for (int i = 0; i < SAMPLE_SIZE; i++) + { + Serial.print(_samples[i], 6); + + // Seperate the audio values with commas, at the last value + // send a newline + Serial.print(i == SAMPLE_SIZE - 1 ? '\n' : ','); + } + } + + /** + * @brief Sets up the serial port and the sample capture object + */ + void setup() + { + // Start the serial connection so the captured audio data can be output + Serial.begin(115200); + + // Start the sample capture object to listen for audio and callback when + // it has a full set of samples + sampleCapture.init(onSamples); + + // Wait 3 seconds for everything to get started + delay(3000); + } + + /** + * @brief Runs continuously capturing audio data and writing it to + * the serial port + */ + void loop() + { + // check to see if we have audio data + if (_ready) + { + // If we do, mark it as read ready for the next loop + _ready = false; + + // Process the samples + procesSamples(); + } + + // Sleep to allow background microphone processing + // Each sample is 2 seconds, so sleeping for 1 second is fine + delay(1000); + } + ``` + + This code sets up the sample capture object in the `setup` function, registering a callback that is called with the 128 samples every time audio is detected. When the callback is called, a copy of the data is made, and a flag is set to indicate data is ready. This is then detected in the `loop` function, and written to the console in a call to `procesSamples`. + +1. Save the file + +## Run the code + +Now the code is written, it can be deployed to the Arduino device and run to capture data. + +### Upload the code to the device + + 1. Connect your Arduino board to your PC or Mac using a USB cable + +1. From Visual Studio Code, launch the Command Palette. This is a pop-up menu that allows you to run actions from VS Code as well as any extensions installed. + + 1. If you are using Windows or Linux, press `Ctrl+Shift+p` + 1. If you are using macOS, press `Command+Shift+p` + +1. Search for `PlatformIO: Upload` by typing in the box, then select that option + + ![The upload option](../../../images/vscode-command-palette-platformio-upload.png) + +The relevant tooling will be installed, and the code will be compiled for the device. Once built, it will be uploaded to the Arduino. You will see the progress in the VS Code terminal. + +```output +Linking .pio/build/nano33ble/firmware.elf +Checking size .pio/build/nano33ble/firmware.elf +Building .pio/build/nano33ble/firmware.bin +Advanced Memory Usage is available via "PlatformIO Home > Project Inspect" +RAM: [== ] 16.8% (used 44080 bytes from 262144 bytes) +Flash: [= ] 8.2% (used 80236 bytes from 983040 bytes) +================================================ [SUCCESS] Took 12.28 seconds ================================================ +``` + +### Monitor the output + +The serial output of the Arduino is the USB cable that is used to connect it to your PC or Mac. You can monitor the data being sent using the PlatformIO serial monitor. + +1. Launch the serial monitor using one of these methods: + * Open the VS Code Command Palette, search for `PlatformIO: Serial Monitor` and select this option + + ![The serial monitor command palette option](../../../images/vscode-command-palette-platformio-serial-monitor.png) + + * Select the Serial Monitor button from the status bar + + ![The serial monitor button](../../../images/vscode-status-bar-platformio-serial-monitor-button.png) + +The serial monitor will listen for all messages from the device. You'll see nothing in the output to start with. + +### Capture audio data + +To train the model, you will need multiple audio samples for each label you want to classify - for example if you want to detect the difference between "Yes" and "No", you will need multiple samples for the word "Yes", and multiple for the word "No". Ideally you'd want between 15-30 samples per label. + +1. Make the relevant sound into the microphone on the Arduino device, either by speaking in to it, or playing the relevant audio. You will see a line of samples appear on the serial monitor output. + +1. Repeat this multiple times, slightly adjusting the position of the microphone relative to the audio. You want 15-30 samples. Each new sample will be a new line. + + ```output + > Executing task: pio device monitor < + + --- Available filters and text transformations: colorize, debug, default, direct, hexlify, log2file, nocontrol, printable, send_on_enter, time + --- More details at http://bit.ly/pio-monitor-filters + --- Miniterm on /dev/cu.usbmodem101 9600,8,N,1 --- + --- Quit: Ctrl+C | Menu: Ctrl+T | Help: Ctrl+T followed by Ctrl+H --- + 25.599998,42.820000,13.540000,31.559999,43.279999,45.919998,46.059998,44.919998,40.459999,29.840000,30.500000,27.799999,23.459999,19.139999,16.980000,9.559999,8.860000,6.260000,0.000000,3.620000,0.000000,3.620000,8.080000,11.440000,12.540000,15.339999,14.020000,14.020000,12.000000,10.860000,8.860000,6.260000,3.620000,0.000000,0.000000,0.000000,3.620000,3.620000,6.260000,8.860000,9.559999,10.860000,11.440000,12.540000,13.040000,13.540000,14.020000,14.920000,15.780000,16.180000,16.580000,16.580000,16.980000,16.980000,17.359999,16.980000,16.580000,16.180000,15.780000,14.920000,14.480000,13.540000,13.040000,12.540000,11.440000,10.860000,10.219999,9.559999,8.860000,8.080000,8.080000,7.240000,7.240000,6.260000,6.260000,5.100000,5.100000,5.100000,5.100000,3.620000,3.620000,3.620000,3.620000,3.620000,3.620000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000 + 38.299999,34.139999,31.139999,27.799999,19.480000,7.240000,7.240000,8.860000,17.719999,20.779999,23.740000,21.400000,21.100000,20.779999,18.459999,15.339999,10.860000,5.100000,3.620000,0.000000,0.000000,0.000000,0.000000,0.000000,3.620000,3.620000,5.100000,5.100000,6.260000,8.080000,9.559999,11.440000,12.540000,14.020000,14.480000,15.339999,15.780000,16.180000,16.980000,17.359999,17.719999,18.100000,17.719999,17.719999,16.980000,16.580000,15.780000,15.339999,14.480000,13.540000,12.540000,12.000000,10.860000,10.219999,8.860000,8.860000,8.080000,7.240000,7.240000,6.260000,5.100000,5.100000,5.100000,3.620000,3.620000,3.620000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000 + ``` + +1. Copy all the data from the serial monitor into a text file. Save this as `