Merge pull request #64 from jimbobbennett/main

Adding TinyML lab

.gitignore

@@ -140,4 +140,5 @@ obj
 # Node stuff
 node_modules
 
-~$*.docx
+~$*.docx
+*.a

labs/README.md

@@ -22,6 +22,12 @@ These labs cover running AI workloads either in the cloud from the IoT device, o
 * [Assembly line QA](./ai-edge/vision/manufacturing-part-check/) - A prototype of an AI image classification based quality assurance tool for manufacturing showing how to detect broken parts on an assembly line using AI, controlled from another device
 * [Speech](./ai-edge/speech) - Speech to text, text to speech and speech translation using a Raspberry Pi and USB microphone/speaker
 
+## TinyML
+
+These labs cover training tiny machine learning workloads and running them on embedded hardware such as Adruino microcontrollers.
+
+* [Audio classifier](./tiny-ml/audio-classifier/) - a TinyML lab to capture audio data, use it to train a model, then classify audio data using that model on an Arduino Nano 33 Sense BLE board.
+
 ## Digital Agriculture
 
 These labs cover scenarios around digital agriculture.

labs/ai-edge/vision/manufacturing-part-check/code/analytics/esp-eye/src/ImageHandler.cpp

@@ -64,7 +64,7 @@ bool ImageHandler::SavePhoto(camera_fb_t *frameBuffer, const char *fileName)
 // Checks if the photo has been saved by validating the file size
 bool ImageHandler::CheckPhoto(const char *fileName)
 {
-    // Open the file from teh SPIFFS file system
+    // Open the file from the SPIFFS file system
     File f_pic = SPIFFS.open(fileName);
 
     // Get the file size

labs/ai-edge/vision/manufacturing-part-check/code/custom-vision/src/ImageHandler.cpp

@@ -64,7 +64,7 @@ bool ImageHandler::SavePhoto(camera_fb_t *frameBuffer, const char *fileName)
 // Checks if the photo has been saved by validating the file size
 bool ImageHandler::CheckPhoto(const char *fileName)
 {
-    // Open the file from teh SPIFFS file system
+    // Open the file from the SPIFFS file system
     File f_pic = SPIFFS.open(fileName);
 
     // Get the file size

labs/ai-edge/vision/manufacturing-part-check/code/image-capture/src/ImageHandler.cpp

@@ -64,7 +64,7 @@ bool ImageHandler::SavePhoto(camera_fb_t *frameBuffer, const char *fileName)
 // Checks if the photo has been saved by validating the file size
 bool ImageHandler::CheckPhoto(const char *fileName)
 {
-    // Open the file from teh SPIFFS file system
+    // Open the file from the SPIFFS file system
     File f_pic = SPIFFS.open(fileName);
 
     // Get the file size

labs/ai-edge/vision/manufacturing-part-check/code/pi-control/esp-eye/src/ImageHandler.cpp

@@ -64,7 +64,7 @@ bool ImageHandler::SavePhoto(camera_fb_t *frameBuffer, const char *fileName)
 // Checks if the photo has been saved by validating the file size
 bool ImageHandler::CheckPhoto(const char *fileName)
 {
-    // Open the file from teh SPIFFS file system
+    // Open the file from the SPIFFS file system
     File f_pic = SPIFFS.open(fileName);
 
     // Get the file size

labs/ai-edge/vision/manufacturing-part-check/steps/image-capture.md

@@ -77,7 +77,7 @@ To access the camera and host a web server, you can use some pre-existing Arduin
       ESP Async WebServer
     ```
 
-    This will add the [Espressif ESP32 Camera driver](https://github.com/espressif/esp32-camera) and the [ESP Async WebServer](https://github.com/me-no-dev/ESPAsyncWebServer) libraries to the project. The ESP32 camera driver is a library to talk to the camera on the ESP-EYE board, and ht eESP Async WebServer allows you to run a web server on the ESP board.
+    This will add the [Espressif ESP32 Camera driver](https://github.com/espressif/esp32-camera) and the [ESP Async WebServer](https://github.com/me-no-dev/ESPAsyncWebServer) libraries to the project. The ESP32 camera driver is a library to talk to the camera on the ESP-EYE board, and the ESP Async WebServer allows you to run a web server on the ESP board.
 
 Next time the app is compiled, it will bring in these libraries and compile them into the build.
 
@@ -362,7 +362,7 @@ Just like code for interacting with the camera, the code for interacting with th
     // Checks if the photo has been saved by validating the file size
     bool ImageHandler::CheckPhoto(const char *fileName)
     {
-        // Open the file from teh SPIFFS file system
+        // Open the file from the SPIFFS file system
         File f_pic = SPIFFS.open(fileName);
 
         // Get the file size

labs/tiny-ml/audio-classifier/README.md

@@ -0,0 +1,48 @@
+# TinyML Audio classifier
+
+This folder contains a lab with multiple parts working towards an audio classifier running on an Arduino Nano 33 BLE Sense microcontroller, taking advantage of the built-in microphone.
+
+> This lab was inspired by [Better word classification with Arduino Nano 33 BLE Sense and Machine Learning](https://eloquentarduino.github.io/2020/08/better-word-classification-with-arduino-33-ble-sense-and-machine-learning/) by [Eloquent Arduino](https://eloquentarduino.github.io/about-me/).
+
+| Author | [Jim Bennett](https://github.com/JimBobBennett) |
+|:---|:---|
+| Target platform   | <ul><li>Arduino Nano 33 BLE Sense</li></ul> |
+| Hardware required | <ul><li>Arduino Nano 33 BLE Sense</li><li>USB cable</li></ul> |
+| Software required | <ul><li>[Visual Studio Code](http://code.visualstudio.com?WT.mc_id=academic-7372-jabenn)</li><li>[PlatformIO](https://marketplace.visualstudio.com/items?itemName=platformio.platformio-ide&WT.mc_id=academic-7372-jabenn)</li></ul>|
+| Programming Language | <ul><li>C++</li><li>Python</li></ul> |
+| Prerequisites | Basic proficiency in using VS Code, C++ and Python.<br>If you want to learn Python, check out these free resources:<br><ul><li>[Python for beginners video series on Channel9](https://channel9.msdn.com/Series/Intro-to-Python-Development?WT.mc_id=academic-7372-jabenn)</li><li>[Take your first steps with Python learning path on Microsoft Learn](https://docs.microsoft.com/learn/paths/python-first-steps/?WT.mc_id=academic-7372-jabenn)</li></ul> |
+| Date | February 2021 |
+| Learning Objectives | <ul><li>Capture audio data suitable for TinyML using an Arduino device</li><li>Train a TinyML model using Python</li><li>Classify audio using TinyML on an Arduino device</li></ul> |
+| Time to complete | 1 hours |
+
+## The lab parts
+
+This lab has the following parts
+
+1. Program the Arduino device for audio capture and capture training data
+1. Train a ML model using the training data
+1. Program the Arduino device to classify audio
+
+## Audio classification
+
+Audo classification is the process of classifying a sound based on labelled training data. For example - you could train a model by using multiple recordings of someone saying the word "Yes" labelled as `Yes`, and multiple recordings of someone saying the word "No" labelled as `No`. The model could then take a new sound recording and classify it as either `Yes` or `No`.
+
+This lab starts by coding the Arduino to record multiple samples that are labelled, then these labelled samples are used to train a model, which is then add to device code that runs on the microcontroller to classify new audio data.
+
+The classifier you will create here needs at least 2 labels in the model, and will pick the most probable one to classify the audio.
+
+## TinyML
+
+TinyML is the coming together of machine learning and embedded systems. It involves training ML models that are tiny - that is substantially smaller than the models historically created, and able to run on microcontrollers with limited memory and power. It was originally defined as ML models that can run using less than 1mW of power, but has become a general term for running ML on microcontrollers.
+
+Microcontrollers have memory limits usually in the kilobytes, meaning traditional ML models that are many megabytes in size cannot even be installed on the device, let alone run. By using TinyML models you can bring the world of ML to a microcontroller. An audio classifier (a model that can distinguish between multiple sounds) or a wake word model (a model that can detect one specific sound, such as the command to wake a smart device up), for example, can be compressed to less than 20KB using TinyML.
+
+There is a trade off - with smaller models you lose accuracy, but this is an acceptable tradeoff for the advantages of smaller, low powered models. For example, if you were creating a voice activated smart device you would want it to consume as little power as possible waiting for a wake word, and only then powering up to listen for more instructions and using bigger models, or even sending data to the cloud. If you are using audio to classify animal behavior on a smart collar, you want long battery life and the device to be as small and light-weight as possible to reduce the inconvenience to the animal being tracked.
+
+In this lab, the device in question is an [Arduino Nano 33 BLE Sense microcontroller](https://store.arduino.cc/usa/nano-33-ble-sense) - a board that has a built in microphone that can be used to detect and classify audio signals. It has 256KB of memory available.
+
+## Labs
+
+1. [Create the audio capture tool](./steps/audio-capture.md)
+1. [Train the TinyML model](./steps/train-model.md)
+1. [Create the audio classifier](./steps/audio-classifier.md)

labs/tiny-ml/audio-classifier/code/audio-capture/.gitignore

@@ -0,0 +1,5 @@
+.pio
+.vscode/.browse.c_cpp.db*
+.vscode/c_cpp_properties.json
+.vscode/launch.json
+.vscode/ipch

labs/tiny-ml/audio-classifier/code/audio-capture/.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    // See http://go.microsoft.com/fwlink/?LinkId=827846
+    // for the documentation about the extensions.json format
+    "recommendations": [
+        "platformio.platformio-ide"
+    ]
+}

labs/tiny-ml/audio-classifier/code/audio-capture/README.md

@@ -0,0 +1,30 @@
+# Audio Capture
+
+This folder contains a [PlatformIO](https://platformio.org/platformio-ide) project to build an audio capture app that runs on an [Arduino Nano 33 BLE Sense](https://store.arduino.cc/usa/nano-33-ble-sense) board.
+
+You can read more about this project in the [top level README.md file](../README.md).
+
+To build and deploy this project, you will need to open this folder in [Visual Studio Code](https://code.visualstudio.com/?WT.mc_id=academic-7372-jabenn) with the [PlatformIO extension](https://platformio.org/platformio-ide) installed. You will then be able to build the project and upload to a Arduino Nano 33 BLE Sense board.
+
+## Building this project
+
+This project has a library dependency on the ARM CMSIS static library. You will need to download this from the ARM GitHub repo.
+
+* Head to [the CMSIS GCC libs folder in the GitHub repo](https://github.com/ARM-software/CMSIS_5/tree/5.7.0/CMSIS/DSP/Lib/GCC)
+* Download `libarm_cortexM4l_math.a` from that repo page
+* Add the file to the root folder of this project
+* Build and upload as normal
+
+## Running the project
+
+Once the project is running on the board, it will listen for audio and output RMS values to the serial port.
+
+* Connect to the serial monitor to view the audio values
+* Make the one of the relevant noises into the microphone of the board. Pause after each noise and you will see a line output to the serial monitor.
+* Repeat 15-30 times for that one noise
+* Copy the values from the serial monitor into a CSV file named using the name of the label (for eample if you were capturing numbers you would put the data for one into `one.csv`, for two into `two.csv` and so on).
+* Clear the serial output and repeat the above steps for all the noises you are interested in
+
+## Processing the output
+
+Refer to the [top level README.md file](../README.md) for instructions on how to process and use the output to classify noises.

labs/tiny-ml/audio-classifier/code/audio-capture/include/README

@@ -0,0 +1,39 @@
+
+This directory is intended for project header files.
+
+A header file is a file containing C declarations and macro definitions
+to be shared between several project source files. You request the use of a
+header file in your project source file (C, C++, etc) located in `src` folder
+by including it, with the C preprocessing directive `#include'.
+
+```src/main.c
+
+#include "header.h"
+
+int main (void)
+{
+ ...
+}
+```
+
+Including a header file produces the same results as copying the header file
+into each source file that needs it. Such copying would be time-consuming
+and error-prone. With a header file, the related declarations appear
+in only one place. If they need to be changed, they can be changed in one
+place, and programs that include the header file will automatically use the
+new version when next recompiled. The header file eliminates the labor of
+finding and changing all the copies as well as the risk that a failure to
+find one copy will result in inconsistencies within a program.
+
+In C, the usual convention is to give header files names that end with `.h'.
+It is most portable to use only letters, digits, dashes, and underscores in
+header file names, and at most one dot.
+
+Read more about using header files in official GCC documentation:
+
+* Include Syntax
+* Include Operation
+* Once-Only Headers
+* Computed Includes
+
+https://gcc.gnu.org/onlinedocs/cpp/Header-Files.html

labs/tiny-ml/audio-classifier/code/audio-capture/platformio.ini

@@ -0,0 +1,18 @@
+; PlatformIO Project Configuration File
+;
+;   Build options: build flags, source filter
+;   Upload options: custom upload port, speed and extra flags
+;   Library options: dependencies, extra library storages
+;   Advanced options: extra scripting
+;
+; Please visit documentation for the other options and examples
+; https://docs.platformio.org/page/projectconf.html
+
+[env:nano33ble]
+platform = nordicnrf52
+board = nano33ble
+framework = arduino
+build_flags = 
+    -L. 
+    -l arm_cortexM4l_math
+    -w

labs/tiny-ml/audio-classifier/code/audio-capture/src/main.cpp

@@ -0,0 +1,101 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+/**
+ * Audio Capture
+ * 
+ * This program listens on the microphone capturing audio data and writing it out as
+ * CSV file lines to the serial port.
+ * 
+ * Audio data is captured as a full 512 byte data buffer (1/64 second of audio data).
+ * Each buffer is then converted into a root mean square value of the buffer to create a smaller
+ * representation of the audio.
+ * 
+ * If this root mean square value is over a threshold (i.e. not silence), then the next SAMPLE_SIZE
+ * buffers are captured and the RMS values calculated. Once a full set of samples is retrieved, a
+ * callback function is invoked, passing this set of samples.
+ * 
+ * The default for SAMPLE_SIZE is 128 - 2 seconds of audio data. You can configure this in the 
+ * sample_capture.h header file.
+ * 
+ * This essentially converts audio data to a smaller representation for use with training TinyML models.
+ * For example - 2 seconds of audio * becomes 128 4-byte float values (512 bytes) instead of
+ * 32,768 2-byte integerss (65,536 bytes).
+ * 
+ * The output sent to the serial port can be saved as a CSV file and used to train the model
+ * 
+ */
+
+#include "sample_capture.h"
+
+// A helper class that captures audio from the microphone
+SampleCapture sampleCapture;
+
+// A buffer used to store data read from the Sample Capture class
+float _samples[SAMPLE_SIZE];
+
+// Tracks if we have samples ready to log to the serial port
+bool _ready;
+
+/**
+ * @brief A callback that is called whenever the sample capture object has a full buffer of audio
+ * RMS values ready for processing
+ */
+void onSamples(float *samples)
+{
+	memcpy(_samples, samples, SAMPLE_SIZE * sizeof(float));
+	_ready = true;
+}
+
+/**
+ * @brief Process the samples, writing them out to the serial port
+ */
+void procesSamples()
+{
+    // print the audio data to serial port
+    for (int i = 0; i < SAMPLE_SIZE; i++)
+    {
+        Serial.print(_samples[i], 6);
+
+        // Seperate the audio values with commas, at the last value
+        // send a newline
+        Serial.print(i == SAMPLE_SIZE - 1 ? '\n' : ',');
+    }
+}
+
+/**
+ * @brief Sets up the serial port and the sample capture object
+ */
+void setup()
+{
+    // Start the serial connection so the captured audio data can be output
+    Serial.begin(115200);
+
+    // Start the sample capture object to listen for audio and callback when
+	// it has a full set of samples
+    sampleCapture.init(onSamples);
+
+    // Wait 3 seconds for everything to get started
+    delay(3000);
+}
+
+/**
+ * @brief Runs continuously capturing audio data and writing it to
+ * the serial port
+ */
+void loop()
+{
+    // check to see if we have audio data
+    if (_ready)
+    {
+		// If we do, mark it as read ready for the next loop
+		_ready = false;
+
+        // Process the samples
+        procesSamples();
+    }
+
+    // Sleep to allow background microphone processing
+	// Each sample is 2 seconds, so sleeping for 1 second is fine
+    delay(1000);
+}