add discogs demo from private repo

mrutherford92 · Jun 14, 2022 · 961a44f · 961a44f
1 parent b95d6d5
commit 961a44f
Show file tree

Hide file tree

Showing 36 changed files with 1,936 additions and 0 deletions.
diff --git a/examples/demos/discogs-autotagging/Dockerfile b/examples/demos/discogs-autotagging/Dockerfile
@@ -0,0 +1,24 @@
+FROM node:14.18.1
+
+WORKDIR /usr/src/app
+
+# COPY . .
+
+RUN apt-get -y update
+RUN apt-get -y upgrade
+RUN apt-get install -y ffmpeg
+
+COPY ./server ./server
+WORKDIR /usr/src/app/server
+RUN npm ci --only=production
+ENV PORT=8000
+
+COPY ./views/dist/ ./public
+# WORKDIR /usr/src/app/views
+# RUN npm ci --only=production
+# RUN npm run build
+
+EXPOSE 8000
+
+# WORKDIR /usr/src/app/server
+CMD node server.js
diff --git a/examples/demos/discogs-autotagging/README.md b/examples/demos/discogs-autotagging/README.md
@@ -0,0 +1,14 @@
+# Music autotagging with Discogs Effnet
+Demo showing realtime music autotagging based on the Discogs taxonomy (400 subgenre tags). 
+
+## To start project locally (Dev version):
+- `cd server`
+- `npm run dev`
+- `cd ../views`
+- `npm run dev`
+
+## To build for deployment:
+- `cd views`
+- `npm run build`
+- `docker build -t discogs-demo:latest .`
+- `docker run --rm -it -p 8000:8000 discogs-demo:latest node server.js`
diff --git a/examples/demos/discogs-autotagging/server/package.json b/examples/demos/discogs-autotagging/server/package.json
@@ -0,0 +1,21 @@
+{
+  "name": "youtube-streaming-test",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "dev": "nodemon server.js",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Jorge Marcos Fernandez",
+  "license": "ISC",
+  "dependencies": {
+    "cors": "^2.8.5",
+    "ejs": "^3.1.6",
+    "express": "^4.17.1",
+    "youtube-audio-stream": "^0.3.32"
+  },
+  "devDependencies": {
+    "nodemon": "^2.0.15"
+  }
+}
diff --git a/examples/demos/discogs-autotagging/server/server.js b/examples/demos/discogs-autotagging/server/server.js
@@ -0,0 +1,31 @@
+const express = require('express');
+const path = require('path');
+const cors = require('cors');
+// const http = require('http');
+const process = require('process');
+const app = express();
+
+const stream = require('youtube-audio-stream');
+
+const corsOptions = {
+    origin: ['https://192.168.1.131:3000', 'https://localhost:3000', 'https://localhost:4173'],
+    optionsSuccessStatus: 200
+}
+
+app.get('/stream/:videoId', cors(corsOptions), async function (req, res) {
+    try {
+        res.set({'Content-Type': 'audio/mpeg'});
+        stream(req.params.videoId).pipe(res);
+    } catch (exception) {
+        res.status(500).send(exception)
+    }
+});
+
+// app.use(express.static(path.join(__dirname, '../views/dist')));
+app.use('/', express.static('public'));
+
+
+// const server = http.createServer(app);
+app.listen(process.env.PORT || 8000, '0.0.0.0', () => {
+    console.log(`server is running on port ${process.env.PORT || 8000}`)
+});
diff --git a/examples/demos/discogs-autotagging/views/README.md b/examples/demos/discogs-autotagging/views/README.md
@@ -0,0 +1,4 @@
+# Music autotagging with Discogs Effnet
+
+model currently in use:
+- 211115-202040_resnet18
diff --git a/examples/demos/discogs-autotagging/views/index.html b/examples/demos/discogs-autotagging/views/index.html
@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" href="/favicon.ico" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>DiscogsEffNet Music Classification Demo</title>
+  </head>
+  <body>
+    <div id="app"></div>
+    <script type="module" src="/src/main.js"></script>
+  </body>
+</html>
diff --git a/examples/demos/discogs-autotagging/views/package.json b/examples/demos/discogs-autotagging/views/package.json
@@ -0,0 +1,26 @@
+{
+  "name": "discogseffnet-demo",
+  "version": "0.0.0",
+  "scripts": {
+    "dev": "vite --host",
+    "build": "vite build && cp src/audio/model-tfjs/*.bin dist/assets",
+    "serve": "vite preview"
+  },
+  "dependencies": {
+    "@tensorflow/tfjs": "^3.11.0",
+    "d3": "^7.1.1",
+    "d3-cloud": "^1.2.5",
+    "essentia.js": "^0.1.3",
+    "mathjs": "^10.6.1",
+    "pico-emitter": "^1.1.0",
+    "seedrandom": "^3.0.5",
+    "tiny-emitter": "^2.1.0",
+    "uuid": "^8.3.2",
+    "vue": "^3.2.16"
+  },
+  "devDependencies": {
+    "@vitejs/plugin-vue": "^1.9.3",
+    "sass": "^1.50.1",
+    "vite": "^2.9.10"
+  }
+}
diff --git a/examples/demos/discogs-autotagging/views/public/favicon.ico b/examples/demos/discogs-autotagging/views/public/favicon.ico
diff --git a/examples/demos/discogs-autotagging/views/src/App.vue b/examples/demos/discogs-autotagging/views/src/App.vue
@@ -0,0 +1,49 @@
+<template>
+	<main>
+		<YTStream></YTStream>
+		<PredictionsDisplay></PredictionsDisplay>
+	</main>
+</template>
+
+<script>
+import YTStream from './components/YTStream.vue';
+import PredictionsDisplay from './components/PredictionsDisplay.vue';
+
+export default {
+    components: { YTStream, PredictionsDisplay }
+}
+</script>
+
+<style>
+@import url('https://fonts.googleapis.com/css2?family=Montserrat:ital,wght@0,300;0,400;0,500;0,600;1,300&display=swap');
+
+* {
+	box-sizing: border-box;
+	--color-bg: 25, 25, 25;
+	--border-radius: 5px;
+	--color-primary: 242, 242, 242;
+	--color-grey: 117, 117, 117;
+	--action-color: #19ee9c;
+	--btn-shadow: 0 2px 4px black;
+	font-family: 'Montserrat', sans-serif;
+}
+
+#app {
+	-webkit-font-smoothing: antialiased;
+	-moz-osx-font-smoothing: grayscale;
+	text-align: center;
+	/* color: #2c3e50; */
+}
+
+main {
+	display: flex;
+	flex-direction: row;
+	justify-content: center;
+	height: 100vh;
+}
+
+body {
+	background-color: rgb(var(--color-bg));
+	color: white;
+}
+</style>
diff --git a/examples/demos/discogs-autotagging/views/src/assets/logo.png b/examples/demos/discogs-autotagging/views/src/assets/logo.png
diff --git a/examples/demos/discogs-autotagging/views/src/audio/ActivationSmoother.js b/examples/demos/discogs-autotagging/views/src/audio/ActivationSmoother.js
@@ -0,0 +1,65 @@
+import { EffnetDiscogsLabels as discogsLabels } from './labels.js';
+import { median } from 'mathjs';
+
+class ActivationSmoother {
+    #memory;
+    #size;
+    constructor (memorySize = 3) {
+        this.#memory = {};
+        this.#size = memorySize;
+
+        for (let label of discogsLabels) {
+            this.#memory[label] = Array(memorySize).fill(0);
+        }
+        console.log('Smoother init:\n');
+        console.log(this.#memory);
+    }
+
+    push (activations) {
+        if (activations.length < discogsLabels.length) throw RangeError;
+
+        if (this.#size === 1) {
+            console.log('smoother: size 1');
+            return activations;
+        }
+
+        return discogsLabels.map( (l, i) => {
+            // if (i==0) console.log('before shift', this.#memory[l]);
+            this.#memory[l].shift();
+            // if (i==0) console.log('before push', this.#memory[l]);
+            this.#memory[l].push(activations[i]);
+            // console.log('after push', this.#memory[l]);
+            const activationMedian = median(this.#memory[l]);
+            // console.log('after median', this.#memory[l]);
+            // this.#memory[l][this.#size-1] = activationMedian;
+            return activationMedian;
+        })
+    }
+
+    set memorySize (newSize) {
+        if (newSize === this.#size) return; // do nothing
+        if (newSize < 1) newSize = 1;
+        if (newSize > this.#size) {
+            const diff = newSize - this.#size; // to add
+            for (let label of discogsLabels) {
+                this.#memory[label].splice(0, 0, ...Array(diff).fill(0));
+            }
+            console.log(this.#memory);
+        }
+        if (newSize < this.#size) {
+            const diff = this.#size - newSize; // to remove
+            for (let label of discogsLabels) {
+                this.#memory[label].splice(0, diff);
+            }
+            console.log(this.#memory);
+        }
+        this.#size = newSize;
+        console.info(`Smoother now using memory size: ${this.#size}`);
+    }
+
+    get memorySize () {
+        return this.#size;
+    }
+}
+
+export default new ActivationSmoother();
diff --git a/examples/demos/discogs-autotagging/views/src/audio/feature-extractor.js b/examples/demos/discogs-autotagging/views/src/audio/feature-extractor.js
@@ -0,0 +1,109 @@
+// import { EssentiaWASM, EssentiaModel } from "essentia.js";
+import { EssentiaWASM } from "https://cdn.jsdelivr.net/npm/[email protected]/dist/essentia-wasm.es.js";
+import * as EssentiaModel from "https://cdn.jsdelivr.net/npm/[email protected]/dist/essentia.js-model.es.js";
+// retrieved from https://raw.githubusercontent.com/GoogleChromeLabs/web-audio-samples/main/audio-worklet/design-pattern/lib/wasm-audio-helper.js
+// import { RingBuffer } from "./wasm-audio-helper.js";
+
+class PatchHop {
+    constructor(patchSize, ratio) {
+        this.size = Math.floor(patchSize * ratio);
+        this.frameCount = 0;
+    }
+
+    incrementFrame() {
+        this.frameCount++;
+        this.frameCount %= this.size;
+    }
+
+    readyToHop() {
+        if (this.frameCount === 0) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+}
+
+function getZeroMatrix(x, y) {
+    let matrix = new Array(x);
+    for (let f = 0; f < x; f++) {
+        matrix[f] = new Array(y);
+        matrix[f].fill(0);
+    }
+    return matrix;
+}
+
+class FeatureExtractProcessor extends AudioWorkletProcessor {
+    constructor() {
+        super();
+        this._frameSize = 512;
+        this._hopSize = 256;
+        this._channelCount = 1;
+        this._patchHop = new PatchHop(128, 0.5); // if patchSize at 16kHz and 256 hopSize corresponds to about 3s of audio, this would jump by 1s
+        this._extractor = new EssentiaModel.EssentiaTFInputExtractor(EssentiaWASM, 'musicnn'); 
+        this._features = {
+            melSpectrum: getZeroMatrix(128, 96), // init melSpectrum 187x96 matrix with zeros
+            frameSize: 128,
+            melBandsSize: 96,
+            patchSize: 128
+        };
+
+        // buffersize mismatch helpers
+        this._hopRingBuffer = new RingBuffer(this._hopSize, this._channelCount);
+        this._frameRingBuffer = new RingBuffer(this._frameSize, this._channelCount);
+        this._hopData = [new Float32Array(this._hopSize)];
+        this._frameData = [new Float32Array(this._frameSize)];
+
+        // init zero-pad frameData so we have 512 values upon the very first 256 samples we get in
+        this._hopData[0].fill(0);
+
+        // setup worker comms
+        this._workerPort = undefined;
+        this._workerPortAvailable = false;
+        this.port.onmessage = (msg) => {
+            if (msg.data.port) {
+                console.info('Worklet received port from main!\n', msg.data.port);
+                this._workerPort = msg.data.port;
+                this._workerPortAvailable = true;
+                this._workerPort.postMessage({request: "check", check: "Received 2-way port from worker" });
+            }
+        }
+    }
+
+    process(inputList) {
+        let input = inputList[0];
+        if (!input[0]) {
+            console.info("worklet: empty input buffer");
+            return true;
+        }
+
+        this._hopRingBuffer.push(input);
+
+        if (this._hopRingBuffer.framesAvailable >= this._hopSize) {
+
+            this._frameRingBuffer.push(this._hopData); // always push the previous hopData samples to create overlap of hopSize
+            this._hopRingBuffer.pull(this._hopData);
+            this._frameRingBuffer.push(this._hopData); // push new hopData samples
+
+            if (this._frameRingBuffer.framesAvailable >= this._frameSize) {
+                // console.count('frame');
+                this._frameRingBuffer.pull(this._frameData);
+                this._features.melSpectrum.push(this._extractor.compute(this._frameData[0]).melSpectrum);
+                this._features.melSpectrum.shift();
+                this._patchHop.incrementFrame();
+                if (this._patchHop.readyToHop() && this._workerPort) {
+                    // send features to Worker for inference
+                    // console.info('Computed new patch of features\n', this._features);
+                    this._workerPort.postMessage({
+                        request: "features",
+                        features: this._features
+                    });
+                }
+            }
+        }
+
+        return true;
+    }
+}
+
+registerProcessor("feature-extract-processor", FeatureExtractProcessor);