Skip to content

Commit

Permalink
Merge pull request MTG#90 from MTG/onsets-demo-improvements
Browse files Browse the repository at this point in the history
Onsets demo improvements
  • Loading branch information
jmarcosfer authored Feb 9, 2022
2 parents e58c58a + 12fc635 commit 143e373
Show file tree
Hide file tree
Showing 20 changed files with 844 additions and 242 deletions.
3 changes: 2 additions & 1 deletion examples/demos/onsets/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
"private": true,
"scripts": {
"dev": "cross-env NODE_ENV=development webpack-dev-server --open --hot",
"build": "cross-env NODE_ENV=production webpack --progress"
"build": "cross-env NODE_ENV=production webpack --progress",
"postinstall": "npx patch-package"
},
"browserslist": [
"> 1%",
Expand Down
20 changes: 20 additions & 0 deletions examples/demos/onsets/patches/audio-encoder+1.0.4.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
diff --git a/node_modules/audio-encoder/src/encodeWav.js b/node_modules/audio-encoder/src/encodeWav.js
index 51a531c..3b96798 100644
--- a/node_modules/audio-encoder/src/encodeWav.js
+++ b/node_modules/audio-encoder/src/encodeWav.js
@@ -82,9 +82,12 @@ function encodeWav(audioBuffer, cb) {

var blob = new Blob([uint8], { type: 'audio/x-wav' })

- setTimeout(function () {
- return cb(blob);
- }, 30);
+ // return new Promise((resolve, _) => {
+ // setTimeout(function () {
+ // resolve(blob);
+ // }, 30);
+ // });
+ return blob;
}

module.exports = encodeWav;
69 changes: 69 additions & 0 deletions examples/demos/onsets/patches/freesound+0.0.4.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
diff --git a/node_modules/freesound/freesound.js b/node_modules/freesound/freesound.js
index 879d67a..25a1ca1 100644
--- a/node_modules/freesound/freesound.js
+++ b/node_modules/freesound/freesound.js
@@ -57,46 +57,24 @@
uri = uri +"?"+ paramStr;
}

- if (typeof module !== 'undefined'){ // node.js
- var http = require("http");
- var options = {
- host: host,
- path: uri.substring(uri.indexOf("/",8),uri.length), // first '/' after 'http://'
- port: '80',
- method: method,
- headers: {'Authorization': authHeader}
- };
- var req = http.request(options,function(res){
- res.setEncoding('utf8');
- res.on('data', function (data){
- if([200,201,202].indexOf(res.statusCode)>=0)
- success(wrapper?wrapper(data):data);
- else
- error(data);
- });
- });
- req.on('error', error).end();
- }
- else{ // browser
- var xhr;
- try {xhr = new XMLHttpRequest();}
- catch (e) {xhr = new ActiveXObject('Microsoft.XMLHTTP');}
-
- xhr.onreadystatechange = function(){
- if (xhr.readyState === 4 && [200,201,202].indexOf(xhr.status)>=0){
- var data = eval("(" + xhr.responseText + ")");
- if(success) success(wrapper?wrapper(data):data);
- }
- else if (xhr.readyState === 4 && xhr.status !== 200){
- if(error) error(xhr.statusText);
- }
- };
- xhr.open(method, uri);
- xhr.setRequestHeader('Authorization',authHeader);
- if(content_type!==undefined)
- xhr.setRequestHeader('Content-Type',content_type);
- xhr.send(data);
- }
+ var xhr;
+ try {xhr = new XMLHttpRequest();}
+ catch (e) {xhr = new ActiveXObject('Microsoft.XMLHTTP');}
+
+ xhr.onreadystatechange = function(){
+ if (xhr.readyState === 4 && [200,201,202].indexOf(xhr.status)>=0){
+ var data = eval("(" + xhr.responseText + ")");
+ if(success) success(wrapper?wrapper(data):data);
+ }
+ else if (xhr.readyState === 4 && xhr.status !== 200){
+ if(error) error(xhr.statusText);
+ }
+ };
+ xhr.open(method, uri);
+ xhr.setRequestHeader('Authorization',authHeader);
+ if(content_type!==undefined)
+ xhr.setRequestHeader('Content-Type',content_type);
+ xhr.send(data);
};
var checkOauth = function(){
if(authHeader.indexOf("Bearer")==-1)
4 changes: 2 additions & 2 deletions examples/demos/onsets/public/0.build.worker.js

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion examples/demos/onsets/public/0.build.worker.js.map

Large diffs are not rendered by default.

70 changes: 35 additions & 35 deletions examples/demos/onsets/public/build.js

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion examples/demos/onsets/public/build.js.map

Large diffs are not rendered by default.

80 changes: 69 additions & 11 deletions examples/demos/onsets/src/App.vue
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<template>
<div id="app">
<div id="instructions" v-if="!instructionsClosed">
<div id="instructions" v-show="!instructionsClosed">
<!-- using placeholder GIFs -->
<instructions-modal :img-link="instructionsGifs"
@closed="instructionsClosed=true">
Expand All @@ -9,13 +9,13 @@
<template slot="dismiss">Got it!</template>
</instructions-modal>
</div>
<main v-if="instructionsClosed" class="d-flex flex-column justify-content-between align-items-center">
<main v-show="instructionsClosed" class="d-flex flex-column align-items-center">
<demos-header></demos-header>
<section id="middle-screen" class="d-flex flex-column justify-content-around align-items-center container-fluid">
<section id="middle-screen" class="d-flex flex-column align-items-center container-fluid">
<browse-display></browse-display>
<algorithm-controls></algorithm-controls>
<algorithm-controls :init="algorithmParameters"></algorithm-controls>
</section>
<demos-footer></demos-footer>
<demos-footer class="mt-auto"></demos-footer>
</main>
</div>
</template>
Expand All @@ -29,6 +29,7 @@ import DemosHeader from './components/DemosHeader.vue';
import BrowseDisplay from './components/BrowseDisplayPanel.vue';
import AlgorithmControls from './components/AlgorithmControls.vue';
import audioURL from './assets/acoustic-drums.wav';
import gif1 from './assets/onset-instructions-1.gif';
import gif2 from './assets/onset-instructions-2.gif';
import gif3 from './assets/onset-instructions-3.gif';
Expand All @@ -41,11 +42,71 @@ export default {
return {
instructionsClosed: false,
audioUploaded: false,
instructionsGifs: [gif1, gif2, gif3, gif4]
instructionsGifs: [gif1, gif2, gif3, gif4],
url: new URL(window.location),
algorithmParameters: {}
}
},
created () {
// EventBus.$on()
EventBus.$on('sound-selected', sound => {
if (sound.id !== '') {
this.url.searchParams.set('id', sound.id);
window.history.pushState({}, '', this.url);
}
})
EventBus.$on('algo-params-updated', params => {
let searchParams = new URLSearchParams(params);
for (let p of searchParams) {
this.url.searchParams.set(p[0], p[1]);
}
window.history.pushState({}, '', this.url);
})
this.initAlgoParams();
},
mounted () {
function loadSound () {
// initialize sound
if (this.url.searchParams.has('id')) {
// grab from URL params (Freesound-hosted)
console.log("grab from URL params (Freesound-hosted)")
const id = this.url.searchParams.get('id');
EventBus.$emit('sound-selected', {name: '', url: '', id: id, user: '', fsLink: '', license: ''});
} else {
// load self-hosted file
console.log("load self-hosted file")
EventBus.$emit("sound-selected", {name: 'acoustic-drums', url: audioURL, id: '', user: '', fsLink: '', license: ''});
}
}
this.$nextTick(loadSound);
},
methods: {
initAlgoParams () {
let algoParams = {
frameSize: 1024,
hopSize: 512,
odfs: ["hfc","complex"],
odfsWeights: [0.5,0.5],
sensitivity: 0.65
};
if (this.url.searchParams.has('frameSize')) {
for (let p of this.url.searchParams) {
if (p[0] === 'id') continue;
let paramVal = p[1].split(',').map( v => p[0] === 'odfs' ? v : Number(v))
if (p[0] === 'odfs' || p[0] === 'odfsWeights') {
algoParams[p[0]] = paramVal;
continue;
}
algoParams[p[0]] = paramVal[0];
}
console.info('found algo params in URL!', algoParams);
}
this.algorithmParameters = algoParams;
EventBus.$emit('algo-params-init', algoParams);
}
}
}
</script>
Expand All @@ -54,12 +115,9 @@ export default {
main {
height: 100vh;
width: 100vw;
overflow: scroll;
}
#middle-screen {
flex-grow: 1;
}
// #algo-controls {
// height: 200px;
// }
</style>
2 changes: 1 addition & 1 deletion examples/demos/onsets/src/assets/styles/globals.scss
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// $main-bg: #c4c4c4;
$primary: #E4454A;
$carousel-control-color: #000;

@import '~bootstrap/scss/bootstrap.scss';
@import '~bootstrap-vue/src/index.scss';
Expand Down
34 changes: 19 additions & 15 deletions examples/demos/onsets/src/components/AlgorithmControls.vue
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
@pointerleave="currentlyHovered='none'"></proportion-slider>
</div>
<div class="col-3 py-2">
<b-card id="info-panel" class="h-100 text-secondary" :sub-title="quickHelpTitle" body-class="small">
<b-card id="info-panel" class="h-100 text-secondary" :sub-title="quickHelpTitle">
<b-card-text>{{quickHelpText}}</b-card-text>
</b-card>
</div>
Expand All @@ -45,33 +45,37 @@ import ProportionSlider from "./ProportionSlider.vue";
import EventBus from "../core/event-bus";
const quickhelpContents = {
"frame-size": "Size of the audio frames used for analysis. A bigger frame will have greater frequency resolution, but poorer temporal resolution, and viceversa.",
"hop-size": "Rate at which audio is cut in frames for analysis. Given as a percentage of the frame size. At 100% there's no overlap between consecutive frames. Lower values mean higher frame rate, thus greater temporal resolution.",
sensitivity: "Regulates the threshold for onset detection. Higher values tend to produce more false positives. Increase it if you know that your chosen audio has onsets but none are being displayed.",
odf: "Four functions are available to use as the basis for onset detection. They can be combined and be given different ratios such that some have a greater effect than others. At least one has to be selected.",
HFC: "This function computes the high frequency content (HFC) of an audio spectrum. Particularly good for percussive events. ",
Complex: "Detects changes in magnitude and phase. Emphasizes significant energy changes in the magnitude spectrum and/or deviation from the expected values in the phase spectrum, caused by changes in pitch.",
Flux: "Characterizes changes in magnitude as the Euclidean distance of the difference between consecutive magnitude spectrum frames.",
"Complex Phase": "Similar to 'Complex' but considers only phase changes, weighted by magnitude. Good for tonal sounds such as bowed string, but tends to over-detect percussive events.",
"frame-size": "Size of audio chunks for analysis. Controls frequency resolution. Increase this when using the 'Complex' or 'Complex Phase' detection functions and a pitched sound.",
"hop-size": "Audio analysis frame rate, given as a percentage of the frame size. Lower values result in increased temporal resolution, but longer analysis time.",
sensitivity: "Regulates the threshold for onset detection. Higher values tend to produce more false positives. Increase it if you know that your chosen audio has onsets but few or none are being displayed.",
odf: "Four functions are available to use for onset detection. They can be combined and be given different ratios so that some have a greater effect than others on the results. At least one has to be selected.",
HFC: "This function computes the High Frequency Content (HFC) of a sound's spectrum. Great for detecting percussive events. ",
Complex: "Reacts to deviations in pitch and changes in the frequency components of the sound. Measures spectral differences of both magnitude and phase between frames.",
Flux: "Measures how quickly the frequency components of the sound are changing over time. Related to changes in timbre, works well with synthetic or found sound with no clear transients.",
"Complex Phase": "Similar to 'Complex' but considers only changes in phase, weighted by magnitude. Good for tonal sounds such as bowed string, but tends to over-detect percussive events.",
none: "Hover over any of the controls for more info."
};
const odfsNames = ['hfc', 'complex', 'flux', 'complex_phase'];
export default {
components: { ExpSlider, LinearSlider, ProportionSlider },
props: {
init: Object
},
data () {
return {
frameSize: 1024,
hopSizePercentage: 50,
frameSize: this.init.frameSize,
hopSizePercentage: this.init.hopSize * 100 / this.init.frameSize,
odfs: {
on: {
names: ["hfc", "complex"],
values: [0.5, 0.5]
names: this.init.odfs,
values: this.init.odfsWeights
},
off: {
names: ["flux", "complex_phase"]
names: odfsNames.filter( odf => !this.init.odfs.includes(odf) )
}
},
sensitivity: 0.3,
sensitivity: this.init.sensitivity,
paramsChanged: false,
currentlyHovered: "none"
}
Expand Down
43 changes: 38 additions & 5 deletions examples/demos/onsets/src/components/AudioDisplay.vue
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,19 @@
<b-icon icon="volume-up-fill" v-show="soundOn"></b-icon>
</b-button>
</b-button-group>
<b-button-group v-if="receivedSound">
<b-link v-if="soundData.fsLink !== ''" :href="soundData.fsLink" target="_blank">
{{soundData.name}} - {{soundData.user}}
</b-link>
<p v-else>
{{soundData.name}}
</p>
<license-logo v-if="licenseType" :license-type="licenseType" color="#E4454A" style="margin-left: 1em;"></license-logo>
</b-button-group>
<b-button-group>
<b-button id="download" @click="handleDownload" variant="light" :disabled="!downloadEnabled">
<b-icon icon="download"></b-icon>
Download
Download slices
</b-button>
</b-button-group>
</div>
Expand All @@ -32,10 +41,10 @@ import EventBus from '../core/event-bus';
import WaveSurfer from 'wavesurfer.js';
import MarkersPlugin from 'wavesurfer.js/dist/plugin/wavesurfer.markers';
import RegionsPlugin from 'wavesurfer.js/dist/plugin/wavesurfer.regions';
import audioURL from '../assets/acoustic-drums.wav';
import LicenseLogo from './LicenseLogo.vue';
export default {
components: {LicenseLogo},
data () {
return {
isPlaying: false,
Expand All @@ -46,6 +55,8 @@ export default {
sliceRegions: [],
waitingOnsets: false,
waitingOnsetsMsg: "Finding onsets...",
soundData: null,
receivedSound: false,
height: 0
}
},
Expand Down Expand Up @@ -102,6 +113,14 @@ export default {
ev.stopPropagation();
region.play();
})
},
redraw () {
setTimeout(() => {
this.wavesurfer.clearMarkers();
this.wavesurfer.clearRegions();
this.drawOnsets();
this.drawOnsetSlices();
}, 150);
}
},
watch: {
Expand All @@ -116,8 +135,21 @@ export default {
return true;
}
return false;
},
licenseType () {
if (this.soundData.license == '') return null;
if (this.soundData.license.includes('/zero/')) return 'zero';
if (this.soundData.license.includes('/by/')) return 'by';
if (this.soundData.license.includes('by-nc')) return 'by-nc';
return 'sampling';
}
},
created () {
window.addEventListener('resize', () => {
if (this.wavesurfer) this.redraw()
});
},
mounted () {
this.height = this.$el.querySelector("#audio-display").clientHeight;
let setPause = () => {
Expand All @@ -129,6 +161,9 @@ export default {
this.waitingOnsets = true;
this.onsetPositions = [];
this.soundData = sound;
this.receivedSound = true;
if (this.wavesurfer) {
this.wavesurfer.destroy();
}
Expand Down Expand Up @@ -171,8 +206,6 @@ export default {
this.waitingOnsetsMsg = "Recalculating...";
this.waitingOnsets = true;
})
EventBus.$emit("sound-selected", audioURL);
}
}
</script>
Expand Down
Loading

0 comments on commit 143e373

Please sign in to comment.