improve interface

hlorenzi · Mar 9, 2024 · 0a8cb59 · 0a8cb59
1 parent 0bd7e12
commit 0a8cb59
Show file tree

Hide file tree

Showing 6 changed files with 147 additions and 50 deletions.
diff --git a/index.html b/index.html
@@ -6,10 +6,19 @@
 	<head>
 		<meta charset="utf-8"/>
 		<title>Vowel Analysis</title>
+
+		<meta
+			name="viewport"
+			content="initial-scale=1, width=device-width, shrink-to-fit=yes">
 	</head>
 
 
 	<style>
+		* {
+			box-sizing: border-box;
+        	text-size-adjust: none;
+		}
+
 		html, body {
 			margin: 0;
 			padding: 0;

diff --git a/src/AnalysisChart.tsx b/src/AnalysisChart.tsx
@@ -20,7 +20,7 @@ export function AnalysisChart(props: {
     return <canvas
         ref={ canvas }
         style={{
-            "width": "700px",
+            "width": "min(700px, calc(100dvw - 2em))",
             "height": "120px",
             "border-radius": "0.5em",
         }}

diff --git a/src/VowelChart.tsx b/src/VowelChart.tsx
@@ -17,6 +17,8 @@ export function VowelChart(props: {
         synth: props.synth,
         mouseDown: false,
         mousePosNormalized: { x: 0, y: 0 },
+        mousePosFormants: { f1: 0, f2: 0 },
+        mousePath: [],
     }
 
     const [width, setWidth] = Solid.createSignal(700)
@@ -29,52 +31,91 @@ export function VowelChart(props: {
         state.canvas.addEventListener("mousedown", (ev) => mouseDown(state, ev))
         window.addEventListener("mousemove", (ev) => mouseMove(state, ev))
         window.addEventListener("mouseup", (ev) => mouseUp(state, ev))
-        draw(state)
+        state.canvas.addEventListener("touchstart", (ev) => mouseDown(state, ev));
+        window.addEventListener("touchend", (ev) => mouseUp(state, ev));
+        window.addEventListener("touchcancel", (ev) => mouseUp(state, ev));
+        window.addEventListener("touchmove", (ev) => mouseMove(state, ev));
+        window.requestAnimationFrame(() => draw(state))
     })
 
 
     return <canvas
         ref={ state.canvas }
         style={{
-            "width": "700px",
-            "height": "400px",
+            "width": "min(700px, calc(100dvw - 2em))",
+            "height": "min(400px, 60dvh)",
             "border-radius": "0.5em",
         }}
     />
 }
 
 
+interface Position
+{
+    x: number
+    y: number
+}
+
+
+interface PathPoint
+{
+    x: number
+    y: number
+    timer: number
+}
+
+
 interface State
 {
     canvas: HTMLCanvasElement
     ctx: CanvasRenderingContext2D
     synth: VowelSynth
 
     mouseDown: boolean
-    mousePosNormalized: {
-        x: number,
-        y: number,
+    mousePosNormalized: Position
+    mousePosFormants: {
+        f1: number
+        f2: number
     }
+    mousePath: PathPoint[]
 }
 
 
 function updateMousePos(
     state: State,
-    ev: MouseEvent)
+    ev: MouseEvent | TouchEvent)
 {
     const rect = state.canvas.getBoundingClientRect()
 
-    const x = (ev.clientX - rect.x) / rect.width
-    const y = (ev.clientY - rect.y) / rect.height
+    const [clientX, clientY] = "touches" in ev ?
+        [ev.touches.item(0)?.clientX ?? 0, ev.touches.item(0)?.clientY ?? 0] :
+        [ev.clientX, ev.clientY]
+
+    const x = (clientX - rect.x) / rect.width
+    const y = (clientY - rect.y) / rect.height
 
     state.mousePosNormalized = { x, y }
 
-    const f2 = mapViewToValue(1 - x, Data.f2Min, Data.f2Max)
-    const f1 = mapViewToValue(y, Data.f1Min, Data.f1Max)
+    state.mousePosFormants = {
+        f1: Math.floor(mapViewToValue(y, Data.f1Min, Data.f1Max)),
+        f2: Math.floor(mapViewToValue(1 - x, Data.f2Min, Data.f2Max)),
+    }
 
-    console.log(f1.toFixed(0), f2.toFixed(0))
+    /*console.log(
+        "mouse:",
+        "F1 = ", state.mousePosFormants.f1, "Hz",
+        "F2 = ", state.mousePosFormants.f2, "Hz")*/
 
-    state.synth.setFrequencies(f1, f2)
+    if (state.mouseDown)
+    {
+        state.mousePath.push({ ...state.mousePosNormalized, timer: 1 })
+        while (state.mousePath.length > 200)
+            state.mousePath.splice(0, 1)
+    }
+
+    state.synth.setFrequencies(
+        state.mousePosFormants.f1,
+        state.mousePosFormants.f2)
 }
 
 
@@ -94,36 +135,34 @@ function mapValueToView(x: number, min: number, max: number)
 
 function mouseDown(
     state: State,
-    ev: MouseEvent)
+    ev: MouseEvent | TouchEvent)
 {
     ev.preventDefault()
     state.mouseDown = true
     state.synth.resume()
     state.synth.setGain(1)
+    state.mousePath = []
     updateMousePos(state, ev)
-    draw(state)
 }
 
 
 function mouseMove(
     state: State,
-    ev: MouseEvent)
+    ev: MouseEvent | TouchEvent)
 {
     ev.preventDefault()
     updateMousePos(state, ev)
-    draw(state)
 }
 
 
 function mouseUp(
     state: State,
-    ev: MouseEvent)
+    ev: MouseEvent | TouchEvent)
 {
     ev.preventDefault()
     state.mouseDown = false
     state.synth.setGain(0)
     updateMousePos(state, ev)
-    draw(state)
 }
 
 
@@ -145,9 +184,14 @@ const ipaVowels = [
 ]
 
 
+const isMobile = window.matchMedia("(pointer: coarse)").matches
+
+
 function draw(
     state: State)
 {
+    window.requestAnimationFrame(() => draw(state))
+
     const pixelRatio = window.devicePixelRatio
     const rect = state.canvas.getBoundingClientRect()
     const w =
@@ -183,18 +227,21 @@ function draw(
     state.ctx.lineWidth = 1
     state.ctx.strokeStyle = "#aaa"
     state.ctx.fillStyle = "#aaa"
-    state.ctx.font = "0.75em Times New Roman"
+    state.ctx.font = `${ isMobile ? "1.5em" : "0.75em" } Times New Roman`
     state.ctx.fillStyle = "#000"
     state.ctx.textAlign = "left"
     state.ctx.textBaseline = "top"
-    for (let freq = Data.f2Min + 500; freq <= Data.f2Max; freq += 500)
+    for (let freq = Data.f2Min + 500; freq < Data.f2Max; freq += 500)
     {
         const x = Math.floor(w - mapValueToView(freq, Data.f2Min, Data.f2Max) * w)
         state.ctx.beginPath()
         state.ctx.moveTo(x, 0)
         state.ctx.lineTo(x, h)
         state.ctx.stroke()
-        state.ctx.fillText(freq + " Hz", x + 2, 2)
+        state.ctx.fillText(
+            freq == Data.f2Min + 500 ? `F2 = ${ freq } Hz` : `${ freq }`,
+            x + 2,
+            2)
     }
     state.ctx.textBaseline = "bottom"
     for (let freq = Data.f1Min + 200; freq <= Data.f1Max; freq += 200)
@@ -204,11 +251,14 @@ function draw(
         state.ctx.moveTo(0, y)
         state.ctx.lineTo(w, y)
         state.ctx.stroke()
-        state.ctx.fillText(freq + " Hz", 2, y - 2)
+        state.ctx.fillText(
+            freq == Data.f1Max ? `F1 = ${ freq } Hz` : `${ freq }`,
+            2,
+            y - 2)
     }
 
     // Draw IPA symbols
-    state.ctx.font = "2em Times New Roman"
+    state.ctx.font = `${ isMobile ? "2.5em" : "2em" } Times New Roman`
     state.ctx.fillStyle = "#000"
     state.ctx.textAlign = "center"
     state.ctx.textBaseline = "middle"
@@ -219,19 +269,56 @@ function draw(
         state.ctx.fillText(vowel.symbol, x, y)
     }
 
+    // Draw mouse path
+    state.ctx.strokeStyle = "#048"
+    state.ctx.lineWidth = 2
+    for (let p = 1; p < state.mousePath.length; p++)
+    {
+        state.ctx.beginPath()
+        const timer = Math.max(0, state.mousePath[p].timer)
+        state.ctx.strokeStyle = `rgb(0 40 255 / ${ 0.25 + 0.75 * timer })`
+
+        const pA = state.mousePath[p - 1]
+        const pB = state.mousePath[p]
+        state.ctx.moveTo(pA.x * w, pA.y * h)
+        state.ctx.lineTo(pB.x * w, pB.y * h)
+
+        const vecX = (pB.x - pA.x) * w
+        const vecY = (pB.y - pA.y) * h
+        const vecMagn = Math.sqrt(vecX * vecX + vecY * vecY)
+        const vecXN = vecX / vecMagn
+        const vecYN = vecY / vecMagn
+        const crossSize = 3
+        state.ctx.moveTo(pA.x * w - vecYN * crossSize, pA.y * h + vecXN * crossSize)
+        state.ctx.lineTo(pA.x * w + vecYN * crossSize, pA.y * h - vecXN * crossSize)
+
+        state.ctx.stroke()
+    }
+
+    state.mousePath.forEach((p) => p.timer -= 1 / 30)
+
     // Draw mouse
     if (state.mouseDown)
     {
-        state.ctx.strokeStyle = "#048"
+        state.ctx.strokeStyle = "rgb(0 40 255)"
         state.ctx.lineWidth = 2
         state.ctx.beginPath()
         state.ctx.arc(
             state.mousePosNormalized.x * w,
             state.mousePosNormalized.y * h,
-            5,
+            2,
             0,
             Math.PI * 2)
         state.ctx.stroke()
+
+        state.ctx.font = `${ isMobile ? "1.5em" : "0.75em" } Times New Roman`
+        state.ctx.textAlign = "center"
+        state.ctx.textBaseline = "bottom"
+        state.ctx.fillText(
+            `(${ state.mousePosFormants.f1 }, ` +
+            `${ state.mousePosFormants.f2 } Hz)`,
+            state.mousePosNormalized.x * w,
+            state.mousePosNormalized.y * h + (isMobile ? -120 : -10))
     }
 
     state.ctx.restore()

diff --git a/src/formantExtractor.ts b/src/formantExtractor.ts
@@ -2,17 +2,20 @@ import { forwardLinearPrediction } from "./lpc.ts"
 import { findRoots, Complex } from "./roots.ts"
 
 
+// From: https://www.mathworks.com/help/signal/ug/formant-estimation-with-lpc-coefficients.html
 export function extractFormants(
     sample: Float32Array,
     samplingFrequency: number)
     : number[]
 {
-    //console.log("length", sample.length, "freq", samplingFrequency, "ratio", sample.length / samplingFrequency)
-    const sampleWindowed = sample.map((s, i) => s * hammingWindow(i, sample.length))
-    const sampleFiltered = preemphasisFilter(sampleWindowed)
-    //console.log(preemphasisFilter(new Float32Array([1, 1, 1, 1])))
+    const sampleWindowed =
+        sample.map((s, i) => s * hammingWindow(i, sample.length))
 
-    const lpc = forwardLinearPrediction(sampleFiltered, 8)
+    const sampleFiltered =
+        preemphasisFilter(sampleWindowed)
+
+    const lpc =
+        forwardLinearPrediction(sampleFiltered, 8)
 
     const roots = findRoots(lpc)
         .filter(c => c.imag >= 0)
@@ -23,13 +26,13 @@ export function extractFormants(
     const frequencies = angles
         .map(a => a * (samplingFrequency / (2 * Math.PI)))
 
-    const complexMagnitude = (c: Complex) => Math.sqrt(c.real * c.real + c.imag * c.imag)
+    const complexMagnitude =
+        (c: Complex) => Math.sqrt(c.real * c.real + c.imag * c.imag)
 
     const bandwidths = roots
         .map(r => -0.5 * (samplingFrequency / (2 * Math.PI)) * Math.log(complexMagnitude(r)))
 
     const formants = []
-    //console.log(frequencies.map(f => f.toFixed(0).padStart(4)), bandwidths.map(b => b.toFixed(0).padStart(4)))
     for (let i = 0; i < angles.length; i++)
     {
         const frequency = frequencies[i]

diff --git a/src/main.tsx b/src/main.tsx
@@ -19,8 +19,8 @@ function Page()
     }}>
         <br/>
         <br/>
-        Click and drag to synthesize vowel sounds.<br/>
-        The blue bars on the bottom chart indicate formant frequencies. (Work-in-progress)
+        Click and drag to synthesize vowel sounds via formant frequencies.<br/><br/>
+        The blue bars on the bottom chart shows formant frequencies extracted from the waveform data. (Not working properly)
         <br/>
         <br/>
         <VowelChart synth={ synth }/>

diff --git a/src/vowelSynth.ts b/src/vowelSynth.ts
@@ -104,19 +104,17 @@ export class VowelSynth
     {
         if (this.nodeMicSrc)
             return
-
-        return new Promise<void>((resolve, reject) => {
-
-            const callback = (stream: MediaStream) => {
-                this.nodeMicSrc = this.ctx.createMediaStreamSource(stream)
-                this.nodeMicSrc.connect(this.nodeMicFilter)
-                resolve()
-            }
-
-            navigator.getUserMedia(
-                { audio: true },
-                callback,
-                reject)
-        })
+
+        try
+        {
+            const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
+            this.nodeMicSrc = this.ctx.createMediaStreamSource(stream)
+            this.nodeMicSrc.connect(this.nodeMicFilter)
+        }
+        catch (e)
+        {
+            window.alert("Could not open the microphone.\n\n" + e)
+            throw e
+        }
     }
 }