Skip to content

Commit 0e95d2c

Browse files
urishurish
authored
Merge pull request #13 from rejunity/audio-output
Audio output
2 parents 1ff79bb + 833653d commit 0e95d2c

File tree

11 files changed

+801
-9
lines changed

11 files changed

+801
-9
lines changed

‎index.html‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,10 @@
4242
<main>
4343
<div id="code-editor"></div>
4444
<div id="vga-canvas-container">
45-
<span id="fps-display">FPS: <span id="fps-count">00</span></span>
45+
<div>
46+
<span id="audio-latency-display">Audio latency: <span id="audio-latency-ms">00</span> ms </span>
47+
<span id="fps-display">FPS: <span id="fps-count">00</span></span>
48+
</div>
4649
<div id="input-values">
4750
ui_in:
4851
<button>0</button>
@@ -53,6 +56,7 @@
5356
<button>5</button>
5457
<button>6</button>
5558
<button>7</button>
59+
<button>Audio</button>
5660
</div>
5761
<canvas width="736" height="520" id="vga-canvas"></canvas>
5862
</div>

‎public/resampler.js‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// Copyright (C) 2024, Tiny Tapeout LTD
3+
// Author: Renaldas Zioma, Uri Shaked
4+
5+
class AudioResamplerProcessor extends AudioWorkletProcessor {
6+
constructor() {
7+
super();
8+
9+
// Define buffer properties
10+
this.downsampleFactor = 1;
11+
this.ringBufferSize = 16_384 * this.downsampleFactor; // stores approximately 5 frames of audio data at 192 kHz
12+
this.ringBuffer = new Float32Array(this.ringBufferSize); // ring-buffer helps to amortise uneven framerate and
13+
// keeps re-sampler from starving or overflowing
14+
this.writeIndex = 0; // Index where new samples are written
15+
this.readIndex = 0; // Index where samples are read
16+
this.previousSample = 0.0;
17+
this.ringBuffer.fill(this.previousSample);
18+
19+
this.downsampleBuffer = new Float32Array(128 * this.downsampleFactor);
20+
21+
// Listen to messages from the main thread
22+
this.port.onmessage = this.handleMessage.bind(this);
23+
24+
this.expectedFPS = 60.0;
25+
this.currentFPS = this.expectedFPS;
26+
console.log("Audio WebWorker started");
27+
}
28+
29+
handleMessage(event) {
30+
const data = event.data;
31+
32+
// Handle incoming audio samples and write to the ring buffer
33+
if (data.type === 'samples') {
34+
this.currentFPS = data.fps;
35+
this.downsampleFactor = data.downsampleFactor;
36+
const samples = data.samples;
37+
for (let i = 0; i < samples.length; i++) {
38+
if ((this.writeIndex + 1) % this.ringBufferSize == this.readIndex)
39+
{
40+
this.port.postMessage([this.ringBufferSize, 1.0]);
41+
console.log("Buffer is full. Dropping", samples.length - i, "incomming samples!");
42+
break; // Skip incomming samples when ring-buffer is full
43+
}
44+
if (this.writeIndex == this.readIndex)
45+
this.ringBuffer[(this.writeIndex - 1) % this.ringBufferSize] = samples[i];
46+
this.ringBuffer[this.writeIndex] = samples[i];
47+
this.writeIndex = (this.writeIndex + 1) % this.ringBufferSize; // Wrap around
48+
}
49+
50+
const samplesAvailable = (this.writeIndex - this.readIndex + this.ringBufferSize) % this.ringBufferSize;
51+
this.port.postMessage([samplesAvailable, samplesAvailable / this.ringBufferSize]);
52+
}
53+
else if (data.type === 'reset') {
54+
this.ringBuffer.fill(this.previousSample);
55+
this.readIndex = 0;
56+
this.writeIndex = 0;
57+
}
58+
}
59+
60+
// Linear interpolation for resampling
61+
interpolate(buffer, index1, index2, frac) {
62+
return (1 - frac) * buffer[index1] + frac * buffer[index2];
63+
}
64+
65+
// Process function that resamples the data from the ring buffer to match output size
66+
process(inputs, outputs) {
67+
const output = outputs[0]; // Mono output (1 channel)
68+
const outputData = output[0]; // Get the output data array
69+
70+
const playbackRate = this.currentFPS / this.expectedFPS;
71+
const borderSamples = 2;
72+
const samplesRequired = Math.round(outputData.length * playbackRate * this.downsampleFactor) + borderSamples;
73+
74+
// example when samplesRequired = 8 + 2 border samples
75+
// (border samples are marked as 'b' below)
76+
//
77+
// 3 subsequent invocations of process():
78+
//
79+
// ringBuffer: b01234567b01234567b01234567b.
80+
// process#0 ^........^ | <- sampling window
81+
// process#1 ^........^ | <- sampling window
82+
// process#2 ^........^| <- sampling window
83+
// WRITE pointer--`
84+
85+
// process#0 ^--READ pointer
86+
// process#1 ^--READ pointer
87+
// process#2 ^--READ pointer
88+
// after process#2 ^--READ pointer
89+
90+
const samplesAvailable = (this.writeIndex - this.readIndex + this.ringBufferSize) % this.ringBufferSize;
91+
if (samplesAvailable < borderSamples)
92+
{
93+
for (let i = 0; i < outputData.length; i++)
94+
outputData[i] = this.previousSample;
95+
console.log("Buffer is empty. Using previous sample value " + this.previousSample.toFixed(3));
96+
return true;
97+
}
98+
99+
const samplesConsumed = Math.min(samplesRequired, samplesAvailable) - borderSamples;
100+
101+
if (this.downsampleBuffer.length != outputData.length * this.downsampleFactor);
102+
this.downsampleBuffer = new Float32Array(outputData.length * this.downsampleFactor);
103+
104+
// Calculate resampling ratio
105+
const ratio = samplesConsumed / this.downsampleBuffer.length;
106+
107+
// Fill the output buffer by resampling from the ring buffer
108+
for (let i = 0; i < this.downsampleBuffer.length; i++) {
109+
const floatPos = 0.5 + ratio * (i + 0.5); // use sample centroids, thus +0.5
110+
const intPos = Math.floor(floatPos);
111+
const nextIntPos = intPos + 1;
112+
const frac = floatPos - intPos; // fractional part for interpolation
113+
114+
// Resample with linear interpolation
115+
this.downsampleBuffer[i] = this.interpolate(this.ringBuffer,
116+
(this.readIndex + intPos) % this.ringBufferSize,
117+
(this.readIndex + nextIntPos) % this.ringBufferSize, frac);
118+
}
119+
120+
// Optional (if audio context does not support 192 kHz) downsample to output buffer
121+
const N = this.downsampleFactor;
122+
if (N > 1) {
123+
for (let i = 0; i < outputData.length; i++) {
124+
let acc = this.downsampleBuffer[i*N];
125+
for (let j = 1; j < N; j++)
126+
acc += this.downsampleBuffer[i*N + j];
127+
outputData[i] = acc / N;
128+
}
129+
} else {
130+
for (let i = 0; i < outputData.length; i++)
131+
outputData[i] = this.downsampleBuffer[i];
132+
133+
}
134+
135+
// Store last sample as a future fallback value in case
136+
// if data would not be ready for the next process() call
137+
this.previousSample = outputData[outputData.length - 1];
138+
139+
// Update readIndex to match how many samples were consumed
140+
this.readIndex = (this.readIndex + samplesConsumed) % this.ringBufferSize;
141+
142+
return true; // return true to keep the processor alive
143+
}
144+
}
145+
146+
// Register the processor
147+
registerProcessor('resampler', AudioResamplerProcessor);
148+

‎src/AudioPlayer.ts‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// Copyright (C) 2024, Tiny Tapeout LTD
3+
// Author: Renaldas Zioma, Uri Shaked
4+
5+
export class AudioPlayer {
6+
private audioCtx : AudioContext;
7+
private resamplerNode : AudioWorkletNode;
8+
9+
private downsampleIntFactor = 1;
10+
private downsampleFracFactor = 1;
11+
12+
constructor(private readonly sampleRate: number,
13+
private readonly fps: number,
14+
stateListener = null,
15+
private readonly bufferSize: number = 200) {
16+
this.audioCtx = new AudioContext({sampleRate:sampleRate, latencyHint:'interactive'});
17+
// Optional downsampling is used in case when audio context does not support 192 kHz
18+
// for example when context playback rate is 44.1 kHz:
19+
this.downsampleFracFactor = sampleRate / this.audioCtx.sampleRate;// 4.35 = 192_000 / 44_100
20+
this.downsampleIntFactor = Math.floor(this.downsampleFracFactor); // 4
21+
this.downsampleFracFactor /= this.downsampleIntFactor; // 1.088 ~~ 48_000 / 44_100
22+
23+
this.audioCtx.audioWorklet.addModule(new URL('/resampler.js', import.meta.url)).then(() => {
24+
25+
this.resamplerNode = new AudioWorkletNode(this.audioCtx, 'resampler');
26+
this.resamplerNode.connect(this.audioCtx.destination);
27+
28+
this.resamplerNode.port.onmessage = this.handleMessage.bind(this);
29+
30+
this.audioCtx.resume().then(() => {
31+
console.log('Audio playback started');
32+
});
33+
});
34+
35+
this.audioCtx.onstatechange = stateListener;
36+
}
37+
38+
readonly latencyInMilliseconds = 0.0;
39+
handleMessage(event) {
40+
const getEffectiveLatency = (audioContext) => {
41+
return audioContext.outputLatency || audioContext.baseLatency || 0;
42+
}
43+
44+
const samplesInBuffer = event.data[0];
45+
this.latencyInMilliseconds = samplesInBuffer / this.sampleRate * 1000.0;
46+
this.latencyInMilliseconds += getEffectiveLatency(this.audioCtx) * 1000.0;
47+
48+
const bufferOccupancy = event.data[1];
49+
if (this.resumeScheduled && bufferOccupancy > 0.25) // resume playback once resampler's
50+
{ // buffer is at least 25% full
51+
this.audioCtx.resume();
52+
this.resumeScheduled = false;
53+
}
54+
}
55+
56+
private writeIndex = 0;
57+
readonly buffer = new Float32Array(this.bufferSize); // larger buffer reduces the communication overhead with the worker thread
58+
// however, if buffer is too large it could lead to worker thread starving
59+
feed(value: number, current_fps: number) {
60+
if (this.writeIndex >= this.bufferSize) {
61+
if (this.resamplerNode != null)
62+
{
63+
this.resamplerNode.port.postMessage({
64+
type: 'samples',
65+
samples: this.buffer,
66+
fps: current_fps * this.downsampleFracFactor,
67+
downsampleFactor: this.downsampleIntFactor,
68+
});
69+
}
70+
this.writeIndex = 0;
71+
}
72+
73+
this.buffer[this.writeIndex] = value;
74+
this.writeIndex++;
75+
}
76+
77+
private resumeScheduled = false;
78+
resume() {
79+
// Pre-feed buffers before resuming playback to avoid starving playback
80+
this.resumeScheduled = true;
81+
if (this.resamplerNode != null)
82+
{
83+
this.resamplerNode.port.postMessage({
84+
type: 'reset'
85+
});
86+
}
87+
}
88+
89+
suspend() {
90+
this.resumeScheduled = false;
91+
this.audioCtx.suspend();
92+
if (this.resamplerNode != null)
93+
{
94+
this.resamplerNode.port.postMessage({
95+
type: 'reset'
96+
});
97+
}
98+
}
99+
100+
isRunning() {
101+
return (this.audioCtx.state === "running");
102+
}
103+
needsFeeding() {
104+
return this.isRunning() || this.resumeScheduled;
105+
}
106+
107+
}
108+

‎src/FPSCounter.ts‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ export class FPSCounter {
33
private index = 0;
44
private lastTime = -1;
55
private pauseTime = -1;
6+
readonly fps = 0;
67

78
constructor() {}
89

@@ -18,6 +19,12 @@ export class FPSCounter {
1819
this.samples[this.index++ % this.samples.length] = time - this.lastTime;
1920
}
2021
this.lastTime = time;
22+
23+
if (this.index > 0) {
24+
const slice = this.samples.slice(0, this.index);
25+
const avgDelta = slice.reduce((a, b) => a + b, 0) / slice.length;
26+
this.fps = 1000 / avgDelta;
27+
}
2128
}
2229

2330
pause(time: number) {
@@ -34,12 +41,6 @@ export class FPSCounter {
3441
}
3542

3643
getFPS() {
37-
if (this.index === 0) {
38-
// Not enough data yet
39-
return 0;
40-
}
41-
const slice = this.samples.slice(0, this.index);
42-
const avgDelta = slice.reduce((a, b) => a + b, 0) / slice.length;
43-
return 1000 / avgDelta;
44+
return this.fps;
4445
}
4546
}

‎src/examples/index.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,6 @@ import { logo } from './logo';
55
import { conway } from './conway';
66
import { checkers } from './checkers';
77
import { drop } from './drop';
8+
import { music } from './music';
89

9-
export const examples: Project[] = [stripes, balls, logo, conway, checkers, drop];
10+
export const examples: Project[] = [music, stripes, balls, logo, conway, checkers, drop];

0 commit comments

Comments
 (0)