New audio! (#335)

* Outputs audio at chip frequency
* Spits it out as buffers to a web audio thread that then (terribly) downsamples it.
* Various queues and stats
* Horrible hacks to try and keep things in sync.

README.md

@@ -90,6 +90,7 @@ site to make it smaller and faster to load when it's deployed to [https://bbc.xa
   Doesn't support the sth: pseudo URL unlike `disc` and `tape`, but if given a ZIP file will attempt to use the `.rom`
   file assumed to be within.
 - (mostly internal use) `logFdcCommands`, `logFdcStateChanges` - turn on logging in the disc controller.
+- `audioDebug=true` turns on some audio debug graphs.
 
 ## Patches
 

index.html

@@ -214,6 +214,7 @@
           <div class="sidebar bottom"><img src="images/placeholder.png" alt="" /></div>
         </div>
       </div>
+      <canvas id="audio-stats" width="400" height="100"></canvas>
       <div id="leds">
         <table>
           <thead>

package.json

@@ -19,6 +19,7 @@
     "bootstrap": "^5.3.3",
     "bootswatch": "^5.3.3",
     "jquery": "^3.7.1",
+    "smoothie": "^1.36.1",
     "underscore": "^1.13.7"
   },
   "devDependencies": {

src/jsbeeb.css

@@ -1,3 +1,6 @@
+body {
+  overflow: hidden;
+}
 #outer {
   display: block;
 }
@@ -133,6 +136,12 @@ span.accesskey {
   position: fixed;
 }
 
+#audio-stats {
+  bottom: 0;
+  right: 0;
+  position: fixed;
+}
+
 /* hide the LED headers when the LEDs would likely cover the screen */
 @media screen and (max-width: 1200px) and (max-height: 700px) {
   #leds thead {
@@ -344,3 +353,13 @@ small {
 .btn:disabled {
   pointer-events: auto;
 }
+
+div.smoothie-chart-tooltip {
+  background: #444;
+  padding: 1em;
+  margin-top: 20px;
+  font-family: consolas, serif;
+  color: white;
+  font-size: 10px;
+  pointer-events: none;
+}

src/main.js

@@ -294,7 +294,9 @@ video = new Video(model.isMaster, canvas.fb32, function paint(minx, miny, maxx,
 });
 if (parsedQuery.fakeVideo !== undefined) video = new FakeVideo();
 
-const audioHandler = new AudioHandler($("#audio-warning"), audioFilterFreq, audioFilterQ, noSeek);
+const audioStatsNode = document.getElementById("audio-stats");
+const audioHandler = new AudioHandler($("#audio-warning"), audioStatsNode, audioFilterFreq, audioFilterQ, noSeek);
+if (!parsedQuery.audioDebug) audioStatsNode.style.display = "none";
 // Firefox will report that audio is suspended even when it will
 // start playing without user interaction, so we need to delay a
 // little to get a reliable indication.
@@ -1502,6 +1504,7 @@ function draw(now) {
         window.requestAnimationFrame(draw);
     }
 
+    audioHandler.soundChip.catchUp();
     gamepad.update(processor.sysvia);
     syncLights();
     if (last !== 0) {

src/soundchip.js

@@ -1,6 +1,4 @@
-const FloatType = typeof Float64Array !== "undefined" ? Float64Array : Float32Array;
-
-const volumeTable = new FloatType(16);
+const volumeTable = new Float32Array(16);
 (() => {
     let f = 1.0;
     for (let i = 0; i < 15; ++i) {
@@ -11,31 +9,32 @@ const volumeTable = new FloatType(16);
 })();
 
 function makeSineTable(attenuation) {
-    const sineTable = new FloatType(8192);
+    const sineTable = new Float32Array(8192);
     for (let i = 0; i < sineTable.length; ++i) {
         sineTable[i] = Math.sin((2 * Math.PI * i) / sineTable.length) * attenuation;
     }
     return sineTable;
 }
 
 export class SoundChip {
-    constructor(sampleRate) {
-        this.cpuFreq = 1 / (2 * 1000 * 1000); // TODO hacky here
+    constructor(onBuffer) {
+        this._onBuffer = onBuffer;
         // 4MHz input signal. Internal divide-by-8
         this.soundchipFreq = 4000000.0 / 8;
+        const sampleRate = this.soundchipFreq;
         // Square wave changes every time a counter hits zero: A full wave needs to be 2x counter zeros.
         this.waveDecrementPerSecond = this.soundchipFreq / 2;
         // Each sample in the buffer represents (1/sampleRate) time, so each time
         // we generate a sample, we need to decrement the counters by this amount:
         this.sampleDecrement = this.waveDecrementPerSecond / sampleRate;
         // How many samples are generated per CPU cycle.
-        this.samplesPerCycle = sampleRate * this.cpuFreq;
+        this.samplesPerCycle = sampleRate / 2000000;
         this.minCyclesWELow = 14; // Somewhat empirically derived; Repton 2 has only 14 cycles between WE low and WE high (@0x2caa)
 
         this.registers = new Uint16Array(4);
-        this.counter = new FloatType(4);
+        this.counter = new Float32Array(4);
         this.outputBit = [false, false, false, false];
-        this.volume = new FloatType(4);
+        this.volume = new Float32Array(4);
         this.generators = [
             this.toneChannel.bind(this),
             this.toneChannel.bind(this),
@@ -59,7 +58,7 @@ export class SoundChip {
 
         this.residual = 0;
         this.position = 0;
-        this.buffer = new FloatType(4096);
+        this.buffer = new Float32Array(512);
 
         this.latchedRegister = 0;
         this.slowDataBus = 0;
@@ -204,16 +203,24 @@ export class SoundChip {
         }
     }
 
-    advance(time) {
-        const num = time * this.samplesPerCycle + this.residual;
+    advance(cycles) {
+        const num = cycles * this.samplesPerCycle + this.residual;
         let rounded = num | 0;
         this.residual = num - rounded;
-        if (this.position + rounded >= this.buffer.length) {
-            rounded = this.buffer.length - this.position;
+        const bufferLength = this.buffer.length;
+        while (rounded > 0) {
+            const leftInBuffer = bufferLength - this.position;
+            const numSamplesToGenerate = Math.min(rounded, leftInBuffer);
+            this.generate(this.buffer, this.position, numSamplesToGenerate);
+            this.position += numSamplesToGenerate;
+            rounded -= numSamplesToGenerate;
+
+            if (this.position === bufferLength) {
+                this._onBuffer(this.buffer);
+                this.buffer = new Float32Array(bufferLength);
+                this.position = 0;
+            }
         }
-        if (rounded === 0) return;
-        this.generate(this.buffer, this.position, rounded);
-        this.position += rounded;
     }
 
     poke(value) {

src/web/audio-handler.js

@@ -1,27 +1,52 @@
+import rendererUrl from "./audio-renderer.js?url";
+import music500WorkletUrl from "../music5000-worklet.js?url";
+import { SmoothieChart, TimeSeries } from "smoothie";
 import { FakeSoundChip, SoundChip } from "../soundchip.js";
 import { DdNoise, FakeDdNoise } from "../ddnoise.js";
 import { Music5000, FakeMusic5000 } from "../music5000.js";
 
 export class AudioHandler {
-    constructor(warningNode, audioFilterFreq, audioFilterQ, noSeek) {
+    constructor(warningNode, statsNode, audioFilterFreq, audioFilterQ, noSeek) {
         this.warningNode = warningNode;
-
+        this.warningNode.toggle(false);
+        this.chart = new SmoothieChart({
+            tooltip: true,
+            labels: { precision: 0 },
+            yRangeFunction: (range) => {
+                return { min: 0, max: range.max };
+            },
+        });
+        this.stats = {};
+        this._addStat("queueSize", { strokeStyle: "rgb(51,126,108)" });
+        this._addStat("queueAge", { strokeStyle: "rgb(162,119,22)" });
+        this.chart.streamTo(statsNode, 100);
         /*global webkitAudioContext*/
         this.audioContext =
             typeof AudioContext !== "undefined"
                 ? new AudioContext()
                 : typeof webkitAudioContext !== "undefined"
                   ? new webkitAudioContext()
                   : null;
-        if (this.audioContext) {
+        this._jsAudioNode = null;
+        if (this.audioContext && this.audioContext.audioWorklet) {
             this.audioContext.onstatechange = () => this.checkStatus();
-            // TODO: try and remove the dependency on this being created first? maybe? like, why should the soundchip
-            //  care what renderer we have? Perhaps we can pick a sample rate and then use playback speed of the
-            //  js audio node to match real time with the output.
-            this.soundChip = new SoundChip(this.audioContext.sampleRate);
+            this.soundChip = new SoundChip((buffer, time) => this._onBuffer(buffer, time));
             this.ddNoise = noSeek ? new FakeDdNoise() : new DdNoise(this.audioContext);
-            this._setup(audioFilterFreq, audioFilterQ);
+            this._setup(audioFilterFreq, audioFilterQ).then();
         } else {
+            if (this.audioContext && !this.audioContext.audioWorklet) {
+                this.audioContext = null;
+                console.log("Unable to initialise audio: no audio worklet API");
+                this.warningNode.toggle(true);
+                const localhost = new URL(window.location);
+                localhost.hostname = "localhost";
+                this.warningNode.html(
+                    `No audio worklet API was found - there will be no audio. 
+                    If you are running a local jsbeeb, you must either use a host of
+                    <a href="${localhost}">localhost</a>, 
+                    or serve the content over <em>https</em>.`,
+                );
+            }
             this.soundChip = new FakeSoundChip();
             this.ddNoise = new FakeDdNoise();
         }
@@ -41,7 +66,7 @@ export class AudioHandler {
             this.audioContextM5000.onstatechange = () => this.checkStatus();
             this.music5000 = new Music5000((buffer) => this._onBufferMusic5000(buffer));
 
-            this.audioContextM5000.audioWorklet.addModule("./music5000-worklet.js").then(() => {
+            this.audioContextM5000.audioWorklet.addModule(music500WorkletUrl).then(() => {
                 this._music5000workletnode = new AudioWorkletNode(this.audioContextM5000, "music5000", {
                     outputChannelCount: [2],
                 });
@@ -52,30 +77,41 @@ export class AudioHandler {
         }
     }
 
-    _setup(audioFilterFreq, audioFilterQ) {
-        // NB must be assigned to some kind of object else it seems to get GC'd by Safari...
-        // TODO consider using a newer API. AudioWorkletNode? Harder to do two-way conversations there. Maybe needs
-        //  a AudioBufferSourceNode and pingponging between buffers?
-        this._jsAudioNode = this.audioContext.createScriptProcessor(2048, 0, 1);
-        this._jsAudioNode.onaudioprocess = (event) => {
-            const outBuffer = event.outputBuffer;
-            const chan = outBuffer.getChannelData(0);
-            this.soundChip.render(chan, 0, chan.length);
-        };
-
+    async _setup(audioFilterFreq, audioFilterQ) {
+        await this.audioContext.audioWorklet.addModule(rendererUrl);
         if (audioFilterFreq !== 0) {
-            this.soundChip.filterNode = this.audioContext.createBiquadFilter();
-            this.soundChip.filterNode.type = "lowpass";
-            this.soundChip.filterNode.frequency.value = audioFilterFreq;
-            this.soundChip.filterNode.Q.value = audioFilterQ;
-            this._jsAudioNode.connect(this.soundChip.filterNode);
-            this.soundChip.filterNode.connect(this.audioContext.destination);
+            const filterNode = this.audioContext.createBiquadFilter();
+            filterNode.type = "lowpass";
+            filterNode.frequency.value = audioFilterFreq;
+            filterNode.Q.value = audioFilterQ;
+            this._audioDestination = filterNode;
+            filterNode.connect(this.audioContext.destination);
         } else {
-            this.soundChip._jsAudioNode.connect(this.audioContext.destination);
+            this._audioDestination = this.audioContext.destination;
         }
+
+        this._jsAudioNode = new AudioWorkletNode(this.audioContext, "sound-chip-processor");
+        this._jsAudioNode.connect(this._audioDestination);
+        this._jsAudioNode.port.onmessage = (event) => {
+            const now = Date.now();
+            for (const stat of Object.keys(event.data)) {
+                if (this.stats[stat]) this.stats[stat].append(now, event.data[stat]);
+            }
+        };
     }
-    // Recent browsers, particularly Safari and Chrome, require a user
-    // interaction in order to enable sound playback.
+
+    _addStat(stat, info) {
+        const timeSeries = new TimeSeries();
+        this.stats[stat] = timeSeries;
+        info.tooltipLabel = stat;
+        this.chart.addTimeSeries(timeSeries, info);
+    }
+
+    _onBuffer(buffer) {
+        if (this._jsAudioNode) this._jsAudioNode.port.postMessage({ time: Date.now(), buffer }, [buffer.buffer]);
+    }
+
+    // Recent browsers, particularly Safari and Chrome, require a user interaction in order to enable sound playback.
     async tryResume() {
         if (this.audioContext) await this.audioContext.resume();
         if (this.audioContextM5000) await this.audioContextM5000.resume();