Merge pull request #386 from Mindgamesnl/feature/preloading-plus-comm…

…ands

Fix syncing, prebuffering, and add command

client/public/metadata.json

@@ -1 +1 @@
-{"buildMajor":1,"buildMinor":125,"buildRevision":194,"buildTag":"dev","buildDate":"Sun Jan 14 2024","build":"1.125.194 dev"}
+{"buildMajor":1,"buildMinor":125,"buildRevision":195,"buildTag":"dev","buildDate":"Thu Jan 25 2024","build":"1.125.195 dev"}

client/src/client/services/media/objects/Channel.jsx

@@ -143,7 +143,7 @@ export class Channel {
   tick() {
     // tick all sounds
     this.sounds.forEach((sound) => {
-      sound.tick();
+      sound.tick.bind(sound)();
     });
   }
 

client/src/client/services/media/objects/Sound.jsx

@@ -1,4 +1,3 @@
-import { GetAudio } from '../../../util/AudioFactory';
 import { AUDIO_ENDPOINTS, AudioSourceProcessor } from '../../../util/AudioSourceProcessor';
 import { TimeService } from '../../time/TimeService';
 import { SocketManager } from '../../socket/SocketModule';
@@ -7,6 +6,7 @@ import { ReportError } from '../../../util/ErrorReporter';
 import { getGlobalState } from '../../../../state/store';
 import { debugLog } from '../../debugging/DebugService';
 import { isDomainOfficial } from '../../../config/MagicValues';
+import { AudioPreloader } from '../../preloading/AudioPreloader';
 
 export class Sound extends AudioSourceProcessor {
   constructor(opts = {}) {
@@ -36,20 +36,20 @@ export class Sound extends AudioSourceProcessor {
 
   whenInitialized(f) {
     if (this.loaded) {
-      f();
+      f.bind(this)();
     } else {
       this.initCallbacks.push(f);
     }
   }
 
-  async load(source, allowCaching = true) {
+  async load(source) {
     if (this.startedLoading) return;
     this.startedLoading = true;
     this.rawSource = source;
 
+    this.soundElement = await AudioPreloader.getResource(source);
     source = await this.translate(source);
 
-    this.soundElement = await GetAudio(source, true, allowCaching);
     // mute default
     if (this.options.startMuted) {
       this.soundElement.volume = 0;
@@ -141,7 +141,7 @@ export class Sound extends AudioSourceProcessor {
         debugLog(`Ready state is ${this.soundElement.readyState}, metadata is available`);
         this.loaded = true;
         for (let i = 0; i < this.initCallbacks.length; i++) {
-          const shouldStop = this.initCallbacks[i]();
+          const shouldStop = this.initCallbacks[i].bind(this)();
           if (shouldStop) {
             debugLog('Stopping init callbacks');
             this.initCallbacks = [];
@@ -286,10 +286,12 @@ export class Sound extends AudioSourceProcessor {
       // debugLog('Starting synced media');
       const start = new Date(date);
       const predictedNow = TimeService.getPredictedTime();
-      let seconds = (predictedNow - start) / 1000;
+      let seconds = ((predictedNow.getTime() / 1000) - (start.getTime() / 1000)) / 1000;
 
       // add at startAt timestamp to the seconds to still apply the offset
-      seconds += this.startAtMillis / 1000;
+      if (this.startAtMillis) {
+        seconds += this.startAtMillis / 1000;
+      }
 
       // debugLog(`Started ${seconds} ago`);
       const length = this.soundElement.duration;

client/src/client/services/preloading/AudioPreloader.jsx

@@ -0,0 +1,98 @@
+import { AudioSourceProcessor } from '../../util/AudioSourceProcessor';
+import { PreloadedMedia } from './PreloadedMedia';
+import { debugLog, feedDebugValue } from '../debugging/DebugService';
+import { DebugStatistic } from '../debugging/DebugStatistic';
+
+export const AudioPreloader = new class IAudPreload {
+  constructor() {
+    this.sourceRewriter = new AudioSourceProcessor();
+    this.namespaces = {}; // each namespace has a list of media
+  }
+
+  async fetch(source, namespace) {
+    source = await this.sourceRewriter.translate(source);
+    const media = new PreloadedMedia(source, namespace);
+
+    if (this.namespaces[namespace] == null) {
+      this.namespaces[namespace] = [];
+    }
+
+    this.namespaces[namespace].push(media);
+    this.submitStatistic();
+  }
+
+  drop(namespace) {
+    // does the namespace exist?
+    if (this.namespaces[namespace] == null) {
+      return;
+    }
+
+    // loop through all media in the namespace
+    this.namespaces[namespace].forEach((media) => {
+      // remove the media from the source
+      media.preDelete();
+    });
+
+    // delete the namespace
+    delete this.namespaces[namespace];
+    this.submitStatistic();
+  }
+
+  findAndRemoveMedia(source) {
+    // loop through all namespaces
+    // eslint-disable-next-line no-restricted-syntax
+    for (const namespace in this.namespaces) {
+      // eslint-disable-next-line no-prototype-builtins
+      if (this.namespaces.hasOwnProperty(namespace)) {
+        // loop through all media in the namespace
+        // eslint-disable-next-line no-restricted-syntax
+        for (const media of this.namespaces[namespace]) {
+          // does the media match the source?
+          if (media.source === source) {
+            // this is the one we want! now also remove it from the namespace
+            const countPre = this.namespaces[namespace].length;
+            this.namespaces[namespace].splice(this.namespaces[namespace].indexOf(media), 1);
+            const countPost = this.namespaces[namespace].length;
+            if (countPre === countPost) {
+              // eslint-disable-next-line no-console
+              console.warn('Could not remove media from namespace');
+            } else {
+              this.submitStatistic();
+            }
+            return media;
+          }
+        }
+      }
+    }
+
+    return null;
+  }
+
+  async getResource(source) {
+    source = await this.sourceRewriter.translate(source);
+
+    // find a preloaded media that matches the source
+    let media = this.findAndRemoveMedia(source);
+
+    if (media == null) {
+      // create new
+      media = new PreloadedMedia(source, null);
+    } else {
+      debugLog(`Using preloaded media, found ${media.source} in namespace ${media.namespace}, and it already has ready state ${media.audio.readyState}`);
+    }
+
+    return media.audio;
+  }
+
+  submitStatistic() {
+    let count = 0;
+    // eslint-disable-next-line no-restricted-syntax
+    for (const namespace in this.namespaces) {
+      // eslint-disable-next-line no-prototype-builtins
+      if (this.namespaces.hasOwnProperty(namespace)) {
+        count += this.namespaces[namespace].length;
+      }
+    }
+    feedDebugValue(DebugStatistic.PRELOADED_SOUNDS, count);
+  }
+}();

client/src/client/services/preloading/PreloadedMedia.jsx

@@ -0,0 +1,17 @@
+export class PreloadedMedia {
+  constructor(source, namespace) {
+    this.source = source;
+    this.namespace = namespace;
+
+    const soundElement = new Audio();
+    soundElement.autoplay = false;
+    soundElement.src = source;
+    soundElement.load();
+
+    this.audio = soundElement;
+  }
+
+  preDelete() {
+    // optionally do something before the media is deleted
+  }
+}

client/src/client/services/socket/handlers/HandlePrefetch.jsx

@@ -1,20 +1,29 @@
-import { ClearPrefetchedMedia, PreFetch } from '../../../util/AudioFactory';
 import { getGlobalState } from '../../../../state/store';
+import { AudioPreloader } from '../../preloading/AudioPreloader';
 
 export function handlePrefetchPacket(data) {
-  if (data.clear) {
-    // clear all prefetched bullshit
+  const { clear, source } = data;
+  let { origin } = data;
+  // clear = bool, whether the origin context should be cleared
+  // origin = string, the origin context
+  // source = untranslated media source
+
+  // if origin is null, default to global
+  if (origin == null) {
+    origin = 'global';
+  }
+
+  if (clear) {
     setTimeout(() => {
-      ClearPrefetchedMedia();
-    }, 2500);
+      AudioPreloader.drop(origin);
+    }, 500);
   } else {
     if (!getGlobalState().settings.prefetchMedia) {
       return;
     }
-    const toFetch = data.source;
-    // fetch a file
+
     setTimeout(() => {
-      PreFetch(toFetch);
-    }, 2500);
+      AudioPreloader.fetch(source, origin);
+    }, 500);
   }
 }

client/src/metadata.json

@@ -1 +1 @@
-{"buildMajor":1,"buildMinor":125,"buildRevision":194,"buildTag":"dev","buildDate":"Sun Jan 14 2024","build":"1.125.194 dev"}
+{"buildMajor":1,"buildMinor":125,"buildRevision":195,"buildTag":"dev","buildDate":"Thu Jan 25 2024","build":"1.125.195 dev"}

plugin/src/main/bash/data.bin

@@ -1 +1 @@
-BUILD_NUM="1056"
+BUILD_NUM="1058"

plugin/src/main/java/com/craftmend/openaudiomc/api/interfaces/Client.java

@@ -60,4 +60,9 @@ public interface Client {
      */
     boolean isModerating();
 
+    /**
+     * @param source The source to preload
+     */
+    void preloadMedia(String source);
+
 }

plugin/src/main/java/com/craftmend/openaudiomc/generic/client/objects/ClientConnection.java

@@ -22,9 +22,11 @@
 import com.craftmend.openaudiomc.generic.networking.interfaces.Authenticatable;
 import com.craftmend.openaudiomc.generic.networking.interfaces.NetworkingService;
 import com.craftmend.openaudiomc.generic.networking.packets.client.media.PacketClientCreateMedia;
+import com.craftmend.openaudiomc.generic.networking.packets.client.media.PacketClientPreFetch;
 import com.craftmend.openaudiomc.generic.networking.packets.client.ui.PacketClientModerationStatus;
 import com.craftmend.openaudiomc.generic.networking.packets.client.ui.PacketClientProtocolRevisionPacket;
 import com.craftmend.openaudiomc.generic.networking.packets.client.ui.PacketClientSetVolume;
+import com.craftmend.openaudiomc.generic.networking.payloads.client.media.ClientPreFetchPayload;
 import com.craftmend.openaudiomc.generic.rest.Task;
 import com.craftmend.openaudiomc.generic.node.packets.ClientConnectedPacket;
 import com.craftmend.openaudiomc.generic.node.packets.ClientDisconnectedPacket;
@@ -288,4 +290,10 @@ public void forcefullyDisableMicrophone(boolean disabled) {
     public boolean isModerating() {
         return session.isModerating();
     }
+
+    @Override
+    public void preloadMedia(String source) {
+        ClientPreFetchPayload payload = new ClientPreFetchPayload(source, "api", false);
+        sendPacket(new PacketClientPreFetch(payload));
+    }
 }

plugin/src/main/java/com/craftmend/openaudiomc/generic/commands/CommandService.java

@@ -35,7 +35,8 @@ public CommandService() {
                 new ModulesSubCommand(),
                 new ClientsSubCommand(),
                 new StopSubCommand(),
-                new PlaySubCommand()
+                new PlaySubCommand(),
+                new PreloadSubCommand()
         );
     }
 

plugin/src/main/java/com/craftmend/openaudiomc/generic/commands/subcommands/PreloadSubCommand.java

@@ -0,0 +1,47 @@
+package com.craftmend.openaudiomc.generic.commands.subcommands;
+
+import com.craftmend.openaudiomc.api.interfaces.Client;
+import com.craftmend.openaudiomc.generic.client.objects.ClientConnection;
+import com.craftmend.openaudiomc.generic.commands.interfaces.SubCommand;
+import com.craftmend.openaudiomc.generic.commands.objects.Argument;
+import com.craftmend.openaudiomc.generic.networking.packets.client.media.PacketClientPreFetch;
+import com.craftmend.openaudiomc.generic.networking.payloads.client.media.ClientPreFetchPayload;
+import com.craftmend.openaudiomc.generic.platform.OaColor;
+import com.craftmend.openaudiomc.generic.user.User;
+
+import java.util.Optional;
+
+public class PreloadSubCommand extends SubCommand {
+
+    public PreloadSubCommand() {
+        super("preload");
+        registerArguments(
+                new Argument("<selector> <source>", "Attempt to preload a sound for all players in a selection")
+        );
+    }
+
+    @Override
+    public void onExecute(User sender, String[] args) {
+        if (args.length == 0) {
+            sender.makeExecuteCommand("oa help " + getCommand());
+            return;
+        }
+
+        if (args.length == 2) {
+            int affected = 0;
+
+            ClientPreFetchPayload payload = new ClientPreFetchPayload(args[1], "command", false);
+
+            for (User<?> user : resolveSelector(sender, args[0])) {
+                Optional<Client> client = user.findClient();
+                if (client.isPresent()) {
+                    if (client.get().isConnected()) affected++;
+                    ClientConnection clientConnection = (ClientConnection) client.get();
+                    clientConnection.sendPacket(new PacketClientPreFetch(payload));
+                }
+            }
+            message(sender, OaColor.GREEN + "Requested " + affected + " web-clients to preload the sound");
+            return;
+        }
+    }
+}

plugin/src/main/java/com/craftmend/openaudiomc/generic/networking/packets/client/media/PacketClientPreFetch.java

@@ -7,11 +7,15 @@
 public class PacketClientPreFetch extends AbstractPacket {
 
     public PacketClientPreFetch(String source) {
-        super(new ClientPreFetchPayload(source, false), PacketChannel.CLIENT_OUT_PREFETCH, null);
+        super(new ClientPreFetchPayload(source, "automatic", false), PacketChannel.CLIENT_OUT_PREFETCH, null);
     }
 
     public PacketClientPreFetch(boolean clear) {
-        super(new ClientPreFetchPayload(null, clear), PacketChannel.CLIENT_OUT_PREFETCH, null);
+        super(new ClientPreFetchPayload(null, "automatic", clear), PacketChannel.CLIENT_OUT_PREFETCH, null);
+    }
+
+    public PacketClientPreFetch(ClientPreFetchPayload payload) {
+        super(payload, PacketChannel.CLIENT_OUT_PREFETCH, null);
     }
 
 }

plugin/src/main/java/com/craftmend/openaudiomc/generic/networking/payloads/client/media/ClientPreFetchPayload.java

@@ -11,6 +11,7 @@
 public class ClientPreFetchPayload extends AbstractPacketPayload {
 
     private String source;
+    private String origin = "automatic"; // or command, if invoked manually
     private boolean clear = false;
 
 }

plugin/src/main/resources/data.bin

@@ -1 +1 @@
-BUILD_NUM="1056"
+BUILD_NUM="1058"