Draft: Augment audio system with the ability to have captions

Robust.Client/Animations/AnimationTrackPlaySound.cs

@@ -40,11 +40,7 @@ public override (int KeyFrameIndex, float FramePlayingTime)
                 var keyFrame = KeyFrames[keyFrameIndex];
 
                 var audioParams = keyFrame.AudioParamsFunc.Invoke();
-                var audio = new SoundPathSpecifier(keyFrame.Resource)
-                {
-                    Params = audioParams
-                };
-                IoCManager.Resolve<IEntitySystemManager>().GetEntitySystem<AudioSystem>().PlayEntity(audio, Filter.Local(), entity, true);
+                IoCManager.Resolve<IEntitySystemManager>().GetEntitySystem<AudioSystem>().PlayEntity(keyFrame.Specifier, Filter.Local(), entity, true, audioParams);
             }
 
             return (keyFrameIndex, playingTime);
@@ -55,7 +51,7 @@ public struct KeyFrame
             /// <summary>
             ///     The RSI state to play when this keyframe gets triggered.
             /// </summary>
-            public readonly string Resource;
+            public readonly ResolvedSoundSpecifier Specifier;
 
             /// <summary>
             ///     A function that returns the audio parameter to be used.
@@ -69,9 +65,9 @@ public struct KeyFrame
             /// </summary>
             public readonly float KeyTime;
 
-            public KeyFrame(string resource, float keyTime, Func<AudioParams>? audioParams = null)
+            public KeyFrame(ResolvedSoundSpecifier specifier, float keyTime, Func<AudioParams>? audioParams = null)
             {
-                Resource = resource;
+                Specifier = specifier;
                 KeyTime = keyTime;
                 AudioParamsFunc = audioParams ?? (() => AudioParams.Default);
             }

Robust.Client/ComponentTrees/CaptionTreeSystem.cs

@@ -0,0 +1,46 @@
+using System.Numerics;
+using Robust.Client.GameObjects;
+using Robust.Shared.ComponentTrees;
+using Robust.Shared.GameObjects;
+using Robust.Shared.Maths;
+using Robust.Shared.Physics;
+using Robust.Shared.Audio.Components;
+using Robust.Shared.IoC;
+
+namespace Robust.Client.ComponentTrees;
+
+public sealed class CaptionTreeSystem : ComponentTreeSystem<CaptionTreeComponent, CaptionComponent>
+{
+    [Dependency] private readonly IEntityManager _entityManager = default!;
+
+    #region Component Tree Overrides
+    protected override bool DoFrameUpdate => false;
+    protected override bool DoTickUpdate => true;
+    protected override bool Recursive => true;
+    protected override int InitialCapacity => 128;
+
+    protected override Box2 ExtractAabb(in ComponentTreeEntry<CaptionComponent> entry, Vector2 pos, Angle rot)
+    {
+        if (_entityManager.TryGetComponent(entry.Uid, out AudioComponent? audio))
+        {
+            var radius = audio.MaxDistance;
+            var radiusVec = new Vector2(radius, radius);
+            return new Box2(pos - radiusVec, pos + radiusVec);
+        }
+        return default;
+    }
+
+    protected override Box2 ExtractAabb(in ComponentTreeEntry<CaptionComponent> entry)
+    {
+        if (entry.Component.TreeUid == null)
+            return default;
+
+        var pos = XformSystem.GetRelativePosition(
+            entry.Transform,
+            entry.Component.TreeUid.Value,
+            GetEntityQuery<TransformComponent>());
+
+        return ExtractAabb(in entry, pos, default);
+    }
+    #endregion
+}

Robust.Server/Audio/AudioSystem.cs

@@ -81,27 +81,27 @@ private void AddAudioFilter(EntityUid uid, AudioComponent component, Filter filt
     }
 
     /// <inheritdoc />
-    public override (EntityUid Entity, AudioComponent Component)? PlayGlobal(string? filename, Filter playerFilter, bool recordReplay, AudioParams? audioParams = null)
+    public override (EntityUid Entity, AudioComponent Component)? PlayGlobal(ResolvedSoundSpecifier? specifier, Filter playerFilter, bool recordReplay, AudioParams? audioParams = null)
     {
-        if (string.IsNullOrEmpty(filename))
+        if (specifier is null)
             return null;
 
-        var entity = SetupAudio(filename, audioParams);
+        var entity = SetupAudio(specifier, audioParams);
         AddAudioFilter(entity, entity.Comp, playerFilter);
         entity.Comp.Global = true;
         return (entity, entity.Comp);
     }
 
     /// <inheritdoc />
-    public override (EntityUid Entity, AudioComponent Component)? PlayEntity(string? filename, Filter playerFilter, EntityUid uid, bool recordReplay, AudioParams? audioParams = null)
+    public override (EntityUid Entity, AudioComponent Component)? PlayEntity(ResolvedSoundSpecifier? specifier, Filter playerFilter, EntityUid uid, bool recordReplay, AudioParams? audioParams = null)
     {
-        if (string.IsNullOrEmpty(filename))
+        if (specifier is null)
             return null;
 
         if (TerminatingOrDeleted(uid))
             return null;
 
-        var entity = SetupAudio(filename, audioParams);
+        var entity = SetupAudio(specifier, audioParams);
         // Move it after setting it up
         XformSystem.SetCoordinates(entity, new EntityCoordinates(uid, Vector2.Zero));
         AddAudioFilter(entity, entity.Comp, playerFilter);
@@ -110,24 +110,24 @@ public override (EntityUid Entity, AudioComponent Component)? PlayEntity(string?
     }
 
     /// <inheritdoc />
-    public override (EntityUid Entity, AudioComponent Component)? PlayPvs(string? filename, EntityUid uid, AudioParams? audioParams = null)
+    public override (EntityUid Entity, AudioComponent Component)? PlayPvs(ResolvedSoundSpecifier? specifier, EntityUid uid, AudioParams? audioParams = null)
     {
-        if (string.IsNullOrEmpty(filename))
+        if (specifier is null)
             return null;
 
         if (TerminatingOrDeleted(uid))
             return null;
 
-        var entity = SetupAudio(filename, audioParams);
+        var entity = SetupAudio(specifier, audioParams);
         XformSystem.SetCoordinates(entity, new EntityCoordinates(uid, Vector2.Zero));
 
         return (entity, entity.Comp);
     }
 
     /// <inheritdoc />
-    public override (EntityUid Entity, AudioComponent Component)? PlayStatic(string? filename, Filter playerFilter, EntityCoordinates coordinates, bool recordReplay, AudioParams? audioParams = null)
+    public override (EntityUid Entity, AudioComponent Component)? PlayStatic(ResolvedSoundSpecifier? specifier, Filter playerFilter, EntityCoordinates coordinates, bool recordReplay, AudioParams? audioParams = null)
     {
-        if (string.IsNullOrEmpty(filename))
+        if (specifier is null)
             return null;
 
         if (TerminatingOrDeleted(coordinates.EntityId))
@@ -139,18 +139,18 @@ public override (EntityUid Entity, AudioComponent Component)? PlayStatic(string?
         if (!coordinates.IsValid(EntityManager))
             return null;
 
-        var entity = SetupAudio(filename, audioParams);
+        var entity = SetupAudio(specifier, audioParams);
         XformSystem.SetCoordinates(entity, coordinates);
         AddAudioFilter(entity, entity.Comp, playerFilter);
 
         return (entity, entity.Comp);
     }
 
     /// <inheritdoc />
-    public override (EntityUid Entity, AudioComponent Component)? PlayPvs(string? filename, EntityCoordinates coordinates,
+    public override (EntityUid Entity, AudioComponent Component)? PlayPvs(ResolvedSoundSpecifier? specifier, EntityCoordinates coordinates,
         AudioParams? audioParams = null)
     {
-        if (string.IsNullOrEmpty(filename))
+        if (specifier is null)
             return null;
 
         if (TerminatingOrDeleted(coordinates.EntityId))
@@ -163,7 +163,7 @@ public override (EntityUid Entity, AudioComponent Component)? PlayPvs(string? fi
             return null;
 
         // TODO: Transform TryFindGridAt mess + optimisation required.
-        var entity = SetupAudio(filename, audioParams);
+        var entity = SetupAudio(specifier, audioParams);
         XformSystem.SetCoordinates(entity, coordinates);
 
         return (entity, entity.Comp);
@@ -186,7 +186,7 @@ public override (EntityUid Entity, AudioComponent Component)? PlayPredicted(Soun
         if (sound == null)
             return null;
 
-        var audio = PlayPvs(GetSound(sound), source, audioParams ?? sound.Params);
+        var audio = PlayPvs(ResolveSound(sound), source, audioParams ?? sound.Params);
 
         if (audio == null)
             return null;
@@ -201,7 +201,7 @@ public override (EntityUid Entity, AudioComponent Component)? PlayPredicted(Soun
         if (sound == null)
             return null;
 
-        var audio = PlayPvs(GetSound(sound), coordinates, audioParams ?? sound.Params);
+        var audio = PlayPvs(ResolveSound(sound), coordinates, audioParams ?? sound.Params);
 
         if (audio == null)
             return null;
@@ -210,38 +210,38 @@ public override (EntityUid Entity, AudioComponent Component)? PlayPredicted(Soun
         return audio;
     }
 
-    public override (EntityUid Entity, AudioComponent Component)? PlayGlobal(string? filename, ICommonSession recipient, AudioParams? audioParams = null)
+    public override (EntityUid Entity, AudioComponent Component)? PlayGlobal(ResolvedSoundSpecifier? filename, ICommonSession recipient, AudioParams? audioParams = null)
     {
         return PlayGlobal(filename, Filter.SinglePlayer(recipient), false, audioParams);
     }
 
-    public override (EntityUid Entity, AudioComponent Component)? PlayGlobal(string? filename, EntityUid recipient, AudioParams? audioParams = null)
+    public override (EntityUid Entity, AudioComponent Component)? PlayGlobal(ResolvedSoundSpecifier? filename, EntityUid recipient, AudioParams? audioParams = null)
     {
         if (TryComp(recipient, out ActorComponent? actor))
             return PlayGlobal(filename, actor.PlayerSession, audioParams);
 
         return null;
     }
 
-    public override (EntityUid Entity, AudioComponent Component)? PlayEntity(string? filename, ICommonSession recipient, EntityUid uid, AudioParams? audioParams = null)
+    public override (EntityUid Entity, AudioComponent Component)? PlayEntity(ResolvedSoundSpecifier? filename, ICommonSession recipient, EntityUid uid, AudioParams? audioParams = null)
     {
         return PlayEntity(filename, Filter.SinglePlayer(recipient), uid, false, audioParams);
     }
 
-    public override (EntityUid Entity, AudioComponent Component)? PlayEntity(string? filename, EntityUid recipient, EntityUid uid, AudioParams? audioParams = null)
+    public override (EntityUid Entity, AudioComponent Component)? PlayEntity(ResolvedSoundSpecifier? filename, EntityUid recipient, EntityUid uid, AudioParams? audioParams = null)
     {
         if (TryComp(recipient, out ActorComponent? actor))
             return PlayEntity(filename, actor.PlayerSession, uid, audioParams);
 
         return null;
     }
 
-    public override (EntityUid Entity, AudioComponent Component)? PlayStatic(string? filename, ICommonSession recipient, EntityCoordinates coordinates, AudioParams? audioParams = null)
+    public override (EntityUid Entity, AudioComponent Component)? PlayStatic(ResolvedSoundSpecifier? filename, ICommonSession recipient, EntityCoordinates coordinates, AudioParams? audioParams = null)
     {
         return PlayStatic(filename, Filter.SinglePlayer(recipient), coordinates, false, audioParams);
     }
 
-    public override (EntityUid Entity, AudioComponent Component)? PlayStatic(string? filename, EntityUid recipient, EntityCoordinates coordinates, AudioParams? audioParams = null)
+    public override (EntityUid Entity, AudioComponent Component)? PlayStatic(ResolvedSoundSpecifier? filename, EntityUid recipient, EntityCoordinates coordinates, AudioParams? audioParams = null)
     {
         if (TryComp(recipient, out ActorComponent? actor))
             return PlayStatic(filename, actor.PlayerSession, coordinates, audioParams);

Robust.Shared/Audio/Components/CaptionComponent.cs

@@ -0,0 +1,30 @@
+using Robust.Shared.GameObjects;
+using Robust.Shared.Localization;
+using Robust.Shared.ViewVariables;
+using Robust.Shared.Physics;
+using Robust.Shared.ComponentTrees;
+using Robust.Shared.Serialization.Manager.Attributes;
+using Robust.Shared.GameStates;
+
+namespace Robust.Shared.Audio.Components;
+
+/// <summary>
+/// Stores the caption data for an audio entity.
+/// </summary>
+[RegisterComponent, NetworkedComponent, AutoGenerateComponentState(true)]
+public sealed partial class CaptionComponent : Component, IComponentTreeEntry<CaptionComponent>
+{
+    [DataField, AutoNetworkedField]
+    public LocId? Caption { get; set; }
+
+    [ViewVariables(VVAccess.ReadOnly)]
+    public string? LocalizedCaption => Caption != null ? Loc.GetString(Caption) : null;
+
+    public EntityUid? TreeUid { get; set; }
+
+    public DynamicTree<ComponentTreeEntry<CaptionComponent>>? Tree { get; set; }
+
+    public bool AddToTree => true;
+
+    public bool TreeUpdateQueued { get; set; }
+}

Robust.Shared/Audio/Components/CaptionTreeComponent.cs

@@ -0,0 +1,14 @@
+using Robust.Shared.Physics;
+using Robust.Shared.ComponentTrees;
+using Robust.Shared.GameObjects;
+
+namespace Robust.Shared.Audio.Components;
+
+/// <summary>
+/// Samples nearby <see cref="CaptionComponent"/>.
+/// </summary>
+[RegisterComponent]
+public sealed partial class CaptionTreeComponent : Component, IComponentTreeComponent<CaptionComponent>
+{
+    public DynamicTree<ComponentTreeEntry<CaptionComponent>> Tree { get; set; } = default!;
+}

Robust.Shared/Audio/ResolvedSoundSpecifier.cs

@@ -0,0 +1,90 @@
+using System;
+using JetBrains.Annotations;
+using Robust.Shared.Utility;
+using Robust.Shared.Serialization;
+using Robust.Shared.Serialization.Manager.Attributes;
+using Robust.Shared.Prototypes;
+
+namespace Robust.Shared.Audio;
+
+/// <summary>
+/// Represents a path to a sound resource, either as a literal path or as a collection ID and index.
+/// </summary>
+/// <seealso cref="ResolvedPathSpecifier"/>
+/// <seealso cref="ResolvedCollectionSpecifier"/>
+[Serializable, NetSerializable]
+public abstract partial class ResolvedSoundSpecifier {
+    [Obsolete("String literals for sounds are deprecated, use a SoundSpecifier or ResolvedSoundSpecifier as appropriate instead")]
+    public static implicit operator ResolvedSoundSpecifier(string s) => new ResolvedPathSpecifier(s);
+    [Obsolete("String literals for sounds are deprecated, use a SoundSpecifier or ResolvedSoundSpecifier as appropriate instead")]
+    public static implicit operator ResolvedSoundSpecifier(ResPath s) => new ResolvedPathSpecifier(s);
+
+    /// <summary>
+    /// Returns whether <c>s</c> is null, or if it contains an empty path/collection ID.
+    /// </summary>
+    public static bool IsNullOrEmpty(ResolvedSoundSpecifier? s) {
+        return s switch {
+            null => true,
+            ResolvedPathSpecifier path => path.Path.ToString() == "",
+            ResolvedCollectionSpecifier collection => string.IsNullOrEmpty(collection.Collection),
+            _ => throw new ArgumentOutOfRangeException("s", s, "argument is not a ResolvedPathSpecifier or a ResolvedCollectionSpecifier"),
+        };
+    }
+}
+
+/// <summary>
+/// Represents a path to a sound resource as a literal path.
+/// </summary>
+/// <seealso cref="ResolvedCollectionSpecifier"/>
+[Serializable, NetSerializable]
+public sealed partial class ResolvedPathSpecifier : ResolvedSoundSpecifier {
+    /// <summary>
+    /// The resource path of the sound.
+    /// </summary>
+    public ResPath Path { get; private set; }
+
+    override public string ToString() =>
+        $"ResolvedPathSpecifier({Path})";
+
+    [UsedImplicitly]
+    private ResolvedPathSpecifier()
+    {
+    }
+    public ResolvedPathSpecifier(ResPath path)
+    {
+        Path = path;
+    }
+    public ResolvedPathSpecifier(string path) : this(new ResPath(path))
+    {
+    }
+}
+
+/// <summary>
+/// Represents a path to a sound resource as a collection ID and index.
+/// </summary>
+/// <seealso cref="ResolvedPathSpecifier"/>
+[Serializable, NetSerializable]
+public sealed partial class ResolvedCollectionSpecifier : ResolvedSoundSpecifier {
+    /// <summary>
+    /// The ID of the <see cref="SoundCollectionPrototype">sound collection</see> to look up.
+    /// </summary>
+    public ProtoId<SoundCollectionPrototype>? Collection { get; private set; }
+    /// <summary>
+    /// The index of the file in the associated sound collection to play.
+    /// </summary>
+    public int Index { get; private set; }
+
+    override public string ToString() =>
+        $"ResolvedCollectionSpecifier({Collection}, {Index})";
+
+    [UsedImplicitly]
+    private ResolvedCollectionSpecifier()
+    {
+    }
+
+    public ResolvedCollectionSpecifier(string collection, int index)
+    {
+        Collection = collection;
+        Index = index;
+    }
+}

Robust.Shared/Audio/SoundCollectionPrototype.cs

@@ -3,6 +3,7 @@
 using Robust.Shared.Serialization.Manager.Attributes;
 using Robust.Shared.Utility;
 using Robust.Shared.ViewVariables;
+using Robust.Shared.Localization;
 
 namespace Robust.Shared.Audio;
 
@@ -13,6 +14,9 @@ public sealed partial class SoundCollectionPrototype : IPrototype
     [IdDataField]
     public string ID { get; private set; } = default!;
 
+    [DataField]
+    public LocId? Caption { get; private set; }
+
     [DataField("files")]
     public List<ResPath> PickFiles { get; private set; } = new();
 }