Augment audio system with the ability to have captions

Author: sowelipililimute

Robust.Client/Audio/AudioSystem.cs

@@ -43,6 +43,10 @@ public sealed partial class AudioSystem : SharedAudioSystem
     [Dependency] private readonly SharedMapSystem _maps = default!;
     [Dependency] private readonly SharedTransformSystem _xformSys = default!;
     [Dependency] private readonly SharedPhysicsSystem _physics = default!;
+    [Dependency] private readonly IEntityManager _entityManager = default!;
+
+    public event Action<CaptionComponent>? OnSubtitledAudioStart;
+    public event Action<CaptionComponent>? OnSubtitledAudioEnd;
 
     /// <summary>
     /// Per-tick cache of relevant streams.
@@ -172,17 +176,23 @@ public void SetMasterVolume(float value)
 
     private void OnAudioPaused(EntityUid uid, AudioComponent component, ref EntityPausedEvent args)
     {
+        if (_entityManager.TryGetComponent(uid, out CaptionComponent? caption))
+            OnSubtitledAudioEnd?.Invoke(caption);
         component.Pause();
     }
 
     protected override void OnAudioUnpaused(EntityUid uid, AudioComponent component, ref EntityUnpausedEvent args)
     {
+        if (_entityManager.TryGetComponent(uid, out CaptionComponent? caption))
+            OnSubtitledAudioStart?.Invoke(caption);
         base.OnAudioUnpaused(uid, component, ref args);
         component.StartPlaying();
     }
 
     private void OnAudioStartup(EntityUid uid, AudioComponent component, ComponentStartup args)
     {
+        if (_entityManager.TryGetComponent(uid, out CaptionComponent? caption))
+            OnSubtitledAudioStart?.Invoke(caption);
         if (!Timing.ApplyingState && !Timing.IsFirstTimePredicted)
         {
             return;
@@ -247,6 +257,8 @@ private void OnAudioShutdown(EntityUid uid, AudioComponent component, ComponentS
         component.Source.Dispose();
 
         RemoveAudioLimit(component.FileName);
+        if (_entityManager.TryGetComponent(uid, out CaptionComponent? caption))
+            OnSubtitledAudioEnd?.Invoke(caption);
     }
 
     private void OnAudioAttenuation(int obj)

Robust.Client/ComponentTrees/CaptionTreeSystem.cs

@@ -0,0 +1,46 @@
+using System.Numerics;
+using Robust.Client.GameObjects;
+using Robust.Shared.ComponentTrees;
+using Robust.Shared.GameObjects;
+using Robust.Shared.Maths;
+using Robust.Shared.Physics;
+using Robust.Shared.Audio.Components;
+using Robust.Shared.IoC;
+
+namespace Robust.Client.ComponentTrees;
+
+public sealed class CaptionTreeSystem : ComponentTreeSystem<CaptionTreeComponent, CaptionComponent>
+{
+    [Dependency] private readonly IEntityManager _entityManager = default!;
+
+    #region Component Tree Overrides
+    protected override bool DoFrameUpdate => false;
+    protected override bool DoTickUpdate => true;
+    protected override bool Recursive => true;
+    protected override int InitialCapacity => 128;
+
+    protected override Box2 ExtractAabb(in ComponentTreeEntry<CaptionComponent> entry, Vector2 pos, Angle rot)
+    {
+        if (_entityManager.TryGetComponent(entry.Uid, out AudioComponent? audio))
+        {
+            var radius = audio.MaxDistance;
+            var radiusVec = new Vector2(radius, radius);
+            return new Box2(pos - radiusVec, pos + radiusVec);
+        }
+        return default;
+    }
+
+    protected override Box2 ExtractAabb(in ComponentTreeEntry<CaptionComponent> entry)
+    {
+        if (entry.Component.TreeUid == null)
+            return default;
+
+        var pos = XformSystem.GetRelativePosition(
+            entry.Transform,
+            entry.Component.TreeUid.Value,
+            GetEntityQuery<TransformComponent>());
+
+        return ExtractAabb(in entry, pos, default);
+    }
+    #endregion
+}

Robust.Shared/Audio/Components/CaptionComponent.cs

@@ -0,0 +1,30 @@
+using Robust.Shared.GameObjects;
+using Robust.Shared.Localization;
+using Robust.Shared.ViewVariables;
+using Robust.Shared.Physics;
+using Robust.Shared.ComponentTrees;
+using Robust.Shared.Serialization.Manager.Attributes;
+using Robust.Shared.GameStates;
+
+namespace Robust.Shared.Audio.Components;
+
+/// <summary>
+/// Stores the caption data for an audio entity.
+/// </summary>
+[RegisterComponent, NetworkedComponent, AutoGenerateComponentState(true)]
+public sealed partial class CaptionComponent : Component, IComponentTreeEntry<CaptionComponent>
+{
+    [DataField, AutoNetworkedField]
+    public LocId? Caption { get; set; }
+
+    [ViewVariables(VVAccess.ReadOnly)]
+    public string? LocalizedCaption => Caption != null ? Loc.GetString(Caption) : null;
+
+    public EntityUid? TreeUid { get; set; }
+
+    public DynamicTree<ComponentTreeEntry<CaptionComponent>>? Tree { get; set; }
+
+    public bool AddToTree => true;
+
+    public bool TreeUpdateQueued { get; set; }
+}

Robust.Shared/Audio/Components/CaptionTreeComponent.cs

@@ -0,0 +1,14 @@
+using Robust.Shared.Physics;
+using Robust.Shared.ComponentTrees;
+using Robust.Shared.GameObjects;
+
+namespace Robust.Shared.Audio.Components;
+
+/// <summary>
+/// Samples nearby <see cref="CaptionComponent"/>.
+/// </summary>
+[RegisterComponent]
+public sealed partial class CaptionTreeComponent : Component, IComponentTreeComponent<CaptionComponent>
+{
+    public DynamicTree<ComponentTreeEntry<CaptionComponent>> Tree { get; set; } = default!;
+}

Robust.Shared/Audio/SoundCollectionPrototype.cs

@@ -3,6 +3,7 @@
 using Robust.Shared.Serialization.Manager.Attributes;
 using Robust.Shared.Utility;
 using Robust.Shared.ViewVariables;
+using Robust.Shared.Localization;
 
 namespace Robust.Shared.Audio;
 
@@ -13,6 +14,9 @@ public sealed partial class SoundCollectionPrototype : IPrototype
     [IdDataField]
     public string ID { get; private set; } = default!;
 
+    [DataField]
+    public LocId? Caption { get; private set; }
+
     [DataField("files")]
     public List<ResPath> PickFiles { get; private set; } = new();
 }

Robust.Shared/Audio/Systems/SharedAudioSystem.cs

@@ -332,6 +332,15 @@ protected Entity<AudioComponent> SetupAudio(ResolvedSoundSpecifier? specifier, A
         comp.FileName = fileName ?? string.Empty;
         comp.Params = audioParams.Value;
         comp.AudioStart = Timing.CurTime;
+        if (specifier is ResolvedCollectionSpecifier collection && collection.Collection is string collectionID)
+        {
+            var caption = ProtoMan.Index<SoundCollectionPrototype>(collectionID).Caption;
+            if (caption is not null)
+            {
+                var capt = AddComp<CaptionComponent>(uid);
+                capt.Caption = caption;
+            }
+        }
 
         if (!audioParams.Value.Loop)
         {