Allow recording and saving audio (#1836)

* move MediaUri into a media folder

* handle saving files locally

* store recordings as a file with a generated name

* prevent reporting an infinite duration which can crash the browser due to how the slider works

* generate a filename based on the time and guess the extension by mimetype

* save files locally using harmony resources

* show an error for files which are too big

* handle file saving and upload to lexbox

* expose a media files gql endpoint

* attempt to upload pending media files on sync

* setup permission manager for android blazor web client

Author: hahn-kev

backend/FwHeadless/LexboxFwDataMediaAdapter.cs

@@ -4,7 +4,9 @@
 using LexCore.Exceptions;
 using Microsoft.Extensions.Options;
 using MiniLcm;
+using MiniLcm.Media;
 using SIL.LCModel;
+using MediaFile = LexCore.Entities.MediaFile;
 
 namespace FwHeadless;
 
@@ -13,17 +15,17 @@ public class LexboxFwDataMediaAdapter(IOptions<FwHeadlessConfig> config, MediaFi
 {
     public MediaUri MediaUriFromPath(string path, LcmCache cache)
     {
-        var fullPath = Path.Join(cache.LangProject.LinkedFilesRootDir, path);
-        if (!File.Exists(fullPath)) return MediaUri.NotFound;
-        return MediaUriForMediaFile(mediaFileService.FindMediaFile(config.Value.LexboxProjectId(cache), fullPath));
+        if (!Path.IsPathRooted(path)) throw new ArgumentException("Path must be absolute, " + path, nameof(path));
+        if (!File.Exists(path)) return MediaUri.NotFound;
+        return MediaUriForMediaFile(mediaFileService.FindMediaFile(config.Value.LexboxProjectId(cache), path));
     }
 
-    public string PathFromMediaUri(MediaUri mediaUri, LcmCache cache)
+    public string? PathFromMediaUri(MediaUri mediaUri, LcmCache cache)
     {
-        var mediaFile = mediaFileService.FindMediaFile(mediaUri.FileId) ??
-                        throw new NotFoundException($"Unable to find file {mediaUri.FileId}.", nameof(MediaFile));
+        var mediaFile = mediaFileService.FindMediaFile(mediaUri.FileId);
+        if (mediaFile is null) return null;
         var fullFilePath = Path.Join(cache.ProjectId.ProjectFolder, mediaFile.Filename);
-        return Path.GetRelativePath(cache.LangProject.LinkedFilesRootDir, fullFilePath);
+        return fullFilePath;
     }
 
     private MediaUri MediaUriForMediaFile(MediaFile mediaFile)

backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/FwDataTestsKernel.cs

@@ -11,6 +11,8 @@ public static IServiceCollection AddTestFwDataBridge(this IServiceCollection ser
     {
         services.AddFwDataBridge();
         services.TryAddSingleton<IConfiguration>(_ => new ConfigurationRoot([]));
+        //this path is typically not used for projects (they're in memory) but it is used for media
+        services.Configure<FwDataBridgeConfig>(config => config.ProjectsFolder = Path.GetFullPath(Path.Combine(".", "fw-test-projects")));
         if (mockProjectLoader)
         {
             services.AddSingleton<MockFwProjectLoader>();

backend/FwLite/FwDataMiniLcmBridge.Tests/Fixtures/ProjectLoaderFixture.cs

@@ -11,11 +11,14 @@ public class ProjectLoaderFixture : IDisposable
     private readonly ServiceProvider _serviceProvider;
     private readonly IOptions<FwDataBridgeConfig> _config;
     public MockFwProjectLoader MockFwProjectLoader { get; }
+    public IServiceProvider Services => _serviceProvider;
 
     public ProjectLoaderFixture()
     {
         //todo make mock of IProjectLoader so we can load from test projects
-        var provider = new ServiceCollection().AddTestFwDataBridge().BuildServiceProvider();
+        var provider = new ServiceCollection()
+            .AddTestFwDataBridge()
+            .BuildServiceProvider();
         _serviceProvider = provider;
         _fwDataFactory = provider.GetRequiredService<FwDataFactory>();
         MockFwProjectLoader = provider.GetRequiredService<MockFwProjectLoader>();

backend/FwLite/FwDataMiniLcmBridge.Tests/MediaFileTests.cs

@@ -1,6 +1,8 @@
 using FwDataMiniLcmBridge.Api;
 using FwDataMiniLcmBridge.Media;
 using FwDataMiniLcmBridge.Tests.Fixtures;
+using Microsoft.Extensions.DependencyInjection;
+using MiniLcm.Media;
 using MiniLcm.Models;
 using SIL.LCModel.Infrastructure;
 
@@ -11,9 +13,11 @@ public class MediaFileTests : IAsyncLifetime
 {
     private readonly FwDataMiniLcmApi _api;
     private readonly WritingSystemId _audioWs = "en-Zxxx-x-audio";
+    private IMediaAdapter _mediaAdapter;
 
     public MediaFileTests(ProjectLoaderFixture fixture)
     {
+        _mediaAdapter = fixture.Services.GetRequiredService<IMediaAdapter>();
         _api = fixture.NewProjectApi("media-file-test", "en", "en");
     }
 
@@ -60,10 +64,10 @@ private async Task<Guid> AddFileDirectly(string fileName, string? contents, bool
 
     private async Task<Guid> StoreFileContentsAsync(string fileName, string? contents)
     {
-        var fwFilePath = Path.Combine(FwDataMiniLcmApi.AudioVisualFolder, fileName);
-        var filePath = Path.Combine(_api.Cache.LangProject.LinkedFilesRootDir, fwFilePath);
+        var filePath = Path.Combine(_api.Cache.LangProject.LinkedFilesRootDir, FwDataMiniLcmApi.AudioVisualFolder, fileName);
         await File.WriteAllTextAsync(filePath, contents);
-        return LocalMediaAdapter.NewGuidV5(fwFilePath);
+        //using media adapter to ensure it's cache is updated with the new file
+        return _mediaAdapter.MediaUriFromPath(filePath, _api.Cache).FileId;
     }
 
     private string GetFwAudioValue(Guid id)
@@ -78,7 +82,7 @@ private string GetFwAudioValue(Guid id)
     public async Task GetEntry_MapsFilePathsFromAudioWs()
     {
         var fileName = "MapsAFileReferenceIntoAMediaUri.txt";
-        var fileGuid = LocalMediaAdapter.NewGuidV5(Path.Combine(FwDataMiniLcmApi.AudioVisualFolder, fileName));
+        var fileGuid = LocalMediaAdapter.NewGuidV5(Path.Combine(_api.Cache.LangProject.LinkedFilesRootDir, FwDataMiniLcmApi.AudioVisualFolder, fileName));
         var entryId = await AddFileDirectly(fileName, "test");
 
         var entry = await _api.GetEntry(entryId);

backend/FwLite/FwDataMiniLcmBridge.Tests/MiniLcmTests/MediaTests.cs

@@ -0,0 +1,34 @@
+using FwDataMiniLcmBridge.Api;
+using FwDataMiniLcmBridge.Tests.Fixtures;
+
+namespace FwDataMiniLcmBridge.Tests.MiniLcmTests;
+
+[Collection(ProjectLoaderFixture.Name)]
+public class MediaTests : MediaTestsBase
+{
+    private readonly ProjectLoaderFixture _fixture;
+
+    public MediaTests(ProjectLoaderFixture fixture)
+    {
+        _fixture = fixture;
+    }
+
+    protected override Task<IMiniLcmApi> NewApi()
+    {
+        return Task.FromResult<IMiniLcmApi>(_fixture.NewProjectApi("media-test", "en", "en"));
+    }
+
+    public override async Task InitializeAsync()
+    {
+        await base.InitializeAsync();
+        var projectFolder = ((FwDataMiniLcmApi)Api).Cache.LangProject.LinkedFilesRootDir;
+        Directory.CreateDirectory(projectFolder);
+    }
+
+    public override async Task DisposeAsync()
+    {
+        var projectFolder = ((FwDataMiniLcmApi)Api).Cache.ProjectId.ProjectFolder;
+        if (Directory.Exists(projectFolder)) Directory.Delete(projectFolder, true);
+        await base.DisposeAsync();
+    }
+}

backend/FwLite/FwDataMiniLcmBridge/Api/FwDataMiniLcmApi.cs

@@ -9,6 +9,7 @@
 using Microsoft.Extensions.Options;
 using MiniLcm;
 using MiniLcm.Exceptions;
+using MiniLcm.Media;
 using MiniLcm.Models;
 using MiniLcm.SyncHelpers;
 using MiniLcm.Validators;
@@ -718,14 +719,17 @@ private string ToMediaUri(string tsString)
         //rooted media paths aren't supported
         if (Path.IsPathRooted(tsString))
             throw new ArgumentException("Media path must be relative", nameof(tsString));
-        return mediaAdapter.MediaUriFromPath(Path.Combine(AudioVisualFolder, tsString), Cache).ToString();
+        var fullFilePath = Path.Join(Cache.LangProject.LinkedFilesRootDir, AudioVisualFolder, tsString);
+        return mediaAdapter.MediaUriFromPath(fullFilePath, Cache).ToString();
     }
 
-    internal string FromMediaUri(string mediaUri)
+    internal string FromMediaUri(string mediaUriString)
     {
         //path includes `AudioVisual` currently
-        var path = mediaAdapter.PathFromMediaUri(new MediaUri(mediaUri), Cache);
-        return Path.GetRelativePath(AudioVisualFolder, path);
+        MediaUri mediaUri = new MediaUri(mediaUriString);
+        var path = mediaAdapter.PathFromMediaUri(mediaUri, Cache);
+        if (path is null) throw new NotFoundException($"Unable to find file {mediaUri.FileId}.", nameof(MediaFile));
+        return Path.GetRelativePath(Path.Join(Cache.LangProject.LinkedFilesRootDir, AudioVisualFolder), path);
     }
 
     internal RichString? ToRichString(ITsString? tsString)
@@ -817,7 +821,7 @@ public IAsyncEnumerable<Entry> SearchEntries(string query, QueryOptions? options
     {
         if (string.IsNullOrEmpty(query)) return null;
         return entry => entry.CitationForm.SearchValue(query) ||
-                        entry.LexemeFormOA.Form.SearchValue(query) ||
+                        entry.LexemeFormOA?.Form.SearchValue(query) is true ||
                         entry.AllSenses.Any(s => s.Gloss.SearchValue(query));
     }
 
@@ -1536,8 +1540,55 @@ private static void ValidateOwnership(ILexExampleSentence lexExampleSentence, Gu
     public Task<ReadFileResponse> GetFileStream(MediaUri mediaUri)
     {
         if (mediaUri == MediaUri.NotFound) return Task.FromResult(new ReadFileResponse(ReadFileResult.NotFound));
-        string fullPath = Path.Combine(Cache.LangProject.LinkedFilesRootDir, mediaAdapter.PathFromMediaUri(mediaUri, Cache));
+        var pathFromMediaUri = mediaAdapter.PathFromMediaUri(mediaUri, Cache);
+        if (pathFromMediaUri is not {Length: > 0}) return Task.FromResult(new ReadFileResponse(ReadFileResult.NotFound));
+        string fullPath = Path.Combine(Cache.LangProject.LinkedFilesRootDir, pathFromMediaUri);
         if (!File.Exists(fullPath)) return Task.FromResult(new ReadFileResponse(ReadFileResult.NotFound));
         return Task.FromResult(new ReadFileResponse(File.OpenRead(fullPath), Path.GetFileName(fullPath)));
     }
+
+    public async Task<UploadFileResponse> SaveFile(Stream stream, LcmFileMetadata metadata)
+    {
+        if (stream.SafeLength() > MediaFile.MaxFileSize) return new UploadFileResponse(UploadFileResult.TooBig);
+        var fullPath = Path.Combine(Cache.LangProject.LinkedFilesRootDir, TypeToLinkedFolder(metadata.MimeType), Path.GetFileName(metadata.Filename));
+
+        if (File.Exists(fullPath))
+            return new UploadFileResponse(mediaAdapter.MediaUriFromPath(fullPath, Cache), savedToLexbox: false, newResource: false);
+        var directory = Path.GetDirectoryName(fullPath);
+        if (directory is not null)
+        {
+            try
+            {
+                Directory.CreateDirectory(directory);
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Failed to create directory {Directory} for file {Filename}", directory, metadata.Filename);
+                return new UploadFileResponse($"Failed to create directory: {ex.Message}");
+            }
+        }
+
+        try
+        {
+            await using var fileStream = File.Create(fullPath);
+            await stream.CopyToAsync(fileStream);
+            return new UploadFileResponse(mediaAdapter.MediaUriFromPath(fullPath, Cache), savedToLexbox: false, newResource: true);
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Failed to save file {Filename} to {Path}", metadata.Filename, fullPath);
+            return new UploadFileResponse($"Failed to save file: {ex.Message}");
+        }
+    }
+
+    private string TypeToLinkedFolder(string mimeType)
+    {
+        return mimeType switch
+        {
+            { } s when s.StartsWith("audio/") => AudioVisualFolder,
+            { } s when s.StartsWith("video/") => AudioVisualFolder,
+            { } s when s.StartsWith("image/") => "Pictures",
+            _ => "Others"
+        };
+    }
 }

backend/FwLite/FwDataMiniLcmBridge/Api/UpdateProxy/UpdateDictionaryProxy.cs

@@ -1,6 +1,7 @@
 using System.Collections;
 using System.Diagnostics.CodeAnalysis;
 using MiniLcm;
+using MiniLcm.Media;
 using MiniLcm.Models;
 using SIL.LCModel.Core.KernelInterfaces;
 using SIL.LCModel.Core.Text;

backend/FwLite/FwDataMiniLcmBridge/LexEntryFilterMapProvider.cs

@@ -28,7 +28,8 @@ public class LexEntryFilterMapProvider : EntryFilterMapProvider<ILexEntry>
     public override Expression<Func<ILexEntry, string, object?>> EntrySensesGloss => (entry, ws) => entry.AllSenses.Select(s => s.PickText(s.Gloss, ws));
     public override Expression<Func<ILexEntry, string, object?>> EntrySensesDefinition => (entry, ws) => entry.AllSenses.Select(s => s.PickText(s.Definition, ws));
     public override Expression<Func<ILexEntry, string, object?>> EntryNote => (entry, ws) => entry.PickText(entry.Comment, ws);
-    public override Expression<Func<ILexEntry, string, object?>> EntryLexemeForm => (entry, ws) => entry.PickText(entry.LexemeFormOA.Form, ws);
+    public override Expression<Func<ILexEntry, string, object?>> EntryLexemeForm => (entry, ws) =>
+        entry.LexemeFormOA == null ? null : entry.PickText(entry.LexemeFormOA.Form, ws);
     public override Expression<Func<ILexEntry, string, object?>> EntryCitationForm => (entry, ws) => entry.PickText(entry.CitationForm, ws);
     public override Expression<Func<ILexEntry, string, object?>> EntryLiteralMeaning => (entry, ws) => entry.PickText(entry.LiteralMeaning, ws);
     public override Expression<Func<ILexEntry, object?>> EntryComplexFormTypes => e => EmptyToNull(e.ComplexFormEntryRefs.SelectMany(r => r.ComplexEntryTypesRS));

backend/FwLite/FwDataMiniLcmBridge/Media/IMediaAdapter.cs

@@ -1,4 +1,5 @@
 using MiniLcm;
+using MiniLcm.Media;
 using SIL.LCModel;
 
 namespace FwDataMiniLcmBridge.Media;
@@ -8,7 +9,7 @@ public interface IMediaAdapter
     /// <summary>
     /// get the MediaUri representing a file, can be used later to get the path back
     /// </summary>
-    /// <param name="path">the path relative to LinkedFiles to find the file at</param>
+    /// <param name="path">the full file path must be inside the project LinkedFiles directory</param>
     /// <param name="cache">the current project</param>
     /// <returns>a media uri which can later be used to get the path</returns>
     MediaUri MediaUriFromPath(string path, LcmCache cache);
@@ -17,6 +18,6 @@ public interface IMediaAdapter
     /// </summary>
     /// <param name="mediaUri"></param>
     /// <param name="cache"></param>
-    /// <returns>the path to the file represented by the mediaUri, relative to the LinkedFiles directory in the given project</returns>
-    string PathFromMediaUri(MediaUri mediaUri, LcmCache cache);
+    /// <returns>the full path to the file represented by the mediaUri, will return null when it can't find the file</returns>
+    string? PathFromMediaUri(MediaUri mediaUri, LcmCache cache);
 }

backend/FwLite/FwDataMiniLcmBridge/Media/LocalMediaAdapter.cs

@@ -2,6 +2,7 @@
 using Microsoft.Extensions.Caching.Memory;
 using MiniLcm;
 using MiniLcm.Exceptions;
+using MiniLcm.Media;
 using SIL.LCModel;
 using UUIDNext;
 
@@ -21,19 +22,38 @@ private Dictionary<Guid, string> Paths(LcmCache cache)
                 entry.SlidingExpiration = TimeSpan.FromMinutes(10);
                 return Directory
                     .EnumerateFiles(cache.LangProject.LinkedFilesRootDir, "*", SearchOption.AllDirectories)
-                    .Select(file => Path.GetRelativePath(cache.LangProject.LinkedFilesRootDir, file))
-                    .ToDictionary(file => MediaUriFromPath(file, cache).FileId, file => file);
+                    .ToDictionary(file => PathToUri(file).FileId, file => file);
             }) ?? throw new Exception("Failed to get paths");
     }
 
     //path is expected to be relative to the LinkedFilesRootDir
     public MediaUri MediaUriFromPath(string path, LcmCache cache)
     {
-        if (!File.Exists(Path.Combine(cache.LangProject.LinkedFilesRootDir, path))) return MediaUri.NotFound;
+        EnsureCorrectRootFolder(path, cache);
+        if (!File.Exists(path)) return MediaUri.NotFound;
+        var uri = PathToUri(path);
+        //this may be a new file, so we need to add it to the cache
+        Paths(cache)[uri.FileId] = path;
+        return uri;
+    }
+
+    private void EnsureCorrectRootFolder(string path, LcmCache cache)
+    {
+        if (Path.IsPathRooted(path))
+        {
+            if (path.StartsWith(cache.LangProject.LinkedFilesRootDir)) return;
+            throw new ArgumentException("Path must be in the LinkedFilesRootDir", nameof(path));
+        }
+
+        throw new ArgumentException("Path must be absolute, " + path, nameof(path));
+    }
+
+    private static MediaUri PathToUri(string path)
+    {
         return new MediaUri(NewGuidV5(path), LocalMediaAuthority);
     }
 
-    public string PathFromMediaUri(MediaUri mediaUri, LcmCache cache)
+    public string? PathFromMediaUri(MediaUri mediaUri, LcmCache cache)
     {
         var paths = Paths(cache);
         if (mediaUri.Authority != LocalMediaAuthority) throw new ArgumentException("MediaUri must be local", nameof(mediaUri));
@@ -42,7 +62,7 @@ public string PathFromMediaUri(MediaUri mediaUri, LcmCache cache)
             return path;
         }
 
-        throw new NotFoundException("Media not found: " + mediaUri.FileId, "MedaiUri");
+        return null;
     }
 
     // produces the same Guid for the same input name