Dropped support for FileService

GrillBot · Oct 25, 2023 · fe2142b · fe2142b
1 parent 419e72f
commit fe2142b
Show file tree

Hide file tree

Showing 14 changed files with 246 additions and 77 deletions.
diff --git a/src/GrillBot.App/Actions/Api/V1/AuditLog/GetAuditLogList.cs b/src/GrillBot.App/Actions/Api/V1/AuditLog/GetAuditLogList.cs
@@ -1,6 +1,8 @@
 using System.Text.Json;
 using AutoMapper;
+using GrillBot.App.Helpers;
 using GrillBot.Common.Extensions.Discord;
+using GrillBot.Common.FileStorage;
 using GrillBot.Common.Models;
 using GrillBot.Core.Extensions;
 using GrillBot.Core.Models.Pagination;
@@ -12,6 +14,7 @@
 using GrillBot.Data.Models.API.AuditLog.Preview;
 using GrillBot.Database.Services.Repository;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Azure;
 using File = GrillBot.Data.Models.API.AuditLog.File;
 using SearchModels = GrillBot.Core.Services.AuditLog.Models.Response.Search;
 
@@ -21,22 +24,25 @@ public class GetAuditLogList : ApiAction
 {
     private GrillBotDatabaseBuilder DatabaseBuilder { get; }
     private IMapper Mapper { get; }
-    private IFileServiceClient FileServiceClient { get; }
     private IAuditLogServiceClient AuditLogServiceClient { get; }
     private IDiscordClient DiscordClient { get; }
+    private BlobManagerFactoryHelper BlobManagerFactoryHelper { get; }
 
     private Dictionary<string, Database.Entity.User> CachedUsers { get; } = new();
     private Dictionary<string, Database.Entity.Guild> CachedGuilds { get; } = new();
     private Dictionary<string, Dictionary<string, Database.Entity.GuildChannel>> CachedChannels { get; } = new(); // Dictionary<GuildId, Dictionary<ChannelId, Channel>>
 
-    public GetAuditLogList(ApiRequestContext apiContext, GrillBotDatabaseBuilder databaseBuilder, IMapper mapper, IFileServiceClient fileServiceClient, IAuditLogServiceClient auditLogServiceClient,
-        IDiscordClient discordClient) : base(apiContext)
+    private BlobManager BlobManager { get; set; } = null!;
+    private BlobManager LegacyBlobManager { get; set; } = null!;
+
+    public GetAuditLogList(ApiRequestContext apiContext, GrillBotDatabaseBuilder databaseBuilder, IMapper mapper, IAuditLogServiceClient auditLogServiceClient, IDiscordClient discordClient,
+        BlobManagerFactoryHelper blobManagerFactoryHelper) : base(apiContext)
     {
         DatabaseBuilder = databaseBuilder;
         Mapper = mapper;
-        FileServiceClient = fileServiceClient;
         AuditLogServiceClient = auditLogServiceClient;
         DiscordClient = discordClient;
+        BlobManagerFactoryHelper = blobManagerFactoryHelper;
     }
 
     public async Task<PaginatedResponse<LogListItem>> ProcessAsync(SearchRequest request)
@@ -47,6 +53,12 @@ public async Task<PaginatedResponse<LogListItem>> ProcessAsync(SearchRequest req
         if (response.ValidationErrors is not null)
             throw CreateValidationExceptions(response.ValidationErrors);
 
+        if (response.Response!.Data.Exists(o => o.Files.Count > 0))
+        {
+            BlobManager = await BlobManagerFactoryHelper.CreateAsync(BlobConstants.AuditLogDeletedAttachments);
+            LegacyBlobManager = await BlobManagerFactoryHelper.CreateAsync("production");
+        }
+
         await using var repository = DatabaseBuilder.CreateRepository();
         return await PaginatedResponse<LogListItem>.CopyAndMapAsync(response.Response!, async entity => await MapListItemAsync(repository, entity));
     }
@@ -89,12 +101,10 @@ private async Task<LogListItem> MapListItemAsync(GrillBotRepository repository,
             Type = item.Type,
             CreatedAt = item.CreatedAt.ToLocalTime(),
             IsDetailAvailable = item.IsDetailAvailable,
-            Id = item.Id
+            Id = item.Id,
+            Files = item.Files.ConvertAll(o => ConvertFile(o, item))
         };
 
-        foreach (var file in item.Files)
-            result.Files.Add(await ConvertFileAsync(file));
-
         if (!string.IsNullOrEmpty(item.GuildId))
         {
             var guild = await ResolveGuildAsync(repository, item.GuildId);
@@ -252,14 +262,17 @@ private async Task<LogListItem> MapListItemAsync(GrillBotRepository repository,
         return null;
     }
 
-    private async Task<File> ConvertFileAsync(SearchModels.File file)
+    private File ConvertFile(SearchModels.File file, SearchModels.LogListItem item)
     {
-        var link = await FileServiceClient.GenerateLinkAsync(file.Filename);
+        // TODO Hack until all old files has been deleted.
+        var migrationDate = new DateTime(2023, 10, 25, 12, 00, 00, DateTimeKind.Utc);
+        var usedManager = item.CreatedAt >= migrationDate ? BlobManager : LegacyBlobManager;
+        var link = usedManager.GenerateSasLink(file.Filename, 1);
 
         return new File
         {
             Filename = file.Filename,
-            Link = link!,
+            Link = link ?? "about:blank",
             Size = file.Size
         };
     }

diff --git a/src/GrillBot.App/Actions/Api/V1/AuditLog/RemoveItem.cs b/src/GrillBot.App/Actions/Api/V1/AuditLog/RemoveItem.cs
@@ -1,22 +1,23 @@
-using GrillBot.Common.Managers.Localization;
+using GrillBot.App.Helpers;
+using GrillBot.Common.FileStorage;
+using GrillBot.Common.Managers.Localization;
 using GrillBot.Common.Models;
 using GrillBot.Core.Exceptions;
 using GrillBot.Core.Services.AuditLog;
-using GrillBot.Core.Services.FileService;
 
 namespace GrillBot.App.Actions.Api.V1.AuditLog;
 
 public class RemoveItem : ApiAction
 {
     private ITextsManager Texts { get; }
-    private IFileServiceClient FileServiceClient { get; }
     private IAuditLogServiceClient AuditLogServiceClient { get; }
+    private BlobManagerFactoryHelper BlobManagerFactoryHelper { get; }
 
-    public RemoveItem(ApiRequestContext apiContext, ITextsManager texts, IFileServiceClient fileServiceClient, IAuditLogServiceClient auditLogServiceClient) : base(apiContext)
+    public RemoveItem(ApiRequestContext apiContext, ITextsManager texts, IAuditLogServiceClient auditLogServiceClient, BlobManagerFactoryHelper blobManagerFactoryHelper) : base(apiContext)
     {
         Texts = texts;
-        FileServiceClient = fileServiceClient;
         AuditLogServiceClient = auditLogServiceClient;
+        BlobManagerFactoryHelper = blobManagerFactoryHelper;
     }
 
     public async Task ProcessAsync(Guid id)
@@ -25,7 +26,16 @@ public async Task ProcessAsync(Guid id)
         if (!response.Exists)
             throw new NotFoundException(Texts["AuditLog/RemoveItem/NotFound", ApiContext.Language]);
 
-        foreach (var file in response.FilesToDelete)
-            await FileServiceClient.DeleteFileAsync(file);
+        if (response.FilesToDelete.Count == 0)
+            return;
+
+        var manager = await BlobManagerFactoryHelper.CreateAsync(BlobConstants.AuditLogDeletedAttachments);
+        var legacyManager = await BlobManagerFactoryHelper.CreateAsync("production");
+
+        foreach (var filename in response.FilesToDelete)
+        {
+            await manager.DeleteAsync(filename);
+            await legacyManager.DeleteAsync(filename);
+        }
     }
 }
diff --git a/src/GrillBot.App/Actions/Api/V1/Dashboard/GetServicesList.cs b/src/GrillBot.App/Actions/Api/V1/Dashboard/GetServicesList.cs
@@ -2,7 +2,6 @@
 using GrillBot.Common.Models;
 using GrillBot.Core.Services.AuditLog;
 using GrillBot.Core.Services.Common;
-using GrillBot.Core.Services.FileService;
 using GrillBot.Core.Services.Graphics;
 using GrillBot.Core.Services.ImageProcessing;
 using GrillBot.Core.Services.PointsService;
@@ -15,20 +14,18 @@ public class GetServicesList : ApiAction
 {
     private IGraphicsClient GraphicsClient { get; }
     private IRubbergodServiceClient RubbergodServiceClient { get; }
-    private IFileServiceClient FileServiceClient { get; }
     private IPointsServiceClient PointsServiceClient { get; }
     private IImageProcessingClient ImageProcessingClient { get; }
     private IAuditLogServiceClient AuditLogServiceClient { get; }
     private LoggingManager LoggingManager { get; }
 
     private List<Exception> Errors { get; } = new();
 
-    public GetServicesList(ApiRequestContext apiContext, LoggingManager logging, IGraphicsClient graphicsClient, IRubbergodServiceClient rubbergodServiceClient, IFileServiceClient fileServiceClient,
-        IPointsServiceClient pointsServiceClient, IImageProcessingClient imageProcessingClient, IAuditLogServiceClient auditLogServiceClient) : base(apiContext)
+    public GetServicesList(ApiRequestContext apiContext, LoggingManager logging, IGraphicsClient graphicsClient, IRubbergodServiceClient rubbergodServiceClient, IPointsServiceClient pointsServiceClient,
+        IImageProcessingClient imageProcessingClient, IAuditLogServiceClient auditLogServiceClient) : base(apiContext)
     {
         GraphicsClient = graphicsClient;
         RubbergodServiceClient = rubbergodServiceClient;
-        FileServiceClient = fileServiceClient;
         ImageProcessingClient = imageProcessingClient;
         PointsServiceClient = pointsServiceClient;
         AuditLogServiceClient = auditLogServiceClient;
@@ -41,7 +38,6 @@ public async Task<List<DashboardService>> ProcessAsync()
 
         await AddServiceStatusAsync(services, "graphics", GraphicsClient);
         await AddServiceStatusAsync(services, "rubbergod", RubbergodServiceClient);
-        await AddServiceStatusAsync(services, "file", FileServiceClient);
         await AddServiceStatusAsync(services, "points", PointsServiceClient);
         await AddServiceStatusAsync(services, "image-processing", ImageProcessingClient);
         await AddServiceStatusAsync(services, "audit-log", AuditLogServiceClient);

diff --git a/src/GrillBot.App/Actions/Api/V1/Services/GetServiceInfo.cs b/src/GrillBot.App/Actions/Api/V1/Services/GetServiceInfo.cs
@@ -16,18 +16,16 @@ public class GetServiceInfo : ApiAction
 {
     private IGraphicsClient GraphicsClient { get; }
     private IRubbergodServiceClient RubbergodServiceClient { get; }
-    private IFileServiceClient FileServiceClient { get; }
     private LoggingManager LoggingManager { get; }
     private IPointsServiceClient PointsServiceClient { get; }
     private IImageProcessingClient ImageProcessingClient { get; }
     private IAuditLogServiceClient AuditLogServiceClient { get; }
 
-    public GetServiceInfo(ApiRequestContext apiContext, IGraphicsClient graphicsClient, IRubbergodServiceClient rubbergodServiceClient, IFileServiceClient fileServiceClient,
-        LoggingManager loggingManager, IPointsServiceClient pointsServiceClient, IImageProcessingClient imageProcessingClient, IAuditLogServiceClient auditLogServiceClient) : base(apiContext)
+    public GetServiceInfo(ApiRequestContext apiContext, IGraphicsClient graphicsClient, IRubbergodServiceClient rubbergodServiceClient, LoggingManager loggingManager, IPointsServiceClient pointsServiceClient,
+        IImageProcessingClient imageProcessingClient, IAuditLogServiceClient auditLogServiceClient) : base(apiContext)
     {
         GraphicsClient = graphicsClient;
         RubbergodServiceClient = rubbergodServiceClient;
-        FileServiceClient = fileServiceClient;
         LoggingManager = loggingManager;
         PointsServiceClient = pointsServiceClient;
         ImageProcessingClient = imageProcessingClient;
@@ -53,7 +51,6 @@ private IClient PickClient(string id)
         return id switch
         {
             "rubbergod" => RubbergodServiceClient,
-            "file" => FileServiceClient,
             "graphics" => GraphicsClient,
             "points" => PointsServiceClient,
             "image-processing" => ImageProcessingClient,
@@ -70,7 +67,6 @@ private async Task SetDiagnosticsInfo(ServiceInfo info, IClient client)
             {
                 IRubbergodServiceClient => await RubbergodServiceClient.GetDiagAsync(),
                 IGraphicsClient => await GetGraphicsServiceInfo(),
-                IFileServiceClient => await FileServiceClient.GetDiagAsync(),
                 IPointsServiceClient => await PointsServiceClient.GetDiagAsync(),
                 IImageProcessingClient => await ImageProcessingClient.GetDiagAsync(),
                 IAuditLogServiceClient => await AuditLogServiceClient.GetDiagAsync(),

diff --git a/src/GrillBot.App/Handlers/MessageDeleted/AuditMessageDeletedHandler.cs b/src/GrillBot.App/Handlers/MessageDeleted/AuditMessageDeletedHandler.cs
@@ -3,26 +3,26 @@
 using GrillBot.Common.Managers.Events.Contracts;
 using GrillBot.Core.Services.AuditLog;
 using GrillBot.Core.Services.AuditLog.Enums;
-using GrillBot.Core.Services.AuditLog.Models;
 using GrillBot.Core.Services.AuditLog.Models.Request.CreateItems;
-using GrillBot.Core.Services.FileService;
 using GrillBot.Core.Extensions;
-using Microsoft.AspNetCore.StaticFiles;
+using GrillBot.Common.FileStorage;
+using Azure.Storage.Blobs.Models;
+using Azure;
 
 namespace GrillBot.App.Handlers.MessageDeleted;
 
 public class AuditMessageDeletedHandler : AuditLogServiceHandler, IMessageDeletedEvent
 {
     private IMessageCacheManager MessageCache { get; }
     private DownloadHelper DownloadHelper { get; }
-    private IFileServiceClient FileServiceClient { get; }
+    private BlobManagerFactoryHelper BlobManagerFactoryHelper { get; }
 
-    public AuditMessageDeletedHandler(IMessageCacheManager messageCache, DownloadHelper downloadHelper, IFileServiceClient fileServiceClient, IAuditLogServiceClient auditLogServiceClient) :
+    public AuditMessageDeletedHandler(IMessageCacheManager messageCache, DownloadHelper downloadHelper, IAuditLogServiceClient auditLogServiceClient, BlobManagerFactoryHelper blobManagerFactoryHelper) :
         base(auditLogServiceClient)
     {
         MessageCache = messageCache;
         DownloadHelper = downloadHelper;
-        FileServiceClient = fileServiceClient;
+        BlobManagerFactoryHelper = blobManagerFactoryHelper;
     }
 
     public async Task ProcessAsync(Cacheable<IMessage, ulong> cachedMessage, Cacheable<IMessageChannel, ulong> cachedChannel)
@@ -52,11 +52,12 @@ private async Task<List<FileRequest>> GetAndStoreAttachmentsAsync(IMessage messa
         if (message.Attachments.Count == 0)
             return files;
 
-        var contentTypeProvider = new FileExtensionContentTypeProvider();
+        var manager = await BlobManagerFactoryHelper.CreateAsync(BlobConstants.AuditLogDeletedAttachments);
+
         foreach (var attachment in message.Attachments)
         {
             var content = await DownloadHelper.DownloadAsync(attachment);
-            if (content == null) continue;
+            if (content is null) continue;
 
             var extension = Path.GetExtension(attachment.Filename);
             var filenameWithoutExtension = Path.GetFileNameWithoutExtension(attachment.Filename).Cut(100, true);
@@ -67,10 +68,16 @@ private async Task<List<FileRequest>> GetAndStoreAttachmentsAsync(IMessage messa
                 Size = attachment.Size
             };
 
-            var contentType = contentTypeProvider.TryGetContentType(file.Filename, out var type) ? type : "application/octet-stream";
-            await FileServiceClient.UploadFileAsync(file.Filename, content, contentType);
-
             files.Add(file);
+
+            try
+            {
+                await manager.UploadAsync(file.Filename, content);
+            }
+            catch (RequestFailedException ex) when (ex.ErrorCode == BlobErrorCode.BlobAlreadyExists)
+            {
+                // Can ignore.
+            }
         }
 
         return files;

diff --git a/src/GrillBot.App/Helpers/BlobManagerFactoryHelper.cs b/src/GrillBot.App/Helpers/BlobManagerFactoryHelper.cs
@@ -0,0 +1,23 @@
+using GrillBot.Common.FileStorage;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.Hosting;
+
+namespace GrillBot.App.Helpers;
+
+public class BlobManagerFactoryHelper
+{
+    private BlobManagerFactory Factory { get; }
+    private IWebHostEnvironment Environment { get; }
+
+    public BlobManagerFactoryHelper(BlobManagerFactory blobManagerFactory, IWebHostEnvironment environment)
+    {
+        Factory = blobManagerFactory;
+        Environment = environment;
+    }
+
+    private string CreateShortcut()
+        => Environment.IsDevelopment() ? "dev" : "prod";
+
+    public async Task<BlobManager> CreateAsync(string containerName)
+        => await Factory.CreateAsync($"{containerName}-{CreateShortcut()}");
+}
diff --git a/src/GrillBot.App/Helpers/HelperExtensions.cs b/src/GrillBot.App/Helpers/HelperExtensions.cs
@@ -11,7 +11,8 @@ public static IServiceCollection AddHelpers(this IServiceCollection services)
             .AddScoped<DownloadHelper>()
             .AddScoped<ChannelHelper>()
             .AddScoped<UnverifyHelper>()
-            .AddScoped<EmoteSuggestionHelper>();
+            .AddScoped<EmoteSuggestionHelper>()
+            .AddSingleton<BlobManagerFactoryHelper>();
 
         return services;
     }

diff --git a/src/GrillBot.App/Jobs/AuditLogClearingJob.cs b/src/GrillBot.App/Jobs/AuditLogClearingJob.cs
@@ -1,11 +1,10 @@
 using System.IO.Compression;
 using System.Xml.Linq;
+using GrillBot.App.Helpers;
 using GrillBot.App.Jobs.Abstractions;
 using GrillBot.Common.FileStorage;
-using GrillBot.Core.Extensions;
 using GrillBot.Core.Services.AuditLog;
 using GrillBot.Core.Services.AuditLog.Models.Response;
-using GrillBot.Core.Services.FileService;
 using GrillBot.Database.Entity;
 using GrillBot.Database.Services.Repository;
 using Quartz;
@@ -17,16 +16,16 @@ public class AuditLogClearingJob : ArchivationJobBase
 {
     private GrillBotDatabaseBuilder DbFactory { get; }
     private FileStorageFactory FileStorage { get; }
-    private IFileServiceClient FileServiceClient { get; }
     private IAuditLogServiceClient AuditLogServiceClient { get; }
+    private BlobManagerFactoryHelper BlobManagerFactoryHelper { get; }
 
-    public AuditLogClearingJob(GrillBotDatabaseBuilder dbFactory, IServiceProvider serviceProvider, FileStorageFactory fileStorage, IFileServiceClient fileServiceClient,
-        IAuditLogServiceClient auditLogServiceClient) : base(serviceProvider)
+    public AuditLogClearingJob(GrillBotDatabaseBuilder dbFactory, IServiceProvider serviceProvider, FileStorageFactory fileStorage, IAuditLogServiceClient auditLogServiceClient,
+        BlobManagerFactoryHelper blobManagerFactoryHelper) : base(serviceProvider)
     {
         DbFactory = dbFactory;
         FileStorage = fileStorage;
-        FileServiceClient = fileServiceClient;
         AuditLogServiceClient = auditLogServiceClient;
+        BlobManagerFactoryHelper = blobManagerFactoryHelper;
     }
 
     protected override async Task RunAsync(IJobExecutionContext context)
@@ -101,10 +100,24 @@ private async Task<string> StoreDataAsync(XElement xml, IEnumerable<string> file
 
     private async Task AddFilesToArchiveAsync(IEnumerable<string> files, ZipArchive archive)
     {
+        if (!files.Any())
+            return;
+
+        var manager = await BlobManagerFactoryHelper.CreateAsync(BlobConstants.AuditLogDeletedAttachments);
+        var legacyManager = await BlobManagerFactoryHelper.CreateAsync("production");
+
         foreach (var file in files)
         {
-            var fileContent = await FileServiceClient.DownloadFileAsync(file);
-            if (fileContent is null) continue;
+            var useLegacy = false;
+            var fileContent = await manager.DownloadAsync(file);
+
+            if (fileContent is null)
+            {
+                fileContent = await legacyManager.DownloadAsync(file);
+                useLegacy = true;
+
+                if (fileContent is null) continue;
+            }
 
             var entry = archive.CreateEntry(file);
             entry.LastWriteTime = DateTimeOffset.UtcNow;
@@ -113,7 +126,10 @@ private async Task AddFilesToArchiveAsync(IEnumerable<string> files, ZipArchive
             await using var archiveStream = entry.Open();
             await ms.CopyToAsync(archiveStream);
 
-            await FileServiceClient.DeleteFileAsync(file);
+            if (useLegacy)
+                await legacyManager.DeleteAsync(file);
+            else
+                await manager.DeleteAsync(file);
         }
     }