From 99a205ddc0c48dbe3e0ae3d0ce5bcdff8b1ddec5 Mon Sep 17 00:00:00 2001 From: Michal Halabica Date: Fri, 9 Feb 2024 09:46:54 +0100 Subject: [PATCH] Serialize archived data to json --- .../Jobs/Abstractions/ArchivationJobBase.cs | 66 +++++++++++-------- src/GrillBot.App/Jobs/AuditLogClearingJob.cs | 64 +++++++++--------- .../Jobs/UnverifyLogArchivationJob.cs | 57 ++++++++-------- src/GrillBot.Common/GrillBot.Common.csproj | 2 +- 4 files changed, 101 insertions(+), 88 deletions(-) diff --git a/src/GrillBot.App/Jobs/Abstractions/ArchivationJobBase.cs b/src/GrillBot.App/Jobs/Abstractions/ArchivationJobBase.cs index dcd54d7d..8887a4e2 100644 --- a/src/GrillBot.App/Jobs/Abstractions/ArchivationJobBase.cs +++ b/src/GrillBot.App/Jobs/Abstractions/ArchivationJobBase.cs @@ -17,60 +17,70 @@ protected ArchivationJobBase(IServiceProvider serviceProvider) : base(servicePro DatabaseBuilder = serviceProvider.GetRequiredService(); } - protected static IEnumerable CreateMetadata(int count) + protected static JArray TransformGuilds(IEnumerable guilds) { - yield return new XAttribute("CreatedAt", DateTime.Now.ToString("o")); - yield return new XAttribute("Count", count); + var guildObjects = guilds + .Where(o => o != null) + .DistinctBy(o => o!.Id) + .Select(o => new JObject + { + ["Id"] = o!.Id, + ["Name"] = o.Name + }); + + return new JArray(guildObjects); } - protected static IEnumerable TransformGuilds(IEnumerable guilds) + protected static JArray TransformUsers(IEnumerable users) { - return guilds - .Where(o => o != null) + var userObjects = users + .Where(o => o is not null) .DistinctBy(o => o!.Id) - .Select(o => new XElement("Guild", new XAttribute("Id", o!.Id), new XAttribute("Name", o.Name))); - } + .Select(u => TransformUser(u!)); - protected static IEnumerable TransformUsers(IEnumerable users) - => users.Where(o => o is not null).DistinctBy(o => o!.Id).Select(u => TransformUser(u!)); + return new JArray(userObjects); + } - protected static IEnumerable TransformGuildUsers(IEnumerable guildUsers) + protected static JArray TransformGuildUsers(IEnumerable guildUsers) { - return guildUsers.DistinctBy(o => $"{o.UserId}/{o.GuildId}").Select(u => + var userObjects = guildUsers.DistinctBy(o => $"{o.UserId}/{o.GuildId}").Select(u => { var user = TransformUser(u.User!); - user.Name = "GuildUser"; - - user.Add(new XAttribute("GuildId", u.GuildId)); - user.Attribute("FullName")!.Value = u.DisplayName ?? ""; + user["GuildId"] = u.GuildId; + user["FullName"] = u.DisplayName ?? ""; if (!string.IsNullOrEmpty(u.UsedInviteCode)) - user.Add(new XAttribute("UsedInviteCode", u.UsedInviteCode)); + user["UsedInviteCode"] = u.UsedInviteCode; return user; }); + + return new JArray(userObjects); } - private static XElement TransformUser(Database.Entity.User user) + private static JObject TransformUser(Database.Entity.User user) { - var element = new XElement( - "User", - new XAttribute("Id", user.Id), - new XAttribute("FullName", user.Username) - ); + var json = new JObject + { + ["Id"] = user.Id, + ["FullName"] = user.Username + }; if (user.Flags > 0) - element.Add(new XAttribute("Flags", user.Flags)); + json["Flags"] = user.Flags; - return element; + return json; } - protected static async Task AddXmlToZipAsync(ZipArchive archive, XElement xml, string xmlName) + protected static async Task AddJsonToZipAsync(ZipArchive archive, JObject json, string jsonName) { - var entry = archive.CreateEntry(xmlName); + var entry = archive.CreateEntry(jsonName); entry.LastWriteTime = DateTimeOffset.Now; await using var entryStream = entry.Open(); - await xml.SaveAsync(entryStream, SaveOptions.OmitDuplicateNamespaces | SaveOptions.DisableFormatting, CancellationToken.None); + await using var streamWriter = new StreamWriter(entryStream); + + await streamWriter.WriteAsync(json.ToString(Formatting.None)); + await streamWriter.FlushAsync(); } } diff --git a/src/GrillBot.App/Jobs/AuditLogClearingJob.cs b/src/GrillBot.App/Jobs/AuditLogClearingJob.cs index c8c61b42..f757f6e1 100644 --- a/src/GrillBot.App/Jobs/AuditLogClearingJob.cs +++ b/src/GrillBot.App/Jobs/AuditLogClearingJob.cs @@ -27,54 +27,54 @@ protected override async Task RunAsync(IJobExecutionContext context) if (archivationResult is null) return; - var xmlData = XElement.Parse(archivationResult.Xml); + var jsonData = JObject.Parse(archivationResult.Content); await using var repository = DatabaseBuilder.CreateRepository(); - await ProcessGuildsAsync(repository, archivationResult.GuildIds, xmlData); - await ProcessChannelsAsync(repository, archivationResult.ChannelIds, xmlData); - await ProcessUsersAsync(repository, archivationResult.UserIds, xmlData); + await ProcessGuildsAsync(repository, archivationResult.GuildIds, jsonData); + await ProcessChannelsAsync(repository, archivationResult.ChannelIds, jsonData); + await ProcessUsersAsync(repository, archivationResult.UserIds, jsonData); - var zipSize = await StoreDataAsync(xmlData, archivationResult.Files); - var xmlSize = Encoding.UTF8.GetBytes(xmlData.ToString()).Length.Bytes().ToString(); + var zipSize = await StoreDataAsync(jsonData, archivationResult.Files); + var xmlSize = Encoding.UTF8.GetBytes(jsonData.ToString(Formatting.None)).Length.Bytes().ToString(); var formattedZipSize = zipSize.Bytes().ToString(); await AuditLogServiceClient.BulkDeleteAsync(archivationResult.Ids); context.Result = BuildReport(archivationResult, xmlSize, formattedZipSize); } - private static IEnumerable TransformChannels(IEnumerable channels) + private static IEnumerable TransformChannels(IEnumerable channels) { return channels .Where(o => o is not null) .DistinctBy(o => $"{o!.ChannelId}/{o.GuildId}").Select(ch => { - var channel = new XElement("Channel"); - channel.Add( - new XAttribute("Id", ch!.ChannelId), - new XAttribute("Name", ch.Name), - new XAttribute("Type", ch.ChannelType.ToString()), - new XAttribute("GuildId", ch.GuildId) - ); + var channel = new JObject + { + ["Id"] = ch!.ChannelId, + ["Name"] = ch.Name, + ["Type"] = ch.ChannelType.ToString(), + ["GuildId"] = ch.GuildId + }; if (ch.UserPermissionsCount > 0) - channel.Add(new XAttribute("UserPermissionsCount", ch.UserPermissionsCount)); + channel["UserPermissionsCount"] = ch.UserPermissionsCount; if (ch.RolePermissionsCount > 0) - channel.Add(new XAttribute("RolePermissionsCount", ch.RolePermissionsCount)); + channel["RolePermissionsCount"] = ch.RolePermissionsCount; if (ch.Flags > 0) - channel.Add(new XAttribute("Flags", ch.Flags)); + channel["Flags"] = ch.Flags; if (!string.IsNullOrEmpty(ch.ParentChannelId)) - channel.Add(new XAttribute("ParentChannelId", ch.ParentChannelId)); + channel["ParentChannelId"] = ch.ParentChannelId; return channel; }); } - private async Task StoreDataAsync(XElement xml, IEnumerable files) + private async Task StoreDataAsync(JObject json, IEnumerable files) { - var xmlBaseName = $"AuditLog_{DateTime.Now:yyyyMMdd_HHmmss}"; + var jsonBaseName = $"AuditLog_{DateTime.Now:yyyyMMdd_HHmmss}"; var temporaryPath = Path.GetTempPath(); - var zipName = $"{xmlBaseName}.zip"; + var zipName = $"{jsonBaseName}.zip"; var zipPath = Path.Combine(temporaryPath, zipName); long archiveSize; @@ -82,11 +82,11 @@ private async Task StoreDataAsync(XElement xml, IEnumerable files) { using (var zipArchive = ZipFile.Open(zipPath, ZipArchiveMode.Create)) { - await AddXmlToZipAsync(zipArchive, xml, $"{xmlBaseName}.xml"); + await AddJsonToZipAsync(zipArchive, json, $"{jsonBaseName}.json"); await AddFilesToArchiveAsync(files, zipArchive); } - using var reader = File.OpenRead(zipPath); + await using var reader = File.OpenRead(zipPath); var archiveManager = await BlobManagerFactoryHelper.CreateAsync(BlobConstants.AuditLogArchives); await archiveManager.UploadAsync(zipName, reader); } @@ -126,7 +126,7 @@ private async Task AddFilesToArchiveAsync(IEnumerable files, ZipArchive var entry = archive.CreateEntry(file); entry.LastWriteTime = DateTimeOffset.UtcNow; - using var ms = new MemoryStream(fileContent); + await using var ms = new MemoryStream(fileContent); await using var archiveStream = entry.Open(); await ms.CopyToAsync(archiveStream); @@ -137,38 +137,38 @@ private async Task AddFilesToArchiveAsync(IEnumerable files, ZipArchive } } - private static async Task ProcessGuildsAsync(GrillBotRepository repository, List guildIds, XContainer xmlData) + private static async Task ProcessGuildsAsync(GrillBotRepository repository, List guildIds, JObject json) { var guilds = new List(); foreach (var guildChunk in guildIds.Chunk(100)) guilds.AddRange(await repository.Guild.GetGuildsByIdsAsync(guildChunk.ToList())); - xmlData.Add(TransformGuilds(guilds)); + json["Guilds"] = new JArray(TransformGuilds(guilds)); } - private static async Task ProcessChannelsAsync(GrillBotRepository repository, List channelIds, XContainer xmlData) + private static async Task ProcessChannelsAsync(GrillBotRepository repository, List channelIds, JObject json) { var channels = new List(); foreach (var channelChunk in channelIds.Chunk(100)) channels.AddRange(await repository.Channel.GetChannelsByIdsAsync(channelChunk.ToList())); - xmlData.Add(TransformChannels(channels)); + json["Channels"] = new JArray(TransformChannels(channels)); } - private static async Task ProcessUsersAsync(GrillBotRepository repository, List userIds, XContainer xmlData) + private static async Task ProcessUsersAsync(GrillBotRepository repository, List userIds, JObject json) { var users = new List(); foreach (var userChunk in userIds.Chunk(100)) users.AddRange(await repository.User.GetUsersByIdsAsync(userChunk.ToList())); - xmlData.Add(TransformUsers(users)); + json["Users"] = new JArray(TransformUsers(users)); } - private static string BuildReport(ArchivationResult result, string xmlSize, string zipSize) + private static string BuildReport(ArchivationResult result, string jsonSize, string zipSize) { var totalFilesSize = result.TotalFilesSize.Bytes().ToString(); var builder = new StringBuilder() - .AppendFormat("Items: {0}, Files: {1} ({2}), XmlSize: {3}, ZipSize: {4}", result.ItemsCount, result.Files.Count, totalFilesSize, xmlSize, zipSize) + .AppendFormat("Items: {0}, Files: {1} ({2}), JsonSize: {3}, ZipSize: {4}", result.ItemsCount, result.Files.Count, totalFilesSize, jsonSize, zipSize) .AppendLine() .AppendLine(); diff --git a/src/GrillBot.App/Jobs/UnverifyLogArchivationJob.cs b/src/GrillBot.App/Jobs/UnverifyLogArchivationJob.cs index 86faf61b..233f2172 100644 --- a/src/GrillBot.App/Jobs/UnverifyLogArchivationJob.cs +++ b/src/GrillBot.App/Jobs/UnverifyLogArchivationJob.cs @@ -1,5 +1,4 @@ using System.IO.Compression; -using System.Xml.Linq; using GrillBot.App.Helpers; using GrillBot.App.Jobs.Abstractions; using GrillBot.Common.FileStorage; @@ -29,50 +28,54 @@ protected override async Task RunAsync(IJobExecutionContext context) return; var data = await repository.Unverify.GetLogsForArchivationAsync(expirationMilestone); - var logRoot = new XElement("UnverifyLog"); - - logRoot.Add(CreateMetadata(data.Count)); - logRoot.Add(TransformGuilds(data.Select(o => o.Guild))); + var logRoot = new JObject + { + ["CreatedAt"] = DateTime.Now.ToString("o"), + ["Count"] = data.Count, + ["Guilds"] = TransformGuilds(data.Select(o => o.Guild)) + }; var users = data .Select(o => o.FromUser!) .Concat(data.Select(o => o.ToUser!)) .Where(o => o is not null) .DistinctBy(o => $"{o!.UserId}/{o.GuildId}"); - logRoot.Add(TransformGuildUsers(users)); + logRoot.Add("Users", TransformGuildUsers(users)); + var items = new JArray(); foreach (var item in data) { - var element = new XElement("Item"); - - element.Add( - new XAttribute("Id", item.Id), - new XAttribute("Operation", item.Operation.ToString()), - new XAttribute("GuildId", item.GuildId), - new XAttribute("FromUserId", item.FromUserId), - new XAttribute("ToUserId", item.ToUserId), - new XAttribute("CreatedAt", item.CreatedAt.ToString("o")), - new XElement("Data", item.Data) - ); - - logRoot.Add(element); + var jsonElement = new JObject + { + ["Id"] = item.Id, + ["Operation"] = item.Operation.ToString(), + ["GuildId"] = item.GuildId, + ["FromUserId"] = item.FromUserId, + ["ToUserId"] = item.ToUserId, + ["CreatedAt"] = item.CreatedAt.ToString("o"), + ["Data"] = item.Data + }; + + items.Add(jsonElement); repository.Remove(item); } + logRoot.Add("Items", items); + var archiveSize = await SaveDataAsync(logRoot); await repository.CommitAsync(); - var xmlSize = Encoding.UTF8.GetBytes(logRoot.ToString()).Length.Bytes().ToString(); + var xmlSize = Encoding.UTF8.GetBytes(logRoot.ToString(Formatting.None)).Length.Bytes().ToString(); var zipSize = archiveSize.Bytes().ToString(); context.Result = BuildReport(xmlSize, zipSize, data); } - private async Task SaveDataAsync(XElement xml) + private async Task SaveDataAsync(JObject json) { - var xmlBaseName = $"UnverifyLog_{DateTime.Now:yyyyMMdd_HHmmss}"; + var jsonBaseName = $"UnverifyLog_{DateTime.Now:yyyyMMdd_HHmmss}"; var temporaryPath = Path.GetTempPath(); - var zipName = $"{xmlBaseName}.zip"; + var zipName = $"{jsonBaseName}.zip"; var zipPath = Path.Combine(temporaryPath, zipName); long archiveSize; @@ -80,10 +83,10 @@ private async Task SaveDataAsync(XElement xml) { using (var zipArchive = ZipFile.Open(zipPath, ZipArchiveMode.Create)) { - await AddXmlToZipAsync(zipArchive, xml, $"{xmlBaseName}.xml"); + await AddJsonToZipAsync(zipArchive, json, $"{jsonBaseName}.json"); } - using var reader = File.OpenRead(zipPath); + await using var reader = File.OpenRead(zipPath); var archiveManager = await BlobManagerFactoryHelper.CreateAsync(BlobConstants.UnverifyLogArchives); await archiveManager.UploadAsync(zipName, reader); } @@ -98,10 +101,10 @@ private async Task SaveDataAsync(XElement xml) return archiveSize; } - private static string BuildReport(string xmlSize, string zipSize, List data) + private static string BuildReport(string jsonSize, string zipSize, List data) { var builder = new StringBuilder() - .AppendFormat("Items: {0}, XmlSize: {1}, ZipSize: {0}", xmlSize, zipSize) + .AppendFormat("Items: {0}, JsonSize: {1}, ZipSize: {0}", jsonSize, zipSize) .AppendLine() .AppendLine(); diff --git a/src/GrillBot.Common/GrillBot.Common.csproj b/src/GrillBot.Common/GrillBot.Common.csproj index eab31bd4..cba2dfa2 100644 --- a/src/GrillBot.Common/GrillBot.Common.csproj +++ b/src/GrillBot.Common/GrillBot.Common.csproj @@ -19,7 +19,7 @@ - +