Skip to content

Commit

Permalink
Serialize archived data to json
Browse files Browse the repository at this point in the history
  • Loading branch information
@Misha12
Misha12 committed Feb 9, 2024
1 parent 40416b0 commit 99a205d
Show file tree
Hide file tree
Showing 4 changed files with 101 additions and 88 deletions.
66 changes: 38 additions & 28 deletions src/GrillBot.App/Jobs/Abstractions/ArchivationJobBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,60 +17,70 @@ protected ArchivationJobBase(IServiceProvider serviceProvider) : base(servicePro
DatabaseBuilder = serviceProvider.GetRequiredService<GrillBotDatabaseBuilder>();
}

protected static IEnumerable<XAttribute> CreateMetadata(int count)
protected static JArray TransformGuilds(IEnumerable<Database.Entity.Guild?> guilds)
{
yield return new XAttribute("CreatedAt", DateTime.Now.ToString("o"));
yield return new XAttribute("Count", count);
var guildObjects = guilds
.Where(o => o != null)
.DistinctBy(o => o!.Id)
.Select(o => new JObject
{
["Id"] = o!.Id,
["Name"] = o.Name
});

return new JArray(guildObjects);
}

protected static IEnumerable<XElement> TransformGuilds(IEnumerable<Database.Entity.Guild?> guilds)
protected static JArray TransformUsers(IEnumerable<Database.Entity.User?> users)
{
return guilds
.Where(o => o != null)
var userObjects = users
.Where(o => o is not null)
.DistinctBy(o => o!.Id)
.Select(o => new XElement("Guild", new XAttribute("Id", o!.Id), new XAttribute("Name", o.Name)));
}
.Select(u => TransformUser(u!));

protected static IEnumerable<XElement> TransformUsers(IEnumerable<Database.Entity.User?> users)
=> users.Where(o => o is not null).DistinctBy(o => o!.Id).Select(u => TransformUser(u!));
return new JArray(userObjects);
}

protected static IEnumerable<XElement> TransformGuildUsers(IEnumerable<Database.Entity.GuildUser> guildUsers)
protected static JArray TransformGuildUsers(IEnumerable<Database.Entity.GuildUser> guildUsers)
{
return guildUsers.DistinctBy(o => $"{o.UserId}/{o.GuildId}").Select(u =>
var userObjects = guildUsers.DistinctBy(o => $"{o.UserId}/{o.GuildId}").Select(u =>
{
var user = TransformUser(u.User!);
user.Name = "GuildUser";
user.Add(new XAttribute("GuildId", u.GuildId));
user.Attribute("FullName")!.Value = u.DisplayName ?? "";
user["GuildId"] = u.GuildId;
user["FullName"] = u.DisplayName ?? "";
if (!string.IsNullOrEmpty(u.UsedInviteCode))
user.Add(new XAttribute("UsedInviteCode", u.UsedInviteCode));
user["UsedInviteCode"] = u.UsedInviteCode;
return user;
});

return new JArray(userObjects);
}

private static XElement TransformUser(Database.Entity.User user)
private static JObject TransformUser(Database.Entity.User user)
{
var element = new XElement(
"User",
new XAttribute("Id", user.Id),
new XAttribute("FullName", user.Username)
);
var json = new JObject
{
["Id"] = user.Id,
["FullName"] = user.Username
};

if (user.Flags > 0)
element.Add(new XAttribute("Flags", user.Flags));
json["Flags"] = user.Flags;

return element;
return json;
}

protected static async Task AddXmlToZipAsync(ZipArchive archive, XElement xml, string xmlName)
protected static async Task AddJsonToZipAsync(ZipArchive archive, JObject json, string jsonName)
{
var entry = archive.CreateEntry(xmlName);
var entry = archive.CreateEntry(jsonName);
entry.LastWriteTime = DateTimeOffset.Now;

await using var entryStream = entry.Open();
await xml.SaveAsync(entryStream, SaveOptions.OmitDuplicateNamespaces | SaveOptions.DisableFormatting, CancellationToken.None);
await using var streamWriter = new StreamWriter(entryStream);

await streamWriter.WriteAsync(json.ToString(Formatting.None));
await streamWriter.FlushAsync();
}
}
64 changes: 32 additions & 32 deletions src/GrillBot.App/Jobs/AuditLogClearingJob.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,66 +27,66 @@ protected override async Task RunAsync(IJobExecutionContext context)
if (archivationResult is null)
return;

var xmlData = XElement.Parse(archivationResult.Xml);
var jsonData = JObject.Parse(archivationResult.Content);

await using var repository = DatabaseBuilder.CreateRepository();

await ProcessGuildsAsync(repository, archivationResult.GuildIds, xmlData);
await ProcessChannelsAsync(repository, archivationResult.ChannelIds, xmlData);
await ProcessUsersAsync(repository, archivationResult.UserIds, xmlData);
await ProcessGuildsAsync(repository, archivationResult.GuildIds, jsonData);
await ProcessChannelsAsync(repository, archivationResult.ChannelIds, jsonData);
await ProcessUsersAsync(repository, archivationResult.UserIds, jsonData);

var zipSize = await StoreDataAsync(xmlData, archivationResult.Files);
var xmlSize = Encoding.UTF8.GetBytes(xmlData.ToString()).Length.Bytes().ToString();
var zipSize = await StoreDataAsync(jsonData, archivationResult.Files);
var xmlSize = Encoding.UTF8.GetBytes(jsonData.ToString(Formatting.None)).Length.Bytes().ToString();
var formattedZipSize = zipSize.Bytes().ToString();

await AuditLogServiceClient.BulkDeleteAsync(archivationResult.Ids);
context.Result = BuildReport(archivationResult, xmlSize, formattedZipSize);
}

private static IEnumerable<XElement> TransformChannels(IEnumerable<GuildChannel?> channels)
private static IEnumerable<JObject> TransformChannels(IEnumerable<GuildChannel?> channels)
{
return channels
.Where(o => o is not null)
.DistinctBy(o => $"{o!.ChannelId}/{o.GuildId}").Select(ch =>
{
var channel = new XElement("Channel");
channel.Add(
new XAttribute("Id", ch!.ChannelId),
new XAttribute("Name", ch.Name),
new XAttribute("Type", ch.ChannelType.ToString()),
new XAttribute("GuildId", ch.GuildId)
);
var channel = new JObject
{
["Id"] = ch!.ChannelId,
["Name"] = ch.Name,
["Type"] = ch.ChannelType.ToString(),
["GuildId"] = ch.GuildId
};
if (ch.UserPermissionsCount > 0)
channel.Add(new XAttribute("UserPermissionsCount", ch.UserPermissionsCount));
channel["UserPermissionsCount"] = ch.UserPermissionsCount;
if (ch.RolePermissionsCount > 0)
channel.Add(new XAttribute("RolePermissionsCount", ch.RolePermissionsCount));
channel["RolePermissionsCount"] = ch.RolePermissionsCount;
if (ch.Flags > 0)
channel.Add(new XAttribute("Flags", ch.Flags));
channel["Flags"] = ch.Flags;
if (!string.IsNullOrEmpty(ch.ParentChannelId))
channel.Add(new XAttribute("ParentChannelId", ch.ParentChannelId));
channel["ParentChannelId"] = ch.ParentChannelId;
return channel;
});
}

private async Task<long> StoreDataAsync(XElement xml, IEnumerable<string> files)
private async Task<long> StoreDataAsync(JObject json, IEnumerable<string> files)
{
var xmlBaseName = $"AuditLog_{DateTime.Now:yyyyMMdd_HHmmss}";
var jsonBaseName = $"AuditLog_{DateTime.Now:yyyyMMdd_HHmmss}";
var temporaryPath = Path.GetTempPath();
var zipName = $"{xmlBaseName}.zip";
var zipName = $"{jsonBaseName}.zip";
var zipPath = Path.Combine(temporaryPath, zipName);
long archiveSize;

try
{
using (var zipArchive = ZipFile.Open(zipPath, ZipArchiveMode.Create))
{
await AddXmlToZipAsync(zipArchive, xml, $"{xmlBaseName}.xml");
await AddJsonToZipAsync(zipArchive, json, $"{jsonBaseName}.json");
await AddFilesToArchiveAsync(files, zipArchive);
}

using var reader = File.OpenRead(zipPath);
await using var reader = File.OpenRead(zipPath);
var archiveManager = await BlobManagerFactoryHelper.CreateAsync(BlobConstants.AuditLogArchives);
await archiveManager.UploadAsync(zipName, reader);
}
Expand Down Expand Up @@ -126,7 +126,7 @@ private async Task AddFilesToArchiveAsync(IEnumerable<string> files, ZipArchive
var entry = archive.CreateEntry(file);
entry.LastWriteTime = DateTimeOffset.UtcNow;

using var ms = new MemoryStream(fileContent);
await using var ms = new MemoryStream(fileContent);
await using var archiveStream = entry.Open();
await ms.CopyToAsync(archiveStream);

Expand All @@ -137,38 +137,38 @@ private async Task AddFilesToArchiveAsync(IEnumerable<string> files, ZipArchive
}
}

private static async Task ProcessGuildsAsync(GrillBotRepository repository, List<string> guildIds, XContainer xmlData)
private static async Task ProcessGuildsAsync(GrillBotRepository repository, List<string> guildIds, JObject json)
{
var guilds = new List<Guild>();
foreach (var guildChunk in guildIds.Chunk(100))
guilds.AddRange(await repository.Guild.GetGuildsByIdsAsync(guildChunk.ToList()));
xmlData.Add(TransformGuilds(guilds));
json["Guilds"] = new JArray(TransformGuilds(guilds));
}

private static async Task ProcessChannelsAsync(GrillBotRepository repository, List<string> channelIds, XContainer xmlData)
private static async Task ProcessChannelsAsync(GrillBotRepository repository, List<string> channelIds, JObject json)
{
var channels = new List<GuildChannel?>();
foreach (var channelChunk in channelIds.Chunk(100))
channels.AddRange(await repository.Channel.GetChannelsByIdsAsync(channelChunk.ToList()));

xmlData.Add(TransformChannels(channels));
json["Channels"] = new JArray(TransformChannels(channels));
}

private static async Task ProcessUsersAsync(GrillBotRepository repository, List<string> userIds, XContainer xmlData)
private static async Task ProcessUsersAsync(GrillBotRepository repository, List<string> userIds, JObject json)
{
var users = new List<User?>();
foreach (var userChunk in userIds.Chunk(100))
users.AddRange(await repository.User.GetUsersByIdsAsync(userChunk.ToList()));

xmlData.Add(TransformUsers(users));
json["Users"] = new JArray(TransformUsers(users));
}

private static string BuildReport(ArchivationResult result, string xmlSize, string zipSize)
private static string BuildReport(ArchivationResult result, string jsonSize, string zipSize)
{
var totalFilesSize = result.TotalFilesSize.Bytes().ToString();

var builder = new StringBuilder()
.AppendFormat("Items: {0}, Files: {1} ({2}), XmlSize: {3}, ZipSize: {4}", result.ItemsCount, result.Files.Count, totalFilesSize, xmlSize, zipSize)
.AppendFormat("Items: {0}, Files: {1} ({2}), JsonSize: {3}, ZipSize: {4}", result.ItemsCount, result.Files.Count, totalFilesSize, jsonSize, zipSize)
.AppendLine()
.AppendLine();

Expand Down
57 changes: 30 additions & 27 deletions src/GrillBot.App/Jobs/UnverifyLogArchivationJob.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
using System.IO.Compression;
using System.Xml.Linq;
using GrillBot.App.Helpers;
using GrillBot.App.Jobs.Abstractions;
using GrillBot.Common.FileStorage;
Expand Down Expand Up @@ -29,61 +28,65 @@ protected override async Task RunAsync(IJobExecutionContext context)
return;

var data = await repository.Unverify.GetLogsForArchivationAsync(expirationMilestone);
var logRoot = new XElement("UnverifyLog");

logRoot.Add(CreateMetadata(data.Count));
logRoot.Add(TransformGuilds(data.Select(o => o.Guild)));
var logRoot = new JObject
{
["CreatedAt"] = DateTime.Now.ToString("o"),
["Count"] = data.Count,
["Guilds"] = TransformGuilds(data.Select(o => o.Guild))
};

var users = data
.Select(o => o.FromUser!)
.Concat(data.Select(o => o.ToUser!))
.Where(o => o is not null)
.DistinctBy(o => $"{o!.UserId}/{o.GuildId}");
logRoot.Add(TransformGuildUsers(users));
logRoot.Add("Users", TransformGuildUsers(users));

var items = new JArray();
foreach (var item in data)
{
var element = new XElement("Item");

element.Add(
new XAttribute("Id", item.Id),
new XAttribute("Operation", item.Operation.ToString()),
new XAttribute("GuildId", item.GuildId),
new XAttribute("FromUserId", item.FromUserId),
new XAttribute("ToUserId", item.ToUserId),
new XAttribute("CreatedAt", item.CreatedAt.ToString("o")),
new XElement("Data", item.Data)
);

logRoot.Add(element);
var jsonElement = new JObject
{
["Id"] = item.Id,
["Operation"] = item.Operation.ToString(),
["GuildId"] = item.GuildId,
["FromUserId"] = item.FromUserId,
["ToUserId"] = item.ToUserId,
["CreatedAt"] = item.CreatedAt.ToString("o"),
["Data"] = item.Data
};

items.Add(jsonElement);
repository.Remove(item);
}

logRoot.Add("Items", items);

var archiveSize = await SaveDataAsync(logRoot);
await repository.CommitAsync();

var xmlSize = Encoding.UTF8.GetBytes(logRoot.ToString()).Length.Bytes().ToString();
var xmlSize = Encoding.UTF8.GetBytes(logRoot.ToString(Formatting.None)).Length.Bytes().ToString();
var zipSize = archiveSize.Bytes().ToString();

context.Result = BuildReport(xmlSize, zipSize, data);
}

private async Task<long> SaveDataAsync(XElement xml)
private async Task<long> SaveDataAsync(JObject json)
{
var xmlBaseName = $"UnverifyLog_{DateTime.Now:yyyyMMdd_HHmmss}";
var jsonBaseName = $"UnverifyLog_{DateTime.Now:yyyyMMdd_HHmmss}";
var temporaryPath = Path.GetTempPath();
var zipName = $"{xmlBaseName}.zip";
var zipName = $"{jsonBaseName}.zip";
var zipPath = Path.Combine(temporaryPath, zipName);
long archiveSize;

try
{
using (var zipArchive = ZipFile.Open(zipPath, ZipArchiveMode.Create))
{
await AddXmlToZipAsync(zipArchive, xml, $"{xmlBaseName}.xml");
await AddJsonToZipAsync(zipArchive, json, $"{jsonBaseName}.json");
}

using var reader = File.OpenRead(zipPath);
await using var reader = File.OpenRead(zipPath);
var archiveManager = await BlobManagerFactoryHelper.CreateAsync(BlobConstants.UnverifyLogArchives);
await archiveManager.UploadAsync(zipName, reader);
}
Expand All @@ -98,10 +101,10 @@ private async Task<long> SaveDataAsync(XElement xml)
return archiveSize;
}

private static string BuildReport(string xmlSize, string zipSize, List<Database.Entity.UnverifyLog> data)
private static string BuildReport(string jsonSize, string zipSize, List<Database.Entity.UnverifyLog> data)
{
var builder = new StringBuilder()
.AppendFormat("Items: {0}, XmlSize: {1}, ZipSize: {0}", xmlSize, zipSize)
.AppendFormat("Items: {0}, JsonSize: {1}, ZipSize: {0}", jsonSize, zipSize)
.AppendLine()
.AppendLine();

Expand Down
2 changes: 1 addition & 1 deletion src/GrillBot.Common/GrillBot.Common.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
<ItemGroup>
<PackageReference Include="CircularBuffer" Version="1.4.0" />
<PackageReference Include="Discord.Net" Version="3.13.0" />
<PackageReference Include="GrillBot.Core.Services" Version="1.4.8" />
<PackageReference Include="GrillBot.Core.Services" Version="1.4.9" />
<PackageReference Include="Humanizer.Core.cs" Version="2.14.1" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="7.0.4" />
<PackageReference Include="Microsoft.Extensions.Http" Version="7.0.0" />
Expand Down

0 comments on commit 99a205d

Please sign in to comment.