Skip to content

Commit

Permalink
Merge pull request #561 from TheTrackerCouncil/fix-sprite-download
Browse files Browse the repository at this point in the history 
Fix sprite downloads
  • Loading branch information
@MattEqualsCoder
MattEqualsCoder authored Aug 24, 2024
2 parents 094513e + 4fa63e9 commit a4f315b
Show file tree
Hide file tree
Showing 5 changed files with 71 additions and 45 deletions.
5 changes: 0 additions & 5 deletions src/TrackerCouncil.Smz3.Data/Options/GeneralOptions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,11 +226,6 @@ public string? TwitchId
/// </summary>
public List<ConfigSource> ConfigSources { get; set; } = new();

/// <summary>
/// Dictionary of previously downloaded hashes
/// </summary>
public Dictionary<string, string> SpriteHashes { get; set; } = new();

/// <summary>
/// Device to be used for push to talk mode
/// </summary>
Expand Down
96 changes: 67 additions & 29 deletions src/TrackerCouncil.Smz3.Data/Services/GitHubSpriteDownloaderService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using TrackerCouncil.Smz3.Data.Options;
using YamlDotNet.Serialization;

namespace TrackerCouncil.Smz3.Data.Services;

Expand All @@ -19,13 +20,11 @@ public class GitHubSpriteDownloaderService : IGitHubSpriteDownloaderService
{
private ILogger<GitHubSpriteDownloaderService> _logger;
private string _spriteFolder;
private OptionsFactory _optionsFactory;
private CancellationTokenSource _cts = new();

public GitHubSpriteDownloaderService(ILogger<GitHubSpriteDownloaderService> logger, OptionsFactory optionsFactory)
public GitHubSpriteDownloaderService(ILogger<GitHubSpriteDownloaderService> logger)
{
_logger = logger;
_optionsFactory = optionsFactory;
_spriteFolder = Sprite.SpritePath;
_logger.LogInformation("Sprite path: {Path} | {OtherPath}", Sprite.SpritePath, AppContext.BaseDirectory);
}
Expand All @@ -34,7 +33,7 @@ public GitHubSpriteDownloaderService(ILogger<GitHubSpriteDownloaderService> logg

public event EventHandler<SpriteDownloadUpdateEventArgs>? SpriteDownloadUpdate;

public async Task<IDictionary<string, string>?> GetSpritesToDownloadAsync(string owner, string repo, TimeSpan? timeout = null)
public async Task<IDictionary<string, string>?> GetSpritesToDownloadAsync(string owner, string repo, TimeSpan? timeout = null, bool ignoreNoPreviousHashes = false)
{
var sprites = await GetGitHubSpritesAsync(owner, repo, timeout);
if (sprites == null)
Expand All @@ -43,6 +42,13 @@ public GitHubSpriteDownloaderService(ILogger<GitHubSpriteDownloaderService> logg
}

var previousHashes = GetPreviousSpriteHashes();

// If there are no previous hashes, don't download any sprites
if (!ignoreNoPreviousHashes && previousHashes.Count == 0)
{
return new Dictionary<string, string>();
}

var toDownload = new ConcurrentDictionary<string, string>();

Parallel.ForEach(sprites, parallelOptions: new ParallelOptions() { MaxDegreeOfParallelism = 4 },
Expand All @@ -66,7 +72,7 @@ public GitHubSpriteDownloaderService(ILogger<GitHubSpriteDownloaderService> logg
public async Task DownloadSpritesAsync(string owner, string repo, IDictionary<string, string>? spritesToDownload = null, TimeSpan? timeout = null)
{
spritesToDownload ??= await GetSpritesToDownloadAsync(owner, repo, timeout);
if (spritesToDownload == null)
if (spritesToDownload == null || spritesToDownload.Count == 0)
{
return;
}
Expand All @@ -78,13 +84,13 @@ public async Task DownloadSpritesAsync(string owner, string repo, IDictionary<st
Directory.CreateDirectory(_spriteFolder);
}

var previousHashes = GetPreviousSpriteHashes();
var added = new ConcurrentDictionary<string, string>();
var spriteHashes = GetPreviousSpriteHashes();
var addedHashes = new ConcurrentDictionary<string, string>();

var total = spritesToDownload.Count;
var completed = 0;

if (spritesToDownload?.Any() == true)
if (spritesToDownload.Any())
{
await Parallel.ForEachAsync(spritesToDownload, parallelOptions: new ParallelOptions() { MaxDegreeOfParallelism = 4, CancellationToken = _cts.Token},
async (spriteData, _) =>
Expand All @@ -96,45 +102,70 @@ public async Task DownloadSpritesAsync(string owner, string repo, IDictionary<st
if (successful)
{
added[localPath] = currentHash;
addedHashes[localPath] = currentHash;
}
completed++;
SpriteDownloadUpdate.Invoke(this, new SpriteDownloadUpdateEventArgs(completed, total));
SpriteDownloadUpdate?.Invoke(this, new SpriteDownloadUpdateEventArgs(completed, total));
});
}

if (File.Exists(Path.Combine(_spriteFolder, "sprites.json")))
foreach (var addedSprite in addedHashes)
{
File.Delete(Path.Combine(_spriteFolder, "sprites.json"));
spriteHashes[addedSprite.Key] = addedSprite.Value;
}

foreach (var addedSprite in added)
{
previousHashes[addedSprite.Key] = addedSprite.Value;
}

_optionsFactory.Create().GeneralOptions.SpriteHashes = previousHashes;
_optionsFactory.Create().Save();
SaveSpriteHashYaml(spriteHashes);
}

private Dictionary<string, string> GetPreviousSpriteHashes()
{
if (File.Exists(Path.Combine(_spriteFolder, "sprites.json")))
if (File.Exists(GitHubSpriteJsonFilePath))
{
var spriteJson = File.ReadAllText(Path.Combine(_spriteFolder, "sprites.json"));
var spriteJson = File.ReadAllText(GitHubSpriteJsonFilePath);
var tree = JsonSerializer.Deserialize<GitHubTree>(spriteJson);

if (tree?.tree?.Any() == true)
if (tree?.tree == null || tree.tree.Count == 0)
{
_logger.LogInformation("Loading previous sprite hashes from {Path}", Path.Combine(_spriteFolder, "sprites.json"));
return tree.tree
.Where(IsValidSpriteFile)
.ToDictionary(x => ConvertGitHubPath(x.path), x => x.sha);
File.Delete(GitHubSpriteJsonFilePath);
return [];
}

_logger.LogInformation("Loading previous sprite hashes from {Path}", GitHubSpriteJsonFilePath);

var toReturn = tree.tree
.Where(IsValidSpriteFile)
.ToDictionary(x => ConvertGitHubPath(x.path), x => x.sha);

SaveSpriteHashYaml(toReturn);
File.Delete(GitHubSpriteJsonFilePath);

return toReturn;
}
else if (File.Exists(SpriteHashYamlFilePath))
{
var yamlText = File.ReadAllText(SpriteHashYamlFilePath);
var serializer = new DeserializerBuilder()
.IgnoreUnmatchedProperties()
.Build();
try
{
return serializer.Deserialize<Dictionary<string, string>>(yamlText);
}
catch (Exception e)
{
_logger.LogError(e, "Error deserializing sprite hash yaml file {Path}", SpriteHashYamlFilePath);
}
}

return _optionsFactory.Create().GeneralOptions.SpriteHashes;
return [];
}

private void SaveSpriteHashYaml(Dictionary<string, string> hashes)
{
var serializer = new Serializer();
var yamlText = serializer.Serialize(hashes);
File.WriteAllText(SpriteHashYamlFilePath, yamlText);
}

private async Task<bool> DownloadFileAsync(string destination, string url, int attempts = 2)
Expand Down Expand Up @@ -197,7 +228,6 @@ private async Task<bool> DownloadFileAsync(string destination, string url, int a

_logger.LogInformation("Retrieved {Count} file data from GitHub", tree.tree.Count);


return tree.tree
.Where(IsValidSpriteFile)
.ToDictionary(x => x.path, x => x.sha);
Expand Down Expand Up @@ -227,12 +257,20 @@ private class GitHubTree
public List<GitHubFile>? tree { get; set; }
}

public class GitHubFile
private class GitHubFile
{
public required string path { get; set; }
public required string mode { get; set; }
public required string type { get; set; }
public required string sha { get; set; }
public required string url { get; set; }
}

#if DEBUG
private string SpriteHashYamlFilePath => Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "SMZ3CasRandomizer", "sprite-hashes-debug.yml");
#else
private string SpriteHashYamlFilePath => Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "SMZ3CasRandomizer", "sprite-hashes.yml");
#endif

private string GitHubSpriteJsonFilePath => Path.Combine(_spriteFolder, "sprites.json");
}
3 changes: 2 additions & 1 deletion src/TrackerCouncil.Smz3.Data/Services/IGitHubSpriteDownloaderService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@ public interface IGitHubSpriteDownloaderService
/// <param name="owner">The GitHub repository owner</param>
/// <param name="repo">The GitHub repository to download</param>
/// <param name="timeout">How long to timeout from the api call</param>
/// <param name="ignoreNoPreviousHashes">If downloads should be triggered, even if no previous hashes exist</param>
/// <returns>A dictionary of paths on GitHub and their git hashes</returns>
public Task<IDictionary<string, string>?> GetSpritesToDownloadAsync(string owner, string repo, TimeSpan? timeout = null);
public Task<IDictionary<string, string>?> GetSpritesToDownloadAsync(string owner, string repo, TimeSpan? timeout = null, bool ignoreNoPreviousHashes = false);

/// <summary>
/// Downloads sprites from GitHub to the folder
Expand Down
4 changes: 2 additions & 2 deletions src/TrackerCouncil.Smz3.Data/Services/OptionsWindowService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,12 +192,12 @@ private async Task UpdateConfigsAsync()

private async Task UpdateSpritesAsync()
{
var sprites = await gitHubSpriteDownloaderService.GetSpritesToDownloadAsync("TheTrackerCouncil", "SMZ3CasSprites");
var sprites = await gitHubSpriteDownloaderService.GetSpritesToDownloadAsync("TheTrackerCouncil", "SMZ3CasSprites", null, true);

if (sprites?.Any() == true)
{
SpriteDownloadStarted?.Invoke(this, EventArgs.Empty);
await gitHubSpriteDownloaderService.DownloadSpritesAsync("TheTrackerCouncil", "SMZ3CasSprites");
await gitHubSpriteDownloaderService.DownloadSpritesAsync("TheTrackerCouncil", "SMZ3CasSprites", sprites);
SpriteDownloadEnded?.Invoke(this, EventArgs.Empty);
}
}
Expand Down
8 changes: 0 additions & 8 deletions src/TrackerCouncil.Smz3.UI/Services/MainWindowService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,14 +113,6 @@ public async Task DownloadSpritesAsync()
return;
}

#if DEBUG
// Skip sprite download when debugging if the folder already exists
if (Directory.Exists(Sprite.SpritePath))
{
return;
}
#endif

var toDownload = await gitHubSpriteDownloaderService.GetSpritesToDownloadAsync("TheTrackerCouncil", "SMZ3CasSprites");

if (toDownload is not { Count: > 4 })
Expand Down

0 comments on commit a4f315b

Please sign in to comment.