From 52ce5d05c1c91235ef045e0f6ff766baba6bebb8 Mon Sep 17 00:00:00 2001
From: ScrubN <72096833+ScrubN@users.noreply.github.com>
Date: Mon, 3 Jun 2024 22:34:14 -0400
Subject: [PATCH 01/21] Create destination file immediately
---
TwitchDownloaderCore/Chat/ChatHtml.cs | 13 +----
TwitchDownloaderCore/Chat/ChatJson.cs | 15 +----
TwitchDownloaderCore/Chat/ChatText.cs | 12 +---
TwitchDownloaderCore/ChatDownloader.cs | 16 +++++-
TwitchDownloaderCore/ChatRenderer.cs | 56 ++++++++++---------
TwitchDownloaderCore/ChatUpdater.cs | 16 +++++-
TwitchDownloaderCore/ClipDownloader.cs | 48 +++++++++-------
.../Options/ChatDownloadOptions.cs | 5 +-
.../Options/ChatRenderOptions.cs | 1 +
.../Options/ChatUpdateOptions.cs | 5 +-
.../Options/ClipDownloadOptions.cs | 6 +-
.../Options/TsMergeOptions.cs | 6 +-
.../Options/VideoDownloadOptions.cs | 3 +-
TwitchDownloaderCore/TsMerger.cs | 15 ++++-
TwitchDownloaderCore/TwitchHelper.cs | 27 ++++++++-
TwitchDownloaderCore/VideoDownloader.cs | 28 ++++++----
16 files changed, 165 insertions(+), 107 deletions(-)
diff --git a/TwitchDownloaderCore/Chat/ChatHtml.cs b/TwitchDownloaderCore/Chat/ChatHtml.cs
index f00d51d0..96272477 100644
--- a/TwitchDownloaderCore/Chat/ChatHtml.cs
+++ b/TwitchDownloaderCore/Chat/ChatHtml.cs
@@ -18,10 +18,8 @@ public static class ChatHtml
///
/// Serializes a chat Html file.
///
- public static async Task SerializeAsync(string filePath, ChatRoot chatRoot, ITaskLogger logger, bool embedData = true, CancellationToken cancellationToken = default)
+ public static async Task SerializeAsync(FileStream fileStream, string filePath, ChatRoot chatRoot, ITaskLogger logger, bool embedData = true, CancellationToken cancellationToken = default)
{
- ArgumentNullException.ThrowIfNull(filePath, nameof(filePath));
-
Dictionary thirdEmoteData = new();
await BuildThirdPartyDictionary(chatRoot, embedData, thirdEmoteData, logger, cancellationToken);
@@ -35,14 +33,7 @@ public static async Task SerializeAsync(string filePath, ChatRoot chatRoot, ITas
using var templateStream = new MemoryStream(Properties.Resources.chat_template);
using var templateReader = new StreamReader(templateStream);
- var outputDirectory = Directory.GetParent(Path.GetFullPath(filePath))!;
- if (!outputDirectory.Exists)
- {
- TwitchHelper.CreateDirectory(outputDirectory.FullName);
- }
-
- await using var fs = File.Open(filePath, FileMode.Create, FileAccess.Write, FileShare.Read);
- await using var sw = new StreamWriter(fs);
+ await using var sw = new StreamWriter(fileStream);
while (!templateReader.EndOfStream)
{
diff --git a/TwitchDownloaderCore/Chat/ChatJson.cs b/TwitchDownloaderCore/Chat/ChatJson.cs
index af80a634..137f67b4 100644
--- a/TwitchDownloaderCore/Chat/ChatJson.cs
+++ b/TwitchDownloaderCore/Chat/ChatJson.cs
@@ -244,25 +244,16 @@ private static async Task UpgradeChatJson(ChatRoot chatRoot)
///
/// Asynchronously serializes a chat json file.
///
- public static async Task SerializeAsync(string filePath, ChatRoot chatRoot, ChatCompression compression, CancellationToken cancellationToken)
+ public static async Task SerializeAsync(FileStream fileStream, ChatRoot chatRoot, ChatCompression compression, CancellationToken cancellationToken)
{
- ArgumentNullException.ThrowIfNull(chatRoot, nameof(chatRoot));
-
- var outputDirectory = Directory.GetParent(Path.GetFullPath(filePath))!;
- if (!outputDirectory.Exists)
- {
- TwitchHelper.CreateDirectory(outputDirectory.FullName);
- }
-
- await using var fs = File.Open(filePath, FileMode.Create, FileAccess.Write, FileShare.Read);
switch (compression)
{
case ChatCompression.None:
- await JsonSerializer.SerializeAsync(fs, chatRoot, _jsonSerializerOptions, cancellationToken);
+ await JsonSerializer.SerializeAsync(fileStream, chatRoot, _jsonSerializerOptions, cancellationToken);
break;
case ChatCompression.Gzip:
{
- await using var gs = new GZipStream(fs, CompressionLevel.SmallestSize);
+ await using var gs = new GZipStream(fileStream, CompressionLevel.SmallestSize);
await JsonSerializer.SerializeAsync(gs, chatRoot, _jsonSerializerOptions, cancellationToken);
break;
}
diff --git a/TwitchDownloaderCore/Chat/ChatText.cs b/TwitchDownloaderCore/Chat/ChatText.cs
index 31930e81..9b6f66ad 100644
--- a/TwitchDownloaderCore/Chat/ChatText.cs
+++ b/TwitchDownloaderCore/Chat/ChatText.cs
@@ -11,17 +11,9 @@ public static class ChatText
///
/// Serializes a chat plain text file.
///
- public static async Task SerializeAsync(string filePath, ChatRoot chatRoot, TimestampFormat timeFormat)
+ public static async Task SerializeAsync(FileStream fileStream, ChatRoot chatRoot, TimestampFormat timeFormat)
{
- ArgumentNullException.ThrowIfNull(filePath, nameof(filePath));
-
- var outputDirectory = Directory.GetParent(Path.GetFullPath(filePath))!;
- if (!outputDirectory.Exists)
- {
- TwitchHelper.CreateDirectory(outputDirectory.FullName);
- }
-
- await using var sw = new StreamWriter(filePath);
+ await using var sw = new StreamWriter(fileStream);
foreach (var comment in chatRoot.comments)
{
var username = comment.commenter.display_name;
diff --git a/TwitchDownloaderCore/ChatDownloader.cs b/TwitchDownloaderCore/ChatDownloader.cs
index d2cffeb4..c9f916bc 100644
--- a/TwitchDownloaderCore/ChatDownloader.cs
+++ b/TwitchDownloaderCore/ChatDownloader.cs
@@ -251,6 +251,16 @@ public async Task DownloadAsync(CancellationToken cancellationToken)
throw new NullReferenceException("Null or empty video/clip ID");
}
+ var outputFileInfo = TwitchHelper.ClaimFile(downloadOptions.Filename, downloadOptions.FileOverwriteCallback, _progress);
+ if (outputFileInfo is null)
+ {
+ _progress.LogWarning("No destination file was provided, aborting.");
+ return;
+ }
+
+ // Open the destination file so that it exists in the filesystem.
+ await using var outputFs = outputFileInfo.Open(FileMode.Create, FileAccess.Write, FileShare.Read);
+
DownloadType downloadType = downloadOptions.Id.All(char.IsDigit) ? DownloadType.Video : DownloadType.Clip;
ChatRoot chatRoot = new()
@@ -519,13 +529,13 @@ public async Task DownloadAsync(CancellationToken cancellationToken)
switch (downloadOptions.DownloadFormat)
{
case ChatFormat.Json:
- await ChatJson.SerializeAsync(downloadOptions.Filename, chatRoot, downloadOptions.Compression, cancellationToken);
+ await ChatJson.SerializeAsync(outputFs, chatRoot, downloadOptions.Compression, cancellationToken);
break;
case ChatFormat.Html:
- await ChatHtml.SerializeAsync(downloadOptions.Filename, chatRoot, _progress, downloadOptions.EmbedData, cancellationToken);
+ await ChatHtml.SerializeAsync(outputFs, outputFileInfo.FullName, chatRoot, _progress, downloadOptions.EmbedData, cancellationToken);
break;
case ChatFormat.Text:
- await ChatText.SerializeAsync(downloadOptions.Filename, chatRoot, downloadOptions.TimeFormat);
+ await ChatText.SerializeAsync(outputFs, chatRoot, downloadOptions.TimeFormat);
break;
default:
throw new NotSupportedException($"{downloadOptions.DownloadFormat} is not a supported output format.");
diff --git a/TwitchDownloaderCore/ChatRenderer.cs b/TwitchDownloaderCore/ChatRenderer.cs
index 99850a26..a2def73b 100644
--- a/TwitchDownloaderCore/ChatRenderer.cs
+++ b/TwitchDownloaderCore/ChatRenderer.cs
@@ -71,6 +71,24 @@ public ChatRenderer(ChatRenderOptions chatRenderOptions, ITaskProgress progress)
public async Task RenderVideoAsync(CancellationToken cancellationToken)
{
+ var outputFileInfo = TwitchHelper.ClaimFile(renderOptions.OutputFile, renderOptions.FileOverwriteCallback, _progress);
+ var maskFileInfo = renderOptions.GenerateMask ? TwitchHelper.ClaimFile(renderOptions.MaskFile, renderOptions.FileOverwriteCallback, _progress) : null;
+ if (outputFileInfo is null)
+ {
+ _progress.LogWarning("No destination file was provided, aborting.");
+ return;
+ }
+
+ if (renderOptions.GenerateMask && maskFileInfo is null)
+ {
+ _progress.LogWarning("No mask file was provided, aborting.");
+ return;
+ }
+
+ // Open the destination files so that they exist in the filesystem.
+ await using var outputFs = outputFileInfo.Open(FileMode.Create, FileAccess.Write, FileShare.Read);
+ await using var maskFs = maskFileInfo?.Open(FileMode.Create, FileAccess.Write, FileShare.Read);
+
_progress.SetStatus("Fetching Images [1/2]");
await Task.Run(() => FetchScaledImages(cancellationToken), cancellationToken);
@@ -107,20 +125,17 @@ public async Task RenderVideoAsync(CancellationToken cancellationToken)
(int startTick, int totalTicks) = GetVideoTicks();
- var renderFileDirectory = Directory.GetParent(Path.GetFullPath(renderOptions.OutputFile))!;
- if (!renderFileDirectory.Exists)
- {
- TwitchHelper.CreateDirectory(renderFileDirectory.FullName);
- }
-
- if (File.Exists(renderOptions.OutputFile))
- File.Delete(renderOptions.OutputFile);
+ // Delete the files as it is not guaranteed that the overwrite flag is passed in the FFmpeg args.
+ outputFs.Close();
+ if (outputFileInfo.Exists)
+ outputFileInfo.Delete();
- if (renderOptions.GenerateMask && File.Exists(renderOptions.MaskFile))
- File.Delete(renderOptions.MaskFile);
+ maskFs?.Close();
+ if (renderOptions.GenerateMask && maskFileInfo!.Exists)
+ maskFileInfo.Delete();
- FfmpegProcess ffmpegProcess = GetFfmpegProcess(0, false);
- FfmpegProcess maskProcess = renderOptions.GenerateMask ? GetFfmpegProcess(0, true) : null;
+ FfmpegProcess ffmpegProcess = GetFfmpegProcess(outputFileInfo);
+ FfmpegProcess maskProcess = renderOptions.GenerateMask ? GetFfmpegProcess(maskFileInfo) : null;
_progress.SetTemplateStatus(@"Rendering Video {0}% ({1:h\hm\ms\s} Elapsed | {2:h\hm\ms\s} Remaining)", 0, TimeSpan.Zero, TimeSpan.Zero);
try
@@ -323,22 +338,9 @@ private static void SetFrameMask(SKBitmap frame)
}
}
- private FfmpegProcess GetFfmpegProcess(int partNumber, bool isMask)
+ private FfmpegProcess GetFfmpegProcess(FileInfo fileInfo)
{
- string savePath;
- if (partNumber == 0)
- {
- if (isMask)
- savePath = renderOptions.MaskFile;
- else
- savePath = renderOptions.OutputFile;
- }
- else
- {
- savePath = Path.Combine(renderOptions.TempFolder, Path.GetRandomFileName() + (isMask ? "_mask" : "") + Path.GetExtension(renderOptions.OutputFile));
- }
-
- savePath = Path.GetFullPath(savePath);
+ string savePath = fileInfo.FullName;
string inputArgs = new StringBuilder(renderOptions.InputArgs)
.Replace("{fps}", renderOptions.Framerate.ToString())
diff --git a/TwitchDownloaderCore/ChatUpdater.cs b/TwitchDownloaderCore/ChatUpdater.cs
index 8e99366b..3317b06c 100644
--- a/TwitchDownloaderCore/ChatUpdater.cs
+++ b/TwitchDownloaderCore/ChatUpdater.cs
@@ -32,6 +32,16 @@ public ChatUpdater(ChatUpdateOptions updateOptions, ITaskProgress progress)
public async Task UpdateAsync(CancellationToken cancellationToken)
{
+ var outputFileInfo = TwitchHelper.ClaimFile(_updateOptions.OutputFile, _updateOptions.FileOverwriteCallback, _progress);
+ if (outputFileInfo is null)
+ {
+ _progress.LogWarning("No destination file was provided, aborting.");
+ return;
+ }
+
+ // Open the destination file so that it exists in the filesystem.
+ await using var outputFs = outputFileInfo.Open(FileMode.Create, FileAccess.Write, FileShare.Read);
+
chatRoot.FileInfo = new() { Version = ChatRootVersion.CurrentVersion, CreatedAt = chatRoot.FileInfo.CreatedAt, UpdatedAt = DateTime.Now };
if (!Path.GetExtension(_updateOptions.InputFile.Replace(".gz", ""))!.Equals(".json", StringComparison.OrdinalIgnoreCase))
{
@@ -70,13 +80,13 @@ public async Task UpdateAsync(CancellationToken cancellationToken)
switch (_updateOptions.OutputFormat)
{
case ChatFormat.Json:
- await ChatJson.SerializeAsync(_updateOptions.OutputFile, chatRoot, _updateOptions.Compression, cancellationToken);
+ await ChatJson.SerializeAsync(outputFs, chatRoot, _updateOptions.Compression, cancellationToken);
break;
case ChatFormat.Html:
- await ChatHtml.SerializeAsync(_updateOptions.OutputFile, chatRoot, _progress, chatRoot.embeddedData != null && (chatRoot.embeddedData.firstParty?.Count > 0 || chatRoot.embeddedData.twitchBadges?.Count > 0), cancellationToken);
+ await ChatHtml.SerializeAsync(outputFs, outputFileInfo.FullName, chatRoot, _progress, chatRoot.embeddedData != null && (chatRoot.embeddedData.firstParty?.Count > 0 || chatRoot.embeddedData.twitchBadges?.Count > 0), cancellationToken);
break; // If there is embedded data, it's almost guaranteed to be first party emotes or badges.
case ChatFormat.Text:
- await ChatText.SerializeAsync(_updateOptions.OutputFile, chatRoot, _updateOptions.TextTimestampFormat);
+ await ChatText.SerializeAsync(outputFs, chatRoot, _updateOptions.TextTimestampFormat);
break;
default:
throw new NotSupportedException($"{_updateOptions.OutputFormat} is not a supported output format.");
diff --git a/TwitchDownloaderCore/ClipDownloader.cs b/TwitchDownloaderCore/ClipDownloader.cs
index a6171eda..90db73dd 100644
--- a/TwitchDownloaderCore/ClipDownloader.cs
+++ b/TwitchDownloaderCore/ClipDownloader.cs
@@ -31,6 +31,17 @@ public ClipDownloader(ClipDownloadOptions clipDownloadOptions, ITaskProgress pro
public async Task DownloadAsync(CancellationToken cancellationToken)
{
+
+ var outputFileInfo = TwitchHelper.ClaimFile(downloadOptions.Filename, downloadOptions.FileOverwriteCallback, _progress);
+ if (outputFileInfo is null)
+ {
+ _progress.LogWarning("No destination file was provided, aborting.");
+ return;
+ }
+
+ // Open the destination file so that it exists in the filesystem.
+ await using var outputFs = outputFileInfo.Open(FileMode.Create, FileAccess.Write, FileShare.Read);
+
_progress.SetStatus("Fetching Clip Info");
var downloadUrl = await GetDownloadUrl();
@@ -38,23 +49,11 @@ public async Task DownloadAsync(CancellationToken cancellationToken)
cancellationToken.ThrowIfCancellationRequested();
- var clipDirectory = Directory.GetParent(Path.GetFullPath(downloadOptions.Filename))!;
- if (!clipDirectory.Exists)
- {
- TwitchHelper.CreateDirectory(clipDirectory.FullName);
- }
-
_progress.SetTemplateStatus("Downloading Clip {0}%", 0);
- void DownloadProgressHandler(StreamCopyProgress streamProgress)
- {
- var percent = (int)(streamProgress.BytesCopied / (double)streamProgress.SourceLength * 100);
- _progress.ReportProgress(percent);
- }
-
if (!downloadOptions.EncodeMetadata)
{
- await DownloadFileTaskAsync(downloadUrl, downloadOptions.Filename, downloadOptions.ThrottleKib, new Progress(DownloadProgressHandler), cancellationToken);
+ await DownloadFileTaskAsync(downloadUrl, outputFs, downloadOptions.ThrottleKib, new Progress(DownloadProgressHandler), cancellationToken);
return;
}
@@ -66,16 +65,21 @@ void DownloadProgressHandler(StreamCopyProgress streamProgress)
var tempFile = Path.Combine(downloadOptions.TempFolder, $"{downloadOptions.Id}_{DateTimeOffset.UtcNow.Ticks}.mp4");
try
{
- await DownloadFileTaskAsync(downloadUrl, tempFile, downloadOptions.ThrottleKib, new Progress(DownloadProgressHandler), cancellationToken);
+ await using (var tempFileStream = File.Open(tempFile, FileMode.Create, FileAccess.Write, FileShare.Read))
+ {
+ await DownloadFileTaskAsync(downloadUrl, tempFileStream, downloadOptions.ThrottleKib, new Progress(DownloadProgressHandler), cancellationToken);
+ }
+
+ outputFs.Close();
_progress.SetTemplateStatus("Encoding Clip Metadata {0}%", 0);
var clipChapter = TwitchHelper.GenerateClipChapter(clipInfo.data.clip);
- await EncodeClipWithMetadata(tempFile, downloadOptions.Filename, clipInfo.data.clip, clipChapter, cancellationToken);
+ await EncodeClipWithMetadata(tempFile, outputFileInfo.FullName, clipInfo.data.clip, clipChapter, cancellationToken);
- if (!File.Exists(downloadOptions.Filename))
+ if (!outputFileInfo.Exists)
{
- File.Move(tempFile, downloadOptions.Filename);
+ File.Move(tempFile, outputFileInfo.FullName);
_progress.LogError("Unable to serialize metadata. The download has been completed without custom metadata.");
}
@@ -85,6 +89,12 @@ void DownloadProgressHandler(StreamCopyProgress streamProgress)
{
File.Delete(tempFile);
}
+
+ void DownloadProgressHandler(StreamCopyProgress streamProgress)
+ {
+ var percent = (int)(streamProgress.BytesCopied / (double)streamProgress.SourceLength * 100);
+ _progress.ReportProgress(percent);
+ }
}
private async Task GetDownloadUrl()
@@ -120,7 +130,7 @@ private async Task GetDownloadUrl()
return downloadUrl + "?sig=" + clip.playbackAccessToken.signature + "&token=" + HttpUtility.UrlEncode(clip.playbackAccessToken.value);
}
- private static async Task DownloadFileTaskAsync(string url, string destinationFile, int throttleKib, IProgress progress, CancellationToken cancellationToken)
+ private static async Task DownloadFileTaskAsync(string url, FileStream fs, int throttleKib, IProgress progress, CancellationToken cancellationToken)
{
var request = new HttpRequestMessage(HttpMethod.Get, url);
using var response = await HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
@@ -130,13 +140,11 @@ private static async Task DownloadFileTaskAsync(string url, string destinationFi
if (throttleKib == -1)
{
- await using var fs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read);
await using var contentStream = await response.Content.ReadAsStreamAsync(cancellationToken);
await contentStream.ProgressCopyToAsync(fs, contentLength, progress, cancellationToken).ConfigureAwait(false);
}
else
{
- await using var fs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read);
await using var contentStream = await response.Content.ReadAsStreamAsync(cancellationToken);
await using var throttledStream = new ThrottledStream(contentStream, throttleKib);
await throttledStream.ProgressCopyToAsync(fs, contentLength, progress, cancellationToken).ConfigureAwait(false);
diff --git a/TwitchDownloaderCore/Options/ChatDownloadOptions.cs b/TwitchDownloaderCore/Options/ChatDownloadOptions.cs
index bf6ee5ca..3b5118dc 100644
--- a/TwitchDownloaderCore/Options/ChatDownloadOptions.cs
+++ b/TwitchDownloaderCore/Options/ChatDownloadOptions.cs
@@ -1,4 +1,6 @@
-using TwitchDownloaderCore.Tools;
+using System;
+using System.IO;
+using TwitchDownloaderCore.Tools;
namespace TwitchDownloaderCore.Options
{
@@ -34,5 +36,6 @@ public string FileExtension
}
}
public string TempFolder { get; set; }
+ public Func FileOverwriteCallback { get; set; } = info => info;
}
}
diff --git a/TwitchDownloaderCore/Options/ChatRenderOptions.cs b/TwitchDownloaderCore/Options/ChatRenderOptions.cs
index bdfc1883..6a049121 100644
--- a/TwitchDownloaderCore/Options/ChatRenderOptions.cs
+++ b/TwitchDownloaderCore/Options/ChatRenderOptions.cs
@@ -92,5 +92,6 @@ public string MaskFile
public EmojiVendor EmojiVendor { get; set; } = EmojiVendor.GoogleNotoColor;
public int[] TimestampWidths { get; set; }
public bool AdjustUsernameVisibility { get; set; }
+ public Func FileOverwriteCallback { get; set; } = info => info;
}
}
diff --git a/TwitchDownloaderCore/Options/ChatUpdateOptions.cs b/TwitchDownloaderCore/Options/ChatUpdateOptions.cs
index 13333990..9e9b3332 100644
--- a/TwitchDownloaderCore/Options/ChatUpdateOptions.cs
+++ b/TwitchDownloaderCore/Options/ChatUpdateOptions.cs
@@ -1,4 +1,6 @@
-using TwitchDownloaderCore.Tools;
+using System;
+using System.IO;
+using TwitchDownloaderCore.Tools;
namespace TwitchDownloaderCore.Options
{
@@ -33,5 +35,6 @@ public string FileExtension
}
}
public string TempFolder { get; set; }
+ public Func FileOverwriteCallback { get; set; } = info => info;
}
}
diff --git a/TwitchDownloaderCore/Options/ClipDownloadOptions.cs b/TwitchDownloaderCore/Options/ClipDownloadOptions.cs
index 9f8c6ce9..3a0a1620 100644
--- a/TwitchDownloaderCore/Options/ClipDownloadOptions.cs
+++ b/TwitchDownloaderCore/Options/ClipDownloadOptions.cs
@@ -1,4 +1,7 @@
-namespace TwitchDownloaderCore.Options
+using System;
+using System.IO;
+
+namespace TwitchDownloaderCore.Options
{
public class ClipDownloadOptions
{
@@ -9,5 +12,6 @@ public class ClipDownloadOptions
public string TempFolder { get; set; }
public bool EncodeMetadata { get; set; }
public string FfmpegPath { get; set; }
+ public Func FileOverwriteCallback { get; set; } = info => info;
}
}
\ No newline at end of file
diff --git a/TwitchDownloaderCore/Options/TsMergeOptions.cs b/TwitchDownloaderCore/Options/TsMergeOptions.cs
index 8dd41e29..e44da205 100644
--- a/TwitchDownloaderCore/Options/TsMergeOptions.cs
+++ b/TwitchDownloaderCore/Options/TsMergeOptions.cs
@@ -1,8 +1,12 @@
-namespace TwitchDownloaderCore.Options
+using System;
+using System.IO;
+
+namespace TwitchDownloaderCore.Options
{
public class TsMergeOptions
{
public string OutputFile { get; set; }
public string InputFile { get; set; }
+ public Func FileOverwriteCallback { get; set; } = info => info;
}
}
diff --git a/TwitchDownloaderCore/Options/VideoDownloadOptions.cs b/TwitchDownloaderCore/Options/VideoDownloadOptions.cs
index 19151a58..70df7f0b 100644
--- a/TwitchDownloaderCore/Options/VideoDownloadOptions.cs
+++ b/TwitchDownloaderCore/Options/VideoDownloadOptions.cs
@@ -18,5 +18,6 @@ public class VideoDownloadOptions
public string FfmpegPath { get; set; }
public string TempFolder { get; set; }
public Func CacheCleanerCallback { get; set; }
+ public Func FileOverwriteCallback { get; set; } = info => info;
}
-}
+}
\ No newline at end of file
diff --git a/TwitchDownloaderCore/TsMerger.cs b/TwitchDownloaderCore/TsMerger.cs
index 16d01f6d..eda1693a 100644
--- a/TwitchDownloaderCore/TsMerger.cs
+++ b/TwitchDownloaderCore/TsMerger.cs
@@ -26,6 +26,16 @@ public async Task MergeAsync(CancellationToken cancellationToken)
throw new FileNotFoundException("Input file does not exist");
}
+ var outputFileInfo = TwitchHelper.ClaimFile(mergeOptions.OutputFile, mergeOptions.FileOverwriteCallback, _progress);
+ if (outputFileInfo is null)
+ {
+ _progress.LogWarning("No destination file was provided, aborting.");
+ return;
+ }
+
+ // Open the destination file so that it exists in the filesystem.
+ await using var outputFs = outputFileInfo.Open(FileMode.Create, FileAccess.Write, FileShare.Read);
+
var isM3U8 = false;
var fileList = new List();
await using (var fs = File.Open(mergeOptions.InputFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
@@ -54,7 +64,7 @@ public async Task MergeAsync(CancellationToken cancellationToken)
_progress.SetTemplateStatus("Combining Parts {0}% [2/2]", 0);
- await CombineVideoParts(fileList, cancellationToken);
+ await CombineVideoParts(fileList, outputFs, cancellationToken);
_progress.ReportProgress(100);
}
@@ -111,7 +121,7 @@ private static async Task VerifyVideoPart(string filePath)
return true;
}
- private async Task CombineVideoParts(IReadOnlyCollection fileList, CancellationToken cancellationToken)
+ private async Task CombineVideoParts(IReadOnlyCollection fileList, FileStream outputStream, CancellationToken cancellationToken)
{
DriveInfo outputDrive = DriveHelper.GetOutputDrive(mergeOptions.OutputFile);
string outputFile = mergeOptions.OutputFile;
@@ -119,7 +129,6 @@ private async Task CombineVideoParts(IReadOnlyCollection fileList, Cance
int partCount = fileList.Count;
int doneCount = 0;
- await using var outputStream = new FileStream(outputFile, FileMode.Create, FileAccess.Write, FileShare.Read);
foreach (var partFile in fileList)
{
await DriveHelper.WaitForDrive(outputDrive, _progress, cancellationToken);
diff --git a/TwitchDownloaderCore/TwitchHelper.cs b/TwitchDownloaderCore/TwitchHelper.cs
index 03dbbf81..303554d4 100644
--- a/TwitchDownloaderCore/TwitchHelper.cs
+++ b/TwitchDownloaderCore/TwitchHelper.cs
@@ -1,7 +1,7 @@
using SkiaSharp;
using System;
-using System.Buffers;
using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.IO.Compression;
using System.Linq;
@@ -854,6 +854,31 @@ where comments
return returnList;
}
+ [return: MaybeNull]
+ public static FileInfo ClaimFile(string path, Func fileAlreadyExistsCallback, ITaskLogger logger)
+ {
+ var fileInfo = new FileInfo(path);
+ if (fileInfo.Exists)
+ {
+ if (fileAlreadyExistsCallback is null)
+ {
+ logger.LogWarning($"{nameof(fileAlreadyExistsCallback)} was null.");
+ }
+ else
+ {
+ fileInfo = fileAlreadyExistsCallback(fileInfo);
+ }
+ }
+
+ var directory = fileInfo.Directory;
+ if (directory is not null && !directory.Exists)
+ {
+ CreateDirectory(directory.FullName);
+ }
+
+ return fileInfo;
+ }
+
public static DirectoryInfo CreateDirectory(string path)
{
DirectoryInfo directoryInfo = Directory.CreateDirectory(path);
diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs
index 0262f5dc..0a8439b7 100644
--- a/TwitchDownloaderCore/VideoDownloader.cs
+++ b/TwitchDownloaderCore/VideoDownloader.cs
@@ -39,6 +39,16 @@ public VideoDownloader(VideoDownloadOptions videoDownloadOptions, ITaskProgress
public async Task DownloadAsync(CancellationToken cancellationToken)
{
+ var outputFileInfo = TwitchHelper.ClaimFile(downloadOptions.Filename, downloadOptions.FileOverwriteCallback, _progress);
+ if (outputFileInfo is null)
+ {
+ _progress.LogWarning("No destination file was provided, aborting.");
+ return;
+ }
+
+ // Open the destination file so that it exists in the filesystem.
+ await using var outputFs = outputFileInfo.Open(FileMode.Create, FileAccess.Write, FileShare.Read);
+
await TwitchHelper.CleanupAbandonedVideoCaches(downloadOptions.TempFolder, downloadOptions.CacheCleanerCallback, _progress);
string downloadFolder = Path.Combine(
@@ -49,8 +59,6 @@ public async Task DownloadAsync(CancellationToken cancellationToken)
try
{
- ServicePointManager.DefaultConnectionLimit = downloadOptions.DownloadThreads;
-
GqlVideoResponse videoInfoResponse = await TwitchHelper.GetVideoInfo(downloadOptions.Id);
if (videoInfoResponse.data.video == null)
{
@@ -100,17 +108,13 @@ await FfmpegMetadata.SerializeAsync(metadataPath, videoInfo.owner.displayName, d
videoInfo.description?.Replace(" \n", "\n").Replace("\n\n", "\n").TrimEnd(), downloadOptions.TrimBeginning ? downloadOptions.TrimBeginningTime : TimeSpan.Zero,
videoChapterResponse.data.video.moments.edges, cancellationToken);
- var finalizedFileDirectory = Directory.GetParent(Path.GetFullPath(downloadOptions.Filename))!;
- if (!finalizedFileDirectory.Exists)
- {
- TwitchHelper.CreateDirectory(finalizedFileDirectory.FullName);
- }
+ outputFs.Close();
int ffmpegExitCode;
var ffmpegRetries = 0;
do
{
- ffmpegExitCode = await Task.Run(() => RunFfmpegVideoCopy(downloadFolder, metadataPath, startOffset, seekDuration > TimeSpan.Zero ? seekDuration : videoLength), cancellationToken);
+ ffmpegExitCode = await Task.Run(() => RunFfmpegVideoCopy(downloadFolder, outputFileInfo, metadataPath, startOffset, seekDuration > TimeSpan.Zero ? seekDuration : videoLength), cancellationToken);
if (ffmpegExitCode != 0)
{
_progress.LogError($"Failed to finalize video (code {ffmpegExitCode}), retrying in 10 seconds...");
@@ -118,7 +122,7 @@ await FfmpegMetadata.SerializeAsync(metadataPath, videoInfo.owner.displayName, d
}
} while (ffmpegExitCode != 0 && ffmpegRetries++ < 1);
- if (ffmpegExitCode != 0 || !File.Exists(downloadOptions.Filename))
+ if (ffmpegExitCode != 0 || !outputFileInfo.Exists)
{
_shouldClearCache = false;
throw new Exception($"Failed to finalize video. The download cache has not been cleared and can be found at {downloadFolder} along with a log file.");
@@ -329,7 +333,7 @@ private static bool VerifyVideoPart(string filePath)
return true;
}
- private int RunFfmpegVideoCopy(string downloadFolder, string metadataPath, TimeSpan startOffset, TimeSpan seekDuration)
+ private int RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string metadataPath, TimeSpan startOffset, TimeSpan seekDuration)
{
var process = new Process
{
@@ -338,7 +342,7 @@ private int RunFfmpegVideoCopy(string downloadFolder, string metadataPath, TimeS
FileName = downloadOptions.FfmpegPath,
Arguments = string.Format(
"-hide_banner -stats -y -avoid_negative_ts make_zero " + (downloadOptions.TrimBeginning ? "-ss {2} " : "") + "-i \"{0}\" -i \"{1}\" -map_metadata 1 -analyzeduration {3} -probesize {3} " + (downloadOptions.TrimEnding ? "-t {4} " : "") + "-c:v copy \"{5}\"",
- Path.Combine(downloadFolder, "output.ts"), metadataPath, startOffset.TotalSeconds.ToString(CultureInfo.InvariantCulture), int.MaxValue, seekDuration.TotalSeconds.ToString(CultureInfo.InvariantCulture), Path.GetFullPath(downloadOptions.Filename)),
+ Path.Combine(tempFolder, "output.ts"), metadataPath, startOffset.TotalSeconds.ToString(CultureInfo.InvariantCulture), int.MaxValue, seekDuration.TotalSeconds.ToString(CultureInfo.InvariantCulture), outputFile.FullName),
UseShellExecute = false,
CreateNoWindow = true,
RedirectStandardInput = false,
@@ -363,7 +367,7 @@ private int RunFfmpegVideoCopy(string downloadFolder, string metadataPath, TimeS
process.Start();
process.BeginErrorReadLine();
- using var logWriter = File.AppendText(Path.Combine(downloadFolder, "ffmpegLog.txt"));
+ using var logWriter = File.AppendText(Path.Combine(tempFolder, "ffmpegLog.txt"));
do // We cannot handle logging inside the ErrorDataReceived lambda because more than 1 can come in at once and cause a race condition. lay295#598
{
Thread.Sleep(100);
From c89531d4d2633119724daaef23b1e72544505fd0 Mon Sep 17 00:00:00 2001
From: ScrubN <72096833+ScrubN@users.noreply.github.com>
Date: Wed, 5 Jun 2024 00:47:47 -0400
Subject: [PATCH 02/21] Fix potential NRE
---
TwitchDownloaderCore/TwitchHelper.cs | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/TwitchDownloaderCore/TwitchHelper.cs b/TwitchDownloaderCore/TwitchHelper.cs
index 303554d4..5ee7ddec 100644
--- a/TwitchDownloaderCore/TwitchHelper.cs
+++ b/TwitchDownloaderCore/TwitchHelper.cs
@@ -867,10 +867,11 @@ public static FileInfo ClaimFile(string path, Func fileAlrea
else
{
fileInfo = fileAlreadyExistsCallback(fileInfo);
+ logger.LogVerbose($"{path} will be renamed to {fileInfo?.FullName}.");
}
}
- var directory = fileInfo.Directory;
+ var directory = fileInfo?.Directory;
if (directory is not null && !directory.Exists)
{
CreateDirectory(directory.FullName);
From fc6b4aab4d865b551e711f5bb592e7259f668a7b Mon Sep 17 00:00:00 2001
From: ScrubN <72096833+ScrubN@users.noreply.github.com>
Date: Wed, 5 Jun 2024 00:51:12 -0400
Subject: [PATCH 03/21] Implement FileOverwriteHandler for CLI
---
.../Models/{LogLevel.cs => Enums.cs} | 8 +++
.../Models/IFileOverwriteArgs.cs | 10 ++++
.../Modes/Arguments/ChatDownloadArgs.cs | 4 +-
.../Modes/Arguments/ChatRenderArgs.cs | 4 +-
.../Modes/Arguments/ChatUpdateArgs.cs | 4 +-
.../Modes/Arguments/ClipDownloadArgs.cs | 5 +-
.../Modes/Arguments/TsMergeArgs.cs | 5 +-
.../Modes/Arguments/VideoDownloadArgs.cs | 4 +-
TwitchDownloaderCLI/Modes/DownloadClip.cs | 8 +--
TwitchDownloaderCLI/Modes/DownloadVideo.cs | 8 +--
.../Tools/FileOverwriteHandler.cs | 52 +++++++++++++++++++
TwitchDownloaderCore/Tools/FilenameService.cs | 19 +++++++
12 files changed, 119 insertions(+), 12 deletions(-)
rename TwitchDownloaderCLI/Models/{LogLevel.cs => Enums.cs} (71%)
create mode 100644 TwitchDownloaderCLI/Models/IFileOverwriteArgs.cs
create mode 100644 TwitchDownloaderCLI/Tools/FileOverwriteHandler.cs
diff --git a/TwitchDownloaderCLI/Models/LogLevel.cs b/TwitchDownloaderCLI/Models/Enums.cs
similarity index 71%
rename from TwitchDownloaderCLI/Models/LogLevel.cs
rename to TwitchDownloaderCLI/Models/Enums.cs
index e472e17a..6b6cd172 100644
--- a/TwitchDownloaderCLI/Models/LogLevel.cs
+++ b/TwitchDownloaderCLI/Models/Enums.cs
@@ -13,4 +13,12 @@ internal enum LogLevel
Error = 1 << 5,
Ffmpeg = 1 << 6,
}
+
+ public enum OverwriteBehavior
+ {
+ Overwrite,
+ Exit,
+ Rename,
+ Prompt,
+ }
}
\ No newline at end of file
diff --git a/TwitchDownloaderCLI/Models/IFileOverwriteArgs.cs b/TwitchDownloaderCLI/Models/IFileOverwriteArgs.cs
new file mode 100644
index 00000000..d31f2d7f
--- /dev/null
+++ b/TwitchDownloaderCLI/Models/IFileOverwriteArgs.cs
@@ -0,0 +1,10 @@
+using CommandLine;
+
+namespace TwitchDownloaderCLI.Models
+{
+ public interface IFileOverwriteArgs
+ {
+ [Option("overwrite", Default = OverwriteBehavior.Prompt, HelpText = ". Valid values are: Overwrite, Exit, Rename, Prompt.")]
+ public OverwriteBehavior OverwriteBehavior { get; set; }
+ }
+}
\ No newline at end of file
diff --git a/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs
index 2d4eaadb..5512a965 100644
--- a/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs
+++ b/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs
@@ -5,7 +5,7 @@
namespace TwitchDownloaderCLI.Modes.Arguments
{
[Verb("chatdownload", HelpText = "Downloads the chat from a VOD or clip")]
- internal sealed class ChatDownloadArgs : TwitchDownloaderArgs
+ internal sealed class ChatDownloadArgs : TwitchDownloaderArgs, IFileOverwriteArgs
{
[Option('u', "id", Required = true, HelpText = "The ID or URL of the VOD or clip to download that chat of.")]
public string Id { get; set; }
@@ -45,5 +45,7 @@ internal sealed class ChatDownloadArgs : TwitchDownloaderArgs
[Option("temp-path", Default = "", HelpText = "Path to temporary folder to use for cache.")]
public string TempFolder { get; set; }
+
+ public OverwriteBehavior OverwriteBehavior { get; set; }
}
}
\ No newline at end of file
diff --git a/TwitchDownloaderCLI/Modes/Arguments/ChatRenderArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/ChatRenderArgs.cs
index f8afe990..c34317dd 100644
--- a/TwitchDownloaderCLI/Modes/Arguments/ChatRenderArgs.cs
+++ b/TwitchDownloaderCLI/Modes/Arguments/ChatRenderArgs.cs
@@ -4,7 +4,7 @@
namespace TwitchDownloaderCLI.Modes.Arguments
{
[Verb("chatrender", HelpText = "Renders a chat JSON as a video")]
- internal sealed class ChatRenderArgs : TwitchDownloaderArgs
+ internal sealed class ChatRenderArgs : TwitchDownloaderArgs, IFileOverwriteArgs
{
[Option('i', "input", Required = true, HelpText = "Path to JSON chat file input.")]
public string InputFile { get; set; }
@@ -152,5 +152,7 @@ internal sealed class ChatRenderArgs : TwitchDownloaderArgs
[Option("scale-highlight-indent", Default = 1.0, HelpText = "Number to scale highlight indent size (sub messages).")]
public double ScaleAccentIndent { get; set; }
+
+ public OverwriteBehavior OverwriteBehavior { get; set; }
}
}
\ No newline at end of file
diff --git a/TwitchDownloaderCLI/Modes/Arguments/ChatUpdateArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/ChatUpdateArgs.cs
index 7818dc6d..8aa8405f 100644
--- a/TwitchDownloaderCLI/Modes/Arguments/ChatUpdateArgs.cs
+++ b/TwitchDownloaderCLI/Modes/Arguments/ChatUpdateArgs.cs
@@ -5,7 +5,7 @@
namespace TwitchDownloaderCLI.Modes.Arguments
{
[Verb("chatupdate", HelpText = "Updates the embedded emotes, badges, bits, and trims a chat JSON and/or converts a JSON chat to another format.")]
- internal sealed class ChatUpdateArgs : TwitchDownloaderArgs
+ internal sealed class ChatUpdateArgs : TwitchDownloaderArgs, IFileOverwriteArgs
{
[Option('i', "input", Required = true, HelpText = "Path to input file. Valid extensions are: .json, .json.gz.")]
public string InputFile { get; set; }
@@ -42,5 +42,7 @@ internal sealed class ChatUpdateArgs : TwitchDownloaderArgs
[Option("temp-path", Default = "", HelpText = "Path to temporary folder to use for cache.")]
public string TempFolder { get; set; }
+
+ public OverwriteBehavior OverwriteBehavior { get; set; }
}
}
diff --git a/TwitchDownloaderCLI/Modes/Arguments/ClipDownloadArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/ClipDownloadArgs.cs
index ebbdf978..4502fd1b 100644
--- a/TwitchDownloaderCLI/Modes/Arguments/ClipDownloadArgs.cs
+++ b/TwitchDownloaderCLI/Modes/Arguments/ClipDownloadArgs.cs
@@ -1,9 +1,10 @@
using CommandLine;
+using TwitchDownloaderCLI.Models;
namespace TwitchDownloaderCLI.Modes.Arguments
{
[Verb("clipdownload", HelpText = "Downloads a clip from Twitch")]
- internal sealed class ClipDownloadArgs : TwitchDownloaderArgs
+ internal sealed class ClipDownloadArgs : TwitchDownloaderArgs, IFileOverwriteArgs
{
[Option('u', "id", Required = true, HelpText = "The ID or URL of the clip to download.")]
public string Id { get; set; }
@@ -25,5 +26,7 @@ internal sealed class ClipDownloadArgs : TwitchDownloaderArgs
[Option("temp-path", Default = "", HelpText = "Path to temporary caching folder.")]
public string TempFolder { get; set; }
+
+ public OverwriteBehavior OverwriteBehavior { get; set; }
}
}
\ No newline at end of file
diff --git a/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs
index a19bdf0b..24ad1b09 100644
--- a/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs
+++ b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs
@@ -1,14 +1,17 @@
using CommandLine;
+using TwitchDownloaderCLI.Models;
namespace TwitchDownloaderCLI.Modes.Arguments
{
[Verb("tsmerge", HelpText = "Concatenates multiple .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) files into a single file")]
- internal sealed class TsMergeArgs : TwitchDownloaderArgs
+ internal sealed class TsMergeArgs : TwitchDownloaderArgs, IFileOverwriteArgs
{
[Option('i', "input", Required = true, HelpText = "Path a text file containing the absolute paths of the files to concatenate, separated by newlines. M3U/M3U8 is also supported.")]
public string InputList { get; set; }
[Option('o', "output", Required = true, HelpText = "Path to output file.")]
public string OutputFile { get; set; }
+
+ public OverwriteBehavior OverwriteBehavior { get; set; }
}
}
diff --git a/TwitchDownloaderCLI/Modes/Arguments/VideoDownloadArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/VideoDownloadArgs.cs
index d0b93b6c..d150cc92 100644
--- a/TwitchDownloaderCLI/Modes/Arguments/VideoDownloadArgs.cs
+++ b/TwitchDownloaderCLI/Modes/Arguments/VideoDownloadArgs.cs
@@ -4,7 +4,7 @@
namespace TwitchDownloaderCLI.Modes.Arguments
{
[Verb("videodownload", HelpText = "Downloads a stream VOD from Twitch")]
- internal sealed class VideoDownloadArgs : TwitchDownloaderArgs
+ internal sealed class VideoDownloadArgs : TwitchDownloaderArgs, IFileOverwriteArgs
{
[Option('u', "id", Required = true, HelpText = "The ID or URL of the VOD to download.")]
public string Id { get; set; }
@@ -35,5 +35,7 @@ internal sealed class VideoDownloadArgs : TwitchDownloaderArgs
[Option("temp-path", Default = "", HelpText = "Path to temporary caching folder.")]
public string TempFolder { get; set; }
+
+ public OverwriteBehavior OverwriteBehavior { get; set; }
}
}
diff --git a/TwitchDownloaderCLI/Modes/DownloadClip.cs b/TwitchDownloaderCLI/Modes/DownloadClip.cs
index 8f2f3a28..a1cc97bf 100644
--- a/TwitchDownloaderCLI/Modes/DownloadClip.cs
+++ b/TwitchDownloaderCLI/Modes/DownloadClip.cs
@@ -21,13 +21,14 @@ internal static void Download(ClipDownloadArgs inputOptions)
FfmpegHandler.DetectFfmpeg(inputOptions.FfmpegPath, progress);
}
- var downloadOptions = GetDownloadOptions(inputOptions, progress);
+ var overwriteHandler = new FileOverwriteHandler(inputOptions);
+ var downloadOptions = GetDownloadOptions(inputOptions, overwriteHandler, progress);
var clipDownloader = new ClipDownloader(downloadOptions, progress);
clipDownloader.DownloadAsync(new CancellationToken()).Wait();
}
- private static ClipDownloadOptions GetDownloadOptions(ClipDownloadArgs inputOptions, ITaskLogger logger)
+ private static ClipDownloadOptions GetDownloadOptions(ClipDownloadArgs inputOptions, FileOverwriteHandler overwriteHandler, ITaskLogger logger)
{
if (inputOptions.Id is null)
{
@@ -50,7 +51,8 @@ private static ClipDownloadOptions GetDownloadOptions(ClipDownloadArgs inputOpti
ThrottleKib = inputOptions.ThrottleKib,
FfmpegPath = string.IsNullOrWhiteSpace(inputOptions.FfmpegPath) ? FfmpegHandler.FfmpegExecutableName : Path.GetFullPath(inputOptions.FfmpegPath),
EncodeMetadata = inputOptions.EncodeMetadata!.Value,
- TempFolder = inputOptions.TempFolder
+ TempFolder = inputOptions.TempFolder,
+ FileOverwriteCallback = overwriteHandler.HandleOverwriteCallback,
};
return downloadOptions;
diff --git a/TwitchDownloaderCLI/Modes/DownloadVideo.cs b/TwitchDownloaderCLI/Modes/DownloadVideo.cs
index 9a9bc596..ef907f78 100644
--- a/TwitchDownloaderCLI/Modes/DownloadVideo.cs
+++ b/TwitchDownloaderCLI/Modes/DownloadVideo.cs
@@ -19,13 +19,14 @@ internal static void Download(VideoDownloadArgs inputOptions)
FfmpegHandler.DetectFfmpeg(inputOptions.FfmpegPath, progress);
- var downloadOptions = GetDownloadOptions(inputOptions, progress);
+ var overwriteHandler = new FileOverwriteHandler(inputOptions);
+ var downloadOptions = GetDownloadOptions(inputOptions, overwriteHandler, progress);
var videoDownloader = new VideoDownloader(downloadOptions, progress);
videoDownloader.DownloadAsync(new CancellationToken()).Wait();
}
- private static VideoDownloadOptions GetDownloadOptions(VideoDownloadArgs inputOptions, ITaskLogger logger)
+ private static VideoDownloadOptions GetDownloadOptions(VideoDownloadArgs inputOptions, FileOverwriteHandler overwriteHandler, ITaskLogger logger)
{
if (inputOptions.Id is null)
{
@@ -76,7 +77,8 @@ private static VideoDownloadOptions GetDownloadOptions(VideoDownloadArgs inputOp
"Run 'TwitchDownloaderCLI cache help' for more information.");
return Array.Empty();
- }
+ },
+ FileOverwriteCallback = overwriteHandler.HandleOverwriteCallback,
};
return downloadOptions;
diff --git a/TwitchDownloaderCLI/Tools/FileOverwriteHandler.cs b/TwitchDownloaderCLI/Tools/FileOverwriteHandler.cs
new file mode 100644
index 00000000..e3457430
--- /dev/null
+++ b/TwitchDownloaderCLI/Tools/FileOverwriteHandler.cs
@@ -0,0 +1,52 @@
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.IO;
+using TwitchDownloaderCLI.Models;
+using TwitchDownloaderCore.Tools;
+
+namespace TwitchDownloaderCLI.Tools
+{
+ internal class FileOverwriteHandler
+ {
+ private readonly IFileOverwriteArgs _overwriteArgs;
+
+ public FileOverwriteHandler(IFileOverwriteArgs overwriteArgs)
+ {
+ _overwriteArgs = overwriteArgs;
+ }
+
+ [return: MaybeNull]
+ public FileInfo HandleOverwriteCallback(FileInfo fileInfo)
+ {
+ return _overwriteArgs.OverwriteBehavior switch
+ {
+ OverwriteBehavior.Overwrite => fileInfo,
+ OverwriteBehavior.Exit => null,
+ OverwriteBehavior.Rename => FilenameService.GetNonCollidingName(fileInfo),
+ OverwriteBehavior.Prompt => PromptUser(fileInfo),
+ _ => throw new ArgumentOutOfRangeException(nameof(_overwriteArgs.OverwriteBehavior), _overwriteArgs.OverwriteBehavior, null)
+ };
+ }
+
+ [return: MaybeNull]
+ private static FileInfo PromptUser(FileInfo fileInfo)
+ {
+ Console.WriteLine($"{fileInfo.FullName} already exists.");
+
+ while (true)
+ {
+ Console.Write("[O] Overwrite / [R] Rename / [E] Exit: ");
+ var userInput = Console.ReadLine()!.Trim().ToLower();
+ switch (userInput)
+ {
+ case "o" or "overwrite":
+ return fileInfo;
+ case "e" or "exit":
+ return null;
+ case "r" or "rename":
+ return FilenameService.GetNonCollidingName(fileInfo);
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/TwitchDownloaderCore/Tools/FilenameService.cs b/TwitchDownloaderCore/Tools/FilenameService.cs
index 46dd0245..7ce61e45 100644
--- a/TwitchDownloaderCore/Tools/FilenameService.cs
+++ b/TwitchDownloaderCore/Tools/FilenameService.cs
@@ -86,5 +86,24 @@ private static string[] GetTemplateSubfolders(ref string fullPath)
private static readonly char[] FilenameInvalidChars = Path.GetInvalidFileNameChars();
private static string RemoveInvalidFilenameChars(string filename) => filename.ReplaceAny(FilenameInvalidChars, '_');
+
+ public static FileInfo GetNonCollidingName(FileInfo fileInfo)
+ {
+ var fi = fileInfo;
+
+ var parentDir = Path.GetDirectoryName(fi.FullName)!;
+ var oldName = Path.GetFileNameWithoutExtension(fi.Name.AsSpan());
+ var extension = Path.GetExtension(fi.Name.AsSpan());
+
+ var i = 1;
+ while (fi.Exists)
+ {
+ var newName = Path.Combine(parentDir, $"{oldName} ({i}){extension}");
+ fi = new FileInfo(newName);
+ i++;
+ }
+
+ return fi;
+ }
}
}
\ No newline at end of file
From e84de6d425e1002ca76616b697278255640aaf0d Mon Sep 17 00:00:00 2001
From: ScrubN <72096833+ScrubN@users.noreply.github.com>
Date: Wed, 5 Jun 2024 00:51:43 -0400
Subject: [PATCH 04/21] Add GetNonCollidingName tests
---
.../ToolTests/FilenameServiceTests.cs | 40 +++++++++++++++++++
1 file changed, 40 insertions(+)
diff --git a/TwitchDownloaderCore.Tests/ToolTests/FilenameServiceTests.cs b/TwitchDownloaderCore.Tests/ToolTests/FilenameServiceTests.cs
index 18084153..94626cfe 100644
--- a/TwitchDownloaderCore.Tests/ToolTests/FilenameServiceTests.cs
+++ b/TwitchDownloaderCore.Tests/ToolTests/FilenameServiceTests.cs
@@ -151,5 +151,45 @@ public void DoesNotInterpretBogusTemplateParameter()
Assert.Equal(EXPECTED, result);
}
+
+ [Fact]
+ public void GetNonCollidingNameWorks_WhenNoCollisionExists()
+ {
+ var expected = Path.Combine(Path.GetTempPath(), "foo.txt");
+ var path = Path.Combine(Path.GetTempPath(), "foo.txt");
+ var fileInfo = new FileInfo(path);
+
+ try
+ {
+ var actual = FilenameService.GetNonCollidingName(fileInfo);
+
+ Assert.Equal(expected, actual.FullName);
+ }
+ finally
+ {
+ File.Delete(path);
+ }
+ }
+
+ [Fact]
+ public void GetNonCollidingNameWorks_WhenCollisionExists()
+ {
+ var expected = Path.Combine(Path.GetTempPath(), "foo (1).txt");
+ var path = Path.Combine(Path.GetTempPath(), "foo.txt");
+ var fileInfo = new FileInfo(path);
+
+ try
+ {
+ fileInfo.Create().Close();
+
+ var actual = FilenameService.GetNonCollidingName(fileInfo);
+
+ Assert.Equal(expected, actual.FullName);
+ }
+ finally
+ {
+ File.Delete(path);
+ }
+ }
}
}
\ No newline at end of file
From 8432bdbddc952db4d457fa9f26ac614eb225d730 Mon Sep 17 00:00:00 2001
From: ScrubN <72096833+ScrubN@users.noreply.github.com>
Date: Wed, 5 Jun 2024 01:35:31 -0400
Subject: [PATCH 05/21] TwitchDownloaderArgs -> ITwitchDownloaderArgs &
rearrange arg interfaces due to help text ordering
---
TwitchDownloaderCLI/Modes/Arguments/CacheArgs.cs | 7 ++++++-
TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs | 5 ++++-
TwitchDownloaderCLI/Modes/Arguments/ChatRenderArgs.cs | 5 ++++-
TwitchDownloaderCLI/Modes/Arguments/ChatUpdateArgs.cs | 5 ++++-
TwitchDownloaderCLI/Modes/Arguments/ClipDownloadArgs.cs | 5 ++++-
TwitchDownloaderCLI/Modes/Arguments/FfmpegArgs.cs | 7 ++++++-
.../{Models => Modes/Arguments}/IFileOverwriteArgs.cs | 5 +++--
.../{TwitchDownloaderArgs.cs => ITwitchDownloaderArgs.cs} | 2 +-
TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs | 5 ++++-
TwitchDownloaderCLI/Modes/Arguments/VideoDownloadArgs.cs | 5 ++++-
TwitchDownloaderCLI/Program.cs | 4 ++--
TwitchDownloaderCLI/Tools/FileOverwriteHandler.cs | 1 +
12 files changed, 43 insertions(+), 13 deletions(-)
rename TwitchDownloaderCLI/{Models => Modes/Arguments}/IFileOverwriteArgs.cs (65%)
rename TwitchDownloaderCLI/Modes/Arguments/{TwitchDownloaderArgs.cs => ITwitchDownloaderArgs.cs} (91%)
diff --git a/TwitchDownloaderCLI/Modes/Arguments/CacheArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/CacheArgs.cs
index 1f9c89bc..92c66843 100644
--- a/TwitchDownloaderCLI/Modes/Arguments/CacheArgs.cs
+++ b/TwitchDownloaderCLI/Modes/Arguments/CacheArgs.cs
@@ -1,14 +1,19 @@
using CommandLine;
+using TwitchDownloaderCLI.Models;
namespace TwitchDownloaderCLI.Modes.Arguments
{
[Verb("cache", HelpText = "Manage the working cache")]
- internal sealed class CacheArgs : TwitchDownloaderArgs
+ internal sealed class CacheArgs : ITwitchDownloaderArgs
{
[Option('c', "clear", Default = false, Required = false, HelpText = "Clears the default cache folder.")]
public bool ClearCache { get; set; }
[Option("force-clear", Default = false, Required = false, HelpText = "Clears the default cache folder, bypassing the confirmation prompt")]
public bool ForceClearCache { get; set; }
+
+ // Interface args
+ public bool? ShowBanner { get; set; }
+ public LogLevel LogLevel { get; set; }
}
}
diff --git a/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs
index 5512a965..1609c97b 100644
--- a/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs
+++ b/TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs
@@ -5,7 +5,7 @@
namespace TwitchDownloaderCLI.Modes.Arguments
{
[Verb("chatdownload", HelpText = "Downloads the chat from a VOD or clip")]
- internal sealed class ChatDownloadArgs : TwitchDownloaderArgs, IFileOverwriteArgs
+ internal sealed class ChatDownloadArgs : IFileOverwriteArgs, ITwitchDownloaderArgs
{
[Option('u', "id", Required = true, HelpText = "The ID or URL of the VOD or clip to download that chat of.")]
public string Id { get; set; }
@@ -46,6 +46,9 @@ internal sealed class ChatDownloadArgs : TwitchDownloaderArgs, IFileOverwriteArg
[Option("temp-path", Default = "", HelpText = "Path to temporary folder to use for cache.")]
public string TempFolder { get; set; }
+ // Interface args
public OverwriteBehavior OverwriteBehavior { get; set; }
+ public bool? ShowBanner { get; set; }
+ public LogLevel LogLevel { get; set; }
}
}
\ No newline at end of file
diff --git a/TwitchDownloaderCLI/Modes/Arguments/ChatRenderArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/ChatRenderArgs.cs
index c34317dd..48fbf0c2 100644
--- a/TwitchDownloaderCLI/Modes/Arguments/ChatRenderArgs.cs
+++ b/TwitchDownloaderCLI/Modes/Arguments/ChatRenderArgs.cs
@@ -4,7 +4,7 @@
namespace TwitchDownloaderCLI.Modes.Arguments
{
[Verb("chatrender", HelpText = "Renders a chat JSON as a video")]
- internal sealed class ChatRenderArgs : TwitchDownloaderArgs, IFileOverwriteArgs
+ internal sealed class ChatRenderArgs : IFileOverwriteArgs, ITwitchDownloaderArgs
{
[Option('i', "input", Required = true, HelpText = "Path to JSON chat file input.")]
public string InputFile { get; set; }
@@ -153,6 +153,9 @@ internal sealed class ChatRenderArgs : TwitchDownloaderArgs, IFileOverwriteArgs
[Option("scale-highlight-indent", Default = 1.0, HelpText = "Number to scale highlight indent size (sub messages).")]
public double ScaleAccentIndent { get; set; }
+ // Interface args
public OverwriteBehavior OverwriteBehavior { get; set; }
+ public bool? ShowBanner { get; set; }
+ public LogLevel LogLevel { get; set; }
}
}
\ No newline at end of file
diff --git a/TwitchDownloaderCLI/Modes/Arguments/ChatUpdateArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/ChatUpdateArgs.cs
index 8aa8405f..3d39d9bc 100644
--- a/TwitchDownloaderCLI/Modes/Arguments/ChatUpdateArgs.cs
+++ b/TwitchDownloaderCLI/Modes/Arguments/ChatUpdateArgs.cs
@@ -5,7 +5,7 @@
namespace TwitchDownloaderCLI.Modes.Arguments
{
[Verb("chatupdate", HelpText = "Updates the embedded emotes, badges, bits, and trims a chat JSON and/or converts a JSON chat to another format.")]
- internal sealed class ChatUpdateArgs : TwitchDownloaderArgs, IFileOverwriteArgs
+ internal sealed class ChatUpdateArgs : IFileOverwriteArgs, ITwitchDownloaderArgs
{
[Option('i', "input", Required = true, HelpText = "Path to input file. Valid extensions are: .json, .json.gz.")]
public string InputFile { get; set; }
@@ -43,6 +43,9 @@ internal sealed class ChatUpdateArgs : TwitchDownloaderArgs, IFileOverwriteArgs
[Option("temp-path", Default = "", HelpText = "Path to temporary folder to use for cache.")]
public string TempFolder { get; set; }
+ // Interface args
public OverwriteBehavior OverwriteBehavior { get; set; }
+ public bool? ShowBanner { get; set; }
+ public LogLevel LogLevel { get; set; }
}
}
diff --git a/TwitchDownloaderCLI/Modes/Arguments/ClipDownloadArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/ClipDownloadArgs.cs
index 4502fd1b..c1cc51fa 100644
--- a/TwitchDownloaderCLI/Modes/Arguments/ClipDownloadArgs.cs
+++ b/TwitchDownloaderCLI/Modes/Arguments/ClipDownloadArgs.cs
@@ -4,7 +4,7 @@
namespace TwitchDownloaderCLI.Modes.Arguments
{
[Verb("clipdownload", HelpText = "Downloads a clip from Twitch")]
- internal sealed class ClipDownloadArgs : TwitchDownloaderArgs, IFileOverwriteArgs
+ internal sealed class ClipDownloadArgs : IFileOverwriteArgs, ITwitchDownloaderArgs
{
[Option('u', "id", Required = true, HelpText = "The ID or URL of the clip to download.")]
public string Id { get; set; }
@@ -27,6 +27,9 @@ internal sealed class ClipDownloadArgs : TwitchDownloaderArgs, IFileOverwriteArg
[Option("temp-path", Default = "", HelpText = "Path to temporary caching folder.")]
public string TempFolder { get; set; }
+ // Interface args
public OverwriteBehavior OverwriteBehavior { get; set; }
+ public bool? ShowBanner { get; set; }
+ public LogLevel LogLevel { get; set; }
}
}
\ No newline at end of file
diff --git a/TwitchDownloaderCLI/Modes/Arguments/FfmpegArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/FfmpegArgs.cs
index c3988428..2d2ff505 100644
--- a/TwitchDownloaderCLI/Modes/Arguments/FfmpegArgs.cs
+++ b/TwitchDownloaderCLI/Modes/Arguments/FfmpegArgs.cs
@@ -1,11 +1,16 @@
using CommandLine;
+using TwitchDownloaderCLI.Models;
namespace TwitchDownloaderCLI.Modes.Arguments
{
[Verb("ffmpeg", HelpText = "Manage standalone ffmpeg")]
- internal sealed class FfmpegArgs : TwitchDownloaderArgs
+ internal sealed class FfmpegArgs : ITwitchDownloaderArgs
{
[Option('d', "download", Default = false, Required = false, HelpText = "Downloads FFmpeg as a standalone file.")]
public bool DownloadFfmpeg { get; set; }
+
+ // Interface args
+ public bool? ShowBanner { get; set; }
+ public LogLevel LogLevel { get; set; }
}
}
diff --git a/TwitchDownloaderCLI/Models/IFileOverwriteArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/IFileOverwriteArgs.cs
similarity index 65%
rename from TwitchDownloaderCLI/Models/IFileOverwriteArgs.cs
rename to TwitchDownloaderCLI/Modes/Arguments/IFileOverwriteArgs.cs
index d31f2d7f..68a8bed6 100644
--- a/TwitchDownloaderCLI/Models/IFileOverwriteArgs.cs
+++ b/TwitchDownloaderCLI/Modes/Arguments/IFileOverwriteArgs.cs
@@ -1,8 +1,9 @@
using CommandLine;
+using TwitchDownloaderCLI.Models;
-namespace TwitchDownloaderCLI.Models
+namespace TwitchDownloaderCLI.Modes.Arguments
{
- public interface IFileOverwriteArgs
+ internal interface IFileOverwriteArgs
{
[Option("overwrite", Default = OverwriteBehavior.Prompt, HelpText = ". Valid values are: Overwrite, Exit, Rename, Prompt.")]
public OverwriteBehavior OverwriteBehavior { get; set; }
diff --git a/TwitchDownloaderCLI/Modes/Arguments/TwitchDownloaderArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/ITwitchDownloaderArgs.cs
similarity index 91%
rename from TwitchDownloaderCLI/Modes/Arguments/TwitchDownloaderArgs.cs
rename to TwitchDownloaderCLI/Modes/Arguments/ITwitchDownloaderArgs.cs
index 07945f8e..21b09a3d 100644
--- a/TwitchDownloaderCLI/Modes/Arguments/TwitchDownloaderArgs.cs
+++ b/TwitchDownloaderCLI/Modes/Arguments/ITwitchDownloaderArgs.cs
@@ -3,7 +3,7 @@
namespace TwitchDownloaderCLI.Modes.Arguments
{
- internal abstract class TwitchDownloaderArgs
+ internal interface ITwitchDownloaderArgs
{
[Option("banner", Default = true, HelpText = "Displays a banner containing version and copyright information.")]
public bool? ShowBanner { get; set; }
diff --git a/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs
index 24ad1b09..51c74496 100644
--- a/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs
+++ b/TwitchDownloaderCLI/Modes/Arguments/TsMergeArgs.cs
@@ -4,7 +4,7 @@
namespace TwitchDownloaderCLI.Modes.Arguments
{
[Verb("tsmerge", HelpText = "Concatenates multiple .ts/.tsv/.tsa/.m2t/.m2ts (MPEG Transport Stream) files into a single file")]
- internal sealed class TsMergeArgs : TwitchDownloaderArgs, IFileOverwriteArgs
+ internal sealed class TsMergeArgs : IFileOverwriteArgs, ITwitchDownloaderArgs
{
[Option('i', "input", Required = true, HelpText = "Path a text file containing the absolute paths of the files to concatenate, separated by newlines. M3U/M3U8 is also supported.")]
public string InputList { get; set; }
@@ -12,6 +12,9 @@ internal sealed class TsMergeArgs : TwitchDownloaderArgs, IFileOverwriteArgs
[Option('o', "output", Required = true, HelpText = "Path to output file.")]
public string OutputFile { get; set; }
+ // Interface args
public OverwriteBehavior OverwriteBehavior { get; set; }
+ public bool? ShowBanner { get; set; }
+ public LogLevel LogLevel { get; set; }
}
}
diff --git a/TwitchDownloaderCLI/Modes/Arguments/VideoDownloadArgs.cs b/TwitchDownloaderCLI/Modes/Arguments/VideoDownloadArgs.cs
index d150cc92..00a473b7 100644
--- a/TwitchDownloaderCLI/Modes/Arguments/VideoDownloadArgs.cs
+++ b/TwitchDownloaderCLI/Modes/Arguments/VideoDownloadArgs.cs
@@ -4,7 +4,7 @@
namespace TwitchDownloaderCLI.Modes.Arguments
{
[Verb("videodownload", HelpText = "Downloads a stream VOD from Twitch")]
- internal sealed class VideoDownloadArgs : TwitchDownloaderArgs, IFileOverwriteArgs
+ internal sealed class VideoDownloadArgs : IFileOverwriteArgs, ITwitchDownloaderArgs
{
[Option('u', "id", Required = true, HelpText = "The ID or URL of the VOD to download.")]
public string Id { get; set; }
@@ -36,6 +36,9 @@ internal sealed class VideoDownloadArgs : TwitchDownloaderArgs, IFileOverwriteAr
[Option("temp-path", Default = "", HelpText = "Path to temporary caching folder.")]
public string TempFolder { get; set; }
+ // Interface args
public OverwriteBehavior OverwriteBehavior { get; set; }
+ public bool? ShowBanner { get; set; }
+ public LogLevel LogLevel { get; set; }
}
}
diff --git a/TwitchDownloaderCLI/Program.cs b/TwitchDownloaderCLI/Program.cs
index 3149b62d..070da004 100644
--- a/TwitchDownloaderCLI/Program.cs
+++ b/TwitchDownloaderCLI/Program.cs
@@ -29,7 +29,7 @@ private static void Main(string[] args)
parserResult.WithNotParsed(errors => WriteHelpText(errors, parserResult, parser.Settings));
CoreLicensor.EnsureFilesExist(AppContext.BaseDirectory);
- WriteApplicationBanner((TwitchDownloaderArgs)parserResult.Value);
+ WriteApplicationBanner((ITwitchDownloaderArgs)parserResult.Value);
parserResult
.WithParsed(DownloadVideo.Download)
@@ -74,7 +74,7 @@ private static void WriteHelpText(IEnumerable errors, ParserResult