Clip download progress (#782)

* Add progress reporting to clip downloader

* Fix oversight with clip metadata FFmpeg args

* Fix crash when not providing a specific FFmpeg path to the CLI clip downloader

* Add Span/ReadOnlySpan Read/Write overrides to ThrottledStream.cs

Author: ScrubN

TwitchDownloaderCLI/Modes/DownloadClip.cs

@@ -1,4 +1,5 @@
 using System;
+using System.IO;
 using System.Text.RegularExpressions;
 using System.Threading;
 using TwitchDownloaderCLI.Modes.Arguments;
@@ -48,7 +49,7 @@ private static ClipDownloadOptions GetDownloadOptions(ClipDownloadArgs inputOpti
                 Filename = inputOptions.OutputFile,
                 Quality = inputOptions.Quality,
                 ThrottleKib = inputOptions.ThrottleKib,
-                FfmpegPath = inputOptions.FfmpegPath,
+                FfmpegPath = string.IsNullOrWhiteSpace(inputOptions.FfmpegPath) ? FfmpegHandler.FfmpegExecutableName : Path.GetFullPath(inputOptions.FfmpegPath),
                 EncodeMetadata = inputOptions.EncodeMetadata!.Value,
                 TempFolder = inputOptions.TempFolder
             };

TwitchDownloaderCore/ClipDownloader.cs

@@ -6,6 +6,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using System.Web;
+using TwitchDownloaderCore.Extensions;
 using TwitchDownloaderCore.Options;
 using TwitchDownloaderCore.Tools;
 
@@ -40,12 +41,18 @@ public async Task DownloadAsync(CancellationToken cancellationToken)
                 TwitchHelper.CreateDirectory(clipDirectory.FullName);
             }
 
-            _progress.Report(new ProgressReport(ReportType.NewLineStatus, "Downloading Clip"));
+            _progress.Report(new ProgressReport(ReportType.NewLineStatus, "Downloading Clip 0%"));
+
+            void DownloadProgressHandler(StreamCopyProgress streamProgress)
+            {
+                var percent = (int)(streamProgress.BytesCopied / (double)streamProgress.SourceLength * 100);
+                _progress.Report(new ProgressReport(ReportType.SameLineStatus, $"Downloading Clip {percent}%"));
+                _progress.Report(new ProgressReport(percent));
+            }
 
             if (!downloadOptions.EncodeMetadata)
             {
-                await DownloadFileTaskAsync(downloadUrl, downloadOptions.Filename, downloadOptions.ThrottleKib, cancellationToken);
-                _progress.Report(new ProgressReport(100));
+                await DownloadFileTaskAsync(downloadUrl, downloadOptions.Filename, downloadOptions.ThrottleKib, new Progress<StreamCopyProgress>(DownloadProgressHandler), cancellationToken);
                 return;
             }
 
@@ -57,12 +64,14 @@ public async Task DownloadAsync(CancellationToken cancellationToken)
             var tempFile = Path.Combine(downloadOptions.TempFolder, $"clip_{DateTimeOffset.Now.ToUnixTimeMilliseconds()}_{Path.GetRandomFileName()}");
             try
             {
-                await DownloadFileTaskAsync(downloadUrl, tempFile, downloadOptions.ThrottleKib, cancellationToken);
+                await DownloadFileTaskAsync(downloadUrl, tempFile, downloadOptions.ThrottleKib, new Progress<StreamCopyProgress>(DownloadProgressHandler), cancellationToken);
 
-                _progress.Report(new ProgressReport(ReportType.NewLineStatus, "Encoding Clip Metadata"));
+                _progress.Report(new ProgressReport(ReportType.NewLineStatus, "Encoding Clip Metadata 0%"));
+                _progress.Report(new ProgressReport(0));
 
                 await EncodeClipMetadata(tempFile, downloadOptions.Filename, cancellationToken);
 
+                _progress.Report(new ProgressReport(ReportType.SameLineStatus, "Encoding Clip Metadata 100%"));
                 _progress.Report(new ProgressReport(100));
             }
             finally
@@ -103,22 +112,26 @@ private async Task<string> GetDownloadUrl()
             return downloadUrl + "?sig=" + listLinks[0].data.clip.playbackAccessToken.signature + "&token=" + HttpUtility.UrlEncode(listLinks[0].data.clip.playbackAccessToken.value);
         }
 
-        private static async Task DownloadFileTaskAsync(string url, string destinationFile, int throttleKib, CancellationToken cancellationToken)
+        private static async Task DownloadFileTaskAsync(string url, string destinationFile, int throttleKib, IProgress<StreamCopyProgress> progress, CancellationToken cancellationToken)
         {
             var request = new HttpRequestMessage(HttpMethod.Get, url);
             using var response = await HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
             response.EnsureSuccessStatusCode();
 
+            var contentLength = response.Content.Headers.ContentLength;
+
             if (throttleKib == -1)
             {
                 await using var fs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read);
-                await response.Content.CopyToAsync(fs, cancellationToken).ConfigureAwait(false);
+                await using var contentStream = await response.Content.ReadAsStreamAsync(cancellationToken);
+                await contentStream.ProgressCopyToAsync(fs, contentLength, progress, cancellationToken).ConfigureAwait(false);
             }
             else
             {
-                await using var throttledStream = new ThrottledStream(await response.Content.ReadAsStreamAsync(cancellationToken), throttleKib);
                 await using var fs = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.Read);
-                await throttledStream.CopyToAsync(fs, cancellationToken).ConfigureAwait(false);
+                await using var contentStream = await response.Content.ReadAsStreamAsync(cancellationToken);
+                await using var throttledStream = new ThrottledStream(contentStream, throttleKib);
+                await throttledStream.ProgressCopyToAsync(fs, contentLength, progress, cancellationToken).ConfigureAwait(false);
             }
         }
 
@@ -137,7 +150,7 @@ await FfmpegMetadata.SerializeAsync(metadataFile, clipInfo.data.clip.broadcaster
                     StartInfo =
                     {
                         FileName = downloadOptions.FfmpegPath,
-                        Arguments = $"-i \"{inputFile}\" -i \"{metadataFile}\" -map_metadata 1 -c copy \"{destinationFile}\"",
+                        Arguments = $"-i \"{inputFile}\" -i \"{metadataFile}\" -map_metadata 1 -y -c copy \"{destinationFile}\"",
                         UseShellExecute = false,
                         CreateNoWindow = true,
                         RedirectStandardInput = false,

TwitchDownloaderCore/Extensions/StreamExtensions.cs

@@ -0,0 +1,50 @@
+using System;
+using System.Buffers;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace TwitchDownloaderCore.Extensions
+{
+    public record struct StreamCopyProgress(long SourceLength, long BytesCopied);
+
+    public static class StreamExtensions
+    {
+        // The default size from Stream.GetCopyBufferSize() is 81_920.
+        private const int STREAM_DEFAULT_BUFFER_LENGTH = 81_920;
+
+        public static async Task ProgressCopyToAsync(this Stream source, Stream destination, long? sourceLength, IProgress<StreamCopyProgress> progress = null,
+            CancellationToken cancellationToken = default)
+        {
+            if (!sourceLength.HasValue)
+            {
+                await source.CopyToAsync(destination, cancellationToken);
+                return;
+            }
+
+            var rentedBuffer = ArrayPool<byte>.Shared.Rent(STREAM_DEFAULT_BUFFER_LENGTH);
+            var buffer = rentedBuffer.AsMemory(0, STREAM_DEFAULT_BUFFER_LENGTH);
+            var totalBytesRead = 0L;
+
+            try
+            {
+                var bytesRead = 0;
+                do
+                {
+                    bytesRead = await source.ReadAsync(buffer, cancellationToken).ConfigureAwait(false);
+                    if (bytesRead == 0)
+                        continue;
+
+                    await destination.WriteAsync(buffer[..bytesRead], cancellationToken).ConfigureAwait(false);
+                    totalBytesRead += bytesRead;
+
+                    progress?.Report(new StreamCopyProgress(sourceLength!.Value, totalBytesRead));
+                } while (bytesRead != 0);
+            }
+            finally
+            {
+                ArrayPool<byte>.Shared.Return(rentedBuffer);
+            }
+        }
+    }
+}

TwitchDownloaderCore/Tools/ThrottledStream.cs

@@ -50,6 +50,14 @@ public override int Read(byte[] buffer, int offset, int count)
             return read;
         }
 
+        public override int Read(Span<byte> buffer)
+        {
+            var newCount = GetBytesToReturn(buffer.Length);
+            var read = BaseStream.Read(buffer[..newCount]);
+            Interlocked.Add(ref _totalBytesRead, read);
+            return read;
+        }
+
         public override long Seek(long offset, SeekOrigin origin)
         {
             return BaseStream.Seek(offset, origin);
@@ -59,6 +67,8 @@ public override void SetLength(long value) { }
 
         public override void Write(byte[] buffer, int offset, int count) { }
 
+        public override void Write(ReadOnlySpan<byte> buffer) { }
+
         private int GetBytesToReturn(int count)
         {
             return GetBytesToReturnAsync(count).GetAwaiter().GetResult();