Merge pull request #35 from dotnet-campus/t/lindexi/request

lindexi · web-flow · commit 2d62cc81bdde · 2020-12-22T19:44:48.000+08:00
加入慢网测试
diff --git a/src/FileDownloader.Tests/SegmentFileDownloaderTest.cs b/src/FileDownloader.Tests/SegmentFileDownloaderTest.cs
@@ -1,6 +1,11 @@
 ﻿using System;
+using System.Diagnostics;
 using System.IO;
 using System.Net;
+using System.Net.Cache;
+using System.Runtime.CompilerServices;
+using System.Runtime.Serialization;
+using System.Threading;
 using System.Threading.Tasks;
 using dotnetCampus.FileDownloader;
 using Microsoft.Extensions.Logging;
@@ -34,6 +39,83 @@ public void CreateWebRequest()
 
                 mock.Verify(downloader => downloader.CreateWebRequest(It.IsAny<string>()), Times.AtLeastOnce);
             });
+
+            "测试进入慢网环境下载，能成功下载文件".Test(() =>
+            {
+                var url = $"https://blog.lindexi.com";
+                var file = new FileInfo(Path.GetTempFileName());
+                var slowlySegmentFileDownloader = new SlowSegmentFileDownloader(url, file);
+                var task = slowlySegmentFileDownloader.DownloadFileAsync();
+
+                Task.WaitAny(task, Task.Delay(TimeSpan.FromSeconds(20)));
+
+                if (task.IsCompleted)
+                {
+                    file.Refresh();
+                    Assert.AreEqual(100, file.Length);
+                }
+                else
+                {
+                    // 测试设备太渣
+                }
+            });
+        }
+
+        class SlowStream : Stream
+        {
+            public override void Flush()
+            {
+            }
+
+            public override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
+            {
+                return Task.FromResult(Read(buffer, offset, count));
+            }
+
+            public override int Read(byte[] buffer, int offset, int count)
+            {
+                Thread.Sleep(100);
+                buffer[0] = 0xFF;
+                return 1;
+            }
+
+            public override long Seek(long offset, SeekOrigin origin)
+            {
+                return offset;
+            }
+
+            public override void SetLength(long value)
+            {
+                throw new NotImplementedException();
+            }
+
+            public override void Write(byte[] buffer, int offset, int count)
+            {
+                throw new NotImplementedException();
+            }
+
+            public override bool CanRead => true;
+            public override bool CanSeek => false;
+            public override bool CanWrite => false;
+            public override long Length => 100;
+            public override long Position { get; set; }
+        }
+
+        class SlowSegmentFileDownloader : SegmentFileDownloader
+        {
+            public SlowSegmentFileDownloader(string url, FileInfo file, ILogger<SegmentFileDownloader>? logger = null, IProgress<DownloadProgress>? progress = null, ISharedArrayPool? sharedArrayPool = null, int bufferLength = UInt16.MaxValue, TimeSpan? stepTimeOut = null) : base(url, file, logger, progress, sharedArrayPool, bufferLength, stepTimeOut)
+            {
+            }
+
+            protected override WebRequest CreateWebRequest(string url)
+            {
+                var fakeWebResponse = new FakeWebResponse()
+                {
+                    Stream = new SlowStream()
+                };
+
+                return new FakeHttpWebRequest(fakeWebResponse);
+            }
         }
 
         class FakeSegmentFileDownloader : SegmentFileDownloader
@@ -49,21 +131,64 @@ public FakeSegmentFileDownloader(IMockSegmentFileDownloader mockSegmentFileDownl
 
             public IMockSegmentFileDownloader MockSegmentFileDownloader { get; }
 
-            protected override HttpWebRequest CreateWebRequest(string url)
+            protected override WebRequest CreateWebRequest(string url)
             {
                 return MockSegmentFileDownloader.CreateWebRequest(url);
             }
 
-            protected override HttpWebRequest OnWebRequestSet(HttpWebRequest webRequest)
+            protected override WebRequest OnWebRequestSet(WebRequest webRequest)
             {
                 return MockSegmentFileDownloader.OnWebRequestSet(webRequest);
             }
         }
 
         public interface IMockSegmentFileDownloader
         {
-            HttpWebRequest CreateWebRequest(string url);
-            HttpWebRequest OnWebRequestSet(HttpWebRequest webRequest);
+            WebRequest CreateWebRequest(string url);
+            WebRequest OnWebRequestSet(WebRequest webRequest);
+        }
+    }
+
+    class FakeHttpWebRequest : WebRequest
+    {
+        public FakeHttpWebRequest(FakeWebResponse fakeWebResponse)
+        {
+            FakeWebResponse = fakeWebResponse;
+        }
+
+        public override Task<WebResponse> GetResponseAsync()
+        {
+            return Task.FromResult(GetResponse());
+        }
+
+        public override string Method { get; set; }
+        public override RequestCachePolicy CachePolicy { get; set; }
+        public override string ConnectionGroupName { get; set; }
+        public override long ContentLength { get; set; }
+        public override string ContentType { get; set; }
+        public override ICredentials Credentials { get; set; }
+        public override WebHeaderCollection Headers { get; set; }
+        public override bool PreAuthenticate { get; set; }
+        public override Uri RequestUri { get; }
+        public override int Timeout { get; set; }
+        public override bool UseDefaultCredentials { get; set; }
+
+        private FakeWebResponse FakeWebResponse { get; }
+
+        public override WebResponse GetResponse()
+        {
+            return FakeWebResponse;
+        }
+    }
+
+    class FakeWebResponse : WebResponse
+    {
+        public Stream Stream { set; get; }
+        public override long ContentLength => Stream.Length;
+
+        public override Stream GetResponseStream()
+        {
+            return Stream;
         }
     }
 }
diff --git a/src/dotnetCampus.FileDownloader/SegmentFileDownloader.cs b/src/dotnetCampus.FileDownloader/SegmentFileDownloader.cs
@@ -4,6 +4,7 @@
 using System.Net;
 using System.Threading;
 using System.Threading.Tasks;
+using dotnetCampus.FileDownloader.Utils;
 using dotnetCampus.Threading;
 using Microsoft.Extensions.Logging;
 
@@ -200,7 +201,7 @@ public async Task DownloadFileAsync()
             if (supportSegment)
             {
                 // 先根据文件的大小，大概是 1M 让一个线程下载，至少需要开两个线程，最多是 10 个线程
-                threadCount = Math.Max(Math.Min(2, (int) (contentLength / 1024 / 1024)), MaxThreadCount);
+                threadCount = Math.Min(MaxThreadCount, Math.Max(2, (int) (contentLength / 1024 / 1024)));
             }
             else
             {
@@ -253,18 +254,18 @@ public async Task DownloadFileAsync()
         }
 
         /// <summary>
-        /// 通过 Url 创建出对应的 HttpWebRequest 实例
+        /// 通过 Url 创建出对应的 <see cref="WebRequest"/> 实例
         /// </summary>
         /// <param name="url"></param>
         /// <returns></returns>
-        protected virtual HttpWebRequest CreateWebRequest(string url) => (HttpWebRequest) WebRequest.Create(url);
+        protected virtual WebRequest CreateWebRequest(string url) => (WebRequest) WebRequest.Create(url);
 
         /// <summary>
-        /// 在 <see cref="HttpWebRequest"/> 经过了应用设置之后调用，应用的设置包括下载的 Range 等值，调用这个方法之后的下一步将会是使用这个方法的返回值去下载文件
+        /// 在 <see cref="WebRequest"/> 经过了应用设置之后调用，应用的设置包括下载的 Range 等值，调用这个方法之后的下一步将会是使用这个方法的返回值去下载文件
         /// </summary>
         /// <param name="webRequest"></param>
         /// <returns></returns>
-        protected virtual HttpWebRequest OnWebRequestSet(HttpWebRequest webRequest) => webRequest;
+        protected virtual WebRequest OnWebRequestSet(WebRequest webRequest) => webRequest;
 
         /// <summary>
         /// 这是给我自己开发调试用的
@@ -277,7 +278,7 @@ private void LogDebugInternal(string message, params object[] args)
             _logger.LogDebug(message, args);
         }
 
-        private async Task<WebResponse?> GetWebResponseAsync(Action<HttpWebRequest>? action = null)
+        private async Task<WebResponse?> GetWebResponseAsync(Action<WebRequest>? action = null)
         {
             var id = Interlocked.Increment(ref _idGenerator);
 
@@ -297,7 +298,12 @@ private void LogDebugInternal(string message, params object[] args)
                     // 即使下载速度再慢，只有要在下载，也不能算超时
                     // 如果下载 BufferLength 长度 默认 65535 字节时间超过 10 秒，基本上也断开也差不多
                     webRequest.Timeout = (int) StepTimeOut.TotalMilliseconds;
-                    webRequest.ReadWriteTimeout = (int) StepTimeOut.TotalMilliseconds;
+
+                    if (webRequest is HttpWebRequest httpWebRequest)
+                    {
+                        // ReadWriteTimeout设置的是从建立连接开始，到下载数据完毕所历经的时间
+                        httpWebRequest.ReadWriteTimeout = (int) StepTimeOut.TotalMilliseconds;
+                    }
 
                     LogDebugInternal("[GetWebResponseAsync] [{0}] Enter action.", id);
                     action?.Invoke(webRequest);
@@ -367,7 +373,9 @@ protected virtual Task<WebResponse> GetResponseAsync(WebRequest request)
             // 为什么不使用 StartPoint 而是使用 CurrentDownloadPoint 是因为需要处理重试
 
             var response = await GetWebResponseAsync(webRequest =>
-                webRequest.AddRange(downloadSegment.CurrentDownloadPoint, downloadSegment.RequirementDownloadPoint));
+            {
+                webRequest.AddRange(downloadSegment.CurrentDownloadPoint, downloadSegment.RequirementDownloadPoint);
+            });
             return response;
         }
 
diff --git a/src/dotnetCampus.FileDownloader/Utils/WebRequestExtension.cs b/src/dotnetCampus.FileDownloader/Utils/WebRequestExtension.cs
@@ -0,0 +1,90 @@
+﻿using System;
+using System.Globalization;
+using System.Net;
+
+namespace dotnetCampus.FileDownloader.Utils
+{
+    static class WebRequestExtension
+    {
+        public static void AddRange(this WebRequest webRequest, int from, int to)
+        {
+            webRequest.AddRange("bytes", from, to);
+        }
+
+  
+        public static void AddRange(this WebRequest webRequest, long from, long to)
+        {
+            webRequest.AddRange("bytes", from, to);
+        }
+
+        public static void AddRange(this WebRequest webRequest, string rangeSpecifier, long from, long to)
+        {
+            if (rangeSpecifier == null)
+            {
+                throw new ArgumentNullException(nameof(rangeSpecifier));
+            }
+            if ((from < 0) || (to < 0))
+            {
+                throw new ArgumentOutOfRangeException(from < 0 ? nameof(from) : nameof(to), "Range 太小了");
+            }
+            if (from > to)
+            {
+                throw new ArgumentOutOfRangeException(nameof(from), "传入的 From 比 To 大");
+            }
+         
+            if (!AddRange(webRequest,rangeSpecifier, from.ToString(NumberFormatInfo.InvariantInfo), to.ToString(NumberFormatInfo.InvariantInfo)))
+            {
+                throw new InvalidOperationException();
+            }
+        }
+
+        public static void AddRange(this WebRequest webRequest, string rangeSpecifier, int range)
+        {
+            webRequest.AddRange(rangeSpecifier, (long) range);
+        }
+
+        public static void AddRange(this WebRequest webRequest, string rangeSpecifier, long range)
+        {
+            if (rangeSpecifier == null)
+            {
+                throw new ArgumentNullException(nameof(rangeSpecifier));
+            }
+
+            if (!AddRange(webRequest,rangeSpecifier, range.ToString(NumberFormatInfo.InvariantInfo), (range >= 0) ? "" : null))
+            {
+                throw new InvalidOperationException();
+            }
+        }
+
+        private static bool AddRange(WebRequest webRequest,string rangeSpecifier, string from, string? to)
+        {
+            var webHeaderCollection = webRequest.Headers;
+            string? curRange = webHeaderCollection[HttpKnownHeaderNames.Range];
+
+            if ((curRange == null) || (curRange.Length == 0))
+            {
+                curRange = rangeSpecifier + "=";
+            }
+            else
+            {
+                if (!string.Equals(curRange.Substring(0, curRange.IndexOf('=')), rangeSpecifier, StringComparison.OrdinalIgnoreCase))
+                {
+                    return false;
+                }
+                curRange = string.Empty;
+            }
+            curRange += @from;
+            if (to != null)
+            {
+                curRange += "-" + to;
+            }
+            webHeaderCollection[HttpKnownHeaderNames.Range] = curRange;
+            return true;
+        }
+
+        internal static class HttpKnownHeaderNames
+        {
+            public const string Range = "Range";
+        }
+    }
+}
