From 2c2a0fcbcda47160d7495318e61e3870fe3a54bf Mon Sep 17 00:00:00 2001 From: Jakub Szczyrk <35535734+Szczyrk@users.noreply.github.com> Date: Mon, 20 Sep 2021 16:38:28 +0200 Subject: [PATCH 1/9] Reduced number of events sent for version integration check. (#167) Co-authored-by: Jakub Szczyrk --- .../AppUpdater/Commands/CheckVersionIntegrityCommand.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/CheckVersionIntegrityCommand.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/CheckVersionIntegrityCommand.cs index f6ebb4d0..b763b37e 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/CheckVersionIntegrityCommand.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/CheckVersionIntegrityCommand.cs @@ -115,10 +115,16 @@ private void ExecuteInternal(CancellationToken cancellationToken) _status.IsActive.Value = false; } + private int _eventSendCount; private FileIntegrity CheckFile(AppContentSummaryFile file) { Action onVerificationFailed = () => { + _eventSendCount++; + if (_eventSendCount > 32) + { + return; + } PatcherStatistics.DispatchSendEvent(PatcherStatistics.Event.FileVerificationFailed, new PatcherStatistics.OptionalParams { FileName = file.Path, From 58aa39aa0d251bda6ade6b23cbfb7049d4833612 Mon Sep 17 00:00:00 2001 From: Jakub Szczyrk <35535734+Szczyrk@users.noreply.github.com> Date: Mon, 20 Sep 2021 16:47:10 +0200 Subject: [PATCH 2/9] Caching for diff and content app summary (#171) Co-authored-by: Jakub Szczyrk --- .../Scripts/AppData/Remote/RemoteMetaData.cs | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/Assets/PatchKit Patcher/Scripts/AppData/Remote/RemoteMetaData.cs b/Assets/PatchKit Patcher/Scripts/AppData/Remote/RemoteMetaData.cs index 7853df6b..4376846b 100644 --- a/Assets/PatchKit Patcher/Scripts/AppData/Remote/RemoteMetaData.cs +++ b/Assets/PatchKit Patcher/Scripts/AppData/Remote/RemoteMetaData.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using JetBrains.Annotations; using PatchKit.Api; using PatchKit.Api.Models.Main; @@ -17,6 +18,9 @@ public class RemoteMetaData : IRemoteMetaData private readonly MainApiConnection _mainApiConnection; private readonly KeysApiConnection _keysApiConnection; + public Dictionary _cacheAppContentSummary = new Dictionary(); + public Dictionary _cacheAppDiffSummary = new Dictionary(); + public RemoteMetaData([NotNull] string appSecret, [NotNull] IRequestTimeoutCalculator requestTimeoutCalculator) { if (string.IsNullOrEmpty(appSecret)) @@ -77,8 +81,16 @@ public AppContentSummary GetContentSummary(int versionId, CancellationToken canc { Checks.ArgumentValidVersionId(versionId, "versionId"); DebugLogger.Log(string.Format("Getting content summary of version with id {0}.", versionId)); + AppContentSummary appContentSummary; + if (_cacheAppContentSummary.TryGetValue(versionId, out appContentSummary)) + { + return appContentSummary; + } - return _mainApiConnection.GetAppVersionContentSummary(_appSecret, versionId, cancellationToken); + appContentSummary = + _mainApiConnection.GetAppVersionContentSummary(_appSecret, versionId, cancellationToken); + _cacheAppContentSummary.Add(versionId, appContentSummary); + return appContentSummary; } public AppDiffSummary GetDiffSummary(int versionId, CancellationToken cancellationToken) @@ -86,7 +98,16 @@ public AppDiffSummary GetDiffSummary(int versionId, CancellationToken cancellati Checks.ArgumentValidVersionId(versionId, "versionId"); DebugLogger.Log(string.Format("Getting diff summary of version with id {0}.", versionId)); - return _mainApiConnection.GetAppVersionDiffSummary(_appSecret, versionId, cancellationToken); + AppDiffSummary appDiffSummary; + if (_cacheAppDiffSummary.TryGetValue(versionId, out appDiffSummary)) + { + return appDiffSummary; + } + + appDiffSummary = + _mainApiConnection.GetAppVersionDiffSummary(_appSecret, versionId, cancellationToken); + _cacheAppDiffSummary.Add(versionId, appDiffSummary); + return appDiffSummary; } public string GetKeySecret(string key, string cachedKeySecret, CancellationToken cancellationToken) From c1969616188bc849c4ba79e6414514b0489c270a Mon Sep 17 00:00:00 2001 From: Jakub Szczyrk <35535734+Szczyrk@users.noreply.github.com> Date: Mon, 20 Sep 2021 16:49:34 +0200 Subject: [PATCH 3/9] The retry button resumes the update (#177) Co-authored-by: Jakub Szczyrk --- .../Scripts/UI/Dialogs/ErrorDialog.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Assets/PatchKit Patcher/Scripts/UI/Dialogs/ErrorDialog.cs b/Assets/PatchKit Patcher/Scripts/UI/Dialogs/ErrorDialog.cs index e32368b1..d8b80aa3 100644 --- a/Assets/PatchKit Patcher/Scripts/UI/Dialogs/ErrorDialog.cs +++ b/Assets/PatchKit Patcher/Scripts/UI/Dialogs/ErrorDialog.cs @@ -1,5 +1,7 @@ -using PatchKit.Unity.Patcher.Cancellation; +using System.Collections; +using PatchKit.Unity.Patcher.Cancellation; using PatchKit.Unity.Utilities; +using UnityEngine; using UnityEngine.UI; namespace PatchKit.Unity.Patcher.UI.Dialogs @@ -11,6 +13,16 @@ public class ErrorDialog : Dialog public void Confirm() { OnDisplayed(); + StartCoroutine(Retry()); + } + + public IEnumerator Retry() + { + while (Patcher.Instance.State.Value != PatcherState.WaitingForUserDecision) + { + yield return new WaitForSeconds(1); + } + Patcher.Instance.SetUserDecision(Patcher.UserDecision.InstallApp); } public void Display(PatcherError error, CancellationToken cancellationToken) From 4bddaecd70c68da741a47416fb2ac50aaf264b26 Mon Sep 17 00:00:00 2001 From: Jakub Szczyrk <35535734+Szczyrk@users.noreply.github.com> Date: Wed, 22 Sep 2021 13:45:01 +0200 Subject: [PATCH 4/9] Bugfix/v3.17.x.x/ Repair cost and decision (#169) * Fix repair cost - consider block size. Decision repair always when invalid files. * Refactoring * Refactoring Co-authored-by: Jakub Szczyrk --- .../Scripts/AppUpdater/AppRepairer.cs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/AppRepairer.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/AppRepairer.cs index ad7d4e83..c2790ca4 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/AppRepairer.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/AppRepairer.cs @@ -1,10 +1,6 @@ using System; -using System.IO; using System.Linq; -using System.Linq.Expressions; -using System.Threading; using System.Collections.Generic; -using PatchKit.Unity.Patcher.Cancellation; using PatchKit.Unity.Patcher.Debug; using PatchKit.Unity.Patcher.AppUpdater.Commands; using PatchKit.Unity.Patcher.AppUpdater.Status; @@ -34,7 +30,6 @@ public class AppRepairer private const double IncreaseRepairCost = 1.5d; - public AppRepairer(AppUpdaterContext context, UpdaterStatus status) { DebugLogger.LogConstructor(); @@ -162,17 +157,16 @@ int installedVersionId private IEnumerable FilesNeedFixing(VersionIntegrity results) { - var missingFiles = results.Files.Where(f => f.Status == FileIntegrityStatus.MissingData); - var invalidSizeFiles = results.Files.Where(f => f.Status == FileIntegrityStatus.InvalidSize); + var invalidFiles = results.Files.Where(f => f.Status != FileIntegrityStatus.Ok); - return missingFiles.Concat(invalidSizeFiles); + return invalidFiles; } private long CalculateRepairCost(AppContentSummary contentSummary, IEnumerable filesToRepair) { return filesToRepair .Select(f => contentSummary.Files.FirstOrDefault(e => e.Path == f.FileName)) - .Sum(f => f.Size); + .Sum(f => Math.Max(contentSummary.Chunks.Size, f.Size)); } private void ReinstallContent(PatchKit.Unity.Patcher.Cancellation.CancellationToken cancellationToken) From 541527674d633afa9befb0395150f5efcd1512ff Mon Sep 17 00:00:00 2001 From: Jakub Szczyrk <35535734+Szczyrk@users.noreply.github.com> Date: Wed, 22 Sep 2021 13:45:28 +0200 Subject: [PATCH 5/9] Restriction of hash repair for a large number of files (#175) * Restriction of hash repair for a large number of files * Refactoring * Refactoring Co-authored-by: Jakub Szczyrk --- .../Scripts/AppUpdater/AppUpdaterContentStrategy.cs | 12 +++++++++++- .../AppUpdater/Commands/CheckDiskSpaceCommand.cs | 1 + .../AppUpdater/Commands/CheckPathLengthCommand.cs | 5 ++--- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterContentStrategy.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterContentStrategy.cs index 54aa324a..ebd0e271 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterContentStrategy.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterContentStrategy.cs @@ -15,6 +15,8 @@ public class AppUpdaterContentStrategy: IAppUpdaterStrategy private readonly UpdaterStatus _status; + private const int RepairCheckHashesMaxFilesCount = 10000; + private bool _updateHasBeenCalled; public bool RepairOnError { get; set; } @@ -119,7 +121,15 @@ public void Update(CancellationToken cancellationToken) DebugLogger.Log("Content installed with errors, requesting repair"); var appRepairer = new AppRepairer(_context, _status); - appRepairer.CheckHashes = true; + if (_context.App.RemoteMetaData.GetContentSummary(latestVersionId, cancellationToken).Files.Length < RepairCheckHashesMaxFilesCount) + { + appRepairer.CheckHashes = true; + } + else + { + appRepairer.CheckHashes = false; + } + if (!appRepairer.Perform(cancellationToken)) { diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/CheckDiskSpaceCommand.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/CheckDiskSpaceCommand.cs index efe0c955..1249a8dd 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/CheckDiskSpaceCommand.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/CheckDiskSpaceCommand.cs @@ -66,6 +66,7 @@ public void Execute(CancellationToken cancellationToken) _status.IsActive.Value = true; _status.IsIdle.Value = true; + DebugLogger.Log("Check Disk Space Command Execute..."); try { long availableDiskSpace = -1; diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/CheckPathLengthCommand.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/CheckPathLengthCommand.cs index 150012c2..f43737a2 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/CheckPathLengthCommand.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/CheckPathLengthCommand.cs @@ -27,13 +27,12 @@ public void Execute(CancellationToken cancellationToken) { _status.IsActive.Value = true; _status.IsIdle.Value = true; - + DebugLogger.Log("Check Path Length Command Execute..."); try { - string pathFile; foreach (AppContentSummaryFile contentSummaryFile in _contentSummary.Value.Files) { - pathFile = Path.Combine(_localDirectoryPath, contentSummaryFile.Path); + string pathFile = Path.Combine(_localDirectoryPath, contentSummaryFile.Path); if (pathFile.Length > 259) { From ec1d9c1cb6cb8f606a38386af96651c7112674c9 Mon Sep 17 00:00:00 2001 From: Jakub Szczyrk <35535734+Szczyrk@users.noreply.github.com> Date: Wed, 22 Sep 2021 13:47:26 +0200 Subject: [PATCH 6/9] Remove check version for diff (#168) * Remove check version for diff * VersionIntegrity was used instead of ICheckVersionIntegrityCommand Co-authored-by: Jakub Szczyrk --- .../Scripts/AppUpdater/AppRepairer.cs | 10 ++++++---- .../Scripts/AppUpdater/AppUpdater.cs | 4 +++- .../AppUpdater/AppUpdaterStrategyResolver.cs | 16 +++++----------- .../AppUpdater/IAppUpdaterStrategyResolver.cs | 3 ++- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/AppRepairer.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/AppRepairer.cs index c2790ca4..e9d3100f 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/AppRepairer.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/AppRepairer.cs @@ -14,6 +14,8 @@ public class AppRepairer public readonly AppUpdaterContext Context; + public VersionIntegrity VersionIntegrity; + // set to true if you wish to check file hashes public bool CheckHashes = false; @@ -60,8 +62,8 @@ public bool Perform(PatchKit.Unity.Patcher.Cancellation.CancellationToken cancel // retry count reached, let's check for the last time if data is ok, but without repairing int installedVersionId = Context.App.GetInstalledVersionId(); - VersionIntegrity results = CheckIntegrity(cancellationToken, installedVersionId); - var filesNeedFixing = FilesNeedFixing(results); + VersionIntegrity = CheckIntegrity(cancellationToken, installedVersionId); + var filesNeedFixing = FilesNeedFixing(VersionIntegrity); if (filesNeedFixing.Count() == 0) { @@ -79,8 +81,8 @@ private bool PerformInternal(PatchKit.Unity.Patcher.Cancellation.CancellationTok { int installedVersionId = Context.App.GetInstalledVersionId(); - VersionIntegrity results = CheckIntegrity(cancellationToken, installedVersionId); - var filesNeedFixing = FilesNeedFixing(results); + VersionIntegrity = CheckIntegrity(cancellationToken, installedVersionId); + var filesNeedFixing = FilesNeedFixing(VersionIntegrity); if (filesNeedFixing.Count() == 0) { diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdater.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdater.cs index 76982f13..a337becd 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdater.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdater.cs @@ -27,6 +27,7 @@ public class AppUpdater private bool _updateHasBeenCalled; private bool _uninstallHasBeenCalled; private bool _verifyFilesHasBeenCalled; + private VersionIntegrity _versionIntegrity; public IReadOnlyUpdaterStatus Status { @@ -47,6 +48,7 @@ private void PreUpdate(PatchKit.Unity.Patcher.Cancellation.CancellationToken can { var appRepairer = new AppRepairer(Context, _status); appRepairer.Perform(cancellationToken); + _versionIntegrity = appRepairer.VersionIntegrity; } public void Update(PatchKit.Unity.Patcher.Cancellation.CancellationToken cancellationToken) @@ -60,7 +62,7 @@ public void Update(PatchKit.Unity.Patcher.Cancellation.CancellationToken cancell DebugLogger.Log("Updating."); - StrategyType type = _strategyResolver.Resolve(Context, cancellationToken); + StrategyType type = _strategyResolver.Resolve(Context, _versionIntegrity, cancellationToken); _strategy = _strategyResolver.Create(type, Context); try diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterStrategyResolver.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterStrategyResolver.cs index 24fc3d02..da723835 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterStrategyResolver.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterStrategyResolver.cs @@ -59,7 +59,7 @@ public StrategyType GetFallbackStrategy(StrategyType strategyType) } } - public StrategyType Resolve([NotNull] AppUpdaterContext context, CancellationToken cancellationToken) + public StrategyType Resolve([NotNull] AppUpdaterContext context, VersionIntegrity versionIntegrity, CancellationToken cancellationToken) { if (context == null) { @@ -129,7 +129,7 @@ public StrategyType Resolve([NotNull] AppUpdaterContext context, CancellationTok { _logger.LogDebug("Checking consitency before allowing diff update..."); - if (!IsVersionIntegral(contentSize, context, cancellationToken)) + if (!IsVersionIntegral(contentSize, versionIntegrity, context, cancellationToken)) { if (IsRepairPossible(context, cancellationToken)) { @@ -195,7 +195,7 @@ private bool DoesVersionSupportDiffUpdates(AppUpdaterContext context, int versio return versionId + 1 < lowestVersionWithDiffId; } - private bool IsVersionIntegral(long contentSize, AppUpdaterContext context, CancellationToken cancellationToken) + private bool IsVersionIntegral(long contentSize, VersionIntegrity versionIntegrity, AppUpdaterContext context, CancellationToken cancellationToken) { var commandFactory = new AppUpdaterCommandFactory(); int installedVersionId = context.App.GetInstalledVersionId(); @@ -205,18 +205,12 @@ private bool IsVersionIntegral(long contentSize, AppUpdaterContext context, Canc bool isCheckingHash = contentSize < context.Configuration.HashSizeThreshold; _logger.LogTrace("isCheckingHash = " + isCheckingHash); - var checkVersionIntegrity = commandFactory.CreateCheckVersionIntegrityCommand( - installedVersionId, context, isCheckingHash, true, cancellationToken); - - checkVersionIntegrity.Prepare(_status, cancellationToken); - checkVersionIntegrity.Execute(CancellationToken.Empty); - - bool isValid = checkVersionIntegrity.Results.Files.All( + bool isValid = versionIntegrity.Files.All( fileIntegrity => fileIntegrity.Status == FileIntegrityStatus.Ok); if (!isValid) { - foreach (var fileIntegrity in checkVersionIntegrity.Results.Files) + foreach (var fileIntegrity in versionIntegrity.Files) { if (fileIntegrity.Status != FileIntegrityStatus.Ok) { diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/IAppUpdaterStrategyResolver.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/IAppUpdaterStrategyResolver.cs index ce9f2314..5a3bbeb1 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/IAppUpdaterStrategyResolver.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/IAppUpdaterStrategyResolver.cs @@ -1,11 +1,12 @@ using System; +using PatchKit.Unity.Patcher.AppUpdater.Commands; using PatchKit.Unity.Patcher.Cancellation; namespace PatchKit.Unity.Patcher.AppUpdater { public interface IAppUpdaterStrategyResolver { - StrategyType Resolve(AppUpdaterContext context, CancellationToken cancellationToken); + StrategyType Resolve(AppUpdaterContext context, VersionIntegrity versionIntegrity, CancellationToken cancellationToken); IAppUpdaterStrategy Create(StrategyType type, AppUpdaterContext context); From c76393ee6de2fbc9ee5c24fb10d9e97164f5a939 Mon Sep 17 00:00:00 2001 From: Jakub Szczyrk <35535734+Szczyrk@users.noreply.github.com> Date: Wed, 22 Sep 2021 14:34:06 +0200 Subject: [PATCH 7/9] Features/v3.17.x.x Improved diff update (#172) * Unpacking only added and modified files. For unchanged files only entry change at app_data.json. Refactoring LocalMetaData * Refactoring * Refactoring Co-authored-by: Jakub Szczyrk --- Assets/Editor/Tests/LocalMetaDataTest.cs | 6 +- .../Scripts/AppData/Local/ILocalMetaData.cs | 5 +- .../Scripts/AppData/Local/LocalMetaData.cs | 31 ++-- .../AppUpdater/AppUpdaterRepairStrategy.cs | 7 +- .../Commands/InstallContentCommand.cs | 8 +- .../AppUpdater/Commands/InstallDiffCommand.cs | 135 +++++++++--------- 6 files changed, 101 insertions(+), 91 deletions(-) diff --git a/Assets/Editor/Tests/LocalMetaDataTest.cs b/Assets/Editor/Tests/LocalMetaDataTest.cs index 33cb32b6..0d7e2880 100644 --- a/Assets/Editor/Tests/LocalMetaDataTest.cs +++ b/Assets/Editor/Tests/LocalMetaDataTest.cs @@ -31,7 +31,7 @@ public void CreateMetaDataFile() var localMetaData = new LocalMetaData(_filePath, _deprecatedFilePath); - localMetaData.RegisterEntry("test", 1, 0, true); + localMetaData.RegisterEntry("test", 1, 0); Assert.True(File.Exists(_filePath)); } @@ -41,8 +41,8 @@ public void SaveValidFileSinglePass() { var localMetaData = new LocalMetaData(_filePath, _deprecatedFilePath); - localMetaData.RegisterEntry("a", 1, 0 , true); - localMetaData.RegisterEntry("b", 2, 0 , true); + localMetaData.RegisterEntry("a", 1, 0); + localMetaData.RegisterEntry("b", 2, 0); var localMetaData2 = new LocalMetaData(_filePath, _deprecatedFilePath); diff --git a/Assets/PatchKit Patcher/Scripts/AppData/Local/ILocalMetaData.cs b/Assets/PatchKit Patcher/Scripts/AppData/Local/ILocalMetaData.cs index 4b402321..55d6d364 100644 --- a/Assets/PatchKit Patcher/Scripts/AppData/Local/ILocalMetaData.cs +++ b/Assets/PatchKit Patcher/Scripts/AppData/Local/ILocalMetaData.cs @@ -17,8 +17,7 @@ public interface ILocalMetaData /// Name of the entry. /// The version id. /// Size of entry. - /// If set to true, it is last entry. - void RegisterEntry(string entryName, int versionId, long entrySize, bool isLastEntry); + void RegisterEntry(string entryName, int versionId, long? entrySize); /// /// Unregisters the entry. @@ -52,5 +51,7 @@ public interface ILocalMetaData string GetMainExecutable(); string MainExecutableArgs { get; } + + void SaveData(); } } \ No newline at end of file diff --git a/Assets/PatchKit Patcher/Scripts/AppData/Local/LocalMetaData.cs b/Assets/PatchKit Patcher/Scripts/AppData/Local/LocalMetaData.cs index 5a45594e..6c23ff0a 100644 --- a/Assets/PatchKit Patcher/Scripts/AppData/Local/LocalMetaData.cs +++ b/Assets/PatchKit Patcher/Scripts/AppData/Local/LocalMetaData.cs @@ -21,8 +21,10 @@ public class LocalMetaData : ILocalMetaData { private readonly ILogger _logger; private long _unsavedEntriesSize = 0; + private int _unsavedEntriesCount = 0; private const string DeprecatedCachePatchKitKey = "patchkit-key"; private const long UnsavedEntriesSizeLimit = 104857600; //100MiB + private const long UnsavedEntriesCountLimit = 100; /// /// Data structure stored in file. @@ -81,7 +83,7 @@ public string[] GetRegisteredEntries() return _data.FileVersionIds.Select(pair => pair.Key).ToArray(); } - public void RegisterEntry([NotNull] string entryName, int versionId, long entrySize, bool isLastEntry) + public void RegisterEntry([NotNull] string entryName, int versionId, long? entrySize) { if (entryName == null) { @@ -102,10 +104,9 @@ public void RegisterEntry([NotNull] string entryName, int versionId, long entryS try { _logger.LogDebug(string.Format("Registering entry {0} as version {1}.", entryName, versionId)); - _data.FileVersionIds[entryName] = versionId; - - if (ShouldSaveEntry(entrySize, isLastEntry)) + + if (ShouldSaveEntry(entrySize)) { SaveData(); } @@ -119,19 +120,22 @@ public void RegisterEntry([NotNull] string entryName, int versionId, long entryS } } - private bool ShouldSaveEntry(long entrySize, bool isLastEntry) + private bool ShouldSaveEntry(long? entrySize) { - if (isLastEntry) + if (entrySize.HasValue) { - return true; + _unsavedEntriesSize += entrySize.Value; + if (_unsavedEntriesSize > UnsavedEntriesSizeLimit) + { + return true; + } } - _unsavedEntriesSize += entrySize; - if (_unsavedEntriesSize > UnsavedEntriesSizeLimit) + if (++_unsavedEntriesCount > UnsavedEntriesCountLimit) { - _unsavedEntriesSize = 0; return true; } + return false; } @@ -201,7 +205,7 @@ public string GetProductKey() { return _data.ProductKey; } - + public void SetMainExecutableAndArgs(string mainExecutable, string mainExecutableArgs) { if (_data.MainExecutable != mainExecutable || @@ -233,10 +237,11 @@ private void CreateDataDir() } } - private void SaveData() + public void SaveData() { _logger.LogDebug(string.Format("Saving data to {0}", _filePath)); - + _unsavedEntriesSize = 0; + _unsavedEntriesCount = 0; CreateDataDir(); Files.WriteAllText(_filePath, JsonConvert.SerializeObject(_data, Formatting.None)); diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterRepairStrategy.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterRepairStrategy.cs index 8b070d7e..915d2466 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterRepairStrategy.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterRepairStrategy.cs @@ -97,18 +97,17 @@ ICheckVersionIntegrityCommand checkVersionIntegrityCommand if (actualFileHash != file.Hash) { FileOperations.Delete(localPath, cancellationToken); - _context.App.LocalMetaData.RegisterEntry(fileName, installedVersionId, file.Size, - i == filesIntegrityWithInvalidVersion.Length - 1); + _context.App.LocalMetaData.RegisterEntry(fileName, installedVersionId, file.Size); filesIntegrityWithInvalidVersion[i].Status = FileIntegrityStatus.MissingData; } else { - _context.App.LocalMetaData.RegisterEntry(fileName, installedVersionId, file.Size, - i == filesIntegrityWithInvalidVersion.Length - 1); + _context.App.LocalMetaData.RegisterEntry(fileName, installedVersionId, file.Size); filesIntegrityWithInvalidVersion[i].Status = FileIntegrityStatus.Ok; } } + _context.App.LocalMetaData.SaveData(); Pack1Meta.FileEntry[] brokenFiles = filesIntegrity // Filter only files with invalid size, hash or missing entirely .Where(f => f.Status == FileIntegrityStatus.InvalidHash diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/InstallContentCommand.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/InstallContentCommand.cs index 9a55faa9..a40f4aa9 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/InstallContentCommand.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/InstallContentCommand.cs @@ -153,7 +153,7 @@ public override void Execute(CancellationToken cancellationToken) } else { - InstallFile(sourceFile, cancellationToken, i == _versionContentSummary.Files.Length - 1); + InstallFile(sourceFile, cancellationToken); } } else @@ -165,6 +165,8 @@ public override void Execute(CancellationToken cancellationToken) _copyFilesStatus.Description.Value = string.Format("Installing ({0}/{1})...", i + 1, _versionContentSummary.Files.Length); } + _localMetaData.SaveData(); + _copyFilesStatus.Progress.Value = 1.0; _copyFilesStatus.IsActive.Value = false; }); @@ -186,7 +188,7 @@ private IUnarchiver CreateUnrachiver(string destinationDir, MapHashExtractedFile } } - private void InstallFile(SourceFile sourceFile, CancellationToken cancellationToken, bool isLastEntry) + private void InstallFile(SourceFile sourceFile, CancellationToken cancellationToken) { DebugLogger.Log(string.Format("Installing file {0}", sourceFile.Name)); @@ -211,7 +213,7 @@ private void InstallFile(SourceFile sourceFile, CancellationToken cancellationTo } #endif FileOperations.Move(sourceFile.FullHashPath, destinationFilePath, cancellationToken); - _localMetaData.RegisterEntry(sourceFile.Name, _versionId, sourceFile.Size, isLastEntry); + _localMetaData.RegisterEntry(sourceFile.Name, _versionId, sourceFile.Size); } struct SourceFile diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/InstallDiffCommand.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/InstallDiffCommand.cs index 557d8dd4..0dc988c9 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/InstallDiffCommand.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/InstallDiffCommand.cs @@ -189,12 +189,13 @@ public override void Execute(CancellationToken cancellationToken) ProcessRemovedFiles(cancellationToken); } - TemporaryDirectory.ExecuteIn(_packagePath + ".temp_diff_" + Path.GetRandomFileName(), (tempDiffDir) => - { - _logger.LogTrace("tempDiffDir = " + tempDiffDir.Path); + TemporaryDirectory.ExecuteIn(_packagePath + ".temp_diff_" + Path.GetRandomFileName(), + (tempDiffDir) => + { + _logger.LogTrace("tempDiffDir = " + tempDiffDir.Path); - ProcessModifiedFiles(packageDir.Path, usedSuffix, tempDiffDir, cancellationToken); - }); + ProcessModifiedFiles(packageDir.Path, usedSuffix, tempDiffDir, cancellationToken); + }); DeleteEmptyMacAppDirectories(cancellationToken); }); @@ -250,10 +251,19 @@ private void ReadPack1MetaFile() _logger.LogTrace("pack1Meta.iv = " + _pack1Meta.Iv); _logger.LogTrace("pack1Meta.version = " + _pack1Meta.Version); _logger.LogTrace("pack1Meta.encryption = " + _pack1Meta.Encryption); - for (int i = 0; i < _pack1Meta.Files.Length; i++) + + var filesMeta = _pack1Meta.Files; + List fileEntries = new List(); + for (int i = 0; i < filesMeta.Length; i++) { - _logger.LogTrace(string.Format("pack1Meta.files[{0}] = {1}", i, _pack1Meta.Files[i])); + if (!_diffSummary.UnchangedFiles.Contains(filesMeta[i].Name)) + { + fileEntries.Add(filesMeta[i]); + _logger.LogTrace(string.Format("pack1Meta.files[{0}] = {1}", i, _pack1Meta.Files[i])); + } } + + _pack1Meta.Files = fileEntries.ToArray(); } private void UnarchivePackage(string packageDirPath, out string usedSuffix, CancellationToken cancellationToken) @@ -281,9 +291,11 @@ private void UnarchivePackage(string packageDirPath, out string usedSuffix, Canc var entryMinProgress = (entry - 1) / (double) amount; var entryMaxProgress = entry / (double) amount; - _unarchivePackageStatusReporter.Progress.Value = entryMinProgress + (entryMaxProgress - entryMinProgress) * entryProgress; + _unarchivePackageStatusReporter.Progress.Value = + entryMinProgress + (entryMaxProgress - entryMinProgress) * entryProgress; - _unarchivePackageStatusReporter.Description.Value = string.Format("Unarchiving package ({0}/{1})...", entry, amount); + _unarchivePackageStatusReporter.Description.Value = + string.Format("Unarchiving package ({0}/{1})...", entry, amount); }; unarchiver.Unarchive(cancellationToken); @@ -303,7 +315,8 @@ private IUnarchiver CreateUnrachiver(string destinationDir, out string usedSuffi return new ZipUnarchiver(_packagePath, destinationDir, _mapHashExtractedFiles, _packagePassword); case "pack1": usedSuffix = Suffix; - return new Pack1Unarchiver(_packagePath, _pack1Meta, destinationDir, _mapHashExtractedFiles, _packagePassword, Suffix); + return new Pack1Unarchiver(_packagePath, _pack1Meta, destinationDir, _mapHashExtractedFiles, + _packagePassword, Suffix); default: throw new UnknownPackageCompressionModeException(string.Format("Unknown compression method: {0}", _diffSummary.CompressionMethod)); @@ -439,6 +452,8 @@ private void ProcessAddedFiles(string packageDirPath, string suffix, _addFilesStatusReporter.Description.Value = string.Format("Adding new files ({0}/{1})...", i + 1, _diffSummary.AddedFiles.Length); } + _localMetaData.SaveData(); + _addFilesStatusReporter.Progress.Value = 1.0; _addFilesStatusReporter.IsActive.Value = false; @@ -469,7 +484,9 @@ private void AddFile(string fileName, string packageDirPath, string suffix, Canc #if UNITY_STANDALONE_WIN if (filePath.Length > 259) { - throw new FilePathTooLongException(string.Format("Cannot install file {0}, the destination path length has exceeded Windows path length limit (260).", filePath)); + throw new FilePathTooLongException(string.Format( + "Cannot install file {0}, the destination path length has exceeded Windows path length limit (260).", + filePath)); } #endif string nameHash; @@ -483,7 +500,7 @@ private void AddFile(string fileName, string packageDirPath, string suffix, Canc throw new MissingFileFromPackageException(string.Format("Cannot find file {0} in diff package.", fileName)); } - + _logger.LogDebug("Creating file parent directories in local data..."); var fileParentDirPath = Path.GetDirectoryName(filePath); _logger.LogTrace("fileParentDirPath = " + fileParentDirPath); @@ -496,9 +513,8 @@ private void AddFile(string fileName, string packageDirPath, string suffix, Canc FileOperations.Copy(sourceFilePath, filePath, true, cancellationToken); _logger.LogDebug("File copied to local data."); - _localMetaData.RegisterEntry(fileName, _versionId, - _contentSummary.Files.First(x => x.Path == fileName).Size, - fileIndex == _diffSummary.AddedFiles.Length - 1); + _localMetaData.RegisterEntry(fileName, _versionId, + _contentSummary.Files.First(x => x.Path == fileName).Size); _logger.LogDebug("Add file entry processed."); } @@ -516,21 +532,35 @@ private void ProcessModifiedFiles(string packageDirPath, string suffix, Temporar _modifiedFilesStatusReporter.IsActive.Value = true; _modifiedFilesStatusReporter.Description.Value = "Applying diffs..."; - for (int i = 0; i < _diffSummary.ModifiedFiles.Length; i++) + var entryNames = _pack1Meta.Files.Select(f => f.Name).Where(f => _diffSummary.ModifiedFiles.Contains(f)) + .ToArray(); + for (int i = 0; i < entryNames.Length; i++) { cancellationToken.ThrowIfCancellationRequested(); - var entryName = _diffSummary.ModifiedFiles[i]; + var entryName = entryNames[i]; if (!entryName.EndsWith("/")) { PatchFile(entryName, packageDirPath, suffix, tempDiffDir, cancellationToken, i); } - _modifiedFilesStatusReporter.Progress.Value = (i + 1) / (double) _diffSummary.ModifiedFiles.Length; - _modifiedFilesStatusReporter.Description.Value = string.Format("Applying diffs ({0}/{1})...", i + 1, _diffSummary.ModifiedFiles.Length); + _modifiedFilesStatusReporter.Progress.Value = (i + 1) / (double) entryNames.Length; + _modifiedFilesStatusReporter.Description.Value = + string.Format("Applying diffs ({0}/{1})...", i + 1, entryNames.Length); } + _localMetaData.SaveData(); + + _logger.LogDebug("Register entry for unchanged files."); + _modifiedFilesStatusReporter.Description.Value = "Applying diffs..."; + + foreach (string fileName in _diffSummary.UnchangedFiles) + { + _localMetaData.RegisterEntry(fileName, _versionId, null); + } + + _localMetaData.SaveData(); _modifiedFilesStatusReporter.Progress.Value = 1.0; _modifiedFilesStatusReporter.IsActive.Value = false; @@ -563,69 +593,42 @@ private void PatchFile( fileName, _versionId - 1, fileVersion)); } - _logger.LogDebug("Checking whether patching file content is necessary..."); - if (IsPatchingFileContentNecessary(fileName)) + string nameHash; + if (_mapHashExtractedFiles.TryGetHash(fileName, out nameHash)) { - _logger.LogDebug("Patching is necessary. Generating new file with patched content..."); + var sourceDeltaFilePath = Path.Combine(packageDirPath, nameHash + suffix); + _logger.LogTrace("sourceDeltaFilePath = " + sourceDeltaFilePath); - string nameHash; - if (_mapHashExtractedFiles.TryGetHash(fileName, out nameHash)) + if (!File.Exists(sourceDeltaFilePath)) { - var sourceDeltaFilePath = Path.Combine(packageDirPath, nameHash + suffix); - _logger.LogTrace("sourceDeltaFilePath = " + sourceDeltaFilePath); - - if (!File.Exists(sourceDeltaFilePath)) - { - throw new MissingFileFromPackageException(string.Format( - "Cannot find delta file {0} in diff package.", - fileName)); - } + throw new MissingFileFromPackageException(string.Format( + "Cannot find delta file {0} in diff package.", + fileName)); + } - var newFilePath = tempDiffDir.GetUniquePath(); - _logger.LogTrace("newFilePath = " + newFilePath); + var newFilePath = tempDiffDir.GetUniquePath(); + _logger.LogTrace("newFilePath = " + newFilePath); - var filePatcher = new FilePatcher(filePath, sourceDeltaFilePath, newFilePath); - filePatcher.Patch(); + var filePatcher = new FilePatcher(filePath, sourceDeltaFilePath, newFilePath); + filePatcher.Patch(); - _logger.LogDebug("New file generated. Deleting old file in local data..."); - FileOperations.Delete(filePath, cancellationToken); + _logger.LogDebug("New file generated. Deleting old file in local data..."); + FileOperations.Delete(filePath, cancellationToken); - _logger.LogDebug("Old file deleted. Moving new file to local data..."); - FileOperations.Move(newFilePath, filePath, cancellationToken); + _logger.LogDebug("Old file deleted. Moving new file to local data..."); + FileOperations.Move(newFilePath, filePath, cancellationToken); - _logger.LogDebug("New file moved."); - } - else - { - throw new InstallerException(string.Format("Cannot find hash for file {0} in mapHash.", fileName)); - } + _logger.LogDebug("New file moved."); } else { - _logger.LogDebug("Patching is not necessary. File content is the same as in previous version."); + throw new InstallerException(string.Format("Cannot find hash for file {0} in mapHash.", fileName)); } - - _localMetaData.RegisterEntry(fileName, _versionId, - _contentSummary.Files.First(x => x.Path == fileName).Size, - fileIndex == _diffSummary.ModifiedFiles.Length - 1); + _localMetaData.RegisterEntry(fileName, _versionId, null); _logger.LogDebug("Patch file entry processed."); } - private bool IsPatchingFileContentNecessary(string fileName) - { - if (_diffSummary.UnchangedFiles.Contains(fileName)) - { - return false; - } - - //TODO: Throw exceptions if file is not present in any of both content summaries - var fileHash = _contentSummary.Files.First(x => x.Path == fileName).Hash; - var previousFileHash = _previousContentSummary.Files.First(x => x.Path == fileName).Hash; - - return fileHash != previousFileHash; - } - // TODO: Temporary solution for situation when .app directory is not deleted private void DeleteEmptyMacAppDirectories(CancellationToken cancellationToken) { From 0d9e2ab5cda2a3420ff259aaca308c926f4f5947 Mon Sep 17 00:00:00 2001 From: Jakub Szczyrk Date: Thu, 23 Sep 2021 13:08:02 +0200 Subject: [PATCH 8/9] Removed processing of contnet summary files during diff. Added retries to install files during diff. Added caching of broken files during diff and a subsequent attempt to repair these files. --- .../AppData/FileSystem/RetryStrategy.cs | 4 +- .../Scripts/AppUpdater/AppRepairer.cs | 43 +++++++-- .../AppUpdater/AppUpdaterDiffStrategy.cs | 20 +++- .../AppUpdater/AppUpdaterRepairStrategy.cs | 31 ++++-- .../Commands/AppUpdaterCommandFactory.cs | 2 +- .../Commands/CheckVersionIntegrityCommand.cs | 12 +++ .../Commands/IInstallDiffCommand.cs | 5 +- .../AppUpdater/Commands/InstallDiffCommand.cs | 95 ++++++++++++++----- .../AppUpdater/Commands/RepairFilesCommand.cs | 11 ++- Assets/PatchKit Patcher/Scripts/Version.cs | 2 +- CHANGELOG.md | 21 ++++ 11 files changed, 199 insertions(+), 47 deletions(-) diff --git a/Assets/PatchKit Patcher/Scripts/AppData/FileSystem/RetryStrategy.cs b/Assets/PatchKit Patcher/Scripts/AppData/FileSystem/RetryStrategy.cs index 9d40bb33..203baff9 100644 --- a/Assets/PatchKit Patcher/Scripts/AppData/FileSystem/RetryStrategy.cs +++ b/Assets/PatchKit Patcher/Scripts/AppData/FileSystem/RetryStrategy.cs @@ -12,8 +12,8 @@ public class RetryStrategy : IRequestRetryStrategy { private static readonly DebugLogger DebugLogger = new DebugLogger(typeof(RetryStrategy)); - public const int DefaultTryCount = 10; - public const int DefaultDelayMsec = 500; + public const int DefaultTryCount = 5; + public const int DefaultDelayMsec = 10; private readonly int _tryCount; private int _currentTry = 0; diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/AppRepairer.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/AppRepairer.cs index e9d3100f..a267364e 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/AppRepairer.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/AppRepairer.cs @@ -32,6 +32,9 @@ public class AppRepairer private const double IncreaseRepairCost = 1.5d; + private string[] _brokenFiles; + + public AppRepairer(AppUpdaterContext context, UpdaterStatus status) { DebugLogger.LogConstructor(); @@ -44,6 +47,20 @@ public AppRepairer(AppUpdaterContext context, UpdaterStatus status) _strategyResolver = new AppUpdaterStrategyResolver(_status); _commandFactory = new AppUpdaterCommandFactory(); } + + public AppRepairer(AppUpdaterContext context, UpdaterStatus status, string[] brokenFiles) + { + DebugLogger.LogConstructor(); + + Checks.ArgumentNotNull(context, "context"); + + Context = context; + _status = status; + _brokenFiles = brokenFiles; + + _strategyResolver = new AppUpdaterStrategyResolver(_status); + _commandFactory = new AppUpdaterCommandFactory(); + } // returns true if data is valid (was valid from the start or successfull repair was performed) public bool Perform(PatchKit.Unity.Patcher.Cancellation.CancellationToken cancellationToken) @@ -62,8 +79,8 @@ public bool Perform(PatchKit.Unity.Patcher.Cancellation.CancellationToken cancel // retry count reached, let's check for the last time if data is ok, but without repairing int installedVersionId = Context.App.GetInstalledVersionId(); - VersionIntegrity = CheckIntegrity(cancellationToken, installedVersionId); - var filesNeedFixing = FilesNeedFixing(VersionIntegrity); + VersionIntegrity results = CheckIntegrity(cancellationToken, installedVersionId); + var filesNeedFixing = FilesNeedFixing(results); if (filesNeedFixing.Count() == 0) { @@ -81,15 +98,23 @@ private bool PerformInternal(PatchKit.Unity.Patcher.Cancellation.CancellationTok { int installedVersionId = Context.App.GetInstalledVersionId(); - VersionIntegrity = CheckIntegrity(cancellationToken, installedVersionId); - var filesNeedFixing = FilesNeedFixing(VersionIntegrity); - - if (filesNeedFixing.Count() == 0) + if (_brokenFiles == null) { - DebugLogger.Log("No missing or invalid size files."); - return true; + VersionIntegrity = CheckIntegrity(cancellationToken, installedVersionId); } + else + { + VersionIntegrity = new CheckVersionIntegrityCommand(_brokenFiles).Results; + } + + var filesNeedFixing = FilesNeedFixing(VersionIntegrity); + if (filesNeedFixing.Count() == 0) + { + DebugLogger.Log("No missing or invalid size files."); + return true; + } + // need to collect some data about the application to calculate the repair cost and make decisions int latestVersionId = Context.App.GetLatestVersionId(true, cancellationToken); @@ -126,7 +151,7 @@ AppContentSummary latestVersionContentSummary else if (repairCost < contentSize) { DebugLogger.Log(string.Format("Repair cost {0} is smaller than content cost {1}, repairing...", repairCost, contentSize)); - IAppUpdaterStrategy repairStrategy = _strategyResolver.Create(StrategyType.Repair, Context); + IAppUpdaterStrategy repairStrategy = new AppUpdaterRepairStrategy(Context, _status, VersionIntegrity); repairStrategy.Update(cancellationToken); } else diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterDiffStrategy.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterDiffStrategy.cs index 63d47348..ec94b4e2 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterDiffStrategy.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterDiffStrategy.cs @@ -1,8 +1,11 @@ using System; using System.Collections.Generic; +using System.Linq; +using PatchKit.Unity.Patcher.AppData.Local; using PatchKit.Unity.Patcher.AppUpdater.Commands; using PatchKit.Unity.Patcher.AppUpdater.Status; using PatchKit.Unity.Patcher.AppData.Remote; +using PatchKit.Unity.Patcher.AppData.Remote.Downloaders; using PatchKit.Unity.Patcher.Cancellation; using PatchKit.Unity.Patcher.Debug; @@ -114,7 +117,6 @@ public void Update(CancellationToken cancellationToken) try { - PatcherStatistics.DispatchSendEvent(PatcherStatistics.Event.PatchDownloadStarted, optionalParams); diffCommands.Download.Command.Execute(cancellationToken); PatcherStatistics.DispatchSendEvent(PatcherStatistics.Event.PatchDownloadSucceeded, optionalParams); @@ -129,8 +131,22 @@ public void Update(CancellationToken cancellationToken) PatcherStatistics.DispatchSendEvent(PatcherStatistics.Event.PatchDownloadFailed, optionalParams); throw; } - + diffCommands.Install.Execute(cancellationToken); + + var brokenFiles = diffCommands.Install.GetBrokenFiles(); + if (brokenFiles.Length == 0) + { + DebugLogger.Log("Nothing to repair."); + break; + } + DebugLogger.Log(string.Format("Broken files count: {0}", brokenFiles.Length)); + + AppRepairer appRepairer = new AppRepairer(_context, _status, brokenFiles); + if (!appRepairer.Perform(cancellationToken)) + { + throw new CannotRepairDiskFilesException("Failed to validate/repair disk files"); + } } _context.App.DownloadDirectory.Clear(); diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterRepairStrategy.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterRepairStrategy.cs index 915d2466..0404ac85 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterRepairStrategy.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterRepairStrategy.cs @@ -21,6 +21,7 @@ public class AppUpdaterRepairStrategy: IAppUpdaterStrategy private readonly UpdaterStatus _status; private readonly ILogger _logger; + private VersionIntegrity _versionIntegrity; // not used public bool RepairOnError { get; set; } @@ -34,6 +35,17 @@ public AppUpdaterRepairStrategy(AppUpdaterContext context, UpdaterStatus status) _logger = PatcherLogManager.DefaultLogger; } + + public AppUpdaterRepairStrategy(AppUpdaterContext context, UpdaterStatus status, VersionIntegrity versionIntegrity) + { + Assert.IsNotNull(context, "Context is null"); + + _context = context; + _status = status; + _versionIntegrity = versionIntegrity; + + _logger = PatcherLogManager.DefaultLogger; + } public StrategyType GetStrategyType() { @@ -72,15 +84,20 @@ public void Update(CancellationToken cancellationToken) var downloader = new HttpDownloader(metaDestination, resource.GetMetaUrls()); downloader.Download(cancellationToken); - ICheckVersionIntegrityCommand checkVersionIntegrityCommand - = commandFactory.CreateCheckVersionIntegrityCommand(installedVersionId, _context, true, true, cancellationToken); + if (_versionIntegrity == null) + { + ICheckVersionIntegrityCommand checkVersionIntegrityCommand + = commandFactory.CreateCheckVersionIntegrityCommand(installedVersionId, _context, true, true, + cancellationToken); - checkVersionIntegrityCommand.Prepare(_status, cancellationToken); - checkVersionIntegrityCommand.Execute(cancellationToken); + checkVersionIntegrityCommand.Prepare(_status, cancellationToken); + checkVersionIntegrityCommand.Execute(cancellationToken); + _versionIntegrity = checkVersionIntegrityCommand.Results; + } var meta = Pack1Meta.ParseFromFile(metaDestination); - FileIntegrity[] filesIntegrity = checkVersionIntegrityCommand.Results.Files; + FileIntegrity[] filesIntegrity = _versionIntegrity.Files; var contentSummary = _context.App.RemoteMetaData.GetContentSummary(installedVersionId, cancellationToken); @@ -110,9 +127,7 @@ ICheckVersionIntegrityCommand checkVersionIntegrityCommand _context.App.LocalMetaData.SaveData(); Pack1Meta.FileEntry[] brokenFiles = filesIntegrity // Filter only files with invalid size, hash or missing entirely - .Where(f => f.Status == FileIntegrityStatus.InvalidHash - || f.Status == FileIntegrityStatus.InvalidSize - || f.Status == FileIntegrityStatus.MissingData) + .Where(f => f.Status != FileIntegrityStatus.Ok) // Map to file entires from meta .Select(integrity => meta.Files.SingleOrDefault(file => file.Name == integrity.FileName)) // Filter only regular files diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/AppUpdaterCommandFactory.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/AppUpdaterCommandFactory.cs index 96d41007..28d30346 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/AppUpdaterCommandFactory.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/AppUpdaterCommandFactory.cs @@ -44,7 +44,7 @@ public IRepairFilesCommand CreateRepairFilesCommand(int versionId, AppUpdaterCon var packagePath = context.App.DownloadDirectory.GetContentPackagePath(versionId); var packagePassword = context.App.RemoteData.GetContentPackageResourcePassword(versionId); - return new RepairFilesCommand(resource, meta, brokenFiles, packagePath, packagePassword, context.App.LocalDirectory); + return new RepairFilesCommand(resource, meta, brokenFiles, packagePath, packagePassword, context.App.LocalDirectory, context.App.LocalMetaData, versionId); } public IInstallContentCommand CreateInstallContentCommand(int versionId, AppUpdaterContext context, CancellationToken cancellationToken) diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/CheckVersionIntegrityCommand.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/CheckVersionIntegrityCommand.cs index b763b37e..3e525945 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/CheckVersionIntegrityCommand.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/CheckVersionIntegrityCommand.cs @@ -42,6 +42,12 @@ public CheckVersionIntegrityCommand(int versionId, AppContentSummary versionSumm _isCheckingSize = isCheckingSize; _isCheckingHash = isCheckingHash; } + + public CheckVersionIntegrityCommand(string[] brokenFiles) + { + Results = new VersionIntegrity(brokenFiles.Select(file => + new FileIntegrity(file, FileIntegrityStatus.MissingData)).ToArray()); + } public override void Prepare(UpdaterStatus status, CancellationToken cancellationToken) { @@ -135,12 +141,14 @@ private FileIntegrity CheckFile(AppContentSummaryFile file) string localPath = _localDirectory.Path.PathCombine(file.Path); if (!File.Exists(localPath)) { + DebugLogger.LogWarning("File " + localPath + " not exist"); onVerificationFailed(); return new FileIntegrity(file.Path, FileIntegrityStatus.MissingData); } if (!_localMetaData.IsEntryRegistered(file.Path)) { + DebugLogger.LogWarning("File " + file.Path + " is not entry registered"); onVerificationFailed(); return new FileIntegrity(file.Path, FileIntegrityStatus.MissingMetaData); } @@ -148,6 +156,7 @@ private FileIntegrity CheckFile(AppContentSummaryFile file) int actualVersionId = _localMetaData.GetEntryVersionId(file.Path); if (actualVersionId != _versionId) { + DebugLogger.LogWarning("File " + file.Path + " version is " + actualVersionId + ", expected " + _versionId); onVerificationFailed(); return FileIntegrity.InvalidVersion(_versionId, actualVersionId, file.Path); } @@ -157,6 +166,7 @@ private FileIntegrity CheckFile(AppContentSummaryFile file) long actualSize = new FileInfo(localPath).Length; if (actualSize != file.Size) { + DebugLogger.LogWarning("File " + localPath + " size is " + actualSize + ", expected " + file.Size); onVerificationFailed(); return FileIntegrity.InvalidSize(file.Size, actualSize, file.Path); } @@ -167,6 +177,8 @@ private FileIntegrity CheckFile(AppContentSummaryFile file) string actualFileHash = HashCalculator.ComputeFileHash(localPath); if (actualFileHash != file.Hash) { + DebugLogger.LogWarning("File " + localPath + " hash is " + actualFileHash + ", expected " + file.Hash); + onVerificationFailed(); return FileIntegrity.InvalidHash(file.Hash, actualFileHash, file.Path); } diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/IInstallDiffCommand.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/IInstallDiffCommand.cs index 7dee5577..1bc1878d 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/IInstallDiffCommand.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/IInstallDiffCommand.cs @@ -1,6 +1,9 @@ -namespace PatchKit.Unity.Patcher.AppUpdater.Commands +using PatchKit.Unity.Patcher.AppData.Local; + +namespace PatchKit.Unity.Patcher.AppUpdater.Commands { public interface IInstallDiffCommand : IAppUpdaterCommand { + string[] GetBrokenFiles(); } } \ No newline at end of file diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/InstallDiffCommand.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/InstallDiffCommand.cs index 0dc988c9..fd7146c7 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/InstallDiffCommand.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/InstallDiffCommand.cs @@ -20,6 +20,10 @@ namespace PatchKit.Unity.Patcher.AppUpdater.Commands public class InstallDiffCommand : BaseAppUpdaterCommand, IInstallDiffCommand { private const string Suffix = "_"; // FIX: Bug #714 + private const long EntriesSize = 1048576; //1MiB + private const int Whenever = 100; + private const int Ratio = 10; + private int _checkNumber = 100; private readonly ILogger _logger; private readonly string _packagePath; @@ -40,6 +44,7 @@ public class InstallDiffCommand : BaseAppUpdaterCommand, IInstallDiffCommand private AppDiffSummary _diffSummary; private Pack1Meta _pack1Meta; private MapHashExtractedFiles _mapHashExtractedFiles; + private List _brokenFiles = new List(); public InstallDiffCommand([NotNull] string packagePath, string packageMetaPath, string packagePassword, int versionId, @@ -343,7 +348,8 @@ private void ProcessRemovedFiles(CancellationToken cancellationToken) counter++; _removeFilesStatusReporter.Progress.Value = counter / (double) _diffSummary.RemovedFiles.Length; - _removeFilesStatusReporter.Description.Value = string.Format("Removing old files ({0}/{1})...", counter, _diffSummary.RemovedFiles.Length); + _removeFilesStatusReporter.Description.Value = string.Format("Removing old files ({0}/{1})...", counter, + _diffSummary.RemovedFiles.Length); } foreach (var dirName in dirNames) @@ -354,7 +360,8 @@ private void ProcessRemovedFiles(CancellationToken cancellationToken) counter++; _removeFilesStatusReporter.Progress.Value = counter / (double) _diffSummary.RemovedFiles.Length; - _removeFilesStatusReporter.Description.Value = string.Format("Removing old files ({0}/{1})...", counter, _diffSummary.RemovedFiles.Length); + _removeFilesStatusReporter.Description.Value = string.Format("Removing old files ({0}/{1})...", counter, + _diffSummary.RemovedFiles.Length); } _removeFilesStatusReporter.Progress.Value = 1.0; @@ -435,21 +442,41 @@ private void ProcessAddedFiles(string packageDirPath, string suffix, for (int i = 0; i < _diffSummary.AddedFiles.Length; i++) { - cancellationToken.ThrowIfCancellationRequested(); + try + { + cancellationToken.ThrowIfCancellationRequested(); - var entryName = _diffSummary.AddedFiles[i]; + var entryName = _diffSummary.AddedFiles[i]; - if (entryName.EndsWith("/")) - { - AddDirectory(entryName, cancellationToken); + if (entryName.EndsWith("/")) + { + AddDirectory(entryName, cancellationToken); + } + else + { + RetryStrategy.TryExecute(() => AddFile(entryName, packageDirPath, suffix, cancellationToken), + cancellationToken); + } + + _addFilesStatusReporter.Progress.Value = (i + 1) / (double) _diffSummary.AddedFiles.Length; + _addFilesStatusReporter.Description.Value = string.Format("Adding new files ({0}/{1})...", i + 1, + _diffSummary.AddedFiles.Length); } - else + catch (Exception e) { - AddFile(entryName, packageDirPath, suffix, cancellationToken, i); + string brokenFile = _diffSummary.AddedFiles[i]; + _logger.LogWarning("Broken file during modifying: " + brokenFile + " Exception: " + e); + _brokenFiles.Add(brokenFile); + int brokenFilesCount = _brokenFiles.Count; + if (brokenFilesCount > _checkNumber) + { + _checkNumber += Whenever; + if (brokenFilesCount > (i - brokenFilesCount) * Ratio) + { + throw; + } + } } - - _addFilesStatusReporter.Progress.Value = (i + 1) / (double) _diffSummary.AddedFiles.Length; - _addFilesStatusReporter.Description.Value = string.Format("Adding new files ({0}/{1})...", i + 1, _diffSummary.AddedFiles.Length); } _localMetaData.SaveData(); @@ -474,8 +501,7 @@ private void AddDirectory(string dirName, CancellationToken cancellationToken) _logger.LogDebug("Add directory entry processed."); } - private void AddFile(string fileName, string packageDirPath, string suffix, CancellationToken cancellationToken, - int fileIndex) + private void AddFile(string fileName, string packageDirPath, string suffix, CancellationToken cancellationToken) { _logger.LogDebug(string.Format("Processing add file entry {0}", fileName)); @@ -536,18 +562,38 @@ private void ProcessModifiedFiles(string packageDirPath, string suffix, Temporar .ToArray(); for (int i = 0; i < entryNames.Length; i++) { - cancellationToken.ThrowIfCancellationRequested(); + try + { + cancellationToken.ThrowIfCancellationRequested(); - var entryName = entryNames[i]; + var entryName = entryNames[i]; - if (!entryName.EndsWith("/")) + if (!entryName.EndsWith("/")) + { + RetryStrategy.TryExecute( + () => PatchFile(entryName, packageDirPath, suffix, tempDiffDir, cancellationToken, i), + cancellationToken); + } + + _modifiedFilesStatusReporter.Progress.Value = (i + 1) / (double) entryName.Length; + _modifiedFilesStatusReporter.Description.Value = + string.Format("Applying diffs ({0}/{1})...", i + 1, entryName.Length); + } + catch (Exception e) { - PatchFile(entryName, packageDirPath, suffix, tempDiffDir, cancellationToken, i); + string brokenFile = entryNames[i]; + _logger.LogWarning("Broken file during modifying: " + brokenFile + " Exception: " + e); + _brokenFiles.Add(brokenFile); + int brokenFilesCount = _brokenFiles.Count; + if (brokenFilesCount > _checkNumber) + { + _checkNumber += Whenever; + if (brokenFilesCount > (i - brokenFilesCount) * Ratio) + { + throw; + } + } } - - _modifiedFilesStatusReporter.Progress.Value = (i + 1) / (double) entryNames.Length; - _modifiedFilesStatusReporter.Description.Value = - string.Format("Applying diffs ({0}/{1})...", i + 1, entryNames.Length); } _localMetaData.SaveData(); @@ -662,5 +708,10 @@ private static bool IsEmptyMacAppDirectory(string dirPath) dirPath.EndsWith(".app") && Directory.GetFiles(dirPath, "*", SearchOption.AllDirectories).Length == 0; } + + public string[] GetBrokenFiles() + { + return _brokenFiles.ToArray(); + } } } \ No newline at end of file diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/RepairFilesCommand.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/RepairFilesCommand.cs index e2471904..2bea6bea 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/RepairFilesCommand.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/RepairFilesCommand.cs @@ -31,6 +31,8 @@ private struct EntryStatus private Pack1Meta.FileEntry[] _entries; private ILocalDirectory _localData; + private readonly ILocalMetaData _localMetaData; + private readonly int _versionId; private string _packagePath; private string _packagePassword; @@ -48,7 +50,9 @@ public RepairFilesCommand( Pack1Meta.FileEntry[] fileEntries, string destinationPackagePath, string packagePassword, - ILocalDirectory localData) + ILocalDirectory localData, + ILocalMetaData localMetaData, + int versionId) { _resource = resource; _meta = meta; @@ -58,6 +62,8 @@ public RepairFilesCommand( _packagePassword = packagePassword; _localData = localData; + _localMetaData = localMetaData; + _versionId = versionId; _logger = PatcherLogManager.DefaultLogger; } @@ -131,6 +137,7 @@ public override void Execute(CancellationToken cancellationToken) { EmplaceFile(Path.Combine(unarchivePath, nameHash + _unpackingSuffix), Path.Combine(_localData.Path, entry.Name), cancellationToken); + _localMetaData.RegisterEntry(entry.Name, _versionId, entry.Size.Value); } else { @@ -140,6 +147,8 @@ public override void Execute(CancellationToken cancellationToken) repairStatus.IsActive.Value = false; }); } + + _localMetaData.SaveData(); } public override void Prepare(UpdaterStatus status, CancellationToken cancellationToken) diff --git a/Assets/PatchKit Patcher/Scripts/Version.cs b/Assets/PatchKit Patcher/Scripts/Version.cs index 04e98fc5..ff72ed0f 100644 --- a/Assets/PatchKit Patcher/Scripts/Version.cs +++ b/Assets/PatchKit Patcher/Scripts/Version.cs @@ -4,7 +4,7 @@ public static class Version { public const int Major = 3; public const int Minor = 17; - public const int Patch = 9; + public const int Patch = 10; public const int Hotfix = 0; public static string Value diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d4ca879..46565673 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,27 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [3.17.10.0] +### Added +- Caching for diff and content app summary +- Restriction of hash repair for a large number of files +- Reduced number of events sent for version integration check +- Retries to install files during diff +- Caching of broken files during diff and a subsequent attempt to repair these files. + +### Changed +- Repair cost and decision + +### Fixed +- The retry button resumes the update + +### Removed +- Check version for diff +- Processing of contnet summary files during diff + +### Fixed +- The retry button resumes the update + ## [3.17.9.0] ### Added - Support HDPI for debug menu From b6fe458df78003c9a49554ad47c6ef6539f7aa67 Mon Sep 17 00:00:00 2001 From: Jakub Szczyrk Date: Mon, 27 Sep 2021 15:00:25 +0200 Subject: [PATCH 9/9] Added environment variables for debugging a broken diff install --- .../AppUpdater/Commands/InstallDiffCommand.cs | 31 ++++++++++++++++++- .../Scripts/Debug/EnvironmentVariables.cs | 6 ++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/InstallDiffCommand.cs b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/InstallDiffCommand.cs index fd7146c7..fd618d16 100644 --- a/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/InstallDiffCommand.cs +++ b/Assets/PatchKit Patcher/Scripts/AppUpdater/Commands/InstallDiffCommand.cs @@ -45,6 +45,8 @@ public class InstallDiffCommand : BaseAppUpdaterCommand, IInstallDiffCommand private Pack1Meta _pack1Meta; private MapHashExtractedFiles _mapHashExtractedFiles; private List _brokenFiles = new List(); + private int _numberOfFilesToCrashWhenModifyingDiffEnvironmentVariable; + private int _numberOfFilesToCrashWhenAddingDiffEnvironmentVariable; public InstallDiffCommand([NotNull] string packagePath, string packageMetaPath, string packagePassword, int versionId, @@ -139,6 +141,25 @@ public override void Prepare([NotNull] UpdaterStatus status, CancellationToken c status.RegisterOperation(_removeFilesStatusReporter); _logger.LogDebug("Diff installation prepared."); + + string tmp; + if (EnvironmentInfo.TryReadEnvironmentVariable( + EnvironmentVariables.NumberOfFilesToCrashWhenModifyingDiffEnvironmentVariable, + out tmp)) + { + if (int.TryParse(tmp, out _numberOfFilesToCrashWhenModifyingDiffEnvironmentVariable)) + { + _logger.LogDebug(string.Format("Number of files to crash when modifying diff {0}", _numberOfFilesToCrashWhenModifyingDiffEnvironmentVariable)); + } + } + + if (EnvironmentInfo.TryReadEnvironmentVariable(EnvironmentVariables.NumberOfFilesToCrashWhenAddingDiffEnvironmentVariable, out tmp)) + { + if (int.TryParse(tmp, out _numberOfFilesToCrashWhenAddingDiffEnvironmentVariable)) + { + _logger.LogDebug(string.Format("Number of files to crash when adding diff {0}", _numberOfFilesToCrashWhenAddingDiffEnvironmentVariable)); + } + } } catch (Exception e) { @@ -448,6 +469,10 @@ private void ProcessAddedFiles(string packageDirPath, string suffix, var entryName = _diffSummary.AddedFiles[i]; + if (i < _numberOfFilesToCrashWhenAddingDiffEnvironmentVariable) + throw new MissingFileFromPackageException(string.Format("Cannot find file {0} in diff package. Controlled crash.", + entryName)); + if (entryName.EndsWith("/")) { AddDirectory(entryName, cancellationToken); @@ -465,7 +490,7 @@ private void ProcessAddedFiles(string packageDirPath, string suffix, catch (Exception e) { string brokenFile = _diffSummary.AddedFiles[i]; - _logger.LogWarning("Broken file during modifying: " + brokenFile + " Exception: " + e); + _logger.LogWarning("Broken file during adding: " + brokenFile + " Exception: " + e); _brokenFiles.Add(brokenFile); int brokenFilesCount = _brokenFiles.Count; if (brokenFilesCount > _checkNumber) @@ -568,6 +593,10 @@ private void ProcessModifiedFiles(string packageDirPath, string suffix, Temporar var entryName = entryNames[i]; + if (i < _numberOfFilesToCrashWhenModifyingDiffEnvironmentVariable) + throw new MissingFileFromPackageException(string.Format("Cannot find file {0} in diff package. Controlled crash.", + entryName)); + if (!entryName.EndsWith("/")) { RetryStrategy.TryExecute( diff --git a/Assets/PatchKit Patcher/Scripts/Debug/EnvironmentVariables.cs b/Assets/PatchKit Patcher/Scripts/Debug/EnvironmentVariables.cs index b87a7be0..54c0027a 100644 --- a/Assets/PatchKit Patcher/Scripts/Debug/EnvironmentVariables.cs +++ b/Assets/PatchKit Patcher/Scripts/Debug/EnvironmentVariables.cs @@ -9,6 +9,12 @@ public static class EnvironmentVariables public const string KeysUrlEnvironmentVariable = "PK_PATCHER_KEYS_URL"; public const string KeepFilesOnErrorEnvironmentVariable = "PK_PATCHER_KEEP_FILES_ON_ERROR"; + public const string NumberOfFilesToCrashWhenAddingDiffEnvironmentVariable = + "PK_PATCHER_NUMBER_OF_FILES_TO_CRASH_WHEN_ADDING_DIFF"; + + public const string NumberOfFilesToCrashWhenModifyingDiffEnvironmentVariable = + "PK_PATCHER_NUMBER_OF_FILES_TO_CRASH_WHEN_MODIFYING_DIFF"; + // will corrupt every 10th file public const string CorruptFilesOnUnpack10 = "PK_PATCHER_CORRUPT_UNPACK_10";