Improved diff installation

Assets/Editor/Tests/LocalMetaDataTest.cs

@@ -31,7 +31,7 @@ public void CreateMetaDataFile()
 
         var localMetaData = new LocalMetaData(_filePath, _deprecatedFilePath);
 
-        localMetaData.RegisterEntry("test", 1, 0, true);
+        localMetaData.RegisterEntry("test", 1, 0);
 
         Assert.True(File.Exists(_filePath));
     }
@@ -41,8 +41,8 @@ public void SaveValidFileSinglePass()
     {
         var localMetaData = new LocalMetaData(_filePath, _deprecatedFilePath);
 
-        localMetaData.RegisterEntry("a", 1, 0 , true);
-        localMetaData.RegisterEntry("b", 2, 0 , true);
+        localMetaData.RegisterEntry("a", 1, 0);
+        localMetaData.RegisterEntry("b", 2, 0);
 
         var localMetaData2 = new LocalMetaData(_filePath, _deprecatedFilePath);
 

Assets/PatchKit Patcher/Scripts/AppData/FileSystem/RetryStrategy.cs

@@ -12,8 +12,8 @@ public class RetryStrategy : IRequestRetryStrategy
     {
         private static readonly DebugLogger DebugLogger = new DebugLogger(typeof(RetryStrategy));
 
-        public const int DefaultTryCount = 10;
-        public const int DefaultDelayMsec = 500;
+        public const int DefaultTryCount = 5;
+        public const int DefaultDelayMsec = 10;
 
         private readonly int _tryCount;
         private int _currentTry = 0;

Assets/PatchKit Patcher/Scripts/AppData/Local/ILocalMetaData.cs

@@ -17,8 +17,7 @@ public interface ILocalMetaData
         /// <param name="entryName">Name of the entry.</param>
         /// <param name="versionId">The version id.</param>
         /// <param name="entrySize">Size of entry.</param>
-        /// <param name="isLastEntry">If set to true, it is last entry.</param>
-        void RegisterEntry(string entryName, int versionId, long entrySize, bool isLastEntry);
+        void RegisterEntry(string entryName, int versionId, long? entrySize);
 
         /// <summary>
         /// Unregisters the entry.
@@ -52,5 +51,7 @@ public interface ILocalMetaData
         string GetMainExecutable();
 
         string MainExecutableArgs { get; }
+
+        void SaveData();
     }
 }

Assets/PatchKit Patcher/Scripts/AppData/Local/LocalMetaData.cs

@@ -21,8 +21,10 @@ public class LocalMetaData : ILocalMetaData
     {
         private readonly ILogger _logger;
         private long _unsavedEntriesSize = 0;
+        private int _unsavedEntriesCount = 0;
         private const string DeprecatedCachePatchKitKey = "patchkit-key";
         private const long UnsavedEntriesSizeLimit = 104857600; //100MiB
+        private const long UnsavedEntriesCountLimit = 100;
 
         /// <summary>
         /// Data structure stored in file.
@@ -81,7 +83,7 @@ public string[] GetRegisteredEntries()
             return _data.FileVersionIds.Select(pair => pair.Key).ToArray();
         }
 
-        public void RegisterEntry([NotNull] string entryName, int versionId, long entrySize, bool isLastEntry)
+        public void RegisterEntry([NotNull] string entryName, int versionId, long? entrySize)
         {
             if (entryName == null)
             {
@@ -102,10 +104,9 @@ public void RegisterEntry([NotNull] string entryName, int versionId, long entryS
             try
             {
                 _logger.LogDebug(string.Format("Registering entry {0} as version {1}.", entryName, versionId));
-
                 _data.FileVersionIds[entryName] = versionId;
-
-                if (ShouldSaveEntry(entrySize, isLastEntry))
+     
+                if (ShouldSaveEntry(entrySize))
                 {
                     SaveData();
                 }
@@ -119,19 +120,22 @@ public void RegisterEntry([NotNull] string entryName, int versionId, long entryS
             }
         }
 
-        private bool ShouldSaveEntry(long entrySize, bool isLastEntry)
+        private bool ShouldSaveEntry(long? entrySize)
         {
-            if (isLastEntry)
+            if (entrySize.HasValue)
             {
-                return true;
+                _unsavedEntriesSize += entrySize.Value;
+                if (_unsavedEntriesSize > UnsavedEntriesSizeLimit)
+                {
+                    return true;
+                }
             }
 
-            _unsavedEntriesSize += entrySize;
-            if (_unsavedEntriesSize > UnsavedEntriesSizeLimit)
+            if (++_unsavedEntriesCount > UnsavedEntriesCountLimit)
             {
-                _unsavedEntriesSize = 0;
                 return true;
             }
+
             return false;
         }
 
@@ -201,7 +205,7 @@ public string GetProductKey()
         {
             return _data.ProductKey;
         }
-        
+
         public void SetMainExecutableAndArgs(string mainExecutable, string mainExecutableArgs)
         {
             if (_data.MainExecutable != mainExecutable ||
@@ -233,10 +237,11 @@ private void CreateDataDir()
             }
         }
 
-        private void SaveData()
+        public void SaveData()
         {
             _logger.LogDebug(string.Format("Saving data to {0}", _filePath));
-
+            _unsavedEntriesSize = 0;
+            _unsavedEntriesCount = 0;
             CreateDataDir();
             Files.WriteAllText(_filePath, JsonConvert.SerializeObject(_data, Formatting.None));
 

Assets/PatchKit Patcher/Scripts/AppData/Remote/RemoteMetaData.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using JetBrains.Annotations;
 using PatchKit.Api;
 using PatchKit.Api.Models.Main;
@@ -17,6 +18,9 @@ public class RemoteMetaData : IRemoteMetaData
         private readonly MainApiConnection _mainApiConnection;
         private readonly KeysApiConnection _keysApiConnection;
 
+        public Dictionary<int, AppContentSummary> _cacheAppContentSummary = new Dictionary<int, AppContentSummary>();
+        public Dictionary<int, AppDiffSummary> _cacheAppDiffSummary = new Dictionary<int, AppDiffSummary>();
+
         public RemoteMetaData([NotNull] string appSecret, [NotNull] IRequestTimeoutCalculator requestTimeoutCalculator)
         {
             if (string.IsNullOrEmpty(appSecret))
@@ -77,16 +81,33 @@ public AppContentSummary GetContentSummary(int versionId, CancellationToken canc
         {
             Checks.ArgumentValidVersionId(versionId, "versionId");
             DebugLogger.Log(string.Format("Getting content summary of version with id {0}.", versionId));
+            AppContentSummary appContentSummary;
+            if (_cacheAppContentSummary.TryGetValue(versionId, out appContentSummary))
+            {
+                return appContentSummary;
+            }
 
-            return _mainApiConnection.GetAppVersionContentSummary(_appSecret, versionId, cancellationToken);
+            appContentSummary =
+                _mainApiConnection.GetAppVersionContentSummary(_appSecret, versionId, cancellationToken);
+            _cacheAppContentSummary.Add(versionId, appContentSummary);
+            return appContentSummary;
         }
 
         public AppDiffSummary GetDiffSummary(int versionId, CancellationToken cancellationToken)
         {
             Checks.ArgumentValidVersionId(versionId, "versionId");
             DebugLogger.Log(string.Format("Getting diff summary of version with id {0}.", versionId));
 
-            return _mainApiConnection.GetAppVersionDiffSummary(_appSecret, versionId, cancellationToken);
+            AppDiffSummary appDiffSummary;
+            if (_cacheAppDiffSummary.TryGetValue(versionId, out appDiffSummary))
+            {
+                return appDiffSummary;
+            }
+
+            appDiffSummary =
+                _mainApiConnection.GetAppVersionDiffSummary(_appSecret, versionId, cancellationToken);
+            _cacheAppDiffSummary.Add(versionId, appDiffSummary);
+            return appDiffSummary;
         }
 
         public string GetKeySecret(string key, string cachedKeySecret, CancellationToken cancellationToken)

Assets/PatchKit Patcher/Scripts/AppUpdater/AppRepairer.cs

@@ -1,10 +1,6 @@
 using System;
-using System.IO;
 using System.Linq;
-using System.Linq.Expressions;
-using System.Threading;
 using System.Collections.Generic;
-using PatchKit.Unity.Patcher.Cancellation;
 using PatchKit.Unity.Patcher.Debug;
 using PatchKit.Unity.Patcher.AppUpdater.Commands;
 using PatchKit.Unity.Patcher.AppUpdater.Status;
@@ -18,6 +14,8 @@ public class AppRepairer
 
         public readonly AppUpdaterContext Context;
 
+        public VersionIntegrity VersionIntegrity;
+
         // set to true if you wish to check file hashes
         public bool CheckHashes = false;
 
@@ -34,6 +32,8 @@ public class AppRepairer
 
         private const double IncreaseRepairCost = 1.5d;
 
+        private string[] _brokenFiles;
+
 
         public AppRepairer(AppUpdaterContext context, UpdaterStatus status)
         {
@@ -47,6 +47,20 @@ public AppRepairer(AppUpdaterContext context, UpdaterStatus status)
             _strategyResolver = new AppUpdaterStrategyResolver(_status);
             _commandFactory = new AppUpdaterCommandFactory();
         }
+
+        public AppRepairer(AppUpdaterContext context, UpdaterStatus status, string[] brokenFiles)
+        {
+            DebugLogger.LogConstructor();
+
+            Checks.ArgumentNotNull(context, "context");
+
+            Context = context;
+            _status = status;
+            _brokenFiles = brokenFiles;
+
+            _strategyResolver = new AppUpdaterStrategyResolver(_status);
+            _commandFactory = new AppUpdaterCommandFactory();
+        }
 
         // returns true if data is valid (was valid from the start or successfull repair was performed)
         public bool Perform(PatchKit.Unity.Patcher.Cancellation.CancellationToken cancellationToken)
@@ -84,15 +98,23 @@ private bool PerformInternal(PatchKit.Unity.Patcher.Cancellation.CancellationTok
         {
             int installedVersionId = Context.App.GetInstalledVersionId();
 
-            VersionIntegrity results = CheckIntegrity(cancellationToken, installedVersionId);
-            var filesNeedFixing = FilesNeedFixing(results);
-
-            if (filesNeedFixing.Count() == 0)
+            if (_brokenFiles == null)
             {
-                DebugLogger.Log("No missing or invalid size files.");
-                return true;
+                VersionIntegrity = CheckIntegrity(cancellationToken, installedVersionId);
             }
+            else
+            {
+                VersionIntegrity = new CheckVersionIntegrityCommand(_brokenFiles).Results;
+            }
+
+            var filesNeedFixing = FilesNeedFixing(VersionIntegrity);
 
+                if (filesNeedFixing.Count() == 0)
+                {
+                    DebugLogger.Log("No missing or invalid size files.");
+                    return true;
+                }
+
             // need to collect some data about the application to calculate the repair cost and make decisions
 
             int latestVersionId = Context.App.GetLatestVersionId(true, cancellationToken);
@@ -129,7 +151,7 @@ AppContentSummary latestVersionContentSummary
             else if (repairCost < contentSize)
             {
                 DebugLogger.Log(string.Format("Repair cost {0} is smaller than content cost {1}, repairing...", repairCost, contentSize));
-                IAppUpdaterStrategy repairStrategy = _strategyResolver.Create(StrategyType.Repair, Context);
+                IAppUpdaterStrategy repairStrategy = new AppUpdaterRepairStrategy(Context, _status, VersionIntegrity);
                 repairStrategy.Update(cancellationToken);
             }
             else
@@ -162,17 +184,16 @@ int installedVersionId
 
         private IEnumerable<FileIntegrity> FilesNeedFixing(VersionIntegrity results)
         {
-            var missingFiles = results.Files.Where(f => f.Status == FileIntegrityStatus.MissingData);
-            var invalidSizeFiles = results.Files.Where(f => f.Status == FileIntegrityStatus.InvalidSize);
+            var invalidFiles = results.Files.Where(f => f.Status != FileIntegrityStatus.Ok);
 
-            return missingFiles.Concat(invalidSizeFiles);
+            return invalidFiles;
         }
 
         private long CalculateRepairCost(AppContentSummary contentSummary, IEnumerable<FileIntegrity> filesToRepair)
         {
             return filesToRepair
                 .Select(f => contentSummary.Files.FirstOrDefault(e => e.Path == f.FileName))
-                .Sum(f => f.Size);
+                .Sum(f => Math.Max(contentSummary.Chunks.Size, f.Size));
         }
 
         private void ReinstallContent(PatchKit.Unity.Patcher.Cancellation.CancellationToken cancellationToken)

Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdater.cs

@@ -27,6 +27,7 @@ public class AppUpdater
         private bool _updateHasBeenCalled;
         private bool _uninstallHasBeenCalled;
         private bool _verifyFilesHasBeenCalled;
+        private VersionIntegrity _versionIntegrity;
 
         public IReadOnlyUpdaterStatus Status
         {
@@ -47,6 +48,7 @@ private void PreUpdate(PatchKit.Unity.Patcher.Cancellation.CancellationToken can
         {
             var appRepairer = new AppRepairer(Context, _status);
             appRepairer.Perform(cancellationToken);
+            _versionIntegrity = appRepairer.VersionIntegrity;
         }
 
         public void Update(PatchKit.Unity.Patcher.Cancellation.CancellationToken cancellationToken)
@@ -60,7 +62,7 @@ public void Update(PatchKit.Unity.Patcher.Cancellation.CancellationToken cancell
 
             DebugLogger.Log("Updating.");
 
-            StrategyType type = _strategyResolver.Resolve(Context, cancellationToken);
+            StrategyType type = _strategyResolver.Resolve(Context, _versionIntegrity, cancellationToken);
             _strategy = _strategyResolver.Create(type, Context);
 
             try

Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterContentStrategy.cs

@@ -15,6 +15,8 @@ public class AppUpdaterContentStrategy: IAppUpdaterStrategy
 
         private readonly UpdaterStatus _status;
 
+        private const int RepairCheckHashesMaxFilesCount = 10000;
+
         private bool _updateHasBeenCalled;
 
         public bool RepairOnError { get; set; }
@@ -119,7 +121,15 @@ public void Update(CancellationToken cancellationToken)
                 DebugLogger.Log("Content installed with errors, requesting repair");
 
                 var appRepairer = new AppRepairer(_context, _status);
-                appRepairer.CheckHashes = true;
+                if (_context.App.RemoteMetaData.GetContentSummary(latestVersionId, cancellationToken).Files.Length < RepairCheckHashesMaxFilesCount)
+                {
+                    appRepairer.CheckHashes = true;
+                }
+                else
+                {
+                    appRepairer.CheckHashes = false;
+                }
+
 
                 if (!appRepairer.Perform(cancellationToken))
                 {

Assets/PatchKit Patcher/Scripts/AppUpdater/AppUpdaterDiffStrategy.cs

@@ -1,8 +1,11 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
+using PatchKit.Unity.Patcher.AppData.Local;
 using PatchKit.Unity.Patcher.AppUpdater.Commands;
 using PatchKit.Unity.Patcher.AppUpdater.Status;
 using PatchKit.Unity.Patcher.AppData.Remote;
+using PatchKit.Unity.Patcher.AppData.Remote.Downloaders;
 using PatchKit.Unity.Patcher.Cancellation;
 using PatchKit.Unity.Patcher.Debug;
 
@@ -114,7 +117,6 @@ public void Update(CancellationToken cancellationToken)
 
                 try
                 {
-
                     PatcherStatistics.DispatchSendEvent(PatcherStatistics.Event.PatchDownloadStarted, optionalParams);
                     diffCommands.Download.Command.Execute(cancellationToken);
                     PatcherStatistics.DispatchSendEvent(PatcherStatistics.Event.PatchDownloadSucceeded, optionalParams);
@@ -129,8 +131,22 @@ public void Update(CancellationToken cancellationToken)
                     PatcherStatistics.DispatchSendEvent(PatcherStatistics.Event.PatchDownloadFailed, optionalParams);
                     throw;
                 }
-
+                
                 diffCommands.Install.Execute(cancellationToken);
+
+                var brokenFiles = diffCommands.Install.GetBrokenFiles();
+                if (brokenFiles.Length == 0)
+                {
+                    DebugLogger.Log("Nothing to repair.");
+                    break;
+                }
+                DebugLogger.Log(string.Format("Broken files count: {0}", brokenFiles.Length));
+
+                AppRepairer appRepairer = new AppRepairer(_context, _status, brokenFiles);
+                if (!appRepairer.Perform(cancellationToken))
+                {
+                    throw new CannotRepairDiskFilesException("Failed to validate/repair disk files");
+                }
             }
 
             _context.App.DownloadDirectory.Clear();