Merge pull request #245 from giginet/shared-restored-cache-to-other-storages

Add automatic cache sharing for restored frameworks

Author: ikesyo

Sources/ScipioKit/Producer/Cache/CacheSystem.swift

@@ -48,7 +48,7 @@ public struct SwiftPMCacheKey: CacheKey {
 }
 
 struct CacheSystem: Sendable {
-    static let defaultParalellNumber = 8
+    static let defaultParallelNumber = 8
     private let outputDirectory: URL
     private let fileSystem: any FileSystem
 
@@ -92,7 +92,7 @@ struct CacheSystem: Sendable {
     }
 
     private func cacheFrameworks(_ targets: Set<CacheTarget>, to storage: some CacheStorage) async {
-        let chunked = targets.chunks(ofCount: storage.parallelNumber ?? CacheSystem.defaultParalellNumber)
+        let chunked = targets.chunks(ofCount: storage.parallelNumber ?? CacheSystem.defaultParallelNumber)
 
         let storageName = storage.displayName
         for chunk in chunked {

Sources/ScipioKit/Producer/Cache/LocalDiskCacheStorage.swift

@@ -1,7 +1,7 @@
 import Foundation
 import ScipioStorage
 
-struct LocalDiskCacheStorage: CacheStorage {
+struct LocalDiskCacheStorage: CacheStorage, Equatable {
     private let fileSystem: any FileSystem
 
     var parallelNumber: Int? { nil }
@@ -19,6 +19,11 @@ struct LocalDiskCacheStorage: CacheStorage {
         self.fileSystem = fileSystem
     }
 
+    // MARK: - Equatable
+    static func == (lhs: LocalDiskCacheStorage, rhs: LocalDiskCacheStorage) -> Bool {
+        lhs.baseURL == rhs.baseURL
+    }
+
     private func buildBaseDirectoryPath() throws -> URL {
         let cacheDir: URL
         if let baseURL {

Sources/ScipioKit/Producer/FrameworkProducer.swift

@@ -100,12 +100,25 @@ struct FrameworkProducer {
                 // no-op
                 targetGraph.remove(valid)
             } else {
-                let restored = await restoreAllAvailableCachesIfNeeded(
+                let restoredSetsToSourceStorage = await restoreAllAvailableCachesIfNeeded(
                     availableTargets: targets.subtracting(valid),
                     to: storagesWithConsumer,
                     cacheSystem: cacheSystem
                 )
-                let skipTargets = valid.union(restored)
+
+                let allRestoredTargets = restoredSetsToSourceStorage.keys.reduce(into: Set<CacheSystem.CacheTarget>()) { result, targetSet in
+                    result.formUnion(targetSet)
+                }
+
+                if !allRestoredTargets.isEmpty {
+                    await shareRestoredCachesToProducers(
+                        allRestoredTargets,
+                        restoredSetsToSourceStorage: restoredSetsToSourceStorage,
+                        cacheSystem: cacheSystem
+                    )
+                }
+
+                let skipTargets = valid.union(allRestoredTargets)
                 targetGraph.remove(skipTargets)
             }
             dependencyGraphToBuild = targetGraph
@@ -137,7 +150,7 @@ struct FrameworkProducer {
         availableTargets: Set<CacheSystem.CacheTarget>,
         cacheSystem: CacheSystem
     ) async -> Set<CacheSystem.CacheTarget> {
-        let chunked = availableTargets.chunks(ofCount: CacheSystem.defaultParalellNumber)
+        let chunked = availableTargets.chunks(ofCount: CacheSystem.defaultParallelNumber)
 
         var validFrameworks: Set<CacheSystem.CacheTarget> = []
         for chunk in chunked {
@@ -147,7 +160,7 @@ struct FrameworkProducer {
                         do {
                             let product = target.buildProduct
                             let frameworkName = product.frameworkName
-                            let outputPath = outputDir.appendingPathComponent(frameworkName)
+                            let outputPath = outputDir.appending(component: frameworkName)
                             let exists = fileSystem.exists(outputPath)
                             guard exists else { return nil }
 
@@ -184,9 +197,9 @@ struct FrameworkProducer {
         availableTargets: Set<CacheSystem.CacheTarget>,
         to storages: [any CacheStorage],
         cacheSystem: CacheSystem
-    ) async -> Set<CacheSystem.CacheTarget> {
+    ) async -> [Set<CacheSystem.CacheTarget>: any CacheStorage] {
         var remainingTargets = availableTargets
-        var restored: Set<CacheSystem.CacheTarget> = []
+        var restoredSetsToSourceStorage: [Set<CacheSystem.CacheTarget>: any CacheStorage] = [:]
 
         for index in storages.indices {
             let storage = storages[index]
@@ -209,7 +222,11 @@ struct FrameworkProducer {
                 from: storage,
                 cacheSystem: cacheSystem
             )
-            restored.formUnion(restoredPerStorage)
+
+            // Record which storage restored which set of targets
+            if !restoredPerStorage.isEmpty {
+                restoredSetsToSourceStorage[restoredPerStorage] = storage
+            }
 
             logger.info(
                 "⏸️ Restoration finished with cache storage: \(logSuffix)",
@@ -224,15 +241,15 @@ struct FrameworkProducer {
         }
 
         logger.info("⏹️ Restoration finished", metadata: .color(.green))
-        return restored
+        return restoredSetsToSourceStorage
     }
 
     private func restoreCaches(
         for targets: Set<CacheSystem.CacheTarget>,
         from cacheStorage: any CacheStorage,
         cacheSystem: CacheSystem
     ) async -> Set<CacheSystem.CacheTarget> {
-        let chunked = targets.chunks(ofCount: cacheStorage.parallelNumber ?? CacheSystem.defaultParalellNumber)
+        let chunked = targets.chunks(ofCount: cacheStorage.parallelNumber ?? CacheSystem.defaultParallelNumber)
 
         var restored: Set<CacheSystem.CacheTarget> = []
         for chunk in chunked {
@@ -369,6 +386,89 @@ struct FrameworkProducer {
             logger.warning("⚠️ Could not create VersionFile. This framework will not be cached.", metadata: .color(.yellow))
         }
     }
+
+    private func shareRestoredCachesToProducers(
+        _ allRestoredTargets: Set<CacheSystem.CacheTarget>,
+        restoredSetsToSourceStorage: [Set<CacheSystem.CacheTarget>: any CacheStorage],
+        cacheSystem: CacheSystem
+    ) async {
+        let storagesWithProducer = cachePolicies.storages(for: .producer)
+        guard !storagesWithProducer.isEmpty else { return }
+
+        logger.info(
+            "🔄 Sharing \(allRestoredTargets.count) restored framework(s) to other cache storages",
+            metadata: .color(.blue)
+        )
+
+        for storage in storagesWithProducer {
+            // Filter targets to exclude those that were restored from this storage
+            var targetsToShare = allRestoredTargets
+
+            // Remove targets that were restored from this storage
+            for (restoredSet, sourceStorage) in restoredSetsToSourceStorage where areStoragesEqual(sourceStorage, storage) {
+                targetsToShare.subtract(restoredSet)
+            }
+
+            if !targetsToShare.isEmpty {
+                await shareCachesToStorage(targetsToShare, to: storage, cacheSystem: cacheSystem)
+            }
+        }
+
+        logger.info("⏹️ Sharing to other cache storages finished", metadata: .color(.green))
+    }
+
+    private func areStoragesEqual(_ lhs: any CacheStorage, _ rhs: any CacheStorage) -> Bool {
+        // ProjectCacheStorage instances are always considered the same
+        if lhs is ProjectCacheStorage && rhs is ProjectCacheStorage {
+            return true
+        }
+
+        // For LocalDiskCacheStorage (value type), use Equatable comparison
+        if let lhsLocal = lhs as? LocalDiskCacheStorage,
+           let rhsLocal = rhs as? LocalDiskCacheStorage {
+            return lhsLocal == rhsLocal
+        }
+
+        // For reference types (actors, classes), use ObjectIdentifier comparison
+        return ObjectIdentifier(lhs as AnyObject) == ObjectIdentifier(rhs as AnyObject)
+    }
+
+    private func shareCachesToStorage(
+        _ targets: Set<CacheSystem.CacheTarget>,
+        to storage: any CacheStorage,
+        cacheSystem: CacheSystem
+    ) async {
+        let chunked = targets.chunks(ofCount: storage.parallelNumber ?? CacheSystem.defaultParallelNumber)
+
+        for chunk in chunked {
+            await withTaskGroup(of: Void.self) { group in
+                for target in chunk {
+                    group.addTask {
+                        do {
+                            let cacheKey = try await cacheSystem.calculateCacheKey(of: target)
+                            let hasCache = try await storage.existsValidCache(for: cacheKey)
+                            guard !hasCache else { return }
+
+                            let frameworkName = target.buildProduct.frameworkName
+                            let frameworkPath = outputDir.appending(component: frameworkName)
+
+                            logger.info(
+                                "🔄 Share \(frameworkName) to cache storage: \(storage.displayName)",
+                                metadata: .color(.blue)
+                            )
+                            try await storage.cacheFramework(frameworkPath, for: cacheKey)
+                        } catch {
+                            logger.warning(
+                                "⚠️ Failed to share cache to \(storage.displayName): \(error.localizedDescription)",
+                                metadata: .color(.yellow)
+                            )
+                        }
+                    }
+                }
+                await group.waitForAll()
+            }
+        }
+    }
 }
 
 extension [Runner.Options.CachePolicy] {