[#1475] Adopt transaction data requests

CHANGELOG.md

@@ -14,6 +14,21 @@ and this library adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   `Synchronizer.exchangeRateUSDStream`. Prices are queried over Tor (to hide the wallet's
   IP address).
 
+## Changed
+
+### [#1475] Adopt transaction data requests
+- The transaction history is now processed using `transaction data requests`, which are fetched every 1,000 blocks during longer syncs or with each sync loop when a new block is mined.
+
+## Checkpoints
+
+Mainnet
+
+````
+Sources/ZcashLightClientKit/Resources/checkpoints/mainnet/2562500.json
+...
+Sources/ZcashLightClientKit/Resources/checkpoints/mainnet/2610000.json
+````
+
 # 2.1.12 - 2024-07-04
 
 ## Fixed

Example/ZcashLightClientSample/ZcashLightClientSample.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

@@ -176,7 +176,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/zcash-hackworks/zcash-light-client-ffi",
       "state" : {
-        "revision" : "4de1b42f99aebfc5e4f0340da8a66a4f719db9a6"
+        "revision" : "6521af353b63299d32d010195302dac50074f918",
+        "version" : "0.9.0"
       }
     }
   ],

Package.swift

@@ -16,9 +16,7 @@ let package = Package(
     dependencies: [
         .package(url: "https://github.com/grpc/grpc-swift.git", from: "1.19.1"),
         .package(url: "https://github.com/stephencelis/SQLite.swift.git", from: "0.14.1"),
-        // .package(url: "https://github.com/zcash-hackworks/zcash-light-client-ffi", exact: "0.8.1")
-        // Compiled from 2516a94f8bdc540d951c38b66e9c07e2b8c29cb4
-        .package(url: "https://github.com/zcash-hackworks/zcash-light-client-ffi", revision: "4de1b42f99aebfc5e4f0340da8a66a4f719db9a6")
+        .package(url: "https://github.com/zcash-hackworks/zcash-light-client-ffi", exact: "0.9.0")
     ],
     targets: [
         .target(

Sources/ZcashLightClientKit/Block/Actions/UpdateSubtreeRootsAction.swift

@@ -30,7 +30,7 @@ extension UpdateSubtreeRootsAction: Action {
 
         logger.debug("Attempt to get subtree roots, this may fail because lightwalletd may not support Spend before Sync.")
         let stream = service.getSubtreeRoots(request)
-        
+
         var saplingRoots: [SubtreeRoot] = []
 
         do {
@@ -39,6 +39,8 @@ extension UpdateSubtreeRootsAction: Action {
             }
         } catch ZcashError.serviceSubtreeRootsStreamFailed(LightWalletServiceError.timeOut) {
             throw ZcashError.serviceSubtreeRootsStreamFailed(LightWalletServiceError.timeOut)
+        } catch {
+            await context.update(state: .updateChainTip)
         }
 
         logger.debug("Sapling tree has \(saplingRoots.count) subtrees")

Sources/ZcashLightClientKit/Block/Download/BlockDownloaderService.swift

@@ -43,7 +43,7 @@ protocol BlockDownloaderService {
     Gets the transaction for the Id given
     - Parameter txId: Data representing the transaction Id
     */
-    func fetchTransaction(txId: Data) async throws -> ZcashTransaction.Fetched
+    func fetchTransaction(txId: Data) async throws -> (tx: ZcashTransaction.Fetched?, status: TransactionStatus)
 
     func fetchUnspentTransactionOutputs(tAddress: String, startHeight: BlockHeight) -> AsyncThrowingStream<UnspentTransactionOutputEntity, Error>
 
@@ -111,7 +111,7 @@ extension BlockDownloaderServiceImpl: BlockDownloaderService {
         try await self.storage.latestHeight()
     }
 
-    func fetchTransaction(txId: Data) async throws -> ZcashTransaction.Fetched {
+    func fetchTransaction(txId: Data) async throws -> (tx: ZcashTransaction.Fetched?, status: TransactionStatus) {
         try await lightwalletService.fetchTransaction(txId: txId)
     }
 }

Sources/ZcashLightClientKit/Block/Enhance/BlockEnhancer.swift

@@ -60,28 +60,12 @@
     let rustBackend: ZcashRustBackendWelding
     let transactionRepository: TransactionRepository
     let metrics: SDKMetrics
+    let service: LightWalletService
     let logger: Logger
-
-    private func enhance(transaction: ZcashTransaction.Overview) async throws -> ZcashTransaction.Overview {
-        logger.debug("Zoom.... Enhance... Tx: \(transaction.rawID.toHexStringTxId())")
-
-        let fetchedTransaction = try await blockDownloaderService.fetchTransaction(txId: transaction.rawID)
-
-        let transactionID = fetchedTransaction.rawID.toHexStringTxId()
-        let block = String(describing: transaction.minedHeight)
-        logger.debug("Decrypting and storing transaction id: \(transactionID) block: \(block)")
-
-        try await rustBackend.decryptAndStoreTransaction(
-            txBytes: fetchedTransaction.raw.bytes,
-            minedHeight: Int32(fetchedTransaction.minedHeight)
-        )
-
-        return try await transactionRepository.find(rawID: fetchedTransaction.rawID)
-    }
 }
 
 extension BlockEnhancerImpl: BlockEnhancer {
     func enhance(at range: CompactBlockRange, didEnhance: @escaping (EnhancementProgress) async -> Void) async throws -> [ZcashTransaction.Overview]? {
         try Task.checkCancellation()
 
         logger.debug("Started Enhancing range: \(range)")
@@ -91,55 +75,75 @@
 
         // fetch transactions
         do {
-            let startTime = Date()
-            let transactions = try await transactionRepository.find(in: range, limit: Int.max, kind: .all)
+            let transactionDataRequests = try await rustBackend.transactionDataRequests()
 
-            guard !transactions.isEmpty else {
-                logger.debug("no transactions detected on range: \(range.lowerBound)...\(range.upperBound)")
-                logger.sync("No transactions detected on range: \(range.lowerBound)...\(range.upperBound)")
+            guard !transactionDataRequests.isEmpty else {
+                logger.debug("No transaction data requests detected.")
+                logger.sync("No transaction data requests detected.")
                 return nil
             }
 
-            let chainTipHeight = try await blockDownloaderService.latestBlockHeight()
-
-            let newlyMinedLowerBound = chainTipHeight - ZcashSDK.expiryOffset
-
-            let newlyMinedRange = newlyMinedLowerBound...chainTipHeight
-
-            for index in 0 ..< transactions.count {
-                let transaction = transactions[index]
+            for index in 0 ..< transactionDataRequests.count {
+                let transactionDataRequest = transactionDataRequests[index]
                 var retry = true
-                
+
                 while retry && retries < maxRetries {
                     try Task.checkCancellation()
                     do {
-                        let confirmedTx = try await enhance(transaction: transaction)
-                        retry = false
-
-                        let progress = EnhancementProgress(
-                            totalTransactions: transactions.count,
-                            enhancedTransactions: index + 1,
-                            lastFoundTransaction: confirmedTx,
-                            range: range,
-                            newlyMined: confirmedTx.isSentTransaction && newlyMinedRange.contains(confirmedTx.minedHeight ?? 0)
-                        )
-
-                        await didEnhance(progress)
+                        switch transactionDataRequest {
+                        case .getStatus(let txId):
+                            let response = try await blockDownloaderService.fetchTransaction(txId: txId.data)
+                            retry = false
+
+                            if let fetchedTransaction = response.tx {
+                                try await rustBackend.setTransactionStatus(txId: fetchedTransaction.rawID, status: response.status)
+                            }
+
+                        case .enhancement(let txId):
+                            let response = try await blockDownloaderService.fetchTransaction(txId: txId.data)
+                            retry = false
+
+                            if response.status == .txidNotRecognized {
+                                try await rustBackend.setTransactionStatus(txId: txId.data, status: .txidNotRecognized)
+                            } else if let fetchedTransaction = response.tx {
+                                try await rustBackend.decryptAndStoreTransaction(
+                                    txBytes: fetchedTransaction.raw.bytes,
+                                    minedHeight: fetchedTransaction.minedHeight
+                                )
+                            }
+
+                        case .spendsFromAddress(let sfa):
+                            guard let blockRangeEnd = sfa.blockRangeEnd else {
+                                logger.error("spendsFromAddress \(sfa) is missing blockRangeEnd, ignoring the request.")
+                                continue
+                            }
+
+                            var filter = TransparentAddressBlockFilter()
+                            filter.address = sfa.address
+                            filter.range = BlockRange(startHeight: Int(sfa.blockRangeStart), endHeight: Int(blockRangeEnd - 1))
+
+                            let stream = service.getTaddressTxids(filter)
+
+                            for try await rawTransaction in stream {
+                                let minedHeight = (rawTransaction.height == 0 || rawTransaction.height > UInt32.max) 
+                                ? nil : UInt32(rawTransaction.height)
+
+                                try await rustBackend.decryptAndStoreTransaction(
+                                    txBytes: rawTransaction.data.bytes,
+                                    minedHeight: minedHeight
+                                )
+                            }
+                            retry = false
+                        }
                     } catch {
                         retries += 1
-                        logger.error("could not enhance txId \(transaction.rawID.toHexStringTxId()) - Error: \(error)")
+                        logger.error("could not enhance transactionDataRequest \(transactionDataRequest) - Error: \(error)")
                         if retries > maxRetries {
                             throw error
                         }
                     }
                 }
             }
-
-            let endTime = Date()
-            let diff = endTime.timeIntervalSince1970 - startTime.timeIntervalSince1970
-            let logMsg = "Enhanced \(transactions.count) transaction(s) in \(diff) for range \(range.lowerBound)...\(range.upperBound)"
-            logger.sync(logMsg)
-            metrics.actionDetail(logMsg, for: .enhance)
         } catch {
             logger.error("error enhancing transactions! \(error)")
             throw error

Sources/ZcashLightClientKit/Entity/TransactionEntity.swift

@@ -94,7 +94,7 @@
     /// Used when fetching blocks from the lightwalletd
     struct Fetched {
         public let rawID: Data
-        public let minedHeight: BlockHeight
+        public let minedHeight: UInt32?
         public let raw: Data
     }
 }
@@ -124,7 +124,7 @@
             if
                 let outputRecipient = try row.get(Column.toAddress),
                 let metadata = DerivationTool.getAddressMetadata(outputRecipient)
             {
                 recipient = TransactionRecipient.address(try Recipient(outputRecipient, network: metadata.networkType))
             } else if let toAccount = try row.get(Column.toAccount) {
                 recipient = .internalAccount(UInt32(toAccount))

Sources/ZcashLightClientKit/Error/ZcashError.swift

@@ -61,6 +61,9 @@ public enum ZcashError: Equatable, Error {
     /// LightWalletService.getSubtreeRoots failed.
     /// ZSRVC0009
     case serviceSubtreeRootsStreamFailed(_ error: LightWalletServiceError)
+    /// LightWalletService.getTaddressTxids failed.
+    /// ZSRVC0010
+    case serviceGetTaddressTxidsFailed(_ error: LightWalletServiceError)
     /// SimpleConnectionProvider init of Connection failed.
     /// ZSCPC0001
     case simpleConnectionProvider(_ error: Error)
@@ -352,6 +355,11 @@ public enum ZcashError: Equatable, Error {
     /// sourcery: code="ZRUST0063"
     /// ZRUST0063
     case rustTorClientGet(_ rustError: String)
+    /// Error from rust layer when calling ZcashRustBackend.transactionDataRequests
+    /// - `rustError` contains error generated by the rust layer.
+    /// sourcery: code="ZRUST0064"
+    /// ZRUST0064
+    case rustTransactionDataRequests(_ rustError: String)
     /// SQLite query failed when fetching all accounts from the database.
     /// - `sqliteError` is error produced by SQLite library.
     /// ZADAO0001
@@ -646,6 +654,7 @@ public enum ZcashError: Equatable, Error {
         case .serviceFetchUTXOsFailed: return "LightWalletService.fetchUTXOs failed."
         case .serviceBlockStreamFailed: return "LightWalletService.blockStream failed."
         case .serviceSubtreeRootsStreamFailed: return "LightWalletService.getSubtreeRoots failed."
+        case .serviceGetTaddressTxidsFailed: return "LightWalletService.getTaddressTxids failed."
         case .simpleConnectionProvider: return "SimpleConnectionProvider init of Connection failed."
         case .saplingParamsInvalidSpendParams: return "Downloaded file with sapling spending parameters isn't valid."
         case .saplingParamsInvalidOutputParams: return "Downloaded file with sapling output parameters isn't valid."
@@ -720,6 +729,7 @@ public enum ZcashError: Equatable, Error {
         case .rustPutOrchardSubtreeRoots: return "Error from rust layer when calling ZcashRustBackend.putOrchardSubtreeRoots"
         case .rustTorClientInit: return "Error from rust layer when calling TorClient.init"
         case .rustTorClientGet: return "Error from rust layer when calling TorClient.get"
+        case .rustTransactionDataRequests: return "Error from rust layer when calling ZcashRustBackend.transactionDataRequests"
         case .accountDAOGetAll: return "SQLite query failed when fetching all accounts from the database."
         case .accountDAOGetAllCantDecode: return "Fetched accounts from SQLite but can't decode them."
         case .accountDAOFindBy: return "SQLite query failed when seaching for accounts in the database."
@@ -829,6 +839,7 @@ public enum ZcashError: Equatable, Error {
         case .serviceFetchUTXOsFailed: return .serviceFetchUTXOsFailed
         case .serviceBlockStreamFailed: return .serviceBlockStreamFailed
         case .serviceSubtreeRootsStreamFailed: return .serviceSubtreeRootsStreamFailed
+        case .serviceGetTaddressTxidsFailed: return .serviceGetTaddressTxidsFailed
         case .simpleConnectionProvider: return .simpleConnectionProvider
         case .saplingParamsInvalidSpendParams: return .saplingParamsInvalidSpendParams
         case .saplingParamsInvalidOutputParams: return .saplingParamsInvalidOutputParams
@@ -903,6 +914,7 @@ public enum ZcashError: Equatable, Error {
         case .rustPutOrchardSubtreeRoots: return .rustPutOrchardSubtreeRoots
         case .rustTorClientInit: return .rustTorClientInit
         case .rustTorClientGet: return .rustTorClientGet
+        case .rustTransactionDataRequests: return .rustTransactionDataRequests
         case .accountDAOGetAll: return .accountDAOGetAll
         case .accountDAOGetAllCantDecode: return .accountDAOGetAllCantDecode
         case .accountDAOFindBy: return .accountDAOFindBy

Sources/ZcashLightClientKit/Error/ZcashErrorCode.swift

@@ -41,6 +41,8 @@ public enum ZcashErrorCode: String {
     case serviceBlockStreamFailed = "ZSRVC0000"
     /// LightWalletService.getSubtreeRoots failed.
     case serviceSubtreeRootsStreamFailed = "ZSRVC0009"
+    /// LightWalletService.getTaddressTxids failed.
+    case serviceGetTaddressTxidsFailed = "ZSRVC0010"
     /// SimpleConnectionProvider init of Connection failed.
     case simpleConnectionProvider = "ZSCPC0001"
     /// Downloaded file with sapling spending parameters isn't valid.
@@ -189,6 +191,8 @@ public enum ZcashErrorCode: String {
     case rustTorClientInit = "ZRUST0062"
     /// Error from rust layer when calling TorClient.get
     case rustTorClientGet = "ZRUST0063"
+    /// Error from rust layer when calling ZcashRustBackend.transactionDataRequests
+    case rustTransactionDataRequests = "ZRUST0064"
     /// SQLite query failed when fetching all accounts from the database.
     case accountDAOGetAll = "ZADAO0001"
     /// Fetched accounts from SQLite but can't decode them.

Sources/ZcashLightClientKit/Error/ZcashErrorCodeDefinition.swift

@@ -80,6 +80,9 @@ enum ZcashErrorDefinition {
     /// LightWalletService.getSubtreeRoots failed.
     // sourcery: code="ZSRVC0009"
     case serviceSubtreeRootsStreamFailed(_ error: LightWalletServiceError)
+    /// LightWalletService.getTaddressTxids failed.
+    // sourcery: code="ZSRVC0010"
+    case serviceGetTaddressTxidsFailed(_ error: LightWalletServiceError)
 
     // MARK: SQLite connection
 
@@ -375,6 +378,10 @@ enum ZcashErrorDefinition {
     /// - `rustError` contains error generated by the rust layer.
     /// sourcery: code="ZRUST0063"
     case rustTorClientGet(_ rustError: String)
+    /// Error from rust layer when calling ZcashRustBackend.transactionDataRequests
+    /// - `rustError` contains error generated by the rust layer.
+    /// sourcery: code="ZRUST0064"
+    case rustTransactionDataRequests(_ rustError: String)
 
     // MARK: - Account DAO
 

Sources/ZcashLightClientKit/Model/Proposal.swift

@@ -35,7 +35,7 @@ public extension Proposal {
     /// passed to `Synchronizer.createProposedTransactions`. It should never be called in
     /// production code.
     static func testOnlyFakeProposal(totalFee: UInt64) -> Self {
-        var ffiProposal = FfiProposal()
+        let ffiProposal = FfiProposal()
         var balance = FfiTransactionBalance()
 
         balance.feeRequired = totalFee

Sources/ZcashLightClientKit/Model/TransactionDataRequest.swift

@@ -0,0 +1,26 @@
+//
+//  TransactionDataRequest.swift
+//
+//
+//  Created by Lukáš Korba on 08-15-2024.
+//
+
+import Foundation
+
+struct SpendsFromAddress: Equatable {
+    let address: String
+    let blockRangeStart: UInt32
+    let blockRangeEnd: UInt32?
+}
+
+enum TransactionDataRequest: Equatable {
+    case getStatus([UInt8])
+    case enhancement([UInt8])
+    case spendsFromAddress(SpendsFromAddress)
+}
+
+enum TransactionStatus: Equatable {
+    case txidNotRecognized
+    case notInMainChain
+    case mined(BlockHeight)
+}