Fix deep hash and signing

.gitignore

@@ -3,3 +3,4 @@
 /Packages
 /*.xcodeproj
 xcuserdata/
+Tests/Dave-key

Package.swift

@@ -26,6 +26,6 @@ let package = Package(
             name: "ArweaveTests",
             dependencies: ["Arweave"],
             path: "Tests",
-            resources: [.process("test-key.json")])
+            resources: [.process("test-key.json"), .process("dave-key")])
     ]
 )

Sources/API/HttpClient.swift

@@ -2,9 +2,9 @@ import Foundation
 
 extension String: Error { }
 
-struct HttpResponse {
-    let data: Data
-    let statusCode: Int
+public struct HttpResponse {
+    public let data: Data
+    public let statusCode: Int
 }
 
 struct HttpClient {
@@ -23,6 +23,7 @@ struct HttpClient {
 
         if case .transactionStatus = target.route {}
         else if statusCode != 200 {
+            print("\(httpResponse.debugDescription)")
             throw "Bad response code \(statusCode)"
         }
 

Sources/KeyUtilities.swift

@@ -6,14 +6,79 @@ extension Digest {
     var data: Data { Data(bytes) }
 }
 
-extension String {
+public extension String {
     var base64URLEncoded: String {
         Data(utf8).base64URLEncodedString()
     }
+
+    var base64URLDecoded: String {
+        guard let data = Data(base64URLEncoded: self) else { return "" }
+        return String(data: data, encoding: .utf8) ?? ""
+    }
 }
 
 extension Array where Element == Data {
     var combined: Data {
        reduce(.init(), +)
     }
 }
+
+func concatData(data: [Data]) -> Data {
+    var length = 0
+    for d in data {
+        length += d.count
+    }
+
+    var temp = Data(capacity: length)
+    for d in data {
+        temp.append(d)
+    }
+
+    return temp
+}
+
+func deepHash(data: DeepHashChunk) -> Data {
+    if case .dataArray(let dataArray) = data {
+        let tag = concatData(data: [
+            "list".data(using: .utf8)!,
+            String(dataArray.count).data(using: .utf8)!
+        ])
+
+        return deepHashChunks(chunks: dataArray, acc: SHA384.hash(data: tag).data)
+    }
+
+    if case .data(let dataItem) = data {
+        let tag = concatData(data: [
+            "blob".data(using: .utf8)!,
+            dataItem.count.description.data(using: .utf8)!
+        ])
+
+        let taggedHash = concatData(data: [
+            SHA384.hash(data: tag).data,
+            SHA384.hash(data: dataItem).data
+        ])
+
+        return SHA384.hash(data: taggedHash).data
+    }
+
+    return "".data(using: .utf8)!
+}
+
+func deepHashChunks(chunks: DeepHashChunks, acc: Data) -> Data {
+    if chunks.count < 1 {
+        return acc
+    }
+
+    let hashPair = concatData(data: [
+        acc,
+        deepHash(data: chunks.first!)
+    ])
+    let newAcc = SHA384.hash(data: hashPair).data
+    return deepHashChunks(chunks: Array(chunks[1..<chunks.count]), acc: newAcc)
+}
+
+enum DeepHashChunk {
+    case data(Data)
+    case dataArray(DeepHashChunks)
+}
+typealias DeepHashChunks = Array<DeepHashChunk>

Sources/Merkle.swift

@@ -0,0 +1,268 @@
+//
+//  File.swift
+//
+//
+//  Created by David Choi on 11/6/21.
+//
+
+import Foundation
+import CryptoKit
+import zlib
+
+struct Chunk {
+    let dataHash: Data
+    let minByteRange: Int
+    let maxByteRange: Int
+}
+
+class BranchNode {
+    var id: Data
+    var type: MerkleNode? = nil
+    let byteRange: Int
+    var maxByteRange: Int
+    let leftChild: MerkleNode?
+    let rightChild: MerkleNode?
+
+    init(id: Data, byteRange: Int, maxByteRange: Int, leftChild: MerkleNode? = nil, rightChild: MerkleNode? = nil) {
+        self.id = id
+        self.byteRange = byteRange
+        self.maxByteRange = maxByteRange
+        self.leftChild = leftChild
+        self.rightChild = rightChild
+    }
+}
+
+class LeafNode {
+    var id: Data
+    let dataHash: Data
+    var type: MerkleNode? = nil
+    let minByteRange: Int
+    var maxByteRange: Int
+
+    init(id: Data, dataHash: Data, minByteRange: Int, maxByteRange: Int) {
+        self.id = id
+        self.dataHash = dataHash
+        self.minByteRange = minByteRange
+        self.maxByteRange = maxByteRange
+    }
+}
+
+enum MerkleNode {
+    case branchNode(BranchNode)
+    case leafNode(LeafNode)
+}
+
+enum BranchOrLeafError: Error {
+    case UnknownNodeType
+}
+
+struct Proof {
+    let offset: Int
+    let proof: Data
+}
+
+let MAX_CHUNK_SIZE = 256 * 1024
+let MIN_CHUNK_SIZE = 32 * 1024
+let NOTE_SIZE = 32
+let HASH_SIZE = 32
+
+func chunkData(data: Data) -> [Chunk] {
+    var chunks = [Chunk]()
+
+    var rest = data
+    var cursor = 0
+
+    while(rest.count >= MAX_CHUNK_SIZE) {
+        var chunkSize = MAX_CHUNK_SIZE
+
+        let nextChunkSize = rest.count - MAX_CHUNK_SIZE
+        if (nextChunkSize > 0 && nextChunkSize < MIN_CHUNK_SIZE) {
+            chunkSize = Int(Double(rest.count / 2).rounded())
+        }
+
+        let chunk = rest.subdata(in: 0..<chunkSize)
+        let dataHash = SHA256.hash(data: chunk)
+        cursor += chunk.count
+        chunks.append(Chunk(dataHash: dataHash.data, minByteRange: cursor - chunk.count, maxByteRange: cursor))
+        rest = rest.subdata(in: (chunkSize <= 0 ? 0 : chunkSize - 1)..<rest.count)
+    }
+
+    chunks.append(Chunk(dataHash: SHA256.hash(data: rest).data, minByteRange: cursor, maxByteRange: cursor + rest.count))
+    return chunks
+}
+
+func generateLeaves(chunks: [Chunk]) -> [LeafNode] {
+    return chunks.map { chunk in
+        var idData = [Data]()
+        idData.append(chunk.dataHash)
+        idData.append(intToBuffer(note: chunk.maxByteRange))
+
+        let leaf = LeafNode(
+            id: hashId(data: idData),
+            dataHash: chunk.dataHash,
+            minByteRange: chunk.minByteRange,
+            maxByteRange: chunk.maxByteRange
+        )
+        leaf.type = MerkleNode.leafNode(leaf)
+        return leaf
+    }
+}
+
+func hashId(data: [Data]) -> Data {
+    let data = concatBuffers(buffers: data)
+    return Data(SHA256.hash(data: data))
+}
+
+func intToBuffer(note: Int) -> Data {
+    var note = note
+    var buffer = Data(capacity: NOTE_SIZE)
+
+    for i in stride(from: buffer.count - 1, through: 0, by: -1) {
+        let byte = note % 256
+        buffer[i] = UInt8(byte)
+        note = (note - byte) / 256
+    }
+
+    return buffer
+}
+
+// of leafs or branches
+func buildLayers(nodes: [MerkleNode], level: Int = 0) throws -> MerkleNode {
+    let nodesCount = nodes.count
+    if nodesCount < 2 {
+        return try hashBranch(left: nodes[0])
+    }
+
+    var nextLayer = [MerkleNode]()
+
+    for i in stride(from: 0, to: nodesCount, by: 2) {
+        nextLayer.append(try hashBranch(left: nodes[i], right: i + 1 < nodesCount ? nodes[i + 1] : nil))
+    }
+
+    return try buildLayers(nodes: nextLayer, level: level + 1)
+}
+
+func generateTransactionChunks(data: Data) throws -> Chunks {
+    var chunks = chunkData(data: data)
+    let leaves = generateLeaves(chunks: chunks)
+    let root = try buildLayers(nodes: leaves.map { leaf in
+        leaf.type!
+    })
+    var proofs = generateProofs(root: root)
+
+    if chunks.count > 0 {
+        let lastChunk = chunks.last
+        if ((lastChunk!.maxByteRange - lastChunk!.minByteRange) == 0) {
+            chunks.remove(at: chunks.count - 1)
+            proofs.remove(at: proofs.count - 1)
+        }
+    }
+
+    var rootId: Data?
+    switch root {
+    case .leafNode(let leaf):
+        rootId = leaf.id
+    case .branchNode(let branch):
+        rootId = branch.id
+    }
+    return Chunks(data_root: rootId!, chunks: chunks, proofs: proofs)
+}
+
+func generateProofs(root: MerkleNode) -> [Proof] {
+    var proofs: [Proof] = [Proof]()
+    do {
+        proofs = try resolveBranchProofs(node: root)
+    } catch {
+        print("failed to resolve branch proofs \(error)")
+    }
+    return proofs
+}
+
+func resolveBranchProofs(node: MerkleNode, proof: Data = Data(), depth: Int = 0) throws -> [Proof] {
+    if case .leafNode(let leaf) = node {
+        let dataHash = leaf.dataHash
+        return [
+            Proof(offset: leaf.maxByteRange - 1, proof: concatBuffers(buffers: [proof, dataHash, intToBuffer(note: leaf.maxByteRange)]))
+        ]
+    }
+
+    if case .branchNode(let branch) = node {
+        var buffers = [
+            proof,
+            intToBuffer(note: branch.byteRange)
+        ]
+        if let leftChild = branch.leftChild {
+            if case .leafNode(let leftLeaf) = leftChild {
+                buffers.append(leftLeaf.id)
+            } else if case .branchNode(let leftBranch) = leftChild {
+                buffers.append(leftBranch.id)
+            }
+        }
+        if let rightChild = branch.rightChild {
+            if case .leafNode(let rightLeaf) = rightChild {
+                buffers.append(rightLeaf.id)
+            } else if case .branchNode(let rightBranch) = rightChild {
+                buffers.append(rightBranch.id)
+            }
+        }
+        let partialProof = concatBuffers(buffers: buffers)
+
+        var resolvedProofs = [[Proof]]()
+        if let leftChild = branch.leftChild {
+            resolvedProofs.append(try resolveBranchProofs(node: leftChild, proof: partialProof, depth: depth + 1))
+        }
+        if let rightChild = branch.rightChild {
+            resolvedProofs.append(try resolveBranchProofs(node: rightChild, proof: partialProof, depth: depth + 1))
+        }
+        return Array(resolvedProofs.joined())
+    }
+
+    throw BranchOrLeafError.UnknownNodeType
+}
+
+func hashBranch(left: MerkleNode, right: MerkleNode? = nil) throws -> MerkleNode {
+    if right == nil {
+        switch left {
+        case .leafNode(let leaf):
+            let branch = BranchNode(id: leaf.id, byteRange: leaf.maxByteRange, maxByteRange: leaf.maxByteRange, leftChild: nil, rightChild: nil)
+            branch.type = MerkleNode.branchNode(branch)
+            return branch.type!
+        case .branchNode(let branch):
+            return branch.type!
+        }
+    } else {
+        var leftLeaf: LeafNode?
+        var leftBranch: BranchNode?
+        var rightLeaf: LeafNode?
+        var rightBranch: BranchNode?
+
+        switch left {
+        case .leafNode(let leaf):
+            leftLeaf = leaf
+        case .branchNode(let branch):
+            leftBranch = branch
+        }
+
+        switch right! {
+        case .leafNode(let leaf):
+            rightLeaf = leaf
+        case .branchNode(let branch):
+            rightBranch = branch
+        }
+
+        let branch = BranchNode(
+            id: hashId(data: [
+                hashId(data: [leftLeaf != nil ? leftLeaf!.id : leftBranch!.id]),
+                hashId(data: [rightLeaf != nil ? rightLeaf!.id : rightBranch!.id]),
+                hashId(data: [intToBuffer(note: leftLeaf != nil ? leftLeaf!.maxByteRange : leftBranch!.maxByteRange)]),
+            ]),
+            byteRange: leftLeaf != nil ? leftLeaf!.maxByteRange : leftBranch!.maxByteRange,
+            maxByteRange: rightLeaf != nil ? rightLeaf!.maxByteRange : rightBranch!.maxByteRange,
+            leftChild: left,
+            rightChild: right
+        )
+        branch.type = MerkleNode.branchNode(branch)
+
+        return branch.type!
+    }
+}