From 6235bafa2b5dc0ffa834f6980f37a95e3cf55bd9 Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Mon, 17 Apr 2023 17:01:56 +0100 Subject: [PATCH 01/10] feat: getPath with carScope add getPath method as a generator that returns blocks for the targeted dag and all blocks traversed while resolving a cid+path string supports carScope to specify what blocks to return for the resolved dag - 'all': return the entire dag starting at path. (default) - 'block': return the block identified by the path. - 'file': Mimic gateway semantics: Return All blocks for a multi-block file or just enough blocks to enumerate a dir/map but not the dir contents. see: https://github.com/web3-storage/freeway/issues/33 see: https://github.com/web3-storage/freeway/issues/34 TODO: - [] find out how to identify the boundaries of a unixfs hamt (unixfs-exported seems to define it as "not having an empty or null Link.Name after the first 2 chars are stripped, which seems risky... what happens if the actual dir listing has 2 char long link names? see: https://github.com/ipfs/js-ipfs-unixfs/blob/e853049bd63d6773442e1540ae49b6a443ca8672/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts#L20-L42 License: MIT Signed-off-by: Oli Evans --- bin.js | 7 +- index.d.ts | 18 ++- index.js | 56 ++++++-- test/getPath.test.js | 248 ++++++++++++++++++++++++++++++++++ test.js => test/index.test.js | 11 +- 5 files changed, 323 insertions(+), 17 deletions(-) create mode 100644 test/getPath.test.js rename test.js => test/index.test.js (93%) diff --git a/bin.js b/bin.js index a3f400a..5cf31c1 100755 --- a/bin.js +++ b/bin.js @@ -56,8 +56,9 @@ cli.command('get ') .describe('Fetch a DAG from the peer. Outputs a CAR file.') .option('-p, --peer', 'Address of peer to fetch data from.') .option('-t, --timeout', 'Timeout in milliseconds.', TIMEOUT) - .action(async (cid, { peer, timeout }) => { - cid = CID.parse(cid) + .action(async (cidPath, { peer, timeout }) => { + const [cidStr] = cidPath.replace(/^\/ipfs\//, '').split('/') + const cid = CID.parse(cidStr) const controller = new TimeoutController(timeout) const libp2p = await getLibp2p() const dagula = await Dagula.fromNetwork(libp2p, { peer }) @@ -66,7 +67,7 @@ cli.command('get ') let error ;(async () => { try { - for await (const block of dagula.get(cid, { signal: controller.signal })) { + for await (const block of dagula.getPath(cidPath, { signal: controller.signal })) { controller.reset() await writer.put(block) } diff --git a/index.d.ts b/index.d.ts index cbb323e..dbcaa22 100644 --- a/index.d.ts +++ b/index.d.ts @@ -25,11 +25,21 @@ export interface Network { handle: (protocol: string | string[], handler: StreamHandler) => Promise } +export type CarScope = 'all'|'file'|'block' + +export interface CarScopeOptions { + carScope?: CarScope +} + export interface IDagula { /** * Get a complete DAG. */ get (cid: CID|string, options?: AbortOptions): AsyncIterableIterator + /** + * Get a DAG for a cid+path + */ + getPath (cidPath: string, options?: AbortOptions & CarScopeOptions): AsyncIterableIterator /** * Get a single block. */ @@ -41,7 +51,7 @@ export interface IDagula { /** * Emit nodes for all path segements and get UnixFS files and directories */ - walkUnixfsPath (path: CID|string, options?: AbortOptions): Promise + walkUnixfsPath (path: CID|string, options?: AbortOptions): AsyncIterableIterator } export declare class Dagula implements IDagula { @@ -50,6 +60,10 @@ export declare class Dagula implements IDagula { * Get a complete DAG. */ get (cid: CID|string, options?: AbortOptions): AsyncIterableIterator + /** + * Get a DAG for a cid+path + */ + getPath (cidPath: string, options?: AbortOptions & CarScopeOptions): AsyncIterableIterator /** * Get a single block. */ @@ -61,7 +75,7 @@ export declare class Dagula implements IDagula { /** * Emit nodes for all path segements and get UnixFS files and directories */ - walkUnixfsPath (path: CID|string, options?: AbortOptions): Promise + walkUnixfsPath (path: CID|string, options?: AbortOptions): AsyncIterableIterator /** * Create a new Dagula instance from the passed libp2p Network interface. */ diff --git a/index.js b/index.js index ad6a8bf..30f1bb0 100644 --- a/index.js +++ b/index.js @@ -66,14 +66,14 @@ export class Dagula { } /** - * @param {import('multiformats').CID|string} cid + * @param {CID[]|CID|string} cid * @param {{ signal?: AbortSignal }} [options] */ async * get (cid, options = {}) { cid = typeof cid === 'string' ? CID.parse(cid) : cid log('getting DAG %s', cid) - let cids = [cid] - while (true) { + let cids = Array.isArray(cid) ? cid : [cid] + while (cids.length > 0) { log('fetching %d CIDs', cids.length) const fetchBlocks = transform(cids.length, async cid => { return this.getBlock(cid, { signal: options.signal }) @@ -98,6 +98,43 @@ export class Dagula { } } + /** + * @param {string} cidPath + * @param {object} [options] + * @param {AbortSignal} [options.signal] + * @param {'all'|'file'|'block'} [options.carScope] control how many layers of the dag are returned + * 'all': return the entire dag starting at path. (default) + * 'block': return the block identified by the path. + * 'file': Mimic gateway semantics: Return All blocks for a multi-block file or just enough blocks to enumerate a dir/map but not the dir contents. + * e.g. Where path points to a single block file, all three selectors would return the same thing. + * e.g. where path points to a sharded hamt: 'file' returns the blocks of the hamt so the dir can be listed. 'block' returns the root block of the hamt. + */ + async * getPath (cidPath, options = {}) { + const carScope = options.carScope ?? 'all' + /** @type {import('ipfs-unixfs-exporter').UnixFSEntry} */ + let base + for await (const item of this.walkUnixfsPath(cidPath, { signal: options.signal })) { + base = item + yield item + } + if (carScope === 'all' || (carScope === 'file' && base.type !== 'directory')) { + // fetch the entire dag rooted at the end of the provided path + const links = base.node.Links?.map(l => l.Hash) || [] + if (links.length) { + yield * this.get(links, { signal: options.signal }) + } + } + // non-files, like directories, and IPLD Maps only return blocks necessary for their enumeration + if (carScope === 'file' && base.type === 'directory') { + // the single block for the root has already been yielded. For a hamt we must fetch all the blocks of the (current) hamt. + if (base.unixfs.type === 'hamt-sharded-directory') { + // TODO: how to determine the boudary of a hamt + throw new Error('hamt-sharded-directory is unsupported') + } + // otherwise a dir is a single block, so we're done. + } + } + /** * @param {import('multiformats').CID|string} cid * @param {{ signal?: AbortSignal }} [options] @@ -133,11 +170,11 @@ export class Dagula { } /** - * @param {string|import('multiformats').CID} path + * @param {string} cidPath * @param {{ signal?: AbortSignal }} [options] */ - async * walkUnixfsPath (path, options = {}) { - log('walking unixfs %s', path) + async * walkUnixfsPath (cidPath, options = {}) { + log('walking unixfs %s', cidPath) const blockstore = { /** * @param {CID} cid @@ -148,7 +185,10 @@ export class Dagula { return block.bytes } } - - yield * walkPath(path, blockstore, { signal: options.signal }) + for await (const entry of walkPath(cidPath, blockstore, { signal: options.signal })) { + /** @type {Uint8Array} */ + const bytes = entry.node.Links ? dagPb.encode(entry.node) : entry.node + yield { ...entry, bytes } + } } } diff --git a/test/getPath.test.js b/test/getPath.test.js new file mode 100644 index 0000000..ac485e4 --- /dev/null +++ b/test/getPath.test.js @@ -0,0 +1,248 @@ +import test from 'ava' +import { createLibp2p } from 'libp2p' +import { webSockets } from '@libp2p/websockets' +import { noise } from '@chainsafe/libp2p-noise' +import { mplex } from '@libp2p/mplex' +import { MemoryBlockstore } from 'blockstore-core/memory' +import { fromString } from 'uint8arrays' +import * as raw from 'multiformats/codecs/raw' +import * as dagPB from '@ipld/dag-pb' +import { UnixFS } from 'ipfs-unixfs' +import { sha256 } from 'multiformats/hashes/sha2' +import { CID } from 'multiformats/cid' +import * as Block from 'multiformats/block' +import { Miniswap, BITSWAP_PROTOCOL } from 'miniswap' +import { Dagula } from '../index.js' +import { getLibp2p } from '../p2p.js' + +test('should getPath', async t => { + // should return all blocks in path and all blocks for resolved target of path + const serverBlockstore = new MemoryBlockstore() + const filePart1 = await Block.decode({ codec: raw, bytes: fromString(`MORE TEST DATA ${Date.now()}`), hasher: sha256 }) + const filePart2 = await Block.decode({ codec: raw, bytes: fromString(`EVEN MORE TEST DATA ${Date.now()}`), hasher: sha256 }) + const fileNode = await Block.encode({ + codec: dagPB, + hasher: sha256, + value: { + Data: new UnixFS({ type: 'file' }).marshal(), + Links: [ + { Name: '0', Hash: filePart1.cid }, + { Name: '1', Hash: filePart2.cid } + ] + } + }) + + const dirNode = await Block.encode({ + codec: dagPB, + hasher: sha256, + value: { + Data: new UnixFS({ type: 'directory' }).marshal(), + Links: [ + { Name: 'foo', Hash: fileNode.cid }, + { Name: 'other', Hash: CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') } + ] + } + }) + + for (const { cid, bytes } of [filePart1, filePart2, fileNode, dirNode]) { + await serverBlockstore.put(cid, bytes) + } + + const server = await createLibp2p({ + addresses: { listen: ['/ip4/127.0.0.1/tcp/0/ws'] }, + transports: [webSockets()], + streamMuxers: [mplex()], + connectionEncryption: [noise()] + }) + + const miniswap = new Miniswap(serverBlockstore) + server.handle(BITSWAP_PROTOCOL, miniswap.handler) + + await server.start() + + const libp2p = await getLibp2p() + const dagula = await Dagula.fromNetwork(libp2p, { peer: server.getMultiaddrs()[0] }) + const entries = [] + for await (const entry of dagula.getPath(`${dirNode.cid}/foo`)) { + entries.push(entry) + } + // did not try and return block for `other` + t.is(entries.length, 4) + t.deepEqual(entries.at(0).cid, dirNode.cid) + t.deepEqual(entries.at(0).bytes, dirNode.bytes) + t.deepEqual(entries.at(1).cid, fileNode.cid) + t.deepEqual(entries.at(1).bytes, fileNode.bytes) + t.deepEqual(entries.at(2).cid, filePart1.cid) + t.deepEqual(entries.at(2).bytes, filePart1.bytes) + t.deepEqual(entries.at(3).cid, filePart2.cid) + t.deepEqual(entries.at(3).bytes, filePart2.bytes) +}) + +test('should getPath on file with carScope=file', async t => { + // return all blocks in path and all blocks for resolved target of path + const serverBlockstore = new MemoryBlockstore() + const filePart1 = await Block.decode({ codec: raw, bytes: fromString(`MORE TEST DATA ${Date.now()}`), hasher: sha256 }) + const filePart2 = await Block.decode({ codec: raw, bytes: fromString(`EVEN MORE TEST DATA ${Date.now()}`), hasher: sha256 }) + const fileNode = await Block.encode({ + codec: dagPB, + hasher: sha256, + value: { + Data: new UnixFS({ type: 'file' }).marshal(), + Links: [ + { Name: '0', Hash: filePart1.cid }, + { Name: '1', Hash: filePart2.cid } + ] + } + }) + + const dirNode = await Block.encode({ + codec: dagPB, + hasher: sha256, + value: { + Data: new UnixFS({ type: 'directory' }).marshal(), + Links: [ + { Name: 'foo', Hash: fileNode.cid }, + { Name: 'other', Hash: CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') } + ] + } + }) + + for (const { cid, bytes } of [filePart1, filePart2, fileNode, dirNode]) { + await serverBlockstore.put(cid, bytes) + } + + const server = await createLibp2p({ + addresses: { listen: ['/ip4/127.0.0.1/tcp/0/ws'] }, + transports: [webSockets()], + streamMuxers: [mplex()], + connectionEncryption: [noise()] + }) + + const miniswap = new Miniswap(serverBlockstore) + server.handle(BITSWAP_PROTOCOL, miniswap.handler) + + await server.start() + + const libp2p = await getLibp2p() + const dagula = await Dagula.fromNetwork(libp2p, { peer: server.getMultiaddrs()[0] }) + const entries = [] + const carScope = 'file' + for await (const entry of dagula.getPath(`${dirNode.cid}/foo`, { carScope })) { + entries.push(entry) + } + // did not try and return block for `other` + t.is(entries.length, 4) + t.deepEqual(entries.at(0).cid, dirNode.cid) + t.deepEqual(entries.at(0).bytes, dirNode.bytes) + t.deepEqual(entries.at(1).cid, fileNode.cid) + t.deepEqual(entries.at(1).bytes, fileNode.bytes) + t.deepEqual(entries.at(2).cid, filePart1.cid) + t.deepEqual(entries.at(2).bytes, filePart1.bytes) + t.deepEqual(entries.at(3).cid, filePart2.cid) + t.deepEqual(entries.at(3).bytes, filePart2.bytes) +}) + +test('should getPath on file with carScope=block', async t => { + // return all blocks in path and all blocks for resolved target of path + const serverBlockstore = new MemoryBlockstore() + const filePart1 = await Block.decode({ codec: raw, bytes: fromString(`MORE TEST DATA ${Date.now()}`), hasher: sha256 }) + const filePart2 = await Block.decode({ codec: raw, bytes: fromString(`EVEN MORE TEST DATA ${Date.now()}`), hasher: sha256 }) + const fileNode = await Block.encode({ + codec: dagPB, + hasher: sha256, + value: { + Data: new UnixFS({ type: 'file' }).marshal(), + Links: [ + { Name: '0', Hash: filePart1.cid }, + { Name: '1', Hash: filePart2.cid } + ] + } + }) + + const dirNode = await Block.encode({ + codec: dagPB, + hasher: sha256, + value: { + Data: new UnixFS({ type: 'directory' }).marshal(), + Links: [ + { Name: 'foo', Hash: fileNode.cid }, + { Name: 'other', Hash: CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') } + ] + } + }) + + for (const { cid, bytes } of [filePart1, filePart2, fileNode, dirNode]) { + await serverBlockstore.put(cid, bytes) + } + + const server = await createLibp2p({ + addresses: { listen: ['/ip4/127.0.0.1/tcp/0/ws'] }, + transports: [webSockets()], + streamMuxers: [mplex()], + connectionEncryption: [noise()] + }) + + const miniswap = new Miniswap(serverBlockstore) + server.handle(BITSWAP_PROTOCOL, miniswap.handler) + + await server.start() + + const libp2p = await getLibp2p() + const dagula = await Dagula.fromNetwork(libp2p, { peer: server.getMultiaddrs()[0] }) + const entries = [] + const carScope = 'block' + for await (const entry of dagula.getPath(`${dirNode.cid}/foo`, { carScope })) { + entries.push(entry) + } + // did not try and return block for `other` + t.is(entries.length, 2) + t.deepEqual(entries.at(0).cid, dirNode.cid) + t.deepEqual(entries.at(0).bytes, dirNode.bytes) + t.deepEqual(entries.at(1).cid, fileNode.cid) + t.deepEqual(entries.at(1).bytes, fileNode.bytes) +}) + +test('should getPath on dir with carScope=file', async t => { + // return all blocks in path. as it's a dir, it should stop there + const serverBlockstore = new MemoryBlockstore() + const file = await Block.decode({ codec: raw, bytes: fromString(`MORE TEST DATA ${Date.now()}`), hasher: sha256 }) + + const dirNode = await Block.encode({ + codec: dagPB, + hasher: sha256, + value: { + Data: new UnixFS({ type: 'directory' }).marshal(), + Links: [ + { Name: 'foo', Hash: file.cid }, + { Name: 'other', Hash: CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') } + ] + } + }) + + for (const { cid, bytes } of [file, dirNode]) { + await serverBlockstore.put(cid, bytes) + } + + const server = await createLibp2p({ + addresses: { listen: ['/ip4/127.0.0.1/tcp/0/ws'] }, + transports: [webSockets()], + streamMuxers: [mplex()], + connectionEncryption: [noise()] + }) + + const miniswap = new Miniswap(serverBlockstore) + server.handle(BITSWAP_PROTOCOL, miniswap.handler) + + await server.start() + + const libp2p = await getLibp2p() + const dagula = await Dagula.fromNetwork(libp2p, { peer: server.getMultiaddrs()[0] }) + const entries = [] + for await (const entry of dagula.getPath(`${dirNode.cid}`, { carScope: 'file' })) { + entries.push(entry) + } + // only return the dir if carScope=file and target is a dir + t.is(entries.length, 1) + t.deepEqual(entries.at(0).cid, dirNode.cid) + t.deepEqual(entries.at(0).bytes, dirNode.bytes) +}) diff --git a/test.js b/test/index.test.js similarity index 93% rename from test.js rename to test/index.test.js index 4d3032e..51946ff 100644 --- a/test.js +++ b/test/index.test.js @@ -12,8 +12,8 @@ import { sha256 } from 'multiformats/hashes/sha2' import { CID } from 'multiformats/cid' import { Miniswap, BITSWAP_PROTOCOL } from 'miniswap' import { TimeoutController } from 'timeout-abort-controller' -import { Dagula } from './index.js' -import { getLibp2p } from './p2p.js' +import { Dagula } from '../index.js' +import { getLibp2p } from '../p2p.js' test('should fetch a single CID', async t => { // create blockstore and add data @@ -74,8 +74,11 @@ test('should walk a unixfs path', async t => { for await (const entry of dagula.walkUnixfsPath(`${dirCid}/${linkName}`)) { entries.push(entry) } - t.is(entries.at(0).cid.toString(), dirCid.toString()) - t.is(entries.at(1).cid.toString(), cid.toString()) + t.is(entries.length, 2) + t.deepEqual(entries.at(0).cid, dirCid) + t.deepEqual(entries.at(0).bytes, dirBytes) + t.deepEqual(entries.at(1).cid, cid) + t.deepEqual(entries.at(1).bytes, data) }) test('should abort a fetch', async t => { From b7898eeac51709abe058d11869a94987ccf8424c Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Mon, 17 Apr 2023 17:20:16 +0100 Subject: [PATCH 02/10] chore: fix doc License: MIT Signed-off-by: Oli Evans --- index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 30f1bb0..571eb8e 100644 --- a/index.js +++ b/index.js @@ -126,12 +126,12 @@ export class Dagula { } // non-files, like directories, and IPLD Maps only return blocks necessary for their enumeration if (carScope === 'file' && base.type === 'directory') { - // the single block for the root has already been yielded. For a hamt we must fetch all the blocks of the (current) hamt. + // the single block for the root has already been yielded. + // For a hamt we must fetch all the blocks of the (current) hamt. if (base.unixfs.type === 'hamt-sharded-directory') { - // TODO: how to determine the boudary of a hamt + // TODO: how to determine the boundary of a hamt throw new Error('hamt-sharded-directory is unsupported') } - // otherwise a dir is a single block, so we're done. } } From 3b758416cc2fb36b506dbca8bb3bc0023b3eac0b Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Tue, 18 Apr 2023 11:52:44 +0100 Subject: [PATCH 03/10] chore: simplify tests License: MIT Signed-off-by: Oli Evans --- test/_libp2p.js | 33 +++++++++++++++++ test/getPath.test.js | 84 ++++++-------------------------------------- test/index.test.js | 71 +++++++------------------------------ 3 files changed, 56 insertions(+), 132 deletions(-) create mode 100644 test/_libp2p.js diff --git a/test/_libp2p.js b/test/_libp2p.js new file mode 100644 index 0000000..ba7603e --- /dev/null +++ b/test/_libp2p.js @@ -0,0 +1,33 @@ +import { createLibp2p } from 'libp2p' +import { webSockets } from '@libp2p/websockets' +import { noise } from '@chainsafe/libp2p-noise' +import { mplex } from '@libp2p/mplex' +import { MemoryBlockstore } from 'blockstore-core/memory' +import { Miniswap, BITSWAP_PROTOCOL } from 'miniswap' + +/** + * @param {import('..').Block[]} + */ +export async function startBitswapPeer (blocks = []) { + const libp2p = await createLibp2p({ + addresses: { listen: ['/ip4/127.0.0.1/tcp/0/ws'] }, + transports: [webSockets()], + streamMuxers: [mplex()], + connectionEncryption: [noise()] + }) + + const bs = new MemoryBlockstore() + for (const { cid, bytes } of blocks) { + bs.put(cid, bytes) + } + + const miniswap = new Miniswap(bs) + libp2p.handle(BITSWAP_PROTOCOL, miniswap.handler) + + await libp2p.start() + + return { + libp2p, + bs + } +} diff --git a/test/getPath.test.js b/test/getPath.test.js index ac485e4..a72ebda 100644 --- a/test/getPath.test.js +++ b/test/getPath.test.js @@ -1,9 +1,4 @@ import test from 'ava' -import { createLibp2p } from 'libp2p' -import { webSockets } from '@libp2p/websockets' -import { noise } from '@chainsafe/libp2p-noise' -import { mplex } from '@libp2p/mplex' -import { MemoryBlockstore } from 'blockstore-core/memory' import { fromString } from 'uint8arrays' import * as raw from 'multiformats/codecs/raw' import * as dagPB from '@ipld/dag-pb' @@ -11,13 +6,12 @@ import { UnixFS } from 'ipfs-unixfs' import { sha256 } from 'multiformats/hashes/sha2' import { CID } from 'multiformats/cid' import * as Block from 'multiformats/block' -import { Miniswap, BITSWAP_PROTOCOL } from 'miniswap' import { Dagula } from '../index.js' import { getLibp2p } from '../p2p.js' +import { startBitswapPeer } from './_libp2p.js' test('should getPath', async t => { // should return all blocks in path and all blocks for resolved target of path - const serverBlockstore = new MemoryBlockstore() const filePart1 = await Block.decode({ codec: raw, bytes: fromString(`MORE TEST DATA ${Date.now()}`), hasher: sha256 }) const filePart2 = await Block.decode({ codec: raw, bytes: fromString(`EVEN MORE TEST DATA ${Date.now()}`), hasher: sha256 }) const fileNode = await Block.encode({ @@ -44,24 +38,10 @@ test('should getPath', async t => { } }) - for (const { cid, bytes } of [filePart1, filePart2, fileNode, dirNode]) { - await serverBlockstore.put(cid, bytes) - } - - const server = await createLibp2p({ - addresses: { listen: ['/ip4/127.0.0.1/tcp/0/ws'] }, - transports: [webSockets()], - streamMuxers: [mplex()], - connectionEncryption: [noise()] - }) - - const miniswap = new Miniswap(serverBlockstore) - server.handle(BITSWAP_PROTOCOL, miniswap.handler) - - await server.start() + const peer = await startBitswapPeer([filePart1, filePart2, fileNode, dirNode]) const libp2p = await getLibp2p() - const dagula = await Dagula.fromNetwork(libp2p, { peer: server.getMultiaddrs()[0] }) + const dagula = await Dagula.fromNetwork(libp2p, { peer: peer.libp2p.getMultiaddrs()[0] }) const entries = [] for await (const entry of dagula.getPath(`${dirNode.cid}/foo`)) { entries.push(entry) @@ -80,7 +60,6 @@ test('should getPath', async t => { test('should getPath on file with carScope=file', async t => { // return all blocks in path and all blocks for resolved target of path - const serverBlockstore = new MemoryBlockstore() const filePart1 = await Block.decode({ codec: raw, bytes: fromString(`MORE TEST DATA ${Date.now()}`), hasher: sha256 }) const filePart2 = await Block.decode({ codec: raw, bytes: fromString(`EVEN MORE TEST DATA ${Date.now()}`), hasher: sha256 }) const fileNode = await Block.encode({ @@ -107,24 +86,11 @@ test('should getPath on file with carScope=file', async t => { } }) - for (const { cid, bytes } of [filePart1, filePart2, fileNode, dirNode]) { - await serverBlockstore.put(cid, bytes) - } - - const server = await createLibp2p({ - addresses: { listen: ['/ip4/127.0.0.1/tcp/0/ws'] }, - transports: [webSockets()], - streamMuxers: [mplex()], - connectionEncryption: [noise()] - }) - - const miniswap = new Miniswap(serverBlockstore) - server.handle(BITSWAP_PROTOCOL, miniswap.handler) - - await server.start() + const peer = await startBitswapPeer([filePart1, filePart2, fileNode, dirNode]) const libp2p = await getLibp2p() - const dagula = await Dagula.fromNetwork(libp2p, { peer: server.getMultiaddrs()[0] }) + const dagula = await Dagula.fromNetwork(libp2p, { peer: peer.libp2p.getMultiaddrs()[0] }) + const entries = [] const carScope = 'file' for await (const entry of dagula.getPath(`${dirNode.cid}/foo`, { carScope })) { @@ -144,7 +110,6 @@ test('should getPath on file with carScope=file', async t => { test('should getPath on file with carScope=block', async t => { // return all blocks in path and all blocks for resolved target of path - const serverBlockstore = new MemoryBlockstore() const filePart1 = await Block.decode({ codec: raw, bytes: fromString(`MORE TEST DATA ${Date.now()}`), hasher: sha256 }) const filePart2 = await Block.decode({ codec: raw, bytes: fromString(`EVEN MORE TEST DATA ${Date.now()}`), hasher: sha256 }) const fileNode = await Block.encode({ @@ -171,24 +136,10 @@ test('should getPath on file with carScope=block', async t => { } }) - for (const { cid, bytes } of [filePart1, filePart2, fileNode, dirNode]) { - await serverBlockstore.put(cid, bytes) - } - - const server = await createLibp2p({ - addresses: { listen: ['/ip4/127.0.0.1/tcp/0/ws'] }, - transports: [webSockets()], - streamMuxers: [mplex()], - connectionEncryption: [noise()] - }) - - const miniswap = new Miniswap(serverBlockstore) - server.handle(BITSWAP_PROTOCOL, miniswap.handler) - - await server.start() + const peer = await startBitswapPeer([filePart1, filePart2, fileNode, dirNode]) const libp2p = await getLibp2p() - const dagula = await Dagula.fromNetwork(libp2p, { peer: server.getMultiaddrs()[0] }) + const dagula = await Dagula.fromNetwork(libp2p, { peer: peer.libp2p.getMultiaddrs()[0] }) const entries = [] const carScope = 'block' for await (const entry of dagula.getPath(`${dirNode.cid}/foo`, { carScope })) { @@ -204,7 +155,6 @@ test('should getPath on file with carScope=block', async t => { test('should getPath on dir with carScope=file', async t => { // return all blocks in path. as it's a dir, it should stop there - const serverBlockstore = new MemoryBlockstore() const file = await Block.decode({ codec: raw, bytes: fromString(`MORE TEST DATA ${Date.now()}`), hasher: sha256 }) const dirNode = await Block.encode({ @@ -219,24 +169,10 @@ test('should getPath on dir with carScope=file', async t => { } }) - for (const { cid, bytes } of [file, dirNode]) { - await serverBlockstore.put(cid, bytes) - } - - const server = await createLibp2p({ - addresses: { listen: ['/ip4/127.0.0.1/tcp/0/ws'] }, - transports: [webSockets()], - streamMuxers: [mplex()], - connectionEncryption: [noise()] - }) - - const miniswap = new Miniswap(serverBlockstore) - server.handle(BITSWAP_PROTOCOL, miniswap.handler) - - await server.start() + const peer = await startBitswapPeer([file, dirNode]) const libp2p = await getLibp2p() - const dagula = await Dagula.fromNetwork(libp2p, { peer: server.getMultiaddrs()[0] }) + const dagula = await Dagula.fromNetwork(libp2p, { peer: peer.libp2p.getMultiaddrs()[0] }) const entries = [] for await (const entry of dagula.getPath(`${dirNode.cid}`, { carScope: 'file' })) { entries.push(entry) diff --git a/test/index.test.js b/test/index.test.js index 51946ff..04cdfa0 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -1,75 +1,41 @@ import test from 'ava' -import { createLibp2p } from 'libp2p' -import { webSockets } from '@libp2p/websockets' -import { noise } from '@chainsafe/libp2p-noise' -import { mplex } from '@libp2p/mplex' -import { MemoryBlockstore } from 'blockstore-core/memory' import { fromString, toString } from 'uint8arrays' import * as raw from 'multiformats/codecs/raw' import * as dagPB from '@ipld/dag-pb' import { UnixFS } from 'ipfs-unixfs' import { sha256 } from 'multiformats/hashes/sha2' import { CID } from 'multiformats/cid' -import { Miniswap, BITSWAP_PROTOCOL } from 'miniswap' import { TimeoutController } from 'timeout-abort-controller' import { Dagula } from '../index.js' import { getLibp2p } from '../p2p.js' +import { startBitswapPeer } from './_libp2p.js' test('should fetch a single CID', async t => { - // create blockstore and add data - const serverBlockstore = new MemoryBlockstore() - const data = fromString(`TEST DATA ${Date.now()}`) - const hash = await sha256.digest(data) + const bytes = fromString(`TEST DATA ${Date.now()}`) + const hash = await sha256.digest(bytes) const cid = CID.create(1, raw.code, hash) - await serverBlockstore.put(cid, data) - - const server = await createLibp2p({ - addresses: { listen: ['/ip4/127.0.0.1/tcp/0/ws'] }, - transports: [webSockets()], - streamMuxers: [mplex()], - connectionEncryption: [noise()] - }) - - const miniswap = new Miniswap(serverBlockstore) - server.handle(BITSWAP_PROTOCOL, miniswap.handler) - - await server.start() + const peer = await startBitswapPeer([{ cid, bytes }]) const libp2p = await getLibp2p() - const dagula = await Dagula.fromNetwork(libp2p, { peer: server.getMultiaddrs()[0] }) + const dagula = await Dagula.fromNetwork(libp2p, { peer: peer.libp2p.getMultiaddrs()[0] }) for await (const block of dagula.get(cid)) { t.is(block.cid.toString(), cid.toString()) - t.is(toString(block.bytes), toString(data)) + t.is(toString(block.bytes), toString(bytes)) } }) test('should walk a unixfs path', async t => { - // create blockstore and add data - const serverBlockstore = new MemoryBlockstore() - const data = fromString(`TEST DATA ${Date.now()}`) - const hash = await sha256.digest(data) + const bytes = fromString(`TEST DATA ${Date.now()}`) + const hash = await sha256.digest(bytes) const cid = CID.create(1, raw.code, hash) - await serverBlockstore.put(cid, data) const linkName = 'foo' const dirData = new UnixFS({ type: 'directory' }).marshal() const dirBytes = dagPB.encode(dagPB.prepare({ Data: dirData, Links: [{ Name: linkName, Hash: cid }] })) const dirCid = CID.create(1, dagPB.code, await sha256.digest(dirBytes)) - await serverBlockstore.put(dirCid, dirBytes) - - const server = await createLibp2p({ - addresses: { listen: ['/ip4/127.0.0.1/tcp/0/ws'] }, - transports: [webSockets()], - streamMuxers: [mplex()], - connectionEncryption: [noise()] - }) - - const miniswap = new Miniswap(serverBlockstore) - server.handle(BITSWAP_PROTOCOL, miniswap.handler) - - await server.start() + const peer = await startBitswapPeer([{ cid: dirCid, bytes: dirBytes }, { cid, bytes }]) const libp2p = await getLibp2p() - const dagula = await Dagula.fromNetwork(libp2p, { peer: server.getMultiaddrs()[0] }) + const dagula = await Dagula.fromNetwork(libp2p, { peer: peer.libp2p.getMultiaddrs()[0] }) const entries = [] for await (const entry of dagula.walkUnixfsPath(`${dirCid}/${linkName}`)) { entries.push(entry) @@ -78,24 +44,13 @@ test('should walk a unixfs path', async t => { t.deepEqual(entries.at(0).cid, dirCid) t.deepEqual(entries.at(0).bytes, dirBytes) t.deepEqual(entries.at(1).cid, cid) - t.deepEqual(entries.at(1).bytes, data) + t.deepEqual(entries.at(1).bytes, bytes) }) test('should abort a fetch', async t => { - const server = await createLibp2p({ - addresses: { listen: ['/ip4/127.0.0.1/tcp/0/ws'] }, - transports: [webSockets()], - streamMuxers: [mplex()], - connectionEncryption: [noise()] - }) - - const miniswap = new Miniswap(new MemoryBlockstore()) - server.handle(BITSWAP_PROTOCOL, miniswap.handler) - - await server.start() - + const peer = await startBitswapPeer() const libp2p = await getLibp2p() - const dagula = await Dagula.fromNetwork(libp2p, { peer: server.getMultiaddrs()[0] }) + const dagula = await Dagula.fromNetwork(libp2p, { peer: peer.libp2p.getMultiaddrs()[0] }) // not in the blockstore so will hang indefinitely const cid = 'bafkreig7tekltu2k2bci74rpbyrruft4e7nrepzo4z36ie4n2bado5ru74' const controller = new TimeoutController(1_000) From 2efdcf98904b68ff561e846401a5b6858c3c4427 Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Tue, 18 Apr 2023 12:04:20 +0100 Subject: [PATCH 04/10] chore: always use encode. remove uint8arrays dep License: MIT Signed-off-by: Oli Evans --- package-lock.json | 18 +----------------- package.json | 3 +-- test/getPath.test.js | 6 +++--- test/index.test.js | 2 +- 4 files changed, 6 insertions(+), 23 deletions(-) diff --git a/package-lock.json b/package-lock.json index abd3b93..562ee70 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,8 +44,7 @@ "blockstore-core": "^1.0.5", "ipfs-unixfs": "^11.0.0", "miniswap": "^2.0.0", - "standard": "^17.0.0", - "uint8arrays": "^3.0.0" + "standard": "^17.0.0" } }, "node_modules/@achingbrain/ip-address": { @@ -6669,21 +6668,6 @@ "npm": ">=7.0.0" } }, - "node_modules/uint8arrays": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-3.1.1.tgz", - "integrity": "sha512-+QJa8QRnbdXVpHYjLoTpJIdCTiw9Ir62nocClWuXIq2JIh4Uta0cQsTSpFL678p2CN8B+XSApwcU+pQEqVpKWg==", - "dev": true, - "dependencies": { - "multiformats": "^9.4.2" - } - }, - "node_modules/uint8arrays/node_modules/multiformats": { - "version": "9.9.0", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-9.9.0.tgz", - "integrity": "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==", - "dev": true - }, "node_modules/unbox-primitive": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", diff --git a/package.json b/package.json index b0fb063..6bf6d9b 100644 --- a/package.json +++ b/package.json @@ -49,8 +49,7 @@ "blockstore-core": "^1.0.5", "ipfs-unixfs": "^11.0.0", "miniswap": "^2.0.0", - "standard": "^17.0.0", - "uint8arrays": "^3.0.0" + "standard": "^17.0.0" }, "types": "./index.d.ts", "repository": { diff --git a/test/getPath.test.js b/test/getPath.test.js index a72ebda..6c37595 100644 --- a/test/getPath.test.js +++ b/test/getPath.test.js @@ -1,5 +1,5 @@ import test from 'ava' -import { fromString } from 'uint8arrays' +import { fromString } from 'multiformats/bytes' import * as raw from 'multiformats/codecs/raw' import * as dagPB from '@ipld/dag-pb' import { UnixFS } from 'ipfs-unixfs' @@ -12,8 +12,8 @@ import { startBitswapPeer } from './_libp2p.js' test('should getPath', async t => { // should return all blocks in path and all blocks for resolved target of path - const filePart1 = await Block.decode({ codec: raw, bytes: fromString(`MORE TEST DATA ${Date.now()}`), hasher: sha256 }) - const filePart2 = await Block.decode({ codec: raw, bytes: fromString(`EVEN MORE TEST DATA ${Date.now()}`), hasher: sha256 }) + const filePart1 = await Block.encode({ codec: raw, value: fromString(`MORE TEST DATA ${Date.now()}`), hasher: sha256 }) + const filePart2 = await Block.encode({ codec: raw, value: fromString(`EVEN MORE TEST DATA ${Date.now()}`), hasher: sha256 }) const fileNode = await Block.encode({ codec: dagPB, hasher: sha256, diff --git a/test/index.test.js b/test/index.test.js index 04cdfa0..e02ea02 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -1,5 +1,5 @@ import test from 'ava' -import { fromString, toString } from 'uint8arrays' +import { fromString, toString } from 'multiformats/bytes' import * as raw from 'multiformats/codecs/raw' import * as dagPB from '@ipld/dag-pb' import { UnixFS } from 'ipfs-unixfs' From 59c666b7ffe3bf4eee9c44acde9c9a7ed9534a1d Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Wed, 19 Apr 2023 20:28:07 +0100 Subject: [PATCH 05/10] feat: car-scope for hamt-sharded-directory License: MIT Signed-off-by: Oli Evans --- index.js | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/index.js b/index.js index 571eb8e..7f955c8 100644 --- a/index.js +++ b/index.js @@ -67,18 +67,21 @@ export class Dagula { /** * @param {CID[]|CID|string} cid - * @param {{ signal?: AbortSignal }} [options] + * @param {object} [options] + * @param {AbortSignal} [options.signal] + * @param {(block: import('multiformats').BlockView) => CID[]} [options.search] */ async * get (cid, options = {}) { cid = typeof cid === 'string' ? CID.parse(cid) : cid log('getting DAG %s', cid) let cids = Array.isArray(cid) ? cid : [cid] + const search = options.search || breadthFirstSearch() while (cids.length > 0) { log('fetching %d CIDs', cids.length) const fetchBlocks = transform(cids.length, async cid => { return this.getBlock(cid, { signal: options.signal }) }) - const nextCids = [] + let nextCids = [] for await (const { cid, bytes } of fetchBlocks(cids)) { const decoder = this.#decoders[cid.code] if (!decoder) { @@ -88,11 +91,8 @@ export class Dagula { log('decoding block %s', cid) const block = await Block.decode({ bytes, codec: decoder, hasher }) yield block - for (const [, cid] of block.links()) { - nextCids.push(cid) - } + nextCids = nextCids.concat(search(block)) } - if (!nextCids.length) break log('%d CIDs in links', nextCids.length) cids = nextCids } @@ -129,8 +129,7 @@ export class Dagula { // the single block for the root has already been yielded. // For a hamt we must fetch all the blocks of the (current) hamt. if (base.unixfs.type === 'hamt-sharded-directory') { - // TODO: how to determine the boundary of a hamt - throw new Error('hamt-sharded-directory is unsupported') + yield * this.get(base.cid, { search: hamtSearch, signal: options.signal }) } } } @@ -192,3 +191,23 @@ export class Dagula { } } } + +/** + * + */ +export function breadthFirstSearch (linkFilter = () => true) { + /** + * @param {import('multiformats').BlockView} block + */ + return function (block) { + const nextCids = [] + for (const link of block.links()) { + if (linkFilter(link)) { + nextCids.push(link[1]) + } + } + return nextCids + } +} + +export const hamtSearch = breadthFirstSearch(([name]) => name.length === 2) From 3acbe114277840e4352d15615babc73f9d157aaf Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Fri, 21 Apr 2023 16:38:07 +0100 Subject: [PATCH 06/10] chore: add test for hamt-sharded-dir License: MIT Signed-off-by: Oli Evans --- index.js | 7 ++++-- package-lock.json | 56 ++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + test/getPath.test.js | 52 +++++++++++++++++++++++++++++++++------- 4 files changed, 106 insertions(+), 10 deletions(-) diff --git a/index.js b/index.js index 7f955c8..5ceff94 100644 --- a/index.js +++ b/index.js @@ -118,8 +118,8 @@ export class Dagula { yield item } if (carScope === 'all' || (carScope === 'file' && base.type !== 'directory')) { - // fetch the entire dag rooted at the end of the provided path const links = base.node.Links?.map(l => l.Hash) || [] + // fetch the entire dag rooted at the end of the provided path if (links.length) { yield * this.get(links, { signal: options.signal }) } @@ -129,7 +129,10 @@ export class Dagula { // the single block for the root has already been yielded. // For a hamt we must fetch all the blocks of the (current) hamt. if (base.unixfs.type === 'hamt-sharded-directory') { - yield * this.get(base.cid, { search: hamtSearch, signal: options.signal }) + const hamtLinks = base.node.Links?.filter(l => l.Name.length === 2).map(l => l.Hash) || [] + if (hamtLinks.length) { + yield * this.get(hamtLinks, { search: hamtSearch, signal: options.signal }) + } } } } diff --git a/package-lock.json b/package-lock.json index 562ee70..fd4e179 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,6 +40,7 @@ "dagula": "bin.js" }, "devDependencies": { + "@ipld/unixfs": "^2.1.1", "ava": "^4.3.1", "blockstore-core": "^1.0.5", "ipfs-unixfs": "^11.0.0", @@ -288,6 +289,22 @@ "npm": ">=7.0.0" } }, + "node_modules/@ipld/unixfs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@ipld/unixfs/-/unixfs-2.1.1.tgz", + "integrity": "sha512-g3gr/3XvfQs4x2VFjlICae09ul5fbWCKRInN6Vgeot2+GH0h/krr3PqZCIo4dy4Ou2mQOsIddxUvG8UZ4p9SbQ==", + "dev": true, + "dependencies": { + "@ipld/dag-pb": "^4.0.0", + "@multiformats/murmur3": "^2.1.3", + "@perma/map": "^1.0.2", + "@web-std/stream": "1.0.1", + "actor": "^2.3.1", + "multiformats": "^11.0.1", + "protobufjs": "^7.1.2", + "rabin-rs": "^2.1.0" + } + }, "node_modules/@libp2p/crypto": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@libp2p/crypto/-/crypto-1.0.11.tgz", @@ -1099,6 +1116,15 @@ "node": ">= 8" } }, + "node_modules/@perma/map": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@perma/map/-/map-1.0.2.tgz", + "integrity": "sha512-hujwGOY6yTYnpf5YAtpD5MJAI1kcsVPqyN0lxG8Sampf/InO3jmX/MlJCHCGFPpPqB5JyO5WNnL+tUs1Umqe0A==", + "dev": true, + "dependencies": { + "murmurhash3js-revisited": "^3.0.0" + } + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -1295,6 +1321,15 @@ "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.1.tgz", "integrity": "sha512-xoDlM2S4ortawSWORYqsdU+2rxdh4LRW9ytc3zmT37RIKQh6IHyKwwtKhKis9ah8ol07DCkZxPt8BBvPjC6v4g==" }, + "node_modules/@web-std/stream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@web-std/stream/-/stream-1.0.1.tgz", + "integrity": "sha512-tsz4Y0WNDgFA5jwLSeV7/UV5rfMIlj0cPsSLVfTihjaVW0OJPd5NxJ3le1B3yLyqqzRpeG5OAfJAADLc4VoGTA==", + "dev": true, + "dependencies": { + "web-streams-polyfill": "^3.1.1" + } + }, "node_modules/abortable-iterator": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/abortable-iterator/-/abortable-iterator-4.0.2.tgz", @@ -1334,6 +1369,12 @@ "node": ">=0.4.0" } }, + "node_modules/actor": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/actor/-/actor-2.3.1.tgz", + "integrity": "sha512-ST/3wnvcP2tKDXnum7nLCLXm+/rsf8vPocXH2Fre6D8FQwNkGDd4JEitBlXj007VQJfiGYRQvXqwOBZVi+JtRg==", + "dev": true + }, "node_modules/aggregate-error": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-4.0.1.tgz", @@ -5789,6 +5830,12 @@ } ] }, + "node_modules/rabin-rs": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/rabin-rs/-/rabin-rs-2.1.0.tgz", + "integrity": "sha512-5y72gAXPzIBsAMHcpxZP8eMDuDT98qMP1BqSDHRbHkJJXEgWIN1lA47LxUqzsK6jknOJtgfkQr9v+7qMlFDm6g==", + "dev": true + }, "node_modules/rate-limiter-flexible": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/rate-limiter-flexible/-/rate-limiter-flexible-2.4.1.tgz", @@ -6720,6 +6767,15 @@ "resolved": "https://registry.npmjs.org/varint/-/varint-6.0.0.tgz", "integrity": "sha512-cXEIW6cfr15lFv563k4GuVuW/fiwjknytD37jIOLSdSWuOI6WnO/oKwmP2FQTU2l01LP8/M5TSAJpzUaGe3uWg==" }, + "node_modules/web-streams-polyfill": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", + "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, "node_modules/well-known-symbols": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz", diff --git a/package.json b/package.json index 6bf6d9b..170feb9 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "timeout-abort-controller": "^3.0.0" }, "devDependencies": { + "@ipld/unixfs": "^2.1.1", "ava": "^4.3.1", "blockstore-core": "^1.0.5", "ipfs-unixfs": "^11.0.0", diff --git a/test/getPath.test.js b/test/getPath.test.js index 6c37595..e796da2 100644 --- a/test/getPath.test.js +++ b/test/getPath.test.js @@ -2,7 +2,9 @@ import test from 'ava' import { fromString } from 'multiformats/bytes' import * as raw from 'multiformats/codecs/raw' import * as dagPB from '@ipld/dag-pb' -import { UnixFS } from 'ipfs-unixfs' +import { UnixFS as UnixFSv1 } from 'ipfs-unixfs' +import * as UnixFS from '@ipld/unixfs' +import { TransformStream } from 'node:stream/web' import { sha256 } from 'multiformats/hashes/sha2' import { CID } from 'multiformats/cid' import * as Block from 'multiformats/block' @@ -18,7 +20,7 @@ test('should getPath', async t => { codec: dagPB, hasher: sha256, value: { - Data: new UnixFS({ type: 'file' }).marshal(), + Data: new UnixFSv1({ type: 'file' }).marshal(), Links: [ { Name: '0', Hash: filePart1.cid }, { Name: '1', Hash: filePart2.cid } @@ -30,7 +32,7 @@ test('should getPath', async t => { codec: dagPB, hasher: sha256, value: { - Data: new UnixFS({ type: 'directory' }).marshal(), + Data: new UnixFSv1({ type: 'directory' }).marshal(), Links: [ { Name: 'foo', Hash: fileNode.cid }, { Name: 'other', Hash: CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') } @@ -66,7 +68,7 @@ test('should getPath on file with carScope=file', async t => { codec: dagPB, hasher: sha256, value: { - Data: new UnixFS({ type: 'file' }).marshal(), + Data: new UnixFSv1({ type: 'file' }).marshal(), Links: [ { Name: '0', Hash: filePart1.cid }, { Name: '1', Hash: filePart2.cid } @@ -78,7 +80,7 @@ test('should getPath on file with carScope=file', async t => { codec: dagPB, hasher: sha256, value: { - Data: new UnixFS({ type: 'directory' }).marshal(), + Data: new UnixFSv1({ type: 'directory' }).marshal(), Links: [ { Name: 'foo', Hash: fileNode.cid }, { Name: 'other', Hash: CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') } @@ -116,7 +118,7 @@ test('should getPath on file with carScope=block', async t => { codec: dagPB, hasher: sha256, value: { - Data: new UnixFS({ type: 'file' }).marshal(), + Data: new UnixFSv1({ type: 'file' }).marshal(), Links: [ { Name: '0', Hash: filePart1.cid }, { Name: '1', Hash: filePart2.cid } @@ -128,7 +130,7 @@ test('should getPath on file with carScope=block', async t => { codec: dagPB, hasher: sha256, value: { - Data: new UnixFS({ type: 'directory' }).marshal(), + Data: new UnixFSv1({ type: 'directory' }).marshal(), Links: [ { Name: 'foo', Hash: fileNode.cid }, { Name: 'other', Hash: CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') } @@ -161,7 +163,7 @@ test('should getPath on dir with carScope=file', async t => { codec: dagPB, hasher: sha256, value: { - Data: new UnixFS({ type: 'directory' }).marshal(), + Data: new UnixFSv1({ type: 'directory' }).marshal(), Links: [ { Name: 'foo', Hash: file.cid }, { Name: 'other', Hash: CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') } @@ -182,3 +184,37 @@ test('should getPath on dir with carScope=file', async t => { t.deepEqual(entries.at(0).cid, dirNode.cid) t.deepEqual(entries.at(0).bytes, dirNode.bytes) }) + +test('should getPath on hamt sharded dir with carScope=file', async t => { + const { readable, writable } = new TransformStream(undefined, UnixFS.withCapacity(1048576 * 32)) + const writer = writable.getWriter() + + const file = UnixFS.createFileWriter({ writer }) + file.write(new TextEncoder().encode('HELP')) + const fileLink = await file.close() + + const dir = UnixFS.createShardedDirectoryWriter({ writer }) + dir.set('foo', fileLink) + const dirLink = await dir.close() + + writer.close() + + const blocks = [] + for await (const block of readable) { + blocks.push(block) + console.log(block.cid, block.value?.entries) + } + + const peer = await startBitswapPeer(blocks) + + const libp2p = await getLibp2p() + const dagula = await Dagula.fromNetwork(libp2p, { peer: peer.libp2p.getMultiaddrs()[0] }) + const entries = [] + for await (const entry of dagula.getPath(`${dirLink.cid}`, { carScope: 'file' })) { + entries.push(entry) + } + + // only return the dir if carScope=file and target is a dir + t.is(entries.length, 1) + t.deepEqual(entries.at(0).cid, dirLink.cid) +}) From 2769c69f7e6601a69f7240f36fc309c7dffa77bc Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Fri, 28 Apr 2023 15:43:04 +0100 Subject: [PATCH 07/10] chore: fix hamt traversal License: MIT Signed-off-by: Oli Evans --- index.d.ts | 4 +- index.js | 66 ++++++-- package-lock.json | 368 ++++++++++++++++++------------------------- package.json | 2 +- test/getPath.test.js | 87 +++++++++- test/index.test.js | 2 - 6 files changed, 293 insertions(+), 236 deletions(-) diff --git a/index.d.ts b/index.d.ts index 9a3ae7e..04d1f49 100644 --- a/index.d.ts +++ b/index.d.ts @@ -56,7 +56,7 @@ export interface IDagula { /** * Emit nodes for all path segements and get UnixFS files and directories */ - walkUnixfsPath (path: CID|string, options?: AbortOptions): AsyncIterableIterator + walkUnixfsPath (path: CID|string, options?: AbortOptions): AsyncIterableIterator } export declare class Dagula implements IDagula { @@ -80,5 +80,5 @@ export declare class Dagula implements IDagula { /** * Emit nodes for all path segements and get UnixFS files and directories */ - walkUnixfsPath (path: CID|string, options?: AbortOptions): AsyncIterableIterator + walkUnixfsPath (path: CID|string, options?: AbortOptions): AsyncIterableIterator } diff --git a/index.js b/index.js index 2492401..4be0f44 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,6 @@ import debug from 'debug' -import * as dagPB from '@ipld/dag-pb' import { CID } from 'multiformats/cid' +import * as dagPB from '@ipld/dag-pb' import * as Block from 'multiformats/block' import { exporter, walkPath } from 'ipfs-unixfs-exporter' import { transform } from 'streaming-iterables' @@ -84,6 +84,10 @@ export class Dagula { } /** + * Yield all blocks traversed to resovlve the ipfs path + * then use carScope to determine the set of blocks of the targeted dag to yield. + * Yield all blocks by default. Use carSope: 'block' to limit it to just the termimal block. + * * @param {string} cidPath * @param {object} [options] * @param {AbortSignal} [options.signal] @@ -96,12 +100,40 @@ export class Dagula { */ async * getPath (cidPath, options = {}) { const carScope = options.carScope ?? 'all' - /** @type {import('ipfs-unixfs-exporter').UnixFSEntry} */ + + /** + * The resolved dag root at the terminus of the cidPath + * @type {import('ipfs-unixfs-exporter').UnixFSEntry} + * */ let base - for await (const item of this.walkUnixfsPath(cidPath, { signal: options.signal })) { + + /** + * cache of blocks required to resove the cidPath + * @type {import('./index').Block[]} */ + let traversed = [] + + /** + * Adapter for unixfs-exporter to track the blocks it loads as it resolves the path. + * `walkPath` emits a single unixfs entry for multibock structures, but we need the individual blocks. + * TODO: port logic to @web3-storage/ipfs-path to make this less ugly. + */ + const mockstore = { + /** + * @param {CID} cid + * @param {{ signal?: AbortSignal }} [options] + */ + get: async (cid, options) => { + const block = await this.getBlock(cid, options) + traversed.push(block) + return block.bytes + } + } + for await (const item of walkPath(cidPath, mockstore, { signal: options.signal })) { base = item - yield item + yield * traversed + traversed = [] } + if (carScope === 'all' || (carScope === 'file' && base.type !== 'directory')) { const links = base.node.Links?.map(l => l.Hash) || [] // fetch the entire dag rooted at the end of the provided path @@ -172,16 +204,15 @@ export class Dagula { return block.bytes } } - for await (const entry of walkPath(cidPath, blockstore, { signal: options.signal })) { - /** @type {Uint8Array} */ - const bytes = entry.node.Links ? dagPB.encode(entry.node) : entry.node - yield { ...entry, bytes } - } + yield * walkPath(cidPath, blockstore, { signal: options.signal }) } } /** - * + * Create a search function that given a decoded Block + * will return an array of CIDs to fetch next. + * + * @param {([name, cid]: [string, Link]) => boolean} linkFilter */ export function breadthFirstSearch (linkFilter = () => true) { /** @@ -189,9 +220,18 @@ export function breadthFirstSearch (linkFilter = () => true) { */ return function (block) { const nextCids = [] - for (const link of block.links()) { - if (linkFilter(link)) { - nextCids.push(link[1]) + if (block.cid.code === dagPB.code) { + for (const { Name, Hash } of block.value.Links) { + if (linkFilter([Name, Hash])) { + nextCids.push(Hash) + } + } + } else { + // links() paths dagPb in the ipld style so name is e.g `Links/0/Hash`, and not what we want here. + for (const link of block.links()) { + if (linkFilter(link)) { + nextCids.push(link[1]) + } } } return nextCids diff --git a/package-lock.json b/package-lock.json index e9dcc34..15da0d7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,7 +43,7 @@ }, "devDependencies": { "@ipld/unixfs": "^2.1.1", - "ava": "^4.3.1", + "ava": "^5.2.0", "blockstore-core": "^1.0.5", "ipfs-unixfs": "^11.0.0", "miniswap": "^2.0.1", @@ -1411,15 +1411,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/array-union": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", - "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/array.prototype.flat": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.1.tgz", @@ -1500,22 +1491,22 @@ } }, "node_modules/ava": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/ava/-/ava-4.3.3.tgz", - "integrity": "sha512-9Egq/d9R74ExrWohHeqUlexjDbgZJX5jA1Wq4KCTqc3wIfpGEK79zVy4rBtofJ9YKIxs4PzhJ8BgbW5PlAYe6w==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ava/-/ava-5.2.0.tgz", + "integrity": "sha512-W8yxFXJr/P68JP55eMpQIa6AiXhCX3VeuajM8nolyWNExcMDD6rnIWKTjw0B/+GkFHBIaN6Jd0LtcMThcoqVfg==", "dev": true, "dependencies": { - "acorn": "^8.7.1", + "acorn": "^8.8.1", "acorn-walk": "^8.2.0", - "ansi-styles": "^6.1.0", + "ansi-styles": "^6.2.1", "arrgv": "^1.0.2", "arrify": "^3.0.0", "callsites": "^4.0.0", "cbor": "^8.1.0", - "chalk": "^5.0.1", + "chalk": "^5.2.0", "chokidar": "^3.5.3", "chunkd": "^2.0.1", - "ci-info": "^3.3.1", + "ci-info": "^3.7.1", "ci-parallel-vars": "^1.0.1", "clean-yaml-object": "^0.1.0", "cli-truncate": "^3.1.0", @@ -1524,10 +1515,10 @@ "concordance": "^5.0.4", "currently-unhandled": "^0.4.1", "debug": "^4.3.4", - "del": "^6.1.1", - "emittery": "^0.11.0", - "figures": "^4.0.1", - "globby": "^13.1.1", + "del": "^7.0.0", + "emittery": "^1.0.1", + "figures": "^5.0.0", + "globby": "^13.1.3", "ignore-by-default": "^2.1.0", "indent-string": "^5.0.0", "is-error": "^2.2.2", @@ -1537,25 +1528,25 @@ "mem": "^9.0.2", "ms": "^2.1.3", "p-event": "^5.0.1", - "p-map": "^5.4.0", + "p-map": "^5.5.0", "picomatch": "^2.3.1", "pkg-conf": "^4.0.0", "plur": "^5.1.0", - "pretty-ms": "^7.0.1", + "pretty-ms": "^8.0.0", "resolve-cwd": "^3.0.0", "slash": "^3.0.0", - "stack-utils": "^2.0.5", + "stack-utils": "^2.0.6", "strip-ansi": "^7.0.1", "supertap": "^3.0.1", - "temp-dir": "^2.0.0", - "write-file-atomic": "^4.0.1", - "yargs": "^17.5.1" + "temp-dir": "^3.0.0", + "write-file-atomic": "^5.0.0", + "yargs": "^17.6.2" }, "bin": { "ava": "entrypoints/cli.mjs" }, "engines": { - "node": ">=12.22 <13 || >=14.17 <15 || >=16.4 <17 || >=18" + "node": ">=14.19 <15 || >=16.15 <17 || >=18" }, "peerDependencies": { "@ava/typescript": "*" @@ -2160,88 +2151,46 @@ } }, "node_modules/del": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/del/-/del-6.1.1.tgz", - "integrity": "sha512-ua8BhapfP0JUJKC/zV9yHHDW/rDoDxP4Zhn3AkA6/xT6gY7jYXJiaeyBZznYVujhZZET+UgcbZiQ7sN3WqcImg==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/del/-/del-7.0.0.tgz", + "integrity": "sha512-tQbV/4u5WVB8HMJr08pgw0b6nG4RGt/tj+7Numvq+zqcvUFeMaIWWOUFltiU+6go8BSO2/ogsB4EasDaj0y68Q==", "dev": true, "dependencies": { - "globby": "^11.0.1", - "graceful-fs": "^4.2.4", - "is-glob": "^4.0.1", - "is-path-cwd": "^2.2.0", - "is-path-inside": "^3.0.2", - "p-map": "^4.0.0", + "globby": "^13.1.2", + "graceful-fs": "^4.2.10", + "is-glob": "^4.0.3", + "is-path-cwd": "^3.0.0", + "is-path-inside": "^4.0.0", + "p-map": "^5.5.0", "rimraf": "^3.0.2", - "slash": "^3.0.0" + "slash": "^4.0.0" }, "engines": { - "node": ">=10" + "node": ">=14.16" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/del/node_modules/aggregate-error": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", - "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", - "dev": true, - "dependencies": { - "clean-stack": "^2.0.0", - "indent-string": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/del/node_modules/clean-stack": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", - "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/del/node_modules/globby": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", - "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", + "node_modules/del/node_modules/is-path-inside": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-4.0.0.tgz", + "integrity": "sha512-lJJV/5dYS+RcL8uQdBDW9c9uWFLLBNRyFhnAKXw5tVqLlKZ4RMGZKv+YQ/IA3OhD+RpbJa1LLFM1FQPGyIXvOA==", "dev": true, - "dependencies": { - "array-union": "^2.1.0", - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.9", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^3.0.0" - }, "engines": { - "node": ">=10" + "node": ">=12" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/del/node_modules/indent-string": { + "node_modules/del/node_modules/slash": { "version": "4.0.0", - "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", - "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/del/node_modules/p-map": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", - "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", + "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", "dev": true, - "dependencies": { - "aggregate-error": "^3.0.0" - }, "engines": { - "node": ">=10" + "node": ">=12" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -2307,12 +2256,12 @@ "dev": true }, "node_modules/emittery": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.11.0.tgz", - "integrity": "sha512-S/7tzL6v5i+4iJd627Nhv9cLFIo5weAIlGccqJFpnBoDB8U1TF2k5tez4J/QNuxyyhWuFqHg1L84Kd3m7iXg6g==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/emittery/-/emittery-1.0.1.tgz", + "integrity": "sha512-2ID6FdrMD9KDLldGesP6317G78K7km/kMcwItRtVFva7I/cSEOIaLpewaUb+YLXVwdAp3Ctfxh/V5zIl1sj7dQ==", "dev": true, "engines": { - "node": ">=12" + "node": ">=14.16" }, "funding": { "url": "https://github.com/sindresorhus/emittery?sponsor=1" @@ -3210,16 +3159,16 @@ } }, "node_modules/figures": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/figures/-/figures-4.0.1.tgz", - "integrity": "sha512-rElJwkA/xS04Vfg+CaZodpso7VqBknOYbzi6I76hI4X80RUjkSxO2oAyPmGbuXUppywjqndOrQDl817hDnI++w==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-5.0.0.tgz", + "integrity": "sha512-ej8ksPF4x6e5wvK9yevct0UCXh8TTFlWGVLlgjZuoBH1HwjIfKE/IdL5mq89sFA7zELi1VhKpmtDnrs7zWyeyg==", "dev": true, "dependencies": { "escape-string-regexp": "^5.0.0", "is-unicode-supported": "^1.2.0" }, "engines": { - "node": ">=12" + "node": ">=14" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -4098,12 +4047,15 @@ } }, "node_modules/is-path-cwd": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-2.2.0.tgz", - "integrity": "sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-3.0.0.tgz", + "integrity": "sha512-kyiNFFLU0Ampr6SDZitD/DwUo4Zs1nSdnygUBqsu3LooL00Qvb5j+UnvApUn/TTj1J3OuE6BTdQ5rudKmU2ZaA==", "dev": true, "engines": { - "node": ">=6" + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/is-path-inside": { @@ -5493,12 +5445,15 @@ } }, "node_modules/parse-ms": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", - "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-3.0.0.tgz", + "integrity": "sha512-Tpb8Z7r7XbbtBTrM9UhpkzzaMrqA2VXMT3YChzYltwV3P3pM6t8wl7TvpMnSTosz1aQAdVib7kdoys7vYOPerw==", "dev": true, "engines": { - "node": ">=6" + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/path-exists": { @@ -5609,15 +5564,15 @@ } }, "node_modules/pretty-ms": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-7.0.1.tgz", - "integrity": "sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-8.0.0.tgz", + "integrity": "sha512-ASJqOugUF1bbzI35STMBUpZqdfYKlJugy6JBziGi2EE+AL5JPJGSzvpeVXojxrr0ViUYoToUjb5kjSEGf7Y83Q==", "dev": true, "dependencies": { - "parse-ms": "^2.1.0" + "parse-ms": "^3.0.0" }, "engines": { - "node": ">=10" + "node": ">=14.16" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -6449,12 +6404,12 @@ } }, "node_modules/temp-dir": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-2.0.0.tgz", - "integrity": "sha512-aoBAniQmmwtcKp/7BzsH8Cxzv8OL736p7v1ihGb5e9DJ9kTwGWHrQrVB5+lfVDzfGrdRzXch+ig7LHaY1JTOrg==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-3.0.0.tgz", + "integrity": "sha512-nHc6S/bwIilKHNRgK/3jlhDoIHcp45YgyiwcAk46Tr0LfEqGBVpmiAyuiuxeVE44m3mXnEeVhaipLOEWmH+Njw==", "dev": true, "engines": { - "node": ">=8" + "node": ">=14.16" } }, "node_modules/text-table": { @@ -6823,16 +6778,28 @@ "dev": true }, "node_modules/write-file-atomic": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", - "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-5.0.1.tgz", + "integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==", "dev": true, "dependencies": { "imurmurhash": "^0.1.4", - "signal-exit": "^3.0.7" + "signal-exit": "^4.0.1" }, "engines": { - "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/write-file-atomic/node_modules/signal-exit": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.0.1.tgz", + "integrity": "sha512-uUWsN4aOxJAS8KOuf3QMyFtgm1pkb6I+KRZbRF/ghdf5T7sM+B1lLLzPDxswUjkmHyxQAVzEgG35E3NzDM9GVw==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/ws": { @@ -8065,12 +8032,6 @@ "is-string": "^1.0.7" } }, - "array-union": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", - "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", - "dev": true - }, "array.prototype.flat": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.1.tgz", @@ -8130,22 +8091,22 @@ } }, "ava": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/ava/-/ava-4.3.3.tgz", - "integrity": "sha512-9Egq/d9R74ExrWohHeqUlexjDbgZJX5jA1Wq4KCTqc3wIfpGEK79zVy4rBtofJ9YKIxs4PzhJ8BgbW5PlAYe6w==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ava/-/ava-5.2.0.tgz", + "integrity": "sha512-W8yxFXJr/P68JP55eMpQIa6AiXhCX3VeuajM8nolyWNExcMDD6rnIWKTjw0B/+GkFHBIaN6Jd0LtcMThcoqVfg==", "dev": true, "requires": { - "acorn": "^8.7.1", + "acorn": "^8.8.1", "acorn-walk": "^8.2.0", - "ansi-styles": "^6.1.0", + "ansi-styles": "^6.2.1", "arrgv": "^1.0.2", "arrify": "^3.0.0", "callsites": "^4.0.0", "cbor": "^8.1.0", - "chalk": "^5.0.1", + "chalk": "^5.2.0", "chokidar": "^3.5.3", "chunkd": "^2.0.1", - "ci-info": "^3.3.1", + "ci-info": "^3.7.1", "ci-parallel-vars": "^1.0.1", "clean-yaml-object": "^0.1.0", "cli-truncate": "^3.1.0", @@ -8154,10 +8115,10 @@ "concordance": "^5.0.4", "currently-unhandled": "^0.4.1", "debug": "^4.3.4", - "del": "^6.1.1", - "emittery": "^0.11.0", - "figures": "^4.0.1", - "globby": "^13.1.1", + "del": "^7.0.0", + "emittery": "^1.0.1", + "figures": "^5.0.0", + "globby": "^13.1.3", "ignore-by-default": "^2.1.0", "indent-string": "^5.0.0", "is-error": "^2.2.2", @@ -8167,19 +8128,19 @@ "mem": "^9.0.2", "ms": "^2.1.3", "p-event": "^5.0.1", - "p-map": "^5.4.0", + "p-map": "^5.5.0", "picomatch": "^2.3.1", "pkg-conf": "^4.0.0", "plur": "^5.1.0", - "pretty-ms": "^7.0.1", + "pretty-ms": "^8.0.0", "resolve-cwd": "^3.0.0", "slash": "^3.0.0", - "stack-utils": "^2.0.5", + "stack-utils": "^2.0.6", "strip-ansi": "^7.0.1", "supertap": "^3.0.1", - "temp-dir": "^2.0.0", - "write-file-atomic": "^4.0.1", - "yargs": "^17.5.1" + "temp-dir": "^3.0.0", + "write-file-atomic": "^5.0.0", + "yargs": "^17.6.2" } }, "available-typed-arrays": { @@ -8624,65 +8585,32 @@ } }, "del": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/del/-/del-6.1.1.tgz", - "integrity": "sha512-ua8BhapfP0JUJKC/zV9yHHDW/rDoDxP4Zhn3AkA6/xT6gY7jYXJiaeyBZznYVujhZZET+UgcbZiQ7sN3WqcImg==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/del/-/del-7.0.0.tgz", + "integrity": "sha512-tQbV/4u5WVB8HMJr08pgw0b6nG4RGt/tj+7Numvq+zqcvUFeMaIWWOUFltiU+6go8BSO2/ogsB4EasDaj0y68Q==", "dev": true, "requires": { - "globby": "^11.0.1", - "graceful-fs": "^4.2.4", - "is-glob": "^4.0.1", - "is-path-cwd": "^2.2.0", - "is-path-inside": "^3.0.2", - "p-map": "^4.0.0", + "globby": "^13.1.2", + "graceful-fs": "^4.2.10", + "is-glob": "^4.0.3", + "is-path-cwd": "^3.0.0", + "is-path-inside": "^4.0.0", + "p-map": "^5.5.0", "rimraf": "^3.0.2", - "slash": "^3.0.0" + "slash": "^4.0.0" }, "dependencies": { - "aggregate-error": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", - "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", - "dev": true, - "requires": { - "clean-stack": "^2.0.0", - "indent-string": "^4.0.0" - } - }, - "clean-stack": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", - "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", - "dev": true - }, - "globby": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", - "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", - "dev": true, - "requires": { - "array-union": "^2.1.0", - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.9", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^3.0.0" - } - }, - "indent-string": { + "is-path-inside": { "version": "4.0.0", - "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", - "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-4.0.0.tgz", + "integrity": "sha512-lJJV/5dYS+RcL8uQdBDW9c9uWFLLBNRyFhnAKXw5tVqLlKZ4RMGZKv+YQ/IA3OhD+RpbJa1LLFM1FQPGyIXvOA==", "dev": true }, - "p-map": { + "slash": { "version": "4.0.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", - "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", - "dev": true, - "requires": { - "aggregate-error": "^3.0.0" - } + "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", + "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", + "dev": true } } }, @@ -8730,9 +8658,9 @@ "dev": true }, "emittery": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.11.0.tgz", - "integrity": "sha512-S/7tzL6v5i+4iJd627Nhv9cLFIo5weAIlGccqJFpnBoDB8U1TF2k5tez4J/QNuxyyhWuFqHg1L84Kd3m7iXg6g==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/emittery/-/emittery-1.0.1.tgz", + "integrity": "sha512-2ID6FdrMD9KDLldGesP6317G78K7km/kMcwItRtVFva7I/cSEOIaLpewaUb+YLXVwdAp3Ctfxh/V5zIl1sj7dQ==", "dev": true }, "emoji-regex": { @@ -9377,9 +9305,9 @@ } }, "figures": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/figures/-/figures-4.0.1.tgz", - "integrity": "sha512-rElJwkA/xS04Vfg+CaZodpso7VqBknOYbzi6I76hI4X80RUjkSxO2oAyPmGbuXUppywjqndOrQDl817hDnI++w==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-5.0.0.tgz", + "integrity": "sha512-ej8ksPF4x6e5wvK9yevct0UCXh8TTFlWGVLlgjZuoBH1HwjIfKE/IdL5mq89sFA7zELi1VhKpmtDnrs7zWyeyg==", "dev": true, "requires": { "escape-string-regexp": "^5.0.0", @@ -10004,9 +9932,9 @@ } }, "is-path-cwd": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-2.2.0.tgz", - "integrity": "sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-3.0.0.tgz", + "integrity": "sha512-kyiNFFLU0Ampr6SDZitD/DwUo4Zs1nSdnygUBqsu3LooL00Qvb5j+UnvApUn/TTj1J3OuE6BTdQ5rudKmU2ZaA==", "dev": true }, "is-path-inside": { @@ -11000,9 +10928,9 @@ } }, "parse-ms": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", - "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-3.0.0.tgz", + "integrity": "sha512-Tpb8Z7r7XbbtBTrM9UhpkzzaMrqA2VXMT3YChzYltwV3P3pM6t8wl7TvpMnSTosz1aQAdVib7kdoys7vYOPerw==", "dev": true }, "path-exists": { @@ -11077,12 +11005,12 @@ "dev": true }, "pretty-ms": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-7.0.1.tgz", - "integrity": "sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-8.0.0.tgz", + "integrity": "sha512-ASJqOugUF1bbzI35STMBUpZqdfYKlJugy6JBziGi2EE+AL5JPJGSzvpeVXojxrr0ViUYoToUjb5kjSEGf7Y83Q==", "dev": true, "requires": { - "parse-ms": "^2.1.0" + "parse-ms": "^3.0.0" } }, "private-ip": { @@ -11664,9 +11592,9 @@ "dev": true }, "temp-dir": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-2.0.0.tgz", - "integrity": "sha512-aoBAniQmmwtcKp/7BzsH8Cxzv8OL736p7v1ihGb5e9DJ9kTwGWHrQrVB5+lfVDzfGrdRzXch+ig7LHaY1JTOrg==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-3.0.0.tgz", + "integrity": "sha512-nHc6S/bwIilKHNRgK/3jlhDoIHcp45YgyiwcAk46Tr0LfEqGBVpmiAyuiuxeVE44m3mXnEeVhaipLOEWmH+Njw==", "dev": true }, "text-table": { @@ -11946,13 +11874,21 @@ "dev": true }, "write-file-atomic": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", - "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-5.0.1.tgz", + "integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==", "dev": true, "requires": { "imurmurhash": "^0.1.4", - "signal-exit": "^3.0.7" + "signal-exit": "^4.0.1" + }, + "dependencies": { + "signal-exit": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.0.1.tgz", + "integrity": "sha512-uUWsN4aOxJAS8KOuf3QMyFtgm1pkb6I+KRZbRF/ghdf5T7sM+B1lLLzPDxswUjkmHyxQAVzEgG35E3NzDM9GVw==", + "dev": true + } } }, "ws": { diff --git a/package.json b/package.json index c093f77..3e37766 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ }, "devDependencies": { "@ipld/unixfs": "^2.1.1", - "ava": "^4.3.1", + "ava": "^5.2.0", "blockstore-core": "^1.0.5", "ipfs-unixfs": "^11.0.0", "miniswap": "^2.0.1", diff --git a/test/getPath.test.js b/test/getPath.test.js index d790ddb..f00337a 100644 --- a/test/getPath.test.js +++ b/test/getPath.test.js @@ -10,6 +10,7 @@ import { CID } from 'multiformats/cid' import * as Block from 'multiformats/block' import { getLibp2p, fromNetwork } from '../p2p.js' import { startBitswapPeer } from './_libp2p.js' +import { hasher } from 'multiformats' test('should getPath', async t => { // should return all blocks in path and all blocks for resolved target of path @@ -184,7 +185,7 @@ test('should getPath on dir with carScope=file', async t => { t.deepEqual(entries.at(0).bytes, dirNode.bytes) }) -test('should getPath on hamt sharded dir with carScope=file', async t => { +test('should getPath to a hamt dir with carScope=file', async t => { const { readable, writable } = new TransformStream(undefined, UnixFS.withCapacity(1048576 * 32)) const writer = writable.getWriter() @@ -201,7 +202,6 @@ test('should getPath on hamt sharded dir with carScope=file', async t => { const blocks = [] for await (const block of readable) { blocks.push(block) - console.log(block.cid, block.value?.entries) } const peer = await startBitswapPeer(blocks) @@ -217,3 +217,86 @@ test('should getPath on hamt sharded dir with carScope=file', async t => { t.is(entries.length, 1) t.deepEqual(entries.at(0).cid, dirLink.cid) }) + +test('should getPath to a sharded hamt dir with carScope=file', async t => { + const { readable, writable } = new TransformStream(undefined, UnixFS.withCapacity(1048576 * 32)) + const writer = writable.getWriter() + + const file = UnixFS.createFileWriter({ writer }) + file.write(new TextEncoder().encode('HELP')) + const fileLink = await file.close() + + const dir = UnixFS.createShardedDirectoryWriter({ writer }) + // make a bunch of links to force some imtermediate hamt shards + for (const x of Array.from(Array(250), (_, i) => i)) { + dir.set(`empty-${x}`, { + cid: CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn'), + dagByteLength: 0 + }) + } + dir.set('foo', fileLink) + const dirLink = await dir.close() + + writer.close() + + const blocks = [] + for await (const block of readable) { + blocks.push(block) + } + + const peer = await startBitswapPeer(blocks) + + const libp2p = await getLibp2p() + const dagula = await fromNetwork(libp2p, { peer: peer.libp2p.getMultiaddrs()[0] }) + const res = [] + for await (const block of dagula.getPath(`${dirLink.cid}`, { carScope: 'file' })) { + res.push(block) + } + + // return only the dir if carScope=file and target is a dir. file block should be missing + t.is(res.length, blocks.length - 1, 'all blocks for sharded dir were included') + t.deepEqual(res[0].cid, dirLink.cid, 'first block is root of dir') + t.false(res.some(b => b.cid.toString() === fileLink.cid.toString()), 'linked file was not returned because carScope: file') +}) + +test('should getPath through sharded hamt dir with carScope=file', async t => { + const { readable, writable } = new TransformStream(undefined, UnixFS.withCapacity(1048576 * 32)) + const writer = writable.getWriter() + + const file = UnixFS.createFileWriter({ writer }) + file.write(new TextEncoder().encode('HELP')) + const fileLink = await file.close() + + const dir = UnixFS.createShardedDirectoryWriter({ writer }) + // make a bunch of links to force some imtermediate hamt shards + for (const x of Array.from(Array(1000), (_, i) => i)) { + dir.set(`empty-${x}`, { + cid: CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn'), + dagByteLength: 0 + }) + } + dir.set('foo', fileLink) + const dirLink = await dir.close() + + writer.close() + + const blocks = [] + for await (const block of readable) { + blocks.push(block) + } + + const peer = await startBitswapPeer(blocks) + + const libp2p = await getLibp2p() + const dagula = await fromNetwork(libp2p, { peer: peer.libp2p.getMultiaddrs()[0] }) + + const res = [] + for await (const block of dagula.getPath(`${dirLink.cid}/foo`, { carScope: 'file' })) { + res.push(block) + } + + // only return the hamt root, hamt shard, and file block + t.is(res.length, 3) + t.deepEqual(res.at(0).cid, dirLink.cid) + t.deepEqual(res.at(2).cid, fileLink.cid) +}) diff --git a/test/index.test.js b/test/index.test.js index ab872f2..fe18738 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -56,9 +56,7 @@ test('should walk a unixfs path', async t => { } t.is(entries.length, 2) t.deepEqual(entries.at(0).cid, dirCid) - t.deepEqual(entries.at(0).bytes, dirBytes) t.deepEqual(entries.at(1).cid, cid) - t.deepEqual(entries.at(1).bytes, bytes) }) test('should abort a fetch', async t => { From e942e8272f47927b9a3b13a83f20f51b452af774 Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Fri, 28 Apr 2023 15:54:10 +0100 Subject: [PATCH 08/10] chore: lint License: MIT Signed-off-by: Oli Evans --- index.js | 10 +++++----- test/getPath.test.js | 3 +-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/index.js b/index.js index 4be0f44..b49e3ab 100644 --- a/index.js +++ b/index.js @@ -87,7 +87,7 @@ export class Dagula { * Yield all blocks traversed to resovlve the ipfs path * then use carScope to determine the set of blocks of the targeted dag to yield. * Yield all blocks by default. Use carSope: 'block' to limit it to just the termimal block. - * + * * @param {string} cidPath * @param {object} [options] * @param {AbortSignal} [options.signal] @@ -103,7 +103,7 @@ export class Dagula { /** * The resolved dag root at the terminus of the cidPath - * @type {import('ipfs-unixfs-exporter').UnixFSEntry} + * @type {import('ipfs-unixfs-exporter').UnixFSEntry} * */ let base @@ -112,7 +112,7 @@ export class Dagula { * @type {import('./index').Block[]} */ let traversed = [] - /** + /** * Adapter for unixfs-exporter to track the blocks it loads as it resolves the path. * `walkPath` emits a single unixfs entry for multibock structures, but we need the individual blocks. * TODO: port logic to @web3-storage/ipfs-path to make this less ugly. @@ -209,9 +209,9 @@ export class Dagula { } /** - * Create a search function that given a decoded Block + * Create a search function that given a decoded Block * will return an array of CIDs to fetch next. - * + * * @param {([name, cid]: [string, Link]) => boolean} linkFilter */ export function breadthFirstSearch (linkFilter = () => true) { diff --git a/test/getPath.test.js b/test/getPath.test.js index f00337a..41a200d 100644 --- a/test/getPath.test.js +++ b/test/getPath.test.js @@ -10,7 +10,6 @@ import { CID } from 'multiformats/cid' import * as Block from 'multiformats/block' import { getLibp2p, fromNetwork } from '../p2p.js' import { startBitswapPeer } from './_libp2p.js' -import { hasher } from 'multiformats' test('should getPath', async t => { // should return all blocks in path and all blocks for resolved target of path @@ -289,7 +288,7 @@ test('should getPath through sharded hamt dir with carScope=file', async t => { const libp2p = await getLibp2p() const dagula = await fromNetwork(libp2p, { peer: peer.libp2p.getMultiaddrs()[0] }) - + const res = [] for await (const block of dagula.getPath(`${dirLink.cid}/foo`, { carScope: 'file' })) { res.push(block) From 4aed254b477e4ad5b39643777e1ef79330560007 Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Fri, 28 Apr 2023 21:43:55 +0100 Subject: [PATCH 09/10] chore: typos License: MIT Signed-off-by: Oli Evans --- index.js | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/index.js b/index.js index b49e3ab..3236f4c 100644 --- a/index.js +++ b/index.js @@ -84,9 +84,11 @@ export class Dagula { } /** - * Yield all blocks traversed to resovlve the ipfs path - * then use carScope to determine the set of blocks of the targeted dag to yield. - * Yield all blocks by default. Use carSope: 'block' to limit it to just the termimal block. + * Yield all blocks traversed to resolve the ipfs path. + * Then use carScope to determine the set of blocks of the targeted dag to yield. + * Yield all blocks by default. + * Use carScope: 'block' to yield the termimal block. + * Use carScope: 'file' to yield all the blocks of a unixfs file, or enough blocks to list a directory. * * @param {string} cidPath * @param {object} [options] @@ -95,8 +97,8 @@ export class Dagula { * 'all': return the entire dag starting at path. (default) * 'block': return the block identified by the path. * 'file': Mimic gateway semantics: Return All blocks for a multi-block file or just enough blocks to enumerate a dir/map but not the dir contents. - * e.g. Where path points to a single block file, all three selectors would return the same thing. - * e.g. where path points to a sharded hamt: 'file' returns the blocks of the hamt so the dir can be listed. 'block' returns the root block of the hamt. + * Where path points to a single block file, all three selectors would return the same thing. + * where path points to a sharded hamt: 'file' returns the blocks of the hamt so the dir can be listed. 'block' returns the root block of the hamt. */ async * getPath (cidPath, options = {}) { const carScope = options.carScope ?? 'all' @@ -108,16 +110,16 @@ export class Dagula { let base /** - * cache of blocks required to resove the cidPath + * Cache of blocks required to resove the cidPath * @type {import('./index').Block[]} */ let traversed = [] /** * Adapter for unixfs-exporter to track the blocks it loads as it resolves the path. - * `walkPath` emits a single unixfs entry for multibock structures, but we need the individual blocks. + * `walkPath` emits a single unixfs entry for multiblock structures, but we need the individual blocks. * TODO: port logic to @web3-storage/ipfs-path to make this less ugly. */ - const mockstore = { + const blockstore = { /** * @param {CID} cid * @param {{ signal?: AbortSignal }} [options] @@ -128,7 +130,7 @@ export class Dagula { return block.bytes } } - for await (const item of walkPath(cidPath, mockstore, { signal: options.signal })) { + for await (const item of walkPath(cidPath, blockstore, { signal: options.signal })) { base = item yield * traversed traversed = [] From 26f019a10a93082245a80a024e6272a307d307a4 Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Mon, 1 May 2023 13:20:20 +0100 Subject: [PATCH 10/10] Apply suggestions from code review Co-authored-by: Alan Shaw --- index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 3236f4c..ebf8a9f 100644 --- a/index.js +++ b/index.js @@ -106,12 +106,13 @@ export class Dagula { /** * The resolved dag root at the terminus of the cidPath * @type {import('ipfs-unixfs-exporter').UnixFSEntry} - * */ + */ let base /** * Cache of blocks required to resove the cidPath - * @type {import('./index').Block[]} */ + * @type {import('./index').Block[]} + */ let traversed = [] /**