diff --git a/src/client/callers/index.ts b/src/client/callers/index.ts index c283cf22e..d89931efd 100644 --- a/src/client/callers/index.ts +++ b/src/client/callers/index.ts @@ -67,7 +67,7 @@ import vaultsSecretsDelete from './vaultsSecretsDelete'; import vaultsSecretsEdit from './vaultsSecretsEdit'; import vaultsSecretsEnv from './vaultsSecretsEnv'; import vaultsSecretsGet from './vaultsSecretsGet'; -import vaultsSecretsGetFileTree from './vaultsSecretsGetFileTree'; +import vaultsSecretsList from './vaultsSecretsList'; import vaultsSecretsMkdir from './vaultsSecretsMkdir'; import vaultsSecretsNew from './vaultsSecretsNew'; import vaultsSecretsNewDir from './vaultsSecretsNewDir'; @@ -148,7 +148,7 @@ const clientManifest = { vaultsSecretsEdit, vaultsSecretsEnv, vaultsSecretsGet, - vaultsSecretsGetFileTree, + vaultsSecretsList, vaultsSecretsMkdir, vaultsSecretsNew, vaultsSecretsNewDir, @@ -228,7 +228,7 @@ export { vaultsSecretsEdit, vaultsSecretsEnv, vaultsSecretsGet, - vaultsSecretsGetFileTree, + vaultsSecretsList, vaultsSecretsMkdir, vaultsSecretsNew, vaultsSecretsNewDir, diff --git a/src/client/callers/vaultsSecretsGetFileTree.ts b/src/client/callers/vaultsSecretsGetFileTree.ts deleted file mode 100644 index f3b024ba6..000000000 --- a/src/client/callers/vaultsSecretsGetFileTree.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { HandlerTypes } from '@matrixai/rpc'; -import type VaultsSecretsGetFileTree from '../handlers/VaultsSecretsGetFileTree'; -import { ServerCaller } from '@matrixai/rpc'; - -type CallerTypes = HandlerTypes; - -const vaultsSecretsGetFileTree = new ServerCaller< - CallerTypes['input'], - CallerTypes['output'] ->(); - -export default vaultsSecretsGetFileTree; diff --git a/src/client/callers/vaultsSecretsList.ts b/src/client/callers/vaultsSecretsList.ts new file mode 100644 index 000000000..c8bba49ae --- /dev/null +++ b/src/client/callers/vaultsSecretsList.ts @@ -0,0 +1,12 @@ +import type { HandlerTypes } from '@matrixai/rpc'; +import type VaultsSecretsList from '../handlers/VaultsSecretsList'; +import { ServerCaller } from '@matrixai/rpc'; + +type CallerTypes = HandlerTypes; + +const vaultsSecretsList = new ServerCaller< + CallerTypes['input'], + CallerTypes['output'] +>(); + +export default vaultsSecretsList; diff --git a/src/client/handlers/VaultsSecretsGetFileTree.ts b/src/client/handlers/VaultsSecretsList.ts similarity index 86% rename from src/client/handlers/VaultsSecretsGetFileTree.ts rename to src/client/handlers/VaultsSecretsList.ts index d39bf89b4..501d2cbc7 100644 --- a/src/client/handlers/VaultsSecretsGetFileTree.ts +++ b/src/client/handlers/VaultsSecretsList.ts @@ -1,11 +1,11 @@ import type { DB } from '@matrixai/db'; import type VaultManager from '../../vaults/VaultManager'; import type { ClientRPCRequestParams, ClientRPCResponseResult } from '../types'; -import type { SecretFilesMessage, VaultFileNode } from '../types'; +import type { SecretFilesMessage, SecretFile } from '../types'; +import type { StatEncoded } from '../../vaults/types'; import path from 'path'; import { ServerHandler } from '@matrixai/rpc'; import { generateStats } from '../../vaults/fileTree'; -import { StatEncoded } from '../../vaults/types'; import * as vaultsUtils from '../../vaults/utils'; import * as vaultsErrors from '../../vaults/errors'; import * as clientErrors from '../errors'; @@ -16,12 +16,12 @@ class VaultsSecretsGetFileTree extends ServerHandler< db: DB; }, ClientRPCRequestParams, - ClientRPCResponseResult + ClientRPCResponseResult > { public async *handle( input: ClientRPCRequestParams, _cancel: any, - ): AsyncGenerator, void, void> { + ): AsyncGenerator, void, void> { const { vaultManager, db } = this.container; const vaultId = await db.withTransactionF(async (tran) => { const vaultIdFromName = await vaultManager.getVaultId( @@ -36,14 +36,14 @@ class VaultsSecretsGetFileTree extends ServerHandler< yield* vaultManager.withVaultsG([vaultId], (vault) => { return vault.readG(async function* (fs): AsyncGenerator< - VaultFileNode, + SecretFile, void, void > { try { // @ts-ignore: While the types don't fully match, it matches enough for our usage. - let files: Array = await fs.promises.readdir(input.pattern); - files = files.map((file) => path.join(input.pattern, file)); + let files: Array = await fs.promises.readdir(input.path); + files = files.map((file) => path.join(input.path, file)); for await (const file of files) { try { @@ -74,7 +74,7 @@ class VaultsSecretsGetFileTree extends ServerHandler< } } catch (e) { throw new clientErrors.ErrorClientFSReadFailed( - `Failed to read directory: ${input.pattern}`, + `Failed to read directory: ${input.path}`, { cause: e }, ); } diff --git a/src/client/handlers/index.ts b/src/client/handlers/index.ts index 351c0932e..a6aebaceb 100644 --- a/src/client/handlers/index.ts +++ b/src/client/handlers/index.ts @@ -84,7 +84,7 @@ import VaultsSecretsDelete from './VaultsSecretsDelete'; import VaultsSecretsEdit from './VaultsSecretsEdit'; import VaultsSecretsEnv from './VaultsSecretsEnv'; import VaultsSecretsGet from './VaultsSecretsGet'; -import VaultsSecretsGetFileTree from './VaultsSecretsGetFileTree'; +import VaultsSecretsList from './VaultsSecretsList'; import VaultsSecretsMkdir from './VaultsSecretsMkdir'; import VaultsSecretsNew from './VaultsSecretsNew'; import VaultsSecretsNewDir from './VaultsSecretsNewDir'; @@ -188,7 +188,7 @@ const serverManifest = (container: { vaultsSecretsEdit: new VaultsSecretsEdit(container), vaultsSecretsEnv: new VaultsSecretsEnv(container), vaultsSecretsGet: new VaultsSecretsGet(container), - vaultsSecretsGetFileTree: new VaultsSecretsGetFileTree(container), + vaultsSecretsList: new VaultsSecretsList(container), vaultsSecretsMkdir: new VaultsSecretsMkdir(container), vaultsSecretsNew: new VaultsSecretsNew(container), vaultsSecretsNewDir: new VaultsSecretsNewDir(container), @@ -270,7 +270,7 @@ export { VaultsSecretsEdit, VaultsSecretsEnv, VaultsSecretsGet, - VaultsSecretsGetFileTree, + VaultsSecretsList, VaultsSecretsMkdir, VaultsSecretsNew, VaultsSecretsNewDir, diff --git a/src/client/types.ts b/src/client/types.ts index d1f2016d9..08c1467fe 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -306,7 +306,7 @@ type VaultsLatestVersionMessage = { // Secrets type SecretFilesMessage = VaultIdentifierMessage & { - pattern: string; + path: string; yieldStats: boolean; }; @@ -336,7 +336,7 @@ type SecretRenameMessage = SecretIdentifierMessage & { newSecretName: string; }; -type VaultFileNode = { +type SecretFile = { path: string; type: 'FILE' | 'DIRECTORY'; stat?: StatEncoded; @@ -433,7 +433,7 @@ export type { SecretMkdirMessage, SecretDirMessage, SecretRenameMessage, - VaultFileNode, + SecretFile, SecretStatMessage, SignatureMessage, OverrideRPClientType, diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 6e65acc6a..cd4dbeda4 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -10,7 +10,6 @@ import os from 'os'; import process from 'process'; import path from 'path'; import nodesEvents from 'events'; -import { ReadableStream } from 'stream/web'; import lexi from 'lexicographic-integer'; import { PromiseCancellable } from '@matrixai/async-cancellable'; import { timedCancellable } from '@matrixai/contexts/dist/functions'; @@ -542,38 +541,6 @@ function setMaxListeners( nodesEvents.setMaxListeners(limit, target); } -function asyncGeneratorToReadableStream( - generator: AsyncGenerator, -): ReadableStream { - return new ReadableStream({ - pull: async (controller) => { - const { done, value } = await generator.next(); - if (done === true) return controller.close(); - controller.enqueue(value); - }, - cancel: async (reason) => { - if (typeof generator.throw === 'function') { - generator.throw(reason).catch(() => {}); - } - }, - }); -} - -async function* readableStreamToAsyncGenerator( - stream: ReadableStream, -): AsyncGenerator { - const reader = stream.getReader(); - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - yield value; - } - } finally { - reader.releaseLock(); - } -} - export { AsyncFunction, GeneratorFunction, @@ -614,6 +581,4 @@ export { isBufferSource, yieldMicro, setMaxListeners, - asyncGeneratorToReadableStream, - readableStreamToAsyncGenerator, }; diff --git a/src/vaults/fileTree.ts b/src/vaults/fileTree.ts index 697e37484..f9a5de850 100644 --- a/src/vaults/fileTree.ts +++ b/src/vaults/fileTree.ts @@ -12,7 +12,7 @@ import type { HeaderContent, } from './types'; import path from 'path'; -import { TransformStream } from 'stream/web'; +import { ReadableStream, TransformStream } from 'stream/web'; import { minimatch } from 'minimatch'; import { JSONParser, TokenizerError } from '@streamparser/json'; import * as vaultsUtils from './utils'; @@ -334,17 +334,16 @@ async function* encodeContent( } /** - * Takes an AsyncGenerator and serializes it into a `AsyncGenerator` + * Takes an AsyncGenerator and serializes it into a `ReadableStream` * @param fs * @param treeGen - An AsyncGenerator that yields the files and directories of a file tree. * @param yieldContents - Toggles sending the contents of files after the file tree. */ -// TODO: change name? -async function* serializerStreamFactory( +function serializerStreamFactory( fs: FileSystem | FileSystemReadable, treeGen: AsyncGenerator, yieldContents: boolean = true, -): AsyncGenerator { +): ReadableStream { const files: Array<[number, string]> = []; let treeDataGen: AsyncGenerator | undefined = treeGen; let contentsGen: AsyncGenerator | undefined = @@ -369,38 +368,53 @@ async function* serializerStreamFactory( await treeDataGen?.throw(reason).catch(() => {}); await contentsGen?.throw(reason).catch(() => {}); } - try { - yield generateGenericHeader({ type: HeaderType.TREE }); - while (true) { - if (treeDataGen != null) { - const result = await treeGen.next(); - if (!result.done) { - // If a file, add to the file list to encode contents later - if (result.value.type === 'FILE') { - files.push([result.value.iNode, result.value.path]); + return new ReadableStream({ + start: (controller) => { + controller.enqueue( + generateGenericHeader({ + type: HeaderType.TREE, + }), + ); + }, + pull: async (controller) => { + try { + if (treeDataGen != null) { + const result = await treeGen.next(); + if (!result.done) { + // If a file, add to the file list to encode contents later + if (result.value.type === 'FILE') { + files.push([result.value.iNode, result.value.path]); + } + // Normal tree nodes are just serialized and converted to `UInt8Array` + const jsonSerialized = JSON.stringify(result.value); + controller.enqueue( + vaultsUtils.bufferToUint8ArrayCopyless( + Buffer.from(jsonSerialized, 'utf-8'), + ), + ); + } else { + const treeDoneMessage = JSON.stringify({ type: 'DONE' }); + controller.enqueue( + vaultsUtils.bufferToUint8ArrayCopyless( + Buffer.from(treeDoneMessage, 'utf-8'), + ), + ); + treeDataGen = undefined; } - // Normal tree nodes are just serialized and converted to `UInt8Array` - const jsonSerialized = JSON.stringify(result.value); - yield vaultsUtils.bufferToUint8ArrayCopyless( - Buffer.from(jsonSerialized, 'utf-8'), - ); } else { - const treeDoneMessage = JSON.stringify({ type: 'DONE' }); - yield vaultsUtils.bufferToUint8ArrayCopyless( - Buffer.from(treeDoneMessage, 'utf-8'), - ); - treeDataGen = undefined; + const contentDataChunk = await getNextContentChunk(); + if (contentDataChunk == null) return controller.close(); + controller.enqueue(contentDataChunk); } - } else { - const contentDataChunk = await getNextContentChunk(); - if (contentDataChunk == null) return; - yield contentDataChunk; + } catch (e) { + await cleanup(e); + return controller.error(e); } - } - } catch (e) { - await cleanup(e); - throw e; - } + }, + cancel: async (reason) => { + await cleanup(reason); + }, + }); } /** diff --git a/src/vaults/types.ts b/src/vaults/types.ts index d768ac3b8..a4609718c 100644 --- a/src/vaults/types.ts +++ b/src/vaults/types.ts @@ -155,7 +155,6 @@ type INode = number; type StatEncoded = { isSymbolicLink: boolean; - symbolicLinkTarget?: string, dev: number; ino: number; mode: number; diff --git a/tests/client/handlers/vaults.test.ts b/tests/client/handlers/vaults.test.ts index 593c84e96..f88dc9f2c 100644 --- a/tests/client/handlers/vaults.test.ts +++ b/tests/client/handlers/vaults.test.ts @@ -35,7 +35,7 @@ import { VaultsSecretsEdit, VaultsSecretsEnv, VaultsSecretsGet, - VaultsSecretsGetFileTree, + VaultsSecretsList, VaultsSecretsMkdir, VaultsSecretsNewDir, VaultsSecretsNew, @@ -56,7 +56,7 @@ import { vaultsSecretsEdit, vaultsSecretsEnv, vaultsSecretsGet, - vaultsSecretsGetFileTree, + vaultsSecretsList, vaultsSecretsMkdir, vaultsSecretsNew, vaultsSecretsNewDir, @@ -1489,7 +1489,7 @@ describe('vaultsSecretsNewDir and vaultsSecretsList', () => { let webSocketClient: WebSocketClient; let rpcClient: RPCClient<{ vaultsSecretsNewDir: typeof vaultsSecretsNewDir; - vaultsSecretsGetFileTree: typeof vaultsSecretsGetFileTree; + vaultsSecretsList: typeof vaultsSecretsList; }>; let vaultManager: VaultManager; beforeEach(async () => { @@ -1533,7 +1533,7 @@ describe('vaultsSecretsNewDir and vaultsSecretsList', () => { fs, vaultManager, }), - vaultsSecretsGetFileTree: new VaultsSecretsGetFileTree({ + vaultsSecretsList: new VaultsSecretsList({ db, vaultManager, }), @@ -1551,7 +1551,7 @@ describe('vaultsSecretsNewDir and vaultsSecretsList', () => { rpcClient = new RPCClient({ manifest: { vaultsSecretsNewDir, - vaultsSecretsGetFileTree, + vaultsSecretsList, }, streamFactory: () => webSocketClient.connection.newStream(), toError: networkUtils.toError, @@ -1590,9 +1590,9 @@ describe('vaultsSecretsNewDir and vaultsSecretsList', () => { expect(addResponse.success).toBeTruthy(); await expect(async () => { - const files = await rpcClient.methods.vaultsSecretsGetFileTree({ + const files = await rpcClient.methods.vaultsSecretsList({ nameOrId: vaultsIdEncoded, - pattern: 'doesntExist', + path: 'doesntExist', yieldStats: false, }); try { @@ -1602,10 +1602,9 @@ describe('vaultsSecretsNewDir and vaultsSecretsList', () => { } }).rejects.toThrow(clientErrors.ErrorClientFSReadFailed); - // List secrets with names of directories - const secrets = await rpcClient.methods.vaultsSecretsGetFileTree({ + const secrets = await rpcClient.methods.vaultsSecretsList({ nameOrId: vaultsIdEncoded, - pattern: 'secretDir', + path: 'secretDir', yieldStats: false, });