From 4b997488ea4095a7aa92f9a4d6f6132e23e127a4 Mon Sep 17 00:00:00 2001 From: Aryan Jassal Date: Tue, 6 Aug 2024 16:58:35 +1000 Subject: [PATCH] feat: adding unix-like ls for secrets wip: working on adding unix-like ls for secrets chore: added custom type and pattern matching chore: updated caller for VaultsSecretsList wip: working on passing filetree via rpc feat: revamped the entire VaultsSecretsList handler chore: updated handler name chore: working on fixing read: 0 error chore: renamed VaultsSecretsList to VaultsSecretsGetFileTree fix: fixed problem with `withVaultsG` where `this` wasn't defined [ci skip] chore: switched from RawHandler to ServerCaller chore: cleaned up code and started work on param parser feat: added parsers for parameters [ci skip] chore: updated all tests referencing old serializerStreamFactory feat: switched over the rpc handler from rawhandler to stremhandler feat: stopped using globWalk for reading and listing files chore: added error handling for invalid patterns chore: added error handling for invalid patterns chore: added symbolic link information to the file stats chore: cleaned up irrelevant code chore: fixed tests chore: updated types chore: added comments for stream ending prematurely bug: working out why errors arent being caught by expect chore: updated tests and added new kinds of errors chore: added complement for 'ErrorSecretsIsDirectory' --- src/client/handlers/VaultsSecretsList.ts | 70 ++++++++++++++---------- src/client/types.ts | 13 +++-- src/vaults/VaultManager.ts | 3 +- src/vaults/errors.ts | 12 ++++ src/vaults/fileTree.ts | 12 ++++ tests/client/handlers/vaults.test.ts | 30 +++++++--- 6 files changed, 100 insertions(+), 40 deletions(-) diff --git a/src/client/handlers/VaultsSecretsList.ts b/src/client/handlers/VaultsSecretsList.ts index c1e1dfbfa..277fcdd90 100644 --- a/src/client/handlers/VaultsSecretsList.ts +++ b/src/client/handlers/VaultsSecretsList.ts @@ -2,55 +2,69 @@ import type { DB } from '@matrixai/db'; import type { ClientRPCRequestParams, ClientRPCResponseResult, - SecretNameMessage, - VaultIdentifierMessage, + SecretFilesMessage, + SecretIdentifierMessage, } from '../types'; import type VaultManager from '../../vaults/VaultManager'; +import path from 'path'; import { ServerHandler } from '@matrixai/rpc'; import * as vaultsUtils from '../../vaults/utils'; import * as vaultsErrors from '../../vaults/errors'; -import * as vaultOps from '../../vaults/VaultOps'; class VaultsSecretsList extends ServerHandler< { vaultManager: VaultManager; db: DB; }, - ClientRPCRequestParams, - ClientRPCResponseResult + ClientRPCRequestParams, + ClientRPCResponseResult > { public async *handle( - input: ClientRPCRequestParams, - _cancel, - _meta, - ctx, - ): AsyncGenerator> { - if (ctx.signal.aborted) throw ctx.signal.reason; + input: ClientRPCRequestParams, + _cancel: any, + ): AsyncGenerator, void, void> { const { vaultManager, db } = this.container; - const secrets = await db.withTransactionF(async (tran) => { + const vaultId = await db.withTransactionF(async (tran) => { const vaultIdFromName = await vaultManager.getVaultId( input.nameOrId, tran, ); const vaultId = vaultIdFromName ?? vaultsUtils.decodeVaultId(input.nameOrId); - if (vaultId == null) { - throw new vaultsErrors.ErrorVaultsVaultUndefined(); - } - return await vaultManager.withVaults( - [vaultId], - async (vault) => { - return await vaultOps.listSecrets(vault); - }, - tran, - ); + if (vaultId == null) throw new vaultsErrors.ErrorVaultsVaultUndefined(); + return vaultId; + }); + + yield* vaultManager.withVaultsG([vaultId], (vault) => { + return vault.readG(async function* (fs): AsyncGenerator< + SecretFilesMessage, + void, + void + > { + let files: Array; + try { + files = await fs.promises.readdir(input.secretName); + } catch (e) { + if (e.code === 'ENOENT') { + throw new vaultsErrors.ErrorSecretsDirectoryUndefined(e.message, { + cause: e, + }); + } + if (e.code === 'ENOTDIR') { + throw new vaultsErrors.ErrorSecretsIsSecret(e.message, { + cause: e, + }); + } + throw e; + } + for await (const file of files) { + const filePath = path.join(input.secretName, file.toString()); + const stat = await fs.promises.stat(filePath); + const type = stat.isFile() ? 'FILE' : 'DIRECTORY'; + yield { path: filePath, type: type }; + } + }); }); - for (const secret of secrets) { - if (ctx.signal.aborted) throw ctx.signal.reason; - yield { - secretName: secret, - }; - } } } diff --git a/src/client/types.ts b/src/client/types.ts index ee4055057..0b125b18d 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -300,12 +300,11 @@ type VaultsLatestVersionMessage = { }; // Secrets - -type SecretNameMessage = { +type SecretPathMessage = { secretName: string; }; -type SecretIdentifierMessage = VaultIdentifierMessage & SecretNameMessage; +type SecretIdentifierMessage = VaultIdentifierMessage & SecretPathMessage; // Contains binary content as a binary string 'toString('binary')' type ContentMessage = { @@ -327,6 +326,11 @@ type SecretRenameMessage = SecretIdentifierMessage & { newSecretName: string; }; +type SecretFilesMessage = { + path: string; + type: 'FILE' | 'DIRECTORY'; +}; + // Stat is the 'JSON.stringify version of the file stat type SecretStatMessage = { stat: { @@ -410,13 +414,14 @@ export type { VaultsScanMessage, VaultsVersionMessage, VaultsLatestVersionMessage, - SecretNameMessage, + SecretPathMessage, SecretIdentifierMessage, ContentMessage, SecretContentMessage, SecretMkdirMessage, SecretDirMessage, SecretRenameMessage, + SecretFilesMessage, SecretStatMessage, SignatureMessage, OverrideRPClientType, diff --git a/src/vaults/VaultManager.ts b/src/vaults/VaultManager.ts index 21c34c324..30a7183bd 100644 --- a/src/vaults/VaultManager.ts +++ b/src/vaults/VaultManager.ts @@ -1058,13 +1058,14 @@ class VaultManager { }, ); // Running the function with locking + const vaultThis = this; return yield* this.vaultLocks.withG( ...vaultLocks, async function* (): AsyncGenerator { // Getting the vaults while locked const vaults = await Promise.all( vaultIds.map(async (vaultId) => { - return await this.getVault(vaultId, tran); + return await vaultThis.getVault(vaultId, tran); }), ); return yield* g(...vaults); diff --git a/src/vaults/errors.ts b/src/vaults/errors.ts index b887846c4..a6874fe66 100644 --- a/src/vaults/errors.ts +++ b/src/vaults/errors.ts @@ -118,6 +118,16 @@ class ErrorSecretsIsDirectory extends ErrorSecrets { exitCode = sysexits.USAGE; } +class ErrorSecretsIsSecret extends ErrorSecrets { + static description = 'Is a secret and not a directory'; + exitCode = sysexits.USAGE; +} + +class ErrorSecretsDirectoryUndefined extends ErrorSecrets { + static description = 'Directory does not exist'; + exitCode = sysexits.USAGE; +} + export { ErrorVaults, ErrorVaultManagerRunning, @@ -144,4 +154,6 @@ export { ErrorSecretsSecretUndefined, ErrorSecretsSecretDefined, ErrorSecretsIsDirectory, + ErrorSecretsIsSecret, + ErrorSecretsDirectoryUndefined, }; diff --git a/src/vaults/fileTree.ts b/src/vaults/fileTree.ts index 7180adaf6..31bc05f35 100644 --- a/src/vaults/fileTree.ts +++ b/src/vaults/fileTree.ts @@ -542,8 +542,20 @@ function parserTransformStreamFactory(): TransformStream< }; jsonParser.write(initialChunk); }; + /* Check if any chunks have been processed. If the stream is being flushed + * without processing any chunks, then something went wrong with the stream. + */ + let processedChunks: boolean = false; return new TransformStream({ + flush: (controller) => { + if (!processedChunks) { + controller.error( + new validationErrors.ErrorParse('Stream ended prematurely'), + ); + } + }, transform: (chunk, controller) => { + if (chunk.byteLength > 0) processedChunks = true; switch (phase) { case 'START': { workingBuffer = vaultsUtils.uint8ArrayConcat([workingBuffer, chunk]); diff --git a/tests/client/handlers/vaults.test.ts b/tests/client/handlers/vaults.test.ts index af274acea..b847ee018 100644 --- a/tests/client/handlers/vaults.test.ts +++ b/tests/client/handlers/vaults.test.ts @@ -1587,16 +1587,32 @@ describe('vaultsSecretsNewDir and vaultsSecretsList', () => { dirName: secretDir, }); expect(addResponse.success).toBeTruthy(); - // List secrets - const listResponse = await rpcClient.methods.vaultsSecretsList({ + + const noFiles = await rpcClient.methods.vaultsSecretsList({ nameOrId: vaultsIdEncoded, + secretName: 'doesntExist', }); - const secrets: Array = []; - for await (const secret of listResponse) { - secrets.push(secret.secretName); + + await expect(async () => { + try { + for await (const _ of noFiles); + } catch (e) { + throw e.cause; + } + }).rejects.toThrow(vaultsErrors.ErrorSecretsDirectoryUndefined); + + const secrets = await rpcClient.methods.vaultsSecretsList({ + nameOrId: vaultsIdEncoded, + secretName: 'secretDir', + }); + + // Extract secret file paths + const parsedFiles: Array = []; + for await (const file of secrets) { + parsedFiles.push(file.path); } - expect(secrets.sort()).toStrictEqual( - secretList.map((secret) => path.join('secretDir', secret)).sort(), + expect(parsedFiles).toIncludeAllMembers( + secretList.map((secret) => path.join('secretDir', secret)), ); }); });