feat: adding unix-like ls for secrets

wip: working on adding unix-like ls for secrets

chore: added custom type and pattern matching

chore: updated caller for VaultsSecretsList

wip: working on passing filetree via rpc

feat: revamped the entire VaultsSecretsList handler

chore: updated handler name

chore: working on fixing read: 0 error

chore: renamed VaultsSecretsList to VaultsSecretsGetFileTree

fix: fixed problem with `withVaultsG` where `this` wasn't defined

[ci skip]

chore: switched from RawHandler to ServerCaller

chore: cleaned up code and started work on param parser

feat: added parsers for parameters

[ci skip]

chore: updated all tests referencing old serializerStreamFactory

feat: switched over the rpc handler from rawhandler to stremhandler

feat: stopped using globWalk for reading and listing files

chore: added error handling for invalid patterns

chore: added error handling for invalid patterns

chore: added symbolic link information to the file stats

chore: cleaned up irrelevant code

chore: fixed tests

chore: updated types

chore: added comments for stream ending prematurely

bug: working out why errors arent being caught by expect

chore: updated tests and added new kinds of errors

chore: added complement for 'ErrorSecretsIsDirectory'

src/client/handlers/VaultsSecretsList.ts

@@ -2,55 +2,69 @@ import type { DB } from '@matrixai/db';
 import type {
   ClientRPCRequestParams,
   ClientRPCResponseResult,
-  SecretNameMessage,
-  VaultIdentifierMessage,
+  SecretFilesMessage,
+  SecretIdentifierMessage,
 } from '../types';
 import type VaultManager from '../../vaults/VaultManager';
+import path from 'path';
 import { ServerHandler } from '@matrixai/rpc';
 import * as vaultsUtils from '../../vaults/utils';
 import * as vaultsErrors from '../../vaults/errors';
-import * as vaultOps from '../../vaults/VaultOps';
 
 class VaultsSecretsList extends ServerHandler<
   {
     vaultManager: VaultManager;
     db: DB;
   },
-  ClientRPCRequestParams<VaultIdentifierMessage>,
-  ClientRPCResponseResult<SecretNameMessage>
+  ClientRPCRequestParams<SecretIdentifierMessage>,
+  ClientRPCResponseResult<SecretFilesMessage>
 > {
   public async *handle(
-    input: ClientRPCRequestParams<VaultIdentifierMessage>,
-    _cancel,
-    _meta,
-    ctx,
-  ): AsyncGenerator<ClientRPCResponseResult<SecretNameMessage>> {
-    if (ctx.signal.aborted) throw ctx.signal.reason;
+    input: ClientRPCRequestParams<SecretIdentifierMessage>,
+    _cancel: any,
+  ): AsyncGenerator<ClientRPCResponseResult<SecretFilesMessage>, void, void> {
     const { vaultManager, db } = this.container;
-    const secrets = await db.withTransactionF(async (tran) => {
+    const vaultId = await db.withTransactionF(async (tran) => {
       const vaultIdFromName = await vaultManager.getVaultId(
         input.nameOrId,
         tran,
       );
       const vaultId =
         vaultIdFromName ?? vaultsUtils.decodeVaultId(input.nameOrId);
-      if (vaultId == null) {
-        throw new vaultsErrors.ErrorVaultsVaultUndefined();
-      }
-      return await vaultManager.withVaults(
-        [vaultId],
-        async (vault) => {
-          return await vaultOps.listSecrets(vault);
-        },
-        tran,
-      );
+      if (vaultId == null) throw new vaultsErrors.ErrorVaultsVaultUndefined();
+      return vaultId;
+    });
+
+    yield* vaultManager.withVaultsG([vaultId], (vault) => {
+      return vault.readG(async function* (fs): AsyncGenerator<
+        SecretFilesMessage,
+        void,
+        void
+      > {
+        let files: Array<string | Buffer>;
+        try {
+          files = await fs.promises.readdir(input.secretName);
+        } catch (e) {
+          if (e.code === 'ENOENT') {
+            throw new vaultsErrors.ErrorSecretsDirectoryUndefined(e.message, {
+              cause: e,
+            });
+          }
+          if (e.code === 'ENOTDIR') {
+            throw new vaultsErrors.ErrorSecretsIsSecret(e.message, {
+              cause: e,
+            });
+          }
+          throw e;
+        }
+        for await (const file of files) {
+          const filePath = path.join(input.secretName, file.toString());
+          const stat = await fs.promises.stat(filePath);
+          const type = stat.isFile() ? 'FILE' : 'DIRECTORY';
+          yield { path: filePath, type: type };
+        }
+      });
     });
-    for (const secret of secrets) {
-      if (ctx.signal.aborted) throw ctx.signal.reason;
-      yield {
-        secretName: secret,
-      };
-    }
   }
 }
 

src/client/types.ts

@@ -300,12 +300,11 @@ type VaultsLatestVersionMessage = {
 };
 
 // Secrets
-
-type SecretNameMessage = {
+type SecretPathMessage = {
   secretName: string;
 };
 
-type SecretIdentifierMessage = VaultIdentifierMessage & SecretNameMessage;
+type SecretIdentifierMessage = VaultIdentifierMessage & SecretPathMessage;
 
 // Contains binary content as a binary string 'toString('binary')'
 type ContentMessage = {
@@ -327,6 +326,11 @@ type SecretRenameMessage = SecretIdentifierMessage & {
   newSecretName: string;
 };
 
+type SecretFilesMessage = {
+  path: string;
+  type: 'FILE' | 'DIRECTORY';
+};
+
 // Stat is the 'JSON.stringify version of the file stat
 type SecretStatMessage = {
   stat: {
@@ -410,13 +414,14 @@ export type {
   VaultsScanMessage,
   VaultsVersionMessage,
   VaultsLatestVersionMessage,
-  SecretNameMessage,
+  SecretPathMessage,
   SecretIdentifierMessage,
   ContentMessage,
   SecretContentMessage,
   SecretMkdirMessage,
   SecretDirMessage,
   SecretRenameMessage,
+  SecretFilesMessage,
   SecretStatMessage,
   SignatureMessage,
   OverrideRPClientType,

src/vaults/VaultManager.ts

@@ -1058,13 +1058,14 @@ class VaultManager {
       },
     );
     // Running the function with locking
+    const vaultThis = this;
     return yield* this.vaultLocks.withG(
       ...vaultLocks,
       async function* (): AsyncGenerator<T, Treturn, Tnext> {
         // Getting the vaults while locked
         const vaults = await Promise.all(
           vaultIds.map(async (vaultId) => {
-            return await this.getVault(vaultId, tran);
+            return await vaultThis.getVault(vaultId, tran);
           }),
         );
         return yield* g(...vaults);

src/vaults/errors.ts

@@ -118,6 +118,16 @@ class ErrorSecretsIsDirectory<T> extends ErrorSecrets<T> {
   exitCode = sysexits.USAGE;
 }
 
+class ErrorSecretsIsSecret<T> extends ErrorSecrets<T> {
+  static description = 'Is a secret and not a directory';
+  exitCode = sysexits.USAGE;
+}
+
+class ErrorSecretsDirectoryUndefined<T> extends ErrorSecrets<T> {
+  static description = 'Directory does not exist';
+  exitCode = sysexits.USAGE;
+}
+
 export {
   ErrorVaults,
   ErrorVaultManagerRunning,
@@ -144,4 +154,6 @@ export {
   ErrorSecretsSecretUndefined,
   ErrorSecretsSecretDefined,
   ErrorSecretsIsDirectory,
+  ErrorSecretsIsSecret,
+  ErrorSecretsDirectoryUndefined,
 };

src/vaults/fileTree.ts

@@ -542,8 +542,20 @@ function parserTransformStreamFactory(): TransformStream<
     };
     jsonParser.write(initialChunk);
   };
+  /* Check if any chunks have been processed. If the stream is being flushed
+   * without processing any chunks, then something went wrong with the stream.
+   */
+  let processedChunks: boolean = false;
   return new TransformStream<Uint8Array, TreeNode | ContentNode | Uint8Array>({
+    flush: (controller) => {
+      if (!processedChunks) {
+        controller.error(
+          new validationErrors.ErrorParse('Stream ended prematurely'),
+        );
+      }
+    },
     transform: (chunk, controller) => {
+      if (chunk.byteLength > 0) processedChunks = true;
       switch (phase) {
         case 'START': {
           workingBuffer = vaultsUtils.uint8ArrayConcat([workingBuffer, chunk]);

tests/client/handlers/vaults.test.ts

@@ -1587,16 +1587,32 @@ describe('vaultsSecretsNewDir and vaultsSecretsList', () => {
       dirName: secretDir,
     });
     expect(addResponse.success).toBeTruthy();
-    // List secrets
-    const listResponse = await rpcClient.methods.vaultsSecretsList({
+
+    const noFiles = await rpcClient.methods.vaultsSecretsList({
       nameOrId: vaultsIdEncoded,
+      secretName: 'doesntExist',
     });
-    const secrets: Array<string> = [];
-    for await (const secret of listResponse) {
-      secrets.push(secret.secretName);
+
+    await expect(async () => {
+      try {
+        for await (const _ of noFiles);
+      } catch (e) {
+        throw e.cause;
+      }
+    }).rejects.toThrow(vaultsErrors.ErrorSecretsDirectoryUndefined);
+
+    const secrets = await rpcClient.methods.vaultsSecretsList({
+      nameOrId: vaultsIdEncoded,
+      secretName: 'secretDir',
+    });
+
+    // Extract secret file paths
+    const parsedFiles: Array<string> = [];
+    for await (const file of secrets) {
+      parsedFiles.push(file.path);
     }
-    expect(secrets.sort()).toStrictEqual(
-      secretList.map((secret) => path.join('secretDir', secret)).sort(),
+    expect(parsedFiles).toIncludeAllMembers(
+      secretList.map((secret) => path.join('secretDir', secret)),
     );
   });
 });