add pagination to list api

src/handlers/get.js

@@ -11,6 +11,7 @@
  */
 import { getSource } from '../routes/source.js';
 import getList from '../routes/list.js';
+import getListPaginated from '../routes/list-paginated.js';
 import logout from '../routes/logout.js';
 import { getConfig } from '../routes/config.js';
 import { getVersionSource, getVersionList } from '../routes/version.js';
@@ -24,13 +25,14 @@ function getRobots() {
   return { body, status: 200 };
 }
 
-export default async function getHandler({ env, daCtx }) {
+export default async function getHandler({ req, env, daCtx }) {
   const { path } = daCtx;
 
   if (path.startsWith('/favicon.ico')) return get404();
   if (path.startsWith('/robots.txt')) return getRobots();
 
   if (path.startsWith('/source')) return getSource({ env, daCtx });
+  if (path.startsWith('/list-paginated')) return getListPaginated({ req, env, daCtx });
   if (path.startsWith('/list')) return getList({ env, daCtx });
   if (path.startsWith('/config')) return getConfig({ env, daCtx });
   if (path.startsWith('/versionlist')) return getVersionList({ env, daCtx });

src/index.js

@@ -36,7 +36,7 @@ export default {
         respObj = await headHandler({ env, daCtx });
         break;
       case 'GET':
-        respObj = await getHandler({ env, daCtx });
+        respObj = await getHandler({ req, env, daCtx });
         break;
       case 'PUT':
         respObj = await postHandler({ req, env, daCtx });

src/routes/list-paginated.js

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import listBuckets from '../storage/bucket/list.js';
+import { listObjectsPaginated } from '../storage/object/list.js';
+import { getChildRules, hasPermission } from '../utils/auth.js';
+
+export default async function getListPaginated({ req, env, daCtx }) {
+  if (!daCtx.org) return listBuckets(env, daCtx);
+  if (!hasPermission(daCtx, daCtx.key, 'read')) return { status: 403 };
+
+  // Get the child rules of the current folder and store this in daCtx.aclCtx
+  getChildRules(daCtx);
+
+  const { searchParams } = new URL(req.url);
+  const limit = Number.parseInt(searchParams.get('limit'), 10);
+  const offset = Number.parseInt(searchParams.get('offset'), 10);
+
+  function numOrUndef(num) {
+    return Number.isNaN(num) ? undefined : num;
+  }
+
+  return /* await */ listObjectsPaginated(env, daCtx, numOrUndef(limit), numOrUndef(offset));
+}

src/storage/object/list.js

@@ -15,18 +15,77 @@ import {
 } from '@aws-sdk/client-s3';
 
 import getS3Config from '../utils/config.js';
-import formatList from '../utils/list.js';
+import formatList, { formatPaginatedList } from '../utils/list.js';
 
-function buildInput({ org, key, maxKeys }) {
+const LIST_LIMIT = 5000;
+
+function buildInput({
+  org, key, maxKeys, continuationToken,
+}) {
   const input = {
     Bucket: `${org}-content`,
     Prefix: key ? `${key}/` : null,
     Delimiter: '/',
   };
   if (maxKeys) input.MaxKeys = maxKeys;
+  if (continuationToken) input.ContinuationToken = continuationToken;
   return input;
 }
 
+async function scanFiles({
+  daCtx, env, offset, limit,
+}) {
+  const config = getS3Config(env);
+  const client = new S3Client(config);
+
+  let continuationToken = null;
+  let visibleFiles = [];
+  const fetchedItems = [];
+  const fetchedPrefixes = [];
+
+  while (visibleFiles.length < offset + limit) {
+    const remainingKeys = offset + limit - visibleFiles.length;
+    // fetch 25 extra to account for some hidden files (reduce likelihood of continuation token)
+    const numKeysToFetch = Math.min(1000, remainingKeys + 25);
+
+    const input = buildInput({ ...daCtx, maxKeys: numKeysToFetch, continuationToken });
+    const command = new ListObjectsV2Command(input);
+
+    const resp = await client.send(command);
+    continuationToken = resp.NextContinuationToken;
+
+    fetchedItems.push(...(resp.Contents ?? []));
+    fetchedPrefixes.push(...(resp.CommonPrefixes ?? []));
+    visibleFiles = formatPaginatedList(fetchedItems, fetchedPrefixes, daCtx);
+
+    if (!continuationToken) break;
+  }
+
+  return visibleFiles.slice(offset, offset + limit);
+}
+
+export async function listObjectsPaginated(env, daCtx, maxKeys = 1000, offset = 0) {
+  if (offset + maxKeys > LIST_LIMIT) {
+    return { status: 400 };
+  }
+
+  try {
+    const files = await scanFiles({
+      daCtx, env, limit: maxKeys, offset,
+    });
+    return {
+      body: JSON.stringify({
+        offset,
+        limit: maxKeys,
+        data: files,
+      }),
+      status: 200,
+    };
+  } catch (e) {
+    return { body: '', status: 404 };
+  }
+}
+
 export default async function listObjects(env, daCtx, maxKeys) {
   const config = getS3Config(env);
   const client = new S3Client(config);
@@ -35,7 +94,6 @@ export default async function listObjects(env, daCtx, maxKeys) {
   const command = new ListObjectsV2Command(input);
   try {
     const resp = await client.send(command);
-    // console.log(resp);
     const body = formatList(resp, daCtx);
     return {
       body: JSON.stringify(body),

src/storage/utils/list.js

@@ -13,63 +13,92 @@ import {
   ListObjectsV2Command,
 } from '@aws-sdk/client-s3';
 
-export default function formatList(resp, daCtx) {
+function mapPrefixes(CommonPrefixes, daCtx) {
+  return CommonPrefixes?.map((prefix) => {
+    const name = prefix.Prefix.slice(0, -1).split('/').pop();
+    const splitName = name.split('.');
+
+    // Do not add any extension folders
+    if (splitName.length > 1) return null;
+
+    const path = `/${daCtx.org}/${prefix.Prefix.slice(0, -1)}`;
+
+    return { path, name };
+  }).filter((x) => !!x) ?? [];
+}
+
+function mapContents(Contents, folders, daCtx) {
+  return Contents?.map((content) => {
+    let key = content.Key;
+    const itemName = key.split('/').pop();
+    const splitName = itemName.split('.');
+    // file.jpg.props should not be a part of the list
+    // hidden files (.props) should not be a part of this list
+    if (splitName.length !== 2) return null;
+
+    const [name, ext, props] = splitName;
+
+    // Do not show any props sidecar files
+    if (props) return null;
+
+    // See if the folder is already in the list
+    if (ext === 'props') {
+      if (folders.some((item) => item.name === name && !item.ext)) return null;
+
+      // Remove props from the key so it can look like a folder
+      key = key.replace('.props', '');
+    }
+
+    // Do not show any hidden files.
+    if (!name) return null;
+    const item = { path: `/${daCtx.org}/${key}`, name };
+    if (ext !== 'props') {
+      item.ext = ext;
+      item.lastModified = content.LastModified.getTime();
+    }
+
+    return item;
+  }).filter((x) => !!x) ?? [];
+}
+
+// Performs the same as formatList, but doesn't sort (returns exactly how it was
+// sorted in the S3 client response)
+// This prevents bugs when sorting across pages of the paginated api response
+// However, the order is slightly different to the formatList return value
+// for strings with the same prefix but different length
+export function formatPaginatedList(items, prefixes, daCtx) {
   function compare(a, b) {
-    if (a.name < b.name) return -1;
-    if (a.name > b.name) return 1;
+    const aN = a.name;
+    const bN = b.name;
+    if (aN.startsWith(bN) || bN.startsWith(aN)) return bN.length - aN.length;
+    if (aN < bN) return -1;
+    if (aN > bN) return 1;
     return undefined;
   }
 
-  const { CommonPrefixes, Contents } = resp;
+  const folders = mapPrefixes(prefixes, daCtx);
+  const files = mapContents(items, folders, daCtx);
 
   const combined = [];
+  combined.push(...files, ...folders);
 
-  if (CommonPrefixes) {
-    CommonPrefixes.forEach((prefix) => {
-      const name = prefix.Prefix.slice(0, -1).split('/').pop();
-      const splitName = name.split('.');
-
-      // Do not add any extension folders
-      if (splitName.length > 1) return;
+  return combined.sort(compare);
+}
 
-      const path = `/${daCtx.org}/${prefix.Prefix.slice(0, -1)}`;
-      combined.push({ path, name });
-    });
+export default function formatList(resp, daCtx) {
+  function compare(a, b) {
+    if (a.name < b.name) return -1;
+    if (a.name > b.name) return 1;
+    return undefined;
   }
 
-  if (Contents) {
-    Contents.forEach((content) => {
-      const itemName = content.Key.split('/').pop();
-      const splitName = itemName.split('.');
-      // file.jpg.props should not be a part of the list
-      // hidden files (.props) should not be a part of this list
-      if (splitName.length !== 2) return;
-
-      const [name, ext, props] = splitName;
-
-      // Do not show any props sidecar files
-      if (props) return;
-
-      if (ext === 'props') {
-        // Do not add if it already exists as a folder (does not have an extension)
-        if (combined.some((item) => item.name === name && !item.ext)) return;
-
-        // Remove props from the key so it can look like a folder
-        // eslint-disable-next-line no-param-reassign
-        content.Key = content.Key.replace('.props', '');
-      }
-
-      // Do not show any hidden files.
-      if (!name) return;
-      const item = { path: `/${daCtx.org}/${content.Key}`, name };
-      if (ext !== 'props') {
-        item.ext = ext;
-        item.lastModified = content.LastModified.getTime();
-      }
-
-      combined.push(item);
-    });
-  }
+  const { CommonPrefixes, Contents } = resp;
+
+  const folders = mapPrefixes(CommonPrefixes, daCtx);
+  const files = mapContents(Contents, folders, daCtx);
+
+  const combined = [];
+  combined.push(...files, ...folders);
 
   return combined.sort(compare);
 }

test/routes/list-paginated.test.js

@@ -0,0 +1,95 @@
+/*
+ * Copyright 2025 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import assert from 'assert';
+import esmock from 'esmock';
+
+describe('List Route', () => {
+  it('Test getListPaginated with permissions', async () => {
+    const loCalled = [];
+    const listObjectsPaginated = (e, c) => {
+      loCalled.push({ e, c });
+      return {};
+    }
+
+    const ctx = { org: 'foo', key: 'q/q/q' };
+    const hasPermission = (c, k, a) => {
+      if (k === 'q/q/q' && a === 'read') {
+        return false;
+      }
+      return true;
+    }
+
+    const getListPaginated = await esmock(
+      '../../src/routes/list-paginated.js', {
+        '../../src/storage/object/list.js': {
+          listObjectsPaginated
+        },
+        '../../src/utils/auth.js': {
+          hasPermission
+        }
+      }
+    );
+
+    const req = {
+      url: new URL('https://admin.da.live/list/foo/bar'),
+    }
+
+    const resp = await getListPaginated({ req, env: {}, daCtx: ctx, aclCtx: {} });
+    assert.strictEqual(403, resp.status);
+    assert.strictEqual(0, loCalled.length);
+
+    const aclCtx = { pathLookup: new Map() };
+    await getListPaginated({ req, env: {}, daCtx: { org: 'bar', key: 'q/q', users: [], aclCtx }});
+    assert.strictEqual(1, loCalled.length);
+    assert.strictEqual('q/q', loCalled[0].c.key);
+
+    const childRules = aclCtx.childRules;
+    assert.strictEqual(1, childRules.length);
+    assert(childRules[0].startsWith('/q/q/**='), 'Should have defined some child rule');
+  });
+
+  it('parses request params', async () => {
+    const loCalled = [];
+    const listObjectsPaginated = (e, c, limit, offset) => {
+      console.log({offset, limit})
+      loCalled.push({ offset, limit });
+      return {};
+    }
+
+    const hasPermission = () => true;
+
+    const getListPaginated = await esmock(
+      '../../src/routes/list-paginated.js', {
+        '../../src/storage/object/list.js': {
+          listObjectsPaginated
+        },
+        '../../src/utils/auth.js': {
+          hasPermission,
+          getChildRules: () => {}
+        }
+      }
+    );
+
+    const ctx = { org: 'foo', };
+    const reqs = [
+      { url: 'https://admin.da.live/list/foo/bar?limit=12&offset=1' },
+      { url: 'https://admin.da.live/list/foo/bar?limit=asdf&offset=17' },
+      { url: 'https://admin.da.live/list/foo/bar?limit=12&offset=asdf' },
+    ];
+    await getListPaginated({ req: reqs[0], env: {}, daCtx: ctx, aclCtx: {} });
+    assert.deepStrictEqual(loCalled[0], { limit: 12, offset: 1 });
+    await getListPaginated({ req: reqs[1], env: {}, daCtx: ctx, aclCtx: {} });
+    assert.deepStrictEqual(loCalled[1], { limit: undefined, offset: 17 });
+    await getListPaginated({ req: reqs[2], env: {}, daCtx: ctx, aclCtx: {} });
+    assert.deepStrictEqual(loCalled[2], { limit: 12, offset: undefined });
+  });
+});