Skip to content
This repository has been archived by the owner on Feb 1, 2022. It is now read-only.

Commit

Permalink
feat: support sourcing from buckets with over 1000 objects (#20)
Browse files Browse the repository at this point in the history
* adds ContinuationToken and extra error handling

* updates list objects error handling

Co-Authored-By: Robin Métral <[email protected]>

* updates empty bucket error handling

Co-Authored-By: Robin Métral <[email protected]>

* adds types for getS3ListObjects params

Co-Authored-By: Robin Métral <[email protected]>

* adds types for listAllS3Items params

Co-Authored-By: Robin Métral <[email protected]>

* use optional chaining operator for contents check

Co-Authored-By: Robin Métral <[email protected]>

* uses ObjectType for allS3Items

Co-Authored-By: Robin Métral <[email protected]>

* feat: adds suggested changes from PR to use ContinuationToken

* fix: runs prettier to correct indent

* fix: adds type back to allBucketsObjects

* corrects type on allBucketsObjects

Co-Authored-By: Robin Métral <[email protected]>

* Corrects check for data.Contents

Co-Authored-By: Robin Métral <[email protected]>

Co-authored-by: Robin Métral <[email protected]>
  • Loading branch information
LpmRaven and robinmetral authored Apr 14, 2020
1 parent d0fda52 commit ec9fc17
Showing 1 changed file with 50 additions and 22 deletions.
72 changes: 50 additions & 22 deletions src/gatsby-node.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,34 +28,62 @@ export async function sourceNodes(
// get objects
const s3 = new AWS.S3();

const listObjects = async bucket => {
// todo improve this call
// see https://stackoverflow.com/a/49888947
const response = await s3
.listObjectsV2({
Bucket: bucket
// todo handle continuation token
// ContinuationToken: token,
})
.promise();

// add bucket key
const objects = response.Contents?.reduce((acc: ObjectType[], cur) => {
const object: ObjectType = { ...cur, Bucket: bucket };
acc.push(object);
return acc;
}, []);

return objects;
const getS3ListObjects = async (params: {
Bucket: string;
ContinuationToken?: string;
}) => {
return await s3
.listObjectsV2(params)
.promise()
.catch(error => {
reporter.error(
`Error listing S3 objects on bucket "${params.Bucket}": ${error}`
);
});
};

const listAllS3Objects = async (bucket: string) => {
const allS3Objects: ObjectType[] = [];

const data = await getS3ListObjects({ Bucket: bucket });

if (data && data.Contents) {
data.Contents.forEach(object => {
allS3Objects.push({ ...object, Bucket: bucket });
});
} else {
reporter.error(
`Error processing objects from bucket "${bucket}". Is it empty?`
);
}

let nextToken = data && data.IsTruncated && data.NextContinuationToken;

while (nextToken) {
const data = await getS3ListObjects({
Bucket: bucket,
ContinuationToken: nextToken
});

if (data && data.Contents) {
data.Contents.forEach(object => {
allS3Objects.push({ ...object, Bucket: bucket });
});
}
nextToken = data && data.IsTruncated && data.NextContinuationToken;
}

return allS3Objects;
};

try {
let objects: Array<any> = await Promise.all(
buckets.map(bucket => listObjects(bucket))
const allBucketsObjects: ObjectType[][] = await Promise.all(
buckets.map(bucket => listAllS3Objects(bucket))
);

// flatten objects
// flat() is not supported in node 10
objects = [].concat(...objects);
const objects = allBucketsObjects.reduce((acc, val) => acc.concat(val), []);

// create file nodes
// todo touch nodes if they exist already
Expand Down

0 comments on commit ec9fc17

Please sign in to comment.