From d91ed0f8de3b373a8720807e1142cd7796259766 Mon Sep 17 00:00:00 2001 From: Kelvin Luck Date: Fri, 8 Jul 2022 23:35:06 +0100 Subject: [PATCH 1/4] fix: Optimise upload when `allowOverwrite` is true There is no need to try and figure out if a revision has already been deployed if we are going to overwrite it anyway. Let's save ourselves the network request! --- lib/s3.js | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/s3.js b/lib/s3.js index c821371..66313f9 100644 --- a/lib/s3.js +++ b/lib/s3.js @@ -62,6 +62,7 @@ module.exports = CoreObject.extend({ var isGzipped = gzippedFilePaths.indexOf(options.filePattern) !== -1; var isBrotliCompressed = brotliCompressedFilePaths.indexOf(options.filePattern) !== -1; var serverSideEncryption = options.serverSideEncryption; + var checkForOverwrite = RSVP.resolve(); var params = { Bucket: bucket, @@ -83,13 +84,17 @@ module.exports = CoreObject.extend({ params.ContentEncoding = 'br'; } - return this.findRevision(options) - .then(function(found) { - if (found !== undefined && !allowOverwrite) { - return RSVP.reject("REVISION ALREADY UPLOADED! (set `allowOverwrite: true` if you want to support overwriting revisions)"); - } - return RSVP.resolve(); - }) + if (!allowOverwrite) { + checkForOverwrite = this.findRevision(options) + .then(function(found) { + if (found !== undefined) { + return RSVP.reject("REVISION ALREADY UPLOADED! (set `allowOverwrite: true` if you want to support overwriting revisions)"); + } + return RSVP.resolve(); + }) + } + + return checkForOverwrite .then(readFile.bind(this, options.filePath)) .then(function(fileContents) { params.Body = fileContents; From 6b2b57387b26ed139bc0668ffb11bbd54351c8a1 Mon Sep 17 00:00:00 2001 From: Kelvin Luck Date: Mon, 4 Jul 2022 21:53:36 +0100 Subject: [PATCH 2/4] feat: Introduce an optional escape hatch for `listAllObjects` There are cases where we might want to page through the list of objects available on S3 but we don't need to keep going once we've found what we are looking for. This commit updates `listAllObjects` to accept an `until` argument - if this is provided then when a new page of results is loaded we check if `some` of them match the `until` - if so we don't bother loading another page. A concrete use-case for this is in #120 - sometimes we just want to find the currently active revision so we only need to loop through the "pages" on S3 until we do. --- lib/s3.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/s3.js b/lib/s3.js index 66313f9..2e6d180 100644 --- a/lib/s3.js +++ b/lib/s3.js @@ -179,7 +179,7 @@ module.exports = CoreObject.extend({ }); }, - listAllObjects: function(options) { + listAllObjects: function(options, until) { var client = this._client; var listObjects = RSVP.denodeify(client.listObjects.bind(client)); var allRevisions = []; @@ -188,13 +188,14 @@ module.exports = CoreObject.extend({ return listObjects(options).then(function(response) { [].push.apply(allRevisions, response.Contents); - if (response.IsTruncated) { - var nextMarker = response.Contents[response.Contents.length - 1].Key; - options.Marker = nextMarker; - return listObjectRecursively(options); - } else { + var isComplete = !response.IsTruncated || (until && response.Contents.some(until)); + + if (isComplete) { return allRevisions; } + var nextMarker = response.Contents[response.Contents.length - 1].Key; + options.Marker = nextMarker; + return listObjectRecursively(options); }); } From 5d346ef9a8288ede9a951e01a1205824dde08d44 Mon Sep 17 00:00:00 2001 From: Kelvin Luck Date: Sat, 9 Jul 2022 00:01:46 +0100 Subject: [PATCH 3/4] feat: Define `fetchRevisionsFunc` and `fetchInitialRevisionsFunc` on config So that they can be overridden by call sites who are looking to optimise performance (listing revisions from S3 can be expensive - especially so when you have many deployments deployed) --- index.js | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/index.js b/index.js index bb1edaa..176b508 100644 --- a/index.js +++ b/index.js @@ -34,6 +34,26 @@ module.exports = { brotliCompressedFiles: function(context) { return context.brotliCompressedFiles || []; }, + fetchRevisionsFunc: function(context) { + return function() { + return this._list(context) + .then(function(revisions) { + return { + revisions: revisions + }; + }); + } + }, + fetchInitialRevisionsFunc: function(context) { + return function() { + return this._list(context) + .then(function(revisions) { + return { + initialRevisions: revisions + }; + }); + } + }, allowOverwrite: false }, @@ -106,22 +126,12 @@ module.exports = { return s3.activate(options); }, - fetchRevisions: function(context) { - return this._list(context) - .then(function(revisions) { - return { - revisions: revisions - }; - }); + fetchRevisions: function(/* context */) { + return this.readConfig('fetchRevisionsFunc').call(this); }, - fetchInitialRevisions: function(context) { - return this._list(context) - .then(function(revisions) { - return { - initialRevisions: revisions - }; - }); + fetchInitialRevisions: function(/* context */) { + return this.readConfig('fetchInitialRevisionsFunc').call(this); }, _list: function(/* context */) { From c42a08670ccf77d77b459464ce3dc37b1d61eacb Mon Sep 17 00:00:00 2001 From: Kelvin Luck Date: Sat, 9 Jul 2022 00:29:31 +0100 Subject: [PATCH 4/4] [DOC] Add some documentation for the new functions --- README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/README.md b/README.md index 79e6d4a..352c164 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,38 @@ If you are using DigitalOcean spaces you need to set this setting to `false`. *Default:* `true` +### fetchInitialRevisionsFunc + +The function that is called when `fetchInitialRevisions` is called on this plugin (e.g. when a user runs `ember deploy`). + +The main purpose of this function is to gather the "before" state of your s3 buckets so that you can produce a useful change or +audit log. + +The default implementation of this will loop over all revisions found in S3. This can be slow if you have a long history of +deployments. You may find that in your usages that you don't need to gather this information in this way so you may want to +overwrite it to just return an empty array e.g. + +```js +fetchInitialRevisionsFunc(/* context */) { + return () => ({ initialRevisions: [] }); +} +```` + +### fetchRevisionsFunc + +The function that is called when `fetchRevisions` is called on this plugin (e.g. when a user runs `ember deploy:list`). + + +The default implementation of this will loop over all revisions found in S3. This can be slow if you have a long history of +deployments. You may find that in your usages there are times that it's not ncessary to load this data so you may want to +overwrite it to just return an empty array e.g. + +```js +fetchRevisionsFunc(/* context */) { + return () => ({ revisions: [] }); +}, +```` + ### How do I activate a revision? A user can activate a revision by either: