From 5651d86adaf3fbb4e5931131134c1b4f8642ecb8 Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Thu, 15 Aug 2024 16:50:46 -0700
Subject: [PATCH 01/22] jq expression to assist with monitoring log file during
 migration

---
 monitor-migration/trace-migration-ops.jq | 29 ++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 monitor-migration/trace-migration-ops.jq

diff --git a/monitor-migration/trace-migration-ops.jq b/monitor-migration/trace-migration-ops.jq
new file mode 100644
index 0000000..5001b27
--- /dev/null
+++ b/monitor-migration/trace-migration-ops.jq
@@ -0,0 +1,29 @@
+#
+# The expression requires explicitly passing `pubid_field` argument
+# to specify the column name from the input file that contains the
+# public_id of the migrated asset.
+#
+# Example:
+# --------
+# $ tail -f log.jsonl | \
+#     jq --arg pubid_field "<Public Id Column Name>" \ -r -f trace-migration-ops.jq
+#
+def pad_right(n): . + " " * (n - length);
+
+select(.flow == "payload") |
+"\t" as $separator |
+if .input | has($pubid_field) | not then
+    error("Input does not contain the specified field: '\($pubid_field)'")
+end |
+.input[$pubid_field] as $public_id |
+if .summary.status == "MIGRATED" then
+    if .response | has("overwritten") then
+        ["🟡", ("overwritten"      | pad_right(20)), $public_id] | join($separator)
+    elif .response | has("existing") then
+        ["⚪️", ("skipped existing" | pad_right(20)), $public_id] | join($separator)
+    else
+        ["🟢", ("created"          | pad_right(20)), $public_id] | join($separator)
+    end
+else
+    ["🔴", ("failed" | pad_right(20)), $public_id, (.summary.err | tostring)] | join($separator)
+end

From 3ed692010ba8d3707e75b5c8aa198737b28583d1 Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Fri, 16 Aug 2024 12:21:33 -0700
Subject: [PATCH 02/22] Added script to count ops and trace failed. Removed jq
 expression to avoid logic duplication and simplify

---
 monitor-migration/count-logged-ops.sh    | 36 ++++++++++++++++++++++++
 monitor-migration/trace-failed-ops.sh    |  8 ++++++
 monitor-migration/trace-migration-ops.jq | 29 -------------------
 3 files changed, 44 insertions(+), 29 deletions(-)
 create mode 100755 monitor-migration/count-logged-ops.sh
 create mode 100755 monitor-migration/trace-failed-ops.sh
 delete mode 100644 monitor-migration/trace-migration-ops.jq

diff --git a/monitor-migration/count-logged-ops.sh b/monitor-migration/count-logged-ops.sh
new file mode 100755
index 0000000..9303cc9
--- /dev/null
+++ b/monitor-migration/count-logged-ops.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# File to be monitored
+LOGFILE=$1
+
+# Processing payload entries from the log to produce tally
+grep -F '"flow":"payload"' $LOGFILE | \
+awk '
+    BEGIN {
+        # Explicitly initialize variables to 0
+        created = 0
+        overwritten = 0
+        existing = 0
+        failure = 0
+    }    
+    {
+        migrated = index($0, "\"status\":\"MIGRATED\"")
+        if (migrated) {
+            if (index($0, "\"existing\":true")) {
+                existing++
+            } else if (index($0, "\"overwritten\":true")) {
+                overwritten++
+            } else {
+                created++
+            }
+        } else {
+            failure++
+        }
+    }    
+    END {
+        print "🟢 Created            : ", created
+        print "🟡 Overwritten        : ", overwritten
+        print "⚪️ Existing (skipped) : ", existing
+        print "🔴 Failed             : ", failure
+    }
+'
\ No newline at end of file
diff --git a/monitor-migration/trace-failed-ops.sh b/monitor-migration/trace-failed-ops.sh
new file mode 100755
index 0000000..40f8a74
--- /dev/null
+++ b/monitor-migration/trace-failed-ops.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Log file to be scanned
+LOGFILE=$1
+
+# Processing payload entries with status different from MIGRATED
+grep -F '"flow":"payload"' $LOGFILE | grep -F -v '"status":"MIGRATED"' | \
+jq -r '(.summary.err | tostring)'
diff --git a/monitor-migration/trace-migration-ops.jq b/monitor-migration/trace-migration-ops.jq
deleted file mode 100644
index 5001b27..0000000
--- a/monitor-migration/trace-migration-ops.jq
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# The expression requires explicitly passing `pubid_field` argument
-# to specify the column name from the input file that contains the
-# public_id of the migrated asset.
-#
-# Example:
-# --------
-# $ tail -f log.jsonl | \
-#     jq --arg pubid_field "<Public Id Column Name>" \ -r -f trace-migration-ops.jq
-#
-def pad_right(n): . + " " * (n - length);
-
-select(.flow == "payload") |
-"\t" as $separator |
-if .input | has($pubid_field) | not then
-    error("Input does not contain the specified field: '\($pubid_field)'")
-end |
-.input[$pubid_field] as $public_id |
-if .summary.status == "MIGRATED" then
-    if .response | has("overwritten") then
-        ["🟡", ("overwritten"      | pad_right(20)), $public_id] | join($separator)
-    elif .response | has("existing") then
-        ["⚪️", ("skipped existing" | pad_right(20)), $public_id] | join($separator)
-    else
-        ["🟢", ("created"          | pad_right(20)), $public_id] | join($separator)
-    end
-else
-    ["🔴", ("failed" | pad_right(20)), $public_id, (.summary.err | tostring)] | join($separator)
-end

From 516857efa84863b4511edd34e28cae20e77d37d1 Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Fri, 16 Aug 2024 12:24:12 -0700
Subject: [PATCH 03/22] Renamed for consistency and to express purpose

---
 monitor-migration/{count-logged-ops.sh => count-log-ops.sh}       | 0
 .../{trace-failed-ops.sh => trace-failed-log-ops.sh}              | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename monitor-migration/{count-logged-ops.sh => count-log-ops.sh} (100%)
 rename monitor-migration/{trace-failed-ops.sh => trace-failed-log-ops.sh} (100%)

diff --git a/monitor-migration/count-logged-ops.sh b/monitor-migration/count-log-ops.sh
similarity index 100%
rename from monitor-migration/count-logged-ops.sh
rename to monitor-migration/count-log-ops.sh
diff --git a/monitor-migration/trace-failed-ops.sh b/monitor-migration/trace-failed-log-ops.sh
similarity index 100%
rename from monitor-migration/trace-failed-ops.sh
rename to monitor-migration/trace-failed-log-ops.sh

From f8fc192005eb824cf528eeb630454e0512740f9b Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Thu, 22 Aug 2024 16:54:33 -0700
Subject: [PATCH 04/22] Added check for resolved operation on initial migration

---
 test/end2end/migration-basic/e2e-migration-basic.test.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/end2end/migration-basic/e2e-migration-basic.test.js b/test/end2end/migration-basic/e2e-migration-basic.test.js
index b3e35a5..74dcc1a 100644
--- a/test/end2end/migration-basic/e2e-migration-basic.test.js
+++ b/test/end2end/migration-basic/e2e-migration-basic.test.js
@@ -168,6 +168,7 @@ describe('End-to-end migration basic', () => {
         const testReportEntry = testReportEntries[0];
         expect(testReportEntry.Cld_PublicId).toEqual(public_id);
         expect(testReportEntry.Cld_Status).toEqual('MIGRATED');
+        expect(testReportEntry.Cld_Operation).toEqual('Uploaded');
     });
 
     test.each(

From dee6e278244d5cc99ba6bf8ee96fe2ba927b2bca Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Thu, 22 Aug 2024 17:47:09 -0700
Subject: [PATCH 05/22] Extracted re-usable logic to produce test migration
 records

---
 .../e2e-migration-basic.test.js               | 78 ++-----------------
 .../yield-e2e-migration-records.js            | 73 +++++++++++++++++
 2 files changed, 81 insertions(+), 70 deletions(-)
 create mode 100644 test/end2end/migration-basic/yield-e2e-migration-records.js

diff --git a/test/end2end/migration-basic/e2e-migration-basic.test.js b/test/end2end/migration-basic/e2e-migration-basic.test.js
index 74dcc1a..c9f97f3 100644
--- a/test/end2end/migration-basic/e2e-migration-basic.test.js
+++ b/test/end2end/migration-basic/e2e-migration-basic.test.js
@@ -1,74 +1,12 @@
 const fs = require('fs');
 const path = require('path');
 const testAppInput = require('../app-input');
-const testResources = require('../../resources');
 const testAppFlow = require('../test-invoke-app-flow');
 const migrationPayload = require('../../../lib/payload/migrate');
+const testMigrationRecords = require('./yield-e2e-migration-records');
 
 const INPUT_CSV_FILE = path.join(__dirname, 'input.csv');
 const TEST_OUTPUT_FOLDER = path.join(__dirname, 'test-output');
-//
-// Persisted to CSV file used as input for the end-to-end test
-// Keys are used as asset public_ids
-// Values are expanded into CSV columns 
-//
-// Split into positive / negative to allow referencing separately in the tests
-//
-const _TEST_INPUT_POSITIVE = {
-    test_http_remote_asset_small    : {Ref: 'https://res.cloudinary.com/cld-sol-demo/image/upload/sample.jpg'},
-    test_local_asset_small_relpath  : {Ref: testResources.getAssetPathRelativeToAppRoot('sample.jpg')},
-    test_local_asset_small_fullpath : {Ref: testResources.getAssetFullPath('sample.jpg')},
-    test_local_asset_large          : {Ref: testResources.LARGE_VIDEO_FILE_FULLPATH},
-}
-
-const _TEST_INPUT_NEGATIVE = {
-    remote_test_asset_does_not_exist : {Ref: 'https://res.cloudinary.com/cld-sol-demo/image/upload/this-asset-does-not-exist.png'},
-    local_test_asset_does_not_exist  : {Ref: testResources.getAssetFullPath('this-asset-does-not-exist.jpg')},
-}
-
-// Adding bulk tests
-const _TEST_CASE_BULK_SIZE = 100; // Number of records to generate for each test case
-
-const _BULK_TEST_CASES_POSITIVE_REMOTE = new Object();
-for (let i = 0; i < _TEST_CASE_BULK_SIZE; i++) {
-    _BULK_TEST_CASES_POSITIVE_REMOTE[`test_http_remote_asset_small_${i}`] = {Ref: 'https://res.cloudinary.com/cld-sol-demo/image/upload/sample.jpg'};
-}
-
-const _BULK_TEST_CASES_POSITIVE_LOCAL = new Object();
-for (let i = 0; i < _TEST_CASE_BULK_SIZE; i++) {
-    _BULK_TEST_CASES_POSITIVE_LOCAL[`test_local_asset_small_relpath_${i}`] = {Ref: testResources.getAssetPathRelativeToAppRoot('sample.jpg')};
-}
-
-const _BULK_TEST_CASES_NEGATIVE_REMOTE = new Object();
-for (let i = 0; i < _TEST_CASE_BULK_SIZE; i++) {
-    _BULK_TEST_CASES_NEGATIVE_REMOTE[`remote_test_asset_does_not_exist_${i}`] = {Ref: 'https://res.cloudinary.com/cld-sol-demo/image/upload/this-asset-does-not-exist.png'};
-}
-
-const _BULK_TEST_CASES_NEGATIVE_LOCAL = new Object();
-for (let i = 0; i < _TEST_CASE_BULK_SIZE; i++) {
-    _BULK_TEST_CASES_NEGATIVE_LOCAL[`local_test_asset_does_not_exist_${i}`] = {Ref: testResources.getAssetFullPath('this-asset-does-not-exist.jpg')};
-}
-
-const TEST_INPUT = {
-    ..._TEST_INPUT_POSITIVE,
-    ..._TEST_INPUT_NEGATIVE,
-    ..._BULK_TEST_CASES_POSITIVE_REMOTE,
-    ..._BULK_TEST_CASES_POSITIVE_LOCAL,
-    ..._BULK_TEST_CASES_NEGATIVE_REMOTE,
-    ..._BULK_TEST_CASES_NEGATIVE_LOCAL,
-};
-
-const _TEST_INPUT_POSITIVE_CASES = {
-    ..._TEST_INPUT_POSITIVE,
-    ..._BULK_TEST_CASES_POSITIVE_REMOTE,
-    ..._BULK_TEST_CASES_POSITIVE_LOCAL,
-};
-
-const _TEST_INPUT_NEGATIVE_CASES = {
-    ..._TEST_INPUT_NEGATIVE,
-    ..._BULK_TEST_CASES_NEGATIVE_REMOTE,
-    ..._BULK_TEST_CASES_NEGATIVE_LOCAL,
-};
 
 // Mocking the CSV input to API payload conversion logic to match the 
 // produced CSV input for the test
@@ -108,7 +46,7 @@ describe('End-to-end migration basic', () => {
         
         console.log('Serializing test input to CSV file...');
         testAppInput.testInput2CsvFile({
-            test_input: TEST_INPUT,
+            test_input: testMigrationRecords.ALL_RECORDS,
             csv_file_path: INPUT_CSV_FILE,
         });
 
@@ -140,7 +78,7 @@ describe('End-to-end migration basic', () => {
     });
 
     test.each(
-        Object.keys(TEST_INPUT)
+        Object.keys(testMigrationRecords.ALL_RECORDS)
     )('Should produce single log record for asset %s', (public_id) => {
         const testLogEntries = __TEST_LOG.getEntriesByPublicId(public_id);
         expect(testLogEntries.length).toEqual(1);
@@ -148,11 +86,11 @@ describe('End-to-end migration basic', () => {
 
     it('Should produce report file with a record for each input', () => {
         expect(fs.existsSync(__TEST_REPORT.getPath())).toBe(true);
-        expect(__TEST_REPORT.getLength()).toEqual(Object.keys(TEST_INPUT).length);
+        expect(__TEST_REPORT.getLength()).toEqual(Object.keys(testMigrationRecords.ALL_RECORDS).length);
     });
 
     test.each(
-        Object.keys(_TEST_INPUT_POSITIVE_CASES)
+        Object.keys(testMigrationRecords.POSITIVE_ONLY)
     )('Should successfully migrate valid asset %s', (public_id) => {
         const testLogEntries = __TEST_LOG.getEntriesByPublicId(public_id);
         const testLogEntry = testLogEntries[0];
@@ -161,7 +99,7 @@ describe('End-to-end migration basic', () => {
     });
 
     test.each(
-        Object.keys(_TEST_INPUT_POSITIVE_CASES)
+        Object.keys(testMigrationRecords.POSITIVE_ONLY)
     )('Should produce report record for migrated asset %s', (public_id) => {
         const testReportEntries = __TEST_REPORT.getEntriesByPublicId(public_id);
         expect(testReportEntries.length).toEqual(1);
@@ -172,7 +110,7 @@ describe('End-to-end migration basic', () => {
     });
 
     test.each(
-        Object.keys(_TEST_INPUT_NEGATIVE_CASES)
+        Object.keys(testMigrationRecords.NEGATIVE_ONLY)
     )('Should report errors for invalid asset %s', (public_id) => {
         const testLogEntries = __TEST_LOG.getEntriesByPublicId(public_id);
         const testLogEntry = testLogEntries[0];
@@ -181,7 +119,7 @@ describe('End-to-end migration basic', () => {
     });
 
     test.each(
-        Object.keys(_TEST_INPUT_NEGATIVE_CASES)
+        Object.keys(testMigrationRecords.NEGATIVE_ONLY)
     )('Should produce report record for failed asset %s', (public_id) => {
         const testReportEntries = __TEST_REPORT.getEntriesByPublicId(public_id);
         expect(testReportEntries.length).toEqual(1);
diff --git a/test/end2end/migration-basic/yield-e2e-migration-records.js b/test/end2end/migration-basic/yield-e2e-migration-records.js
new file mode 100644
index 0000000..0f86bb3
--- /dev/null
+++ b/test/end2end/migration-basic/yield-e2e-migration-records.js
@@ -0,0 +1,73 @@
+/*
+    Yields the migration records for the E2E tests
+*/
+const testResources = require('../../resources');
+
+//
+// Persisted to CSV file used as input for the end-to-end test
+// Keys are to be used as asset public_ids
+// Values will be expanded into CSV columns 
+//
+// Split into positive / negative to allow referencing separately in the tests
+//
+const _TEST_INPUT_POSITIVE = {
+    test_http_remote_asset_small    : {Ref: 'https://res.cloudinary.com/cld-sol-demo/image/upload/sample.jpg'},
+    test_local_asset_small_relpath  : {Ref: testResources.getAssetPathRelativeToAppRoot('sample.jpg')},
+    test_local_asset_small_fullpath : {Ref: testResources.getAssetFullPath('sample.jpg')},
+    test_local_asset_large          : {Ref: testResources.LARGE_VIDEO_FILE_FULLPATH},
+}
+
+const _TEST_INPUT_NEGATIVE = {
+    remote_test_asset_does_not_exist : {Ref: 'https://res.cloudinary.com/cld-sol-demo/image/upload/this-asset-does-not-exist.png'},
+    local_test_asset_does_not_exist  : {Ref: testResources.getAssetFullPath('this-asset-does-not-exist.jpg')},
+}
+
+// Adding bulk tests
+const _TEST_CASE_BULK_SIZE = 100; // Number of records to generate for each test case
+
+const _BULK_TEST_CASES_POSITIVE_REMOTE = new Object();
+for (let i = 0; i < _TEST_CASE_BULK_SIZE; i++) {
+    _BULK_TEST_CASES_POSITIVE_REMOTE[`test_http_remote_asset_small_${i}`] = {Ref: 'https://res.cloudinary.com/cld-sol-demo/image/upload/sample.jpg'};
+}
+
+const _BULK_TEST_CASES_POSITIVE_LOCAL = new Object();
+for (let i = 0; i < _TEST_CASE_BULK_SIZE; i++) {
+    _BULK_TEST_CASES_POSITIVE_LOCAL[`test_local_asset_small_relpath_${i}`] = {Ref: testResources.getAssetPathRelativeToAppRoot('sample.jpg')};
+}
+
+const _BULK_TEST_CASES_NEGATIVE_REMOTE = new Object();
+for (let i = 0; i < _TEST_CASE_BULK_SIZE; i++) {
+    _BULK_TEST_CASES_NEGATIVE_REMOTE[`remote_test_asset_does_not_exist_${i}`] = {Ref: 'https://res.cloudinary.com/cld-sol-demo/image/upload/this-asset-does-not-exist.png'};
+}
+
+const _BULK_TEST_CASES_NEGATIVE_LOCAL = new Object();
+for (let i = 0; i < _TEST_CASE_BULK_SIZE; i++) {
+    _BULK_TEST_CASES_NEGATIVE_LOCAL[`local_test_asset_does_not_exist_${i}`] = {Ref: testResources.getAssetFullPath('this-asset-does-not-exist.jpg')};
+}
+
+const TEST_INPUT = {
+    ..._TEST_INPUT_POSITIVE,
+    ..._TEST_INPUT_NEGATIVE,
+    ..._BULK_TEST_CASES_POSITIVE_REMOTE,
+    ..._BULK_TEST_CASES_POSITIVE_LOCAL,
+    ..._BULK_TEST_CASES_NEGATIVE_REMOTE,
+    ..._BULK_TEST_CASES_NEGATIVE_LOCAL,
+};
+
+const _TEST_INPUT_POSITIVE_CASES = {
+    ..._TEST_INPUT_POSITIVE,
+    ..._BULK_TEST_CASES_POSITIVE_REMOTE,
+    ..._BULK_TEST_CASES_POSITIVE_LOCAL,
+};
+
+const _TEST_INPUT_NEGATIVE_CASES = {
+    ..._TEST_INPUT_NEGATIVE,
+    ..._BULK_TEST_CASES_NEGATIVE_REMOTE,
+    ..._BULK_TEST_CASES_NEGATIVE_LOCAL,
+};
+
+module.exports = {
+    ALL_RECORDS     : TEST_INPUT,
+    POSITIVE_ONLY: _TEST_INPUT_POSITIVE_CASES,
+    NEGATIVE_ONLY: _TEST_INPUT_NEGATIVE_CASES,
+};
\ No newline at end of file

From 5c1e442bdff9654fb05c4bfcaf78a1b0a7021a3d Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Thu, 22 Aug 2024 18:14:38 -0700
Subject: [PATCH 06/22] Reorganized test invocation (to ensure order for
 end-to-end tests). Expanded readme to reflect.

---
 README.md                     |  3 +++
 jest.config.js                |  6 -----
 package.json                  |  2 +-
 readme/dev/readme.md          | 13 +++++++++
 test/__global-e2e-setup.js    | 14 ----------
 test/__global-e2e-teardown.js | 14 ----------
 test/jest.run-all-tests.js    | 50 +++++++++++++++++++++++++++++++++++
 7 files changed, 67 insertions(+), 35 deletions(-)
 delete mode 100644 jest.config.js
 create mode 100644 readme/dev/readme.md
 delete mode 100644 test/__global-e2e-setup.js
 delete mode 100644 test/__global-e2e-teardown.js
 create mode 100644 test/jest.run-all-tests.js

diff --git a/README.md b/README.md
index 559c30e..ac0f119 100644
--- a/README.md
+++ b/README.md
@@ -28,3 +28,6 @@ Follow these steps to successfully migrate your assets:
 3. [⚙️ Configure the Script](./readme/configure.md) - Customize the script's settings for your specific migration needs.
 4. [🚚 Run the Script and Obtain the Report](./readme/run-migration-obtain-report.md) - Execute the script and review the migration report.
 5. [🔄 Iterate for Failed Migrations](./readme/identify-reattempt-failed.md) - Identify failed asset migrations and rerun the script to fix them.
+
+# How to Tweak It
+Things to know are covered in the [🧑‍💻 dev readme](./readme/dev/readme.md).
\ No newline at end of file
diff --git a/jest.config.js b/jest.config.js
deleted file mode 100644
index 67009b0..0000000
--- a/jest.config.js
+++ /dev/null
@@ -1,6 +0,0 @@
-// Jest configuration as per https://jestjs.io/docs/configuration
-
-module.exports = {
-    globalSetup: './test/__global-e2e-setup.js',
-    globalTeardown: './test/__global-e2e-teardown.js',
-}
\ No newline at end of file
diff --git a/package.json b/package.json
index 04830ce..5fb856f 100644
--- a/package.json
+++ b/package.json
@@ -19,7 +19,7 @@
     "jest": "^29.6.2"
   },
   "scripts": {
-    "test": "jest --verbose"
+    "test": "node ./test/jest.run-all-tests.js"
   },
   "version": "2.0.1"
 }
diff --git a/readme/dev/readme.md b/readme/dev/readme.md
new file mode 100644
index 0000000..3c93246
--- /dev/null
+++ b/readme/dev/readme.md
@@ -0,0 +1,13 @@
+# Things to know about 
+
+## Layout
+
+### App Components
+All of the app components (and tests for them) are expected to be under the `lib` folder.
+
+This is important because tests are explicitly invoked from under the folder
+
+### App Tests
+Test modules are maintained under the `test` folder.
+
+End to end tests require to be executed in certain order (see the `test/jest.run-all-tests.js` file)
\ No newline at end of file
diff --git a/test/__global-e2e-setup.js b/test/__global-e2e-setup.js
deleted file mode 100644
index d9b3f41..0000000
--- a/test/__global-e2e-setup.js
+++ /dev/null
@@ -1,14 +0,0 @@
-const testEnv = require('./end2end/test-env');
-const resources = require('./resources');
-
-// Global setup for Jest.
-//
-// Provisions new temp "sandbox" product environment before running tests 
-module.exports = async () => {
-    console.log('\nGLOBAL SETUP: START');
-    await testEnv.setupNewSandboxCloud_Async();
-    console.log('Downloading large video asset');
-    await resources.createLargeVideoTestAsset_Async();
-    console.log('GLOBAL SETUP: DONE\n');
-};
-
diff --git a/test/__global-e2e-teardown.js b/test/__global-e2e-teardown.js
deleted file mode 100644
index a088fbc..0000000
--- a/test/__global-e2e-teardown.js
+++ /dev/null
@@ -1,14 +0,0 @@
-const testEnv = require('./end2end/test-env');
-const resources = require('./resources');
-
-// Global teardown for Jest.
-//
-// Deletes the .env file for the previously provisioned temp 
-// "sandbox" product environment
-module.exports = async () => { 
-    console.log('\nGLOBAL TEARDOWN: START');
-    testEnv.teardown();
-    console.log('Deleting large video asset');
-    await resources.cleanupLargeVideoTestAsset_Async();
-    console.log('GLOBAL TEARDOWN: DONE');
-}
\ No newline at end of file
diff --git a/test/jest.run-all-tests.js b/test/jest.run-all-tests.js
new file mode 100644
index 0000000..4175b75
--- /dev/null
+++ b/test/jest.run-all-tests.js
@@ -0,0 +1,50 @@
+/**
+ * @fileoverview  Explicitly invokes test modules to ensure correct order.
+ * 
+ * Some end-to-end tests are dependent on the order of execution (initial migration, testing overwrites, testing existing assets)
+*/
+const { execSync } = require('child_process');
+
+const testEnv = require('./end2end/test-env');
+const resources = require('./resources');
+
+// Setup and teardown
+async function runGlobalSetup_Async() {
+    console.log('\nGLOBAL SETUP: START');
+    await testEnv.setupNewSandboxCloud_Async();
+    console.log('Downloading large video asset');
+    await resources.createLargeVideoTestAsset_Async();
+    console.log('GLOBAL SETUP: DONE\n');
+};
+
+async function runGlobalTeardown_Async() { 
+    console.log('\nGLOBAL TEARDOWN: START');
+    testEnv.teardown();
+    console.log('Deleting large video asset');
+    await resources.cleanupLargeVideoTestAsset_Async();
+    console.log('GLOBAL TEARDOWN: DONE');
+}
+
+// Helper function to run tests from a path or module
+function runTestsFrom(testPath) {
+    console.log(`Running test from: ${testPath}`);
+    execSync(`npx jest ${testPath}`, {stdio: 'inherit'});
+}
+
+
+// Invoking tests
+(async () => {
+    await runGlobalSetup_Async();
+
+    
+    // It is assumed that all component tests are next to components
+    // under the 'lib' folder
+    runTestsFrom('./lib/');
+
+    // Running end-to-end tests in order
+    runTestsFrom('./test/end2end/migration-basic/e2e-migration-basic.test.js');
+    
+
+    await runGlobalTeardown_Async();
+})();
+

From 06d95f3bf17931324c43340166bf467a0dce0b2e Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Thu, 22 Aug 2024 18:26:19 -0700
Subject: [PATCH 07/22] Reorganized tests to allow for sequential execution for
 end-to-end tests

---
 .../e2e-initial-migration.test.js}                          | 4 ++--
 .../{migration-basic => }/yield-e2e-migration-records.js    | 2 +-
 test/jest.run-all-tests.js                                  | 6 +++---
 3 files changed, 6 insertions(+), 6 deletions(-)
 rename test/end2end/{migration-basic/e2e-migration-basic.test.js => 001-test-initial-migration/e2e-initial-migration.test.js} (97%)
 rename test/end2end/{migration-basic => }/yield-e2e-migration-records.js (98%)

diff --git a/test/end2end/migration-basic/e2e-migration-basic.test.js b/test/end2end/001-test-initial-migration/e2e-initial-migration.test.js
similarity index 97%
rename from test/end2end/migration-basic/e2e-migration-basic.test.js
rename to test/end2end/001-test-initial-migration/e2e-initial-migration.test.js
index c9f97f3..8bc0c3d 100644
--- a/test/end2end/migration-basic/e2e-migration-basic.test.js
+++ b/test/end2end/001-test-initial-migration/e2e-initial-migration.test.js
@@ -3,7 +3,7 @@ const path = require('path');
 const testAppInput = require('../app-input');
 const testAppFlow = require('../test-invoke-app-flow');
 const migrationPayload = require('../../../lib/payload/migrate');
-const testMigrationRecords = require('./yield-e2e-migration-records');
+const testMigrationRecords = require('../yield-e2e-migration-records');
 
 const INPUT_CSV_FILE = path.join(__dirname, 'input.csv');
 const TEST_OUTPUT_FOLDER = path.join(__dirname, 'test-output');
@@ -38,7 +38,7 @@ async function cleanup() {
 let __TEST_LOG = null;
 let __TEST_REPORT = null;
 
-describe('End-to-end migration basic', () => {
+describe('Initial asset migration', () => {
     beforeAll(async () => {
         console.log('Preparing test environment');
         // Ensuring there are no artifacts from prior test run that could interfere
diff --git a/test/end2end/migration-basic/yield-e2e-migration-records.js b/test/end2end/yield-e2e-migration-records.js
similarity index 98%
rename from test/end2end/migration-basic/yield-e2e-migration-records.js
rename to test/end2end/yield-e2e-migration-records.js
index 0f86bb3..62ff127 100644
--- a/test/end2end/migration-basic/yield-e2e-migration-records.js
+++ b/test/end2end/yield-e2e-migration-records.js
@@ -1,7 +1,7 @@
 /*
     Yields the migration records for the E2E tests
 */
-const testResources = require('../../resources');
+const testResources = require('../resources');
 
 //
 // Persisted to CSV file used as input for the end-to-end test
diff --git a/test/jest.run-all-tests.js b/test/jest.run-all-tests.js
index 4175b75..b03ec12 100644
--- a/test/jest.run-all-tests.js
+++ b/test/jest.run-all-tests.js
@@ -28,7 +28,7 @@ async function runGlobalTeardown_Async() {
 // Helper function to run tests from a path or module
 function runTestsFrom(testPath) {
     console.log(`Running test from: ${testPath}`);
-    execSync(`npx jest ${testPath}`, {stdio: 'inherit'});
+    execSync(`npx jest --verbose "${testPath}"`, {stdio: 'inherit'});
 }
 
 
@@ -42,8 +42,8 @@ function runTestsFrom(testPath) {
     runTestsFrom('./lib/');
 
     // Running end-to-end tests in order
-    runTestsFrom('./test/end2end/migration-basic/e2e-migration-basic.test.js');
-    
+    runTestsFrom('./test/end2end/001-test-initial-migration/');
+
 
     await runGlobalTeardown_Async();
 })();

From d445faff60997bbd1662fa1c68941a8be30134e3 Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Thu, 22 Aug 2024 18:30:07 -0700
Subject: [PATCH 08/22] Reorganized tests to simplify structure

---
 .../001-initial-migration.test.js}                              | 0
 test/jest.run-all-tests.js                                      | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename test/end2end/{001-test-initial-migration/e2e-initial-migration.test.js => tests/001-initial-migration.test.js} (100%)

diff --git a/test/end2end/001-test-initial-migration/e2e-initial-migration.test.js b/test/end2end/tests/001-initial-migration.test.js
similarity index 100%
rename from test/end2end/001-test-initial-migration/e2e-initial-migration.test.js
rename to test/end2end/tests/001-initial-migration.test.js
diff --git a/test/jest.run-all-tests.js b/test/jest.run-all-tests.js
index b03ec12..e1b677f 100644
--- a/test/jest.run-all-tests.js
+++ b/test/jest.run-all-tests.js
@@ -42,7 +42,7 @@ function runTestsFrom(testPath) {
     runTestsFrom('./lib/');
 
     // Running end-to-end tests in order
-    runTestsFrom('./test/end2end/001-test-initial-migration/');
+    runTestsFrom('./test/end2end/tests/001-initial-migration.test.js');
 
 
     await runGlobalTeardown_Async();

From df4413f9c29e48cd0462dd4f3df62bda3a4a87b4 Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Thu, 22 Aug 2024 18:38:04 -0700
Subject: [PATCH 09/22] Added test for overwriting enabled in end-to-end tests

---
 .../tests/002-overwriting-enabled.test.js     | 88 +++++++++++++++++++
 test/jest.run-all-tests.js                    |  2 +
 2 files changed, 90 insertions(+)
 create mode 100644 test/end2end/tests/002-overwriting-enabled.test.js

diff --git a/test/end2end/tests/002-overwriting-enabled.test.js b/test/end2end/tests/002-overwriting-enabled.test.js
new file mode 100644
index 0000000..cd3b4c3
--- /dev/null
+++ b/test/end2end/tests/002-overwriting-enabled.test.js
@@ -0,0 +1,88 @@
+const fs = require('fs');
+const path = require('path');
+const testAppInput = require('../app-input');
+const testAppFlow = require('../test-invoke-app-flow');
+const migrationPayload = require('../../../lib/payload/migrate');
+const testMigrationRecords = require('../yield-e2e-migration-records');
+
+const INPUT_CSV_FILE = path.join(__dirname, 'input.csv');
+const TEST_OUTPUT_FOLDER = path.join(__dirname, 'test-output');
+
+// Mocking the CSV input to API payload conversion logic to match the 
+// produced CSV input for the test
+jest.mock('../../../__input-to-api-payload', () => {
+    return {
+        input2ApiPayload: jest.fn((csvRec) => {
+            return {
+                file: csvRec.Ref,
+                options: {
+                    public_id: csvRec.public_id,
+                    unique_filename: false,
+                    resource_type: 'auto',
+                    type: 'upload',        
+                }
+            };
+        })
+    };
+});
+
+
+async function cleanup() {
+    await testAppFlow.testCleanup_Async({
+        input_csv_file: INPUT_CSV_FILE,
+        test_output_folder: TEST_OUTPUT_FOLDER,
+    });
+}
+
+// Variables to reference records from the parsed migration log and report files
+let __TEST_LOG = null;
+let __TEST_REPORT = null;
+
+describe('Overwriting enabled', () => {
+    beforeAll(async () => {
+        console.log('Preparing test environment');
+        // Ensuring there are no artifacts from prior test run that could interfere
+        await cleanup();
+        
+        console.log('Serializing test input to CSV file...');
+        testAppInput.testInput2CsvFile({
+            test_input: testMigrationRecords.POSITIVE_ONLY,
+            csv_file_path: INPUT_CSV_FILE,
+        });
+
+        // Invoking the main loop for E2E testing
+        const { testLog, testReport } = await testAppFlow.invokeMainLoopForTest_Async(
+            { // Mocking CLI args
+                fromCsvFile: INPUT_CSV_FILE,
+                maxConcurrentUploads: 2,
+                outputFolder: TEST_OUTPUT_FOLDER,
+            },
+            { // Mocking CLI command
+                name: () => 'migrate',
+            },
+            migrationPayload
+        );
+        
+        __TEST_LOG = testLog;
+        __TEST_REPORT = testReport;
+
+        console.log('Done preparing test environment');
+    }, 5*60*1000); // Explicitly setting timeout to allow for execution of the migration loop
+
+    afterAll(async () => {
+        await cleanup();
+    });
+
+    test.each(
+        Object.keys(testMigrationRecords.POSITIVE_ONLY)
+    )('Should report existing asset as overwritten %s', (public_id) => {
+        const testReportEntries = __TEST_REPORT.getEntriesByPublicId(public_id);
+        expect(testReportEntries.length).toEqual(1);
+        const testReportEntry = testReportEntries[0];
+        expect(testReportEntry.Cld_PublicId).toEqual(public_id);
+        expect(testReportEntry.Cld_Status).toEqual('MIGRATED');
+        expect(testReportEntry.Cld_Operation).toEqual('Overwritten');
+    });
+
+});
+
diff --git a/test/jest.run-all-tests.js b/test/jest.run-all-tests.js
index e1b677f..aee66f8 100644
--- a/test/jest.run-all-tests.js
+++ b/test/jest.run-all-tests.js
@@ -7,6 +7,7 @@ const { execSync } = require('child_process');
 
 const testEnv = require('./end2end/test-env');
 const resources = require('./resources');
+const { run } = require('jest');
 
 // Setup and teardown
 async function runGlobalSetup_Async() {
@@ -43,6 +44,7 @@ function runTestsFrom(testPath) {
 
     // Running end-to-end tests in order
     runTestsFrom('./test/end2end/tests/001-initial-migration.test.js');
+    runTestsFrom('./test/end2end/tests/002-overwriting-enabled.test.js');
 
 
     await runGlobalTeardown_Async();

From 4d826adfb5c193b17f2de0aa7d67f7f19d17310c Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Thu, 22 Aug 2024 18:57:51 -0700
Subject: [PATCH 10/22] Removed redundant message

---
 test/jest.run-all-tests.js | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/test/jest.run-all-tests.js b/test/jest.run-all-tests.js
index aee66f8..cf421b5 100644
--- a/test/jest.run-all-tests.js
+++ b/test/jest.run-all-tests.js
@@ -28,7 +28,6 @@ async function runGlobalTeardown_Async() {
 
 // Helper function to run tests from a path or module
 function runTestsFrom(testPath) {
-    console.log(`Running test from: ${testPath}`);
     execSync(`npx jest --verbose "${testPath}"`, {stdio: 'inherit'});
 }
 
@@ -44,8 +43,7 @@ function runTestsFrom(testPath) {
 
     // Running end-to-end tests in order
     runTestsFrom('./test/end2end/tests/001-initial-migration.test.js');
-    runTestsFrom('./test/end2end/tests/002-overwriting-enabled.test.js');
-
+    runTestsFrom('./test/end2end/tests/002-overwriting-enabled.test.js');    
 
     await runGlobalTeardown_Async();
 })();

From 6983001deab7c33369f6de266bede0cbec0fd473 Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Thu, 22 Aug 2024 18:58:22 -0700
Subject: [PATCH 11/22] Optimized tests execution. Increased concurrently for
 large batch. Reduce batch size for overwrite enabled test.

---
 .../tests/001-initial-migration.test.js        |  2 +-
 .../tests/002-overwriting-enabled.test.js      | 18 ++++++++++++++++--
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/test/end2end/tests/001-initial-migration.test.js b/test/end2end/tests/001-initial-migration.test.js
index 8bc0c3d..79abeee 100644
--- a/test/end2end/tests/001-initial-migration.test.js
+++ b/test/end2end/tests/001-initial-migration.test.js
@@ -54,7 +54,7 @@ describe('Initial asset migration', () => {
         const { testLog, testReport } = await testAppFlow.invokeMainLoopForTest_Async(
             { // Mocking CLI args
                 fromCsvFile: INPUT_CSV_FILE,
-                maxConcurrentUploads: 2,
+                maxConcurrentUploads: 10,
                 outputFolder: TEST_OUTPUT_FOLDER,
             },
             { // Mocking CLI command
diff --git a/test/end2end/tests/002-overwriting-enabled.test.js b/test/end2end/tests/002-overwriting-enabled.test.js
index cd3b4c3..d8f6e1d 100644
--- a/test/end2end/tests/002-overwriting-enabled.test.js
+++ b/test/end2end/tests/002-overwriting-enabled.test.js
@@ -38,6 +38,20 @@ async function cleanup() {
 let __TEST_LOG = null;
 let __TEST_REPORT = null;
 
+// Reduce number of test records to speed up execution
+function reduceTestRecords(toNoOfRecords, fromTestDataObj) {
+    reducedBatchSize = Math.min(toNoOfRecords, Object.keys(fromTestDataObj).length);
+    let reducedTestData = {};
+    let keys = Object.keys(fromTestDataObj);
+    for (let i = 0; i < reducedBatchSize; i++) {
+        testDataKey = keys[i];
+        reducedTestData[testDataKey] = fromTestDataObj[testDataKey];
+    }
+    return reducedTestData;
+}
+
+let __TEST_DATA = reduceTestRecords(10, testMigrationRecords.POSITIVE_ONLY);
+
 describe('Overwriting enabled', () => {
     beforeAll(async () => {
         console.log('Preparing test environment');
@@ -46,7 +60,7 @@ describe('Overwriting enabled', () => {
         
         console.log('Serializing test input to CSV file...');
         testAppInput.testInput2CsvFile({
-            test_input: testMigrationRecords.POSITIVE_ONLY,
+            test_input: __TEST_DATA,
             csv_file_path: INPUT_CSV_FILE,
         });
 
@@ -74,7 +88,7 @@ describe('Overwriting enabled', () => {
     });
 
     test.each(
-        Object.keys(testMigrationRecords.POSITIVE_ONLY)
+        Object.keys(__TEST_DATA)
     )('Should report existing asset as overwritten %s', (public_id) => {
         const testReportEntries = __TEST_REPORT.getEntriesByPublicId(public_id);
         expect(testReportEntries.length).toEqual(1);

From d07ee55d32b2ca503dcb1b509344c0d595b3ac7c Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Thu, 22 Aug 2024 19:04:42 -0700
Subject: [PATCH 12/22] Updated max concurrent uploads to 20 (typical value for
 bulk migrations)

---
 test/end2end/tests/001-initial-migration.test.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/end2end/tests/001-initial-migration.test.js b/test/end2end/tests/001-initial-migration.test.js
index 79abeee..6d04113 100644
--- a/test/end2end/tests/001-initial-migration.test.js
+++ b/test/end2end/tests/001-initial-migration.test.js
@@ -54,7 +54,7 @@ describe('Initial asset migration', () => {
         const { testLog, testReport } = await testAppFlow.invokeMainLoopForTest_Async(
             { // Mocking CLI args
                 fromCsvFile: INPUT_CSV_FILE,
-                maxConcurrentUploads: 10,
+                maxConcurrentUploads: 20,
                 outputFolder: TEST_OUTPUT_FOLDER,
             },
             { // Mocking CLI command

From 7bcebeb8edd7dbbdd02d1ea846964daa1de53f2a Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Thu, 22 Aug 2024 19:05:07 -0700
Subject: [PATCH 13/22] Extracted re-usable function to reduce size of the test
 batch

---
 .../tests/002-overwriting-enabled.test.js        | 16 ++++------------
 test/end2end/yield-e2e-migration-records.js      | 15 +++++++++++++++
 2 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/test/end2end/tests/002-overwriting-enabled.test.js b/test/end2end/tests/002-overwriting-enabled.test.js
index d8f6e1d..d80bb52 100644
--- a/test/end2end/tests/002-overwriting-enabled.test.js
+++ b/test/end2end/tests/002-overwriting-enabled.test.js
@@ -39,18 +39,10 @@ let __TEST_LOG = null;
 let __TEST_REPORT = null;
 
 // Reduce number of test records to speed up execution
-function reduceTestRecords(toNoOfRecords, fromTestDataObj) {
-    reducedBatchSize = Math.min(toNoOfRecords, Object.keys(fromTestDataObj).length);
-    let reducedTestData = {};
-    let keys = Object.keys(fromTestDataObj);
-    for (let i = 0; i < reducedBatchSize; i++) {
-        testDataKey = keys[i];
-        reducedTestData[testDataKey] = fromTestDataObj[testDataKey];
-    }
-    return reducedTestData;
-}
-
-let __TEST_DATA = reduceTestRecords(10, testMigrationRecords.POSITIVE_ONLY);
+let __TEST_DATA = testMigrationRecords.reduceTestRecords(
+    10, 
+    testMigrationRecords.POSITIVE_ONLY
+);
 
 describe('Overwriting enabled', () => {
     beforeAll(async () => {
diff --git a/test/end2end/yield-e2e-migration-records.js b/test/end2end/yield-e2e-migration-records.js
index 62ff127..1570aac 100644
--- a/test/end2end/yield-e2e-migration-records.js
+++ b/test/end2end/yield-e2e-migration-records.js
@@ -66,8 +66,23 @@ const _TEST_INPUT_NEGATIVE_CASES = {
     ..._BULK_TEST_CASES_NEGATIVE_LOCAL,
 };
 
+// Helper function to reduce the number of test records 
+// when a smaller batch is needed for testing
+function reduceTestRecords(toNoOfRecords, fromTestDataObj) {
+    reducedBatchSize = Math.min(toNoOfRecords, Object.keys(fromTestDataObj).length);
+    let reducedTestData = {};
+    let keys = Object.keys(fromTestDataObj);
+    for (let i = 0; i < reducedBatchSize; i++) {
+        testDataKey = keys[i];
+        reducedTestData[testDataKey] = fromTestDataObj[testDataKey];
+    }
+    return reducedTestData;
+}
+
+
 module.exports = {
     ALL_RECORDS     : TEST_INPUT,
     POSITIVE_ONLY: _TEST_INPUT_POSITIVE_CASES,
     NEGATIVE_ONLY: _TEST_INPUT_NEGATIVE_CASES,
+    reduceTestRecords: reduceTestRecords,
 };
\ No newline at end of file

From 2884b86d88bac9fd9d0a669a127c0c92365d8f2d Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Thu, 22 Aug 2024 19:07:47 -0700
Subject: [PATCH 14/22] Added end-to-end test when overwrite on upload is
 disabled

---
 .../tests/003-overwriting-disabled.test.js    | 96 +++++++++++++++++++
 test/jest.run-all-tests.js                    |  3 +-
 2 files changed, 98 insertions(+), 1 deletion(-)
 create mode 100644 test/end2end/tests/003-overwriting-disabled.test.js

diff --git a/test/end2end/tests/003-overwriting-disabled.test.js b/test/end2end/tests/003-overwriting-disabled.test.js
new file mode 100644
index 0000000..76b60b0
--- /dev/null
+++ b/test/end2end/tests/003-overwriting-disabled.test.js
@@ -0,0 +1,96 @@
+const fs = require('fs');
+const path = require('path');
+const testAppInput = require('../app-input');
+const testAppFlow = require('../test-invoke-app-flow');
+const migrationPayload = require('../../../lib/payload/migrate');
+const testMigrationRecords = require('../yield-e2e-migration-records');
+
+const INPUT_CSV_FILE = path.join(__dirname, 'input.csv');
+const TEST_OUTPUT_FOLDER = path.join(__dirname, 'test-output');
+
+// Mocking the CSV input to API payload conversion logic to match the 
+// produced CSV input for the test
+jest.mock('../../../__input-to-api-payload', () => {
+    return {
+        input2ApiPayload: jest.fn((csvRec) => {
+            return {
+                file: csvRec.Ref,
+                options: {
+                    public_id: csvRec.public_id,
+                    unique_filename: false,
+                    overwrite: false, // << Disable overwriting assets
+                    resource_type: 'auto',
+                    type: 'upload',        
+                }
+            };
+        })
+    };
+});
+
+
+async function cleanup() {
+    await testAppFlow.testCleanup_Async({
+        input_csv_file: INPUT_CSV_FILE,
+        test_output_folder: TEST_OUTPUT_FOLDER,
+    });
+}
+
+// Variables to reference records from the parsed migration log and report files
+let __TEST_LOG = null;
+let __TEST_REPORT = null;
+
+// Reduce number of test records to speed up execution
+
+let __TEST_DATA = testMigrationRecords.reduceTestRecords(
+    10, 
+    testMigrationRecords.POSITIVE_ONLY
+);
+
+describe('Overwriting enabled', () => {
+    beforeAll(async () => {
+        console.log('Preparing test environment');
+        // Ensuring there are no artifacts from prior test run that could interfere
+        await cleanup();
+        
+        console.log('Serializing test input to CSV file...');
+        testAppInput.testInput2CsvFile({
+            test_input: __TEST_DATA,
+            csv_file_path: INPUT_CSV_FILE,
+        });
+
+        // Invoking the main loop for E2E testing
+        const { testLog, testReport } = await testAppFlow.invokeMainLoopForTest_Async(
+            { // Mocking CLI args
+                fromCsvFile: INPUT_CSV_FILE,
+                maxConcurrentUploads: 2,
+                outputFolder: TEST_OUTPUT_FOLDER,
+            },
+            { // Mocking CLI command
+                name: () => 'migrate',
+            },
+            migrationPayload
+        );
+        
+        __TEST_LOG = testLog;
+        __TEST_REPORT = testReport;
+
+        console.log('Done preparing test environment');
+    }, 5*60*1000); // Explicitly setting timeout to allow for execution of the migration loop
+
+    afterAll(async () => {
+        await cleanup();
+    });
+
+    test.each(
+        Object.keys(__TEST_DATA)
+    )('Should report existing asset as overwritten %s', (public_id) => {
+        const testReportEntries = __TEST_REPORT.getEntriesByPublicId(public_id);
+        expect(testReportEntries.length).toEqual(1);
+        const testReportEntry = testReportEntries[0];
+        expect(testReportEntry.Cld_PublicId).toEqual(public_id);
+        expect(testReportEntry.Cld_Status).toEqual('MIGRATED');
+        expect(testReportEntry.Cld_Operation).toEqual('SkippedAlreadyExists');
+    });
+
+});
+
diff --git a/test/jest.run-all-tests.js b/test/jest.run-all-tests.js
index cf421b5..1160a10 100644
--- a/test/jest.run-all-tests.js
+++ b/test/jest.run-all-tests.js
@@ -43,7 +43,8 @@ function runTestsFrom(testPath) {
 
     // Running end-to-end tests in order
     runTestsFrom('./test/end2end/tests/001-initial-migration.test.js');
-    runTestsFrom('./test/end2end/tests/002-overwriting-enabled.test.js');    
+    runTestsFrom('./test/end2end/tests/002-overwriting-enabled.test.js');
+    runTestsFrom('./test/end2end/tests/003-overwriting-disabled.test.js');
 
     await runGlobalTeardown_Async();
 })();

From 1d8bb0fd9282772af26dfff539c0e3aa83d2ccbe Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Thu, 22 Aug 2024 19:27:39 -0700
Subject: [PATCH 15/22] Added explicit reporting for those assets that already
 exist and were not overwritten due to `overwite:false` set on upload api call

---
 __log-to-report.js | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/__log-to-report.js b/__log-to-report.js
index a24c2d0..7299b5a 100644
--- a/__log-to-report.js
+++ b/__log-to-report.js
@@ -46,7 +46,12 @@ function extractMigrationFlowRecord(logLine) {
             }
             migrationSummaryRec.Cld_Error = errInfo;
         } else {
-            migrationSummaryRec.Cld_Operation = logRec.response.overwritten ? 'Overwritten' : 'Uploaded';
+            // Resolving operation
+            let resolvedMigrationOp = 'Uploaded';
+            if (logRec.response.existing    === true) { resolvedMigrationOp = 'SkippedAlreadyExists' }
+            if (logRec.response.overwritten === true) { resolvedMigrationOp = 'Overwritten' }
+ 
+            migrationSummaryRec.Cld_Operation = resolvedMigrationOp;
             migrationSummaryRec.Cld_PublicId = logRec.response.public_id;
             migrationSummaryRec.Cld_Etag = logRec.response.etag;
         }

From a4f1a1e5c12aec7e1eaf1d998ebb212a2a9c070f Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Thu, 22 Aug 2024 19:28:10 -0700
Subject: [PATCH 16/22] Ensured teardown is executed when a test fails

---
 test/jest.run-all-tests.js | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/test/jest.run-all-tests.js b/test/jest.run-all-tests.js
index 1160a10..11fd141 100644
--- a/test/jest.run-all-tests.js
+++ b/test/jest.run-all-tests.js
@@ -34,18 +34,21 @@ function runTestsFrom(testPath) {
 
 // Invoking tests
 (async () => {
-    await runGlobalSetup_Async();
-
-    
-    // It is assumed that all component tests are next to components
-    // under the 'lib' folder
-    runTestsFrom('./lib/');
-
-    // Running end-to-end tests in order
-    runTestsFrom('./test/end2end/tests/001-initial-migration.test.js');
-    runTestsFrom('./test/end2end/tests/002-overwriting-enabled.test.js');
-    runTestsFrom('./test/end2end/tests/003-overwriting-disabled.test.js');
-
-    await runGlobalTeardown_Async();
+    try{
+        await runGlobalSetup_Async();
+
+        // It is assumed that all component tests are next to components
+        // under the 'lib' folder
+        runTestsFrom('./lib/');
+
+        // Running end-to-end tests in order
+        runTestsFrom('./test/end2end/tests/001-initial-migration.test.js');
+        runTestsFrom('./test/end2end/tests/002-overwriting-enabled.test.js');
+        runTestsFrom('./test/end2end/tests/003-overwriting-disabled.test.js');
+    } catch (error) {
+        console.error('Error running tests:', error);
+    } finally {
+        await runGlobalTeardown_Async();
+    }
 })();
 

From 4d522ea6ec95c46b00df629db33a5d7c9a4fd624 Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Thu, 22 Aug 2024 19:28:32 -0700
Subject: [PATCH 17/22] Added more info on end-to-end tests implementation

---
 readme/dev/readme.md | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/readme/dev/readme.md b/readme/dev/readme.md
index 3c93246..912d7e9 100644
--- a/readme/dev/readme.md
+++ b/readme/dev/readme.md
@@ -8,6 +8,16 @@ All of the app components (and tests for them) are expected to be under the `lib
 This is important because tests are explicitly invoked from under the folder
 
 ### App Tests
+jest test framework is used. Use `npm test` to run all tests (unit tests and end-to-end tests).
+
+Custom configuration is used - see the `scripts.test` in `package.json`.
+
 Test modules are maintained under the `test` folder.
 
-End to end tests require to be executed in certain order (see the `test/jest.run-all-tests.js` file)
\ No newline at end of file
+
+## End2End Tests
+End to end tests require to be executed in certain order (see the `test/jest.run-all-tests.js` file).
+
+- a temporary cloud is provisioned when running tests
+- if you want to use a different cloud (for debugging purposes, for example) - place `.env` file with Cloudinary credentials under the `test/end2end` folder
+    * then the test migration operations will be performed against that cloud

From 550ff4075f8f4ea2f3e04e86ff2cea06df95c8af Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Tue, 12 Nov 2024 17:14:16 -0800
Subject: [PATCH 18/22] Updated sample payload with more examples, added
 comments

---
 __input-to-api-payload.js | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/__input-to-api-payload.js b/__input-to-api-payload.js
index 7be7051..2bff6ed 100644
--- a/__input-to-api-payload.js
+++ b/__input-to-api-payload.js
@@ -21,16 +21,26 @@
  *  - options: options for the Cloudinary Upload API call
  */
 exports.input2ApiPayload = function(csvRec) {
-    const file = csvRec.Url;
-    const options = {
-        public_id: csvRec.Id,
-        unique_filename: false,
-        resource_type: 'auto',
-        type: 'upload',
-        tags: csvRec.Tags,
+    // Pass value from 'Url' column with the asset URLs or paths
+    const file = csvRec['Url'];
+    
+    // Optional parameters for the Cloudinary API
+    const options = {                       
+        public_id:       csvRec['Id'],            // Pass value from 'Id' column to be used as public_id
+        unique_filename: false,                   // Do not add random suffix to the public_id
+        resource_type:   'auto',                  // Let Cloudinary determine the resource type
+        overwrite:       false,                   // Do not overwrite the asset with same public_id if it already exists
+        type:            'upload',                // Explicitly set delivery type
+        tags:            csvRec['Tags'],          // Pass value from 'Tags' column as tags
+
         context: {
-            caption: csvRec.Description,
-        }
+            caption: csvRec['Description'],       // Pass value from 'Description' column as contextual metadata
+        },
+
+        metadata: {
+            sample_field: csvRec['SampleField'],  // Pass value from 'SampleField' column into the structured metadata field
+                                                  // with external_id of 'sample_field'
+        },
     };
 
     return { file, options };

From 8f34ff29868a6daeffecced704884a61ac771fd6 Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Tue, 12 Nov 2024 17:28:45 -0800
Subject: [PATCH 19/22] Updated README with instructions for scripts to monitor
 an ongoing migration

---
 readme/run-migration-obtain-report.md | 35 +++++++++++++++++++++------
 1 file changed, 28 insertions(+), 7 deletions(-)

diff --git a/readme/run-migration-obtain-report.md b/readme/run-migration-obtain-report.md
index 37aa030..c118083 100644
--- a/readme/run-migration-obtain-report.md
+++ b/readme/run-migration-obtain-report.md
@@ -35,17 +35,38 @@ node ./cld-bulk.js migrate \
 
 # Monitoring for errors
 
-The migration script keeps updating the `log.json` file in the specified output folder.
+The migration script keeps updating the `log.jsonl` file in the specified output folder.
 
 Oftentime when errors do occur it is helpful to know what types of errors those are (network "hiccups" or incorrect upload API parameters).
 
-If you would like to monitor for errors in the log file during the script execution you can adjust the following command:
+To assist with monitoring an ongoing bulk migration the following scripts are available with the tool.
+
+## Script to count tally of operations performed so far
 
 ```bash
-#
-# Make sure to replace <input-field-name> with a column name from the input CSV file 
-# (for example, the one you use to pass `public_id` for the asset)
-#
+# Assumes you are in the root folder of the repository
+monitor-migration/count-log-ops.sh  path/to/ongoing/migration/log.jsonl
+```
+
+Provides output such as:
+```
+🟢 Created            : 987
+🟡 Overwritten        : 65
+⚪️ Existing (skipped) : 43
+🔴 Failed             : 21
+```
 
-tail -f log.jsonl | jq -r 'select(.summary.status != "MIGRATED") | [.input.<input-field-name>, .summary.status, "--", .summary.err] | join("  ")'
+## Script to output information for all failed operations
+```bash
+# Assumes you are in the root folder of the repository
+monitor-migration/trace-failed-log-ops.sh  path/to/ongoing/migration/log.jsonl
+```
+
+Provides output such as:
+```
+{"error":{"message":"Request Timeout","http_code":499,"name":"TimeoutError"}}
+{"message":"Error in loading https://test.img/url - 503 Service Unavailable","name":"Error","http_code":400}
+{"error":{"message":"Request Timeout","http_code":499,"name":"TimeoutError"}}
+{"error":{"message":"Request Timeout","http_code":499,"name":"TimeoutError"}}
+{"message":"Server returned unexpected status code - 502","http_code":502,"name":"UnexpectedResponse"}
 ```

From d00fbad6da918404134fa89c620fd3208124c970 Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Tue, 12 Nov 2024 17:30:16 -0800
Subject: [PATCH 20/22] Used `npm audit fix` to update outdated dependencies

---
 package-lock.json | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 989a3f5..fdc6a6e 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1326,12 +1326,13 @@
       }
     },
     "node_modules/braces": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
-      "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
+      "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "fill-range": "^7.0.1"
+        "fill-range": "^7.1.1"
       },
       "engines": {
         "node": ">=8"
@@ -1850,10 +1851,11 @@
       }
     },
     "node_modules/fill-range": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
-      "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
+      "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "to-regex-range": "^5.0.1"
       },
@@ -2094,6 +2096,7 @@
       "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
       "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
       "dev": true,
+      "license": "MIT",
       "engines": {
         "node": ">=0.12.0"
       }
@@ -2966,12 +2969,13 @@
       "dev": true
     },
     "node_modules/micromatch": {
-      "version": "4.0.5",
-      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz",
-      "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==",
+      "version": "4.0.8",
+      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
+      "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "braces": "^3.0.2",
+        "braces": "^3.0.3",
         "picomatch": "^2.3.1"
       },
       "engines": {
@@ -3649,6 +3653,7 @@
       "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
       "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "is-number": "^7.0.0"
       },

From fe515bfa8c98a6d61526b00703e75d68ae00a503 Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Tue, 12 Nov 2024 17:40:48 -0800
Subject: [PATCH 21/22] Updated tool version

---
 cld-bulk.js       | 2 +-
 package-lock.json | 4 ++--
 package.json      | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cld-bulk.js b/cld-bulk.js
index b470c13..26032d5 100755
--- a/cld-bulk.js
+++ b/cld-bulk.js
@@ -57,7 +57,7 @@ function configureProgram(program) {
     program
         .name('cld-bulk')
         .description('Extensible CLI tool to efficiently translate CSV file records into Cloudinary API operations')
-        .version('2.0.1');
+        .version('2.1.0');
 }
 
 
diff --git a/package-lock.json b/package-lock.json
index fdc6a6e..999e511 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "cld-bulk",
-  "version": "2.0.1",
+  "version": "2.1.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "cld-bulk",
-      "version": "2.0.1",
+      "version": "2.1.0",
       "dependencies": {
         "async": "^3.2.4",
         "bunyan": "^1.8.15",
diff --git a/package.json b/package.json
index 5fb856f..49cec3f 100644
--- a/package.json
+++ b/package.json
@@ -21,5 +21,5 @@
   "scripts": {
     "test": "node ./test/jest.run-all-tests.js"
   },
-  "version": "2.0.1"
+  "version": "2.1.0"
 }

From 75c1d63c8913122c63ef2dc4dc8ea34b9c44587e Mon Sep 17 00:00:00 2001
From: achumachenko-cloudinary <andriy.chumachenko@cloudinary.com>
Date: Tue, 12 Nov 2024 17:48:11 -0800
Subject: [PATCH 22/22] Updated changelog

---
 CHANGELOG.md | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9c6f0bb..96898cb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,17 @@
 # CHANGELOG
 
+
+[2.1.0] / 2024-11-12
+====================
+
+### Added
+- Scripts to monitor logs for an ongoing bulk operation with instructions in README
+- Explicitly reporting API responses with `existing` property as `SkippedAlreadyExists` value for the `Cld_Operation` column in a migration report file
+
+### Changed
+- Updated outdated dependencies
+
+
 [2.0.1] / 2024-02-09
 ====================