Merge pull request #11 from cloudinary/2.1.0-release

2.1.0 release

CHANGELOG.md

@@ -1,5 +1,17 @@
 # CHANGELOG
 
+
+[2.1.0] / 2024-11-12
+====================
+
+### Added
+- Scripts to monitor logs for an ongoing bulk operation with instructions in README
+- Explicitly reporting API responses with `existing` property as `SkippedAlreadyExists` value for the `Cld_Operation` column in a migration report file
+
+### Changed
+- Updated outdated dependencies
+
+
 [2.0.1] / 2024-02-09
 ====================
 

README.md

@@ -28,3 +28,6 @@ Follow these steps to successfully migrate your assets:
 3. [⚙️ Configure the Script](./readme/configure.md) - Customize the script's settings for your specific migration needs.
 4. [🚚 Run the Script and Obtain the Report](./readme/run-migration-obtain-report.md) - Execute the script and review the migration report.
 5. [🔄 Iterate for Failed Migrations](./readme/identify-reattempt-failed.md) - Identify failed asset migrations and rerun the script to fix them.
+
+# How to Tweak It
+Things to know are covered in the [🧑‍💻 dev readme](./readme/dev/readme.md).

__input-to-api-payload.js

@@ -21,16 +21,26 @@
  *  - options: options for the Cloudinary Upload API call
  */
 exports.input2ApiPayload = function(csvRec) {
-    const file = csvRec.Url;
-    const options = {
-        public_id: csvRec.Id,
-        unique_filename: false,
-        resource_type: 'auto',
-        type: 'upload',
-        tags: csvRec.Tags,
+    // Pass value from 'Url' column with the asset URLs or paths
+    const file = csvRec['Url'];
+
+    // Optional parameters for the Cloudinary API
+    const options = {                       
+        public_id:       csvRec['Id'],            // Pass value from 'Id' column to be used as public_id
+        unique_filename: false,                   // Do not add random suffix to the public_id
+        resource_type:   'auto',                  // Let Cloudinary determine the resource type
+        overwrite:       false,                   // Do not overwrite the asset with same public_id if it already exists
+        type:            'upload',                // Explicitly set delivery type
+        tags:            csvRec['Tags'],          // Pass value from 'Tags' column as tags
+
         context: {
-            caption: csvRec.Description,
-        }
+            caption: csvRec['Description'],       // Pass value from 'Description' column as contextual metadata
+        },
+
+        metadata: {
+            sample_field: csvRec['SampleField'],  // Pass value from 'SampleField' column into the structured metadata field
+                                                  // with external_id of 'sample_field'
+        },
     };
 
     return { file, options };

__log-to-report.js

@@ -46,7 +46,12 @@ function extractMigrationFlowRecord(logLine) {
             }
             migrationSummaryRec.Cld_Error = errInfo;
         } else {
-            migrationSummaryRec.Cld_Operation = logRec.response.overwritten ? 'Overwritten' : 'Uploaded';
+            // Resolving operation
+            let resolvedMigrationOp = 'Uploaded';
+            if (logRec.response.existing    === true) { resolvedMigrationOp = 'SkippedAlreadyExists' }
+            if (logRec.response.overwritten === true) { resolvedMigrationOp = 'Overwritten' }
+
+            migrationSummaryRec.Cld_Operation = resolvedMigrationOp;
             migrationSummaryRec.Cld_PublicId = logRec.response.public_id;
             migrationSummaryRec.Cld_Etag = logRec.response.etag;
         }

cld-bulk.js

@@ -57,7 +57,7 @@ function configureProgram(program) {
     program
         .name('cld-bulk')
         .description('Extensible CLI tool to efficiently translate CSV file records into Cloudinary API operations')
-        .version('2.0.1');
+        .version('2.1.0');
 }
 
 

monitor-migration/count-log-ops.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# File to be monitored
+LOGFILE=$1
+
+# Processing payload entries from the log to produce tally
+grep -F '"flow":"payload"' $LOGFILE | \
+awk '
+    BEGIN {
+        # Explicitly initialize variables to 0
+        created = 0
+        overwritten = 0
+        existing = 0
+        failure = 0
+    }    
+    {
+        migrated = index($0, "\"status\":\"MIGRATED\"")
+        if (migrated) {
+            if (index($0, "\"existing\":true")) {
+                existing++
+            } else if (index($0, "\"overwritten\":true")) {
+                overwritten++
+            } else {
+                created++
+            }
+        } else {
+            failure++
+        }
+    }    
+    END {
+        print "🟢 Created            : ", created
+        print "🟡 Overwritten        : ", overwritten
+        print "⚪️ Existing (skipped) : ", existing
+        print "🔴 Failed             : ", failure
+    }
+'

monitor-migration/trace-failed-log-ops.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Log file to be scanned
+LOGFILE=$1
+
+# Processing payload entries with status different from MIGRATED
+grep -F '"flow":"payload"' $LOGFILE | grep -F -v '"status":"MIGRATED"' | \
+jq -r '(.summary.err | tostring)'

package.json

@@ -19,7 +19,7 @@
     "jest": "^29.6.2"
   },
   "scripts": {
-    "test": "jest --verbose"
+    "test": "node ./test/jest.run-all-tests.js"
   },
-  "version": "2.0.1"
+  "version": "2.1.0"
 }

readme/dev/readme.md

@@ -0,0 +1,23 @@
+# Things to know about 
+
+## Layout
+
+### App Components
+All of the app components (and tests for them) are expected to be under the `lib` folder.
+
+This is important because tests are explicitly invoked from under the folder
+
+### App Tests
+jest test framework is used. Use `npm test` to run all tests (unit tests and end-to-end tests).
+
+Custom configuration is used - see the `scripts.test` in `package.json`.
+
+Test modules are maintained under the `test` folder.
+
+
+## End2End Tests
+End to end tests require to be executed in certain order (see the `test/jest.run-all-tests.js` file).
+
+- a temporary cloud is provisioned when running tests
+- if you want to use a different cloud (for debugging purposes, for example) - place `.env` file with Cloudinary credentials under the `test/end2end` folder
+    * then the test migration operations will be performed against that cloud

readme/run-migration-obtain-report.md

@@ -35,17 +35,38 @@ node ./cld-bulk.js migrate \
 
 # Monitoring for errors
 
-The migration script keeps updating the `log.json` file in the specified output folder.
+The migration script keeps updating the `log.jsonl` file in the specified output folder.
 
 Oftentime when errors do occur it is helpful to know what types of errors those are (network "hiccups" or incorrect upload API parameters).
 
-If you would like to monitor for errors in the log file during the script execution you can adjust the following command:
+To assist with monitoring an ongoing bulk migration the following scripts are available with the tool.
+
+## Script to count tally of operations performed so far
 
 ```bash
-#
-# Make sure to replace <input-field-name> with a column name from the input CSV file 
-# (for example, the one you use to pass `public_id` for the asset)
-#
+# Assumes you are in the root folder of the repository
+monitor-migration/count-log-ops.sh  path/to/ongoing/migration/log.jsonl
+```
+
+Provides output such as:
+```
+🟢 Created            : 987
+🟡 Overwritten        : 65
+⚪️ Existing (skipped) : 43
+🔴 Failed             : 21
+```
 
-tail -f log.jsonl | jq -r 'select(.summary.status != "MIGRATED") | [.input.<input-field-name>, .summary.status, "--", .summary.err] | join("  ")'
+## Script to output information for all failed operations
+```bash
+# Assumes you are in the root folder of the repository
+monitor-migration/trace-failed-log-ops.sh  path/to/ongoing/migration/log.jsonl
+```
+
+Provides output such as:
+```
+{"error":{"message":"Request Timeout","http_code":499,"name":"TimeoutError"}}
+{"message":"Error in loading https://test.img/url - 503 Service Unavailable","name":"Error","http_code":400}
+{"error":{"message":"Request Timeout","http_code":499,"name":"TimeoutError"}}
+{"error":{"message":"Request Timeout","http_code":499,"name":"TimeoutError"}}
+{"message":"Server returned unexpected status code - 502","http_code":502,"name":"UnexpectedResponse"}
 ```