CLD-48-organization-content #43
+411
−66
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| 'use strict'; | ||
|
|
||
| /** | ||
| * Creates valid paath argument and uploads a single asset | ||
| * | ||
| * @param {string} folder - current processing folder | ||
| * @param {Object} file - current file | ||
| * @returns {boolean} - true/false | ||
| */ | ||
| function doFile(folder, file) { | ||
| var cloudinarySvc = require('~/cartridge/scripts/service/cldUpload'); | ||
| var cloudinaryConstants = require('~/cartridge/scripts/util/cloudinaryConstants'); | ||
| var jobStepHelpers = require('~/cartridge/scripts/helpers/jobStepHelpers'); | ||
| var cloudinaryUtils = require('~/cartridge/scripts/util/cloudinaryUtils'); | ||
| var jobLogger = require('dw/system').Logger.getLogger('Cloudinary', 'UPLOAD'); | ||
|
|
||
| var url; | ||
|
|
||
| try { | ||
| if (file.path && cloudinaryUtils.isVideo(file.path, cloudinaryConstants)) { | ||
| folder = cloudinaryConstants.CLD_ORG_CONTENT_VIDEO_PATH; | ||
| } else { | ||
| folder = cloudinaryConstants.CLD_ORG_CONTENT_IMAGE_PATH; | ||
| } | ||
|
|
||
| url = cloudinaryConstants.HOST_NAME + cloudinaryConstants.ORG_CONTENT_DW_URL + file.path; | ||
| jobLogger.info('Now uploading file: {0}', file.getName()); | ||
| var assetPublicID = jobStepHelpers.getAssetRelURL(file.toString()); | ||
| var changedAssetIds = jobStepHelpers.changePublicIdAndCloudFolder(assetPublicID, folder); | ||
| assetPublicID = changedAssetIds.assetPublicID; | ||
| var svcArgs = jobStepHelpers.getCldUploadSvcArgs(); | ||
| jobStepHelpers.uploadFile(cloudinaryConstants, url, null, folder, assetPublicID, null, svcArgs, null); | ||
|
There was a problem hiding this comment. Is this where we get 401? Can you share the log line, filename and the timestamp so I can triage it in the Cloudinary logs? There was a problem hiding this comment. @asad-rafter Thanks for the code change. Please share the output of it There was a problem hiding this comment. @yuval-cloudinary output means the logs you want? There was a problem hiding this comment. yes @asad-rafter , the new logs with the 401 There was a problem hiding this comment. There was a problem hiding this comment. @yuval-cloudinary we are still getting the 401, cloudinary is still trying to access the content and it returns the 401. I have attached the latest logs. There was a problem hiding this comment. @asad-rafter Was the 401 issue resolved after removing the environment protection? |
||
|
|
||
| return true; | ||
| } catch (err) { | ||
| jobLogger.error('Processing failure: {0}', err); | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Loops over folder/file list recursively and uploads each asset | ||
| * | ||
| * @param {Object} folder - the current folder | ||
| * @param {array} arrFilelist - list of files (optional) | ||
| * @param {string} syncMode - full/delta | ||
| * @param {string} cloudinaryConstants - organization constants | ||
| */ | ||
| var processFolder = function (folder, arrFilelist, syncMode, lastJobExecution, cloudinaryUtils, cloudinaryConstants, jobLogger) { | ||
| var counter; | ||
| var file; | ||
| var files = folder.listFiles(); | ||
| var filelist = arrFilelist || []; | ||
| counter = files.length; | ||
|
|
||
| while (counter > 0) { | ||
| file = files[counter - 1]; | ||
| if (file.isDirectory()) { | ||
| jobLogger.debug('** Now processing folder: {0}', file.getName()); | ||
| filelist = processFolder(file, null, syncMode, lastJobExecution, cloudinaryUtils, cloudinaryConstants, jobLogger); | ||
| } else { | ||
| filelist.push(file); | ||
| if (cloudinaryUtils.validFile(file.getName(), cloudinaryConstants)) { | ||
| jobLogger.debug('** Now processing file: {0}', file.getName()); | ||
| // If this is a delta job, skip files already processed | ||
| if (syncMode === 'DELTA') { | ||
| if (file.lastModified() > lastJobExecution) { | ||
| doFile(folder.getFullPath(), file); | ||
| } | ||
| } else { | ||
| doFile(folder.getFullPath(), file); | ||
| } | ||
| } | ||
| } | ||
| counter--; | ||
| } | ||
| }; | ||
|
|
||
| /** | ||
| * Job's starting point uploads all image, video and raws | ||
| * present at organization level into Cloudinary DAM | ||
| * | ||
| * @param {Object} args - arguments (executionMode, debugCount) | ||
| */ | ||
| function start(args) { | ||
| var System = require('dw/system/System'); | ||
| var Transaction = require('dw/system/Transaction'); | ||
|
|
||
| var cloudinaryConstants = require('~/cartridge/scripts/util/cloudinaryConstants'); | ||
| var jobStepHelpers = require('~/cartridge/scripts/helpers/jobStepHelpers'); | ||
| var cloudinaryUtils = require('~/cartridge/scripts/util/cloudinaryUtils'); | ||
| var jobLogger = require('dw/system').Logger.getLogger('Cloudinary', 'UPLOAD'); | ||
| var File = require('dw/io/File'); | ||
|
|
||
| var debugCounter = args.debugCounter || 0; | ||
| var orgContentFolder; | ||
| var lastJobExecution = new Date(cloudinaryConstants.CLD_LAST_SYNC); | ||
| var resource; | ||
| var resources; | ||
|
|
||
| try { | ||
| if (cloudinaryConstants.CLD_ENABLED) { | ||
| if (jobStepHelpers.isStepDisabled(args)) { | ||
| return new Status(Status.OK, 'OK', 'Step disabled, skip it...'); | ||
| } | ||
| orgContentFolder = new File(cloudinaryConstants.FORWARD_SLASH + File.STATIC + cloudinaryConstants.FORWARD_SLASH); | ||
| resources = orgContentFolder.listFiles(); | ||
|
|
||
| if (resources) { | ||
| Transaction.wrap(function () { | ||
| var runTime = new Date(); | ||
| System.preferences.custom.CLDLastSyncJobExecution = runTime; | ||
| }); | ||
| for (var idx = 0; idx < resources.length; idx++) { | ||
| resource = resources[idx]; | ||
| if (resource.isFile() && cloudinaryUtils.validFile(resource.getName(), cloudinaryConstants)) { | ||
| // if the job is running in "delta" mode skip already processed files | ||
| if (args.CLDSyncMode === 'DELTA') { | ||
| if (resource.lastModified() > lastJobExecution) { | ||
| doFile(resource.getName(), resource); | ||
| } | ||
| } else { | ||
| doFile(resource.getName(), resource); | ||
| } | ||
| } else { | ||
| processFolder(resource, null, args.CLDSyncMode, lastJobExecution, cloudinaryUtils, cloudinaryConstants, jobLogger); | ||
| } | ||
|
|
||
| if (debugCounter !== 0) { | ||
| debugCounter--; | ||
| if (debugCounter === 0) { | ||
| break; | ||
| } | ||
| } | ||
| } | ||
| }; | ||
| } else { | ||
| jobLogger.error('Cloudinary is disabled currently at organization level'); | ||
| } | ||
| } catch (ex) { | ||
| jobLogger.error('Error occurred while uploading organization static content on cloudinary, error : {0}, {1}, {2}', ex, ex.fileName, ex.lineNumber); | ||
| } | ||
| }; | ||
|
|
||
| module.exports = { | ||
| Start: start | ||
| }; | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@asad-rafter Let's revert this line for log message consistency