Add v3.31.3

YannickRe · Aug 31, 2020 · b924fae · b924fae
1 parent fe7c0c2
commit b924fae
Show file tree

Hide file tree

Showing 21 changed files with 622 additions and 550 deletions.
diff --git a/...dark-7ca0db005c1563c214d1f9a674ce346d.css → ...dark-238f8ca6b00a0d7814a199863b3cf875.css b/...dark-7ca0db005c1563c214d1f9a674ce346d.css → ...dark-238f8ca6b00a0d7814a199863b3cf875.css
diff --git a/...t.min-1e3cb8721f8cba88875c1d2c6eb575b1.js → ...t.min-0b46f9f865564fa84cad2aa0dd328a32.js b/...t.min-1e3cb8721f8cba88875c1d2c6eb575b1.js → ...t.min-0b46f9f865564fa84cad2aa0dd328a32.js
diff --git a/....min-77f6fffdad4f632d189ac7ea76ae9997.css → ....min-47936cacd85040b08580d15467323266.css b/....min-77f6fffdad4f632d189ac7ea76ae9997.css → ....min-47936cacd85040b08580d15467323266.css
diff --git a/...r.min-7db2bd11ae29fc44c51dfce4ad6641c5.js → ...r.min-5fd46fff106ec80fe8ce28621e77f569.js b/...r.min-7db2bd11ae29fc44c51dfce4ad6641c5.js → ...r.min-5fd46fff106ec80fe8ce28621e77f569.js
diff --git a/core/server/api/canary/members.js b/core/server/api/canary/members.js
@@ -7,6 +7,7 @@ const config = require('../../../shared/config');
 const models = require('../../models');
 const membersService = require('../../services/members');
 const doImport = require('../../services/members/importer');
+const memberLabelsImporter = require('../../services/members/importer/labels');
 const settingsCache = require('../../services/settings/cache');
 const {i18n} = require('../../lib/common');
 const logging = require('../../../shared/logging');
@@ -125,75 +126,11 @@ const sanitizeInput = async (members) => {
     return {
         sanitized,
         invalidCount,
-        validationErrors
+        validationErrors,
+        duplicateStripeCustomersCount
     };
 };
 
-function serializeMemberLabels(labels) {
-    if (_.isString(labels)) {
-        if (labels === '') {
-            return [];
-        }
-
-        return [{
-            name: labels.trim()
-        }];
-    } else if (labels) {
-        return labels.filter((label) => {
-            return !!label;
-        }).map((label) => {
-            if (_.isString(label)) {
-                return {
-                    name: label.trim()
-                };
-            }
-            return label;
-        });
-    }
-    return [];
-}
-
-const findOrCreateLabels = async (labels, options) => {
-    const api = require('./index');
-
-    return await Promise.all(labels.map((label) => {
-        return models.Label.findOne({name: label.name}).then((existingLabel) => {
-            if (existingLabel) {
-                return existingLabel;
-            }
-
-            return api.labels.add.query({
-                data: {
-                    labels: [label]
-                },
-                options: {
-                    context: options.context
-                }
-            }).catch((error) => {
-                if (error.errorType === 'ValidationError') {
-                    return;
-                }
-
-                throw error;
-            });
-        });
-    }));
-};
-
-const getUniqueMemberLabels = (members) => {
-    const allLabels = [];
-
-    members.forEach((member) => {
-        const labels = (member.labels && member.labels.split(',')) || [];
-
-        if (labels.length) {
-            allLabels.push(...labels);
-        }
-    });
-
-    return _.uniq(allLabels);
-};
-
 module.exports = {
     docName: 'members',
 
@@ -528,30 +465,17 @@ module.exports = {
             };
             let duplicateStripeCustomerIdCount = 0;
 
-            // NOTE: custom labels have to be created in advance otherwise there are conflicts
-            //       when processing member creation in parallel later on in import process
-            const importSetLabels = serializeMemberLabels(frame.data.labels);
-            await findOrCreateLabels(importSetLabels, frame.options);
-
-            // NOTE: adding an import label allows for imports to be "undone" via bulk delete
-            let importLabel;
-            if (frame.data.members.length) {
-                const siteTimezone = settingsCache.get('timezone');
-                const name = `Import ${moment().tz(siteTimezone).format('YYYY-MM-DD HH:mm')}`;
-                const result = await findOrCreateLabels([{name}], frame.options);
-                importLabel = result[0] && result[0].toJSON();
-
-                importSetLabels.push(importLabel);
-            }
-
-            // NOTE: member-specific labels have to be pre-created as they cause conflicts when processed
-            //       in parallel
-            const memberLabels = serializeMemberLabels(getUniqueMemberLabels(frame.data.members));
-            await findOrCreateLabels(memberLabels, frame.options);
+            let {importSetLabels, importLabel} = await memberLabelsImporter.handleAllLabels(
+                frame.data.labels,
+                frame.data.members,
+                settingsCache.get('timezone'),
+                frame.options
+            );
 
             return Promise.resolve().then(async () => {
-                const {sanitized, invalidCount, validationErrors} = await sanitizeInput(frame.data.members);
+                const {sanitized, invalidCount, validationErrors, duplicateStripeCustomersCount} = await sanitizeInput(frame.data.members);
                 invalid.count += invalidCount;
+                duplicateStripeCustomerIdCount = duplicateStripeCustomersCount;
 
                 if (validationErrors.length) {
                     invalid.errors.push(...validationErrors);
@@ -560,7 +484,7 @@ module.exports = {
                 return Promise.map(sanitized, ((entry) => {
                     const api = require('./index');
                     entry.labels = (entry.labels && entry.labels.split(',')) || [];
-                    const entryLabels = serializeMemberLabels(entry.labels);
+                    const entryLabels = memberLabelsImporter.serializeMemberLabels(entry.labels);
                     const mergedLabels = _.unionBy(entryLabels, importSetLabels, 'name');
 
                     cleanupUndefined(entry);
@@ -634,7 +558,7 @@ module.exports = {
 
                 invalid.errors = outputErrors;
 
-                if (imported.count === 0 && importLabel) {
+                if (imported.count === 0 && importLabel && importLabel.generated) {
                     await models.Label.destroy(Object.assign({}, {id: importLabel.id}, frame.options));
                     importLabel = null;
                 }
@@ -681,41 +605,25 @@ module.exports = {
 
             const createdBy = contextUser(frame.options);
 
-            // NOTE: custom labels have to be created in advance otherwise there are conflicts
-            //       when processing member creation in parallel later on in import process
-            const importSetLabels = serializeMemberLabels(frame.data.labels);
-
-            // NOTE: adding an import label allows for imports to be "undone" via bulk delete
-            let importLabel;
-            if (frame.data.members.length) {
-                const siteTimezone = settingsCache.get('timezone');
-                const name = `Import ${moment().tz(siteTimezone).format('YYYY-MM-DD HH:mm')}`;
-                const result = await findOrCreateLabels([{name}], frame.options);
-                importLabel = result[0] && result[0].toJSON();
-
-                importSetLabels.push(importLabel);
-            }
-
-            const importSetLabelModels = await findOrCreateLabels(importSetLabels, frame.options);
-
-            // NOTE: member-specific labels have to be pre-created as they cause conflicts when processed
-            //       in parallel
-            const memberLabels = serializeMemberLabels(getUniqueMemberLabels(frame.data.members));
-            const memberLabelModels = await findOrCreateLabels(memberLabels, frame.options);
-
-            const allLabelModels = [...importSetLabelModels, ...memberLabelModels].filter(model => model !== undefined);
+            let {allLabels, importSetLabels, importLabel} = await memberLabelsImporter.handleAllLabels(
+                frame.data.labels,
+                frame.data.members,
+                settingsCache.get('timezone'),
+                frame.options
+            );
 
             return Promise.resolve().then(async () => {
-                const {sanitized, invalidCount, validationErrors} = await sanitizeInput(frame.data.members);
+                const {sanitized, invalidCount, validationErrors, duplicateStripeCustomersCount} = await sanitizeInput(frame.data.members);
                 invalid.count += invalidCount;
+                duplicateStripeCustomerIdCount = duplicateStripeCustomersCount;
 
                 if (validationErrors.length) {
                     invalid.errors.push(...validationErrors);
                 }
 
                 return doImport({
                     members: sanitized,
-                    allLabelModels,
+                    labels: allLabels,
                     importSetLabels,
                     createdBy
                 });
@@ -746,7 +654,7 @@ module.exports = {
 
                 invalid.errors = outputErrors;
 
-                if (imported.count === 0 && importLabel) {
+                if (imported.count === 0 && importLabel && importLabel.generated) {
                     await models.Label.destroy(Object.assign({}, {id: importLabel.id}, frame.options));
                     importLabel = null;
                 }

diff --git a/core/server/api/canary/utils/validators/input/schemas/members-upload.json b/core/server/api/canary/utils/validators/input/schemas/members-upload.json
@@ -27,8 +27,8 @@
                 "maxLength": 2000
             },
             "subscribed": {
-                "type": ["string"],
-                "enum": ["true", "false", "TRUE", "FALSE", ""]
+                "type": ["string", "null"],
+                "enum": ["true", "false", "TRUE", "FALSE", "", null]
             },
             "labels": {
                 "type": ["string", "null"]
@@ -41,8 +41,8 @@
                 "type": ["string", "null"]
             },
             "complimentary_plan": {
-                "type": ["string"],
-                "enum": ["true", "false", "TRUE", "FALSE", ""]
+                "type": ["string", "null"],
+                "enum": ["true", "false", "TRUE", "FALSE", "", null]
             }
         }
     }

diff --git a/core/server/api/v2/utils/serializers/output/utils/clean.js b/core/server/api/v2/utils/serializers/output/utils/clean.js
@@ -12,7 +12,8 @@ const tag = (attrs, frame) => {
             'name',
             'slug',
             'url',
-            'visibility'
+            'visibility',
+            'count'
         ]);
 
         // We are standardising on returning null from the Content API for any empty values
@@ -25,6 +26,8 @@ const tag = (attrs, frame) => {
         if (contentAttrs.description === '') {
             contentAttrs.description = null;
         }
+
+        return contentAttrs;
     }
 
     return _.pick(attrs, [
@@ -38,7 +41,8 @@ const tag = (attrs, frame) => {
         'slug',
         'updated_at',
         'url',
-        'visibility'
+        'visibility',
+        'count'
     ]);
 };
 

diff --git a/...vices/members/importer/bulk-operations.js → core/server/models/base/bulk-operations.js b/...vices/members/importer/bulk-operations.js → core/server/models/base/bulk-operations.js
@@ -1,5 +1,7 @@
 const _ = require('lodash');
-const db = require('../../../data/db');
+const errors = require('@tryghost/errors');
+const db = require('../../data/db');
+const logging = require('../../../shared/logging');
 
 const CHUNK_SIZE = 100;
 
@@ -9,8 +11,9 @@ async function insertChunkSequential(table, chunk, result) {
             await db.knex(table).insert(record);
             result.successful += 1;
         } catch (err) {
+            err.errorDetails = record;
             result.errors.push(err);
-            result.unsuccessfulIds.push(record.id);
+            result.unsuccessfulRecords.push(record);
             result.unsuccessful += 1;
         }
     }
@@ -29,7 +32,7 @@ async function insert(table, data) {
     const result = {
         successful: 0,
         unsuccessful: 0,
-        unsuccessfulIds: [],
+        unsuccessfulRecords: [],
         errors: []
     };
 
@@ -41,13 +44,20 @@ async function insert(table, data) {
 }
 
 async function delChunkSequential(table, chunk, result) {
-    for (const record of chunk) {
+    for (const id of chunk) {
         try {
-            await db.knex(table).where('id', record).del();
+            await db.knex(table).where('id', id).del();
             result.successful += 1;
         } catch (err) {
-            result.errors.push(err);
-            result.unsuccessfulIds.push(record);
+            const importError = new errors.DataImportError({
+                message: `Failed to remove entry from ${table}`,
+                context: `Entry id: ${id}`,
+                err: err
+            });
+            logging.error(importError);
+
+            result.errors.push(importError);
+            result.unsuccessfulIds.push(id);
             result.unsuccessful += 1;
         }
     }