diff --git a/package.json b/package.json index 44765a650..fb5a762fc 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "compile": "tsc", "build": "node ./scripts/dump_git_info.js && react-scripts build", "react-start": "node ./scripts/dump_git_info.js && react-scripts start", - "test": "react-scripts test --env=jsdom --silent", + "test": "react-scripts test --env=jsdom", "eject": "react-scripts eject", "webpack:dev": "webpack --config ./config/webpack.dev.js", "webpack:prod": "webpack --config ./config/webpack.prod.js", @@ -108,6 +108,7 @@ "enzyme-adapter-react-16": "^1.15.1", "eslint-utils": "^1.4.3", "foreman": "^3.0.1", + "jest-enzyme": "^7.1.2", "kind-of": "^6.0.3", "mime": "^2.4.6", "minimist": "^1.2.2", diff --git a/src/common/queueMap/queue.ts b/src/common/queueMap/queue.ts new file mode 100644 index 000000000..5cbf77549 --- /dev/null +++ b/src/common/queueMap/queue.ts @@ -0,0 +1,16 @@ +export type Args = any[]; + +export interface IQueue { + queue: Args[]; + isLooping: boolean; + promise?: Promise; +} + +export class Queue implements IQueue { + queue: Args[]; + isLooping: boolean; + constructor() { + this.queue = []; + this.isLooping = false; + } +} diff --git a/src/common/queueMap/queueMap.test.ts b/src/common/queueMap/queueMap.test.ts new file mode 100644 index 000000000..14e547b82 --- /dev/null +++ b/src/common/queueMap/queueMap.test.ts @@ -0,0 +1,114 @@ +import QueueMap from "./queueMap"; + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +describe("QueueMap", () => { + test("dequeueUntilLast", () => { + const queueMap = new QueueMap(); + const queueId = "1"; + const a = ["a", 1]; + const b = ["b", 2]; + queueMap.enque(queueId, a); + queueMap.enque(queueId, b); + queueMap.dequeueUntilLast(queueId); + const { queue } = queueMap.getQueueById(queueId); + expect([b]).toEqual(queue); + }) + test("call enque while looping items in the queue", async () => { + const queueMap = new QueueMap(); + const mockWrite = jest.fn(); + const queueId = "1"; + const sleepThenReturn = ms => async (...params) => { + await mockWrite(...params); + await sleep(ms); + } + const a = ["a", 1]; + const b = ["b", 2]; + const c = ["c", 3]; + const d = ["d", 4]; + const expected = [b, d] + queueMap.enque(queueId, a); + queueMap.enque(queueId, b); + queueMap.on(queueId, sleepThenReturn(1000), params => params); + queueMap.enque(queueId, c); + queueMap.enque(queueId, d); + await sleep(2000); + expect(mockWrite.mock.calls.length).toBe(2); + expect([mockWrite.mock.calls[0], mockWrite.mock.calls[1]]).toEqual(expected); + }) + test("prevent call on twice.", async () => { + const queueMap = new QueueMap(); + const queueId = "1"; + const mockWrite = jest.fn(); + const sleepThenReturn = ms => async (...params) => { + await mockWrite(...params); + await sleep(ms); + } + const a = ["a", 1]; + const b = ["b", 2]; + const c = ["c", 3]; + const d = ["d", 4]; + const expected = [b, d] + queueMap.enque(queueId, a); + queueMap.enque(queueId, b); + queueMap.on(queueId, sleepThenReturn(1000), params => params); + queueMap.enque(queueId, c); + queueMap.on(queueId, sleepThenReturn(1000), params => params); + queueMap.enque(queueId, d); + await sleep(2000); + expect(mockWrite.mock.calls.length).toBe(2); + expect([mockWrite.mock.calls[0], mockWrite.mock.calls[1]]).toEqual(expected); + }) + test("read last element.", async () => { + const queueMap = new QueueMap(); + const queueId = "1"; + const f = jest.fn(); + const sleepThenReturn = ms => async (...params) => { + await f(...params); + await sleep(ms); + } + const a = ["a", 1]; + const b = ["b", 2]; + const c = ["c", 3]; + const d = ["d", 4]; + queueMap.enque(queueId, a); + queueMap.enque(queueId, b); + queueMap.on(queueId, sleepThenReturn(1000), params => params); + queueMap.enque(queueId, c); + queueMap.enque(queueId, d); + expect(queueMap.getLast(queueId)).toEqual(d); + }) + test("delete after write finished", async () => { + const mockCallback = jest.fn(); + const mockWrite = jest.fn(); + const queueMap = new QueueMap(); + const queueId = "1"; + const mockAsync = ms => async (...params) => { + await mockWrite(...params); + await sleep(ms); + } + const a = ["a", 1]; + const b = ["b", 2]; + const c = ["c", 3]; + const d = ["d", 4]; + queueMap.enque(queueId, a); + queueMap.enque(queueId, b); + queueMap.on(queueId, mockAsync(1000)); + queueMap.enque(queueId, c); + queueMap.enque(queueId, d); + const args = [a, b]; + queueMap.callAfterLoop(queueId, mockCallback, args); + await sleep(3000); + expect(mockCallback.mock.calls.length).toBe(1); + expect(mockCallback.mock.calls[0]).toEqual(args); + }) + test("can call callback finished", async () => { + const mockCallback = jest.fn(); + const queueMap = new QueueMap(); + const queueId = "1"; + queueMap.callAfterLoop(queueId, mockCallback); + expect(mockCallback.mock.calls.length).toBe(1); + }) +}) diff --git a/src/common/queueMap/queueMap.ts b/src/common/queueMap/queueMap.ts new file mode 100644 index 000000000..5d7527ede --- /dev/null +++ b/src/common/queueMap/queueMap.ts @@ -0,0 +1,123 @@ +import { Args, IQueue, Queue } from "./queue"; + +interface IQueueMap { + [id: string]: IQueue; +} + +export default class QueueMap { + queueById: IQueueMap; + constructor() { + this.queueById = {}; + } + + /** + * Get a copy of IQueueMap from QueueMap + * @return QueueMap - IQueueMap + */ + getMap = (): IQueueMap => { + return { ...this.queueById }; + } + + /** + * Get the IQueue by id. Create a new IQueue while get null. + * @param id - id of the queue + * @return IQueue + */ + getQueueById = (id: string): IQueue => { + if (!this.queueById.hasOwnProperty(id)) { + this.queueById[id] = new Queue(); + } + return this.queueById[id]; + } + + /** + * Find a queue by id, then enqueue an object into the queue. + * @param id - id of the queue + * @param args - list of argument + */ + enque = (id: string, args: Args) => { + const { queue } = this.getQueueById(id); + queue.push(args); + } + + /** + * @param id - id of the queue + * @return - dequeued object + */ + dequeue = (id: string): Args => { + const { queue } = this.getQueueById(id); + return queue.shift(); + } + + /** + * Find a queue by id then dequeue. Then clear objects before the last one. + * @param id - id of the queue + * @return - dequeue object + */ + dequeueUntilLast = (id: string): Args => { + let ret = []; + const { queue } = this.getQueueById(id); + while (queue.length > 1) { + ret = queue.shift(); + } + return ret; + } + + /** Find and return the last element in the queue + * @param id - id of the queue + * @return last element in the queue + */ + getLast = (id: string): Args => { + const { queue } = this.getQueueById(id); + if (queue.length) { + return queue[queue.length - 1]; + } + return []; + } + + /** + * loop to use last element as parameters to call async method. + * will prevent this function call while the queue is already looping by another function. + * @param id - id of the queue + * @param method - async method to call + * @param paramsHandler - process dequeue object to method parameters + * @param errorHandler - handle async method error + */ + on = (id: string, method: (...args: any[]) => void, paramsHandler = (params) => params, errorHandler = console.error) => { + const q = this.getQueueById(id); + const loop = async () => { + q.isLooping = true; + while (q.queue.length) { + this.dequeueUntilLast(id); + const args = this.getLast(id); + const params = args.map(paramsHandler); + try { + await method(...params); + } catch (err) { + errorHandler(err); + } + this.dequeue(id); + } + q.isLooping = false; + } + if (q.isLooping === false) { + q.promise = loop(); + } + } + + /** + * call the callback function after loop finished + * @param id - id of the queue + * @param callback - callback after loop finished + * @param args - callback arguments + */ + callAfterLoop = async (id: string, callback: (...args: any[]) => void, args: Args = []) => { + const q = this.getQueueById(id); + if (q.promise) { + await q.promise; + } + await callback(...args); + } +} + +export const queueMap = new QueueMap(); diff --git a/src/common/queueMap/withQueueMap.ts b/src/common/queueMap/withQueueMap.ts new file mode 100644 index 000000000..f276badfc --- /dev/null +++ b/src/common/queueMap/withQueueMap.ts @@ -0,0 +1,44 @@ +import { IStorageProvider } from "../../providers/storage/storageProviderFactory"; +import { constants } from "../constants"; +import { queueMap } from "./queueMap"; + +// tslint:disable-next-line +export function withQueueMap(constructor: T) { + return class extends constructor { + isQueuedFile = (filePath: string = ""): boolean => { + return filePath.endsWith(constants.labelFileExtension); + } + + writeText = async (filePath: string, contents: string): Promise => { + const parentWriteText = super.writeText.bind(this); + if (this.isQueuedFile(filePath)) { + queueMap.enque(filePath, [filePath, contents]); + queueMap.on(filePath, parentWriteText); + return; + } + return await parentWriteText(filePath, contents); + } + + readText = async (filePath: string, ignoreNotFound?: boolean): Promise => { + const parentReadText = super.readText.bind(this); + if (this.isQueuedFile(filePath)) { + const args = queueMap.getLast(filePath); + if (args.length >= 2) { + const contents = args[1] || ""; + return (async () => contents)() + } + } + return parentReadText(filePath, ignoreNotFound); + } + + deleteFile = async (filePath: string, ignoreNotFound?: boolean, ignoreForbidden?: boolean) => { + // Expect this function is not called too often or may cause race with readText. + const parentDeleteFile = super.deleteFile.bind(this); + if (this.isQueuedFile(filePath)) { + await queueMap.callAfterLoop(filePath, parentDeleteFile, [filePath, ignoreNotFound, ignoreForbidden]) + return; + } + parentDeleteFile(filePath, ignoreNotFound, ignoreForbidden); + } + } +} diff --git a/src/providers/storage/azureBlobStorage.ts b/src/providers/storage/azureBlobStorage.ts index c8eacb5db..a24c0fb3a 100644 --- a/src/providers/storage/azureBlobStorage.ts +++ b/src/providers/storage/azureBlobStorage.ts @@ -7,6 +7,7 @@ import { AppError, AssetState, AssetType, ErrorCode, IAsset, StorageType, ILabel import { throwUnhandledRejectionForEdge } from "../../react/components/common/errorHandler/errorHandler"; import { AssetService } from "../../services/assetService"; import { IStorageProvider } from "./storageProviderFactory"; +import {withQueueMap} from "../../common/queueMap/withQueueMap" /** * Options for Azure Cloud Storage @@ -21,6 +22,7 @@ export interface IAzureCloudStorageOptions { /** * Storage Provider for Azure Blob Storage */ +@withQueueMap export class AzureBlobStorage implements IStorageProvider { /** * Storage type diff --git a/src/redux/actions/projectActions.ts b/src/redux/actions/projectActions.ts index 931af06c6..d21967b21 100644 --- a/src/redux/actions/projectActions.ts +++ b/src/redux/actions/projectActions.ts @@ -286,9 +286,9 @@ export function saveAssetMetadata( return async (dispatch: Dispatch) => { const assetService = new AssetService(project); - const savedMetadata = await assetService.save(newAssetMetadata); - dispatch(saveAssetMetadataAction(savedMetadata)); - return { ...savedMetadata }; + assetService.save(newAssetMetadata); + dispatch(saveAssetMetadataAction(newAssetMetadata)); + return { ...newAssetMetadata }; }; } diff --git a/src/services/assetService.ts b/src/services/assetService.ts index 040711ad0..e8eb3fdbe 100644 --- a/src/services/assetService.ts +++ b/src/services/assetService.ts @@ -268,7 +268,6 @@ export class AssetService { this.project.sourceConnection.providerOptions, ); } - return this.storageProviderInstance; } @@ -745,7 +744,7 @@ export class AssetService { * @param project to get assets and connect to file system. * @returns updated project */ - public static checkAndUpdateSchema = async(project: IProject): Promise => { + public static checkAndUpdateSchema = async (project: IProject): Promise => { let shouldAssetsUpdate = false; let updatedProject; const { assets } = project; diff --git a/yarn.lock b/yarn.lock index 9536be798..6a3bd3a10 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1797,6 +1797,13 @@ dependencies: "@types/node" "*" +"@types/cheerio@^0.22.22": + version "0.22.28" + resolved "https://registry.yarnpkg.com/@types/cheerio/-/cheerio-0.22.28.tgz#90808aabb44fec40fa2950f4c72351e3e4eb065b" + integrity sha512-ehUMGSW5IeDxJjbru4awKYMlKGmo1wSSGUVqXtYwlgmUM8X1a0PZttEIm6yEY7vHsY/hh6iPnklF213G0UColw== + dependencies: + "@types/node" "*" + "@types/color-name@^1.1.1": version "1.1.1" resolved "https://registry.yarnpkg.com/@types/color-name/-/color-name-1.1.1.tgz#1c1261bbeaa10a8055bbc5d8ab84b7b2afc846a0" @@ -3627,6 +3634,11 @@ cipher-base@^1.0.0, cipher-base@^1.0.1, cipher-base@^1.0.3: inherits "^2.0.1" safe-buffer "^5.0.1" +circular-json-es6@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/circular-json-es6/-/circular-json-es6-2.0.2.tgz#e4f4a093e49fb4b6aba1157365746112a78bd344" + integrity sha512-ODYONMMNb3p658Zv+Pp+/XPa5s6q7afhz3Tzyvo+VRh9WIrJ64J76ZC4GQxnlye/NesTn09jvOiuE8+xxfpwhQ== + class-utils@^0.3.5: version "0.3.6" resolved "https://registry.yarnpkg.com/class-utils/-/class-utils-0.3.6.tgz#f93369ae8b9a7ce02fd41faad0ca83033190c463" @@ -4438,6 +4450,13 @@ deep-diff@^0.3.5: resolved "https://registry.yarnpkg.com/deep-diff/-/deep-diff-0.3.8.tgz#c01de63efb0eec9798801d40c7e0dae25b582c84" integrity sha1-wB3mPvsO7JeYgB1Ax+Da4ltYLIQ= +deep-equal-ident@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/deep-equal-ident/-/deep-equal-ident-1.1.1.tgz#06f4b89e53710cd6cea4a7781c7a956642de8dc9" + integrity sha1-BvS4nlNxDNbOpKd4HHqVZkLejck= + dependencies: + lodash.isequal "^3.0" + deep-equal@^1.0.1, deep-equal@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.1.1.tgz#b5c98c942ceffaf7cb051e24e1434a25a2e6076a" @@ -4966,6 +4985,14 @@ enzyme-adapter-utils@^1.13.0: prop-types "^15.7.2" semver "^5.7.1" +enzyme-matchers@^7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/enzyme-matchers/-/enzyme-matchers-7.1.2.tgz#d80530a61f22d28bb993dd7588abba38bd4de282" + integrity sha512-03WqAg2XDl7id9rARIO97HQ1JIw9F2heJ3R4meGu/13hx0ULTDEgl0E67MGl2Uq1jq1DyRnJfto1/VSzskdV5A== + dependencies: + circular-json-es6 "^2.0.1" + deep-equal-ident "^1.1.1" + enzyme-shallow-equal@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/enzyme-shallow-equal/-/enzyme-shallow-equal-1.0.1.tgz#7afe03db3801c9b76de8440694096412a8d9d49e" @@ -4974,6 +5001,15 @@ enzyme-shallow-equal@^1.0.1: has "^1.0.3" object-is "^1.0.2" +enzyme-to-json@^3.3.0: + version "3.6.1" + resolved "https://registry.yarnpkg.com/enzyme-to-json/-/enzyme-to-json-3.6.1.tgz#d60740950bc7ca6384dfe6fe405494ec5df996bc" + integrity sha512-15tXuONeq5ORoZjV/bUo2gbtZrN2IH+Z6DvL35QmZyKHgbY1ahn6wcnLd9Xv9OjiwbAXiiP8MRZwbZrCv1wYNg== + dependencies: + "@types/cheerio" "^0.22.22" + lodash "^4.17.15" + react-is "^16.12.0" + enzyme@^3.10.0: version "3.11.0" resolved "https://registry.yarnpkg.com/enzyme/-/enzyme-3.11.0.tgz#71d680c580fe9349f6f5ac6c775bc3e6b7a79c28" @@ -7387,6 +7423,13 @@ jest-each@^24.9.0: jest-util "^24.9.0" pretty-format "^24.9.0" +jest-environment-enzyme@^7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/jest-environment-enzyme/-/jest-environment-enzyme-7.1.2.tgz#4561f26a719e8e87ce8c9a6d3f540a92663ba8d5" + integrity sha512-3tfaYAzO7qZSRrv+srQnfK16Vu5XwH/pHi8FpoqSHjKKngbHzXf7aBCBuWh8y3w0OtknHRfDMFrC60Khj+g1hA== + dependencies: + jest-environment-jsdom "^24.0.0" + jest-environment-jsdom-fourteen@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/jest-environment-jsdom-fourteen/-/jest-environment-jsdom-fourteen-1.0.1.tgz#4cd0042f58b4ab666950d96532ecb2fc188f96fb" @@ -7399,7 +7442,7 @@ jest-environment-jsdom-fourteen@1.0.1: jest-util "^24.0.0" jsdom "^14.1.0" -jest-environment-jsdom@^24.9.0: +jest-environment-jsdom@^24.0.0, jest-environment-jsdom@^24.9.0: version "24.9.0" resolved "https://registry.yarnpkg.com/jest-environment-jsdom/-/jest-environment-jsdom-24.9.0.tgz#4b0806c7fc94f95edb369a69cc2778eec2b7375b" integrity sha512-Zv9FV9NBRzLuALXjvRijO2351DRQeLYXtpD4xNvfoVFw21IOKNhZAEUKcbiEtjTkm2GsJ3boMVgkaR7rN8qetA== @@ -7422,6 +7465,15 @@ jest-environment-node@^24.9.0: jest-mock "^24.9.0" jest-util "^24.9.0" +jest-enzyme@^7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/jest-enzyme/-/jest-enzyme-7.1.2.tgz#91a10b2d3be1b56c0d65b34286e5bdc41ab4ba3d" + integrity sha512-j+jkph3t5hGBS12eOldpfsnERYRCHi4c/0KWPMnqRPoJJXvCpLIc5th1MHl0xDznQDXVU0AHUXg3rqMrf8vGpA== + dependencies: + enzyme-matchers "^7.1.2" + enzyme-to-json "^3.3.0" + jest-environment-enzyme "^7.1.2" + jest-get-type@^24.9.0: version "24.9.0" resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-24.9.0.tgz#1684a0c8a50f2e4901b6644ae861f579eed2ef0e" @@ -8128,6 +8180,25 @@ locate-path@^5.0.0: dependencies: p-locate "^4.1.0" +lodash._baseisequal@^3.0.0: + version "3.0.7" + resolved "https://registry.yarnpkg.com/lodash._baseisequal/-/lodash._baseisequal-3.0.7.tgz#d8025f76339d29342767dcc887ce5cb95a5b51f1" + integrity sha1-2AJfdjOdKTQnZ9zIh85cuVpbUfE= + dependencies: + lodash.isarray "^3.0.0" + lodash.istypedarray "^3.0.0" + lodash.keys "^3.0.0" + +lodash._bindcallback@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/lodash._bindcallback/-/lodash._bindcallback-3.0.1.tgz#e531c27644cf8b57a99e17ed95b35c748789392e" + integrity sha1-5THCdkTPi1epnhftlbNcdIeJOS4= + +lodash._getnative@^3.0.0: + version "3.9.1" + resolved "https://registry.yarnpkg.com/lodash._getnative/-/lodash._getnative-3.9.1.tgz#570bc7dede46d61cdcde687d65d3eecbaa3aaff5" + integrity sha1-VwvH3t5G1hzc3mh9ZdPuy6o6r/U= + lodash._reinterpolate@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz#0ccf2d89166af03b3663c796538b75ac6e114d9d" @@ -8143,6 +8214,24 @@ lodash.flattendeep@^4.4.0: resolved "https://registry.yarnpkg.com/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz#fb030917f86a3134e5bc9bec0d69e0013ddfedb2" integrity sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI= +lodash.isarguments@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a" + integrity sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo= + +lodash.isarray@^3.0.0: + version "3.0.4" + resolved "https://registry.yarnpkg.com/lodash.isarray/-/lodash.isarray-3.0.4.tgz#79e4eb88c36a8122af86f844aa9bcd851b5fbb55" + integrity sha1-eeTriMNqgSKvhvhEqpvNhRtfu1U= + +lodash.isequal@^3.0: + version "3.0.4" + resolved "https://registry.yarnpkg.com/lodash.isequal/-/lodash.isequal-3.0.4.tgz#1c35eb3b6ef0cd1ff51743e3ea3cf7fdffdacb64" + integrity sha1-HDXrO27wzR/1F0Pj6jz3/f/ay2Q= + dependencies: + lodash._baseisequal "^3.0.0" + lodash._bindcallback "^3.0.0" + lodash.isequal@^4.5.0: version "4.5.0" resolved "https://registry.yarnpkg.com/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0" @@ -8163,6 +8252,20 @@ lodash.isplainobject@^4.0.6: resolved "https://registry.yarnpkg.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb" integrity sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs= +lodash.istypedarray@^3.0.0: + version "3.0.6" + resolved "https://registry.yarnpkg.com/lodash.istypedarray/-/lodash.istypedarray-3.0.6.tgz#c9a477498607501d8e8494d283b87c39281cef62" + integrity sha1-yaR3SYYHUB2OhJTSg7h8OSgc72I= + +lodash.keys@^3.0.0: + version "3.1.2" + resolved "https://registry.yarnpkg.com/lodash.keys/-/lodash.keys-3.1.2.tgz#4dbc0472b156be50a0b286855d1bd0b0c656098a" + integrity sha1-TbwEcrFWvlCgsoaFXRvQsMZWCYo= + dependencies: + lodash._getnative "^3.0.0" + lodash.isarguments "^3.0.0" + lodash.isarray "^3.0.0" + lodash.memoize@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/lodash.memoize/-/lodash.memoize-4.1.2.tgz#bcc6c49a42a2840ed997f323eada5ecd182e0bfe"