From 39c7a008e8310107e258df9ae9e195b01078e333 Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Fri, 14 Aug 2020 08:06:37 +0900 Subject: [PATCH 01/18] Reproduce --- .github/workflows/release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d4a1fcf1..aab72758 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -91,7 +91,6 @@ jobs: name: Run satackey/action-docker-layer-caching@${{ steps.extract.outputs.branch }} with: key: docker-layer-caching-${{ matrix.inspect_image }}-sha:${{ github.sha }}-{hash} - restore-keys: docker-layer-caching-${{ matrix.inspect_image }}-sha:${{ github.sha }}- - run: ${{ matrix.build_command }} @@ -119,7 +118,7 @@ jobs: - uses: ./action-dlc name: Run satackey/action-docker-layer-caching@${{ steps.extract.outputs.branch }} with: - key: docker-layer-caching-${{ matrix.inspect_image }}-sha:${{ github.sha }}-{hash} + key: never-restored-docker-layer-caching-${{ matrix.inspect_image }}-sha:${{ github.sha }}-{hash} restore-keys: docker-layer-caching-${{ matrix.inspect_image }}-sha:${{ github.sha }}- skip-save: 'true' From 494c4705f987d7b54f89c3cfe6b448011798b46e Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Fri, 14 Aug 2020 09:15:39 +0900 Subject: [PATCH 02/18] Do not fail with missing layer cache --- src/LayerCache.ts | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/src/LayerCache.ts b/src/LayerCache.ts index 5251ef68..0ca739e2 100644 --- a/src/LayerCache.ts +++ b/src/LayerCache.ts @@ -21,6 +21,9 @@ class LayerCache { // manifests: Manifests = [] concurrency: number = 4 + static ERROR_CACHE_ALREAD_EXISTS_STR = `Cache already exists` + static ERROR_LAYER_CACHE_NOT_FOUND_STR = `Layer cache not found` + constructor(ids: string[]) { // this.repotag = repotag this.ids = ids @@ -84,7 +87,7 @@ class LayerCache { this.getUnpackedTarDir(), ] core.info(`Start storing root cache, key: ${rootKey}, dir: ${paths}`) - const cacheId = await LayerCache.dismissCacheAlreadyExistsError(cache.saveCache(paths, rootKey)) + const cacheId = await LayerCache.dismissError(cache.saveCache(paths, rootKey), LayerCache.ERROR_CACHE_ALREAD_EXISTS_STR, -1) core.info(`Stored root cache, key: ${rootKey}, id: ${cacheId}`) return cacheId !== -1 ? cacheId : undefined } @@ -122,20 +125,19 @@ class LayerCache { return result } - static async dismissCacheAlreadyExistsError(promise: Promise): Promise { - let result = -1 + static async dismissError(promise: Promise, dismissStr: string, defaultResult: T): Promise { try { - result = await promise - return result + return await promise } catch (e) { core.debug(`catch error: ${e.toString()}`) - if (typeof e.message !== 'string' || !e.message.includes(`Cache already exists`)) { + if (typeof e.message !== 'string' || !e.message.includes(dismissStr)) { core.error(`Unexpected error: ${e.toString()}`) throw e } - core.info(`info: Cache already exists: ${e.toString()}`) + + core.info(`${dismissStr}: ${e.toString()}`) core.debug(e) - return result + return defaultResult } } @@ -144,7 +146,7 @@ class LayerCache { const key = this.genSingleLayerStoreKey(id) core.info(`Start storing layer cache: ${key}`) - const cacheId = await LayerCache.dismissCacheAlreadyExistsError(cache.saveCache([path], key)) + const cacheId = await LayerCache.dismissError(cache.saveCache([path], key), LayerCache.ERROR_CACHE_ALREAD_EXISTS_STR, -1) core.info(`Stored layer cache, key: ${key}, id: ${cacheId}`) return cacheId @@ -183,20 +185,24 @@ class LayerCache { return this.restoredOriginalKey } - private async restoreLayers() { + private async restoreLayers(): Promise { const pool = new PromisePool(this.concurrency) - const restoredLayerKeysThatMayContainUndefined = await Promise.all( - (await this.getLayerIds()).map( + try { + await (await this.getLayerIds()).map( layerId => { return pool.open(() => this.restoreSingleLayerBy(layerId)) } ) - ) + } catch (e) { + if (typeof e.message === `string` && e.message.includes(LayerCache.ERROR_LAYER_CACHE_NOT_FOUND_STR)) { + core.info(e.message) + return false + } + throw e + } - core.debug(JSON.stringify({ log: `restoreLayers`, restoredLayerKeysThatMayContainUndefined })) - const FailedToRestore = (restored: string | undefined) => restored === undefined - return restoredLayerKeysThatMayContainUndefined.filter(FailedToRestore).length === 0 + return true } private async restoreSingleLayerBy(id: string): Promise { @@ -205,7 +211,7 @@ class LayerCache { const result = await cache.restoreCache([this.genSingleLayerStorePath(id)], this.genSingleLayerStoreKey(id)) if (result == null) { - throw new Error(`Layer cache not found: ${JSON.stringify({ id })}`) + throw new Error(`${LayerCache.ERROR_LAYER_CACHE_NOT_FOUND_STR}: ${JSON.stringify({ id })}`) } return result From 51cdf87c3c3633edeef557ee482d686119f0b991 Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Fri, 14 Aug 2020 14:08:50 +0900 Subject: [PATCH 03/18] Use manifest.json to generate hash --- src/LayerCache.ts | 86 +++++++++++++++++++++++++---------------------- src/Tar.ts | 11 ++++++ 2 files changed, 57 insertions(+), 40 deletions(-) diff --git a/src/LayerCache.ts b/src/LayerCache.ts index 0ca739e2..a3674ed3 100644 --- a/src/LayerCache.ts +++ b/src/LayerCache.ts @@ -5,27 +5,22 @@ import * as core from '@actions/core' import * as cache from '@actions/cache' import { ExecOptions } from '@actions/exec/lib/interfaces' import { promises as fs } from 'fs' -import { assertManifests, Manifest, Manifests } from './Tar' +import { assertManifests, Manifest, Manifests, loadManifests, loadRawManifests } from './Tar' import format from 'string-format' import PromisePool from 'native-promise-pool' class LayerCache { - // repotag: string - ids: string[] - unformattedOrigianlKey: string = '' - restoredOriginalKey?: string - // tarFile: string = '' + ids: string[] = [] + unformattedSaveKey: string = '' + restoredRootKey: string = '' imagesDir: string = path.resolve(`${process.cwd()}/./.action-docker-layer-caching-docker_images`) enabledParallel = true - // unpackedTarDir: string = '' - // manifests: Manifests = [] concurrency: number = 4 static ERROR_CACHE_ALREAD_EXISTS_STR = `Cache already exists` static ERROR_LAYER_CACHE_NOT_FOUND_STR = `Layer cache not found` constructor(ids: string[]) { - // this.repotag = repotag this.ids = ids } @@ -39,10 +34,8 @@ class LayerCache { } async store(key: string) { - this.unformattedOrigianlKey = key - // this.originalKeyToStore = format(key, { - // hash: this.getIdhashesPathFriendly() - // }) + this.unformattedSaveKey = key + await this.saveImageAsUnpacked() if (this.enabledParallel) { await this.separateAllLayerCaches() @@ -76,13 +69,11 @@ class LayerCache { } private async getManifests() { - const manifests = JSON.parse((await fs.readFile(`${this.getUnpackedTarDir()}/manifest.json`)).toString()) - assertManifests(manifests) - return manifests + return loadManifests(this.getUnpackedTarDir()) } private async storeRoot() { - const rootKey = this.getRootKey() + const rootKey = await this.generateRootSaveKey() const paths = [ this.getUnpackedTarDir(), ] @@ -143,7 +134,7 @@ class LayerCache { private async storeSingleLayerBy(id: string): Promise { const path = this.genSingleLayerStorePath(id) - const key = this.genSingleLayerStoreKey(id) + const key = await this.generateSingleLayerSaveKey(id) core.info(`Start storing layer cache: ${key}`) const cacheId = await LayerCache.dismissError(cache.saveCache([path], key), LayerCache.ERROR_CACHE_ALREAD_EXISTS_STR, -1) @@ -155,8 +146,6 @@ class LayerCache { // --- async restore(key: string, restoreKeys?: string[]) { - this.unformattedOrigianlKey = key - // const restoreKeysIncludedRootKey = [key, ...(restoreKeys !== undefined ? restoreKeys : [])] const restoredCacheKey = await this.restoreRoot(restoreKeys) if (restoredCacheKey === undefined) { core.info(`Root cache could not be found. aborting.`) @@ -175,14 +164,13 @@ class LayerCache { } private async restoreRoot(restoreKeys?: string[]): Promise { - core.debug(`Trying to restore root cache: ${ JSON.stringify({ primaryKey: this.getRootKey(), restoreKeys, dir: this.getUnpackedTarDir() }) }`) - const restoredCacheKeyMayUndefined = await cache.restoreCache([this.getUnpackedTarDir()], this.getRootKey(), restoreKeys) - core.debug(`restoredCacheKeyMayUndefined: ${restoredCacheKeyMayUndefined}`) - if (restoredCacheKeyMayUndefined === undefined) { + core.debug(`Trying to restore root cache: ${ JSON.stringify({ restoreKeys, dir: this.getUnpackedTarDir() }) }`) + const restoredRootKey = await cache.restoreCache([this.getUnpackedTarDir()], ``, restoreKeys) + core.debug(`restoredRootKey: ${restoredRootKey}`) + if (restoredRootKey === undefined) { return undefined } - this.restoredOriginalKey = restoredCacheKeyMayUndefined.replace(/-root$/, '') - return this.restoredOriginalKey + this.restoredRootKey = restoredRootKey } private async restoreLayers(): Promise { @@ -208,7 +196,7 @@ class LayerCache { private async restoreSingleLayerBy(id: string): Promise { core.debug(JSON.stringify({ log: `restoreSingleLayerBy`, id })) - const result = await cache.restoreCache([this.genSingleLayerStorePath(id)], this.genSingleLayerStoreKey(id)) + const result = await cache.restoreCache([this.genSingleLayerStorePath(id)], await this.recoverSingleLayerKey(id)) if (result == null) { throw new Error(`${LayerCache.ERROR_LAYER_CACHE_NOT_FOUND_STR}: ${JSON.stringify({ id })}`) @@ -247,33 +235,51 @@ class LayerCache { return 'image' } - getIdhashesPathFriendly(): string { - const result = crypto.createHash(`sha256`).update(this.ids.join(`-`), `utf8`).digest(`hex`) + async getIdhashesPathFriendly(): Promise { + const result = crypto.createHash(`sha256`).update((await this.getLayerIds()).join(`-`), `utf8`).digest(`hex`) core.debug(JSON.stringify({ log: `getIdhashesPathFriendly`, result })) return result } - getRootKey(): string { - return `${this.getFormattedOriginalCacheKey()}-root` - } genSingleLayerStorePath(id: string) { return `${this.getLayerCachesDir()}/${id}/layer.tar` } - genSingleLayerStoreKey(id: string) { - const singleLayerStoreKey = this.getFormattedOriginalCacheKey(id) - core.debug(JSON.stringify({ log: `genSingleLayerStoreKey`, singleLayerStoreKey, id })) - return `layer-${singleLayerStoreKey}` + async generateRootHashFromManifest(): Promise { + const manifest = await loadRawManifests(this.getUnpackedTarDir()) + return crypto.createHash(`sha256`).update(manifest, `utf8`).digest(`hex`) } - getFormattedOriginalCacheKey(hashThatMayBeSpecified?: string) { - const hash = hashThatMayBeSpecified !== undefined ? hashThatMayBeSpecified : this.getIdhashesPathFriendly() - const result = format(this.unformattedOrigianlKey, { hash }) - core.debug(JSON.stringify({ log: `getFormattedOriginalCacheKey`, hash, hashThatMayBeSpecified, result })) + async generateRootSaveKey(): Promise { + const rootHash = await this.generateRootHashFromManifest() + const formatted = await this.getFormattedSaveKey(rootHash) + core.debug(JSON.stringify({ log: `generateRootSaveKey`, rootHash, formatted })) + return `${formatted}-root` + } + + async generateSingleLayerSaveKey(id: string) { + const formatted = await this.getFormattedSaveKey(id) + core.debug(JSON.stringify({ log: `generateSingleLayerSaveKey`, formatted, id })) + return `layer-${formatted}` + } + + async recoverSingleLayerKey(id: string) { + const unformatted = await this.recoverUnformattedSaveKey() + return format(unformatted, { hash: id }) + } + + async getFormattedSaveKey(hash: string) { + const result = format(this.unformattedSaveKey, { hash }) + core.debug(JSON.stringify({ log: `getFormattedSaveKey`, hash, result })) return result } + async recoverUnformattedSaveKey() { + const hash = await loadRawManifests(this.getUnpackedTarDir()) + return this.restoredRootKey.replace(hash, `{hash}`) + } + async getLayerTarFiles(): Promise { const getTarFilesFromManifest = (manifest: Manifest) => manifest.Layers diff --git a/src/Tar.ts b/src/Tar.ts index f5ef8cec..46db0f49 100644 --- a/src/Tar.ts +++ b/src/Tar.ts @@ -1,4 +1,5 @@ import { assertType } from 'typescript-is' +import { promises as fs } from 'fs' export interface Manifest { Config: string @@ -11,3 +12,13 @@ export type Manifests = Manifest[] export function assertManifests(x: unknown): asserts x is Manifests { assertType(x) } + +export async function loadRawManifests(path: string) { + return (await fs.readFile(`${path}/manifests.json`)).toString() +} +export async function loadManifests(path: string) { + const raw = await loadRawManifests(path) + const manifests = JSON.parse(raw.toString()) + assertManifests(manifests) + return manifests +} From 99d7f1def3ae963d63ecae8184dbab27a973759c Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Fri, 14 Aug 2020 20:08:10 +0900 Subject: [PATCH 04/18] update --- main.ts | 6 +++++- post.ts | 26 ++++++++++++-------------- src/ImageDetector.ts | 24 ++++++++++++------------ 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/main.ts b/main.ts index 65ad632a..8dc996b5 100644 --- a/main.ts +++ b/main.ts @@ -8,7 +8,9 @@ const main = async () => { const primaryKey = core.getInput(`key`, { required: true }) const restoreKeys = core.getInput(`restore-keys`, { required: false }).split(`\n`).filter(key => key !== ``) - core.saveState(`already-existing-images`, JSON.stringify(await new ImageDetector().getExistingImages())) + const imageDetector = new ImageDetector() + + const alreadyExistingImages = await imageDetector.getExistingImages() const layerCache = new LayerCache([]) layerCache.concurrency = parseInt(core.getInput(`concurrency`, { required: true }), 10) @@ -16,6 +18,8 @@ const main = async () => { await layerCache.cleanUp() core.saveState(`restored-key`, JSON.stringify(restoredKey !== undefined ? restoredKey : '')) + core.saveState(`already-existing-images`, JSON.stringify(alreadyExistingImages)) + core.saveState(`restored-images`, JSON.stringify(imageDetector.getImagesShouldSave(alreadyExistingImages))) } main().catch(e => { diff --git a/post.ts b/post.ts index 529e8f2d..21b89cbe 100644 --- a/post.ts +++ b/post.ts @@ -12,26 +12,24 @@ const main = async () => { } const primaryKey = core.getInput('key', { required: true }) - const restoredKey = JSON.parse(core.getState(`restored-key`)) as string - const rawAlreadyExistingImages = core.getState(`already-existing-images`) - assertType(rawAlreadyExistingImages) - const alreadyExistingImages = JSON.parse(rawAlreadyExistingImages) + const restoredKey = JSON.parse(core.getState(`restored-key`)) + const alreadyExistingImages = JSON.parse(core.getState(`already-existing-images`)) + const restoredImages = JSON.parse(core.getState(`restored-images`)) + + assertType(restoredKey) assertType(alreadyExistingImages) + assertType(restoredImages) const imageDetector = new ImageDetector() - imageDetector.registerAlreadyExistedImages(alreadyExistingImages) - await imageDetector.getExistingImages() - core.debug(JSON.stringify({ imageIdsToSave: imageDetector.getImagesShouldSave() })) - const layerCache = new LayerCache(imageDetector.getImagesShouldSave()) - layerCache.concurrency = parseInt(core.getInput(`concurrency`, { required: true }), 10) - - layerCache.unformattedOrigianlKey = primaryKey - core.debug(JSON.stringify({ restoredKey, formattedOriginalCacheKey: layerCache.getFormattedOriginalCacheKey()})) - if (restoredKey !== `` && restoredKey === layerCache.getFormattedOriginalCacheKey()) { - core.info(`Key ${restoredKey} already exists, skip storing.`) + if (await imageDetector.checkIfImageHasAdded(restoredImages)) { + core.info(`Key ${restoredKey} already exists, not saving cache.`) return } + + const layerCache = new LayerCache(await imageDetector.getImagesShouldSave(alreadyExistingImages)) + layerCache.concurrency = parseInt(core.getInput(`concurrency`, { required: true }), 10) + await layerCache.store(primaryKey) await layerCache.cleanUp() } diff --git a/src/ImageDetector.ts b/src/ImageDetector.ts index 4b7079c6..1ab23033 100644 --- a/src/ImageDetector.ts +++ b/src/ImageDetector.ts @@ -2,24 +2,24 @@ import exec from 'actions-exec-listener' import * as core from '@actions/core' export class ImageDetector { - alreadyExistedImages: Set = new Set([]) - existingImages: Set = new Set([]) - registerAlreadyExistedImages(images: string[]) { - images.forEach(image => this.alreadyExistedImages.add(image)) - } - async getExistingImages(): Promise { + const existingSet = new Set([]) const ids = (await exec.exec(`docker image ls -q`, [], { silent: true })).stdoutStr.split(`\n`).filter(id => id !== ``) const repotags = (await exec.exec(`sh -c "docker image ls --format '{{ .Repository }}:{{ .Tag }}' --filter 'dangling=false'"`, [], { silent: true })).stdoutStr.split(`\n`).filter(id => id !== ``); core.debug(JSON.stringify({ log: "getExistingImages", ids, repotags })); - ([...ids, ...repotags]).forEach(image => this.existingImages.add(image)) - core.debug(JSON.stringify({ existingImages: this.existingImages })) - return Array.from(this.existingImages) + ([...ids, ...repotags]).forEach(image => existingSet.add(image)) + core.debug(JSON.stringify({ existingSet })) + return Array.from(existingSet) } - getImagesShouldSave(): string[] { - const resultSet = new Set(this.existingImages.values()) - this.alreadyExistedImages.forEach(image => resultSet.delete(image)) + async getImagesShouldSave(alreadRegisteredImages: string[]): Promise { + const resultSet = new Set(await this.getExistingImages()) + alreadRegisteredImages.forEach(image => resultSet.delete(image)) return Array.from(resultSet) } + + async checkIfImageHasAdded(restoredImages: string[]): Promise { + const existing = await this.getExistingImages() + return JSON.stringify(restoredImages) === JSON.stringify(existing) + } } From 771109efb844b856fbf6f5cf1cf7f81c9389eaad Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Fri, 14 Aug 2020 20:10:25 +0900 Subject: [PATCH 05/18] fix TypeGuardError: validation failed at restoredImages: expected an array, found: {} --- main.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.ts b/main.ts index 8dc996b5..bc2699bb 100644 --- a/main.ts +++ b/main.ts @@ -19,7 +19,7 @@ const main = async () => { core.saveState(`restored-key`, JSON.stringify(restoredKey !== undefined ? restoredKey : '')) core.saveState(`already-existing-images`, JSON.stringify(alreadyExistingImages)) - core.saveState(`restored-images`, JSON.stringify(imageDetector.getImagesShouldSave(alreadyExistingImages))) + core.saveState(`restored-images`, JSON.stringify(await imageDetector.getImagesShouldSave(alreadyExistingImages))) } main().catch(e => { From 5a49c4916d640f993f18e177ce3ec13b046c138e Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Fri, 14 Aug 2020 20:18:03 +0900 Subject: [PATCH 06/18] Fix Error: ENOENT: no such file or directory, open manifests.json --- main.ts | 1 - post.ts | 1 - src/Tar.ts | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/main.ts b/main.ts index bc2699bb..25efb9e2 100644 --- a/main.ts +++ b/main.ts @@ -4,7 +4,6 @@ import { LayerCache } from './src/LayerCache' import { ImageDetector } from './src/ImageDetector' const main = async () => { - // const repotag = core.getInput(`repotag`, { required: true }) const primaryKey = core.getInput(`key`, { required: true }) const restoreKeys = core.getInput(`restore-keys`, { required: false }).split(`\n`).filter(key => key !== ``) diff --git a/post.ts b/post.ts index 21b89cbe..2d56eb22 100644 --- a/post.ts +++ b/post.ts @@ -1,5 +1,4 @@ import * as core from '@actions/core' -import exec from 'actions-exec-listener' import { LayerCache } from './src/LayerCache' import { ImageDetector } from './src/ImageDetector' diff --git a/src/Tar.ts b/src/Tar.ts index 46db0f49..99653304 100644 --- a/src/Tar.ts +++ b/src/Tar.ts @@ -14,7 +14,7 @@ export function assertManifests(x: unknown): asserts x is Manifests { } export async function loadRawManifests(path: string) { - return (await fs.readFile(`${path}/manifests.json`)).toString() + return (await fs.readFile(`${path}/manifest.json`)).toString() } export async function loadManifests(path: string) { const raw = await loadRawManifests(path) From 6d9e2d26b5e6fcb9053837d4b46916d411ca73f4 Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Fri, 14 Aug 2020 20:19:43 +0900 Subject: [PATCH 07/18] Use hello-world for test --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aab72758..4e1b416f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,7 +46,7 @@ jobs: return { inspect_image: [ 'test_project_node', - 'amazon/aws-cli', + 'hello-world', ], include: [ { @@ -54,9 +54,9 @@ jobs: prepare_command: 'docker-compose -f test_project/docker-compose.yml -p test_project pull', build_command: 'docker-compose -f test_project/docker-compose.yml -p test_project build', }, { - inspect_image: 'amazon/aws-cli', + inspect_image: 'hello-world', prepare_command: ':', - build_command: 'docker pull amazon/aws-cli', + build_command: 'docker pull hello-world', }, ], } From 1facf521f5a4333418c006ff193c85bf457a3127 Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Fri, 14 Aug 2020 20:26:53 +0900 Subject: [PATCH 08/18] Fix Root cache could not be found. aborting. --- src/LayerCache.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/LayerCache.ts b/src/LayerCache.ts index a3674ed3..bd999c3c 100644 --- a/src/LayerCache.ts +++ b/src/LayerCache.ts @@ -171,6 +171,8 @@ class LayerCache { return undefined } this.restoredRootKey = restoredRootKey + + return restoredRootKey } private async restoreLayers(): Promise { From 5ffbffb5d2ed080a1b0829547c4d590b6d45eb95 Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Fri, 14 Aug 2020 20:42:24 +0900 Subject: [PATCH 09/18] Fix There was an error when attempting to execute the process '/usr/bin/find'. This may indicate the process failed to start. Error: spawn /usr/bin/find ENOENT --- src/LayerCache.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/LayerCache.ts b/src/LayerCache.ts index bd999c3c..580c9674 100644 --- a/src/LayerCache.ts +++ b/src/LayerCache.ts @@ -92,7 +92,8 @@ class LayerCache { } private async moveLayerTarsInDir(fromDir: string, toDir: string) { - const layerTars = (await exec.exec(`find . -name layer.tar`, [], { cwd: fromDir, silent: true })).stdoutStr.split(`\n`).filter(tar => tar !== '') + const layerTars = (await fs.readdir(fromDir)).filter(path => path.endsWith(`/layer.tar`)) + const moveLayer = async (layer: string) => { const from = path.resolve(`${fromDir}/${layer}`) const to = path.resolve(`${toDir}/${layer}`) From 9cce89503bc0e67a8f3f430d2ee33dae411bdb77 Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Fri, 14 Aug 2020 23:23:24 +0900 Subject: [PATCH 10/18] Fix: ENOENT: no such file or directory, scandir 'image-layers' ENOENT: no such file or directory, scandir '/home/runner/work/action-docker-layer-caching/action-docker-layer-caching/.action-docker-layer-caching-docker_images/image-layers' --- src/LayerCache.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/LayerCache.ts b/src/LayerCache.ts index 580c9674..ce74d6d6 100644 --- a/src/LayerCache.ts +++ b/src/LayerCache.ts @@ -179,15 +179,19 @@ class LayerCache { private async restoreLayers(): Promise { const pool = new PromisePool(this.concurrency) + const tasks = (await this.getLayerIds()).map( + layerId => pool.open(() => this.restoreSingleLayerBy(layerId)) + ) + try { - await (await this.getLayerIds()).map( - layerId => { - return pool.open(() => this.restoreSingleLayerBy(layerId)) - } - ) + await Promise.all(tasks) } catch (e) { if (typeof e.message === `string` && e.message.includes(LayerCache.ERROR_LAYER_CACHE_NOT_FOUND_STR)) { core.info(e.message) + + // Avoid UnhandledPromiseRejectionWarning + tasks.map(task => task.catch(core.info)) + return false } throw e @@ -280,7 +284,7 @@ class LayerCache { async recoverUnformattedSaveKey() { const hash = await loadRawManifests(this.getUnpackedTarDir()) - return this.restoredRootKey.replace(hash, `{hash}`) + return this.restoredRootKey.replace(hash, `{hash}`).replace(/-root$/, ``) } async getLayerTarFiles(): Promise { From 120939d349c55d125e1c68d67b5f2a36c0ebf305 Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Fri, 14 Aug 2020 23:29:44 +0900 Subject: [PATCH 11/18] Fix: Layer cache not found --- src/LayerCache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LayerCache.ts b/src/LayerCache.ts index ce74d6d6..9fbf7ef9 100644 --- a/src/LayerCache.ts +++ b/src/LayerCache.ts @@ -273,7 +273,7 @@ class LayerCache { async recoverSingleLayerKey(id: string) { const unformatted = await this.recoverUnformattedSaveKey() - return format(unformatted, { hash: id }) + return format(`layer-${unformatted}`, { hash: id }) } async getFormattedSaveKey(hash: string) { From 952c36ca04691571ec294a02462227c6b0e06cd7 Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Fri, 14 Aug 2020 23:45:22 +0900 Subject: [PATCH 12/18] fix recoverUnformattedSaveKey --- src/LayerCache.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/LayerCache.ts b/src/LayerCache.ts index 9fbf7ef9..4cc91cbc 100644 --- a/src/LayerCache.ts +++ b/src/LayerCache.ts @@ -242,13 +242,6 @@ class LayerCache { return 'image' } - async getIdhashesPathFriendly(): Promise { - const result = crypto.createHash(`sha256`).update((await this.getLayerIds()).join(`-`), `utf8`).digest(`hex`) - core.debug(JSON.stringify({ log: `getIdhashesPathFriendly`, result })) - return result - } - - genSingleLayerStorePath(id: string) { return `${this.getLayerCachesDir()}/${id}/layer.tar` } @@ -283,7 +276,9 @@ class LayerCache { } async recoverUnformattedSaveKey() { - const hash = await loadRawManifests(this.getUnpackedTarDir()) + const hash = await this.generateRootHashFromManifest() + core.debug(JSON.stringify({ log: `recoverUnformattedSaveKey`, hash})) + return this.restoredRootKey.replace(hash, `{hash}`).replace(/-root$/, ``) } From 56b8c097f410362eb678b0b828926f7963181e4f Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Sat, 15 Aug 2020 07:42:32 +0900 Subject: [PATCH 13/18] recursiveReaddir --- package.json | 2 ++ src/LayerCache.ts | 5 +++-- yarn.lock | 16 +++++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 09b00d06..07c23c17 100644 --- a/package.json +++ b/package.json @@ -8,9 +8,11 @@ "dependencies": { "@actions/cache": "^1.0.2", "@actions/core": "^1.2.4", + "@types/recursive-readdir": "^2.2.0", "actions-exec-listener": "^0.0.2", "crypto": "^1.0.1", "native-promise-pool": "^3.13.0", + "recursive-readdir": "^2.2.2", "string-format": "^2.0.0", "typescript-is": "^0.16.3" }, diff --git a/src/LayerCache.ts b/src/LayerCache.ts index 4cc91cbc..83876acd 100644 --- a/src/LayerCache.ts +++ b/src/LayerCache.ts @@ -5,7 +5,8 @@ import * as core from '@actions/core' import * as cache from '@actions/cache' import { ExecOptions } from '@actions/exec/lib/interfaces' import { promises as fs } from 'fs' -import { assertManifests, Manifest, Manifests, loadManifests, loadRawManifests } from './Tar' +import recursiveReaddir from 'recursive-readdir' +import { Manifest, loadManifests, loadRawManifests } from './Tar' import format from 'string-format' import PromisePool from 'native-promise-pool' @@ -92,7 +93,7 @@ class LayerCache { } private async moveLayerTarsInDir(fromDir: string, toDir: string) { - const layerTars = (await fs.readdir(fromDir)).filter(path => path.endsWith(`/layer.tar`)) + const layerTars = (await recursiveReaddir(fromDir)).filter(path => path.endsWith(`/layer.tar`)) const moveLayer = async (layer: string) => { const from = path.resolve(`${fromDir}/${layer}`) diff --git a/yarn.lock b/yarn.lock index 86826840..7a6c72b2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -186,6 +186,13 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-14.0.27.tgz#a151873af5a5e851b51b3b065c9e63390a9e0eb1" integrity sha512-kVrqXhbclHNHGu9ztnAwSncIgJv/FaxmzXJvGXNdcCpV1b8u1/Mi6z6m0vwy0LzKeXFTPLH0NzwmoJ3fNCIq0g== +"@types/recursive-readdir@^2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@types/recursive-readdir/-/recursive-readdir-2.2.0.tgz#b39cd5474fd58ea727fe434d5c68b7a20ba9121c" + integrity sha512-HGk753KRu2N4mWduovY4BLjYq4jTOL29gV2OfGdGxHcPSWGFkC5RRIdk+VTs5XmYd7MVAD+JwKrcb5+5Y7FOCg== + dependencies: + "@types/node" "*" + "@types/string-format@^2.0.0": version "2.0.0" resolved "https://registry.yarnpkg.com/@types/string-format/-/string-format-2.0.0.tgz#c1588f507be7b8ef5eb5074a41e48e4538f3f6d5" @@ -322,7 +329,7 @@ mime-types@^2.1.12: dependencies: mime-db "1.44.0" -minimatch@^3.0.4: +minimatch@3.0.4, minimatch@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== @@ -364,6 +371,13 @@ punycode@^2.1.1: resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== +recursive-readdir@^2.2.2: + version "2.2.2" + resolved "https://registry.yarnpkg.com/recursive-readdir/-/recursive-readdir-2.2.2.tgz#9946fb3274e1628de6e36b2f6714953b4845094f" + integrity sha512-nRCcW9Sj7NuZwa2XvH9co8NPeXUBhZP7CRKJtU+cS6PW9FpCIFoI5ib0NT1ZrbNuPoRy0ylyCaUL8Gih4LSyFg== + dependencies: + minimatch "3.0.4" + reflect-metadata@>=0.1.12: version "0.1.13" resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.13.tgz#67ae3ca57c972a2aa1642b10fe363fe32d49dc08" From 826028e230966751d6039c31a081faf321e474b0 Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Sat, 15 Aug 2020 07:53:56 +0900 Subject: [PATCH 14/18] replace fromDir --- src/LayerCache.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/LayerCache.ts b/src/LayerCache.ts index 83876acd..552b4c76 100644 --- a/src/LayerCache.ts +++ b/src/LayerCache.ts @@ -93,7 +93,10 @@ class LayerCache { } private async moveLayerTarsInDir(fromDir: string, toDir: string) { - const layerTars = (await recursiveReaddir(fromDir)).filter(path => path.endsWith(`/layer.tar`)) + const fromDirTrailingSlash = fromDir.endsWith(`/`) ? fromDir : `${fromDir}/` + const layerTars = (await recursiveReaddir(fromDir)) + .filter(path => path.endsWith(`/layer.tar`)) + .map(path => path.replace(fromDirTrailingSlash, ``)) const moveLayer = async (layer: string) => { const from = path.resolve(`${fromDir}/${layer}`) From 5158e54bbc5b7cfbeb392e41d21bbf49bfd9a798 Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Sat, 15 Aug 2020 07:59:55 +0900 Subject: [PATCH 15/18] Update test --- .github/workflows/release.yml | 4 ++-- src/LayerCache.ts | 3 --- test_project/Dockerfile | 26 ++++---------------------- test_project/docker-compose.yml | 10 ++++------ 4 files changed, 10 insertions(+), 33 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4e1b416f..ffeac0b0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,12 +45,12 @@ jobs: script: | return { inspect_image: [ - 'test_project_node', + 'test_project_scratch', 'hello-world', ], include: [ { - inspect_image: 'test_project_node', + inspect_image: 'test_project_scratch', prepare_command: 'docker-compose -f test_project/docker-compose.yml -p test_project pull', build_command: 'docker-compose -f test_project/docker-compose.yml -p test_project build', }, { diff --git a/src/LayerCache.ts b/src/LayerCache.ts index 552b4c76..29f9c7a6 100644 --- a/src/LayerCache.ts +++ b/src/LayerCache.ts @@ -26,10 +26,7 @@ class LayerCache { } async exec(command: string, args?: string[], options?: ExecOptions) { - const argsStr = args != null ? args.join(' ') : '' - core.startGroup(`${command} ${argsStr}`) const result = await exec.exec(command, args, options) - core.endGroup() return result } diff --git a/test_project/Dockerfile b/test_project/Dockerfile index cd52b105..7da685b0 100644 --- a/test_project/Dockerfile +++ b/test_project/Dockerfile @@ -1,23 +1,5 @@ -FROM node:12-alpine as curl-env -RUN set -x \ - apk update && \ - apk add curl && \ - mkdir -p /src && \ - curl -o /src/install.sh -L https://yarnpkg.com/install.sh && \ - chmod +x /src/install.sh && \ - apk del curl +FROM alpine AS data +RUN date > /now.txt -FROM node:12-alpine - -COPY --from=curl-env /src/install.sh /tmp/install.sh -RUN set -x \ - apk update && \ - apk add curl && \ - /tmp/install.sh && \ - apk del curl - -WORKDIR /app -COPY package.json yarn.lock ./ -RUN yarn install --frozen-lockfile - -# COPY . ./ +FROM scratch +COPY --from=data /now.txt /data_stage_built_at.txt diff --git a/test_project/docker-compose.yml b/test_project/docker-compose.yml index 9437dfa8..67129bb6 100644 --- a/test_project/docker-compose.yml +++ b/test_project/docker-compose.yml @@ -1,9 +1,7 @@ version: '3' services: - node: - build: - context: ../ - dockerfile: ./test_project/Dockerfile - mysql_pull_only: - image: mysql:8.0 + scratch: + build: '.' + hello_world: + image: hello-world From 01fea7ee253a8ffe95af78d2593a2e6c30d4560a Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Sat, 15 Aug 2020 10:13:29 +0900 Subject: [PATCH 16/18] primary key --- src/LayerCache.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/LayerCache.ts b/src/LayerCache.ts index 29f9c7a6..7c1768d9 100644 --- a/src/LayerCache.ts +++ b/src/LayerCache.ts @@ -147,8 +147,8 @@ class LayerCache { // --- - async restore(key: string, restoreKeys?: string[]) { - const restoredCacheKey = await this.restoreRoot(restoreKeys) + async restore(primaryKey: string, restoreKeys?: string[]) { + const restoredCacheKey = await this.restoreRoot(primaryKey, restoreKeys) if (restoredCacheKey === undefined) { core.info(`Root cache could not be found. aborting.`) return undefined @@ -165,9 +165,9 @@ class LayerCache { return restoredCacheKey } - private async restoreRoot(restoreKeys?: string[]): Promise { + private async restoreRoot(primaryKey: string, restoreKeys?: string[]): Promise { core.debug(`Trying to restore root cache: ${ JSON.stringify({ restoreKeys, dir: this.getUnpackedTarDir() }) }`) - const restoredRootKey = await cache.restoreCache([this.getUnpackedTarDir()], ``, restoreKeys) + const restoredRootKey = await cache.restoreCache([this.getUnpackedTarDir()], primaryKey, restoreKeys) core.debug(`restoredRootKey: ${restoredRootKey}`) if (restoredRootKey === undefined) { return undefined From a9b1350927d1cdac6827f66c008b17cc95cde1b2 Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Sat, 15 Aug 2020 11:43:20 +0900 Subject: [PATCH 17/18] Use __dirname for saving path --- src/LayerCache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LayerCache.ts b/src/LayerCache.ts index 7c1768d9..3c8b72a1 100644 --- a/src/LayerCache.ts +++ b/src/LayerCache.ts @@ -14,7 +14,7 @@ class LayerCache { ids: string[] = [] unformattedSaveKey: string = '' restoredRootKey: string = '' - imagesDir: string = path.resolve(`${process.cwd()}/./.action-docker-layer-caching-docker_images`) + imagesDir: string = path.resolve(`${__dirname}/../.action-docker-layer-caching-docker_images`) enabledParallel = true concurrency: number = 4 From 50cf92b5bbc883ec91374ee3e7e356a0ea741784 Mon Sep 17 00:00:00 2001 From: satackey <21271711+satackey@users.noreply.github.com> Date: Sat, 15 Aug 2020 15:49:23 +0900 Subject: [PATCH 18/18] update moveLayerTarsInDir --- src/LayerCache.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/LayerCache.ts b/src/LayerCache.ts index 3c8b72a1..54c7464f 100644 --- a/src/LayerCache.ts +++ b/src/LayerCache.ts @@ -90,10 +90,9 @@ class LayerCache { } private async moveLayerTarsInDir(fromDir: string, toDir: string) { - const fromDirTrailingSlash = fromDir.endsWith(`/`) ? fromDir : `${fromDir}/` const layerTars = (await recursiveReaddir(fromDir)) .filter(path => path.endsWith(`/layer.tar`)) - .map(path => path.replace(fromDirTrailingSlash, ``)) + .map(path => path.replace(`${fromDir}/`, ``)) const moveLayer = async (layer: string) => { const from = path.resolve(`${fromDir}/${layer}`)