diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d4a1fcf1..ffeac0b0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,18 +45,18 @@ jobs: script: | return { inspect_image: [ - 'test_project_node', - 'amazon/aws-cli', + 'test_project_scratch', + 'hello-world', ], include: [ { - inspect_image: 'test_project_node', + inspect_image: 'test_project_scratch', prepare_command: 'docker-compose -f test_project/docker-compose.yml -p test_project pull', build_command: 'docker-compose -f test_project/docker-compose.yml -p test_project build', }, { - inspect_image: 'amazon/aws-cli', + inspect_image: 'hello-world', prepare_command: ':', - build_command: 'docker pull amazon/aws-cli', + build_command: 'docker pull hello-world', }, ], } @@ -91,7 +91,6 @@ jobs: name: Run satackey/action-docker-layer-caching@${{ steps.extract.outputs.branch }} with: key: docker-layer-caching-${{ matrix.inspect_image }}-sha:${{ github.sha }}-{hash} - restore-keys: docker-layer-caching-${{ matrix.inspect_image }}-sha:${{ github.sha }}- - run: ${{ matrix.build_command }} @@ -119,7 +118,7 @@ jobs: - uses: ./action-dlc name: Run satackey/action-docker-layer-caching@${{ steps.extract.outputs.branch }} with: - key: docker-layer-caching-${{ matrix.inspect_image }}-sha:${{ github.sha }}-{hash} + key: never-restored-docker-layer-caching-${{ matrix.inspect_image }}-sha:${{ github.sha }}-{hash} restore-keys: docker-layer-caching-${{ matrix.inspect_image }}-sha:${{ github.sha }}- skip-save: 'true' diff --git a/main.ts b/main.ts index 65ad632a..25efb9e2 100644 --- a/main.ts +++ b/main.ts @@ -4,11 +4,12 @@ import { LayerCache } from './src/LayerCache' import { ImageDetector } from './src/ImageDetector' const main = async () => { - // const repotag = core.getInput(`repotag`, { required: true }) const primaryKey = core.getInput(`key`, { required: true }) const restoreKeys = core.getInput(`restore-keys`, { required: false }).split(`\n`).filter(key => key !== ``) - core.saveState(`already-existing-images`, JSON.stringify(await new ImageDetector().getExistingImages())) + const imageDetector = new ImageDetector() + + const alreadyExistingImages = await imageDetector.getExistingImages() const layerCache = new LayerCache([]) layerCache.concurrency = parseInt(core.getInput(`concurrency`, { required: true }), 10) @@ -16,6 +17,8 @@ const main = async () => { await layerCache.cleanUp() core.saveState(`restored-key`, JSON.stringify(restoredKey !== undefined ? restoredKey : '')) + core.saveState(`already-existing-images`, JSON.stringify(alreadyExistingImages)) + core.saveState(`restored-images`, JSON.stringify(await imageDetector.getImagesShouldSave(alreadyExistingImages))) } main().catch(e => { diff --git a/package.json b/package.json index 09b00d06..07c23c17 100644 --- a/package.json +++ b/package.json @@ -8,9 +8,11 @@ "dependencies": { "@actions/cache": "^1.0.2", "@actions/core": "^1.2.4", + "@types/recursive-readdir": "^2.2.0", "actions-exec-listener": "^0.0.2", "crypto": "^1.0.1", "native-promise-pool": "^3.13.0", + "recursive-readdir": "^2.2.2", "string-format": "^2.0.0", "typescript-is": "^0.16.3" }, diff --git a/post.ts b/post.ts index 529e8f2d..2d56eb22 100644 --- a/post.ts +++ b/post.ts @@ -1,5 +1,4 @@ import * as core from '@actions/core' -import exec from 'actions-exec-listener' import { LayerCache } from './src/LayerCache' import { ImageDetector } from './src/ImageDetector' @@ -12,26 +11,24 @@ const main = async () => { } const primaryKey = core.getInput('key', { required: true }) - const restoredKey = JSON.parse(core.getState(`restored-key`)) as string - const rawAlreadyExistingImages = core.getState(`already-existing-images`) - assertType(rawAlreadyExistingImages) - const alreadyExistingImages = JSON.parse(rawAlreadyExistingImages) + const restoredKey = JSON.parse(core.getState(`restored-key`)) + const alreadyExistingImages = JSON.parse(core.getState(`already-existing-images`)) + const restoredImages = JSON.parse(core.getState(`restored-images`)) + + assertType(restoredKey) assertType(alreadyExistingImages) + assertType(restoredImages) const imageDetector = new ImageDetector() - imageDetector.registerAlreadyExistedImages(alreadyExistingImages) - await imageDetector.getExistingImages() - core.debug(JSON.stringify({ imageIdsToSave: imageDetector.getImagesShouldSave() })) - const layerCache = new LayerCache(imageDetector.getImagesShouldSave()) - layerCache.concurrency = parseInt(core.getInput(`concurrency`, { required: true }), 10) - - layerCache.unformattedOrigianlKey = primaryKey - core.debug(JSON.stringify({ restoredKey, formattedOriginalCacheKey: layerCache.getFormattedOriginalCacheKey()})) - if (restoredKey !== `` && restoredKey === layerCache.getFormattedOriginalCacheKey()) { - core.info(`Key ${restoredKey} already exists, skip storing.`) + if (await imageDetector.checkIfImageHasAdded(restoredImages)) { + core.info(`Key ${restoredKey} already exists, not saving cache.`) return } + + const layerCache = new LayerCache(await imageDetector.getImagesShouldSave(alreadyExistingImages)) + layerCache.concurrency = parseInt(core.getInput(`concurrency`, { required: true }), 10) + await layerCache.store(primaryKey) await layerCache.cleanUp() } diff --git a/src/ImageDetector.ts b/src/ImageDetector.ts index 4b7079c6..1ab23033 100644 --- a/src/ImageDetector.ts +++ b/src/ImageDetector.ts @@ -2,24 +2,24 @@ import exec from 'actions-exec-listener' import * as core from '@actions/core' export class ImageDetector { - alreadyExistedImages: Set = new Set([]) - existingImages: Set = new Set([]) - registerAlreadyExistedImages(images: string[]) { - images.forEach(image => this.alreadyExistedImages.add(image)) - } - async getExistingImages(): Promise { + const existingSet = new Set([]) const ids = (await exec.exec(`docker image ls -q`, [], { silent: true })).stdoutStr.split(`\n`).filter(id => id !== ``) const repotags = (await exec.exec(`sh -c "docker image ls --format '{{ .Repository }}:{{ .Tag }}' --filter 'dangling=false'"`, [], { silent: true })).stdoutStr.split(`\n`).filter(id => id !== ``); core.debug(JSON.stringify({ log: "getExistingImages", ids, repotags })); - ([...ids, ...repotags]).forEach(image => this.existingImages.add(image)) - core.debug(JSON.stringify({ existingImages: this.existingImages })) - return Array.from(this.existingImages) + ([...ids, ...repotags]).forEach(image => existingSet.add(image)) + core.debug(JSON.stringify({ existingSet })) + return Array.from(existingSet) } - getImagesShouldSave(): string[] { - const resultSet = new Set(this.existingImages.values()) - this.alreadyExistedImages.forEach(image => resultSet.delete(image)) + async getImagesShouldSave(alreadRegisteredImages: string[]): Promise { + const resultSet = new Set(await this.getExistingImages()) + alreadRegisteredImages.forEach(image => resultSet.delete(image)) return Array.from(resultSet) } + + async checkIfImageHasAdded(restoredImages: string[]): Promise { + const existing = await this.getExistingImages() + return JSON.stringify(restoredImages) === JSON.stringify(existing) + } } diff --git a/src/LayerCache.ts b/src/LayerCache.ts index 5251ef68..54c7464f 100644 --- a/src/LayerCache.ts +++ b/src/LayerCache.ts @@ -5,41 +5,35 @@ import * as core from '@actions/core' import * as cache from '@actions/cache' import { ExecOptions } from '@actions/exec/lib/interfaces' import { promises as fs } from 'fs' -import { assertManifests, Manifest, Manifests } from './Tar' +import recursiveReaddir from 'recursive-readdir' +import { Manifest, loadManifests, loadRawManifests } from './Tar' import format from 'string-format' import PromisePool from 'native-promise-pool' class LayerCache { - // repotag: string - ids: string[] - unformattedOrigianlKey: string = '' - restoredOriginalKey?: string - // tarFile: string = '' - imagesDir: string = path.resolve(`${process.cwd()}/./.action-docker-layer-caching-docker_images`) + ids: string[] = [] + unformattedSaveKey: string = '' + restoredRootKey: string = '' + imagesDir: string = path.resolve(`${__dirname}/../.action-docker-layer-caching-docker_images`) enabledParallel = true - // unpackedTarDir: string = '' - // manifests: Manifests = [] concurrency: number = 4 + static ERROR_CACHE_ALREAD_EXISTS_STR = `Cache already exists` + static ERROR_LAYER_CACHE_NOT_FOUND_STR = `Layer cache not found` + constructor(ids: string[]) { - // this.repotag = repotag this.ids = ids } async exec(command: string, args?: string[], options?: ExecOptions) { - const argsStr = args != null ? args.join(' ') : '' - core.startGroup(`${command} ${argsStr}`) const result = await exec.exec(command, args, options) - core.endGroup() return result } async store(key: string) { - this.unformattedOrigianlKey = key - // this.originalKeyToStore = format(key, { - // hash: this.getIdhashesPathFriendly() - // }) + this.unformattedSaveKey = key + await this.saveImageAsUnpacked() if (this.enabledParallel) { await this.separateAllLayerCaches() @@ -73,18 +67,16 @@ class LayerCache { } private async getManifests() { - const manifests = JSON.parse((await fs.readFile(`${this.getUnpackedTarDir()}/manifest.json`)).toString()) - assertManifests(manifests) - return manifests + return loadManifests(this.getUnpackedTarDir()) } private async storeRoot() { - const rootKey = this.getRootKey() + const rootKey = await this.generateRootSaveKey() const paths = [ this.getUnpackedTarDir(), ] core.info(`Start storing root cache, key: ${rootKey}, dir: ${paths}`) - const cacheId = await LayerCache.dismissCacheAlreadyExistsError(cache.saveCache(paths, rootKey)) + const cacheId = await LayerCache.dismissError(cache.saveCache(paths, rootKey), LayerCache.ERROR_CACHE_ALREAD_EXISTS_STR, -1) core.info(`Stored root cache, key: ${rootKey}, id: ${cacheId}`) return cacheId !== -1 ? cacheId : undefined } @@ -98,7 +90,10 @@ class LayerCache { } private async moveLayerTarsInDir(fromDir: string, toDir: string) { - const layerTars = (await exec.exec(`find . -name layer.tar`, [], { cwd: fromDir, silent: true })).stdoutStr.split(`\n`).filter(tar => tar !== '') + const layerTars = (await recursiveReaddir(fromDir)) + .filter(path => path.endsWith(`/layer.tar`)) + .map(path => path.replace(`${fromDir}/`, ``)) + const moveLayer = async (layer: string) => { const from = path.resolve(`${fromDir}/${layer}`) const to = path.resolve(`${toDir}/${layer}`) @@ -122,29 +117,28 @@ class LayerCache { return result } - static async dismissCacheAlreadyExistsError(promise: Promise): Promise { - let result = -1 + static async dismissError(promise: Promise, dismissStr: string, defaultResult: T): Promise { try { - result = await promise - return result + return await promise } catch (e) { core.debug(`catch error: ${e.toString()}`) - if (typeof e.message !== 'string' || !e.message.includes(`Cache already exists`)) { + if (typeof e.message !== 'string' || !e.message.includes(dismissStr)) { core.error(`Unexpected error: ${e.toString()}`) throw e } - core.info(`info: Cache already exists: ${e.toString()}`) + + core.info(`${dismissStr}: ${e.toString()}`) core.debug(e) - return result + return defaultResult } } private async storeSingleLayerBy(id: string): Promise { const path = this.genSingleLayerStorePath(id) - const key = this.genSingleLayerStoreKey(id) + const key = await this.generateSingleLayerSaveKey(id) core.info(`Start storing layer cache: ${key}`) - const cacheId = await LayerCache.dismissCacheAlreadyExistsError(cache.saveCache([path], key)) + const cacheId = await LayerCache.dismissError(cache.saveCache([path], key), LayerCache.ERROR_CACHE_ALREAD_EXISTS_STR, -1) core.info(`Stored layer cache, key: ${key}, id: ${cacheId}`) return cacheId @@ -152,10 +146,8 @@ class LayerCache { // --- - async restore(key: string, restoreKeys?: string[]) { - this.unformattedOrigianlKey = key - // const restoreKeysIncludedRootKey = [key, ...(restoreKeys !== undefined ? restoreKeys : [])] - const restoredCacheKey = await this.restoreRoot(restoreKeys) + async restore(primaryKey: string, restoreKeys?: string[]) { + const restoredCacheKey = await this.restoreRoot(primaryKey, restoreKeys) if (restoredCacheKey === undefined) { core.info(`Root cache could not be found. aborting.`) return undefined @@ -172,40 +164,49 @@ class LayerCache { return restoredCacheKey } - private async restoreRoot(restoreKeys?: string[]): Promise { - core.debug(`Trying to restore root cache: ${ JSON.stringify({ primaryKey: this.getRootKey(), restoreKeys, dir: this.getUnpackedTarDir() }) }`) - const restoredCacheKeyMayUndefined = await cache.restoreCache([this.getUnpackedTarDir()], this.getRootKey(), restoreKeys) - core.debug(`restoredCacheKeyMayUndefined: ${restoredCacheKeyMayUndefined}`) - if (restoredCacheKeyMayUndefined === undefined) { + private async restoreRoot(primaryKey: string, restoreKeys?: string[]): Promise { + core.debug(`Trying to restore root cache: ${ JSON.stringify({ restoreKeys, dir: this.getUnpackedTarDir() }) }`) + const restoredRootKey = await cache.restoreCache([this.getUnpackedTarDir()], primaryKey, restoreKeys) + core.debug(`restoredRootKey: ${restoredRootKey}`) + if (restoredRootKey === undefined) { return undefined } - this.restoredOriginalKey = restoredCacheKeyMayUndefined.replace(/-root$/, '') - return this.restoredOriginalKey + this.restoredRootKey = restoredRootKey + + return restoredRootKey } - private async restoreLayers() { + private async restoreLayers(): Promise { const pool = new PromisePool(this.concurrency) - const restoredLayerKeysThatMayContainUndefined = await Promise.all( - (await this.getLayerIds()).map( - layerId => { - return pool.open(() => this.restoreSingleLayerBy(layerId)) - } - ) + const tasks = (await this.getLayerIds()).map( + layerId => pool.open(() => this.restoreSingleLayerBy(layerId)) ) - core.debug(JSON.stringify({ log: `restoreLayers`, restoredLayerKeysThatMayContainUndefined })) - const FailedToRestore = (restored: string | undefined) => restored === undefined - return restoredLayerKeysThatMayContainUndefined.filter(FailedToRestore).length === 0 + try { + await Promise.all(tasks) + } catch (e) { + if (typeof e.message === `string` && e.message.includes(LayerCache.ERROR_LAYER_CACHE_NOT_FOUND_STR)) { + core.info(e.message) + + // Avoid UnhandledPromiseRejectionWarning + tasks.map(task => task.catch(core.info)) + + return false + } + throw e + } + + return true } private async restoreSingleLayerBy(id: string): Promise { core.debug(JSON.stringify({ log: `restoreSingleLayerBy`, id })) - const result = await cache.restoreCache([this.genSingleLayerStorePath(id)], this.genSingleLayerStoreKey(id)) + const result = await cache.restoreCache([this.genSingleLayerStorePath(id)], await this.recoverSingleLayerKey(id)) if (result == null) { - throw new Error(`Layer cache not found: ${JSON.stringify({ id })}`) + throw new Error(`${LayerCache.ERROR_LAYER_CACHE_NOT_FOUND_STR}: ${JSON.stringify({ id })}`) } return result @@ -241,33 +242,46 @@ class LayerCache { return 'image' } - getIdhashesPathFriendly(): string { - const result = crypto.createHash(`sha256`).update(this.ids.join(`-`), `utf8`).digest(`hex`) - core.debug(JSON.stringify({ log: `getIdhashesPathFriendly`, result })) - return result + genSingleLayerStorePath(id: string) { + return `${this.getLayerCachesDir()}/${id}/layer.tar` } - getRootKey(): string { - return `${this.getFormattedOriginalCacheKey()}-root` + async generateRootHashFromManifest(): Promise { + const manifest = await loadRawManifests(this.getUnpackedTarDir()) + return crypto.createHash(`sha256`).update(manifest, `utf8`).digest(`hex`) } - genSingleLayerStorePath(id: string) { - return `${this.getLayerCachesDir()}/${id}/layer.tar` + async generateRootSaveKey(): Promise { + const rootHash = await this.generateRootHashFromManifest() + const formatted = await this.getFormattedSaveKey(rootHash) + core.debug(JSON.stringify({ log: `generateRootSaveKey`, rootHash, formatted })) + return `${formatted}-root` } - genSingleLayerStoreKey(id: string) { - const singleLayerStoreKey = this.getFormattedOriginalCacheKey(id) - core.debug(JSON.stringify({ log: `genSingleLayerStoreKey`, singleLayerStoreKey, id })) - return `layer-${singleLayerStoreKey}` + async generateSingleLayerSaveKey(id: string) { + const formatted = await this.getFormattedSaveKey(id) + core.debug(JSON.stringify({ log: `generateSingleLayerSaveKey`, formatted, id })) + return `layer-${formatted}` + } + + async recoverSingleLayerKey(id: string) { + const unformatted = await this.recoverUnformattedSaveKey() + return format(`layer-${unformatted}`, { hash: id }) } - getFormattedOriginalCacheKey(hashThatMayBeSpecified?: string) { - const hash = hashThatMayBeSpecified !== undefined ? hashThatMayBeSpecified : this.getIdhashesPathFriendly() - const result = format(this.unformattedOrigianlKey, { hash }) - core.debug(JSON.stringify({ log: `getFormattedOriginalCacheKey`, hash, hashThatMayBeSpecified, result })) + async getFormattedSaveKey(hash: string) { + const result = format(this.unformattedSaveKey, { hash }) + core.debug(JSON.stringify({ log: `getFormattedSaveKey`, hash, result })) return result } + async recoverUnformattedSaveKey() { + const hash = await this.generateRootHashFromManifest() + core.debug(JSON.stringify({ log: `recoverUnformattedSaveKey`, hash})) + + return this.restoredRootKey.replace(hash, `{hash}`).replace(/-root$/, ``) + } + async getLayerTarFiles(): Promise { const getTarFilesFromManifest = (manifest: Manifest) => manifest.Layers diff --git a/src/Tar.ts b/src/Tar.ts index f5ef8cec..99653304 100644 --- a/src/Tar.ts +++ b/src/Tar.ts @@ -1,4 +1,5 @@ import { assertType } from 'typescript-is' +import { promises as fs } from 'fs' export interface Manifest { Config: string @@ -11,3 +12,13 @@ export type Manifests = Manifest[] export function assertManifests(x: unknown): asserts x is Manifests { assertType(x) } + +export async function loadRawManifests(path: string) { + return (await fs.readFile(`${path}/manifest.json`)).toString() +} +export async function loadManifests(path: string) { + const raw = await loadRawManifests(path) + const manifests = JSON.parse(raw.toString()) + assertManifests(manifests) + return manifests +} diff --git a/test_project/Dockerfile b/test_project/Dockerfile index cd52b105..7da685b0 100644 --- a/test_project/Dockerfile +++ b/test_project/Dockerfile @@ -1,23 +1,5 @@ -FROM node:12-alpine as curl-env -RUN set -x \ - apk update && \ - apk add curl && \ - mkdir -p /src && \ - curl -o /src/install.sh -L https://yarnpkg.com/install.sh && \ - chmod +x /src/install.sh && \ - apk del curl +FROM alpine AS data +RUN date > /now.txt -FROM node:12-alpine - -COPY --from=curl-env /src/install.sh /tmp/install.sh -RUN set -x \ - apk update && \ - apk add curl && \ - /tmp/install.sh && \ - apk del curl - -WORKDIR /app -COPY package.json yarn.lock ./ -RUN yarn install --frozen-lockfile - -# COPY . ./ +FROM scratch +COPY --from=data /now.txt /data_stage_built_at.txt diff --git a/test_project/docker-compose.yml b/test_project/docker-compose.yml index 9437dfa8..67129bb6 100644 --- a/test_project/docker-compose.yml +++ b/test_project/docker-compose.yml @@ -1,9 +1,7 @@ version: '3' services: - node: - build: - context: ../ - dockerfile: ./test_project/Dockerfile - mysql_pull_only: - image: mysql:8.0 + scratch: + build: '.' + hello_world: + image: hello-world diff --git a/yarn.lock b/yarn.lock index 86826840..7a6c72b2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -186,6 +186,13 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-14.0.27.tgz#a151873af5a5e851b51b3b065c9e63390a9e0eb1" integrity sha512-kVrqXhbclHNHGu9ztnAwSncIgJv/FaxmzXJvGXNdcCpV1b8u1/Mi6z6m0vwy0LzKeXFTPLH0NzwmoJ3fNCIq0g== +"@types/recursive-readdir@^2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@types/recursive-readdir/-/recursive-readdir-2.2.0.tgz#b39cd5474fd58ea727fe434d5c68b7a20ba9121c" + integrity sha512-HGk753KRu2N4mWduovY4BLjYq4jTOL29gV2OfGdGxHcPSWGFkC5RRIdk+VTs5XmYd7MVAD+JwKrcb5+5Y7FOCg== + dependencies: + "@types/node" "*" + "@types/string-format@^2.0.0": version "2.0.0" resolved "https://registry.yarnpkg.com/@types/string-format/-/string-format-2.0.0.tgz#c1588f507be7b8ef5eb5074a41e48e4538f3f6d5" @@ -322,7 +329,7 @@ mime-types@^2.1.12: dependencies: mime-db "1.44.0" -minimatch@^3.0.4: +minimatch@3.0.4, minimatch@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== @@ -364,6 +371,13 @@ punycode@^2.1.1: resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== +recursive-readdir@^2.2.2: + version "2.2.2" + resolved "https://registry.yarnpkg.com/recursive-readdir/-/recursive-readdir-2.2.2.tgz#9946fb3274e1628de6e36b2f6714953b4845094f" + integrity sha512-nRCcW9Sj7NuZwa2XvH9co8NPeXUBhZP7CRKJtU+cS6PW9FpCIFoI5ib0NT1ZrbNuPoRy0ylyCaUL8Gih4LSyFg== + dependencies: + minimatch "3.0.4" + reflect-metadata@>=0.1.12: version "0.1.13" resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.13.tgz#67ae3ca57c972a2aa1642b10fe363fe32d49dc08"