From 5894dfe0a730f9dd5e11f81dde3b58b3e5527711 Mon Sep 17 00:00:00 2001 From: Maxim Medvedev <redrathnure@gmail.com> Date: Tue, 24 Dec 2024 11:41:02 +0100 Subject: [PATCH 01/12] Server: use git tag info for local builds --- packages/tools/buildServerDocker.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/tools/buildServerDocker.ts b/packages/tools/buildServerDocker.ts index 479bb49b26a..b761ae41525 100644 --- a/packages/tools/buildServerDocker.ts +++ b/packages/tools/buildServerDocker.ts @@ -24,13 +24,13 @@ export function getIsPreRelease(_tagName: string): boolean { async function main() { const argv = require('yargs').argv as Argv; - if (!argv.tagName) throw new Error('--tag-name not provided'); + if (!argv.tagName) console.info('No `--tag-name` was specified. A latest git tag will be used instead.'); if (!argv.repository) throw new Error('--repository not provided'); const dryRun = !!argv.dryRun; const pushImages = !!argv.pushImages; const repository = argv.repository; - const tagName = argv.tagName; + const tagName = argv.tagName || `server-${await execCommand('git describe --tags --match v*', { showStdout: false })}`; const isPreRelease = getIsPreRelease(tagName); const imageVersion = getVersionFromTag(tagName, isPreRelease); const buildDate = moment(new Date().getTime()).format('YYYY-MM-DDTHH:mm:ssZ'); From 0ca867422628fff9ea7f62ceeb3163e6533e87ef Mon Sep 17 00:00:00 2001 From: Maxim Medvedev <redrathnure@gmail.com> Date: Tue, 24 Dec 2024 11:46:21 +0100 Subject: [PATCH 02/12] Server: docker BuildX is valid term --- packages/tools/cspell/dictionary4.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/tools/cspell/dictionary4.txt b/packages/tools/cspell/dictionary4.txt index 92b620e94b9..5f8d5f6d0f1 100644 --- a/packages/tools/cspell/dictionary4.txt +++ b/packages/tools/cspell/dictionary4.txt @@ -161,3 +161,4 @@ unwatcher pedr Slotozilla keyshortcuts +buildx From d9bca55f6f8f680d84cc8f6307c6914c62960a5f Mon Sep 17 00:00:00 2001 From: Maxim Medvedev <redrathnure@gmail.com> Date: Tue, 24 Dec 2024 11:46:31 +0100 Subject: [PATCH 03/12] Server: improve version parsing --- packages/tools/buildServerDocker.test.ts | 11 +++++++++-- packages/tools/buildServerDocker.ts | 15 ++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/packages/tools/buildServerDocker.test.ts b/packages/tools/buildServerDocker.test.ts index 332d23423bf..de01b6b8333 100644 --- a/packages/tools/buildServerDocker.test.ts +++ b/packages/tools/buildServerDocker.test.ts @@ -6,8 +6,15 @@ describe('buildServerDocker', () => { type TestCase = [string, boolean, string]; const testCases: TestCase[] = [ - ['server-v1.1.2-beta', true, '1.1.2-beta'], - ['server-v1.1.2', false, '1.1.2'], + ['server-v1.2.3-beta', true, '1.2.3-beta'], + ['server-v1.2.3-beta', false, '1.2.3'], + ['server-v1.2.3', false, '1.2.3'], + ['server-v1.2.3-zxc', true, '1.2.3-beta.zxc'], + ['server-v1.2.3-zxc', false, '1.2.3'], + ['server-v1.2.3-4-zxc', true, '1.2.3-beta.4.zxc'], + ['server-v1.2.3-4-zxc', false, '1.2.3'], + ['server-1.2.3-4-zxc', true, '1.2.3-beta.4.zxc'], + ['server-1.2.3-4-zxc', false, '1.2.3'], ]; for (const testCase of testCases) { diff --git a/packages/tools/buildServerDocker.ts b/packages/tools/buildServerDocker.ts index b761ae41525..3f6938d2f38 100644 --- a/packages/tools/buildServerDocker.ts +++ b/packages/tools/buildServerDocker.ts @@ -11,8 +11,12 @@ interface Argv { export function getVersionFromTag(tagName: string, isPreRelease: boolean): string { const s = tagName.split('-'); - const suffix = isPreRelease ? '-beta' : ''; - return s[1].substr(1) + suffix; + const mainVersion = s[1].replace(/^(v)/, ''); + const metaComponents = s.slice(2).filter(item => item !== 'beta'); + + // Append `git describe` components for pre release images. Mostly for case without `tagName` arg + const suffix = isPreRelease ? `-beta${metaComponents.length > 0 ? `.${metaComponents.join('.')}` : ''}` : ''; + return mainVersion + suffix; } export function getIsPreRelease(_tagName: string): boolean { @@ -43,10 +47,15 @@ async function main() { const buildArgs = `--build-arg BUILD_DATE="${buildDate}" --build-arg REVISION="${revision}" --build-arg VERSION="${imageVersion}"`; const dockerTags: string[] = []; const versionPart = imageVersion.split('.'); + const patchVersionPart = versionPart[2].split('-')[0]; dockerTags.push(isPreRelease ? 'beta' : 'latest'); dockerTags.push(versionPart[0] + (isPreRelease ? '-beta' : '')); dockerTags.push(`${versionPart[0]}.${versionPart[1]}${isPreRelease ? '-beta' : ''}`); - dockerTags.push(imageVersion); + dockerTags.push(`${versionPart[0]}.${versionPart[1]}.${patchVersionPart}${isPreRelease ? '-beta' : ''}`); + if (dockerTags.indexOf(imageVersion) < 0) { + dockerTags.push(imageVersion); + } + process.chdir(rootDir); console.info(`Running from: ${process.cwd()}`); From d9c79f39aa599e4279c3fa4476415101e5477e5a Mon Sep 17 00:00:00 2001 From: Maxim Medvedev <redrathnure@gmail.com> Date: Sun, 5 Jan 2025 12:31:05 +0100 Subject: [PATCH 04/12] Server: improve version and source args management --- Dockerfile.server | 5 +++-- packages/tools/buildServerDocker.ts | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Dockerfile.server b/Dockerfile.server index 076da5859ad..b9daac40adb 100644 --- a/Dockerfile.server +++ b/Dockerfile.server @@ -81,10 +81,11 @@ CMD ["yarn", "start-prod"] ARG BUILD_DATE ARG REVISION ARG VERSION +ARG SOURCE LABEL org.opencontainers.image.created="$BUILD_DATE" \ org.opencontainers.image.title="Joplin Server" \ org.opencontainers.image.description="Docker image for Joplin Server" \ org.opencontainers.image.url="https://joplinapp.org/" \ org.opencontainers.image.revision="$REVISION" \ - org.opencontainers.image.source="https://github.com/laurent22/joplin.git" \ - org.opencontainers.image.version="${VERSION}" + org.opencontainers.image.source="$SOURCE" \ + org.opencontainers.image.version="$VERSION" diff --git a/packages/tools/buildServerDocker.ts b/packages/tools/buildServerDocker.ts index 3f6938d2f38..44b56092632 100644 --- a/packages/tools/buildServerDocker.ts +++ b/packages/tools/buildServerDocker.ts @@ -35,6 +35,8 @@ async function main() { const pushImages = !!argv.pushImages; const repository = argv.repository; const tagName = argv.tagName || `server-${await execCommand('git describe --tags --match v*', { showStdout: false })}`; + const source = 'https://github.com/laurent22/joplin.git'; + const isPreRelease = getIsPreRelease(tagName); const imageVersion = getVersionFromTag(tagName, isPreRelease); const buildDate = moment(new Date().getTime()).format('YYYY-MM-DDTHH:mm:ssZ'); @@ -44,7 +46,7 @@ async function main() { } catch (error) { console.info('Could not get git commit: metadata revision field will be empty'); } - const buildArgs = `--build-arg BUILD_DATE="${buildDate}" --build-arg REVISION="${revision}" --build-arg VERSION="${imageVersion}"`; + const buildArgs = `--build-arg BUILD_DATE="${buildDate}" --build-arg REVISION="${revision}" --build-arg VERSION="${imageVersion}" --build-arg SOURCE="${source}"`; const dockerTags: string[] = []; const versionPart = imageVersion.split('.'); const patchVersionPart = versionPart[2].split('-')[0]; From 3b0215e7ea64110bec1073a076c4e21fedcc70f2 Mon Sep 17 00:00:00 2001 From: Maxim Medvedev <redrathnure@gmail.com> Date: Sun, 5 Jan 2025 12:41:26 +0100 Subject: [PATCH 05/12] Server: use docker BuildX --- packages/tools/buildServerDocker.ts | 43 ++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/packages/tools/buildServerDocker.ts b/packages/tools/buildServerDocker.ts index 44b56092632..3b4ab441f6c 100644 --- a/packages/tools/buildServerDocker.ts +++ b/packages/tools/buildServerDocker.ts @@ -7,6 +7,8 @@ interface Argv { pushImages?: boolean; repository?: string; tagName?: string; + platform?: string; + source?: string; } export function getVersionFromTag(tagName: string, isPreRelease: boolean): string { @@ -35,6 +37,7 @@ async function main() { const pushImages = !!argv.pushImages; const repository = argv.repository; const tagName = argv.tagName || `server-${await execCommand('git describe --tags --match v*', { showStdout: false })}`; + const platform = argv.platform || 'linux/amd64'; const source = 'https://github.com/laurent22/joplin.git'; const isPreRelease = getIsPreRelease(tagName); @@ -46,14 +49,20 @@ async function main() { } catch (error) { console.info('Could not get git commit: metadata revision field will be empty'); } - const buildArgs = `--build-arg BUILD_DATE="${buildDate}" --build-arg REVISION="${revision}" --build-arg VERSION="${imageVersion}" --build-arg SOURCE="${source}"`; + + const buildArgs = []; + buildArgs.push(`BUILD_DATE="${buildDate}"`); + buildArgs.push(`REVISION="${revision}"`); + buildArgs.push(`VERSION="${imageVersion}"`); + buildArgs.push(`SOURCE="${source}"`); + const dockerTags: string[] = []; - const versionPart = imageVersion.split('.'); - const patchVersionPart = versionPart[2].split('-')[0]; + const versionParts = imageVersion.split('.'); + const patchVersionPart = versionParts[2].split('-')[0]; dockerTags.push(isPreRelease ? 'beta' : 'latest'); - dockerTags.push(versionPart[0] + (isPreRelease ? '-beta' : '')); - dockerTags.push(`${versionPart[0]}.${versionPart[1]}${isPreRelease ? '-beta' : ''}`); - dockerTags.push(`${versionPart[0]}.${versionPart[1]}.${patchVersionPart}${isPreRelease ? '-beta' : ''}`); + dockerTags.push(versionParts[0] + (isPreRelease ? '-beta' : '')); + dockerTags.push(`${versionParts[0]}.${versionParts[1]}${isPreRelease ? '-beta' : ''}`); + dockerTags.push(`${versionParts[0]}.${versionParts[1]}.${patchVersionPart}${isPreRelease ? '-beta' : ''}`); if (dockerTags.indexOf(imageVersion) < 0) { dockerTags.push(imageVersion); } @@ -62,24 +71,32 @@ async function main() { process.chdir(rootDir); console.info(`Running from: ${process.cwd()}`); + console.info('repository:', repository); console.info('tagName:', tagName); + console.info('platform:', platform); console.info('pushImages:', pushImages); console.info('imageVersion:', imageVersion); console.info('isPreRelease:', isPreRelease); console.info('Docker tags:', dockerTags.join(', ')); - const dockerCommand = `docker build --progress=plain -t "${repository}:${imageVersion}" ${buildArgs} -f Dockerfile.server .`; + const cliArgs = ['--progress=plain']; + cliArgs.push(`--platform ${platform}`); + cliArgs.push(...dockerTags.map(tag => `--tag "${repository}:${tag}"`)); + cliArgs.push(...buildArgs.map(arg => `--build-arg ${arg}`)); + if (pushImages) { + cliArgs.push('--push'); + } + cliArgs.push('-f Dockerfile.server'); + cliArgs.push('.'); + + const dockerCommand = `docker buildx build ${cliArgs.join(' ')}`; + + console.info('exec:', dockerCommand); if (dryRun) { - console.info(dockerCommand); return; } await execCommand(dockerCommand); - - for (const tag of dockerTags) { - await execCommand(`docker tag "${repository}:${imageVersion}" "${repository}:${tag}"`); - if (pushImages) await execCommand(`docker push ${repository}:${tag}`); - } } if (require.main === module) { From 9bc2fff44a718382115df485abddbd7328927dd2 Mon Sep 17 00:00:00 2001 From: Maxim Medvedev <redrathnure@gmail.com> Date: Tue, 24 Dec 2024 11:47:12 +0100 Subject: [PATCH 06/12] Server: cache mounts for yarn --- Dockerfile.server | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile.server b/Dockerfile.server index b9daac40adb..8c1742827e9 100644 --- a/Dockerfile.server +++ b/Dockerfile.server @@ -47,9 +47,9 @@ RUN sed --in-place '/onenote-converter/d' ./packages/lib/package.json # Note that `yarn install` ignores `NODE_ENV=production` and will install dev # dependencies too, but this is fine because we need them to build the app. -RUN BUILD_SEQUENCIAL=1 yarn install --inline-builds \ - && yarn cache clean \ - && rm -rf .yarn/berry +RUN --mount=type=cache,target=/build/.yarn/cache --mount=type=cache,target=/build/.yarn/berry/cache\ + BUILD_SEQUENCIAL=1 yarn config set cacheFolder /build/.yarn/cache \ + && yarn install --inline-builds # ============================================================================= # Final stage - we copy only the relevant files from the build stage and start From 4418212a9b5fc3fda882557e4e876a0f837df6db Mon Sep 17 00:00:00 2001 From: Maxim Medvedev <redrathnure@gmail.com> Date: Sun, 5 Jan 2025 14:07:15 +0100 Subject: [PATCH 07/12] Server: add arg to force `latest` flag `--addLatestTag` will force `latest` tag creation even for pre-release versions --- packages/tools/buildServerDocker.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/tools/buildServerDocker.ts b/packages/tools/buildServerDocker.ts index 3b4ab441f6c..29663126514 100644 --- a/packages/tools/buildServerDocker.ts +++ b/packages/tools/buildServerDocker.ts @@ -9,6 +9,7 @@ interface Argv { tagName?: string; platform?: string; source?: string; + addLatestTag?: boolean; } export function getVersionFromTag(tagName: string, isPreRelease: boolean): string { @@ -34,6 +35,7 @@ async function main() { if (!argv.repository) throw new Error('--repository not provided'); const dryRun = !!argv.dryRun; + const addLatestTag = !!argv.addLatestTag; const pushImages = !!argv.pushImages; const repository = argv.repository; const tagName = argv.tagName || `server-${await execCommand('git describe --tags --match v*', { showStdout: false })}`; @@ -66,6 +68,9 @@ async function main() { if (dockerTags.indexOf(imageVersion) < 0) { dockerTags.push(imageVersion); } + if (addLatestTag && dockerTags.indexOf('latest') < 0) { + dockerTags.push('latest'); + } process.chdir(rootDir); From 1ebe52044aa006fc9e0eff69ed52a8c86a0078bf Mon Sep 17 00:00:00 2001 From: Maxim Medvedev <redrathnure@gmail.com> Date: Sun, 5 Jan 2025 14:49:28 +0100 Subject: [PATCH 08/12] Server: add proper arg parsion for `yarn buildServerDocker` Just run `yarn buildServerDocker` --- packages/tools/buildServerDocker.ts | 60 +++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 7 deletions(-) diff --git a/packages/tools/buildServerDocker.ts b/packages/tools/buildServerDocker.ts index 29663126514..b1f99249f2d 100644 --- a/packages/tools/buildServerDocker.ts +++ b/packages/tools/buildServerDocker.ts @@ -12,6 +12,53 @@ interface Argv { addLatestTag?: boolean; } +function parseArgv(): Argv { + return require('yargs') + .scriptName('yarn buildServerDocker') + .usage('$0 --repository OWNER/IMAGE [args]') + .option('r', { + alias: 'repository', + describe: 'Target image repository. Usually in format `OWNER/NAME`', + demandOption: true, + type: 'string', + }) + .option('t', { + alias: 'tagName', + describe: 'Base image tag. Usually should be in format `server-v1.2.3` or `server-v1.2.3-beta`. The latest `server-v*` git tag will be used by default.', + type: 'string', + }) + .option('l', { + alias: 'addLatestTag', + describe: 'Add `latest` tag even for pre-release images.', + type: 'boolean', + default: false, + }) + .option('platform', { + describe: 'Comma separated list of target image platforms.', + type: 'string', + default: 'linux/amd64', + }) + .option('source', { + describe: 'Source Git repository for the images.', + type: 'string', + default: 'https://github.com/laurent22/joplin.git', + }) + .option('p', { + alias: 'pushImages', + describe: 'Publish images to target repository.', + type: 'boolean', + default: false, + }) + .option('dryRun', { + alias: 'dryRun', + describe: 'Do not call docker, just show command instead.', + type: 'boolean', + default: false, + }) + .help() + .argv as Argv; +} + export function getVersionFromTag(tagName: string, isPreRelease: boolean): string { const s = tagName.split('-'); const mainVersion = s[1].replace(/^(v)/, ''); @@ -30,17 +77,16 @@ export function getIsPreRelease(_tagName: string): boolean { } async function main() { - const argv = require('yargs').argv as Argv; + const argv = parseArgv(); if (!argv.tagName) console.info('No `--tag-name` was specified. A latest git tag will be used instead.'); - if (!argv.repository) throw new Error('--repository not provided'); - const dryRun = !!argv.dryRun; - const addLatestTag = !!argv.addLatestTag; - const pushImages = !!argv.pushImages; + const dryRun = argv.dryRun; + const addLatestTag = argv.addLatestTag; + const pushImages = argv.pushImages; const repository = argv.repository; const tagName = argv.tagName || `server-${await execCommand('git describe --tags --match v*', { showStdout: false })}`; - const platform = argv.platform || 'linux/amd64'; - const source = 'https://github.com/laurent22/joplin.git'; + const platform = argv.platform; + const source = argv.source; const isPreRelease = getIsPreRelease(tagName); const imageVersion = getVersionFromTag(tagName, isPreRelease); From dd0ccd7e0361a4d5b52617ca7ced14c876c4d756 Mon Sep 17 00:00:00 2001 From: Maxim Medvedev <redrathnure@gmail.com> Date: Sun, 5 Jan 2025 14:59:45 +0100 Subject: [PATCH 09/12] Server: rename `beta` to `latest-beta` tag --- packages/tools/buildServerDocker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tools/buildServerDocker.ts b/packages/tools/buildServerDocker.ts index b1f99249f2d..64caf410669 100644 --- a/packages/tools/buildServerDocker.ts +++ b/packages/tools/buildServerDocker.ts @@ -107,7 +107,7 @@ async function main() { const dockerTags: string[] = []; const versionParts = imageVersion.split('.'); const patchVersionPart = versionParts[2].split('-')[0]; - dockerTags.push(isPreRelease ? 'beta' : 'latest'); + dockerTags.push(isPreRelease ? 'latest-beta' : 'latest'); dockerTags.push(versionParts[0] + (isPreRelease ? '-beta' : '')); dockerTags.push(`${versionParts[0]}.${versionParts[1]}${isPreRelease ? '-beta' : ''}`); dockerTags.push(`${versionParts[0]}.${versionParts[1]}.${patchVersionPart}${isPreRelease ? '-beta' : ''}`); From c7e9fce580aa331a29f0a49eebdf872d4108aaa7 Mon Sep 17 00:00:00 2001 From: Maxim Medvedev <redrathnure@gmail.com> Date: Thu, 9 Jan 2025 19:55:25 +0100 Subject: [PATCH 10/12] Server: adjust CLI help test --- packages/tools/buildServerDocker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tools/buildServerDocker.ts b/packages/tools/buildServerDocker.ts index 64caf410669..0e35bc4750a 100644 --- a/packages/tools/buildServerDocker.ts +++ b/packages/tools/buildServerDocker.ts @@ -34,7 +34,7 @@ function parseArgv(): Argv { default: false, }) .option('platform', { - describe: 'Comma separated list of target image platforms.', + describe: 'Comma separated list of target image platforms. E.g. `linux/amd64` or `linux/amd64,linux/arm64`', type: 'string', default: 'linux/amd64', }) From fb2e52099c2aebf83ddfe2af81c22a7c56928101 Mon Sep 17 00:00:00 2001 From: Maxim Medvedev <redrathnure@gmail.com> Date: Thu, 9 Jan 2025 20:04:08 +0100 Subject: [PATCH 11/12] Server: fix build ARM64 images --- Dockerfile.server | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile.server b/Dockerfile.server index 8c1742827e9..2ab61b9404f 100644 --- a/Dockerfile.server +++ b/Dockerfile.server @@ -7,6 +7,9 @@ FROM node:18 AS builder RUN apt-get update \ && apt-get install -y \ python3 tini \ + # needed for node-canvas for ARM32 platform. + # See also https://github.com/Automattic/node-canvas/wiki/Installation:-Ubuntu-and-other-Debian-based-systems + libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev \ && rm -rf /var/lib/apt/lists/* # Enables Yarn From efc749129b81721d6a754ddb707e126cb54c959a Mon Sep 17 00:00:00 2001 From: Maxim Medvedev <redrathnure@gmail.com> Date: Thu, 9 Jan 2025 20:10:14 +0100 Subject: [PATCH 12/12] Server: add docker-buildx-plugin package to main workflow --- .github/workflows/github-actions-main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/github-actions-main.yml b/.github/workflows/github-actions-main.yml index ca2e938271b..7d09b7d45bf 100644 --- a/.github/workflows/github-actions-main.yml +++ b/.github/workflows/github-actions-main.yml @@ -65,7 +65,7 @@ jobs: "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update || true - sudo apt-get install -y docker-ce docker-ce-cli containerd.io + sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin - uses: actions/checkout@v4 - uses: olegtarasov/get-tag@v2.1.3 @@ -165,7 +165,7 @@ jobs: "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update || true - sudo apt-get install -y docker-ce docker-ce-cli containerd.io + sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin - uses: actions/checkout@v4