Server: buildx support for Docker images (#11582)

Co-authored-by: Laurent Cozic <laurent22@users.noreply.github.com>

.github/workflows/github-actions-main.yml

@@ -30,7 +30,7 @@ jobs:
               "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
               $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
           sudo apt-get update || true
-          sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin
 
       # Login to Docker only if we're on a server release tag. If we run this on
       # a pull request it will fail because the PR doesn't have access to
@@ -106,7 +106,7 @@ jobs:
               "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
               $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
           sudo apt-get update || true
-          sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin
 
       - uses: actions/checkout@v4
 

Dockerfile.server

@@ -7,6 +7,9 @@ FROM node:18 AS builder
 RUN apt-get update \
     && apt-get install -y \
     python3 tini \
+    # needed for node-canvas for ARM32 platform.
+    # See also https://github.com/Automattic/node-canvas/wiki/Installation:-Ubuntu-and-other-Debian-based-systems  
+    libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev \
     && rm -rf /var/lib/apt/lists/*
 
 # Enables Yarn
@@ -47,9 +50,9 @@ RUN sed --in-place '/onenote-converter/d' ./packages/lib/package.json
 # Note that `yarn install` ignores `NODE_ENV=production` and will install dev
 # dependencies too, but this is fine because we need them to build the app.
 
-RUN BUILD_SEQUENCIAL=1 yarn install --inline-builds \
-    && yarn cache clean \
-    && rm -rf .yarn/berry
+RUN --mount=type=cache,target=/build/.yarn/cache --mount=type=cache,target=/build/.yarn/berry/cache\
+    BUILD_SEQUENCIAL=1 yarn config set cacheFolder /build/.yarn/cache \
+    && yarn install --inline-builds
 
 # =============================================================================
 # Final stage - we copy only the relevant files from the build stage and start
@@ -81,10 +84,11 @@ CMD ["yarn", "start-prod"]
 ARG BUILD_DATE
 ARG REVISION
 ARG VERSION
+ARG SOURCE
 LABEL org.opencontainers.image.created="$BUILD_DATE" \
       org.opencontainers.image.title="Joplin Server" \
       org.opencontainers.image.description="Docker image for Joplin Server" \
       org.opencontainers.image.url="https://joplinapp.org/" \
       org.opencontainers.image.revision="$REVISION" \
-      org.opencontainers.image.source="https://github.com/laurent22/joplin.git" \
-      org.opencontainers.image.version="${VERSION}"
+      org.opencontainers.image.source="$SOURCE" \
+      org.opencontainers.image.version="$VERSION"

packages/tools/buildServerDocker.test.ts

@@ -6,8 +6,15 @@ describe('buildServerDocker', () => {
 		type TestCase = [string, boolean, string];
 
 		const testCases: TestCase[] = [
-			['server-v1.1.2-beta', true, '1.1.2-beta'],
-			['server-v1.1.2', false, '1.1.2'],
+			['server-v1.2.3-beta', true, '1.2.3-beta'],
+			['server-v1.2.3-beta', false, '1.2.3'],
+			['server-v1.2.3', false, '1.2.3'],
+			['server-v1.2.3-zxc', true, '1.2.3-beta.zxc'],
+			['server-v1.2.3-zxc', false, '1.2.3'],
+			['server-v1.2.3-4-zxc', true, '1.2.3-beta.4.zxc'],
+			['server-v1.2.3-4-zxc', false, '1.2.3'],
+			['server-1.2.3-4-zxc', true, '1.2.3-beta.4.zxc'],
+			['server-1.2.3-4-zxc', false, '1.2.3'],
 		];
 
 		for (const testCase of testCases) {

packages/tools/buildServerDocker.ts

@@ -7,12 +7,66 @@ interface Argv {
 	pushImages?: boolean;
 	repository?: string;
 	tagName?: string;
+	platform?: string;
+	source?: string;
+	addLatestTag?: boolean;
+}
+
+function parseArgv(): Argv {
+	return require('yargs')
+		.scriptName('yarn buildServerDocker')
+		.usage('$0 --repository OWNER/IMAGE [args]')
+		.option('r', {
+			alias: 'repository',
+			describe: 'Target image repository. Usually in format `OWNER/NAME`',
+			demandOption: true,
+			type: 'string',
+		})
+		.option('t', {
+			alias: 'tagName',
+			describe: 'Base image tag. Usually should be in format `server-v1.2.3` or `server-v1.2.3-beta`. The latest `server-v*` git tag will be used by default.',
+			type: 'string',
+		})
+		.option('l', {
+			alias: 'addLatestTag',
+			describe: 'Add `latest` tag even for pre-release images.',
+			type: 'boolean',
+			default: false,
+		})
+		.option('platform', {
+			describe: 'Comma separated list of target image platforms. E.g. `linux/amd64` or `linux/amd64,linux/arm64`',
+			type: 'string',
+			default: 'linux/amd64',
+		})
+		.option('source', {
+			describe: 'Source Git repository for the images.',
+			type: 'string',
+			default: 'https://github.com/laurent22/joplin.git',
+		})
+		.option('p', {
+			alias: 'pushImages',
+			describe: 'Publish images to target repository.',
+			type: 'boolean',
+			default: false,
+		})
+		.option('dryRun', {
+			alias: 'dryRun',
+			describe: 'Do not call docker, just show command instead.',
+			type: 'boolean',
+			default: false,
+		})
+		.help()
+		.argv as Argv;
 }
 
 export function getVersionFromTag(tagName: string, isPreRelease: boolean): string {
 	const s = tagName.split('-');
-	const suffix = isPreRelease ? '-beta' : '';
-	return s[1].substr(1) + suffix;
+	const mainVersion = s[1].replace(/^(v)/, '');
+	const metaComponents = s.slice(2).filter(item => item !== 'beta');
+
+	// Append `git describe` components for pre release images. Mostly for case without `tagName` arg
+	const suffix = isPreRelease ? `-beta${metaComponents.length > 0 ? `.${metaComponents.join('.')}` : ''}` : '';
+	return mainVersion + suffix;
 }
 
 export function getIsPreRelease(_tagName: string): boolean {
@@ -23,14 +77,17 @@ export function getIsPreRelease(_tagName: string): boolean {
 }
 
 async function main() {
-	const argv = require('yargs').argv as Argv;
-	if (!argv.tagName) throw new Error('--tag-name not provided');
-	if (!argv.repository) throw new Error('--repository not provided');
+	const argv = parseArgv();
+	if (!argv.tagName) console.info('No `--tag-name` was specified. A latest git tag will be used instead.');
 
-	const dryRun = !!argv.dryRun;
-	const pushImages = !!argv.pushImages;
+	const dryRun = argv.dryRun;
+	const addLatestTag = argv.addLatestTag;
+	const pushImages = argv.pushImages;
 	const repository = argv.repository;
-	const tagName = argv.tagName;
+	const tagName = argv.tagName || `server-${await execCommand('git describe --tags --match v*', { showStdout: false })}`;
+	const platform = argv.platform;
+	const source = argv.source;
+
 	const isPreRelease = getIsPreRelease(tagName);
 	const imageVersion = getVersionFromTag(tagName, isPreRelease);
 	const buildDate = moment(new Date().getTime()).format('YYYY-MM-DDTHH:mm:ssZ');
@@ -40,35 +97,57 @@ async function main() {
 	} catch (error) {
 		console.info('Could not get git commit: metadata revision field will be empty');
 	}
-	const buildArgs = `--build-arg BUILD_DATE="${buildDate}" --build-arg REVISION="${revision}" --build-arg VERSION="${imageVersion}"`;
+
+	const buildArgs = [];
+	buildArgs.push(`BUILD_DATE="${buildDate}"`);
+	buildArgs.push(`REVISION="${revision}"`);
+	buildArgs.push(`VERSION="${imageVersion}"`);
+	buildArgs.push(`SOURCE="${source}"`);
+
 	const dockerTags: string[] = [];
-	const versionPart = imageVersion.split('.');
-	dockerTags.push(isPreRelease ? 'beta' : 'latest');
-	dockerTags.push(versionPart[0] + (isPreRelease ? '-beta' : ''));
-	dockerTags.push(`${versionPart[0]}.${versionPart[1]}${isPreRelease ? '-beta' : ''}`);
-	dockerTags.push(imageVersion);
+	const versionParts = imageVersion.split('.');
+	const patchVersionPart = versionParts[2].split('-')[0];
+	dockerTags.push(isPreRelease ? 'latest-beta' : 'latest');
+	dockerTags.push(versionParts[0] + (isPreRelease ? '-beta' : ''));
+	dockerTags.push(`${versionParts[0]}.${versionParts[1]}${isPreRelease ? '-beta' : ''}`);
+	dockerTags.push(`${versionParts[0]}.${versionParts[1]}.${patchVersionPart}${isPreRelease ? '-beta' : ''}`);
+	if (dockerTags.indexOf(imageVersion) < 0) {
+		dockerTags.push(imageVersion);
+	}
+	if (addLatestTag && dockerTags.indexOf('latest') < 0) {
+		dockerTags.push('latest');
+	}
+
 
 	process.chdir(rootDir);
 	console.info(`Running from: ${process.cwd()}`);
 
+	console.info('repository:', repository);
 	console.info('tagName:', tagName);
+	console.info('platform:', platform);
 	console.info('pushImages:', pushImages);
 	console.info('imageVersion:', imageVersion);
 	console.info('isPreRelease:', isPreRelease);
 	console.info('Docker tags:', dockerTags.join(', '));
 
-	const dockerCommand = `docker build --progress=plain -t "${repository}:${imageVersion}" ${buildArgs} -f Dockerfile.server .`;
+	const cliArgs = ['--progress=plain'];
+	cliArgs.push(`--platform ${platform}`);
+	cliArgs.push(...dockerTags.map(tag => `--tag "${repository}:${tag}"`));
+	cliArgs.push(...buildArgs.map(arg => `--build-arg ${arg}`));
+	if (pushImages) {
+		cliArgs.push('--push');
+	}
+	cliArgs.push('-f Dockerfile.server');
+	cliArgs.push('.');
+
+	const dockerCommand = `docker buildx build ${cliArgs.join(' ')}`;
+
+	console.info('exec:', dockerCommand);
 	if (dryRun) {
-		console.info(dockerCommand);
 		return;
 	}
 
 	await execCommand(dockerCommand);
-
-	for (const tag of dockerTags) {
-		await execCommand(`docker tag "${repository}:${imageVersion}" "${repository}:${tag}"`);
-		if (pushImages) await execCommand(`docker push ${repository}:${tag}`);
-	}
 }
 
 if (require.main === module) {

packages/tools/cspell/dictionary4.txt

@@ -161,6 +161,7 @@ unwatcher
 pedr
 Slotozilla
 keyshortcuts
+buildx
 atag
 HMAC
 Siri