Merge pull request #551 from omacom/ci/daily-builder-images

Publish tested daily package builder images

.github/workflows/builder-images.yml

@@ -0,0 +1,118 @@
+name: Refresh builder images
+
+on:
+  schedule:
+    - cron: '23 4 * * *'
+  push:
+    branches: [master]
+    paths:
+      - build/**
+      - bin/builder-image
+      - helpers/paths.sh
+      - helpers/docker-helpers.sh
+      - tests/build-isolation.sh
+      - .github/workflows/builder-images.yml
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - build/**
+      - bin/builder-image
+      - helpers/paths.sh
+      - helpers/docker-helpers.sh
+      - tests/build-isolation.sh
+      - .github/workflows/builder-images.yml
+
+# Complete each refresh before another can replace its tested image tags.
+concurrency:
+  group: builder-images-${{ github.event.pull_request.number || 'master' }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  # Exercise proposed image changes on native runners with a read-only token.
+  # Publishing is a separate master-only job with its own write permission.
+  validate:
+    if: github.event_name == 'pull_request'
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: x86_64
+            runner: ubuntu-24.04
+          - arch: aarch64
+            runner: ubuntu-24.04-arm
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 60
+    env:
+      CONTAINER_ENGINE: docker
+      CANDIDATE_IMAGE: omarchy-builder-candidate:${{ matrix.arch }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Build a fresh environment
+        run: bin/builder-image build --arch "${{ matrix.arch }}" --mirror edge --tag "$CANDIDATE_IMAGE" --fresh
+      - name: Test isolated package builds
+        env:
+          TEST_BUILDER_IMAGE: ${{ env.CANDIDATE_IMAGE }}
+        run: tests/build-isolation.sh
+
+  refresh:
+    if: github.repository == 'omacom/omarchy-pkgs' && github.ref == 'refs/heads/master'
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: x86_64
+            runner: ubuntu-24.04
+          - arch: aarch64
+            runner: ubuntu-24.04-arm
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 60
+    permissions:
+      contents: read
+      packages: write
+    env:
+      CONTAINER_ENGINE: docker
+      REGISTRY_IMAGE: ghcr.io/omacom/omarchy-pkg-builder
+      CANDIDATE_IMAGE: omarchy-builder-candidate:${{ matrix.arch }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Build a fresh environment
+        run: bin/builder-image build --arch "${{ matrix.arch }}" --mirror edge --tag "$CANDIDATE_IMAGE" --fresh
+      - name: Test isolated package builds
+        env:
+          TEST_BUILDER_IMAGE: ${{ env.CANDIDATE_IMAGE }}
+        run: tests/build-isolation.sh
+      - name: Publish tested image
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_ACTOR: ${{ github.actor }}
+          DOCKER_CONFIG: ${{ runner.temp }}/builder-registry-auth
+        run: |
+          set -euo pipefail
+          mkdir -p "$DOCKER_CONFIG"
+          trap 'rm -rf "$DOCKER_CONFIG"' EXIT
+          printf '%s' "$GH_TOKEN" | docker login ghcr.io --username "$GH_ACTOR" --password-stdin
+          key=$(bin/builder-image key --arch "${{ matrix.arch }}" --mirror edge)
+          version="$REGISTRY_IMAGE:$key-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
+          docker tag "$CANDIDATE_IMAGE" "$version"
+          docker push "$version"
+          # GHCR creates new packages private. Do not advertise an image to
+          # fork PRs until it is public. This is a one-time package setting.
+          anonymous_config=$(mktemp -d "$RUNNER_TEMP/builder-anonymous.XXXXXX")
+          if ! DOCKER_CONFIG="$anonymous_config" docker manifest inspect "$version" >/dev/null; then
+            rm -rf "$anonymous_config"
+            echo "::error::Make the omacom/omarchy-pkg-builder GHCR package public, then rerun this job. The previous matching image remains selected."
+            exit 1
+          fi
+          rm -rf "$anonymous_config"
+          docker tag "$CANDIDATE_IMAGE" "$REGISTRY_IMAGE:$key"
+          docker push "$REGISTRY_IMAGE:$key"
+          digest=$(docker image inspect "$version" --format '{{index .RepoDigests 0}}')
+          printf '### Builder image (%s)\n\nInput key: `%s`\n\nImage: `%s`\n' \
+            "${{ matrix.arch }}" "$key" "$digest" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/test.yml

@@ -35,6 +35,9 @@ jobs:
       - name: Test PR workflow approval
         run: node --test tests/pr-workflow-approval.cjs
 
+      - name: Test builder images
+        run: node --test tests/builder-image.cjs
+
       # An Arch container for vercmp: version ordering has to be decided by
       # the same comparator pacman uses on users' machines.
       - name: Run self-tests

README.md

@@ -825,6 +825,45 @@ using real containers and pacman transactions. It uses the prepared builder
 image, or an image named by `TEST_BUILDER_IMAGE`; CI builds the small fixture
 image in `tests/build-isolation.Dockerfile`.
 
+### Daily builder images
+
+`Refresh builder images` builds fresh `edge` environments daily at 04:23 UTC,
+when their inputs change on `master`, and on manual dispatch. x86_64 and
+aarch64 build on native GitHub-hosted runners, without occupying the DO
+package-builder pool. Each candidate must pass `tests/build-isolation.sh`,
+including real package builds, before publication to
+`ghcr.io/omacom/omarchy-pkg-builder`. Only `master` in this repository can
+publish; PR workflows cannot replace the shared images.
+PRs that change image inputs also build and test both candidates on native
+runners, with a read-only token and no registry publication.
+
+The compatibility tag contains the architecture, mirror, and a hash of the
+entire `build/` context, including executable bits and symlink targets but
+excluding checkout timestamps and ownership. This deliberately invalidates
+images when mounted build scripts change too. `v1` identifies the image build
+contract; change it if the invocation or compatibility rules change. Each
+successful refresh also gets a run-specific tag for diagnosis and rollback.
+A failed build, isolation test, or push leaves the previous compatible image
+selected. Scheduled builds use `--pull --no-cache` so unchanged Dockerfiles
+still pick up fresh Arch packages.
+
+To build and test a candidate locally:
+
+```bash
+bin/builder-image key --arch x86_64 --mirror edge
+bin/builder-image build --arch x86_64 --mirror edge --tag builder-candidate:test --fresh
+CONTAINER_ENGINE=docker TEST_BUILDER_IMAGE=builder-candidate:test tests/build-isolation.sh
+```
+
+The workflow uses its repository `GITHUB_TOKEN` with `packages: write`; no
+registry PAT is needed. **First publication needs one package setting:** GHCR
+creates the package private. In the `omacom/omarchy-pkg-builder` package
+settings, change visibility to **Public**, then rerun the failed refresh job.
+The workflow checks anonymous registry access before advancing the compatible
+tag, so fork PRs will not be directed to an image they cannot pull. Subsequent
+refreshes preserve that package visibility. This change only produces images;
+package jobs keep their existing behavior until image consumption is enabled.
+
 ## Version Management
 
 Packages are only rebuilt if:

bin/builder-image

@@ -0,0 +1,66 @@
+#!/bin/bash
+# Build a reusable package environment from this checkout's own inputs.
+set -euo pipefail
+
+BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..")
+source "$BUILD_ROOT/helpers/paths.sh"
+
+usage() {
+  echo "Usage: bin/builder-image {key|build} [--arch x86_64|aarch64] [--mirror edge|rc|stable] [--tag IMAGE] [--fresh]"
+}
+
+command=${1:-}
+[[ $# -eq 0 ]] || shift
+tag=""
+fresh=false
+while (( $# )); do
+  case "$1" in
+    --arch) ARCH=${2:?Missing architecture}; shift 2 ;;
+    --mirror) MIRROR=${2:?Missing mirror}; shift 2 ;;
+    --tag) tag=${2:?Missing image tag}; shift 2 ;;
+    --fresh) fresh=true; shift ;;
+    *) usage >&2; exit 1 ;;
+  esac
+done
+require_valid_arch "$ARCH"
+validate_mirror "$MIRROR" || { echo "Invalid mirror: $MIRROR" >&2; exit 1; }
+case "$command" in key|build) ;; *) usage >&2; exit 1 ;; esac
+if [[ $command == key && ( -n $tag || $fresh == true ) ]]; then
+  usage >&2
+  exit 1
+fi
+
+# Include the whole build context, conservatively including mounted build
+# scripts too. Normalize timestamps and ownership so fresh checkouts agree;
+# retain file contents, names, executable bits and symlink targets. Bump v1
+# if the image build invocation or this compatibility contract changes.
+hash=$(tar --sort=name --mtime=@0 --owner=0 --group=0 --numeric-owner \
+  --format=gnu -cf - -C "$BUILD_DIR" . | sha256sum | cut -d' ' -f1)
+key="v1-$ARCH-$MIRROR-$hash"
+if [[ $command == key ]]; then
+  echo "$key"
+  exit 0
+fi
+
+source "$BUILD_ROOT/helpers/message-helpers.sh"
+source "$BUILD_ROOT/helpers/docker-helpers.sh"
+check_engine
+platform=$(get_platform_arg "$ARCH")
+tag=${tag:-omarchy-pkg-builder:latest-$ARCH-$MIRROR}
+revision=$(git -C "$BUILD_ROOT" rev-parse HEAD 2>/dev/null || echo unknown)
+args=("$platform" --build-arg "MIRROR=$MIRROR"
+  --label "org.omarchy.builder.key=$key"
+  --label "org.opencontainers.image.source=https://github.com/omacom/omarchy-pkgs"
+  --label "org.opencontainers.image.revision=$revision"
+  --label "org.opencontainers.image.created=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+  --tag "$tag" --file "$BUILD_DIR/Dockerfile")
+if [[ $fresh == true ]]; then
+  # A daily build must refresh Arch even when its Dockerfile has not changed.
+  args+=(--no-cache)
+  if [[ $CONTAINER_ENGINE == docker ]]; then args+=(--pull); else args+=(--pull=always); fi
+fi
+if [[ $CONTAINER_ENGINE == docker ]]; then
+  docker buildx build --load "${args[@]}" "$BUILD_DIR"
+else
+  podman build "${args[@]}" "$BUILD_DIR"
+fi

tests/builder-image.cjs

@@ -0,0 +1,130 @@
+const assert = require('node:assert/strict');
+const { spawnSync } = require('node:child_process');
+const { chmodSync, cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, utimesSync, writeFileSync } = require('node:fs');
+const { tmpdir } = require('node:os');
+const { join } = require('node:path');
+const { test } = require('node:test');
+
+const root = join(__dirname, '..');
+const workflow = readFileSync(join(root, '.github/workflows/builder-images.yml'), 'utf8');
+
+function fixture(t) {
+  const directory = mkdtempSync(join(tmpdir(), 'builder-image-test-'));
+  t.after(() => rmSync(directory, { recursive: true, force: true }));
+  mkdirSync(join(directory, 'bin'));
+  mkdirSync(join(directory, 'build'));
+  cpSync(join(root, 'helpers'), join(directory, 'helpers'), { recursive: true });
+  cpSync(join(root, 'bin/builder-image'), join(directory, 'bin/builder-image'));
+  writeFileSync(join(directory, 'build/Dockerfile'), 'FROM scratch\nCOPY input /input\n');
+  writeFileSync(join(directory, 'build/input'), 'original input\n');
+  const engine = join(directory, 'engine');
+  mkdirSync(engine);
+  const log = join(directory, 'engine.jsonl');
+  writeFileSync(join(engine, 'docker'), `#!/usr/bin/env node
+const fs = require('node:fs');
+const args = process.argv.slice(2);
+fs.appendFileSync(process.env.ENGINE_LOG, JSON.stringify(args) + '\\n');
+if (args[0] === 'manifest' && process.env.PRIVATE_IMAGE === '1') process.exit(1);
+if (args[0] === 'push' && process.env.PUSH_FAIL === '1') process.exit(1);
+if (args[0] === 'image' && args[1] === 'inspect') console.log('ghcr.io/omacom/omarchy-pkg-builder@sha256:' + 'a'.repeat(64));
+`);
+  chmodSync(join(engine, 'docker'), 0o755);
+  const env = {
+    ...process.env, PATH: `${engine}:${process.env.PATH}`, CONTAINER_ENGINE: 'docker',
+    ENGINE_LOG: log, ARCH: 'x86_64', MIRROR: 'edge',
+  };
+  const run = (args, extraEnv = {}) => spawnSync(join(directory, 'bin/builder-image'), args, {
+    cwd: directory, env: { ...env, ...extraEnv }, encoding: 'utf8',
+  });
+  const key = (...args) => {
+    const result = run(['key', ...args]);
+    assert.equal(result.status, 0, result.stderr);
+    return result.stdout.trim();
+  };
+  const calls = () => readFileSync(log, 'utf8').trim().split('\n').filter(Boolean).map(JSON.parse);
+  return { directory, env, run, key, calls };
+}
+
+test('image keys are stable across checkout location and timestamp changes', t => {
+  const a = fixture(t);
+  const b = fixture(t);
+  const key = a.key();
+  assert.match(key, /^v1-x86_64-edge-[a-f0-9]{64}$/);
+  utimesSync(join(b.directory, 'build/input'), new Date(0), new Date(0));
+  assert.equal(b.key(), key);
+});
+
+test('image keys separate architectures, mirrors, content, modes and symlink targets', t => {
+  const f = fixture(t);
+  const keys = new Set([f.key(), f.key('--arch', 'aarch64'), f.key('--mirror', 'rc'), f.key('--mirror', 'stable')]);
+  writeFileSync(join(f.directory, 'build/input'), 'new input\n');
+  keys.add(f.key());
+  chmodSync(join(f.directory, 'build/input'), 0o755);
+  keys.add(f.key());
+  symlinkSync('input', join(f.directory, 'build/link'));
+  keys.add(f.key());
+  rmSync(join(f.directory, 'build/link'));
+  symlinkSync('Dockerfile', join(f.directory, 'build/link'));
+  keys.add(f.key());
+  assert.equal(keys.size, 8);
+  mkdirSync(join(f.directory, 'pkgbuilds/example'), { recursive: true });
+  const key = f.key();
+  writeFileSync(join(f.directory, 'pkgbuilds/example/PKGBUILD'), 'pkgver=2\n');
+  assert.equal(f.key(), key, 'package changes must not invalidate the build environment');
+});
+
+test('fresh builds refresh package layers and record their compatibility key', t => {
+  const f = fixture(t);
+  const key = f.key('--arch', 'aarch64');
+  const result = f.run(['build', '--arch', 'aarch64', '--tag', 'candidate:test', '--fresh']);
+  assert.equal(result.status, 0, result.stderr);
+  const build = f.calls().find(args => args[0] === 'buildx');
+  assert.ok(build.includes('--no-cache'));
+  assert.ok(build.includes('--pull'));
+  assert.ok(build.includes('--load'));
+  assert.ok(build.includes('--platform=linux/arm64'));
+  assert.ok(build.includes(`org.omarchy.builder.key=${key}`));
+  assert.ok(build.includes('candidate:test'));
+});
+
+test('invalid targets fail before starting an image build', t => {
+  const f = fixture(t);
+  for (const args of [['key', '--arch', 'invalid'], ['build', '--mirror', 'invalid'], ['key', '--fresh']]) {
+    assert.notEqual(f.run(args).status, 0);
+  }
+});
+
+function publish(f, extraEnv = {}) {
+  const script = workflow.split('      - name: Publish tested image\n')[1].split('        run: |\n')[1]
+    .replaceAll('${{ matrix.arch }}', 'x86_64');
+  return spawnSync('bash', ['-e', '-o', 'pipefail', '-c', script], {
+    cwd: f.directory, encoding: 'utf8', env: {
+      ...f.env, REGISTRY_IMAGE: 'ghcr.io/omacom/omarchy-pkg-builder', CANDIDATE_IMAGE: 'candidate:test',
+      GH_TOKEN: 'fixture', GH_ACTOR: 'fixture', DOCKER_CONFIG: join(f.directory, 'auth'),
+      RUNNER_TEMP: f.directory, GITHUB_RUN_ID: '123', GITHUB_RUN_ATTEMPT: '1',
+      GITHUB_STEP_SUMMARY: join(f.directory, 'summary'), ...extraEnv,
+    },
+  });
+}
+
+test('a public tested image gets a version tag before the compatible-image tag advances', t => {
+  const f = fixture(t);
+  const key = f.key();
+  const result = publish(f);
+  assert.equal(result.status, 0, result.stderr);
+  const calls = f.calls();
+  assert.deepEqual(calls.filter(args => args[0] === 'push').map(args => args[1]), [
+    `ghcr.io/omacom/omarchy-pkg-builder:${key}-123-1`, `ghcr.io/omacom/omarchy-pkg-builder:${key}`,
+  ]);
+  assert.ok(calls.findIndex(args => args[0] === 'manifest') < calls.findLastIndex(args => args[0] === 'push'));
+});
+
+test('a private image or failed push never replaces the previous compatible-image tag', t => {
+  for (const extraEnv of [{ PRIVATE_IMAGE: '1' }, { PUSH_FAIL: '1' }]) {
+    const f = fixture(t);
+    const key = f.key();
+    const result = publish(f, extraEnv);
+    assert.notEqual(result.status, 0);
+    assert.equal(f.calls().some(args => args[0] === 'push' && args[1] === `ghcr.io/omacom/omarchy-pkg-builder:${key}`), false);
+  }
+});