From aa64ec9a4f63e22ad328336317b5aa28d6586e0b Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Wed, 23 Sep 2026 23:26:34 -0400 Subject: [PATCH] Track Debian QEMU revisions through verified immutable snapshots --- .github/workflows/test.yml | 1 + bin/sync-upstream | 16 ++- .../qemu-user-static/.omarchy/package.json | 3 +- .../qemu-user-static/.omarchy/upstream.py | 120 ++++++++++++++++++ .../qemu-user-static/.omarchy/upstream.sh | 3 + pkgbuilds/qemu-user-static/PKGBUILD | 15 ++- pkgbuilds/qemu-user-static/README.md | 14 +- tests/qemu-upstream.py | 115 +++++++++++++++++ 8 files changed, 272 insertions(+), 15 deletions(-) create mode 100644 pkgbuilds/qemu-user-static/.omarchy/upstream.py create mode 100644 pkgbuilds/qemu-user-static/.omarchy/upstream.sh create mode 100644 tests/qemu-upstream.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 29d18b0..7bcbb22 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,6 +50,7 @@ jobs: pacman -Syu --noconfirm git jq python libarchive python tests/oma-service-removal.py python tests/upstream-watch.py + python tests/qemu-upstream.py ./bin/sync-upstream self-test ./bin/sync-rebuilds --self-test ./bin/omarchy-pkgs self-test diff --git a/bin/sync-upstream b/bin/sync-upstream index 2f84506..360a1b2 100755 --- a/bin/sync-upstream +++ b/bin/sync-upstream @@ -31,7 +31,10 @@ pkgbuilds//.omarchy/upstream.sh, a hook that reports JSON on stdout: } Architecture keys become sha256sums_ in the PKGBUILD; the key "any" means -the unsuffixed sha256sums array. An empty object ({}) reports no update. +the unsuffixed sha256sums array. An empty object ({}) reports no update. Optional "variables" maps existing +underscore-prefixed release scalars to values containing only letters, digits, +periods, underscores, plus, colon and hyphen. They are verified and written in +the same atomic replacement as pkgver and checksums. When the reported version is newer than the checked-in one, pkgver and the listed checksum arrays are rewritten and pkgrel is reset to 1. @@ -209,6 +212,10 @@ validate_release() { and (.sha256sums | to_entries | all( .value | type == "array" and length > 0 and all(test("\\A[0-9a-f]{64}\\z")) )) + and (if has("variables") then (.variables | type == "object" and (to_entries | all( + (.key | test("\\A_[a-z][a-z0-9_]*\\z")) and + (.value | type == "string" and test("\\A[A-Za-z0-9._+:-]+\\z")) + ))) else true end) and (if has("published_at") then (.published_at | type == "string" and length > 0) else true end) ' <<<"$release" >/dev/null } @@ -312,6 +319,13 @@ apply_release() { set_pkgbuild_scalar "$scratch" pkgver "$pkgver" || exit 1 set_pkgbuild_scalar "$scratch" pkgrel 1 || exit 1 + local field value + while IFS=$'\t' read -r field value; do + [[ -n "$field" ]] || continue + set_pkgbuild_scalar "$scratch" "$field" "$value" || exit 1 + [[ $(bash -c 'source "$1"; printf "%s" "${!2}"' _ "$scratch" "$field") == "$value" ]] || exit 1 + done < <(jq -r '(.variables // {}) | to_entries[] | [.key, .value] | @tsv' <<<"$release") + verify_pkgbuild "$scratch" "$release" "$pkgver" "${arrays[@]}" || exit 1 ); then rm -f "$scratch" diff --git a/pkgbuilds/qemu-user-static/.omarchy/package.json b/pkgbuilds/qemu-user-static/.omarchy/package.json index 283d159..2a9719d 100644 --- a/pkgbuilds/qemu-user-static/.omarchy/package.json +++ b/pkgbuilds/qemu-user-static/.omarchy/package.json @@ -1,4 +1,3 @@ { - "source": "local", - "sync": false + "source": "local" } diff --git a/pkgbuilds/qemu-user-static/.omarchy/upstream.py b/pkgbuilds/qemu-user-static/.omarchy/upstream.py new file mode 100644 index 0000000..341d9c1 --- /dev/null +++ b/pkgbuilds/qemu-user-static/.omarchy/upstream.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Track Debian 13 ARM64 QEMU revisions, verifying an immutable snapshot first.""" +import functools +import hashlib +import json +import lzma +from pathlib import Path +import re +import subprocess +import sys +import urllib.parse +import urllib.request + +FEEDS = [ + 'https://deb.debian.org/debian/dists/trixie/main/binary-arm64/Packages.xz', + 'https://deb.debian.org/debian/dists/trixie-updates/main/binary-arm64/Packages.xz', + 'https://security.debian.org/debian-security/dists/trixie-security/main/binary-arm64/Packages.xz', +] + +def fetch(url): + with urllib.request.urlopen(url, timeout=120) as response: + if not response.url.startswith('https://'): + raise ValueError('Insecure redirect') + return response.read() + +def parts(version): + match = re.fullmatch(r'(?:(\d+):)?([0-9][A-Za-z0-9.+~]*?)-([A-Za-z0-9.+~]+)', version) + if not match: + raise ValueError('Unsupported Debian version: ' + version) + return int(match[1] or '0'), match[2], match[3] + +def segment_cmp(a, b): + # Debian policy: tilde precedes everything, then end/digits, letters, + # then other characters; digit runs are compared numerically. + def order(c): + if c == '~': return -1 + if not c or c.isdigit(): return 0 + return ord(c) if c.isalpha() else ord(c) + 256 + while a or b: + while (a and not a[0].isdigit()) or (b and not b[0].isdigit()): + x, y = order(a[:1]), order(b[:1]) + if x != y: return (x > y) - (x < y) + a, b = a[1:], b[1:] + x = re.match(r'\d*', a)[0] + y = re.match(r'\d*', b)[0] + nx, ny = int(x or '0'), int(y or '0') + if nx != ny: return (nx > ny) - (nx < ny) + a, b = a[len(x):], b[len(y):] + return 0 + +def compare(a, b): + ea, ua, ra = parts(a) + eb, ub, rb = parts(b) + return (ea > eb) - (ea < eb) or segment_cmp(ua, ub) or segment_cmp(ra, rb) + +def package_version(version): + epoch, upstream, revision = parts(version) + # Stable releases only. Fail visibly on prerelease/repack conventions that + # need a reviewed Arch ordering rather than silently misordering them. + if '~' in version: + raise ValueError('Debian prerelease needs a reviewed Arch version mapping') + return f'{epoch}.{upstream}.{revision}' + +def records(data): + found = [] + for stanza in re.split(r'\n\s*\n', lzma.decompress(data).decode()): + fields = dict(re.findall(r'^([A-Za-z0-9-]+): (.*)$', stanza, re.M)) + if fields.get('Package') != 'qemu-user': continue + if fields.get('Architecture') != 'arm64': raise ValueError('Wrong architecture') + parts(fields['Version']) + if not re.fullmatch(r'[0-9a-f]{64}', fields.get('SHA256', '')): + raise ValueError('Missing/malformed SHA256') + if not re.fullmatch(r'[1-9][0-9]*', fields.get('Size', '')): + raise ValueError('Missing/malformed package size') + found.append(fields) + return found + +def discover(current, current_pkgver, current_hash, get=fetch): + candidates = [row for url in FEEDS for row in records(get(url))] + if not candidates: raise ValueError('No ARM64 qemu-user package in Debian feeds') + selected = max(candidates, key=functools.cmp_to_key(lambda a,b: compare(a['Version'], b['Version']))) + version, checksum = selected['Version'], selected['SHA256'] + if any(row['SHA256'] != checksum for row in candidates if row['Version'] == version): + raise ValueError('Debian feeds disagree on the selected checksum') + ordering = compare(version, current) + if ordering < 0: raise ValueError('Debian feeds are older than the pinned recipe') + if ordering == 0: + if checksum != current_hash: raise ValueError('Checksum changed for the pinned Debian revision') + return {} + pkgver = package_version(version) + if int(subprocess.check_output(['vercmp', pkgver, current_pkgver], text=True)) <= 0: + raise ValueError('New Debian revision does not advance Arch version; review mapping') + api = 'https://snapshot.debian.org/mr/binary/qemu-user/' + urllib.parse.quote(version, safe='') + '/binfiles' + document = json.loads(get(api)) + if document.get('binary') != 'qemu-user' or document.get('binary_version') != version: + raise ValueError('Snapshot revision differs') + hashes = {row['hash'] for row in document['result'] if row['architecture'] == 'arm64'} + if len(hashes) != 1 or not re.fullmatch(r'[0-9a-f]{40}', next(iter(hashes), '')): + raise ValueError('Missing/ambiguous ARM64 snapshot') + snapshot = hashes.pop() + archive = get('https://snapshot.debian.org/file/' + snapshot) + if len(archive) != int(selected['Size']) or hashlib.sha256(archive).hexdigest() != checksum or hashlib.sha1(archive).hexdigest() != snapshot: + raise ValueError('Snapshot bytes differ from Debian package metadata') + return {'pkgver': pkgver, 'variables': {'_debver': version, '_snapshot': snapshot}, + 'sha256sums': {'aarch64': [checksum]}} + +def main(): + recipe = Path('PKGBUILD').read_text() + def scalar(name): + match = re.search(r'^' + name + r'=[\"\']?([^\"\'\n]+)', recipe, re.M) + if not match: raise ValueError('Missing recipe scalar: ' + name) + return match[1] + checksum = re.search(r"^sha256sums_aarch64=\('([a-f0-9]{64})'\)", recipe, re.M) + if not checksum: raise ValueError('Missing current ARM checksum') + print(json.dumps(discover(scalar('_debver'), scalar('pkgver'), checksum[1]))) + +if __name__ == '__main__': + try: main() + except Exception as error: + sys.exit('QEMU Debian update failed: ' + str(error)) diff --git a/pkgbuilds/qemu-user-static/.omarchy/upstream.sh b/pkgbuilds/qemu-user-static/.omarchy/upstream.sh new file mode 100644 index 0000000..829dbc9 --- /dev/null +++ b/pkgbuilds/qemu-user-static/.omarchy/upstream.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +exec python3 .omarchy/upstream.py diff --git a/pkgbuilds/qemu-user-static/PKGBUILD b/pkgbuilds/qemu-user-static/PKGBUILD index 8a21f11..22b64cc 100644 --- a/pkgbuilds/qemu-user-static/PKGBUILD +++ b/pkgbuilds/qemu-user-static/PKGBUILD @@ -5,8 +5,12 @@ pkgbase=qemu-user-static pkgname=('qemu-user-static' 'qemu-user-static-binfmt') -pkgver=10.0.11 -_debver="1:10.0.11+ds-0+deb13u1+b1" +# One-time epoch moves from upstream-only to full Debian revision ordering. +# pkgver records Debian epoch.upstream.revision; updates also verify vercmp. +epoch=1 +pkgver=1.10.0.13+ds.0+deb13u1 +_debver=1:10.0.13+ds-0+deb13u1 +_snapshot=02b553a4532192bc999d15408d1e56d7b254906f pkgrel=1 pkgdesc="A generic and open source machine emulator and virtualizer - static user-mode emulation" arch=('aarch64') @@ -14,10 +18,11 @@ url="https://www.qemu.org" license=('GPL-2.0-only' 'LGPL-2.1-only') source=("qemu-binfmt-conf.sh") # Debian removes superseded pool files. Pin the official archive snapshot so -# the same checksum-verified binary remains rebuildable. -source_aarch64=("qemu-user_${pkgver}_arm64.deb::https://snapshot.debian.org/archive/debian/20260713T202617Z/pool/main/q/qemu/qemu-user_${_debver#*:}_arm64.deb") +# the same checksum-verified binary remains rebuildable. _snapshot is the +# content-addressed snapshot SHA1; SHA256 below remains the integrity check. +source_aarch64=("qemu-user_${pkgver}_arm64.deb::https://snapshot.debian.org/file/$_snapshot") sha256sums=('5e12ef484b8364e1ba7bae4acda837d2e6320ffbeb7fab341034124553cdb137') -sha256sums_aarch64=('6c9483063bf60f37fe181ead911251deabe40c893ea5829bc34b0b7b88913b6a') +sha256sums_aarch64=('c73711af02b97cd5e2667e735d9c06890c57165517110ca4e646c95a7083158d') noextract=("qemu-user_${pkgver}_arm64.deb") prepare() { diff --git a/pkgbuilds/qemu-user-static/README.md b/pkgbuilds/qemu-user-static/README.md index daeb0ea..d30002c 100644 --- a/pkgbuilds/qemu-user-static/README.md +++ b/pkgbuilds/qemu-user-static/README.md @@ -1,13 +1,13 @@ -# ARM static QEMU candidate +# ARM static QEMU -This split recipe provides `qemu-user-static` and `qemu-user-static-binfmt` on aarch64, where Arch Linux ARM does not supply the static emulator packages requested by Omarchy's default package set. It repackages Debian's checksum-pinned ARM64 binary; it does not compile QEMU or replace upstream's build-worker emulation setup. +This aarch64-only split recipe provides qemu-user-static and qemu-user-static-binfmt using Debian’s static ARM64 binaries. It preserves Marcelo’s recipe, imported through omarchy-mac/omarchy-pkgs-aarch64#61, including licensing and binfmt behavior. It does not change x86 packages or the build workers. -Imported from Marcelo's `maralcbr/omarchy-pkgs` revision `83973903b7deb9b56ce75f02b432fba0561d6293`, through the validated candidate recipe in `omarchy-mac/omarchy-pkgs-aarch64` (#61). Debian copyright and upstream licenses are retained. The immutable Debian snapshot keeps the pinned binary available when the rolling pool removes old revisions. +## Updates -## Draft readiness gates +The package-local upstream hook checks Debian 13 (trixie), trixie-updates, and trixie-security ARM64 indexes. It compares full Debian versions, including epochs, security revisions and binary rebuilds, and resolves the selected binary through snapshot.debian.org’s API. The immutable content-addressed archive must match the index’s size and SHA256 as well as the snapshot SHA1 before an update is returned. -Automatic sync is deliberately disabled in this initial draft. This does **not** detect new releases or security updates. Do not mark this PR ready until a package-local, fixture-tested update integration handles Debian's full version (epoch, upstream version, Debian security revision and architecture rebuild), immutable archive location and SHA256 together. A new Debian revision at the same QEMU version must produce a forward package version; repeated checks must be idempotent. Missing or malformed metadata, unavailable snapshots and checksum inconsistencies must leave the recipe unchanged and fail visibly. +A one-time Arch epoch of 1 moves away from the previous upstream-only version. pkgver encodes Debian epoch.upstream.revision. The hook requires both Debian ordering and pacman vercmp to advance; unfamiliar prerelease conventions or ordering discrepancies fail for manual review. Debian distribution upgrades are explicit recipe changes, not automatic jumps to testing/unstable. -The existing simple custom-hook contract updates pkgver and SHA256 arrays but cannot update `_debver` and the snapshot timestamp together. Do not add a misleading upstream-version-only watch. Review the smallest compatible integration with upstream before removing the hold. Once enabled, updates should join upstream's existing six-hour update PR workflow; GitHub notification settings determine who sees those PRs. Until then, check Debian's package and security updates manually. +The existing six-hour upstream update PR workflow discovers the hook. Its pkgver, _debver, _snapshot and ARM checksum are applied atomically through the normal sync interface. Repeated checks are idempotent. Feed failures, malformed metadata, conflicting checksums and unavailable/corrupt snapshots fail visibly before recipe mutation. No update installs packages or registers binfmt rules. -Before readiness, build both outputs using upstream's ARM builder and test installation in a disposable ARM VM. Verify AArch64 static ELF executables, exact interpreter paths and ownership, the absence of native AArch64 binfmt rules, execution of a known foreign-architecture program both directly and through binfmt, and clean package removal. Never register binfmt rules on the developer host or assume a container isolates that kernel state. Recipe generation only writes rules into the package staging directory. +Offline fixtures cover version/epoch/security/binNMU updates, security-feed selection, unchanged versions, metadata and snapshot failures, atomic application, and hostile/missing scalar rejection. Existing GUI-independent VM qualification and packaging evidence are recorded in the PR; new binaries still receive build and runtime checks before publication. diff --git a/tests/qemu-upstream.py b/tests/qemu-upstream.py new file mode 100644 index 0000000..30a54a2 --- /dev/null +++ b/tests/qemu-upstream.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +import hashlib +import importlib.util +import json +import lzma +import re +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +PACKAGE = ROOT / 'pkgbuilds/qemu-user-static' +spec = importlib.util.spec_from_file_location('qemu', PACKAGE / '.omarchy/upstream.py') +q = importlib.util.module_from_spec(spec) +spec.loader.exec_module(q) +CURRENT = '1:10.0.11+ds-0+deb13u1+b1' + +class Updates(unittest.TestCase): + def fixtures(self, version): + blob = b'fixture package bytes' + sha = hashlib.sha256(blob).hexdigest() + snap = hashlib.sha1(blob).hexdigest() + row = f'Package: qemu-user\nArchitecture: arm64\nVersion: {version}\nSHA256: {sha}\nSize: {len(blob)}\n' + feeds = {url: lzma.compress(row.encode()) for url in q.FEEDS} + def get(url): + if url in feeds: return feeds[url] + if '/mr/' in url: + return json.dumps({'binary':'qemu-user', 'binary_version':version, + 'result':[{'architecture':'arm64','hash':snap}]}).encode() + return blob + return get, feeds, sha + + def test_versions(self): + for old, new in [(CURRENT,'1:10.0.11+ds-0+deb13u1+b2'), + (CURRENT,'1:10.0.11+ds-0+deb13u2'), + (CURRENT,'1:10.0.12+ds-1'), + (CURRENT,'2:9.0.0+ds-1')]: + with self.subTest(new=new): + get,_,_=self.fixtures(new) + result=q.discover(old,q.package_version(old),'0'*64,get) + self.assertEqual(result['variables']['_debver'],new) + self.assertGreater(q.compare(new,old),0) + self.assertLess(q.compare('1:10.0~rc1-1','1:10.0-1'),0) + self.assertLess(q.compare('1:10.0-2','1:10.0-10'),0) + + def test_security_wins(self): + newer='1:10.0.11+ds-0+deb13u2' + get,feeds,_=self.fixtures(newer) + _,oldfeeds,_=self.fixtures(CURRENT) + feeds[q.FEEDS[0]]=oldfeeds[q.FEEDS[0]] + self.assertEqual(q.discover(CURRENT,q.package_version(CURRENT),'0'*64,get)['variables']['_debver'],newer) + + def test_unchanged(self): + get,_,sha=self.fixtures(CURRENT) + self.assertEqual(q.discover(CURRENT,q.package_version(CURRENT),sha,get),{}) + with self.assertRaisesRegex(ValueError,'Checksum changed'): + q.discover(CURRENT,q.package_version(CURRENT),'0'*64,get) + + def test_bad_metadata(self): + get,feeds,_=self.fixtures('1:10.0.12+ds-1') + feeds[q.FEEDS[0]]=lzma.compress(b'Package: qemu-user\nArchitecture: arm64\nVersion: invalid\n') + with self.assertRaises(ValueError): q.discover(CURRENT,q.package_version(CURRENT),'0'*64,get) + + def test_snapshot_failures(self): + for failure in ['missing','bytes','metadata']: + get,_,_=self.fixtures('1:10.0.12+ds-1') + def bad(url): + if '/file/' in url: + if failure=='missing': raise OSError('unavailable snapshot') + if failure=='bytes': return b'corrupt' + if '/mr/' in url and failure=='metadata': return b'{}' + return get(url) + with self.subTest(failure=failure), self.assertRaises((ValueError,OSError)): + q.discover(CURRENT,q.package_version(CURRENT),'0'*64,bad) + + def sync(self, recipe, release): + with tempfile.TemporaryDirectory() as tmp: + p=Path(tmp)/'fixture'; (p/'.omarchy').mkdir(parents=True) + (p/'PKGBUILD').write_text(recipe) + (p/'.omarchy/package.json').write_text('{"source":"local"}') + (p/'.omarchy/upstream.sh').write_text("#!/bin/bash\ncat <<'JSON'\n"+json.dumps(release)+"\nJSON\n") + # Load production functions, excluding only the command dispatch. + prefix=(ROOT/'bin/sync-upstream').read_text().split('if [[ ${#SPECIFIC_PACKAGES[@]} -gt 0 &&')[0] + prefix=prefix.replace('BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..")', 'BUILD_ROOT='+str(ROOT)) + command=prefix+'\nPKGBUILDS_DIR='+tmp+'\nSPECIFIC_MODE=true\nsync_package fixture\n((FAILED == 0))\n' + result=subprocess.run(['bash','-c',command],capture_output=True,text=True) + return result,(p/'PKGBUILD').read_text() + + def test_atomic_sync_and_idempotence(self): + get,_,_=self.fixtures('1:10.0.11+ds-0+deb13u2') + release=q.discover(CURRENT,q.package_version(CURRENT),'0'*64,get) + before=(PACKAGE/'PKGBUILD').read_text() + before=re.sub(r'^pkgver=.*$', 'pkgver='+q.package_version(CURRENT), before, flags=re.M) + before=re.sub(r'^_debver=.*$', '_debver='+CURRENT, before, flags=re.M) + result,after=self.sync(before,release) + self.assertEqual(result.returncode,0,result.stdout+result.stderr) + self.assertIn('_debver='+release['variables']['_debver'],after) + self.assertIn('_snapshot='+release['variables']['_snapshot'],after) + self.assertIn(release['sha256sums']['aarch64'][0],after) + result,again=self.sync(after,release) + self.assertEqual(result.returncode,0,result.stdout+result.stderr) + self.assertEqual(again,after) + + def test_invalid_variables_leave_recipe_unchanged(self): + before=(PACKAGE/'PKGBUILD').read_text() + for variables in [{'_debver':'$(touch /tmp/qemu-injection)'}, {'pkgver':'2'}, + {'_absent':'1'}, {'_snapshot':'abc\ncommand'}, {'_snapshot':'https://bad'}]: + with self.subTest(variables=variables): + result,after=self.sync(before,{'pkgver':'99','variables':variables,'sha256sums':{'aarch64':['a'*64]}}) + self.assertNotEqual(result.returncode,0) + self.assertEqual(before,after) + +if __name__=='__main__': unittest.main()