Track Debian QEMU revisions through verified immutable snapshots

.github/workflows/test.yml

@@ -50,6 +50,7 @@ jobs:
               pacman -Syu --noconfirm git jq python libarchive
               python tests/oma-service-removal.py
               python tests/upstream-watch.py
+              python tests/qemu-upstream.py
               ./bin/sync-upstream self-test
               ./bin/sync-rebuilds --self-test
               ./bin/omarchy-pkgs self-test

bin/sync-upstream

@@ -31,7 +31,10 @@ pkgbuilds/<package>/.omarchy/upstream.sh, a hook that reports JSON on stdout:
   }
 
 Architecture keys become sha256sums_<arch> in the PKGBUILD; the key "any" means
-the unsuffixed sha256sums array. An empty object ({}) reports no update.
+the unsuffixed sha256sums array. An empty object ({}) reports no update. Optional "variables" maps existing
+underscore-prefixed release scalars to values containing only letters, digits,
+periods, underscores, plus, colon and hyphen. They are verified and written in
+the same atomic replacement as pkgver and checksums.
 
 When the reported version is newer than the checked-in one, pkgver and the
 listed checksum arrays are rewritten and pkgrel is reset to 1.
@@ -209,6 +212,10 @@ validate_release() {
     and (.sha256sums | to_entries | all(
       .value | type == "array" and length > 0 and all(test("\\A[0-9a-f]{64}\\z"))
     ))
+    and (if has("variables") then (.variables | type == "object" and (to_entries | all(
+      (.key | test("\\A_[a-z][a-z0-9_]*\\z")) and
+      (.value | type == "string" and test("\\A[A-Za-z0-9._+:-]+\\z"))
+    ))) else true end)
     and (if has("published_at") then (.published_at | type == "string" and length > 0) else true end)
   ' <<<"$release" >/dev/null
 }
@@ -312,6 +319,13 @@ apply_release() {
     set_pkgbuild_scalar "$scratch" pkgver "$pkgver" || exit 1
     set_pkgbuild_scalar "$scratch" pkgrel 1 || exit 1
 
+    local field value
+    while IFS=$'\t' read -r field value; do
+      [[ -n "$field" ]] || continue
+      set_pkgbuild_scalar "$scratch" "$field" "$value" || exit 1
+      [[ $(bash -c 'source "$1"; printf "%s" "${!2}"' _ "$scratch" "$field") == "$value" ]] || exit 1
+    done < <(jq -r '(.variables // {}) | to_entries[] | [.key, .value] | @tsv' <<<"$release")
+
     verify_pkgbuild "$scratch" "$release" "$pkgver" "${arrays[@]}" || exit 1
   ); then
     rm -f "$scratch"

pkgbuilds/qemu-user-static/.omarchy/package.json

@@ -1,4 +1,3 @@
 {
-  "source": "local",
-  "sync": false
+  "source": "local"
 }

pkgbuilds/qemu-user-static/.omarchy/upstream.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+"""Track Debian 13 ARM64 QEMU revisions, verifying an immutable snapshot first."""
+import functools
+import hashlib
+import json
+import lzma
+from pathlib import Path
+import re
+import subprocess
+import sys
+import urllib.parse
+import urllib.request
+
+FEEDS = [
+    'https://deb.debian.org/debian/dists/trixie/main/binary-arm64/Packages.xz',
+    'https://deb.debian.org/debian/dists/trixie-updates/main/binary-arm64/Packages.xz',
+    'https://security.debian.org/debian-security/dists/trixie-security/main/binary-arm64/Packages.xz',
+]
+
+def fetch(url):
+    with urllib.request.urlopen(url, timeout=120) as response:
+        if not response.url.startswith('https://'):
+            raise ValueError('Insecure redirect')
+        return response.read()
+
+def parts(version):
+    match = re.fullmatch(r'(?:(\d+):)?([0-9][A-Za-z0-9.+~]*?)-([A-Za-z0-9.+~]+)', version)
+    if not match:
+        raise ValueError('Unsupported Debian version: ' + version)
+    return int(match[1] or '0'), match[2], match[3]
+
+def segment_cmp(a, b):
+    # Debian policy: tilde precedes everything, then end/digits, letters,
+    # then other characters; digit runs are compared numerically.
+    def order(c):
+        if c == '~': return -1
+        if not c or c.isdigit(): return 0
+        return ord(c) if c.isalpha() else ord(c) + 256
+    while a or b:
+        while (a and not a[0].isdigit()) or (b and not b[0].isdigit()):
+            x, y = order(a[:1]), order(b[:1])
+            if x != y: return (x > y) - (x < y)
+            a, b = a[1:], b[1:]
+        x = re.match(r'\d*', a)[0]
+        y = re.match(r'\d*', b)[0]
+        nx, ny = int(x or '0'), int(y or '0')
+        if nx != ny: return (nx > ny) - (nx < ny)
+        a, b = a[len(x):], b[len(y):]
+    return 0
+
+def compare(a, b):
+    ea, ua, ra = parts(a)
+    eb, ub, rb = parts(b)
+    return (ea > eb) - (ea < eb) or segment_cmp(ua, ub) or segment_cmp(ra, rb)
+
+def package_version(version):
+    epoch, upstream, revision = parts(version)
+    # Stable releases only. Fail visibly on prerelease/repack conventions that
+    # need a reviewed Arch ordering rather than silently misordering them.
+    if '~' in version:
+        raise ValueError('Debian prerelease needs a reviewed Arch version mapping')
+    return f'{epoch}.{upstream}.{revision}'
+
+def records(data):
+    found = []
+    for stanza in re.split(r'\n\s*\n', lzma.decompress(data).decode()):
+        fields = dict(re.findall(r'^([A-Za-z0-9-]+): (.*)$', stanza, re.M))
+        if fields.get('Package') != 'qemu-user': continue
+        if fields.get('Architecture') != 'arm64': raise ValueError('Wrong architecture')
+        parts(fields['Version'])
+        if not re.fullmatch(r'[0-9a-f]{64}', fields.get('SHA256', '')):
+            raise ValueError('Missing/malformed SHA256')
+        if not re.fullmatch(r'[1-9][0-9]*', fields.get('Size', '')):
+            raise ValueError('Missing/malformed package size')
+        found.append(fields)
+    return found
+
+def discover(current, current_pkgver, current_hash, get=fetch):
+    candidates = [row for url in FEEDS for row in records(get(url))]
+    if not candidates: raise ValueError('No ARM64 qemu-user package in Debian feeds')
+    selected = max(candidates, key=functools.cmp_to_key(lambda a,b: compare(a['Version'], b['Version'])))
+    version, checksum = selected['Version'], selected['SHA256']
+    if any(row['SHA256'] != checksum for row in candidates if row['Version'] == version):
+        raise ValueError('Debian feeds disagree on the selected checksum')
+    ordering = compare(version, current)
+    if ordering < 0: raise ValueError('Debian feeds are older than the pinned recipe')
+    if ordering == 0:
+        if checksum != current_hash: raise ValueError('Checksum changed for the pinned Debian revision')
+        return {}
+    pkgver = package_version(version)
+    if int(subprocess.check_output(['vercmp', pkgver, current_pkgver], text=True)) <= 0:
+        raise ValueError('New Debian revision does not advance Arch version; review mapping')
+    api = 'https://snapshot.debian.org/mr/binary/qemu-user/' + urllib.parse.quote(version, safe='') + '/binfiles'
+    document = json.loads(get(api))
+    if document.get('binary') != 'qemu-user' or document.get('binary_version') != version:
+        raise ValueError('Snapshot revision differs')
+    hashes = {row['hash'] for row in document['result'] if row['architecture'] == 'arm64'}
+    if len(hashes) != 1 or not re.fullmatch(r'[0-9a-f]{40}', next(iter(hashes), '')):
+        raise ValueError('Missing/ambiguous ARM64 snapshot')
+    snapshot = hashes.pop()
+    archive = get('https://snapshot.debian.org/file/' + snapshot)
+    if len(archive) != int(selected['Size']) or hashlib.sha256(archive).hexdigest() != checksum or hashlib.sha1(archive).hexdigest() != snapshot:
+        raise ValueError('Snapshot bytes differ from Debian package metadata')
+    return {'pkgver': pkgver, 'variables': {'_debver': version, '_snapshot': snapshot},
+            'sha256sums': {'aarch64': [checksum]}}
+
+def main():
+    recipe = Path('PKGBUILD').read_text()
+    def scalar(name):
+        match = re.search(r'^' + name + r'=[\"\']?([^\"\'\n]+)', recipe, re.M)
+        if not match: raise ValueError('Missing recipe scalar: ' + name)
+        return match[1]
+    checksum = re.search(r"^sha256sums_aarch64=\('([a-f0-9]{64})'\)", recipe, re.M)
+    if not checksum: raise ValueError('Missing current ARM checksum')
+    print(json.dumps(discover(scalar('_debver'), scalar('pkgver'), checksum[1])))
+
+if __name__ == '__main__':
+    try: main()
+    except Exception as error:
+        sys.exit('QEMU Debian update failed: ' + str(error))

pkgbuilds/qemu-user-static/.omarchy/upstream.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -euo pipefail
+exec python3 .omarchy/upstream.py

pkgbuilds/qemu-user-static/PKGBUILD

@@ -5,8 +5,12 @@
 
 pkgbase=qemu-user-static
 pkgname=('qemu-user-static' 'qemu-user-static-binfmt')
-pkgver=10.0.11
-_debver="1:10.0.11+ds-0+deb13u1+b1"
+# One-time epoch moves from upstream-only to full Debian revision ordering.
+# pkgver records Debian epoch.upstream.revision; updates also verify vercmp.
+epoch=1
+pkgver=1.10.0.13+ds.0+deb13u1
+_debver=1:10.0.13+ds-0+deb13u1
+_snapshot=02b553a4532192bc999d15408d1e56d7b254906f
 pkgrel=1
 pkgdesc="A generic and open source machine emulator and virtualizer - static user-mode emulation"
 arch=('aarch64')
@@ -14,10 +18,11 @@ url="https://www.qemu.org"
 license=('GPL-2.0-only' 'LGPL-2.1-only')
 source=("qemu-binfmt-conf.sh")
 # Debian removes superseded pool files. Pin the official archive snapshot so
-# the same checksum-verified binary remains rebuildable.
-source_aarch64=("qemu-user_${pkgver}_arm64.deb::https://snapshot.debian.org/archive/debian/20260713T202617Z/pool/main/q/qemu/qemu-user_${_debver#*:}_arm64.deb")
+# the same checksum-verified binary remains rebuildable. _snapshot is the
+# content-addressed snapshot SHA1; SHA256 below remains the integrity check.
+source_aarch64=("qemu-user_${pkgver}_arm64.deb::https://snapshot.debian.org/file/$_snapshot")
 sha256sums=('5e12ef484b8364e1ba7bae4acda837d2e6320ffbeb7fab341034124553cdb137')
-sha256sums_aarch64=('6c9483063bf60f37fe181ead911251deabe40c893ea5829bc34b0b7b88913b6a')
+sha256sums_aarch64=('c73711af02b97cd5e2667e735d9c06890c57165517110ca4e646c95a7083158d')
 noextract=("qemu-user_${pkgver}_arm64.deb")
 
 prepare() {

pkgbuilds/qemu-user-static/README.md

@@ -1,13 +1,13 @@
-# ARM static QEMU candidate
+# ARM static QEMU
 
-This split recipe provides `qemu-user-static` and `qemu-user-static-binfmt` on aarch64, where Arch Linux ARM does not supply the static emulator packages requested by Omarchy's default package set. It repackages Debian's checksum-pinned ARM64 binary; it does not compile QEMU or replace upstream's build-worker emulation setup.
+This aarch64-only split recipe provides qemu-user-static and qemu-user-static-binfmt using Debian’s static ARM64 binaries. It preserves Marcelo’s recipe, imported through omarchy-mac/omarchy-pkgs-aarch64#61, including licensing and binfmt behavior. It does not change x86 packages or the build workers.
 
-Imported from Marcelo's `maralcbr/omarchy-pkgs` revision `83973903b7deb9b56ce75f02b432fba0561d6293`, through the validated candidate recipe in `omarchy-mac/omarchy-pkgs-aarch64` (#61). Debian copyright and upstream licenses are retained. The immutable Debian snapshot keeps the pinned binary available when the rolling pool removes old revisions.
+## Updates
 
-## Draft readiness gates
+The package-local upstream hook checks Debian 13 (trixie), trixie-updates, and trixie-security ARM64 indexes. It compares full Debian versions, including epochs, security revisions and binary rebuilds, and resolves the selected binary through snapshot.debian.org’s API. The immutable content-addressed archive must match the index’s size and SHA256 as well as the snapshot SHA1 before an update is returned.
 
-Automatic sync is deliberately disabled in this initial draft. This does **not** detect new releases or security updates. Do not mark this PR ready until a package-local, fixture-tested update integration handles Debian's full version (epoch, upstream version, Debian security revision and architecture rebuild), immutable archive location and SHA256 together. A new Debian revision at the same QEMU version must produce a forward package version; repeated checks must be idempotent. Missing or malformed metadata, unavailable snapshots and checksum inconsistencies must leave the recipe unchanged and fail visibly.
+A one-time Arch epoch of 1 moves away from the previous upstream-only version. pkgver encodes Debian epoch.upstream.revision. The hook requires both Debian ordering and pacman vercmp to advance; unfamiliar prerelease conventions or ordering discrepancies fail for manual review. Debian distribution upgrades are explicit recipe changes, not automatic jumps to testing/unstable.
 
-The existing simple custom-hook contract updates pkgver and SHA256 arrays but cannot update `_debver` and the snapshot timestamp together. Do not add a misleading upstream-version-only watch. Review the smallest compatible integration with upstream before removing the hold. Once enabled, updates should join upstream's existing six-hour update PR workflow; GitHub notification settings determine who sees those PRs. Until then, check Debian's package and security updates manually.
+The existing six-hour upstream update PR workflow discovers the hook. Its pkgver, _debver, _snapshot and ARM checksum are applied atomically through the normal sync interface. Repeated checks are idempotent. Feed failures, malformed metadata, conflicting checksums and unavailable/corrupt snapshots fail visibly before recipe mutation. No update installs packages or registers binfmt rules.
 
-Before readiness, build both outputs using upstream's ARM builder and test installation in a disposable ARM VM. Verify AArch64 static ELF executables, exact interpreter paths and ownership, the absence of native AArch64 binfmt rules, execution of a known foreign-architecture program both directly and through binfmt, and clean package removal. Never register binfmt rules on the developer host or assume a container isolates that kernel state. Recipe generation only writes rules into the package staging directory.
+Offline fixtures cover version/epoch/security/binNMU updates, security-feed selection, unchanged versions, metadata and snapshot failures, atomic application, and hostile/missing scalar rejection. Existing GUI-independent VM qualification and packaging evidence are recorded in the PR; new binaries still receive build and runtime checks before publication.

tests/qemu-upstream.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+import hashlib
+import importlib.util
+import json
+import lzma
+import re
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[1]
+PACKAGE = ROOT / 'pkgbuilds/qemu-user-static'
+spec = importlib.util.spec_from_file_location('qemu', PACKAGE / '.omarchy/upstream.py')
+q = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(q)
+CURRENT = '1:10.0.11+ds-0+deb13u1+b1'
+
+class Updates(unittest.TestCase):
+    def fixtures(self, version):
+        blob = b'fixture package bytes'
+        sha = hashlib.sha256(blob).hexdigest()
+        snap = hashlib.sha1(blob).hexdigest()
+        row = f'Package: qemu-user\nArchitecture: arm64\nVersion: {version}\nSHA256: {sha}\nSize: {len(blob)}\n'
+        feeds = {url: lzma.compress(row.encode()) for url in q.FEEDS}
+        def get(url):
+            if url in feeds: return feeds[url]
+            if '/mr/' in url:
+                return json.dumps({'binary':'qemu-user', 'binary_version':version,
+                  'result':[{'architecture':'arm64','hash':snap}]}).encode()
+            return blob
+        return get, feeds, sha
+
+    def test_versions(self):
+        for old, new in [(CURRENT,'1:10.0.11+ds-0+deb13u1+b2'),
+                         (CURRENT,'1:10.0.11+ds-0+deb13u2'),
+                         (CURRENT,'1:10.0.12+ds-1'),
+                         (CURRENT,'2:9.0.0+ds-1')]:
+            with self.subTest(new=new):
+                get,_,_=self.fixtures(new)
+                result=q.discover(old,q.package_version(old),'0'*64,get)
+                self.assertEqual(result['variables']['_debver'],new)
+                self.assertGreater(q.compare(new,old),0)
+        self.assertLess(q.compare('1:10.0~rc1-1','1:10.0-1'),0)
+        self.assertLess(q.compare('1:10.0-2','1:10.0-10'),0)
+
+    def test_security_wins(self):
+        newer='1:10.0.11+ds-0+deb13u2'
+        get,feeds,_=self.fixtures(newer)
+        _,oldfeeds,_=self.fixtures(CURRENT)
+        feeds[q.FEEDS[0]]=oldfeeds[q.FEEDS[0]]
+        self.assertEqual(q.discover(CURRENT,q.package_version(CURRENT),'0'*64,get)['variables']['_debver'],newer)
+
+    def test_unchanged(self):
+        get,_,sha=self.fixtures(CURRENT)
+        self.assertEqual(q.discover(CURRENT,q.package_version(CURRENT),sha,get),{})
+        with self.assertRaisesRegex(ValueError,'Checksum changed'):
+            q.discover(CURRENT,q.package_version(CURRENT),'0'*64,get)
+
+    def test_bad_metadata(self):
+        get,feeds,_=self.fixtures('1:10.0.12+ds-1')
+        feeds[q.FEEDS[0]]=lzma.compress(b'Package: qemu-user\nArchitecture: arm64\nVersion: invalid\n')
+        with self.assertRaises(ValueError): q.discover(CURRENT,q.package_version(CURRENT),'0'*64,get)
+
+    def test_snapshot_failures(self):
+        for failure in ['missing','bytes','metadata']:
+            get,_,_=self.fixtures('1:10.0.12+ds-1')
+            def bad(url):
+                if '/file/' in url:
+                    if failure=='missing': raise OSError('unavailable snapshot')
+                    if failure=='bytes': return b'corrupt'
+                if '/mr/' in url and failure=='metadata': return b'{}'
+                return get(url)
+            with self.subTest(failure=failure), self.assertRaises((ValueError,OSError)):
+                q.discover(CURRENT,q.package_version(CURRENT),'0'*64,bad)
+
+    def sync(self, recipe, release):
+        with tempfile.TemporaryDirectory() as tmp:
+            p=Path(tmp)/'fixture'; (p/'.omarchy').mkdir(parents=True)
+            (p/'PKGBUILD').write_text(recipe)
+            (p/'.omarchy/package.json').write_text('{"source":"local"}')
+            (p/'.omarchy/upstream.sh').write_text("#!/bin/bash\ncat <<'JSON'\n"+json.dumps(release)+"\nJSON\n")
+            # Load production functions, excluding only the command dispatch.
+            prefix=(ROOT/'bin/sync-upstream').read_text().split('if [[ ${#SPECIFIC_PACKAGES[@]} -gt 0 &&')[0]
+            prefix=prefix.replace('BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..")', 'BUILD_ROOT='+str(ROOT))
+            command=prefix+'\nPKGBUILDS_DIR='+tmp+'\nSPECIFIC_MODE=true\nsync_package fixture\n((FAILED == 0))\n'
+            result=subprocess.run(['bash','-c',command],capture_output=True,text=True)
+            return result,(p/'PKGBUILD').read_text()
+
+    def test_atomic_sync_and_idempotence(self):
+        get,_,_=self.fixtures('1:10.0.11+ds-0+deb13u2')
+        release=q.discover(CURRENT,q.package_version(CURRENT),'0'*64,get)
+        before=(PACKAGE/'PKGBUILD').read_text()
+        before=re.sub(r'^pkgver=.*$', 'pkgver='+q.package_version(CURRENT), before, flags=re.M)
+        before=re.sub(r'^_debver=.*$', '_debver='+CURRENT, before, flags=re.M)
+        result,after=self.sync(before,release)
+        self.assertEqual(result.returncode,0,result.stdout+result.stderr)
+        self.assertIn('_debver='+release['variables']['_debver'],after)
+        self.assertIn('_snapshot='+release['variables']['_snapshot'],after)
+        self.assertIn(release['sha256sums']['aarch64'][0],after)
+        result,again=self.sync(after,release)
+        self.assertEqual(result.returncode,0,result.stdout+result.stderr)
+        self.assertEqual(again,after)
+
+    def test_invalid_variables_leave_recipe_unchanged(self):
+        before=(PACKAGE/'PKGBUILD').read_text()
+        for variables in [{'_debver':'$(touch /tmp/qemu-injection)'}, {'pkgver':'2'},
+                          {'_absent':'1'}, {'_snapshot':'abc\ncommand'}, {'_snapshot':'https://bad'}]:
+            with self.subTest(variables=variables):
+                result,after=self.sync(before,{'pkgver':'99','variables':variables,'sha256sums':{'aarch64':['a'*64]}})
+                self.assertNotEqual(result.returncode,0)
+                self.assertEqual(before,after)
+
+if __name__=='__main__': unittest.main()