diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7bcbb22..29d18b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,7 +50,6 @@ jobs: pacman -Syu --noconfirm git jq python libarchive python tests/oma-service-removal.py python tests/upstream-watch.py - python tests/qemu-upstream.py ./bin/sync-upstream self-test ./bin/sync-rebuilds --self-test ./bin/omarchy-pkgs self-test diff --git a/bin/sync-upstream b/bin/sync-upstream index 360a1b2..2f84506 100755 --- a/bin/sync-upstream +++ b/bin/sync-upstream @@ -31,10 +31,7 @@ pkgbuilds//.omarchy/upstream.sh, a hook that reports JSON on stdout: } Architecture keys become sha256sums_ in the PKGBUILD; the key "any" means -the unsuffixed sha256sums array. An empty object ({}) reports no update. Optional "variables" maps existing -underscore-prefixed release scalars to values containing only letters, digits, -periods, underscores, plus, colon and hyphen. They are verified and written in -the same atomic replacement as pkgver and checksums. +the unsuffixed sha256sums array. An empty object ({}) reports no update. When the reported version is newer than the checked-in one, pkgver and the listed checksum arrays are rewritten and pkgrel is reset to 1. @@ -212,10 +209,6 @@ validate_release() { and (.sha256sums | to_entries | all( .value | type == "array" and length > 0 and all(test("\\A[0-9a-f]{64}\\z")) )) - and (if has("variables") then (.variables | type == "object" and (to_entries | all( - (.key | test("\\A_[a-z][a-z0-9_]*\\z")) and - (.value | type == "string" and test("\\A[A-Za-z0-9._+:-]+\\z")) - ))) else true end) and (if has("published_at") then (.published_at | type == "string" and length > 0) else true end) ' <<<"$release" >/dev/null } @@ -319,13 +312,6 @@ apply_release() { set_pkgbuild_scalar "$scratch" pkgver "$pkgver" || exit 1 set_pkgbuild_scalar "$scratch" pkgrel 1 || exit 1 - local field value - while IFS=$'\t' read -r field value; do - [[ -n "$field" ]] || continue - set_pkgbuild_scalar "$scratch" "$field" "$value" || exit 1 - [[ $(bash -c 'source "$1"; printf "%s" "${!2}"' _ "$scratch" "$field") == "$value" ]] || exit 1 - done < <(jq -r '(.variables // {}) | to_entries[] | [.key, .value] | @tsv' <<<"$release") - verify_pkgbuild "$scratch" "$release" "$pkgver" "${arrays[@]}" || exit 1 ); then rm -f "$scratch" diff --git a/pkgbuilds/qemu-user-static/.omarchy/upstream.py b/pkgbuilds/qemu-user-static/.omarchy/upstream.py deleted file mode 100644 index 341d9c1..0000000 --- a/pkgbuilds/qemu-user-static/.omarchy/upstream.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python3 -"""Track Debian 13 ARM64 QEMU revisions, verifying an immutable snapshot first.""" -import functools -import hashlib -import json -import lzma -from pathlib import Path -import re -import subprocess -import sys -import urllib.parse -import urllib.request - -FEEDS = [ - 'https://deb.debian.org/debian/dists/trixie/main/binary-arm64/Packages.xz', - 'https://deb.debian.org/debian/dists/trixie-updates/main/binary-arm64/Packages.xz', - 'https://security.debian.org/debian-security/dists/trixie-security/main/binary-arm64/Packages.xz', -] - -def fetch(url): - with urllib.request.urlopen(url, timeout=120) as response: - if not response.url.startswith('https://'): - raise ValueError('Insecure redirect') - return response.read() - -def parts(version): - match = re.fullmatch(r'(?:(\d+):)?([0-9][A-Za-z0-9.+~]*?)-([A-Za-z0-9.+~]+)', version) - if not match: - raise ValueError('Unsupported Debian version: ' + version) - return int(match[1] or '0'), match[2], match[3] - -def segment_cmp(a, b): - # Debian policy: tilde precedes everything, then end/digits, letters, - # then other characters; digit runs are compared numerically. - def order(c): - if c == '~': return -1 - if not c or c.isdigit(): return 0 - return ord(c) if c.isalpha() else ord(c) + 256 - while a or b: - while (a and not a[0].isdigit()) or (b and not b[0].isdigit()): - x, y = order(a[:1]), order(b[:1]) - if x != y: return (x > y) - (x < y) - a, b = a[1:], b[1:] - x = re.match(r'\d*', a)[0] - y = re.match(r'\d*', b)[0] - nx, ny = int(x or '0'), int(y or '0') - if nx != ny: return (nx > ny) - (nx < ny) - a, b = a[len(x):], b[len(y):] - return 0 - -def compare(a, b): - ea, ua, ra = parts(a) - eb, ub, rb = parts(b) - return (ea > eb) - (ea < eb) or segment_cmp(ua, ub) or segment_cmp(ra, rb) - -def package_version(version): - epoch, upstream, revision = parts(version) - # Stable releases only. Fail visibly on prerelease/repack conventions that - # need a reviewed Arch ordering rather than silently misordering them. - if '~' in version: - raise ValueError('Debian prerelease needs a reviewed Arch version mapping') - return f'{epoch}.{upstream}.{revision}' - -def records(data): - found = [] - for stanza in re.split(r'\n\s*\n', lzma.decompress(data).decode()): - fields = dict(re.findall(r'^([A-Za-z0-9-]+): (.*)$', stanza, re.M)) - if fields.get('Package') != 'qemu-user': continue - if fields.get('Architecture') != 'arm64': raise ValueError('Wrong architecture') - parts(fields['Version']) - if not re.fullmatch(r'[0-9a-f]{64}', fields.get('SHA256', '')): - raise ValueError('Missing/malformed SHA256') - if not re.fullmatch(r'[1-9][0-9]*', fields.get('Size', '')): - raise ValueError('Missing/malformed package size') - found.append(fields) - return found - -def discover(current, current_pkgver, current_hash, get=fetch): - candidates = [row for url in FEEDS for row in records(get(url))] - if not candidates: raise ValueError('No ARM64 qemu-user package in Debian feeds') - selected = max(candidates, key=functools.cmp_to_key(lambda a,b: compare(a['Version'], b['Version']))) - version, checksum = selected['Version'], selected['SHA256'] - if any(row['SHA256'] != checksum for row in candidates if row['Version'] == version): - raise ValueError('Debian feeds disagree on the selected checksum') - ordering = compare(version, current) - if ordering < 0: raise ValueError('Debian feeds are older than the pinned recipe') - if ordering == 0: - if checksum != current_hash: raise ValueError('Checksum changed for the pinned Debian revision') - return {} - pkgver = package_version(version) - if int(subprocess.check_output(['vercmp', pkgver, current_pkgver], text=True)) <= 0: - raise ValueError('New Debian revision does not advance Arch version; review mapping') - api = 'https://snapshot.debian.org/mr/binary/qemu-user/' + urllib.parse.quote(version, safe='') + '/binfiles' - document = json.loads(get(api)) - if document.get('binary') != 'qemu-user' or document.get('binary_version') != version: - raise ValueError('Snapshot revision differs') - hashes = {row['hash'] for row in document['result'] if row['architecture'] == 'arm64'} - if len(hashes) != 1 or not re.fullmatch(r'[0-9a-f]{40}', next(iter(hashes), '')): - raise ValueError('Missing/ambiguous ARM64 snapshot') - snapshot = hashes.pop() - archive = get('https://snapshot.debian.org/file/' + snapshot) - if len(archive) != int(selected['Size']) or hashlib.sha256(archive).hexdigest() != checksum or hashlib.sha1(archive).hexdigest() != snapshot: - raise ValueError('Snapshot bytes differ from Debian package metadata') - return {'pkgver': pkgver, 'variables': {'_debver': version, '_snapshot': snapshot}, - 'sha256sums': {'aarch64': [checksum]}} - -def main(): - recipe = Path('PKGBUILD').read_text() - def scalar(name): - match = re.search(r'^' + name + r'=[\"\']?([^\"\'\n]+)', recipe, re.M) - if not match: raise ValueError('Missing recipe scalar: ' + name) - return match[1] - checksum = re.search(r"^sha256sums_aarch64=\('([a-f0-9]{64})'\)", recipe, re.M) - if not checksum: raise ValueError('Missing current ARM checksum') - print(json.dumps(discover(scalar('_debver'), scalar('pkgver'), checksum[1]))) - -if __name__ == '__main__': - try: main() - except Exception as error: - sys.exit('QEMU Debian update failed: ' + str(error)) diff --git a/pkgbuilds/qemu-user-static/.omarchy/upstream.sh b/pkgbuilds/qemu-user-static/.omarchy/upstream.sh deleted file mode 100644 index 829dbc9..0000000 --- a/pkgbuilds/qemu-user-static/.omarchy/upstream.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -set -euo pipefail -exec python3 .omarchy/upstream.py diff --git a/pkgbuilds/qemu-user-static/PKGBUILD b/pkgbuilds/qemu-user-static/PKGBUILD index 22b64cc..8d983a8 100644 --- a/pkgbuilds/qemu-user-static/PKGBUILD +++ b/pkgbuilds/qemu-user-static/PKGBUILD @@ -6,7 +6,8 @@ pkgbase=qemu-user-static pkgname=('qemu-user-static' 'qemu-user-static-binfmt') # One-time epoch moves from upstream-only to full Debian revision ordering. -# pkgver records Debian epoch.upstream.revision; updates also verify vercmp. +# pkgver records Debian epoch.upstream.revision. Updates are reviewed pins: +# bump _debver, pkgver, _snapshot and the checksum together. epoch=1 pkgver=1.10.0.13+ds.0+deb13u1 _debver=1:10.0.13+ds-0+deb13u1 @@ -55,13 +56,16 @@ package_qemu-user-static-binfmt() { install -dm755 "$pkgdir/usr/lib/binfmt.d" # HOST_ARCH keeps the native architecture out of the rule set regardless # of the build host, so the package is identical under native and QEMU builds. + # --ignore-family keeps 32-bit ARM: the script groups it with aarch64, but + # Apple Silicon cannot execute AArch32 natively. HOST_ARCH=aarch64 sh "$srcdir/qemu-binfmt-conf.sh" \ --systemd ALL \ --exportdir "$pkgdir/usr/lib/binfmt.d/" \ --qemu-path /usr/bin \ --qemu-suffix -static \ --persistent yes \ - --preserve-argv0 yes + --preserve-argv0 yes \ + --ignore-family yes local conf for conf in "$pkgdir"/usr/lib/binfmt.d/*.conf; do mv "$conf" "${conf%.conf}-static.conf" diff --git a/pkgbuilds/qemu-user-static/README.md b/pkgbuilds/qemu-user-static/README.md index d30002c..f487ccd 100644 --- a/pkgbuilds/qemu-user-static/README.md +++ b/pkgbuilds/qemu-user-static/README.md @@ -4,10 +4,8 @@ This aarch64-only split recipe provides qemu-user-static and qemu-user-static-bi ## Updates -The package-local upstream hook checks Debian 13 (trixie), trixie-updates, and trixie-security ARM64 indexes. It compares full Debian versions, including epochs, security revisions and binary rebuilds, and resolves the selected binary through snapshot.debian.org’s API. The immutable content-addressed archive must match the index’s size and SHA256 as well as the snapshot SHA1 before an update is returned. +Updates are reviewed pins, not automatic. To move to a new Debian 13 revision, update `_debver`, `pkgver` (Debian epoch.upstream.revision), `_snapshot` (the snapshot.debian.org SHA1 of the ARM64 `qemu-user` archive) and `sha256sums_aarch64` together, and confirm both Debian ordering and `vercmp` advance. A one-time Arch epoch of 1 moves away from the previous upstream-only version. -A one-time Arch epoch of 1 moves away from the previous upstream-only version. pkgver encodes Debian epoch.upstream.revision. The hook requires both Debian ordering and pacman vercmp to advance; unfamiliar prerelease conventions or ordering discrepancies fail for manual review. Debian distribution upgrades are explicit recipe changes, not automatic jumps to testing/unstable. +## binfmt rules -The existing six-hour upstream update PR workflow discovers the hook. Its pkgver, _debver, _snapshot and ARM checksum are applied atomically through the normal sync interface. Repeated checks are idempotent. Feed failures, malformed metadata, conflicting checksums and unavailable/corrupt snapshots fail visibly before recipe mutation. No update installs packages or registers binfmt rules. - -Offline fixtures cover version/epoch/security/binNMU updates, security-feed selection, unchanged versions, metadata and snapshot failures, atomic application, and hostile/missing scalar rejection. Existing GUI-independent VM qualification and packaging evidence are recorded in the PR; new binaries still receive build and runtime checks before publication. +The rules are generated with `--ignore-family yes`, so 32-bit ARM binaries are registered even though the script groups them with aarch64; Apple Silicon has no AArch32 execution. Native aarch64 stays excluded. Rules use the persistent and preserve-argv0 flags and never the credential flag. diff --git a/tests/qemu-upstream.py b/tests/qemu-upstream.py deleted file mode 100644 index 30a54a2..0000000 --- a/tests/qemu-upstream.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python3 -import hashlib -import importlib.util -import json -import lzma -import re -from pathlib import Path -import subprocess -import tempfile -import unittest -from unittest.mock import patch - -ROOT = Path(__file__).resolve().parents[1] -PACKAGE = ROOT / 'pkgbuilds/qemu-user-static' -spec = importlib.util.spec_from_file_location('qemu', PACKAGE / '.omarchy/upstream.py') -q = importlib.util.module_from_spec(spec) -spec.loader.exec_module(q) -CURRENT = '1:10.0.11+ds-0+deb13u1+b1' - -class Updates(unittest.TestCase): - def fixtures(self, version): - blob = b'fixture package bytes' - sha = hashlib.sha256(blob).hexdigest() - snap = hashlib.sha1(blob).hexdigest() - row = f'Package: qemu-user\nArchitecture: arm64\nVersion: {version}\nSHA256: {sha}\nSize: {len(blob)}\n' - feeds = {url: lzma.compress(row.encode()) for url in q.FEEDS} - def get(url): - if url in feeds: return feeds[url] - if '/mr/' in url: - return json.dumps({'binary':'qemu-user', 'binary_version':version, - 'result':[{'architecture':'arm64','hash':snap}]}).encode() - return blob - return get, feeds, sha - - def test_versions(self): - for old, new in [(CURRENT,'1:10.0.11+ds-0+deb13u1+b2'), - (CURRENT,'1:10.0.11+ds-0+deb13u2'), - (CURRENT,'1:10.0.12+ds-1'), - (CURRENT,'2:9.0.0+ds-1')]: - with self.subTest(new=new): - get,_,_=self.fixtures(new) - result=q.discover(old,q.package_version(old),'0'*64,get) - self.assertEqual(result['variables']['_debver'],new) - self.assertGreater(q.compare(new,old),0) - self.assertLess(q.compare('1:10.0~rc1-1','1:10.0-1'),0) - self.assertLess(q.compare('1:10.0-2','1:10.0-10'),0) - - def test_security_wins(self): - newer='1:10.0.11+ds-0+deb13u2' - get,feeds,_=self.fixtures(newer) - _,oldfeeds,_=self.fixtures(CURRENT) - feeds[q.FEEDS[0]]=oldfeeds[q.FEEDS[0]] - self.assertEqual(q.discover(CURRENT,q.package_version(CURRENT),'0'*64,get)['variables']['_debver'],newer) - - def test_unchanged(self): - get,_,sha=self.fixtures(CURRENT) - self.assertEqual(q.discover(CURRENT,q.package_version(CURRENT),sha,get),{}) - with self.assertRaisesRegex(ValueError,'Checksum changed'): - q.discover(CURRENT,q.package_version(CURRENT),'0'*64,get) - - def test_bad_metadata(self): - get,feeds,_=self.fixtures('1:10.0.12+ds-1') - feeds[q.FEEDS[0]]=lzma.compress(b'Package: qemu-user\nArchitecture: arm64\nVersion: invalid\n') - with self.assertRaises(ValueError): q.discover(CURRENT,q.package_version(CURRENT),'0'*64,get) - - def test_snapshot_failures(self): - for failure in ['missing','bytes','metadata']: - get,_,_=self.fixtures('1:10.0.12+ds-1') - def bad(url): - if '/file/' in url: - if failure=='missing': raise OSError('unavailable snapshot') - if failure=='bytes': return b'corrupt' - if '/mr/' in url and failure=='metadata': return b'{}' - return get(url) - with self.subTest(failure=failure), self.assertRaises((ValueError,OSError)): - q.discover(CURRENT,q.package_version(CURRENT),'0'*64,bad) - - def sync(self, recipe, release): - with tempfile.TemporaryDirectory() as tmp: - p=Path(tmp)/'fixture'; (p/'.omarchy').mkdir(parents=True) - (p/'PKGBUILD').write_text(recipe) - (p/'.omarchy/package.json').write_text('{"source":"local"}') - (p/'.omarchy/upstream.sh').write_text("#!/bin/bash\ncat <<'JSON'\n"+json.dumps(release)+"\nJSON\n") - # Load production functions, excluding only the command dispatch. - prefix=(ROOT/'bin/sync-upstream').read_text().split('if [[ ${#SPECIFIC_PACKAGES[@]} -gt 0 &&')[0] - prefix=prefix.replace('BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..")', 'BUILD_ROOT='+str(ROOT)) - command=prefix+'\nPKGBUILDS_DIR='+tmp+'\nSPECIFIC_MODE=true\nsync_package fixture\n((FAILED == 0))\n' - result=subprocess.run(['bash','-c',command],capture_output=True,text=True) - return result,(p/'PKGBUILD').read_text() - - def test_atomic_sync_and_idempotence(self): - get,_,_=self.fixtures('1:10.0.11+ds-0+deb13u2') - release=q.discover(CURRENT,q.package_version(CURRENT),'0'*64,get) - before=(PACKAGE/'PKGBUILD').read_text() - before=re.sub(r'^pkgver=.*$', 'pkgver='+q.package_version(CURRENT), before, flags=re.M) - before=re.sub(r'^_debver=.*$', '_debver='+CURRENT, before, flags=re.M) - result,after=self.sync(before,release) - self.assertEqual(result.returncode,0,result.stdout+result.stderr) - self.assertIn('_debver='+release['variables']['_debver'],after) - self.assertIn('_snapshot='+release['variables']['_snapshot'],after) - self.assertIn(release['sha256sums']['aarch64'][0],after) - result,again=self.sync(after,release) - self.assertEqual(result.returncode,0,result.stdout+result.stderr) - self.assertEqual(again,after) - - def test_invalid_variables_leave_recipe_unchanged(self): - before=(PACKAGE/'PKGBUILD').read_text() - for variables in [{'_debver':'$(touch /tmp/qemu-injection)'}, {'pkgver':'2'}, - {'_absent':'1'}, {'_snapshot':'abc\ncommand'}, {'_snapshot':'https://bad'}]: - with self.subTest(variables=variables): - result,after=self.sync(before,{'pkgver':'99','variables':variables,'sha256sums':{'aarch64':['a'*64]}}) - self.assertNotEqual(result.returncode,0) - self.assertEqual(before,after) - -if __name__=='__main__': unittest.main()