diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..0eb40d9 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,31 @@ +name: Tests + +on: + pull_request: + push: + branches: [master] + workflow_dispatch: + +jobs: + self-tests: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + # An Arch container for vercmp: version ordering has to be decided by + # the same comparator pacman uses on users' machines. + - name: Run self-tests + run: | + docker run --rm \ + -v "$PWD:/workspace:ro" \ + -w /workspace \ + archlinux:base-devel bash -lc ' + set -euo pipefail + pacman -Syu --noconfirm jq + ./bin/sync-upstream self-test + ./bin/omarchy-pkgs self-test + ' diff --git a/README.md b/README.md index 3530dab..6ba724c 100644 --- a/README.md +++ b/README.md @@ -258,8 +258,41 @@ bin/sync-upstream openai-codex-desktop # Update specific packages Some vendors publish a release feed of their own that is faster and more precise than the AUR packaging of it. Those packages are `source: local` — Omarchy owns -the PKGBUILD — and provide `.omarchy/upstream.sh`, a hook that reports the newest -upstream release as JSON on stdout: +the PKGBUILD — and declare where releases come from in one of two ways. + +A vendor shipping tagged GitHub releases with a checksum manifest asset is pure +data, declared as `upstream` in `.omarchy/package.json` with no code at all: + +```json +"upstream": { + "github": "jdx/mise", + "checksums": "SHASUMS256.txt", + "assets": { + "x86_64": "mise-{tag}-linux-x64.tar.xz", + "aarch64": "mise-{tag}-linux-arm64.tar.xz" + } +} +``` + +`{tag}` and `{pkgver}` interpolate into asset names; a leading `v` on the tag is +stripped for `pkgver`; drafts and prereleases are ignored. Only the 100 most +recent releases are considered. The provider fails closed on anything it cannot +read — an unusable tag, timestamp, or checksum stops the sync rather than being +skipped. + +A package may also declare `"min_release_age": "24h"` (`s`/`m`/`h`/`d` suffix or +bare seconds) to quarantine fresh releases until maintainers have had time to +pull a bad or compromised one. The newest release that has cleared the window +ships, so a fast release cadence cannot starve updates. The window is enforced +centrally: whatever reports the release must prove its age via `published_at`, +or the sync fails. A maintainer deliberately shipping inside the window runs +`BYPASS_MIN_RELEASE_AGE=1 bin/sync-upstream ` locally and merges the +result through a normal PR; scheduled automation never sets the bypass. + +A vendor whose feed fits no convention (a Debian package index, a bare +version.txt) instead provides `.omarchy/upstream.sh`, a hook that reports the +newest upstream release as JSON on stdout — declaring both an `upstream` block +and a hook is an error: ```json { @@ -281,7 +314,11 @@ back cannot walk the repository backwards. Hooks should read checksums from whatever manifest the vendor publishes rather than downloading the artifacts — see `pkgbuilds/openai-codex-desktop/.omarchy/upstream.sh`, which reads OpenAI's Debian package index and never fetches the 750 MB of debs -it describes. +it describes. Hooks honoring `min_release_age` receive the window as +`MIN_RELEASE_AGE_SECONDS` and report `published_at` alongside `pkgver`. + +`bin/sync-upstream self-test` runs offline fixture tests over the release +selection, quarantine backstop, duration parsing, and manifest validation. ### Sync Rebuild Triggers @@ -462,7 +499,9 @@ Minimal examples: Fields: -- `source`: `aur` or `local`. A `local` package can still follow an upstream release with an `.omarchy/upstream.sh` hook. +- `source`: `aur` or `local`. A `local` package can still follow an upstream release, either declaratively via `upstream` or with an `.omarchy/upstream.sh` hook. +- `upstream`: optional for `local` packages whose vendor ships tagged GitHub releases with a checksum manifest asset. `{ "github": "owner/repo", "checksums": "SHASUMS256.txt", "assets": { "": "name-{tag}.tar.xz" } }` — see [Sync Upstream Releases](#sync-upstream-releases). Mutually exclusive with `.omarchy/upstream.sh`. +- `min_release_age`: optional quarantine for upstream releases (`"24h"`, `"2d"`, or bare seconds). The newest release older than the window ships; anything younger waits, and a release whose age cannot be proven fails the sync. Bypass deliberately with `BYPASS_MIN_RELEASE_AGE=1 bin/sync-upstream `. - `sync`: optional for AUR packages; defaults to `true`. Set `false` for AUR-origin packages that Omarchy maintains manually. - `aur`: optional AUR package name when it differs from the local package directory, usually for split packages. - `release_ring`: optional. `fast` means the package is built directly for stable as well as edge. Packages without a ring build in edge and reach stable through tested artifact promotion (`bin/repo migrate`). diff --git a/bin/sync-upstream b/bin/sync-upstream index 93e6f0d..a0da5f3 100755 --- a/bin/sync-upstream +++ b/bin/sync-upstream @@ -5,6 +5,7 @@ BUILD_ROOT=$(realpath "${BASH_SOURCE[0]%/*}/..") source "$BUILD_ROOT/helpers/message-helpers.sh" source "$BUILD_ROOT/helpers/paths.sh" source "$BUILD_ROOT/helpers/package-metadata.sh" +source "$BUILD_ROOT/helpers/upstream-github.sh" TEMP_DIR=$(mktemp -d) trap 'rm -rf "$TEMP_DIR"' EXIT @@ -17,7 +18,10 @@ Usage: $0 [PACKAGE...] Update packages that track an upstream vendor release feed instead of the AUR. -A package opts in by providing pkgbuilds//.omarchy/upstream.sh, a hook +A package whose upstream ships tagged GitHub releases with a checksum manifest +opts in declaratively, via "upstream" in .omarchy/package.json (see +helpers/upstream-github.sh for the schema); no code needed. Anything with a +bespoke feed provides pkgbuilds//.omarchy/upstream.sh instead, a hook that reports the newest upstream release as JSON on stdout: { @@ -31,11 +35,25 @@ the unsuffixed sha256sums array. An empty object ({}) reports no update. When the reported version is newer than the checked-in one, pkgver and the listed checksum arrays are rewritten and pkgrel is reset to 1. +A package may declare "min_release_age" in .omarchy/package.json ("24h", "2d", +or bare seconds) to quarantine fresh releases until maintainers have had time +to pull a bad or compromised one. The window is exported to the hook as +MIN_RELEASE_AGE_SECONDS so it can select the newest release that has already +cleared it, and enforced here as a backstop: the hook must then report +"published_at" (ISO 8601), and a release younger than the window is treated +as no update. A maintainer shipping an emergency update inside the window +runs: BYPASS_MIN_RELEASE_AGE=1 $0 . Scheduled automation never sets +the bypass, so the resulting change still goes through a reviewed PR. + Arguments: PACKAGE One or more package names to update (optional) +Commands: + self-test Run the offline fixture tests for release selection, the + quarantine backstop, and metadata parsing + Examples: - $0 # Update every package with an upstream hook + $0 # Update every package with an upstream source $0 openai-codex-desktop # Update specific packages EOF } @@ -143,6 +161,26 @@ set_pkgbuild_array() { mv "$rewritten" "$pkgbuild" } +# Backstop verdict for a reported release against min_release_age. Returns 0 +# when old enough (or no policy is set, or the bypass is deliberate), 1 when +# the release is younger than the window, 2 when the report carries no usable +# published_at and the age cannot be established at all. +release_age_status() { + local release="$1" min_age="$2" + (( min_age > 0 )) || return 0 + [[ "${BYPASS_MIN_RELEASE_AGE:-}" != "1" ]] || return 0 + local published_at published_epoch + published_at=$(jq -r '.published_at // empty' <<<"$release") + # Strict ISO 8601 before GNU date sees it: date also accepts relative + # expressions like "2 days ago", which would let a buggy hook fabricate an + # age instead of failing closed. + if [[ ! "$published_at" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?(Z|[+-][0-9]{2}:?[0-9]{2})$ ]] \ + || ! published_epoch=$(date --date="$published_at" +%s 2>/dev/null); then + return 2 + fi + (( $(date +%s) - published_epoch >= min_age )) || return 1 +} + # pacman's own comparator, because nothing else agrees with it at the corners: # sort -V calls 1.0a newer than 1.0, vercmp calls it older, and pacman is what # decides whether a published package is an upgrade. @@ -170,6 +208,7 @@ validate_release() { and (.sha256sums | to_entries | all( .value | type == "array" and length > 0 and all(test("\\A[0-9a-f]{64}\\z")) )) + and (if has("published_at") then (.published_at | type == "string" and length > 0) else true end) ' <<<"$release" >/dev/null } @@ -293,21 +332,57 @@ sync_package() { return 0 fi - if [[ ! -f "$hook" ]]; then + local github_repo has_upstream=false + github_repo=$(package_upstream_github_repo "$package_dir") + if package_has_upstream_provider "$package_dir"; then + has_upstream=true + fi + + # A present-but-unusable declaration fails loudly; treating it like "no + # upstream source" would silently drop the package from scheduled runs. + if [[ "$has_upstream" == true && -z "$github_repo" ]]; then + print_error "Package $package has an unusable upstream declaration (needs a github owner/repo)" + ((++FAILED)) + return 0 + fi + + if [[ -n "$github_repo" && -f "$hook" ]]; then + print_error "Package $package declares both upstream.github and an upstream.sh hook; keep exactly one" + ((++FAILED)) + return 0 + fi + + if [[ -z "$github_repo" && ! -f "$hook" ]]; then if [[ "$SPECIFIC_MODE" == true ]]; then - print_error "Package $package is missing .omarchy/upstream.sh" + print_error "Package $package has neither an upstream declaration nor .omarchy/upstream.sh" ((++FAILED)) else - print_info "Skipping $package: no upstream hook" + print_info "Skipping $package: no upstream source" ((++SKIPPED)) fi return 0 fi + local min_age + if ! min_age=$(package_min_release_age_seconds "$package_dir"); then + print_error "Invalid min_release_age in $package_dir/.omarchy/package.json" + ((++FAILED)) + return 0 + fi + print_info "Checking $package for upstream releases..." local release - if ! release=$(cd "$package_dir" && PACKAGE_NAME="$package" bash .omarchy/upstream.sh); then + if [[ -n "$github_repo" ]]; then + if ! release=$(github_upstream_release "$package_dir" "$min_age"); then + print_error "GitHub release provider failed for $package" + ((++FAILED)) + return 0 + fi + elif ! release=$(cd "$package_dir" && PACKAGE_NAME="$package" \ + MIN_RELEASE_AGE_SECONDS="$min_age" \ + BYPASS_MIN_RELEASE_AGE="${BYPASS_MIN_RELEASE_AGE:-}" \ + bash .omarchy/upstream.sh); then print_error "Upstream hook failed for $package" ((++FAILED)) return 0 @@ -331,6 +406,24 @@ sync_package() { return 0 fi + # Backstop for min_release_age: the selection already honors the window, + # but a provider or hook bug must not be able to ship a release younger + # than the policy. + local age_status=0 + release_age_status "$release" "$min_age" || age_status=$? + case "$age_status" in + 1) + print_warning " Reported release is inside the ${min_age}s minimum release age; leaving it alone" + ((++SKIPPED)) + return 0 + ;; + 2) + print_error "min_release_age is set for $package but its source reported no usable published_at; refusing an unverifiable release" + ((++FAILED)) + return 0 + ;; + esac + local pkgver current_pkgver pkgver=$(jq -r '.pkgver' <<<"$release") current_pkgver=$(get_pkgver "$package_dir") @@ -363,6 +456,233 @@ sync_package() { ((++UPDATED)) } +# Offline fixture tests: the network fetches in helpers/upstream-github.sh +# are swapped for fixture readers, everything else runs the production code +# paths. Covers release selection (fallback past quarantined releases, +# draft/prerelease filtering, bypass, unchanged version), failure paths +# (unusable tags/timestamps, missing checksums), checksum template mapping +# for both architectures, the min_release_age backstop, the duration parser, +# and manifest validation. +cmd_self_test() { + local failures=0 + + check() { + local desc="$1" expected="$2" got="$3" + if [[ "$expected" == "$got" ]]; then + echo " ok: $desc" + else + echo " FAIL: $desc (expected '$expected', got '$got')" + failures=$((failures + 1)) + fi + } + + local pkg="$TEMP_DIR/selftest-pkg" + mkdir -p "$pkg/.omarchy" + printf 'pkgver=1.0.0\npkgrel=1\n' > "$pkg/PKGBUILD" + cat > "$pkg/.omarchy/package.json" <<'EOF' +{ + "source": "local", + "min_release_age": "24h", + "upstream": { + "github": "example/tool", + "checksums": "SHASUMS256.txt", + "assets": { + "x86_64": "tool-{tag}-x64.tar.xz", + "aarch64": "tool-v{pkgver}-arm64.tar.xz" + } + } +} +EOF + + local young old2d old3d + young=$(date -u -d '1 hour ago' +%Y-%m-%dT%H:%M:%SZ) + old2d=$(date -u -d '2 days ago' +%Y-%m-%dT%H:%M:%SZ) + old3d=$(date -u -d '3 days ago' +%Y-%m-%dT%H:%M:%SZ) + + # v2.0.0 is inside the 24h window; v1.9.9/v1.9.8 are a prerelease and a + # draft that would outrank v1.9.0 if the filters failed. + local sum_x19 sum_a19 sum_x20 sum_a20 + sum_x19=$(printf 'a%.0s' {1..64}) + sum_a19=$(printf 'b%.0s' {1..64}) + sum_x20=$(printf 'c%.0s' {1..64}) + sum_a20=$(printf 'd%.0s' {1..64}) + + FIXTURE_RELEASES=$(jq -n --arg young "$young" --arg old2 "$old2d" --arg old3 "$old3d" '[ + {tag_name: "v2.0.0", published_at: $young, draft: false, prerelease: false}, + {tag_name: "v1.9.9", published_at: $old2, draft: false, prerelease: true}, + {tag_name: "v1.9.8", published_at: $old2, draft: true, prerelease: false}, + {tag_name: "v1.9.0", published_at: $old2, draft: false, prerelease: false}, + {tag_name: "v1.8.0", published_at: $old3, draft: false, prerelease: false} + ]') + FIXTURE_CHECKSUMS=$(printf '%s\n' \ + "$sum_x19 ./tool-v1.9.0-x64.tar.xz" \ + "$sum_a19 tool-v1.9.0-arm64.tar.xz" \ + "$sum_x20 *tool-v2.0.0-x64.tar.xz" \ + "$sum_a20 tool-v2.0.0-arm64.tar.xz") + + github_fetch_releases() { printf '%s' "$FIXTURE_RELEASES"; } + github_fetch_checksums() { printf '%s\n' "$FIXTURE_CHECKSUMS"; } + + echo "Release selection:" + local out + out=$(github_upstream_release "$pkg" 86400 2>/dev/null) || out="" + check "quarantine falls back past the young v2.0.0" "1.9.0" "$(jq -r '.pkgver // ""' <<<"$out")" + check "selected release reports its published_at" "$old2d" "$(jq -r '.published_at // ""' <<<"$out")" + check "x86_64 checksum via {tag} template and ./ prefix" "$sum_x19" "$(jq -r '.sha256sums.x86_64[0] // ""' <<<"$out")" + check "aarch64 checksum via {pkgver} template" "$sum_a19" "$(jq -r '.sha256sums.aarch64[0] // ""' <<<"$out")" + + out=$(github_upstream_release "$pkg" 0 2>/dev/null) || out="" + check "no policy selects the newest stable release" "2.0.0" "$(jq -r '.pkgver // ""' <<<"$out")" + check "prerelease v1.9.9 and draft v1.9.8 are never selected" "" "$(jq -r 'select(.pkgver == "1.9.9" or .pkgver == "1.9.8") | .pkgver' <<<"$out")" + + out=$(BYPASS_MIN_RELEASE_AGE=1 github_upstream_release "$pkg" 86400 2>/dev/null) || out="" + check "bypass lifts the quarantine" "2.0.0" "$(jq -r '.pkgver // ""' <<<"$out")" + check "x86_64 checksum via * binary-mode prefix" "$sum_x20" "$(jq -r '.sha256sums.x86_64[0] // ""' <<<"$out")" + + out=$(github_upstream_release "$pkg" 8640000 2>/dev/null) || out="" + check "everything quarantined reports no update" "{}" "$(jq -c . <<<"$out")" + + printf 'pkgver=1.9.0\npkgrel=1\n' > "$pkg/PKGBUILD" + out=$(github_upstream_release "$pkg" 86400 2>/dev/null) || out="" + check "already checked in reports no update" "{}" "$(jq -c . <<<"$out")" + printf 'pkgver=1.0.0\npkgrel=1\n' > "$pkg/PKGBUILD" + + echo "Failure paths:" + local rc + FIXTURE_RELEASES=$(jq -n '[{tag_name: "v1.9.0", published_at: "not-a-date", draft: false, prerelease: false}]') + rc=0; github_upstream_release "$pkg" 86400 >/dev/null 2>&1 || rc=$? + check "invalid published_at fails the sync" "1" "$rc" + + FIXTURE_RELEASES=$(jq -n --arg old "$old2d" '[{tag_name: "release 1.9!", published_at: $old, draft: false, prerelease: false}]') + rc=0; github_upstream_release "$pkg" 86400 >/dev/null 2>&1 || rc=$? + check "unusable tag fails the sync" "1" "$rc" + + FIXTURE_RELEASES=$(jq -n --arg old "$old2d" '[{tag_name: "v1.9.0", published_at: $old, draft: false, prerelease: false}]') + FIXTURE_CHECKSUMS="$sum_x19 ./tool-v1.9.0-x64.tar.xz" + rc=0; github_upstream_release "$pkg" 86400 >/dev/null 2>&1 || rc=$? + check "missing aarch64 checksum fails the sync" "1" "$rc" + + echo "Quarantine backstop:" + local rel st + rel=$(jq -n --arg p "$old2d" '{pkgver: "1.9.0", published_at: $p, sha256sums: {}}') + st=0; release_age_status "$rel" 86400 || st=$? + check "old enough passes" "0" "$st" + rel=$(jq -n --arg p "$young" '{pkgver: "2.0.0", published_at: $p, sha256sums: {}}') + st=0; release_age_status "$rel" 86400 || st=$? + check "too young is held" "1" "$st" + st=0; release_age_status "$rel" 0 || st=$? + check "no policy passes anything" "0" "$st" + st=0; BYPASS_MIN_RELEASE_AGE=1 release_age_status "$rel" 86400 || st=$? + check "deliberate bypass passes" "0" "$st" + rel=$(jq -n '{pkgver: "2.0.0", sha256sums: {}}') + st=0; release_age_status "$rel" 86400 || st=$? + check "missing published_at is unprovable" "2" "$st" + rel=$(jq -n '{pkgver: "2.0.0", published_at: "2 days ago", sha256sums: {}}') + st=0; release_age_status "$rel" 86400 || st=$? + check "relative-date expression is unprovable, not an age" "2" "$st" + rel=$(jq -n --arg p "$(date -u -d '2 days ago' +%Y-%m-%dT%H:%M:%S+00:00)" '{pkgver: "1.9.0", published_at: $p, sha256sums: {}}') + st=0; release_age_status "$rel" 86400 || st=$? + check "numeric-offset ISO timestamp passes" "0" "$st" + + echo "Duration parser:" + local agepkg="$TEMP_DIR/selftest-age" + mkdir -p "$agepkg/.omarchy" + check_age() { + local json_value="$1" expected="$2" got + jq -n "{source: \"local\", min_release_age: $json_value}" > "$agepkg/.omarchy/package.json" + got=$(package_min_release_age_seconds "$agepkg") || got="" + check "min_release_age $json_value" "$expected" "$got" + } + check_age '"24h"' 86400 + check_age '"90m"' 5400 + check_age '"2d"' 172800 + check_age '3600' 3600 + check_age '"600s"' 600 + check_age '"010h"' 36000 + check_age '"abc"' "" + check_age '"24hh"' "" + check_age 'false' "" + check_age '""' "" + check_age '"9999999999"' "" + + echo "Manifest validation:" + printf 'pkgver=1.0.0\n' > "$agepkg/PKGBUILD" + local vst + echo '{"source": "local", "upstream": false}' > "$agepkg/.omarchy/package.json" + vst=0; validate_package_metadata "$agepkg" >/dev/null || vst=$? + check "upstream: false is rejected" "1" "$vst" + echo '{"source": "local", "upstream": {"github": "example/tool"}}' > "$agepkg/.omarchy/package.json" + vst=0; validate_package_metadata "$agepkg" >/dev/null || vst=$? + check "upstream without checksums/assets is rejected" "1" "$vst" + cp "$pkg/.omarchy/package.json" "$agepkg/.omarchy/package.json" + vst=0; validate_package_metadata "$agepkg" >/dev/null || vst=$? + check "the real declaration shape is accepted" "0" "$vst" + + # End to end over the real mise-bin package: its checked-in metadata and + # PKGBUILD, the full sync_package path (selection, validation, backstop, + # rewrite, read-back verification), with only the two network fetches + # replaced by mise-shaped fixtures. + echo "End-to-end sync_package with the checked-in mise-bin metadata:" + local e2e_root="$TEMP_DIR/e2e-pkgbuilds" + mkdir -p "$e2e_root" + cp -a "$BUILD_ROOT/pkgbuilds/mise-bin" "$e2e_root/mise-bin" + + # Fixture versions extend the checked-in pkgver so they stay newer no + # matter what version the real package is at when the test runs. + local mise_current mise_aged mise_fresh mise_x64 mise_a64 + mise_current=$(grep -m1 '^pkgver=' "$e2e_root/mise-bin/PKGBUILD" | cut -d= -f2- | tr -d "\"'") + mise_aged="${mise_current}.90" + mise_fresh="${mise_current}.91" + mise_x64=$(printf 'e%.0s' {1..64}) + mise_a64=$(printf 'f%.0s' {1..64}) + FIXTURE_RELEASES=$(jq -n --arg young "$young" --arg old2 "$old2d" \ + --arg aged "v$mise_aged" --arg fresh "v$mise_fresh" '[ + {tag_name: $fresh, published_at: $young, draft: false, prerelease: false}, + {tag_name: $aged, published_at: $old2, draft: false, prerelease: false} + ]') + FIXTURE_CHECKSUMS=$(printf '%s\n' \ + "$mise_x64 ./mise-v$mise_aged-linux-x64.tar.xz" \ + "$mise_a64 ./mise-v$mise_aged-linux-arm64.tar.xz") + + local prev_updated=$UPDATED prev_failed=$FAILED + PKGBUILDS_DIR="$e2e_root" sync_package mise-bin >/dev/null 2>&1 || true + check "sync_package updates without failures" "updated=1 failed=0" \ + "updated=$((UPDATED - prev_updated)) failed=$((FAILED - prev_failed))" + check "the 24h manifest policy holds v$mise_fresh and ships v$mise_aged" "$mise_aged" \ + "$(grep -m1 '^pkgver=' "$e2e_root/mise-bin/PKGBUILD" | cut -d= -f2-)" + check "pkgrel resets to 1" "1" \ + "$(grep -m1 '^pkgrel=' "$e2e_root/mise-bin/PKGBUILD" | cut -d= -f2-)" + check "x86_64 checksum lands in the PKGBUILD" "sha256sums_x86_64=('$mise_x64')" \ + "$(grep -m1 '^sha256sums_x86_64=' "$e2e_root/mise-bin/PKGBUILD")" + check "aarch64 checksum lands in the PKGBUILD" "sha256sums_aarch64=('$mise_a64')" \ + "$(grep -m1 '^sha256sums_aarch64=' "$e2e_root/mise-bin/PKGBUILD")" + + # A malformed declaration must fail the run loudly, and still be discovered. + local badpkg="$e2e_root/selftest-broken" + mkdir -p "$badpkg/.omarchy" + printf 'pkgver=1.0.0\npkgrel=1\n' > "$badpkg/PKGBUILD" + echo '{"source": "local", "upstream": false}' > "$badpkg/.omarchy/package.json" + check "malformed upstream stays discoverable for scheduled runs" "yes" \ + "$(PKGBUILDS_DIR="$e2e_root" packages_for_upstream_sync | grep -qx selftest-broken && echo yes || echo no)" + prev_failed=$FAILED + PKGBUILDS_DIR="$e2e_root" sync_package selftest-broken >/dev/null 2>&1 || true + check "malformed upstream fails the sync instead of skipping" "1" "$((FAILED - prev_failed))" + FAILED=0 + + echo "" + if [[ "$failures" -eq 0 ]]; then + print_success "Self-test passed" + else + print_error "$failures self-test failure(s)" + exit 1 + fi +} + +if [[ ${#SPECIFIC_PACKAGES[@]} -gt 0 && "${SPECIFIC_PACKAGES[0]}" == "self-test" ]]; then + cmd_self_test + exit 0 +fi + if [[ ${#SPECIFIC_PACKAGES[@]} -gt 0 ]]; then SPECIFIC_MODE=true for package in "${SPECIFIC_PACKAGES[@]}"; do diff --git a/helpers/package-metadata.sh b/helpers/package-metadata.sh index 83ed37a..f73cc19 100644 --- a/helpers/package-metadata.sh +++ b/helpers/package-metadata.sh @@ -12,6 +12,8 @@ # { "source": "aur", "pkgrel": { "suffix": 1, "offset": 1 } } # { "source": "aur", "rebuild_on": ["qt6-base"] } # { "source": "local" } +# { "source": "local", "min_release_age": "24h" } +# { "source": "local", "upstream": { "github": "owner/repo", "checksums": "SHASUMS256.txt", "assets": { "x86_64": "name-{tag}-x64.tar.xz" } } } # # bin/sync-aur also writes upstream_commit for AUR-backed packages, and # bin/sync-rebuilds writes rebuilt_against for packages declaring rebuild_on. @@ -78,6 +80,43 @@ package_is_fast_ring() { [[ "$(package_release_ring "$pkgdir")" == "fast" ]] } +# Quarantine window for upstream releases, in seconds. Accepts a bare number +# of seconds or a number suffixed s/m/h/d ("24h", "2d"). Unset means 0 (no +# hold); an unparseable value -- including a non-string/non-number JSON type +# like false -- returns 1 so callers fail closed instead of silently dropping +# the hold. At most 9 digits: enough for three decades in seconds, and small +# enough that no suffix multiplication can overflow 64-bit arithmetic. +package_min_release_age_seconds() { + local pkgdir="$1" metadata raw + metadata=$(metadata_file_for_dir "$pkgdir") + if [[ ! -f "$metadata" ]]; then + echo 0 + return 0 + fi + # A present-but-empty value maps to "unparseable", not to "absent": only a + # missing key means no hold, so '"min_release_age": ""' cannot silently + # disable the quarantine. + raw=$(jq -r ' + if has("min_release_age") | not then "" + elif (.min_release_age | type) == "string" or (.min_release_age | type) == "number" then + .min_release_age | tostring | if . == "" then "unparseable" else . end + else "unparseable" end + ' "$metadata") + if [[ -z "$raw" ]]; then + echo 0 + return 0 + fi + [[ "$raw" =~ ^([0-9]{1,9})([smhd]?)$ ]] || return 1 + # Forced base 10: bash arithmetic would otherwise read "010" as octal. + local n=$((10#${BASH_REMATCH[1]})) + case "${BASH_REMATCH[2]}" in + ""|s) echo "$n" ;; + m) echo $((n * 60)) ;; + h) echo $((n * 3600)) ;; + d) echo $((n * 86400)) ;; + esac +} + package_build_skipped() { local pkgdir="$1" local metadata skip_build @@ -142,9 +181,18 @@ package_has_upstream_hook() { [[ -f "$pkgdir/.omarchy/upstream.sh" ]] } +package_has_upstream_provider() { + local pkgdir="$1" metadata + metadata=$(metadata_file_for_dir "$pkgdir") + [[ -f "$metadata" ]] || return 1 + # Any upstream key counts, valid or not: a malformed declaration must reach + # bin/sync-upstream and fail loudly there, not vanish from discovery. + jq -e 'has("upstream")' "$metadata" >/dev/null +} + packages_for_upstream_sync() { package_dirs | while IFS= read -r pkgdir; do - if package_has_upstream_hook "$pkgdir"; then + if package_has_upstream_hook "$pkgdir" || package_has_upstream_provider "$pkgdir"; then basename "$pkgdir" fi done @@ -313,6 +361,28 @@ validate_package_metadata() { *) echo "invalid release_ring for $(basename "$pkgdir"): $ring"; return 1 ;; esac + if ! package_min_release_age_seconds "$pkgdir" >/dev/null; then + echo "invalid min_release_age for $(basename "$pkgdir"): must be a number with optional s/m/h/d suffix" + return 1 + fi + + # `has` rather than `// {}`: jq's // treats false as absent, which would + # let "upstream": false slip through as an empty declaration. + if ! jq -e ' + if has("upstream") | not then true + elif (.upstream | type) != "object" then false + else .upstream | + ((.github // "") | type == "string" and test("\\A[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\\z")) + and ((.checksums // "") | type == "string" and length > 0) + and ((.assets // {}) | type == "object" and length > 0 and (to_entries | all( + (.key | test("\\A[a-z0-9_]+\\z")) and (.value | type == "string" and length > 0) + ))) + end + ' "$metadata" >/dev/null; then + echo "invalid upstream for $(basename "$pkgdir"): needs github owner/repo, checksums asset name, and an assets arch->name map" + return 1 + fi + pkgrel_type=$(jq -r 'if has("pkgrel") then .pkgrel | type else "missing" end' "$metadata") case "$pkgrel_type" in object|missing) ;; diff --git a/helpers/upstream-github.sh b/helpers/upstream-github.sh new file mode 100644 index 0000000..e37e6ab --- /dev/null +++ b/helpers/upstream-github.sh @@ -0,0 +1,161 @@ +# GitHub-releases upstream provider for bin/sync-upstream. +# +# A package whose upstream ships tagged GitHub releases with a checksum +# manifest asset needs no upstream.sh hook: the whole feed is data, declared +# in .omarchy/package.json -- +# +# "upstream": { +# "github": "jdx/mise", +# "checksums": "SHASUMS256.txt", +# "assets": { +# "x86_64": "mise-{tag}-linux-x64.tar.xz", +# "aarch64": "mise-{tag}-linux-arm64.tar.xz" +# } +# } +# +# {tag} and {pkgver} interpolate into asset names; tags may carry a leading +# "v", which is stripped for pkgver. Drafts and prereleases are ignored. The +# provider emits the same JSON contract as an upstream.sh hook, so +# bin/sync-upstream's validation and min_release_age backstop apply +# unchanged; a feed that fits no convention keeps a bespoke upstream.sh. + +package_upstream_github_repo() { + local pkgdir="$1" + # `objects` drops a non-object upstream value (validation rejects those + # separately) instead of erroring the jq pipeline. + package_metadata_value "$pkgdir" '(.upstream? | objects | .github)' "" +} + +# Fetches sit behind functions so the self-test can replace them with fixture +# readers; everything below the fetch is deterministic and testable offline. +# Only the 100 most recent releases are considered -- a bounded search, not +# pagination. Quarantined releases report no update and wait for the next +# run; a page with no stable release at all (drafts and prereleases only) +# fails the sync instead, because a provider-tracked feed suddenly shipping +# nothing stable is an anomaly worth a loud error, not a silent skip. +github_fetch_releases() { + local repo="$1" + curl -fsSL "https://api.github.com/repos/$repo/releases?per_page=100" +} + +github_fetch_checksums() { + local repo="$1" tag="$2" asset="$3" + curl -fsSL "https://github.com/$repo/releases/download/$tag/$asset" +} + +# Emits the newest qualifying release as hook-contract JSON. min_release_age +# is honored during selection (newest release older than the window wins, +# even when a younger one exists) and BYPASS_MIN_RELEASE_AGE=1 lifts it. +# Unusable tags or timestamps anywhere in the feed fail the sync rather than +# being skipped: a feed this provider cannot fully read is a feed it should +# not silently choose from. +github_upstream_release() { + local package_dir="$1" min_age="${2:-0}" + local metadata repo checksums_name + metadata=$(metadata_file_for_dir "$package_dir") + + repo=$(jq -r '(.upstream? | objects | .github) // ""' "$metadata") + if [[ ! "$repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then + echo "invalid upstream.github repository: '${repo:-}'" >&2 + return 1 + fi + checksums_name=$(jq -r '(.upstream? | objects | .checksums) // ""' "$metadata") + if [[ -z "$checksums_name" ]]; then + echo "upstream.checksums names the checksum manifest asset and is required" >&2 + return 1 + fi + local arches + mapfile -t arches < <(jq -r '(.upstream? | objects | .assets) // {} | keys[]' "$metadata") + if [[ ${#arches[@]} -eq 0 ]]; then + echo "upstream.assets must map at least one architecture to an asset name" >&2 + return 1 + fi + + local releases now + now=$(date +%s) + if ! releases=$(github_fetch_releases "$repo"); then + echo "could not fetch the release feed for $repo" >&2 + return 1 + fi + + local candidates=0 best_tag="" best_pkgver="" best_published_at="" + local tag published_at pkgver published_epoch + while IFS=$'\t' read -r tag published_at; do + if [[ ! "$tag" =~ ^v?([A-Za-z0-9._+]+)$ ]]; then + echo "$repo release has an unusable tag: ${tag:-}" >&2 + return 1 + fi + pkgver=${BASH_REMATCH[1]} + + if [[ -z "$published_at" ]] || ! published_epoch=$(date --date="$published_at" +%s 2>/dev/null); then + echo "$repo release $tag has an invalid published_at: ${published_at:-}" >&2 + return 1 + fi + candidates=$((candidates + 1)) + + if (( now - published_epoch < min_age )); then + if [[ "${BYPASS_MIN_RELEASE_AGE:-}" == "1" ]]; then + echo "Bypassing release-age gate for $repo $tag" >&2 + else + continue + fi + fi + + if [[ -z "$best_pkgver" ]] || [[ "$(vercmp "$pkgver" "$best_pkgver")" -gt 0 ]]; then + best_tag=$tag + best_pkgver=$pkgver + best_published_at=$published_at + fi + # `// ""` rather than `// empty`: empty would drop the field and shift the + # columns, so a malformed row could masquerade as a different one instead + # of tripping the per-field checks above. + done < <(jq -r '.[] | select((.draft or .prerelease) | not) | [.tag_name // "", .published_at // ""] | @tsv' <<<"$releases") + + if (( candidates == 0 )); then + echo "no stable releases found in the feed for $repo" >&2 + return 1 + fi + if [[ -z "$best_tag" ]]; then + echo "every recent $repo release is still inside the release-age quarantine; skipping" >&2 + echo '{}' + return 0 + fi + + # Already checked in: report no update instead of re-fetching checksums. + local current_pkgver + current_pkgver=$(grep -m1 '^pkgver=' "$package_dir/PKGBUILD" | cut -d= -f2- | tr -d "\"'") + if [[ "$best_pkgver" == "$current_pkgver" ]]; then + echo '{}' + return 0 + fi + + local checksums + if ! checksums=$(github_fetch_checksums "$repo" "$best_tag" "$checksums_name"); then + echo "could not fetch $checksums_name for $repo $best_tag" >&2 + return 1 + fi + + local jq_args=(--arg pkgver "$best_pkgver" --arg published_at "$best_published_at") + local jq_filter='{pkgver: $pkgver, published_at: $published_at, sha256sums: {}}' + local arch template filename checksum + for arch in "${arches[@]}"; do + if [[ ! "$arch" =~ ^[a-z0-9_]+$ ]]; then + echo "invalid architecture key in upstream.assets: '$arch'" >&2 + return 1 + fi + template=$(jq -r --arg arch "$arch" '.upstream.assets[$arch]' "$metadata") + filename=${template//\{pkgver\}/$best_pkgver} + filename=${filename//\{tag\}/$best_tag} + # Manifest lines are " ", with the name sometimes prefixed + # "./" (sha256sum of a local path) or "*" (binary-mode marker). + checksum=$(awk -v f="$filename" '$2 == f || $2 == "./" f || $2 == "*" f { print $1; exit }' <<<"$checksums") + if [[ ! "$checksum" =~ ^[0-9a-f]{64}$ ]]; then + echo "no valid checksum for $filename in $repo $best_tag $checksums_name" >&2 + return 1 + fi + jq_args+=(--arg "sum_$arch" "$checksum") + jq_filter+=" | .sha256sums[\"$arch\"] = [\$sum_$arch]" + done + + jq -n "${jq_args[@]}" "$jq_filter" +} diff --git a/pkgbuilds/mise-bin/.omarchy/package.json b/pkgbuilds/mise-bin/.omarchy/package.json index db153c3..f90090e 100644 --- a/pkgbuilds/mise-bin/.omarchy/package.json +++ b/pkgbuilds/mise-bin/.omarchy/package.json @@ -1,4 +1,13 @@ { "source": "local", - "release_ring": "fast" + "release_ring": "fast", + "min_release_age": "24h", + "upstream": { + "github": "jdx/mise", + "checksums": "SHASUMS256.txt", + "assets": { + "x86_64": "mise-{tag}-linux-x64.tar.xz", + "aarch64": "mise-{tag}-linux-arm64.tar.xz" + } + } } diff --git a/pkgbuilds/mise-bin/.omarchy/upstream.sh b/pkgbuilds/mise-bin/.omarchy/upstream.sh deleted file mode 100644 index 365063d..0000000 --- a/pkgbuilds/mise-bin/.omarchy/upstream.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -set -euo pipefail - -repo="jdx/mise" -release=$(curl -fsSL "https://api.github.com/repos/$repo/releases/latest") -tag=$(jq -r '.tag_name // empty' <<<"$release") -published_at=$(jq -r '.published_at // empty' <<<"$release") - -if [[ ! "$tag" =~ ^v([A-Za-z0-9._+]+)$ ]]; then - echo "Latest mise release has an invalid tag: ${tag:-}" >&2 - exit 1 -fi - -if [[ -z "$published_at" ]] || ! published_epoch=$(date --date="$published_at" +%s); then - echo "Latest mise release has an invalid published_at: ${published_at:-}" >&2 - exit 1 -fi - -# Keep a compromised mise release from reaching Omarchy before there has been -# a full day for maintainers and the community to notice and pull it. -minimum_release_age_seconds=$((24 * 60 * 60)) -now=$(date +%s) -if (( now - published_epoch < minimum_release_age_seconds )); then - if [[ "${MISE_BIN_BYPASS_RELEASE_AGE:-}" == "1" ]]; then - echo "Bypassing mise release-age gate for $tag" >&2 - else - echo '{}' - exit 0 - fi -fi - -pkgver=${BASH_REMATCH[1]} -checksums=$(curl -fsSL \ - "https://github.com/$repo/releases/download/$tag/SHASUMS256.txt") - -checksum_for() { - local filename=$1 - local checksum - checksum=$(awk -v filename="./$filename" '$2 == filename { print $1 }' <<<"$checksums") - - if [[ ! "$checksum" =~ ^[0-9a-f]{64}$ ]]; then - echo "No valid checksum found for $filename in $tag" >&2 - exit 1 - fi - - echo "$checksum" -} - -x86_64=$(checksum_for "mise-v${pkgver}-linux-x64.tar.xz") -aarch64=$(checksum_for "mise-v${pkgver}-linux-arm64.tar.xz") - -jq -n \ - --arg pkgver "$pkgver" \ - --arg x86_64 "$x86_64" \ - --arg aarch64 "$aarch64" \ - '{pkgver: $pkgver, sha256sums: {x86_64: [$x86_64], aarch64: [$aarch64]}}'