Run the timers every 5 minutes, with overlap and failure guards
A push reaching the mirror should take minutes, not up to six hours. All four units now fire every 5 minutes, staggered a minute apart. Three guards make that cadence safe: - Scheduled runs take the release lock NON-BLOCKING (try_release_lock) and skip the tick when a build is running. Blocking would stack one stalled process per tick behind a long build and stampede when it finished. Manual commands still wait, as an operator expects. - check-versions takes the lock too, and now owns its git pull (--pull, passed by the unit) instead of an ExecStartPre: at this cadence an unlocked pull would swap PKGBUILDs out from under a running build. - A failed release records .build-failed-<channel> and backs off exponentially (10m, 20m, 40m … capped at 6h) rather than rebuilding the same broken tree every 5 minutes. Any new commit clears the backoff, since a push is the most likely fix. Idle ticks exit without output so the journal keeps showing the runs that matter, and bin/repo timers reports backoff state — a paused channel is otherwise indistinguishable from an idle one.
This commit is contained in:
1 parent
9d86123264
commit
71e72e581b
10 files changed
+243
-57
No files matched your search
+56
-17
@@ -6,39 +6,78 @@
|
||||
#
|
||||
# The lock lives beside the published tree (REPO_ROOT), not the checkout, so
|
||||
# the primary checkout and the rc branch worktree contend on the same file.
|
||||
# Reentrant across child scripts: acquire_release_lock exports
|
||||
# OMARCHY_RELEASE_LOCK_HELD, and children skip acquisition when they see it
|
||||
# (the flock fd is inherited, so the lock stays held for the whole tree).
|
||||
# Reentrant across child scripts: acquiring exports OMARCHY_RELEASE_LOCK_HELD,
|
||||
# and children skip acquisition when they see it (the flock fd is inherited,
|
||||
# so the lock stays held for the whole tree).
|
||||
#
|
||||
# Two acquisition modes:
|
||||
# acquire_release_lock waits — for humans, who want the command to run
|
||||
# try_release_lock fails immediately — for timers, which must never
|
||||
# queue up behind a long build and stampede when it
|
||||
# finishes
|
||||
|
||||
RELEASE_LOCK_FD=9
|
||||
|
||||
release_lock_file() {
|
||||
echo "${REPO_ROOT:-$BUILD_ROOT/pkgs.omarchy.org}/.release.lock"
|
||||
}
|
||||
|
||||
release_lock_holder() {
|
||||
tail -1 "$(release_lock_file)" 2>/dev/null
|
||||
}
|
||||
|
||||
# True when the recorded holder is a live process. A holder line left behind by
|
||||
# a killed run describes nothing that is still running.
|
||||
release_lock_is_held() {
|
||||
local holder pid
|
||||
holder=$(release_lock_holder) || return 1
|
||||
[[ -n "$holder" ]] || return 1
|
||||
pid=$(sed -n 's/^pid \([0-9]\+\).*/\1/p' <<<"$holder")
|
||||
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null
|
||||
}
|
||||
|
||||
_release_lock_open() {
|
||||
local lock_file
|
||||
lock_file=$(release_lock_file)
|
||||
mkdir -p "$(dirname "$lock_file")"
|
||||
eval "exec $RELEASE_LOCK_FD>>\"\$lock_file\""
|
||||
}
|
||||
|
||||
_release_lock_record() {
|
||||
local lock_file
|
||||
lock_file=$(release_lock_file)
|
||||
# Truncate first so a crashed holder's stale line does not linger.
|
||||
: >"$lock_file"
|
||||
echo "pid $$ ($0) since $(date '+%Y-%m-%d %H:%M:%S')" >>"$lock_file"
|
||||
export OMARCHY_RELEASE_LOCK_HELD=1
|
||||
}
|
||||
|
||||
acquire_release_lock() {
|
||||
local timeout="${1:-3600}"
|
||||
|
||||
if [[ -n "${OMARCHY_RELEASE_LOCK_HELD:-}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
[[ -n "${OMARCHY_RELEASE_LOCK_HELD:-}" ]] && return 0
|
||||
|
||||
local lock_file="${REPO_ROOT:-$BUILD_ROOT/pkgs.omarchy.org}/.release.lock"
|
||||
mkdir -p "$(dirname "$lock_file")"
|
||||
|
||||
eval "exec $RELEASE_LOCK_FD>>\"\$lock_file\""
|
||||
_release_lock_open
|
||||
|
||||
if ! flock -n "$RELEASE_LOCK_FD"; then
|
||||
local holder
|
||||
holder=$(cat "$lock_file" 2>/dev/null | tail -1)
|
||||
holder=$(release_lock_holder)
|
||||
echo "Waiting for release lock (up to ${timeout}s)${holder:+ — held by: $holder}" >&2
|
||||
if ! flock -w "$timeout" "$RELEASE_LOCK_FD"; then
|
||||
echo "Could not acquire release lock within ${timeout}s: $lock_file" >&2
|
||||
echo "Could not acquire release lock within ${timeout}s: $(release_lock_file)" >&2
|
||||
echo "If no release is actually running, remove the file and retry." >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Record the holder for the "waiting for" message above. Truncate first so a
|
||||
# crashed holder's stale line does not linger once we own the lock.
|
||||
: >"$lock_file"
|
||||
echo "pid $$ ($0) since $(date '+%Y-%m-%d %H:%M:%S')" >>"$lock_file"
|
||||
_release_lock_record
|
||||
}
|
||||
|
||||
export OMARCHY_RELEASE_LOCK_HELD=1
|
||||
# Non-blocking. Returns 1 immediately when another run holds the lock, so
|
||||
# scheduled work can skip this tick instead of piling up.
|
||||
try_release_lock() {
|
||||
[[ -n "${OMARCHY_RELEASE_LOCK_HELD:-}" ]] && return 0
|
||||
_release_lock_open
|
||||
flock -n "$RELEASE_LOCK_FD" || return 1
|
||||
_release_lock_record
|
||||
}
|
||||
Reference in new issue
Block a user