upstream-watch: probe redirect watches past a staged rollout

The dropbox watch failed with "no matching releases" on some runs
(e.g. actions run 37236931512). It is not the network: Dropbox's download
redirect sends a share of requests to a newer build the pattern rejects
on purpose. On 2026-10-04, 96 of 100 HEADs ended at
dropbox-lnx.x86_64-272.4.3798 and 4 at 274.3.4801, which is not an x.4.y
stable build. The redirect provider made one probe, so one unlucky
answer failed the whole watch.

It now probes up to five times and keeps the first final URL that
matches, and curl gets --retry 2 as Fetcher.file already has. Live,
100 discovers against Dropbox all found 272.4.3798 in 103 probes. Two
tests cover a rollout answer before the stable one and every probe
missing.
This commit is contained in:
Emir Beganovic committed 2026-10-05 00:06:22 +02:00
1 parent 075f20f401
commit 811a149728
2 files changed
+30 -2

No files matched your search

+12 -2
View File
@@ -26,6 +26,8 @@ VERSION = re.compile(r"[A-Za-z0-9][A-Za-z0-9._+]*\Z")
SCALAR = re.compile(r"[A-Za-z0-9._+/-]+\Z")
SUM = re.compile(r"(md5|sha1|sha224|sha256|sha384|sha512|b2)sums(_[a-z0-9_]+)?\Z")
HASHES = {"b2": "blake2b"}
# How many times a redirect watch probes before it calls the feed unmatched.
REDIRECT_PROBES = 5
def run(args, **kwargs):
@@ -286,8 +288,16 @@ def discover(watch, fetch):
values.update({name: json_path(data, path) for name, path in watch.get("fields", {}).items()})
results.append(candidate(watch, values))
elif provider == "redirect":
final_url = run(["curl", "--proto", "=https", "--proto-redir", "=https", "-fsSLI", "--max-time", "60", "-o", "/dev/null", "-w", "%{url_effective}", feed], text=True)
results.extend(matches(watch, final_url))
# A download redirect can be a staged rollout: some requests land on a
# newer build the pattern deliberately rejects (Dropbox, 2026-10-04:
# 4 of 100 HEADs ended at 274.3.4801 instead of 272.4.3798), so one
# probe that misses is not an answer. Keep the first one that matches.
for _ in range(REDIRECT_PROBES):
final_url = run(["curl", "--proto", "=https", "--proto-redir", "=https", "-fsSLI", "--max-time", "60", "--retry", "2",
"-o", "/dev/null", "-w", "%{url_effective}", feed], text=True)
results.extend(matches(watch, final_url))
if results:
break
elif provider == "regex":
text = fetch.text(feed)
if watch.get("unescape_json"):
+18
View File
@@ -160,6 +160,24 @@ b2sums=('old' 'local-b2')
output = w.replace_array(text, 'sha256sums', ['abc'])
self.assertEqual(output, "sha256sums=('abc') # trailing\npackage() { :; }\n")
def test_redirect_probes_past_a_staged_rollout(self):
watch = {"redirect": "https://example.test/download",
"pattern": r"tool-(?P<version>[0-9]+\.4\.[0-9]+)\.tar\.gz"}
rollout = "https://cdn.example.test/tool-274.3.4801.tar.gz"
stable = "https://cdn.example.test/tool-272.4.3798.tar.gz"
with patch.object(w, 'run', side_effect=[rollout, rollout, stable]) as probe:
releases = w.discover(watch, self.fetch)
self.assertEqual([r["pkgver"] for r in releases], ["272.4.3798"])
self.assertEqual(probe.call_count, 3)
def test_redirect_gives_up_after_its_probes(self):
watch = {"redirect": "https://example.test/download",
"pattern": r"tool-(?P<version>[0-9]+\.4\.[0-9]+)\.tar\.gz"}
with patch.object(w, 'run', return_value="https://cdn.example.test/tool-274.3.4801.tar.gz") as probe, \
self.assertRaisesRegex(ValueError, 'no matching releases'):
w.discover(watch, self.fetch)
self.assertEqual(probe.call_count, w.REDIRECT_PROBES)
def test_git_hash_matches_makepkg(self):
repo = self.root / 'git'
repo.mkdir()