From 811a149728b4765a27ed7ea3257748dfb20731a8 Mon Sep 17 00:00:00 2001 From: Emir Beganovic Date: Mon, 5 Oct 2026 00:01:23 +0200 Subject: [PATCH] upstream-watch: probe redirect watches past a staged rollout The dropbox watch failed with "no matching releases" on some runs (e.g. actions run 37236931512). It is not the network: Dropbox's download redirect sends a share of requests to a newer build the pattern rejects on purpose. On 2026-10-04, 96 of 100 HEADs ended at dropbox-lnx.x86_64-272.4.3798 and 4 at 274.3.4801, which is not an x.4.y stable build. The redirect provider made one probe, so one unlucky answer failed the whole watch. It now probes up to five times and keeps the first final URL that matches, and curl gets --retry 2 as Fetcher.file already has. Live, 100 discovers against Dropbox all found 272.4.3798 in 103 probes. Two tests cover a rollout answer before the stable one and every probe missing. --- helpers/upstream-watch.py | 14 ++++++++++++-- tests/upstream-watch.py | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/helpers/upstream-watch.py b/helpers/upstream-watch.py index f8468ab..7ab28a0 100644 --- a/helpers/upstream-watch.py +++ b/helpers/upstream-watch.py @@ -26,6 +26,8 @@ VERSION = re.compile(r"[A-Za-z0-9][A-Za-z0-9._+]*\Z") SCALAR = re.compile(r"[A-Za-z0-9._+/-]+\Z") SUM = re.compile(r"(md5|sha1|sha224|sha256|sha384|sha512|b2)sums(_[a-z0-9_]+)?\Z") HASHES = {"b2": "blake2b"} +# How many times a redirect watch probes before it calls the feed unmatched. +REDIRECT_PROBES = 5 def run(args, **kwargs): @@ -286,8 +288,16 @@ def discover(watch, fetch): values.update({name: json_path(data, path) for name, path in watch.get("fields", {}).items()}) results.append(candidate(watch, values)) elif provider == "redirect": - final_url = run(["curl", "--proto", "=https", "--proto-redir", "=https", "-fsSLI", "--max-time", "60", "-o", "/dev/null", "-w", "%{url_effective}", feed], text=True) - results.extend(matches(watch, final_url)) + # A download redirect can be a staged rollout: some requests land on a + # newer build the pattern deliberately rejects (Dropbox, 2026-10-04: + # 4 of 100 HEADs ended at 274.3.4801 instead of 272.4.3798), so one + # probe that misses is not an answer. Keep the first one that matches. + for _ in range(REDIRECT_PROBES): + final_url = run(["curl", "--proto", "=https", "--proto-redir", "=https", "-fsSLI", "--max-time", "60", "--retry", "2", + "-o", "/dev/null", "-w", "%{url_effective}", feed], text=True) + results.extend(matches(watch, final_url)) + if results: + break elif provider == "regex": text = fetch.text(feed) if watch.get("unescape_json"): diff --git a/tests/upstream-watch.py b/tests/upstream-watch.py index 2921f7c..65dfd9a 100644 --- a/tests/upstream-watch.py +++ b/tests/upstream-watch.py @@ -160,6 +160,24 @@ b2sums=('old' 'local-b2') output = w.replace_array(text, 'sha256sums', ['abc']) self.assertEqual(output, "sha256sums=('abc') # trailing\npackage() { :; }\n") + def test_redirect_probes_past_a_staged_rollout(self): + watch = {"redirect": "https://example.test/download", + "pattern": r"tool-(?P[0-9]+\.4\.[0-9]+)\.tar\.gz"} + rollout = "https://cdn.example.test/tool-274.3.4801.tar.gz" + stable = "https://cdn.example.test/tool-272.4.3798.tar.gz" + with patch.object(w, 'run', side_effect=[rollout, rollout, stable]) as probe: + releases = w.discover(watch, self.fetch) + self.assertEqual([r["pkgver"] for r in releases], ["272.4.3798"]) + self.assertEqual(probe.call_count, 3) + + def test_redirect_gives_up_after_its_probes(self): + watch = {"redirect": "https://example.test/download", + "pattern": r"tool-(?P[0-9]+\.4\.[0-9]+)\.tar\.gz"} + with patch.object(w, 'run', return_value="https://cdn.example.test/tool-274.3.4801.tar.gz") as probe, \ + self.assertRaisesRegex(ValueError, 'no matching releases'): + w.discover(watch, self.fetch) + self.assertEqual(probe.call_count, w.REDIRECT_PROBES) + def test_git_hash_matches_makepkg(self): repo = self.root / 'git' repo.mkdir()