upstream-watch: probe redirect watches past a staged rollout
The dropbox watch failed with "no matching releases" on some runs (e.g. actions run 37236931512). It is not the network: Dropbox's download redirect sends a share of requests to a newer build the pattern rejects on purpose. On 2026-10-04, 96 of 100 HEADs ended at dropbox-lnx.x86_64-272.4.3798 and 4 at 274.3.4801, which is not an x.4.y stable build. The redirect provider made one probe, so one unlucky answer failed the whole watch. It now probes up to five times and keeps the first final URL that matches, and curl gets --retry 2 as Fetcher.file already has. Live, 100 discovers against Dropbox all found 272.4.3798 in 103 probes. Two tests cover a rollout answer before the stable one and every probe missing.
This commit is contained in:
1 parent
075f20f401
commit
811a149728
2 files changed
+30
-2
No files matched your search
@@ -26,6 +26,8 @@ VERSION = re.compile(r"[A-Za-z0-9][A-Za-z0-9._+]*\Z")
|
||||
SCALAR = re.compile(r"[A-Za-z0-9._+/-]+\Z")
|
||||
SUM = re.compile(r"(md5|sha1|sha224|sha256|sha384|sha512|b2)sums(_[a-z0-9_]+)?\Z")
|
||||
HASHES = {"b2": "blake2b"}
|
||||
# How many times a redirect watch probes before it calls the feed unmatched.
|
||||
REDIRECT_PROBES = 5
|
||||
|
||||
|
||||
def run(args, **kwargs):
|
||||
@@ -286,8 +288,16 @@ def discover(watch, fetch):
|
||||
values.update({name: json_path(data, path) for name, path in watch.get("fields", {}).items()})
|
||||
results.append(candidate(watch, values))
|
||||
elif provider == "redirect":
|
||||
final_url = run(["curl", "--proto", "=https", "--proto-redir", "=https", "-fsSLI", "--max-time", "60", "-o", "/dev/null", "-w", "%{url_effective}", feed], text=True)
|
||||
results.extend(matches(watch, final_url))
|
||||
# A download redirect can be a staged rollout: some requests land on a
|
||||
# newer build the pattern deliberately rejects (Dropbox, 2026-10-04:
|
||||
# 4 of 100 HEADs ended at 274.3.4801 instead of 272.4.3798), so one
|
||||
# probe that misses is not an answer. Keep the first one that matches.
|
||||
for _ in range(REDIRECT_PROBES):
|
||||
final_url = run(["curl", "--proto", "=https", "--proto-redir", "=https", "-fsSLI", "--max-time", "60", "--retry", "2",
|
||||
"-o", "/dev/null", "-w", "%{url_effective}", feed], text=True)
|
||||
results.extend(matches(watch, final_url))
|
||||
if results:
|
||||
break
|
||||
elif provider == "regex":
|
||||
text = fetch.text(feed)
|
||||
if watch.get("unescape_json"):
|
||||
|
||||
@@ -160,6 +160,24 @@ b2sums=('old' 'local-b2')
|
||||
output = w.replace_array(text, 'sha256sums', ['abc'])
|
||||
self.assertEqual(output, "sha256sums=('abc') # trailing\npackage() { :; }\n")
|
||||
|
||||
def test_redirect_probes_past_a_staged_rollout(self):
|
||||
watch = {"redirect": "https://example.test/download",
|
||||
"pattern": r"tool-(?P<version>[0-9]+\.4\.[0-9]+)\.tar\.gz"}
|
||||
rollout = "https://cdn.example.test/tool-274.3.4801.tar.gz"
|
||||
stable = "https://cdn.example.test/tool-272.4.3798.tar.gz"
|
||||
with patch.object(w, 'run', side_effect=[rollout, rollout, stable]) as probe:
|
||||
releases = w.discover(watch, self.fetch)
|
||||
self.assertEqual([r["pkgver"] for r in releases], ["272.4.3798"])
|
||||
self.assertEqual(probe.call_count, 3)
|
||||
|
||||
def test_redirect_gives_up_after_its_probes(self):
|
||||
watch = {"redirect": "https://example.test/download",
|
||||
"pattern": r"tool-(?P<version>[0-9]+\.4\.[0-9]+)\.tar\.gz"}
|
||||
with patch.object(w, 'run', return_value="https://cdn.example.test/tool-274.3.4801.tar.gz") as probe, \
|
||||
self.assertRaisesRegex(ValueError, 'no matching releases'):
|
||||
w.discover(watch, self.fetch)
|
||||
self.assertEqual(probe.call_count, w.REDIRECT_PROBES)
|
||||
|
||||
def test_git_hash_matches_makepkg(self):
|
||||
repo = self.root / 'git'
|
||||
repo.mkdir()
|
||||
|
||||
Reference in new issue
Block a user