fix(python): retry rejected prev_chain_hash within a bounded submit budget; release 2.0.1
Claude-Session: https://claude.ai/code/session_01QoUjdc7HGKTsgj5GhG8HvF
This commit is contained in:
1 parent
d94989b400
commit
9aaf08bf51
3 files changed
+194
-53
No files matched your search
+1
-1
@@ -1,3 +1,3 @@
|
||||
"""Canonical Elydora package version."""
|
||||
|
||||
__version__ = "2.0.0"
|
||||
__version__ = "2.0.1"
|
||||
@@ -51,6 +51,8 @@ PRIVATE_KEY_PATH = os.path.join(ELYDORA_DIR, AGENT_ID, "private.key")
|
||||
CHAIN_STATE_PATH = os.path.join(ELYDORA_DIR, AGENT_ID, "chain-state.json")
|
||||
ERROR_LOG_PATH = os.path.join(ELYDORA_DIR, AGENT_ID, "error.log")
|
||||
ZERO_CHAIN_HASH = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
||||
MAX_CHAIN_ATTEMPTS = 5
|
||||
SUBMIT_BUDGET_SECONDS = 4.0
|
||||
|
||||
{PROTECTED_RUNTIME_READER}
|
||||
|
||||
@@ -245,27 +247,44 @@ def read_event():
|
||||
return event
|
||||
|
||||
|
||||
def submit_operation(event, runtime):
|
||||
nested = event.get("tool_result")
|
||||
if not isinstance(nested, dict):
|
||||
nested = event.get("tool_call")
|
||||
if not isinstance(nested, dict):
|
||||
nested = {{}}
|
||||
tool_name = event.get("tool_name", event.get("toolName", event.get("name", nested.get("name", "unknown"))))
|
||||
tool_input = event.get("tool_input", event.get("toolInput", event.get("input", nested.get("input", {{}}))))
|
||||
session_id = event.get(
|
||||
"conversation_id",
|
||||
event.get("session_id", event.get("sessionId", event.get("session", event.get("taskId", "unknown")))),
|
||||
def post_operation(operation, runtime, deadline):
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise RuntimeError("Audit API retry budget exhausted")
|
||||
url = runtime.get("base_url", "https://api.elydora.com").rstrip("/") + "/v1/operations"
|
||||
headers = {{"Content-Type": "application/json", "Accept": "application/json"}}
|
||||
if runtime.get("token"):
|
||||
headers["Authorization"] = "Bearer " + runtime["token"]
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=json.dumps(operation).encode("utf-8"),
|
||||
headers=headers,
|
||||
method="POST",
|
||||
)
|
||||
previous = read_chain_state()
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=min(5.0, remaining)):
|
||||
return None
|
||||
except urllib.error.HTTPError as error:
|
||||
body = error.read().decode("utf-8")
|
||||
if error.code == 400 and body:
|
||||
try:
|
||||
failure = json.loads(body)
|
||||
except (json.JSONDecodeError, ValueError) as parse_error:
|
||||
raise RuntimeError("Audit API returned invalid error JSON: " + str(parse_error)) from parse_error
|
||||
details = failure.get("error", {{}}) if isinstance(failure, dict) else {{}}
|
||||
if details.get("code") == "PREV_HASH_MISMATCH":
|
||||
match = re.search(
|
||||
r'Expected prev_chain_hash "([A-Za-z0-9_-]{{43}})"',
|
||||
str(details.get("message", "")),
|
||||
)
|
||||
if match:
|
||||
return match.group(1)
|
||||
raise RuntimeError("Audit API returned HTTP " + str(error.code)) from error
|
||||
|
||||
|
||||
def build_operation(runtime, seed, payload, payload_hash, tool_name, session_id, previous):
|
||||
operation_id = generate_uuidv7()
|
||||
issued_at = int(time.time() * 1000)
|
||||
payload = event if NATIVE_PAYLOAD else {{
|
||||
"tool_name": tool_name,
|
||||
"tool_input": tool_input,
|
||||
"session_id": session_id,
|
||||
}}
|
||||
payload_hash = compute_payload_hash(payload)
|
||||
chain_hash = compute_chain_hash(previous, payload_hash, operation_id, issued_at)
|
||||
unsigned = {{
|
||||
"op_version": "1.0",
|
||||
@@ -283,44 +302,47 @@ def submit_operation(event, runtime):
|
||||
"prev_chain_hash": previous,
|
||||
"agent_pubkey_kid": runtime["kid"],
|
||||
}}
|
||||
signature = sign_ed25519(
|
||||
read_private_key(),
|
||||
jcs_canonicalize(unsigned).encode("utf-8"),
|
||||
)
|
||||
signature = sign_ed25519(seed, jcs_canonicalize(unsigned).encode("utf-8"))
|
||||
operation = dict(unsigned)
|
||||
operation.update({{"chain_hash": chain_hash, "signature": signature}})
|
||||
url = runtime.get("base_url", "https://api.elydora.com").rstrip("/") + "/v1/operations"
|
||||
headers = {{"Content-Type": "application/json", "Accept": "application/json"}}
|
||||
if runtime.get("token"):
|
||||
headers["Authorization"] = "Bearer " + runtime["token"]
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=json.dumps(operation).encode("utf-8"),
|
||||
headers=headers,
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=5):
|
||||
write_chain_state(chain_hash)
|
||||
except urllib.error.HTTPError as error:
|
||||
body = error.read().decode("utf-8")
|
||||
if error.code == 400 and body:
|
||||
try:
|
||||
failure = json.loads(body)
|
||||
except (json.JSONDecodeError, ValueError) as parse_error:
|
||||
raise RuntimeError("Audit API returned invalid error JSON: " + str(parse_error)) from parse_error
|
||||
details = failure.get("error", {{}}) if isinstance(failure, dict) else {{}}
|
||||
if details.get("code") == "PREV_HASH_MISMATCH":
|
||||
match = re.search(
|
||||
r'Expected prev_chain_hash "([A-Za-z0-9_-]{{43}})"',
|
||||
str(details.get("message", "")),
|
||||
)
|
||||
if match:
|
||||
write_chain_state(match.group(1))
|
||||
log_error("Chain hash resynced to server: " + match.group(1))
|
||||
raise RuntimeError("Audit API returned HTTP " + str(error.code)) from error
|
||||
return operation, chain_hash
|
||||
|
||||
|
||||
def submit_operation(event, runtime):
|
||||
nested = event.get("tool_result")
|
||||
if not isinstance(nested, dict):
|
||||
nested = event.get("tool_call")
|
||||
if not isinstance(nested, dict):
|
||||
nested = {{}}
|
||||
tool_name = event.get("tool_name", event.get("toolName", event.get("name", nested.get("name", "unknown"))))
|
||||
tool_input = event.get("tool_input", event.get("toolInput", event.get("input", nested.get("input", {{}}))))
|
||||
session_id = event.get(
|
||||
"conversation_id",
|
||||
event.get("session_id", event.get("sessionId", event.get("session", event.get("taskId", "unknown")))),
|
||||
)
|
||||
payload = event if NATIVE_PAYLOAD else {{
|
||||
"tool_name": tool_name,
|
||||
"tool_input": tool_input,
|
||||
"session_id": session_id,
|
||||
}}
|
||||
payload_hash = compute_payload_hash(payload)
|
||||
seed = read_private_key()
|
||||
deadline = time.monotonic() + SUBMIT_BUDGET_SECONDS
|
||||
previous = read_chain_state()
|
||||
attempt = 0
|
||||
while True:
|
||||
attempt += 1
|
||||
operation, chain_hash = build_operation(runtime, seed, payload, payload_hash, tool_name, session_id, previous)
|
||||
expected = post_operation(operation, runtime, deadline)
|
||||
if expected is None:
|
||||
write_chain_state(chain_hash)
|
||||
return
|
||||
write_chain_state(expected)
|
||||
log_error("Chain hash resynced to server: " + expected)
|
||||
if attempt >= MAX_CHAIN_ATTEMPTS:
|
||||
raise RuntimeError("Audit API rejected prev_chain_hash " + str(attempt) + " times")
|
||||
previous = expected
|
||||
|
||||
def main():
|
||||
try:
|
||||
submit_operation(read_event(), read_runtime_config())
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from threading import Thread
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -144,3 +148,118 @@ def test_runtime_failures_are_observable_and_fail_open(
|
||||
log = (fixture.agent_dir / "error.log").read_text(encoding="utf-8")
|
||||
assert "invalid JSON" in log
|
||||
assert "refused" in log.lower() or "urlopen" in log.lower()
|
||||
|
||||
|
||||
class _MismatchApi(BaseHTTPRequestHandler):
|
||||
submissions: list[dict[str, Any]] = []
|
||||
rejections = 1
|
||||
delay = 0.0
|
||||
expected = "Rxlf4j36C3KvIQ3hWuOkX698BR5iDypUFuB70JjEuvM"
|
||||
|
||||
@classmethod
|
||||
def reset(cls, rejections: int, delay: float = 0.0) -> None:
|
||||
cls.submissions = []
|
||||
cls.rejections = rejections
|
||||
cls.delay = delay
|
||||
|
||||
def _respond(self, status: int, value: object) -> None:
|
||||
body = json.dumps(value).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_GET(self) -> None:
|
||||
self._respond(200, {"agent": {"status": "active"}})
|
||||
|
||||
def do_POST(self) -> None:
|
||||
raw = self.rfile.read(int(self.headers.get("Content-Length", "0")))
|
||||
cls = type(self)
|
||||
cls.submissions.append(json.loads(raw))
|
||||
count = len(cls.submissions)
|
||||
if cls.delay:
|
||||
time.sleep(cls.delay)
|
||||
if count > cls.rejections:
|
||||
self._respond(202, {"receipt": {"seq_no": count}})
|
||||
return
|
||||
expected = cls.expected if cls.rejections == 1 else str(count).rjust(43, "A")
|
||||
self._respond(400, {"error": {
|
||||
"code": "PREV_HASH_MISMATCH",
|
||||
"message": f'Expected prev_chain_hash "{expected}", got "x".',
|
||||
}})
|
||||
|
||||
def log_message(self, _format: str, *_args: object) -> None:
|
||||
return
|
||||
|
||||
|
||||
def _mismatch_server(rejections: int, delay: float = 0.0) -> ThreadingHTTPServer:
|
||||
_MismatchApi.reset(rejections, delay)
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), _MismatchApi)
|
||||
Thread(target=server.serve_forever, daemon=True).start()
|
||||
return server
|
||||
|
||||
|
||||
def test_audit_retries_rejected_prev_chain_hash_with_server_value(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
server = _mismatch_server(1)
|
||||
fixture = prepare_fixture(monkeypatch, tmp_path, base_url=server_base_url(server))
|
||||
try:
|
||||
fixture.install()
|
||||
audit = managed_handler(fixture.settings(), "PostToolUse")
|
||||
result = run_handler(
|
||||
audit, encoded(official_payload("PostToolUse", tool_response={"stdout": "ok"})), fixture
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
submissions = _MismatchApi.submissions
|
||||
assert len(submissions) == 2
|
||||
assert submissions[1]["prev_chain_hash"] == _MismatchApi.expected
|
||||
assert submissions[1]["operation_id"] != submissions[0]["operation_id"]
|
||||
assert submissions[1]["nonce"] != submissions[0]["nonce"]
|
||||
state = json.loads((fixture.agent_dir / "chain-state.json").read_text(encoding="utf-8"))
|
||||
assert state["prev_chain_hash"] == submissions[1]["chain_hash"]
|
||||
assert "resynced to server: Rxlf" in (fixture.agent_dir / "error.log").read_text(encoding="utf-8")
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def test_audit_stops_after_five_rejected_prev_chain_hash_attempts(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
server = _mismatch_server(99)
|
||||
fixture = prepare_fixture(monkeypatch, tmp_path, base_url=server_base_url(server))
|
||||
try:
|
||||
fixture.install()
|
||||
audit = managed_handler(fixture.settings(), "PostToolUse")
|
||||
result = run_handler(
|
||||
audit, encoded(official_payload("PostToolUse", tool_response={"stdout": "ok"})), fixture
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert len(_MismatchApi.submissions) == 5
|
||||
assert _MismatchApi.submissions[4]["prev_chain_hash"] == "4".rjust(43, "A")
|
||||
assert "rejected prev_chain_hash 5 times" in (fixture.agent_dir / "error.log").read_text(encoding="utf-8")
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def test_audit_stops_retrying_when_the_submit_budget_is_spent(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
server = _mismatch_server(99, delay=2.6)
|
||||
fixture = prepare_fixture(monkeypatch, tmp_path, base_url=server_base_url(server))
|
||||
try:
|
||||
fixture.install()
|
||||
audit = managed_handler(fixture.settings(), "PostToolUse")
|
||||
started = time.monotonic()
|
||||
result = run_handler(
|
||||
audit, encoded(official_payload("PostToolUse", tool_response={"stdout": "ok"})), fixture
|
||||
)
|
||||
elapsed = time.monotonic() - started
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert elapsed < 7.5, elapsed
|
||||
assert 2 <= len(_MismatchApi.submissions) <= 3
|
||||
log = (fixture.agent_dir / "error.log").read_text(encoding="utf-8").lower()
|
||||
assert "timed out" in log or "retry budget exhausted" in log
|
||||
finally:
|
||||
server.shutdown()
|
||||
Reference in new issue
Block a user