fix(python): retry rejected prev_chain_hash within a bounded submit budget; release 2.0.1
Claude-Session: https://claude.ai/code/session_01QoUjdc7HGKTsgj5GhG8HvF
This commit is contained in:
1 parent
d94989b400
commit
9aaf08bf51
3 files changed
+194
-53
No files matched your search
+1
-1
@@ -1,3 +1,3 @@
|
|||||||
"""Canonical Elydora package version."""
|
"""Canonical Elydora package version."""
|
||||||
|
|
||||||
__version__ = "2.0.0"
|
__version__ = "2.0.1"
|
||||||
@@ -51,6 +51,8 @@ PRIVATE_KEY_PATH = os.path.join(ELYDORA_DIR, AGENT_ID, "private.key")
|
|||||||
CHAIN_STATE_PATH = os.path.join(ELYDORA_DIR, AGENT_ID, "chain-state.json")
|
CHAIN_STATE_PATH = os.path.join(ELYDORA_DIR, AGENT_ID, "chain-state.json")
|
||||||
ERROR_LOG_PATH = os.path.join(ELYDORA_DIR, AGENT_ID, "error.log")
|
ERROR_LOG_PATH = os.path.join(ELYDORA_DIR, AGENT_ID, "error.log")
|
||||||
ZERO_CHAIN_HASH = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
ZERO_CHAIN_HASH = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
|
||||||
|
MAX_CHAIN_ATTEMPTS = 5
|
||||||
|
SUBMIT_BUDGET_SECONDS = 4.0
|
||||||
|
|
||||||
{PROTECTED_RUNTIME_READER}
|
{PROTECTED_RUNTIME_READER}
|
||||||
|
|
||||||
@@ -245,27 +247,44 @@ def read_event():
|
|||||||
return event
|
return event
|
||||||
|
|
||||||
|
|
||||||
def submit_operation(event, runtime):
|
def post_operation(operation, runtime, deadline):
|
||||||
nested = event.get("tool_result")
|
remaining = deadline - time.monotonic()
|
||||||
if not isinstance(nested, dict):
|
if remaining <= 0:
|
||||||
nested = event.get("tool_call")
|
raise RuntimeError("Audit API retry budget exhausted")
|
||||||
if not isinstance(nested, dict):
|
url = runtime.get("base_url", "https://api.elydora.com").rstrip("/") + "/v1/operations"
|
||||||
nested = {{}}
|
headers = {{"Content-Type": "application/json", "Accept": "application/json"}}
|
||||||
tool_name = event.get("tool_name", event.get("toolName", event.get("name", nested.get("name", "unknown"))))
|
if runtime.get("token"):
|
||||||
tool_input = event.get("tool_input", event.get("toolInput", event.get("input", nested.get("input", {{}}))))
|
headers["Authorization"] = "Bearer " + runtime["token"]
|
||||||
session_id = event.get(
|
request = urllib.request.Request(
|
||||||
"conversation_id",
|
url,
|
||||||
event.get("session_id", event.get("sessionId", event.get("session", event.get("taskId", "unknown")))),
|
data=json.dumps(operation).encode("utf-8"),
|
||||||
|
headers=headers,
|
||||||
|
method="POST",
|
||||||
)
|
)
|
||||||
previous = read_chain_state()
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=min(5.0, remaining)):
|
||||||
|
return None
|
||||||
|
except urllib.error.HTTPError as error:
|
||||||
|
body = error.read().decode("utf-8")
|
||||||
|
if error.code == 400 and body:
|
||||||
|
try:
|
||||||
|
failure = json.loads(body)
|
||||||
|
except (json.JSONDecodeError, ValueError) as parse_error:
|
||||||
|
raise RuntimeError("Audit API returned invalid error JSON: " + str(parse_error)) from parse_error
|
||||||
|
details = failure.get("error", {{}}) if isinstance(failure, dict) else {{}}
|
||||||
|
if details.get("code") == "PREV_HASH_MISMATCH":
|
||||||
|
match = re.search(
|
||||||
|
r'Expected prev_chain_hash "([A-Za-z0-9_-]{{43}})"',
|
||||||
|
str(details.get("message", "")),
|
||||||
|
)
|
||||||
|
if match:
|
||||||
|
return match.group(1)
|
||||||
|
raise RuntimeError("Audit API returned HTTP " + str(error.code)) from error
|
||||||
|
|
||||||
|
|
||||||
|
def build_operation(runtime, seed, payload, payload_hash, tool_name, session_id, previous):
|
||||||
operation_id = generate_uuidv7()
|
operation_id = generate_uuidv7()
|
||||||
issued_at = int(time.time() * 1000)
|
issued_at = int(time.time() * 1000)
|
||||||
payload = event if NATIVE_PAYLOAD else {{
|
|
||||||
"tool_name": tool_name,
|
|
||||||
"tool_input": tool_input,
|
|
||||||
"session_id": session_id,
|
|
||||||
}}
|
|
||||||
payload_hash = compute_payload_hash(payload)
|
|
||||||
chain_hash = compute_chain_hash(previous, payload_hash, operation_id, issued_at)
|
chain_hash = compute_chain_hash(previous, payload_hash, operation_id, issued_at)
|
||||||
unsigned = {{
|
unsigned = {{
|
||||||
"op_version": "1.0",
|
"op_version": "1.0",
|
||||||
@@ -283,44 +302,47 @@ def submit_operation(event, runtime):
|
|||||||
"prev_chain_hash": previous,
|
"prev_chain_hash": previous,
|
||||||
"agent_pubkey_kid": runtime["kid"],
|
"agent_pubkey_kid": runtime["kid"],
|
||||||
}}
|
}}
|
||||||
signature = sign_ed25519(
|
signature = sign_ed25519(seed, jcs_canonicalize(unsigned).encode("utf-8"))
|
||||||
read_private_key(),
|
|
||||||
jcs_canonicalize(unsigned).encode("utf-8"),
|
|
||||||
)
|
|
||||||
operation = dict(unsigned)
|
operation = dict(unsigned)
|
||||||
operation.update({{"chain_hash": chain_hash, "signature": signature}})
|
operation.update({{"chain_hash": chain_hash, "signature": signature}})
|
||||||
url = runtime.get("base_url", "https://api.elydora.com").rstrip("/") + "/v1/operations"
|
return operation, chain_hash
|
||||||
headers = {{"Content-Type": "application/json", "Accept": "application/json"}}
|
|
||||||
if runtime.get("token"):
|
|
||||||
headers["Authorization"] = "Bearer " + runtime["token"]
|
|
||||||
request = urllib.request.Request(
|
|
||||||
url,
|
|
||||||
data=json.dumps(operation).encode("utf-8"),
|
|
||||||
headers=headers,
|
|
||||||
method="POST",
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
with urllib.request.urlopen(request, timeout=5):
|
|
||||||
write_chain_state(chain_hash)
|
|
||||||
except urllib.error.HTTPError as error:
|
|
||||||
body = error.read().decode("utf-8")
|
|
||||||
if error.code == 400 and body:
|
|
||||||
try:
|
|
||||||
failure = json.loads(body)
|
|
||||||
except (json.JSONDecodeError, ValueError) as parse_error:
|
|
||||||
raise RuntimeError("Audit API returned invalid error JSON: " + str(parse_error)) from parse_error
|
|
||||||
details = failure.get("error", {{}}) if isinstance(failure, dict) else {{}}
|
|
||||||
if details.get("code") == "PREV_HASH_MISMATCH":
|
|
||||||
match = re.search(
|
|
||||||
r'Expected prev_chain_hash "([A-Za-z0-9_-]{{43}})"',
|
|
||||||
str(details.get("message", "")),
|
|
||||||
)
|
|
||||||
if match:
|
|
||||||
write_chain_state(match.group(1))
|
|
||||||
log_error("Chain hash resynced to server: " + match.group(1))
|
|
||||||
raise RuntimeError("Audit API returned HTTP " + str(error.code)) from error
|
|
||||||
|
|
||||||
|
|
||||||
|
def submit_operation(event, runtime):
|
||||||
|
nested = event.get("tool_result")
|
||||||
|
if not isinstance(nested, dict):
|
||||||
|
nested = event.get("tool_call")
|
||||||
|
if not isinstance(nested, dict):
|
||||||
|
nested = {{}}
|
||||||
|
tool_name = event.get("tool_name", event.get("toolName", event.get("name", nested.get("name", "unknown"))))
|
||||||
|
tool_input = event.get("tool_input", event.get("toolInput", event.get("input", nested.get("input", {{}}))))
|
||||||
|
session_id = event.get(
|
||||||
|
"conversation_id",
|
||||||
|
event.get("session_id", event.get("sessionId", event.get("session", event.get("taskId", "unknown")))),
|
||||||
|
)
|
||||||
|
payload = event if NATIVE_PAYLOAD else {{
|
||||||
|
"tool_name": tool_name,
|
||||||
|
"tool_input": tool_input,
|
||||||
|
"session_id": session_id,
|
||||||
|
}}
|
||||||
|
payload_hash = compute_payload_hash(payload)
|
||||||
|
seed = read_private_key()
|
||||||
|
deadline = time.monotonic() + SUBMIT_BUDGET_SECONDS
|
||||||
|
previous = read_chain_state()
|
||||||
|
attempt = 0
|
||||||
|
while True:
|
||||||
|
attempt += 1
|
||||||
|
operation, chain_hash = build_operation(runtime, seed, payload, payload_hash, tool_name, session_id, previous)
|
||||||
|
expected = post_operation(operation, runtime, deadline)
|
||||||
|
if expected is None:
|
||||||
|
write_chain_state(chain_hash)
|
||||||
|
return
|
||||||
|
write_chain_state(expected)
|
||||||
|
log_error("Chain hash resynced to server: " + expected)
|
||||||
|
if attempt >= MAX_CHAIN_ATTEMPTS:
|
||||||
|
raise RuntimeError("Audit API rejected prev_chain_hash " + str(attempt) + " times")
|
||||||
|
previous = expected
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
try:
|
try:
|
||||||
submit_operation(read_event(), read_runtime_config())
|
submit_operation(read_event(), read_runtime_config())
|
||||||
|
|||||||
@@ -1,7 +1,11 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import time
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from threading import Thread
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
@@ -144,3 +148,118 @@ def test_runtime_failures_are_observable_and_fail_open(
|
|||||||
log = (fixture.agent_dir / "error.log").read_text(encoding="utf-8")
|
log = (fixture.agent_dir / "error.log").read_text(encoding="utf-8")
|
||||||
assert "invalid JSON" in log
|
assert "invalid JSON" in log
|
||||||
assert "refused" in log.lower() or "urlopen" in log.lower()
|
assert "refused" in log.lower() or "urlopen" in log.lower()
|
||||||
|
|
||||||
|
|
||||||
|
class _MismatchApi(BaseHTTPRequestHandler):
|
||||||
|
submissions: list[dict[str, Any]] = []
|
||||||
|
rejections = 1
|
||||||
|
delay = 0.0
|
||||||
|
expected = "Rxlf4j36C3KvIQ3hWuOkX698BR5iDypUFuB70JjEuvM"
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def reset(cls, rejections: int, delay: float = 0.0) -> None:
|
||||||
|
cls.submissions = []
|
||||||
|
cls.rejections = rejections
|
||||||
|
cls.delay = delay
|
||||||
|
|
||||||
|
def _respond(self, status: int, value: object) -> None:
|
||||||
|
body = json.dumps(value).encode()
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
self._respond(200, {"agent": {"status": "active"}})
|
||||||
|
|
||||||
|
def do_POST(self) -> None:
|
||||||
|
raw = self.rfile.read(int(self.headers.get("Content-Length", "0")))
|
||||||
|
cls = type(self)
|
||||||
|
cls.submissions.append(json.loads(raw))
|
||||||
|
count = len(cls.submissions)
|
||||||
|
if cls.delay:
|
||||||
|
time.sleep(cls.delay)
|
||||||
|
if count > cls.rejections:
|
||||||
|
self._respond(202, {"receipt": {"seq_no": count}})
|
||||||
|
return
|
||||||
|
expected = cls.expected if cls.rejections == 1 else str(count).rjust(43, "A")
|
||||||
|
self._respond(400, {"error": {
|
||||||
|
"code": "PREV_HASH_MISMATCH",
|
||||||
|
"message": f'Expected prev_chain_hash "{expected}", got "x".',
|
||||||
|
}})
|
||||||
|
|
||||||
|
def log_message(self, _format: str, *_args: object) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
def _mismatch_server(rejections: int, delay: float = 0.0) -> ThreadingHTTPServer:
|
||||||
|
_MismatchApi.reset(rejections, delay)
|
||||||
|
server = ThreadingHTTPServer(("127.0.0.1", 0), _MismatchApi)
|
||||||
|
Thread(target=server.serve_forever, daemon=True).start()
|
||||||
|
return server
|
||||||
|
|
||||||
|
|
||||||
|
def test_audit_retries_rejected_prev_chain_hash_with_server_value(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
server = _mismatch_server(1)
|
||||||
|
fixture = prepare_fixture(monkeypatch, tmp_path, base_url=server_base_url(server))
|
||||||
|
try:
|
||||||
|
fixture.install()
|
||||||
|
audit = managed_handler(fixture.settings(), "PostToolUse")
|
||||||
|
result = run_handler(
|
||||||
|
audit, encoded(official_payload("PostToolUse", tool_response={"stdout": "ok"})), fixture
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
submissions = _MismatchApi.submissions
|
||||||
|
assert len(submissions) == 2
|
||||||
|
assert submissions[1]["prev_chain_hash"] == _MismatchApi.expected
|
||||||
|
assert submissions[1]["operation_id"] != submissions[0]["operation_id"]
|
||||||
|
assert submissions[1]["nonce"] != submissions[0]["nonce"]
|
||||||
|
state = json.loads((fixture.agent_dir / "chain-state.json").read_text(encoding="utf-8"))
|
||||||
|
assert state["prev_chain_hash"] == submissions[1]["chain_hash"]
|
||||||
|
assert "resynced to server: Rxlf" in (fixture.agent_dir / "error.log").read_text(encoding="utf-8")
|
||||||
|
finally:
|
||||||
|
server.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
def test_audit_stops_after_five_rejected_prev_chain_hash_attempts(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
server = _mismatch_server(99)
|
||||||
|
fixture = prepare_fixture(monkeypatch, tmp_path, base_url=server_base_url(server))
|
||||||
|
try:
|
||||||
|
fixture.install()
|
||||||
|
audit = managed_handler(fixture.settings(), "PostToolUse")
|
||||||
|
result = run_handler(
|
||||||
|
audit, encoded(official_payload("PostToolUse", tool_response={"stdout": "ok"})), fixture
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert len(_MismatchApi.submissions) == 5
|
||||||
|
assert _MismatchApi.submissions[4]["prev_chain_hash"] == "4".rjust(43, "A")
|
||||||
|
assert "rejected prev_chain_hash 5 times" in (fixture.agent_dir / "error.log").read_text(encoding="utf-8")
|
||||||
|
finally:
|
||||||
|
server.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
def test_audit_stops_retrying_when_the_submit_budget_is_spent(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
server = _mismatch_server(99, delay=2.6)
|
||||||
|
fixture = prepare_fixture(monkeypatch, tmp_path, base_url=server_base_url(server))
|
||||||
|
try:
|
||||||
|
fixture.install()
|
||||||
|
audit = managed_handler(fixture.settings(), "PostToolUse")
|
||||||
|
started = time.monotonic()
|
||||||
|
result = run_handler(
|
||||||
|
audit, encoded(official_payload("PostToolUse", tool_response={"stdout": "ok"})), fixture
|
||||||
|
)
|
||||||
|
elapsed = time.monotonic() - started
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert elapsed < 7.5, elapsed
|
||||||
|
assert 2 <= len(_MismatchApi.submissions) <= 3
|
||||||
|
log = (fixture.agent_dir / "error.log").read_text(encoding="utf-8").lower()
|
||||||
|
assert "timed out" in log or "retry budget exhausted" in log
|
||||||
|
finally:
|
||||||
|
server.shutdown()
|
||||||
Reference in new issue
Block a user