fix(python): compare-and-advance chain state under a per-agent lock; release 2.0.2
Claude-Session: https://claude.ai/code/session_01QoUjdc7HGKTsgj5GhG8HvF
This commit is contained in:
1 parent
9aaf08bf51
commit
54e65eb56a
3 files changed
+149
-4
No files matched your search
+1
-1
@@ -1,3 +1,3 @@
|
||||
"""Canonical Elydora package version."""
|
||||
|
||||
__version__ = "2.0.1"
|
||||
__version__ = "2.0.2"
|
||||
@@ -185,6 +185,77 @@ def write_chain_state(chain_hash):
|
||||
)
|
||||
|
||||
|
||||
def lock_owner_alive(lock_path):
|
||||
if os.name == "nt":
|
||||
return False
|
||||
try:
|
||||
with open(lock_path, "r", encoding="ascii") as handle:
|
||||
owner = int(handle.read().strip() or "0")
|
||||
except (FileNotFoundError, ValueError):
|
||||
return False
|
||||
if owner <= 0:
|
||||
return False
|
||||
try:
|
||||
os.kill(owner, 0)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return True
|
||||
return True
|
||||
|
||||
|
||||
def with_chain_lock(update):
|
||||
lock_path = CHAIN_STATE_PATH + ".lock"
|
||||
deadline = time.monotonic() + 1.0
|
||||
while True:
|
||||
try:
|
||||
descriptor = os.open(lock_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
os.write(descriptor, str(os.getpid()).encode("ascii"))
|
||||
except FileExistsError:
|
||||
if lock_owner_alive(lock_path):
|
||||
if time.monotonic() > deadline:
|
||||
raise RuntimeError("Chain state lock timed out: " + lock_path)
|
||||
time.sleep(0.01)
|
||||
continue
|
||||
try:
|
||||
age = time.time() - os.stat(lock_path).st_mtime
|
||||
except FileNotFoundError:
|
||||
age = 0.0
|
||||
if age > 5.0:
|
||||
reclaimed = lock_path + "." + str(os.getpid()) + ".stale"
|
||||
try:
|
||||
os.rename(lock_path, reclaimed)
|
||||
os.unlink(reclaimed)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
continue
|
||||
if time.monotonic() > deadline:
|
||||
raise RuntimeError("Chain state lock timed out: " + lock_path)
|
||||
time.sleep(0.01)
|
||||
continue
|
||||
try:
|
||||
return update()
|
||||
finally:
|
||||
try:
|
||||
current = os.lstat(lock_path)
|
||||
except FileNotFoundError:
|
||||
current = None
|
||||
own = os.fstat(descriptor)
|
||||
os.close(descriptor)
|
||||
if current is not None and current.st_ino == own.st_ino and current.st_dev == own.st_dev:
|
||||
os.unlink(lock_path)
|
||||
|
||||
|
||||
def advance_chain_state(from_hash, to_hash):
|
||||
def update():
|
||||
if read_chain_state() != from_hash:
|
||||
return False
|
||||
write_chain_state(to_hash)
|
||||
return True
|
||||
|
||||
return with_chain_lock(update)
|
||||
|
||||
|
||||
def read_runtime_config():
|
||||
try:
|
||||
raw = read_protected_file(
|
||||
@@ -335,13 +406,13 @@ def submit_operation(event, runtime):
|
||||
operation, chain_hash = build_operation(runtime, seed, payload, payload_hash, tool_name, session_id, previous)
|
||||
expected = post_operation(operation, runtime, deadline)
|
||||
if expected is None:
|
||||
write_chain_state(chain_hash)
|
||||
advance_chain_state(previous, chain_hash)
|
||||
return
|
||||
write_chain_state(expected)
|
||||
advance_chain_state(previous, expected)
|
||||
log_error("Chain hash resynced to server: " + expected)
|
||||
if attempt >= MAX_CHAIN_ATTEMPTS:
|
||||
raise RuntimeError("Audit API rejected prev_chain_hash " + str(attempt) + " times")
|
||||
previous = expected
|
||||
previous = read_chain_state()
|
||||
|
||||
def main():
|
||||
try:
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
@@ -16,6 +17,7 @@ from claudecode_support import (
|
||||
run_handler,
|
||||
server_base_url,
|
||||
start_api_server,
|
||||
write_json,
|
||||
)
|
||||
|
||||
|
||||
@@ -154,6 +156,7 @@ class _MismatchApi(BaseHTTPRequestHandler):
|
||||
submissions: list[dict[str, Any]] = []
|
||||
rejections = 1
|
||||
delay = 0.0
|
||||
state_path: Path | None = None
|
||||
expected = "Rxlf4j36C3KvIQ3hWuOkX698BR5iDypUFuB70JjEuvM"
|
||||
|
||||
@classmethod
|
||||
@@ -161,6 +164,7 @@ class _MismatchApi(BaseHTTPRequestHandler):
|
||||
cls.submissions = []
|
||||
cls.rejections = rejections
|
||||
cls.delay = delay
|
||||
cls.state_path = None
|
||||
|
||||
def _respond(self, status: int, value: object) -> None:
|
||||
body = json.dumps(value).encode()
|
||||
@@ -180,6 +184,8 @@ class _MismatchApi(BaseHTTPRequestHandler):
|
||||
count = len(cls.submissions)
|
||||
if cls.delay:
|
||||
time.sleep(cls.delay)
|
||||
if cls.state_path is not None:
|
||||
write_json(cls.state_path, {"prev_chain_hash": "B" * 43})
|
||||
if count > cls.rejections:
|
||||
self._respond(202, {"receipt": {"seq_no": count}})
|
||||
return
|
||||
@@ -263,3 +269,71 @@ def test_audit_stops_retrying_when_the_submit_budget_is_spent(
|
||||
assert "timed out" in log or "retry budget exhausted" in log
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def test_audit_does_not_regress_a_chain_head_advanced_by_a_concurrent_hook(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
server = _mismatch_server(0)
|
||||
fixture = prepare_fixture(monkeypatch, tmp_path, base_url=server_base_url(server))
|
||||
try:
|
||||
fixture.install()
|
||||
_MismatchApi.state_path = fixture.agent_dir / "chain-state.json"
|
||||
audit = managed_handler(fixture.settings(), "PostToolUse")
|
||||
result = run_handler(
|
||||
audit, encoded(official_payload("PostToolUse", tool_response={"stdout": "ok"})), fixture
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert _MismatchApi.submissions[0]["prev_chain_hash"] == "A" * 43
|
||||
state = json.loads((fixture.agent_dir / "chain-state.json").read_text(encoding="utf-8"))
|
||||
assert state["prev_chain_hash"] == "B" * 43
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def test_audit_clears_a_stale_chain_state_lock_and_proceeds(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
server = _mismatch_server(0)
|
||||
fixture = prepare_fixture(monkeypatch, tmp_path, base_url=server_base_url(server))
|
||||
try:
|
||||
fixture.install()
|
||||
lock_path = fixture.agent_dir / "chain-state.json.lock"
|
||||
lock_path.touch(mode=0o600)
|
||||
stale = time.time() - 10
|
||||
os.utime(lock_path, (stale, stale))
|
||||
audit = managed_handler(fixture.settings(), "PostToolUse")
|
||||
result = run_handler(
|
||||
audit, encoded(official_payload("PostToolUse", tool_response={"stdout": "ok"})), fixture
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert len(_MismatchApi.submissions) == 1
|
||||
assert not lock_path.exists()
|
||||
state = json.loads((fixture.agent_dir / "chain-state.json").read_text(encoding="utf-8"))
|
||||
assert state["prev_chain_hash"] == _MismatchApi.submissions[0]["chain_hash"]
|
||||
finally:
|
||||
server.shutdown()
|
||||
|
||||
|
||||
def test_audit_does_not_reclaim_a_lock_whose_owner_is_alive(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
server = _mismatch_server(0)
|
||||
fixture = prepare_fixture(monkeypatch, tmp_path, base_url=server_base_url(server))
|
||||
try:
|
||||
fixture.install()
|
||||
lock_path = fixture.agent_dir / "chain-state.json.lock"
|
||||
lock_path.touch(mode=0o600)
|
||||
lock_path.write_text(str(os.getpid()), encoding="ascii")
|
||||
stale = time.time() - 10
|
||||
os.utime(lock_path, (stale, stale))
|
||||
audit = managed_handler(fixture.settings(), "PostToolUse")
|
||||
result = run_handler(
|
||||
audit, encoded(official_payload("PostToolUse", tool_response={"stdout": "ok"})), fixture
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert len(_MismatchApi.submissions) == 1
|
||||
assert lock_path.exists()
|
||||
assert "lock timed out" in (fixture.agent_dir / "error.log").read_text(encoding="utf-8")
|
||||
finally:
|
||||
server.shutdown()
|
||||
Reference in new issue
Block a user