fix(python): compare-and-advance chain state under a per-agent lock; release 2.0.2
Claude-Session: https://claude.ai/code/session_01QoUjdc7HGKTsgj5GhG8HvF
This commit is contained in:
1 parent
9aaf08bf51
commit
54e65eb56a
3 files changed
+149
-4
No files matched your search
+1
-1
@@ -1,3 +1,3 @@
|
|||||||
"""Canonical Elydora package version."""
|
"""Canonical Elydora package version."""
|
||||||
|
|
||||||
__version__ = "2.0.1"
|
__version__ = "2.0.2"
|
||||||
@@ -185,6 +185,77 @@ def write_chain_state(chain_hash):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def lock_owner_alive(lock_path):
|
||||||
|
if os.name == "nt":
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
with open(lock_path, "r", encoding="ascii") as handle:
|
||||||
|
owner = int(handle.read().strip() or "0")
|
||||||
|
except (FileNotFoundError, ValueError):
|
||||||
|
return False
|
||||||
|
if owner <= 0:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
os.kill(owner, 0)
|
||||||
|
except ProcessLookupError:
|
||||||
|
return False
|
||||||
|
except PermissionError:
|
||||||
|
return True
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def with_chain_lock(update):
|
||||||
|
lock_path = CHAIN_STATE_PATH + ".lock"
|
||||||
|
deadline = time.monotonic() + 1.0
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
descriptor = os.open(lock_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||||
|
os.write(descriptor, str(os.getpid()).encode("ascii"))
|
||||||
|
except FileExistsError:
|
||||||
|
if lock_owner_alive(lock_path):
|
||||||
|
if time.monotonic() > deadline:
|
||||||
|
raise RuntimeError("Chain state lock timed out: " + lock_path)
|
||||||
|
time.sleep(0.01)
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
age = time.time() - os.stat(lock_path).st_mtime
|
||||||
|
except FileNotFoundError:
|
||||||
|
age = 0.0
|
||||||
|
if age > 5.0:
|
||||||
|
reclaimed = lock_path + "." + str(os.getpid()) + ".stale"
|
||||||
|
try:
|
||||||
|
os.rename(lock_path, reclaimed)
|
||||||
|
os.unlink(reclaimed)
|
||||||
|
except FileNotFoundError:
|
||||||
|
pass
|
||||||
|
continue
|
||||||
|
if time.monotonic() > deadline:
|
||||||
|
raise RuntimeError("Chain state lock timed out: " + lock_path)
|
||||||
|
time.sleep(0.01)
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
return update()
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
current = os.lstat(lock_path)
|
||||||
|
except FileNotFoundError:
|
||||||
|
current = None
|
||||||
|
own = os.fstat(descriptor)
|
||||||
|
os.close(descriptor)
|
||||||
|
if current is not None and current.st_ino == own.st_ino and current.st_dev == own.st_dev:
|
||||||
|
os.unlink(lock_path)
|
||||||
|
|
||||||
|
|
||||||
|
def advance_chain_state(from_hash, to_hash):
|
||||||
|
def update():
|
||||||
|
if read_chain_state() != from_hash:
|
||||||
|
return False
|
||||||
|
write_chain_state(to_hash)
|
||||||
|
return True
|
||||||
|
|
||||||
|
return with_chain_lock(update)
|
||||||
|
|
||||||
|
|
||||||
def read_runtime_config():
|
def read_runtime_config():
|
||||||
try:
|
try:
|
||||||
raw = read_protected_file(
|
raw = read_protected_file(
|
||||||
@@ -335,13 +406,13 @@ def submit_operation(event, runtime):
|
|||||||
operation, chain_hash = build_operation(runtime, seed, payload, payload_hash, tool_name, session_id, previous)
|
operation, chain_hash = build_operation(runtime, seed, payload, payload_hash, tool_name, session_id, previous)
|
||||||
expected = post_operation(operation, runtime, deadline)
|
expected = post_operation(operation, runtime, deadline)
|
||||||
if expected is None:
|
if expected is None:
|
||||||
write_chain_state(chain_hash)
|
advance_chain_state(previous, chain_hash)
|
||||||
return
|
return
|
||||||
write_chain_state(expected)
|
advance_chain_state(previous, expected)
|
||||||
log_error("Chain hash resynced to server: " + expected)
|
log_error("Chain hash resynced to server: " + expected)
|
||||||
if attempt >= MAX_CHAIN_ATTEMPTS:
|
if attempt >= MAX_CHAIN_ATTEMPTS:
|
||||||
raise RuntimeError("Audit API rejected prev_chain_hash " + str(attempt) + " times")
|
raise RuntimeError("Audit API rejected prev_chain_hash " + str(attempt) + " times")
|
||||||
previous = expected
|
previous = read_chain_state()
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
import time
|
import time
|
||||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -16,6 +17,7 @@ from claudecode_support import (
|
|||||||
run_handler,
|
run_handler,
|
||||||
server_base_url,
|
server_base_url,
|
||||||
start_api_server,
|
start_api_server,
|
||||||
|
write_json,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -154,6 +156,7 @@ class _MismatchApi(BaseHTTPRequestHandler):
|
|||||||
submissions: list[dict[str, Any]] = []
|
submissions: list[dict[str, Any]] = []
|
||||||
rejections = 1
|
rejections = 1
|
||||||
delay = 0.0
|
delay = 0.0
|
||||||
|
state_path: Path | None = None
|
||||||
expected = "Rxlf4j36C3KvIQ3hWuOkX698BR5iDypUFuB70JjEuvM"
|
expected = "Rxlf4j36C3KvIQ3hWuOkX698BR5iDypUFuB70JjEuvM"
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -161,6 +164,7 @@ class _MismatchApi(BaseHTTPRequestHandler):
|
|||||||
cls.submissions = []
|
cls.submissions = []
|
||||||
cls.rejections = rejections
|
cls.rejections = rejections
|
||||||
cls.delay = delay
|
cls.delay = delay
|
||||||
|
cls.state_path = None
|
||||||
|
|
||||||
def _respond(self, status: int, value: object) -> None:
|
def _respond(self, status: int, value: object) -> None:
|
||||||
body = json.dumps(value).encode()
|
body = json.dumps(value).encode()
|
||||||
@@ -180,6 +184,8 @@ class _MismatchApi(BaseHTTPRequestHandler):
|
|||||||
count = len(cls.submissions)
|
count = len(cls.submissions)
|
||||||
if cls.delay:
|
if cls.delay:
|
||||||
time.sleep(cls.delay)
|
time.sleep(cls.delay)
|
||||||
|
if cls.state_path is not None:
|
||||||
|
write_json(cls.state_path, {"prev_chain_hash": "B" * 43})
|
||||||
if count > cls.rejections:
|
if count > cls.rejections:
|
||||||
self._respond(202, {"receipt": {"seq_no": count}})
|
self._respond(202, {"receipt": {"seq_no": count}})
|
||||||
return
|
return
|
||||||
@@ -263,3 +269,71 @@ def test_audit_stops_retrying_when_the_submit_budget_is_spent(
|
|||||||
assert "timed out" in log or "retry budget exhausted" in log
|
assert "timed out" in log or "retry budget exhausted" in log
|
||||||
finally:
|
finally:
|
||||||
server.shutdown()
|
server.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
def test_audit_does_not_regress_a_chain_head_advanced_by_a_concurrent_hook(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
server = _mismatch_server(0)
|
||||||
|
fixture = prepare_fixture(monkeypatch, tmp_path, base_url=server_base_url(server))
|
||||||
|
try:
|
||||||
|
fixture.install()
|
||||||
|
_MismatchApi.state_path = fixture.agent_dir / "chain-state.json"
|
||||||
|
audit = managed_handler(fixture.settings(), "PostToolUse")
|
||||||
|
result = run_handler(
|
||||||
|
audit, encoded(official_payload("PostToolUse", tool_response={"stdout": "ok"})), fixture
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert _MismatchApi.submissions[0]["prev_chain_hash"] == "A" * 43
|
||||||
|
state = json.loads((fixture.agent_dir / "chain-state.json").read_text(encoding="utf-8"))
|
||||||
|
assert state["prev_chain_hash"] == "B" * 43
|
||||||
|
finally:
|
||||||
|
server.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
def test_audit_clears_a_stale_chain_state_lock_and_proceeds(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
server = _mismatch_server(0)
|
||||||
|
fixture = prepare_fixture(monkeypatch, tmp_path, base_url=server_base_url(server))
|
||||||
|
try:
|
||||||
|
fixture.install()
|
||||||
|
lock_path = fixture.agent_dir / "chain-state.json.lock"
|
||||||
|
lock_path.touch(mode=0o600)
|
||||||
|
stale = time.time() - 10
|
||||||
|
os.utime(lock_path, (stale, stale))
|
||||||
|
audit = managed_handler(fixture.settings(), "PostToolUse")
|
||||||
|
result = run_handler(
|
||||||
|
audit, encoded(official_payload("PostToolUse", tool_response={"stdout": "ok"})), fixture
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert len(_MismatchApi.submissions) == 1
|
||||||
|
assert not lock_path.exists()
|
||||||
|
state = json.loads((fixture.agent_dir / "chain-state.json").read_text(encoding="utf-8"))
|
||||||
|
assert state["prev_chain_hash"] == _MismatchApi.submissions[0]["chain_hash"]
|
||||||
|
finally:
|
||||||
|
server.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
def test_audit_does_not_reclaim_a_lock_whose_owner_is_alive(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
server = _mismatch_server(0)
|
||||||
|
fixture = prepare_fixture(monkeypatch, tmp_path, base_url=server_base_url(server))
|
||||||
|
try:
|
||||||
|
fixture.install()
|
||||||
|
lock_path = fixture.agent_dir / "chain-state.json.lock"
|
||||||
|
lock_path.touch(mode=0o600)
|
||||||
|
lock_path.write_text(str(os.getpid()), encoding="ascii")
|
||||||
|
stale = time.time() - 10
|
||||||
|
os.utime(lock_path, (stale, stale))
|
||||||
|
audit = managed_handler(fixture.settings(), "PostToolUse")
|
||||||
|
result = run_handler(
|
||||||
|
audit, encoded(official_payload("PostToolUse", tool_response={"stdout": "ok"})), fixture
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert len(_MismatchApi.submissions) == 1
|
||||||
|
assert lock_path.exists()
|
||||||
|
assert "lock timed out" in (fixture.agent_dir / "error.log").read_text(encoding="utf-8")
|
||||||
|
finally:
|
||||||
|
server.shutdown()
|
||||||
Reference in new issue
Block a user