fix(python): make retries safe and deterministic
This commit is contained in:
1 parent
f64d00673c
commit
82e5807752
8 files changed
+566
-49
No files matched your search
@@ -48,6 +48,9 @@ This repository publishes the open-source server, console, integration catalog,
|
|||||||
- Keep functions focused on one ownership boundary.
|
- Keep functions focused on one ownership boundary.
|
||||||
- Propagate unexpected errors to the CLI or request boundary.
|
- Propagate unexpected errors to the CLI or request boundary.
|
||||||
- Keep technical claims executable through tests or generated validation.
|
- Keep technical claims executable through tests or generated validation.
|
||||||
|
- Interpret Python SDK `max_retries` as retries after the initial attempt and reject negative or non-integer values.
|
||||||
|
- Retry RFC-idempotent Python SDK requests automatically; retry non-idempotent requests only when the transport proves the request was never sent.
|
||||||
|
- Honor valid `Retry-After` delay-seconds and HTTP-date values, and release retryable responses before sleeping.
|
||||||
- Avoid compatibility shims without a named public contract.
|
- Avoid compatibility shims without a named public contract.
|
||||||
- Preserve the minimum runtime versions declared by each package.
|
- Preserve the minimum runtime versions declared by each package.
|
||||||
|
|
||||||
|
|||||||
@@ -129,11 +129,15 @@ client = ElydoraClient(
|
|||||||
private_key="<seed>", # Base64url-encoded Ed25519 seed
|
private_key="<seed>", # Base64url-encoded Ed25519 seed
|
||||||
base_url="https://...", # API base URL (default: https://api.elydora.com)
|
base_url="https://...", # API base URL (default: https://api.elydora.com)
|
||||||
ttl_ms=30000, # Operation TTL in ms (default: 30000)
|
ttl_ms=30000, # Operation TTL in ms (default: 30000)
|
||||||
max_retries=3, # Max retries on transient failures (default: 3)
|
max_retries=3, # Retries after the initial attempt (default: 3)
|
||||||
token="<api-token>", # Optional API token
|
token="<api-token>", # Optional API token
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Automatic retries apply to RFC-idempotent methods.
|
||||||
|
Connection failures that prove a request was never sent can also retry non-idempotent methods.
|
||||||
|
Retryable responses honor both `Retry-After` seconds and HTTP-date values.
|
||||||
|
|
||||||
### Authentication
|
### Authentication
|
||||||
|
|
||||||
Authentication uses Better Auth. Register and sign in via the Elydora console or the Better Auth endpoints, then issue a long-lived API token for SDK use:
|
Authentication uses Better Auth. Register and sign in via the Elydora console or the Better Auth endpoints, then issue a long-lived API token for SDK use:
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
"""Asynchronous HTTP execution with the Elydora retry contract."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
from typing import Any, Awaitable, Callable, Dict, Optional
|
||||||
|
|
||||||
|
import aiohttp
|
||||||
|
|
||||||
|
from ._retry import can_retry, log_retry, retry_delay_seconds, should_retry_response
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
async def request_with_retries(
|
||||||
|
session: aiohttp.ClientSession,
|
||||||
|
method: str,
|
||||||
|
url: str,
|
||||||
|
*,
|
||||||
|
path: str,
|
||||||
|
max_retries: int,
|
||||||
|
response_handler: Callable[[aiohttp.ClientResponse], Awaitable[Any]],
|
||||||
|
json_body: Any = None,
|
||||||
|
params: Optional[Dict[str, str]] = None,
|
||||||
|
headers: Optional[Dict[str, str]] = None,
|
||||||
|
) -> Any:
|
||||||
|
retry_count = 0
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
delay: Optional[float] = None
|
||||||
|
reason = ""
|
||||||
|
async with session.request(
|
||||||
|
method,
|
||||||
|
url,
|
||||||
|
json=json_body,
|
||||||
|
params=params,
|
||||||
|
headers=headers,
|
||||||
|
timeout=aiohttp.ClientTimeout(total=30),
|
||||||
|
) as response:
|
||||||
|
if should_retry_response(
|
||||||
|
method, response.status, retry_count, max_retries
|
||||||
|
):
|
||||||
|
delay = retry_delay_seconds(
|
||||||
|
retry_count, response.headers.get("Retry-After")
|
||||||
|
)
|
||||||
|
reason = f"http_{response.status}"
|
||||||
|
else:
|
||||||
|
return await response_handler(response)
|
||||||
|
if delay is not None:
|
||||||
|
log_retry(logger, method, path, retry_count, max_retries, delay, reason)
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
retry_count += 1
|
||||||
|
continue
|
||||||
|
except aiohttp.ClientConnectorError as error:
|
||||||
|
if can_retry(
|
||||||
|
method,
|
||||||
|
retry_count,
|
||||||
|
max_retries,
|
||||||
|
request_known_unprocessed=True,
|
||||||
|
):
|
||||||
|
delay = retry_delay_seconds(retry_count)
|
||||||
|
log_retry(
|
||||||
|
logger,
|
||||||
|
method,
|
||||||
|
path,
|
||||||
|
retry_count,
|
||||||
|
max_retries,
|
||||||
|
delay,
|
||||||
|
type(error).__name__,
|
||||||
|
)
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
retry_count += 1
|
||||||
|
continue
|
||||||
|
raise
|
||||||
|
except (aiohttp.ClientConnectionError, asyncio.TimeoutError) as error:
|
||||||
|
if can_retry(method, retry_count, max_retries):
|
||||||
|
delay = retry_delay_seconds(retry_count)
|
||||||
|
log_retry(
|
||||||
|
logger,
|
||||||
|
method,
|
||||||
|
path,
|
||||||
|
retry_count,
|
||||||
|
max_retries,
|
||||||
|
delay,
|
||||||
|
type(error).__name__,
|
||||||
|
)
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
retry_count += 1
|
||||||
|
continue
|
||||||
|
raise
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
"""Shared retry policy for the synchronous and asynchronous clients."""
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from email.utils import parsedate_to_datetime
|
||||||
|
from logging import Logger
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
IDEMPOTENT_METHODS = frozenset({"DELETE", "GET", "HEAD", "OPTIONS", "PUT"})
|
||||||
|
RETRYABLE_STATUS_CODES = frozenset({408, 429, 500, 502, 503, 504})
|
||||||
|
MAX_BACKOFF_SECONDS = 8
|
||||||
|
|
||||||
|
|
||||||
|
def require_max_retries(value: object) -> int:
|
||||||
|
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
||||||
|
raise ValueError("max_retries must be a non-negative integer")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def can_retry(
|
||||||
|
method: str,
|
||||||
|
retry_count: int,
|
||||||
|
max_retries: int,
|
||||||
|
*,
|
||||||
|
request_known_unprocessed: bool = False,
|
||||||
|
) -> bool:
|
||||||
|
if retry_count >= max_retries:
|
||||||
|
return False
|
||||||
|
return request_known_unprocessed or method.upper() in IDEMPOTENT_METHODS
|
||||||
|
|
||||||
|
|
||||||
|
def should_retry_response(
|
||||||
|
method: str,
|
||||||
|
status_code: int,
|
||||||
|
retry_count: int,
|
||||||
|
max_retries: int,
|
||||||
|
) -> bool:
|
||||||
|
return status_code in RETRYABLE_STATUS_CODES and can_retry(
|
||||||
|
method, retry_count, max_retries
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def retry_delay_seconds(
|
||||||
|
retry_count: int,
|
||||||
|
retry_after: Optional[str] = None,
|
||||||
|
*,
|
||||||
|
now: Optional[datetime] = None,
|
||||||
|
) -> float:
|
||||||
|
parsed_delay = _parse_retry_after(retry_after, now=now)
|
||||||
|
if parsed_delay is not None:
|
||||||
|
return parsed_delay
|
||||||
|
return float(min(2**retry_count, MAX_BACKOFF_SECONDS))
|
||||||
|
|
||||||
|
|
||||||
|
def log_retry(
|
||||||
|
logger: Logger,
|
||||||
|
method: str,
|
||||||
|
path: str,
|
||||||
|
retry_count: int,
|
||||||
|
max_retries: int,
|
||||||
|
delay: float,
|
||||||
|
reason: str,
|
||||||
|
) -> None:
|
||||||
|
logger.warning(
|
||||||
|
"Retrying Elydora request method=%s path=%s retry=%d/%d "
|
||||||
|
"delay_seconds=%s reason=%s",
|
||||||
|
method.upper(),
|
||||||
|
path,
|
||||||
|
retry_count + 1,
|
||||||
|
max_retries,
|
||||||
|
delay,
|
||||||
|
reason,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_retry_after(
|
||||||
|
value: Optional[str],
|
||||||
|
*,
|
||||||
|
now: Optional[datetime],
|
||||||
|
) -> Optional[float]:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
normalized = value.strip()
|
||||||
|
if normalized.isascii() and normalized.isdecimal():
|
||||||
|
return float(int(normalized))
|
||||||
|
|
||||||
|
try:
|
||||||
|
retry_at = parsedate_to_datetime(normalized)
|
||||||
|
except (TypeError, ValueError, OverflowError):
|
||||||
|
return None
|
||||||
|
if retry_at.tzinfo is None:
|
||||||
|
retry_at = retry_at.replace(tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
current_time = now or datetime.now(timezone.utc)
|
||||||
|
return max(0.0, (retry_at - current_time).total_seconds())
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
"""Synchronous HTTP execution with the Elydora retry contract."""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
from typing import Any, Callable, Dict, Optional
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from ._retry import can_retry, log_retry, retry_delay_seconds, should_retry_response
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def request_with_retries(
|
||||||
|
session: requests.Session,
|
||||||
|
method: str,
|
||||||
|
url: str,
|
||||||
|
*,
|
||||||
|
path: str,
|
||||||
|
max_retries: int,
|
||||||
|
response_handler: Callable[[requests.Response], Any],
|
||||||
|
json_body: Any = None,
|
||||||
|
params: Optional[Dict[str, str]] = None,
|
||||||
|
headers: Optional[Dict[str, str]] = None,
|
||||||
|
) -> Any:
|
||||||
|
retry_count = 0
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
response = session.request(
|
||||||
|
method,
|
||||||
|
url,
|
||||||
|
json=json_body,
|
||||||
|
params=params,
|
||||||
|
headers=headers,
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
if should_retry_response(
|
||||||
|
method, response.status_code, retry_count, max_retries
|
||||||
|
):
|
||||||
|
delay = retry_delay_seconds(
|
||||||
|
retry_count, response.headers.get("Retry-After")
|
||||||
|
)
|
||||||
|
response.close()
|
||||||
|
log_retry(
|
||||||
|
logger,
|
||||||
|
method,
|
||||||
|
path,
|
||||||
|
retry_count,
|
||||||
|
max_retries,
|
||||||
|
delay,
|
||||||
|
f"http_{response.status_code}",
|
||||||
|
)
|
||||||
|
time.sleep(delay)
|
||||||
|
retry_count += 1
|
||||||
|
continue
|
||||||
|
return response_handler(response)
|
||||||
|
except requests.exceptions.ConnectTimeout as error:
|
||||||
|
if can_retry(
|
||||||
|
method,
|
||||||
|
retry_count,
|
||||||
|
max_retries,
|
||||||
|
request_known_unprocessed=True,
|
||||||
|
):
|
||||||
|
delay = retry_delay_seconds(retry_count)
|
||||||
|
log_retry(
|
||||||
|
logger,
|
||||||
|
method,
|
||||||
|
path,
|
||||||
|
retry_count,
|
||||||
|
max_retries,
|
||||||
|
delay,
|
||||||
|
type(error).__name__,
|
||||||
|
)
|
||||||
|
time.sleep(delay)
|
||||||
|
retry_count += 1
|
||||||
|
continue
|
||||||
|
raise
|
||||||
|
except (
|
||||||
|
requests.exceptions.ConnectionError,
|
||||||
|
requests.exceptions.Timeout,
|
||||||
|
) as error:
|
||||||
|
if can_retry(method, retry_count, max_retries):
|
||||||
|
delay = retry_delay_seconds(retry_count)
|
||||||
|
log_retry(
|
||||||
|
logger,
|
||||||
|
method,
|
||||||
|
path,
|
||||||
|
retry_count,
|
||||||
|
max_retries,
|
||||||
|
delay,
|
||||||
|
type(error).__name__,
|
||||||
|
)
|
||||||
|
time.sleep(delay)
|
||||||
|
retry_count += 1
|
||||||
|
continue
|
||||||
|
raise
|
||||||
@@ -3,12 +3,13 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import time
|
import time
|
||||||
import asyncio
|
|
||||||
import warnings
|
import warnings
|
||||||
from typing import Any, Dict, List, Optional, Union
|
from typing import Any, Dict, Optional, Union
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
|
|
||||||
|
from ._async_http import request_with_retries
|
||||||
|
from ._retry import require_max_retries
|
||||||
from .crypto import compute_chain_hash, compute_payload_hash, sign_eor
|
from .crypto import compute_chain_hash, compute_payload_hash, sign_eor
|
||||||
from .errors import ElydoraError
|
from .errors import ElydoraError
|
||||||
from .integration_types import require_integration_type
|
from .integration_types import require_integration_type
|
||||||
@@ -71,7 +72,7 @@ class AsyncElydoraClient:
|
|||||||
self.private_key = private_key
|
self.private_key = private_key
|
||||||
self.base_url = base_url.rstrip("/")
|
self.base_url = base_url.rstrip("/")
|
||||||
self.ttl_ms = ttl_ms
|
self.ttl_ms = ttl_ms
|
||||||
self.max_retries = max_retries
|
self.max_retries = require_max_retries(max_retries)
|
||||||
self.token = token
|
self.token = token
|
||||||
|
|
||||||
self._prev_chain_hash = GENESIS_CHAIN_HASH
|
self._prev_chain_hash = GENESIS_CHAIN_HASH
|
||||||
@@ -119,23 +120,17 @@ class AsyncElydoraClient:
|
|||||||
hdrs = headers or self._headers()
|
hdrs = headers or self._headers()
|
||||||
session = await self._get_session()
|
session = await self._get_session()
|
||||||
|
|
||||||
last_exc: Optional[Exception] = None
|
return await request_with_retries(
|
||||||
for attempt in range(self.max_retries):
|
session,
|
||||||
try:
|
method,
|
||||||
async with session.request(
|
url,
|
||||||
method, url, json=json_body, params=params, headers=hdrs, timeout=aiohttp.ClientTimeout(total=30)
|
path=path,
|
||||||
) as resp:
|
max_retries=self.max_retries,
|
||||||
return await self._handle_response(resp)
|
response_handler=self._handle_response,
|
||||||
except ElydoraError:
|
json_body=json_body,
|
||||||
raise
|
params=params,
|
||||||
except (aiohttp.ClientConnectionError, asyncio.TimeoutError) as exc:
|
headers=hdrs,
|
||||||
last_exc = exc
|
)
|
||||||
if attempt < self.max_retries - 1:
|
|
||||||
await asyncio.sleep(min(2 ** attempt, 8))
|
|
||||||
continue
|
|
||||||
raise
|
|
||||||
|
|
||||||
raise last_exc # type: ignore[misc]
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
async def _handle_response(resp: aiohttp.ClientResponse) -> Any:
|
async def _handle_response(resp: aiohttp.ClientResponse) -> Any:
|
||||||
|
|||||||
@@ -4,10 +4,12 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import time
|
import time
|
||||||
import warnings
|
import warnings
|
||||||
from typing import Any, Dict, List, Optional, Union
|
from typing import Any, Dict, Optional, Union
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
|
from ._retry import require_max_retries
|
||||||
|
from ._sync_http import request_with_retries
|
||||||
from .crypto import compute_chain_hash, compute_payload_hash, sign_eor
|
from .crypto import compute_chain_hash, compute_payload_hash, sign_eor
|
||||||
from .errors import ElydoraError
|
from .errors import ElydoraError
|
||||||
from .integration_types import require_integration_type
|
from .integration_types import require_integration_type
|
||||||
@@ -70,7 +72,7 @@ class ElydoraClient:
|
|||||||
self.private_key = private_key
|
self.private_key = private_key
|
||||||
self.base_url = base_url.rstrip("/")
|
self.base_url = base_url.rstrip("/")
|
||||||
self.ttl_ms = ttl_ms
|
self.ttl_ms = ttl_ms
|
||||||
self.max_retries = max_retries
|
self.max_retries = require_max_retries(max_retries)
|
||||||
self.token = token
|
self.token = token
|
||||||
|
|
||||||
self._prev_chain_hash = GENESIS_CHAIN_HASH
|
self._prev_chain_hash = GENESIS_CHAIN_HASH
|
||||||
@@ -107,33 +109,17 @@ class ElydoraClient:
|
|||||||
url = f"{self.base_url}{path}"
|
url = f"{self.base_url}{path}"
|
||||||
hdrs = headers or self._headers()
|
hdrs = headers or self._headers()
|
||||||
|
|
||||||
last_exc: Optional[Exception] = None
|
return request_with_retries(
|
||||||
for attempt in range(self.max_retries):
|
self._session,
|
||||||
try:
|
method,
|
||||||
resp = self._session.request(
|
url,
|
||||||
method, url, json=json_body, params=params, headers=hdrs, timeout=30
|
path=path,
|
||||||
|
max_retries=self.max_retries,
|
||||||
|
response_handler=self._handle_response,
|
||||||
|
json_body=json_body,
|
||||||
|
params=params,
|
||||||
|
headers=hdrs,
|
||||||
)
|
)
|
||||||
# Retry on 429 or 5xx
|
|
||||||
if resp.status_code == 429 or resp.status_code >= 500:
|
|
||||||
if attempt < self.max_retries - 1:
|
|
||||||
retry_after = resp.headers.get("Retry-After")
|
|
||||||
if retry_after and retry_after.isdigit():
|
|
||||||
delay = int(retry_after)
|
|
||||||
else:
|
|
||||||
delay = min(2 ** attempt, 8)
|
|
||||||
time.sleep(delay)
|
|
||||||
continue
|
|
||||||
return self._handle_response(resp)
|
|
||||||
except ElydoraError:
|
|
||||||
raise
|
|
||||||
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as exc:
|
|
||||||
last_exc = exc
|
|
||||||
if attempt < self.max_retries - 1:
|
|
||||||
time.sleep(min(2 ** attempt, 8))
|
|
||||||
continue
|
|
||||||
raise
|
|
||||||
|
|
||||||
raise last_exc # type: ignore[misc]
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _handle_response(resp: requests.Response) -> Any:
|
def _handle_response(resp: requests.Response) -> Any:
|
||||||
|
|||||||
@@ -0,0 +1,249 @@
|
|||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import threading
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from email.utils import format_datetime
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
from typing import Dict, List, Tuple, cast
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from elydora import AsyncElydoraClient, ElydoraClient, ElydoraError
|
||||||
|
from elydora._retry import (
|
||||||
|
can_retry,
|
||||||
|
retry_delay_seconds,
|
||||||
|
should_retry_response,
|
||||||
|
)
|
||||||
|
|
||||||
|
ScriptedResponse = Tuple[int, Dict[str, str], object]
|
||||||
|
|
||||||
|
|
||||||
|
class ScriptedServer(ThreadingHTTPServer):
|
||||||
|
responses: List[ScriptedResponse]
|
||||||
|
methods: List[str]
|
||||||
|
|
||||||
|
|
||||||
|
class ScriptedHandler(BaseHTTPRequestHandler):
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
self._respond()
|
||||||
|
|
||||||
|
def do_POST(self) -> None:
|
||||||
|
content_length = int(self.headers.get("Content-Length", "0"))
|
||||||
|
if content_length:
|
||||||
|
self.rfile.read(content_length)
|
||||||
|
self._respond()
|
||||||
|
|
||||||
|
def _respond(self) -> None:
|
||||||
|
server = cast(ScriptedServer, self.server)
|
||||||
|
server.methods.append(self.command)
|
||||||
|
status, headers, body = server.responses.pop(0)
|
||||||
|
encoded = json.dumps(body).encode()
|
||||||
|
self.send_response(status)
|
||||||
|
for name, value in headers.items():
|
||||||
|
self.send_header(name, value)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Content-Length", str(len(encoded)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(encoded)
|
||||||
|
|
||||||
|
def log_message(self, _format: str, *args: object) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def scripted_server():
|
||||||
|
server = ScriptedServer(("127.0.0.1", 0), ScriptedHandler)
|
||||||
|
server.responses = []
|
||||||
|
server.methods = []
|
||||||
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||||
|
thread.start()
|
||||||
|
try:
|
||||||
|
yield server
|
||||||
|
finally:
|
||||||
|
server.shutdown()
|
||||||
|
server.server_close()
|
||||||
|
thread.join()
|
||||||
|
|
||||||
|
|
||||||
|
def base_url(server: ScriptedServer) -> str:
|
||||||
|
host, port = server.server_address
|
||||||
|
return f"http://{host}:{port}"
|
||||||
|
|
||||||
|
|
||||||
|
def error_response(status: int, retry_after: str = "") -> ScriptedResponse:
|
||||||
|
headers = {"Retry-After": retry_after} if retry_after else {}
|
||||||
|
return (
|
||||||
|
status,
|
||||||
|
headers,
|
||||||
|
{"error": {"code": "INTERNAL_ERROR", "message": "temporary failure"}},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_retry_policy_matches_http_semantics() -> None:
|
||||||
|
assert should_retry_response("GET", 503, 0, 1)
|
||||||
|
assert should_retry_response("delete", 429, 0, 1)
|
||||||
|
assert not should_retry_response("POST", 503, 0, 1)
|
||||||
|
assert not should_retry_response("GET", 501, 0, 1)
|
||||||
|
assert not should_retry_response("GET", 503, 1, 1)
|
||||||
|
assert can_retry("POST", 0, 1, request_known_unprocessed=True)
|
||||||
|
|
||||||
|
|
||||||
|
def test_retry_after_supports_seconds_and_http_dates() -> None:
|
||||||
|
now = datetime(2026, 7, 19, 12, 0, tzinfo=timezone.utc)
|
||||||
|
retry_at = format_datetime(now + timedelta(seconds=30), usegmt=True)
|
||||||
|
|
||||||
|
assert retry_delay_seconds(0, "15", now=now) == 15
|
||||||
|
assert retry_delay_seconds(0, retry_at, now=now) == 30
|
||||||
|
assert retry_delay_seconds(2, "invalid", now=now) == 4
|
||||||
|
assert retry_delay_seconds(4, None, now=now) == 8
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("value", [-1, True, 1.5, "2"])
|
||||||
|
def test_clients_reject_invalid_retry_configuration(value: object) -> None:
|
||||||
|
with pytest.raises(ValueError, match="non-negative integer"):
|
||||||
|
ElydoraClient("org", "agent", "key", max_retries=value) # type: ignore[arg-type]
|
||||||
|
with pytest.raises(ValueError, match="non-negative integer"):
|
||||||
|
AsyncElydoraClient("org", "agent", "key", max_retries=value) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def test_zero_retries_still_performs_one_sync_request(
|
||||||
|
scripted_server: ScriptedServer,
|
||||||
|
) -> None:
|
||||||
|
scripted_server.responses = [(200, {}, {"user_id": "user-1"})]
|
||||||
|
client = ElydoraClient(
|
||||||
|
"org", "agent", "key", base_url=base_url(scripted_server), max_retries=0
|
||||||
|
)
|
||||||
|
|
||||||
|
assert client.get_me()["user_id"] == "user-1"
|
||||||
|
assert scripted_server.methods == ["GET"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_zero_retries_still_performs_one_async_request(
|
||||||
|
scripted_server: ScriptedServer,
|
||||||
|
) -> None:
|
||||||
|
scripted_server.responses = [(200, {}, {"user_id": "user-1"})]
|
||||||
|
client = AsyncElydoraClient(
|
||||||
|
"org", "agent", "key", base_url=base_url(scripted_server), max_retries=0
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
response = await client.get_me()
|
||||||
|
assert response["user_id"] == "user-1"
|
||||||
|
assert scripted_server.methods == ["GET"]
|
||||||
|
finally:
|
||||||
|
await client.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_sync_client_retries_idempotent_responses(
|
||||||
|
scripted_server: ScriptedServer,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
caplog: pytest.LogCaptureFixture,
|
||||||
|
) -> None:
|
||||||
|
scripted_server.responses = [
|
||||||
|
error_response(503, "0"),
|
||||||
|
error_response(500),
|
||||||
|
(200, {}, {"user_id": "user-1"}),
|
||||||
|
]
|
||||||
|
delays: List[float] = []
|
||||||
|
monkeypatch.setattr("elydora._sync_http.time.sleep", delays.append)
|
||||||
|
caplog.set_level(logging.WARNING, logger="elydora._sync_http")
|
||||||
|
client = ElydoraClient(
|
||||||
|
"org", "agent", "key", base_url=base_url(scripted_server), max_retries=2
|
||||||
|
)
|
||||||
|
|
||||||
|
assert client.get_me()["user_id"] == "user-1"
|
||||||
|
assert scripted_server.methods == ["GET", "GET", "GET"]
|
||||||
|
assert delays == [0.0, 2.0]
|
||||||
|
retry_logs = [
|
||||||
|
record.getMessage()
|
||||||
|
for record in caplog.records
|
||||||
|
if record.name == "elydora._sync_http"
|
||||||
|
]
|
||||||
|
assert "retry=1/2 delay_seconds=0.0 reason=http_503" in retry_logs[0]
|
||||||
|
assert "retry=2/2 delay_seconds=2.0 reason=http_500" in retry_logs[1]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_async_client_retries_idempotent_responses(
|
||||||
|
scripted_server: ScriptedServer,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
caplog: pytest.LogCaptureFixture,
|
||||||
|
) -> None:
|
||||||
|
scripted_server.responses = [
|
||||||
|
error_response(429, "0"),
|
||||||
|
error_response(502),
|
||||||
|
(200, {}, {"user_id": "user-1"}),
|
||||||
|
]
|
||||||
|
delays: List[float] = []
|
||||||
|
|
||||||
|
async def record_sleep(delay: float) -> None:
|
||||||
|
delays.append(delay)
|
||||||
|
|
||||||
|
monkeypatch.setattr("elydora._async_http.asyncio.sleep", record_sleep)
|
||||||
|
caplog.set_level(logging.WARNING, logger="elydora._async_http")
|
||||||
|
client = AsyncElydoraClient(
|
||||||
|
"org", "agent", "key", base_url=base_url(scripted_server), max_retries=2
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
response = await client.get_me()
|
||||||
|
assert response["user_id"] == "user-1"
|
||||||
|
assert scripted_server.methods == ["GET", "GET", "GET"]
|
||||||
|
assert delays == [0.0, 2.0]
|
||||||
|
retry_logs = [
|
||||||
|
record.getMessage()
|
||||||
|
for record in caplog.records
|
||||||
|
if record.name == "elydora._async_http"
|
||||||
|
]
|
||||||
|
assert "retry=1/2 delay_seconds=0.0 reason=http_429" in retry_logs[0]
|
||||||
|
assert "retry=2/2 delay_seconds=2.0 reason=http_502" in retry_logs[1]
|
||||||
|
finally:
|
||||||
|
await client.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_sync_client_does_not_replay_non_idempotent_requests(
|
||||||
|
scripted_server: ScriptedServer,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
scripted_server.responses = [
|
||||||
|
error_response(503),
|
||||||
|
(200, {}, {"token": "duplicate"}),
|
||||||
|
]
|
||||||
|
delays: List[float] = []
|
||||||
|
monkeypatch.setattr("elydora._sync_http.time.sleep", delays.append)
|
||||||
|
client = ElydoraClient(
|
||||||
|
"org", "agent", "key", base_url=base_url(scripted_server), max_retries=3
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ElydoraError) as error:
|
||||||
|
client.issue_token()
|
||||||
|
assert error.value.status_code == 503
|
||||||
|
assert scripted_server.methods == ["POST"]
|
||||||
|
assert delays == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_async_client_does_not_replay_non_idempotent_requests(
|
||||||
|
scripted_server: ScriptedServer,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
scripted_server.responses = [
|
||||||
|
error_response(503),
|
||||||
|
(200, {}, {"token": "duplicate"}),
|
||||||
|
]
|
||||||
|
delays: List[float] = []
|
||||||
|
|
||||||
|
async def record_sleep(delay: float) -> None:
|
||||||
|
delays.append(delay)
|
||||||
|
|
||||||
|
monkeypatch.setattr("elydora._async_http.asyncio.sleep", record_sleep)
|
||||||
|
client = AsyncElydoraClient(
|
||||||
|
"org", "agent", "key", base_url=base_url(scripted_server), max_retries=3
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with pytest.raises(ElydoraError) as error:
|
||||||
|
await client.issue_token()
|
||||||
|
assert error.value.status_code == 503
|
||||||
|
assert scripted_server.methods == ["POST"]
|
||||||
|
assert delays == []
|
||||||
|
finally:
|
||||||
|
await client.close()
|
||||||
Reference in new issue
Block a user