mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-17 03:59:28 +00:00
fix: a body that stops mid-read is transient, and a round that dies still says so
Round 5 died on `http.client.IncompleteRead(0 bytes read)` and left no record at all. Two defects, both ours. The retry classified only connection-level failures. IncompleteRead and RemoteDisconnected arrive while the *body* is being read, after the connection succeeded, so they are not URLError -- they escaped both the retry and the error wrapper and walked out of the round. The network is allowed to be unreliable; losing a paid round to it is not the provider's defect. And the exception path was wrapped around the stages that come *after* the proposal is applied, because those are the ones with something to revert. The stages before it had already spent money, and a round that leaves no record is a round nobody can account for. Every outcome is journalled now, with the exception type in the reason. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
cde57e4293
commit
3ebcc6729a
@@ -224,7 +224,7 @@ def test_a_stage_that_raises_is_reverted_journalled_and_re_raised(world) -> None
|
||||
_run(world, _proposal(), build)
|
||||
assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n"
|
||||
outcomes = [(item.outcome, item.reason) for item in world["journal"].rounds()]
|
||||
assert outcomes == [("reverted", "stage_failed")]
|
||||
assert outcomes == [("reverted", "stage_failed: RuntimeError")] # the exception type is part of the record
|
||||
assert (world["out"] / "rounds" / "1" / "scores.json").is_file()
|
||||
|
||||
|
||||
@@ -406,3 +406,23 @@ def test_a_round_journals_why_the_proposer_had_nothing_to_say(world) -> None: #
|
||||
)
|
||||
assert record.outcome == "rejected"
|
||||
assert record.reason == "no_proposal: meta_agent_failed: RuntimeError"
|
||||
|
||||
|
||||
def test_a_round_that_dies_before_it_applies_anything_is_still_journalled(world) -> None: # type: ignore[no-untyped-def]
|
||||
"""Measured: round 5 died on a network read during its *first* stage and left no record at all.
|
||||
|
||||
The exception path was wrapped around the stages after the proposal is applied, because those are the ones that
|
||||
have something to revert. The ones before it had already spent money."""
|
||||
|
||||
def build(config: FamilyConfig): # type: ignore[no-untyped-def]
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
raise ConnectionError("the body stopped arriving")
|
||||
|
||||
return scan
|
||||
|
||||
with pytest.raises(ConnectionError):
|
||||
_run(world, _proposal(), build)
|
||||
outcomes = [(item.outcome, item.reason) for item in world["journal"].rounds()]
|
||||
assert outcomes == [("reverted", "stage_failed: ConnectionError")]
|
||||
assert (world["out"] / "rounds" / "1").is_dir()
|
||||
assert (world["configs"] / "injection" / "checklist.md").read_text() == "- old\n"
|
||||
|
||||
@@ -63,3 +63,53 @@ def test_non_retryable_value_error_propagates_immediately() -> None:
|
||||
|
||||
with pytest.raises(OpenRouterError):
|
||||
call_with_retry(op, attempts=3, base_delay=0, sleep=_no_sleep)
|
||||
|
||||
|
||||
def test_a_body_that_stops_mid_read_is_transient() -> None:
|
||||
"""Measured: an evolve round died on `http.client.IncompleteRead(0 bytes read)`.
|
||||
|
||||
The retry classified only connection-level failures, so a response whose *body* stopped arriving escaped both
|
||||
the retry and the error wrapper, and the exception walked out of the round. The network is allowed to be
|
||||
unreliable; losing a paid round to it is our defect, not the provider's."""
|
||||
import http.client
|
||||
|
||||
from openultrasast.provider.openrouter import OpenRouterError, _is_transient, call_with_retry
|
||||
|
||||
assert _is_transient(http.client.IncompleteRead(b"")) is True
|
||||
assert _is_transient(http.client.RemoteDisconnected("closed")) is True
|
||||
assert _is_transient(ValueError("a bug in our own code")) is False
|
||||
|
||||
attempts = []
|
||||
|
||||
def flaky() -> str:
|
||||
attempts.append(1)
|
||||
if len(attempts) < 3:
|
||||
raise http.client.IncompleteRead(b"")
|
||||
return "ok"
|
||||
|
||||
assert call_with_retry(flaky, attempts=3, base_delay=0.0, sleep=lambda _s: None) == "ok"
|
||||
assert len(attempts) == 3
|
||||
assert issubclass(OpenRouterError, Exception)
|
||||
|
||||
|
||||
def test_a_read_failure_is_reported_as_a_provider_error_not_a_raw_http_exception() -> None:
|
||||
import http.client
|
||||
import urllib.error
|
||||
|
||||
from openultrasast.provider.openrouter import OpenRouterChatClient, OpenRouterError
|
||||
|
||||
client = OpenRouterChatClient(api_key="k", base_url="https://example.invalid", max_attempts=1)
|
||||
|
||||
def boom(*_args: object, **_kwargs: object) -> object:
|
||||
raise http.client.IncompleteRead(b"")
|
||||
|
||||
import openultrasast.provider.openrouter as module
|
||||
|
||||
original = module.urllib.request.urlopen
|
||||
module.urllib.request.urlopen = boom # type: ignore[assignment]
|
||||
try:
|
||||
with pytest.raises(OpenRouterError, match="IncompleteRead"):
|
||||
client.complete_chat_raw(model="m", messages=[{"role": "user", "content": "x"}])
|
||||
finally:
|
||||
module.urllib.request.urlopen = original # type: ignore[assignment]
|
||||
assert urllib.error.URLError is not None
|
||||
|
||||
Reference in New Issue
Block a user