mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
fix: a body that stops mid-read is transient, and a round that dies still says so
Round 5 died on `http.client.IncompleteRead(0 bytes read)` and left no record at all. Two defects, both ours. The retry classified only connection-level failures. IncompleteRead and RemoteDisconnected arrive while the *body* is being read, after the connection succeeded, so they are not URLError -- they escaped both the retry and the error wrapper and walked out of the round. The network is allowed to be unreliable; losing a paid round to it is not the provider's defect. And the exception path was wrapped around the stages that come *after* the proposal is applied, because those are the ones with something to revert. The stages before it had already spent money, and a round that leaves no record is a round nobody can account for. Every outcome is journalled now, with the exception type in the reason. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
cde57e4293
commit
3ebcc6729a
@@ -406,10 +406,25 @@ def run_learning_round(
|
||||
spent = spent_usd() if spent_usd is not None else 0.0
|
||||
return not within_budget(spent, cost_cap_usd)
|
||||
|
||||
before_scan = scan_factory(config)
|
||||
before_train = _score(batch, before_scan, family)
|
||||
before_hold = _score(holdout, before_scan, family)
|
||||
record = _round_record(round_number, family, None, taxonomy, config)
|
||||
try:
|
||||
before_scan = scan_factory(config)
|
||||
before_train = _score(batch, before_scan, family)
|
||||
before_hold = _score(holdout, before_scan, family)
|
||||
except Exception as exc:
|
||||
# These stages run before anything is applied, so there is nothing to revert — but they have already spent
|
||||
# money, and a round that leaves no record is a round nobody can account for. Measured: round 5 died on a
|
||||
# network read during its first stage and vanished.
|
||||
_finish(
|
||||
journal,
|
||||
directory,
|
||||
record,
|
||||
outcome="reverted",
|
||||
reason=f"stage_failed: {type(exc).__name__}",
|
||||
cost=spent_usd() if spent_usd is not None else 0.0,
|
||||
trajectories=trajectories,
|
||||
)
|
||||
raise
|
||||
if _over_cap():
|
||||
return _finish(
|
||||
journal, directory, record, outcome="reverted_cost", reason="cost_cap_exceeded", cost=spent, trajectories=trajectories
|
||||
@@ -551,7 +566,7 @@ def run_learning_round(
|
||||
version=version,
|
||||
trajectories=trajectories,
|
||||
)
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
# A stage that raises has already spent money and already changed the tree. Both facts belong in the
|
||||
# journal before the exception leaves, or the next round starts from a state nothing describes.
|
||||
snapshot.restore()
|
||||
@@ -560,7 +575,7 @@ def run_learning_round(
|
||||
directory,
|
||||
record,
|
||||
outcome="reverted",
|
||||
reason="stage_failed",
|
||||
reason=f"stage_failed: {type(exc).__name__}",
|
||||
proposal=proposal,
|
||||
cost=spent,
|
||||
trajectories=trajectories,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -20,10 +21,14 @@ _T = TypeVar("_T")
|
||||
|
||||
|
||||
def _is_transient(exc: Exception) -> bool:
|
||||
"""A network failure worth retrying (rate-limit, 5xx, connection error, timeout)."""
|
||||
"""A network failure worth retrying: rate-limit, 5xx, connection error, timeout, or a body that stopped.
|
||||
|
||||
`http.client.IncompleteRead` and `RemoteDisconnected` arrive while the *body* is being read, after the
|
||||
connection succeeded, so they are not `URLError`. Measured: one of them ended an evolve round that had already
|
||||
been paid for."""
|
||||
if isinstance(exc, urllib.error.HTTPError):
|
||||
return exc.code in _TRANSIENT_STATUS
|
||||
return isinstance(exc, urllib.error.URLError | TimeoutError)
|
||||
return isinstance(exc, urllib.error.URLError | TimeoutError | http.client.HTTPException)
|
||||
|
||||
|
||||
def call_with_retry(
|
||||
@@ -108,8 +113,8 @@ class OpenRouterChatClient:
|
||||
|
||||
try:
|
||||
return call_with_retry(_do, attempts=self.max_attempts, base_delay=self.retry_base_delay)
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
|
||||
raise OpenRouterError(f"OpenRouter chat request failed: {exc}") from exc
|
||||
except (urllib.error.URLError, TimeoutError, http.client.HTTPException, json.JSONDecodeError) as exc:
|
||||
raise OpenRouterError(f"OpenRouter chat request failed: {type(exc).__name__}: {exc}") from exc
|
||||
|
||||
def complete_json(self, *, model: str, messages: list[dict[str, str]], timeout_seconds: int = 60) -> object:
|
||||
message = self.complete_chat(model=model, messages=messages, timeout_seconds=timeout_seconds)
|
||||
@@ -151,8 +156,8 @@ class OpenRouterEmbeddingClient:
|
||||
|
||||
try:
|
||||
response_payload = call_with_retry(_do, attempts=self.max_attempts, base_delay=self.retry_base_delay)
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
|
||||
raise OpenRouterError(f"OpenRouter embedding request failed: {exc}") from exc
|
||||
except (urllib.error.URLError, TimeoutError, http.client.HTTPException, json.JSONDecodeError) as exc:
|
||||
raise OpenRouterError(f"OpenRouter embedding request failed: {type(exc).__name__}: {exc}") from exc
|
||||
return parse_embedding_response(response_payload)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user