fix: a body that stops mid-read is transient, and a round that dies still says so

Round 5 died on `http.client.IncompleteRead(0 bytes read)` and left no record
at all. Two defects, both ours.

The retry classified only connection-level failures. IncompleteRead and
RemoteDisconnected arrive while the *body* is being read, after the connection
succeeded, so they are not URLError -- they escaped both the retry and the
error wrapper and walked out of the round. The network is allowed to be
unreliable; losing a paid round to it is not the provider's defect.

And the exception path was wrapped around the stages that come *after* the
proposal is applied, because those are the ones with something to revert. The
stages before it had already spent money, and a round that leaves no record is
a round nobody can account for. Every outcome is journalled now, with the
exception type in the reason.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
Marius Ciepluch
2026-09-07 07:37:41 +02:00
co-authored by Claude Opus 5
parent cde57e4293
commit 3ebcc6729a
4 changed files with 102 additions and 12 deletions
+20 -5
View File
@@ -406,10 +406,25 @@ def run_learning_round(
spent = spent_usd() if spent_usd is not None else 0.0
return not within_budget(spent, cost_cap_usd)
before_scan = scan_factory(config)
before_train = _score(batch, before_scan, family)
before_hold = _score(holdout, before_scan, family)
record = _round_record(round_number, family, None, taxonomy, config)
try:
before_scan = scan_factory(config)
before_train = _score(batch, before_scan, family)
before_hold = _score(holdout, before_scan, family)
except Exception as exc:
# These stages run before anything is applied, so there is nothing to revert — but they have already spent
# money, and a round that leaves no record is a round nobody can account for. Measured: round 5 died on a
# network read during its first stage and vanished.
_finish(
journal,
directory,
record,
outcome="reverted",
reason=f"stage_failed: {type(exc).__name__}",
cost=spent_usd() if spent_usd is not None else 0.0,
trajectories=trajectories,
)
raise
if _over_cap():
return _finish(
journal, directory, record, outcome="reverted_cost", reason="cost_cap_exceeded", cost=spent, trajectories=trajectories
@@ -551,7 +566,7 @@ def run_learning_round(
version=version,
trajectories=trajectories,
)
except Exception:
except Exception as exc:
# A stage that raises has already spent money and already changed the tree. Both facts belong in the
# journal before the exception leaves, or the next round starts from a state nothing describes.
snapshot.restore()
@@ -560,7 +575,7 @@ def run_learning_round(
directory,
record,
outcome="reverted",
reason="stage_failed",
reason=f"stage_failed: {type(exc).__name__}",
proposal=proposal,
cost=spent,
trajectories=trajectories,
+11 -6
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import http.client
import json
import os
import re
@@ -20,10 +21,14 @@ _T = TypeVar("_T")
def _is_transient(exc: Exception) -> bool:
"""A network failure worth retrying (rate-limit, 5xx, connection error, timeout)."""
"""A network failure worth retrying: rate-limit, 5xx, connection error, timeout, or a body that stopped.
`http.client.IncompleteRead` and `RemoteDisconnected` arrive while the *body* is being read, after the
connection succeeded, so they are not `URLError`. Measured: one of them ended an evolve round that had already
been paid for."""
if isinstance(exc, urllib.error.HTTPError):
return exc.code in _TRANSIENT_STATUS
return isinstance(exc, urllib.error.URLError | TimeoutError)
return isinstance(exc, urllib.error.URLError | TimeoutError | http.client.HTTPException)
def call_with_retry(
@@ -108,8 +113,8 @@ class OpenRouterChatClient:
try:
return call_with_retry(_do, attempts=self.max_attempts, base_delay=self.retry_base_delay)
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
raise OpenRouterError(f"OpenRouter chat request failed: {exc}") from exc
except (urllib.error.URLError, TimeoutError, http.client.HTTPException, json.JSONDecodeError) as exc:
raise OpenRouterError(f"OpenRouter chat request failed: {type(exc).__name__}: {exc}") from exc
def complete_json(self, *, model: str, messages: list[dict[str, str]], timeout_seconds: int = 60) -> object:
message = self.complete_chat(model=model, messages=messages, timeout_seconds=timeout_seconds)
@@ -151,8 +156,8 @@ class OpenRouterEmbeddingClient:
try:
response_payload = call_with_retry(_do, attempts=self.max_attempts, base_delay=self.retry_base_delay)
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
raise OpenRouterError(f"OpenRouter embedding request failed: {exc}") from exc
except (urllib.error.URLError, TimeoutError, http.client.HTTPException, json.JSONDecodeError) as exc:
raise OpenRouterError(f"OpenRouter embedding request failed: {type(exc).__name__}: {exc}") from exc
return parse_embedding_response(response_payload)