feat(learning-harness): thinking is a measured condition, and every round

records whether its decision beat noise

Two things the maintainer was right about.

Thinking was disabled so `temperature: 0` would not be silently ignored --
a trade that only pays if it buys determinism. Round zero says it did not:
11 of 20 injection pairs disagreed with themselves across five runs, so the
variance is in the agentic path, not the decoder. We paid the reasoning cost
on a reasoning model and got nothing. It is now `[learning] thinking`,
recorded in the artifact and part of its key so the two modes cannot be read
as one number. The default stays off only because that is what round zero
was measured under.

And the deeper point: the spec built `sign_test` and `reliable_change` and
the acceptance rule calls neither -- `decide` compares raw deltas. On a
detector with that floor a raw delta is mostly noise, and a rule that
brittle is exactly what makes a deterministic decoder look necessary. It is
not; the alternative is an acceptance rule that models the score as noisy,
the way reflective-evolution and Bayesian prompt optimizers do. Changing
Req 9.5 is a maintainer's call, so this does not change the decision: every
round now journals the two-sided sign test over the pairs that moved, and
after four rounds the journal will say whether any decision could be told
from a coin flip.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

src/openultrasast/cli.py

@@ -1328,6 +1328,7 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy:
             # a run over the memory-safety slice reports nothing at all (Req 8.4).
             slices=(args.slice,) if args.slice != "all" else DEFAULT_SLICES,
             spent_usd=spent,
+            thinking=config.learning.thinking,
         )
         print(f"learning baseline cost: ${report.cost_usd:.4f}")
         print(f"learning baseline {model}: {len(report.floors)} families, k={report.k_runs} -> {out / 'baseline'}")
@@ -1343,7 +1344,7 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy:
         model=model,
         journal=journal,
         archive=Archive(out / "archive.jsonl"),
-        floors=load_noise_floors(out, model, (args.slice,) if args.slice != "all" else DEFAULT_SLICES),
+        floors=load_noise_floors(out, model, (args.slice,) if args.slice != "all" else DEFAULT_SLICES, config.learning.thinking),
         out_dir=out,
         cost_cap_usd=args.cost_cap_usd or config.learning.round_cost_cap_usd,
         minibatch=config.learning.minibatch,

src/openultrasast/config.py

@@ -150,6 +150,12 @@ class LearningConfig:
     detector_model: str | None = None
     judge_model: str | None = None
     cutoff_date: str | None = None  # rows fixed after this date are reported as a separate slice
+    # Whether the detector model reasons before answering. Off by default because that is what round zero was
+    # measured under, not because it is known to be better: the provider silently ignores `temperature` while
+    # thinking, and thinking was disabled to make `temperature: 0` mean something. It did not — the measured
+    # disagreement between runs comes from the agentic path, not the decoder — so which mode wins is a question
+    # for a second baseline rather than for a default (Req 8.3).
+    thinking: bool = False
 
 
 @dataclass(frozen=True)
@@ -391,6 +397,7 @@ def _load_learning(value: object) -> LearningConfig:
         detector_model=_string(data.get("detector_model")),
         judge_model=_string(data.get("judge_model")),
         cutoff_date=_string(data.get("cutoff_date")),
+        thinking=bool(data.get("thinking", False)),
     )
 
 

src/openultrasast/learning/acceptance.py

@@ -55,6 +55,35 @@ def decide(
     return Verdict(accepted=True, reason="improved")
 
 
+@dataclass(frozen=True)
+class Reliability:
+    """Whether a stage's movement could be told from a coin flip."""
+
+    better: int
+    worse: int
+    p_value: float
+    reliable: bool
+
+    def to_dict(self) -> dict[str, object]:
+        return {"better": self.better, "worse": self.worse, "p_value": self.p_value, "reliable": self.reliable}
+
+
+def reliability(*, better: int, worse: int, alpha: float = 0.05) -> Reliability:
+    """The two-sided sign test over the pairs that changed, for the record rather than for the decision.
+
+    `decide` compares raw deltas, as Req 9.5 specifies. On a detector whose measured floor says half its pairs
+    disagree with themselves between runs, a raw delta is mostly noise — which is what makes a deterministic
+    decoder look necessary. It is not: the alternative is an acceptance rule that models the score as noisy, the
+    way reflective-evolution and Bayesian prompt optimizers do, and keeps per-instance winners on a frontier
+    instead of demanding a single global improvement. Changing the rule is a maintainer's call, so every round
+    records this beside its decision and the journal can be read to answer the question with data.
+    """
+    from .scoring import sign_test
+
+    p_value = sign_test(better, worse)
+    return Reliability(better=better, worse=worse, p_value=p_value, reliable=bool(p_value < alpha and better != worse))
+
+
 def _gates(family: str, floors: Mapping[str, NoiseFloor]) -> bool:
     floor = floors.get(family)
     return floor.gates if floor is not None else True
@@ -114,4 +143,4 @@ def bump_version(directory: Path, version: str) -> None:
     settings.write_text("\n".join(rewritten) + "\n", encoding="utf-8")
 
 
-__all__ = ["DirectorySnapshot", "Verdict", "bump_version", "decide", "digest_configs", "within_budget"]
+__all__ = ["DirectorySnapshot", "Reliability", "Verdict", "bump_version", "decide", "digest_configs", "reliability", "within_budget"]

src/openultrasast/learning/endpoint.py

@@ -154,10 +154,16 @@ def resolve_chat_endpoint(config: ResolvedConfig, *, override: ChatClient | None
     if configured or key_env:
         key = os.environ.get(key_env or DEEPSEEK_KEY_ENV, "")
         base_url = configured or DEEPSEEK_BASE_URL
-        return _deepseek_client(key, base_url, model, provider=_provider_of(base_url))
+        return _deepseek_client(key, base_url, model, provider=_provider_of(base_url), thinking=config.learning.thinking)
     deepseek_key = os.environ.get(DEEPSEEK_KEY_ENV)
     if deepseek_key:
-        return _deepseek_client(deepseek_key, os.environ.get(DEEPSEEK_BASE_ENV, DEEPSEEK_BASE_URL), model, provider="deepseek")
+        return _deepseek_client(
+            deepseek_key,
+            os.environ.get(DEEPSEEK_BASE_ENV, DEEPSEEK_BASE_URL),
+            model,
+            provider="deepseek",
+            thinking=config.learning.thinking,
+        )
     if os.environ.get("OPENROUTER_API_KEY"):
         try:
             client = OpenRouterChatClient.from_env()
@@ -175,10 +181,10 @@ def resolve_models(config: ResolvedConfig) -> tuple[str, str]:
     return detector, judge
 
 
-def _deepseek_client(key: str, base_url: str, model: str, *, provider: Provider) -> tuple[ChatClient, ChatEndpoint]:
-    endpoint = ChatEndpoint(provider=provider, base_url=base_url.rstrip("/"), thinking=True, prices=price_of(model))
+def _deepseek_client(key: str, base_url: str, model: str, *, provider: Provider, thinking: bool = False) -> tuple[ChatClient, ChatEndpoint]:
+    endpoint = ChatEndpoint(provider=provider, base_url=base_url.rstrip("/"), thinking=thinking, prices=price_of(model))
     client = OpenRouterChatClient(api_key=key, base_url=endpoint.base_url)
-    return DeepSeekChatClient(client, endpoint=endpoint), endpoint
+    return DeepSeekChatClient(client, disable_thinking=not thinking, endpoint=endpoint), endpoint
 
 
 def _provider_of(base_url: str) -> Provider:

src/openultrasast/learning/journal.py

@@ -87,6 +87,12 @@ class RoundRecord:
     cost_usd: float = 0.0
     taxonomy_version: str = ""
     config_version: str = ""
+    # Pairs that moved each way in each stage, so the record can say whether the decision was distinguishable
+    # from noise. The decision itself is Req 9.5's raw-delta rule; this is the evidence beside it.
+    train_better: int = 0
+    train_worse: int = 0
+    holdout_better: int = 0
+    holdout_worse: int = 0
 
     def to_dict(self) -> dict[str, object]:
         return {
@@ -105,8 +111,17 @@ class RoundRecord:
             "cost_usd": self.cost_usd,
             "taxonomy_version": self.taxonomy_version,
             "config_version": self.config_version,
+            **{f"train_{key}": value for key, value in self._reliability(self.train_better, self.train_worse).items()},
+            **{f"holdout_{key}": value for key, value in self._reliability(self.holdout_better, self.holdout_worse).items()},
         }
 
+    @staticmethod
+    def _reliability(better: int, worse: int) -> dict[str, object]:
+        from .acceptance import reliability
+
+        result = reliability(better=better, worse=worse)
+        return {"better": result.better, "worse": result.worse, "p_value": result.p_value, "reliable": result.reliable}
+
     @staticmethod
     def from_dict(payload: Mapping[str, object]) -> RoundRecord:
         attribution = payload.get("attribution")

src/openultrasast/learning/rounds.py

@@ -66,6 +66,7 @@ class BaselineReport:
     k_runs: int
     taxonomy_version: str
     slices: tuple[str, ...]
+    thinking: bool = False  # the condition that produced these numbers, so two modes cannot be read as one
     floors: dict[str, NoiseFloor] = field(default_factory=dict)
     metrics: dict[str, dict[str, object]] = field(default_factory=dict)
     # `<slice>/<family>`: the budgets stay per family, but vibe-py and agent-vfc are different worlds and one
@@ -80,6 +81,7 @@ class BaselineReport:
             "k_runs": self.k_runs,
             "taxonomy_version": self.taxonomy_version,
             "slices": list(self.slices),
+            "thinking": self.thinking,
             "cost_usd": self.cost_usd,
             "floors": {name: floor.to_dict() for name, floor in sorted(self.floors.items())},
             "metrics": {name: dict(block) for name, block in sorted(self.metrics.items())},
@@ -100,6 +102,7 @@ def run_baseline(
     prompt: str | None = None,
     spent_usd: Callable[[], float] | None = None,
     cutoff: str = "",
+    thinking: bool = False,
 ) -> BaselineReport:
     """Clone one detector into every family, measure each family **with its own configuration**, write the artifacts.
 
@@ -146,6 +149,7 @@ def run_baseline(
         k_runs=k_runs,
         taxonomy_version=taxonomy.version,
         slices=tuple(slices),
+        thinking=thinking,
         floors=floors,
         metrics={name: block.to_dict() for name, block in metrics.items()},
         per_slice={name: block.to_dict() for name, block in aggregate(scores, taxonomy=taxonomy, per_slice=True, cutoff=cutoff).items()},
@@ -198,7 +202,8 @@ def compare_baselines(out_dir: Path) -> dict[str, dict[str, dict[str, object]]]:
         payload = json.loads(report.read_text(encoding="utf-8"))
         model = str(payload.get("model", directory.name))
         slices = tuple(str(item) for item in (payload.get("slices") or ()))
-        label = model if not slices or slices == DEFAULT_SLICES else f"{model} ({', '.join(slices)})"
+        conditions = [*(() if not payload.get("thinking") else ("thinking",)), *(() if slices == DEFAULT_SLICES else slices)]
+        label = f"{model} ({', '.join(conditions)})" if conditions else model
         table[label] = {family: dict(block) for family, block in (payload.get("metrics") or {}).items()}
     return table
 
@@ -227,18 +232,21 @@ def _family_of(case: PairCase, taxonomy: FamilyTaxonomy) -> str:
     return answer.families[0] if answer.families else "unknown"
 
 
-def _artifact_key(model: str, slices: Sequence[str]) -> str:
+def _artifact_key(model: str, slices: Sequence[str], thinking: bool = False) -> str:
     """What identifies a baseline run: the detector model, and the slice selection when it is not the default.
 
     Keyed by model alone, a run over the memory-safety slice silently replaces the run over the web families and
     the published table shows one of them as if it were everything (Req 8.4)."""
-    if tuple(slices) == DEFAULT_SLICES:
-        return _slug(model)
-    return f"{_slug(model)}-{_slug('-'.join(sorted(slices)))}"
+    parts = [_slug(model)]
+    if thinking:
+        parts.append("thinking")
+    if tuple(slices) != DEFAULT_SLICES:
+        parts.append(_slug("-".join(sorted(slices))))
+    return "-".join(parts)
 
 
 def _write(out_dir: Path, report: BaselineReport) -> None:
-    directory = out_dir / "baseline" / _artifact_key(report.model, report.slices)
+    directory = out_dir / "baseline" / _artifact_key(report.model, report.slices, report.thinking)
     directory.mkdir(parents=True, exist_ok=True)
     payload = report.to_dict()
     (directory / "noise-floors.json").write_text(
@@ -252,9 +260,9 @@ def _slug(model: str) -> str:
     return re.sub(r"[^A-Za-z0-9._-]+", "-", model).strip("-") or "unnamed"
 
 
-def load_noise_floors(out_dir: Path, model: str, slices: Sequence[str] = ()) -> dict[str, NoiseFloor]:
+def load_noise_floors(out_dir: Path, model: str, slices: Sequence[str] = (), thinking: bool = False) -> dict[str, NoiseFloor]:
     """The budgets a later round must stay inside, as round zero measured them for this model."""
-    path = out_dir / "baseline" / _artifact_key(model, slices or DEFAULT_SLICES) / "noise-floors.json"
+    path = out_dir / "baseline" / _artifact_key(model, slices or DEFAULT_SLICES, thinking) / "noise-floors.json"
     if not path.is_file():
         return {}
     payload = json.loads(path.read_text(encoding="utf-8"))
@@ -405,6 +413,8 @@ def run_learning_round(
         train_delta = _delta(before_train, after_train)
         holdout_delta = _delta(before_hold, after_hold)
         flipped = _flipped(before_train + before_hold, after_train + after_hold)
+        train_moves = _moves(before_train, after_train)
+        holdout_moves = _moves(before_hold, after_hold)
         attribution = Attribution(
             flipped_predicted=len(flipped),
             flipped_unpredicted=len(flipped_unpredicted),
@@ -438,6 +448,7 @@ def run_learning_round(
                 holdout_delta=holdout_delta,
                 sweep=sweep,
                 attribution=attribution,
+                moves=(train_moves, holdout_moves),
                 flipped=flipped,
                 unpredicted=flipped_unpredicted,
                 scores=after_train + after_hold,
@@ -456,6 +467,7 @@ def run_learning_round(
             holdout_delta=holdout_delta,
             sweep=sweep,
             attribution=attribution,
+            moves=(train_moves, holdout_moves),
             flipped=flipped,
             unpredicted=flipped_unpredicted,
             scores=after_train + after_hold,
@@ -543,6 +555,21 @@ def _delta(before: Sequence[PairFamilyScore], after: Sequence[PairFamilyScore])
     return (now - was) / len(scorable)
 
 
+def _moves(before: Sequence[PairFamilyScore], after: Sequence[PairFamilyScore]) -> tuple[int, int]:
+    """(pairs that became correct, pairs that stopped being correct) — what the sign test is run over."""
+    was = {item.pair: item.outcome for item in before if item.outcome != "unscorable"}
+    better = worse = 0
+    for item in after:
+        previous = was.get(item.pair)
+        if previous is None or item.outcome == "unscorable":
+            continue
+        if previous != "pair_correct" and item.outcome == "pair_correct":
+            better += 1
+        elif previous == "pair_correct" and item.outcome != "pair_correct":
+            worse += 1
+    return better, worse
+
+
 def _flipped(before: Sequence[PairFamilyScore], after: Sequence[PairFamilyScore]) -> list[str]:
     """Pairs this round turned right, by name and in a stable order."""
     was = {item.pair: item.outcome for item in before}
@@ -577,6 +604,7 @@ def _finish(
     holdout_delta: float = 0.0,
     sweep: Mapping[str, int] | None = None,
     attribution: Attribution | None = None,
+    moves: tuple[tuple[int, int], tuple[int, int]] = ((0, 0), (0, 0)),
     flipped: Sequence[str] = (),
     unpredicted: Sequence[str] = (),
     scores: Sequence[PairFamilyScore] = (),
@@ -595,6 +623,10 @@ def _finish(
         sweep=dict(sweep or {}),
         attribution=attribution or Attribution(),
         cost_usd=cost,
+        train_better=moves[0][0],
+        train_worse=moves[0][1],
+        holdout_better=moves[1][0],
+        holdout_worse=moves[1][1],
     )
     if version is not None:
         record = RoundRecord(**{**record.__dict__, "config_version": version})