From b57b1a35d85ec3c6b5d59dc671fa1303ed2626a8 Mon Sep 17 00:00:00 2001
From: Marius Ciepluch <11855163+norandom@users.noreply.github.com>
Date: Sun, 6 Sep 2026 15:12:09 +0200
Subject: [PATCH] feat(learning-harness): thinking is a measured condition, and
 every round records whether its decision beat noise

Two things the maintainer was right about.

Thinking was disabled so `temperature: 0` would not be silently ignored --
a trade that only pays if it buys determinism. Round zero says it did not:
11 of 20 injection pairs disagreed with themselves across five runs, so the
variance is in the agentic path, not the decoder. We paid the reasoning cost
on a reasoning model and got nothing. It is now `[learning] thinking`,
recorded in the artifact and part of its key so the two modes cannot be read
as one number. The default stays off only because that is what round zero
was measured under.

And the deeper point: the spec built `sign_test` and `reliable_change` and
the acceptance rule calls neither -- `decide` compares raw deltas. On a
detector with that floor a raw delta is mostly noise, and a rule that
brittle is exactly what makes a deterministic decoder look necessary. It is
not; the alternative is an acceptance rule that models the score as noisy,
the way reflective-evolution and Bayesian prompt optimizers do. Changing
Req 9.5 is a maintainer's call, so this does not change the decision: every
round now journals the two-sided sign test over the pairs that moved, and
after four rounds the journal will say whether any decision could be told
from a coin flip.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
---
 src/openultrasast/cli.py                 |  3 +-
 src/openultrasast/config.py              |  7 +++
 src/openultrasast/learning/acceptance.py | 31 ++++++++++-
 src/openultrasast/learning/endpoint.py   | 16 ++++--
 src/openultrasast/learning/journal.py    | 15 ++++++
 src/openultrasast/learning/rounds.py     | 48 ++++++++++++++---
 tests/test_learning_reliability.py       | 56 +++++++++++++++++++
 tests/test_learning_thinking.py          | 69 ++++++++++++++++++++++++
 8 files changed, 230 insertions(+), 15 deletions(-)
 create mode 100644 tests/test_learning_reliability.py
 create mode 100644 tests/test_learning_thinking.py

diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py
index 77f1f59..168827d 100644
--- a/src/openultrasast/cli.py
+++ b/src/openultrasast/cli.py
@@ -1328,6 +1328,7 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy:
             # a run over the memory-safety slice reports nothing at all (Req 8.4).
             slices=(args.slice,) if args.slice != "all" else DEFAULT_SLICES,
             spent_usd=spent,
+            thinking=config.learning.thinking,
         )
         print(f"learning baseline cost: ${report.cost_usd:.4f}")
         print(f"learning baseline {model}: {len(report.floors)} families, k={report.k_runs} -> {out / 'baseline'}")
@@ -1343,7 +1344,7 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy:
         model=model,
         journal=journal,
         archive=Archive(out / "archive.jsonl"),
-        floors=load_noise_floors(out, model, (args.slice,) if args.slice != "all" else DEFAULT_SLICES),
+        floors=load_noise_floors(out, model, (args.slice,) if args.slice != "all" else DEFAULT_SLICES, config.learning.thinking),
         out_dir=out,
         cost_cap_usd=args.cost_cap_usd or config.learning.round_cost_cap_usd,
         minibatch=config.learning.minibatch,
diff --git a/src/openultrasast/config.py b/src/openultrasast/config.py
index 75bc9e0..b47e00d 100644
--- a/src/openultrasast/config.py
+++ b/src/openultrasast/config.py
@@ -150,6 +150,12 @@ class LearningConfig:
     detector_model: str | None = None
     judge_model: str | None = None
     cutoff_date: str | None = None  # rows fixed after this date are reported as a separate slice
+    # Whether the detector model reasons before answering. Off by default because that is what round zero was
+    # measured under, not because it is known to be better: the provider silently ignores `temperature` while
+    # thinking, and thinking was disabled to make `temperature: 0` mean something. It did not — the measured
+    # disagreement between runs comes from the agentic path, not the decoder — so which mode wins is a question
+    # for a second baseline rather than for a default (Req 8.3).
+    thinking: bool = False
 
 
 @dataclass(frozen=True)
@@ -391,6 +397,7 @@ def _load_learning(value: object) -> LearningConfig:
         detector_model=_string(data.get("detector_model")),
         judge_model=_string(data.get("judge_model")),
         cutoff_date=_string(data.get("cutoff_date")),
+        thinking=bool(data.get("thinking", False)),
     )
 
 
diff --git a/src/openultrasast/learning/acceptance.py b/src/openultrasast/learning/acceptance.py
index 5d49494..46a1ebe 100644
--- a/src/openultrasast/learning/acceptance.py
+++ b/src/openultrasast/learning/acceptance.py
@@ -55,6 +55,35 @@ def decide(
     return Verdict(accepted=True, reason="improved")
 
 
+@dataclass(frozen=True)
+class Reliability:
+    """Whether a stage's movement could be told from a coin flip."""
+
+    better: int
+    worse: int
+    p_value: float
+    reliable: bool
+
+    def to_dict(self) -> dict[str, object]:
+        return {"better": self.better, "worse": self.worse, "p_value": self.p_value, "reliable": self.reliable}
+
+
+def reliability(*, better: int, worse: int, alpha: float = 0.05) -> Reliability:
+    """The two-sided sign test over the pairs that changed, for the record rather than for the decision.
+
+    `decide` compares raw deltas, as Req 9.5 specifies. On a detector whose measured floor says half its pairs
+    disagree with themselves between runs, a raw delta is mostly noise — which is what makes a deterministic
+    decoder look necessary. It is not: the alternative is an acceptance rule that models the score as noisy, the
+    way reflective-evolution and Bayesian prompt optimizers do, and keeps per-instance winners on a frontier
+    instead of demanding a single global improvement. Changing the rule is a maintainer's call, so every round
+    records this beside its decision and the journal can be read to answer the question with data.
+    """
+    from .scoring import sign_test
+
+    p_value = sign_test(better, worse)
+    return Reliability(better=better, worse=worse, p_value=p_value, reliable=bool(p_value < alpha and better != worse))
+
+
 def _gates(family: str, floors: Mapping[str, NoiseFloor]) -> bool:
     floor = floors.get(family)
     return floor.gates if floor is not None else True
@@ -114,4 +143,4 @@ def bump_version(directory: Path, version: str) -> None:
     settings.write_text("\n".join(rewritten) + "\n", encoding="utf-8")
 
 
-__all__ = ["DirectorySnapshot", "Verdict", "bump_version", "decide", "digest_configs", "within_budget"]
+__all__ = ["DirectorySnapshot", "Reliability", "Verdict", "bump_version", "decide", "digest_configs", "reliability", "within_budget"]
diff --git a/src/openultrasast/learning/endpoint.py b/src/openultrasast/learning/endpoint.py
index 4d95807..a734d30 100644
--- a/src/openultrasast/learning/endpoint.py
+++ b/src/openultrasast/learning/endpoint.py
@@ -154,10 +154,16 @@ def resolve_chat_endpoint(config: ResolvedConfig, *, override: ChatClient | None
     if configured or key_env:
         key = os.environ.get(key_env or DEEPSEEK_KEY_ENV, "")
         base_url = configured or DEEPSEEK_BASE_URL
-        return _deepseek_client(key, base_url, model, provider=_provider_of(base_url))
+        return _deepseek_client(key, base_url, model, provider=_provider_of(base_url), thinking=config.learning.thinking)
     deepseek_key = os.environ.get(DEEPSEEK_KEY_ENV)
     if deepseek_key:
-        return _deepseek_client(deepseek_key, os.environ.get(DEEPSEEK_BASE_ENV, DEEPSEEK_BASE_URL), model, provider="deepseek")
+        return _deepseek_client(
+            deepseek_key,
+            os.environ.get(DEEPSEEK_BASE_ENV, DEEPSEEK_BASE_URL),
+            model,
+            provider="deepseek",
+            thinking=config.learning.thinking,
+        )
     if os.environ.get("OPENROUTER_API_KEY"):
         try:
             client = OpenRouterChatClient.from_env()
@@ -175,10 +181,10 @@ def resolve_models(config: ResolvedConfig) -> tuple[str, str]:
     return detector, judge
 
 
-def _deepseek_client(key: str, base_url: str, model: str, *, provider: Provider) -> tuple[ChatClient, ChatEndpoint]:
-    endpoint = ChatEndpoint(provider=provider, base_url=base_url.rstrip("/"), thinking=True, prices=price_of(model))
+def _deepseek_client(key: str, base_url: str, model: str, *, provider: Provider, thinking: bool = False) -> tuple[ChatClient, ChatEndpoint]:
+    endpoint = ChatEndpoint(provider=provider, base_url=base_url.rstrip("/"), thinking=thinking, prices=price_of(model))
     client = OpenRouterChatClient(api_key=key, base_url=endpoint.base_url)
-    return DeepSeekChatClient(client, endpoint=endpoint), endpoint
+    return DeepSeekChatClient(client, disable_thinking=not thinking, endpoint=endpoint), endpoint
 
 
 def _provider_of(base_url: str) -> Provider:
diff --git a/src/openultrasast/learning/journal.py b/src/openultrasast/learning/journal.py
index acd51a1..a14f808 100644
--- a/src/openultrasast/learning/journal.py
+++ b/src/openultrasast/learning/journal.py
@@ -87,6 +87,12 @@ class RoundRecord:
     cost_usd: float = 0.0
     taxonomy_version: str = ""
     config_version: str = ""
+    # Pairs that moved each way in each stage, so the record can say whether the decision was distinguishable
+    # from noise. The decision itself is Req 9.5's raw-delta rule; this is the evidence beside it.
+    train_better: int = 0
+    train_worse: int = 0
+    holdout_better: int = 0
+    holdout_worse: int = 0
 
     def to_dict(self) -> dict[str, object]:
         return {
@@ -105,8 +111,17 @@ class RoundRecord:
             "cost_usd": self.cost_usd,
             "taxonomy_version": self.taxonomy_version,
             "config_version": self.config_version,
+            **{f"train_{key}": value for key, value in self._reliability(self.train_better, self.train_worse).items()},
+            **{f"holdout_{key}": value for key, value in self._reliability(self.holdout_better, self.holdout_worse).items()},
         }
 
+    @staticmethod
+    def _reliability(better: int, worse: int) -> dict[str, object]:
+        from .acceptance import reliability
+
+        result = reliability(better=better, worse=worse)
+        return {"better": result.better, "worse": result.worse, "p_value": result.p_value, "reliable": result.reliable}
+
     @staticmethod
     def from_dict(payload: Mapping[str, object]) -> RoundRecord:
         attribution = payload.get("attribution")
diff --git a/src/openultrasast/learning/rounds.py b/src/openultrasast/learning/rounds.py
index 0815690..68bbfb7 100644
--- a/src/openultrasast/learning/rounds.py
+++ b/src/openultrasast/learning/rounds.py
@@ -66,6 +66,7 @@ class BaselineReport:
     k_runs: int
     taxonomy_version: str
     slices: tuple[str, ...]
+    thinking: bool = False  # the condition that produced these numbers, so two modes cannot be read as one
     floors: dict[str, NoiseFloor] = field(default_factory=dict)
     metrics: dict[str, dict[str, object]] = field(default_factory=dict)
     # `<slice>/<family>`: the budgets stay per family, but vibe-py and agent-vfc are different worlds and one
@@ -80,6 +81,7 @@ class BaselineReport:
             "k_runs": self.k_runs,
             "taxonomy_version": self.taxonomy_version,
             "slices": list(self.slices),
+            "thinking": self.thinking,
             "cost_usd": self.cost_usd,
             "floors": {name: floor.to_dict() for name, floor in sorted(self.floors.items())},
             "metrics": {name: dict(block) for name, block in sorted(self.metrics.items())},
@@ -100,6 +102,7 @@ def run_baseline(
     prompt: str | None = None,
     spent_usd: Callable[[], float] | None = None,
     cutoff: str = "",
+    thinking: bool = False,
 ) -> BaselineReport:
     """Clone one detector into every family, measure each family **with its own configuration**, write the artifacts.
 
@@ -146,6 +149,7 @@ def run_baseline(
         k_runs=k_runs,
         taxonomy_version=taxonomy.version,
         slices=tuple(slices),
+        thinking=thinking,
         floors=floors,
         metrics={name: block.to_dict() for name, block in metrics.items()},
         per_slice={name: block.to_dict() for name, block in aggregate(scores, taxonomy=taxonomy, per_slice=True, cutoff=cutoff).items()},
@@ -198,7 +202,8 @@ def compare_baselines(out_dir: Path) -> dict[str, dict[str, dict[str, object]]]:
         payload = json.loads(report.read_text(encoding="utf-8"))
         model = str(payload.get("model", directory.name))
         slices = tuple(str(item) for item in (payload.get("slices") or ()))
-        label = model if not slices or slices == DEFAULT_SLICES else f"{model} ({', '.join(slices)})"
+        conditions = [*(() if not payload.get("thinking") else ("thinking",)), *(() if slices == DEFAULT_SLICES else slices)]
+        label = f"{model} ({', '.join(conditions)})" if conditions else model
         table[label] = {family: dict(block) for family, block in (payload.get("metrics") or {}).items()}
     return table
 
@@ -227,18 +232,21 @@ def _family_of(case: PairCase, taxonomy: FamilyTaxonomy) -> str:
     return answer.families[0] if answer.families else "unknown"
 
 
-def _artifact_key(model: str, slices: Sequence[str]) -> str:
+def _artifact_key(model: str, slices: Sequence[str], thinking: bool = False) -> str:
     """What identifies a baseline run: the detector model, and the slice selection when it is not the default.
 
     Keyed by model alone, a run over the memory-safety slice silently replaces the run over the web families and
     the published table shows one of them as if it were everything (Req 8.4)."""
-    if tuple(slices) == DEFAULT_SLICES:
-        return _slug(model)
-    return f"{_slug(model)}-{_slug('-'.join(sorted(slices)))}"
+    parts = [_slug(model)]
+    if thinking:
+        parts.append("thinking")
+    if tuple(slices) != DEFAULT_SLICES:
+        parts.append(_slug("-".join(sorted(slices))))
+    return "-".join(parts)
 
 
 def _write(out_dir: Path, report: BaselineReport) -> None:
-    directory = out_dir / "baseline" / _artifact_key(report.model, report.slices)
+    directory = out_dir / "baseline" / _artifact_key(report.model, report.slices, report.thinking)
     directory.mkdir(parents=True, exist_ok=True)
     payload = report.to_dict()
     (directory / "noise-floors.json").write_text(
@@ -252,9 +260,9 @@ def _slug(model: str) -> str:
     return re.sub(r"[^A-Za-z0-9._-]+", "-", model).strip("-") or "unnamed"
 
 
-def load_noise_floors(out_dir: Path, model: str, slices: Sequence[str] = ()) -> dict[str, NoiseFloor]:
+def load_noise_floors(out_dir: Path, model: str, slices: Sequence[str] = (), thinking: bool = False) -> dict[str, NoiseFloor]:
     """The budgets a later round must stay inside, as round zero measured them for this model."""
-    path = out_dir / "baseline" / _artifact_key(model, slices or DEFAULT_SLICES) / "noise-floors.json"
+    path = out_dir / "baseline" / _artifact_key(model, slices or DEFAULT_SLICES, thinking) / "noise-floors.json"
     if not path.is_file():
         return {}
     payload = json.loads(path.read_text(encoding="utf-8"))
@@ -405,6 +413,8 @@ def run_learning_round(
         train_delta = _delta(before_train, after_train)
         holdout_delta = _delta(before_hold, after_hold)
         flipped = _flipped(before_train + before_hold, after_train + after_hold)
+        train_moves = _moves(before_train, after_train)
+        holdout_moves = _moves(before_hold, after_hold)
         attribution = Attribution(
             flipped_predicted=len(flipped),
             flipped_unpredicted=len(flipped_unpredicted),
@@ -438,6 +448,7 @@ def run_learning_round(
                 holdout_delta=holdout_delta,
                 sweep=sweep,
                 attribution=attribution,
+                moves=(train_moves, holdout_moves),
                 flipped=flipped,
                 unpredicted=flipped_unpredicted,
                 scores=after_train + after_hold,
@@ -456,6 +467,7 @@ def run_learning_round(
             holdout_delta=holdout_delta,
             sweep=sweep,
             attribution=attribution,
+            moves=(train_moves, holdout_moves),
             flipped=flipped,
             unpredicted=flipped_unpredicted,
             scores=after_train + after_hold,
@@ -543,6 +555,21 @@ def _delta(before: Sequence[PairFamilyScore], after: Sequence[PairFamilyScore])
     return (now - was) / len(scorable)
 
 
+def _moves(before: Sequence[PairFamilyScore], after: Sequence[PairFamilyScore]) -> tuple[int, int]:
+    """(pairs that became correct, pairs that stopped being correct) — what the sign test is run over."""
+    was = {item.pair: item.outcome for item in before if item.outcome != "unscorable"}
+    better = worse = 0
+    for item in after:
+        previous = was.get(item.pair)
+        if previous is None or item.outcome == "unscorable":
+            continue
+        if previous != "pair_correct" and item.outcome == "pair_correct":
+            better += 1
+        elif previous == "pair_correct" and item.outcome != "pair_correct":
+            worse += 1
+    return better, worse
+
+
 def _flipped(before: Sequence[PairFamilyScore], after: Sequence[PairFamilyScore]) -> list[str]:
     """Pairs this round turned right, by name and in a stable order."""
     was = {item.pair: item.outcome for item in before}
@@ -577,6 +604,7 @@ def _finish(
     holdout_delta: float = 0.0,
     sweep: Mapping[str, int] | None = None,
     attribution: Attribution | None = None,
+    moves: tuple[tuple[int, int], tuple[int, int]] = ((0, 0), (0, 0)),
     flipped: Sequence[str] = (),
     unpredicted: Sequence[str] = (),
     scores: Sequence[PairFamilyScore] = (),
@@ -595,6 +623,10 @@ def _finish(
         sweep=dict(sweep or {}),
         attribution=attribution or Attribution(),
         cost_usd=cost,
+        train_better=moves[0][0],
+        train_worse=moves[0][1],
+        holdout_better=moves[1][0],
+        holdout_worse=moves[1][1],
     )
     if version is not None:
         record = RoundRecord(**{**record.__dict__, "config_version": version})
diff --git a/tests/test_learning_reliability.py b/tests/test_learning_reliability.py
new file mode 100644
index 0000000..abda25d
--- /dev/null
+++ b/tests/test_learning_reliability.py
@@ -0,0 +1,56 @@
+"""Every round says whether its decision was distinguishable from noise (learning-harness, Req 9.5).
+
+The spec built `sign_test` and `reliable_change` and the acceptance rule never calls either: `decide` compares raw
+deltas. On a detector where 11 of 20 pairs disagree with themselves between runs, a raw delta is mostly noise, and
+a rule that brittle is exactly what makes someone reach for a deterministic decoder to prop it up.
+
+Changing Req 9.5 is a maintainer's call, so this does not change the decision — it records, beside every decision,
+the two-sided sign test over the pairs that actually changed. After a few rounds the journal says plainly whether
+any of those decisions could be told from a coin flip.
+"""
+
+from __future__ import annotations
+
+from openultrasast.learning.acceptance import reliability
+
+
+def test_a_clean_sweep_of_the_minibatch_is_distinguishable_from_noise() -> None:
+    result = reliability(better=8, worse=0)
+    assert result.better == 8 and result.worse == 0
+    assert result.p_value < 0.05 and result.reliable is True
+
+
+def test_two_pairs_moving_the_right_way_is_not() -> None:
+    """n = 2 cannot reach p < 0.05 whatever it does, and a rule that treats it as evidence is measuring nothing."""
+    result = reliability(better=2, worse=0)
+    assert result.p_value == 0.5 and result.reliable is False
+
+
+def test_a_round_that_changed_nothing_reports_no_evidence_rather_than_agreement() -> None:
+    result = reliability(better=0, worse=0)
+    assert result.p_value == 1.0 and result.reliable is False
+
+
+def test_an_even_split_is_never_reliable_however_large() -> None:
+    assert reliability(better=20, worse=20).reliable is False
+
+
+def test_the_round_journals_the_reliability_of_its_own_decision() -> None:
+    from openultrasast.learning.journal import RoundRecord
+
+    record = RoundRecord(
+        round=1,
+        family="injection",
+        hypothesis="h",
+        levers=("checklist",),
+        predicted_affected=(),
+        predicted_at_risk=(),
+        outcome="accepted",
+        train_better=8,
+        train_worse=0,
+        holdout_better=1,
+        holdout_worse=0,
+    )
+    payload = record.to_dict()
+    assert payload["train_reliable"] is True and payload["train_p_value"] < 0.05
+    assert payload["holdout_reliable"] is False, "one pair is not evidence, and the record says so"
diff --git a/tests/test_learning_thinking.py b/tests/test_learning_thinking.py
new file mode 100644
index 0000000..cf2fcfa
--- /dev/null
+++ b/tests/test_learning_thinking.py
@@ -0,0 +1,69 @@
+"""Thinking is a measured condition, not a hidden default (learning-harness, Req 8.3).
+
+Thinking was disabled so that `temperature: 0` would not be silently ignored — a trade that only pays if it buys
+determinism. Round zero measured 11 of 20 injection pairs disagreeing with themselves across five runs, so it did
+not: the variance comes from the agentic path, not the decoder. Which mode is better is a question the harness is
+built to answer, so it is a knob whose value the artifact records and whose runs cannot overwrite each other.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from openultrasast.config import load_config
+
+
+def test_thinking_is_configurable_and_off_by_default(tmp_path: Path) -> None:
+    """The default preserves what round zero was measured under; changing it silently would make the two runs
+    incomparable while looking like one number."""
+    assert load_config(None).learning.thinking is False
+    path = tmp_path / "openultrasast.toml"
+    path.write_text("[learning]\nthinking = true\n")
+    assert load_config(path).learning.thinking is True
+
+
+def test_the_endpoint_records_the_mode_it_ran_in(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    from openultrasast.learning.endpoint import resolve_chat_endpoint
+
+    monkeypatch.setenv("DEEPSEEK_API_KEY", "k")
+    monkeypatch.delenv("OPENULTRASAST_HUNTER_CLIENT", raising=False)
+    path = tmp_path / "openultrasast.toml"
+    path.write_text("[learning]\nthinking = true\n")
+    client, endpoint = resolve_chat_endpoint(load_config(path))  # type: ignore[misc]
+    assert endpoint.thinking is True
+    assert client._disable_thinking is False  # type: ignore[attr-defined]
+    path.write_text("[learning]\nthinking = false\n")
+    client, endpoint = resolve_chat_endpoint(load_config(path))  # type: ignore[misc]
+    assert endpoint.thinking is False and client._disable_thinking is True  # type: ignore[attr-defined]
+
+
+def test_the_two_modes_are_two_baselines_not_one_overwriting_the_other(tmp_path: Path) -> None:
+    """The condition that produced a number is part of what identifies it."""
+    import sys
+
+    sys.path.insert(0, str(Path(__file__).parent))
+    from test_learning_rounds import _always, _case, _steady  # noqa: PLC0415
+
+    from openultrasast.learning.families import load_families
+    from openultrasast.learning.rounds import compare_baselines, run_baseline
+
+    out = tmp_path / "out"
+    for thinking in (False, True):
+        run_baseline(
+            [_case(tmp_path, "a")],
+            taxonomy=load_families(),
+            configs_dir=tmp_path / "configs",
+            scan_factory=_always(_steady),
+            model="deepseek-v4-flash",
+            out_dir=out,
+            k_runs=3,
+            thinking=thinking,
+        )
+    written = sorted(path.name for path in (out / "baseline").iterdir())
+    assert written == ["deepseek-v4-flash", "deepseek-v4-flash-thinking"], written
+    assert set(compare_baselines(out)) == {"deepseek-v4-flash", "deepseek-v4-flash (thinking)"}
+    report = json.loads((out / "baseline" / "deepseek-v4-flash-thinking" / "report.json").read_text())
+    assert report["thinking"] is True