feat(learning-harness): thinking is a measured condition, and every round

records whether its decision beat noise

Two things the maintainer was right about.

Thinking was disabled so `temperature: 0` would not be silently ignored --
a trade that only pays if it buys determinism. Round zero says it did not:
11 of 20 injection pairs disagreed with themselves across five runs, so the
variance is in the agentic path, not the decoder. We paid the reasoning cost
on a reasoning model and got nothing. It is now `[learning] thinking`,
recorded in the artifact and part of its key so the two modes cannot be read
as one number. The default stays off only because that is what round zero
was measured under.

And the deeper point: the spec built `sign_test` and `reliable_change` and
the acceptance rule calls neither -- `decide` compares raw deltas. On a
detector with that floor a raw delta is mostly noise, and a rule that
brittle is exactly what makes a deterministic decoder look necessary. It is
not; the alternative is an acceptance rule that models the score as noisy,
the way reflective-evolution and Bayesian prompt optimizers do. Changing
Req 9.5 is a maintainer's call, so this does not change the decision: every
round now journals the two-sided sign test over the pairs that moved, and
after four rounds the journal will say whether any decision could be told
from a coin flip.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG

tests/test_learning_reliability.py

@@ -0,0 +1,56 @@
+"""Every round says whether its decision was distinguishable from noise (learning-harness, Req 9.5).
+
+The spec built `sign_test` and `reliable_change` and the acceptance rule never calls either: `decide` compares raw
+deltas. On a detector where 11 of 20 pairs disagree with themselves between runs, a raw delta is mostly noise, and
+a rule that brittle is exactly what makes someone reach for a deterministic decoder to prop it up.
+
+Changing Req 9.5 is a maintainer's call, so this does not change the decision — it records, beside every decision,
+the two-sided sign test over the pairs that actually changed. After a few rounds the journal says plainly whether
+any of those decisions could be told from a coin flip.
+"""
+
+from __future__ import annotations
+
+from openultrasast.learning.acceptance import reliability
+
+
+def test_a_clean_sweep_of_the_minibatch_is_distinguishable_from_noise() -> None:
+    result = reliability(better=8, worse=0)
+    assert result.better == 8 and result.worse == 0
+    assert result.p_value < 0.05 and result.reliable is True
+
+
+def test_two_pairs_moving_the_right_way_is_not() -> None:
+    """n = 2 cannot reach p < 0.05 whatever it does, and a rule that treats it as evidence is measuring nothing."""
+    result = reliability(better=2, worse=0)
+    assert result.p_value == 0.5 and result.reliable is False
+
+
+def test_a_round_that_changed_nothing_reports_no_evidence_rather_than_agreement() -> None:
+    result = reliability(better=0, worse=0)
+    assert result.p_value == 1.0 and result.reliable is False
+
+
+def test_an_even_split_is_never_reliable_however_large() -> None:
+    assert reliability(better=20, worse=20).reliable is False
+
+
+def test_the_round_journals_the_reliability_of_its_own_decision() -> None:
+    from openultrasast.learning.journal import RoundRecord
+
+    record = RoundRecord(
+        round=1,
+        family="injection",
+        hypothesis="h",
+        levers=("checklist",),
+        predicted_affected=(),
+        predicted_at_risk=(),
+        outcome="accepted",
+        train_better=8,
+        train_worse=0,
+        holdout_better=1,
+        holdout_worse=0,
+    )
+    payload = record.to_dict()
+    assert payload["train_reliable"] is True and payload["train_p_value"] < 0.05
+    assert payload["holdout_reliable"] is False, "one pair is not evidence, and the record says so"

tests/test_learning_thinking.py

@@ -0,0 +1,69 @@
+"""Thinking is a measured condition, not a hidden default (learning-harness, Req 8.3).
+
+Thinking was disabled so that `temperature: 0` would not be silently ignored — a trade that only pays if it buys
+determinism. Round zero measured 11 of 20 injection pairs disagreeing with themselves across five runs, so it did
+not: the variance comes from the agentic path, not the decoder. Which mode is better is a question the harness is
+built to answer, so it is a knob whose value the artifact records and whose runs cannot overwrite each other.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from openultrasast.config import load_config
+
+
+def test_thinking_is_configurable_and_off_by_default(tmp_path: Path) -> None:
+    """The default preserves what round zero was measured under; changing it silently would make the two runs
+    incomparable while looking like one number."""
+    assert load_config(None).learning.thinking is False
+    path = tmp_path / "openultrasast.toml"
+    path.write_text("[learning]\nthinking = true\n")
+    assert load_config(path).learning.thinking is True
+
+
+def test_the_endpoint_records_the_mode_it_ran_in(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    from openultrasast.learning.endpoint import resolve_chat_endpoint
+
+    monkeypatch.setenv("DEEPSEEK_API_KEY", "k")
+    monkeypatch.delenv("OPENULTRASAST_HUNTER_CLIENT", raising=False)
+    path = tmp_path / "openultrasast.toml"
+    path.write_text("[learning]\nthinking = true\n")
+    client, endpoint = resolve_chat_endpoint(load_config(path))  # type: ignore[misc]
+    assert endpoint.thinking is True
+    assert client._disable_thinking is False  # type: ignore[attr-defined]
+    path.write_text("[learning]\nthinking = false\n")
+    client, endpoint = resolve_chat_endpoint(load_config(path))  # type: ignore[misc]
+    assert endpoint.thinking is False and client._disable_thinking is True  # type: ignore[attr-defined]
+
+
+def test_the_two_modes_are_two_baselines_not_one_overwriting_the_other(tmp_path: Path) -> None:
+    """The condition that produced a number is part of what identifies it."""
+    import sys
+
+    sys.path.insert(0, str(Path(__file__).parent))
+    from test_learning_rounds import _always, _case, _steady  # noqa: PLC0415
+
+    from openultrasast.learning.families import load_families
+    from openultrasast.learning.rounds import compare_baselines, run_baseline
+
+    out = tmp_path / "out"
+    for thinking in (False, True):
+        run_baseline(
+            [_case(tmp_path, "a")],
+            taxonomy=load_families(),
+            configs_dir=tmp_path / "configs",
+            scan_factory=_always(_steady),
+            model="deepseek-v4-flash",
+            out_dir=out,
+            k_runs=3,
+            thinking=thinking,
+        )
+    written = sorted(path.name for path in (out / "baseline").iterdir())
+    assert written == ["deepseek-v4-flash", "deepseek-v4-flash-thinking"], written
+    assert set(compare_baselines(out)) == {"deepseek-v4-flash", "deepseek-v4-flash (thinking)"}
+    report = json.loads((out / "baseline" / "deepseek-v4-flash-thinking" / "report.json").read_text())
+    assert report["thinking"] is True