diff --git a/src/openultrasast/cli.py b/src/openultrasast/cli.py index 77f1f59..168827d 100644 --- a/src/openultrasast/cli.py +++ b/src/openultrasast/cli.py @@ -1328,6 +1328,7 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy: # a run over the memory-safety slice reports nothing at all (Req 8.4). slices=(args.slice,) if args.slice != "all" else DEFAULT_SLICES, spent_usd=spent, + thinking=config.learning.thinking, ) print(f"learning baseline cost: ${report.cost_usd:.4f}") print(f"learning baseline {model}: {len(report.floors)} families, k={report.k_runs} -> {out / 'baseline'}") @@ -1343,7 +1344,7 @@ def _learning_run(args: argparse.Namespace, cases: Sequence[PairCase], taxonomy: model=model, journal=journal, archive=Archive(out / "archive.jsonl"), - floors=load_noise_floors(out, model, (args.slice,) if args.slice != "all" else DEFAULT_SLICES), + floors=load_noise_floors(out, model, (args.slice,) if args.slice != "all" else DEFAULT_SLICES, config.learning.thinking), out_dir=out, cost_cap_usd=args.cost_cap_usd or config.learning.round_cost_cap_usd, minibatch=config.learning.minibatch, diff --git a/src/openultrasast/config.py b/src/openultrasast/config.py index 75bc9e0..b47e00d 100644 --- a/src/openultrasast/config.py +++ b/src/openultrasast/config.py @@ -150,6 +150,12 @@ class LearningConfig: detector_model: str | None = None judge_model: str | None = None cutoff_date: str | None = None # rows fixed after this date are reported as a separate slice + # Whether the detector model reasons before answering. Off by default because that is what round zero was + # measured under, not because it is known to be better: the provider silently ignores `temperature` while + # thinking, and thinking was disabled to make `temperature: 0` mean something. It did not — the measured + # disagreement between runs comes from the agentic path, not the decoder — so which mode wins is a question + # for a second baseline rather than for a default (Req 8.3). + thinking: bool = False @dataclass(frozen=True) @@ -391,6 +397,7 @@ def _load_learning(value: object) -> LearningConfig: detector_model=_string(data.get("detector_model")), judge_model=_string(data.get("judge_model")), cutoff_date=_string(data.get("cutoff_date")), + thinking=bool(data.get("thinking", False)), ) diff --git a/src/openultrasast/learning/acceptance.py b/src/openultrasast/learning/acceptance.py index 5d49494..46a1ebe 100644 --- a/src/openultrasast/learning/acceptance.py +++ b/src/openultrasast/learning/acceptance.py @@ -55,6 +55,35 @@ def decide( return Verdict(accepted=True, reason="improved") +@dataclass(frozen=True) +class Reliability: + """Whether a stage's movement could be told from a coin flip.""" + + better: int + worse: int + p_value: float + reliable: bool + + def to_dict(self) -> dict[str, object]: + return {"better": self.better, "worse": self.worse, "p_value": self.p_value, "reliable": self.reliable} + + +def reliability(*, better: int, worse: int, alpha: float = 0.05) -> Reliability: + """The two-sided sign test over the pairs that changed, for the record rather than for the decision. + + `decide` compares raw deltas, as Req 9.5 specifies. On a detector whose measured floor says half its pairs + disagree with themselves between runs, a raw delta is mostly noise — which is what makes a deterministic + decoder look necessary. It is not: the alternative is an acceptance rule that models the score as noisy, the + way reflective-evolution and Bayesian prompt optimizers do, and keeps per-instance winners on a frontier + instead of demanding a single global improvement. Changing the rule is a maintainer's call, so every round + records this beside its decision and the journal can be read to answer the question with data. + """ + from .scoring import sign_test + + p_value = sign_test(better, worse) + return Reliability(better=better, worse=worse, p_value=p_value, reliable=bool(p_value < alpha and better != worse)) + + def _gates(family: str, floors: Mapping[str, NoiseFloor]) -> bool: floor = floors.get(family) return floor.gates if floor is not None else True @@ -114,4 +143,4 @@ def bump_version(directory: Path, version: str) -> None: settings.write_text("\n".join(rewritten) + "\n", encoding="utf-8") -__all__ = ["DirectorySnapshot", "Verdict", "bump_version", "decide", "digest_configs", "within_budget"] +__all__ = ["DirectorySnapshot", "Reliability", "Verdict", "bump_version", "decide", "digest_configs", "reliability", "within_budget"] diff --git a/src/openultrasast/learning/endpoint.py b/src/openultrasast/learning/endpoint.py index 4d95807..a734d30 100644 --- a/src/openultrasast/learning/endpoint.py +++ b/src/openultrasast/learning/endpoint.py @@ -154,10 +154,16 @@ def resolve_chat_endpoint(config: ResolvedConfig, *, override: ChatClient | None if configured or key_env: key = os.environ.get(key_env or DEEPSEEK_KEY_ENV, "") base_url = configured or DEEPSEEK_BASE_URL - return _deepseek_client(key, base_url, model, provider=_provider_of(base_url)) + return _deepseek_client(key, base_url, model, provider=_provider_of(base_url), thinking=config.learning.thinking) deepseek_key = os.environ.get(DEEPSEEK_KEY_ENV) if deepseek_key: - return _deepseek_client(deepseek_key, os.environ.get(DEEPSEEK_BASE_ENV, DEEPSEEK_BASE_URL), model, provider="deepseek") + return _deepseek_client( + deepseek_key, + os.environ.get(DEEPSEEK_BASE_ENV, DEEPSEEK_BASE_URL), + model, + provider="deepseek", + thinking=config.learning.thinking, + ) if os.environ.get("OPENROUTER_API_KEY"): try: client = OpenRouterChatClient.from_env() @@ -175,10 +181,10 @@ def resolve_models(config: ResolvedConfig) -> tuple[str, str]: return detector, judge -def _deepseek_client(key: str, base_url: str, model: str, *, provider: Provider) -> tuple[ChatClient, ChatEndpoint]: - endpoint = ChatEndpoint(provider=provider, base_url=base_url.rstrip("/"), thinking=True, prices=price_of(model)) +def _deepseek_client(key: str, base_url: str, model: str, *, provider: Provider, thinking: bool = False) -> tuple[ChatClient, ChatEndpoint]: + endpoint = ChatEndpoint(provider=provider, base_url=base_url.rstrip("/"), thinking=thinking, prices=price_of(model)) client = OpenRouterChatClient(api_key=key, base_url=endpoint.base_url) - return DeepSeekChatClient(client, endpoint=endpoint), endpoint + return DeepSeekChatClient(client, disable_thinking=not thinking, endpoint=endpoint), endpoint def _provider_of(base_url: str) -> Provider: diff --git a/src/openultrasast/learning/journal.py b/src/openultrasast/learning/journal.py index acd51a1..a14f808 100644 --- a/src/openultrasast/learning/journal.py +++ b/src/openultrasast/learning/journal.py @@ -87,6 +87,12 @@ class RoundRecord: cost_usd: float = 0.0 taxonomy_version: str = "" config_version: str = "" + # Pairs that moved each way in each stage, so the record can say whether the decision was distinguishable + # from noise. The decision itself is Req 9.5's raw-delta rule; this is the evidence beside it. + train_better: int = 0 + train_worse: int = 0 + holdout_better: int = 0 + holdout_worse: int = 0 def to_dict(self) -> dict[str, object]: return { @@ -105,8 +111,17 @@ class RoundRecord: "cost_usd": self.cost_usd, "taxonomy_version": self.taxonomy_version, "config_version": self.config_version, + **{f"train_{key}": value for key, value in self._reliability(self.train_better, self.train_worse).items()}, + **{f"holdout_{key}": value for key, value in self._reliability(self.holdout_better, self.holdout_worse).items()}, } + @staticmethod + def _reliability(better: int, worse: int) -> dict[str, object]: + from .acceptance import reliability + + result = reliability(better=better, worse=worse) + return {"better": result.better, "worse": result.worse, "p_value": result.p_value, "reliable": result.reliable} + @staticmethod def from_dict(payload: Mapping[str, object]) -> RoundRecord: attribution = payload.get("attribution") diff --git a/src/openultrasast/learning/rounds.py b/src/openultrasast/learning/rounds.py index 0815690..68bbfb7 100644 --- a/src/openultrasast/learning/rounds.py +++ b/src/openultrasast/learning/rounds.py @@ -66,6 +66,7 @@ class BaselineReport: k_runs: int taxonomy_version: str slices: tuple[str, ...] + thinking: bool = False # the condition that produced these numbers, so two modes cannot be read as one floors: dict[str, NoiseFloor] = field(default_factory=dict) metrics: dict[str, dict[str, object]] = field(default_factory=dict) # `/`: the budgets stay per family, but vibe-py and agent-vfc are different worlds and one @@ -80,6 +81,7 @@ class BaselineReport: "k_runs": self.k_runs, "taxonomy_version": self.taxonomy_version, "slices": list(self.slices), + "thinking": self.thinking, "cost_usd": self.cost_usd, "floors": {name: floor.to_dict() for name, floor in sorted(self.floors.items())}, "metrics": {name: dict(block) for name, block in sorted(self.metrics.items())}, @@ -100,6 +102,7 @@ def run_baseline( prompt: str | None = None, spent_usd: Callable[[], float] | None = None, cutoff: str = "", + thinking: bool = False, ) -> BaselineReport: """Clone one detector into every family, measure each family **with its own configuration**, write the artifacts. @@ -146,6 +149,7 @@ def run_baseline( k_runs=k_runs, taxonomy_version=taxonomy.version, slices=tuple(slices), + thinking=thinking, floors=floors, metrics={name: block.to_dict() for name, block in metrics.items()}, per_slice={name: block.to_dict() for name, block in aggregate(scores, taxonomy=taxonomy, per_slice=True, cutoff=cutoff).items()}, @@ -198,7 +202,8 @@ def compare_baselines(out_dir: Path) -> dict[str, dict[str, dict[str, object]]]: payload = json.loads(report.read_text(encoding="utf-8")) model = str(payload.get("model", directory.name)) slices = tuple(str(item) for item in (payload.get("slices") or ())) - label = model if not slices or slices == DEFAULT_SLICES else f"{model} ({', '.join(slices)})" + conditions = [*(() if not payload.get("thinking") else ("thinking",)), *(() if slices == DEFAULT_SLICES else slices)] + label = f"{model} ({', '.join(conditions)})" if conditions else model table[label] = {family: dict(block) for family, block in (payload.get("metrics") or {}).items()} return table @@ -227,18 +232,21 @@ def _family_of(case: PairCase, taxonomy: FamilyTaxonomy) -> str: return answer.families[0] if answer.families else "unknown" -def _artifact_key(model: str, slices: Sequence[str]) -> str: +def _artifact_key(model: str, slices: Sequence[str], thinking: bool = False) -> str: """What identifies a baseline run: the detector model, and the slice selection when it is not the default. Keyed by model alone, a run over the memory-safety slice silently replaces the run over the web families and the published table shows one of them as if it were everything (Req 8.4).""" - if tuple(slices) == DEFAULT_SLICES: - return _slug(model) - return f"{_slug(model)}-{_slug('-'.join(sorted(slices)))}" + parts = [_slug(model)] + if thinking: + parts.append("thinking") + if tuple(slices) != DEFAULT_SLICES: + parts.append(_slug("-".join(sorted(slices)))) + return "-".join(parts) def _write(out_dir: Path, report: BaselineReport) -> None: - directory = out_dir / "baseline" / _artifact_key(report.model, report.slices) + directory = out_dir / "baseline" / _artifact_key(report.model, report.slices, report.thinking) directory.mkdir(parents=True, exist_ok=True) payload = report.to_dict() (directory / "noise-floors.json").write_text( @@ -252,9 +260,9 @@ def _slug(model: str) -> str: return re.sub(r"[^A-Za-z0-9._-]+", "-", model).strip("-") or "unnamed" -def load_noise_floors(out_dir: Path, model: str, slices: Sequence[str] = ()) -> dict[str, NoiseFloor]: +def load_noise_floors(out_dir: Path, model: str, slices: Sequence[str] = (), thinking: bool = False) -> dict[str, NoiseFloor]: """The budgets a later round must stay inside, as round zero measured them for this model.""" - path = out_dir / "baseline" / _artifact_key(model, slices or DEFAULT_SLICES) / "noise-floors.json" + path = out_dir / "baseline" / _artifact_key(model, slices or DEFAULT_SLICES, thinking) / "noise-floors.json" if not path.is_file(): return {} payload = json.loads(path.read_text(encoding="utf-8")) @@ -405,6 +413,8 @@ def run_learning_round( train_delta = _delta(before_train, after_train) holdout_delta = _delta(before_hold, after_hold) flipped = _flipped(before_train + before_hold, after_train + after_hold) + train_moves = _moves(before_train, after_train) + holdout_moves = _moves(before_hold, after_hold) attribution = Attribution( flipped_predicted=len(flipped), flipped_unpredicted=len(flipped_unpredicted), @@ -438,6 +448,7 @@ def run_learning_round( holdout_delta=holdout_delta, sweep=sweep, attribution=attribution, + moves=(train_moves, holdout_moves), flipped=flipped, unpredicted=flipped_unpredicted, scores=after_train + after_hold, @@ -456,6 +467,7 @@ def run_learning_round( holdout_delta=holdout_delta, sweep=sweep, attribution=attribution, + moves=(train_moves, holdout_moves), flipped=flipped, unpredicted=flipped_unpredicted, scores=after_train + after_hold, @@ -543,6 +555,21 @@ def _delta(before: Sequence[PairFamilyScore], after: Sequence[PairFamilyScore]) return (now - was) / len(scorable) +def _moves(before: Sequence[PairFamilyScore], after: Sequence[PairFamilyScore]) -> tuple[int, int]: + """(pairs that became correct, pairs that stopped being correct) — what the sign test is run over.""" + was = {item.pair: item.outcome for item in before if item.outcome != "unscorable"} + better = worse = 0 + for item in after: + previous = was.get(item.pair) + if previous is None or item.outcome == "unscorable": + continue + if previous != "pair_correct" and item.outcome == "pair_correct": + better += 1 + elif previous == "pair_correct" and item.outcome != "pair_correct": + worse += 1 + return better, worse + + def _flipped(before: Sequence[PairFamilyScore], after: Sequence[PairFamilyScore]) -> list[str]: """Pairs this round turned right, by name and in a stable order.""" was = {item.pair: item.outcome for item in before} @@ -577,6 +604,7 @@ def _finish( holdout_delta: float = 0.0, sweep: Mapping[str, int] | None = None, attribution: Attribution | None = None, + moves: tuple[tuple[int, int], tuple[int, int]] = ((0, 0), (0, 0)), flipped: Sequence[str] = (), unpredicted: Sequence[str] = (), scores: Sequence[PairFamilyScore] = (), @@ -595,6 +623,10 @@ def _finish( sweep=dict(sweep or {}), attribution=attribution or Attribution(), cost_usd=cost, + train_better=moves[0][0], + train_worse=moves[0][1], + holdout_better=moves[1][0], + holdout_worse=moves[1][1], ) if version is not None: record = RoundRecord(**{**record.__dict__, "config_version": version}) diff --git a/tests/test_learning_reliability.py b/tests/test_learning_reliability.py new file mode 100644 index 0000000..abda25d --- /dev/null +++ b/tests/test_learning_reliability.py @@ -0,0 +1,56 @@ +"""Every round says whether its decision was distinguishable from noise (learning-harness, Req 9.5). + +The spec built `sign_test` and `reliable_change` and the acceptance rule never calls either: `decide` compares raw +deltas. On a detector where 11 of 20 pairs disagree with themselves between runs, a raw delta is mostly noise, and +a rule that brittle is exactly what makes someone reach for a deterministic decoder to prop it up. + +Changing Req 9.5 is a maintainer's call, so this does not change the decision — it records, beside every decision, +the two-sided sign test over the pairs that actually changed. After a few rounds the journal says plainly whether +any of those decisions could be told from a coin flip. +""" + +from __future__ import annotations + +from openultrasast.learning.acceptance import reliability + + +def test_a_clean_sweep_of_the_minibatch_is_distinguishable_from_noise() -> None: + result = reliability(better=8, worse=0) + assert result.better == 8 and result.worse == 0 + assert result.p_value < 0.05 and result.reliable is True + + +def test_two_pairs_moving_the_right_way_is_not() -> None: + """n = 2 cannot reach p < 0.05 whatever it does, and a rule that treats it as evidence is measuring nothing.""" + result = reliability(better=2, worse=0) + assert result.p_value == 0.5 and result.reliable is False + + +def test_a_round_that_changed_nothing_reports_no_evidence_rather_than_agreement() -> None: + result = reliability(better=0, worse=0) + assert result.p_value == 1.0 and result.reliable is False + + +def test_an_even_split_is_never_reliable_however_large() -> None: + assert reliability(better=20, worse=20).reliable is False + + +def test_the_round_journals_the_reliability_of_its_own_decision() -> None: + from openultrasast.learning.journal import RoundRecord + + record = RoundRecord( + round=1, + family="injection", + hypothesis="h", + levers=("checklist",), + predicted_affected=(), + predicted_at_risk=(), + outcome="accepted", + train_better=8, + train_worse=0, + holdout_better=1, + holdout_worse=0, + ) + payload = record.to_dict() + assert payload["train_reliable"] is True and payload["train_p_value"] < 0.05 + assert payload["holdout_reliable"] is False, "one pair is not evidence, and the record says so" diff --git a/tests/test_learning_thinking.py b/tests/test_learning_thinking.py new file mode 100644 index 0000000..cf2fcfa --- /dev/null +++ b/tests/test_learning_thinking.py @@ -0,0 +1,69 @@ +"""Thinking is a measured condition, not a hidden default (learning-harness, Req 8.3). + +Thinking was disabled so that `temperature: 0` would not be silently ignored — a trade that only pays if it buys +determinism. Round zero measured 11 of 20 injection pairs disagreeing with themselves across five runs, so it did +not: the variance comes from the agentic path, not the decoder. Which mode is better is a question the harness is +built to answer, so it is a knob whose value the artifact records and whose runs cannot overwrite each other. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from openultrasast.config import load_config + + +def test_thinking_is_configurable_and_off_by_default(tmp_path: Path) -> None: + """The default preserves what round zero was measured under; changing it silently would make the two runs + incomparable while looking like one number.""" + assert load_config(None).learning.thinking is False + path = tmp_path / "openultrasast.toml" + path.write_text("[learning]\nthinking = true\n") + assert load_config(path).learning.thinking is True + + +def test_the_endpoint_records_the_mode_it_ran_in(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from openultrasast.learning.endpoint import resolve_chat_endpoint + + monkeypatch.setenv("DEEPSEEK_API_KEY", "k") + monkeypatch.delenv("OPENULTRASAST_HUNTER_CLIENT", raising=False) + path = tmp_path / "openultrasast.toml" + path.write_text("[learning]\nthinking = true\n") + client, endpoint = resolve_chat_endpoint(load_config(path)) # type: ignore[misc] + assert endpoint.thinking is True + assert client._disable_thinking is False # type: ignore[attr-defined] + path.write_text("[learning]\nthinking = false\n") + client, endpoint = resolve_chat_endpoint(load_config(path)) # type: ignore[misc] + assert endpoint.thinking is False and client._disable_thinking is True # type: ignore[attr-defined] + + +def test_the_two_modes_are_two_baselines_not_one_overwriting_the_other(tmp_path: Path) -> None: + """The condition that produced a number is part of what identifies it.""" + import sys + + sys.path.insert(0, str(Path(__file__).parent)) + from test_learning_rounds import _always, _case, _steady # noqa: PLC0415 + + from openultrasast.learning.families import load_families + from openultrasast.learning.rounds import compare_baselines, run_baseline + + out = tmp_path / "out" + for thinking in (False, True): + run_baseline( + [_case(tmp_path, "a")], + taxonomy=load_families(), + configs_dir=tmp_path / "configs", + scan_factory=_always(_steady), + model="deepseek-v4-flash", + out_dir=out, + k_runs=3, + thinking=thinking, + ) + written = sorted(path.name for path in (out / "baseline").iterdir()) + assert written == ["deepseek-v4-flash", "deepseek-v4-flash-thinking"], written + assert set(compare_baselines(out)) == {"deepseek-v4-flash", "deepseek-v4-flash (thinking)"} + report = json.loads((out / "baseline" / "deepseek-v4-flash-thinking" / "report.json").read_text()) + assert report["thinking"] is True