mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 19:59:26 +00:00
Marius CiepluchandClaude Opus 5
b57b1a35d8
records whether its decision beat noise Two things the maintainer was right about. Thinking was disabled so `temperature: 0` would not be silently ignored -- a trade that only pays if it buys determinism. Round zero says it did not: 11 of 20 injection pairs disagreed with themselves across five runs, so the variance is in the agentic path, not the decoder. We paid the reasoning cost on a reasoning model and got nothing. It is now `[learning] thinking`, recorded in the artifact and part of its key so the two modes cannot be read as one number. The default stays off only because that is what round zero was measured under. And the deeper point: the spec built `sign_test` and `reliable_change` and the acceptance rule calls neither -- `decide` compares raw deltas. On a detector with that floor a raw delta is mostly noise, and a rule that brittle is exactly what makes a deterministic decoder look necessary. It is not; the alternative is an acceptance rule that models the score as noisy, the way reflective-evolution and Bayesian prompt optimizers do. Changing Req 9.5 is a maintainer's call, so this does not change the decision: every round now journals the two-sided sign test over the pairs that moved, and after four rounds the journal will say whether any decision could be told from a coin flip. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
57 lines
2.3 KiB
Python
57 lines
2.3 KiB
Python
"""Every round says whether its decision was distinguishable from noise (learning-harness, Req 9.5).
|
|
|
|
The spec built `sign_test` and `reliable_change` and the acceptance rule never calls either: `decide` compares raw
|
|
deltas. On a detector where 11 of 20 pairs disagree with themselves between runs, a raw delta is mostly noise, and
|
|
a rule that brittle is exactly what makes someone reach for a deterministic decoder to prop it up.
|
|
|
|
Changing Req 9.5 is a maintainer's call, so this does not change the decision — it records, beside every decision,
|
|
the two-sided sign test over the pairs that actually changed. After a few rounds the journal says plainly whether
|
|
any of those decisions could be told from a coin flip.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from openultrasast.learning.acceptance import reliability
|
|
|
|
|
|
def test_a_clean_sweep_of_the_minibatch_is_distinguishable_from_noise() -> None:
|
|
result = reliability(better=8, worse=0)
|
|
assert result.better == 8 and result.worse == 0
|
|
assert result.p_value < 0.05 and result.reliable is True
|
|
|
|
|
|
def test_two_pairs_moving_the_right_way_is_not() -> None:
|
|
"""n = 2 cannot reach p < 0.05 whatever it does, and a rule that treats it as evidence is measuring nothing."""
|
|
result = reliability(better=2, worse=0)
|
|
assert result.p_value == 0.5 and result.reliable is False
|
|
|
|
|
|
def test_a_round_that_changed_nothing_reports_no_evidence_rather_than_agreement() -> None:
|
|
result = reliability(better=0, worse=0)
|
|
assert result.p_value == 1.0 and result.reliable is False
|
|
|
|
|
|
def test_an_even_split_is_never_reliable_however_large() -> None:
|
|
assert reliability(better=20, worse=20).reliable is False
|
|
|
|
|
|
def test_the_round_journals_the_reliability_of_its_own_decision() -> None:
|
|
from openultrasast.learning.journal import RoundRecord
|
|
|
|
record = RoundRecord(
|
|
round=1,
|
|
family="injection",
|
|
hypothesis="h",
|
|
levers=("checklist",),
|
|
predicted_affected=(),
|
|
predicted_at_risk=(),
|
|
outcome="accepted",
|
|
train_better=8,
|
|
train_worse=0,
|
|
holdout_better=1,
|
|
holdout_worse=0,
|
|
)
|
|
payload = record.to_dict()
|
|
assert payload["train_reliable"] is True and payload["train_p_value"] < 0.05
|
|
assert payload["holdout_reliable"] is False, "one pair is not evidence, and the record says so"
|