mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 11:49:28 +00:00
fix(learning-harness): the round-3 review findings, remediated RED-first
Eight of nine group-3 tasks were rejected. Each defect was hiding a number, and the numbers are why the loop was not learning: - The fixed side was judged against the *vulnerable* side's function ranges. 191 of 227 pairs with a resolvable labeled span (84%) have different spans on the two sides, so a leak was recorded as silence, always in the direction that flatters Youden. - Round zero measured every family with the generalist. `family:unknown` earns no credit against any real family, so every family but `unknown` baselined at a structural zero and every noise floor with it. - The cross-family sweep counted absolute failures instead of negative flips. On a corpus where most pairs fail at round zero that rejects every round for damage it did not do. Unscorable rows voted too. - K runs was declared and never threaded; `pairs --k-runs` was parsed and discarded, and its only test grepped `--help`. - The meta-agent proposer was a stub, so `learning round` could never propose anything. It now composes MetaAgent.evolve against a scratch copy of the family directory, and `--proposals FILE` runs a recorded sequence offline. - `refused_holdout` was a vocabulary word no code path could produce, and the covering test accepted either outcome. - Cost was metered once, after everything had been paid for; a stage that raised left no record at all. - The corrected lever number was published beside the earlier one rather than in place of it, and the test named for that property never checked. - Refusals were `continue` statements: `Refusal.degradation()` existed and nothing called it. A train-split identical twin was reported as `split:train`, which is untrue. Test repairs that each changed a number: two split tests measured an `x = 1` fixture that teaches nothing whatever the rule says; two CLI scoreboard tests asserted against all-zero output produced by a stub that answers twice and puts its finding outside the labeled function. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01T7XD7x5KonMdmGRkp63EQG
This commit is contained in:
co-authored by
Claude Opus 5
parent
c2d9c4ad01
commit
1843ef9a02
@@ -134,21 +134,29 @@ def test_a_change_that_helps_nothing_is_rejected_and_reverted(world) -> None: #
|
||||
|
||||
|
||||
def test_a_round_is_rejected_when_the_sweep_finds_another_family_below_its_floor(world) -> None: # type: ignore[no-untyped-def]
|
||||
"""The sweep is a floor check over every other family, not a causal claim about this change.
|
||||
"""The sweep counts pairs this change turned from right to wrong in another family (Req 9.5).
|
||||
|
||||
Per-family directories make a change that reaches another family structurally impossible, so what the
|
||||
sweep really guards is a family that is failing for any reason at the moment this round would land.
|
||||
It used to count every failing pair, which rejected a round for damage it did not do; the pair below is right
|
||||
before the change and wrong after it, which is the only thing that may reject a round on another family's
|
||||
behalf. The other arm — a family that was already failing must not reject — lives in
|
||||
`tests/test_learning_round_evidence.py`.
|
||||
"""
|
||||
world["cases"].append(_case(world["configs"].parent, "other-hold", split="holdout", family="access_control"))
|
||||
world["floors"]["access_control"] = NoiseFloor(
|
||||
family="access_control", k_runs=5, pairs=1, flaky_pairs=0, negative_flip_rate=0.0, budget_flips=0
|
||||
)
|
||||
landed = {"yes": False}
|
||||
|
||||
def build(config: FamilyConfig): # type: ignore[no-untyped-def]
|
||||
def scan(root: Path) -> list[StaticFinding]:
|
||||
if root.name != "vuln" or config.family == "access_control":
|
||||
return [] # access control finds nothing, so its holdout pair is below its floor
|
||||
return [_finding("injection")] if config.checklist.strip() == "- ask about quoting" else []
|
||||
if root.name != "vuln":
|
||||
return []
|
||||
if config.family == "access_control":
|
||||
return [] if landed["yes"] else [_finding("access_control")]
|
||||
if config.checklist.strip() == "- ask about quoting":
|
||||
landed["yes"] = True # the injection change lands, and access control stops finding its pair
|
||||
return [_finding("injection")]
|
||||
return []
|
||||
|
||||
return scan
|
||||
|
||||
@@ -181,8 +189,7 @@ def test_a_round_over_its_cost_cap_is_reverted_and_says_so(world) -> None: # ty
|
||||
|
||||
def test_a_proposal_that_reaches_beyond_its_family_never_runs(world) -> None: # type: ignore[no-untyped-def]
|
||||
record = _run(world, Proposal(hypothesis="h", lever="checklist", change={"../escape.md": "x"}), {})
|
||||
assert record.outcome == "refused_holdout" or record.outcome == "rejected"
|
||||
assert "outside" in record.reason
|
||||
assert record.outcome == "rejected" and "outside" in record.reason # the holdout refusal has its own test
|
||||
assert not (world["configs"].parent / "escape.md").exists()
|
||||
|
||||
|
||||
@@ -202,7 +209,7 @@ def test_the_round_directory_holds_what_a_replay_needs(world) -> None: # type:
|
||||
directory = world["out"] / "rounds" / "1"
|
||||
assert (directory / "proposal.json").is_file() and (directory / "scores.json").is_file()
|
||||
attribution = json.loads((directory / "attribution.json").read_text())
|
||||
assert attribution["flipped_predicted"] == ["hold-a"] or attribution["flipped_predicted"] == ["train-a", "hold-a"]
|
||||
assert attribution["flipped_predicted"] == ["hold-a", "train-a"] # the round is deterministic; one exact answer
|
||||
assert attribution["flipped_unpredicted"] == []
|
||||
trajectories = (directory / "trajectories.jsonl").read_text()
|
||||
assert "REDACTED" in trajectories and "B" * 24 not in trajectories
|
||||
|
||||
Reference in New Issue
Block a user